root/ompi/mca/osc/rdma/osc_rdma_module.c

DEFINITIONS

1. ompi_osc_module_add_peer
2. ompi_osc_rdma_free

   1 /* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
   2 /*
   3  * Copyright (c) 2004-2005 The Trustees of Indiana University.
   4  *                         All rights reserved.
   5  * Copyright (c) 2004-2005 The Trustees of the University of Tennessee.
   6  *                         All rights reserved.
   7  * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
   8  *                         University of Stuttgart.  All rights reserved.
   9  * Copyright (c) 2004-2005 The Regents of the University of California.
  10  *                         All rights reserved.
  11  * Copyright (c) 2007-2016 Los Alamos National Security, LLC. All rights
  12  *                         reserved.
  13  * Copyright (c) 2012-2013 Sandia National Laboratories.  All rights reserved.
  14  * Copyright (c) 2017      The University of Tennessee and The University
  15  *                         of Tennessee Research Foundation.  All rights
  16  *                         reserved.
  17  * $COPYRIGHT$
  18  *
  19  * Additional copyrights may follow
  20  *
  21  * $HEADER$
  22  */
  23 
  24 #include "osc_rdma.h"
  25 #include "osc_rdma_lock.h"
  26 
  27 #include "mpi.h"
  28 
  29 int ompi_osc_module_add_peer (ompi_osc_rdma_module_t *module, ompi_osc_rdma_peer_t *peer)
  30 {
  31     int ret = OMPI_SUCCESS;
  32 
  33     if (NULL == module->peer_array) {
  34         ret = opal_hash_table_set_value_uint32 (&module->peer_hash, peer->rank, (void *) peer);
  35     } else {
  36         module->peer_array[peer->rank] = peer;
  37     }
  38 
  39     return ret;
  40 }
  41 
  42 int ompi_osc_rdma_free(ompi_win_t *win)
  43 {
  44     int ret = OMPI_SUCCESS;
  45     ompi_osc_rdma_module_t *module = GET_MODULE(win);
  46     ompi_osc_rdma_peer_t *peer;
  47     uint32_t key;
  48     void *node;
  49 
  50     if (NULL == module) {
  51         return OMPI_SUCCESS;
  52     }
  53 
  54     while (module->pending_ops) {
  55         ompi_osc_rdma_progress (module);
  56     }
  57 
  58     if (NULL != module->comm) {
  59         opal_output_verbose(1, ompi_osc_base_framework.framework_output,
  60                             "rdma component destroying window with id %d",
  61                             ompi_comm_get_cid(module->comm));
  62 
  63         /* finish with a barrier */
  64         if (ompi_group_size(win->w_group) > 1) {
  65             (void) module->comm->c_coll->coll_barrier (module->comm,
  66                                                       module->comm->c_coll->coll_barrier_module);
  67         }
  68 
  69         /* remove from component information */
  70         OPAL_THREAD_LOCK(&mca_osc_rdma_component.lock);
  71         opal_hash_table_remove_value_uint32(&mca_osc_rdma_component.modules,
  72                                             ompi_comm_get_cid(module->comm));
  73         OPAL_THREAD_UNLOCK(&mca_osc_rdma_component.lock);
  74     }
  75 
  76     win->w_osc_module = NULL;
  77 
  78     if (module->state) {
  79         int region_count = module->state->region_count & 0xffffffffL;
  80         if (NULL != module->dynamic_handles) {
  81             for (int i = 0 ; i < region_count ; ++i) {
  82                 ompi_osc_rdma_deregister (module, module->dynamic_handles[i].btl_handle);
  83             }
  84 
  85             free (module->dynamic_handles);
  86         }
  87     }
  88 
  89     OBJ_DESTRUCT(&module->outstanding_locks);
  90     OBJ_DESTRUCT(&module->lock);
  91     OBJ_DESTRUCT(&module->peer_lock);
  92     OBJ_DESTRUCT(&module->all_sync);
  93 
  94     ompi_osc_rdma_deregister (module, module->state_handle);
  95     ompi_osc_rdma_deregister (module, module->base_handle);
  96 
  97     OPAL_LIST_DESTRUCT(&module->pending_posts);
  98 
  99     if (NULL != module->rdma_frag) {
 100         ompi_osc_rdma_deregister (module, module->rdma_frag->handle);
 101     }
 102 
 103     /* remove all cached peers */
 104     if (NULL == module->peer_array) {
 105         ret = opal_hash_table_get_first_key_uint32 (&module->peer_hash, &key, (void **) &peer, &node);
 106         while (OPAL_SUCCESS == ret) {
 107             OBJ_RELEASE(peer);
 108             ret = opal_hash_table_get_next_key_uint32 (&module->peer_hash, &key, (void **) &peer,
 109                                                        node, &node);
 110         }
 111 
 112         OBJ_DESTRUCT(&module->peer_hash);
 113     } else if (NULL != module->comm) {
 114         for (int i = 0 ; i < ompi_comm_size (module->comm) ; ++i) {
 115             if (NULL != module->peer_array[i]) {
 116                 OBJ_RELEASE(module->peer_array[i]);
 117             }
 118         }
 119     }
 120 
 121     if (module->local_leaders && MPI_COMM_NULL != module->local_leaders) {
 122         ompi_comm_free (&module->local_leaders);
 123     }
 124 
 125     if (module->shared_comm && MPI_COMM_NULL != module->shared_comm) {
 126         ompi_comm_free (&module->shared_comm);
 127     }
 128 
 129     if (module->comm && MPI_COMM_NULL != module->comm) {
 130         ompi_comm_free (&module->comm);
 131     }
 132 
 133     if (module->segment_base) {
 134         opal_shmem_segment_detach (&module->seg_ds);
 135         module->segment_base = NULL;
 136     }
 137 
 138     free (module->peer_array);
 139     free (module->outstanding_lock_array);
 140     free (module->free_after);
 141     free (module);
 142 
 143     return OMPI_SUCCESS;
 144 }