root/opal/mca/btl/ugni/btl_ugni_progress_thread.c

DEFINITIONS

1. mca_btl_ugni_prog_thread_fn
2. mca_btl_ugni_spawn_progress_thread
3. mca_btl_ugni_kill_progress_thread

   1 /* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
   2 /*
   3  * Copyright (c) 2011-2017 Los Alamos National Security, LLC. All rights
   4  *                         reserved.
   5  * Copyright (c) 2011      UT-Battelle, LLC. All rights reserved.
   6  * $COPYRIGHT$
   7  *
   8  * Additional copyrights may follow
   9  *
  10  * $HEADER$
  11  */
  12 
  13 #include "opal_config.h"
  14 
  15 #include "btl_ugni.h"
  16 #include "btl_ugni_frag.h"
  17 #include "btl_ugni_smsg.h"
  18 
  19 #include "opal/include/opal/align.h"
  20 
  21 
  22 static pthread_t mca_btl_ugni_progress_thread_id;
  23 
  24 static volatile int stop_progress_thread = 0;
  25 
  26 unsigned int mca_btl_ugni_progress_thread_wakeups = 0;
  27 
  28 static void *mca_btl_ugni_prog_thread_fn(void * data)
  29 {
  30     uint32_t which;
  31     gni_return_t status;
  32     gni_cq_handle_t cq_vec[1 + MCA_BTL_UGNI_MAX_DEV_HANDLES];
  33 
  34     struct mca_btl_ugni_module_t *btl = (mca_btl_ugni_module_t *)data;
  35     int cq_count = 1 + mca_btl_ugni_component.virtual_device_count;
  36 
  37     /*
  38      * need to block signals
  39      */
  40 
  41     cq_vec[0] = btl->smsg_remote_irq_cq;
  42     for (int i = 0 ; i < mca_btl_ugni_component.virtual_device_count ; ++i) {
  43         cq_vec[i + 1] = btl->devices[i].dev_rdma_local_irq_cq.gni_handle;
  44     }
  45 
  46     while (stop_progress_thread == 0) {
  47 
  48         /*
  49          * this ugni call doesn't need a lock
  50          */
  51 
  52         status = GNI_CqVectorMonitor(cq_vec,
  53                                      cq_count,
  54                                      -1,
  55                                      &which);
  56 
  57         if (status == GNI_RC_NOT_DONE) continue;
  58 
  59         if ((status == GNI_RC_SUCCESS) && (stop_progress_thread == 0)) {
  60             mca_btl_ugni_progress_thread_wakeups++;
  61             opal_progress();
  62         }
  63     }
  64 
  65     return (void *) (intptr_t) OPAL_SUCCESS;
  66 }
  67 
  68 int mca_btl_ugni_spawn_progress_thread(struct mca_btl_base_module_t *btl)
  69 {
  70     int rc, ret=OPAL_SUCCESS;
  71     pthread_attr_t attr;
  72 
  73     pthread_attr_init(&attr);
  74     rc = pthread_attr_setdetachstate(&attr, PTHREAD_CREATE_DETACHED);
  75     if (0 != rc) {
  76         BTL_ERROR(("btl/ugni pthread_attr_setdetachstate returned %s ",strerror(rc)));
  77         ret = OPAL_ERROR;
  78         goto fn_exit;
  79     }
  80 
  81     rc = pthread_create(&mca_btl_ugni_progress_thread_id,
  82                         &attr, mca_btl_ugni_prog_thread_fn, (void *)btl);
  83     if (0 != rc) {
  84         BTL_ERROR(("btl/ugni pthread_create returned %s ",strerror(rc)));
  85         ret = OPAL_ERROR;
  86         goto fn_exit;
  87     }
  88 
  89     rc = pthread_attr_destroy(&attr);
  90     if (0 != rc) {
  91         BTL_ERROR(("btl/ugni pthread_attr_destory returned %s ",strerror(rc)));
  92         ret = OPAL_ERROR;
  93     }
  94 
  95    fn_exit:
  96     return ret;
  97 }
  98 
  99 int mca_btl_ugni_kill_progress_thread(void)
 100 {
 101     int ret=OPAL_SUCCESS;
 102     void *thread_rc;
 103 
 104     stop_progress_thread = 1;
 105 
 106     /*
 107      * post a CQ to myself to wake my thread up
 108      */
 109 
 110     ret = mca_btl_ugni_post_cqwrite (mca_btl_ugni_component.modules[0].local_ep,
 111                                      &mca_btl_ugni_component.modules[0].devices[0].dev_rdma_local_cq,
 112                                      mca_btl_ugni_component.modules[0].devices[0].smsg_irq_mhndl,
 113                                      0xdead, NULL, NULL, NULL);
 114     /*
 115      * TODO: if error returned, need to kill off thread manually
 116      */
 117     if (OPAL_SUCCESS != ret) {
 118         /* force the thread to exit */
 119         pthread_cancel (mca_btl_ugni_progress_thread_id);
 120         goto fn_exit;
 121     }
 122 
 123     pthread_join (mca_btl_ugni_progress_thread_id, &thread_rc);
 124     if (0 != (intptr_t) thread_rc) {
 125         BTL_ERROR(("btl/ugni error returned from progress thread: %d", (int) (intptr_t) thread_rc));
 126         ret = (int)(intptr_t) thread_rc;
 127     }
 128 
 129    fn_exit:
 130     return ret;
 131 }
 132