root/opal/mca/btl/usnic/btl_usnic_endpoint.c

DEFINITIONS

1. endpoint_construct
2. endpoint_destruct
3. opal_btl_usnic_flush_endpoint

   1 /*
   2  * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
   3  *                         University Research and Technology
   4  *                         Corporation.  All rights reserved.
   5  * Copyright (c) 2004-2005 The University of Tennessee and The University
   6  *                         of Tennessee Research Foundation.  All rights
   7  *                         reserved.
   8  * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
   9  *                         University of Stuttgart.  All rights reserved.
  10  * Copyright (c) 2004-2005 The Regents of the University of California.
  11  *                         All rights reserved.
  12  * Copyright (c) 2006      Sandia National Laboratories. All rights
  13  *                         reserved.
  14  * Copyright (c) 2007      The Regents of the University of California.
  15  *                         All rights reserved.
  16  * Copyright (c) 2013-2014 Cisco Systems, Inc.  All rights reserved.
  17  * Copyright (c) 2015      Intel, Inc. All rights reserved
  18  * $COPYRIGHT$
  19  *
  20  * Additional copyrights may follow
  21  *
  22  * $HEADER$
  23  */
  24 
  25 #include "opal_config.h"
  26 
  27 #include <stdio.h>
  28 #include <errno.h>
  29 #include <string.h>
  30 #include <sys/types.h>
  31 #include <unistd.h>
  32 
  33 #include "opal/prefetch.h"
  34 #include "opal/types.h"
  35 
  36 #include "btl_usnic.h"
  37 #include "btl_usnic_endpoint.h"
  38 #include "btl_usnic_module.h"
  39 #include "btl_usnic_frag.h"
  40 #include "btl_usnic_proc.h"
  41 #include "btl_usnic_util.h"
  42 #include "btl_usnic_ack.h"
  43 #include "btl_usnic_send.h"
  44 
  45 /*
  46  * Construct/destruct an endpoint structure.
  47  */
  48 static void endpoint_construct(mca_btl_base_endpoint_t* endpoint)
  49 {
  50     int i;
  51 
  52     endpoint->endpoint_module = NULL;
  53     endpoint->endpoint_proc = NULL;
  54     endpoint->endpoint_proc_index = -1;
  55     endpoint->endpoint_exiting = false;
  56     endpoint->endpoint_connectivity_checked = false;
  57     endpoint->endpoint_on_all_endpoints = false;
  58 
  59     for (i = 0; i < USNIC_NUM_CHANNELS; ++i) {
  60         endpoint->endpoint_remote_modex.ports[i] = 0;
  61         endpoint->endpoint_remote_addrs[i] = FI_ADDR_NOTAVAIL;
  62     }
  63 
  64     endpoint->endpoint_send_credits = 8;
  65 
  66     /* list of fragments queued to be sent */
  67     OBJ_CONSTRUCT(&endpoint->endpoint_frag_send_queue, opal_list_t);
  68 
  69     endpoint->endpoint_next_frag_id = 1;
  70     endpoint->endpoint_acktime = 0;
  71 
  72     /* endpoint starts not-ready-to-send */
  73     endpoint->endpoint_ready_to_send = 0;
  74     endpoint->endpoint_ack_needed = false;
  75 
  76     /* clear sent/received sequence number array */
  77     memset(endpoint->endpoint_sent_segs, 0,
  78            sizeof(endpoint->endpoint_sent_segs));
  79     memset(endpoint->endpoint_rcvd_segs, 0,
  80            sizeof(endpoint->endpoint_rcvd_segs));
  81 
  82     /*
  83      * Make a new OPAL hotel for this module
  84      * "hotel" is a construct used for triggering segment retransmission
  85      * due to timeout
  86      */
  87     OBJ_CONSTRUCT(&endpoint->endpoint_hotel, opal_hotel_t);
  88     opal_hotel_init(&endpoint->endpoint_hotel,
  89                     WINDOW_SIZE,
  90                     opal_sync_event_base,
  91                     mca_btl_usnic_component.retrans_timeout,
  92                     0,
  93                     opal_btl_usnic_ack_timeout);
  94 
  95     /* Setup this endpoint's list links */
  96     OBJ_CONSTRUCT(&(endpoint->endpoint_ack_li), opal_list_item_t);
  97     OBJ_CONSTRUCT(&(endpoint->endpoint_endpoint_li), opal_list_item_t);
  98     endpoint->endpoint_ack_needed = false;
  99 
 100     /* fragment reassembly info */
 101     endpoint->endpoint_rx_frag_info =
 102         calloc(sizeof(struct opal_btl_usnic_rx_frag_info_t), MAX_ACTIVE_FRAGS);
 103     assert(NULL != endpoint->endpoint_rx_frag_info);
 104     if (OPAL_UNLIKELY(endpoint->endpoint_rx_frag_info == NULL)) {
 105         BTL_ERROR(("calloc returned NULL -- this should not happen!"));
 106         opal_btl_usnic_exit(endpoint->endpoint_module);
 107         /* Does not return */
 108     }
 109 }
 110 
 111 static void endpoint_destruct(mca_btl_base_endpoint_t* endpoint)
 112 {
 113     opal_btl_usnic_proc_t *proc;
 114 
 115     if (endpoint->endpoint_ack_needed) {
 116         opal_btl_usnic_remove_from_endpoints_needing_ack(endpoint);
 117     }
 118     OBJ_DESTRUCT(&(endpoint->endpoint_ack_li));
 119 
 120     /* Remove the endpoint from the all_endpoints list */
 121     opal_btl_usnic_module_t *module = endpoint->endpoint_module;
 122     opal_mutex_lock(&module->all_endpoints_lock);
 123     if (endpoint->endpoint_on_all_endpoints) {
 124         opal_list_remove_item(&module->all_endpoints,
 125                               &endpoint->endpoint_endpoint_li);
 126         endpoint->endpoint_on_all_endpoints = false;
 127     }
 128     opal_mutex_unlock(&module->all_endpoints_lock);
 129     OBJ_DESTRUCT(&(endpoint->endpoint_endpoint_li));
 130 
 131     if (endpoint->endpoint_hotel.rooms != NULL) {
 132         OBJ_DESTRUCT(&(endpoint->endpoint_hotel));
 133     }
 134 
 135     OBJ_DESTRUCT(&endpoint->endpoint_frag_send_queue);
 136 
 137     /* release owning proc */
 138     proc = endpoint->endpoint_proc;
 139     if (NULL != proc) {
 140         proc->proc_endpoints[endpoint->endpoint_proc_index] = NULL;
 141         OBJ_RELEASE(proc);
 142     }
 143 
 144     free(endpoint->endpoint_rx_frag_info);
 145 }
 146 
 147 OBJ_CLASS_INSTANCE(opal_btl_usnic_endpoint_t,
 148                    opal_list_item_t,
 149                    endpoint_construct,
 150                    endpoint_destruct);
 151 
 152 /*
 153  * Forcibly drain all pending output on an endpoint, without waiting for
 154  * actual completion.
 155  */
 156 void
 157 opal_btl_usnic_flush_endpoint(
 158     opal_btl_usnic_endpoint_t *endpoint)
 159 {
 160     opal_btl_usnic_send_frag_t *frag;
 161 
 162     /* First, free all pending fragments */
 163     while (!opal_list_is_empty(&endpoint->endpoint_frag_send_queue)) {
 164         frag = (opal_btl_usnic_send_frag_t *)opal_list_remove_first(
 165                 &endpoint->endpoint_frag_send_queue);
 166 
 167         /* _cond still needs to check ownership, but make sure the
 168          * fragment is marked as done.
 169          */
 170         frag->sf_ack_bytes_left = 0;
 171         frag->sf_seg_post_cnt = 0;
 172         opal_btl_usnic_send_frag_return_cond(endpoint->endpoint_module, frag);
 173     }
 174 
 175     /* Now, ACK everything that is pending */
 176     opal_btl_usnic_handle_ack(endpoint, endpoint->endpoint_next_seq_to_send-1);
 177 }