root/opal/mca/btl/usnic/btl_usnic_send.h

INCLUDED FROM

DEFINITIONS

1. opal_btl_usnic_check_rts
2. opal_btl_usnic_post_segment
3. opal_btl_usnic_post_ack
4. opal_btl_usnic_endpoint_send_segment
5. opal_btl_usnic_endpoint_enqueue_frag
6. opal_btl_usnic_release_send_segment

   1 /*
   2  * Copyright (c) 2013-2017 Cisco Systems, Inc.  All rights reserved.
   3  * $COPYRIGHT$
   4  *
   5  * Additional copyrights may follow
   6  *
   7  * $HEADER$
   8  */
   9 
  10 #ifndef BTL_USNIC_SEND_H
  11 #define BTL_USNIC_SEND_H
  12 
  13 #include "btl_usnic.h"
  14 #include "btl_usnic_frag.h"
  15 #include "btl_usnic_ack.h"
  16 #if MSGDEBUG1
  17 #include "btl_usnic_util.h"
  18 #endif
  19 
  20 /*
  21  * Check if conditions are right, and if so, put endpoint on
  22  * list of endpoints that have sends to be done
  23  */
  24 static inline void
  25 opal_btl_usnic_check_rts(
  26     opal_btl_usnic_endpoint_t *endpoint)
  27 {
  28     /*
  29      * If endpoint not already ready,
  30      * and has packets to send,
  31      * and it has send credits,
  32      * and its retransmission window is open,
  33      * make it ready
  34      */
  35     if (!endpoint->endpoint_ready_to_send &&
  36         !opal_list_is_empty(&endpoint->endpoint_frag_send_queue) &&
  37          endpoint->endpoint_send_credits > 0 &&
  38          WINDOW_OPEN(endpoint)) {
  39         opal_list_append(&endpoint->endpoint_module->endpoints_with_sends,
  40                 &endpoint->super);
  41         endpoint->endpoint_ready_to_send = true;
  42 #if MSGDEBUG1
  43         opal_output(0, "make endpoint %p RTS\n", (void*)endpoint);
  44     } else {
  45         opal_output(0, "rts:%d empty:%d cred:%d open%d\n",
  46                 endpoint->endpoint_ready_to_send,
  47                 opal_list_is_empty(&endpoint->endpoint_frag_send_queue),
  48                 endpoint->endpoint_send_credits,
  49                 WINDOW_OPEN(endpoint));
  50 #endif
  51     }
  52 }
  53 
  54 /*
  55  * Common point for posting a segment.
  56  *
  57  * ASSUMES THAT THE CALLER HAS ALREADY CHECKED TO SEE IF WE HAVE
  58  * A SEND CREDIT!
  59  */
  60 static inline void
  61 opal_btl_usnic_post_segment(
  62     opal_btl_usnic_module_t *module,
  63     opal_btl_usnic_endpoint_t *endpoint,
  64     opal_btl_usnic_send_segment_t *sseg)
  65 {
  66     int ret;
  67 
  68     /* get channel and remote channel */
  69     opal_btl_usnic_channel_id_t channel_id = sseg->ss_channel;
  70     opal_btl_usnic_channel_t *channel = &module->mod_channels[channel_id];
  71 
  72 #if MSGDEBUG1
  73     opal_output(0, "post_send: type=%s, ep=%p, remote_addr=%p, addr=%p, len=%"
  74                 PRIsize_t,
  75                 usnic_seg_type_str(sseg->ss_base.us_type),
  76                 (void*) channel->ep,
  77                 (void*) endpoint->endpoint_remote_addrs[channel_id],
  78                 (void*) sseg->ss_ptr,
  79                 sseg->ss_len);
  80 #endif
  81 
  82     assert(channel_id == USNIC_DATA_CHANNEL);
  83     assert(channel->credits > 1);
  84 
  85     /* Send the segment */
  86     ret = fi_send(channel->ep,
  87             sseg->ss_ptr,
  88             sseg->ss_len + mca_btl_usnic_component.prefix_send_offset,
  89             NULL,
  90             endpoint->endpoint_remote_addrs[channel_id],
  91             sseg);
  92     if (OPAL_UNLIKELY(0 != ret)) {
  93         opal_btl_usnic_util_abort("fi_send() failed",
  94                                   __FILE__, __LINE__);
  95         /* Never returns */
  96     }
  97 
  98     /* track # of time non-ACKs are posted */
  99     if (sseg->ss_base.us_type != OPAL_BTL_USNIC_SEG_ACK) {
 100         ++sseg->ss_send_posted;
 101         ++sseg->ss_parent_frag->sf_seg_post_cnt;
 102     }
 103 
 104     /* Stats */
 105     ++module->stats.num_total_sends;
 106     ++channel->num_channel_sends;
 107     --channel->credits;
 108 }
 109 
 110 /*
 111  * Common point for posting an ACK
 112  *
 113  * ASSUMES THAT THE CALLER HAS ALREADY CHECKED TO SEE IF WE HAVE
 114  * A SEND CREDIT!
 115  */
 116 static inline void
 117 opal_btl_usnic_post_ack(
 118     opal_btl_usnic_module_t *module,
 119     opal_btl_usnic_endpoint_t *endpoint,
 120     opal_btl_usnic_send_segment_t *sseg)
 121 {
 122     int ret;
 123 
 124     /* get channel and remote channel */
 125     opal_btl_usnic_channel_id_t channel_id = sseg->ss_channel;
 126     opal_btl_usnic_channel_t *channel = &module->mod_channels[channel_id];
 127 
 128 #if MSGDEBUG1
 129     opal_output(0, "post_send ACK: type=%s, ep=%p, remote_addr=%p, addr=%p, len=%"
 130                 PRIsize_t,
 131                 usnic_seg_type_str(sseg->ss_base.us_type),
 132                 (void*) channel->ep,
 133                 (void*) endpoint->endpoint_remote_addrs[channel_id],
 134                 (void*) sseg->ss_ptr,
 135                 sseg->ss_len);
 136 #endif
 137 
 138     assert(channel_id == USNIC_PRIORITY_CHANNEL);
 139     assert(channel->credits > 1);
 140 
 141     ret = fi_send(channel->ep,
 142             sseg->ss_ptr,
 143             sseg->ss_len + mca_btl_usnic_component.prefix_send_offset,
 144             NULL,
 145             endpoint->endpoint_remote_addrs[channel_id],
 146             sseg);
 147     if (OPAL_UNLIKELY(0 != ret)) {
 148         opal_btl_usnic_util_abort("fi_send() failed",
 149                                   __FILE__, __LINE__);
 150         /* Never returns */
 151     }
 152 
 153     /* Stats */
 154     ++module->stats.num_total_sends;
 155     ++channel->num_channel_sends;
 156     --channel->credits;
 157 }
 158 
 159 /*
 160  * Post a send to the work queue
 161  */
 162 static inline void
 163 opal_btl_usnic_endpoint_send_segment(
 164     opal_btl_usnic_module_t *module,
 165     opal_btl_usnic_send_segment_t *sseg)
 166 {
 167     opal_btl_usnic_send_frag_t *frag;
 168     opal_btl_usnic_endpoint_t *endpoint;
 169     uint16_t sfi;
 170 
 171     frag = sseg->ss_parent_frag;
 172     endpoint = frag->sf_endpoint;
 173 
 174     /* Do we have room in the endpoint's sender window?
 175 
 176        Sender window:
 177 
 178                        |-------- WINDOW_SIZE ----------|
 179                       +---------------------------------+
 180                       |         next_seq_to_send        |
 181                       |     somewhere in this range     |
 182                      ^+---------------------------------+
 183                      |
 184                      +-- ack_seq_rcvd: one less than the window left edge
 185 
 186        Assuming that next_seq_to_send is > ack_seq_rcvd (verified
 187        by assert), then the good condition to send is:
 188 
 189             next_seq_to_send <= ack_seq_rcvd + WINDOW_SIZE
 190 
 191        And therefore the bad condition is
 192 
 193             next_seq_to_send > ack_seq_rcvd + WINDOW_SIZE
 194     */
 195     assert(SEQ_GT(endpoint->endpoint_next_seq_to_send,
 196                   endpoint->endpoint_ack_seq_rcvd));
 197     assert(WINDOW_OPEN(endpoint));
 198 
 199     /* Assign sequence number and increment */
 200     sseg->ss_base.us_btl_header->pkt_seq =
 201         endpoint->endpoint_next_seq_to_send++;
 202 
 203     /* Fill in remote address to indicate PUT or not */
 204     sseg->ss_base.us_btl_header->put_addr =
 205         frag->sf_base.uf_remote_seg[0].seg_addr.pval;
 206 
 207     /* piggy-back an ACK if needed */
 208     opal_btl_usnic_piggyback_ack(endpoint, sseg);
 209 
 210 #if MSGDEBUG1
 211     {
 212         char local_ip[32];
 213         char remote_ip[32];
 214 
 215         opal_btl_usnic_snprintf_ipv4_addr(local_ip, sizeof(local_ip),
 216                                           module->local_modex.ipv4_addr,
 217                                           module->local_modex.netmask);
 218         opal_btl_usnic_snprintf_ipv4_addr(remote_ip, sizeof(remote_ip),
 219                                           endpoint->endpoint_remote_modex.ipv4_addr,
 220                                           endpoint->endpoint_remote_modex.netmask);
 221 
 222         opal_output(0, "--> Sending %s: seq: %" UDSEQ ", sender: 0x%016lx from device %s, IP %s, port %u, seg %p, room %d, wc len %u, remote IP %s, port %u",
 223             (sseg->ss_parent_frag->sf_base.uf_type == OPAL_BTL_USNIC_FRAG_LARGE_SEND)?
 224                     "CHUNK" : "FRAG",
 225                     sseg->ss_base.us_btl_header->pkt_seq,
 226                     sseg->ss_base.us_btl_header->sender,
 227                     endpoint->endpoint_module->linux_device_name,
 228                     local_ip,
 229                     module->local_modex.ports[sseg->ss_channel],
 230                     (void*)sseg,
 231                     sseg->ss_hotel_room,
 232                     sseg->ss_ptr,
 233                     remote_ip,
 234                     endpoint->endpoint_remote_modex.ports[sseg->ss_channel]);
 235     }
 236 #endif
 237 
 238     /* do the actual send */
 239     opal_btl_usnic_post_segment(module, endpoint, sseg);
 240 
 241     /* Stash this segment in an array on the endpoint that is the same
 242        length as the sender's window (i.e., WINDOW_SIZE) until it
 243        receives its ACK.  To find a unique slot in this array, use
 244        (seq % WINDOW_SIZE). */
 245     sfi = WINDOW_SIZE_MOD(sseg->ss_base.us_btl_header->pkt_seq);
 246     assert(NULL == endpoint->endpoint_sent_segs[sfi]);
 247     endpoint->endpoint_sent_segs[sfi] = sseg;
 248     sseg->ss_ack_pending = true;
 249 
 250     /* bookkeeping */
 251     --endpoint->endpoint_send_credits;
 252 
 253     /* Stats */
 254     if (sseg->ss_parent_frag->sf_base.uf_type
 255             == OPAL_BTL_USNIC_FRAG_LARGE_SEND) {
 256         ++module->stats.num_chunk_sends;
 257     } else {
 258         ++module->stats.num_frag_sends;
 259     }
 260 
 261     /* If we have room in the sender's window, we also have room in
 262        endpoint hotel */
 263     opal_hotel_checkin_with_res(&endpoint->endpoint_hotel, sseg,
 264             &sseg->ss_hotel_room);
 265 }
 266 
 267 /*
 268  * This enqueues a fragment send into the system.  A send of a fragment
 269  * may result in the sending of multiple segments
 270  */
 271 static inline int
 272 opal_btl_usnic_endpoint_enqueue_frag(
 273     opal_btl_usnic_endpoint_t *endpoint,
 274     opal_btl_usnic_send_frag_t *frag)
 275 {
 276 #if MSGDEBUG1
 277     opal_output(0, "enq_frag: frag=%p, endpoint=%p, %s, len=%lu\n",
 278             (void*)frag, (void*)endpoint,
 279             usnic_frag_type(frag->sf_base.uf_type),
 280             (long unsigned)frag->sf_base.uf_base.des_src->seg_len);
 281     if (frag->sf_base.uf_type == OPAL_BTL_USNIC_FRAG_LARGE_SEND) {
 282         opal_btl_usnic_large_send_frag_t *lfrag;
 283         lfrag = (opal_btl_usnic_large_send_frag_t *)frag;
 284         opal_output(0, "   large size=%zd\n", lfrag->lsf_base.sf_size);
 285     }
 286 #endif
 287 
 288     /* add to tail of in-progress list */
 289     opal_list_append(&endpoint->endpoint_frag_send_queue,
 290             &frag->sf_base.uf_base.super.super);
 291 
 292     /* possibly make this endpoint ready to send again */
 293     opal_btl_usnic_check_rts(endpoint);
 294 
 295     return OPAL_SUCCESS;
 296 }
 297 
 298 static inline void
 299 opal_btl_usnic_release_send_segment(
 300     opal_btl_usnic_module_t *module,
 301     opal_btl_usnic_send_frag_t *frag,
 302     opal_btl_usnic_send_segment_t *sseg)
 303 {
 304     /* We only return CHUNK segments because they are the only send-style
 305      * segments that are dynamically allocated.
 306      */
 307     if (sseg->ss_base.us_type == OPAL_BTL_USNIC_SEG_CHUNK) {
 308         opal_btl_usnic_chunk_segment_return(module, sseg);
 309     }
 310 }
 311 
 312 void opal_btl_usnic_frag_complete(opal_btl_usnic_send_frag_t *frag);
 313 
 314 void opal_btl_usnic_frag_send_complete(opal_btl_usnic_module_t *module,
 315                                     opal_btl_usnic_send_segment_t *sseg);
 316 
 317 void opal_btl_usnic_chunk_send_complete(opal_btl_usnic_module_t *module,
 318                                     opal_btl_usnic_send_segment_t *sseg);
 319 
 320 int
 321 opal_btl_usnic_finish_put_or_send(
 322     opal_btl_usnic_module_t *module,
 323     opal_btl_usnic_endpoint_t *endpoint,
 324     opal_btl_usnic_send_frag_t *frag,
 325     mca_btl_base_tag_t tag)
 326 __opal_attribute_noinline__;
 327 
 328 #endif /* BTL_USNIC_SEND_H */