1 /*
2 * Copyright (c) 2004-2008 The Trustees of Indiana University and Indiana
3 * University Research and Technology
4 * Corporation. All rights reserved.
5 * Copyright (c) 2004-2011 The University of Tennessee and The University
6 * of Tennessee Research Foundation. All rights
7 * reserved.
8 * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
9 * University of Stuttgart. All rights reserved.
10 * Copyright (c) 2004-2005 The Regents of the University of California.
11 * All rights reserved.
12 * Copyright (c) 2006 Sandia National Laboratories. All rights
13 * reserved.
14 * Copyright (c) 2008-2019 Cisco Systems, Inc. All rights reserved
15 * Copyright (c) 2012 Los Alamos National Security, LLC. All rights
16 * reserved.
17 * $COPYRIGHT$
18 *
19 * Additional copyrights may follow
20 *
21 * $HEADER$
22 */
23
24 #include "opal_config.h"
25
26 #include "opal_stdint.h"
27
28 #include "opal/constants.h"
29
30 #include "opal/mca/btl/btl.h"
31 #include "opal/mca/btl/base/base.h"
32
33 #include "btl_usnic.h"
34 #include "btl_usnic_frag.h"
35 #include "btl_usnic_util.h"
36 #include "btl_usnic_send.h"
37 #include "btl_usnic_ack.h"
38
39
40 /*
41 * This function is called when a send of a segment completes that is
42 * the one-and-only segment of an MPI message. Return the WQE and
43 * also return the segment if no ACK pending.
44 */
45 void
46 opal_btl_usnic_frag_send_complete(opal_btl_usnic_module_t *module,
47 opal_btl_usnic_send_segment_t *sseg)
48 {
49 opal_btl_usnic_send_frag_t *frag;
50
51 frag = sseg->ss_parent_frag;
52
53 /* Reap a frag that was sent */
54 --sseg->ss_send_posted;
55 --frag->sf_seg_post_cnt;
56
57 /* checks for returnability made inside */
58 opal_btl_usnic_endpoint_t *ep = frag->sf_endpoint;
59 opal_btl_usnic_send_frag_return_cond(module, frag);
60
61 // In a short frag segment, the sseg is embedded in the frag. So
62 // there's no need to return the sseg (because we already returned
63 // the frag).
64
65 /* do bookkeeping */
66 ++ep->endpoint_send_credits;
67
68 /* see if this endpoint needs to be made ready-to-send */
69 opal_btl_usnic_check_rts(ep);
70
71 ++module->mod_channels[sseg->ss_channel].credits;
72 }
73
74 /*
75 * This function is called when a send segment completes that is part
76 * of a larger MPI message (ie., there may still be other chunk
77 * segments that have not yet completed sending). Return the WQE and
78 * also return the segment if no ACK pending.
79 */
80 void
81 opal_btl_usnic_chunk_send_complete(opal_btl_usnic_module_t *module,
82 opal_btl_usnic_send_segment_t *sseg)
83 {
84 opal_btl_usnic_send_frag_t *frag;
85
86 frag = sseg->ss_parent_frag;
87
88 /* Reap a frag that was sent */
89 --sseg->ss_send_posted;
90 --frag->sf_seg_post_cnt;
91
92 if (sseg->ss_send_posted == 0 && !sseg->ss_ack_pending) {
93 opal_btl_usnic_release_send_segment(module, frag, sseg);
94 }
95
96 /* done with whole fragment? */
97 /* checks for returnability made inside */
98 opal_btl_usnic_send_frag_return_cond(module, frag);
99
100 /* do bookkeeping */
101 ++frag->sf_endpoint->endpoint_send_credits;
102
103 /* see if this endpoint needs to be made ready-to-send */
104 opal_btl_usnic_check_rts(frag->sf_endpoint);
105
106 ++module->mod_channels[sseg->ss_channel].credits;
107 }
108
109 /* Responsible for completing non-fastpath parts of a put or send operation,
110 * including initializing any large frag bookkeeping fields and enqueuing the
111 * frag on the endpoint.
112 *
113 * This routine lives in this file to help prevent automatic inlining by the
114 * compiler.
115 *
116 * The "tag" only applies to sends.
117 */
118 int
119 opal_btl_usnic_finish_put_or_send(
120 opal_btl_usnic_module_t *module,
121 opal_btl_usnic_endpoint_t *endpoint,
122 opal_btl_usnic_send_frag_t *frag,
123 mca_btl_base_tag_t tag)
124 {
125 int rc;
126 opal_btl_usnic_small_send_frag_t *sfrag;
127 opal_btl_usnic_send_segment_t *sseg;
128
129 /*
130 * If this is small, need to do the copyin now.
131 * We don't do this earlier in case we got lucky and were
132 * able to do an inline send. We did not, so here we are...
133 */
134 if (frag->sf_base.uf_type == OPAL_BTL_USNIC_FRAG_SMALL_SEND) {
135
136 sfrag = (opal_btl_usnic_small_send_frag_t *)frag;
137 sseg = &sfrag->ssf_segment;
138
139 /* Copy in user data if there is any, collapsing 2 segments into 1.
140 * We already packed via the convertor if necessary, so we only need to
141 * handle the simple memcpy case here.
142 */
143 if (frag->sf_base.uf_base.USNIC_SEND_LOCAL_COUNT > 1) {
144 /* no convertor */
145 assert(NULL != frag->sf_base.uf_local_seg[1].seg_addr.pval);
146
147 memcpy(((char *)(intptr_t)frag->sf_base.uf_local_seg[0].seg_addr.lval +
148 frag->sf_base.uf_local_seg[0].seg_len),
149 frag->sf_base.uf_local_seg[1].seg_addr.pval,
150 frag->sf_base.uf_local_seg[1].seg_len);
151
152 /* update 1st segment length */
153 frag->sf_base.uf_base.USNIC_SEND_LOCAL_COUNT = 1;
154 frag->sf_base.uf_local_seg[0].seg_len +=
155 frag->sf_base.uf_local_seg[1].seg_len;
156 }
157
158 sseg->ss_len = sizeof(opal_btl_usnic_btl_header_t) + frag->sf_size;
159
160 /* use standard channel */
161 sseg->ss_channel = USNIC_DATA_CHANNEL;
162 sseg->ss_base.us_btl_header->tag = tag;
163 } else {
164 opal_btl_usnic_large_send_frag_t *lfrag;
165
166 /* Save info about the frag so that future invocations of
167 * usnic_handle_large_send can generate segments to put on the wire. */
168 lfrag = (opal_btl_usnic_large_send_frag_t *)frag;
169 lfrag->lsf_tag = tag;
170 lfrag->lsf_cur_offset = 0;
171 lfrag->lsf_cur_ptr = lfrag->lsf_des_src[0].seg_addr.pval;
172 lfrag->lsf_cur_sge = 0;
173 lfrag->lsf_bytes_left_in_sge = lfrag->lsf_des_src[0].seg_len;
174 lfrag->lsf_bytes_left = frag->sf_size;
175
176 if (lfrag->lsf_pack_on_the_fly) {
177 lfrag->lsf_pack_bytes_left = frag->sf_size;
178 } else {
179 /* we pre-packed the convertor into a chain in prepare_src */
180 lfrag->lsf_pack_bytes_left = 0;
181 }
182 }
183
184 /* queue this fragment into the send engine */
185 rc = opal_btl_usnic_endpoint_enqueue_frag(endpoint, frag);
186 return rc;
187 }