This source file includes following definitions.
- _predefined_data
- _contiguous_loop
- _copy_content_same_ddt
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16 #ifdef HAVE_ALLOCA_H
17 #include <alloca.h>
18 #endif
19
20 #if !defined(MEM_OP_NAME)
21 #error
22 #endif
23 #if !defined(MEM_OP)
24 #error
25 #endif
26
27 #ifndef STRINGIFY
28 # define STRINGIFY_(arg) #arg
29 # define STRINGIFY(arg) STRINGIFY_(arg)
30 #endif
31
32 #ifndef DT_CONCAT
33 # define DT__CONCAT(a, b) a##b
34 # define DT_CONCAT(a, b) DT__CONCAT(a, b)
35 #endif
36
37
38 #define _predefined_data DT_CONCAT(MEM_OP_NAME,_predefined_data)
39 #define _contiguous_loop DT_CONCAT(MEM_OP_NAME,_contiguous_loop)
40 #define _copy_content_same_ddt DT_CONCAT(MEM_OP_NAME,_copy_content_same_ddt)
41
42 static inline void _predefined_data( const dt_elem_desc_t* ELEM,
43 const opal_datatype_t* DATATYPE,
44 unsigned char* SOURCE_BASE,
45 size_t TOTAL_COUNT,
46 size_t COUNT,
47 unsigned char* SOURCE,
48 unsigned char* DESTINATION,
49 size_t* SPACE )
50 {
51 size_t _copy_count = (COUNT);
52 size_t _copy_blength;
53 const ddt_elem_desc_t* _elem = &((ELEM)->elem);
54 unsigned char* _source = (SOURCE) + _elem->disp;
55 unsigned char* _destination = (DESTINATION) + _elem->disp;
56
57 _copy_blength = opal_datatype_basicDatatypes[_elem->common.type]->size;
58
59 if( _copy_blength == (size_t)_elem->extent ) {
60 _copy_blength *= _copy_count;
61 OPAL_DATATYPE_SAFEGUARD_POINTER( _source, _copy_blength, (SOURCE_BASE),
62 (DATATYPE), (TOTAL_COUNT) );
63
64 DO_DEBUG( opal_output( 0, "copy 1. %s( %p, %p, %" PRIsize_t " ) => space %" PRIsize_t "\n",
65 STRINGIFY(MEM_OP_NAME), (void*)_destination, (void*)_source, _copy_blength, *(SPACE) ); );
66 MEM_OP( _destination, _source, _copy_blength );
67 _source += _copy_blength;
68 _destination += _copy_blength;
69 } else {
70 for(size_t _i = 0; _i < _copy_count; _i++ ) {
71 OPAL_DATATYPE_SAFEGUARD_POINTER( _source, _copy_blength, (SOURCE_BASE),
72 (DATATYPE), (TOTAL_COUNT) );
73 DO_DEBUG( opal_output( 0, "copy 2. %s( %p, %p, %lu ) => space %lu\n",
74 STRINGIFY(MEM_OP_NAME), (void*)_destination, (void*)_source, (unsigned long)_copy_blength, (unsigned long)(*(SPACE) - (_i * _copy_blength)) ); );
75 MEM_OP( _destination, _source, _copy_blength );
76 _source += _elem->extent;
77 _destination += _elem->extent;
78 }
79 _copy_blength *= _copy_count;
80 }
81 *(SPACE) -= _copy_blength;
82 }
83
84 static inline void _contiguous_loop( const dt_elem_desc_t* ELEM,
85 const opal_datatype_t* DATATYPE,
86 unsigned char* SOURCE_BASE,
87 size_t TOTAL_COUNT,
88 size_t COUNT,
89 unsigned char* SOURCE,
90 unsigned char* DESTINATION,
91 size_t* SPACE )
92 {
93 ddt_loop_desc_t *_loop = (ddt_loop_desc_t*)(ELEM);
94 ddt_endloop_desc_t* _end_loop = (ddt_endloop_desc_t*)((ELEM) + _loop->items);
95 unsigned char* _source = (SOURCE) + _end_loop->first_elem_disp;
96 unsigned char* _destination = (DESTINATION) + _end_loop->first_elem_disp;
97 size_t _copy_loops = (COUNT);
98
99 if( _loop->extent == (ptrdiff_t)_end_loop->size ) {
100 _copy_loops *= _end_loop->size;
101 OPAL_DATATYPE_SAFEGUARD_POINTER( _source, _copy_loops, (SOURCE_BASE),
102 (DATATYPE), (TOTAL_COUNT) );
103 MEM_OP( _destination, _source, _copy_loops );
104 } else {
105 for(size_t _i = 0; _i < _copy_loops; _i++ ) {
106 OPAL_DATATYPE_SAFEGUARD_POINTER( _source, _end_loop->size, (SOURCE_BASE),
107 (DATATYPE), (TOTAL_COUNT) );
108 DO_DEBUG( opal_output( 0, "copy 3. %s( %p, %p, %" PRIsize_t " ) => space %" PRIsize_t "\n",
109 STRINGIFY(MEM_OP_NAME), (void*)_destination, (void*)_source, _end_loop->size, *(SPACE) - _i * _end_loop->size ); );
110 MEM_OP( _destination, _source, _end_loop->size );
111 _source += _loop->extent;
112 _destination += _loop->extent;
113 }
114 _copy_loops *= _end_loop->size;
115 }
116 *(SPACE) -= _copy_loops;
117 }
118
119 static inline int32_t _copy_content_same_ddt( const opal_datatype_t* datatype, int32_t count,
120 char* destination_base, char* source_base )
121 {
122 dt_stack_t* pStack;
123 int32_t stack_pos;
124 uint32_t pos_desc;
125 uint32_t count_desc;
126 dt_elem_desc_t* description;
127 dt_elem_desc_t* pElem;
128 size_t iov_len_local;
129 unsigned char *source = (unsigned char*)source_base,
130 *destination = (unsigned char*)destination_base;
131
132 DO_DEBUG( opal_output( 0, "_copy_content_same_ddt( %p, %d, dst %p, src %p )\n",
133 (void*)datatype, count, (void*)destination_base, (void*)source_base ); );
134
135 iov_len_local = count * datatype->size;
136
137
138
139
140 if( datatype->flags & OPAL_DATATYPE_FLAG_CONTIGUOUS ) {
141 ptrdiff_t extent = (datatype->ub - datatype->lb);
142
143
144
145 destination += datatype->true_lb;
146 source += datatype->true_lb;
147 if( (ptrdiff_t)datatype->size == extent ) {
148 size_t total_length = iov_len_local;
149 size_t memop_chunk = opal_datatype_memop_block_size;
150 while( total_length > 0 ) {
151 if( memop_chunk > total_length ) memop_chunk = total_length;
152 OPAL_DATATYPE_SAFEGUARD_POINTER( destination, memop_chunk,
153 (unsigned char*)destination_base, datatype, count );
154 OPAL_DATATYPE_SAFEGUARD_POINTER( source, memop_chunk,
155 (unsigned char*)source_base, datatype, count );
156 DO_DEBUG( opal_output( 0, "copy c1. %s( %p, %p, %lu ) => space %lu\n",
157 STRINGIFY(MEM_OP_NAME), (void*)destination, (void*)source, (unsigned long)memop_chunk, (unsigned long)total_length ); );
158 MEM_OP( destination, source, memop_chunk );
159 destination += memop_chunk;
160 source += memop_chunk;
161 total_length -= memop_chunk;
162 }
163 return 0;
164 }
165 for( pos_desc = 0; (int32_t)pos_desc < count; pos_desc++ ) {
166 OPAL_DATATYPE_SAFEGUARD_POINTER( destination, datatype->size,
167 (unsigned char*)destination_base, datatype, count );
168 OPAL_DATATYPE_SAFEGUARD_POINTER( source, datatype->size,
169 (unsigned char*)source_base, datatype, count );
170 DO_DEBUG( opal_output( 0, "copy c2. %s( %p, %p, %lu ) => space %lu\n",
171 STRINGIFY(MEM_OP_NAME), (void*)destination, (void*)source, (unsigned long)datatype->size,
172 (unsigned long)(iov_len_local - (pos_desc * datatype->size)) ); );
173 MEM_OP( destination, source, datatype->size );
174 destination += extent;
175 source += extent;
176 }
177 return 0;
178 }
179
180 pStack = (dt_stack_t*)alloca( sizeof(dt_stack_t) * (datatype->loops + 1) );
181 pStack->count = count;
182 pStack->index = -1;
183 pStack->disp = 0;
184 pos_desc = 0;
185 stack_pos = 0;
186
187 if( datatype->opt_desc.desc != NULL ) {
188 description = datatype->opt_desc.desc;
189 } else {
190 description = datatype->desc.desc;
191 }
192
193 if( description[0].elem.common.type == OPAL_DATATYPE_LOOP )
194 count_desc = description[0].loop.loops;
195 else
196 count_desc = description[0].elem.count;
197 pElem = &(description[pos_desc]);
198
199 while( 1 ) {
200 while( OPAL_LIKELY(pElem->elem.common.flags & OPAL_DATATYPE_FLAG_DATA) ) {
201
202 _predefined_data( pElem, datatype, (unsigned char*)source_base, count, count_desc,
203 source, destination, &iov_len_local );
204 pos_desc++;
205 UPDATE_INTERNAL_COUNTERS( description, pos_desc, pElem, count_desc );
206 }
207 if( OPAL_DATATYPE_END_LOOP == pElem->elem.common.type ) {
208 DO_DEBUG( opal_output( 0, "copy end_loop count %" PRIsize_t " stack_pos %d pos_desc %d disp %ld space %lu\n",
209 pStack->count, stack_pos, pos_desc, pStack->disp, (unsigned long)iov_len_local ); );
210 if( --(pStack->count) == 0 ) {
211 if( stack_pos == 0 ) {
212 assert( iov_len_local == 0 );
213 return 0;
214 }
215 stack_pos--;
216 pStack--;
217 pos_desc++;
218 } else {
219 pos_desc = pStack->index + 1;
220 if( pStack->index == -1 ) {
221 pStack->disp += (datatype->ub - datatype->lb);
222 } else {
223 assert( OPAL_DATATYPE_LOOP == description[pStack->index].loop.common.type );
224 pStack->disp += description[pStack->index].loop.extent;
225 }
226 }
227 source = (unsigned char*)source_base + pStack->disp;
228 destination = (unsigned char*)destination_base + pStack->disp;
229 UPDATE_INTERNAL_COUNTERS( description, pos_desc, pElem, count_desc );
230 DO_DEBUG( opal_output( 0, "copy new_loop count %" PRIsize_t " stack_pos %d pos_desc %d disp %ld space %lu\n",
231 pStack->count, stack_pos, pos_desc, pStack->disp, (unsigned long)iov_len_local ); );
232 }
233 if( OPAL_DATATYPE_LOOP == pElem->elem.common.type ) {
234 ptrdiff_t local_disp = (ptrdiff_t)source;
235 if( pElem->loop.common.flags & OPAL_DATATYPE_FLAG_CONTIGUOUS ) {
236 _contiguous_loop( pElem, datatype, (unsigned char*)source_base, count, count_desc,
237 source, destination, &iov_len_local );
238 pos_desc += pElem->loop.items + 1;
239 goto update_loop_description;
240 }
241 local_disp = (ptrdiff_t)source - local_disp;
242 PUSH_STACK( pStack, stack_pos, pos_desc, OPAL_DATATYPE_LOOP, count_desc,
243 pStack->disp + local_disp);
244 pos_desc++;
245 update_loop_description:
246 source = (unsigned char*)source_base + pStack->disp;
247 destination = (unsigned char*)destination_base + pStack->disp;
248 UPDATE_INTERNAL_COUNTERS( description, pos_desc, pElem, count_desc );
249 DDT_DUMP_STACK( pStack, stack_pos, pElem, "advance loop" );
250 continue;
251 }
252 }
253 }