1 #if HAVE_CONFIG_H
2 # include "config.h"
3 #endif
4
5 /* $Id: memlock.c,v 1.24.2.3 2007-08-29 17:32:32 manoj Exp $ */
6 #include "armcip.h"
7 #include "locks.h"
8 #include "copy.h"
9 #include "memlock.h"
10 #if HAVE_STDIO_H
11 # include <stdio.h>
12 #endif
13
14 #define DEBUG_ 0
15 #define INVALID_VAL -9999999
16
17 #ifdef DATA_SERVER
18 # define CORRECT_PTR
19 #endif
20 static size_t armci_mem_offset=0;
21
22 /* We start by using table: assign address of local variable set to 1
23 * On shmem systems, this addres is overwritten by a shared memory location
24 * when memlock array is allocated in armci_init
25 * Therefore, any process within shmem node can reset armci_use_memlock_table
26 * to "not used" when offset changes. Since the variable is in shmem, everybody
27 * on that SMP node will see the change and use the same locking functions
28 */
29 int init_use_memlock_table=1;
30 int *armci_use_memlock_table=&init_use_memlock_table;
31
32 static int locked_slot=INVALID_VAL;
33
34 volatile double armci_dummy_work=0.;
35 void **memlock_table_array;
36
37 /* constants for cache line alignment */
38 #ifdef SOLARIS
39 # define CALGN 32
40 # define LOG_CALGN 5
41 #else
42 # define CALGN 64
43 # define LOG_CALGN 6
44 #endif
45
46 #define ALIGN_ADDRESS(x) (char*)((((unsigned long)x) >> LOG_CALGN) << LOG_CALGN)
47 #ifdef CRAY_T3E
48 #pragma _CRI cache_align table
49 #endif
50 static memlock_t table[MAX_SLOTS];
51
52 #if defined(SGIALTIX) || (defined(CRAY_SHMEM) && defined(CRAY_XT))
53 #define MAX_SEGS 512
54 armci_memoffset_t armci_memoffset_table[MAX_SEGS];
55 static short int seg_count=0;
56 static short int new_seg=0;
57 #endif
58
59 /*\ simple locking scheme that ignores addresses
60 \*/
armci_lockmem_(void * pstart,void * pend,int proc)61 void armci_lockmem_(void *pstart, void *pend, int proc)
62 {
63 #ifdef BGML
64 bgml_lockmem(pstart, pend, proc);
65 #else
66
67 #if defined(CLUSTER) && !defined(SGIALTIX)
68 int lock = (proc-armci_clus_info[armci_clus_id(proc)].master)%NUM_LOCKS;
69 #else
70 int lock = 0;
71 #endif
72
73 if(DEBUG_){
74 printf("%d: armci_lockmem_ proc=%d lock=%d\n",armci_me,proc,lock);
75 fflush(stdout);
76 }
77
78 NATIVE_LOCK(lock,proc);
79 # ifdef LAPI
80 {
81 extern int kevin_ok;
82 kevin_ok=0;
83 }
84 # endif
85 if(DEBUG_){
86 printf("%d: armci_lockmem_ done\n",armci_me);
87 fflush(stdout);
88 }
89 #endif
90 }
91
armci_unlockmem_(int proc)92 void armci_unlockmem_(int proc)
93 {
94 #ifdef BGML
95 bgml_unlockmem(proc);
96 #else
97
98 #if defined(CLUSTER) && !defined(SGIALTIX)
99 int lock = (proc-armci_clus_info[armci_clus_id(proc)].master)%NUM_LOCKS;
100 #else
101 int lock = 0;
102 #endif
103 NATIVE_UNLOCK(lock,proc);
104 # ifdef LAPI
105 {
106 extern int kevin_ok;
107 kevin_ok=1;
108 }
109 # endif
110 #endif
111 }
112
113
114
115 /*\ idle for a time proportional to factor
116 \*/
armci_waitsome(int factor)117 void armci_waitsome(int factor)
118 {
119 int i=factor*100000;
120
121 if(factor <= 1) armci_dummy_work =0.;
122 if(factor < 1) return;
123 while(--i){
124 armci_dummy_work = armci_dummy_work + 1./(double)i;
125 }
126 }
127
128 #ifdef SGIALTIX
129 #include <mpp/shmem.h>
130 #endif
131
132 /*\ acquire exclusive LOCK to MEMORY area <pstart,pend> owned by process "proc"
133 * . only one area can be locked at a time by the calling process
134 * . must unlock it with armci_unlockmem
135 \*/
armci_lockmem(void * start,void * end,int proc)136 void armci_lockmem(void *start, void *end, int proc)
137 {
138 #ifdef ARMCIX
139 ARMCIX_Lockmem (start, end, proc);
140 #else
141 register void* pstart, *pend;
142 register int slot, avail=0;
143 int turn=0, conflict=0;
144 memlock_t *memlock_table;
145 #if defined(CLUSTER) && !defined(SGIALTIX)
146 int lock = (proc-armci_clus_info[armci_clus_id(proc)].master)%NUM_LOCKS;
147 #else
148 int lock = 0;
149 #endif
150
151 #ifdef CORRECT_PTR
152 if(! *armci_use_memlock_table){
153 /* if offset invalid, use dumb locking scheme ignoring addresses */
154 armci_lockmem_(start, end, proc);
155 return;
156 }
157
158 # ifndef SGIALTIX
159 /* when processes are attached to a shmem region at different addresses,
160 * addresses written to memlock table must be adjusted to the node master
161 */
162 if(armci_mem_offset){
163 start = armci_mem_offset + (char*)start;
164 end = armci_mem_offset + (char*)end;
165 }
166 # endif
167 #endif
168
169 if(DEBUG_){
170 printf("%d: calling armci_lockmem for %d range %p -%p\n",
171 armci_me, proc, start,end);
172 fflush(stdout);
173 }
174 memlock_table = (memlock_t*)memlock_table_array[proc];
175
176
177 #ifdef ALIGN_ADDRESS
178 /* align address range on cache line boundary to avoid false sharing */
179 pstart = ALIGN_ADDRESS(start);
180 pend = CALGN -1 + ALIGN_ADDRESS(end);
181 #else
182 pstart=start;
183 pend =end;
184 #endif
185
186 #ifdef CRAY_SHMEM
187 { /* adjust according the remote process raw address */
188 long bytes = (long) ((char*)pend-(char*)pstart);
189 extern void* armci_shmalloc_remote_addr(void *ptr, int proc);
190 pstart = armci_shmalloc_remote_addr(pstart, proc);
191 pend = (char*)pstart + bytes;
192 }
193 #endif
194 #ifdef SGIALTIX
195 if (proc == armci_me) {
196 pstart = shmem_ptr(pstart,armci_me);
197 pend = shmem_ptr(pend,armci_me);
198 }
199 /* In SGI Altix processes are attached to a shmem region at different
200 addresses. Addresses written to memlock table must be adjusted to
201 the node master
202 */
203 if(ARMCI_Uses_shm()){
204 int i, seg_id=-1;
205 size_t tile_size,offset;
206 void *start_addr, *end_addr;
207 for(i=0; i<seg_count; i++) {
208 tile_size = armci_memoffset_table[i].tile_size;
209 start_addr = (void*) ((char*)armci_memoffset_table[i].seg_addr +
210 proc*tile_size);
211 end_addr = (void*) ((char*)start_addr +
212 armci_memoffset_table[i].seg_size);
213 /* CHECK: because of too much "span" in armci_lockmem_patch in
214 * strided.c, it is not possible to have condition as (commented):*/
215 /*if(pstart>=start_addr && pend<=end_addr) {seg_id=i; break;}*/
216 if(pstart >= start_addr && pstart <= end_addr) {seg_id=i; break;}
217 }
218 if(seg_id==-1) armci_die("armci_lockmem: Invalid segment", seg_id);
219
220 offset = armci_memoffset_table[seg_id].mem_offset;
221 pstart = ((char*)pstart + offset);
222 pend = ((char*)pend + offset);
223 }
224 #endif
225
226 while(1){
227 NATIVE_LOCK(lock,proc);
228
229 armci_get(memlock_table, table, sizeof(table), proc);
230 /* armci_copy(memlock_table, table, sizeof(table));*/
231
232 /* inspect the table */
233 conflict = 0; avail =-1;
234 for(slot = 0; slot < MAX_SLOTS; slot ++){
235
236 /* nonzero starting address means the slot is occupied */
237 if(table[slot].start == NULL){
238
239 /* remember a free slot to store address range */
240 avail = slot;
241
242 }else{
243 /*check for conflict: overlap between stored and current range*/
244 if( (pstart >= table[slot].start && pstart <= table[slot].end)
245 || (pend >= table[slot].start && pend <= table[slot].end) ){
246
247 conflict = 1;
248 break;
249
250 }
251 /*
252 printf("%d: locking %ld-%ld (%d) conflict\n",
253 armci_me, */
254 }
255 }
256
257 if(avail != -1 && !conflict) break;
258
259 NATIVE_UNLOCK(lock,proc);
260 armci_waitsome( ++turn );
261
262 }
263
264 /* we got the memory lock: enter address into the table */
265 table[avail].start = pstart;
266 table[avail].end = pend;
267 armci_put(table+avail,memlock_table+avail,sizeof(memlock_t),proc);
268
269 FENCE_NODE(proc);
270
271 NATIVE_UNLOCK(lock,proc);
272 locked_slot = avail;
273 #endif /* ! ARMCIX */
274 }
275
276
277 /*\ release lock to the memory area locked by previous call to armci_lockemem
278 \*/
armci_unlockmem(int proc)279 void armci_unlockmem(int proc)
280 {
281 #ifdef ARMCIX
282 ARMCIX_Unlockmem (proc);
283 #else
284
285 void *null[2] = {NULL,NULL};
286 memlock_t *memlock_table;
287
288 #ifdef CORRECT_PTR
289 if(! *armci_use_memlock_table){
290 /* if offset invalid, use dumb locking scheme ignoring addresses */
291 armci_unlockmem_(proc);
292 return;
293 }
294 #endif
295
296 #ifdef DEBUG
297 if(locked_slot == INVALID_VAL) armci_die("armci_unlock: empty",0);
298 if(locked_slot >= MAX_SLOTS || locked_slot <0)
299 armci_die("armci_unlock: corrupted slot?",locked_slot);
300 #endif
301
302 memlock_table = (memlock_t*)memlock_table_array[proc];
303 armci_put(null,&memlock_table[locked_slot].start,2*sizeof(void*),proc);
304 #endif /* ! ARMCIX */
305 }
306
307
308
309 /*\ based on address for set by master, determine correction for
310 * memory addresses set in memlock table
311 * if the correction/offset ever changes stop using memlock table locking
312 \*/
armci_set_mem_offset(void * ptr)313 void armci_set_mem_offset(void *ptr)
314 {
315 size_t off;
316 static int first_time=1;
317 volatile void *ref_ptr;
318
319 /* do not care if memlock not used */
320 if(! *armci_use_memlock_table) return;
321
322 if(!ptr) armci_die("armci_set_mem_offset : null ptr",0);
323 ref_ptr = *(void**)ptr;
324 off = (size_t)((char*)ref_ptr - (char*)ptr);
325
326 if(first_time){
327
328 armci_mem_offset =off;
329 first_time =0;
330 if(DEBUG_){
331 printf("%d memlock offset=%ld ref=%p ptr=%p\n",armci_me,
332 (long)armci_mem_offset, ref_ptr, ptr); fflush(stdout);
333 }
334
335 }else{
336 if(armci_mem_offset != off){
337 *armci_use_memlock_table =0;
338 fprintf(stderr, "%d: WARNING:armci_set_mem_offset: offset changed %ld to %ld\n",
339 armci_me, (long)armci_mem_offset, (long)off); fflush(stdout);
340 }
341 }
342 }
343
344 #ifdef SGIALTIX
345 /* SGI Altix Stuff */
armci_altix_gettilesize(void * ptr,void ** ptr_arr,size_t * tile_size)346 static void armci_altix_gettilesize(void *ptr, void **ptr_arr,
347 size_t *tile_size) {
348 int i;
349 size_t diff=0;
350 for(i=0; i<armci_nproc; i++) {
351 ptr_arr[i]=shmem_ptr(ptr,i);
352 if(i>0) diff = (size_t)((char*)ptr_arr[i]-(char*)ptr_arr[i-1]);
353 if(i>1 && diff!=*tile_size)
354 armci_die("armci_memoffset_table_newentry:Inconsistent tile size",
355 armci_me);
356 *tile_size = diff;
357 }
358 }
359
armci_memoffset_table_newentry(void * ptr,size_t seg_size)360 void armci_memoffset_table_newentry(void *ptr, size_t seg_size) {
361
362 void **ptr_arr;
363 void *master_addr = NULL;
364 size_t tile_size=0, offset=0;
365
366 if(!ptr) armci_die("armci_memoffset_table_newentry : null ptr",0);
367
368 if(seg_count >= MAX_SEGS) /* CHECK: make it dynamic */
369 armci_die("armci_altix_allocate: Increase MAX_SEGS > 512", armci_me);
370
371 if(armci_me == armci_master) master_addr = shmem_ptr(ptr, armci_me);
372 armci_msg_brdcst(&master_addr, sizeof(void*), armci_master);
373
374 ptr_arr = (void**)malloc(armci_nproc*sizeof(void*));
375 armci_altix_gettilesize(ptr, ptr_arr, &tile_size);
376 offset = (size_t)((char*)master_addr - (char*)ptr_arr[armci_master]);
377
378 /* enter in memoffset table */
379 armci_memoffset_table[seg_count].seg_addr = ptr_arr[armci_master];
380 armci_memoffset_table[seg_count].seg_size = seg_size;
381 armci_memoffset_table[seg_count].tile_size = tile_size;
382 armci_memoffset_table[seg_count].mem_offset = offset;
383
384 #if DEBUG_
385 printf("%d: addr=%p seg_size=%ld tile_size=%ld offset=%ld\n", armci_me,
386 ptr_arr[armci_master], seg_size, tile_size, offset);
387 #endif
388
389 ++seg_count;
390 free(ptr_arr);
391 }
392 #endif
393
394 #if defined(CRAY_SHMEM) && defined(CRAY_XT)
395 /* CRAY-CRAY_XT stuff */
armci_cray_gettilesize(void * ptr,void ** ptr_arr,size_t * tile_size)396 static void armci_cray_gettilesize(void *ptr, void **ptr_arr,
397 size_t *tile_size) {
398 int i;
399 size_t diff=0;
400 for(i=0; i<armci_nproc; i++) {
401 ptr_arr[i]=ptr;
402 if(i>0) diff = (size_t)((char*)ptr_arr[i]-(char*)ptr_arr[i-1]);
403 if(i>1 && diff!=*tile_size)
404 armci_die("armci_memoffset_table_newentry:Inconsistent tile size",
405 armci_me);
406 *tile_size = diff;
407 }
408 }
409
armci_memoffset_table_newentry(void * ptr,size_t seg_size)410 void armci_memoffset_table_newentry(void *ptr, size_t seg_size) {
411
412 void **ptr_arr;
413 void *master_addr = NULL;
414 size_t tile_size=0, offset=0;
415
416 if(!ptr) armci_die("armci_memoffset_table_newentry : null ptr",0);
417
418 if(seg_count >= MAX_SEGS) /* CHECK: make it dynamic */
419 armci_die("armci_cary_allocate: Increase MAX_SEGS > 512", armci_me);
420
421 if(armci_me == armci_master) master_addr = ptr;
422 armci_msg_brdcst(&master_addr, sizeof(void*), armci_master);
423
424 ptr_arr = (void**)malloc(armci_nproc*sizeof(void*));
425 armci_cray_gettilesize(ptr, ptr_arr, &tile_size);
426 offset = (size_t)((char*)master_addr - (char*)ptr_arr[armci_master]);
427
428 /* enter in memoffset table */
429 armci_memoffset_table[seg_count].seg_addr = ptr_arr[armci_master];
430 armci_memoffset_table[seg_count].seg_size = seg_size;
431 armci_memoffset_table[seg_count].tile_size = tile_size;
432 armci_memoffset_table[seg_count].mem_offset = offset;
433
434 #if DEBUG_
435 printf("%d: addr=%p seg_size=%ld tile_size=%ld offset=%ld\n", armci_me,
436 ptr_arr[armci_master], seg_size, tile_size, offset);
437 #endif
438
439 ++seg_count;
440 free(ptr_arr);
441 }
442 #endif
443