1 /*****************************************************************************
2 * *
3 * Copyright (c) 2003-2006 Intel Corporation. *
4 * All rights reserved. *
5 * *
6 *****************************************************************************
7
8 This code is covered by the Community Source License (CPL), version
9 1.0 as published by IBM and reproduced in the file "license.txt" in the
10 "license" subdirectory. Redistribution in source and binary form, with
11 or without modification, is permitted ONLY within the regulations
12 contained in above mentioned license.
13
14 Use of the name and trademark "Intel(R) MPI Benchmarks" is allowed ONLY
15 within the regulations of the "License for Use of "Intel(R) MPI
16 Benchmarks" Name and Trademark" as reproduced in the file
17 "use-of-trademark-license.txt" in the "license" subdirectory.
18
19 THE PROGRAM IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OR
20 CONDITIONS OF ANY KIND, EITHER EXPRESS OR IMPLIED INCLUDING, WITHOUT
21 LIMITATION, ANY WARRANTIES OR CONDITIONS OF TITLE, NON-INFRINGEMENT,
22 MERCHANTABILITY OR FITNESS FOR A PARTICULAR PURPOSE. Each Recipient is
23 solely responsible for determining the appropriateness of using and
24 distributing the Program and assumes all risks associated with its
25 exercise of rights under this Agreement, including but not limited to
26 the risks and costs of program errors, compliance with applicable
27 laws, damage to or loss of data, programs or equipment, and
28 unavailability or interruption of operations.
29
30 EXCEPT AS EXPRESSLY SET FORTH IN THIS AGREEMENT, NEITHER RECIPIENT NOR
31 ANY CONTRIBUTORS SHALL HAVE ANY LIABILITY FOR ANY DIRECT, INDIRECT,
32 INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING
33 WITHOUT LIMITATION LOST PROFITS), HOWEVER CAUSED AND ON ANY THEORY OF
34 LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
35 NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OR
36 DISTRIBUTION OF THE PROGRAM OR THE EXERCISE OF ANY RIGHTS GRANTED
37 HEREUNDER, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGES.
38
39 EXPORT LAWS: THIS LICENSE ADDS NO RESTRICTIONS TO THE EXPORT LAWS OF
40 YOUR JURISDICTION. It is licensee's responsibility to comply with any
41 export regulations applicable in licensee's jurisdiction. Under
42 CURRENT U.S. export regulations this software is eligible for export
43 from the U.S. and can be downloaded by or otherwise exported or
44 reexported worldwide EXCEPT to U.S. embargoed destinations which
45 include Cuba, Iraq, Libya, North Korea, Iran, Syria, Sudan,
46 Afghanistan and any other country to which the U.S. has embargoed
47 goods and services.
48
49 ***************************************************************************
50
51 For more documentation than found here, see
52
53 [1] doc/ReadMe_IMB.txt
54
55 [2] Intel (R) MPI Benchmarks
56 Users Guide and Methodology Description
57 In
58 doc/IMB_ug.pdf
59
60 File: IMB_write.c
61
62 Implemented functions:
63
64 IMB_write_shared;
65 IMB_write_indv;
66 IMB_write_expl;
67 IMB_write_ij;
68 IMB_iwrite_ij;
69
70 ***************************************************************************/
71
72
73
74
75 #include "IMB_declare.h"
76 #include "IMB_benchmark.h"
77
78 #include "IMB_prototypes.h"
79
80
81 /*************************************************************************/
82
83
84
IMB_write_shared(struct comm_info * c_info,int size,int n_sample,MODES RUN_MODE,double * time)85 void IMB_write_shared(struct comm_info* c_info, int size, int n_sample,
86 MODES RUN_MODE, double* time)
87 /*
88
89
90 MPI-IO benchmark kernel
91 Driver for write benchmarks with shared file pointers
92
93
94
95 Input variables:
96
97 -c_info (type struct comm_info*)
98 Collection of all base data for MPI;
99 see [1] for more information
100
101
102 -size (type int)
103 Basic message size in bytes
104
105 -n_sample (type int)
106 Number of repetitions (for timing accuracy)
107
108 -RUN_MODE (type MODES)
109 Mode (aggregate/non aggregate; blocking/nonblocking);
110 see "IMB_benchmark.h" for definition
111
112
113 Output variables:
114
115 -time (type double*)
116 Timing result per sample
117
118
119 */
120 {
121 if( RUN_MODE->AGGREGATE )
122 IMB_write_ij(c_info, size, shared, RUN_MODE->type, 1, n_sample, 1, time);
123 else
124 IMB_write_ij(c_info, size, shared, RUN_MODE->type, n_sample, 1, 0, time);
125
126 if( RUN_MODE->NONBLOCKING )
127 {
128 MPI_File_close(&c_info->fh);
129 IMB_open_file(c_info);
130
131 if( RUN_MODE->AGGREGATE )
132 IMB_iwrite_ij(c_info, size, shared, RUN_MODE->type, 1, n_sample, 1, 1, time+1);
133 else
134 IMB_iwrite_ij(c_info, size, shared, RUN_MODE->type, n_sample, 1, 0, 1, time+1);
135 }
136 }
137
138 /*************************************************************************/
139
140
141
IMB_write_indv(struct comm_info * c_info,int size,int n_sample,MODES RUN_MODE,double * time)142 void IMB_write_indv(struct comm_info* c_info, int size, int n_sample,
143 MODES RUN_MODE, double* time)
144 /*
145
146
147 MPI-IO benchmark kernel
148 Driver for write benchmarks with individual file pointers
149
150
151
152 Input variables:
153
154 -c_info (type struct comm_info*)
155 Collection of all base data for MPI;
156 see [1] for more information
157
158
159 -size (type int)
160 Basic message size in bytes
161
162 -n_sample (type int)
163 Number of repetitions (for timing accuracy)
164
165 -RUN_MODE (type MODES)
166 Mode (aggregate/non aggregate; blocking/nonblocking);
167 see "IMB_benchmark.h" for definition
168
169
170 Output variables:
171
172 -time (type double*)
173 Timing result per sample
174
175
176 */
177 {
178 if( RUN_MODE->AGGREGATE )
179 IMB_write_ij(c_info, size, indv_block, RUN_MODE->type, 1, n_sample, 1, time);
180 else
181 IMB_write_ij(c_info, size, indv_block, RUN_MODE->type, n_sample, 1, 0, time);
182
183 if( RUN_MODE->NONBLOCKING )
184 {
185 MPI_File_close(&c_info->fh);
186 IMB_open_file(c_info);
187
188 if( RUN_MODE->AGGREGATE )
189 IMB_iwrite_ij(c_info, size, indv_block, RUN_MODE->type, 1, n_sample, 1, 1, time+1);
190 else
191 IMB_iwrite_ij(c_info, size, indv_block, RUN_MODE->type, n_sample, 1, 0, 1, time+1);
192 }
193
194 }
195
196 /*************************************************************************/
197
198
199
IMB_write_expl(struct comm_info * c_info,int size,int n_sample,MODES RUN_MODE,double * time)200 void IMB_write_expl(struct comm_info* c_info, int size, int n_sample,
201 MODES RUN_MODE, double* time)
202 /*
203
204
205 MPI-IO benchmark kernel
206 Driver for write benchmarks with explicit offsets
207
208
209
210 Input variables:
211
212 -c_info (type struct comm_info*)
213 Collection of all base data for MPI;
214 see [1] for more information
215
216
217 -size (type int)
218 Basic message size in bytes
219
220 -n_sample (type int)
221 Number of repetitions (for timing accuracy)
222
223 -RUN_MODE (type MODES)
224 Mode (aggregate/non aggregate; blocking/nonblocking);
225 see "IMB_benchmark.h" for definition
226
227
228 Output variables:
229
230 -time (type double*)
231 Timing result per sample
232
233
234 */
235 {
236 if( RUN_MODE->AGGREGATE )
237 IMB_write_ij(c_info, size, explicit, RUN_MODE->type, 1, n_sample, 1, time);
238 else
239 IMB_write_ij(c_info, size, explicit, RUN_MODE->type, n_sample, 1, 0, time);
240
241 if( RUN_MODE->NONBLOCKING )
242 {
243 MPI_File_close(&c_info->fh);
244 IMB_open_file(c_info);
245
246 if( RUN_MODE->AGGREGATE )
247 IMB_iwrite_ij(c_info, size, explicit, RUN_MODE->type, 1, n_sample, 1, 1, time+1);
248 else
249 IMB_iwrite_ij(c_info, size, explicit, RUN_MODE->type, n_sample, 1, 0, 1, time+1);
250 }
251 }
252
253
IMB_write_ij(struct comm_info * c_info,int size,POSITIONING pos,BTYPES type,int i_sample,int j_sample,int time_inner,double * time)254 void IMB_write_ij(struct comm_info* c_info, int size, POSITIONING pos,
255 BTYPES type, int i_sample, int j_sample,
256 int time_inner, double* time)
257 /*
258
259
260 Calls the proper write functions, blocking case
261
262 (See IMB_read_ij for documentation of calling sequence)
263
264
265
266 */
267 {
268 int i, j;
269 int Locsize,Totalsize,Ioffs;
270 MPI_Status stat;
271 MPI_Offset Offset;
272
273 ierr = 0;
274
275 *time=0.;
276 if( c_info->File_rank >= 0 )
277 {
278
279 int (* GEN_File_write)(MPI_File fh, void* buf, int count,
280 MPI_Datatype datatype, MPI_Status *status);
281 int (* GEN_File_write_shared)
282 (MPI_File fh, void* buf, int count,
283 MPI_Datatype datatype, MPI_Status *status);
284 int (* GEN_File_write_at)
285 (MPI_File fh, MPI_Offset offset, void* buf, int count,
286 MPI_Datatype datatype, MPI_Status *status);
287
288 #ifdef CHECK
289 int chk_mode;
290
291 if( pos == shared && type != Collective) chk_mode = -3;
292 else chk_mode = -2;
293
294 defect = 0.;
295 #endif
296
297 if (type == Collective )
298 {
299 GEN_File_write = MPI_File_write_all;
300 GEN_File_write_shared = MPI_File_write_ordered;
301 GEN_File_write_at = MPI_File_write_at_all;
302 #ifdef DEBUG
303 fprintf(unit,"Collective output\n");
304 #endif
305 }
306 else
307 {
308 GEN_File_write = MPI_File_write;
309 GEN_File_write_shared = MPI_File_write_shared;
310 GEN_File_write_at = MPI_File_write_at;
311 #ifdef DEBUG
312 fprintf(unit,"Non collective output\n");
313 #endif
314 }
315
316 Locsize = c_info->split.Locsize;
317 Totalsize = c_info->split.Totalsize;
318 Offset = (MPI_Offset)c_info->split.Offset;
319
320 /*
321 !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
322 BLOCKING COLLECTIVE/NON COLLECTIVE OUTPUT CASES COMBINED
323 (function pointer GEN_File_write_XXX
324 either standard or collective MPI_File_write_XXX
325 !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
326 */
327
328 for(i=0; i<N_BARR; i++) MPI_Barrier(c_info->File_comm);
329
330 if( !time_inner ) *time = MPI_Wtime();
331
332 for( i=0; i<i_sample; i++ )
333 {
334
335
336 if( time_inner ) *time = MPI_Wtime();
337
338 if( pos == indv_block )
339 for ( j=0; j<j_sample; j++ )
340 {
341
342 ierr=GEN_File_write(c_info->fh,c_info->s_buffer,Locsize,c_info->etype,&stat);
343 MPI_ERRHAND(ierr);
344
345 DIAGNOSTICS("Write standard ",c_info,c_info->s_buffer,Locsize,Totalsize,i+j,pos);
346
347 }
348
349 else if ( pos == explicit )
350 for ( j=0; j<j_sample; j++ )
351 {
352
353 Offset = c_info->split.Offset+(MPI_Offset)((i+j)*Totalsize);
354
355 ierr=GEN_File_write_at
356 (c_info->fh, Offset, c_info->s_buffer,Locsize,c_info->etype,&stat);
357 MPI_ERRHAND(ierr);
358
359 DIAGNOSTICS("Write explicit ",c_info,c_info->s_buffer,Locsize,Totalsize,i+j,pos);
360
361 }
362
363 else if ( pos == shared )
364 for ( j=0; j<j_sample; j++ )
365 {
366
367 ierr=GEN_File_write_shared
368 (c_info->fh,c_info->s_buffer,Locsize,c_info->etype,&stat);
369 MPI_ERRHAND(ierr);
370
371 DIAGNOSTICS("Write shared ",c_info,c_info->s_buffer,Locsize,Totalsize,i+j,pos);
372
373 }
374
375 MPI_File_sync(c_info->fh);
376
377 if( time_inner ) *time = (MPI_Wtime() - *time)/(i_sample*j_sample);
378
379 CHK_DIFF("Write_xxx",c_info, c_info->r_buffer, 0,
380 Locsize, Totalsize, asize,
381 put, pos, j_sample, time_inner ? -1 : i,
382 chk_mode, &defect);
383 CHK_STOP;
384
385 }
386
387 if( !time_inner ) *time = (MPI_Wtime() - *time)/(i_sample*j_sample);
388
389 } /* end if (File_comm ) */
390 }
391
392
393
IMB_iwrite_ij(struct comm_info * c_info,int size,POSITIONING pos,BTYPES type,int i_sample,int j_sample,int time_inner,int do_ovrlp,double * time)394 void IMB_iwrite_ij(struct comm_info* c_info, int size, POSITIONING pos,
395 BTYPES type, int i_sample, int j_sample,
396 int time_inner, int do_ovrlp, double* time)
397 /*
398
399
400 Calls the proper write functions, non blocking case
401
402 (See IMB_read_ij for documentation of calling sequence)
403
404
405
406 */
407 {
408 int i, j;
409 int Locsize,Totalsize,Ioffs;
410 MPI_Offset Offset;
411
412 MPI_Status* STAT, stat;
413 MPI_Request* REQUESTS;
414
415 ierr = 0;
416
417 *time=0;
418
419 if( c_info->File_rank >= 0 )
420 {
421 #ifdef CHECK
422 int chk_mode;
423
424 if( pos == shared && type != Collective) chk_mode = -3;
425 else chk_mode = -2;
426
427 defect = 0.;
428 #endif
429
430
431 Locsize = c_info->split.Locsize;
432 Totalsize = c_info->split.Totalsize;
433 Offset = (MPI_Offset)c_info->split.Offset;
434
435
436 if(type == Collective )
437
438 /*
439 !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
440 NON BLOCKING COLLECTIVE OUTPUT CASES
441 !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
442 */
443
444 {
445 for(i=0; i<N_BARR; i++) MPI_Barrier(c_info->File_comm);
446
447 *time = MPI_Wtime();
448
449 if( pos == indv_block )
450
451 for ( j=0; j<i_sample*j_sample; j++ )
452 {
453
454 ierr=MPI_File_write_all_begin
455 (c_info->fh,c_info->s_buffer,Locsize,c_info->etype);
456 MPI_ERRHAND(ierr);
457 DIAGNOSTICS("IWrite coll. ",c_info,c_info->s_buffer,Locsize,Totalsize,j,pos);
458
459 if( do_ovrlp )
460 IMB_cpu_exploit(TARGET_CPU_SECS,0);
461
462 ierr=MPI_File_write_all_end
463 (c_info->fh,c_info->s_buffer,&stat);
464 MPI_ERRHAND(ierr);
465
466 }
467
468 else if ( pos == explicit )
469
470 for ( j=0; j<i_sample*j_sample; j++ )
471 {
472
473 Offset = c_info->split.Offset+(MPI_Offset)(j*Totalsize);
474
475 ierr=MPI_File_write_at_all_begin
476 (c_info->fh,Offset,c_info->s_buffer,Locsize,c_info->etype);
477 MPI_ERRHAND(ierr);
478
479 DIAGNOSTICS("IWrite expl coll. ",c_info,c_info->s_buffer,Locsize,Totalsize,j,pos);
480
481 if( do_ovrlp )
482 IMB_cpu_exploit(TARGET_CPU_SECS,0);
483
484 ierr=MPI_File_write_at_all_end
485 (c_info->fh,c_info->s_buffer,&stat);
486 MPI_ERRHAND(ierr);
487
488
489 }
490
491 else if ( pos == shared )
492
493 for ( j=0; j<i_sample*j_sample; j++ )
494 {
495
496 ierr=MPI_File_write_ordered_begin
497 (c_info->fh,c_info->s_buffer,Locsize,c_info->etype);
498 MPI_ERRHAND(ierr);
499
500 DIAGNOSTICS("IWrite shared coll. ",c_info,c_info->s_buffer,Locsize,Totalsize,j,pos);
501
502 if( do_ovrlp )
503 IMB_cpu_exploit(TARGET_CPU_SECS,0);
504
505 ierr=MPI_File_write_ordered_end
506 (c_info->fh,c_info->s_buffer,&stat);
507 MPI_ERRHAND(ierr);
508
509 }
510
511 MPI_File_sync(c_info->fh);
512
513
514 *time = (MPI_Wtime() - *time)/(i_sample*j_sample);
515
516 CHK_DIFF("Coll. IWrite_xxx",c_info, c_info->r_buffer, 0,
517 Locsize, Totalsize, asize,
518 put, pos, i_sample*j_sample, -1,
519 chk_mode, &defect);
520 CHK_STOP;
521
522 }
523
524 else /* type non Collective */
525
526 /*
527 !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
528 NON BLOCKING NON COLLECTIVE OUTPUT CASES
529 !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
530 */
531
532 {
533
534 REQUESTS = (MPI_Request*)IMB_v_alloc(j_sample*sizeof(MPI_Request), "IWrite_ij");
535 STAT = (MPI_Status *)IMB_v_alloc(j_sample*sizeof(MPI_Status ), "IWrite_ij");
536
537 for( j=0; j<j_sample; j++ ) REQUESTS[j]=MPI_REQUEST_NULL;
538
539 for(i=0; i<N_BARR; i++) MPI_Barrier(c_info->File_comm);
540
541 if( !time_inner ) *time = MPI_Wtime();
542
543 for( i=0; i<i_sample; i++ )
544 {
545
546
547 if( time_inner ) *time = MPI_Wtime();
548
549 if( pos == indv_block )
550 for ( j=0; j<j_sample; j++ )
551 {
552
553 ierr=MPI_File_iwrite(c_info->fh,c_info->s_buffer,Locsize,c_info->etype,&REQUESTS[j]);
554 MPI_ERRHAND(ierr);
555 DIAGNOSTICS("IWrite standard ",c_info,c_info->s_buffer,Locsize,Totalsize,i+j,pos);
556
557 }
558
559 else if ( pos == explicit )
560 for ( j=0; j<j_sample; j++ )
561 {
562
563 Offset = c_info->split.Offset+(MPI_Offset)((i+j)*Totalsize);
564
565 ierr=MPI_File_iwrite_at
566 (c_info->fh,Offset,c_info->s_buffer,Locsize,c_info->etype,&REQUESTS[j]);
567 MPI_ERRHAND(ierr);
568
569 DIAGNOSTICS("IWrite expl ",c_info,c_info->s_buffer,Locsize,Totalsize,i+j,pos);
570
571
572 }
573
574 else if ( pos == shared )
575 for ( j=0; j<j_sample; j++ )
576 {
577
578 ierr=MPI_File_iwrite_shared
579 (c_info->fh,c_info->s_buffer,Locsize,c_info->etype,&REQUESTS[j]);
580 MPI_ERRHAND(ierr);
581
582 DIAGNOSTICS("IWrite shared ",c_info,c_info->s_buffer,Locsize,Totalsize,i+j,pos);
583
584 }
585
586
587 if( do_ovrlp )
588 for ( j=0; j<j_sample; j++ )
589 IMB_cpu_exploit(TARGET_CPU_SECS,0);
590
591 if( j_sample == 1 )
592 MPI_Wait(REQUESTS,STAT);
593 else
594 MPI_Waitall(j_sample,REQUESTS,STAT);
595
596
597 MPI_File_sync(c_info->fh);
598
599
600 if( time_inner ) *time = (MPI_Wtime() - *time)/(i_sample*j_sample);
601
602 CHK_DIFF("IWrite_xxx",c_info, c_info->r_buffer, 0,
603 Locsize, Totalsize, asize,
604 put, pos, j_sample, time_inner ? -1 : i,
605 chk_mode, &defect);
606 CHK_STOP;
607
608 }
609 if( !time_inner ) *time = (MPI_Wtime() - *time)/(i_sample*j_sample);
610
611 free (REQUESTS);
612 free (STAT);
613
614 }
615
616
617 } /* end if (File_comm ) */
618 }
619