1 /* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil ; -*- */
2 /*
3  *
4  *   Copyright (C) 1997 University of Chicago.
5  *   See COPYRIGHT notice in top-level directory.
6  */
7 
8 #include "ad_nfs.h"
9 #include "adio_extern.h"
10 
ADIOI_NFS_ReadContig(ADIO_File fd,void * buf,int count,MPI_Datatype datatype,int file_ptr_type,ADIO_Offset offset,ADIO_Status * status,int * error_code)11 void ADIOI_NFS_ReadContig(ADIO_File fd, void *buf, int count,
12                      MPI_Datatype datatype, int file_ptr_type,
13 		     ADIO_Offset offset, ADIO_Status *status, int *error_code)
14 {
15     int err=-1;
16     MPI_Count datatype_size, len;
17     static char myname[] = "ADIOI_NFS_READCONTIG";
18 
19     MPI_Type_size_x(datatype, &datatype_size);
20     len = datatype_size * count;
21 
22     if (file_ptr_type == ADIO_EXPLICIT_OFFSET) {
23 	if (fd->fp_sys_posn != offset) {
24 #ifdef ADIOI_MPE_LOGGING
25             MPE_Log_event( ADIOI_MPE_lseek_a, 0, NULL );
26 #endif
27 	    lseek(fd->fd_sys, offset, SEEK_SET);
28 #ifdef ADIOI_MPE_LOGGING
29             MPE_Log_event( ADIOI_MPE_lseek_b, 0, NULL );
30 #endif
31         }
32 	if (fd->atomicity)
33 	    ADIOI_WRITE_LOCK(fd, offset, SEEK_SET, len);
34 	else ADIOI_READ_LOCK(fd, offset, SEEK_SET, len);
35 #ifdef ADIOI_MPE_LOGGING
36         MPE_Log_event( ADIOI_MPE_read_a, 0, NULL );
37 #endif
38 	err = read(fd->fd_sys, buf, len);
39 #ifdef ADIOI_MPE_LOGGING
40         MPE_Log_event( ADIOI_MPE_read_b, 0, NULL );
41 #endif
42 	ADIOI_UNLOCK(fd, offset, SEEK_SET, len);
43 	fd->fp_sys_posn = offset + err;
44 	/* individual file pointer not updated */
45     }
46     else {  /* read from curr. location of ind. file pointer */
47 	offset = fd->fp_ind;
48 	if (fd->fp_sys_posn != fd->fp_ind) {
49 #ifdef ADIOI_MPE_LOGGING
50             MPE_Log_event( ADIOI_MPE_lseek_a, 0, NULL );
51 #endif
52 	    lseek(fd->fd_sys, fd->fp_ind, SEEK_SET);
53 #ifdef ADIOI_MPE_LOGGING
54             MPE_Log_event( ADIOI_MPE_lseek_b, 0, NULL );
55 #endif
56         }
57 	if (fd->atomicity)
58 	    ADIOI_WRITE_LOCK(fd, offset, SEEK_SET, len);
59 	else ADIOI_READ_LOCK(fd, offset, SEEK_SET, len);
60 #ifdef ADIOI_MPE_LOGGING
61         MPE_Log_event( ADIOI_MPE_read_a, 0, NULL );
62 #endif
63 	err = read(fd->fd_sys, buf, len);
64 #ifdef ADIOI_MPE_LOGGING
65         MPE_Log_event( ADIOI_MPE_read_b, 0, NULL );
66 #endif
67 	ADIOI_UNLOCK(fd, offset, SEEK_SET, len);
68 	fd->fp_ind += err;
69 	fd->fp_sys_posn = fd->fp_ind;
70     }
71 
72     /* --BEGIN ERROR HANDLING-- */
73     if (err == -1) {
74 	*error_code = MPIO_Err_create_code(MPI_SUCCESS, MPIR_ERR_RECOVERABLE,
75 					   myname, __LINE__, MPI_ERR_IO,
76 					   "**io", "**io %s", strerror(errno));
77 	return;
78     }
79     /* --END ERROR HANDLING-- */
80 
81 #ifdef HAVE_STATUS_SET_BYTES
82     MPIR_Status_set_bytes(status, datatype, err);
83 #endif
84 
85     *error_code = MPI_SUCCESS;
86 }
87 
88 
89 
90 #ifdef ADIOI_MPE_LOGGING
91 #define ADIOI_BUFFERED_READ \
92 { \
93     if (req_off >= readbuf_off + readbuf_len) { \
94 	readbuf_off = req_off; \
95 	readbuf_len = (int) (ADIOI_MIN(max_bufsize, end_offset-readbuf_off+1));\
96         MPE_Log_event( ADIOI_MPE_lseek_a, 0, NULL ); \
97 	lseek(fd->fd_sys, readbuf_off, SEEK_SET);\
98         MPE_Log_event( ADIOI_MPE_lseek_b, 0, NULL ); \
99         if (!(fd->atomicity)) ADIOI_READ_LOCK(fd, readbuf_off, SEEK_SET, readbuf_len);\
100         MPE_Log_event( ADIOI_MPE_read_a, 0, NULL ); \
101         err = read(fd->fd_sys, readbuf, readbuf_len);\
102         MPE_Log_event( ADIOI_MPE_read_b, 0, NULL ); \
103         if (!(fd->atomicity)) ADIOI_UNLOCK(fd, readbuf_off, SEEK_SET, readbuf_len);\
104         if (err == -1) err_flag = 1; \
105     } \
106     while (req_len > readbuf_off + readbuf_len - req_off) { \
107 	partial_read = (int) (readbuf_off + readbuf_len - req_off); \
108 	tmp_buf = (char *) ADIOI_Malloc(partial_read); \
109 	memcpy(tmp_buf, readbuf+readbuf_len-partial_read, partial_read); \
110 	ADIOI_Free(readbuf); \
111 	readbuf = (char *) ADIOI_Malloc(partial_read + max_bufsize); \
112 	memcpy(readbuf, tmp_buf, partial_read); \
113 	ADIOI_Free(tmp_buf); \
114 	readbuf_off += readbuf_len-partial_read; \
115 	readbuf_len = (int) (partial_read + ADIOI_MIN(max_bufsize, \
116 				       end_offset-readbuf_off+1)); \
117         MPE_Log_event( ADIOI_MPE_lseek_a, 0, NULL ); \
118 	lseek(fd->fd_sys, readbuf_off+partial_read, SEEK_SET);\
119         MPE_Log_event( ADIOI_MPE_lseek_b, 0, NULL ); \
120         if (!(fd->atomicity)) ADIOI_READ_LOCK(fd, readbuf_off+partial_read, SEEK_SET, readbuf_len-partial_read);\
121         MPE_Log_event( ADIOI_MPE_read_a, 0, NULL ); \
122         err = read(fd->fd_sys, readbuf+partial_read, readbuf_len-partial_read);\
123         MPE_Log_event( ADIOI_MPE_read_b, 0, NULL ); \
124         if (!(fd->atomicity)) ADIOI_UNLOCK(fd, readbuf_off+partial_read, SEEK_SET, readbuf_len-partial_read);\
125         if (err == -1) err_flag = 1; \
126     } \
127     memcpy((char *)buf + userbuf_off, readbuf+req_off-readbuf_off, req_len); \
128 }
129 #else
130 #define ADIOI_BUFFERED_READ \
131 { \
132     if (req_off >= readbuf_off + readbuf_len) { \
133 	readbuf_off = req_off; \
134 	readbuf_len = (int) (ADIOI_MIN(max_bufsize, end_offset-readbuf_off+1));\
135 	lseek(fd->fd_sys, readbuf_off, SEEK_SET);\
136         if (!(fd->atomicity)) ADIOI_READ_LOCK(fd, readbuf_off, SEEK_SET, readbuf_len);\
137         err = read(fd->fd_sys, readbuf, readbuf_len);\
138         if (!(fd->atomicity)) ADIOI_UNLOCK(fd, readbuf_off, SEEK_SET, readbuf_len);\
139         if (err == -1) err_flag = 1; \
140     } \
141     while (req_len > readbuf_off + readbuf_len - req_off) { \
142 	partial_read = (int) (readbuf_off + readbuf_len - req_off); \
143 	tmp_buf = (char *) ADIOI_Malloc(partial_read); \
144 	memcpy(tmp_buf, readbuf+readbuf_len-partial_read, partial_read); \
145 	ADIOI_Free(readbuf); \
146 	readbuf = (char *) ADIOI_Malloc(partial_read + max_bufsize); \
147 	memcpy(readbuf, tmp_buf, partial_read); \
148 	ADIOI_Free(tmp_buf); \
149 	readbuf_off += readbuf_len-partial_read; \
150 	readbuf_len = (int) (partial_read + ADIOI_MIN(max_bufsize, \
151 				       end_offset-readbuf_off+1)); \
152 	lseek(fd->fd_sys, readbuf_off+partial_read, SEEK_SET);\
153         if (!(fd->atomicity)) ADIOI_READ_LOCK(fd, readbuf_off+partial_read, SEEK_SET, readbuf_len-partial_read);\
154         err = read(fd->fd_sys, readbuf+partial_read, readbuf_len-partial_read);\
155         if (!(fd->atomicity)) ADIOI_UNLOCK(fd, readbuf_off+partial_read, SEEK_SET, readbuf_len-partial_read);\
156         if (err == -1) err_flag = 1; \
157     } \
158     memcpy((char *)buf + userbuf_off, readbuf+req_off-readbuf_off, req_len); \
159 }
160 #endif
161 
162 
ADIOI_NFS_ReadStrided(ADIO_File fd,void * buf,int count,MPI_Datatype datatype,int file_ptr_type,ADIO_Offset offset,ADIO_Status * status,int * error_code)163 void ADIOI_NFS_ReadStrided(ADIO_File fd, void *buf, int count,
164                        MPI_Datatype datatype, int file_ptr_type,
165                        ADIO_Offset offset, ADIO_Status *status, int
166                        *error_code)
167 {
168 /* offset is in units of etype relative to the filetype. */
169 
170     ADIOI_Flatlist_node *flat_buf, *flat_file;
171     int i, j, k, err=-1, brd_size, st_index=0;
172     int num, size, sum, n_etypes_in_filetype, size_in_filetype;
173     MPI_Count bufsize;
174     int n_filetypes, etype_in_filetype;
175     ADIO_Offset abs_off_in_filetype=0;
176     int req_len, partial_read;
177     MPI_Count filetype_size, etype_size, buftype_size;
178     MPI_Aint filetype_extent, buftype_extent, lb;
179     int buf_count, buftype_is_contig, filetype_is_contig;
180     ADIO_Offset userbuf_off;
181     ADIO_Offset off, req_off, disp, end_offset=0, readbuf_off, start_off;
182     char *readbuf, *tmp_buf, *value;
183     int st_n_filetypes, readbuf_len;
184     ADIO_Offset frd_size=0, new_frd_size, st_frd_size;
185     int new_brd_size, err_flag=0, info_flag, max_bufsize;
186 
187     static char myname[] = "ADIOI_NFS_READSTRIDED";
188 
189     ADIOI_Datatype_iscontig(datatype, &buftype_is_contig);
190     ADIOI_Datatype_iscontig(fd->filetype, &filetype_is_contig);
191 
192     MPI_Type_size_x(fd->filetype, &filetype_size);
193     if ( ! filetype_size ) {
194 #ifdef HAVE_STATUS_SET_BYTES
195 	MPIR_Status_set_bytes(status, datatype, 0);
196 #endif
197 	*error_code = MPI_SUCCESS;
198 	return;
199     }
200 
201     MPI_Type_get_extent(fd->filetype, &lb, &filetype_extent);
202     MPI_Type_size_x(datatype, &buftype_size);
203     MPI_Type_get_extent(datatype, &lb, &buftype_extent);
204     etype_size = fd->etype_size;
205 
206     bufsize = buftype_size * count;
207 
208 /* get max_bufsize from the info object. */
209 
210     value = (char *) ADIOI_Malloc((MPI_MAX_INFO_VAL+1)*sizeof(char));
211     ADIOI_Info_get(fd->info, "ind_rd_buffer_size", MPI_MAX_INFO_VAL, value,
212                  &info_flag);
213     max_bufsize = atoi(value);
214     ADIOI_Free(value);
215 
216     if (!buftype_is_contig && filetype_is_contig) {
217 
218 /* noncontiguous in memory, contiguous in file. */
219 
220 	flat_buf = ADIOI_Flatten_and_find(datatype);
221 
222         off = (file_ptr_type == ADIO_INDIVIDUAL) ? fd->fp_ind :
223                  fd->disp + etype_size * offset;
224 
225 	start_off = off;
226 	end_offset = off + bufsize - 1;
227         readbuf_off = off;
228         readbuf = (char *) ADIOI_Malloc(max_bufsize);
229         readbuf_len = (int) (ADIOI_MIN(max_bufsize, end_offset-readbuf_off+1));
230 
231 /* if atomicity is true, lock (exclusive) the region to be accessed */
232         if (fd->atomicity)
233             ADIOI_WRITE_LOCK(fd, start_off, SEEK_SET, end_offset-start_off+1);
234 
235 #ifdef ADIOI_MPE_LOGGING
236         MPE_Log_event( ADIOI_MPE_lseek_a, 0, NULL );
237 #endif
238 	lseek(fd->fd_sys, readbuf_off, SEEK_SET);
239 #ifdef ADIOI_MPE_LOGGING
240         MPE_Log_event( ADIOI_MPE_lseek_b, 0, NULL );
241 #endif
242         if (!(fd->atomicity)) ADIOI_READ_LOCK(fd, readbuf_off, SEEK_SET, readbuf_len);
243 #ifdef ADIOI_MPE_LOGGING
244         MPE_Log_event( ADIOI_MPE_read_a, 0, NULL );
245 #endif
246         err = read(fd->fd_sys, readbuf, readbuf_len);
247 #ifdef ADIOI_MPE_LOGGING
248         MPE_Log_event( ADIOI_MPE_read_b, 0, NULL );
249 #endif
250         if (!(fd->atomicity)) ADIOI_UNLOCK(fd, readbuf_off, SEEK_SET, readbuf_len);
251         if (err == -1) err_flag = 1;
252 
253         for (j=0; j<count; j++)
254             for (i=0; i<flat_buf->count; i++) {
255                 userbuf_off = j*buftype_extent + flat_buf->indices[i];
256 		req_off = off;
257 		req_len = flat_buf->blocklens[i];
258 		ADIOI_BUFFERED_READ
259                 off += flat_buf->blocklens[i];
260             }
261 
262         if (fd->atomicity)
263             ADIOI_UNLOCK(fd, start_off, SEEK_SET, end_offset-start_off+1);
264 
265         if (file_ptr_type == ADIO_INDIVIDUAL) fd->fp_ind = off;
266 
267 	ADIOI_Free(readbuf); /* malloced in the buffered_read macro */
268 
269 	if (err_flag) {
270 	    *error_code = MPIO_Err_create_code(MPI_SUCCESS,
271 					       MPIR_ERR_RECOVERABLE, myname,
272 					       __LINE__, MPI_ERR_IO, "**io",
273 					       "**io %s", strerror(errno));
274 	}
275 	else *error_code = MPI_SUCCESS;
276     }
277 
278     else {  /* noncontiguous in file */
279 
280 /* filetype already flattened in ADIO_Open */
281 	flat_file = ADIOI_Flatlist;
282 	while (flat_file->type != fd->filetype) flat_file = flat_file->next;
283 	disp = fd->disp;
284 
285 	if (file_ptr_type == ADIO_INDIVIDUAL) {
286           /* Wei-keng reworked type processing to be a bit more efficient */
287            offset       = fd->fp_ind - disp;
288            n_filetypes  = (offset - flat_file->indices[0]) / filetype_extent;
289           offset -= (ADIO_Offset)n_filetypes * filetype_extent;
290           /* now offset is local to this extent */
291 
292            /* find the block where offset is located, skip blocklens[i]==0 */
293            for (i=0; i<flat_file->count; i++) {
294                ADIO_Offset dist;
295                if (flat_file->blocklens[i] == 0) continue;
296                dist = flat_file->indices[i] + flat_file->blocklens[i] - offset;
297                /* frd_size is from offset to the end of block i */
298               if (dist == 0) {
299                   i++;
300                   offset   = flat_file->indices[i];
301                   frd_size = flat_file->blocklens[i];
302                   break;
303               }
304               if (dist > 0 ) {
305                    frd_size = dist;
306 		   break;
307               }
308           }
309            st_index = i;  /* starting index in flat_file->indices[] */
310            offset += disp + (ADIO_Offset)n_filetypes*filetype_extent;
311        }
312 	else {
313 	    n_etypes_in_filetype = filetype_size/etype_size;
314 	    n_filetypes = (int) (offset / n_etypes_in_filetype);
315 	    etype_in_filetype = (int) (offset % n_etypes_in_filetype);
316 	    size_in_filetype = etype_in_filetype * etype_size;
317 
318 	    sum = 0;
319 	    for (i=0; i<flat_file->count; i++) {
320 		sum += flat_file->blocklens[i];
321 		if (sum > size_in_filetype) {
322 		    st_index = i;
323 		    frd_size = sum - size_in_filetype;
324 		    abs_off_in_filetype = flat_file->indices[i] +
325 			size_in_filetype - (sum - flat_file->blocklens[i]);
326 		    break;
327 		}
328 	    }
329 
330 	    /* abs. offset in bytes in the file */
331 	    offset = disp + (ADIO_Offset) n_filetypes*filetype_extent +
332 		    abs_off_in_filetype;
333 	}
334 
335         start_off = offset;
336 
337        /* Wei-keng Liao: read request is within a single flat_file contig
338         * block e.g. with subarray types that actually describe the whole
339         * array */
340        if (buftype_is_contig && bufsize <= frd_size) {
341             ADIO_ReadContig(fd, buf, bufsize, MPI_BYTE, ADIO_EXPLICIT_OFFSET,
342                              offset, status, error_code);
343 
344            if (file_ptr_type == ADIO_INDIVIDUAL) {
345                 /* update MPI-IO file pointer to point to the first byte that
346                 * can be accessed in the fileview. */
347                fd->fp_ind = offset + bufsize;
348                if (bufsize == frd_size) {
349                    do {
350                        st_index++;
351                        if (st_index == flat_file->count) {
352                            st_index = 0;
353                            n_filetypes++;
354                        }
355                     } while (flat_file->blocklens[st_index] == 0);
356                    fd->fp_ind = disp + flat_file->indices[st_index]
357                                + n_filetypes*filetype_extent;
358                }
359            }
360            fd->fp_sys_posn = -1;   /* set it to null. */
361 #ifdef HAVE_STATUS_SET_BYTES
362            MPIR_Status_set_bytes(status, datatype, bufsize);
363 #endif
364             return;
365        }
366 
367        /* Calculate end_offset, the last byte-offset that will be accessed.
368          e.g., if start_offset=0 and 100 bytes to be read, end_offset=99*/
369 
370 	st_frd_size = frd_size;
371 	st_n_filetypes = n_filetypes;
372 	i = 0;
373 	j = st_index;
374 	off = offset;
375 	frd_size = ADIOI_MIN(st_frd_size, bufsize);
376 	while (i < bufsize) {
377 	    i += frd_size;
378 	    end_offset = off + frd_size - 1;
379             j = (j+1) % flat_file->count;
380             n_filetypes += (j == 0) ? 1 : 0;
381             while (flat_file->blocklens[j]==0) {
382                j = (j+1) % flat_file->count;
383                n_filetypes += (j == 0) ? 1 : 0;
384 	    }
385 
386 	    off = disp + flat_file->indices[j] + (ADIO_Offset) n_filetypes*filetype_extent;
387 	    frd_size = ADIOI_MIN(flat_file->blocklens[j], bufsize-i);
388 	}
389 
390 /* if atomicity is true, lock (exclusive) the region to be accessed */
391         if (fd->atomicity)
392             ADIOI_WRITE_LOCK(fd, start_off, SEEK_SET, end_offset-start_off+1);
393 
394         /* initial read into readbuf */
395 	readbuf_off = offset;
396 	readbuf = (char *) ADIOI_Malloc(max_bufsize);
397 	readbuf_len = (int) (ADIOI_MIN(max_bufsize, end_offset-readbuf_off+1));
398 
399 #ifdef ADIOI_MPE_LOGGING
400         MPE_Log_event( ADIOI_MPE_lseek_a, 0, NULL );
401 #endif
402 	lseek(fd->fd_sys, offset, SEEK_SET);
403 #ifdef ADIOI_MPE_LOGGING
404         MPE_Log_event( ADIOI_MPE_lseek_b, 0, NULL );
405 #endif
406         if (!(fd->atomicity)) ADIOI_READ_LOCK(fd, offset, SEEK_SET, readbuf_len);
407 #ifdef ADIOI_MPE_LOGGING
408         MPE_Log_event( ADIOI_MPE_read_a, 0, NULL );
409 #endif
410         err = read(fd->fd_sys, readbuf, readbuf_len);
411 #ifdef ADIOI_MPE_LOGGING
412         MPE_Log_event( ADIOI_MPE_read_b, 0, NULL );
413 #endif
414         if (!(fd->atomicity)) ADIOI_UNLOCK(fd, offset, SEEK_SET, readbuf_len);
415 
416         if (err == -1) err_flag = 1;
417 
418 	if (buftype_is_contig && !filetype_is_contig) {
419 
420 /* contiguous in memory, noncontiguous in file. should be the most
421    common case. */
422 
423 	    i = 0;
424 	    j = st_index;
425 	    off = offset;
426 	    n_filetypes = st_n_filetypes;
427 	    frd_size = ADIOI_MIN(st_frd_size, bufsize);
428 	    while (i < bufsize) {
429                 if (frd_size) {
430                     /* TYPE_UB and TYPE_LB can result in
431                        frd_size = 0. save system call in such cases */
432 		    /* lseek(fd->fd_sys, off, SEEK_SET);
433 		    err = read(fd->fd_sys, ((char *) buf) + i, frd_size);*/
434 
435 		    req_off = off;
436 		    req_len = frd_size;
437 		    userbuf_off = i;
438 		    ADIOI_BUFFERED_READ
439 		}
440 		i += frd_size;
441 
442                 if (off + frd_size < disp + flat_file->indices[j] +
443                    flat_file->blocklens[j] + (ADIO_Offset) n_filetypes*filetype_extent)
444                        off += frd_size;
445                 /* did not reach end of contiguous block in filetype.
446                    no more I/O needed. off is incremented by frd_size. */
447                 else {
448                     j = (j+1) % flat_file->count;
449                     n_filetypes += (j == 0) ? 1 : 0;
450                     while (flat_file->blocklens[j]==0) {
451                         j = (j+1) % flat_file->count;
452                         n_filetypes += (j == 0) ? 1 : 0;
453                     }
454 		    off = disp + flat_file->indices[j] +
455                                         (ADIO_Offset) n_filetypes*filetype_extent;
456 		    frd_size = ADIOI_MIN(flat_file->blocklens[j], bufsize-i);
457 		}
458 	    }
459 	}
460 	else {
461 /* noncontiguous in memory as well as in file */
462 
463             ADIO_Offset i;
464 	    flat_buf = ADIOI_Flatten_and_find(datatype);
465 
466 	    k = num = buf_count = 0;
467 	    i = flat_buf->indices[0];
468 	    j = st_index;
469 	    off = offset;
470 	    n_filetypes = st_n_filetypes;
471 	    frd_size = st_frd_size;
472 	    brd_size = flat_buf->blocklens[0];
473 
474 	    while (num < bufsize) {
475 		size = ADIOI_MIN(frd_size, brd_size);
476 		if (size) {
477 		    /* lseek(fd->fd_sys, off, SEEK_SET);
478 		    err = read(fd->fd_sys, ((char *) buf) + i, size); */
479 
480 		    req_off = off;
481 		    req_len = size;
482 		    userbuf_off = i;
483 		    ADIOI_BUFFERED_READ
484 		}
485 
486 		new_frd_size = frd_size;
487 		new_brd_size = brd_size;
488 
489 		if (size == frd_size) {
490 /* reached end of contiguous block in file */
491                     j = (j+1) % flat_file->count;
492                     n_filetypes += (j == 0) ? 1 : 0;
493                     while (flat_file->blocklens[j]==0) {
494                         j = (j+1) % flat_file->count;
495                         n_filetypes += (j == 0) ? 1 : 0;
496                     }
497 		    off = disp + flat_file->indices[j] +
498                                               (ADIO_Offset) n_filetypes*filetype_extent;
499 
500 		    new_frd_size = flat_file->blocklens[j];
501 		    if (size != brd_size) {
502 			i += size;
503 			new_brd_size -= size;
504 		    }
505 		}
506 
507 		if (size == brd_size) {
508 /* reached end of contiguous block in memory */
509 
510 		    k = (k + 1)%flat_buf->count;
511 		    buf_count++;
512 		    i = buftype_extent*(buf_count/flat_buf->count) +
513 			flat_buf->indices[k];
514 		    new_brd_size = flat_buf->blocklens[k];
515 		    if (size != frd_size) {
516 			off += size;
517 			new_frd_size -= size;
518 		    }
519 		}
520 		num += size;
521 		frd_size = new_frd_size;
522                 brd_size = new_brd_size;
523 	    }
524 	}
525 
526         if (fd->atomicity)
527             ADIOI_UNLOCK(fd, start_off, SEEK_SET, end_offset-start_off+1);
528 
529 	if (file_ptr_type == ADIO_INDIVIDUAL) fd->fp_ind = off;
530 
531 	ADIOI_Free(readbuf); /* malloced in the buffered_read macro */
532 
533 	if (err_flag) {
534 	    *error_code = MPIO_Err_create_code(MPI_SUCCESS,
535 					       MPIR_ERR_RECOVERABLE, myname,
536 					       __LINE__, MPI_ERR_IO, "**io",
537 					       "**io %s", strerror(errno));
538 	}
539 	else *error_code = MPI_SUCCESS;
540     }
541 
542     fd->fp_sys_posn = -1;   /* set it to null. */
543 
544 #ifdef HAVE_STATUS_SET_BYTES
545     MPIR_Status_set_bytes(status, datatype, bufsize);
546 /* This is a temporary way of filling in status. The right way is to
547    keep track of how much data was actually read and placed in buf
548    by ADIOI_BUFFERED_READ. */
549 #endif
550 
551     if (!buftype_is_contig) ADIOI_Delete_flattened(datatype);
552 }
553