1 /*
2 * (C) Copyright 2005- ECMWF.
3 *
4 * This software is licensed under the terms of the Apache Licence Version 2.0
5 * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0.
6 *
7 * In applying this licence, ECMWF does not waive the privileges and immunities granted to it by
8 * virtue of its status as an intergovernmental organisation nor does it submit to any jurisdiction.
9 */
10
11 /*
12 *
13 * Description: file pool
14 *
15 */
16
17 #include "grib_api_internal.h"
18 #define GRIB_MAX_OPENED_FILES 200
19
20 #if GRIB_PTHREADS
21 static pthread_once_t once = PTHREAD_ONCE_INIT;
22 static pthread_mutex_t mutex1 = PTHREAD_MUTEX_INITIALIZER;
23
init()24 static void init()
25 {
26 pthread_mutexattr_t attr;
27 pthread_mutexattr_init(&attr);
28 pthread_mutexattr_settype(&attr, PTHREAD_MUTEX_RECURSIVE);
29 pthread_mutex_init(&mutex1, &attr);
30 pthread_mutexattr_destroy(&attr);
31 }
32 #elif GRIB_OMP_THREADS
33 static int once = 0;
34 static omp_nest_lock_t mutex1;
35
init()36 static void init()
37 {
38 GRIB_OMP_CRITICAL(lock_grib_filepool_c)
39 {
40 if (once == 0) {
41 omp_init_nest_lock(&mutex1);
42 once = 1;
43 }
44 }
45 }
46 #endif
47
48 static short next_id = 0;
49
50 /* Note: A fast cut-down version of strcmp which does NOT return -1 */
51 /* 0 means input strings are equal and 1 means not equal */
grib_inline_strcmp(const char * a,const char * b)52 GRIB_INLINE static int grib_inline_strcmp(const char* a, const char* b)
53 {
54 if (*a != *b)
55 return 1;
56 while ((*a != 0 && *b != 0) && *(a) == *(b)) {
57 a++;
58 b++;
59 }
60 return (*a == 0 && *b == 0) ? 0 : 1;
61 }
62
63 static grib_file_pool file_pool = {
64 0, /* grib_context* context;*/
65 0, /* grib_file* first;*/
66 0, /* grib_file* current; */
67 0, /* size_t size;*/
68 0, /* int number_of_opened_files;*/
69 GRIB_MAX_OPENED_FILES /* int max_opened_files; */
70 };
71
grib_file_pool_clean()72 void grib_file_pool_clean()
73 {
74 grib_file *file, *next;
75
76 if (!file_pool.first)
77 return;
78
79 file = file_pool.first;
80 while (file) {
81 next = file->next;
82 grib_file_delete(file);
83 file = next;
84 }
85 }
86
grib_file_pool_change_id()87 static void grib_file_pool_change_id()
88 {
89 grib_file* file;
90
91 if (!file_pool.first)
92 return;
93
94 file = file_pool.first;
95 while (file) {
96 file->id += 1000;
97 file = file->next;
98 }
99 }
100
grib_read_file(grib_context * c,FILE * fh,int * err)101 static grib_file* grib_read_file(grib_context* c, FILE* fh, int* err)
102 {
103 short marker = 0;
104 short id = 0;
105 grib_file* file;
106 *err = grib_read_short(fh, &marker);
107 if (!marker)
108 return NULL;
109
110 file = (grib_file*)grib_context_malloc_clear(c, sizeof(grib_file));
111 file->buffer = 0;
112 file->name = grib_read_string(c, fh, err);
113 if (*err)
114 return NULL;
115
116 *err = grib_read_short(fh, &id);
117 file->id = id;
118 if (*err)
119 return NULL;
120
121 file->next = grib_read_file(c, fh, err);
122 if (*err)
123 return NULL;
124
125 return file;
126 }
127
grib_write_file(FILE * fh,grib_file * file)128 static int grib_write_file(FILE* fh, grib_file* file)
129 {
130 int err = 0;
131
132 if (!file)
133 return grib_write_null_marker(fh);
134
135 err = grib_write_not_null_marker(fh);
136 if (err)
137 return err;
138
139 err = grib_write_string(fh, file->name);
140 if (err)
141 return err;
142
143 err = grib_write_short(fh, (short)file->id);
144 if (err)
145 return err;
146
147 return grib_write_file(fh, file->next);
148 }
149
grib_file_pool_get_files()150 grib_file* grib_file_pool_get_files()
151 {
152 return file_pool.first;
153 }
154
grib_file_pool_read(grib_context * c,FILE * fh)155 int grib_file_pool_read(grib_context* c, FILE* fh)
156 {
157 int err = 0;
158 short marker = 0;
159 grib_file* file;
160
161 if (!c)
162 c = grib_context_get_default();
163
164 err = grib_read_short(fh, &marker);
165 if (!marker) {
166 grib_context_log(c, GRIB_LOG_ERROR,
167 "Unable to find file information in index file\n");
168 return GRIB_INVALID_FILE;
169 }
170
171 grib_file_pool_change_id();
172 file = file_pool.first;
173
174 while (file->next)
175 file = file->next;
176
177 file->next = grib_read_file(c, fh, &err);
178 if (err)
179 return err;
180
181 return GRIB_SUCCESS;
182 }
183
grib_file_pool_write(FILE * fh)184 int grib_file_pool_write(FILE* fh)
185 {
186 int err = 0;
187 if (!file_pool.first)
188 return grib_write_null_marker(fh);
189
190 err = grib_write_not_null_marker(fh);
191 if (err)
192 return err;
193
194 return grib_write_file(fh, file_pool.first);
195 }
196
grib_file_open(const char * filename,const char * mode,int * err)197 grib_file* grib_file_open(const char* filename, const char* mode, int* err)
198 {
199 grib_file *file = 0, *prev = 0;
200 int same_mode = 0;
201 int is_new = 0;
202 GRIB_MUTEX_INIT_ONCE(&once, &init);
203
204 if (!file_pool.context)
205 file_pool.context = grib_context_get_default();
206
207 if (file_pool.current && !grib_inline_strcmp(filename, file_pool.current->name)) {
208 file = file_pool.current;
209 }
210 else {
211 GRIB_MUTEX_LOCK(&mutex1);
212 file = file_pool.first;
213 while (file) {
214 if (!grib_inline_strcmp(filename, file->name))
215 break;
216 prev = file;
217 file = file->next;
218 }
219 if (!file) {
220 is_new = 1;
221 file = grib_file_new(file_pool.context, filename, err);
222 if (prev)
223 prev->next = file;
224 file_pool.current = file;
225 if (!prev)
226 file_pool.first = file;
227 file_pool.size++;
228 }
229 GRIB_MUTEX_UNLOCK(&mutex1);
230 }
231
232 if (file->mode)
233 same_mode = grib_inline_strcmp(mode, file->mode) ? 0 : 1;
234 if (file->handle && same_mode) {
235 *err = 0;
236 return file;
237 }
238
239 GRIB_MUTEX_LOCK(&mutex1);
240 if (!same_mode && file->handle) {
241 fclose(file->handle);
242 }
243
244 if (!file->handle) {
245 if (!is_new && *mode == 'w') {
246 file->handle = fopen(file->name, "a");
247 }
248 else {
249 file->handle = fopen(file->name, mode);
250 }
251
252 if (!file->handle) {
253 grib_context_log(file->context, GRIB_LOG_PERROR, "grib_file_open: cannot open file %s", file->name);
254 *err = GRIB_IO_PROBLEM;
255 GRIB_MUTEX_UNLOCK(&mutex1);
256 return NULL;
257 }
258 if (file->mode) free(file->mode);
259 file->mode = strdup(mode);
260 if (file_pool.context->io_buffer_size) {
261 #ifdef POSIX_MEMALIGN
262 if (posix_memalign((void**)&(file->buffer), sysconf(_SC_PAGESIZE), file_pool.context->io_buffer_size)) {
263 grib_context_log(file->context, GRIB_LOG_FATAL, "posix_memalign unable to allocate io_buffer\n");
264 }
265 #else
266 file->buffer = (void*)malloc(file_pool.context->io_buffer_size);
267 if (!file->buffer) {
268 grib_context_log(file->context, GRIB_LOG_FATAL, "Unable to allocate io_buffer\n");
269 }
270 #endif
271 setvbuf(file->handle, file->buffer, _IOFBF, file_pool.context->io_buffer_size);
272 }
273
274 file_pool.number_of_opened_files++;
275 }
276
277 GRIB_MUTEX_UNLOCK(&mutex1);
278 return file;
279 }
280
grib_file_pool_delete_file(grib_file * file)281 void grib_file_pool_delete_file(grib_file* file)
282 {
283 grib_file* prev = NULL;
284 GRIB_MUTEX_INIT_ONCE(&once, &init);
285 GRIB_MUTEX_LOCK(&mutex1);
286
287 if (file == file_pool.first) {
288 file_pool.first = file->next;
289 file_pool.current = file->next;
290 }
291 else {
292 prev = file_pool.first;
293 file_pool.current = file_pool.first;
294 while (prev) {
295 if (prev->next == file)
296 break;
297 prev = prev->next;
298 }
299 DebugAssert(prev);
300 if (prev) {
301 prev->next = file->next;
302 }
303 }
304
305 if (file->handle) {
306 file_pool.number_of_opened_files--;
307 }
308 grib_file_delete(file);
309 GRIB_MUTEX_UNLOCK(&mutex1);
310 }
311
grib_file_close(const char * filename,int force,int * err)312 void grib_file_close(const char* filename, int force, int* err)
313 {
314 grib_file* file = NULL;
315 grib_context* context = grib_context_get_default();
316
317 /* Performance: keep the files open to avoid opening and closing files when writing the output. */
318 /* So only call fclose() when too many files are open. */
319 /* Also see ECC-411 */
320 int do_close = (file_pool.number_of_opened_files > context->file_pool_max_opened_files);
321 if (force == 1)
322 do_close = 1; /* Can be overridden with the force argument */
323
324 if (do_close) {
325 /*printf("+++++++++++++ closing file %s (n=%d)\n",filename, file_pool.number_of_opened_files);*/
326 GRIB_MUTEX_INIT_ONCE(&once, &init);
327 GRIB_MUTEX_LOCK(&mutex1);
328 file = grib_get_file(filename, err);
329 if (file->handle) {
330 if (fclose(file->handle) != 0) {
331 *err = GRIB_IO_PROBLEM;
332 }
333 if (file->buffer) {
334 free(file->buffer);
335 file->buffer = 0;
336 }
337 file->handle = NULL;
338 file_pool.number_of_opened_files--;
339 }
340 GRIB_MUTEX_UNLOCK(&mutex1);
341 }
342 }
343
grib_file_close_all(int * err)344 void grib_file_close_all(int* err)
345 {
346 grib_file* file = NULL;
347 if (!file_pool.first)
348 return;
349
350 GRIB_MUTEX_INIT_ONCE(&once, &init);
351 GRIB_MUTEX_LOCK(&mutex1);
352
353 file = file_pool.first;
354 while (file) {
355 if (file->handle) {
356 if (fclose(file->handle) != 0) {
357 *err = GRIB_IO_PROBLEM;
358 }
359 file->handle = NULL;
360 }
361 file = file->next;
362 }
363
364 GRIB_MUTEX_UNLOCK(&mutex1);
365 }
366
grib_get_file(const char * filename,int * err)367 grib_file* grib_get_file(const char* filename, int* err)
368 {
369 grib_file* file = NULL;
370
371 if (file_pool.current->name && !grib_inline_strcmp(filename, file_pool.current->name)) {
372 return file_pool.current;
373 }
374
375 file = file_pool.first;
376 while (file) {
377 if (!grib_inline_strcmp(filename, file->name))
378 break;
379 file = file->next;
380 }
381 if (!file)
382 file = grib_file_new(0, filename, err);
383
384 return file;
385 }
386
grib_find_file(short id)387 grib_file* grib_find_file(short id)
388 {
389 grib_file* file = NULL;
390
391 if (file_pool.current->name && id == file_pool.current->id) {
392 return file_pool.current;
393 }
394
395 file = file_pool.first;
396 while (file) {
397 if (id == file->id)
398 break;
399 file = file->next;
400 }
401
402 return file;
403 }
404
grib_file_new(grib_context * c,const char * name,int * err)405 grib_file* grib_file_new(grib_context* c, const char* name, int* err)
406 {
407 grib_file* file;
408
409 if (!c)
410 c = grib_context_get_default();
411
412 file = (grib_file*)grib_context_malloc_clear(c, sizeof(grib_file));
413
414 if (!file) {
415 grib_context_log(c, GRIB_LOG_ERROR, "grib_file_new: unable to allocate memory");
416 *err = GRIB_OUT_OF_MEMORY;
417 return NULL;
418 }
419 GRIB_MUTEX_INIT_ONCE(&once, &init);
420
421 file->name = strdup(name);
422 file->id = next_id;
423
424 GRIB_MUTEX_LOCK(&mutex1);
425 next_id++;
426 GRIB_MUTEX_UNLOCK(&mutex1);
427
428 file->mode = 0;
429 file->handle = 0;
430 file->refcount = 0;
431 file->context = c;
432 file->next = 0;
433 file->buffer = 0;
434 return file;
435 }
436
grib_file_delete(grib_file * file)437 void grib_file_delete(grib_file* file)
438 {
439 {
440 if (!file)
441 return;
442 }
443 GRIB_MUTEX_INIT_ONCE(&once, &init);
444 GRIB_MUTEX_LOCK(&mutex1);
445 /* GRIB-803: cannot call fclose yet! Causes crash */
446 /* TODO: Set handle to NULL in filepool too */
447 #if 0
448 if (file->handle) {
449 if (fclose(file->handle) != 0) {
450 perror(file->name);
451 }
452 }
453 #endif
454 if (file->name)
455 free(file->name);
456 if (file->mode)
457 free(file->mode);
458
459 if (file->buffer) {
460 free(file->buffer);
461 }
462 grib_context_free(file->context, file);
463 file = NULL;
464 GRIB_MUTEX_UNLOCK(&mutex1);
465 }
466