1 #include "cdi_fdb.h"
2
3 int cdi_fdb_dummy;
4
5 #ifdef HAVE_LIBFDB5
6
7 #include <string.h>
8 #include <stdlib.h>
9
10 #include "error.h"
11 #include "cdi_int.h"
12
13 void ensureBufferSize(size_t requiredSize, size_t *curSize, void **buffer);
14
decode_fdbitem(const char * fdbItem,KeyValueEntry * keyValue)15 void decode_fdbitem(const char *fdbItem, KeyValueEntry *keyValue)
16 {
17 keyValue->item = strdup(fdbItem);
18 char *pItem = keyValue->item;
19 int numKeys = 0;
20 char **itemKeys = keyValue->keys;
21 char **itemValues = keyValue->values;
22 int len = strlen(pItem);
23 int start = (*pItem == '{');
24 itemKeys[0] = pItem + start;
25 for (int i = start; i < len; i++)
26 {
27 if (pItem[i] == ',')
28 {
29 pItem[i] = 0;
30 numKeys++;
31 itemKeys[numKeys] = &pItem[++i];
32 }
33 else if (pItem[i] == '}')
34 {
35 pItem[i] = 0;
36 numKeys++;
37 if (pItem[i + 1] == '{')
38 {
39 itemKeys[numKeys] = &pItem[i + 2];
40 i += 2;
41 }
42 }
43 else if (i > start && i == (len - 1))
44 {
45 numKeys++;
46 }
47 }
48
49 keyValue->numKeys = numKeys;
50 //for (int i = 0; i < numKeys; i++) printf("%d <%s>\n", i, itemKeys[i]);
51
52 for (int i = 0; i < numKeys; i++)
53 {
54 char *itemKey = itemKeys[i];
55 len = strlen(itemKey);
56 for (int k = 0; k < len; k++)
57 if (itemKey[k] == '=')
58 {
59 itemKey[k] = 0;
60 itemValues[i] = &itemKey[k + 1];
61 break;
62 }
63
64 //printf("key <%s> value <%s>\n", itemKeys[i], itemValues[i]);
65 }
66 }
67
68 static
fdb_request_add1(fdb_request_t * req,const char * param,const char * value)69 int fdb_request_add1(fdb_request_t *req, const char *param, const char *value)
70 {
71 return fdb_request_add(req, param, &value, 1);
72 }
73
74 static
fdbitem_to_request(const char * fdbItem,fdb_request_t * request)75 void fdbitem_to_request(const char *fdbItem, fdb_request_t *request)
76 {
77 KeyValueEntry keyValue;
78 keyValue.item = NULL;
79 decode_fdbitem(fdbItem, &keyValue);
80
81 for (int i = 0; i < keyValue.numKeys; i++)
82 {
83 // printf("key <%s> value <%s>\n", keyValue.keys[i], keyValue.values[i]);
84 fdb_request_add1(request, keyValue.keys[i], keyValue.values[i]);
85 }
86
87 if (keyValue.item) free(keyValue.item);
88 }
89
90
fdb_fill_itemlist(fdb_handle_t * fdb,fdb_request_t * request,char *** itemList)91 int fdb_fill_itemlist(fdb_handle_t *fdb, fdb_request_t *request, char ***itemList)
92 {
93 const char **item = (const char **) malloc(sizeof(const char*));
94
95 fdb_listiterator_t *it;
96 fdb_new_listiterator(&it);
97
98 fdb_list(fdb, request, it);
99
100 int numItems = 0;
101 while (true)
102 {
103 bool exist;
104 fdb_listiterator_next(it, &exist, item);
105 if (!exist) break;
106
107 numItems++;
108 }
109 if (CDI_Debug) Message("numItems = %d", numItems);
110
111 if (*itemList == NULL) *itemList = (char **) malloc(numItems * sizeof(char*));
112
113 fdb_list(fdb, request, it);
114
115 int itemNum = 0;
116 while (true)
117 {
118 bool exist;
119 fdb_listiterator_next(it, &exist, item);
120 if (!exist) break;
121
122 (*itemList)[itemNum++] = strdup(*item);
123 }
124
125 fdb_delete_listiterator(it);
126
127 free(item);
128
129 return numItems;
130 }
131
132
fdb_read_record(fdb_handle_t * fdb,char * item,size_t * buffersize,void ** gribbuffer)133 long fdb_read_record(fdb_handle_t *fdb, char *item, size_t *buffersize, void **gribbuffer)
134 {
135 //Message("%s", item);
136
137 fdb_datareader_t *dataReader = NULL;
138 fdb_new_datareader(&dataReader);
139 fdb_request_t *singleRequest = NULL;
140 fdb_new_request(&singleRequest);
141 fdbitem_to_request(item, singleRequest);
142 int status = fdb_retrieve(fdb, singleRequest, dataReader);
143 fdb_delete_request(singleRequest);
144 if (status != FDB_SUCCESS) Error("fdb_retrieve failed!");
145
146 long recordSize = 0;
147 fdb_datareader_open(dataReader, &recordSize);
148 if (recordSize == 0) Error("fdb_datareader empty!");
149
150 ensureBufferSize(recordSize, buffersize, gribbuffer);
151
152 long readSize = 0;
153 fdb_datareader_read(dataReader, *gribbuffer, recordSize, &readSize);
154 // printf("fdb_datareader_read: size=%ld/%ld\n", recordSize, readSize);
155 if (readSize != recordSize) Error("fdb_datareader_read failed!");
156
157 fdb_datareader_close(dataReader);
158 fdb_delete_datareader(dataReader);
159
160 return recordSize;
161 }
162
163 static
check_numKey(const char * key,int numKeys,int numItems)164 int check_numKey(const char *key, int numKeys, int numItems)
165 {
166 if (numKeys == 0)
167 {
168 Warning("Key %s is missing in all of the FDB records!", key);
169 return -1;
170 }
171 else if (numKeys < numItems)
172 {
173 Warning("Key %s is missing in some of the FDB records!", key);
174 return -2;
175 }
176
177 return 0;
178 }
179
180
check_keyvalueList(int numItems,KeyValueEntry * keyValueList)181 int check_keyvalueList(int numItems, KeyValueEntry *keyValueList)
182 {
183 const char *searchKeys[] = {"date", "time", "param", "levtype"};
184 const int numSearchKeys = sizeof(searchKeys) / sizeof(searchKeys[0]);
185 int searchKeysCount[numSearchKeys];
186 for (int k = 0; k < numSearchKeys; k++) searchKeysCount[k] = 0;
187
188 for (int i = 0; i < numItems; i++)
189 {
190 int numKeys = keyValueList[i].numKeys;
191 char **itemKeys = keyValueList[i].keys;
192 for (int k = 0; k < numSearchKeys; k++)
193 {
194 for (int j = 0; j < numKeys; j++)
195 {
196 if (strcmp(itemKeys[j], searchKeys[k]) == 0)
197 {
198 searchKeysCount[k]++;
199 break;
200 }
201 }
202 }
203 }
204
205 int status = 0;
206 for (int k = 0; k < numSearchKeys; k++)
207 if (check_numKey(searchKeys[k], searchKeysCount[k], numItems) != 0)
208 status = -1;
209
210 return status;
211 }
212
213
214 typedef struct
215 {
216 int date, time, param, levtype;
217 int level;
218 } CmpKeys;
219
record_info_entry_init(RecordInfoEntry * recordInfo)220 void record_info_entry_init(RecordInfoEntry *recordInfo)
221 {
222 recordInfo->date = 0;
223 recordInfo->time = 0;
224 recordInfo->param = 0;
225 recordInfo->levtype = 0;
226 recordInfo->level = 0;
227 }
228
229 static
set_cmpkeys(RecordInfoEntry * recordInfo)230 CmpKeys set_cmpkeys(RecordInfoEntry *recordInfo)
231 {
232 CmpKeys cmpKeys;
233 cmpKeys.date = recordInfo->date;
234 cmpKeys.time = recordInfo->time;
235 cmpKeys.param = recordInfo->param;
236 cmpKeys.levtype = recordInfo->levtype;
237 cmpKeys.level = recordInfo->level;
238 return cmpKeys;
239 }
240
241 static
compare_cmpkeys(const CmpKeys * cmpKeys1,const CmpKeys * cmpKeys2)242 int compare_cmpkeys(const CmpKeys *cmpKeys1, const CmpKeys *cmpKeys2)
243 {
244 if (cmpKeys1->date == cmpKeys2->date &&
245 cmpKeys1->time == cmpKeys2->time &&
246 cmpKeys1->param == cmpKeys2->param &&
247 cmpKeys1->levtype == cmpKeys2->levtype &&
248 cmpKeys1->level == cmpKeys2->level)
249 return 0;
250
251 return -1;
252 }
253
254
get_num_records(int numItems,RecordInfoEntry * recordInfoList)255 int get_num_records(int numItems, RecordInfoEntry *recordInfoList)
256 {
257 const int date = recordInfoList[0].date;
258 const int time = recordInfoList[0].time;
259
260 int numRecords = 0;
261 for (int i = 0; i < numItems; i++)
262 {
263 if (date == recordInfoList[i].date && time == recordInfoList[i].time)
264 numRecords++;
265 else
266 break;
267 }
268
269 CmpKeys cmpKeys0 = set_cmpkeys(&recordInfoList[0]);
270 for (int i = 1; i < numRecords; i++)
271 {
272 CmpKeys cmpKeys = set_cmpkeys(&recordInfoList[i]);
273 if (compare_cmpkeys(&cmpKeys0, &cmpKeys) == 0)
274 {
275 numRecords = i;
276 break;
277 }
278 }
279
280 return numRecords;
281 }
282
283 enum {levTypeUndef = 0, levTypeSFC, levTypeML, levTypePL};
284
285 static
get_ilevtype(const char * levtype)286 int get_ilevtype(const char *levtype)
287 {
288 int ilevtype = levTypeUndef;
289
290 if (strcmp(levtype, "sfc") == 0) ilevtype = levTypeSFC;
291 else if (strcmp(levtype, "ml") == 0) ilevtype = levTypeML;
292 else if (strcmp(levtype, "pl") == 0) ilevtype = levTypeML;
293
294 return ilevtype;
295 }
296
297
decode_keyvalue(KeyValueEntry * keyValue,RecordInfoEntry * recordInfo)298 void decode_keyvalue(KeyValueEntry *keyValue, RecordInfoEntry *recordInfo)
299 {
300 char **itemKeys = keyValue->keys;
301 char **itemValues = keyValue->values;
302 int numKeys = keyValue->numKeys;
303 for (int i = 0; i < numKeys; i++)
304 {
305 //printf("key <%s> value <%s>\n", itemKeys[i], itemValues[i]);
306 if (strcmp(itemKeys[i], "date") == 0) recordInfo->date = atoi(itemValues[i]);
307 else if (strcmp(itemKeys[i], "time") == 0) recordInfo->time = atoi(itemValues[i]);
308 else if (strcmp(itemKeys[i], "param") == 0) recordInfo->param = atoi(itemValues[i]);
309 else if (strcmp(itemKeys[i], "levtype") == 0) recordInfo->levtype = get_ilevtype(itemValues[i]);
310 else if (strcmp(itemKeys[i], "levelist") == 0) recordInfo->level = atoi(itemValues[i]);
311 }
312 }
313
314
remove_duplicate_timesteps(RecordInfoEntry * recordInfoList,int numRecords,int numTimesteps,int * timestepRecordOffset)315 int remove_duplicate_timesteps(RecordInfoEntry *recordInfoList, int numRecords, int numTimesteps, int *timestepRecordOffset)
316 {
317 int numTimestepsNew = numTimesteps;
318
319 int date = recordInfoList[0].date;
320 int time = recordInfoList[0].time;
321
322 for (int i = 1; i < numTimesteps; ++i)
323 {
324 int k = 0;
325 for (k = 0; k < numTimesteps; k++)
326 {
327 const int index = (i + k) * numRecords;
328 if (date != recordInfoList[index].date || time != recordInfoList[index].time) break;
329 }
330
331 int index = i * numRecords;
332 if (k > 0 && k < numTimesteps)
333 {
334 index = (i + k) * numRecords;
335 int n = k;
336 for (k = 0; k < n; k++)
337 {
338 Message("Skip timestep %d", i + k + 1);
339 numTimestepsNew--;
340 for (int j = i; j < numTimestepsNew; j++) timestepRecordOffset[j] = timestepRecordOffset[j + 1];
341 }
342 i += k;
343 if (i >= numTimesteps) break;
344 }
345
346 date = recordInfoList[index].date;
347 time = recordInfoList[index].time;
348 }
349
350 return numTimestepsNew;
351 }
352
353
create_fdb_request(const char * filename)354 fdb_request_t *create_fdb_request(const char *filename)
355 {
356 size_t len = strlen(filename);
357 if (len == 4) Error("Empty FDB request!");
358
359 KeyValueEntry keyValue;
360 keyValue.item = NULL;
361 decode_fdbitem(filename + 4, &keyValue);
362
363 if (keyValue.numKeys == 0) Error("Empty FDB request!");
364
365 fdb_request_t *request = NULL;
366 fdb_new_request(&request);
367
368 bool classDefined = false;
369 bool streamDefined = false;
370 bool expverDefined = false;
371 for (int i = 0; i < keyValue.numKeys; i++)
372 {
373 if (strcmp(keyValue.keys[i], "class") == 0) classDefined = true;
374 else if (strcmp(keyValue.keys[i], "stream") == 0) streamDefined = true;
375 else if (strcmp(keyValue.keys[i], "expver") == 0) expverDefined = true;
376
377 fdb_request_add1(request, keyValue.keys[i], keyValue.values[i]);
378 }
379
380 if (!classDefined) Error("FDB parameter <class> undefined!");
381 if (!streamDefined) Error("FDB parameter <stream> undefined!");
382 if (!expverDefined) Error("FDB parameter <expver> undefined!");
383
384 /*
385 fdb_request_add1(request, "class", "ea");
386 fdb_request_add1(request, "expver", "0001");
387 fdb_request_add1(request, "stream", "oper");
388 fdb_request_add1(request, "domain", "g");
389 fdb_request_add1(request, "date", "20180601");
390 // fdb_request_add1(request, "time", "1800");
391 fdb_request_add1(request, "type", "an");
392 fdb_request_add1(request, "levtype", "sfc");
393 fdb_request_add1(request, "step", "0");
394 fdb_request_add1(request, "param", "139");
395 // fdb_request_add1(request, "levelist", "300");
396 */
397 if (keyValue.item) free(keyValue.item);
398
399 return request;
400 }
401
402 #endif
403