1 #include "cdi_fdb.h"
2 
3 int cdi_fdb_dummy;
4 
5 #ifdef HAVE_LIBFDB5
6 
7 #include <string.h>
8 #include <stdlib.h>
9 
10 #include "error.h"
11 #include "cdi_int.h"
12 
13 void ensureBufferSize(size_t requiredSize, size_t *curSize, void **buffer);
14 
decode_fdbitem(const char * fdbItem,KeyValueEntry * keyValue)15 void decode_fdbitem(const char *fdbItem, KeyValueEntry *keyValue)
16 {
17   keyValue->item = strdup(fdbItem);
18   char *pItem = keyValue->item;
19   int numKeys = 0;
20   char **itemKeys = keyValue->keys;
21   char **itemValues = keyValue->values;
22   int len = strlen(pItem);
23   int start = (*pItem == '{');
24   itemKeys[0] = pItem + start;
25   for (int i = start; i < len; i++)
26     {
27       if (pItem[i] == ',')
28         {
29           pItem[i] = 0;
30           numKeys++;
31           itemKeys[numKeys] = &pItem[++i];
32         }
33       else if (pItem[i] == '}')
34         {
35           pItem[i] = 0;
36           numKeys++;
37           if (pItem[i + 1] == '{')
38             {
39               itemKeys[numKeys] = &pItem[i + 2];
40               i += 2;
41             }
42         }
43       else if (i > start && i == (len - 1))
44         {
45           numKeys++;
46         }
47     }
48 
49   keyValue->numKeys = numKeys;
50   //for (int i = 0; i < numKeys; i++)  printf("%d <%s>\n", i, itemKeys[i]);
51 
52   for (int i = 0; i < numKeys; i++)
53     {
54       char *itemKey = itemKeys[i];
55       len = strlen(itemKey);
56       for (int k = 0; k < len; k++)
57         if (itemKey[k] == '=')
58           {
59             itemKey[k] = 0;
60             itemValues[i] = &itemKey[k + 1];
61             break;
62           }
63 
64       //printf("key <%s> value <%s>\n", itemKeys[i], itemValues[i]);
65     }
66 }
67 
68 static
fdb_request_add1(fdb_request_t * req,const char * param,const char * value)69 int fdb_request_add1(fdb_request_t *req, const char *param, const char *value)
70 {
71   return fdb_request_add(req, param, &value, 1);
72 }
73 
74 static
fdbitem_to_request(const char * fdbItem,fdb_request_t * request)75 void fdbitem_to_request(const char *fdbItem, fdb_request_t *request)
76 {
77   KeyValueEntry keyValue;
78   keyValue.item = NULL;
79   decode_fdbitem(fdbItem, &keyValue);
80 
81   for (int i = 0; i < keyValue.numKeys; i++)
82     {
83       // printf("key <%s> value <%s>\n", keyValue.keys[i], keyValue.values[i]);
84       fdb_request_add1(request, keyValue.keys[i], keyValue.values[i]);
85     }
86 
87   if (keyValue.item) free(keyValue.item);
88 }
89 
90 
fdb_fill_itemlist(fdb_handle_t * fdb,fdb_request_t * request,char *** itemList)91 int fdb_fill_itemlist(fdb_handle_t *fdb, fdb_request_t *request, char ***itemList)
92 {
93   const char **item = (const char **) malloc(sizeof(const char*));
94 
95   fdb_listiterator_t *it;
96   fdb_new_listiterator(&it);
97 
98   fdb_list(fdb, request, it);
99 
100   int numItems = 0;
101   while (true)
102     {
103       bool exist;
104       fdb_listiterator_next(it, &exist, item);
105       if (!exist) break;
106 
107       numItems++;
108     }
109   if (CDI_Debug) Message("numItems = %d", numItems);
110 
111   if (*itemList == NULL) *itemList = (char **) malloc(numItems * sizeof(char*));
112 
113   fdb_list(fdb, request, it);
114 
115   int itemNum = 0;
116   while (true)
117     {
118       bool exist;
119       fdb_listiterator_next(it, &exist, item);
120       if (!exist) break;
121 
122       (*itemList)[itemNum++] = strdup(*item);
123     }
124 
125   fdb_delete_listiterator(it);
126 
127   free(item);
128 
129   return numItems;
130 }
131 
132 
fdb_read_record(fdb_handle_t * fdb,char * item,size_t * buffersize,void ** gribbuffer)133 long fdb_read_record(fdb_handle_t *fdb, char *item, size_t *buffersize, void **gribbuffer)
134 {
135   //Message("%s", item);
136 
137   fdb_datareader_t *dataReader = NULL;
138   fdb_new_datareader(&dataReader);
139   fdb_request_t *singleRequest = NULL;
140   fdb_new_request(&singleRequest);
141   fdbitem_to_request(item, singleRequest);
142   int status = fdb_retrieve(fdb, singleRequest, dataReader);
143   fdb_delete_request(singleRequest);
144   if (status != FDB_SUCCESS) Error("fdb_retrieve failed!");
145 
146   long recordSize = 0;
147   fdb_datareader_open(dataReader, &recordSize);
148   if (recordSize == 0) Error("fdb_datareader empty!");
149 
150   ensureBufferSize(recordSize, buffersize, gribbuffer);
151 
152   long readSize = 0;
153   fdb_datareader_read(dataReader, *gribbuffer, recordSize, &readSize);
154   // printf("fdb_datareader_read: size=%ld/%ld\n", recordSize, readSize);
155   if (readSize != recordSize) Error("fdb_datareader_read failed!");
156 
157   fdb_datareader_close(dataReader);
158   fdb_delete_datareader(dataReader);
159 
160   return recordSize;
161 }
162 
163 static
check_numKey(const char * key,int numKeys,int numItems)164 int check_numKey(const char *key, int numKeys, int numItems)
165 {
166   if (numKeys == 0)
167     {
168       Warning("Key %s is missing in all of the FDB records!", key);
169       return -1;
170     }
171   else if (numKeys < numItems)
172     {
173       Warning("Key %s is missing in some of the FDB records!", key);
174       return -2;
175     }
176 
177   return 0;
178 }
179 
180 
check_keyvalueList(int numItems,KeyValueEntry * keyValueList)181 int check_keyvalueList(int numItems, KeyValueEntry *keyValueList)
182 {
183   const char *searchKeys[] = {"date", "time", "param", "levtype"};
184   const int numSearchKeys = sizeof(searchKeys) / sizeof(searchKeys[0]);
185   int searchKeysCount[numSearchKeys];
186   for (int k = 0; k < numSearchKeys; k++) searchKeysCount[k] = 0;
187 
188   for (int i = 0; i < numItems; i++)
189     {
190       int numKeys = keyValueList[i].numKeys;
191       char **itemKeys = keyValueList[i].keys;
192       for (int k = 0; k < numSearchKeys; k++)
193         {
194           for (int j = 0; j < numKeys; j++)
195             {
196               if (strcmp(itemKeys[j], searchKeys[k]) == 0)
197                 {
198                   searchKeysCount[k]++;
199                   break;
200                 }
201             }
202         }
203     }
204 
205   int status = 0;
206   for (int k = 0; k < numSearchKeys; k++)
207     if (check_numKey(searchKeys[k], searchKeysCount[k], numItems) != 0)
208       status = -1;
209 
210   return status;
211 }
212 
213 
214 typedef struct
215 {
216   int date, time, param, levtype;
217   int level;
218 } CmpKeys;
219 
record_info_entry_init(RecordInfoEntry * recordInfo)220 void record_info_entry_init(RecordInfoEntry *recordInfo)
221 {
222   recordInfo->date = 0;
223   recordInfo->time = 0;
224   recordInfo->param = 0;
225   recordInfo->levtype = 0;
226   recordInfo->level = 0;
227 }
228 
229 static
set_cmpkeys(RecordInfoEntry * recordInfo)230 CmpKeys set_cmpkeys(RecordInfoEntry *recordInfo)
231 {
232   CmpKeys cmpKeys;
233   cmpKeys.date = recordInfo->date;
234   cmpKeys.time = recordInfo->time;
235   cmpKeys.param = recordInfo->param;
236   cmpKeys.levtype = recordInfo->levtype;
237   cmpKeys.level = recordInfo->level;
238   return cmpKeys;
239 }
240 
241 static
compare_cmpkeys(const CmpKeys * cmpKeys1,const CmpKeys * cmpKeys2)242 int compare_cmpkeys(const CmpKeys *cmpKeys1, const CmpKeys *cmpKeys2)
243 {
244   if (cmpKeys1->date == cmpKeys2->date &&
245       cmpKeys1->time == cmpKeys2->time &&
246       cmpKeys1->param == cmpKeys2->param &&
247       cmpKeys1->levtype == cmpKeys2->levtype &&
248       cmpKeys1->level == cmpKeys2->level)
249     return 0;
250 
251   return -1;
252 }
253 
254 
get_num_records(int numItems,RecordInfoEntry * recordInfoList)255 int get_num_records(int numItems, RecordInfoEntry *recordInfoList)
256 {
257   const int date = recordInfoList[0].date;
258   const int time = recordInfoList[0].time;
259 
260   int numRecords = 0;
261   for (int i = 0; i < numItems; i++)
262     {
263       if (date == recordInfoList[i].date && time == recordInfoList[i].time)
264         numRecords++;
265       else
266         break;
267     }
268 
269   CmpKeys cmpKeys0 = set_cmpkeys(&recordInfoList[0]);
270   for (int i = 1; i < numRecords; i++)
271     {
272       CmpKeys cmpKeys = set_cmpkeys(&recordInfoList[i]);
273       if (compare_cmpkeys(&cmpKeys0, &cmpKeys) == 0)
274         {
275           numRecords = i;
276           break;
277         }
278     }
279 
280   return numRecords;
281 }
282 
283 enum {levTypeUndef = 0, levTypeSFC, levTypeML, levTypePL};
284 
285 static
get_ilevtype(const char * levtype)286 int get_ilevtype(const char *levtype)
287 {
288   int ilevtype = levTypeUndef;
289 
290   if      (strcmp(levtype, "sfc") == 0) ilevtype = levTypeSFC;
291   else if (strcmp(levtype, "ml") == 0) ilevtype = levTypeML;
292   else if (strcmp(levtype, "pl") == 0) ilevtype = levTypeML;
293 
294   return ilevtype;
295 }
296 
297 
decode_keyvalue(KeyValueEntry * keyValue,RecordInfoEntry * recordInfo)298 void decode_keyvalue(KeyValueEntry *keyValue, RecordInfoEntry *recordInfo)
299 {
300   char **itemKeys = keyValue->keys;
301   char **itemValues = keyValue->values;
302   int numKeys = keyValue->numKeys;
303   for (int i = 0; i < numKeys; i++)
304     {
305       //printf("key <%s> value <%s>\n", itemKeys[i], itemValues[i]);
306       if      (strcmp(itemKeys[i], "date") == 0) recordInfo->date = atoi(itemValues[i]);
307       else if (strcmp(itemKeys[i], "time") == 0) recordInfo->time = atoi(itemValues[i]);
308       else if (strcmp(itemKeys[i], "param") == 0) recordInfo->param = atoi(itemValues[i]);
309       else if (strcmp(itemKeys[i], "levtype") == 0) recordInfo->levtype = get_ilevtype(itemValues[i]);
310       else if (strcmp(itemKeys[i], "levelist") == 0) recordInfo->level = atoi(itemValues[i]);
311    }
312 }
313 
314 
remove_duplicate_timesteps(RecordInfoEntry * recordInfoList,int numRecords,int numTimesteps,int * timestepRecordOffset)315 int remove_duplicate_timesteps(RecordInfoEntry *recordInfoList, int numRecords, int numTimesteps, int *timestepRecordOffset)
316 {
317   int numTimestepsNew = numTimesteps;
318 
319   int date = recordInfoList[0].date;
320   int time = recordInfoList[0].time;
321 
322   for (int i = 1; i < numTimesteps; ++i)
323     {
324       int k = 0;
325       for (k = 0; k < numTimesteps; k++)
326         {
327           const int index = (i + k) * numRecords;
328           if (date != recordInfoList[index].date || time != recordInfoList[index].time) break;
329         }
330 
331       int index = i * numRecords;
332       if (k > 0 && k < numTimesteps)
333         {
334           index = (i + k) * numRecords;
335           int n = k;
336           for (k = 0; k < n; k++)
337             {
338               Message("Skip timestep %d", i + k + 1);
339               numTimestepsNew--;
340               for (int j = i; j < numTimestepsNew; j++) timestepRecordOffset[j] = timestepRecordOffset[j + 1];
341             }
342           i += k;
343           if (i >= numTimesteps) break;
344         }
345 
346       date = recordInfoList[index].date;
347       time = recordInfoList[index].time;
348     }
349 
350   return numTimestepsNew;
351 }
352 
353 
create_fdb_request(const char * filename)354 fdb_request_t *create_fdb_request(const char *filename)
355 {
356   size_t len = strlen(filename);
357   if (len == 4) Error("Empty FDB request!");
358 
359   KeyValueEntry keyValue;
360   keyValue.item = NULL;
361   decode_fdbitem(filename + 4, &keyValue);
362 
363   if (keyValue.numKeys == 0) Error("Empty FDB request!");
364 
365   fdb_request_t *request = NULL;
366   fdb_new_request(&request);
367 
368   bool classDefined = false;
369   bool streamDefined = false;
370   bool expverDefined = false;
371   for (int i = 0; i < keyValue.numKeys; i++)
372     {
373       if      (strcmp(keyValue.keys[i], "class") == 0) classDefined = true;
374       else if (strcmp(keyValue.keys[i], "stream") == 0) streamDefined = true;
375       else if (strcmp(keyValue.keys[i], "expver") == 0) expverDefined = true;
376 
377       fdb_request_add1(request, keyValue.keys[i], keyValue.values[i]);
378     }
379 
380   if (!classDefined) Error("FDB parameter <class> undefined!");
381   if (!streamDefined) Error("FDB parameter <stream> undefined!");
382   if (!expverDefined) Error("FDB parameter <expver> undefined!");
383 
384   /*
385   fdb_request_add1(request, "class", "ea");
386   fdb_request_add1(request, "expver", "0001");
387   fdb_request_add1(request, "stream", "oper");
388   fdb_request_add1(request, "domain", "g");
389   fdb_request_add1(request, "date", "20180601");
390   // fdb_request_add1(request, "time", "1800");
391   fdb_request_add1(request, "type", "an");
392   fdb_request_add1(request, "levtype", "sfc");
393   fdb_request_add1(request, "step", "0");
394   fdb_request_add1(request, "param", "139");
395   // fdb_request_add1(request, "levelist", "300");
396   */
397   if (keyValue.item) free(keyValue.item);
398 
399   return request;
400 }
401 
402 #endif
403