1 /*********************************************************************
2 * Copyright 2018, UCAR/Unidata
3 * See netcdf/COPYRIGHT file for copying and redistribution conditions.
4 *********************************************************************/
5
6 #include "includes.h"
7 #include "nc_iter.h"
8 #include "odom.h"
9 #include "ncoffsets.h"
10 #include "netcdf_aux.h"
11
12 /**************************************************/
13 /* Code for generating data lists*/
14 /**************************************************/
15 /* For datalist constant rules: see the rules on the man page */
16
17 /* Forward*/
18 static void generate_array(Symbol*,Bytebuffer*,Datalist*,Generator*,Writer);
19 static void generate_arrayr(Symbol*,Bytebuffer*,Datalist*,Odometer*,int,Datalist*,Generator*);
20 static void generate_primdata(Symbol*, NCConstant*, Bytebuffer*, Datalist* fillsrc, Generator*);
21 static void generate_fieldarray(Symbol*, NCConstant*, Dimset*, Bytebuffer*, Datalist* fillsrc, Generator*);
22
23 /* Mnemonics */
24 #define VLENLIST1
25 #define FIELDARRAY 1
26
27 /**************************************************/
28
29
30 int
generator_getstate(Generator * generator,void ** statep)31 generator_getstate(Generator* generator ,void** statep)
32 {
33 if(statep) *statep = (void*)generator->globalstate;
34 return 1;
35 }
36
generator_reset(Generator * generator,void * state)37 int generator_reset(Generator* generator, void* state)
38 {
39 generator->globalstate = state;
40 return 1;
41 }
42
43 #ifdef IGNORe
44 static void
checkodom(Odometer * odom)45 checkodom(Odometer* odom)
46 {
47 int i;
48 for(i=0;i<odom->rank;i++) {
49 ASSERT(odom->index[i] == odom->start[i]+odom->count[i]);
50 }
51 }
52 #endif
53
54 /**************************************************/
55
56 void
generate_attrdata(Symbol * asym,Generator * generator,Writer writer,Bytebuffer * codebuf)57 generate_attrdata(Symbol* asym, Generator* generator, Writer writer, Bytebuffer* codebuf)
58 {
59 Symbol* basetype = asym->typ.basetype;
60 nc_type typecode = basetype->typ.typecode;
61
62 if(typecode == NC_CHAR) {
63 gen_charattr(asym->data,codebuf);
64 } else {
65 int uid;
66 size_t count;
67 generator->listbegin(generator,asym,NULL,LISTATTR,asym->data->length,codebuf,&uid);
68 for(count=0;count<asym->data->length;count++) {
69 NCConstant* con = datalistith(asym->data,count);
70 generator->list(generator,asym,NULL,LISTATTR,uid,count,codebuf);
71 generate_basetype(asym->typ.basetype,con,codebuf,NULL,generator);
72 }
73 generator->listend(generator,asym,NULL,LISTATTR,uid,count,codebuf);
74 }
75 writer(generator,asym,codebuf,0,NULL,NULL);
76 }
77
78 void
generate_vardata(Symbol * vsym,Generator * generator,Writer writer,Bytebuffer * code)79 generate_vardata(Symbol* vsym, Generator* generator, Writer writer, Bytebuffer* code)
80 {
81 Dimset* dimset = &vsym->typ.dimset;
82 int rank = dimset->ndims;
83 Symbol* basetype = vsym->typ.basetype;
84 Datalist* filler = getfiller(vsym);
85
86 if(vsym->data == NULL) return;
87
88 /* give the buffer a running start to be large enough*/
89 if(!bbSetalloc(code, nciterbuffersize))
90 return;
91
92 if(rank == 0) {/*scalar case*/
93 NCConstant* c0 = datalistith(vsym->data,0);
94 generate_basetype(basetype,c0,code,filler,generator);
95 writer(generator,vsym,code,0,NULL,NULL);
96 } else {/*rank > 0*/
97 #if 0
98 /* First, create an odometer using all of the dimensions */
99 odom = newodometer(dimset,NULL,NULL);
100 start = odometerstartvector(odom);
101 count = odometercountvector(odom);
102 #endif
103 generate_array(vsym,code,filler,generator,writer);
104 }
105 }
106
107
108 /**
109
110 The basic idea is to split the set of dimensions into
111 groups and iterate over each group by recursion.
112
113 A group is defined as the range of indices starting at an
114 unlimited dimension up to (but not including) the next
115 unlimited.
116
117 The first group starts at index 0, even if dimension 0 is not
118 unlimited. The last group is everything from the last
119 unlimited dimension thru the last dimension (index rank-1).
120
121 */
122
123 static void
generate_array(Symbol * vsym,Bytebuffer * code,Datalist * filler,Generator * generator,Writer writer)124 generate_array(Symbol* vsym,
125 Bytebuffer* code,
126 Datalist* filler,
127 Generator* generator,
128 Writer writer
129 )
130 {
131 Dimset* dimset = &vsym->typ.dimset;
132 int rank = dimset->ndims;
133 Symbol* basetype = vsym->typ.basetype;
134 nc_type typecode = basetype->typ.typecode;
135 nciter_t iter;
136 int firstunlim = findunlimited(dimset,1);
137 int nunlim = countunlimited(dimset);
138 int isnc3unlim = (nunlim <= 1 && (firstunlim == 0 || firstunlim == rank)); /* netcdf-3 case of at most 1 unlim in 0th dimension */
139
140 ASSERT(rank > 0);
141
142 if(isnc3unlim) {
143 /* Handle NC_CHAR case separately */
144 if(typecode == NC_CHAR) {
145 Odometer* odom = newodometer(dimset,NULL,NULL);
146 Bytebuffer* charbuf = bbNew();
147 gen_chararray(dimset,0,vsym->data,charbuf,filler);
148 generator->charconstant(generator,vsym,code,charbuf);
149 /* Create an odometer to get the dimension info */
150 writer(generator,vsym,code,odom->rank,odom->start,odom->count);
151 #if 0
152 writer(generator,vsym,code,odom->rank,0,bbLength(charbuf));
153 #endif
154 bbFree(charbuf);
155 odometerfree(odom);
156 } else { /* typecode != NC_CHAR */
157 Odometer* odom = newodometer(dimset,NULL,NULL);
158 /* Case: dim 1..rank-1 are not unlimited, dim 0 might be */
159 size_t offset = 0; /* where are we in the data list */
160 size_t nelems = 0; /* # of data list items to generate */
161 /* Create an iterator and odometer and just walk the datalist */
162 nc_get_iter(vsym,nciterbuffersize,&iter);
163 for(;;offset+=nelems) {
164 int i,uid;
165 nelems=nc_next_iter(&iter,odometerstartvector(odom),odometercountvector(odom));
166 if(nelems == 0)
167 break;
168 bbClear(code);
169 generator->listbegin(generator,vsym,NULL,LISTDATA,vsym->data->length,code,&uid);
170 for(i=0;i<nelems;i++) {
171 NCConstant* con = datalistith(vsym->data,i+offset);
172 generator->list(generator,vsym,NULL,LISTDATA,uid,i,code);
173 generate_basetype(basetype,con,code,filler,generator);
174 }
175 generator->listend(generator,vsym,NULL,LISTDATA,uid,i,code);
176 writer(generator,vsym,code,rank,odom->start,odom->count);
177 }
178 odometerfree(odom);
179 }
180 } else { /* Hard case: multiple unlimited dimensions or unlim in dim > 0*/
181 Odometer* odom = newodometer(dimset,NULL,NULL);
182 /* Setup iterator and odometer */
183 nc_get_iter(vsym,NC_MAX_UINT,&iter); /* effectively infinite */
184 for(;;) {/* iterate in nelem chunks */
185 /* get nelems count and modify odometer */
186 size_t nelems=nc_next_iter(&iter,odom->start,odom->count);
187 if(nelems == 0) break;
188 generate_arrayr(vsym,code,vsym->data,
189 odom,
190 /*dim index=*/0,
191 filler,generator
192 );
193 writer(generator,vsym,code,odom->rank,odom->start,odom->count);
194 }
195 odometerfree(odom);
196 }
197 }
198
199 /**
200 The basic idea is to split the set of dimensions into groups
201 and iterate over each group. A group is defined as the
202 range of indices starting at an unlimited dimension up to
203 (but not including) the next unlimited. The first group
204 starts at index 0, even if dimension 0 is not unlimited.
205 The last group is everything from the last unlimited
206 dimension thru the last dimension (index rank-1).
207 */
208 static void
generate_arrayr(Symbol * vsym,Bytebuffer * code,Datalist * list,Odometer * odom,int dimindex,Datalist * filler,Generator * generator)209 generate_arrayr(Symbol* vsym,
210 Bytebuffer* code,
211 Datalist* list,
212 Odometer* odom,
213 int dimindex,
214 Datalist* filler,
215 Generator* generator
216 )
217 {
218 int uid,i;
219 Symbol* basetype = vsym->typ.basetype;
220 Dimset* dimset = &vsym->typ.dimset;
221 int rank = dimset->ndims;
222 int lastunlimited = findlastunlimited(dimset);
223 int nextunlimited = findunlimited(dimset,dimindex+1);
224 int typecode = basetype->typ.typecode;
225 int islastgroup = (lastunlimited == rank || dimindex >= lastunlimited || dimindex == rank-1);
226 Odometer* subodom = NULL;
227
228 ASSERT(rank > 0);
229 ASSERT((dimindex >= 0 && dimindex < rank));
230
231 if(islastgroup) {
232 /* Handle NC_CHAR case separately */
233 if(typecode == NC_CHAR) {
234 Bytebuffer* charbuf = bbNew();
235 gen_chararray(dimset,dimindex,list,charbuf,filler);
236 generator->charconstant(generator,vsym,code,charbuf);
237 bbFree(charbuf);
238 } else {
239 /* build a special odometer to walk the last few dimensions */
240 subodom = newsubodometer(odom,dimset,dimindex,rank);
241 generator->listbegin(generator,vsym,NULL,LISTDATA,list->length,code,&uid);
242 for(i=0;odometermore(subodom);i++) {
243 size_t offset = odometeroffset(subodom);
244 NCConstant* con = datalistith(list,offset);
245 generator->list(generator,vsym,NULL,LISTDATA,uid,i,code);
246 generate_basetype(basetype,con,code,filler,generator);
247 odometerincr(subodom);
248 }
249 generator->listend(generator,vsym,NULL,LISTDATA,uid,i,code);
250 odometerfree(subodom); subodom = NULL;
251 }
252 } else {/* !islastgroup */
253 /* Our datalist must be a list of compounds representing
254 the next unlimited; so walk the subarray from this index
255 up to next unlimited.
256 */
257 ASSERT((dimindex < nextunlimited));
258 ASSERT((isunlimited(dimset,nextunlimited)));
259 /* build a sub odometer */
260 subodom = newsubodometer(odom,dimset,dimindex,nextunlimited);
261 for(i=0;odometermore(subodom);i++) {
262 size_t offset = odometeroffset(subodom);
263 NCConstant* con = datalistith(list,offset);
264 if(con == NULL || con->nctype == NC_FILL) {
265 if(filler == NULL)
266 filler = getfiller(vsym);
267 generate_arrayr(vsym,code,filler,odom,nextunlimited,NULL,generator);
268
269 } else if(islistconst(con)) {
270 Datalist* sublist = compoundfor(con);
271 generate_arrayr(vsym,code,sublist,odom,nextunlimited,filler,generator);
272 } else {
273 semerror(constline(con),"Expected {...} representing unlimited list");
274 return;
275 }
276 odometerincr(subodom);
277 }
278 odometerfree(subodom); subodom = NULL;
279 }
280 if(subodom != NULL)
281 odometerfree(subodom);
282 return;
283 }
284
285 /* Generate an instance of the basetype using the value of con*/
286 void
generate_basetype(Symbol * tsym,NCConstant * con,Bytebuffer * codebuf,Datalist * filler,Generator * generator)287 generate_basetype(Symbol* tsym, NCConstant* con, Bytebuffer* codebuf, Datalist* filler, Generator* generator)
288 {
289 Datalist* data;
290 int offsetbase = 0;
291
292 switch (tsym->subclass) {
293
294 case NC_ENUM:
295 case NC_OPAQUE:
296 case NC_PRIM:
297 if(islistconst(con)) {
298 semerror(constline(con),"Expected primitive found {..}");
299 }
300 generate_primdata(tsym,con,codebuf,filler,generator);
301 break;
302
303 case NC_COMPOUND: {
304 int i,uid, nfields, dllen;
305 if(con == NULL || isfillconst(con)) {
306 Datalist* fill = (filler==NULL?getfiller(tsym):filler);
307 ASSERT(fill->length == 1);
308 con = fill->data[0];
309 if(!islistconst(con)) {
310 if(con)
311 semerror(con->lineno,"Compound data fill value is not enclosed in {..}");
312 else
313 semerror(0,"Compound data fill value not enclosed in {..}, con is NULL.");
314 }
315 }
316
317 if(!con) { /* fail on null compound. */
318 semerror(constline(con),"NULL compound data.");
319 break;
320 }
321
322 if(!islistconst(con)) {/* fail on no compound*/
323 semerror(constline(con),"Compound data must be enclosed in {..}");
324 }
325
326 data = con->value.compoundv;
327 nfields = listlength(tsym->subnodes);
328 dllen = datalistlen(data);
329 if(dllen > nfields) {
330 semerror(con->lineno,"Datalist longer than the number of compound fields");
331 break;
332 }
333 generator->listbegin(generator,tsym,&offsetbase,LISTCOMPOUND,listlength(tsym->subnodes),codebuf,&uid);
334 for(i=0;i<nfields;i++) {
335 Symbol* field = (Symbol*)listget(tsym->subnodes,i);
336 con = datalistith(data,i);
337 generator->list(generator,field,&offsetbase,LISTCOMPOUND,uid,i,codebuf);
338 generate_basetype(field,con,codebuf,NULL,generator);
339 }
340 generator->listend(generator,tsym,&offsetbase,LISTCOMPOUND,uid,i,codebuf);
341 } break;
342
343 case NC_VLEN: {
344 Bytebuffer* vlenbuf;
345 int uid;
346 size_t count;
347
348 if(con == NULL || isfillconst(con)) {
349 Datalist* fill = (filler==NULL?getfiller(tsym):filler);
350 ASSERT(fill->length == 1);
351 con = fill->data[0];
352 if(con->nctype != NC_COMPOUND) {
353 semerror(con->lineno,"Vlen data fill value is not enclosed in {..}");
354 }
355 }
356
357 if(!islistconst(con)) {
358 semerror(constline(con),"Vlen data must be enclosed in {..}");
359 }
360 data = con->value.compoundv;
361 /* generate the nc_vlen_t instance*/
362 vlenbuf = bbNew();
363 if(tsym->typ.basetype->typ.typecode == NC_CHAR) {
364 gen_charseq(data,vlenbuf);
365 generator->vlenstring(generator,tsym,vlenbuf,&uid,&count);
366 } else {
367 generator->listbegin(generator,tsym,NULL,LISTVLEN,data->length,codebuf,&uid);
368 for(count=0;count<data->length;count++) {
369 NCConstant* con;
370 generator->list(generator,tsym,NULL,LISTVLEN,uid,count,vlenbuf);
371 con = datalistith(data,count);
372 generate_basetype(tsym->typ.basetype,con,vlenbuf,NULL,generator);
373 }
374 generator->listend(generator,tsym,NULL,LISTVLEN,uid,count,codebuf,(void*)vlenbuf);
375 }
376 generator->vlendecl(generator,tsym,codebuf,uid,count,vlenbuf); /* Will extract contents of vlenbuf */
377 bbFree(vlenbuf);
378 } break;
379
380 case NC_FIELD:
381 if(tsym->typ.dimset.ndims > 0) {
382 /* Verify that we have a sublist (or fill situation) */
383 if(con != NULL && !isfillconst(con) && !islistconst(con))
384 semerror(constline(con),"Dimensioned fields must be enclose in {...}");
385 generate_fieldarray(tsym->typ.basetype,con,&tsym->typ.dimset,codebuf,filler,generator);
386 } else {
387 generate_basetype(tsym->typ.basetype,con,codebuf,NULL,generator);
388 }
389 break;
390
391 default: PANIC1("generate_basetype: unexpected subclass %d",tsym->subclass);
392 }
393 }
394
395 /* Used only for structure field arrays*/
396 static void
generate_fieldarray(Symbol * basetype,NCConstant * con,Dimset * dimset,Bytebuffer * codebuf,Datalist * filler,Generator * generator)397 generate_fieldarray(Symbol* basetype, NCConstant* con, Dimset* dimset,
398 Bytebuffer* codebuf, Datalist* filler, Generator* generator)
399 {
400 int i;
401 int chartype = (basetype->typ.typecode == NC_CHAR);
402 Datalist* data;
403 int rank = rankfor(dimset);
404
405 ASSERT(dimset->ndims > 0);
406
407 if(con != NULL && !isfillconst(con))
408 data = con->value.compoundv;
409 else
410 data = NULL;
411
412 if(chartype) {
413 Bytebuffer* charbuf = bbNew();
414 gen_chararray(dimset,0,data,charbuf,filler);
415 generator->charconstant(generator,basetype,codebuf,charbuf);
416 bbFree(charbuf);
417 } else {
418 int uid;
419 size_t xproduct = crossproduct(dimset,0,rank); /* compute total number of elements */
420 generator->listbegin(generator,basetype,NULL,LISTFIELDARRAY,xproduct,codebuf,&uid);
421 for(i=0;i<xproduct;i++) {
422 con = (data == NULL ? NULL : datalistith(data,i));
423 generator->list(generator,basetype,NULL,LISTFIELDARRAY,uid,i,codebuf);
424 generate_basetype(basetype,con,codebuf,NULL,generator);
425 }
426 generator->listend(generator,basetype,NULL,LISTFIELDARRAY,uid,i,codebuf);
427 }
428 }
429
430
431 /* An opaque string value might not conform
432 to the size of the opaque to which it is being
433 assigned. Normalize it to match the required
434 opaque length (in bytes).
435 Note that the string is a sequence of nibbles (4 bits).
436 */
437 static void
normalizeopaquelength(NCConstant * prim,unsigned long nbytes)438 normalizeopaquelength(NCConstant* prim, unsigned long nbytes)
439 {
440 int nnibs = 2*nbytes;
441 ASSERT(prim->nctype==NC_OPAQUE);
442 if(prim->value.opaquev.len == nnibs) {
443 /* do nothing*/
444 } else if(prim->value.opaquev.len > nnibs) { /* truncate*/
445 prim->value.opaquev.stringv[nnibs] = '\0';
446 prim->value.opaquev.len = nnibs;
447 } else {/* prim->value.opaquev.len < nnibs => expand*/
448 char* s;
449 s = (char*)ecalloc(nnibs+1);
450 memset(s,'0',nnibs); /* Fill with '0' characters */
451 memcpy(s,prim->value.opaquev.stringv,prim->value.opaquev.len);
452 s[nnibs] = '\0';
453 efree(prim->value.opaquev.stringv);
454 prim->value.opaquev.stringv=s;
455 prim->value.opaquev.len = nnibs;
456 }
457 }
458
459 static void
generate_primdata(Symbol * basetype,NCConstant * prim,Bytebuffer * codebuf,Datalist * filler,Generator * generator)460 generate_primdata(Symbol* basetype, NCConstant* prim, Bytebuffer* codebuf,
461 Datalist* filler, Generator* generator)
462 {
463 NCConstant* target;
464 int match;
465
466 if(prim == NULL || isfillconst(prim)) {
467 Datalist* fill = (filler==NULL?getfiller(basetype):filler);
468 ASSERT(fill->length == 1);
469 prim = datalistith(fill,0);
470 }
471
472 ASSERT((prim->nctype != NC_COMPOUND));
473
474 /* Verify that the constant is consistent with the type */
475 match = 1;
476 switch (prim->nctype) {
477 case NC_CHAR:
478 case NC_BYTE:
479 case NC_SHORT:
480 case NC_INT:
481 case NC_FLOAT:
482 case NC_DOUBLE:
483 case NC_UBYTE:
484 case NC_USHORT:
485 case NC_UINT:
486 case NC_INT64:
487 case NC_UINT64:
488 case NC_STRING:
489 match = (basetype->subclass == NC_PRIM ? 1 : 0);
490 break;
491
492 #ifdef USE_NETCDF4
493 case NC_NIL:
494 match = (basetype->subclass == NC_PRIM && basetype->typ.typecode == NC_STRING ? 1 : 0);
495 break;
496
497 case NC_OPAQUE:
498 /* OPAQUE is also consistent with numbers */
499 match = (basetype->subclass == NC_OPAQUE
500 || basetype->subclass == NC_PRIM ? 1 : 0);
501 break;
502 case NC_ECONST:
503 match = (basetype->subclass == NC_ENUM ? 1 : 0);
504 if(match) {
505 /* Make sure this econst belongs to this enum */
506 Symbol* ec = prim->value.enumv;
507 Symbol* en = ec->container;
508 match = (en == basetype);
509 }
510 break;
511 #endif
512 default:
513 match = 0;
514 }
515 if(!match) {
516 semerror(constline(prim),"Data value is not consistent with the expected type: %s",
517 basetype->name);
518 }
519
520 target = nullconst();
521 target->nctype = basetype->typ.typecode;
522
523 if(target->nctype != NC_ECONST) {
524 convert1(prim,target);
525 }
526
527 switch (target->nctype) {
528 case NC_ECONST:
529 if(basetype->subclass != NC_ENUM) {
530 semerror(constline(prim),"Conversion to enum not supported (yet)");
531 } break;
532 case NC_OPAQUE:
533 normalizeopaquelength(target,basetype->typ.size);
534 break;
535 default:
536 break;
537 }
538 generator->constant(generator,basetype,target,codebuf);
539 reclaimconstant(target);
540 target = NULL;
541 return;
542 }
543