1 /***********************************************************************
2 * pc_schema.c
3 *
4 * Pointclound schema handling. Parse and emit the XML format for
5 * representing packed multidimensional point data.
6 *
7 * PgSQL Pointcloud is free and open source software provided
8 * by the Government of Canada
9 * Copyright (c) 2013 Natural Resources Canada
10 *
11 ***********************************************************************/
12
13 #include <libxml/parser.h>
14 #include <libxml/xpath.h>
15 #include <libxml/xpathInternals.h>
16
17 #include "pc_api_internal.h"
18 #include "stringbuffer.h"
19
20 static char *INTERPRETATION_STRINGS[NUM_INTERPRETATIONS] =
21 {
22 "unknown",
23 "int8_t", "uint8_t",
24 "int16_t", "uint16_t",
25 "int32_t", "uint32_t",
26 "int64_t", "uint64_t",
27 "double", "float"
28 };
29
30 static size_t INTERPRETATION_SIZES[NUM_INTERPRETATIONS] =
31 {
32 -1, /* PC_UNKNOWN */
33 1, 1, /* PC_INT8, PC_UINT8, */
34 2, 2, /* PC_INT16, PC_UINT16 */
35 4, 4, /* PC_INT32, PC_UINT32 */
36 8, 8, /* PC_INT64, PC_UINT64 */
37 8, 4 /* PC_DOUBLE, PC_FLOAT */
38 };
39
40
41 /** Convert XML string token to type interpretation number */
42 const char *
pc_interpretation_string(uint32_t interp)43 pc_interpretation_string(uint32_t interp)
44 {
45 if ( interp < NUM_INTERPRETATIONS )
46 return INTERPRETATION_STRINGS[interp];
47 else
48 return "unknown";
49 }
50
51
52 /** Convert XML string token to type interpretation number */
53 static int
pc_interpretation_number(const char * str)54 pc_interpretation_number(const char *str)
55 {
56 if ( str[0] == 'i' || str[0] == 'I' )
57 {
58 if ( str[3] == '8' )
59 return PC_INT8;
60 if ( str[3] == '1' )
61 return PC_INT16;
62 if ( str[3] == '3' )
63 return PC_INT32;
64 if ( str[3] == '6' )
65 return PC_INT64;
66 }
67 else if ( str[0] == 'u' || str[0] == 'U' )
68 {
69 if ( str[4] == '8' )
70 return PC_UINT8;
71 if ( str[4] == '1' )
72 return PC_UINT16;
73 if ( str[4] == '3' )
74 return PC_UINT32;
75 if ( str[4] == '6' )
76 return PC_UINT64;
77 }
78 else if ( str[0] == 'd' || str[0] == 'D' )
79 {
80 return PC_DOUBLE;
81 }
82 else if ( str[0] == 'f' || str[0] == 'F' )
83 {
84 return PC_FLOAT;
85 }
86 else
87 return PC_UNKNOWN;
88
89 return PC_UNKNOWN;
90 }
91
92 const char*
pc_compression_name(int num)93 pc_compression_name(int num)
94 {
95 switch (num)
96 {
97 case PC_NONE:
98 return "none";
99 case PC_DIMENSIONAL:
100 return "dimensional";
101 case PC_LAZPERF:
102 return "laz";
103 default:
104 return "UNKNOWN";
105 }
106 }
107
108 static int
pc_compression_number(const char * str)109 pc_compression_number(const char *str)
110 {
111 if ( ! str )
112 return PC_NONE;
113
114 if ( (str[0] == 'd' || str[0] == 'D') &&
115 (strcasecmp(str, "dimensional") == 0) )
116 {
117 return PC_DIMENSIONAL;
118 }
119
120 if ( (str[0] == 'l' || str[0] == 'L') &&
121 (strcasecmp(str, "laz") == 0) )
122 {
123 return PC_LAZPERF;
124 }
125
126 if ( (str[0] == 'n' || str[0] == 'N') &&
127 (strcasecmp(str, "none") == 0) )
128 {
129 return PC_NONE;
130 }
131
132 return PC_NONE;
133 }
134
135 /** Convert type interpretation number size in bytes */
136 size_t
pc_interpretation_size(uint32_t interp)137 pc_interpretation_size(uint32_t interp)
138 {
139 if ( interp < NUM_INTERPRETATIONS )
140 {
141 return INTERPRETATION_SIZES[interp];
142 }
143 else
144 {
145 pcerror("pc_interpretation_size: invalid interpretation");
146 return 0;
147 }
148 }
149
150 /** Allocate clean memory for a PCDIMENSION struct */
151 static PCDIMENSION*
pc_dimension_new()152 pc_dimension_new()
153 {
154 PCDIMENSION *pcd = pcalloc(sizeof(PCDIMENSION));
155 /* Default scaling value is 1! */
156 pcd->scale = 1.0;
157 return pcd;
158 }
159
160 static PCDIMENSION*
pc_dimension_clone(const PCDIMENSION * dim)161 pc_dimension_clone(const PCDIMENSION *dim)
162 {
163 PCDIMENSION *pcd = pc_dimension_new();
164 /* Copy all the inline data */
165 memcpy(pcd, dim, sizeof(PCDIMENSION));
166 /* Copy the referenced data */
167 if ( dim->name ) pcd->name = pcstrdup(dim->name);
168 if ( dim->description ) pcd->description = pcstrdup(dim->description);
169 return pcd;
170 }
171
172 /** Release the memory behind the PCDIMENSION struct */
173 static void
pc_dimension_free(PCDIMENSION * pcd)174 pc_dimension_free(PCDIMENSION *pcd)
175 {
176 /* Assumption: No memory in the dimension is owned somewhere else */
177 if ( pcd->description )
178 pcfree(pcd->description);
179 if ( pcd->name )
180 pcfree(pcd->name);
181 pcfree(pcd);
182 }
183
184 static PCSCHEMA*
pc_schema_new(uint32_t ndims)185 pc_schema_new(uint32_t ndims)
186 {
187 PCSCHEMA *pcs = pcalloc(sizeof(PCSCHEMA));
188 pcs->dims = pcalloc(sizeof(PCDIMENSION*) * ndims);
189 pcs->namehash = create_string_hashtable();
190 pcs->ndims = ndims;
191 /* pcalloc memsets to 0, so xdim,ydim,zdim and mdim are already NULL */
192 return pcs;
193 }
194
195 /** Complete the byte offsets of dimensions from the ordered sizes */
196 static void
pc_schema_calculate_byteoffsets(PCSCHEMA * pcs)197 pc_schema_calculate_byteoffsets(PCSCHEMA *pcs)
198 {
199 int i;
200 size_t byteoffset = 0;
201 for ( i = 0; i < pcs->ndims; i++ )
202 {
203 if ( pcs->dims[i] )
204 {
205 pcs->dims[i]->byteoffset = byteoffset;
206 pcs->dims[i]->size = pc_interpretation_size(pcs->dims[i]->interpretation);
207 byteoffset += pcs->dims[i]->size;
208 }
209 }
210 pcs->size = byteoffset;
211 }
212
213 void
pc_schema_set_dimension(PCSCHEMA * s,PCDIMENSION * d)214 pc_schema_set_dimension(PCSCHEMA *s, PCDIMENSION *d)
215 {
216 s->dims[d->position] = d;
217 if ( d->name ) hashtable_insert(s->namehash, d->name, d);
218 pc_schema_calculate_byteoffsets(s);
219 }
220
221
222 PCSCHEMA*
pc_schema_clone(const PCSCHEMA * s)223 pc_schema_clone(const PCSCHEMA *s)
224 {
225 int i;
226 PCSCHEMA *pcs = pc_schema_new(s->ndims);
227 pcs->pcid = s->pcid;
228 pcs->srid = s->srid;
229 pcs->compression = s->compression;
230 for ( i = 0; i < pcs->ndims; i++ )
231 {
232 if ( s->dims[i] )
233 {
234 pc_schema_set_dimension(pcs, pc_dimension_clone(s->dims[i]));
235 }
236 }
237 pcs->xdim = s->xdim ? pcs->dims[s->xdim->position] : NULL;
238 pcs->ydim = s->ydim ? pcs->dims[s->ydim->position] : NULL;
239 pcs->zdim = s->zdim ? pcs->dims[s->zdim->position] : NULL;
240 pcs->mdim = s->mdim ? pcs->dims[s->mdim->position] : NULL;
241 pc_schema_calculate_byteoffsets(pcs);
242 return pcs;
243 }
244
245
246 /** Release the memory behind the PCSCHEMA struct */
247 void
pc_schema_free(PCSCHEMA * pcs)248 pc_schema_free(PCSCHEMA *pcs)
249 {
250 int i;
251
252 for ( i = 0; i < pcs->ndims; i++ )
253 {
254 if ( pcs->dims[i] )
255 {
256 pc_dimension_free(pcs->dims[i]);
257 pcs->dims[i] = 0;
258 }
259 }
260 pcfree(pcs->dims);
261
262 if ( pcs->namehash )
263 hashtable_destroy(pcs->namehash, 0);
264
265 pcfree(pcs);
266 }
267
268 /** Convert a PCSCHEMA to a human-readable JSON string */
269 char *
pc_schema_to_json(const PCSCHEMA * pcs)270 pc_schema_to_json(const PCSCHEMA *pcs)
271 {
272 int i;
273 char *str;
274 stringbuffer_t *sb = stringbuffer_create();
275 stringbuffer_append(sb, "{");
276
277 if ( pcs->pcid )
278 stringbuffer_aprintf(sb, "\"pcid\" : %d,\n", pcs->pcid);
279 if ( pcs->srid )
280 stringbuffer_aprintf(sb, "\"srid\" : %d,\n", pcs->srid);
281 if ( pcs->compression )
282 stringbuffer_aprintf(sb, "\"compression\" : %d,\n", pcs->compression);
283
284
285 if ( pcs->ndims )
286 {
287
288 stringbuffer_append(sb, "\"dims\" : [\n");
289
290 for ( i = 0; i < pcs->ndims; i++ )
291 {
292 if ( pcs->dims[i] )
293 {
294 PCDIMENSION *d = pcs->dims[i];
295
296 if ( i ) stringbuffer_append(sb, ",");
297 stringbuffer_append(sb, "\n { \n");
298
299 if ( d->name )
300 stringbuffer_aprintf(sb, " \"name\" : \"%s\",\n", d->name);
301 if ( d->description )
302 stringbuffer_aprintf(sb, " \"description\" : \"%s\",\n", d->description);
303
304 stringbuffer_aprintf(sb, " \"size\" : %d,\n", d->size);
305 stringbuffer_aprintf(sb, " \"byteoffset\" : %d,\n", d->byteoffset);
306 stringbuffer_aprintf(sb, " \"scale\" : %g,\n", d->scale);
307 stringbuffer_aprintf(sb, " \"interpretation\" : \"%s\",\n", pc_interpretation_string(d->interpretation));
308 stringbuffer_aprintf(sb, " \"offset\" : %g,\n", d->offset);
309
310 stringbuffer_aprintf(sb, " \"active\" : %d\n", d->active);
311 stringbuffer_append(sb, " }");
312 }
313 }
314 stringbuffer_append(sb, "\n]\n");
315 }
316 stringbuffer_append(sb, "}\n");
317 str = stringbuffer_getstringcopy(sb);
318 stringbuffer_destroy(sb);
319 return str;
320 }
321
pc_schema_check_xyzm(PCSCHEMA * s)322 void pc_schema_check_xyzm(PCSCHEMA *s)
323 {
324 int i;
325 for ( i = 0; i < s->ndims; i++ )
326 {
327 char *dimname = s->dims[i]->name;
328 if ( ! dimname ) continue;
329 if ( strcasecmp(dimname, "X") == 0 ||
330 strcasecmp(dimname, "Longitude") == 0 ||
331 strcasecmp(dimname, "Lon") == 0 )
332 {
333 s->xdim = s->dims[i];
334 continue;
335 }
336 if ( strcasecmp(dimname, "Y") == 0 ||
337 strcasecmp(dimname, "Latitude") == 0 ||
338 strcasecmp(dimname, "Lat") == 0 )
339 {
340 s->ydim = s->dims[i];
341 continue;
342 }
343 if ( strcasecmp(dimname, "Z") == 0 ||
344 strcasecmp(dimname, "H") == 0 ||
345 strcasecmp(dimname, "Height") == 0 )
346 {
347 s->zdim = s->dims[i];
348 continue;
349 }
350 if ( strcasecmp(dimname, "M") == 0 ||
351 strcasecmp(dimname, "T") == 0 ||
352 strcasecmp(dimname, "Time") == 0 ||
353 strcasecmp(dimname, "GPSTime") == 0 )
354 {
355 s->mdim = s->dims[i];
356 continue;
357 }
358 }
359 }
360
361 static char *
xml_node_get_content(xmlNodePtr node)362 xml_node_get_content(xmlNodePtr node)
363 {
364 xmlNodePtr cur = node->children;
365 if ( cur )
366 {
367 do
368 {
369 if ( cur->type == XML_TEXT_NODE )
370 {
371 return (char*)(cur->content);
372 }
373 }
374 while ( (cur = cur->next) );
375 }
376 return "";
377 }
378
379 /** Population a PCSCHEMA struct from the XML representation */
380 PCSCHEMA *
pc_schema_from_xml(const char * xml_str)381 pc_schema_from_xml(const char *xml_str)
382 {
383 xmlDocPtr xml_doc = NULL;
384 xmlNodePtr xml_root = NULL;
385 xmlNsPtr xml_ns = NULL;
386 xmlXPathContextPtr xpath_ctx = NULL;
387 xmlXPathObjectPtr xpath_obj = NULL;
388 xmlNodeSetPtr nodes;
389 PCSCHEMA *s = NULL;
390 const char *xml_ptr = xml_str;
391
392 /* Roll forward to start of XML string */
393 while( (*xml_ptr != '\0') && (*xml_ptr != '<') )
394 {
395 xml_ptr++;
396 }
397
398 size_t xml_size = strlen(xml_ptr);
399 static xmlChar *xpath_str = (xmlChar*)("/pc:PointCloudSchema/pc:dimension");
400 static xmlChar *xpath_metadata_str = (xmlChar*)("/pc:PointCloudSchema/pc:metadata/Metadata");
401
402
403 /* Parse XML doc */
404 xmlInitParser();
405 xml_doc = xmlReadMemory(xml_ptr, xml_size, NULL, NULL, 0);
406 if ( ! xml_doc )
407 {
408 pcwarn("unable to parse schema XML");
409 goto cleanup;
410 }
411
412 /* Capture the namespace */
413 xml_root = xmlDocGetRootElement(xml_doc);
414 if ( xml_root->ns )
415 xml_ns = xml_root->ns;
416
417 /* Create xpath evaluation context */
418 xpath_ctx = xmlXPathNewContext(xml_doc);
419 if ( ! xpath_ctx )
420 {
421 pcwarn("unable to create new XPath context to read schema XML");
422 goto cleanup;
423 }
424
425 /* Register the root namespace if there is one */
426 if ( xml_ns )
427 xmlXPathRegisterNs(xpath_ctx, (xmlChar*)"pc", xml_ns->href);
428
429 /* Evaluate xpath expression */
430 xpath_obj = xmlXPathEvalExpression(xpath_str, xpath_ctx);
431 if ( ! xpath_obj )
432 {
433 pcwarn("unable to evaluate xpath expression \"%s\" against schema XML", xpath_str);
434 goto cleanup;
435 }
436
437 /* Iterate on the dimensions we found */
438 if ( (nodes = xpath_obj->nodesetval) )
439 {
440 int ndims = nodes->nodeNr;
441 int i;
442 s = pc_schema_new(ndims);
443
444 for ( i = 0; i < ndims; i++ )
445 {
446 /* This is a "dimension" */
447 if ( nodes->nodeTab[i]->type == XML_ELEMENT_NODE )
448 {
449 xmlNodePtr cur = nodes->nodeTab[i];
450 xmlNodePtr child;
451 PCDIMENSION *d = pc_dimension_new();
452
453 /* These are the values of the dimension */
454 for ( child = cur->children; child; child = child->next )
455 {
456 if ( child->type == XML_ELEMENT_NODE && child->children != NULL)
457 {
458 char *content = (char*)(child->children->content);
459 char *name = (char*)(child->name);
460 if ( strcmp(name, "name") == 0 )
461 d->name = pcstrdup(content);
462 else if ( strcmp(name, "description") == 0 )
463 d->description = pcstrdup(content);
464 else if ( strcmp(name, "size") == 0 )
465 d->size = atoi(content);
466 else if ( strcmp(name, "active") == 0 )
467 d->active = atoi(content);
468 else if ( strcmp(name, "position") == 0 )
469 d->position = atoi(content) - 1;
470 else if ( strcmp(name, "interpretation") == 0 )
471 d->interpretation = pc_interpretation_number(content);
472 else if ( strcmp(name, "scale") == 0 )
473 d->scale = atof(content);
474 else if ( strcmp(name, "offset") == 0 )
475 d->offset = atof(content);
476 else if ( strcmp(name, "uuid") == 0 )
477 /* Ignore this tag for now */ {}
478 else if ( strcmp(name, "parent_uuid") == 0 )
479 /* Ignore this tag for now */ {}
480 else
481 pcinfo("unhandled schema type element \"%s\" encountered", name);
482 }
483 }
484
485 /* Convert interprestation to size */
486 d->size = pc_interpretation_size(d->interpretation);
487
488 /* Store the dimension in the schema */
489 if ( d->position >= ndims )
490 {
491 pcwarn("schema dimension states position \"%d\", but number of XML dimensions is \"%d\"", d->position + 1, ndims);
492 pc_dimension_free(d);
493 goto cleanup;
494
495 }
496 else if ( s->dims[d->position] )
497 {
498 pcwarn("schema dimension at position \"%d\" is declared twice", d->position + 1, ndims);
499 pc_dimension_free(d);
500 goto cleanup;
501 }
502 pc_schema_set_dimension(s, d);
503 }
504 }
505
506 /* Complete the byte offsets of dimensions from the ordered sizes */
507 pc_schema_calculate_byteoffsets(s);
508 /* Check XYZM positions */
509 pc_schema_check_xyzm(s);
510 }
511
512 xmlXPathFreeObject(xpath_obj);
513
514 /* SEARCH FOR METADATA ENTRIES */
515 xpath_obj = xmlXPathEvalExpression(xpath_metadata_str, xpath_ctx);
516 if ( ! xpath_obj )
517 {
518 pcwarn("unable to evaluate xpath expression \"%s\" against schema XML", xpath_metadata_str);
519 goto cleanup;
520 }
521
522 /* Iterate on the <Metadata> we find */
523 if ( (nodes = xpath_obj->nodesetval) )
524 {
525 int i;
526
527 for ( i = 0; i < nodes->nodeNr; i++ )
528 {
529 char *metadata_name = "";
530 char *metadata_value = "";
531 /* Read the metadata name and value from the node */
532 /* <Metadata name="somename">somevalue</Metadata> */
533 xmlNodePtr cur = nodes->nodeTab[i];
534 if ( cur->type == XML_ELEMENT_NODE && strcmp((char*)(cur->name), "Metadata") == 0 )
535 {
536 metadata_name = (char*)xmlGetProp(cur, (xmlChar*)"name");
537 metadata_value = xml_node_get_content(cur);
538 }
539
540 /* Store the compression type on the schema */
541 if ( strcmp(metadata_name, "compression") == 0 )
542 {
543 int compression = pc_compression_number(metadata_value);
544 if ( compression >= 0 )
545 {
546 s->compression = compression;
547 }
548 }
549 xmlFree(metadata_name);
550 }
551 }
552
553 cleanup:
554 if ( s && ! pc_schema_is_valid(s) )
555 {
556 pc_schema_free(s);
557 s = NULL;
558 }
559
560 if ( xpath_obj )
561 xmlXPathFreeObject(xpath_obj);
562 if ( xpath_ctx )
563 xmlXPathFreeContext(xpath_ctx);
564 if ( xml_doc )
565 xmlFreeDoc(xml_doc);
566 xmlCleanupParser();
567
568 return s;
569 }
570
571 uint32_t
pc_schema_is_valid(const PCSCHEMA * s)572 pc_schema_is_valid(const PCSCHEMA *s)
573 {
574 int i;
575
576 if ( ! s->xdim )
577 {
578 pcwarn("schema does not include an X coordinate");
579 return PC_FALSE;
580 }
581
582 if ( ! s->ydim )
583 {
584 pcwarn("schema does not include a Y coordinate");
585 return PC_FALSE;
586 }
587
588 if ( ! s->ndims )
589 {
590 pcwarn("schema has no dimensions");
591 return PC_FALSE;
592 }
593
594 for ( i = 0; i < s->ndims; i++ )
595 {
596 if ( ! s->dims[i] )
597 {
598 pcwarn("schema is missing a dimension at position %d", i);
599 return PC_FALSE;
600 }
601 }
602
603 return PC_TRUE;
604 }
605
606 PCDIMENSION *
pc_schema_get_dimension(const PCSCHEMA * s,uint32_t dim)607 pc_schema_get_dimension(const PCSCHEMA *s, uint32_t dim)
608 {
609 if ( s && s->ndims > dim )
610 {
611 return s->dims[dim];
612 }
613 return NULL;
614 }
615
616 PCDIMENSION *
pc_schema_get_dimension_by_name(const PCSCHEMA * s,const char * name)617 pc_schema_get_dimension_by_name(const PCSCHEMA *s, const char *name)
618 {
619 if ( ! ( s && s->namehash ) )
620 return NULL;
621
622 return hashtable_search(s->namehash, name);
623 }
624
625 size_t
pc_schema_get_size(const PCSCHEMA * s)626 pc_schema_get_size(const PCSCHEMA *s)
627 {
628 return s->size;
629 }
630
631
632 /**
633 * Return true if the schemas have the same dimensions with the same
634 * interpretations and at the same locations. The scales and offsets
635 * may be different though. Otherwise return false.
636 */
637 uint32_t
pc_schema_same_dimensions(const PCSCHEMA * s1,const PCSCHEMA * s2)638 pc_schema_same_dimensions(const PCSCHEMA *s1, const PCSCHEMA *s2)
639 {
640 size_t i;
641
642 if ( s1->ndims != s2->ndims )
643 return PC_FALSE;
644
645 for ( i = 0; i < s1->ndims; i++ )
646 {
647 PCDIMENSION *s1dim = s1->dims[i];
648 PCDIMENSION *s2dim = s2->dims[i];
649
650 if ( strcasecmp(s1dim->name, s2dim->name) != 0 )
651 return PC_FALSE;
652
653 if ( s1dim->interpretation != s2dim->interpretation )
654 return PC_FALSE;
655 }
656
657 return PC_TRUE;
658 }
659
660
661 /**
662 * Return false if s1 and s2 don't have the same srids, or if there are dimensions
663 * in s2 that are also in s1 but don't have the same interpretations, scales or
664 * offsets. Otherwise return true. The function is used to determine if
665 * re-interpretating the patch data is required when changing from one schema
666 * (s1) to another (s2).
667 */
668 uint32_t
pc_schema_same_interpretations(const PCSCHEMA * s1,const PCSCHEMA * s2)669 pc_schema_same_interpretations(const PCSCHEMA *s1, const PCSCHEMA *s2)
670 {
671 size_t i;
672
673 if ( s1->srid != s2->srid )
674 return PC_FALSE;
675
676 for ( i = 0; i < s2->ndims; i++ )
677 {
678 PCDIMENSION *s2dim = s2->dims[i];
679 PCDIMENSION *s1dim = pc_schema_get_dimension_by_name(s1, s2dim->name);
680
681 if ( s1dim )
682 {
683 if ( s1dim->interpretation != s2dim->interpretation )
684 return PC_FALSE;
685
686 if ( s1dim->scale != s2dim->scale )
687 return PC_FALSE;
688
689 if ( s1dim->offset != s2dim->offset )
690 return PC_FALSE;
691 }
692 }
693
694 return PC_TRUE;
695 }
696