1 /******************************************************************************
2 * Copyright (c) 2011, Howard Butler, hobu.inc@gmail.com *
3 * All rights reserved.
4 *
5 * Redistribution and use in source and binary forms, with or without
6 * modification, are permitted provided that the following
7 * conditions are met:
8 *
9 * * Redistributions of source code must retain the above copyright
10 * notice, this list of conditions and the following disclaimer.
11 * * Redistributions in binary form must reproduce the above copyright
12 * notice, this list of conditions and the following disclaimer in
13 * the documentation and/or other materials provided
14 * with the distribution.
15 * * Neither the name of Hobu, Inc. or Flaxen Geo Consulting nor the
16 * names of its contributors may be used to endorse or promote
17 * products derived from this software without specific prior
18 * written permission.
19 *
20 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
21 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
22 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
23 * FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE
24 * COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
25 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
26 * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS
27 * OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
28 * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
29 * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT
30 * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY
31 * OF SUCH DAMAGE.
32 ****************************************************************************/
33
34 #include <pdal/XMLSchema.hpp>
35 #include <pdal/PipelineWriter.hpp>
36 #include <pdal/PDALUtils.hpp>
37
38 #include <sstream>
39 #include <iostream>
40 #include <iostream>
41 #include <list>
42 #include <cstdlib>
43 #include <map>
44 #include <algorithm>
45 #include <functional>
46
47 #include <string.h>
48 #include <stdlib.h>
49
50 namespace
51 {
52
53 /**
54 void print_element_names(xmlNode * a_node)
55 {
56
57 xmlNode *cur_node = NULL;
58
59 for (cur_node = a_node; cur_node; cur_node = cur_node->next)
60 {
61 if (cur_node->type == XML_ELEMENT_NODE)
62 {
63 printf("node type: Element, name: %s\n", cur_node->name);
64 }
65 print_element_names(cur_node->children);
66 }
67 }
68 **/
69
70 } // anonymous namespace
71
72 namespace pdal
73 {
74
OCISchemaStructuredErrorHandler(void *,xmlErrorPtr error)75 void OCISchemaStructuredErrorHandler
76 (void * /*userData*/, xmlErrorPtr error)
77 {
78 std::ostringstream oss;
79
80 oss << "XML error: '" << error->message <<"' ";
81
82 if (error->str1)
83 oss << " extra info1: '" << error->str1 << "' ";
84 if (error->str2)
85 oss << " extra info2: '" << error->str2 << "' ";
86 if (error->str3)
87 oss << " extra info3: '" << error->str3 << "' ";
88 oss << "on line " << error->line;
89
90 if (error->ctxt)
91 {
92 xmlParserCtxtPtr ctxt = (xmlParserCtxtPtr) error->ctxt;
93 xmlParserInputPtr input = ctxt->input;
94
95 xmlParserPrintFileContext(input);
96 }
97 std::cerr << oss.str() << std::endl;
98 }
99
OCISchemaParserStructuredErrorHandler(void *,xmlErrorPtr error)100 void OCISchemaParserStructuredErrorHandler
101 (void * /*userData*/, xmlErrorPtr error)
102 {
103 std::cerr << "Schema parsing error: '" << error->message << "' " <<
104 "on line " << error->line << std::endl;
105 }
106
OCISchemaValidationStructuredErrorHandler(void *,xmlErrorPtr error)107 void OCISchemaValidationStructuredErrorHandler
108 (void * /*userData*/, xmlErrorPtr error)
109 {
110 std::cerr << "Schema validation error: '" << error->message << "' " <<
111 "on line " << error->line << std::endl;
112 }
113
OCISchemaValidityError(void *,const char * message,...)114 void OCISchemaValidityError
115 (void * /*ctx*/, const char* message, ...)
116 {
117 const int ERROR_MESSAGE_SIZE = 256;
118 char error[ERROR_MESSAGE_SIZE];
119 va_list arg_ptr;
120
121 va_start(arg_ptr, message);
122 vsnprintf(error, ERROR_MESSAGE_SIZE, message, arg_ptr);
123 va_end(arg_ptr);
124
125 std::cerr << "Schema validity error: '" << error << "' " << std::endl;
126 }
127
OCISchemaValidityDebug(void *,const char * message,...)128 void OCISchemaValidityDebug
129 (void * /*ctx*/, const char* message, ...)
130 {
131 const int ERROR_MESSAGE_SIZE = 256;
132 char error[ERROR_MESSAGE_SIZE];
133 va_list arg_ptr;
134
135 va_start(arg_ptr, message);
136 vsnprintf(error, ERROR_MESSAGE_SIZE, message, arg_ptr);
137 va_end(arg_ptr);
138
139 std::cout << "Schema validity debug: '" << error << "' " << "\n";
140 }
141
142
OCISchemaGenericErrorHandler(void *,const char * message,...)143 void OCISchemaGenericErrorHandler
144 (void * /*ctx*/, const char* message, ...)
145 {
146 const int ERROR_MESSAGE_SIZE = 256;
147 char error[ERROR_MESSAGE_SIZE];
148 va_list arg_ptr;
149
150 va_start(arg_ptr, message);
151 vsnprintf(error, ERROR_MESSAGE_SIZE, message, arg_ptr);
152 va_end(arg_ptr);
153
154 std::ostringstream oss;
155
156 std::cerr << "Generic error: '" << error << "'" << std::endl;
157 }
158
XMLSchema(std::string xml,std::string xsd,Orientation orientation)159 XMLSchema::XMLSchema(std::string xml, std::string xsd,
160 Orientation orientation) : m_orientation(orientation)
161 {
162 xmlDocPtr doc = init(xml, xsd);
163 if (doc)
164 {
165 load(doc);
166 xmlFreeDoc(doc);
167 }
168 }
169
170
XMLSchema(const XMLDimList & dims,MetadataNode m,Orientation orientation)171 XMLSchema::XMLSchema(const XMLDimList& dims, MetadataNode m,
172 Orientation orientation) : m_orientation(orientation), m_dims(dims),
173 m_metadata(m)
174 {}
175
176
XMLSchema(const PointLayoutPtr & layout,MetadataNode m,Orientation orientation)177 XMLSchema::XMLSchema(const PointLayoutPtr& layout, MetadataNode m,
178 Orientation orientation) : m_orientation(orientation), m_metadata(m)
179 {
180 DimTypeList dimTypes = layout->dimTypes();
181 for (DimType& d : dimTypes)
182 m_dims.push_back(XMLDim(d, layout->dimName(d.m_id)));
183 }
184
185
xml() const186 std::string XMLSchema::xml() const
187 {
188 xmlBuffer *b = xmlBufferCreate();
189 xmlTextWriterPtr w = xmlNewTextWriterMemory(b, 0);
190
191 xmlTextWriterSetIndent(w, 1);
192 xmlTextWriterStartDocument(w, NULL, "utf-8", NULL);
193 xmlTextWriterStartElementNS(w, (const xmlChar*)"pc",
194 (const xmlChar*)"PointCloudSchema", NULL);
195 xmlTextWriterWriteAttributeNS(w, (const xmlChar*) "xmlns",
196 (const xmlChar*)"pc", NULL,
197 (const xmlChar*)"http://pointcloud.org/schemas/PC/");
198 xmlTextWriterWriteAttributeNS(w, (const xmlChar*)"xmlns",
199 (const xmlChar*)"xsi", NULL,
200 (const xmlChar*)"http://www.w3.org/2001/XMLSchema-instance");
201
202 writeXml(w);
203
204 xmlTextWriterEndElement(w);
205 xmlTextWriterEndDocument(w);
206
207 std::string output((const char *)b->content, b->use);
208 xmlFreeTextWriter(w);
209 xmlBufferFree(b);
210
211 return output;
212 }
213
214
dimTypes() const215 DimTypeList XMLSchema::dimTypes() const
216 {
217 DimTypeList dimTypes;
218
219 for (auto di = m_dims.begin(); di != m_dims.end(); ++di)
220 dimTypes.push_back(di->m_dimType);
221 return dimTypes;
222 }
223
224
init(const std::string & xml,const std::string & xsd)225 xmlDocPtr XMLSchema::init(const std::string& xml, const std::string& xsd)
226 {
227 xmlParserOption parserOption(XML_PARSE_NONET);
228
229 LIBXML_TEST_VERSION
230
231 xmlSetGenericErrorFunc(m_global_context,
232 (xmlGenericErrorFunc)&OCISchemaGenericErrorHandler);
233 xmlSetStructuredErrorFunc(m_global_context,
234 (xmlStructuredErrorFunc)&OCISchemaStructuredErrorHandler);
235
236 xmlDocPtr doc = xmlReadMemory(xml.c_str(), xml.size(), NULL, NULL,
237 parserOption);
238
239 if (xsd.size() && !validate(doc, xsd))
240 {
241 xmlFreeDoc(doc);
242 doc = NULL;
243 std::cerr << "Document did not validate against schema." << std::endl;
244 }
245 return doc;
246 }
247
248
validate(xmlDocPtr doc,const std::string & xsd)249 bool XMLSchema::validate(xmlDocPtr doc, const std::string& xsd)
250 {
251 xmlParserOption parserOption(XML_PARSE_NONET);
252
253 xmlDocPtr schemaDoc = xmlReadMemory(xsd.c_str(), xsd.size(),
254 NULL, NULL, parserOption);
255 xmlSchemaParserCtxtPtr parserCtxt = xmlSchemaNewDocParserCtxt(schemaDoc);
256 xmlSchemaSetParserStructuredErrors(parserCtxt,
257 &OCISchemaParserStructuredErrorHandler, m_global_context);
258 xmlSchemaPtr schema = xmlSchemaParse(parserCtxt);
259 xmlSchemaValidCtxtPtr validCtxt = xmlSchemaNewValidCtxt(schema);
260 xmlSchemaSetValidErrors(validCtxt, &OCISchemaValidityError,
261 &OCISchemaValidityDebug, m_global_context);
262 bool valid = (xmlSchemaValidateDoc(validCtxt, doc) == 0);
263
264 xmlFreeDoc(schemaDoc);
265 xmlSchemaFreeParserCtxt(parserCtxt);
266 xmlSchemaFree(schema);
267 xmlSchemaFreeValidCtxt(validCtxt);
268
269 return valid;
270 }
271
272
remapOldNames(const std::string & input)273 std::string XMLSchema::remapOldNames(const std::string& input)
274 {
275 if (Utils::iequals(input, "Unnamed field 512") ||
276 Utils::iequals(input, "Chipper Point ID"))
277 return std::string("Chipper:PointID");
278
279 if (Utils::iequals(input, "Unnamed field 513") ||
280 Utils::iequals(input, "Chipper Block ID"))
281 return std::string("Chipper:BlockID");
282
283 return input;
284 }
285
286
loadMetadata(xmlNode * startNode,MetadataNode & input)287 bool XMLSchema::loadMetadata(xmlNode *startNode, MetadataNode& input)
288 {
289 // Expect metadata in the following form
290 // We are going to skip the root element because we are
291 // expecting to be given one with our input
292 // <pc:metadata>
293 // <Metadata name="root" type="">
294 // <Metadata name="compression" type="string">lazperf</Metadata>
295 // <Metadata name="version" type="string">1.0</Metadata>
296 // </Metadata>
297 // </pc:metadata>
298
299 xmlNode *node = startNode;
300 for (node = startNode; node; node = node->next)
301 {
302 if (node->type != XML_ELEMENT_NODE)
303 continue;
304 if (std::string((const char*)node->name) == "Metadata")
305 {
306 char *fieldname =
307 (char*)xmlGetProp(node, (const xmlChar*)"name");
308 char *description =
309 (char*)xmlGetProp(node, (const xmlChar*) "description");
310 char *text = (char*)xmlNodeGetContent(node);
311
312 if (!Utils::iequals(fieldname, "root"))
313 {
314 if (!fieldname)
315 {
316 std::cerr << "Unable to read metadata for node '" <<
317 (const char*)node->name << "' no \"name\" was given";
318 return false;
319 }
320 input.add(fieldname, text ? text : "",
321 description ? description : "");
322 }
323 xmlFree(fieldname);
324 xmlFree(description);
325 xmlFree(text);
326 }
327 loadMetadata(node->children, input);
328 }
329 return true;
330 }
331
332
load(xmlDocPtr doc)333 bool XMLSchema::load(xmlDocPtr doc)
334 {
335 xmlNode* root = xmlDocGetRootElement(doc);
336 // print_element_names(root);
337
338 if (!Utils::iequals((const char*)root->name, "PointCloudSchema"))
339 {
340 std::cerr << "First node of document was not named 'PointCloudSchema'";
341 return false;
342 }
343
344 const unsigned SENTINEL_POS = 100000;
345 unsigned missingPos = SENTINEL_POS + 1;
346
347 xmlNode* dimension = root->children;
348 pdal::Metadata metadata;
349 for (xmlNode *dimension = root->children; dimension;
350 dimension = dimension->next)
351 {
352 // Read off orientation setting
353 if (std::string((const char*)dimension->name) == "orientation")
354 {
355 xmlChar* n = xmlNodeListGetString(doc, dimension->children, 1);
356 if (!n)
357 {
358 std::cerr << "Unable to fetch orientation.\n";
359 return false;
360 }
361 std::string orientation = std::string((const char*)n);
362 xmlFree(n);
363
364 if (Utils::iequals(orientation, "dimension"))
365 m_orientation = Orientation::DimensionMajor;
366 else
367 m_orientation = Orientation::PointMajor;
368 continue;
369 }
370
371 if (std::string((const char*)dimension->name) == "metadata")
372 {
373 m_metadata = MetadataNode("root");
374 if (!loadMetadata(dimension, m_metadata))
375 return false;
376 continue;
377 }
378
379 if (dimension->type != XML_ELEMENT_NODE ||
380 !Utils::iequals((const char*)dimension->name, "dimension"))
381 continue;
382
383 XMLDim dim;
384 dim.m_position = SENTINEL_POS;
385 for (xmlNode *properties = dimension->children; properties;
386 properties = properties->next)
387 {
388 if (properties->type != XML_ELEMENT_NODE)
389 continue;
390
391 std::string propName = (const char *)properties->name;
392 propName = Utils::tolower(propName);
393 if (propName == "name")
394 {
395 xmlChar *n = xmlNodeListGetString(doc, properties->children, 1);
396 if (!n)
397 {
398 std::cerr << "Unable to fetch name from XML node.";
399 return false;
400 }
401 dim.m_name = remapOldNames(std::string((const char*)n));
402 xmlFree(n);
403 }
404 if (propName == "description")
405 {
406 xmlChar* n = xmlNodeListGetString(doc, properties->children, 1);
407 if (!n)
408 {
409 std::cerr << "Unable to fetch description.\n";
410 return false;
411 }
412 dim.m_description = std::string((const char*)n);
413 xmlFree(n);
414 }
415 if (propName == "interpretation")
416 {
417 xmlChar* n = xmlNodeListGetString(doc, properties->children, 1);
418 if (!n)
419 {
420 std::cerr << "Unable to fetch interpretation.\n";
421 return false;
422 }
423 dim.m_dimType.m_type = Dimension::type((const char*)n);
424 xmlFree(n);
425 }
426 if (propName == "minimum")
427 {
428 xmlChar* n = xmlGetProp(properties, (const xmlChar*) "value");
429 if (!n)
430 {
431 std::cerr << "Unable to fetch minimum value.\n";
432 return false;
433 }
434 dim.m_min = std::atof((const char*)n);
435 xmlFree(n);
436 }
437 if (propName == "maximum")
438 {
439 xmlChar* n = xmlGetProp(properties, (const xmlChar*) "value");
440 if (!n)
441 {
442 std::cerr << "Unable to fetch maximum value.\n";
443 return false;
444 }
445 dim.m_max = std::atof((const char*)n);
446 xmlFree(n);
447 }
448 if (propName == "position")
449 {
450 xmlChar* n = xmlNodeListGetString(doc, properties->children, 1);
451 if (!n)
452 {
453 std::cerr << "Unable to fetch position value.\n";
454 return false;
455 }
456 dim.m_position = std::atoi((const char*)n);
457 xmlFree(n);
458 }
459 if (propName == "offset")
460 {
461 xmlChar* n = xmlNodeListGetString(doc, properties->children, 1);
462 if (!n)
463 {
464 std::cerr << "Unable to fetch offset value!";
465 return false;
466 }
467 dim.m_dimType.m_xform.m_offset.set((const char*)n);
468 xmlFree(n);
469 }
470 if (propName == "scale")
471 {
472 xmlChar* n = xmlNodeListGetString(doc, properties->children, 1);
473 if (!n)
474 {
475 std::cerr << "Unable to fetch scale value!";
476 return false;
477 }
478 dim.m_dimType.m_xform.m_scale.set((const char*)n);
479 xmlFree(n);
480 }
481 }
482 // If we don't have a position, set it to some value larger than all
483 // previous values.
484 if (dim.m_position == SENTINEL_POS)
485 dim.m_position = missingPos++;
486 m_dims.push_back(dim);
487 }
488 std::sort(m_dims.begin(), m_dims.end());
489
490 // Renumber dimension positions to be 1..N
491 for (unsigned pos = 0; pos < m_dims.size(); pos++)
492 m_dims[pos].m_position = pos + 1;
493
494 return true;
495 }
496
497
xmlDim(Dimension::Id id)498 XMLDim& XMLSchema::xmlDim(Dimension::Id id)
499 {
500 static XMLDim nullDim;
501
502 for (auto di = m_dims.begin(); di != m_dims.end(); ++di)
503 if (di->m_dimType.m_id == id)
504 return *di;
505 return nullDim;
506 }
507
508
xmlDim(Dimension::Id id) const509 const XMLDim& XMLSchema::xmlDim(Dimension::Id id) const
510 {
511 static XMLDim nullDim;
512
513 for (auto di = m_dims.begin(); di != m_dims.end(); ++di)
514 if (di->m_dimType.m_id == id)
515 return *di;
516 return nullDim;
517 }
518
519
xmlDim(const std::string & name)520 XMLDim& XMLSchema::xmlDim(const std::string& name)
521 {
522 static XMLDim nullDim;
523
524 for (auto di = m_dims.begin(); di != m_dims.end(); ++di)
525 if (di->m_name == name)
526 return *di;
527 return nullDim;
528 }
529
530
531 namespace
532 {
533
addMetadataEntry(xmlTextWriterPtr w,const MetadataNode & input)534 void addMetadataEntry(xmlTextWriterPtr w, const MetadataNode& input)
535 {
536
537 std::function<void (const MetadataNode&) > collectMetadata = [&] (const MetadataNode& node)
538 {
539 xmlTextWriterStartElement(w, (const xmlChar*)"Metadata");
540 xmlTextWriterWriteAttribute(w, (const xmlChar*)"name", (const xmlChar*)node.name().c_str());
541 xmlTextWriterWriteAttribute(w, (const xmlChar*)"type", (const xmlChar*)node.type().c_str());
542 xmlTextWriterWriteString(w, (const xmlChar*) node.value().c_str());
543 for (auto& m : node.children())
544 if (!m.empty())
545 collectMetadata(m);
546
547 xmlTextWriterEndElement(w);
548 };
549
550 xmlTextWriterStartElementNS(w, (const xmlChar*)"pc",
551 (const xmlChar*)"metadata", NULL);
552
553 for (auto& m : input.children())
554 if (!m.empty())
555 collectMetadata(m);
556
557 xmlTextWriterEndElement(w);
558 xmlTextWriterFlush(w);
559 }
560
561
562 } // anonymous namespace
563
564
writeXml(xmlTextWriterPtr w) const565 void XMLSchema::writeXml(xmlTextWriterPtr w) const
566 {
567 int pos = 0;
568 for (auto di = m_dims.begin(); di != m_dims.end(); ++di, ++pos)
569 {
570 xmlTextWriterStartElementNS(w, (const xmlChar*)"pc",
571 (const xmlChar*)"dimension", NULL);
572
573 std::ostringstream position;
574 position << (pos + 1);
575 xmlTextWriterWriteElementNS(w, (const xmlChar*)"pc",
576 (const xmlChar*)"position", NULL,
577 (const xmlChar*)position.str().c_str());
578
579 std::ostringstream size;
580 size << Dimension::size(di->m_dimType.m_type);
581 xmlTextWriterWriteElementNS(w, (const xmlChar*)"pc",
582 (const xmlChar*)"size", NULL, (const xmlChar*)size.str().c_str());
583
584 std::string description = Dimension::description(di->m_dimType.m_id);
585 if (description.size())
586 xmlTextWriterWriteElementNS(w, (const xmlChar*)"pc",
587 (const xmlChar*)"description", NULL,
588 (const xmlChar*)description.c_str());
589
590 XForm xform = di->m_dimType.m_xform;
591 if (xform.nonstandard())
592 {
593 std::ostringstream out;
594 out.precision(15);
595
596 out << xform.m_scale.m_val;
597 std::string scale = out.str();
598
599 out.str(std::string());
600 out << xform.m_offset.m_val;
601 std::string offset = out.str();
602
603 out << xform.m_scale.m_val;
604 xmlTextWriterWriteElementNS(w, (const xmlChar*)"pc",
605 (const xmlChar *)"scale", NULL,
606 (const xmlChar *)scale.data());
607 xmlTextWriterWriteElementNS(w, (const xmlChar*)"pc",
608 (const xmlChar *)"offset", NULL,
609 (const xmlChar *)offset.data());
610 }
611
612 std::string name = di->m_name;
613 if (name.size())
614 xmlTextWriterWriteElementNS(w, (const xmlChar*)"pc",
615 (const xmlChar*)"name", NULL, (const xmlChar*)name.c_str());
616
617 xmlTextWriterWriteElementNS(w, (const xmlChar*)"pc",
618 (const xmlChar*)"interpretation", NULL,
619 (const xmlChar*)
620 Dimension::interpretationName(di->m_dimType.m_type).c_str());
621
622 xmlTextWriterWriteElementNS(w, (const xmlChar*)"pc",
623 (const xmlChar*)"active", NULL, (const xmlChar*)"true");
624
625 xmlTextWriterEndElement(w);
626 xmlTextWriterFlush(w);
627 }
628 std::ostringstream orientation;
629 if (m_orientation == Orientation::PointMajor)
630 orientation << "point";
631 if (m_orientation == Orientation::DimensionMajor)
632 orientation << "dimension";
633 if (!m_metadata.empty())
634 {
635 addMetadataEntry(w, m_metadata);
636 }
637 xmlTextWriterWriteElementNS(w, (const xmlChar*) "pc",
638 (const xmlChar*)"orientation", NULL,
639 (const xmlChar*)orientation.str().c_str());
640
641 xmlTextWriterWriteElementNS(w, (const xmlChar*)"pc",
642 (const xmlChar*)"version", NULL,
643 (const xmlChar*)PDAL_XML_SCHEMA_VERSION);
644
645 xmlTextWriterEndElement(w);
646 xmlTextWriterFlush(w);
647 }
648
649 } // namespace pdal
650