1 /******************************************************************************
2 * Copyright (c) 2011, Howard Butler, hobu.inc@gmail.com *
3 * All rights reserved.
4 *
5 * Redistribution and use in source and binary forms, with or without
6 * modification, are permitted provided that the following
7 * conditions are met:
8 *
9 *     * Redistributions of source code must retain the above copyright
10 *       notice, this list of conditions and the following disclaimer.
11 *     * Redistributions in binary form must reproduce the above copyright
12 *       notice, this list of conditions and the following disclaimer in
13 *       the documentation and/or other materials provided
14 *       with the distribution.
15 *     * Neither the name of Hobu, Inc. or Flaxen Geo Consulting nor the
16 *       names of its contributors may be used to endorse or promote
17 *       products derived from this software without specific prior
18 *       written permission.
19 *
20 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
21 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
22 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
23 * FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE
24 * COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
25 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
26 * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS
27 * OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
28 * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
29 * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT
30 * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY
31 * OF SUCH DAMAGE.
32 ****************************************************************************/
33 
34 #include <pdal/XMLSchema.hpp>
35 #include <pdal/PipelineWriter.hpp>
36 #include <pdal/PDALUtils.hpp>
37 
38 #include <sstream>
39 #include <iostream>
40 #include <iostream>
41 #include <list>
42 #include <cstdlib>
43 #include <map>
44 #include <algorithm>
45 #include <functional>
46 
47 #include <string.h>
48 #include <stdlib.h>
49 
50 namespace
51 {
52 
53 /**
54 void print_element_names(xmlNode * a_node)
55 {
56 
57     xmlNode *cur_node = NULL;
58 
59     for (cur_node = a_node; cur_node; cur_node = cur_node->next)
60     {
61         if (cur_node->type == XML_ELEMENT_NODE)
62         {
63             printf("node type: Element, name: %s\n", cur_node->name);
64         }
65         print_element_names(cur_node->children);
66     }
67 }
68 **/
69 
70 } // anonymous namespace
71 
72 namespace pdal
73 {
74 
OCISchemaStructuredErrorHandler(void *,xmlErrorPtr error)75 void OCISchemaStructuredErrorHandler
76 (void * /*userData*/, xmlErrorPtr error)
77 {
78     std::ostringstream oss;
79 
80     oss << "XML error: '" << error->message <<"' ";
81 
82     if (error->str1)
83         oss << " extra info1: '" << error->str1 << "' ";
84     if (error->str2)
85         oss << " extra info2: '" << error->str2 << "' ";
86     if (error->str3)
87         oss << " extra info3: '" << error->str3 << "' ";
88     oss << "on line " << error->line;
89 
90     if (error->ctxt)
91     {
92         xmlParserCtxtPtr ctxt = (xmlParserCtxtPtr) error->ctxt;
93         xmlParserInputPtr input = ctxt->input;
94 
95         xmlParserPrintFileContext(input);
96     }
97     std::cerr << oss.str() << std::endl;
98 }
99 
OCISchemaParserStructuredErrorHandler(void *,xmlErrorPtr error)100 void OCISchemaParserStructuredErrorHandler
101 (void * /*userData*/, xmlErrorPtr error)
102 {
103     std::cerr << "Schema parsing error: '" << error->message << "' " <<
104         "on line " << error->line << std::endl;
105 }
106 
OCISchemaValidationStructuredErrorHandler(void *,xmlErrorPtr error)107 void OCISchemaValidationStructuredErrorHandler
108 (void * /*userData*/, xmlErrorPtr error)
109 {
110     std::cerr << "Schema validation error: '" << error->message << "' " <<
111         "on line " << error->line << std::endl;
112 }
113 
OCISchemaValidityError(void *,const char * message,...)114 void OCISchemaValidityError
115 (void * /*ctx*/, const char* message, ...)
116 {
117     const int ERROR_MESSAGE_SIZE = 256;
118     char error[ERROR_MESSAGE_SIZE];
119     va_list arg_ptr;
120 
121     va_start(arg_ptr, message);
122     vsnprintf(error, ERROR_MESSAGE_SIZE, message, arg_ptr);
123     va_end(arg_ptr);
124 
125     std::cerr << "Schema validity error: '" << error << "' " << std::endl;
126 }
127 
OCISchemaValidityDebug(void *,const char * message,...)128 void OCISchemaValidityDebug
129 (void * /*ctx*/, const char* message, ...)
130 {
131     const int ERROR_MESSAGE_SIZE = 256;
132     char error[ERROR_MESSAGE_SIZE];
133     va_list arg_ptr;
134 
135     va_start(arg_ptr, message);
136     vsnprintf(error, ERROR_MESSAGE_SIZE, message, arg_ptr);
137     va_end(arg_ptr);
138 
139     std::cout << "Schema validity debug: '" << error << "' " << "\n";
140 }
141 
142 
OCISchemaGenericErrorHandler(void *,const char * message,...)143 void OCISchemaGenericErrorHandler
144 (void * /*ctx*/, const char* message, ...)
145 {
146     const int ERROR_MESSAGE_SIZE = 256;
147     char error[ERROR_MESSAGE_SIZE];
148     va_list arg_ptr;
149 
150     va_start(arg_ptr, message);
151     vsnprintf(error, ERROR_MESSAGE_SIZE, message, arg_ptr);
152     va_end(arg_ptr);
153 
154     std::ostringstream oss;
155 
156     std::cerr << "Generic error: '" << error << "'" << std::endl;
157 }
158 
XMLSchema(std::string xml,std::string xsd,Orientation orientation)159 XMLSchema::XMLSchema(std::string xml, std::string xsd,
160     Orientation orientation) : m_orientation(orientation)
161 {
162     xmlDocPtr doc = init(xml, xsd);
163     if (doc)
164     {
165         load(doc);
166         xmlFreeDoc(doc);
167     }
168 }
169 
170 
XMLSchema(const XMLDimList & dims,MetadataNode m,Orientation orientation)171 XMLSchema::XMLSchema(const XMLDimList& dims, MetadataNode m,
172     Orientation orientation) : m_orientation(orientation), m_dims(dims),
173     m_metadata(m)
174 {}
175 
176 
XMLSchema(const PointLayoutPtr & layout,MetadataNode m,Orientation orientation)177 XMLSchema::XMLSchema(const PointLayoutPtr& layout, MetadataNode m,
178     Orientation orientation) : m_orientation(orientation), m_metadata(m)
179 {
180     DimTypeList dimTypes = layout->dimTypes();
181     for (DimType& d : dimTypes)
182         m_dims.push_back(XMLDim(d, layout->dimName(d.m_id)));
183 }
184 
185 
xml() const186 std::string XMLSchema::xml() const
187 {
188     xmlBuffer *b = xmlBufferCreate();
189     xmlTextWriterPtr w = xmlNewTextWriterMemory(b, 0);
190 
191     xmlTextWriterSetIndent(w, 1);
192     xmlTextWriterStartDocument(w, NULL, "utf-8", NULL);
193     xmlTextWriterStartElementNS(w, (const xmlChar*)"pc",
194         (const xmlChar*)"PointCloudSchema", NULL);
195     xmlTextWriterWriteAttributeNS(w, (const xmlChar*) "xmlns",
196         (const xmlChar*)"pc", NULL,
197         (const xmlChar*)"http://pointcloud.org/schemas/PC/");
198     xmlTextWriterWriteAttributeNS(w, (const xmlChar*)"xmlns",
199         (const xmlChar*)"xsi", NULL,
200         (const xmlChar*)"http://www.w3.org/2001/XMLSchema-instance");
201 
202     writeXml(w);
203 
204     xmlTextWriterEndElement(w);
205     xmlTextWriterEndDocument(w);
206 
207     std::string output((const char *)b->content, b->use);
208     xmlFreeTextWriter(w);
209     xmlBufferFree(b);
210 
211     return output;
212 }
213 
214 
dimTypes() const215 DimTypeList XMLSchema::dimTypes() const
216 {
217     DimTypeList dimTypes;
218 
219     for (auto di = m_dims.begin(); di != m_dims.end(); ++di)
220         dimTypes.push_back(di->m_dimType);
221     return dimTypes;
222 }
223 
224 
init(const std::string & xml,const std::string & xsd)225 xmlDocPtr XMLSchema::init(const std::string& xml, const std::string& xsd)
226 {
227     xmlParserOption parserOption(XML_PARSE_NONET);
228 
229     LIBXML_TEST_VERSION
230 
231     xmlSetGenericErrorFunc(m_global_context,
232         (xmlGenericErrorFunc)&OCISchemaGenericErrorHandler);
233     xmlSetStructuredErrorFunc(m_global_context,
234         (xmlStructuredErrorFunc)&OCISchemaStructuredErrorHandler);
235 
236     xmlDocPtr doc = xmlReadMemory(xml.c_str(), xml.size(), NULL, NULL,
237         parserOption);
238 
239     if (xsd.size() && !validate(doc, xsd))
240     {
241         xmlFreeDoc(doc);
242         doc = NULL;
243         std::cerr << "Document did not validate against schema." << std::endl;
244     }
245     return doc;
246 }
247 
248 
validate(xmlDocPtr doc,const std::string & xsd)249 bool XMLSchema::validate(xmlDocPtr doc, const std::string& xsd)
250 {
251     xmlParserOption parserOption(XML_PARSE_NONET);
252 
253     xmlDocPtr schemaDoc = xmlReadMemory(xsd.c_str(), xsd.size(),
254         NULL, NULL, parserOption);
255     xmlSchemaParserCtxtPtr parserCtxt = xmlSchemaNewDocParserCtxt(schemaDoc);
256     xmlSchemaSetParserStructuredErrors(parserCtxt,
257         &OCISchemaParserStructuredErrorHandler, m_global_context);
258     xmlSchemaPtr schema = xmlSchemaParse(parserCtxt);
259     xmlSchemaValidCtxtPtr validCtxt = xmlSchemaNewValidCtxt(schema);
260     xmlSchemaSetValidErrors(validCtxt, &OCISchemaValidityError,
261         &OCISchemaValidityDebug, m_global_context);
262     bool valid = (xmlSchemaValidateDoc(validCtxt, doc) == 0);
263 
264     xmlFreeDoc(schemaDoc);
265     xmlSchemaFreeParserCtxt(parserCtxt);
266     xmlSchemaFree(schema);
267     xmlSchemaFreeValidCtxt(validCtxt);
268 
269     return valid;
270 }
271 
272 
remapOldNames(const std::string & input)273 std::string XMLSchema::remapOldNames(const std::string& input)
274 {
275     if (Utils::iequals(input, "Unnamed field 512") ||
276             Utils::iequals(input, "Chipper Point ID"))
277         return std::string("Chipper:PointID");
278 
279     if (Utils::iequals(input, "Unnamed field 513") ||
280             Utils::iequals(input, "Chipper Block ID"))
281         return std::string("Chipper:BlockID");
282 
283     return input;
284 }
285 
286 
loadMetadata(xmlNode * startNode,MetadataNode & input)287 bool XMLSchema::loadMetadata(xmlNode *startNode, MetadataNode& input)
288 {
289 //     Expect metadata in the following form
290 //     We are going to skip the root element because we are
291 //     expecting to be given one with our input
292 //     <pc:metadata>
293 //         <Metadata name="root" type="">
294 //             <Metadata name="compression" type="string">lazperf</Metadata>
295 //             <Metadata name="version" type="string">1.0</Metadata>
296 //         </Metadata>
297 //     </pc:metadata>
298 
299     xmlNode *node = startNode;
300     for (node = startNode; node; node = node->next)
301     {
302         if (node->type != XML_ELEMENT_NODE)
303             continue;
304         if (std::string((const char*)node->name) == "Metadata")
305         {
306             char *fieldname =
307                 (char*)xmlGetProp(node, (const xmlChar*)"name");
308             char *description =
309                 (char*)xmlGetProp(node, (const xmlChar*) "description");
310             char *text = (char*)xmlNodeGetContent(node);
311 
312             if (!Utils::iequals(fieldname, "root"))
313             {
314                 if (!fieldname)
315                 {
316                     std::cerr << "Unable to read metadata for node '" <<
317                         (const char*)node->name << "' no \"name\" was given";
318                     return false;
319                 }
320                 input.add(fieldname, text ? text : "",
321                     description ? description : "");
322             }
323             xmlFree(fieldname);
324             xmlFree(description);
325             xmlFree(text);
326         }
327         loadMetadata(node->children, input);
328     }
329     return true;
330 }
331 
332 
load(xmlDocPtr doc)333 bool XMLSchema::load(xmlDocPtr doc)
334 {
335     xmlNode* root = xmlDocGetRootElement(doc);
336     // print_element_names(root);
337 
338     if (!Utils::iequals((const char*)root->name, "PointCloudSchema"))
339     {
340         std::cerr << "First node of document was not named 'PointCloudSchema'";
341         return false;
342     }
343 
344     const unsigned SENTINEL_POS = 100000;
345     unsigned missingPos = SENTINEL_POS + 1;
346 
347     xmlNode* dimension = root->children;
348     pdal::Metadata metadata;
349     for (xmlNode *dimension = root->children; dimension;
350         dimension = dimension->next)
351     {
352         // Read off orientation setting
353         if (std::string((const char*)dimension->name) == "orientation")
354         {
355             xmlChar* n = xmlNodeListGetString(doc, dimension->children, 1);
356             if (!n)
357             {
358                 std::cerr << "Unable to fetch orientation.\n";
359                 return false;
360             }
361             std::string orientation = std::string((const char*)n);
362             xmlFree(n);
363 
364             if (Utils::iequals(orientation, "dimension"))
365                 m_orientation = Orientation::DimensionMajor;
366             else
367                 m_orientation = Orientation::PointMajor;
368             continue;
369         }
370 
371         if (std::string((const char*)dimension->name) == "metadata")
372         {
373             m_metadata = MetadataNode("root");
374             if (!loadMetadata(dimension, m_metadata))
375                 return false;
376             continue;
377         }
378 
379         if (dimension->type != XML_ELEMENT_NODE ||
380             !Utils::iequals((const char*)dimension->name, "dimension"))
381             continue;
382 
383         XMLDim dim;
384         dim.m_position = SENTINEL_POS;
385         for (xmlNode *properties = dimension->children; properties;
386             properties = properties->next)
387         {
388             if (properties->type != XML_ELEMENT_NODE)
389                 continue;
390 
391             std::string propName = (const char *)properties->name;
392             propName = Utils::tolower(propName);
393             if (propName == "name")
394             {
395                 xmlChar *n = xmlNodeListGetString(doc, properties->children, 1);
396                 if (!n)
397                 {
398                     std::cerr << "Unable to fetch name from XML node.";
399                     return false;
400                 }
401                 dim.m_name = remapOldNames(std::string((const char*)n));
402                 xmlFree(n);
403             }
404             if (propName == "description")
405             {
406                 xmlChar* n = xmlNodeListGetString(doc, properties->children, 1);
407                 if (!n)
408                 {
409                     std::cerr << "Unable to fetch description.\n";
410                     return false;
411                 }
412                 dim.m_description = std::string((const char*)n);
413                 xmlFree(n);
414             }
415             if (propName == "interpretation")
416             {
417                 xmlChar* n = xmlNodeListGetString(doc, properties->children, 1);
418                 if (!n)
419                 {
420                     std::cerr << "Unable to fetch interpretation.\n";
421                     return false;
422                 }
423                 dim.m_dimType.m_type = Dimension::type((const char*)n);
424                 xmlFree(n);
425             }
426             if (propName == "minimum")
427             {
428                 xmlChar* n = xmlGetProp(properties, (const xmlChar*) "value");
429                 if (!n)
430                 {
431                     std::cerr << "Unable to fetch minimum value.\n";
432                     return false;
433                 }
434                 dim.m_min = std::atof((const char*)n);
435                 xmlFree(n);
436             }
437             if (propName == "maximum")
438             {
439                 xmlChar* n = xmlGetProp(properties, (const xmlChar*) "value");
440                 if (!n)
441                 {
442                     std::cerr << "Unable to fetch maximum value.\n";
443                     return false;
444                 }
445                 dim.m_max = std::atof((const char*)n);
446                 xmlFree(n);
447             }
448             if (propName == "position")
449             {
450                 xmlChar* n = xmlNodeListGetString(doc, properties->children, 1);
451                 if (!n)
452                 {
453                     std::cerr << "Unable to fetch position value.\n";
454                     return false;
455                 }
456                 dim.m_position = std::atoi((const char*)n);
457                 xmlFree(n);
458             }
459             if (propName == "offset")
460             {
461                 xmlChar* n = xmlNodeListGetString(doc, properties->children, 1);
462                 if (!n)
463                 {
464                     std::cerr << "Unable to fetch offset value!";
465                     return false;
466                 }
467                 dim.m_dimType.m_xform.m_offset.set((const char*)n);
468                 xmlFree(n);
469             }
470             if (propName == "scale")
471             {
472                 xmlChar* n = xmlNodeListGetString(doc, properties->children, 1);
473                 if (!n)
474                 {
475                     std::cerr << "Unable to fetch scale value!";
476                     return false;
477                 }
478                 dim.m_dimType.m_xform.m_scale.set((const char*)n);
479                 xmlFree(n);
480             }
481         }
482         // If we don't have a position, set it to some value larger than all
483         // previous values.
484         if (dim.m_position == SENTINEL_POS)
485             dim.m_position = missingPos++;
486         m_dims.push_back(dim);
487     }
488     std::sort(m_dims.begin(), m_dims.end());
489 
490     // Renumber dimension positions to be 1..N
491     for (unsigned pos = 0; pos < m_dims.size(); pos++)
492         m_dims[pos].m_position = pos + 1;
493 
494     return true;
495 }
496 
497 
xmlDim(Dimension::Id id)498 XMLDim& XMLSchema::xmlDim(Dimension::Id id)
499 {
500     static XMLDim nullDim;
501 
502     for (auto di = m_dims.begin(); di != m_dims.end(); ++di)
503         if (di->m_dimType.m_id == id)
504             return *di;
505     return nullDim;
506 }
507 
508 
xmlDim(Dimension::Id id) const509 const XMLDim& XMLSchema::xmlDim(Dimension::Id id) const
510 {
511     static XMLDim nullDim;
512 
513     for (auto di = m_dims.begin(); di != m_dims.end(); ++di)
514         if (di->m_dimType.m_id == id)
515             return *di;
516     return nullDim;
517 }
518 
519 
xmlDim(const std::string & name)520 XMLDim& XMLSchema::xmlDim(const std::string& name)
521 {
522     static XMLDim nullDim;
523 
524     for (auto di = m_dims.begin(); di != m_dims.end(); ++di)
525         if (di->m_name == name)
526             return *di;
527     return nullDim;
528 }
529 
530 
531 namespace
532 {
533 
addMetadataEntry(xmlTextWriterPtr w,const MetadataNode & input)534 void addMetadataEntry(xmlTextWriterPtr w, const MetadataNode& input)
535 {
536 
537     std::function<void (const MetadataNode&) >  collectMetadata = [&] (const MetadataNode& node)
538     {
539         xmlTextWriterStartElement(w, (const xmlChar*)"Metadata");
540         xmlTextWriterWriteAttribute(w, (const xmlChar*)"name",  (const xmlChar*)node.name().c_str());
541         xmlTextWriterWriteAttribute(w, (const xmlChar*)"type",  (const xmlChar*)node.type().c_str());
542         xmlTextWriterWriteString(w, (const xmlChar*) node.value().c_str());
543         for (auto& m : node.children())
544             if (!m.empty())
545                 collectMetadata(m);
546 
547         xmlTextWriterEndElement(w);
548     };
549 
550     xmlTextWriterStartElementNS(w, (const xmlChar*)"pc",
551         (const xmlChar*)"metadata", NULL);
552 
553     for (auto& m : input.children())
554         if (!m.empty())
555             collectMetadata(m);
556 
557      xmlTextWriterEndElement(w);
558         xmlTextWriterFlush(w);
559 }
560 
561 
562 } // anonymous namespace
563 
564 
writeXml(xmlTextWriterPtr w) const565 void XMLSchema::writeXml(xmlTextWriterPtr w) const
566 {
567     int pos = 0;
568     for (auto di = m_dims.begin(); di != m_dims.end(); ++di, ++pos)
569     {
570         xmlTextWriterStartElementNS(w, (const xmlChar*)"pc",
571             (const xmlChar*)"dimension", NULL);
572 
573         std::ostringstream position;
574         position << (pos + 1);
575         xmlTextWriterWriteElementNS(w, (const xmlChar*)"pc",
576             (const xmlChar*)"position", NULL,
577             (const xmlChar*)position.str().c_str());
578 
579         std::ostringstream size;
580         size << Dimension::size(di->m_dimType.m_type);
581         xmlTextWriterWriteElementNS(w, (const xmlChar*)"pc",
582             (const xmlChar*)"size", NULL, (const xmlChar*)size.str().c_str());
583 
584         std::string description = Dimension::description(di->m_dimType.m_id);
585         if (description.size())
586             xmlTextWriterWriteElementNS(w, (const xmlChar*)"pc",
587                 (const xmlChar*)"description", NULL,
588                 (const xmlChar*)description.c_str());
589 
590         XForm xform = di->m_dimType.m_xform;
591         if (xform.nonstandard())
592         {
593             std::ostringstream out;
594             out.precision(15);
595 
596             out << xform.m_scale.m_val;
597             std::string scale = out.str();
598 
599             out.str(std::string());
600             out << xform.m_offset.m_val;
601             std::string offset = out.str();
602 
603             out << xform.m_scale.m_val;
604             xmlTextWriterWriteElementNS(w, (const xmlChar*)"pc",
605                 (const xmlChar *)"scale", NULL,
606                 (const xmlChar *)scale.data());
607             xmlTextWriterWriteElementNS(w, (const xmlChar*)"pc",
608                 (const xmlChar *)"offset", NULL,
609                 (const xmlChar *)offset.data());
610         }
611 
612         std::string name = di->m_name;
613         if (name.size())
614             xmlTextWriterWriteElementNS(w, (const xmlChar*)"pc",
615                 (const xmlChar*)"name", NULL, (const xmlChar*)name.c_str());
616 
617         xmlTextWriterWriteElementNS(w, (const xmlChar*)"pc",
618             (const xmlChar*)"interpretation", NULL,
619             (const xmlChar*)
620                 Dimension::interpretationName(di->m_dimType.m_type).c_str());
621 
622         xmlTextWriterWriteElementNS(w, (const xmlChar*)"pc",
623             (const xmlChar*)"active", NULL, (const xmlChar*)"true");
624 
625         xmlTextWriterEndElement(w);
626         xmlTextWriterFlush(w);
627     }
628     std::ostringstream orientation;
629     if (m_orientation == Orientation::PointMajor)
630         orientation << "point";
631     if (m_orientation == Orientation::DimensionMajor)
632         orientation << "dimension";
633     if (!m_metadata.empty())
634     {
635         addMetadataEntry(w, m_metadata);
636     }
637     xmlTextWriterWriteElementNS(w, (const xmlChar*) "pc",
638         (const xmlChar*)"orientation", NULL,
639         (const xmlChar*)orientation.str().c_str());
640 
641     xmlTextWriterWriteElementNS(w, (const xmlChar*)"pc",
642         (const xmlChar*)"version", NULL,
643         (const xmlChar*)PDAL_XML_SCHEMA_VERSION);
644 
645     xmlTextWriterEndElement(w);
646     xmlTextWriterFlush(w);
647 }
648 
649 } // namespace pdal
650