1 /*
2     serializer.cpp: Helper class to serialize the application state to a .PLY file
3 
4     This file is part of the implementation of
5 
6         Instant Field-Aligned Meshes
7         Wenzel Jakob, Daniele Panozzo, Marco Tarini, and Olga Sorkine-Hornung
8         In ACM Transactions on Graphics (Proc. SIGGRAPH Asia 2015)
9 
10     All rights reserved. Use of this source code is governed by a
11     BSD-style license that can be found in the LICENSE.txt file.
12 */
13 
14 #include "serializer.h"
15 #include <set>
16 
17 extern "C" {
18     #include "rply.h"
19 }
20 
Serializer()21 Serializer::Serializer() : mCompatibilityMode(false) { mPrefixStack.push(""); }
22 
isSerializedFile(const std::string & filename)23 bool Serializer::isSerializedFile(const std::string &filename) {
24     auto message_cb = [](p_ply ply, const char *msg) { /* ignore */};
25     p_ply ply = ply_open(filename.c_str(), message_cb, 0, nullptr);
26     if (!ply)
27         return false;
28     if (!ply_read_header(ply)) {
29         ply_close(ply);
30         return false;
31     }
32     bool is_serialized = false;
33     const char *comment = nullptr;
34     while ((comment = ply_get_next_comment(ply, comment))) {
35         if (strcmp(comment, "Instant Meshes Application State") == 0)
36             is_serialized = true;
37     }
38     ply_close(ply);
39     return is_serialized;
40 }
41 
getKeys() const42 std::vector<std::string> Serializer::getKeys() const {
43     const std::string &prefix = mPrefixStack.top();
44     std::vector<std::string> result;
45     for (auto const &kv : mData) {
46         if (kv.first.substr(0, prefix.length()) == prefix)
47             result.push_back(kv.first.substr(prefix.length()));
48     }
49     return result;
50 }
51 
52 struct CallbackState {
53     const ProgressCallback &progress;
54     void *data;
55     std::string name;
56     size_t offset, total = 0;
57     bool visited = false;
58 
CallbackStateCallbackState59     CallbackState(const ProgressCallback &progress, void *data,
60         const std::string &name, size_t offset)
61         : progress(progress), data(data), name(name), offset(offset) {}
62 };
63 
64 
Serializer(const std::string & filename,bool compatibilityMode,const ProgressCallback & progress)65 Serializer::Serializer(const std::string &filename, bool compatibilityMode, const ProgressCallback &progress)
66     : mCompatibilityMode(compatibilityMode) {
67     auto message_cb = [](p_ply ply, const char *msg) { cerr << "rply: " << msg << endl; };
68 
69     mPrefixStack.push("");
70     Timer<> timer;
71     cout << "Unserializing application state from \"" << filename << "\" .. ";
72     cout.flush();
73 
74     p_ply ply = ply_open(filename.c_str(), message_cb, 0, nullptr);
75     if (!ply)
76         throw std::runtime_error("Serializer: unable to open PLY file \"" + filename + "\"!");
77 
78     if (!ply_read_header(ply)) {
79         ply_close(ply);
80         throw std::runtime_error("Serializer: unable to open PLY header of \"" + filename + "\"!");
81     }
82 
83     p_ply_element element = nullptr;
84 
85     #define IMPLEMENT(type) \
86         case Variant::Matrix_##type: \
87             const_cast<Eigen::Matrix<type, Eigen::Dynamic, Eigen::Dynamic>&>(*data->matrix_##type)(coord, index) = (type) ply_get_argument_value(argument); \
88             break; \
89         \
90         case Variant::List_##type: {\
91                 ply_get_argument_property(argument, nullptr, &length, &value_index); \
92                 std::vector<std::vector<type>> &vec = \
93                     const_cast<std::vector<std::vector<type>>&>(*data->list_##type); \
94                 if (value_index >= 0) \
95                     vec[index][value_index] = (type) ply_get_argument_value(argument); \
96                 else \
97                     vec[index].resize(length); \
98             } \
99             break;
100 
101 
102     auto rply_element_cb = [](p_ply_argument argument) -> int {
103         CallbackState *state; long index, coord, length, value_index;
104         ply_get_argument_user_data(argument, (void **) &state, &coord);
105         Variant *data = (Variant *) state->data;
106         ply_get_argument_element(argument, nullptr, &index);
107         switch (data->type_id) {
108             IMPLEMENT(uint8_t);
109             IMPLEMENT(uint16_t);
110             IMPLEMENT(uint32_t);
111             IMPLEMENT(float);
112             IMPLEMENT(double);
113             default:
114                 throw std::runtime_error("Unexpected data type while unserializing! (1)");
115         }
116         if (state->progress && !state->visited) {
117             state->progress("Loading field \"" + state->name + "\"", state->offset / (float) state->total);
118             state->visited = true;
119         }
120         return 1;
121     };
122     #undef IMPLEMENT
123 
124     std::vector<CallbackState *> callbackStates;
125     try {
126         /* Inspect the structure of the PLY file */
127         size_t totalSize = 0;
128         while ((element = ply_get_next_element(ply, element)) != nullptr) {
129             const char *element_name;
130             long cols = 0, rows = 0;
131 
132             ply_get_element_info(element, &element_name, &cols);
133             e_ply_type type = PLY_UINT8;
134             bool fail = false, list = false;
135 
136             p_ply_property property = nullptr;
137             while ((property = ply_get_next_property(element, property)) != nullptr) {
138                 e_ply_type property_type, length_type, list_type;
139                 const char *property_name;
140                 ply_get_property_info(property, &property_name, &property_type,
141                                       &length_type, &list_type);
142                 if (property_type == PLY_LIST) {
143                     property_type = list_type;
144                     list = true;
145                 }
146                 if (rows == 0)
147                     type = property_type;
148                 else if (type != property_type)
149                     fail = true;
150                 if (cols == 0)
151                     continue;
152                 Variant &variant = mData[element_name];
153                 CallbackState *state = new CallbackState(progress, &variant, element_name, totalSize);
154                 callbackStates.push_back(state);
155                 if (!ply_set_read_cb(ply, element_name, property_name, rply_element_cb, state, rows++)) {
156                     ply_close(ply);
157                     throw std::runtime_error(
158                         "Serializer: could not register read callback for " + std::string(element_name) +
159                         "." + std::string(property_name) + " in PLY file \"" + filename + "\"!");
160                 }
161             }
162             if (rows == 0 && cols == 0)
163                 rows = 1;
164 
165             if (fail)
166                 throw std::runtime_error("Serializer: unsupported data format in \"" + filename + "\"!");
167 
168             Variant &variant = mData[std::string(element_name)];
169             totalSize += cols;
170 
171             #define IMPLEMENT(ply_type, type) \
172                 case ply_type: \
173                     if (list) { \
174                         auto l = new std::vector<std::vector<type>>(); \
175                         variant.type_id = Variant::Type::List_##type; \
176                         variant.list_##type = l; \
177                         l->resize(cols); \
178                     } else { \
179                         auto mat = new Eigen::Matrix<type, Eigen::Dynamic, Eigen::Dynamic>(rows, cols); \
180                         variant.type_id = Variant::Type::Matrix_##type; \
181                         variant.matrix_##type = mat; \
182                     } \
183                     break;
184 
185             switch (type) {
186                 IMPLEMENT(PLY_UINT8, uint8_t)
187                 IMPLEMENT(PLY_UINT16, uint16_t)
188                 IMPLEMENT(PLY_UINT32, uint32_t)
189                 IMPLEMENT(PLY_FLOAT32, float)
190                 IMPLEMENT(PLY_FLOAT64, double)
191                 default:
192                     throw std::runtime_error("Unexpected data type while unserializing! (2)");
193             }
194             #undef IMPLEMENT
195         }
196         for (auto c : callbackStates)
197             c->total = totalSize;
198 
199         if (!ply_read(ply)) {
200             ply_close(ply);
201             throw std::runtime_error("Error while loading application state from \"" + filename + "\"!");
202         }
203     } catch (...) {
204         for (auto c : callbackStates)
205             delete c;
206         throw;
207     }
208     ply_close(ply);
209     for (auto c : callbackStates)
210         delete c;
211 
212     cout << "done. (took " << timeString(timer.value()) << ")" << endl;
213 }
214 
~Serializer()215 Serializer::~Serializer() {
216     #define IMPLEMENT(type) \
217         case Variant::Type::List_##type: delete kv.second.list_##type; break; \
218         case Variant::Type::Matrix_##type: delete kv.second.matrix_##type; break;
219 
220     for (auto const &kv : mData) {
221         switch (kv.second.type_id) {
222             IMPLEMENT(uint8_t);
223             IMPLEMENT(uint16_t);
224             IMPLEMENT(uint32_t);
225             IMPLEMENT(float);
226             IMPLEMENT(double);
227             default: break;
228         }
229     }
230 
231     #undef IMPLEMENT
232 }
233 
write(const std::string & filename,const ProgressCallback & progress)234 void Serializer::write(const std::string &filename, const ProgressCallback &progress) {
235     auto message_cb = [](p_ply ply, const char *msg) { cerr << "rply: " << msg << endl; };
236 
237     Timer<> timer;
238     cout << "Writing application state to \"" << filename << "\" .. ";
239     cout.flush();
240 
241     p_ply ply = ply_create(filename.c_str(), PLY_DEFAULT, message_cb, 0, nullptr);
242     if (!ply)
243         throw std::runtime_error("Unable to write PLY file!");
244 
245     ply_add_comment(ply, "Instant Meshes Application State");
246 
247     for (auto const &kv : mData) {
248         #define IMPLEMENT(ply_type, type) \
249             case Variant::Type::Matrix_##type: \
250                 ply_add_element(ply, kv.first.c_str(), kv.second.matrix_##type->cols()); \
251                 if (kv.second.matrix_##type->rows() <= 1) \
252                     ply_add_scalar_property(ply, "value", ply_type); \
253             else \
254                     for (uint32_t i=0; i<kv.second.matrix_##type->rows(); ++i) \
255                         ply_add_scalar_property(ply, ("value_" + std::to_string(i)).c_str(), ply_type); \
256                 break; \
257             case Variant::Type::List_##type: { \
258                     ply_add_element(ply, kv.first.c_str(), kv.second.list_##type->size()); \
259                     size_t max_size = 0; \
260                     for (size_t i=0; i<kv.second.list_##type->size(); ++i) \
261                         max_size = std::max(max_size, kv.second.list_##type->operator[](i).size()); \
262                     ply_add_list_property(ply, "value", max_size < 256 ? PLY_UINT8 : (max_size < 65536 ? PLY_UINT16 : PLY_UINT32), ply_type); \
263                 } \
264                 break;
265 
266         switch (kv.second.type_id) {
267             IMPLEMENT(PLY_UINT8, uint8_t)
268             IMPLEMENT(PLY_UINT16, uint16_t)
269             IMPLEMENT(PLY_UINT32, uint32_t)
270             IMPLEMENT(PLY_FLOAT32, float)
271             IMPLEMENT(PLY_FLOAT64, double)
272             default:
273                 throw std::runtime_error("Unexpected data type while serializing! (1)");
274                 break;
275         }
276 
277         #undef IMPLEMENT
278     }
279 
280     ply_write_header(ply);
281 
282     #define IMPLEMENT(type) \
283         case Variant::Type::Matrix_##type: \
284             for (uint32_t i=0; i<kv.second.matrix_##type->size(); ++i) \
285                 ply_write(ply, kv.second.matrix_##type->data()[i]); \
286             written += kv.second.matrix_##type->size() * sizeof(type); \
287             break; \
288         case Variant::Type::List_##type: { \
289                 const std::vector<std::vector<type>> &list = *kv.second.list_##type; \
290                 for (uint32_t i=0; i<list.size(); ++i) { \
291                     ply_write(ply, list[i].size()); \
292                     for (uint32_t j=0; j<list[i].size(); ++j) \
293                         ply_write(ply, list[i][j]); \
294                     written += list[i].size() * sizeof(type) + sizeof(uint32_t); \
295                 } \
296             } \
297             break;
298 
299     size_t size = totalSize(), written = 0;
300     for (auto const &kv : mData) {
301         switch (kv.second.type_id) {
302             IMPLEMENT(uint8_t);
303             IMPLEMENT(uint16_t);
304             IMPLEMENT(uint32_t);
305             IMPLEMENT(float);
306             IMPLEMENT(double);
307             default:
308                 throw std::runtime_error("Unexpected data type while serializing! (2)");
309                 break;
310         }
311         if (progress)
312             progress("Writing \"" + kv.first  + "\"", written / (Float) size);
313     }
314 
315     #undef IMPLEMENT
316 
317     ply_close(ply);
318     cout << "done. (took " << timeString(timer.value()) << ")" << endl;
319 }
320 
totalSize() const321 size_t Serializer::totalSize() const {
322     size_t result = 0;
323 
324     #define IMPLEMENT(type) \
325         case Variant::Type::Matrix_##type: \
326             result += kv.second.matrix_##type->size() * sizeof(type); \
327             break; \
328         case Variant::Type::List_##type: {\
329                 const std::vector<std::vector<type>> &list = *kv.second.list_##type; \
330                 for (uint32_t i=0; i<list.size(); ++i) \
331                     result += list[i].size() * sizeof(type) + sizeof(uint32_t); \
332             } \
333             break;
334 
335     for (auto const &kv : mData) {
336         switch (kv.second.type_id) {
337             IMPLEMENT(uint8_t);
338             IMPLEMENT(uint16_t);
339             IMPLEMENT(uint32_t);
340             IMPLEMENT(float);
341             IMPLEMENT(double);
342         }
343     }
344 
345     #undef IMPLEMENT
346     return result;
347 }
348 
349 
diff(const Serializer & other) const350 bool Serializer::diff(const Serializer &other) const {
351     std::set<std::string> keys;
352     for (auto const &kv : mData)
353         keys.insert(kv.first);
354     for (auto const &kv : other.mData)
355         keys.insert(kv.first);
356 
357     bool diff = false;
358 
359     for (const std::string &key : keys) {
360         auto it1 = mData.find(key);
361         if (it1 == mData.end()) {
362             cout << "Element " << key << " does not exist in serializer 1." << endl;
363             diff = true;
364             continue;
365         }
366         auto it2 = other.mData.find(key);
367         if (it2 == other.mData.end()) {
368             cout << "Element " << key << " does not exist in serializer 2." << endl;
369             diff = true;
370             continue;
371         }
372         const Variant &v1 = it1->second;
373         const Variant &v2 = it2->second;
374         if (v1.type_id != v2.type_id) {
375             cout << "Element " << key << " have different types." << endl;
376             diff = true;
377             continue;
378         }
379 
380         #define IMPLEMENT(type) \
381             case Variant::Type::Matrix_##type: \
382                 if (v1.matrix_##type->cols() != v2.matrix_##type->cols() || \
383                     v1.matrix_##type->rows() != v2.matrix_##type->rows()) \
384                     result = true; \
385                 else \
386                     result = *(v1.matrix_##type) != *(v2.matrix_##type); \
387                 break; \
388             case Variant::Type::List_##type: \
389                 result = *(v1.list_##type) != *(v2.list_##type); \
390                 break;
391 
392         bool result = false;
393         switch (v1.type_id) {
394             IMPLEMENT(uint8_t);
395             IMPLEMENT(uint16_t);
396             IMPLEMENT(uint32_t);
397             IMPLEMENT(float);
398             IMPLEMENT(double);
399             default:
400                 throw std::runtime_error("Unexpected data type while diffing!");
401                 break;
402         }
403 
404         #undef IMPLEMENT
405 
406         if (result) {
407             cout << "Element " << key << " differs." << endl;
408             diff = true;
409         }
410     }
411 
412     return diff;
413 }
414 
operator <<(std::ostream & os,const Serializer & state)415 std::ostream &operator<<(std::ostream &os, const Serializer &state) {
416     os << "Serializer[";
417     bool first = true;
418     for (auto const &kv : state.mData) {
419         const Serializer::Variant &v = kv.second;
420         if (!first)
421             cout << ",";
422         first = false;
423         cout << endl;
424         std::string tname, value;
425 
426         #define IMPLEMENT(type) \
427             case Serializer::Variant::Type::Matrix_##type: { \
428                     const Eigen::Matrix<type, Eigen::Dynamic, Eigen::Dynamic> &mat = *(v.matrix_##type); \
429                     if (mat.size() == 1) { \
430                         tname = #type; \
431                         value = std::to_string(mat(0, 0)); \
432                     } else if (mat.cols() == 1 && mat.rows() <= 4) { \
433                         tname = "vec<" #type ">"; \
434                         value = "["; \
435                         for (int i=0; i<mat.rows(); ++i) { \
436                             value += std::to_string(mat(i, 0)); \
437                             if (i+1<mat.rows()) \
438                                 value += ", "; \
439                         } \
440                         value += "]"; \
441                     } else { \
442                         tname = std::string(mat.cols() == 1 ? "vec<" : "mat<") + #type + std::string(">"); \
443                         value = "data[" + std::to_string(mat.rows()) + "x" + \
444                                           std::to_string(mat.cols()) + "]"; \
445                     } \
446                 } \
447                 break; \
448             case Serializer::Variant::Type::List_##type: \
449                 tname = #type "**"; \
450                 value = "data[" + std::to_string(v.list_##type->size()) + "][]"; \
451                 break; \
452 
453         switch (v.type_id) {
454             IMPLEMENT(uint8_t);
455             IMPLEMENT(uint16_t);
456             IMPLEMENT(uint32_t);
457             IMPLEMENT(float);
458             IMPLEMENT(double);
459             default:
460                 throw std::runtime_error("Unexpected type in stream operator");
461         }
462         os << "\t" << tname << " " << kv.first << " = " << value;
463     }
464     os << endl << "]";
465     return os;
466 }
467