1 /**
2  * @file   array_schema.cc
3  *
4  * @section LICENSE
5  *
6  * The MIT License
7  *
8  * @copyright Copyright (c) 2018-2021 TileDB, Inc.
9  *
10  * Permission is hereby granted, free of charge, to any person obtaining a copy
11  * of this software and associated documentation files (the "Software"), to deal
12  * in the Software without restriction, including without limitation the rights
13  * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
14  * copies of the Software, and to permit persons to whom the Software is
15  * furnished to do so, subject to the following conditions:
16  *
17  * The above copyright notice and this permission notice shall be included in
18  * all copies or substantial portions of the Software.
19  *
20  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
21  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
22  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
23  * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
24  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
25  * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
26  * THE SOFTWARE.
27  *
28  * @section DESCRIPTION
29  *
30  * This file defines serialization functions for ArraySchema.
31  */
32 #ifdef TILEDB_SERIALIZATION
33 #include <capnp/compat/json.h>
34 #include <capnp/serialize.h>
35 #include "tiledb/sm/serialization/capnp_utils.h"
36 #endif
37 
38 #include "tiledb/common/heap_memory.h"
39 #include "tiledb/common/logger.h"
40 #include "tiledb/sm/array/array.h"
41 #include "tiledb/sm/array_schema/attribute.h"
42 #include "tiledb/sm/array_schema/dimension.h"
43 #include "tiledb/sm/array_schema/domain.h"
44 #include "tiledb/sm/enums/array_type.h"
45 #include "tiledb/sm/enums/compressor.h"
46 #include "tiledb/sm/enums/datatype.h"
47 #include "tiledb/sm/enums/filter_option.h"
48 #include "tiledb/sm/enums/filter_type.h"
49 #include "tiledb/sm/enums/layout.h"
50 #include "tiledb/sm/enums/serialization_type.h"
51 #include "tiledb/sm/misc/constants.h"
52 #include "tiledb/sm/serialization/array_schema.h"
53 
54 #include <set>
55 
56 using namespace tiledb::common;
57 
58 namespace tiledb {
59 namespace sm {
60 namespace serialization {
61 
62 #ifdef TILEDB_SERIALIZATION
63 
filter_pipeline_to_capnp(const FilterPipeline * filter_pipeline,capnp::FilterPipeline::Builder * filter_pipeline_builder)64 Status filter_pipeline_to_capnp(
65     const FilterPipeline* filter_pipeline,
66     capnp::FilterPipeline::Builder* filter_pipeline_builder) {
67   if (filter_pipeline == nullptr)
68     return LOG_STATUS(Status::SerializationError(
69         "Error serializing filter pipeline; filter pipeline is null."));
70 
71   const unsigned num_filters = filter_pipeline->size();
72   if (num_filters == 0)
73     return Status::Ok();
74 
75   auto filter_list_builder = filter_pipeline_builder->initFilters(num_filters);
76   for (unsigned i = 0; i < num_filters; i++) {
77     const auto* filter = filter_pipeline->get_filter(i);
78     auto filter_builder = filter_list_builder[i];
79     filter_builder.setType(filter_type_str(filter->type()));
80 
81     switch (filter->type()) {
82       case FilterType::FILTER_BIT_WIDTH_REDUCTION: {
83         uint32_t window;
84         RETURN_NOT_OK(
85             filter->get_option(FilterOption::BIT_WIDTH_MAX_WINDOW, &window));
86         auto data = filter_builder.initData();
87         data.setUint32(window);
88         break;
89       }
90       case FilterType::FILTER_POSITIVE_DELTA: {
91         uint32_t window;
92         RETURN_NOT_OK(filter->get_option(
93             FilterOption::POSITIVE_DELTA_MAX_WINDOW, &window));
94         auto data = filter_builder.initData();
95         data.setUint32(window);
96         break;
97       }
98       case FilterType::FILTER_GZIP:
99       case FilterType::FILTER_ZSTD:
100       case FilterType::FILTER_LZ4:
101       case FilterType::FILTER_RLE:
102       case FilterType::FILTER_BZIP2:
103       case FilterType::FILTER_DOUBLE_DELTA: {
104         int32_t level;
105         RETURN_NOT_OK(
106             filter->get_option(FilterOption::COMPRESSION_LEVEL, &level));
107         auto data = filter_builder.initData();
108         data.setInt32(level);
109         break;
110       }
111       default:
112         break;
113     }
114   }
115 
116   return Status::Ok();
117 }
118 
filter_pipeline_from_capnp(const capnp::FilterPipeline::Reader & filter_pipeline_reader,tdb_unique_ptr<FilterPipeline> * filter_pipeline)119 Status filter_pipeline_from_capnp(
120     const capnp::FilterPipeline::Reader& filter_pipeline_reader,
121     tdb_unique_ptr<FilterPipeline>* filter_pipeline) {
122   filter_pipeline->reset(tdb_new(FilterPipeline));
123   if (!filter_pipeline_reader.hasFilters())
124     return Status::Ok();
125 
126   auto filter_list_reader = filter_pipeline_reader.getFilters();
127   for (auto filter_reader : filter_list_reader) {
128     FilterType type = FilterType::FILTER_NONE;
129     RETURN_NOT_OK(filter_type_enum(filter_reader.getType().cStr(), &type));
130     tdb_unique_ptr<Filter> filter(Filter::create(type));
131     if (filter == nullptr)
132       return LOG_STATUS(Status::SerializationError(
133           "Error deserializing filter pipeline; failed to create filter."));
134 
135     switch (filter->type()) {
136       case FilterType::FILTER_BIT_WIDTH_REDUCTION: {
137         auto data = filter_reader.getData();
138         uint32_t window = data.getUint32();
139         RETURN_NOT_OK(
140             filter->set_option(FilterOption::BIT_WIDTH_MAX_WINDOW, &window));
141         break;
142       }
143       case FilterType::FILTER_POSITIVE_DELTA: {
144         auto data = filter_reader.getData();
145         uint32_t window = data.getUint32();
146         RETURN_NOT_OK(filter->set_option(
147             FilterOption::POSITIVE_DELTA_MAX_WINDOW, &window));
148         break;
149       }
150       case FilterType::FILTER_GZIP:
151       case FilterType::FILTER_ZSTD:
152       case FilterType::FILTER_LZ4:
153       case FilterType::FILTER_RLE:
154       case FilterType::FILTER_BZIP2:
155       case FilterType::FILTER_DOUBLE_DELTA: {
156         auto data = filter_reader.getData();
157         int32_t level = data.getInt32();
158         RETURN_NOT_OK(
159             filter->set_option(FilterOption::COMPRESSION_LEVEL, &level));
160         break;
161       }
162       default:
163         break;
164     }
165 
166     RETURN_NOT_OK((*filter_pipeline)->add_filter(*filter));
167   }
168 
169   return Status::Ok();
170 }
171 
attribute_to_capnp(const Attribute * attribute,capnp::Attribute::Builder * attribute_builder)172 Status attribute_to_capnp(
173     const Attribute* attribute, capnp::Attribute::Builder* attribute_builder) {
174   if (attribute == nullptr)
175     return LOG_STATUS(Status::SerializationError(
176         "Error serializing attribute; attribute is null."));
177 
178   attribute_builder->setName(attribute->name());
179   attribute_builder->setType(datatype_str(attribute->type()));
180   attribute_builder->setCellValNum(attribute->cell_val_num());
181   attribute_builder->setNullable(attribute->nullable());
182 
183   // Get the fill value from `attribute`.
184   const void* fill_value;
185   uint64_t fill_value_size;
186   uint8_t fill_validity = true;
187   if (!attribute->nullable())
188     RETURN_NOT_OK(attribute->get_fill_value(&fill_value, &fill_value_size));
189   else
190     RETURN_NOT_OK(attribute->get_fill_value(
191         &fill_value, &fill_value_size, &fill_validity));
192 
193   // Copy the fill value buffer into a capnp vector of bytes.
194   auto capnpFillValue = kj::Vector<uint8_t>();
195   capnpFillValue.addAll(kj::ArrayPtr<uint8_t>(
196       const_cast<uint8_t*>(reinterpret_cast<const uint8_t*>(fill_value)),
197       fill_value_size));
198 
199   // Store the fill value vector of bytes.
200   attribute_builder->setFillValue(capnpFillValue.asPtr());
201 
202   // Set the validity fill value.
203   attribute_builder->setFillValueValidity(fill_validity ? true : false);
204 
205   const auto& filters = attribute->filters();
206   auto filter_pipeline_builder = attribute_builder->initFilterPipeline();
207   RETURN_NOT_OK(filter_pipeline_to_capnp(&filters, &filter_pipeline_builder));
208 
209   return Status::Ok();
210 }
211 
attribute_from_capnp(const capnp::Attribute::Reader & attribute_reader,tdb_unique_ptr<Attribute> * attribute)212 Status attribute_from_capnp(
213     const capnp::Attribute::Reader& attribute_reader,
214     tdb_unique_ptr<Attribute>* attribute) {
215   Datatype datatype = Datatype::ANY;
216   RETURN_NOT_OK(datatype_enum(attribute_reader.getType(), &datatype));
217 
218   attribute->reset(tdb_new(Attribute, attribute_reader.getName(), datatype));
219   RETURN_NOT_OK(
220       (*attribute)->set_cell_val_num(attribute_reader.getCellValNum()));
221 
222   // Set nullable.
223   const bool nullable = attribute_reader.getNullable();
224   RETURN_NOT_OK((*attribute)->set_nullable(nullable));
225 
226   // Set the fill value.
227   if (attribute_reader.hasFillValue()) {
228     auto fill_value = attribute_reader.getFillValue();
229     if (nullable) {
230       (*attribute)
231           ->set_fill_value(
232               fill_value.asBytes().begin(),
233               fill_value.size(),
234               attribute_reader.getFillValueValidity());
235     } else {
236       (*attribute)
237           ->set_fill_value(fill_value.asBytes().begin(), fill_value.size());
238     }
239   }
240 
241   // Set filter pipelines.
242   if (attribute_reader.hasFilterPipeline()) {
243     auto filter_pipeline_reader = attribute_reader.getFilterPipeline();
244     tdb_unique_ptr<FilterPipeline> filters;
245     RETURN_NOT_OK(filter_pipeline_from_capnp(filter_pipeline_reader, &filters));
246     RETURN_NOT_OK((*attribute)->set_filter_pipeline(filters.get()));
247   }
248 
249   // Set nullable.
250   RETURN_NOT_OK((*attribute)->set_nullable(attribute_reader.getNullable()));
251 
252   return Status::Ok();
253 }
254 
dimension_to_capnp(const Dimension * dimension,capnp::Dimension::Builder * dimension_builder)255 Status dimension_to_capnp(
256     const Dimension* dimension, capnp::Dimension::Builder* dimension_builder) {
257   if (dimension == nullptr)
258     return LOG_STATUS(Status::SerializationError(
259         "Error serializing dimension; dimension is null."));
260 
261   dimension_builder->setName(dimension->name());
262   dimension_builder->setType(datatype_str(dimension->type()));
263   dimension_builder->setNullTileExtent(!dimension->tile_extent());
264 
265   // Only set the domain if its not empty/null. String dimensions have null
266   // domains
267   if (!dimension->domain().empty()) {
268     auto domain_builder = dimension_builder->initDomain();
269     RETURN_NOT_OK(utils::set_capnp_array_ptr(
270         domain_builder, dimension->type(), dimension->domain().data(), 2));
271   }
272 
273   // Only set the tile extent if its not empty
274   if (dimension->tile_extent()) {
275     auto tile_extent_builder = dimension_builder->initTileExtent();
276     RETURN_NOT_OK(utils::set_capnp_scalar(
277         tile_extent_builder,
278         dimension->type(),
279         dimension->tile_extent().data()));
280   }
281 
282   // Set filters
283   const FilterPipeline& coords_filters = dimension->filters();
284   capnp::FilterPipeline::Builder filters_builder =
285       dimension_builder->initFilterPipeline();
286   RETURN_NOT_OK(filter_pipeline_to_capnp(&coords_filters, &filters_builder));
287   return Status::Ok();
288 }
289 
dimension_from_capnp(const capnp::Dimension::Reader & dimension_reader,tdb_unique_ptr<Dimension> * dimension)290 Status dimension_from_capnp(
291     const capnp::Dimension::Reader& dimension_reader,
292     tdb_unique_ptr<Dimension>* dimension) {
293   Datatype dim_type = Datatype::ANY;
294   RETURN_NOT_OK(datatype_enum(dimension_reader.getType().cStr(), &dim_type));
295   dimension->reset(tdb_new(Dimension, dimension_reader.getName(), dim_type));
296 
297   if (dimension_reader.hasDomain()) {
298     auto domain_reader = dimension_reader.getDomain();
299     Buffer domain_buffer;
300     RETURN_NOT_OK(
301         utils::copy_capnp_list(domain_reader, dim_type, &domain_buffer));
302     RETURN_NOT_OK((*dimension)->set_domain_unsafe(domain_buffer.data()));
303   }
304 
305   if (dimension_reader.hasFilterPipeline()) {
306     auto reader = dimension_reader.getFilterPipeline();
307     tdb_unique_ptr<FilterPipeline> filters;
308     RETURN_NOT_OK(filter_pipeline_from_capnp(reader, &filters));
309     RETURN_NOT_OK((*dimension)->set_filter_pipeline(filters.get()));
310   }
311 
312   if (!dimension_reader.getNullTileExtent()) {
313     auto tile_extent_reader = dimension_reader.getTileExtent();
314     switch (dim_type) {
315       case Datatype::INT8: {
316         auto val = tile_extent_reader.getInt8();
317         RETURN_NOT_OK((*dimension)->set_tile_extent(&val));
318         break;
319       }
320       case Datatype::UINT8: {
321         auto val = tile_extent_reader.getUint8();
322         RETURN_NOT_OK((*dimension)->set_tile_extent(&val));
323         break;
324       }
325       case Datatype::INT16: {
326         auto val = tile_extent_reader.getInt16();
327         RETURN_NOT_OK((*dimension)->set_tile_extent(&val));
328         break;
329       }
330       case Datatype::UINT16: {
331         auto val = tile_extent_reader.getUint16();
332         RETURN_NOT_OK((*dimension)->set_tile_extent(&val));
333         break;
334       }
335       case Datatype::INT32: {
336         auto val = tile_extent_reader.getInt32();
337         RETURN_NOT_OK((*dimension)->set_tile_extent(&val));
338         break;
339       }
340       case Datatype::UINT32: {
341         auto val = tile_extent_reader.getUint32();
342         RETURN_NOT_OK((*dimension)->set_tile_extent(&val));
343         break;
344       }
345       case Datatype::DATETIME_YEAR:
346       case Datatype::DATETIME_MONTH:
347       case Datatype::DATETIME_WEEK:
348       case Datatype::DATETIME_DAY:
349       case Datatype::DATETIME_HR:
350       case Datatype::DATETIME_MIN:
351       case Datatype::DATETIME_SEC:
352       case Datatype::DATETIME_MS:
353       case Datatype::DATETIME_US:
354       case Datatype::DATETIME_NS:
355       case Datatype::DATETIME_PS:
356       case Datatype::DATETIME_FS:
357       case Datatype::DATETIME_AS:
358       case Datatype::TIME_HR:
359       case Datatype::TIME_MIN:
360       case Datatype::TIME_SEC:
361       case Datatype::TIME_MS:
362       case Datatype::TIME_US:
363       case Datatype::TIME_NS:
364       case Datatype::TIME_PS:
365       case Datatype::TIME_FS:
366       case Datatype::TIME_AS:
367       case Datatype::INT64: {
368         auto val = tile_extent_reader.getInt64();
369         RETURN_NOT_OK((*dimension)->set_tile_extent(&val));
370         break;
371       }
372       case Datatype::UINT64: {
373         auto val = tile_extent_reader.getUint64();
374         RETURN_NOT_OK((*dimension)->set_tile_extent(&val));
375         break;
376       }
377       case Datatype::FLOAT32: {
378         auto val = tile_extent_reader.getFloat32();
379         RETURN_NOT_OK((*dimension)->set_tile_extent(&val));
380         break;
381       }
382       case Datatype::FLOAT64: {
383         auto val = tile_extent_reader.getFloat64();
384         RETURN_NOT_OK((*dimension)->set_tile_extent(&val));
385         break;
386       }
387       default:
388         return LOG_STATUS(Status::SerializationError(
389             "Error deserializing dimension; unknown datatype."));
390     }
391   }
392 
393   return Status::Ok();
394 }
395 
domain_to_capnp(const Domain * domain,capnp::Domain::Builder * domainBuilder)396 Status domain_to_capnp(
397     const Domain* domain, capnp::Domain::Builder* domainBuilder) {
398   if (domain == nullptr)
399     return LOG_STATUS(Status::SerializationError(
400         "Error serializing domain; domain is null."));
401 
402   domainBuilder->setType(datatype_str(domain->dimension(0)->type()));
403   domainBuilder->setTileOrder(layout_str(domain->tile_order()));
404   domainBuilder->setCellOrder(layout_str(domain->cell_order()));
405 
406   const unsigned ndims = domain->dim_num();
407   auto dimensions_builder = domainBuilder->initDimensions(ndims);
408   for (unsigned i = 0; i < ndims; i++) {
409     auto dim_builder = dimensions_builder[i];
410     RETURN_NOT_OK(dimension_to_capnp(domain->dimension(i), &dim_builder));
411   }
412 
413   return Status::Ok();
414 }
415 
domain_from_capnp(const capnp::Domain::Reader & domain_reader,tdb_unique_ptr<Domain> * domain)416 Status domain_from_capnp(
417     const capnp::Domain::Reader& domain_reader,
418     tdb_unique_ptr<Domain>* domain) {
419   Datatype datatype = Datatype::ANY;
420   RETURN_NOT_OK(datatype_enum(domain_reader.getType(), &datatype));
421   domain->reset(tdb_new(Domain));
422 
423   auto dimensions = domain_reader.getDimensions();
424   for (auto dimension : dimensions) {
425     tdb_unique_ptr<Dimension> dim;
426     RETURN_NOT_OK(dimension_from_capnp(dimension, &dim));
427     RETURN_NOT_OK((*domain)->add_dimension(dim.get()));
428   }
429 
430   return Status::Ok();
431 }
432 
array_schema_to_capnp(const ArraySchema * array_schema,capnp::ArraySchema::Builder * array_schema_builder,const bool client_side)433 Status array_schema_to_capnp(
434     const ArraySchema* array_schema,
435     capnp::ArraySchema::Builder* array_schema_builder,
436     const bool client_side) {
437   if (array_schema == nullptr)
438     return LOG_STATUS(Status::SerializationError(
439         "Error serializing array schema; array schema is null."));
440 
441   // Only set the URI if client side
442   if (client_side)
443     array_schema_builder->setUri(array_schema->array_uri().to_string());
444   auto v = kj::heapArray<int32_t>(1);
445   v[0] = array_schema->version();
446   array_schema_builder->setVersion(v);
447   array_schema_builder->setArrayType(
448       array_type_str(array_schema->array_type()));
449   array_schema_builder->setTileOrder(layout_str(array_schema->tile_order()));
450   array_schema_builder->setCellOrder(layout_str(array_schema->cell_order()));
451   array_schema_builder->setCapacity(array_schema->capacity());
452   array_schema_builder->setAllowsDuplicates(array_schema->allows_dups());
453 
454   // Set coordinate filters
455   const FilterPipeline& coords_filters = array_schema->coords_filters();
456   capnp::FilterPipeline::Builder coords_filters_builder =
457       array_schema_builder->initCoordsFilterPipeline();
458   RETURN_NOT_OK(
459       filter_pipeline_to_capnp(&coords_filters, &coords_filters_builder));
460 
461   // Set offset filters
462   const FilterPipeline& offsets_filters =
463       array_schema->cell_var_offsets_filters();
464   capnp::FilterPipeline::Builder offsets_filters_builder =
465       array_schema_builder->initOffsetFilterPipeline();
466   RETURN_NOT_OK(
467       filter_pipeline_to_capnp(&offsets_filters, &offsets_filters_builder));
468 
469   // Set validity filters
470   const FilterPipeline& validity_filters =
471       array_schema->cell_validity_filters();
472   capnp::FilterPipeline::Builder validity_filters_builder =
473       array_schema_builder->initValidityFilterPipeline();
474   RETURN_NOT_OK(
475       filter_pipeline_to_capnp(&validity_filters, &validity_filters_builder));
476 
477   // Domain
478   auto domain_builder = array_schema_builder->initDomain();
479   RETURN_NOT_OK(domain_to_capnp(array_schema->domain(), &domain_builder));
480 
481   // Attributes
482   const unsigned num_attrs = array_schema->attribute_num();
483   auto attributes_buidler = array_schema_builder->initAttributes(num_attrs);
484   for (size_t i = 0; i < num_attrs; i++) {
485     auto attribute_builder = attributes_buidler[i];
486     RETURN_NOT_OK(
487         attribute_to_capnp(array_schema->attribute(i), &attribute_builder));
488   }
489 
490   return Status::Ok();
491 }
492 
array_schema_from_capnp(const capnp::ArraySchema::Reader & schema_reader,tdb_unique_ptr<ArraySchema> * array_schema)493 Status array_schema_from_capnp(
494     const capnp::ArraySchema::Reader& schema_reader,
495     tdb_unique_ptr<ArraySchema>* array_schema) {
496   ArrayType array_type = ArrayType::DENSE;
497   RETURN_NOT_OK(array_type_enum(schema_reader.getArrayType(), &array_type));
498   array_schema->reset(tdb_new(ArraySchema, array_type));
499 
500   Layout layout = Layout::ROW_MAJOR;
501   RETURN_NOT_OK(layout_enum(schema_reader.getTileOrder().cStr(), &layout));
502   (*array_schema)->set_tile_order(layout);
503   RETURN_NOT_OK(layout_enum(schema_reader.getCellOrder().cStr(), &layout));
504 
505   if (schema_reader.hasUri())
506     (*array_schema)->set_array_uri(URI(schema_reader.getUri().cStr()));
507 
508   (*array_schema)->set_cell_order(layout);
509   (*array_schema)->set_capacity(schema_reader.getCapacity());
510   (*array_schema)->set_allows_dups(schema_reader.getAllowsDuplicates());
511   // Pre 1.8 TileDB serialized the version as the library version
512   // This would have been a list of size 3, so only set the version
513   // if the list size is 1, meaning tiledb 1.8 or later
514   if (schema_reader.hasVersion() && schema_reader.getVersion().size() == 1) {
515     (*array_schema)->set_version(schema_reader.getVersion()[0]);
516   }
517 
518   auto domain_reader = schema_reader.getDomain();
519   tdb_unique_ptr<Domain> domain;
520   RETURN_NOT_OK(domain_from_capnp(domain_reader, &domain));
521   RETURN_NOT_OK((*array_schema)->set_domain(domain.get()));
522 
523   // Set coords filter pipelines
524   if (schema_reader.hasCoordsFilterPipeline()) {
525     auto reader = schema_reader.getCoordsFilterPipeline();
526     tdb_unique_ptr<FilterPipeline> filters;
527     RETURN_NOT_OK(filter_pipeline_from_capnp(reader, &filters));
528     RETURN_NOT_OK((*array_schema)->set_coords_filter_pipeline(filters.get()));
529   }
530 
531   // Set offsets filter pipelines
532   if (schema_reader.hasOffsetFilterPipeline()) {
533     auto reader = schema_reader.getOffsetFilterPipeline();
534     tdb_unique_ptr<FilterPipeline> filters;
535     RETURN_NOT_OK(filter_pipeline_from_capnp(reader, &filters));
536     RETURN_NOT_OK(
537         (*array_schema)->set_cell_var_offsets_filter_pipeline(filters.get()));
538   }
539 
540   // Set validity filter pipelines
541   if (schema_reader.hasValidityFilterPipeline()) {
542     auto reader = schema_reader.getValidityFilterPipeline();
543     tdb_unique_ptr<FilterPipeline> filters;
544     RETURN_NOT_OK(filter_pipeline_from_capnp(reader, &filters));
545     RETURN_NOT_OK(
546         (*array_schema)->set_cell_validity_filter_pipeline(filters.get()));
547   }
548 
549   // Set attributes
550   auto attributes_reader = schema_reader.getAttributes();
551   for (auto attr_reader : attributes_reader) {
552     tdb_unique_ptr<Attribute> attribute;
553     RETURN_NOT_OK(attribute_from_capnp(attr_reader, &attribute));
554     RETURN_NOT_OK((*array_schema)->add_attribute(attribute.get(), false));
555   }
556 
557   // Initialize
558   RETURN_NOT_OK((*array_schema)->init());
559 
560   return Status::Ok();
561 }
562 
array_schema_serialize(ArraySchema * array_schema,SerializationType serialize_type,Buffer * serialized_buffer,const bool client_side)563 Status array_schema_serialize(
564     ArraySchema* array_schema,
565     SerializationType serialize_type,
566     Buffer* serialized_buffer,
567     const bool client_side) {
568   try {
569     ::capnp::MallocMessageBuilder message;
570     capnp::ArraySchema::Builder arraySchemaBuilder =
571         message.initRoot<capnp::ArraySchema>();
572     RETURN_NOT_OK(
573         array_schema_to_capnp(array_schema, &arraySchemaBuilder, client_side));
574 
575     serialized_buffer->reset_size();
576     serialized_buffer->reset_offset();
577 
578     switch (serialize_type) {
579       case SerializationType::JSON: {
580         ::capnp::JsonCodec json;
581         kj::String capnp_json = json.encode(arraySchemaBuilder);
582         const auto json_len = capnp_json.size();
583         const char nul = '\0';
584         // size does not include needed null terminator, so add +1
585         RETURN_NOT_OK(serialized_buffer->realloc(json_len + 1));
586         RETURN_NOT_OK(serialized_buffer->write(capnp_json.cStr(), json_len));
587         RETURN_NOT_OK(serialized_buffer->write(&nul, 1));
588         break;
589       }
590       case SerializationType::CAPNP: {
591         kj::Array<::capnp::word> protomessage = messageToFlatArray(message);
592         kj::ArrayPtr<const char> message_chars = protomessage.asChars();
593         const auto nbytes = message_chars.size();
594         RETURN_NOT_OK(serialized_buffer->realloc(nbytes));
595         RETURN_NOT_OK(serialized_buffer->write(message_chars.begin(), nbytes));
596         break;
597       }
598       default: {
599         return LOG_STATUS(Status::SerializationError(
600             "Error serializing array schema; Unknown serialization type "
601             "passed"));
602       }
603     }
604 
605   } catch (kj::Exception& e) {
606     return LOG_STATUS(Status::SerializationError(
607         "Error serializing array schema; kj::Exception: " +
608         std::string(e.getDescription().cStr())));
609   } catch (std::exception& e) {
610     return LOG_STATUS(Status::SerializationError(
611         "Error serializing array schema; exception " + std::string(e.what())));
612   }
613 
614   return Status::Ok();
615 }
616 
array_schema_deserialize(ArraySchema ** array_schema,SerializationType serialize_type,const Buffer & serialized_buffer)617 Status array_schema_deserialize(
618     ArraySchema** array_schema,
619     SerializationType serialize_type,
620     const Buffer& serialized_buffer) {
621   try {
622     tdb_unique_ptr<ArraySchema> decoded_array_schema = nullptr;
623 
624     switch (serialize_type) {
625       case SerializationType::JSON: {
626         ::capnp::JsonCodec json;
627         ::capnp::MallocMessageBuilder message_builder;
628         capnp::ArraySchema::Builder array_schema_builder =
629             message_builder.initRoot<capnp::ArraySchema>();
630         json.decode(
631             kj::StringPtr(static_cast<const char*>(serialized_buffer.data())),
632             array_schema_builder);
633         capnp::ArraySchema::Reader array_schema_reader =
634             array_schema_builder.asReader();
635         RETURN_NOT_OK(array_schema_from_capnp(
636             array_schema_reader, &decoded_array_schema));
637         break;
638       }
639       case SerializationType::CAPNP: {
640         const auto mBytes =
641             reinterpret_cast<const kj::byte*>(serialized_buffer.data());
642         ::capnp::FlatArrayMessageReader reader(kj::arrayPtr(
643             reinterpret_cast<const ::capnp::word*>(mBytes),
644             serialized_buffer.size() / sizeof(::capnp::word)));
645         capnp::ArraySchema::Reader array_schema_reader =
646             reader.getRoot<capnp::ArraySchema>();
647         RETURN_NOT_OK(array_schema_from_capnp(
648             array_schema_reader, &decoded_array_schema));
649         break;
650       }
651       default: {
652         return LOG_STATUS(Status::SerializationError(
653             "Error deserializing array schema; Unknown serialization type "
654             "passed"));
655       }
656     }
657 
658     if (decoded_array_schema == nullptr)
659       return LOG_STATUS(Status::SerializationError(
660           "Error serializing array schema; deserialized schema is null"));
661 
662     *array_schema = decoded_array_schema.release();
663   } catch (kj::Exception& e) {
664     return LOG_STATUS(Status::SerializationError(
665         "Error deserializing array schema; kj::Exception: " +
666         std::string(e.getDescription().cStr())));
667   } catch (std::exception& e) {
668     return LOG_STATUS(Status::SerializationError(
669         "Error deserializing array schema; exception " +
670         std::string(e.what())));
671   }
672 
673   return Status::Ok();
674 }
675 
nonempty_domain_serialize(const Dimension * dimension,const void * nonempty_domain,bool is_empty,SerializationType serialize_type,Buffer * serialized_buffer)676 Status nonempty_domain_serialize(
677     const Dimension* dimension,
678     const void* nonempty_domain,
679     bool is_empty,
680     SerializationType serialize_type,
681     Buffer* serialized_buffer) {
682   if (!is_empty && nonempty_domain == nullptr)
683     return LOG_STATUS(Status::SerializationError(
684         "Error serializing nonempty domain; nonempty domain is null."));
685 
686   try {
687     // Serialize
688     ::capnp::MallocMessageBuilder message;
689     auto builder = message.initRoot<capnp::NonEmptyDomain>();
690     builder.setIsEmpty(is_empty);
691 
692     if (!is_empty) {
693       auto subarray_builder = builder.initNonEmptyDomain();
694       RETURN_NOT_OK(utils::serialize_coords(
695           subarray_builder, dimension, nonempty_domain));
696     }
697 
698     // Copy to buffer
699     serialized_buffer->reset_size();
700     serialized_buffer->reset_offset();
701 
702     switch (serialize_type) {
703       case SerializationType::JSON: {
704         ::capnp::JsonCodec json;
705         kj::String capnp_json = json.encode(builder);
706         const auto json_len = capnp_json.size();
707         const char nul = '\0';
708         // size does not include needed null terminator, so add +1
709         RETURN_NOT_OK(serialized_buffer->realloc(json_len + 1));
710         RETURN_NOT_OK(serialized_buffer->write(capnp_json.cStr(), json_len));
711         RETURN_NOT_OK(serialized_buffer->write(&nul, 1));
712         break;
713       }
714       case SerializationType::CAPNP: {
715         kj::Array<::capnp::word> protomessage = messageToFlatArray(message);
716         kj::ArrayPtr<const char> message_chars = protomessage.asChars();
717         const auto nbytes = message_chars.size();
718         RETURN_NOT_OK(serialized_buffer->realloc(nbytes));
719         RETURN_NOT_OK(serialized_buffer->write(message_chars.begin(), nbytes));
720         break;
721       }
722       default: {
723         return LOG_STATUS(Status::SerializationError(
724             "Error serializing nonempty domain; Unknown serialization type "
725             "passed"));
726       }
727     }
728 
729   } catch (kj::Exception& e) {
730     return LOG_STATUS(Status::SerializationError(
731         "Error serializing nonempty domain; kj::Exception: " +
732         std::string(e.getDescription().cStr())));
733   } catch (std::exception& e) {
734     return LOG_STATUS(Status::SerializationError(
735         "Error serializing nonempty domain; exception " +
736         std::string(e.what())));
737   }
738 
739   return Status::Ok();
740 }
741 
nonempty_domain_deserialize(const Dimension * dimension,const Buffer & serialized_buffer,SerializationType serialize_type,void * nonempty_domain,bool * is_empty)742 Status nonempty_domain_deserialize(
743     const Dimension* dimension,
744     const Buffer& serialized_buffer,
745     SerializationType serialize_type,
746     void* nonempty_domain,
747     bool* is_empty) {
748   if (nonempty_domain == nullptr)
749     return LOG_STATUS(Status::SerializationError(
750         "Error deserializing nonempty domain; nonempty domain is null."));
751 
752   try {
753     switch (serialize_type) {
754       case SerializationType::JSON: {
755         ::capnp::JsonCodec json;
756         ::capnp::MallocMessageBuilder message_builder;
757         auto builder = message_builder.initRoot<capnp::NonEmptyDomain>();
758         json.decode(
759             kj::StringPtr(static_cast<const char*>(serialized_buffer.data())),
760             builder);
761         auto reader = builder.asReader();
762 
763         // Deserialize
764         *is_empty = reader.getIsEmpty();
765         if (!*is_empty) {
766           void* subarray;
767           RETURN_NOT_OK(utils::deserialize_coords(
768               reader.getNonEmptyDomain(), dimension, &subarray));
769           std::memcpy(nonempty_domain, subarray, 2 * dimension->coord_size());
770           tdb_free(subarray);
771         }
772 
773         break;
774       }
775       case SerializationType::CAPNP: {
776         const auto mBytes =
777             reinterpret_cast<const kj::byte*>(serialized_buffer.data());
778         ::capnp::FlatArrayMessageReader msg_reader(kj::arrayPtr(
779             reinterpret_cast<const ::capnp::word*>(mBytes),
780             serialized_buffer.size() / sizeof(::capnp::word)));
781         auto reader = msg_reader.getRoot<capnp::NonEmptyDomain>();
782 
783         // Deserialize
784         *is_empty = reader.getIsEmpty();
785         if (!*is_empty) {
786           void* subarray;
787           RETURN_NOT_OK(utils::deserialize_coords(
788               reader.getNonEmptyDomain(), dimension, &subarray));
789           std::memcpy(nonempty_domain, subarray, 2 * dimension->coord_size());
790           tdb_free(subarray);
791         }
792 
793         break;
794       }
795       default: {
796         return LOG_STATUS(Status::SerializationError(
797             "Error deserializing nonempty domain; Unknown serialization type "
798             "passed"));
799       }
800     }
801   } catch (kj::Exception& e) {
802     return LOG_STATUS(Status::SerializationError(
803         "Error deserializing nonempty domain; kj::Exception: " +
804         std::string(e.getDescription().cStr())));
805   } catch (std::exception& e) {
806     return LOG_STATUS(Status::SerializationError(
807         "Error deserializing nonempty domain; exception " +
808         std::string(e.what())));
809   }
810 
811   return Status::Ok();
812 }
813 
nonempty_domain_serialize(const Array * array,const void * nonempty_domain,bool is_empty,SerializationType serialize_type,Buffer * serialized_buffer)814 Status nonempty_domain_serialize(
815     const Array* array,
816     const void* nonempty_domain,
817     bool is_empty,
818     SerializationType serialize_type,
819     Buffer* serialized_buffer) {
820   if (!is_empty && nonempty_domain == nullptr)
821     return LOG_STATUS(Status::SerializationError(
822         "Error serializing nonempty domain; nonempty domain is null."));
823 
824   const auto* schema = array->array_schema_latest();
825   if (schema == nullptr)
826     return LOG_STATUS(Status::SerializationError(
827         "Error serializing nonempty domain; array schema is null."));
828 
829   try {
830     // Serialize
831     ::capnp::MallocMessageBuilder message;
832     auto builder = message.initRoot<capnp::NonEmptyDomain>();
833     builder.setIsEmpty(is_empty);
834 
835     if (!is_empty) {
836       auto subarray_builder = builder.initNonEmptyDomain();
837       RETURN_NOT_OK(
838           utils::serialize_subarray(subarray_builder, schema, nonempty_domain));
839     }
840 
841     // Copy to buffer
842     serialized_buffer->reset_size();
843     serialized_buffer->reset_offset();
844 
845     switch (serialize_type) {
846       case SerializationType::JSON: {
847         ::capnp::JsonCodec json;
848         kj::String capnp_json = json.encode(builder);
849         const auto json_len = capnp_json.size();
850         const char nul = '\0';
851         // size does not include needed null terminator, so add +1
852         RETURN_NOT_OK(serialized_buffer->realloc(json_len + 1));
853         RETURN_NOT_OK(serialized_buffer->write(capnp_json.cStr(), json_len));
854         RETURN_NOT_OK(serialized_buffer->write(&nul, 1));
855         break;
856       }
857       case SerializationType::CAPNP: {
858         kj::Array<::capnp::word> protomessage = messageToFlatArray(message);
859         kj::ArrayPtr<const char> message_chars = protomessage.asChars();
860         const auto nbytes = message_chars.size();
861         RETURN_NOT_OK(serialized_buffer->realloc(nbytes));
862         RETURN_NOT_OK(serialized_buffer->write(message_chars.begin(), nbytes));
863         break;
864       }
865       default: {
866         return LOG_STATUS(Status::SerializationError(
867             "Error serializing nonempty domain; Unknown serialization type "
868             "passed"));
869       }
870     }
871 
872   } catch (kj::Exception& e) {
873     return LOG_STATUS(Status::SerializationError(
874         "Error serializing nonempty domain; kj::Exception: " +
875         std::string(e.getDescription().cStr())));
876   } catch (std::exception& e) {
877     return LOG_STATUS(Status::SerializationError(
878         "Error serializing nonempty domain; exception " +
879         std::string(e.what())));
880   }
881 
882   return Status::Ok();
883 }
884 
nonempty_domain_deserialize(const Array * array,const Buffer & serialized_buffer,SerializationType serialize_type,void * nonempty_domain,bool * is_empty)885 Status nonempty_domain_deserialize(
886     const Array* array,
887     const Buffer& serialized_buffer,
888     SerializationType serialize_type,
889     void* nonempty_domain,
890     bool* is_empty) {
891   if (nonempty_domain == nullptr)
892     return LOG_STATUS(Status::SerializationError(
893         "Error deserializing nonempty domain; nonempty domain is null."));
894 
895   const auto* schema = array->array_schema_latest();
896   if (schema == nullptr)
897     return LOG_STATUS(Status::SerializationError(
898         "Error deserializing nonempty domain; array schema is null."));
899 
900   try {
901     switch (serialize_type) {
902       case SerializationType::JSON: {
903         ::capnp::JsonCodec json;
904         ::capnp::MallocMessageBuilder message_builder;
905         auto builder = message_builder.initRoot<capnp::NonEmptyDomain>();
906         json.decode(
907             kj::StringPtr(static_cast<const char*>(serialized_buffer.data())),
908             builder);
909         auto reader = builder.asReader();
910 
911         // Deserialize
912         *is_empty = reader.getIsEmpty();
913         if (!*is_empty) {
914           void* subarray;
915           RETURN_NOT_OK(utils::deserialize_subarray(
916               reader.getNonEmptyDomain(), schema, &subarray));
917           std::memcpy(
918               nonempty_domain,
919               subarray,
920               2 * schema->dimension(0)->coord_size());
921           tdb_free(subarray);
922         }
923 
924         break;
925       }
926       case SerializationType::CAPNP: {
927         const auto mBytes =
928             reinterpret_cast<const kj::byte*>(serialized_buffer.data());
929         ::capnp::FlatArrayMessageReader msg_reader(kj::arrayPtr(
930             reinterpret_cast<const ::capnp::word*>(mBytes),
931             serialized_buffer.size() / sizeof(::capnp::word)));
932         auto reader = msg_reader.getRoot<capnp::NonEmptyDomain>();
933 
934         // Deserialize
935         *is_empty = reader.getIsEmpty();
936         if (!*is_empty) {
937           void* subarray;
938           RETURN_NOT_OK(utils::deserialize_subarray(
939               reader.getNonEmptyDomain(), schema, &subarray));
940           std::memcpy(
941               nonempty_domain,
942               subarray,
943               2 * schema->dimension(0)->coord_size());
944           tdb_free(subarray);
945         }
946 
947         break;
948       }
949       default: {
950         return LOG_STATUS(Status::SerializationError(
951             "Error deserializing nonempty domain; Unknown serialization type "
952             "passed"));
953       }
954     }
955   } catch (kj::Exception& e) {
956     return LOG_STATUS(Status::SerializationError(
957         "Error deserializing nonempty domain; kj::Exception: " +
958         std::string(e.getDescription().cStr())));
959   } catch (std::exception& e) {
960     return LOG_STATUS(Status::SerializationError(
961         "Error deserializing nonempty domain; exception " +
962         std::string(e.what())));
963   }
964 
965   return Status::Ok();
966 }
967 
nonempty_domain_serialize(Array * array,SerializationType serialize_type,Buffer * serialized_buffer)968 Status nonempty_domain_serialize(
969     Array* array, SerializationType serialize_type, Buffer* serialized_buffer) {
970   const auto* schema = array->array_schema_latest();
971   if (schema == nullptr)
972     return LOG_STATUS(Status::SerializationError(
973         "Error serializing nonempty domain; array schema is null."));
974 
975   try {
976     // Serialize
977     ::capnp::MallocMessageBuilder message;
978     auto builder = message.initRoot<capnp::NonEmptyDomainList>();
979 
980     RETURN_NOT_OK(utils::serialize_non_empty_domain(builder, array));
981 
982     // Copy to buffer
983     serialized_buffer->reset_size();
984     serialized_buffer->reset_offset();
985 
986     switch (serialize_type) {
987       case SerializationType::JSON: {
988         ::capnp::JsonCodec json;
989         kj::String capnp_json = json.encode(builder);
990         const auto json_len = capnp_json.size();
991         const char nul = '\0';
992         // size does not include needed null terminator, so add +1
993         RETURN_NOT_OK(serialized_buffer->realloc(json_len + 1));
994         RETURN_NOT_OK(serialized_buffer->write(capnp_json.cStr(), json_len));
995         RETURN_NOT_OK(serialized_buffer->write(&nul, 1));
996         break;
997       }
998       case SerializationType::CAPNP: {
999         kj::Array<::capnp::word> protomessage = messageToFlatArray(message);
1000         kj::ArrayPtr<const char> message_chars = protomessage.asChars();
1001         const auto nbytes = message_chars.size();
1002         RETURN_NOT_OK(serialized_buffer->realloc(nbytes));
1003         RETURN_NOT_OK(serialized_buffer->write(message_chars.begin(), nbytes));
1004         break;
1005       }
1006       default: {
1007         return LOG_STATUS(Status::SerializationError(
1008             "Error serializing nonempty domain; Unknown serialization type "
1009             "passed"));
1010       }
1011     }
1012 
1013   } catch (kj::Exception& e) {
1014     return LOG_STATUS(Status::SerializationError(
1015         "Error serializing nonempty domain; kj::Exception: " +
1016         std::string(e.getDescription().cStr())));
1017   } catch (std::exception& e) {
1018     return LOG_STATUS(Status::SerializationError(
1019         "Error serializing nonempty domain; exception " +
1020         std::string(e.what())));
1021   }
1022 
1023   return Status::Ok();
1024 }
1025 
nonempty_domain_deserialize(Array * array,const Buffer & serialized_buffer,SerializationType serialize_type)1026 Status nonempty_domain_deserialize(
1027     Array* array,
1028     const Buffer& serialized_buffer,
1029     SerializationType serialize_type) {
1030   try {
1031     switch (serialize_type) {
1032       case SerializationType::JSON: {
1033         ::capnp::JsonCodec json;
1034         ::capnp::MallocMessageBuilder message_builder;
1035         auto builder = message_builder.initRoot<capnp::NonEmptyDomainList>();
1036         json.decode(
1037             kj::StringPtr(static_cast<const char*>(serialized_buffer.data())),
1038             builder);
1039         auto reader = builder.asReader();
1040 
1041         // Deserialize
1042         RETURN_NOT_OK(utils::deserialize_non_empty_domain(reader, array));
1043         break;
1044       }
1045       case SerializationType::CAPNP: {
1046         const auto mBytes =
1047             reinterpret_cast<const kj::byte*>(serialized_buffer.data());
1048         ::capnp::FlatArrayMessageReader msg_reader(kj::arrayPtr(
1049             reinterpret_cast<const ::capnp::word*>(mBytes),
1050             serialized_buffer.size() / sizeof(::capnp::word)));
1051         auto reader = msg_reader.getRoot<capnp::NonEmptyDomainList>();
1052 
1053         // Deserialize
1054         RETURN_NOT_OK(utils::deserialize_non_empty_domain(reader, array));
1055         break;
1056       }
1057       default: {
1058         return LOG_STATUS(Status::SerializationError(
1059             "Error deserializing nonempty domain; Unknown serialization type "
1060             "passed"));
1061       }
1062     }
1063   } catch (kj::Exception& e) {
1064     return LOG_STATUS(Status::SerializationError(
1065         "Error deserializing nonempty domain; kj::Exception: " +
1066         std::string(e.getDescription().cStr())));
1067   } catch (std::exception& e) {
1068     return LOG_STATUS(Status::SerializationError(
1069         "Error deserializing nonempty domain; exception " +
1070         std::string(e.what())));
1071   }
1072 
1073   return Status::Ok();
1074 }
1075 
max_buffer_sizes_serialize(Array * array,const void * subarray,SerializationType serialize_type,Buffer * serialized_buffer)1076 Status max_buffer_sizes_serialize(
1077     Array* array,
1078     const void* subarray,
1079     SerializationType serialize_type,
1080     Buffer* serialized_buffer) {
1081   const auto* schema = array->array_schema_latest();
1082   if (schema == nullptr)
1083     return LOG_STATUS(Status::SerializationError(
1084         "Error serializing max buffer sizes; array schema is null."));
1085 
1086   try {
1087     // Serialize
1088     ::capnp::MallocMessageBuilder message;
1089     auto builder = message.initRoot<capnp::MaxBufferSizes>();
1090 
1091     // Get all attribute names including coords
1092     const auto& attrs = schema->attributes();
1093     std::set<std::string> attr_names;
1094     attr_names.insert(constants::coords);
1095     for (const auto* a : attrs)
1096       attr_names.insert(a->name());
1097 
1098     // Get max buffer size for each attribute from the given Array instance
1099     // and serialize it.
1100     auto max_buffer_sizes_builder =
1101         builder.initMaxBufferSizes(attr_names.size());
1102     size_t i = 0;
1103     for (const auto& attr_name : attr_names) {
1104       bool var_size =
1105           attr_name != constants::coords && schema->var_size(attr_name);
1106       auto max_buffer_size_builder = max_buffer_sizes_builder[i++];
1107       max_buffer_size_builder.setAttribute(attr_name);
1108 
1109       if (var_size) {
1110         uint64_t offset_bytes, data_bytes;
1111         RETURN_NOT_OK(array->get_max_buffer_size(
1112             attr_name.c_str(), subarray, &offset_bytes, &data_bytes));
1113         max_buffer_size_builder.setOffsetBytes(offset_bytes);
1114         max_buffer_size_builder.setDataBytes(data_bytes);
1115       } else {
1116         uint64_t data_bytes;
1117         RETURN_NOT_OK(array->get_max_buffer_size(
1118             attr_name.c_str(), subarray, &data_bytes));
1119         max_buffer_size_builder.setOffsetBytes(0);
1120         max_buffer_size_builder.setDataBytes(data_bytes);
1121       }
1122     }
1123 
1124     // Copy to buffer
1125     serialized_buffer->reset_size();
1126     serialized_buffer->reset_offset();
1127 
1128     switch (serialize_type) {
1129       case SerializationType::JSON: {
1130         ::capnp::JsonCodec json;
1131         kj::String capnp_json = json.encode(builder);
1132         const auto json_len = capnp_json.size();
1133         const char nul = '\0';
1134         // size does not include needed null terminator, so add +1
1135         RETURN_NOT_OK(serialized_buffer->realloc(json_len + 1));
1136         RETURN_NOT_OK(serialized_buffer->write(capnp_json.cStr(), json_len));
1137         RETURN_NOT_OK(serialized_buffer->write(&nul, 1));
1138         break;
1139       }
1140       case SerializationType::CAPNP: {
1141         kj::Array<::capnp::word> protomessage = messageToFlatArray(message);
1142         kj::ArrayPtr<const char> message_chars = protomessage.asChars();
1143         const auto nbytes = message_chars.size();
1144         RETURN_NOT_OK(serialized_buffer->realloc(nbytes));
1145         RETURN_NOT_OK(serialized_buffer->write(message_chars.begin(), nbytes));
1146         break;
1147       }
1148       default: {
1149         return LOG_STATUS(Status::SerializationError(
1150             "Error serializing max buffer sizes; Unknown serialization type "
1151             "passed"));
1152       }
1153     }
1154 
1155   } catch (kj::Exception& e) {
1156     return LOG_STATUS(Status::SerializationError(
1157         "Error serializing max buffer sizes; kj::Exception: " +
1158         std::string(e.getDescription().cStr())));
1159   } catch (std::exception& e) {
1160     return LOG_STATUS(Status::SerializationError(
1161         "Error serializing max buffer sizes; exception " +
1162         std::string(e.what())));
1163   }
1164 
1165   return Status::Ok();
1166 }
1167 
max_buffer_sizes_deserialize(const ArraySchema * schema,const Buffer & serialized_buffer,SerializationType serialize_type,std::unordered_map<std::string,std::pair<uint64_t,uint64_t>> * buffer_sizes)1168 Status max_buffer_sizes_deserialize(
1169     const ArraySchema* schema,
1170     const Buffer& serialized_buffer,
1171     SerializationType serialize_type,
1172     std::unordered_map<std::string, std::pair<uint64_t, uint64_t>>*
1173         buffer_sizes) {
1174   if (schema == nullptr)
1175     return LOG_STATUS(Status::SerializationError(
1176         "Error deserializing max buffer sizes; array schema is null."));
1177 
1178   try {
1179     switch (serialize_type) {
1180       case SerializationType::JSON: {
1181         ::capnp::JsonCodec json;
1182         ::capnp::MallocMessageBuilder message_builder;
1183         auto builder = message_builder.initRoot<capnp::MaxBufferSizes>();
1184         json.decode(
1185             kj::StringPtr(static_cast<const char*>(serialized_buffer.data())),
1186             builder);
1187         auto reader = builder.asReader();
1188 
1189         // Deserialize
1190         auto max_buffer_sizes_reader = reader.getMaxBufferSizes();
1191         const size_t num_max_buffer_sizes = max_buffer_sizes_reader.size();
1192         for (size_t i = 0; i < num_max_buffer_sizes; i++) {
1193           auto max_buffer_size_reader = max_buffer_sizes_reader[i];
1194           std::string attribute = max_buffer_size_reader.getAttribute();
1195           uint64_t offset_size = max_buffer_size_reader.getOffsetBytes();
1196           uint64_t data_size = max_buffer_size_reader.getDataBytes();
1197 
1198           if (attribute == constants::coords || !schema->var_size(attribute)) {
1199             (*buffer_sizes)[attribute] = std::make_pair(data_size, 0);
1200           } else {
1201             (*buffer_sizes)[attribute] = std::make_pair(offset_size, data_size);
1202           }
1203         }
1204 
1205         break;
1206       }
1207       case SerializationType::CAPNP: {
1208         const auto mBytes =
1209             reinterpret_cast<const kj::byte*>(serialized_buffer.data());
1210         ::capnp::FlatArrayMessageReader msg_reader(kj::arrayPtr(
1211             reinterpret_cast<const ::capnp::word*>(mBytes),
1212             serialized_buffer.size() / sizeof(::capnp::word)));
1213         auto reader = msg_reader.getRoot<capnp::MaxBufferSizes>();
1214 
1215         // Deserialize
1216         auto max_buffer_sizes_reader = reader.getMaxBufferSizes();
1217         const size_t num_max_buffer_sizes = max_buffer_sizes_reader.size();
1218         for (size_t i = 0; i < num_max_buffer_sizes; i++) {
1219           auto max_buffer_size_reader = max_buffer_sizes_reader[i];
1220           std::string attribute = max_buffer_size_reader.getAttribute();
1221           uint64_t offset_size = max_buffer_size_reader.getOffsetBytes();
1222           uint64_t data_size = max_buffer_size_reader.getDataBytes();
1223 
1224           if (attribute == constants::coords || !schema->var_size(attribute)) {
1225             (*buffer_sizes)[attribute] = std::make_pair(data_size, 0);
1226           } else {
1227             (*buffer_sizes)[attribute] = std::make_pair(offset_size, data_size);
1228           }
1229         }
1230 
1231         break;
1232       }
1233       default: {
1234         return LOG_STATUS(Status::SerializationError(
1235             "Error deserializing max buffer sizes; Unknown serialization type "
1236             "passed"));
1237       }
1238     }
1239   } catch (kj::Exception& e) {
1240     return LOG_STATUS(Status::SerializationError(
1241         "Error deserializing max buffer sizes; kj::Exception: " +
1242         std::string(e.getDescription().cStr())));
1243   } catch (std::exception& e) {
1244     return LOG_STATUS(Status::SerializationError(
1245         "Error deserializing max buffer sizes; exception " +
1246         std::string(e.what())));
1247   }
1248 
1249   return Status::Ok();
1250 }
1251 
array_metadata_serialize(Array * array,SerializationType serialize_type,Buffer * serialized_buffer)1252 Status array_metadata_serialize(
1253     Array* array, SerializationType serialize_type, Buffer* serialized_buffer) {
1254   if (array == nullptr)
1255     return LOG_STATUS(Status::SerializationError(
1256         "Error serializing array metadata; array instance is null"));
1257 
1258   Metadata* metadata;
1259 
1260   RETURN_NOT_OK(array->metadata(&metadata));
1261 
1262   if (metadata == nullptr)
1263     return LOG_STATUS(Status::SerializationError(
1264         "Error serializing array metadata; array metadata instance is null"));
1265 
1266   try {
1267     // Serialize
1268     ::capnp::MallocMessageBuilder message;
1269     auto builder = message.initRoot<capnp::ArrayMetadata>();
1270     auto entries_builder = builder.initEntries(metadata->num());
1271     size_t i = 0;
1272     for (auto it = metadata->begin(); it != metadata->end(); ++it) {
1273       auto entry_builder = entries_builder[i++];
1274       const auto& entry = it->second;
1275       auto datatype = static_cast<Datatype>(entry.type_);
1276       entry_builder.setKey(it->first);
1277       entry_builder.setType(datatype_str(datatype));
1278       entry_builder.setValueNum(entry.num_);
1279       entry_builder.setValue(kj::arrayPtr(
1280           static_cast<const uint8_t*>(entry.value_.data()),
1281           entry.value_.size()));
1282       entry_builder.setDel(entry.del_ == 1);
1283     }
1284 
1285     // Copy to buffer
1286     serialized_buffer->reset_size();
1287     serialized_buffer->reset_offset();
1288 
1289     switch (serialize_type) {
1290       case SerializationType::JSON: {
1291         ::capnp::JsonCodec json;
1292         kj::String capnp_json = json.encode(builder);
1293         const auto json_len = capnp_json.size();
1294         const char nul = '\0';
1295         // size does not include needed null terminator, so add +1
1296         RETURN_NOT_OK(serialized_buffer->realloc(json_len + 1));
1297         RETURN_NOT_OK(serialized_buffer->write(capnp_json.cStr(), json_len));
1298         RETURN_NOT_OK(serialized_buffer->write(&nul, 1));
1299         break;
1300       }
1301       case SerializationType::CAPNP: {
1302         kj::Array<::capnp::word> protomessage = messageToFlatArray(message);
1303         kj::ArrayPtr<const char> message_chars = protomessage.asChars();
1304         const auto nbytes = message_chars.size();
1305         RETURN_NOT_OK(serialized_buffer->realloc(nbytes));
1306         RETURN_NOT_OK(serialized_buffer->write(message_chars.begin(), nbytes));
1307         break;
1308       }
1309       default: {
1310         return LOG_STATUS(Status::SerializationError(
1311             "Error serializing array metadata; Unknown serialization type "
1312             "passed"));
1313       }
1314     }
1315 
1316   } catch (kj::Exception& e) {
1317     return LOG_STATUS(Status::SerializationError(
1318         "Error serializing array metadata; kj::Exception: " +
1319         std::string(e.getDescription().cStr())));
1320   } catch (std::exception& e) {
1321     return LOG_STATUS(Status::SerializationError(
1322         "Error serializing array metadata; exception " +
1323         std::string(e.what())));
1324   }
1325 
1326   return Status::Ok();
1327 }
1328 
array_metadata_deserialize(Array * array,SerializationType serialize_type,const Buffer & serialized_buffer)1329 Status array_metadata_deserialize(
1330     Array* array,
1331     SerializationType serialize_type,
1332     const Buffer& serialized_buffer) {
1333   if (array == nullptr)
1334     return LOG_STATUS(Status::SerializationError(
1335         "Error deserializing array metadata; null array instance given."));
1336   if (array->metadata() == nullptr)
1337     return LOG_STATUS(Status::SerializationError(
1338         "Error deserializing array metadata; null metadata instance."));
1339 
1340   Metadata* metadata = array->metadata();
1341 
1342   try {
1343     switch (serialize_type) {
1344       case SerializationType::JSON: {
1345         ::capnp::JsonCodec json;
1346         ::capnp::MallocMessageBuilder message_builder;
1347         auto builder = message_builder.initRoot<capnp::ArrayMetadata>();
1348         json.decode(
1349             kj::StringPtr(static_cast<const char*>(serialized_buffer.data())),
1350             builder);
1351         auto reader = builder.asReader();
1352 
1353         // Deserialize
1354         auto entries_reader = reader.getEntries();
1355         const size_t num_entries = entries_reader.size();
1356         for (size_t i = 0; i < num_entries; i++) {
1357           auto entry_reader = entries_reader[i];
1358           std::string key = entry_reader.getKey();
1359           Datatype type = Datatype::UINT8;
1360           RETURN_NOT_OK(datatype_enum(entry_reader.getType(), &type));
1361           uint32_t value_num = entry_reader.getValueNum();
1362 
1363           auto value_ptr = entry_reader.getValue();
1364           const void* value = (void*)value_ptr.begin();
1365           if (value_ptr.size() != datatype_size(type) * value_num)
1366             return LOG_STATUS(Status::SerializationError(
1367                 "Error deserializing array metadata; value size sanity check "
1368                 "failed."));
1369 
1370           if (entry_reader.getDel()) {
1371             RETURN_NOT_OK(metadata->del(key.c_str()));
1372           } else {
1373             RETURN_NOT_OK(metadata->put(key.c_str(), type, value_num, value));
1374           }
1375         }
1376 
1377         break;
1378       }
1379       case SerializationType::CAPNP: {
1380         const auto mBytes =
1381             reinterpret_cast<const kj::byte*>(serialized_buffer.data());
1382         ::capnp::FlatArrayMessageReader msg_reader(kj::arrayPtr(
1383             reinterpret_cast<const ::capnp::word*>(mBytes),
1384             serialized_buffer.size() / sizeof(::capnp::word)));
1385         auto reader = msg_reader.getRoot<capnp::ArrayMetadata>();
1386 
1387         // Deserialize
1388         auto entries_reader = reader.getEntries();
1389         const size_t num_entries = entries_reader.size();
1390         for (size_t i = 0; i < num_entries; i++) {
1391           auto entry_reader = entries_reader[i];
1392           std::string key = entry_reader.getKey();
1393           Datatype type = Datatype::UINT8;
1394           RETURN_NOT_OK(datatype_enum(entry_reader.getType(), &type));
1395           uint32_t value_num = entry_reader.getValueNum();
1396 
1397           auto value_ptr = entry_reader.getValue();
1398           const void* value = (void*)value_ptr.begin();
1399           if (value_ptr.size() != datatype_size(type) * value_num)
1400             return LOG_STATUS(Status::SerializationError(
1401                 "Error deserializing array metadata; value size sanity check "
1402                 "failed."));
1403 
1404           if (entry_reader.getDel()) {
1405             RETURN_NOT_OK(metadata->del(key.c_str()));
1406           } else {
1407             RETURN_NOT_OK(metadata->put(key.c_str(), type, value_num, value));
1408           }
1409         }
1410 
1411         break;
1412       }
1413       default: {
1414         return LOG_STATUS(Status::SerializationError(
1415             "Error deserializing array metadata; Unknown serialization type "
1416             "passed"));
1417       }
1418     }
1419   } catch (kj::Exception& e) {
1420     return LOG_STATUS(Status::SerializationError(
1421         "Error deserializing array metadata; kj::Exception: " +
1422         std::string(e.getDescription().cStr())));
1423   } catch (std::exception& e) {
1424     return LOG_STATUS(Status::SerializationError(
1425         "Error deserializing array metadata; exception " +
1426         std::string(e.what())));
1427   }
1428 
1429   return Status::Ok();
1430 }
1431 
1432 #else
1433 
1434 Status array_schema_serialize(
1435     ArraySchema*, SerializationType, Buffer*, const bool) {
1436   return LOG_STATUS(Status::SerializationError(
1437       "Cannot serialize; serialization not enabled."));
1438 }
1439 
1440 Status array_schema_deserialize(
1441     ArraySchema**, SerializationType, const Buffer&) {
1442   return LOG_STATUS(Status::SerializationError(
1443       "Cannot serialize; serialization not enabled."));
1444 }
1445 
1446 Status nonempty_domain_serialize(Array*, SerializationType, Buffer*) {
1447   return LOG_STATUS(Status::SerializationError(
1448       "Cannot serialize; serialization not enabled."));
1449 }
1450 
1451 Status nonempty_domain_deserialize(Array*, const Buffer&, SerializationType) {
1452   return LOG_STATUS(Status::SerializationError(
1453       "Cannot serialize; serialization not enabled."));
1454 }
1455 
1456 Status nonempty_domain_serialize(
1457     const Array*, const void*, bool, SerializationType, Buffer*) {
1458   return LOG_STATUS(Status::SerializationError(
1459       "Cannot serialize; serialization not enabled."));
1460 }
1461 
1462 Status nonempty_domain_deserialize(
1463     const Array*, const Buffer&, SerializationType, void*, bool*) {
1464   return LOG_STATUS(Status::SerializationError(
1465       "Cannot serialize; serialization not enabled."));
1466 }
1467 
1468 Status max_buffer_sizes_serialize(
1469     Array*, const void*, SerializationType, Buffer*) {
1470   return LOG_STATUS(Status::SerializationError(
1471       "Cannot serialize; serialization not enabled."));
1472 }
1473 
1474 Status max_buffer_sizes_deserialize(
1475     const ArraySchema*,
1476     const Buffer&,
1477     SerializationType,
1478     std::unordered_map<std::string, std::pair<uint64_t, uint64_t>>*) {
1479   return LOG_STATUS(Status::SerializationError(
1480       "Cannot serialize; serialization not enabled."));
1481 }
1482 
1483 Status array_metadata_serialize(Array*, SerializationType, Buffer*) {
1484   return LOG_STATUS(Status::SerializationError(
1485       "Cannot serialize; serialization not enabled."));
1486 }
1487 
1488 Status array_metadata_deserialize(Array*, SerializationType, const Buffer&) {
1489   return LOG_STATUS(Status::SerializationError(
1490       "Cannot serialize; serialization not enabled."));
1491 }
1492 
1493 #endif  // TILEDB_SERIALIZATION
1494 
1495 }  // namespace serialization
1496 }  // namespace sm
1497 }  // namespace tiledb
1498