1 /**
2 * @file constants.cc
3 *
4 * @section LICENSE
5 *
6 * The MIT License
7 *
8 * @copyright Copyright (c) 2017-2021 TileDB, Inc.
9 *
10 * Permission is hereby granted, free of charge, to any person obtaining a copy
11 * of this software and associated documentation files (the "Software"), to deal
12 * in the Software without restriction, including without limitation the rights
13 * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
14 * copies of the Software, and to permit persons to whom the Software is
15 * furnished to do so, subject to the following conditions:
16 *
17 * The above copyright notice and this permission notice shall be included in
18 * all copies or substantial portions of the Software.
19 *
20 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
21 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
22 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
23 * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
24 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
25 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
26 * THE SOFTWARE.
27 *
28 * @section DESCRIPTION
29 *
30 * This file defines the TileDB constants.
31 */
32
33 #include <cstdint>
34 #include <limits>
35 #include <thread>
36
37 #include "tiledb/sm/c_api/tiledb_version.h"
38
39 // Include files for platform path max definition.
40 #ifdef _WIN32
41 #include "tiledb/sm/misc/win_constants.h"
42 #elif __APPLE__
43 #include <sys/syslimits.h>
44 #else
45 #include <limits.h>
46 #endif
47
48 #include "tiledb/sm/enums/compressor.h"
49 #include "tiledb/sm/enums/datatype.h"
50 #include "tiledb/sm/enums/serialization_type.h"
51 #include "tiledb/sm/misc/utils.h"
52
53 namespace tiledb {
54 namespace sm {
55
56 namespace constants {
57
58 /**
59 * Reduction factor (must be in [0.0, 1.0]) for the multi_range subarray
60 * split by the partitioner. If the number is equal to 0.3, then this
61 * means that the number of ranges will be reduced by 30%.
62 */
63 const double multi_range_reduction_in_split = 0.3;
64
65 /** Amplification factor for the result size estimation. */
66 const double est_result_size_amplification = 1.0;
67
68 /** Default fanout for RTrees. */
69 const unsigned rtree_fanout = 10;
70
71 /** The array schema file name. */
72 const std::string array_schema_filename = "__array_schema.tdb";
73
74 /** The array schema folder name. */
75 const std::string array_schema_folder_name = "__schema";
76
77 /** The array metadata folder name. */
78 const std::string array_metadata_folder_name = "__meta";
79
80 /** The fragment metadata file name. */
81 const std::string fragment_metadata_filename = "__fragment_metadata.tdb";
82
83 /** The default tile capacity. */
84 const uint64_t capacity = 10000;
85
86 /** The size of a variable cell offset. */
87 const uint64_t cell_var_offset_size = sizeof(uint64_t);
88
89 /** The type of a variable cell offset. */
90 const Datatype cell_var_offset_type = Datatype::UINT64;
91
92 /** The size of a validity cell. */
93 const uint64_t cell_validity_size = sizeof(uint8_t);
94
95 /** The type of a validity cell. */
96 const Datatype cell_validity_type = Datatype::UINT8;
97
98 /** A special value indicating variable size. */
99 const uint64_t var_size = std::numeric_limits<uint64_t>::max();
100
101 /** The default compressor for the offsets of variable-sized cells. */
102 Compressor cell_var_offsets_compression = Compressor::ZSTD;
103
104 /** The default compression level for the offsets of variable-sized cells. */
105 int cell_var_offsets_compression_level = -1;
106
107 /** The default compressor for the validity value cells. */
108 Compressor cell_validity_compression = Compressor::RLE;
109
110 /** The default compression level for the validity value cells. */
111 int cell_validity_compression_level = -1;
112
113 /** Special name reserved for the coordinates attribute. */
114 const std::string coords = "__coords";
115
116 /** The default compressor for the coordinates. */
117 Compressor coords_compression = Compressor::ZSTD;
118
119 /** The default compressor for real coordinates. */
120 Compressor real_coords_compression = Compressor::ZSTD;
121
122 /** The default compression level for the coordinates. */
123 int coords_compression_level = -1;
124
125 /** The filelock name. */
126 const std::string filelock_name = "__lock.tdb";
127
128 /** The special value for an empty int32. */
129 const int empty_int32 = std::numeric_limits<int32_t>::min();
130
131 /** The special value for an empty int64. */
132 const int64_t empty_int64 = std::numeric_limits<int64_t>::min();
133
134 /** The special value for an empty float32. */
135 const float empty_float32 = std::numeric_limits<float>::quiet_NaN();
136
137 /** The special value for an empty float64. */
138 const double empty_float64 = std::numeric_limits<double>::quiet_NaN();
139
140 /** The special value for an empty char. */
141 const char empty_char = std::numeric_limits<char>::min();
142
143 /** The special value for an empty int8. */
144 const int8_t empty_int8 = std::numeric_limits<int8_t>::min();
145
146 /** The special value for an empty uint8. */
147 const uint8_t empty_uint8 = std::numeric_limits<uint8_t>::max();
148
149 /** The special value for an empty int16. */
150 const int16_t empty_int16 = std::numeric_limits<int16_t>::min();
151
152 /** The special value for an empty uint16. */
153 const uint16_t empty_uint16 = std::numeric_limits<uint16_t>::max();
154
155 /** The special value for an empty uint32. */
156 const uint32_t empty_uint32 = std::numeric_limits<uint32_t>::max();
157
158 /** The special value for an empty uint64. */
159 const uint64_t empty_uint64 = std::numeric_limits<uint64_t>::max();
160
161 /** The special value for an empty ASCII. */
162 const uint8_t empty_ascii = 0;
163
164 /** The special value for an empty UTF8. */
165 const uint8_t empty_utf8 = 0;
166
167 /** The special value for an empty UTF16. */
168 const uint16_t empty_utf16 = 0;
169
170 /** The special value for an empty UTF32. */
171 const uint32_t empty_utf32 = 0;
172
173 /** The special value for an empty UCS2. */
174 const uint16_t empty_ucs2 = 0;
175
176 /** The special value for an empty UCS4. */
177 const uint32_t empty_ucs4 = 0;
178
179 /** The special value for an empty ANY. */
180 const uint8_t empty_any = 0;
181
182 /** The file suffix used in TileDB. */
183 const std::string file_suffix = ".tdb";
184
185 /** Suffix for the special vacuum files used in TileDB. */
186 const std::string vacuum_file_suffix = ".vac";
187
188 /** Suffix for the special ok files used in TileDB. */
189 const std::string ok_file_suffix = ".ok";
190
191 /** Suffix for the special metadata files used in TileDB. */
192 const std::string meta_file_suffix = ".meta";
193
194 /** Default datatype for a generic tile. */
195 const Datatype generic_tile_datatype = Datatype::CHAR;
196
197 /** Default compressor for a generic tile. */
198 Compressor generic_tile_compressor = Compressor::GZIP;
199
200 /** Default compression level for a generic tile. */
201 int generic_tile_compression_level = 1;
202
203 /** Default cell size for a generic tile. */
204 uint64_t generic_tile_cell_size = sizeof(char);
205
206 /** The group file name. */
207 const std::string group_filename = "__tiledb_group.tdb";
208
209 /** The maximum number of bytes written in a single I/O. */
210 const uint64_t max_write_bytes = std::numeric_limits<int>::max();
211
212 /** The maximum file path length (depending on platform). */
213 #ifndef _WIN32
214 const uint32_t path_max_len = PATH_MAX;
215 #endif
216
217 /** Special value indicating a variable number of elements. */
218 const uint32_t var_num = std::numeric_limits<unsigned int>::max();
219
220 /** String describing no compression. */
221 const std::string no_compression_str = "NO_COMPRESSION";
222
223 /** Empty String **/
224 const std::string empty_str = "";
225
226 /** TILEDB_READ Query String **/
227 const std::string query_type_read_str = "READ";
228
229 /** TILEDB_WRITE Query String **/
230 const std::string query_type_write_str = "WRITE";
231
232 /** TILEDB_FAILED Query String **/
233 const std::string query_status_failed_str = "FAILED";
234
235 /** TILEDB_COMPLETED Query String **/
236 const std::string query_status_completed_str = "COMPLETED";
237
238 /** TILEDB_INPROGRESS Query String **/
239 const std::string query_status_inprogress_str = "INPROGRESS";
240
241 /** TILEDB_INCOMPLETE Query String **/
242 const std::string query_status_incomplete_str = "INCOMPLETE";
243
244 /** TILEDB_UNINITIALIZED Query String **/
245 const std::string query_status_uninitialized_str = "UNINITIALIZED";
246
247 /** TILEDB_LT Query Condition Op String **/
248 const std::string query_condition_op_lt_str = "LT";
249
250 /** TILEDB_LE Query Condition Op String **/
251 const std::string query_condition_op_le_str = "LE";
252
253 /** TILEDB_GT Query Condition Op String **/
254 const std::string query_condition_op_gt_str = "GT";
255
256 /** TILEDB_GE Query Condition Op String **/
257 const std::string query_condition_op_ge_str = "GE";
258
259 /** TILEDB_EQ Query Condition Op String **/
260 const std::string query_condition_op_eq_str = "EQ";
261
262 /** TILEDB_NE Query Condition Op String **/
263 const std::string query_condition_op_ne_str = "NE";
264
265 /** TILEDB_AND Query Condition Combination Op String **/
266 const std::string query_condition_combination_op_and_str = "AND";
267
268 /** TILEDB_OR Query Condition Combination Op String **/
269 const std::string query_condition_combination_op_or_str = "OR";
270
271 /** TILEDB_NOT Query Condition Combination Op String **/
272 const std::string query_condition_combination_op_not_str = "NOT";
273
274 /** TILEDB_COMPRESSION Filter type string */
275 const std::string filter_type_compression_str = "COMPRESSION";
276
277 /** String describing no encryption. */
278 const std::string no_encryption_str = "NO_ENCRYPTION";
279
280 /** String describing AES_256_GCM. */
281 const std::string aes_256_gcm_str = "AES_256_GCM";
282
283 /** TILEDB_JSON **/
284 const std::string serialization_type_json_str = "JSON";
285
286 /** TILEDB_CAPNP **/
287 const std::string serialization_type_capnp_str = "CAPNP";
288
289 /** String describing GZIP. */
290 const std::string gzip_str = "GZIP";
291
292 /** String describing ZSTD. */
293 const std::string zstd_str = "ZSTD";
294
295 /** String describing LZ4. */
296 const std::string lz4_str = "LZ4";
297
298 /** String describing RLE. */
299 const std::string rle_str = "RLE";
300
301 /** String describing BZIP2. */
302 const std::string bzip2_str = "BZIP2";
303
304 /** String describing DOUBLE_DELTA. */
305 const std::string double_delta_str = "DOUBLE_DELTA";
306
307 /** String describing FILTER_NONE. */
308 const std::string filter_none_str = "NONE";
309
310 /** String describing FILTER_BIT_WIDTH_REDUCTION. */
311 const std::string filter_bit_width_reduction_str = "BIT_WIDTH_REDUCTION";
312
313 /** String describing FILTER_BITSHUFFLE. */
314 const std::string filter_bitshuffle_str = "BITSHUFFLE";
315
316 /** String describing FILTER_BYTESHUFFLE. */
317 const std::string filter_byteshuffle_str = "BYTESHUFFLE";
318
319 /** String describing FILTER_POSITIVE_DELTA. */
320 const std::string filter_positive_delta_str = "POSITIVE_DELTA";
321
322 /** String describing FILTER_CHECKSUM_MD5. */
323 const std::string filter_checksum_md5_str = "CHECKSUM_MD5";
324
325 /** String describing FILTER_CHECKSUM_SHA256. */
326 const std::string filter_checksum_sha256_str = "CHECKSUM_SHA256";
327
328 /** The string representation for FilterOption type compression_level. */
329 const std::string filter_option_compression_level_str = "COMPRESSION_LEVEL";
330
331 /** The string representation for FilterOption type bit_width_max_window. */
332 const std::string filter_option_bit_width_max_window_str =
333 "BIT_WIDTH_MAX_WINDOW";
334
335 /** The string representation for FilterOption type positive_delta_max_window.
336 */
337 const std::string filter_option_positive_delta_max_window_str =
338 "POSITIVE_DELTA_MAX_WINDOW";
339
340 /** The string representation for type int32. */
341 const std::string int32_str = "INT32";
342
343 /** The string representation for type int64. */
344 const std::string int64_str = "INT64";
345
346 /** The string representation for type float32. */
347 const std::string float32_str = "FLOAT32";
348
349 /** The string representation for type float64. */
350 const std::string float64_str = "FLOAT64";
351
352 /** The string representation for type char. */
353 const std::string char_str = "CHAR";
354
355 /** The string representation for type int8. */
356 const std::string int8_str = "INT8";
357
358 /** The string representation for type uint8. */
359 const std::string uint8_str = "UINT8";
360
361 /** The string representation for type int16. */
362 const std::string int16_str = "INT16";
363
364 /** The string representation for type uint16. */
365 const std::string uint16_str = "UINT16";
366
367 /** The string representation for type uint32. */
368 const std::string uint32_str = "UINT32";
369
370 /** The string representation for type uint64. */
371 const std::string uint64_str = "UINT64";
372
373 /** The string representation for type STRING_ASCII. */
374 const std::string string_ascii_str = "STRING_ASCII";
375
376 /** The string representation for type STRING_UTF8. */
377 const std::string string_utf8_str = "STRING_UTF8";
378
379 /** The string representation for type STRING_UTF16. */
380 const std::string string_utf16_str = "STRING_UTF16";
381
382 /** The string representation for type STRING_UTF32. */
383 const std::string string_utf32_str = "STRING_UTF32";
384
385 /** The string representation for type STRING_UCS2. */
386 const std::string string_ucs2_str = "STRING_UCS2";
387
388 /** The string representation for type STRING_UCS4. */
389 const std::string string_ucs4_str = "STRING_UCS4";
390
391 /** The string representation for type ANY. */
392 const std::string any_str = "ANY";
393
394 /** The string representation for type DATETIME_YEAR. */
395 const std::string datetime_year_str = "DATETIME_YEAR";
396
397 /** The string representation for type DATETIME_MONTH. */
398 const std::string datetime_month_str = "DATETIME_MONTH";
399
400 /** The string representation for type DATETIME_WEEK. */
401 const std::string datetime_week_str = "DATETIME_WEEK";
402
403 /** The string representation for type DATETIME_DAY. */
404 const std::string datetime_day_str = "DATETIME_DAY";
405
406 /** The string representation for type DATETIME_HR. */
407 const std::string datetime_hr_str = "DATETIME_HR";
408
409 /** The string representation for type DATETIME_MIN. */
410 const std::string datetime_min_str = "DATETIME_MIN";
411
412 /** The string representation for type DATETIME_SEC. */
413 const std::string datetime_sec_str = "DATETIME_SEC";
414
415 /** The string representation for type DATETIME_MS. */
416 const std::string datetime_ms_str = "DATETIME_MS";
417
418 /** The string representation for type DATETIME_US. */
419 const std::string datetime_us_str = "DATETIME_US";
420
421 /** The string representation for type DATETIME_NS. */
422 const std::string datetime_ns_str = "DATETIME_NS";
423
424 /** The string representation for type DATETIME_PS. */
425 const std::string datetime_ps_str = "DATETIME_PS";
426
427 /** The string representation for type DATETIME_FS. */
428 const std::string datetime_fs_str = "DATETIME_FS";
429
430 /** The string representation for type DATETIME_AS. */
431 const std::string datetime_as_str = "DATETIME_AS";
432
433 /** The string representation for type TIME_HR. */
434 const std::string time_hr_str = "TIME_HR";
435
436 /** The string representation for type TIME_MIN. */
437 const std::string time_min_str = "TIME_MIN";
438
439 /** The string representation for type TIME_SEC. */
440 const std::string time_sec_str = "TIME_SEC";
441
442 /** The string representation for type TIME_MS. */
443 const std::string time_ms_str = "TIME_MS";
444
445 /** The string representation for type TIME_US. */
446 const std::string time_us_str = "TIME_US";
447
448 /** The string representation for type TIME_NS. */
449 const std::string time_ns_str = "TIME_NS";
450
451 /** The string representation for type TIME_PS. */
452 const std::string time_ps_str = "TIME_PS";
453
454 /** The string representation for type TIME_FS. */
455 const std::string time_fs_str = "TIME_FS";
456
457 /** The string representation for type TIME_AS. */
458 const std::string time_as_str = "TIME_AS";
459
460 /** The string representation for the dense array type. */
461 const std::string dense_str = "dense";
462
463 /** The string representation for the sparse array type. */
464 const std::string sparse_str = "sparse";
465
466 /** The string representation for the column-major layout. */
467 const std::string col_major_str = "col-major";
468
469 /** The string representation for the row-major layout. */
470 const std::string row_major_str = "row-major";
471
472 /** The string representation for the global order layout. */
473 const std::string global_order_str = "global-order";
474
475 /** The string representation for the unordered layout. */
476 const std::string unordered_str = "unordered";
477
478 /** The string representation for the Hilbert layout. */
479 const std::string hilbert_str = "hilbert";
480
481 /** The string representation of null. */
482 const std::string null_str = "null";
483
484 /** The string representation for object type invalid. */
485 const std::string object_type_invalid_str = "INVALID";
486
487 /** The string representation for object type group. */
488 const std::string object_type_group_str = "GROUP";
489
490 /** The string representation for object type array. */
491 const std::string object_type_array_str = "ARRAY";
492
493 /** The string representation for filesystem type hdfs. */
494 const std::string filesystem_type_hdfs_str = "HDFS";
495
496 /** The string representation for filesystem type s3. */
497 const std::string filesystem_type_s3_str = "S3";
498
499 /** The string representation for filesystem type Azure. */
500 const std::string filesystem_type_azure_str = "AZURE";
501
502 /** The string representation for filesystem type GCS. */
503 const std::string filesystem_type_gcs_str = "GCS";
504
505 /** The string representation for in-memory filesystem */
506 const std::string filesystem_type_mem_str = "MEM";
507
508 /** The string representation for WalkOrder preorder. */
509 const std::string walkorder_preorder_str = "PREORDER";
510
511 /** The string representation for WalkOrder postorder. */
512 const std::string walkorder_postorder_str = "POSTORDER";
513
514 /** The string representation for VFSMode read. */
515 const std::string vfsmode_read_str = "VFS_READ";
516
517 /** The string representation for VFSMode write. */
518 const std::string vfsmode_write_str = "VFS_WRITE";
519
520 /** The string representation for VFSMode append. */
521 const std::string vfsmode_append_str = "VFS_APPEND";
522
523 /** The version in format { major, minor, revision }. */
524 const int32_t library_version[3] = {
525 TILEDB_VERSION_MAJOR, TILEDB_VERSION_MINOR, TILEDB_VERSION_PATCH};
526
527 /** The TileDB serialization format version number. */
528 const uint32_t format_version = 10;
529
530 /** The lowest version supported for back compat writes. */
531 const uint32_t back_compat_writes_min_format_version = 7;
532
533 /** The maximum size of a tile chunk (unit of compression) in bytes. */
534 const uint64_t max_tile_chunk_size = 64 * 1024;
535
536 /**
537 * Maximum splitting depth on a string dimension. For a maximum depth
538 * of D, the maximum number of splits is 2^D.
539 */
540 const uint64_t max_string_dim_split_depth = 7;
541
542 /** Maximum number of attempts to wait for an S3 response. */
543 const unsigned int s3_max_attempts = 100;
544
545 /** Milliseconds of wait time between S3 attempts. */
546 const unsigned int s3_attempt_sleep_ms = 100;
547
548 /** Maximum number of attempts to wait for an Azure response. */
549 const unsigned int azure_max_attempts = 10;
550
551 /** Milliseconds of wait time between Azure attempts. */
552 const unsigned int azure_attempt_sleep_ms = 1000;
553
554 /** Maximum number of attempts to wait for a GCS response. */
555 const unsigned int gcs_max_attempts = 100;
556
557 /** Milliseconds of wait time between GCS attempts. */
558 const unsigned int gcs_attempt_sleep_ms = 1000;
559
560 /** An allocation tag used for logging. */
561 const std::string s3_allocation_tag = "TileDB";
562
563 /** Prefix indicating a special name reserved by TileDB. */
564 const std::string special_name_prefix = "__";
565
566 /** Number of milliseconds between watchdog thread wakeups. */
567 const unsigned watchdog_thread_sleep_ms = 1000;
568
569 /** Maximum number of concurrent attribute reads. */
570 const unsigned concurrent_attr_reads = 2;
571
572 /** The redirection header key in REST response. */
573 extern const std::string redirection_header_key = "location";
574
fill_value(Datatype type)575 const void* fill_value(Datatype type) {
576 switch (type) {
577 case Datatype::INT8:
578 return &constants::empty_int8;
579 case Datatype::UINT8:
580 return &constants::empty_uint8;
581 case Datatype::INT16:
582 return &constants::empty_int16;
583 case Datatype::UINT16:
584 return &constants::empty_uint16;
585 case Datatype::INT32:
586 return &constants::empty_int32;
587 case Datatype::UINT32:
588 return &constants::empty_uint32;
589 case Datatype::INT64:
590 return &constants::empty_int64;
591 case Datatype::UINT64:
592 return &constants::empty_uint64;
593 case Datatype::FLOAT32:
594 return &constants::empty_float32;
595 case Datatype::FLOAT64:
596 return &constants::empty_float64;
597 case Datatype::CHAR:
598 return &constants::empty_char;
599 case Datatype::ANY:
600 return &constants::empty_any;
601 case Datatype::STRING_ASCII:
602 return &constants::empty_ascii;
603 case Datatype::STRING_UTF8:
604 return &constants::empty_utf8;
605 case Datatype::STRING_UTF16:
606 return &constants::empty_utf16;
607 case Datatype::STRING_UTF32:
608 return &constants::empty_utf32;
609 case Datatype::STRING_UCS2:
610 return &constants::empty_ucs2;
611 case Datatype::STRING_UCS4:
612 return &constants::empty_ucs4;
613 case Datatype::DATETIME_YEAR:
614 case Datatype::DATETIME_MONTH:
615 case Datatype::DATETIME_WEEK:
616 case Datatype::DATETIME_DAY:
617 case Datatype::DATETIME_HR:
618 case Datatype::DATETIME_MIN:
619 case Datatype::DATETIME_SEC:
620 case Datatype::DATETIME_MS:
621 case Datatype::DATETIME_US:
622 case Datatype::DATETIME_NS:
623 case Datatype::DATETIME_PS:
624 case Datatype::DATETIME_FS:
625 case Datatype::DATETIME_AS:
626 case Datatype::TIME_HR:
627 case Datatype::TIME_MIN:
628 case Datatype::TIME_SEC:
629 case Datatype::TIME_MS:
630 case Datatype::TIME_US:
631 case Datatype::TIME_NS:
632 case Datatype::TIME_PS:
633 case Datatype::TIME_FS:
634 case Datatype::TIME_AS:
635 return &constants::empty_int64;
636 }
637
638 return nullptr;
639 }
640
641 #ifdef __linux__
642 /** Possible certificate files; stop after finding one.
643 * inspired by
644 * https://github.com/golang/go/blob/f0e940ebc985661f54d31c8d9ba31a553b87041b/src/crypto/x509/root_linux.go
645 */
646 const std::array<std::string, 6> cert_files_linux = {
647 "/etc/ssl/certs/ca-certificates.crt", // Debian/Ubuntu/Gentoo etc.
648 "/etc/pki/tls/certs/ca-bundle.crt", // Fedora/RHEL 6
649 "/etc/ssl/ca-bundle.pem", // OpenSUSE
650 "/etc/pki/tls/cacert.pem", // OpenELEC
651 "/etc/pki/ca-trust/extracted/pem/tls-ca-bundle.pem", // CentOS/RHEL 7
652 "/etc/ssl/cert.pem" // Alpine Linux
653 };
654 #endif
655
656 const std::string config_delimiter = ",";
657 } // namespace constants
658
659 } // namespace sm
660 } // namespace tiledb
661