1 /**
2 * @file fragment_info.cc
3 *
4 * @section LICENSE
5 *
6 * The MIT License
7 *
8 * @copyright Copyright (c) 2020-2021 TileDB, Inc.
9 *
10 * Permission is hereby granted, free of charge, to any person obtaining a copy
11 * of this software and associated documentation files (the "Software"), to deal
12 * in the Software without restriction, including without limitation the rights
13 * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
14 * copies of the Software, and to permit persons to whom the Software is
15 * furnished to do so, subject to the following conditions:
16 *
17 * The above copyright notice and this permission notice shall be included in
18 * all copies or substantial portions of the Software.
19 *
20 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
21 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
22 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
23 * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
24 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
25 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
26 * THE SOFTWARE.
27 *
28 * @section DESCRIPTION
29 *
30 * This file implements the FragmentInfo class.
31 */
32
33 #include "tiledb/sm/fragment/fragment_info.h"
34 #include "tiledb/common/logger.h"
35 #include "tiledb/sm/array/array.h"
36 #include "tiledb/sm/enums/encryption_type.h"
37 #include "tiledb/sm/global_state/unit_test_config.h"
38 #include "tiledb/sm/misc/utils.h"
39
40 using namespace tiledb::sm;
41 using namespace tiledb::common;
42
43 /* ****************************** */
44 /* CONSTRUCTORS & DESTRUCTORS */
45 /* ****************************** */
46
FragmentInfo()47 FragmentInfo::FragmentInfo()
48 : storage_manager_(nullptr)
49 , array_(nullptr)
50 , unconsolidated_metadata_num_(0) {
51 }
52
FragmentInfo(const URI & array_uri,StorageManager * storage_manager)53 FragmentInfo::FragmentInfo(
54 const URI& array_uri, StorageManager* storage_manager)
55 : array_uri_(array_uri)
56 , storage_manager_(storage_manager)
57 , array_(tdb_new(Array, array_uri_, storage_manager_))
58 , unconsolidated_metadata_num_(0) {
59 }
60
~FragmentInfo()61 FragmentInfo::~FragmentInfo() {
62 if (array_ != nullptr) {
63 if (array_->is_open())
64 array_->close();
65 tdb_delete(array_);
66 }
67 }
68
FragmentInfo(const FragmentInfo & fragment_info)69 FragmentInfo::FragmentInfo(const FragmentInfo& fragment_info)
70 : FragmentInfo() {
71 auto clone = fragment_info.clone();
72 swap(clone);
73 }
74
FragmentInfo(FragmentInfo && fragment_info)75 FragmentInfo::FragmentInfo(FragmentInfo&& fragment_info)
76 : FragmentInfo() {
77 swap(fragment_info);
78 }
79
operator =(const FragmentInfo & fragment_info)80 FragmentInfo& FragmentInfo::operator=(const FragmentInfo& fragment_info) {
81 auto clone = fragment_info.clone();
82 swap(clone);
83 return *this;
84 }
85
operator =(FragmentInfo && fragment_info)86 FragmentInfo& FragmentInfo::operator=(FragmentInfo&& fragment_info) {
87 swap(fragment_info);
88 return *this;
89 }
90
91 /* ********************************* */
92 /* API */
93 /* ********************************* */
94
append(const SingleFragmentInfo & fragment)95 void FragmentInfo::append(const SingleFragmentInfo& fragment) {
96 fragments_.emplace_back(fragment);
97 }
98
expand_anterior_ndrange(const Domain * domain,const NDRange & range)99 void FragmentInfo::expand_anterior_ndrange(
100 const Domain* domain, const NDRange& range) {
101 domain->expand_ndrange(range, &anterior_ndrange_);
102 }
103
clear()104 void FragmentInfo::clear() {
105 fragments_.clear();
106 anterior_ndrange_.clear();
107 }
108
dump(FILE * out) const109 void FragmentInfo::dump(FILE* out) const {
110 if (out == nullptr)
111 out = stdout;
112
113 std::stringstream ss;
114 ss << "- Fragment num: " << fragments_.size() << "\n";
115 ss << "- Unconsolidated metadata num: " << unconsolidated_metadata_num_
116 << "\n";
117 ss << "- To vacuum num: " << to_vacuum_.size() << "\n";
118
119 if (!to_vacuum_.empty()) {
120 ss << "- To vacuum URIs:\n";
121 for (const auto& v : to_vacuum_)
122 ss << " > " << v.c_str() << "\n";
123 }
124
125 fprintf(out, "%s", ss.str().c_str());
126
127 for (uint32_t i = 0; i < (uint32_t)fragments_.size(); ++i) {
128 fprintf(out, "- Fragment #%u:\n", i + 1);
129 fragments_[i].dump(dim_types_, out);
130 }
131 }
132
get_dense(uint32_t fid,int32_t * dense) const133 Status FragmentInfo::get_dense(uint32_t fid, int32_t* dense) const {
134 if (dense == nullptr)
135 return LOG_STATUS(Status::FragmentInfoError(
136 "Cannot check if fragment is dense; Dense argument cannot be null"));
137
138 if (fid >= fragments_.size())
139 return LOG_STATUS(Status::FragmentInfoError(
140 "Cannot check if fragment is dense; Invalid fragment index"));
141
142 *dense = (int32_t)!fragments_[fid].sparse();
143
144 return Status::Ok();
145 }
146
get_sparse(uint32_t fid,int32_t * sparse) const147 Status FragmentInfo::get_sparse(uint32_t fid, int32_t* sparse) const {
148 if (sparse == nullptr)
149 return LOG_STATUS(Status::FragmentInfoError(
150 "Cannot check if fragment is sparse; Sparse argument cannot be null"));
151
152 if (fid >= fragments_.size())
153 return LOG_STATUS(Status::FragmentInfoError(
154 "Cannot check if fragment is sparse; Invalid fragment index"));
155
156 *sparse = (int32_t)fragments_[fid].sparse();
157
158 return Status::Ok();
159 }
160
fragment_num() const161 uint32_t FragmentInfo::fragment_num() const {
162 return (uint32_t)fragments_.size();
163 }
164
get_cell_num(uint32_t fid,uint64_t * cell_num) const165 Status FragmentInfo::get_cell_num(uint32_t fid, uint64_t* cell_num) const {
166 if (cell_num == nullptr)
167 return LOG_STATUS(Status::FragmentInfoError(
168 "Cannot get fragment URI; Cell number argument cannot be null"));
169
170 if (fid >= fragments_.size())
171 return LOG_STATUS(Status::FragmentInfoError(
172 "Cannot get fragment URI; Invalid fragment index"));
173
174 *cell_num = fragments_[fid].cell_num();
175
176 return Status::Ok();
177 }
178
get_fragment_size(uint32_t fid,uint64_t * size) const179 Status FragmentInfo::get_fragment_size(uint32_t fid, uint64_t* size) const {
180 if (size == nullptr)
181 return LOG_STATUS(Status::FragmentInfoError(
182 "Cannot get fragment URI; Size argument cannot be null"));
183
184 if (fid >= fragments_.size())
185 return LOG_STATUS(Status::FragmentInfoError(
186 "Cannot get fragment URI; Invalid fragment index"));
187
188 *size = fragments_[fid].fragment_size();
189
190 return Status::Ok();
191 }
192
get_fragment_uri(uint32_t fid,const char ** uri) const193 Status FragmentInfo::get_fragment_uri(uint32_t fid, const char** uri) const {
194 if (uri == nullptr)
195 return LOG_STATUS(Status::FragmentInfoError(
196 "Cannot get fragment URI; URI argument cannot be null"));
197
198 if (fid >= fragments_.size())
199 return LOG_STATUS(Status::FragmentInfoError(
200 "Cannot get fragment URI; Invalid fragment index"));
201
202 *uri = fragments_[fid].uri().c_str();
203
204 return Status::Ok();
205 }
206
get_to_vacuum_uri(uint32_t fid,const char ** uri) const207 Status FragmentInfo::get_to_vacuum_uri(uint32_t fid, const char** uri) const {
208 if (uri == nullptr)
209 return LOG_STATUS(Status::FragmentInfoError(
210 "Cannot get URI of fragment to vacuum; URI argument cannot be null"));
211
212 if (fid >= to_vacuum_.size())
213 return LOG_STATUS(Status::FragmentInfoError(
214 "Cannot get URI of fragment to vacuum; Invalid fragment index"));
215
216 *uri = to_vacuum_[fid].c_str();
217
218 return Status::Ok();
219 }
220
get_timestamp_range(uint32_t fid,uint64_t * start,uint64_t * end) const221 Status FragmentInfo::get_timestamp_range(
222 uint32_t fid, uint64_t* start, uint64_t* end) const {
223 if (start == nullptr)
224 return LOG_STATUS(Status::FragmentInfoError(
225 "Cannot get timestamp range; Start argument cannot be null"));
226
227 if (end == nullptr)
228 return LOG_STATUS(Status::FragmentInfoError(
229 "Cannot get timestamp range; End argument cannot be null"));
230
231 if (fid >= fragments_.size())
232 return LOG_STATUS(Status::FragmentInfoError(
233 "Cannot get fragment URI; Invalid fragment index"));
234
235 auto range = fragments_[fid].timestamp_range();
236 *start = range.first;
237 *end = range.second;
238
239 return Status::Ok();
240 }
241
get_non_empty_domain(uint32_t fid,uint32_t did,void * domain) const242 Status FragmentInfo::get_non_empty_domain(
243 uint32_t fid, uint32_t did, void* domain) const {
244 if (domain == nullptr)
245 return LOG_STATUS(Status::FragmentInfoError(
246 "Cannot get non-empty domain; Domain argument cannot be null"));
247
248 if (fid >= fragments_.size())
249 return LOG_STATUS(Status::FragmentInfoError(
250 "Cannot get non-empty domain; Invalid fragment index"));
251
252 const auto& non_empty_domain = fragments_[fid].non_empty_domain();
253
254 if (did >= non_empty_domain.size())
255 return LOG_STATUS(Status::FragmentInfoError(
256 "Cannot get non-empty domain; Invalid dimension index"));
257
258 if (non_empty_domain[did].var_size())
259 return LOG_STATUS(Status::FragmentInfoError(
260 "Cannot get non-empty domain; Dimension is variable-sized"));
261
262 assert(!non_empty_domain[did].empty());
263 std::memcpy(
264 domain, non_empty_domain[did].data(), non_empty_domain[did].size());
265
266 return Status::Ok();
267 }
268
get_non_empty_domain(uint32_t fid,const char * dim_name,void * domain) const269 Status FragmentInfo::get_non_empty_domain(
270 uint32_t fid, const char* dim_name, void* domain) const {
271 if (dim_name == nullptr)
272 return LOG_STATUS(Status::FragmentInfoError(
273 "Cannot get non-empty domain; Dimension name argument cannot be null"));
274
275 uint32_t did;
276 for (did = 0; did < dim_names_.size(); ++did) {
277 if (dim_name == dim_names_[did]) {
278 break;
279 }
280 }
281
282 // Dimension name not found
283 if (did == dim_names_.size()) {
284 auto msg =
285 std::string("Cannot get non-empty domain; Invalid dimension name '") +
286 dim_name + "'";
287 return LOG_STATUS(Status::FragmentInfoError(msg));
288 }
289
290 return get_non_empty_domain(fid, did, domain);
291 }
292
get_non_empty_domain_var_size(uint32_t fid,uint32_t did,uint64_t * start_size,uint64_t * end_size) const293 Status FragmentInfo::get_non_empty_domain_var_size(
294 uint32_t fid,
295 uint32_t did,
296 uint64_t* start_size,
297 uint64_t* end_size) const {
298 if (start_size == nullptr)
299 return LOG_STATUS(
300 Status::FragmentInfoError("Cannot get non-empty domain var size; Start "
301 "size argument cannot be null"));
302
303 if (end_size == nullptr)
304 return LOG_STATUS(
305 Status::FragmentInfoError("Cannot get non-empty domain var size; End "
306 "size argument cannot be null"));
307
308 if (fid >= fragments_.size())
309 return LOG_STATUS(Status::FragmentInfoError(
310 "Cannot get non-empty domain var size; Invalid fragment index"));
311
312 const auto& non_empty_domain = fragments_[fid].non_empty_domain();
313
314 if (did >= non_empty_domain.size())
315 return LOG_STATUS(Status::FragmentInfoError(
316 "Cannot get non-empty domain var size; Invalid dimension index"));
317
318 if (!non_empty_domain[did].var_size())
319 return LOG_STATUS(Status::FragmentInfoError(
320 "Cannot get non-empty domain var size; Dimension is fixed sized"));
321
322 assert(!non_empty_domain[did].empty());
323 *start_size = non_empty_domain[did].start_size();
324 *end_size = non_empty_domain[did].end_size();
325
326 return Status::Ok();
327 }
328
get_non_empty_domain_var_size(uint32_t fid,const char * dim_name,uint64_t * start_size,uint64_t * end_size) const329 Status FragmentInfo::get_non_empty_domain_var_size(
330 uint32_t fid,
331 const char* dim_name,
332 uint64_t* start_size,
333 uint64_t* end_size) const {
334 if (dim_name == nullptr)
335 return LOG_STATUS(
336 Status::FragmentInfoError("Cannot get non-empty domain var size; "
337 "Dimension name argument cannot be null"));
338
339 uint32_t did;
340 for (did = 0; did < dim_names_.size(); ++did) {
341 if (dim_name == dim_names_[did]) {
342 break;
343 }
344 }
345
346 // Dimension name not found
347 if (did == dim_names_.size()) {
348 auto msg =
349 std::string(
350 "Cannot get non-empty domain var size; Invalid dimension name '") +
351 dim_name + "'";
352 return LOG_STATUS(Status::FragmentInfoError(msg));
353 }
354
355 return get_non_empty_domain_var_size(fid, did, start_size, end_size);
356 }
357
get_non_empty_domain_var(uint32_t fid,uint32_t did,void * start,void * end) const358 Status FragmentInfo::get_non_empty_domain_var(
359 uint32_t fid, uint32_t did, void* start, void* end) const {
360 if (start == nullptr)
361 return LOG_STATUS(
362 Status::FragmentInfoError("Cannot get non-empty domain var; Domain "
363 "start argument cannot be null"));
364
365 if (end == nullptr)
366 return LOG_STATUS(Status::FragmentInfoError(
367 "Cannot get non-empty domain var; Domain end argument cannot be null"));
368
369 if (fid >= fragments_.size())
370 return LOG_STATUS(Status::FragmentInfoError(
371 "Cannot get non-empty domain var; Invalid fragment index"));
372
373 const auto& non_empty_domain = fragments_[fid].non_empty_domain();
374
375 if (did >= non_empty_domain.size())
376 return LOG_STATUS(Status::FragmentInfoError(
377 "Cannot get non-empty domain var; Invalid dimension index"));
378
379 if (!non_empty_domain[did].var_size())
380 return LOG_STATUS(Status::FragmentInfoError(
381 "Cannot get non-empty domain var; Dimension is fixed-sized"));
382
383 assert(!non_empty_domain[did].empty());
384 std::memcpy(
385 start, non_empty_domain[did].start(), non_empty_domain[did].start_size());
386 std::memcpy(
387 end, non_empty_domain[did].end(), non_empty_domain[did].end_size());
388
389 return Status::Ok();
390 }
391
get_non_empty_domain_var(uint32_t fid,const char * dim_name,void * start,void * end) const392 Status FragmentInfo::get_non_empty_domain_var(
393 uint32_t fid, const char* dim_name, void* start, void* end) const {
394 if (dim_name == nullptr)
395 return LOG_STATUS(
396 Status::FragmentInfoError("Cannot get non-empty domain var; Dimension "
397 "name argument cannot be null"));
398
399 uint32_t did;
400 for (did = 0; did < dim_names_.size(); ++did) {
401 if (dim_name == dim_names_[did]) {
402 break;
403 }
404 }
405
406 // Dimension name not found
407 if (did == dim_names_.size()) {
408 auto msg =
409 std::string(
410 "Cannot get non-empty domain var; Invalid dimension name '") +
411 dim_name + "'";
412 return LOG_STATUS(Status::FragmentInfoError(msg));
413 }
414
415 return get_non_empty_domain_var(fid, did, start, end);
416 }
417
get_mbr_num(uint32_t fid,uint64_t * mbr_num)418 Status FragmentInfo::get_mbr_num(uint32_t fid, uint64_t* mbr_num) {
419 if (mbr_num == nullptr)
420 return LOG_STATUS(Status::FragmentInfoError(
421 "Cannot get fragment URI; MBR number argument cannot be null"));
422
423 if (fid >= fragments_.size())
424 return LOG_STATUS(Status::FragmentInfoError(
425 "Cannot get fragment URI; Invalid fragment index"));
426
427 if (!fragments_[fid].sparse()) {
428 *mbr_num = 0;
429 return Status::Ok();
430 }
431
432 auto meta = fragments_[fid].meta();
433 RETURN_NOT_OK(meta->load_rtree(*array_->encryption_key()));
434 *mbr_num = meta->mbrs().size();
435
436 return Status::Ok();
437 }
438
get_mbr(uint32_t fid,uint32_t mid,uint32_t did,void * mbr)439 Status FragmentInfo::get_mbr(
440 uint32_t fid, uint32_t mid, uint32_t did, void* mbr) {
441 if (mbr == nullptr)
442 return LOG_STATUS(Status::FragmentInfoError(
443 "Cannot get MBR; mbr argument cannot be null"));
444
445 if (fid >= fragments_.size())
446 return LOG_STATUS(
447 Status::FragmentInfoError("Cannot get MBR; Invalid fragment index"));
448
449 if (!fragments_[fid].sparse())
450 return LOG_STATUS(
451 Status::FragmentInfoError("Cannot get MBR; Fragment is not sparse"));
452
453 auto meta = fragments_[fid].meta();
454 RETURN_NOT_OK(meta->load_rtree(*array_->encryption_key()));
455 const auto& mbrs = meta->mbrs();
456
457 if (mid >= mbrs.size())
458 return LOG_STATUS(
459 Status::FragmentInfoError("Cannot get MBR; Invalid MBR index"));
460
461 const auto& minimum_bounding_rectangle = mbrs[mid];
462 if (did >= minimum_bounding_rectangle.size())
463 return LOG_STATUS(
464 Status::FragmentInfoError("Cannot get MBR; Invalid dimension index"));
465
466 if (minimum_bounding_rectangle[did].var_size())
467 return LOG_STATUS(Status::FragmentInfoError(
468 "Cannot get MBR; Dimension is variable-sized"));
469
470 assert(!minimum_bounding_rectangle[did].empty());
471 std::memcpy(
472 mbr,
473 minimum_bounding_rectangle[did].data(),
474 minimum_bounding_rectangle[did].size());
475
476 return Status::Ok();
477 }
478
get_mbr(uint32_t fid,uint32_t mid,const char * dim_name,void * mbr)479 Status FragmentInfo::get_mbr(
480 uint32_t fid, uint32_t mid, const char* dim_name, void* mbr) {
481 if (dim_name == nullptr)
482 return LOG_STATUS(Status::FragmentInfoError(
483 "Cannot get non-empty domain; Dimension name argument cannot be null"));
484
485 uint32_t did;
486 for (did = 0; did < dim_names_.size(); ++did) {
487 if (dim_name == dim_names_[did]) {
488 break;
489 }
490 }
491
492 // Dimension name not found
493 if (did == dim_names_.size()) {
494 auto msg =
495 std::string("Cannot get non-empty domain; Invalid dimension name '") +
496 dim_name + "'";
497 return LOG_STATUS(Status::FragmentInfoError(msg));
498 }
499
500 return get_mbr(fid, mid, did, mbr);
501 }
502
get_mbr_var_size(uint32_t fid,uint32_t mid,uint32_t did,uint64_t * start_size,uint64_t * end_size)503 Status FragmentInfo::get_mbr_var_size(
504 uint32_t fid,
505 uint32_t mid,
506 uint32_t did,
507 uint64_t* start_size,
508 uint64_t* end_size) {
509 if (start_size == nullptr)
510 return LOG_STATUS(
511 Status::FragmentInfoError("Cannot get MBR var size; Start "
512 "size argument cannot be null"));
513
514 if (end_size == nullptr)
515 return LOG_STATUS(
516 Status::FragmentInfoError("Cannot get MBR var size; End "
517 "size argument cannot be null"));
518
519 if (fid >= fragments_.size())
520 return LOG_STATUS(Status::FragmentInfoError(
521 "Cannot get MBR var size; Invalid fragment index"));
522
523 if (!fragments_[fid].sparse())
524 return LOG_STATUS(
525 Status::FragmentInfoError("Cannot get MBR; Fragment is not sparse"));
526
527 auto meta = fragments_[fid].meta();
528 RETURN_NOT_OK(meta->load_rtree(*array_->encryption_key()));
529 const auto& mbrs = meta->mbrs();
530
531 if (mid >= mbrs.size())
532 return LOG_STATUS(
533 Status::FragmentInfoError("Cannot get MBR; Invalid mbr index"));
534
535 const auto& minimum_bounding_rectangle = mbrs[mid];
536
537 if (did >= minimum_bounding_rectangle.size())
538 return LOG_STATUS(Status::FragmentInfoError(
539 "Cannot get MBR var size; Invalid dimension index"));
540
541 if (!minimum_bounding_rectangle[did].var_size())
542 return LOG_STATUS(Status::FragmentInfoError(
543 "Cannot get MBR var size; Dimension is fixed sized"));
544
545 assert(!minimum_bounding_rectangle[did].empty());
546 *start_size = minimum_bounding_rectangle[did].start_size();
547 *end_size = minimum_bounding_rectangle[did].end_size();
548
549 return Status::Ok();
550 }
551
get_mbr_var_size(uint32_t fid,uint32_t mid,const char * dim_name,uint64_t * start_size,uint64_t * end_size)552 Status FragmentInfo::get_mbr_var_size(
553 uint32_t fid,
554 uint32_t mid,
555 const char* dim_name,
556 uint64_t* start_size,
557 uint64_t* end_size) {
558 if (dim_name == nullptr)
559 return LOG_STATUS(
560 Status::FragmentInfoError("Cannot get MBR var size; "
561 "Dimension name argument cannot be null"));
562
563 uint32_t did;
564 for (did = 0; did < dim_names_.size(); ++did) {
565 if (dim_name == dim_names_[did]) {
566 break;
567 }
568 }
569
570 // Dimension name not found
571 if (did == dim_names_.size()) {
572 auto msg =
573 std::string("Cannot get MBR var size; Invalid dimension name '") +
574 dim_name + "'";
575 return LOG_STATUS(Status::FragmentInfoError(msg));
576 }
577
578 return get_mbr_var_size(fid, mid, did, start_size, end_size);
579 }
580
get_mbr_var(uint32_t fid,uint32_t mid,uint32_t did,void * start,void * end)581 Status FragmentInfo::get_mbr_var(
582 uint32_t fid, uint32_t mid, uint32_t did, void* start, void* end) {
583 if (start == nullptr)
584 return LOG_STATUS(
585 Status::FragmentInfoError("Cannot get non-empty domain var; Domain "
586 "start argument cannot be null"));
587
588 if (end == nullptr)
589 return LOG_STATUS(Status::FragmentInfoError(
590 "Cannot get non-empty domain var; Domain end argument cannot be null"));
591
592 if (fid >= fragments_.size())
593 return LOG_STATUS(Status::FragmentInfoError(
594 "Cannot get non-empty domain var; Invalid fragment index"));
595
596 if (!fragments_[fid].sparse())
597 return LOG_STATUS(
598 Status::FragmentInfoError("Cannot get MBR; Fragment is not sparse"));
599
600 auto meta = fragments_[fid].meta();
601 RETURN_NOT_OK(meta->load_rtree(*array_->encryption_key()));
602 const auto& mbrs = meta->mbrs();
603
604 if (mid >= mbrs.size())
605 return LOG_STATUS(
606 Status::FragmentInfoError("Cannot get MBR; Invalid mbr index"));
607
608 const auto& minimum_bounding_rectangle = mbrs[mid];
609
610 if (did >= minimum_bounding_rectangle.size())
611 return LOG_STATUS(Status::FragmentInfoError(
612 "Cannot get non-empty domain var; Invalid dimension index"));
613
614 if (!minimum_bounding_rectangle[did].var_size())
615 return LOG_STATUS(Status::FragmentInfoError(
616 "Cannot get non-empty domain var; Dimension is fixed-sized"));
617
618 assert(!minimum_bounding_rectangle[did].empty());
619 std::memcpy(
620 start,
621 minimum_bounding_rectangle[did].start(),
622 minimum_bounding_rectangle[did].start_size());
623 std::memcpy(
624 end,
625 minimum_bounding_rectangle[did].end(),
626 minimum_bounding_rectangle[did].end_size());
627
628 return Status::Ok();
629 }
630
get_mbr_var(uint32_t fid,uint32_t mid,const char * dim_name,void * start,void * end)631 Status FragmentInfo::get_mbr_var(
632 uint32_t fid, uint32_t mid, const char* dim_name, void* start, void* end) {
633 if (dim_name == nullptr)
634 return LOG_STATUS(
635 Status::FragmentInfoError("Cannot get non-empty domain var; Dimension "
636 "name argument cannot be null"));
637
638 uint32_t did;
639 for (did = 0; did < dim_names_.size(); ++did) {
640 if (dim_name == dim_names_[did]) {
641 break;
642 }
643 }
644
645 // Dimension name not found
646 if (did == dim_names_.size()) {
647 auto msg =
648 std::string(
649 "Cannot get non-empty domain var; Invalid dimension name '") +
650 dim_name + "'";
651 return LOG_STATUS(Status::FragmentInfoError(msg));
652 }
653
654 return get_mbr_var(fid, mid, did, start, end);
655 }
656
get_version(uint32_t fid,uint32_t * version) const657 Status FragmentInfo::get_version(uint32_t fid, uint32_t* version) const {
658 if (version == nullptr)
659 return LOG_STATUS(Status::FragmentInfoError(
660 "Cannot get version; Version argument cannot be null"));
661
662 if (fid >= fragments_.size())
663 return LOG_STATUS(Status::FragmentInfoError(
664 "Cannot get version; Invalid fragment index"));
665
666 *version = fragments_[fid].format_version();
667
668 return Status::Ok();
669 }
670
get_array_schema(uint32_t fid,ArraySchema ** array_schema)671 Status FragmentInfo::get_array_schema(
672 uint32_t fid, ArraySchema** array_schema) {
673 if (array_schema == nullptr)
674 return LOG_STATUS(Status::FragmentInfoError(
675 "Cannot get array schema; schema argument cannot be null"));
676
677 if (fid >= fragments_.size())
678 return LOG_STATUS(Status::FragmentInfoError(
679 "Cannot get array schema; Invalid fragment index"));
680 URI schema_uri;
681 uint32_t version = fragments_[fid].format_version();
682 if (version >= 10) {
683 schema_uri = array_uri_.join_path(constants::array_schema_folder_name)
684 .join_path(fragments_[fid].array_schema_name());
685 } else {
686 schema_uri = array_uri_.join_path(constants::array_schema_filename);
687 }
688
689 EncryptionKey encryption_key;
690 RETURN_NOT_OK(storage_manager_->load_array_schema_from_uri(
691 schema_uri, encryption_key, array_schema));
692
693 return Status::Ok();
694 }
695
get_array_schema_name(uint32_t fid,const char ** schema_name)696 Status FragmentInfo::get_array_schema_name(
697 uint32_t fid, const char** schema_name) {
698 if (schema_name == nullptr)
699 return LOG_STATUS(Status::FragmentInfoError(
700 "Cannot get array schema URI; schema name argument cannot be null"));
701
702 if (fid >= fragments_.size())
703 return LOG_STATUS(Status::FragmentInfoError(
704 "Cannot get array schema name; Invalid fragment index"));
705
706 uint32_t version = fragments_[fid].format_version();
707 if (version >= 10) {
708 *schema_name = fragments_[fid].array_schema_name().c_str();
709 } else {
710 *schema_name = constants::array_schema_filename.c_str();
711 }
712
713 return Status::Ok();
714 }
715
has_consolidated_metadata(uint32_t fid,int32_t * has) const716 Status FragmentInfo::has_consolidated_metadata(
717 uint32_t fid, int32_t* has) const {
718 if (has == nullptr)
719 return LOG_STATUS(
720 Status::FragmentInfoError("Cannot check if fragment has consolidated "
721 "metadata; Has argument cannot be null"));
722
723 if (fid >= fragments_.size())
724 return LOG_STATUS(
725 Status::FragmentInfoError("Cannot check if fragment has consolidated "
726 "metadata; Invalid fragment index"));
727
728 *has = fragments_[fid].has_consolidated_footer();
729
730 return Status::Ok();
731 }
732
array_open(const Config & config,EncryptionType encryption_type,const void * encryption_key,uint32_t key_length) const733 Status FragmentInfo::array_open(
734 const Config& config,
735 EncryptionType encryption_type,
736 const void* encryption_key,
737 uint32_t key_length) const {
738 if (array_->is_open())
739 array_->close();
740
741 if (encryption_type == EncryptionType::NO_ENCRYPTION) {
742 bool found = false;
743 std::string encryption_key_from_cfg =
744 config.get("sm.encryption_key", &found);
745 assert(found);
746 std::string encryption_type_from_cfg =
747 config.get("sm.encryption_type", &found);
748 assert(found);
749 auto [st, et] = encryption_type_enum(encryption_type_from_cfg);
750 RETURN_NOT_OK(st);
751 encryption_type = et.value();
752 EncryptionKey encryption_key_cfg;
753 uint32_t key_length = 0;
754
755 if (encryption_key_from_cfg.empty()) {
756 RETURN_NOT_OK(encryption_key_cfg.set_key(encryption_type, nullptr, 0));
757 } else {
758 if (EncryptionKey::is_valid_key_length(
759 encryption_type,
760 static_cast<uint32_t>(encryption_key_from_cfg.size()))) {
761 const UnitTestConfig& unit_test_cfg = UnitTestConfig::instance();
762 if (unit_test_cfg.array_encryption_key_length.is_set()) {
763 key_length = unit_test_cfg.array_encryption_key_length.get();
764 } else {
765 key_length = static_cast<uint32_t>(encryption_key_from_cfg.size());
766 }
767 }
768 }
769 RETURN_NOT_OK(array_->open_without_fragments(
770 encryption_type,
771 (const void*)encryption_key_from_cfg.c_str(),
772 key_length));
773 } else {
774 RETURN_NOT_OK(array_->open_without_fragments(
775 encryption_type, encryption_key, key_length));
776 }
777
778 return Status::Ok();
779 }
780
load(const Config & config,EncryptionType encryption_type,const void * encryption_key,uint32_t key_length)781 Status FragmentInfo::load(
782 const Config& config,
783 EncryptionType encryption_type,
784 const void* encryption_key,
785 uint32_t key_length) {
786 bool is_array;
787 RETURN_NOT_OK(storage_manager_->is_array(array_uri_, &is_array));
788 if (!is_array) {
789 auto msg = std::string("Cannot load fragment info; Array '") +
790 array_uri_.to_string() + "' does not exist";
791 return LOG_STATUS(Status::FragmentInfoError(msg));
792 }
793
794 RETURN_NOT_OK(
795 array_open(config, encryption_type, encryption_key, key_length));
796
797 auto timestamp = utils::time::timestamp_now_ms();
798 RETURN_NOT_OK_ELSE(
799 storage_manager_->get_fragment_info(*array_, 0, timestamp, this, true),
800 array_->close());
801
802 unconsolidated_metadata_num_ = 0;
803 for (const auto& f : fragments_)
804 unconsolidated_metadata_num_ += (uint32_t)!f.has_consolidated_footer();
805
806 return Status::Ok();
807 }
808
set_dim_info(const std::vector<std::string> & dim_names,const std::vector<Datatype> & dim_types)809 void FragmentInfo::set_dim_info(
810 const std::vector<std::string>& dim_names,
811 const std::vector<Datatype>& dim_types) {
812 dim_names_ = dim_names;
813 dim_types_ = dim_types;
814 }
815
set_to_vacuum(const std::vector<URI> & to_vacuum)816 void FragmentInfo::set_to_vacuum(const std::vector<URI>& to_vacuum) {
817 to_vacuum_ = to_vacuum;
818 }
819
fragments() const820 const std::vector<SingleFragmentInfo>& FragmentInfo::fragments() const {
821 return fragments_;
822 }
823
anterior_ndrange() const824 const NDRange& FragmentInfo::anterior_ndrange() const {
825 return anterior_ndrange_;
826 }
827
to_vacuum_num() const828 uint32_t FragmentInfo::to_vacuum_num() const {
829 return (uint32_t)to_vacuum_.size();
830 }
831
unconsolidated_metadata_num() const832 uint32_t FragmentInfo::unconsolidated_metadata_num() const {
833 return unconsolidated_metadata_num_;
834 }
835
836 /* ********************************* */
837 /* PRIVATE METHODS */
838 /* ********************************* */
839
clone() const840 FragmentInfo FragmentInfo::clone() const {
841 FragmentInfo clone;
842 clone.array_uri_ = array_uri_;
843 clone.dim_names_ = dim_names_;
844 clone.dim_types_ = dim_types_;
845 clone.fragments_ = fragments_;
846 clone.storage_manager_ = storage_manager_;
847 clone.to_vacuum_ = to_vacuum_;
848 clone.unconsolidated_metadata_num_ = unconsolidated_metadata_num_;
849 clone.anterior_ndrange_ = anterior_ndrange_;
850
851 return clone;
852 }
853
swap(FragmentInfo & fragment_info)854 void FragmentInfo::swap(FragmentInfo& fragment_info) {
855 std::swap(array_uri_, fragment_info.array_uri_);
856 std::swap(dim_names_, fragment_info.dim_names_);
857 std::swap(dim_types_, fragment_info.dim_types_);
858 std::swap(fragments_, fragment_info.fragments_);
859 std::swap(storage_manager_, fragment_info.storage_manager_);
860 std::swap(to_vacuum_, fragment_info.to_vacuum_);
861 std::swap(
862 unconsolidated_metadata_num_, fragment_info.unconsolidated_metadata_num_);
863 std::swap(anterior_ndrange_, fragment_info.anterior_ndrange_);
864 }
865