1 /**
2 * @file sparse_index_reader_base.h
3 *
4 * @section LICENSE
5 *
6 * The MIT License
7 *
8 * @copyright Copyright (c) 2017-2021 TileDB, Inc.
9 *
10 * Permission is hereby granted, free of charge, to any person obtaining a copy
11 * of this software and associated documentation files (the "Software"), to deal
12 * in the Software without restriction, including without limitation the rights
13 * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
14 * copies of the Software, and to permit persons to whom the Software is
15 * furnished to do so, subject to the following conditions:
16 *
17 * The above copyright notice and this permission notice shall be included in
18 * all copies or substantial portions of the Software.
19 *
20 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
21 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
22 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
23 * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
24 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
25 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
26 * THE SOFTWARE.
27 *
28 * @section DESCRIPTION
29 *
30 * This file defines class SparseIndexReaderBase.
31 */
32
33 #ifndef TILEDB_SPARSE_INDEX_READER_BASE_H
34 #define TILEDB_SPARSE_INDEX_READER_BASE_H
35
36 #include <queue>
37 #include "reader_base.h"
38 #include "tiledb/common/status.h"
39 #include "tiledb/sm/array_schema/dimension.h"
40 #include "tiledb/sm/misc/types.h"
41 #include "tiledb/sm/misc/utils.h"
42 #include "tiledb/sm/query/query_condition.h"
43 #include "tiledb/sm/query/result_cell_slab.h"
44
45 namespace tiledb {
46 namespace sm {
47
48 class Array;
49 class ArraySchema;
50 class OpenArrayMemoryTracker;
51 class StorageManager;
52 class Subarray;
53
ceil_div(uint64_t a,uint64_t b)54 constexpr uint64_t ceil_div(uint64_t a, uint64_t b) {
55 auto div = static_cast<float>(a) / static_cast<float>(b);
56 return (static_cast<float>(static_cast<int32_t>(div)) == div) ?
57 static_cast<int32_t>(div) :
58 static_cast<int32_t>(div) + ((div > 0) ? 1 : 0);
59 }
60
61 /** Processes read queries. */
62 class SparseIndexReaderBase : public ReaderBase {
63 public:
64 /* ********************************* */
65 /* TYPE DEFINITIONS */
66 /* ********************************* */
67
68 /** The state for a read sparse global order query. */
69 struct ReadState {
70 /** The result cell slabs currently in process. */
71 std::vector<ResultCellSlab> result_cell_slabs_;
72
73 /** The tile index inside of each fragments. */
74 std::vector<std::pair<uint64_t, uint64_t>> frag_tile_idx_;
75
76 /** Is the reader done with the query. */
77 bool done_adding_result_tiles_;
78 };
79
80 /* ********************************* */
81 /* CONSTRUCTORS & DESTRUCTORS */
82 /* ********************************* */
83
84 /** Constructor. */
85 SparseIndexReaderBase(
86 stats::Stats* stats,
87 tdb_shared_ptr<Logger> logger,
88 StorageManager* storage_manager,
89 Array* array,
90 Config& config,
91 std::unordered_map<std::string, QueryBuffer>& buffers,
92 Subarray& subarray,
93 Layout layout,
94 QueryCondition& condition);
95
96 /** Destructor. */
97 ~SparseIndexReaderBase() = default;
98
99 /* ********************************* */
100 /* PUBLIC METHODS */
101 /* ********************************* */
102
103 /** Returns the current read state. */
104 const ReadState* read_state() const;
105
106 /** Returns the current read state. */
107 ReadState* read_state();
108
109 /** Clears the result tiles. Used by serialization. */
110 virtual Status clear_result_tiles() = 0;
111
112 /** Add a result tile with no memory budget checks. Used by serialization. */
113 virtual ResultTile* add_result_tile_unsafe(
114 unsigned f, uint64_t t, const Domain* domain) = 0;
115
116 protected:
117 /* ********************************* */
118 /* PROTECTED ATTRIBUTES */
119 /* ********************************* */
120
121 /** Read state. */
122 ReadState read_state_;
123
124 /** Have we loaded all thiles for this fragment. */
125 std::vector<bool> all_tiles_loaded_;
126
127 /** Dimension names. */
128 std::vector<std::string> dim_names_;
129
130 /** Are dimensions var sized. */
131 std::vector<bool> is_dim_var_size_;
132
133 /** Reverse sorted vector, per fragments, of tiles ranges in the subarray, if
134 * set. */
135 std::vector<std::vector<std::pair<uint64_t, uint64_t>>> result_tile_ranges_;
136
137 /** Have ve loaded the initial data. */
138 bool initial_data_loaded_;
139
140 /** Total memory budget. */
141 uint64_t memory_budget_;
142
143 /** Mutex protecting memory budget variables. */
144 std::mutex mem_budget_mtx_;
145
146 /** Memory tracker object for the array. */
147 OpenArrayMemoryTracker* array_memory_tracker_;
148
149 /** Memory used for coordinates tiles. */
150 uint64_t memory_used_for_coords_total_;
151
152 /** Memory used for query condition tiles. */
153 uint64_t memory_used_qc_tiles_;
154
155 /** Memory used for result cell slabs. */
156 uint64_t memory_used_rcs_;
157
158 /** Memory used for result tiles. */
159 uint64_t memory_used_result_tiles_;
160
161 /** Memory used for result tile ranges. */
162 uint64_t memory_used_result_tile_ranges_;
163
164 /** How much of the memory budget is reserved for coords. */
165 double memory_budget_ratio_coords_;
166
167 /** How much of the memory budget is reserved for query condition. */
168 double memory_budget_ratio_query_condition_;
169
170 /** How much of the memory budget is reserved for tile ranges. */
171 double memory_budget_ratio_tile_ranges_;
172
173 /** How much of the memory budget is reserved for array data. */
174 double memory_budget_ratio_array_data_;
175
176 /** How much of the memory budget is reserved for result tiles. */
177 double memory_budget_ratio_result_tiles_;
178
179 /** How much of the memory budget is reserved for result cell slabs. */
180 double memory_budget_ratio_rcs_;
181
182 /** Indicate if the coordinates are loaded for the result tiles. */
183 bool coords_loaded_;
184
185 /* ********************************* */
186 /* PROTECTED METHODS */
187 /* ********************************* */
188
189 /** Get the coordinate tiles size for a dimension. */
190 Status get_coord_tiles_size(
191 unsigned dim_num, unsigned f, uint64_t t, uint64_t* tiles_size);
192
193 /** Load tile offsets and result tile ranges. */
194 Status load_initial_data();
195
196 /** Compute the result bitmap for a tile. */
197 Status compute_coord_tiles_result_bitmap(
198 ResultTile* tile,
199 uint64_t range_idx,
200 std::vector<uint8_t>* coord_tiles_result_bitmap);
201
202 /** Resize the output buffers to the correct size after copying. */
203 Status resize_output_buffers();
204
205 /**
206 * Adds an extra offset in the end of the offsets buffer indicating the
207 * returned data size if an attribute is var-sized.
208 */
209 Status add_extra_offset();
210
211 /** Remove a result tile range for a specific fragment */
212 void remove_result_tile_range(uint64_t f);
213
214 /** Remove a result tile range for a specific range */
215 void remove_range_result_tile_range(uint64_t r);
216 };
217
218 } // namespace sm
219 } // namespace tiledb
220
221 #endif // TILEDB_SPARSE_INDEX_READER_BASE_H