1 /*
2 Portions Copyright (c) 2016-Present, Facebook, Inc.
3 Portions Copyright (c) 2012,2013 Monty Program Ab
4
5 This program is free software; you can redistribute it and/or modify
6 it under the terms of the GNU General Public License as published by
7 the Free Software Foundation; version 2 of the License.
8
9 This program is distributed in the hope that it will be useful,
10 but WITHOUT ANY WARRANTY; without even the implied warranty of
11 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12 GNU General Public License for more details.
13
14 You should have received a copy of the GNU General Public License
15 along with this program; if not, write to the Free Software
16 Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02111-1301 USA */
17 #pragma once
18
19 #include <algorithm>
20 #include <string>
21 #include <vector>
22
23 #ifdef _WIN32
24 #include <stdlib.h>
25 #define htobe64 _byteswap_uint64
26 #define be64toh _byteswap_uint64
27 #define htobe32 _byteswap_ulong
28 #define be32toh _byteswap_ulong
29 #define htobe16 _byteswap_ushort
30 #define be16toh _byteswap_ushort
31 #endif
32
33 #if defined(__APPLE__)
34 #include <libkern/OSByteOrder.h>
35 #define htobe64(x) OSSwapHostToBigInt64(x)
36 #define be64toh(x) OSSwapBigToHostInt64(x)
37 #define htobe32(x) OSSwapHostToBigInt32(x)
38 #define be32toh(x) OSSwapBigToHostInt32(x)
39 #define htobe16(x) OSSwapHostToBigInt16(x)
40 #define be16toh(x) OSSwapBigToHostInt16(x)
41 #endif
42
43 /* MySQL header files */
44 #include "./my_global.h"
45
46 /* MyRocks header files */
47 #include "./rdb_global.h"
48 #include "./rdb_utils.h"
49
50 /* RocksDB header files */
51 #include "rocksdb/slice.h"
52 #include "rocksdb/status.h"
53
54 namespace myrocks {
55
56 /*
57 Basic composition functions for a network buffer presented as a MySQL String
58 ("netstr") which stores data in Network Byte Order (Big Endian).
59 */
60
rdb_netstr_append_uint64(my_core::String * const out_netstr,const uint64 val)61 inline void rdb_netstr_append_uint64(my_core::String *const out_netstr,
62 const uint64 val) {
63 DBUG_ASSERT(out_netstr != nullptr);
64
65 // Convert from host machine byte order (usually Little Endian) to network
66 // byte order (Big Endian).
67 uint64 net_val = htobe64(val);
68 out_netstr->append(reinterpret_cast<char *>(&net_val), sizeof(net_val));
69 }
70
rdb_netstr_append_uint32(my_core::String * const out_netstr,const uint32 val)71 inline void rdb_netstr_append_uint32(my_core::String *const out_netstr,
72 const uint32 val) {
73 DBUG_ASSERT(out_netstr != nullptr);
74
75 // Convert from host machine byte order (usually Little Endian) to network
76 // byte order (Big Endian).
77 uint32 net_val = htobe32(val);
78 out_netstr->append(reinterpret_cast<char *>(&net_val), sizeof(net_val));
79 }
80
rdb_netstr_append_uint16(my_core::String * const out_netstr,const uint16 val)81 inline void rdb_netstr_append_uint16(my_core::String *const out_netstr,
82 const uint16 val) {
83 DBUG_ASSERT(out_netstr != nullptr);
84
85 // Convert from host machine byte order (usually Little Endian) to network
86 // byte order (Big Endian).
87 uint16 net_val = htobe16(val);
88 out_netstr->append(reinterpret_cast<char *>(&net_val), sizeof(net_val));
89 }
90
91 /*
92 Basic network buffer ("netbuf") write helper functions.
93 */
94
rdb_netbuf_store_uint64(uchar * const dst_netbuf,const uint64 n)95 inline void rdb_netbuf_store_uint64(uchar *const dst_netbuf, const uint64 n) {
96 DBUG_ASSERT(dst_netbuf != nullptr);
97
98 // Convert from host byte order (usually Little Endian) to network byte order
99 // (Big Endian).
100 uint64 net_val = htobe64(n);
101 memcpy(dst_netbuf, &net_val, sizeof(net_val));
102 }
103
rdb_netbuf_store_uint32(uchar * const dst_netbuf,const uint32 n)104 inline void rdb_netbuf_store_uint32(uchar *const dst_netbuf, const uint32 n) {
105 DBUG_ASSERT(dst_netbuf != nullptr);
106
107 // Convert from host byte order (usually Little Endian) to network byte order
108 // (Big Endian).
109 uint32 net_val = htobe32(n);
110 memcpy(dst_netbuf, &net_val, sizeof(net_val));
111 }
112
rdb_netbuf_store_uint16(uchar * const dst_netbuf,const uint16 n)113 inline void rdb_netbuf_store_uint16(uchar *const dst_netbuf, const uint16 n) {
114 DBUG_ASSERT(dst_netbuf != nullptr);
115
116 // Convert from host byte order (usually Little Endian) to network byte order
117 // (Big Endian).
118 uint16 net_val = htobe16(n);
119 memcpy(dst_netbuf, &net_val, sizeof(net_val));
120 }
121
rdb_netbuf_store_byte(uchar * const dst_netbuf,const uchar c)122 inline void rdb_netbuf_store_byte(uchar *const dst_netbuf, const uchar c) {
123 DBUG_ASSERT(dst_netbuf != nullptr);
124
125 *dst_netbuf = c;
126 }
127
rdb_netbuf_store_index(uchar * const dst_netbuf,const uint32 number)128 inline void rdb_netbuf_store_index(uchar *const dst_netbuf,
129 const uint32 number) {
130 DBUG_ASSERT(dst_netbuf != nullptr);
131
132 rdb_netbuf_store_uint32(dst_netbuf, number);
133 }
134
135 /*
136 Basic conversion helper functions from network byte order (Big Endian) to host
137 machine byte order (usually Little Endian).
138 */
139
rdb_netbuf_to_uint64(const uchar * const netbuf)140 inline uint64 rdb_netbuf_to_uint64(const uchar *const netbuf) {
141 DBUG_ASSERT(netbuf != nullptr);
142
143 uint64 net_val;
144 memcpy(&net_val, netbuf, sizeof(net_val));
145
146 // Convert from network byte order (Big Endian) to host machine byte order
147 // (usually Little Endian).
148 return be64toh(net_val);
149 }
150
rdb_netbuf_to_uint32(const uchar * const netbuf)151 inline uint32 rdb_netbuf_to_uint32(const uchar *const netbuf) {
152 DBUG_ASSERT(netbuf != nullptr);
153
154 uint32 net_val;
155 memcpy(&net_val, netbuf, sizeof(net_val));
156
157 // Convert from network byte order (Big Endian) to host machine byte order
158 // (usually Little Endian).
159 return be32toh(net_val);
160 }
161
rdb_netbuf_to_uint16(const uchar * const netbuf)162 inline uint16 rdb_netbuf_to_uint16(const uchar *const netbuf) {
163 DBUG_ASSERT(netbuf != nullptr);
164
165 uint16 net_val;
166 memcpy(&net_val, netbuf, sizeof(net_val));
167
168 // Convert from network byte order (Big Endian) to host machine byte order
169 // (usually Little Endian).
170 return be16toh(net_val);
171 }
172
rdb_netbuf_to_byte(const uchar * const netbuf)173 inline uchar rdb_netbuf_to_byte(const uchar *const netbuf) {
174 DBUG_ASSERT(netbuf != nullptr);
175
176 return (uchar)netbuf[0];
177 }
178
179 /*
180 Basic network buffer ("netbuf") read helper functions.
181 Network buffer stores data in Network Byte Order (Big Endian).
182 NB: The netbuf is passed as an input/output param, hence after reading,
183 the netbuf pointer gets advanced to the following byte.
184 */
185
rdb_netbuf_read_uint64(const uchar ** netbuf_ptr)186 inline uint64 rdb_netbuf_read_uint64(const uchar **netbuf_ptr) {
187 DBUG_ASSERT(netbuf_ptr != nullptr);
188
189 // Convert from network byte order (Big Endian) to host machine byte order
190 // (usually Little Endian).
191 const uint64 host_val = rdb_netbuf_to_uint64(*netbuf_ptr);
192
193 // Advance pointer.
194 *netbuf_ptr += sizeof(host_val);
195
196 return host_val;
197 }
198
rdb_netbuf_read_uint32(const uchar ** netbuf_ptr)199 inline uint32 rdb_netbuf_read_uint32(const uchar **netbuf_ptr) {
200 DBUG_ASSERT(netbuf_ptr != nullptr);
201
202 // Convert from network byte order (Big Endian) to host machine byte order
203 // (usually Little Endian).
204 const uint32 host_val = rdb_netbuf_to_uint32(*netbuf_ptr);
205
206 // Advance pointer.
207 *netbuf_ptr += sizeof(host_val);
208
209 return host_val;
210 }
211
rdb_netbuf_read_uint16(const uchar ** netbuf_ptr)212 inline uint16 rdb_netbuf_read_uint16(const uchar **netbuf_ptr) {
213 DBUG_ASSERT(netbuf_ptr != nullptr);
214
215 // Convert from network byte order (Big Endian) to host machine byte order
216 // (usually Little Endian).
217 const uint16 host_val = rdb_netbuf_to_uint16(*netbuf_ptr);
218
219 // Advance pointer.
220 *netbuf_ptr += sizeof(host_val);
221
222 return host_val;
223 }
224
rdb_netbuf_read_gl_index(const uchar ** netbuf_ptr,GL_INDEX_ID * const gl_index_id)225 inline void rdb_netbuf_read_gl_index(const uchar **netbuf_ptr,
226 GL_INDEX_ID *const gl_index_id) {
227 DBUG_ASSERT(gl_index_id != nullptr);
228 DBUG_ASSERT(netbuf_ptr != nullptr);
229
230 gl_index_id->cf_id = rdb_netbuf_read_uint32(netbuf_ptr);
231 gl_index_id->index_id = rdb_netbuf_read_uint32(netbuf_ptr);
232 }
233
234 /*
235 A simple string reader:
236 - it keeps position within the string that we read from
237 - it prevents one from reading beyond the end of the string.
238 */
239
240 class Rdb_string_reader {
241 const char *m_ptr;
242 uint m_len;
243
244 private:
245 Rdb_string_reader &operator=(const Rdb_string_reader &) = default;
246
247 public:
248 Rdb_string_reader(const Rdb_string_reader &) = default;
249 /* named constructor */
read_or_empty(const rocksdb::Slice * const slice)250 static Rdb_string_reader read_or_empty(const rocksdb::Slice *const slice) {
251 if (!slice) {
252 return Rdb_string_reader("");
253 } else {
254 return Rdb_string_reader(slice);
255 }
256 }
257
Rdb_string_reader(const std::string & str)258 explicit Rdb_string_reader(const std::string &str) {
259 m_len = str.length();
260 if (m_len) {
261 m_ptr = &str.at(0);
262 } else {
263 /*
264 One can a create a Rdb_string_reader for reading from an empty string
265 (although attempts to read anything will fail).
266 We must not access str.at(0), since len==0, we can set ptr to any
267 value.
268 */
269 m_ptr = nullptr;
270 }
271 }
272
Rdb_string_reader(const rocksdb::Slice * const slice)273 explicit Rdb_string_reader(const rocksdb::Slice *const slice) {
274 m_ptr = slice->data();
275 m_len = slice->size();
276 }
277
278 /*
279 Read the next @param size bytes. Returns pointer to the bytes read, or
280 nullptr if the remaining string doesn't have that many bytes.
281 */
read(const uint size)282 const char *read(const uint size) {
283 const char *res;
284 if (m_len < size) {
285 res = nullptr;
286 } else {
287 res = m_ptr;
288 m_ptr += size;
289 m_len -= size;
290 }
291 return res;
292 }
293
read_uint8(uint * const res)294 bool read_uint8(uint *const res) {
295 const uchar *p;
296 if (!(p = reinterpret_cast<const uchar *>(read(1)))) {
297 return true; // error
298 } else {
299 *res = *p;
300 return false; // Ok
301 }
302 }
303
read_uint16(uint * const res)304 bool read_uint16(uint *const res) {
305 const uchar *p;
306 if (!(p = reinterpret_cast<const uchar *>(read(2)))) {
307 return true; // error
308 } else {
309 *res = rdb_netbuf_to_uint16(p);
310 return false; // Ok
311 }
312 }
313
read_uint64(uint64 * const res)314 bool read_uint64(uint64 *const res) {
315 const uchar *p;
316 if (!(p = reinterpret_cast<const uchar *>(read(sizeof(uint64))))) {
317 return true; // error
318 } else {
319 *res = rdb_netbuf_to_uint64(p);
320 return false; // Ok
321 }
322 }
323
remaining_bytes()324 uint remaining_bytes() const { return m_len; }
325
326 /*
327 Return pointer to data that will be read by next read() call (if there is
328 nothing left to read, returns pointer to beyond the end of previous read()
329 call)
330 */
get_current_ptr()331 const char *get_current_ptr() const { return m_ptr; }
332 };
333
334 /*
335 @brief
336 A buffer one can write the data to.
337
338 @detail
339 Suggested usage pattern:
340
341 writer->clear();
342 writer->write_XXX(...);
343 ...
344 // Ok, writer->ptr() points to the data written so far,
345 // and writer->get_current_pos() is the length of the data
346
347 */
348
349 class Rdb_string_writer {
350 std::vector<uchar> m_data;
351
352 public:
353 Rdb_string_writer(const Rdb_string_writer &) = delete;
354 Rdb_string_writer &operator=(const Rdb_string_writer &) = delete;
355 Rdb_string_writer() = default;
356
clear()357 void clear() { m_data.clear(); }
write_uint8(const uint val)358 void write_uint8(const uint val) {
359 m_data.push_back(static_cast<uchar>(val));
360 }
361
write_uint16(const uint val)362 void write_uint16(const uint val) {
363 const auto size = m_data.size();
364 m_data.resize(size + 2);
365 rdb_netbuf_store_uint16(m_data.data() + size, val);
366 }
367
write_uint32(const uint val)368 void write_uint32(const uint val) {
369 const auto size = m_data.size();
370 m_data.resize(size + 4);
371 rdb_netbuf_store_uint32(m_data.data() + size, val);
372 }
373
write(const uchar * const new_data,const size_t len)374 void write(const uchar *const new_data, const size_t len) {
375 DBUG_ASSERT(new_data != nullptr);
376 m_data.insert(m_data.end(), new_data, new_data + len);
377 }
378
ptr()379 uchar *ptr() { return m_data.data(); }
get_current_pos()380 size_t get_current_pos() const { return m_data.size(); }
381
write_uint8_at(const size_t pos,const uint new_val)382 void write_uint8_at(const size_t pos, const uint new_val) {
383 // This function will only overwrite what was written
384 DBUG_ASSERT(pos < get_current_pos());
385 m_data.data()[pos] = new_val;
386 }
387
write_uint16_at(const size_t pos,const uint new_val)388 void write_uint16_at(const size_t pos, const uint new_val) {
389 // This function will only overwrite what was written
390 DBUG_ASSERT(pos < get_current_pos() && (pos + 1) < get_current_pos());
391 rdb_netbuf_store_uint16(m_data.data() + pos, new_val);
392 }
393
truncate(const size_t pos)394 void truncate(const size_t pos) {
395 DBUG_ASSERT(pos < m_data.size());
396 m_data.resize(pos);
397 }
398
399 void allocate(const size_t len, const uchar val = 0) {
400 DBUG_ASSERT(len > 0);
401 m_data.resize(m_data.size() + len, val);
402 }
403
404 /*
405 An awful hack to deallocate the buffer without relying on the deconstructor.
406 This is needed to suppress valgrind errors in rocksdb.partition
407 */
free()408 void free() { std::vector<uchar>().swap(m_data); }
409 };
410
411 /*
412 A helper class for writing bits into Rdb_string_writer.
413
414 The class assumes (but doesn't check) that nobody tries to write
415 anything to the Rdb_string_writer that it is writing to.
416 */
417 class Rdb_bit_writer {
418 Rdb_string_writer *m_writer;
419 uchar m_offset;
420
421 public:
422 Rdb_bit_writer(const Rdb_bit_writer &) = delete;
423 Rdb_bit_writer &operator=(const Rdb_bit_writer &) = delete;
424
Rdb_bit_writer(Rdb_string_writer * writer_arg)425 explicit Rdb_bit_writer(Rdb_string_writer *writer_arg)
426 : m_writer(writer_arg), m_offset(0) {}
427
write(uint size,const uint value)428 void write(uint size, const uint value) {
429 DBUG_ASSERT((value & ((1 << size) - 1)) == value);
430
431 while (size > 0) {
432 if (m_offset == 0) {
433 m_writer->write_uint8(0);
434 }
435 // number of bits to put in this byte
436 const uint bits = std::min(size, (uint)(8 - m_offset));
437 uchar *const last_byte =
438 m_writer->ptr() + m_writer->get_current_pos() - 1;
439 *last_byte |= (uchar)((value >> (size - bits)) & ((1 << bits) - 1))
440 << m_offset;
441 size -= bits;
442 m_offset = (m_offset + bits) & 0x7;
443 }
444 }
445 };
446
447 class Rdb_bit_reader {
448 const uchar *m_cur;
449 uchar m_offset;
450 uint m_ret;
451 Rdb_string_reader *const m_reader;
452
453 public:
454 Rdb_bit_reader(const Rdb_bit_reader &) = delete;
455 Rdb_bit_reader &operator=(const Rdb_bit_reader &) = delete;
456
Rdb_bit_reader(Rdb_string_reader * const reader)457 explicit Rdb_bit_reader(Rdb_string_reader *const reader)
458 : m_cur(nullptr), m_offset(0), m_reader(reader) {}
459
460 // Returns a pointer to an uint containing the bits read. On subsequent
461 // reads, the value being pointed to will be overwritten. Returns nullptr
462 // on failure.
read(uint size)463 uint *read(uint size) {
464 m_ret = 0;
465 DBUG_ASSERT(size <= 32);
466
467 while (size > 0) {
468 if (m_offset == 0) {
469 m_cur = (const uchar *)m_reader->read(1);
470 if (m_cur == nullptr) {
471 return nullptr;
472 }
473 }
474 // how many bits from the current byte?
475 const uint bits = std::min((uint)(8 - m_offset), size);
476 m_ret <<= bits;
477 m_ret |= (*m_cur >> m_offset) & ((1 << bits) - 1);
478 size -= bits;
479 m_offset = (m_offset + bits) & 0x7;
480 }
481
482 return &m_ret;
483 }
484 };
485
486 template <size_t buf_length>
487 class Rdb_buf_writer {
488 public:
489 Rdb_buf_writer(const Rdb_buf_writer &) = delete;
490 Rdb_buf_writer &operator=(const Rdb_buf_writer &) = delete;
Rdb_buf_writer()491 Rdb_buf_writer() { reset(); }
492
write_uint32(const uint32 n)493 void write_uint32(const uint32 n) {
494 DBUG_ASSERT(m_ptr + sizeof(n) <= m_buf.data() + buf_length);
495 rdb_netbuf_store_uint32(m_ptr, n);
496 m_ptr += sizeof(n);
497 }
498
write_uint64(const uint64 n)499 void write_uint64(const uint64 n) {
500 DBUG_ASSERT(m_ptr + sizeof(n) <= m_buf.data() + buf_length);
501 rdb_netbuf_store_uint64(m_ptr, n);
502 m_ptr += sizeof(n);
503 }
504
write_uint16(const uint16 n)505 void write_uint16(const uint16 n) {
506 DBUG_ASSERT(m_ptr + sizeof(n) <= m_buf.data() + buf_length);
507 rdb_netbuf_store_uint16(m_ptr, n);
508 m_ptr += sizeof(n);
509 }
510
write_byte(const uchar c)511 void write_byte(const uchar c) {
512 DBUG_ASSERT(m_ptr + sizeof(c) <= m_buf.data() + buf_length);
513 rdb_netbuf_store_byte(m_ptr, c);
514 m_ptr += sizeof(c);
515 }
516
write_index(const uint32 n)517 void write_index(const uint32 n) { write_uint32(n); }
518
write(const char * buf,const size_t size)519 void write(const char *buf, const size_t size) {
520 DBUG_ASSERT(m_ptr + size <= m_buf.data() + buf_length);
521 memcpy(m_ptr, buf, size);
522 m_ptr += size;
523 }
524
write(const uchar * buf,const size_t size)525 void write(const uchar *buf, const size_t size) {
526 DBUG_ASSERT(m_ptr + size <= m_buf.data() + buf_length);
527 memcpy(m_ptr, buf, size);
528 m_ptr += size;
529 }
530
reset()531 void reset() { m_ptr = m_buf.data(); }
532
data()533 const char *data() const {
534 return reinterpret_cast<const char *>(m_buf.data());
535 }
536
capacity()537 size_t capacity() { return buf_length; }
538
539 /** Returns actual size of the buffer that has data */
size()540 size_t size() { return m_ptr - m_buf.data(); }
541
to_slice()542 rocksdb::Slice to_slice() { return rocksdb::Slice(data(), size()); }
543
544 private:
545 std::array<uchar, buf_length> m_buf;
546 uchar *m_ptr;
547 };
548
549 } // namespace myrocks
550