1 /* 2 Portions Copyright (c) 2016-Present, Facebook, Inc. 3 Portions Copyright (c) 2012,2013 Monty Program Ab 4 5 This program is free software; you can redistribute it and/or modify 6 it under the terms of the GNU General Public License as published by 7 the Free Software Foundation; version 2 of the License. 8 9 This program is distributed in the hope that it will be useful, 10 but WITHOUT ANY WARRANTY; without even the implied warranty of 11 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 12 GNU General Public License for more details. 13 14 You should have received a copy of the GNU General Public License 15 along with this program; if not, write to the Free Software 16 Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02111-1301 USA */ 17 #pragma once 18 19 #include <algorithm> 20 #include <string> 21 #include <vector> 22 23 #ifdef _WIN32 24 #include <stdlib.h> 25 #define htobe64 _byteswap_uint64 26 #define be64toh _byteswap_uint64 27 #define htobe32 _byteswap_ulong 28 #define be32toh _byteswap_ulong 29 #define htobe16 _byteswap_ushort 30 #define be16toh _byteswap_ushort 31 #endif 32 33 #if defined(__APPLE__) 34 #include <libkern/OSByteOrder.h> 35 #define htobe64(x) OSSwapHostToBigInt64(x) 36 #define be64toh(x) OSSwapBigToHostInt64(x) 37 #define htobe32(x) OSSwapHostToBigInt32(x) 38 #define be32toh(x) OSSwapBigToHostInt32(x) 39 #define htobe16(x) OSSwapHostToBigInt16(x) 40 #define be16toh(x) OSSwapBigToHostInt16(x) 41 #endif 42 43 /* MySQL header files */ 44 #include "./my_global.h" 45 46 /* MyRocks header files */ 47 #include "./rdb_global.h" 48 #include "./rdb_utils.h" 49 50 /* RocksDB header files */ 51 #include "rocksdb/slice.h" 52 #include "rocksdb/status.h" 53 54 namespace myrocks { 55 56 /* 57 Basic composition functions for a network buffer presented as a MySQL String 58 ("netstr") which stores data in Network Byte Order (Big Endian). 59 */ 60 61 inline void rdb_netstr_append_uint64(my_core::String *const out_netstr, 62 const uint64 val) { 63 DBUG_ASSERT(out_netstr != nullptr); 64 65 // Convert from host machine byte order (usually Little Endian) to network 66 // byte order (Big Endian). 67 uint64 net_val = htobe64(val); 68 out_netstr->append(reinterpret_cast<char *>(&net_val), sizeof(net_val)); 69 } 70 71 inline void rdb_netstr_append_uint32(my_core::String *const out_netstr, 72 const uint32 val) { 73 DBUG_ASSERT(out_netstr != nullptr); 74 75 // Convert from host machine byte order (usually Little Endian) to network 76 // byte order (Big Endian). 77 uint32 net_val = htobe32(val); 78 out_netstr->append(reinterpret_cast<char *>(&net_val), sizeof(net_val)); 79 } 80 81 inline void rdb_netstr_append_uint16(my_core::String *const out_netstr, 82 const uint16 val) { 83 DBUG_ASSERT(out_netstr != nullptr); 84 85 // Convert from host machine byte order (usually Little Endian) to network 86 // byte order (Big Endian). 87 uint16 net_val = htobe16(val); 88 out_netstr->append(reinterpret_cast<char *>(&net_val), sizeof(net_val)); 89 } 90 91 /* 92 Basic network buffer ("netbuf") write helper functions. 93 */ 94 95 inline void rdb_netbuf_store_uint64(uchar *const dst_netbuf, const uint64 n) { 96 DBUG_ASSERT(dst_netbuf != nullptr); 97 98 // Convert from host byte order (usually Little Endian) to network byte order 99 // (Big Endian). 100 uint64 net_val = htobe64(n); 101 memcpy(dst_netbuf, &net_val, sizeof(net_val)); 102 } 103 104 inline void rdb_netbuf_store_uint32(uchar *const dst_netbuf, const uint32 n) { 105 DBUG_ASSERT(dst_netbuf != nullptr); 106 107 // Convert from host byte order (usually Little Endian) to network byte order 108 // (Big Endian). 109 uint32 net_val = htobe32(n); 110 memcpy(dst_netbuf, &net_val, sizeof(net_val)); 111 } 112 113 inline void rdb_netbuf_store_uint16(uchar *const dst_netbuf, const uint16 n) { 114 DBUG_ASSERT(dst_netbuf != nullptr); 115 116 // Convert from host byte order (usually Little Endian) to network byte order 117 // (Big Endian). 118 uint16 net_val = htobe16(n); 119 memcpy(dst_netbuf, &net_val, sizeof(net_val)); 120 } 121 122 inline void rdb_netbuf_store_byte(uchar *const dst_netbuf, const uchar c) { 123 DBUG_ASSERT(dst_netbuf != nullptr); 124 125 *dst_netbuf = c; 126 } 127 128 inline void rdb_netbuf_store_index(uchar *const dst_netbuf, 129 const uint32 number) { 130 DBUG_ASSERT(dst_netbuf != nullptr); 131 132 rdb_netbuf_store_uint32(dst_netbuf, number); 133 } 134 135 /* 136 Basic conversion helper functions from network byte order (Big Endian) to host 137 machine byte order (usually Little Endian). 138 */ 139 140 inline uint64 rdb_netbuf_to_uint64(const uchar *const netbuf) { 141 DBUG_ASSERT(netbuf != nullptr); 142 143 uint64 net_val; 144 memcpy(&net_val, netbuf, sizeof(net_val)); 145 146 // Convert from network byte order (Big Endian) to host machine byte order 147 // (usually Little Endian). 148 return be64toh(net_val); 149 } 150 151 inline uint32 rdb_netbuf_to_uint32(const uchar *const netbuf) { 152 DBUG_ASSERT(netbuf != nullptr); 153 154 uint32 net_val; 155 memcpy(&net_val, netbuf, sizeof(net_val)); 156 157 // Convert from network byte order (Big Endian) to host machine byte order 158 // (usually Little Endian). 159 return be32toh(net_val); 160 } 161 162 inline uint16 rdb_netbuf_to_uint16(const uchar *const netbuf) { 163 DBUG_ASSERT(netbuf != nullptr); 164 165 uint16 net_val; 166 memcpy(&net_val, netbuf, sizeof(net_val)); 167 168 // Convert from network byte order (Big Endian) to host machine byte order 169 // (usually Little Endian). 170 return be16toh(net_val); 171 } 172 173 inline uchar rdb_netbuf_to_byte(const uchar *const netbuf) { 174 DBUG_ASSERT(netbuf != nullptr); 175 176 return (uchar)netbuf[0]; 177 } 178 179 /* 180 Basic network buffer ("netbuf") read helper functions. 181 Network buffer stores data in Network Byte Order (Big Endian). 182 NB: The netbuf is passed as an input/output param, hence after reading, 183 the netbuf pointer gets advanced to the following byte. 184 */ 185 186 inline uint64 rdb_netbuf_read_uint64(const uchar **netbuf_ptr) { 187 DBUG_ASSERT(netbuf_ptr != nullptr); 188 189 // Convert from network byte order (Big Endian) to host machine byte order 190 // (usually Little Endian). 191 const uint64 host_val = rdb_netbuf_to_uint64(*netbuf_ptr); 192 193 // Advance pointer. 194 *netbuf_ptr += sizeof(host_val); 195 196 return host_val; 197 } 198 199 inline uint32 rdb_netbuf_read_uint32(const uchar **netbuf_ptr) { 200 DBUG_ASSERT(netbuf_ptr != nullptr); 201 202 // Convert from network byte order (Big Endian) to host machine byte order 203 // (usually Little Endian). 204 const uint32 host_val = rdb_netbuf_to_uint32(*netbuf_ptr); 205 206 // Advance pointer. 207 *netbuf_ptr += sizeof(host_val); 208 209 return host_val; 210 } 211 212 inline uint16 rdb_netbuf_read_uint16(const uchar **netbuf_ptr) { 213 DBUG_ASSERT(netbuf_ptr != nullptr); 214 215 // Convert from network byte order (Big Endian) to host machine byte order 216 // (usually Little Endian). 217 const uint16 host_val = rdb_netbuf_to_uint16(*netbuf_ptr); 218 219 // Advance pointer. 220 *netbuf_ptr += sizeof(host_val); 221 222 return host_val; 223 } 224 225 inline void rdb_netbuf_read_gl_index(const uchar **netbuf_ptr, 226 GL_INDEX_ID *const gl_index_id) { 227 DBUG_ASSERT(gl_index_id != nullptr); 228 DBUG_ASSERT(netbuf_ptr != nullptr); 229 230 gl_index_id->cf_id = rdb_netbuf_read_uint32(netbuf_ptr); 231 gl_index_id->index_id = rdb_netbuf_read_uint32(netbuf_ptr); 232 } 233 234 /* 235 A simple string reader: 236 - it keeps position within the string that we read from 237 - it prevents one from reading beyond the end of the string. 238 */ 239 240 class Rdb_string_reader { 241 const char *m_ptr; 242 uint m_len; 243 244 private: 245 Rdb_string_reader &operator=(const Rdb_string_reader &) = default; 246 247 public: 248 Rdb_string_reader(const Rdb_string_reader &) = default; 249 /* named constructor */ 250 static Rdb_string_reader read_or_empty(const rocksdb::Slice *const slice) { 251 if (!slice) { 252 return Rdb_string_reader(""); 253 } else { 254 return Rdb_string_reader(slice); 255 } 256 } 257 258 explicit Rdb_string_reader(const std::string &str) { 259 m_len = str.length(); 260 if (m_len) { 261 m_ptr = &str.at(0); 262 } else { 263 /* 264 One can a create a Rdb_string_reader for reading from an empty string 265 (although attempts to read anything will fail). 266 We must not access str.at(0), since len==0, we can set ptr to any 267 value. 268 */ 269 m_ptr = nullptr; 270 } 271 } 272 273 explicit Rdb_string_reader(const rocksdb::Slice *const slice) { 274 m_ptr = slice->data(); 275 m_len = slice->size(); 276 } 277 278 /* 279 Read the next @param size bytes. Returns pointer to the bytes read, or 280 nullptr if the remaining string doesn't have that many bytes. 281 */ 282 const char *read(const uint size) { 283 const char *res; 284 if (m_len < size) { 285 res = nullptr; 286 } else { 287 res = m_ptr; 288 m_ptr += size; 289 m_len -= size; 290 } 291 return res; 292 } 293 294 bool read_uint8(uint *const res) { 295 const uchar *p; 296 if (!(p = reinterpret_cast<const uchar *>(read(1)))) { 297 return true; // error 298 } else { 299 *res = *p; 300 return false; // Ok 301 } 302 } 303 304 bool read_uint16(uint *const res) { 305 const uchar *p; 306 if (!(p = reinterpret_cast<const uchar *>(read(2)))) { 307 return true; // error 308 } else { 309 *res = rdb_netbuf_to_uint16(p); 310 return false; // Ok 311 } 312 } 313 314 bool read_uint64(uint64 *const res) { 315 const uchar *p; 316 if (!(p = reinterpret_cast<const uchar *>(read(sizeof(uint64))))) { 317 return true; // error 318 } else { 319 *res = rdb_netbuf_to_uint64(p); 320 return false; // Ok 321 } 322 } 323 324 uint remaining_bytes() const { return m_len; } 325 326 /* 327 Return pointer to data that will be read by next read() call (if there is 328 nothing left to read, returns pointer to beyond the end of previous read() 329 call) 330 */ 331 const char *get_current_ptr() const { return m_ptr; } 332 }; 333 334 /* 335 @brief 336 A buffer one can write the data to. 337 338 @detail 339 Suggested usage pattern: 340 341 writer->clear(); 342 writer->write_XXX(...); 343 ... 344 // Ok, writer->ptr() points to the data written so far, 345 // and writer->get_current_pos() is the length of the data 346 347 */ 348 349 class Rdb_string_writer { 350 std::vector<uchar> m_data; 351 352 public: 353 Rdb_string_writer(const Rdb_string_writer &) = delete; 354 Rdb_string_writer &operator=(const Rdb_string_writer &) = delete; 355 Rdb_string_writer() = default; 356 357 void clear() { m_data.clear(); } 358 void write_uint8(const uint val) { 359 m_data.push_back(static_cast<uchar>(val)); 360 } 361 362 void write_uint16(const uint val) { 363 const auto size = m_data.size(); 364 m_data.resize(size + 2); 365 rdb_netbuf_store_uint16(m_data.data() + size, val); 366 } 367 368 void write_uint32(const uint val) { 369 const auto size = m_data.size(); 370 m_data.resize(size + 4); 371 rdb_netbuf_store_uint32(m_data.data() + size, val); 372 } 373 374 void write(const uchar *const new_data, const size_t len) { 375 DBUG_ASSERT(new_data != nullptr); 376 m_data.insert(m_data.end(), new_data, new_data + len); 377 } 378 379 uchar *ptr() { return m_data.data(); } 380 size_t get_current_pos() const { return m_data.size(); } 381 382 void write_uint8_at(const size_t pos, const uint new_val) { 383 // This function will only overwrite what was written 384 DBUG_ASSERT(pos < get_current_pos()); 385 m_data.data()[pos] = new_val; 386 } 387 388 void write_uint16_at(const size_t pos, const uint new_val) { 389 // This function will only overwrite what was written 390 DBUG_ASSERT(pos < get_current_pos() && (pos + 1) < get_current_pos()); 391 rdb_netbuf_store_uint16(m_data.data() + pos, new_val); 392 } 393 394 void truncate(const size_t pos) { 395 DBUG_ASSERT(pos < m_data.size()); 396 m_data.resize(pos); 397 } 398 399 void allocate(const size_t len, const uchar val = 0) { 400 DBUG_ASSERT(len > 0); 401 m_data.resize(m_data.size() + len, val); 402 } 403 404 /* 405 An awful hack to deallocate the buffer without relying on the deconstructor. 406 This is needed to suppress valgrind errors in rocksdb.partition 407 */ 408 void free() { std::vector<uchar>().swap(m_data); } 409 }; 410 411 /* 412 A helper class for writing bits into Rdb_string_writer. 413 414 The class assumes (but doesn't check) that nobody tries to write 415 anything to the Rdb_string_writer that it is writing to. 416 */ 417 class Rdb_bit_writer { 418 Rdb_string_writer *m_writer; 419 uchar m_offset; 420 421 public: 422 Rdb_bit_writer(const Rdb_bit_writer &) = delete; 423 Rdb_bit_writer &operator=(const Rdb_bit_writer &) = delete; 424 425 explicit Rdb_bit_writer(Rdb_string_writer *writer_arg) 426 : m_writer(writer_arg), m_offset(0) {} 427 428 void write(uint size, const uint value) { 429 DBUG_ASSERT((value & ((1 << size) - 1)) == value); 430 431 while (size > 0) { 432 if (m_offset == 0) { 433 m_writer->write_uint8(0); 434 } 435 // number of bits to put in this byte 436 const uint bits = std::min(size, (uint)(8 - m_offset)); 437 uchar *const last_byte = 438 m_writer->ptr() + m_writer->get_current_pos() - 1; 439 *last_byte |= (uchar)((value >> (size - bits)) & ((1 << bits) - 1)) 440 << m_offset; 441 size -= bits; 442 m_offset = (m_offset + bits) & 0x7; 443 } 444 } 445 }; 446 447 class Rdb_bit_reader { 448 const uchar *m_cur; 449 uchar m_offset; 450 uint m_ret; 451 Rdb_string_reader *const m_reader; 452 453 public: 454 Rdb_bit_reader(const Rdb_bit_reader &) = delete; 455 Rdb_bit_reader &operator=(const Rdb_bit_reader &) = delete; 456 457 explicit Rdb_bit_reader(Rdb_string_reader *const reader) 458 : m_cur(nullptr), m_offset(0), m_reader(reader) {} 459 460 // Returns a pointer to an uint containing the bits read. On subsequent 461 // reads, the value being pointed to will be overwritten. Returns nullptr 462 // on failure. 463 uint *read(uint size) { 464 m_ret = 0; 465 DBUG_ASSERT(size <= 32); 466 467 while (size > 0) { 468 if (m_offset == 0) { 469 m_cur = (const uchar *)m_reader->read(1); 470 if (m_cur == nullptr) { 471 return nullptr; 472 } 473 } 474 // how many bits from the current byte? 475 const uint bits = std::min((uint)(8 - m_offset), size); 476 m_ret <<= bits; 477 m_ret |= (*m_cur >> m_offset) & ((1 << bits) - 1); 478 size -= bits; 479 m_offset = (m_offset + bits) & 0x7; 480 } 481 482 return &m_ret; 483 } 484 }; 485 486 template <size_t buf_length> 487 class Rdb_buf_writer { 488 public: 489 Rdb_buf_writer(const Rdb_buf_writer &) = delete; 490 Rdb_buf_writer &operator=(const Rdb_buf_writer &) = delete; 491 Rdb_buf_writer() { reset(); } 492 493 void write_uint32(const uint32 n) { 494 DBUG_ASSERT(m_ptr + sizeof(n) <= m_buf.data() + buf_length); 495 rdb_netbuf_store_uint32(m_ptr, n); 496 m_ptr += sizeof(n); 497 } 498 499 void write_uint64(const uint64 n) { 500 DBUG_ASSERT(m_ptr + sizeof(n) <= m_buf.data() + buf_length); 501 rdb_netbuf_store_uint64(m_ptr, n); 502 m_ptr += sizeof(n); 503 } 504 505 void write_uint16(const uint16 n) { 506 DBUG_ASSERT(m_ptr + sizeof(n) <= m_buf.data() + buf_length); 507 rdb_netbuf_store_uint16(m_ptr, n); 508 m_ptr += sizeof(n); 509 } 510 511 void write_byte(const uchar c) { 512 DBUG_ASSERT(m_ptr + sizeof(c) <= m_buf.data() + buf_length); 513 rdb_netbuf_store_byte(m_ptr, c); 514 m_ptr += sizeof(c); 515 } 516 517 void write_index(const uint32 n) { write_uint32(n); } 518 519 void write(const char *buf, const size_t size) { 520 DBUG_ASSERT(m_ptr + size <= m_buf.data() + buf_length); 521 memcpy(m_ptr, buf, size); 522 m_ptr += size; 523 } 524 525 void write(const uchar *buf, const size_t size) { 526 DBUG_ASSERT(m_ptr + size <= m_buf.data() + buf_length); 527 memcpy(m_ptr, buf, size); 528 m_ptr += size; 529 } 530 531 void reset() { m_ptr = m_buf.data(); } 532 533 const char *data() const { 534 return reinterpret_cast<const char *>(m_buf.data()); 535 } 536 537 size_t capacity() { return buf_length; } 538 539 /** Returns actual size of the buffer that has data */ 540 size_t size() { return m_ptr - m_buf.data(); } 541 542 rocksdb::Slice to_slice() { return rocksdb::Slice(data(), size()); } 543 544 private: 545 std::array<uchar, buf_length> m_buf; 546 uchar *m_ptr; 547 }; 548 549 } // namespace myrocks 550