1 /***************************************************************************** 2 3 Copyright (c) 2005, 2016, Oracle and/or its affiliates. All Rights Reserved. 4 Copyright (c) 2012, Facebook Inc. 5 6 This program is free software; you can redistribute it and/or modify 7 it under the terms of the GNU General Public License, version 2.0, 8 as published by the Free Software Foundation. 9 10 This program is also distributed with certain software (including 11 but not limited to OpenSSL) that is licensed under separate terms, 12 as designated in a particular file or component or in included license 13 documentation. The authors of MySQL hereby grant you an additional 14 permission to link the program and your derivative works with the 15 separately licensed software that they have included with MySQL. 16 17 This program is distributed in the hope that it will be useful, 18 but WITHOUT ANY WARRANTY; without even the implied warranty of 19 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 20 GNU General Public License, version 2.0, for more details. 21 22 You should have received a copy of the GNU General Public License along with 23 this program; if not, write to the Free Software Foundation, Inc., 24 51 Franklin Street, Suite 500, Boston, MA 02110-1335 USA 25 26 *****************************************************************************/ 27 28 /**************************************************//** 29 @file include/page0zip.h 30 Compressed page interface 31 32 Created June 2005 by Marko Makela 33 *******************************************************/ 34 35 #ifndef page0zip_h 36 #define page0zip_h 37 38 #ifdef UNIV_MATERIALIZE 39 # undef UNIV_INLINE 40 # define UNIV_INLINE 41 #endif 42 43 #include "mtr0types.h" 44 #include "page0types.h" 45 #include "buf0types.h" 46 #include "dict0types.h" 47 #include "srv0srv.h" 48 #include "trx0types.h" 49 #include "mem0mem.h" 50 51 /* Compression level to be used by zlib. Settable by user. */ 52 extern uint page_zip_level; 53 54 /* Default compression level. */ 55 #define DEFAULT_COMPRESSION_LEVEL 6 56 57 /* Whether or not to log compressed page images to avoid possible 58 compression algorithm changes in zlib. */ 59 extern my_bool page_zip_log_pages; 60 61 /**********************************************************************//** 62 Determine the size of a compressed page in bytes. 63 @return size in bytes */ 64 UNIV_INLINE 65 ulint 66 page_zip_get_size( 67 /*==============*/ 68 const page_zip_des_t* page_zip) /*!< in: compressed page */ 69 MY_ATTRIBUTE((nonnull, pure)); 70 /**********************************************************************//** 71 Set the size of a compressed page in bytes. */ 72 UNIV_INLINE 73 void 74 page_zip_set_size( 75 /*==============*/ 76 page_zip_des_t* page_zip, /*!< in/out: compressed page */ 77 ulint size); /*!< in: size in bytes */ 78 79 #ifndef UNIV_HOTBACKUP 80 /**********************************************************************//** 81 Determine if a record is so big that it needs to be stored externally. 82 @return FALSE if the entire record can be stored locally on the page */ 83 UNIV_INLINE 84 ibool 85 page_zip_rec_needs_ext( 86 /*===================*/ 87 ulint rec_size, /*!< in: length of the record in bytes */ 88 ulint comp, /*!< in: nonzero=compact format */ 89 ulint n_fields, /*!< in: number of fields in the record; 90 ignored if zip_size == 0 */ 91 ulint zip_size) /*!< in: compressed page size in bytes, or 0 */ 92 MY_ATTRIBUTE((const)); 93 94 /**********************************************************************//** 95 Determine the guaranteed free space on an empty page. 96 @return minimum payload size on the page */ 97 UNIV_INTERN 98 ulint 99 page_zip_empty_size( 100 /*================*/ 101 ulint n_fields, /*!< in: number of columns in the index */ 102 ulint zip_size) /*!< in: compressed page size in bytes */ 103 MY_ATTRIBUTE((const)); 104 #endif /* !UNIV_HOTBACKUP */ 105 106 /**********************************************************************//** 107 Initialize a compressed page descriptor. */ 108 UNIV_INLINE 109 void 110 page_zip_des_init( 111 /*==============*/ 112 page_zip_des_t* page_zip); /*!< in/out: compressed page 113 descriptor */ 114 115 /**********************************************************************//** 116 Configure the zlib allocator to use the given memory heap. */ 117 UNIV_INTERN 118 void 119 page_zip_set_alloc( 120 /*===============*/ 121 void* stream, /*!< in/out: zlib stream */ 122 mem_heap_t* heap); /*!< in: memory heap to use */ 123 124 /**********************************************************************//** 125 Compress a page. 126 @return TRUE on success, FALSE on failure; page_zip will be left 127 intact on failure. */ 128 UNIV_INTERN 129 ibool 130 page_zip_compress( 131 /*==============*/ 132 page_zip_des_t* page_zip,/*!< in: size; out: data, n_blobs, 133 m_start, m_end, m_nonempty */ 134 const page_t* page, /*!< in: uncompressed page */ 135 dict_index_t* index, /*!< in: index of the B-tree node */ 136 ulint level, /*!< in: compression level */ 137 mtr_t* mtr) /*!< in: mini-transaction, or NULL */ 138 MY_ATTRIBUTE((nonnull(1,2,3))); 139 140 /**********************************************************************//** 141 Decompress a page. This function should tolerate errors on the compressed 142 page. Instead of letting assertions fail, it will return FALSE if an 143 inconsistency is detected. 144 @return TRUE on success, FALSE on failure */ 145 UNIV_INTERN 146 ibool 147 page_zip_decompress( 148 /*================*/ 149 page_zip_des_t* page_zip,/*!< in: data, ssize; 150 out: m_start, m_end, m_nonempty, n_blobs */ 151 page_t* page, /*!< out: uncompressed page, may be trashed */ 152 ibool all) /*!< in: TRUE=decompress the whole page; 153 FALSE=verify but do not copy some 154 page header fields that should not change 155 after page creation */ 156 MY_ATTRIBUTE((nonnull(1,2))); 157 158 #ifdef UNIV_DEBUG 159 /**********************************************************************//** 160 Validate a compressed page descriptor. 161 @return TRUE if ok */ 162 UNIV_INLINE 163 ibool 164 page_zip_simple_validate( 165 /*=====================*/ 166 const page_zip_des_t* page_zip); /*!< in: compressed page 167 descriptor */ 168 #endif /* UNIV_DEBUG */ 169 170 #ifdef UNIV_ZIP_DEBUG 171 /**********************************************************************//** 172 Check that the compressed and decompressed pages match. 173 @return TRUE if valid, FALSE if not */ 174 UNIV_INTERN 175 ibool 176 page_zip_validate_low( 177 /*==================*/ 178 const page_zip_des_t* page_zip,/*!< in: compressed page */ 179 const page_t* page, /*!< in: uncompressed page */ 180 const dict_index_t* index, /*!< in: index of the page, if known */ 181 ibool sloppy) /*!< in: FALSE=strict, 182 TRUE=ignore the MIN_REC_FLAG */ 183 MY_ATTRIBUTE((nonnull(1,2))); 184 /**********************************************************************//** 185 Check that the compressed and decompressed pages match. */ 186 UNIV_INTERN 187 ibool 188 page_zip_validate( 189 /*==============*/ 190 const page_zip_des_t* page_zip,/*!< in: compressed page */ 191 const page_t* page, /*!< in: uncompressed page */ 192 const dict_index_t* index) /*!< in: index of the page, if known */ 193 MY_ATTRIBUTE((nonnull(1,2))); 194 #endif /* UNIV_ZIP_DEBUG */ 195 196 /**********************************************************************//** 197 Determine how big record can be inserted without recompressing the page. 198 @return a positive number indicating the maximum size of a record 199 whose insertion is guaranteed to succeed, or zero or negative */ 200 UNIV_INLINE 201 lint 202 page_zip_max_ins_size( 203 /*==================*/ 204 const page_zip_des_t* page_zip,/*!< in: compressed page */ 205 ibool is_clust)/*!< in: TRUE if clustered index */ 206 MY_ATTRIBUTE((nonnull, pure)); 207 208 /**********************************************************************//** 209 Determine if enough space is available in the modification log. 210 @return TRUE if page_zip_write_rec() will succeed */ 211 UNIV_INLINE 212 ibool 213 page_zip_available( 214 /*===============*/ 215 const page_zip_des_t* page_zip,/*!< in: compressed page */ 216 ibool is_clust,/*!< in: TRUE if clustered index */ 217 ulint length, /*!< in: combined size of the record */ 218 ulint create) /*!< in: nonzero=add the record to 219 the heap */ 220 MY_ATTRIBUTE((nonnull, pure)); 221 222 /**********************************************************************//** 223 Write data to the uncompressed header portion of a page. The data must 224 already have been written to the uncompressed page. */ 225 UNIV_INLINE 226 void 227 page_zip_write_header( 228 /*==================*/ 229 page_zip_des_t* page_zip,/*!< in/out: compressed page */ 230 const byte* str, /*!< in: address on the uncompressed page */ 231 ulint length, /*!< in: length of the data */ 232 mtr_t* mtr) /*!< in: mini-transaction, or NULL */ 233 MY_ATTRIBUTE((nonnull(1,2))); 234 235 /**********************************************************************//** 236 Write an entire record on the compressed page. The data must already 237 have been written to the uncompressed page. */ 238 UNIV_INTERN 239 void 240 page_zip_write_rec( 241 /*===============*/ 242 page_zip_des_t* page_zip,/*!< in/out: compressed page */ 243 const byte* rec, /*!< in: record being written */ 244 dict_index_t* index, /*!< in: the index the record belongs to */ 245 const ulint* offsets,/*!< in: rec_get_offsets(rec, index) */ 246 ulint create) /*!< in: nonzero=insert, zero=update */ 247 MY_ATTRIBUTE((nonnull)); 248 249 /***********************************************************//** 250 Parses a log record of writing a BLOB pointer of a record. 251 @return end of log record or NULL */ 252 UNIV_INTERN 253 byte* 254 page_zip_parse_write_blob_ptr( 255 /*==========================*/ 256 byte* ptr, /*!< in: redo log buffer */ 257 byte* end_ptr,/*!< in: redo log buffer end */ 258 page_t* page, /*!< in/out: uncompressed page */ 259 page_zip_des_t* page_zip);/*!< in/out: compressed page */ 260 261 /**********************************************************************//** 262 Write a BLOB pointer of a record on the leaf page of a clustered index. 263 The information must already have been updated on the uncompressed page. */ 264 UNIV_INTERN 265 void 266 page_zip_write_blob_ptr( 267 /*====================*/ 268 page_zip_des_t* page_zip,/*!< in/out: compressed page */ 269 const byte* rec, /*!< in/out: record whose data is being 270 written */ 271 dict_index_t* index, /*!< in: index of the page */ 272 const ulint* offsets,/*!< in: rec_get_offsets(rec, index) */ 273 ulint n, /*!< in: column index */ 274 mtr_t* mtr) /*!< in: mini-transaction handle, 275 or NULL if no logging is needed */ 276 MY_ATTRIBUTE((nonnull(1,2,3,4))); 277 278 /***********************************************************//** 279 Parses a log record of writing the node pointer of a record. 280 @return end of log record or NULL */ 281 UNIV_INTERN 282 byte* 283 page_zip_parse_write_node_ptr( 284 /*==========================*/ 285 byte* ptr, /*!< in: redo log buffer */ 286 byte* end_ptr,/*!< in: redo log buffer end */ 287 page_t* page, /*!< in/out: uncompressed page */ 288 page_zip_des_t* page_zip);/*!< in/out: compressed page */ 289 290 /**********************************************************************//** 291 Write the node pointer of a record on a non-leaf compressed page. */ 292 UNIV_INTERN 293 void 294 page_zip_write_node_ptr( 295 /*====================*/ 296 page_zip_des_t* page_zip,/*!< in/out: compressed page */ 297 byte* rec, /*!< in/out: record */ 298 ulint size, /*!< in: data size of rec */ 299 ulint ptr, /*!< in: node pointer */ 300 mtr_t* mtr) /*!< in: mini-transaction, or NULL */ 301 MY_ATTRIBUTE((nonnull(1,2))); 302 303 /**********************************************************************//** 304 Write the trx_id and roll_ptr of a record on a B-tree leaf node page. */ 305 UNIV_INTERN 306 void 307 page_zip_write_trx_id_and_roll_ptr( 308 /*===============================*/ 309 page_zip_des_t* page_zip,/*!< in/out: compressed page */ 310 byte* rec, /*!< in/out: record */ 311 const ulint* offsets,/*!< in: rec_get_offsets(rec, index) */ 312 ulint trx_id_col,/*!< in: column number of TRX_ID in rec */ 313 trx_id_t trx_id, /*!< in: transaction identifier */ 314 roll_ptr_t roll_ptr)/*!< in: roll_ptr */ 315 MY_ATTRIBUTE((nonnull)); 316 317 /**********************************************************************//** 318 Write the "deleted" flag of a record on a compressed page. The flag must 319 already have been written on the uncompressed page. */ 320 UNIV_INTERN 321 void 322 page_zip_rec_set_deleted( 323 /*=====================*/ 324 page_zip_des_t* page_zip,/*!< in/out: compressed page */ 325 const byte* rec, /*!< in: record on the uncompressed page */ 326 ulint flag) /*!< in: the deleted flag (nonzero=TRUE) */ 327 MY_ATTRIBUTE((nonnull)); 328 329 /**********************************************************************//** 330 Write the "owned" flag of a record on a compressed page. The n_owned field 331 must already have been written on the uncompressed page. */ 332 UNIV_INTERN 333 void 334 page_zip_rec_set_owned( 335 /*===================*/ 336 page_zip_des_t* page_zip,/*!< in/out: compressed page */ 337 const byte* rec, /*!< in: record on the uncompressed page */ 338 ulint flag) /*!< in: the owned flag (nonzero=TRUE) */ 339 MY_ATTRIBUTE((nonnull)); 340 341 /**********************************************************************//** 342 Insert a record to the dense page directory. */ 343 UNIV_INTERN 344 void 345 page_zip_dir_insert( 346 /*================*/ 347 page_zip_des_t* page_zip,/*!< in/out: compressed page */ 348 const byte* prev_rec,/*!< in: record after which to insert */ 349 const byte* free_rec,/*!< in: record from which rec was 350 allocated, or NULL */ 351 byte* rec); /*!< in: record to insert */ 352 353 /**********************************************************************//** 354 Shift the dense page directory and the array of BLOB pointers 355 when a record is deleted. */ 356 UNIV_INTERN 357 void 358 page_zip_dir_delete( 359 /*================*/ 360 page_zip_des_t* page_zip, /*!< in/out: compressed page */ 361 byte* rec, /*!< in: deleted record */ 362 const dict_index_t* index, /*!< in: index of rec */ 363 const ulint* offsets, /*!< in: rec_get_offsets(rec) */ 364 const byte* free) /*!< in: previous start of 365 the free list */ 366 MY_ATTRIBUTE((nonnull(1,2,3,4))); 367 368 /**********************************************************************//** 369 Add a slot to the dense page directory. */ 370 UNIV_INTERN 371 void 372 page_zip_dir_add_slot( 373 /*==================*/ 374 page_zip_des_t* page_zip, /*!< in/out: compressed page */ 375 ulint is_clustered) /*!< in: nonzero for clustered index, 376 zero for others */ 377 MY_ATTRIBUTE((nonnull)); 378 379 /***********************************************************//** 380 Parses a log record of writing to the header of a page. 381 @return end of log record or NULL */ 382 UNIV_INTERN 383 byte* 384 page_zip_parse_write_header( 385 /*========================*/ 386 byte* ptr, /*!< in: redo log buffer */ 387 byte* end_ptr,/*!< in: redo log buffer end */ 388 page_t* page, /*!< in/out: uncompressed page */ 389 page_zip_des_t* page_zip);/*!< in/out: compressed page */ 390 391 /**********************************************************************//** 392 Write data to the uncompressed header portion of a page. The data must 393 already have been written to the uncompressed page. 394 However, the data portion of the uncompressed page may differ from 395 the compressed page when a record is being inserted in 396 page_cur_insert_rec_low(). */ 397 UNIV_INLINE 398 void 399 page_zip_write_header( 400 /*==================*/ 401 page_zip_des_t* page_zip,/*!< in/out: compressed page */ 402 const byte* str, /*!< in: address on the uncompressed page */ 403 ulint length, /*!< in: length of the data */ 404 mtr_t* mtr) /*!< in: mini-transaction, or NULL */ 405 MY_ATTRIBUTE((nonnull(1,2))); 406 407 /**********************************************************************//** 408 Reorganize and compress a page. This is a low-level operation for 409 compressed pages, to be used when page_zip_compress() fails. 410 On success, a redo log entry MLOG_ZIP_PAGE_COMPRESS will be written. 411 The function btr_page_reorganize() should be preferred whenever possible. 412 IMPORTANT: if page_zip_reorganize() is invoked on a leaf page of a 413 non-clustered index, the caller must update the insert buffer free 414 bits in the same mini-transaction in such a way that the modification 415 will be redo-logged. 416 @return TRUE on success, FALSE on failure; page_zip will be left 417 intact on failure, but page will be overwritten. */ 418 UNIV_INTERN 419 ibool 420 page_zip_reorganize( 421 /*================*/ 422 buf_block_t* block, /*!< in/out: page with compressed page; 423 on the compressed page, in: size; 424 out: data, n_blobs, 425 m_start, m_end, m_nonempty */ 426 dict_index_t* index, /*!< in: index of the B-tree node */ 427 mtr_t* mtr) /*!< in: mini-transaction */ 428 MY_ATTRIBUTE((nonnull)); 429 #ifndef UNIV_HOTBACKUP 430 /**********************************************************************//** 431 Copy the records of a page byte for byte. Do not copy the page header 432 or trailer, except those B-tree header fields that are directly 433 related to the storage of records. Also copy PAGE_MAX_TRX_ID. 434 NOTE: The caller must update the lock table and the adaptive hash index. */ 435 UNIV_INTERN 436 void 437 page_zip_copy_recs( 438 /*===============*/ 439 page_zip_des_t* page_zip, /*!< out: copy of src_zip 440 (n_blobs, m_start, m_end, 441 m_nonempty, data[0..size-1]) */ 442 page_t* page, /*!< out: copy of src */ 443 const page_zip_des_t* src_zip, /*!< in: compressed page */ 444 const page_t* src, /*!< in: page */ 445 dict_index_t* index, /*!< in: index of the B-tree */ 446 mtr_t* mtr) /*!< in: mini-transaction */ 447 MY_ATTRIBUTE((nonnull)); 448 #endif /* !UNIV_HOTBACKUP */ 449 450 /**********************************************************************//** 451 Parses a log record of compressing an index page. 452 @return end of log record or NULL */ 453 UNIV_INTERN 454 byte* 455 page_zip_parse_compress( 456 /*====================*/ 457 byte* ptr, /*!< in: buffer */ 458 byte* end_ptr,/*!< in: buffer end */ 459 page_t* page, /*!< out: uncompressed page */ 460 page_zip_des_t* page_zip)/*!< out: compressed page */ 461 MY_ATTRIBUTE((nonnull(1,2))); 462 463 /**********************************************************************//** 464 Calculate the compressed page checksum. 465 @return page checksum */ 466 UNIV_INTERN 467 ulint 468 page_zip_calc_checksum( 469 /*===================*/ 470 const void* data, /*!< in: compressed page */ 471 ulint size, /*!< in: size of compressed page */ 472 srv_checksum_algorithm_t algo) /*!< in: algorithm to use */ 473 MY_ATTRIBUTE((nonnull)); 474 475 /**********************************************************************//** 476 Verify a compressed page's checksum. 477 @return TRUE if the stored checksum is valid according to the value of 478 innodb_checksum_algorithm */ 479 UNIV_INTERN 480 ibool 481 page_zip_verify_checksum( 482 /*=====================*/ 483 const void* data, /*!< in: compressed page */ 484 ulint size); /*!< in: size of compressed page */ 485 /**********************************************************************//** 486 Write a log record of compressing an index page without the data on the page. */ 487 UNIV_INLINE 488 void 489 page_zip_compress_write_log_no_data( 490 /*================================*/ 491 ulint level, /*!< in: compression level */ 492 const page_t* page, /*!< in: page that is compressed */ 493 dict_index_t* index, /*!< in: index */ 494 mtr_t* mtr); /*!< in: mtr */ 495 /**********************************************************************//** 496 Parses a log record of compressing an index page without the data. 497 @return end of log record or NULL */ 498 UNIV_INLINE 499 byte* 500 page_zip_parse_compress_no_data( 501 /*============================*/ 502 byte* ptr, /*!< in: buffer */ 503 byte* end_ptr, /*!< in: buffer end */ 504 page_t* page, /*!< in: uncompressed page */ 505 page_zip_des_t* page_zip, /*!< out: compressed page */ 506 dict_index_t* index) /*!< in: index */ 507 MY_ATTRIBUTE((nonnull(1,2))); 508 509 /**********************************************************************//** 510 Reset the counters used for filling 511 INFORMATION_SCHEMA.innodb_cmp_per_index. */ 512 UNIV_INLINE 513 void 514 page_zip_reset_stat_per_index(); 515 /*===========================*/ 516 517 #ifndef UNIV_HOTBACKUP 518 /** Check if a pointer to an uncompressed page matches a compressed page. 519 When we IMPORT a tablespace the blocks and accompanying frames are allocted 520 from outside the buffer pool. 521 @param ptr pointer to an uncompressed page frame 522 @param page_zip compressed page descriptor 523 @return TRUE if ptr and page_zip refer to the same block */ 524 # define PAGE_ZIP_MATCH(ptr, page_zip) \ 525 (((page_zip)->m_external \ 526 && (page_align(ptr) + UNIV_PAGE_SIZE == (page_zip)->data)) \ 527 || buf_frame_get_page_zip(ptr) == (page_zip)) 528 #else /* !UNIV_HOTBACKUP */ 529 /** Check if a pointer to an uncompressed page matches a compressed page. 530 @param ptr pointer to an uncompressed page frame 531 @param page_zip compressed page descriptor 532 @return TRUE if ptr and page_zip refer to the same block */ 533 # define PAGE_ZIP_MATCH(ptr, page_zip) \ 534 (page_align(ptr) + UNIV_PAGE_SIZE == (page_zip)->data) 535 #endif /* !UNIV_HOTBACKUP */ 536 537 #ifdef UNIV_MATERIALIZE 538 # undef UNIV_INLINE 539 # define UNIV_INLINE UNIV_INLINE_ORIGINAL 540 #endif 541 542 #ifndef UNIV_NONINL 543 # include "page0zip.ic" 544 #endif 545 546 #endif /* page0zip_h */ 547