1 /*
2 Copyright (c) 2000, 2019, Oracle and/or its affiliates. All rights reserved.
3
4 This program is free software; you can redistribute it and/or modify
5 it under the terms of the GNU General Public License, version 2.0,
6 as published by the Free Software Foundation.
7
8 This program is also distributed with certain software (including
9 but not limited to OpenSSL) that is licensed under separate terms,
10 as designated in a particular file or component or in included license
11 documentation. The authors of MySQL hereby grant you an additional
12 permission to link the program and your derivative works with the
13 separately licensed software that they have included with MySQL.
14
15 This program is distributed in the hope that it will be useful,
16 but WITHOUT ANY WARRANTY; without even the implied warranty of
17 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
18 GNU General Public License, version 2.0, for more details.
19
20 You should have received a copy of the GNU General Public License
21 along with this program; if not, write to the Free Software
22 Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA */
23
24 /* Describe, check and repair of MyISAM tables */
25
26 /*
27 About checksum calculation.
28
29 There are two types of checksums. Table checksum and row checksum.
30
31 Row checksum is an additional byte at the end of dynamic length
32 records. It must be calculated if the table is configured for them.
33 Otherwise they must not be used. The variable
34 MYISAM_SHARE::calc_checksum determines if row checksums are used.
35 MI_INFO::checksum is used as temporary storage during row handling.
36 For parallel repair we must assure that only one thread can use this
37 variable. There is no problem on the write side as this is done by one
38 thread only. But when checking a record after read this could go
39 wrong. But since all threads read through a common read buffer, it is
40 sufficient if only one thread checks it.
41
42 Table checksum is an eight byte value in the header of the index file.
43 It can be calculated even if row checksums are not used. The variable
44 MI_CHECK::glob_crc is calculated over all records.
45 MI_SORT_PARAM::calc_checksum determines if this should be done. This
46 variable is not part of MI_CHECK because it must be set per thread for
47 parallel repair. The global glob_crc must be changed by one thread
48 only. And it is sufficient to calculate the checksum once only.
49 */
50
51 #include "my_config.h"
52
53 #include <assert.h>
54 #include <errno.h>
55 #include <fcntl.h>
56 #include <limits.h>
57 #include <stdarg.h>
58 #include <sys/types.h>
59 #include <time.h>
60
61 #include <algorithm>
62
63 #include "m_ctype.h"
64 #include "my_byteorder.h"
65 #include "my_compiler.h"
66 #include "my_dbug.h"
67 #include "my_double2ulonglong.h"
68 #include "my_getopt.h"
69 #include "my_inttypes.h"
70 #include "my_io.h"
71 #include "my_macros.h"
72 #include "my_pointer_arithmetic.h"
73 #include "storage/myisam/ftdefs.h"
74 #include "storage/myisam/myisam_sys.h"
75 #ifdef HAVE_SYS_MMAN_H
76 #include <sys/mman.h>
77 #endif
78 #include "storage/myisam/rt_index.h"
79
80 /* Functions defined in this file */
81
82 static int check_k_link(MI_CHECK *param, MI_INFO *info, uint nr);
83 static int chk_index(MI_CHECK *param, MI_INFO *info, MI_KEYDEF *keyinfo,
84 my_off_t page, uchar *buff, ha_rows *keys,
85 ha_checksum *key_checksum, uint level);
86 static uint isam_key_length(MI_INFO *info, MI_KEYDEF *keyinfo);
87 static ha_checksum calc_checksum(ha_rows count);
88 static int writekeys(MI_SORT_PARAM *sort_param);
89 static int sort_one_index(MI_CHECK *param, MI_INFO *info, MI_KEYDEF *keyinfo,
90 my_off_t pagepos, File new_file);
91 static int sort_key_read(MI_SORT_PARAM *sort_param, void *key);
92 static int sort_ft_key_read(MI_SORT_PARAM *sort_param, void *key);
93 static int sort_get_next_record(MI_SORT_PARAM *sort_param);
94 static int sort_key_cmp(void *cmp_arg, unsigned char *a, unsigned char *b);
95 static int sort_ft_key_write(MI_SORT_PARAM *sort_param, const void *a);
96 static int sort_key_write(MI_SORT_PARAM *sort_param, const void *a);
97 static my_off_t get_record_for_key(MI_INFO *info, MI_KEYDEF *keyinfo,
98 const uchar *key);
99 static int sort_insert_key(MI_SORT_PARAM *sort_param,
100 SORT_KEY_BLOCKS *key_block, const uchar *key,
101 my_off_t prev_block);
102 static int sort_delete_record(MI_SORT_PARAM *sort_param);
103 /*static int flush_pending_blocks(MI_CHECK *param);*/
104 static SORT_KEY_BLOCKS *alloc_key_blocks(MI_CHECK *param, uint blocks,
105 uint buffer_length);
106 static ha_checksum mi_byte_checksum(const uchar *buf, uint length);
107 static void set_data_file_type(SORT_INFO *sort_info, MYISAM_SHARE *share);
108 static HA_KEYSEG *ha_find_null(HA_KEYSEG *keyseg, const uchar *a);
109
myisamchk_init(MI_CHECK * param)110 void myisamchk_init(MI_CHECK *param) {
111 memset(param, 0, sizeof(*param));
112 param->opt_follow_links = true;
113 param->keys_in_use = ~(ulonglong)0;
114 param->search_after_block = HA_OFFSET_ERROR;
115 param->auto_increment_value = 0;
116 param->use_buffers = USE_BUFFER_INIT;
117 param->read_buffer_length = READ_BUFFER_INIT;
118 param->write_buffer_length = READ_BUFFER_INIT;
119 param->sort_buffer_length = SORT_BUFFER_INIT;
120 param->sort_key_blocks = BUFFERS_WHEN_SORTING;
121 param->tmpfile_createflag = O_RDWR | O_TRUNC | O_EXCL;
122 param->myf_rw = MYF(MY_NABP | MY_WME | MY_WAIT_IF_FULL);
123 param->start_check_pos = 0;
124 param->max_record_length = LLONG_MAX;
125 param->key_cache_block_size = KEY_CACHE_BLOCK_SIZE;
126 param->stats_method = MI_STATS_METHOD_NULLS_NOT_EQUAL;
127 param->need_print_msg_lock = false;
128 }
129
130 /* Check the status flags for the table */
131
chk_status(MI_CHECK * param,MI_INFO * info)132 int chk_status(MI_CHECK *param, MI_INFO *info) {
133 MYISAM_SHARE *share = info->s;
134
135 if (mi_is_crashed_on_repair(info))
136 mi_check_print_warning(param,
137 "Table is marked as crashed and last repair failed");
138 else if (mi_is_crashed(info))
139 mi_check_print_warning(param, "Table is marked as crashed");
140 if (share->state.open_count != (uint)(info->s->global_changed ? 1 : 0)) {
141 /* Don't count this as a real warning, as check can correct this ! */
142 uint save = param->warning_printed;
143 mi_check_print_warning(
144 param,
145 share->state.open_count == 1
146 ? "%d client is using or hasn't closed the table properly"
147 : "%d clients are using or haven't closed the table properly",
148 share->state.open_count);
149 /* If this will be fixed by the check, forget the warning */
150 if (param->testflag & T_UPDATE_STATE) param->warning_printed = save;
151 }
152 return 0;
153 }
154
155 /* Check delete links */
156
chk_del(MI_CHECK * param,MI_INFO * info,uint test_flag)157 int chk_del(MI_CHECK *param, MI_INFO *info, uint test_flag) {
158 ha_rows i;
159 uint delete_link_length;
160 my_off_t empty, next_link, old_link = 0;
161 char buff[22], buff2[22];
162 DBUG_TRACE;
163
164 param->record_checksum = 0;
165 delete_link_length =
166 ((info->s->options & HA_OPTION_PACK_RECORD) ? 20
167 : info->s->rec_reflength + 1);
168
169 if (!(test_flag & T_SILENT)) puts("- check record delete-chain");
170
171 next_link = info->s->state.dellink;
172 if (info->state->del == 0) {
173 if (test_flag & T_VERBOSE) {
174 puts("No recordlinks");
175 }
176 } else {
177 if (test_flag & T_VERBOSE) printf("Recordlinks: ");
178 empty = 0;
179 for (i = info->state->del; i > 0L && next_link != HA_OFFSET_ERROR; i--) {
180 if (*killed_ptr(param)) return 1;
181 if (test_flag & T_VERBOSE) printf(" %9s", llstr(next_link, buff));
182 if (next_link >= info->state->data_file_length) goto wrong;
183 if (mysql_file_pread(info->dfile, (uchar *)buff, delete_link_length,
184 next_link, MYF(MY_NABP))) {
185 if (test_flag & T_VERBOSE) puts("");
186 mi_check_print_error(param, "Can't read delete-link at filepos: %s",
187 llstr(next_link, buff));
188 return 1;
189 }
190 if (*buff != '\0') {
191 if (test_flag & T_VERBOSE) puts("");
192 mi_check_print_error(param, "Record at pos: %s is not remove-marked",
193 llstr(next_link, buff));
194 goto wrong;
195 }
196 if (info->s->options & HA_OPTION_PACK_RECORD) {
197 my_off_t prev_link = mi_sizekorr(pointer_cast<uchar *>(buff) + 12);
198 if (empty && prev_link != old_link) {
199 if (test_flag & T_VERBOSE) puts("");
200 mi_check_print_error(
201 param,
202 "Deleted block at %s doesn't point back at previous delete link",
203 llstr(next_link, buff2));
204 goto wrong;
205 }
206 old_link = next_link;
207 next_link = mi_sizekorr(pointer_cast<uchar *>(buff) + 4);
208 empty += mi_uint3korr(pointer_cast<uchar *>(buff) + 1);
209 } else {
210 param->record_checksum += (ha_checksum)next_link;
211 next_link = _mi_rec_pos(info->s, (uchar *)buff + 1);
212 empty += info->s->base.pack_reclength;
213 }
214 }
215 if (test_flag & T_VERBOSE) puts("\n");
216 if (empty != info->state->empty) {
217 mi_check_print_warning(
218 param, "Found %s deleted space in delete link chain. Should be %s",
219 llstr(empty, buff2), llstr(info->state->empty, buff));
220 }
221 if (next_link != HA_OFFSET_ERROR) {
222 mi_check_print_error(
223 param,
224 "Found more than the expected %s deleted rows in delete link chain",
225 llstr(info->state->del, buff));
226 goto wrong;
227 }
228 if (i != 0) {
229 mi_check_print_error(
230 param, "Found %s deleted rows in delete link chain. Should be %s",
231 llstr(info->state->del - i, buff2), llstr(info->state->del, buff));
232 goto wrong;
233 }
234 }
235 return 0;
236
237 wrong:
238 param->testflag |= T_RETRY_WITHOUT_QUICK;
239 if (test_flag & T_VERBOSE) puts("");
240 mi_check_print_error(param, "record delete-link-chain corrupted");
241 return 1;
242 } /* chk_del */
243
244 /* Check delete links in index file */
245
check_k_link(MI_CHECK * param,MI_INFO * info,uint nr)246 static int check_k_link(MI_CHECK *param, MI_INFO *info, uint nr) {
247 my_off_t next_link;
248 uint block_size = (nr + 1) * MI_MIN_KEY_BLOCK_LENGTH;
249 ha_rows records;
250 char llbuff[21], llbuff2[21];
251 uchar *buff;
252 DBUG_TRACE;
253 DBUG_PRINT("enter", ("block_size: %u", block_size));
254
255 if (param->testflag & T_VERBOSE)
256 printf("block_size %4u:", block_size); /* purecov: tested */
257
258 next_link = info->s->state.key_del[nr];
259 records = (ha_rows)(info->state->key_file_length / block_size);
260 while (next_link != HA_OFFSET_ERROR && records > 0) {
261 if (*killed_ptr(param)) return 1;
262 if (param->testflag & T_VERBOSE) printf("%16s", llstr(next_link, llbuff));
263
264 /* Key blocks must lay within the key file length entirely. */
265 if (next_link + block_size > info->state->key_file_length) {
266 /* purecov: begin tested */
267 mi_check_print_error(param,
268 "Invalid key block position: %s "
269 "key block size: %u file_length: %s",
270 llstr(next_link, llbuff), block_size,
271 llstr(info->state->key_file_length, llbuff2));
272 return 1;
273 /* purecov: end */
274 }
275
276 /* Key blocks must be aligned at MI_MIN_KEY_BLOCK_LENGTH. */
277 if (next_link & (MI_MIN_KEY_BLOCK_LENGTH - 1)) {
278 /* purecov: begin tested */
279 mi_check_print_error(param,
280 "Mis-aligned key block: %s "
281 "minimum key block length: %u",
282 llstr(next_link, llbuff), MI_MIN_KEY_BLOCK_LENGTH);
283 return 1;
284 /* purecov: end */
285 }
286
287 /*
288 Read the key block with MI_MIN_KEY_BLOCK_LENGTH to find next link.
289 If the key cache block size is smaller than block_size, we can so
290 avoid unecessary eviction of cache block.
291 */
292 if (!(buff = key_cache_read(info->s->key_cache, keycache_thread_var(),
293 info->s->kfile, next_link, DFLT_INIT_HITS,
294 (uchar *)info->buff, MI_MIN_KEY_BLOCK_LENGTH,
295 MI_MIN_KEY_BLOCK_LENGTH, 1))) {
296 /* purecov: begin tested */
297 mi_check_print_error(param, "key cache read error for block: %s",
298 llstr(next_link, llbuff));
299 return 1;
300 /* purecov: end */
301 }
302 next_link = mi_sizekorr(buff);
303 records--;
304 param->key_file_blocks += block_size;
305 }
306 if (param->testflag & T_VERBOSE) {
307 if (next_link != HA_OFFSET_ERROR)
308 printf("%16s\n", llstr(next_link, llbuff));
309 else
310 puts("");
311 }
312 return next_link != HA_OFFSET_ERROR;
313 } /* check_k_link */
314
315 /* Check sizes of files */
316
chk_size(MI_CHECK * param,MI_INFO * info)317 int chk_size(MI_CHECK *param, MI_INFO *info) {
318 int error = 0;
319 my_off_t skr, size;
320 DBUG_TRACE;
321
322 if (!(param->testflag & T_SILENT)) puts("- check file-size");
323
324 /* The following is needed if called externally (not from myisamchk) */
325 flush_key_blocks(info->s->key_cache, keycache_thread_var(), info->s->kfile,
326 FLUSH_FORCE_WRITE);
327
328 size = mysql_file_seek(info->s->kfile, 0L, MY_SEEK_END, MYF(0));
329 if ((skr = (my_off_t)info->state->key_file_length) != size) {
330 /* Don't give error if file generated by myisampack */
331 if (skr > size && mi_is_any_key_active(info->s->state.key_map)) {
332 error = 1;
333 mi_check_print_error(param,
334 "Size of indexfile is: %lld Should be: %lld",
335 size, skr);
336 } else
337 mi_check_print_warning(
338 param, "Size of indexfile is: %lld Should be: %lld", size, skr);
339 }
340 if (!(param->testflag & T_VERY_SILENT) &&
341 !(info->s->options & HA_OPTION_COMPRESS_RECORD) &&
342 ulonglong2double(info->state->key_file_length) >
343 ulonglong2double(info->s->base.margin_key_file_length) * 0.9)
344 mi_check_print_warning(param, "Keyfile is almost full, %lld of %lld used",
345 info->state->key_file_length,
346 info->s->base.max_key_file_length - 1);
347
348 size = mysql_file_seek(info->dfile, 0L, MY_SEEK_END, MYF(0));
349 skr = (my_off_t)info->state->data_file_length;
350 if (info->s->options & HA_OPTION_COMPRESS_RECORD) skr += MEMMAP_EXTRA_MARGIN;
351 if (skr != size) {
352 info->state->data_file_length = size; /* Skip other errors */
353 if (skr > size && skr != size + MEMMAP_EXTRA_MARGIN) {
354 error = 1;
355 mi_check_print_error(param,
356 "Size of datafile is: %lld Should be: %lld",
357 size, skr);
358 param->testflag |= T_RETRY_WITHOUT_QUICK;
359 } else {
360 mi_check_print_warning(
361 param, "Size of datafile is: %lld Should be: %lld", size, skr);
362 }
363 }
364 if (!(param->testflag & T_VERY_SILENT) &&
365 !(info->s->options & HA_OPTION_COMPRESS_RECORD) &&
366 ulonglong2double(info->state->data_file_length) >
367 (ulonglong2double(info->s->base.max_data_file_length) * 0.9))
368 mi_check_print_warning(param, "Datafile is almost full, %lld of %lld used",
369 info->state->data_file_length,
370 info->s->base.max_data_file_length - 1);
371 return error;
372 } /* chk_size */
373
374 /* Check keys */
375
chk_key(MI_CHECK * param,MI_INFO * info)376 int chk_key(MI_CHECK *param, MI_INFO *info) {
377 uint key, found_keys = 0, full_text_keys = 0, result = 0;
378 ha_rows keys;
379 ha_checksum old_record_checksum, init_checksum;
380 my_off_t all_keydata, all_totaldata, key_totlength, length;
381 ulong *rec_per_key_part;
382 uint number_of_rec_per_key_estimates = 0;
383 MYISAM_SHARE *share = info->s;
384 MI_KEYDEF *keyinfo;
385 char buff[22], buff2[22];
386 DBUG_TRACE;
387
388 if (!(param->testflag & T_SILENT)) puts("- check key delete-chain");
389
390 param->key_file_blocks = info->s->base.keystart;
391 for (key = 0; key < info->s->state.header.max_block_size_index; key++)
392 if (check_k_link(param, info, key)) {
393 if (param->testflag & T_VERBOSE) puts("");
394 mi_check_print_error(param, "key delete-link-chain corrupted");
395 return -1;
396 }
397
398 if (!(param->testflag & T_SILENT)) puts("- check index reference");
399
400 all_keydata = all_totaldata = key_totlength = 0;
401 old_record_checksum = 0;
402 init_checksum = param->record_checksum;
403 if (!(share->options & (HA_OPTION_PACK_RECORD | HA_OPTION_COMPRESS_RECORD)))
404 old_record_checksum =
405 calc_checksum(info->state->records + info->state->del - 1) *
406 share->base.pack_reclength;
407 rec_per_key_part = param->rec_per_key_part;
408 for (key = 0, keyinfo = &share->keyinfo[0]; key < share->base.keys;
409 rec_per_key_part += number_of_rec_per_key_estimates, key++, keyinfo++) {
410 /*
411 R-tree indexes have 1 key part (column) and 4 key segments. Only
412 one rec_per_key estimate should be produced for those indexes.
413
414 B-tree indexes have the same number of segments as key parts
415 (columns). Generate one rec_per_key estimate per key part.
416 */
417 if (keyinfo->flag & HA_SPATIAL)
418 number_of_rec_per_key_estimates = 1;
419 else
420 number_of_rec_per_key_estimates = keyinfo->keysegs;
421
422 param->key_crc[key] = 0;
423 if (!mi_is_key_active(share->state.key_map, key)) {
424 /* Remember old statistics for key */
425 memcpy((char *)rec_per_key_part,
426 (char *)(share->state.rec_per_key_part +
427 (uint)(rec_per_key_part - param->rec_per_key_part)),
428 number_of_rec_per_key_estimates * sizeof(*rec_per_key_part));
429 continue;
430 }
431 found_keys++;
432
433 param->record_checksum = init_checksum;
434
435 memset(¶m->unique_count, 0, sizeof(param->unique_count));
436 memset(¶m->notnull_count, 0, sizeof(param->notnull_count));
437
438 if ((!(param->testflag & T_SILENT)))
439 printf("- check data record references index: %d\n", key + 1);
440 if (keyinfo->flag & (HA_FULLTEXT | HA_SPATIAL)) full_text_keys++;
441 if (share->state.key_root[key] == HA_OFFSET_ERROR &&
442 (info->state->records == 0 || keyinfo->flag & HA_FULLTEXT))
443 goto do_stat;
444 if (!_mi_fetch_keypage(info, keyinfo, share->state.key_root[key],
445 DFLT_INIT_HITS, info->buff, 0)) {
446 mi_check_print_error(param, "Can't read indexpage from filepos: %s",
447 llstr(share->state.key_root[key], buff));
448 if (!(param->testflag & T_INFO)) return -1;
449 result = -1;
450 continue;
451 }
452 param->key_file_blocks += keyinfo->block_length;
453 keys = 0;
454 param->keydata = param->totaldata = 0;
455 param->key_blocks = 0;
456 param->max_level = 0;
457 if (chk_index(param, info, keyinfo, share->state.key_root[key], info->buff,
458 &keys, param->key_crc + key, 1))
459 return -1;
460 if (!(keyinfo->flag & (HA_FULLTEXT | HA_SPATIAL))) {
461 if (keys != info->state->records) {
462 mi_check_print_error(param, "Found %s keys of %s", llstr(keys, buff),
463 llstr(info->state->records, buff2));
464 if (!(param->testflag & T_INFO)) return -1;
465 result = -1;
466 continue;
467 }
468 if (found_keys - full_text_keys == 1 &&
469 ((share->options &
470 (HA_OPTION_PACK_RECORD | HA_OPTION_COMPRESS_RECORD)) ||
471 (param->testflag & T_DONT_CHECK_CHECKSUM)))
472 old_record_checksum = param->record_checksum;
473 else if (old_record_checksum != param->record_checksum) {
474 if (key)
475 mi_check_print_error(
476 param, "Key %u doesn't point at same records that key 1",
477 key + 1);
478 else
479 mi_check_print_error(param, "Key 1 doesn't point at all records");
480 if (!(param->testflag & T_INFO)) return -1;
481 result = -1;
482 continue;
483 }
484 }
485 if ((uint)share->base.auto_key - 1 == key) {
486 /* Check that auto_increment key is bigger than max key value */
487 ulonglong auto_increment;
488 info->lastinx = key;
489 _mi_read_key_record(info, 0L, info->rec_buff);
490 auto_increment = retrieve_auto_increment(info, info->rec_buff);
491 if (auto_increment > info->s->state.auto_increment) {
492 mi_check_print_warning(param,
493 "Auto-increment value: %s is smaller "
494 "than max used value: %s",
495 llstr(info->s->state.auto_increment, buff2),
496 llstr(auto_increment, buff));
497 }
498 if (param->testflag & T_AUTO_INC) {
499 info->s->state.auto_increment =
500 std::max(info->s->state.auto_increment, auto_increment);
501 info->s->state.auto_increment = std::max(info->s->state.auto_increment,
502 param->auto_increment_value);
503 }
504
505 /* Check that there isn't a row with auto_increment = 0 in the table */
506 mi_extra(info, HA_EXTRA_KEYREAD, nullptr);
507 memset(info->lastkey, 0, keyinfo->seg->length);
508 if (!mi_rkey(info, info->rec_buff, key, (const uchar *)info->lastkey,
509 (key_part_map)1, HA_READ_KEY_EXACT)) {
510 /* Don't count this as a real warning, as myisamchk can't correct it */
511 uint save = param->warning_printed;
512 mi_check_print_warning(param,
513 "Found row where the auto_increment "
514 "column has the value 0");
515 param->warning_printed = save;
516 }
517 mi_extra(info, HA_EXTRA_NO_KEYREAD, nullptr);
518 }
519
520 length =
521 (my_off_t)isam_key_length(info, keyinfo) * keys + param->key_blocks * 2;
522 if (param->testflag & T_INFO && param->totaldata != 0L && keys != 0L)
523 printf(
524 "Key: %2d: Keyblocks used: %3d%% Packed: %4d%% Max levels: %2d\n",
525 key + 1,
526 (int)(my_off_t2double(param->keydata) * 100.0 /
527 my_off_t2double(param->totaldata)),
528 (int)((my_off_t2double(length) - my_off_t2double(param->keydata)) *
529 100.0 / my_off_t2double(length)),
530 param->max_level);
531 all_keydata += param->keydata;
532 all_totaldata += param->totaldata;
533 key_totlength += length;
534
535 do_stat:
536 if (param->testflag & T_STATISTICS)
537 update_key_parts(keyinfo, rec_per_key_part, param->unique_count,
538 param->stats_method == MI_STATS_METHOD_IGNORE_NULLS
539 ? param->notnull_count
540 : nullptr,
541 (ulonglong)info->state->records);
542 }
543 if (param->testflag & T_INFO) {
544 if (all_totaldata != 0L && found_keys > 0)
545 printf("Total: Keyblocks used: %3d%% Packed: %4d%%\n\n",
546 (int)(my_off_t2double(all_keydata) * 100.0 /
547 my_off_t2double(all_totaldata)),
548 (int)((my_off_t2double(key_totlength) -
549 my_off_t2double(all_keydata)) *
550 100.0 / my_off_t2double(key_totlength)));
551 else if (all_totaldata != 0L && mi_is_any_key_active(share->state.key_map))
552 puts("");
553 }
554 if (param->key_file_blocks != info->state->key_file_length &&
555 param->keys_in_use != ~(ulonglong)0)
556 mi_check_print_warning(param, "Some data are unreferenced in keyfile");
557 if (found_keys != full_text_keys)
558 param->record_checksum =
559 old_record_checksum - init_checksum; /* Remove delete links */
560 else
561 param->record_checksum = 0;
562 return result;
563 } /* chk_key */
564
chk_index_down(MI_CHECK * param,MI_INFO * info,MI_KEYDEF * keyinfo,my_off_t page,uchar * buff,ha_rows * keys,ha_checksum * key_checksum,uint level)565 static int chk_index_down(MI_CHECK *param, MI_INFO *info, MI_KEYDEF *keyinfo,
566 my_off_t page, uchar *buff, ha_rows *keys,
567 ha_checksum *key_checksum, uint level) {
568 char llbuff[22], llbuff2[22];
569 DBUG_TRACE;
570
571 /* Key blocks must lay within the key file length entirely. */
572 if (page + keyinfo->block_length > info->state->key_file_length) {
573 /* purecov: begin tested */
574 /* Give it a chance to fit in the real file size. */
575 my_off_t max_length =
576 mysql_file_seek(info->s->kfile, 0L, MY_SEEK_END, MYF(0));
577 mi_check_print_error(param,
578 "Invalid key block position: %s "
579 "key block size: %u file_length: %s",
580 llstr(page, llbuff), keyinfo->block_length,
581 llstr(info->state->key_file_length, llbuff2));
582 if (page + keyinfo->block_length > max_length) goto err;
583 /* Fix the remebered key file length. */
584 info->state->key_file_length =
585 (max_length & ~(my_off_t)(keyinfo->block_length - 1));
586 /* purecov: end */
587 }
588
589 /* Key blocks must be aligned at MI_MIN_KEY_BLOCK_LENGTH. */
590 if (page & (MI_MIN_KEY_BLOCK_LENGTH - 1)) {
591 /* purecov: begin tested */
592 mi_check_print_error(param,
593 "Mis-aligned key block: %s "
594 "minimum key block length: %u",
595 llstr(page, llbuff), MI_MIN_KEY_BLOCK_LENGTH);
596 goto err;
597 /* purecov: end */
598 }
599
600 if (!_mi_fetch_keypage(info, keyinfo, page, DFLT_INIT_HITS, buff, 0)) {
601 mi_check_print_error(param, "Can't read key from filepos: %s",
602 llstr(page, llbuff));
603 goto err;
604 }
605 param->key_file_blocks += keyinfo->block_length;
606 if (chk_index(param, info, keyinfo, page, buff, keys, key_checksum, level))
607 goto err;
608
609 return 0;
610
611 /* purecov: begin tested */
612 err:
613 return 1;
614 /* purecov: end */
615 }
616
617 /*
618 "Ignore NULLs" statistics collection method: process first index tuple.
619
620 SYNOPSIS
621 mi_collect_stats_nonulls_first()
622 keyseg IN Array of key part descriptions
623 notnull INOUT Array, notnull[i] = (number of {keypart1...keypart_i}
624 tuples that don't contain NULLs)
625 key IN Key values tuple
626
627 DESCRIPTION
628 Process the first index tuple - find out which prefix tuples don't
629 contain NULLs, and update the array of notnull counters accordingly.
630 */
631
mi_collect_stats_nonulls_first(HA_KEYSEG * keyseg,ulonglong * notnull,const uchar * key)632 static void mi_collect_stats_nonulls_first(HA_KEYSEG *keyseg,
633 ulonglong *notnull,
634 const uchar *key) {
635 uint first_null, kp;
636 first_null = (uint)(ha_find_null(keyseg, key) - keyseg);
637 /*
638 All prefix tuples that don't include keypart_{first_null} are not-null
639 tuples (and all others aren't), increment counters for them.
640 */
641 for (kp = 0; kp < first_null; kp++) notnull[kp]++;
642 }
643
644 /*
645 "Ignore NULLs" statistics collection method: process next index tuple.
646
647 SYNOPSIS
648 mi_collect_stats_nonulls_next()
649 keyseg IN Array of key part descriptions
650 notnull INOUT Array, notnull[i] = (number of {keypart1...keypart_i}
651 tuples that don't contain NULLs)
652 prev_key IN Previous key values tuple
653 last_key IN Next key values tuple
654
655 DESCRIPTION
656 Process the next index tuple:
657 1. Find out which prefix tuples of last_key don't contain NULLs, and
658 update the array of notnull counters accordingly.
659 2. Find the first keypart number where the prev_key and last_key tuples
660 are different(A), or last_key has NULL value(B), and return it, so the
661 caller can count number of unique tuples for each key prefix. We don't
662 need (B) to be counted, and that is compensated back in
663 update_key_parts().
664
665 RETURN
666 1 + number of first keypart where values differ or last_key tuple has NULL
667 */
668
mi_collect_stats_nonulls_next(HA_KEYSEG * keyseg,ulonglong * notnull,uchar * prev_key,const uchar * last_key)669 static int mi_collect_stats_nonulls_next(HA_KEYSEG *keyseg, ulonglong *notnull,
670 uchar *prev_key,
671 const uchar *last_key) {
672 uint diffs[2];
673 uint first_null_seg, kp;
674 HA_KEYSEG *seg;
675
676 /*
677 Find the first keypart where values are different or either of them is
678 NULL. We get results in diffs array:
679 diffs[0]= 1 + number of first different keypart
680 diffs[1]=offset: (last_key + diffs[1]) points to first value in
681 last_key that is NULL or different from corresponding
682 value in prev_key.
683 */
684 ha_key_cmp(keyseg, prev_key, last_key, USE_WHOLE_KEY,
685 SEARCH_FIND | SEARCH_NULL_ARE_NOT_EQUAL, diffs);
686 seg = keyseg + diffs[0] - 1;
687
688 /* Find first NULL in last_key */
689 first_null_seg = (uint)(ha_find_null(seg, last_key + diffs[1]) - keyseg);
690 for (kp = 0; kp < first_null_seg; kp++) notnull[kp]++;
691
692 /*
693 Return 1+ number of first key part where values differ. Don't care if
694 these were NULLs and not .... We compensate for that in
695 update_key_parts.
696 */
697 return diffs[0];
698 }
699
700 /* Check if index is ok */
701
chk_index(MI_CHECK * param,MI_INFO * info,MI_KEYDEF * keyinfo,my_off_t page,uchar * buff,ha_rows * keys,ha_checksum * key_checksum,uint level)702 static int chk_index(MI_CHECK *param, MI_INFO *info, MI_KEYDEF *keyinfo,
703 my_off_t page, uchar *buff, ha_rows *keys,
704 ha_checksum *key_checksum, uint level) {
705 int flag;
706 uint used_length, comp_flag, nod_flag, key_length = 0;
707 uchar key[HA_MAX_POSSIBLE_KEY_BUFF], *temp_buff, *keypos, *old_keypos,
708 *endpos;
709 my_off_t next_page, record;
710 char llbuff[22];
711 uint diff_pos[2];
712 DBUG_TRACE;
713 DBUG_DUMP("buff", (uchar *)buff, mi_getint(buff));
714
715 /* TODO: implement appropriate check for RTree keys */
716 if (keyinfo->flag & HA_SPATIAL) return 0;
717
718 if (!(temp_buff = (uchar *)my_alloca((uint)keyinfo->block_length))) {
719 mi_check_print_error(param, "Not enough memory for keyblock");
720 return -1;
721 }
722
723 if (keyinfo->flag & HA_NOSAME)
724 comp_flag = SEARCH_FIND | SEARCH_UPDATE; /* Not real duplicates */
725 else
726 comp_flag = SEARCH_SAME; /* Keys in positionorder */
727 nod_flag = mi_test_if_nod(buff);
728 used_length = mi_getint(buff);
729 keypos = buff + 2 + nod_flag;
730 endpos = buff + used_length;
731
732 param->keydata += used_length;
733 param->totaldata += keyinfo->block_length; /* INFO */
734 param->key_blocks++;
735 if (level > param->max_level) param->max_level = level;
736
737 if (used_length > keyinfo->block_length) {
738 mi_check_print_error(param, "Wrong pageinfo at page: %s",
739 llstr(page, llbuff));
740 goto err;
741 }
742 for (;;) {
743 if (*killed_ptr(param)) goto err;
744 memcpy((char *)info->lastkey, (char *)key, key_length);
745 info->lastkey_length = key_length;
746 if (nod_flag) {
747 next_page = _mi_kpos(nod_flag, keypos);
748 if (chk_index_down(param, info, keyinfo, next_page, temp_buff, keys,
749 key_checksum, level + 1))
750 goto err;
751 }
752 old_keypos = keypos;
753 if (keypos >= endpos || (key_length = (*keyinfo->get_key)(
754 keyinfo, nod_flag, &keypos, key)) == 0)
755 break;
756 if (keypos > endpos) {
757 mi_check_print_error(param, "Wrong key block length at page: %s",
758 llstr(page, llbuff));
759 goto err;
760 }
761 if ((*keys)++ &&
762 (flag = ha_key_cmp(keyinfo->seg, info->lastkey, key, key_length,
763 comp_flag, diff_pos)) >= 0) {
764 DBUG_DUMP("old", (uchar *)info->lastkey, info->lastkey_length);
765 DBUG_DUMP("new", (uchar *)key, key_length);
766 DBUG_DUMP("new_in_page", (uchar *)old_keypos,
767 (uint)(keypos - old_keypos));
768
769 if (comp_flag & SEARCH_FIND && flag == 0)
770 mi_check_print_error(param, "Found duplicated key at page %s",
771 llstr(page, llbuff));
772 else
773 mi_check_print_error(param, "Key in wrong position at page %s",
774 llstr(page, llbuff));
775 goto err;
776 }
777 if (param->testflag & T_STATISTICS) {
778 if (*keys != 1L) /* not first_key */
779 {
780 if (param->stats_method == MI_STATS_METHOD_NULLS_NOT_EQUAL)
781 ha_key_cmp(keyinfo->seg, info->lastkey, key, USE_WHOLE_KEY,
782 SEARCH_FIND | SEARCH_NULL_ARE_NOT_EQUAL, diff_pos);
783 else if (param->stats_method == MI_STATS_METHOD_IGNORE_NULLS) {
784 diff_pos[0] = mi_collect_stats_nonulls_next(
785 keyinfo->seg, param->notnull_count, info->lastkey, key);
786 }
787 param->unique_count[diff_pos[0] - 1]++;
788 } else {
789 if (param->stats_method == MI_STATS_METHOD_IGNORE_NULLS)
790 mi_collect_stats_nonulls_first(keyinfo->seg, param->notnull_count,
791 key);
792 }
793 }
794 (*key_checksum) +=
795 mi_byte_checksum((uchar *)key, key_length - info->s->rec_reflength);
796 record = _mi_dpos(info, 0, key + key_length);
797 if (keyinfo->flag & HA_FULLTEXT) /* special handling for ft2 */
798 {
799 uint off;
800 int subkeys;
801 get_key_full_length_rdonly(off, key);
802 subkeys = ft_sintXkorr(key + off);
803 if (subkeys < 0) {
804 ha_rows tmp_keys = 0;
805 if (chk_index_down(param, info, &info->s->ft2_keyinfo, record,
806 temp_buff, &tmp_keys, key_checksum, 1))
807 goto err;
808 if (tmp_keys + subkeys) {
809 mi_check_print_error(param,
810 "Number of words in the 2nd level tree "
811 "does not match the number in the header. "
812 "Parent word in on the page %s, offset %u",
813 llstr(page, llbuff), (uint)(old_keypos - buff));
814 goto err;
815 }
816 (*keys) += tmp_keys - 1;
817 continue;
818 }
819 /* fall through */
820 }
821 if (record >= info->state->data_file_length) {
822 #ifndef DBUG_OFF
823 char llbuff2[22], llbuff3[22];
824 #endif
825 mi_check_print_error(
826 param, "Found key at page %s that points to record outside datafile",
827 llstr(page, llbuff));
828 DBUG_PRINT("test", ("page: %s record: %s filelength: %s",
829 llstr(page, llbuff), llstr(record, llbuff2),
830 llstr(info->state->data_file_length, llbuff3)));
831 DBUG_DUMP("key", (uchar *)key, key_length);
832 DBUG_DUMP("new_in_page", (uchar *)old_keypos,
833 (uint)(keypos - old_keypos));
834 goto err;
835 }
836 param->record_checksum += (ha_checksum)record;
837 }
838 if (keypos != endpos) {
839 mi_check_print_error(param,
840 "Keyblock size at page %s is not correct. Block "
841 "length: %d key length: %d",
842 llstr(page, llbuff), used_length,
843 (int)(keypos - buff));
844 goto err;
845 }
846 return 0;
847 err:
848 return 1;
849 } /* chk_index */
850
851 /* Calculate a checksum of 1+2+3+4...N = N*(N+1)/2 without overflow */
852
calc_checksum(ha_rows count)853 static ha_checksum calc_checksum(ha_rows count) {
854 ulonglong sum, a, b;
855 DBUG_TRACE;
856
857 sum = 0;
858 a = count;
859 b = count + 1;
860 if (a & 1)
861 b >>= 1;
862 else
863 a >>= 1;
864 while (b) {
865 if (b & 1) sum += a;
866 a <<= 1;
867 b >>= 1;
868 }
869 DBUG_PRINT("exit", ("sum: %lx", (ulong)sum));
870 return (ha_checksum)sum;
871 } /* calc_checksum */
872
873 /* Calc length of key in normal isam */
874
isam_key_length(MI_INFO * info,MI_KEYDEF * keyinfo)875 static uint isam_key_length(MI_INFO *info, MI_KEYDEF *keyinfo) {
876 uint length;
877 HA_KEYSEG *keyseg;
878 DBUG_TRACE;
879
880 length = info->s->rec_reflength;
881 for (keyseg = keyinfo->seg; keyseg->type; keyseg++) length += keyseg->length;
882
883 DBUG_PRINT("exit", ("length: %d", length));
884 return length;
885 } /* key_length */
886
887 /* Check that record-link is ok */
888
chk_data_link(MI_CHECK * param,MI_INFO * info,int extend)889 int chk_data_link(MI_CHECK *param, MI_INFO *info, int extend) {
890 int error, got_error, flag;
891 uint key, left_length = 0, b_type, field;
892 ha_rows records, del_blocks;
893 my_off_t used, empty, pos, splits, start_recpos = 0, del_length, link_used,
894 start_block;
895 uchar *record = nullptr, *to = nullptr;
896 char llbuff[22], llbuff2[22], llbuff3[22];
897 ha_checksum intern_record_checksum;
898 ha_checksum key_checksum[HA_MAX_POSSIBLE_KEY];
899 bool static_row_size;
900 MI_KEYDEF *keyinfo;
901 MI_BLOCK_INFO block_info;
902 DBUG_TRACE;
903
904 if (!(param->testflag & T_SILENT)) {
905 if (extend)
906 puts("- check records and index references");
907 else
908 puts("- check record links");
909 }
910
911 if (!mi_alloc_rec_buff(info, -1, &record)) {
912 mi_check_print_error(param, "Not enough memory for record");
913 return -1;
914 }
915 records = del_blocks = 0;
916 used = link_used = splits = del_length = 0;
917 intern_record_checksum = param->glob_crc = 0;
918 got_error = error = 0;
919 empty = info->s->pack.header_length;
920
921 /* Check how to calculate checksum of rows */
922 static_row_size = true;
923 if (info->s->data_file_type == COMPRESSED_RECORD) {
924 for (field = 0; field < info->s->base.fields; field++) {
925 if (info->s->rec[field].base_type == FIELD_BLOB ||
926 info->s->rec[field].base_type == FIELD_VARCHAR) {
927 static_row_size = false;
928 break;
929 }
930 }
931 }
932
933 pos = my_b_tell(¶m->read_cache);
934 memset(key_checksum, 0, info->s->base.keys * sizeof(key_checksum[0]));
935 while (pos < info->state->data_file_length) {
936 if (*killed_ptr(param)) goto err2;
937 switch (info->s->data_file_type) {
938 case STATIC_RECORD:
939 if (my_b_read(¶m->read_cache, (uchar *)record,
940 info->s->base.pack_reclength))
941 goto err;
942 start_recpos = pos;
943 pos += info->s->base.pack_reclength;
944 splits++;
945 if (*record == '\0') {
946 del_blocks++;
947 del_length += info->s->base.pack_reclength;
948 continue; /* Record removed */
949 }
950 param->glob_crc += mi_static_checksum(info, record);
951 used += info->s->base.pack_reclength;
952 break;
953 case DYNAMIC_RECORD:
954 flag = block_info.second_read = 0;
955 block_info.next_filepos = pos;
956 do {
957 if (_mi_read_cache(¶m->read_cache, (uchar *)block_info.header,
958 (start_block = block_info.next_filepos),
959 sizeof(block_info.header),
960 (flag ? 0 : READING_NEXT) | READING_HEADER))
961 goto err;
962 if (start_block & (MI_DYN_ALIGN_SIZE - 1)) {
963 mi_check_print_error(param, "Wrong aligned block at %s",
964 llstr(start_block, llbuff));
965 goto err2;
966 }
967 b_type = _mi_get_block_info(&block_info, -1, start_block);
968 if (b_type & (BLOCK_DELETED | BLOCK_ERROR | BLOCK_SYNC_ERROR |
969 BLOCK_FATAL_ERROR)) {
970 if (b_type & BLOCK_SYNC_ERROR) {
971 if (flag) {
972 mi_check_print_error(param, "Unexpected byte: %d at link: %s",
973 (int)block_info.header[0],
974 llstr(start_block, llbuff));
975 goto err2;
976 }
977 pos = block_info.filepos + block_info.block_len;
978 goto next;
979 }
980 if (b_type & BLOCK_DELETED) {
981 if (block_info.block_len < info->s->base.min_block_length) {
982 mi_check_print_error(
983 param, "Deleted block with impossible length %lu at %s",
984 block_info.block_len, llstr(pos, llbuff));
985 goto err2;
986 }
987 if ((block_info.next_filepos != HA_OFFSET_ERROR &&
988 block_info.next_filepos >= info->state->data_file_length) ||
989 (block_info.prev_filepos != HA_OFFSET_ERROR &&
990 block_info.prev_filepos >= info->state->data_file_length)) {
991 mi_check_print_error(
992 param, "Delete link points outside datafile at %s",
993 llstr(pos, llbuff));
994 goto err2;
995 }
996 del_blocks++;
997 del_length += block_info.block_len;
998 pos = block_info.filepos + block_info.block_len;
999 splits++;
1000 goto next;
1001 }
1002 mi_check_print_error(
1003 param, "Wrong bytesec: %d-%d-%d at linkstart: %s",
1004 block_info.header[0], block_info.header[1],
1005 block_info.header[2], llstr(start_block, llbuff));
1006 goto err2;
1007 }
1008 if (info->state->data_file_length <
1009 block_info.filepos + block_info.block_len) {
1010 mi_check_print_error(
1011 param, "Recordlink that points outside datafile at %s",
1012 llstr(pos, llbuff));
1013 got_error = 1;
1014 break;
1015 }
1016 splits++;
1017 if (!flag++) /* First block */
1018 {
1019 start_recpos = pos;
1020 pos = block_info.filepos + block_info.block_len;
1021 if (block_info.rec_len > (uint)info->s->base.max_pack_length) {
1022 mi_check_print_error(param, "Found too long record (%lu) at %s",
1023 (ulong)block_info.rec_len,
1024 llstr(start_recpos, llbuff));
1025 got_error = 1;
1026 break;
1027 }
1028 if (info->s->base.blobs) {
1029 if (!(to = mi_alloc_rec_buff(info, block_info.rec_len,
1030 &info->rec_buff))) {
1031 mi_check_print_error(
1032 param, "Not enough memory (%lu) for blob at %s",
1033 (ulong)block_info.rec_len, llstr(start_recpos, llbuff));
1034 got_error = 1;
1035 break;
1036 }
1037 } else
1038 to = info->rec_buff;
1039 left_length = block_info.rec_len;
1040 }
1041 if (left_length < block_info.data_len) {
1042 mi_check_print_error(param, "Found too long record (%lu) at %s",
1043 (ulong)block_info.data_len,
1044 llstr(start_recpos, llbuff));
1045 got_error = 1;
1046 break;
1047 }
1048 if (_mi_read_cache(¶m->read_cache, (uchar *)to,
1049 block_info.filepos, (uint)block_info.data_len,
1050 flag == 1 ? READING_NEXT : 0))
1051 goto err;
1052 to += block_info.data_len;
1053 link_used += block_info.filepos - start_block;
1054 used += block_info.filepos - start_block + block_info.data_len;
1055 empty += block_info.block_len - block_info.data_len;
1056 left_length -= block_info.data_len;
1057 if (left_length) {
1058 if (b_type & BLOCK_LAST) {
1059 mi_check_print_error(
1060 param, "Wrong record length %s of %s at %s",
1061 llstr(block_info.rec_len - left_length, llbuff),
1062 llstr(block_info.rec_len, llbuff2),
1063 llstr(start_recpos, llbuff3));
1064 got_error = 1;
1065 break;
1066 }
1067 if (info->state->data_file_length < block_info.next_filepos) {
1068 mi_check_print_error(
1069 param,
1070 "Found next-recordlink that points outside datafile at %s",
1071 llstr(block_info.filepos, llbuff));
1072 got_error = 1;
1073 break;
1074 }
1075 }
1076 } while (left_length);
1077 if (!got_error) {
1078 if (_mi_rec_unpack(info, record, info->rec_buff,
1079 block_info.rec_len) == MY_FILE_ERROR) {
1080 mi_check_print_error(param, "Found wrong record at %s",
1081 llstr(start_recpos, llbuff));
1082 got_error = 1;
1083 } else {
1084 info->checksum = mi_checksum(info, record);
1085 if (param->testflag & (T_EXTEND | T_MEDIUM | T_VERBOSE)) {
1086 if (_mi_rec_check(info, record, info->rec_buff,
1087 block_info.rec_len, info->s->calc_checksum)) {
1088 mi_check_print_error(param, "Found wrong packed record at %s",
1089 llstr(start_recpos, llbuff));
1090 got_error = 1;
1091 }
1092 }
1093 if (!got_error) param->glob_crc += info->checksum;
1094 }
1095 } else if (!flag)
1096 pos = block_info.filepos + block_info.block_len;
1097 break;
1098 case COMPRESSED_RECORD:
1099 if (_mi_read_cache(¶m->read_cache, (uchar *)block_info.header, pos,
1100 info->s->pack.ref_length, READING_NEXT))
1101 goto err;
1102 start_recpos = pos;
1103 splits++;
1104 (void)_mi_pack_get_block_info(info, &info->bit_buff, &block_info,
1105 &info->rec_buff, -1, start_recpos);
1106 pos = block_info.filepos + block_info.rec_len;
1107 if (block_info.rec_len < (uint)info->s->min_pack_length ||
1108 block_info.rec_len > (uint)info->s->max_pack_length) {
1109 mi_check_print_error(param,
1110 "Found block with wrong recordlength: %ld at %s",
1111 block_info.rec_len, llstr(start_recpos, llbuff));
1112 got_error = 1;
1113 break;
1114 }
1115 if (_mi_read_cache(¶m->read_cache, (uchar *)info->rec_buff,
1116 block_info.filepos, block_info.rec_len,
1117 READING_NEXT))
1118 goto err;
1119 if (_mi_pack_rec_unpack(info, &info->bit_buff, record, info->rec_buff,
1120 block_info.rec_len)) {
1121 mi_check_print_error(param, "Found wrong record at %s",
1122 llstr(start_recpos, llbuff));
1123 got_error = 1;
1124 }
1125 if (static_row_size)
1126 param->glob_crc += mi_static_checksum(info, record);
1127 else
1128 param->glob_crc += mi_checksum(info, record);
1129 link_used += (block_info.filepos - start_recpos);
1130 used += (pos - start_recpos);
1131 break;
1132 case BLOCK_RECORD:
1133 assert(0); /* Impossible */
1134 } /* switch */
1135 if (!got_error) {
1136 intern_record_checksum += (ha_checksum)start_recpos;
1137 records++;
1138 if (param->testflag & T_WRITE_LOOP && records % WRITE_COUNT == 0) {
1139 printf("%s\r", llstr(records, llbuff));
1140 (void)fflush(stdout);
1141 }
1142
1143 /* Check if keys match the record */
1144
1145 for (key = 0, keyinfo = info->s->keyinfo; key < info->s->base.keys;
1146 key++, keyinfo++) {
1147 if (mi_is_key_active(info->s->state.key_map, key)) {
1148 if (!(keyinfo->flag & HA_FULLTEXT)) {
1149 uint key_length =
1150 _mi_make_key(info, key, info->lastkey, record, start_recpos);
1151 if (extend) {
1152 /* We don't need to lock the key tree here as we don't allow
1153 concurrent threads when running myisamchk
1154 */
1155 int search_result =
1156 (keyinfo->flag & HA_SPATIAL)
1157 ? rtree_find_first(info, key, info->lastkey, key_length,
1158 MBR_EQUAL | MBR_DATA)
1159 : _mi_search(info, keyinfo, info->lastkey, key_length,
1160 SEARCH_SAME, info->s->state.key_root[key]);
1161 if (search_result) {
1162 mi_check_print_error(param,
1163 "Record at: %10s "
1164 "Can't find key for index: %2d",
1165 llstr(start_recpos, llbuff), key + 1);
1166 if (error++ > MAXERR || !(param->testflag & T_VERBOSE))
1167 goto err2;
1168 }
1169 } else
1170 key_checksum[key] +=
1171 mi_byte_checksum((uchar *)info->lastkey, key_length);
1172 }
1173 }
1174 }
1175 } else {
1176 got_error = 0;
1177 if (error++ > MAXERR || !(param->testflag & T_VERBOSE)) goto err2;
1178 }
1179 next:; /* Next record */
1180 }
1181 if (param->testflag & T_WRITE_LOOP) {
1182 (void)fputs(" \r", stdout);
1183 (void)fflush(stdout);
1184 }
1185 if (records != info->state->records) {
1186 mi_check_print_error(param,
1187 "Record-count is not ok; is %lld Should be: %lld",
1188 records, info->state->records);
1189 error = 1;
1190 } else if (param->record_checksum &&
1191 param->record_checksum != intern_record_checksum) {
1192 mi_check_print_error(param,
1193 "Keypointers and record positions doesn't match");
1194 error = 1;
1195 } else if (param->glob_crc != info->state->checksum &&
1196 (info->s->options &
1197 (HA_OPTION_CHECKSUM | HA_OPTION_COMPRESS_RECORD))) {
1198 mi_check_print_warning(param,
1199 "Record checksum is not the same as checksum stored "
1200 "in the index file\n");
1201 error = 1;
1202 } else if (!extend) {
1203 for (key = 0; key < info->s->base.keys; key++) {
1204 if (key_checksum[key] != param->key_crc[key] &&
1205 !(info->s->keyinfo[key].flag & (HA_FULLTEXT | HA_SPATIAL))) {
1206 mi_check_print_error(
1207 param, "Checksum for key: %2d doesn't match checksum for records",
1208 key + 1);
1209 error = 1;
1210 }
1211 }
1212 }
1213
1214 if (del_length != info->state->empty) {
1215 mi_check_print_warning(param, "Found %s deleted space. Should be %s",
1216 llstr(del_length, llbuff2),
1217 llstr(info->state->empty, llbuff));
1218 }
1219 if (used + empty + del_length != info->state->data_file_length) {
1220 mi_check_print_warning(
1221 param, "Found %s record-data and %s unused data and %s deleted-data",
1222 llstr(used, llbuff), llstr(empty, llbuff2), llstr(del_length, llbuff3));
1223 mi_check_print_warning(param, "Total %s, Should be: %s",
1224 llstr((used + empty + del_length), llbuff),
1225 llstr(info->state->data_file_length, llbuff2));
1226 }
1227 if (del_blocks != info->state->del) {
1228 mi_check_print_warning(
1229 param, "Found %10s deleted blocks Should be: %s",
1230 llstr(del_blocks, llbuff), llstr(info->state->del, llbuff2));
1231 }
1232 if (splits != info->s->state.split) {
1233 mi_check_print_warning(param, "Found %lld key parts. Should be: %lld",
1234 splits, info->s->state.split);
1235 }
1236 if (param->testflag & T_INFO) {
1237 if (param->warning_printed || param->error_printed) puts("");
1238 if (used != 0 && !param->error_printed) {
1239 printf("Records:%18s M.recordlength:%9lu Packed:%14.0f%%\n",
1240 llstr(records, llbuff), (long)((used - link_used) / records),
1241 (info->s->base.blobs
1242 ? 0.0
1243 : (ulonglong2double((ulonglong)info->s->base.reclength *
1244 records) -
1245 my_off_t2double(used)) /
1246 ulonglong2double((ulonglong)info->s->base.reclength *
1247 records) *
1248 100.0));
1249 printf(
1250 "Recordspace used:%9.0f%% Empty space:%12d%% Blocks/Record: "
1251 "%6.2f\n",
1252 (ulonglong2double(used - link_used) /
1253 ulonglong2double(used - link_used + empty) * 100.0),
1254 (!records ? 100
1255 : (int)(ulonglong2double(del_length + empty) /
1256 my_off_t2double(used) * 100.0)),
1257 ulonglong2double(splits - del_blocks) / records);
1258 }
1259 printf("Record blocks:%12s Delete blocks:%10s\n",
1260 llstr(splits - del_blocks, llbuff), llstr(del_blocks, llbuff2));
1261 printf("Record data: %12s Deleted data: %10s\n",
1262 llstr(used - link_used, llbuff), llstr(del_length, llbuff2));
1263 printf("Lost space: %12s Linkdata: %10s\n", llstr(empty, llbuff),
1264 llstr(link_used, llbuff2));
1265 }
1266 my_free(mi_get_rec_buff_ptr(info, record));
1267 return error;
1268 err:
1269 mi_check_print_error(param,
1270 "got error: %d when reading datafile at record: %s",
1271 my_errno(), llstr(records, llbuff));
1272 err2:
1273 my_free(mi_get_rec_buff_ptr(info, record));
1274 param->testflag |= T_RETRY_WITHOUT_QUICK;
1275 return 1;
1276 } /* chk_data_link */
1277
1278 /**
1279 @brief Drop all indexes
1280
1281 @param[in] param check parameters
1282 @param[in] info MI_INFO handle
1283 @param[in] force if to force drop all indexes
1284
1285 @return status
1286 @retval 0 OK
1287 @retval != 0 Error
1288
1289 @note
1290 Once allocated, index blocks remain part of the key file forever.
1291 When indexes are disabled, no block is freed. When enabling indexes,
1292 no block is freed either. The new indexes are create from new
1293 blocks. (Bug #4692)
1294
1295 Before recreating formerly disabled indexes, the unused blocks
1296 must be freed. There are two options to do this:
1297 - Follow the tree of disabled indexes, add all blocks to the
1298 deleted blocks chain. Would require a lot of random I/O.
1299 - Drop all blocks by clearing all index root pointers and all
1300 delete chain pointers and resetting key_file_length to the end
1301 of the index file header. This requires to recreate all indexes,
1302 even those that may still be intact.
1303 The second method is probably faster in most cases.
1304
1305 When disabling indexes, MySQL disables either all indexes or all
1306 non-unique indexes. When MySQL [re-]enables disabled indexes
1307 (T_CREATE_MISSING_KEYS), then we either have "lost" blocks in the
1308 index file, or there are no non-unique indexes. In the latter case,
1309 mi_repair*() would not be called as there would be no disabled
1310 indexes.
1311
1312 If there would be more unique indexes than disabled (non-unique)
1313 indexes, we could do the first method. But this is not implemented
1314 yet. By now we drop and recreate all indexes when repair is called.
1315
1316 However, there is an exception. Sometimes MySQL disables non-unique
1317 indexes when the table is empty (e.g. when copying a table in
1318 mysql_alter_table()). When enabling the non-unique indexes, they
1319 are still empty. So there is no index block that can be lost. This
1320 optimization is implemented in this function.
1321
1322 Note that in normal repair (T_CREATE_MISSING_KEYS not set) we
1323 recreate all enabled indexes unconditonally. We do not change the
1324 key_map. Otherwise we invert the key map temporarily (outside of
1325 this function) and recreate the then "seemingly" enabled indexes.
1326 When we cannot use the optimization, and drop all indexes, we
1327 pretend that all indexes were disabled. By the inversion, we will
1328 then recrate all indexes.
1329 */
1330
mi_drop_all_indexes(MI_CHECK * param,MI_INFO * info,bool force)1331 static int mi_drop_all_indexes(MI_CHECK *param, MI_INFO *info, bool force) {
1332 MYISAM_SHARE *share = info->s;
1333 MI_STATE_INFO *state = &share->state;
1334 uint i;
1335 int error;
1336 DBUG_TRACE;
1337
1338 /*
1339 If any of the disabled indexes has a key block assigned, we must
1340 drop and recreate all indexes to avoid losing index blocks.
1341
1342 If we want to recreate disabled indexes only _and_ all of these
1343 indexes are empty, we don't need to recreate the existing indexes.
1344 */
1345 if (!force && (param->testflag & T_CREATE_MISSING_KEYS)) {
1346 DBUG_PRINT("repair", ("creating missing indexes"));
1347 for (i = 0; i < share->base.keys; i++) {
1348 DBUG_PRINT("repair", ("index #: %u key_root: 0x%lx active: %d", i,
1349 (long)state->key_root[i],
1350 mi_is_key_active(state->key_map, i)));
1351 if ((state->key_root[i] != HA_OFFSET_ERROR) &&
1352 !mi_is_key_active(state->key_map, i)) {
1353 /*
1354 This index has at least one key block and it is disabled.
1355 We would lose its block(s) if would just recreate it.
1356 So we need to drop and recreate all indexes.
1357 */
1358 DBUG_PRINT("repair", ("nonempty and disabled: recreate all"));
1359 break;
1360 }
1361 }
1362 if (i >= share->base.keys) {
1363 /*
1364 All of the disabled indexes are empty. We can just recreate them.
1365 Flush dirty blocks of this index file from key cache and remove
1366 all blocks of this index file from key cache.
1367 */
1368 DBUG_PRINT("repair", ("all disabled are empty: create missing"));
1369 error = flush_key_blocks(share->key_cache, keycache_thread_var(),
1370 share->kfile, FLUSH_FORCE_WRITE);
1371 goto end;
1372 }
1373 /*
1374 We do now drop all indexes and declare them disabled. With the
1375 T_CREATE_MISSING_KEYS flag, mi_repair*() will recreate all
1376 disabled indexes and enable them.
1377 */
1378 mi_clear_all_keys_active(state->key_map);
1379 DBUG_PRINT("repair", ("declared all indexes disabled"));
1380 }
1381
1382 /* Remove all key blocks of this index file from key cache. */
1383 if ((error = flush_key_blocks(share->key_cache, keycache_thread_var(),
1384 share->kfile, FLUSH_IGNORE_CHANGED)))
1385 goto end; /* purecov: inspected */
1386
1387 /* Clear index root block pointers. */
1388 for (i = 0; i < share->base.keys; i++) state->key_root[i] = HA_OFFSET_ERROR;
1389
1390 /* Clear the delete chains. */
1391 for (i = 0; i < state->header.max_block_size_index; i++)
1392 state->key_del[i] = HA_OFFSET_ERROR;
1393
1394 /* Reset index file length to end of index file header. */
1395 info->state->key_file_length = share->base.keystart;
1396
1397 DBUG_PRINT("repair", ("dropped all indexes"));
1398 /* error= 0; set by last (error= flush_key_bocks()). */
1399
1400 end:
1401 return error;
1402 }
1403
1404 /* Recover old table by reading each record and writing all keys */
1405 /* Save new datafile-name in temp_filename */
1406
mi_repair(MI_CHECK * param,MI_INFO * info,char * name,int rep_quick,bool no_copy_stat)1407 int mi_repair(MI_CHECK *param, MI_INFO *info, char *name, int rep_quick,
1408 bool no_copy_stat) {
1409 int error, got_error;
1410 ha_rows start_records, new_header_length;
1411 my_off_t del;
1412 File new_file;
1413 MYISAM_SHARE *share = info->s;
1414 char llbuff[22], llbuff2[22];
1415 SORT_INFO sort_info;
1416 MI_SORT_PARAM sort_param;
1417 DBUG_TRACE;
1418
1419 memset(&sort_info, 0, sizeof(sort_info));
1420 memset(&sort_param, 0, sizeof(sort_param));
1421 start_records = info->state->records;
1422 new_header_length =
1423 (param->testflag & T_UNPACK) ? 0L : share->pack.header_length;
1424 got_error = 1;
1425 new_file = -1;
1426 sort_param.sort_info = &sort_info;
1427
1428 if (!(param->testflag & T_SILENT)) {
1429 printf("- recovering (with keycache) MyISAM-table '%s'\n", name);
1430 printf("Data records: %s\n", llstr(info->state->records, llbuff));
1431 }
1432 param->testflag |= T_REP; /* for easy checking */
1433
1434 if (info->s->options & (HA_OPTION_CHECKSUM | HA_OPTION_COMPRESS_RECORD))
1435 param->testflag |= T_CALC_CHECKSUM;
1436
1437 DBUG_ASSERT(param->use_buffers < SIZE_T_MAX);
1438
1439 if (!param->using_global_keycache)
1440 (void)init_key_cache(dflt_key_cache, param->key_cache_block_size,
1441 (size_t)param->use_buffers, 0, 0);
1442
1443 if (init_io_cache(¶m->read_cache, info->dfile,
1444 (uint)param->read_buffer_length, READ_CACHE,
1445 share->pack.header_length, true, MYF(MY_WME))) {
1446 memset(&info->rec_cache, 0, sizeof(info->rec_cache));
1447 goto err;
1448 }
1449 if (!rep_quick)
1450 if (init_io_cache(&info->rec_cache, -1, (uint)param->write_buffer_length,
1451 WRITE_CACHE, new_header_length, true,
1452 MYF(MY_WME | MY_WAIT_IF_FULL)))
1453 goto err;
1454 info->opt_flag |= WRITE_CACHE_USED;
1455 if (!mi_alloc_rec_buff(info, -1, &sort_param.record) ||
1456 !mi_alloc_rec_buff(info, -1, &sort_param.rec_buff)) {
1457 mi_check_print_error(param, "Not enough memory for extra record");
1458 goto err;
1459 }
1460
1461 if (!rep_quick) {
1462 /* Get real path for data file */
1463 if ((new_file = mysql_file_create(
1464 mi_key_file_datatmp,
1465 fn_format(param->temp_filename, share->data_file_name, "",
1466 DATA_TMP_EXT, 2 + 4),
1467 0, param->tmpfile_createflag, MYF(0))) < 0) {
1468 mi_check_print_error(param, "Can't create new tempfile: '%s'",
1469 param->temp_filename);
1470 goto err;
1471 }
1472 if (new_header_length && filecopy(param, new_file, info->dfile, 0L,
1473 new_header_length, "datafile-header"))
1474 goto err;
1475 info->s->state.dellink = HA_OFFSET_ERROR;
1476 info->rec_cache.file = new_file;
1477 if (param->testflag & T_UNPACK) {
1478 share->options &= ~HA_OPTION_COMPRESS_RECORD;
1479 mi_int2store(share->state.header.options, share->options);
1480 }
1481 }
1482 sort_info.info = info;
1483 sort_info.param = param;
1484 sort_param.read_cache = param->read_cache;
1485 sort_param.pos = sort_param.max_pos = share->pack.header_length;
1486 sort_param.filepos = new_header_length;
1487 param->read_cache.end_of_file = sort_info.filelength =
1488 mysql_file_seek(info->dfile, 0L, MY_SEEK_END, MYF(0));
1489 sort_info.dupp = 0;
1490 sort_param.fix_datafile = (bool)(!rep_quick);
1491 sort_param.master = true;
1492 sort_info.max_records = ~(ha_rows)0;
1493
1494 set_data_file_type(&sort_info, share);
1495 del = info->state->del;
1496 info->state->records = info->state->del = share->state.split = 0;
1497 info->state->empty = 0;
1498 param->glob_crc = 0;
1499 if (param->testflag & T_CALC_CHECKSUM) sort_param.calc_checksum = true;
1500
1501 info->update = (short)(HA_STATE_CHANGED | HA_STATE_ROW_CHANGED);
1502
1503 /* This function always recreates all enabled indexes. */
1504 if (param->testflag & T_CREATE_MISSING_KEYS)
1505 mi_set_all_keys_active(share->state.key_map, share->base.keys);
1506 mi_drop_all_indexes(param, info, true);
1507
1508 /* Re-create all keys, which are set in key_map. */
1509 while (!(error = sort_get_next_record(&sort_param))) {
1510 if (writekeys(&sort_param)) {
1511 if (my_errno() != HA_ERR_FOUND_DUPP_KEY) goto err;
1512 DBUG_DUMP("record", (uchar *)sort_param.record,
1513 share->base.pack_reclength);
1514 mi_check_print_info(
1515 param,
1516 "Duplicate key %2d for record at %10s against new record at %10s",
1517 info->errkey + 1, llstr(sort_param.start_recpos, llbuff),
1518 llstr(info->dupp_key_pos, llbuff2));
1519 if (param->testflag & T_VERBOSE) {
1520 (void)_mi_make_key(info, (uint)info->errkey, info->lastkey,
1521 sort_param.record, 0L);
1522 _mi_print_key(stdout, share->keyinfo[info->errkey].seg, info->lastkey,
1523 USE_WHOLE_KEY);
1524 }
1525 sort_info.dupp++;
1526 if ((param->testflag & (T_FORCE_UNIQUENESS | T_QUICK)) == T_QUICK) {
1527 param->testflag |= T_RETRY_WITHOUT_QUICK;
1528 param->error_printed = 1;
1529 goto err;
1530 }
1531 continue;
1532 }
1533 if (sort_write_record(&sort_param)) goto err;
1534 }
1535 if (error > 0 || write_data_suffix(&sort_info, (bool)!rep_quick) ||
1536 flush_io_cache(&info->rec_cache) || param->read_cache.error < 0)
1537 goto err;
1538
1539 if (param->testflag & T_WRITE_LOOP) {
1540 (void)fputs(" \r", stdout);
1541 (void)fflush(stdout);
1542 }
1543 if (mysql_file_chsize(share->kfile, info->state->key_file_length, 0,
1544 MYF(0))) {
1545 mi_check_print_warning(param, "Can't change size of indexfile, error: %d",
1546 my_errno());
1547 goto err;
1548 }
1549
1550 if (rep_quick && del + sort_info.dupp != info->state->del) {
1551 mi_check_print_error(param,
1552 "Couldn't fix table with quick recovery: Found wrong "
1553 "number of deleted records");
1554 mi_check_print_error(param, "Run recovery again without -q");
1555 got_error = 1;
1556 param->retry_repair = true;
1557 param->testflag |= T_RETRY_WITHOUT_QUICK;
1558 goto err;
1559 }
1560 if (param->testflag & T_SAFE_REPAIR) {
1561 /* Don't repair if we loosed more than one row */
1562 if (info->state->records + 1 < start_records) {
1563 info->state->records = start_records;
1564 got_error = 1;
1565 goto err;
1566 }
1567 }
1568
1569 if (!rep_quick) {
1570 mysql_file_close(info->dfile, MYF(0));
1571 info->dfile = new_file;
1572 info->state->data_file_length = sort_param.filepos;
1573 share->state.version = (ulong)time((time_t *)nullptr); /* Force reopen */
1574 } else {
1575 info->state->data_file_length = sort_param.max_pos;
1576 }
1577 if (param->testflag & T_CALC_CHECKSUM)
1578 info->state->checksum = param->glob_crc;
1579
1580 if (!(param->testflag & T_SILENT)) {
1581 if (start_records != info->state->records)
1582 printf("Data records: %s\n", llstr(info->state->records, llbuff));
1583 if (sort_info.dupp)
1584 mi_check_print_warning(param, "%s records have been removed",
1585 llstr(sort_info.dupp, llbuff));
1586 }
1587
1588 got_error = 0;
1589 /* If invoked by external program that uses thr_lock */
1590 if (&share->state.state != info->state)
1591 memcpy(&share->state.state, info->state, sizeof(*info->state));
1592
1593 err:
1594 if (!got_error) {
1595 /* Replace the actual file with the temporary file */
1596 if (new_file >= 0) {
1597 myf flags = 0;
1598 if (param->testflag & T_BACKUP_DATA) flags |= MY_REDEL_MAKE_BACKUP;
1599 if (no_copy_stat) flags |= MY_REDEL_NO_COPY_STAT;
1600 mysql_file_close(new_file, MYF(0));
1601 info->dfile = new_file = -1;
1602 /*
1603 On Windows, the old data file cannot be deleted if it is either
1604 open, or memory mapped. Closing the file won't remove the memory
1605 map implicilty on Windows. We closed the data file, but we keep
1606 the MyISAM table open. A memory map will be closed on the final
1607 mi_close() only. So we need to unmap explicitly here. After
1608 renaming the new file under the hook, we couldn't use the map of
1609 the old file any more anyway.
1610 */
1611 if (info->s->file_map) {
1612 (void)my_munmap((char *)info->s->file_map,
1613 (size_t)info->s->mmaped_length);
1614 info->s->file_map = nullptr;
1615 }
1616 if (change_to_newfile(share->data_file_name, MI_NAME_DEXT, DATA_TMP_EXT,
1617 flags) ||
1618 mi_open_datafile(info, share, name, -1))
1619 got_error = 1;
1620
1621 param->retry_repair = false;
1622 }
1623 }
1624 if (got_error) {
1625 if (!param->error_printed)
1626 mi_check_print_error(param, "%d for record at pos %s", my_errno(),
1627 llstr(sort_param.start_recpos, llbuff));
1628 if (new_file >= 0) {
1629 (void)mysql_file_close(new_file, MYF(0));
1630 (void)mysql_file_delete(mi_key_file_datatmp, param->temp_filename,
1631 MYF(MY_WME));
1632 info->rec_cache.file = -1; /* don't flush data to new_file, it's closed */
1633 }
1634 mi_mark_crashed_on_repair(info);
1635 }
1636 my_free(mi_get_rec_buff_ptr(info, sort_param.rec_buff));
1637 my_free(mi_get_rec_buff_ptr(info, sort_param.record));
1638 my_free(sort_info.buff);
1639 (void)end_io_cache(¶m->read_cache);
1640 info->opt_flag &= ~(READ_CACHE_USED | WRITE_CACHE_USED);
1641 (void)end_io_cache(&info->rec_cache);
1642 got_error |= flush_blocks(param, share->key_cache, share->kfile);
1643 if (!got_error && param->testflag & T_UNPACK) {
1644 share->state.header.options[0] &= (uchar)~HA_OPTION_COMPRESS_RECORD;
1645 share->pack.header_length = 0;
1646 share->data_file_type = sort_info.new_data_file_type;
1647 }
1648 share->state.changed |=
1649 (STATE_NOT_OPTIMIZED_KEYS | STATE_NOT_SORTED_PAGES | STATE_NOT_ANALYZED);
1650 return got_error;
1651 }
1652
1653 /* Uppate keyfile when doing repair */
1654
writekeys(MI_SORT_PARAM * sort_param)1655 static int writekeys(MI_SORT_PARAM *sort_param) {
1656 uint i;
1657 uchar *key;
1658 MI_INFO *info = sort_param->sort_info->info;
1659 uchar *buff = sort_param->record;
1660 my_off_t filepos = sort_param->filepos;
1661 DBUG_TRACE;
1662
1663 key = info->lastkey + info->s->base.max_key_length;
1664 for (i = 0; i < info->s->base.keys; i++) {
1665 if (mi_is_key_active(info->s->state.key_map, i)) {
1666 if (info->s->keyinfo[i].flag & HA_FULLTEXT) {
1667 if (_mi_ft_add(info, i, key, buff, filepos)) goto err;
1668 } else if (info->s->keyinfo[i].flag & HA_SPATIAL) {
1669 uint key_length = _mi_make_key(info, i, key, buff, filepos);
1670 if (rtree_insert(info, i, key, key_length)) goto err;
1671 } else {
1672 uint key_length = _mi_make_key(info, i, key, buff, filepos);
1673 if (_mi_ck_write(info, i, key, key_length)) goto err;
1674 }
1675 }
1676 }
1677 return 0;
1678
1679 err:
1680 if (my_errno() == HA_ERR_FOUND_DUPP_KEY) {
1681 info->errkey = (int)i; /* This key was found */
1682 while (i-- > 0) {
1683 if (mi_is_key_active(info->s->state.key_map, i)) {
1684 if (info->s->keyinfo[i].flag & HA_FULLTEXT) {
1685 if (_mi_ft_del(info, i, key, buff, filepos)) break;
1686 } else {
1687 uint key_length = _mi_make_key(info, i, key, buff, filepos);
1688 if (_mi_ck_delete(info, i, key, key_length)) break;
1689 }
1690 }
1691 }
1692 }
1693 /* Remove checksum that was added to glob_crc in sort_get_next_record */
1694 if (sort_param->calc_checksum)
1695 sort_param->sort_info->param->glob_crc -= info->checksum;
1696 DBUG_PRINT("error", ("errno: %d", my_errno()));
1697 return -1;
1698 } /* writekeys */
1699
1700 /* Change all key-pointers that points to a records */
1701
movepoint(MI_INFO * info,uchar * record,my_off_t oldpos,my_off_t newpos,uint prot_key)1702 int movepoint(MI_INFO *info, uchar *record, my_off_t oldpos, my_off_t newpos,
1703 uint prot_key) {
1704 uint i;
1705 uchar *key;
1706 uint key_length;
1707 DBUG_TRACE;
1708
1709 key = info->lastkey + info->s->base.max_key_length;
1710 for (i = 0; i < info->s->base.keys; i++) {
1711 if (i != prot_key && mi_is_key_active(info->s->state.key_map, i)) {
1712 key_length = _mi_make_key(info, i, key, record, oldpos);
1713 if (info->s->keyinfo[i].flag & HA_NOSAME) { /* Change pointer direct */
1714 uint nod_flag;
1715 MI_KEYDEF *keyinfo;
1716 keyinfo = info->s->keyinfo + i;
1717 if (_mi_search(info, keyinfo, key, USE_WHOLE_KEY,
1718 (uint)(SEARCH_SAME | SEARCH_SAVE_BUFF),
1719 info->s->state.key_root[i]))
1720 return -1;
1721 nod_flag = mi_test_if_nod(info->buff);
1722 _mi_dpointer(info, info->int_keypos - nod_flag - info->s->rec_reflength,
1723 newpos);
1724 if (_mi_write_keypage(info, keyinfo, info->last_keypage, DFLT_INIT_HITS,
1725 info->buff))
1726 return -1;
1727 } else { /* Change old key to new */
1728 if (_mi_ck_delete(info, i, key, key_length)) return -1;
1729 key_length = _mi_make_key(info, i, key, record, newpos);
1730 if (_mi_ck_write(info, i, key, key_length)) return -1;
1731 }
1732 }
1733 }
1734 return 0;
1735 } /* movepoint */
1736
1737 /* Flush all changed blocks to disk */
1738
flush_blocks(MI_CHECK * param,KEY_CACHE * key_cache,File file)1739 int flush_blocks(MI_CHECK *param, KEY_CACHE *key_cache, File file) {
1740 if (flush_key_blocks(key_cache, keycache_thread_var(), file, FLUSH_RELEASE)) {
1741 mi_check_print_error(param, "%d when trying to write bufferts", my_errno());
1742 return (1);
1743 }
1744 if (!param->using_global_keycache) end_key_cache(key_cache, true);
1745 return 0;
1746 } /* flush_blocks */
1747
1748 /* Sort index for more efficent reads */
1749
mi_sort_index(MI_CHECK * param,MI_INFO * info,char * name,bool no_copy_stat)1750 int mi_sort_index(MI_CHECK *param, MI_INFO *info, char *name,
1751 bool no_copy_stat) {
1752 uint key;
1753 MI_KEYDEF *keyinfo;
1754 File new_file;
1755 my_off_t index_pos[HA_MAX_POSSIBLE_KEY];
1756 uint r_locks, w_locks;
1757 int old_lock;
1758 MYISAM_SHARE *share = info->s;
1759 MI_STATE_INFO old_state;
1760 DBUG_TRACE;
1761
1762 /* cannot sort index files with R-tree indexes */
1763 for (key = 0, keyinfo = &share->keyinfo[0]; key < share->base.keys;
1764 key++, keyinfo++)
1765 if (keyinfo->key_alg == HA_KEY_ALG_RTREE) return 0;
1766
1767 if (!(param->testflag & T_SILENT))
1768 printf("- Sorting index for MyISAM-table '%s'\n", name);
1769
1770 /* Get real path for index file */
1771 fn_format(param->temp_filename, name, "", MI_NAME_IEXT, 2 + 4 + 32);
1772 if ((new_file = mysql_file_create(
1773 mi_key_file_datatmp,
1774 fn_format(param->temp_filename, param->temp_filename, "",
1775 INDEX_TMP_EXT, 2 + 4),
1776 0, param->tmpfile_createflag, MYF(0))) <= 0) {
1777 mi_check_print_error(param, "Can't create new tempfile: '%s'",
1778 param->temp_filename);
1779 return -1;
1780 }
1781 if (filecopy(param, new_file, share->kfile, 0L, (ulong)share->base.keystart,
1782 "headerblock"))
1783 goto err;
1784
1785 param->new_file_pos = share->base.keystart;
1786 for (key = 0, keyinfo = &share->keyinfo[0]; key < share->base.keys;
1787 key++, keyinfo++) {
1788 if (!mi_is_key_active(info->s->state.key_map, key)) {
1789 /* Since the key is not active, this should not be read, but we
1790 initialize it anyway to silence a Valgrind warn when passing that
1791 chunk of memory to pwrite(). */
1792 index_pos[key] = HA_OFFSET_ERROR;
1793 continue;
1794 }
1795
1796 if (share->state.key_root[key] != HA_OFFSET_ERROR) {
1797 index_pos[key] = param->new_file_pos; /* Write first block here */
1798 if (sort_one_index(param, info, keyinfo, share->state.key_root[key],
1799 new_file))
1800 goto err;
1801 } else
1802 index_pos[key] = HA_OFFSET_ERROR; /* No blocks */
1803 }
1804
1805 /* Flush key cache for this file if we are calling this outside myisamchk */
1806 flush_key_blocks(share->key_cache, keycache_thread_var(), share->kfile,
1807 FLUSH_IGNORE_CHANGED);
1808
1809 share->state.version = (ulong)time((time_t *)nullptr);
1810 old_state = share->state; /* save state if not stored */
1811 r_locks = share->r_locks;
1812 w_locks = share->w_locks;
1813 old_lock = info->lock_type;
1814
1815 /* Put same locks as old file */
1816 share->r_locks = share->w_locks = share->tot_locks = 0;
1817 (void)_mi_writeinfo(info, WRITEINFO_UPDATE_KEYFILE);
1818 (void)mysql_file_close(share->kfile, MYF(MY_WME));
1819 share->kfile = -1;
1820 (void)mysql_file_close(new_file, MYF(MY_WME));
1821 if (change_to_newfile(share->index_file_name, MI_NAME_IEXT, INDEX_TMP_EXT,
1822 no_copy_stat ? MYF(MY_REDEL_NO_COPY_STAT) : MYF(0)) ||
1823 mi_open_keyfile(share))
1824 goto err2;
1825 info->lock_type = F_UNLCK; /* Force mi_readinfo to lock */
1826 _mi_readinfo(info, F_WRLCK, 0); /* Will lock the table */
1827 info->lock_type = old_lock;
1828 share->r_locks = r_locks;
1829 share->w_locks = w_locks;
1830 share->tot_locks = r_locks + w_locks;
1831 share->state = old_state; /* Restore old state */
1832
1833 info->state->key_file_length = param->new_file_pos;
1834 info->update = (short)(HA_STATE_CHANGED | HA_STATE_ROW_CHANGED);
1835 for (key = 0; key < info->s->base.keys; key++)
1836 info->s->state.key_root[key] = index_pos[key];
1837 for (key = 0; key < info->s->state.header.max_block_size_index; key++)
1838 info->s->state.key_del[key] = HA_OFFSET_ERROR;
1839
1840 info->s->state.changed &= ~STATE_NOT_SORTED_PAGES;
1841 return 0;
1842
1843 err:
1844 (void)mysql_file_close(new_file, MYF(MY_WME));
1845 err2:
1846 (void)mysql_file_delete(mi_key_file_datatmp, param->temp_filename,
1847 MYF(MY_WME));
1848 return -1;
1849 } /* mi_sort_index */
1850
1851 /* Sort records recursive using one index */
1852
sort_one_index(MI_CHECK * param,MI_INFO * info,MI_KEYDEF * keyinfo,my_off_t pagepos,File new_file)1853 static int sort_one_index(MI_CHECK *param, MI_INFO *info, MI_KEYDEF *keyinfo,
1854 my_off_t pagepos, File new_file) {
1855 uint length, nod_flag, used_length, key_length;
1856 uchar *buff, *keypos, *endpos;
1857 uchar key[HA_MAX_POSSIBLE_KEY_BUFF];
1858 my_off_t new_page_pos, next_page;
1859 char llbuff[22];
1860 DBUG_TRACE;
1861
1862 /* cannot walk over R-tree indices */
1863 DBUG_ASSERT(keyinfo->key_alg != HA_KEY_ALG_RTREE);
1864 new_page_pos = param->new_file_pos;
1865 param->new_file_pos += keyinfo->block_length;
1866
1867 if (!(buff = (uchar *)my_alloca((uint)keyinfo->block_length))) {
1868 mi_check_print_error(param, "Not enough memory for key block");
1869 return -1;
1870 }
1871 if (!_mi_fetch_keypage(info, keyinfo, pagepos, DFLT_INIT_HITS, buff, 0)) {
1872 mi_check_print_error(param, "Can't read key block from filepos: %s",
1873 llstr(pagepos, llbuff));
1874 goto err;
1875 }
1876 if ((nod_flag = mi_test_if_nod(buff)) || keyinfo->flag & HA_FULLTEXT) {
1877 used_length = mi_getint(buff);
1878 keypos = buff + 2 + nod_flag;
1879 endpos = buff + used_length;
1880 for (;;) {
1881 if (nod_flag) {
1882 next_page = _mi_kpos(nod_flag, keypos);
1883 _mi_kpointer(info, keypos - nod_flag,
1884 param->new_file_pos); /* Save new pos */
1885 if (sort_one_index(param, info, keyinfo, next_page, new_file)) {
1886 DBUG_PRINT(
1887 "error",
1888 ("From page: %ld, keyoffset: %lu used_length: %d",
1889 (ulong)pagepos, (ulong)(keypos - buff), (int)used_length));
1890 DBUG_DUMP("buff", (uchar *)buff, used_length);
1891 goto err;
1892 }
1893 }
1894 if (keypos >= endpos || (key_length = (*keyinfo->get_key)(
1895 keyinfo, nod_flag, &keypos, key)) == 0)
1896 break;
1897 DBUG_ASSERT(keypos <= endpos);
1898 if (keyinfo->flag & HA_FULLTEXT) {
1899 uint off;
1900 int subkeys;
1901 get_key_full_length_rdonly(off, key);
1902 subkeys = ft_sintXkorr(key + off);
1903 if (subkeys < 0) {
1904 next_page = _mi_dpos(info, 0, key + key_length);
1905 _mi_dpointer(info, keypos - nod_flag - info->s->rec_reflength,
1906 param->new_file_pos); /* Save new pos */
1907 if (sort_one_index(param, info, &info->s->ft2_keyinfo, next_page,
1908 new_file))
1909 goto err;
1910 }
1911 }
1912 }
1913 }
1914
1915 /* Fill block with zero and write it to the new index file */
1916 length = mi_getint(buff);
1917 memset(buff + length, 0, keyinfo->block_length - length);
1918 if (mysql_file_pwrite(new_file, (uchar *)buff, (uint)keyinfo->block_length,
1919 new_page_pos, MYF(MY_NABP | MY_WAIT_IF_FULL))) {
1920 mi_check_print_error(param, "Can't write indexblock, error: %d",
1921 my_errno());
1922 goto err;
1923 }
1924 return 0;
1925 err:
1926 return 1;
1927 } /* sort_one_index */
1928
1929 /*
1930 Let temporary file replace old file.
1931 This assumes that the new file was created in the same
1932 directory as given by realpath(filename).
1933 This will ensure that any symlinks that are used will still work.
1934 Copy stats from old file to new file, deletes orignal and
1935 changes new file name to old file name
1936 */
1937
change_to_newfile(const char * filename,const char * old_ext,const char * new_ext,myf MyFlags)1938 int change_to_newfile(const char *filename, const char *old_ext,
1939 const char *new_ext, myf MyFlags) {
1940 char old_filename[FN_REFLEN], new_filename[FN_REFLEN];
1941 /* Get real path to filename */
1942 (void)fn_format(old_filename, filename, "", old_ext, 2 + 4 + 32);
1943 return my_redel(old_filename,
1944 fn_format(new_filename, old_filename, "", new_ext, 2 + 4),
1945 MYF(MY_WME | MY_LINK_WARNING | MyFlags));
1946 } /* change_to_newfile */
1947
1948 /* Locks a whole file */
1949 /* Gives an error-message if file can't be locked */
1950
lock_file(MI_CHECK * param,File file,int lock_type,const char * filetype,const char * filename)1951 int lock_file(MI_CHECK *param, File file, int lock_type, const char *filetype,
1952 const char *filename) {
1953 if (my_lock(file, lock_type,
1954 param->testflag & T_WAIT_FOREVER
1955 ? MYF(MY_SEEK_NOT_DONE)
1956 : MYF(MY_SEEK_NOT_DONE | MY_DONT_WAIT))) {
1957 mi_check_print_error(param, " %d when locking %s '%s'", my_errno(),
1958 filetype, filename);
1959 param->error_printed = 2; /* Don't give that data is crashed */
1960 return 1;
1961 }
1962 return 0;
1963 } /* lock_file */
1964
1965 /* Copy a block between two files */
1966
filecopy(MI_CHECK * param,File to,File from,my_off_t start,my_off_t length,const char * type)1967 int filecopy(MI_CHECK *param, File to, File from, my_off_t start,
1968 my_off_t length, const char *type) {
1969 char tmp_buff[IO_SIZE], *buff;
1970 ulong buff_length;
1971 DBUG_TRACE;
1972
1973 buff_length = std::min<ulong>(param->write_buffer_length, length);
1974 if (!(buff =
1975 (char *)my_malloc(mi_key_memory_filecopy, buff_length, MYF(0)))) {
1976 buff = tmp_buff;
1977 buff_length = IO_SIZE;
1978 }
1979
1980 mysql_file_seek(from, start, MY_SEEK_SET, MYF(0));
1981 while (length > buff_length) {
1982 if (mysql_file_read(from, (uchar *)buff, buff_length, MYF(MY_NABP)) ||
1983 mysql_file_write(to, (uchar *)buff, buff_length, param->myf_rw))
1984 goto err;
1985 length -= buff_length;
1986 }
1987 if (mysql_file_read(from, (uchar *)buff, (uint)length, MYF(MY_NABP)) ||
1988 mysql_file_write(to, (uchar *)buff, (uint)length, param->myf_rw))
1989 goto err;
1990 if (buff != tmp_buff) my_free(buff);
1991 return 0;
1992 err:
1993 if (buff != tmp_buff) my_free(buff);
1994 mi_check_print_error(param, "Can't copy %s to tempfile, error %d", type,
1995 my_errno());
1996 return 1;
1997 }
1998
1999 /*
2000 Repair table or given index using sorting
2001
2002 SYNOPSIS
2003 mi_repair_by_sort()
2004 param Repair parameters
2005 info MyISAM handler to repair
2006 name Name of table (for warnings)
2007 rep_quick set to <> 0 if we should not change data file
2008 no_copy_stat Don't copy file stats from old to new file,
2009 assume that new file was created with correct stats
2010
2011 RESULT
2012 0 ok
2013 <>0 Error
2014 */
2015
mi_repair_by_sort(MI_CHECK * param,MI_INFO * info,const char * name,int rep_quick,bool no_copy_stat)2016 int mi_repair_by_sort(MI_CHECK *param, MI_INFO *info, const char *name,
2017 int rep_quick, bool no_copy_stat) {
2018 int got_error;
2019 uint i;
2020 ulong length;
2021 ha_rows start_records;
2022 my_off_t new_header_length, del;
2023 File new_file;
2024 MI_SORT_PARAM sort_param;
2025 MYISAM_SHARE *share = info->s;
2026 HA_KEYSEG *keyseg;
2027 ulong *rec_per_key_part;
2028 char llbuff[22];
2029 SORT_INFO sort_info;
2030 ulonglong key_map = 0;
2031 DBUG_TRACE;
2032
2033 start_records = info->state->records;
2034 got_error = 1;
2035 new_file = -1;
2036 new_header_length =
2037 (param->testflag & T_UNPACK) ? 0 : share->pack.header_length;
2038 if (!(param->testflag & T_SILENT)) {
2039 printf("- recovering (with sort) MyISAM-table '%s'\n", name);
2040 printf("Data records: %s\n", llstr(start_records, llbuff));
2041 }
2042 param->testflag |= T_REP; /* for easy checking */
2043
2044 if (info->s->options & (HA_OPTION_CHECKSUM | HA_OPTION_COMPRESS_RECORD))
2045 param->testflag |= T_CALC_CHECKSUM;
2046
2047 memset(&sort_info, 0, sizeof(sort_info));
2048 memset(&sort_param, 0, sizeof(sort_param));
2049 if (!(sort_info.key_block =
2050 alloc_key_blocks(param, (uint)param->sort_key_blocks,
2051 share->base.max_key_block_length)) ||
2052 init_io_cache(¶m->read_cache, info->dfile,
2053 (uint)param->read_buffer_length, READ_CACHE,
2054 share->pack.header_length, true, MYF(MY_WME)) ||
2055 (!rep_quick &&
2056 init_io_cache(&info->rec_cache, info->dfile,
2057 (uint)param->write_buffer_length, WRITE_CACHE,
2058 new_header_length, true,
2059 MYF(MY_WME | MY_WAIT_IF_FULL) & param->myf_rw)))
2060 goto err;
2061 sort_info.key_block_end = sort_info.key_block + param->sort_key_blocks;
2062 info->opt_flag |= WRITE_CACHE_USED;
2063 info->rec_cache.file = info->dfile; /* for sort_delete_record */
2064
2065 if (!mi_alloc_rec_buff(info, -1, &sort_param.record) ||
2066 !mi_alloc_rec_buff(info, -1, &sort_param.rec_buff)) {
2067 mi_check_print_error(param, "Not enough memory for extra record");
2068 goto err;
2069 }
2070 if (!rep_quick) {
2071 /* Get real path for data file */
2072 if ((new_file = mysql_file_create(
2073 mi_key_file_datatmp,
2074 fn_format(param->temp_filename, share->data_file_name, "",
2075 DATA_TMP_EXT, 2 + 4),
2076 0, param->tmpfile_createflag, MYF(0))) < 0) {
2077 mi_check_print_error(param, "Can't create new tempfile: '%s'",
2078 param->temp_filename);
2079 goto err;
2080 }
2081 if (new_header_length && filecopy(param, new_file, info->dfile, 0L,
2082 new_header_length, "datafile-header"))
2083 goto err;
2084 if (param->testflag & T_UNPACK) {
2085 share->options &= ~HA_OPTION_COMPRESS_RECORD;
2086 mi_int2store(share->state.header.options, share->options);
2087 }
2088 share->state.dellink = HA_OFFSET_ERROR;
2089 info->rec_cache.file = new_file;
2090 }
2091
2092 info->update = (short)(HA_STATE_CHANGED | HA_STATE_ROW_CHANGED);
2093
2094 /* Optionally drop indexes and optionally modify the key_map. */
2095 mi_drop_all_indexes(param, info, false);
2096 key_map = share->state.key_map;
2097 if (param->testflag & T_CREATE_MISSING_KEYS) {
2098 /* Invert the copied key_map to recreate all disabled indexes. */
2099 key_map = ~key_map;
2100 }
2101
2102 sort_info.info = info;
2103 sort_info.param = param;
2104
2105 set_data_file_type(&sort_info, share);
2106 sort_param.filepos = new_header_length;
2107 sort_info.dupp = 0;
2108 sort_info.buff = nullptr;
2109 param->read_cache.end_of_file = sort_info.filelength =
2110 mysql_file_seek(param->read_cache.file, 0L, MY_SEEK_END, MYF(0));
2111
2112 sort_param.wordlist = nullptr;
2113 init_alloc_root(mi_key_memory_MI_SORT_PARAM_wordroot, &sort_param.wordroot,
2114 FTPARSER_MEMROOT_ALLOC_SIZE, 0);
2115
2116 if (share->data_file_type == DYNAMIC_RECORD)
2117 length =
2118 std::max(share->base.min_pack_length + 1, share->base.min_block_length);
2119 else if (share->data_file_type == COMPRESSED_RECORD)
2120 length = share->base.min_block_length;
2121 else
2122 length = share->base.pack_reclength;
2123 sort_info.max_records = ((param->testflag & T_CREATE_MISSING_KEYS)
2124 ? info->state->records
2125 : (ha_rows)(sort_info.filelength / length + 1));
2126 sort_param.key_cmp = sort_key_cmp;
2127 sort_param.tmpdir = param->tmpdir;
2128 sort_param.sort_info = &sort_info;
2129 sort_param.fix_datafile = (bool)(!rep_quick);
2130 sort_param.master = true;
2131
2132 del = info->state->del;
2133 param->glob_crc = 0;
2134 if (param->testflag & T_CALC_CHECKSUM) sort_param.calc_checksum = true;
2135
2136 rec_per_key_part = param->rec_per_key_part;
2137 for (sort_param.key = 0; sort_param.key < share->base.keys;
2138 rec_per_key_part += sort_param.keyinfo->keysegs, sort_param.key++) {
2139 sort_param.read_cache = param->read_cache;
2140 sort_param.keyinfo = share->keyinfo + sort_param.key;
2141 sort_param.seg = sort_param.keyinfo->seg;
2142 /*
2143 Skip this index if it is marked disabled in the copied
2144 (and possibly inverted) key_map.
2145 */
2146 if (!mi_is_key_active(key_map, sort_param.key)) {
2147 /* Remember old statistics for key */
2148 memcpy((char *)rec_per_key_part,
2149 (char *)(share->state.rec_per_key_part +
2150 (uint)(rec_per_key_part - param->rec_per_key_part)),
2151 sort_param.keyinfo->keysegs * sizeof(*rec_per_key_part));
2152 DBUG_PRINT("repair",
2153 ("skipping seemingly disabled index #: %u", sort_param.key));
2154 continue;
2155 }
2156
2157 if ((!(param->testflag & T_SILENT)))
2158 printf("- Fixing index %d\n", sort_param.key + 1);
2159 sort_param.max_pos = sort_param.pos = share->pack.header_length;
2160 keyseg = sort_param.seg;
2161 memset(sort_param.unique, 0, sizeof(sort_param.unique));
2162 sort_param.key_length = share->rec_reflength;
2163 for (i = 0; keyseg[i].type != HA_KEYTYPE_END; i++) {
2164 sort_param.key_length += keyseg[i].length;
2165 if (keyseg[i].flag & HA_SPACE_PACK)
2166 sort_param.key_length += get_pack_length(keyseg[i].length);
2167 if (keyseg[i].flag & (HA_BLOB_PART | HA_VAR_LENGTH_PART))
2168 sort_param.key_length += 2 + (keyseg[i].length >= 127);
2169 if (keyseg[i].flag & HA_NULL_PART) sort_param.key_length++;
2170 }
2171 info->state->records = info->state->del = share->state.split = 0;
2172 info->state->empty = 0;
2173
2174 if (sort_param.keyinfo->flag & HA_FULLTEXT) {
2175 uint ft_max_word_len_for_sort =
2176 FT_MAX_WORD_LEN_FOR_SORT * sort_param.keyinfo->seg->charset->mbmaxlen;
2177 sort_param.key_length += ft_max_word_len_for_sort - HA_FT_MAXBYTELEN;
2178 /*
2179 fulltext indexes may have much more entries than the
2180 number of rows in the table. We estimate the number here.
2181 */
2182 if (sort_param.keyinfo->parser == &ft_default_parser) {
2183 /*
2184 for built-in parser the number of generated index entries
2185 cannot be larger than the size of the data file divided
2186 by the minimal word's length
2187 */
2188 sort_info.max_records =
2189 (ha_rows)(sort_info.filelength / ft_min_word_len + 1);
2190 } else {
2191 /*
2192 for external plugin parser we cannot tell anything at all :(
2193 so, we'll use all the sort memory and start from ~10 buffpeks.
2194 (see _create_index_by_sort)
2195 */
2196 sort_info.max_records =
2197 10 * std::max(param->sort_buffer_length, MIN_SORT_BUFFER) /
2198 sort_param.key_length;
2199 }
2200
2201 sort_param.key_read = sort_ft_key_read;
2202 sort_param.key_write = sort_ft_key_write;
2203 } else {
2204 sort_param.key_read = sort_key_read;
2205 sort_param.key_write = sort_key_write;
2206 }
2207
2208 if (_create_index_by_sort(&sort_param,
2209 (bool)(!(param->testflag & T_VERBOSE)),
2210 param->sort_buffer_length)) {
2211 param->retry_repair = true;
2212 goto err;
2213 }
2214 /* No need to calculate checksum again. */
2215 sort_param.calc_checksum = false;
2216 free_root(&sort_param.wordroot, MYF(0));
2217
2218 /* Set for next loop */
2219 sort_info.max_records = (ha_rows)info->state->records;
2220
2221 if (param->testflag & T_STATISTICS)
2222 update_key_parts(sort_param.keyinfo, rec_per_key_part, sort_param.unique,
2223 param->stats_method == MI_STATS_METHOD_IGNORE_NULLS
2224 ? sort_param.notnull
2225 : nullptr,
2226 (ulonglong)info->state->records);
2227 /* Enable this index in the permanent (not the copied) key_map. */
2228 mi_set_key_active(share->state.key_map, sort_param.key);
2229 DBUG_PRINT("repair", ("set enabled index #: %u", sort_param.key));
2230
2231 if (sort_param.fix_datafile) {
2232 param->read_cache.end_of_file = sort_param.filepos;
2233 if (write_data_suffix(&sort_info, true) || end_io_cache(&info->rec_cache))
2234 goto err;
2235 if (param->testflag & T_SAFE_REPAIR) {
2236 /* Don't repair if we loosed more than one row */
2237 if (info->state->records + 1 < start_records) {
2238 info->state->records = start_records;
2239 goto err;
2240 }
2241 }
2242 share->state.state.data_file_length = info->state->data_file_length =
2243 sort_param.filepos;
2244 /* Only whole records */
2245 share->state.version = (ulong)time((time_t *)nullptr);
2246 mysql_file_close(info->dfile, MYF(0));
2247 info->dfile = new_file;
2248 share->data_file_type = sort_info.new_data_file_type;
2249 share->pack.header_length = (ulong)new_header_length;
2250 sort_param.fix_datafile = false;
2251 } else
2252 info->state->data_file_length = sort_param.max_pos;
2253
2254 param->read_cache.file = info->dfile; /* re-init read cache */
2255 reinit_io_cache(¶m->read_cache, READ_CACHE, share->pack.header_length,
2256 true, true);
2257 }
2258
2259 if (param->testflag & T_WRITE_LOOP) {
2260 (void)fputs(" \r", stdout);
2261 (void)fflush(stdout);
2262 }
2263
2264 if (rep_quick && del + sort_info.dupp != info->state->del) {
2265 mi_check_print_error(param,
2266 "Couldn't fix table with quick recovery: Found wrong "
2267 "number of deleted records");
2268 mi_check_print_error(param, "Run recovery again without -q");
2269 got_error = 1;
2270 param->retry_repair = true;
2271 param->testflag |= T_RETRY_WITHOUT_QUICK;
2272 goto err;
2273 }
2274
2275 if (rep_quick & T_FORCE_UNIQUENESS) {
2276 my_off_t skr =
2277 info->state->data_file_length +
2278 (share->options & HA_OPTION_COMPRESS_RECORD ? MEMMAP_EXTRA_MARGIN : 0);
2279 if (skr != sort_info.filelength)
2280 if (mysql_file_chsize(info->dfile, skr, 0, MYF(0)))
2281 mi_check_print_warning(
2282 param, "Can't change size of datafile, error: %d", my_errno());
2283 }
2284 if (param->testflag & T_CALC_CHECKSUM)
2285 info->state->checksum = param->glob_crc;
2286
2287 if (mysql_file_chsize(share->kfile, info->state->key_file_length, 0, MYF(0)))
2288 mi_check_print_warning(param, "Can't change size of indexfile, error: %d",
2289 my_errno());
2290
2291 if (!(param->testflag & T_SILENT)) {
2292 if (start_records != info->state->records)
2293 printf("Data records: %s\n", llstr(info->state->records, llbuff));
2294 if (sort_info.dupp)
2295 mi_check_print_warning(param, "%s records have been removed",
2296 llstr(sort_info.dupp, llbuff));
2297 }
2298 got_error = 0;
2299
2300 if (&share->state.state != info->state)
2301 memcpy(&share->state.state, info->state, sizeof(*info->state));
2302
2303 err:
2304 got_error |= flush_blocks(param, share->key_cache, share->kfile);
2305 (void)end_io_cache(&info->rec_cache);
2306 if (!got_error) {
2307 /* Replace the actual file with the temporary file */
2308 if (new_file >= 0) {
2309 myf flags = 0;
2310 if (param->testflag & T_BACKUP_DATA) flags |= MY_REDEL_MAKE_BACKUP;
2311 if (no_copy_stat) flags |= MY_REDEL_NO_COPY_STAT;
2312 mysql_file_close(new_file, MYF(0));
2313 info->dfile = new_file = -1;
2314 if (change_to_newfile(share->data_file_name, MI_NAME_DEXT, DATA_TMP_EXT,
2315 flags) ||
2316 mi_open_datafile(info, share, name, -1))
2317 got_error = 1;
2318 }
2319 }
2320 if (got_error) {
2321 if (!param->error_printed)
2322 mi_check_print_error(param, "%d when fixing table", my_errno());
2323 if (new_file >= 0) {
2324 (void)mysql_file_close(new_file, MYF(0));
2325 (void)mysql_file_delete(mi_key_file_datatmp, param->temp_filename,
2326 MYF(MY_WME));
2327 if (info->dfile == new_file) /* Retry with key cache */
2328 if (unlikely(mi_open_datafile(info, share, name, -1)))
2329 param->retry_repair = false; /* Safety */
2330 }
2331 mi_mark_crashed_on_repair(info);
2332 } else if (key_map == share->state.key_map)
2333 share->state.changed &= ~STATE_NOT_OPTIMIZED_KEYS;
2334 share->state.changed |= STATE_NOT_SORTED_PAGES;
2335
2336 my_free(mi_get_rec_buff_ptr(info, sort_param.rec_buff));
2337 my_free(mi_get_rec_buff_ptr(info, sort_param.record));
2338 my_free(sort_info.key_block);
2339 my_free(sort_info.ft_buf);
2340 my_free(sort_info.buff);
2341 (void)end_io_cache(¶m->read_cache);
2342 info->opt_flag &= ~(READ_CACHE_USED | WRITE_CACHE_USED);
2343 if (!got_error && (param->testflag & T_UNPACK)) {
2344 share->state.header.options[0] &= (uchar)~HA_OPTION_COMPRESS_RECORD;
2345 share->pack.header_length = 0;
2346 }
2347 return got_error;
2348 }
2349
2350 /*
2351 Threaded repair of table using sorting
2352
2353 SYNOPSIS
2354 mi_repair_parallel()
2355 param Repair parameters
2356 info MyISAM handler to repair
2357 name Name of table (for warnings)
2358 rep_quick set to <> 0 if we should not change data file
2359 no_copy_stat Don't copy file stats from old to new file,
2360 assume that new file was created with correct stats
2361
2362 DESCRIPTION
2363 Same as mi_repair_by_sort but do it multithreaded
2364 Each key is handled by a separate thread.
2365 TODO: make a number of threads a parameter
2366
2367 In parallel repair we use one thread per index. There are two modes:
2368
2369 Quick
2370
2371 Only the indexes are rebuilt. All threads share a read buffer.
2372 Every thread that needs fresh data in the buffer enters the shared
2373 cache lock. The last thread joining the lock reads the buffer from
2374 the data file and wakes all other threads.
2375
2376 Non-quick
2377
2378 The data file is rebuilt and all indexes are rebuilt to point to
2379 the new record positions. One thread is the master thread. It
2380 reads from the old data file and writes to the new data file. It
2381 also creates one of the indexes. The other threads read from a
2382 buffer which is filled by the master. If they need fresh data,
2383 they enter the shared cache lock. If the masters write buffer is
2384 full, it flushes it to the new data file and enters the shared
2385 cache lock too. When all threads joined in the lock, the master
2386 copies its write buffer to the read buffer for the other threads
2387 and wakes them.
2388
2389 RESULT
2390 0 ok
2391 <>0 Error
2392 */
2393
mi_repair_parallel(MI_CHECK * param,MI_INFO * info,const char * name,int rep_quick,bool no_copy_stat)2394 int mi_repair_parallel(MI_CHECK *param, MI_INFO *info, const char *name,
2395 int rep_quick, bool no_copy_stat) {
2396 int got_error;
2397 uint i, key, total_key_length, istep;
2398 ulong rec_length;
2399 ha_rows start_records;
2400 my_off_t new_header_length, del;
2401 File new_file;
2402 MI_SORT_PARAM *sort_param = nullptr;
2403 MYISAM_SHARE *share = info->s;
2404 ulong *rec_per_key_part;
2405 HA_KEYSEG *keyseg;
2406 char llbuff[22];
2407 IO_CACHE new_data_cache; /* For non-quick repair. */
2408 IO_CACHE_SHARE io_share;
2409 SORT_INFO sort_info;
2410 ulonglong key_map = 0;
2411 my_thread_attr_t thr_attr;
2412 ulong max_pack_reclength;
2413 int error;
2414 DBUG_TRACE;
2415
2416 memset(&new_data_cache, 0, sizeof(IO_CACHE));
2417 start_records = info->state->records;
2418 got_error = 1;
2419 new_file = -1;
2420 new_header_length =
2421 (param->testflag & T_UNPACK) ? 0 : share->pack.header_length;
2422 if (!(param->testflag & T_SILENT)) {
2423 printf("- parallel recovering (with sort) MyISAM-table '%s'\n", name);
2424 printf("Data records: %s\n", llstr(start_records, llbuff));
2425 }
2426 param->testflag |= T_REP; /* for easy checking */
2427
2428 if (info->s->options & (HA_OPTION_CHECKSUM | HA_OPTION_COMPRESS_RECORD))
2429 param->testflag |= T_CALC_CHECKSUM;
2430
2431 /*
2432 Quick repair (not touching data file, rebuilding indexes):
2433 {
2434 Read cache is (MI_CHECK *param)->read_cache using info->dfile.
2435 }
2436
2437 Non-quick repair (rebuilding data file and indexes):
2438 {
2439 Master thread:
2440
2441 Read cache is (MI_CHECK *param)->read_cache using info->dfile.
2442 Write cache is (MI_INFO *info)->rec_cache using new_file.
2443
2444 Slave threads:
2445
2446 Read cache is new_data_cache synced to master rec_cache.
2447
2448 The final assignment of the filedescriptor for rec_cache is done
2449 after the cache creation.
2450
2451 Don't check file size on new_data_cache, as the resulting file size
2452 is not known yet.
2453
2454 As rec_cache and new_data_cache are synced, write_buffer_length is
2455 used for the read cache 'new_data_cache'. Both start at the same
2456 position 'new_header_length'.
2457 }
2458 */
2459 DBUG_PRINT("info", ("is quick repair: %d", rep_quick));
2460 memset(&sort_info, 0, sizeof(sort_info));
2461 /* Initialize pthread structures before goto err. */
2462 mysql_mutex_init(mi_key_mutex_MI_SORT_INFO_mutex, &sort_info.mutex,
2463 MY_MUTEX_INIT_FAST);
2464 mysql_cond_init(mi_key_cond_MI_SORT_INFO_cond, &sort_info.cond);
2465 mysql_mutex_init(mi_key_mutex_MI_CHECK_print_msg, ¶m->print_msg_mutex,
2466 MY_MUTEX_INIT_FAST);
2467 param->need_print_msg_lock = true;
2468
2469 if (!(sort_info.key_block =
2470 alloc_key_blocks(param, (uint)param->sort_key_blocks,
2471 share->base.max_key_block_length)) ||
2472 init_io_cache(¶m->read_cache, info->dfile,
2473 (uint)param->read_buffer_length, READ_CACHE,
2474 share->pack.header_length, true, MYF(MY_WME)) ||
2475 (!rep_quick &&
2476 (init_io_cache(&info->rec_cache, info->dfile,
2477 (uint)param->write_buffer_length, WRITE_CACHE,
2478 new_header_length, true,
2479 MYF(MY_WME | MY_WAIT_IF_FULL) & param->myf_rw) ||
2480 init_io_cache(&new_data_cache, -1, (uint)param->write_buffer_length,
2481 READ_CACHE, new_header_length, true,
2482 MYF(MY_WME | MY_DONT_CHECK_FILESIZE)))))
2483 goto err;
2484 sort_info.key_block_end = sort_info.key_block + param->sort_key_blocks;
2485 info->opt_flag |= WRITE_CACHE_USED;
2486 info->rec_cache.file = info->dfile; /* for sort_delete_record */
2487
2488 if (!rep_quick) {
2489 /* Get real path for data file */
2490 if ((new_file = mysql_file_create(
2491 mi_key_file_datatmp,
2492 fn_format(param->temp_filename, share->data_file_name, "",
2493 DATA_TMP_EXT, 2 + 4),
2494 0, param->tmpfile_createflag, MYF(0))) < 0) {
2495 mi_check_print_error(param, "Can't create new tempfile: '%s'",
2496 param->temp_filename);
2497 goto err;
2498 }
2499 if (new_header_length && filecopy(param, new_file, info->dfile, 0L,
2500 new_header_length, "datafile-header"))
2501 goto err;
2502 if (param->testflag & T_UNPACK) {
2503 share->options &= ~HA_OPTION_COMPRESS_RECORD;
2504 mi_int2store(share->state.header.options, share->options);
2505 }
2506 share->state.dellink = HA_OFFSET_ERROR;
2507 info->rec_cache.file = new_file;
2508 }
2509
2510 info->update = (short)(HA_STATE_CHANGED | HA_STATE_ROW_CHANGED);
2511
2512 /* Optionally drop indexes and optionally modify the key_map. */
2513 mi_drop_all_indexes(param, info, false);
2514 key_map = share->state.key_map;
2515 if (param->testflag & T_CREATE_MISSING_KEYS) {
2516 /* Invert the copied key_map to recreate all disabled indexes. */
2517 key_map = ~key_map;
2518 }
2519
2520 sort_info.info = info;
2521 sort_info.param = param;
2522
2523 set_data_file_type(&sort_info, share);
2524 sort_info.dupp = 0;
2525 sort_info.buff = nullptr;
2526 param->read_cache.end_of_file = sort_info.filelength =
2527 mysql_file_seek(param->read_cache.file, 0L, MY_SEEK_END, MYF(0));
2528
2529 if (share->data_file_type == DYNAMIC_RECORD)
2530 rec_length =
2531 std::max(share->base.min_pack_length + 1, share->base.min_block_length);
2532 else if (share->data_file_type == COMPRESSED_RECORD)
2533 rec_length = share->base.min_block_length;
2534 else
2535 rec_length = share->base.pack_reclength;
2536 /*
2537 +1 below is required hack for parallel repair mode.
2538 The info->state->records value, that is compared later
2539 to sort_info.max_records and cannot exceed it, is
2540 increased in sort_key_write. In mi_repair_by_sort, sort_key_write
2541 is called after sort_key_read, where the comparison is performed,
2542 but in parallel mode master thread can call sort_key_write
2543 before some other repair thread calls sort_key_read.
2544 Furthermore I'm not even sure +1 would be enough.
2545 May be sort_info.max_records shold be always set to max value in
2546 parallel mode.
2547 */
2548 sort_info.max_records =
2549 ((param->testflag & T_CREATE_MISSING_KEYS)
2550 ? info->state->records + 1
2551 : (ha_rows)(sort_info.filelength / rec_length + 1));
2552
2553 del = info->state->del;
2554 param->glob_crc = 0;
2555 /* for compressed tables */
2556 max_pack_reclength = share->base.pack_reclength;
2557 if (share->options & HA_OPTION_COMPRESS_RECORD)
2558 max_pack_reclength = std::max(max_pack_reclength, share->max_pack_length);
2559 if (!(sort_param = (MI_SORT_PARAM *)my_malloc(
2560 mi_key_memory_MI_SORT_PARAM,
2561 (uint)share->base.keys *
2562 (sizeof(MI_SORT_PARAM) + max_pack_reclength),
2563 MYF(MY_ZEROFILL)))) {
2564 mi_check_print_error(param, "Not enough memory for key!");
2565 goto err;
2566 }
2567 total_key_length = 0;
2568 rec_per_key_part = param->rec_per_key_part;
2569 info->state->records = info->state->del = share->state.split = 0;
2570 info->state->empty = 0;
2571
2572 for (i = key = 0, istep = 1; key < share->base.keys;
2573 rec_per_key_part += sort_param[i].keyinfo->keysegs, i += istep, key++) {
2574 sort_param[i].key = key;
2575 sort_param[i].keyinfo = share->keyinfo + key;
2576 sort_param[i].seg = sort_param[i].keyinfo->seg;
2577 /*
2578 Skip this index if it is marked disabled in the copied
2579 (and possibly inverted) key_map.
2580 */
2581 if (!mi_is_key_active(key_map, key)) {
2582 /* Remember old statistics for key */
2583 memcpy((char *)rec_per_key_part,
2584 (char *)(share->state.rec_per_key_part +
2585 (uint)(rec_per_key_part - param->rec_per_key_part)),
2586 sort_param[i].keyinfo->keysegs * sizeof(*rec_per_key_part));
2587 istep = 0;
2588 continue;
2589 }
2590 istep = 1;
2591 if ((!(param->testflag & T_SILENT))) printf("- Fixing index %d\n", key + 1);
2592 if (sort_param[i].keyinfo->flag & HA_FULLTEXT) {
2593 sort_param[i].key_read = sort_ft_key_read;
2594 sort_param[i].key_write = sort_ft_key_write;
2595 } else {
2596 sort_param[i].key_read = sort_key_read;
2597 sort_param[i].key_write = sort_key_write;
2598 }
2599 sort_param[i].key_cmp = sort_key_cmp;
2600 sort_param[i].tmpdir = param->tmpdir;
2601 sort_param[i].sort_info = &sort_info;
2602 sort_param[i].master = false;
2603 sort_param[i].fix_datafile = false;
2604 sort_param[i].calc_checksum = false;
2605
2606 sort_param[i].filepos = new_header_length;
2607 sort_param[i].max_pos = sort_param[i].pos = share->pack.header_length;
2608
2609 sort_param[i].record =
2610 (((uchar *)(sort_param + share->base.keys)) + (max_pack_reclength * i));
2611 if (!mi_alloc_rec_buff(info, -1, &sort_param[i].rec_buff)) {
2612 mi_check_print_error(param, "Not enough memory!");
2613 goto err;
2614 }
2615
2616 sort_param[i].key_length = share->rec_reflength;
2617 for (keyseg = sort_param[i].seg; keyseg->type != HA_KEYTYPE_END; keyseg++) {
2618 sort_param[i].key_length += keyseg->length;
2619 if (keyseg->flag & HA_SPACE_PACK)
2620 sort_param[i].key_length += get_pack_length(keyseg->length);
2621 if (keyseg->flag & (HA_BLOB_PART | HA_VAR_LENGTH_PART))
2622 sort_param[i].key_length += 2 + (keyseg->length >= 127);
2623 if (keyseg->flag & HA_NULL_PART) sort_param[i].key_length++;
2624 }
2625 total_key_length += sort_param[i].key_length;
2626
2627 if (sort_param[i].keyinfo->flag & HA_FULLTEXT) {
2628 uint ft_max_word_len_for_sort =
2629 FT_MAX_WORD_LEN_FOR_SORT *
2630 sort_param[i].keyinfo->seg->charset->mbmaxlen;
2631 sort_param[i].key_length += ft_max_word_len_for_sort - HA_FT_MAXBYTELEN;
2632 init_alloc_root(mi_key_memory_MI_SORT_PARAM_wordroot,
2633 &sort_param[i].wordroot, FTPARSER_MEMROOT_ALLOC_SIZE, 0);
2634 }
2635 }
2636 sort_info.total_keys = i;
2637 sort_param[0].master = true;
2638 sort_param[0].fix_datafile = (bool)(!rep_quick);
2639 sort_param[0].calc_checksum = (param->testflag & T_CALC_CHECKSUM);
2640
2641 if (!ftparser_alloc_param(info)) goto err;
2642
2643 sort_info.got_error = 0;
2644 mysql_mutex_lock(&sort_info.mutex);
2645
2646 /*
2647 Initialize the I/O cache share for use with the read caches and, in
2648 case of non-quick repair, the write cache. When all threads join on
2649 the cache lock, the writer copies the write cache contents to the
2650 read caches.
2651 */
2652 if (i > 1) {
2653 if (rep_quick)
2654 init_io_cache_share(¶m->read_cache, &io_share, nullptr, i);
2655 else
2656 init_io_cache_share(&new_data_cache, &io_share, &info->rec_cache, i);
2657 } else
2658 io_share.total_threads = 0; /* share not used */
2659
2660 (void)my_thread_attr_init(&thr_attr);
2661 (void)my_thread_attr_setdetachstate(&thr_attr, MY_THREAD_CREATE_DETACHED);
2662
2663 for (i = 0; i < sort_info.total_keys; i++) {
2664 /*
2665 Copy the properly initialized IO_CACHE structure so that every
2666 thread has its own copy. In quick mode param->read_cache is shared
2667 for use by all threads. In non-quick mode all threads but the
2668 first copy the shared new_data_cache, which is synchronized to the
2669 write cache of the first thread. The first thread copies
2670 param->read_cache, which is not shared.
2671 */
2672 sort_param[i].read_cache =
2673 ((rep_quick || !i) ? param->read_cache : new_data_cache);
2674 DBUG_PRINT("io_cache_share",
2675 ("thread: %u read_cache: %p", i, &sort_param[i].read_cache));
2676
2677 sort_param[i].sortbuff_size =
2678 param->sort_buffer_length / sort_info.total_keys;
2679 if ((error = mysql_thread_create(
2680 mi_key_thread_find_all_keys, &sort_param[i].thr, &thr_attr,
2681 thr_find_all_keys, (void *)(sort_param + i)))) {
2682 mi_check_print_error(param, "Cannot start a repair thread (errno= %d)",
2683 error);
2684 /* Cleanup: Detach from the share. Avoid others to be blocked. */
2685 if (io_share.total_threads) remove_io_thread(&sort_param[i].read_cache);
2686 DBUG_PRINT("error", ("Cannot start a repair thread"));
2687 sort_info.got_error = 1;
2688 } else
2689 sort_info.threads_running++;
2690 }
2691 (void)my_thread_attr_destroy(&thr_attr);
2692
2693 /* waiting for all threads to finish */
2694 while (sort_info.threads_running)
2695 mysql_cond_wait(&sort_info.cond, &sort_info.mutex);
2696 mysql_mutex_unlock(&sort_info.mutex);
2697
2698 if ((got_error = thr_write_keys(sort_param))) {
2699 param->retry_repair = true;
2700 goto err;
2701 }
2702 got_error = 1; /* Assume the following may go wrong */
2703
2704 if (sort_param[0].fix_datafile) {
2705 /*
2706 Append some nuls to the end of a memory mapped file. Destroy the
2707 write cache. The master thread did already detach from the share
2708 by remove_io_thread() in sort.c:thr_find_all_keys().
2709 */
2710 if (write_data_suffix(&sort_info, true) || end_io_cache(&info->rec_cache))
2711 goto err;
2712 if (param->testflag & T_SAFE_REPAIR) {
2713 /* Don't repair if we loosed more than one row */
2714 if (info->state->records + 1 < start_records) {
2715 info->state->records = start_records;
2716 goto err;
2717 }
2718 }
2719 share->state.state.data_file_length = info->state->data_file_length =
2720 sort_param->filepos;
2721 /* Only whole records */
2722 share->state.version = (ulong)time((time_t *)nullptr);
2723
2724 /*
2725 Exchange the data file descriptor of the table, so that we use the
2726 new file from now on.
2727 */
2728 mysql_file_close(info->dfile, MYF(0));
2729 info->dfile = new_file;
2730
2731 share->data_file_type = sort_info.new_data_file_type;
2732 share->pack.header_length = (ulong)new_header_length;
2733 } else
2734 info->state->data_file_length = sort_param->max_pos;
2735
2736 if (rep_quick && del + sort_info.dupp != info->state->del) {
2737 mi_check_print_error(param,
2738 "Couldn't fix table with quick recovery: Found wrong "
2739 "number of deleted records");
2740 mi_check_print_error(param, "Run recovery again without -q");
2741 param->retry_repair = true;
2742 param->testflag |= T_RETRY_WITHOUT_QUICK;
2743 goto err;
2744 }
2745
2746 if (rep_quick & T_FORCE_UNIQUENESS) {
2747 my_off_t skr =
2748 info->state->data_file_length +
2749 (share->options & HA_OPTION_COMPRESS_RECORD ? MEMMAP_EXTRA_MARGIN : 0);
2750 if (skr != sort_info.filelength)
2751 if (mysql_file_chsize(info->dfile, skr, 0, MYF(0)))
2752 mi_check_print_warning(
2753 param, "Can't change size of datafile, error: %d", my_errno());
2754 }
2755 if (param->testflag & T_CALC_CHECKSUM)
2756 info->state->checksum = param->glob_crc;
2757
2758 if (mysql_file_chsize(share->kfile, info->state->key_file_length, 0, MYF(0)))
2759 mi_check_print_warning(param, "Can't change size of indexfile, error: %d",
2760 my_errno());
2761
2762 if (!(param->testflag & T_SILENT)) {
2763 if (start_records != info->state->records)
2764 printf("Data records: %s\n", llstr(info->state->records, llbuff));
2765 if (sort_info.dupp)
2766 mi_check_print_warning(param, "%s records have been removed",
2767 llstr(sort_info.dupp, llbuff));
2768 }
2769 got_error = 0;
2770
2771 if (&share->state.state != info->state)
2772 memcpy(&share->state.state, info->state, sizeof(*info->state));
2773
2774 err:
2775 got_error |= flush_blocks(param, share->key_cache, share->kfile);
2776 /*
2777 Destroy the write cache. The master thread did already detach from
2778 the share by remove_io_thread() or it was not yet started (if the
2779 error happend before creating the thread).
2780 */
2781 (void)end_io_cache(&info->rec_cache);
2782 /*
2783 Destroy the new data cache in case of non-quick repair. All slave
2784 threads did either detach from the share by remove_io_thread()
2785 already or they were not yet started (if the error happend before
2786 creating the threads).
2787 */
2788 if (!rep_quick) (void)end_io_cache(&new_data_cache);
2789 if (!got_error) {
2790 /* Replace the actual file with the temporary file */
2791 if (new_file >= 0) {
2792 myf flags = 0;
2793 if (param->testflag & T_BACKUP_DATA) flags |= MY_REDEL_MAKE_BACKUP;
2794 if (no_copy_stat) flags |= MY_REDEL_NO_COPY_STAT;
2795 mysql_file_close(new_file, MYF(0));
2796 info->dfile = new_file = -1;
2797 if (change_to_newfile(share->data_file_name, MI_NAME_DEXT, DATA_TMP_EXT,
2798 flags) ||
2799 mi_open_datafile(info, share, name, -1))
2800 got_error = 1;
2801 }
2802 }
2803 if (got_error) {
2804 if (!param->error_printed)
2805 mi_check_print_error(param, "%d when fixing table", my_errno());
2806 if (new_file >= 0) {
2807 (void)mysql_file_close(new_file, MYF(0));
2808 (void)mysql_file_delete(mi_key_file_datatmp, param->temp_filename,
2809 MYF(MY_WME));
2810 if (info->dfile == new_file) /* Retry with key cache */
2811 if (unlikely(mi_open_datafile(info, share, name, -1)))
2812 param->retry_repair = false; /* Safety */
2813 }
2814 mi_mark_crashed_on_repair(info);
2815 } else if (key_map == share->state.key_map)
2816 share->state.changed &= ~STATE_NOT_OPTIMIZED_KEYS;
2817 share->state.changed |= STATE_NOT_SORTED_PAGES;
2818
2819 mysql_cond_destroy(&sort_info.cond);
2820 mysql_mutex_destroy(&sort_info.mutex);
2821 mysql_mutex_destroy(¶m->print_msg_mutex);
2822 param->need_print_msg_lock = false;
2823
2824 my_free(sort_info.ft_buf);
2825 my_free(sort_info.key_block);
2826 my_free(sort_param);
2827 my_free(sort_info.buff);
2828 (void)end_io_cache(¶m->read_cache);
2829 info->opt_flag &= ~(READ_CACHE_USED | WRITE_CACHE_USED);
2830 if (!got_error && (param->testflag & T_UNPACK)) {
2831 share->state.header.options[0] &= (uchar)~HA_OPTION_COMPRESS_RECORD;
2832 share->pack.header_length = 0;
2833 }
2834 return got_error;
2835 }
2836
2837 /* Read next record and return next key */
2838
sort_key_read(MI_SORT_PARAM * sort_param,void * key)2839 static int sort_key_read(MI_SORT_PARAM *sort_param, void *key) {
2840 int error;
2841 SORT_INFO *sort_info = sort_param->sort_info;
2842 MI_INFO *info = sort_info->info;
2843 DBUG_TRACE;
2844
2845 if ((error = sort_get_next_record(sort_param))) return error;
2846 if (info->state->records == sort_info->max_records) {
2847 mi_check_print_error(sort_info->param,
2848 "Key %d - Found too many records; Can't continue",
2849 sort_param->key + 1);
2850 return 1;
2851 }
2852 sort_param->real_key_length =
2853 (info->s->rec_reflength + _mi_make_key(info, sort_param->key,
2854 (uchar *)key, sort_param->record,
2855 sort_param->filepos));
2856 return sort_write_record(sort_param);
2857 } /* sort_key_read */
2858
sort_ft_key_read(MI_SORT_PARAM * sort_param,void * key)2859 static int sort_ft_key_read(MI_SORT_PARAM *sort_param, void *key) {
2860 int error;
2861 SORT_INFO *sort_info = sort_param->sort_info;
2862 MI_INFO *info = sort_info->info;
2863 FT_WORD *wptr = nullptr;
2864 DBUG_TRACE;
2865
2866 if (!sort_param->wordlist) {
2867 for (;;) {
2868 free_root(&sort_param->wordroot, MYF(MY_MARK_BLOCKS_FREE));
2869 if ((error = sort_get_next_record(sort_param))) return error;
2870 if (!(wptr = _mi_ft_parserecord(info, sort_param->key, sort_param->record,
2871 &sort_param->wordroot)))
2872 return 1;
2873 if (wptr->pos) break;
2874 error = sort_write_record(sort_param);
2875 }
2876 sort_param->wordptr = sort_param->wordlist = wptr;
2877 } else {
2878 error = 0;
2879 wptr = (FT_WORD *)(sort_param->wordptr);
2880 }
2881
2882 sort_param->real_key_length =
2883 (info->s->rec_reflength + _ft_make_key(info, sort_param->key,
2884 (uchar *)key, wptr++,
2885 sort_param->filepos));
2886 if (!wptr->pos) {
2887 free_root(&sort_param->wordroot, MYF(MY_MARK_BLOCKS_FREE));
2888 sort_param->wordlist = nullptr;
2889 error = sort_write_record(sort_param);
2890 } else
2891 sort_param->wordptr = (void *)wptr;
2892
2893 return error;
2894 } /* sort_ft_key_read */
2895
2896 /*
2897 Read next record from file using parameters in sort_info.
2898
2899 SYNOPSIS
2900 sort_get_next_record()
2901 sort_param Information about and for the sort process
2902
2903 NOTE
2904
2905 Dynamic Records With Non-Quick Parallel Repair
2906
2907 For non-quick parallel repair we use a synchronized read/write
2908 cache. This means that one thread is the master who fixes the data
2909 file by reading each record from the old data file and writing it
2910 to the new data file. By doing this the records in the new data
2911 file are written contiguously. Whenever the write buffer is full,
2912 it is copied to the read buffer. The slaves read from the read
2913 buffer, which is not associated with a file. Thus read_cache.file
2914 is -1. When using _mi_read_cache(), the slaves must always set
2915 flag to READING_NEXT so that the function never tries to read from
2916 file. This is safe because the records are contiguous. There is no
2917 need to read outside the cache. This condition is evaluated in the
2918 variable 'parallel_flag' for quick reference. read_cache.file must
2919 be >= 0 in every other case.
2920
2921 RETURN
2922 -1 end of file
2923 0 ok
2924 > 0 error
2925 */
2926
sort_get_next_record(MI_SORT_PARAM * sort_param)2927 static int sort_get_next_record(MI_SORT_PARAM *sort_param) {
2928 int searching;
2929 int parallel_flag;
2930 uint found_record, b_type, left_length;
2931 my_off_t pos;
2932 uchar *to = nullptr;
2933 MI_BLOCK_INFO block_info;
2934 SORT_INFO *sort_info = sort_param->sort_info;
2935 MI_CHECK *param = sort_info->param;
2936 MI_INFO *info = sort_info->info;
2937 MYISAM_SHARE *share = info->s;
2938 char llbuff[22], llbuff2[22];
2939 DBUG_TRACE;
2940
2941 if (*killed_ptr(param)) return 1;
2942
2943 switch (share->data_file_type) {
2944 case STATIC_RECORD:
2945 for (;;) {
2946 if (my_b_read(&sort_param->read_cache, sort_param->record,
2947 share->base.pack_reclength)) {
2948 if (sort_param->read_cache.error) param->out_flag |= O_DATA_LOST;
2949 param->retry_repair = true;
2950 param->testflag |= T_RETRY_WITHOUT_QUICK;
2951 return -1;
2952 }
2953 sort_param->start_recpos = sort_param->pos;
2954 if (!sort_param->fix_datafile) {
2955 sort_param->filepos = sort_param->pos;
2956 if (sort_param->master) share->state.split++;
2957 }
2958 sort_param->max_pos = (sort_param->pos += share->base.pack_reclength);
2959 if (*sort_param->record) {
2960 if (sort_param->calc_checksum)
2961 param->glob_crc +=
2962 (info->checksum = mi_static_checksum(info, sort_param->record));
2963 return 0;
2964 }
2965 if (!sort_param->fix_datafile && sort_param->master) {
2966 info->state->del++;
2967 info->state->empty += share->base.pack_reclength;
2968 }
2969 }
2970 case DYNAMIC_RECORD:
2971 to = nullptr;
2972 pos = sort_param->pos;
2973 searching = (sort_param->fix_datafile && (param->testflag & T_EXTEND));
2974 parallel_flag = (sort_param->read_cache.file < 0) ? READING_NEXT : 0;
2975 for (;;) {
2976 found_record = block_info.second_read = 0;
2977 left_length = 1;
2978 if (searching) {
2979 pos = MY_ALIGN(pos, MI_DYN_ALIGN_SIZE);
2980 param->testflag |= T_RETRY_WITHOUT_QUICK;
2981 sort_param->start_recpos = pos;
2982 }
2983 do {
2984 if (pos > sort_param->max_pos) sort_param->max_pos = pos;
2985 if (pos & (MI_DYN_ALIGN_SIZE - 1)) {
2986 if ((param->testflag & T_VERBOSE) || searching == 0)
2987 mi_check_print_info(param, "Wrong aligned block at %s",
2988 llstr(pos, llbuff));
2989 if (searching) goto try_next;
2990 }
2991 if (found_record && pos == param->search_after_block)
2992 mi_check_print_info(param, "Block: %s used by record at %s",
2993 llstr(param->search_after_block, llbuff),
2994 llstr(sort_param->start_recpos, llbuff2));
2995 if (_mi_read_cache(&sort_param->read_cache,
2996 (uchar *)block_info.header, pos,
2997 MI_BLOCK_INFO_HEADER_LENGTH,
2998 (!found_record ? READING_NEXT : 0) |
2999 parallel_flag | READING_HEADER)) {
3000 if (found_record) {
3001 mi_check_print_info(
3002 param, "Can't read whole record at %s (errno: %d)",
3003 llstr(sort_param->start_recpos, llbuff), errno);
3004 goto try_next;
3005 }
3006 return -1;
3007 }
3008 if (searching && !sort_param->fix_datafile) {
3009 param->error_printed = 1;
3010 param->retry_repair = true;
3011 param->testflag |= T_RETRY_WITHOUT_QUICK;
3012 return 1; /* Something wrong with data */
3013 }
3014 b_type = _mi_get_block_info(&block_info, -1, pos);
3015 if ((b_type & (BLOCK_ERROR | BLOCK_FATAL_ERROR)) ||
3016 ((b_type & BLOCK_FIRST) &&
3017 (block_info.rec_len < (uint)share->base.min_pack_length ||
3018 block_info.rec_len > (uint)share->base.max_pack_length))) {
3019 uint i;
3020 if (param->testflag & T_VERBOSE || searching == 0)
3021 mi_check_print_info(param,
3022 "Wrong bytesec: %3d-%3d-%3d at %10s; Skipped",
3023 block_info.header[0], block_info.header[1],
3024 block_info.header[2], llstr(pos, llbuff));
3025 if (found_record) goto try_next;
3026 block_info.second_read = 0;
3027 searching = 1;
3028 /* Search after block in read header string */
3029 for (i = MI_DYN_ALIGN_SIZE; i < MI_BLOCK_INFO_HEADER_LENGTH;
3030 i += MI_DYN_ALIGN_SIZE)
3031 if (block_info.header[i] >= 1 &&
3032 block_info.header[i] <= MI_MAX_DYN_HEADER_BYTE)
3033 break;
3034 pos += (ulong)i;
3035 sort_param->start_recpos = pos;
3036 continue;
3037 }
3038 if (b_type & BLOCK_DELETED) {
3039 bool error = false;
3040 if (block_info.block_len + (uint)(block_info.filepos - pos) <
3041 share->base.min_block_length) {
3042 if (!searching)
3043 mi_check_print_info(
3044 param, "Deleted block with impossible length %lu at %s",
3045 block_info.block_len, llstr(pos, llbuff));
3046 error = true;
3047 } else {
3048 if ((block_info.next_filepos != HA_OFFSET_ERROR &&
3049 block_info.next_filepos >= info->state->data_file_length) ||
3050 (block_info.prev_filepos != HA_OFFSET_ERROR &&
3051 block_info.prev_filepos >= info->state->data_file_length)) {
3052 if (!searching)
3053 mi_check_print_info(
3054 param, "Delete link points outside datafile at %s",
3055 llstr(pos, llbuff));
3056 error = true;
3057 }
3058 }
3059 if (error) {
3060 if (found_record) goto try_next;
3061 searching = 1;
3062 pos += MI_DYN_ALIGN_SIZE;
3063 sort_param->start_recpos = pos;
3064 block_info.second_read = 0;
3065 continue;
3066 }
3067 } else {
3068 if (block_info.block_len + (uint)(block_info.filepos - pos) <
3069 share->base.min_block_length ||
3070 block_info.block_len >
3071 (uint)share->base.max_pack_length + MI_SPLIT_LENGTH) {
3072 if (!searching)
3073 mi_check_print_info(
3074 param,
3075 "Found block with impossible length %lu at %s; Skipped",
3076 block_info.block_len + (uint)(block_info.filepos - pos),
3077 llstr(pos, llbuff));
3078 if (found_record) goto try_next;
3079 searching = 1;
3080 pos += MI_DYN_ALIGN_SIZE;
3081 sort_param->start_recpos = pos;
3082 block_info.second_read = 0;
3083 continue;
3084 }
3085 }
3086 if (b_type & (BLOCK_DELETED | BLOCK_SYNC_ERROR)) {
3087 if (!sort_param->fix_datafile && sort_param->master &&
3088 (b_type & BLOCK_DELETED)) {
3089 info->state->empty += block_info.block_len;
3090 info->state->del++;
3091 share->state.split++;
3092 }
3093 if (found_record) goto try_next;
3094 if (searching) {
3095 pos += MI_DYN_ALIGN_SIZE;
3096 sort_param->start_recpos = pos;
3097 } else
3098 pos = block_info.filepos + block_info.block_len;
3099 block_info.second_read = 0;
3100 continue;
3101 }
3102
3103 if (!sort_param->fix_datafile && sort_param->master)
3104 share->state.split++;
3105 if (!found_record++) {
3106 sort_param->find_length = left_length = block_info.rec_len;
3107 sort_param->start_recpos = pos;
3108 if (!sort_param->fix_datafile)
3109 sort_param->filepos = sort_param->start_recpos;
3110 if (sort_param->fix_datafile && (param->testflag & T_EXTEND))
3111 sort_param->pos = block_info.filepos + 1;
3112 else
3113 sort_param->pos = block_info.filepos + block_info.block_len;
3114 if (share->base.blobs) {
3115 if (!(to = mi_alloc_rec_buff(info, block_info.rec_len,
3116 &(sort_param->rec_buff)))) {
3117 if (param->max_record_length >= block_info.rec_len) {
3118 mi_check_print_error(
3119 param, "Not enough memory for blob at %s (need %lu)",
3120 llstr(sort_param->start_recpos, llbuff),
3121 (ulong)block_info.rec_len);
3122 return 1;
3123 } else {
3124 mi_check_print_info(param,
3125 "Not enough memory for blob at %s (need "
3126 "%lu); Row skipped",
3127 llstr(sort_param->start_recpos, llbuff),
3128 (ulong)block_info.rec_len);
3129 goto try_next;
3130 }
3131 }
3132 } else
3133 to = sort_param->rec_buff;
3134 }
3135 if (left_length < block_info.data_len || !block_info.data_len) {
3136 mi_check_print_info(
3137 param, "Found block with too small length at %s; Skipped",
3138 llstr(sort_param->start_recpos, llbuff));
3139 goto try_next;
3140 }
3141 if (block_info.filepos + block_info.data_len >
3142 sort_param->read_cache.end_of_file) {
3143 mi_check_print_info(
3144 param, "Found block that points outside data file at %s",
3145 llstr(sort_param->start_recpos, llbuff));
3146 goto try_next;
3147 }
3148 /*
3149 Copy information that is already read. Avoid accessing data
3150 below the cache start. This could happen if the header
3151 streched over the end of the previous buffer contents.
3152 */
3153 {
3154 uint header_len = (uint)(block_info.filepos - pos);
3155 uint prefetch_len = (MI_BLOCK_INFO_HEADER_LENGTH - header_len);
3156
3157 if (prefetch_len > block_info.data_len)
3158 prefetch_len = block_info.data_len;
3159 if (prefetch_len) {
3160 memcpy(to, block_info.header + header_len, prefetch_len);
3161 block_info.filepos += prefetch_len;
3162 block_info.data_len -= prefetch_len;
3163 left_length -= prefetch_len;
3164 to += prefetch_len;
3165 }
3166 }
3167 if (block_info.data_len &&
3168 _mi_read_cache(
3169 &sort_param->read_cache, to, block_info.filepos,
3170 block_info.data_len,
3171 (found_record == 1 ? READING_NEXT : 0) | parallel_flag)) {
3172 mi_check_print_info(
3173 param, "Read error for block at: %s (error: %d); Skipped",
3174 llstr(block_info.filepos, llbuff), my_errno());
3175 goto try_next;
3176 }
3177 left_length -= block_info.data_len;
3178 to += block_info.data_len;
3179 pos = block_info.next_filepos;
3180 if (pos == HA_OFFSET_ERROR && left_length) {
3181 mi_check_print_info(
3182 param, "Wrong block with wrong total length starting at %s",
3183 llstr(sort_param->start_recpos, llbuff));
3184 goto try_next;
3185 }
3186 if (pos + MI_BLOCK_INFO_HEADER_LENGTH >
3187 sort_param->read_cache.end_of_file) {
3188 mi_check_print_info(
3189 param, "Found link that points at %s (outside data file) at %s",
3190 llstr(pos, llbuff2), llstr(sort_param->start_recpos, llbuff));
3191 goto try_next;
3192 }
3193 } while (left_length);
3194
3195 if (_mi_rec_unpack(info, sort_param->record, sort_param->rec_buff,
3196 sort_param->find_length) != MY_FILE_ERROR) {
3197 if (sort_param->read_cache.error < 0) return 1;
3198 if (sort_param->calc_checksum)
3199 info->checksum = mi_checksum(info, sort_param->record);
3200 if ((param->testflag & (T_EXTEND | T_REP)) || searching) {
3201 if (_mi_rec_check(info, sort_param->record, sort_param->rec_buff,
3202 sort_param->find_length,
3203 (param->testflag & T_QUICK) &&
3204 sort_param->calc_checksum &&
3205 info->s->calc_checksum)) {
3206 mi_check_print_info(param, "Found wrong packed record at %s",
3207 llstr(sort_param->start_recpos, llbuff));
3208 goto try_next;
3209 }
3210 }
3211 if (sort_param->calc_checksum) param->glob_crc += info->checksum;
3212 return 0;
3213 }
3214 if (!searching)
3215 mi_check_print_info(param, "Key %d - Found wrong stored record at %s",
3216 sort_param->key + 1,
3217 llstr(sort_param->start_recpos, llbuff));
3218 try_next:
3219 pos = (sort_param->start_recpos += MI_DYN_ALIGN_SIZE);
3220 searching = 1;
3221 }
3222 case COMPRESSED_RECORD:
3223 for (searching = 0;; searching = 1, sort_param->pos++) {
3224 if (_mi_read_cache(&sort_param->read_cache, (uchar *)block_info.header,
3225 sort_param->pos, share->pack.ref_length,
3226 READING_NEXT))
3227 return -1;
3228 if (searching && !sort_param->fix_datafile) {
3229 param->error_printed = 1;
3230 param->retry_repair = true;
3231 param->testflag |= T_RETRY_WITHOUT_QUICK;
3232 return 1; /* Something wrong with data */
3233 }
3234 sort_param->start_recpos = sort_param->pos;
3235 if (_mi_pack_get_block_info(info, &sort_param->bit_buff, &block_info,
3236 &sort_param->rec_buff, -1, sort_param->pos))
3237 return -1;
3238 if (!block_info.rec_len && sort_param->pos + MEMMAP_EXTRA_MARGIN ==
3239 sort_param->read_cache.end_of_file)
3240 return -1;
3241 if (block_info.rec_len < (uint)share->min_pack_length ||
3242 block_info.rec_len > (uint)share->max_pack_length) {
3243 if (!searching)
3244 mi_check_print_info(
3245 param, "Found block with wrong recordlength: %ld at %s\n",
3246 block_info.rec_len, llstr(sort_param->pos, llbuff));
3247 continue;
3248 }
3249 if (_mi_read_cache(&sort_param->read_cache,
3250 (uchar *)sort_param->rec_buff, block_info.filepos,
3251 block_info.rec_len, READING_NEXT)) {
3252 if (!searching)
3253 mi_check_print_info(param, "Couldn't read whole record from %s",
3254 llstr(sort_param->pos, llbuff));
3255 continue;
3256 }
3257 if (_mi_pack_rec_unpack(info, &sort_param->bit_buff, sort_param->record,
3258 sort_param->rec_buff, block_info.rec_len)) {
3259 if (!searching)
3260 mi_check_print_info(param, "Found wrong record at %s",
3261 llstr(sort_param->pos, llbuff));
3262 continue;
3263 }
3264 if (!sort_param->fix_datafile) {
3265 sort_param->filepos = sort_param->pos;
3266 if (sort_param->master) share->state.split++;
3267 }
3268 sort_param->max_pos =
3269 (sort_param->pos = block_info.filepos + block_info.rec_len);
3270 info->packed_length = block_info.rec_len;
3271 if (sort_param->calc_checksum)
3272 param->glob_crc +=
3273 (info->checksum = mi_checksum(info, sort_param->record));
3274 return 0;
3275 }
3276 case BLOCK_RECORD:
3277 assert(0); /* Impossible */
3278 }
3279 return 1; /* Impossible */
3280 }
3281
3282 /*
3283 Write record to new file.
3284
3285 SYNOPSIS
3286 sort_write_record()
3287 sort_param Sort parameters.
3288
3289 NOTE
3290 This is only called by a master thread if parallel repair is used.
3291
3292 RETURN
3293 0 OK
3294 1 Error
3295 */
3296
sort_write_record(MI_SORT_PARAM * sort_param)3297 int sort_write_record(MI_SORT_PARAM *sort_param) {
3298 int flag;
3299 uint length;
3300 ulong block_length, reclength;
3301 uchar *from;
3302 uchar block_buff[8];
3303 SORT_INFO *sort_info = sort_param->sort_info;
3304 MI_CHECK *param = sort_info->param;
3305 MI_INFO *info = sort_info->info;
3306 MYISAM_SHARE *share = info->s;
3307 DBUG_TRACE;
3308
3309 if (sort_param->fix_datafile) {
3310 switch (sort_info->new_data_file_type) {
3311 case STATIC_RECORD:
3312 if (my_b_write(&info->rec_cache, sort_param->record,
3313 share->base.pack_reclength)) {
3314 mi_check_print_error(param, "%d when writing to datafile",
3315 my_errno());
3316 return 1;
3317 }
3318 sort_param->filepos += share->base.pack_reclength;
3319 info->s->state.split++;
3320 /* sort_info->param->glob_crc+=mi_static_checksum(info,
3321 * sort_param->record); */
3322 break;
3323 case DYNAMIC_RECORD:
3324 if (!info->blobs)
3325 from = sort_param->rec_buff;
3326 else {
3327 /* must be sure that local buffer is big enough */
3328 reclength = info->s->base.pack_reclength +
3329 _my_calc_total_blob_length(info, sort_param->record) +
3330 ALIGN_SIZE(MI_MAX_DYN_BLOCK_HEADER) + MI_SPLIT_LENGTH +
3331 MI_DYN_DELETE_BLOCK_HEADER;
3332 if (sort_info->buff_length < reclength) {
3333 if (!(sort_info->buff = (uchar *)my_realloc(
3334 mi_key_memory_SORT_INFO_buffer, sort_info->buff,
3335 (uint)reclength,
3336 MYF(MY_FREE_ON_ERROR | MY_ALLOW_ZERO_PTR))))
3337 return 1;
3338 sort_info->buff_length = reclength;
3339 }
3340 from = sort_info->buff + ALIGN_SIZE(MI_MAX_DYN_BLOCK_HEADER);
3341 }
3342 /* We can use info->checksum here as only one thread calls this. */
3343 info->checksum = mi_checksum(info, sort_param->record);
3344 reclength = _mi_rec_pack(info, from, sort_param->record);
3345 flag = 0;
3346 /* sort_info->param->glob_crc+=info->checksum; */
3347
3348 do {
3349 block_length = reclength + 3 + reclength >= (65520 - 3);
3350 if (block_length < share->base.min_block_length)
3351 block_length = share->base.min_block_length;
3352 info->update |= HA_STATE_WRITE_AT_END;
3353 block_length = MY_ALIGN(block_length, MI_DYN_ALIGN_SIZE);
3354 if (block_length > MI_MAX_BLOCK_LENGTH)
3355 block_length = MI_MAX_BLOCK_LENGTH;
3356 if (_mi_write_part_record(info, 0L, block_length,
3357 sort_param->filepos + block_length, &from,
3358 &reclength, &flag)) {
3359 mi_check_print_error(param, "%d when writing to datafile",
3360 my_errno());
3361 return 1;
3362 }
3363 sort_param->filepos += block_length;
3364 info->s->state.split++;
3365 } while (reclength);
3366 /* sort_info->param->glob_crc+=info->checksum; */
3367 break;
3368 case COMPRESSED_RECORD:
3369 reclength = info->packed_length;
3370 length =
3371 save_pack_length((uint)share->pack.version, block_buff, reclength);
3372 if (info->s->base.blobs)
3373 length += save_pack_length((uint)share->pack.version,
3374 block_buff + length, info->blob_length);
3375 if (my_b_write(&info->rec_cache, block_buff, length) ||
3376 my_b_write(&info->rec_cache, (uchar *)sort_param->rec_buff,
3377 reclength)) {
3378 mi_check_print_error(param, "%d when writing to datafile",
3379 my_errno());
3380 return 1;
3381 }
3382 /* sort_info->param->glob_crc+=info->checksum; */
3383 sort_param->filepos += reclength + length;
3384 info->s->state.split++;
3385 break;
3386 case BLOCK_RECORD:
3387 assert(0); /* Impossible */
3388 }
3389 }
3390 if (sort_param->master) {
3391 info->state->records++;
3392 if ((param->testflag & T_WRITE_LOOP) &&
3393 (info->state->records % WRITE_COUNT) == 0) {
3394 char llbuff[22];
3395 printf("%s\r", llstr(info->state->records, llbuff));
3396 (void)fflush(stdout);
3397 }
3398 }
3399 return 0;
3400 } /* sort_write_record */
3401
3402 /* Compare two keys from _create_index_by_sort */
3403
sort_key_cmp(void * cmp_arg,uchar * u_a,uchar * u_b)3404 static int sort_key_cmp(void *cmp_arg, uchar *u_a, uchar *u_b) {
3405 MI_SORT_PARAM *sort_param = (MI_SORT_PARAM *)cmp_arg;
3406 void *a = u_a;
3407 void *b = u_b;
3408 uint not_used[2];
3409 return (ha_key_cmp(sort_param->seg, *((uchar **)a), *((uchar **)b),
3410 USE_WHOLE_KEY, SEARCH_SAME, not_used));
3411 } /* sort_key_cmp */
3412
sort_key_write(MI_SORT_PARAM * sort_param,const void * a)3413 static int sort_key_write(MI_SORT_PARAM *sort_param, const void *a) {
3414 uint diff_pos[2];
3415 char llbuff[22], llbuff2[22];
3416 SORT_INFO *sort_info = sort_param->sort_info;
3417 MI_CHECK *param = sort_info->param;
3418 int cmp;
3419
3420 if (sort_info->key_block->inited) {
3421 cmp = ha_key_cmp(sort_param->seg, sort_info->key_block->lastkey,
3422 static_cast<const uchar *>(a), USE_WHOLE_KEY,
3423 SEARCH_FIND | SEARCH_UPDATE, diff_pos);
3424 if (param->stats_method == MI_STATS_METHOD_NULLS_NOT_EQUAL)
3425 ha_key_cmp(sort_param->seg, sort_info->key_block->lastkey,
3426 static_cast<const uchar *>(a), USE_WHOLE_KEY,
3427 SEARCH_FIND | SEARCH_NULL_ARE_NOT_EQUAL, diff_pos);
3428 else if (param->stats_method == MI_STATS_METHOD_IGNORE_NULLS) {
3429 diff_pos[0] = mi_collect_stats_nonulls_next(
3430 sort_param->seg, sort_param->notnull, sort_info->key_block->lastkey,
3431 static_cast<const uchar *>(a));
3432 }
3433 sort_param->unique[diff_pos[0] - 1]++;
3434 } else {
3435 cmp = -1;
3436 if (param->stats_method == MI_STATS_METHOD_IGNORE_NULLS)
3437 mi_collect_stats_nonulls_first(sort_param->seg, sort_param->notnull,
3438 static_cast<const uchar *>(a));
3439 }
3440 if ((sort_param->keyinfo->flag & HA_NOSAME) && cmp == 0) {
3441 sort_info->dupp++;
3442 sort_info->info->lastpos = get_record_for_key(
3443 sort_info->info, sort_param->keyinfo, static_cast<const uchar *>(a));
3444 mi_check_print_warning(
3445 param, "Duplicate key for record at %10s against record at %10s",
3446 llstr(sort_info->info->lastpos, llbuff),
3447 llstr(get_record_for_key(sort_info->info, sort_param->keyinfo,
3448 sort_info->key_block->lastkey),
3449 llbuff2));
3450 param->testflag |= T_RETRY_WITHOUT_QUICK;
3451 if (sort_info->param->testflag & T_VERBOSE)
3452 _mi_print_key(stdout, sort_param->seg, static_cast<const uchar *>(a),
3453 USE_WHOLE_KEY);
3454 return (sort_delete_record(sort_param));
3455 }
3456 #ifndef DBUG_OFF
3457 if (cmp > 0) {
3458 mi_check_print_error(param,
3459 "Internal error: Keys are not in order from sort");
3460 return (1);
3461 }
3462 #endif
3463 return (sort_insert_key(sort_param, sort_info->key_block,
3464 static_cast<const uchar *>(a), HA_OFFSET_ERROR));
3465 } /* sort_key_write */
3466
sort_ft_buf_flush(MI_SORT_PARAM * sort_param)3467 int sort_ft_buf_flush(MI_SORT_PARAM *sort_param) {
3468 SORT_INFO *sort_info = sort_param->sort_info;
3469 SORT_KEY_BLOCKS *key_block = sort_info->key_block;
3470 MYISAM_SHARE *share = sort_info->info->s;
3471 uint val_off, val_len;
3472 int error;
3473 SORT_FT_BUF *ft_buf = sort_info->ft_buf;
3474 uchar *from, *to;
3475
3476 val_len = share->ft2_keyinfo.keylength;
3477 get_key_full_length_rdonly(val_off, ft_buf->lastkey);
3478 to = ft_buf->lastkey + val_off;
3479
3480 if (ft_buf->buf) {
3481 /* flushing first-level tree */
3482 error = sort_insert_key(sort_param, key_block, ft_buf->lastkey,
3483 HA_OFFSET_ERROR);
3484 for (from = to + val_len; !error && from < ft_buf->buf; from += val_len) {
3485 memcpy(to, from, val_len);
3486 error = sort_insert_key(sort_param, key_block, ft_buf->lastkey,
3487 HA_OFFSET_ERROR);
3488 }
3489 return error;
3490 }
3491 /* flushing second-level tree keyblocks */
3492 error = flush_pending_blocks(sort_param);
3493 /* updating lastkey with second-level tree info */
3494 ft_intXstore(ft_buf->lastkey + val_off, -ft_buf->count);
3495 _mi_dpointer(sort_info->info, ft_buf->lastkey + val_off + HA_FT_WLEN,
3496 share->state.key_root[sort_param->key]);
3497 /* restoring first level tree data in sort_info/sort_param */
3498 sort_info->key_block =
3499 sort_info->key_block_end - sort_info->param->sort_key_blocks;
3500 sort_param->keyinfo = share->keyinfo + sort_param->key;
3501 share->state.key_root[sort_param->key] = HA_OFFSET_ERROR;
3502 /* writing lastkey in first-level tree */
3503 return error ? error
3504 : sort_insert_key(sort_param, sort_info->key_block,
3505 ft_buf->lastkey, HA_OFFSET_ERROR);
3506 }
3507
sort_ft_key_write(MI_SORT_PARAM * sort_param,const void * a)3508 static int sort_ft_key_write(MI_SORT_PARAM *sort_param, const void *a) {
3509 uint a_len, val_off, val_len, error;
3510 uchar *p;
3511 SORT_INFO *sort_info = sort_param->sort_info;
3512 SORT_FT_BUF *ft_buf = sort_info->ft_buf;
3513 SORT_KEY_BLOCKS *key_block = sort_info->key_block;
3514
3515 val_len = HA_FT_WLEN + sort_info->info->s->rec_reflength;
3516 get_key_full_length_rdonly(a_len, static_cast<const uchar *>(a));
3517
3518 if (!ft_buf) {
3519 /*
3520 use two-level tree only if key_reflength fits in rec_reflength place
3521 and row format is NOT static - for _mi_dpointer not to garble offsets
3522 */
3523 if ((sort_info->info->s->base.key_reflength <=
3524 sort_info->info->s->rec_reflength) &&
3525 (sort_info->info->s->options &
3526 (HA_OPTION_PACK_RECORD | HA_OPTION_COMPRESS_RECORD)))
3527 ft_buf = (SORT_FT_BUF *)my_malloc(
3528 mi_key_memory_SORT_FT_BUF,
3529 sort_param->keyinfo->block_length + sizeof(SORT_FT_BUF), MYF(MY_WME));
3530
3531 if (!ft_buf) {
3532 sort_param->key_write = sort_key_write;
3533 return sort_key_write(sort_param, a);
3534 }
3535 sort_info->ft_buf = ft_buf;
3536 goto word_init_ft_buf; /* no need to duplicate the code */
3537 }
3538 get_key_full_length_rdonly(val_off, ft_buf->lastkey);
3539
3540 if (ha_compare_text(sort_param->seg->charset,
3541 static_cast<const uchar *>(a) + 1, a_len - 1,
3542 ft_buf->lastkey + 1, val_off - 1, false) == 0) {
3543 if (!ft_buf->buf) /* store in second-level tree */
3544 {
3545 ft_buf->count++;
3546 return sort_insert_key(sort_param, key_block,
3547 static_cast<const uchar *>(a) + a_len,
3548 HA_OFFSET_ERROR);
3549 }
3550
3551 /* storing the key in the buffer. */
3552 memcpy(ft_buf->buf, static_cast<const char *>(a) + a_len, val_len);
3553 ft_buf->buf += val_len;
3554 if (ft_buf->buf < ft_buf->end) return 0;
3555
3556 /* converting to two-level tree */
3557 p = ft_buf->lastkey + val_off;
3558
3559 while (key_block->inited) key_block++;
3560 sort_info->key_block = key_block;
3561 sort_param->keyinfo = &sort_info->info->s->ft2_keyinfo;
3562 ft_buf->count = (uint)(ft_buf->buf - p) / val_len;
3563
3564 /* flushing buffer to second-level tree */
3565 for (error = 0; !error && p < ft_buf->buf; p += val_len)
3566 error = sort_insert_key(sort_param, key_block, p, HA_OFFSET_ERROR);
3567 ft_buf->buf = nullptr;
3568 return error;
3569 }
3570
3571 /* flushing buffer */
3572 if ((error = sort_ft_buf_flush(sort_param))) return error;
3573
3574 word_init_ft_buf:
3575 a_len += val_len;
3576 memcpy(ft_buf->lastkey, a, a_len);
3577 ft_buf->buf = ft_buf->lastkey + a_len;
3578 /*
3579 32 is just a safety margin here
3580 (at least max(val_len, sizeof(nod_flag)) should be there).
3581 May be better performance could be achieved if we'd put
3582 (sort_info->keyinfo->block_length-32)/XXX
3583 instead.
3584 TODO: benchmark the best value for XXX.
3585 */
3586 ft_buf->end = ft_buf->lastkey + (sort_param->keyinfo->block_length - 32);
3587 return 0;
3588 } /* sort_ft_key_write */
3589
3590 /* get pointer to record from a key */
3591
get_record_for_key(MI_INFO * info,MI_KEYDEF * keyinfo,const uchar * key)3592 static my_off_t get_record_for_key(MI_INFO *info, MI_KEYDEF *keyinfo,
3593 const uchar *key) {
3594 return _mi_dpos(info, 0, key + _mi_keylength(keyinfo, key));
3595 } /* get_record_for_key */
3596
3597 /* Insert a key in sort-key-blocks */
3598
sort_insert_key(MI_SORT_PARAM * sort_param,SORT_KEY_BLOCKS * key_block,const uchar * key,my_off_t prev_block)3599 static int sort_insert_key(MI_SORT_PARAM *sort_param,
3600 SORT_KEY_BLOCKS *key_block, const uchar *key,
3601 my_off_t prev_block) {
3602 uint a_length, t_length, nod_flag;
3603 my_off_t filepos, key_file_length;
3604 uchar *anc_buff, *lastkey;
3605 MI_KEY_PARAM s_temp;
3606 MI_INFO *info;
3607 MI_KEYDEF *keyinfo = sort_param->keyinfo;
3608 SORT_INFO *sort_info = sort_param->sort_info;
3609 MI_CHECK *param = sort_info->param;
3610 DBUG_TRACE;
3611
3612 anc_buff = key_block->buff;
3613 info = sort_info->info;
3614 lastkey = key_block->lastkey;
3615 nod_flag =
3616 (key_block == sort_info->key_block ? 0 : info->s->base.key_reflength);
3617
3618 if (!key_block->inited) {
3619 key_block->inited = 1;
3620 if (key_block == sort_info->key_block_end) {
3621 mi_check_print_error(
3622 param, "To many key-block-levels; Try increasing sort_key_blocks");
3623 return 1;
3624 }
3625 a_length = 2 + nod_flag;
3626 key_block->end_pos = anc_buff + 2;
3627 lastkey = nullptr; /* No previous key in block */
3628 } else
3629 a_length = mi_getint(anc_buff);
3630
3631 /* Save pointer to previous block */
3632 if (nod_flag) _mi_kpointer(info, key_block->end_pos, prev_block);
3633
3634 t_length = (*keyinfo->pack_key)(keyinfo, nod_flag, (uchar *)nullptr, lastkey,
3635 lastkey, key, &s_temp);
3636 (*keyinfo->store_key)(keyinfo, key_block->end_pos + nod_flag, &s_temp);
3637 a_length += t_length;
3638 mi_putint(anc_buff, a_length, nod_flag);
3639 key_block->end_pos += t_length;
3640 if (a_length <= keyinfo->block_length) {
3641 (void)_mi_move_key(keyinfo, key_block->lastkey, key);
3642 key_block->last_length = a_length - t_length;
3643 return 0;
3644 }
3645
3646 /* Fill block with end-zero and write filled block */
3647 mi_putint(anc_buff, key_block->last_length, nod_flag);
3648 memset(anc_buff + key_block->last_length, 0,
3649 keyinfo->block_length - key_block->last_length);
3650 key_file_length = info->state->key_file_length;
3651 if ((filepos = _mi_new(info, keyinfo, DFLT_INIT_HITS)) == HA_OFFSET_ERROR)
3652 return 1;
3653
3654 /* If we read the page from the key cache, we have to write it back to it */
3655 if (key_file_length == info->state->key_file_length) {
3656 if (_mi_write_keypage(info, keyinfo, filepos, DFLT_INIT_HITS, anc_buff))
3657 return 1;
3658 } else if (mysql_file_pwrite(info->s->kfile, (uchar *)anc_buff,
3659 (uint)keyinfo->block_length, filepos,
3660 param->myf_rw))
3661 return 1;
3662 DBUG_DUMP("buff", (uchar *)anc_buff, mi_getint(anc_buff));
3663
3664 /* Write separator-key to block in next level */
3665 if (sort_insert_key(sort_param, key_block + 1, key_block->lastkey, filepos))
3666 return 1;
3667
3668 /* clear old block and write new key in it */
3669 key_block->inited = 0;
3670 return sort_insert_key(sort_param, key_block, key, prev_block);
3671 } /* sort_insert_key */
3672
3673 /* Delete record when we found a duplicated key */
3674
sort_delete_record(MI_SORT_PARAM * sort_param)3675 static int sort_delete_record(MI_SORT_PARAM *sort_param) {
3676 uint i;
3677 int old_file, error;
3678 uchar *key;
3679 SORT_INFO *sort_info = sort_param->sort_info;
3680 MI_CHECK *param = sort_info->param;
3681 MI_INFO *info = sort_info->info;
3682 DBUG_TRACE;
3683
3684 if ((param->testflag & (T_FORCE_UNIQUENESS | T_QUICK)) == T_QUICK) {
3685 mi_check_print_error(param,
3686 "Quick-recover aborted; Run recovery without switch "
3687 "-q or with switch -qq");
3688 return 1;
3689 }
3690 if (info->s->options & HA_OPTION_COMPRESS_RECORD) {
3691 mi_check_print_error(param,
3692 "Recover aborted; Can't run standard recovery on "
3693 "compressed tables with errors in data-file. Use "
3694 "switch 'myisamchk --safe-recover' to fix it\n");
3695 ;
3696 return 1;
3697 }
3698
3699 old_file = info->dfile;
3700 info->dfile = info->rec_cache.file;
3701 if (sort_info->current_key) {
3702 key = info->lastkey + info->s->base.max_key_length;
3703 if ((error = (*info->s->read_rnd)(info, sort_param->record, info->lastpos,
3704 false)) &&
3705 error != HA_ERR_RECORD_DELETED) {
3706 mi_check_print_error(param, "Can't read record to be removed");
3707 info->dfile = old_file;
3708 return 1;
3709 }
3710
3711 for (i = 0; i < sort_info->current_key; i++) {
3712 uint key_length =
3713 _mi_make_key(info, i, key, sort_param->record, info->lastpos);
3714 if (_mi_ck_delete(info, i, key, key_length)) {
3715 mi_check_print_error(
3716 param, "Can't delete key %d from record to be removed", i + 1);
3717 info->dfile = old_file;
3718 return 1;
3719 }
3720 }
3721 if (sort_param->calc_checksum)
3722 param->glob_crc -= (*info->s->calc_checksum)(info, sort_param->record);
3723 }
3724 error = flush_io_cache(&info->rec_cache) || (*info->s->delete_record)(info);
3725 info->dfile = old_file; /* restore actual value */
3726 info->state->records--;
3727 return error;
3728 } /* sort_delete_record */
3729
3730 /* Fix all pending blocks and flush everything to disk */
3731
flush_pending_blocks(MI_SORT_PARAM * sort_param)3732 int flush_pending_blocks(MI_SORT_PARAM *sort_param) {
3733 uint nod_flag, length;
3734 my_off_t filepos, key_file_length;
3735 SORT_KEY_BLOCKS *key_block;
3736 SORT_INFO *sort_info = sort_param->sort_info;
3737 myf myf_rw = sort_info->param->myf_rw;
3738 MI_INFO *info = sort_info->info;
3739 MI_KEYDEF *keyinfo = sort_param->keyinfo;
3740 DBUG_TRACE;
3741
3742 filepos = HA_OFFSET_ERROR; /* if empty file */
3743 nod_flag = 0;
3744 for (key_block = sort_info->key_block; key_block->inited; key_block++) {
3745 key_block->inited = 0;
3746 length = mi_getint(key_block->buff);
3747 if (nod_flag) _mi_kpointer(info, key_block->end_pos, filepos);
3748 key_file_length = info->state->key_file_length;
3749 memset(key_block->buff + length, 0, keyinfo->block_length - length);
3750 if ((filepos = _mi_new(info, keyinfo, DFLT_INIT_HITS)) == HA_OFFSET_ERROR)
3751 return 1;
3752
3753 /* If we read the page from the key cache, we have to write it back */
3754 if (key_file_length == info->state->key_file_length) {
3755 if (_mi_write_keypage(info, keyinfo, filepos, DFLT_INIT_HITS,
3756 key_block->buff))
3757 return 1;
3758 } else if (mysql_file_pwrite(info->s->kfile, (uchar *)key_block->buff,
3759 (uint)keyinfo->block_length, filepos, myf_rw))
3760 return 1;
3761 DBUG_DUMP("buff", (uchar *)key_block->buff, length);
3762 nod_flag = 1;
3763 }
3764 info->s->state.key_root[sort_param->key] =
3765 filepos; /* Last is root for tree */
3766 return 0;
3767 } /* flush_pending_blocks */
3768
3769 /* alloc space and pointers for key_blocks */
3770
alloc_key_blocks(MI_CHECK * param,uint blocks,uint buffer_length)3771 static SORT_KEY_BLOCKS *alloc_key_blocks(MI_CHECK *param, uint blocks,
3772 uint buffer_length) {
3773 uint i;
3774 SORT_KEY_BLOCKS *block;
3775 DBUG_TRACE;
3776
3777 if (!(block = (SORT_KEY_BLOCKS *)my_malloc(
3778 mi_key_memory_SORT_KEY_BLOCKS,
3779 (sizeof(SORT_KEY_BLOCKS) + buffer_length + IO_SIZE) * blocks,
3780 MYF(0)))) {
3781 mi_check_print_error(param, "Not enough memory for sort-key-blocks");
3782 return nullptr;
3783 }
3784 for (i = 0; i < blocks; i++) {
3785 block[i].inited = 0;
3786 block[i].buff = (uchar *)(block + blocks) + (buffer_length + IO_SIZE) * i;
3787 }
3788 return block;
3789 } /* alloc_key_blocks */
3790
3791 /* Check if file is almost full */
3792
test_if_almost_full(MI_INFO * info)3793 int test_if_almost_full(MI_INFO *info) {
3794 if (info->s->options & HA_OPTION_COMPRESS_RECORD) return 0;
3795 return mysql_file_seek(info->s->kfile, 0L, MY_SEEK_END, MYF(0)) / 10 * 9 >
3796 (my_off_t)info->s->base.max_key_file_length ||
3797 mysql_file_seek(info->dfile, 0L, MY_SEEK_END, MYF(0)) / 10 * 9 >
3798 (my_off_t)info->s->base.max_data_file_length;
3799 }
3800
3801 /* Recreate table with bigger more alloced record-data */
3802
recreate_table(MI_CHECK * param,MI_INFO ** org_info,char * filename)3803 int recreate_table(MI_CHECK *param, MI_INFO **org_info, char *filename) {
3804 int error;
3805 MYISAM_SHARE share;
3806 MI_KEYDEF *keyinfo, *key, *key_end;
3807 HA_KEYSEG *keysegs, *keyseg;
3808 MI_COLUMNDEF *recdef, *rec, *end;
3809 MI_UNIQUEDEF *uniquedef, *u_ptr, *u_end;
3810 MI_STATUS_INFO status_info;
3811 uint unpack, key_parts;
3812 ha_rows max_records;
3813 ulonglong file_length, tmp_length;
3814 MI_CREATE_INFO create_info;
3815 DBUG_TRACE;
3816
3817 error = 1; /* Default error */
3818 const MI_INFO &info = **org_info;
3819 status_info = (*org_info)->state[0];
3820 share = *(*org_info)->s;
3821 unpack = (share.options & HA_OPTION_COMPRESS_RECORD) &&
3822 (param->testflag & T_UNPACK);
3823 if (!(keyinfo = (MI_KEYDEF *)my_alloca(sizeof(MI_KEYDEF) * share.base.keys)))
3824 return 0;
3825 memcpy((uchar *)keyinfo, (uchar *)share.keyinfo,
3826 (size_t)(sizeof(MI_KEYDEF) * share.base.keys));
3827
3828 key_parts = share.base.all_key_parts;
3829 if (!(keysegs = (HA_KEYSEG *)my_alloca(sizeof(HA_KEYSEG) *
3830 (key_parts + share.base.keys)))) {
3831 return 1;
3832 }
3833 if (!(recdef = (MI_COLUMNDEF *)my_alloca(sizeof(MI_COLUMNDEF) *
3834 (share.base.fields + 1)))) {
3835 return 1;
3836 }
3837 if (!(uniquedef = (MI_UNIQUEDEF *)my_alloca(
3838 sizeof(MI_UNIQUEDEF) * (share.state.header.uniques + 1)))) {
3839 return 1;
3840 }
3841
3842 /* Copy the column definitions */
3843 memcpy((uchar *)recdef, (uchar *)share.rec,
3844 (size_t)(sizeof(MI_COLUMNDEF) * (share.base.fields + 1)));
3845 if (unpack && !(share.options & HA_OPTION_PACK_RECORD)) {
3846 for (rec = recdef, end = recdef + share.base.fields; rec != end; rec++) {
3847 if (rec->type != FIELD_BLOB && rec->type != FIELD_VARCHAR &&
3848 rec->type != FIELD_CHECK)
3849 rec->type = (int)FIELD_NORMAL;
3850 }
3851 }
3852
3853 /* Change the new key to point at the saved key segments */
3854 memcpy((uchar *)keysegs, (uchar *)share.keyparts,
3855 (size_t)(sizeof(HA_KEYSEG) *
3856 (key_parts + share.base.keys + share.state.header.uniques)));
3857 keyseg = keysegs;
3858 for (key = keyinfo, key_end = keyinfo + share.base.keys; key != key_end;
3859 key++) {
3860 key->seg = keyseg;
3861 for (; keyseg->type; keyseg++) {
3862 if (param->language)
3863 keyseg->language = param->language; /* change language */
3864 }
3865 keyseg++; /* Skip end pointer */
3866 }
3867
3868 /* Copy the unique definitions and change them to point at the new key
3869 segments*/
3870 memcpy((uchar *)uniquedef, (uchar *)share.uniqueinfo,
3871 (size_t)(sizeof(MI_UNIQUEDEF) * (share.state.header.uniques)));
3872 for (u_ptr = uniquedef, u_end = uniquedef + share.state.header.uniques;
3873 u_ptr != u_end; u_ptr++) {
3874 u_ptr->seg = keyseg;
3875 keyseg += u_ptr->keysegs + 1;
3876 }
3877 unpack = (share.options & HA_OPTION_COMPRESS_RECORD) &&
3878 (param->testflag & T_UNPACK);
3879 share.options &= ~HA_OPTION_TEMP_COMPRESS_RECORD;
3880
3881 file_length = (ulonglong)mysql_file_seek(info.dfile, 0L, MY_SEEK_END, MYF(0));
3882 tmp_length = file_length + file_length / 10;
3883 file_length = std::max(file_length, param->max_data_file_length);
3884 file_length = std::max(file_length, tmp_length);
3885 file_length =
3886 std::max(file_length, ulonglong(share.base.max_data_file_length));
3887
3888 if (share.options & HA_OPTION_COMPRESS_RECORD)
3889 share.base.records = max_records = info.state->records;
3890 else if (!(share.options & HA_OPTION_PACK_RECORD))
3891 max_records = (ha_rows)(file_length / share.base.pack_reclength);
3892 else
3893 max_records = 0;
3894
3895 (void)mi_close(*org_info);
3896 memset(&create_info, 0, sizeof(create_info));
3897 create_info.max_rows = max_records;
3898 create_info.reloc_rows = share.base.reloc;
3899 create_info.old_options =
3900 (share.options | (unpack ? HA_OPTION_TEMP_COMPRESS_RECORD : 0));
3901
3902 create_info.data_file_length = file_length;
3903 create_info.auto_increment = share.state.auto_increment;
3904 create_info.language =
3905 (param->language ? param->language : share.state.header.language);
3906 create_info.key_file_length = status_info.key_file_length;
3907 /*
3908 Allow for creating an auto_increment key. This has an effect only if
3909 an auto_increment key exists in the original table.
3910 */
3911 create_info.with_auto_increment = true;
3912 /* We don't have to handle symlinks here because we are using
3913 HA_DONT_TOUCH_DATA */
3914 if (mi_create(filename, share.base.keys - share.state.header.uniques, keyinfo,
3915 share.base.fields, recdef, share.state.header.uniques,
3916 uniquedef, &create_info, HA_DONT_TOUCH_DATA)) {
3917 mi_check_print_error(
3918 param, "Got error %d when trying to recreate indexfile", my_errno());
3919 goto end;
3920 }
3921 *org_info =
3922 mi_open(filename, O_RDWR,
3923 (param->testflag & T_WAIT_FOREVER)
3924 ? HA_OPEN_WAIT_IF_LOCKED
3925 : (param->testflag & T_DESCRIPT) ? HA_OPEN_IGNORE_IF_LOCKED
3926 : HA_OPEN_ABORT_IF_LOCKED);
3927 if (!*org_info) {
3928 mi_check_print_error(
3929 param, "Got error %d when trying to open re-created indexfile",
3930 my_errno());
3931 goto end;
3932 }
3933 /* We are modifing */
3934 (*org_info)->s->options &= ~HA_OPTION_READ_ONLY_DATA;
3935 (void)_mi_readinfo(*org_info, F_WRLCK, 0);
3936 (*org_info)->state->records = status_info.records;
3937 if (share.state.create_time)
3938 (*org_info)->s->state.create_time = share.state.create_time;
3939 (*org_info)->s->state.unique = (*org_info)->this_unique = share.state.unique;
3940 (*org_info)->state->checksum = status_info.checksum;
3941 (*org_info)->state->del = status_info.del;
3942 (*org_info)->s->state.dellink = share.state.dellink;
3943 (*org_info)->state->empty = status_info.empty;
3944 (*org_info)->state->data_file_length = status_info.data_file_length;
3945 if (update_state_info(param, *org_info,
3946 UPDATE_TIME | UPDATE_STAT | UPDATE_OPEN_COUNT))
3947 goto end;
3948 error = 0;
3949 end:
3950 return error;
3951 }
3952
3953 /* write suffix to data file if neaded */
3954
write_data_suffix(SORT_INFO * sort_info,bool fix_datafile)3955 int write_data_suffix(SORT_INFO *sort_info, bool fix_datafile) {
3956 MI_INFO *info = sort_info->info;
3957
3958 if (info->s->options & HA_OPTION_COMPRESS_RECORD && fix_datafile) {
3959 uchar buff[MEMMAP_EXTRA_MARGIN];
3960 memset(buff, 0, sizeof(buff));
3961 if (my_b_write(&info->rec_cache, buff, sizeof(buff))) {
3962 mi_check_print_error(sort_info->param, "%d when writing to datafile",
3963 my_errno());
3964 return 1;
3965 }
3966 sort_info->param->read_cache.end_of_file += sizeof(buff);
3967 }
3968 return 0;
3969 }
3970
3971 /* Update state and myisamchk_time of indexfile */
3972
update_state_info(MI_CHECK * param,MI_INFO * info,uint update)3973 int update_state_info(MI_CHECK *param, MI_INFO *info, uint update) {
3974 MYISAM_SHARE *share = info->s;
3975
3976 if (update & UPDATE_OPEN_COUNT) {
3977 share->state.open_count = 0;
3978 share->global_changed = false;
3979 }
3980 if (update & UPDATE_STAT) {
3981 uint i, key_parts = mi_uint2korr(share->state.header.key_parts);
3982 share->state.rec_per_key_rows = info->state->records;
3983 share->state.changed &= ~STATE_NOT_ANALYZED;
3984 if (info->state->records) {
3985 for (i = 0; i < key_parts; i++) {
3986 if (!(share->state.rec_per_key_part[i] = param->rec_per_key_part[i]))
3987 share->state.changed |= STATE_NOT_ANALYZED;
3988 }
3989 }
3990 }
3991 if (update & (UPDATE_STAT | UPDATE_SORT | UPDATE_TIME | UPDATE_AUTO_INC)) {
3992 if (update & UPDATE_TIME) {
3993 share->state.check_time = (long)time((time_t *)nullptr);
3994 if (!share->state.create_time)
3995 share->state.create_time = share->state.check_time;
3996 }
3997 /*
3998 When tables are locked we haven't synched the share state and the
3999 real state for a while so we better do it here before synching
4000 the share state to disk. Only when table is write locked is it
4001 necessary to perform this synch.
4002 */
4003 if (info->lock_type == F_WRLCK) share->state.state = *info->state;
4004 if (mi_state_info_write(share->kfile, &share->state, 1 + 2)) goto err;
4005 share->changed = false;
4006 }
4007 { /* Force update of status */
4008 int error;
4009 uint r_locks = share->r_locks, w_locks = share->w_locks;
4010 share->r_locks = share->w_locks = share->tot_locks = 0;
4011
4012 DBUG_EXECUTE_IF("simulate_incorrect_share_wlock_value",
4013 DEBUG_SYNC_C("after_share_wlock_set_to_0"););
4014
4015 error = _mi_writeinfo(info, WRITEINFO_NO_UNLOCK);
4016 share->r_locks = r_locks;
4017 share->w_locks = w_locks;
4018 share->tot_locks = r_locks + w_locks;
4019 if (!error) return 0;
4020 }
4021 err:
4022 mi_check_print_error(param, "%d when updating keyfile", my_errno());
4023 return 1;
4024 }
4025
4026 /*
4027 Update auto increment value for a table
4028 When setting the 'repair_only' flag we only want to change the
4029 old auto_increment value if its wrong (smaller than some given key).
4030 The reason is that we shouldn't change the auto_increment value
4031 for a table without good reason when only doing a repair; If the
4032 user have inserted and deleted rows, the auto_increment value
4033 may be bigger than the biggest current row and this is ok.
4034
4035 If repair_only is not set, we will update the flag to the value in
4036 param->auto_increment is bigger than the biggest key.
4037 */
4038
update_auto_increment_key(MI_CHECK * param,MI_INFO * info,bool repair_only)4039 void update_auto_increment_key(MI_CHECK *param, MI_INFO *info,
4040 bool repair_only) {
4041 uchar *record = nullptr;
4042 DBUG_TRACE;
4043
4044 if (!info->s->base.auto_key ||
4045 !mi_is_key_active(info->s->state.key_map, info->s->base.auto_key - 1)) {
4046 if (!(param->testflag & T_VERY_SILENT))
4047 mi_check_print_info(param,
4048 "Table: %s doesn't have an auto increment key\n",
4049 param->isam_file_name);
4050 return;
4051 }
4052 if (!(param->testflag & T_SILENT) && !(param->testflag & T_REP))
4053 printf("Updating MyISAM file: %s\n", param->isam_file_name);
4054 /*
4055 We have to use an allocated buffer instead of info->rec_buff as
4056 _mi_put_key_in_record() may use info->rec_buff
4057 */
4058 if (!mi_alloc_rec_buff(info, -1, &record)) {
4059 mi_check_print_error(param, "Not enough memory for extra record");
4060 return;
4061 }
4062
4063 mi_extra(info, HA_EXTRA_KEYREAD, nullptr);
4064 if (mi_rlast(info, record, info->s->base.auto_key - 1)) {
4065 if (my_errno() != HA_ERR_END_OF_FILE) {
4066 mi_extra(info, HA_EXTRA_NO_KEYREAD, nullptr);
4067 my_free(mi_get_rec_buff_ptr(info, record));
4068 mi_check_print_error(param, "%d when reading last record", my_errno());
4069 return;
4070 }
4071 if (!repair_only)
4072 info->s->state.auto_increment = param->auto_increment_value;
4073 } else {
4074 ulonglong auto_increment = retrieve_auto_increment(info, record);
4075 info->s->state.auto_increment =
4076 std::max(info->s->state.auto_increment, auto_increment);
4077 if (!repair_only)
4078 info->s->state.auto_increment =
4079 std::max(info->s->state.auto_increment, param->auto_increment_value);
4080 }
4081 mi_extra(info, HA_EXTRA_NO_KEYREAD, nullptr);
4082 my_free(mi_get_rec_buff_ptr(info, record));
4083 update_state_info(param, info, UPDATE_AUTO_INC);
4084 }
4085
4086 /*
4087 Update statistics for each part of an index
4088
4089 SYNOPSIS
4090 update_key_parts()
4091 keyinfo IN Index information (only key->keysegs used)
4092 rec_per_key_part OUT Store statistics here
4093 unique IN Array of (#distinct tuples)
4094 notnull_tuples IN Array of (#tuples), or NULL
4095 records Number of records in the table
4096
4097 DESCRIPTION
4098 This function is called produce index statistics values from unique and
4099 notnull_tuples arrays after these arrays were produced with sequential
4100 index scan (the scan is done in two places: chk_index() and
4101 sort_key_write()).
4102
4103 This function handles all 3 index statistics collection methods.
4104
4105 Unique is an array:
4106 unique[0]= (#different values of {keypart1}) - 1
4107 unique[1]= (#different values of {keypart1,keypart2} tuple)-unique[0]-1
4108 ...
4109
4110 For MI_STATS_METHOD_IGNORE_NULLS method, notnull_tuples is an array too:
4111 notnull_tuples[0]= (#of {keypart1} tuples such that keypart1 is not NULL)
4112 notnull_tuples[1]= (#of {keypart1,keypart2} tuples such that all
4113 keypart{i} are not NULL)
4114 ...
4115 For all other statistics collection methods notnull_tuples==NULL.
4116
4117 Output is an array:
4118 rec_per_key_part[k] =
4119 = E(#records in the table such that keypart_1=c_1 AND ... AND
4120 keypart_k=c_k for arbitrary constants c_1 ... c_k)
4121
4122 = {assuming that values have uniform distribution and index contains all
4123 tuples from the domain (or that {c_1, ..., c_k} tuple is choosen from
4124 index tuples}
4125
4126 = #tuples-in-the-index / #distinct-tuples-in-the-index.
4127
4128 The #tuples-in-the-index and #distinct-tuples-in-the-index have different
4129 meaning depending on which statistics collection method is used:
4130
4131 MI_STATS_METHOD_* how are nulls compared? which tuples are counted?
4132 NULLS_EQUAL NULL == NULL all tuples in table
4133 NULLS_NOT_EQUAL NULL != NULL all tuples in table
4134 IGNORE_NULLS n/a tuples that don't have NULLs
4135 */
4136
update_key_parts(MI_KEYDEF * keyinfo,ulong * rec_per_key_part,ulonglong * unique,ulonglong * notnull,ulonglong records)4137 void update_key_parts(MI_KEYDEF *keyinfo, ulong *rec_per_key_part,
4138 ulonglong *unique, ulonglong *notnull,
4139 ulonglong records) {
4140 ulonglong count = 0, tmp, unique_tuples;
4141 ulonglong tuples = records;
4142 uint parts;
4143 uint maxparts;
4144
4145 if (keyinfo->flag & HA_SPATIAL)
4146 maxparts = 1; /* Only 1 key part (but 4 segments) */
4147 else
4148 maxparts = keyinfo->keysegs; /* parts == segments == columns */
4149
4150 for (parts = 0; parts < maxparts; parts++) {
4151 count += unique[parts];
4152 unique_tuples = count + 1;
4153 if (notnull) {
4154 tuples = notnull[parts];
4155 /*
4156 #(unique_tuples not counting tuples with NULLs) =
4157 #(unique_tuples counting tuples with NULLs as different) -
4158 #(tuples with NULLs)
4159 */
4160 unique_tuples -= (records - notnull[parts]);
4161 }
4162
4163 if (unique_tuples == 0)
4164 tmp = 1;
4165 else if (count == 0)
4166 tmp = tuples; /* 1 unique tuple */
4167 else
4168 tmp = (tuples + unique_tuples / 2) / unique_tuples;
4169
4170 /*
4171 for some weird keys (e.g. FULLTEXT) tmp can be <1 here.
4172 let's ensure it is not
4173 */
4174 tmp = std::max(tmp, 1ULL);
4175 if (tmp >= (ulonglong) ~(ulong)0) tmp = (ulonglong) ~(ulong)0;
4176
4177 *rec_per_key_part = (ulong)tmp;
4178 rec_per_key_part++;
4179 }
4180 }
4181
mi_byte_checksum(const uchar * buf,uint length)4182 static ha_checksum mi_byte_checksum(const uchar *buf, uint length) {
4183 ha_checksum crc;
4184 const uchar *end = buf + length;
4185 for (crc = 0; buf != end; buf++) {
4186 crc = (crc << 1) | (crc >> (8 * sizeof(ha_checksum) - 1));
4187 crc += *buf;
4188 }
4189 return crc;
4190 }
4191
mi_too_big_key_for_sort(MI_KEYDEF * key,ha_rows rows)4192 static bool mi_too_big_key_for_sort(MI_KEYDEF *key, ha_rows rows) {
4193 uint key_maxlength = key->maxlength;
4194 if (key->flag & HA_FULLTEXT) {
4195 uint ft_max_word_len_for_sort =
4196 FT_MAX_WORD_LEN_FOR_SORT * key->seg->charset->mbmaxlen;
4197 key_maxlength += ft_max_word_len_for_sort - HA_FT_MAXBYTELEN;
4198 }
4199 return (key->flag & HA_SPATIAL) ||
4200 (key->flag & (HA_BINARY_PACK_KEY | HA_VAR_LENGTH_KEY | HA_FULLTEXT) &&
4201 ((ulonglong)rows * key_maxlength > myisam_max_temp_length));
4202 }
4203
4204 /*
4205 Deactivate all not unique index that can be recreated fast
4206 These include packed keys on which sorting will use more temporary
4207 space than the max allowed file length or for which the unpacked keys
4208 will take much more space than packed keys.
4209 Note that 'rows' may be zero for the case when we don't know how many
4210 rows we will put into the file.
4211 */
4212
mi_disable_non_unique_index(MI_INFO * info,ha_rows rows)4213 void mi_disable_non_unique_index(MI_INFO *info, ha_rows rows) {
4214 MYISAM_SHARE *share = info->s;
4215 MI_KEYDEF *key = share->keyinfo;
4216 uint i;
4217
4218 DBUG_ASSERT(info->state->records == 0 &&
4219 (!rows || rows >= MI_MIN_ROWS_TO_DISABLE_INDEXES));
4220 for (i = 0; i < share->base.keys; i++, key++) {
4221 if (!(key->flag & (HA_NOSAME | HA_SPATIAL | HA_AUTO_KEY)) &&
4222 !mi_too_big_key_for_sort(key, rows) &&
4223 info->s->base.auto_key != i + 1) {
4224 mi_clear_key_active(share->state.key_map, i);
4225 info->update |= HA_STATE_CHANGED;
4226 }
4227 }
4228 }
4229
4230 /*
4231 Return true if we can use repair by sorting
4232 One can set the force argument to force to use sorting
4233 even if the temporary file would be quite big!
4234 */
4235
mi_test_if_sort_rep(MI_INFO * info,ha_rows rows,ulonglong key_map,bool force)4236 bool mi_test_if_sort_rep(MI_INFO *info, ha_rows rows, ulonglong key_map,
4237 bool force) {
4238 MYISAM_SHARE *share = info->s;
4239 MI_KEYDEF *key = share->keyinfo;
4240 uint i;
4241
4242 /*
4243 mi_repair_by_sort only works if we have at least one key. If we don't
4244 have any keys, we should use the normal repair.
4245 */
4246 if (!mi_is_any_key_active(key_map)) return false; /* Can't use sort */
4247 if (!force) {
4248 for (i = 0; i < share->base.keys; i++, key++) {
4249 if (mi_too_big_key_for_sort(key, rows)) return false;
4250 }
4251 }
4252 return true;
4253 }
4254
set_data_file_type(SORT_INFO * sort_info,MYISAM_SHARE * share)4255 static void set_data_file_type(SORT_INFO *sort_info, MYISAM_SHARE *share) {
4256 if ((sort_info->new_data_file_type = share->data_file_type) ==
4257 COMPRESSED_RECORD &&
4258 sort_info->param->testflag & T_UNPACK) {
4259 MYISAM_SHARE tmp;
4260
4261 if (share->options & HA_OPTION_PACK_RECORD)
4262 sort_info->new_data_file_type = DYNAMIC_RECORD;
4263 else
4264 sort_info->new_data_file_type = STATIC_RECORD;
4265
4266 /* Set delete_function for sort_delete_record() */
4267 memcpy((char *)&tmp, share, sizeof(*share));
4268 tmp.options = ~HA_OPTION_COMPRESS_RECORD;
4269 mi_setup_functions(&tmp);
4270 share->delete_record = tmp.delete_record;
4271 }
4272 }
4273
4274 /*
4275 Find the first NULL value in index-suffix values tuple
4276
4277 SYNOPSIS
4278 ha_find_null()
4279 keyseg Array of keyparts for key suffix
4280 a Key suffix value tuple
4281
4282 DESCRIPTION
4283 Find the first NULL value in index-suffix values tuple.
4284
4285 TODO
4286 Consider optimizing this function or its use so we don't search for
4287 NULL values in completely NOT NULL index suffixes.
4288
4289 RETURN
4290 First key part that has NULL as value in values tuple, or the last key
4291 part (with keyseg->type==HA_TYPE_END) if values tuple doesn't contain
4292 NULLs.
4293 */
4294
ha_find_null(HA_KEYSEG * keyseg,const uchar * a)4295 static HA_KEYSEG *ha_find_null(HA_KEYSEG *keyseg, const uchar *a) {
4296 for (; (enum ha_base_keytype)keyseg->type != HA_KEYTYPE_END; keyseg++) {
4297 const uchar *end;
4298 if (keyseg->null_bit) {
4299 if (!*a++) return keyseg;
4300 }
4301 end = a + keyseg->length;
4302
4303 switch ((enum ha_base_keytype)keyseg->type) {
4304 case HA_KEYTYPE_TEXT:
4305 case HA_KEYTYPE_BINARY:
4306 case HA_KEYTYPE_BIT:
4307 if (keyseg->flag & HA_SPACE_PACK) {
4308 int a_length = get_key_length(&a);
4309 a += a_length;
4310 break;
4311 } else
4312 a = end;
4313 break;
4314 case HA_KEYTYPE_VARTEXT1:
4315 case HA_KEYTYPE_VARTEXT2:
4316 case HA_KEYTYPE_VARBINARY1:
4317 case HA_KEYTYPE_VARBINARY2: {
4318 int a_length = get_key_length(&a);
4319 a += a_length;
4320 break;
4321 }
4322 case HA_KEYTYPE_NUM:
4323 if (keyseg->flag & HA_SPACE_PACK) {
4324 int alength = *a++;
4325 end = a + alength;
4326 }
4327 a = end;
4328 break;
4329 case HA_KEYTYPE_INT8:
4330 case HA_KEYTYPE_SHORT_INT:
4331 case HA_KEYTYPE_USHORT_INT:
4332 case HA_KEYTYPE_LONG_INT:
4333 case HA_KEYTYPE_ULONG_INT:
4334 case HA_KEYTYPE_INT24:
4335 case HA_KEYTYPE_UINT24:
4336 case HA_KEYTYPE_LONGLONG:
4337 case HA_KEYTYPE_ULONGLONG:
4338 case HA_KEYTYPE_FLOAT:
4339 case HA_KEYTYPE_DOUBLE:
4340 a = end;
4341 break;
4342 case HA_KEYTYPE_END: /* purecov: inspected */
4343 /* keep compiler happy */
4344 DBUG_ASSERT(0);
4345 break;
4346 }
4347 }
4348 return keyseg;
4349 }
4350