1 /*
2 Copyright (c) 2005, 2016, Oracle and/or its affiliates. All rights reserved.
3
4 This program is free software; you can redistribute it and/or modify
5 it under the terms of the GNU General Public License, version 2.0,
6 as published by the Free Software Foundation.
7
8 This program is also distributed with certain software (including
9 but not limited to OpenSSL) that is licensed under separate terms,
10 as designated in a particular file or component or in included license
11 documentation. The authors of MySQL hereby grant you an additional
12 permission to link the program and your derivative works with the
13 separately licensed software that they have included with MySQL.
14
15 This program is distributed in the hope that it will be useful,
16 but WITHOUT ANY WARRANTY; without even the implied warranty of
17 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
18 GNU General Public License, version 2.0, for more details.
19
20 You should have received a copy of the GNU General Public License
21 along with this program; if not, write to the Free Software
22 Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA
23 */
24
25 /*
26 InnoDB offline file checksum utility. 85% of the code in this utility
27 is included from the InnoDB codebase.
28
29 The final 15% was originally written by Mark Smith of Danga
30 Interactive, Inc. <junior@danga.com>
31
32 Published with a permission.
33 */
34
35 #include <my_config.h>
36 #include <my_global.h>
37 #include <stdio.h>
38 #include <stdlib.h>
39 #include <time.h>
40 #include <sys/types.h>
41 #include <sys/stat.h>
42 #ifndef __WIN__
43 # include <unistd.h>
44 #endif
45 #include <my_getopt.h>
46 #include <m_string.h>
47 #include <welcome_copyright_notice.h> /* ORACLE_WELCOME_COPYRIGHT_NOTICE */
48
49 /* Only parts of these files are included from the InnoDB codebase.
50 The parts not included are excluded by #ifndef UNIV_INNOCHECKSUM. */
51
52 #include "univ.i" /* include all of this */
53
54 #include "buf0checksum.h" /* buf_calc_page_*() */
55 #include "dict0mem.h" /* DICT_TF_BITS et al */
56 #include "fil0fil.h" /* FIL_* */
57 #include "page0zip.h" /* page_zip_*() */
58 #include "fsp0fsp.h" /* fsp_flags_get_page_size() &
59 fsp_flags_get_zip_size() */
60 #include "mach0data.h" /* mach_read_from_4() */
61 #include "ut0crc32.h" /* ut_crc32_init() */
62
63 #ifdef UNIV_NONINL
64 # include "fsp0fsp.ic"
65 # include "mach0data.ic"
66 # include "ut0rnd.ic"
67 #endif
68
69 /* Global variables */
70 static my_bool verbose;
71 static my_bool debug;
72 static my_bool just_count;
73 static ullint start_page;
74 static ullint end_page;
75 static ullint do_page;
76 static my_bool use_end_page;
77 static my_bool do_one_page;
78 static my_bool display_format;
79 ulong srv_page_size; /* replaces declaration in srv0srv.c */
80 static ulong physical_page_size; /* Page size in bytes on disk. */
81 static ulong logical_page_size; /* Page size when uncompressed. */
82
83 /* Get the page size of the filespace from the filespace header. */
84 static
85 my_bool
get_page_size(FILE * f,byte * buf,ulong * logical_page_size,ulong * physical_page_size,bool * compressed)86 get_page_size(
87 /*==========*/
88 FILE* f, /*!< in: file pointer, must be open
89 and set to start of file */
90 byte* buf, /*!< in: buffer used to read the page */
91 ulong* logical_page_size, /*!< out: Logical/Uncompressed page size */
92 ulong* physical_page_size, /*!< out: Physical/Commpressed page size */
93 bool* compressed) /*!< out: compression flag */
94 {
95 ulong flags;
96
97 ulong bytes= ulong(fread(buf, 1, UNIV_PAGE_SIZE_MIN, f));
98
99 if (ferror(f))
100 {
101 perror("Error reading file header");
102 return FALSE;
103 }
104
105 if (bytes != UNIV_PAGE_SIZE_MIN)
106 {
107 fprintf(stderr, "Error; Was not able to read the minimum page size ");
108 fprintf(stderr, "of %d bytes. Bytes read was %lu\n", UNIV_PAGE_SIZE_MIN, bytes);
109 return FALSE;
110 }
111
112 rewind(f);
113
114 flags = mach_read_from_4(buf + FIL_PAGE_DATA + FSP_SPACE_FLAGS);
115
116 /* srv_page_size is used by InnoDB code as UNIV_PAGE_SIZE */
117 srv_page_size = *logical_page_size = fsp_flags_get_page_size(flags);
118
119 /* fsp_flags_get_zip_size() will return zero if not compressed. */
120 *physical_page_size = fsp_flags_get_zip_size(flags);
121 *compressed = (*physical_page_size != 0);
122 if (*physical_page_size == 0)
123 *physical_page_size= *logical_page_size;
124
125 return TRUE;
126 }
127
128 #ifdef __WIN__
129 /***********************************************//*
130 @param [in] error error no. from the getLastError().
131
132 @retval error message corresponding to error no.
133 */
134 static
135 char*
win32_error_message(int error)136 win32_error_message(
137 int error)
138 {
139 static char err_msg[1024] = {'\0'};
140 FormatMessage(FORMAT_MESSAGE_FROM_SYSTEM,
141 NULL, error, MAKELANGID(LANG_NEUTRAL, SUBLANG_DEFAULT),
142 (LPTSTR)err_msg, sizeof(err_msg), NULL );
143
144 return (err_msg);
145 }
146 #endif /* __WIN__ */
147
148 /***********************************************//*
149 @param [in] name name of file.
150 @retval file pointer; file pointer is NULL when error occured.
151 */
152
153 FILE*
open_file(const char * name)154 open_file(
155 const char* name)
156 {
157 int fd; /* file descriptor. */
158 FILE* fil_in;
159 #ifdef __WIN__
160 HANDLE hFile; /* handle to open file. */
161 DWORD access; /* define access control */
162 int flags; /* define the mode for file
163 descriptor */
164
165 access = GENERIC_READ;
166 flags = _O_RDONLY | _O_BINARY;
167 hFile = CreateFile(
168 (LPCTSTR) name, access, 0L, NULL,
169 OPEN_EXISTING, NULL, NULL);
170
171 if (hFile == INVALID_HANDLE_VALUE) {
172 /* print the error message. */
173 fprintf(stderr, "Filename::%s %s\n",
174 win32_error_message(GetLastError()));
175
176 return (NULL);
177 }
178
179 /* get the file descriptor. */
180 fd= _open_osfhandle((intptr_t)hFile, flags);
181 #else /* __WIN__ */
182
183 int create_flag;
184 create_flag = O_RDONLY;
185
186 fd = open(name, create_flag);
187 if ( -1 == fd) {
188 perror("open");
189 return (NULL);
190 }
191
192 #endif /* __WIN__ */
193
194 fil_in = fdopen(fd, "rb");
195
196 return (fil_in);
197 }
198
199 /* command line argument to do page checks (that's it) */
200 /* another argument to specify page ranges... seek to right spot and go from there */
201
202 static struct my_option innochecksum_options[] =
203 {
204 {"help", '?', "Displays this help and exits.",
205 0, 0, 0, GET_NO_ARG, NO_ARG, 0, 0, 0, 0, 0, 0},
206 {"info", 'I', "Synonym for --help.",
207 0, 0, 0, GET_NO_ARG, NO_ARG, 0, 0, 0, 0, 0, 0},
208 {"version", 'V', "Displays version information and exits.",
209 0, 0, 0, GET_NO_ARG, NO_ARG, 0, 0, 0, 0, 0, 0},
210 {"verbose", 'v', "Verbose (prints progress every 5 seconds).",
211 &verbose, &verbose, 0, GET_BOOL, NO_ARG, 0, 0, 0, 0, 0, 0},
212 {"debug", 'd', "Debug mode (prints checksums for each page, implies verbose).",
213 &debug, &debug, 0, GET_BOOL, NO_ARG, 0, 0, 0, 0, 0, 0},
214 {"count", 'c', "Print the count of pages in the file.",
215 &just_count, &just_count, 0, GET_BOOL, NO_ARG, 0, 0, 0, 0, 0, 0},
216 {"format_info", 'f', "Display information about the file format and exit",
217 0, 0, 0, GET_NO_ARG, NO_ARG, 0, 0, 0, 0, 0, 0},
218 {"start_page", 's', "Start on this page number (0 based).",
219 &start_page, &start_page, 0, GET_ULL, REQUIRED_ARG,
220 0, 0, ULONGLONG_MAX, 0, 1, 0},
221 {"end_page", 'e', "End at this page number (0 based).",
222 &end_page, &end_page, 0, GET_ULL, REQUIRED_ARG,
223 0, 0, ULONGLONG_MAX, 0, 1, 0},
224 {"page", 'p', "Check only this page (0 based).",
225 &do_page, &do_page, 0, GET_ULL, REQUIRED_ARG,
226 0, 0, ULONGLONG_MAX, 0, 1, 0},
227
228 {0, 0, 0, 0, 0, 0, GET_NO_ARG, NO_ARG, 0, 0, 0, 0, 0, 0}
229 };
230
print_version(void)231 static void print_version(void)
232 {
233 printf("%s Ver %s, for %s (%s)\n",
234 my_progname, INNODB_VERSION_STR,
235 SYSTEM_TYPE, MACHINE_TYPE);
236 }
237
usage(void)238 static void usage(void)
239 {
240 print_version();
241 puts(ORACLE_WELCOME_COPYRIGHT_NOTICE("2000"));
242 printf("InnoDB offline file checksum utility.\n");
243 printf("Usage: %s [-c] [-s <start page>] [-e <end page>] [-p <page>] [-v] [-d] <filename>\n", my_progname);
244 my_print_help(innochecksum_options);
245 my_print_variables(innochecksum_options);
246 }
247
248 extern "C" my_bool
innochecksum_get_one_option(int optid,const struct my_option * opt MY_ATTRIBUTE ((unused)),char * argument MY_ATTRIBUTE ((unused)))249 innochecksum_get_one_option(
250 /*========================*/
251 int optid,
252 const struct my_option *opt MY_ATTRIBUTE((unused)),
253 char *argument MY_ATTRIBUTE((unused)))
254 {
255 switch (optid) {
256 case 'd':
257 verbose=1; /* debug implies verbose... */
258 break;
259 case 'e':
260 use_end_page= 1;
261 break;
262 case 'f':
263 display_format= 1;
264 break;
265 case 'p':
266 end_page= start_page= do_page;
267 use_end_page= 1;
268 do_one_page= 1;
269 break;
270 case 'V':
271 print_version();
272 exit(0);
273 break;
274 case 'I':
275 case '?':
276 usage();
277 exit(0);
278 break;
279 }
280 return 0;
281 }
282
get_options(int * argc,char *** argv)283 static int get_options(
284 /*===================*/
285 int *argc,
286 char ***argv)
287 {
288 int ho_error;
289
290 if ((ho_error=handle_options(argc, argv, innochecksum_options, innochecksum_get_one_option)))
291 exit(ho_error);
292
293 /* The next arg must be the filename */
294 if (!*argc)
295 {
296 usage();
297 return 1;
298 }
299 return 0;
300 } /* get_options */
301
302 static
303 void
display_format_info(uchar * page)304 display_format_info(uchar *page)
305 {
306 ulint page_type;
307 ulint flags;
308
309 /* Read page type. Pre-5.1.7 InnoDB always have zero in FIL_PAGE_TYPE for the
310 first page, later versions initialize it to FIL_PAGE_TYPE_FSP_HDR. */
311 page_type= mach_read_from_2(page + FIL_PAGE_TYPE);
312
313 /* Read FSP flags from the page header. */
314 flags = mach_read_from_4(page + FSP_HEADER_OFFSET + FSP_SPACE_FLAGS);
315
316 if (!page_type)
317 {
318 printf("Detected file format: Antelope (pre-5.1.7).\n");
319 if (flags != 0) {
320 printf("But FSP_SPACE_FLAGS is non-zero: %lu. Corrupted tablespace?\n",
321 flags);
322 }
323 }
324 else if (page_type == FIL_PAGE_TYPE_FSP_HDR)
325 {
326 ulint zip_size = fsp_flags_get_zip_size(flags);
327
328 if (!flags)
329 {
330 printf("Detected file format: Antelope (5.1.7 or newer).\n");
331 }
332 else if (DICT_TF_HAS_ATOMIC_BLOBS(flags))
333 {
334 printf("Detected file format: Barracuda ");
335 if (!zip_size)
336 printf("(not compressed).\n");
337 else
338 printf("(compressed with KEY_BLOCK_SIZE=%lu).\n", zip_size);
339 }
340 else
341 printf("Unknown file format flags: %lu\n", flags);
342 }
343 else
344 {
345 printf("Bogus FIL_PAGE_TYPE value: %lu. Cannot detect the file format.\n",
346 page_type);
347 }
348 }
349
main(int argc,char ** argv)350 int main(int argc, char **argv)
351 {
352 FILE* f; /* our input file */
353 char* filename; /* our input filename. */
354 unsigned char buf[UNIV_PAGE_SIZE_MAX]; /* Buffer to store pages read */
355 ulong bytes; /* bytes read count */
356 ulint ct; /* current page number (0 based) */
357 time_t now; /* current time */
358 time_t lastt; /* last time */
359 ulint oldcsum, oldcsumfield, csum, csumfield, crc32, logseq, logseqfield;
360
361 /* ulints for checksum storage */
362 /* stat, to get file size. */
363 #ifdef __WIN__
364 struct _stat64 st;
365 #else
366 struct stat st;
367 #endif
368 unsigned long long int size; /* size of file (has to be 64 bits) */
369 ulint pages; /* number of pages in file */
370 off_t offset= 0;
371 bool compressed;
372
373 printf("InnoDB offline file checksum utility.\n");
374
375 ut_crc32_init();
376
377 MY_INIT(argv[0]);
378
379 if (get_options(&argc,&argv))
380 exit(1);
381
382 if (verbose)
383 my_print_variables(innochecksum_options);
384
385 /* The file name is not optional */
386 filename = *argv;
387 if (*filename == '\0')
388 {
389 fprintf(stderr, "Error; File name missing\n");
390 return 1;
391 }
392
393 /* stat the file to get size and page count */
394 #ifdef __WIN__
395 if (_stat64(filename, &st))
396 #else
397 if (stat(filename, &st))
398 #endif
399 {
400 fprintf(stderr, "Error; %s cannot be found\n", filename);
401 return 1;
402 }
403 size= st.st_size;
404
405 /* Open the file for reading */
406 f= open_file(filename);
407 if (f == NULL) {
408 return 1;
409 }
410
411 if (!get_page_size(f, buf, &logical_page_size, &physical_page_size,
412 &compressed))
413 {
414 return 1;
415 }
416
417 if (compressed)
418 {
419 printf("Table is compressed\n");
420 printf("Key block size is %lu\n", physical_page_size);
421 }
422 else
423 {
424 printf("Table is uncompressed\n");
425 printf("Page size is %lu\n", physical_page_size);
426 }
427
428 pages= (ulint) (size / physical_page_size);
429
430 if (just_count)
431 {
432 if (verbose)
433 printf("Number of pages: ");
434 printf("%lu\n", pages);
435 return 0;
436 }
437 else if (verbose)
438 {
439 printf("file %s = %llu bytes (%lu pages)...\n", filename, size, pages);
440 if (do_one_page)
441 printf("InnoChecksum; checking page %llu\n", do_page);
442 else
443 printf("InnoChecksum; checking pages in range %llu to %llu\n", start_page, use_end_page ? end_page : (pages - 1));
444 }
445
446 /* seek to the necessary position, ignore with -f as we only need to read the
447 first page */
448 if (start_page && !display_format)
449 {
450
451 offset= (off_t)start_page * (off_t)physical_page_size;
452
453 #ifdef __WIN__
454 if (_fseeki64(f, offset, SEEK_SET)) {
455 #else
456 if (fseeko(f, offset, SEEK_SET)) {
457 #endif /* __WIN__ */
458 perror("Error; Unable to seek to necessary offset");
459 return 1;
460 }
461 }
462
463 /* main checksumming loop */
464 ct= start_page;
465 lastt= 0;
466 while (!feof(f))
467 {
468 bytes= ulong(fread(buf, 1, physical_page_size, f));
469 if (!bytes && feof(f))
470 return 0;
471
472 if (ferror(f))
473 {
474 fprintf(stderr, "Error reading %lu bytes", physical_page_size);
475 perror(" ");
476 return 1;
477 }
478 if (bytes != physical_page_size)
479 {
480 fprintf(stderr, "Error; bytes read (%lu) doesn't match page size (%lu)\n", bytes, physical_page_size);
481 return 1;
482 }
483
484 if (display_format)
485 {
486 /* for -f, analyze only the first page and exit */
487 display_format_info(buf);
488 return 0;
489 }
490
491 if (compressed) {
492 /* compressed pages */
493 if (!page_zip_verify_checksum(buf, physical_page_size)) {
494 fprintf(stderr, "Fail; page %lu invalid (fails compressed page checksum).\n", ct);
495 return 1;
496 }
497 } else {
498 /* check the "stored log sequence numbers" */
499 logseq= mach_read_from_4(buf + FIL_PAGE_LSN + 4);
500 logseqfield= mach_read_from_4(buf + logical_page_size - FIL_PAGE_END_LSN_OLD_CHKSUM + 4);
501 if (debug)
502 printf("page %lu: log sequence number: first = %lu; second = %lu\n", ct, logseq, logseqfield);
503 if (logseq != logseqfield)
504 {
505 fprintf(stderr, "Fail; page %lu invalid (fails log sequence number check)\n", ct);
506 return 1;
507 }
508
509 /* check old method of checksumming */
510 oldcsum= buf_calc_page_old_checksum(buf);
511 oldcsumfield= mach_read_from_4(buf + logical_page_size - FIL_PAGE_END_LSN_OLD_CHKSUM);
512 if (debug)
513 printf("page %lu: old style: calculated = %lu; recorded = %lu\n", ct, oldcsum, oldcsumfield);
514 if (oldcsumfield != mach_read_from_4(buf + FIL_PAGE_LSN) && oldcsumfield != oldcsum)
515 {
516 fprintf(stderr, "Fail; page %lu invalid (fails old style checksum)\n", ct);
517 return 1;
518 }
519
520 /* now check the new method */
521 csum= buf_calc_page_new_checksum(buf);
522 crc32= buf_calc_page_crc32(buf);
523 csumfield= mach_read_from_4(buf + FIL_PAGE_SPACE_OR_CHKSUM);
524 if (debug)
525 printf("page %lu: new style: calculated = %lu; crc32 = %lu; recorded = %lu\n",
526 ct, csum, crc32, csumfield);
527 if (csumfield != 0 && crc32 != csumfield && csum != csumfield)
528 {
529 fprintf(stderr, "Fail; page %lu invalid (fails innodb and crc32 checksum)\n", ct);
530 return 1;
531 }
532 }
533
534 /* end if this was the last page we were supposed to check */
535 if (use_end_page && (ct >= end_page))
536 return 0;
537
538 /* do counter increase and progress printing */
539 ct++;
540 if (verbose)
541 {
542 if (ct % 64 == 0)
543 {
544 now= time(0);
545 if (!lastt) lastt= now;
546 if (now - lastt >= 1)
547 {
548 printf("page %lu okay: %.3f%% done\n", (ct - 1), (float) ct / pages * 100);
549 lastt= now;
550 }
551 }
552 }
553 }
554 return 0;
555 }
556
557