1 /*-
2 * Copyright (c) 1996, 2020 Oracle and/or its affiliates. All rights reserved.
3 *
4 * See the file LICENSE for license information.
5 *
6 * $Id: db_upg.c,v a79fcece62a1 2016/06/06 14:15:33 yong $
7 */
8
9 #include "db_config.h"
10
11 #include "db_int.h"
12 #include "dbinc/db_page.h"
13 #include "dbinc/db_swap.h"
14 #include "dbinc/btree.h"
15 #include "dbinc/hash.h"
16 #include "dbinc/heap.h"
17 #include "dbinc/qam.h"
18
19 /*
20 * __db_upgrade_pp --
21 * DB->upgrade pre/post processing.
22 *
23 * PUBLIC: int __db_upgrade_pp __P((DB *, const char *, u_int32_t));
24 */
25 int
__db_upgrade_pp(dbp,fname,flags)26 __db_upgrade_pp(dbp, fname, flags)
27 DB *dbp;
28 const char *fname;
29 u_int32_t flags;
30 {
31 #ifdef HAVE_UPGRADE_SUPPORT
32 DB_THREAD_INFO *ip;
33 ENV *env;
34 int ret;
35
36 env = dbp->env;
37
38 if ((ret = __db_fchk(env, "DB->upgrade", flags, DB_DUPSORT)) != 0)
39 return (ret);
40
41 ENV_ENTER(env, ip);
42 ret = __db_upgrade(dbp, fname, flags);
43
44 #ifdef HAVE_SLICES
45 if (ret == 0)
46 ret = __db_slice_process(dbp, fname, flags,
47 __db_upgrade_pp, "db_upgrade");
48 #endif
49
50 ENV_LEAVE(env, ip);
51 return (ret);
52 #else
53 COMPQUIET(dbp, NULL);
54 COMPQUIET(fname, NULL);
55 COMPQUIET(flags, 0);
56
57 __db_errx(dbp->env, DB_STR("0665", "upgrade not supported"));
58 return (EINVAL);
59 #endif
60 }
61
62 #ifdef HAVE_UPGRADE_SUPPORT
63 static int (* const func_31_list[P_PAGETYPE_MAX])
64 __P((DB *, char *, u_int32_t, DB_FH *, PAGE *, int *)) = {
65 NULL, /* P_INVALID */
66 NULL, /* __P_DUPLICATE */
67 __ham_31_hash, /* P_HASH_UNSORTED */
68 NULL, /* P_IBTREE */
69 NULL, /* P_IRECNO */
70 __bam_31_lbtree, /* P_LBTREE */
71 NULL, /* P_LRECNO */
72 NULL, /* P_OVERFLOW */
73 __ham_31_hashmeta, /* P_HASHMETA */
74 __bam_31_btreemeta, /* P_BTREEMETA */
75 NULL, /* P_QAMMETA */
76 NULL, /* P_QAMDATA */
77 NULL, /* P_LDUP */
78 NULL, /* P_HASH */
79 NULL, /* P_HEAPMETA */
80 NULL, /* P_HEAP */
81 NULL, /* P_IHEAP */
82 };
83
84 static int (* const func_46_list[P_PAGETYPE_MAX])
85 __P((DB *, char *, u_int32_t, DB_FH *, PAGE *, int *)) = {
86 NULL, /* P_INVALID */
87 NULL, /* __P_DUPLICATE */
88 __ham_46_hash, /* P_HASH_UNSORTED */
89 NULL, /* P_IBTREE */
90 NULL, /* P_IRECNO */
91 NULL, /* P_LBTREE */
92 NULL, /* P_LRECNO */
93 NULL, /* P_OVERFLOW */
94 __ham_46_hashmeta, /* P_HASHMETA */
95 NULL, /* P_BTREEMETA */
96 NULL, /* P_QAMMETA */
97 NULL, /* P_QAMDATA */
98 NULL, /* P_LDUP */
99 NULL, /* P_HASH */
100 NULL, /* P_HEAPMETA */
101 NULL, /* P_HEAP */
102 NULL, /* P_IHEAP */
103 };
104
105 static int (* const func_60_list[P_PAGETYPE_MAX])
106 __P((DB *, char *, u_int32_t, DB_FH *, PAGE *, int *)) = {
107 NULL, /* P_INVALID */
108 NULL, /* __P_DUPLICATE */
109 NULL, /* P_HASH_UNSORTED */
110 NULL, /* P_IBTREE */
111 NULL, /* P_IRECNO */
112 __bam_60_lbtree, /* P_LBTREE */
113 NULL, /* P_LRECNO */
114 NULL, /* P_OVERFLOW */
115 __ham_60_hashmeta, /* P_HASHMETA */
116 __bam_60_btreemeta, /* P_BTREEMETA */
117 NULL, /* P_QAMMETA */
118 NULL, /* P_QAMDATA */
119 NULL, /* P_LDUP */
120 __ham_60_hash, /* P_HASH */
121 __heap_60_heapmeta, /* P_HEAPMETA */
122 __heap_60_heap, /* P_HEAP */
123 NULL, /* P_IHEAP */
124 };
125
126 static int __db_set_lastpgno __P((DB *, char *, DB_FH *));
127
128 /*
129 * __db_upgrade --
130 * Upgrade an existing database.
131 *
132 * PUBLIC: int __db_upgrade __P((DB *, const char *, u_int32_t));
133 */
134 int
__db_upgrade(dbp,fname,flags)135 __db_upgrade(dbp, fname, flags)
136 DB *dbp;
137 const char *fname;
138 u_int32_t flags;
139 {
140 DBMETA *meta, *dbmeta;
141 DB_FH *fhp;
142 ENV *env;
143 size_t n;
144 int ret, t_ret, use_mp_open;
145 u_int8_t mbuf[256], tmpflags;
146 char *real_name;
147
148 use_mp_open = 0;
149 env = dbp->env;
150 fhp = NULL;
151
152 /* Get the real backing file name. */
153 if ((ret = __db_appname(env,
154 DB_APP_DATA, fname, NULL, &real_name)) != 0)
155 return (ret);
156
157 /* Open the file. */
158 if ((ret = __os_open(env, real_name, 0, 0, 0, &fhp)) != 0) {
159 __db_err(env, ret, "%s", real_name);
160 return (ret);
161 }
162
163 /* Initialize the feedback. */
164 if (dbp->db_feedback != NULL)
165 dbp->db_feedback(dbp, DB_UPGRADE, 0);
166
167 /*
168 * Read the metadata page. We read 256 bytes, which is larger than
169 * any access method's metadata page and smaller than any disk sector.
170 */
171 if ((ret = __os_read(env, fhp, mbuf, sizeof(mbuf), &n)) != 0)
172 goto err;
173
174 dbmeta = (DBMETA *)mbuf;
175 switch (dbmeta->magic) {
176 case DB_BTREEMAGIC:
177 switch (dbmeta->version) {
178 case 6:
179 /*
180 * Before V7 not all pages had page types, so we do the
181 * single meta-data page by hand.
182 */
183 if ((ret =
184 __bam_30_btreemeta(dbp, real_name, mbuf)) != 0)
185 goto err;
186 if ((ret = __os_seek(env, fhp, 0, 0, 0)) != 0)
187 goto err;
188 if ((ret = __os_write(env, fhp, mbuf, 256, &n)) != 0)
189 goto err;
190 /* FALLTHROUGH */
191 case 7:
192 /*
193 * We need the page size to do more. Rip it out of
194 * the meta-data page.
195 */
196 memcpy(&dbp->pgsize, mbuf + 20, sizeof(u_int32_t));
197
198 if ((ret = __db_page_pass(dbp, real_name, flags,
199 func_31_list, fhp, DB_UPGRADE)) != 0)
200 goto err;
201 /* FALLTHROUGH */
202 case 8:
203 if ((ret =
204 __db_set_lastpgno(dbp, real_name, fhp)) != 0)
205 goto err;
206 /* FALLTHROUGH */
207 case 9:
208 /*
209 * Various blob ids and size use two u_int32_t values
210 * to represent 64 bit integers in early 6.0. Change
211 * those values to 64 bit integers.
212 */
213 /*
214 * Read the encrypt_alg and chksum fields from the
215 * metadata page.
216 */
217 meta = (DBMETA *)mbuf;
218 if (FLD_ISSET(meta->metaflags, DBMETA_CHKSUM))
219 F_SET(dbp, DB_AM_CHKSUM);
220 if (meta->encrypt_alg != 0) {
221 if (!CRYPTO_ON(dbp->env)) {
222 ret = USR_ERR(env, EINVAL);
223 __db_errx(env, DB_STR("0667",
224 "Attempt to upgrade an encrypted database without providing a password."));
225 goto err;
226 }
227 F_SET(dbp, DB_AM_ENCRYPT);
228 }
229 memcpy(&dbp->pgsize,
230 &meta->pagesize, sizeof(u_int32_t));
231 if ((ret = __db_page_pass(dbp, real_name, flags,
232 func_60_list, fhp, DB_UPGRADE)) != 0)
233 goto err;
234 /* FALLTHROUGH */
235 case 10:
236 break;
237 default:
238 __db_errx(env, DB_STR_A("1009",
239 "%s: unsupported btree version: %lu", "%s %lu"),
240 real_name, (u_long)dbmeta->version);
241 ret = DB_OLD_VERSION;
242 goto err;
243 }
244 break;
245 case DB_HASHMAGIC:
246 switch (dbmeta->version) {
247 case 4:
248 case 5:
249 /*
250 * Before V6 not all pages had page types, so we do the
251 * single meta-data page by hand.
252 */
253 if ((ret =
254 __ham_30_hashmeta(dbp, real_name, mbuf)) != 0)
255 goto err;
256 if ((ret = __os_seek(env, fhp, 0, 0, 0)) != 0)
257 goto err;
258 if ((ret = __os_write(env, fhp, mbuf, 256, &n)) != 0)
259 goto err;
260
261 /*
262 * Before V6, we created hash pages one by one as they
263 * were needed, using hashhdr.ovfl_point to reserve
264 * a block of page numbers for them. A consequence
265 * of this was that, if no overflow pages had been
266 * created, the current doubling might extend past
267 * the end of the database file.
268 *
269 * In DB 3.X, we now create all the hash pages
270 * belonging to a doubling atomically; it's not
271 * safe to just save them for later, because when
272 * we create an overflow page we'll just create
273 * a new last page (whatever that may be). Grow
274 * the database to the end of the current doubling.
275 */
276 if ((ret =
277 __ham_30_sizefix(dbp, fhp, real_name, mbuf)) != 0)
278 goto err;
279 /* FALLTHROUGH */
280 case 6:
281 /*
282 * We need the page size to do more. Rip it out of
283 * the meta-data page.
284 */
285 memcpy(&dbp->pgsize, mbuf + 20, sizeof(u_int32_t));
286
287 if ((ret = __db_page_pass(dbp, real_name, flags,
288 func_31_list, fhp, DB_UPGRADE)) != 0)
289 goto err;
290 /* FALLTHROUGH */
291 case 7:
292 if ((ret =
293 __db_set_lastpgno(dbp, real_name, fhp)) != 0)
294 goto err;
295 /* FALLTHROUGH */
296 case 8:
297 /*
298 * Any upgrade that has proceeded this far has metadata
299 * pages compatible with hash version 8 metadata pages,
300 * so casting mbuf to a dbmeta is safe.
301 * If a newer revision moves the pagesize, checksum or
302 * encrypt_alg flags in the metadata, then the
303 * extraction of the fields will need to use hard coded
304 * offsets.
305 */
306 meta = (DBMETA*)mbuf;
307 /*
308 * We need the page size to do more. Extract it from
309 * the meta-data page.
310 */
311 memcpy(&dbp->pgsize, &meta->pagesize,
312 sizeof(u_int32_t));
313 /*
314 * Rip out metadata and encrypt_alg fields from the
315 * metadata page. So the upgrade can know how big
316 * the page metadata pre-amble is. Any upgrade that has
317 * proceeded this far has metadata pages compatible
318 * with hash version 8 metadata pages, so extracting
319 * the fields is safe.
320 */
321 memcpy(&tmpflags, &meta->metaflags, sizeof(u_int8_t));
322 if (FLD_ISSET(tmpflags, DBMETA_CHKSUM))
323 F_SET(dbp, DB_AM_CHKSUM);
324 memcpy(&tmpflags, &meta->encrypt_alg, sizeof(u_int8_t));
325 if (tmpflags != 0) {
326 if (!CRYPTO_ON(dbp->env)) {
327 ret = USR_ERR(env, EINVAL);
328 __db_errx(env, DB_STR("0667",
329 "Attempt to upgrade an encrypted database without providing a password."));
330 goto err;
331 }
332 F_SET(dbp, DB_AM_ENCRYPT);
333 }
334
335 /*
336 * This is ugly. It is necessary to have a usable
337 * mpool in the dbp to upgrade from an unsorted
338 * to a sorted hash database. The mpool file is used
339 * to resolve offpage key items, which are needed to
340 * determine sort order. Having mpool open and access
341 * the file does not affect the page pass, since the
342 * page pass only updates DB_HASH_UNSORTED pages
343 * in-place, and the mpool file is only used to read
344 * OFFPAGE items.
345 * XXX DB_HASH_UNSORTED no longer exists. since ~db-4.4.
346 * Is this code, and the lesser versions above, needed?
347 */
348 use_mp_open = 1;
349 if ((ret = __os_closehandle(env, fhp)) != 0)
350 return (ret);
351 dbp->type = DB_HASH;
352 if ((ret = __env_mpool(dbp, fname,
353 DB_AM_NOT_DURABLE | DB_AM_VERIFYING)) != 0)
354 return (ret);
355 fhp = dbp->mpf->fhp;
356
357 /* Do the actual conversion pass. */
358 if ((ret = __db_page_pass(dbp, real_name, flags,
359 func_46_list, fhp, DB_UPGRADE)) != 0)
360 goto err;
361
362 /* FALLTHROUGH */
363 case 9:
364 /*
365 * Various blob ids and size use two u_int32_t values
366 * to represent 64 bit integers in early 6.0. Change
367 * those values to 64 bit integers.
368 */
369 meta = (DBMETA*)mbuf;
370 memcpy(&dbp->pgsize,
371 &meta->pagesize, sizeof(u_int32_t));
372 /*
373 * Read the encrypt_alg and chksum fields from the
374 * metadata page.
375 */
376 if (FLD_ISSET(meta->metaflags, DBMETA_CHKSUM))
377 F_SET(dbp, DB_AM_CHKSUM);
378 if (meta->encrypt_alg != 0) {
379 if (!CRYPTO_ON(dbp->env)) {
380 ret = USR_ERR(env, EINVAL);
381 __db_errx(env, DB_STR("0667",
382 "Attempt to upgrade an encrypted database without providing a password."));
383 goto err;
384 }
385 F_SET(dbp, DB_AM_ENCRYPT);
386 }
387 if ((ret = __db_page_pass(dbp, real_name, flags,
388 func_60_list, fhp, DB_UPGRADE)) != 0)
389 goto err;
390 /* FALLTHROUGH */
391 case 10:
392 break;
393 default:
394 __db_errx(env, DB_STR_A("1126",
395 "%s: unsupported hash version: %lu", "%s %lu"),
396 real_name, (u_long)dbmeta->version);
397 ret = DB_OLD_VERSION;
398 goto err;
399 }
400 break;
401 case DB_HEAPMAGIC:
402 switch (dbmeta->version) {
403 case 1:
404 /*
405 * Various blob ids and size use two u_int32_t values
406 * to represent 64 bit integers in early 6.0. Change
407 * those values to 64 bit integers.
408 */
409 meta = (DBMETA*)mbuf;
410 memcpy(&dbp->pgsize,
411 &meta->pagesize, sizeof(u_int32_t));
412 /*
413 * Read the encrypt_alg and chksum fields from the
414 * metadata page.
415 */
416 if (FLD_ISSET(meta->metaflags, DBMETA_CHKSUM))
417 F_SET(dbp, DB_AM_CHKSUM);
418 if (meta->encrypt_alg != 0) {
419 if (!CRYPTO_ON(dbp->env)) {
420 ret = USR_ERR(env, EINVAL);
421 __db_errx(env, DB_STR("0667",
422 "Attempt to upgrade an encrypted database without providing a password."));
423 goto err;
424 }
425 F_SET(dbp, DB_AM_ENCRYPT);
426 }
427 if ((ret = __db_page_pass(dbp, real_name, flags,
428 func_60_list, fhp, DB_UPGRADE)) != 0)
429 goto err;
430 /* FALLTHROUGH */
431 case 2:
432 break;
433 default:
434 __db_errx(env, DB_STR_A("0776",
435 "%s: unsupported heap version: %lu",
436 "%s %lu"), real_name,
437 (u_long)dbmeta->version);
438 ret = DB_OLD_VERSION;
439 goto err;
440 }
441 break;
442 case DB_QAMMAGIC:
443 switch (dbmeta->version) {
444 case 1:
445 /*
446 * If we're in a Queue database, the only page that
447 * needs upgrading is the meta-database page, don't
448 * bother with a full pass.
449 */
450 if ((ret = __qam_31_qammeta(dbp, real_name, mbuf)) != 0)
451 return (ret);
452 /* FALLTHROUGH */
453 case 2:
454 if ((ret = __qam_32_qammeta(dbp, real_name, mbuf)) != 0)
455 return (ret);
456 if ((ret = __os_seek(env, fhp, 0, 0, 0)) != 0)
457 goto err;
458 if ((ret = __os_write(env, fhp, mbuf, 256, &n)) != 0)
459 goto err;
460 /* FALLTHROUGH */
461 case 3:
462 case 4:
463 break;
464 default:
465 __db_errx(env, DB_STR_A("0669",
466 "%s: unsupported queue version: %lu",
467 "%s %lu"), real_name,
468 (u_long)dbmeta->version);
469 ret = DB_OLD_VERSION;
470 goto err;
471 }
472 break;
473 default:
474 M_32_SWAP(dbmeta->magic);
475 switch (dbmeta->magic) {
476 case DB_BTREEMAGIC:
477 case DB_HASHMAGIC:
478 case DB_HEAPMAGIC:
479 case DB_QAMMAGIC:
480 __db_errx(env, DB_STR_A("0670",
481 "%s: DB->upgrade only supported on native byte-order systems",
482 "%s"), real_name);
483 break;
484 default:
485 __db_errx(env, DB_STR_A("0671",
486 "%s: unrecognized file type", "%s"), real_name);
487 break;
488 }
489 ret = USR_ERR(env, EINVAL);
490 goto err;
491 }
492
493 ret = __os_fsync(env, fhp);
494
495 /*
496 * If mp_open was used, then rely on the database close to clean up
497 * any file handles.
498 */
499 err: if (use_mp_open == 0 && fhp != NULL &&
500 (t_ret = __os_closehandle(env, fhp)) != 0 && ret == 0)
501 ret = t_ret;
502 __os_free(env, real_name);
503
504 /* We're done. */
505 if (dbp->db_feedback != NULL)
506 dbp->db_feedback(dbp, DB_UPGRADE, 100);
507
508 return (ret);
509 }
510
511 /*
512 * __db_set_lastpgno --
513 * Update the meta->last_pgno field.
514 *
515 * Code assumes that we do not have checksums/crypto on the page.
516 */
517 static int
__db_set_lastpgno(dbp,real_name,fhp)518 __db_set_lastpgno(dbp, real_name, fhp)
519 DB *dbp;
520 char *real_name;
521 DB_FH *fhp;
522 {
523 DBMETA meta;
524 ENV *env;
525 int ret;
526 size_t n;
527
528 env = dbp->env;
529 if ((ret = __os_seek(env, fhp, 0, 0, 0)) != 0)
530 return (ret);
531 if ((ret = __os_read(env, fhp, &meta, sizeof(meta), &n)) != 0)
532 return (ret);
533 dbp->pgsize = meta.pagesize;
534 if ((ret = __db_lastpgno(dbp, real_name, fhp, &meta.last_pgno)) != 0)
535 return (ret);
536 if ((ret = __os_seek(env, fhp, 0, 0, 0)) != 0)
537 return (ret);
538 if ((ret = __os_write(env, fhp, &meta, sizeof(meta), &n)) != 0)
539 return (ret);
540
541 return (0);
542 }
543 #endif /* HAVE_UPGRADE_SUPPORT */
544
545 /*
546 * __db_page_pass --
547 * Walk the pages of the database, doing whatever needs it.
548 *
549 * PUBLIC: int __db_page_pass __P((DB *, char *, u_int32_t, int (* const [])
550 * PUBLIC: (DB *, char *, u_int32_t, DB_FH *, PAGE *, int *), DB_FH *,
551 * PUBLIC: int));
552 */
553 int
__db_page_pass(dbp,real_name,flags,fl,fhp,feedback_code)554 __db_page_pass(dbp, real_name, flags, fl, fhp, feedback_code)
555 DB *dbp;
556 char *real_name;
557 u_int32_t flags;
558 int (* const fl[P_PAGETYPE_MAX])
559 __P((DB *, char *, u_int32_t, DB_FH *, PAGE *, int *));
560 DB_FH *fhp;
561 int feedback_code;
562 {
563 ENV *env;
564 PAGE *page;
565 db_pgno_t i, pgno_last;
566 size_t n;
567 int dirty, ret;
568
569 env = dbp->env;
570
571 /* Determine the last page of the file. */
572 if ((ret = __db_lastpgno(dbp, real_name, fhp, &pgno_last)) != 0)
573 return (ret);
574
575 /* Allocate memory for a single page. */
576 if ((ret = __os_malloc(env, dbp->pgsize, &page)) != 0)
577 return (ret);
578
579 /* Walk the file, calling the underlying conversion functions. */
580 for (i = 0; i < pgno_last; ++i) {
581 if (dbp->db_feedback != NULL)
582 dbp->db_feedback(
583 dbp, feedback_code, (int)((i * 100)/pgno_last));
584 if ((ret = __os_seek(env, fhp, i, dbp->pgsize, 0)) != 0)
585 break;
586 if ((ret = __os_read(env, fhp, page, dbp->pgsize, &n)) != 0)
587 break;
588 dirty = 0;
589 /* Always decrypt the page. */
590 if ((ret = __db_decrypt_pg(env, dbp, page)) != 0)
591 break;
592 if (fl[TYPE(page)] != NULL && (ret = fl[TYPE(page)]
593 (dbp, real_name, flags, fhp, page, &dirty)) != 0)
594 break;
595 if (dirty) {
596 if ((ret = __db_encrypt_and_checksum_pg(
597 env, dbp, page)) != 0)
598 break;
599 if ((ret =
600 __os_seek(env, fhp, i, dbp->pgsize, 0)) != 0)
601 break;
602 if ((ret = __os_write(env,
603 fhp, page, dbp->pgsize, &n)) != 0)
604 break;
605 }
606 }
607
608 __os_free(dbp->env, page);
609 return (ret);
610 }
611
612 /*
613 * __db_lastpgno --
614 * Return the current last page number of the file.
615 *
616 * PUBLIC: int __db_lastpgno __P((DB *, char *, DB_FH *, db_pgno_t *));
617 */
618 int
__db_lastpgno(dbp,real_name,fhp,pgno_lastp)619 __db_lastpgno(dbp, real_name, fhp, pgno_lastp)
620 DB *dbp;
621 char *real_name;
622 DB_FH *fhp;
623 db_pgno_t *pgno_lastp;
624 {
625 ENV *env;
626 db_pgno_t pgno_last;
627 u_int32_t mbytes, bytes;
628 int ret;
629
630 env = dbp->env;
631
632 if ((ret = __os_ioinfo(env,
633 real_name, fhp, &mbytes, &bytes, NULL)) != 0) {
634 __db_err(env, ret, "%s", real_name);
635 return (ret);
636 }
637
638 /* Page sizes have to be a power-of-two. */
639 if (bytes % dbp->pgsize != 0) {
640 __db_errx(env, DB_STR_A("0672",
641 "%s: file size not a multiple of the pagesize", "%s"),
642 real_name);
643 return (EINVAL);
644 }
645 pgno_last = mbytes * (MEGABYTE / dbp->pgsize);
646 pgno_last += bytes / dbp->pgsize;
647
648 *pgno_lastp = pgno_last;
649 return (0);
650 }
651