1 /*- 2 * See the file LICENSE for redistribution information. 3 * 4 * Copyright (c) 2002, 2014 Oracle and/or its affiliates. All rights reserved. 5 * 6 */ 7 8 package com.sleepycat.je.cleaner; 9 10 import java.util.ArrayList; 11 import java.util.Arrays; 12 import java.util.Collection; 13 import java.util.Collections; 14 import java.util.Iterator; 15 import java.util.List; 16 import java.util.Set; 17 import java.util.SortedMap; 18 import java.util.TreeMap; 19 import java.util.logging.Level; 20 import java.util.logging.Logger; 21 22 import com.sleepycat.je.CacheMode; 23 import com.sleepycat.je.DatabaseConfig; 24 import com.sleepycat.je.DatabaseEntry; 25 import com.sleepycat.je.DatabaseException; 26 import com.sleepycat.je.DbInternal; 27 import com.sleepycat.je.EnvironmentFailureException; 28 import com.sleepycat.je.OperationStatus; 29 import com.sleepycat.je.TransactionConfig; 30 import com.sleepycat.je.dbi.CursorImpl; 31 import com.sleepycat.je.dbi.DatabaseId; 32 import com.sleepycat.je.dbi.DatabaseImpl; 33 import com.sleepycat.je.dbi.DbTree; 34 import com.sleepycat.je.dbi.DbType; 35 import com.sleepycat.je.dbi.EnvironmentImpl; 36 import com.sleepycat.je.dbi.MemoryBudget; 37 import com.sleepycat.je.dbi.StartupTracker; 38 import com.sleepycat.je.log.LogManager; 39 import com.sleepycat.je.log.ReplicationContext; 40 import com.sleepycat.je.log.entry.LNLogEntry; 41 import com.sleepycat.je.tree.BIN; 42 import com.sleepycat.je.tree.FileSummaryLN; 43 import com.sleepycat.je.tree.MapLN; 44 import com.sleepycat.je.tree.Tree; 45 import com.sleepycat.je.tree.TreeLocation; 46 import com.sleepycat.je.txn.BasicLocker; 47 import com.sleepycat.je.txn.LockType; 48 import com.sleepycat.je.txn.Locker; 49 import com.sleepycat.je.txn.Txn; 50 import com.sleepycat.je.utilint.DbLsn; 51 import com.sleepycat.je.utilint.LoggerUtils; 52 53 /** 54 * The UP tracks utilization summary information for all log files. 55 * 56 * <p>Unlike the UtilizationTracker, the UP is not accessed under the log write 57 * latch and is instead synchronized on itself for protecting the cache. It is 58 * not accessed during the primary data access path, except for when flushing 59 * (writing) file summary LNs. This occurs in the following cases: 60 * <ol> 61 * <li>The summary information is flushed at the end of a checkpoint. This 62 * allows tracking to occur in memory in between checkpoints, and replayed 63 * during recovery.</li> 64 * <li>When committing the truncateDatabase and removeDatabase operations, the 65 * summary information is flushed because detail tracking for those operations 66 * is not replayed during recovery</li> 67 * <li>The evictor will ask the UtilizationTracker to flush the largest summary 68 * if the memory taken by the tracker exeeds its budget.</li> 69 * </ol> 70 * 71 * <p>The cache is populated by the RecoveryManager just before performing the 72 * initial checkpoint. The UP must be open and populated in order to respond 73 * to requests to flush summaries and to evict tracked detail, even if the 74 * cleaner is disabled.</p> 75 * 76 * <p>WARNING: While synchronized on this object, eviction is not permitted. 77 * If it were, this could cause deadlocks because the order of locking would be 78 * the UP object and then the evictor. During normal eviction the order is to 79 * first lock the evictor and then the UP, when evicting tracked detail.</p> 80 * 81 * <p>The methods in this class synchronize to protect the cached summary 82 * information. Some methods also access the UP database. However, because 83 * eviction must not occur while synchronized, UP database access is not 84 * performed while synchronized except in one case: when inserting a new 85 * summary record. In that case we disallow eviction during the database 86 * operation.</p> 87 */ 88 public class UtilizationProfile { 89 90 private final EnvironmentImpl env; 91 private final UtilizationTracker tracker; 92 private DatabaseImpl fileSummaryDb; 93 private SortedMap<Long, FileSummary> fileSummaryMap; 94 private boolean cachePopulated; 95 private final Logger logger; 96 97 /** 98 * Creates an empty UP. 99 */ UtilizationProfile(EnvironmentImpl env, UtilizationTracker tracker)100 public UtilizationProfile(EnvironmentImpl env, 101 UtilizationTracker tracker) { 102 this.env = env; 103 this.tracker = tracker; 104 fileSummaryMap = new TreeMap<Long, FileSummary>(); 105 106 logger = LoggerUtils.getLogger(getClass()); 107 } 108 109 /** 110 * Returns the number of files in the profile. 111 */ getNumberOfFiles()112 synchronized int getNumberOfFiles() { 113 return fileSummaryMap.size(); 114 } 115 116 /** 117 * Returns an approximation of the total log size. Used for stats. 118 */ getTotalLogSize()119 long getTotalLogSize() { 120 121 /* Start with the size from the profile. */ 122 long size = 0; 123 synchronized (this) { 124 for (FileSummary summary : fileSummaryMap.values()) { 125 size += summary.totalSize; 126 } 127 } 128 129 /* 130 * Add sizes that are known to the tracker but are not yet in the 131 * profile. The FileSummary.totalSize field is the delta for new 132 * log entries added. Typically the last log file is only one that 133 * will have a delta, but previous files may also not have been added 134 * to the profile yet. 135 */ 136 for (TrackedFileSummary summary : tracker.getTrackedFiles()) { 137 size += summary.totalSize; 138 } 139 140 return size; 141 } 142 143 /** 144 * Gets the base summary from the cached map. Add the tracked summary, if 145 * one exists, to the base summary. Sets all entries obsolete, if the file 146 * is in the migrateFiles set. 147 */ getFileSummary(Long file)148 private synchronized FileSummary getFileSummary(Long file) { 149 150 /* Get base summary. */ 151 FileSummary summary = fileSummaryMap.get(file); 152 153 /* Add tracked summary */ 154 TrackedFileSummary trackedSummary = tracker.getTrackedFile(file); 155 if (trackedSummary != null) { 156 FileSummary totals = new FileSummary(); 157 totals.add(summary); 158 totals.add(trackedSummary); 159 summary = totals; 160 } 161 162 return summary; 163 } 164 165 /** 166 * Count the given locally tracked info as obsolete and then log the file 167 * and database info. 168 */ flushLocalTracker(LocalUtilizationTracker localTracker)169 public void flushLocalTracker(LocalUtilizationTracker localTracker) 170 throws DatabaseException { 171 172 /* Count tracked info under the log write latch. */ 173 env.getLogManager().transferToUtilizationTracker(localTracker); 174 175 /* Write out the modified file and database info. */ 176 flushFileUtilization(localTracker.getTrackedFiles()); 177 flushDbUtilization(localTracker); 178 } 179 180 /** 181 * Flush a FileSummaryLN node for each given TrackedFileSummary. 182 */ flushFileUtilization(Collection<TrackedFileSummary> activeFiles)183 public void flushFileUtilization 184 (Collection<TrackedFileSummary> activeFiles) 185 throws DatabaseException { 186 187 /* Utilization flushing may be disabled for unittests. */ 188 if (!DbInternal.getCheckpointUP 189 (env.getConfigManager().getEnvironmentConfig())) { 190 return; 191 } 192 193 /* Write out the modified file summaries. */ 194 for (TrackedFileSummary activeFile : activeFiles) { 195 long fileNum = activeFile.getFileNumber(); 196 TrackedFileSummary tfs = tracker.getTrackedFile(fileNum); 197 if (tfs != null) { 198 flushFileSummary(tfs); 199 } 200 } 201 } 202 203 /** 204 * Flush a MapLN for each database that has dirty utilization in the given 205 * tracker. 206 */ flushDbUtilization(LocalUtilizationTracker localTracker)207 private void flushDbUtilization(LocalUtilizationTracker localTracker) 208 throws DatabaseException { 209 210 /* Utilization flushing may be disabled for unittests. */ 211 if (!DbInternal.getCheckpointUP 212 (env.getConfigManager().getEnvironmentConfig())) { 213 return; 214 } 215 216 /* Write out the modified MapLNs. */ 217 Iterator<Object> dbs = localTracker.getTrackedDbs().iterator(); 218 while (dbs.hasNext()) { 219 DatabaseImpl db = (DatabaseImpl) dbs.next(); 220 if (!db.isDeleted() && db.isDirty()) { 221 env.getDbTree().modifyDbRoot(db); 222 } 223 } 224 } 225 226 /** 227 * Returns a copy of the current file summary map, optionally including 228 * tracked summary information, for use by the DbSpace utility and by unit 229 * tests. The returned map's key is a Long file number and its value is a 230 * FileSummary. 231 */ 232 public synchronized SortedMap<Long, FileSummary> getFileSummaryMap(boolean includeTrackedFiles)233 getFileSummaryMap(boolean includeTrackedFiles) { 234 235 assert cachePopulated; 236 237 if (includeTrackedFiles) { 238 239 /* 240 * Copy the fileSummaryMap to a new map, adding in the tracked 241 * summary information for each entry. 242 */ 243 TreeMap<Long, FileSummary> map = new TreeMap<Long, FileSummary>(); 244 for (Long file : fileSummaryMap.keySet()) { 245 FileSummary summary = getFileSummary(file); 246 map.put(file, summary); 247 } 248 249 /* Add tracked files that are not in fileSummaryMap yet. */ 250 for (TrackedFileSummary summary : tracker.getTrackedFiles()) { 251 Long fileNum = Long.valueOf(summary.getFileNumber()); 252 if (!map.containsKey(fileNum)) { 253 map.put(fileNum, summary); 254 } 255 } 256 return map; 257 } else { 258 return new TreeMap<Long, FileSummary>(fileSummaryMap); 259 } 260 } 261 262 /** 263 * Returns unprotected file summary map for testing. 264 */ getMapForTesting()265 SortedMap<Long, FileSummary> getMapForTesting() { 266 return fileSummaryMap; 267 } 268 269 /** 270 * Clears the cache of file summary info. The cache is not automatically 271 * repopulated, so this method should currently be called only by close. 272 */ clearCache()273 private synchronized void clearCache() { 274 275 int memorySize = fileSummaryMap.size() * 276 MemoryBudget.UTILIZATION_PROFILE_ENTRY; 277 MemoryBudget mb = env.getMemoryBudget(); 278 mb.updateAdminMemoryUsage(0 - memorySize); 279 280 fileSummaryMap = new TreeMap<Long, FileSummary>(); 281 cachePopulated = false; 282 } 283 284 /** 285 * Removes a file from the MapLN utilization info, the utilization database 286 * and the profile, after it has been determined that the file does not 287 * exist. 288 */ removeFile(Long fileNum, Set<DatabaseId> databases)289 void removeFile(Long fileNum, Set<DatabaseId> databases) 290 throws DatabaseException { 291 292 removePerDbMetadata(Collections.singleton(fileNum), databases); 293 removePerFileMetadata(fileNum); 294 } 295 296 /** 297 * Removes a file from the utilization database and the profile. 298 * 299 * For a given file, this method should be called after calling 300 * removePerDbMetadata. We update the MapLNs before deleting 301 * FileSummaryLNs in case there is an error during this process. If a 302 * FileSummaryLN exists, we will redo this process during the next recovery 303 * (populateCache). 304 */ removePerFileMetadata(Long fileNum)305 void removePerFileMetadata(Long fileNum) 306 throws DatabaseException { 307 308 /* Synchronize to update the cache. */ 309 synchronized (this) { 310 assert cachePopulated; 311 312 /* Remove from the cache. */ 313 FileSummary oldSummary = fileSummaryMap.remove(fileNum); 314 if (oldSummary != null) { 315 MemoryBudget mb = env.getMemoryBudget(); 316 mb.updateAdminMemoryUsage 317 (0 - MemoryBudget.UTILIZATION_PROFILE_ENTRY); 318 } 319 } 320 321 /* Do not synchronize during LN deletion, to permit eviction. */ 322 deleteFileSummary(fileNum); 323 } 324 325 /** 326 * Updates all MapLNs to remove the DbFileSummary for the given set of 327 * file. This method performs eviction and is not synchronized. 328 * 329 * This method is optimally called with a set of files that will 330 * subsequently be passed to removePerFileMetadata. When a set of files is 331 * being deleted, this prevents writing a MapLN more than once when more 332 * than one file contains entries for that database. 333 * 334 * For a given file, this method should be called before calling 335 * removePerFileMetadata. We update the MapLNs before deleting 336 * FileSummaryLNs in case there is an error during this process. If a 337 * FileSummaryLN exists, we will redo this process during the next recovery 338 * (populateCache). 339 */ removePerDbMetadata(final Collection<Long> fileNums, final Set<DatabaseId> databases)340 void removePerDbMetadata(final Collection<Long> fileNums, 341 final Set<DatabaseId> databases) 342 throws DatabaseException { 343 344 final LogManager logManager = env.getLogManager(); 345 final DbTree dbTree = env.getDbTree(); 346 /* Only call logMapTreeRoot once for ID and NAME DBs. */ 347 DatabaseImpl idDatabase = dbTree.getDb(DbTree.ID_DB_ID); 348 DatabaseImpl nameDatabase = dbTree.getDb(DbTree.NAME_DB_ID); 349 boolean logRoot = false; 350 if (logManager.removeDbFileSummaries(idDatabase, fileNums)) { 351 logRoot = true; 352 } 353 if (logManager.removeDbFileSummaries(nameDatabase, fileNums)) { 354 logRoot = true; 355 } 356 if (logRoot) { 357 env.logMapTreeRoot(); 358 } 359 /* Use DB ID set if available to avoid full scan of ID DB. */ 360 if (databases != null) { 361 for (DatabaseId dbId : databases) { 362 if (!dbId.equals(DbTree.ID_DB_ID) && 363 !dbId.equals(DbTree.NAME_DB_ID)) { 364 DatabaseImpl db = dbTree.getDb(dbId); 365 try { 366 if (db != null && 367 logManager.removeDbFileSummaries(db, fileNums)) { 368 dbTree.modifyDbRoot(db); 369 } 370 } finally { 371 dbTree.releaseDb(db); 372 } 373 } 374 } 375 } else { 376 377 /* 378 * Use LockType.NONE for traversing the ID DB so that a lock is not 379 * held when calling modifyDbRoot, which must release locks to 380 * handle deadlocks. 381 */ 382 CursorImpl.traverseDbWithCursor(idDatabase, 383 LockType.NONE, 384 true /*allowEviction*/, 385 new CursorImpl.WithCursor() { 386 public boolean withCursor(CursorImpl cursor, 387 DatabaseEntry key, 388 DatabaseEntry data) 389 throws DatabaseException { 390 391 MapLN mapLN = 392 (MapLN) cursor.lockAndGetCurrentLN(LockType.NONE); 393 394 if (mapLN != null) { 395 DatabaseImpl db = mapLN.getDatabase(); 396 if (logManager.removeDbFileSummaries(db, fileNums)) { 397 398 /* 399 * Because we're using dirty-read, silently do 400 * nothing if the DB does not exist 401 * (mustExist=false). 402 */ 403 dbTree.modifyDbRoot 404 (db, DbLsn.NULL_LSN /*ifBeforeLsn*/, 405 false /*mustExist*/); 406 } 407 } 408 return true; 409 } 410 }); 411 } 412 } 413 414 /** 415 * Deletes all FileSummaryLNs for the file. This method performs eviction 416 * and is not synchronized. 417 */ deleteFileSummary(final Long fileNum)418 private void deleteFileSummary(final Long fileNum) 419 throws DatabaseException { 420 421 Locker locker = null; 422 CursorImpl cursor = null; 423 try { 424 locker = BasicLocker.createBasicLocker(env, false /*noWait*/); 425 cursor = new CursorImpl(fileSummaryDb, locker); 426 /* Perform eviction in unsynchronized methods. */ 427 cursor.setAllowEviction(true); 428 429 DatabaseEntry keyEntry = new DatabaseEntry(); 430 DatabaseEntry dataEntry = new DatabaseEntry(); 431 long fileNumVal = fileNum.longValue(); 432 433 /* Do not return data to avoid a fetch of the existing LN. */ 434 dataEntry.setPartial(0, 0, true); 435 436 /* Search by file number. */ 437 OperationStatus status = OperationStatus.SUCCESS; 438 if (getFirstFSLN 439 (cursor, fileNumVal, keyEntry, dataEntry, LockType.WRITE)) { 440 status = OperationStatus.SUCCESS; 441 } else { 442 status = OperationStatus.NOTFOUND; 443 } 444 445 /* Delete all LNs for this file number. */ 446 while (status == OperationStatus.SUCCESS && 447 fileNumVal == 448 FileSummaryLN.getFileNumber(keyEntry.getData())) { 449 450 /* Perform eviction once per operation. */ 451 env.daemonEviction(true /*backgroundIO*/); 452 453 /* 454 * Eviction after deleting is not necessary since we did not 455 * fetch the LN. 456 */ 457 cursor.deleteCurrentRecord(ReplicationContext.NO_REPLICATE); 458 459 status = cursor.getNext( 460 keyEntry, dataEntry, LockType.WRITE, 461 false /*dirtyReadAll*/, true /*forward*/, 462 false /*isLatched*/, null /*rangeConstraint*/); 463 } 464 } finally { 465 if (cursor != null) { 466 cursor.close(); 467 } 468 if (locker != null) { 469 locker.operationEnd(); 470 } 471 } 472 473 /* Explicitly remove the file from the tracker. */ 474 TrackedFileSummary tfs = tracker.getTrackedFile(fileNum); 475 if (tfs != null) { 476 env.getLogManager().removeTrackedFile(tfs); 477 } 478 } 479 480 /** 481 * Updates and stores the FileSummary for a given tracked file, if flushing 482 * of the summary is allowed. 483 */ flushFileSummary(TrackedFileSummary tfs)484 public void flushFileSummary(TrackedFileSummary tfs) 485 throws DatabaseException { 486 487 if (tfs.getAllowFlush()) { 488 putFileSummary(tfs); 489 } 490 } 491 492 /** 493 * Updates and stores the FileSummary for a given tracked file. This 494 * method is synchronized and may not perform eviction. 495 */ putFileSummary(TrackedFileSummary tfs)496 private synchronized PackedOffsets putFileSummary(TrackedFileSummary tfs) 497 throws DatabaseException { 498 499 if (env.isReadOnly()) { 500 throw EnvironmentFailureException.unexpectedState 501 ("Cannot write file summary in a read-only environment"); 502 } 503 504 if (tfs.isEmpty()) { 505 return null; // no delta 506 } 507 508 if (!cachePopulated) { 509 /* Db does not exist and this is a read-only environment. */ 510 return null; 511 } 512 513 long fileNum = tfs.getFileNumber(); 514 Long fileNumLong = Long.valueOf(fileNum); 515 516 /* Get existing file summary or create an empty one. */ 517 FileSummary summary = fileSummaryMap.get(fileNumLong); 518 if (summary == null) { 519 520 /* 521 * An obsolete node may have been counted after its file was 522 * deleted, for example, when compressing a BIN. Do not insert a 523 * new profile record if no corresponding log file exists. But if 524 * the file number is greater than the last known file, this is a 525 * new file that has been buffered but not yet flushed to disk; in 526 * that case we should insert a new profile record. 527 */ 528 if (!fileSummaryMap.isEmpty() && 529 fileNum < fileSummaryMap.lastKey() && 530 !env.getFileManager().isFileValid(fileNum)) { 531 532 /* 533 * File was deleted by the cleaner. Remove it from the 534 * UtilizationTracker and return. Note that a file is normally 535 * removed from the tracker by FileSummaryLN.writeToLog method 536 * when it is called via insertFileSummary below. [#15512] 537 */ 538 env.getLogManager().removeTrackedFile(tfs); 539 return null; 540 } 541 542 summary = new FileSummary(); 543 } 544 545 /* 546 * The key discriminator is a sequence that must be increasing over the 547 * life of the file. We use the sum of all entries counted. We must 548 * add the tracked and current summaries here to calculate the key. 549 */ 550 FileSummary tmp = new FileSummary(); 551 tmp.add(summary); 552 tmp.add(tfs); 553 int sequence = tmp.getEntriesCounted(); 554 555 /* Insert an LN with the existing and tracked summary info. */ 556 FileSummaryLN ln = new FileSummaryLN(summary); 557 ln.setTrackedSummary(tfs); 558 insertFileSummary(ln, fileNum, sequence); 559 560 /* Cache the updated summary object. */ 561 summary = ln.getBaseSummary(); 562 563 if (fileSummaryMap.put(fileNumLong, summary) == null) { 564 MemoryBudget mb = env.getMemoryBudget(); 565 mb.updateAdminMemoryUsage(MemoryBudget.UTILIZATION_PROFILE_ENTRY); 566 } 567 568 return ln.getObsoleteOffsets(); 569 } 570 571 /** 572 * Returns the stored/packed obsolete offsets offsets for the given file. 573 * 574 * @param logUpdate if true, log any updates to the utilization profile. If 575 * false, only retrieve the new information. 576 */ getObsoleteDetail(Long fileNum, boolean logUpdate)577 PackedOffsets getObsoleteDetail(Long fileNum, boolean logUpdate) 578 throws DatabaseException { 579 580 final PackedOffsets packedOffsets = new PackedOffsets(); 581 582 /* Return if no detail is being tracked. */ 583 if (!env.getCleaner().trackDetail) { 584 return packedOffsets; 585 } 586 587 assert cachePopulated; 588 589 final long fileNumVal = fileNum.longValue(); 590 final List<long[]> list = new ArrayList<long[]>(); 591 592 /* 593 * Get a TrackedFileSummary that cannot be flushed (evicted) while we 594 * gather obsolete offsets. 595 */ 596 final TrackedFileSummary tfs = 597 env.getLogManager().getUnflushableTrackedSummary(fileNumVal); 598 try { 599 /* Read the summary db. */ 600 final Locker locker = 601 BasicLocker.createBasicLocker(env, false /*noWait*/); 602 final CursorImpl cursor = new CursorImpl(fileSummaryDb, locker); 603 try { 604 /* Perform eviction in unsynchronized methods. */ 605 cursor.setAllowEviction(true); 606 607 final DatabaseEntry keyEntry = new DatabaseEntry(); 608 final DatabaseEntry dataEntry = new DatabaseEntry(); 609 610 /* Search by file number. */ 611 OperationStatus status = OperationStatus.SUCCESS; 612 if (!getFirstFSLN(cursor, fileNumVal, keyEntry, dataEntry, 613 LockType.NONE)) { 614 status = OperationStatus.NOTFOUND; 615 } 616 617 /* Read all LNs for this file number. */ 618 while (status == OperationStatus.SUCCESS) { 619 620 /* Perform eviction once per operation. */ 621 env.daemonEviction(true /*backgroundIO*/); 622 623 final FileSummaryLN ln = (FileSummaryLN) 624 cursor.lockAndGetCurrentLN(LockType.NONE); 625 626 if (ln != null) { 627 /* Stop if the file number changes. */ 628 if (fileNumVal != 629 ln.getFileNumber(keyEntry.getData())) { 630 break; 631 } 632 633 final PackedOffsets offsets = ln.getObsoleteOffsets(); 634 if (offsets != null) { 635 list.add(offsets.toArray()); 636 } 637 638 /* Always evict after using a file summary LN. */ 639 cursor.evict(); 640 } 641 642 status = cursor.getNext( 643 keyEntry, dataEntry, LockType.NONE, 644 false /*dirtyReadAll*/, true /*forward*/, 645 false /*isLatched*/, null /*rangeConstraint*/); 646 } 647 } finally { 648 cursor.close(); 649 locker.operationEnd(); 650 } 651 652 /* 653 * Write out tracked detail, if any, and add its offsets to the 654 * list. 655 */ 656 if (!tfs.isEmpty()) { 657 if (logUpdate) { 658 final PackedOffsets offsets = putFileSummary(tfs); 659 if (offsets != null) { 660 list.add(offsets.toArray()); 661 } 662 } else { 663 final long[] offsetList = tfs.getObsoleteOffsets(); 664 if (offsetList != null) { 665 list.add(offsetList); 666 } 667 } 668 } 669 } finally { 670 /* Allow flushing of TFS when all offsets have been gathered. */ 671 tfs.setAllowFlush(true); 672 } 673 674 /* Merge all offsets into a single array and pack the result. */ 675 int size = 0; 676 for (int i = 0; i < list.size(); i += 1) { 677 final long[] a = list.get(i); 678 size += a.length; 679 } 680 final long[] offsets = new long[size]; 681 int index = 0; 682 for (int i = 0; i < list.size(); i += 1) { 683 long[] a = list.get(i); 684 System.arraycopy(a, 0, offsets, index, a.length); 685 index += a.length; 686 } 687 assert index == offsets.length; 688 689 packedOffsets.pack(offsets); 690 return packedOffsets; 691 } 692 693 /** 694 * Populate the profile for file selection. This method performs eviction 695 * and is not synchronized. It must be called before recovery is complete 696 * so that synchronization is unnecessary. It must be called before the 697 * recovery checkpoint so that the checkpoint can flush file summary 698 * information. 699 */ populateCache(StartupTracker.Counter counter)700 public boolean populateCache(StartupTracker.Counter counter) 701 throws DatabaseException { 702 703 assert !cachePopulated; 704 705 /* Open the file summary db on first use. */ 706 if (!openFileSummaryDatabase()) { 707 /* Db does not exist and this is a read-only environment. */ 708 return false; 709 } 710 711 int oldMemorySize = fileSummaryMap.size() * 712 MemoryBudget.UTILIZATION_PROFILE_ENTRY; 713 714 /* 715 * It is possible to have an undeleted FileSummaryLN in the database 716 * for a deleted log file if we crash after deleting a file but before 717 * deleting the FileSummaryLN. Iterate through all FileSummaryLNs and 718 * add them to the cache if their corresponding log file exists. But 719 * delete those records that have no corresponding log file. 720 */ 721 Long[] existingFiles = env.getFileManager().getAllFileNumbers(); 722 Locker locker = null; 723 CursorImpl cursor = null; 724 try { 725 locker = BasicLocker.createBasicLocker(env, false /*noWait*/); 726 cursor = new CursorImpl(fileSummaryDb, locker); 727 /* Perform eviction in unsynchronized methods. */ 728 cursor.setAllowEviction(true); 729 730 DatabaseEntry keyEntry = new DatabaseEntry(); 731 DatabaseEntry dataEntry = new DatabaseEntry(); 732 733 if (cursor.positionFirstOrLast(true)) { 734 735 /* Retrieve the first record. */ 736 OperationStatus status = cursor.lockAndGetCurrent( 737 keyEntry, dataEntry, LockType.NONE, 738 false /*dirtyReadAll*/, 739 true /*isLatched*/, true /*unlatch*/); 740 741 if (status != OperationStatus.SUCCESS) { 742 /* The record we're pointing at may be deleted. */ 743 status = cursor.getNext( 744 keyEntry, dataEntry, LockType.NONE, 745 false /*dirtyReadAll*/, true /*forward*/, 746 false /*isLatched*/, null /*rangeConstraint*/); 747 } 748 749 while (status == OperationStatus.SUCCESS) { 750 counter.incNumRead(); 751 752 /* 753 * Perform eviction once per operation. Pass false for 754 * backgroundIO because this is done during recovery and 755 * there is no reason to sleep. 756 */ 757 env.daemonEviction(false /*backgroundIO*/); 758 759 FileSummaryLN ln = (FileSummaryLN) 760 cursor.lockAndGetCurrentLN(LockType.NONE); 761 762 if (ln == null) { 763 /* Advance past a cleaned record. */ 764 status = cursor.getNext( 765 keyEntry, dataEntry, LockType.NONE, 766 false /*dirtyReadAll*/, 767 true /*forward*/, false /*isLatched*/, 768 null /*rangeConstraint*/); 769 continue; 770 } 771 772 byte[] keyBytes = keyEntry.getData(); 773 boolean isOldVersion = ln.hasStringKey(keyBytes); 774 long fileNum = ln.getFileNumber(keyBytes); 775 Long fileNumLong = Long.valueOf(fileNum); 776 777 if (Arrays.binarySearch(existingFiles, fileNumLong) >= 0) { 778 counter.incNumProcessed(); 779 780 /* File exists, cache the FileSummaryLN. */ 781 FileSummary summary = ln.getBaseSummary(); 782 fileSummaryMap.put(fileNumLong, summary); 783 784 /* 785 * Update old version records to the new version. A 786 * zero sequence number is used to distinguish the 787 * converted records and to ensure that later records 788 * will have a greater sequence number. 789 */ 790 if (isOldVersion && !env.isReadOnly()) { 791 insertFileSummary(ln, fileNum, 0); 792 cursor.deleteCurrentRecord( 793 ReplicationContext.NO_REPLICATE); 794 } else { 795 /* Always evict after using a file summary LN. */ 796 cursor.evict(); 797 } 798 } else { 799 800 /* 801 * File does not exist, remove the summary from the map 802 * and delete all FileSummaryLN records. 803 */ 804 counter.incNumDeleted(); 805 806 fileSummaryMap.remove(fileNumLong); 807 808 if (!env.isReadOnly()) { 809 removePerDbMetadata 810 (Collections.singleton(fileNumLong), 811 null /*databases*/); 812 if (isOldVersion) { 813 cursor.latchBIN(); 814 cursor.deleteCurrentRecord( 815 ReplicationContext.NO_REPLICATE); 816 } else { 817 deleteFileSummary(fileNumLong); 818 } 819 } 820 821 /* 822 * Do not evict after deleting since the compressor 823 * would have to fetch it again. 824 */ 825 } 826 827 /* Go on to the next entry. */ 828 if (isOldVersion) { 829 830 /* Advance past the single old version record. */ 831 status = cursor.getNext( 832 keyEntry, dataEntry, LockType.NONE, 833 false /*dirtyReadAll*/, 834 true /*forward*/, false /*isLatched*/, 835 null /*rangeConstraint*/); 836 } else { 837 838 /* 839 * Skip over other records for this file by adding one 840 * to the file number and doing a range search. 841 */ 842 if (!getFirstFSLN 843 (cursor, 844 fileNum + 1, 845 keyEntry, dataEntry, 846 LockType.NONE)) { 847 status = OperationStatus.NOTFOUND; 848 } 849 } 850 } 851 } 852 } finally { 853 if (cursor != null) { 854 /* positionFirstOrLast may leave BIN latched. */ 855 cursor.close(); 856 } 857 if (locker != null) { 858 locker.operationEnd(); 859 } 860 861 int newMemorySize = fileSummaryMap.size() * 862 MemoryBudget.UTILIZATION_PROFILE_ENTRY; 863 MemoryBudget mb = env.getMemoryBudget(); 864 mb.updateAdminMemoryUsage(newMemorySize - oldMemorySize); 865 } 866 867 cachePopulated = true; 868 return true; 869 } 870 871 /** 872 * Positions at the most recent LN for the given file number. 873 */ getFirstFSLN(CursorImpl cursor, long fileNum, DatabaseEntry keyEntry, DatabaseEntry dataEntry, LockType lockType)874 private boolean getFirstFSLN(CursorImpl cursor, 875 long fileNum, 876 DatabaseEntry keyEntry, 877 DatabaseEntry dataEntry, 878 LockType lockType) 879 throws DatabaseException { 880 881 byte[] keyBytes = FileSummaryLN.makePartialKey(fileNum); 882 keyEntry.setData(keyBytes); 883 884 cursor.reset(); 885 886 try { 887 int result = cursor.searchRange(keyEntry, null /*comparator*/); 888 889 if ((result & CursorImpl.FOUND) == 0) { 890 return false; 891 } 892 893 boolean exactKeyMatch = ((result & CursorImpl.EXACT_KEY) != 0); 894 895 if (exactKeyMatch && 896 cursor.lockAndGetCurrent( 897 keyEntry, dataEntry, lockType, false /*dirtyReadAll*/, 898 true /*isLatched*/, false /*unlatch*/) != 899 OperationStatus.KEYEMPTY) { 900 return true; 901 } 902 } finally { 903 cursor.releaseBIN(); 904 } 905 906 /* Always evict after using a file summary LN. */ 907 cursor.evict(); 908 909 OperationStatus status = cursor.getNext( 910 keyEntry, dataEntry, lockType, false /*dirtyReadAll*/, 911 true /*forward*/, false /*isLatched*/, null /*rangeConstraint*/); 912 913 return status == OperationStatus.SUCCESS; 914 } 915 916 /** 917 * If the file summary db is already open, return, otherwise attempt to 918 * open it. If the environment is read-only and the database doesn't 919 * exist, return false. If the environment is read-write the database will 920 * be created if it doesn't exist. 921 */ openFileSummaryDatabase()922 private boolean openFileSummaryDatabase() 923 throws DatabaseException { 924 925 if (fileSummaryDb != null) { 926 return true; 927 } 928 DbTree dbTree = env.getDbTree(); 929 Locker autoTxn = null; 930 boolean operationOk = false; 931 try { 932 autoTxn = Txn.createLocalAutoTxn(env, new TransactionConfig()); 933 934 /* 935 * releaseDb is not called after this getDb or createDb because we 936 * want to prohibit eviction of this database until the environment 937 * is closed. 938 */ 939 DatabaseImpl db = dbTree.getDb 940 (autoTxn, DbType.UTILIZATION.getInternalName(), null); 941 if (db == null) { 942 if (env.isReadOnly()) { 943 return false; 944 } 945 DatabaseConfig dbConfig = new DatabaseConfig(); 946 dbConfig.setReplicated(false); 947 db = dbTree.createInternalDb 948 (autoTxn, DbType.UTILIZATION.getInternalName(), 949 dbConfig); 950 } 951 fileSummaryDb = db; 952 operationOk = true; 953 return true; 954 } finally { 955 if (autoTxn != null) { 956 autoTxn.operationEnd(operationOk); 957 } 958 } 959 } 960 961 /** 962 * For unit testing. 963 */ getFileSummaryDb()964 public DatabaseImpl getFileSummaryDb() { 965 return fileSummaryDb; 966 } 967 968 /** 969 * Insert the given LN with the given key values. This method is 970 * synchronized and may not perform eviction. 971 * 972 * Is public only for unit testing. 973 */ insertFileSummary( FileSummaryLN ln, long fileNum, int sequence)974 synchronized boolean insertFileSummary( 975 FileSummaryLN ln, 976 long fileNum, 977 int sequence) 978 throws DatabaseException { 979 980 byte[] keyBytes = FileSummaryLN.makeFullKey(fileNum, sequence); 981 982 Locker locker = null; 983 CursorImpl cursor = null; 984 try { 985 locker = BasicLocker.createBasicLocker(env, false /*noWait*/); 986 cursor = new CursorImpl(fileSummaryDb, locker); 987 988 /* Insert the LN. */ 989 OperationStatus status = cursor.insertRecord( 990 keyBytes, ln, false /*blindInsertion*/, 991 ReplicationContext.NO_REPLICATE); 992 993 if (status == OperationStatus.KEYEXIST) { 994 LoggerUtils.traceAndLog 995 (logger, env, Level.SEVERE, 996 "Cleaner duplicate key sequence file=0x" + 997 Long.toHexString(fileNum) + " sequence=0x" + 998 Long.toHexString(sequence)); 999 return false; 1000 } 1001 1002 /* Always evict after using a file summary LN. */ 1003 cursor.evict(); 1004 return true; 1005 } finally { 1006 if (cursor != null) { 1007 cursor.close(); 1008 } 1009 if (locker != null) { 1010 locker.operationEnd(); 1011 } 1012 } 1013 } 1014 1015 /** 1016 * Checks that all FSLN offsets are indeed obsolete. Assumes that the 1017 * system is quiesent (does not lock LNs). This method is not synchronized 1018 * (because it doesn't access fileSummaryMap) and eviction is allowed. 1019 * 1020 * @return true if no verification failures. 1021 */ verifyFileSummaryDatabase()1022 public boolean verifyFileSummaryDatabase() 1023 throws DatabaseException { 1024 1025 DatabaseEntry key = new DatabaseEntry(); 1026 DatabaseEntry data = new DatabaseEntry(); 1027 1028 openFileSummaryDatabase(); 1029 Locker locker = null; 1030 CursorImpl cursor = null; 1031 boolean ok = true; 1032 1033 try { 1034 locker = BasicLocker.createBasicLocker(env, false /*noWait*/); 1035 cursor = new CursorImpl(fileSummaryDb, locker); 1036 cursor.setAllowEviction(true); 1037 1038 if (cursor.positionFirstOrLast(true)) { 1039 1040 OperationStatus status = cursor.lockAndGetCurrent( 1041 key, data, LockType.NONE, false /*dirtyReadAll*/, 1042 true /*isLatched*/, true /*unlatch*/); 1043 1044 /* Iterate over all file summary lns. */ 1045 while (status == OperationStatus.SUCCESS) { 1046 1047 /* Perform eviction once per operation. */ 1048 env.daemonEviction(true /*backgroundIO*/); 1049 1050 FileSummaryLN ln = (FileSummaryLN) 1051 cursor.lockAndGetCurrentLN(LockType.NONE); 1052 1053 if (ln != null) { 1054 long fileNumVal = ln.getFileNumber(key.getData()); 1055 PackedOffsets offsets = ln.getObsoleteOffsets(); 1056 1057 /* 1058 * Check every offset in the fsln to make sure it's 1059 * truely obsolete. 1060 */ 1061 if (offsets != null) { 1062 long[] vals = offsets.toArray(); 1063 for (int i = 0; i < vals.length; i++) { 1064 long lsn = DbLsn.makeLsn(fileNumVal, vals[i]); 1065 if (!verifyLsnIsObsolete(lsn)) { 1066 ok = false; 1067 } 1068 } 1069 } 1070 1071 cursor.evict(); 1072 } 1073 1074 status = cursor.getNext( 1075 key, data, LockType.NONE, false /*dirtyReadAll*/, 1076 true /*forward*/, false /*isLatched*/, 1077 null /*rangeConstraint*/); 1078 } 1079 } 1080 } finally { 1081 if (cursor != null) { 1082 cursor.close(); 1083 } 1084 if (locker != null) { 1085 locker.operationEnd(); 1086 } 1087 } 1088 1089 return ok; 1090 } 1091 1092 /* 1093 * Return true if the LN at this lsn is obsolete. 1094 */ verifyLsnIsObsolete(long lsn)1095 private boolean verifyLsnIsObsolete(long lsn) 1096 throws DatabaseException { 1097 1098 /* Read the whole entry out of the log. */ 1099 Object o = env.getLogManager().getLogEntryHandleFileNotFound(lsn); 1100 if (!(o instanceof LNLogEntry)) { 1101 return true; 1102 } 1103 LNLogEntry<?> entry = (LNLogEntry<?>) o; 1104 1105 /* Find the owning database. */ 1106 DatabaseId dbId = entry.getDbId(); 1107 DatabaseImpl db = env.getDbTree().getDb(dbId); 1108 1109 /* 1110 * Search down to the bottom most level for the parent of this LN. 1111 */ 1112 BIN bin = null; 1113 try { 1114 /* 1115 * The whole database is gone, so this LN is obsolete. No need 1116 * to worry about delete cleanup; this is just verification and 1117 * no cleaning is done. 1118 */ 1119 if (db == null || db.isDeleted()) { 1120 return true; 1121 } 1122 1123 if (entry.isImmediatelyObsolete(db)) { 1124 return true; 1125 } 1126 1127 entry.postFetchInit(db); 1128 1129 Tree tree = db.getTree(); 1130 TreeLocation location = new TreeLocation(); 1131 boolean parentFound = tree.getParentBINForChildLN( 1132 location, entry.getKey(), false /*splitsAllowed*/, 1133 false /*blindDeltaOps*/, CacheMode.UNCHANGED); 1134 1135 bin = location.bin; 1136 int index = location.index; 1137 1138 /* Is bin latched ? */ 1139 if (!parentFound) { 1140 return true; 1141 } 1142 1143 /* 1144 * Now we're at the BIN parent for this LN. If knownDeleted, LN is 1145 * deleted and can be purged. 1146 */ 1147 if (bin.isEntryKnownDeleted(index)) { 1148 return true; 1149 } 1150 1151 if (bin.getLsn(index) != lsn) { 1152 return true; 1153 } 1154 1155 /* Oh no -- this lsn is in the tree. */ 1156 /* should print, or trace? */ 1157 System.err.println("lsn " + DbLsn.getNoFormatString(lsn)+ 1158 " was found in tree."); 1159 return false; 1160 } finally { 1161 env.getDbTree().releaseDb(db); 1162 if (bin != null) { 1163 bin.releaseLatch(); 1164 } 1165 } 1166 } 1167 1168 /** 1169 * Update memory budgets when this profile is closed and will never be 1170 * accessed again. 1171 */ close()1172 void close() { 1173 clearCache(); 1174 if (fileSummaryDb != null) { 1175 fileSummaryDb.releaseTreeAdminMemory(); 1176 } 1177 } 1178 } 1179