1 /* 2 * CDDL HEADER START 3 * 4 * The contents of this file are subject to the terms of the 5 * Common Development and Distribution License (the "License"). 6 * You may not use this file except in compliance with the License. 7 * 8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 9 * or http://www.opensolaris.org/os/licensing. 10 * See the License for the specific language governing permissions 11 * and limitations under the License. 12 * 13 * When distributing Covered Code, include this CDDL HEADER in each 14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 15 * If applicable, add the following below this CDDL HEADER, with the 16 * fields enclosed by brackets "[]" replaced with your own identifying 17 * information: Portions Copyright [yyyy] [name of copyright owner] 18 * 19 * CDDL HEADER END 20 */ 21 /* 22 * Copyright (c) 2005, 2010, Oracle and/or its affiliates. All rights reserved. 23 * Copyright (c) 2018, Joyent, Inc. 24 * Copyright (c) 2011, 2018 by Delphix. All rights reserved. 25 * Copyright (c) 2014 by Saso Kiselkov. All rights reserved. 26 * Copyright 2017 Nexenta Systems, Inc. All rights reserved. 27 */ 28 29 /* 30 * DVA-based Adjustable Replacement Cache 31 * 32 * While much of the theory of operation used here is 33 * based on the self-tuning, low overhead replacement cache 34 * presented by Megiddo and Modha at FAST 2003, there are some 35 * significant differences: 36 * 37 * 1. The Megiddo and Modha model assumes any page is evictable. 38 * Pages in its cache cannot be "locked" into memory. This makes 39 * the eviction algorithm simple: evict the last page in the list. 40 * This also make the performance characteristics easy to reason 41 * about. Our cache is not so simple. At any given moment, some 42 * subset of the blocks in the cache are un-evictable because we 43 * have handed out a reference to them. Blocks are only evictable 44 * when there are no external references active. This makes 45 * eviction far more problematic: we choose to evict the evictable 46 * blocks that are the "lowest" in the list. 47 * 48 * There are times when it is not possible to evict the requested 49 * space. In these circumstances we are unable to adjust the cache 50 * size. To prevent the cache growing unbounded at these times we 51 * implement a "cache throttle" that slows the flow of new data 52 * into the cache until we can make space available. 53 * 54 * 2. The Megiddo and Modha model assumes a fixed cache size. 55 * Pages are evicted when the cache is full and there is a cache 56 * miss. Our model has a variable sized cache. It grows with 57 * high use, but also tries to react to memory pressure from the 58 * operating system: decreasing its size when system memory is 59 * tight. 60 * 61 * 3. The Megiddo and Modha model assumes a fixed page size. All 62 * elements of the cache are therefore exactly the same size. So 63 * when adjusting the cache size following a cache miss, its simply 64 * a matter of choosing a single page to evict. In our model, we 65 * have variable sized cache blocks (rangeing from 512 bytes to 66 * 128K bytes). We therefore choose a set of blocks to evict to make 67 * space for a cache miss that approximates as closely as possible 68 * the space used by the new block. 69 * 70 * See also: "ARC: A Self-Tuning, Low Overhead Replacement Cache" 71 * by N. Megiddo & D. Modha, FAST 2003 72 */ 73 74 /* 75 * The locking model: 76 * 77 * A new reference to a cache buffer can be obtained in two 78 * ways: 1) via a hash table lookup using the DVA as a key, 79 * or 2) via one of the ARC lists. The arc_read() interface 80 * uses method 1, while the internal ARC algorithms for 81 * adjusting the cache use method 2. We therefore provide two 82 * types of locks: 1) the hash table lock array, and 2) the 83 * ARC list locks. 84 * 85 * Buffers do not have their own mutexes, rather they rely on the 86 * hash table mutexes for the bulk of their protection (i.e. most 87 * fields in the arc_buf_hdr_t are protected by these mutexes). 88 * 89 * buf_hash_find() returns the appropriate mutex (held) when it 90 * locates the requested buffer in the hash table. It returns 91 * NULL for the mutex if the buffer was not in the table. 92 * 93 * buf_hash_remove() expects the appropriate hash mutex to be 94 * already held before it is invoked. 95 * 96 * Each ARC state also has a mutex which is used to protect the 97 * buffer list associated with the state. When attempting to 98 * obtain a hash table lock while holding an ARC list lock you 99 * must use: mutex_tryenter() to avoid deadlock. Also note that 100 * the active state mutex must be held before the ghost state mutex. 101 * 102 * Note that the majority of the performance stats are manipulated 103 * with atomic operations. 104 * 105 * The L2ARC uses the l2ad_mtx on each vdev for the following: 106 * 107 * - L2ARC buflist creation 108 * - L2ARC buflist eviction 109 * - L2ARC write completion, which walks L2ARC buflists 110 * - ARC header destruction, as it removes from L2ARC buflists 111 * - ARC header release, as it removes from L2ARC buflists 112 */ 113 114 /* 115 * ARC operation: 116 * 117 * Every block that is in the ARC is tracked by an arc_buf_hdr_t structure. 118 * This structure can point either to a block that is still in the cache or to 119 * one that is only accessible in an L2 ARC device, or it can provide 120 * information about a block that was recently evicted. If a block is 121 * only accessible in the L2ARC, then the arc_buf_hdr_t only has enough 122 * information to retrieve it from the L2ARC device. This information is 123 * stored in the l2arc_buf_hdr_t sub-structure of the arc_buf_hdr_t. A block 124 * that is in this state cannot access the data directly. 125 * 126 * Blocks that are actively being referenced or have not been evicted 127 * are cached in the L1ARC. The L1ARC (l1arc_buf_hdr_t) is a structure within 128 * the arc_buf_hdr_t that will point to the data block in memory. A block can 129 * only be read by a consumer if it has an l1arc_buf_hdr_t. The L1ARC 130 * caches data in two ways -- in a list of ARC buffers (arc_buf_t) and 131 * also in the arc_buf_hdr_t's private physical data block pointer (b_pabd). 132 * 133 * The L1ARC's data pointer may or may not be uncompressed. The ARC has the 134 * ability to store the physical data (b_pabd) associated with the DVA of the 135 * arc_buf_hdr_t. Since the b_pabd is a copy of the on-disk physical block, 136 * it will match its on-disk compression characteristics. This behavior can be 137 * disabled by setting 'zfs_compressed_arc_enabled' to B_FALSE. When the 138 * compressed ARC functionality is disabled, the b_pabd will point to an 139 * uncompressed version of the on-disk data. 140 * 141 * Data in the L1ARC is not accessed by consumers of the ARC directly. Each 142 * arc_buf_hdr_t can have multiple ARC buffers (arc_buf_t) which reference it. 143 * Each ARC buffer (arc_buf_t) is being actively accessed by a specific ARC 144 * consumer. The ARC will provide references to this data and will keep it 145 * cached until it is no longer in use. The ARC caches only the L1ARC's physical 146 * data block and will evict any arc_buf_t that is no longer referenced. The 147 * amount of memory consumed by the arc_buf_ts' data buffers can be seen via the 148 * "overhead_size" kstat. 149 * 150 * Depending on the consumer, an arc_buf_t can be requested in uncompressed or 151 * compressed form. The typical case is that consumers will want uncompressed 152 * data, and when that happens a new data buffer is allocated where the data is 153 * decompressed for them to use. Currently the only consumer who wants 154 * compressed arc_buf_t's is "zfs send", when it streams data exactly as it 155 * exists on disk. When this happens, the arc_buf_t's data buffer is shared 156 * with the arc_buf_hdr_t. 157 * 158 * Here is a diagram showing an arc_buf_hdr_t referenced by two arc_buf_t's. The 159 * first one is owned by a compressed send consumer (and therefore references 160 * the same compressed data buffer as the arc_buf_hdr_t) and the second could be 161 * used by any other consumer (and has its own uncompressed copy of the data 162 * buffer). 163 * 164 * arc_buf_hdr_t 165 * +-----------+ 166 * | fields | 167 * | common to | 168 * | L1- and | 169 * | L2ARC | 170 * +-----------+ 171 * | l2arc_buf_hdr_t 172 * | | 173 * +-----------+ 174 * | l1arc_buf_hdr_t 175 * | | arc_buf_t 176 * | b_buf +------------>+-----------+ arc_buf_t 177 * | b_pabd +-+ |b_next +---->+-----------+ 178 * +-----------+ | |-----------| |b_next +-->NULL 179 * | |b_comp = T | +-----------+ 180 * | |b_data +-+ |b_comp = F | 181 * | +-----------+ | |b_data +-+ 182 * +->+------+ | +-----------+ | 183 * compressed | | | | 184 * data | |<--------------+ | uncompressed 185 * +------+ compressed, | data 186 * shared +-->+------+ 187 * data | | 188 * | | 189 * +------+ 190 * 191 * When a consumer reads a block, the ARC must first look to see if the 192 * arc_buf_hdr_t is cached. If the hdr is cached then the ARC allocates a new 193 * arc_buf_t and either copies uncompressed data into a new data buffer from an 194 * existing uncompressed arc_buf_t, decompresses the hdr's b_pabd buffer into a 195 * new data buffer, or shares the hdr's b_pabd buffer, depending on whether the 196 * hdr is compressed and the desired compression characteristics of the 197 * arc_buf_t consumer. If the arc_buf_t ends up sharing data with the 198 * arc_buf_hdr_t and both of them are uncompressed then the arc_buf_t must be 199 * the last buffer in the hdr's b_buf list, however a shared compressed buf can 200 * be anywhere in the hdr's list. 201 * 202 * The diagram below shows an example of an uncompressed ARC hdr that is 203 * sharing its data with an arc_buf_t (note that the shared uncompressed buf is 204 * the last element in the buf list): 205 * 206 * arc_buf_hdr_t 207 * +-----------+ 208 * | | 209 * | | 210 * | | 211 * +-----------+ 212 * l2arc_buf_hdr_t| | 213 * | | 214 * +-----------+ 215 * l1arc_buf_hdr_t| | 216 * | | arc_buf_t (shared) 217 * | b_buf +------------>+---------+ arc_buf_t 218 * | | |b_next +---->+---------+ 219 * | b_pabd +-+ |---------| |b_next +-->NULL 220 * +-----------+ | | | +---------+ 221 * | |b_data +-+ | | 222 * | +---------+ | |b_data +-+ 223 * +->+------+ | +---------+ | 224 * | | | | 225 * uncompressed | | | | 226 * data +------+ | | 227 * ^ +->+------+ | 228 * | uncompressed | | | 229 * | data | | | 230 * | +------+ | 231 * +---------------------------------+ 232 * 233 * Writing to the ARC requires that the ARC first discard the hdr's b_pabd 234 * since the physical block is about to be rewritten. The new data contents 235 * will be contained in the arc_buf_t. As the I/O pipeline performs the write, 236 * it may compress the data before writing it to disk. The ARC will be called 237 * with the transformed data and will bcopy the transformed on-disk block into 238 * a newly allocated b_pabd. Writes are always done into buffers which have 239 * either been loaned (and hence are new and don't have other readers) or 240 * buffers which have been released (and hence have their own hdr, if there 241 * were originally other readers of the buf's original hdr). This ensures that 242 * the ARC only needs to update a single buf and its hdr after a write occurs. 243 * 244 * When the L2ARC is in use, it will also take advantage of the b_pabd. The 245 * L2ARC will always write the contents of b_pabd to the L2ARC. This means 246 * that when compressed ARC is enabled that the L2ARC blocks are identical 247 * to the on-disk block in the main data pool. This provides a significant 248 * advantage since the ARC can leverage the bp's checksum when reading from the 249 * L2ARC to determine if the contents are valid. However, if the compressed 250 * ARC is disabled, then the L2ARC's block must be transformed to look 251 * like the physical block in the main data pool before comparing the 252 * checksum and determining its validity. 253 */ 254 255 #include <sys/spa.h> 256 #include <sys/zio.h> 257 #include <sys/spa_impl.h> 258 #include <sys/zio_compress.h> 259 #include <sys/zio_checksum.h> 260 #include <sys/zfs_context.h> 261 #include <sys/arc.h> 262 #include <sys/refcount.h> 263 #include <sys/vdev.h> 264 #include <sys/vdev_impl.h> 265 #include <sys/dsl_pool.h> 266 #include <sys/zio_checksum.h> 267 #include <sys/multilist.h> 268 #include <sys/abd.h> 269 #ifdef _KERNEL 270 #include <sys/vmsystm.h> 271 #include <vm/anon.h> 272 #include <sys/fs/swapnode.h> 273 #include <sys/dnlc.h> 274 #endif 275 #include <sys/callb.h> 276 #include <sys/kstat.h> 277 #include <sys/zthr.h> 278 #include <zfs_fletcher.h> 279 #include <sys/aggsum.h> 280 #include <sys/cityhash.h> 281 282 #ifndef _KERNEL 283 /* set with ZFS_DEBUG=watch, to enable watchpoints on frozen buffers */ 284 boolean_t arc_watch = B_FALSE; 285 int arc_procfd; 286 #endif 287 288 /* 289 * This thread's job is to keep enough free memory in the system, by 290 * calling arc_kmem_reap_now() plus arc_shrink(), which improves 291 * arc_available_memory(). 292 */ 293 static zthr_t *arc_reap_zthr; 294 295 /* 296 * This thread's job is to keep arc_size under arc_c, by calling 297 * arc_adjust(), which improves arc_is_overflowing(). 298 */ 299 static zthr_t *arc_adjust_zthr; 300 301 static kmutex_t arc_adjust_lock; 302 static kcondvar_t arc_adjust_waiters_cv; 303 static boolean_t arc_adjust_needed = B_FALSE; 304 305 uint_t arc_reduce_dnlc_percent = 3; 306 307 /* 308 * The number of headers to evict in arc_evict_state_impl() before 309 * dropping the sublist lock and evicting from another sublist. A lower 310 * value means we're more likely to evict the "correct" header (i.e. the 311 * oldest header in the arc state), but comes with higher overhead 312 * (i.e. more invocations of arc_evict_state_impl()). 313 */ 314 int zfs_arc_evict_batch_limit = 10; 315 316 /* number of seconds before growing cache again */ 317 int arc_grow_retry = 60; 318 319 /* 320 * Minimum time between calls to arc_kmem_reap_soon(). Note that this will 321 * be converted to ticks, so with the default hz=100, a setting of 15 ms 322 * will actually wait 2 ticks, or 20ms. 323 */ 324 int arc_kmem_cache_reap_retry_ms = 1000; 325 326 /* shift of arc_c for calculating overflow limit in arc_get_data_impl */ 327 int zfs_arc_overflow_shift = 8; 328 329 /* shift of arc_c for calculating both min and max arc_p */ 330 int arc_p_min_shift = 4; 331 332 /* log2(fraction of arc to reclaim) */ 333 int arc_shrink_shift = 7; 334 335 /* 336 * log2(fraction of ARC which must be free to allow growing). 337 * I.e. If there is less than arc_c >> arc_no_grow_shift free memory, 338 * when reading a new block into the ARC, we will evict an equal-sized block 339 * from the ARC. 340 * 341 * This must be less than arc_shrink_shift, so that when we shrink the ARC, 342 * we will still not allow it to grow. 343 */ 344 int arc_no_grow_shift = 5; 345 346 347 /* 348 * minimum lifespan of a prefetch block in clock ticks 349 * (initialized in arc_init()) 350 */ 351 static int arc_min_prefetch_lifespan; 352 353 /* 354 * If this percent of memory is free, don't throttle. 355 */ 356 int arc_lotsfree_percent = 10; 357 358 static boolean_t arc_initialized; 359 360 /* 361 * The arc has filled available memory and has now warmed up. 362 */ 363 static boolean_t arc_warm; 364 365 /* 366 * log2 fraction of the zio arena to keep free. 367 */ 368 int arc_zio_arena_free_shift = 2; 369 370 /* 371 * These tunables are for performance analysis. 372 */ 373 uint64_t zfs_arc_max; 374 uint64_t zfs_arc_min; 375 uint64_t zfs_arc_meta_limit = 0; 376 uint64_t zfs_arc_meta_min = 0; 377 int zfs_arc_grow_retry = 0; 378 int zfs_arc_shrink_shift = 0; 379 int zfs_arc_p_min_shift = 0; 380 int zfs_arc_average_blocksize = 8 * 1024; /* 8KB */ 381 382 /* 383 * ARC dirty data constraints for arc_tempreserve_space() throttle 384 */ 385 uint_t zfs_arc_dirty_limit_percent = 50; /* total dirty data limit */ 386 uint_t zfs_arc_anon_limit_percent = 25; /* anon block dirty limit */ 387 uint_t zfs_arc_pool_dirty_percent = 20; /* each pool's anon allowance */ 388 389 boolean_t zfs_compressed_arc_enabled = B_TRUE; 390 391 /* 392 * Note that buffers can be in one of 6 states: 393 * ARC_anon - anonymous (discussed below) 394 * ARC_mru - recently used, currently cached 395 * ARC_mru_ghost - recentely used, no longer in cache 396 * ARC_mfu - frequently used, currently cached 397 * ARC_mfu_ghost - frequently used, no longer in cache 398 * ARC_l2c_only - exists in L2ARC but not other states 399 * When there are no active references to the buffer, they are 400 * are linked onto a list in one of these arc states. These are 401 * the only buffers that can be evicted or deleted. Within each 402 * state there are multiple lists, one for meta-data and one for 403 * non-meta-data. Meta-data (indirect blocks, blocks of dnodes, 404 * etc.) is tracked separately so that it can be managed more 405 * explicitly: favored over data, limited explicitly. 406 * 407 * Anonymous buffers are buffers that are not associated with 408 * a DVA. These are buffers that hold dirty block copies 409 * before they are written to stable storage. By definition, 410 * they are "ref'd" and are considered part of arc_mru 411 * that cannot be freed. Generally, they will aquire a DVA 412 * as they are written and migrate onto the arc_mru list. 413 * 414 * The ARC_l2c_only state is for buffers that are in the second 415 * level ARC but no longer in any of the ARC_m* lists. The second 416 * level ARC itself may also contain buffers that are in any of 417 * the ARC_m* states - meaning that a buffer can exist in two 418 * places. The reason for the ARC_l2c_only state is to keep the 419 * buffer header in the hash table, so that reads that hit the 420 * second level ARC benefit from these fast lookups. 421 */ 422 423 typedef struct arc_state { 424 /* 425 * list of evictable buffers 426 */ 427 multilist_t *arcs_list[ARC_BUFC_NUMTYPES]; 428 /* 429 * total amount of evictable data in this state 430 */ 431 zfs_refcount_t arcs_esize[ARC_BUFC_NUMTYPES]; 432 /* 433 * total amount of data in this state; this includes: evictable, 434 * non-evictable, ARC_BUFC_DATA, and ARC_BUFC_METADATA. 435 */ 436 zfs_refcount_t arcs_size; 437 } arc_state_t; 438 439 /* The 6 states: */ 440 static arc_state_t ARC_anon; 441 static arc_state_t ARC_mru; 442 static arc_state_t ARC_mru_ghost; 443 static arc_state_t ARC_mfu; 444 static arc_state_t ARC_mfu_ghost; 445 static arc_state_t ARC_l2c_only; 446 447 typedef struct arc_stats { 448 kstat_named_t arcstat_hits; 449 kstat_named_t arcstat_misses; 450 kstat_named_t arcstat_demand_data_hits; 451 kstat_named_t arcstat_demand_data_misses; 452 kstat_named_t arcstat_demand_metadata_hits; 453 kstat_named_t arcstat_demand_metadata_misses; 454 kstat_named_t arcstat_prefetch_data_hits; 455 kstat_named_t arcstat_prefetch_data_misses; 456 kstat_named_t arcstat_prefetch_metadata_hits; 457 kstat_named_t arcstat_prefetch_metadata_misses; 458 kstat_named_t arcstat_mru_hits; 459 kstat_named_t arcstat_mru_ghost_hits; 460 kstat_named_t arcstat_mfu_hits; 461 kstat_named_t arcstat_mfu_ghost_hits; 462 kstat_named_t arcstat_deleted; 463 /* 464 * Number of buffers that could not be evicted because the hash lock 465 * was held by another thread. The lock may not necessarily be held 466 * by something using the same buffer, since hash locks are shared 467 * by multiple buffers. 468 */ 469 kstat_named_t arcstat_mutex_miss; 470 /* 471 * Number of buffers skipped when updating the access state due to the 472 * header having already been released after acquiring the hash lock. 473 */ 474 kstat_named_t arcstat_access_skip; 475 /* 476 * Number of buffers skipped because they have I/O in progress, are 477 * indirect prefetch buffers that have not lived long enough, or are 478 * not from the spa we're trying to evict from. 479 */ 480 kstat_named_t arcstat_evict_skip; 481 /* 482 * Number of times arc_evict_state() was unable to evict enough 483 * buffers to reach it's target amount. 484 */ 485 kstat_named_t arcstat_evict_not_enough; 486 kstat_named_t arcstat_evict_l2_cached; 487 kstat_named_t arcstat_evict_l2_eligible; 488 kstat_named_t arcstat_evict_l2_ineligible; 489 kstat_named_t arcstat_evict_l2_skip; 490 kstat_named_t arcstat_hash_elements; 491 kstat_named_t arcstat_hash_elements_max; 492 kstat_named_t arcstat_hash_collisions; 493 kstat_named_t arcstat_hash_chains; 494 kstat_named_t arcstat_hash_chain_max; 495 kstat_named_t arcstat_p; 496 kstat_named_t arcstat_c; 497 kstat_named_t arcstat_c_min; 498 kstat_named_t arcstat_c_max; 499 /* Not updated directly; only synced in arc_kstat_update. */ 500 kstat_named_t arcstat_size; 501 /* 502 * Number of compressed bytes stored in the arc_buf_hdr_t's b_pabd. 503 * Note that the compressed bytes may match the uncompressed bytes 504 * if the block is either not compressed or compressed arc is disabled. 505 */ 506 kstat_named_t arcstat_compressed_size; 507 /* 508 * Uncompressed size of the data stored in b_pabd. If compressed 509 * arc is disabled then this value will be identical to the stat 510 * above. 511 */ 512 kstat_named_t arcstat_uncompressed_size; 513 /* 514 * Number of bytes stored in all the arc_buf_t's. This is classified 515 * as "overhead" since this data is typically short-lived and will 516 * be evicted from the arc when it becomes unreferenced unless the 517 * zfs_keep_uncompressed_metadata or zfs_keep_uncompressed_level 518 * values have been set (see comment in dbuf.c for more information). 519 */ 520 kstat_named_t arcstat_overhead_size; 521 /* 522 * Number of bytes consumed by internal ARC structures necessary 523 * for tracking purposes; these structures are not actually 524 * backed by ARC buffers. This includes arc_buf_hdr_t structures 525 * (allocated via arc_buf_hdr_t_full and arc_buf_hdr_t_l2only 526 * caches), and arc_buf_t structures (allocated via arc_buf_t 527 * cache). 528 * Not updated directly; only synced in arc_kstat_update. 529 */ 530 kstat_named_t arcstat_hdr_size; 531 /* 532 * Number of bytes consumed by ARC buffers of type equal to 533 * ARC_BUFC_DATA. This is generally consumed by buffers backing 534 * on disk user data (e.g. plain file contents). 535 * Not updated directly; only synced in arc_kstat_update. 536 */ 537 kstat_named_t arcstat_data_size; 538 /* 539 * Number of bytes consumed by ARC buffers of type equal to 540 * ARC_BUFC_METADATA. This is generally consumed by buffers 541 * backing on disk data that is used for internal ZFS 542 * structures (e.g. ZAP, dnode, indirect blocks, etc). 543 * Not updated directly; only synced in arc_kstat_update. 544 */ 545 kstat_named_t arcstat_metadata_size; 546 /* 547 * Number of bytes consumed by various buffers and structures 548 * not actually backed with ARC buffers. This includes bonus 549 * buffers (allocated directly via zio_buf_* functions), 550 * dmu_buf_impl_t structures (allocated via dmu_buf_impl_t 551 * cache), and dnode_t structures (allocated via dnode_t cache). 552 * Not updated directly; only synced in arc_kstat_update. 553 */ 554 kstat_named_t arcstat_other_size; 555 /* 556 * Total number of bytes consumed by ARC buffers residing in the 557 * arc_anon state. This includes *all* buffers in the arc_anon 558 * state; e.g. data, metadata, evictable, and unevictable buffers 559 * are all included in this value. 560 * Not updated directly; only synced in arc_kstat_update. 561 */ 562 kstat_named_t arcstat_anon_size; 563 /* 564 * Number of bytes consumed by ARC buffers that meet the 565 * following criteria: backing buffers of type ARC_BUFC_DATA, 566 * residing in the arc_anon state, and are eligible for eviction 567 * (e.g. have no outstanding holds on the buffer). 568 * Not updated directly; only synced in arc_kstat_update. 569 */ 570 kstat_named_t arcstat_anon_evictable_data; 571 /* 572 * Number of bytes consumed by ARC buffers that meet the 573 * following criteria: backing buffers of type ARC_BUFC_METADATA, 574 * residing in the arc_anon state, and are eligible for eviction 575 * (e.g. have no outstanding holds on the buffer). 576 * Not updated directly; only synced in arc_kstat_update. 577 */ 578 kstat_named_t arcstat_anon_evictable_metadata; 579 /* 580 * Total number of bytes consumed by ARC buffers residing in the 581 * arc_mru state. This includes *all* buffers in the arc_mru 582 * state; e.g. data, metadata, evictable, and unevictable buffers 583 * are all included in this value. 584 * Not updated directly; only synced in arc_kstat_update. 585 */ 586 kstat_named_t arcstat_mru_size; 587 /* 588 * Number of bytes consumed by ARC buffers that meet the 589 * following criteria: backing buffers of type ARC_BUFC_DATA, 590 * residing in the arc_mru state, and are eligible for eviction 591 * (e.g. have no outstanding holds on the buffer). 592 * Not updated directly; only synced in arc_kstat_update. 593 */ 594 kstat_named_t arcstat_mru_evictable_data; 595 /* 596 * Number of bytes consumed by ARC buffers that meet the 597 * following criteria: backing buffers of type ARC_BUFC_METADATA, 598 * residing in the arc_mru state, and are eligible for eviction 599 * (e.g. have no outstanding holds on the buffer). 600 * Not updated directly; only synced in arc_kstat_update. 601 */ 602 kstat_named_t arcstat_mru_evictable_metadata; 603 /* 604 * Total number of bytes that *would have been* consumed by ARC 605 * buffers in the arc_mru_ghost state. The key thing to note 606 * here, is the fact that this size doesn't actually indicate 607 * RAM consumption. The ghost lists only consist of headers and 608 * don't actually have ARC buffers linked off of these headers. 609 * Thus, *if* the headers had associated ARC buffers, these 610 * buffers *would have* consumed this number of bytes. 611 * Not updated directly; only synced in arc_kstat_update. 612 */ 613 kstat_named_t arcstat_mru_ghost_size; 614 /* 615 * Number of bytes that *would have been* consumed by ARC 616 * buffers that are eligible for eviction, of type 617 * ARC_BUFC_DATA, and linked off the arc_mru_ghost state. 618 * Not updated directly; only synced in arc_kstat_update. 619 */ 620 kstat_named_t arcstat_mru_ghost_evictable_data; 621 /* 622 * Number of bytes that *would have been* consumed by ARC 623 * buffers that are eligible for eviction, of type 624 * ARC_BUFC_METADATA, and linked off the arc_mru_ghost state. 625 * Not updated directly; only synced in arc_kstat_update. 626 */ 627 kstat_named_t arcstat_mru_ghost_evictable_metadata; 628 /* 629 * Total number of bytes consumed by ARC buffers residing in the 630 * arc_mfu state. This includes *all* buffers in the arc_mfu 631 * state; e.g. data, metadata, evictable, and unevictable buffers 632 * are all included in this value. 633 * Not updated directly; only synced in arc_kstat_update. 634 */ 635 kstat_named_t arcstat_mfu_size; 636 /* 637 * Number of bytes consumed by ARC buffers that are eligible for 638 * eviction, of type ARC_BUFC_DATA, and reside in the arc_mfu 639 * state. 640 * Not updated directly; only synced in arc_kstat_update. 641 */ 642 kstat_named_t arcstat_mfu_evictable_data; 643 /* 644 * Number of bytes consumed by ARC buffers that are eligible for 645 * eviction, of type ARC_BUFC_METADATA, and reside in the 646 * arc_mfu state. 647 * Not updated directly; only synced in arc_kstat_update. 648 */ 649 kstat_named_t arcstat_mfu_evictable_metadata; 650 /* 651 * Total number of bytes that *would have been* consumed by ARC 652 * buffers in the arc_mfu_ghost state. See the comment above 653 * arcstat_mru_ghost_size for more details. 654 * Not updated directly; only synced in arc_kstat_update. 655 */ 656 kstat_named_t arcstat_mfu_ghost_size; 657 /* 658 * Number of bytes that *would have been* consumed by ARC 659 * buffers that are eligible for eviction, of type 660 * ARC_BUFC_DATA, and linked off the arc_mfu_ghost state. 661 * Not updated directly; only synced in arc_kstat_update. 662 */ 663 kstat_named_t arcstat_mfu_ghost_evictable_data; 664 /* 665 * Number of bytes that *would have been* consumed by ARC 666 * buffers that are eligible for eviction, of type 667 * ARC_BUFC_METADATA, and linked off the arc_mru_ghost state. 668 * Not updated directly; only synced in arc_kstat_update. 669 */ 670 kstat_named_t arcstat_mfu_ghost_evictable_metadata; 671 kstat_named_t arcstat_l2_hits; 672 kstat_named_t arcstat_l2_misses; 673 kstat_named_t arcstat_l2_feeds; 674 kstat_named_t arcstat_l2_rw_clash; 675 kstat_named_t arcstat_l2_read_bytes; 676 kstat_named_t arcstat_l2_write_bytes; 677 kstat_named_t arcstat_l2_writes_sent; 678 kstat_named_t arcstat_l2_writes_done; 679 kstat_named_t arcstat_l2_writes_error; 680 kstat_named_t arcstat_l2_writes_lock_retry; 681 kstat_named_t arcstat_l2_evict_lock_retry; 682 kstat_named_t arcstat_l2_evict_reading; 683 kstat_named_t arcstat_l2_evict_l1cached; 684 kstat_named_t arcstat_l2_free_on_write; 685 kstat_named_t arcstat_l2_abort_lowmem; 686 kstat_named_t arcstat_l2_cksum_bad; 687 kstat_named_t arcstat_l2_io_error; 688 kstat_named_t arcstat_l2_lsize; 689 kstat_named_t arcstat_l2_psize; 690 /* Not updated directly; only synced in arc_kstat_update. */ 691 kstat_named_t arcstat_l2_hdr_size; 692 kstat_named_t arcstat_memory_throttle_count; 693 /* Not updated directly; only synced in arc_kstat_update. */ 694 kstat_named_t arcstat_meta_used; 695 kstat_named_t arcstat_meta_limit; 696 kstat_named_t arcstat_meta_max; 697 kstat_named_t arcstat_meta_min; 698 kstat_named_t arcstat_sync_wait_for_async; 699 kstat_named_t arcstat_demand_hit_predictive_prefetch; 700 } arc_stats_t; 701 702 static arc_stats_t arc_stats = { 703 { "hits", KSTAT_DATA_UINT64 }, 704 { "misses", KSTAT_DATA_UINT64 }, 705 { "demand_data_hits", KSTAT_DATA_UINT64 }, 706 { "demand_data_misses", KSTAT_DATA_UINT64 }, 707 { "demand_metadata_hits", KSTAT_DATA_UINT64 }, 708 { "demand_metadata_misses", KSTAT_DATA_UINT64 }, 709 { "prefetch_data_hits", KSTAT_DATA_UINT64 }, 710 { "prefetch_data_misses", KSTAT_DATA_UINT64 }, 711 { "prefetch_metadata_hits", KSTAT_DATA_UINT64 }, 712 { "prefetch_metadata_misses", KSTAT_DATA_UINT64 }, 713 { "mru_hits", KSTAT_DATA_UINT64 }, 714 { "mru_ghost_hits", KSTAT_DATA_UINT64 }, 715 { "mfu_hits", KSTAT_DATA_UINT64 }, 716 { "mfu_ghost_hits", KSTAT_DATA_UINT64 }, 717 { "deleted", KSTAT_DATA_UINT64 }, 718 { "mutex_miss", KSTAT_DATA_UINT64 }, 719 { "access_skip", KSTAT_DATA_UINT64 }, 720 { "evict_skip", KSTAT_DATA_UINT64 }, 721 { "evict_not_enough", KSTAT_DATA_UINT64 }, 722 { "evict_l2_cached", KSTAT_DATA_UINT64 }, 723 { "evict_l2_eligible", KSTAT_DATA_UINT64 }, 724 { "evict_l2_ineligible", KSTAT_DATA_UINT64 }, 725 { "evict_l2_skip", KSTAT_DATA_UINT64 }, 726 { "hash_elements", KSTAT_DATA_UINT64 }, 727 { "hash_elements_max", KSTAT_DATA_UINT64 }, 728 { "hash_collisions", KSTAT_DATA_UINT64 }, 729 { "hash_chains", KSTAT_DATA_UINT64 }, 730 { "hash_chain_max", KSTAT_DATA_UINT64 }, 731 { "p", KSTAT_DATA_UINT64 }, 732 { "c", KSTAT_DATA_UINT64 }, 733 { "c_min", KSTAT_DATA_UINT64 }, 734 { "c_max", KSTAT_DATA_UINT64 }, 735 { "size", KSTAT_DATA_UINT64 }, 736 { "compressed_size", KSTAT_DATA_UINT64 }, 737 { "uncompressed_size", KSTAT_DATA_UINT64 }, 738 { "overhead_size", KSTAT_DATA_UINT64 }, 739 { "hdr_size", KSTAT_DATA_UINT64 }, 740 { "data_size", KSTAT_DATA_UINT64 }, 741 { "metadata_size", KSTAT_DATA_UINT64 }, 742 { "other_size", KSTAT_DATA_UINT64 }, 743 { "anon_size", KSTAT_DATA_UINT64 }, 744 { "anon_evictable_data", KSTAT_DATA_UINT64 }, 745 { "anon_evictable_metadata", KSTAT_DATA_UINT64 }, 746 { "mru_size", KSTAT_DATA_UINT64 }, 747 { "mru_evictable_data", KSTAT_DATA_UINT64 }, 748 { "mru_evictable_metadata", KSTAT_DATA_UINT64 }, 749 { "mru_ghost_size", KSTAT_DATA_UINT64 }, 750 { "mru_ghost_evictable_data", KSTAT_DATA_UINT64 }, 751 { "mru_ghost_evictable_metadata", KSTAT_DATA_UINT64 }, 752 { "mfu_size", KSTAT_DATA_UINT64 }, 753 { "mfu_evictable_data", KSTAT_DATA_UINT64 }, 754 { "mfu_evictable_metadata", KSTAT_DATA_UINT64 }, 755 { "mfu_ghost_size", KSTAT_DATA_UINT64 }, 756 { "mfu_ghost_evictable_data", KSTAT_DATA_UINT64 }, 757 { "mfu_ghost_evictable_metadata", KSTAT_DATA_UINT64 }, 758 { "l2_hits", KSTAT_DATA_UINT64 }, 759 { "l2_misses", KSTAT_DATA_UINT64 }, 760 { "l2_feeds", KSTAT_DATA_UINT64 }, 761 { "l2_rw_clash", KSTAT_DATA_UINT64 }, 762 { "l2_read_bytes", KSTAT_DATA_UINT64 }, 763 { "l2_write_bytes", KSTAT_DATA_UINT64 }, 764 { "l2_writes_sent", KSTAT_DATA_UINT64 }, 765 { "l2_writes_done", KSTAT_DATA_UINT64 }, 766 { "l2_writes_error", KSTAT_DATA_UINT64 }, 767 { "l2_writes_lock_retry", KSTAT_DATA_UINT64 }, 768 { "l2_evict_lock_retry", KSTAT_DATA_UINT64 }, 769 { "l2_evict_reading", KSTAT_DATA_UINT64 }, 770 { "l2_evict_l1cached", KSTAT_DATA_UINT64 }, 771 { "l2_free_on_write", KSTAT_DATA_UINT64 }, 772 { "l2_abort_lowmem", KSTAT_DATA_UINT64 }, 773 { "l2_cksum_bad", KSTAT_DATA_UINT64 }, 774 { "l2_io_error", KSTAT_DATA_UINT64 }, 775 { "l2_size", KSTAT_DATA_UINT64 }, 776 { "l2_asize", KSTAT_DATA_UINT64 }, 777 { "l2_hdr_size", KSTAT_DATA_UINT64 }, 778 { "memory_throttle_count", KSTAT_DATA_UINT64 }, 779 { "arc_meta_used", KSTAT_DATA_UINT64 }, 780 { "arc_meta_limit", KSTAT_DATA_UINT64 }, 781 { "arc_meta_max", KSTAT_DATA_UINT64 }, 782 { "arc_meta_min", KSTAT_DATA_UINT64 }, 783 { "sync_wait_for_async", KSTAT_DATA_UINT64 }, 784 { "demand_hit_predictive_prefetch", KSTAT_DATA_UINT64 }, 785 }; 786 787 #define ARCSTAT(stat) (arc_stats.stat.value.ui64) 788 789 #define ARCSTAT_INCR(stat, val) \ 790 atomic_add_64(&arc_stats.stat.value.ui64, (val)) 791 792 #define ARCSTAT_BUMP(stat) ARCSTAT_INCR(stat, 1) 793 #define ARCSTAT_BUMPDOWN(stat) ARCSTAT_INCR(stat, -1) 794 795 #define ARCSTAT_MAX(stat, val) { \ 796 uint64_t m; \ 797 while ((val) > (m = arc_stats.stat.value.ui64) && \ 798 (m != atomic_cas_64(&arc_stats.stat.value.ui64, m, (val)))) \ 799 continue; \ 800 } 801 802 #define ARCSTAT_MAXSTAT(stat) \ 803 ARCSTAT_MAX(stat##_max, arc_stats.stat.value.ui64) 804 805 /* 806 * We define a macro to allow ARC hits/misses to be easily broken down by 807 * two separate conditions, giving a total of four different subtypes for 808 * each of hits and misses (so eight statistics total). 809 */ 810 #define ARCSTAT_CONDSTAT(cond1, stat1, notstat1, cond2, stat2, notstat2, stat) \ 811 if (cond1) { \ 812 if (cond2) { \ 813 ARCSTAT_BUMP(arcstat_##stat1##_##stat2##_##stat); \ 814 } else { \ 815 ARCSTAT_BUMP(arcstat_##stat1##_##notstat2##_##stat); \ 816 } \ 817 } else { \ 818 if (cond2) { \ 819 ARCSTAT_BUMP(arcstat_##notstat1##_##stat2##_##stat); \ 820 } else { \ 821 ARCSTAT_BUMP(arcstat_##notstat1##_##notstat2##_##stat);\ 822 } \ 823 } 824 825 kstat_t *arc_ksp; 826 static arc_state_t *arc_anon; 827 static arc_state_t *arc_mru; 828 static arc_state_t *arc_mru_ghost; 829 static arc_state_t *arc_mfu; 830 static arc_state_t *arc_mfu_ghost; 831 static arc_state_t *arc_l2c_only; 832 833 /* 834 * There are several ARC variables that are critical to export as kstats -- 835 * but we don't want to have to grovel around in the kstat whenever we wish to 836 * manipulate them. For these variables, we therefore define them to be in 837 * terms of the statistic variable. This assures that we are not introducing 838 * the possibility of inconsistency by having shadow copies of the variables, 839 * while still allowing the code to be readable. 840 */ 841 #define arc_p ARCSTAT(arcstat_p) /* target size of MRU */ 842 #define arc_c ARCSTAT(arcstat_c) /* target size of cache */ 843 #define arc_c_min ARCSTAT(arcstat_c_min) /* min target cache size */ 844 #define arc_c_max ARCSTAT(arcstat_c_max) /* max target cache size */ 845 #define arc_meta_limit ARCSTAT(arcstat_meta_limit) /* max size for metadata */ 846 #define arc_meta_min ARCSTAT(arcstat_meta_min) /* min size for metadata */ 847 #define arc_meta_max ARCSTAT(arcstat_meta_max) /* max size of metadata */ 848 849 /* compressed size of entire arc */ 850 #define arc_compressed_size ARCSTAT(arcstat_compressed_size) 851 /* uncompressed size of entire arc */ 852 #define arc_uncompressed_size ARCSTAT(arcstat_uncompressed_size) 853 /* number of bytes in the arc from arc_buf_t's */ 854 #define arc_overhead_size ARCSTAT(arcstat_overhead_size) 855 856 /* 857 * There are also some ARC variables that we want to export, but that are 858 * updated so often that having the canonical representation be the statistic 859 * variable causes a performance bottleneck. We want to use aggsum_t's for these 860 * instead, but still be able to export the kstat in the same way as before. 861 * The solution is to always use the aggsum version, except in the kstat update 862 * callback. 863 */ 864 aggsum_t arc_size; 865 aggsum_t arc_meta_used; 866 aggsum_t astat_data_size; 867 aggsum_t astat_metadata_size; 868 aggsum_t astat_hdr_size; 869 aggsum_t astat_other_size; 870 aggsum_t astat_l2_hdr_size; 871 872 static int arc_no_grow; /* Don't try to grow cache size */ 873 static hrtime_t arc_growtime; 874 static uint64_t arc_tempreserve; 875 static uint64_t arc_loaned_bytes; 876 877 typedef struct arc_callback arc_callback_t; 878 879 struct arc_callback { 880 void *acb_private; 881 arc_done_func_t *acb_done; 882 arc_buf_t *acb_buf; 883 boolean_t acb_compressed; 884 zio_t *acb_zio_dummy; 885 arc_callback_t *acb_next; 886 }; 887 888 typedef struct arc_write_callback arc_write_callback_t; 889 890 struct arc_write_callback { 891 void *awcb_private; 892 arc_done_func_t *awcb_ready; 893 arc_done_func_t *awcb_children_ready; 894 arc_done_func_t *awcb_physdone; 895 arc_done_func_t *awcb_done; 896 arc_buf_t *awcb_buf; 897 }; 898 899 /* 900 * ARC buffers are separated into multiple structs as a memory saving measure: 901 * - Common fields struct, always defined, and embedded within it: 902 * - L2-only fields, always allocated but undefined when not in L2ARC 903 * - L1-only fields, only allocated when in L1ARC 904 * 905 * Buffer in L1 Buffer only in L2 906 * +------------------------+ +------------------------+ 907 * | arc_buf_hdr_t | | arc_buf_hdr_t | 908 * | | | | 909 * | | | | 910 * | | | | 911 * +------------------------+ +------------------------+ 912 * | l2arc_buf_hdr_t | | l2arc_buf_hdr_t | 913 * | (undefined if L1-only) | | | 914 * +------------------------+ +------------------------+ 915 * | l1arc_buf_hdr_t | 916 * | | 917 * | | 918 * | | 919 * | | 920 * +------------------------+ 921 * 922 * Because it's possible for the L2ARC to become extremely large, we can wind 923 * up eating a lot of memory in L2ARC buffer headers, so the size of a header 924 * is minimized by only allocating the fields necessary for an L1-cached buffer 925 * when a header is actually in the L1 cache. The sub-headers (l1arc_buf_hdr and 926 * l2arc_buf_hdr) are embedded rather than allocated separately to save a couple 927 * words in pointers. arc_hdr_realloc() is used to switch a header between 928 * these two allocation states. 929 */ 930 typedef struct l1arc_buf_hdr { 931 kmutex_t b_freeze_lock; 932 zio_cksum_t *b_freeze_cksum; 933 #ifdef ZFS_DEBUG 934 /* 935 * Used for debugging with kmem_flags - by allocating and freeing 936 * b_thawed when the buffer is thawed, we get a record of the stack 937 * trace that thawed it. 938 */ 939 void *b_thawed; 940 #endif 941 942 arc_buf_t *b_buf; 943 uint32_t b_bufcnt; 944 /* for waiting on writes to complete */ 945 kcondvar_t b_cv; 946 uint8_t b_byteswap; 947 948 /* protected by arc state mutex */ 949 arc_state_t *b_state; 950 multilist_node_t b_arc_node; 951 952 /* updated atomically */ 953 clock_t b_arc_access; 954 955 /* self protecting */ 956 zfs_refcount_t b_refcnt; 957 958 arc_callback_t *b_acb; 959 abd_t *b_pabd; 960 } l1arc_buf_hdr_t; 961 962 typedef struct l2arc_dev l2arc_dev_t; 963 964 typedef struct l2arc_buf_hdr { 965 /* protected by arc_buf_hdr mutex */ 966 l2arc_dev_t *b_dev; /* L2ARC device */ 967 uint64_t b_daddr; /* disk address, offset byte */ 968 969 list_node_t b_l2node; 970 } l2arc_buf_hdr_t; 971 972 struct arc_buf_hdr { 973 /* protected by hash lock */ 974 dva_t b_dva; 975 uint64_t b_birth; 976 977 arc_buf_contents_t b_type; 978 arc_buf_hdr_t *b_hash_next; 979 arc_flags_t b_flags; 980 981 /* 982 * This field stores the size of the data buffer after 983 * compression, and is set in the arc's zio completion handlers. 984 * It is in units of SPA_MINBLOCKSIZE (e.g. 1 == 512 bytes). 985 * 986 * While the block pointers can store up to 32MB in their psize 987 * field, we can only store up to 32MB minus 512B. This is due 988 * to the bp using a bias of 1, whereas we use a bias of 0 (i.e. 989 * a field of zeros represents 512B in the bp). We can't use a 990 * bias of 1 since we need to reserve a psize of zero, here, to 991 * represent holes and embedded blocks. 992 * 993 * This isn't a problem in practice, since the maximum size of a 994 * buffer is limited to 16MB, so we never need to store 32MB in 995 * this field. Even in the upstream illumos code base, the 996 * maximum size of a buffer is limited to 16MB. 997 */ 998 uint16_t b_psize; 999 1000 /* 1001 * This field stores the size of the data buffer before 1002 * compression, and cannot change once set. It is in units 1003 * of SPA_MINBLOCKSIZE (e.g. 2 == 1024 bytes) 1004 */ 1005 uint16_t b_lsize; /* immutable */ 1006 uint64_t b_spa; /* immutable */ 1007 1008 /* L2ARC fields. Undefined when not in L2ARC. */ 1009 l2arc_buf_hdr_t b_l2hdr; 1010 /* L1ARC fields. Undefined when in l2arc_only state */ 1011 l1arc_buf_hdr_t b_l1hdr; 1012 }; 1013 1014 #define GHOST_STATE(state) \ 1015 ((state) == arc_mru_ghost || (state) == arc_mfu_ghost || \ 1016 (state) == arc_l2c_only) 1017 1018 #define HDR_IN_HASH_TABLE(hdr) ((hdr)->b_flags & ARC_FLAG_IN_HASH_TABLE) 1019 #define HDR_IO_IN_PROGRESS(hdr) ((hdr)->b_flags & ARC_FLAG_IO_IN_PROGRESS) 1020 #define HDR_IO_ERROR(hdr) ((hdr)->b_flags & ARC_FLAG_IO_ERROR) 1021 #define HDR_PREFETCH(hdr) ((hdr)->b_flags & ARC_FLAG_PREFETCH) 1022 #define HDR_COMPRESSION_ENABLED(hdr) \ 1023 ((hdr)->b_flags & ARC_FLAG_COMPRESSED_ARC) 1024 1025 #define HDR_L2CACHE(hdr) ((hdr)->b_flags & ARC_FLAG_L2CACHE) 1026 #define HDR_L2_READING(hdr) \ 1027 (((hdr)->b_flags & ARC_FLAG_IO_IN_PROGRESS) && \ 1028 ((hdr)->b_flags & ARC_FLAG_HAS_L2HDR)) 1029 #define HDR_L2_WRITING(hdr) ((hdr)->b_flags & ARC_FLAG_L2_WRITING) 1030 #define HDR_L2_EVICTED(hdr) ((hdr)->b_flags & ARC_FLAG_L2_EVICTED) 1031 #define HDR_L2_WRITE_HEAD(hdr) ((hdr)->b_flags & ARC_FLAG_L2_WRITE_HEAD) 1032 #define HDR_SHARED_DATA(hdr) ((hdr)->b_flags & ARC_FLAG_SHARED_DATA) 1033 1034 #define HDR_ISTYPE_METADATA(hdr) \ 1035 ((hdr)->b_flags & ARC_FLAG_BUFC_METADATA) 1036 #define HDR_ISTYPE_DATA(hdr) (!HDR_ISTYPE_METADATA(hdr)) 1037 1038 #define HDR_HAS_L1HDR(hdr) ((hdr)->b_flags & ARC_FLAG_HAS_L1HDR) 1039 #define HDR_HAS_L2HDR(hdr) ((hdr)->b_flags & ARC_FLAG_HAS_L2HDR) 1040 1041 /* For storing compression mode in b_flags */ 1042 #define HDR_COMPRESS_OFFSET (highbit64(ARC_FLAG_COMPRESS_0) - 1) 1043 1044 #define HDR_GET_COMPRESS(hdr) ((enum zio_compress)BF32_GET((hdr)->b_flags, \ 1045 HDR_COMPRESS_OFFSET, SPA_COMPRESSBITS)) 1046 #define HDR_SET_COMPRESS(hdr, cmp) BF32_SET((hdr)->b_flags, \ 1047 HDR_COMPRESS_OFFSET, SPA_COMPRESSBITS, (cmp)); 1048 1049 #define ARC_BUF_LAST(buf) ((buf)->b_next == NULL) 1050 #define ARC_BUF_SHARED(buf) ((buf)->b_flags & ARC_BUF_FLAG_SHARED) 1051 #define ARC_BUF_COMPRESSED(buf) ((buf)->b_flags & ARC_BUF_FLAG_COMPRESSED) 1052 1053 /* 1054 * Other sizes 1055 */ 1056 1057 #define HDR_FULL_SIZE ((int64_t)sizeof (arc_buf_hdr_t)) 1058 #define HDR_L2ONLY_SIZE ((int64_t)offsetof(arc_buf_hdr_t, b_l1hdr)) 1059 1060 /* 1061 * Hash table routines 1062 */ 1063 1064 #define HT_LOCK_PAD 64 1065 1066 struct ht_lock { 1067 kmutex_t ht_lock; 1068 #ifdef _KERNEL 1069 unsigned char pad[(HT_LOCK_PAD - sizeof (kmutex_t))]; 1070 #endif 1071 }; 1072 1073 #define BUF_LOCKS 256 1074 typedef struct buf_hash_table { 1075 uint64_t ht_mask; 1076 arc_buf_hdr_t **ht_table; 1077 struct ht_lock ht_locks[BUF_LOCKS]; 1078 } buf_hash_table_t; 1079 1080 static buf_hash_table_t buf_hash_table; 1081 1082 #define BUF_HASH_INDEX(spa, dva, birth) \ 1083 (buf_hash(spa, dva, birth) & buf_hash_table.ht_mask) 1084 #define BUF_HASH_LOCK_NTRY(idx) (buf_hash_table.ht_locks[idx & (BUF_LOCKS-1)]) 1085 #define BUF_HASH_LOCK(idx) (&(BUF_HASH_LOCK_NTRY(idx).ht_lock)) 1086 #define HDR_LOCK(hdr) \ 1087 (BUF_HASH_LOCK(BUF_HASH_INDEX(hdr->b_spa, &hdr->b_dva, hdr->b_birth))) 1088 1089 uint64_t zfs_crc64_table[256]; 1090 1091 /* 1092 * Level 2 ARC 1093 */ 1094 1095 #define L2ARC_WRITE_SIZE (8 * 1024 * 1024) /* initial write max */ 1096 #define L2ARC_HEADROOM 2 /* num of writes */ 1097 /* 1098 * If we discover during ARC scan any buffers to be compressed, we boost 1099 * our headroom for the next scanning cycle by this percentage multiple. 1100 */ 1101 #define L2ARC_HEADROOM_BOOST 200 1102 #define L2ARC_FEED_SECS 1 /* caching interval secs */ 1103 #define L2ARC_FEED_MIN_MS 200 /* min caching interval ms */ 1104 1105 #define l2arc_writes_sent ARCSTAT(arcstat_l2_writes_sent) 1106 #define l2arc_writes_done ARCSTAT(arcstat_l2_writes_done) 1107 1108 /* L2ARC Performance Tunables */ 1109 uint64_t l2arc_write_max = L2ARC_WRITE_SIZE; /* default max write size */ 1110 uint64_t l2arc_write_boost = L2ARC_WRITE_SIZE; /* extra write during warmup */ 1111 uint64_t l2arc_headroom = L2ARC_HEADROOM; /* number of dev writes */ 1112 uint64_t l2arc_headroom_boost = L2ARC_HEADROOM_BOOST; 1113 uint64_t l2arc_feed_secs = L2ARC_FEED_SECS; /* interval seconds */ 1114 uint64_t l2arc_feed_min_ms = L2ARC_FEED_MIN_MS; /* min interval milliseconds */ 1115 boolean_t l2arc_noprefetch = B_TRUE; /* don't cache prefetch bufs */ 1116 boolean_t l2arc_feed_again = B_TRUE; /* turbo warmup */ 1117 boolean_t l2arc_norw = B_TRUE; /* no reads during writes */ 1118 1119 /* 1120 * L2ARC Internals 1121 */ 1122 struct l2arc_dev { 1123 vdev_t *l2ad_vdev; /* vdev */ 1124 spa_t *l2ad_spa; /* spa */ 1125 uint64_t l2ad_hand; /* next write location */ 1126 uint64_t l2ad_start; /* first addr on device */ 1127 uint64_t l2ad_end; /* last addr on device */ 1128 boolean_t l2ad_first; /* first sweep through */ 1129 boolean_t l2ad_writing; /* currently writing */ 1130 kmutex_t l2ad_mtx; /* lock for buffer list */ 1131 list_t l2ad_buflist; /* buffer list */ 1132 list_node_t l2ad_node; /* device list node */ 1133 zfs_refcount_t l2ad_alloc; /* allocated bytes */ 1134 }; 1135 1136 static list_t L2ARC_dev_list; /* device list */ 1137 static list_t *l2arc_dev_list; /* device list pointer */ 1138 static kmutex_t l2arc_dev_mtx; /* device list mutex */ 1139 static l2arc_dev_t *l2arc_dev_last; /* last device used */ 1140 static list_t L2ARC_free_on_write; /* free after write buf list */ 1141 static list_t *l2arc_free_on_write; /* free after write list ptr */ 1142 static kmutex_t l2arc_free_on_write_mtx; /* mutex for list */ 1143 static uint64_t l2arc_ndev; /* number of devices */ 1144 1145 typedef struct l2arc_read_callback { 1146 arc_buf_hdr_t *l2rcb_hdr; /* read header */ 1147 blkptr_t l2rcb_bp; /* original blkptr */ 1148 zbookmark_phys_t l2rcb_zb; /* original bookmark */ 1149 int l2rcb_flags; /* original flags */ 1150 abd_t *l2rcb_abd; /* temporary buffer */ 1151 } l2arc_read_callback_t; 1152 1153 typedef struct l2arc_write_callback { 1154 l2arc_dev_t *l2wcb_dev; /* device info */ 1155 arc_buf_hdr_t *l2wcb_head; /* head of write buflist */ 1156 } l2arc_write_callback_t; 1157 1158 typedef struct l2arc_data_free { 1159 /* protected by l2arc_free_on_write_mtx */ 1160 abd_t *l2df_abd; 1161 size_t l2df_size; 1162 arc_buf_contents_t l2df_type; 1163 list_node_t l2df_list_node; 1164 } l2arc_data_free_t; 1165 1166 static kmutex_t l2arc_feed_thr_lock; 1167 static kcondvar_t l2arc_feed_thr_cv; 1168 static uint8_t l2arc_thread_exit; 1169 1170 static abd_t *arc_get_data_abd(arc_buf_hdr_t *, uint64_t, void *); 1171 static void *arc_get_data_buf(arc_buf_hdr_t *, uint64_t, void *); 1172 static void arc_get_data_impl(arc_buf_hdr_t *, uint64_t, void *); 1173 static void arc_free_data_abd(arc_buf_hdr_t *, abd_t *, uint64_t, void *); 1174 static void arc_free_data_buf(arc_buf_hdr_t *, void *, uint64_t, void *); 1175 static void arc_free_data_impl(arc_buf_hdr_t *hdr, uint64_t size, void *tag); 1176 static void arc_hdr_free_pabd(arc_buf_hdr_t *); 1177 static void arc_hdr_alloc_pabd(arc_buf_hdr_t *); 1178 static void arc_access(arc_buf_hdr_t *, kmutex_t *); 1179 static boolean_t arc_is_overflowing(); 1180 static void arc_buf_watch(arc_buf_t *); 1181 1182 static arc_buf_contents_t arc_buf_type(arc_buf_hdr_t *); 1183 static uint32_t arc_bufc_to_flags(arc_buf_contents_t); 1184 static inline void arc_hdr_set_flags(arc_buf_hdr_t *hdr, arc_flags_t flags); 1185 static inline void arc_hdr_clear_flags(arc_buf_hdr_t *hdr, arc_flags_t flags); 1186 1187 static boolean_t l2arc_write_eligible(uint64_t, arc_buf_hdr_t *); 1188 static void l2arc_read_done(zio_t *); 1189 1190 1191 /* 1192 * We use Cityhash for this. It's fast, and has good hash properties without 1193 * requiring any large static buffers. 1194 */ 1195 static uint64_t 1196 buf_hash(uint64_t spa, const dva_t *dva, uint64_t birth) 1197 { 1198 return (cityhash4(spa, dva->dva_word[0], dva->dva_word[1], birth)); 1199 } 1200 1201 #define HDR_EMPTY(hdr) \ 1202 ((hdr)->b_dva.dva_word[0] == 0 && \ 1203 (hdr)->b_dva.dva_word[1] == 0) 1204 1205 #define HDR_EQUAL(spa, dva, birth, hdr) \ 1206 ((hdr)->b_dva.dva_word[0] == (dva)->dva_word[0]) && \ 1207 ((hdr)->b_dva.dva_word[1] == (dva)->dva_word[1]) && \ 1208 ((hdr)->b_birth == birth) && ((hdr)->b_spa == spa) 1209 1210 static void 1211 buf_discard_identity(arc_buf_hdr_t *hdr) 1212 { 1213 hdr->b_dva.dva_word[0] = 0; 1214 hdr->b_dva.dva_word[1] = 0; 1215 hdr->b_birth = 0; 1216 } 1217 1218 static arc_buf_hdr_t * 1219 buf_hash_find(uint64_t spa, const blkptr_t *bp, kmutex_t **lockp) 1220 { 1221 const dva_t *dva = BP_IDENTITY(bp); 1222 uint64_t birth = BP_PHYSICAL_BIRTH(bp); 1223 uint64_t idx = BUF_HASH_INDEX(spa, dva, birth); 1224 kmutex_t *hash_lock = BUF_HASH_LOCK(idx); 1225 arc_buf_hdr_t *hdr; 1226 1227 mutex_enter(hash_lock); 1228 for (hdr = buf_hash_table.ht_table[idx]; hdr != NULL; 1229 hdr = hdr->b_hash_next) { 1230 if (HDR_EQUAL(spa, dva, birth, hdr)) { 1231 *lockp = hash_lock; 1232 return (hdr); 1233 } 1234 } 1235 mutex_exit(hash_lock); 1236 *lockp = NULL; 1237 return (NULL); 1238 } 1239 1240 /* 1241 * Insert an entry into the hash table. If there is already an element 1242 * equal to elem in the hash table, then the already existing element 1243 * will be returned and the new element will not be inserted. 1244 * Otherwise returns NULL. 1245 * If lockp == NULL, the caller is assumed to already hold the hash lock. 1246 */ 1247 static arc_buf_hdr_t * 1248 buf_hash_insert(arc_buf_hdr_t *hdr, kmutex_t **lockp) 1249 { 1250 uint64_t idx = BUF_HASH_INDEX(hdr->b_spa, &hdr->b_dva, hdr->b_birth); 1251 kmutex_t *hash_lock = BUF_HASH_LOCK(idx); 1252 arc_buf_hdr_t *fhdr; 1253 uint32_t i; 1254 1255 ASSERT(!DVA_IS_EMPTY(&hdr->b_dva)); 1256 ASSERT(hdr->b_birth != 0); 1257 ASSERT(!HDR_IN_HASH_TABLE(hdr)); 1258 1259 if (lockp != NULL) { 1260 *lockp = hash_lock; 1261 mutex_enter(hash_lock); 1262 } else { 1263 ASSERT(MUTEX_HELD(hash_lock)); 1264 } 1265 1266 for (fhdr = buf_hash_table.ht_table[idx], i = 0; fhdr != NULL; 1267 fhdr = fhdr->b_hash_next, i++) { 1268 if (HDR_EQUAL(hdr->b_spa, &hdr->b_dva, hdr->b_birth, fhdr)) 1269 return (fhdr); 1270 } 1271 1272 hdr->b_hash_next = buf_hash_table.ht_table[idx]; 1273 buf_hash_table.ht_table[idx] = hdr; 1274 arc_hdr_set_flags(hdr, ARC_FLAG_IN_HASH_TABLE); 1275 1276 /* collect some hash table performance data */ 1277 if (i > 0) { 1278 ARCSTAT_BUMP(arcstat_hash_collisions); 1279 if (i == 1) 1280 ARCSTAT_BUMP(arcstat_hash_chains); 1281 1282 ARCSTAT_MAX(arcstat_hash_chain_max, i); 1283 } 1284 1285 ARCSTAT_BUMP(arcstat_hash_elements); 1286 ARCSTAT_MAXSTAT(arcstat_hash_elements); 1287 1288 return (NULL); 1289 } 1290 1291 static void 1292 buf_hash_remove(arc_buf_hdr_t *hdr) 1293 { 1294 arc_buf_hdr_t *fhdr, **hdrp; 1295 uint64_t idx = BUF_HASH_INDEX(hdr->b_spa, &hdr->b_dva, hdr->b_birth); 1296 1297 ASSERT(MUTEX_HELD(BUF_HASH_LOCK(idx))); 1298 ASSERT(HDR_IN_HASH_TABLE(hdr)); 1299 1300 hdrp = &buf_hash_table.ht_table[idx]; 1301 while ((fhdr = *hdrp) != hdr) { 1302 ASSERT3P(fhdr, !=, NULL); 1303 hdrp = &fhdr->b_hash_next; 1304 } 1305 *hdrp = hdr->b_hash_next; 1306 hdr->b_hash_next = NULL; 1307 arc_hdr_clear_flags(hdr, ARC_FLAG_IN_HASH_TABLE); 1308 1309 /* collect some hash table performance data */ 1310 ARCSTAT_BUMPDOWN(arcstat_hash_elements); 1311 1312 if (buf_hash_table.ht_table[idx] && 1313 buf_hash_table.ht_table[idx]->b_hash_next == NULL) 1314 ARCSTAT_BUMPDOWN(arcstat_hash_chains); 1315 } 1316 1317 /* 1318 * Global data structures and functions for the buf kmem cache. 1319 */ 1320 static kmem_cache_t *hdr_full_cache; 1321 static kmem_cache_t *hdr_l2only_cache; 1322 static kmem_cache_t *buf_cache; 1323 1324 static void 1325 buf_fini(void) 1326 { 1327 int i; 1328 1329 kmem_free(buf_hash_table.ht_table, 1330 (buf_hash_table.ht_mask + 1) * sizeof (void *)); 1331 for (i = 0; i < BUF_LOCKS; i++) 1332 mutex_destroy(&buf_hash_table.ht_locks[i].ht_lock); 1333 kmem_cache_destroy(hdr_full_cache); 1334 kmem_cache_destroy(hdr_l2only_cache); 1335 kmem_cache_destroy(buf_cache); 1336 } 1337 1338 /* 1339 * Constructor callback - called when the cache is empty 1340 * and a new buf is requested. 1341 */ 1342 /* ARGSUSED */ 1343 static int 1344 hdr_full_cons(void *vbuf, void *unused, int kmflag) 1345 { 1346 arc_buf_hdr_t *hdr = vbuf; 1347 1348 bzero(hdr, HDR_FULL_SIZE); 1349 cv_init(&hdr->b_l1hdr.b_cv, NULL, CV_DEFAULT, NULL); 1350 zfs_refcount_create(&hdr->b_l1hdr.b_refcnt); 1351 mutex_init(&hdr->b_l1hdr.b_freeze_lock, NULL, MUTEX_DEFAULT, NULL); 1352 multilist_link_init(&hdr->b_l1hdr.b_arc_node); 1353 arc_space_consume(HDR_FULL_SIZE, ARC_SPACE_HDRS); 1354 1355 return (0); 1356 } 1357 1358 /* ARGSUSED */ 1359 static int 1360 hdr_l2only_cons(void *vbuf, void *unused, int kmflag) 1361 { 1362 arc_buf_hdr_t *hdr = vbuf; 1363 1364 bzero(hdr, HDR_L2ONLY_SIZE); 1365 arc_space_consume(HDR_L2ONLY_SIZE, ARC_SPACE_L2HDRS); 1366 1367 return (0); 1368 } 1369 1370 /* ARGSUSED */ 1371 static int 1372 buf_cons(void *vbuf, void *unused, int kmflag) 1373 { 1374 arc_buf_t *buf = vbuf; 1375 1376 bzero(buf, sizeof (arc_buf_t)); 1377 mutex_init(&buf->b_evict_lock, NULL, MUTEX_DEFAULT, NULL); 1378 arc_space_consume(sizeof (arc_buf_t), ARC_SPACE_HDRS); 1379 1380 return (0); 1381 } 1382 1383 /* 1384 * Destructor callback - called when a cached buf is 1385 * no longer required. 1386 */ 1387 /* ARGSUSED */ 1388 static void 1389 hdr_full_dest(void *vbuf, void *unused) 1390 { 1391 arc_buf_hdr_t *hdr = vbuf; 1392 1393 ASSERT(HDR_EMPTY(hdr)); 1394 cv_destroy(&hdr->b_l1hdr.b_cv); 1395 zfs_refcount_destroy(&hdr->b_l1hdr.b_refcnt); 1396 mutex_destroy(&hdr->b_l1hdr.b_freeze_lock); 1397 ASSERT(!multilist_link_active(&hdr->b_l1hdr.b_arc_node)); 1398 arc_space_return(HDR_FULL_SIZE, ARC_SPACE_HDRS); 1399 } 1400 1401 /* ARGSUSED */ 1402 static void 1403 hdr_l2only_dest(void *vbuf, void *unused) 1404 { 1405 arc_buf_hdr_t *hdr = vbuf; 1406 1407 ASSERT(HDR_EMPTY(hdr)); 1408 arc_space_return(HDR_L2ONLY_SIZE, ARC_SPACE_L2HDRS); 1409 } 1410 1411 /* ARGSUSED */ 1412 static void 1413 buf_dest(void *vbuf, void *unused) 1414 { 1415 arc_buf_t *buf = vbuf; 1416 1417 mutex_destroy(&buf->b_evict_lock); 1418 arc_space_return(sizeof (arc_buf_t), ARC_SPACE_HDRS); 1419 } 1420 1421 /* 1422 * Reclaim callback -- invoked when memory is low. 1423 */ 1424 /* ARGSUSED */ 1425 static void 1426 hdr_recl(void *unused) 1427 { 1428 dprintf("hdr_recl called\n"); 1429 /* 1430 * umem calls the reclaim func when we destroy the buf cache, 1431 * which is after we do arc_fini(). 1432 */ 1433 if (arc_initialized) 1434 zthr_wakeup(arc_reap_zthr); 1435 } 1436 1437 static void 1438 buf_init(void) 1439 { 1440 uint64_t *ct; 1441 uint64_t hsize = 1ULL << 12; 1442 int i, j; 1443 1444 /* 1445 * The hash table is big enough to fill all of physical memory 1446 * with an average block size of zfs_arc_average_blocksize (default 8K). 1447 * By default, the table will take up 1448 * totalmem * sizeof(void*) / 8K (1MB per GB with 8-byte pointers). 1449 */ 1450 while (hsize * zfs_arc_average_blocksize < physmem * PAGESIZE) 1451 hsize <<= 1; 1452 retry: 1453 buf_hash_table.ht_mask = hsize - 1; 1454 buf_hash_table.ht_table = 1455 kmem_zalloc(hsize * sizeof (void*), KM_NOSLEEP); 1456 if (buf_hash_table.ht_table == NULL) { 1457 ASSERT(hsize > (1ULL << 8)); 1458 hsize >>= 1; 1459 goto retry; 1460 } 1461 1462 hdr_full_cache = kmem_cache_create("arc_buf_hdr_t_full", HDR_FULL_SIZE, 1463 0, hdr_full_cons, hdr_full_dest, hdr_recl, NULL, NULL, 0); 1464 hdr_l2only_cache = kmem_cache_create("arc_buf_hdr_t_l2only", 1465 HDR_L2ONLY_SIZE, 0, hdr_l2only_cons, hdr_l2only_dest, hdr_recl, 1466 NULL, NULL, 0); 1467 buf_cache = kmem_cache_create("arc_buf_t", sizeof (arc_buf_t), 1468 0, buf_cons, buf_dest, NULL, NULL, NULL, 0); 1469 1470 for (i = 0; i < 256; i++) 1471 for (ct = zfs_crc64_table + i, *ct = i, j = 8; j > 0; j--) 1472 *ct = (*ct >> 1) ^ (-(*ct & 1) & ZFS_CRC64_POLY); 1473 1474 for (i = 0; i < BUF_LOCKS; i++) { 1475 mutex_init(&buf_hash_table.ht_locks[i].ht_lock, 1476 NULL, MUTEX_DEFAULT, NULL); 1477 } 1478 } 1479 1480 /* 1481 * This is the size that the buf occupies in memory. If the buf is compressed, 1482 * it will correspond to the compressed size. You should use this method of 1483 * getting the buf size unless you explicitly need the logical size. 1484 */ 1485 int32_t 1486 arc_buf_size(arc_buf_t *buf) 1487 { 1488 return (ARC_BUF_COMPRESSED(buf) ? 1489 HDR_GET_PSIZE(buf->b_hdr) : HDR_GET_LSIZE(buf->b_hdr)); 1490 } 1491 1492 int32_t 1493 arc_buf_lsize(arc_buf_t *buf) 1494 { 1495 return (HDR_GET_LSIZE(buf->b_hdr)); 1496 } 1497 1498 enum zio_compress 1499 arc_get_compression(arc_buf_t *buf) 1500 { 1501 return (ARC_BUF_COMPRESSED(buf) ? 1502 HDR_GET_COMPRESS(buf->b_hdr) : ZIO_COMPRESS_OFF); 1503 } 1504 1505 #define ARC_MINTIME (hz>>4) /* 62 ms */ 1506 1507 static inline boolean_t 1508 arc_buf_is_shared(arc_buf_t *buf) 1509 { 1510 boolean_t shared = (buf->b_data != NULL && 1511 buf->b_hdr->b_l1hdr.b_pabd != NULL && 1512 abd_is_linear(buf->b_hdr->b_l1hdr.b_pabd) && 1513 buf->b_data == abd_to_buf(buf->b_hdr->b_l1hdr.b_pabd)); 1514 IMPLY(shared, HDR_SHARED_DATA(buf->b_hdr)); 1515 IMPLY(shared, ARC_BUF_SHARED(buf)); 1516 IMPLY(shared, ARC_BUF_COMPRESSED(buf) || ARC_BUF_LAST(buf)); 1517 1518 /* 1519 * It would be nice to assert arc_can_share() too, but the "hdr isn't 1520 * already being shared" requirement prevents us from doing that. 1521 */ 1522 1523 return (shared); 1524 } 1525 1526 /* 1527 * Free the checksum associated with this header. If there is no checksum, this 1528 * is a no-op. 1529 */ 1530 static inline void 1531 arc_cksum_free(arc_buf_hdr_t *hdr) 1532 { 1533 ASSERT(HDR_HAS_L1HDR(hdr)); 1534 mutex_enter(&hdr->b_l1hdr.b_freeze_lock); 1535 if (hdr->b_l1hdr.b_freeze_cksum != NULL) { 1536 kmem_free(hdr->b_l1hdr.b_freeze_cksum, sizeof (zio_cksum_t)); 1537 hdr->b_l1hdr.b_freeze_cksum = NULL; 1538 } 1539 mutex_exit(&hdr->b_l1hdr.b_freeze_lock); 1540 } 1541 1542 /* 1543 * Return true iff at least one of the bufs on hdr is not compressed. 1544 */ 1545 static boolean_t 1546 arc_hdr_has_uncompressed_buf(arc_buf_hdr_t *hdr) 1547 { 1548 for (arc_buf_t *b = hdr->b_l1hdr.b_buf; b != NULL; b = b->b_next) { 1549 if (!ARC_BUF_COMPRESSED(b)) { 1550 return (B_TRUE); 1551 } 1552 } 1553 return (B_FALSE); 1554 } 1555 1556 /* 1557 * If we've turned on the ZFS_DEBUG_MODIFY flag, verify that the buf's data 1558 * matches the checksum that is stored in the hdr. If there is no checksum, 1559 * or if the buf is compressed, this is a no-op. 1560 */ 1561 static void 1562 arc_cksum_verify(arc_buf_t *buf) 1563 { 1564 arc_buf_hdr_t *hdr = buf->b_hdr; 1565 zio_cksum_t zc; 1566 1567 if (!(zfs_flags & ZFS_DEBUG_MODIFY)) 1568 return; 1569 1570 if (ARC_BUF_COMPRESSED(buf)) { 1571 ASSERT(hdr->b_l1hdr.b_freeze_cksum == NULL || 1572 arc_hdr_has_uncompressed_buf(hdr)); 1573 return; 1574 } 1575 1576 ASSERT(HDR_HAS_L1HDR(hdr)); 1577 1578 mutex_enter(&hdr->b_l1hdr.b_freeze_lock); 1579 if (hdr->b_l1hdr.b_freeze_cksum == NULL || HDR_IO_ERROR(hdr)) { 1580 mutex_exit(&hdr->b_l1hdr.b_freeze_lock); 1581 return; 1582 } 1583 1584 fletcher_2_native(buf->b_data, arc_buf_size(buf), NULL, &zc); 1585 if (!ZIO_CHECKSUM_EQUAL(*hdr->b_l1hdr.b_freeze_cksum, zc)) 1586 panic("buffer modified while frozen!"); 1587 mutex_exit(&hdr->b_l1hdr.b_freeze_lock); 1588 } 1589 1590 static boolean_t 1591 arc_cksum_is_equal(arc_buf_hdr_t *hdr, zio_t *zio) 1592 { 1593 enum zio_compress compress = BP_GET_COMPRESS(zio->io_bp); 1594 boolean_t valid_cksum; 1595 1596 ASSERT(!BP_IS_EMBEDDED(zio->io_bp)); 1597 VERIFY3U(BP_GET_PSIZE(zio->io_bp), ==, HDR_GET_PSIZE(hdr)); 1598 1599 /* 1600 * We rely on the blkptr's checksum to determine if the block 1601 * is valid or not. When compressed arc is enabled, the l2arc 1602 * writes the block to the l2arc just as it appears in the pool. 1603 * This allows us to use the blkptr's checksum to validate the 1604 * data that we just read off of the l2arc without having to store 1605 * a separate checksum in the arc_buf_hdr_t. However, if compressed 1606 * arc is disabled, then the data written to the l2arc is always 1607 * uncompressed and won't match the block as it exists in the main 1608 * pool. When this is the case, we must first compress it if it is 1609 * compressed on the main pool before we can validate the checksum. 1610 */ 1611 if (!HDR_COMPRESSION_ENABLED(hdr) && compress != ZIO_COMPRESS_OFF) { 1612 ASSERT3U(HDR_GET_COMPRESS(hdr), ==, ZIO_COMPRESS_OFF); 1613 uint64_t lsize = HDR_GET_LSIZE(hdr); 1614 uint64_t csize; 1615 1616 abd_t *cdata = abd_alloc_linear(HDR_GET_PSIZE(hdr), B_TRUE); 1617 csize = zio_compress_data(compress, zio->io_abd, 1618 abd_to_buf(cdata), lsize); 1619 1620 ASSERT3U(csize, <=, HDR_GET_PSIZE(hdr)); 1621 if (csize < HDR_GET_PSIZE(hdr)) { 1622 /* 1623 * Compressed blocks are always a multiple of the 1624 * smallest ashift in the pool. Ideally, we would 1625 * like to round up the csize to the next 1626 * spa_min_ashift but that value may have changed 1627 * since the block was last written. Instead, 1628 * we rely on the fact that the hdr's psize 1629 * was set to the psize of the block when it was 1630 * last written. We set the csize to that value 1631 * and zero out any part that should not contain 1632 * data. 1633 */ 1634 abd_zero_off(cdata, csize, HDR_GET_PSIZE(hdr) - csize); 1635 csize = HDR_GET_PSIZE(hdr); 1636 } 1637 zio_push_transform(zio, cdata, csize, HDR_GET_PSIZE(hdr), NULL); 1638 } 1639 1640 /* 1641 * Block pointers always store the checksum for the logical data. 1642 * If the block pointer has the gang bit set, then the checksum 1643 * it represents is for the reconstituted data and not for an 1644 * individual gang member. The zio pipeline, however, must be able to 1645 * determine the checksum of each of the gang constituents so it 1646 * treats the checksum comparison differently than what we need 1647 * for l2arc blocks. This prevents us from using the 1648 * zio_checksum_error() interface directly. Instead we must call the 1649 * zio_checksum_error_impl() so that we can ensure the checksum is 1650 * generated using the correct checksum algorithm and accounts for the 1651 * logical I/O size and not just a gang fragment. 1652 */ 1653 valid_cksum = (zio_checksum_error_impl(zio->io_spa, zio->io_bp, 1654 BP_GET_CHECKSUM(zio->io_bp), zio->io_abd, zio->io_size, 1655 zio->io_offset, NULL) == 0); 1656 zio_pop_transforms(zio); 1657 return (valid_cksum); 1658 } 1659 1660 /* 1661 * Given a buf full of data, if ZFS_DEBUG_MODIFY is enabled this computes a 1662 * checksum and attaches it to the buf's hdr so that we can ensure that the buf 1663 * isn't modified later on. If buf is compressed or there is already a checksum 1664 * on the hdr, this is a no-op (we only checksum uncompressed bufs). 1665 */ 1666 static void 1667 arc_cksum_compute(arc_buf_t *buf) 1668 { 1669 arc_buf_hdr_t *hdr = buf->b_hdr; 1670 1671 if (!(zfs_flags & ZFS_DEBUG_MODIFY)) 1672 return; 1673 1674 ASSERT(HDR_HAS_L1HDR(hdr)); 1675 1676 mutex_enter(&buf->b_hdr->b_l1hdr.b_freeze_lock); 1677 if (hdr->b_l1hdr.b_freeze_cksum != NULL) { 1678 ASSERT(arc_hdr_has_uncompressed_buf(hdr)); 1679 mutex_exit(&hdr->b_l1hdr.b_freeze_lock); 1680 return; 1681 } else if (ARC_BUF_COMPRESSED(buf)) { 1682 mutex_exit(&hdr->b_l1hdr.b_freeze_lock); 1683 return; 1684 } 1685 1686 ASSERT(!ARC_BUF_COMPRESSED(buf)); 1687 hdr->b_l1hdr.b_freeze_cksum = kmem_alloc(sizeof (zio_cksum_t), 1688 KM_SLEEP); 1689 fletcher_2_native(buf->b_data, arc_buf_size(buf), NULL, 1690 hdr->b_l1hdr.b_freeze_cksum); 1691 mutex_exit(&hdr->b_l1hdr.b_freeze_lock); 1692 arc_buf_watch(buf); 1693 } 1694 1695 #ifndef _KERNEL 1696 typedef struct procctl { 1697 long cmd; 1698 prwatch_t prwatch; 1699 } procctl_t; 1700 #endif 1701 1702 /* ARGSUSED */ 1703 static void 1704 arc_buf_unwatch(arc_buf_t *buf) 1705 { 1706 #ifndef _KERNEL 1707 if (arc_watch) { 1708 int result; 1709 procctl_t ctl; 1710 ctl.cmd = PCWATCH; 1711 ctl.prwatch.pr_vaddr = (uintptr_t)buf->b_data; 1712 ctl.prwatch.pr_size = 0; 1713 ctl.prwatch.pr_wflags = 0; 1714 result = write(arc_procfd, &ctl, sizeof (ctl)); 1715 ASSERT3U(result, ==, sizeof (ctl)); 1716 } 1717 #endif 1718 } 1719 1720 /* ARGSUSED */ 1721 static void 1722 arc_buf_watch(arc_buf_t *buf) 1723 { 1724 #ifndef _KERNEL 1725 if (arc_watch) { 1726 int result; 1727 procctl_t ctl; 1728 ctl.cmd = PCWATCH; 1729 ctl.prwatch.pr_vaddr = (uintptr_t)buf->b_data; 1730 ctl.prwatch.pr_size = arc_buf_size(buf); 1731 ctl.prwatch.pr_wflags = WA_WRITE; 1732 result = write(arc_procfd, &ctl, sizeof (ctl)); 1733 ASSERT3U(result, ==, sizeof (ctl)); 1734 } 1735 #endif 1736 } 1737 1738 static arc_buf_contents_t 1739 arc_buf_type(arc_buf_hdr_t *hdr) 1740 { 1741 arc_buf_contents_t type; 1742 if (HDR_ISTYPE_METADATA(hdr)) { 1743 type = ARC_BUFC_METADATA; 1744 } else { 1745 type = ARC_BUFC_DATA; 1746 } 1747 VERIFY3U(hdr->b_type, ==, type); 1748 return (type); 1749 } 1750 1751 boolean_t 1752 arc_is_metadata(arc_buf_t *buf) 1753 { 1754 return (HDR_ISTYPE_METADATA(buf->b_hdr) != 0); 1755 } 1756 1757 static uint32_t 1758 arc_bufc_to_flags(arc_buf_contents_t type) 1759 { 1760 switch (type) { 1761 case ARC_BUFC_DATA: 1762 /* metadata field is 0 if buffer contains normal data */ 1763 return (0); 1764 case ARC_BUFC_METADATA: 1765 return (ARC_FLAG_BUFC_METADATA); 1766 default: 1767 break; 1768 } 1769 panic("undefined ARC buffer type!"); 1770 return ((uint32_t)-1); 1771 } 1772 1773 void 1774 arc_buf_thaw(arc_buf_t *buf) 1775 { 1776 arc_buf_hdr_t *hdr = buf->b_hdr; 1777 1778 ASSERT3P(hdr->b_l1hdr.b_state, ==, arc_anon); 1779 ASSERT(!HDR_IO_IN_PROGRESS(hdr)); 1780 1781 arc_cksum_verify(buf); 1782 1783 /* 1784 * Compressed buffers do not manipulate the b_freeze_cksum or 1785 * allocate b_thawed. 1786 */ 1787 if (ARC_BUF_COMPRESSED(buf)) { 1788 ASSERT(hdr->b_l1hdr.b_freeze_cksum == NULL || 1789 arc_hdr_has_uncompressed_buf(hdr)); 1790 return; 1791 } 1792 1793 ASSERT(HDR_HAS_L1HDR(hdr)); 1794 arc_cksum_free(hdr); 1795 1796 mutex_enter(&hdr->b_l1hdr.b_freeze_lock); 1797 #ifdef ZFS_DEBUG 1798 if (zfs_flags & ZFS_DEBUG_MODIFY) { 1799 if (hdr->b_l1hdr.b_thawed != NULL) 1800 kmem_free(hdr->b_l1hdr.b_thawed, 1); 1801 hdr->b_l1hdr.b_thawed = kmem_alloc(1, KM_SLEEP); 1802 } 1803 #endif 1804 1805 mutex_exit(&hdr->b_l1hdr.b_freeze_lock); 1806 1807 arc_buf_unwatch(buf); 1808 } 1809 1810 void 1811 arc_buf_freeze(arc_buf_t *buf) 1812 { 1813 arc_buf_hdr_t *hdr = buf->b_hdr; 1814 kmutex_t *hash_lock; 1815 1816 if (!(zfs_flags & ZFS_DEBUG_MODIFY)) 1817 return; 1818 1819 if (ARC_BUF_COMPRESSED(buf)) { 1820 ASSERT(hdr->b_l1hdr.b_freeze_cksum == NULL || 1821 arc_hdr_has_uncompressed_buf(hdr)); 1822 return; 1823 } 1824 1825 hash_lock = HDR_LOCK(hdr); 1826 mutex_enter(hash_lock); 1827 1828 ASSERT(HDR_HAS_L1HDR(hdr)); 1829 ASSERT(hdr->b_l1hdr.b_freeze_cksum != NULL || 1830 hdr->b_l1hdr.b_state == arc_anon); 1831 arc_cksum_compute(buf); 1832 mutex_exit(hash_lock); 1833 } 1834 1835 /* 1836 * The arc_buf_hdr_t's b_flags should never be modified directly. Instead, 1837 * the following functions should be used to ensure that the flags are 1838 * updated in a thread-safe way. When manipulating the flags either 1839 * the hash_lock must be held or the hdr must be undiscoverable. This 1840 * ensures that we're not racing with any other threads when updating 1841 * the flags. 1842 */ 1843 static inline void 1844 arc_hdr_set_flags(arc_buf_hdr_t *hdr, arc_flags_t flags) 1845 { 1846 ASSERT(MUTEX_HELD(HDR_LOCK(hdr)) || HDR_EMPTY(hdr)); 1847 hdr->b_flags |= flags; 1848 } 1849 1850 static inline void 1851 arc_hdr_clear_flags(arc_buf_hdr_t *hdr, arc_flags_t flags) 1852 { 1853 ASSERT(MUTEX_HELD(HDR_LOCK(hdr)) || HDR_EMPTY(hdr)); 1854 hdr->b_flags &= ~flags; 1855 } 1856 1857 /* 1858 * Setting the compression bits in the arc_buf_hdr_t's b_flags is 1859 * done in a special way since we have to clear and set bits 1860 * at the same time. Consumers that wish to set the compression bits 1861 * must use this function to ensure that the flags are updated in 1862 * thread-safe manner. 1863 */ 1864 static void 1865 arc_hdr_set_compress(arc_buf_hdr_t *hdr, enum zio_compress cmp) 1866 { 1867 ASSERT(MUTEX_HELD(HDR_LOCK(hdr)) || HDR_EMPTY(hdr)); 1868 1869 /* 1870 * Holes and embedded blocks will always have a psize = 0 so 1871 * we ignore the compression of the blkptr and set the 1872 * arc_buf_hdr_t's compression to ZIO_COMPRESS_OFF. 1873 * Holes and embedded blocks remain anonymous so we don't 1874 * want to uncompress them. Mark them as uncompressed. 1875 */ 1876 if (!zfs_compressed_arc_enabled || HDR_GET_PSIZE(hdr) == 0) { 1877 arc_hdr_clear_flags(hdr, ARC_FLAG_COMPRESSED_ARC); 1878 HDR_SET_COMPRESS(hdr, ZIO_COMPRESS_OFF); 1879 ASSERT(!HDR_COMPRESSION_ENABLED(hdr)); 1880 ASSERT3U(HDR_GET_COMPRESS(hdr), ==, ZIO_COMPRESS_OFF); 1881 } else { 1882 arc_hdr_set_flags(hdr, ARC_FLAG_COMPRESSED_ARC); 1883 HDR_SET_COMPRESS(hdr, cmp); 1884 ASSERT3U(HDR_GET_COMPRESS(hdr), ==, cmp); 1885 ASSERT(HDR_COMPRESSION_ENABLED(hdr)); 1886 } 1887 } 1888 1889 /* 1890 * Looks for another buf on the same hdr which has the data decompressed, copies 1891 * from it, and returns true. If no such buf exists, returns false. 1892 */ 1893 static boolean_t 1894 arc_buf_try_copy_decompressed_data(arc_buf_t *buf) 1895 { 1896 arc_buf_hdr_t *hdr = buf->b_hdr; 1897 boolean_t copied = B_FALSE; 1898 1899 ASSERT(HDR_HAS_L1HDR(hdr)); 1900 ASSERT3P(buf->b_data, !=, NULL); 1901 ASSERT(!ARC_BUF_COMPRESSED(buf)); 1902 1903 for (arc_buf_t *from = hdr->b_l1hdr.b_buf; from != NULL; 1904 from = from->b_next) { 1905 /* can't use our own data buffer */ 1906 if (from == buf) { 1907 continue; 1908 } 1909 1910 if (!ARC_BUF_COMPRESSED(from)) { 1911 bcopy(from->b_data, buf->b_data, arc_buf_size(buf)); 1912 copied = B_TRUE; 1913 break; 1914 } 1915 } 1916 1917 /* 1918 * There were no decompressed bufs, so there should not be a 1919 * checksum on the hdr either. 1920 */ 1921 EQUIV(!copied, hdr->b_l1hdr.b_freeze_cksum == NULL); 1922 1923 return (copied); 1924 } 1925 1926 /* 1927 * Given a buf that has a data buffer attached to it, this function will 1928 * efficiently fill the buf with data of the specified compression setting from 1929 * the hdr and update the hdr's b_freeze_cksum if necessary. If the buf and hdr 1930 * are already sharing a data buf, no copy is performed. 1931 * 1932 * If the buf is marked as compressed but uncompressed data was requested, this 1933 * will allocate a new data buffer for the buf, remove that flag, and fill the 1934 * buf with uncompressed data. You can't request a compressed buf on a hdr with 1935 * uncompressed data, and (since we haven't added support for it yet) if you 1936 * want compressed data your buf must already be marked as compressed and have 1937 * the correct-sized data buffer. 1938 */ 1939 static int 1940 arc_buf_fill(arc_buf_t *buf, boolean_t compressed) 1941 { 1942 arc_buf_hdr_t *hdr = buf->b_hdr; 1943 boolean_t hdr_compressed = (HDR_GET_COMPRESS(hdr) != ZIO_COMPRESS_OFF); 1944 dmu_object_byteswap_t bswap = hdr->b_l1hdr.b_byteswap; 1945 1946 ASSERT3P(buf->b_data, !=, NULL); 1947 IMPLY(compressed, hdr_compressed); 1948 IMPLY(compressed, ARC_BUF_COMPRESSED(buf)); 1949 1950 if (hdr_compressed == compressed) { 1951 if (!arc_buf_is_shared(buf)) { 1952 abd_copy_to_buf(buf->b_data, hdr->b_l1hdr.b_pabd, 1953 arc_buf_size(buf)); 1954 } 1955 } else { 1956 ASSERT(hdr_compressed); 1957 ASSERT(!compressed); 1958 ASSERT3U(HDR_GET_LSIZE(hdr), !=, HDR_GET_PSIZE(hdr)); 1959 1960 /* 1961 * If the buf is sharing its data with the hdr, unlink it and 1962 * allocate a new data buffer for the buf. 1963 */ 1964 if (arc_buf_is_shared(buf)) { 1965 ASSERT(ARC_BUF_COMPRESSED(buf)); 1966 1967 /* We need to give the buf it's own b_data */ 1968 buf->b_flags &= ~ARC_BUF_FLAG_SHARED; 1969 buf->b_data = 1970 arc_get_data_buf(hdr, HDR_GET_LSIZE(hdr), buf); 1971 arc_hdr_clear_flags(hdr, ARC_FLAG_SHARED_DATA); 1972 1973 /* Previously overhead was 0; just add new overhead */ 1974 ARCSTAT_INCR(arcstat_overhead_size, HDR_GET_LSIZE(hdr)); 1975 } else if (ARC_BUF_COMPRESSED(buf)) { 1976 /* We need to reallocate the buf's b_data */ 1977 arc_free_data_buf(hdr, buf->b_data, HDR_GET_PSIZE(hdr), 1978 buf); 1979 buf->b_data = 1980 arc_get_data_buf(hdr, HDR_GET_LSIZE(hdr), buf); 1981 1982 /* We increased the size of b_data; update overhead */ 1983 ARCSTAT_INCR(arcstat_overhead_size, 1984 HDR_GET_LSIZE(hdr) - HDR_GET_PSIZE(hdr)); 1985 } 1986 1987 /* 1988 * Regardless of the buf's previous compression settings, it 1989 * should not be compressed at the end of this function. 1990 */ 1991 buf->b_flags &= ~ARC_BUF_FLAG_COMPRESSED; 1992 1993 /* 1994 * Try copying the data from another buf which already has a 1995 * decompressed version. If that's not possible, it's time to 1996 * bite the bullet and decompress the data from the hdr. 1997 */ 1998 if (arc_buf_try_copy_decompressed_data(buf)) { 1999 /* Skip byteswapping and checksumming (already done) */ 2000 ASSERT3P(hdr->b_l1hdr.b_freeze_cksum, !=, NULL); 2001 return (0); 2002 } else { 2003 int error = zio_decompress_data(HDR_GET_COMPRESS(hdr), 2004 hdr->b_l1hdr.b_pabd, buf->b_data, 2005 HDR_GET_PSIZE(hdr), HDR_GET_LSIZE(hdr)); 2006 2007 /* 2008 * Absent hardware errors or software bugs, this should 2009 * be impossible, but log it anyway so we can debug it. 2010 */ 2011 if (error != 0) { 2012 zfs_dbgmsg( 2013 "hdr %p, compress %d, psize %d, lsize %d", 2014 hdr, HDR_GET_COMPRESS(hdr), 2015 HDR_GET_PSIZE(hdr), HDR_GET_LSIZE(hdr)); 2016 return (SET_ERROR(EIO)); 2017 } 2018 } 2019 } 2020 2021 /* Byteswap the buf's data if necessary */ 2022 if (bswap != DMU_BSWAP_NUMFUNCS) { 2023 ASSERT(!HDR_SHARED_DATA(hdr)); 2024 ASSERT3U(bswap, <, DMU_BSWAP_NUMFUNCS); 2025 dmu_ot_byteswap[bswap].ob_func(buf->b_data, HDR_GET_LSIZE(hdr)); 2026 } 2027 2028 /* Compute the hdr's checksum if necessary */ 2029 arc_cksum_compute(buf); 2030 2031 return (0); 2032 } 2033 2034 int 2035 arc_decompress(arc_buf_t *buf) 2036 { 2037 return (arc_buf_fill(buf, B_FALSE)); 2038 } 2039 2040 /* 2041 * Return the size of the block, b_pabd, that is stored in the arc_buf_hdr_t. 2042 */ 2043 static uint64_t 2044 arc_hdr_size(arc_buf_hdr_t *hdr) 2045 { 2046 uint64_t size; 2047 2048 if (HDR_GET_COMPRESS(hdr) != ZIO_COMPRESS_OFF && 2049 HDR_GET_PSIZE(hdr) > 0) { 2050 size = HDR_GET_PSIZE(hdr); 2051 } else { 2052 ASSERT3U(HDR_GET_LSIZE(hdr), !=, 0); 2053 size = HDR_GET_LSIZE(hdr); 2054 } 2055 return (size); 2056 } 2057 2058 /* 2059 * Increment the amount of evictable space in the arc_state_t's refcount. 2060 * We account for the space used by the hdr and the arc buf individually 2061 * so that we can add and remove them from the refcount individually. 2062 */ 2063 static void 2064 arc_evictable_space_increment(arc_buf_hdr_t *hdr, arc_state_t *state) 2065 { 2066 arc_buf_contents_t type = arc_buf_type(hdr); 2067 2068 ASSERT(HDR_HAS_L1HDR(hdr)); 2069 2070 if (GHOST_STATE(state)) { 2071 ASSERT0(hdr->b_l1hdr.b_bufcnt); 2072 ASSERT3P(hdr->b_l1hdr.b_buf, ==, NULL); 2073 ASSERT3P(hdr->b_l1hdr.b_pabd, ==, NULL); 2074 (void) zfs_refcount_add_many(&state->arcs_esize[type], 2075 HDR_GET_LSIZE(hdr), hdr); 2076 return; 2077 } 2078 2079 ASSERT(!GHOST_STATE(state)); 2080 if (hdr->b_l1hdr.b_pabd != NULL) { 2081 (void) zfs_refcount_add_many(&state->arcs_esize[type], 2082 arc_hdr_size(hdr), hdr); 2083 } 2084 for (arc_buf_t *buf = hdr->b_l1hdr.b_buf; buf != NULL; 2085 buf = buf->b_next) { 2086 if (arc_buf_is_shared(buf)) 2087 continue; 2088 (void) zfs_refcount_add_many(&state->arcs_esize[type], 2089 arc_buf_size(buf), buf); 2090 } 2091 } 2092 2093 /* 2094 * Decrement the amount of evictable space in the arc_state_t's refcount. 2095 * We account for the space used by the hdr and the arc buf individually 2096 * so that we can add and remove them from the refcount individually. 2097 */ 2098 static void 2099 arc_evictable_space_decrement(arc_buf_hdr_t *hdr, arc_state_t *state) 2100 { 2101 arc_buf_contents_t type = arc_buf_type(hdr); 2102 2103 ASSERT(HDR_HAS_L1HDR(hdr)); 2104 2105 if (GHOST_STATE(state)) { 2106 ASSERT0(hdr->b_l1hdr.b_bufcnt); 2107 ASSERT3P(hdr->b_l1hdr.b_buf, ==, NULL); 2108 ASSERT3P(hdr->b_l1hdr.b_pabd, ==, NULL); 2109 (void) zfs_refcount_remove_many(&state->arcs_esize[type], 2110 HDR_GET_LSIZE(hdr), hdr); 2111 return; 2112 } 2113 2114 ASSERT(!GHOST_STATE(state)); 2115 if (hdr->b_l1hdr.b_pabd != NULL) { 2116 (void) zfs_refcount_remove_many(&state->arcs_esize[type], 2117 arc_hdr_size(hdr), hdr); 2118 } 2119 for (arc_buf_t *buf = hdr->b_l1hdr.b_buf; buf != NULL; 2120 buf = buf->b_next) { 2121 if (arc_buf_is_shared(buf)) 2122 continue; 2123 (void) zfs_refcount_remove_many(&state->arcs_esize[type], 2124 arc_buf_size(buf), buf); 2125 } 2126 } 2127 2128 /* 2129 * Add a reference to this hdr indicating that someone is actively 2130 * referencing that memory. When the refcount transitions from 0 to 1, 2131 * we remove it from the respective arc_state_t list to indicate that 2132 * it is not evictable. 2133 */ 2134 static void 2135 add_reference(arc_buf_hdr_t *hdr, void *tag) 2136 { 2137 ASSERT(HDR_HAS_L1HDR(hdr)); 2138 if (!MUTEX_HELD(HDR_LOCK(hdr))) { 2139 ASSERT(hdr->b_l1hdr.b_state == arc_anon); 2140 ASSERT(zfs_refcount_is_zero(&hdr->b_l1hdr.b_refcnt)); 2141 ASSERT3P(hdr->b_l1hdr.b_buf, ==, NULL); 2142 } 2143 2144 arc_state_t *state = hdr->b_l1hdr.b_state; 2145 2146 if ((zfs_refcount_add(&hdr->b_l1hdr.b_refcnt, tag) == 1) && 2147 (state != arc_anon)) { 2148 /* We don't use the L2-only state list. */ 2149 if (state != arc_l2c_only) { 2150 multilist_remove(state->arcs_list[arc_buf_type(hdr)], 2151 hdr); 2152 arc_evictable_space_decrement(hdr, state); 2153 } 2154 /* remove the prefetch flag if we get a reference */ 2155 arc_hdr_clear_flags(hdr, ARC_FLAG_PREFETCH); 2156 } 2157 } 2158 2159 /* 2160 * Remove a reference from this hdr. When the reference transitions from 2161 * 1 to 0 and we're not anonymous, then we add this hdr to the arc_state_t's 2162 * list making it eligible for eviction. 2163 */ 2164 static int 2165 remove_reference(arc_buf_hdr_t *hdr, kmutex_t *hash_lock, void *tag) 2166 { 2167 int cnt; 2168 arc_state_t *state = hdr->b_l1hdr.b_state; 2169 2170 ASSERT(HDR_HAS_L1HDR(hdr)); 2171 ASSERT(state == arc_anon || MUTEX_HELD(hash_lock)); 2172 ASSERT(!GHOST_STATE(state)); 2173 2174 /* 2175 * arc_l2c_only counts as a ghost state so we don't need to explicitly 2176 * check to prevent usage of the arc_l2c_only list. 2177 */ 2178 if (((cnt = zfs_refcount_remove(&hdr->b_l1hdr.b_refcnt, tag)) == 0) && 2179 (state != arc_anon)) { 2180 multilist_insert(state->arcs_list[arc_buf_type(hdr)], hdr); 2181 ASSERT3U(hdr->b_l1hdr.b_bufcnt, >, 0); 2182 arc_evictable_space_increment(hdr, state); 2183 } 2184 return (cnt); 2185 } 2186 2187 /* 2188 * Move the supplied buffer to the indicated state. The hash lock 2189 * for the buffer must be held by the caller. 2190 */ 2191 static void 2192 arc_change_state(arc_state_t *new_state, arc_buf_hdr_t *hdr, 2193 kmutex_t *hash_lock) 2194 { 2195 arc_state_t *old_state; 2196 int64_t refcnt; 2197 uint32_t bufcnt; 2198 boolean_t update_old, update_new; 2199 arc_buf_contents_t buftype = arc_buf_type(hdr); 2200 2201 /* 2202 * We almost always have an L1 hdr here, since we call arc_hdr_realloc() 2203 * in arc_read() when bringing a buffer out of the L2ARC. However, the 2204 * L1 hdr doesn't always exist when we change state to arc_anon before 2205 * destroying a header, in which case reallocating to add the L1 hdr is 2206 * pointless. 2207 */ 2208 if (HDR_HAS_L1HDR(hdr)) { 2209 old_state = hdr->b_l1hdr.b_state; 2210 refcnt = zfs_refcount_count(&hdr->b_l1hdr.b_refcnt); 2211 bufcnt = hdr->b_l1hdr.b_bufcnt; 2212 update_old = (bufcnt > 0 || hdr->b_l1hdr.b_pabd != NULL); 2213 } else { 2214 old_state = arc_l2c_only; 2215 refcnt = 0; 2216 bufcnt = 0; 2217 update_old = B_FALSE; 2218 } 2219 update_new = update_old; 2220 2221 ASSERT(MUTEX_HELD(hash_lock)); 2222 ASSERT3P(new_state, !=, old_state); 2223 ASSERT(!GHOST_STATE(new_state) || bufcnt == 0); 2224 ASSERT(old_state != arc_anon || bufcnt <= 1); 2225 2226 /* 2227 * If this buffer is evictable, transfer it from the 2228 * old state list to the new state list. 2229 */ 2230 if (refcnt == 0) { 2231 if (old_state != arc_anon && old_state != arc_l2c_only) { 2232 ASSERT(HDR_HAS_L1HDR(hdr)); 2233 multilist_remove(old_state->arcs_list[buftype], hdr); 2234 2235 if (GHOST_STATE(old_state)) { 2236 ASSERT0(bufcnt); 2237 ASSERT3P(hdr->b_l1hdr.b_buf, ==, NULL); 2238 update_old = B_TRUE; 2239 } 2240 arc_evictable_space_decrement(hdr, old_state); 2241 } 2242 if (new_state != arc_anon && new_state != arc_l2c_only) { 2243 2244 /* 2245 * An L1 header always exists here, since if we're 2246 * moving to some L1-cached state (i.e. not l2c_only or 2247 * anonymous), we realloc the header to add an L1hdr 2248 * beforehand. 2249 */ 2250 ASSERT(HDR_HAS_L1HDR(hdr)); 2251 multilist_insert(new_state->arcs_list[buftype], hdr); 2252 2253 if (GHOST_STATE(new_state)) { 2254 ASSERT0(bufcnt); 2255 ASSERT3P(hdr->b_l1hdr.b_buf, ==, NULL); 2256 update_new = B_TRUE; 2257 } 2258 arc_evictable_space_increment(hdr, new_state); 2259 } 2260 } 2261 2262 ASSERT(!HDR_EMPTY(hdr)); 2263 if (new_state == arc_anon && HDR_IN_HASH_TABLE(hdr)) 2264 buf_hash_remove(hdr); 2265 2266 /* adjust state sizes (ignore arc_l2c_only) */ 2267 2268 if (update_new && new_state != arc_l2c_only) { 2269 ASSERT(HDR_HAS_L1HDR(hdr)); 2270 if (GHOST_STATE(new_state)) { 2271 ASSERT0(bufcnt); 2272 2273 /* 2274 * When moving a header to a ghost state, we first 2275 * remove all arc buffers. Thus, we'll have a 2276 * bufcnt of zero, and no arc buffer to use for 2277 * the reference. As a result, we use the arc 2278 * header pointer for the reference. 2279 */ 2280 (void) zfs_refcount_add_many(&new_state->arcs_size, 2281 HDR_GET_LSIZE(hdr), hdr); 2282 ASSERT3P(hdr->b_l1hdr.b_pabd, ==, NULL); 2283 } else { 2284 uint32_t buffers = 0; 2285 2286 /* 2287 * Each individual buffer holds a unique reference, 2288 * thus we must remove each of these references one 2289 * at a time. 2290 */ 2291 for (arc_buf_t *buf = hdr->b_l1hdr.b_buf; buf != NULL; 2292 buf = buf->b_next) { 2293 ASSERT3U(bufcnt, !=, 0); 2294 buffers++; 2295 2296 /* 2297 * When the arc_buf_t is sharing the data 2298 * block with the hdr, the owner of the 2299 * reference belongs to the hdr. Only 2300 * add to the refcount if the arc_buf_t is 2301 * not shared. 2302 */ 2303 if (arc_buf_is_shared(buf)) 2304 continue; 2305 2306 (void) zfs_refcount_add_many( 2307 &new_state->arcs_size, 2308 arc_buf_size(buf), buf); 2309 } 2310 ASSERT3U(bufcnt, ==, buffers); 2311 2312 if (hdr->b_l1hdr.b_pabd != NULL) { 2313 (void) zfs_refcount_add_many( 2314 &new_state->arcs_size, 2315 arc_hdr_size(hdr), hdr); 2316 } else { 2317 ASSERT(GHOST_STATE(old_state)); 2318 } 2319 } 2320 } 2321 2322 if (update_old && old_state != arc_l2c_only) { 2323 ASSERT(HDR_HAS_L1HDR(hdr)); 2324 if (GHOST_STATE(old_state)) { 2325 ASSERT0(bufcnt); 2326 ASSERT3P(hdr->b_l1hdr.b_pabd, ==, NULL); 2327 2328 /* 2329 * When moving a header off of a ghost state, 2330 * the header will not contain any arc buffers. 2331 * We use the arc header pointer for the reference 2332 * which is exactly what we did when we put the 2333 * header on the ghost state. 2334 */ 2335 2336 (void) zfs_refcount_remove_many(&old_state->arcs_size, 2337 HDR_GET_LSIZE(hdr), hdr); 2338 } else { 2339 uint32_t buffers = 0; 2340 2341 /* 2342 * Each individual buffer holds a unique reference, 2343 * thus we must remove each of these references one 2344 * at a time. 2345 */ 2346 for (arc_buf_t *buf = hdr->b_l1hdr.b_buf; buf != NULL; 2347 buf = buf->b_next) { 2348 ASSERT3U(bufcnt, !=, 0); 2349 buffers++; 2350 2351 /* 2352 * When the arc_buf_t is sharing the data 2353 * block with the hdr, the owner of the 2354 * reference belongs to the hdr. Only 2355 * add to the refcount if the arc_buf_t is 2356 * not shared. 2357 */ 2358 if (arc_buf_is_shared(buf)) 2359 continue; 2360 2361 (void) zfs_refcount_remove_many( 2362 &old_state->arcs_size, arc_buf_size(buf), 2363 buf); 2364 } 2365 ASSERT3U(bufcnt, ==, buffers); 2366 ASSERT3P(hdr->b_l1hdr.b_pabd, !=, NULL); 2367 (void) zfs_refcount_remove_many( 2368 &old_state->arcs_size, arc_hdr_size(hdr), hdr); 2369 } 2370 } 2371 2372 if (HDR_HAS_L1HDR(hdr)) 2373 hdr->b_l1hdr.b_state = new_state; 2374 2375 /* 2376 * L2 headers should never be on the L2 state list since they don't 2377 * have L1 headers allocated. 2378 */ 2379 ASSERT(multilist_is_empty(arc_l2c_only->arcs_list[ARC_BUFC_DATA]) && 2380 multilist_is_empty(arc_l2c_only->arcs_list[ARC_BUFC_METADATA])); 2381 } 2382 2383 void 2384 arc_space_consume(uint64_t space, arc_space_type_t type) 2385 { 2386 ASSERT(type >= 0 && type < ARC_SPACE_NUMTYPES); 2387 2388 switch (type) { 2389 case ARC_SPACE_DATA: 2390 aggsum_add(&astat_data_size, space); 2391 break; 2392 case ARC_SPACE_META: 2393 aggsum_add(&astat_metadata_size, space); 2394 break; 2395 case ARC_SPACE_OTHER: 2396 aggsum_add(&astat_other_size, space); 2397 break; 2398 case ARC_SPACE_HDRS: 2399 aggsum_add(&astat_hdr_size, space); 2400 break; 2401 case ARC_SPACE_L2HDRS: 2402 aggsum_add(&astat_l2_hdr_size, space); 2403 break; 2404 } 2405 2406 if (type != ARC_SPACE_DATA) 2407 aggsum_add(&arc_meta_used, space); 2408 2409 aggsum_add(&arc_size, space); 2410 } 2411 2412 void 2413 arc_space_return(uint64_t space, arc_space_type_t type) 2414 { 2415 ASSERT(type >= 0 && type < ARC_SPACE_NUMTYPES); 2416 2417 switch (type) { 2418 case ARC_SPACE_DATA: 2419 aggsum_add(&astat_data_size, -space); 2420 break; 2421 case ARC_SPACE_META: 2422 aggsum_add(&astat_metadata_size, -space); 2423 break; 2424 case ARC_SPACE_OTHER: 2425 aggsum_add(&astat_other_size, -space); 2426 break; 2427 case ARC_SPACE_HDRS: 2428 aggsum_add(&astat_hdr_size, -space); 2429 break; 2430 case ARC_SPACE_L2HDRS: 2431 aggsum_add(&astat_l2_hdr_size, -space); 2432 break; 2433 } 2434 2435 if (type != ARC_SPACE_DATA) { 2436 ASSERT(aggsum_compare(&arc_meta_used, space) >= 0); 2437 /* 2438 * We use the upper bound here rather than the precise value 2439 * because the arc_meta_max value doesn't need to be 2440 * precise. It's only consumed by humans via arcstats. 2441 */ 2442 if (arc_meta_max < aggsum_upper_bound(&arc_meta_used)) 2443 arc_meta_max = aggsum_upper_bound(&arc_meta_used); 2444 aggsum_add(&arc_meta_used, -space); 2445 } 2446 2447 ASSERT(aggsum_compare(&arc_size, space) >= 0); 2448 aggsum_add(&arc_size, -space); 2449 } 2450 2451 /* 2452 * Given a hdr and a buf, returns whether that buf can share its b_data buffer 2453 * with the hdr's b_pabd. 2454 */ 2455 static boolean_t 2456 arc_can_share(arc_buf_hdr_t *hdr, arc_buf_t *buf) 2457 { 2458 /* 2459 * The criteria for sharing a hdr's data are: 2460 * 1. the hdr's compression matches the buf's compression 2461 * 2. the hdr doesn't need to be byteswapped 2462 * 3. the hdr isn't already being shared 2463 * 4. the buf is either compressed or it is the last buf in the hdr list 2464 * 2465 * Criterion #4 maintains the invariant that shared uncompressed 2466 * bufs must be the final buf in the hdr's b_buf list. Reading this, you 2467 * might ask, "if a compressed buf is allocated first, won't that be the 2468 * last thing in the list?", but in that case it's impossible to create 2469 * a shared uncompressed buf anyway (because the hdr must be compressed 2470 * to have the compressed buf). You might also think that #3 is 2471 * sufficient to make this guarantee, however it's possible 2472 * (specifically in the rare L2ARC write race mentioned in 2473 * arc_buf_alloc_impl()) there will be an existing uncompressed buf that 2474 * is sharable, but wasn't at the time of its allocation. Rather than 2475 * allow a new shared uncompressed buf to be created and then shuffle 2476 * the list around to make it the last element, this simply disallows 2477 * sharing if the new buf isn't the first to be added. 2478 */ 2479 ASSERT3P(buf->b_hdr, ==, hdr); 2480 boolean_t hdr_compressed = HDR_GET_COMPRESS(hdr) != ZIO_COMPRESS_OFF; 2481 boolean_t buf_compressed = ARC_BUF_COMPRESSED(buf) != 0; 2482 return (buf_compressed == hdr_compressed && 2483 hdr->b_l1hdr.b_byteswap == DMU_BSWAP_NUMFUNCS && 2484 !HDR_SHARED_DATA(hdr) && 2485 (ARC_BUF_LAST(buf) || ARC_BUF_COMPRESSED(buf))); 2486 } 2487 2488 /* 2489 * Allocate a buf for this hdr. If you care about the data that's in the hdr, 2490 * or if you want a compressed buffer, pass those flags in. Returns 0 if the 2491 * copy was made successfully, or an error code otherwise. 2492 */ 2493 static int 2494 arc_buf_alloc_impl(arc_buf_hdr_t *hdr, void *tag, boolean_t compressed, 2495 boolean_t fill, arc_buf_t **ret) 2496 { 2497 arc_buf_t *buf; 2498 2499 ASSERT(HDR_HAS_L1HDR(hdr)); 2500 ASSERT3U(HDR_GET_LSIZE(hdr), >, 0); 2501 VERIFY(hdr->b_type == ARC_BUFC_DATA || 2502 hdr->b_type == ARC_BUFC_METADATA); 2503 ASSERT3P(ret, !=, NULL); 2504 ASSERT3P(*ret, ==, NULL); 2505 2506 buf = *ret = kmem_cache_alloc(buf_cache, KM_PUSHPAGE); 2507 buf->b_hdr = hdr; 2508 buf->b_data = NULL; 2509 buf->b_next = hdr->b_l1hdr.b_buf; 2510 buf->b_flags = 0; 2511 2512 add_reference(hdr, tag); 2513 2514 /* 2515 * We're about to change the hdr's b_flags. We must either 2516 * hold the hash_lock or be undiscoverable. 2517 */ 2518 ASSERT(MUTEX_HELD(HDR_LOCK(hdr)) || HDR_EMPTY(hdr)); 2519 2520 /* 2521 * Only honor requests for compressed bufs if the hdr is actually 2522 * compressed. 2523 */ 2524 if (compressed && HDR_GET_COMPRESS(hdr) != ZIO_COMPRESS_OFF) 2525 buf->b_flags |= ARC_BUF_FLAG_COMPRESSED; 2526 2527 /* 2528 * If the hdr's data can be shared then we share the data buffer and 2529 * set the appropriate bit in the hdr's b_flags to indicate the hdr is 2530 * sharing it's b_pabd with the arc_buf_t. Otherwise, we allocate a new 2531 * buffer to store the buf's data. 2532 * 2533 * There are two additional restrictions here because we're sharing 2534 * hdr -> buf instead of the usual buf -> hdr. First, the hdr can't be 2535 * actively involved in an L2ARC write, because if this buf is used by 2536 * an arc_write() then the hdr's data buffer will be released when the 2537 * write completes, even though the L2ARC write might still be using it. 2538 * Second, the hdr's ABD must be linear so that the buf's user doesn't 2539 * need to be ABD-aware. 2540 */ 2541 boolean_t can_share = arc_can_share(hdr, buf) && !HDR_L2_WRITING(hdr) && 2542 abd_is_linear(hdr->b_l1hdr.b_pabd); 2543 2544 /* Set up b_data and sharing */ 2545 if (can_share) { 2546 buf->b_data = abd_to_buf(hdr->b_l1hdr.b_pabd); 2547 buf->b_flags |= ARC_BUF_FLAG_SHARED; 2548 arc_hdr_set_flags(hdr, ARC_FLAG_SHARED_DATA); 2549 } else { 2550 buf->b_data = 2551 arc_get_data_buf(hdr, arc_buf_size(buf), buf); 2552 ARCSTAT_INCR(arcstat_overhead_size, arc_buf_size(buf)); 2553 } 2554 VERIFY3P(buf->b_data, !=, NULL); 2555 2556 hdr->b_l1hdr.b_buf = buf; 2557 hdr->b_l1hdr.b_bufcnt += 1; 2558 2559 /* 2560 * If the user wants the data from the hdr, we need to either copy or 2561 * decompress the data. 2562 */ 2563 if (fill) { 2564 return (arc_buf_fill(buf, ARC_BUF_COMPRESSED(buf) != 0)); 2565 } 2566 2567 return (0); 2568 } 2569 2570 static char *arc_onloan_tag = "onloan"; 2571 2572 static inline void 2573 arc_loaned_bytes_update(int64_t delta) 2574 { 2575 atomic_add_64(&arc_loaned_bytes, delta); 2576 2577 /* assert that it did not wrap around */ 2578 ASSERT3S(atomic_add_64_nv(&arc_loaned_bytes, 0), >=, 0); 2579 } 2580 2581 /* 2582 * Loan out an anonymous arc buffer. Loaned buffers are not counted as in 2583 * flight data by arc_tempreserve_space() until they are "returned". Loaned 2584 * buffers must be returned to the arc before they can be used by the DMU or 2585 * freed. 2586 */ 2587 arc_buf_t * 2588 arc_loan_buf(spa_t *spa, boolean_t is_metadata, int size) 2589 { 2590 arc_buf_t *buf = arc_alloc_buf(spa, arc_onloan_tag, 2591 is_metadata ? ARC_BUFC_METADATA : ARC_BUFC_DATA, size); 2592 2593 arc_loaned_bytes_update(arc_buf_size(buf)); 2594 2595 return (buf); 2596 } 2597 2598 arc_buf_t * 2599 arc_loan_compressed_buf(spa_t *spa, uint64_t psize, uint64_t lsize, 2600 enum zio_compress compression_type) 2601 { 2602 arc_buf_t *buf = arc_alloc_compressed_buf(spa, arc_onloan_tag, 2603 psize, lsize, compression_type); 2604 2605 arc_loaned_bytes_update(arc_buf_size(buf)); 2606 2607 return (buf); 2608 } 2609 2610 2611 /* 2612 * Return a loaned arc buffer to the arc. 2613 */ 2614 void 2615 arc_return_buf(arc_buf_t *buf, void *tag) 2616 { 2617 arc_buf_hdr_t *hdr = buf->b_hdr; 2618 2619 ASSERT3P(buf->b_data, !=, NULL); 2620 ASSERT(HDR_HAS_L1HDR(hdr)); 2621 (void) zfs_refcount_add(&hdr->b_l1hdr.b_refcnt, tag); 2622 (void) zfs_refcount_remove(&hdr->b_l1hdr.b_refcnt, arc_onloan_tag); 2623 2624 arc_loaned_bytes_update(-arc_buf_size(buf)); 2625 } 2626 2627 /* Detach an arc_buf from a dbuf (tag) */ 2628 void 2629 arc_loan_inuse_buf(arc_buf_t *buf, void *tag) 2630 { 2631 arc_buf_hdr_t *hdr = buf->b_hdr; 2632 2633 ASSERT3P(buf->b_data, !=, NULL); 2634 ASSERT(HDR_HAS_L1HDR(hdr)); 2635 (void) zfs_refcount_add(&hdr->b_l1hdr.b_refcnt, arc_onloan_tag); 2636 (void) zfs_refcount_remove(&hdr->b_l1hdr.b_refcnt, tag); 2637 2638 arc_loaned_bytes_update(arc_buf_size(buf)); 2639 } 2640 2641 static void 2642 l2arc_free_abd_on_write(abd_t *abd, size_t size, arc_buf_contents_t type) 2643 { 2644 l2arc_data_free_t *df = kmem_alloc(sizeof (*df), KM_SLEEP); 2645 2646 df->l2df_abd = abd; 2647 df->l2df_size = size; 2648 df->l2df_type = type; 2649 mutex_enter(&l2arc_free_on_write_mtx); 2650 list_insert_head(l2arc_free_on_write, df); 2651 mutex_exit(&l2arc_free_on_write_mtx); 2652 } 2653 2654 static void 2655 arc_hdr_free_on_write(arc_buf_hdr_t *hdr) 2656 { 2657 arc_state_t *state = hdr->b_l1hdr.b_state; 2658 arc_buf_contents_t type = arc_buf_type(hdr); 2659 uint64_t size = arc_hdr_size(hdr); 2660 2661 /* protected by hash lock, if in the hash table */ 2662 if (multilist_link_active(&hdr->b_l1hdr.b_arc_node)) { 2663 ASSERT(zfs_refcount_is_zero(&hdr->b_l1hdr.b_refcnt)); 2664 ASSERT(state != arc_anon && state != arc_l2c_only); 2665 2666 (void) zfs_refcount_remove_many(&state->arcs_esize[type], 2667 size, hdr); 2668 } 2669 (void) zfs_refcount_remove_many(&state->arcs_size, size, hdr); 2670 if (type == ARC_BUFC_METADATA) { 2671 arc_space_return(size, ARC_SPACE_META); 2672 } else { 2673 ASSERT(type == ARC_BUFC_DATA); 2674 arc_space_return(size, ARC_SPACE_DATA); 2675 } 2676 2677 l2arc_free_abd_on_write(hdr->b_l1hdr.b_pabd, size, type); 2678 } 2679 2680 /* 2681 * Share the arc_buf_t's data with the hdr. Whenever we are sharing the 2682 * data buffer, we transfer the refcount ownership to the hdr and update 2683 * the appropriate kstats. 2684 */ 2685 static void 2686 arc_share_buf(arc_buf_hdr_t *hdr, arc_buf_t *buf) 2687 { 2688 arc_state_t *state = hdr->b_l1hdr.b_state; 2689 2690 ASSERT(arc_can_share(hdr, buf)); 2691 ASSERT3P(hdr->b_l1hdr.b_pabd, ==, NULL); 2692 ASSERT(MUTEX_HELD(HDR_LOCK(hdr)) || HDR_EMPTY(hdr)); 2693 2694 /* 2695 * Start sharing the data buffer. We transfer the 2696 * refcount ownership to the hdr since it always owns 2697 * the refcount whenever an arc_buf_t is shared. 2698 */ 2699 zfs_refcount_transfer_ownership(&state->arcs_size, buf, hdr); 2700 hdr->b_l1hdr.b_pabd = abd_get_from_buf(buf->b_data, arc_buf_size(buf)); 2701 abd_take_ownership_of_buf(hdr->b_l1hdr.b_pabd, 2702 HDR_ISTYPE_METADATA(hdr)); 2703 arc_hdr_set_flags(hdr, ARC_FLAG_SHARED_DATA); 2704 buf->b_flags |= ARC_BUF_FLAG_SHARED; 2705 2706 /* 2707 * Since we've transferred ownership to the hdr we need 2708 * to increment its compressed and uncompressed kstats and 2709 * decrement the overhead size. 2710 */ 2711 ARCSTAT_INCR(arcstat_compressed_size, arc_hdr_size(hdr)); 2712 ARCSTAT_INCR(arcstat_uncompressed_size, HDR_GET_LSIZE(hdr)); 2713 ARCSTAT_INCR(arcstat_overhead_size, -arc_buf_size(buf)); 2714 } 2715 2716 static void 2717 arc_unshare_buf(arc_buf_hdr_t *hdr, arc_buf_t *buf) 2718 { 2719 arc_state_t *state = hdr->b_l1hdr.b_state; 2720 2721 ASSERT(arc_buf_is_shared(buf)); 2722 ASSERT3P(hdr->b_l1hdr.b_pabd, !=, NULL); 2723 ASSERT(MUTEX_HELD(HDR_LOCK(hdr)) || HDR_EMPTY(hdr)); 2724 2725 /* 2726 * We are no longer sharing this buffer so we need 2727 * to transfer its ownership to the rightful owner. 2728 */ 2729 zfs_refcount_transfer_ownership(&state->arcs_size, hdr, buf); 2730 arc_hdr_clear_flags(hdr, ARC_FLAG_SHARED_DATA); 2731 abd_release_ownership_of_buf(hdr->b_l1hdr.b_pabd); 2732 abd_put(hdr->b_l1hdr.b_pabd); 2733 hdr->b_l1hdr.b_pabd = NULL; 2734 buf->b_flags &= ~ARC_BUF_FLAG_SHARED; 2735 2736 /* 2737 * Since the buffer is no longer shared between 2738 * the arc buf and the hdr, count it as overhead. 2739 */ 2740 ARCSTAT_INCR(arcstat_compressed_size, -arc_hdr_size(hdr)); 2741 ARCSTAT_INCR(arcstat_uncompressed_size, -HDR_GET_LSIZE(hdr)); 2742 ARCSTAT_INCR(arcstat_overhead_size, arc_buf_size(buf)); 2743 } 2744 2745 /* 2746 * Remove an arc_buf_t from the hdr's buf list and return the last 2747 * arc_buf_t on the list. If no buffers remain on the list then return 2748 * NULL. 2749 */ 2750 static arc_buf_t * 2751 arc_buf_remove(arc_buf_hdr_t *hdr, arc_buf_t *buf) 2752 { 2753 ASSERT(HDR_HAS_L1HDR(hdr)); 2754 ASSERT(MUTEX_HELD(HDR_LOCK(hdr)) || HDR_EMPTY(hdr)); 2755 2756 arc_buf_t **bufp = &hdr->b_l1hdr.b_buf; 2757 arc_buf_t *lastbuf = NULL; 2758 2759 /* 2760 * Remove the buf from the hdr list and locate the last 2761 * remaining buffer on the list. 2762 */ 2763 while (*bufp != NULL) { 2764 if (*bufp == buf) 2765 *bufp = buf->b_next; 2766 2767 /* 2768 * If we've removed a buffer in the middle of 2769 * the list then update the lastbuf and update 2770 * bufp. 2771 */ 2772 if (*bufp != NULL) { 2773 lastbuf = *bufp; 2774 bufp = &(*bufp)->b_next; 2775 } 2776 } 2777 buf->b_next = NULL; 2778 ASSERT3P(lastbuf, !=, buf); 2779 IMPLY(hdr->b_l1hdr.b_bufcnt > 0, lastbuf != NULL); 2780 IMPLY(hdr->b_l1hdr.b_bufcnt > 0, hdr->b_l1hdr.b_buf != NULL); 2781 IMPLY(lastbuf != NULL, ARC_BUF_LAST(lastbuf)); 2782 2783 return (lastbuf); 2784 } 2785 2786 /* 2787 * Free up buf->b_data and pull the arc_buf_t off of the the arc_buf_hdr_t's 2788 * list and free it. 2789 */ 2790 static void 2791 arc_buf_destroy_impl(arc_buf_t *buf) 2792 { 2793 arc_buf_hdr_t *hdr = buf->b_hdr; 2794 2795 /* 2796 * Free up the data associated with the buf but only if we're not 2797 * sharing this with the hdr. If we are sharing it with the hdr, the 2798 * hdr is responsible for doing the free. 2799 */ 2800 if (buf->b_data != NULL) { 2801 /* 2802 * We're about to change the hdr's b_flags. We must either 2803 * hold the hash_lock or be undiscoverable. 2804 */ 2805 ASSERT(MUTEX_HELD(HDR_LOCK(hdr)) || HDR_EMPTY(hdr)); 2806 2807 arc_cksum_verify(buf); 2808 arc_buf_unwatch(buf); 2809 2810 if (arc_buf_is_shared(buf)) { 2811 arc_hdr_clear_flags(hdr, ARC_FLAG_SHARED_DATA); 2812 } else { 2813 uint64_t size = arc_buf_size(buf); 2814 arc_free_data_buf(hdr, buf->b_data, size, buf); 2815 ARCSTAT_INCR(arcstat_overhead_size, -size); 2816 } 2817 buf->b_data = NULL; 2818 2819 ASSERT(hdr->b_l1hdr.b_bufcnt > 0); 2820 hdr->b_l1hdr.b_bufcnt -= 1; 2821 } 2822 2823 arc_buf_t *lastbuf = arc_buf_remove(hdr, buf); 2824 2825 if (ARC_BUF_SHARED(buf) && !ARC_BUF_COMPRESSED(buf)) { 2826 /* 2827 * If the current arc_buf_t is sharing its data buffer with the 2828 * hdr, then reassign the hdr's b_pabd to share it with the new 2829 * buffer at the end of the list. The shared buffer is always 2830 * the last one on the hdr's buffer list. 2831 * 2832 * There is an equivalent case for compressed bufs, but since 2833 * they aren't guaranteed to be the last buf in the list and 2834 * that is an exceedingly rare case, we just allow that space be 2835 * wasted temporarily. 2836 */ 2837 if (lastbuf != NULL) { 2838 /* Only one buf can be shared at once */ 2839 VERIFY(!arc_buf_is_shared(lastbuf)); 2840 /* hdr is uncompressed so can't have compressed buf */ 2841 VERIFY(!ARC_BUF_COMPRESSED(lastbuf)); 2842 2843 ASSERT3P(hdr->b_l1hdr.b_pabd, !=, NULL); 2844 arc_hdr_free_pabd(hdr); 2845 2846 /* 2847 * We must setup a new shared block between the 2848 * last buffer and the hdr. The data would have 2849 * been allocated by the arc buf so we need to transfer 2850 * ownership to the hdr since it's now being shared. 2851 */ 2852 arc_share_buf(hdr, lastbuf); 2853 } 2854 } else if (HDR_SHARED_DATA(hdr)) { 2855 /* 2856 * Uncompressed shared buffers are always at the end 2857 * of the list. Compressed buffers don't have the 2858 * same requirements. This makes it hard to 2859 * simply assert that the lastbuf is shared so 2860 * we rely on the hdr's compression flags to determine 2861 * if we have a compressed, shared buffer. 2862 */ 2863 ASSERT3P(lastbuf, !=, NULL); 2864 ASSERT(arc_buf_is_shared(lastbuf) || 2865 HDR_GET_COMPRESS(hdr) != ZIO_COMPRESS_OFF); 2866 } 2867 2868 /* 2869 * Free the checksum if we're removing the last uncompressed buf from 2870 * this hdr. 2871 */ 2872 if (!arc_hdr_has_uncompressed_buf(hdr)) { 2873 arc_cksum_free(hdr); 2874 } 2875 2876 /* clean up the buf */ 2877 buf->b_hdr = NULL; 2878 kmem_cache_free(buf_cache, buf); 2879 } 2880 2881 static void 2882 arc_hdr_alloc_pabd(arc_buf_hdr_t *hdr) 2883 { 2884 ASSERT3U(HDR_GET_LSIZE(hdr), >, 0); 2885 ASSERT(HDR_HAS_L1HDR(hdr)); 2886 ASSERT(!HDR_SHARED_DATA(hdr)); 2887 2888 ASSERT3P(hdr->b_l1hdr.b_pabd, ==, NULL); 2889 hdr->b_l1hdr.b_pabd = arc_get_data_abd(hdr, arc_hdr_size(hdr), hdr); 2890 hdr->b_l1hdr.b_byteswap = DMU_BSWAP_NUMFUNCS; 2891 ASSERT3P(hdr->b_l1hdr.b_pabd, !=, NULL); 2892 2893 ARCSTAT_INCR(arcstat_compressed_size, arc_hdr_size(hdr)); 2894 ARCSTAT_INCR(arcstat_uncompressed_size, HDR_GET_LSIZE(hdr)); 2895 } 2896 2897 static void 2898 arc_hdr_free_pabd(arc_buf_hdr_t *hdr) 2899 { 2900 ASSERT(HDR_HAS_L1HDR(hdr)); 2901 ASSERT3P(hdr->b_l1hdr.b_pabd, !=, NULL); 2902 2903 /* 2904 * If the hdr is currently being written to the l2arc then 2905 * we defer freeing the data by adding it to the l2arc_free_on_write 2906 * list. The l2arc will free the data once it's finished 2907 * writing it to the l2arc device. 2908 */ 2909 if (HDR_L2_WRITING(hdr)) { 2910 arc_hdr_free_on_write(hdr); 2911 ARCSTAT_BUMP(arcstat_l2_free_on_write); 2912 } else { 2913 arc_free_data_abd(hdr, hdr->b_l1hdr.b_pabd, 2914 arc_hdr_size(hdr), hdr); 2915 } 2916 hdr->b_l1hdr.b_pabd = NULL; 2917 hdr->b_l1hdr.b_byteswap = DMU_BSWAP_NUMFUNCS; 2918 2919 ARCSTAT_INCR(arcstat_compressed_size, -arc_hdr_size(hdr)); 2920 ARCSTAT_INCR(arcstat_uncompressed_size, -HDR_GET_LSIZE(hdr)); 2921 } 2922 2923 static arc_buf_hdr_t * 2924 arc_hdr_alloc(uint64_t spa, int32_t psize, int32_t lsize, 2925 enum zio_compress compression_type, arc_buf_contents_t type) 2926 { 2927 arc_buf_hdr_t *hdr; 2928 2929 VERIFY(type == ARC_BUFC_DATA || type == ARC_BUFC_METADATA); 2930 2931 hdr = kmem_cache_alloc(hdr_full_cache, KM_PUSHPAGE); 2932 ASSERT(HDR_EMPTY(hdr)); 2933 ASSERT3P(hdr->b_l1hdr.b_freeze_cksum, ==, NULL); 2934 ASSERT3P(hdr->b_l1hdr.b_thawed, ==, NULL); 2935 HDR_SET_PSIZE(hdr, psize); 2936 HDR_SET_LSIZE(hdr, lsize); 2937 hdr->b_spa = spa; 2938 hdr->b_type = type; 2939 hdr->b_flags = 0; 2940 arc_hdr_set_flags(hdr, arc_bufc_to_flags(type) | ARC_FLAG_HAS_L1HDR); 2941 arc_hdr_set_compress(hdr, compression_type); 2942 2943 hdr->b_l1hdr.b_state = arc_anon; 2944 hdr->b_l1hdr.b_arc_access = 0; 2945 hdr->b_l1hdr.b_bufcnt = 0; 2946 hdr->b_l1hdr.b_buf = NULL; 2947 2948 /* 2949 * Allocate the hdr's buffer. This will contain either 2950 * the compressed or uncompressed data depending on the block 2951 * it references and compressed arc enablement. 2952 */ 2953 arc_hdr_alloc_pabd(hdr); 2954 ASSERT(zfs_refcount_is_zero(&hdr->b_l1hdr.b_refcnt)); 2955 2956 return (hdr); 2957 } 2958 2959 /* 2960 * Transition between the two allocation states for the arc_buf_hdr struct. 2961 * The arc_buf_hdr struct can be allocated with (hdr_full_cache) or without 2962 * (hdr_l2only_cache) the fields necessary for the L1 cache - the smaller 2963 * version is used when a cache buffer is only in the L2ARC in order to reduce 2964 * memory usage. 2965 */ 2966 static arc_buf_hdr_t * 2967 arc_hdr_realloc(arc_buf_hdr_t *hdr, kmem_cache_t *old, kmem_cache_t *new) 2968 { 2969 ASSERT(HDR_HAS_L2HDR(hdr)); 2970 2971 arc_buf_hdr_t *nhdr; 2972 l2arc_dev_t *dev = hdr->b_l2hdr.b_dev; 2973 2974 ASSERT((old == hdr_full_cache && new == hdr_l2only_cache) || 2975 (old == hdr_l2only_cache && new == hdr_full_cache)); 2976 2977 nhdr = kmem_cache_alloc(new, KM_PUSHPAGE); 2978 2979 ASSERT(MUTEX_HELD(HDR_LOCK(hdr))); 2980 buf_hash_remove(hdr); 2981 2982 bcopy(hdr, nhdr, HDR_L2ONLY_SIZE); 2983 2984 if (new == hdr_full_cache) { 2985 arc_hdr_set_flags(nhdr, ARC_FLAG_HAS_L1HDR); 2986 /* 2987 * arc_access and arc_change_state need to be aware that a 2988 * header has just come out of L2ARC, so we set its state to 2989 * l2c_only even though it's about to change. 2990 */ 2991 nhdr->b_l1hdr.b_state = arc_l2c_only; 2992 2993 /* Verify previous threads set to NULL before freeing */ 2994 ASSERT3P(nhdr->b_l1hdr.b_pabd, ==, NULL); 2995 } else { 2996 ASSERT3P(hdr->b_l1hdr.b_buf, ==, NULL); 2997 ASSERT0(hdr->b_l1hdr.b_bufcnt); 2998 ASSERT3P(hdr->b_l1hdr.b_freeze_cksum, ==, NULL); 2999 3000 /* 3001 * If we've reached here, We must have been called from 3002 * arc_evict_hdr(), as such we should have already been 3003 * removed from any ghost list we were previously on 3004 * (which protects us from racing with arc_evict_state), 3005 * thus no locking is needed during this check. 3006 */ 3007 ASSERT(!multilist_link_active(&hdr->b_l1hdr.b_arc_node)); 3008 3009 /* 3010 * A buffer must not be moved into the arc_l2c_only 3011 * state if it's not finished being written out to the 3012 * l2arc device. Otherwise, the b_l1hdr.b_pabd field 3013 * might try to be accessed, even though it was removed. 3014 */ 3015 VERIFY(!HDR_L2_WRITING(hdr)); 3016 VERIFY3P(hdr->b_l1hdr.b_pabd, ==, NULL); 3017 3018 #ifdef ZFS_DEBUG 3019 if (hdr->b_l1hdr.b_thawed != NULL) { 3020 kmem_free(hdr->b_l1hdr.b_thawed, 1); 3021 hdr->b_l1hdr.b_thawed = NULL; 3022 } 3023 #endif 3024 3025 arc_hdr_clear_flags(nhdr, ARC_FLAG_HAS_L1HDR); 3026 } 3027 /* 3028 * The header has been reallocated so we need to re-insert it into any 3029 * lists it was on. 3030 */ 3031 (void) buf_hash_insert(nhdr, NULL); 3032 3033 ASSERT(list_link_active(&hdr->b_l2hdr.b_l2node)); 3034 3035 mutex_enter(&dev->l2ad_mtx); 3036 3037 /* 3038 * We must place the realloc'ed header back into the list at 3039 * the same spot. Otherwise, if it's placed earlier in the list, 3040 * l2arc_write_buffers() could find it during the function's 3041 * write phase, and try to write it out to the l2arc. 3042 */ 3043 list_insert_after(&dev->l2ad_buflist, hdr, nhdr); 3044 list_remove(&dev->l2ad_buflist, hdr); 3045 3046 mutex_exit(&dev->l2ad_mtx); 3047 3048 /* 3049 * Since we're using the pointer address as the tag when 3050 * incrementing and decrementing the l2ad_alloc refcount, we 3051 * must remove the old pointer (that we're about to destroy) and 3052 * add the new pointer to the refcount. Otherwise we'd remove 3053 * the wrong pointer address when calling arc_hdr_destroy() later. 3054 */ 3055 3056 (void) zfs_refcount_remove_many(&dev->l2ad_alloc, arc_hdr_size(hdr), 3057 hdr); 3058 (void) zfs_refcount_add_many(&dev->l2ad_alloc, arc_hdr_size(nhdr), 3059 nhdr); 3060 3061 buf_discard_identity(hdr); 3062 kmem_cache_free(old, hdr); 3063 3064 return (nhdr); 3065 } 3066 3067 /* 3068 * Allocate a new arc_buf_hdr_t and arc_buf_t and return the buf to the caller. 3069 * The buf is returned thawed since we expect the consumer to modify it. 3070 */ 3071 arc_buf_t * 3072 arc_alloc_buf(spa_t *spa, void *tag, arc_buf_contents_t type, int32_t size) 3073 { 3074 arc_buf_hdr_t *hdr = arc_hdr_alloc(spa_load_guid(spa), size, size, 3075 ZIO_COMPRESS_OFF, type); 3076 ASSERT(!MUTEX_HELD(HDR_LOCK(hdr))); 3077 3078 arc_buf_t *buf = NULL; 3079 VERIFY0(arc_buf_alloc_impl(hdr, tag, B_FALSE, B_FALSE, &buf)); 3080 arc_buf_thaw(buf); 3081 3082 return (buf); 3083 } 3084 3085 /* 3086 * Allocate a compressed buf in the same manner as arc_alloc_buf. Don't use this 3087 * for bufs containing metadata. 3088 */ 3089 arc_buf_t * 3090 arc_alloc_compressed_buf(spa_t *spa, void *tag, uint64_t psize, uint64_t lsize, 3091 enum zio_compress compression_type) 3092 { 3093 ASSERT3U(lsize, >, 0); 3094 ASSERT3U(lsize, >=, psize); 3095 ASSERT(compression_type > ZIO_COMPRESS_OFF); 3096 ASSERT(compression_type < ZIO_COMPRESS_FUNCTIONS); 3097 3098 arc_buf_hdr_t *hdr = arc_hdr_alloc(spa_load_guid(spa), psize, lsize, 3099 compression_type, ARC_BUFC_DATA); 3100 ASSERT(!MUTEX_HELD(HDR_LOCK(hdr))); 3101 3102 arc_buf_t *buf = NULL; 3103 VERIFY0(arc_buf_alloc_impl(hdr, tag, B_TRUE, B_FALSE, &buf)); 3104 arc_buf_thaw(buf); 3105 ASSERT3P(hdr->b_l1hdr.b_freeze_cksum, ==, NULL); 3106 3107 if (!arc_buf_is_shared(buf)) { 3108 /* 3109 * To ensure that the hdr has the correct data in it if we call 3110 * arc_decompress() on this buf before it's been written to 3111 * disk, it's easiest if we just set up sharing between the 3112 * buf and the hdr. 3113 */ 3114 ASSERT(!abd_is_linear(hdr->b_l1hdr.b_pabd)); 3115 arc_hdr_free_pabd(hdr); 3116 arc_share_buf(hdr, buf); 3117 } 3118 3119 return (buf); 3120 } 3121 3122 static void 3123 arc_hdr_l2hdr_destroy(arc_buf_hdr_t *hdr) 3124 { 3125 l2arc_buf_hdr_t *l2hdr = &hdr->b_l2hdr; 3126 l2arc_dev_t *dev = l2hdr->b_dev; 3127 uint64_t psize = arc_hdr_size(hdr); 3128 3129 ASSERT(MUTEX_HELD(&dev->l2ad_mtx)); 3130 ASSERT(HDR_HAS_L2HDR(hdr)); 3131 3132 list_remove(&dev->l2ad_buflist, hdr); 3133 3134 ARCSTAT_INCR(arcstat_l2_psize, -psize); 3135 ARCSTAT_INCR(arcstat_l2_lsize, -HDR_GET_LSIZE(hdr)); 3136 3137 vdev_space_update(dev->l2ad_vdev, -psize, 0, 0); 3138 3139 (void) zfs_refcount_remove_many(&dev->l2ad_alloc, psize, hdr); 3140 arc_hdr_clear_flags(hdr, ARC_FLAG_HAS_L2HDR); 3141 } 3142 3143 static void 3144 arc_hdr_destroy(arc_buf_hdr_t *hdr) 3145 { 3146 if (HDR_HAS_L1HDR(hdr)) { 3147 ASSERT(hdr->b_l1hdr.b_buf == NULL || 3148 hdr->b_l1hdr.b_bufcnt > 0); 3149 ASSERT(zfs_refcount_is_zero(&hdr->b_l1hdr.b_refcnt)); 3150 ASSERT3P(hdr->b_l1hdr.b_state, ==, arc_anon); 3151 } 3152 ASSERT(!HDR_IO_IN_PROGRESS(hdr)); 3153 ASSERT(!HDR_IN_HASH_TABLE(hdr)); 3154 3155 if (!HDR_EMPTY(hdr)) 3156 buf_discard_identity(hdr); 3157 3158 if (HDR_HAS_L2HDR(hdr)) { 3159 l2arc_dev_t *dev = hdr->b_l2hdr.b_dev; 3160 boolean_t buflist_held = MUTEX_HELD(&dev->l2ad_mtx); 3161 3162 if (!buflist_held) 3163 mutex_enter(&dev->l2ad_mtx); 3164 3165 /* 3166 * Even though we checked this conditional above, we 3167 * need to check this again now that we have the 3168 * l2ad_mtx. This is because we could be racing with 3169 * another thread calling l2arc_evict() which might have 3170 * destroyed this header's L2 portion as we were waiting 3171 * to acquire the l2ad_mtx. If that happens, we don't 3172 * want to re-destroy the header's L2 portion. 3173 */ 3174 if (HDR_HAS_L2HDR(hdr)) 3175 arc_hdr_l2hdr_destroy(hdr); 3176 3177 if (!buflist_held) 3178 mutex_exit(&dev->l2ad_mtx); 3179 } 3180 3181 if (HDR_HAS_L1HDR(hdr)) { 3182 arc_cksum_free(hdr); 3183 3184 while (hdr->b_l1hdr.b_buf != NULL) 3185 arc_buf_destroy_impl(hdr->b_l1hdr.b_buf); 3186 3187 #ifdef ZFS_DEBUG 3188 if (hdr->b_l1hdr.b_thawed != NULL) { 3189 kmem_free(hdr->b_l1hdr.b_thawed, 1); 3190 hdr->b_l1hdr.b_thawed = NULL; 3191 } 3192 #endif 3193 3194 if (hdr->b_l1hdr.b_pabd != NULL) { 3195 arc_hdr_free_pabd(hdr); 3196 } 3197 } 3198 3199 ASSERT3P(hdr->b_hash_next, ==, NULL); 3200 if (HDR_HAS_L1HDR(hdr)) { 3201 ASSERT(!multilist_link_active(&hdr->b_l1hdr.b_arc_node)); 3202 ASSERT3P(hdr->b_l1hdr.b_acb, ==, NULL); 3203 kmem_cache_free(hdr_full_cache, hdr); 3204 } else { 3205 kmem_cache_free(hdr_l2only_cache, hdr); 3206 } 3207 } 3208 3209 void 3210 arc_buf_destroy(arc_buf_t *buf, void* tag) 3211 { 3212 arc_buf_hdr_t *hdr = buf->b_hdr; 3213 kmutex_t *hash_lock = HDR_LOCK(hdr); 3214 3215 if (hdr->b_l1hdr.b_state == arc_anon) { 3216 ASSERT3U(hdr->b_l1hdr.b_bufcnt, ==, 1); 3217 ASSERT(!HDR_IO_IN_PROGRESS(hdr)); 3218 VERIFY0(remove_reference(hdr, NULL, tag)); 3219 arc_hdr_destroy(hdr); 3220 return; 3221 } 3222 3223 mutex_enter(hash_lock); 3224 ASSERT3P(hdr, ==, buf->b_hdr); 3225 ASSERT(hdr->b_l1hdr.b_bufcnt > 0); 3226 ASSERT3P(hash_lock, ==, HDR_LOCK(hdr)); 3227 ASSERT3P(hdr->b_l1hdr.b_state, !=, arc_anon); 3228 ASSERT3P(buf->b_data, !=, NULL); 3229 3230 (void) remove_reference(hdr, hash_lock, tag); 3231 arc_buf_destroy_impl(buf); 3232 mutex_exit(hash_lock); 3233 } 3234 3235 /* 3236 * Evict the arc_buf_hdr that is provided as a parameter. The resultant 3237 * state of the header is dependent on it's state prior to entering this 3238 * function. The following transitions are possible: 3239 * 3240 * - arc_mru -> arc_mru_ghost 3241 * - arc_mfu -> arc_mfu_ghost 3242 * - arc_mru_ghost -> arc_l2c_only 3243 * - arc_mru_ghost -> deleted 3244 * - arc_mfu_ghost -> arc_l2c_only 3245 * - arc_mfu_ghost -> deleted 3246 */ 3247 static int64_t 3248 arc_evict_hdr(arc_buf_hdr_t *hdr, kmutex_t *hash_lock) 3249 { 3250 arc_state_t *evicted_state, *state; 3251 int64_t bytes_evicted = 0; 3252 3253 ASSERT(MUTEX_HELD(hash_lock)); 3254 ASSERT(HDR_HAS_L1HDR(hdr)); 3255 3256 state = hdr->b_l1hdr.b_state; 3257 if (GHOST_STATE(state)) { 3258 ASSERT(!HDR_IO_IN_PROGRESS(hdr)); 3259 ASSERT3P(hdr->b_l1hdr.b_buf, ==, NULL); 3260 3261 /* 3262 * l2arc_write_buffers() relies on a header's L1 portion 3263 * (i.e. its b_pabd field) during it's write phase. 3264 * Thus, we cannot push a header onto the arc_l2c_only 3265 * state (removing it's L1 piece) until the header is 3266 * done being written to the l2arc. 3267 */ 3268 if (HDR_HAS_L2HDR(hdr) && HDR_L2_WRITING(hdr)) { 3269 ARCSTAT_BUMP(arcstat_evict_l2_skip); 3270 return (bytes_evicted); 3271 } 3272 3273 ARCSTAT_BUMP(arcstat_deleted); 3274 bytes_evicted += HDR_GET_LSIZE(hdr); 3275 3276 DTRACE_PROBE1(arc__delete, arc_buf_hdr_t *, hdr); 3277 3278 ASSERT3P(hdr->b_l1hdr.b_pabd, ==, NULL); 3279 if (HDR_HAS_L2HDR(hdr)) { 3280 /* 3281 * This buffer is cached on the 2nd Level ARC; 3282 * don't destroy the header. 3283 */ 3284 arc_change_state(arc_l2c_only, hdr, hash_lock); 3285 /* 3286 * dropping from L1+L2 cached to L2-only, 3287 * realloc to remove the L1 header. 3288 */ 3289 hdr = arc_hdr_realloc(hdr, hdr_full_cache, 3290 hdr_l2only_cache); 3291 } else { 3292 arc_change_state(arc_anon, hdr, hash_lock); 3293 arc_hdr_destroy(hdr); 3294 } 3295 return (bytes_evicted); 3296 } 3297 3298 ASSERT(state == arc_mru || state == arc_mfu); 3299 evicted_state = (state == arc_mru) ? arc_mru_ghost : arc_mfu_ghost; 3300 3301 /* prefetch buffers have a minimum lifespan */ 3302 if (HDR_IO_IN_PROGRESS(hdr) || 3303 ((hdr->b_flags & (ARC_FLAG_PREFETCH | ARC_FLAG_INDIRECT)) && 3304 ddi_get_lbolt() - hdr->b_l1hdr.b_arc_access < 3305 arc_min_prefetch_lifespan)) { 3306 ARCSTAT_BUMP(arcstat_evict_skip); 3307 return (bytes_evicted); 3308 } 3309 3310 ASSERT0(zfs_refcount_count(&hdr->b_l1hdr.b_refcnt)); 3311 while (hdr->b_l1hdr.b_buf) { 3312 arc_buf_t *buf = hdr->b_l1hdr.b_buf; 3313 if (!mutex_tryenter(&buf->b_evict_lock)) { 3314 ARCSTAT_BUMP(arcstat_mutex_miss); 3315 break; 3316 } 3317 if (buf->b_data != NULL) 3318 bytes_evicted += HDR_GET_LSIZE(hdr); 3319 mutex_exit(&buf->b_evict_lock); 3320 arc_buf_destroy_impl(buf); 3321 } 3322 3323 if (HDR_HAS_L2HDR(hdr)) { 3324 ARCSTAT_INCR(arcstat_evict_l2_cached, HDR_GET_LSIZE(hdr)); 3325 } else { 3326 if (l2arc_write_eligible(hdr->b_spa, hdr)) { 3327 ARCSTAT_INCR(arcstat_evict_l2_eligible, 3328 HDR_GET_LSIZE(hdr)); 3329 } else { 3330 ARCSTAT_INCR(arcstat_evict_l2_ineligible, 3331 HDR_GET_LSIZE(hdr)); 3332 } 3333 } 3334 3335 if (hdr->b_l1hdr.b_bufcnt == 0) { 3336 arc_cksum_free(hdr); 3337 3338 bytes_evicted += arc_hdr_size(hdr); 3339 3340 /* 3341 * If this hdr is being evicted and has a compressed 3342 * buffer then we discard it here before we change states. 3343 * This ensures that the accounting is updated correctly 3344 * in arc_free_data_impl(). 3345 */ 3346 arc_hdr_free_pabd(hdr); 3347 3348 arc_change_state(evicted_state, hdr, hash_lock); 3349 ASSERT(HDR_IN_HASH_TABLE(hdr)); 3350 arc_hdr_set_flags(hdr, ARC_FLAG_IN_HASH_TABLE); 3351 DTRACE_PROBE1(arc__evict, arc_buf_hdr_t *, hdr); 3352 } 3353 3354 return (bytes_evicted); 3355 } 3356 3357 static uint64_t 3358 arc_evict_state_impl(multilist_t *ml, int idx, arc_buf_hdr_t *marker, 3359 uint64_t spa, int64_t bytes) 3360 { 3361 multilist_sublist_t *mls; 3362 uint64_t bytes_evicted = 0; 3363 arc_buf_hdr_t *hdr; 3364 kmutex_t *hash_lock; 3365 int evict_count = 0; 3366 3367 ASSERT3P(marker, !=, NULL); 3368 IMPLY(bytes < 0, bytes == ARC_EVICT_ALL); 3369 3370 mls = multilist_sublist_lock(ml, idx); 3371 3372 for (hdr = multilist_sublist_prev(mls, marker); hdr != NULL; 3373 hdr = multilist_sublist_prev(mls, marker)) { 3374 if ((bytes != ARC_EVICT_ALL && bytes_evicted >= bytes) || 3375 (evict_count >= zfs_arc_evict_batch_limit)) 3376 break; 3377 3378 /* 3379 * To keep our iteration location, move the marker 3380 * forward. Since we're not holding hdr's hash lock, we 3381 * must be very careful and not remove 'hdr' from the 3382 * sublist. Otherwise, other consumers might mistake the 3383 * 'hdr' as not being on a sublist when they call the 3384 * multilist_link_active() function (they all rely on 3385 * the hash lock protecting concurrent insertions and 3386 * removals). multilist_sublist_move_forward() was 3387 * specifically implemented to ensure this is the case 3388 * (only 'marker' will be removed and re-inserted). 3389 */ 3390 multilist_sublist_move_forward(mls, marker); 3391 3392 /* 3393 * The only case where the b_spa field should ever be 3394 * zero, is the marker headers inserted by 3395 * arc_evict_state(). It's possible for multiple threads 3396 * to be calling arc_evict_state() concurrently (e.g. 3397 * dsl_pool_close() and zio_inject_fault()), so we must 3398 * skip any markers we see from these other threads. 3399 */ 3400 if (hdr->b_spa == 0) 3401 continue; 3402 3403 /* we're only interested in evicting buffers of a certain spa */ 3404 if (spa != 0 && hdr->b_spa != spa) { 3405 ARCSTAT_BUMP(arcstat_evict_skip); 3406 continue; 3407 } 3408 3409 hash_lock = HDR_LOCK(hdr); 3410 3411 /* 3412 * We aren't calling this function from any code path 3413 * that would already be holding a hash lock, so we're 3414 * asserting on this assumption to be defensive in case 3415 * this ever changes. Without this check, it would be 3416 * possible to incorrectly increment arcstat_mutex_miss 3417 * below (e.g. if the code changed such that we called 3418 * this function with a hash lock held). 3419 */ 3420 ASSERT(!MUTEX_HELD(hash_lock)); 3421 3422 if (mutex_tryenter(hash_lock)) { 3423 uint64_t evicted = arc_evict_hdr(hdr, hash_lock); 3424 mutex_exit(hash_lock); 3425 3426 bytes_evicted += evicted; 3427 3428 /* 3429 * If evicted is zero, arc_evict_hdr() must have 3430 * decided to skip this header, don't increment 3431 * evict_count in this case. 3432 */ 3433 if (evicted != 0) 3434 evict_count++; 3435 3436 /* 3437 * If arc_size isn't overflowing, signal any 3438 * threads that might happen to be waiting. 3439 * 3440 * For each header evicted, we wake up a single 3441 * thread. If we used cv_broadcast, we could 3442 * wake up "too many" threads causing arc_size 3443 * to significantly overflow arc_c; since 3444 * arc_get_data_impl() doesn't check for overflow 3445 * when it's woken up (it doesn't because it's 3446 * possible for the ARC to be overflowing while 3447 * full of un-evictable buffers, and the 3448 * function should proceed in this case). 3449 * 3450 * If threads are left sleeping, due to not 3451 * using cv_broadcast here, they will be woken 3452 * up via cv_broadcast in arc_adjust_cb() just 3453 * before arc_adjust_zthr sleeps. 3454 */ 3455 mutex_enter(&arc_adjust_lock); 3456 if (!arc_is_overflowing()) 3457 cv_signal(&arc_adjust_waiters_cv); 3458 mutex_exit(&arc_adjust_lock); 3459 } else { 3460 ARCSTAT_BUMP(arcstat_mutex_miss); 3461 } 3462 } 3463 3464 multilist_sublist_unlock(mls); 3465 3466 return (bytes_evicted); 3467 } 3468 3469 /* 3470 * Evict buffers from the given arc state, until we've removed the 3471 * specified number of bytes. Move the removed buffers to the 3472 * appropriate evict state. 3473 * 3474 * This function makes a "best effort". It skips over any buffers 3475 * it can't get a hash_lock on, and so, may not catch all candidates. 3476 * It may also return without evicting as much space as requested. 3477 * 3478 * If bytes is specified using the special value ARC_EVICT_ALL, this 3479 * will evict all available (i.e. unlocked and evictable) buffers from 3480 * the given arc state; which is used by arc_flush(). 3481 */ 3482 static uint64_t 3483 arc_evict_state(arc_state_t *state, uint64_t spa, int64_t bytes, 3484 arc_buf_contents_t type) 3485 { 3486 uint64_t total_evicted = 0; 3487 multilist_t *ml = state->arcs_list[type]; 3488 int num_sublists; 3489 arc_buf_hdr_t **markers; 3490 3491 IMPLY(bytes < 0, bytes == ARC_EVICT_ALL); 3492 3493 num_sublists = multilist_get_num_sublists(ml); 3494 3495 /* 3496 * If we've tried to evict from each sublist, made some 3497 * progress, but still have not hit the target number of bytes 3498 * to evict, we want to keep trying. The markers allow us to 3499 * pick up where we left off for each individual sublist, rather 3500 * than starting from the tail each time. 3501 */ 3502 markers = kmem_zalloc(sizeof (*markers) * num_sublists, KM_SLEEP); 3503 for (int i = 0; i < num_sublists; i++) { 3504 markers[i] = kmem_cache_alloc(hdr_full_cache, KM_SLEEP); 3505 3506 /* 3507 * A b_spa of 0 is used to indicate that this header is 3508 * a marker. This fact is used in arc_adjust_type() and 3509 * arc_evict_state_impl(). 3510 */ 3511 markers[i]->b_spa = 0; 3512 3513 multilist_sublist_t *mls = multilist_sublist_lock(ml, i); 3514 multilist_sublist_insert_tail(mls, markers[i]); 3515 multilist_sublist_unlock(mls); 3516 } 3517 3518 /* 3519 * While we haven't hit our target number of bytes to evict, or 3520 * we're evicting all available buffers. 3521 */ 3522 while (total_evicted < bytes || bytes == ARC_EVICT_ALL) { 3523 /* 3524 * Start eviction using a randomly selected sublist, 3525 * this is to try and evenly balance eviction across all 3526 * sublists. Always starting at the same sublist 3527 * (e.g. index 0) would cause evictions to favor certain 3528 * sublists over others. 3529 */ 3530 int sublist_idx = multilist_get_random_index(ml); 3531 uint64_t scan_evicted = 0; 3532 3533 for (int i = 0; i < num_sublists; i++) { 3534 uint64_t bytes_remaining; 3535 uint64_t bytes_evicted; 3536 3537 if (bytes == ARC_EVICT_ALL) 3538 bytes_remaining = ARC_EVICT_ALL; 3539 else if (total_evicted < bytes) 3540 bytes_remaining = bytes - total_evicted; 3541 else 3542 break; 3543 3544 bytes_evicted = arc_evict_state_impl(ml, sublist_idx, 3545 markers[sublist_idx], spa, bytes_remaining); 3546 3547 scan_evicted += bytes_evicted; 3548 total_evicted += bytes_evicted; 3549 3550 /* we've reached the end, wrap to the beginning */ 3551 if (++sublist_idx >= num_sublists) 3552 sublist_idx = 0; 3553 } 3554 3555 /* 3556 * If we didn't evict anything during this scan, we have 3557 * no reason to believe we'll evict more during another 3558 * scan, so break the loop. 3559 */ 3560 if (scan_evicted == 0) { 3561 /* This isn't possible, let's make that obvious */ 3562 ASSERT3S(bytes, !=, 0); 3563 3564 /* 3565 * When bytes is ARC_EVICT_ALL, the only way to 3566 * break the loop is when scan_evicted is zero. 3567 * In that case, we actually have evicted enough, 3568 * so we don't want to increment the kstat. 3569 */ 3570 if (bytes != ARC_EVICT_ALL) { 3571 ASSERT3S(total_evicted, <, bytes); 3572 ARCSTAT_BUMP(arcstat_evict_not_enough); 3573 } 3574 3575 break; 3576 } 3577 } 3578 3579 for (int i = 0; i < num_sublists; i++) { 3580 multilist_sublist_t *mls = multilist_sublist_lock(ml, i); 3581 multilist_sublist_remove(mls, markers[i]); 3582 multilist_sublist_unlock(mls); 3583 3584 kmem_cache_free(hdr_full_cache, markers[i]); 3585 } 3586 kmem_free(markers, sizeof (*markers) * num_sublists); 3587 3588 return (total_evicted); 3589 } 3590 3591 /* 3592 * Flush all "evictable" data of the given type from the arc state 3593 * specified. This will not evict any "active" buffers (i.e. referenced). 3594 * 3595 * When 'retry' is set to B_FALSE, the function will make a single pass 3596 * over the state and evict any buffers that it can. Since it doesn't 3597 * continually retry the eviction, it might end up leaving some buffers 3598 * in the ARC due to lock misses. 3599 * 3600 * When 'retry' is set to B_TRUE, the function will continually retry the 3601 * eviction until *all* evictable buffers have been removed from the 3602 * state. As a result, if concurrent insertions into the state are 3603 * allowed (e.g. if the ARC isn't shutting down), this function might 3604 * wind up in an infinite loop, continually trying to evict buffers. 3605 */ 3606 static uint64_t 3607 arc_flush_state(arc_state_t *state, uint64_t spa, arc_buf_contents_t type, 3608 boolean_t retry) 3609 { 3610 uint64_t evicted = 0; 3611 3612 while (zfs_refcount_count(&state->arcs_esize[type]) != 0) { 3613 evicted += arc_evict_state(state, spa, ARC_EVICT_ALL, type); 3614 3615 if (!retry) 3616 break; 3617 } 3618 3619 return (evicted); 3620 } 3621 3622 /* 3623 * Evict the specified number of bytes from the state specified, 3624 * restricting eviction to the spa and type given. This function 3625 * prevents us from trying to evict more from a state's list than 3626 * is "evictable", and to skip evicting altogether when passed a 3627 * negative value for "bytes". In contrast, arc_evict_state() will 3628 * evict everything it can, when passed a negative value for "bytes". 3629 */ 3630 static uint64_t 3631 arc_adjust_impl(arc_state_t *state, uint64_t spa, int64_t bytes, 3632 arc_buf_contents_t type) 3633 { 3634 int64_t delta; 3635 3636 if (bytes > 0 && zfs_refcount_count(&state->arcs_esize[type]) > 0) { 3637 delta = MIN(zfs_refcount_count(&state->arcs_esize[type]), 3638 bytes); 3639 return (arc_evict_state(state, spa, delta, type)); 3640 } 3641 3642 return (0); 3643 } 3644 3645 /* 3646 * Evict metadata buffers from the cache, such that arc_meta_used is 3647 * capped by the arc_meta_limit tunable. 3648 */ 3649 static uint64_t 3650 arc_adjust_meta(uint64_t meta_used) 3651 { 3652 uint64_t total_evicted = 0; 3653 int64_t target; 3654 3655 /* 3656 * If we're over the meta limit, we want to evict enough 3657 * metadata to get back under the meta limit. We don't want to 3658 * evict so much that we drop the MRU below arc_p, though. If 3659 * we're over the meta limit more than we're over arc_p, we 3660 * evict some from the MRU here, and some from the MFU below. 3661 */ 3662 target = MIN((int64_t)(meta_used - arc_meta_limit), 3663 (int64_t)(zfs_refcount_count(&arc_anon->arcs_size) + 3664 zfs_refcount_count(&arc_mru->arcs_size) - arc_p)); 3665 3666 total_evicted += arc_adjust_impl(arc_mru, 0, target, ARC_BUFC_METADATA); 3667 3668 /* 3669 * Similar to the above, we want to evict enough bytes to get us 3670 * below the meta limit, but not so much as to drop us below the 3671 * space allotted to the MFU (which is defined as arc_c - arc_p). 3672 */ 3673 target = MIN((int64_t)(meta_used - arc_meta_limit), 3674 (int64_t)(zfs_refcount_count(&arc_mfu->arcs_size) - 3675 (arc_c - arc_p))); 3676 3677 total_evicted += arc_adjust_impl(arc_mfu, 0, target, ARC_BUFC_METADATA); 3678 3679 return (total_evicted); 3680 } 3681 3682 /* 3683 * Return the type of the oldest buffer in the given arc state 3684 * 3685 * This function will select a random sublist of type ARC_BUFC_DATA and 3686 * a random sublist of type ARC_BUFC_METADATA. The tail of each sublist 3687 * is compared, and the type which contains the "older" buffer will be 3688 * returned. 3689 */ 3690 static arc_buf_contents_t 3691 arc_adjust_type(arc_state_t *state) 3692 { 3693 multilist_t *data_ml = state->arcs_list[ARC_BUFC_DATA]; 3694 multilist_t *meta_ml = state->arcs_list[ARC_BUFC_METADATA]; 3695 int data_idx = multilist_get_random_index(data_ml); 3696 int meta_idx = multilist_get_random_index(meta_ml); 3697 multilist_sublist_t *data_mls; 3698 multilist_sublist_t *meta_mls; 3699 arc_buf_contents_t type; 3700 arc_buf_hdr_t *data_hdr; 3701 arc_buf_hdr_t *meta_hdr; 3702 3703 /* 3704 * We keep the sublist lock until we're finished, to prevent 3705 * the headers from being destroyed via arc_evict_state(). 3706 */ 3707 data_mls = multilist_sublist_lock(data_ml, data_idx); 3708 meta_mls = multilist_sublist_lock(meta_ml, meta_idx); 3709 3710 /* 3711 * These two loops are to ensure we skip any markers that 3712 * might be at the tail of the lists due to arc_evict_state(). 3713 */ 3714 3715 for (data_hdr = multilist_sublist_tail(data_mls); data_hdr != NULL; 3716 data_hdr = multilist_sublist_prev(data_mls, data_hdr)) { 3717 if (data_hdr->b_spa != 0) 3718 break; 3719 } 3720 3721 for (meta_hdr = multilist_sublist_tail(meta_mls); meta_hdr != NULL; 3722 meta_hdr = multilist_sublist_prev(meta_mls, meta_hdr)) { 3723 if (meta_hdr->b_spa != 0) 3724 break; 3725 } 3726 3727 if (data_hdr == NULL && meta_hdr == NULL) { 3728 type = ARC_BUFC_DATA; 3729 } else if (data_hdr == NULL) { 3730 ASSERT3P(meta_hdr, !=, NULL); 3731 type = ARC_BUFC_METADATA; 3732 } else if (meta_hdr == NULL) { 3733 ASSERT3P(data_hdr, !=, NULL); 3734 type = ARC_BUFC_DATA; 3735 } else { 3736 ASSERT3P(data_hdr, !=, NULL); 3737 ASSERT3P(meta_hdr, !=, NULL); 3738 3739 /* The headers can't be on the sublist without an L1 header */ 3740 ASSERT(HDR_HAS_L1HDR(data_hdr)); 3741 ASSERT(HDR_HAS_L1HDR(meta_hdr)); 3742 3743 if (data_hdr->b_l1hdr.b_arc_access < 3744 meta_hdr->b_l1hdr.b_arc_access) { 3745 type = ARC_BUFC_DATA; 3746 } else { 3747 type = ARC_BUFC_METADATA; 3748 } 3749 } 3750 3751 multilist_sublist_unlock(meta_mls); 3752 multilist_sublist_unlock(data_mls); 3753 3754 return (type); 3755 } 3756 3757 /* 3758 * Evict buffers from the cache, such that arc_size is capped by arc_c. 3759 */ 3760 static uint64_t 3761 arc_adjust(void) 3762 { 3763 uint64_t total_evicted = 0; 3764 uint64_t bytes; 3765 int64_t target; 3766 uint64_t asize = aggsum_value(&arc_size); 3767 uint64_t ameta = aggsum_value(&arc_meta_used); 3768 3769 /* 3770 * If we're over arc_meta_limit, we want to correct that before 3771 * potentially evicting data buffers below. 3772 */ 3773 total_evicted += arc_adjust_meta(ameta); 3774 3775 /* 3776 * Adjust MRU size 3777 * 3778 * If we're over the target cache size, we want to evict enough 3779 * from the list to get back to our target size. We don't want 3780 * to evict too much from the MRU, such that it drops below 3781 * arc_p. So, if we're over our target cache size more than 3782 * the MRU is over arc_p, we'll evict enough to get back to 3783 * arc_p here, and then evict more from the MFU below. 3784 */ 3785 target = MIN((int64_t)(asize - arc_c), 3786 (int64_t)(zfs_refcount_count(&arc_anon->arcs_size) + 3787 zfs_refcount_count(&arc_mru->arcs_size) + ameta - arc_p)); 3788 3789 /* 3790 * If we're below arc_meta_min, always prefer to evict data. 3791 * Otherwise, try to satisfy the requested number of bytes to 3792 * evict from the type which contains older buffers; in an 3793 * effort to keep newer buffers in the cache regardless of their 3794 * type. If we cannot satisfy the number of bytes from this 3795 * type, spill over into the next type. 3796 */ 3797 if (arc_adjust_type(arc_mru) == ARC_BUFC_METADATA && 3798 ameta > arc_meta_min) { 3799 bytes = arc_adjust_impl(arc_mru, 0, target, ARC_BUFC_METADATA); 3800 total_evicted += bytes; 3801 3802 /* 3803 * If we couldn't evict our target number of bytes from 3804 * metadata, we try to get the rest from data. 3805 */ 3806 target -= bytes; 3807 3808 total_evicted += 3809 arc_adjust_impl(arc_mru, 0, target, ARC_BUFC_DATA); 3810 } else { 3811 bytes = arc_adjust_impl(arc_mru, 0, target, ARC_BUFC_DATA); 3812 total_evicted += bytes; 3813 3814 /* 3815 * If we couldn't evict our target number of bytes from 3816 * data, we try to get the rest from metadata. 3817 */ 3818 target -= bytes; 3819 3820 total_evicted += 3821 arc_adjust_impl(arc_mru, 0, target, ARC_BUFC_METADATA); 3822 } 3823 3824 /* 3825 * Adjust MFU size 3826 * 3827 * Now that we've tried to evict enough from the MRU to get its 3828 * size back to arc_p, if we're still above the target cache 3829 * size, we evict the rest from the MFU. 3830 */ 3831 target = asize - arc_c; 3832 3833 if (arc_adjust_type(arc_mfu) == ARC_BUFC_METADATA && 3834 ameta > arc_meta_min) { 3835 bytes = arc_adjust_impl(arc_mfu, 0, target, ARC_BUFC_METADATA); 3836 total_evicted += bytes; 3837 3838 /* 3839 * If we couldn't evict our target number of bytes from 3840 * metadata, we try to get the rest from data. 3841 */ 3842 target -= bytes; 3843 3844 total_evicted += 3845 arc_adjust_impl(arc_mfu, 0, target, ARC_BUFC_DATA); 3846 } else { 3847 bytes = arc_adjust_impl(arc_mfu, 0, target, ARC_BUFC_DATA); 3848 total_evicted += bytes; 3849 3850 /* 3851 * If we couldn't evict our target number of bytes from 3852 * data, we try to get the rest from data. 3853 */ 3854 target -= bytes; 3855 3856 total_evicted += 3857 arc_adjust_impl(arc_mfu, 0, target, ARC_BUFC_METADATA); 3858 } 3859 3860 /* 3861 * Adjust ghost lists 3862 * 3863 * In addition to the above, the ARC also defines target values 3864 * for the ghost lists. The sum of the mru list and mru ghost 3865 * list should never exceed the target size of the cache, and 3866 * the sum of the mru list, mfu list, mru ghost list, and mfu 3867 * ghost list should never exceed twice the target size of the 3868 * cache. The following logic enforces these limits on the ghost 3869 * caches, and evicts from them as needed. 3870 */ 3871 target = zfs_refcount_count(&arc_mru->arcs_size) + 3872 zfs_refcount_count(&arc_mru_ghost->arcs_size) - arc_c; 3873 3874 bytes = arc_adjust_impl(arc_mru_ghost, 0, target, ARC_BUFC_DATA); 3875 total_evicted += bytes; 3876 3877 target -= bytes; 3878 3879 total_evicted += 3880 arc_adjust_impl(arc_mru_ghost, 0, target, ARC_BUFC_METADATA); 3881 3882 /* 3883 * We assume the sum of the mru list and mfu list is less than 3884 * or equal to arc_c (we enforced this above), which means we 3885 * can use the simpler of the two equations below: 3886 * 3887 * mru + mfu + mru ghost + mfu ghost <= 2 * arc_c 3888 * mru ghost + mfu ghost <= arc_c 3889 */ 3890 target = zfs_refcount_count(&arc_mru_ghost->arcs_size) + 3891 zfs_refcount_count(&arc_mfu_ghost->arcs_size) - arc_c; 3892 3893 bytes = arc_adjust_impl(arc_mfu_ghost, 0, target, ARC_BUFC_DATA); 3894 total_evicted += bytes; 3895 3896 target -= bytes; 3897 3898 total_evicted += 3899 arc_adjust_impl(arc_mfu_ghost, 0, target, ARC_BUFC_METADATA); 3900 3901 return (total_evicted); 3902 } 3903 3904 void 3905 arc_flush(spa_t *spa, boolean_t retry) 3906 { 3907 uint64_t guid = 0; 3908 3909 /* 3910 * If retry is B_TRUE, a spa must not be specified since we have 3911 * no good way to determine if all of a spa's buffers have been 3912 * evicted from an arc state. 3913 */ 3914 ASSERT(!retry || spa == 0); 3915 3916 if (spa != NULL) 3917 guid = spa_load_guid(spa); 3918 3919 (void) arc_flush_state(arc_mru, guid, ARC_BUFC_DATA, retry); 3920 (void) arc_flush_state(arc_mru, guid, ARC_BUFC_METADATA, retry); 3921 3922 (void) arc_flush_state(arc_mfu, guid, ARC_BUFC_DATA, retry); 3923 (void) arc_flush_state(arc_mfu, guid, ARC_BUFC_METADATA, retry); 3924 3925 (void) arc_flush_state(arc_mru_ghost, guid, ARC_BUFC_DATA, retry); 3926 (void) arc_flush_state(arc_mru_ghost, guid, ARC_BUFC_METADATA, retry); 3927 3928 (void) arc_flush_state(arc_mfu_ghost, guid, ARC_BUFC_DATA, retry); 3929 (void) arc_flush_state(arc_mfu_ghost, guid, ARC_BUFC_METADATA, retry); 3930 } 3931 3932 static void 3933 arc_reduce_target_size(int64_t to_free) 3934 { 3935 uint64_t asize = aggsum_value(&arc_size); 3936 if (arc_c > arc_c_min) { 3937 3938 if (arc_c > arc_c_min + to_free) 3939 atomic_add_64(&arc_c, -to_free); 3940 else 3941 arc_c = arc_c_min; 3942 3943 atomic_add_64(&arc_p, -(arc_p >> arc_shrink_shift)); 3944 if (asize < arc_c) 3945 arc_c = MAX(asize, arc_c_min); 3946 if (arc_p > arc_c) 3947 arc_p = (arc_c >> 1); 3948 ASSERT(arc_c >= arc_c_min); 3949 ASSERT((int64_t)arc_p >= 0); 3950 } 3951 3952 if (asize > arc_c) { 3953 /* See comment in arc_adjust_cb_check() on why lock+flag */ 3954 mutex_enter(&arc_adjust_lock); 3955 arc_adjust_needed = B_TRUE; 3956 mutex_exit(&arc_adjust_lock); 3957 zthr_wakeup(arc_adjust_zthr); 3958 } 3959 } 3960 3961 typedef enum free_memory_reason_t { 3962 FMR_UNKNOWN, 3963 FMR_NEEDFREE, 3964 FMR_LOTSFREE, 3965 FMR_SWAPFS_MINFREE, 3966 FMR_PAGES_PP_MAXIMUM, 3967 FMR_HEAP_ARENA, 3968 FMR_ZIO_ARENA, 3969 } free_memory_reason_t; 3970 3971 int64_t last_free_memory; 3972 free_memory_reason_t last_free_reason; 3973 3974 /* 3975 * Additional reserve of pages for pp_reserve. 3976 */ 3977 int64_t arc_pages_pp_reserve = 64; 3978 3979 /* 3980 * Additional reserve of pages for swapfs. 3981 */ 3982 int64_t arc_swapfs_reserve = 64; 3983 3984 /* 3985 * Return the amount of memory that can be consumed before reclaim will be 3986 * needed. Positive if there is sufficient free memory, negative indicates 3987 * the amount of memory that needs to be freed up. 3988 */ 3989 static int64_t 3990 arc_available_memory(void) 3991 { 3992 int64_t lowest = INT64_MAX; 3993 int64_t n; 3994 free_memory_reason_t r = FMR_UNKNOWN; 3995 3996 #ifdef _KERNEL 3997 if (needfree > 0) { 3998 n = PAGESIZE * (-needfree); 3999 if (n < lowest) { 4000 lowest = n; 4001 r = FMR_NEEDFREE; 4002 } 4003 } 4004 4005 /* 4006 * check that we're out of range of the pageout scanner. It starts to 4007 * schedule paging if freemem is less than lotsfree and needfree. 4008 * lotsfree is the high-water mark for pageout, and needfree is the 4009 * number of needed free pages. We add extra pages here to make sure 4010 * the scanner doesn't start up while we're freeing memory. 4011 */ 4012 n = PAGESIZE * (freemem - lotsfree - needfree - desfree); 4013 if (n < lowest) { 4014 lowest = n; 4015 r = FMR_LOTSFREE; 4016 } 4017 4018 /* 4019 * check to make sure that swapfs has enough space so that anon 4020 * reservations can still succeed. anon_resvmem() checks that the 4021 * availrmem is greater than swapfs_minfree, and the number of reserved 4022 * swap pages. We also add a bit of extra here just to prevent 4023 * circumstances from getting really dire. 4024 */ 4025 n = PAGESIZE * (availrmem - swapfs_minfree - swapfs_reserve - 4026 desfree - arc_swapfs_reserve); 4027 if (n < lowest) { 4028 lowest = n; 4029 r = FMR_SWAPFS_MINFREE; 4030 } 4031 4032 4033 /* 4034 * Check that we have enough availrmem that memory locking (e.g., via 4035 * mlock(3C) or memcntl(2)) can still succeed. (pages_pp_maximum 4036 * stores the number of pages that cannot be locked; when availrmem 4037 * drops below pages_pp_maximum, page locking mechanisms such as 4038 * page_pp_lock() will fail.) 4039 */ 4040 n = PAGESIZE * (availrmem - pages_pp_maximum - 4041 arc_pages_pp_reserve); 4042 if (n < lowest) { 4043 lowest = n; 4044 r = FMR_PAGES_PP_MAXIMUM; 4045 } 4046 4047 #if defined(__i386) 4048 /* 4049 * If we're on an i386 platform, it's possible that we'll exhaust the 4050 * kernel heap space before we ever run out of available physical 4051 * memory. Most checks of the size of the heap_area compare against 4052 * tune.t_minarmem, which is the minimum available real memory that we 4053 * can have in the system. However, this is generally fixed at 25 pages 4054 * which is so low that it's useless. In this comparison, we seek to 4055 * calculate the total heap-size, and reclaim if more than 3/4ths of the 4056 * heap is allocated. (Or, in the calculation, if less than 1/4th is 4057 * free) 4058 */ 4059 n = (int64_t)vmem_size(heap_arena, VMEM_FREE) - 4060 (vmem_size(heap_arena, VMEM_FREE | VMEM_ALLOC) >> 2); 4061 if (n < lowest) { 4062 lowest = n; 4063 r = FMR_HEAP_ARENA; 4064 } 4065 #endif 4066 4067 /* 4068 * If zio data pages are being allocated out of a separate heap segment, 4069 * then enforce that the size of available vmem for this arena remains 4070 * above about 1/4th (1/(2^arc_zio_arena_free_shift)) free. 4071 * 4072 * Note that reducing the arc_zio_arena_free_shift keeps more virtual 4073 * memory (in the zio_arena) free, which can avoid memory 4074 * fragmentation issues. 4075 */ 4076 if (zio_arena != NULL) { 4077 n = (int64_t)vmem_size(zio_arena, VMEM_FREE) - 4078 (vmem_size(zio_arena, VMEM_ALLOC) >> 4079 arc_zio_arena_free_shift); 4080 if (n < lowest) { 4081 lowest = n; 4082 r = FMR_ZIO_ARENA; 4083 } 4084 } 4085 #else 4086 /* Every 100 calls, free a small amount */ 4087 if (spa_get_random(100) == 0) 4088 lowest = -1024; 4089 #endif 4090 4091 last_free_memory = lowest; 4092 last_free_reason = r; 4093 4094 return (lowest); 4095 } 4096 4097 4098 /* 4099 * Determine if the system is under memory pressure and is asking 4100 * to reclaim memory. A return value of B_TRUE indicates that the system 4101 * is under memory pressure and that the arc should adjust accordingly. 4102 */ 4103 static boolean_t 4104 arc_reclaim_needed(void) 4105 { 4106 return (arc_available_memory() < 0); 4107 } 4108 4109 static void 4110 arc_kmem_reap_soon(void) 4111 { 4112 size_t i; 4113 kmem_cache_t *prev_cache = NULL; 4114 kmem_cache_t *prev_data_cache = NULL; 4115 extern kmem_cache_t *zio_buf_cache[]; 4116 extern kmem_cache_t *zio_data_buf_cache[]; 4117 extern kmem_cache_t *range_seg_cache; 4118 extern kmem_cache_t *abd_chunk_cache; 4119 4120 #ifdef _KERNEL 4121 if (aggsum_compare(&arc_meta_used, arc_meta_limit) >= 0) { 4122 /* 4123 * We are exceeding our meta-data cache limit. 4124 * Purge some DNLC entries to release holds on meta-data. 4125 */ 4126 dnlc_reduce_cache((void *)(uintptr_t)arc_reduce_dnlc_percent); 4127 } 4128 #if defined(__i386) 4129 /* 4130 * Reclaim unused memory from all kmem caches. 4131 */ 4132 kmem_reap(); 4133 #endif 4134 #endif 4135 4136 for (i = 0; i < SPA_MAXBLOCKSIZE >> SPA_MINBLOCKSHIFT; i++) { 4137 if (zio_buf_cache[i] != prev_cache) { 4138 prev_cache = zio_buf_cache[i]; 4139 kmem_cache_reap_soon(zio_buf_cache[i]); 4140 } 4141 if (zio_data_buf_cache[i] != prev_data_cache) { 4142 prev_data_cache = zio_data_buf_cache[i]; 4143 kmem_cache_reap_soon(zio_data_buf_cache[i]); 4144 } 4145 } 4146 kmem_cache_reap_soon(abd_chunk_cache); 4147 kmem_cache_reap_soon(buf_cache); 4148 kmem_cache_reap_soon(hdr_full_cache); 4149 kmem_cache_reap_soon(hdr_l2only_cache); 4150 kmem_cache_reap_soon(range_seg_cache); 4151 4152 if (zio_arena != NULL) { 4153 /* 4154 * Ask the vmem arena to reclaim unused memory from its 4155 * quantum caches. 4156 */ 4157 vmem_qcache_reap(zio_arena); 4158 } 4159 } 4160 4161 /* ARGSUSED */ 4162 static boolean_t 4163 arc_adjust_cb_check(void *arg, zthr_t *zthr) 4164 { 4165 /* 4166 * This is necessary in order for the mdb ::arc dcmd to 4167 * show up to date information. Since the ::arc command 4168 * does not call the kstat's update function, without 4169 * this call, the command may show stale stats for the 4170 * anon, mru, mru_ghost, mfu, and mfu_ghost lists. Even 4171 * with this change, the data might be up to 1 second 4172 * out of date(the arc_adjust_zthr has a maximum sleep 4173 * time of 1 second); but that should suffice. The 4174 * arc_state_t structures can be queried directly if more 4175 * accurate information is needed. 4176 */ 4177 if (arc_ksp != NULL) 4178 arc_ksp->ks_update(arc_ksp, KSTAT_READ); 4179 4180 /* 4181 * We have to rely on arc_get_data_impl() to tell us when to adjust, 4182 * rather than checking if we are overflowing here, so that we are 4183 * sure to not leave arc_get_data_impl() waiting on 4184 * arc_adjust_waiters_cv. If we have become "not overflowing" since 4185 * arc_get_data_impl() checked, we need to wake it up. We could 4186 * broadcast the CV here, but arc_get_data_impl() may have not yet 4187 * gone to sleep. We would need to use a mutex to ensure that this 4188 * function doesn't broadcast until arc_get_data_impl() has gone to 4189 * sleep (e.g. the arc_adjust_lock). However, the lock ordering of 4190 * such a lock would necessarily be incorrect with respect to the 4191 * zthr_lock, which is held before this function is called, and is 4192 * held by arc_get_data_impl() when it calls zthr_wakeup(). 4193 */ 4194 return (arc_adjust_needed); 4195 } 4196 4197 /* 4198 * Keep arc_size under arc_c by running arc_adjust which evicts data 4199 * from the ARC. 4200 */ 4201 /* ARGSUSED */ 4202 static void 4203 arc_adjust_cb(void *arg, zthr_t *zthr) 4204 { 4205 uint64_t evicted = 0; 4206 4207 /* Evict from cache */ 4208 evicted = arc_adjust(); 4209 4210 /* 4211 * If evicted is zero, we couldn't evict anything 4212 * via arc_adjust(). This could be due to hash lock 4213 * collisions, but more likely due to the majority of 4214 * arc buffers being unevictable. Therefore, even if 4215 * arc_size is above arc_c, another pass is unlikely to 4216 * be helpful and could potentially cause us to enter an 4217 * infinite loop. Additionally, zthr_iscancelled() is 4218 * checked here so that if the arc is shutting down, the 4219 * broadcast will wake any remaining arc adjust waiters. 4220 */ 4221 mutex_enter(&arc_adjust_lock); 4222 arc_adjust_needed = !zthr_iscancelled(arc_adjust_zthr) && 4223 evicted > 0 && aggsum_compare(&arc_size, arc_c) > 0; 4224 if (!arc_adjust_needed) { 4225 /* 4226 * We're either no longer overflowing, or we 4227 * can't evict anything more, so we should wake 4228 * up any waiters. 4229 */ 4230 cv_broadcast(&arc_adjust_waiters_cv); 4231 } 4232 mutex_exit(&arc_adjust_lock); 4233 } 4234 4235 /* ARGSUSED */ 4236 static boolean_t 4237 arc_reap_cb_check(void *arg, zthr_t *zthr) 4238 { 4239 int64_t free_memory = arc_available_memory(); 4240 4241 /* 4242 * If a kmem reap is already active, don't schedule more. We must 4243 * check for this because kmem_cache_reap_soon() won't actually 4244 * block on the cache being reaped (this is to prevent callers from 4245 * becoming implicitly blocked by a system-wide kmem reap -- which, 4246 * on a system with many, many full magazines, can take minutes). 4247 */ 4248 if (!kmem_cache_reap_active() && 4249 free_memory < 0) { 4250 arc_no_grow = B_TRUE; 4251 arc_warm = B_TRUE; 4252 /* 4253 * Wait at least zfs_grow_retry (default 60) seconds 4254 * before considering growing. 4255 */ 4256 arc_growtime = gethrtime() + SEC2NSEC(arc_grow_retry); 4257 return (B_TRUE); 4258 } else if (free_memory < arc_c >> arc_no_grow_shift) { 4259 arc_no_grow = B_TRUE; 4260 } else if (gethrtime() >= arc_growtime) { 4261 arc_no_grow = B_FALSE; 4262 } 4263 4264 return (B_FALSE); 4265 } 4266 4267 /* 4268 * Keep enough free memory in the system by reaping the ARC's kmem 4269 * caches. To cause more slabs to be reapable, we may reduce the 4270 * target size of the cache (arc_c), causing the arc_adjust_cb() 4271 * to free more buffers. 4272 */ 4273 /* ARGSUSED */ 4274 static void 4275 arc_reap_cb(void *arg, zthr_t *zthr) 4276 { 4277 int64_t free_memory; 4278 4279 /* 4280 * Kick off asynchronous kmem_reap()'s of all our caches. 4281 */ 4282 arc_kmem_reap_soon(); 4283 4284 /* 4285 * Wait at least arc_kmem_cache_reap_retry_ms between 4286 * arc_kmem_reap_soon() calls. Without this check it is possible to 4287 * end up in a situation where we spend lots of time reaping 4288 * caches, while we're near arc_c_min. Waiting here also gives the 4289 * subsequent free memory check a chance of finding that the 4290 * asynchronous reap has already freed enough memory, and we don't 4291 * need to call arc_reduce_target_size(). 4292 */ 4293 delay((hz * arc_kmem_cache_reap_retry_ms + 999) / 1000); 4294 4295 /* 4296 * Reduce the target size as needed to maintain the amount of free 4297 * memory in the system at a fraction of the arc_size (1/128th by 4298 * default). If oversubscribed (free_memory < 0) then reduce the 4299 * target arc_size by the deficit amount plus the fractional 4300 * amount. If free memory is positive but less then the fractional 4301 * amount, reduce by what is needed to hit the fractional amount. 4302 */ 4303 free_memory = arc_available_memory(); 4304 4305 int64_t to_free = 4306 (arc_c >> arc_shrink_shift) - free_memory; 4307 if (to_free > 0) { 4308 #ifdef _KERNEL 4309 to_free = MAX(to_free, ptob(needfree)); 4310 #endif 4311 arc_reduce_target_size(to_free); 4312 } 4313 } 4314 4315 /* 4316 * Adapt arc info given the number of bytes we are trying to add and 4317 * the state that we are comming from. This function is only called 4318 * when we are adding new content to the cache. 4319 */ 4320 static void 4321 arc_adapt(int bytes, arc_state_t *state) 4322 { 4323 int mult; 4324 uint64_t arc_p_min = (arc_c >> arc_p_min_shift); 4325 int64_t mrug_size = zfs_refcount_count(&arc_mru_ghost->arcs_size); 4326 int64_t mfug_size = zfs_refcount_count(&arc_mfu_ghost->arcs_size); 4327 4328 if (state == arc_l2c_only) 4329 return; 4330 4331 ASSERT(bytes > 0); 4332 /* 4333 * Adapt the target size of the MRU list: 4334 * - if we just hit in the MRU ghost list, then increase 4335 * the target size of the MRU list. 4336 * - if we just hit in the MFU ghost list, then increase 4337 * the target size of the MFU list by decreasing the 4338 * target size of the MRU list. 4339 */ 4340 if (state == arc_mru_ghost) { 4341 mult = (mrug_size >= mfug_size) ? 1 : (mfug_size / mrug_size); 4342 mult = MIN(mult, 10); /* avoid wild arc_p adjustment */ 4343 4344 arc_p = MIN(arc_c - arc_p_min, arc_p + bytes * mult); 4345 } else if (state == arc_mfu_ghost) { 4346 uint64_t delta; 4347 4348 mult = (mfug_size >= mrug_size) ? 1 : (mrug_size / mfug_size); 4349 mult = MIN(mult, 10); 4350 4351 delta = MIN(bytes * mult, arc_p); 4352 arc_p = MAX(arc_p_min, arc_p - delta); 4353 } 4354 ASSERT((int64_t)arc_p >= 0); 4355 4356 /* 4357 * Wake reap thread if we do not have any available memory 4358 */ 4359 if (arc_reclaim_needed()) { 4360 zthr_wakeup(arc_reap_zthr); 4361 return; 4362 } 4363 4364 4365 if (arc_no_grow) 4366 return; 4367 4368 if (arc_c >= arc_c_max) 4369 return; 4370 4371 /* 4372 * If we're within (2 * maxblocksize) bytes of the target 4373 * cache size, increment the target cache size 4374 */ 4375 if (aggsum_compare(&arc_size, arc_c - (2ULL << SPA_MAXBLOCKSHIFT)) > 4376 0) { 4377 atomic_add_64(&arc_c, (int64_t)bytes); 4378 if (arc_c > arc_c_max) 4379 arc_c = arc_c_max; 4380 else if (state == arc_anon) 4381 atomic_add_64(&arc_p, (int64_t)bytes); 4382 if (arc_p > arc_c) 4383 arc_p = arc_c; 4384 } 4385 ASSERT((int64_t)arc_p >= 0); 4386 } 4387 4388 /* 4389 * Check if arc_size has grown past our upper threshold, determined by 4390 * zfs_arc_overflow_shift. 4391 */ 4392 static boolean_t 4393 arc_is_overflowing(void) 4394 { 4395 /* Always allow at least one block of overflow */ 4396 uint64_t overflow = MAX(SPA_MAXBLOCKSIZE, 4397 arc_c >> zfs_arc_overflow_shift); 4398 4399 /* 4400 * We just compare the lower bound here for performance reasons. Our 4401 * primary goals are to make sure that the arc never grows without 4402 * bound, and that it can reach its maximum size. This check 4403 * accomplishes both goals. The maximum amount we could run over by is 4404 * 2 * aggsum_borrow_multiplier * NUM_CPUS * the average size of a block 4405 * in the ARC. In practice, that's in the tens of MB, which is low 4406 * enough to be safe. 4407 */ 4408 return (aggsum_lower_bound(&arc_size) >= arc_c + overflow); 4409 } 4410 4411 static abd_t * 4412 arc_get_data_abd(arc_buf_hdr_t *hdr, uint64_t size, void *tag) 4413 { 4414 arc_buf_contents_t type = arc_buf_type(hdr); 4415 4416 arc_get_data_impl(hdr, size, tag); 4417 if (type == ARC_BUFC_METADATA) { 4418 return (abd_alloc(size, B_TRUE)); 4419 } else { 4420 ASSERT(type == ARC_BUFC_DATA); 4421 return (abd_alloc(size, B_FALSE)); 4422 } 4423 } 4424 4425 static void * 4426 arc_get_data_buf(arc_buf_hdr_t *hdr, uint64_t size, void *tag) 4427 { 4428 arc_buf_contents_t type = arc_buf_type(hdr); 4429 4430 arc_get_data_impl(hdr, size, tag); 4431 if (type == ARC_BUFC_METADATA) { 4432 return (zio_buf_alloc(size)); 4433 } else { 4434 ASSERT(type == ARC_BUFC_DATA); 4435 return (zio_data_buf_alloc(size)); 4436 } 4437 } 4438 4439 /* 4440 * Allocate a block and return it to the caller. If we are hitting the 4441 * hard limit for the cache size, we must sleep, waiting for the eviction 4442 * thread to catch up. If we're past the target size but below the hard 4443 * limit, we'll only signal the reclaim thread and continue on. 4444 */ 4445 static void 4446 arc_get_data_impl(arc_buf_hdr_t *hdr, uint64_t size, void *tag) 4447 { 4448 arc_state_t *state = hdr->b_l1hdr.b_state; 4449 arc_buf_contents_t type = arc_buf_type(hdr); 4450 4451 arc_adapt(size, state); 4452 4453 /* 4454 * If arc_size is currently overflowing, and has grown past our 4455 * upper limit, we must be adding data faster than the evict 4456 * thread can evict. Thus, to ensure we don't compound the 4457 * problem by adding more data and forcing arc_size to grow even 4458 * further past it's target size, we halt and wait for the 4459 * eviction thread to catch up. 4460 * 4461 * It's also possible that the reclaim thread is unable to evict 4462 * enough buffers to get arc_size below the overflow limit (e.g. 4463 * due to buffers being un-evictable, or hash lock collisions). 4464 * In this case, we want to proceed regardless if we're 4465 * overflowing; thus we don't use a while loop here. 4466 */ 4467 if (arc_is_overflowing()) { 4468 mutex_enter(&arc_adjust_lock); 4469 4470 /* 4471 * Now that we've acquired the lock, we may no longer be 4472 * over the overflow limit, lets check. 4473 * 4474 * We're ignoring the case of spurious wake ups. If that 4475 * were to happen, it'd let this thread consume an ARC 4476 * buffer before it should have (i.e. before we're under 4477 * the overflow limit and were signalled by the reclaim 4478 * thread). As long as that is a rare occurrence, it 4479 * shouldn't cause any harm. 4480 */ 4481 if (arc_is_overflowing()) { 4482 arc_adjust_needed = B_TRUE; 4483 zthr_wakeup(arc_adjust_zthr); 4484 (void) cv_wait(&arc_adjust_waiters_cv, 4485 &arc_adjust_lock); 4486 } 4487 mutex_exit(&arc_adjust_lock); 4488 } 4489 4490 VERIFY3U(hdr->b_type, ==, type); 4491 if (type == ARC_BUFC_METADATA) { 4492 arc_space_consume(size, ARC_SPACE_META); 4493 } else { 4494 arc_space_consume(size, ARC_SPACE_DATA); 4495 } 4496 4497 /* 4498 * Update the state size. Note that ghost states have a 4499 * "ghost size" and so don't need to be updated. 4500 */ 4501 if (!GHOST_STATE(state)) { 4502 4503 (void) zfs_refcount_add_many(&state->arcs_size, size, tag); 4504 4505 /* 4506 * If this is reached via arc_read, the link is 4507 * protected by the hash lock. If reached via 4508 * arc_buf_alloc, the header should not be accessed by 4509 * any other thread. And, if reached via arc_read_done, 4510 * the hash lock will protect it if it's found in the 4511 * hash table; otherwise no other thread should be 4512 * trying to [add|remove]_reference it. 4513 */ 4514 if (multilist_link_active(&hdr->b_l1hdr.b_arc_node)) { 4515 ASSERT(zfs_refcount_is_zero(&hdr->b_l1hdr.b_refcnt)); 4516 (void) zfs_refcount_add_many(&state->arcs_esize[type], 4517 size, tag); 4518 } 4519 4520 /* 4521 * If we are growing the cache, and we are adding anonymous 4522 * data, and we have outgrown arc_p, update arc_p 4523 */ 4524 if (aggsum_compare(&arc_size, arc_c) < 0 && 4525 hdr->b_l1hdr.b_state == arc_anon && 4526 (zfs_refcount_count(&arc_anon->arcs_size) + 4527 zfs_refcount_count(&arc_mru->arcs_size) > arc_p)) 4528 arc_p = MIN(arc_c, arc_p + size); 4529 } 4530 } 4531 4532 static void 4533 arc_free_data_abd(arc_buf_hdr_t *hdr, abd_t *abd, uint64_t size, void *tag) 4534 { 4535 arc_free_data_impl(hdr, size, tag); 4536 abd_free(abd); 4537 } 4538 4539 static void 4540 arc_free_data_buf(arc_buf_hdr_t *hdr, void *buf, uint64_t size, void *tag) 4541 { 4542 arc_buf_contents_t type = arc_buf_type(hdr); 4543 4544 arc_free_data_impl(hdr, size, tag); 4545 if (type == ARC_BUFC_METADATA) { 4546 zio_buf_free(buf, size); 4547 } else { 4548 ASSERT(type == ARC_BUFC_DATA); 4549 zio_data_buf_free(buf, size); 4550 } 4551 } 4552 4553 /* 4554 * Free the arc data buffer. 4555 */ 4556 static void 4557 arc_free_data_impl(arc_buf_hdr_t *hdr, uint64_t size, void *tag) 4558 { 4559 arc_state_t *state = hdr->b_l1hdr.b_state; 4560 arc_buf_contents_t type = arc_buf_type(hdr); 4561 4562 /* protected by hash lock, if in the hash table */ 4563 if (multilist_link_active(&hdr->b_l1hdr.b_arc_node)) { 4564 ASSERT(zfs_refcount_is_zero(&hdr->b_l1hdr.b_refcnt)); 4565 ASSERT(state != arc_anon && state != arc_l2c_only); 4566 4567 (void) zfs_refcount_remove_many(&state->arcs_esize[type], 4568 size, tag); 4569 } 4570 (void) zfs_refcount_remove_many(&state->arcs_size, size, tag); 4571 4572 VERIFY3U(hdr->b_type, ==, type); 4573 if (type == ARC_BUFC_METADATA) { 4574 arc_space_return(size, ARC_SPACE_META); 4575 } else { 4576 ASSERT(type == ARC_BUFC_DATA); 4577 arc_space_return(size, ARC_SPACE_DATA); 4578 } 4579 } 4580 4581 /* 4582 * This routine is called whenever a buffer is accessed. 4583 * NOTE: the hash lock is dropped in this function. 4584 */ 4585 static void 4586 arc_access(arc_buf_hdr_t *hdr, kmutex_t *hash_lock) 4587 { 4588 clock_t now; 4589 4590 ASSERT(MUTEX_HELD(hash_lock)); 4591 ASSERT(HDR_HAS_L1HDR(hdr)); 4592 4593 if (hdr->b_l1hdr.b_state == arc_anon) { 4594 /* 4595 * This buffer is not in the cache, and does not 4596 * appear in our "ghost" list. Add the new buffer 4597 * to the MRU state. 4598 */ 4599 4600 ASSERT0(hdr->b_l1hdr.b_arc_access); 4601 hdr->b_l1hdr.b_arc_access = ddi_get_lbolt(); 4602 DTRACE_PROBE1(new_state__mru, arc_buf_hdr_t *, hdr); 4603 arc_change_state(arc_mru, hdr, hash_lock); 4604 4605 } else if (hdr->b_l1hdr.b_state == arc_mru) { 4606 now = ddi_get_lbolt(); 4607 4608 /* 4609 * If this buffer is here because of a prefetch, then either: 4610 * - clear the flag if this is a "referencing" read 4611 * (any subsequent access will bump this into the MFU state). 4612 * or 4613 * - move the buffer to the head of the list if this is 4614 * another prefetch (to make it less likely to be evicted). 4615 */ 4616 if (HDR_PREFETCH(hdr)) { 4617 if (zfs_refcount_count(&hdr->b_l1hdr.b_refcnt) == 0) { 4618 /* link protected by hash lock */ 4619 ASSERT(multilist_link_active( 4620 &hdr->b_l1hdr.b_arc_node)); 4621 } else { 4622 arc_hdr_clear_flags(hdr, ARC_FLAG_PREFETCH); 4623 ARCSTAT_BUMP(arcstat_mru_hits); 4624 } 4625 hdr->b_l1hdr.b_arc_access = now; 4626 return; 4627 } 4628 4629 /* 4630 * This buffer has been "accessed" only once so far, 4631 * but it is still in the cache. Move it to the MFU 4632 * state. 4633 */ 4634 if (now > hdr->b_l1hdr.b_arc_access + ARC_MINTIME) { 4635 /* 4636 * More than 125ms have passed since we 4637 * instantiated this buffer. Move it to the 4638 * most frequently used state. 4639 */ 4640 hdr->b_l1hdr.b_arc_access = now; 4641 DTRACE_PROBE1(new_state__mfu, arc_buf_hdr_t *, hdr); 4642 arc_change_state(arc_mfu, hdr, hash_lock); 4643 } 4644 ARCSTAT_BUMP(arcstat_mru_hits); 4645 } else if (hdr->b_l1hdr.b_state == arc_mru_ghost) { 4646 arc_state_t *new_state; 4647 /* 4648 * This buffer has been "accessed" recently, but 4649 * was evicted from the cache. Move it to the 4650 * MFU state. 4651 */ 4652 4653 if (HDR_PREFETCH(hdr)) { 4654 new_state = arc_mru; 4655 if (zfs_refcount_count(&hdr->b_l1hdr.b_refcnt) > 0) 4656 arc_hdr_clear_flags(hdr, ARC_FLAG_PREFETCH); 4657 DTRACE_PROBE1(new_state__mru, arc_buf_hdr_t *, hdr); 4658 } else { 4659 new_state = arc_mfu; 4660 DTRACE_PROBE1(new_state__mfu, arc_buf_hdr_t *, hdr); 4661 } 4662 4663 hdr->b_l1hdr.b_arc_access = ddi_get_lbolt(); 4664 arc_change_state(new_state, hdr, hash_lock); 4665 4666 ARCSTAT_BUMP(arcstat_mru_ghost_hits); 4667 } else if (hdr->b_l1hdr.b_state == arc_mfu) { 4668 /* 4669 * This buffer has been accessed more than once and is 4670 * still in the cache. Keep it in the MFU state. 4671 * 4672 * NOTE: an add_reference() that occurred when we did 4673 * the arc_read() will have kicked this off the list. 4674 * If it was a prefetch, we will explicitly move it to 4675 * the head of the list now. 4676 */ 4677 if ((HDR_PREFETCH(hdr)) != 0) { 4678 ASSERT(zfs_refcount_is_zero(&hdr->b_l1hdr.b_refcnt)); 4679 /* link protected by hash_lock */ 4680 ASSERT(multilist_link_active(&hdr->b_l1hdr.b_arc_node)); 4681 } 4682 ARCSTAT_BUMP(arcstat_mfu_hits); 4683 hdr->b_l1hdr.b_arc_access = ddi_get_lbolt(); 4684 } else if (hdr->b_l1hdr.b_state == arc_mfu_ghost) { 4685 arc_state_t *new_state = arc_mfu; 4686 /* 4687 * This buffer has been accessed more than once but has 4688 * been evicted from the cache. Move it back to the 4689 * MFU state. 4690 */ 4691 4692 if (HDR_PREFETCH(hdr)) { 4693 /* 4694 * This is a prefetch access... 4695 * move this block back to the MRU state. 4696 */ 4697 ASSERT0(zfs_refcount_count(&hdr->b_l1hdr.b_refcnt)); 4698 new_state = arc_mru; 4699 } 4700 4701 hdr->b_l1hdr.b_arc_access = ddi_get_lbolt(); 4702 DTRACE_PROBE1(new_state__mfu, arc_buf_hdr_t *, hdr); 4703 arc_change_state(new_state, hdr, hash_lock); 4704 4705 ARCSTAT_BUMP(arcstat_mfu_ghost_hits); 4706 } else if (hdr->b_l1hdr.b_state == arc_l2c_only) { 4707 /* 4708 * This buffer is on the 2nd Level ARC. 4709 */ 4710 4711 hdr->b_l1hdr.b_arc_access = ddi_get_lbolt(); 4712 DTRACE_PROBE1(new_state__mfu, arc_buf_hdr_t *, hdr); 4713 arc_change_state(arc_mfu, hdr, hash_lock); 4714 } else { 4715 ASSERT(!"invalid arc state"); 4716 } 4717 } 4718 4719 /* 4720 * This routine is called by dbuf_hold() to update the arc_access() state 4721 * which otherwise would be skipped for entries in the dbuf cache. 4722 */ 4723 void 4724 arc_buf_access(arc_buf_t *buf) 4725 { 4726 mutex_enter(&buf->b_evict_lock); 4727 arc_buf_hdr_t *hdr = buf->b_hdr; 4728 4729 /* 4730 * Avoid taking the hash_lock when possible as an optimization. 4731 * The header must be checked again under the hash_lock in order 4732 * to handle the case where it is concurrently being released. 4733 */ 4734 if (hdr->b_l1hdr.b_state == arc_anon || HDR_EMPTY(hdr)) { 4735 mutex_exit(&buf->b_evict_lock); 4736 return; 4737 } 4738 4739 kmutex_t *hash_lock = HDR_LOCK(hdr); 4740 mutex_enter(hash_lock); 4741 4742 if (hdr->b_l1hdr.b_state == arc_anon || HDR_EMPTY(hdr)) { 4743 mutex_exit(hash_lock); 4744 mutex_exit(&buf->b_evict_lock); 4745 ARCSTAT_BUMP(arcstat_access_skip); 4746 return; 4747 } 4748 4749 mutex_exit(&buf->b_evict_lock); 4750 4751 ASSERT(hdr->b_l1hdr.b_state == arc_mru || 4752 hdr->b_l1hdr.b_state == arc_mfu); 4753 4754 DTRACE_PROBE1(arc__hit, arc_buf_hdr_t *, hdr); 4755 arc_access(hdr, hash_lock); 4756 mutex_exit(hash_lock); 4757 4758 ARCSTAT_BUMP(arcstat_hits); 4759 ARCSTAT_CONDSTAT(!HDR_PREFETCH(hdr), 4760 demand, prefetch, !HDR_ISTYPE_METADATA(hdr), data, metadata, hits); 4761 } 4762 4763 /* a generic arc_done_func_t which you can use */ 4764 /* ARGSUSED */ 4765 void 4766 arc_bcopy_func(zio_t *zio, arc_buf_t *buf, void *arg) 4767 { 4768 if (zio == NULL || zio->io_error == 0) 4769 bcopy(buf->b_data, arg, arc_buf_size(buf)); 4770 arc_buf_destroy(buf, arg); 4771 } 4772 4773 /* a generic arc_done_func_t */ 4774 void 4775 arc_getbuf_func(zio_t *zio, arc_buf_t *buf, void *arg) 4776 { 4777 arc_buf_t **bufp = arg; 4778 if (buf == NULL) { 4779 ASSERT(zio == NULL || zio->io_error != 0); 4780 *bufp = NULL; 4781 } else { 4782 ASSERT(zio == NULL || zio->io_error == 0); 4783 *bufp = buf; 4784 ASSERT(buf->b_data != NULL); 4785 } 4786 } 4787 4788 static void 4789 arc_hdr_verify(arc_buf_hdr_t *hdr, blkptr_t *bp) 4790 { 4791 if (BP_IS_HOLE(bp) || BP_IS_EMBEDDED(bp)) { 4792 ASSERT3U(HDR_GET_PSIZE(hdr), ==, 0); 4793 ASSERT3U(HDR_GET_COMPRESS(hdr), ==, ZIO_COMPRESS_OFF); 4794 } else { 4795 if (HDR_COMPRESSION_ENABLED(hdr)) { 4796 ASSERT3U(HDR_GET_COMPRESS(hdr), ==, 4797 BP_GET_COMPRESS(bp)); 4798 } 4799 ASSERT3U(HDR_GET_LSIZE(hdr), ==, BP_GET_LSIZE(bp)); 4800 ASSERT3U(HDR_GET_PSIZE(hdr), ==, BP_GET_PSIZE(bp)); 4801 } 4802 } 4803 4804 static void 4805 arc_read_done(zio_t *zio) 4806 { 4807 arc_buf_hdr_t *hdr = zio->io_private; 4808 kmutex_t *hash_lock = NULL; 4809 arc_callback_t *callback_list; 4810 arc_callback_t *acb; 4811 boolean_t freeable = B_FALSE; 4812 boolean_t no_zio_error = (zio->io_error == 0); 4813 4814 /* 4815 * The hdr was inserted into hash-table and removed from lists 4816 * prior to starting I/O. We should find this header, since 4817 * it's in the hash table, and it should be legit since it's 4818 * not possible to evict it during the I/O. The only possible 4819 * reason for it not to be found is if we were freed during the 4820 * read. 4821 */ 4822 if (HDR_IN_HASH_TABLE(hdr)) { 4823 ASSERT3U(hdr->b_birth, ==, BP_PHYSICAL_BIRTH(zio->io_bp)); 4824 ASSERT3U(hdr->b_dva.dva_word[0], ==, 4825 BP_IDENTITY(zio->io_bp)->dva_word[0]); 4826 ASSERT3U(hdr->b_dva.dva_word[1], ==, 4827 BP_IDENTITY(zio->io_bp)->dva_word[1]); 4828 4829 arc_buf_hdr_t *found = buf_hash_find(hdr->b_spa, zio->io_bp, 4830 &hash_lock); 4831 4832 ASSERT((found == hdr && 4833 DVA_EQUAL(&hdr->b_dva, BP_IDENTITY(zio->io_bp))) || 4834 (found == hdr && HDR_L2_READING(hdr))); 4835 ASSERT3P(hash_lock, !=, NULL); 4836 } 4837 4838 if (no_zio_error) { 4839 /* byteswap if necessary */ 4840 if (BP_SHOULD_BYTESWAP(zio->io_bp)) { 4841 if (BP_GET_LEVEL(zio->io_bp) > 0) { 4842 hdr->b_l1hdr.b_byteswap = DMU_BSWAP_UINT64; 4843 } else { 4844 hdr->b_l1hdr.b_byteswap = 4845 DMU_OT_BYTESWAP(BP_GET_TYPE(zio->io_bp)); 4846 } 4847 } else { 4848 hdr->b_l1hdr.b_byteswap = DMU_BSWAP_NUMFUNCS; 4849 } 4850 } 4851 4852 arc_hdr_clear_flags(hdr, ARC_FLAG_L2_EVICTED); 4853 if (l2arc_noprefetch && HDR_PREFETCH(hdr)) 4854 arc_hdr_clear_flags(hdr, ARC_FLAG_L2CACHE); 4855 4856 callback_list = hdr->b_l1hdr.b_acb; 4857 ASSERT3P(callback_list, !=, NULL); 4858 4859 if (hash_lock && no_zio_error && hdr->b_l1hdr.b_state == arc_anon) { 4860 /* 4861 * Only call arc_access on anonymous buffers. This is because 4862 * if we've issued an I/O for an evicted buffer, we've already 4863 * called arc_access (to prevent any simultaneous readers from 4864 * getting confused). 4865 */ 4866 arc_access(hdr, hash_lock); 4867 } 4868 4869 /* 4870 * If a read request has a callback (i.e. acb_done is not NULL), then we 4871 * make a buf containing the data according to the parameters which were 4872 * passed in. The implementation of arc_buf_alloc_impl() ensures that we 4873 * aren't needlessly decompressing the data multiple times. 4874 */ 4875 int callback_cnt = 0; 4876 for (acb = callback_list; acb != NULL; acb = acb->acb_next) { 4877 if (!acb->acb_done) 4878 continue; 4879 4880 /* This is a demand read since prefetches don't use callbacks */ 4881 callback_cnt++; 4882 4883 if (no_zio_error) { 4884 int error = arc_buf_alloc_impl(hdr, acb->acb_private, 4885 acb->acb_compressed, zio->io_error == 0, 4886 &acb->acb_buf); 4887 if (error != 0) { 4888 /* 4889 * Decompression failed. Set io_error 4890 * so that when we call acb_done (below), 4891 * we will indicate that the read failed. 4892 * Note that in the unusual case where one 4893 * callback is compressed and another 4894 * uncompressed, we will mark all of them 4895 * as failed, even though the uncompressed 4896 * one can't actually fail. In this case, 4897 * the hdr will not be anonymous, because 4898 * if there are multiple callbacks, it's 4899 * because multiple threads found the same 4900 * arc buf in the hash table. 4901 */ 4902 zio->io_error = error; 4903 } 4904 } 4905 } 4906 /* 4907 * If there are multiple callbacks, we must have the hash lock, 4908 * because the only way for multiple threads to find this hdr is 4909 * in the hash table. This ensures that if there are multiple 4910 * callbacks, the hdr is not anonymous. If it were anonymous, 4911 * we couldn't use arc_buf_destroy() in the error case below. 4912 */ 4913 ASSERT(callback_cnt < 2 || hash_lock != NULL); 4914 4915 hdr->b_l1hdr.b_acb = NULL; 4916 arc_hdr_clear_flags(hdr, ARC_FLAG_IO_IN_PROGRESS); 4917 if (callback_cnt == 0) { 4918 ASSERT(HDR_PREFETCH(hdr)); 4919 ASSERT0(hdr->b_l1hdr.b_bufcnt); 4920 ASSERT3P(hdr->b_l1hdr.b_pabd, !=, NULL); 4921 } 4922 4923 ASSERT(zfs_refcount_is_zero(&hdr->b_l1hdr.b_refcnt) || 4924 callback_list != NULL); 4925 4926 if (no_zio_error) { 4927 arc_hdr_verify(hdr, zio->io_bp); 4928 } else { 4929 arc_hdr_set_flags(hdr, ARC_FLAG_IO_ERROR); 4930 if (hdr->b_l1hdr.b_state != arc_anon) 4931 arc_change_state(arc_anon, hdr, hash_lock); 4932 if (HDR_IN_HASH_TABLE(hdr)) 4933 buf_hash_remove(hdr); 4934 freeable = zfs_refcount_is_zero(&hdr->b_l1hdr.b_refcnt); 4935 } 4936 4937 /* 4938 * Broadcast before we drop the hash_lock to avoid the possibility 4939 * that the hdr (and hence the cv) might be freed before we get to 4940 * the cv_broadcast(). 4941 */ 4942 cv_broadcast(&hdr->b_l1hdr.b_cv); 4943 4944 if (hash_lock != NULL) { 4945 mutex_exit(hash_lock); 4946 } else { 4947 /* 4948 * This block was freed while we waited for the read to 4949 * complete. It has been removed from the hash table and 4950 * moved to the anonymous state (so that it won't show up 4951 * in the cache). 4952 */ 4953 ASSERT3P(hdr->b_l1hdr.b_state, ==, arc_anon); 4954 freeable = zfs_refcount_is_zero(&hdr->b_l1hdr.b_refcnt); 4955 } 4956 4957 /* execute each callback and free its structure */ 4958 while ((acb = callback_list) != NULL) { 4959 if (acb->acb_done != NULL) { 4960 if (zio->io_error != 0 && acb->acb_buf != NULL) { 4961 /* 4962 * If arc_buf_alloc_impl() fails during 4963 * decompression, the buf will still be 4964 * allocated, and needs to be freed here. 4965 */ 4966 arc_buf_destroy(acb->acb_buf, acb->acb_private); 4967 acb->acb_buf = NULL; 4968 } 4969 acb->acb_done(zio, acb->acb_buf, acb->acb_private); 4970 } 4971 4972 if (acb->acb_zio_dummy != NULL) { 4973 acb->acb_zio_dummy->io_error = zio->io_error; 4974 zio_nowait(acb->acb_zio_dummy); 4975 } 4976 4977 callback_list = acb->acb_next; 4978 kmem_free(acb, sizeof (arc_callback_t)); 4979 } 4980 4981 if (freeable) 4982 arc_hdr_destroy(hdr); 4983 } 4984 4985 /* 4986 * "Read" the block at the specified DVA (in bp) via the 4987 * cache. If the block is found in the cache, invoke the provided 4988 * callback immediately and return. Note that the `zio' parameter 4989 * in the callback will be NULL in this case, since no IO was 4990 * required. If the block is not in the cache pass the read request 4991 * on to the spa with a substitute callback function, so that the 4992 * requested block will be added to the cache. 4993 * 4994 * If a read request arrives for a block that has a read in-progress, 4995 * either wait for the in-progress read to complete (and return the 4996 * results); or, if this is a read with a "done" func, add a record 4997 * to the read to invoke the "done" func when the read completes, 4998 * and return; or just return. 4999 * 5000 * arc_read_done() will invoke all the requested "done" functions 5001 * for readers of this block. 5002 */ 5003 int 5004 arc_read(zio_t *pio, spa_t *spa, const blkptr_t *bp, arc_done_func_t *done, 5005 void *private, zio_priority_t priority, int zio_flags, 5006 arc_flags_t *arc_flags, const zbookmark_phys_t *zb) 5007 { 5008 arc_buf_hdr_t *hdr = NULL; 5009 kmutex_t *hash_lock = NULL; 5010 zio_t *rzio; 5011 uint64_t guid = spa_load_guid(spa); 5012 boolean_t compressed_read = (zio_flags & ZIO_FLAG_RAW) != 0; 5013 5014 ASSERT(!BP_IS_EMBEDDED(bp) || 5015 BPE_GET_ETYPE(bp) == BP_EMBEDDED_TYPE_DATA); 5016 5017 top: 5018 if (!BP_IS_EMBEDDED(bp)) { 5019 /* 5020 * Embedded BP's have no DVA and require no I/O to "read". 5021 * Create an anonymous arc buf to back it. 5022 */ 5023 hdr = buf_hash_find(guid, bp, &hash_lock); 5024 } 5025 5026 if (hdr != NULL && HDR_HAS_L1HDR(hdr) && hdr->b_l1hdr.b_pabd != NULL) { 5027 arc_buf_t *buf = NULL; 5028 *arc_flags |= ARC_FLAG_CACHED; 5029 5030 if (HDR_IO_IN_PROGRESS(hdr)) { 5031 5032 if ((hdr->b_flags & ARC_FLAG_PRIO_ASYNC_READ) && 5033 priority == ZIO_PRIORITY_SYNC_READ) { 5034 /* 5035 * This sync read must wait for an 5036 * in-progress async read (e.g. a predictive 5037 * prefetch). Async reads are queued 5038 * separately at the vdev_queue layer, so 5039 * this is a form of priority inversion. 5040 * Ideally, we would "inherit" the demand 5041 * i/o's priority by moving the i/o from 5042 * the async queue to the synchronous queue, 5043 * but there is currently no mechanism to do 5044 * so. Track this so that we can evaluate 5045 * the magnitude of this potential performance 5046 * problem. 5047 * 5048 * Note that if the prefetch i/o is already 5049 * active (has been issued to the device), 5050 * the prefetch improved performance, because 5051 * we issued it sooner than we would have 5052 * without the prefetch. 5053 */ 5054 DTRACE_PROBE1(arc__sync__wait__for__async, 5055 arc_buf_hdr_t *, hdr); 5056 ARCSTAT_BUMP(arcstat_sync_wait_for_async); 5057 } 5058 if (hdr->b_flags & ARC_FLAG_PREDICTIVE_PREFETCH) { 5059 arc_hdr_clear_flags(hdr, 5060 ARC_FLAG_PREDICTIVE_PREFETCH); 5061 } 5062 5063 if (*arc_flags & ARC_FLAG_WAIT) { 5064 cv_wait(&hdr->b_l1hdr.b_cv, hash_lock); 5065 mutex_exit(hash_lock); 5066 goto top; 5067 } 5068 ASSERT(*arc_flags & ARC_FLAG_NOWAIT); 5069 5070 if (done) { 5071 arc_callback_t *acb = NULL; 5072 5073 acb = kmem_zalloc(sizeof (arc_callback_t), 5074 KM_SLEEP); 5075 acb->acb_done = done; 5076 acb->acb_private = private; 5077 acb->acb_compressed = compressed_read; 5078 if (pio != NULL) 5079 acb->acb_zio_dummy = zio_null(pio, 5080 spa, NULL, NULL, NULL, zio_flags); 5081 5082 ASSERT3P(acb->acb_done, !=, NULL); 5083 acb->acb_next = hdr->b_l1hdr.b_acb; 5084 hdr->b_l1hdr.b_acb = acb; 5085 mutex_exit(hash_lock); 5086 return (0); 5087 } 5088 mutex_exit(hash_lock); 5089 return (0); 5090 } 5091 5092 ASSERT(hdr->b_l1hdr.b_state == arc_mru || 5093 hdr->b_l1hdr.b_state == arc_mfu); 5094 5095 if (done) { 5096 if (hdr->b_flags & ARC_FLAG_PREDICTIVE_PREFETCH) { 5097 /* 5098 * This is a demand read which does not have to 5099 * wait for i/o because we did a predictive 5100 * prefetch i/o for it, which has completed. 5101 */ 5102 DTRACE_PROBE1( 5103 arc__demand__hit__predictive__prefetch, 5104 arc_buf_hdr_t *, hdr); 5105 ARCSTAT_BUMP( 5106 arcstat_demand_hit_predictive_prefetch); 5107 arc_hdr_clear_flags(hdr, 5108 ARC_FLAG_PREDICTIVE_PREFETCH); 5109 } 5110 ASSERT(!BP_IS_EMBEDDED(bp) || !BP_IS_HOLE(bp)); 5111 5112 /* Get a buf with the desired data in it. */ 5113 VERIFY0(arc_buf_alloc_impl(hdr, private, 5114 compressed_read, B_TRUE, &buf)); 5115 } else if (*arc_flags & ARC_FLAG_PREFETCH && 5116 zfs_refcount_count(&hdr->b_l1hdr.b_refcnt) == 0) { 5117 arc_hdr_set_flags(hdr, ARC_FLAG_PREFETCH); 5118 } 5119 DTRACE_PROBE1(arc__hit, arc_buf_hdr_t *, hdr); 5120 arc_access(hdr, hash_lock); 5121 if (*arc_flags & ARC_FLAG_L2CACHE) 5122 arc_hdr_set_flags(hdr, ARC_FLAG_L2CACHE); 5123 mutex_exit(hash_lock); 5124 ARCSTAT_BUMP(arcstat_hits); 5125 ARCSTAT_CONDSTAT(!HDR_PREFETCH(hdr), 5126 demand, prefetch, !HDR_ISTYPE_METADATA(hdr), 5127 data, metadata, hits); 5128 5129 if (done) 5130 done(NULL, buf, private); 5131 } else { 5132 uint64_t lsize = BP_GET_LSIZE(bp); 5133 uint64_t psize = BP_GET_PSIZE(bp); 5134 arc_callback_t *acb; 5135 vdev_t *vd = NULL; 5136 uint64_t addr = 0; 5137 boolean_t devw = B_FALSE; 5138 uint64_t size; 5139 5140 if (hdr == NULL) { 5141 /* this block is not in the cache */ 5142 arc_buf_hdr_t *exists = NULL; 5143 arc_buf_contents_t type = BP_GET_BUFC_TYPE(bp); 5144 hdr = arc_hdr_alloc(spa_load_guid(spa), psize, lsize, 5145 BP_GET_COMPRESS(bp), type); 5146 5147 if (!BP_IS_EMBEDDED(bp)) { 5148 hdr->b_dva = *BP_IDENTITY(bp); 5149 hdr->b_birth = BP_PHYSICAL_BIRTH(bp); 5150 exists = buf_hash_insert(hdr, &hash_lock); 5151 } 5152 if (exists != NULL) { 5153 /* somebody beat us to the hash insert */ 5154 mutex_exit(hash_lock); 5155 buf_discard_identity(hdr); 5156 arc_hdr_destroy(hdr); 5157 goto top; /* restart the IO request */ 5158 } 5159 } else { 5160 /* 5161 * This block is in the ghost cache. If it was L2-only 5162 * (and thus didn't have an L1 hdr), we realloc the 5163 * header to add an L1 hdr. 5164 */ 5165 if (!HDR_HAS_L1HDR(hdr)) { 5166 hdr = arc_hdr_realloc(hdr, hdr_l2only_cache, 5167 hdr_full_cache); 5168 } 5169 ASSERT3P(hdr->b_l1hdr.b_pabd, ==, NULL); 5170 ASSERT(GHOST_STATE(hdr->b_l1hdr.b_state)); 5171 ASSERT(!HDR_IO_IN_PROGRESS(hdr)); 5172 ASSERT(zfs_refcount_is_zero(&hdr->b_l1hdr.b_refcnt)); 5173 ASSERT3P(hdr->b_l1hdr.b_buf, ==, NULL); 5174 ASSERT3P(hdr->b_l1hdr.b_freeze_cksum, ==, NULL); 5175 5176 /* 5177 * This is a delicate dance that we play here. 5178 * This hdr is in the ghost list so we access it 5179 * to move it out of the ghost list before we 5180 * initiate the read. If it's a prefetch then 5181 * it won't have a callback so we'll remove the 5182 * reference that arc_buf_alloc_impl() created. We 5183 * do this after we've called arc_access() to 5184 * avoid hitting an assert in remove_reference(). 5185 */ 5186 arc_access(hdr, hash_lock); 5187 arc_hdr_alloc_pabd(hdr); 5188 } 5189 ASSERT3P(hdr->b_l1hdr.b_pabd, !=, NULL); 5190 size = arc_hdr_size(hdr); 5191 5192 /* 5193 * If compression is enabled on the hdr, then will do 5194 * RAW I/O and will store the compressed data in the hdr's 5195 * data block. Otherwise, the hdr's data block will contain 5196 * the uncompressed data. 5197 */ 5198 if (HDR_GET_COMPRESS(hdr) != ZIO_COMPRESS_OFF) { 5199 zio_flags |= ZIO_FLAG_RAW; 5200 } 5201 5202 if (*arc_flags & ARC_FLAG_PREFETCH) 5203 arc_hdr_set_flags(hdr, ARC_FLAG_PREFETCH); 5204 if (*arc_flags & ARC_FLAG_L2CACHE) 5205 arc_hdr_set_flags(hdr, ARC_FLAG_L2CACHE); 5206 if (BP_GET_LEVEL(bp) > 0) 5207 arc_hdr_set_flags(hdr, ARC_FLAG_INDIRECT); 5208 if (*arc_flags & ARC_FLAG_PREDICTIVE_PREFETCH) 5209 arc_hdr_set_flags(hdr, ARC_FLAG_PREDICTIVE_PREFETCH); 5210 ASSERT(!GHOST_STATE(hdr->b_l1hdr.b_state)); 5211 5212 acb = kmem_zalloc(sizeof (arc_callback_t), KM_SLEEP); 5213 acb->acb_done = done; 5214 acb->acb_private = private; 5215 acb->acb_compressed = compressed_read; 5216 5217 ASSERT3P(hdr->b_l1hdr.b_acb, ==, NULL); 5218 hdr->b_l1hdr.b_acb = acb; 5219 arc_hdr_set_flags(hdr, ARC_FLAG_IO_IN_PROGRESS); 5220 5221 if (HDR_HAS_L2HDR(hdr) && 5222 (vd = hdr->b_l2hdr.b_dev->l2ad_vdev) != NULL) { 5223 devw = hdr->b_l2hdr.b_dev->l2ad_writing; 5224 addr = hdr->b_l2hdr.b_daddr; 5225 /* 5226 * Lock out L2ARC device removal. 5227 */ 5228 if (vdev_is_dead(vd) || 5229 !spa_config_tryenter(spa, SCL_L2ARC, vd, RW_READER)) 5230 vd = NULL; 5231 } 5232 5233 if (priority == ZIO_PRIORITY_ASYNC_READ) 5234 arc_hdr_set_flags(hdr, ARC_FLAG_PRIO_ASYNC_READ); 5235 else 5236 arc_hdr_clear_flags(hdr, ARC_FLAG_PRIO_ASYNC_READ); 5237 5238 if (hash_lock != NULL) 5239 mutex_exit(hash_lock); 5240 5241 /* 5242 * At this point, we have a level 1 cache miss. Try again in 5243 * L2ARC if possible. 5244 */ 5245 ASSERT3U(HDR_GET_LSIZE(hdr), ==, lsize); 5246 5247 DTRACE_PROBE4(arc__miss, arc_buf_hdr_t *, hdr, blkptr_t *, bp, 5248 uint64_t, lsize, zbookmark_phys_t *, zb); 5249 ARCSTAT_BUMP(arcstat_misses); 5250 ARCSTAT_CONDSTAT(!HDR_PREFETCH(hdr), 5251 demand, prefetch, !HDR_ISTYPE_METADATA(hdr), 5252 data, metadata, misses); 5253 5254 if (vd != NULL && l2arc_ndev != 0 && !(l2arc_norw && devw)) { 5255 /* 5256 * Read from the L2ARC if the following are true: 5257 * 1. The L2ARC vdev was previously cached. 5258 * 2. This buffer still has L2ARC metadata. 5259 * 3. This buffer isn't currently writing to the L2ARC. 5260 * 4. The L2ARC entry wasn't evicted, which may 5261 * also have invalidated the vdev. 5262 * 5. This isn't prefetch and l2arc_noprefetch is set. 5263 */ 5264 if (HDR_HAS_L2HDR(hdr) && 5265 !HDR_L2_WRITING(hdr) && !HDR_L2_EVICTED(hdr) && 5266 !(l2arc_noprefetch && HDR_PREFETCH(hdr))) { 5267 l2arc_read_callback_t *cb; 5268 abd_t *abd; 5269 uint64_t asize; 5270 5271 DTRACE_PROBE1(l2arc__hit, arc_buf_hdr_t *, hdr); 5272 ARCSTAT_BUMP(arcstat_l2_hits); 5273 5274 cb = kmem_zalloc(sizeof (l2arc_read_callback_t), 5275 KM_SLEEP); 5276 cb->l2rcb_hdr = hdr; 5277 cb->l2rcb_bp = *bp; 5278 cb->l2rcb_zb = *zb; 5279 cb->l2rcb_flags = zio_flags; 5280 5281 asize = vdev_psize_to_asize(vd, size); 5282 if (asize != size) { 5283 abd = abd_alloc_for_io(asize, 5284 HDR_ISTYPE_METADATA(hdr)); 5285 cb->l2rcb_abd = abd; 5286 } else { 5287 abd = hdr->b_l1hdr.b_pabd; 5288 } 5289 5290 ASSERT(addr >= VDEV_LABEL_START_SIZE && 5291 addr + asize <= vd->vdev_psize - 5292 VDEV_LABEL_END_SIZE); 5293 5294 /* 5295 * l2arc read. The SCL_L2ARC lock will be 5296 * released by l2arc_read_done(). 5297 * Issue a null zio if the underlying buffer 5298 * was squashed to zero size by compression. 5299 */ 5300 ASSERT3U(HDR_GET_COMPRESS(hdr), !=, 5301 ZIO_COMPRESS_EMPTY); 5302 rzio = zio_read_phys(pio, vd, addr, 5303 asize, abd, 5304 ZIO_CHECKSUM_OFF, 5305 l2arc_read_done, cb, priority, 5306 zio_flags | ZIO_FLAG_DONT_CACHE | 5307 ZIO_FLAG_CANFAIL | 5308 ZIO_FLAG_DONT_PROPAGATE | 5309 ZIO_FLAG_DONT_RETRY, B_FALSE); 5310 DTRACE_PROBE2(l2arc__read, vdev_t *, vd, 5311 zio_t *, rzio); 5312 ARCSTAT_INCR(arcstat_l2_read_bytes, size); 5313 5314 if (*arc_flags & ARC_FLAG_NOWAIT) { 5315 zio_nowait(rzio); 5316 return (0); 5317 } 5318 5319 ASSERT(*arc_flags & ARC_FLAG_WAIT); 5320 if (zio_wait(rzio) == 0) 5321 return (0); 5322 5323 /* l2arc read error; goto zio_read() */ 5324 } else { 5325 DTRACE_PROBE1(l2arc__miss, 5326 arc_buf_hdr_t *, hdr); 5327 ARCSTAT_BUMP(arcstat_l2_misses); 5328 if (HDR_L2_WRITING(hdr)) 5329 ARCSTAT_BUMP(arcstat_l2_rw_clash); 5330 spa_config_exit(spa, SCL_L2ARC, vd); 5331 } 5332 } else { 5333 if (vd != NULL) 5334 spa_config_exit(spa, SCL_L2ARC, vd); 5335 if (l2arc_ndev != 0) { 5336 DTRACE_PROBE1(l2arc__miss, 5337 arc_buf_hdr_t *, hdr); 5338 ARCSTAT_BUMP(arcstat_l2_misses); 5339 } 5340 } 5341 5342 rzio = zio_read(pio, spa, bp, hdr->b_l1hdr.b_pabd, size, 5343 arc_read_done, hdr, priority, zio_flags, zb); 5344 5345 if (*arc_flags & ARC_FLAG_WAIT) 5346 return (zio_wait(rzio)); 5347 5348 ASSERT(*arc_flags & ARC_FLAG_NOWAIT); 5349 zio_nowait(rzio); 5350 } 5351 return (0); 5352 } 5353 5354 /* 5355 * Notify the arc that a block was freed, and thus will never be used again. 5356 */ 5357 void 5358 arc_freed(spa_t *spa, const blkptr_t *bp) 5359 { 5360 arc_buf_hdr_t *hdr; 5361 kmutex_t *hash_lock; 5362 uint64_t guid = spa_load_guid(spa); 5363 5364 ASSERT(!BP_IS_EMBEDDED(bp)); 5365 5366 hdr = buf_hash_find(guid, bp, &hash_lock); 5367 if (hdr == NULL) 5368 return; 5369 5370 /* 5371 * We might be trying to free a block that is still doing I/O 5372 * (i.e. prefetch) or has a reference (i.e. a dedup-ed, 5373 * dmu_sync-ed block). If this block is being prefetched, then it 5374 * would still have the ARC_FLAG_IO_IN_PROGRESS flag set on the hdr 5375 * until the I/O completes. A block may also have a reference if it is 5376 * part of a dedup-ed, dmu_synced write. The dmu_sync() function would 5377 * have written the new block to its final resting place on disk but 5378 * without the dedup flag set. This would have left the hdr in the MRU 5379 * state and discoverable. When the txg finally syncs it detects that 5380 * the block was overridden in open context and issues an override I/O. 5381 * Since this is a dedup block, the override I/O will determine if the 5382 * block is already in the DDT. If so, then it will replace the io_bp 5383 * with the bp from the DDT and allow the I/O to finish. When the I/O 5384 * reaches the done callback, dbuf_write_override_done, it will 5385 * check to see if the io_bp and io_bp_override are identical. 5386 * If they are not, then it indicates that the bp was replaced with 5387 * the bp in the DDT and the override bp is freed. This allows 5388 * us to arrive here with a reference on a block that is being 5389 * freed. So if we have an I/O in progress, or a reference to 5390 * this hdr, then we don't destroy the hdr. 5391 */ 5392 if (!HDR_HAS_L1HDR(hdr) || (!HDR_IO_IN_PROGRESS(hdr) && 5393 zfs_refcount_is_zero(&hdr->b_l1hdr.b_refcnt))) { 5394 arc_change_state(arc_anon, hdr, hash_lock); 5395 arc_hdr_destroy(hdr); 5396 mutex_exit(hash_lock); 5397 } else { 5398 mutex_exit(hash_lock); 5399 } 5400 5401 } 5402 5403 /* 5404 * Release this buffer from the cache, making it an anonymous buffer. This 5405 * must be done after a read and prior to modifying the buffer contents. 5406 * If the buffer has more than one reference, we must make 5407 * a new hdr for the buffer. 5408 */ 5409 void 5410 arc_release(arc_buf_t *buf, void *tag) 5411 { 5412 arc_buf_hdr_t *hdr = buf->b_hdr; 5413 5414 /* 5415 * It would be nice to assert that if it's DMU metadata (level > 5416 * 0 || it's the dnode file), then it must be syncing context. 5417 * But we don't know that information at this level. 5418 */ 5419 5420 mutex_enter(&buf->b_evict_lock); 5421 5422 ASSERT(HDR_HAS_L1HDR(hdr)); 5423 5424 /* 5425 * We don't grab the hash lock prior to this check, because if 5426 * the buffer's header is in the arc_anon state, it won't be 5427 * linked into the hash table. 5428 */ 5429 if (hdr->b_l1hdr.b_state == arc_anon) { 5430 mutex_exit(&buf->b_evict_lock); 5431 ASSERT(!HDR_IO_IN_PROGRESS(hdr)); 5432 ASSERT(!HDR_IN_HASH_TABLE(hdr)); 5433 ASSERT(!HDR_HAS_L2HDR(hdr)); 5434 ASSERT(HDR_EMPTY(hdr)); 5435 5436 ASSERT3U(hdr->b_l1hdr.b_bufcnt, ==, 1); 5437 ASSERT3S(zfs_refcount_count(&hdr->b_l1hdr.b_refcnt), ==, 1); 5438 ASSERT(!list_link_active(&hdr->b_l1hdr.b_arc_node)); 5439 5440 hdr->b_l1hdr.b_arc_access = 0; 5441 5442 /* 5443 * If the buf is being overridden then it may already 5444 * have a hdr that is not empty. 5445 */ 5446 buf_discard_identity(hdr); 5447 arc_buf_thaw(buf); 5448 5449 return; 5450 } 5451 5452 kmutex_t *hash_lock = HDR_LOCK(hdr); 5453 mutex_enter(hash_lock); 5454 5455 /* 5456 * This assignment is only valid as long as the hash_lock is 5457 * held, we must be careful not to reference state or the 5458 * b_state field after dropping the lock. 5459 */ 5460 arc_state_t *state = hdr->b_l1hdr.b_state; 5461 ASSERT3P(hash_lock, ==, HDR_LOCK(hdr)); 5462 ASSERT3P(state, !=, arc_anon); 5463 5464 /* this buffer is not on any list */ 5465 ASSERT3S(zfs_refcount_count(&hdr->b_l1hdr.b_refcnt), >, 0); 5466 5467 if (HDR_HAS_L2HDR(hdr)) { 5468 mutex_enter(&hdr->b_l2hdr.b_dev->l2ad_mtx); 5469 5470 /* 5471 * We have to recheck this conditional again now that 5472 * we're holding the l2ad_mtx to prevent a race with 5473 * another thread which might be concurrently calling 5474 * l2arc_evict(). In that case, l2arc_evict() might have 5475 * destroyed the header's L2 portion as we were waiting 5476 * to acquire the l2ad_mtx. 5477 */ 5478 if (HDR_HAS_L2HDR(hdr)) 5479 arc_hdr_l2hdr_destroy(hdr); 5480 5481 mutex_exit(&hdr->b_l2hdr.b_dev->l2ad_mtx); 5482 } 5483 5484 /* 5485 * Do we have more than one buf? 5486 */ 5487 if (hdr->b_l1hdr.b_bufcnt > 1) { 5488 arc_buf_hdr_t *nhdr; 5489 uint64_t spa = hdr->b_spa; 5490 uint64_t psize = HDR_GET_PSIZE(hdr); 5491 uint64_t lsize = HDR_GET_LSIZE(hdr); 5492 enum zio_compress compress = HDR_GET_COMPRESS(hdr); 5493 arc_buf_contents_t type = arc_buf_type(hdr); 5494 VERIFY3U(hdr->b_type, ==, type); 5495 5496 ASSERT(hdr->b_l1hdr.b_buf != buf || buf->b_next != NULL); 5497 (void) remove_reference(hdr, hash_lock, tag); 5498 5499 if (arc_buf_is_shared(buf) && !ARC_BUF_COMPRESSED(buf)) { 5500 ASSERT3P(hdr->b_l1hdr.b_buf, !=, buf); 5501 ASSERT(ARC_BUF_LAST(buf)); 5502 } 5503 5504 /* 5505 * Pull the data off of this hdr and attach it to 5506 * a new anonymous hdr. Also find the last buffer 5507 * in the hdr's buffer list. 5508 */ 5509 arc_buf_t *lastbuf = arc_buf_remove(hdr, buf); 5510 ASSERT3P(lastbuf, !=, NULL); 5511 5512 /* 5513 * If the current arc_buf_t and the hdr are sharing their data 5514 * buffer, then we must stop sharing that block. 5515 */ 5516 if (arc_buf_is_shared(buf)) { 5517 VERIFY(!arc_buf_is_shared(lastbuf)); 5518 5519 /* 5520 * First, sever the block sharing relationship between 5521 * buf and the arc_buf_hdr_t. 5522 */ 5523 arc_unshare_buf(hdr, buf); 5524 5525 /* 5526 * Now we need to recreate the hdr's b_pabd. Since we 5527 * have lastbuf handy, we try to share with it, but if 5528 * we can't then we allocate a new b_pabd and copy the 5529 * data from buf into it. 5530 */ 5531 if (arc_can_share(hdr, lastbuf)) { 5532 arc_share_buf(hdr, lastbuf); 5533 } else { 5534 arc_hdr_alloc_pabd(hdr); 5535 abd_copy_from_buf(hdr->b_l1hdr.b_pabd, 5536 buf->b_data, psize); 5537 } 5538 VERIFY3P(lastbuf->b_data, !=, NULL); 5539 } else if (HDR_SHARED_DATA(hdr)) { 5540 /* 5541 * Uncompressed shared buffers are always at the end 5542 * of the list. Compressed buffers don't have the 5543 * same requirements. This makes it hard to 5544 * simply assert that the lastbuf is shared so 5545 * we rely on the hdr's compression flags to determine 5546 * if we have a compressed, shared buffer. 5547 */ 5548 ASSERT(arc_buf_is_shared(lastbuf) || 5549 HDR_GET_COMPRESS(hdr) != ZIO_COMPRESS_OFF); 5550 ASSERT(!ARC_BUF_SHARED(buf)); 5551 } 5552 ASSERT3P(hdr->b_l1hdr.b_pabd, !=, NULL); 5553 ASSERT3P(state, !=, arc_l2c_only); 5554 5555 (void) zfs_refcount_remove_many(&state->arcs_size, 5556 arc_buf_size(buf), buf); 5557 5558 if (zfs_refcount_is_zero(&hdr->b_l1hdr.b_refcnt)) { 5559 ASSERT3P(state, !=, arc_l2c_only); 5560 (void) zfs_refcount_remove_many( 5561 &state->arcs_esize[type], 5562 arc_buf_size(buf), buf); 5563 } 5564 5565 hdr->b_l1hdr.b_bufcnt -= 1; 5566 arc_cksum_verify(buf); 5567 arc_buf_unwatch(buf); 5568 5569 mutex_exit(hash_lock); 5570 5571 /* 5572 * Allocate a new hdr. The new hdr will contain a b_pabd 5573 * buffer which will be freed in arc_write(). 5574 */ 5575 nhdr = arc_hdr_alloc(spa, psize, lsize, compress, type); 5576 ASSERT3P(nhdr->b_l1hdr.b_buf, ==, NULL); 5577 ASSERT0(nhdr->b_l1hdr.b_bufcnt); 5578 ASSERT0(zfs_refcount_count(&nhdr->b_l1hdr.b_refcnt)); 5579 VERIFY3U(nhdr->b_type, ==, type); 5580 ASSERT(!HDR_SHARED_DATA(nhdr)); 5581 5582 nhdr->b_l1hdr.b_buf = buf; 5583 nhdr->b_l1hdr.b_bufcnt = 1; 5584 (void) zfs_refcount_add(&nhdr->b_l1hdr.b_refcnt, tag); 5585 buf->b_hdr = nhdr; 5586 5587 mutex_exit(&buf->b_evict_lock); 5588 (void) zfs_refcount_add_many(&arc_anon->arcs_size, 5589 arc_buf_size(buf), buf); 5590 } else { 5591 mutex_exit(&buf->b_evict_lock); 5592 ASSERT(zfs_refcount_count(&hdr->b_l1hdr.b_refcnt) == 1); 5593 /* protected by hash lock, or hdr is on arc_anon */ 5594 ASSERT(!multilist_link_active(&hdr->b_l1hdr.b_arc_node)); 5595 ASSERT(!HDR_IO_IN_PROGRESS(hdr)); 5596 arc_change_state(arc_anon, hdr, hash_lock); 5597 hdr->b_l1hdr.b_arc_access = 0; 5598 mutex_exit(hash_lock); 5599 5600 buf_discard_identity(hdr); 5601 arc_buf_thaw(buf); 5602 } 5603 } 5604 5605 int 5606 arc_released(arc_buf_t *buf) 5607 { 5608 int released; 5609 5610 mutex_enter(&buf->b_evict_lock); 5611 released = (buf->b_data != NULL && 5612 buf->b_hdr->b_l1hdr.b_state == arc_anon); 5613 mutex_exit(&buf->b_evict_lock); 5614 return (released); 5615 } 5616 5617 #ifdef ZFS_DEBUG 5618 int 5619 arc_referenced(arc_buf_t *buf) 5620 { 5621 int referenced; 5622 5623 mutex_enter(&buf->b_evict_lock); 5624 referenced = (zfs_refcount_count(&buf->b_hdr->b_l1hdr.b_refcnt)); 5625 mutex_exit(&buf->b_evict_lock); 5626 return (referenced); 5627 } 5628 #endif 5629 5630 static void 5631 arc_write_ready(zio_t *zio) 5632 { 5633 arc_write_callback_t *callback = zio->io_private; 5634 arc_buf_t *buf = callback->awcb_buf; 5635 arc_buf_hdr_t *hdr = buf->b_hdr; 5636 uint64_t psize = BP_IS_HOLE(zio->io_bp) ? 0 : BP_GET_PSIZE(zio->io_bp); 5637 5638 ASSERT(HDR_HAS_L1HDR(hdr)); 5639 ASSERT(!zfs_refcount_is_zero(&buf->b_hdr->b_l1hdr.b_refcnt)); 5640 ASSERT(hdr->b_l1hdr.b_bufcnt > 0); 5641 5642 /* 5643 * If we're reexecuting this zio because the pool suspended, then 5644 * cleanup any state that was previously set the first time the 5645 * callback was invoked. 5646 */ 5647 if (zio->io_flags & ZIO_FLAG_REEXECUTED) { 5648 arc_cksum_free(hdr); 5649 arc_buf_unwatch(buf); 5650 if (hdr->b_l1hdr.b_pabd != NULL) { 5651 if (arc_buf_is_shared(buf)) { 5652 arc_unshare_buf(hdr, buf); 5653 } else { 5654 arc_hdr_free_pabd(hdr); 5655 } 5656 } 5657 } 5658 ASSERT3P(hdr->b_l1hdr.b_pabd, ==, NULL); 5659 ASSERT(!HDR_SHARED_DATA(hdr)); 5660 ASSERT(!arc_buf_is_shared(buf)); 5661 5662 callback->awcb_ready(zio, buf, callback->awcb_private); 5663 5664 if (HDR_IO_IN_PROGRESS(hdr)) 5665 ASSERT(zio->io_flags & ZIO_FLAG_REEXECUTED); 5666 5667 arc_cksum_compute(buf); 5668 arc_hdr_set_flags(hdr, ARC_FLAG_IO_IN_PROGRESS); 5669 5670 enum zio_compress compress; 5671 if (BP_IS_HOLE(zio->io_bp) || BP_IS_EMBEDDED(zio->io_bp)) { 5672 compress = ZIO_COMPRESS_OFF; 5673 } else { 5674 ASSERT3U(HDR_GET_LSIZE(hdr), ==, BP_GET_LSIZE(zio->io_bp)); 5675 compress = BP_GET_COMPRESS(zio->io_bp); 5676 } 5677 HDR_SET_PSIZE(hdr, psize); 5678 arc_hdr_set_compress(hdr, compress); 5679 5680 5681 /* 5682 * Fill the hdr with data. If the hdr is compressed, the data we want 5683 * is available from the zio, otherwise we can take it from the buf. 5684 * 5685 * We might be able to share the buf's data with the hdr here. However, 5686 * doing so would cause the ARC to be full of linear ABDs if we write a 5687 * lot of shareable data. As a compromise, we check whether scattered 5688 * ABDs are allowed, and assume that if they are then the user wants 5689 * the ARC to be primarily filled with them regardless of the data being 5690 * written. Therefore, if they're allowed then we allocate one and copy 5691 * the data into it; otherwise, we share the data directly if we can. 5692 */ 5693 if (zfs_abd_scatter_enabled || !arc_can_share(hdr, buf)) { 5694 arc_hdr_alloc_pabd(hdr); 5695 5696 /* 5697 * Ideally, we would always copy the io_abd into b_pabd, but the 5698 * user may have disabled compressed ARC, thus we must check the 5699 * hdr's compression setting rather than the io_bp's. 5700 */ 5701 if (HDR_GET_COMPRESS(hdr) != ZIO_COMPRESS_OFF) { 5702 ASSERT3U(BP_GET_COMPRESS(zio->io_bp), !=, 5703 ZIO_COMPRESS_OFF); 5704 ASSERT3U(psize, >, 0); 5705 5706 abd_copy(hdr->b_l1hdr.b_pabd, zio->io_abd, psize); 5707 } else { 5708 ASSERT3U(zio->io_orig_size, ==, arc_hdr_size(hdr)); 5709 5710 abd_copy_from_buf(hdr->b_l1hdr.b_pabd, buf->b_data, 5711 arc_buf_size(buf)); 5712 } 5713 } else { 5714 ASSERT3P(buf->b_data, ==, abd_to_buf(zio->io_orig_abd)); 5715 ASSERT3U(zio->io_orig_size, ==, arc_buf_size(buf)); 5716 ASSERT3U(hdr->b_l1hdr.b_bufcnt, ==, 1); 5717 5718 arc_share_buf(hdr, buf); 5719 } 5720 5721 arc_hdr_verify(hdr, zio->io_bp); 5722 } 5723 5724 static void 5725 arc_write_children_ready(zio_t *zio) 5726 { 5727 arc_write_callback_t *callback = zio->io_private; 5728 arc_buf_t *buf = callback->awcb_buf; 5729 5730 callback->awcb_children_ready(zio, buf, callback->awcb_private); 5731 } 5732 5733 /* 5734 * The SPA calls this callback for each physical write that happens on behalf 5735 * of a logical write. See the comment in dbuf_write_physdone() for details. 5736 */ 5737 static void 5738 arc_write_physdone(zio_t *zio) 5739 { 5740 arc_write_callback_t *cb = zio->io_private; 5741 if (cb->awcb_physdone != NULL) 5742 cb->awcb_physdone(zio, cb->awcb_buf, cb->awcb_private); 5743 } 5744 5745 static void 5746 arc_write_done(zio_t *zio) 5747 { 5748 arc_write_callback_t *callback = zio->io_private; 5749 arc_buf_t *buf = callback->awcb_buf; 5750 arc_buf_hdr_t *hdr = buf->b_hdr; 5751 5752 ASSERT3P(hdr->b_l1hdr.b_acb, ==, NULL); 5753 5754 if (zio->io_error == 0) { 5755 arc_hdr_verify(hdr, zio->io_bp); 5756 5757 if (BP_IS_HOLE(zio->io_bp) || BP_IS_EMBEDDED(zio->io_bp)) { 5758 buf_discard_identity(hdr); 5759 } else { 5760 hdr->b_dva = *BP_IDENTITY(zio->io_bp); 5761 hdr->b_birth = BP_PHYSICAL_BIRTH(zio->io_bp); 5762 } 5763 } else { 5764 ASSERT(HDR_EMPTY(hdr)); 5765 } 5766 5767 /* 5768 * If the block to be written was all-zero or compressed enough to be 5769 * embedded in the BP, no write was performed so there will be no 5770 * dva/birth/checksum. The buffer must therefore remain anonymous 5771 * (and uncached). 5772 */ 5773 if (!HDR_EMPTY(hdr)) { 5774 arc_buf_hdr_t *exists; 5775 kmutex_t *hash_lock; 5776 5777 ASSERT3U(zio->io_error, ==, 0); 5778 5779 arc_cksum_verify(buf); 5780 5781 exists = buf_hash_insert(hdr, &hash_lock); 5782 if (exists != NULL) { 5783 /* 5784 * This can only happen if we overwrite for 5785 * sync-to-convergence, because we remove 5786 * buffers from the hash table when we arc_free(). 5787 */ 5788 if (zio->io_flags & ZIO_FLAG_IO_REWRITE) { 5789 if (!BP_EQUAL(&zio->io_bp_orig, zio->io_bp)) 5790 panic("bad overwrite, hdr=%p exists=%p", 5791 (void *)hdr, (void *)exists); 5792 ASSERT(zfs_refcount_is_zero( 5793 &exists->b_l1hdr.b_refcnt)); 5794 arc_change_state(arc_anon, exists, hash_lock); 5795 mutex_exit(hash_lock); 5796 arc_hdr_destroy(exists); 5797 exists = buf_hash_insert(hdr, &hash_lock); 5798 ASSERT3P(exists, ==, NULL); 5799 } else if (zio->io_flags & ZIO_FLAG_NOPWRITE) { 5800 /* nopwrite */ 5801 ASSERT(zio->io_prop.zp_nopwrite); 5802 if (!BP_EQUAL(&zio->io_bp_orig, zio->io_bp)) 5803 panic("bad nopwrite, hdr=%p exists=%p", 5804 (void *)hdr, (void *)exists); 5805 } else { 5806 /* Dedup */ 5807 ASSERT(hdr->b_l1hdr.b_bufcnt == 1); 5808 ASSERT(hdr->b_l1hdr.b_state == arc_anon); 5809 ASSERT(BP_GET_DEDUP(zio->io_bp)); 5810 ASSERT(BP_GET_LEVEL(zio->io_bp) == 0); 5811 } 5812 } 5813 arc_hdr_clear_flags(hdr, ARC_FLAG_IO_IN_PROGRESS); 5814 /* if it's not anon, we are doing a scrub */ 5815 if (exists == NULL && hdr->b_l1hdr.b_state == arc_anon) 5816 arc_access(hdr, hash_lock); 5817 mutex_exit(hash_lock); 5818 } else { 5819 arc_hdr_clear_flags(hdr, ARC_FLAG_IO_IN_PROGRESS); 5820 } 5821 5822 ASSERT(!zfs_refcount_is_zero(&hdr->b_l1hdr.b_refcnt)); 5823 callback->awcb_done(zio, buf, callback->awcb_private); 5824 5825 abd_put(zio->io_abd); 5826 kmem_free(callback, sizeof (arc_write_callback_t)); 5827 } 5828 5829 zio_t * 5830 arc_write(zio_t *pio, spa_t *spa, uint64_t txg, blkptr_t *bp, arc_buf_t *buf, 5831 boolean_t l2arc, const zio_prop_t *zp, arc_done_func_t *ready, 5832 arc_done_func_t *children_ready, arc_done_func_t *physdone, 5833 arc_done_func_t *done, void *private, zio_priority_t priority, 5834 int zio_flags, const zbookmark_phys_t *zb) 5835 { 5836 arc_buf_hdr_t *hdr = buf->b_hdr; 5837 arc_write_callback_t *callback; 5838 zio_t *zio; 5839 zio_prop_t localprop = *zp; 5840 5841 ASSERT3P(ready, !=, NULL); 5842 ASSERT3P(done, !=, NULL); 5843 ASSERT(!HDR_IO_ERROR(hdr)); 5844 ASSERT(!HDR_IO_IN_PROGRESS(hdr)); 5845 ASSERT3P(hdr->b_l1hdr.b_acb, ==, NULL); 5846 ASSERT3U(hdr->b_l1hdr.b_bufcnt, >, 0); 5847 if (l2arc) 5848 arc_hdr_set_flags(hdr, ARC_FLAG_L2CACHE); 5849 if (ARC_BUF_COMPRESSED(buf)) { 5850 /* 5851 * We're writing a pre-compressed buffer. Make the 5852 * compression algorithm requested by the zio_prop_t match 5853 * the pre-compressed buffer's compression algorithm. 5854 */ 5855 localprop.zp_compress = HDR_GET_COMPRESS(hdr); 5856 5857 ASSERT3U(HDR_GET_LSIZE(hdr), !=, arc_buf_size(buf)); 5858 zio_flags |= ZIO_FLAG_RAW; 5859 } 5860 callback = kmem_zalloc(sizeof (arc_write_callback_t), KM_SLEEP); 5861 callback->awcb_ready = ready; 5862 callback->awcb_children_ready = children_ready; 5863 callback->awcb_physdone = physdone; 5864 callback->awcb_done = done; 5865 callback->awcb_private = private; 5866 callback->awcb_buf = buf; 5867 5868 /* 5869 * The hdr's b_pabd is now stale, free it now. A new data block 5870 * will be allocated when the zio pipeline calls arc_write_ready(). 5871 */ 5872 if (hdr->b_l1hdr.b_pabd != NULL) { 5873 /* 5874 * If the buf is currently sharing the data block with 5875 * the hdr then we need to break that relationship here. 5876 * The hdr will remain with a NULL data pointer and the 5877 * buf will take sole ownership of the block. 5878 */ 5879 if (arc_buf_is_shared(buf)) { 5880 arc_unshare_buf(hdr, buf); 5881 } else { 5882 arc_hdr_free_pabd(hdr); 5883 } 5884 VERIFY3P(buf->b_data, !=, NULL); 5885 arc_hdr_set_compress(hdr, ZIO_COMPRESS_OFF); 5886 } 5887 ASSERT(!arc_buf_is_shared(buf)); 5888 ASSERT3P(hdr->b_l1hdr.b_pabd, ==, NULL); 5889 5890 zio = zio_write(pio, spa, txg, bp, 5891 abd_get_from_buf(buf->b_data, HDR_GET_LSIZE(hdr)), 5892 HDR_GET_LSIZE(hdr), arc_buf_size(buf), &localprop, arc_write_ready, 5893 (children_ready != NULL) ? arc_write_children_ready : NULL, 5894 arc_write_physdone, arc_write_done, callback, 5895 priority, zio_flags, zb); 5896 5897 return (zio); 5898 } 5899 5900 static int 5901 arc_memory_throttle(spa_t *spa, uint64_t reserve, uint64_t txg) 5902 { 5903 #ifdef _KERNEL 5904 uint64_t available_memory = ptob(freemem); 5905 5906 #if defined(__i386) 5907 available_memory = 5908 MIN(available_memory, vmem_size(heap_arena, VMEM_FREE)); 5909 #endif 5910 5911 if (freemem > physmem * arc_lotsfree_percent / 100) 5912 return (0); 5913 5914 if (txg > spa->spa_lowmem_last_txg) { 5915 spa->spa_lowmem_last_txg = txg; 5916 spa->spa_lowmem_page_load = 0; 5917 } 5918 /* 5919 * If we are in pageout, we know that memory is already tight, 5920 * the arc is already going to be evicting, so we just want to 5921 * continue to let page writes occur as quickly as possible. 5922 */ 5923 if (curproc == proc_pageout) { 5924 if (spa->spa_lowmem_page_load > 5925 MAX(ptob(minfree), available_memory) / 4) 5926 return (SET_ERROR(ERESTART)); 5927 /* Note: reserve is inflated, so we deflate */ 5928 atomic_add_64(&spa->spa_lowmem_page_load, reserve / 8); 5929 return (0); 5930 } else if (spa->spa_lowmem_page_load > 0 && arc_reclaim_needed()) { 5931 /* memory is low, delay before restarting */ 5932 ARCSTAT_INCR(arcstat_memory_throttle_count, 1); 5933 return (SET_ERROR(EAGAIN)); 5934 } 5935 spa->spa_lowmem_page_load = 0; 5936 #endif /* _KERNEL */ 5937 return (0); 5938 } 5939 5940 void 5941 arc_tempreserve_clear(uint64_t reserve) 5942 { 5943 atomic_add_64(&arc_tempreserve, -reserve); 5944 ASSERT((int64_t)arc_tempreserve >= 0); 5945 } 5946 5947 int 5948 arc_tempreserve_space(spa_t *spa, uint64_t reserve, uint64_t txg) 5949 { 5950 int error; 5951 uint64_t anon_size; 5952 5953 if (reserve > arc_c/4 && !arc_no_grow) 5954 arc_c = MIN(arc_c_max, reserve * 4); 5955 if (reserve > arc_c) 5956 return (SET_ERROR(ENOMEM)); 5957 5958 /* 5959 * Don't count loaned bufs as in flight dirty data to prevent long 5960 * network delays from blocking transactions that are ready to be 5961 * assigned to a txg. 5962 */ 5963 5964 /* assert that it has not wrapped around */ 5965 ASSERT3S(atomic_add_64_nv(&arc_loaned_bytes, 0), >=, 0); 5966 5967 anon_size = MAX((int64_t)(zfs_refcount_count(&arc_anon->arcs_size) - 5968 arc_loaned_bytes), 0); 5969 5970 /* 5971 * Writes will, almost always, require additional memory allocations 5972 * in order to compress/encrypt/etc the data. We therefore need to 5973 * make sure that there is sufficient available memory for this. 5974 */ 5975 error = arc_memory_throttle(spa, reserve, txg); 5976 if (error != 0) 5977 return (error); 5978 5979 /* 5980 * Throttle writes when the amount of dirty data in the cache 5981 * gets too large. We try to keep the cache less than half full 5982 * of dirty blocks so that our sync times don't grow too large. 5983 * 5984 * In the case of one pool being built on another pool, we want 5985 * to make sure we don't end up throttling the lower (backing) 5986 * pool when the upper pool is the majority contributor to dirty 5987 * data. To insure we make forward progress during throttling, we 5988 * also check the current pool's net dirty data and only throttle 5989 * if it exceeds zfs_arc_pool_dirty_percent of the anonymous dirty 5990 * data in the cache. 5991 * 5992 * Note: if two requests come in concurrently, we might let them 5993 * both succeed, when one of them should fail. Not a huge deal. 5994 */ 5995 uint64_t total_dirty = reserve + arc_tempreserve + anon_size; 5996 uint64_t spa_dirty_anon = spa_dirty_data(spa); 5997 5998 if (total_dirty > arc_c * zfs_arc_dirty_limit_percent / 100 && 5999 anon_size > arc_c * zfs_arc_anon_limit_percent / 100 && 6000 spa_dirty_anon > anon_size * zfs_arc_pool_dirty_percent / 100) { 6001 uint64_t meta_esize = 6002 zfs_refcount_count( 6003 &arc_anon->arcs_esize[ARC_BUFC_METADATA]); 6004 uint64_t data_esize = 6005 zfs_refcount_count(&arc_anon->arcs_esize[ARC_BUFC_DATA]); 6006 dprintf("failing, arc_tempreserve=%lluK anon_meta=%lluK " 6007 "anon_data=%lluK tempreserve=%lluK arc_c=%lluK\n", 6008 arc_tempreserve >> 10, meta_esize >> 10, 6009 data_esize >> 10, reserve >> 10, arc_c >> 10); 6010 return (SET_ERROR(ERESTART)); 6011 } 6012 atomic_add_64(&arc_tempreserve, reserve); 6013 return (0); 6014 } 6015 6016 static void 6017 arc_kstat_update_state(arc_state_t *state, kstat_named_t *size, 6018 kstat_named_t *evict_data, kstat_named_t *evict_metadata) 6019 { 6020 size->value.ui64 = zfs_refcount_count(&state->arcs_size); 6021 evict_data->value.ui64 = 6022 zfs_refcount_count(&state->arcs_esize[ARC_BUFC_DATA]); 6023 evict_metadata->value.ui64 = 6024 zfs_refcount_count(&state->arcs_esize[ARC_BUFC_METADATA]); 6025 } 6026 6027 static int 6028 arc_kstat_update(kstat_t *ksp, int rw) 6029 { 6030 arc_stats_t *as = ksp->ks_data; 6031 6032 if (rw == KSTAT_WRITE) { 6033 return (EACCES); 6034 } else { 6035 arc_kstat_update_state(arc_anon, 6036 &as->arcstat_anon_size, 6037 &as->arcstat_anon_evictable_data, 6038 &as->arcstat_anon_evictable_metadata); 6039 arc_kstat_update_state(arc_mru, 6040 &as->arcstat_mru_size, 6041 &as->arcstat_mru_evictable_data, 6042 &as->arcstat_mru_evictable_metadata); 6043 arc_kstat_update_state(arc_mru_ghost, 6044 &as->arcstat_mru_ghost_size, 6045 &as->arcstat_mru_ghost_evictable_data, 6046 &as->arcstat_mru_ghost_evictable_metadata); 6047 arc_kstat_update_state(arc_mfu, 6048 &as->arcstat_mfu_size, 6049 &as->arcstat_mfu_evictable_data, 6050 &as->arcstat_mfu_evictable_metadata); 6051 arc_kstat_update_state(arc_mfu_ghost, 6052 &as->arcstat_mfu_ghost_size, 6053 &as->arcstat_mfu_ghost_evictable_data, 6054 &as->arcstat_mfu_ghost_evictable_metadata); 6055 6056 ARCSTAT(arcstat_size) = aggsum_value(&arc_size); 6057 ARCSTAT(arcstat_meta_used) = aggsum_value(&arc_meta_used); 6058 ARCSTAT(arcstat_data_size) = aggsum_value(&astat_data_size); 6059 ARCSTAT(arcstat_metadata_size) = 6060 aggsum_value(&astat_metadata_size); 6061 ARCSTAT(arcstat_hdr_size) = aggsum_value(&astat_hdr_size); 6062 ARCSTAT(arcstat_other_size) = aggsum_value(&astat_other_size); 6063 ARCSTAT(arcstat_l2_hdr_size) = aggsum_value(&astat_l2_hdr_size); 6064 } 6065 6066 return (0); 6067 } 6068 6069 /* 6070 * This function *must* return indices evenly distributed between all 6071 * sublists of the multilist. This is needed due to how the ARC eviction 6072 * code is laid out; arc_evict_state() assumes ARC buffers are evenly 6073 * distributed between all sublists and uses this assumption when 6074 * deciding which sublist to evict from and how much to evict from it. 6075 */ 6076 unsigned int 6077 arc_state_multilist_index_func(multilist_t *ml, void *obj) 6078 { 6079 arc_buf_hdr_t *hdr = obj; 6080 6081 /* 6082 * We rely on b_dva to generate evenly distributed index 6083 * numbers using buf_hash below. So, as an added precaution, 6084 * let's make sure we never add empty buffers to the arc lists. 6085 */ 6086 ASSERT(!HDR_EMPTY(hdr)); 6087 6088 /* 6089 * The assumption here, is the hash value for a given 6090 * arc_buf_hdr_t will remain constant throughout it's lifetime 6091 * (i.e. it's b_spa, b_dva, and b_birth fields don't change). 6092 * Thus, we don't need to store the header's sublist index 6093 * on insertion, as this index can be recalculated on removal. 6094 * 6095 * Also, the low order bits of the hash value are thought to be 6096 * distributed evenly. Otherwise, in the case that the multilist 6097 * has a power of two number of sublists, each sublists' usage 6098 * would not be evenly distributed. 6099 */ 6100 return (buf_hash(hdr->b_spa, &hdr->b_dva, hdr->b_birth) % 6101 multilist_get_num_sublists(ml)); 6102 } 6103 6104 static void 6105 arc_state_init(void) 6106 { 6107 arc_anon = &ARC_anon; 6108 arc_mru = &ARC_mru; 6109 arc_mru_ghost = &ARC_mru_ghost; 6110 arc_mfu = &ARC_mfu; 6111 arc_mfu_ghost = &ARC_mfu_ghost; 6112 arc_l2c_only = &ARC_l2c_only; 6113 6114 arc_mru->arcs_list[ARC_BUFC_METADATA] = 6115 multilist_create(sizeof (arc_buf_hdr_t), 6116 offsetof(arc_buf_hdr_t, b_l1hdr.b_arc_node), 6117 arc_state_multilist_index_func); 6118 arc_mru->arcs_list[ARC_BUFC_DATA] = 6119 multilist_create(sizeof (arc_buf_hdr_t), 6120 offsetof(arc_buf_hdr_t, b_l1hdr.b_arc_node), 6121 arc_state_multilist_index_func); 6122 arc_mru_ghost->arcs_list[ARC_BUFC_METADATA] = 6123 multilist_create(sizeof (arc_buf_hdr_t), 6124 offsetof(arc_buf_hdr_t, b_l1hdr.b_arc_node), 6125 arc_state_multilist_index_func); 6126 arc_mru_ghost->arcs_list[ARC_BUFC_DATA] = 6127 multilist_create(sizeof (arc_buf_hdr_t), 6128 offsetof(arc_buf_hdr_t, b_l1hdr.b_arc_node), 6129 arc_state_multilist_index_func); 6130 arc_mfu->arcs_list[ARC_BUFC_METADATA] = 6131 multilist_create(sizeof (arc_buf_hdr_t), 6132 offsetof(arc_buf_hdr_t, b_l1hdr.b_arc_node), 6133 arc_state_multilist_index_func); 6134 arc_mfu->arcs_list[ARC_BUFC_DATA] = 6135 multilist_create(sizeof (arc_buf_hdr_t), 6136 offsetof(arc_buf_hdr_t, b_l1hdr.b_arc_node), 6137 arc_state_multilist_index_func); 6138 arc_mfu_ghost->arcs_list[ARC_BUFC_METADATA] = 6139 multilist_create(sizeof (arc_buf_hdr_t), 6140 offsetof(arc_buf_hdr_t, b_l1hdr.b_arc_node), 6141 arc_state_multilist_index_func); 6142 arc_mfu_ghost->arcs_list[ARC_BUFC_DATA] = 6143 multilist_create(sizeof (arc_buf_hdr_t), 6144 offsetof(arc_buf_hdr_t, b_l1hdr.b_arc_node), 6145 arc_state_multilist_index_func); 6146 arc_l2c_only->arcs_list[ARC_BUFC_METADATA] = 6147 multilist_create(sizeof (arc_buf_hdr_t), 6148 offsetof(arc_buf_hdr_t, b_l1hdr.b_arc_node), 6149 arc_state_multilist_index_func); 6150 arc_l2c_only->arcs_list[ARC_BUFC_DATA] = 6151 multilist_create(sizeof (arc_buf_hdr_t), 6152 offsetof(arc_buf_hdr_t, b_l1hdr.b_arc_node), 6153 arc_state_multilist_index_func); 6154 6155 zfs_refcount_create(&arc_anon->arcs_esize[ARC_BUFC_METADATA]); 6156 zfs_refcount_create(&arc_anon->arcs_esize[ARC_BUFC_DATA]); 6157 zfs_refcount_create(&arc_mru->arcs_esize[ARC_BUFC_METADATA]); 6158 zfs_refcount_create(&arc_mru->arcs_esize[ARC_BUFC_DATA]); 6159 zfs_refcount_create(&arc_mru_ghost->arcs_esize[ARC_BUFC_METADATA]); 6160 zfs_refcount_create(&arc_mru_ghost->arcs_esize[ARC_BUFC_DATA]); 6161 zfs_refcount_create(&arc_mfu->arcs_esize[ARC_BUFC_METADATA]); 6162 zfs_refcount_create(&arc_mfu->arcs_esize[ARC_BUFC_DATA]); 6163 zfs_refcount_create(&arc_mfu_ghost->arcs_esize[ARC_BUFC_METADATA]); 6164 zfs_refcount_create(&arc_mfu_ghost->arcs_esize[ARC_BUFC_DATA]); 6165 zfs_refcount_create(&arc_l2c_only->arcs_esize[ARC_BUFC_METADATA]); 6166 zfs_refcount_create(&arc_l2c_only->arcs_esize[ARC_BUFC_DATA]); 6167 6168 zfs_refcount_create(&arc_anon->arcs_size); 6169 zfs_refcount_create(&arc_mru->arcs_size); 6170 zfs_refcount_create(&arc_mru_ghost->arcs_size); 6171 zfs_refcount_create(&arc_mfu->arcs_size); 6172 zfs_refcount_create(&arc_mfu_ghost->arcs_size); 6173 zfs_refcount_create(&arc_l2c_only->arcs_size); 6174 6175 aggsum_init(&arc_meta_used, 0); 6176 aggsum_init(&arc_size, 0); 6177 aggsum_init(&astat_data_size, 0); 6178 aggsum_init(&astat_metadata_size, 0); 6179 aggsum_init(&astat_hdr_size, 0); 6180 aggsum_init(&astat_other_size, 0); 6181 aggsum_init(&astat_l2_hdr_size, 0); 6182 } 6183 6184 static void 6185 arc_state_fini(void) 6186 { 6187 zfs_refcount_destroy(&arc_anon->arcs_esize[ARC_BUFC_METADATA]); 6188 zfs_refcount_destroy(&arc_anon->arcs_esize[ARC_BUFC_DATA]); 6189 zfs_refcount_destroy(&arc_mru->arcs_esize[ARC_BUFC_METADATA]); 6190 zfs_refcount_destroy(&arc_mru->arcs_esize[ARC_BUFC_DATA]); 6191 zfs_refcount_destroy(&arc_mru_ghost->arcs_esize[ARC_BUFC_METADATA]); 6192 zfs_refcount_destroy(&arc_mru_ghost->arcs_esize[ARC_BUFC_DATA]); 6193 zfs_refcount_destroy(&arc_mfu->arcs_esize[ARC_BUFC_METADATA]); 6194 zfs_refcount_destroy(&arc_mfu->arcs_esize[ARC_BUFC_DATA]); 6195 zfs_refcount_destroy(&arc_mfu_ghost->arcs_esize[ARC_BUFC_METADATA]); 6196 zfs_refcount_destroy(&arc_mfu_ghost->arcs_esize[ARC_BUFC_DATA]); 6197 zfs_refcount_destroy(&arc_l2c_only->arcs_esize[ARC_BUFC_METADATA]); 6198 zfs_refcount_destroy(&arc_l2c_only->arcs_esize[ARC_BUFC_DATA]); 6199 6200 zfs_refcount_destroy(&arc_anon->arcs_size); 6201 zfs_refcount_destroy(&arc_mru->arcs_size); 6202 zfs_refcount_destroy(&arc_mru_ghost->arcs_size); 6203 zfs_refcount_destroy(&arc_mfu->arcs_size); 6204 zfs_refcount_destroy(&arc_mfu_ghost->arcs_size); 6205 zfs_refcount_destroy(&arc_l2c_only->arcs_size); 6206 6207 multilist_destroy(arc_mru->arcs_list[ARC_BUFC_METADATA]); 6208 multilist_destroy(arc_mru_ghost->arcs_list[ARC_BUFC_METADATA]); 6209 multilist_destroy(arc_mfu->arcs_list[ARC_BUFC_METADATA]); 6210 multilist_destroy(arc_mfu_ghost->arcs_list[ARC_BUFC_METADATA]); 6211 multilist_destroy(arc_mru->arcs_list[ARC_BUFC_DATA]); 6212 multilist_destroy(arc_mru_ghost->arcs_list[ARC_BUFC_DATA]); 6213 multilist_destroy(arc_mfu->arcs_list[ARC_BUFC_DATA]); 6214 multilist_destroy(arc_mfu_ghost->arcs_list[ARC_BUFC_DATA]); 6215 6216 aggsum_fini(&arc_meta_used); 6217 aggsum_fini(&arc_size); 6218 aggsum_fini(&astat_data_size); 6219 aggsum_fini(&astat_metadata_size); 6220 aggsum_fini(&astat_hdr_size); 6221 aggsum_fini(&astat_other_size); 6222 aggsum_fini(&astat_l2_hdr_size); 6223 } 6224 6225 uint64_t 6226 arc_max_bytes(void) 6227 { 6228 return (arc_c_max); 6229 } 6230 6231 void 6232 arc_init(void) 6233 { 6234 /* 6235 * allmem is "all memory that we could possibly use". 6236 */ 6237 #ifdef _KERNEL 6238 uint64_t allmem = ptob(physmem - swapfs_minfree); 6239 #else 6240 uint64_t allmem = (physmem * PAGESIZE) / 2; 6241 #endif 6242 mutex_init(&arc_adjust_lock, NULL, MUTEX_DEFAULT, NULL); 6243 cv_init(&arc_adjust_waiters_cv, NULL, CV_DEFAULT, NULL); 6244 6245 /* Convert seconds to clock ticks */ 6246 arc_min_prefetch_lifespan = 1 * hz; 6247 6248 /* set min cache to 1/32 of all memory, or 64MB, whichever is more */ 6249 arc_c_min = MAX(allmem / 32, 64 << 20); 6250 /* set max to 3/4 of all memory, or all but 1GB, whichever is more */ 6251 if (allmem >= 1 << 30) 6252 arc_c_max = allmem - (1 << 30); 6253 else 6254 arc_c_max = arc_c_min; 6255 arc_c_max = MAX(allmem * 3 / 4, arc_c_max); 6256 6257 /* 6258 * In userland, there's only the memory pressure that we artificially 6259 * create (see arc_available_memory()). Don't let arc_c get too 6260 * small, because it can cause transactions to be larger than 6261 * arc_c, causing arc_tempreserve_space() to fail. 6262 */ 6263 #ifndef _KERNEL 6264 arc_c_min = arc_c_max / 2; 6265 #endif 6266 6267 /* 6268 * Allow the tunables to override our calculations if they are 6269 * reasonable (ie. over 64MB) 6270 */ 6271 if (zfs_arc_max > 64 << 20 && zfs_arc_max < allmem) { 6272 arc_c_max = zfs_arc_max; 6273 arc_c_min = MIN(arc_c_min, arc_c_max); 6274 } 6275 if (zfs_arc_min > 64 << 20 && zfs_arc_min <= arc_c_max) 6276 arc_c_min = zfs_arc_min; 6277 6278 arc_c = arc_c_max; 6279 arc_p = (arc_c >> 1); 6280 6281 /* limit meta-data to 1/4 of the arc capacity */ 6282 arc_meta_limit = arc_c_max / 4; 6283 6284 #ifdef _KERNEL 6285 /* 6286 * Metadata is stored in the kernel's heap. Don't let us 6287 * use more than half the heap for the ARC. 6288 */ 6289 arc_meta_limit = MIN(arc_meta_limit, 6290 vmem_size(heap_arena, VMEM_ALLOC | VMEM_FREE) / 2); 6291 #endif 6292 6293 /* Allow the tunable to override if it is reasonable */ 6294 if (zfs_arc_meta_limit > 0 && zfs_arc_meta_limit <= arc_c_max) 6295 arc_meta_limit = zfs_arc_meta_limit; 6296 6297 if (arc_c_min < arc_meta_limit / 2 && zfs_arc_min == 0) 6298 arc_c_min = arc_meta_limit / 2; 6299 6300 if (zfs_arc_meta_min > 0) { 6301 arc_meta_min = zfs_arc_meta_min; 6302 } else { 6303 arc_meta_min = arc_c_min / 2; 6304 } 6305 6306 if (zfs_arc_grow_retry > 0) 6307 arc_grow_retry = zfs_arc_grow_retry; 6308 6309 if (zfs_arc_shrink_shift > 0) 6310 arc_shrink_shift = zfs_arc_shrink_shift; 6311 6312 /* 6313 * Ensure that arc_no_grow_shift is less than arc_shrink_shift. 6314 */ 6315 if (arc_no_grow_shift >= arc_shrink_shift) 6316 arc_no_grow_shift = arc_shrink_shift - 1; 6317 6318 if (zfs_arc_p_min_shift > 0) 6319 arc_p_min_shift = zfs_arc_p_min_shift; 6320 6321 /* if kmem_flags are set, lets try to use less memory */ 6322 if (kmem_debugging()) 6323 arc_c = arc_c / 2; 6324 if (arc_c < arc_c_min) 6325 arc_c = arc_c_min; 6326 6327 arc_state_init(); 6328 6329 /* 6330 * The arc must be "uninitialized", so that hdr_recl() (which is 6331 * registered by buf_init()) will not access arc_reap_zthr before 6332 * it is created. 6333 */ 6334 ASSERT(!arc_initialized); 6335 buf_init(); 6336 6337 arc_ksp = kstat_create("zfs", 0, "arcstats", "misc", KSTAT_TYPE_NAMED, 6338 sizeof (arc_stats) / sizeof (kstat_named_t), KSTAT_FLAG_VIRTUAL); 6339 6340 if (arc_ksp != NULL) { 6341 arc_ksp->ks_data = &arc_stats; 6342 arc_ksp->ks_update = arc_kstat_update; 6343 kstat_install(arc_ksp); 6344 } 6345 6346 arc_adjust_zthr = zthr_create(arc_adjust_cb_check, 6347 arc_adjust_cb, NULL); 6348 arc_reap_zthr = zthr_create_timer(arc_reap_cb_check, 6349 arc_reap_cb, NULL, SEC2NSEC(1)); 6350 6351 arc_initialized = B_TRUE; 6352 arc_warm = B_FALSE; 6353 6354 /* 6355 * Calculate maximum amount of dirty data per pool. 6356 * 6357 * If it has been set by /etc/system, take that. 6358 * Otherwise, use a percentage of physical memory defined by 6359 * zfs_dirty_data_max_percent (default 10%) with a cap at 6360 * zfs_dirty_data_max_max (default 4GB). 6361 */ 6362 if (zfs_dirty_data_max == 0) { 6363 zfs_dirty_data_max = physmem * PAGESIZE * 6364 zfs_dirty_data_max_percent / 100; 6365 zfs_dirty_data_max = MIN(zfs_dirty_data_max, 6366 zfs_dirty_data_max_max); 6367 } 6368 } 6369 6370 void 6371 arc_fini(void) 6372 { 6373 /* Use B_TRUE to ensure *all* buffers are evicted */ 6374 arc_flush(NULL, B_TRUE); 6375 6376 arc_initialized = B_FALSE; 6377 6378 if (arc_ksp != NULL) { 6379 kstat_delete(arc_ksp); 6380 arc_ksp = NULL; 6381 } 6382 6383 (void) zthr_cancel(arc_adjust_zthr); 6384 zthr_destroy(arc_adjust_zthr); 6385 6386 (void) zthr_cancel(arc_reap_zthr); 6387 zthr_destroy(arc_reap_zthr); 6388 6389 mutex_destroy(&arc_adjust_lock); 6390 cv_destroy(&arc_adjust_waiters_cv); 6391 6392 /* 6393 * buf_fini() must proceed arc_state_fini() because buf_fin() may 6394 * trigger the release of kmem magazines, which can callback to 6395 * arc_space_return() which accesses aggsums freed in act_state_fini(). 6396 */ 6397 buf_fini(); 6398 arc_state_fini(); 6399 6400 ASSERT0(arc_loaned_bytes); 6401 } 6402 6403 /* 6404 * Level 2 ARC 6405 * 6406 * The level 2 ARC (L2ARC) is a cache layer in-between main memory and disk. 6407 * It uses dedicated storage devices to hold cached data, which are populated 6408 * using large infrequent writes. The main role of this cache is to boost 6409 * the performance of random read workloads. The intended L2ARC devices 6410 * include short-stroked disks, solid state disks, and other media with 6411 * substantially faster read latency than disk. 6412 * 6413 * +-----------------------+ 6414 * | ARC | 6415 * +-----------------------+ 6416 * | ^ ^ 6417 * | | | 6418 * l2arc_feed_thread() arc_read() 6419 * | | | 6420 * | l2arc read | 6421 * V | | 6422 * +---------------+ | 6423 * | L2ARC | | 6424 * +---------------+ | 6425 * | ^ | 6426 * l2arc_write() | | 6427 * | | | 6428 * V | | 6429 * +-------+ +-------+ 6430 * | vdev | | vdev | 6431 * | cache | | cache | 6432 * +-------+ +-------+ 6433 * +=========+ .-----. 6434 * : L2ARC : |-_____-| 6435 * : devices : | Disks | 6436 * +=========+ `-_____-' 6437 * 6438 * Read requests are satisfied from the following sources, in order: 6439 * 6440 * 1) ARC 6441 * 2) vdev cache of L2ARC devices 6442 * 3) L2ARC devices 6443 * 4) vdev cache of disks 6444 * 5) disks 6445 * 6446 * Some L2ARC device types exhibit extremely slow write performance. 6447 * To accommodate for this there are some significant differences between 6448 * the L2ARC and traditional cache design: 6449 * 6450 * 1. There is no eviction path from the ARC to the L2ARC. Evictions from 6451 * the ARC behave as usual, freeing buffers and placing headers on ghost 6452 * lists. The ARC does not send buffers to the L2ARC during eviction as 6453 * this would add inflated write latencies for all ARC memory pressure. 6454 * 6455 * 2. The L2ARC attempts to cache data from the ARC before it is evicted. 6456 * It does this by periodically scanning buffers from the eviction-end of 6457 * the MFU and MRU ARC lists, copying them to the L2ARC devices if they are 6458 * not already there. It scans until a headroom of buffers is satisfied, 6459 * which itself is a buffer for ARC eviction. If a compressible buffer is 6460 * found during scanning and selected for writing to an L2ARC device, we 6461 * temporarily boost scanning headroom during the next scan cycle to make 6462 * sure we adapt to compression effects (which might significantly reduce 6463 * the data volume we write to L2ARC). The thread that does this is 6464 * l2arc_feed_thread(), illustrated below; example sizes are included to 6465 * provide a better sense of ratio than this diagram: 6466 * 6467 * head --> tail 6468 * +---------------------+----------+ 6469 * ARC_mfu |:::::#:::::::::::::::|o#o###o###|-->. # already on L2ARC 6470 * +---------------------+----------+ | o L2ARC eligible 6471 * ARC_mru |:#:::::::::::::::::::|#o#ooo####|-->| : ARC buffer 6472 * +---------------------+----------+ | 6473 * 15.9 Gbytes ^ 32 Mbytes | 6474 * headroom | 6475 * l2arc_feed_thread() 6476 * | 6477 * l2arc write hand <--[oooo]--' 6478 * | 8 Mbyte 6479 * | write max 6480 * V 6481 * +==============================+ 6482 * L2ARC dev |####|#|###|###| |####| ... | 6483 * +==============================+ 6484 * 32 Gbytes 6485 * 6486 * 3. If an ARC buffer is copied to the L2ARC but then hit instead of 6487 * evicted, then the L2ARC has cached a buffer much sooner than it probably 6488 * needed to, potentially wasting L2ARC device bandwidth and storage. It is 6489 * safe to say that this is an uncommon case, since buffers at the end of 6490 * the ARC lists have moved there due to inactivity. 6491 * 6492 * 4. If the ARC evicts faster than the L2ARC can maintain a headroom, 6493 * then the L2ARC simply misses copying some buffers. This serves as a 6494 * pressure valve to prevent heavy read workloads from both stalling the ARC 6495 * with waits and clogging the L2ARC with writes. This also helps prevent 6496 * the potential for the L2ARC to churn if it attempts to cache content too 6497 * quickly, such as during backups of the entire pool. 6498 * 6499 * 5. After system boot and before the ARC has filled main memory, there are 6500 * no evictions from the ARC and so the tails of the ARC_mfu and ARC_mru 6501 * lists can remain mostly static. Instead of searching from tail of these 6502 * lists as pictured, the l2arc_feed_thread() will search from the list heads 6503 * for eligible buffers, greatly increasing its chance of finding them. 6504 * 6505 * The L2ARC device write speed is also boosted during this time so that 6506 * the L2ARC warms up faster. Since there have been no ARC evictions yet, 6507 * there are no L2ARC reads, and no fear of degrading read performance 6508 * through increased writes. 6509 * 6510 * 6. Writes to the L2ARC devices are grouped and sent in-sequence, so that 6511 * the vdev queue can aggregate them into larger and fewer writes. Each 6512 * device is written to in a rotor fashion, sweeping writes through 6513 * available space then repeating. 6514 * 6515 * 7. The L2ARC does not store dirty content. It never needs to flush 6516 * write buffers back to disk based storage. 6517 * 6518 * 8. If an ARC buffer is written (and dirtied) which also exists in the 6519 * L2ARC, the now stale L2ARC buffer is immediately dropped. 6520 * 6521 * The performance of the L2ARC can be tweaked by a number of tunables, which 6522 * may be necessary for different workloads: 6523 * 6524 * l2arc_write_max max write bytes per interval 6525 * l2arc_write_boost extra write bytes during device warmup 6526 * l2arc_noprefetch skip caching prefetched buffers 6527 * l2arc_headroom number of max device writes to precache 6528 * l2arc_headroom_boost when we find compressed buffers during ARC 6529 * scanning, we multiply headroom by this 6530 * percentage factor for the next scan cycle, 6531 * since more compressed buffers are likely to 6532 * be present 6533 * l2arc_feed_secs seconds between L2ARC writing 6534 * 6535 * Tunables may be removed or added as future performance improvements are 6536 * integrated, and also may become zpool properties. 6537 * 6538 * There are three key functions that control how the L2ARC warms up: 6539 * 6540 * l2arc_write_eligible() check if a buffer is eligible to cache 6541 * l2arc_write_size() calculate how much to write 6542 * l2arc_write_interval() calculate sleep delay between writes 6543 * 6544 * These three functions determine what to write, how much, and how quickly 6545 * to send writes. 6546 */ 6547 6548 static boolean_t 6549 l2arc_write_eligible(uint64_t spa_guid, arc_buf_hdr_t *hdr) 6550 { 6551 /* 6552 * A buffer is *not* eligible for the L2ARC if it: 6553 * 1. belongs to a different spa. 6554 * 2. is already cached on the L2ARC. 6555 * 3. has an I/O in progress (it may be an incomplete read). 6556 * 4. is flagged not eligible (zfs property). 6557 */ 6558 if (hdr->b_spa != spa_guid || HDR_HAS_L2HDR(hdr) || 6559 HDR_IO_IN_PROGRESS(hdr) || !HDR_L2CACHE(hdr)) 6560 return (B_FALSE); 6561 6562 return (B_TRUE); 6563 } 6564 6565 static uint64_t 6566 l2arc_write_size(void) 6567 { 6568 uint64_t size; 6569 6570 /* 6571 * Make sure our globals have meaningful values in case the user 6572 * altered them. 6573 */ 6574 size = l2arc_write_max; 6575 if (size == 0) { 6576 cmn_err(CE_NOTE, "Bad value for l2arc_write_max, value must " 6577 "be greater than zero, resetting it to the default (%d)", 6578 L2ARC_WRITE_SIZE); 6579 size = l2arc_write_max = L2ARC_WRITE_SIZE; 6580 } 6581 6582 if (arc_warm == B_FALSE) 6583 size += l2arc_write_boost; 6584 6585 return (size); 6586 6587 } 6588 6589 static clock_t 6590 l2arc_write_interval(clock_t began, uint64_t wanted, uint64_t wrote) 6591 { 6592 clock_t interval, next, now; 6593 6594 /* 6595 * If the ARC lists are busy, increase our write rate; if the 6596 * lists are stale, idle back. This is achieved by checking 6597 * how much we previously wrote - if it was more than half of 6598 * what we wanted, schedule the next write much sooner. 6599 */ 6600 if (l2arc_feed_again && wrote > (wanted / 2)) 6601 interval = (hz * l2arc_feed_min_ms) / 1000; 6602 else 6603 interval = hz * l2arc_feed_secs; 6604 6605 now = ddi_get_lbolt(); 6606 next = MAX(now, MIN(now + interval, began + interval)); 6607 6608 return (next); 6609 } 6610 6611 /* 6612 * Cycle through L2ARC devices. This is how L2ARC load balances. 6613 * If a device is returned, this also returns holding the spa config lock. 6614 */ 6615 static l2arc_dev_t * 6616 l2arc_dev_get_next(void) 6617 { 6618 l2arc_dev_t *first, *next = NULL; 6619 6620 /* 6621 * Lock out the removal of spas (spa_namespace_lock), then removal 6622 * of cache devices (l2arc_dev_mtx). Once a device has been selected, 6623 * both locks will be dropped and a spa config lock held instead. 6624 */ 6625 mutex_enter(&spa_namespace_lock); 6626 mutex_enter(&l2arc_dev_mtx); 6627 6628 /* if there are no vdevs, there is nothing to do */ 6629 if (l2arc_ndev == 0) 6630 goto out; 6631 6632 first = NULL; 6633 next = l2arc_dev_last; 6634 do { 6635 /* loop around the list looking for a non-faulted vdev */ 6636 if (next == NULL) { 6637 next = list_head(l2arc_dev_list); 6638 } else { 6639 next = list_next(l2arc_dev_list, next); 6640 if (next == NULL) 6641 next = list_head(l2arc_dev_list); 6642 } 6643 6644 /* if we have come back to the start, bail out */ 6645 if (first == NULL) 6646 first = next; 6647 else if (next == first) 6648 break; 6649 6650 } while (vdev_is_dead(next->l2ad_vdev)); 6651 6652 /* if we were unable to find any usable vdevs, return NULL */ 6653 if (vdev_is_dead(next->l2ad_vdev)) 6654 next = NULL; 6655 6656 l2arc_dev_last = next; 6657 6658 out: 6659 mutex_exit(&l2arc_dev_mtx); 6660 6661 /* 6662 * Grab the config lock to prevent the 'next' device from being 6663 * removed while we are writing to it. 6664 */ 6665 if (next != NULL) 6666 spa_config_enter(next->l2ad_spa, SCL_L2ARC, next, RW_READER); 6667 mutex_exit(&spa_namespace_lock); 6668 6669 return (next); 6670 } 6671 6672 /* 6673 * Free buffers that were tagged for destruction. 6674 */ 6675 static void 6676 l2arc_do_free_on_write() 6677 { 6678 list_t *buflist; 6679 l2arc_data_free_t *df, *df_prev; 6680 6681 mutex_enter(&l2arc_free_on_write_mtx); 6682 buflist = l2arc_free_on_write; 6683 6684 for (df = list_tail(buflist); df; df = df_prev) { 6685 df_prev = list_prev(buflist, df); 6686 ASSERT3P(df->l2df_abd, !=, NULL); 6687 abd_free(df->l2df_abd); 6688 list_remove(buflist, df); 6689 kmem_free(df, sizeof (l2arc_data_free_t)); 6690 } 6691 6692 mutex_exit(&l2arc_free_on_write_mtx); 6693 } 6694 6695 /* 6696 * A write to a cache device has completed. Update all headers to allow 6697 * reads from these buffers to begin. 6698 */ 6699 static void 6700 l2arc_write_done(zio_t *zio) 6701 { 6702 l2arc_write_callback_t *cb; 6703 l2arc_dev_t *dev; 6704 list_t *buflist; 6705 arc_buf_hdr_t *head, *hdr, *hdr_prev; 6706 kmutex_t *hash_lock; 6707 int64_t bytes_dropped = 0; 6708 6709 cb = zio->io_private; 6710 ASSERT3P(cb, !=, NULL); 6711 dev = cb->l2wcb_dev; 6712 ASSERT3P(dev, !=, NULL); 6713 head = cb->l2wcb_head; 6714 ASSERT3P(head, !=, NULL); 6715 buflist = &dev->l2ad_buflist; 6716 ASSERT3P(buflist, !=, NULL); 6717 DTRACE_PROBE2(l2arc__iodone, zio_t *, zio, 6718 l2arc_write_callback_t *, cb); 6719 6720 if (zio->io_error != 0) 6721 ARCSTAT_BUMP(arcstat_l2_writes_error); 6722 6723 /* 6724 * All writes completed, or an error was hit. 6725 */ 6726 top: 6727 mutex_enter(&dev->l2ad_mtx); 6728 for (hdr = list_prev(buflist, head); hdr; hdr = hdr_prev) { 6729 hdr_prev = list_prev(buflist, hdr); 6730 6731 hash_lock = HDR_LOCK(hdr); 6732 6733 /* 6734 * We cannot use mutex_enter or else we can deadlock 6735 * with l2arc_write_buffers (due to swapping the order 6736 * the hash lock and l2ad_mtx are taken). 6737 */ 6738 if (!mutex_tryenter(hash_lock)) { 6739 /* 6740 * Missed the hash lock. We must retry so we 6741 * don't leave the ARC_FLAG_L2_WRITING bit set. 6742 */ 6743 ARCSTAT_BUMP(arcstat_l2_writes_lock_retry); 6744 6745 /* 6746 * We don't want to rescan the headers we've 6747 * already marked as having been written out, so 6748 * we reinsert the head node so we can pick up 6749 * where we left off. 6750 */ 6751 list_remove(buflist, head); 6752 list_insert_after(buflist, hdr, head); 6753 6754 mutex_exit(&dev->l2ad_mtx); 6755 6756 /* 6757 * We wait for the hash lock to become available 6758 * to try and prevent busy waiting, and increase 6759 * the chance we'll be able to acquire the lock 6760 * the next time around. 6761 */ 6762 mutex_enter(hash_lock); 6763 mutex_exit(hash_lock); 6764 goto top; 6765 } 6766 6767 /* 6768 * We could not have been moved into the arc_l2c_only 6769 * state while in-flight due to our ARC_FLAG_L2_WRITING 6770 * bit being set. Let's just ensure that's being enforced. 6771 */ 6772 ASSERT(HDR_HAS_L1HDR(hdr)); 6773 6774 if (zio->io_error != 0) { 6775 /* 6776 * Error - drop L2ARC entry. 6777 */ 6778 list_remove(buflist, hdr); 6779 arc_hdr_clear_flags(hdr, ARC_FLAG_HAS_L2HDR); 6780 6781 ARCSTAT_INCR(arcstat_l2_psize, -arc_hdr_size(hdr)); 6782 ARCSTAT_INCR(arcstat_l2_lsize, -HDR_GET_LSIZE(hdr)); 6783 6784 bytes_dropped += arc_hdr_size(hdr); 6785 (void) zfs_refcount_remove_many(&dev->l2ad_alloc, 6786 arc_hdr_size(hdr), hdr); 6787 } 6788 6789 /* 6790 * Allow ARC to begin reads and ghost list evictions to 6791 * this L2ARC entry. 6792 */ 6793 arc_hdr_clear_flags(hdr, ARC_FLAG_L2_WRITING); 6794 6795 mutex_exit(hash_lock); 6796 } 6797 6798 atomic_inc_64(&l2arc_writes_done); 6799 list_remove(buflist, head); 6800 ASSERT(!HDR_HAS_L1HDR(head)); 6801 kmem_cache_free(hdr_l2only_cache, head); 6802 mutex_exit(&dev->l2ad_mtx); 6803 6804 vdev_space_update(dev->l2ad_vdev, -bytes_dropped, 0, 0); 6805 6806 l2arc_do_free_on_write(); 6807 6808 kmem_free(cb, sizeof (l2arc_write_callback_t)); 6809 } 6810 6811 /* 6812 * A read to a cache device completed. Validate buffer contents before 6813 * handing over to the regular ARC routines. 6814 */ 6815 static void 6816 l2arc_read_done(zio_t *zio) 6817 { 6818 l2arc_read_callback_t *cb; 6819 arc_buf_hdr_t *hdr; 6820 kmutex_t *hash_lock; 6821 boolean_t valid_cksum; 6822 6823 ASSERT3P(zio->io_vd, !=, NULL); 6824 ASSERT(zio->io_flags & ZIO_FLAG_DONT_PROPAGATE); 6825 6826 spa_config_exit(zio->io_spa, SCL_L2ARC, zio->io_vd); 6827 6828 cb = zio->io_private; 6829 ASSERT3P(cb, !=, NULL); 6830 hdr = cb->l2rcb_hdr; 6831 ASSERT3P(hdr, !=, NULL); 6832 6833 hash_lock = HDR_LOCK(hdr); 6834 mutex_enter(hash_lock); 6835 ASSERT3P(hash_lock, ==, HDR_LOCK(hdr)); 6836 6837 /* 6838 * If the data was read into a temporary buffer, 6839 * move it and free the buffer. 6840 */ 6841 if (cb->l2rcb_abd != NULL) { 6842 ASSERT3U(arc_hdr_size(hdr), <, zio->io_size); 6843 if (zio->io_error == 0) { 6844 abd_copy(hdr->b_l1hdr.b_pabd, cb->l2rcb_abd, 6845 arc_hdr_size(hdr)); 6846 } 6847 6848 /* 6849 * The following must be done regardless of whether 6850 * there was an error: 6851 * - free the temporary buffer 6852 * - point zio to the real ARC buffer 6853 * - set zio size accordingly 6854 * These are required because zio is either re-used for 6855 * an I/O of the block in the case of the error 6856 * or the zio is passed to arc_read_done() and it 6857 * needs real data. 6858 */ 6859 abd_free(cb->l2rcb_abd); 6860 zio->io_size = zio->io_orig_size = arc_hdr_size(hdr); 6861 zio->io_abd = zio->io_orig_abd = hdr->b_l1hdr.b_pabd; 6862 } 6863 6864 ASSERT3P(zio->io_abd, !=, NULL); 6865 6866 /* 6867 * Check this survived the L2ARC journey. 6868 */ 6869 ASSERT3P(zio->io_abd, ==, hdr->b_l1hdr.b_pabd); 6870 zio->io_bp_copy = cb->l2rcb_bp; /* XXX fix in L2ARC 2.0 */ 6871 zio->io_bp = &zio->io_bp_copy; /* XXX fix in L2ARC 2.0 */ 6872 6873 valid_cksum = arc_cksum_is_equal(hdr, zio); 6874 if (valid_cksum && zio->io_error == 0 && !HDR_L2_EVICTED(hdr)) { 6875 mutex_exit(hash_lock); 6876 zio->io_private = hdr; 6877 arc_read_done(zio); 6878 } else { 6879 mutex_exit(hash_lock); 6880 /* 6881 * Buffer didn't survive caching. Increment stats and 6882 * reissue to the original storage device. 6883 */ 6884 if (zio->io_error != 0) { 6885 ARCSTAT_BUMP(arcstat_l2_io_error); 6886 } else { 6887 zio->io_error = SET_ERROR(EIO); 6888 } 6889 if (!valid_cksum) 6890 ARCSTAT_BUMP(arcstat_l2_cksum_bad); 6891 6892 /* 6893 * If there's no waiter, issue an async i/o to the primary 6894 * storage now. If there *is* a waiter, the caller must 6895 * issue the i/o in a context where it's OK to block. 6896 */ 6897 if (zio->io_waiter == NULL) { 6898 zio_t *pio = zio_unique_parent(zio); 6899 6900 ASSERT(!pio || pio->io_child_type == ZIO_CHILD_LOGICAL); 6901 6902 zio_nowait(zio_read(pio, zio->io_spa, zio->io_bp, 6903 hdr->b_l1hdr.b_pabd, zio->io_size, arc_read_done, 6904 hdr, zio->io_priority, cb->l2rcb_flags, 6905 &cb->l2rcb_zb)); 6906 } 6907 } 6908 6909 kmem_free(cb, sizeof (l2arc_read_callback_t)); 6910 } 6911 6912 /* 6913 * This is the list priority from which the L2ARC will search for pages to 6914 * cache. This is used within loops (0..3) to cycle through lists in the 6915 * desired order. This order can have a significant effect on cache 6916 * performance. 6917 * 6918 * Currently the metadata lists are hit first, MFU then MRU, followed by 6919 * the data lists. This function returns a locked list, and also returns 6920 * the lock pointer. 6921 */ 6922 static multilist_sublist_t * 6923 l2arc_sublist_lock(int list_num) 6924 { 6925 multilist_t *ml = NULL; 6926 unsigned int idx; 6927 6928 ASSERT(list_num >= 0 && list_num <= 3); 6929 6930 switch (list_num) { 6931 case 0: 6932 ml = arc_mfu->arcs_list[ARC_BUFC_METADATA]; 6933 break; 6934 case 1: 6935 ml = arc_mru->arcs_list[ARC_BUFC_METADATA]; 6936 break; 6937 case 2: 6938 ml = arc_mfu->arcs_list[ARC_BUFC_DATA]; 6939 break; 6940 case 3: 6941 ml = arc_mru->arcs_list[ARC_BUFC_DATA]; 6942 break; 6943 } 6944 6945 /* 6946 * Return a randomly-selected sublist. This is acceptable 6947 * because the caller feeds only a little bit of data for each 6948 * call (8MB). Subsequent calls will result in different 6949 * sublists being selected. 6950 */ 6951 idx = multilist_get_random_index(ml); 6952 return (multilist_sublist_lock(ml, idx)); 6953 } 6954 6955 /* 6956 * Evict buffers from the device write hand to the distance specified in 6957 * bytes. This distance may span populated buffers, it may span nothing. 6958 * This is clearing a region on the L2ARC device ready for writing. 6959 * If the 'all' boolean is set, every buffer is evicted. 6960 */ 6961 static void 6962 l2arc_evict(l2arc_dev_t *dev, uint64_t distance, boolean_t all) 6963 { 6964 list_t *buflist; 6965 arc_buf_hdr_t *hdr, *hdr_prev; 6966 kmutex_t *hash_lock; 6967 uint64_t taddr; 6968 6969 buflist = &dev->l2ad_buflist; 6970 6971 if (!all && dev->l2ad_first) { 6972 /* 6973 * This is the first sweep through the device. There is 6974 * nothing to evict. 6975 */ 6976 return; 6977 } 6978 6979 if (dev->l2ad_hand >= (dev->l2ad_end - (2 * distance))) { 6980 /* 6981 * When nearing the end of the device, evict to the end 6982 * before the device write hand jumps to the start. 6983 */ 6984 taddr = dev->l2ad_end; 6985 } else { 6986 taddr = dev->l2ad_hand + distance; 6987 } 6988 DTRACE_PROBE4(l2arc__evict, l2arc_dev_t *, dev, list_t *, buflist, 6989 uint64_t, taddr, boolean_t, all); 6990 6991 top: 6992 mutex_enter(&dev->l2ad_mtx); 6993 for (hdr = list_tail(buflist); hdr; hdr = hdr_prev) { 6994 hdr_prev = list_prev(buflist, hdr); 6995 6996 hash_lock = HDR_LOCK(hdr); 6997 6998 /* 6999 * We cannot use mutex_enter or else we can deadlock 7000 * with l2arc_write_buffers (due to swapping the order 7001 * the hash lock and l2ad_mtx are taken). 7002 */ 7003 if (!mutex_tryenter(hash_lock)) { 7004 /* 7005 * Missed the hash lock. Retry. 7006 */ 7007 ARCSTAT_BUMP(arcstat_l2_evict_lock_retry); 7008 mutex_exit(&dev->l2ad_mtx); 7009 mutex_enter(hash_lock); 7010 mutex_exit(hash_lock); 7011 goto top; 7012 } 7013 7014 /* 7015 * A header can't be on this list if it doesn't have L2 header. 7016 */ 7017 ASSERT(HDR_HAS_L2HDR(hdr)); 7018 7019 /* Ensure this header has finished being written. */ 7020 ASSERT(!HDR_L2_WRITING(hdr)); 7021 ASSERT(!HDR_L2_WRITE_HEAD(hdr)); 7022 7023 if (!all && (hdr->b_l2hdr.b_daddr >= taddr || 7024 hdr->b_l2hdr.b_daddr < dev->l2ad_hand)) { 7025 /* 7026 * We've evicted to the target address, 7027 * or the end of the device. 7028 */ 7029 mutex_exit(hash_lock); 7030 break; 7031 } 7032 7033 if (!HDR_HAS_L1HDR(hdr)) { 7034 ASSERT(!HDR_L2_READING(hdr)); 7035 /* 7036 * This doesn't exist in the ARC. Destroy. 7037 * arc_hdr_destroy() will call list_remove() 7038 * and decrement arcstat_l2_lsize. 7039 */ 7040 arc_change_state(arc_anon, hdr, hash_lock); 7041 arc_hdr_destroy(hdr); 7042 } else { 7043 ASSERT(hdr->b_l1hdr.b_state != arc_l2c_only); 7044 ARCSTAT_BUMP(arcstat_l2_evict_l1cached); 7045 /* 7046 * Invalidate issued or about to be issued 7047 * reads, since we may be about to write 7048 * over this location. 7049 */ 7050 if (HDR_L2_READING(hdr)) { 7051 ARCSTAT_BUMP(arcstat_l2_evict_reading); 7052 arc_hdr_set_flags(hdr, ARC_FLAG_L2_EVICTED); 7053 } 7054 7055 arc_hdr_l2hdr_destroy(hdr); 7056 } 7057 mutex_exit(hash_lock); 7058 } 7059 mutex_exit(&dev->l2ad_mtx); 7060 } 7061 7062 /* 7063 * Find and write ARC buffers to the L2ARC device. 7064 * 7065 * An ARC_FLAG_L2_WRITING flag is set so that the L2ARC buffers are not valid 7066 * for reading until they have completed writing. 7067 * The headroom_boost is an in-out parameter used to maintain headroom boost 7068 * state between calls to this function. 7069 * 7070 * Returns the number of bytes actually written (which may be smaller than 7071 * the delta by which the device hand has changed due to alignment). 7072 */ 7073 static uint64_t 7074 l2arc_write_buffers(spa_t *spa, l2arc_dev_t *dev, uint64_t target_sz) 7075 { 7076 arc_buf_hdr_t *hdr, *hdr_prev, *head; 7077 uint64_t write_asize, write_psize, write_lsize, headroom; 7078 boolean_t full; 7079 l2arc_write_callback_t *cb; 7080 zio_t *pio, *wzio; 7081 uint64_t guid = spa_load_guid(spa); 7082 7083 ASSERT3P(dev->l2ad_vdev, !=, NULL); 7084 7085 pio = NULL; 7086 write_lsize = write_asize = write_psize = 0; 7087 full = B_FALSE; 7088 head = kmem_cache_alloc(hdr_l2only_cache, KM_PUSHPAGE); 7089 arc_hdr_set_flags(head, ARC_FLAG_L2_WRITE_HEAD | ARC_FLAG_HAS_L2HDR); 7090 7091 /* 7092 * Copy buffers for L2ARC writing. 7093 */ 7094 for (int try = 0; try <= 3; try++) { 7095 multilist_sublist_t *mls = l2arc_sublist_lock(try); 7096 uint64_t passed_sz = 0; 7097 7098 /* 7099 * L2ARC fast warmup. 7100 * 7101 * Until the ARC is warm and starts to evict, read from the 7102 * head of the ARC lists rather than the tail. 7103 */ 7104 if (arc_warm == B_FALSE) 7105 hdr = multilist_sublist_head(mls); 7106 else 7107 hdr = multilist_sublist_tail(mls); 7108 7109 headroom = target_sz * l2arc_headroom; 7110 if (zfs_compressed_arc_enabled) 7111 headroom = (headroom * l2arc_headroom_boost) / 100; 7112 7113 for (; hdr; hdr = hdr_prev) { 7114 kmutex_t *hash_lock; 7115 7116 if (arc_warm == B_FALSE) 7117 hdr_prev = multilist_sublist_next(mls, hdr); 7118 else 7119 hdr_prev = multilist_sublist_prev(mls, hdr); 7120 7121 hash_lock = HDR_LOCK(hdr); 7122 if (!mutex_tryenter(hash_lock)) { 7123 /* 7124 * Skip this buffer rather than waiting. 7125 */ 7126 continue; 7127 } 7128 7129 passed_sz += HDR_GET_LSIZE(hdr); 7130 if (passed_sz > headroom) { 7131 /* 7132 * Searched too far. 7133 */ 7134 mutex_exit(hash_lock); 7135 break; 7136 } 7137 7138 if (!l2arc_write_eligible(guid, hdr)) { 7139 mutex_exit(hash_lock); 7140 continue; 7141 } 7142 7143 /* 7144 * We rely on the L1 portion of the header below, so 7145 * it's invalid for this header to have been evicted out 7146 * of the ghost cache, prior to being written out. The 7147 * ARC_FLAG_L2_WRITING bit ensures this won't happen. 7148 */ 7149 ASSERT(HDR_HAS_L1HDR(hdr)); 7150 7151 ASSERT3U(HDR_GET_PSIZE(hdr), >, 0); 7152 ASSERT3P(hdr->b_l1hdr.b_pabd, !=, NULL); 7153 ASSERT3U(arc_hdr_size(hdr), >, 0); 7154 uint64_t psize = arc_hdr_size(hdr); 7155 uint64_t asize = vdev_psize_to_asize(dev->l2ad_vdev, 7156 psize); 7157 7158 if ((write_asize + asize) > target_sz) { 7159 full = B_TRUE; 7160 mutex_exit(hash_lock); 7161 break; 7162 } 7163 7164 if (pio == NULL) { 7165 /* 7166 * Insert a dummy header on the buflist so 7167 * l2arc_write_done() can find where the 7168 * write buffers begin without searching. 7169 */ 7170 mutex_enter(&dev->l2ad_mtx); 7171 list_insert_head(&dev->l2ad_buflist, head); 7172 mutex_exit(&dev->l2ad_mtx); 7173 7174 cb = kmem_alloc( 7175 sizeof (l2arc_write_callback_t), KM_SLEEP); 7176 cb->l2wcb_dev = dev; 7177 cb->l2wcb_head = head; 7178 pio = zio_root(spa, l2arc_write_done, cb, 7179 ZIO_FLAG_CANFAIL); 7180 } 7181 7182 hdr->b_l2hdr.b_dev = dev; 7183 hdr->b_l2hdr.b_daddr = dev->l2ad_hand; 7184 arc_hdr_set_flags(hdr, 7185 ARC_FLAG_L2_WRITING | ARC_FLAG_HAS_L2HDR); 7186 7187 mutex_enter(&dev->l2ad_mtx); 7188 list_insert_head(&dev->l2ad_buflist, hdr); 7189 mutex_exit(&dev->l2ad_mtx); 7190 7191 (void) zfs_refcount_add_many(&dev->l2ad_alloc, psize, 7192 hdr); 7193 7194 /* 7195 * Normally the L2ARC can use the hdr's data, but if 7196 * we're sharing data between the hdr and one of its 7197 * bufs, L2ARC needs its own copy of the data so that 7198 * the ZIO below can't race with the buf consumer. 7199 * Another case where we need to create a copy of the 7200 * data is when the buffer size is not device-aligned 7201 * and we need to pad the block to make it such. 7202 * That also keeps the clock hand suitably aligned. 7203 * 7204 * To ensure that the copy will be available for the 7205 * lifetime of the ZIO and be cleaned up afterwards, we 7206 * add it to the l2arc_free_on_write queue. 7207 */ 7208 abd_t *to_write; 7209 if (!HDR_SHARED_DATA(hdr) && psize == asize) { 7210 to_write = hdr->b_l1hdr.b_pabd; 7211 } else { 7212 to_write = abd_alloc_for_io(asize, 7213 HDR_ISTYPE_METADATA(hdr)); 7214 abd_copy(to_write, hdr->b_l1hdr.b_pabd, psize); 7215 if (asize != psize) { 7216 abd_zero_off(to_write, psize, 7217 asize - psize); 7218 } 7219 l2arc_free_abd_on_write(to_write, asize, 7220 arc_buf_type(hdr)); 7221 } 7222 wzio = zio_write_phys(pio, dev->l2ad_vdev, 7223 hdr->b_l2hdr.b_daddr, asize, to_write, 7224 ZIO_CHECKSUM_OFF, NULL, hdr, 7225 ZIO_PRIORITY_ASYNC_WRITE, 7226 ZIO_FLAG_CANFAIL, B_FALSE); 7227 7228 write_lsize += HDR_GET_LSIZE(hdr); 7229 DTRACE_PROBE2(l2arc__write, vdev_t *, dev->l2ad_vdev, 7230 zio_t *, wzio); 7231 7232 write_psize += psize; 7233 write_asize += asize; 7234 dev->l2ad_hand += asize; 7235 7236 mutex_exit(hash_lock); 7237 7238 (void) zio_nowait(wzio); 7239 } 7240 7241 multilist_sublist_unlock(mls); 7242 7243 if (full == B_TRUE) 7244 break; 7245 } 7246 7247 /* No buffers selected for writing? */ 7248 if (pio == NULL) { 7249 ASSERT0(write_lsize); 7250 ASSERT(!HDR_HAS_L1HDR(head)); 7251 kmem_cache_free(hdr_l2only_cache, head); 7252 return (0); 7253 } 7254 7255 ASSERT3U(write_asize, <=, target_sz); 7256 ARCSTAT_BUMP(arcstat_l2_writes_sent); 7257 ARCSTAT_INCR(arcstat_l2_write_bytes, write_psize); 7258 ARCSTAT_INCR(arcstat_l2_lsize, write_lsize); 7259 ARCSTAT_INCR(arcstat_l2_psize, write_psize); 7260 vdev_space_update(dev->l2ad_vdev, write_psize, 0, 0); 7261 7262 /* 7263 * Bump device hand to the device start if it is approaching the end. 7264 * l2arc_evict() will already have evicted ahead for this case. 7265 */ 7266 if (dev->l2ad_hand >= (dev->l2ad_end - target_sz)) { 7267 dev->l2ad_hand = dev->l2ad_start; 7268 dev->l2ad_first = B_FALSE; 7269 } 7270 7271 dev->l2ad_writing = B_TRUE; 7272 (void) zio_wait(pio); 7273 dev->l2ad_writing = B_FALSE; 7274 7275 return (write_asize); 7276 } 7277 7278 /* 7279 * This thread feeds the L2ARC at regular intervals. This is the beating 7280 * heart of the L2ARC. 7281 */ 7282 /* ARGSUSED */ 7283 static void 7284 l2arc_feed_thread(void *unused) 7285 { 7286 callb_cpr_t cpr; 7287 l2arc_dev_t *dev; 7288 spa_t *spa; 7289 uint64_t size, wrote; 7290 clock_t begin, next = ddi_get_lbolt(); 7291 7292 CALLB_CPR_INIT(&cpr, &l2arc_feed_thr_lock, callb_generic_cpr, FTAG); 7293 7294 mutex_enter(&l2arc_feed_thr_lock); 7295 7296 while (l2arc_thread_exit == 0) { 7297 CALLB_CPR_SAFE_BEGIN(&cpr); 7298 (void) cv_timedwait(&l2arc_feed_thr_cv, &l2arc_feed_thr_lock, 7299 next); 7300 CALLB_CPR_SAFE_END(&cpr, &l2arc_feed_thr_lock); 7301 next = ddi_get_lbolt() + hz; 7302 7303 /* 7304 * Quick check for L2ARC devices. 7305 */ 7306 mutex_enter(&l2arc_dev_mtx); 7307 if (l2arc_ndev == 0) { 7308 mutex_exit(&l2arc_dev_mtx); 7309 continue; 7310 } 7311 mutex_exit(&l2arc_dev_mtx); 7312 begin = ddi_get_lbolt(); 7313 7314 /* 7315 * This selects the next l2arc device to write to, and in 7316 * doing so the next spa to feed from: dev->l2ad_spa. This 7317 * will return NULL if there are now no l2arc devices or if 7318 * they are all faulted. 7319 * 7320 * If a device is returned, its spa's config lock is also 7321 * held to prevent device removal. l2arc_dev_get_next() 7322 * will grab and release l2arc_dev_mtx. 7323 */ 7324 if ((dev = l2arc_dev_get_next()) == NULL) 7325 continue; 7326 7327 spa = dev->l2ad_spa; 7328 ASSERT3P(spa, !=, NULL); 7329 7330 /* 7331 * If the pool is read-only then force the feed thread to 7332 * sleep a little longer. 7333 */ 7334 if (!spa_writeable(spa)) { 7335 next = ddi_get_lbolt() + 5 * l2arc_feed_secs * hz; 7336 spa_config_exit(spa, SCL_L2ARC, dev); 7337 continue; 7338 } 7339 7340 /* 7341 * Avoid contributing to memory pressure. 7342 */ 7343 if (arc_reclaim_needed()) { 7344 ARCSTAT_BUMP(arcstat_l2_abort_lowmem); 7345 spa_config_exit(spa, SCL_L2ARC, dev); 7346 continue; 7347 } 7348 7349 ARCSTAT_BUMP(arcstat_l2_feeds); 7350 7351 size = l2arc_write_size(); 7352 7353 /* 7354 * Evict L2ARC buffers that will be overwritten. 7355 */ 7356 l2arc_evict(dev, size, B_FALSE); 7357 7358 /* 7359 * Write ARC buffers. 7360 */ 7361 wrote = l2arc_write_buffers(spa, dev, size); 7362 7363 /* 7364 * Calculate interval between writes. 7365 */ 7366 next = l2arc_write_interval(begin, size, wrote); 7367 spa_config_exit(spa, SCL_L2ARC, dev); 7368 } 7369 7370 l2arc_thread_exit = 0; 7371 cv_broadcast(&l2arc_feed_thr_cv); 7372 CALLB_CPR_EXIT(&cpr); /* drops l2arc_feed_thr_lock */ 7373 thread_exit(); 7374 } 7375 7376 boolean_t 7377 l2arc_vdev_present(vdev_t *vd) 7378 { 7379 l2arc_dev_t *dev; 7380 7381 mutex_enter(&l2arc_dev_mtx); 7382 for (dev = list_head(l2arc_dev_list); dev != NULL; 7383 dev = list_next(l2arc_dev_list, dev)) { 7384 if (dev->l2ad_vdev == vd) 7385 break; 7386 } 7387 mutex_exit(&l2arc_dev_mtx); 7388 7389 return (dev != NULL); 7390 } 7391 7392 /* 7393 * Add a vdev for use by the L2ARC. By this point the spa has already 7394 * validated the vdev and opened it. 7395 */ 7396 void 7397 l2arc_add_vdev(spa_t *spa, vdev_t *vd) 7398 { 7399 l2arc_dev_t *adddev; 7400 7401 ASSERT(!l2arc_vdev_present(vd)); 7402 7403 /* 7404 * Create a new l2arc device entry. 7405 */ 7406 adddev = kmem_zalloc(sizeof (l2arc_dev_t), KM_SLEEP); 7407 adddev->l2ad_spa = spa; 7408 adddev->l2ad_vdev = vd; 7409 adddev->l2ad_start = VDEV_LABEL_START_SIZE; 7410 adddev->l2ad_end = VDEV_LABEL_START_SIZE + vdev_get_min_asize(vd); 7411 adddev->l2ad_hand = adddev->l2ad_start; 7412 adddev->l2ad_first = B_TRUE; 7413 adddev->l2ad_writing = B_FALSE; 7414 7415 mutex_init(&adddev->l2ad_mtx, NULL, MUTEX_DEFAULT, NULL); 7416 /* 7417 * This is a list of all ARC buffers that are still valid on the 7418 * device. 7419 */ 7420 list_create(&adddev->l2ad_buflist, sizeof (arc_buf_hdr_t), 7421 offsetof(arc_buf_hdr_t, b_l2hdr.b_l2node)); 7422 7423 vdev_space_update(vd, 0, 0, adddev->l2ad_end - adddev->l2ad_hand); 7424 zfs_refcount_create(&adddev->l2ad_alloc); 7425 7426 /* 7427 * Add device to global list 7428 */ 7429 mutex_enter(&l2arc_dev_mtx); 7430 list_insert_head(l2arc_dev_list, adddev); 7431 atomic_inc_64(&l2arc_ndev); 7432 mutex_exit(&l2arc_dev_mtx); 7433 } 7434 7435 /* 7436 * Remove a vdev from the L2ARC. 7437 */ 7438 void 7439 l2arc_remove_vdev(vdev_t *vd) 7440 { 7441 l2arc_dev_t *dev, *nextdev, *remdev = NULL; 7442 7443 /* 7444 * Find the device by vdev 7445 */ 7446 mutex_enter(&l2arc_dev_mtx); 7447 for (dev = list_head(l2arc_dev_list); dev; dev = nextdev) { 7448 nextdev = list_next(l2arc_dev_list, dev); 7449 if (vd == dev->l2ad_vdev) { 7450 remdev = dev; 7451 break; 7452 } 7453 } 7454 ASSERT3P(remdev, !=, NULL); 7455 7456 /* 7457 * Remove device from global list 7458 */ 7459 list_remove(l2arc_dev_list, remdev); 7460 l2arc_dev_last = NULL; /* may have been invalidated */ 7461 atomic_dec_64(&l2arc_ndev); 7462 mutex_exit(&l2arc_dev_mtx); 7463 7464 /* 7465 * Clear all buflists and ARC references. L2ARC device flush. 7466 */ 7467 l2arc_evict(remdev, 0, B_TRUE); 7468 list_destroy(&remdev->l2ad_buflist); 7469 mutex_destroy(&remdev->l2ad_mtx); 7470 zfs_refcount_destroy(&remdev->l2ad_alloc); 7471 kmem_free(remdev, sizeof (l2arc_dev_t)); 7472 } 7473 7474 void 7475 l2arc_init(void) 7476 { 7477 l2arc_thread_exit = 0; 7478 l2arc_ndev = 0; 7479 l2arc_writes_sent = 0; 7480 l2arc_writes_done = 0; 7481 7482 mutex_init(&l2arc_feed_thr_lock, NULL, MUTEX_DEFAULT, NULL); 7483 cv_init(&l2arc_feed_thr_cv, NULL, CV_DEFAULT, NULL); 7484 mutex_init(&l2arc_dev_mtx, NULL, MUTEX_DEFAULT, NULL); 7485 mutex_init(&l2arc_free_on_write_mtx, NULL, MUTEX_DEFAULT, NULL); 7486 7487 l2arc_dev_list = &L2ARC_dev_list; 7488 l2arc_free_on_write = &L2ARC_free_on_write; 7489 list_create(l2arc_dev_list, sizeof (l2arc_dev_t), 7490 offsetof(l2arc_dev_t, l2ad_node)); 7491 list_create(l2arc_free_on_write, sizeof (l2arc_data_free_t), 7492 offsetof(l2arc_data_free_t, l2df_list_node)); 7493 } 7494 7495 void 7496 l2arc_fini(void) 7497 { 7498 /* 7499 * This is called from dmu_fini(), which is called from spa_fini(); 7500 * Because of this, we can assume that all l2arc devices have 7501 * already been removed when the pools themselves were removed. 7502 */ 7503 7504 l2arc_do_free_on_write(); 7505 7506 mutex_destroy(&l2arc_feed_thr_lock); 7507 cv_destroy(&l2arc_feed_thr_cv); 7508 mutex_destroy(&l2arc_dev_mtx); 7509 mutex_destroy(&l2arc_free_on_write_mtx); 7510 7511 list_destroy(l2arc_dev_list); 7512 list_destroy(l2arc_free_on_write); 7513 } 7514 7515 void 7516 l2arc_start(void) 7517 { 7518 if (!(spa_mode_global & FWRITE)) 7519 return; 7520 7521 (void) thread_create(NULL, 0, l2arc_feed_thread, NULL, 0, &p0, 7522 TS_RUN, minclsyspri); 7523 } 7524 7525 void 7526 l2arc_stop(void) 7527 { 7528 if (!(spa_mode_global & FWRITE)) 7529 return; 7530 7531 mutex_enter(&l2arc_feed_thr_lock); 7532 cv_signal(&l2arc_feed_thr_cv); /* kick thread out of startup */ 7533 l2arc_thread_exit = 1; 7534 while (l2arc_thread_exit != 0) 7535 cv_wait(&l2arc_feed_thr_cv, &l2arc_feed_thr_lock); 7536 mutex_exit(&l2arc_feed_thr_lock); 7537 } 7538