xref: /illumos-gate/usr/src/uts/common/fs/zfs/arc.c (revision 144f99a5)
1 /*
2  * CDDL HEADER START
3  *
4  * The contents of this file are subject to the terms of the
5  * Common Development and Distribution License (the "License").
6  * You may not use this file except in compliance with the License.
7  *
8  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9  * or http://www.opensolaris.org/os/licensing.
10  * See the License for the specific language governing permissions
11  * and limitations under the License.
12  *
13  * When distributing Covered Code, include this CDDL HEADER in each
14  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15  * If applicable, add the following below this CDDL HEADER, with the
16  * fields enclosed by brackets "[]" replaced with your own identifying
17  * information: Portions Copyright [yyyy] [name of copyright owner]
18  *
19  * CDDL HEADER END
20  */
21 /*
22  * Copyright (c) 2005, 2010, Oracle and/or its affiliates. All rights reserved.
23  * Copyright (c) 2012, Joyent, Inc. All rights reserved.
24  * Copyright (c) 2011, 2014 by Delphix. All rights reserved.
25  * Copyright (c) 2014 by Saso Kiselkov. All rights reserved.
26  * Copyright 2014 Nexenta Systems, Inc.  All rights reserved.
27  */
28 
29 /*
30  * DVA-based Adjustable Replacement Cache
31  *
32  * While much of the theory of operation used here is
33  * based on the self-tuning, low overhead replacement cache
34  * presented by Megiddo and Modha at FAST 2003, there are some
35  * significant differences:
36  *
37  * 1. The Megiddo and Modha model assumes any page is evictable.
38  * Pages in its cache cannot be "locked" into memory.  This makes
39  * the eviction algorithm simple: evict the last page in the list.
40  * This also make the performance characteristics easy to reason
41  * about.  Our cache is not so simple.  At any given moment, some
42  * subset of the blocks in the cache are un-evictable because we
43  * have handed out a reference to them.  Blocks are only evictable
44  * when there are no external references active.  This makes
45  * eviction far more problematic:  we choose to evict the evictable
46  * blocks that are the "lowest" in the list.
47  *
48  * There are times when it is not possible to evict the requested
49  * space.  In these circumstances we are unable to adjust the cache
50  * size.  To prevent the cache growing unbounded at these times we
51  * implement a "cache throttle" that slows the flow of new data
52  * into the cache until we can make space available.
53  *
54  * 2. The Megiddo and Modha model assumes a fixed cache size.
55  * Pages are evicted when the cache is full and there is a cache
56  * miss.  Our model has a variable sized cache.  It grows with
57  * high use, but also tries to react to memory pressure from the
58  * operating system: decreasing its size when system memory is
59  * tight.
60  *
61  * 3. The Megiddo and Modha model assumes a fixed page size. All
62  * elements of the cache are therefore exactly the same size.  So
63  * when adjusting the cache size following a cache miss, its simply
64  * a matter of choosing a single page to evict.  In our model, we
65  * have variable sized cache blocks (rangeing from 512 bytes to
66  * 128K bytes).  We therefore choose a set of blocks to evict to make
67  * space for a cache miss that approximates as closely as possible
68  * the space used by the new block.
69  *
70  * See also:  "ARC: A Self-Tuning, Low Overhead Replacement Cache"
71  * by N. Megiddo & D. Modha, FAST 2003
72  */
73 
74 /*
75  * The locking model:
76  *
77  * A new reference to a cache buffer can be obtained in two
78  * ways: 1) via a hash table lookup using the DVA as a key,
79  * or 2) via one of the ARC lists.  The arc_read() interface
80  * uses method 1, while the internal arc algorithms for
81  * adjusting the cache use method 2.  We therefore provide two
82  * types of locks: 1) the hash table lock array, and 2) the
83  * arc list locks.
84  *
85  * Buffers do not have their own mutexes, rather they rely on the
86  * hash table mutexes for the bulk of their protection (i.e. most
87  * fields in the arc_buf_hdr_t are protected by these mutexes).
88  *
89  * buf_hash_find() returns the appropriate mutex (held) when it
90  * locates the requested buffer in the hash table.  It returns
91  * NULL for the mutex if the buffer was not in the table.
92  *
93  * buf_hash_remove() expects the appropriate hash mutex to be
94  * already held before it is invoked.
95  *
96  * Each arc state also has a mutex which is used to protect the
97  * buffer list associated with the state.  When attempting to
98  * obtain a hash table lock while holding an arc list lock you
99  * must use: mutex_tryenter() to avoid deadlock.  Also note that
100  * the active state mutex must be held before the ghost state mutex.
101  *
102  * Arc buffers may have an associated eviction callback function.
103  * This function will be invoked prior to removing the buffer (e.g.
104  * in arc_do_user_evicts()).  Note however that the data associated
105  * with the buffer may be evicted prior to the callback.  The callback
106  * must be made with *no locks held* (to prevent deadlock).  Additionally,
107  * the users of callbacks must ensure that their private data is
108  * protected from simultaneous callbacks from arc_clear_callback()
109  * and arc_do_user_evicts().
110  *
111  * Note that the majority of the performance stats are manipulated
112  * with atomic operations.
113  *
114  * The L2ARC uses the l2ad_mtx on each vdev for the following:
115  *
116  *	- L2ARC buflist creation
117  *	- L2ARC buflist eviction
118  *	- L2ARC write completion, which walks L2ARC buflists
119  *	- ARC header destruction, as it removes from L2ARC buflists
120  *	- ARC header release, as it removes from L2ARC buflists
121  */
122 
123 #include <sys/spa.h>
124 #include <sys/zio.h>
125 #include <sys/zio_compress.h>
126 #include <sys/zfs_context.h>
127 #include <sys/arc.h>
128 #include <sys/refcount.h>
129 #include <sys/vdev.h>
130 #include <sys/vdev_impl.h>
131 #include <sys/dsl_pool.h>
132 #include <sys/multilist.h>
133 #ifdef _KERNEL
134 #include <sys/vmsystm.h>
135 #include <vm/anon.h>
136 #include <sys/fs/swapnode.h>
137 #include <sys/dnlc.h>
138 #endif
139 #include <sys/callb.h>
140 #include <sys/kstat.h>
141 #include <zfs_fletcher.h>
142 
143 #ifndef _KERNEL
144 /* set with ZFS_DEBUG=watch, to enable watchpoints on frozen buffers */
145 boolean_t arc_watch = B_FALSE;
146 int arc_procfd;
147 #endif
148 
149 static kmutex_t		arc_reclaim_lock;
150 static kcondvar_t	arc_reclaim_thread_cv;
151 static boolean_t	arc_reclaim_thread_exit;
152 static kcondvar_t	arc_reclaim_waiters_cv;
153 
154 static kmutex_t		arc_user_evicts_lock;
155 static kcondvar_t	arc_user_evicts_cv;
156 static boolean_t	arc_user_evicts_thread_exit;
157 
158 uint_t arc_reduce_dnlc_percent = 3;
159 
160 /*
161  * The number of headers to evict in arc_evict_state_impl() before
162  * dropping the sublist lock and evicting from another sublist. A lower
163  * value means we're more likely to evict the "correct" header (i.e. the
164  * oldest header in the arc state), but comes with higher overhead
165  * (i.e. more invocations of arc_evict_state_impl()).
166  */
167 int zfs_arc_evict_batch_limit = 10;
168 
169 /*
170  * The number of sublists used for each of the arc state lists. If this
171  * is not set to a suitable value by the user, it will be configured to
172  * the number of CPUs on the system in arc_init().
173  */
174 int zfs_arc_num_sublists_per_state = 0;
175 
176 /* number of seconds before growing cache again */
177 static int		arc_grow_retry = 60;
178 
179 /* shift of arc_c for calculating overflow limit in arc_get_data_buf */
180 int		zfs_arc_overflow_shift = 8;
181 
182 /* shift of arc_c for calculating both min and max arc_p */
183 static int		arc_p_min_shift = 4;
184 
185 /* log2(fraction of arc to reclaim) */
186 static int		arc_shrink_shift = 7;
187 
188 /*
189  * log2(fraction of ARC which must be free to allow growing).
190  * I.e. If there is less than arc_c >> arc_no_grow_shift free memory,
191  * when reading a new block into the ARC, we will evict an equal-sized block
192  * from the ARC.
193  *
194  * This must be less than arc_shrink_shift, so that when we shrink the ARC,
195  * we will still not allow it to grow.
196  */
197 int			arc_no_grow_shift = 5;
198 
199 
200 /*
201  * minimum lifespan of a prefetch block in clock ticks
202  * (initialized in arc_init())
203  */
204 static int		arc_min_prefetch_lifespan;
205 
206 /*
207  * If this percent of memory is free, don't throttle.
208  */
209 int arc_lotsfree_percent = 10;
210 
211 static int arc_dead;
212 
213 /*
214  * The arc has filled available memory and has now warmed up.
215  */
216 static boolean_t arc_warm;
217 
218 /*
219  * These tunables are for performance analysis.
220  */
221 uint64_t zfs_arc_max;
222 uint64_t zfs_arc_min;
223 uint64_t zfs_arc_meta_limit = 0;
224 uint64_t zfs_arc_meta_min = 0;
225 int zfs_arc_grow_retry = 0;
226 int zfs_arc_shrink_shift = 0;
227 int zfs_arc_p_min_shift = 0;
228 int zfs_disable_dup_eviction = 0;
229 int zfs_arc_average_blocksize = 8 * 1024; /* 8KB */
230 
231 /*
232  * Note that buffers can be in one of 6 states:
233  *	ARC_anon	- anonymous (discussed below)
234  *	ARC_mru		- recently used, currently cached
235  *	ARC_mru_ghost	- recentely used, no longer in cache
236  *	ARC_mfu		- frequently used, currently cached
237  *	ARC_mfu_ghost	- frequently used, no longer in cache
238  *	ARC_l2c_only	- exists in L2ARC but not other states
239  * When there are no active references to the buffer, they are
240  * are linked onto a list in one of these arc states.  These are
241  * the only buffers that can be evicted or deleted.  Within each
242  * state there are multiple lists, one for meta-data and one for
243  * non-meta-data.  Meta-data (indirect blocks, blocks of dnodes,
244  * etc.) is tracked separately so that it can be managed more
245  * explicitly: favored over data, limited explicitly.
246  *
247  * Anonymous buffers are buffers that are not associated with
248  * a DVA.  These are buffers that hold dirty block copies
249  * before they are written to stable storage.  By definition,
250  * they are "ref'd" and are considered part of arc_mru
251  * that cannot be freed.  Generally, they will aquire a DVA
252  * as they are written and migrate onto the arc_mru list.
253  *
254  * The ARC_l2c_only state is for buffers that are in the second
255  * level ARC but no longer in any of the ARC_m* lists.  The second
256  * level ARC itself may also contain buffers that are in any of
257  * the ARC_m* states - meaning that a buffer can exist in two
258  * places.  The reason for the ARC_l2c_only state is to keep the
259  * buffer header in the hash table, so that reads that hit the
260  * second level ARC benefit from these fast lookups.
261  */
262 
263 typedef struct arc_state {
264 	/*
265 	 * list of evictable buffers
266 	 */
267 	multilist_t arcs_list[ARC_BUFC_NUMTYPES];
268 	/*
269 	 * total amount of evictable data in this state
270 	 */
271 	uint64_t arcs_lsize[ARC_BUFC_NUMTYPES];
272 	/*
273 	 * total amount of data in this state; this includes: evictable,
274 	 * non-evictable, ARC_BUFC_DATA, and ARC_BUFC_METADATA.
275 	 */
276 	uint64_t arcs_size;
277 } arc_state_t;
278 
279 /* The 6 states: */
280 static arc_state_t ARC_anon;
281 static arc_state_t ARC_mru;
282 static arc_state_t ARC_mru_ghost;
283 static arc_state_t ARC_mfu;
284 static arc_state_t ARC_mfu_ghost;
285 static arc_state_t ARC_l2c_only;
286 
287 typedef struct arc_stats {
288 	kstat_named_t arcstat_hits;
289 	kstat_named_t arcstat_misses;
290 	kstat_named_t arcstat_demand_data_hits;
291 	kstat_named_t arcstat_demand_data_misses;
292 	kstat_named_t arcstat_demand_metadata_hits;
293 	kstat_named_t arcstat_demand_metadata_misses;
294 	kstat_named_t arcstat_prefetch_data_hits;
295 	kstat_named_t arcstat_prefetch_data_misses;
296 	kstat_named_t arcstat_prefetch_metadata_hits;
297 	kstat_named_t arcstat_prefetch_metadata_misses;
298 	kstat_named_t arcstat_mru_hits;
299 	kstat_named_t arcstat_mru_ghost_hits;
300 	kstat_named_t arcstat_mfu_hits;
301 	kstat_named_t arcstat_mfu_ghost_hits;
302 	kstat_named_t arcstat_deleted;
303 	/*
304 	 * Number of buffers that could not be evicted because the hash lock
305 	 * was held by another thread.  The lock may not necessarily be held
306 	 * by something using the same buffer, since hash locks are shared
307 	 * by multiple buffers.
308 	 */
309 	kstat_named_t arcstat_mutex_miss;
310 	/*
311 	 * Number of buffers skipped because they have I/O in progress, are
312 	 * indrect prefetch buffers that have not lived long enough, or are
313 	 * not from the spa we're trying to evict from.
314 	 */
315 	kstat_named_t arcstat_evict_skip;
316 	/*
317 	 * Number of times arc_evict_state() was unable to evict enough
318 	 * buffers to reach it's target amount.
319 	 */
320 	kstat_named_t arcstat_evict_not_enough;
321 	kstat_named_t arcstat_evict_l2_cached;
322 	kstat_named_t arcstat_evict_l2_eligible;
323 	kstat_named_t arcstat_evict_l2_ineligible;
324 	kstat_named_t arcstat_evict_l2_skip;
325 	kstat_named_t arcstat_hash_elements;
326 	kstat_named_t arcstat_hash_elements_max;
327 	kstat_named_t arcstat_hash_collisions;
328 	kstat_named_t arcstat_hash_chains;
329 	kstat_named_t arcstat_hash_chain_max;
330 	kstat_named_t arcstat_p;
331 	kstat_named_t arcstat_c;
332 	kstat_named_t arcstat_c_min;
333 	kstat_named_t arcstat_c_max;
334 	kstat_named_t arcstat_size;
335 	/*
336 	 * Number of bytes consumed by internal ARC structures necessary
337 	 * for tracking purposes; these structures are not actually
338 	 * backed by ARC buffers. This includes arc_buf_hdr_t structures
339 	 * (allocated via arc_buf_hdr_t_full and arc_buf_hdr_t_l2only
340 	 * caches), and arc_buf_t structures (allocated via arc_buf_t
341 	 * cache).
342 	 */
343 	kstat_named_t arcstat_hdr_size;
344 	/*
345 	 * Number of bytes consumed by ARC buffers of type equal to
346 	 * ARC_BUFC_DATA. This is generally consumed by buffers backing
347 	 * on disk user data (e.g. plain file contents).
348 	 */
349 	kstat_named_t arcstat_data_size;
350 	/*
351 	 * Number of bytes consumed by ARC buffers of type equal to
352 	 * ARC_BUFC_METADATA. This is generally consumed by buffers
353 	 * backing on disk data that is used for internal ZFS
354 	 * structures (e.g. ZAP, dnode, indirect blocks, etc).
355 	 */
356 	kstat_named_t arcstat_metadata_size;
357 	/*
358 	 * Number of bytes consumed by various buffers and structures
359 	 * not actually backed with ARC buffers. This includes bonus
360 	 * buffers (allocated directly via zio_buf_* functions),
361 	 * dmu_buf_impl_t structures (allocated via dmu_buf_impl_t
362 	 * cache), and dnode_t structures (allocated via dnode_t cache).
363 	 */
364 	kstat_named_t arcstat_other_size;
365 	/*
366 	 * Total number of bytes consumed by ARC buffers residing in the
367 	 * arc_anon state. This includes *all* buffers in the arc_anon
368 	 * state; e.g. data, metadata, evictable, and unevictable buffers
369 	 * are all included in this value.
370 	 */
371 	kstat_named_t arcstat_anon_size;
372 	/*
373 	 * Number of bytes consumed by ARC buffers that meet the
374 	 * following criteria: backing buffers of type ARC_BUFC_DATA,
375 	 * residing in the arc_anon state, and are eligible for eviction
376 	 * (e.g. have no outstanding holds on the buffer).
377 	 */
378 	kstat_named_t arcstat_anon_evictable_data;
379 	/*
380 	 * Number of bytes consumed by ARC buffers that meet the
381 	 * following criteria: backing buffers of type ARC_BUFC_METADATA,
382 	 * residing in the arc_anon state, and are eligible for eviction
383 	 * (e.g. have no outstanding holds on the buffer).
384 	 */
385 	kstat_named_t arcstat_anon_evictable_metadata;
386 	/*
387 	 * Total number of bytes consumed by ARC buffers residing in the
388 	 * arc_mru state. This includes *all* buffers in the arc_mru
389 	 * state; e.g. data, metadata, evictable, and unevictable buffers
390 	 * are all included in this value.
391 	 */
392 	kstat_named_t arcstat_mru_size;
393 	/*
394 	 * Number of bytes consumed by ARC buffers that meet the
395 	 * following criteria: backing buffers of type ARC_BUFC_DATA,
396 	 * residing in the arc_mru state, and are eligible for eviction
397 	 * (e.g. have no outstanding holds on the buffer).
398 	 */
399 	kstat_named_t arcstat_mru_evictable_data;
400 	/*
401 	 * Number of bytes consumed by ARC buffers that meet the
402 	 * following criteria: backing buffers of type ARC_BUFC_METADATA,
403 	 * residing in the arc_mru state, and are eligible for eviction
404 	 * (e.g. have no outstanding holds on the buffer).
405 	 */
406 	kstat_named_t arcstat_mru_evictable_metadata;
407 	/*
408 	 * Total number of bytes that *would have been* consumed by ARC
409 	 * buffers in the arc_mru_ghost state. The key thing to note
410 	 * here, is the fact that this size doesn't actually indicate
411 	 * RAM consumption. The ghost lists only consist of headers and
412 	 * don't actually have ARC buffers linked off of these headers.
413 	 * Thus, *if* the headers had associated ARC buffers, these
414 	 * buffers *would have* consumed this number of bytes.
415 	 */
416 	kstat_named_t arcstat_mru_ghost_size;
417 	/*
418 	 * Number of bytes that *would have been* consumed by ARC
419 	 * buffers that are eligible for eviction, of type
420 	 * ARC_BUFC_DATA, and linked off the arc_mru_ghost state.
421 	 */
422 	kstat_named_t arcstat_mru_ghost_evictable_data;
423 	/*
424 	 * Number of bytes that *would have been* consumed by ARC
425 	 * buffers that are eligible for eviction, of type
426 	 * ARC_BUFC_METADATA, and linked off the arc_mru_ghost state.
427 	 */
428 	kstat_named_t arcstat_mru_ghost_evictable_metadata;
429 	/*
430 	 * Total number of bytes consumed by ARC buffers residing in the
431 	 * arc_mfu state. This includes *all* buffers in the arc_mfu
432 	 * state; e.g. data, metadata, evictable, and unevictable buffers
433 	 * are all included in this value.
434 	 */
435 	kstat_named_t arcstat_mfu_size;
436 	/*
437 	 * Number of bytes consumed by ARC buffers that are eligible for
438 	 * eviction, of type ARC_BUFC_DATA, and reside in the arc_mfu
439 	 * state.
440 	 */
441 	kstat_named_t arcstat_mfu_evictable_data;
442 	/*
443 	 * Number of bytes consumed by ARC buffers that are eligible for
444 	 * eviction, of type ARC_BUFC_METADATA, and reside in the
445 	 * arc_mfu state.
446 	 */
447 	kstat_named_t arcstat_mfu_evictable_metadata;
448 	/*
449 	 * Total number of bytes that *would have been* consumed by ARC
450 	 * buffers in the arc_mfu_ghost state. See the comment above
451 	 * arcstat_mru_ghost_size for more details.
452 	 */
453 	kstat_named_t arcstat_mfu_ghost_size;
454 	/*
455 	 * Number of bytes that *would have been* consumed by ARC
456 	 * buffers that are eligible for eviction, of type
457 	 * ARC_BUFC_DATA, and linked off the arc_mfu_ghost state.
458 	 */
459 	kstat_named_t arcstat_mfu_ghost_evictable_data;
460 	/*
461 	 * Number of bytes that *would have been* consumed by ARC
462 	 * buffers that are eligible for eviction, of type
463 	 * ARC_BUFC_METADATA, and linked off the arc_mru_ghost state.
464 	 */
465 	kstat_named_t arcstat_mfu_ghost_evictable_metadata;
466 	kstat_named_t arcstat_l2_hits;
467 	kstat_named_t arcstat_l2_misses;
468 	kstat_named_t arcstat_l2_feeds;
469 	kstat_named_t arcstat_l2_rw_clash;
470 	kstat_named_t arcstat_l2_read_bytes;
471 	kstat_named_t arcstat_l2_write_bytes;
472 	kstat_named_t arcstat_l2_writes_sent;
473 	kstat_named_t arcstat_l2_writes_done;
474 	kstat_named_t arcstat_l2_writes_error;
475 	kstat_named_t arcstat_l2_writes_lock_retry;
476 	kstat_named_t arcstat_l2_evict_lock_retry;
477 	kstat_named_t arcstat_l2_evict_reading;
478 	kstat_named_t arcstat_l2_evict_l1cached;
479 	kstat_named_t arcstat_l2_free_on_write;
480 	kstat_named_t arcstat_l2_cdata_free_on_write;
481 	kstat_named_t arcstat_l2_abort_lowmem;
482 	kstat_named_t arcstat_l2_cksum_bad;
483 	kstat_named_t arcstat_l2_io_error;
484 	kstat_named_t arcstat_l2_size;
485 	kstat_named_t arcstat_l2_asize;
486 	kstat_named_t arcstat_l2_hdr_size;
487 	kstat_named_t arcstat_l2_compress_successes;
488 	kstat_named_t arcstat_l2_compress_zeros;
489 	kstat_named_t arcstat_l2_compress_failures;
490 	kstat_named_t arcstat_memory_throttle_count;
491 	kstat_named_t arcstat_duplicate_buffers;
492 	kstat_named_t arcstat_duplicate_buffers_size;
493 	kstat_named_t arcstat_duplicate_reads;
494 	kstat_named_t arcstat_meta_used;
495 	kstat_named_t arcstat_meta_limit;
496 	kstat_named_t arcstat_meta_max;
497 	kstat_named_t arcstat_meta_min;
498 } arc_stats_t;
499 
500 static arc_stats_t arc_stats = {
501 	{ "hits",			KSTAT_DATA_UINT64 },
502 	{ "misses",			KSTAT_DATA_UINT64 },
503 	{ "demand_data_hits",		KSTAT_DATA_UINT64 },
504 	{ "demand_data_misses",		KSTAT_DATA_UINT64 },
505 	{ "demand_metadata_hits",	KSTAT_DATA_UINT64 },
506 	{ "demand_metadata_misses",	KSTAT_DATA_UINT64 },
507 	{ "prefetch_data_hits",		KSTAT_DATA_UINT64 },
508 	{ "prefetch_data_misses",	KSTAT_DATA_UINT64 },
509 	{ "prefetch_metadata_hits",	KSTAT_DATA_UINT64 },
510 	{ "prefetch_metadata_misses",	KSTAT_DATA_UINT64 },
511 	{ "mru_hits",			KSTAT_DATA_UINT64 },
512 	{ "mru_ghost_hits",		KSTAT_DATA_UINT64 },
513 	{ "mfu_hits",			KSTAT_DATA_UINT64 },
514 	{ "mfu_ghost_hits",		KSTAT_DATA_UINT64 },
515 	{ "deleted",			KSTAT_DATA_UINT64 },
516 	{ "mutex_miss",			KSTAT_DATA_UINT64 },
517 	{ "evict_skip",			KSTAT_DATA_UINT64 },
518 	{ "evict_not_enough",		KSTAT_DATA_UINT64 },
519 	{ "evict_l2_cached",		KSTAT_DATA_UINT64 },
520 	{ "evict_l2_eligible",		KSTAT_DATA_UINT64 },
521 	{ "evict_l2_ineligible",	KSTAT_DATA_UINT64 },
522 	{ "evict_l2_skip",		KSTAT_DATA_UINT64 },
523 	{ "hash_elements",		KSTAT_DATA_UINT64 },
524 	{ "hash_elements_max",		KSTAT_DATA_UINT64 },
525 	{ "hash_collisions",		KSTAT_DATA_UINT64 },
526 	{ "hash_chains",		KSTAT_DATA_UINT64 },
527 	{ "hash_chain_max",		KSTAT_DATA_UINT64 },
528 	{ "p",				KSTAT_DATA_UINT64 },
529 	{ "c",				KSTAT_DATA_UINT64 },
530 	{ "c_min",			KSTAT_DATA_UINT64 },
531 	{ "c_max",			KSTAT_DATA_UINT64 },
532 	{ "size",			KSTAT_DATA_UINT64 },
533 	{ "hdr_size",			KSTAT_DATA_UINT64 },
534 	{ "data_size",			KSTAT_DATA_UINT64 },
535 	{ "metadata_size",		KSTAT_DATA_UINT64 },
536 	{ "other_size",			KSTAT_DATA_UINT64 },
537 	{ "anon_size",			KSTAT_DATA_UINT64 },
538 	{ "anon_evictable_data",	KSTAT_DATA_UINT64 },
539 	{ "anon_evictable_metadata",	KSTAT_DATA_UINT64 },
540 	{ "mru_size",			KSTAT_DATA_UINT64 },
541 	{ "mru_evictable_data",		KSTAT_DATA_UINT64 },
542 	{ "mru_evictable_metadata",	KSTAT_DATA_UINT64 },
543 	{ "mru_ghost_size",		KSTAT_DATA_UINT64 },
544 	{ "mru_ghost_evictable_data",	KSTAT_DATA_UINT64 },
545 	{ "mru_ghost_evictable_metadata", KSTAT_DATA_UINT64 },
546 	{ "mfu_size",			KSTAT_DATA_UINT64 },
547 	{ "mfu_evictable_data",		KSTAT_DATA_UINT64 },
548 	{ "mfu_evictable_metadata",	KSTAT_DATA_UINT64 },
549 	{ "mfu_ghost_size",		KSTAT_DATA_UINT64 },
550 	{ "mfu_ghost_evictable_data",	KSTAT_DATA_UINT64 },
551 	{ "mfu_ghost_evictable_metadata", KSTAT_DATA_UINT64 },
552 	{ "l2_hits",			KSTAT_DATA_UINT64 },
553 	{ "l2_misses",			KSTAT_DATA_UINT64 },
554 	{ "l2_feeds",			KSTAT_DATA_UINT64 },
555 	{ "l2_rw_clash",		KSTAT_DATA_UINT64 },
556 	{ "l2_read_bytes",		KSTAT_DATA_UINT64 },
557 	{ "l2_write_bytes",		KSTAT_DATA_UINT64 },
558 	{ "l2_writes_sent",		KSTAT_DATA_UINT64 },
559 	{ "l2_writes_done",		KSTAT_DATA_UINT64 },
560 	{ "l2_writes_error",		KSTAT_DATA_UINT64 },
561 	{ "l2_writes_lock_retry",	KSTAT_DATA_UINT64 },
562 	{ "l2_evict_lock_retry",	KSTAT_DATA_UINT64 },
563 	{ "l2_evict_reading",		KSTAT_DATA_UINT64 },
564 	{ "l2_evict_l1cached",		KSTAT_DATA_UINT64 },
565 	{ "l2_free_on_write",		KSTAT_DATA_UINT64 },
566 	{ "l2_cdata_free_on_write",	KSTAT_DATA_UINT64 },
567 	{ "l2_abort_lowmem",		KSTAT_DATA_UINT64 },
568 	{ "l2_cksum_bad",		KSTAT_DATA_UINT64 },
569 	{ "l2_io_error",		KSTAT_DATA_UINT64 },
570 	{ "l2_size",			KSTAT_DATA_UINT64 },
571 	{ "l2_asize",			KSTAT_DATA_UINT64 },
572 	{ "l2_hdr_size",		KSTAT_DATA_UINT64 },
573 	{ "l2_compress_successes",	KSTAT_DATA_UINT64 },
574 	{ "l2_compress_zeros",		KSTAT_DATA_UINT64 },
575 	{ "l2_compress_failures",	KSTAT_DATA_UINT64 },
576 	{ "memory_throttle_count",	KSTAT_DATA_UINT64 },
577 	{ "duplicate_buffers",		KSTAT_DATA_UINT64 },
578 	{ "duplicate_buffers_size",	KSTAT_DATA_UINT64 },
579 	{ "duplicate_reads",		KSTAT_DATA_UINT64 },
580 	{ "arc_meta_used",		KSTAT_DATA_UINT64 },
581 	{ "arc_meta_limit",		KSTAT_DATA_UINT64 },
582 	{ "arc_meta_max",		KSTAT_DATA_UINT64 },
583 	{ "arc_meta_min",		KSTAT_DATA_UINT64 }
584 };
585 
586 #define	ARCSTAT(stat)	(arc_stats.stat.value.ui64)
587 
588 #define	ARCSTAT_INCR(stat, val) \
589 	atomic_add_64(&arc_stats.stat.value.ui64, (val))
590 
591 #define	ARCSTAT_BUMP(stat)	ARCSTAT_INCR(stat, 1)
592 #define	ARCSTAT_BUMPDOWN(stat)	ARCSTAT_INCR(stat, -1)
593 
594 #define	ARCSTAT_MAX(stat, val) {					\
595 	uint64_t m;							\
596 	while ((val) > (m = arc_stats.stat.value.ui64) &&		\
597 	    (m != atomic_cas_64(&arc_stats.stat.value.ui64, m, (val))))	\
598 		continue;						\
599 }
600 
601 #define	ARCSTAT_MAXSTAT(stat) \
602 	ARCSTAT_MAX(stat##_max, arc_stats.stat.value.ui64)
603 
604 /*
605  * We define a macro to allow ARC hits/misses to be easily broken down by
606  * two separate conditions, giving a total of four different subtypes for
607  * each of hits and misses (so eight statistics total).
608  */
609 #define	ARCSTAT_CONDSTAT(cond1, stat1, notstat1, cond2, stat2, notstat2, stat) \
610 	if (cond1) {							\
611 		if (cond2) {						\
612 			ARCSTAT_BUMP(arcstat_##stat1##_##stat2##_##stat); \
613 		} else {						\
614 			ARCSTAT_BUMP(arcstat_##stat1##_##notstat2##_##stat); \
615 		}							\
616 	} else {							\
617 		if (cond2) {						\
618 			ARCSTAT_BUMP(arcstat_##notstat1##_##stat2##_##stat); \
619 		} else {						\
620 			ARCSTAT_BUMP(arcstat_##notstat1##_##notstat2##_##stat);\
621 		}							\
622 	}
623 
624 kstat_t			*arc_ksp;
625 static arc_state_t	*arc_anon;
626 static arc_state_t	*arc_mru;
627 static arc_state_t	*arc_mru_ghost;
628 static arc_state_t	*arc_mfu;
629 static arc_state_t	*arc_mfu_ghost;
630 static arc_state_t	*arc_l2c_only;
631 
632 /*
633  * There are several ARC variables that are critical to export as kstats --
634  * but we don't want to have to grovel around in the kstat whenever we wish to
635  * manipulate them.  For these variables, we therefore define them to be in
636  * terms of the statistic variable.  This assures that we are not introducing
637  * the possibility of inconsistency by having shadow copies of the variables,
638  * while still allowing the code to be readable.
639  */
640 #define	arc_size	ARCSTAT(arcstat_size)	/* actual total arc size */
641 #define	arc_p		ARCSTAT(arcstat_p)	/* target size of MRU */
642 #define	arc_c		ARCSTAT(arcstat_c)	/* target size of cache */
643 #define	arc_c_min	ARCSTAT(arcstat_c_min)	/* min target cache size */
644 #define	arc_c_max	ARCSTAT(arcstat_c_max)	/* max target cache size */
645 #define	arc_meta_limit	ARCSTAT(arcstat_meta_limit) /* max size for metadata */
646 #define	arc_meta_min	ARCSTAT(arcstat_meta_min) /* min size for metadata */
647 #define	arc_meta_used	ARCSTAT(arcstat_meta_used) /* size of metadata */
648 #define	arc_meta_max	ARCSTAT(arcstat_meta_max) /* max size of metadata */
649 
650 #define	L2ARC_IS_VALID_COMPRESS(_c_) \
651 	((_c_) == ZIO_COMPRESS_LZ4 || (_c_) == ZIO_COMPRESS_EMPTY)
652 
653 static int		arc_no_grow;	/* Don't try to grow cache size */
654 static uint64_t		arc_tempreserve;
655 static uint64_t		arc_loaned_bytes;
656 
657 typedef struct arc_callback arc_callback_t;
658 
659 struct arc_callback {
660 	void			*acb_private;
661 	arc_done_func_t		*acb_done;
662 	arc_buf_t		*acb_buf;
663 	zio_t			*acb_zio_dummy;
664 	arc_callback_t		*acb_next;
665 };
666 
667 typedef struct arc_write_callback arc_write_callback_t;
668 
669 struct arc_write_callback {
670 	void		*awcb_private;
671 	arc_done_func_t	*awcb_ready;
672 	arc_done_func_t	*awcb_physdone;
673 	arc_done_func_t	*awcb_done;
674 	arc_buf_t	*awcb_buf;
675 };
676 
677 /*
678  * ARC buffers are separated into multiple structs as a memory saving measure:
679  *   - Common fields struct, always defined, and embedded within it:
680  *       - L2-only fields, always allocated but undefined when not in L2ARC
681  *       - L1-only fields, only allocated when in L1ARC
682  *
683  *           Buffer in L1                     Buffer only in L2
684  *    +------------------------+          +------------------------+
685  *    | arc_buf_hdr_t          |          | arc_buf_hdr_t          |
686  *    |                        |          |                        |
687  *    |                        |          |                        |
688  *    |                        |          |                        |
689  *    +------------------------+          +------------------------+
690  *    | l2arc_buf_hdr_t        |          | l2arc_buf_hdr_t        |
691  *    | (undefined if L1-only) |          |                        |
692  *    +------------------------+          +------------------------+
693  *    | l1arc_buf_hdr_t        |
694  *    |                        |
695  *    |                        |
696  *    |                        |
697  *    |                        |
698  *    +------------------------+
699  *
700  * Because it's possible for the L2ARC to become extremely large, we can wind
701  * up eating a lot of memory in L2ARC buffer headers, so the size of a header
702  * is minimized by only allocating the fields necessary for an L1-cached buffer
703  * when a header is actually in the L1 cache. The sub-headers (l1arc_buf_hdr and
704  * l2arc_buf_hdr) are embedded rather than allocated separately to save a couple
705  * words in pointers. arc_hdr_realloc() is used to switch a header between
706  * these two allocation states.
707  */
708 typedef struct l1arc_buf_hdr {
709 	kmutex_t		b_freeze_lock;
710 #ifdef ZFS_DEBUG
711 	/*
712 	 * used for debugging wtih kmem_flags - by allocating and freeing
713 	 * b_thawed when the buffer is thawed, we get a record of the stack
714 	 * trace that thawed it.
715 	 */
716 	void			*b_thawed;
717 #endif
718 
719 	arc_buf_t		*b_buf;
720 	uint32_t		b_datacnt;
721 	/* for waiting on writes to complete */
722 	kcondvar_t		b_cv;
723 
724 	/* protected by arc state mutex */
725 	arc_state_t		*b_state;
726 	multilist_node_t	b_arc_node;
727 
728 	/* updated atomically */
729 	clock_t			b_arc_access;
730 
731 	/* self protecting */
732 	refcount_t		b_refcnt;
733 
734 	arc_callback_t		*b_acb;
735 	/* temporary buffer holder for in-flight compressed data */
736 	void			*b_tmp_cdata;
737 } l1arc_buf_hdr_t;
738 
739 typedef struct l2arc_dev l2arc_dev_t;
740 
741 typedef struct l2arc_buf_hdr {
742 	/* protected by arc_buf_hdr mutex */
743 	l2arc_dev_t		*b_dev;		/* L2ARC device */
744 	uint64_t		b_daddr;	/* disk address, offset byte */
745 	/* real alloc'd buffer size depending on b_compress applied */
746 	int32_t			b_asize;
747 
748 	list_node_t		b_l2node;
749 } l2arc_buf_hdr_t;
750 
751 struct arc_buf_hdr {
752 	/* protected by hash lock */
753 	dva_t			b_dva;
754 	uint64_t		b_birth;
755 	/*
756 	 * Even though this checksum is only set/verified when a buffer is in
757 	 * the L1 cache, it needs to be in the set of common fields because it
758 	 * must be preserved from the time before a buffer is written out to
759 	 * L2ARC until after it is read back in.
760 	 */
761 	zio_cksum_t		*b_freeze_cksum;
762 
763 	arc_buf_hdr_t		*b_hash_next;
764 	arc_flags_t		b_flags;
765 
766 	/* immutable */
767 	int32_t			b_size;
768 	uint64_t		b_spa;
769 
770 	/* L2ARC fields. Undefined when not in L2ARC. */
771 	l2arc_buf_hdr_t		b_l2hdr;
772 	/* L1ARC fields. Undefined when in l2arc_only state */
773 	l1arc_buf_hdr_t		b_l1hdr;
774 };
775 
776 static arc_buf_t *arc_eviction_list;
777 static arc_buf_hdr_t arc_eviction_hdr;
778 
779 #define	GHOST_STATE(state)	\
780 	((state) == arc_mru_ghost || (state) == arc_mfu_ghost ||	\
781 	(state) == arc_l2c_only)
782 
783 #define	HDR_IN_HASH_TABLE(hdr)	((hdr)->b_flags & ARC_FLAG_IN_HASH_TABLE)
784 #define	HDR_IO_IN_PROGRESS(hdr)	((hdr)->b_flags & ARC_FLAG_IO_IN_PROGRESS)
785 #define	HDR_IO_ERROR(hdr)	((hdr)->b_flags & ARC_FLAG_IO_ERROR)
786 #define	HDR_PREFETCH(hdr)	((hdr)->b_flags & ARC_FLAG_PREFETCH)
787 #define	HDR_FREED_IN_READ(hdr)	((hdr)->b_flags & ARC_FLAG_FREED_IN_READ)
788 #define	HDR_BUF_AVAILABLE(hdr)	((hdr)->b_flags & ARC_FLAG_BUF_AVAILABLE)
789 
790 #define	HDR_L2CACHE(hdr)	((hdr)->b_flags & ARC_FLAG_L2CACHE)
791 #define	HDR_L2COMPRESS(hdr)	((hdr)->b_flags & ARC_FLAG_L2COMPRESS)
792 #define	HDR_L2_READING(hdr)	\
793 	    (((hdr)->b_flags & ARC_FLAG_IO_IN_PROGRESS) &&	\
794 	    ((hdr)->b_flags & ARC_FLAG_HAS_L2HDR))
795 #define	HDR_L2_WRITING(hdr)	((hdr)->b_flags & ARC_FLAG_L2_WRITING)
796 #define	HDR_L2_EVICTED(hdr)	((hdr)->b_flags & ARC_FLAG_L2_EVICTED)
797 #define	HDR_L2_WRITE_HEAD(hdr)	((hdr)->b_flags & ARC_FLAG_L2_WRITE_HEAD)
798 
799 #define	HDR_ISTYPE_METADATA(hdr)	\
800 	    ((hdr)->b_flags & ARC_FLAG_BUFC_METADATA)
801 #define	HDR_ISTYPE_DATA(hdr)	(!HDR_ISTYPE_METADATA(hdr))
802 
803 #define	HDR_HAS_L1HDR(hdr)	((hdr)->b_flags & ARC_FLAG_HAS_L1HDR)
804 #define	HDR_HAS_L2HDR(hdr)	((hdr)->b_flags & ARC_FLAG_HAS_L2HDR)
805 
806 /* For storing compression mode in b_flags */
807 #define	HDR_COMPRESS_OFFSET	24
808 #define	HDR_COMPRESS_NBITS	7
809 
810 #define	HDR_GET_COMPRESS(hdr)	((enum zio_compress)BF32_GET(hdr->b_flags, \
811 	    HDR_COMPRESS_OFFSET, HDR_COMPRESS_NBITS))
812 #define	HDR_SET_COMPRESS(hdr, cmp) BF32_SET(hdr->b_flags, \
813 	    HDR_COMPRESS_OFFSET, HDR_COMPRESS_NBITS, (cmp))
814 
815 /*
816  * Other sizes
817  */
818 
819 #define	HDR_FULL_SIZE ((int64_t)sizeof (arc_buf_hdr_t))
820 #define	HDR_L2ONLY_SIZE ((int64_t)offsetof(arc_buf_hdr_t, b_l1hdr))
821 
822 /*
823  * Hash table routines
824  */
825 
826 #define	HT_LOCK_PAD	64
827 
828 struct ht_lock {
829 	kmutex_t	ht_lock;
830 #ifdef _KERNEL
831 	unsigned char	pad[(HT_LOCK_PAD - sizeof (kmutex_t))];
832 #endif
833 };
834 
835 #define	BUF_LOCKS 256
836 typedef struct buf_hash_table {
837 	uint64_t ht_mask;
838 	arc_buf_hdr_t **ht_table;
839 	struct ht_lock ht_locks[BUF_LOCKS];
840 } buf_hash_table_t;
841 
842 static buf_hash_table_t buf_hash_table;
843 
844 #define	BUF_HASH_INDEX(spa, dva, birth) \
845 	(buf_hash(spa, dva, birth) & buf_hash_table.ht_mask)
846 #define	BUF_HASH_LOCK_NTRY(idx) (buf_hash_table.ht_locks[idx & (BUF_LOCKS-1)])
847 #define	BUF_HASH_LOCK(idx)	(&(BUF_HASH_LOCK_NTRY(idx).ht_lock))
848 #define	HDR_LOCK(hdr) \
849 	(BUF_HASH_LOCK(BUF_HASH_INDEX(hdr->b_spa, &hdr->b_dva, hdr->b_birth)))
850 
851 uint64_t zfs_crc64_table[256];
852 
853 /*
854  * Level 2 ARC
855  */
856 
857 #define	L2ARC_WRITE_SIZE	(8 * 1024 * 1024)	/* initial write max */
858 #define	L2ARC_HEADROOM		2			/* num of writes */
859 /*
860  * If we discover during ARC scan any buffers to be compressed, we boost
861  * our headroom for the next scanning cycle by this percentage multiple.
862  */
863 #define	L2ARC_HEADROOM_BOOST	200
864 #define	L2ARC_FEED_SECS		1		/* caching interval secs */
865 #define	L2ARC_FEED_MIN_MS	200		/* min caching interval ms */
866 
867 #define	l2arc_writes_sent	ARCSTAT(arcstat_l2_writes_sent)
868 #define	l2arc_writes_done	ARCSTAT(arcstat_l2_writes_done)
869 
870 /* L2ARC Performance Tunables */
871 uint64_t l2arc_write_max = L2ARC_WRITE_SIZE;	/* default max write size */
872 uint64_t l2arc_write_boost = L2ARC_WRITE_SIZE;	/* extra write during warmup */
873 uint64_t l2arc_headroom = L2ARC_HEADROOM;	/* number of dev writes */
874 uint64_t l2arc_headroom_boost = L2ARC_HEADROOM_BOOST;
875 uint64_t l2arc_feed_secs = L2ARC_FEED_SECS;	/* interval seconds */
876 uint64_t l2arc_feed_min_ms = L2ARC_FEED_MIN_MS;	/* min interval milliseconds */
877 boolean_t l2arc_noprefetch = B_TRUE;		/* don't cache prefetch bufs */
878 boolean_t l2arc_feed_again = B_TRUE;		/* turbo warmup */
879 boolean_t l2arc_norw = B_TRUE;			/* no reads during writes */
880 
881 /*
882  * L2ARC Internals
883  */
884 struct l2arc_dev {
885 	vdev_t			*l2ad_vdev;	/* vdev */
886 	spa_t			*l2ad_spa;	/* spa */
887 	uint64_t		l2ad_hand;	/* next write location */
888 	uint64_t		l2ad_start;	/* first addr on device */
889 	uint64_t		l2ad_end;	/* last addr on device */
890 	uint64_t		l2ad_evict;	/* last addr eviction reached */
891 	boolean_t		l2ad_first;	/* first sweep through */
892 	boolean_t		l2ad_writing;	/* currently writing */
893 	kmutex_t		l2ad_mtx;	/* lock for buffer list */
894 	list_t			l2ad_buflist;	/* buffer list */
895 	list_node_t		l2ad_node;	/* device list node */
896 };
897 
898 static list_t L2ARC_dev_list;			/* device list */
899 static list_t *l2arc_dev_list;			/* device list pointer */
900 static kmutex_t l2arc_dev_mtx;			/* device list mutex */
901 static l2arc_dev_t *l2arc_dev_last;		/* last device used */
902 static list_t L2ARC_free_on_write;		/* free after write buf list */
903 static list_t *l2arc_free_on_write;		/* free after write list ptr */
904 static kmutex_t l2arc_free_on_write_mtx;	/* mutex for list */
905 static uint64_t l2arc_ndev;			/* number of devices */
906 
907 typedef struct l2arc_read_callback {
908 	arc_buf_t		*l2rcb_buf;		/* read buffer */
909 	spa_t			*l2rcb_spa;		/* spa */
910 	blkptr_t		l2rcb_bp;		/* original blkptr */
911 	zbookmark_phys_t	l2rcb_zb;		/* original bookmark */
912 	int			l2rcb_flags;		/* original flags */
913 	enum zio_compress	l2rcb_compress;		/* applied compress */
914 } l2arc_read_callback_t;
915 
916 typedef struct l2arc_write_callback {
917 	l2arc_dev_t	*l2wcb_dev;		/* device info */
918 	arc_buf_hdr_t	*l2wcb_head;		/* head of write buflist */
919 } l2arc_write_callback_t;
920 
921 typedef struct l2arc_data_free {
922 	/* protected by l2arc_free_on_write_mtx */
923 	void		*l2df_data;
924 	size_t		l2df_size;
925 	void		(*l2df_func)(void *, size_t);
926 	list_node_t	l2df_list_node;
927 } l2arc_data_free_t;
928 
929 static kmutex_t l2arc_feed_thr_lock;
930 static kcondvar_t l2arc_feed_thr_cv;
931 static uint8_t l2arc_thread_exit;
932 
933 static void arc_get_data_buf(arc_buf_t *);
934 static void arc_access(arc_buf_hdr_t *, kmutex_t *);
935 static boolean_t arc_is_overflowing();
936 static void arc_buf_watch(arc_buf_t *);
937 
938 static arc_buf_contents_t arc_buf_type(arc_buf_hdr_t *);
939 static uint32_t arc_bufc_to_flags(arc_buf_contents_t);
940 
941 static boolean_t l2arc_write_eligible(uint64_t, arc_buf_hdr_t *);
942 static void l2arc_read_done(zio_t *);
943 
944 static boolean_t l2arc_compress_buf(arc_buf_hdr_t *);
945 static void l2arc_decompress_zio(zio_t *, arc_buf_hdr_t *, enum zio_compress);
946 static void l2arc_release_cdata_buf(arc_buf_hdr_t *);
947 
948 static uint64_t
949 buf_hash(uint64_t spa, const dva_t *dva, uint64_t birth)
950 {
951 	uint8_t *vdva = (uint8_t *)dva;
952 	uint64_t crc = -1ULL;
953 	int i;
954 
955 	ASSERT(zfs_crc64_table[128] == ZFS_CRC64_POLY);
956 
957 	for (i = 0; i < sizeof (dva_t); i++)
958 		crc = (crc >> 8) ^ zfs_crc64_table[(crc ^ vdva[i]) & 0xFF];
959 
960 	crc ^= (spa>>8) ^ birth;
961 
962 	return (crc);
963 }
964 
965 #define	BUF_EMPTY(buf)						\
966 	((buf)->b_dva.dva_word[0] == 0 &&			\
967 	(buf)->b_dva.dva_word[1] == 0)
968 
969 #define	BUF_EQUAL(spa, dva, birth, buf)				\
970 	((buf)->b_dva.dva_word[0] == (dva)->dva_word[0]) &&	\
971 	((buf)->b_dva.dva_word[1] == (dva)->dva_word[1]) &&	\
972 	((buf)->b_birth == birth) && ((buf)->b_spa == spa)
973 
974 static void
975 buf_discard_identity(arc_buf_hdr_t *hdr)
976 {
977 	hdr->b_dva.dva_word[0] = 0;
978 	hdr->b_dva.dva_word[1] = 0;
979 	hdr->b_birth = 0;
980 }
981 
982 static arc_buf_hdr_t *
983 buf_hash_find(uint64_t spa, const blkptr_t *bp, kmutex_t **lockp)
984 {
985 	const dva_t *dva = BP_IDENTITY(bp);
986 	uint64_t birth = BP_PHYSICAL_BIRTH(bp);
987 	uint64_t idx = BUF_HASH_INDEX(spa, dva, birth);
988 	kmutex_t *hash_lock = BUF_HASH_LOCK(idx);
989 	arc_buf_hdr_t *hdr;
990 
991 	mutex_enter(hash_lock);
992 	for (hdr = buf_hash_table.ht_table[idx]; hdr != NULL;
993 	    hdr = hdr->b_hash_next) {
994 		if (BUF_EQUAL(spa, dva, birth, hdr)) {
995 			*lockp = hash_lock;
996 			return (hdr);
997 		}
998 	}
999 	mutex_exit(hash_lock);
1000 	*lockp = NULL;
1001 	return (NULL);
1002 }
1003 
1004 /*
1005  * Insert an entry into the hash table.  If there is already an element
1006  * equal to elem in the hash table, then the already existing element
1007  * will be returned and the new element will not be inserted.
1008  * Otherwise returns NULL.
1009  * If lockp == NULL, the caller is assumed to already hold the hash lock.
1010  */
1011 static arc_buf_hdr_t *
1012 buf_hash_insert(arc_buf_hdr_t *hdr, kmutex_t **lockp)
1013 {
1014 	uint64_t idx = BUF_HASH_INDEX(hdr->b_spa, &hdr->b_dva, hdr->b_birth);
1015 	kmutex_t *hash_lock = BUF_HASH_LOCK(idx);
1016 	arc_buf_hdr_t *fhdr;
1017 	uint32_t i;
1018 
1019 	ASSERT(!DVA_IS_EMPTY(&hdr->b_dva));
1020 	ASSERT(hdr->b_birth != 0);
1021 	ASSERT(!HDR_IN_HASH_TABLE(hdr));
1022 
1023 	if (lockp != NULL) {
1024 		*lockp = hash_lock;
1025 		mutex_enter(hash_lock);
1026 	} else {
1027 		ASSERT(MUTEX_HELD(hash_lock));
1028 	}
1029 
1030 	for (fhdr = buf_hash_table.ht_table[idx], i = 0; fhdr != NULL;
1031 	    fhdr = fhdr->b_hash_next, i++) {
1032 		if (BUF_EQUAL(hdr->b_spa, &hdr->b_dva, hdr->b_birth, fhdr))
1033 			return (fhdr);
1034 	}
1035 
1036 	hdr->b_hash_next = buf_hash_table.ht_table[idx];
1037 	buf_hash_table.ht_table[idx] = hdr;
1038 	hdr->b_flags |= ARC_FLAG_IN_HASH_TABLE;
1039 
1040 	/* collect some hash table performance data */
1041 	if (i > 0) {
1042 		ARCSTAT_BUMP(arcstat_hash_collisions);
1043 		if (i == 1)
1044 			ARCSTAT_BUMP(arcstat_hash_chains);
1045 
1046 		ARCSTAT_MAX(arcstat_hash_chain_max, i);
1047 	}
1048 
1049 	ARCSTAT_BUMP(arcstat_hash_elements);
1050 	ARCSTAT_MAXSTAT(arcstat_hash_elements);
1051 
1052 	return (NULL);
1053 }
1054 
1055 static void
1056 buf_hash_remove(arc_buf_hdr_t *hdr)
1057 {
1058 	arc_buf_hdr_t *fhdr, **hdrp;
1059 	uint64_t idx = BUF_HASH_INDEX(hdr->b_spa, &hdr->b_dva, hdr->b_birth);
1060 
1061 	ASSERT(MUTEX_HELD(BUF_HASH_LOCK(idx)));
1062 	ASSERT(HDR_IN_HASH_TABLE(hdr));
1063 
1064 	hdrp = &buf_hash_table.ht_table[idx];
1065 	while ((fhdr = *hdrp) != hdr) {
1066 		ASSERT(fhdr != NULL);
1067 		hdrp = &fhdr->b_hash_next;
1068 	}
1069 	*hdrp = hdr->b_hash_next;
1070 	hdr->b_hash_next = NULL;
1071 	hdr->b_flags &= ~ARC_FLAG_IN_HASH_TABLE;
1072 
1073 	/* collect some hash table performance data */
1074 	ARCSTAT_BUMPDOWN(arcstat_hash_elements);
1075 
1076 	if (buf_hash_table.ht_table[idx] &&
1077 	    buf_hash_table.ht_table[idx]->b_hash_next == NULL)
1078 		ARCSTAT_BUMPDOWN(arcstat_hash_chains);
1079 }
1080 
1081 /*
1082  * Global data structures and functions for the buf kmem cache.
1083  */
1084 static kmem_cache_t *hdr_full_cache;
1085 static kmem_cache_t *hdr_l2only_cache;
1086 static kmem_cache_t *buf_cache;
1087 
1088 static void
1089 buf_fini(void)
1090 {
1091 	int i;
1092 
1093 	kmem_free(buf_hash_table.ht_table,
1094 	    (buf_hash_table.ht_mask + 1) * sizeof (void *));
1095 	for (i = 0; i < BUF_LOCKS; i++)
1096 		mutex_destroy(&buf_hash_table.ht_locks[i].ht_lock);
1097 	kmem_cache_destroy(hdr_full_cache);
1098 	kmem_cache_destroy(hdr_l2only_cache);
1099 	kmem_cache_destroy(buf_cache);
1100 }
1101 
1102 /*
1103  * Constructor callback - called when the cache is empty
1104  * and a new buf is requested.
1105  */
1106 /* ARGSUSED */
1107 static int
1108 hdr_full_cons(void *vbuf, void *unused, int kmflag)
1109 {
1110 	arc_buf_hdr_t *hdr = vbuf;
1111 
1112 	bzero(hdr, HDR_FULL_SIZE);
1113 	cv_init(&hdr->b_l1hdr.b_cv, NULL, CV_DEFAULT, NULL);
1114 	refcount_create(&hdr->b_l1hdr.b_refcnt);
1115 	mutex_init(&hdr->b_l1hdr.b_freeze_lock, NULL, MUTEX_DEFAULT, NULL);
1116 	multilist_link_init(&hdr->b_l1hdr.b_arc_node);
1117 	arc_space_consume(HDR_FULL_SIZE, ARC_SPACE_HDRS);
1118 
1119 	return (0);
1120 }
1121 
1122 /* ARGSUSED */
1123 static int
1124 hdr_l2only_cons(void *vbuf, void *unused, int kmflag)
1125 {
1126 	arc_buf_hdr_t *hdr = vbuf;
1127 
1128 	bzero(hdr, HDR_L2ONLY_SIZE);
1129 	arc_space_consume(HDR_L2ONLY_SIZE, ARC_SPACE_L2HDRS);
1130 
1131 	return (0);
1132 }
1133 
1134 /* ARGSUSED */
1135 static int
1136 buf_cons(void *vbuf, void *unused, int kmflag)
1137 {
1138 	arc_buf_t *buf = vbuf;
1139 
1140 	bzero(buf, sizeof (arc_buf_t));
1141 	mutex_init(&buf->b_evict_lock, NULL, MUTEX_DEFAULT, NULL);
1142 	arc_space_consume(sizeof (arc_buf_t), ARC_SPACE_HDRS);
1143 
1144 	return (0);
1145 }
1146 
1147 /*
1148  * Destructor callback - called when a cached buf is
1149  * no longer required.
1150  */
1151 /* ARGSUSED */
1152 static void
1153 hdr_full_dest(void *vbuf, void *unused)
1154 {
1155 	arc_buf_hdr_t *hdr = vbuf;
1156 
1157 	ASSERT(BUF_EMPTY(hdr));
1158 	cv_destroy(&hdr->b_l1hdr.b_cv);
1159 	refcount_destroy(&hdr->b_l1hdr.b_refcnt);
1160 	mutex_destroy(&hdr->b_l1hdr.b_freeze_lock);
1161 	ASSERT(!multilist_link_active(&hdr->b_l1hdr.b_arc_node));
1162 	arc_space_return(HDR_FULL_SIZE, ARC_SPACE_HDRS);
1163 }
1164 
1165 /* ARGSUSED */
1166 static void
1167 hdr_l2only_dest(void *vbuf, void *unused)
1168 {
1169 	arc_buf_hdr_t *hdr = vbuf;
1170 
1171 	ASSERT(BUF_EMPTY(hdr));
1172 	arc_space_return(HDR_L2ONLY_SIZE, ARC_SPACE_L2HDRS);
1173 }
1174 
1175 /* ARGSUSED */
1176 static void
1177 buf_dest(void *vbuf, void *unused)
1178 {
1179 	arc_buf_t *buf = vbuf;
1180 
1181 	mutex_destroy(&buf->b_evict_lock);
1182 	arc_space_return(sizeof (arc_buf_t), ARC_SPACE_HDRS);
1183 }
1184 
1185 /*
1186  * Reclaim callback -- invoked when memory is low.
1187  */
1188 /* ARGSUSED */
1189 static void
1190 hdr_recl(void *unused)
1191 {
1192 	dprintf("hdr_recl called\n");
1193 	/*
1194 	 * umem calls the reclaim func when we destroy the buf cache,
1195 	 * which is after we do arc_fini().
1196 	 */
1197 	if (!arc_dead)
1198 		cv_signal(&arc_reclaim_thread_cv);
1199 }
1200 
1201 static void
1202 buf_init(void)
1203 {
1204 	uint64_t *ct;
1205 	uint64_t hsize = 1ULL << 12;
1206 	int i, j;
1207 
1208 	/*
1209 	 * The hash table is big enough to fill all of physical memory
1210 	 * with an average block size of zfs_arc_average_blocksize (default 8K).
1211 	 * By default, the table will take up
1212 	 * totalmem * sizeof(void*) / 8K (1MB per GB with 8-byte pointers).
1213 	 */
1214 	while (hsize * zfs_arc_average_blocksize < physmem * PAGESIZE)
1215 		hsize <<= 1;
1216 retry:
1217 	buf_hash_table.ht_mask = hsize - 1;
1218 	buf_hash_table.ht_table =
1219 	    kmem_zalloc(hsize * sizeof (void*), KM_NOSLEEP);
1220 	if (buf_hash_table.ht_table == NULL) {
1221 		ASSERT(hsize > (1ULL << 8));
1222 		hsize >>= 1;
1223 		goto retry;
1224 	}
1225 
1226 	hdr_full_cache = kmem_cache_create("arc_buf_hdr_t_full", HDR_FULL_SIZE,
1227 	    0, hdr_full_cons, hdr_full_dest, hdr_recl, NULL, NULL, 0);
1228 	hdr_l2only_cache = kmem_cache_create("arc_buf_hdr_t_l2only",
1229 	    HDR_L2ONLY_SIZE, 0, hdr_l2only_cons, hdr_l2only_dest, hdr_recl,
1230 	    NULL, NULL, 0);
1231 	buf_cache = kmem_cache_create("arc_buf_t", sizeof (arc_buf_t),
1232 	    0, buf_cons, buf_dest, NULL, NULL, NULL, 0);
1233 
1234 	for (i = 0; i < 256; i++)
1235 		for (ct = zfs_crc64_table + i, *ct = i, j = 8; j > 0; j--)
1236 			*ct = (*ct >> 1) ^ (-(*ct & 1) & ZFS_CRC64_POLY);
1237 
1238 	for (i = 0; i < BUF_LOCKS; i++) {
1239 		mutex_init(&buf_hash_table.ht_locks[i].ht_lock,
1240 		    NULL, MUTEX_DEFAULT, NULL);
1241 	}
1242 }
1243 
1244 /*
1245  * Transition between the two allocation states for the arc_buf_hdr struct.
1246  * The arc_buf_hdr struct can be allocated with (hdr_full_cache) or without
1247  * (hdr_l2only_cache) the fields necessary for the L1 cache - the smaller
1248  * version is used when a cache buffer is only in the L2ARC in order to reduce
1249  * memory usage.
1250  */
1251 static arc_buf_hdr_t *
1252 arc_hdr_realloc(arc_buf_hdr_t *hdr, kmem_cache_t *old, kmem_cache_t *new)
1253 {
1254 	ASSERT(HDR_HAS_L2HDR(hdr));
1255 
1256 	arc_buf_hdr_t *nhdr;
1257 	l2arc_dev_t *dev = hdr->b_l2hdr.b_dev;
1258 
1259 	ASSERT((old == hdr_full_cache && new == hdr_l2only_cache) ||
1260 	    (old == hdr_l2only_cache && new == hdr_full_cache));
1261 
1262 	nhdr = kmem_cache_alloc(new, KM_PUSHPAGE);
1263 
1264 	ASSERT(MUTEX_HELD(HDR_LOCK(hdr)));
1265 	buf_hash_remove(hdr);
1266 
1267 	bcopy(hdr, nhdr, HDR_L2ONLY_SIZE);
1268 	if (new == hdr_full_cache) {
1269 		nhdr->b_flags |= ARC_FLAG_HAS_L1HDR;
1270 		/*
1271 		 * arc_access and arc_change_state need to be aware that a
1272 		 * header has just come out of L2ARC, so we set its state to
1273 		 * l2c_only even though it's about to change.
1274 		 */
1275 		nhdr->b_l1hdr.b_state = arc_l2c_only;
1276 
1277 		/* Verify previous threads set to NULL before freeing */
1278 		ASSERT3P(nhdr->b_l1hdr.b_tmp_cdata, ==, NULL);
1279 	} else {
1280 		ASSERT(hdr->b_l1hdr.b_buf == NULL);
1281 		ASSERT0(hdr->b_l1hdr.b_datacnt);
1282 
1283 		/*
1284 		 * If we've reached here, We must have been called from
1285 		 * arc_evict_hdr(), as such we should have already been
1286 		 * removed from any ghost list we were previously on
1287 		 * (which protects us from racing with arc_evict_state),
1288 		 * thus no locking is needed during this check.
1289 		 */
1290 		ASSERT(!multilist_link_active(&hdr->b_l1hdr.b_arc_node));
1291 
1292 		/*
1293 		 * A buffer must not be moved into the arc_l2c_only
1294 		 * state if it's not finished being written out to the
1295 		 * l2arc device. Otherwise, the b_l1hdr.b_tmp_cdata field
1296 		 * might try to be accessed, even though it was removed.
1297 		 */
1298 		VERIFY(!HDR_L2_WRITING(hdr));
1299 		VERIFY3P(hdr->b_l1hdr.b_tmp_cdata, ==, NULL);
1300 
1301 		nhdr->b_flags &= ~ARC_FLAG_HAS_L1HDR;
1302 	}
1303 	/*
1304 	 * The header has been reallocated so we need to re-insert it into any
1305 	 * lists it was on.
1306 	 */
1307 	(void) buf_hash_insert(nhdr, NULL);
1308 
1309 	ASSERT(list_link_active(&hdr->b_l2hdr.b_l2node));
1310 
1311 	mutex_enter(&dev->l2ad_mtx);
1312 
1313 	/*
1314 	 * We must place the realloc'ed header back into the list at
1315 	 * the same spot. Otherwise, if it's placed earlier in the list,
1316 	 * l2arc_write_buffers() could find it during the function's
1317 	 * write phase, and try to write it out to the l2arc.
1318 	 */
1319 	list_insert_after(&dev->l2ad_buflist, hdr, nhdr);
1320 	list_remove(&dev->l2ad_buflist, hdr);
1321 
1322 	mutex_exit(&dev->l2ad_mtx);
1323 
1324 	buf_discard_identity(hdr);
1325 	hdr->b_freeze_cksum = NULL;
1326 	kmem_cache_free(old, hdr);
1327 
1328 	return (nhdr);
1329 }
1330 
1331 
1332 #define	ARC_MINTIME	(hz>>4) /* 62 ms */
1333 
1334 static void
1335 arc_cksum_verify(arc_buf_t *buf)
1336 {
1337 	zio_cksum_t zc;
1338 
1339 	if (!(zfs_flags & ZFS_DEBUG_MODIFY))
1340 		return;
1341 
1342 	mutex_enter(&buf->b_hdr->b_l1hdr.b_freeze_lock);
1343 	if (buf->b_hdr->b_freeze_cksum == NULL || HDR_IO_ERROR(buf->b_hdr)) {
1344 		mutex_exit(&buf->b_hdr->b_l1hdr.b_freeze_lock);
1345 		return;
1346 	}
1347 	fletcher_2_native(buf->b_data, buf->b_hdr->b_size, &zc);
1348 	if (!ZIO_CHECKSUM_EQUAL(*buf->b_hdr->b_freeze_cksum, zc))
1349 		panic("buffer modified while frozen!");
1350 	mutex_exit(&buf->b_hdr->b_l1hdr.b_freeze_lock);
1351 }
1352 
1353 static int
1354 arc_cksum_equal(arc_buf_t *buf)
1355 {
1356 	zio_cksum_t zc;
1357 	int equal;
1358 
1359 	mutex_enter(&buf->b_hdr->b_l1hdr.b_freeze_lock);
1360 	fletcher_2_native(buf->b_data, buf->b_hdr->b_size, &zc);
1361 	equal = ZIO_CHECKSUM_EQUAL(*buf->b_hdr->b_freeze_cksum, zc);
1362 	mutex_exit(&buf->b_hdr->b_l1hdr.b_freeze_lock);
1363 
1364 	return (equal);
1365 }
1366 
1367 static void
1368 arc_cksum_compute(arc_buf_t *buf, boolean_t force)
1369 {
1370 	if (!force && !(zfs_flags & ZFS_DEBUG_MODIFY))
1371 		return;
1372 
1373 	mutex_enter(&buf->b_hdr->b_l1hdr.b_freeze_lock);
1374 	if (buf->b_hdr->b_freeze_cksum != NULL) {
1375 		mutex_exit(&buf->b_hdr->b_l1hdr.b_freeze_lock);
1376 		return;
1377 	}
1378 	buf->b_hdr->b_freeze_cksum = kmem_alloc(sizeof (zio_cksum_t), KM_SLEEP);
1379 	fletcher_2_native(buf->b_data, buf->b_hdr->b_size,
1380 	    buf->b_hdr->b_freeze_cksum);
1381 	mutex_exit(&buf->b_hdr->b_l1hdr.b_freeze_lock);
1382 	arc_buf_watch(buf);
1383 }
1384 
1385 #ifndef _KERNEL
1386 typedef struct procctl {
1387 	long cmd;
1388 	prwatch_t prwatch;
1389 } procctl_t;
1390 #endif
1391 
1392 /* ARGSUSED */
1393 static void
1394 arc_buf_unwatch(arc_buf_t *buf)
1395 {
1396 #ifndef _KERNEL
1397 	if (arc_watch) {
1398 		int result;
1399 		procctl_t ctl;
1400 		ctl.cmd = PCWATCH;
1401 		ctl.prwatch.pr_vaddr = (uintptr_t)buf->b_data;
1402 		ctl.prwatch.pr_size = 0;
1403 		ctl.prwatch.pr_wflags = 0;
1404 		result = write(arc_procfd, &ctl, sizeof (ctl));
1405 		ASSERT3U(result, ==, sizeof (ctl));
1406 	}
1407 #endif
1408 }
1409 
1410 /* ARGSUSED */
1411 static void
1412 arc_buf_watch(arc_buf_t *buf)
1413 {
1414 #ifndef _KERNEL
1415 	if (arc_watch) {
1416 		int result;
1417 		procctl_t ctl;
1418 		ctl.cmd = PCWATCH;
1419 		ctl.prwatch.pr_vaddr = (uintptr_t)buf->b_data;
1420 		ctl.prwatch.pr_size = buf->b_hdr->b_size;
1421 		ctl.prwatch.pr_wflags = WA_WRITE;
1422 		result = write(arc_procfd, &ctl, sizeof (ctl));
1423 		ASSERT3U(result, ==, sizeof (ctl));
1424 	}
1425 #endif
1426 }
1427 
1428 static arc_buf_contents_t
1429 arc_buf_type(arc_buf_hdr_t *hdr)
1430 {
1431 	if (HDR_ISTYPE_METADATA(hdr)) {
1432 		return (ARC_BUFC_METADATA);
1433 	} else {
1434 		return (ARC_BUFC_DATA);
1435 	}
1436 }
1437 
1438 static uint32_t
1439 arc_bufc_to_flags(arc_buf_contents_t type)
1440 {
1441 	switch (type) {
1442 	case ARC_BUFC_DATA:
1443 		/* metadata field is 0 if buffer contains normal data */
1444 		return (0);
1445 	case ARC_BUFC_METADATA:
1446 		return (ARC_FLAG_BUFC_METADATA);
1447 	default:
1448 		break;
1449 	}
1450 	panic("undefined ARC buffer type!");
1451 	return ((uint32_t)-1);
1452 }
1453 
1454 void
1455 arc_buf_thaw(arc_buf_t *buf)
1456 {
1457 	if (zfs_flags & ZFS_DEBUG_MODIFY) {
1458 		if (buf->b_hdr->b_l1hdr.b_state != arc_anon)
1459 			panic("modifying non-anon buffer!");
1460 		if (HDR_IO_IN_PROGRESS(buf->b_hdr))
1461 			panic("modifying buffer while i/o in progress!");
1462 		arc_cksum_verify(buf);
1463 	}
1464 
1465 	mutex_enter(&buf->b_hdr->b_l1hdr.b_freeze_lock);
1466 	if (buf->b_hdr->b_freeze_cksum != NULL) {
1467 		kmem_free(buf->b_hdr->b_freeze_cksum, sizeof (zio_cksum_t));
1468 		buf->b_hdr->b_freeze_cksum = NULL;
1469 	}
1470 
1471 #ifdef ZFS_DEBUG
1472 	if (zfs_flags & ZFS_DEBUG_MODIFY) {
1473 		if (buf->b_hdr->b_l1hdr.b_thawed != NULL)
1474 			kmem_free(buf->b_hdr->b_l1hdr.b_thawed, 1);
1475 		buf->b_hdr->b_l1hdr.b_thawed = kmem_alloc(1, KM_SLEEP);
1476 	}
1477 #endif
1478 
1479 	mutex_exit(&buf->b_hdr->b_l1hdr.b_freeze_lock);
1480 
1481 	arc_buf_unwatch(buf);
1482 }
1483 
1484 void
1485 arc_buf_freeze(arc_buf_t *buf)
1486 {
1487 	kmutex_t *hash_lock;
1488 
1489 	if (!(zfs_flags & ZFS_DEBUG_MODIFY))
1490 		return;
1491 
1492 	hash_lock = HDR_LOCK(buf->b_hdr);
1493 	mutex_enter(hash_lock);
1494 
1495 	ASSERT(buf->b_hdr->b_freeze_cksum != NULL ||
1496 	    buf->b_hdr->b_l1hdr.b_state == arc_anon);
1497 	arc_cksum_compute(buf, B_FALSE);
1498 	mutex_exit(hash_lock);
1499 
1500 }
1501 
1502 static void
1503 add_reference(arc_buf_hdr_t *hdr, kmutex_t *hash_lock, void *tag)
1504 {
1505 	ASSERT(HDR_HAS_L1HDR(hdr));
1506 	ASSERT(MUTEX_HELD(hash_lock));
1507 	arc_state_t *state = hdr->b_l1hdr.b_state;
1508 
1509 	if ((refcount_add(&hdr->b_l1hdr.b_refcnt, tag) == 1) &&
1510 	    (state != arc_anon)) {
1511 		/* We don't use the L2-only state list. */
1512 		if (state != arc_l2c_only) {
1513 			arc_buf_contents_t type = arc_buf_type(hdr);
1514 			uint64_t delta = hdr->b_size * hdr->b_l1hdr.b_datacnt;
1515 			multilist_t *list = &state->arcs_list[type];
1516 			uint64_t *size = &state->arcs_lsize[type];
1517 
1518 			multilist_remove(list, hdr);
1519 
1520 			if (GHOST_STATE(state)) {
1521 				ASSERT0(hdr->b_l1hdr.b_datacnt);
1522 				ASSERT3P(hdr->b_l1hdr.b_buf, ==, NULL);
1523 				delta = hdr->b_size;
1524 			}
1525 			ASSERT(delta > 0);
1526 			ASSERT3U(*size, >=, delta);
1527 			atomic_add_64(size, -delta);
1528 		}
1529 		/* remove the prefetch flag if we get a reference */
1530 		hdr->b_flags &= ~ARC_FLAG_PREFETCH;
1531 	}
1532 }
1533 
1534 static int
1535 remove_reference(arc_buf_hdr_t *hdr, kmutex_t *hash_lock, void *tag)
1536 {
1537 	int cnt;
1538 	arc_state_t *state = hdr->b_l1hdr.b_state;
1539 
1540 	ASSERT(HDR_HAS_L1HDR(hdr));
1541 	ASSERT(state == arc_anon || MUTEX_HELD(hash_lock));
1542 	ASSERT(!GHOST_STATE(state));
1543 
1544 	/*
1545 	 * arc_l2c_only counts as a ghost state so we don't need to explicitly
1546 	 * check to prevent usage of the arc_l2c_only list.
1547 	 */
1548 	if (((cnt = refcount_remove(&hdr->b_l1hdr.b_refcnt, tag)) == 0) &&
1549 	    (state != arc_anon)) {
1550 		arc_buf_contents_t type = arc_buf_type(hdr);
1551 		multilist_t *list = &state->arcs_list[type];
1552 		uint64_t *size = &state->arcs_lsize[type];
1553 
1554 		multilist_insert(list, hdr);
1555 
1556 		ASSERT(hdr->b_l1hdr.b_datacnt > 0);
1557 		atomic_add_64(size, hdr->b_size *
1558 		    hdr->b_l1hdr.b_datacnt);
1559 	}
1560 	return (cnt);
1561 }
1562 
1563 /*
1564  * Move the supplied buffer to the indicated state. The hash lock
1565  * for the buffer must be held by the caller.
1566  */
1567 static void
1568 arc_change_state(arc_state_t *new_state, arc_buf_hdr_t *hdr,
1569     kmutex_t *hash_lock)
1570 {
1571 	arc_state_t *old_state;
1572 	int64_t refcnt;
1573 	uint32_t datacnt;
1574 	uint64_t from_delta, to_delta;
1575 	arc_buf_contents_t buftype = arc_buf_type(hdr);
1576 
1577 	/*
1578 	 * We almost always have an L1 hdr here, since we call arc_hdr_realloc()
1579 	 * in arc_read() when bringing a buffer out of the L2ARC.  However, the
1580 	 * L1 hdr doesn't always exist when we change state to arc_anon before
1581 	 * destroying a header, in which case reallocating to add the L1 hdr is
1582 	 * pointless.
1583 	 */
1584 	if (HDR_HAS_L1HDR(hdr)) {
1585 		old_state = hdr->b_l1hdr.b_state;
1586 		refcnt = refcount_count(&hdr->b_l1hdr.b_refcnt);
1587 		datacnt = hdr->b_l1hdr.b_datacnt;
1588 	} else {
1589 		old_state = arc_l2c_only;
1590 		refcnt = 0;
1591 		datacnt = 0;
1592 	}
1593 
1594 	ASSERT(MUTEX_HELD(hash_lock));
1595 	ASSERT3P(new_state, !=, old_state);
1596 	ASSERT(refcnt == 0 || datacnt > 0);
1597 	ASSERT(!GHOST_STATE(new_state) || datacnt == 0);
1598 	ASSERT(old_state != arc_anon || datacnt <= 1);
1599 
1600 	from_delta = to_delta = datacnt * hdr->b_size;
1601 
1602 	/*
1603 	 * If this buffer is evictable, transfer it from the
1604 	 * old state list to the new state list.
1605 	 */
1606 	if (refcnt == 0) {
1607 		if (old_state != arc_anon && old_state != arc_l2c_only) {
1608 			uint64_t *size = &old_state->arcs_lsize[buftype];
1609 
1610 			ASSERT(HDR_HAS_L1HDR(hdr));
1611 			multilist_remove(&old_state->arcs_list[buftype], hdr);
1612 
1613 			/*
1614 			 * If prefetching out of the ghost cache,
1615 			 * we will have a non-zero datacnt.
1616 			 */
1617 			if (GHOST_STATE(old_state) && datacnt == 0) {
1618 				/* ghost elements have a ghost size */
1619 				ASSERT(hdr->b_l1hdr.b_buf == NULL);
1620 				from_delta = hdr->b_size;
1621 			}
1622 			ASSERT3U(*size, >=, from_delta);
1623 			atomic_add_64(size, -from_delta);
1624 		}
1625 		if (new_state != arc_anon && new_state != arc_l2c_only) {
1626 			uint64_t *size = &new_state->arcs_lsize[buftype];
1627 
1628 			/*
1629 			 * An L1 header always exists here, since if we're
1630 			 * moving to some L1-cached state (i.e. not l2c_only or
1631 			 * anonymous), we realloc the header to add an L1hdr
1632 			 * beforehand.
1633 			 */
1634 			ASSERT(HDR_HAS_L1HDR(hdr));
1635 			multilist_insert(&new_state->arcs_list[buftype], hdr);
1636 
1637 			/* ghost elements have a ghost size */
1638 			if (GHOST_STATE(new_state)) {
1639 				ASSERT0(datacnt);
1640 				ASSERT(hdr->b_l1hdr.b_buf == NULL);
1641 				to_delta = hdr->b_size;
1642 			}
1643 			atomic_add_64(size, to_delta);
1644 		}
1645 	}
1646 
1647 	ASSERT(!BUF_EMPTY(hdr));
1648 	if (new_state == arc_anon && HDR_IN_HASH_TABLE(hdr))
1649 		buf_hash_remove(hdr);
1650 
1651 	/* adjust state sizes (ignore arc_l2c_only) */
1652 	if (to_delta && new_state != arc_l2c_only)
1653 		atomic_add_64(&new_state->arcs_size, to_delta);
1654 	if (from_delta && old_state != arc_l2c_only) {
1655 		ASSERT3U(old_state->arcs_size, >=, from_delta);
1656 		atomic_add_64(&old_state->arcs_size, -from_delta);
1657 	}
1658 	if (HDR_HAS_L1HDR(hdr))
1659 		hdr->b_l1hdr.b_state = new_state;
1660 
1661 	/*
1662 	 * L2 headers should never be on the L2 state list since they don't
1663 	 * have L1 headers allocated.
1664 	 */
1665 	ASSERT(multilist_is_empty(&arc_l2c_only->arcs_list[ARC_BUFC_DATA]) &&
1666 	    multilist_is_empty(&arc_l2c_only->arcs_list[ARC_BUFC_METADATA]));
1667 }
1668 
1669 void
1670 arc_space_consume(uint64_t space, arc_space_type_t type)
1671 {
1672 	ASSERT(type >= 0 && type < ARC_SPACE_NUMTYPES);
1673 
1674 	switch (type) {
1675 	case ARC_SPACE_DATA:
1676 		ARCSTAT_INCR(arcstat_data_size, space);
1677 		break;
1678 	case ARC_SPACE_META:
1679 		ARCSTAT_INCR(arcstat_metadata_size, space);
1680 		break;
1681 	case ARC_SPACE_OTHER:
1682 		ARCSTAT_INCR(arcstat_other_size, space);
1683 		break;
1684 	case ARC_SPACE_HDRS:
1685 		ARCSTAT_INCR(arcstat_hdr_size, space);
1686 		break;
1687 	case ARC_SPACE_L2HDRS:
1688 		ARCSTAT_INCR(arcstat_l2_hdr_size, space);
1689 		break;
1690 	}
1691 
1692 	if (type != ARC_SPACE_DATA)
1693 		ARCSTAT_INCR(arcstat_meta_used, space);
1694 
1695 	atomic_add_64(&arc_size, space);
1696 }
1697 
1698 void
1699 arc_space_return(uint64_t space, arc_space_type_t type)
1700 {
1701 	ASSERT(type >= 0 && type < ARC_SPACE_NUMTYPES);
1702 
1703 	switch (type) {
1704 	case ARC_SPACE_DATA:
1705 		ARCSTAT_INCR(arcstat_data_size, -space);
1706 		break;
1707 	case ARC_SPACE_META:
1708 		ARCSTAT_INCR(arcstat_metadata_size, -space);
1709 		break;
1710 	case ARC_SPACE_OTHER:
1711 		ARCSTAT_INCR(arcstat_other_size, -space);
1712 		break;
1713 	case ARC_SPACE_HDRS:
1714 		ARCSTAT_INCR(arcstat_hdr_size, -space);
1715 		break;
1716 	case ARC_SPACE_L2HDRS:
1717 		ARCSTAT_INCR(arcstat_l2_hdr_size, -space);
1718 		break;
1719 	}
1720 
1721 	if (type != ARC_SPACE_DATA) {
1722 		ASSERT(arc_meta_used >= space);
1723 		if (arc_meta_max < arc_meta_used)
1724 			arc_meta_max = arc_meta_used;
1725 		ARCSTAT_INCR(arcstat_meta_used, -space);
1726 	}
1727 
1728 	ASSERT(arc_size >= space);
1729 	atomic_add_64(&arc_size, -space);
1730 }
1731 
1732 arc_buf_t *
1733 arc_buf_alloc(spa_t *spa, int32_t size, void *tag, arc_buf_contents_t type)
1734 {
1735 	arc_buf_hdr_t *hdr;
1736 	arc_buf_t *buf;
1737 
1738 	ASSERT3U(size, >, 0);
1739 	hdr = kmem_cache_alloc(hdr_full_cache, KM_PUSHPAGE);
1740 	ASSERT(BUF_EMPTY(hdr));
1741 	ASSERT3P(hdr->b_freeze_cksum, ==, NULL);
1742 	hdr->b_size = size;
1743 	hdr->b_spa = spa_load_guid(spa);
1744 
1745 	buf = kmem_cache_alloc(buf_cache, KM_PUSHPAGE);
1746 	buf->b_hdr = hdr;
1747 	buf->b_data = NULL;
1748 	buf->b_efunc = NULL;
1749 	buf->b_private = NULL;
1750 	buf->b_next = NULL;
1751 
1752 	hdr->b_flags = arc_bufc_to_flags(type);
1753 	hdr->b_flags |= ARC_FLAG_HAS_L1HDR;
1754 
1755 	hdr->b_l1hdr.b_buf = buf;
1756 	hdr->b_l1hdr.b_state = arc_anon;
1757 	hdr->b_l1hdr.b_arc_access = 0;
1758 	hdr->b_l1hdr.b_datacnt = 1;
1759 	hdr->b_l1hdr.b_tmp_cdata = NULL;
1760 
1761 	arc_get_data_buf(buf);
1762 	ASSERT(refcount_is_zero(&hdr->b_l1hdr.b_refcnt));
1763 	(void) refcount_add(&hdr->b_l1hdr.b_refcnt, tag);
1764 
1765 	return (buf);
1766 }
1767 
1768 static char *arc_onloan_tag = "onloan";
1769 
1770 /*
1771  * Loan out an anonymous arc buffer. Loaned buffers are not counted as in
1772  * flight data by arc_tempreserve_space() until they are "returned". Loaned
1773  * buffers must be returned to the arc before they can be used by the DMU or
1774  * freed.
1775  */
1776 arc_buf_t *
1777 arc_loan_buf(spa_t *spa, int size)
1778 {
1779 	arc_buf_t *buf;
1780 
1781 	buf = arc_buf_alloc(spa, size, arc_onloan_tag, ARC_BUFC_DATA);
1782 
1783 	atomic_add_64(&arc_loaned_bytes, size);
1784 	return (buf);
1785 }
1786 
1787 /*
1788  * Return a loaned arc buffer to the arc.
1789  */
1790 void
1791 arc_return_buf(arc_buf_t *buf, void *tag)
1792 {
1793 	arc_buf_hdr_t *hdr = buf->b_hdr;
1794 
1795 	ASSERT(buf->b_data != NULL);
1796 	ASSERT(HDR_HAS_L1HDR(hdr));
1797 	(void) refcount_add(&hdr->b_l1hdr.b_refcnt, tag);
1798 	(void) refcount_remove(&hdr->b_l1hdr.b_refcnt, arc_onloan_tag);
1799 
1800 	atomic_add_64(&arc_loaned_bytes, -hdr->b_size);
1801 }
1802 
1803 /* Detach an arc_buf from a dbuf (tag) */
1804 void
1805 arc_loan_inuse_buf(arc_buf_t *buf, void *tag)
1806 {
1807 	arc_buf_hdr_t *hdr = buf->b_hdr;
1808 
1809 	ASSERT(buf->b_data != NULL);
1810 	ASSERT(HDR_HAS_L1HDR(hdr));
1811 	(void) refcount_add(&hdr->b_l1hdr.b_refcnt, arc_onloan_tag);
1812 	(void) refcount_remove(&hdr->b_l1hdr.b_refcnt, tag);
1813 	buf->b_efunc = NULL;
1814 	buf->b_private = NULL;
1815 
1816 	atomic_add_64(&arc_loaned_bytes, hdr->b_size);
1817 }
1818 
1819 static arc_buf_t *
1820 arc_buf_clone(arc_buf_t *from)
1821 {
1822 	arc_buf_t *buf;
1823 	arc_buf_hdr_t *hdr = from->b_hdr;
1824 	uint64_t size = hdr->b_size;
1825 
1826 	ASSERT(HDR_HAS_L1HDR(hdr));
1827 	ASSERT(hdr->b_l1hdr.b_state != arc_anon);
1828 
1829 	buf = kmem_cache_alloc(buf_cache, KM_PUSHPAGE);
1830 	buf->b_hdr = hdr;
1831 	buf->b_data = NULL;
1832 	buf->b_efunc = NULL;
1833 	buf->b_private = NULL;
1834 	buf->b_next = hdr->b_l1hdr.b_buf;
1835 	hdr->b_l1hdr.b_buf = buf;
1836 	arc_get_data_buf(buf);
1837 	bcopy(from->b_data, buf->b_data, size);
1838 
1839 	/*
1840 	 * This buffer already exists in the arc so create a duplicate
1841 	 * copy for the caller.  If the buffer is associated with user data
1842 	 * then track the size and number of duplicates.  These stats will be
1843 	 * updated as duplicate buffers are created and destroyed.
1844 	 */
1845 	if (HDR_ISTYPE_DATA(hdr)) {
1846 		ARCSTAT_BUMP(arcstat_duplicate_buffers);
1847 		ARCSTAT_INCR(arcstat_duplicate_buffers_size, size);
1848 	}
1849 	hdr->b_l1hdr.b_datacnt += 1;
1850 	return (buf);
1851 }
1852 
1853 void
1854 arc_buf_add_ref(arc_buf_t *buf, void* tag)
1855 {
1856 	arc_buf_hdr_t *hdr;
1857 	kmutex_t *hash_lock;
1858 
1859 	/*
1860 	 * Check to see if this buffer is evicted.  Callers
1861 	 * must verify b_data != NULL to know if the add_ref
1862 	 * was successful.
1863 	 */
1864 	mutex_enter(&buf->b_evict_lock);
1865 	if (buf->b_data == NULL) {
1866 		mutex_exit(&buf->b_evict_lock);
1867 		return;
1868 	}
1869 	hash_lock = HDR_LOCK(buf->b_hdr);
1870 	mutex_enter(hash_lock);
1871 	hdr = buf->b_hdr;
1872 	ASSERT(HDR_HAS_L1HDR(hdr));
1873 	ASSERT3P(hash_lock, ==, HDR_LOCK(hdr));
1874 	mutex_exit(&buf->b_evict_lock);
1875 
1876 	ASSERT(hdr->b_l1hdr.b_state == arc_mru ||
1877 	    hdr->b_l1hdr.b_state == arc_mfu);
1878 
1879 	add_reference(hdr, hash_lock, tag);
1880 	DTRACE_PROBE1(arc__hit, arc_buf_hdr_t *, hdr);
1881 	arc_access(hdr, hash_lock);
1882 	mutex_exit(hash_lock);
1883 	ARCSTAT_BUMP(arcstat_hits);
1884 	ARCSTAT_CONDSTAT(!HDR_PREFETCH(hdr),
1885 	    demand, prefetch, !HDR_ISTYPE_METADATA(hdr),
1886 	    data, metadata, hits);
1887 }
1888 
1889 static void
1890 arc_buf_free_on_write(void *data, size_t size,
1891     void (*free_func)(void *, size_t))
1892 {
1893 	l2arc_data_free_t *df;
1894 
1895 	df = kmem_alloc(sizeof (*df), KM_SLEEP);
1896 	df->l2df_data = data;
1897 	df->l2df_size = size;
1898 	df->l2df_func = free_func;
1899 	mutex_enter(&l2arc_free_on_write_mtx);
1900 	list_insert_head(l2arc_free_on_write, df);
1901 	mutex_exit(&l2arc_free_on_write_mtx);
1902 }
1903 
1904 /*
1905  * Free the arc data buffer.  If it is an l2arc write in progress,
1906  * the buffer is placed on l2arc_free_on_write to be freed later.
1907  */
1908 static void
1909 arc_buf_data_free(arc_buf_t *buf, void (*free_func)(void *, size_t))
1910 {
1911 	arc_buf_hdr_t *hdr = buf->b_hdr;
1912 
1913 	if (HDR_L2_WRITING(hdr)) {
1914 		arc_buf_free_on_write(buf->b_data, hdr->b_size, free_func);
1915 		ARCSTAT_BUMP(arcstat_l2_free_on_write);
1916 	} else {
1917 		free_func(buf->b_data, hdr->b_size);
1918 	}
1919 }
1920 
1921 static void
1922 arc_buf_l2_cdata_free(arc_buf_hdr_t *hdr)
1923 {
1924 	ASSERT(HDR_HAS_L2HDR(hdr));
1925 	ASSERT(MUTEX_HELD(&hdr->b_l2hdr.b_dev->l2ad_mtx));
1926 
1927 	/*
1928 	 * The b_tmp_cdata field is linked off of the b_l1hdr, so if
1929 	 * that doesn't exist, the header is in the arc_l2c_only state,
1930 	 * and there isn't anything to free (it's already been freed).
1931 	 */
1932 	if (!HDR_HAS_L1HDR(hdr))
1933 		return;
1934 
1935 	/*
1936 	 * The header isn't being written to the l2arc device, thus it
1937 	 * shouldn't have a b_tmp_cdata to free.
1938 	 */
1939 	if (!HDR_L2_WRITING(hdr)) {
1940 		ASSERT3P(hdr->b_l1hdr.b_tmp_cdata, ==, NULL);
1941 		return;
1942 	}
1943 
1944 	/*
1945 	 * The header does not have compression enabled. This can be due
1946 	 * to the buffer not being compressible, or because we're
1947 	 * freeing the buffer before the second phase of
1948 	 * l2arc_write_buffer() has started (which does the compression
1949 	 * step). In either case, b_tmp_cdata does not point to a
1950 	 * separately compressed buffer, so there's nothing to free (it
1951 	 * points to the same buffer as the arc_buf_t's b_data field).
1952 	 */
1953 	if (HDR_GET_COMPRESS(hdr) == ZIO_COMPRESS_OFF) {
1954 		hdr->b_l1hdr.b_tmp_cdata = NULL;
1955 		return;
1956 	}
1957 
1958 	/*
1959 	 * There's nothing to free since the buffer was all zero's and
1960 	 * compressed to a zero length buffer.
1961 	 */
1962 	if (HDR_GET_COMPRESS(hdr) == ZIO_COMPRESS_EMPTY) {
1963 		ASSERT3P(hdr->b_l1hdr.b_tmp_cdata, ==, NULL);
1964 		return;
1965 	}
1966 
1967 	ASSERT(L2ARC_IS_VALID_COMPRESS(HDR_GET_COMPRESS(hdr)));
1968 
1969 	arc_buf_free_on_write(hdr->b_l1hdr.b_tmp_cdata,
1970 	    hdr->b_size, zio_data_buf_free);
1971 
1972 	ARCSTAT_BUMP(arcstat_l2_cdata_free_on_write);
1973 	hdr->b_l1hdr.b_tmp_cdata = NULL;
1974 }
1975 
1976 /*
1977  * Free up buf->b_data and if 'remove' is set, then pull the
1978  * arc_buf_t off of the the arc_buf_hdr_t's list and free it.
1979  */
1980 static void
1981 arc_buf_destroy(arc_buf_t *buf, boolean_t remove)
1982 {
1983 	arc_buf_t **bufp;
1984 
1985 	/* free up data associated with the buf */
1986 	if (buf->b_data != NULL) {
1987 		arc_state_t *state = buf->b_hdr->b_l1hdr.b_state;
1988 		uint64_t size = buf->b_hdr->b_size;
1989 		arc_buf_contents_t type = arc_buf_type(buf->b_hdr);
1990 
1991 		arc_cksum_verify(buf);
1992 		arc_buf_unwatch(buf);
1993 
1994 		if (type == ARC_BUFC_METADATA) {
1995 			arc_buf_data_free(buf, zio_buf_free);
1996 			arc_space_return(size, ARC_SPACE_META);
1997 		} else {
1998 			ASSERT(type == ARC_BUFC_DATA);
1999 			arc_buf_data_free(buf, zio_data_buf_free);
2000 			arc_space_return(size, ARC_SPACE_DATA);
2001 		}
2002 
2003 		/* protected by hash lock, if in the hash table */
2004 		if (multilist_link_active(&buf->b_hdr->b_l1hdr.b_arc_node)) {
2005 			uint64_t *cnt = &state->arcs_lsize[type];
2006 
2007 			ASSERT(refcount_is_zero(
2008 			    &buf->b_hdr->b_l1hdr.b_refcnt));
2009 			ASSERT(state != arc_anon && state != arc_l2c_only);
2010 
2011 			ASSERT3U(*cnt, >=, size);
2012 			atomic_add_64(cnt, -size);
2013 		}
2014 		ASSERT3U(state->arcs_size, >=, size);
2015 		atomic_add_64(&state->arcs_size, -size);
2016 		buf->b_data = NULL;
2017 
2018 		/*
2019 		 * If we're destroying a duplicate buffer make sure
2020 		 * that the appropriate statistics are updated.
2021 		 */
2022 		if (buf->b_hdr->b_l1hdr.b_datacnt > 1 &&
2023 		    HDR_ISTYPE_DATA(buf->b_hdr)) {
2024 			ARCSTAT_BUMPDOWN(arcstat_duplicate_buffers);
2025 			ARCSTAT_INCR(arcstat_duplicate_buffers_size, -size);
2026 		}
2027 		ASSERT(buf->b_hdr->b_l1hdr.b_datacnt > 0);
2028 		buf->b_hdr->b_l1hdr.b_datacnt -= 1;
2029 	}
2030 
2031 	/* only remove the buf if requested */
2032 	if (!remove)
2033 		return;
2034 
2035 	/* remove the buf from the hdr list */
2036 	for (bufp = &buf->b_hdr->b_l1hdr.b_buf; *bufp != buf;
2037 	    bufp = &(*bufp)->b_next)
2038 		continue;
2039 	*bufp = buf->b_next;
2040 	buf->b_next = NULL;
2041 
2042 	ASSERT(buf->b_efunc == NULL);
2043 
2044 	/* clean up the buf */
2045 	buf->b_hdr = NULL;
2046 	kmem_cache_free(buf_cache, buf);
2047 }
2048 
2049 static void
2050 arc_hdr_destroy(arc_buf_hdr_t *hdr)
2051 {
2052 	if (HDR_HAS_L1HDR(hdr)) {
2053 		ASSERT(hdr->b_l1hdr.b_buf == NULL ||
2054 		    hdr->b_l1hdr.b_datacnt > 0);
2055 		ASSERT(refcount_is_zero(&hdr->b_l1hdr.b_refcnt));
2056 		ASSERT3P(hdr->b_l1hdr.b_state, ==, arc_anon);
2057 	}
2058 	ASSERT(!HDR_IO_IN_PROGRESS(hdr));
2059 	ASSERT(!HDR_IN_HASH_TABLE(hdr));
2060 
2061 	if (HDR_HAS_L2HDR(hdr)) {
2062 		l2arc_buf_hdr_t *l2hdr = &hdr->b_l2hdr;
2063 		boolean_t buflist_held = MUTEX_HELD(&l2hdr->b_dev->l2ad_mtx);
2064 
2065 		if (!buflist_held) {
2066 			mutex_enter(&l2hdr->b_dev->l2ad_mtx);
2067 			l2hdr = &hdr->b_l2hdr;
2068 		}
2069 
2070 		list_remove(&l2hdr->b_dev->l2ad_buflist, hdr);
2071 
2072 		/*
2073 		 * We don't want to leak the b_tmp_cdata buffer that was
2074 		 * allocated in l2arc_write_buffers()
2075 		 */
2076 		arc_buf_l2_cdata_free(hdr);
2077 
2078 		ARCSTAT_INCR(arcstat_l2_size, -hdr->b_size);
2079 		ARCSTAT_INCR(arcstat_l2_asize, -l2hdr->b_asize);
2080 
2081 		if (!buflist_held)
2082 			mutex_exit(&l2hdr->b_dev->l2ad_mtx);
2083 
2084 		hdr->b_flags &= ~ARC_FLAG_HAS_L2HDR;
2085 	}
2086 
2087 	if (!BUF_EMPTY(hdr))
2088 		buf_discard_identity(hdr);
2089 
2090 	if (hdr->b_freeze_cksum != NULL) {
2091 		kmem_free(hdr->b_freeze_cksum, sizeof (zio_cksum_t));
2092 		hdr->b_freeze_cksum = NULL;
2093 	}
2094 
2095 	if (HDR_HAS_L1HDR(hdr)) {
2096 		while (hdr->b_l1hdr.b_buf) {
2097 			arc_buf_t *buf = hdr->b_l1hdr.b_buf;
2098 
2099 			if (buf->b_efunc != NULL) {
2100 				mutex_enter(&arc_user_evicts_lock);
2101 				mutex_enter(&buf->b_evict_lock);
2102 				ASSERT(buf->b_hdr != NULL);
2103 				arc_buf_destroy(hdr->b_l1hdr.b_buf, FALSE);
2104 				hdr->b_l1hdr.b_buf = buf->b_next;
2105 				buf->b_hdr = &arc_eviction_hdr;
2106 				buf->b_next = arc_eviction_list;
2107 				arc_eviction_list = buf;
2108 				mutex_exit(&buf->b_evict_lock);
2109 				cv_signal(&arc_user_evicts_cv);
2110 				mutex_exit(&arc_user_evicts_lock);
2111 			} else {
2112 				arc_buf_destroy(hdr->b_l1hdr.b_buf, TRUE);
2113 			}
2114 		}
2115 #ifdef ZFS_DEBUG
2116 		if (hdr->b_l1hdr.b_thawed != NULL) {
2117 			kmem_free(hdr->b_l1hdr.b_thawed, 1);
2118 			hdr->b_l1hdr.b_thawed = NULL;
2119 		}
2120 #endif
2121 	}
2122 
2123 	ASSERT3P(hdr->b_hash_next, ==, NULL);
2124 	if (HDR_HAS_L1HDR(hdr)) {
2125 		ASSERT(!multilist_link_active(&hdr->b_l1hdr.b_arc_node));
2126 		ASSERT3P(hdr->b_l1hdr.b_acb, ==, NULL);
2127 		kmem_cache_free(hdr_full_cache, hdr);
2128 	} else {
2129 		kmem_cache_free(hdr_l2only_cache, hdr);
2130 	}
2131 }
2132 
2133 void
2134 arc_buf_free(arc_buf_t *buf, void *tag)
2135 {
2136 	arc_buf_hdr_t *hdr = buf->b_hdr;
2137 	int hashed = hdr->b_l1hdr.b_state != arc_anon;
2138 
2139 	ASSERT(buf->b_efunc == NULL);
2140 	ASSERT(buf->b_data != NULL);
2141 
2142 	if (hashed) {
2143 		kmutex_t *hash_lock = HDR_LOCK(hdr);
2144 
2145 		mutex_enter(hash_lock);
2146 		hdr = buf->b_hdr;
2147 		ASSERT3P(hash_lock, ==, HDR_LOCK(hdr));
2148 
2149 		(void) remove_reference(hdr, hash_lock, tag);
2150 		if (hdr->b_l1hdr.b_datacnt > 1) {
2151 			arc_buf_destroy(buf, TRUE);
2152 		} else {
2153 			ASSERT(buf == hdr->b_l1hdr.b_buf);
2154 			ASSERT(buf->b_efunc == NULL);
2155 			hdr->b_flags |= ARC_FLAG_BUF_AVAILABLE;
2156 		}
2157 		mutex_exit(hash_lock);
2158 	} else if (HDR_IO_IN_PROGRESS(hdr)) {
2159 		int destroy_hdr;
2160 		/*
2161 		 * We are in the middle of an async write.  Don't destroy
2162 		 * this buffer unless the write completes before we finish
2163 		 * decrementing the reference count.
2164 		 */
2165 		mutex_enter(&arc_user_evicts_lock);
2166 		(void) remove_reference(hdr, NULL, tag);
2167 		ASSERT(refcount_is_zero(&hdr->b_l1hdr.b_refcnt));
2168 		destroy_hdr = !HDR_IO_IN_PROGRESS(hdr);
2169 		mutex_exit(&arc_user_evicts_lock);
2170 		if (destroy_hdr)
2171 			arc_hdr_destroy(hdr);
2172 	} else {
2173 		if (remove_reference(hdr, NULL, tag) > 0)
2174 			arc_buf_destroy(buf, TRUE);
2175 		else
2176 			arc_hdr_destroy(hdr);
2177 	}
2178 }
2179 
2180 boolean_t
2181 arc_buf_remove_ref(arc_buf_t *buf, void* tag)
2182 {
2183 	arc_buf_hdr_t *hdr = buf->b_hdr;
2184 	kmutex_t *hash_lock = HDR_LOCK(hdr);
2185 	boolean_t no_callback = (buf->b_efunc == NULL);
2186 
2187 	if (hdr->b_l1hdr.b_state == arc_anon) {
2188 		ASSERT(hdr->b_l1hdr.b_datacnt == 1);
2189 		arc_buf_free(buf, tag);
2190 		return (no_callback);
2191 	}
2192 
2193 	mutex_enter(hash_lock);
2194 	hdr = buf->b_hdr;
2195 	ASSERT(hdr->b_l1hdr.b_datacnt > 0);
2196 	ASSERT3P(hash_lock, ==, HDR_LOCK(hdr));
2197 	ASSERT(hdr->b_l1hdr.b_state != arc_anon);
2198 	ASSERT(buf->b_data != NULL);
2199 
2200 	(void) remove_reference(hdr, hash_lock, tag);
2201 	if (hdr->b_l1hdr.b_datacnt > 1) {
2202 		if (no_callback)
2203 			arc_buf_destroy(buf, TRUE);
2204 	} else if (no_callback) {
2205 		ASSERT(hdr->b_l1hdr.b_buf == buf && buf->b_next == NULL);
2206 		ASSERT(buf->b_efunc == NULL);
2207 		hdr->b_flags |= ARC_FLAG_BUF_AVAILABLE;
2208 	}
2209 	ASSERT(no_callback || hdr->b_l1hdr.b_datacnt > 1 ||
2210 	    refcount_is_zero(&hdr->b_l1hdr.b_refcnt));
2211 	mutex_exit(hash_lock);
2212 	return (no_callback);
2213 }
2214 
2215 int32_t
2216 arc_buf_size(arc_buf_t *buf)
2217 {
2218 	return (buf->b_hdr->b_size);
2219 }
2220 
2221 /*
2222  * Called from the DMU to determine if the current buffer should be
2223  * evicted. In order to ensure proper locking, the eviction must be initiated
2224  * from the DMU. Return true if the buffer is associated with user data and
2225  * duplicate buffers still exist.
2226  */
2227 boolean_t
2228 arc_buf_eviction_needed(arc_buf_t *buf)
2229 {
2230 	arc_buf_hdr_t *hdr;
2231 	boolean_t evict_needed = B_FALSE;
2232 
2233 	if (zfs_disable_dup_eviction)
2234 		return (B_FALSE);
2235 
2236 	mutex_enter(&buf->b_evict_lock);
2237 	hdr = buf->b_hdr;
2238 	if (hdr == NULL) {
2239 		/*
2240 		 * We are in arc_do_user_evicts(); let that function
2241 		 * perform the eviction.
2242 		 */
2243 		ASSERT(buf->b_data == NULL);
2244 		mutex_exit(&buf->b_evict_lock);
2245 		return (B_FALSE);
2246 	} else if (buf->b_data == NULL) {
2247 		/*
2248 		 * We have already been added to the arc eviction list;
2249 		 * recommend eviction.
2250 		 */
2251 		ASSERT3P(hdr, ==, &arc_eviction_hdr);
2252 		mutex_exit(&buf->b_evict_lock);
2253 		return (B_TRUE);
2254 	}
2255 
2256 	if (hdr->b_l1hdr.b_datacnt > 1 && HDR_ISTYPE_DATA(hdr))
2257 		evict_needed = B_TRUE;
2258 
2259 	mutex_exit(&buf->b_evict_lock);
2260 	return (evict_needed);
2261 }
2262 
2263 /*
2264  * Evict the arc_buf_hdr that is provided as a parameter. The resultant
2265  * state of the header is dependent on it's state prior to entering this
2266  * function. The following transitions are possible:
2267  *
2268  *    - arc_mru -> arc_mru_ghost
2269  *    - arc_mfu -> arc_mfu_ghost
2270  *    - arc_mru_ghost -> arc_l2c_only
2271  *    - arc_mru_ghost -> deleted
2272  *    - arc_mfu_ghost -> arc_l2c_only
2273  *    - arc_mfu_ghost -> deleted
2274  */
2275 static int64_t
2276 arc_evict_hdr(arc_buf_hdr_t *hdr, kmutex_t *hash_lock)
2277 {
2278 	arc_state_t *evicted_state, *state;
2279 	int64_t bytes_evicted = 0;
2280 
2281 	ASSERT(MUTEX_HELD(hash_lock));
2282 	ASSERT(HDR_HAS_L1HDR(hdr));
2283 
2284 	state = hdr->b_l1hdr.b_state;
2285 	if (GHOST_STATE(state)) {
2286 		ASSERT(!HDR_IO_IN_PROGRESS(hdr));
2287 		ASSERT(hdr->b_l1hdr.b_buf == NULL);
2288 
2289 		/*
2290 		 * l2arc_write_buffers() relies on a header's L1 portion
2291 		 * (i.e. it's b_tmp_cdata field) during it's write phase.
2292 		 * Thus, we cannot push a header onto the arc_l2c_only
2293 		 * state (removing it's L1 piece) until the header is
2294 		 * done being written to the l2arc.
2295 		 */
2296 		if (HDR_HAS_L2HDR(hdr) && HDR_L2_WRITING(hdr)) {
2297 			ARCSTAT_BUMP(arcstat_evict_l2_skip);
2298 			return (bytes_evicted);
2299 		}
2300 
2301 		ARCSTAT_BUMP(arcstat_deleted);
2302 		bytes_evicted += hdr->b_size;
2303 
2304 		DTRACE_PROBE1(arc__delete, arc_buf_hdr_t *, hdr);
2305 
2306 		if (HDR_HAS_L2HDR(hdr)) {
2307 			/*
2308 			 * This buffer is cached on the 2nd Level ARC;
2309 			 * don't destroy the header.
2310 			 */
2311 			arc_change_state(arc_l2c_only, hdr, hash_lock);
2312 			/*
2313 			 * dropping from L1+L2 cached to L2-only,
2314 			 * realloc to remove the L1 header.
2315 			 */
2316 			hdr = arc_hdr_realloc(hdr, hdr_full_cache,
2317 			    hdr_l2only_cache);
2318 		} else {
2319 			arc_change_state(arc_anon, hdr, hash_lock);
2320 			arc_hdr_destroy(hdr);
2321 		}
2322 		return (bytes_evicted);
2323 	}
2324 
2325 	ASSERT(state == arc_mru || state == arc_mfu);
2326 	evicted_state = (state == arc_mru) ? arc_mru_ghost : arc_mfu_ghost;
2327 
2328 	/* prefetch buffers have a minimum lifespan */
2329 	if (HDR_IO_IN_PROGRESS(hdr) ||
2330 	    ((hdr->b_flags & (ARC_FLAG_PREFETCH | ARC_FLAG_INDIRECT)) &&
2331 	    ddi_get_lbolt() - hdr->b_l1hdr.b_arc_access <
2332 	    arc_min_prefetch_lifespan)) {
2333 		ARCSTAT_BUMP(arcstat_evict_skip);
2334 		return (bytes_evicted);
2335 	}
2336 
2337 	ASSERT0(refcount_count(&hdr->b_l1hdr.b_refcnt));
2338 	ASSERT3U(hdr->b_l1hdr.b_datacnt, >, 0);
2339 	while (hdr->b_l1hdr.b_buf) {
2340 		arc_buf_t *buf = hdr->b_l1hdr.b_buf;
2341 		if (!mutex_tryenter(&buf->b_evict_lock)) {
2342 			ARCSTAT_BUMP(arcstat_mutex_miss);
2343 			break;
2344 		}
2345 		if (buf->b_data != NULL)
2346 			bytes_evicted += hdr->b_size;
2347 		if (buf->b_efunc != NULL) {
2348 			mutex_enter(&arc_user_evicts_lock);
2349 			arc_buf_destroy(buf, FALSE);
2350 			hdr->b_l1hdr.b_buf = buf->b_next;
2351 			buf->b_hdr = &arc_eviction_hdr;
2352 			buf->b_next = arc_eviction_list;
2353 			arc_eviction_list = buf;
2354 			cv_signal(&arc_user_evicts_cv);
2355 			mutex_exit(&arc_user_evicts_lock);
2356 			mutex_exit(&buf->b_evict_lock);
2357 		} else {
2358 			mutex_exit(&buf->b_evict_lock);
2359 			arc_buf_destroy(buf, TRUE);
2360 		}
2361 	}
2362 
2363 	if (HDR_HAS_L2HDR(hdr)) {
2364 		ARCSTAT_INCR(arcstat_evict_l2_cached, hdr->b_size);
2365 	} else {
2366 		if (l2arc_write_eligible(hdr->b_spa, hdr))
2367 			ARCSTAT_INCR(arcstat_evict_l2_eligible, hdr->b_size);
2368 		else
2369 			ARCSTAT_INCR(arcstat_evict_l2_ineligible, hdr->b_size);
2370 	}
2371 
2372 	if (hdr->b_l1hdr.b_datacnt == 0) {
2373 		arc_change_state(evicted_state, hdr, hash_lock);
2374 		ASSERT(HDR_IN_HASH_TABLE(hdr));
2375 		hdr->b_flags |= ARC_FLAG_IN_HASH_TABLE;
2376 		hdr->b_flags &= ~ARC_FLAG_BUF_AVAILABLE;
2377 		DTRACE_PROBE1(arc__evict, arc_buf_hdr_t *, hdr);
2378 	}
2379 
2380 	return (bytes_evicted);
2381 }
2382 
2383 static uint64_t
2384 arc_evict_state_impl(multilist_t *ml, int idx, arc_buf_hdr_t *marker,
2385     uint64_t spa, int64_t bytes)
2386 {
2387 	multilist_sublist_t *mls;
2388 	uint64_t bytes_evicted = 0;
2389 	arc_buf_hdr_t *hdr;
2390 	kmutex_t *hash_lock;
2391 	int evict_count = 0;
2392 
2393 	ASSERT3P(marker, !=, NULL);
2394 	IMPLY(bytes < 0, bytes == ARC_EVICT_ALL);
2395 
2396 	mls = multilist_sublist_lock(ml, idx);
2397 
2398 	for (hdr = multilist_sublist_prev(mls, marker); hdr != NULL;
2399 	    hdr = multilist_sublist_prev(mls, marker)) {
2400 		if ((bytes != ARC_EVICT_ALL && bytes_evicted >= bytes) ||
2401 		    (evict_count >= zfs_arc_evict_batch_limit))
2402 			break;
2403 
2404 		/*
2405 		 * To keep our iteration location, move the marker
2406 		 * forward. Since we're not holding hdr's hash lock, we
2407 		 * must be very careful and not remove 'hdr' from the
2408 		 * sublist. Otherwise, other consumers might mistake the
2409 		 * 'hdr' as not being on a sublist when they call the
2410 		 * multilist_link_active() function (they all rely on
2411 		 * the hash lock protecting concurrent insertions and
2412 		 * removals). multilist_sublist_move_forward() was
2413 		 * specifically implemented to ensure this is the case
2414 		 * (only 'marker' will be removed and re-inserted).
2415 		 */
2416 		multilist_sublist_move_forward(mls, marker);
2417 
2418 		/*
2419 		 * The only case where the b_spa field should ever be
2420 		 * zero, is the marker headers inserted by
2421 		 * arc_evict_state(). It's possible for multiple threads
2422 		 * to be calling arc_evict_state() concurrently (e.g.
2423 		 * dsl_pool_close() and zio_inject_fault()), so we must
2424 		 * skip any markers we see from these other threads.
2425 		 */
2426 		if (hdr->b_spa == 0)
2427 			continue;
2428 
2429 		/* we're only interested in evicting buffers of a certain spa */
2430 		if (spa != 0 && hdr->b_spa != spa) {
2431 			ARCSTAT_BUMP(arcstat_evict_skip);
2432 			continue;
2433 		}
2434 
2435 		hash_lock = HDR_LOCK(hdr);
2436 
2437 		/*
2438 		 * We aren't calling this function from any code path
2439 		 * that would already be holding a hash lock, so we're
2440 		 * asserting on this assumption to be defensive in case
2441 		 * this ever changes. Without this check, it would be
2442 		 * possible to incorrectly increment arcstat_mutex_miss
2443 		 * below (e.g. if the code changed such that we called
2444 		 * this function with a hash lock held).
2445 		 */
2446 		ASSERT(!MUTEX_HELD(hash_lock));
2447 
2448 		if (mutex_tryenter(hash_lock)) {
2449 			uint64_t evicted = arc_evict_hdr(hdr, hash_lock);
2450 			mutex_exit(hash_lock);
2451 
2452 			bytes_evicted += evicted;
2453 
2454 			/*
2455 			 * If evicted is zero, arc_evict_hdr() must have
2456 			 * decided to skip this header, don't increment
2457 			 * evict_count in this case.
2458 			 */
2459 			if (evicted != 0)
2460 				evict_count++;
2461 
2462 			/*
2463 			 * If arc_size isn't overflowing, signal any
2464 			 * threads that might happen to be waiting.
2465 			 *
2466 			 * For each header evicted, we wake up a single
2467 			 * thread. If we used cv_broadcast, we could
2468 			 * wake up "too many" threads causing arc_size
2469 			 * to significantly overflow arc_c; since
2470 			 * arc_get_data_buf() doesn't check for overflow
2471 			 * when it's woken up (it doesn't because it's
2472 			 * possible for the ARC to be overflowing while
2473 			 * full of un-evictable buffers, and the
2474 			 * function should proceed in this case).
2475 			 *
2476 			 * If threads are left sleeping, due to not
2477 			 * using cv_broadcast, they will be woken up
2478 			 * just before arc_reclaim_thread() sleeps.
2479 			 */
2480 			mutex_enter(&arc_reclaim_lock);
2481 			if (!arc_is_overflowing())
2482 				cv_signal(&arc_reclaim_waiters_cv);
2483 			mutex_exit(&arc_reclaim_lock);
2484 		} else {
2485 			ARCSTAT_BUMP(arcstat_mutex_miss);
2486 		}
2487 	}
2488 
2489 	multilist_sublist_unlock(mls);
2490 
2491 	return (bytes_evicted);
2492 }
2493 
2494 /*
2495  * Evict buffers from the given arc state, until we've removed the
2496  * specified number of bytes. Move the removed buffers to the
2497  * appropriate evict state.
2498  *
2499  * This function makes a "best effort". It skips over any buffers
2500  * it can't get a hash_lock on, and so, may not catch all candidates.
2501  * It may also return without evicting as much space as requested.
2502  *
2503  * If bytes is specified using the special value ARC_EVICT_ALL, this
2504  * will evict all available (i.e. unlocked and evictable) buffers from
2505  * the given arc state; which is used by arc_flush().
2506  */
2507 static uint64_t
2508 arc_evict_state(arc_state_t *state, uint64_t spa, int64_t bytes,
2509     arc_buf_contents_t type)
2510 {
2511 	uint64_t total_evicted = 0;
2512 	multilist_t *ml = &state->arcs_list[type];
2513 	int num_sublists;
2514 	arc_buf_hdr_t **markers;
2515 
2516 	IMPLY(bytes < 0, bytes == ARC_EVICT_ALL);
2517 
2518 	num_sublists = multilist_get_num_sublists(ml);
2519 
2520 	/*
2521 	 * If we've tried to evict from each sublist, made some
2522 	 * progress, but still have not hit the target number of bytes
2523 	 * to evict, we want to keep trying. The markers allow us to
2524 	 * pick up where we left off for each individual sublist, rather
2525 	 * than starting from the tail each time.
2526 	 */
2527 	markers = kmem_zalloc(sizeof (*markers) * num_sublists, KM_SLEEP);
2528 	for (int i = 0; i < num_sublists; i++) {
2529 		markers[i] = kmem_cache_alloc(hdr_full_cache, KM_SLEEP);
2530 
2531 		/*
2532 		 * A b_spa of 0 is used to indicate that this header is
2533 		 * a marker. This fact is used in arc_adjust_type() and
2534 		 * arc_evict_state_impl().
2535 		 */
2536 		markers[i]->b_spa = 0;
2537 
2538 		multilist_sublist_t *mls = multilist_sublist_lock(ml, i);
2539 		multilist_sublist_insert_tail(mls, markers[i]);
2540 		multilist_sublist_unlock(mls);
2541 	}
2542 
2543 	/*
2544 	 * While we haven't hit our target number of bytes to evict, or
2545 	 * we're evicting all available buffers.
2546 	 */
2547 	while (total_evicted < bytes || bytes == ARC_EVICT_ALL) {
2548 		/*
2549 		 * Start eviction using a randomly selected sublist,
2550 		 * this is to try and evenly balance eviction across all
2551 		 * sublists. Always starting at the same sublist
2552 		 * (e.g. index 0) would cause evictions to favor certain
2553 		 * sublists over others.
2554 		 */
2555 		int sublist_idx = multilist_get_random_index(ml);
2556 		uint64_t scan_evicted = 0;
2557 
2558 		for (int i = 0; i < num_sublists; i++) {
2559 			uint64_t bytes_remaining;
2560 			uint64_t bytes_evicted;
2561 
2562 			if (bytes == ARC_EVICT_ALL)
2563 				bytes_remaining = ARC_EVICT_ALL;
2564 			else if (total_evicted < bytes)
2565 				bytes_remaining = bytes - total_evicted;
2566 			else
2567 				break;
2568 
2569 			bytes_evicted = arc_evict_state_impl(ml, sublist_idx,
2570 			    markers[sublist_idx], spa, bytes_remaining);
2571 
2572 			scan_evicted += bytes_evicted;
2573 			total_evicted += bytes_evicted;
2574 
2575 			/* we've reached the end, wrap to the beginning */
2576 			if (++sublist_idx >= num_sublists)
2577 				sublist_idx = 0;
2578 		}
2579 
2580 		/*
2581 		 * If we didn't evict anything during this scan, we have
2582 		 * no reason to believe we'll evict more during another
2583 		 * scan, so break the loop.
2584 		 */
2585 		if (scan_evicted == 0) {
2586 			/* This isn't possible, let's make that obvious */
2587 			ASSERT3S(bytes, !=, 0);
2588 
2589 			/*
2590 			 * When bytes is ARC_EVICT_ALL, the only way to
2591 			 * break the loop is when scan_evicted is zero.
2592 			 * In that case, we actually have evicted enough,
2593 			 * so we don't want to increment the kstat.
2594 			 */
2595 			if (bytes != ARC_EVICT_ALL) {
2596 				ASSERT3S(total_evicted, <, bytes);
2597 				ARCSTAT_BUMP(arcstat_evict_not_enough);
2598 			}
2599 
2600 			break;
2601 		}
2602 	}
2603 
2604 	for (int i = 0; i < num_sublists; i++) {
2605 		multilist_sublist_t *mls = multilist_sublist_lock(ml, i);
2606 		multilist_sublist_remove(mls, markers[i]);
2607 		multilist_sublist_unlock(mls);
2608 
2609 		kmem_cache_free(hdr_full_cache, markers[i]);
2610 	}
2611 	kmem_free(markers, sizeof (*markers) * num_sublists);
2612 
2613 	return (total_evicted);
2614 }
2615 
2616 /*
2617  * Flush all "evictable" data of the given type from the arc state
2618  * specified. This will not evict any "active" buffers (i.e. referenced).
2619  *
2620  * When 'retry' is set to FALSE, the function will make a single pass
2621  * over the state and evict any buffers that it can. Since it doesn't
2622  * continually retry the eviction, it might end up leaving some buffers
2623  * in the ARC due to lock misses.
2624  *
2625  * When 'retry' is set to TRUE, the function will continually retry the
2626  * eviction until *all* evictable buffers have been removed from the
2627  * state. As a result, if concurrent insertions into the state are
2628  * allowed (e.g. if the ARC isn't shutting down), this function might
2629  * wind up in an infinite loop, continually trying to evict buffers.
2630  */
2631 static uint64_t
2632 arc_flush_state(arc_state_t *state, uint64_t spa, arc_buf_contents_t type,
2633     boolean_t retry)
2634 {
2635 	uint64_t evicted = 0;
2636 
2637 	while (state->arcs_lsize[type] != 0) {
2638 		evicted += arc_evict_state(state, spa, ARC_EVICT_ALL, type);
2639 
2640 		if (!retry)
2641 			break;
2642 	}
2643 
2644 	return (evicted);
2645 }
2646 
2647 /*
2648  * Evict the specified number of bytes from the state specified,
2649  * restricting eviction to the spa and type given. This function
2650  * prevents us from trying to evict more from a state's list than
2651  * is "evictable", and to skip evicting altogether when passed a
2652  * negative value for "bytes". In contrast, arc_evict_state() will
2653  * evict everything it can, when passed a negative value for "bytes".
2654  */
2655 static uint64_t
2656 arc_adjust_impl(arc_state_t *state, uint64_t spa, int64_t bytes,
2657     arc_buf_contents_t type)
2658 {
2659 	int64_t delta;
2660 
2661 	if (bytes > 0 && state->arcs_lsize[type] > 0) {
2662 		delta = MIN(state->arcs_lsize[type], bytes);
2663 		return (arc_evict_state(state, spa, delta, type));
2664 	}
2665 
2666 	return (0);
2667 }
2668 
2669 /*
2670  * Evict metadata buffers from the cache, such that arc_meta_used is
2671  * capped by the arc_meta_limit tunable.
2672  */
2673 static uint64_t
2674 arc_adjust_meta(void)
2675 {
2676 	uint64_t total_evicted = 0;
2677 	int64_t target;
2678 
2679 	/*
2680 	 * If we're over the meta limit, we want to evict enough
2681 	 * metadata to get back under the meta limit. We don't want to
2682 	 * evict so much that we drop the MRU below arc_p, though. If
2683 	 * we're over the meta limit more than we're over arc_p, we
2684 	 * evict some from the MRU here, and some from the MFU below.
2685 	 */
2686 	target = MIN((int64_t)(arc_meta_used - arc_meta_limit),
2687 	    (int64_t)(arc_anon->arcs_size + arc_mru->arcs_size - arc_p));
2688 
2689 	total_evicted += arc_adjust_impl(arc_mru, 0, target, ARC_BUFC_METADATA);
2690 
2691 	/*
2692 	 * Similar to the above, we want to evict enough bytes to get us
2693 	 * below the meta limit, but not so much as to drop us below the
2694 	 * space alloted to the MFU (which is defined as arc_c - arc_p).
2695 	 */
2696 	target = MIN((int64_t)(arc_meta_used - arc_meta_limit),
2697 	    (int64_t)(arc_mfu->arcs_size - (arc_c - arc_p)));
2698 
2699 	total_evicted += arc_adjust_impl(arc_mfu, 0, target, ARC_BUFC_METADATA);
2700 
2701 	return (total_evicted);
2702 }
2703 
2704 /*
2705  * Return the type of the oldest buffer in the given arc state
2706  *
2707  * This function will select a random sublist of type ARC_BUFC_DATA and
2708  * a random sublist of type ARC_BUFC_METADATA. The tail of each sublist
2709  * is compared, and the type which contains the "older" buffer will be
2710  * returned.
2711  */
2712 static arc_buf_contents_t
2713 arc_adjust_type(arc_state_t *state)
2714 {
2715 	multilist_t *data_ml = &state->arcs_list[ARC_BUFC_DATA];
2716 	multilist_t *meta_ml = &state->arcs_list[ARC_BUFC_METADATA];
2717 	int data_idx = multilist_get_random_index(data_ml);
2718 	int meta_idx = multilist_get_random_index(meta_ml);
2719 	multilist_sublist_t *data_mls;
2720 	multilist_sublist_t *meta_mls;
2721 	arc_buf_contents_t type;
2722 	arc_buf_hdr_t *data_hdr;
2723 	arc_buf_hdr_t *meta_hdr;
2724 
2725 	/*
2726 	 * We keep the sublist lock until we're finished, to prevent
2727 	 * the headers from being destroyed via arc_evict_state().
2728 	 */
2729 	data_mls = multilist_sublist_lock(data_ml, data_idx);
2730 	meta_mls = multilist_sublist_lock(meta_ml, meta_idx);
2731 
2732 	/*
2733 	 * These two loops are to ensure we skip any markers that
2734 	 * might be at the tail of the lists due to arc_evict_state().
2735 	 */
2736 
2737 	for (data_hdr = multilist_sublist_tail(data_mls); data_hdr != NULL;
2738 	    data_hdr = multilist_sublist_prev(data_mls, data_hdr)) {
2739 		if (data_hdr->b_spa != 0)
2740 			break;
2741 	}
2742 
2743 	for (meta_hdr = multilist_sublist_tail(meta_mls); meta_hdr != NULL;
2744 	    meta_hdr = multilist_sublist_prev(meta_mls, meta_hdr)) {
2745 		if (meta_hdr->b_spa != 0)
2746 			break;
2747 	}
2748 
2749 	if (data_hdr == NULL && meta_hdr == NULL) {
2750 		type = ARC_BUFC_DATA;
2751 	} else if (data_hdr == NULL) {
2752 		ASSERT3P(meta_hdr, !=, NULL);
2753 		type = ARC_BUFC_METADATA;
2754 	} else if (meta_hdr == NULL) {
2755 		ASSERT3P(data_hdr, !=, NULL);
2756 		type = ARC_BUFC_DATA;
2757 	} else {
2758 		ASSERT3P(data_hdr, !=, NULL);
2759 		ASSERT3P(meta_hdr, !=, NULL);
2760 
2761 		/* The headers can't be on the sublist without an L1 header */
2762 		ASSERT(HDR_HAS_L1HDR(data_hdr));
2763 		ASSERT(HDR_HAS_L1HDR(meta_hdr));
2764 
2765 		if (data_hdr->b_l1hdr.b_arc_access <
2766 		    meta_hdr->b_l1hdr.b_arc_access) {
2767 			type = ARC_BUFC_DATA;
2768 		} else {
2769 			type = ARC_BUFC_METADATA;
2770 		}
2771 	}
2772 
2773 	multilist_sublist_unlock(meta_mls);
2774 	multilist_sublist_unlock(data_mls);
2775 
2776 	return (type);
2777 }
2778 
2779 /*
2780  * Evict buffers from the cache, such that arc_size is capped by arc_c.
2781  */
2782 static uint64_t
2783 arc_adjust(void)
2784 {
2785 	uint64_t total_evicted = 0;
2786 	uint64_t bytes;
2787 	int64_t target;
2788 
2789 	/*
2790 	 * If we're over arc_meta_limit, we want to correct that before
2791 	 * potentially evicting data buffers below.
2792 	 */
2793 	total_evicted += arc_adjust_meta();
2794 
2795 	/*
2796 	 * Adjust MRU size
2797 	 *
2798 	 * If we're over the target cache size, we want to evict enough
2799 	 * from the list to get back to our target size. We don't want
2800 	 * to evict too much from the MRU, such that it drops below
2801 	 * arc_p. So, if we're over our target cache size more than
2802 	 * the MRU is over arc_p, we'll evict enough to get back to
2803 	 * arc_p here, and then evict more from the MFU below.
2804 	 */
2805 	target = MIN((int64_t)(arc_size - arc_c),
2806 	    (int64_t)(arc_anon->arcs_size + arc_mru->arcs_size + arc_meta_used -
2807 	    arc_p));
2808 
2809 	/*
2810 	 * If we're below arc_meta_min, always prefer to evict data.
2811 	 * Otherwise, try to satisfy the requested number of bytes to
2812 	 * evict from the type which contains older buffers; in an
2813 	 * effort to keep newer buffers in the cache regardless of their
2814 	 * type. If we cannot satisfy the number of bytes from this
2815 	 * type, spill over into the next type.
2816 	 */
2817 	if (arc_adjust_type(arc_mru) == ARC_BUFC_METADATA &&
2818 	    arc_meta_used > arc_meta_min) {
2819 		bytes = arc_adjust_impl(arc_mru, 0, target, ARC_BUFC_METADATA);
2820 		total_evicted += bytes;
2821 
2822 		/*
2823 		 * If we couldn't evict our target number of bytes from
2824 		 * metadata, we try to get the rest from data.
2825 		 */
2826 		target -= bytes;
2827 
2828 		total_evicted +=
2829 		    arc_adjust_impl(arc_mru, 0, target, ARC_BUFC_DATA);
2830 	} else {
2831 		bytes = arc_adjust_impl(arc_mru, 0, target, ARC_BUFC_DATA);
2832 		total_evicted += bytes;
2833 
2834 		/*
2835 		 * If we couldn't evict our target number of bytes from
2836 		 * data, we try to get the rest from metadata.
2837 		 */
2838 		target -= bytes;
2839 
2840 		total_evicted +=
2841 		    arc_adjust_impl(arc_mru, 0, target, ARC_BUFC_METADATA);
2842 	}
2843 
2844 	/*
2845 	 * Adjust MFU size
2846 	 *
2847 	 * Now that we've tried to evict enough from the MRU to get its
2848 	 * size back to arc_p, if we're still above the target cache
2849 	 * size, we evict the rest from the MFU.
2850 	 */
2851 	target = arc_size - arc_c;
2852 
2853 	if (arc_adjust_type(arc_mru) == ARC_BUFC_METADATA &&
2854 	    arc_meta_used > arc_meta_min) {
2855 		bytes = arc_adjust_impl(arc_mfu, 0, target, ARC_BUFC_METADATA);
2856 		total_evicted += bytes;
2857 
2858 		/*
2859 		 * If we couldn't evict our target number of bytes from
2860 		 * metadata, we try to get the rest from data.
2861 		 */
2862 		target -= bytes;
2863 
2864 		total_evicted +=
2865 		    arc_adjust_impl(arc_mfu, 0, target, ARC_BUFC_DATA);
2866 	} else {
2867 		bytes = arc_adjust_impl(arc_mfu, 0, target, ARC_BUFC_DATA);
2868 		total_evicted += bytes;
2869 
2870 		/*
2871 		 * If we couldn't evict our target number of bytes from
2872 		 * data, we try to get the rest from data.
2873 		 */
2874 		target -= bytes;
2875 
2876 		total_evicted +=
2877 		    arc_adjust_impl(arc_mfu, 0, target, ARC_BUFC_METADATA);
2878 	}
2879 
2880 	/*
2881 	 * Adjust ghost lists
2882 	 *
2883 	 * In addition to the above, the ARC also defines target values
2884 	 * for the ghost lists. The sum of the mru list and mru ghost
2885 	 * list should never exceed the target size of the cache, and
2886 	 * the sum of the mru list, mfu list, mru ghost list, and mfu
2887 	 * ghost list should never exceed twice the target size of the
2888 	 * cache. The following logic enforces these limits on the ghost
2889 	 * caches, and evicts from them as needed.
2890 	 */
2891 	target = arc_mru->arcs_size + arc_mru_ghost->arcs_size - arc_c;
2892 
2893 	bytes = arc_adjust_impl(arc_mru_ghost, 0, target, ARC_BUFC_DATA);
2894 	total_evicted += bytes;
2895 
2896 	target -= bytes;
2897 
2898 	total_evicted +=
2899 	    arc_adjust_impl(arc_mru_ghost, 0, target, ARC_BUFC_METADATA);
2900 
2901 	/*
2902 	 * We assume the sum of the mru list and mfu list is less than
2903 	 * or equal to arc_c (we enforced this above), which means we
2904 	 * can use the simpler of the two equations below:
2905 	 *
2906 	 *	mru + mfu + mru ghost + mfu ghost <= 2 * arc_c
2907 	 *		    mru ghost + mfu ghost <= arc_c
2908 	 */
2909 	target = arc_mru_ghost->arcs_size + arc_mfu_ghost->arcs_size - arc_c;
2910 
2911 	bytes = arc_adjust_impl(arc_mfu_ghost, 0, target, ARC_BUFC_DATA);
2912 	total_evicted += bytes;
2913 
2914 	target -= bytes;
2915 
2916 	total_evicted +=
2917 	    arc_adjust_impl(arc_mfu_ghost, 0, target, ARC_BUFC_METADATA);
2918 
2919 	return (total_evicted);
2920 }
2921 
2922 static void
2923 arc_do_user_evicts(void)
2924 {
2925 	mutex_enter(&arc_user_evicts_lock);
2926 	while (arc_eviction_list != NULL) {
2927 		arc_buf_t *buf = arc_eviction_list;
2928 		arc_eviction_list = buf->b_next;
2929 		mutex_enter(&buf->b_evict_lock);
2930 		buf->b_hdr = NULL;
2931 		mutex_exit(&buf->b_evict_lock);
2932 		mutex_exit(&arc_user_evicts_lock);
2933 
2934 		if (buf->b_efunc != NULL)
2935 			VERIFY0(buf->b_efunc(buf->b_private));
2936 
2937 		buf->b_efunc = NULL;
2938 		buf->b_private = NULL;
2939 		kmem_cache_free(buf_cache, buf);
2940 		mutex_enter(&arc_user_evicts_lock);
2941 	}
2942 	mutex_exit(&arc_user_evicts_lock);
2943 }
2944 
2945 void
2946 arc_flush(spa_t *spa, boolean_t retry)
2947 {
2948 	uint64_t guid = 0;
2949 
2950 	/*
2951 	 * If retry is TRUE, a spa must not be specified since we have
2952 	 * no good way to determine if all of a spa's buffers have been
2953 	 * evicted from an arc state.
2954 	 */
2955 	ASSERT(!retry || spa == 0);
2956 
2957 	if (spa != NULL)
2958 		guid = spa_load_guid(spa);
2959 
2960 	(void) arc_flush_state(arc_mru, guid, ARC_BUFC_DATA, retry);
2961 	(void) arc_flush_state(arc_mru, guid, ARC_BUFC_METADATA, retry);
2962 
2963 	(void) arc_flush_state(arc_mfu, guid, ARC_BUFC_DATA, retry);
2964 	(void) arc_flush_state(arc_mfu, guid, ARC_BUFC_METADATA, retry);
2965 
2966 	(void) arc_flush_state(arc_mru_ghost, guid, ARC_BUFC_DATA, retry);
2967 	(void) arc_flush_state(arc_mru_ghost, guid, ARC_BUFC_METADATA, retry);
2968 
2969 	(void) arc_flush_state(arc_mfu_ghost, guid, ARC_BUFC_DATA, retry);
2970 	(void) arc_flush_state(arc_mfu_ghost, guid, ARC_BUFC_METADATA, retry);
2971 
2972 	arc_do_user_evicts();
2973 	ASSERT(spa || arc_eviction_list == NULL);
2974 }
2975 
2976 void
2977 arc_shrink(int64_t to_free)
2978 {
2979 	if (arc_c > arc_c_min) {
2980 
2981 		if (arc_c > arc_c_min + to_free)
2982 			atomic_add_64(&arc_c, -to_free);
2983 		else
2984 			arc_c = arc_c_min;
2985 
2986 		atomic_add_64(&arc_p, -(arc_p >> arc_shrink_shift));
2987 		if (arc_c > arc_size)
2988 			arc_c = MAX(arc_size, arc_c_min);
2989 		if (arc_p > arc_c)
2990 			arc_p = (arc_c >> 1);
2991 		ASSERT(arc_c >= arc_c_min);
2992 		ASSERT((int64_t)arc_p >= 0);
2993 	}
2994 
2995 	if (arc_size > arc_c)
2996 		(void) arc_adjust();
2997 }
2998 
2999 typedef enum free_memory_reason_t {
3000 	FMR_UNKNOWN,
3001 	FMR_NEEDFREE,
3002 	FMR_LOTSFREE,
3003 	FMR_SWAPFS_MINFREE,
3004 	FMR_PAGES_PP_MAXIMUM,
3005 	FMR_HEAP_ARENA,
3006 	FMR_ZIO_ARENA,
3007 } free_memory_reason_t;
3008 
3009 int64_t last_free_memory;
3010 free_memory_reason_t last_free_reason;
3011 
3012 /*
3013  * Additional reserve of pages for pp_reserve.
3014  */
3015 int64_t arc_pages_pp_reserve = 64;
3016 
3017 /*
3018  * Additional reserve of pages for swapfs.
3019  */
3020 int64_t arc_swapfs_reserve = 64;
3021 
3022 /*
3023  * Return the amount of memory that can be consumed before reclaim will be
3024  * needed.  Positive if there is sufficient free memory, negative indicates
3025  * the amount of memory that needs to be freed up.
3026  */
3027 static int64_t
3028 arc_available_memory(void)
3029 {
3030 	int64_t lowest = INT64_MAX;
3031 	int64_t n;
3032 	free_memory_reason_t r = FMR_UNKNOWN;
3033 
3034 #ifdef _KERNEL
3035 	if (needfree > 0) {
3036 		n = PAGESIZE * (-needfree);
3037 		if (n < lowest) {
3038 			lowest = n;
3039 			r = FMR_NEEDFREE;
3040 		}
3041 	}
3042 
3043 	/*
3044 	 * check that we're out of range of the pageout scanner.  It starts to
3045 	 * schedule paging if freemem is less than lotsfree and needfree.
3046 	 * lotsfree is the high-water mark for pageout, and needfree is the
3047 	 * number of needed free pages.  We add extra pages here to make sure
3048 	 * the scanner doesn't start up while we're freeing memory.
3049 	 */
3050 	n = PAGESIZE * (freemem - lotsfree - needfree - desfree);
3051 	if (n < lowest) {
3052 		lowest = n;
3053 		r = FMR_LOTSFREE;
3054 	}
3055 
3056 	/*
3057 	 * check to make sure that swapfs has enough space so that anon
3058 	 * reservations can still succeed. anon_resvmem() checks that the
3059 	 * availrmem is greater than swapfs_minfree, and the number of reserved
3060 	 * swap pages.  We also add a bit of extra here just to prevent
3061 	 * circumstances from getting really dire.
3062 	 */
3063 	n = PAGESIZE * (availrmem - swapfs_minfree - swapfs_reserve -
3064 	    desfree - arc_swapfs_reserve);
3065 	if (n < lowest) {
3066 		lowest = n;
3067 		r = FMR_SWAPFS_MINFREE;
3068 	}
3069 
3070 
3071 	/*
3072 	 * Check that we have enough availrmem that memory locking (e.g., via
3073 	 * mlock(3C) or memcntl(2)) can still succeed.  (pages_pp_maximum
3074 	 * stores the number of pages that cannot be locked; when availrmem
3075 	 * drops below pages_pp_maximum, page locking mechanisms such as
3076 	 * page_pp_lock() will fail.)
3077 	 */
3078 	n = PAGESIZE * (availrmem - pages_pp_maximum -
3079 	    arc_pages_pp_reserve);
3080 	if (n < lowest) {
3081 		lowest = n;
3082 		r = FMR_PAGES_PP_MAXIMUM;
3083 	}
3084 
3085 #if defined(__i386)
3086 	/*
3087 	 * If we're on an i386 platform, it's possible that we'll exhaust the
3088 	 * kernel heap space before we ever run out of available physical
3089 	 * memory.  Most checks of the size of the heap_area compare against
3090 	 * tune.t_minarmem, which is the minimum available real memory that we
3091 	 * can have in the system.  However, this is generally fixed at 25 pages
3092 	 * which is so low that it's useless.  In this comparison, we seek to
3093 	 * calculate the total heap-size, and reclaim if more than 3/4ths of the
3094 	 * heap is allocated.  (Or, in the calculation, if less than 1/4th is
3095 	 * free)
3096 	 */
3097 	n = vmem_size(heap_arena, VMEM_FREE) -
3098 	    (vmem_size(heap_arena, VMEM_FREE | VMEM_ALLOC) >> 2);
3099 	if (n < lowest) {
3100 		lowest = n;
3101 		r = FMR_HEAP_ARENA;
3102 	}
3103 #endif
3104 
3105 	/*
3106 	 * If zio data pages are being allocated out of a separate heap segment,
3107 	 * then enforce that the size of available vmem for this arena remains
3108 	 * above about 1/16th free.
3109 	 *
3110 	 * Note: The 1/16th arena free requirement was put in place
3111 	 * to aggressively evict memory from the arc in order to avoid
3112 	 * memory fragmentation issues.
3113 	 */
3114 	if (zio_arena != NULL) {
3115 		n = vmem_size(zio_arena, VMEM_FREE) -
3116 		    (vmem_size(zio_arena, VMEM_ALLOC) >> 4);
3117 		if (n < lowest) {
3118 			lowest = n;
3119 			r = FMR_ZIO_ARENA;
3120 		}
3121 	}
3122 #else
3123 	/* Every 100 calls, free a small amount */
3124 	if (spa_get_random(100) == 0)
3125 		lowest = -1024;
3126 #endif
3127 
3128 	last_free_memory = lowest;
3129 	last_free_reason = r;
3130 
3131 	return (lowest);
3132 }
3133 
3134 
3135 /*
3136  * Determine if the system is under memory pressure and is asking
3137  * to reclaim memory. A return value of TRUE indicates that the system
3138  * is under memory pressure and that the arc should adjust accordingly.
3139  */
3140 static boolean_t
3141 arc_reclaim_needed(void)
3142 {
3143 	return (arc_available_memory() < 0);
3144 }
3145 
3146 static void
3147 arc_kmem_reap_now(void)
3148 {
3149 	size_t			i;
3150 	kmem_cache_t		*prev_cache = NULL;
3151 	kmem_cache_t		*prev_data_cache = NULL;
3152 	extern kmem_cache_t	*zio_buf_cache[];
3153 	extern kmem_cache_t	*zio_data_buf_cache[];
3154 	extern kmem_cache_t	*range_seg_cache;
3155 
3156 #ifdef _KERNEL
3157 	if (arc_meta_used >= arc_meta_limit) {
3158 		/*
3159 		 * We are exceeding our meta-data cache limit.
3160 		 * Purge some DNLC entries to release holds on meta-data.
3161 		 */
3162 		dnlc_reduce_cache((void *)(uintptr_t)arc_reduce_dnlc_percent);
3163 	}
3164 #if defined(__i386)
3165 	/*
3166 	 * Reclaim unused memory from all kmem caches.
3167 	 */
3168 	kmem_reap();
3169 #endif
3170 #endif
3171 
3172 	for (i = 0; i < SPA_MAXBLOCKSIZE >> SPA_MINBLOCKSHIFT; i++) {
3173 		if (zio_buf_cache[i] != prev_cache) {
3174 			prev_cache = zio_buf_cache[i];
3175 			kmem_cache_reap_now(zio_buf_cache[i]);
3176 		}
3177 		if (zio_data_buf_cache[i] != prev_data_cache) {
3178 			prev_data_cache = zio_data_buf_cache[i];
3179 			kmem_cache_reap_now(zio_data_buf_cache[i]);
3180 		}
3181 	}
3182 	kmem_cache_reap_now(buf_cache);
3183 	kmem_cache_reap_now(hdr_full_cache);
3184 	kmem_cache_reap_now(hdr_l2only_cache);
3185 	kmem_cache_reap_now(range_seg_cache);
3186 
3187 	if (zio_arena != NULL) {
3188 		/*
3189 		 * Ask the vmem arena to reclaim unused memory from its
3190 		 * quantum caches.
3191 		 */
3192 		vmem_qcache_reap(zio_arena);
3193 	}
3194 }
3195 
3196 /*
3197  * Threads can block in arc_get_data_buf() waiting for this thread to evict
3198  * enough data and signal them to proceed. When this happens, the threads in
3199  * arc_get_data_buf() are sleeping while holding the hash lock for their
3200  * particular arc header. Thus, we must be careful to never sleep on a
3201  * hash lock in this thread. This is to prevent the following deadlock:
3202  *
3203  *  - Thread A sleeps on CV in arc_get_data_buf() holding hash lock "L",
3204  *    waiting for the reclaim thread to signal it.
3205  *
3206  *  - arc_reclaim_thread() tries to acquire hash lock "L" using mutex_enter,
3207  *    fails, and goes to sleep forever.
3208  *
3209  * This possible deadlock is avoided by always acquiring a hash lock
3210  * using mutex_tryenter() from arc_reclaim_thread().
3211  */
3212 static void
3213 arc_reclaim_thread(void)
3214 {
3215 	clock_t			growtime = 0;
3216 	callb_cpr_t		cpr;
3217 
3218 	CALLB_CPR_INIT(&cpr, &arc_reclaim_lock, callb_generic_cpr, FTAG);
3219 
3220 	mutex_enter(&arc_reclaim_lock);
3221 	while (!arc_reclaim_thread_exit) {
3222 		int64_t free_memory = arc_available_memory();
3223 		uint64_t evicted = 0;
3224 
3225 		mutex_exit(&arc_reclaim_lock);
3226 
3227 		if (free_memory < 0) {
3228 
3229 			arc_no_grow = B_TRUE;
3230 			arc_warm = B_TRUE;
3231 
3232 			/*
3233 			 * Wait at least zfs_grow_retry (default 60) seconds
3234 			 * before considering growing.
3235 			 */
3236 			growtime = ddi_get_lbolt() + (arc_grow_retry * hz);
3237 
3238 			arc_kmem_reap_now();
3239 
3240 			/*
3241 			 * If we are still low on memory, shrink the ARC
3242 			 * so that we have arc_shrink_min free space.
3243 			 */
3244 			free_memory = arc_available_memory();
3245 
3246 			int64_t to_free =
3247 			    (arc_c >> arc_shrink_shift) - free_memory;
3248 			if (to_free > 0) {
3249 #ifdef _KERNEL
3250 				to_free = MAX(to_free, ptob(needfree));
3251 #endif
3252 				arc_shrink(to_free);
3253 			}
3254 		} else if (free_memory < arc_c >> arc_no_grow_shift) {
3255 			arc_no_grow = B_TRUE;
3256 		} else if (ddi_get_lbolt() >= growtime) {
3257 			arc_no_grow = B_FALSE;
3258 		}
3259 
3260 		evicted = arc_adjust();
3261 
3262 		mutex_enter(&arc_reclaim_lock);
3263 
3264 		/*
3265 		 * If evicted is zero, we couldn't evict anything via
3266 		 * arc_adjust(). This could be due to hash lock
3267 		 * collisions, but more likely due to the majority of
3268 		 * arc buffers being unevictable. Therefore, even if
3269 		 * arc_size is above arc_c, another pass is unlikely to
3270 		 * be helpful and could potentially cause us to enter an
3271 		 * infinite loop.
3272 		 */
3273 		if (arc_size <= arc_c || evicted == 0) {
3274 			/*
3275 			 * We're either no longer overflowing, or we
3276 			 * can't evict anything more, so we should wake
3277 			 * up any threads before we go to sleep.
3278 			 */
3279 			cv_broadcast(&arc_reclaim_waiters_cv);
3280 
3281 			/*
3282 			 * Block until signaled, or after one second (we
3283 			 * might need to perform arc_kmem_reap_now()
3284 			 * even if we aren't being signalled)
3285 			 */
3286 			CALLB_CPR_SAFE_BEGIN(&cpr);
3287 			(void) cv_timedwait(&arc_reclaim_thread_cv,
3288 			    &arc_reclaim_lock, ddi_get_lbolt() + hz);
3289 			CALLB_CPR_SAFE_END(&cpr, &arc_reclaim_lock);
3290 		}
3291 	}
3292 
3293 	arc_reclaim_thread_exit = FALSE;
3294 	cv_broadcast(&arc_reclaim_thread_cv);
3295 	CALLB_CPR_EXIT(&cpr);		/* drops arc_reclaim_lock */
3296 	thread_exit();
3297 }
3298 
3299 static void
3300 arc_user_evicts_thread(void)
3301 {
3302 	callb_cpr_t cpr;
3303 
3304 	CALLB_CPR_INIT(&cpr, &arc_user_evicts_lock, callb_generic_cpr, FTAG);
3305 
3306 	mutex_enter(&arc_user_evicts_lock);
3307 	while (!arc_user_evicts_thread_exit) {
3308 		mutex_exit(&arc_user_evicts_lock);
3309 
3310 		arc_do_user_evicts();
3311 
3312 		/*
3313 		 * This is necessary in order for the mdb ::arc dcmd to
3314 		 * show up to date information. Since the ::arc command
3315 		 * does not call the kstat's update function, without
3316 		 * this call, the command may show stale stats for the
3317 		 * anon, mru, mru_ghost, mfu, and mfu_ghost lists. Even
3318 		 * with this change, the data might be up to 1 second
3319 		 * out of date; but that should suffice. The arc_state_t
3320 		 * structures can be queried directly if more accurate
3321 		 * information is needed.
3322 		 */
3323 		if (arc_ksp != NULL)
3324 			arc_ksp->ks_update(arc_ksp, KSTAT_READ);
3325 
3326 		mutex_enter(&arc_user_evicts_lock);
3327 
3328 		/*
3329 		 * Block until signaled, or after one second (we need to
3330 		 * call the arc's kstat update function regularly).
3331 		 */
3332 		CALLB_CPR_SAFE_BEGIN(&cpr);
3333 		(void) cv_timedwait(&arc_user_evicts_cv,
3334 		    &arc_user_evicts_lock, ddi_get_lbolt() + hz);
3335 		CALLB_CPR_SAFE_END(&cpr, &arc_user_evicts_lock);
3336 	}
3337 
3338 	arc_user_evicts_thread_exit = FALSE;
3339 	cv_broadcast(&arc_user_evicts_cv);
3340 	CALLB_CPR_EXIT(&cpr);		/* drops arc_user_evicts_lock */
3341 	thread_exit();
3342 }
3343 
3344 /*
3345  * Adapt arc info given the number of bytes we are trying to add and
3346  * the state that we are comming from.  This function is only called
3347  * when we are adding new content to the cache.
3348  */
3349 static void
3350 arc_adapt(int bytes, arc_state_t *state)
3351 {
3352 	int mult;
3353 	uint64_t arc_p_min = (arc_c >> arc_p_min_shift);
3354 
3355 	if (state == arc_l2c_only)
3356 		return;
3357 
3358 	ASSERT(bytes > 0);
3359 	/*
3360 	 * Adapt the target size of the MRU list:
3361 	 *	- if we just hit in the MRU ghost list, then increase
3362 	 *	  the target size of the MRU list.
3363 	 *	- if we just hit in the MFU ghost list, then increase
3364 	 *	  the target size of the MFU list by decreasing the
3365 	 *	  target size of the MRU list.
3366 	 */
3367 	if (state == arc_mru_ghost) {
3368 		mult = ((arc_mru_ghost->arcs_size >= arc_mfu_ghost->arcs_size) ?
3369 		    1 : (arc_mfu_ghost->arcs_size/arc_mru_ghost->arcs_size));
3370 		mult = MIN(mult, 10); /* avoid wild arc_p adjustment */
3371 
3372 		arc_p = MIN(arc_c - arc_p_min, arc_p + bytes * mult);
3373 	} else if (state == arc_mfu_ghost) {
3374 		uint64_t delta;
3375 
3376 		mult = ((arc_mfu_ghost->arcs_size >= arc_mru_ghost->arcs_size) ?
3377 		    1 : (arc_mru_ghost->arcs_size/arc_mfu_ghost->arcs_size));
3378 		mult = MIN(mult, 10);
3379 
3380 		delta = MIN(bytes * mult, arc_p);
3381 		arc_p = MAX(arc_p_min, arc_p - delta);
3382 	}
3383 	ASSERT((int64_t)arc_p >= 0);
3384 
3385 	if (arc_reclaim_needed()) {
3386 		cv_signal(&arc_reclaim_thread_cv);
3387 		return;
3388 	}
3389 
3390 	if (arc_no_grow)
3391 		return;
3392 
3393 	if (arc_c >= arc_c_max)
3394 		return;
3395 
3396 	/*
3397 	 * If we're within (2 * maxblocksize) bytes of the target
3398 	 * cache size, increment the target cache size
3399 	 */
3400 	if (arc_size > arc_c - (2ULL << SPA_MAXBLOCKSHIFT)) {
3401 		atomic_add_64(&arc_c, (int64_t)bytes);
3402 		if (arc_c > arc_c_max)
3403 			arc_c = arc_c_max;
3404 		else if (state == arc_anon)
3405 			atomic_add_64(&arc_p, (int64_t)bytes);
3406 		if (arc_p > arc_c)
3407 			arc_p = arc_c;
3408 	}
3409 	ASSERT((int64_t)arc_p >= 0);
3410 }
3411 
3412 /*
3413  * Check if arc_size has grown past our upper threshold, determined by
3414  * zfs_arc_overflow_shift.
3415  */
3416 static boolean_t
3417 arc_is_overflowing(void)
3418 {
3419 	/* Always allow at least one block of overflow */
3420 	uint64_t overflow = MAX(SPA_MAXBLOCKSIZE,
3421 	    arc_c >> zfs_arc_overflow_shift);
3422 
3423 	return (arc_size >= arc_c + overflow);
3424 }
3425 
3426 /*
3427  * The buffer, supplied as the first argument, needs a data block. If we
3428  * are hitting the hard limit for the cache size, we must sleep, waiting
3429  * for the eviction thread to catch up. If we're past the target size
3430  * but below the hard limit, we'll only signal the reclaim thread and
3431  * continue on.
3432  */
3433 static void
3434 arc_get_data_buf(arc_buf_t *buf)
3435 {
3436 	arc_state_t		*state = buf->b_hdr->b_l1hdr.b_state;
3437 	uint64_t		size = buf->b_hdr->b_size;
3438 	arc_buf_contents_t	type = arc_buf_type(buf->b_hdr);
3439 
3440 	arc_adapt(size, state);
3441 
3442 	/*
3443 	 * If arc_size is currently overflowing, and has grown past our
3444 	 * upper limit, we must be adding data faster than the evict
3445 	 * thread can evict. Thus, to ensure we don't compound the
3446 	 * problem by adding more data and forcing arc_size to grow even
3447 	 * further past it's target size, we halt and wait for the
3448 	 * eviction thread to catch up.
3449 	 *
3450 	 * It's also possible that the reclaim thread is unable to evict
3451 	 * enough buffers to get arc_size below the overflow limit (e.g.
3452 	 * due to buffers being un-evictable, or hash lock collisions).
3453 	 * In this case, we want to proceed regardless if we're
3454 	 * overflowing; thus we don't use a while loop here.
3455 	 */
3456 	if (arc_is_overflowing()) {
3457 		mutex_enter(&arc_reclaim_lock);
3458 
3459 		/*
3460 		 * Now that we've acquired the lock, we may no longer be
3461 		 * over the overflow limit, lets check.
3462 		 *
3463 		 * We're ignoring the case of spurious wake ups. If that
3464 		 * were to happen, it'd let this thread consume an ARC
3465 		 * buffer before it should have (i.e. before we're under
3466 		 * the overflow limit and were signalled by the reclaim
3467 		 * thread). As long as that is a rare occurrence, it
3468 		 * shouldn't cause any harm.
3469 		 */
3470 		if (arc_is_overflowing()) {
3471 			cv_signal(&arc_reclaim_thread_cv);
3472 			cv_wait(&arc_reclaim_waiters_cv, &arc_reclaim_lock);
3473 		}
3474 
3475 		mutex_exit(&arc_reclaim_lock);
3476 	}
3477 
3478 	if (type == ARC_BUFC_METADATA) {
3479 		buf->b_data = zio_buf_alloc(size);
3480 		arc_space_consume(size, ARC_SPACE_META);
3481 	} else {
3482 		ASSERT(type == ARC_BUFC_DATA);
3483 		buf->b_data = zio_data_buf_alloc(size);
3484 		arc_space_consume(size, ARC_SPACE_DATA);
3485 	}
3486 
3487 	/*
3488 	 * Update the state size.  Note that ghost states have a
3489 	 * "ghost size" and so don't need to be updated.
3490 	 */
3491 	if (!GHOST_STATE(buf->b_hdr->b_l1hdr.b_state)) {
3492 		arc_buf_hdr_t *hdr = buf->b_hdr;
3493 
3494 		atomic_add_64(&hdr->b_l1hdr.b_state->arcs_size, size);
3495 
3496 		/*
3497 		 * If this is reached via arc_read, the link is
3498 		 * protected by the hash lock. If reached via
3499 		 * arc_buf_alloc, the header should not be accessed by
3500 		 * any other thread. And, if reached via arc_read_done,
3501 		 * the hash lock will protect it if it's found in the
3502 		 * hash table; otherwise no other thread should be
3503 		 * trying to [add|remove]_reference it.
3504 		 */
3505 		if (multilist_link_active(&hdr->b_l1hdr.b_arc_node)) {
3506 			ASSERT(refcount_is_zero(&hdr->b_l1hdr.b_refcnt));
3507 			atomic_add_64(&hdr->b_l1hdr.b_state->arcs_lsize[type],
3508 			    size);
3509 		}
3510 		/*
3511 		 * If we are growing the cache, and we are adding anonymous
3512 		 * data, and we have outgrown arc_p, update arc_p
3513 		 */
3514 		if (arc_size < arc_c && hdr->b_l1hdr.b_state == arc_anon &&
3515 		    arc_anon->arcs_size + arc_mru->arcs_size > arc_p)
3516 			arc_p = MIN(arc_c, arc_p + size);
3517 	}
3518 }
3519 
3520 /*
3521  * This routine is called whenever a buffer is accessed.
3522  * NOTE: the hash lock is dropped in this function.
3523  */
3524 static void
3525 arc_access(arc_buf_hdr_t *hdr, kmutex_t *hash_lock)
3526 {
3527 	clock_t now;
3528 
3529 	ASSERT(MUTEX_HELD(hash_lock));
3530 	ASSERT(HDR_HAS_L1HDR(hdr));
3531 
3532 	if (hdr->b_l1hdr.b_state == arc_anon) {
3533 		/*
3534 		 * This buffer is not in the cache, and does not
3535 		 * appear in our "ghost" list.  Add the new buffer
3536 		 * to the MRU state.
3537 		 */
3538 
3539 		ASSERT0(hdr->b_l1hdr.b_arc_access);
3540 		hdr->b_l1hdr.b_arc_access = ddi_get_lbolt();
3541 		DTRACE_PROBE1(new_state__mru, arc_buf_hdr_t *, hdr);
3542 		arc_change_state(arc_mru, hdr, hash_lock);
3543 
3544 	} else if (hdr->b_l1hdr.b_state == arc_mru) {
3545 		now = ddi_get_lbolt();
3546 
3547 		/*
3548 		 * If this buffer is here because of a prefetch, then either:
3549 		 * - clear the flag if this is a "referencing" read
3550 		 *   (any subsequent access will bump this into the MFU state).
3551 		 * or
3552 		 * - move the buffer to the head of the list if this is
3553 		 *   another prefetch (to make it less likely to be evicted).
3554 		 */
3555 		if (HDR_PREFETCH(hdr)) {
3556 			if (refcount_count(&hdr->b_l1hdr.b_refcnt) == 0) {
3557 				/* link protected by hash lock */
3558 				ASSERT(multilist_link_active(
3559 				    &hdr->b_l1hdr.b_arc_node));
3560 			} else {
3561 				hdr->b_flags &= ~ARC_FLAG_PREFETCH;
3562 				ARCSTAT_BUMP(arcstat_mru_hits);
3563 			}
3564 			hdr->b_l1hdr.b_arc_access = now;
3565 			return;
3566 		}
3567 
3568 		/*
3569 		 * This buffer has been "accessed" only once so far,
3570 		 * but it is still in the cache. Move it to the MFU
3571 		 * state.
3572 		 */
3573 		if (now > hdr->b_l1hdr.b_arc_access + ARC_MINTIME) {
3574 			/*
3575 			 * More than 125ms have passed since we
3576 			 * instantiated this buffer.  Move it to the
3577 			 * most frequently used state.
3578 			 */
3579 			hdr->b_l1hdr.b_arc_access = now;
3580 			DTRACE_PROBE1(new_state__mfu, arc_buf_hdr_t *, hdr);
3581 			arc_change_state(arc_mfu, hdr, hash_lock);
3582 		}
3583 		ARCSTAT_BUMP(arcstat_mru_hits);
3584 	} else if (hdr->b_l1hdr.b_state == arc_mru_ghost) {
3585 		arc_state_t	*new_state;
3586 		/*
3587 		 * This buffer has been "accessed" recently, but
3588 		 * was evicted from the cache.  Move it to the
3589 		 * MFU state.
3590 		 */
3591 
3592 		if (HDR_PREFETCH(hdr)) {
3593 			new_state = arc_mru;
3594 			if (refcount_count(&hdr->b_l1hdr.b_refcnt) > 0)
3595 				hdr->b_flags &= ~ARC_FLAG_PREFETCH;
3596 			DTRACE_PROBE1(new_state__mru, arc_buf_hdr_t *, hdr);
3597 		} else {
3598 			new_state = arc_mfu;
3599 			DTRACE_PROBE1(new_state__mfu, arc_buf_hdr_t *, hdr);
3600 		}
3601 
3602 		hdr->b_l1hdr.b_arc_access = ddi_get_lbolt();
3603 		arc_change_state(new_state, hdr, hash_lock);
3604 
3605 		ARCSTAT_BUMP(arcstat_mru_ghost_hits);
3606 	} else if (hdr->b_l1hdr.b_state == arc_mfu) {
3607 		/*
3608 		 * This buffer has been accessed more than once and is
3609 		 * still in the cache.  Keep it in the MFU state.
3610 		 *
3611 		 * NOTE: an add_reference() that occurred when we did
3612 		 * the arc_read() will have kicked this off the list.
3613 		 * If it was a prefetch, we will explicitly move it to
3614 		 * the head of the list now.
3615 		 */
3616 		if ((HDR_PREFETCH(hdr)) != 0) {
3617 			ASSERT(refcount_is_zero(&hdr->b_l1hdr.b_refcnt));
3618 			/* link protected by hash_lock */
3619 			ASSERT(multilist_link_active(&hdr->b_l1hdr.b_arc_node));
3620 		}
3621 		ARCSTAT_BUMP(arcstat_mfu_hits);
3622 		hdr->b_l1hdr.b_arc_access = ddi_get_lbolt();
3623 	} else if (hdr->b_l1hdr.b_state == arc_mfu_ghost) {
3624 		arc_state_t	*new_state = arc_mfu;
3625 		/*
3626 		 * This buffer has been accessed more than once but has
3627 		 * been evicted from the cache.  Move it back to the
3628 		 * MFU state.
3629 		 */
3630 
3631 		if (HDR_PREFETCH(hdr)) {
3632 			/*
3633 			 * This is a prefetch access...
3634 			 * move this block back to the MRU state.
3635 			 */
3636 			ASSERT0(refcount_count(&hdr->b_l1hdr.b_refcnt));
3637 			new_state = arc_mru;
3638 		}
3639 
3640 		hdr->b_l1hdr.b_arc_access = ddi_get_lbolt();
3641 		DTRACE_PROBE1(new_state__mfu, arc_buf_hdr_t *, hdr);
3642 		arc_change_state(new_state, hdr, hash_lock);
3643 
3644 		ARCSTAT_BUMP(arcstat_mfu_ghost_hits);
3645 	} else if (hdr->b_l1hdr.b_state == arc_l2c_only) {
3646 		/*
3647 		 * This buffer is on the 2nd Level ARC.
3648 		 */
3649 
3650 		hdr->b_l1hdr.b_arc_access = ddi_get_lbolt();
3651 		DTRACE_PROBE1(new_state__mfu, arc_buf_hdr_t *, hdr);
3652 		arc_change_state(arc_mfu, hdr, hash_lock);
3653 	} else {
3654 		ASSERT(!"invalid arc state");
3655 	}
3656 }
3657 
3658 /* a generic arc_done_func_t which you can use */
3659 /* ARGSUSED */
3660 void
3661 arc_bcopy_func(zio_t *zio, arc_buf_t *buf, void *arg)
3662 {
3663 	if (zio == NULL || zio->io_error == 0)
3664 		bcopy(buf->b_data, arg, buf->b_hdr->b_size);
3665 	VERIFY(arc_buf_remove_ref(buf, arg));
3666 }
3667 
3668 /* a generic arc_done_func_t */
3669 void
3670 arc_getbuf_func(zio_t *zio, arc_buf_t *buf, void *arg)
3671 {
3672 	arc_buf_t **bufp = arg;
3673 	if (zio && zio->io_error) {
3674 		VERIFY(arc_buf_remove_ref(buf, arg));
3675 		*bufp = NULL;
3676 	} else {
3677 		*bufp = buf;
3678 		ASSERT(buf->b_data);
3679 	}
3680 }
3681 
3682 static void
3683 arc_read_done(zio_t *zio)
3684 {
3685 	arc_buf_hdr_t	*hdr;
3686 	arc_buf_t	*buf;
3687 	arc_buf_t	*abuf;	/* buffer we're assigning to callback */
3688 	kmutex_t	*hash_lock = NULL;
3689 	arc_callback_t	*callback_list, *acb;
3690 	int		freeable = FALSE;
3691 
3692 	buf = zio->io_private;
3693 	hdr = buf->b_hdr;
3694 
3695 	/*
3696 	 * The hdr was inserted into hash-table and removed from lists
3697 	 * prior to starting I/O.  We should find this header, since
3698 	 * it's in the hash table, and it should be legit since it's
3699 	 * not possible to evict it during the I/O.  The only possible
3700 	 * reason for it not to be found is if we were freed during the
3701 	 * read.
3702 	 */
3703 	if (HDR_IN_HASH_TABLE(hdr)) {
3704 		ASSERT3U(hdr->b_birth, ==, BP_PHYSICAL_BIRTH(zio->io_bp));
3705 		ASSERT3U(hdr->b_dva.dva_word[0], ==,
3706 		    BP_IDENTITY(zio->io_bp)->dva_word[0]);
3707 		ASSERT3U(hdr->b_dva.dva_word[1], ==,
3708 		    BP_IDENTITY(zio->io_bp)->dva_word[1]);
3709 
3710 		arc_buf_hdr_t *found = buf_hash_find(hdr->b_spa, zio->io_bp,
3711 		    &hash_lock);
3712 
3713 		ASSERT((found == NULL && HDR_FREED_IN_READ(hdr) &&
3714 		    hash_lock == NULL) ||
3715 		    (found == hdr &&
3716 		    DVA_EQUAL(&hdr->b_dva, BP_IDENTITY(zio->io_bp))) ||
3717 		    (found == hdr && HDR_L2_READING(hdr)));
3718 	}
3719 
3720 	hdr->b_flags &= ~ARC_FLAG_L2_EVICTED;
3721 	if (l2arc_noprefetch && HDR_PREFETCH(hdr))
3722 		hdr->b_flags &= ~ARC_FLAG_L2CACHE;
3723 
3724 	/* byteswap if necessary */
3725 	callback_list = hdr->b_l1hdr.b_acb;
3726 	ASSERT(callback_list != NULL);
3727 	if (BP_SHOULD_BYTESWAP(zio->io_bp) && zio->io_error == 0) {
3728 		dmu_object_byteswap_t bswap =
3729 		    DMU_OT_BYTESWAP(BP_GET_TYPE(zio->io_bp));
3730 		arc_byteswap_func_t *func = BP_GET_LEVEL(zio->io_bp) > 0 ?
3731 		    byteswap_uint64_array :
3732 		    dmu_ot_byteswap[bswap].ob_func;
3733 		func(buf->b_data, hdr->b_size);
3734 	}
3735 
3736 	arc_cksum_compute(buf, B_FALSE);
3737 	arc_buf_watch(buf);
3738 
3739 	if (hash_lock && zio->io_error == 0 &&
3740 	    hdr->b_l1hdr.b_state == arc_anon) {
3741 		/*
3742 		 * Only call arc_access on anonymous buffers.  This is because
3743 		 * if we've issued an I/O for an evicted buffer, we've already
3744 		 * called arc_access (to prevent any simultaneous readers from
3745 		 * getting confused).
3746 		 */
3747 		arc_access(hdr, hash_lock);
3748 	}
3749 
3750 	/* create copies of the data buffer for the callers */
3751 	abuf = buf;
3752 	for (acb = callback_list; acb; acb = acb->acb_next) {
3753 		if (acb->acb_done) {
3754 			if (abuf == NULL) {
3755 				ARCSTAT_BUMP(arcstat_duplicate_reads);
3756 				abuf = arc_buf_clone(buf);
3757 			}
3758 			acb->acb_buf = abuf;
3759 			abuf = NULL;
3760 		}
3761 	}
3762 	hdr->b_l1hdr.b_acb = NULL;
3763 	hdr->b_flags &= ~ARC_FLAG_IO_IN_PROGRESS;
3764 	ASSERT(!HDR_BUF_AVAILABLE(hdr));
3765 	if (abuf == buf) {
3766 		ASSERT(buf->b_efunc == NULL);
3767 		ASSERT(hdr->b_l1hdr.b_datacnt == 1);
3768 		hdr->b_flags |= ARC_FLAG_BUF_AVAILABLE;
3769 	}
3770 
3771 	ASSERT(refcount_is_zero(&hdr->b_l1hdr.b_refcnt) ||
3772 	    callback_list != NULL);
3773 
3774 	if (zio->io_error != 0) {
3775 		hdr->b_flags |= ARC_FLAG_IO_ERROR;
3776 		if (hdr->b_l1hdr.b_state != arc_anon)
3777 			arc_change_state(arc_anon, hdr, hash_lock);
3778 		if (HDR_IN_HASH_TABLE(hdr))
3779 			buf_hash_remove(hdr);
3780 		freeable = refcount_is_zero(&hdr->b_l1hdr.b_refcnt);
3781 	}
3782 
3783 	/*
3784 	 * Broadcast before we drop the hash_lock to avoid the possibility
3785 	 * that the hdr (and hence the cv) might be freed before we get to
3786 	 * the cv_broadcast().
3787 	 */
3788 	cv_broadcast(&hdr->b_l1hdr.b_cv);
3789 
3790 	if (hash_lock != NULL) {
3791 		mutex_exit(hash_lock);
3792 	} else {
3793 		/*
3794 		 * This block was freed while we waited for the read to
3795 		 * complete.  It has been removed from the hash table and
3796 		 * moved to the anonymous state (so that it won't show up
3797 		 * in the cache).
3798 		 */
3799 		ASSERT3P(hdr->b_l1hdr.b_state, ==, arc_anon);
3800 		freeable = refcount_is_zero(&hdr->b_l1hdr.b_refcnt);
3801 	}
3802 
3803 	/* execute each callback and free its structure */
3804 	while ((acb = callback_list) != NULL) {
3805 		if (acb->acb_done)
3806 			acb->acb_done(zio, acb->acb_buf, acb->acb_private);
3807 
3808 		if (acb->acb_zio_dummy != NULL) {
3809 			acb->acb_zio_dummy->io_error = zio->io_error;
3810 			zio_nowait(acb->acb_zio_dummy);
3811 		}
3812 
3813 		callback_list = acb->acb_next;
3814 		kmem_free(acb, sizeof (arc_callback_t));
3815 	}
3816 
3817 	if (freeable)
3818 		arc_hdr_destroy(hdr);
3819 }
3820 
3821 /*
3822  * "Read" the block at the specified DVA (in bp) via the
3823  * cache.  If the block is found in the cache, invoke the provided
3824  * callback immediately and return.  Note that the `zio' parameter
3825  * in the callback will be NULL in this case, since no IO was
3826  * required.  If the block is not in the cache pass the read request
3827  * on to the spa with a substitute callback function, so that the
3828  * requested block will be added to the cache.
3829  *
3830  * If a read request arrives for a block that has a read in-progress,
3831  * either wait for the in-progress read to complete (and return the
3832  * results); or, if this is a read with a "done" func, add a record
3833  * to the read to invoke the "done" func when the read completes,
3834  * and return; or just return.
3835  *
3836  * arc_read_done() will invoke all the requested "done" functions
3837  * for readers of this block.
3838  */
3839 int
3840 arc_read(zio_t *pio, spa_t *spa, const blkptr_t *bp, arc_done_func_t *done,
3841     void *private, zio_priority_t priority, int zio_flags,
3842     arc_flags_t *arc_flags, const zbookmark_phys_t *zb)
3843 {
3844 	arc_buf_hdr_t *hdr = NULL;
3845 	arc_buf_t *buf = NULL;
3846 	kmutex_t *hash_lock = NULL;
3847 	zio_t *rzio;
3848 	uint64_t guid = spa_load_guid(spa);
3849 
3850 	ASSERT(!BP_IS_EMBEDDED(bp) ||
3851 	    BPE_GET_ETYPE(bp) == BP_EMBEDDED_TYPE_DATA);
3852 
3853 top:
3854 	if (!BP_IS_EMBEDDED(bp)) {
3855 		/*
3856 		 * Embedded BP's have no DVA and require no I/O to "read".
3857 		 * Create an anonymous arc buf to back it.
3858 		 */
3859 		hdr = buf_hash_find(guid, bp, &hash_lock);
3860 	}
3861 
3862 	if (hdr != NULL && HDR_HAS_L1HDR(hdr) && hdr->b_l1hdr.b_datacnt > 0) {
3863 
3864 		*arc_flags |= ARC_FLAG_CACHED;
3865 
3866 		if (HDR_IO_IN_PROGRESS(hdr)) {
3867 
3868 			if (*arc_flags & ARC_FLAG_WAIT) {
3869 				cv_wait(&hdr->b_l1hdr.b_cv, hash_lock);
3870 				mutex_exit(hash_lock);
3871 				goto top;
3872 			}
3873 			ASSERT(*arc_flags & ARC_FLAG_NOWAIT);
3874 
3875 			if (done) {
3876 				arc_callback_t	*acb = NULL;
3877 
3878 				acb = kmem_zalloc(sizeof (arc_callback_t),
3879 				    KM_SLEEP);
3880 				acb->acb_done = done;
3881 				acb->acb_private = private;
3882 				if (pio != NULL)
3883 					acb->acb_zio_dummy = zio_null(pio,
3884 					    spa, NULL, NULL, NULL, zio_flags);
3885 
3886 				ASSERT(acb->acb_done != NULL);
3887 				acb->acb_next = hdr->b_l1hdr.b_acb;
3888 				hdr->b_l1hdr.b_acb = acb;
3889 				add_reference(hdr, hash_lock, private);
3890 				mutex_exit(hash_lock);
3891 				return (0);
3892 			}
3893 			mutex_exit(hash_lock);
3894 			return (0);
3895 		}
3896 
3897 		ASSERT(hdr->b_l1hdr.b_state == arc_mru ||
3898 		    hdr->b_l1hdr.b_state == arc_mfu);
3899 
3900 		if (done) {
3901 			add_reference(hdr, hash_lock, private);
3902 			/*
3903 			 * If this block is already in use, create a new
3904 			 * copy of the data so that we will be guaranteed
3905 			 * that arc_release() will always succeed.
3906 			 */
3907 			buf = hdr->b_l1hdr.b_buf;
3908 			ASSERT(buf);
3909 			ASSERT(buf->b_data);
3910 			if (HDR_BUF_AVAILABLE(hdr)) {
3911 				ASSERT(buf->b_efunc == NULL);
3912 				hdr->b_flags &= ~ARC_FLAG_BUF_AVAILABLE;
3913 			} else {
3914 				buf = arc_buf_clone(buf);
3915 			}
3916 
3917 		} else if (*arc_flags & ARC_FLAG_PREFETCH &&
3918 		    refcount_count(&hdr->b_l1hdr.b_refcnt) == 0) {
3919 			hdr->b_flags |= ARC_FLAG_PREFETCH;
3920 		}
3921 		DTRACE_PROBE1(arc__hit, arc_buf_hdr_t *, hdr);
3922 		arc_access(hdr, hash_lock);
3923 		if (*arc_flags & ARC_FLAG_L2CACHE)
3924 			hdr->b_flags |= ARC_FLAG_L2CACHE;
3925 		if (*arc_flags & ARC_FLAG_L2COMPRESS)
3926 			hdr->b_flags |= ARC_FLAG_L2COMPRESS;
3927 		mutex_exit(hash_lock);
3928 		ARCSTAT_BUMP(arcstat_hits);
3929 		ARCSTAT_CONDSTAT(!HDR_PREFETCH(hdr),
3930 		    demand, prefetch, !HDR_ISTYPE_METADATA(hdr),
3931 		    data, metadata, hits);
3932 
3933 		if (done)
3934 			done(NULL, buf, private);
3935 	} else {
3936 		uint64_t size = BP_GET_LSIZE(bp);
3937 		arc_callback_t *acb;
3938 		vdev_t *vd = NULL;
3939 		uint64_t addr = 0;
3940 		boolean_t devw = B_FALSE;
3941 		enum zio_compress b_compress = ZIO_COMPRESS_OFF;
3942 		int32_t b_asize = 0;
3943 
3944 		if (hdr == NULL) {
3945 			/* this block is not in the cache */
3946 			arc_buf_hdr_t *exists = NULL;
3947 			arc_buf_contents_t type = BP_GET_BUFC_TYPE(bp);
3948 			buf = arc_buf_alloc(spa, size, private, type);
3949 			hdr = buf->b_hdr;
3950 			if (!BP_IS_EMBEDDED(bp)) {
3951 				hdr->b_dva = *BP_IDENTITY(bp);
3952 				hdr->b_birth = BP_PHYSICAL_BIRTH(bp);
3953 				exists = buf_hash_insert(hdr, &hash_lock);
3954 			}
3955 			if (exists != NULL) {
3956 				/* somebody beat us to the hash insert */
3957 				mutex_exit(hash_lock);
3958 				buf_discard_identity(hdr);
3959 				(void) arc_buf_remove_ref(buf, private);
3960 				goto top; /* restart the IO request */
3961 			}
3962 
3963 			/* if this is a prefetch, we don't have a reference */
3964 			if (*arc_flags & ARC_FLAG_PREFETCH) {
3965 				(void) remove_reference(hdr, hash_lock,
3966 				    private);
3967 				hdr->b_flags |= ARC_FLAG_PREFETCH;
3968 			}
3969 			if (*arc_flags & ARC_FLAG_L2CACHE)
3970 				hdr->b_flags |= ARC_FLAG_L2CACHE;
3971 			if (*arc_flags & ARC_FLAG_L2COMPRESS)
3972 				hdr->b_flags |= ARC_FLAG_L2COMPRESS;
3973 			if (BP_GET_LEVEL(bp) > 0)
3974 				hdr->b_flags |= ARC_FLAG_INDIRECT;
3975 		} else {
3976 			/*
3977 			 * This block is in the ghost cache. If it was L2-only
3978 			 * (and thus didn't have an L1 hdr), we realloc the
3979 			 * header to add an L1 hdr.
3980 			 */
3981 			if (!HDR_HAS_L1HDR(hdr)) {
3982 				hdr = arc_hdr_realloc(hdr, hdr_l2only_cache,
3983 				    hdr_full_cache);
3984 			}
3985 
3986 			ASSERT(GHOST_STATE(hdr->b_l1hdr.b_state));
3987 			ASSERT(!HDR_IO_IN_PROGRESS(hdr));
3988 			ASSERT(refcount_is_zero(&hdr->b_l1hdr.b_refcnt));
3989 			ASSERT3P(hdr->b_l1hdr.b_buf, ==, NULL);
3990 
3991 			/* if this is a prefetch, we don't have a reference */
3992 			if (*arc_flags & ARC_FLAG_PREFETCH)
3993 				hdr->b_flags |= ARC_FLAG_PREFETCH;
3994 			else
3995 				add_reference(hdr, hash_lock, private);
3996 			if (*arc_flags & ARC_FLAG_L2CACHE)
3997 				hdr->b_flags |= ARC_FLAG_L2CACHE;
3998 			if (*arc_flags & ARC_FLAG_L2COMPRESS)
3999 				hdr->b_flags |= ARC_FLAG_L2COMPRESS;
4000 			buf = kmem_cache_alloc(buf_cache, KM_PUSHPAGE);
4001 			buf->b_hdr = hdr;
4002 			buf->b_data = NULL;
4003 			buf->b_efunc = NULL;
4004 			buf->b_private = NULL;
4005 			buf->b_next = NULL;
4006 			hdr->b_l1hdr.b_buf = buf;
4007 			ASSERT0(hdr->b_l1hdr.b_datacnt);
4008 			hdr->b_l1hdr.b_datacnt = 1;
4009 			arc_get_data_buf(buf);
4010 			arc_access(hdr, hash_lock);
4011 		}
4012 
4013 		ASSERT(!GHOST_STATE(hdr->b_l1hdr.b_state));
4014 
4015 		acb = kmem_zalloc(sizeof (arc_callback_t), KM_SLEEP);
4016 		acb->acb_done = done;
4017 		acb->acb_private = private;
4018 
4019 		ASSERT(hdr->b_l1hdr.b_acb == NULL);
4020 		hdr->b_l1hdr.b_acb = acb;
4021 		hdr->b_flags |= ARC_FLAG_IO_IN_PROGRESS;
4022 
4023 		if (HDR_HAS_L2HDR(hdr) &&
4024 		    (vd = hdr->b_l2hdr.b_dev->l2ad_vdev) != NULL) {
4025 			devw = hdr->b_l2hdr.b_dev->l2ad_writing;
4026 			addr = hdr->b_l2hdr.b_daddr;
4027 			b_compress = HDR_GET_COMPRESS(hdr);
4028 			b_asize = hdr->b_l2hdr.b_asize;
4029 			/*
4030 			 * Lock out device removal.
4031 			 */
4032 			if (vdev_is_dead(vd) ||
4033 			    !spa_config_tryenter(spa, SCL_L2ARC, vd, RW_READER))
4034 				vd = NULL;
4035 		}
4036 
4037 		if (hash_lock != NULL)
4038 			mutex_exit(hash_lock);
4039 
4040 		/*
4041 		 * At this point, we have a level 1 cache miss.  Try again in
4042 		 * L2ARC if possible.
4043 		 */
4044 		ASSERT3U(hdr->b_size, ==, size);
4045 		DTRACE_PROBE4(arc__miss, arc_buf_hdr_t *, hdr, blkptr_t *, bp,
4046 		    uint64_t, size, zbookmark_phys_t *, zb);
4047 		ARCSTAT_BUMP(arcstat_misses);
4048 		ARCSTAT_CONDSTAT(!HDR_PREFETCH(hdr),
4049 		    demand, prefetch, !HDR_ISTYPE_METADATA(hdr),
4050 		    data, metadata, misses);
4051 
4052 		if (vd != NULL && l2arc_ndev != 0 && !(l2arc_norw && devw)) {
4053 			/*
4054 			 * Read from the L2ARC if the following are true:
4055 			 * 1. The L2ARC vdev was previously cached.
4056 			 * 2. This buffer still has L2ARC metadata.
4057 			 * 3. This buffer isn't currently writing to the L2ARC.
4058 			 * 4. The L2ARC entry wasn't evicted, which may
4059 			 *    also have invalidated the vdev.
4060 			 * 5. This isn't prefetch and l2arc_noprefetch is set.
4061 			 */
4062 			if (HDR_HAS_L2HDR(hdr) &&
4063 			    !HDR_L2_WRITING(hdr) && !HDR_L2_EVICTED(hdr) &&
4064 			    !(l2arc_noprefetch && HDR_PREFETCH(hdr))) {
4065 				l2arc_read_callback_t *cb;
4066 
4067 				DTRACE_PROBE1(l2arc__hit, arc_buf_hdr_t *, hdr);
4068 				ARCSTAT_BUMP(arcstat_l2_hits);
4069 
4070 				cb = kmem_zalloc(sizeof (l2arc_read_callback_t),
4071 				    KM_SLEEP);
4072 				cb->l2rcb_buf = buf;
4073 				cb->l2rcb_spa = spa;
4074 				cb->l2rcb_bp = *bp;
4075 				cb->l2rcb_zb = *zb;
4076 				cb->l2rcb_flags = zio_flags;
4077 				cb->l2rcb_compress = b_compress;
4078 
4079 				ASSERT(addr >= VDEV_LABEL_START_SIZE &&
4080 				    addr + size < vd->vdev_psize -
4081 				    VDEV_LABEL_END_SIZE);
4082 
4083 				/*
4084 				 * l2arc read.  The SCL_L2ARC lock will be
4085 				 * released by l2arc_read_done().
4086 				 * Issue a null zio if the underlying buffer
4087 				 * was squashed to zero size by compression.
4088 				 */
4089 				if (b_compress == ZIO_COMPRESS_EMPTY) {
4090 					rzio = zio_null(pio, spa, vd,
4091 					    l2arc_read_done, cb,
4092 					    zio_flags | ZIO_FLAG_DONT_CACHE |
4093 					    ZIO_FLAG_CANFAIL |
4094 					    ZIO_FLAG_DONT_PROPAGATE |
4095 					    ZIO_FLAG_DONT_RETRY);
4096 				} else {
4097 					rzio = zio_read_phys(pio, vd, addr,
4098 					    b_asize, buf->b_data,
4099 					    ZIO_CHECKSUM_OFF,
4100 					    l2arc_read_done, cb, priority,
4101 					    zio_flags | ZIO_FLAG_DONT_CACHE |
4102 					    ZIO_FLAG_CANFAIL |
4103 					    ZIO_FLAG_DONT_PROPAGATE |
4104 					    ZIO_FLAG_DONT_RETRY, B_FALSE);
4105 				}
4106 				DTRACE_PROBE2(l2arc__read, vdev_t *, vd,
4107 				    zio_t *, rzio);
4108 				ARCSTAT_INCR(arcstat_l2_read_bytes, b_asize);
4109 
4110 				if (*arc_flags & ARC_FLAG_NOWAIT) {
4111 					zio_nowait(rzio);
4112 					return (0);
4113 				}
4114 
4115 				ASSERT(*arc_flags & ARC_FLAG_WAIT);
4116 				if (zio_wait(rzio) == 0)
4117 					return (0);
4118 
4119 				/* l2arc read error; goto zio_read() */
4120 			} else {
4121 				DTRACE_PROBE1(l2arc__miss,
4122 				    arc_buf_hdr_t *, hdr);
4123 				ARCSTAT_BUMP(arcstat_l2_misses);
4124 				if (HDR_L2_WRITING(hdr))
4125 					ARCSTAT_BUMP(arcstat_l2_rw_clash);
4126 				spa_config_exit(spa, SCL_L2ARC, vd);
4127 			}
4128 		} else {
4129 			if (vd != NULL)
4130 				spa_config_exit(spa, SCL_L2ARC, vd);
4131 			if (l2arc_ndev != 0) {
4132 				DTRACE_PROBE1(l2arc__miss,
4133 				    arc_buf_hdr_t *, hdr);
4134 				ARCSTAT_BUMP(arcstat_l2_misses);
4135 			}
4136 		}
4137 
4138 		rzio = zio_read(pio, spa, bp, buf->b_data, size,
4139 		    arc_read_done, buf, priority, zio_flags, zb);
4140 
4141 		if (*arc_flags & ARC_FLAG_WAIT)
4142 			return (zio_wait(rzio));
4143 
4144 		ASSERT(*arc_flags & ARC_FLAG_NOWAIT);
4145 		zio_nowait(rzio);
4146 	}
4147 	return (0);
4148 }
4149 
4150 void
4151 arc_set_callback(arc_buf_t *buf, arc_evict_func_t *func, void *private)
4152 {
4153 	ASSERT(buf->b_hdr != NULL);
4154 	ASSERT(buf->b_hdr->b_l1hdr.b_state != arc_anon);
4155 	ASSERT(!refcount_is_zero(&buf->b_hdr->b_l1hdr.b_refcnt) ||
4156 	    func == NULL);
4157 	ASSERT(buf->b_efunc == NULL);
4158 	ASSERT(!HDR_BUF_AVAILABLE(buf->b_hdr));
4159 
4160 	buf->b_efunc = func;
4161 	buf->b_private = private;
4162 }
4163 
4164 /*
4165  * Notify the arc that a block was freed, and thus will never be used again.
4166  */
4167 void
4168 arc_freed(spa_t *spa, const blkptr_t *bp)
4169 {
4170 	arc_buf_hdr_t *hdr;
4171 	kmutex_t *hash_lock;
4172 	uint64_t guid = spa_load_guid(spa);
4173 
4174 	ASSERT(!BP_IS_EMBEDDED(bp));
4175 
4176 	hdr = buf_hash_find(guid, bp, &hash_lock);
4177 	if (hdr == NULL)
4178 		return;
4179 	if (HDR_BUF_AVAILABLE(hdr)) {
4180 		arc_buf_t *buf = hdr->b_l1hdr.b_buf;
4181 		add_reference(hdr, hash_lock, FTAG);
4182 		hdr->b_flags &= ~ARC_FLAG_BUF_AVAILABLE;
4183 		mutex_exit(hash_lock);
4184 
4185 		arc_release(buf, FTAG);
4186 		(void) arc_buf_remove_ref(buf, FTAG);
4187 	} else {
4188 		mutex_exit(hash_lock);
4189 	}
4190 
4191 }
4192 
4193 /*
4194  * Clear the user eviction callback set by arc_set_callback(), first calling
4195  * it if it exists.  Because the presence of a callback keeps an arc_buf cached
4196  * clearing the callback may result in the arc_buf being destroyed.  However,
4197  * it will not result in the *last* arc_buf being destroyed, hence the data
4198  * will remain cached in the ARC. We make a copy of the arc buffer here so
4199  * that we can process the callback without holding any locks.
4200  *
4201  * It's possible that the callback is already in the process of being cleared
4202  * by another thread.  In this case we can not clear the callback.
4203  *
4204  * Returns B_TRUE if the callback was successfully called and cleared.
4205  */
4206 boolean_t
4207 arc_clear_callback(arc_buf_t *buf)
4208 {
4209 	arc_buf_hdr_t *hdr;
4210 	kmutex_t *hash_lock;
4211 	arc_evict_func_t *efunc = buf->b_efunc;
4212 	void *private = buf->b_private;
4213 
4214 	mutex_enter(&buf->b_evict_lock);
4215 	hdr = buf->b_hdr;
4216 	if (hdr == NULL) {
4217 		/*
4218 		 * We are in arc_do_user_evicts().
4219 		 */
4220 		ASSERT(buf->b_data == NULL);
4221 		mutex_exit(&buf->b_evict_lock);
4222 		return (B_FALSE);
4223 	} else if (buf->b_data == NULL) {
4224 		/*
4225 		 * We are on the eviction list; process this buffer now
4226 		 * but let arc_do_user_evicts() do the reaping.
4227 		 */
4228 		buf->b_efunc = NULL;
4229 		mutex_exit(&buf->b_evict_lock);
4230 		VERIFY0(efunc(private));
4231 		return (B_TRUE);
4232 	}
4233 	hash_lock = HDR_LOCK(hdr);
4234 	mutex_enter(hash_lock);
4235 	hdr = buf->b_hdr;
4236 	ASSERT3P(hash_lock, ==, HDR_LOCK(hdr));
4237 
4238 	ASSERT3U(refcount_count(&hdr->b_l1hdr.b_refcnt), <,
4239 	    hdr->b_l1hdr.b_datacnt);
4240 	ASSERT(hdr->b_l1hdr.b_state == arc_mru ||
4241 	    hdr->b_l1hdr.b_state == arc_mfu);
4242 
4243 	buf->b_efunc = NULL;
4244 	buf->b_private = NULL;
4245 
4246 	if (hdr->b_l1hdr.b_datacnt > 1) {
4247 		mutex_exit(&buf->b_evict_lock);
4248 		arc_buf_destroy(buf, TRUE);
4249 	} else {
4250 		ASSERT(buf == hdr->b_l1hdr.b_buf);
4251 		hdr->b_flags |= ARC_FLAG_BUF_AVAILABLE;
4252 		mutex_exit(&buf->b_evict_lock);
4253 	}
4254 
4255 	mutex_exit(hash_lock);
4256 	VERIFY0(efunc(private));
4257 	return (B_TRUE);
4258 }
4259 
4260 /*
4261  * Release this buffer from the cache, making it an anonymous buffer.  This
4262  * must be done after a read and prior to modifying the buffer contents.
4263  * If the buffer has more than one reference, we must make
4264  * a new hdr for the buffer.
4265  */
4266 void
4267 arc_release(arc_buf_t *buf, void *tag)
4268 {
4269 	arc_buf_hdr_t *hdr = buf->b_hdr;
4270 
4271 	/*
4272 	 * It would be nice to assert that if it's DMU metadata (level >
4273 	 * 0 || it's the dnode file), then it must be syncing context.
4274 	 * But we don't know that information at this level.
4275 	 */
4276 
4277 	mutex_enter(&buf->b_evict_lock);
4278 
4279 	ASSERT(HDR_HAS_L1HDR(hdr));
4280 
4281 	/*
4282 	 * We don't grab the hash lock prior to this check, because if
4283 	 * the buffer's header is in the arc_anon state, it won't be
4284 	 * linked into the hash table.
4285 	 */
4286 	if (hdr->b_l1hdr.b_state == arc_anon) {
4287 		mutex_exit(&buf->b_evict_lock);
4288 		ASSERT(!HDR_IO_IN_PROGRESS(hdr));
4289 		ASSERT(!HDR_IN_HASH_TABLE(hdr));
4290 		ASSERT(!HDR_HAS_L2HDR(hdr));
4291 		ASSERT(BUF_EMPTY(hdr));
4292 
4293 		ASSERT3U(hdr->b_l1hdr.b_datacnt, ==, 1);
4294 		ASSERT3S(refcount_count(&hdr->b_l1hdr.b_refcnt), ==, 1);
4295 		ASSERT(!list_link_active(&hdr->b_l1hdr.b_arc_node));
4296 
4297 		ASSERT3P(buf->b_efunc, ==, NULL);
4298 		ASSERT3P(buf->b_private, ==, NULL);
4299 
4300 		hdr->b_l1hdr.b_arc_access = 0;
4301 		arc_buf_thaw(buf);
4302 
4303 		return;
4304 	}
4305 
4306 	kmutex_t *hash_lock = HDR_LOCK(hdr);
4307 	mutex_enter(hash_lock);
4308 
4309 	/*
4310 	 * This assignment is only valid as long as the hash_lock is
4311 	 * held, we must be careful not to reference state or the
4312 	 * b_state field after dropping the lock.
4313 	 */
4314 	arc_state_t *state = hdr->b_l1hdr.b_state;
4315 	ASSERT3P(hash_lock, ==, HDR_LOCK(hdr));
4316 	ASSERT3P(state, !=, arc_anon);
4317 
4318 	/* this buffer is not on any list */
4319 	ASSERT(refcount_count(&hdr->b_l1hdr.b_refcnt) > 0);
4320 
4321 	if (HDR_HAS_L2HDR(hdr)) {
4322 		ARCSTAT_INCR(arcstat_l2_asize, -hdr->b_l2hdr.b_asize);
4323 		ARCSTAT_INCR(arcstat_l2_size, -hdr->b_size);
4324 
4325 		mutex_enter(&hdr->b_l2hdr.b_dev->l2ad_mtx);
4326 		list_remove(&hdr->b_l2hdr.b_dev->l2ad_buflist, hdr);
4327 
4328 		/*
4329 		 * We don't want to leak the b_tmp_cdata buffer that was
4330 		 * allocated in l2arc_write_buffers()
4331 		 */
4332 		arc_buf_l2_cdata_free(hdr);
4333 
4334 		mutex_exit(&hdr->b_l2hdr.b_dev->l2ad_mtx);
4335 
4336 		hdr->b_flags &= ~ARC_FLAG_HAS_L2HDR;
4337 	}
4338 
4339 	/*
4340 	 * Do we have more than one buf?
4341 	 */
4342 	if (hdr->b_l1hdr.b_datacnt > 1) {
4343 		arc_buf_hdr_t *nhdr;
4344 		arc_buf_t **bufp;
4345 		uint64_t blksz = hdr->b_size;
4346 		uint64_t spa = hdr->b_spa;
4347 		arc_buf_contents_t type = arc_buf_type(hdr);
4348 		uint32_t flags = hdr->b_flags;
4349 
4350 		ASSERT(hdr->b_l1hdr.b_buf != buf || buf->b_next != NULL);
4351 		/*
4352 		 * Pull the data off of this hdr and attach it to
4353 		 * a new anonymous hdr.
4354 		 */
4355 		(void) remove_reference(hdr, hash_lock, tag);
4356 		bufp = &hdr->b_l1hdr.b_buf;
4357 		while (*bufp != buf)
4358 			bufp = &(*bufp)->b_next;
4359 		*bufp = buf->b_next;
4360 		buf->b_next = NULL;
4361 
4362 		ASSERT3P(state, !=, arc_l2c_only);
4363 		ASSERT3U(state->arcs_size, >=, hdr->b_size);
4364 		atomic_add_64(&state->arcs_size, -hdr->b_size);
4365 		if (refcount_is_zero(&hdr->b_l1hdr.b_refcnt)) {
4366 			ASSERT3P(state, !=, arc_l2c_only);
4367 			uint64_t *size = &state->arcs_lsize[type];
4368 			ASSERT3U(*size, >=, hdr->b_size);
4369 			atomic_add_64(size, -hdr->b_size);
4370 		}
4371 
4372 		/*
4373 		 * We're releasing a duplicate user data buffer, update
4374 		 * our statistics accordingly.
4375 		 */
4376 		if (HDR_ISTYPE_DATA(hdr)) {
4377 			ARCSTAT_BUMPDOWN(arcstat_duplicate_buffers);
4378 			ARCSTAT_INCR(arcstat_duplicate_buffers_size,
4379 			    -hdr->b_size);
4380 		}
4381 		hdr->b_l1hdr.b_datacnt -= 1;
4382 		arc_cksum_verify(buf);
4383 		arc_buf_unwatch(buf);
4384 
4385 		mutex_exit(hash_lock);
4386 
4387 		nhdr = kmem_cache_alloc(hdr_full_cache, KM_PUSHPAGE);
4388 		nhdr->b_size = blksz;
4389 		nhdr->b_spa = spa;
4390 
4391 		nhdr->b_flags = flags & ARC_FLAG_L2_WRITING;
4392 		nhdr->b_flags |= arc_bufc_to_flags(type);
4393 		nhdr->b_flags |= ARC_FLAG_HAS_L1HDR;
4394 
4395 		nhdr->b_l1hdr.b_buf = buf;
4396 		nhdr->b_l1hdr.b_datacnt = 1;
4397 		nhdr->b_l1hdr.b_state = arc_anon;
4398 		nhdr->b_l1hdr.b_arc_access = 0;
4399 		nhdr->b_l1hdr.b_tmp_cdata = NULL;
4400 		nhdr->b_freeze_cksum = NULL;
4401 
4402 		(void) refcount_add(&nhdr->b_l1hdr.b_refcnt, tag);
4403 		buf->b_hdr = nhdr;
4404 		mutex_exit(&buf->b_evict_lock);
4405 		atomic_add_64(&arc_anon->arcs_size, blksz);
4406 	} else {
4407 		mutex_exit(&buf->b_evict_lock);
4408 		ASSERT(refcount_count(&hdr->b_l1hdr.b_refcnt) == 1);
4409 		/* protected by hash lock, or hdr is on arc_anon */
4410 		ASSERT(!multilist_link_active(&hdr->b_l1hdr.b_arc_node));
4411 		ASSERT(!HDR_IO_IN_PROGRESS(hdr));
4412 		arc_change_state(arc_anon, hdr, hash_lock);
4413 		hdr->b_l1hdr.b_arc_access = 0;
4414 		mutex_exit(hash_lock);
4415 
4416 		buf_discard_identity(hdr);
4417 		arc_buf_thaw(buf);
4418 	}
4419 	buf->b_efunc = NULL;
4420 	buf->b_private = NULL;
4421 }
4422 
4423 int
4424 arc_released(arc_buf_t *buf)
4425 {
4426 	int released;
4427 
4428 	mutex_enter(&buf->b_evict_lock);
4429 	released = (buf->b_data != NULL &&
4430 	    buf->b_hdr->b_l1hdr.b_state == arc_anon);
4431 	mutex_exit(&buf->b_evict_lock);
4432 	return (released);
4433 }
4434 
4435 #ifdef ZFS_DEBUG
4436 int
4437 arc_referenced(arc_buf_t *buf)
4438 {
4439 	int referenced;
4440 
4441 	mutex_enter(&buf->b_evict_lock);
4442 	referenced = (refcount_count(&buf->b_hdr->b_l1hdr.b_refcnt));
4443 	mutex_exit(&buf->b_evict_lock);
4444 	return (referenced);
4445 }
4446 #endif
4447 
4448 static void
4449 arc_write_ready(zio_t *zio)
4450 {
4451 	arc_write_callback_t *callback = zio->io_private;
4452 	arc_buf_t *buf = callback->awcb_buf;
4453 	arc_buf_hdr_t *hdr = buf->b_hdr;
4454 
4455 	ASSERT(HDR_HAS_L1HDR(hdr));
4456 	ASSERT(!refcount_is_zero(&buf->b_hdr->b_l1hdr.b_refcnt));
4457 	ASSERT(hdr->b_l1hdr.b_datacnt > 0);
4458 	callback->awcb_ready(zio, buf, callback->awcb_private);
4459 
4460 	/*
4461 	 * If the IO is already in progress, then this is a re-write
4462 	 * attempt, so we need to thaw and re-compute the cksum.
4463 	 * It is the responsibility of the callback to handle the
4464 	 * accounting for any re-write attempt.
4465 	 */
4466 	if (HDR_IO_IN_PROGRESS(hdr)) {
4467 		mutex_enter(&hdr->b_l1hdr.b_freeze_lock);
4468 		if (hdr->b_freeze_cksum != NULL) {
4469 			kmem_free(hdr->b_freeze_cksum, sizeof (zio_cksum_t));
4470 			hdr->b_freeze_cksum = NULL;
4471 		}
4472 		mutex_exit(&hdr->b_l1hdr.b_freeze_lock);
4473 	}
4474 	arc_cksum_compute(buf, B_FALSE);
4475 	hdr->b_flags |= ARC_FLAG_IO_IN_PROGRESS;
4476 }
4477 
4478 /*
4479  * The SPA calls this callback for each physical write that happens on behalf
4480  * of a logical write.  See the comment in dbuf_write_physdone() for details.
4481  */
4482 static void
4483 arc_write_physdone(zio_t *zio)
4484 {
4485 	arc_write_callback_t *cb = zio->io_private;
4486 	if (cb->awcb_physdone != NULL)
4487 		cb->awcb_physdone(zio, cb->awcb_buf, cb->awcb_private);
4488 }
4489 
4490 static void
4491 arc_write_done(zio_t *zio)
4492 {
4493 	arc_write_callback_t *callback = zio->io_private;
4494 	arc_buf_t *buf = callback->awcb_buf;
4495 	arc_buf_hdr_t *hdr = buf->b_hdr;
4496 
4497 	ASSERT(hdr->b_l1hdr.b_acb == NULL);
4498 
4499 	if (zio->io_error == 0) {
4500 		if (BP_IS_HOLE(zio->io_bp) || BP_IS_EMBEDDED(zio->io_bp)) {
4501 			buf_discard_identity(hdr);
4502 		} else {
4503 			hdr->b_dva = *BP_IDENTITY(zio->io_bp);
4504 			hdr->b_birth = BP_PHYSICAL_BIRTH(zio->io_bp);
4505 		}
4506 	} else {
4507 		ASSERT(BUF_EMPTY(hdr));
4508 	}
4509 
4510 	/*
4511 	 * If the block to be written was all-zero or compressed enough to be
4512 	 * embedded in the BP, no write was performed so there will be no
4513 	 * dva/birth/checksum.  The buffer must therefore remain anonymous
4514 	 * (and uncached).
4515 	 */
4516 	if (!BUF_EMPTY(hdr)) {
4517 		arc_buf_hdr_t *exists;
4518 		kmutex_t *hash_lock;
4519 
4520 		ASSERT(zio->io_error == 0);
4521 
4522 		arc_cksum_verify(buf);
4523 
4524 		exists = buf_hash_insert(hdr, &hash_lock);
4525 		if (exists != NULL) {
4526 			/*
4527 			 * This can only happen if we overwrite for
4528 			 * sync-to-convergence, because we remove
4529 			 * buffers from the hash table when we arc_free().
4530 			 */
4531 			if (zio->io_flags & ZIO_FLAG_IO_REWRITE) {
4532 				if (!BP_EQUAL(&zio->io_bp_orig, zio->io_bp))
4533 					panic("bad overwrite, hdr=%p exists=%p",
4534 					    (void *)hdr, (void *)exists);
4535 				ASSERT(refcount_is_zero(
4536 				    &exists->b_l1hdr.b_refcnt));
4537 				arc_change_state(arc_anon, exists, hash_lock);
4538 				mutex_exit(hash_lock);
4539 				arc_hdr_destroy(exists);
4540 				exists = buf_hash_insert(hdr, &hash_lock);
4541 				ASSERT3P(exists, ==, NULL);
4542 			} else if (zio->io_flags & ZIO_FLAG_NOPWRITE) {
4543 				/* nopwrite */
4544 				ASSERT(zio->io_prop.zp_nopwrite);
4545 				if (!BP_EQUAL(&zio->io_bp_orig, zio->io_bp))
4546 					panic("bad nopwrite, hdr=%p exists=%p",
4547 					    (void *)hdr, (void *)exists);
4548 			} else {
4549 				/* Dedup */
4550 				ASSERT(hdr->b_l1hdr.b_datacnt == 1);
4551 				ASSERT(hdr->b_l1hdr.b_state == arc_anon);
4552 				ASSERT(BP_GET_DEDUP(zio->io_bp));
4553 				ASSERT(BP_GET_LEVEL(zio->io_bp) == 0);
4554 			}
4555 		}
4556 		hdr->b_flags &= ~ARC_FLAG_IO_IN_PROGRESS;
4557 		/* if it's not anon, we are doing a scrub */
4558 		if (exists == NULL && hdr->b_l1hdr.b_state == arc_anon)
4559 			arc_access(hdr, hash_lock);
4560 		mutex_exit(hash_lock);
4561 	} else {
4562 		hdr->b_flags &= ~ARC_FLAG_IO_IN_PROGRESS;
4563 	}
4564 
4565 	ASSERT(!refcount_is_zero(&hdr->b_l1hdr.b_refcnt));
4566 	callback->awcb_done(zio, buf, callback->awcb_private);
4567 
4568 	kmem_free(callback, sizeof (arc_write_callback_t));
4569 }
4570 
4571 zio_t *
4572 arc_write(zio_t *pio, spa_t *spa, uint64_t txg,
4573     blkptr_t *bp, arc_buf_t *buf, boolean_t l2arc, boolean_t l2arc_compress,
4574     const zio_prop_t *zp, arc_done_func_t *ready, arc_done_func_t *physdone,
4575     arc_done_func_t *done, void *private, zio_priority_t priority,
4576     int zio_flags, const zbookmark_phys_t *zb)
4577 {
4578 	arc_buf_hdr_t *hdr = buf->b_hdr;
4579 	arc_write_callback_t *callback;
4580 	zio_t *zio;
4581 
4582 	ASSERT(ready != NULL);
4583 	ASSERT(done != NULL);
4584 	ASSERT(!HDR_IO_ERROR(hdr));
4585 	ASSERT(!HDR_IO_IN_PROGRESS(hdr));
4586 	ASSERT(hdr->b_l1hdr.b_acb == NULL);
4587 	ASSERT(hdr->b_l1hdr.b_datacnt > 0);
4588 	if (l2arc)
4589 		hdr->b_flags |= ARC_FLAG_L2CACHE;
4590 	if (l2arc_compress)
4591 		hdr->b_flags |= ARC_FLAG_L2COMPRESS;
4592 	callback = kmem_zalloc(sizeof (arc_write_callback_t), KM_SLEEP);
4593 	callback->awcb_ready = ready;
4594 	callback->awcb_physdone = physdone;
4595 	callback->awcb_done = done;
4596 	callback->awcb_private = private;
4597 	callback->awcb_buf = buf;
4598 
4599 	zio = zio_write(pio, spa, txg, bp, buf->b_data, hdr->b_size, zp,
4600 	    arc_write_ready, arc_write_physdone, arc_write_done, callback,
4601 	    priority, zio_flags, zb);
4602 
4603 	return (zio);
4604 }
4605 
4606 static int
4607 arc_memory_throttle(uint64_t reserve, uint64_t txg)
4608 {
4609 #ifdef _KERNEL
4610 	uint64_t available_memory = ptob(freemem);
4611 	static uint64_t page_load = 0;
4612 	static uint64_t last_txg = 0;
4613 
4614 #if defined(__i386)
4615 	available_memory =
4616 	    MIN(available_memory, vmem_size(heap_arena, VMEM_FREE));
4617 #endif
4618 
4619 	if (freemem > physmem * arc_lotsfree_percent / 100)
4620 		return (0);
4621 
4622 	if (txg > last_txg) {
4623 		last_txg = txg;
4624 		page_load = 0;
4625 	}
4626 	/*
4627 	 * If we are in pageout, we know that memory is already tight,
4628 	 * the arc is already going to be evicting, so we just want to
4629 	 * continue to let page writes occur as quickly as possible.
4630 	 */
4631 	if (curproc == proc_pageout) {
4632 		if (page_load > MAX(ptob(minfree), available_memory) / 4)
4633 			return (SET_ERROR(ERESTART));
4634 		/* Note: reserve is inflated, so we deflate */
4635 		page_load += reserve / 8;
4636 		return (0);
4637 	} else if (page_load > 0 && arc_reclaim_needed()) {
4638 		/* memory is low, delay before restarting */
4639 		ARCSTAT_INCR(arcstat_memory_throttle_count, 1);
4640 		return (SET_ERROR(EAGAIN));
4641 	}
4642 	page_load = 0;
4643 #endif
4644 	return (0);
4645 }
4646 
4647 void
4648 arc_tempreserve_clear(uint64_t reserve)
4649 {
4650 	atomic_add_64(&arc_tempreserve, -reserve);
4651 	ASSERT((int64_t)arc_tempreserve >= 0);
4652 }
4653 
4654 int
4655 arc_tempreserve_space(uint64_t reserve, uint64_t txg)
4656 {
4657 	int error;
4658 	uint64_t anon_size;
4659 
4660 	if (reserve > arc_c/4 && !arc_no_grow)
4661 		arc_c = MIN(arc_c_max, reserve * 4);
4662 	if (reserve > arc_c)
4663 		return (SET_ERROR(ENOMEM));
4664 
4665 	/*
4666 	 * Don't count loaned bufs as in flight dirty data to prevent long
4667 	 * network delays from blocking transactions that are ready to be
4668 	 * assigned to a txg.
4669 	 */
4670 	anon_size = MAX((int64_t)(arc_anon->arcs_size - arc_loaned_bytes), 0);
4671 
4672 	/*
4673 	 * Writes will, almost always, require additional memory allocations
4674 	 * in order to compress/encrypt/etc the data.  We therefore need to
4675 	 * make sure that there is sufficient available memory for this.
4676 	 */
4677 	error = arc_memory_throttle(reserve, txg);
4678 	if (error != 0)
4679 		return (error);
4680 
4681 	/*
4682 	 * Throttle writes when the amount of dirty data in the cache
4683 	 * gets too large.  We try to keep the cache less than half full
4684 	 * of dirty blocks so that our sync times don't grow too large.
4685 	 * Note: if two requests come in concurrently, we might let them
4686 	 * both succeed, when one of them should fail.  Not a huge deal.
4687 	 */
4688 
4689 	if (reserve + arc_tempreserve + anon_size > arc_c / 2 &&
4690 	    anon_size > arc_c / 4) {
4691 		dprintf("failing, arc_tempreserve=%lluK anon_meta=%lluK "
4692 		    "anon_data=%lluK tempreserve=%lluK arc_c=%lluK\n",
4693 		    arc_tempreserve>>10,
4694 		    arc_anon->arcs_lsize[ARC_BUFC_METADATA]>>10,
4695 		    arc_anon->arcs_lsize[ARC_BUFC_DATA]>>10,
4696 		    reserve>>10, arc_c>>10);
4697 		return (SET_ERROR(ERESTART));
4698 	}
4699 	atomic_add_64(&arc_tempreserve, reserve);
4700 	return (0);
4701 }
4702 
4703 static void
4704 arc_kstat_update_state(arc_state_t *state, kstat_named_t *size,
4705     kstat_named_t *evict_data, kstat_named_t *evict_metadata)
4706 {
4707 	size->value.ui64 = state->arcs_size;
4708 	evict_data->value.ui64 = state->arcs_lsize[ARC_BUFC_DATA];
4709 	evict_metadata->value.ui64 = state->arcs_lsize[ARC_BUFC_METADATA];
4710 }
4711 
4712 static int
4713 arc_kstat_update(kstat_t *ksp, int rw)
4714 {
4715 	arc_stats_t *as = ksp->ks_data;
4716 
4717 	if (rw == KSTAT_WRITE) {
4718 		return (EACCES);
4719 	} else {
4720 		arc_kstat_update_state(arc_anon,
4721 		    &as->arcstat_anon_size,
4722 		    &as->arcstat_anon_evictable_data,
4723 		    &as->arcstat_anon_evictable_metadata);
4724 		arc_kstat_update_state(arc_mru,
4725 		    &as->arcstat_mru_size,
4726 		    &as->arcstat_mru_evictable_data,
4727 		    &as->arcstat_mru_evictable_metadata);
4728 		arc_kstat_update_state(arc_mru_ghost,
4729 		    &as->arcstat_mru_ghost_size,
4730 		    &as->arcstat_mru_ghost_evictable_data,
4731 		    &as->arcstat_mru_ghost_evictable_metadata);
4732 		arc_kstat_update_state(arc_mfu,
4733 		    &as->arcstat_mfu_size,
4734 		    &as->arcstat_mfu_evictable_data,
4735 		    &as->arcstat_mfu_evictable_metadata);
4736 		arc_kstat_update_state(arc_mfu_ghost,
4737 		    &as->arcstat_mfu_ghost_size,
4738 		    &as->arcstat_mfu_ghost_evictable_data,
4739 		    &as->arcstat_mfu_ghost_evictable_metadata);
4740 	}
4741 
4742 	return (0);
4743 }
4744 
4745 /*
4746  * This function *must* return indices evenly distributed between all
4747  * sublists of the multilist. This is needed due to how the ARC eviction
4748  * code is laid out; arc_evict_state() assumes ARC buffers are evenly
4749  * distributed between all sublists and uses this assumption when
4750  * deciding which sublist to evict from and how much to evict from it.
4751  */
4752 unsigned int
4753 arc_state_multilist_index_func(multilist_t *ml, void *obj)
4754 {
4755 	arc_buf_hdr_t *hdr = obj;
4756 
4757 	/*
4758 	 * We rely on b_dva to generate evenly distributed index
4759 	 * numbers using buf_hash below. So, as an added precaution,
4760 	 * let's make sure we never add empty buffers to the arc lists.
4761 	 */
4762 	ASSERT(!BUF_EMPTY(hdr));
4763 
4764 	/*
4765 	 * The assumption here, is the hash value for a given
4766 	 * arc_buf_hdr_t will remain constant throughout it's lifetime
4767 	 * (i.e. it's b_spa, b_dva, and b_birth fields don't change).
4768 	 * Thus, we don't need to store the header's sublist index
4769 	 * on insertion, as this index can be recalculated on removal.
4770 	 *
4771 	 * Also, the low order bits of the hash value are thought to be
4772 	 * distributed evenly. Otherwise, in the case that the multilist
4773 	 * has a power of two number of sublists, each sublists' usage
4774 	 * would not be evenly distributed.
4775 	 */
4776 	return (buf_hash(hdr->b_spa, &hdr->b_dva, hdr->b_birth) %
4777 	    multilist_get_num_sublists(ml));
4778 }
4779 
4780 void
4781 arc_init(void)
4782 {
4783 	/*
4784 	 * allmem is "all memory that we could possibly use".
4785 	 */
4786 #ifdef _KERNEL
4787 	uint64_t allmem = ptob(physmem - swapfs_minfree);
4788 #else
4789 	uint64_t allmem = (physmem * PAGESIZE) / 2;
4790 #endif
4791 
4792 	mutex_init(&arc_reclaim_lock, NULL, MUTEX_DEFAULT, NULL);
4793 	cv_init(&arc_reclaim_thread_cv, NULL, CV_DEFAULT, NULL);
4794 	cv_init(&arc_reclaim_waiters_cv, NULL, CV_DEFAULT, NULL);
4795 
4796 	mutex_init(&arc_user_evicts_lock, NULL, MUTEX_DEFAULT, NULL);
4797 	cv_init(&arc_user_evicts_cv, NULL, CV_DEFAULT, NULL);
4798 
4799 	/* Convert seconds to clock ticks */
4800 	arc_min_prefetch_lifespan = 1 * hz;
4801 
4802 	/* Start out with 1/8 of all memory */
4803 	arc_c = allmem / 8;
4804 
4805 #ifdef _KERNEL
4806 	/*
4807 	 * On architectures where the physical memory can be larger
4808 	 * than the addressable space (intel in 32-bit mode), we may
4809 	 * need to limit the cache to 1/8 of VM size.
4810 	 */
4811 	arc_c = MIN(arc_c, vmem_size(heap_arena, VMEM_ALLOC | VMEM_FREE) / 8);
4812 #endif
4813 
4814 	/* set min cache to 1/32 of all memory, or 64MB, whichever is more */
4815 	arc_c_min = MAX(allmem / 32, 64 << 20);
4816 	/* set max to 3/4 of all memory, or all but 1GB, whichever is more */
4817 	if (allmem >= 1 << 30)
4818 		arc_c_max = allmem - (1 << 30);
4819 	else
4820 		arc_c_max = arc_c_min;
4821 	arc_c_max = MAX(allmem * 3 / 4, arc_c_max);
4822 
4823 	/*
4824 	 * Allow the tunables to override our calculations if they are
4825 	 * reasonable (ie. over 64MB)
4826 	 */
4827 	if (zfs_arc_max > 64 << 20 && zfs_arc_max < allmem)
4828 		arc_c_max = zfs_arc_max;
4829 	if (zfs_arc_min > 64 << 20 && zfs_arc_min <= arc_c_max)
4830 		arc_c_min = zfs_arc_min;
4831 
4832 	arc_c = arc_c_max;
4833 	arc_p = (arc_c >> 1);
4834 
4835 	/* limit meta-data to 1/4 of the arc capacity */
4836 	arc_meta_limit = arc_c_max / 4;
4837 
4838 	/* Allow the tunable to override if it is reasonable */
4839 	if (zfs_arc_meta_limit > 0 && zfs_arc_meta_limit <= arc_c_max)
4840 		arc_meta_limit = zfs_arc_meta_limit;
4841 
4842 	if (arc_c_min < arc_meta_limit / 2 && zfs_arc_min == 0)
4843 		arc_c_min = arc_meta_limit / 2;
4844 
4845 	if (zfs_arc_meta_min > 0) {
4846 		arc_meta_min = zfs_arc_meta_min;
4847 	} else {
4848 		arc_meta_min = arc_c_min / 2;
4849 	}
4850 
4851 	if (zfs_arc_grow_retry > 0)
4852 		arc_grow_retry = zfs_arc_grow_retry;
4853 
4854 	if (zfs_arc_shrink_shift > 0)
4855 		arc_shrink_shift = zfs_arc_shrink_shift;
4856 
4857 	/*
4858 	 * Ensure that arc_no_grow_shift is less than arc_shrink_shift.
4859 	 */
4860 	if (arc_no_grow_shift >= arc_shrink_shift)
4861 		arc_no_grow_shift = arc_shrink_shift - 1;
4862 
4863 	if (zfs_arc_p_min_shift > 0)
4864 		arc_p_min_shift = zfs_arc_p_min_shift;
4865 
4866 	if (zfs_arc_num_sublists_per_state < 1)
4867 		zfs_arc_num_sublists_per_state = MAX(boot_ncpus, 1);
4868 
4869 	/* if kmem_flags are set, lets try to use less memory */
4870 	if (kmem_debugging())
4871 		arc_c = arc_c / 2;
4872 	if (arc_c < arc_c_min)
4873 		arc_c = arc_c_min;
4874 
4875 	arc_anon = &ARC_anon;
4876 	arc_mru = &ARC_mru;
4877 	arc_mru_ghost = &ARC_mru_ghost;
4878 	arc_mfu = &ARC_mfu;
4879 	arc_mfu_ghost = &ARC_mfu_ghost;
4880 	arc_l2c_only = &ARC_l2c_only;
4881 	arc_size = 0;
4882 
4883 	multilist_create(&arc_mru->arcs_list[ARC_BUFC_METADATA],
4884 	    sizeof (arc_buf_hdr_t),
4885 	    offsetof(arc_buf_hdr_t, b_l1hdr.b_arc_node),
4886 	    zfs_arc_num_sublists_per_state, arc_state_multilist_index_func);
4887 	multilist_create(&arc_mru->arcs_list[ARC_BUFC_DATA],
4888 	    sizeof (arc_buf_hdr_t),
4889 	    offsetof(arc_buf_hdr_t, b_l1hdr.b_arc_node),
4890 	    zfs_arc_num_sublists_per_state, arc_state_multilist_index_func);
4891 	multilist_create(&arc_mru_ghost->arcs_list[ARC_BUFC_METADATA],
4892 	    sizeof (arc_buf_hdr_t),
4893 	    offsetof(arc_buf_hdr_t, b_l1hdr.b_arc_node),
4894 	    zfs_arc_num_sublists_per_state, arc_state_multilist_index_func);
4895 	multilist_create(&arc_mru_ghost->arcs_list[ARC_BUFC_DATA],
4896 	    sizeof (arc_buf_hdr_t),
4897 	    offsetof(arc_buf_hdr_t, b_l1hdr.b_arc_node),
4898 	    zfs_arc_num_sublists_per_state, arc_state_multilist_index_func);
4899 	multilist_create(&arc_mfu->arcs_list[ARC_BUFC_METADATA],
4900 	    sizeof (arc_buf_hdr_t),
4901 	    offsetof(arc_buf_hdr_t, b_l1hdr.b_arc_node),
4902 	    zfs_arc_num_sublists_per_state, arc_state_multilist_index_func);
4903 	multilist_create(&arc_mfu->arcs_list[ARC_BUFC_DATA],
4904 	    sizeof (arc_buf_hdr_t),
4905 	    offsetof(arc_buf_hdr_t, b_l1hdr.b_arc_node),
4906 	    zfs_arc_num_sublists_per_state, arc_state_multilist_index_func);
4907 	multilist_create(&arc_mfu_ghost->arcs_list[ARC_BUFC_METADATA],
4908 	    sizeof (arc_buf_hdr_t),
4909 	    offsetof(arc_buf_hdr_t, b_l1hdr.b_arc_node),
4910 	    zfs_arc_num_sublists_per_state, arc_state_multilist_index_func);
4911 	multilist_create(&arc_mfu_ghost->arcs_list[ARC_BUFC_DATA],
4912 	    sizeof (arc_buf_hdr_t),
4913 	    offsetof(arc_buf_hdr_t, b_l1hdr.b_arc_node),
4914 	    zfs_arc_num_sublists_per_state, arc_state_multilist_index_func);
4915 	multilist_create(&arc_l2c_only->arcs_list[ARC_BUFC_METADATA],
4916 	    sizeof (arc_buf_hdr_t),
4917 	    offsetof(arc_buf_hdr_t, b_l1hdr.b_arc_node),
4918 	    zfs_arc_num_sublists_per_state, arc_state_multilist_index_func);
4919 	multilist_create(&arc_l2c_only->arcs_list[ARC_BUFC_DATA],
4920 	    sizeof (arc_buf_hdr_t),
4921 	    offsetof(arc_buf_hdr_t, b_l1hdr.b_arc_node),
4922 	    zfs_arc_num_sublists_per_state, arc_state_multilist_index_func);
4923 
4924 	buf_init();
4925 
4926 	arc_reclaim_thread_exit = FALSE;
4927 	arc_user_evicts_thread_exit = FALSE;
4928 	arc_eviction_list = NULL;
4929 	bzero(&arc_eviction_hdr, sizeof (arc_buf_hdr_t));
4930 
4931 	arc_ksp = kstat_create("zfs", 0, "arcstats", "misc", KSTAT_TYPE_NAMED,
4932 	    sizeof (arc_stats) / sizeof (kstat_named_t), KSTAT_FLAG_VIRTUAL);
4933 
4934 	if (arc_ksp != NULL) {
4935 		arc_ksp->ks_data = &arc_stats;
4936 		arc_ksp->ks_update = arc_kstat_update;
4937 		kstat_install(arc_ksp);
4938 	}
4939 
4940 	(void) thread_create(NULL, 0, arc_reclaim_thread, NULL, 0, &p0,
4941 	    TS_RUN, minclsyspri);
4942 
4943 	(void) thread_create(NULL, 0, arc_user_evicts_thread, NULL, 0, &p0,
4944 	    TS_RUN, minclsyspri);
4945 
4946 	arc_dead = FALSE;
4947 	arc_warm = B_FALSE;
4948 
4949 	/*
4950 	 * Calculate maximum amount of dirty data per pool.
4951 	 *
4952 	 * If it has been set by /etc/system, take that.
4953 	 * Otherwise, use a percentage of physical memory defined by
4954 	 * zfs_dirty_data_max_percent (default 10%) with a cap at
4955 	 * zfs_dirty_data_max_max (default 4GB).
4956 	 */
4957 	if (zfs_dirty_data_max == 0) {
4958 		zfs_dirty_data_max = physmem * PAGESIZE *
4959 		    zfs_dirty_data_max_percent / 100;
4960 		zfs_dirty_data_max = MIN(zfs_dirty_data_max,
4961 		    zfs_dirty_data_max_max);
4962 	}
4963 }
4964 
4965 void
4966 arc_fini(void)
4967 {
4968 	mutex_enter(&arc_reclaim_lock);
4969 	arc_reclaim_thread_exit = TRUE;
4970 	/*
4971 	 * The reclaim thread will set arc_reclaim_thread_exit back to
4972 	 * FALSE when it is finished exiting; we're waiting for that.
4973 	 */
4974 	while (arc_reclaim_thread_exit) {
4975 		cv_signal(&arc_reclaim_thread_cv);
4976 		cv_wait(&arc_reclaim_thread_cv, &arc_reclaim_lock);
4977 	}
4978 	mutex_exit(&arc_reclaim_lock);
4979 
4980 	mutex_enter(&arc_user_evicts_lock);
4981 	arc_user_evicts_thread_exit = TRUE;
4982 	/*
4983 	 * The user evicts thread will set arc_user_evicts_thread_exit
4984 	 * to FALSE when it is finished exiting; we're waiting for that.
4985 	 */
4986 	while (arc_user_evicts_thread_exit) {
4987 		cv_signal(&arc_user_evicts_cv);
4988 		cv_wait(&arc_user_evicts_cv, &arc_user_evicts_lock);
4989 	}
4990 	mutex_exit(&arc_user_evicts_lock);
4991 
4992 	/* Use TRUE to ensure *all* buffers are evicted */
4993 	arc_flush(NULL, TRUE);
4994 
4995 	arc_dead = TRUE;
4996 
4997 	if (arc_ksp != NULL) {
4998 		kstat_delete(arc_ksp);
4999 		arc_ksp = NULL;
5000 	}
5001 
5002 	mutex_destroy(&arc_reclaim_lock);
5003 	cv_destroy(&arc_reclaim_thread_cv);
5004 	cv_destroy(&arc_reclaim_waiters_cv);
5005 
5006 	mutex_destroy(&arc_user_evicts_lock);
5007 	cv_destroy(&arc_user_evicts_cv);
5008 
5009 	multilist_destroy(&arc_mru->arcs_list[ARC_BUFC_METADATA]);
5010 	multilist_destroy(&arc_mru_ghost->arcs_list[ARC_BUFC_METADATA]);
5011 	multilist_destroy(&arc_mfu->arcs_list[ARC_BUFC_METADATA]);
5012 	multilist_destroy(&arc_mfu_ghost->arcs_list[ARC_BUFC_METADATA]);
5013 	multilist_destroy(&arc_mru->arcs_list[ARC_BUFC_DATA]);
5014 	multilist_destroy(&arc_mru_ghost->arcs_list[ARC_BUFC_DATA]);
5015 	multilist_destroy(&arc_mfu->arcs_list[ARC_BUFC_DATA]);
5016 	multilist_destroy(&arc_mfu_ghost->arcs_list[ARC_BUFC_DATA]);
5017 
5018 	buf_fini();
5019 
5020 	ASSERT0(arc_loaned_bytes);
5021 }
5022 
5023 /*
5024  * Level 2 ARC
5025  *
5026  * The level 2 ARC (L2ARC) is a cache layer in-between main memory and disk.
5027  * It uses dedicated storage devices to hold cached data, which are populated
5028  * using large infrequent writes.  The main role of this cache is to boost
5029  * the performance of random read workloads.  The intended L2ARC devices
5030  * include short-stroked disks, solid state disks, and other media with
5031  * substantially faster read latency than disk.
5032  *
5033  *                 +-----------------------+
5034  *                 |         ARC           |
5035  *                 +-----------------------+
5036  *                    |         ^     ^
5037  *                    |         |     |
5038  *      l2arc_feed_thread()    arc_read()
5039  *                    |         |     |
5040  *                    |  l2arc read   |
5041  *                    V         |     |
5042  *               +---------------+    |
5043  *               |     L2ARC     |    |
5044  *               +---------------+    |
5045  *                   |    ^           |
5046  *          l2arc_write() |           |
5047  *                   |    |           |
5048  *                   V    |           |
5049  *                 +-------+      +-------+
5050  *                 | vdev  |      | vdev  |
5051  *                 | cache |      | cache |
5052  *                 +-------+      +-------+
5053  *                 +=========+     .-----.
5054  *                 :  L2ARC  :    |-_____-|
5055  *                 : devices :    | Disks |
5056  *                 +=========+    `-_____-'
5057  *
5058  * Read requests are satisfied from the following sources, in order:
5059  *
5060  *	1) ARC
5061  *	2) vdev cache of L2ARC devices
5062  *	3) L2ARC devices
5063  *	4) vdev cache of disks
5064  *	5) disks
5065  *
5066  * Some L2ARC device types exhibit extremely slow write performance.
5067  * To accommodate for this there are some significant differences between
5068  * the L2ARC and traditional cache design:
5069  *
5070  * 1. There is no eviction path from the ARC to the L2ARC.  Evictions from
5071  * the ARC behave as usual, freeing buffers and placing headers on ghost
5072  * lists.  The ARC does not send buffers to the L2ARC during eviction as
5073  * this would add inflated write latencies for all ARC memory pressure.
5074  *
5075  * 2. The L2ARC attempts to cache data from the ARC before it is evicted.
5076  * It does this by periodically scanning buffers from the eviction-end of
5077  * the MFU and MRU ARC lists, copying them to the L2ARC devices if they are
5078  * not already there. It scans until a headroom of buffers is satisfied,
5079  * which itself is a buffer for ARC eviction. If a compressible buffer is
5080  * found during scanning and selected for writing to an L2ARC device, we
5081  * temporarily boost scanning headroom during the next scan cycle to make
5082  * sure we adapt to compression effects (which might significantly reduce
5083  * the data volume we write to L2ARC). The thread that does this is
5084  * l2arc_feed_thread(), illustrated below; example sizes are included to
5085  * provide a better sense of ratio than this diagram:
5086  *
5087  *	       head -->                        tail
5088  *	        +---------------------+----------+
5089  *	ARC_mfu |:::::#:::::::::::::::|o#o###o###|-->.   # already on L2ARC
5090  *	        +---------------------+----------+   |   o L2ARC eligible
5091  *	ARC_mru |:#:::::::::::::::::::|#o#ooo####|-->|   : ARC buffer
5092  *	        +---------------------+----------+   |
5093  *	             15.9 Gbytes      ^ 32 Mbytes    |
5094  *	                           headroom          |
5095  *	                                      l2arc_feed_thread()
5096  *	                                             |
5097  *	                 l2arc write hand <--[oooo]--'
5098  *	                         |           8 Mbyte
5099  *	                         |          write max
5100  *	                         V
5101  *		  +==============================+
5102  *	L2ARC dev |####|#|###|###|    |####| ... |
5103  *	          +==============================+
5104  *	                     32 Gbytes
5105  *
5106  * 3. If an ARC buffer is copied to the L2ARC but then hit instead of
5107  * evicted, then the L2ARC has cached a buffer much sooner than it probably
5108  * needed to, potentially wasting L2ARC device bandwidth and storage.  It is
5109  * safe to say that this is an uncommon case, since buffers at the end of
5110  * the ARC lists have moved there due to inactivity.
5111  *
5112  * 4. If the ARC evicts faster than the L2ARC can maintain a headroom,
5113  * then the L2ARC simply misses copying some buffers.  This serves as a
5114  * pressure valve to prevent heavy read workloads from both stalling the ARC
5115  * with waits and clogging the L2ARC with writes.  This also helps prevent
5116  * the potential for the L2ARC to churn if it attempts to cache content too
5117  * quickly, such as during backups of the entire pool.
5118  *
5119  * 5. After system boot and before the ARC has filled main memory, there are
5120  * no evictions from the ARC and so the tails of the ARC_mfu and ARC_mru
5121  * lists can remain mostly static.  Instead of searching from tail of these
5122  * lists as pictured, the l2arc_feed_thread() will search from the list heads
5123  * for eligible buffers, greatly increasing its chance of finding them.
5124  *
5125  * The L2ARC device write speed is also boosted during this time so that
5126  * the L2ARC warms up faster.  Since there have been no ARC evictions yet,
5127  * there are no L2ARC reads, and no fear of degrading read performance
5128  * through increased writes.
5129  *
5130  * 6. Writes to the L2ARC devices are grouped and sent in-sequence, so that
5131  * the vdev queue can aggregate them into larger and fewer writes.  Each
5132  * device is written to in a rotor fashion, sweeping writes through
5133  * available space then repeating.
5134  *
5135  * 7. The L2ARC does not store dirty content.  It never needs to flush
5136  * write buffers back to disk based storage.
5137  *
5138  * 8. If an ARC buffer is written (and dirtied) which also exists in the
5139  * L2ARC, the now stale L2ARC buffer is immediately dropped.
5140  *
5141  * The performance of the L2ARC can be tweaked by a number of tunables, which
5142  * may be necessary for different workloads:
5143  *
5144  *	l2arc_write_max		max write bytes per interval
5145  *	l2arc_write_boost	extra write bytes during device warmup
5146  *	l2arc_noprefetch	skip caching prefetched buffers
5147  *	l2arc_headroom		number of max device writes to precache
5148  *	l2arc_headroom_boost	when we find compressed buffers during ARC
5149  *				scanning, we multiply headroom by this
5150  *				percentage factor for the next scan cycle,
5151  *				since more compressed buffers are likely to
5152  *				be present
5153  *	l2arc_feed_secs		seconds between L2ARC writing
5154  *
5155  * Tunables may be removed or added as future performance improvements are
5156  * integrated, and also may become zpool properties.
5157  *
5158  * There are three key functions that control how the L2ARC warms up:
5159  *
5160  *	l2arc_write_eligible()	check if a buffer is eligible to cache
5161  *	l2arc_write_size()	calculate how much to write
5162  *	l2arc_write_interval()	calculate sleep delay between writes
5163  *
5164  * These three functions determine what to write, how much, and how quickly
5165  * to send writes.
5166  */
5167 
5168 static boolean_t
5169 l2arc_write_eligible(uint64_t spa_guid, arc_buf_hdr_t *hdr)
5170 {
5171 	/*
5172 	 * A buffer is *not* eligible for the L2ARC if it:
5173 	 * 1. belongs to a different spa.
5174 	 * 2. is already cached on the L2ARC.
5175 	 * 3. has an I/O in progress (it may be an incomplete read).
5176 	 * 4. is flagged not eligible (zfs property).
5177 	 */
5178 	if (hdr->b_spa != spa_guid || HDR_HAS_L2HDR(hdr) ||
5179 	    HDR_IO_IN_PROGRESS(hdr) || !HDR_L2CACHE(hdr))
5180 		return (B_FALSE);
5181 
5182 	return (B_TRUE);
5183 }
5184 
5185 static uint64_t
5186 l2arc_write_size(void)
5187 {
5188 	uint64_t size;
5189 
5190 	/*
5191 	 * Make sure our globals have meaningful values in case the user
5192 	 * altered them.
5193 	 */
5194 	size = l2arc_write_max;
5195 	if (size == 0) {
5196 		cmn_err(CE_NOTE, "Bad value for l2arc_write_max, value must "
5197 		    "be greater than zero, resetting it to the default (%d)",
5198 		    L2ARC_WRITE_SIZE);
5199 		size = l2arc_write_max = L2ARC_WRITE_SIZE;
5200 	}
5201 
5202 	if (arc_warm == B_FALSE)
5203 		size += l2arc_write_boost;
5204 
5205 	return (size);
5206 
5207 }
5208 
5209 static clock_t
5210 l2arc_write_interval(clock_t began, uint64_t wanted, uint64_t wrote)
5211 {
5212 	clock_t interval, next, now;
5213 
5214 	/*
5215 	 * If the ARC lists are busy, increase our write rate; if the
5216 	 * lists are stale, idle back.  This is achieved by checking
5217 	 * how much we previously wrote - if it was more than half of
5218 	 * what we wanted, schedule the next write much sooner.
5219 	 */
5220 	if (l2arc_feed_again && wrote > (wanted / 2))
5221 		interval = (hz * l2arc_feed_min_ms) / 1000;
5222 	else
5223 		interval = hz * l2arc_feed_secs;
5224 
5225 	now = ddi_get_lbolt();
5226 	next = MAX(now, MIN(now + interval, began + interval));
5227 
5228 	return (next);
5229 }
5230 
5231 /*
5232  * Cycle through L2ARC devices.  This is how L2ARC load balances.
5233  * If a device is returned, this also returns holding the spa config lock.
5234  */
5235 static l2arc_dev_t *
5236 l2arc_dev_get_next(void)
5237 {
5238 	l2arc_dev_t *first, *next = NULL;
5239 
5240 	/*
5241 	 * Lock out the removal of spas (spa_namespace_lock), then removal
5242 	 * of cache devices (l2arc_dev_mtx).  Once a device has been selected,
5243 	 * both locks will be dropped and a spa config lock held instead.
5244 	 */
5245 	mutex_enter(&spa_namespace_lock);
5246 	mutex_enter(&l2arc_dev_mtx);
5247 
5248 	/* if there are no vdevs, there is nothing to do */
5249 	if (l2arc_ndev == 0)
5250 		goto out;
5251 
5252 	first = NULL;
5253 	next = l2arc_dev_last;
5254 	do {
5255 		/* loop around the list looking for a non-faulted vdev */
5256 		if (next == NULL) {
5257 			next = list_head(l2arc_dev_list);
5258 		} else {
5259 			next = list_next(l2arc_dev_list, next);
5260 			if (next == NULL)
5261 				next = list_head(l2arc_dev_list);
5262 		}
5263 
5264 		/* if we have come back to the start, bail out */
5265 		if (first == NULL)
5266 			first = next;
5267 		else if (next == first)
5268 			break;
5269 
5270 	} while (vdev_is_dead(next->l2ad_vdev));
5271 
5272 	/* if we were unable to find any usable vdevs, return NULL */
5273 	if (vdev_is_dead(next->l2ad_vdev))
5274 		next = NULL;
5275 
5276 	l2arc_dev_last = next;
5277 
5278 out:
5279 	mutex_exit(&l2arc_dev_mtx);
5280 
5281 	/*
5282 	 * Grab the config lock to prevent the 'next' device from being
5283 	 * removed while we are writing to it.
5284 	 */
5285 	if (next != NULL)
5286 		spa_config_enter(next->l2ad_spa, SCL_L2ARC, next, RW_READER);
5287 	mutex_exit(&spa_namespace_lock);
5288 
5289 	return (next);
5290 }
5291 
5292 /*
5293  * Free buffers that were tagged for destruction.
5294  */
5295 static void
5296 l2arc_do_free_on_write()
5297 {
5298 	list_t *buflist;
5299 	l2arc_data_free_t *df, *df_prev;
5300 
5301 	mutex_enter(&l2arc_free_on_write_mtx);
5302 	buflist = l2arc_free_on_write;
5303 
5304 	for (df = list_tail(buflist); df; df = df_prev) {
5305 		df_prev = list_prev(buflist, df);
5306 		ASSERT(df->l2df_data != NULL);
5307 		ASSERT(df->l2df_func != NULL);
5308 		df->l2df_func(df->l2df_data, df->l2df_size);
5309 		list_remove(buflist, df);
5310 		kmem_free(df, sizeof (l2arc_data_free_t));
5311 	}
5312 
5313 	mutex_exit(&l2arc_free_on_write_mtx);
5314 }
5315 
5316 /*
5317  * A write to a cache device has completed.  Update all headers to allow
5318  * reads from these buffers to begin.
5319  */
5320 static void
5321 l2arc_write_done(zio_t *zio)
5322 {
5323 	l2arc_write_callback_t *cb;
5324 	l2arc_dev_t *dev;
5325 	list_t *buflist;
5326 	arc_buf_hdr_t *head, *hdr, *hdr_prev;
5327 	kmutex_t *hash_lock;
5328 	int64_t bytes_dropped = 0;
5329 
5330 	cb = zio->io_private;
5331 	ASSERT(cb != NULL);
5332 	dev = cb->l2wcb_dev;
5333 	ASSERT(dev != NULL);
5334 	head = cb->l2wcb_head;
5335 	ASSERT(head != NULL);
5336 	buflist = &dev->l2ad_buflist;
5337 	ASSERT(buflist != NULL);
5338 	DTRACE_PROBE2(l2arc__iodone, zio_t *, zio,
5339 	    l2arc_write_callback_t *, cb);
5340 
5341 	if (zio->io_error != 0)
5342 		ARCSTAT_BUMP(arcstat_l2_writes_error);
5343 
5344 	/*
5345 	 * All writes completed, or an error was hit.
5346 	 */
5347 top:
5348 	mutex_enter(&dev->l2ad_mtx);
5349 	for (hdr = list_prev(buflist, head); hdr; hdr = hdr_prev) {
5350 		hdr_prev = list_prev(buflist, hdr);
5351 
5352 		hash_lock = HDR_LOCK(hdr);
5353 
5354 		/*
5355 		 * We cannot use mutex_enter or else we can deadlock
5356 		 * with l2arc_write_buffers (due to swapping the order
5357 		 * the hash lock and l2ad_mtx are taken).
5358 		 */
5359 		if (!mutex_tryenter(hash_lock)) {
5360 			/*
5361 			 * Missed the hash lock. We must retry so we
5362 			 * don't leave the ARC_FLAG_L2_WRITING bit set.
5363 			 */
5364 			ARCSTAT_BUMP(arcstat_l2_writes_lock_retry);
5365 
5366 			/*
5367 			 * We don't want to rescan the headers we've
5368 			 * already marked as having been written out, so
5369 			 * we reinsert the head node so we can pick up
5370 			 * where we left off.
5371 			 */
5372 			list_remove(buflist, head);
5373 			list_insert_after(buflist, hdr, head);
5374 
5375 			mutex_exit(&dev->l2ad_mtx);
5376 
5377 			/*
5378 			 * We wait for the hash lock to become available
5379 			 * to try and prevent busy waiting, and increase
5380 			 * the chance we'll be able to acquire the lock
5381 			 * the next time around.
5382 			 */
5383 			mutex_enter(hash_lock);
5384 			mutex_exit(hash_lock);
5385 			goto top;
5386 		}
5387 
5388 		/*
5389 		 * We could not have been moved into the arc_l2c_only
5390 		 * state while in-flight due to our ARC_FLAG_L2_WRITING
5391 		 * bit being set. Let's just ensure that's being enforced.
5392 		 */
5393 		ASSERT(HDR_HAS_L1HDR(hdr));
5394 
5395 		/*
5396 		 * We may have allocated a buffer for L2ARC compression,
5397 		 * we must release it to avoid leaking this data.
5398 		 */
5399 		l2arc_release_cdata_buf(hdr);
5400 
5401 		if (zio->io_error != 0) {
5402 			/*
5403 			 * Error - drop L2ARC entry.
5404 			 */
5405 			list_remove(buflist, hdr);
5406 			hdr->b_flags &= ~ARC_FLAG_HAS_L2HDR;
5407 
5408 			ARCSTAT_INCR(arcstat_l2_asize, -hdr->b_l2hdr.b_asize);
5409 			ARCSTAT_INCR(arcstat_l2_size, -hdr->b_size);
5410 		}
5411 
5412 		/*
5413 		 * Allow ARC to begin reads and ghost list evictions to
5414 		 * this L2ARC entry.
5415 		 */
5416 		hdr->b_flags &= ~ARC_FLAG_L2_WRITING;
5417 
5418 		mutex_exit(hash_lock);
5419 	}
5420 
5421 	atomic_inc_64(&l2arc_writes_done);
5422 	list_remove(buflist, head);
5423 	ASSERT(!HDR_HAS_L1HDR(head));
5424 	kmem_cache_free(hdr_l2only_cache, head);
5425 	mutex_exit(&dev->l2ad_mtx);
5426 
5427 	vdev_space_update(dev->l2ad_vdev, -bytes_dropped, 0, 0);
5428 
5429 	l2arc_do_free_on_write();
5430 
5431 	kmem_free(cb, sizeof (l2arc_write_callback_t));
5432 }
5433 
5434 /*
5435  * A read to a cache device completed.  Validate buffer contents before
5436  * handing over to the regular ARC routines.
5437  */
5438 static void
5439 l2arc_read_done(zio_t *zio)
5440 {
5441 	l2arc_read_callback_t *cb;
5442 	arc_buf_hdr_t *hdr;
5443 	arc_buf_t *buf;
5444 	kmutex_t *hash_lock;
5445 	int equal;
5446 
5447 	ASSERT(zio->io_vd != NULL);
5448 	ASSERT(zio->io_flags & ZIO_FLAG_DONT_PROPAGATE);
5449 
5450 	spa_config_exit(zio->io_spa, SCL_L2ARC, zio->io_vd);
5451 
5452 	cb = zio->io_private;
5453 	ASSERT(cb != NULL);
5454 	buf = cb->l2rcb_buf;
5455 	ASSERT(buf != NULL);
5456 
5457 	hash_lock = HDR_LOCK(buf->b_hdr);
5458 	mutex_enter(hash_lock);
5459 	hdr = buf->b_hdr;
5460 	ASSERT3P(hash_lock, ==, HDR_LOCK(hdr));
5461 
5462 	/*
5463 	 * If the buffer was compressed, decompress it first.
5464 	 */
5465 	if (cb->l2rcb_compress != ZIO_COMPRESS_OFF)
5466 		l2arc_decompress_zio(zio, hdr, cb->l2rcb_compress);
5467 	ASSERT(zio->io_data != NULL);
5468 
5469 	/*
5470 	 * Check this survived the L2ARC journey.
5471 	 */
5472 	equal = arc_cksum_equal(buf);
5473 	if (equal && zio->io_error == 0 && !HDR_L2_EVICTED(hdr)) {
5474 		mutex_exit(hash_lock);
5475 		zio->io_private = buf;
5476 		zio->io_bp_copy = cb->l2rcb_bp;	/* XXX fix in L2ARC 2.0	*/
5477 		zio->io_bp = &zio->io_bp_copy;	/* XXX fix in L2ARC 2.0	*/
5478 		arc_read_done(zio);
5479 	} else {
5480 		mutex_exit(hash_lock);
5481 		/*
5482 		 * Buffer didn't survive caching.  Increment stats and
5483 		 * reissue to the original storage device.
5484 		 */
5485 		if (zio->io_error != 0) {
5486 			ARCSTAT_BUMP(arcstat_l2_io_error);
5487 		} else {
5488 			zio->io_error = SET_ERROR(EIO);
5489 		}
5490 		if (!equal)
5491 			ARCSTAT_BUMP(arcstat_l2_cksum_bad);
5492 
5493 		/*
5494 		 * If there's no waiter, issue an async i/o to the primary
5495 		 * storage now.  If there *is* a waiter, the caller must
5496 		 * issue the i/o in a context where it's OK to block.
5497 		 */
5498 		if (zio->io_waiter == NULL) {
5499 			zio_t *pio = zio_unique_parent(zio);
5500 
5501 			ASSERT(!pio || pio->io_child_type == ZIO_CHILD_LOGICAL);
5502 
5503 			zio_nowait(zio_read(pio, cb->l2rcb_spa, &cb->l2rcb_bp,
5504 			    buf->b_data, zio->io_size, arc_read_done, buf,
5505 			    zio->io_priority, cb->l2rcb_flags, &cb->l2rcb_zb));
5506 		}
5507 	}
5508 
5509 	kmem_free(cb, sizeof (l2arc_read_callback_t));
5510 }
5511 
5512 /*
5513  * This is the list priority from which the L2ARC will search for pages to
5514  * cache.  This is used within loops (0..3) to cycle through lists in the
5515  * desired order.  This order can have a significant effect on cache
5516  * performance.
5517  *
5518  * Currently the metadata lists are hit first, MFU then MRU, followed by
5519  * the data lists.  This function returns a locked list, and also returns
5520  * the lock pointer.
5521  */
5522 static multilist_sublist_t *
5523 l2arc_sublist_lock(int list_num)
5524 {
5525 	multilist_t *ml = NULL;
5526 	unsigned int idx;
5527 
5528 	ASSERT(list_num >= 0 && list_num <= 3);
5529 
5530 	switch (list_num) {
5531 	case 0:
5532 		ml = &arc_mfu->arcs_list[ARC_BUFC_METADATA];
5533 		break;
5534 	case 1:
5535 		ml = &arc_mru->arcs_list[ARC_BUFC_METADATA];
5536 		break;
5537 	case 2:
5538 		ml = &arc_mfu->arcs_list[ARC_BUFC_DATA];
5539 		break;
5540 	case 3:
5541 		ml = &arc_mru->arcs_list[ARC_BUFC_DATA];
5542 		break;
5543 	}
5544 
5545 	/*
5546 	 * Return a randomly-selected sublist. This is acceptable
5547 	 * because the caller feeds only a little bit of data for each
5548 	 * call (8MB). Subsequent calls will result in different
5549 	 * sublists being selected.
5550 	 */
5551 	idx = multilist_get_random_index(ml);
5552 	return (multilist_sublist_lock(ml, idx));
5553 }
5554 
5555 /*
5556  * Evict buffers from the device write hand to the distance specified in
5557  * bytes.  This distance may span populated buffers, it may span nothing.
5558  * This is clearing a region on the L2ARC device ready for writing.
5559  * If the 'all' boolean is set, every buffer is evicted.
5560  */
5561 static void
5562 l2arc_evict(l2arc_dev_t *dev, uint64_t distance, boolean_t all)
5563 {
5564 	list_t *buflist;
5565 	arc_buf_hdr_t *hdr, *hdr_prev;
5566 	kmutex_t *hash_lock;
5567 	uint64_t taddr;
5568 	int64_t bytes_evicted = 0;
5569 
5570 	buflist = &dev->l2ad_buflist;
5571 
5572 	if (!all && dev->l2ad_first) {
5573 		/*
5574 		 * This is the first sweep through the device.  There is
5575 		 * nothing to evict.
5576 		 */
5577 		return;
5578 	}
5579 
5580 	if (dev->l2ad_hand >= (dev->l2ad_end - (2 * distance))) {
5581 		/*
5582 		 * When nearing the end of the device, evict to the end
5583 		 * before the device write hand jumps to the start.
5584 		 */
5585 		taddr = dev->l2ad_end;
5586 	} else {
5587 		taddr = dev->l2ad_hand + distance;
5588 	}
5589 	DTRACE_PROBE4(l2arc__evict, l2arc_dev_t *, dev, list_t *, buflist,
5590 	    uint64_t, taddr, boolean_t, all);
5591 
5592 top:
5593 	mutex_enter(&dev->l2ad_mtx);
5594 	for (hdr = list_tail(buflist); hdr; hdr = hdr_prev) {
5595 		hdr_prev = list_prev(buflist, hdr);
5596 
5597 		hash_lock = HDR_LOCK(hdr);
5598 
5599 		/*
5600 		 * We cannot use mutex_enter or else we can deadlock
5601 		 * with l2arc_write_buffers (due to swapping the order
5602 		 * the hash lock and l2ad_mtx are taken).
5603 		 */
5604 		if (!mutex_tryenter(hash_lock)) {
5605 			/*
5606 			 * Missed the hash lock.  Retry.
5607 			 */
5608 			ARCSTAT_BUMP(arcstat_l2_evict_lock_retry);
5609 			mutex_exit(&dev->l2ad_mtx);
5610 			mutex_enter(hash_lock);
5611 			mutex_exit(hash_lock);
5612 			goto top;
5613 		}
5614 
5615 		if (HDR_L2_WRITE_HEAD(hdr)) {
5616 			/*
5617 			 * We hit a write head node.  Leave it for
5618 			 * l2arc_write_done().
5619 			 */
5620 			list_remove(buflist, hdr);
5621 			mutex_exit(hash_lock);
5622 			continue;
5623 		}
5624 
5625 		if (!all && HDR_HAS_L2HDR(hdr) &&
5626 		    (hdr->b_l2hdr.b_daddr > taddr ||
5627 		    hdr->b_l2hdr.b_daddr < dev->l2ad_hand)) {
5628 			/*
5629 			 * We've evicted to the target address,
5630 			 * or the end of the device.
5631 			 */
5632 			mutex_exit(hash_lock);
5633 			break;
5634 		}
5635 
5636 		ASSERT(HDR_HAS_L2HDR(hdr));
5637 		if (!HDR_HAS_L1HDR(hdr)) {
5638 			ASSERT(!HDR_L2_READING(hdr));
5639 			/*
5640 			 * This doesn't exist in the ARC.  Destroy.
5641 			 * arc_hdr_destroy() will call list_remove()
5642 			 * and decrement arcstat_l2_size.
5643 			 */
5644 			arc_change_state(arc_anon, hdr, hash_lock);
5645 			arc_hdr_destroy(hdr);
5646 		} else {
5647 			ASSERT(hdr->b_l1hdr.b_state != arc_l2c_only);
5648 			ARCSTAT_BUMP(arcstat_l2_evict_l1cached);
5649 			/*
5650 			 * Invalidate issued or about to be issued
5651 			 * reads, since we may be about to write
5652 			 * over this location.
5653 			 */
5654 			if (HDR_L2_READING(hdr)) {
5655 				ARCSTAT_BUMP(arcstat_l2_evict_reading);
5656 				hdr->b_flags |= ARC_FLAG_L2_EVICTED;
5657 			}
5658 
5659 			/* Tell ARC this no longer exists in L2ARC. */
5660 			ARCSTAT_INCR(arcstat_l2_asize, -hdr->b_l2hdr.b_asize);
5661 			ARCSTAT_INCR(arcstat_l2_size, -hdr->b_size);
5662 			hdr->b_flags &= ~ARC_FLAG_HAS_L2HDR;
5663 			list_remove(buflist, hdr);
5664 
5665 			/* Ensure this header has finished being written */
5666 			ASSERT(!HDR_L2_WRITING(hdr));
5667 			ASSERT3P(hdr->b_l1hdr.b_tmp_cdata, ==, NULL);
5668 		}
5669 		mutex_exit(hash_lock);
5670 	}
5671 	mutex_exit(&dev->l2ad_mtx);
5672 
5673 	vdev_space_update(dev->l2ad_vdev, -bytes_evicted, 0, 0);
5674 	dev->l2ad_evict = taddr;
5675 }
5676 
5677 /*
5678  * Find and write ARC buffers to the L2ARC device.
5679  *
5680  * An ARC_FLAG_L2_WRITING flag is set so that the L2ARC buffers are not valid
5681  * for reading until they have completed writing.
5682  * The headroom_boost is an in-out parameter used to maintain headroom boost
5683  * state between calls to this function.
5684  *
5685  * Returns the number of bytes actually written (which may be smaller than
5686  * the delta by which the device hand has changed due to alignment).
5687  */
5688 static uint64_t
5689 l2arc_write_buffers(spa_t *spa, l2arc_dev_t *dev, uint64_t target_sz,
5690     boolean_t *headroom_boost)
5691 {
5692 	arc_buf_hdr_t *hdr, *hdr_prev, *head;
5693 	uint64_t write_asize, write_psize, write_sz, headroom,
5694 	    buf_compress_minsz;
5695 	void *buf_data;
5696 	boolean_t full;
5697 	l2arc_write_callback_t *cb;
5698 	zio_t *pio, *wzio;
5699 	uint64_t guid = spa_load_guid(spa);
5700 	const boolean_t do_headroom_boost = *headroom_boost;
5701 
5702 	ASSERT(dev->l2ad_vdev != NULL);
5703 
5704 	/* Lower the flag now, we might want to raise it again later. */
5705 	*headroom_boost = B_FALSE;
5706 
5707 	pio = NULL;
5708 	write_sz = write_asize = write_psize = 0;
5709 	full = B_FALSE;
5710 	head = kmem_cache_alloc(hdr_l2only_cache, KM_PUSHPAGE);
5711 	head->b_flags |= ARC_FLAG_L2_WRITE_HEAD;
5712 	head->b_flags |= ARC_FLAG_HAS_L2HDR;
5713 
5714 	/*
5715 	 * We will want to try to compress buffers that are at least 2x the
5716 	 * device sector size.
5717 	 */
5718 	buf_compress_minsz = 2 << dev->l2ad_vdev->vdev_ashift;
5719 
5720 	/*
5721 	 * Copy buffers for L2ARC writing.
5722 	 */
5723 	for (int try = 0; try <= 3; try++) {
5724 		multilist_sublist_t *mls = l2arc_sublist_lock(try);
5725 		uint64_t passed_sz = 0;
5726 
5727 		/*
5728 		 * L2ARC fast warmup.
5729 		 *
5730 		 * Until the ARC is warm and starts to evict, read from the
5731 		 * head of the ARC lists rather than the tail.
5732 		 */
5733 		if (arc_warm == B_FALSE)
5734 			hdr = multilist_sublist_head(mls);
5735 		else
5736 			hdr = multilist_sublist_tail(mls);
5737 
5738 		headroom = target_sz * l2arc_headroom;
5739 		if (do_headroom_boost)
5740 			headroom = (headroom * l2arc_headroom_boost) / 100;
5741 
5742 		for (; hdr; hdr = hdr_prev) {
5743 			kmutex_t *hash_lock;
5744 			uint64_t buf_sz;
5745 
5746 			if (arc_warm == B_FALSE)
5747 				hdr_prev = multilist_sublist_next(mls, hdr);
5748 			else
5749 				hdr_prev = multilist_sublist_prev(mls, hdr);
5750 
5751 			hash_lock = HDR_LOCK(hdr);
5752 			if (!mutex_tryenter(hash_lock)) {
5753 				/*
5754 				 * Skip this buffer rather than waiting.
5755 				 */
5756 				continue;
5757 			}
5758 
5759 			passed_sz += hdr->b_size;
5760 			if (passed_sz > headroom) {
5761 				/*
5762 				 * Searched too far.
5763 				 */
5764 				mutex_exit(hash_lock);
5765 				break;
5766 			}
5767 
5768 			if (!l2arc_write_eligible(guid, hdr)) {
5769 				mutex_exit(hash_lock);
5770 				continue;
5771 			}
5772 
5773 			if ((write_sz + hdr->b_size) > target_sz) {
5774 				full = B_TRUE;
5775 				mutex_exit(hash_lock);
5776 				break;
5777 			}
5778 
5779 			if (pio == NULL) {
5780 				/*
5781 				 * Insert a dummy header on the buflist so
5782 				 * l2arc_write_done() can find where the
5783 				 * write buffers begin without searching.
5784 				 */
5785 				mutex_enter(&dev->l2ad_mtx);
5786 				list_insert_head(&dev->l2ad_buflist, head);
5787 				mutex_exit(&dev->l2ad_mtx);
5788 
5789 				cb = kmem_alloc(
5790 				    sizeof (l2arc_write_callback_t), KM_SLEEP);
5791 				cb->l2wcb_dev = dev;
5792 				cb->l2wcb_head = head;
5793 				pio = zio_root(spa, l2arc_write_done, cb,
5794 				    ZIO_FLAG_CANFAIL);
5795 			}
5796 
5797 			/*
5798 			 * Create and add a new L2ARC header.
5799 			 */
5800 			hdr->b_l2hdr.b_dev = dev;
5801 			hdr->b_flags |= ARC_FLAG_L2_WRITING;
5802 			/*
5803 			 * Temporarily stash the data buffer in b_tmp_cdata.
5804 			 * The subsequent write step will pick it up from
5805 			 * there. This is because can't access b_l1hdr.b_buf
5806 			 * without holding the hash_lock, which we in turn
5807 			 * can't access without holding the ARC list locks
5808 			 * (which we want to avoid during compression/writing).
5809 			 */
5810 			HDR_SET_COMPRESS(hdr, ZIO_COMPRESS_OFF);
5811 			hdr->b_l2hdr.b_asize = hdr->b_size;
5812 			hdr->b_l1hdr.b_tmp_cdata = hdr->b_l1hdr.b_buf->b_data;
5813 
5814 			buf_sz = hdr->b_size;
5815 			hdr->b_flags |= ARC_FLAG_HAS_L2HDR;
5816 
5817 			mutex_enter(&dev->l2ad_mtx);
5818 			list_insert_head(&dev->l2ad_buflist, hdr);
5819 			mutex_exit(&dev->l2ad_mtx);
5820 
5821 			/*
5822 			 * Compute and store the buffer cksum before
5823 			 * writing.  On debug the cksum is verified first.
5824 			 */
5825 			arc_cksum_verify(hdr->b_l1hdr.b_buf);
5826 			arc_cksum_compute(hdr->b_l1hdr.b_buf, B_TRUE);
5827 
5828 			mutex_exit(hash_lock);
5829 
5830 			write_sz += buf_sz;
5831 		}
5832 
5833 		multilist_sublist_unlock(mls);
5834 
5835 		if (full == B_TRUE)
5836 			break;
5837 	}
5838 
5839 	/* No buffers selected for writing? */
5840 	if (pio == NULL) {
5841 		ASSERT0(write_sz);
5842 		ASSERT(!HDR_HAS_L1HDR(head));
5843 		kmem_cache_free(hdr_l2only_cache, head);
5844 		return (0);
5845 	}
5846 
5847 	mutex_enter(&dev->l2ad_mtx);
5848 
5849 	/*
5850 	 * Now start writing the buffers. We're starting at the write head
5851 	 * and work backwards, retracing the course of the buffer selector
5852 	 * loop above.
5853 	 */
5854 	for (hdr = list_prev(&dev->l2ad_buflist, head); hdr;
5855 	    hdr = list_prev(&dev->l2ad_buflist, hdr)) {
5856 		uint64_t buf_sz;
5857 
5858 		/*
5859 		 * We rely on the L1 portion of the header below, so
5860 		 * it's invalid for this header to have been evicted out
5861 		 * of the ghost cache, prior to being written out. The
5862 		 * ARC_FLAG_L2_WRITING bit ensures this won't happen.
5863 		 */
5864 		ASSERT(HDR_HAS_L1HDR(hdr));
5865 
5866 		/*
5867 		 * We shouldn't need to lock the buffer here, since we flagged
5868 		 * it as ARC_FLAG_L2_WRITING in the previous step, but we must
5869 		 * take care to only access its L2 cache parameters. In
5870 		 * particular, hdr->l1hdr.b_buf may be invalid by now due to
5871 		 * ARC eviction.
5872 		 */
5873 		hdr->b_l2hdr.b_daddr = dev->l2ad_hand;
5874 
5875 		if ((HDR_L2COMPRESS(hdr)) &&
5876 		    hdr->b_l2hdr.b_asize >= buf_compress_minsz) {
5877 			if (l2arc_compress_buf(hdr)) {
5878 				/*
5879 				 * If compression succeeded, enable headroom
5880 				 * boost on the next scan cycle.
5881 				 */
5882 				*headroom_boost = B_TRUE;
5883 			}
5884 		}
5885 
5886 		/*
5887 		 * Pick up the buffer data we had previously stashed away
5888 		 * (and now potentially also compressed).
5889 		 */
5890 		buf_data = hdr->b_l1hdr.b_tmp_cdata;
5891 		buf_sz = hdr->b_l2hdr.b_asize;
5892 
5893 		/* Compression may have squashed the buffer to zero length. */
5894 		if (buf_sz != 0) {
5895 			uint64_t buf_p_sz;
5896 
5897 			wzio = zio_write_phys(pio, dev->l2ad_vdev,
5898 			    dev->l2ad_hand, buf_sz, buf_data, ZIO_CHECKSUM_OFF,
5899 			    NULL, NULL, ZIO_PRIORITY_ASYNC_WRITE,
5900 			    ZIO_FLAG_CANFAIL, B_FALSE);
5901 
5902 			DTRACE_PROBE2(l2arc__write, vdev_t *, dev->l2ad_vdev,
5903 			    zio_t *, wzio);
5904 			(void) zio_nowait(wzio);
5905 
5906 			write_asize += buf_sz;
5907 			/*
5908 			 * Keep the clock hand suitably device-aligned.
5909 			 */
5910 			buf_p_sz = vdev_psize_to_asize(dev->l2ad_vdev, buf_sz);
5911 			write_psize += buf_p_sz;
5912 			dev->l2ad_hand += buf_p_sz;
5913 		}
5914 	}
5915 
5916 	mutex_exit(&dev->l2ad_mtx);
5917 
5918 	ASSERT3U(write_asize, <=, target_sz);
5919 	ARCSTAT_BUMP(arcstat_l2_writes_sent);
5920 	ARCSTAT_INCR(arcstat_l2_write_bytes, write_asize);
5921 	ARCSTAT_INCR(arcstat_l2_size, write_sz);
5922 	ARCSTAT_INCR(arcstat_l2_asize, write_asize);
5923 	vdev_space_update(dev->l2ad_vdev, write_asize, 0, 0);
5924 
5925 	/*
5926 	 * Bump device hand to the device start if it is approaching the end.
5927 	 * l2arc_evict() will already have evicted ahead for this case.
5928 	 */
5929 	if (dev->l2ad_hand >= (dev->l2ad_end - target_sz)) {
5930 		dev->l2ad_hand = dev->l2ad_start;
5931 		dev->l2ad_evict = dev->l2ad_start;
5932 		dev->l2ad_first = B_FALSE;
5933 	}
5934 
5935 	dev->l2ad_writing = B_TRUE;
5936 	(void) zio_wait(pio);
5937 	dev->l2ad_writing = B_FALSE;
5938 
5939 	return (write_asize);
5940 }
5941 
5942 /*
5943  * Compresses an L2ARC buffer.
5944  * The data to be compressed must be prefilled in l1hdr.b_tmp_cdata and its
5945  * size in l2hdr->b_asize. This routine tries to compress the data and
5946  * depending on the compression result there are three possible outcomes:
5947  * *) The buffer was incompressible. The original l2hdr contents were left
5948  *    untouched and are ready for writing to an L2 device.
5949  * *) The buffer was all-zeros, so there is no need to write it to an L2
5950  *    device. To indicate this situation b_tmp_cdata is NULL'ed, b_asize is
5951  *    set to zero and b_compress is set to ZIO_COMPRESS_EMPTY.
5952  * *) Compression succeeded and b_tmp_cdata was replaced with a temporary
5953  *    data buffer which holds the compressed data to be written, and b_asize
5954  *    tells us how much data there is. b_compress is set to the appropriate
5955  *    compression algorithm. Once writing is done, invoke
5956  *    l2arc_release_cdata_buf on this l2hdr to free this temporary buffer.
5957  *
5958  * Returns B_TRUE if compression succeeded, or B_FALSE if it didn't (the
5959  * buffer was incompressible).
5960  */
5961 static boolean_t
5962 l2arc_compress_buf(arc_buf_hdr_t *hdr)
5963 {
5964 	void *cdata;
5965 	size_t csize, len, rounded;
5966 	ASSERT(HDR_HAS_L2HDR(hdr));
5967 	l2arc_buf_hdr_t *l2hdr = &hdr->b_l2hdr;
5968 
5969 	ASSERT(HDR_HAS_L1HDR(hdr));
5970 	ASSERT(HDR_GET_COMPRESS(hdr) == ZIO_COMPRESS_OFF);
5971 	ASSERT(hdr->b_l1hdr.b_tmp_cdata != NULL);
5972 
5973 	len = l2hdr->b_asize;
5974 	cdata = zio_data_buf_alloc(len);
5975 	ASSERT3P(cdata, !=, NULL);
5976 	csize = zio_compress_data(ZIO_COMPRESS_LZ4, hdr->b_l1hdr.b_tmp_cdata,
5977 	    cdata, l2hdr->b_asize);
5978 
5979 	rounded = P2ROUNDUP(csize, (size_t)SPA_MINBLOCKSIZE);
5980 	if (rounded > csize) {
5981 		bzero((char *)cdata + csize, rounded - csize);
5982 		csize = rounded;
5983 	}
5984 
5985 	if (csize == 0) {
5986 		/* zero block, indicate that there's nothing to write */
5987 		zio_data_buf_free(cdata, len);
5988 		HDR_SET_COMPRESS(hdr, ZIO_COMPRESS_EMPTY);
5989 		l2hdr->b_asize = 0;
5990 		hdr->b_l1hdr.b_tmp_cdata = NULL;
5991 		ARCSTAT_BUMP(arcstat_l2_compress_zeros);
5992 		return (B_TRUE);
5993 	} else if (csize > 0 && csize < len) {
5994 		/*
5995 		 * Compression succeeded, we'll keep the cdata around for
5996 		 * writing and release it afterwards.
5997 		 */
5998 		HDR_SET_COMPRESS(hdr, ZIO_COMPRESS_LZ4);
5999 		l2hdr->b_asize = csize;
6000 		hdr->b_l1hdr.b_tmp_cdata = cdata;
6001 		ARCSTAT_BUMP(arcstat_l2_compress_successes);
6002 		return (B_TRUE);
6003 	} else {
6004 		/*
6005 		 * Compression failed, release the compressed buffer.
6006 		 * l2hdr will be left unmodified.
6007 		 */
6008 		zio_data_buf_free(cdata, len);
6009 		ARCSTAT_BUMP(arcstat_l2_compress_failures);
6010 		return (B_FALSE);
6011 	}
6012 }
6013 
6014 /*
6015  * Decompresses a zio read back from an l2arc device. On success, the
6016  * underlying zio's io_data buffer is overwritten by the uncompressed
6017  * version. On decompression error (corrupt compressed stream), the
6018  * zio->io_error value is set to signal an I/O error.
6019  *
6020  * Please note that the compressed data stream is not checksummed, so
6021  * if the underlying device is experiencing data corruption, we may feed
6022  * corrupt data to the decompressor, so the decompressor needs to be
6023  * able to handle this situation (LZ4 does).
6024  */
6025 static void
6026 l2arc_decompress_zio(zio_t *zio, arc_buf_hdr_t *hdr, enum zio_compress c)
6027 {
6028 	ASSERT(L2ARC_IS_VALID_COMPRESS(c));
6029 
6030 	if (zio->io_error != 0) {
6031 		/*
6032 		 * An io error has occured, just restore the original io
6033 		 * size in preparation for a main pool read.
6034 		 */
6035 		zio->io_orig_size = zio->io_size = hdr->b_size;
6036 		return;
6037 	}
6038 
6039 	if (c == ZIO_COMPRESS_EMPTY) {
6040 		/*
6041 		 * An empty buffer results in a null zio, which means we
6042 		 * need to fill its io_data after we're done restoring the
6043 		 * buffer's contents.
6044 		 */
6045 		ASSERT(hdr->b_l1hdr.b_buf != NULL);
6046 		bzero(hdr->b_l1hdr.b_buf->b_data, hdr->b_size);
6047 		zio->io_data = zio->io_orig_data = hdr->b_l1hdr.b_buf->b_data;
6048 	} else {
6049 		ASSERT(zio->io_data != NULL);
6050 		/*
6051 		 * We copy the compressed data from the start of the arc buffer
6052 		 * (the zio_read will have pulled in only what we need, the
6053 		 * rest is garbage which we will overwrite at decompression)
6054 		 * and then decompress back to the ARC data buffer. This way we
6055 		 * can minimize copying by simply decompressing back over the
6056 		 * original compressed data (rather than decompressing to an
6057 		 * aux buffer and then copying back the uncompressed buffer,
6058 		 * which is likely to be much larger).
6059 		 */
6060 		uint64_t csize;
6061 		void *cdata;
6062 
6063 		csize = zio->io_size;
6064 		cdata = zio_data_buf_alloc(csize);
6065 		bcopy(zio->io_data, cdata, csize);
6066 		if (zio_decompress_data(c, cdata, zio->io_data, csize,
6067 		    hdr->b_size) != 0)
6068 			zio->io_error = EIO;
6069 		zio_data_buf_free(cdata, csize);
6070 	}
6071 
6072 	/* Restore the expected uncompressed IO size. */
6073 	zio->io_orig_size = zio->io_size = hdr->b_size;
6074 }
6075 
6076 /*
6077  * Releases the temporary b_tmp_cdata buffer in an l2arc header structure.
6078  * This buffer serves as a temporary holder of compressed data while
6079  * the buffer entry is being written to an l2arc device. Once that is
6080  * done, we can dispose of it.
6081  */
6082 static void
6083 l2arc_release_cdata_buf(arc_buf_hdr_t *hdr)
6084 {
6085 	enum zio_compress comp = HDR_GET_COMPRESS(hdr);
6086 
6087 	ASSERT(HDR_HAS_L1HDR(hdr));
6088 	ASSERT(comp == ZIO_COMPRESS_OFF || L2ARC_IS_VALID_COMPRESS(comp));
6089 
6090 	if (comp == ZIO_COMPRESS_OFF) {
6091 		/*
6092 		 * In this case, b_tmp_cdata points to the same buffer
6093 		 * as the arc_buf_t's b_data field. We don't want to
6094 		 * free it, since the arc_buf_t will handle that.
6095 		 */
6096 		hdr->b_l1hdr.b_tmp_cdata = NULL;
6097 	} else if (comp == ZIO_COMPRESS_EMPTY) {
6098 		/*
6099 		 * In this case, b_tmp_cdata was compressed to an empty
6100 		 * buffer, thus there's nothing to free and b_tmp_cdata
6101 		 * should have been set to NULL in l2arc_write_buffers().
6102 		 */
6103 		ASSERT3P(hdr->b_l1hdr.b_tmp_cdata, ==, NULL);
6104 	} else {
6105 		/*
6106 		 * If the data was compressed, then we've allocated a
6107 		 * temporary buffer for it, so now we need to release it.
6108 		 */
6109 		ASSERT(hdr->b_l1hdr.b_tmp_cdata != NULL);
6110 		zio_data_buf_free(hdr->b_l1hdr.b_tmp_cdata,
6111 		    hdr->b_size);
6112 		hdr->b_l1hdr.b_tmp_cdata = NULL;
6113 	}
6114 
6115 }
6116 
6117 /*
6118  * This thread feeds the L2ARC at regular intervals.  This is the beating
6119  * heart of the L2ARC.
6120  */
6121 static void
6122 l2arc_feed_thread(void)
6123 {
6124 	callb_cpr_t cpr;
6125 	l2arc_dev_t *dev;
6126 	spa_t *spa;
6127 	uint64_t size, wrote;
6128 	clock_t begin, next = ddi_get_lbolt();
6129 	boolean_t headroom_boost = B_FALSE;
6130 
6131 	CALLB_CPR_INIT(&cpr, &l2arc_feed_thr_lock, callb_generic_cpr, FTAG);
6132 
6133 	mutex_enter(&l2arc_feed_thr_lock);
6134 
6135 	while (l2arc_thread_exit == 0) {
6136 		CALLB_CPR_SAFE_BEGIN(&cpr);
6137 		(void) cv_timedwait(&l2arc_feed_thr_cv, &l2arc_feed_thr_lock,
6138 		    next);
6139 		CALLB_CPR_SAFE_END(&cpr, &l2arc_feed_thr_lock);
6140 		next = ddi_get_lbolt() + hz;
6141 
6142 		/*
6143 		 * Quick check for L2ARC devices.
6144 		 */
6145 		mutex_enter(&l2arc_dev_mtx);
6146 		if (l2arc_ndev == 0) {
6147 			mutex_exit(&l2arc_dev_mtx);
6148 			continue;
6149 		}
6150 		mutex_exit(&l2arc_dev_mtx);
6151 		begin = ddi_get_lbolt();
6152 
6153 		/*
6154 		 * This selects the next l2arc device to write to, and in
6155 		 * doing so the next spa to feed from: dev->l2ad_spa.   This
6156 		 * will return NULL if there are now no l2arc devices or if
6157 		 * they are all faulted.
6158 		 *
6159 		 * If a device is returned, its spa's config lock is also
6160 		 * held to prevent device removal.  l2arc_dev_get_next()
6161 		 * will grab and release l2arc_dev_mtx.
6162 		 */
6163 		if ((dev = l2arc_dev_get_next()) == NULL)
6164 			continue;
6165 
6166 		spa = dev->l2ad_spa;
6167 		ASSERT(spa != NULL);
6168 
6169 		/*
6170 		 * If the pool is read-only then force the feed thread to
6171 		 * sleep a little longer.
6172 		 */
6173 		if (!spa_writeable(spa)) {
6174 			next = ddi_get_lbolt() + 5 * l2arc_feed_secs * hz;
6175 			spa_config_exit(spa, SCL_L2ARC, dev);
6176 			continue;
6177 		}
6178 
6179 		/*
6180 		 * Avoid contributing to memory pressure.
6181 		 */
6182 		if (arc_reclaim_needed()) {
6183 			ARCSTAT_BUMP(arcstat_l2_abort_lowmem);
6184 			spa_config_exit(spa, SCL_L2ARC, dev);
6185 			continue;
6186 		}
6187 
6188 		ARCSTAT_BUMP(arcstat_l2_feeds);
6189 
6190 		size = l2arc_write_size();
6191 
6192 		/*
6193 		 * Evict L2ARC buffers that will be overwritten.
6194 		 */
6195 		l2arc_evict(dev, size, B_FALSE);
6196 
6197 		/*
6198 		 * Write ARC buffers.
6199 		 */
6200 		wrote = l2arc_write_buffers(spa, dev, size, &headroom_boost);
6201 
6202 		/*
6203 		 * Calculate interval between writes.
6204 		 */
6205 		next = l2arc_write_interval(begin, size, wrote);
6206 		spa_config_exit(spa, SCL_L2ARC, dev);
6207 	}
6208 
6209 	l2arc_thread_exit = 0;
6210 	cv_broadcast(&l2arc_feed_thr_cv);
6211 	CALLB_CPR_EXIT(&cpr);		/* drops l2arc_feed_thr_lock */
6212 	thread_exit();
6213 }
6214 
6215 boolean_t
6216 l2arc_vdev_present(vdev_t *vd)
6217 {
6218 	l2arc_dev_t *dev;
6219 
6220 	mutex_enter(&l2arc_dev_mtx);
6221 	for (dev = list_head(l2arc_dev_list); dev != NULL;
6222 	    dev = list_next(l2arc_dev_list, dev)) {
6223 		if (dev->l2ad_vdev == vd)
6224 			break;
6225 	}
6226 	mutex_exit(&l2arc_dev_mtx);
6227 
6228 	return (dev != NULL);
6229 }
6230 
6231 /*
6232  * Add a vdev for use by the L2ARC.  By this point the spa has already
6233  * validated the vdev and opened it.
6234  */
6235 void
6236 l2arc_add_vdev(spa_t *spa, vdev_t *vd)
6237 {
6238 	l2arc_dev_t *adddev;
6239 
6240 	ASSERT(!l2arc_vdev_present(vd));
6241 
6242 	/*
6243 	 * Create a new l2arc device entry.
6244 	 */
6245 	adddev = kmem_zalloc(sizeof (l2arc_dev_t), KM_SLEEP);
6246 	adddev->l2ad_spa = spa;
6247 	adddev->l2ad_vdev = vd;
6248 	adddev->l2ad_start = VDEV_LABEL_START_SIZE;
6249 	adddev->l2ad_end = VDEV_LABEL_START_SIZE + vdev_get_min_asize(vd);
6250 	adddev->l2ad_hand = adddev->l2ad_start;
6251 	adddev->l2ad_evict = adddev->l2ad_start;
6252 	adddev->l2ad_first = B_TRUE;
6253 	adddev->l2ad_writing = B_FALSE;
6254 
6255 	mutex_init(&adddev->l2ad_mtx, NULL, MUTEX_DEFAULT, NULL);
6256 	/*
6257 	 * This is a list of all ARC buffers that are still valid on the
6258 	 * device.
6259 	 */
6260 	list_create(&adddev->l2ad_buflist, sizeof (arc_buf_hdr_t),
6261 	    offsetof(arc_buf_hdr_t, b_l2hdr.b_l2node));
6262 
6263 	vdev_space_update(vd, 0, 0, adddev->l2ad_end - adddev->l2ad_hand);
6264 
6265 	/*
6266 	 * Add device to global list
6267 	 */
6268 	mutex_enter(&l2arc_dev_mtx);
6269 	list_insert_head(l2arc_dev_list, adddev);
6270 	atomic_inc_64(&l2arc_ndev);
6271 	mutex_exit(&l2arc_dev_mtx);
6272 }
6273 
6274 /*
6275  * Remove a vdev from the L2ARC.
6276  */
6277 void
6278 l2arc_remove_vdev(vdev_t *vd)
6279 {
6280 	l2arc_dev_t *dev, *nextdev, *remdev = NULL;
6281 
6282 	/*
6283 	 * Find the device by vdev
6284 	 */
6285 	mutex_enter(&l2arc_dev_mtx);
6286 	for (dev = list_head(l2arc_dev_list); dev; dev = nextdev) {
6287 		nextdev = list_next(l2arc_dev_list, dev);
6288 		if (vd == dev->l2ad_vdev) {
6289 			remdev = dev;
6290 			break;
6291 		}
6292 	}
6293 	ASSERT(remdev != NULL);
6294 
6295 	/*
6296 	 * Remove device from global list
6297 	 */
6298 	list_remove(l2arc_dev_list, remdev);
6299 	l2arc_dev_last = NULL;		/* may have been invalidated */
6300 	atomic_dec_64(&l2arc_ndev);
6301 	mutex_exit(&l2arc_dev_mtx);
6302 
6303 	/*
6304 	 * Clear all buflists and ARC references.  L2ARC device flush.
6305 	 */
6306 	l2arc_evict(remdev, 0, B_TRUE);
6307 	list_destroy(&remdev->l2ad_buflist);
6308 	mutex_destroy(&remdev->l2ad_mtx);
6309 	kmem_free(remdev, sizeof (l2arc_dev_t));
6310 }
6311 
6312 void
6313 l2arc_init(void)
6314 {
6315 	l2arc_thread_exit = 0;
6316 	l2arc_ndev = 0;
6317 	l2arc_writes_sent = 0;
6318 	l2arc_writes_done = 0;
6319 
6320 	mutex_init(&l2arc_feed_thr_lock, NULL, MUTEX_DEFAULT, NULL);
6321 	cv_init(&l2arc_feed_thr_cv, NULL, CV_DEFAULT, NULL);
6322 	mutex_init(&l2arc_dev_mtx, NULL, MUTEX_DEFAULT, NULL);
6323 	mutex_init(&l2arc_free_on_write_mtx, NULL, MUTEX_DEFAULT, NULL);
6324 
6325 	l2arc_dev_list = &L2ARC_dev_list;
6326 	l2arc_free_on_write = &L2ARC_free_on_write;
6327 	list_create(l2arc_dev_list, sizeof (l2arc_dev_t),
6328 	    offsetof(l2arc_dev_t, l2ad_node));
6329 	list_create(l2arc_free_on_write, sizeof (l2arc_data_free_t),
6330 	    offsetof(l2arc_data_free_t, l2df_list_node));
6331 }
6332 
6333 void
6334 l2arc_fini(void)
6335 {
6336 	/*
6337 	 * This is called from dmu_fini(), which is called from spa_fini();
6338 	 * Because of this, we can assume that all l2arc devices have
6339 	 * already been removed when the pools themselves were removed.
6340 	 */
6341 
6342 	l2arc_do_free_on_write();
6343 
6344 	mutex_destroy(&l2arc_feed_thr_lock);
6345 	cv_destroy(&l2arc_feed_thr_cv);
6346 	mutex_destroy(&l2arc_dev_mtx);
6347 	mutex_destroy(&l2arc_free_on_write_mtx);
6348 
6349 	list_destroy(l2arc_dev_list);
6350 	list_destroy(l2arc_free_on_write);
6351 }
6352 
6353 void
6354 l2arc_start(void)
6355 {
6356 	if (!(spa_mode_global & FWRITE))
6357 		return;
6358 
6359 	(void) thread_create(NULL, 0, l2arc_feed_thread, NULL, 0, &p0,
6360 	    TS_RUN, minclsyspri);
6361 }
6362 
6363 void
6364 l2arc_stop(void)
6365 {
6366 	if (!(spa_mode_global & FWRITE))
6367 		return;
6368 
6369 	mutex_enter(&l2arc_feed_thr_lock);
6370 	cv_signal(&l2arc_feed_thr_cv);	/* kick thread out of startup */
6371 	l2arc_thread_exit = 1;
6372 	while (l2arc_thread_exit != 0)
6373 		cv_wait(&l2arc_feed_thr_cv, &l2arc_feed_thr_lock);
6374 	mutex_exit(&l2arc_feed_thr_lock);
6375 }
6376