1096e95c0SMatthew Dillon /*
28e7c4729SMatthew Dillon * (MPSAFE)
38e7c4729SMatthew Dillon *
49de48eadSMatthew Dillon * Copyright (c) 2010,2019 The DragonFly Project. All rights reserved.
5096e95c0SMatthew Dillon *
6096e95c0SMatthew Dillon * This code is derived from software contributed to The DragonFly Project
7096e95c0SMatthew Dillon * by Matthew Dillon <dillon@backplane.com>
8096e95c0SMatthew Dillon *
9096e95c0SMatthew Dillon * Redistribution and use in source and binary forms, with or without
10096e95c0SMatthew Dillon * modification, are permitted provided that the following conditions
11096e95c0SMatthew Dillon * are met:
12096e95c0SMatthew Dillon *
13096e95c0SMatthew Dillon * 1. Redistributions of source code must retain the above copyright
14096e95c0SMatthew Dillon * notice, this list of conditions and the following disclaimer.
15096e95c0SMatthew Dillon * 2. Redistributions in binary form must reproduce the above copyright
16096e95c0SMatthew Dillon * notice, this list of conditions and the following disclaimer in
17096e95c0SMatthew Dillon * the documentation and/or other materials provided with the
18096e95c0SMatthew Dillon * distribution.
19096e95c0SMatthew Dillon * 3. Neither the name of The DragonFly Project nor the names of its
20096e95c0SMatthew Dillon * contributors may be used to endorse or promote products derived
21096e95c0SMatthew Dillon * from this software without specific, prior written permission.
22096e95c0SMatthew Dillon *
23096e95c0SMatthew Dillon * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
24096e95c0SMatthew Dillon * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
25096e95c0SMatthew Dillon * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
26096e95c0SMatthew Dillon * FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE
27096e95c0SMatthew Dillon * COPYRIGHT HOLDERS OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
28096e95c0SMatthew Dillon * INCIDENTAL, SPECIAL, EXEMPLARY OR CONSEQUENTIAL DAMAGES (INCLUDING,
29096e95c0SMatthew Dillon * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
30096e95c0SMatthew Dillon * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
31096e95c0SMatthew Dillon * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
32096e95c0SMatthew Dillon * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT
33096e95c0SMatthew Dillon * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
34096e95c0SMatthew Dillon * SUCH DAMAGE.
35096e95c0SMatthew Dillon */
36096e95c0SMatthew Dillon
37096e95c0SMatthew Dillon /*
38096e95c0SMatthew Dillon * Implement the swapcache daemon. When enabled swap is assumed to be
39096e95c0SMatthew Dillon * configured on a fast storage device such as a SSD. Swap is assigned
40096e95c0SMatthew Dillon * to clean vnode-backed pages in the inactive queue, clustered by object
41096e95c0SMatthew Dillon * if possible, and written out. The swap assignment sticks around even
42096e95c0SMatthew Dillon * after the underlying pages have been recycled.
43096e95c0SMatthew Dillon *
44096e95c0SMatthew Dillon * The daemon manages write bandwidth based on sysctl settings to control
45096e95c0SMatthew Dillon * wear on the SSD.
46096e95c0SMatthew Dillon *
47096e95c0SMatthew Dillon * The vnode strategy code will check for the swap assignments and divert
483ffc7051SMatthew Dillon * reads to the swap device when the data is present in the swapcache.
49096e95c0SMatthew Dillon *
50096e95c0SMatthew Dillon * This operates on both regular files and the block device vnodes used by
51096e95c0SMatthew Dillon * filesystems to manage meta-data.
52096e95c0SMatthew Dillon */
53096e95c0SMatthew Dillon
54096e95c0SMatthew Dillon #include <sys/param.h>
55096e95c0SMatthew Dillon #include <sys/systm.h>
56096e95c0SMatthew Dillon #include <sys/kernel.h>
57096e95c0SMatthew Dillon #include <sys/proc.h>
58096e95c0SMatthew Dillon #include <sys/kthread.h>
59096e95c0SMatthew Dillon #include <sys/resourcevar.h>
60096e95c0SMatthew Dillon #include <sys/signalvar.h>
61096e95c0SMatthew Dillon #include <sys/vnode.h>
62096e95c0SMatthew Dillon #include <sys/vmmeter.h>
63096e95c0SMatthew Dillon #include <sys/sysctl.h>
64497524bfSMatthew Dillon #include <sys/eventhandler.h>
65096e95c0SMatthew Dillon
66096e95c0SMatthew Dillon #include <vm/vm.h>
67096e95c0SMatthew Dillon #include <vm/vm_param.h>
68096e95c0SMatthew Dillon #include <sys/lock.h>
69096e95c0SMatthew Dillon #include <vm/vm_object.h>
70096e95c0SMatthew Dillon #include <vm/vm_page.h>
71096e95c0SMatthew Dillon #include <vm/vm_map.h>
72096e95c0SMatthew Dillon #include <vm/vm_pageout.h>
73096e95c0SMatthew Dillon #include <vm/vm_pager.h>
74096e95c0SMatthew Dillon #include <vm/swap_pager.h>
75096e95c0SMatthew Dillon #include <vm/vm_extern.h>
76096e95c0SMatthew Dillon
77b12defdcSMatthew Dillon #include <sys/spinlock2.h>
78096e95c0SMatthew Dillon #include <vm/vm_page2.h>
79096e95c0SMatthew Dillon
809de48eadSMatthew Dillon struct swmarker {
819de48eadSMatthew Dillon struct vm_object dummy_obj;
829de48eadSMatthew Dillon struct vm_object *save_obj;
839de48eadSMatthew Dillon vm_ooffset_t save_off;
849de48eadSMatthew Dillon };
859de48eadSMatthew Dillon
869de48eadSMatthew Dillon typedef struct swmarker swmarker_t;
879de48eadSMatthew Dillon
88096e95c0SMatthew Dillon /* the kernel process "vm_pageout"*/
89aabd5ce8SMatthew Dillon static int vm_swapcached_flush (vm_page_t m, int isblkdev);
903ffc7051SMatthew Dillon static int vm_swapcache_test(vm_page_t m);
9164949baaSMatthew Dillon static int vm_swapcache_writing_heuristic(void);
9264949baaSMatthew Dillon static int vm_swapcache_writing(vm_page_t marker, int count, int scount);
939de48eadSMatthew Dillon static void vm_swapcache_cleaning(swmarker_t *marker,
94fde6be6aSMatthew Dillon struct vm_object_hash **swindexp);
959de48eadSMatthew Dillon static void vm_swapcache_movemarker(swmarker_t *marker,
96fde6be6aSMatthew Dillon struct vm_object_hash *swindex, vm_object_t object);
97096e95c0SMatthew Dillon struct thread *swapcached_thread;
98096e95c0SMatthew Dillon
99096e95c0SMatthew Dillon SYSCTL_NODE(_vm, OID_AUTO, swapcache, CTLFLAG_RW, NULL, NULL);
100096e95c0SMatthew Dillon
101c504e38eSMatthew Dillon int vm_swapcache_read_enable;
102b396bb03SMatthew Dillon static long vm_swapcache_wtrigger;
103096e95c0SMatthew Dillon static int vm_swapcache_sleep;
10464949baaSMatthew Dillon static int vm_swapcache_maxscan = PQ_L2_SIZE * 8;
10564949baaSMatthew Dillon static int vm_swapcache_maxlaunder = PQ_L2_SIZE * 4;
106096e95c0SMatthew Dillon static int vm_swapcache_data_enable = 0;
107096e95c0SMatthew Dillon static int vm_swapcache_meta_enable = 0;
108e9b56058SMatthew Dillon static int vm_swapcache_maxswappct = 75;
109e527fb6bSMatthew Dillon static int vm_swapcache_hysteresis;
11064949baaSMatthew Dillon static int vm_swapcache_min_hysteresis;
111d4d9221dSMatthew Dillon int vm_swapcache_use_chflags = 0; /* require chflags cache */
1123ffc7051SMatthew Dillon static int64_t vm_swapcache_minburst = 10000000LL; /* 10MB */
1133ffc7051SMatthew Dillon static int64_t vm_swapcache_curburst = 4000000000LL; /* 4G after boot */
1143ffc7051SMatthew Dillon static int64_t vm_swapcache_maxburst = 2000000000LL; /* 2G nominal max */
1153ffc7051SMatthew Dillon static int64_t vm_swapcache_accrate = 100000LL; /* 100K/s */
116096e95c0SMatthew Dillon static int64_t vm_swapcache_write_count;
1173ffc7051SMatthew Dillon static int64_t vm_swapcache_maxfilesize;
118f5f6d247SMatthew Dillon static int64_t vm_swapcache_cleanperobj = 16*1024*1024;
119096e95c0SMatthew Dillon
120096e95c0SMatthew Dillon SYSCTL_INT(_vm_swapcache, OID_AUTO, maxlaunder,
121096e95c0SMatthew Dillon CTLFLAG_RW, &vm_swapcache_maxlaunder, 0, "");
1220bf81261SMatthew Dillon SYSCTL_INT(_vm_swapcache, OID_AUTO, maxscan,
1230bf81261SMatthew Dillon CTLFLAG_RW, &vm_swapcache_maxscan, 0, "");
124c504e38eSMatthew Dillon
125096e95c0SMatthew Dillon SYSCTL_INT(_vm_swapcache, OID_AUTO, data_enable,
126096e95c0SMatthew Dillon CTLFLAG_RW, &vm_swapcache_data_enable, 0, "");
127096e95c0SMatthew Dillon SYSCTL_INT(_vm_swapcache, OID_AUTO, meta_enable,
128096e95c0SMatthew Dillon CTLFLAG_RW, &vm_swapcache_meta_enable, 0, "");
129c504e38eSMatthew Dillon SYSCTL_INT(_vm_swapcache, OID_AUTO, read_enable,
130c504e38eSMatthew Dillon CTLFLAG_RW, &vm_swapcache_read_enable, 0, "");
131e9b56058SMatthew Dillon SYSCTL_INT(_vm_swapcache, OID_AUTO, maxswappct,
132e9b56058SMatthew Dillon CTLFLAG_RW, &vm_swapcache_maxswappct, 0, "");
133e527fb6bSMatthew Dillon SYSCTL_INT(_vm_swapcache, OID_AUTO, hysteresis,
13464949baaSMatthew Dillon CTLFLAG_RD, &vm_swapcache_hysteresis, 0, "");
13564949baaSMatthew Dillon SYSCTL_INT(_vm_swapcache, OID_AUTO, min_hysteresis,
13664949baaSMatthew Dillon CTLFLAG_RW, &vm_swapcache_min_hysteresis, 0, "");
137e9b56058SMatthew Dillon SYSCTL_INT(_vm_swapcache, OID_AUTO, use_chflags,
138e9b56058SMatthew Dillon CTLFLAG_RW, &vm_swapcache_use_chflags, 0, "");
139c504e38eSMatthew Dillon
1403ffc7051SMatthew Dillon SYSCTL_QUAD(_vm_swapcache, OID_AUTO, minburst,
1413ffc7051SMatthew Dillon CTLFLAG_RW, &vm_swapcache_minburst, 0, "");
142c504e38eSMatthew Dillon SYSCTL_QUAD(_vm_swapcache, OID_AUTO, curburst,
143c504e38eSMatthew Dillon CTLFLAG_RW, &vm_swapcache_curburst, 0, "");
144c504e38eSMatthew Dillon SYSCTL_QUAD(_vm_swapcache, OID_AUTO, maxburst,
145c504e38eSMatthew Dillon CTLFLAG_RW, &vm_swapcache_maxburst, 0, "");
1463ffc7051SMatthew Dillon SYSCTL_QUAD(_vm_swapcache, OID_AUTO, maxfilesize,
1473ffc7051SMatthew Dillon CTLFLAG_RW, &vm_swapcache_maxfilesize, 0, "");
148c504e38eSMatthew Dillon SYSCTL_QUAD(_vm_swapcache, OID_AUTO, accrate,
149c504e38eSMatthew Dillon CTLFLAG_RW, &vm_swapcache_accrate, 0, "");
150096e95c0SMatthew Dillon SYSCTL_QUAD(_vm_swapcache, OID_AUTO, write_count,
151096e95c0SMatthew Dillon CTLFLAG_RW, &vm_swapcache_write_count, 0, "");
152f5f6d247SMatthew Dillon SYSCTL_QUAD(_vm_swapcache, OID_AUTO, cleanperobj,
153f5f6d247SMatthew Dillon CTLFLAG_RW, &vm_swapcache_cleanperobj, 0, "");
154096e95c0SMatthew Dillon
155e9b56058SMatthew Dillon #define SWAPMAX(adj) \
156e9b56058SMatthew Dillon ((int64_t)vm_swap_max * (vm_swapcache_maxswappct + (adj)) / 100)
157e9b56058SMatthew Dillon
158096e95c0SMatthew Dillon /*
159497524bfSMatthew Dillon * When shutting down the machine we want to stop swapcache operation
160497524bfSMatthew Dillon * immediately so swap is not accessed after devices have been shuttered.
161497524bfSMatthew Dillon */
162497524bfSMatthew Dillon static void
shutdown_swapcache(void * arg __unused)163497524bfSMatthew Dillon shutdown_swapcache(void *arg __unused)
164497524bfSMatthew Dillon {
165497524bfSMatthew Dillon vm_swapcache_read_enable = 0;
166497524bfSMatthew Dillon vm_swapcache_data_enable = 0;
167497524bfSMatthew Dillon vm_swapcache_meta_enable = 0;
168497524bfSMatthew Dillon wakeup(&vm_swapcache_sleep); /* shortcut 5-second wait */
169497524bfSMatthew Dillon }
170497524bfSMatthew Dillon
171497524bfSMatthew Dillon /*
172096e95c0SMatthew Dillon * vm_swapcached is the high level pageout daemon.
1738e7c4729SMatthew Dillon *
1748e7c4729SMatthew Dillon * No requirements.
175096e95c0SMatthew Dillon */
176096e95c0SMatthew Dillon static void
vm_swapcached_thread(void)177cd8ab232SMatthew Dillon vm_swapcached_thread(void)
178096e95c0SMatthew Dillon {
17900a3fdcaSMatthew Dillon enum { SWAPC_WRITING, SWAPC_CLEANING } state = SWAPC_WRITING;
1803ffc7051SMatthew Dillon enum { SWAPB_BURSTING, SWAPB_RECOVERING } burst = SWAPB_BURSTING;
18151c99c61SMatthew Dillon static struct vm_page page_marker[PQ_L2_SIZE];
1829de48eadSMatthew Dillon static swmarker_t swmarker;
183fde6be6aSMatthew Dillon static struct vm_object_hash *swindex;
184027193ebSMatthew Dillon int q;
185096e95c0SMatthew Dillon
186096e95c0SMatthew Dillon /*
187096e95c0SMatthew Dillon * Thread setup
188096e95c0SMatthew Dillon */
189096e95c0SMatthew Dillon curthread->td_flags |= TDF_SYSTHREAD;
190497524bfSMatthew Dillon EVENTHANDLER_REGISTER(shutdown_pre_sync, shutdown_kproc,
191497524bfSMatthew Dillon swapcached_thread, SHUTDOWN_PRI_FIRST);
192497524bfSMatthew Dillon EVENTHANDLER_REGISTER(shutdown_pre_sync, shutdown_swapcache,
193497524bfSMatthew Dillon NULL, SHUTDOWN_PRI_SECOND);
194096e95c0SMatthew Dillon
195096e95c0SMatthew Dillon /*
19600a3fdcaSMatthew Dillon * Initialize our marker for the inactive scan (SWAPC_WRITING)
197096e95c0SMatthew Dillon */
19800a3fdcaSMatthew Dillon bzero(&page_marker, sizeof(page_marker));
19951c99c61SMatthew Dillon for (q = 0; q < PQ_L2_SIZE; ++q) {
200bc0aa189SMatthew Dillon page_marker[q].flags = PG_FICTITIOUS | PG_MARKER;
201bc0aa189SMatthew Dillon page_marker[q].busy_count = PBUSY_LOCKED;
202027193ebSMatthew Dillon page_marker[q].queue = PQ_INACTIVE + q;
203027193ebSMatthew Dillon page_marker[q].pc = q;
204027193ebSMatthew Dillon page_marker[q].wire_count = 1;
205027193ebSMatthew Dillon vm_page_queues_spin_lock(PQ_INACTIVE + q);
206027193ebSMatthew Dillon TAILQ_INSERT_HEAD(
207027193ebSMatthew Dillon &vm_page_queues[PQ_INACTIVE + q].pl,
208027193ebSMatthew Dillon &page_marker[q], pageq);
209027193ebSMatthew Dillon vm_page_queues_spin_unlock(PQ_INACTIVE + q);
210027193ebSMatthew Dillon }
211b12defdcSMatthew Dillon
21264949baaSMatthew Dillon vm_swapcache_min_hysteresis = 1024;
21364949baaSMatthew Dillon vm_swapcache_hysteresis = vm_swapcache_min_hysteresis;
214b396bb03SMatthew Dillon vm_swapcache_wtrigger = -vm_swapcache_hysteresis;
215096e95c0SMatthew Dillon
21600a3fdcaSMatthew Dillon /*
21700a3fdcaSMatthew Dillon * Initialize our marker for the vm_object scan (SWAPC_CLEANING)
21800a3fdcaSMatthew Dillon */
2197b00fbb4SMatthew Dillon bzero(&swmarker, sizeof(swmarker));
2209de48eadSMatthew Dillon swmarker.dummy_obj.type = OBJT_MARKER;
221fde6be6aSMatthew Dillon swindex = &vm_object_hash[0];
222fde6be6aSMatthew Dillon lwkt_gettoken(&swindex->token);
2235b329e62SMatthew Dillon TAILQ_INSERT_HEAD(&swindex->list, &swmarker.dummy_obj, object_entry);
224fde6be6aSMatthew Dillon lwkt_reltoken(&swindex->token);
225096e95c0SMatthew Dillon
226096e95c0SMatthew Dillon for (;;) {
22764949baaSMatthew Dillon int reached_end;
22864949baaSMatthew Dillon int scount;
22964949baaSMatthew Dillon int count;
23064949baaSMatthew Dillon
231096e95c0SMatthew Dillon /*
232497524bfSMatthew Dillon * Handle shutdown
233497524bfSMatthew Dillon */
234497524bfSMatthew Dillon kproc_suspend_loop();
235497524bfSMatthew Dillon
236497524bfSMatthew Dillon /*
2373da46bd7SMatthew Dillon * Check every 5 seconds when not enabled or if no swap
2383da46bd7SMatthew Dillon * is present.
239096e95c0SMatthew Dillon */
2403da46bd7SMatthew Dillon if ((vm_swapcache_data_enable == 0 &&
241c8ddd5d8SMatthew Dillon vm_swapcache_meta_enable == 0 &&
242c8ddd5d8SMatthew Dillon vm_swap_cache_use <= SWAPMAX(0)) ||
2433da46bd7SMatthew Dillon vm_swap_max == 0) {
244096e95c0SMatthew Dillon tsleep(&vm_swapcache_sleep, 0, "csleep", hz * 5);
245096e95c0SMatthew Dillon continue;
246096e95c0SMatthew Dillon }
247c504e38eSMatthew Dillon
248c504e38eSMatthew Dillon /*
2493da46bd7SMatthew Dillon * Polling rate when enabled is approximately 10 hz.
250c504e38eSMatthew Dillon */
251c504e38eSMatthew Dillon tsleep(&vm_swapcache_sleep, 0, "csleep", hz / 10);
25200a3fdcaSMatthew Dillon
25300a3fdcaSMatthew Dillon /*
25400a3fdcaSMatthew Dillon * State hysteresis. Generate write activity up to 75% of
25500a3fdcaSMatthew Dillon * swap, then clean out swap assignments down to 70%, then
25600a3fdcaSMatthew Dillon * repeat.
25700a3fdcaSMatthew Dillon */
25800a3fdcaSMatthew Dillon if (state == SWAPC_WRITING) {
259e9b56058SMatthew Dillon if (vm_swap_cache_use > SWAPMAX(0))
26000a3fdcaSMatthew Dillon state = SWAPC_CLEANING;
26100a3fdcaSMatthew Dillon } else {
26208fb7a9dSMatthew Dillon if (vm_swap_cache_use < SWAPMAX(-10))
26300a3fdcaSMatthew Dillon state = SWAPC_WRITING;
26400a3fdcaSMatthew Dillon }
26500a3fdcaSMatthew Dillon
26600a3fdcaSMatthew Dillon /*
26700a3fdcaSMatthew Dillon * We are allowed to continue accumulating burst value
2683ffc7051SMatthew Dillon * in either state. Allow the user to set curburst > maxburst
2693ffc7051SMatthew Dillon * for the initial load-in.
27000a3fdcaSMatthew Dillon */
2713ffc7051SMatthew Dillon if (vm_swapcache_curburst < vm_swapcache_maxburst) {
272c504e38eSMatthew Dillon vm_swapcache_curburst += vm_swapcache_accrate / 10;
273c504e38eSMatthew Dillon if (vm_swapcache_curburst > vm_swapcache_maxburst)
274c504e38eSMatthew Dillon vm_swapcache_curburst = vm_swapcache_maxburst;
2753ffc7051SMatthew Dillon }
276c504e38eSMatthew Dillon
277c504e38eSMatthew Dillon /*
27800a3fdcaSMatthew Dillon * We don't want to nickle-and-dime the scan as that will
27900a3fdcaSMatthew Dillon * create unnecessary fragmentation. The minimum burst
28000a3fdcaSMatthew Dillon * is one-seconds worth of accumulation.
281c504e38eSMatthew Dillon */
28264949baaSMatthew Dillon if (state != SWAPC_WRITING) {
2837b00fbb4SMatthew Dillon vm_swapcache_cleaning(&swmarker, &swindex);
28464949baaSMatthew Dillon continue;
28564949baaSMatthew Dillon }
28664949baaSMatthew Dillon if (vm_swapcache_curburst < vm_swapcache_accrate)
28764949baaSMatthew Dillon continue;
28864949baaSMatthew Dillon
28964949baaSMatthew Dillon reached_end = 0;
29064949baaSMatthew Dillon count = vm_swapcache_maxlaunder / PQ_L2_SIZE + 2;
29164949baaSMatthew Dillon scount = vm_swapcache_maxscan / PQ_L2_SIZE + 2;
29264949baaSMatthew Dillon
2933ffc7051SMatthew Dillon if (burst == SWAPB_BURSTING) {
29464949baaSMatthew Dillon if (vm_swapcache_writing_heuristic()) {
29551c99c61SMatthew Dillon for (q = 0; q < PQ_L2_SIZE; ++q) {
29664949baaSMatthew Dillon reached_end +=
297027193ebSMatthew Dillon vm_swapcache_writing(
29864949baaSMatthew Dillon &page_marker[q],
29964949baaSMatthew Dillon count,
30064949baaSMatthew Dillon scount);
30164949baaSMatthew Dillon }
302027193ebSMatthew Dillon }
3033ffc7051SMatthew Dillon if (vm_swapcache_curburst <= 0)
3043ffc7051SMatthew Dillon burst = SWAPB_RECOVERING;
30564949baaSMatthew Dillon } else if (vm_swapcache_curburst > vm_swapcache_minburst) {
30664949baaSMatthew Dillon if (vm_swapcache_writing_heuristic()) {
30751c99c61SMatthew Dillon for (q = 0; q < PQ_L2_SIZE; ++q) {
30864949baaSMatthew Dillon reached_end +=
309027193ebSMatthew Dillon vm_swapcache_writing(
31064949baaSMatthew Dillon &page_marker[q],
31164949baaSMatthew Dillon count,
31264949baaSMatthew Dillon scount);
31364949baaSMatthew Dillon }
314027193ebSMatthew Dillon }
3153ffc7051SMatthew Dillon burst = SWAPB_BURSTING;
3163ffc7051SMatthew Dillon }
31764949baaSMatthew Dillon if (reached_end == PQ_L2_SIZE) {
318b396bb03SMatthew Dillon vm_swapcache_wtrigger = -vm_swapcache_hysteresis;
31900a3fdcaSMatthew Dillon }
32000a3fdcaSMatthew Dillon }
321eccc8ca1SMatthew Dillon
322eccc8ca1SMatthew Dillon /*
323eccc8ca1SMatthew Dillon * Cleanup (NOT REACHED)
324eccc8ca1SMatthew Dillon */
32551c99c61SMatthew Dillon for (q = 0; q < PQ_L2_SIZE; ++q) {
326027193ebSMatthew Dillon vm_page_queues_spin_lock(PQ_INACTIVE + q);
327027193ebSMatthew Dillon TAILQ_REMOVE(
328027193ebSMatthew Dillon &vm_page_queues[PQ_INACTIVE + q].pl,
329027193ebSMatthew Dillon &page_marker[q], pageq);
330027193ebSMatthew Dillon vm_page_queues_spin_unlock(PQ_INACTIVE + q);
331027193ebSMatthew Dillon }
332eccc8ca1SMatthew Dillon
333fde6be6aSMatthew Dillon lwkt_gettoken(&swindex->token);
3345b329e62SMatthew Dillon TAILQ_REMOVE(&swindex->list, &swmarker.dummy_obj, object_entry);
335fde6be6aSMatthew Dillon lwkt_reltoken(&swindex->token);
33600a3fdcaSMatthew Dillon }
337096e95c0SMatthew Dillon
338cd8ab232SMatthew Dillon static struct kproc_desc swpc_kp = {
339cd8ab232SMatthew Dillon "swapcached",
340cd8ab232SMatthew Dillon vm_swapcached_thread,
341cd8ab232SMatthew Dillon &swapcached_thread
342cd8ab232SMatthew Dillon };
343f3f3eadbSSascha Wildner SYSINIT(swapcached, SI_SUB_KTHREAD_PAGE, SI_ORDER_SECOND, kproc_start, &swpc_kp);
344cd8ab232SMatthew Dillon
345096e95c0SMatthew Dillon /*
346fdc53cc7SMatthew Dillon * Deal with an overflow of the heuristic counter or if the user
347fdc53cc7SMatthew Dillon * manually changes the hysteresis.
348fdc53cc7SMatthew Dillon *
349e527fb6bSMatthew Dillon * Try to avoid small incremental pageouts by waiting for enough
350e527fb6bSMatthew Dillon * pages to buildup in the inactive queue to hopefully get a good
351e527fb6bSMatthew Dillon * burst in. This heuristic is bumped by the VM system and reset
352e527fb6bSMatthew Dillon * when our scan hits the end of the queue.
35364949baaSMatthew Dillon *
35464949baaSMatthew Dillon * Return TRUE if we need to take a writing pass.
355e527fb6bSMatthew Dillon */
35664949baaSMatthew Dillon static int
vm_swapcache_writing_heuristic(void)35764949baaSMatthew Dillon vm_swapcache_writing_heuristic(void)
35864949baaSMatthew Dillon {
35964949baaSMatthew Dillon int hyst;
360b396bb03SMatthew Dillon int q;
361b396bb03SMatthew Dillon long adds;
36264949baaSMatthew Dillon
36364949baaSMatthew Dillon hyst = vmstats.v_inactive_count / 4;
36464949baaSMatthew Dillon if (hyst < vm_swapcache_min_hysteresis)
36564949baaSMatthew Dillon hyst = vm_swapcache_min_hysteresis;
36664949baaSMatthew Dillon cpu_ccfence();
36764949baaSMatthew Dillon vm_swapcache_hysteresis = hyst;
36864949baaSMatthew Dillon
369b396bb03SMatthew Dillon adds = 0;
370b396bb03SMatthew Dillon for (q = PQ_INACTIVE; q < PQ_INACTIVE + PQ_L2_SIZE; ++q) {
371b396bb03SMatthew Dillon adds += atomic_swap_long(&vm_page_queues[q].adds, 0);
372b396bb03SMatthew Dillon }
373b396bb03SMatthew Dillon vm_swapcache_wtrigger += adds;
374b396bb03SMatthew Dillon if (vm_swapcache_wtrigger < -hyst)
375b396bb03SMatthew Dillon vm_swapcache_wtrigger = -hyst;
376b396bb03SMatthew Dillon return (vm_swapcache_wtrigger >= 0);
37764949baaSMatthew Dillon }
37864949baaSMatthew Dillon
37964949baaSMatthew Dillon /*
38064949baaSMatthew Dillon * Take a writing pass on one of the inactive queues, return non-zero if
38164949baaSMatthew Dillon * we hit the end of the queue.
38264949baaSMatthew Dillon */
38364949baaSMatthew Dillon static int
vm_swapcache_writing(vm_page_t marker,int count,int scount)38464949baaSMatthew Dillon vm_swapcache_writing(vm_page_t marker, int count, int scount)
38564949baaSMatthew Dillon {
38664949baaSMatthew Dillon vm_object_t object;
38764949baaSMatthew Dillon struct vnode *vp;
38864949baaSMatthew Dillon vm_page_t m;
38964949baaSMatthew Dillon int isblkdev;
390e527fb6bSMatthew Dillon
391e527fb6bSMatthew Dillon /*
392096e95c0SMatthew Dillon * Scan the inactive queue from our marker to locate
393096e95c0SMatthew Dillon * suitable pages to push to the swap cache.
394096e95c0SMatthew Dillon *
395096e95c0SMatthew Dillon * We are looking for clean vnode-backed pages.
396096e95c0SMatthew Dillon */
397027193ebSMatthew Dillon vm_page_queues_spin_lock(marker->queue);
3980bf81261SMatthew Dillon while ((m = TAILQ_NEXT(marker, pageq)) != NULL &&
3990bf81261SMatthew Dillon count > 0 && scount-- > 0) {
400027193ebSMatthew Dillon KKASSERT(m->queue == marker->queue);
401b12defdcSMatthew Dillon
4021de864f0SMatthew Dillon /*
4031de864f0SMatthew Dillon * Stop using swap if paniced, dumping, or dumped.
4041de864f0SMatthew Dillon * Don't try to write if our curburst has been exhausted.
4051de864f0SMatthew Dillon */
4061de864f0SMatthew Dillon if (panicstr || dumping)
4071de864f0SMatthew Dillon break;
408b12defdcSMatthew Dillon if (vm_swapcache_curburst < 0)
409b12defdcSMatthew Dillon break;
4101de864f0SMatthew Dillon
4111de864f0SMatthew Dillon /*
4121de864f0SMatthew Dillon * Move marker
4131de864f0SMatthew Dillon */
414027193ebSMatthew Dillon TAILQ_REMOVE(
415027193ebSMatthew Dillon &vm_page_queues[marker->queue].pl, marker, pageq);
416027193ebSMatthew Dillon TAILQ_INSERT_AFTER(
417027193ebSMatthew Dillon &vm_page_queues[marker->queue].pl, m, marker, pageq);
418f5f6d247SMatthew Dillon
419f5f6d247SMatthew Dillon /*
420f5f6d247SMatthew Dillon * Ignore markers and ignore pages that already have a swap
421f5f6d247SMatthew Dillon * assignment.
422f5f6d247SMatthew Dillon */
4230bf81261SMatthew Dillon if (m->flags & (PG_MARKER | PG_SWAPPED))
424096e95c0SMatthew Dillon continue;
425b12defdcSMatthew Dillon if (vm_page_busy_try(m, TRUE))
426096e95c0SMatthew Dillon continue;
427027193ebSMatthew Dillon vm_page_queues_spin_unlock(marker->queue);
428b12defdcSMatthew Dillon
429b12defdcSMatthew Dillon if ((object = m->object) == NULL) {
430b12defdcSMatthew Dillon vm_page_wakeup(m);
431027193ebSMatthew Dillon vm_page_queues_spin_lock(marker->queue);
432b12defdcSMatthew Dillon continue;
433b12defdcSMatthew Dillon }
434b12defdcSMatthew Dillon vm_object_hold(object);
435b12defdcSMatthew Dillon if (m->object != object) {
436b12defdcSMatthew Dillon vm_object_drop(object);
437b12defdcSMatthew Dillon vm_page_wakeup(m);
438027193ebSMatthew Dillon vm_page_queues_spin_lock(marker->queue);
439b12defdcSMatthew Dillon continue;
440b12defdcSMatthew Dillon }
441b12defdcSMatthew Dillon if (vm_swapcache_test(m)) {
442b12defdcSMatthew Dillon vm_object_drop(object);
443b12defdcSMatthew Dillon vm_page_wakeup(m);
444027193ebSMatthew Dillon vm_page_queues_spin_lock(marker->queue);
445b12defdcSMatthew Dillon continue;
446b12defdcSMatthew Dillon }
447b12defdcSMatthew Dillon
448c504e38eSMatthew Dillon vp = object->handle;
449b12defdcSMatthew Dillon if (vp == NULL) {
450b12defdcSMatthew Dillon vm_object_drop(object);
451b12defdcSMatthew Dillon vm_page_wakeup(m);
452027193ebSMatthew Dillon vm_page_queues_spin_lock(marker->queue);
453c504e38eSMatthew Dillon continue;
454b12defdcSMatthew Dillon }
455d3070b8dSMatthew Dillon
456c504e38eSMatthew Dillon switch(vp->v_type) {
457c504e38eSMatthew Dillon case VREG:
458e9b56058SMatthew Dillon /*
459bfa86281SMatthew Dillon * PG_NOTMETA generically means 'don't swapcache this',
460bfa86281SMatthew Dillon * and HAMMER will set this for regular data buffers
461bfa86281SMatthew Dillon * (and leave it unset for meta-data buffers) as
462bfa86281SMatthew Dillon * appropriate when double buffering is enabled.
463bfa86281SMatthew Dillon */
464b12defdcSMatthew Dillon if (m->flags & PG_NOTMETA) {
465b12defdcSMatthew Dillon vm_object_drop(object);
466b12defdcSMatthew Dillon vm_page_wakeup(m);
467027193ebSMatthew Dillon vm_page_queues_spin_lock(marker->queue);
468bfa86281SMatthew Dillon continue;
469b12defdcSMatthew Dillon }
470bfa86281SMatthew Dillon
471bfa86281SMatthew Dillon /*
472e9b56058SMatthew Dillon * If data_enable is 0 do not try to swapcache data.
473e9b56058SMatthew Dillon * If use_chflags is set then only swapcache data for
474e9b56058SMatthew Dillon * VSWAPCACHE marked vnodes, otherwise any vnode.
475e9b56058SMatthew Dillon */
476e9b56058SMatthew Dillon if (vm_swapcache_data_enable == 0 ||
477e9b56058SMatthew Dillon ((vp->v_flag & VSWAPCACHE) == 0 &&
478e9b56058SMatthew Dillon vm_swapcache_use_chflags)) {
479b12defdcSMatthew Dillon vm_object_drop(object);
480b12defdcSMatthew Dillon vm_page_wakeup(m);
481027193ebSMatthew Dillon vm_page_queues_spin_lock(marker->queue);
482c504e38eSMatthew Dillon continue;
483e9b56058SMatthew Dillon }
484d3070b8dSMatthew Dillon if (vm_swapcache_maxfilesize &&
485d3070b8dSMatthew Dillon object->size >
486d3070b8dSMatthew Dillon (vm_swapcache_maxfilesize >> PAGE_SHIFT)) {
487b12defdcSMatthew Dillon vm_object_drop(object);
488b12defdcSMatthew Dillon vm_page_wakeup(m);
489027193ebSMatthew Dillon vm_page_queues_spin_lock(marker->queue);
490d3070b8dSMatthew Dillon continue;
491d3070b8dSMatthew Dillon }
492aabd5ce8SMatthew Dillon isblkdev = 0;
493c504e38eSMatthew Dillon break;
494c504e38eSMatthew Dillon case VCHR:
495aabd5ce8SMatthew Dillon /*
496bfa86281SMatthew Dillon * PG_NOTMETA generically means 'don't swapcache this',
497bfa86281SMatthew Dillon * and HAMMER will set this for regular data buffers
498bfa86281SMatthew Dillon * (and leave it unset for meta-data buffers) as
499bfa86281SMatthew Dillon * appropriate when double buffering is enabled.
500aabd5ce8SMatthew Dillon */
501b12defdcSMatthew Dillon if (m->flags & PG_NOTMETA) {
502b12defdcSMatthew Dillon vm_object_drop(object);
503b12defdcSMatthew Dillon vm_page_wakeup(m);
504027193ebSMatthew Dillon vm_page_queues_spin_lock(marker->queue);
505aabd5ce8SMatthew Dillon continue;
506b12defdcSMatthew Dillon }
507b12defdcSMatthew Dillon if (vm_swapcache_meta_enable == 0) {
508b12defdcSMatthew Dillon vm_object_drop(object);
509b12defdcSMatthew Dillon vm_page_wakeup(m);
510027193ebSMatthew Dillon vm_page_queues_spin_lock(marker->queue);
511c504e38eSMatthew Dillon continue;
512b12defdcSMatthew Dillon }
513aabd5ce8SMatthew Dillon isblkdev = 1;
514c504e38eSMatthew Dillon break;
515c504e38eSMatthew Dillon default:
516b12defdcSMatthew Dillon vm_object_drop(object);
517b12defdcSMatthew Dillon vm_page_wakeup(m);
518027193ebSMatthew Dillon vm_page_queues_spin_lock(marker->queue);
519c504e38eSMatthew Dillon continue;
520c504e38eSMatthew Dillon }
521096e95c0SMatthew Dillon
522096e95c0SMatthew Dillon
523096e95c0SMatthew Dillon /*
5243ffc7051SMatthew Dillon * Assign swap and initiate I/O.
5253ffc7051SMatthew Dillon *
5263ffc7051SMatthew Dillon * (adjust for the --count which also occurs in the loop)
527096e95c0SMatthew Dillon */
5280bf81261SMatthew Dillon count -= vm_swapcached_flush(m, isblkdev);
529096e95c0SMatthew Dillon
530096e95c0SMatthew Dillon /*
531096e95c0SMatthew Dillon * Setup for next loop using marker.
532096e95c0SMatthew Dillon */
533b12defdcSMatthew Dillon vm_object_drop(object);
534027193ebSMatthew Dillon vm_page_queues_spin_lock(marker->queue);
535096e95c0SMatthew Dillon }
5361e5196f0SMatthew Dillon
5371e5196f0SMatthew Dillon /*
538b12defdcSMatthew Dillon * The marker could wind up at the end, which is ok. If we hit the
539b12defdcSMatthew Dillon * end of the list adjust the heuristic.
5401e5196f0SMatthew Dillon *
5411e5196f0SMatthew Dillon * Earlier inactive pages that were dirty and become clean
5421e5196f0SMatthew Dillon * are typically moved to the end of PQ_INACTIVE by virtue
5431e5196f0SMatthew Dillon * of vfs_vmio_release() when they become unwired from the
5441e5196f0SMatthew Dillon * buffer cache.
5451e5196f0SMatthew Dillon */
546027193ebSMatthew Dillon vm_page_queues_spin_unlock(marker->queue);
54764949baaSMatthew Dillon
54864949baaSMatthew Dillon /*
54964949baaSMatthew Dillon * m invalid but can be used to test for NULL
55064949baaSMatthew Dillon */
55164949baaSMatthew Dillon return (m == NULL);
552096e95c0SMatthew Dillon }
553096e95c0SMatthew Dillon
554096e95c0SMatthew Dillon /*
555b12defdcSMatthew Dillon * Flush the specified page using the swap_pager. The page
556b12defdcSMatthew Dillon * must be busied by the caller and its disposition will become
557b12defdcSMatthew Dillon * the responsibility of this function.
5583ffc7051SMatthew Dillon *
5593ffc7051SMatthew Dillon * Try to collect surrounding pages, including pages which may
5603ffc7051SMatthew Dillon * have already been assigned swap. Try to cluster within a
5613ffc7051SMatthew Dillon * contiguous aligned SMAP_META_PAGES (typ 16 x PAGE_SIZE) block
5623ffc7051SMatthew Dillon * to match what swap_pager_putpages() can do.
5633ffc7051SMatthew Dillon *
5643ffc7051SMatthew Dillon * We also want to try to match against the buffer cache blocksize
5653ffc7051SMatthew Dillon * but we don't really know what it is here. Since the buffer cache
5663ffc7051SMatthew Dillon * wires and unwires pages in groups the fact that we skip wired pages
5673ffc7051SMatthew Dillon * should be sufficient.
5683ffc7051SMatthew Dillon *
5693ffc7051SMatthew Dillon * Returns a count of pages we might have flushed (minimum 1)
570096e95c0SMatthew Dillon */
571096e95c0SMatthew Dillon static
5723ffc7051SMatthew Dillon int
vm_swapcached_flush(vm_page_t m,int isblkdev)573aabd5ce8SMatthew Dillon vm_swapcached_flush(vm_page_t m, int isblkdev)
574096e95c0SMatthew Dillon {
575096e95c0SMatthew Dillon vm_object_t object;
5763ffc7051SMatthew Dillon vm_page_t marray[SWAP_META_PAGES];
5773ffc7051SMatthew Dillon vm_pindex_t basei;
5783ffc7051SMatthew Dillon int rtvals[SWAP_META_PAGES];
5793ffc7051SMatthew Dillon int x;
5803ffc7051SMatthew Dillon int i;
5813ffc7051SMatthew Dillon int j;
5823ffc7051SMatthew Dillon int count;
583b12defdcSMatthew Dillon int error;
584096e95c0SMatthew Dillon
585096e95c0SMatthew Dillon vm_page_io_start(m);
586096e95c0SMatthew Dillon vm_page_protect(m, VM_PROT_READ);
587096e95c0SMatthew Dillon object = m->object;
588b12defdcSMatthew Dillon vm_object_hold(object);
589096e95c0SMatthew Dillon
5903ffc7051SMatthew Dillon /*
5913ffc7051SMatthew Dillon * Try to cluster around (m), keeping in mind that the swap pager
5923ffc7051SMatthew Dillon * can only do SMAP_META_PAGES worth of continguous write.
5933ffc7051SMatthew Dillon */
5943ffc7051SMatthew Dillon x = (int)m->pindex & SWAP_META_MASK;
5953ffc7051SMatthew Dillon marray[x] = m;
5963ffc7051SMatthew Dillon basei = m->pindex;
597b12defdcSMatthew Dillon vm_page_wakeup(m);
5983ffc7051SMatthew Dillon
5993ffc7051SMatthew Dillon for (i = x - 1; i >= 0; --i) {
600b12defdcSMatthew Dillon m = vm_page_lookup_busy_try(object, basei - x + i,
601b12defdcSMatthew Dillon TRUE, &error);
602b12defdcSMatthew Dillon if (error || m == NULL)
6033ffc7051SMatthew Dillon break;
604b12defdcSMatthew Dillon if (vm_swapcache_test(m)) {
605b12defdcSMatthew Dillon vm_page_wakeup(m);
6063ffc7051SMatthew Dillon break;
607b12defdcSMatthew Dillon }
608b12defdcSMatthew Dillon if (isblkdev && (m->flags & PG_NOTMETA)) {
609b12defdcSMatthew Dillon vm_page_wakeup(m);
610aabd5ce8SMatthew Dillon break;
611b12defdcSMatthew Dillon }
6123ffc7051SMatthew Dillon vm_page_io_start(m);
6133ffc7051SMatthew Dillon vm_page_protect(m, VM_PROT_READ);
6143ffc7051SMatthew Dillon if (m->queue - m->pc == PQ_CACHE) {
6153ffc7051SMatthew Dillon vm_page_unqueue_nowakeup(m);
6163ffc7051SMatthew Dillon vm_page_deactivate(m);
6173ffc7051SMatthew Dillon }
6183ffc7051SMatthew Dillon marray[i] = m;
619b12defdcSMatthew Dillon vm_page_wakeup(m);
6203ffc7051SMatthew Dillon }
6213ffc7051SMatthew Dillon ++i;
6223ffc7051SMatthew Dillon
6233ffc7051SMatthew Dillon for (j = x + 1; j < SWAP_META_PAGES; ++j) {
624b12defdcSMatthew Dillon m = vm_page_lookup_busy_try(object, basei - x + j,
625b12defdcSMatthew Dillon TRUE, &error);
626b12defdcSMatthew Dillon if (error || m == NULL)
6273ffc7051SMatthew Dillon break;
628b12defdcSMatthew Dillon if (vm_swapcache_test(m)) {
629b12defdcSMatthew Dillon vm_page_wakeup(m);
6303ffc7051SMatthew Dillon break;
631b12defdcSMatthew Dillon }
632b12defdcSMatthew Dillon if (isblkdev && (m->flags & PG_NOTMETA)) {
633b12defdcSMatthew Dillon vm_page_wakeup(m);
634aabd5ce8SMatthew Dillon break;
635b12defdcSMatthew Dillon }
6363ffc7051SMatthew Dillon vm_page_io_start(m);
6373ffc7051SMatthew Dillon vm_page_protect(m, VM_PROT_READ);
6383ffc7051SMatthew Dillon if (m->queue - m->pc == PQ_CACHE) {
6393ffc7051SMatthew Dillon vm_page_unqueue_nowakeup(m);
6403ffc7051SMatthew Dillon vm_page_deactivate(m);
6413ffc7051SMatthew Dillon }
6423ffc7051SMatthew Dillon marray[j] = m;
643b12defdcSMatthew Dillon vm_page_wakeup(m);
6443ffc7051SMatthew Dillon }
6453ffc7051SMatthew Dillon
6463ffc7051SMatthew Dillon count = j - i;
6473ffc7051SMatthew Dillon vm_object_pip_add(object, count);
6483ffc7051SMatthew Dillon swap_pager_putpages(object, marray + i, count, FALSE, rtvals + i);
6493ffc7051SMatthew Dillon vm_swapcache_write_count += count * PAGE_SIZE;
6503ffc7051SMatthew Dillon vm_swapcache_curburst -= count * PAGE_SIZE;
6513ffc7051SMatthew Dillon
6523ffc7051SMatthew Dillon while (i < j) {
6533ffc7051SMatthew Dillon if (rtvals[i] != VM_PAGER_PEND) {
654b12defdcSMatthew Dillon vm_page_busy_wait(marray[i], FALSE, "swppgfd");
6553ffc7051SMatthew Dillon vm_page_io_finish(marray[i]);
656b12defdcSMatthew Dillon vm_page_wakeup(marray[i]);
657096e95c0SMatthew Dillon vm_object_pip_wakeup(object);
658096e95c0SMatthew Dillon }
6593ffc7051SMatthew Dillon ++i;
6603ffc7051SMatthew Dillon }
661b12defdcSMatthew Dillon vm_object_drop(object);
6623ffc7051SMatthew Dillon return(count);
663096e95c0SMatthew Dillon }
66400a3fdcaSMatthew Dillon
6653ffc7051SMatthew Dillon /*
6663ffc7051SMatthew Dillon * Test whether a VM page is suitable for writing to the swapcache.
6673ffc7051SMatthew Dillon * Does not test m->queue, PG_MARKER, or PG_SWAPPED.
6683ffc7051SMatthew Dillon *
6693ffc7051SMatthew Dillon * Returns 0 on success, 1 on failure
6703ffc7051SMatthew Dillon */
6713ffc7051SMatthew Dillon static int
vm_swapcache_test(vm_page_t m)6723ffc7051SMatthew Dillon vm_swapcache_test(vm_page_t m)
6733ffc7051SMatthew Dillon {
6743ffc7051SMatthew Dillon vm_object_t object;
6753ffc7051SMatthew Dillon
676*831a8507SMatthew Dillon if (m->flags & (PG_UNQUEUED | PG_FICTITIOUS))
6773ffc7051SMatthew Dillon return(1);
678b12defdcSMatthew Dillon if (m->hold_count || m->wire_count)
6793ffc7051SMatthew Dillon return(1);
6803ffc7051SMatthew Dillon if (m->valid != VM_PAGE_BITS_ALL)
6813ffc7051SMatthew Dillon return(1);
6823ffc7051SMatthew Dillon if (m->dirty & m->valid)
6833ffc7051SMatthew Dillon return(1);
6843ffc7051SMatthew Dillon if ((object = m->object) == NULL)
6853ffc7051SMatthew Dillon return(1);
6863ffc7051SMatthew Dillon if (object->type != OBJT_VNODE ||
6873ffc7051SMatthew Dillon (object->flags & OBJ_DEAD)) {
6883ffc7051SMatthew Dillon return(1);
6893ffc7051SMatthew Dillon }
6903ffc7051SMatthew Dillon vm_page_test_dirty(m);
6913ffc7051SMatthew Dillon if (m->dirty & m->valid)
6923ffc7051SMatthew Dillon return(1);
6933ffc7051SMatthew Dillon return(0);
6943ffc7051SMatthew Dillon }
6953ffc7051SMatthew Dillon
6963ffc7051SMatthew Dillon /*
697f5f6d247SMatthew Dillon * Cleaning pass.
698f5f6d247SMatthew Dillon *
699f5f6d247SMatthew Dillon * We clean whole objects up to 16MB
7003ffc7051SMatthew Dillon */
70100a3fdcaSMatthew Dillon static
70200a3fdcaSMatthew Dillon void
vm_swapcache_cleaning(swmarker_t * marker,struct vm_object_hash ** swindexp)7039de48eadSMatthew Dillon vm_swapcache_cleaning(swmarker_t *marker, struct vm_object_hash **swindexp)
70400a3fdcaSMatthew Dillon {
70500a3fdcaSMatthew Dillon vm_object_t object;
70600a3fdcaSMatthew Dillon struct vnode *vp;
70700a3fdcaSMatthew Dillon int count;
7080bf81261SMatthew Dillon int scount;
70900a3fdcaSMatthew Dillon int n;
710eea5ad68SMatthew Dillon int didmove;
71100a3fdcaSMatthew Dillon
71200a3fdcaSMatthew Dillon count = vm_swapcache_maxlaunder;
7130bf81261SMatthew Dillon scount = vm_swapcache_maxscan;
71400a3fdcaSMatthew Dillon
71500a3fdcaSMatthew Dillon /*
71600a3fdcaSMatthew Dillon * Look for vnode objects
71700a3fdcaSMatthew Dillon */
718fde6be6aSMatthew Dillon lwkt_gettoken(&(*swindexp)->token);
7192de4f77eSMatthew Dillon
720eea5ad68SMatthew Dillon didmove = 0;
721b2286ce7SMatthew Dillon outerloop:
7225b329e62SMatthew Dillon while ((object = TAILQ_NEXT(&marker->dummy_obj,
7235b329e62SMatthew Dillon object_entry)) != NULL) {
7242f2d9e58SVenkatesh Srinivas /*
725f5f6d247SMatthew Dillon * We have to skip markers. We cannot hold/drop marker
726f5f6d247SMatthew Dillon * objects!
7272f2d9e58SVenkatesh Srinivas */
728f5f6d247SMatthew Dillon if (object->type == OBJT_MARKER) {
7297b00fbb4SMatthew Dillon vm_swapcache_movemarker(marker, *swindexp, object);
730eea5ad68SMatthew Dillon didmove = 1;
73100a3fdcaSMatthew Dillon continue;
7322f2d9e58SVenkatesh Srinivas }
73300a3fdcaSMatthew Dillon
73400a3fdcaSMatthew Dillon /*
735f5f6d247SMatthew Dillon * Safety, or in case there are millions of VM objects
736f5f6d247SMatthew Dillon * without swapcache backing.
73700a3fdcaSMatthew Dillon */
7380bf81261SMatthew Dillon if (--scount <= 0)
7397b00fbb4SMatthew Dillon goto breakout;
74000a3fdcaSMatthew Dillon
74100a3fdcaSMatthew Dillon /*
742f5f6d247SMatthew Dillon * We must hold the object before potentially yielding.
74300a3fdcaSMatthew Dillon */
744f5f6d247SMatthew Dillon vm_object_hold(object);
745f5f6d247SMatthew Dillon lwkt_yield();
746f5f6d247SMatthew Dillon
747f5f6d247SMatthew Dillon /*
748f5f6d247SMatthew Dillon * Only operate on live VNODE objects that are either
749f5f6d247SMatthew Dillon * VREG or VCHR (VCHR for meta-data).
750f5f6d247SMatthew Dillon */
751f5f6d247SMatthew Dillon if ((object->type != OBJT_VNODE) ||
752f5f6d247SMatthew Dillon ((object->flags & OBJ_DEAD) ||
753f5f6d247SMatthew Dillon object->swblock_count == 0) ||
754f5f6d247SMatthew Dillon ((vp = object->handle) == NULL) ||
755f5f6d247SMatthew Dillon (vp->v_type != VREG && vp->v_type != VCHR)) {
756f5f6d247SMatthew Dillon vm_object_drop(object);
757f5f6d247SMatthew Dillon /* object may be invalid now */
7587b00fbb4SMatthew Dillon vm_swapcache_movemarker(marker, *swindexp, object);
759eea5ad68SMatthew Dillon didmove = 1;
760f5f6d247SMatthew Dillon continue;
761f5f6d247SMatthew Dillon }
762f5f6d247SMatthew Dillon
763f5f6d247SMatthew Dillon /*
764f5f6d247SMatthew Dillon * Reset the object pindex stored in the marker if the
765f5f6d247SMatthew Dillon * working object has changed.
766f5f6d247SMatthew Dillon */
7679de48eadSMatthew Dillon if (marker->save_obj != object || didmove) {
7689de48eadSMatthew Dillon marker->dummy_obj.size = 0;
7699de48eadSMatthew Dillon marker->save_off = 0;
7709de48eadSMatthew Dillon marker->save_obj = object;
771eea5ad68SMatthew Dillon didmove = 0;
772f5f6d247SMatthew Dillon }
77300a3fdcaSMatthew Dillon
77400a3fdcaSMatthew Dillon /*
77500a3fdcaSMatthew Dillon * Look for swblocks starting at our iterator.
77600a3fdcaSMatthew Dillon *
77700a3fdcaSMatthew Dillon * The swap_pager_condfree() function attempts to free
77800a3fdcaSMatthew Dillon * swap space starting at the specified index. The index
77900a3fdcaSMatthew Dillon * will be updated on return. The function will return
78000a3fdcaSMatthew Dillon * a scan factor (NOT the number of blocks freed).
78100a3fdcaSMatthew Dillon *
78200a3fdcaSMatthew Dillon * If it must cut its scan of the object short due to an
78300a3fdcaSMatthew Dillon * excessive number of swblocks, or is able to free the
78400a3fdcaSMatthew Dillon * requested number of blocks, it will return n >= count
78500a3fdcaSMatthew Dillon * and we break and pick it back up on a future attempt.
786f5f6d247SMatthew Dillon *
787f5f6d247SMatthew Dillon * Scan the object linearly and try to batch large sets of
788f5f6d247SMatthew Dillon * blocks that are likely to clean out entire swap radix
789f5f6d247SMatthew Dillon * tree leafs.
79000a3fdcaSMatthew Dillon */
791739be60bSMatthew Dillon lwkt_token_swap();
792fde6be6aSMatthew Dillon lwkt_reltoken(&(*swindexp)->token);
79327b6ee03SMatthew Dillon
7949de48eadSMatthew Dillon n = swap_pager_condfree(object, &marker->dummy_obj.size,
795f5f6d247SMatthew Dillon (count + SWAP_META_MASK) & ~SWAP_META_MASK);
7962f2d9e58SVenkatesh Srinivas
797f5f6d247SMatthew Dillon vm_object_drop(object); /* object may be invalid now */
798fde6be6aSMatthew Dillon lwkt_gettoken(&(*swindexp)->token);
7992f2d9e58SVenkatesh Srinivas
80000a3fdcaSMatthew Dillon /*
801f5f6d247SMatthew Dillon * If we have exhausted the object or deleted our per-pass
802f5f6d247SMatthew Dillon * page limit then move us to the next object. Note that
8035b329e62SMatthew Dillon * the current object may no longer be on the vm_object_entry.
80400a3fdcaSMatthew Dillon */
805f5f6d247SMatthew Dillon if (n <= 0 ||
8069de48eadSMatthew Dillon marker->save_off > vm_swapcache_cleanperobj) {
8077b00fbb4SMatthew Dillon vm_swapcache_movemarker(marker, *swindexp, object);
808eea5ad68SMatthew Dillon didmove = 1;
80900a3fdcaSMatthew Dillon }
81000a3fdcaSMatthew Dillon
81100a3fdcaSMatthew Dillon /*
812f5f6d247SMatthew Dillon * If we have exhausted our max-launder stop for now.
81300a3fdcaSMatthew Dillon */
814f5f6d247SMatthew Dillon count -= n;
8159de48eadSMatthew Dillon marker->save_off += n * PAGE_SIZE;
816f5f6d247SMatthew Dillon if (count < 0)
8177b00fbb4SMatthew Dillon goto breakout;
818f5f6d247SMatthew Dillon }
8197a175765SMatthew Dillon
8207a175765SMatthew Dillon /*
8215b329e62SMatthew Dillon * Iterate vm_object_hash[] hash table
8227a175765SMatthew Dillon */
8235b329e62SMatthew Dillon TAILQ_REMOVE(&(*swindexp)->list, &marker->dummy_obj, object_entry);
824fde6be6aSMatthew Dillon lwkt_reltoken(&(*swindexp)->token);
825fde6be6aSMatthew Dillon if (++*swindexp >= &vm_object_hash[VMOBJ_HSIZE])
826fde6be6aSMatthew Dillon *swindexp = &vm_object_hash[0];
827fde6be6aSMatthew Dillon lwkt_gettoken(&(*swindexp)->token);
8285b329e62SMatthew Dillon TAILQ_INSERT_HEAD(&(*swindexp)->list, &marker->dummy_obj, object_entry);
8297a175765SMatthew Dillon
830fde6be6aSMatthew Dillon if (*swindexp != &vm_object_hash[0])
8317b00fbb4SMatthew Dillon goto outerloop;
8327b00fbb4SMatthew Dillon
8337b00fbb4SMatthew Dillon breakout:
834fde6be6aSMatthew Dillon lwkt_reltoken(&(*swindexp)->token);
83500a3fdcaSMatthew Dillon }
836f5f6d247SMatthew Dillon
837f5f6d247SMatthew Dillon /*
838f5f6d247SMatthew Dillon * Move the marker past the current object. Object can be stale, but we
839f5f6d247SMatthew Dillon * still need it to determine if the marker has to be moved. If the object
840f5f6d247SMatthew Dillon * is still the 'current object' (object after the marker), we hop-scotch
841f5f6d247SMatthew Dillon * the marker past it.
842f5f6d247SMatthew Dillon */
843f5f6d247SMatthew Dillon static void
vm_swapcache_movemarker(swmarker_t * marker,struct vm_object_hash * swindex,vm_object_t object)8449de48eadSMatthew Dillon vm_swapcache_movemarker(swmarker_t *marker, struct vm_object_hash *swindex,
845fde6be6aSMatthew Dillon vm_object_t object)
846f5f6d247SMatthew Dillon {
8475b329e62SMatthew Dillon if (TAILQ_NEXT(&marker->dummy_obj, object_entry) == object) {
8485b329e62SMatthew Dillon TAILQ_REMOVE(&swindex->list, &marker->dummy_obj, object_entry);
8499de48eadSMatthew Dillon TAILQ_INSERT_AFTER(&swindex->list, object,
8505b329e62SMatthew Dillon &marker->dummy_obj, object_entry);
851f5f6d247SMatthew Dillon }
852f5f6d247SMatthew Dillon }
853