1ac04195bSKonstantin Belousov /*-
2796df753SPedro F. Giffuni * SPDX-License-Identifier: (BSD-4-Clause AND MIT-CMU)
3796df753SPedro F. Giffuni *
4ac04195bSKonstantin Belousov * Copyright (c) 1991 Regents of the University of California.
5ac04195bSKonstantin Belousov * All rights reserved.
6ac04195bSKonstantin Belousov * Copyright (c) 1994 John S. Dyson
7ac04195bSKonstantin Belousov * All rights reserved.
8ac04195bSKonstantin Belousov * Copyright (c) 1994 David Greenman
9ac04195bSKonstantin Belousov * All rights reserved.
10ac04195bSKonstantin Belousov * Copyright (c) 2005 Yahoo! Technologies Norway AS
11ac04195bSKonstantin Belousov * All rights reserved.
12ac04195bSKonstantin Belousov *
13ac04195bSKonstantin Belousov * This code is derived from software contributed to Berkeley by
14ac04195bSKonstantin Belousov * The Mach Operating System project at Carnegie-Mellon University.
15ac04195bSKonstantin Belousov *
16ac04195bSKonstantin Belousov * Redistribution and use in source and binary forms, with or without
17ac04195bSKonstantin Belousov * modification, are permitted provided that the following conditions
18ac04195bSKonstantin Belousov * are met:
19ac04195bSKonstantin Belousov * 1. Redistributions of source code must retain the above copyright
20ac04195bSKonstantin Belousov * notice, this list of conditions and the following disclaimer.
21ac04195bSKonstantin Belousov * 2. Redistributions in binary form must reproduce the above copyright
22ac04195bSKonstantin Belousov * notice, this list of conditions and the following disclaimer in the
23ac04195bSKonstantin Belousov * documentation and/or other materials provided with the distribution.
24ac04195bSKonstantin Belousov * 3. All advertising materials mentioning features or use of this software
25ac04195bSKonstantin Belousov * must display the following acknowledgement:
26ac04195bSKonstantin Belousov * This product includes software developed by the University of
27ac04195bSKonstantin Belousov * California, Berkeley and its contributors.
28ac04195bSKonstantin Belousov * 4. Neither the name of the University nor the names of its contributors
29ac04195bSKonstantin Belousov * may be used to endorse or promote products derived from this software
30ac04195bSKonstantin Belousov * without specific prior written permission.
31ac04195bSKonstantin Belousov *
32ac04195bSKonstantin Belousov * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
33ac04195bSKonstantin Belousov * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
34ac04195bSKonstantin Belousov * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
35ac04195bSKonstantin Belousov * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
36ac04195bSKonstantin Belousov * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
37ac04195bSKonstantin Belousov * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
38ac04195bSKonstantin Belousov * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
39ac04195bSKonstantin Belousov * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
40ac04195bSKonstantin Belousov * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
41ac04195bSKonstantin Belousov * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
42ac04195bSKonstantin Belousov * SUCH DAMAGE.
43ac04195bSKonstantin Belousov *
44ac04195bSKonstantin Belousov *
45ac04195bSKonstantin Belousov * Copyright (c) 1987, 1990 Carnegie-Mellon University.
46ac04195bSKonstantin Belousov * All rights reserved.
47ac04195bSKonstantin Belousov *
48ac04195bSKonstantin Belousov * Authors: Avadis Tevanian, Jr., Michael Wayne Young
49ac04195bSKonstantin Belousov *
50ac04195bSKonstantin Belousov * Permission to use, copy, modify and distribute this software and
51ac04195bSKonstantin Belousov * its documentation is hereby granted, provided that both the copyright
52ac04195bSKonstantin Belousov * notice and this permission notice appear in all copies of the
53ac04195bSKonstantin Belousov * software, derivative works or modified versions, and any portions
54ac04195bSKonstantin Belousov * thereof, and that both notices appear in supporting documentation.
55ac04195bSKonstantin Belousov *
56ac04195bSKonstantin Belousov * CARNEGIE MELLON ALLOWS FREE USE OF THIS SOFTWARE IN ITS "AS IS"
57ac04195bSKonstantin Belousov * CONDITION. CARNEGIE MELLON DISCLAIMS ANY LIABILITY OF ANY KIND
58ac04195bSKonstantin Belousov * FOR ANY DAMAGES WHATSOEVER RESULTING FROM THE USE OF THIS SOFTWARE.
59ac04195bSKonstantin Belousov *
60ac04195bSKonstantin Belousov * Carnegie Mellon requests users of this software to return to
61ac04195bSKonstantin Belousov *
62ac04195bSKonstantin Belousov * Software Distribution Coordinator or Software.Distribution@CS.CMU.EDU
63ac04195bSKonstantin Belousov * School of Computer Science
64ac04195bSKonstantin Belousov * Carnegie Mellon University
65ac04195bSKonstantin Belousov * Pittsburgh PA 15213-3890
66ac04195bSKonstantin Belousov *
67ac04195bSKonstantin Belousov * any improvements or extensions that they make and grant Carnegie the
68ac04195bSKonstantin Belousov * rights to redistribute these changes.
69ac04195bSKonstantin Belousov */
70ac04195bSKonstantin Belousov
71ac04195bSKonstantin Belousov #include <sys/cdefs.h>
72ac04195bSKonstantin Belousov #include "opt_kstack_pages.h"
73ac04195bSKonstantin Belousov #include "opt_kstack_max_pages.h"
74ac04195bSKonstantin Belousov #include "opt_vm.h"
75ac04195bSKonstantin Belousov
76ac04195bSKonstantin Belousov #include <sys/param.h>
77ac04195bSKonstantin Belousov #include <sys/systm.h>
78ac04195bSKonstantin Belousov #include <sys/limits.h>
79ac04195bSKonstantin Belousov #include <sys/kernel.h>
80ac04195bSKonstantin Belousov #include <sys/eventhandler.h>
81ac04195bSKonstantin Belousov #include <sys/lock.h>
82ac04195bSKonstantin Belousov #include <sys/mutex.h>
83ac04195bSKonstantin Belousov #include <sys/proc.h>
84ac04195bSKonstantin Belousov #include <sys/kthread.h>
85ac04195bSKonstantin Belousov #include <sys/ktr.h>
86ac04195bSKonstantin Belousov #include <sys/mount.h>
87ac04195bSKonstantin Belousov #include <sys/racct.h>
88ac04195bSKonstantin Belousov #include <sys/resourcevar.h>
8911b57401SHans Petter Selasky #include <sys/refcount.h>
90ac04195bSKonstantin Belousov #include <sys/sched.h>
91ac04195bSKonstantin Belousov #include <sys/sdt.h>
92ac04195bSKonstantin Belousov #include <sys/signalvar.h>
93ac04195bSKonstantin Belousov #include <sys/smp.h>
94ac04195bSKonstantin Belousov #include <sys/time.h>
95ac04195bSKonstantin Belousov #include <sys/vnode.h>
96ac04195bSKonstantin Belousov #include <sys/vmmeter.h>
97ac04195bSKonstantin Belousov #include <sys/rwlock.h>
98ac04195bSKonstantin Belousov #include <sys/sx.h>
99ac04195bSKonstantin Belousov #include <sys/sysctl.h>
100ac04195bSKonstantin Belousov
101ac04195bSKonstantin Belousov #include <vm/vm.h>
102ac04195bSKonstantin Belousov #include <vm/vm_param.h>
103f13fa9dfSMark Johnston #include <vm/vm_kern.h>
104ac04195bSKonstantin Belousov #include <vm/vm_object.h>
105ac04195bSKonstantin Belousov #include <vm/vm_page.h>
106ac04195bSKonstantin Belousov #include <vm/vm_map.h>
107ac04195bSKonstantin Belousov #include <vm/vm_pageout.h>
10841fd4b94SMark Johnston #include <vm/vm_pager.h>
109e8bcf696SMark Johnston #include <vm/vm_phys.h>
110ac04195bSKonstantin Belousov #include <vm/swap_pager.h>
111ac04195bSKonstantin Belousov #include <vm/vm_extern.h>
112ac04195bSKonstantin Belousov #include <vm/uma.h>
113ac04195bSKonstantin Belousov
114ac04195bSKonstantin Belousov /* the kernel process "vm_daemon" */
115ac04195bSKonstantin Belousov static void vm_daemon(void);
116ac04195bSKonstantin Belousov static struct proc *vmproc;
117ac04195bSKonstantin Belousov
118ac04195bSKonstantin Belousov static struct kproc_desc vm_kp = {
119ac04195bSKonstantin Belousov "vmdaemon",
120ac04195bSKonstantin Belousov vm_daemon,
121ac04195bSKonstantin Belousov &vmproc
122ac04195bSKonstantin Belousov };
123ac04195bSKonstantin Belousov SYSINIT(vmdaemon, SI_SUB_KTHREAD_VM, SI_ORDER_FIRST, kproc_start, &vm_kp);
124ac04195bSKonstantin Belousov
125ac04195bSKonstantin Belousov static int vm_swap_enabled = 1;
126ac04195bSKonstantin Belousov static int vm_swap_idle_enabled = 0;
127ac04195bSKonstantin Belousov
128ac04195bSKonstantin Belousov SYSCTL_INT(_vm, VM_SWAPPING_ENABLED, swap_enabled, CTLFLAG_RW,
129ac04195bSKonstantin Belousov &vm_swap_enabled, 0,
130ac04195bSKonstantin Belousov "Enable entire process swapout");
131ac04195bSKonstantin Belousov SYSCTL_INT(_vm, OID_AUTO, swap_idle_enabled, CTLFLAG_RW,
132ac04195bSKonstantin Belousov &vm_swap_idle_enabled, 0,
133ac04195bSKonstantin Belousov "Allow swapout on idle criteria");
134ac04195bSKonstantin Belousov
135ac04195bSKonstantin Belousov /*
136ac04195bSKonstantin Belousov * Swap_idle_threshold1 is the guaranteed swapped in time for a process
137ac04195bSKonstantin Belousov */
138ac04195bSKonstantin Belousov static int swap_idle_threshold1 = 2;
139ac04195bSKonstantin Belousov SYSCTL_INT(_vm, OID_AUTO, swap_idle_threshold1, CTLFLAG_RW,
140ac04195bSKonstantin Belousov &swap_idle_threshold1, 0,
141ac04195bSKonstantin Belousov "Guaranteed swapped in time for a process");
142ac04195bSKonstantin Belousov
143ac04195bSKonstantin Belousov /*
144ac04195bSKonstantin Belousov * Swap_idle_threshold2 is the time that a process can be idle before
145ac04195bSKonstantin Belousov * it will be swapped out, if idle swapping is enabled.
146ac04195bSKonstantin Belousov */
147ac04195bSKonstantin Belousov static int swap_idle_threshold2 = 10;
148ac04195bSKonstantin Belousov SYSCTL_INT(_vm, OID_AUTO, swap_idle_threshold2, CTLFLAG_RW,
149ac04195bSKonstantin Belousov &swap_idle_threshold2, 0,
150ac04195bSKonstantin Belousov "Time before a process will be swapped out");
151ac04195bSKonstantin Belousov
1520f559a9fSEdward Tomasz Napierala static int vm_daemon_timeout = 0;
1530f559a9fSEdward Tomasz Napierala SYSCTL_INT(_vm, OID_AUTO, vmdaemon_timeout, CTLFLAG_RW,
1540f559a9fSEdward Tomasz Napierala &vm_daemon_timeout, 0,
1550f559a9fSEdward Tomasz Napierala "Time between vmdaemon runs");
1560f559a9fSEdward Tomasz Napierala
157ac04195bSKonstantin Belousov static int vm_pageout_req_swapout; /* XXX */
158ac04195bSKonstantin Belousov static int vm_daemon_needed;
159ac04195bSKonstantin Belousov static struct mtx vm_daemon_mtx;
160ac04195bSKonstantin Belousov /* Allow for use by vm_pageout before vm_daemon is initialized. */
161ac04195bSKonstantin Belousov MTX_SYSINIT(vm_daemon, &vm_daemon_mtx, "vm daemon", MTX_DEF);
162ac04195bSKonstantin Belousov
163a70e9a13SKonstantin Belousov static int swapped_cnt;
164c1344d2bSKonstantin Belousov static int swap_inprogress; /* Pending swap-ins done outside swapper. */
165c1344d2bSKonstantin Belousov static int last_swapin;
166a70e9a13SKonstantin Belousov
167ac04195bSKonstantin Belousov static void swapclear(struct proc *);
168ac04195bSKonstantin Belousov static int swapout(struct proc *);
169ac04195bSKonstantin Belousov static void vm_swapout_map_deactivate_pages(vm_map_t, long);
1709be9ea42SMark Johnston static void vm_swapout_object_deactivate(pmap_t, vm_object_t, long);
171ac04195bSKonstantin Belousov static void swapout_procs(int action);
172ac04195bSKonstantin Belousov static void vm_req_vmdaemon(int req);
173ac04195bSKonstantin Belousov static void vm_thread_swapout(struct thread *td);
174ac04195bSKonstantin Belousov
1759be9ea42SMark Johnston static void
vm_swapout_object_deactivate_page(pmap_t pmap,vm_page_t m,bool unmap)1769be9ea42SMark Johnston vm_swapout_object_deactivate_page(pmap_t pmap, vm_page_t m, bool unmap)
1779be9ea42SMark Johnston {
1789be9ea42SMark Johnston
179ac04195bSKonstantin Belousov /*
1803c01c56bSMark Johnston * Ignore unreclaimable wired pages. Repeat the check after busying
1813c01c56bSMark Johnston * since a busy holder may wire the page.
1829be9ea42SMark Johnston */
1833c01c56bSMark Johnston if (vm_page_wired(m) || !vm_page_tryxbusy(m))
1843c01c56bSMark Johnston return;
1853c01c56bSMark Johnston
1869be9ea42SMark Johnston if (vm_page_wired(m) || !pmap_page_exists_quick(pmap, m)) {
1879be9ea42SMark Johnston vm_page_xunbusy(m);
1889be9ea42SMark Johnston return;
1899be9ea42SMark Johnston }
1903c01c56bSMark Johnston if (!pmap_is_referenced(m)) {
1913c01c56bSMark Johnston if (!vm_page_active(m))
1929be9ea42SMark Johnston (void)vm_page_try_remove_all(m);
1933c01c56bSMark Johnston else if (unmap && vm_page_try_remove_all(m))
1949be9ea42SMark Johnston vm_page_deactivate(m);
1953c01c56bSMark Johnston }
1969be9ea42SMark Johnston vm_page_xunbusy(m);
1979be9ea42SMark Johnston }
1989be9ea42SMark Johnston
1999be9ea42SMark Johnston /*
2009be9ea42SMark Johnston * vm_swapout_object_deactivate
201ac04195bSKonstantin Belousov *
202ac04195bSKonstantin Belousov * Deactivate enough pages to satisfy the inactive target
203ac04195bSKonstantin Belousov * requirements.
204ac04195bSKonstantin Belousov *
205ac04195bSKonstantin Belousov * The object and map must be locked.
206ac04195bSKonstantin Belousov */
207ac04195bSKonstantin Belousov static void
vm_swapout_object_deactivate(pmap_t pmap,vm_object_t first_object,long desired)2089be9ea42SMark Johnston vm_swapout_object_deactivate(pmap_t pmap, vm_object_t first_object,
209ac04195bSKonstantin Belousov long desired)
210ac04195bSKonstantin Belousov {
211ac04195bSKonstantin Belousov vm_object_t backing_object, object;
2129be9ea42SMark Johnston vm_page_t m;
2139be9ea42SMark Johnston bool unmap;
214ac04195bSKonstantin Belousov
215ac04195bSKonstantin Belousov VM_OBJECT_ASSERT_LOCKED(first_object);
216ac04195bSKonstantin Belousov if ((first_object->flags & OBJ_FICTITIOUS) != 0)
217ac04195bSKonstantin Belousov return;
218ac04195bSKonstantin Belousov for (object = first_object;; object = backing_object) {
219ac04195bSKonstantin Belousov if (pmap_resident_count(pmap) <= desired)
220ac04195bSKonstantin Belousov goto unlock_return;
221ac04195bSKonstantin Belousov VM_OBJECT_ASSERT_LOCKED(object);
222ac04195bSKonstantin Belousov if ((object->flags & OBJ_UNMANAGED) != 0 ||
223c99d0c58SMark Johnston blockcount_read(&object->paging_in_progress) > 0)
224ac04195bSKonstantin Belousov goto unlock_return;
225ac04195bSKonstantin Belousov
2269be9ea42SMark Johnston unmap = true;
227ac04195bSKonstantin Belousov if (object->shadow_count > 1)
2289be9ea42SMark Johnston unmap = false;
2299be9ea42SMark Johnston
230ac04195bSKonstantin Belousov /*
231ac04195bSKonstantin Belousov * Scan the object's entire memory queue.
232ac04195bSKonstantin Belousov */
2339be9ea42SMark Johnston TAILQ_FOREACH(m, &object->memq, listq) {
234ac04195bSKonstantin Belousov if (pmap_resident_count(pmap) <= desired)
235ac04195bSKonstantin Belousov goto unlock_return;
2369997c048SKonstantin Belousov if (should_yield())
2379997c048SKonstantin Belousov goto unlock_return;
2389be9ea42SMark Johnston vm_swapout_object_deactivate_page(pmap, m, unmap);
239ac04195bSKonstantin Belousov }
240ac04195bSKonstantin Belousov if ((backing_object = object->backing_object) == NULL)
241ac04195bSKonstantin Belousov goto unlock_return;
242ac04195bSKonstantin Belousov VM_OBJECT_RLOCK(backing_object);
243ac04195bSKonstantin Belousov if (object != first_object)
244ac04195bSKonstantin Belousov VM_OBJECT_RUNLOCK(object);
245ac04195bSKonstantin Belousov }
246ac04195bSKonstantin Belousov unlock_return:
247ac04195bSKonstantin Belousov if (object != first_object)
248ac04195bSKonstantin Belousov VM_OBJECT_RUNLOCK(object);
249ac04195bSKonstantin Belousov }
250ac04195bSKonstantin Belousov
251ac04195bSKonstantin Belousov /*
252ac04195bSKonstantin Belousov * deactivate some number of pages in a map, try to do it fairly, but
253ac04195bSKonstantin Belousov * that is really hard to do.
254ac04195bSKonstantin Belousov */
255ac04195bSKonstantin Belousov static void
vm_swapout_map_deactivate_pages(vm_map_t map,long desired)256ac04195bSKonstantin Belousov vm_swapout_map_deactivate_pages(vm_map_t map, long desired)
257ac04195bSKonstantin Belousov {
258ac04195bSKonstantin Belousov vm_map_entry_t tmpe;
259ac04195bSKonstantin Belousov vm_object_t obj, bigobj;
260ac04195bSKonstantin Belousov int nothingwired;
261ac04195bSKonstantin Belousov
2620080a8faSKonstantin Belousov if (!vm_map_trylock_read(map))
263ac04195bSKonstantin Belousov return;
264ac04195bSKonstantin Belousov
265ac04195bSKonstantin Belousov bigobj = NULL;
266ac04195bSKonstantin Belousov nothingwired = TRUE;
267ac04195bSKonstantin Belousov
268ac04195bSKonstantin Belousov /*
269ac04195bSKonstantin Belousov * first, search out the biggest object, and try to free pages from
270ac04195bSKonstantin Belousov * that.
271ac04195bSKonstantin Belousov */
2722288078cSDoug Moore VM_MAP_ENTRY_FOREACH(tmpe, map) {
273ac04195bSKonstantin Belousov if ((tmpe->eflags & MAP_ENTRY_IS_SUB_MAP) == 0) {
274ac04195bSKonstantin Belousov obj = tmpe->object.vm_object;
275ac04195bSKonstantin Belousov if (obj != NULL && VM_OBJECT_TRYRLOCK(obj)) {
276ac04195bSKonstantin Belousov if (obj->shadow_count <= 1 &&
277ac04195bSKonstantin Belousov (bigobj == NULL ||
278ac04195bSKonstantin Belousov bigobj->resident_page_count <
279ac04195bSKonstantin Belousov obj->resident_page_count)) {
280ac04195bSKonstantin Belousov if (bigobj != NULL)
281ac04195bSKonstantin Belousov VM_OBJECT_RUNLOCK(bigobj);
282ac04195bSKonstantin Belousov bigobj = obj;
283ac04195bSKonstantin Belousov } else
284ac04195bSKonstantin Belousov VM_OBJECT_RUNLOCK(obj);
285ac04195bSKonstantin Belousov }
286ac04195bSKonstantin Belousov }
287ac04195bSKonstantin Belousov if (tmpe->wired_count > 0)
288ac04195bSKonstantin Belousov nothingwired = FALSE;
289ac04195bSKonstantin Belousov }
290ac04195bSKonstantin Belousov
291ac04195bSKonstantin Belousov if (bigobj != NULL) {
2929be9ea42SMark Johnston vm_swapout_object_deactivate(map->pmap, bigobj, desired);
293ac04195bSKonstantin Belousov VM_OBJECT_RUNLOCK(bigobj);
294ac04195bSKonstantin Belousov }
295ac04195bSKonstantin Belousov /*
296ac04195bSKonstantin Belousov * Next, hunt around for other pages to deactivate. We actually
297ac04195bSKonstantin Belousov * do this search sort of wrong -- .text first is not the best idea.
298ac04195bSKonstantin Belousov */
2992288078cSDoug Moore VM_MAP_ENTRY_FOREACH(tmpe, map) {
300ac04195bSKonstantin Belousov if (pmap_resident_count(vm_map_pmap(map)) <= desired)
301ac04195bSKonstantin Belousov break;
302ac04195bSKonstantin Belousov if ((tmpe->eflags & MAP_ENTRY_IS_SUB_MAP) == 0) {
303ac04195bSKonstantin Belousov obj = tmpe->object.vm_object;
304ac04195bSKonstantin Belousov if (obj != NULL) {
305ac04195bSKonstantin Belousov VM_OBJECT_RLOCK(obj);
3069be9ea42SMark Johnston vm_swapout_object_deactivate(map->pmap, obj,
3079be9ea42SMark Johnston desired);
308ac04195bSKonstantin Belousov VM_OBJECT_RUNLOCK(obj);
309ac04195bSKonstantin Belousov }
310ac04195bSKonstantin Belousov }
311ac04195bSKonstantin Belousov }
312ac04195bSKonstantin Belousov
313ac04195bSKonstantin Belousov /*
314ac04195bSKonstantin Belousov * Remove all mappings if a process is swapped out, this will free page
315ac04195bSKonstantin Belousov * table pages.
316ac04195bSKonstantin Belousov */
317ac04195bSKonstantin Belousov if (desired == 0 && nothingwired) {
318ac04195bSKonstantin Belousov pmap_remove(vm_map_pmap(map), vm_map_min(map),
319ac04195bSKonstantin Belousov vm_map_max(map));
320ac04195bSKonstantin Belousov }
321ac04195bSKonstantin Belousov
3220080a8faSKonstantin Belousov vm_map_unlock_read(map);
323ac04195bSKonstantin Belousov }
324ac04195bSKonstantin Belousov
325ac04195bSKonstantin Belousov /*
326ac04195bSKonstantin Belousov * Swap out requests
327ac04195bSKonstantin Belousov */
328ac04195bSKonstantin Belousov #define VM_SWAP_NORMAL 1
329ac04195bSKonstantin Belousov #define VM_SWAP_IDLE 2
330ac04195bSKonstantin Belousov
331ac04195bSKonstantin Belousov void
vm_swapout_run(void)332ac04195bSKonstantin Belousov vm_swapout_run(void)
333ac04195bSKonstantin Belousov {
334ac04195bSKonstantin Belousov
335ac04195bSKonstantin Belousov if (vm_swap_enabled)
336ac04195bSKonstantin Belousov vm_req_vmdaemon(VM_SWAP_NORMAL);
337ac04195bSKonstantin Belousov }
338ac04195bSKonstantin Belousov
339ac04195bSKonstantin Belousov /*
340ac04195bSKonstantin Belousov * Idle process swapout -- run once per second when pagedaemons are
341ac04195bSKonstantin Belousov * reclaiming pages.
342ac04195bSKonstantin Belousov */
343ac04195bSKonstantin Belousov void
vm_swapout_run_idle(void)344ac04195bSKonstantin Belousov vm_swapout_run_idle(void)
345ac04195bSKonstantin Belousov {
346ac04195bSKonstantin Belousov static long lsec;
347ac04195bSKonstantin Belousov
348ac04195bSKonstantin Belousov if (!vm_swap_idle_enabled || time_second == lsec)
349ac04195bSKonstantin Belousov return;
350ac04195bSKonstantin Belousov vm_req_vmdaemon(VM_SWAP_IDLE);
351ac04195bSKonstantin Belousov lsec = time_second;
352ac04195bSKonstantin Belousov }
353ac04195bSKonstantin Belousov
354ac04195bSKonstantin Belousov static void
vm_req_vmdaemon(int req)355ac04195bSKonstantin Belousov vm_req_vmdaemon(int req)
356ac04195bSKonstantin Belousov {
357ac04195bSKonstantin Belousov static int lastrun = 0;
358ac04195bSKonstantin Belousov
359ac04195bSKonstantin Belousov mtx_lock(&vm_daemon_mtx);
360ac04195bSKonstantin Belousov vm_pageout_req_swapout |= req;
361ac04195bSKonstantin Belousov if ((ticks > (lastrun + hz)) || (ticks < lastrun)) {
362ac04195bSKonstantin Belousov wakeup(&vm_daemon_needed);
363ac04195bSKonstantin Belousov lastrun = ticks;
364ac04195bSKonstantin Belousov }
365ac04195bSKonstantin Belousov mtx_unlock(&vm_daemon_mtx);
366ac04195bSKonstantin Belousov }
367ac04195bSKonstantin Belousov
368ac04195bSKonstantin Belousov static void
vm_daemon(void)369ac04195bSKonstantin Belousov vm_daemon(void)
370ac04195bSKonstantin Belousov {
371ac04195bSKonstantin Belousov struct rlimit rsslim;
372ac04195bSKonstantin Belousov struct proc *p;
373ac04195bSKonstantin Belousov struct thread *td;
374ac04195bSKonstantin Belousov struct vmspace *vm;
375ac04195bSKonstantin Belousov int breakout, swapout_flags, tryagain, attempts;
376ac04195bSKonstantin Belousov #ifdef RACCT
377ac04195bSKonstantin Belousov uint64_t rsize, ravailable;
3780f559a9fSEdward Tomasz Napierala
3790f559a9fSEdward Tomasz Napierala if (racct_enable && vm_daemon_timeout == 0)
3800f559a9fSEdward Tomasz Napierala vm_daemon_timeout = hz;
381ac04195bSKonstantin Belousov #endif
382ac04195bSKonstantin Belousov
383ac04195bSKonstantin Belousov while (TRUE) {
384ac04195bSKonstantin Belousov mtx_lock(&vm_daemon_mtx);
385ac04195bSKonstantin Belousov msleep(&vm_daemon_needed, &vm_daemon_mtx, PPAUSE, "psleep",
3860f559a9fSEdward Tomasz Napierala vm_daemon_timeout);
387ac04195bSKonstantin Belousov swapout_flags = vm_pageout_req_swapout;
388ac04195bSKonstantin Belousov vm_pageout_req_swapout = 0;
389ac04195bSKonstantin Belousov mtx_unlock(&vm_daemon_mtx);
3905cd29d0fSMark Johnston if (swapout_flags != 0) {
3915cd29d0fSMark Johnston /*
3925cd29d0fSMark Johnston * Drain the per-CPU page queue batches as a deadlock
3935cd29d0fSMark Johnston * avoidance measure.
3945cd29d0fSMark Johnston */
3955cd29d0fSMark Johnston if ((swapout_flags & VM_SWAP_NORMAL) != 0)
396386eba08SMark Johnston vm_page_pqbatch_drain();
397ac04195bSKonstantin Belousov swapout_procs(swapout_flags);
3985cd29d0fSMark Johnston }
399ac04195bSKonstantin Belousov
400ac04195bSKonstantin Belousov /*
401ac04195bSKonstantin Belousov * scan the processes for exceeding their rlimits or if
402ac04195bSKonstantin Belousov * process is swapped out -- deactivate pages
403ac04195bSKonstantin Belousov */
404ac04195bSKonstantin Belousov tryagain = 0;
405ac04195bSKonstantin Belousov attempts = 0;
406ac04195bSKonstantin Belousov again:
407ac04195bSKonstantin Belousov attempts++;
408ac04195bSKonstantin Belousov sx_slock(&allproc_lock);
409ac04195bSKonstantin Belousov FOREACH_PROC_IN_SYSTEM(p) {
410ac04195bSKonstantin Belousov vm_pindex_t limit, size;
411ac04195bSKonstantin Belousov
412ac04195bSKonstantin Belousov /*
413ac04195bSKonstantin Belousov * if this is a system process or if we have already
414ac04195bSKonstantin Belousov * looked at this process, skip it.
415ac04195bSKonstantin Belousov */
416ac04195bSKonstantin Belousov PROC_LOCK(p);
417ac04195bSKonstantin Belousov if (p->p_state != PRS_NORMAL ||
418ac04195bSKonstantin Belousov p->p_flag & (P_INEXEC | P_SYSTEM | P_WEXIT)) {
419ac04195bSKonstantin Belousov PROC_UNLOCK(p);
420ac04195bSKonstantin Belousov continue;
421ac04195bSKonstantin Belousov }
422ac04195bSKonstantin Belousov /*
423ac04195bSKonstantin Belousov * if the process is in a non-running type state,
424ac04195bSKonstantin Belousov * don't touch it.
425ac04195bSKonstantin Belousov */
426ac04195bSKonstantin Belousov breakout = 0;
427ac04195bSKonstantin Belousov FOREACH_THREAD_IN_PROC(p, td) {
428ac04195bSKonstantin Belousov thread_lock(td);
429ac04195bSKonstantin Belousov if (!TD_ON_RUNQ(td) &&
430ac04195bSKonstantin Belousov !TD_IS_RUNNING(td) &&
431ac04195bSKonstantin Belousov !TD_IS_SLEEPING(td) &&
432ac04195bSKonstantin Belousov !TD_IS_SUSPENDED(td)) {
433ac04195bSKonstantin Belousov thread_unlock(td);
434ac04195bSKonstantin Belousov breakout = 1;
435ac04195bSKonstantin Belousov break;
436ac04195bSKonstantin Belousov }
437ac04195bSKonstantin Belousov thread_unlock(td);
438ac04195bSKonstantin Belousov }
439ac04195bSKonstantin Belousov if (breakout) {
440ac04195bSKonstantin Belousov PROC_UNLOCK(p);
441ac04195bSKonstantin Belousov continue;
442ac04195bSKonstantin Belousov }
443ac04195bSKonstantin Belousov /*
444ac04195bSKonstantin Belousov * get a limit
445ac04195bSKonstantin Belousov */
446ac04195bSKonstantin Belousov lim_rlimit_proc(p, RLIMIT_RSS, &rsslim);
447ac04195bSKonstantin Belousov limit = OFF_TO_IDX(
448ac04195bSKonstantin Belousov qmin(rsslim.rlim_cur, rsslim.rlim_max));
449ac04195bSKonstantin Belousov
450ac04195bSKonstantin Belousov /*
451ac04195bSKonstantin Belousov * let processes that are swapped out really be
452ac04195bSKonstantin Belousov * swapped out set the limit to nothing (will force a
453ac04195bSKonstantin Belousov * swap-out.)
454ac04195bSKonstantin Belousov */
455ac04195bSKonstantin Belousov if ((p->p_flag & P_INMEM) == 0)
456ac04195bSKonstantin Belousov limit = 0; /* XXX */
457ac04195bSKonstantin Belousov vm = vmspace_acquire_ref(p);
458ac04195bSKonstantin Belousov _PHOLD_LITE(p);
459ac04195bSKonstantin Belousov PROC_UNLOCK(p);
460ac04195bSKonstantin Belousov if (vm == NULL) {
461ac04195bSKonstantin Belousov PRELE(p);
462ac04195bSKonstantin Belousov continue;
463ac04195bSKonstantin Belousov }
464ac04195bSKonstantin Belousov sx_sunlock(&allproc_lock);
465ac04195bSKonstantin Belousov
466ac04195bSKonstantin Belousov size = vmspace_resident_count(vm);
467ac04195bSKonstantin Belousov if (size >= limit) {
468ac04195bSKonstantin Belousov vm_swapout_map_deactivate_pages(
469ac04195bSKonstantin Belousov &vm->vm_map, limit);
470ac04195bSKonstantin Belousov size = vmspace_resident_count(vm);
471ac04195bSKonstantin Belousov }
472ac04195bSKonstantin Belousov #ifdef RACCT
473ac04195bSKonstantin Belousov if (racct_enable) {
474ac04195bSKonstantin Belousov rsize = IDX_TO_OFF(size);
475ac04195bSKonstantin Belousov PROC_LOCK(p);
476ac04195bSKonstantin Belousov if (p->p_state == PRS_NORMAL)
477ac04195bSKonstantin Belousov racct_set(p, RACCT_RSS, rsize);
478ac04195bSKonstantin Belousov ravailable = racct_get_available(p, RACCT_RSS);
479ac04195bSKonstantin Belousov PROC_UNLOCK(p);
480ac04195bSKonstantin Belousov if (rsize > ravailable) {
481ac04195bSKonstantin Belousov /*
482ac04195bSKonstantin Belousov * Don't be overly aggressive; this
483ac04195bSKonstantin Belousov * might be an innocent process,
484ac04195bSKonstantin Belousov * and the limit could've been exceeded
485ac04195bSKonstantin Belousov * by some memory hog. Don't try
486ac04195bSKonstantin Belousov * to deactivate more than 1/4th
487ac04195bSKonstantin Belousov * of process' resident set size.
488ac04195bSKonstantin Belousov */
489ac04195bSKonstantin Belousov if (attempts <= 8) {
490ac04195bSKonstantin Belousov if (ravailable < rsize -
491ac04195bSKonstantin Belousov (rsize / 4)) {
492ac04195bSKonstantin Belousov ravailable = rsize -
493ac04195bSKonstantin Belousov (rsize / 4);
494ac04195bSKonstantin Belousov }
495ac04195bSKonstantin Belousov }
496ac04195bSKonstantin Belousov vm_swapout_map_deactivate_pages(
497ac04195bSKonstantin Belousov &vm->vm_map,
498ac04195bSKonstantin Belousov OFF_TO_IDX(ravailable));
499ac04195bSKonstantin Belousov /* Update RSS usage after paging out. */
500ac04195bSKonstantin Belousov size = vmspace_resident_count(vm);
501ac04195bSKonstantin Belousov rsize = IDX_TO_OFF(size);
502ac04195bSKonstantin Belousov PROC_LOCK(p);
503ac04195bSKonstantin Belousov if (p->p_state == PRS_NORMAL)
504ac04195bSKonstantin Belousov racct_set(p, RACCT_RSS, rsize);
505ac04195bSKonstantin Belousov PROC_UNLOCK(p);
506ac04195bSKonstantin Belousov if (rsize > ravailable)
507ac04195bSKonstantin Belousov tryagain = 1;
508ac04195bSKonstantin Belousov }
509ac04195bSKonstantin Belousov }
510ac04195bSKonstantin Belousov #endif
511ac04195bSKonstantin Belousov vmspace_free(vm);
512ac04195bSKonstantin Belousov sx_slock(&allproc_lock);
513ac04195bSKonstantin Belousov PRELE(p);
514ac04195bSKonstantin Belousov }
515ac04195bSKonstantin Belousov sx_sunlock(&allproc_lock);
5169997c048SKonstantin Belousov if (tryagain != 0 && attempts <= 10) {
5179997c048SKonstantin Belousov maybe_yield();
518ac04195bSKonstantin Belousov goto again;
519ac04195bSKonstantin Belousov }
520ac04195bSKonstantin Belousov }
5219997c048SKonstantin Belousov }
522ac04195bSKonstantin Belousov
523ac04195bSKonstantin Belousov /*
524ac04195bSKonstantin Belousov * Allow a thread's kernel stack to be paged out.
525ac04195bSKonstantin Belousov */
526ac04195bSKonstantin Belousov static void
vm_thread_swapout(struct thread * td)527ac04195bSKonstantin Belousov vm_thread_swapout(struct thread *td)
528ac04195bSKonstantin Belousov {
529ac04195bSKonstantin Belousov vm_page_t m;
530f13fa9dfSMark Johnston vm_offset_t kaddr;
531f13fa9dfSMark Johnston vm_pindex_t pindex;
532ac04195bSKonstantin Belousov int i, pages;
533*7a79d066SBojan Novković vm_object_t obj;
534ac04195bSKonstantin Belousov
535ac04195bSKonstantin Belousov cpu_thread_swapout(td);
536f13fa9dfSMark Johnston kaddr = td->td_kstack;
537ac04195bSKonstantin Belousov pages = td->td_kstack_pages;
538*7a79d066SBojan Novković obj = vm_thread_kstack_size_to_obj(pages);
539*7a79d066SBojan Novković pindex = vm_kstack_pindex(kaddr, pages);
540f13fa9dfSMark Johnston pmap_qremove(kaddr, pages);
541*7a79d066SBojan Novković VM_OBJECT_WLOCK(obj);
542ac04195bSKonstantin Belousov for (i = 0; i < pages; i++) {
543*7a79d066SBojan Novković m = vm_page_lookup(obj, pindex + i);
544ac04195bSKonstantin Belousov if (m == NULL)
545ac04195bSKonstantin Belousov panic("vm_thread_swapout: kstack already missing?");
546ac04195bSKonstantin Belousov vm_page_dirty(m);
547e9ceb9ddSJeff Roberson vm_page_xunbusy_unchecked(m);
54836ca312dSAlan Cox vm_page_unwire(m, PQ_LAUNDRY);
549ac04195bSKonstantin Belousov }
550*7a79d066SBojan Novković VM_OBJECT_WUNLOCK(obj);
551ac04195bSKonstantin Belousov }
552ac04195bSKonstantin Belousov
553ac04195bSKonstantin Belousov /*
554ac04195bSKonstantin Belousov * Bring the kernel stack for a specified thread back in.
555ac04195bSKonstantin Belousov */
556ac04195bSKonstantin Belousov static void
vm_thread_swapin(struct thread * td,int oom_alloc)557a70e9a13SKonstantin Belousov vm_thread_swapin(struct thread *td, int oom_alloc)
558ac04195bSKonstantin Belousov {
559ac04195bSKonstantin Belousov vm_page_t ma[KSTACK_MAX_PAGES];
560f13fa9dfSMark Johnston vm_offset_t kaddr;
561*7a79d066SBojan Novković vm_object_t obj;
562b8ebd99aSJohn Baldwin int a, count, i, j, pages, rv __diagused;
563ac04195bSKonstantin Belousov
564f13fa9dfSMark Johnston kaddr = td->td_kstack;
565ac04195bSKonstantin Belousov pages = td->td_kstack_pages;
566*7a79d066SBojan Novković obj = vm_thread_kstack_size_to_obj(pages);
567*7a79d066SBojan Novković while (vm_thread_stack_back(kaddr, ma, pages, oom_alloc,
568*7a79d066SBojan Novković td->td_kstack_domain) == ENOMEM)
569*7a79d066SBojan Novković ;
570e258b4a0SKonstantin Belousov for (i = 0; i < pages;) {
571ac04195bSKonstantin Belousov vm_page_assert_xbusied(ma[i]);
5720012f373SJeff Roberson if (vm_page_all_valid(ma[i])) {
573ac04195bSKonstantin Belousov i++;
574ac04195bSKonstantin Belousov continue;
575ac04195bSKonstantin Belousov }
576*7a79d066SBojan Novković vm_object_pip_add(obj, 1);
577ac04195bSKonstantin Belousov for (j = i + 1; j < pages; j++)
5780012f373SJeff Roberson if (vm_page_all_valid(ma[j]))
579ac04195bSKonstantin Belousov break;
580*7a79d066SBojan Novković VM_OBJECT_WLOCK(obj);
581*7a79d066SBojan Novković rv = vm_pager_has_page(obj, ma[i]->pindex, NULL, &a);
582*7a79d066SBojan Novković VM_OBJECT_WUNLOCK(obj);
583ac04195bSKonstantin Belousov KASSERT(rv == 1, ("%s: missing page %p", __func__, ma[i]));
584ac04195bSKonstantin Belousov count = min(a + 1, j - i);
585*7a79d066SBojan Novković rv = vm_pager_get_pages(obj, ma + i, count, NULL, NULL);
586ac04195bSKonstantin Belousov KASSERT(rv == VM_PAGER_OK, ("%s: cannot get kstack for proc %d",
587ac04195bSKonstantin Belousov __func__, td->td_proc->p_pid));
588*7a79d066SBojan Novković vm_object_pip_wakeup(obj);
589ac04195bSKonstantin Belousov i += count;
590ac04195bSKonstantin Belousov }
591f13fa9dfSMark Johnston pmap_qenter(kaddr, ma, pages);
592ac04195bSKonstantin Belousov cpu_thread_swapin(td);
593ac04195bSKonstantin Belousov }
594ac04195bSKonstantin Belousov
595ac04195bSKonstantin Belousov void
faultin(struct proc * p)596ac04195bSKonstantin Belousov faultin(struct proc *p)
597ac04195bSKonstantin Belousov {
598ac04195bSKonstantin Belousov struct thread *td;
599a70e9a13SKonstantin Belousov int oom_alloc;
600ac04195bSKonstantin Belousov
601ac04195bSKonstantin Belousov PROC_LOCK_ASSERT(p, MA_OWNED);
602a70e9a13SKonstantin Belousov
603ac04195bSKonstantin Belousov /*
604ac04195bSKonstantin Belousov * If another process is swapping in this process,
605ac04195bSKonstantin Belousov * just wait until it finishes.
606ac04195bSKonstantin Belousov */
607ac04195bSKonstantin Belousov if (p->p_flag & P_SWAPPINGIN) {
608ac04195bSKonstantin Belousov while (p->p_flag & P_SWAPPINGIN)
609ac04195bSKonstantin Belousov msleep(&p->p_flag, &p->p_mtx, PVM, "faultin", 0);
610ac04195bSKonstantin Belousov return;
611ac04195bSKonstantin Belousov }
612a70e9a13SKonstantin Belousov
613ac04195bSKonstantin Belousov if ((p->p_flag & P_INMEM) == 0) {
614a70e9a13SKonstantin Belousov oom_alloc = (p->p_flag & P_WKILLED) != 0 ? VM_ALLOC_SYSTEM :
615a70e9a13SKonstantin Belousov VM_ALLOC_NORMAL;
616a70e9a13SKonstantin Belousov
617ac04195bSKonstantin Belousov /*
618ac04195bSKonstantin Belousov * Don't let another thread swap process p out while we are
619ac04195bSKonstantin Belousov * busy swapping it in.
620ac04195bSKonstantin Belousov */
621ac04195bSKonstantin Belousov ++p->p_lock;
622ac04195bSKonstantin Belousov p->p_flag |= P_SWAPPINGIN;
623ac04195bSKonstantin Belousov PROC_UNLOCK(p);
624a70e9a13SKonstantin Belousov sx_xlock(&allproc_lock);
625a70e9a13SKonstantin Belousov MPASS(swapped_cnt > 0);
626a70e9a13SKonstantin Belousov swapped_cnt--;
627c1344d2bSKonstantin Belousov if (curthread != &thread0)
628c1344d2bSKonstantin Belousov swap_inprogress++;
629a70e9a13SKonstantin Belousov sx_xunlock(&allproc_lock);
630ac04195bSKonstantin Belousov
631ac04195bSKonstantin Belousov /*
632ac04195bSKonstantin Belousov * We hold no lock here because the list of threads
633ac04195bSKonstantin Belousov * can not change while all threads in the process are
634ac04195bSKonstantin Belousov * swapped out.
635ac04195bSKonstantin Belousov */
636ac04195bSKonstantin Belousov FOREACH_THREAD_IN_PROC(p, td)
637a70e9a13SKonstantin Belousov vm_thread_swapin(td, oom_alloc);
638a70e9a13SKonstantin Belousov
639c1344d2bSKonstantin Belousov if (curthread != &thread0) {
640c1344d2bSKonstantin Belousov sx_xlock(&allproc_lock);
641c1344d2bSKonstantin Belousov MPASS(swap_inprogress > 0);
642c1344d2bSKonstantin Belousov swap_inprogress--;
643c1344d2bSKonstantin Belousov last_swapin = ticks;
644c1344d2bSKonstantin Belousov sx_xunlock(&allproc_lock);
645c1344d2bSKonstantin Belousov }
646ac04195bSKonstantin Belousov PROC_LOCK(p);
647ac04195bSKonstantin Belousov swapclear(p);
648ac04195bSKonstantin Belousov p->p_swtick = ticks;
649ac04195bSKonstantin Belousov
650ac04195bSKonstantin Belousov /* Allow other threads to swap p out now. */
651a70e9a13SKonstantin Belousov wakeup(&p->p_flag);
652ac04195bSKonstantin Belousov --p->p_lock;
653ac04195bSKonstantin Belousov }
654ac04195bSKonstantin Belousov }
655ac04195bSKonstantin Belousov
656ac04195bSKonstantin Belousov /*
657ac04195bSKonstantin Belousov * This swapin algorithm attempts to swap-in processes only if there
658ac04195bSKonstantin Belousov * is enough space for them. Of course, if a process waits for a long
659ac04195bSKonstantin Belousov * time, it will be swapped in anyway.
660ac04195bSKonstantin Belousov */
661a70e9a13SKonstantin Belousov
662a70e9a13SKonstantin Belousov static struct proc *
swapper_selector(bool wkilled_only)663c1344d2bSKonstantin Belousov swapper_selector(bool wkilled_only)
664ac04195bSKonstantin Belousov {
665a70e9a13SKonstantin Belousov struct proc *p, *res;
666ac04195bSKonstantin Belousov struct thread *td;
667c1344d2bSKonstantin Belousov int ppri, pri, slptime, swtime;
668ac04195bSKonstantin Belousov
669a70e9a13SKonstantin Belousov sx_assert(&allproc_lock, SA_SLOCKED);
670a70e9a13SKonstantin Belousov if (swapped_cnt == 0)
671a70e9a13SKonstantin Belousov return (NULL);
672a70e9a13SKonstantin Belousov res = NULL;
673ac04195bSKonstantin Belousov ppri = INT_MIN;
674ac04195bSKonstantin Belousov FOREACH_PROC_IN_SYSTEM(p) {
675ac04195bSKonstantin Belousov PROC_LOCK(p);
676a70e9a13SKonstantin Belousov if (p->p_state == PRS_NEW || (p->p_flag & (P_SWAPPINGOUT |
677a70e9a13SKonstantin Belousov P_SWAPPINGIN | P_INMEM)) != 0) {
678a70e9a13SKonstantin Belousov PROC_UNLOCK(p);
679a70e9a13SKonstantin Belousov continue;
680a70e9a13SKonstantin Belousov }
681a70e9a13SKonstantin Belousov if (p->p_state == PRS_NORMAL && (p->p_flag & P_WKILLED) != 0) {
682a70e9a13SKonstantin Belousov /*
683a70e9a13SKonstantin Belousov * A swapped-out process might have mapped a
684a70e9a13SKonstantin Belousov * large portion of the system's pages as
685a70e9a13SKonstantin Belousov * anonymous memory. There is no other way to
686a70e9a13SKonstantin Belousov * release the memory other than to kill the
687a70e9a13SKonstantin Belousov * process, for which we need to swap it in.
688a70e9a13SKonstantin Belousov */
689a70e9a13SKonstantin Belousov return (p);
690a70e9a13SKonstantin Belousov }
691c1344d2bSKonstantin Belousov if (wkilled_only) {
692ac04195bSKonstantin Belousov PROC_UNLOCK(p);
693ac04195bSKonstantin Belousov continue;
694ac04195bSKonstantin Belousov }
695ac04195bSKonstantin Belousov swtime = (ticks - p->p_swtick) / hz;
696ac04195bSKonstantin Belousov FOREACH_THREAD_IN_PROC(p, td) {
697ac04195bSKonstantin Belousov /*
698ac04195bSKonstantin Belousov * An otherwise runnable thread of a process
699ac04195bSKonstantin Belousov * swapped out has only the TDI_SWAPPED bit set.
700ac04195bSKonstantin Belousov */
701ac04195bSKonstantin Belousov thread_lock(td);
702ac04195bSKonstantin Belousov if (td->td_inhibitors == TDI_SWAPPED) {
703ac04195bSKonstantin Belousov slptime = (ticks - td->td_slptick) / hz;
704ac04195bSKonstantin Belousov pri = swtime + slptime;
705ac04195bSKonstantin Belousov if ((td->td_flags & TDF_SWAPINREQ) == 0)
706ac04195bSKonstantin Belousov pri -= p->p_nice * 8;
707ac04195bSKonstantin Belousov /*
708ac04195bSKonstantin Belousov * if this thread is higher priority
709ac04195bSKonstantin Belousov * and there is enough space, then select
710ac04195bSKonstantin Belousov * this process instead of the previous
711ac04195bSKonstantin Belousov * selection.
712ac04195bSKonstantin Belousov */
713ac04195bSKonstantin Belousov if (pri > ppri) {
714a70e9a13SKonstantin Belousov res = p;
715ac04195bSKonstantin Belousov ppri = pri;
716ac04195bSKonstantin Belousov }
717ac04195bSKonstantin Belousov }
718ac04195bSKonstantin Belousov thread_unlock(td);
719ac04195bSKonstantin Belousov }
720ac04195bSKonstantin Belousov PROC_UNLOCK(p);
721ac04195bSKonstantin Belousov }
722c1344d2bSKonstantin Belousov
723a70e9a13SKonstantin Belousov if (res != NULL)
724a70e9a13SKonstantin Belousov PROC_LOCK(res);
725a70e9a13SKonstantin Belousov return (res);
726a70e9a13SKonstantin Belousov }
727a70e9a13SKonstantin Belousov
728c1344d2bSKonstantin Belousov #define SWAPIN_INTERVAL (MAXSLP * hz / 2)
729c1344d2bSKonstantin Belousov
730c1344d2bSKonstantin Belousov /*
731c1344d2bSKonstantin Belousov * Limit swapper to swap in one non-WKILLED process in MAXSLP/2
732c1344d2bSKonstantin Belousov * interval, assuming that there is:
7332a843ae7SMark Johnston * - at least one domain that is not suffering from a shortage of free memory;
734c1344d2bSKonstantin Belousov * - no parallel swap-ins;
735c1344d2bSKonstantin Belousov * - no other swap-ins in the current SWAPIN_INTERVAL.
736c1344d2bSKonstantin Belousov */
737c1344d2bSKonstantin Belousov static bool
swapper_wkilled_only(void)738c1344d2bSKonstantin Belousov swapper_wkilled_only(void)
739c1344d2bSKonstantin Belousov {
740c1344d2bSKonstantin Belousov
741b0058196SMark Johnston return (vm_page_count_min_set(&all_domains) || swap_inprogress > 0 ||
742c1344d2bSKonstantin Belousov (u_int)(ticks - last_swapin) < SWAPIN_INTERVAL);
743c1344d2bSKonstantin Belousov }
744c1344d2bSKonstantin Belousov
745a70e9a13SKonstantin Belousov void
swapper(void)746a70e9a13SKonstantin Belousov swapper(void)
747a70e9a13SKonstantin Belousov {
748a70e9a13SKonstantin Belousov struct proc *p;
749a70e9a13SKonstantin Belousov
750a70e9a13SKonstantin Belousov for (;;) {
751a70e9a13SKonstantin Belousov sx_slock(&allproc_lock);
752c1344d2bSKonstantin Belousov p = swapper_selector(swapper_wkilled_only());
753ac04195bSKonstantin Belousov sx_sunlock(&allproc_lock);
754ac04195bSKonstantin Belousov
755a70e9a13SKonstantin Belousov if (p == NULL) {
756c1344d2bSKonstantin Belousov tsleep(&proc0, PVM, "swapin", SWAPIN_INTERVAL);
757a70e9a13SKonstantin Belousov } else {
758a70e9a13SKonstantin Belousov PROC_LOCK_ASSERT(p, MA_OWNED);
759ac04195bSKonstantin Belousov
760ac04195bSKonstantin Belousov /*
761a70e9a13SKonstantin Belousov * Another process may be bringing or may have
762a70e9a13SKonstantin Belousov * already brought this process in while we
763a70e9a13SKonstantin Belousov * traverse all threads. Or, this process may
764a70e9a13SKonstantin Belousov * have exited or even being swapped out
765a70e9a13SKonstantin Belousov * again.
766ac04195bSKonstantin Belousov */
767a70e9a13SKonstantin Belousov if (p->p_state == PRS_NORMAL && (p->p_flag & (P_INMEM |
768a70e9a13SKonstantin Belousov P_SWAPPINGOUT | P_SWAPPINGIN)) == 0) {
769ac04195bSKonstantin Belousov faultin(p);
770a70e9a13SKonstantin Belousov }
771ac04195bSKonstantin Belousov PROC_UNLOCK(p);
772a70e9a13SKonstantin Belousov }
773a70e9a13SKonstantin Belousov }
774ac04195bSKonstantin Belousov }
775ac04195bSKonstantin Belousov
776ac04195bSKonstantin Belousov /*
777ac04195bSKonstantin Belousov * First, if any processes have been sleeping or stopped for at least
778ac04195bSKonstantin Belousov * "swap_idle_threshold1" seconds, they are swapped out. If, however,
779ac04195bSKonstantin Belousov * no such processes exist, then the longest-sleeping or stopped
780ac04195bSKonstantin Belousov * process is swapped out. Finally, and only as a last resort, if
781ac04195bSKonstantin Belousov * there are no sleeping or stopped processes, the longest-resident
782ac04195bSKonstantin Belousov * process is swapped out.
783ac04195bSKonstantin Belousov */
784ac04195bSKonstantin Belousov static void
swapout_procs(int action)785ac04195bSKonstantin Belousov swapout_procs(int action)
786ac04195bSKonstantin Belousov {
787ac04195bSKonstantin Belousov struct proc *p;
788ac04195bSKonstantin Belousov struct thread *td;
7897000c588SAlan Cox int slptime;
79033937731SKonstantin Belousov bool didswap, doswap;
79133937731SKonstantin Belousov
79233937731SKonstantin Belousov MPASS((action & (VM_SWAP_NORMAL | VM_SWAP_IDLE)) != 0);
793ac04195bSKonstantin Belousov
794e258b4a0SKonstantin Belousov didswap = false;
795ac04195bSKonstantin Belousov sx_slock(&allproc_lock);
796ac04195bSKonstantin Belousov FOREACH_PROC_IN_SYSTEM(p) {
79733937731SKonstantin Belousov /*
79833937731SKonstantin Belousov * Filter out not yet fully constructed processes. Do
79933937731SKonstantin Belousov * not swap out held processes. Avoid processes which
80033937731SKonstantin Belousov * are system, exiting, execing, traced, already swapped
80133937731SKonstantin Belousov * out or are in the process of being swapped in or out.
80233937731SKonstantin Belousov */
803ac04195bSKonstantin Belousov PROC_LOCK(p);
80433937731SKonstantin Belousov if (p->p_state != PRS_NORMAL || p->p_lock != 0 || (p->p_flag &
80533937731SKonstantin Belousov (P_SYSTEM | P_WEXIT | P_INEXEC | P_STOPPED_SINGLE |
80633937731SKonstantin Belousov P_TRACED | P_SWAPPINGOUT | P_SWAPPINGIN | P_INMEM)) !=
80733937731SKonstantin Belousov P_INMEM) {
808ac04195bSKonstantin Belousov PROC_UNLOCK(p);
809ac04195bSKonstantin Belousov continue;
810ac04195bSKonstantin Belousov }
81133937731SKonstantin Belousov
812ac04195bSKonstantin Belousov /*
81333937731SKonstantin Belousov * Further consideration of this process for swap out
81433937731SKonstantin Belousov * requires iterating over its threads. We release
81533937731SKonstantin Belousov * allproc_lock here so that process creation and
81633937731SKonstantin Belousov * destruction are not blocked while we iterate.
81733937731SKonstantin Belousov *
81833937731SKonstantin Belousov * To later reacquire allproc_lock and resume
81933937731SKonstantin Belousov * iteration over the allproc list, we will first have
82033937731SKonstantin Belousov * to release the lock on the process. We place a
82133937731SKonstantin Belousov * hold on the process so that it remains in the
82233937731SKonstantin Belousov * allproc list while it is unlocked.
823ac04195bSKonstantin Belousov */
824ac04195bSKonstantin Belousov _PHOLD_LITE(p);
825ac04195bSKonstantin Belousov sx_sunlock(&allproc_lock);
826ac04195bSKonstantin Belousov
827ac04195bSKonstantin Belousov /*
82833937731SKonstantin Belousov * Do not swapout a realtime process.
82933937731SKonstantin Belousov * Guarantee swap_idle_threshold1 time in memory.
83033937731SKonstantin Belousov * If the system is under memory stress, or if we are
83133937731SKonstantin Belousov * swapping idle processes >= swap_idle_threshold2,
832ac04195bSKonstantin Belousov * then swap the process out.
833ac04195bSKonstantin Belousov */
83433937731SKonstantin Belousov doswap = true;
83533937731SKonstantin Belousov FOREACH_THREAD_IN_PROC(p, td) {
83633937731SKonstantin Belousov thread_lock(td);
83733937731SKonstantin Belousov slptime = (ticks - td->td_slptick) / hz;
83833937731SKonstantin Belousov if (PRI_IS_REALTIME(td->td_pri_class) ||
83933937731SKonstantin Belousov slptime < swap_idle_threshold1 ||
84033937731SKonstantin Belousov !thread_safetoswapout(td) ||
84133937731SKonstantin Belousov ((action & VM_SWAP_NORMAL) == 0 &&
84233937731SKonstantin Belousov slptime < swap_idle_threshold2))
84333937731SKonstantin Belousov doswap = false;
844ac04195bSKonstantin Belousov thread_unlock(td);
84533937731SKonstantin Belousov if (!doswap)
84633937731SKonstantin Belousov break;
847ac04195bSKonstantin Belousov }
84833937731SKonstantin Belousov if (doswap && swapout(p) == 0)
849e258b4a0SKonstantin Belousov didswap = true;
85033937731SKonstantin Belousov
851ac04195bSKonstantin Belousov PROC_UNLOCK(p);
852a70e9a13SKonstantin Belousov if (didswap) {
853a70e9a13SKonstantin Belousov sx_xlock(&allproc_lock);
854a70e9a13SKonstantin Belousov swapped_cnt++;
855a70e9a13SKonstantin Belousov sx_downgrade(&allproc_lock);
856a70e9a13SKonstantin Belousov } else
857ac04195bSKonstantin Belousov sx_slock(&allproc_lock);
858ac04195bSKonstantin Belousov PRELE(p);
859ac04195bSKonstantin Belousov }
860ac04195bSKonstantin Belousov sx_sunlock(&allproc_lock);
86133937731SKonstantin Belousov
862ac04195bSKonstantin Belousov /*
863ac04195bSKonstantin Belousov * If we swapped something out, and another process needed memory,
864ac04195bSKonstantin Belousov * then wakeup the sched process.
865ac04195bSKonstantin Belousov */
866ac04195bSKonstantin Belousov if (didswap)
867ac04195bSKonstantin Belousov wakeup(&proc0);
868ac04195bSKonstantin Belousov }
869ac04195bSKonstantin Belousov
870ac04195bSKonstantin Belousov static void
swapclear(struct proc * p)871ac04195bSKonstantin Belousov swapclear(struct proc *p)
872ac04195bSKonstantin Belousov {
873ac04195bSKonstantin Belousov struct thread *td;
874ac04195bSKonstantin Belousov
875ac04195bSKonstantin Belousov PROC_LOCK_ASSERT(p, MA_OWNED);
876ac04195bSKonstantin Belousov
877ac04195bSKonstantin Belousov FOREACH_THREAD_IN_PROC(p, td) {
878ac04195bSKonstantin Belousov thread_lock(td);
879ac04195bSKonstantin Belousov td->td_flags |= TDF_INMEM;
880ac04195bSKonstantin Belousov td->td_flags &= ~TDF_SWAPINREQ;
881ac04195bSKonstantin Belousov TD_CLR_SWAPPED(td);
88261a74c5cSJeff Roberson if (TD_CAN_RUN(td)) {
88361a74c5cSJeff Roberson if (setrunnable(td, 0)) {
884ac04195bSKonstantin Belousov #ifdef INVARIANTS
885ac04195bSKonstantin Belousov /*
886ac04195bSKonstantin Belousov * XXX: We just cleared TDI_SWAPPED
887ac04195bSKonstantin Belousov * above and set TDF_INMEM, so this
888ac04195bSKonstantin Belousov * should never happen.
889ac04195bSKonstantin Belousov */
890ac04195bSKonstantin Belousov panic("not waking up swapper");
891ac04195bSKonstantin Belousov #endif
892ac04195bSKonstantin Belousov }
89361a74c5cSJeff Roberson } else
894ac04195bSKonstantin Belousov thread_unlock(td);
895ac04195bSKonstantin Belousov }
896ac04195bSKonstantin Belousov p->p_flag &= ~(P_SWAPPINGIN | P_SWAPPINGOUT);
897ac04195bSKonstantin Belousov p->p_flag |= P_INMEM;
898ac04195bSKonstantin Belousov }
899ac04195bSKonstantin Belousov
900ac04195bSKonstantin Belousov static int
swapout(struct proc * p)901ac04195bSKonstantin Belousov swapout(struct proc *p)
902ac04195bSKonstantin Belousov {
903ac04195bSKonstantin Belousov struct thread *td;
904ac04195bSKonstantin Belousov
905ac04195bSKonstantin Belousov PROC_LOCK_ASSERT(p, MA_OWNED);
906ac04195bSKonstantin Belousov
907ac04195bSKonstantin Belousov /*
908ac04195bSKonstantin Belousov * The states of this process and its threads may have changed
909ac04195bSKonstantin Belousov * by now. Assuming that there is only one pageout daemon thread,
910ac04195bSKonstantin Belousov * this process should still be in memory.
911ac04195bSKonstantin Belousov */
912ac04195bSKonstantin Belousov KASSERT((p->p_flag & (P_INMEM | P_SWAPPINGOUT | P_SWAPPINGIN)) ==
913ac04195bSKonstantin Belousov P_INMEM, ("swapout: lost a swapout race?"));
914ac04195bSKonstantin Belousov
915ac04195bSKonstantin Belousov /*
916e258b4a0SKonstantin Belousov * Remember the resident count.
917ac04195bSKonstantin Belousov */
918ac04195bSKonstantin Belousov p->p_vmspace->vm_swrss = vmspace_resident_count(p->p_vmspace);
919e258b4a0SKonstantin Belousov
920ac04195bSKonstantin Belousov /*
921ac04195bSKonstantin Belousov * Check and mark all threads before we proceed.
922ac04195bSKonstantin Belousov */
923ac04195bSKonstantin Belousov p->p_flag &= ~P_INMEM;
924ac04195bSKonstantin Belousov p->p_flag |= P_SWAPPINGOUT;
925ac04195bSKonstantin Belousov FOREACH_THREAD_IN_PROC(p, td) {
926ac04195bSKonstantin Belousov thread_lock(td);
927ac04195bSKonstantin Belousov if (!thread_safetoswapout(td)) {
928ac04195bSKonstantin Belousov thread_unlock(td);
929ac04195bSKonstantin Belousov swapclear(p);
930ac04195bSKonstantin Belousov return (EBUSY);
931ac04195bSKonstantin Belousov }
932ac04195bSKonstantin Belousov td->td_flags &= ~TDF_INMEM;
933ac04195bSKonstantin Belousov TD_SET_SWAPPED(td);
934ac04195bSKonstantin Belousov thread_unlock(td);
935ac04195bSKonstantin Belousov }
936ac04195bSKonstantin Belousov td = FIRST_THREAD_IN_PROC(p);
937ac04195bSKonstantin Belousov ++td->td_ru.ru_nswap;
938ac04195bSKonstantin Belousov PROC_UNLOCK(p);
939ac04195bSKonstantin Belousov
940ac04195bSKonstantin Belousov /*
941ac04195bSKonstantin Belousov * This list is stable because all threads are now prevented from
942ac04195bSKonstantin Belousov * running. The list is only modified in the context of a running
943ac04195bSKonstantin Belousov * thread in this process.
944ac04195bSKonstantin Belousov */
945ac04195bSKonstantin Belousov FOREACH_THREAD_IN_PROC(p, td)
946ac04195bSKonstantin Belousov vm_thread_swapout(td);
947ac04195bSKonstantin Belousov
948ac04195bSKonstantin Belousov PROC_LOCK(p);
949ac04195bSKonstantin Belousov p->p_flag &= ~P_SWAPPINGOUT;
950ac04195bSKonstantin Belousov p->p_swtick = ticks;
951ac04195bSKonstantin Belousov return (0);
952ac04195bSKonstantin Belousov }
953