1 /*-
2 * SPDX-License-Identifier: (BSD-4-Clause AND MIT-CMU)
3 *
4 * Copyright (c) 1991 Regents of the University of California.
5 * All rights reserved.
6 * Copyright (c) 1994 John S. Dyson
7 * All rights reserved.
8 * Copyright (c) 1994 David Greenman
9 * All rights reserved.
10 * Copyright (c) 2005 Yahoo! Technologies Norway AS
11 * All rights reserved.
12 *
13 * This code is derived from software contributed to Berkeley by
14 * The Mach Operating System project at Carnegie-Mellon University.
15 *
16 * Redistribution and use in source and binary forms, with or without
17 * modification, are permitted provided that the following conditions
18 * are met:
19 * 1. Redistributions of source code must retain the above copyright
20 * notice, this list of conditions and the following disclaimer.
21 * 2. Redistributions in binary form must reproduce the above copyright
22 * notice, this list of conditions and the following disclaimer in the
23 * documentation and/or other materials provided with the distribution.
24 * 3. All advertising materials mentioning features or use of this software
25 * must display the following acknowledgement:
26 * This product includes software developed by the University of
27 * California, Berkeley and its contributors.
28 * 4. Neither the name of the University nor the names of its contributors
29 * may be used to endorse or promote products derived from this software
30 * without specific prior written permission.
31 *
32 * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
33 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
34 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
35 * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
36 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
37 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
38 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
39 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
40 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
41 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
42 * SUCH DAMAGE.
43 *
44 *
45 * Copyright (c) 1987, 1990 Carnegie-Mellon University.
46 * All rights reserved.
47 *
48 * Authors: Avadis Tevanian, Jr., Michael Wayne Young
49 *
50 * Permission to use, copy, modify and distribute this software and
51 * its documentation is hereby granted, provided that both the copyright
52 * notice and this permission notice appear in all copies of the
53 * software, derivative works or modified versions, and any portions
54 * thereof, and that both notices appear in supporting documentation.
55 *
56 * CARNEGIE MELLON ALLOWS FREE USE OF THIS SOFTWARE IN ITS "AS IS"
57 * CONDITION. CARNEGIE MELLON DISCLAIMS ANY LIABILITY OF ANY KIND
58 * FOR ANY DAMAGES WHATSOEVER RESULTING FROM THE USE OF THIS SOFTWARE.
59 *
60 * Carnegie Mellon requests users of this software to return to
61 *
62 * Software Distribution Coordinator or Software.Distribution@CS.CMU.EDU
63 * School of Computer Science
64 * Carnegie Mellon University
65 * Pittsburgh PA 15213-3890
66 *
67 * any improvements or extensions that they make and grant Carnegie the
68 * rights to redistribute these changes.
69 */
70
71 #include <sys/cdefs.h>
72 #include "opt_kstack_pages.h"
73 #include "opt_kstack_max_pages.h"
74 #include "opt_vm.h"
75
76 #include <sys/param.h>
77 #include <sys/systm.h>
78 #include <sys/limits.h>
79 #include <sys/kernel.h>
80 #include <sys/eventhandler.h>
81 #include <sys/lock.h>
82 #include <sys/mutex.h>
83 #include <sys/proc.h>
84 #include <sys/kthread.h>
85 #include <sys/ktr.h>
86 #include <sys/mount.h>
87 #include <sys/racct.h>
88 #include <sys/resourcevar.h>
89 #include <sys/refcount.h>
90 #include <sys/sched.h>
91 #include <sys/sdt.h>
92 #include <sys/signalvar.h>
93 #include <sys/smp.h>
94 #include <sys/time.h>
95 #include <sys/vnode.h>
96 #include <sys/vmmeter.h>
97 #include <sys/rwlock.h>
98 #include <sys/sx.h>
99 #include <sys/sysctl.h>
100
101 #include <vm/vm.h>
102 #include <vm/vm_param.h>
103 #include <vm/vm_kern.h>
104 #include <vm/vm_object.h>
105 #include <vm/vm_page.h>
106 #include <vm/vm_map.h>
107 #include <vm/vm_pageout.h>
108 #include <vm/vm_pager.h>
109 #include <vm/vm_phys.h>
110 #include <vm/swap_pager.h>
111 #include <vm/vm_extern.h>
112 #include <vm/uma.h>
113
114 /* the kernel process "vm_daemon" */
115 static void vm_daemon(void);
116 static struct proc *vmproc;
117
118 static struct kproc_desc vm_kp = {
119 "vmdaemon",
120 vm_daemon,
121 &vmproc
122 };
123 SYSINIT(vmdaemon, SI_SUB_KTHREAD_VM, SI_ORDER_FIRST, kproc_start, &vm_kp);
124
125 static int vm_swap_enabled = 1;
126 static int vm_swap_idle_enabled = 0;
127
128 SYSCTL_INT(_vm, VM_SWAPPING_ENABLED, swap_enabled, CTLFLAG_RW,
129 &vm_swap_enabled, 0,
130 "Enable entire process swapout");
131 SYSCTL_INT(_vm, OID_AUTO, swap_idle_enabled, CTLFLAG_RW,
132 &vm_swap_idle_enabled, 0,
133 "Allow swapout on idle criteria");
134
135 /*
136 * Swap_idle_threshold1 is the guaranteed swapped in time for a process
137 */
138 static int swap_idle_threshold1 = 2;
139 SYSCTL_INT(_vm, OID_AUTO, swap_idle_threshold1, CTLFLAG_RW,
140 &swap_idle_threshold1, 0,
141 "Guaranteed swapped in time for a process");
142
143 /*
144 * Swap_idle_threshold2 is the time that a process can be idle before
145 * it will be swapped out, if idle swapping is enabled.
146 */
147 static int swap_idle_threshold2 = 10;
148 SYSCTL_INT(_vm, OID_AUTO, swap_idle_threshold2, CTLFLAG_RW,
149 &swap_idle_threshold2, 0,
150 "Time before a process will be swapped out");
151
152 static int vm_daemon_timeout = 0;
153 SYSCTL_INT(_vm, OID_AUTO, vmdaemon_timeout, CTLFLAG_RW,
154 &vm_daemon_timeout, 0,
155 "Time between vmdaemon runs");
156
157 static int vm_pageout_req_swapout; /* XXX */
158 static int vm_daemon_needed;
159 static struct mtx vm_daemon_mtx;
160 /* Allow for use by vm_pageout before vm_daemon is initialized. */
161 MTX_SYSINIT(vm_daemon, &vm_daemon_mtx, "vm daemon", MTX_DEF);
162
163 static int swapped_cnt;
164 static int swap_inprogress; /* Pending swap-ins done outside swapper. */
165 static int last_swapin;
166
167 static void swapclear(struct proc *);
168 static int swapout(struct proc *);
169 static void vm_swapout_map_deactivate_pages(vm_map_t, long);
170 static void vm_swapout_object_deactivate(pmap_t, vm_object_t, long);
171 static void swapout_procs(int action);
172 static void vm_req_vmdaemon(int req);
173 static void vm_thread_swapout(struct thread *td);
174
175 static void
vm_swapout_object_deactivate_page(pmap_t pmap,vm_page_t m,bool unmap)176 vm_swapout_object_deactivate_page(pmap_t pmap, vm_page_t m, bool unmap)
177 {
178
179 /*
180 * Ignore unreclaimable wired pages. Repeat the check after busying
181 * since a busy holder may wire the page.
182 */
183 if (vm_page_wired(m) || !vm_page_tryxbusy(m))
184 return;
185
186 if (vm_page_wired(m) || !pmap_page_exists_quick(pmap, m)) {
187 vm_page_xunbusy(m);
188 return;
189 }
190 if (!pmap_is_referenced(m)) {
191 if (!vm_page_active(m))
192 (void)vm_page_try_remove_all(m);
193 else if (unmap && vm_page_try_remove_all(m))
194 vm_page_deactivate(m);
195 }
196 vm_page_xunbusy(m);
197 }
198
199 /*
200 * vm_swapout_object_deactivate
201 *
202 * Deactivate enough pages to satisfy the inactive target
203 * requirements.
204 *
205 * The object and map must be locked.
206 */
207 static void
vm_swapout_object_deactivate(pmap_t pmap,vm_object_t first_object,long desired)208 vm_swapout_object_deactivate(pmap_t pmap, vm_object_t first_object,
209 long desired)
210 {
211 vm_object_t backing_object, object;
212 vm_page_t m;
213 bool unmap;
214
215 VM_OBJECT_ASSERT_LOCKED(first_object);
216 if ((first_object->flags & OBJ_FICTITIOUS) != 0)
217 return;
218 for (object = first_object;; object = backing_object) {
219 if (pmap_resident_count(pmap) <= desired)
220 goto unlock_return;
221 VM_OBJECT_ASSERT_LOCKED(object);
222 if ((object->flags & OBJ_UNMANAGED) != 0 ||
223 blockcount_read(&object->paging_in_progress) > 0)
224 goto unlock_return;
225
226 unmap = true;
227 if (object->shadow_count > 1)
228 unmap = false;
229
230 /*
231 * Scan the object's entire memory queue.
232 */
233 TAILQ_FOREACH(m, &object->memq, listq) {
234 if (pmap_resident_count(pmap) <= desired)
235 goto unlock_return;
236 if (should_yield())
237 goto unlock_return;
238 vm_swapout_object_deactivate_page(pmap, m, unmap);
239 }
240 if ((backing_object = object->backing_object) == NULL)
241 goto unlock_return;
242 VM_OBJECT_RLOCK(backing_object);
243 if (object != first_object)
244 VM_OBJECT_RUNLOCK(object);
245 }
246 unlock_return:
247 if (object != first_object)
248 VM_OBJECT_RUNLOCK(object);
249 }
250
251 /*
252 * deactivate some number of pages in a map, try to do it fairly, but
253 * that is really hard to do.
254 */
255 static void
vm_swapout_map_deactivate_pages(vm_map_t map,long desired)256 vm_swapout_map_deactivate_pages(vm_map_t map, long desired)
257 {
258 vm_map_entry_t tmpe;
259 vm_object_t obj, bigobj;
260 int nothingwired;
261
262 if (!vm_map_trylock_read(map))
263 return;
264
265 bigobj = NULL;
266 nothingwired = TRUE;
267
268 /*
269 * first, search out the biggest object, and try to free pages from
270 * that.
271 */
272 VM_MAP_ENTRY_FOREACH(tmpe, map) {
273 if ((tmpe->eflags & MAP_ENTRY_IS_SUB_MAP) == 0) {
274 obj = tmpe->object.vm_object;
275 if (obj != NULL && VM_OBJECT_TRYRLOCK(obj)) {
276 if (obj->shadow_count <= 1 &&
277 (bigobj == NULL ||
278 bigobj->resident_page_count <
279 obj->resident_page_count)) {
280 if (bigobj != NULL)
281 VM_OBJECT_RUNLOCK(bigobj);
282 bigobj = obj;
283 } else
284 VM_OBJECT_RUNLOCK(obj);
285 }
286 }
287 if (tmpe->wired_count > 0)
288 nothingwired = FALSE;
289 }
290
291 if (bigobj != NULL) {
292 vm_swapout_object_deactivate(map->pmap, bigobj, desired);
293 VM_OBJECT_RUNLOCK(bigobj);
294 }
295 /*
296 * Next, hunt around for other pages to deactivate. We actually
297 * do this search sort of wrong -- .text first is not the best idea.
298 */
299 VM_MAP_ENTRY_FOREACH(tmpe, map) {
300 if (pmap_resident_count(vm_map_pmap(map)) <= desired)
301 break;
302 if ((tmpe->eflags & MAP_ENTRY_IS_SUB_MAP) == 0) {
303 obj = tmpe->object.vm_object;
304 if (obj != NULL) {
305 VM_OBJECT_RLOCK(obj);
306 vm_swapout_object_deactivate(map->pmap, obj,
307 desired);
308 VM_OBJECT_RUNLOCK(obj);
309 }
310 }
311 }
312
313 /*
314 * Remove all mappings if a process is swapped out, this will free page
315 * table pages.
316 */
317 if (desired == 0 && nothingwired) {
318 pmap_remove(vm_map_pmap(map), vm_map_min(map),
319 vm_map_max(map));
320 }
321
322 vm_map_unlock_read(map);
323 }
324
325 /*
326 * Swap out requests
327 */
328 #define VM_SWAP_NORMAL 1
329 #define VM_SWAP_IDLE 2
330
331 void
vm_swapout_run(void)332 vm_swapout_run(void)
333 {
334
335 if (vm_swap_enabled)
336 vm_req_vmdaemon(VM_SWAP_NORMAL);
337 }
338
339 /*
340 * Idle process swapout -- run once per second when pagedaemons are
341 * reclaiming pages.
342 */
343 void
vm_swapout_run_idle(void)344 vm_swapout_run_idle(void)
345 {
346 static long lsec;
347
348 if (!vm_swap_idle_enabled || time_second == lsec)
349 return;
350 vm_req_vmdaemon(VM_SWAP_IDLE);
351 lsec = time_second;
352 }
353
354 static void
vm_req_vmdaemon(int req)355 vm_req_vmdaemon(int req)
356 {
357 static int lastrun = 0;
358
359 mtx_lock(&vm_daemon_mtx);
360 vm_pageout_req_swapout |= req;
361 if ((ticks > (lastrun + hz)) || (ticks < lastrun)) {
362 wakeup(&vm_daemon_needed);
363 lastrun = ticks;
364 }
365 mtx_unlock(&vm_daemon_mtx);
366 }
367
368 static void
vm_daemon(void)369 vm_daemon(void)
370 {
371 struct rlimit rsslim;
372 struct proc *p;
373 struct thread *td;
374 struct vmspace *vm;
375 int breakout, swapout_flags, tryagain, attempts;
376 #ifdef RACCT
377 uint64_t rsize, ravailable;
378
379 if (racct_enable && vm_daemon_timeout == 0)
380 vm_daemon_timeout = hz;
381 #endif
382
383 while (TRUE) {
384 mtx_lock(&vm_daemon_mtx);
385 msleep(&vm_daemon_needed, &vm_daemon_mtx, PPAUSE, "psleep",
386 vm_daemon_timeout);
387 swapout_flags = vm_pageout_req_swapout;
388 vm_pageout_req_swapout = 0;
389 mtx_unlock(&vm_daemon_mtx);
390 if (swapout_flags != 0) {
391 /*
392 * Drain the per-CPU page queue batches as a deadlock
393 * avoidance measure.
394 */
395 if ((swapout_flags & VM_SWAP_NORMAL) != 0)
396 vm_page_pqbatch_drain();
397 swapout_procs(swapout_flags);
398 }
399
400 /*
401 * scan the processes for exceeding their rlimits or if
402 * process is swapped out -- deactivate pages
403 */
404 tryagain = 0;
405 attempts = 0;
406 again:
407 attempts++;
408 sx_slock(&allproc_lock);
409 FOREACH_PROC_IN_SYSTEM(p) {
410 vm_pindex_t limit, size;
411
412 /*
413 * if this is a system process or if we have already
414 * looked at this process, skip it.
415 */
416 PROC_LOCK(p);
417 if (p->p_state != PRS_NORMAL ||
418 p->p_flag & (P_INEXEC | P_SYSTEM | P_WEXIT)) {
419 PROC_UNLOCK(p);
420 continue;
421 }
422 /*
423 * if the process is in a non-running type state,
424 * don't touch it.
425 */
426 breakout = 0;
427 FOREACH_THREAD_IN_PROC(p, td) {
428 thread_lock(td);
429 if (!TD_ON_RUNQ(td) &&
430 !TD_IS_RUNNING(td) &&
431 !TD_IS_SLEEPING(td) &&
432 !TD_IS_SUSPENDED(td)) {
433 thread_unlock(td);
434 breakout = 1;
435 break;
436 }
437 thread_unlock(td);
438 }
439 if (breakout) {
440 PROC_UNLOCK(p);
441 continue;
442 }
443 /*
444 * get a limit
445 */
446 lim_rlimit_proc(p, RLIMIT_RSS, &rsslim);
447 limit = OFF_TO_IDX(
448 qmin(rsslim.rlim_cur, rsslim.rlim_max));
449
450 /*
451 * let processes that are swapped out really be
452 * swapped out set the limit to nothing (will force a
453 * swap-out.)
454 */
455 if ((p->p_flag & P_INMEM) == 0)
456 limit = 0; /* XXX */
457 vm = vmspace_acquire_ref(p);
458 _PHOLD_LITE(p);
459 PROC_UNLOCK(p);
460 if (vm == NULL) {
461 PRELE(p);
462 continue;
463 }
464 sx_sunlock(&allproc_lock);
465
466 size = vmspace_resident_count(vm);
467 if (size >= limit) {
468 vm_swapout_map_deactivate_pages(
469 &vm->vm_map, limit);
470 size = vmspace_resident_count(vm);
471 }
472 #ifdef RACCT
473 if (racct_enable) {
474 rsize = IDX_TO_OFF(size);
475 PROC_LOCK(p);
476 if (p->p_state == PRS_NORMAL)
477 racct_set(p, RACCT_RSS, rsize);
478 ravailable = racct_get_available(p, RACCT_RSS);
479 PROC_UNLOCK(p);
480 if (rsize > ravailable) {
481 /*
482 * Don't be overly aggressive; this
483 * might be an innocent process,
484 * and the limit could've been exceeded
485 * by some memory hog. Don't try
486 * to deactivate more than 1/4th
487 * of process' resident set size.
488 */
489 if (attempts <= 8) {
490 if (ravailable < rsize -
491 (rsize / 4)) {
492 ravailable = rsize -
493 (rsize / 4);
494 }
495 }
496 vm_swapout_map_deactivate_pages(
497 &vm->vm_map,
498 OFF_TO_IDX(ravailable));
499 /* Update RSS usage after paging out. */
500 size = vmspace_resident_count(vm);
501 rsize = IDX_TO_OFF(size);
502 PROC_LOCK(p);
503 if (p->p_state == PRS_NORMAL)
504 racct_set(p, RACCT_RSS, rsize);
505 PROC_UNLOCK(p);
506 if (rsize > ravailable)
507 tryagain = 1;
508 }
509 }
510 #endif
511 vmspace_free(vm);
512 sx_slock(&allproc_lock);
513 PRELE(p);
514 }
515 sx_sunlock(&allproc_lock);
516 if (tryagain != 0 && attempts <= 10) {
517 maybe_yield();
518 goto again;
519 }
520 }
521 }
522
523 /*
524 * Allow a thread's kernel stack to be paged out.
525 */
526 static void
vm_thread_swapout(struct thread * td)527 vm_thread_swapout(struct thread *td)
528 {
529 vm_page_t m;
530 vm_offset_t kaddr;
531 vm_pindex_t pindex;
532 int i, pages;
533 vm_object_t obj;
534
535 cpu_thread_swapout(td);
536 kaddr = td->td_kstack;
537 pages = td->td_kstack_pages;
538 obj = vm_thread_kstack_size_to_obj(pages);
539 pindex = vm_kstack_pindex(kaddr, pages);
540 pmap_qremove(kaddr, pages);
541 VM_OBJECT_WLOCK(obj);
542 for (i = 0; i < pages; i++) {
543 m = vm_page_lookup(obj, pindex + i);
544 if (m == NULL)
545 panic("vm_thread_swapout: kstack already missing?");
546 vm_page_dirty(m);
547 vm_page_xunbusy_unchecked(m);
548 vm_page_unwire(m, PQ_LAUNDRY);
549 }
550 VM_OBJECT_WUNLOCK(obj);
551 }
552
553 /*
554 * Bring the kernel stack for a specified thread back in.
555 */
556 static void
vm_thread_swapin(struct thread * td,int oom_alloc)557 vm_thread_swapin(struct thread *td, int oom_alloc)
558 {
559 vm_page_t ma[KSTACK_MAX_PAGES];
560 vm_offset_t kaddr;
561 vm_object_t obj;
562 int a, count, i, j, pages, rv __diagused;
563
564 kaddr = td->td_kstack;
565 pages = td->td_kstack_pages;
566 obj = vm_thread_kstack_size_to_obj(pages);
567 while (vm_thread_stack_back(kaddr, ma, pages, oom_alloc,
568 td->td_kstack_domain) == ENOMEM)
569 ;
570 for (i = 0; i < pages;) {
571 vm_page_assert_xbusied(ma[i]);
572 if (vm_page_all_valid(ma[i])) {
573 i++;
574 continue;
575 }
576 vm_object_pip_add(obj, 1);
577 for (j = i + 1; j < pages; j++)
578 if (vm_page_all_valid(ma[j]))
579 break;
580 VM_OBJECT_WLOCK(obj);
581 rv = vm_pager_has_page(obj, ma[i]->pindex, NULL, &a);
582 VM_OBJECT_WUNLOCK(obj);
583 KASSERT(rv == 1, ("%s: missing page %p", __func__, ma[i]));
584 count = min(a + 1, j - i);
585 rv = vm_pager_get_pages(obj, ma + i, count, NULL, NULL);
586 KASSERT(rv == VM_PAGER_OK, ("%s: cannot get kstack for proc %d",
587 __func__, td->td_proc->p_pid));
588 vm_object_pip_wakeup(obj);
589 i += count;
590 }
591 pmap_qenter(kaddr, ma, pages);
592 cpu_thread_swapin(td);
593 }
594
595 void
faultin(struct proc * p)596 faultin(struct proc *p)
597 {
598 struct thread *td;
599 int oom_alloc;
600
601 PROC_LOCK_ASSERT(p, MA_OWNED);
602
603 /*
604 * If another process is swapping in this process,
605 * just wait until it finishes.
606 */
607 if (p->p_flag & P_SWAPPINGIN) {
608 while (p->p_flag & P_SWAPPINGIN)
609 msleep(&p->p_flag, &p->p_mtx, PVM, "faultin", 0);
610 return;
611 }
612
613 if ((p->p_flag & P_INMEM) == 0) {
614 oom_alloc = (p->p_flag & P_WKILLED) != 0 ? VM_ALLOC_SYSTEM :
615 VM_ALLOC_NORMAL;
616
617 /*
618 * Don't let another thread swap process p out while we are
619 * busy swapping it in.
620 */
621 ++p->p_lock;
622 p->p_flag |= P_SWAPPINGIN;
623 PROC_UNLOCK(p);
624 sx_xlock(&allproc_lock);
625 MPASS(swapped_cnt > 0);
626 swapped_cnt--;
627 if (curthread != &thread0)
628 swap_inprogress++;
629 sx_xunlock(&allproc_lock);
630
631 /*
632 * We hold no lock here because the list of threads
633 * can not change while all threads in the process are
634 * swapped out.
635 */
636 FOREACH_THREAD_IN_PROC(p, td)
637 vm_thread_swapin(td, oom_alloc);
638
639 if (curthread != &thread0) {
640 sx_xlock(&allproc_lock);
641 MPASS(swap_inprogress > 0);
642 swap_inprogress--;
643 last_swapin = ticks;
644 sx_xunlock(&allproc_lock);
645 }
646 PROC_LOCK(p);
647 swapclear(p);
648 p->p_swtick = ticks;
649
650 /* Allow other threads to swap p out now. */
651 wakeup(&p->p_flag);
652 --p->p_lock;
653 }
654 }
655
656 /*
657 * This swapin algorithm attempts to swap-in processes only if there
658 * is enough space for them. Of course, if a process waits for a long
659 * time, it will be swapped in anyway.
660 */
661
662 static struct proc *
swapper_selector(bool wkilled_only)663 swapper_selector(bool wkilled_only)
664 {
665 struct proc *p, *res;
666 struct thread *td;
667 int ppri, pri, slptime, swtime;
668
669 sx_assert(&allproc_lock, SA_SLOCKED);
670 if (swapped_cnt == 0)
671 return (NULL);
672 res = NULL;
673 ppri = INT_MIN;
674 FOREACH_PROC_IN_SYSTEM(p) {
675 PROC_LOCK(p);
676 if (p->p_state == PRS_NEW || (p->p_flag & (P_SWAPPINGOUT |
677 P_SWAPPINGIN | P_INMEM)) != 0) {
678 PROC_UNLOCK(p);
679 continue;
680 }
681 if (p->p_state == PRS_NORMAL && (p->p_flag & P_WKILLED) != 0) {
682 /*
683 * A swapped-out process might have mapped a
684 * large portion of the system's pages as
685 * anonymous memory. There is no other way to
686 * release the memory other than to kill the
687 * process, for which we need to swap it in.
688 */
689 return (p);
690 }
691 if (wkilled_only) {
692 PROC_UNLOCK(p);
693 continue;
694 }
695 swtime = (ticks - p->p_swtick) / hz;
696 FOREACH_THREAD_IN_PROC(p, td) {
697 /*
698 * An otherwise runnable thread of a process
699 * swapped out has only the TDI_SWAPPED bit set.
700 */
701 thread_lock(td);
702 if (td->td_inhibitors == TDI_SWAPPED) {
703 slptime = (ticks - td->td_slptick) / hz;
704 pri = swtime + slptime;
705 if ((td->td_flags & TDF_SWAPINREQ) == 0)
706 pri -= p->p_nice * 8;
707 /*
708 * if this thread is higher priority
709 * and there is enough space, then select
710 * this process instead of the previous
711 * selection.
712 */
713 if (pri > ppri) {
714 res = p;
715 ppri = pri;
716 }
717 }
718 thread_unlock(td);
719 }
720 PROC_UNLOCK(p);
721 }
722
723 if (res != NULL)
724 PROC_LOCK(res);
725 return (res);
726 }
727
728 #define SWAPIN_INTERVAL (MAXSLP * hz / 2)
729
730 /*
731 * Limit swapper to swap in one non-WKILLED process in MAXSLP/2
732 * interval, assuming that there is:
733 * - at least one domain that is not suffering from a shortage of free memory;
734 * - no parallel swap-ins;
735 * - no other swap-ins in the current SWAPIN_INTERVAL.
736 */
737 static bool
swapper_wkilled_only(void)738 swapper_wkilled_only(void)
739 {
740
741 return (vm_page_count_min_set(&all_domains) || swap_inprogress > 0 ||
742 (u_int)(ticks - last_swapin) < SWAPIN_INTERVAL);
743 }
744
745 void
swapper(void)746 swapper(void)
747 {
748 struct proc *p;
749
750 for (;;) {
751 sx_slock(&allproc_lock);
752 p = swapper_selector(swapper_wkilled_only());
753 sx_sunlock(&allproc_lock);
754
755 if (p == NULL) {
756 tsleep(&proc0, PVM, "swapin", SWAPIN_INTERVAL);
757 } else {
758 PROC_LOCK_ASSERT(p, MA_OWNED);
759
760 /*
761 * Another process may be bringing or may have
762 * already brought this process in while we
763 * traverse all threads. Or, this process may
764 * have exited or even being swapped out
765 * again.
766 */
767 if (p->p_state == PRS_NORMAL && (p->p_flag & (P_INMEM |
768 P_SWAPPINGOUT | P_SWAPPINGIN)) == 0) {
769 faultin(p);
770 }
771 PROC_UNLOCK(p);
772 }
773 }
774 }
775
776 /*
777 * First, if any processes have been sleeping or stopped for at least
778 * "swap_idle_threshold1" seconds, they are swapped out. If, however,
779 * no such processes exist, then the longest-sleeping or stopped
780 * process is swapped out. Finally, and only as a last resort, if
781 * there are no sleeping or stopped processes, the longest-resident
782 * process is swapped out.
783 */
784 static void
swapout_procs(int action)785 swapout_procs(int action)
786 {
787 struct proc *p;
788 struct thread *td;
789 int slptime;
790 bool didswap, doswap;
791
792 MPASS((action & (VM_SWAP_NORMAL | VM_SWAP_IDLE)) != 0);
793
794 didswap = false;
795 sx_slock(&allproc_lock);
796 FOREACH_PROC_IN_SYSTEM(p) {
797 /*
798 * Filter out not yet fully constructed processes. Do
799 * not swap out held processes. Avoid processes which
800 * are system, exiting, execing, traced, already swapped
801 * out or are in the process of being swapped in or out.
802 */
803 PROC_LOCK(p);
804 if (p->p_state != PRS_NORMAL || p->p_lock != 0 || (p->p_flag &
805 (P_SYSTEM | P_WEXIT | P_INEXEC | P_STOPPED_SINGLE |
806 P_TRACED | P_SWAPPINGOUT | P_SWAPPINGIN | P_INMEM)) !=
807 P_INMEM) {
808 PROC_UNLOCK(p);
809 continue;
810 }
811
812 /*
813 * Further consideration of this process for swap out
814 * requires iterating over its threads. We release
815 * allproc_lock here so that process creation and
816 * destruction are not blocked while we iterate.
817 *
818 * To later reacquire allproc_lock and resume
819 * iteration over the allproc list, we will first have
820 * to release the lock on the process. We place a
821 * hold on the process so that it remains in the
822 * allproc list while it is unlocked.
823 */
824 _PHOLD_LITE(p);
825 sx_sunlock(&allproc_lock);
826
827 /*
828 * Do not swapout a realtime process.
829 * Guarantee swap_idle_threshold1 time in memory.
830 * If the system is under memory stress, or if we are
831 * swapping idle processes >= swap_idle_threshold2,
832 * then swap the process out.
833 */
834 doswap = true;
835 FOREACH_THREAD_IN_PROC(p, td) {
836 thread_lock(td);
837 slptime = (ticks - td->td_slptick) / hz;
838 if (PRI_IS_REALTIME(td->td_pri_class) ||
839 slptime < swap_idle_threshold1 ||
840 !thread_safetoswapout(td) ||
841 ((action & VM_SWAP_NORMAL) == 0 &&
842 slptime < swap_idle_threshold2))
843 doswap = false;
844 thread_unlock(td);
845 if (!doswap)
846 break;
847 }
848 if (doswap && swapout(p) == 0)
849 didswap = true;
850
851 PROC_UNLOCK(p);
852 if (didswap) {
853 sx_xlock(&allproc_lock);
854 swapped_cnt++;
855 sx_downgrade(&allproc_lock);
856 } else
857 sx_slock(&allproc_lock);
858 PRELE(p);
859 }
860 sx_sunlock(&allproc_lock);
861
862 /*
863 * If we swapped something out, and another process needed memory,
864 * then wakeup the sched process.
865 */
866 if (didswap)
867 wakeup(&proc0);
868 }
869
870 static void
swapclear(struct proc * p)871 swapclear(struct proc *p)
872 {
873 struct thread *td;
874
875 PROC_LOCK_ASSERT(p, MA_OWNED);
876
877 FOREACH_THREAD_IN_PROC(p, td) {
878 thread_lock(td);
879 td->td_flags |= TDF_INMEM;
880 td->td_flags &= ~TDF_SWAPINREQ;
881 TD_CLR_SWAPPED(td);
882 if (TD_CAN_RUN(td)) {
883 if (setrunnable(td, 0)) {
884 #ifdef INVARIANTS
885 /*
886 * XXX: We just cleared TDI_SWAPPED
887 * above and set TDF_INMEM, so this
888 * should never happen.
889 */
890 panic("not waking up swapper");
891 #endif
892 }
893 } else
894 thread_unlock(td);
895 }
896 p->p_flag &= ~(P_SWAPPINGIN | P_SWAPPINGOUT);
897 p->p_flag |= P_INMEM;
898 }
899
900 static int
swapout(struct proc * p)901 swapout(struct proc *p)
902 {
903 struct thread *td;
904
905 PROC_LOCK_ASSERT(p, MA_OWNED);
906
907 /*
908 * The states of this process and its threads may have changed
909 * by now. Assuming that there is only one pageout daemon thread,
910 * this process should still be in memory.
911 */
912 KASSERT((p->p_flag & (P_INMEM | P_SWAPPINGOUT | P_SWAPPINGIN)) ==
913 P_INMEM, ("swapout: lost a swapout race?"));
914
915 /*
916 * Remember the resident count.
917 */
918 p->p_vmspace->vm_swrss = vmspace_resident_count(p->p_vmspace);
919
920 /*
921 * Check and mark all threads before we proceed.
922 */
923 p->p_flag &= ~P_INMEM;
924 p->p_flag |= P_SWAPPINGOUT;
925 FOREACH_THREAD_IN_PROC(p, td) {
926 thread_lock(td);
927 if (!thread_safetoswapout(td)) {
928 thread_unlock(td);
929 swapclear(p);
930 return (EBUSY);
931 }
932 td->td_flags &= ~TDF_INMEM;
933 TD_SET_SWAPPED(td);
934 thread_unlock(td);
935 }
936 td = FIRST_THREAD_IN_PROC(p);
937 ++td->td_ru.ru_nswap;
938 PROC_UNLOCK(p);
939
940 /*
941 * This list is stable because all threads are now prevented from
942 * running. The list is only modified in the context of a running
943 * thread in this process.
944 */
945 FOREACH_THREAD_IN_PROC(p, td)
946 vm_thread_swapout(td);
947
948 PROC_LOCK(p);
949 p->p_flag &= ~P_SWAPPINGOUT;
950 p->p_swtick = ticks;
951 return (0);
952 }
953