124 static void vm_pageout_mightbe_oom(struct vm_domain *vmd, int page_shortage,
206 static int vm_pageout_launder(struct vm_domain *vmd, int launder,
717 vm_pageout_launder(struct vm_domain *vmd, int launder, bool in_shortfall)  in vm_pageout_launder()  argument
747 	marker = &vmd->vmd_markers[queue];  in vm_pageout_launder()
748 	pq = &vmd->vmd_pagequeues[queue];  in vm_pageout_launder()
984 	struct vm_domain *vmd;  in vm_pageout_laundry_worker()  local
991 	vmd = VM_DOMAIN(domain);  in vm_pageout_laundry_worker()
992 	pq = &vmd->vmd_pagequeues[PQ_LAUNDRY];  in vm_pageout_laundry_worker()
993 	KASSERT(vmd->vmd_segs != 0, ("domain without segments"));  in vm_pageout_laundry_worker()
1004 	(void)EVENTHANDLER_REGISTER(swapon, vm_pageout_swapon, vmd,  in vm_pageout_laundry_worker()
1006 	(void)EVENTHANDLER_REGISTER(swapoff, vm_pageout_swapoff, vmd,  in vm_pageout_laundry_worker()
1028 		else if (shortfall_cycle == 0 || vm_laundry_target(vmd) <= 0) {  in vm_pageout_laundry_worker()
1062 		nclean = vmd->vmd_free_count +  in vm_pageout_laundry_worker()
1063 		    vmd->vmd_pagequeues[PQ_INACTIVE].pq_cnt;  in vm_pageout_laundry_worker()
1064 		ndirty = vmd->vmd_pagequeues[PQ_LAUNDRY].pq_cnt;  in vm_pageout_laundry_worker()
1066 		    vmd->vmd_free_target - vmd->vmd_free_min)) >= nclean) {  in vm_pageout_laundry_worker()
1067 			target = vmd->vmd_background_launder_target;  in vm_pageout_laundry_worker()
1099 			target -= min(vm_pageout_launder(vmd, launder,  in vm_pageout_laundry_worker()
1110 		if (target == 0 && vmd->vmd_laundry_request == VM_LAUNDRY_IDLE)  in vm_pageout_laundry_worker()
1111 			(void)mtx_sleep(&vmd->vmd_laundry_request,  in vm_pageout_laundry_worker()
1119 		if (vmd->vmd_laundry_request == VM_LAUNDRY_SHORTFALL &&  in vm_pageout_laundry_worker()
1121 			shortfall = vm_laundry_target(vmd) +  in vm_pageout_laundry_worker()
1122 			    vmd->vmd_pageout_deficit;  in vm_pageout_laundry_worker()
1128 			vmd->vmd_laundry_request = VM_LAUNDRY_IDLE;  in vm_pageout_laundry_worker()
1129 		nfreed += vmd->vmd_clean_pages_freed;  in vm_pageout_laundry_worker()
1130 		vmd->vmd_clean_pages_freed = 0;  in vm_pageout_laundry_worker()
1148 vm_pageout_active_target(struct vm_domain *vmd)  in vm_pageout_active_target()  argument
1152 	shortage = vmd->vmd_inactive_target + vm_paging_target(vmd) -  in vm_pageout_active_target()
1153 	    (vmd->vmd_pagequeues[PQ_INACTIVE].pq_cnt +  in vm_pageout_active_target()
1154 	    vmd->vmd_pagequeues[PQ_LAUNDRY].pq_cnt / act_scan_laundry_weight);  in vm_pageout_active_target()
1164 vm_pageout_scan_active(struct vm_domain *vmd, int page_shortage)  in vm_pageout_scan_active()  argument
1175 	marker = &vmd->vmd_markers[PQ_ACTIVE];  in vm_pageout_scan_active()
1176 	pq = &vmd->vmd_pagequeues[PQ_ACTIVE];  in vm_pageout_scan_active()
1186 		min_scan *= scan_tick - vmd->vmd_last_active_scan;  in vm_pageout_scan_active()
1191 		vmd->vmd_last_active_scan = scan_tick;  in vm_pageout_scan_active()
1208 	vm_pageout_init_scan(&ss, pq, marker, &vmd->vmd_clock[0], max_scan);  in vm_pageout_scan_active()
1210 		if (__predict_false(m == &vmd->vmd_clock[1])) {  in vm_pageout_scan_active()
1212 			TAILQ_REMOVE(&pq->pq_pl, &vmd->vmd_clock[0], plinks.q);  in vm_pageout_scan_active()
1213 			TAILQ_REMOVE(&pq->pq_pl, &vmd->vmd_clock[1], plinks.q);  in vm_pageout_scan_active()
1214 			TAILQ_INSERT_HEAD(&pq->pq_pl, &vmd->vmd_clock[0],  in vm_pageout_scan_active()
1216 			TAILQ_INSERT_TAIL(&pq->pq_pl, &vmd->vmd_clock[1],  in vm_pageout_scan_active()
1363 	TAILQ_REMOVE(&pq->pq_pl, &vmd->vmd_clock[0], plinks.q);  in vm_pageout_scan_active()
1364 	TAILQ_INSERT_AFTER(&pq->pq_pl, marker, &vmd->vmd_clock[0], plinks.q);  in vm_pageout_scan_active()
1418 vm_pageout_scan_inactive(struct vm_domain *vmd, int page_shortage)  in vm_pageout_scan_inactive()  argument
1451 	pq = &vmd->vmd_pagequeues[PQ_INACTIVE];  in vm_pageout_scan_inactive()
1633 	atomic_add_int(&vmd->vmd_addl_shortage, addl_page_shortage);  in vm_pageout_scan_inactive()
1636 	atomic_add_int(&vmd->vmd_inactive_us,  in vm_pageout_scan_inactive()
1638 	atomic_add_int(&vmd->vmd_inactive_freed,  in vm_pageout_scan_inactive()
1647 vm_pageout_inactive_dispatch(struct vm_domain *vmd, int shortage)  in vm_pageout_inactive_dispatch()  argument
1651 	vmd->vmd_inactive_shortage = shortage;  in vm_pageout_inactive_dispatch()
1658 	threads = vmd->vmd_inactive_threads;  in vm_pageout_inactive_dispatch()
1659 	if (threads > 1 && vmd->vmd_inactive_pps != 0 &&  in vm_pageout_inactive_dispatch()
1660 	    shortage > vmd->vmd_inactive_pps / VM_INACT_SCAN_RATE / 4) {  in vm_pageout_inactive_dispatch()
1661 		vmd->vmd_inactive_shortage /= threads;  in vm_pageout_inactive_dispatch()
1663 		vm_domain_pageout_lock(vmd);  in vm_pageout_inactive_dispatch()
1664 		blockcount_acquire(&vmd->vmd_inactive_starting, threads - 1);  in vm_pageout_inactive_dispatch()
1665 		blockcount_acquire(&vmd->vmd_inactive_running, threads - 1);  in vm_pageout_inactive_dispatch()
1666 		wakeup(&vmd->vmd_inactive_shortage);  in vm_pageout_inactive_dispatch()
1667 		vm_domain_pageout_unlock(vmd);  in vm_pageout_inactive_dispatch()
1671 	vm_pageout_scan_inactive(vmd, vmd->vmd_inactive_shortage + slop);  in vm_pageout_inactive_dispatch()
1677 	blockcount_wait(&vmd->vmd_inactive_running, NULL, "vmpoid", PVM);  in vm_pageout_inactive_dispatch()
1678 	freed = atomic_readandclear_int(&vmd->vmd_inactive_freed);  in vm_pageout_inactive_dispatch()
1686 	us = max(atomic_readandclear_int(&vmd->vmd_inactive_us), 1);  in vm_pageout_inactive_dispatch()
1692 	vmd->vmd_inactive_pps = (vmd->vmd_inactive_pps / 2) + (pps / 2);  in vm_pageout_inactive_dispatch()
1702 vm_pageout_inactive(struct vm_domain *vmd, int shortage, int *addl_shortage)  in vm_pageout_inactive()  argument
1714 	deficit = atomic_readandclear_int(&vmd->vmd_pageout_deficit);  in vm_pageout_inactive()
1720 	page_shortage = vm_pageout_inactive_dispatch(vmd, starting_page_shortage);  in vm_pageout_inactive()
1721 	addl_page_shortage = atomic_readandclear_int(&vmd->vmd_addl_shortage);  in vm_pageout_inactive()
1735 		pq = &vmd->vmd_pagequeues[PQ_LAUNDRY];  in vm_pageout_inactive()
1737 		if (vmd->vmd_laundry_request == VM_LAUNDRY_IDLE &&  in vm_pageout_inactive()
1740 				vmd->vmd_laundry_request = VM_LAUNDRY_SHORTFALL;  in vm_pageout_inactive()
1742 			} else if (vmd->vmd_laundry_request !=  in vm_pageout_inactive()
1744 				vmd->vmd_laundry_request =  in vm_pageout_inactive()
1746 			wakeup(&vmd->vmd_laundry_request);  in vm_pageout_inactive()
1748 		vmd->vmd_clean_pages_freed +=  in vm_pageout_inactive()
1764 	vm_pageout_mightbe_oom(vmd, page_shortage, starting_page_shortage);  in vm_pageout_inactive()
1787 vm_pageout_mightbe_oom(struct vm_domain *vmd, int page_shortage,  in vm_pageout_mightbe_oom()  argument
1794 		vmd->vmd_oom_seq = 0;  in vm_pageout_mightbe_oom()
1796 		vmd->vmd_oom_seq++;  in vm_pageout_mightbe_oom()
1797 	if (vmd->vmd_oom_seq < vm_pageout_oom_seq) {  in vm_pageout_mightbe_oom()
1798 		if (vmd->vmd_oom) {  in vm_pageout_mightbe_oom()
1799 			vmd->vmd_oom = FALSE;  in vm_pageout_mightbe_oom()
1809 	vmd->vmd_oom_seq = 0;  in vm_pageout_mightbe_oom()
1811 	if (vmd->vmd_oom)  in vm_pageout_mightbe_oom()
1814 	vmd->vmd_oom = TRUE;  in vm_pageout_mightbe_oom()
1832 	vmd->vmd_oom = FALSE;  in vm_pageout_mightbe_oom()
2090 	struct vm_domain *vmd;  in vm_pageout_worker()  local
2096 	vmd = VM_DOMAIN(domain);  in vm_pageout_worker()
2106 	KASSERT(vmd->vmd_segs != 0, ("domain without segments"));  in vm_pageout_worker()
2107 	vmd->vmd_last_active_scan = ticks;  in vm_pageout_worker()
2113 		vm_domain_pageout_lock(vmd);  in vm_pageout_worker()
2120 		atomic_store_int(&vmd->vmd_pageout_wanted, 0);  in vm_pageout_worker()
2125 		if (vm_paging_needed(vmd, vmd->vmd_free_count)) {  in vm_pageout_worker()
2131 			vm_domain_pageout_unlock(vmd);  in vm_pageout_worker()
2139 			if (mtx_sleep(&vmd->vmd_pageout_wanted,  in vm_pageout_worker()
2140 			    vm_domain_pageout_lockptr(vmd), PDROP | PVM,  in vm_pageout_worker()
2146 		atomic_store_int(&vmd->vmd_pageout_wanted, 1);  in vm_pageout_worker()
2154 		shortage = pidctrl_daemon(&vmd->vmd_pid, vmd->vmd_free_count);  in vm_pageout_worker()
2156 			ofree = vmd->vmd_free_count;  in vm_pageout_worker()
2157 			if (vm_pageout_lowmem() && vmd->vmd_free_count > ofree)  in vm_pageout_worker()
2158 				shortage -= min(vmd->vmd_free_count - ofree,  in vm_pageout_worker()
2160 			target_met = vm_pageout_inactive(vmd, shortage,  in vm_pageout_worker()
2170 		shortage = vm_pageout_active_target(vmd) + addl_shortage;  in vm_pageout_worker()
2171 		vm_pageout_scan_active(vmd, shortage);  in vm_pageout_worker()
2182 	struct vm_domain *vmd;  in vm_pageout_helper()  local
2186 	vmd = VM_DOMAIN(domain);  in vm_pageout_helper()
2188 	vm_domain_pageout_lock(vmd);  in vm_pageout_helper()
2190 		msleep(&vmd->vmd_inactive_shortage,  in vm_pageout_helper()
2191 		    vm_domain_pageout_lockptr(vmd), PVM, "psleep", 0);  in vm_pageout_helper()
2192 		blockcount_release(&vmd->vmd_inactive_starting, 1);  in vm_pageout_helper()
2194 		vm_domain_pageout_unlock(vmd);  in vm_pageout_helper()
2195 		vm_pageout_scan_inactive(vmd, vmd->vmd_inactive_shortage);  in vm_pageout_helper()
2196 		vm_domain_pageout_lock(vmd);  in vm_pageout_helper()
2202 		blockcount_release(&vmd->vmd_inactive_running, 1);  in vm_pageout_helper()
2207 get_pageout_threads_per_domain(const struct vm_domain *vmd)  in get_pageout_threads_per_domain()  argument
2211 	if (VM_DOMAIN_EMPTY(vmd->vmd_domain))  in get_pageout_threads_per_domain()
2224 	domain_cpus = CPU_COUNT(&cpuset_domain[vmd->vmd_domain]);  in get_pageout_threads_per_domain()
2249 	struct vm_domain *vmd;  in vm_pageout_init_domain()  local
2252 	vmd = VM_DOMAIN(domain);  in vm_pageout_init_domain()
2253 	vmd->vmd_interrupt_free_min = 2;  in vm_pageout_init_domain()
2260 	vmd->vmd_pageout_free_min = 2 * MAXBSIZE / PAGE_SIZE +  in vm_pageout_init_domain()
2261 	    vmd->vmd_interrupt_free_min;  in vm_pageout_init_domain()
2262 	vmd->vmd_free_reserved = vm_pageout_page_count +  in vm_pageout_init_domain()
2263 	    vmd->vmd_pageout_free_min + vmd->vmd_page_count / 768;  in vm_pageout_init_domain()
2264 	vmd->vmd_free_min = vmd->vmd_page_count / 200;  in vm_pageout_init_domain()
2265 	vmd->vmd_free_severe = vmd->vmd_free_min / 2;  in vm_pageout_init_domain()
2266 	vmd->vmd_free_target = 4 * vmd->vmd_free_min + vmd->vmd_free_reserved;  in vm_pageout_init_domain()
2267 	vmd->vmd_free_min += vmd->vmd_free_reserved;  in vm_pageout_init_domain()
2268 	vmd->vmd_free_severe += vmd->vmd_free_reserved;  in vm_pageout_init_domain()
2269 	vmd->vmd_inactive_target = (3 * vmd->vmd_free_target) / 2;  in vm_pageout_init_domain()
2270 	if (vmd->vmd_inactive_target > vmd->vmd_free_count / 3)  in vm_pageout_init_domain()
2271 		vmd->vmd_inactive_target = vmd->vmd_free_count / 3;  in vm_pageout_init_domain()
2277 	vmd->vmd_pageout_wakeup_thresh = (vmd->vmd_free_target / 10) * 9;  in vm_pageout_init_domain()
2284 	vmd->vmd_background_launder_target = (vmd->vmd_free_target -  in vm_pageout_init_domain()
2285 	    vmd->vmd_free_min) / 10;  in vm_pageout_init_domain()
2288 	pidctrl_init(&vmd->vmd_pid, hz / VM_INACT_SCAN_RATE,  in vm_pageout_init_domain()
2289 	    vmd->vmd_free_target, PIDCTRL_BOUND,  in vm_pageout_init_domain()
2291 	oid = SYSCTL_ADD_NODE(NULL, SYSCTL_CHILDREN(vmd->vmd_oid), OID_AUTO,  in vm_pageout_init_domain()
2293 	pidctrl_init_sysctl(&vmd->vmd_pid, SYSCTL_CHILDREN(oid));  in vm_pageout_init_domain()
2295 	vmd->vmd_inactive_threads = get_pageout_threads_per_domain(vmd);  in vm_pageout_init_domain()
2312 		struct vm_domain *vmd;  in vm_pageout_init()  local
2315 		vmd = VM_DOMAIN(i);  in vm_pageout_init()
2316 		vm_cnt.v_free_reserved += vmd->vmd_free_reserved;  in vm_pageout_init()
2317 		vm_cnt.v_free_target += vmd->vmd_free_target;  in vm_pageout_init()
2318 		vm_cnt.v_free_min += vmd->vmd_free_min;  in vm_pageout_init()
2319 		vm_cnt.v_inactive_target += vmd->vmd_inactive_target;  in vm_pageout_init()
2320 		vm_cnt.v_pageout_free_min += vmd->vmd_pageout_free_min;  in vm_pageout_init()
2321 		vm_cnt.v_interrupt_free_min += vmd->vmd_interrupt_free_min;  in vm_pageout_init()
2322 		vm_cnt.v_free_severe += vmd->vmd_free_severe;  in vm_pageout_init()
2323 		freecount += vmd->vmd_free_count;  in vm_pageout_init()
2402 	struct vm_domain *vmd;  in pagedaemon_wakeup()  local
2404 	vmd = VM_DOMAIN(domain);  in pagedaemon_wakeup()
2405 	vm_domain_pageout_assert_unlocked(vmd);  in pagedaemon_wakeup()
2409 	if (atomic_fetchadd_int(&vmd->vmd_pageout_wanted, 1) == 0) {  in pagedaemon_wakeup()
2410 		vm_domain_pageout_lock(vmd);  in pagedaemon_wakeup()
2411 		atomic_store_int(&vmd->vmd_pageout_wanted, 1);  in pagedaemon_wakeup()
2412 		wakeup(&vmd->vmd_pageout_wanted);  in pagedaemon_wakeup()
2413 		vm_domain_pageout_unlock(vmd);  in pagedaemon_wakeup()