428 static void	swap_pager_dealloc(vm_object_t object);
435 		swap_pager_haspage(vm_object_t object, vm_pindex_t pindex, int *before, int *after);
439 static void	swap_pager_update_writecount(vm_object_t object,
441 static void	swap_pager_release_writecount(vm_object_t object,
443 static void	swap_pager_freespace_pgo(vm_object_t object, vm_pindex_t start,
671 swap_pager_init_object(vm_object_t object, void *handle, struct ucred *cred,  in swap_pager_init_object()  argument
680 	object->un_pager.swp.writemappings = 0;  in swap_pager_init_object()
681 	object->handle = handle;  in swap_pager_init_object()
683 		object->cred = cred;  in swap_pager_init_object()
684 		object->charge = size;  in swap_pager_init_object()
693 	vm_object_t object;  in swap_pager_alloc_init()  local
700 	object = vm_object_allocate(otype, OFF_TO_IDX(offset +  in swap_pager_alloc_init()
703 	if (!swap_pager_init_object(object, handle, cred, size, offset)) {  in swap_pager_alloc_init()
704 		vm_object_deallocate(object);  in swap_pager_alloc_init()
707 	return (object);  in swap_pager_alloc_init()
725 	vm_object_t object;  in swap_pager_alloc()  local
735 		object = vm_pager_object_lookup(NOBJLIST(handle), handle);  in swap_pager_alloc()
736 		if (object == NULL) {  in swap_pager_alloc()
737 			object = swap_pager_alloc_init(OBJT_SWAP, handle, cred,  in swap_pager_alloc()
739 			if (object != NULL) {  in swap_pager_alloc()
740 				TAILQ_INSERT_TAIL(NOBJLIST(object->handle),  in swap_pager_alloc()
741 				    object, pager_object_list);  in swap_pager_alloc()
746 		object = swap_pager_alloc_init(OBJT_SWAP, handle, cred,  in swap_pager_alloc()
749 	return (object);  in swap_pager_alloc()
763 swap_pager_dealloc(vm_object_t object)  in swap_pager_dealloc()  argument
766 	VM_OBJECT_ASSERT_WLOCKED(object);  in swap_pager_dealloc()
767 	KASSERT((object->flags & OBJ_DEAD) != 0, ("dealloc of reachable obj"));  in swap_pager_dealloc()
773 	if ((object->flags & OBJ_ANON) == 0 && object->handle != NULL) {  in swap_pager_dealloc()
774 		VM_OBJECT_WUNLOCK(object);  in swap_pager_dealloc()
776 		TAILQ_REMOVE(NOBJLIST(object->handle), object,  in swap_pager_dealloc()
779 		VM_OBJECT_WLOCK(object);  in swap_pager_dealloc()
782 	vm_object_pip_wait(object, "swpdea");  in swap_pager_dealloc()
790 	swp_pager_meta_free_all(object);  in swap_pager_dealloc()
791 	object->handle = NULL;  in swap_pager_dealloc()
792 	object->type = OBJT_DEAD;  in swap_pager_dealloc()
797 	if (object->cred != NULL) {  in swap_pager_dealloc()
798 		swap_release_by_cred(object->charge, object->cred);  in swap_pager_dealloc()
799 		object->charge = 0;  in swap_pager_dealloc()
800 		crfree(object->cred);  in swap_pager_dealloc()
801 		object->cred = NULL;  in swap_pager_dealloc()
807 	vm_object_clear_flag(object, OBJ_SWAP);  in swap_pager_dealloc()
990 swap_pager_freespace(vm_object_t object, vm_pindex_t start, vm_size_t size,  in swap_pager_freespace()  argument
993 	MPASS((object->flags & OBJ_SWAP) != 0);  in swap_pager_freespace()
995 	swp_pager_meta_free(object, start, size, freed);  in swap_pager_freespace()
999 swap_pager_freespace_pgo(vm_object_t object, vm_pindex_t start, vm_size_t size)  in swap_pager_freespace_pgo()  argument
1001 	MPASS((object->flags & OBJ_SWAP) != 0);  in swap_pager_freespace_pgo()
1003 	swp_pager_meta_free(object, start, size, NULL);  in swap_pager_freespace_pgo()
1015 swap_pager_reserve(vm_object_t object, vm_pindex_t start, vm_pindex_t size)  in swap_pager_reserve()  argument
1023 	VM_OBJECT_WLOCK(object);  in swap_pager_reserve()
1028 			swp_pager_meta_free(object, start, i, NULL);  in swap_pager_reserve()
1029 			VM_OBJECT_WUNLOCK(object);  in swap_pager_reserve()
1033 			addr = swp_pager_meta_build(object,  in swap_pager_reserve()
1040 	VM_OBJECT_WUNLOCK(object);  in swap_pager_reserve()
1136 swap_pager_haspage(vm_object_t object, vm_pindex_t pindex, int *before,  in swap_pager_haspage()  argument
1142 	VM_OBJECT_ASSERT_LOCKED(object);  in swap_pager_haspage()
1143 	KASSERT((object->flags & OBJ_SWAP) != 0,  in swap_pager_haspage()
1149 	blk0 = swp_pager_meta_lookup(object, pindex);  in swap_pager_haspage()
1165 			blk = swp_pager_meta_lookup(object, pindex - i);  in swap_pager_haspage()
1177 			blk = swp_pager_meta_lookup(object, pindex + i);  in swap_pager_haspage()
1216 	obj = m->object;  in swap_pager_unswapped()
1240 	KASSERT((m->object->flags & OBJ_SWAP) != 0,  in swap_pager_unswapped()
1243 	sb = SWAP_PCTRIE_LOOKUP(&m->object->un_pager.swp.swp_blks,  in swap_pager_unswapped()
1253 	swp_pager_free_empty_swblk(m->object, sb);  in swap_pager_unswapped()
1268 swap_pager_getpages_locked(vm_object_t object, vm_page_t *ma, int count,  in swap_pager_getpages_locked()  argument
1277 	VM_OBJECT_ASSERT_WLOCKED(object);  in swap_pager_getpages_locked()
1280 	KASSERT((object->flags & OBJ_SWAP) != 0,  in swap_pager_getpages_locked()
1282 	if (!swap_pager_haspage(object, ma[0]->pindex, &maxbehind, &maxahead)) {  in swap_pager_getpages_locked()
1283 		VM_OBJECT_WUNLOCK(object);  in swap_pager_getpages_locked()
1296 	if ((object->flags & (OBJ_SPLIT | OBJ_DEAD)) != 0) {  in swap_pager_getpages_locked()
1328 			p = vm_page_alloc(object, ma[0]->pindex - i,  in swap_pager_getpages_locked()
1339 			p = vm_page_alloc(object,  in swap_pager_getpages_locked()
1352 	vm_object_pip_add(object, count);  in swap_pager_getpages_locked()
1355 	blk = swp_pager_meta_lookup(object, pindex);  in swap_pager_getpages_locked()
1357 	    ("no swap blocking containing %p(%jx)", object, (uintmax_t)pindex));  in swap_pager_getpages_locked()
1359 	VM_OBJECT_WUNLOCK(object);  in swap_pager_getpages_locked()
1402 	VM_OBJECT_WLOCK(object);  in swap_pager_getpages_locked()
1407 		if (VM_OBJECT_SLEEP(object, &object->handle, PSWP,  in swap_pager_getpages_locked()
1414 	VM_OBJECT_WUNLOCK(object);  in swap_pager_getpages_locked()
1434 swap_pager_getpages(vm_object_t object, vm_page_t *ma, int count,  in swap_pager_getpages()  argument
1438 	VM_OBJECT_WLOCK(object);  in swap_pager_getpages()
1439 	return (swap_pager_getpages_locked(object, ma, count, rbehind, rahead));  in swap_pager_getpages()
1449 swap_pager_getpages_async(vm_object_t object, vm_page_t *ma, int count,  in swap_pager_getpages_async()  argument
1454 	r = swap_pager_getpages(object, ma, count, rbehind, rahead);  in swap_pager_getpages_async()
1493 swap_pager_putpages(vm_object_t object, vm_page_t *ma, int count,  in swap_pager_putpages()  argument
1503 	KASSERT(count == 0 || ma[0]->object == object,  in swap_pager_putpages()
1505 	    __func__, object, ma[0]->object));  in swap_pager_putpages()
1507 	VM_OBJECT_WUNLOCK(object);  in swap_pager_putpages()
1540 		VM_OBJECT_WLOCK(object);  in swap_pager_putpages()
1544 			addr = swp_pager_meta_build(mreq->object, mreq->pindex,  in swap_pager_putpages()
1551 		VM_OBJECT_WUNLOCK(object);  in swap_pager_putpages()
1620 	VM_OBJECT_WLOCK(object);  in swap_pager_putpages()
1635 	vm_object_t object = NULL;  in swp_pager_async_iodone()  local
1661 		object = bp->b_pages[0]->object;  in swp_pager_async_iodone()
1662 		VM_OBJECT_WLOCK(object);  in swp_pager_async_iodone()
1679 			wakeup(&object->handle);  in swp_pager_async_iodone()
1755 	if (object != NULL) {  in swp_pager_async_iodone()
1756 		vm_object_pip_wakeupn(object, bp->b_npages);  in swp_pager_async_iodone()
1757 		VM_OBJECT_WUNLOCK(object);  in swp_pager_async_iodone()
1798 swap_pager_swapped_pages(vm_object_t object)  in swap_pager_swapped_pages()  argument
1805 	VM_OBJECT_ASSERT_LOCKED(object);  in swap_pager_swapped_pages()
1807 	if (pctrie_is_empty(&object->un_pager.swp.swp_blks))  in swap_pager_swapped_pages()
1811 	    &object->un_pager.swp.swp_blks, pi)) != NULL;  in swap_pager_swapped_pages()
1828 swap_pager_swapoff_object(struct swdevt *sp, vm_object_t object)  in swap_pager_swapoff_object()  argument
1836 	KASSERT((object->flags & OBJ_SWAP) != 0,  in swap_pager_swapoff_object()
1840 	    &object->un_pager.swp.swp_blks, pi)) != NULL; ) {  in swap_pager_swapoff_object()
1841 		if ((object->flags & OBJ_DEAD) != 0) {  in swap_pager_swapoff_object()
1846 			vm_object_pip_wait(object, "swpoff");  in swap_pager_swapoff_object()
1847 			swp_pager_meta_free_all(object);  in swap_pager_swapoff_object()
1873 			m = vm_page_lookup(object, sb->p + i);  in swap_pager_swapoff_object()
1875 				m = vm_page_alloc(object, sb->p + i,  in swap_pager_swapoff_object()
1882 					VM_OBJECT_SLEEP(object, &object->handle,  in swap_pager_swapoff_object()
1899 			vm_object_pip_add(object, 1);  in swap_pager_swapoff_object()
1901 			rv = swap_pager_getpages_locked(object, &m, 1, NULL,  in swap_pager_swapoff_object()
1906 			VM_OBJECT_WLOCK(object);  in swap_pager_swapoff_object()
1907 			vm_object_pip_wakeupn(object, 1);  in swap_pager_swapoff_object()
1935 	vm_object_t object;  in swap_pager_swapoff()  local
1943 	TAILQ_FOREACH(object, &vm_object_list, object_list) {  in swap_pager_swapoff()
1944 		if ((object->flags & OBJ_SWAP) == 0)  in swap_pager_swapoff()
1948 		VM_OBJECT_WLOCK(object);  in swap_pager_swapoff()
1953 		if ((object->flags & OBJ_DEAD) != 0)  in swap_pager_swapoff()
1963 		if ((object->flags & OBJ_SWAP) == 0)  in swap_pager_swapoff()
1966 		swap_pager_swapoff_object(sp, object);  in swap_pager_swapoff()
1968 		VM_OBJECT_WUNLOCK(object);  in swap_pager_swapoff()
2025 swp_pager_free_empty_swblk(vm_object_t object, struct swblk *sb)  in swp_pager_free_empty_swblk()  argument
2029 		SWAP_PCTRIE_REMOVE(&object->un_pager.swp.swp_blks, sb->p);  in swp_pager_free_empty_swblk()
2042 swp_pager_meta_build(vm_object_t object, vm_pindex_t pindex, daddr_t swapblk)  in swp_pager_meta_build()  argument
2050 	VM_OBJECT_ASSERT_WLOCKED(object);  in swp_pager_meta_build()
2053 	sb = SWAP_PCTRIE_LOOKUP(&object->un_pager.swp.swp_blks, rdpi);  in swp_pager_meta_build()
2069 			VM_OBJECT_WUNLOCK(object);  in swp_pager_meta_build()
2079 			VM_OBJECT_WLOCK(object);  in swp_pager_meta_build()
2080 			sb = SWAP_PCTRIE_LOOKUP(&object->un_pager.swp.swp_blks,  in swp_pager_meta_build()
2092 			    &object->un_pager.swp.swp_blks, sb);  in swp_pager_meta_build()
2099 			VM_OBJECT_WUNLOCK(object);  in swp_pager_meta_build()
2109 			VM_OBJECT_WLOCK(object);  in swp_pager_meta_build()
2110 			sb1 = SWAP_PCTRIE_LOOKUP(&object->un_pager.swp.swp_blks,  in swp_pager_meta_build()
2132 		swp_pager_free_empty_swblk(object, sb);  in swp_pager_meta_build()
2216 swp_pager_meta_free(vm_object_t object, vm_pindex_t pindex, vm_pindex_t count,  in swp_pager_meta_free()  argument
2219 	swp_pager_meta_transfer(object, NULL, pindex, count, freed);  in swp_pager_meta_free()
2241 swp_pager_meta_free_all(vm_object_t object)  in swp_pager_meta_free_all()  argument
2245 	VM_OBJECT_ASSERT_WLOCKED(object);  in swp_pager_meta_free_all()
2248 	SWAP_PCTRIE_RECLAIM_CALLBACK(&object->un_pager.swp.swp_blks,  in swp_pager_meta_free_all()
2265 swp_pager_meta_lookup(vm_object_t object, vm_pindex_t pindex)  in swp_pager_meta_lookup()  argument
2269 	VM_OBJECT_ASSERT_LOCKED(object);  in swp_pager_meta_lookup()
2275 	KASSERT((object->flags & OBJ_SWAP) != 0,  in swp_pager_meta_lookup()
2278 	sb = SWAP_PCTRIE_LOOKUP(&object->un_pager.swp.swp_blks,  in swp_pager_meta_lookup()
2293 swap_pager_find_least(vm_object_t object, vm_pindex_t pindex)  in swap_pager_find_least()  argument
2298 	VM_OBJECT_ASSERT_LOCKED(object);  in swap_pager_find_least()
2299 	MPASS((object->flags & OBJ_SWAP) != 0);  in swap_pager_find_least()
2301 	if (pctrie_is_empty(&object->un_pager.swp.swp_blks))  in swap_pager_find_least()
2302 		return (object->size);  in swap_pager_find_least()
2303 	sb = SWAP_PCTRIE_LOOKUP_GE(&object->un_pager.swp.swp_blks,  in swap_pager_find_least()
2306 		return (object->size);  in swap_pager_find_least()
2312 		sb = SWAP_PCTRIE_LOOKUP_GE(&object->un_pager.swp.swp_blks,  in swap_pager_find_least()
2315 			return (object->size);  in swap_pager_find_least()
2327 	return (object->size);  in swap_pager_find_least()
2759 	vm_object_t object;  in vmspace_swap_count()  local
2771 		object = cur->object.vm_object;  in vmspace_swap_count()
2772 		if (object == NULL || (object->flags & OBJ_SWAP) == 0)  in vmspace_swap_count()
2774 		VM_OBJECT_RLOCK(object);  in vmspace_swap_count()
2775 		if ((object->flags & OBJ_SWAP) == 0)  in vmspace_swap_count()
2781 			    &object->un_pager.swp.swp_blks, pi);  in vmspace_swap_count()
2791 		VM_OBJECT_RUNLOCK(object);  in vmspace_swap_count()
3151 swap_pager_update_writecount(vm_object_t object, vm_offset_t start,  in swap_pager_update_writecount()  argument
3155 	VM_OBJECT_WLOCK(object);  in swap_pager_update_writecount()
3156 	KASSERT((object->flags & OBJ_ANON) == 0,  in swap_pager_update_writecount()
3158 	object->un_pager.swp.writemappings += (vm_ooffset_t)end - start;  in swap_pager_update_writecount()
3159 	VM_OBJECT_WUNLOCK(object);  in swap_pager_update_writecount()
3163 swap_pager_release_writecount(vm_object_t object, vm_offset_t start,  in swap_pager_release_writecount()  argument
3167 	VM_OBJECT_WLOCK(object);  in swap_pager_release_writecount()
3168 	KASSERT((object->flags & OBJ_ANON) == 0,  in swap_pager_release_writecount()
3170 	KASSERT(object->un_pager.swp.writemappings >= (vm_ooffset_t)end - start,  in swap_pager_release_writecount()
3171 	    ("swap obj %p writecount %jx dec %jx", object,  in swap_pager_release_writecount()
3172 	    (uintmax_t)object->un_pager.swp.writemappings,  in swap_pager_release_writecount()
3174 	object->un_pager.swp.writemappings -= (vm_ooffset_t)end - start;  in swap_pager_release_writecount()
3175 	VM_OBJECT_WUNLOCK(object);  in swap_pager_release_writecount()