64 vm_domainset_iter_init(struct vm_domainset_iter *di, struct domainset *ds,  in vm_domainset_iter_init()  argument
68 	di->di_domain = ds;  in vm_domainset_iter_init()
69 	di->di_iter = iter;  in vm_domainset_iter_init()
70 	di->di_policy = ds->ds_policy;  in vm_domainset_iter_init()
71 	DOMAINSET_COPY(&ds->ds_mask, &di->di_valid_mask);  in vm_domainset_iter_init()
72 	if (di->di_policy == DOMAINSET_POLICY_INTERLEAVE) {  in vm_domainset_iter_init()
90 		di->di_offset = pindex;  in vm_domainset_iter_init()
93 	di->di_minskip = true;  in vm_domainset_iter_init()
97 vm_domainset_iter_rr(struct vm_domainset_iter *di, int *domain)  in vm_domainset_iter_rr()  argument
100 	*domain = di->di_domain->ds_order[  in vm_domainset_iter_rr()
101 	    ++(*di->di_iter) % di->di_domain->ds_cnt];  in vm_domainset_iter_rr()
105 vm_domainset_iter_prefer(struct vm_domainset_iter *di, int *domain)  in vm_domainset_iter_prefer()  argument
110 		d = di->di_domain->ds_order[  in vm_domainset_iter_prefer()
111 		    ++(*di->di_iter) % di->di_domain->ds_cnt];  in vm_domainset_iter_prefer()
112 	} while (d == di->di_domain->ds_prefer);  in vm_domainset_iter_prefer()
117 vm_domainset_iter_interleave(struct vm_domainset_iter *di, int *domain)  in vm_domainset_iter_interleave()  argument
121 	d = di->di_offset % di->di_domain->ds_cnt;  in vm_domainset_iter_interleave()
122 	*di->di_iter = d;  in vm_domainset_iter_interleave()
123 	*domain = di->di_domain->ds_order[d];  in vm_domainset_iter_interleave()
127 vm_domainset_iter_next(struct vm_domainset_iter *di, int *domain)  in vm_domainset_iter_next()  argument
130 	KASSERT(di->di_n > 0,  in vm_domainset_iter_next()
131 	    ("vm_domainset_iter_first: Invalid n %d", di->di_n));  in vm_domainset_iter_next()
132 	switch (di->di_policy) {  in vm_domainset_iter_next()
142 		vm_domainset_iter_rr(di, domain);  in vm_domainset_iter_next()
145 		vm_domainset_iter_prefer(di, domain);  in vm_domainset_iter_next()
149 		    di->di_policy);  in vm_domainset_iter_next()
156 vm_domainset_iter_first(struct vm_domainset_iter *di, int *domain)  in vm_domainset_iter_first()  argument
159 	switch (di->di_policy) {  in vm_domainset_iter_first()
162 		if (DOMAINSET_ISSET(*domain, &di->di_valid_mask)) {  in vm_domainset_iter_first()
167 			di->di_n = di->di_domain->ds_cnt + 1;  in vm_domainset_iter_first()
176 		di->di_n = di->di_domain->ds_cnt;  in vm_domainset_iter_first()
177 		vm_domainset_iter_rr(di, domain);  in vm_domainset_iter_first()
180 		*domain = di->di_domain->ds_prefer;  in vm_domainset_iter_first()
181 		di->di_n = di->di_domain->ds_cnt;  in vm_domainset_iter_first()
184 		vm_domainset_iter_interleave(di, domain);  in vm_domainset_iter_first()
185 		di->di_n = di->di_domain->ds_cnt;  in vm_domainset_iter_first()
189 		    di->di_policy);  in vm_domainset_iter_first()
191 	KASSERT(di->di_n > 0,  in vm_domainset_iter_first()
192 	    ("vm_domainset_iter_first: Invalid n %d", di->di_n));  in vm_domainset_iter_first()
198 vm_domainset_iter_page_init(struct vm_domainset_iter *di, struct vm_object *obj,  in vm_domainset_iter_page_init()  argument
212 	vm_domainset_iter_init(di, dr->dr_policy, &dr->dr_iter, obj, pindex);  in vm_domainset_iter_page_init()
213 	di->di_flags = *req;  in vm_domainset_iter_page_init()
214 	*req = (di->di_flags & ~(VM_ALLOC_WAITOK | VM_ALLOC_WAITFAIL)) |  in vm_domainset_iter_page_init()
216 	vm_domainset_iter_first(di, domain);  in vm_domainset_iter_page_init()
218 		vm_domainset_iter_page(di, obj, domain);  in vm_domainset_iter_page_init()
222 vm_domainset_iter_page(struct vm_domainset_iter *di, struct vm_object *obj,  in vm_domainset_iter_page()  argument
225 	if (__predict_false(DOMAINSET_EMPTY(&di->di_valid_mask)))  in vm_domainset_iter_page()
229 	while (--di->di_n != 0) {  in vm_domainset_iter_page()
230 		vm_domainset_iter_next(di, domain);  in vm_domainset_iter_page()
231 		if (DOMAINSET_ISSET(*domain, &di->di_valid_mask) &&  in vm_domainset_iter_page()
232 		    (!di->di_minskip || !vm_page_count_min_domain(*domain)))  in vm_domainset_iter_page()
237 	if (di->di_minskip) {  in vm_domainset_iter_page()
238 		di->di_minskip = false;  in vm_domainset_iter_page()
239 		vm_domainset_iter_first(di, domain);  in vm_domainset_iter_page()
244 	if ((di->di_flags & (VM_ALLOC_WAITOK | VM_ALLOC_WAITFAIL)) == 0)  in vm_domainset_iter_page()
250 	vm_wait_doms(&di->di_valid_mask, 0);  in vm_domainset_iter_page()
253 	if ((di->di_flags & VM_ALLOC_WAITFAIL) != 0)  in vm_domainset_iter_page()
257 	vm_domainset_iter_first(di, domain);  in vm_domainset_iter_page()
263 _vm_domainset_iter_policy_init(struct vm_domainset_iter *di, int *domain,  in _vm_domainset_iter_policy_init()  argument
267 	di->di_flags = *flags;  in _vm_domainset_iter_policy_init()
268 	*flags = (di->di_flags & ~M_WAITOK) | M_NOWAIT;  in _vm_domainset_iter_policy_init()
269 	vm_domainset_iter_first(di, domain);  in _vm_domainset_iter_policy_init()
271 		vm_domainset_iter_policy(di, domain);  in _vm_domainset_iter_policy_init()
275 vm_domainset_iter_policy_init(struct vm_domainset_iter *di,  in vm_domainset_iter_policy_init()  argument
279 	vm_domainset_iter_init(di, ds, &curthread->td_domain.dr_iter, NULL, 0);  in vm_domainset_iter_policy_init()
280 	_vm_domainset_iter_policy_init(di, domain, flags);  in vm_domainset_iter_policy_init()
284 vm_domainset_iter_policy_ref_init(struct vm_domainset_iter *di,  in vm_domainset_iter_policy_ref_init()  argument
288 	vm_domainset_iter_init(di, dr->dr_policy, &dr->dr_iter, NULL, 0);  in vm_domainset_iter_policy_ref_init()
289 	_vm_domainset_iter_policy_init(di, domain, flags);  in vm_domainset_iter_policy_ref_init()
293 vm_domainset_iter_policy(struct vm_domainset_iter *di, int *domain)  in vm_domainset_iter_policy()  argument
295 	if (DOMAINSET_EMPTY(&di->di_valid_mask))  in vm_domainset_iter_policy()
299 	while (--di->di_n != 0) {  in vm_domainset_iter_policy()
300 		vm_domainset_iter_next(di, domain);  in vm_domainset_iter_policy()
301 		if (DOMAINSET_ISSET(*domain, &di->di_valid_mask) &&  in vm_domainset_iter_policy()
302 		    (!di->di_minskip || !vm_page_count_min_domain(*domain)))  in vm_domainset_iter_policy()
307 	if (di->di_minskip) {  in vm_domainset_iter_policy()
308 		di->di_minskip = false;  in vm_domainset_iter_policy()
309 		vm_domainset_iter_first(di, domain);  in vm_domainset_iter_policy()
314 	if ((di->di_flags & M_WAITOK) == 0)  in vm_domainset_iter_policy()
318 	vm_wait_doms(&di->di_valid_mask, 0);  in vm_domainset_iter_policy()
321 	vm_domainset_iter_first(di, domain);  in vm_domainset_iter_policy()
327 vm_domainset_iter_ignore(struct vm_domainset_iter *di, int domain)  in vm_domainset_iter_ignore()  argument
329 	KASSERT(DOMAINSET_ISSET(domain, &di->di_valid_mask),  in vm_domainset_iter_ignore()
331 	    __func__, domain, di));  in vm_domainset_iter_ignore()
332 	DOMAINSET_CLR(domain, &di->di_valid_mask);  in vm_domainset_iter_ignore()
338 vm_domainset_iter_page(struct vm_domainset_iter *di, struct vm_object *obj,  in vm_domainset_iter_page()  argument
346 vm_domainset_iter_page_init(struct vm_domainset_iter *di, struct vm_object *obj,  in vm_domainset_iter_page_init()  argument
354 vm_domainset_iter_policy(struct vm_domainset_iter *di, int *domain)  in vm_domainset_iter_policy()  argument
361 vm_domainset_iter_policy_init(struct vm_domainset_iter *di,  in vm_domainset_iter_policy_init()  argument
369 vm_domainset_iter_policy_ref_init(struct vm_domainset_iter *di,  in vm_domainset_iter_policy_ref_init()  argument
377 vm_domainset_iter_ignore(struct vm_domainset_iter *di __unused,  in vm_domainset_iter_ignore()