xref: /original-bsd/sys/vm/vm_fault.c (revision 5b970e91)
1 /*
2  * Copyright (c) 1991, 1993
3  *	The Regents of the University of California.  All rights reserved.
4  *
5  * This code is derived from software contributed to Berkeley by
6  * The Mach Operating System project at Carnegie-Mellon University.
7  *
8  * %sccs.include.redist.c%
9  *
10  *	@(#)vm_fault.c	8.6 (Berkeley) 05/11/95
11  *
12  *
13  * Copyright (c) 1987, 1990 Carnegie-Mellon University.
14  * All rights reserved.
15  *
16  * Authors: Avadis Tevanian, Jr., Michael Wayne Young
17  *
18  * Permission to use, copy, modify and distribute this software and
19  * its documentation is hereby granted, provided that both the copyright
20  * notice and this permission notice appear in all copies of the
21  * software, derivative works or modified versions, and any portions
22  * thereof, and that both notices appear in supporting documentation.
23  *
24  * CARNEGIE MELLON ALLOWS FREE USE OF THIS SOFTWARE IN ITS "AS IS"
25  * CONDITION.  CARNEGIE MELLON DISCLAIMS ANY LIABILITY OF ANY KIND
26  * FOR ANY DAMAGES WHATSOEVER RESULTING FROM THE USE OF THIS SOFTWARE.
27  *
28  * Carnegie Mellon requests users of this software to return to
29  *
30  *  Software Distribution Coordinator  or  Software.Distribution@CS.CMU.EDU
31  *  School of Computer Science
32  *  Carnegie Mellon University
33  *  Pittsburgh PA 15213-3890
34  *
35  * any improvements or extensions that they make and grant Carnegie the
36  * rights to redistribute these changes.
37  */
38 
39 /*
40  *	Page fault handling module.
41  */
42 
43 #include <sys/param.h>
44 #include <sys/systm.h>
45 
46 #include <vm/vm.h>
47 #include <vm/vm_page.h>
48 #include <vm/vm_pageout.h>
49 
50 /*
51  *	vm_fault:
52  *
53  *	Handle a page fault occuring at the given address,
54  *	requiring the given permissions, in the map specified.
55  *	If successful, the page is inserted into the
56  *	associated physical map.
57  *
58  *	NOTE: the given address should be truncated to the
59  *	proper page address.
60  *
61  *	KERN_SUCCESS is returned if the page fault is handled; otherwise,
62  *	a standard error specifying why the fault is fatal is returned.
63  *
64  *
65  *	The map in question must be referenced, and remains so.
66  *	Caller may hold no locks.
67  */
68 int
vm_fault(map,vaddr,fault_type,change_wiring)69 vm_fault(map, vaddr, fault_type, change_wiring)
70 	vm_map_t	map;
71 	vm_offset_t	vaddr;
72 	vm_prot_t	fault_type;
73 	boolean_t	change_wiring;
74 {
75 	vm_object_t		first_object;
76 	vm_offset_t		first_offset;
77 	vm_map_entry_t		entry;
78 	register vm_object_t	object;
79 	register vm_offset_t	offset;
80 	register vm_page_t	m;
81 	vm_page_t		first_m;
82 	vm_prot_t		prot;
83 	int			result;
84 	boolean_t		wired;
85 	boolean_t		su;
86 	boolean_t		lookup_still_valid;
87 	boolean_t		page_exists;
88 	vm_page_t		old_m;
89 	vm_object_t		next_object;
90 
91 	cnt.v_faults++;		/* needs lock XXX */
92 /*
93  *	Recovery actions
94  */
95 #define	FREE_PAGE(m)	{				\
96 	PAGE_WAKEUP(m);					\
97 	vm_page_lock_queues();				\
98 	vm_page_free(m);				\
99 	vm_page_unlock_queues();			\
100 }
101 
102 #define	RELEASE_PAGE(m)	{				\
103 	PAGE_WAKEUP(m);					\
104 	vm_page_lock_queues();				\
105 	vm_page_activate(m);				\
106 	vm_page_unlock_queues();			\
107 }
108 
109 #define	UNLOCK_MAP	{				\
110 	if (lookup_still_valid) {			\
111 		vm_map_lookup_done(map, entry);		\
112 		lookup_still_valid = FALSE;		\
113 	}						\
114 }
115 
116 #define	UNLOCK_THINGS	{				\
117 	object->paging_in_progress--;			\
118 	vm_object_unlock(object);			\
119 	if (object != first_object) {			\
120 		vm_object_lock(first_object);		\
121 		FREE_PAGE(first_m);			\
122 		first_object->paging_in_progress--;	\
123 		vm_object_unlock(first_object);		\
124 	}						\
125 	UNLOCK_MAP;					\
126 }
127 
128 #define	UNLOCK_AND_DEALLOCATE	{			\
129 	UNLOCK_THINGS;					\
130 	vm_object_deallocate(first_object);		\
131 }
132 
133     RetryFault: ;
134 
135 	/*
136 	 *	Find the backing store object and offset into
137 	 *	it to begin the search.
138 	 */
139 
140 	if ((result = vm_map_lookup(&map, vaddr, fault_type, &entry,
141 			&first_object, &first_offset,
142 			&prot, &wired, &su)) != KERN_SUCCESS) {
143 		return(result);
144 	}
145 	lookup_still_valid = TRUE;
146 
147 	if (wired)
148 		fault_type = prot;
149 
150 	first_m = NULL;
151 
152    	/*
153 	 *	Make a reference to this object to
154 	 *	prevent its disposal while we are messing with
155 	 *	it.  Once we have the reference, the map is free
156 	 *	to be diddled.  Since objects reference their
157 	 *	shadows (and copies), they will stay around as well.
158 	 */
159 
160 	vm_object_lock(first_object);
161 
162 	first_object->ref_count++;
163 	first_object->paging_in_progress++;
164 
165 	/*
166 	 *	INVARIANTS (through entire routine):
167 	 *
168 	 *	1)	At all times, we must either have the object
169 	 *		lock or a busy page in some object to prevent
170 	 *		some other thread from trying to bring in
171 	 *		the same page.
172 	 *
173 	 *		Note that we cannot hold any locks during the
174 	 *		pager access or when waiting for memory, so
175 	 *		we use a busy page then.
176 	 *
177 	 *		Note also that we aren't as concerned about
178 	 *		more than one thead attempting to pager_data_unlock
179 	 *		the same page at once, so we don't hold the page
180 	 *		as busy then, but do record the highest unlock
181 	 *		value so far.  [Unlock requests may also be delivered
182 	 *		out of order.]
183 	 *
184 	 *	2)	Once we have a busy page, we must remove it from
185 	 *		the pageout queues, so that the pageout daemon
186 	 *		will not grab it away.
187 	 *
188 	 *	3)	To prevent another thread from racing us down the
189 	 *		shadow chain and entering a new page in the top
190 	 *		object before we do, we must keep a busy page in
191 	 *		the top object while following the shadow chain.
192 	 *
193 	 *	4)	We must increment paging_in_progress on any object
194 	 *		for which we have a busy page, to prevent
195 	 *		vm_object_collapse from removing the busy page
196 	 *		without our noticing.
197 	 */
198 
199 	/*
200 	 *	Search for the page at object/offset.
201 	 */
202 
203 	object = first_object;
204 	offset = first_offset;
205 
206 	/*
207 	 *	See whether this page is resident
208 	 */
209 
210 	while (TRUE) {
211 		m = vm_page_lookup(object, offset);
212 		if (m != NULL) {
213 			/*
214 			 *	If the page is being brought in,
215 			 *	wait for it and then retry.
216 			 */
217 			if (m->flags & PG_BUSY) {
218 #ifdef DOTHREADS
219 				int	wait_result;
220 
221 				PAGE_ASSERT_WAIT(m, !change_wiring);
222 				UNLOCK_THINGS;
223 				thread_block();
224 				wait_result = current_thread()->wait_result;
225 				vm_object_deallocate(first_object);
226 				if (wait_result != THREAD_AWAKENED)
227 					return(KERN_SUCCESS);
228 				goto RetryFault;
229 #else
230 				PAGE_ASSERT_WAIT(m, !change_wiring);
231 				UNLOCK_THINGS;
232 				cnt.v_intrans++;
233 				thread_block();
234 				vm_object_deallocate(first_object);
235 				goto RetryFault;
236 #endif
237 			}
238 
239 			/*
240 			 *	Remove the page from the pageout daemon's
241 			 *	reach while we play with it.
242 			 */
243 
244 			vm_page_lock_queues();
245 			if (m->flags & PG_INACTIVE) {
246 				TAILQ_REMOVE(&vm_page_queue_inactive, m, pageq);
247 				m->flags &= ~PG_INACTIVE;
248 				cnt.v_inactive_count--;
249 				cnt.v_reactivated++;
250 			}
251 
252 			if (m->flags & PG_ACTIVE) {
253 				TAILQ_REMOVE(&vm_page_queue_active, m, pageq);
254 				m->flags &= ~PG_ACTIVE;
255 				cnt.v_active_count--;
256 			}
257 			vm_page_unlock_queues();
258 
259 			/*
260 			 *	Mark page busy for other threads.
261 			 */
262 			m->flags |= PG_BUSY;
263 			break;
264 		}
265 
266 		if (((object->pager != NULL) &&
267 				(!change_wiring || wired))
268 		    || (object == first_object)) {
269 
270 			/*
271 			 *	Allocate a new page for this object/offset
272 			 *	pair.
273 			 */
274 
275 			m = vm_page_alloc(object, offset);
276 
277 			if (m == NULL) {
278 				UNLOCK_AND_DEALLOCATE;
279 				VM_WAIT;
280 				goto RetryFault;
281 			}
282 		}
283 
284 		if (object->pager != NULL && (!change_wiring || wired)) {
285 			int rv;
286 
287 			/*
288 			 *	Now that we have a busy page, we can
289 			 *	release the object lock.
290 			 */
291 			vm_object_unlock(object);
292 
293 			/*
294 			 *	Call the pager to retrieve the data, if any,
295 			 *	after releasing the lock on the map.
296 			 */
297 			UNLOCK_MAP;
298 			cnt.v_pageins++;
299 			rv = vm_pager_get(object->pager, m, TRUE);
300 
301 			/*
302 			 *	Reaquire the object lock to preserve our
303 			 *	invariant.
304 			 */
305 			vm_object_lock(object);
306 
307 			/*
308 			 *	Found the page.
309 			 *	Leave it busy while we play with it.
310 			 */
311 			if (rv == VM_PAGER_OK) {
312 				/*
313 				 *	Relookup in case pager changed page.
314 				 *	Pager is responsible for disposition
315 				 *	of old page if moved.
316 				 */
317 				m = vm_page_lookup(object, offset);
318 
319 				cnt.v_pgpgin++;
320 				m->flags &= ~PG_FAKE;
321 				m->flags |= PG_CLEAN;
322 				pmap_clear_modify(VM_PAGE_TO_PHYS(m));
323 				break;
324 			}
325 
326 			/*
327 			 * IO error or page outside the range of the pager:
328 			 * cleanup and return an error.
329 			 */
330 			if (rv == VM_PAGER_ERROR || rv == VM_PAGER_BAD) {
331 				FREE_PAGE(m);
332 				UNLOCK_AND_DEALLOCATE;
333 				return(KERN_PROTECTION_FAILURE); /* XXX */
334 			}
335 			/*
336 			 * rv == VM_PAGER_FAIL:
337 			 *
338 			 * Page does not exist at this object/offset.
339 			 * Free the bogus page (waking up anyone waiting
340 			 * for it) and continue on to the next object.
341 			 *
342 			 * If this is the top-level object, we must
343 			 * leave the busy page to prevent another
344 			 * thread from rushing past us, and inserting
345 			 * the page in that object at the same time
346 			 * that we are.
347 			 */
348 			if (object != first_object) {
349 				FREE_PAGE(m);
350 				/* note that `m' is not used after this */
351 			}
352 		}
353 
354 		/*
355 		 * We get here if the object has no pager (or unwiring)
356 		 * or the pager doesn't have the page.
357 		 */
358 		if (object == first_object)
359 			first_m = m;
360 
361 		/*
362 		 *	Move on to the next object.  Lock the next
363 		 *	object before unlocking the current one.
364 		 */
365 
366 		offset += object->shadow_offset;
367 		next_object = object->shadow;
368 		if (next_object == NULL) {
369 			/*
370 			 *	If there's no object left, fill the page
371 			 *	in the top object with zeros.
372 			 */
373 			if (object != first_object) {
374 				object->paging_in_progress--;
375 				vm_object_unlock(object);
376 
377 				object = first_object;
378 				offset = first_offset;
379 				m = first_m;
380 				vm_object_lock(object);
381 			}
382 			first_m = NULL;
383 
384 			vm_page_zero_fill(m);
385 			cnt.v_zfod++;
386 			m->flags &= ~PG_FAKE;
387 			break;
388 		}
389 		else {
390 			vm_object_lock(next_object);
391 			if (object != first_object)
392 				object->paging_in_progress--;
393 			vm_object_unlock(object);
394 			object = next_object;
395 			object->paging_in_progress++;
396 		}
397 	}
398 
399 	if ((m->flags & (PG_ACTIVE | PG_INACTIVE | PG_BUSY)) != PG_BUSY)
400 		panic("vm_fault: active, inactive or !busy after main loop");
401 
402 	/*
403 	 *	PAGE HAS BEEN FOUND.
404 	 *	[Loop invariant still holds -- the object lock
405 	 *	is held.]
406 	 */
407 
408 	old_m = m;	/* save page that would be copied */
409 
410 	/*
411 	 *	If the page is being written, but isn't
412 	 *	already owned by the top-level object,
413 	 *	we have to copy it into a new page owned
414 	 *	by the top-level object.
415 	 */
416 
417 	if (object != first_object) {
418 	    	/*
419 		 *	We only really need to copy if we
420 		 *	want to write it.
421 		 */
422 
423 	    	if (fault_type & VM_PROT_WRITE) {
424 
425 			/*
426 			 *	If we try to collapse first_object at this
427 			 *	point, we may deadlock when we try to get
428 			 *	the lock on an intermediate object (since we
429 			 *	have the bottom object locked).  We can't
430 			 *	unlock the bottom object, because the page
431 			 *	we found may move (by collapse) if we do.
432 			 *
433 			 *	Instead, we first copy the page.  Then, when
434 			 *	we have no more use for the bottom object,
435 			 *	we unlock it and try to collapse.
436 			 *
437 			 *	Note that we copy the page even if we didn't
438 			 *	need to... that's the breaks.
439 			 */
440 
441 		    	/*
442 			 *	We already have an empty page in
443 			 *	first_object - use it.
444 			 */
445 
446 			vm_page_copy(m, first_m);
447 			first_m->flags &= ~PG_FAKE;
448 
449 			/*
450 			 *	If another map is truly sharing this
451 			 *	page with us, we have to flush all
452 			 *	uses of the original page, since we
453 			 *	can't distinguish those which want the
454 			 *	original from those which need the
455 			 *	new copy.
456 			 *
457 			 *	XXX If we know that only one map has
458 			 *	access to this page, then we could
459 			 *	avoid the pmap_page_protect() call.
460 			 */
461 
462 			vm_page_lock_queues();
463 			vm_page_activate(m);
464 			vm_page_deactivate(m);
465 			vm_page_unlock_queues();
466 			/*
467 			 * XXX gag!  The page protect has been moved out
468 			 * of the page queue lock section to avoid a deadlock
469 			 * in the hp300-style (recursive) pmap module.
470 			 * If you were on an MP, p_p_protect might result
471 			 * in a vm_map_pageable(..., TRUE) for the associated
472 			 * page table page.  This would call vm_fault_unwire
473 			 * which would try to lock the page queues.
474 			 * Moving the call out is safe here because the
475 			 * object is still locked and that will prevent
476 			 * the pageout daemon from messing with this page
477 			 * on the inactive list.  (It would move it back to
478 			 * the active list if it were referenced but
479 			 * v_p_deallocate clears the ref bit).
480 			 */
481 			pmap_page_protect(VM_PAGE_TO_PHYS(m), VM_PROT_NONE);
482 
483 			/*
484 			 *	We no longer need the old page or object.
485 			 */
486 			PAGE_WAKEUP(m);
487 			object->paging_in_progress--;
488 			vm_object_unlock(object);
489 
490 			/*
491 			 *	Only use the new page below...
492 			 */
493 
494 			cnt.v_cow_faults++;
495 			m = first_m;
496 			object = first_object;
497 			offset = first_offset;
498 
499 			/*
500 			 *	Now that we've gotten the copy out of the
501 			 *	way, let's try to collapse the top object.
502 			 */
503 			vm_object_lock(object);
504 			/*
505 			 *	But we have to play ugly games with
506 			 *	paging_in_progress to do that...
507 			 */
508 			object->paging_in_progress--;
509 			vm_object_collapse(object);
510 			object->paging_in_progress++;
511 		}
512 		else {
513 		    	prot &= ~VM_PROT_WRITE;
514 			m->flags |= PG_COPYONWRITE;
515 		}
516 	}
517 
518 	if (m->flags & (PG_ACTIVE|PG_INACTIVE))
519 		panic("vm_fault: active or inactive before copy object handling");
520 
521 	/*
522 	 *	If the page is being written, but hasn't been
523 	 *	copied to the copy-object, we have to copy it there.
524 	 */
525     RetryCopy:
526 	if (first_object->copy != NULL) {
527 		vm_object_t copy_object = first_object->copy;
528 		vm_offset_t copy_offset;
529 		vm_page_t copy_m;
530 
531 		/*
532 		 *	We only need to copy if we want to write it.
533 		 */
534 		if ((fault_type & VM_PROT_WRITE) == 0) {
535 			prot &= ~VM_PROT_WRITE;
536 			m->flags |= PG_COPYONWRITE;
537 		}
538 		else {
539 			/*
540 			 *	Try to get the lock on the copy_object.
541 			 */
542 			if (!vm_object_lock_try(copy_object)) {
543 				vm_object_unlock(object);
544 				/* should spin a bit here... */
545 				vm_object_lock(object);
546 				goto RetryCopy;
547 			}
548 
549 			/*
550 			 *	Make another reference to the copy-object,
551 			 *	to keep it from disappearing during the
552 			 *	copy.
553 			 */
554 			copy_object->ref_count++;
555 
556 			/*
557 			 *	Does the page exist in the copy?
558 			 */
559 			copy_offset = first_offset
560 				- copy_object->shadow_offset;
561 			copy_m = vm_page_lookup(copy_object, copy_offset);
562 			if (page_exists = (copy_m != NULL)) {
563 				if (copy_m->flags & PG_BUSY) {
564 #ifdef DOTHREADS
565 					int	wait_result;
566 
567 					/*
568 					 *	If the page is being brought
569 					 *	in, wait for it and then retry.
570 					 */
571 					PAGE_ASSERT_WAIT(copy_m, !change_wiring);
572 					RELEASE_PAGE(m);
573 					copy_object->ref_count--;
574 					vm_object_unlock(copy_object);
575 					UNLOCK_THINGS;
576 					thread_block();
577 					wait_result = current_thread()->wait_result;
578 					vm_object_deallocate(first_object);
579 					if (wait_result != THREAD_AWAKENED)
580 						return(KERN_SUCCESS);
581 					goto RetryFault;
582 #else
583 					/*
584 					 *	If the page is being brought
585 					 *	in, wait for it and then retry.
586 					 */
587 					PAGE_ASSERT_WAIT(copy_m, !change_wiring);
588 					RELEASE_PAGE(m);
589 					copy_object->ref_count--;
590 					vm_object_unlock(copy_object);
591 					UNLOCK_THINGS;
592 					thread_block();
593 					vm_object_deallocate(first_object);
594 					goto RetryFault;
595 #endif
596 				}
597 			}
598 
599 			/*
600 			 *	If the page is not in memory (in the object)
601 			 *	and the object has a pager, we have to check
602 			 *	if the pager has the data in secondary
603 			 *	storage.
604 			 */
605 			if (!page_exists) {
606 
607 				/*
608 				 *	If we don't allocate a (blank) page
609 				 *	here... another thread could try
610 				 *	to page it in, allocate a page, and
611 				 *	then block on the busy page in its
612 				 *	shadow (first_object).  Then we'd
613 				 *	trip over the busy page after we
614 				 *	found that the copy_object's pager
615 				 *	doesn't have the page...
616 				 */
617 				copy_m = vm_page_alloc(copy_object,
618 								copy_offset);
619 				if (copy_m == NULL) {
620 					/*
621 					 *	Wait for a page, then retry.
622 					 */
623 					RELEASE_PAGE(m);
624 					copy_object->ref_count--;
625 					vm_object_unlock(copy_object);
626 					UNLOCK_AND_DEALLOCATE;
627 					VM_WAIT;
628 					goto RetryFault;
629 				}
630 
631 			 	if (copy_object->pager != NULL) {
632 					vm_object_unlock(object);
633 					vm_object_unlock(copy_object);
634 					UNLOCK_MAP;
635 
636 					page_exists = vm_pager_has_page(
637 							copy_object->pager,
638 							(copy_offset + copy_object->paging_offset));
639 
640 					vm_object_lock(copy_object);
641 
642 					/*
643 					 * Since the map is unlocked, someone
644 					 * else could have copied this object
645 					 * and put a different copy_object
646 					 * between the two.  Or, the last
647 					 * reference to the copy-object (other
648 					 * than the one we have) may have
649 					 * disappeared - if that has happened,
650 					 * we don't need to make the copy.
651 					 */
652 					if (copy_object->shadow != object ||
653 					    copy_object->ref_count == 1) {
654 						/*
655 						 *	Gaah... start over!
656 						 */
657 						FREE_PAGE(copy_m);
658 						vm_object_unlock(copy_object);
659 						vm_object_deallocate(copy_object);
660 							/* may block */
661 						vm_object_lock(object);
662 						goto RetryCopy;
663 					}
664 					vm_object_lock(object);
665 
666 					if (page_exists) {
667 						/*
668 						 *	We didn't need the page
669 						 */
670 						FREE_PAGE(copy_m);
671 					}
672 				}
673 			}
674 			if (!page_exists) {
675 				/*
676 				 *	Must copy page into copy-object.
677 				 */
678 				vm_page_copy(m, copy_m);
679 				copy_m->flags &= ~PG_FAKE;
680 
681 				/*
682 				 * Things to remember:
683 				 * 1. The copied page must be marked 'dirty'
684 				 *    so it will be paged out to the copy
685 				 *    object.
686 				 * 2. If the old page was in use by any users
687 				 *    of the copy-object, it must be removed
688 				 *    from all pmaps.  (We can't know which
689 				 *    pmaps use it.)
690 				 */
691 				vm_page_lock_queues();
692 				pmap_page_protect(VM_PAGE_TO_PHYS(old_m),
693 						  VM_PROT_NONE);
694 				copy_m->flags &= ~PG_CLEAN;
695 				vm_page_activate(copy_m);	/* XXX */
696 				vm_page_unlock_queues();
697 
698 				PAGE_WAKEUP(copy_m);
699 			}
700 			/*
701 			 *	The reference count on copy_object must be
702 			 *	at least 2: one for our extra reference,
703 			 *	and at least one from the outside world
704 			 *	(we checked that when we last locked
705 			 *	copy_object).
706 			 */
707 			copy_object->ref_count--;
708 			vm_object_unlock(copy_object);
709 			m->flags &= ~PG_COPYONWRITE;
710 		}
711 	}
712 
713 	if (m->flags & (PG_ACTIVE | PG_INACTIVE))
714 		panic("vm_fault: active or inactive before retrying lookup");
715 
716 	/*
717 	 *	We must verify that the maps have not changed
718 	 *	since our last lookup.
719 	 */
720 
721 	if (!lookup_still_valid) {
722 		vm_object_t	retry_object;
723 		vm_offset_t	retry_offset;
724 		vm_prot_t	retry_prot;
725 
726 		/*
727 		 *	Since map entries may be pageable, make sure we can
728 		 *	take a page fault on them.
729 		 */
730 		vm_object_unlock(object);
731 
732 		/*
733 		 *	To avoid trying to write_lock the map while another
734 		 *	thread has it read_locked (in vm_map_pageable), we
735 		 *	do not try for write permission.  If the page is
736 		 *	still writable, we will get write permission.  If it
737 		 *	is not, or has been marked needs_copy, we enter the
738 		 *	mapping without write permission, and will merely
739 		 *	take another fault.
740 		 */
741 		result = vm_map_lookup(&map, vaddr,
742 				fault_type & ~VM_PROT_WRITE, &entry,
743 				&retry_object, &retry_offset, &retry_prot,
744 				&wired, &su);
745 
746 		vm_object_lock(object);
747 
748 		/*
749 		 *	If we don't need the page any longer, put it on the
750 		 *	active list (the easiest thing to do here).  If no
751 		 *	one needs it, pageout will grab it eventually.
752 		 */
753 
754 		if (result != KERN_SUCCESS) {
755 			RELEASE_PAGE(m);
756 			UNLOCK_AND_DEALLOCATE;
757 			return(result);
758 		}
759 
760 		lookup_still_valid = TRUE;
761 
762 		if ((retry_object != first_object) ||
763 				(retry_offset != first_offset)) {
764 			RELEASE_PAGE(m);
765 			UNLOCK_AND_DEALLOCATE;
766 			goto RetryFault;
767 		}
768 
769 		/*
770 		 *	Check whether the protection has changed or the object
771 		 *	has been copied while we left the map unlocked.
772 		 *	Changing from read to write permission is OK - we leave
773 		 *	the page write-protected, and catch the write fault.
774 		 *	Changing from write to read permission means that we
775 		 *	can't mark the page write-enabled after all.
776 		 */
777 		prot &= retry_prot;
778 		if (m->flags & PG_COPYONWRITE)
779 			prot &= ~VM_PROT_WRITE;
780 	}
781 
782 	/*
783 	 * (the various bits we're fiddling with here are locked by
784 	 * the object's lock)
785 	 */
786 
787 	/* XXX This distorts the meaning of the copy_on_write bit */
788 
789 	if (prot & VM_PROT_WRITE)
790 		m->flags &= ~PG_COPYONWRITE;
791 
792 	/*
793 	 *	It's critically important that a wired-down page be faulted
794 	 *	only once in each map for which it is wired.
795 	 */
796 
797 	if (m->flags & (PG_ACTIVE | PG_INACTIVE))
798 		panic("vm_fault: active or inactive before pmap_enter");
799 
800 	vm_object_unlock(object);
801 
802 	/*
803 	 *	Put this page into the physical map.
804 	 *	We had to do the unlock above because pmap_enter
805 	 *	may cause other faults.   We don't put the
806 	 *	page back on the active queue until later so
807 	 *	that the page-out daemon won't find us (yet).
808 	 */
809 
810 	pmap_enter(map->pmap, vaddr, VM_PAGE_TO_PHYS(m), prot, wired);
811 
812 	/*
813 	 *	If the page is not wired down, then put it where the
814 	 *	pageout daemon can find it.
815 	 */
816 	vm_object_lock(object);
817 	vm_page_lock_queues();
818 	if (change_wiring) {
819 		if (wired)
820 			vm_page_wire(m);
821 		else
822 			vm_page_unwire(m);
823 	}
824 	else
825 		vm_page_activate(m);
826 	vm_page_unlock_queues();
827 
828 	/*
829 	 *	Unlock everything, and return
830 	 */
831 
832 	PAGE_WAKEUP(m);
833 	UNLOCK_AND_DEALLOCATE;
834 
835 	return(KERN_SUCCESS);
836 
837 }
838 
839 /*
840  *	vm_fault_wire:
841  *
842  *	Wire down a range of virtual addresses in a map.
843  */
844 int
vm_fault_wire(map,start,end)845 vm_fault_wire(map, start, end)
846 	vm_map_t	map;
847 	vm_offset_t	start, end;
848 {
849 	register vm_offset_t	va;
850 	register pmap_t		pmap;
851 	int			rv;
852 
853 	pmap = vm_map_pmap(map);
854 
855 	/*
856 	 *	Inform the physical mapping system that the
857 	 *	range of addresses may not fault, so that
858 	 *	page tables and such can be locked down as well.
859 	 */
860 
861 	pmap_pageable(pmap, start, end, FALSE);
862 
863 	/*
864 	 *	We simulate a fault to get the page and enter it
865 	 *	in the physical map.
866 	 */
867 
868 	for (va = start; va < end; va += PAGE_SIZE) {
869 		rv = vm_fault(map, va, VM_PROT_NONE, TRUE);
870 		if (rv) {
871 			if (va != start)
872 				vm_fault_unwire(map, start, va);
873 			return(rv);
874 		}
875 	}
876 	return(KERN_SUCCESS);
877 }
878 
879 
880 /*
881  *	vm_fault_unwire:
882  *
883  *	Unwire a range of virtual addresses in a map.
884  */
885 void
vm_fault_unwire(map,start,end)886 vm_fault_unwire(map, start, end)
887 	vm_map_t	map;
888 	vm_offset_t	start, end;
889 {
890 
891 	register vm_offset_t	va, pa;
892 	register pmap_t		pmap;
893 
894 	pmap = vm_map_pmap(map);
895 
896 	/*
897 	 *	Since the pages are wired down, we must be able to
898 	 *	get their mappings from the physical map system.
899 	 */
900 
901 	vm_page_lock_queues();
902 
903 	for (va = start; va < end; va += PAGE_SIZE) {
904 		pa = pmap_extract(pmap, va);
905 		if (pa == (vm_offset_t) 0) {
906 			panic("unwire: page not in pmap");
907 		}
908 		pmap_change_wiring(pmap, va, FALSE);
909 		vm_page_unwire(PHYS_TO_VM_PAGE(pa));
910 	}
911 	vm_page_unlock_queues();
912 
913 	/*
914 	 *	Inform the physical mapping system that the range
915 	 *	of addresses may fault, so that page tables and
916 	 *	such may be unwired themselves.
917 	 */
918 
919 	pmap_pageable(pmap, start, end, TRUE);
920 
921 }
922 
923 /*
924  *	Routine:
925  *		vm_fault_copy_entry
926  *	Function:
927  *		Copy all of the pages from a wired-down map entry to another.
928  *
929  *	In/out conditions:
930  *		The source and destination maps must be locked for write.
931  *		The source map entry must be wired down (or be a sharing map
932  *		entry corresponding to a main map entry that is wired down).
933  */
934 
935 void
vm_fault_copy_entry(dst_map,src_map,dst_entry,src_entry)936 vm_fault_copy_entry(dst_map, src_map, dst_entry, src_entry)
937 	vm_map_t	dst_map;
938 	vm_map_t	src_map;
939 	vm_map_entry_t	dst_entry;
940 	vm_map_entry_t	src_entry;
941 {
942 
943 	vm_object_t	dst_object;
944 	vm_object_t	src_object;
945 	vm_offset_t	dst_offset;
946 	vm_offset_t	src_offset;
947 	vm_prot_t	prot;
948 	vm_offset_t	vaddr;
949 	vm_page_t	dst_m;
950 	vm_page_t	src_m;
951 
952 #ifdef	lint
953 	src_map++;
954 #endif
955 
956 	src_object = src_entry->object.vm_object;
957 	src_offset = src_entry->offset;
958 
959 	/*
960 	 *	Create the top-level object for the destination entry.
961 	 *	(Doesn't actually shadow anything - we copy the pages
962 	 *	directly.)
963 	 */
964 	dst_object = vm_object_allocate(
965 			(vm_size_t) (dst_entry->end - dst_entry->start));
966 
967 	dst_entry->object.vm_object = dst_object;
968 	dst_entry->offset = 0;
969 
970 	prot  = dst_entry->max_protection;
971 
972 	/*
973 	 *	Loop through all of the pages in the entry's range, copying
974 	 *	each one from the source object (it should be there) to the
975 	 *	destination object.
976 	 */
977 	for (vaddr = dst_entry->start, dst_offset = 0;
978 	     vaddr < dst_entry->end;
979 	     vaddr += PAGE_SIZE, dst_offset += PAGE_SIZE) {
980 
981 		/*
982 		 *	Allocate a page in the destination object
983 		 */
984 		vm_object_lock(dst_object);
985 		do {
986 			dst_m = vm_page_alloc(dst_object, dst_offset);
987 			if (dst_m == NULL) {
988 				vm_object_unlock(dst_object);
989 				VM_WAIT;
990 				vm_object_lock(dst_object);
991 			}
992 		} while (dst_m == NULL);
993 
994 		/*
995 		 *	Find the page in the source object, and copy it in.
996 		 *	(Because the source is wired down, the page will be
997 		 *	in memory.)
998 		 */
999 		vm_object_lock(src_object);
1000 		src_m = vm_page_lookup(src_object, dst_offset + src_offset);
1001 		if (src_m == NULL)
1002 			panic("vm_fault_copy_wired: page missing");
1003 
1004 		vm_page_copy(src_m, dst_m);
1005 
1006 		/*
1007 		 *	Enter it in the pmap...
1008 		 */
1009 		vm_object_unlock(src_object);
1010 		vm_object_unlock(dst_object);
1011 
1012 		pmap_enter(dst_map->pmap, vaddr, VM_PAGE_TO_PHYS(dst_m),
1013 				prot, FALSE);
1014 
1015 		/*
1016 		 *	Mark it no longer busy, and put it on the active list.
1017 		 */
1018 		vm_object_lock(dst_object);
1019 		vm_page_lock_queues();
1020 		vm_page_activate(dst_m);
1021 		vm_page_unlock_queues();
1022 		PAGE_WAKEUP(dst_m);
1023 		vm_object_unlock(dst_object);
1024 	}
1025 
1026 }
1027