xref: /original-bsd/sys/vm/vm_object.c (revision 81351f02)
1 /*
2  * Copyright (c) 1991, 1993
3  *	The Regents of the University of California.  All rights reserved.
4  *
5  * This code is derived from software contributed to Berkeley by
6  * The Mach Operating System project at Carnegie-Mellon University.
7  *
8  * %sccs.include.redist.c%
9  *
10  *	@(#)vm_object.c	8.7 (Berkeley) 05/11/95
11  *
12  *
13  * Copyright (c) 1987, 1990 Carnegie-Mellon University.
14  * All rights reserved.
15  *
16  * Authors: Avadis Tevanian, Jr., Michael Wayne Young
17  *
18  * Permission to use, copy, modify and distribute this software and
19  * its documentation is hereby granted, provided that both the copyright
20  * notice and this permission notice appear in all copies of the
21  * software, derivative works or modified versions, and any portions
22  * thereof, and that both notices appear in supporting documentation.
23  *
24  * CARNEGIE MELLON ALLOWS FREE USE OF THIS SOFTWARE IN ITS "AS IS"
25  * CONDITION.  CARNEGIE MELLON DISCLAIMS ANY LIABILITY OF ANY KIND
26  * FOR ANY DAMAGES WHATSOEVER RESULTING FROM THE USE OF THIS SOFTWARE.
27  *
28  * Carnegie Mellon requests users of this software to return to
29  *
30  *  Software Distribution Coordinator  or  Software.Distribution@CS.CMU.EDU
31  *  School of Computer Science
32  *  Carnegie Mellon University
33  *  Pittsburgh PA 15213-3890
34  *
35  * any improvements or extensions that they make and grant Carnegie the
36  * rights to redistribute these changes.
37  */
38 
39 /*
40  *	Virtual memory object module.
41  */
42 
43 #include <sys/param.h>
44 #include <sys/systm.h>
45 #include <sys/malloc.h>
46 
47 #include <vm/vm.h>
48 #include <vm/vm_page.h>
49 
50 /*
51  *	Virtual memory objects maintain the actual data
52  *	associated with allocated virtual memory.  A given
53  *	page of memory exists within exactly one object.
54  *
55  *	An object is only deallocated when all "references"
56  *	are given up.  Only one "reference" to a given
57  *	region of an object should be writeable.
58  *
59  *	Associated with each object is a list of all resident
60  *	memory pages belonging to that object; this list is
61  *	maintained by the "vm_page" module, and locked by the object's
62  *	lock.
63  *
64  *	Each object also records a "pager" routine which is
65  *	used to retrieve (and store) pages to the proper backing
66  *	storage.  In addition, objects may be backed by other
67  *	objects from which they were virtual-copied.
68  *
69  *	The only items within the object structure which are
70  *	modified after time of creation are:
71  *		reference count		locked by object's lock
72  *		pager routine		locked by object's lock
73  *
74  */
75 
76 struct vm_object	kernel_object_store;
77 struct vm_object	kmem_object_store;
78 
79 #define	VM_OBJECT_HASH_COUNT	157
80 
81 int	vm_cache_max = 100;	/* can patch if necessary */
82 struct	vm_object_hash_head vm_object_hashtable[VM_OBJECT_HASH_COUNT];
83 
84 long	object_collapses = 0;
85 long	object_bypasses  = 0;
86 
87 static void _vm_object_allocate __P((vm_size_t, vm_object_t));
88 
89 /*
90  *	vm_object_init:
91  *
92  *	Initialize the VM objects module.
93  */
94 void
vm_object_init(size)95 vm_object_init(size)
96 	vm_size_t	size;
97 {
98 	register int	i;
99 
100 	TAILQ_INIT(&vm_object_cached_list);
101 	TAILQ_INIT(&vm_object_list);
102 	vm_object_count = 0;
103 	simple_lock_init(&vm_cache_lock);
104 	simple_lock_init(&vm_object_list_lock);
105 
106 	for (i = 0; i < VM_OBJECT_HASH_COUNT; i++)
107 		TAILQ_INIT(&vm_object_hashtable[i]);
108 
109 	kernel_object = &kernel_object_store;
110 	_vm_object_allocate(size, kernel_object);
111 
112 	kmem_object = &kmem_object_store;
113 	_vm_object_allocate(VM_KMEM_SIZE + VM_MBUF_SIZE, kmem_object);
114 }
115 
116 /*
117  *	vm_object_allocate:
118  *
119  *	Returns a new object with the given size.
120  */
121 
122 vm_object_t
vm_object_allocate(size)123 vm_object_allocate(size)
124 	vm_size_t	size;
125 {
126 	register vm_object_t	result;
127 
128 	result = (vm_object_t)
129 		malloc((u_long)sizeof *result, M_VMOBJ, M_WAITOK);
130 
131 	_vm_object_allocate(size, result);
132 
133 	return(result);
134 }
135 
136 static void
_vm_object_allocate(size,object)137 _vm_object_allocate(size, object)
138 	vm_size_t		size;
139 	register vm_object_t	object;
140 {
141 	TAILQ_INIT(&object->memq);
142 	vm_object_lock_init(object);
143 	object->ref_count = 1;
144 	object->resident_page_count = 0;
145 	object->size = size;
146 	object->flags = OBJ_INTERNAL;	/* vm_allocate_with_pager will reset */
147 	object->paging_in_progress = 0;
148 	object->copy = NULL;
149 
150 	/*
151 	 *	Object starts out read-write, with no pager.
152 	 */
153 
154 	object->pager = NULL;
155 	object->paging_offset = 0;
156 	object->shadow = NULL;
157 	object->shadow_offset = (vm_offset_t) 0;
158 
159 	simple_lock(&vm_object_list_lock);
160 	TAILQ_INSERT_TAIL(&vm_object_list, object, object_list);
161 	vm_object_count++;
162 	cnt.v_nzfod += atop(size);
163 	simple_unlock(&vm_object_list_lock);
164 }
165 
166 /*
167  *	vm_object_reference:
168  *
169  *	Gets another reference to the given object.
170  */
171 void
vm_object_reference(object)172 vm_object_reference(object)
173 	register vm_object_t	object;
174 {
175 	if (object == NULL)
176 		return;
177 
178 	vm_object_lock(object);
179 	object->ref_count++;
180 	vm_object_unlock(object);
181 }
182 
183 /*
184  *	vm_object_deallocate:
185  *
186  *	Release a reference to the specified object,
187  *	gained either through a vm_object_allocate
188  *	or a vm_object_reference call.  When all references
189  *	are gone, storage associated with this object
190  *	may be relinquished.
191  *
192  *	No object may be locked.
193  */
194 void
vm_object_deallocate(object)195 vm_object_deallocate(object)
196 	register vm_object_t	object;
197 {
198 	vm_object_t	temp;
199 
200 	while (object != NULL) {
201 
202 		/*
203 		 *	The cache holds a reference (uncounted) to
204 		 *	the object; we must lock it before removing
205 		 *	the object.
206 		 */
207 
208 		vm_object_cache_lock();
209 
210 		/*
211 		 *	Lose the reference
212 		 */
213 		vm_object_lock(object);
214 		if (--(object->ref_count) != 0) {
215 
216 			/*
217 			 *	If there are still references, then
218 			 *	we are done.
219 			 */
220 			vm_object_unlock(object);
221 			vm_object_cache_unlock();
222 			return;
223 		}
224 
225 		/*
226 		 *	See if this object can persist.  If so, enter
227 		 *	it in the cache, then deactivate all of its
228 		 *	pages.
229 		 */
230 
231 		if (object->flags & OBJ_CANPERSIST) {
232 
233 			TAILQ_INSERT_TAIL(&vm_object_cached_list, object,
234 				cached_list);
235 			vm_object_cached++;
236 			vm_object_cache_unlock();
237 
238 			vm_object_deactivate_pages(object);
239 			vm_object_unlock(object);
240 
241 			vm_object_cache_trim();
242 			return;
243 		}
244 
245 		/*
246 		 *	Make sure no one can look us up now.
247 		 */
248 		vm_object_remove(object->pager);
249 		vm_object_cache_unlock();
250 
251 		temp = object->shadow;
252 		vm_object_terminate(object);
253 			/* unlocks and deallocates object */
254 		object = temp;
255 	}
256 }
257 
258 
259 /*
260  *	vm_object_terminate actually destroys the specified object, freeing
261  *	up all previously used resources.
262  *
263  *	The object must be locked.
264  */
265 void
vm_object_terminate(object)266 vm_object_terminate(object)
267 	register vm_object_t	object;
268 {
269 	register vm_page_t	p;
270 	vm_object_t		shadow_object;
271 
272 	/*
273 	 *	Detach the object from its shadow if we are the shadow's
274 	 *	copy.
275 	 */
276 	if ((shadow_object = object->shadow) != NULL) {
277 		vm_object_lock(shadow_object);
278 		if (shadow_object->copy == object)
279 			shadow_object->copy = NULL;
280 #if 0
281 		else if (shadow_object->copy != NULL)
282 			panic("vm_object_terminate: copy/shadow inconsistency");
283 #endif
284 		vm_object_unlock(shadow_object);
285 	}
286 
287 	/*
288 	 * Wait until the pageout daemon is through with the object.
289 	 */
290 	while (object->paging_in_progress) {
291 		vm_object_sleep(object, object, FALSE);
292 		vm_object_lock(object);
293 	}
294 
295 	/*
296 	 * If not an internal object clean all the pages, removing them
297 	 * from paging queues as we go.
298 	 *
299 	 * XXX need to do something in the event of a cleaning error.
300 	 */
301 	if ((object->flags & OBJ_INTERNAL) == 0)
302 		(void) vm_object_page_clean(object, 0, 0, TRUE, TRUE);
303 
304 	/*
305 	 * Now free the pages.
306 	 * For internal objects, this also removes them from paging queues.
307 	 */
308 	while ((p = object->memq.tqh_first) != NULL) {
309 		VM_PAGE_CHECK(p);
310 		vm_page_lock_queues();
311 		vm_page_free(p);
312 		cnt.v_pfree++;
313 		vm_page_unlock_queues();
314 	}
315 	vm_object_unlock(object);
316 
317 	/*
318 	 * Let the pager know object is dead.
319 	 */
320 	if (object->pager != NULL)
321 		vm_pager_deallocate(object->pager);
322 
323 	simple_lock(&vm_object_list_lock);
324 	TAILQ_REMOVE(&vm_object_list, object, object_list);
325 	vm_object_count--;
326 	simple_unlock(&vm_object_list_lock);
327 
328 	/*
329 	 * Free the space for the object.
330 	 */
331 	free((caddr_t)object, M_VMOBJ);
332 }
333 
334 /*
335  *	vm_object_page_clean
336  *
337  *	Clean all dirty pages in the specified range of object.
338  *	If syncio is TRUE, page cleaning is done synchronously.
339  *	If de_queue is TRUE, pages are removed from any paging queue
340  *	they were on, otherwise they are left on whatever queue they
341  *	were on before the cleaning operation began.
342  *
343  *	Odd semantics: if start == end, we clean everything.
344  *
345  *	The object must be locked.
346  *
347  *	Returns TRUE if all was well, FALSE if there was a pager error
348  *	somewhere.  We attempt to clean (and dequeue) all pages regardless
349  *	of where an error occurs.
350  */
351 boolean_t
vm_object_page_clean(object,start,end,syncio,de_queue)352 vm_object_page_clean(object, start, end, syncio, de_queue)
353 	register vm_object_t	object;
354 	register vm_offset_t	start;
355 	register vm_offset_t	end;
356 	boolean_t		syncio;
357 	boolean_t		de_queue;
358 {
359 	register vm_page_t	p;
360 	int onqueue;
361 	boolean_t noerror = TRUE;
362 
363 	if (object == NULL)
364 		return (TRUE);
365 
366 	/*
367 	 * If it is an internal object and there is no pager, attempt to
368 	 * allocate one.  Note that vm_object_collapse may relocate one
369 	 * from a collapsed object so we must recheck afterward.
370 	 */
371 	if ((object->flags & OBJ_INTERNAL) && object->pager == NULL) {
372 		vm_object_collapse(object);
373 		if (object->pager == NULL) {
374 			vm_pager_t pager;
375 
376 			vm_object_unlock(object);
377 			pager = vm_pager_allocate(PG_DFLT, (caddr_t)0,
378 						  object->size, VM_PROT_ALL,
379 						  (vm_offset_t)0);
380 			if (pager)
381 				vm_object_setpager(object, pager, 0, FALSE);
382 			vm_object_lock(object);
383 		}
384 	}
385 	if (object->pager == NULL)
386 		return (FALSE);
387 
388 again:
389 	/*
390 	 * Wait until the pageout daemon is through with the object.
391 	 */
392 	while (object->paging_in_progress) {
393 		vm_object_sleep(object, object, FALSE);
394 		vm_object_lock(object);
395 	}
396 	/*
397 	 * Loop through the object page list cleaning as necessary.
398 	 */
399 	for (p = object->memq.tqh_first; p != NULL; p = p->listq.tqe_next) {
400 		if ((start == end || p->offset >= start && p->offset < end) &&
401 		    !(p->flags & PG_FICTITIOUS)) {
402 			if ((p->flags & PG_CLEAN) &&
403 			    pmap_is_modified(VM_PAGE_TO_PHYS(p)))
404 				p->flags &= ~PG_CLEAN;
405 			/*
406 			 * Remove the page from any paging queue.
407 			 * This needs to be done if either we have been
408 			 * explicitly asked to do so or it is about to
409 			 * be cleaned (see comment below).
410 			 */
411 			if (de_queue || !(p->flags & PG_CLEAN)) {
412 				vm_page_lock_queues();
413 				if (p->flags & PG_ACTIVE) {
414 					TAILQ_REMOVE(&vm_page_queue_active,
415 						     p, pageq);
416 					p->flags &= ~PG_ACTIVE;
417 					cnt.v_active_count--;
418 					onqueue = 1;
419 				} else if (p->flags & PG_INACTIVE) {
420 					TAILQ_REMOVE(&vm_page_queue_inactive,
421 						     p, pageq);
422 					p->flags &= ~PG_INACTIVE;
423 					cnt.v_inactive_count--;
424 					onqueue = -1;
425 				} else
426 					onqueue = 0;
427 				vm_page_unlock_queues();
428 			}
429 			/*
430 			 * To ensure the state of the page doesn't change
431 			 * during the clean operation we do two things.
432 			 * First we set the busy bit and write-protect all
433 			 * mappings to ensure that write accesses to the
434 			 * page block (in vm_fault).  Second, we remove
435 			 * the page from any paging queue to foil the
436 			 * pageout daemon (vm_pageout_scan).
437 			 */
438 			pmap_page_protect(VM_PAGE_TO_PHYS(p), VM_PROT_READ);
439 			if (!(p->flags & PG_CLEAN)) {
440 				p->flags |= PG_BUSY;
441 				object->paging_in_progress++;
442 				vm_object_unlock(object);
443 				/*
444 				 * XXX if put fails we mark the page as
445 				 * clean to avoid an infinite loop.
446 				 * Will loose changes to the page.
447 				 */
448 				if (vm_pager_put(object->pager, p, syncio)) {
449 					printf("%s: pager_put error\n",
450 					       "vm_object_page_clean");
451 					p->flags |= PG_CLEAN;
452 					noerror = FALSE;
453 				}
454 				vm_object_lock(object);
455 				object->paging_in_progress--;
456 				if (!de_queue && onqueue) {
457 					vm_page_lock_queues();
458 					if (onqueue > 0)
459 						vm_page_activate(p);
460 					else
461 						vm_page_deactivate(p);
462 					vm_page_unlock_queues();
463 				}
464 				p->flags &= ~PG_BUSY;
465 				PAGE_WAKEUP(p);
466 				goto again;
467 			}
468 		}
469 	}
470 	return (noerror);
471 }
472 
473 /*
474  *	vm_object_deactivate_pages
475  *
476  *	Deactivate all pages in the specified object.  (Keep its pages
477  *	in memory even though it is no longer referenced.)
478  *
479  *	The object must be locked.
480  */
481 void
vm_object_deactivate_pages(object)482 vm_object_deactivate_pages(object)
483 	register vm_object_t	object;
484 {
485 	register vm_page_t	p, next;
486 
487 	for (p = object->memq.tqh_first; p != NULL; p = next) {
488 		next = p->listq.tqe_next;
489 		vm_page_lock_queues();
490 		vm_page_deactivate(p);
491 		vm_page_unlock_queues();
492 	}
493 }
494 
495 /*
496  *	Trim the object cache to size.
497  */
498 void
vm_object_cache_trim()499 vm_object_cache_trim()
500 {
501 	register vm_object_t	object;
502 
503 	vm_object_cache_lock();
504 	while (vm_object_cached > vm_cache_max) {
505 		object = vm_object_cached_list.tqh_first;
506 		vm_object_cache_unlock();
507 
508 		if (object != vm_object_lookup(object->pager))
509 			panic("vm_object_deactivate: I'm sooo confused.");
510 
511 		pager_cache(object, FALSE);
512 
513 		vm_object_cache_lock();
514 	}
515 	vm_object_cache_unlock();
516 }
517 
518 /*
519  *	vm_object_pmap_copy:
520  *
521  *	Makes all physical pages in the specified
522  *	object range copy-on-write.  No writeable
523  *	references to these pages should remain.
524  *
525  *	The object must *not* be locked.
526  */
527 void
vm_object_pmap_copy(object,start,end)528 vm_object_pmap_copy(object, start, end)
529 	register vm_object_t	object;
530 	register vm_offset_t	start;
531 	register vm_offset_t	end;
532 {
533 	register vm_page_t	p;
534 
535 	if (object == NULL)
536 		return;
537 
538 	vm_object_lock(object);
539 	for (p = object->memq.tqh_first; p != NULL; p = p->listq.tqe_next) {
540 		if ((start <= p->offset) && (p->offset < end)) {
541 			pmap_page_protect(VM_PAGE_TO_PHYS(p), VM_PROT_READ);
542 			p->flags |= PG_COPYONWRITE;
543 		}
544 	}
545 	vm_object_unlock(object);
546 }
547 
548 /*
549  *	vm_object_pmap_remove:
550  *
551  *	Removes all physical pages in the specified
552  *	object range from all physical maps.
553  *
554  *	The object must *not* be locked.
555  */
556 void
vm_object_pmap_remove(object,start,end)557 vm_object_pmap_remove(object, start, end)
558 	register vm_object_t	object;
559 	register vm_offset_t	start;
560 	register vm_offset_t	end;
561 {
562 	register vm_page_t	p;
563 
564 	if (object == NULL)
565 		return;
566 
567 	vm_object_lock(object);
568 	for (p = object->memq.tqh_first; p != NULL; p = p->listq.tqe_next)
569 		if ((start <= p->offset) && (p->offset < end))
570 			pmap_page_protect(VM_PAGE_TO_PHYS(p), VM_PROT_NONE);
571 	vm_object_unlock(object);
572 }
573 
574 /*
575  *	vm_object_copy:
576  *
577  *	Create a new object which is a copy of an existing
578  *	object, and mark all of the pages in the existing
579  *	object 'copy-on-write'.  The new object has one reference.
580  *	Returns the new object.
581  *
582  *	May defer the copy until later if the object is not backed
583  *	up by a non-default pager.
584  */
585 void
vm_object_copy(src_object,src_offset,size,dst_object,dst_offset,src_needs_copy)586 vm_object_copy(src_object, src_offset, size,
587 		    dst_object, dst_offset, src_needs_copy)
588 	register vm_object_t	src_object;
589 	vm_offset_t		src_offset;
590 	vm_size_t		size;
591 	vm_object_t		*dst_object;	/* OUT */
592 	vm_offset_t		*dst_offset;	/* OUT */
593 	boolean_t		*src_needs_copy;	/* OUT */
594 {
595 	register vm_object_t	new_copy;
596 	register vm_object_t	old_copy;
597 	vm_offset_t		new_start, new_end;
598 
599 	register vm_page_t	p;
600 
601 	if (src_object == NULL) {
602 		/*
603 		 *	Nothing to copy
604 		 */
605 		*dst_object = NULL;
606 		*dst_offset = 0;
607 		*src_needs_copy = FALSE;
608 		return;
609 	}
610 
611 	/*
612 	 *	If the object's pager is null_pager or the
613 	 *	default pager, we don't have to make a copy
614 	 *	of it.  Instead, we set the needs copy flag and
615 	 *	make a shadow later.
616 	 */
617 
618 	vm_object_lock(src_object);
619 	if (src_object->pager == NULL ||
620 	    (src_object->flags & OBJ_INTERNAL)) {
621 
622 		/*
623 		 *	Make another reference to the object
624 		 */
625 		src_object->ref_count++;
626 
627 		/*
628 		 *	Mark all of the pages copy-on-write.
629 		 */
630 		for (p = src_object->memq.tqh_first; p; p = p->listq.tqe_next)
631 			if (src_offset <= p->offset &&
632 			    p->offset < src_offset + size)
633 				p->flags |= PG_COPYONWRITE;
634 		vm_object_unlock(src_object);
635 
636 		*dst_object = src_object;
637 		*dst_offset = src_offset;
638 
639 		/*
640 		 *	Must make a shadow when write is desired
641 		 */
642 		*src_needs_copy = TRUE;
643 		return;
644 	}
645 
646 	/*
647 	 *	Try to collapse the object before copying it.
648 	 */
649 	vm_object_collapse(src_object);
650 
651 	/*
652 	 *	If the object has a pager, the pager wants to
653 	 *	see all of the changes.  We need a copy-object
654 	 *	for the changed pages.
655 	 *
656 	 *	If there is a copy-object, and it is empty,
657 	 *	no changes have been made to the object since the
658 	 *	copy-object was made.  We can use the same copy-
659 	 *	object.
660 	 */
661 
662     Retry1:
663 	old_copy = src_object->copy;
664 	if (old_copy != NULL) {
665 		/*
666 		 *	Try to get the locks (out of order)
667 		 */
668 		if (!vm_object_lock_try(old_copy)) {
669 			vm_object_unlock(src_object);
670 
671 			/* should spin a bit here... */
672 			vm_object_lock(src_object);
673 			goto Retry1;
674 		}
675 
676 		if (old_copy->resident_page_count == 0 &&
677 		    old_copy->pager == NULL) {
678 			/*
679 			 *	Return another reference to
680 			 *	the existing copy-object.
681 			 */
682 			old_copy->ref_count++;
683 			vm_object_unlock(old_copy);
684 			vm_object_unlock(src_object);
685 			*dst_object = old_copy;
686 			*dst_offset = src_offset;
687 			*src_needs_copy = FALSE;
688 			return;
689 		}
690 		vm_object_unlock(old_copy);
691 	}
692 	vm_object_unlock(src_object);
693 
694 	/*
695 	 *	If the object has a pager, the pager wants
696 	 *	to see all of the changes.  We must make
697 	 *	a copy-object and put the changed pages there.
698 	 *
699 	 *	The copy-object is always made large enough to
700 	 *	completely shadow the original object, since
701 	 *	it may have several users who want to shadow
702 	 *	the original object at different points.
703 	 */
704 
705 	new_copy = vm_object_allocate(src_object->size);
706 
707     Retry2:
708 	vm_object_lock(src_object);
709 	/*
710 	 *	Copy object may have changed while we were unlocked
711 	 */
712 	old_copy = src_object->copy;
713 	if (old_copy != NULL) {
714 		/*
715 		 *	Try to get the locks (out of order)
716 		 */
717 		if (!vm_object_lock_try(old_copy)) {
718 			vm_object_unlock(src_object);
719 			goto Retry2;
720 		}
721 
722 		/*
723 		 *	Consistency check
724 		 */
725 		if (old_copy->shadow != src_object ||
726 		    old_copy->shadow_offset != (vm_offset_t) 0)
727 			panic("vm_object_copy: copy/shadow inconsistency");
728 
729 		/*
730 		 *	Make the old copy-object shadow the new one.
731 		 *	It will receive no more pages from the original
732 		 *	object.
733 		 */
734 
735 		src_object->ref_count--;	/* remove ref. from old_copy */
736 		old_copy->shadow = new_copy;
737 		new_copy->ref_count++;		/* locking not needed - we
738 						   have the only pointer */
739 		vm_object_unlock(old_copy);	/* done with old_copy */
740 	}
741 
742 	new_start = (vm_offset_t) 0;	/* always shadow original at 0 */
743 	new_end   = (vm_offset_t) new_copy->size; /* for the whole object */
744 
745 	/*
746 	 *	Point the new copy at the existing object.
747 	 */
748 
749 	new_copy->shadow = src_object;
750 	new_copy->shadow_offset = new_start;
751 	src_object->ref_count++;
752 	src_object->copy = new_copy;
753 
754 	/*
755 	 *	Mark all the affected pages of the existing object
756 	 *	copy-on-write.
757 	 */
758 	for (p = src_object->memq.tqh_first; p != NULL; p = p->listq.tqe_next)
759 		if ((new_start <= p->offset) && (p->offset < new_end))
760 			p->flags |= PG_COPYONWRITE;
761 
762 	vm_object_unlock(src_object);
763 
764 	*dst_object = new_copy;
765 	*dst_offset = src_offset - new_start;
766 	*src_needs_copy = FALSE;
767 }
768 
769 /*
770  *	vm_object_shadow:
771  *
772  *	Create a new object which is backed by the
773  *	specified existing object range.  The source
774  *	object reference is deallocated.
775  *
776  *	The new object and offset into that object
777  *	are returned in the source parameters.
778  */
779 
780 void
vm_object_shadow(object,offset,length)781 vm_object_shadow(object, offset, length)
782 	vm_object_t	*object;	/* IN/OUT */
783 	vm_offset_t	*offset;	/* IN/OUT */
784 	vm_size_t	length;
785 {
786 	register vm_object_t	source;
787 	register vm_object_t	result;
788 
789 	source = *object;
790 
791 	/*
792 	 *	Allocate a new object with the given length
793 	 */
794 
795 	if ((result = vm_object_allocate(length)) == NULL)
796 		panic("vm_object_shadow: no object for shadowing");
797 
798 	/*
799 	 *	The new object shadows the source object, adding
800 	 *	a reference to it.  Our caller changes his reference
801 	 *	to point to the new object, removing a reference to
802 	 *	the source object.  Net result: no change of reference
803 	 *	count.
804 	 */
805 	result->shadow = source;
806 
807 	/*
808 	 *	Store the offset into the source object,
809 	 *	and fix up the offset into the new object.
810 	 */
811 
812 	result->shadow_offset = *offset;
813 
814 	/*
815 	 *	Return the new things
816 	 */
817 
818 	*offset = 0;
819 	*object = result;
820 }
821 
822 /*
823  *	Set the specified object's pager to the specified pager.
824  */
825 
826 void
vm_object_setpager(object,pager,paging_offset,read_only)827 vm_object_setpager(object, pager, paging_offset,
828 			read_only)
829 	vm_object_t	object;
830 	vm_pager_t	pager;
831 	vm_offset_t	paging_offset;
832 	boolean_t	read_only;
833 {
834 #ifdef	lint
835 	read_only++;	/* No longer used */
836 #endif
837 
838 	vm_object_lock(object);			/* XXX ? */
839 	object->pager = pager;
840 	object->paging_offset = paging_offset;
841 	vm_object_unlock(object);			/* XXX ? */
842 }
843 
844 /*
845  *	vm_object_hash hashes the pager/id pair.
846  */
847 
848 #define vm_object_hash(pager) \
849 	(((unsigned long)pager)%VM_OBJECT_HASH_COUNT)
850 
851 /*
852  *	vm_object_lookup looks in the object cache for an object with the
853  *	specified pager and paging id.
854  */
855 
856 vm_object_t
vm_object_lookup(pager)857 vm_object_lookup(pager)
858 	vm_pager_t	pager;
859 {
860 	register vm_object_hash_entry_t	entry;
861 	vm_object_t			object;
862 
863 	vm_object_cache_lock();
864 
865 	for (entry = vm_object_hashtable[vm_object_hash(pager)].tqh_first;
866 	     entry != NULL;
867 	     entry = entry->hash_links.tqe_next) {
868 		object = entry->object;
869 		if (object->pager == pager) {
870 			vm_object_lock(object);
871 			if (object->ref_count == 0) {
872 				TAILQ_REMOVE(&vm_object_cached_list, object,
873 					cached_list);
874 				vm_object_cached--;
875 			}
876 			object->ref_count++;
877 			vm_object_unlock(object);
878 			vm_object_cache_unlock();
879 			return(object);
880 		}
881 	}
882 
883 	vm_object_cache_unlock();
884 	return(NULL);
885 }
886 
887 /*
888  *	vm_object_enter enters the specified object/pager/id into
889  *	the hash table.
890  */
891 
892 void
vm_object_enter(object,pager)893 vm_object_enter(object, pager)
894 	vm_object_t	object;
895 	vm_pager_t	pager;
896 {
897 	struct vm_object_hash_head	*bucket;
898 	register vm_object_hash_entry_t	entry;
899 
900 	/*
901 	 *	We don't cache null objects, and we can't cache
902 	 *	objects with the null pager.
903 	 */
904 
905 	if (object == NULL)
906 		return;
907 	if (pager == NULL)
908 		return;
909 
910 	bucket = &vm_object_hashtable[vm_object_hash(pager)];
911 	entry = (vm_object_hash_entry_t)
912 		malloc((u_long)sizeof *entry, M_VMOBJHASH, M_WAITOK);
913 	entry->object = object;
914 	object->flags |= OBJ_CANPERSIST;
915 
916 	vm_object_cache_lock();
917 	TAILQ_INSERT_TAIL(bucket, entry, hash_links);
918 	vm_object_cache_unlock();
919 }
920 
921 /*
922  *	vm_object_remove:
923  *
924  *	Remove the pager from the hash table.
925  *	Note:  This assumes that the object cache
926  *	is locked.  XXX this should be fixed
927  *	by reorganizing vm_object_deallocate.
928  */
929 void
vm_object_remove(pager)930 vm_object_remove(pager)
931 	register vm_pager_t	pager;
932 {
933 	struct vm_object_hash_head	*bucket;
934 	register vm_object_hash_entry_t	entry;
935 	register vm_object_t		object;
936 
937 	bucket = &vm_object_hashtable[vm_object_hash(pager)];
938 
939 	for (entry = bucket->tqh_first;
940 	     entry != NULL;
941 	     entry = entry->hash_links.tqe_next) {
942 		object = entry->object;
943 		if (object->pager == pager) {
944 			TAILQ_REMOVE(bucket, entry, hash_links);
945 			free((caddr_t)entry, M_VMOBJHASH);
946 			break;
947 		}
948 	}
949 }
950 
951 /*
952  *	vm_object_cache_clear removes all objects from the cache.
953  *
954  */
955 void
vm_object_cache_clear()956 vm_object_cache_clear()
957 {
958 	register vm_object_t	object;
959 
960 	/*
961 	 *	Remove each object in the cache by scanning down the
962 	 *	list of cached objects.
963 	 */
964 	vm_object_cache_lock();
965 	while ((object = vm_object_cached_list.tqh_first) != NULL) {
966 		vm_object_cache_unlock();
967 
968 		/*
969 		 * Note: it is important that we use vm_object_lookup
970 		 * to gain a reference, and not vm_object_reference, because
971 		 * the logic for removing an object from the cache lies in
972 		 * lookup.
973 		 */
974 		if (object != vm_object_lookup(object->pager))
975 			panic("vm_object_cache_clear: I'm sooo confused.");
976 		pager_cache(object, FALSE);
977 
978 		vm_object_cache_lock();
979 	}
980 	vm_object_cache_unlock();
981 }
982 
983 boolean_t	vm_object_collapse_allowed = TRUE;
984 /*
985  *	vm_object_collapse:
986  *
987  *	Collapse an object with the object backing it.
988  *	Pages in the backing object are moved into the
989  *	parent, and the backing object is deallocated.
990  *
991  *	Requires that the object be locked and the page
992  *	queues be unlocked.
993  *
994  */
995 void
vm_object_collapse(object)996 vm_object_collapse(object)
997 	register vm_object_t	object;
998 
999 {
1000 	register vm_object_t	backing_object;
1001 	register vm_offset_t	backing_offset;
1002 	register vm_size_t	size;
1003 	register vm_offset_t	new_offset;
1004 	register vm_page_t	p, pp;
1005 
1006 	if (!vm_object_collapse_allowed)
1007 		return;
1008 
1009 	while (TRUE) {
1010 		/*
1011 		 *	Verify that the conditions are right for collapse:
1012 		 *
1013 		 *	The object exists and no pages in it are currently
1014 		 *	being paged out (or have ever been paged out).
1015 		 */
1016 		if (object == NULL ||
1017 		    object->paging_in_progress != 0 ||
1018 		    object->pager != NULL)
1019 			return;
1020 
1021 		/*
1022 		 *		There is a backing object, and
1023 		 */
1024 
1025 		if ((backing_object = object->shadow) == NULL)
1026 			return;
1027 
1028 		vm_object_lock(backing_object);
1029 		/*
1030 		 *	...
1031 		 *		The backing object is not read_only,
1032 		 *		and no pages in the backing object are
1033 		 *		currently being paged out.
1034 		 *		The backing object is internal.
1035 		 */
1036 
1037 		if ((backing_object->flags & OBJ_INTERNAL) == 0 ||
1038 		    backing_object->paging_in_progress != 0) {
1039 			vm_object_unlock(backing_object);
1040 			return;
1041 		}
1042 
1043 		/*
1044 		 *	The backing object can't be a copy-object:
1045 		 *	the shadow_offset for the copy-object must stay
1046 		 *	as 0.  Furthermore (for the 'we have all the
1047 		 *	pages' case), if we bypass backing_object and
1048 		 *	just shadow the next object in the chain, old
1049 		 *	pages from that object would then have to be copied
1050 		 *	BOTH into the (former) backing_object and into the
1051 		 *	parent object.
1052 		 */
1053 		if (backing_object->shadow != NULL &&
1054 		    backing_object->shadow->copy != NULL) {
1055 			vm_object_unlock(backing_object);
1056 			return;
1057 		}
1058 
1059 		/*
1060 		 *	We know that we can either collapse the backing
1061 		 *	object (if the parent is the only reference to
1062 		 *	it) or (perhaps) remove the parent's reference
1063 		 *	to it.
1064 		 */
1065 
1066 		backing_offset = object->shadow_offset;
1067 		size = object->size;
1068 
1069 		/*
1070 		 *	If there is exactly one reference to the backing
1071 		 *	object, we can collapse it into the parent.
1072 		 */
1073 
1074 		if (backing_object->ref_count == 1) {
1075 
1076 			/*
1077 			 *	We can collapse the backing object.
1078 			 *
1079 			 *	Move all in-memory pages from backing_object
1080 			 *	to the parent.  Pages that have been paged out
1081 			 *	will be overwritten by any of the parent's
1082 			 *	pages that shadow them.
1083 			 */
1084 
1085 			while ((p = backing_object->memq.tqh_first) != NULL) {
1086 				new_offset = (p->offset - backing_offset);
1087 
1088 				/*
1089 				 *	If the parent has a page here, or if
1090 				 *	this page falls outside the parent,
1091 				 *	dispose of it.
1092 				 *
1093 				 *	Otherwise, move it as planned.
1094 				 */
1095 
1096 				if (p->offset < backing_offset ||
1097 				    new_offset >= size) {
1098 					vm_page_lock_queues();
1099 					vm_page_free(p);
1100 					vm_page_unlock_queues();
1101 				} else {
1102 				    pp = vm_page_lookup(object, new_offset);
1103 				    if (pp != NULL && !(pp->flags & PG_FAKE)) {
1104 					vm_page_lock_queues();
1105 					vm_page_free(p);
1106 					vm_page_unlock_queues();
1107 				    }
1108 				    else {
1109 					if (pp) {
1110 					    /* may be someone waiting for it */
1111 					    PAGE_WAKEUP(pp);
1112 					    vm_page_lock_queues();
1113 					    vm_page_free(pp);
1114 					    vm_page_unlock_queues();
1115 					}
1116 					vm_page_rename(p, object, new_offset);
1117 				    }
1118 				}
1119 			}
1120 
1121 			/*
1122 			 *	Move the pager from backing_object to object.
1123 			 *
1124 			 *	XXX We're only using part of the paging space
1125 			 *	for keeps now... we ought to discard the
1126 			 *	unused portion.
1127 			 */
1128 
1129 			if (backing_object->pager) {
1130 				object->pager = backing_object->pager;
1131 				object->paging_offset = backing_offset +
1132 					backing_object->paging_offset;
1133 				backing_object->pager = NULL;
1134 			}
1135 
1136 			/*
1137 			 *	Object now shadows whatever backing_object did.
1138 			 *	Note that the reference to backing_object->shadow
1139 			 *	moves from within backing_object to within object.
1140 			 */
1141 
1142 			object->shadow = backing_object->shadow;
1143 			object->shadow_offset += backing_object->shadow_offset;
1144 			if (object->shadow != NULL &&
1145 			    object->shadow->copy != NULL) {
1146 				panic("vm_object_collapse: we collapsed a copy-object!");
1147 			}
1148 			/*
1149 			 *	Discard backing_object.
1150 			 *
1151 			 *	Since the backing object has no pages, no
1152 			 *	pager left, and no object references within it,
1153 			 *	all that is necessary is to dispose of it.
1154 			 */
1155 
1156 			vm_object_unlock(backing_object);
1157 
1158 			simple_lock(&vm_object_list_lock);
1159 			TAILQ_REMOVE(&vm_object_list, backing_object,
1160 			    object_list);
1161 			vm_object_count--;
1162 			simple_unlock(&vm_object_list_lock);
1163 
1164 			free((caddr_t)backing_object, M_VMOBJ);
1165 
1166 			object_collapses++;
1167 		}
1168 		else {
1169 			/*
1170 			 *	If all of the pages in the backing object are
1171 			 *	shadowed by the parent object, the parent
1172 			 *	object no longer has to shadow the backing
1173 			 *	object; it can shadow the next one in the
1174 			 *	chain.
1175 			 *
1176 			 *	The backing object must not be paged out - we'd
1177 			 *	have to check all of the paged-out pages, as
1178 			 *	well.
1179 			 */
1180 
1181 			if (backing_object->pager != NULL) {
1182 				vm_object_unlock(backing_object);
1183 				return;
1184 			}
1185 
1186 			/*
1187 			 *	Should have a check for a 'small' number
1188 			 *	of pages here.
1189 			 */
1190 
1191 			for (p = backing_object->memq.tqh_first;
1192 			     p != NULL;
1193 			     p = p->listq.tqe_next) {
1194 				new_offset = (p->offset - backing_offset);
1195 
1196 				/*
1197 				 *	If the parent has a page here, or if
1198 				 *	this page falls outside the parent,
1199 				 *	keep going.
1200 				 *
1201 				 *	Otherwise, the backing_object must be
1202 				 *	left in the chain.
1203 				 */
1204 
1205 				if (p->offset >= backing_offset &&
1206 				    new_offset < size &&
1207 				    ((pp = vm_page_lookup(object, new_offset))
1208 				      == NULL ||
1209 				     (pp->flags & PG_FAKE))) {
1210 					/*
1211 					 *	Page still needed.
1212 					 *	Can't go any further.
1213 					 */
1214 					vm_object_unlock(backing_object);
1215 					return;
1216 				}
1217 			}
1218 
1219 			/*
1220 			 *	Make the parent shadow the next object
1221 			 *	in the chain.  Deallocating backing_object
1222 			 *	will not remove it, since its reference
1223 			 *	count is at least 2.
1224 			 */
1225 
1226 			object->shadow = backing_object->shadow;
1227 			vm_object_reference(object->shadow);
1228 			object->shadow_offset += backing_object->shadow_offset;
1229 
1230 			/*
1231 			 *	Backing object might have had a copy pointer
1232 			 *	to us.  If it did, clear it.
1233 			 */
1234 			if (backing_object->copy == object) {
1235 				backing_object->copy = NULL;
1236 			}
1237 
1238 			/*	Drop the reference count on backing_object.
1239 			 *	Since its ref_count was at least 2, it
1240 			 *	will not vanish; so we don't need to call
1241 			 *	vm_object_deallocate.
1242 			 */
1243 			backing_object->ref_count--;
1244 			vm_object_unlock(backing_object);
1245 
1246 			object_bypasses ++;
1247 
1248 		}
1249 
1250 		/*
1251 		 *	Try again with this object's new backing object.
1252 		 */
1253 	}
1254 }
1255 
1256 /*
1257  *	vm_object_page_remove: [internal]
1258  *
1259  *	Removes all physical pages in the specified
1260  *	object range from the object's list of pages.
1261  *
1262  *	The object must be locked.
1263  */
1264 void
vm_object_page_remove(object,start,end)1265 vm_object_page_remove(object, start, end)
1266 	register vm_object_t	object;
1267 	register vm_offset_t	start;
1268 	register vm_offset_t	end;
1269 {
1270 	register vm_page_t	p, next;
1271 
1272 	if (object == NULL)
1273 		return;
1274 
1275 	for (p = object->memq.tqh_first; p != NULL; p = next) {
1276 		next = p->listq.tqe_next;
1277 		if ((start <= p->offset) && (p->offset < end)) {
1278 			pmap_page_protect(VM_PAGE_TO_PHYS(p), VM_PROT_NONE);
1279 			vm_page_lock_queues();
1280 			vm_page_free(p);
1281 			vm_page_unlock_queues();
1282 		}
1283 	}
1284 }
1285 
1286 /*
1287  *	Routine:	vm_object_coalesce
1288  *	Function:	Coalesces two objects backing up adjoining
1289  *			regions of memory into a single object.
1290  *
1291  *	returns TRUE if objects were combined.
1292  *
1293  *	NOTE:	Only works at the moment if the second object is NULL -
1294  *		if it's not, which object do we lock first?
1295  *
1296  *	Parameters:
1297  *		prev_object	First object to coalesce
1298  *		prev_offset	Offset into prev_object
1299  *		next_object	Second object into coalesce
1300  *		next_offset	Offset into next_object
1301  *
1302  *		prev_size	Size of reference to prev_object
1303  *		next_size	Size of reference to next_object
1304  *
1305  *	Conditions:
1306  *	The object must *not* be locked.
1307  */
1308 boolean_t
vm_object_coalesce(prev_object,next_object,prev_offset,next_offset,prev_size,next_size)1309 vm_object_coalesce(prev_object, next_object,
1310 			prev_offset, next_offset,
1311 			prev_size, next_size)
1312 
1313 	register vm_object_t	prev_object;
1314 	vm_object_t	next_object;
1315 	vm_offset_t	prev_offset, next_offset;
1316 	vm_size_t	prev_size, next_size;
1317 {
1318 	vm_size_t	newsize;
1319 
1320 #ifdef	lint
1321 	next_offset++;
1322 #endif
1323 
1324 	if (next_object != NULL) {
1325 		return(FALSE);
1326 	}
1327 
1328 	if (prev_object == NULL) {
1329 		return(TRUE);
1330 	}
1331 
1332 	vm_object_lock(prev_object);
1333 
1334 	/*
1335 	 *	Try to collapse the object first
1336 	 */
1337 	vm_object_collapse(prev_object);
1338 
1339 	/*
1340 	 *	Can't coalesce if:
1341 	 *	. more than one reference
1342 	 *	. paged out
1343 	 *	. shadows another object
1344 	 *	. has a copy elsewhere
1345 	 *	(any of which mean that the pages not mapped to
1346 	 *	prev_entry may be in use anyway)
1347 	 */
1348 
1349 	if (prev_object->ref_count > 1 ||
1350 		prev_object->pager != NULL ||
1351 		prev_object->shadow != NULL ||
1352 		prev_object->copy != NULL) {
1353 		vm_object_unlock(prev_object);
1354 		return(FALSE);
1355 	}
1356 
1357 	/*
1358 	 *	Remove any pages that may still be in the object from
1359 	 *	a previous deallocation.
1360 	 */
1361 
1362 	vm_object_page_remove(prev_object,
1363 			prev_offset + prev_size,
1364 			prev_offset + prev_size + next_size);
1365 
1366 	/*
1367 	 *	Extend the object if necessary.
1368 	 */
1369 	newsize = prev_offset + prev_size + next_size;
1370 	if (newsize > prev_object->size)
1371 		prev_object->size = newsize;
1372 
1373 	vm_object_unlock(prev_object);
1374 	return(TRUE);
1375 }
1376 
1377 /*
1378  *	vm_object_print:	[ debug ]
1379  */
1380 void
vm_object_print(object,full)1381 vm_object_print(object, full)
1382 	vm_object_t	object;
1383 	boolean_t	full;
1384 {
1385 	register vm_page_t	p;
1386 	extern indent;
1387 
1388 	register int count;
1389 
1390 	if (object == NULL)
1391 		return;
1392 
1393 	iprintf("Object 0x%x: size=0x%x, res=%d, ref=%d, ",
1394 		(int) object, (int) object->size,
1395 		object->resident_page_count, object->ref_count);
1396 	printf("pager=0x%x+0x%x, shadow=(0x%x)+0x%x\n",
1397 	       (int) object->pager, (int) object->paging_offset,
1398 	       (int) object->shadow, (int) object->shadow_offset);
1399 	printf("cache: next=0x%x, prev=0x%x\n",
1400 	       object->cached_list.tqe_next, object->cached_list.tqe_prev);
1401 
1402 	if (!full)
1403 		return;
1404 
1405 	indent += 2;
1406 	count = 0;
1407 	for (p = object->memq.tqh_first; p != NULL; p = p->listq.tqe_next) {
1408 		if (count == 0)
1409 			iprintf("memory:=");
1410 		else if (count == 6) {
1411 			printf("\n");
1412 			iprintf(" ...");
1413 			count = 0;
1414 		} else
1415 			printf(",");
1416 		count++;
1417 
1418 		printf("(off=0x%x,page=0x%x)", p->offset, VM_PAGE_TO_PHYS(p));
1419 	}
1420 	if (count != 0)
1421 		printf("\n");
1422 	indent -= 2;
1423 }
1424