1 /* $NetBSD: tmpfs_subr.c,v 1.35 2007/07/09 21:10:50 ad Exp $ */
2
3 /*-
4 * SPDX-License-Identifier: BSD-2-Clause
5 *
6 * Copyright (c) 2005 The NetBSD Foundation, Inc.
7 * All rights reserved.
8 *
9 * This code is derived from software contributed to The NetBSD Foundation
10 * by Julio M. Merino Vidal, developed as part of Google's Summer of Code
11 * 2005 program.
12 *
13 * Redistribution and use in source and binary forms, with or without
14 * modification, are permitted provided that the following conditions
15 * are met:
16 * 1. Redistributions of source code must retain the above copyright
17 * notice, this list of conditions and the following disclaimer.
18 * 2. Redistributions in binary form must reproduce the above copyright
19 * notice, this list of conditions and the following disclaimer in the
20 * documentation and/or other materials provided with the distribution.
21 *
22 * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS
23 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
24 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
25 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS
26 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
27 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
28 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
29 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
30 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
31 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
32 * POSSIBILITY OF SUCH DAMAGE.
33 */
34
35 /*
36 * Efficient memory file system supporting functions.
37 */
38
39 #include <sys/param.h>
40 #include <sys/systm.h>
41 #include <sys/dirent.h>
42 #include <sys/fnv_hash.h>
43 #include <sys/lock.h>
44 #include <sys/limits.h>
45 #include <sys/mount.h>
46 #include <sys/namei.h>
47 #include <sys/priv.h>
48 #include <sys/proc.h>
49 #include <sys/random.h>
50 #include <sys/refcount.h>
51 #include <sys/rwlock.h>
52 #include <sys/smr.h>
53 #include <sys/stat.h>
54 #include <sys/sysctl.h>
55 #include <sys/user.h>
56 #include <sys/vnode.h>
57 #include <sys/vmmeter.h>
58
59 #include <vm/vm.h>
60 #include <vm/vm_param.h>
61 #include <vm/vm_object.h>
62 #include <vm/vm_page.h>
63 #include <vm/vm_pageout.h>
64 #include <vm/vm_pager.h>
65 #include <vm/vm_extern.h>
66 #include <vm/swap_pager.h>
67 #include <vm/uma.h>
68
69 #include <fs/tmpfs/tmpfs.h>
70 #include <fs/tmpfs/tmpfs_fifoops.h>
71 #include <fs/tmpfs/tmpfs_vnops.h>
72
73 SYSCTL_NODE(_vfs, OID_AUTO, tmpfs, CTLFLAG_RW | CTLFLAG_MPSAFE, 0,
74 "tmpfs file system");
75
76 static long tmpfs_pages_reserved = TMPFS_PAGES_MINRESERVED;
77 static long tmpfs_pages_avail_init;
78 static int tmpfs_mem_percent = TMPFS_MEM_PERCENT;
79 static void tmpfs_set_reserve_from_percent(void);
80
81 MALLOC_DEFINE(M_TMPFSDIR, "tmpfs dir", "tmpfs dirent structure");
82 static uma_zone_t tmpfs_node_pool;
83 VFS_SMR_DECLARE;
84
85 int tmpfs_pager_type = -1;
86
87 static vm_object_t
tmpfs_pager_alloc(void * handle,vm_ooffset_t size,vm_prot_t prot,vm_ooffset_t offset,struct ucred * cred)88 tmpfs_pager_alloc(void *handle, vm_ooffset_t size, vm_prot_t prot,
89 vm_ooffset_t offset, struct ucred *cred)
90 {
91 vm_object_t object;
92
93 MPASS(handle == NULL);
94 MPASS(offset == 0);
95 object = vm_object_allocate_dyn(tmpfs_pager_type, size,
96 OBJ_COLORED | OBJ_SWAP);
97 if (!swap_pager_init_object(object, NULL, NULL, size, 0)) {
98 vm_object_deallocate(object);
99 object = NULL;
100 }
101 return (object);
102 }
103
104 /*
105 * Make sure tmpfs vnodes with writable mappings can be found on the lazy list.
106 *
107 * This allows for periodic mtime updates while only scanning vnodes which are
108 * plausibly dirty, see tmpfs_update_mtime_lazy.
109 */
110 static void
tmpfs_pager_writecount_recalc(vm_object_t object,vm_offset_t old,vm_offset_t new)111 tmpfs_pager_writecount_recalc(vm_object_t object, vm_offset_t old,
112 vm_offset_t new)
113 {
114 struct vnode *vp;
115
116 VM_OBJECT_ASSERT_WLOCKED(object);
117
118 vp = VM_TO_TMPFS_VP(object);
119
120 /*
121 * Forced unmount?
122 */
123 if (vp == NULL || vp->v_object == NULL) {
124 KASSERT((object->flags & OBJ_TMPFS_VREF) == 0,
125 ("object %p with OBJ_TMPFS_VREF but without vnode",
126 object));
127 VM_OBJECT_WUNLOCK(object);
128 return;
129 }
130
131 if (old == 0) {
132 VNASSERT((object->flags & OBJ_TMPFS_VREF) == 0, vp,
133 ("object without writable mappings has a reference"));
134 VNPASS(vp->v_usecount > 0, vp);
135 } else {
136 VNASSERT((object->flags & OBJ_TMPFS_VREF) != 0, vp,
137 ("object with writable mappings does not "
138 "have a reference"));
139 }
140
141 if (old == new) {
142 VM_OBJECT_WUNLOCK(object);
143 return;
144 }
145
146 if (new == 0) {
147 vm_object_clear_flag(object, OBJ_TMPFS_VREF);
148 VM_OBJECT_WUNLOCK(object);
149 vrele(vp);
150 } else {
151 if ((object->flags & OBJ_TMPFS_VREF) == 0) {
152 vref(vp);
153 vlazy(vp);
154 vm_object_set_flag(object, OBJ_TMPFS_VREF);
155 }
156 VM_OBJECT_WUNLOCK(object);
157 }
158 }
159
160 static void
tmpfs_pager_update_writecount(vm_object_t object,vm_offset_t start,vm_offset_t end)161 tmpfs_pager_update_writecount(vm_object_t object, vm_offset_t start,
162 vm_offset_t end)
163 {
164 vm_offset_t new, old;
165
166 VM_OBJECT_WLOCK(object);
167 KASSERT((object->flags & OBJ_ANON) == 0,
168 ("%s: object %p with OBJ_ANON", __func__, object));
169 old = object->un_pager.swp.writemappings;
170 object->un_pager.swp.writemappings += (vm_ooffset_t)end - start;
171 new = object->un_pager.swp.writemappings;
172 tmpfs_pager_writecount_recalc(object, old, new);
173 VM_OBJECT_ASSERT_UNLOCKED(object);
174 }
175
176 static void
tmpfs_pager_release_writecount(vm_object_t object,vm_offset_t start,vm_offset_t end)177 tmpfs_pager_release_writecount(vm_object_t object, vm_offset_t start,
178 vm_offset_t end)
179 {
180 vm_offset_t new, old;
181
182 VM_OBJECT_WLOCK(object);
183 KASSERT((object->flags & OBJ_ANON) == 0,
184 ("%s: object %p with OBJ_ANON", __func__, object));
185 old = object->un_pager.swp.writemappings;
186 KASSERT(old >= (vm_ooffset_t)end - start,
187 ("tmpfs obj %p writecount %jx dec %jx", object, (uintmax_t)old,
188 (uintmax_t)((vm_ooffset_t)end - start)));
189 object->un_pager.swp.writemappings -= (vm_ooffset_t)end - start;
190 new = object->un_pager.swp.writemappings;
191 tmpfs_pager_writecount_recalc(object, old, new);
192 VM_OBJECT_ASSERT_UNLOCKED(object);
193 }
194
195 static void
tmpfs_pager_getvp(vm_object_t object,struct vnode ** vpp,bool * vp_heldp)196 tmpfs_pager_getvp(vm_object_t object, struct vnode **vpp, bool *vp_heldp)
197 {
198 struct vnode *vp;
199
200 /*
201 * Tmpfs VREG node, which was reclaimed, has tmpfs_pager_type
202 * type. In this case there is no v_writecount to adjust.
203 */
204 if (vp_heldp != NULL)
205 VM_OBJECT_RLOCK(object);
206 else
207 VM_OBJECT_ASSERT_LOCKED(object);
208 if ((object->flags & OBJ_TMPFS) != 0) {
209 vp = VM_TO_TMPFS_VP(object);
210 if (vp != NULL) {
211 *vpp = vp;
212 if (vp_heldp != NULL) {
213 vhold(vp);
214 *vp_heldp = true;
215 }
216 }
217 }
218 if (vp_heldp != NULL)
219 VM_OBJECT_RUNLOCK(object);
220 }
221
222 static void
tmpfs_pager_freespace(vm_object_t obj,vm_pindex_t start,vm_size_t size)223 tmpfs_pager_freespace(vm_object_t obj, vm_pindex_t start, vm_size_t size)
224 {
225 struct tmpfs_node *node;
226 struct tmpfs_mount *tm;
227 vm_size_t c;
228
229 swap_pager_freespace(obj, start, size, &c);
230 if ((obj->flags & OBJ_TMPFS) == 0 || c == 0)
231 return;
232
233 node = obj->un_pager.swp.swp_priv;
234 MPASS(node->tn_type == VREG);
235 tm = node->tn_reg.tn_tmp;
236
237 KASSERT(tm->tm_pages_used >= c,
238 ("tmpfs tm %p pages %jd free %jd", tm,
239 (uintmax_t)tm->tm_pages_used, (uintmax_t)c));
240 atomic_add_long(&tm->tm_pages_used, -c);
241 KASSERT(node->tn_reg.tn_pages >= c,
242 ("tmpfs node %p pages %jd free %jd", node,
243 (uintmax_t)node->tn_reg.tn_pages, (uintmax_t)c));
244 node->tn_reg.tn_pages -= c;
245 }
246
247 static void
tmpfs_page_inserted(vm_object_t obj,vm_page_t m)248 tmpfs_page_inserted(vm_object_t obj, vm_page_t m)
249 {
250 struct tmpfs_node *node;
251 struct tmpfs_mount *tm;
252
253 if ((obj->flags & OBJ_TMPFS) == 0)
254 return;
255
256 node = obj->un_pager.swp.swp_priv;
257 MPASS(node->tn_type == VREG);
258 tm = node->tn_reg.tn_tmp;
259
260 if (!vm_pager_has_page(obj, m->pindex, NULL, NULL)) {
261 atomic_add_long(&tm->tm_pages_used, 1);
262 node->tn_reg.tn_pages += 1;
263 }
264 }
265
266 static void
tmpfs_page_removed(vm_object_t obj,vm_page_t m)267 tmpfs_page_removed(vm_object_t obj, vm_page_t m)
268 {
269 struct tmpfs_node *node;
270 struct tmpfs_mount *tm;
271
272 if ((obj->flags & OBJ_TMPFS) == 0)
273 return;
274
275 node = obj->un_pager.swp.swp_priv;
276 MPASS(node->tn_type == VREG);
277 tm = node->tn_reg.tn_tmp;
278
279 if (!vm_pager_has_page(obj, m->pindex, NULL, NULL)) {
280 KASSERT(tm->tm_pages_used >= 1,
281 ("tmpfs tm %p pages %jd free 1", tm,
282 (uintmax_t)tm->tm_pages_used));
283 atomic_add_long(&tm->tm_pages_used, -1);
284 KASSERT(node->tn_reg.tn_pages >= 1,
285 ("tmpfs node %p pages %jd free 1", node,
286 (uintmax_t)node->tn_reg.tn_pages));
287 node->tn_reg.tn_pages -= 1;
288 }
289 }
290
291 static boolean_t
tmpfs_can_alloc_page(vm_object_t obj,vm_pindex_t pindex)292 tmpfs_can_alloc_page(vm_object_t obj, vm_pindex_t pindex)
293 {
294 struct tmpfs_mount *tm;
295
296 tm = VM_TO_TMPFS_MP(obj);
297 if (tm == NULL || vm_pager_has_page(obj, pindex, NULL, NULL) ||
298 tm->tm_pages_max == 0)
299 return (true);
300 if (tm->tm_pages_max == ULONG_MAX)
301 return (tmpfs_mem_avail() >= 1);
302 return (tm->tm_pages_max > atomic_load_long(&tm->tm_pages_used));
303 }
304
305 struct pagerops tmpfs_pager_ops = {
306 .pgo_kvme_type = KVME_TYPE_VNODE,
307 .pgo_alloc = tmpfs_pager_alloc,
308 .pgo_set_writeable_dirty = vm_object_set_writeable_dirty_,
309 .pgo_update_writecount = tmpfs_pager_update_writecount,
310 .pgo_release_writecount = tmpfs_pager_release_writecount,
311 .pgo_mightbedirty = vm_object_mightbedirty_,
312 .pgo_getvp = tmpfs_pager_getvp,
313 .pgo_freespace = tmpfs_pager_freespace,
314 .pgo_page_inserted = tmpfs_page_inserted,
315 .pgo_page_removed = tmpfs_page_removed,
316 .pgo_can_alloc_page = tmpfs_can_alloc_page,
317 };
318
319 static int
tmpfs_node_ctor(void * mem,int size,void * arg,int flags)320 tmpfs_node_ctor(void *mem, int size, void *arg, int flags)
321 {
322 struct tmpfs_node *node;
323
324 node = mem;
325 node->tn_gen++;
326 node->tn_size = 0;
327 node->tn_status = 0;
328 node->tn_accessed = false;
329 node->tn_flags = 0;
330 node->tn_links = 0;
331 node->tn_vnode = NULL;
332 node->tn_vpstate = 0;
333 return (0);
334 }
335
336 static void
tmpfs_node_dtor(void * mem,int size,void * arg)337 tmpfs_node_dtor(void *mem, int size, void *arg)
338 {
339 struct tmpfs_node *node;
340
341 node = mem;
342 node->tn_type = VNON;
343 }
344
345 static int
tmpfs_node_init(void * mem,int size,int flags)346 tmpfs_node_init(void *mem, int size, int flags)
347 {
348 struct tmpfs_node *node;
349
350 node = mem;
351 node->tn_id = 0;
352 mtx_init(&node->tn_interlock, "tmpfsni", NULL, MTX_DEF | MTX_NEW);
353 node->tn_gen = arc4random();
354 return (0);
355 }
356
357 static void
tmpfs_node_fini(void * mem,int size)358 tmpfs_node_fini(void *mem, int size)
359 {
360 struct tmpfs_node *node;
361
362 node = mem;
363 mtx_destroy(&node->tn_interlock);
364 }
365
366 int
tmpfs_subr_init(void)367 tmpfs_subr_init(void)
368 {
369 tmpfs_pager_type = vm_pager_alloc_dyn_type(&tmpfs_pager_ops,
370 OBJT_SWAP);
371 if (tmpfs_pager_type == -1)
372 return (EINVAL);
373 tmpfs_node_pool = uma_zcreate("TMPFS node",
374 sizeof(struct tmpfs_node), tmpfs_node_ctor, tmpfs_node_dtor,
375 tmpfs_node_init, tmpfs_node_fini, UMA_ALIGN_PTR, 0);
376 VFS_SMR_ZONE_SET(tmpfs_node_pool);
377
378 tmpfs_pages_avail_init = tmpfs_mem_avail();
379 tmpfs_set_reserve_from_percent();
380 return (0);
381 }
382
383 void
tmpfs_subr_uninit(void)384 tmpfs_subr_uninit(void)
385 {
386 if (tmpfs_pager_type != -1)
387 vm_pager_free_dyn_type(tmpfs_pager_type);
388 tmpfs_pager_type = -1;
389 uma_zdestroy(tmpfs_node_pool);
390 }
391
392 static int
sysctl_mem_reserved(SYSCTL_HANDLER_ARGS)393 sysctl_mem_reserved(SYSCTL_HANDLER_ARGS)
394 {
395 int error;
396 long pages, bytes;
397
398 pages = *(long *)arg1;
399 bytes = pages * PAGE_SIZE;
400
401 error = sysctl_handle_long(oidp, &bytes, 0, req);
402 if (error || !req->newptr)
403 return (error);
404
405 pages = bytes / PAGE_SIZE;
406 if (pages < TMPFS_PAGES_MINRESERVED)
407 return (EINVAL);
408
409 *(long *)arg1 = pages;
410 return (0);
411 }
412
413 SYSCTL_PROC(_vfs_tmpfs, OID_AUTO, memory_reserved,
414 CTLTYPE_LONG | CTLFLAG_MPSAFE | CTLFLAG_RW, &tmpfs_pages_reserved, 0,
415 sysctl_mem_reserved, "L",
416 "Amount of available memory and swap below which tmpfs growth stops");
417
418 static int
sysctl_mem_percent(SYSCTL_HANDLER_ARGS)419 sysctl_mem_percent(SYSCTL_HANDLER_ARGS)
420 {
421 int error, percent;
422
423 percent = *(int *)arg1;
424 error = sysctl_handle_int(oidp, &percent, 0, req);
425 if (error || !req->newptr)
426 return (error);
427
428 if ((unsigned) percent > 100)
429 return (EINVAL);
430
431 *(long *)arg1 = percent;
432 tmpfs_set_reserve_from_percent();
433 return (0);
434 }
435
436 static void
tmpfs_set_reserve_from_percent(void)437 tmpfs_set_reserve_from_percent(void)
438 {
439 size_t reserved;
440
441 reserved = tmpfs_pages_avail_init * (100 - tmpfs_mem_percent) / 100;
442 tmpfs_pages_reserved = max(reserved, TMPFS_PAGES_MINRESERVED);
443 }
444
445 SYSCTL_PROC(_vfs_tmpfs, OID_AUTO, memory_percent,
446 CTLTYPE_INT | CTLFLAG_MPSAFE | CTLFLAG_RW, &tmpfs_mem_percent, 0,
447 sysctl_mem_percent, "I",
448 "Percent of available memory that can be used if no size limit");
449
450 static __inline int tmpfs_dirtree_cmp(struct tmpfs_dirent *a,
451 struct tmpfs_dirent *b);
452 RB_PROTOTYPE_STATIC(tmpfs_dir, tmpfs_dirent, uh.td_entries, tmpfs_dirtree_cmp);
453
454 size_t
tmpfs_mem_avail(void)455 tmpfs_mem_avail(void)
456 {
457 size_t avail;
458 long reserved;
459
460 avail = swap_pager_avail + vm_free_count();
461 reserved = atomic_load_long(&tmpfs_pages_reserved);
462 if (__predict_false(avail < reserved))
463 return (0);
464 return (avail - reserved);
465 }
466
467 size_t
tmpfs_pages_used(struct tmpfs_mount * tmp)468 tmpfs_pages_used(struct tmpfs_mount *tmp)
469 {
470 const size_t node_size = sizeof(struct tmpfs_node) +
471 sizeof(struct tmpfs_dirent);
472 size_t meta_pages;
473
474 meta_pages = howmany((uintmax_t)tmp->tm_nodes_inuse * node_size,
475 PAGE_SIZE);
476 return (meta_pages + tmp->tm_pages_used);
477 }
478
479 bool
tmpfs_pages_check_avail(struct tmpfs_mount * tmp,size_t req_pages)480 tmpfs_pages_check_avail(struct tmpfs_mount *tmp, size_t req_pages)
481 {
482 if (tmpfs_mem_avail() < req_pages)
483 return (false);
484
485 if (tmp->tm_pages_max != ULONG_MAX &&
486 tmp->tm_pages_max < req_pages + tmpfs_pages_used(tmp))
487 return (false);
488
489 return (true);
490 }
491
492 static int
tmpfs_partial_page_invalidate(vm_object_t object,vm_pindex_t idx,int base,int end,boolean_t ignerr)493 tmpfs_partial_page_invalidate(vm_object_t object, vm_pindex_t idx, int base,
494 int end, boolean_t ignerr)
495 {
496 vm_page_t m;
497 int rv, error;
498
499 VM_OBJECT_ASSERT_WLOCKED(object);
500 KASSERT(base >= 0, ("%s: base %d", __func__, base));
501 KASSERT(end - base <= PAGE_SIZE, ("%s: base %d end %d", __func__, base,
502 end));
503 error = 0;
504
505 retry:
506 m = vm_page_grab(object, idx, VM_ALLOC_NOCREAT);
507 if (m != NULL) {
508 MPASS(vm_page_all_valid(m));
509 } else if (vm_pager_has_page(object, idx, NULL, NULL)) {
510 m = vm_page_alloc(object, idx, VM_ALLOC_NORMAL |
511 VM_ALLOC_WAITFAIL);
512 if (m == NULL)
513 goto retry;
514 vm_object_pip_add(object, 1);
515 VM_OBJECT_WUNLOCK(object);
516 rv = vm_pager_get_pages(object, &m, 1, NULL, NULL);
517 VM_OBJECT_WLOCK(object);
518 vm_object_pip_wakeup(object);
519 if (rv == VM_PAGER_OK) {
520 /*
521 * Since the page was not resident, and therefore not
522 * recently accessed, immediately enqueue it for
523 * asynchronous laundering. The current operation is
524 * not regarded as an access.
525 */
526 vm_page_launder(m);
527 } else {
528 vm_page_free(m);
529 m = NULL;
530 if (!ignerr)
531 error = EIO;
532 }
533 }
534 if (m != NULL) {
535 pmap_zero_page_area(m, base, end - base);
536 vm_page_set_dirty(m);
537 vm_page_xunbusy(m);
538 }
539
540 return (error);
541 }
542
543 void
tmpfs_ref_node(struct tmpfs_node * node)544 tmpfs_ref_node(struct tmpfs_node *node)
545 {
546 #ifdef INVARIANTS
547 u_int old;
548
549 old =
550 #endif
551 refcount_acquire(&node->tn_refcount);
552 #ifdef INVARIANTS
553 KASSERT(old > 0, ("node %p zero refcount", node));
554 #endif
555 }
556
557 /*
558 * Allocates a new node of type 'type' inside the 'tmp' mount point, with
559 * its owner set to 'uid', its group to 'gid' and its mode set to 'mode',
560 * using the credentials of the process 'p'.
561 *
562 * If the node type is set to 'VDIR', then the parent parameter must point
563 * to the parent directory of the node being created. It may only be NULL
564 * while allocating the root node.
565 *
566 * If the node type is set to 'VBLK' or 'VCHR', then the rdev parameter
567 * specifies the device the node represents.
568 *
569 * If the node type is set to 'VLNK', then the parameter target specifies
570 * the file name of the target file for the symbolic link that is being
571 * created.
572 *
573 * Note that new nodes are retrieved from the available list if it has
574 * items or, if it is empty, from the node pool as long as there is enough
575 * space to create them.
576 *
577 * Returns zero on success or an appropriate error code on failure.
578 */
579 int
tmpfs_alloc_node(struct mount * mp,struct tmpfs_mount * tmp,__enum_uint8 (vtype)type,uid_t uid,gid_t gid,mode_t mode,struct tmpfs_node * parent,const char * target,dev_t rdev,struct tmpfs_node ** node)580 tmpfs_alloc_node(struct mount *mp, struct tmpfs_mount *tmp, __enum_uint8(vtype) type,
581 uid_t uid, gid_t gid, mode_t mode, struct tmpfs_node *parent,
582 const char *target, dev_t rdev, struct tmpfs_node **node)
583 {
584 struct tmpfs_node *nnode;
585 char *symlink;
586 char symlink_smr;
587
588 /* If the root directory of the 'tmp' file system is not yet
589 * allocated, this must be the request to do it. */
590 MPASS(IMPLIES(tmp->tm_root == NULL, parent == NULL && type == VDIR));
591
592 MPASS((type == VLNK) ^ (target == NULL));
593 MPASS((type == VBLK || type == VCHR) ^ (rdev == VNOVAL));
594
595 if (tmp->tm_nodes_inuse >= tmp->tm_nodes_max)
596 return (ENOSPC);
597 if (!tmpfs_pages_check_avail(tmp, 1))
598 return (ENOSPC);
599
600 if ((mp->mnt_kern_flag & MNTK_UNMOUNT) != 0) {
601 /*
602 * When a new tmpfs node is created for fully
603 * constructed mount point, there must be a parent
604 * node, which vnode is locked exclusively. As
605 * consequence, if the unmount is executing in
606 * parallel, vflush() cannot reclaim the parent vnode.
607 * Due to this, the check for MNTK_UNMOUNT flag is not
608 * racy: if we did not see MNTK_UNMOUNT flag, then tmp
609 * cannot be destroyed until node construction is
610 * finished and the parent vnode unlocked.
611 *
612 * Tmpfs does not need to instantiate new nodes during
613 * unmount.
614 */
615 return (EBUSY);
616 }
617 if ((mp->mnt_kern_flag & MNT_RDONLY) != 0)
618 return (EROFS);
619
620 nnode = uma_zalloc_smr(tmpfs_node_pool, M_WAITOK);
621
622 /* Generic initialization. */
623 nnode->tn_type = type;
624 vfs_timestamp(&nnode->tn_atime);
625 nnode->tn_birthtime = nnode->tn_ctime = nnode->tn_mtime =
626 nnode->tn_atime;
627 nnode->tn_uid = uid;
628 nnode->tn_gid = gid;
629 nnode->tn_mode = mode;
630 nnode->tn_id = alloc_unr64(&tmp->tm_ino_unr);
631 nnode->tn_refcount = 1;
632 LIST_INIT(&nnode->tn_extattrs);
633
634 /* Type-specific initialization. */
635 switch (nnode->tn_type) {
636 case VBLK:
637 case VCHR:
638 nnode->tn_rdev = rdev;
639 break;
640
641 case VDIR:
642 RB_INIT(&nnode->tn_dir.tn_dirhead);
643 LIST_INIT(&nnode->tn_dir.tn_dupindex);
644 MPASS(parent != nnode);
645 MPASS(IMPLIES(parent == NULL, tmp->tm_root == NULL));
646 nnode->tn_dir.tn_parent = (parent == NULL) ? nnode : parent;
647 nnode->tn_dir.tn_readdir_lastn = 0;
648 nnode->tn_dir.tn_readdir_lastp = NULL;
649 nnode->tn_links++;
650 TMPFS_NODE_LOCK(nnode->tn_dir.tn_parent);
651 nnode->tn_dir.tn_parent->tn_links++;
652 TMPFS_NODE_UNLOCK(nnode->tn_dir.tn_parent);
653 break;
654
655 case VFIFO:
656 /* FALLTHROUGH */
657 case VSOCK:
658 break;
659
660 case VLNK:
661 MPASS(strlen(target) < MAXPATHLEN);
662 nnode->tn_size = strlen(target);
663
664 symlink = NULL;
665 if (!tmp->tm_nonc) {
666 symlink = cache_symlink_alloc(nnode->tn_size + 1,
667 M_WAITOK);
668 symlink_smr = true;
669 }
670 if (symlink == NULL) {
671 symlink = malloc(nnode->tn_size + 1, M_TMPFSNAME,
672 M_WAITOK);
673 symlink_smr = false;
674 }
675 memcpy(symlink, target, nnode->tn_size + 1);
676
677 /*
678 * Allow safe symlink resolving for lockless lookup.
679 * tmpfs_fplookup_symlink references this comment.
680 *
681 * 1. nnode is not yet visible to the world
682 * 2. both tn_link_target and tn_link_smr get populated
683 * 3. release fence publishes their content
684 * 4. tn_link_target content is immutable until node
685 * destruction, where the pointer gets set to NULL
686 * 5. tn_link_smr is never changed once set
687 *
688 * As a result it is sufficient to issue load consume
689 * on the node pointer to also get the above content
690 * in a stable manner. Worst case tn_link_smr flag
691 * may be set to true despite being stale, while the
692 * target buffer is already cleared out.
693 */
694 atomic_store_ptr(&nnode->tn_link_target, symlink);
695 atomic_store_char((char *)&nnode->tn_link_smr, symlink_smr);
696 atomic_thread_fence_rel();
697 break;
698
699 case VREG:
700 nnode->tn_reg.tn_aobj =
701 vm_pager_allocate(tmpfs_pager_type, NULL, 0,
702 VM_PROT_DEFAULT, 0,
703 NULL /* XXXKIB - tmpfs needs swap reservation */);
704 nnode->tn_reg.tn_aobj->un_pager.swp.swp_priv = nnode;
705 vm_object_set_flag(nnode->tn_reg.tn_aobj, OBJ_TMPFS);
706 nnode->tn_reg.tn_tmp = tmp;
707 nnode->tn_reg.tn_pages = 0;
708 break;
709
710 default:
711 panic("tmpfs_alloc_node: type %p %d", nnode,
712 (int)nnode->tn_type);
713 }
714
715 TMPFS_LOCK(tmp);
716 LIST_INSERT_HEAD(&tmp->tm_nodes_used, nnode, tn_entries);
717 nnode->tn_attached = true;
718 tmp->tm_nodes_inuse++;
719 tmp->tm_refcount++;
720 TMPFS_UNLOCK(tmp);
721
722 *node = nnode;
723 return (0);
724 }
725
726 /*
727 * Destroys the node pointed to by node from the file system 'tmp'.
728 * If the node references a directory, no entries are allowed.
729 */
730 void
tmpfs_free_node(struct tmpfs_mount * tmp,struct tmpfs_node * node)731 tmpfs_free_node(struct tmpfs_mount *tmp, struct tmpfs_node *node)
732 {
733 if (refcount_release_if_not_last(&node->tn_refcount))
734 return;
735
736 TMPFS_LOCK(tmp);
737 TMPFS_NODE_LOCK(node);
738 if (!tmpfs_free_node_locked(tmp, node, false)) {
739 TMPFS_NODE_UNLOCK(node);
740 TMPFS_UNLOCK(tmp);
741 }
742 }
743
744 bool
tmpfs_free_node_locked(struct tmpfs_mount * tmp,struct tmpfs_node * node,bool detach)745 tmpfs_free_node_locked(struct tmpfs_mount *tmp, struct tmpfs_node *node,
746 bool detach)
747 {
748 struct tmpfs_extattr *ea;
749 vm_object_t uobj;
750 char *symlink;
751 bool last;
752
753 TMPFS_MP_ASSERT_LOCKED(tmp);
754 TMPFS_NODE_ASSERT_LOCKED(node);
755
756 last = refcount_release(&node->tn_refcount);
757 if (node->tn_attached && (detach || last)) {
758 MPASS(tmp->tm_nodes_inuse > 0);
759 tmp->tm_nodes_inuse--;
760 LIST_REMOVE(node, tn_entries);
761 node->tn_attached = false;
762 }
763 if (!last)
764 return (false);
765
766 TMPFS_NODE_UNLOCK(node);
767
768 #ifdef INVARIANTS
769 MPASS(node->tn_vnode == NULL);
770 MPASS((node->tn_vpstate & TMPFS_VNODE_ALLOCATING) == 0);
771
772 /*
773 * Make sure this is a node type we can deal with. Everything
774 * is explicitly enumerated without the 'default' clause so
775 * the compiler can throw an error in case a new type is
776 * added.
777 */
778 switch (node->tn_type) {
779 case VBLK:
780 case VCHR:
781 case VDIR:
782 case VFIFO:
783 case VSOCK:
784 case VLNK:
785 case VREG:
786 break;
787 case VNON:
788 case VBAD:
789 case VMARKER:
790 panic("%s: bad type %d for node %p", __func__,
791 (int)node->tn_type, node);
792 }
793 #endif
794
795 while ((ea = LIST_FIRST(&node->tn_extattrs)) != NULL) {
796 LIST_REMOVE(ea, ea_extattrs);
797 tmpfs_extattr_free(ea);
798 }
799
800 switch (node->tn_type) {
801 case VREG:
802 uobj = node->tn_reg.tn_aobj;
803 node->tn_reg.tn_aobj = NULL;
804 if (uobj != NULL) {
805 VM_OBJECT_WLOCK(uobj);
806 KASSERT((uobj->flags & OBJ_TMPFS) != 0,
807 ("tmpfs node %p uobj %p not tmpfs", node, uobj));
808 vm_object_clear_flag(uobj, OBJ_TMPFS);
809 KASSERT(tmp->tm_pages_used >= node->tn_reg.tn_pages,
810 ("tmpfs tmp %p node %p pages %jd free %jd", tmp,
811 node, (uintmax_t)tmp->tm_pages_used,
812 (uintmax_t)node->tn_reg.tn_pages));
813 atomic_add_long(&tmp->tm_pages_used,
814 -node->tn_reg.tn_pages);
815 VM_OBJECT_WUNLOCK(uobj);
816 }
817 tmpfs_free_tmp(tmp);
818
819 /*
820 * vm_object_deallocate() must not be called while
821 * owning tm_allnode_lock, because deallocate might
822 * sleep. Call it after tmpfs_free_tmp() does the
823 * unlock.
824 */
825 if (uobj != NULL)
826 vm_object_deallocate(uobj);
827
828 break;
829 case VLNK:
830 tmpfs_free_tmp(tmp);
831
832 symlink = node->tn_link_target;
833 atomic_store_ptr(&node->tn_link_target, NULL);
834 if (atomic_load_char(&node->tn_link_smr)) {
835 cache_symlink_free(symlink, node->tn_size + 1);
836 } else {
837 free(symlink, M_TMPFSNAME);
838 }
839 break;
840 default:
841 tmpfs_free_tmp(tmp);
842 break;
843 }
844
845 uma_zfree_smr(tmpfs_node_pool, node);
846 return (true);
847 }
848
849 static __inline uint32_t
tmpfs_dirent_hash(const char * name,u_int len)850 tmpfs_dirent_hash(const char *name, u_int len)
851 {
852 uint32_t hash;
853
854 hash = fnv_32_buf(name, len, FNV1_32_INIT + len) & TMPFS_DIRCOOKIE_MASK;
855 #ifdef TMPFS_DEBUG_DIRCOOKIE_DUP
856 hash &= 0xf;
857 #endif
858 if (hash < TMPFS_DIRCOOKIE_MIN)
859 hash += TMPFS_DIRCOOKIE_MIN;
860
861 return (hash);
862 }
863
864 static __inline off_t
tmpfs_dirent_cookie(struct tmpfs_dirent * de)865 tmpfs_dirent_cookie(struct tmpfs_dirent *de)
866 {
867 if (de == NULL)
868 return (TMPFS_DIRCOOKIE_EOF);
869
870 MPASS(de->td_cookie >= TMPFS_DIRCOOKIE_MIN);
871
872 return (de->td_cookie);
873 }
874
875 static __inline boolean_t
tmpfs_dirent_dup(struct tmpfs_dirent * de)876 tmpfs_dirent_dup(struct tmpfs_dirent *de)
877 {
878 return ((de->td_cookie & TMPFS_DIRCOOKIE_DUP) != 0);
879 }
880
881 static __inline boolean_t
tmpfs_dirent_duphead(struct tmpfs_dirent * de)882 tmpfs_dirent_duphead(struct tmpfs_dirent *de)
883 {
884 return ((de->td_cookie & TMPFS_DIRCOOKIE_DUPHEAD) != 0);
885 }
886
887 void
tmpfs_dirent_init(struct tmpfs_dirent * de,const char * name,u_int namelen)888 tmpfs_dirent_init(struct tmpfs_dirent *de, const char *name, u_int namelen)
889 {
890 de->td_hash = de->td_cookie = tmpfs_dirent_hash(name, namelen);
891 memcpy(de->ud.td_name, name, namelen);
892 de->td_namelen = namelen;
893 }
894
895 /*
896 * Allocates a new directory entry for the node node with a name of name.
897 * The new directory entry is returned in *de.
898 *
899 * The link count of node is increased by one to reflect the new object
900 * referencing it.
901 *
902 * Returns zero on success or an appropriate error code on failure.
903 */
904 int
tmpfs_alloc_dirent(struct tmpfs_mount * tmp,struct tmpfs_node * node,const char * name,u_int len,struct tmpfs_dirent ** de)905 tmpfs_alloc_dirent(struct tmpfs_mount *tmp, struct tmpfs_node *node,
906 const char *name, u_int len, struct tmpfs_dirent **de)
907 {
908 struct tmpfs_dirent *nde;
909
910 nde = malloc(sizeof(*nde), M_TMPFSDIR, M_WAITOK);
911 nde->td_node = node;
912 if (name != NULL) {
913 nde->ud.td_name = malloc(len, M_TMPFSNAME, M_WAITOK);
914 tmpfs_dirent_init(nde, name, len);
915 } else
916 nde->td_namelen = 0;
917 if (node != NULL)
918 node->tn_links++;
919
920 *de = nde;
921
922 return (0);
923 }
924
925 /*
926 * Frees a directory entry. It is the caller's responsibility to destroy
927 * the node referenced by it if needed.
928 *
929 * The link count of node is decreased by one to reflect the removal of an
930 * object that referenced it. This only happens if 'node_exists' is true;
931 * otherwise the function will not access the node referred to by the
932 * directory entry, as it may already have been released from the outside.
933 */
934 void
tmpfs_free_dirent(struct tmpfs_mount * tmp,struct tmpfs_dirent * de)935 tmpfs_free_dirent(struct tmpfs_mount *tmp, struct tmpfs_dirent *de)
936 {
937 struct tmpfs_node *node;
938
939 node = de->td_node;
940 if (node != NULL) {
941 MPASS(node->tn_links > 0);
942 node->tn_links--;
943 }
944 if (!tmpfs_dirent_duphead(de) && de->ud.td_name != NULL)
945 free(de->ud.td_name, M_TMPFSNAME);
946 free(de, M_TMPFSDIR);
947 }
948
949 void
tmpfs_destroy_vobject(struct vnode * vp,vm_object_t obj)950 tmpfs_destroy_vobject(struct vnode *vp, vm_object_t obj)
951 {
952 bool want_vrele;
953
954 ASSERT_VOP_ELOCKED(vp, "tmpfs_destroy_vobject");
955 if (vp->v_type != VREG || obj == NULL)
956 return;
957
958 VM_OBJECT_WLOCK(obj);
959 VI_LOCK(vp);
960 vp->v_object = NULL;
961
962 /*
963 * May be going through forced unmount.
964 */
965 want_vrele = false;
966 if ((obj->flags & OBJ_TMPFS_VREF) != 0) {
967 vm_object_clear_flag(obj, OBJ_TMPFS_VREF);
968 want_vrele = true;
969 }
970
971 if (vp->v_writecount < 0)
972 vp->v_writecount = 0;
973 VI_UNLOCK(vp);
974 VM_OBJECT_WUNLOCK(obj);
975 if (want_vrele) {
976 vrele(vp);
977 }
978 }
979
980 /*
981 * Allocates a new vnode for the node node or returns a new reference to
982 * an existing one if the node had already a vnode referencing it. The
983 * resulting locked vnode is returned in *vpp.
984 *
985 * Returns zero on success or an appropriate error code on failure.
986 */
987 int
tmpfs_alloc_vp(struct mount * mp,struct tmpfs_node * node,int lkflag,struct vnode ** vpp)988 tmpfs_alloc_vp(struct mount *mp, struct tmpfs_node *node, int lkflag,
989 struct vnode **vpp)
990 {
991 struct vnode *vp;
992 enum vgetstate vs;
993 struct tmpfs_mount *tm;
994 vm_object_t object;
995 int error;
996
997 error = 0;
998 tm = VFS_TO_TMPFS(mp);
999 TMPFS_NODE_LOCK(node);
1000 tmpfs_ref_node(node);
1001 loop:
1002 TMPFS_NODE_ASSERT_LOCKED(node);
1003 if ((vp = node->tn_vnode) != NULL) {
1004 MPASS((node->tn_vpstate & TMPFS_VNODE_DOOMED) == 0);
1005 if ((node->tn_type == VDIR && node->tn_dir.tn_parent == NULL) ||
1006 (VN_IS_DOOMED(vp) &&
1007 (lkflag & LK_NOWAIT) != 0)) {
1008 TMPFS_NODE_UNLOCK(node);
1009 error = ENOENT;
1010 vp = NULL;
1011 goto out;
1012 }
1013 if (VN_IS_DOOMED(vp)) {
1014 node->tn_vpstate |= TMPFS_VNODE_WRECLAIM;
1015 while ((node->tn_vpstate & TMPFS_VNODE_WRECLAIM) != 0) {
1016 msleep(&node->tn_vnode, TMPFS_NODE_MTX(node),
1017 0, "tmpfsE", 0);
1018 }
1019 goto loop;
1020 }
1021 vs = vget_prep(vp);
1022 TMPFS_NODE_UNLOCK(node);
1023 error = vget_finish(vp, lkflag, vs);
1024 if (error == ENOENT) {
1025 TMPFS_NODE_LOCK(node);
1026 goto loop;
1027 }
1028 if (error != 0) {
1029 vp = NULL;
1030 goto out;
1031 }
1032
1033 /*
1034 * Make sure the vnode is still there after
1035 * getting the interlock to avoid racing a free.
1036 */
1037 if (node->tn_vnode != vp) {
1038 vput(vp);
1039 TMPFS_NODE_LOCK(node);
1040 goto loop;
1041 }
1042
1043 goto out;
1044 }
1045
1046 if ((node->tn_vpstate & TMPFS_VNODE_DOOMED) ||
1047 (node->tn_type == VDIR && node->tn_dir.tn_parent == NULL)) {
1048 TMPFS_NODE_UNLOCK(node);
1049 error = ENOENT;
1050 vp = NULL;
1051 goto out;
1052 }
1053
1054 /*
1055 * otherwise lock the vp list while we call getnewvnode
1056 * since that can block.
1057 */
1058 if (node->tn_vpstate & TMPFS_VNODE_ALLOCATING) {
1059 node->tn_vpstate |= TMPFS_VNODE_WANT;
1060 error = msleep((caddr_t) &node->tn_vpstate,
1061 TMPFS_NODE_MTX(node), 0, "tmpfs_alloc_vp", 0);
1062 if (error != 0)
1063 goto out;
1064 goto loop;
1065 } else
1066 node->tn_vpstate |= TMPFS_VNODE_ALLOCATING;
1067
1068 TMPFS_NODE_UNLOCK(node);
1069
1070 /* Get a new vnode and associate it with our node. */
1071 error = getnewvnode("tmpfs", mp, VFS_TO_TMPFS(mp)->tm_nonc ?
1072 &tmpfs_vnodeop_nonc_entries : &tmpfs_vnodeop_entries, &vp);
1073 if (error != 0)
1074 goto unlock;
1075 MPASS(vp != NULL);
1076
1077 /* lkflag is ignored, the lock is exclusive */
1078 (void) vn_lock(vp, LK_EXCLUSIVE | LK_RETRY);
1079
1080 vp->v_data = node;
1081 vp->v_type = node->tn_type;
1082
1083 /* Type-specific initialization. */
1084 switch (node->tn_type) {
1085 case VBLK:
1086 /* FALLTHROUGH */
1087 case VCHR:
1088 /* FALLTHROUGH */
1089 case VLNK:
1090 /* FALLTHROUGH */
1091 case VSOCK:
1092 break;
1093 case VFIFO:
1094 vp->v_op = &tmpfs_fifoop_entries;
1095 break;
1096 case VREG:
1097 object = node->tn_reg.tn_aobj;
1098 VM_OBJECT_WLOCK(object);
1099 KASSERT((object->flags & OBJ_TMPFS_VREF) == 0,
1100 ("%s: object %p with OBJ_TMPFS_VREF but without vnode",
1101 __func__, object));
1102 VI_LOCK(vp);
1103 KASSERT(vp->v_object == NULL, ("Not NULL v_object in tmpfs"));
1104 vp->v_object = object;
1105 vn_irflag_set_locked(vp, (tm->tm_pgread ? VIRF_PGREAD : 0) |
1106 VIRF_TEXT_REF);
1107 VI_UNLOCK(vp);
1108 VNASSERT((object->flags & OBJ_TMPFS_VREF) == 0, vp,
1109 ("leaked OBJ_TMPFS_VREF"));
1110 if (object->un_pager.swp.writemappings > 0) {
1111 vrefact(vp);
1112 vlazy(vp);
1113 vm_object_set_flag(object, OBJ_TMPFS_VREF);
1114 }
1115 VM_OBJECT_WUNLOCK(object);
1116 break;
1117 case VDIR:
1118 MPASS(node->tn_dir.tn_parent != NULL);
1119 if (node->tn_dir.tn_parent == node)
1120 vp->v_vflag |= VV_ROOT;
1121 break;
1122
1123 default:
1124 panic("tmpfs_alloc_vp: type %p %d", node, (int)node->tn_type);
1125 }
1126 if (vp->v_type != VFIFO)
1127 VN_LOCK_ASHARE(vp);
1128
1129 error = insmntque1(vp, mp);
1130 if (error != 0) {
1131 /* Need to clear v_object for insmntque failure. */
1132 tmpfs_destroy_vobject(vp, vp->v_object);
1133 vp->v_object = NULL;
1134 vp->v_data = NULL;
1135 vp->v_op = &dead_vnodeops;
1136 vgone(vp);
1137 vput(vp);
1138 vp = NULL;
1139 } else {
1140 vn_set_state(vp, VSTATE_CONSTRUCTED);
1141 }
1142
1143 unlock:
1144 TMPFS_NODE_LOCK(node);
1145
1146 MPASS(node->tn_vpstate & TMPFS_VNODE_ALLOCATING);
1147 node->tn_vpstate &= ~TMPFS_VNODE_ALLOCATING;
1148 node->tn_vnode = vp;
1149
1150 if (node->tn_vpstate & TMPFS_VNODE_WANT) {
1151 node->tn_vpstate &= ~TMPFS_VNODE_WANT;
1152 TMPFS_NODE_UNLOCK(node);
1153 wakeup((caddr_t) &node->tn_vpstate);
1154 } else
1155 TMPFS_NODE_UNLOCK(node);
1156
1157 out:
1158 if (error == 0) {
1159 *vpp = vp;
1160
1161 #ifdef INVARIANTS
1162 MPASS(*vpp != NULL);
1163 ASSERT_VOP_LOCKED(*vpp, __func__);
1164 TMPFS_NODE_LOCK(node);
1165 MPASS(*vpp == node->tn_vnode);
1166 TMPFS_NODE_UNLOCK(node);
1167 #endif
1168 }
1169 tmpfs_free_node(tm, node);
1170
1171 return (error);
1172 }
1173
1174 /*
1175 * Destroys the association between the vnode vp and the node it
1176 * references.
1177 */
1178 void
tmpfs_free_vp(struct vnode * vp)1179 tmpfs_free_vp(struct vnode *vp)
1180 {
1181 struct tmpfs_node *node;
1182
1183 node = VP_TO_TMPFS_NODE(vp);
1184
1185 TMPFS_NODE_ASSERT_LOCKED(node);
1186 node->tn_vnode = NULL;
1187 if ((node->tn_vpstate & TMPFS_VNODE_WRECLAIM) != 0)
1188 wakeup(&node->tn_vnode);
1189 node->tn_vpstate &= ~TMPFS_VNODE_WRECLAIM;
1190 vp->v_data = NULL;
1191 }
1192
1193 /*
1194 * Allocates a new file of type 'type' and adds it to the parent directory
1195 * 'dvp'; this addition is done using the component name given in 'cnp'.
1196 * The ownership of the new file is automatically assigned based on the
1197 * credentials of the caller (through 'cnp'), the group is set based on
1198 * the parent directory and the mode is determined from the 'vap' argument.
1199 * If successful, *vpp holds a vnode to the newly created file and zero
1200 * is returned. Otherwise *vpp is NULL and the function returns an
1201 * appropriate error code.
1202 */
1203 int
tmpfs_alloc_file(struct vnode * dvp,struct vnode ** vpp,struct vattr * vap,struct componentname * cnp,const char * target)1204 tmpfs_alloc_file(struct vnode *dvp, struct vnode **vpp, struct vattr *vap,
1205 struct componentname *cnp, const char *target)
1206 {
1207 int error;
1208 struct tmpfs_dirent *de;
1209 struct tmpfs_mount *tmp;
1210 struct tmpfs_node *dnode;
1211 struct tmpfs_node *node;
1212 struct tmpfs_node *parent;
1213
1214 ASSERT_VOP_ELOCKED(dvp, "tmpfs_alloc_file");
1215
1216 tmp = VFS_TO_TMPFS(dvp->v_mount);
1217 dnode = VP_TO_TMPFS_DIR(dvp);
1218 *vpp = NULL;
1219
1220 /* If the entry we are creating is a directory, we cannot overflow
1221 * the number of links of its parent, because it will get a new
1222 * link. */
1223 if (vap->va_type == VDIR) {
1224 /* Ensure that we do not overflow the maximum number of links
1225 * imposed by the system. */
1226 MPASS(dnode->tn_links <= TMPFS_LINK_MAX);
1227 if (dnode->tn_links == TMPFS_LINK_MAX) {
1228 return (EMLINK);
1229 }
1230
1231 parent = dnode;
1232 MPASS(parent != NULL);
1233 } else
1234 parent = NULL;
1235
1236 /* Allocate a node that represents the new file. */
1237 error = tmpfs_alloc_node(dvp->v_mount, tmp, vap->va_type,
1238 cnp->cn_cred->cr_uid, dnode->tn_gid, vap->va_mode, parent,
1239 target, vap->va_rdev, &node);
1240 if (error != 0)
1241 return (error);
1242
1243 /* Allocate a directory entry that points to the new file. */
1244 error = tmpfs_alloc_dirent(tmp, node, cnp->cn_nameptr, cnp->cn_namelen,
1245 &de);
1246 if (error != 0) {
1247 tmpfs_free_node(tmp, node);
1248 return (error);
1249 }
1250
1251 /* Allocate a vnode for the new file. */
1252 error = tmpfs_alloc_vp(dvp->v_mount, node, LK_EXCLUSIVE, vpp);
1253 if (error != 0) {
1254 tmpfs_free_dirent(tmp, de);
1255 tmpfs_free_node(tmp, node);
1256 return (error);
1257 }
1258
1259 /* Now that all required items are allocated, we can proceed to
1260 * insert the new node into the directory, an operation that
1261 * cannot fail. */
1262 if (cnp->cn_flags & ISWHITEOUT)
1263 tmpfs_dir_whiteout_remove(dvp, cnp);
1264 tmpfs_dir_attach(dvp, de);
1265 return (0);
1266 }
1267
1268 struct tmpfs_dirent *
tmpfs_dir_first(struct tmpfs_node * dnode,struct tmpfs_dir_cursor * dc)1269 tmpfs_dir_first(struct tmpfs_node *dnode, struct tmpfs_dir_cursor *dc)
1270 {
1271 struct tmpfs_dirent *de;
1272
1273 de = RB_MIN(tmpfs_dir, &dnode->tn_dir.tn_dirhead);
1274 dc->tdc_tree = de;
1275 if (de != NULL && tmpfs_dirent_duphead(de))
1276 de = LIST_FIRST(&de->ud.td_duphead);
1277 dc->tdc_current = de;
1278
1279 return (dc->tdc_current);
1280 }
1281
1282 struct tmpfs_dirent *
tmpfs_dir_next(struct tmpfs_node * dnode,struct tmpfs_dir_cursor * dc)1283 tmpfs_dir_next(struct tmpfs_node *dnode, struct tmpfs_dir_cursor *dc)
1284 {
1285 struct tmpfs_dirent *de;
1286
1287 MPASS(dc->tdc_tree != NULL);
1288 if (tmpfs_dirent_dup(dc->tdc_current)) {
1289 dc->tdc_current = LIST_NEXT(dc->tdc_current, uh.td_dup.entries);
1290 if (dc->tdc_current != NULL)
1291 return (dc->tdc_current);
1292 }
1293 dc->tdc_tree = dc->tdc_current = RB_NEXT(tmpfs_dir,
1294 &dnode->tn_dir.tn_dirhead, dc->tdc_tree);
1295 if ((de = dc->tdc_current) != NULL && tmpfs_dirent_duphead(de)) {
1296 dc->tdc_current = LIST_FIRST(&de->ud.td_duphead);
1297 MPASS(dc->tdc_current != NULL);
1298 }
1299
1300 return (dc->tdc_current);
1301 }
1302
1303 /* Lookup directory entry in RB-Tree. Function may return duphead entry. */
1304 static struct tmpfs_dirent *
tmpfs_dir_xlookup_hash(struct tmpfs_node * dnode,uint32_t hash)1305 tmpfs_dir_xlookup_hash(struct tmpfs_node *dnode, uint32_t hash)
1306 {
1307 struct tmpfs_dirent *de, dekey;
1308
1309 dekey.td_hash = hash;
1310 de = RB_FIND(tmpfs_dir, &dnode->tn_dir.tn_dirhead, &dekey);
1311 return (de);
1312 }
1313
1314 /* Lookup directory entry by cookie, initialize directory cursor accordingly. */
1315 static struct tmpfs_dirent *
tmpfs_dir_lookup_cookie(struct tmpfs_node * node,off_t cookie,struct tmpfs_dir_cursor * dc)1316 tmpfs_dir_lookup_cookie(struct tmpfs_node *node, off_t cookie,
1317 struct tmpfs_dir_cursor *dc)
1318 {
1319 struct tmpfs_dir *dirhead = &node->tn_dir.tn_dirhead;
1320 struct tmpfs_dirent *de, dekey;
1321
1322 MPASS(cookie >= TMPFS_DIRCOOKIE_MIN);
1323
1324 if (cookie == node->tn_dir.tn_readdir_lastn &&
1325 (de = node->tn_dir.tn_readdir_lastp) != NULL) {
1326 /* Protect against possible race, tn_readdir_last[pn]
1327 * may be updated with only shared vnode lock held. */
1328 if (cookie == tmpfs_dirent_cookie(de))
1329 goto out;
1330 }
1331
1332 if ((cookie & TMPFS_DIRCOOKIE_DUP) != 0) {
1333 LIST_FOREACH(de, &node->tn_dir.tn_dupindex,
1334 uh.td_dup.index_entries) {
1335 MPASS(tmpfs_dirent_dup(de));
1336 if (de->td_cookie == cookie)
1337 goto out;
1338 /* dupindex list is sorted. */
1339 if (de->td_cookie < cookie) {
1340 de = NULL;
1341 goto out;
1342 }
1343 }
1344 MPASS(de == NULL);
1345 goto out;
1346 }
1347
1348 if ((cookie & TMPFS_DIRCOOKIE_MASK) != cookie) {
1349 de = NULL;
1350 } else {
1351 dekey.td_hash = cookie;
1352 /* Recover if direntry for cookie was removed */
1353 de = RB_NFIND(tmpfs_dir, dirhead, &dekey);
1354 }
1355 dc->tdc_tree = de;
1356 dc->tdc_current = de;
1357 if (de != NULL && tmpfs_dirent_duphead(de)) {
1358 dc->tdc_current = LIST_FIRST(&de->ud.td_duphead);
1359 MPASS(dc->tdc_current != NULL);
1360 }
1361 return (dc->tdc_current);
1362
1363 out:
1364 dc->tdc_tree = de;
1365 dc->tdc_current = de;
1366 if (de != NULL && tmpfs_dirent_dup(de))
1367 dc->tdc_tree = tmpfs_dir_xlookup_hash(node,
1368 de->td_hash);
1369 return (dc->tdc_current);
1370 }
1371
1372 /*
1373 * Looks for a directory entry in the directory represented by node.
1374 * 'cnp' describes the name of the entry to look for. Note that the .
1375 * and .. components are not allowed as they do not physically exist
1376 * within directories.
1377 *
1378 * Returns a pointer to the entry when found, otherwise NULL.
1379 */
1380 struct tmpfs_dirent *
tmpfs_dir_lookup(struct tmpfs_node * node,struct tmpfs_node * f,struct componentname * cnp)1381 tmpfs_dir_lookup(struct tmpfs_node *node, struct tmpfs_node *f,
1382 struct componentname *cnp)
1383 {
1384 struct tmpfs_dir_duphead *duphead;
1385 struct tmpfs_dirent *de;
1386 uint32_t hash;
1387
1388 MPASS(IMPLIES(cnp->cn_namelen == 1, cnp->cn_nameptr[0] != '.'));
1389 MPASS(IMPLIES(cnp->cn_namelen == 2, !(cnp->cn_nameptr[0] == '.' &&
1390 cnp->cn_nameptr[1] == '.')));
1391 TMPFS_VALIDATE_DIR(node);
1392
1393 hash = tmpfs_dirent_hash(cnp->cn_nameptr, cnp->cn_namelen);
1394 de = tmpfs_dir_xlookup_hash(node, hash);
1395 if (de != NULL && tmpfs_dirent_duphead(de)) {
1396 duphead = &de->ud.td_duphead;
1397 LIST_FOREACH(de, duphead, uh.td_dup.entries) {
1398 if (TMPFS_DIRENT_MATCHES(de, cnp->cn_nameptr,
1399 cnp->cn_namelen))
1400 break;
1401 }
1402 } else if (de != NULL) {
1403 if (!TMPFS_DIRENT_MATCHES(de, cnp->cn_nameptr,
1404 cnp->cn_namelen))
1405 de = NULL;
1406 }
1407 if (de != NULL && f != NULL && de->td_node != f)
1408 de = NULL;
1409
1410 return (de);
1411 }
1412
1413 /*
1414 * Attach duplicate-cookie directory entry nde to dnode and insert to dupindex
1415 * list, allocate new cookie value.
1416 */
1417 static void
tmpfs_dir_attach_dup(struct tmpfs_node * dnode,struct tmpfs_dir_duphead * duphead,struct tmpfs_dirent * nde)1418 tmpfs_dir_attach_dup(struct tmpfs_node *dnode,
1419 struct tmpfs_dir_duphead *duphead, struct tmpfs_dirent *nde)
1420 {
1421 struct tmpfs_dir_duphead *dupindex;
1422 struct tmpfs_dirent *de, *pde;
1423
1424 dupindex = &dnode->tn_dir.tn_dupindex;
1425 de = LIST_FIRST(dupindex);
1426 if (de == NULL || de->td_cookie < TMPFS_DIRCOOKIE_DUP_MAX) {
1427 if (de == NULL)
1428 nde->td_cookie = TMPFS_DIRCOOKIE_DUP_MIN;
1429 else
1430 nde->td_cookie = de->td_cookie + 1;
1431 MPASS(tmpfs_dirent_dup(nde));
1432 LIST_INSERT_HEAD(dupindex, nde, uh.td_dup.index_entries);
1433 LIST_INSERT_HEAD(duphead, nde, uh.td_dup.entries);
1434 return;
1435 }
1436
1437 /*
1438 * Cookie numbers are near exhaustion. Scan dupindex list for unused
1439 * numbers. dupindex list is sorted in descending order. Keep it so
1440 * after inserting nde.
1441 */
1442 while (1) {
1443 pde = de;
1444 de = LIST_NEXT(de, uh.td_dup.index_entries);
1445 if (de == NULL && pde->td_cookie != TMPFS_DIRCOOKIE_DUP_MIN) {
1446 /*
1447 * Last element of the index doesn't have minimal cookie
1448 * value, use it.
1449 */
1450 nde->td_cookie = TMPFS_DIRCOOKIE_DUP_MIN;
1451 LIST_INSERT_AFTER(pde, nde, uh.td_dup.index_entries);
1452 LIST_INSERT_HEAD(duphead, nde, uh.td_dup.entries);
1453 return;
1454 } else if (de == NULL) {
1455 /*
1456 * We are so lucky have 2^30 hash duplicates in single
1457 * directory :) Return largest possible cookie value.
1458 * It should be fine except possible issues with
1459 * VOP_READDIR restart.
1460 */
1461 nde->td_cookie = TMPFS_DIRCOOKIE_DUP_MAX;
1462 LIST_INSERT_HEAD(dupindex, nde,
1463 uh.td_dup.index_entries);
1464 LIST_INSERT_HEAD(duphead, nde, uh.td_dup.entries);
1465 return;
1466 }
1467 if (de->td_cookie + 1 == pde->td_cookie ||
1468 de->td_cookie >= TMPFS_DIRCOOKIE_DUP_MAX)
1469 continue; /* No hole or invalid cookie. */
1470 nde->td_cookie = de->td_cookie + 1;
1471 MPASS(tmpfs_dirent_dup(nde));
1472 MPASS(pde->td_cookie > nde->td_cookie);
1473 MPASS(nde->td_cookie > de->td_cookie);
1474 LIST_INSERT_BEFORE(de, nde, uh.td_dup.index_entries);
1475 LIST_INSERT_HEAD(duphead, nde, uh.td_dup.entries);
1476 return;
1477 }
1478 }
1479
1480 /*
1481 * Attaches the directory entry de to the directory represented by vp.
1482 * Note that this does not change the link count of the node pointed by
1483 * the directory entry, as this is done by tmpfs_alloc_dirent.
1484 */
1485 void
tmpfs_dir_attach(struct vnode * vp,struct tmpfs_dirent * de)1486 tmpfs_dir_attach(struct vnode *vp, struct tmpfs_dirent *de)
1487 {
1488 struct tmpfs_node *dnode;
1489 struct tmpfs_dirent *xde, *nde;
1490
1491 ASSERT_VOP_ELOCKED(vp, __func__);
1492 MPASS(de->td_namelen > 0);
1493 MPASS(de->td_hash >= TMPFS_DIRCOOKIE_MIN);
1494 MPASS(de->td_cookie == de->td_hash);
1495
1496 dnode = VP_TO_TMPFS_DIR(vp);
1497 dnode->tn_dir.tn_readdir_lastn = 0;
1498 dnode->tn_dir.tn_readdir_lastp = NULL;
1499
1500 MPASS(!tmpfs_dirent_dup(de));
1501 xde = RB_INSERT(tmpfs_dir, &dnode->tn_dir.tn_dirhead, de);
1502 if (xde != NULL && tmpfs_dirent_duphead(xde))
1503 tmpfs_dir_attach_dup(dnode, &xde->ud.td_duphead, de);
1504 else if (xde != NULL) {
1505 /*
1506 * Allocate new duphead. Swap xde with duphead to avoid
1507 * adding/removing elements with the same hash.
1508 */
1509 MPASS(!tmpfs_dirent_dup(xde));
1510 tmpfs_alloc_dirent(VFS_TO_TMPFS(vp->v_mount), NULL, NULL, 0,
1511 &nde);
1512 /* *nde = *xde; XXX gcc 4.2.1 may generate invalid code. */
1513 memcpy(nde, xde, sizeof(*xde));
1514 xde->td_cookie |= TMPFS_DIRCOOKIE_DUPHEAD;
1515 LIST_INIT(&xde->ud.td_duphead);
1516 xde->td_namelen = 0;
1517 xde->td_node = NULL;
1518 tmpfs_dir_attach_dup(dnode, &xde->ud.td_duphead, nde);
1519 tmpfs_dir_attach_dup(dnode, &xde->ud.td_duphead, de);
1520 }
1521 dnode->tn_size += sizeof(struct tmpfs_dirent);
1522 dnode->tn_status |= TMPFS_NODE_CHANGED | TMPFS_NODE_MODIFIED;
1523 dnode->tn_accessed = true;
1524 tmpfs_update(vp);
1525 }
1526
1527 /*
1528 * Detaches the directory entry de from the directory represented by vp.
1529 * Note that this does not change the link count of the node pointed by
1530 * the directory entry, as this is done by tmpfs_free_dirent.
1531 */
1532 void
tmpfs_dir_detach(struct vnode * vp,struct tmpfs_dirent * de)1533 tmpfs_dir_detach(struct vnode *vp, struct tmpfs_dirent *de)
1534 {
1535 struct tmpfs_mount *tmp;
1536 struct tmpfs_dir *head;
1537 struct tmpfs_node *dnode;
1538 struct tmpfs_dirent *xde;
1539
1540 ASSERT_VOP_ELOCKED(vp, __func__);
1541
1542 dnode = VP_TO_TMPFS_DIR(vp);
1543 head = &dnode->tn_dir.tn_dirhead;
1544 dnode->tn_dir.tn_readdir_lastn = 0;
1545 dnode->tn_dir.tn_readdir_lastp = NULL;
1546
1547 if (tmpfs_dirent_dup(de)) {
1548 /* Remove duphead if de was last entry. */
1549 if (LIST_NEXT(de, uh.td_dup.entries) == NULL) {
1550 xde = tmpfs_dir_xlookup_hash(dnode, de->td_hash);
1551 MPASS(tmpfs_dirent_duphead(xde));
1552 } else
1553 xde = NULL;
1554 LIST_REMOVE(de, uh.td_dup.entries);
1555 LIST_REMOVE(de, uh.td_dup.index_entries);
1556 if (xde != NULL) {
1557 if (LIST_EMPTY(&xde->ud.td_duphead)) {
1558 RB_REMOVE(tmpfs_dir, head, xde);
1559 tmp = VFS_TO_TMPFS(vp->v_mount);
1560 MPASS(xde->td_node == NULL);
1561 tmpfs_free_dirent(tmp, xde);
1562 }
1563 }
1564 de->td_cookie = de->td_hash;
1565 } else
1566 RB_REMOVE(tmpfs_dir, head, de);
1567
1568 dnode->tn_size -= sizeof(struct tmpfs_dirent);
1569 dnode->tn_status |= TMPFS_NODE_CHANGED | TMPFS_NODE_MODIFIED;
1570 dnode->tn_accessed = true;
1571 tmpfs_update(vp);
1572 }
1573
1574 void
tmpfs_dir_destroy(struct tmpfs_mount * tmp,struct tmpfs_node * dnode)1575 tmpfs_dir_destroy(struct tmpfs_mount *tmp, struct tmpfs_node *dnode)
1576 {
1577 struct tmpfs_dirent *de, *dde, *nde;
1578
1579 RB_FOREACH_SAFE(de, tmpfs_dir, &dnode->tn_dir.tn_dirhead, nde) {
1580 RB_REMOVE(tmpfs_dir, &dnode->tn_dir.tn_dirhead, de);
1581 /* Node may already be destroyed. */
1582 de->td_node = NULL;
1583 if (tmpfs_dirent_duphead(de)) {
1584 while ((dde = LIST_FIRST(&de->ud.td_duphead)) != NULL) {
1585 LIST_REMOVE(dde, uh.td_dup.entries);
1586 dde->td_node = NULL;
1587 tmpfs_free_dirent(tmp, dde);
1588 }
1589 }
1590 tmpfs_free_dirent(tmp, de);
1591 }
1592 }
1593
1594 /*
1595 * Helper function for tmpfs_readdir. Creates a '.' entry for the given
1596 * directory and returns it in the uio space. The function returns 0
1597 * on success, -1 if there was not enough space in the uio structure to
1598 * hold the directory entry or an appropriate error code if another
1599 * error happens.
1600 */
1601 static int
tmpfs_dir_getdotdent(struct tmpfs_mount * tm,struct tmpfs_node * node,struct uio * uio)1602 tmpfs_dir_getdotdent(struct tmpfs_mount *tm, struct tmpfs_node *node,
1603 struct uio *uio)
1604 {
1605 int error;
1606 struct dirent dent;
1607
1608 TMPFS_VALIDATE_DIR(node);
1609 MPASS(uio->uio_offset == TMPFS_DIRCOOKIE_DOT);
1610
1611 dent.d_fileno = node->tn_id;
1612 dent.d_off = TMPFS_DIRCOOKIE_DOTDOT;
1613 dent.d_type = DT_DIR;
1614 dent.d_namlen = 1;
1615 dent.d_name[0] = '.';
1616 dent.d_reclen = GENERIC_DIRSIZ(&dent);
1617 dirent_terminate(&dent);
1618
1619 if (dent.d_reclen > uio->uio_resid)
1620 error = EJUSTRETURN;
1621 else
1622 error = uiomove(&dent, dent.d_reclen, uio);
1623
1624 tmpfs_set_accessed(tm, node);
1625
1626 return (error);
1627 }
1628
1629 /*
1630 * Helper function for tmpfs_readdir. Creates a '..' entry for the given
1631 * directory and returns it in the uio space. The function returns 0
1632 * on success, -1 if there was not enough space in the uio structure to
1633 * hold the directory entry or an appropriate error code if another
1634 * error happens.
1635 */
1636 static int
tmpfs_dir_getdotdotdent(struct tmpfs_mount * tm,struct tmpfs_node * node,struct uio * uio,off_t next)1637 tmpfs_dir_getdotdotdent(struct tmpfs_mount *tm, struct tmpfs_node *node,
1638 struct uio *uio, off_t next)
1639 {
1640 struct tmpfs_node *parent;
1641 struct dirent dent;
1642 int error;
1643
1644 TMPFS_VALIDATE_DIR(node);
1645 MPASS(uio->uio_offset == TMPFS_DIRCOOKIE_DOTDOT);
1646
1647 /*
1648 * Return ENOENT if the current node is already removed.
1649 */
1650 TMPFS_ASSERT_LOCKED(node);
1651 parent = node->tn_dir.tn_parent;
1652 if (parent == NULL)
1653 return (ENOENT);
1654
1655 dent.d_fileno = parent->tn_id;
1656 dent.d_off = next;
1657 dent.d_type = DT_DIR;
1658 dent.d_namlen = 2;
1659 dent.d_name[0] = '.';
1660 dent.d_name[1] = '.';
1661 dent.d_reclen = GENERIC_DIRSIZ(&dent);
1662 dirent_terminate(&dent);
1663
1664 if (dent.d_reclen > uio->uio_resid)
1665 error = EJUSTRETURN;
1666 else
1667 error = uiomove(&dent, dent.d_reclen, uio);
1668
1669 tmpfs_set_accessed(tm, node);
1670
1671 return (error);
1672 }
1673
1674 /*
1675 * Helper function for tmpfs_readdir. Returns as much directory entries
1676 * as can fit in the uio space. The read starts at uio->uio_offset.
1677 * The function returns 0 on success, -1 if there was not enough space
1678 * in the uio structure to hold the directory entry or an appropriate
1679 * error code if another error happens.
1680 */
1681 int
tmpfs_dir_getdents(struct tmpfs_mount * tm,struct tmpfs_node * node,struct uio * uio,int maxcookies,uint64_t * cookies,int * ncookies)1682 tmpfs_dir_getdents(struct tmpfs_mount *tm, struct tmpfs_node *node,
1683 struct uio *uio, int maxcookies, uint64_t *cookies, int *ncookies)
1684 {
1685 struct tmpfs_dir_cursor dc;
1686 struct tmpfs_dirent *de, *nde;
1687 off_t off;
1688 int error;
1689
1690 TMPFS_VALIDATE_DIR(node);
1691
1692 off = 0;
1693
1694 /*
1695 * Lookup the node from the current offset. The starting offset of
1696 * 0 will lookup both '.' and '..', and then the first real entry,
1697 * or EOF if there are none. Then find all entries for the dir that
1698 * fit into the buffer. Once no more entries are found (de == NULL),
1699 * the offset is set to TMPFS_DIRCOOKIE_EOF, which will cause the next
1700 * call to return 0.
1701 */
1702 switch (uio->uio_offset) {
1703 case TMPFS_DIRCOOKIE_DOT:
1704 error = tmpfs_dir_getdotdent(tm, node, uio);
1705 if (error != 0)
1706 return (error);
1707 uio->uio_offset = off = TMPFS_DIRCOOKIE_DOTDOT;
1708 if (cookies != NULL)
1709 cookies[(*ncookies)++] = off;
1710 /* FALLTHROUGH */
1711 case TMPFS_DIRCOOKIE_DOTDOT:
1712 de = tmpfs_dir_first(node, &dc);
1713 off = tmpfs_dirent_cookie(de);
1714 error = tmpfs_dir_getdotdotdent(tm, node, uio, off);
1715 if (error != 0)
1716 return (error);
1717 uio->uio_offset = off;
1718 if (cookies != NULL)
1719 cookies[(*ncookies)++] = off;
1720 /* EOF. */
1721 if (de == NULL)
1722 return (0);
1723 break;
1724 case TMPFS_DIRCOOKIE_EOF:
1725 return (0);
1726 default:
1727 de = tmpfs_dir_lookup_cookie(node, uio->uio_offset, &dc);
1728 if (de == NULL)
1729 return (EINVAL);
1730 if (cookies != NULL)
1731 off = tmpfs_dirent_cookie(de);
1732 }
1733
1734 /*
1735 * Read as much entries as possible; i.e., until we reach the end of the
1736 * directory or we exhaust uio space.
1737 */
1738 do {
1739 struct dirent d;
1740
1741 /*
1742 * Create a dirent structure representing the current tmpfs_node
1743 * and fill it.
1744 */
1745 if (de->td_node == NULL) {
1746 d.d_fileno = 1;
1747 d.d_type = DT_WHT;
1748 } else {
1749 d.d_fileno = de->td_node->tn_id;
1750 switch (de->td_node->tn_type) {
1751 case VBLK:
1752 d.d_type = DT_BLK;
1753 break;
1754
1755 case VCHR:
1756 d.d_type = DT_CHR;
1757 break;
1758
1759 case VDIR:
1760 d.d_type = DT_DIR;
1761 break;
1762
1763 case VFIFO:
1764 d.d_type = DT_FIFO;
1765 break;
1766
1767 case VLNK:
1768 d.d_type = DT_LNK;
1769 break;
1770
1771 case VREG:
1772 d.d_type = DT_REG;
1773 break;
1774
1775 case VSOCK:
1776 d.d_type = DT_SOCK;
1777 break;
1778
1779 default:
1780 panic("tmpfs_dir_getdents: type %p %d",
1781 de->td_node, (int)de->td_node->tn_type);
1782 }
1783 }
1784 d.d_namlen = de->td_namelen;
1785 MPASS(de->td_namelen < sizeof(d.d_name));
1786 (void)memcpy(d.d_name, de->ud.td_name, de->td_namelen);
1787 d.d_reclen = GENERIC_DIRSIZ(&d);
1788
1789 /*
1790 * Stop reading if the directory entry we are treating is bigger
1791 * than the amount of data that can be returned.
1792 */
1793 if (d.d_reclen > uio->uio_resid) {
1794 error = EJUSTRETURN;
1795 break;
1796 }
1797
1798 nde = tmpfs_dir_next(node, &dc);
1799 d.d_off = tmpfs_dirent_cookie(nde);
1800 dirent_terminate(&d);
1801
1802 /*
1803 * Copy the new dirent structure into the output buffer and
1804 * advance pointers.
1805 */
1806 error = uiomove(&d, d.d_reclen, uio);
1807 if (error == 0) {
1808 de = nde;
1809 if (cookies != NULL) {
1810 off = tmpfs_dirent_cookie(de);
1811 MPASS(*ncookies < maxcookies);
1812 cookies[(*ncookies)++] = off;
1813 }
1814 }
1815 } while (error == 0 && uio->uio_resid > 0 && de != NULL);
1816
1817 /* Skip setting off when using cookies as it is already done above. */
1818 if (cookies == NULL)
1819 off = tmpfs_dirent_cookie(de);
1820
1821 /* Update the offset and cache. */
1822 uio->uio_offset = off;
1823 node->tn_dir.tn_readdir_lastn = off;
1824 node->tn_dir.tn_readdir_lastp = de;
1825
1826 tmpfs_set_accessed(tm, node);
1827 return (error);
1828 }
1829
1830 int
tmpfs_dir_whiteout_add(struct vnode * dvp,struct componentname * cnp)1831 tmpfs_dir_whiteout_add(struct vnode *dvp, struct componentname *cnp)
1832 {
1833 struct tmpfs_dirent *de;
1834 int error;
1835
1836 error = tmpfs_alloc_dirent(VFS_TO_TMPFS(dvp->v_mount), NULL,
1837 cnp->cn_nameptr, cnp->cn_namelen, &de);
1838 if (error != 0)
1839 return (error);
1840 tmpfs_dir_attach(dvp, de);
1841 return (0);
1842 }
1843
1844 void
tmpfs_dir_whiteout_remove(struct vnode * dvp,struct componentname * cnp)1845 tmpfs_dir_whiteout_remove(struct vnode *dvp, struct componentname *cnp)
1846 {
1847 struct tmpfs_dirent *de;
1848
1849 de = tmpfs_dir_lookup(VP_TO_TMPFS_DIR(dvp), NULL, cnp);
1850 MPASS(de != NULL && de->td_node == NULL);
1851 tmpfs_dir_detach(dvp, de);
1852 tmpfs_free_dirent(VFS_TO_TMPFS(dvp->v_mount), de);
1853 }
1854
1855 /*
1856 * Resizes the aobj associated with the regular file pointed to by 'vp' to the
1857 * size 'newsize'. 'vp' must point to a vnode that represents a regular file.
1858 * 'newsize' must be positive.
1859 *
1860 * Returns zero on success or an appropriate error code on failure.
1861 */
1862 int
tmpfs_reg_resize(struct vnode * vp,off_t newsize,boolean_t ignerr)1863 tmpfs_reg_resize(struct vnode *vp, off_t newsize, boolean_t ignerr)
1864 {
1865 struct tmpfs_node *node;
1866 vm_object_t uobj;
1867 vm_pindex_t idx, newpages, oldpages;
1868 off_t oldsize;
1869 int base, error;
1870
1871 MPASS(vp->v_type == VREG);
1872 MPASS(newsize >= 0);
1873
1874 node = VP_TO_TMPFS_NODE(vp);
1875 uobj = node->tn_reg.tn_aobj;
1876
1877 /*
1878 * Convert the old and new sizes to the number of pages needed to
1879 * store them. It may happen that we do not need to do anything
1880 * because the last allocated page can accommodate the change on
1881 * its own.
1882 */
1883 oldsize = node->tn_size;
1884 oldpages = OFF_TO_IDX(oldsize + PAGE_MASK);
1885 MPASS(oldpages == uobj->size);
1886 newpages = OFF_TO_IDX(newsize + PAGE_MASK);
1887
1888 if (__predict_true(newpages == oldpages && newsize >= oldsize)) {
1889 node->tn_size = newsize;
1890 return (0);
1891 }
1892
1893 VM_OBJECT_WLOCK(uobj);
1894 if (newsize < oldsize) {
1895 /*
1896 * Zero the truncated part of the last page.
1897 */
1898 base = newsize & PAGE_MASK;
1899 if (base != 0) {
1900 idx = OFF_TO_IDX(newsize);
1901 error = tmpfs_partial_page_invalidate(uobj, idx, base,
1902 PAGE_SIZE, ignerr);
1903 if (error != 0) {
1904 VM_OBJECT_WUNLOCK(uobj);
1905 return (error);
1906 }
1907 }
1908
1909 /*
1910 * Release any swap space and free any whole pages.
1911 */
1912 if (newpages < oldpages)
1913 vm_object_page_remove(uobj, newpages, 0, 0);
1914 }
1915 uobj->size = newpages;
1916 VM_OBJECT_WUNLOCK(uobj);
1917
1918 node->tn_size = newsize;
1919 return (0);
1920 }
1921
1922 /*
1923 * Punch hole in the aobj associated with the regular file pointed to by 'vp'.
1924 * Requests completely beyond the end-of-file are converted to no-op.
1925 *
1926 * Returns 0 on success or error code from tmpfs_partial_page_invalidate() on
1927 * failure.
1928 */
1929 int
tmpfs_reg_punch_hole(struct vnode * vp,off_t * offset,off_t * length)1930 tmpfs_reg_punch_hole(struct vnode *vp, off_t *offset, off_t *length)
1931 {
1932 struct tmpfs_node *node;
1933 vm_object_t object;
1934 vm_pindex_t pistart, pi, piend;
1935 int startofs, endofs, end;
1936 off_t off, len;
1937 int error;
1938
1939 KASSERT(*length <= OFF_MAX - *offset, ("%s: offset + length overflows",
1940 __func__));
1941 node = VP_TO_TMPFS_NODE(vp);
1942 KASSERT(node->tn_type == VREG, ("%s: node is not regular file",
1943 __func__));
1944 object = node->tn_reg.tn_aobj;
1945 off = *offset;
1946 len = omin(node->tn_size - off, *length);
1947 startofs = off & PAGE_MASK;
1948 endofs = (off + len) & PAGE_MASK;
1949 pistart = OFF_TO_IDX(off);
1950 piend = OFF_TO_IDX(off + len);
1951 pi = OFF_TO_IDX((vm_ooffset_t)off + PAGE_MASK);
1952 error = 0;
1953
1954 /* Handle the case when offset is on or beyond file size. */
1955 if (len <= 0) {
1956 *length = 0;
1957 return (0);
1958 }
1959
1960 VM_OBJECT_WLOCK(object);
1961
1962 /*
1963 * If there is a partial page at the beginning of the hole-punching
1964 * request, fill the partial page with zeroes.
1965 */
1966 if (startofs != 0) {
1967 end = pistart != piend ? PAGE_SIZE : endofs;
1968 error = tmpfs_partial_page_invalidate(object, pistart, startofs,
1969 end, FALSE);
1970 if (error != 0)
1971 goto out;
1972 off += end - startofs;
1973 len -= end - startofs;
1974 }
1975
1976 /*
1977 * Toss away the full pages in the affected area.
1978 */
1979 if (pi < piend) {
1980 vm_object_page_remove(object, pi, piend, 0);
1981 off += IDX_TO_OFF(piend - pi);
1982 len -= IDX_TO_OFF(piend - pi);
1983 }
1984
1985 /*
1986 * If there is a partial page at the end of the hole-punching request,
1987 * fill the partial page with zeroes.
1988 */
1989 if (endofs != 0 && pistart != piend) {
1990 error = tmpfs_partial_page_invalidate(object, piend, 0, endofs,
1991 FALSE);
1992 if (error != 0)
1993 goto out;
1994 off += endofs;
1995 len -= endofs;
1996 }
1997
1998 out:
1999 VM_OBJECT_WUNLOCK(object);
2000 *offset = off;
2001 *length = len;
2002 return (error);
2003 }
2004
2005 void
tmpfs_check_mtime(struct vnode * vp)2006 tmpfs_check_mtime(struct vnode *vp)
2007 {
2008 struct tmpfs_node *node;
2009 struct vm_object *obj;
2010
2011 ASSERT_VOP_ELOCKED(vp, "check_mtime");
2012 if (vp->v_type != VREG)
2013 return;
2014 obj = vp->v_object;
2015 KASSERT(obj->type == tmpfs_pager_type &&
2016 (obj->flags & (OBJ_SWAP | OBJ_TMPFS)) ==
2017 (OBJ_SWAP | OBJ_TMPFS), ("non-tmpfs obj"));
2018 /* unlocked read */
2019 if (obj->generation != obj->cleangeneration) {
2020 VM_OBJECT_WLOCK(obj);
2021 if (obj->generation != obj->cleangeneration) {
2022 obj->cleangeneration = obj->generation;
2023 node = VP_TO_TMPFS_NODE(vp);
2024 node->tn_status |= TMPFS_NODE_MODIFIED |
2025 TMPFS_NODE_CHANGED;
2026 }
2027 VM_OBJECT_WUNLOCK(obj);
2028 }
2029 }
2030
2031 /*
2032 * Change flags of the given vnode.
2033 * Caller should execute tmpfs_update on vp after a successful execution.
2034 * The vnode must be locked on entry and remain locked on exit.
2035 */
2036 int
tmpfs_chflags(struct vnode * vp,u_long flags,struct ucred * cred,struct thread * td)2037 tmpfs_chflags(struct vnode *vp, u_long flags, struct ucred *cred,
2038 struct thread *td)
2039 {
2040 int error;
2041 struct tmpfs_node *node;
2042
2043 ASSERT_VOP_ELOCKED(vp, "chflags");
2044
2045 node = VP_TO_TMPFS_NODE(vp);
2046
2047 if ((flags & ~(SF_APPEND | SF_ARCHIVED | SF_IMMUTABLE | SF_NOUNLINK |
2048 UF_APPEND | UF_ARCHIVE | UF_HIDDEN | UF_IMMUTABLE | UF_NODUMP |
2049 UF_NOUNLINK | UF_OFFLINE | UF_OPAQUE | UF_READONLY | UF_REPARSE |
2050 UF_SPARSE | UF_SYSTEM)) != 0)
2051 return (EOPNOTSUPP);
2052
2053 /* Disallow this operation if the file system is mounted read-only. */
2054 if (vp->v_mount->mnt_flag & MNT_RDONLY)
2055 return (EROFS);
2056
2057 /*
2058 * Callers may only modify the file flags on objects they
2059 * have VADMIN rights for.
2060 */
2061 if ((error = VOP_ACCESS(vp, VADMIN, cred, td)))
2062 return (error);
2063 /*
2064 * Unprivileged processes are not permitted to unset system
2065 * flags, or modify flags if any system flags are set.
2066 */
2067 if (!priv_check_cred(cred, PRIV_VFS_SYSFLAGS)) {
2068 if (node->tn_flags &
2069 (SF_NOUNLINK | SF_IMMUTABLE | SF_APPEND)) {
2070 error = securelevel_gt(cred, 0);
2071 if (error)
2072 return (error);
2073 }
2074 } else {
2075 if (node->tn_flags &
2076 (SF_NOUNLINK | SF_IMMUTABLE | SF_APPEND) ||
2077 ((flags ^ node->tn_flags) & SF_SETTABLE))
2078 return (EPERM);
2079 }
2080 node->tn_flags = flags;
2081 node->tn_status |= TMPFS_NODE_CHANGED;
2082
2083 ASSERT_VOP_ELOCKED(vp, "chflags2");
2084
2085 return (0);
2086 }
2087
2088 /*
2089 * Change access mode on the given vnode.
2090 * Caller should execute tmpfs_update on vp after a successful execution.
2091 * The vnode must be locked on entry and remain locked on exit.
2092 */
2093 int
tmpfs_chmod(struct vnode * vp,mode_t mode,struct ucred * cred,struct thread * td)2094 tmpfs_chmod(struct vnode *vp, mode_t mode, struct ucred *cred,
2095 struct thread *td)
2096 {
2097 int error;
2098 struct tmpfs_node *node;
2099 mode_t newmode;
2100
2101 ASSERT_VOP_ELOCKED(vp, "chmod");
2102 ASSERT_VOP_IN_SEQC(vp);
2103
2104 node = VP_TO_TMPFS_NODE(vp);
2105
2106 /* Disallow this operation if the file system is mounted read-only. */
2107 if (vp->v_mount->mnt_flag & MNT_RDONLY)
2108 return (EROFS);
2109
2110 /* Immutable or append-only files cannot be modified, either. */
2111 if (node->tn_flags & (IMMUTABLE | APPEND))
2112 return (EPERM);
2113
2114 /*
2115 * To modify the permissions on a file, must possess VADMIN
2116 * for that file.
2117 */
2118 if ((error = VOP_ACCESS(vp, VADMIN, cred, td)))
2119 return (error);
2120
2121 /*
2122 * Privileged processes may set the sticky bit on non-directories,
2123 * as well as set the setgid bit on a file with a group that the
2124 * process is not a member of.
2125 */
2126 if (vp->v_type != VDIR && (mode & S_ISTXT)) {
2127 if (priv_check_cred(cred, PRIV_VFS_STICKYFILE))
2128 return (EFTYPE);
2129 }
2130 if (!groupmember(node->tn_gid, cred) && (mode & S_ISGID)) {
2131 error = priv_check_cred(cred, PRIV_VFS_SETGID);
2132 if (error)
2133 return (error);
2134 }
2135
2136 newmode = node->tn_mode & ~ALLPERMS;
2137 newmode |= mode & ALLPERMS;
2138 atomic_store_short(&node->tn_mode, newmode);
2139
2140 node->tn_status |= TMPFS_NODE_CHANGED;
2141
2142 ASSERT_VOP_ELOCKED(vp, "chmod2");
2143
2144 return (0);
2145 }
2146
2147 /*
2148 * Change ownership of the given vnode. At least one of uid or gid must
2149 * be different than VNOVAL. If one is set to that value, the attribute
2150 * is unchanged.
2151 * Caller should execute tmpfs_update on vp after a successful execution.
2152 * The vnode must be locked on entry and remain locked on exit.
2153 */
2154 int
tmpfs_chown(struct vnode * vp,uid_t uid,gid_t gid,struct ucred * cred,struct thread * td)2155 tmpfs_chown(struct vnode *vp, uid_t uid, gid_t gid, struct ucred *cred,
2156 struct thread *td)
2157 {
2158 int error;
2159 struct tmpfs_node *node;
2160 uid_t ouid;
2161 gid_t ogid;
2162 mode_t newmode;
2163
2164 ASSERT_VOP_ELOCKED(vp, "chown");
2165 ASSERT_VOP_IN_SEQC(vp);
2166
2167 node = VP_TO_TMPFS_NODE(vp);
2168
2169 /* Assign default values if they are unknown. */
2170 MPASS(uid != VNOVAL || gid != VNOVAL);
2171 if (uid == VNOVAL)
2172 uid = node->tn_uid;
2173 if (gid == VNOVAL)
2174 gid = node->tn_gid;
2175 MPASS(uid != VNOVAL && gid != VNOVAL);
2176
2177 /* Disallow this operation if the file system is mounted read-only. */
2178 if (vp->v_mount->mnt_flag & MNT_RDONLY)
2179 return (EROFS);
2180
2181 /* Immutable or append-only files cannot be modified, either. */
2182 if (node->tn_flags & (IMMUTABLE | APPEND))
2183 return (EPERM);
2184
2185 /*
2186 * To modify the ownership of a file, must possess VADMIN for that
2187 * file.
2188 */
2189 if ((error = VOP_ACCESS(vp, VADMIN, cred, td)))
2190 return (error);
2191
2192 /*
2193 * To change the owner of a file, or change the group of a file to a
2194 * group of which we are not a member, the caller must have
2195 * privilege.
2196 */
2197 if ((uid != node->tn_uid ||
2198 (gid != node->tn_gid && !groupmember(gid, cred))) &&
2199 (error = priv_check_cred(cred, PRIV_VFS_CHOWN)))
2200 return (error);
2201
2202 ogid = node->tn_gid;
2203 ouid = node->tn_uid;
2204
2205 node->tn_uid = uid;
2206 node->tn_gid = gid;
2207
2208 node->tn_status |= TMPFS_NODE_CHANGED;
2209
2210 if ((node->tn_mode & (S_ISUID | S_ISGID)) != 0 &&
2211 (ouid != uid || ogid != gid)) {
2212 if (priv_check_cred(cred, PRIV_VFS_RETAINSUGID)) {
2213 newmode = node->tn_mode & ~(S_ISUID | S_ISGID);
2214 atomic_store_short(&node->tn_mode, newmode);
2215 }
2216 }
2217
2218 ASSERT_VOP_ELOCKED(vp, "chown2");
2219
2220 return (0);
2221 }
2222
2223 /*
2224 * Change size of the given vnode.
2225 * Caller should execute tmpfs_update on vp after a successful execution.
2226 * The vnode must be locked on entry and remain locked on exit.
2227 */
2228 int
tmpfs_chsize(struct vnode * vp,u_quad_t size,struct ucred * cred,struct thread * td)2229 tmpfs_chsize(struct vnode *vp, u_quad_t size, struct ucred *cred,
2230 struct thread *td)
2231 {
2232 int error;
2233 struct tmpfs_node *node;
2234
2235 ASSERT_VOP_ELOCKED(vp, "chsize");
2236
2237 node = VP_TO_TMPFS_NODE(vp);
2238
2239 /* Decide whether this is a valid operation based on the file type. */
2240 error = 0;
2241 switch (vp->v_type) {
2242 case VDIR:
2243 return (EISDIR);
2244
2245 case VREG:
2246 if (vp->v_mount->mnt_flag & MNT_RDONLY)
2247 return (EROFS);
2248 break;
2249
2250 case VBLK:
2251 /* FALLTHROUGH */
2252 case VCHR:
2253 /* FALLTHROUGH */
2254 case VFIFO:
2255 /*
2256 * Allow modifications of special files even if in the file
2257 * system is mounted read-only (we are not modifying the
2258 * files themselves, but the objects they represent).
2259 */
2260 return (0);
2261
2262 default:
2263 /* Anything else is unsupported. */
2264 return (EOPNOTSUPP);
2265 }
2266
2267 /* Immutable or append-only files cannot be modified, either. */
2268 if (node->tn_flags & (IMMUTABLE | APPEND))
2269 return (EPERM);
2270
2271 error = vn_rlimit_trunc(size, td);
2272 if (error != 0)
2273 return (error);
2274
2275 error = tmpfs_truncate(vp, size);
2276 /*
2277 * tmpfs_truncate will raise the NOTE_EXTEND and NOTE_ATTRIB kevents
2278 * for us, as will update tn_status; no need to do that here.
2279 */
2280
2281 ASSERT_VOP_ELOCKED(vp, "chsize2");
2282
2283 return (error);
2284 }
2285
2286 /*
2287 * Change access and modification times of the given vnode.
2288 * Caller should execute tmpfs_update on vp after a successful execution.
2289 * The vnode must be locked on entry and remain locked on exit.
2290 */
2291 int
tmpfs_chtimes(struct vnode * vp,struct vattr * vap,struct ucred * cred,struct thread * td)2292 tmpfs_chtimes(struct vnode *vp, struct vattr *vap,
2293 struct ucred *cred, struct thread *td)
2294 {
2295 int error;
2296 struct tmpfs_node *node;
2297
2298 ASSERT_VOP_ELOCKED(vp, "chtimes");
2299
2300 node = VP_TO_TMPFS_NODE(vp);
2301
2302 /* Disallow this operation if the file system is mounted read-only. */
2303 if (vp->v_mount->mnt_flag & MNT_RDONLY)
2304 return (EROFS);
2305
2306 /* Immutable or append-only files cannot be modified, either. */
2307 if (node->tn_flags & (IMMUTABLE | APPEND))
2308 return (EPERM);
2309
2310 error = vn_utimes_perm(vp, vap, cred, td);
2311 if (error != 0)
2312 return (error);
2313
2314 if (vap->va_atime.tv_sec != VNOVAL)
2315 node->tn_accessed = true;
2316 if (vap->va_mtime.tv_sec != VNOVAL)
2317 node->tn_status |= TMPFS_NODE_MODIFIED;
2318 if (vap->va_birthtime.tv_sec != VNOVAL)
2319 node->tn_status |= TMPFS_NODE_MODIFIED;
2320 tmpfs_itimes(vp, &vap->va_atime, &vap->va_mtime);
2321 if (vap->va_birthtime.tv_sec != VNOVAL)
2322 node->tn_birthtime = vap->va_birthtime;
2323 ASSERT_VOP_ELOCKED(vp, "chtimes2");
2324
2325 return (0);
2326 }
2327
2328 void
tmpfs_set_status(struct tmpfs_mount * tm,struct tmpfs_node * node,int status)2329 tmpfs_set_status(struct tmpfs_mount *tm, struct tmpfs_node *node, int status)
2330 {
2331
2332 if ((node->tn_status & status) == status || tm->tm_ronly)
2333 return;
2334 TMPFS_NODE_LOCK(node);
2335 node->tn_status |= status;
2336 TMPFS_NODE_UNLOCK(node);
2337 }
2338
2339 void
tmpfs_set_accessed(struct tmpfs_mount * tm,struct tmpfs_node * node)2340 tmpfs_set_accessed(struct tmpfs_mount *tm, struct tmpfs_node *node)
2341 {
2342 if (node->tn_accessed || tm->tm_ronly)
2343 return;
2344 atomic_store_8(&node->tn_accessed, true);
2345 }
2346
2347 /* Sync timestamps */
2348 void
tmpfs_itimes(struct vnode * vp,const struct timespec * acc,const struct timespec * mod)2349 tmpfs_itimes(struct vnode *vp, const struct timespec *acc,
2350 const struct timespec *mod)
2351 {
2352 struct tmpfs_node *node;
2353 struct timespec now;
2354
2355 ASSERT_VOP_LOCKED(vp, "tmpfs_itimes");
2356 node = VP_TO_TMPFS_NODE(vp);
2357
2358 if (!node->tn_accessed &&
2359 (node->tn_status & (TMPFS_NODE_MODIFIED | TMPFS_NODE_CHANGED)) == 0)
2360 return;
2361
2362 vfs_timestamp(&now);
2363 TMPFS_NODE_LOCK(node);
2364 if (node->tn_accessed) {
2365 if (acc == NULL)
2366 acc = &now;
2367 node->tn_atime = *acc;
2368 }
2369 if (node->tn_status & TMPFS_NODE_MODIFIED) {
2370 if (mod == NULL)
2371 mod = &now;
2372 node->tn_mtime = *mod;
2373 }
2374 if (node->tn_status & TMPFS_NODE_CHANGED)
2375 node->tn_ctime = now;
2376 node->tn_status &= ~(TMPFS_NODE_MODIFIED | TMPFS_NODE_CHANGED);
2377 node->tn_accessed = false;
2378 TMPFS_NODE_UNLOCK(node);
2379
2380 /* XXX: FIX? The entropy here is desirable, but the harvesting may be expensive */
2381 random_harvest_queue(node, sizeof(*node), RANDOM_FS_ATIME);
2382 }
2383
2384 int
tmpfs_truncate(struct vnode * vp,off_t length)2385 tmpfs_truncate(struct vnode *vp, off_t length)
2386 {
2387 struct tmpfs_node *node;
2388 int error;
2389
2390 if (length < 0)
2391 return (EINVAL);
2392 if (length > VFS_TO_TMPFS(vp->v_mount)->tm_maxfilesize)
2393 return (EFBIG);
2394
2395 node = VP_TO_TMPFS_NODE(vp);
2396 error = node->tn_size == length ? 0 : tmpfs_reg_resize(vp, length,
2397 FALSE);
2398 if (error == 0)
2399 node->tn_status |= TMPFS_NODE_CHANGED | TMPFS_NODE_MODIFIED;
2400 tmpfs_update(vp);
2401
2402 return (error);
2403 }
2404
2405 static __inline int
tmpfs_dirtree_cmp(struct tmpfs_dirent * a,struct tmpfs_dirent * b)2406 tmpfs_dirtree_cmp(struct tmpfs_dirent *a, struct tmpfs_dirent *b)
2407 {
2408 if (a->td_hash > b->td_hash)
2409 return (1);
2410 else if (a->td_hash < b->td_hash)
2411 return (-1);
2412 return (0);
2413 }
2414
2415 RB_GENERATE_STATIC(tmpfs_dir, tmpfs_dirent, uh.td_entries, tmpfs_dirtree_cmp);
2416