xref: /dragonfly/sys/vfs/hammer2/hammer2_inode.c (revision b0d289c2)

/*
 * Copyright (c) 2011-2014 The DragonFly Project.  All rights reserved.
 *
 * This code is derived from software contributed to The DragonFly Project
 * by Matthew Dillon <dillon@dragonflybsd.org>
 * by Venkatesh Srinivas <vsrinivas@dragonflybsd.org>
 *
 * Redistribution and use in source and binary forms, with or without
 * modification, are permitted provided that the following conditions
 * are met:
 *
 * 1. Redistributions of source code must retain the above copyright
 *    notice, this list of conditions and the following disclaimer.
 * 2. Redistributions in binary form must reproduce the above copyright
 *    notice, this list of conditions and the following disclaimer in
 *    the documentation and/or other materials provided with the
 *    distribution.
 * 3. Neither the name of The DragonFly Project nor the names of its
 *    contributors may be used to endorse or promote products derived
 *    from this software without specific, prior written permission.
 *
 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
 * FOR A PARTICULAR PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE
 * COPYRIGHT HOLDERS OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
 * INCIDENTAL, SPECIAL, EXEMPLARY OR CONSEQUENTIAL DAMAGES (INCLUDING,
 * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
 * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
 * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
 * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT
 * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
 * SUCH DAMAGE.
 */
#include <sys/cdefs.h>
#include <sys/param.h>
#include <sys/systm.h>
#include <sys/types.h>
#include <sys/lock.h>
#include <sys/uuid.h>

#include "hammer2.h"

#define INODE_DEBUG	0

static void hammer2_inode_move_to_hidden(hammer2_trans_t *trans,
					 hammer2_cluster_t **cparentp,
					 hammer2_cluster_t **clusterp,
					 hammer2_tid_t inum);

RB_GENERATE2(hammer2_inode_tree, hammer2_inode, rbnode, hammer2_inode_cmp,
	     hammer2_tid_t, inum);

int
hammer2_inode_cmp(hammer2_inode_t *ip1, hammer2_inode_t *ip2)
{
	if (ip1->inum < ip2->inum)
		return(-1);
	if (ip1->inum > ip2->inum)
		return(1);
	return(0);
}

/*
 * HAMMER2 inode locks
 *
 * HAMMER2 offers shared and exclusive locks on inodes.  Pass a mask of
 * flags for options:
 *
 *	- pass HAMMER2_RESOLVE_SHARED if a shared lock is desired.  The
 *	  inode locking function will automatically set the RDONLY flag.
 *
 *	- pass HAMMER2_RESOLVE_ALWAYS if you need the inode's meta-data.
 *	  Most front-end inode locks do.
 *
 *	- pass HAMMER2_RESOLVE_NEVER if you do not want to require that
 *	  the inode data be resolved.  This is used by the syncthr because
 *	  it can run on an unresolved/out-of-sync cluster, and also by the
 *	  vnode reclamation code to avoid unnecessary I/O (particularly when
 *	  disposing of hundreds of thousands of cached vnodes).
 *
 * The inode locking function locks the inode itself, resolves any stale
 * chains in the inode's cluster, and allocates a fresh copy of the
 * cluster with 1 ref and all the underlying chains locked.
 *
 * ip->cluster will be stable while the inode is locked.
 *
 * NOTE: We don't combine the inode/chain lock because putting away an
 *       inode would otherwise confuse multiple lock holders of the inode.
 *
 * NOTE: In-memory inodes always point to hardlink targets (the actual file),
 *	 and never point to a hardlink pointer.
 *
 * NOTE: If caller passes HAMMER2_RESOLVE_RDONLY the exclusive locking code
 *	 will feel free to reduce the chain set in the cluster as an
 *	 optimization.  It will still be validated against the quorum if
 *	 appropriate, but the optimization might be able to reduce data
 *	 accesses to one node.  This flag is automatically set if the inode
 *	 is locked with HAMMER2_RESOLVE_SHARED.
 */
hammer2_cluster_t *
hammer2_inode_lock(hammer2_inode_t *ip, int how)
{
	hammer2_cluster_t *cluster;

	hammer2_inode_ref(ip);

	/*
	 * Inode structure mutex
	 */
	if (how & HAMMER2_RESOLVE_SHARED) {
		how |= HAMMER2_RESOLVE_RDONLY;
		hammer2_mtx_sh(&ip->lock);
	} else {
		hammer2_mtx_ex(&ip->lock);
	}

	/*
	 * Create a copy of ip->cluster and lock it.  Note that the copy
	 * will have a ref on the cluster AND its chains and we don't want
	 * a second ref to either when we lock it.
	 *
	 * The copy will not have a focus until it is locked.
	 *
	 * Exclusive inode locks set the template focus chain in (ip)
	 * as a hint.  Cluster locks can ALWAYS replace the focus in the
	 * working copy if the hint does not work out, so beware.
	 */
	cluster = hammer2_cluster_copy(&ip->cluster);
	hammer2_cluster_lock(cluster, how);
	hammer2_cluster_resolve(cluster);

	/*
	 * cluster->focus will be set if resolving RESOLVE_ALWAYS, but
	 * only update the cached focus in the inode structure when taking
	 * out an exclusive lock.
	 */
	if ((how & HAMMER2_RESOLVE_SHARED) == 0)
		ip->cluster.focus = cluster->focus;

	/*
	 * Initialize pmp->inode_tid and pmp->modify_tid on first access
	 * to the root of mount that resolves good.
	 * XXX probably not the best place for this.
	 */
	if (ip->pmp->inode_tid == 0 &&
	    cluster->error == 0 && cluster->focus) {
		const hammer2_inode_data_t *ripdata;
		hammer2_pfs_t *pmp = ip->pmp;
		hammer2_blockref_t bref;

		ripdata = &hammer2_cluster_rdata(cluster)->ipdata;
		hammer2_cluster_bref(cluster, &bref);
		pmp->inode_tid = ripdata->pfs_inum + 1;
		pmp->modify_tid = bref.modify_tid;
		kprintf("PMP focus good set nextino=%ld mod=%016jx\n",
			pmp->inode_tid, pmp->modify_tid);

	}
	return (cluster);
}

void
hammer2_inode_unlock(hammer2_inode_t *ip, hammer2_cluster_t *cluster)
{
	if (cluster) {
		hammer2_cluster_unlock(cluster);
		hammer2_cluster_drop(cluster);
	}
	hammer2_mtx_unlock(&ip->lock);
	hammer2_inode_drop(ip);
}

/*
 * Temporarily release a lock held shared or exclusive.  Caller must
 * hold the lock shared or exclusive on call and lock will be released
 * on return.
 *
 * Restore a lock that was temporarily released.
 */
hammer2_mtx_state_t
hammer2_inode_lock_temp_release(hammer2_inode_t *ip)
{
	return hammer2_mtx_temp_release(&ip->lock);
}

void
hammer2_inode_lock_temp_restore(hammer2_inode_t *ip, hammer2_mtx_state_t ostate)
{
	hammer2_mtx_temp_restore(&ip->lock, ostate);
}

/*
 * Upgrade a shared inode lock to exclusive and return.  If the inode lock
 * is already held exclusively this is a NOP.
 *
 * The caller MUST hold the inode lock either shared or exclusive on call
 * and will own the lock exclusively on return.
 *
 * Returns non-zero if the lock was already exclusive prior to the upgrade.
 */
int
hammer2_inode_lock_upgrade(hammer2_inode_t *ip)
{
	int wasexclusive;

	if (mtx_islocked_ex(&ip->lock)) {
		wasexclusive = 1;
	} else {
		hammer2_mtx_unlock(&ip->lock);
		hammer2_mtx_ex(&ip->lock);
		wasexclusive = 0;
	}
	return wasexclusive;
}

/*
 * Downgrade an inode lock from exclusive to shared only if the inode
 * lock was previously shared.  If the inode lock was previously exclusive,
 * this is a NOP.
 */
void
hammer2_inode_lock_downgrade(hammer2_inode_t *ip, int wasexclusive)
{
	if (wasexclusive == 0)
		mtx_downgrade(&ip->lock);
}

/*
 * Lookup an inode by inode number
 */
hammer2_inode_t *
hammer2_inode_lookup(hammer2_pfs_t *pmp, hammer2_tid_t inum)
{
	hammer2_inode_t *ip;

	KKASSERT(pmp);
	if (pmp->spmp_hmp) {
		ip = NULL;
	} else {
		hammer2_spin_ex(&pmp->inum_spin);
		ip = RB_LOOKUP(hammer2_inode_tree, &pmp->inum_tree, inum);
		if (ip)
			hammer2_inode_ref(ip);
		hammer2_spin_unex(&pmp->inum_spin);
	}
	return(ip);
}

/*
 * Adding a ref to an inode is only legal if the inode already has at least
 * one ref.
 *
 * (can be called with spinlock held)
 */
void
hammer2_inode_ref(hammer2_inode_t *ip)
{
	atomic_add_int(&ip->refs, 1);
}

/*
 * Drop an inode reference, freeing the inode when the last reference goes
 * away.
 */
void
hammer2_inode_drop(hammer2_inode_t *ip)
{
	hammer2_pfs_t *pmp;
	hammer2_inode_t *pip;
	u_int refs;

	while (ip) {
		refs = ip->refs;
		cpu_ccfence();
		if (refs == 1) {
			/*
			 * Transition to zero, must interlock with
			 * the inode inumber lookup tree (if applicable).
			 * It should not be possible for anyone to race
			 * the transition to 0.
			 *
			 */
			pmp = ip->pmp;
			KKASSERT(pmp);
			hammer2_spin_ex(&pmp->inum_spin);

			if (atomic_cmpset_int(&ip->refs, 1, 0)) {
				KKASSERT(hammer2_mtx_refs(&ip->lock) == 0);
				if (ip->flags & HAMMER2_INODE_ONRBTREE) {
					atomic_clear_int(&ip->flags,
						     HAMMER2_INODE_ONRBTREE);
					RB_REMOVE(hammer2_inode_tree,
						  &pmp->inum_tree, ip);
				}
				hammer2_spin_unex(&pmp->inum_spin);

				pip = ip->pip;
				ip->pip = NULL;
				ip->pmp = NULL;

				/*
				 * Cleaning out ip->cluster isn't entirely
				 * trivial.
				 */
				hammer2_inode_repoint(ip, NULL, NULL);

				/*
				 * We have to drop pip (if non-NULL) to
				 * dispose of our implied reference from
				 * ip->pip.  We can simply loop on it.
				 */
				kfree(ip, pmp->minode);
				atomic_add_long(&pmp->inmem_inodes, -1);
				ip = pip;
				/* continue with pip (can be NULL) */
			} else {
				hammer2_spin_unex(&ip->pmp->inum_spin);
			}
		} else {
			/*
			 * Non zero transition
			 */
			if (atomic_cmpset_int(&ip->refs, refs, refs - 1))
				break;
		}
	}
}

/*
 * Get the vnode associated with the given inode, allocating the vnode if
 * necessary.  The vnode will be returned exclusively locked.
 *
 * The caller must lock the inode (shared or exclusive).
 *
 * Great care must be taken to avoid deadlocks and vnode acquisition/reclaim
 * races.
 */
struct vnode *
hammer2_igetv(hammer2_inode_t *ip, hammer2_cluster_t *cparent, int *errorp)
{
	const hammer2_inode_data_t *ripdata;
	hammer2_pfs_t *pmp;
	struct vnode *vp;

	pmp = ip->pmp;
	KKASSERT(pmp != NULL);
	*errorp = 0;

	ripdata = &hammer2_cluster_rdata(cparent)->ipdata;

	for (;;) {
		/*
		 * Attempt to reuse an existing vnode assignment.  It is
		 * possible to race a reclaim so the vget() may fail.  The
		 * inode must be unlocked during the vget() to avoid a
		 * deadlock against a reclaim.
		 */
		int wasexclusive;

		vp = ip->vp;
		if (vp) {
			/*
			 * Inode must be unlocked during the vget() to avoid
			 * possible deadlocks, but leave the ip ref intact.
			 *
			 * vnode is held to prevent destruction during the
			 * vget().  The vget() can still fail if we lost
			 * a reclaim race on the vnode.
			 */
			hammer2_mtx_state_t ostate;

			vhold(vp);
			ostate = hammer2_inode_lock_temp_release(ip);
			if (vget(vp, LK_EXCLUSIVE)) {
				vdrop(vp);
				hammer2_inode_lock_temp_restore(ip, ostate);
				continue;
			}
			hammer2_inode_lock_temp_restore(ip, ostate);
			vdrop(vp);
			/* vp still locked and ref from vget */
			if (ip->vp != vp) {
				kprintf("hammer2: igetv race %p/%p\n",
					ip->vp, vp);
				vput(vp);
				continue;
			}
			*errorp = 0;
			break;
		}

		/*
		 * No vnode exists, allocate a new vnode.  Beware of
		 * allocation races.  This function will return an
		 * exclusively locked and referenced vnode.
		 */
		*errorp = getnewvnode(VT_HAMMER2, pmp->mp, &vp, 0, 0);
		if (*errorp) {
			kprintf("hammer2: igetv getnewvnode failed %d\n",
				*errorp);
			vp = NULL;
			break;
		}

		/*
		 * Lock the inode and check for an allocation race.
		 */
		wasexclusive = hammer2_inode_lock_upgrade(ip);
		if (ip->vp != NULL) {
			vp->v_type = VBAD;
			vx_put(vp);
			hammer2_inode_lock_downgrade(ip, wasexclusive);
			continue;
		}

		switch (ripdata->type) {
		case HAMMER2_OBJTYPE_DIRECTORY:
			vp->v_type = VDIR;
			break;
		case HAMMER2_OBJTYPE_REGFILE:
			vp->v_type = VREG;
			vinitvmio(vp, ripdata->size,
				  HAMMER2_LBUFSIZE,
				  (int)ripdata->size & HAMMER2_LBUFMASK);
			break;
		case HAMMER2_OBJTYPE_SOFTLINK:
			/*
			 * XXX for now we are using the generic file_read
			 * and file_write code so we need a buffer cache
			 * association.
			 */
			vp->v_type = VLNK;
			vinitvmio(vp, ripdata->size,
				  HAMMER2_LBUFSIZE,
				  (int)ripdata->size & HAMMER2_LBUFMASK);
			break;
		case HAMMER2_OBJTYPE_CDEV:
			vp->v_type = VCHR;
			/* fall through */
		case HAMMER2_OBJTYPE_BDEV:
			vp->v_ops = &pmp->mp->mnt_vn_spec_ops;
			if (ripdata->type != HAMMER2_OBJTYPE_CDEV)
				vp->v_type = VBLK;
			addaliasu(vp, ripdata->rmajor, ripdata->rminor);
			break;
		case HAMMER2_OBJTYPE_FIFO:
			vp->v_type = VFIFO;
			vp->v_ops = &pmp->mp->mnt_vn_fifo_ops;
			break;
		default:
			panic("hammer2: unhandled objtype %d", ripdata->type);
			break;
		}

		if (ip == pmp->iroot)
			vsetflags(vp, VROOT);

		vp->v_data = ip;
		ip->vp = vp;
		hammer2_inode_ref(ip);		/* vp association */
		hammer2_inode_lock_downgrade(ip, wasexclusive);
		break;
	}

	/*
	 * Return non-NULL vp and *errorp == 0, or NULL vp and *errorp != 0.
	 */
	if (hammer2_debug & 0x0002) {
		kprintf("igetv vp %p refs 0x%08x aux 0x%08x\n",
			vp, vp->v_refcnt, vp->v_auxrefs);
	}
	return (vp);
}

/*
 * Returns the inode associated with the passed-in cluster, creating the
 * inode if necessary and synchronizing it to the passed-in cluster otherwise.
 *
 * The passed-in cluster must be locked and will remain locked on return.
 * The returned inode will be locked and the caller may dispose of both
 * via hammer2_inode_unlock_ex().  However, if the caller needs to resolve
 * a hardlink it must ref/unlock/relock/drop the inode.
 *
 * The hammer2_inode structure regulates the interface between the high level
 * kernel VNOPS API and the filesystem backend (the chains).
 *
 * On return the inode is locked with the supplied cluster.
 */
hammer2_inode_t *
hammer2_inode_get(hammer2_pfs_t *pmp, hammer2_inode_t *dip,
		  hammer2_cluster_t *cluster)
{
	hammer2_inode_t *nip;
	const hammer2_inode_data_t *iptmp;
	const hammer2_inode_data_t *nipdata;

	KKASSERT(cluster == NULL ||
		 hammer2_cluster_type(cluster) == HAMMER2_BREF_TYPE_INODE);
	KKASSERT(pmp);

	/*
	 * Interlocked lookup/ref of the inode.  This code is only needed
	 * when looking up inodes with nlinks != 0 (TODO: optimize out
	 * otherwise and test for duplicates).
	 *
	 * Cluster can be NULL during the initial pfs allocation.
	 */
again:
	while (cluster) {
		iptmp = &hammer2_cluster_rdata(cluster)->ipdata;
		nip = hammer2_inode_lookup(pmp, iptmp->inum);
		if (nip == NULL)
			break;

		hammer2_mtx_ex(&nip->lock);

		/*
		 * Handle SMP race (not applicable to the super-root spmp
		 * which can't index inodes due to duplicative inode numbers).
		 */
		if (pmp->spmp_hmp == NULL &&
		    (nip->flags & HAMMER2_INODE_ONRBTREE) == 0) {
			hammer2_mtx_unlock(&nip->lock);
			hammer2_inode_drop(nip);
			continue;
		}
		hammer2_inode_repoint(nip, NULL, cluster);

		return nip;
	}

	/*
	 * We couldn't find the inode number, create a new inode.
	 */
	nip = kmalloc(sizeof(*nip), pmp->minode, M_WAITOK | M_ZERO);
	spin_init(&nip->cluster_spin, "h2clspin");
	atomic_add_long(&pmp->inmem_inodes, 1);
	hammer2_pfs_memory_inc(pmp);
	hammer2_pfs_memory_wakeup(pmp);
	if (pmp->spmp_hmp)
		nip->flags = HAMMER2_INODE_SROOT;

	/*
	 * Initialize nip's cluster.  A cluster is provided for normal
	 * inodes but typically not for the super-root or PFS inodes.
	 */
	nip->cluster.refs = 1;
	nip->cluster.pmp = pmp;
	nip->cluster.flags |= HAMMER2_CLUSTER_INODE;
	if (cluster) {
		nipdata = &hammer2_cluster_rdata(cluster)->ipdata;
		nip->inum = nipdata->inum;
		nip->size = nipdata->size;
		nip->mtime = nipdata->mtime;
		hammer2_inode_repoint(nip, NULL, cluster);
	} else {
		nip->inum = 1;			/* PFS inum is always 1 XXX */
		/* mtime will be updated when a cluster is available */
	}

	nip->pip = dip;				/* can be NULL */
	if (dip)
		hammer2_inode_ref(dip);	/* ref dip for nip->pip */

	nip->pmp = pmp;

	/*
	 * ref and lock on nip gives it state compatible to after a
	 * hammer2_inode_lock() call.
	 */
	nip->refs = 1;
	hammer2_mtx_init(&nip->lock, "h2inode");
	hammer2_mtx_ex(&nip->lock);
	/* combination of thread lock and chain lock == inode lock */

	/*
	 * Attempt to add the inode.  If it fails we raced another inode
	 * get.  Undo all the work and try again.
	 */
	if (pmp->spmp_hmp == NULL) {
		hammer2_spin_ex(&pmp->inum_spin);
		if (RB_INSERT(hammer2_inode_tree, &pmp->inum_tree, nip)) {
			hammer2_spin_unex(&pmp->inum_spin);
			hammer2_mtx_unlock(&nip->lock);
			hammer2_inode_drop(nip);
			goto again;
		}
		atomic_set_int(&nip->flags, HAMMER2_INODE_ONRBTREE);
		hammer2_spin_unex(&pmp->inum_spin);
	}

	return (nip);
}

/*
 * Create a new inode in the specified directory using the vattr to
 * figure out the type of inode.
 *
 * If no error occurs the new inode with its cluster locked is returned in
 * *nipp, otherwise an error is returned and *nipp is set to NULL.
 *
 * If vap and/or cred are NULL the related fields are not set and the
 * inode type defaults to a directory.  This is used when creating PFSs
 * under the super-root, so the inode number is set to 1 in this case.
 *
 * dip is not locked on entry.
 *
 * NOTE: When used to create a snapshot, the inode is temporarily associated
 *	 with the super-root spmp. XXX should pass new pmp for snapshot.
 */
hammer2_inode_t *
hammer2_inode_create(hammer2_trans_t *trans, hammer2_inode_t *dip,
		     struct vattr *vap, struct ucred *cred,
		     const uint8_t *name, size_t name_len,
		     hammer2_cluster_t **clusterp,
		     int flags, int *errorp)
{
	const hammer2_inode_data_t *dipdata;
	hammer2_inode_data_t *nipdata;
	hammer2_cluster_t *cluster;
	hammer2_cluster_t *cparent;
	hammer2_inode_t *nip;
	hammer2_key_t key_dummy;
	hammer2_key_t lhc;
	int error;
	uid_t xuid;
	uuid_t dip_uid;
	uuid_t dip_gid;
	uint32_t dip_mode;
	uint8_t dip_comp_algo;
	uint8_t dip_check_algo;

	lhc = hammer2_dirhash(name, name_len);
	*errorp = 0;

	/*
	 * Locate the inode or indirect block to create the new
	 * entry in.  At the same time check for key collisions
	 * and iterate until we don't get one.
	 *
	 * NOTE: hidden inodes do not have iterators.
	 */
retry:
	cparent = hammer2_inode_lock(dip, HAMMER2_RESOLVE_ALWAYS);
	dipdata = &hammer2_cluster_rdata(cparent)->ipdata;
	dip_uid = dipdata->uid;
	dip_gid = dipdata->gid;
	dip_mode = dipdata->mode;
	dip_comp_algo = dipdata->comp_algo;
	dip_check_algo = dipdata->check_algo;

	error = 0;
	while (error == 0) {
		cluster = hammer2_cluster_lookup(cparent, &key_dummy,
						 lhc, lhc, 0);
		if (cluster == NULL)
			break;
		if ((lhc & HAMMER2_DIRHASH_VISIBLE) == 0)
			error = ENOSPC;
		if ((lhc & HAMMER2_DIRHASH_LOMASK) == HAMMER2_DIRHASH_LOMASK)
			error = ENOSPC;
		hammer2_cluster_unlock(cluster);
		hammer2_cluster_drop(cluster);
		cluster = NULL;
		++lhc;
	}

	if (error == 0) {
		error = hammer2_cluster_create(trans, cparent, &cluster,
					     lhc, 0,
					     HAMMER2_BREF_TYPE_INODE,
					     HAMMER2_INODE_BYTES,
					     flags);
	}
#if INODE_DEBUG
	kprintf("CREATE INODE %*.*s chain=%p\n",
		(int)name_len, (int)name_len, name,
		(cluster ? cluster->focus : NULL));
#endif

	/*
	 * Cleanup and handle retries.
	 */
	if (error == EAGAIN) {
		hammer2_cluster_ref(cparent);
		hammer2_inode_unlock(dip, cparent);
		hammer2_cluster_wait(cparent);
		hammer2_cluster_drop(cparent);
		goto retry;
	}
	hammer2_inode_unlock(dip, cparent);
	cparent = NULL;

	if (error) {
		KKASSERT(cluster == NULL);
		*errorp = error;
		return (NULL);
	}

	/*
	 * Set up the new inode.
	 *
	 * NOTE: *_get() integrates chain's lock into the inode lock.
	 *
	 * NOTE: Only one new inode can currently be created per
	 *	 transaction.  If the need arises we can adjust
	 *	 hammer2_trans_init() to allow more.
	 *
	 * NOTE: nipdata will have chain's blockset data.
	 */
	KKASSERT(cluster->focus->flags & HAMMER2_CHAIN_MODIFIED);
	nipdata = &hammer2_cluster_wdata(cluster)->ipdata;
	nipdata->inum = trans->inode_tid;
	hammer2_cluster_modsync(cluster);
	nip = hammer2_inode_get(dip->pmp, dip, cluster);
	nipdata = &hammer2_cluster_wdata(cluster)->ipdata;

	if (vap) {
		KKASSERT(trans->inodes_created == 0);
		nipdata->type = hammer2_get_obj_type(vap->va_type);
		nipdata->inum = trans->inode_tid;
		++trans->inodes_created;

		switch (nipdata->type) {
		case HAMMER2_OBJTYPE_CDEV:
		case HAMMER2_OBJTYPE_BDEV:
			nipdata->rmajor = vap->va_rmajor;
			nipdata->rminor = vap->va_rminor;
			break;
		default:
			break;
		}
	} else {
		nipdata->type = HAMMER2_OBJTYPE_DIRECTORY;
		nipdata->inum = 1;
	}

	/* Inherit parent's inode compression mode. */
	nip->comp_heuristic = 0;
	nipdata->comp_algo = dip_comp_algo;
	nipdata->check_algo = dip_check_algo;
	nipdata->version = HAMMER2_INODE_VERSION_ONE;
	hammer2_update_time(&nipdata->ctime);
	nipdata->mtime = nipdata->ctime;
	if (vap)
		nipdata->mode = vap->va_mode;
	nipdata->nlinks = 1;
	if (vap) {
		if (dip && dip->pmp) {
			xuid = hammer2_to_unix_xid(&dip_uid);
			xuid = vop_helper_create_uid(dip->pmp->mp,
						     dip_mode,
						     xuid,
						     cred,
						     &vap->va_mode);
		} else {
			/* super-root has no dip and/or pmp */
			xuid = 0;
		}
		if (vap->va_vaflags & VA_UID_UUID_VALID)
			nipdata->uid = vap->va_uid_uuid;
		else if (vap->va_uid != (uid_t)VNOVAL)
			hammer2_guid_to_uuid(&nipdata->uid, vap->va_uid);
		else
			hammer2_guid_to_uuid(&nipdata->uid, xuid);

		if (vap->va_vaflags & VA_GID_UUID_VALID)
			nipdata->gid = vap->va_gid_uuid;
		else if (vap->va_gid != (gid_t)VNOVAL)
			hammer2_guid_to_uuid(&nipdata->gid, vap->va_gid);
		else if (dip)
			nipdata->gid = dip_gid;
	}

	/*
	 * Regular files and softlinks allow a small amount of data to be
	 * directly embedded in the inode.  This flag will be cleared if
	 * the size is extended past the embedded limit.
	 */
	if (nipdata->type == HAMMER2_OBJTYPE_REGFILE ||
	    nipdata->type == HAMMER2_OBJTYPE_SOFTLINK) {
		nipdata->op_flags |= HAMMER2_OPFLAG_DIRECTDATA;
	}

	KKASSERT(name_len < HAMMER2_INODE_MAXNAME);
	bcopy(name, nipdata->filename, name_len);
	nipdata->name_key = lhc;
	nipdata->name_len = name_len;
	hammer2_cluster_modsync(cluster);
	*clusterp = cluster;

	return (nip);
}

/*
 * The cluster has been removed from the original directory and replaced
 * with a hardlink pointer.  Move the cluster to the specified parent
 * directory, change the filename to "0xINODENUMBER", and adjust the key.
 * The cluster becomes our invisible hardlink target.
 *
 * The original cluster must be deleted on entry.
 */
static
void
hammer2_hardlink_shiftup(hammer2_trans_t *trans, hammer2_cluster_t *cluster,
			hammer2_inode_t *dip, hammer2_cluster_t *dcluster,
			int nlinks, int *errorp)
{
	const hammer2_inode_data_t *iptmp;
	hammer2_inode_data_t *nipdata;
	hammer2_cluster_t *xcluster;
	hammer2_key_t key_dummy;
	hammer2_key_t lhc;
	hammer2_blockref_t bref;

	iptmp = &hammer2_cluster_rdata(cluster)->ipdata;
	lhc = iptmp->inum;
	KKASSERT((lhc & HAMMER2_DIRHASH_VISIBLE) == 0);

	/*
	 * Locate the inode or indirect block to create the new
	 * entry in.  lhc represents the inode number so there is
	 * no collision iteration.
	 *
	 * There should be no key collisions with invisible inode keys.
	 *
	 * WARNING! Must use inode_lock_ex() on dip to handle a stale
	 *	    dip->cluster cache.
	 */
	*errorp = 0;
	xcluster = hammer2_cluster_lookup(dcluster, &key_dummy,
				      lhc, lhc, 0);
	if (xcluster) {
		kprintf("X3 chain %p dip %p dchain %p dip->chain %p\n",
			xcluster->focus, dip, dcluster->focus,
			dip->cluster.focus);
		hammer2_cluster_unlock(xcluster);
		hammer2_cluster_drop(xcluster);
		xcluster = NULL;
		*errorp = ENOSPC;
#if 0
		Debugger("X3");
#endif
	}

	/*
	 * Handle the error case
	 */
	if (*errorp) {
		panic("error2");
		KKASSERT(xcluster == NULL);
		return;
	}

	/*
	 * Use xcluster as a placeholder for (lhc).  Duplicate cluster to the
	 * same target bref as xcluster and then delete xcluster.  The
	 * duplication occurs after xcluster in flush order even though
	 * xcluster is deleted after the duplication. XXX
	 *
	 * WARNING! Duplications (to a different parent) can cause indirect
	 *	    blocks to be inserted, refactor xcluster.
	 *
	 * WARNING! Only key and keybits is extracted from a passed-in bref.
	 */
	hammer2_cluster_bref(cluster, &bref);
	bref.key = lhc;			/* invisible dir entry key */
	bref.keybits = 0;
	hammer2_cluster_rename(trans, &bref, dcluster, cluster, 0);

	/*
	 * cluster is now 'live' again.. adjust the filename.
	 *
	 * Directory entries are inodes but this is a hidden hardlink
	 * target.  The name isn't used but to ease debugging give it
	 * a name after its inode number.
	 */
	hammer2_cluster_modify(trans, cluster, 0);
	nipdata = &hammer2_cluster_wdata(cluster)->ipdata;
	ksnprintf(nipdata->filename, sizeof(nipdata->filename),
		  "0x%016jx", (intmax_t)nipdata->inum);
	nipdata->name_len = strlen(nipdata->filename);
	nipdata->name_key = lhc;
	nipdata->nlinks += nlinks;
	hammer2_cluster_modsync(cluster);
}

/*
 * Connect the target inode represented by (cluster) to the media topology
 * at (dip, name, len).  The caller can pass a rough *chainp, this function
 * will issue lookup()s to position the parent chain properly for the
 * chain insertion.
 *
 * If hlink is TRUE this function creates an OBJTYPE_HARDLINK directory
 * entry instead of connecting (cluster).
 *
 * If hlink is FALSE this function expects (cluster) to be unparented.
 */
int
hammer2_inode_connect(hammer2_trans_t *trans,
		      hammer2_cluster_t **clusterp, int hlink,
		      hammer2_inode_t *dip, hammer2_cluster_t *dcluster,
		      const uint8_t *name, size_t name_len,
		      hammer2_key_t lhc)
{
	hammer2_inode_data_t *wipdata;
	hammer2_cluster_t *ocluster;
	hammer2_cluster_t *ncluster;
	hammer2_key_t key_dummy;
	int error;

	/*
	 * Since ocluster is either disconnected from the topology or
	 * represents a hardlink terminus which is always a parent of or
	 * equal to dip, we should be able to safely lock dip->chain for
	 * our setup.
	 *
	 * WARNING! Must use inode_lock_ex() on dip to handle a stale
	 *	    dip->cluster.
	 *
	 * If name is non-NULL we calculate lhc, else we use the passed-in
	 * lhc.
	 */
	ocluster = *clusterp;

	if (name) {
		lhc = hammer2_dirhash(name, name_len);

		/*
		 * Locate the inode or indirect block to create the new
		 * entry in.  At the same time check for key collisions
		 * and iterate until we don't get one.
		 */
		error = 0;
		while (error == 0) {
			ncluster = hammer2_cluster_lookup(dcluster, &key_dummy,
						      lhc, lhc, 0);
			if (ncluster == NULL)
				break;
			if ((lhc & HAMMER2_DIRHASH_LOMASK) ==
			    HAMMER2_DIRHASH_LOMASK) {
				error = ENOSPC;
			}
			hammer2_cluster_unlock(ncluster);
			hammer2_cluster_drop(ncluster);
			ncluster = NULL;
			++lhc;
		}
	} else {
		/*
		 * Reconnect to specific key (used when moving
		 * unlinked-but-open files into the hidden directory).
		 */
		ncluster = hammer2_cluster_lookup(dcluster, &key_dummy,
						  lhc, lhc, 0);
		KKASSERT(ncluster == NULL);
		error = 0;
	}

	if (error == 0) {
		if (hlink) {
			/*
			 * Hardlink pointer needed, create totally fresh
			 * directory entry.
			 *
			 * We must refactor ocluster because it might have
			 * been shifted into an indirect cluster by the
			 * create.
			 */
			KKASSERT(ncluster == NULL);
			error = hammer2_cluster_create(trans,
						       dcluster, &ncluster,
						       lhc, 0,
						       HAMMER2_BREF_TYPE_INODE,
						       HAMMER2_INODE_BYTES,
						       0);
		} else {
			/*
			 * Reconnect the original cluster under the new name.
			 * Original cluster must have already been deleted by
			 * teh caller.
			 *
			 * WARNING! Can cause held-over clusters to require a
			 *	    refactor.  Fortunately we have none (our
			 *	    locked clusters are passed into and
			 *	    modified by the call).
			 */
			ncluster = ocluster;
			ocluster = NULL;
			error = hammer2_cluster_create(trans,
						       dcluster, &ncluster,
						       lhc, 0,
						       HAMMER2_BREF_TYPE_INODE,
						       HAMMER2_INODE_BYTES,
						       0);
		}
	}

	/*
	 * Unlock stuff.
	 */
	KKASSERT(error != EAGAIN);

	/*
	 * ncluster should be NULL on error, leave ocluster
	 * (ocluster == *clusterp) alone.
	 */
	if (error) {
		KKASSERT(ncluster == NULL);
		return (error);
	}

	/*
	 * Directory entries are inodes so if the name has changed we have
	 * to update the inode.
	 *
	 * When creating an OBJTYPE_HARDLINK entry remember to unlock the
	 * cluster, the caller will access the hardlink via the actual hardlink
	 * target file and not the hardlink pointer entry, so we must still
	 * return ocluster.
	 */
	if (hlink && hammer2_hardlink_enable >= 0) {
		/*
		 * Create the HARDLINK pointer.  oip represents the hardlink
		 * target in this situation.
		 *
		 * We will return ocluster (the hardlink target).
		 */
		hammer2_cluster_modify(trans, ncluster, 0);
		hammer2_cluster_clr_chainflags(ncluster,
					       HAMMER2_CHAIN_UNLINKED);
		KKASSERT(name_len < HAMMER2_INODE_MAXNAME);
		wipdata = &hammer2_cluster_wdata(ncluster)->ipdata;
		bcopy(name, wipdata->filename, name_len);
		wipdata->name_key = lhc;
		wipdata->name_len = name_len;
		wipdata->target_type =
				hammer2_cluster_rdata(ocluster)->ipdata.type;
		wipdata->type = HAMMER2_OBJTYPE_HARDLINK;
		wipdata->inum = hammer2_cluster_rdata(ocluster)->ipdata.inum;
		wipdata->version = HAMMER2_INODE_VERSION_ONE;
		wipdata->nlinks = 1;
		wipdata->op_flags = HAMMER2_OPFLAG_DIRECTDATA;
		hammer2_cluster_modsync(ncluster);
		hammer2_cluster_unlock(ncluster);
		hammer2_cluster_drop(ncluster);
		ncluster = ocluster;
		ocluster = NULL;
	} else {
		/*
		 * ncluster is a duplicate of ocluster at the new location.
		 * We must fixup the name stored in the inode data.
		 * The bref key has already been adjusted by inode_connect().
		 */
		hammer2_cluster_modify(trans, ncluster, 0);
		hammer2_cluster_clr_chainflags(ncluster,
					       HAMMER2_CHAIN_UNLINKED);
		wipdata = &hammer2_cluster_wdata(ncluster)->ipdata;

		KKASSERT(name_len < HAMMER2_INODE_MAXNAME);
		bcopy(name, wipdata->filename, name_len);
		wipdata->name_key = lhc;
		wipdata->name_len = name_len;
		wipdata->nlinks = 1;
		hammer2_cluster_modsync(ncluster);
	}

	/*
	 * We are replacing ocluster with ncluster, unlock ocluster.  In the
	 * case where ocluster is left unchanged the code above sets
	 * ncluster to ocluster and ocluster to NULL, resulting in a NOP here.
	 */
	if (ocluster) {
		hammer2_cluster_unlock(ocluster);
		hammer2_cluster_drop(ocluster);
	}
	*clusterp = ncluster;

	return (0);
}

/*
 * Repoint ip->cluster's chains to cluster's chains and fixup the default
 * focus.  Only valid elements are repointed.  Invalid elements have to be
 * adjusted by the appropriate slave sync threads.
 *
 * Caller must hold the inode and cluster exclusive locked, if not NULL,
 * must also be locked.
 *
 * Cluster may be NULL to clean out any chains in ip->cluster.
 */
void
hammer2_inode_repoint(hammer2_inode_t *ip, hammer2_inode_t *pip,
		      hammer2_cluster_t *cluster)
{
	hammer2_chain_t *dropch[HAMMER2_MAXCLUSTER];
	hammer2_chain_t *ochain;
	hammer2_chain_t *nchain;
	hammer2_inode_t *opip;
	int i;

	bzero(dropch, sizeof(dropch));

	/*
	 * Replace chains in ip->cluster with chains from cluster and
	 * adjust the focus if necessary.
	 *
	 * NOTE: nchain and/or ochain can be NULL due to gaps
	 *	 in the cluster arrays.
	 */
	hammer2_spin_ex(&ip->cluster_spin);
	for (i = 0; cluster && i < cluster->nchains; ++i) {
		/*
		 * Do not replace invalid elements as this might race
		 * syncthr replacements.
		 */
		if (cluster->array[i].flags & HAMMER2_CITEM_INVALID)
			continue;

		/*
		 * Do not replace elements which are the same.  Also handle
		 * element count discrepancies.
		 */
		nchain = cluster->array[i].chain;
		if (i < ip->cluster.nchains) {
			ochain = ip->cluster.array[i].chain;
			if (ochain == nchain)
				continue;
		} else {
			ochain = NULL;
		}

		/*
		 * Make adjustments
		 */
		ip->cluster.array[i].chain = nchain;
		ip->cluster.array[i].flags &= ~HAMMER2_CITEM_INVALID;
		ip->cluster.array[i].flags |= cluster->array[i].flags &
					      HAMMER2_CITEM_INVALID;
		if (nchain)
			hammer2_chain_ref(nchain);
		dropch[i] = ochain;
	}

	/*
	 * Release any left-over chains in ip->cluster.
	 */
	while (i < ip->cluster.nchains) {
		nchain = ip->cluster.array[i].chain;
		if (nchain) {
			ip->cluster.array[i].chain = NULL;
			ip->cluster.array[i].flags |= HAMMER2_CITEM_INVALID;
		}
		dropch[i] = nchain;
		++i;
	}

	/*
	 * Fixup fields.  Note that the inode-embedded cluster is never
	 * directly locked.
	 */
	if (cluster) {
		ip->cluster.nchains = cluster->nchains;
		ip->cluster.focus = cluster->focus;
		ip->cluster.flags = cluster->flags & ~HAMMER2_CLUSTER_LOCKED;
	} else {
		ip->cluster.nchains = 0;
		ip->cluster.focus = NULL;
		ip->cluster.flags &= ~HAMMER2_CLUSTER_ZFLAGS;
	}

	/*
	 * Repoint ip->pip if requested (non-NULL pip).
	 */
	if (pip && ip->pip != pip) {
		opip = ip->pip;
		hammer2_inode_ref(pip);
		ip->pip = pip;
	} else {
		opip = NULL;
	}
	hammer2_spin_unex(&ip->cluster_spin);

	/*
	 * Cleanup outside of spinlock
	 */
	while (--i >= 0) {
		if (dropch[i])
			hammer2_chain_drop(dropch[i]);
	}
	if (opip)
		hammer2_inode_drop(opip);
}

/*
 * Repoint a single element from the cluster to the ip.  Used by the
 * synchronization threads to piecemeal update inodes.  Does not change
 * focus and requires inode to be re-locked to clean-up flags (XXX).
 */
void
hammer2_inode_repoint_one(hammer2_inode_t *ip, hammer2_cluster_t *cluster,
			  int idx)
{
	hammer2_chain_t *ochain;
	hammer2_chain_t *nchain;
	int i;

	hammer2_spin_ex(&ip->cluster_spin);
	KKASSERT(idx < cluster->nchains);
	if (idx < ip->cluster.nchains) {
		ochain = ip->cluster.array[idx].chain;
		nchain = cluster->array[idx].chain;
	} else {
		ochain = NULL;
		nchain = cluster->array[idx].chain;
		ip->cluster.nchains = idx + 1;
		for (i = ip->cluster.nchains; i <= idx; ++i) {
			bzero(&ip->cluster.array[i],
			      sizeof(ip->cluster.array[i]));
			ip->cluster.array[i].flags |= HAMMER2_CITEM_INVALID;
		}
	}
	if (ochain != nchain) {
		/*
		 * Make adjustments.
		 */
		ip->cluster.array[idx].chain = nchain;
		ip->cluster.array[idx].flags &= ~HAMMER2_CITEM_INVALID;
		ip->cluster.array[idx].flags |= cluster->array[idx].flags &
						HAMMER2_CITEM_INVALID;
	}
	hammer2_spin_unex(&ip->cluster_spin);
	if (ochain != nchain) {
		if (nchain)
			hammer2_chain_ref(nchain);
		if (ochain)
			hammer2_chain_drop(ochain);
	}
}

/*
 * Unlink the file from the specified directory inode.  The directory inode
 * does not need to be locked.
 *
 * isdir determines whether a directory/non-directory check should be made.
 * No check is made if isdir is set to -1.
 *
 * isopen specifies whether special unlink-with-open-descriptor handling
 * must be performed.  If set to -1 the caller is deleting a PFS and we
 * check whether the chain is mounted or not (chain->pmp != NULL).  1 is
 * implied if it is mounted.
 *
 * If isopen is 1 and nlinks drops to 0 this function must move the chain
 * to a special hidden directory until last-close occurs on the file.
 *
 * NOTE!  The underlying file can still be active with open descriptors
 *	  or if the chain is being manually held (e.g. for rename).
 *
 *	  The caller is responsible for fixing up ip->chain if e.g. a
 *	  rename occurs (see chain_duplicate()).
 *
 * NOTE!  The chain is not deleted if it is moved to the hidden directory,
 *	  but otherwise will be deleted.
 */
int
hammer2_unlink_file(hammer2_trans_t *trans, hammer2_inode_t *dip,
		    const uint8_t *name, size_t name_len,
		    int isdir, int *hlinkp, struct nchandle *nch,
		    int nlinks)
{
	const hammer2_inode_data_t *ripdata;
	hammer2_inode_data_t *wipdata;
	hammer2_cluster_t *cparent;
	hammer2_cluster_t *hcluster;
	hammer2_cluster_t *hparent;
	hammer2_cluster_t *cluster;
	hammer2_cluster_t *dparent;
	hammer2_cluster_t *dcluster;
	hammer2_key_t key_dummy;
	hammer2_key_t key_next;
	hammer2_key_t lhc;
	int last_link;
	int error;
	int hlink;
	uint8_t type;

	error = 0;
	hlink = 0;
	hcluster = NULL;
	hparent = NULL;
	lhc = hammer2_dirhash(name, name_len);

again:
	/*
	 * Search for the filename in the directory
	 */
	cparent = hammer2_inode_lock(dip, HAMMER2_RESOLVE_ALWAYS);
	cluster = hammer2_cluster_lookup(cparent, &key_next,
				     lhc, lhc + HAMMER2_DIRHASH_LOMASK, 0);
	while (cluster) {
		if (hammer2_cluster_type(cluster) == HAMMER2_BREF_TYPE_INODE) {
			ripdata = &hammer2_cluster_rdata(cluster)->ipdata;
			if (ripdata->name_len == name_len &&
			    bcmp(ripdata->filename, name, name_len) == 0) {
				break;
			}
		}
		cluster = hammer2_cluster_next(cparent, cluster, &key_next,
					       key_next,
					       lhc + HAMMER2_DIRHASH_LOMASK,
					       0);
	}
	hammer2_inode_unlock(dip, NULL);	/* retain cparent */

	/*
	 * Not found or wrong type (isdir < 0 disables the type check).
	 * If a hardlink pointer, type checks use the hardlink target.
	 */
	if (cluster == NULL) {
		error = ENOENT;
		goto done;
	}
	ripdata = &hammer2_cluster_rdata(cluster)->ipdata;
	type = ripdata->type;
	if (type == HAMMER2_OBJTYPE_HARDLINK) {
		hlink = 1;
		type = ripdata->target_type;
	}

	if (type == HAMMER2_OBJTYPE_DIRECTORY && isdir == 0) {
		error = ENOTDIR;
		goto done;
	}
	if (type != HAMMER2_OBJTYPE_DIRECTORY && isdir >= 1) {
		error = EISDIR;
		goto done;
	}

	/*
	 * Hardlink must be resolved.  We can't hold the parent locked
	 * while we do this or we could deadlock.  The physical file will
	 * be located at or above the current directory.
	 *
	 * We loop to reacquire the hardlink origination.
	 *
	 * NOTE: hammer2_hardlink_find() will locate the hardlink target,
	 *	 returning a modified hparent and hcluster.
	 */
	if (ripdata->type == HAMMER2_OBJTYPE_HARDLINK) {
		if (hcluster == NULL) {
			hcluster = cluster;
			cluster = NULL;	/* safety */
			hammer2_cluster_unlock(cparent);
			hammer2_cluster_drop(cparent);
			cparent = NULL; /* safety */
			ripdata = NULL;	/* safety (associated w/cparent) */
			error = hammer2_hardlink_find(dip, &hparent, &hcluster);

			/*
			 * If we couldn't find the hardlink target then some
			 * parent directory containing the hardlink pointer
			 * probably got renamed to above the original target,
			 * a case not yet handled by H2.
			 */
			if (error) {
				kprintf("H2 unlink_file: hardlink target for "
					"\"%s\" not found\n",
					name);
				kprintf("(likely due to known directory "
					"rename bug)\n");
				goto done;
			}
			goto again;
		}
	}

	/*
	 * If this is a directory the directory must be empty.  However, if
	 * isdir < 0 we are doing a rename and the directory does not have
	 * to be empty, and if isdir > 1 we are deleting a PFS/snapshot
	 * and the directory does not have to be empty.
	 *
	 * NOTE: We check the full key range here which covers both visible
	 *	 and invisible entries.  Theoretically there should be no
	 *	 invisible (hardlink target) entries if there are no visible
	 *	 entries.
	 */
	if (type == HAMMER2_OBJTYPE_DIRECTORY && isdir == 1) {
		dparent = hammer2_cluster_lookup_init(cluster, 0);
		dcluster = hammer2_cluster_lookup(dparent, &key_dummy,
					          0, (hammer2_key_t)-1,
					          HAMMER2_LOOKUP_NODATA);
		if (dcluster) {
			hammer2_cluster_unlock(dcluster);
			hammer2_cluster_drop(dcluster);
			hammer2_cluster_lookup_done(dparent);
			error = ENOTEMPTY;
			goto done;
		}
		hammer2_cluster_lookup_done(dparent);
		dparent = NULL;
		/* dcluster NULL */
	}

	/*
	 * If this was a hardlink then (cparent, cluster) is the hardlink
	 * pointer, which we can simply destroy outright.  Discard the
	 * clusters and replace with the hardlink target.
	 */
	if (hcluster) {
		hammer2_cluster_delete(trans, cparent, cluster,
				       HAMMER2_DELETE_PERMANENT);
		hammer2_cluster_unlock(cparent);
		hammer2_cluster_drop(cparent);
		hammer2_cluster_unlock(cluster);
		hammer2_cluster_drop(cluster);
		cparent = hparent;
		cluster = hcluster;
		hparent = NULL;
		hcluster = NULL;
	}

	/*
	 * This leaves us with the hardlink target or non-hardlinked file
	 * or directory in (cparent, cluster).
	 *
	 * Delete the target when nlinks reaches 0 with special handling
	 * to avoid I/O (to avoid actually updating the inode) for the 1->0
	 * transition, if possible.  This optimization makes rm -rf very
	 * fast.
	 *
	 * NOTE! In DragonFly the vnops function calls cache_unlink() after
	 *	 calling us here to clean out the namecache association,
	 *	 (which does not represent a ref for the open-test), and to
	 *	 force finalization of the vnode if/when the last ref gets
	 *	 dropped.
	 *
	 * NOTE! Files are unlinked by rename and then relinked.  nch will be
	 *	 passed as NULL in this situation.  hammer2_inode_connect()
	 *	 will bump nlinks.
	 */
	KKASSERT(cluster != NULL);

	/*
	 * Note: nlinks is negative when decrementing, positive when
	 *	 incrementing.
	 */
	ripdata = &hammer2_cluster_rdata(cluster)->ipdata;
	last_link = (ripdata->nlinks + nlinks == 0);

	if (last_link) {
		/*
		 * Target nlinks has reached 0, file now unlinked (but may
		 * still be open).
		 */
		/* XXX need interlock if mounted
		if ((cluster->focus->flags & HAMMER2_CHAIN_PFSROOT) &&
		    cluster->pmp) {
			error = EINVAL;
			kprintf("hammer2: PFS \"%s\" cannot be deleted "
				"while still mounted\n",
				wipdata->filename);
			goto done;
		}
		*/
		hammer2_cluster_set_chainflags(cluster, HAMMER2_CHAIN_UNLINKED);
		if (nch && cache_isopen(nch)) {
			/*
			 * If an unlinked file is still open we must update
			 * the inodes link count.
			 */
			hammer2_cluster_modify(trans, cluster, 0);
			wipdata = &hammer2_cluster_wdata(cluster)->ipdata;
			ripdata = wipdata;
			wipdata->nlinks += nlinks;
			/* XXX debugging */
			if ((int64_t)wipdata->nlinks < 0) {
				wipdata->nlinks = 0;
			}
			hammer2_cluster_modsync(cluster);
			hammer2_inode_move_to_hidden(trans, &cparent, &cluster,
						     wipdata->inum);
		} else {
			/*
			 * This won't get everything if a vnode is still
			 * present, but the cache_unlink() call the caller
			 * makes will.
			 */
			hammer2_cluster_delete(trans, cparent, cluster,
					       HAMMER2_DELETE_PERMANENT);
		}
	} else if (hlink == 0) {
		/*
		 * In this situation a normal non-hardlinked file (which can
		 * only have nlinks == 1) still has a non-zero nlinks, the
		 * caller must be doing a RENAME operation and so is passing
		 * a nlinks adjustment of 0, and only wishes to remove file
		 * in order to be able to reconnect it under a different name.
		 *
		 * In this situation we do a temporary deletion of the
		 * chain in order to allow the file to be reconnected in
		 * a different location.
		 */
		KKASSERT(nlinks == 0);
		hammer2_cluster_delete(trans, cparent, cluster, 0);
	} else {
		/*
		 * Links remain, must update the inode link count.
		 */
		hammer2_cluster_modify(trans, cluster, 0);
		wipdata = &hammer2_cluster_wdata(cluster)->ipdata;
		ripdata = wipdata;
		wipdata->nlinks += nlinks;
		if ((int64_t)wipdata->nlinks < 0) {	/* XXX debugging */
			wipdata->nlinks = 0;
		}
		hammer2_cluster_modsync(cluster);
	}

	error = 0;
done:
	if (cparent) {
		hammer2_cluster_unlock(cparent);
		hammer2_cluster_drop(cparent);
	}
	if (cluster) {
		hammer2_cluster_unlock(cluster);
		hammer2_cluster_drop(cluster);
	}
	if (hparent) {
		hammer2_cluster_unlock(hparent);
		hammer2_cluster_drop(hparent);
	}
	if (hcluster) {
		hammer2_cluster_unlock(hcluster);
		hammer2_cluster_drop(hcluster);
	}
	if (hlinkp)
		*hlinkp = hlink;

	return error;
}

/*
 * This is called from the mount code to initialize pmp->ihidden
 */
void
hammer2_inode_install_hidden(hammer2_pfs_t *pmp)
{
	hammer2_trans_t trans;
	hammer2_cluster_t *cparent;
	hammer2_cluster_t *cluster;
	hammer2_cluster_t *scan;
	const hammer2_inode_data_t *ripdata;
	hammer2_inode_data_t *wipdata;
	hammer2_key_t key_dummy;
	hammer2_key_t key_next;
	int error;
	int count;
	int dip_check_algo;
	int dip_comp_algo;

	if (pmp->ihidden)
		return;

	/*
	 * Find the hidden directory
	 */
	bzero(&key_dummy, sizeof(key_dummy));
	hammer2_trans_init(&trans, pmp, 0);

	/*
	 * Setup for lookup, retrieve iroot's check and compression
	 * algorithm request which was likely generated by newfs_hammer2.
	 *
	 * The check/comp fields will probably never be used since inodes
	 * are renamed into the hidden directory and not created relative to
	 * the hidden directory, chain creation inherits from bref.methods,
	 * and data chains inherit from their respective file inode *_algo
	 * fields.
	 */
	cparent = hammer2_inode_lock(pmp->iroot, HAMMER2_RESOLVE_ALWAYS);
	ripdata = &hammer2_cluster_rdata(cparent)->ipdata;
	dip_check_algo = ripdata->check_algo;
	dip_comp_algo = ripdata->comp_algo;
	ripdata = NULL;

	cluster = hammer2_cluster_lookup(cparent, &key_dummy,
					 HAMMER2_INODE_HIDDENDIR,
					 HAMMER2_INODE_HIDDENDIR,
					 0);
	if (cluster) {
		pmp->ihidden = hammer2_inode_get(pmp, pmp->iroot, cluster);
		hammer2_inode_ref(pmp->ihidden);

		/*
		 * Remove any unlinked files which were left open as-of
		 * any system crash.
		 *
		 * Don't pass NODATA, we need the inode data so the delete
		 * can do proper statistics updates.
		 */
		count = 0;
		scan = hammer2_cluster_lookup(cluster, &key_next,
					      0, HAMMER2_TID_MAX, 0);
		while (scan) {
			if (hammer2_cluster_type(scan) ==
			    HAMMER2_BREF_TYPE_INODE) {
				hammer2_cluster_delete(&trans, cluster, scan,
						   HAMMER2_DELETE_PERMANENT);
				++count;
			}
			scan = hammer2_cluster_next(cluster, scan, &key_next,
						    0, HAMMER2_TID_MAX, 0);
		}

		hammer2_inode_unlock(pmp->ihidden, cluster);
		hammer2_inode_unlock(pmp->iroot, cparent);
		hammer2_trans_done(&trans);
		kprintf("hammer2: PFS loaded hidden dir, "
			"removed %d dead entries\n", count);
		return;
	}

	/*
	 * Create the hidden directory
	 */
	error = hammer2_cluster_create(&trans, cparent, &cluster,
				       HAMMER2_INODE_HIDDENDIR, 0,
				       HAMMER2_BREF_TYPE_INODE,
				       HAMMER2_INODE_BYTES,
				       0);
	hammer2_inode_unlock(pmp->iroot, cparent);

	hammer2_cluster_modify(&trans, cluster, 0);
	wipdata = &hammer2_cluster_wdata(cluster)->ipdata;
	wipdata->type = HAMMER2_OBJTYPE_DIRECTORY;
	wipdata->inum = HAMMER2_INODE_HIDDENDIR;
	wipdata->nlinks = 1;
	wipdata->comp_algo = dip_comp_algo;
	wipdata->check_algo = dip_check_algo;
	hammer2_cluster_modsync(cluster);
	kprintf("hammer2: PFS root missing hidden directory, creating\n");

	pmp->ihidden = hammer2_inode_get(pmp, pmp->iroot, cluster);
	hammer2_inode_ref(pmp->ihidden);
	hammer2_inode_unlock(pmp->ihidden, cluster);
	hammer2_trans_done(&trans);
}

/*
 * If an open file is unlinked H2 needs to retain the file in the topology
 * to ensure that its backing store is not recovered by the bulk free scan.
 * This also allows us to avoid having to special-case the CHAIN_DELETED flag.
 *
 * To do this the file is moved to a hidden directory in the PFS root and
 * renamed.  The hidden directory must be created if it does not exist.
 */
static
void
hammer2_inode_move_to_hidden(hammer2_trans_t *trans,
			     hammer2_cluster_t **cparentp,
			     hammer2_cluster_t **clusterp,
			     hammer2_tid_t inum)
{
	hammer2_cluster_t *dcluster;
	hammer2_pfs_t *pmp;
	int error;

	pmp = (*clusterp)->pmp;
	KKASSERT(pmp != NULL);
	KKASSERT(pmp->ihidden != NULL);

	hammer2_cluster_delete(trans, *cparentp, *clusterp, 0);
	dcluster = hammer2_inode_lock(pmp->ihidden, HAMMER2_RESOLVE_ALWAYS);
	error = hammer2_inode_connect(trans, clusterp, 0,
				      pmp->ihidden, dcluster,
				      NULL, 0, inum);
	hammer2_inode_unlock(pmp->ihidden, dcluster);
	KKASSERT(error == 0);
}

/*
 * Given an exclusively locked inode and cluster we consolidate the cluster
 * for hardlink creation, adding (nlinks) to the file's link count and
 * potentially relocating the inode to (cdip) which is a parent directory
 * common to both the current location of the inode and the intended new
 * hardlink.
 *
 * Replaces (*clusterp) if consolidation occurred, unlocking the old cluster
 * and returning a new locked cluster.
 *
 * NOTE!  This function will also replace ip->cluster.
 */
int
hammer2_hardlink_consolidate(hammer2_trans_t *trans,
			     hammer2_inode_t *ip,
			     hammer2_cluster_t **clusterp,
			     hammer2_inode_t *cdip,
			     hammer2_cluster_t *cdcluster,
			     int nlinks)
{
	const hammer2_inode_data_t *ripdata;
	hammer2_inode_data_t *wipdata;
	hammer2_cluster_t *cluster;
	hammer2_cluster_t *cparent;
	int error;

	cluster = *clusterp;
	ripdata = &hammer2_cluster_rdata(cluster)->ipdata;
	if (nlinks == 0 &&			/* no hardlink needed */
	    (ripdata->name_key & HAMMER2_DIRHASH_VISIBLE)) {
		return (0);
	}

	if (hammer2_hardlink_enable == 0) {	/* disallow hardlinks */
		hammer2_cluster_unlock(cluster);
		hammer2_cluster_drop(cluster);
		*clusterp = NULL;
		return (ENOTSUP);
	}

	cparent = NULL;

	/*
	 * If no change in the hardlink's target directory is required and
	 * this is already a hardlink target, all we need to do is adjust
	 * the link count.
	 */
	ripdata = &hammer2_cluster_rdata(cluster)->ipdata;
	if (cdip == ip->pip &&
	    (ripdata->name_key & HAMMER2_DIRHASH_VISIBLE) == 0) {
		if (nlinks) {
			hammer2_cluster_modify(trans, cluster, 0);
			wipdata = &hammer2_cluster_wdata(cluster)->ipdata;
			wipdata->nlinks += nlinks;
			hammer2_cluster_modsync(cluster);
			ripdata = wipdata;
		}
		error = 0;
		goto done;
	}

	/*
	 * Cluster is the real inode.  The originating directory is locked
	 * by the caller so we can manipulate it without worrying about races
	 * against other lookups.
	 *
	 * If cluster is visible we need to delete it from the current
	 * location and create a hardlink pointer in its place.  If it is
	 * not visible we need only delete it.  Then later cluster will be
	 * renamed to a parent directory and converted (if necessary) to
	 * a hidden inode (via shiftup).
	 *
	 * NOTE! We must hold cparent locked through the delete/create/rename
	 *	 operation to ensure that other threads block resolving to
	 *	 the same hardlink, otherwise the other threads may not see
	 *	 the hardlink.
	 */
	KKASSERT((cluster->focus->flags & HAMMER2_CHAIN_DELETED) == 0);
	cparent = hammer2_cluster_parent(cluster);

	hammer2_cluster_delete(trans, cparent, cluster, 0);

	ripdata = &hammer2_cluster_rdata(cluster)->ipdata;
	KKASSERT(ripdata->type != HAMMER2_OBJTYPE_HARDLINK);
	if (ripdata->name_key & HAMMER2_DIRHASH_VISIBLE) {
		hammer2_cluster_t *ncluster;
		hammer2_key_t lhc;

		ncluster = NULL;
		lhc = cluster->focus->bref.key;
		error = hammer2_cluster_create(trans, cparent, &ncluster,
					     lhc, 0,
					     HAMMER2_BREF_TYPE_INODE,
					     HAMMER2_INODE_BYTES,
					     0);
		hammer2_cluster_modify(trans, ncluster, 0);
		wipdata = &hammer2_cluster_wdata(ncluster)->ipdata;

		/* wipdata->comp_algo = ripdata->comp_algo; */
		wipdata->comp_algo = 0;
		wipdata->check_algo = 0;
		wipdata->version = HAMMER2_INODE_VERSION_ONE;
		wipdata->inum = ripdata->inum;
		wipdata->target_type = ripdata->type;
		wipdata->type = HAMMER2_OBJTYPE_HARDLINK;
		wipdata->uflags = 0;
		wipdata->rmajor = 0;
		wipdata->rminor = 0;
		wipdata->ctime = 0;
		wipdata->mtime = 0;
		wipdata->atime = 0;
		wipdata->btime = 0;
		bzero(&wipdata->uid, sizeof(wipdata->uid));
		bzero(&wipdata->gid, sizeof(wipdata->gid));
		wipdata->op_flags = HAMMER2_OPFLAG_DIRECTDATA;
		wipdata->cap_flags = 0;
		wipdata->mode = 0;
		wipdata->size = 0;
		wipdata->nlinks = 1;
		wipdata->iparent = 0;	/* XXX */
		wipdata->pfs_type = 0;
		wipdata->pfs_inum = 0;
		bzero(&wipdata->pfs_clid, sizeof(wipdata->pfs_clid));
		bzero(&wipdata->pfs_fsid, sizeof(wipdata->pfs_fsid));
		wipdata->data_quota = 0;
		/* wipdata->data_count = 0; */
		wipdata->inode_quota = 0;
		/* wipdata->inode_count = 0; */
		wipdata->attr_tid = 0;
		wipdata->dirent_tid = 0;
		bzero(&wipdata->u, sizeof(wipdata->u));
		bcopy(ripdata->filename, wipdata->filename, ripdata->name_len);
		wipdata->name_key = ncluster->focus->bref.key;
		wipdata->name_len = ripdata->name_len;
		/* XXX transaction ids */
		hammer2_cluster_modsync(ncluster);
		hammer2_cluster_unlock(ncluster);
		hammer2_cluster_drop(ncluster);
	}
	ripdata = wipdata;

	/*
	 * cluster represents the hardlink target and is now flagged deleted.
	 * duplicate it to the parent directory and adjust nlinks.
	 *
	 * WARNING! The shiftup() call can cause ncluster to be moved into
	 *	    an indirect block, and our ncluster will wind up pointing
	 *	    to the older/original version.
	 */
	KKASSERT(cluster->focus->flags & HAMMER2_CHAIN_DELETED);
	hammer2_hardlink_shiftup(trans, cluster, cdip, cdcluster,
				 nlinks, &error);

	if (error == 0)
		hammer2_inode_repoint(ip, cdip, cluster);

done:
	/*
	 * Cleanup, cluster/ncluster already dealt with.
	 *
	 * Return the shifted cluster in *clusterp.
	 */
	if (cparent) {
		hammer2_cluster_unlock(cparent);
		hammer2_cluster_drop(cparent);
	}
	*clusterp = cluster;

	return (error);
}

/*
 * If (*ochainp) is non-NULL it points to the forward OBJTYPE_HARDLINK
 * inode while (*chainp) points to the resolved (hidden hardlink
 * target) inode.  In this situation when nlinks is 1 we wish to
 * deconsolidate the hardlink, moving it back to the directory that now
 * represents the only remaining link.
 */
int
hammer2_hardlink_deconsolidate(hammer2_trans_t *trans,
			       hammer2_inode_t *dip,
			       hammer2_chain_t **chainp,
			       hammer2_chain_t **ochainp)
{
	if (*ochainp == NULL)
		return (0);
	/* XXX */
	return (0);
}

/*
 * The caller presents a locked cluster with an obj_type of
 * HAMMER2_OBJTYPE_HARDLINK in (*clusterp).  This routine will locate
 * the inode and replace (*clusterp) with a new locked cluster containing
 * the target hardlink, also locked.  The original cluster will be
 * unlocked and released.
 *
 * If cparentp is not NULL a locked cluster representing the hardlink's
 * parent is also returned.
 *
 * If we are unable to locate the hardlink target EIO is returned,
 * (*cparentp) is set to NULL, the original passed-in (*clusterp)
 * will be unlocked and released and (*clusterp) will be set to NULL
 * as well.
 */
int
hammer2_hardlink_find(hammer2_inode_t *dip,
		      hammer2_cluster_t **cparentp,
		      hammer2_cluster_t **clusterp)
{
	const hammer2_inode_data_t *ipdata;
	hammer2_cluster_t *cluster;
	hammer2_cluster_t *cparent;
	hammer2_cluster_t *rcluster;
	hammer2_inode_t *ip;
	hammer2_inode_t *pip;
	hammer2_key_t key_dummy;
	hammer2_key_t lhc;

	cluster = *clusterp;
	pip = dip;
	hammer2_inode_ref(pip);		/* for loop */

	/*
	 * Locate the hardlink.  pip is referenced and not locked.
	 * Unlock and release (*clusterp) after extracting the needed
	 * data.
	 */
	ipdata = &hammer2_cluster_rdata(cluster)->ipdata;
	lhc = ipdata->inum;
	ipdata = NULL;			/* safety */
	hammer2_cluster_unlock(cluster);
	hammer2_cluster_drop(cluster);
	*clusterp = NULL;		/* safety */

	rcluster = NULL;
	cparent = NULL;

	while ((ip = pip) != NULL) {
		cparent = hammer2_inode_lock(ip, HAMMER2_RESOLVE_ALWAYS);
		hammer2_inode_drop(ip);			/* loop */
		KKASSERT(hammer2_cluster_type(cparent) ==
			 HAMMER2_BREF_TYPE_INODE);
		rcluster = hammer2_cluster_lookup(cparent, &key_dummy,
					     lhc, lhc, 0);
		if (rcluster)
			break;
		hammer2_cluster_lookup_done(cparent);	/* discard parent */
		cparent = NULL;				/* safety */
		pip = ip->pip;		/* safe, ip held locked */
		if (pip)
			hammer2_inode_ref(pip);		/* loop */
		hammer2_inode_unlock(ip, NULL);
	}

	/*
	 * chain is locked, ip is locked.  Unlock ip, return the locked
	 * chain.  *ipp is already set w/a ref count and not locked.
	 *
	 * (cparent is already unlocked).
	 */
	*clusterp = rcluster;
	if (rcluster) {
		if (cparentp) {
			*cparentp = cparent;
			hammer2_inode_unlock(ip, NULL);
		} else {
			hammer2_inode_unlock(ip, cparent);
		}
		return (0);
	} else {
		if (cparentp)
			*cparentp = NULL;
		if (ip)
			hammer2_inode_unlock(ip, cparent);
		return (EIO);
	}
}

/*
 * Find the directory common to both fdip and tdip.
 *
 * Returns a held but not locked inode.  Caller typically locks the inode,
 * and when through unlocks AND drops it.
 */
hammer2_inode_t *
hammer2_inode_common_parent(hammer2_inode_t *fdip, hammer2_inode_t *tdip)
{
	hammer2_inode_t *scan1;
	hammer2_inode_t *scan2;

	/*
	 * We used to have a depth field but it complicated matters too
	 * much for directory renames.  So now its ugly.  Check for
	 * simple cases before giving up and doing it the expensive way.
	 *
	 * XXX need a bottom-up topology stability lock
	 */
	if (fdip == tdip || fdip == tdip->pip) {
		hammer2_inode_ref(fdip);
		return(fdip);
	}
	if (fdip->pip == tdip) {
		hammer2_inode_ref(tdip);
		return(tdip);
	}

	/*
	 * XXX not MPSAFE
	 */
	for (scan1 = fdip; scan1->pmp == fdip->pmp; scan1 = scan1->pip) {
		scan2 = tdip;
		while (scan2->pmp == tdip->pmp) {
			if (scan1 == scan2) {
				hammer2_inode_ref(scan1);
				return(scan1);
			}
			scan2 = scan2->pip;
			if (scan2 == NULL)
				break;
		}
	}
	panic("hammer2_inode_common_parent: no common parent %p %p\n",
	      fdip, tdip);
	/* NOT REACHED */
	return(NULL);
}

/*
 * Synchronize the inode's frontend state with the chain state prior
 * to any explicit flush of the inode or any strategy write call.
 *
 * Called with a locked inode.
 */
void
hammer2_inode_fsync(hammer2_trans_t *trans, hammer2_inode_t *ip,
		    hammer2_cluster_t *cparent)
{
	const hammer2_inode_data_t *ripdata;
	hammer2_inode_data_t *wipdata;
	hammer2_cluster_t *dparent;
	hammer2_cluster_t *cluster;
	hammer2_key_t lbase;
	hammer2_key_t key_next;
	int dosync = 0;

	ripdata = &hammer2_cluster_rdata(cparent)->ipdata;    /* target file */

	if (ip->flags & HAMMER2_INODE_MTIME) {
		wipdata = hammer2_cluster_modify_ip(trans, ip, cparent, 0);
		atomic_clear_int(&ip->flags, HAMMER2_INODE_MTIME);
		wipdata->mtime = ip->mtime;
		dosync = 1;
		ripdata = wipdata;
	}
	if ((ip->flags & HAMMER2_INODE_RESIZED) && ip->size < ripdata->size) {
		wipdata = hammer2_cluster_modify_ip(trans, ip, cparent, 0);
		wipdata->size = ip->size;
		dosync = 1;
		ripdata = wipdata;
		atomic_clear_int(&ip->flags, HAMMER2_INODE_RESIZED);

		/*
		 * We must delete any chains beyond the EOF.  The chain
		 * straddling the EOF will be pending in the bioq.
		 */
		lbase = (ripdata->size + HAMMER2_PBUFMASK64) &
			~HAMMER2_PBUFMASK64;
		dparent = hammer2_cluster_lookup_init(&ip->cluster, 0);
		cluster = hammer2_cluster_lookup(dparent, &key_next,
					         lbase, (hammer2_key_t)-1,
						 HAMMER2_LOOKUP_NODATA);
		while (cluster) {
			/*
			 * Degenerate embedded case, nothing to loop on
			 */
			switch (hammer2_cluster_type(cluster)) {
			case HAMMER2_BREF_TYPE_INODE:
				hammer2_cluster_unlock(cluster);
				hammer2_cluster_drop(cluster);
				cluster = NULL;
				break;
			case HAMMER2_BREF_TYPE_DATA:
				hammer2_cluster_delete(trans, dparent, cluster,
						   HAMMER2_DELETE_PERMANENT);
				/* fall through */
			default:
				cluster = hammer2_cluster_next(dparent, cluster,
						   &key_next,
						   key_next, (hammer2_key_t)-1,
						   HAMMER2_LOOKUP_NODATA);
				break;
			}
		}
		hammer2_cluster_lookup_done(dparent);
	} else
	if ((ip->flags & HAMMER2_INODE_RESIZED) && ip->size > ripdata->size) {
		wipdata = hammer2_cluster_modify_ip(trans, ip, cparent, 0);
		wipdata->size = ip->size;
		atomic_clear_int(&ip->flags, HAMMER2_INODE_RESIZED);

		/*
		 * When resizing larger we may not have any direct-data
		 * available.
		 */
		if ((wipdata->op_flags & HAMMER2_OPFLAG_DIRECTDATA) &&
		    ip->size > HAMMER2_EMBEDDED_BYTES) {
			wipdata->op_flags &= ~HAMMER2_OPFLAG_DIRECTDATA;
			bzero(&wipdata->u.blockset,
			      sizeof(wipdata->u.blockset));
		}
		dosync = 1;
		ripdata = wipdata;
	}
	if (dosync)
		hammer2_cluster_modsync(cparent);
}