xref: /linux/drivers/md/dm-vdo/recovery-journal.c (revision 3584240b)

// SPDX-License-Identifier: GPL-2.0-only
/*
 * Copyright 2023 Red Hat
 */

#include "recovery-journal.h"

#include <linux/atomic.h>
#include <linux/bio.h>

#include "logger.h"
#include "memory-alloc.h"
#include "permassert.h"

#include "block-map.h"
#include "completion.h"
#include "constants.h"
#include "data-vio.h"
#include "encodings.h"
#include "io-submitter.h"
#include "slab-depot.h"
#include "types.h"
#include "vdo.h"
#include "vio.h"
#include "wait-queue.h"

static const u64 RECOVERY_COUNT_MASK = 0xff;

/*
 * The number of reserved blocks must be large enough to prevent a new recovery journal
 * block write from overwriting a block which appears to still be a valid head block of the
 * journal. Currently, that means reserving enough space for all 2048 data_vios.
 */
#define RECOVERY_JOURNAL_RESERVED_BLOCKS				\
	((MAXIMUM_VDO_USER_VIOS / RECOVERY_JOURNAL_ENTRIES_PER_BLOCK) + 2)

/**
 * DOC: Lock Counters.
 *
 * A lock_counter is intended to keep all of the locks for the blocks in the recovery journal. The
 * per-zone counters are all kept in a single array which is arranged by zone (i.e. zone 0's lock 0
 * is at index 0, zone 0's lock 1 is at index 1, and zone 1's lock 0 is at index 'locks'. This
 * arrangement is intended to minimize cache-line contention for counters from different zones.
 *
 * The locks are implemented as a single object instead of as a lock counter per lock both to
 * afford this opportunity to reduce cache line contention and also to eliminate the need to have a
 * completion per lock.
 *
 * Lock sets are laid out with the set for recovery journal first, followed by the logical zones,
 * and then the physical zones.
 */

enum lock_counter_state {
	LOCK_COUNTER_STATE_NOT_NOTIFYING,
	LOCK_COUNTER_STATE_NOTIFYING,
	LOCK_COUNTER_STATE_SUSPENDED,
};

/**
 * get_zone_count_ptr() - Get a pointer to the zone count for a given lock on a given zone.
 * @journal: The recovery journal.
 * @lock_number: The lock to get.
 * @zone_type: The zone type whose count is desired.
 *
 * Return: A pointer to the zone count for the given lock and zone.
 */
static inline atomic_t *get_zone_count_ptr(struct recovery_journal *journal,
					   block_count_t lock_number,
					   enum vdo_zone_type zone_type)
{
	return ((zone_type == VDO_ZONE_TYPE_LOGICAL)
		? &journal->lock_counter.logical_zone_counts[lock_number]
		: &journal->lock_counter.physical_zone_counts[lock_number]);
}

/**
 * get_counter() - Get the zone counter for a given lock on a given zone.
 * @journal: The recovery journal.
 * @lock_number: The lock to get.
 * @zone_type: The zone type whose count is desired.
 * @zone_id: The zone index whose count is desired.
 *
 * Return: The counter for the given lock and zone.
 */
static inline u16 *get_counter(struct recovery_journal *journal,
			       block_count_t lock_number, enum vdo_zone_type zone_type,
			       zone_count_t zone_id)
{
	struct lock_counter *counter = &journal->lock_counter;
	block_count_t zone_counter = (counter->locks * zone_id) + lock_number;

	if (zone_type == VDO_ZONE_TYPE_JOURNAL)
		return &counter->journal_counters[zone_counter];

	if (zone_type == VDO_ZONE_TYPE_LOGICAL)
		return &counter->logical_counters[zone_counter];

	return &counter->physical_counters[zone_counter];
}

static atomic_t *get_decrement_counter(struct recovery_journal *journal,
				       block_count_t lock_number)
{
	return &journal->lock_counter.journal_decrement_counts[lock_number];
}

/**
 * is_journal_zone_locked() - Check whether the journal zone is locked for a given lock.
 * @journal: The recovery journal.
 * @lock_number: The lock to check.
 *
 * Return: true if the journal zone is locked.
 */
static bool is_journal_zone_locked(struct recovery_journal *journal,
				   block_count_t lock_number)
{
	u16 journal_value = *get_counter(journal, lock_number, VDO_ZONE_TYPE_JOURNAL, 0);
	u32 decrements = atomic_read(get_decrement_counter(journal, lock_number));

	/* Pairs with barrier in vdo_release_journal_entry_lock() */
	smp_rmb();
	VDO_ASSERT_LOG_ONLY((decrements <= journal_value),
			    "journal zone lock counter must not underflow");
	return (journal_value != decrements);
}

/**
 * vdo_release_recovery_journal_block_reference() - Release a reference to a recovery journal
 *                                                  block.
 * @journal: The recovery journal.
 * @sequence_number: The journal sequence number of the referenced block.
 * @zone_type: The type of the zone making the adjustment.
 * @zone_id: The ID of the zone making the adjustment.
 *
 * If this is the last reference for a given zone type, an attempt will be made to reap the
 * journal.
 */
void vdo_release_recovery_journal_block_reference(struct recovery_journal *journal,
						  sequence_number_t sequence_number,
						  enum vdo_zone_type zone_type,
						  zone_count_t zone_id)
{
	u16 *current_value;
	block_count_t lock_number;
	int prior_state;

	if (sequence_number == 0)
		return;

	lock_number = vdo_get_recovery_journal_block_number(journal, sequence_number);
	current_value = get_counter(journal, lock_number, zone_type, zone_id);

	VDO_ASSERT_LOG_ONLY((*current_value >= 1),
			    "decrement of lock counter must not underflow");
	*current_value -= 1;

	if (zone_type == VDO_ZONE_TYPE_JOURNAL) {
		if (is_journal_zone_locked(journal, lock_number))
			return;
	} else {
		atomic_t *zone_count;

		if (*current_value != 0)
			return;

		zone_count = get_zone_count_ptr(journal, lock_number, zone_type);

		if (atomic_add_return(-1, zone_count) > 0)
			return;
	}

	/*
	 * Extra barriers because this was original developed using a CAS operation that implicitly
	 * had them.
	 */
	smp_mb__before_atomic();
	prior_state = atomic_cmpxchg(&journal->lock_counter.state,
				     LOCK_COUNTER_STATE_NOT_NOTIFYING,
				     LOCK_COUNTER_STATE_NOTIFYING);
	/* same as before_atomic */
	smp_mb__after_atomic();

	if (prior_state != LOCK_COUNTER_STATE_NOT_NOTIFYING)
		return;

	vdo_launch_completion(&journal->lock_counter.completion);
}

static inline struct recovery_journal_block * __must_check get_journal_block(struct list_head *list)
{
	return list_first_entry_or_null(list, struct recovery_journal_block, list_node);
}

/**
 * pop_free_list() - Get a block from the end of the free list.
 * @journal: The journal.
 *
 * Return: The block or NULL if the list is empty.
 */
static struct recovery_journal_block * __must_check pop_free_list(struct recovery_journal *journal)
{
	struct recovery_journal_block *block;

	if (list_empty(&journal->free_tail_blocks))
		return NULL;

	block = list_last_entry(&journal->free_tail_blocks,
				struct recovery_journal_block, list_node);
	list_del_init(&block->list_node);
	return block;
}

/**
 * is_block_dirty() - Check whether a recovery block is dirty.
 * @block: The block to check.
 *
 * Indicates it has any uncommitted entries, which includes both entries not written and entries
 * written but not yet acknowledged.
 *
 * Return: true if the block has any uncommitted entries.
 */
static inline bool __must_check is_block_dirty(const struct recovery_journal_block *block)
{
	return (block->uncommitted_entry_count > 0);
}

/**
 * is_block_empty() - Check whether a journal block is empty.
 * @block: The block to check.
 *
 * Return: true if the block has no entries.
 */
static inline bool __must_check is_block_empty(const struct recovery_journal_block *block)
{
	return (block->entry_count == 0);
}

/**
 * is_block_full() - Check whether a journal block is full.
 * @block: The block to check.
 *
 * Return: true if the block is full.
 */
static inline bool __must_check is_block_full(const struct recovery_journal_block *block)
{
	return ((block == NULL) || (block->journal->entries_per_block == block->entry_count));
}

/**
 * assert_on_journal_thread() - Assert that we are running on the journal thread.
 * @journal: The journal.
 * @function_name: The function doing the check (for logging).
 */
static void assert_on_journal_thread(struct recovery_journal *journal,
				     const char *function_name)
{
	VDO_ASSERT_LOG_ONLY((vdo_get_callback_thread_id() == journal->thread_id),
			    "%s() called on journal thread", function_name);
}

/**
 * continue_waiter() - Release a data_vio from the journal.
 *
 * Invoked whenever a data_vio is to be released from the journal, either because its entry was
 * committed to disk, or because there was an error. Implements waiter_callback_fn.
 */
static void continue_waiter(struct vdo_waiter *waiter, void *context)
{
	continue_data_vio_with_error(vdo_waiter_as_data_vio(waiter), *((int *) context));
}

/**
 * has_block_waiters() - Check whether the journal has any waiters on any blocks.
 * @journal: The journal in question.
 *
 * Return: true if any block has a waiter.
 */
static inline bool has_block_waiters(struct recovery_journal *journal)
{
	struct recovery_journal_block *block = get_journal_block(&journal->active_tail_blocks);

	/*
	 * Either the first active tail block (if it exists) has waiters, or no active tail block
	 * has waiters.
	 */
	return ((block != NULL) &&
		(vdo_waitq_has_waiters(&block->entry_waiters) ||
		 vdo_waitq_has_waiters(&block->commit_waiters)));
}

static void recycle_journal_blocks(struct recovery_journal *journal);
static void recycle_journal_block(struct recovery_journal_block *block);
static void notify_commit_waiters(struct recovery_journal *journal);

/**
 * suspend_lock_counter() - Prevent the lock counter from notifying.
 * @counter: The counter.
 *
 * Return: true if the lock counter was not notifying and hence the suspend was efficacious.
 */
static bool suspend_lock_counter(struct lock_counter *counter)
{
	int prior_state;

	/*
	 * Extra barriers because this was originally developed using a CAS operation that
	 * implicitly had them.
	 */
	smp_mb__before_atomic();
	prior_state = atomic_cmpxchg(&counter->state, LOCK_COUNTER_STATE_NOT_NOTIFYING,
				     LOCK_COUNTER_STATE_SUSPENDED);
	/* same as before_atomic */
	smp_mb__after_atomic();

	return ((prior_state == LOCK_COUNTER_STATE_SUSPENDED) ||
		(prior_state == LOCK_COUNTER_STATE_NOT_NOTIFYING));
}

static inline bool is_read_only(struct recovery_journal *journal)
{
	return vdo_is_read_only(journal->flush_vio->completion.vdo);
}

/**
 * check_for_drain_complete() - Check whether the journal has drained.
 * @journal: The journal which may have just drained.
 */
static void check_for_drain_complete(struct recovery_journal *journal)
{
	int result = VDO_SUCCESS;

	if (is_read_only(journal)) {
		result = VDO_READ_ONLY;
		/*
		 * Clean up any full active blocks which were not written due to read-only mode.
		 *
		 * FIXME: This would probably be better as a short-circuit in write_block().
		 */
		notify_commit_waiters(journal);
		recycle_journal_blocks(journal);

		/* Release any data_vios waiting to be assigned entries. */
		vdo_waitq_notify_all_waiters(&journal->entry_waiters,
					     continue_waiter, &result);
	}

	if (!vdo_is_state_draining(&journal->state) ||
	    journal->reaping ||
	    has_block_waiters(journal) ||
	    vdo_waitq_has_waiters(&journal->entry_waiters) ||
	    !suspend_lock_counter(&journal->lock_counter))
		return;

	if (vdo_is_state_saving(&journal->state)) {
		if (journal->active_block != NULL) {
			VDO_ASSERT_LOG_ONLY(((result == VDO_READ_ONLY) ||
					     !is_block_dirty(journal->active_block)),
					    "journal being saved has clean active block");
			recycle_journal_block(journal->active_block);
		}

		VDO_ASSERT_LOG_ONLY(list_empty(&journal->active_tail_blocks),
				    "all blocks in a journal being saved must be inactive");
	}

	vdo_finish_draining_with_result(&journal->state, result);
}

/**
 * notify_recovery_journal_of_read_only_mode() - Notify a recovery journal that the VDO has gone
 *                                               read-only.
 * @listener: The journal.
 * @parent: The completion to notify in order to acknowledge the notification.
 *
 * Implements vdo_read_only_notification_fn.
 */
static void notify_recovery_journal_of_read_only_mode(void *listener,
						      struct vdo_completion *parent)
{
	check_for_drain_complete(listener);
	vdo_finish_completion(parent);
}

/**
 * enter_journal_read_only_mode() - Put the journal in read-only mode.
 * @journal: The journal which has failed.
 * @error_code: The error result triggering this call.
 *
 * All attempts to add entries after this function is called will fail. All VIOs waiting for
 * commits will be awakened with an error.
 */
static void enter_journal_read_only_mode(struct recovery_journal *journal,
					 int error_code)
{
	vdo_enter_read_only_mode(journal->flush_vio->completion.vdo, error_code);
	check_for_drain_complete(journal);
}

/**
 * vdo_get_recovery_journal_current_sequence_number() - Obtain the recovery journal's current
 *                                                      sequence number.
 * @journal: The journal in question.
 *
 * Exposed only so the block map can be initialized therefrom.
 *
 * Return: The sequence number of the tail block.
 */
sequence_number_t vdo_get_recovery_journal_current_sequence_number(struct recovery_journal *journal)
{
	return journal->tail;
}

/**
 * get_recovery_journal_head() - Get the head of the recovery journal.
 * @journal: The journal.
 *
 * The head is the lowest sequence number of the block map head and the slab journal head.
 *
 * Return: the head of the journal.
 */
static inline sequence_number_t get_recovery_journal_head(const struct recovery_journal *journal)
{
	return min(journal->block_map_head, journal->slab_journal_head);
}

/**
 * compute_recovery_count_byte() - Compute the recovery count byte for a given recovery count.
 * @recovery_count: The recovery count.
 *
 * Return: The byte corresponding to the recovery count.
 */
static inline u8 __must_check compute_recovery_count_byte(u64 recovery_count)
{
	return (u8)(recovery_count & RECOVERY_COUNT_MASK);
}

/**
 * check_slab_journal_commit_threshold() - Check whether the journal is over the threshold, and if
 *                                         so, force the oldest slab journal tail block to commit.
 * @journal: The journal.
 */
static void check_slab_journal_commit_threshold(struct recovery_journal *journal)
{
	block_count_t current_length = journal->tail - journal->slab_journal_head;

	if (current_length > journal->slab_journal_commit_threshold) {
		journal->events.slab_journal_commits_requested++;
		vdo_commit_oldest_slab_journal_tail_blocks(journal->depot,
							   journal->slab_journal_head);
	}
}

static void reap_recovery_journal(struct recovery_journal *journal);
static void assign_entries(struct recovery_journal *journal);

/**
 * finish_reaping() - Finish reaping the journal.
 * @journal: The journal being reaped.
 */
static void finish_reaping(struct recovery_journal *journal)
{
	block_count_t blocks_reaped;
	sequence_number_t old_head = get_recovery_journal_head(journal);

	journal->block_map_head = journal->block_map_reap_head;
	journal->slab_journal_head = journal->slab_journal_reap_head;
	blocks_reaped = get_recovery_journal_head(journal) - old_head;
	journal->available_space += blocks_reaped * journal->entries_per_block;
	journal->reaping = false;
	check_slab_journal_commit_threshold(journal);
	assign_entries(journal);
	check_for_drain_complete(journal);
}

/**
 * complete_reaping() - Finish reaping the journal after flushing the lower layer.
 * @completion: The journal's flush VIO.
 *
 * This is the callback registered in reap_recovery_journal().
 */
static void complete_reaping(struct vdo_completion *completion)
{
	struct recovery_journal *journal = completion->parent;

	finish_reaping(journal);

	/* Try reaping again in case more locks were released while flush was out. */
	reap_recovery_journal(journal);
}

/**
 * handle_flush_error() - Handle an error when flushing the lower layer due to reaping.
 * @completion: The journal's flush VIO.
 */
static void handle_flush_error(struct vdo_completion *completion)
{
	struct recovery_journal *journal = completion->parent;

	vio_record_metadata_io_error(as_vio(completion));
	journal->reaping = false;
	enter_journal_read_only_mode(journal, completion->result);
}

static void flush_endio(struct bio *bio)
{
	struct vio *vio = bio->bi_private;
	struct recovery_journal *journal = vio->completion.parent;

	continue_vio_after_io(vio, complete_reaping, journal->thread_id);
}

/**
 * initialize_journal_state() - Set all journal fields appropriately to start journaling from the
 *                              current active block.
 * @journal: The journal to be reset based on its active block.
 */
static void initialize_journal_state(struct recovery_journal *journal)
{
	journal->append_point.sequence_number = journal->tail;
	journal->last_write_acknowledged = journal->tail;
	journal->block_map_head = journal->tail;
	journal->slab_journal_head = journal->tail;
	journal->block_map_reap_head = journal->tail;
	journal->slab_journal_reap_head = journal->tail;
	journal->block_map_head_block_number =
		vdo_get_recovery_journal_block_number(journal, journal->block_map_head);
	journal->slab_journal_head_block_number =
		vdo_get_recovery_journal_block_number(journal,
						      journal->slab_journal_head);
	journal->available_space =
		(journal->entries_per_block * vdo_get_recovery_journal_length(journal->size));
}

/**
 * vdo_get_recovery_journal_length() - Get the number of usable recovery journal blocks.
 * @journal_size: The size of the recovery journal in blocks.
 *
 * Return: the number of recovery journal blocks usable for entries.
 */
block_count_t vdo_get_recovery_journal_length(block_count_t journal_size)
{
	block_count_t reserved_blocks = journal_size / 4;

	if (reserved_blocks > RECOVERY_JOURNAL_RESERVED_BLOCKS)
		reserved_blocks = RECOVERY_JOURNAL_RESERVED_BLOCKS;
	return (journal_size - reserved_blocks);
}

/**
 * reap_recovery_journal_callback() - Attempt to reap the journal.
 * @completion: The lock counter completion.
 *
 * Attempts to reap the journal now that all the locks on some journal block have been released.
 * This is the callback registered with the lock counter.
 */
static void reap_recovery_journal_callback(struct vdo_completion *completion)
{
	struct recovery_journal *journal = (struct recovery_journal *) completion->parent;
	/*
	 * The acknowledgment must be done before reaping so that there is no race between
	 * acknowledging the notification and unlocks wishing to notify.
	 */
	smp_wmb();
	atomic_set(&journal->lock_counter.state, LOCK_COUNTER_STATE_NOT_NOTIFYING);

	if (vdo_is_state_quiescing(&journal->state)) {
		/*
		 * Don't start reaping when the journal is trying to quiesce. Do check if this
		 * notification is the last thing the is waiting on.
		 */
		check_for_drain_complete(journal);
		return;
	}

	reap_recovery_journal(journal);
	check_slab_journal_commit_threshold(journal);
}

/**
 * initialize_lock_counter() - Initialize a lock counter.
 *
 * @journal: The recovery journal.
 * @vdo: The vdo.
 *
 * Return: VDO_SUCCESS or an error.
 */
static int __must_check initialize_lock_counter(struct recovery_journal *journal,
						struct vdo *vdo)
{
	int result;
	struct thread_config *config = &vdo->thread_config;
	struct lock_counter *counter = &journal->lock_counter;

	result = vdo_allocate(journal->size, u16, __func__, &counter->journal_counters);
	if (result != VDO_SUCCESS)
		return result;

	result = vdo_allocate(journal->size, atomic_t, __func__,
			      &counter->journal_decrement_counts);
	if (result != VDO_SUCCESS)
		return result;

	result = vdo_allocate(journal->size * config->logical_zone_count, u16, __func__,
			      &counter->logical_counters);
	if (result != VDO_SUCCESS)
		return result;

	result = vdo_allocate(journal->size, atomic_t, __func__,
			      &counter->logical_zone_counts);
	if (result != VDO_SUCCESS)
		return result;

	result = vdo_allocate(journal->size * config->physical_zone_count, u16, __func__,
			      &counter->physical_counters);
	if (result != VDO_SUCCESS)
		return result;

	result = vdo_allocate(journal->size, atomic_t, __func__,
			      &counter->physical_zone_counts);
	if (result != VDO_SUCCESS)
		return result;

	vdo_initialize_completion(&counter->completion, vdo,
				  VDO_LOCK_COUNTER_COMPLETION);
	vdo_prepare_completion(&counter->completion, reap_recovery_journal_callback,
			       reap_recovery_journal_callback, config->journal_thread,
			       journal);
	counter->logical_zones = config->logical_zone_count;
	counter->physical_zones = config->physical_zone_count;
	counter->locks = journal->size;
	return VDO_SUCCESS;
}

/**
 * set_journal_tail() - Set the journal's tail sequence number.
 * @journal: The journal whose tail is to be set.
 * @tail: The new tail value.
 */
static void set_journal_tail(struct recovery_journal *journal, sequence_number_t tail)
{
	/* VDO does not support sequence numbers above 1 << 48 in the slab journal. */
	if (tail >= (1ULL << 48))
		enter_journal_read_only_mode(journal, VDO_JOURNAL_OVERFLOW);

	journal->tail = tail;
}

/**
 * initialize_recovery_block() - Initialize a journal block.
 * @vdo: The vdo from which to construct vios.
 * @journal: The journal to which the block will belong.
 * @block: The block to initialize.
 *
 * Return: VDO_SUCCESS or an error.
 */
static int initialize_recovery_block(struct vdo *vdo, struct recovery_journal *journal,
				     struct recovery_journal_block *block)
{
	char *data;
	int result;

	/*
	 * Ensure that a block is large enough to store RECOVERY_JOURNAL_ENTRIES_PER_BLOCK entries.
	 */
	BUILD_BUG_ON(RECOVERY_JOURNAL_ENTRIES_PER_BLOCK >
		     ((VDO_BLOCK_SIZE - sizeof(struct packed_journal_header)) /
		      sizeof(struct packed_recovery_journal_entry)));

	/*
	 * Allocate a full block for the journal block even though not all of the space is used
	 * since the VIO needs to write a full disk block.
	 */
	result = vdo_allocate(VDO_BLOCK_SIZE, char, __func__, &data);
	if (result != VDO_SUCCESS)
		return result;

	result = allocate_vio_components(vdo, VIO_TYPE_RECOVERY_JOURNAL,
					 VIO_PRIORITY_HIGH, block, 1, data, &block->vio);
	if (result != VDO_SUCCESS) {
		vdo_free(data);
		return result;
	}

	list_add_tail(&block->list_node, &journal->free_tail_blocks);
	block->journal = journal;
	return VDO_SUCCESS;
}

/**
 * vdo_decode_recovery_journal() - Make a recovery journal and initialize it with the state that
 *                                 was decoded from the super block.
 *
 * @state: The decoded state of the journal.
 * @nonce: The nonce of the VDO.
 * @vdo: The VDO.
 * @partition: The partition for the journal.
 * @recovery_count: The VDO's number of completed recoveries.
 * @journal_size: The number of blocks in the journal on disk.
 * @journal_ptr: The pointer to hold the new recovery journal.
 *
 * Return: A success or error code.
 */
int vdo_decode_recovery_journal(struct recovery_journal_state_7_0 state, nonce_t nonce,
				struct vdo *vdo, struct partition *partition,
				u64 recovery_count, block_count_t journal_size,
				struct recovery_journal **journal_ptr)
{
	block_count_t i;
	struct recovery_journal *journal;
	int result;

	result = vdo_allocate_extended(struct recovery_journal,
				       RECOVERY_JOURNAL_RESERVED_BLOCKS,
				       struct recovery_journal_block, __func__,
				       &journal);
	if (result != VDO_SUCCESS)
		return result;

	INIT_LIST_HEAD(&journal->free_tail_blocks);
	INIT_LIST_HEAD(&journal->active_tail_blocks);
	vdo_waitq_init(&journal->pending_writes);

	journal->thread_id = vdo->thread_config.journal_thread;
	journal->origin = partition->offset;
	journal->nonce = nonce;
	journal->recovery_count = compute_recovery_count_byte(recovery_count);
	journal->size = journal_size;
	journal->slab_journal_commit_threshold = (journal_size * 2) / 3;
	journal->logical_blocks_used = state.logical_blocks_used;
	journal->block_map_data_blocks = state.block_map_data_blocks;
	journal->entries_per_block = RECOVERY_JOURNAL_ENTRIES_PER_BLOCK;
	set_journal_tail(journal, state.journal_start);
	initialize_journal_state(journal);
	/* TODO: this will have to change if we make initial resume of a VDO a real resume */
	vdo_set_admin_state_code(&journal->state, VDO_ADMIN_STATE_SUSPENDED);

	for (i = 0; i < RECOVERY_JOURNAL_RESERVED_BLOCKS; i++) {
		struct recovery_journal_block *block = &journal->blocks[i];

		result = initialize_recovery_block(vdo, journal, block);
		if (result != VDO_SUCCESS) {
			vdo_free_recovery_journal(journal);
			return result;
		}
	}

	result = initialize_lock_counter(journal, vdo);
	if (result != VDO_SUCCESS) {
		vdo_free_recovery_journal(journal);
		return result;
	}

	result = create_metadata_vio(vdo, VIO_TYPE_RECOVERY_JOURNAL, VIO_PRIORITY_HIGH,
				     journal, NULL, &journal->flush_vio);
	if (result != VDO_SUCCESS) {
		vdo_free_recovery_journal(journal);
		return result;
	}

	result = vdo_register_read_only_listener(vdo, journal,
						 notify_recovery_journal_of_read_only_mode,
						 journal->thread_id);
	if (result != VDO_SUCCESS) {
		vdo_free_recovery_journal(journal);
		return result;
	}

	result = vdo_make_default_thread(vdo, journal->thread_id);
	if (result != VDO_SUCCESS) {
		vdo_free_recovery_journal(journal);
		return result;
	}

	journal->flush_vio->completion.callback_thread_id = journal->thread_id;
	*journal_ptr = journal;
	return VDO_SUCCESS;
}

/**
 * vdo_free_recovery_journal() - Free a recovery journal.
 * @journal: The recovery journal to free.
 */
void vdo_free_recovery_journal(struct recovery_journal *journal)
{
	block_count_t i;

	if (journal == NULL)
		return;

	vdo_free(vdo_forget(journal->lock_counter.logical_zone_counts));
	vdo_free(vdo_forget(journal->lock_counter.physical_zone_counts));
	vdo_free(vdo_forget(journal->lock_counter.journal_counters));
	vdo_free(vdo_forget(journal->lock_counter.journal_decrement_counts));
	vdo_free(vdo_forget(journal->lock_counter.logical_counters));
	vdo_free(vdo_forget(journal->lock_counter.physical_counters));
	free_vio(vdo_forget(journal->flush_vio));

	/*
	 * FIXME: eventually, the journal should be constructed in a quiescent state which
	 *        requires opening before use.
	 */
	if (!vdo_is_state_quiescent(&journal->state)) {
		VDO_ASSERT_LOG_ONLY(list_empty(&journal->active_tail_blocks),
				    "journal being freed has no active tail blocks");
	} else if (!vdo_is_state_saved(&journal->state) &&
		   !list_empty(&journal->active_tail_blocks)) {
		vdo_log_warning("journal being freed has uncommitted entries");
	}

	for (i = 0; i < RECOVERY_JOURNAL_RESERVED_BLOCKS; i++) {
		struct recovery_journal_block *block = &journal->blocks[i];

		vdo_free(vdo_forget(block->vio.data));
		free_vio_components(&block->vio);
	}

	vdo_free(journal);
}

/**
 * vdo_initialize_recovery_journal_post_repair() - Initialize the journal after a repair.
 * @journal: The journal in question.
 * @recovery_count: The number of completed recoveries.
 * @tail: The new tail block sequence number.
 * @logical_blocks_used: The new number of logical blocks used.
 * @block_map_data_blocks: The new number of block map data blocks.
 */
void vdo_initialize_recovery_journal_post_repair(struct recovery_journal *journal,
						 u64 recovery_count,
						 sequence_number_t tail,
						 block_count_t logical_blocks_used,
						 block_count_t block_map_data_blocks)
{
	set_journal_tail(journal, tail + 1);
	journal->recovery_count = compute_recovery_count_byte(recovery_count);
	initialize_journal_state(journal);
	journal->logical_blocks_used = logical_blocks_used;
	journal->block_map_data_blocks = block_map_data_blocks;
}

/**
 * vdo_get_journal_block_map_data_blocks_used() - Get the number of block map pages, allocated from
 *                                                data blocks, currently in use.
 * @journal: The journal in question.
 *
 * Return: The number of block map pages allocated from slabs.
 */
block_count_t vdo_get_journal_block_map_data_blocks_used(struct recovery_journal *journal)
{
	return journal->block_map_data_blocks;
}

/**
 * vdo_get_recovery_journal_thread_id() - Get the ID of a recovery journal's thread.
 * @journal: The journal to query.
 *
 * Return: The ID of the journal's thread.
 */
thread_id_t vdo_get_recovery_journal_thread_id(struct recovery_journal *journal)
{
	return journal->thread_id;
}

/**
 * vdo_open_recovery_journal() - Prepare the journal for new entries.
 * @journal: The journal in question.
 * @depot: The slab depot for this VDO.
 * @block_map: The block map for this VDO.
 */
void vdo_open_recovery_journal(struct recovery_journal *journal,
			       struct slab_depot *depot, struct block_map *block_map)
{
	journal->depot = depot;
	journal->block_map = block_map;
	WRITE_ONCE(journal->state.current_state, VDO_ADMIN_STATE_NORMAL_OPERATION);
}

/**
 * vdo_record_recovery_journal() - Record the state of a recovery journal for encoding in the super
 *                                 block.
 * @journal: the recovery journal.
 *
 * Return: the state of the journal.
 */
struct recovery_journal_state_7_0
vdo_record_recovery_journal(const struct recovery_journal *journal)
{
	struct recovery_journal_state_7_0 state = {
		.logical_blocks_used = journal->logical_blocks_used,
		.block_map_data_blocks = journal->block_map_data_blocks,
	};

	if (vdo_is_state_saved(&journal->state)) {
		/*
		 * If the journal is saved, we should start one past the active block (since the
		 * active block is not guaranteed to be empty).
		 */
		state.journal_start = journal->tail;
	} else {
		/*
		 * When we're merely suspended or have gone read-only, we must record the first
		 * block that might have entries that need to be applied.
		 */
		state.journal_start = get_recovery_journal_head(journal);
	}

	return state;
}

/**
 * get_block_header() - Get a pointer to the packed journal block header in the block buffer.
 * @block: The recovery block.
 *
 * Return: The block's header.
 */
static inline struct packed_journal_header *
get_block_header(const struct recovery_journal_block *block)
{
	return (struct packed_journal_header *) block->vio.data;
}

/**
 * set_active_sector() - Set the current sector of the current block and initialize it.
 * @block: The block to update.
 * @sector: A pointer to the first byte of the new sector.
 */
static void set_active_sector(struct recovery_journal_block *block, void *sector)
{
	block->sector = sector;
	block->sector->check_byte = get_block_header(block)->check_byte;
	block->sector->recovery_count = block->journal->recovery_count;
	block->sector->entry_count = 0;
}

/**
 * advance_tail() - Advance the tail of the journal.
 * @journal: The journal whose tail should be advanced.
 *
 * Return: true if the tail was advanced.
 */
static bool advance_tail(struct recovery_journal *journal)
{
	struct recovery_block_header unpacked;
	struct packed_journal_header *header;
	struct recovery_journal_block *block;

	block = journal->active_block = pop_free_list(journal);
	if (block == NULL)
		return false;

	list_move_tail(&block->list_node, &journal->active_tail_blocks);

	unpacked = (struct recovery_block_header) {
		.metadata_type = VDO_METADATA_RECOVERY_JOURNAL_2,
		.block_map_data_blocks = journal->block_map_data_blocks,
		.logical_blocks_used = journal->logical_blocks_used,
		.nonce = journal->nonce,
		.recovery_count = journal->recovery_count,
		.sequence_number = journal->tail,
		.check_byte = vdo_compute_recovery_journal_check_byte(journal,
								      journal->tail),
	};

	header = get_block_header(block);
	memset(block->vio.data, 0x0, VDO_BLOCK_SIZE);
	block->sequence_number = journal->tail;
	block->entry_count = 0;
	block->uncommitted_entry_count = 0;
	block->block_number = vdo_get_recovery_journal_block_number(journal,
								    journal->tail);

	vdo_pack_recovery_block_header(&unpacked, header);
	set_active_sector(block, vdo_get_journal_block_sector(header, 1));
	set_journal_tail(journal, journal->tail + 1);
	vdo_advance_block_map_era(journal->block_map, journal->tail);
	return true;
}

/**
 * initialize_lock_count() - Initialize the value of the journal zone's counter for a given lock.
 * @journal: The recovery journal.
 *
 * Context: This must be called from the journal zone.
 */
static void initialize_lock_count(struct recovery_journal *journal)
{
	u16 *journal_value;
	block_count_t lock_number = journal->active_block->block_number;
	atomic_t *decrement_counter = get_decrement_counter(journal, lock_number);

	journal_value = get_counter(journal, lock_number, VDO_ZONE_TYPE_JOURNAL, 0);
	VDO_ASSERT_LOG_ONLY((*journal_value == atomic_read(decrement_counter)),
			    "count to be initialized not in use");
	*journal_value = journal->entries_per_block + 1;
	atomic_set(decrement_counter, 0);
}

/**
 * prepare_to_assign_entry() - Prepare the currently active block to receive an entry and check
 *			       whether an entry of the given type may be assigned at this time.
 * @journal: The journal receiving an entry.
 *
 * Return: true if there is space in the journal to store an entry of the specified type.
 */
static bool prepare_to_assign_entry(struct recovery_journal *journal)
{
	if (journal->available_space == 0)
		return false;

	if (is_block_full(journal->active_block) && !advance_tail(journal))
		return false;

	if (!is_block_empty(journal->active_block))
		return true;

	if ((journal->tail - get_recovery_journal_head(journal)) > journal->size) {
		/* Cannot use this block since the journal is full. */
		journal->events.disk_full++;
		return false;
	}

	/*
	 * Don't allow the new block to be reaped until all of its entries have been committed to
	 * the block map and until the journal block has been fully committed as well. Because the
	 * block map update is done only after any slab journal entries have been made, the
	 * per-entry lock for the block map entry serves to protect those as well.
	 */
	initialize_lock_count(journal);
	return true;
}

static void write_blocks(struct recovery_journal *journal);

/**
 * schedule_block_write() - Queue a block for writing.
 * @journal: The journal in question.
 * @block: The block which is now ready to write.
 *
 * The block is expected to be full. If the block is currently writing, this is a noop as the block
 * will be queued for writing when the write finishes. The block must not currently be queued for
 * writing.
 */
static void schedule_block_write(struct recovery_journal *journal,
				 struct recovery_journal_block *block)
{
	if (!block->committing)
		vdo_waitq_enqueue_waiter(&journal->pending_writes, &block->write_waiter);
	/*
	 * At the end of adding entries, or discovering this partial block is now full and ready to
	 * rewrite, we will call write_blocks() and write a whole batch.
	 */
}

/**
 * release_journal_block_reference() - Release a reference to a journal block.
 * @block: The journal block from which to release a reference.
 */
static void release_journal_block_reference(struct recovery_journal_block *block)
{
	vdo_release_recovery_journal_block_reference(block->journal,
						     block->sequence_number,
						     VDO_ZONE_TYPE_JOURNAL, 0);
}

static void update_usages(struct recovery_journal *journal, struct data_vio *data_vio)
{
	if (data_vio->increment_updater.operation == VDO_JOURNAL_BLOCK_MAP_REMAPPING) {
		journal->block_map_data_blocks++;
		return;
	}

	if (data_vio->new_mapped.state != VDO_MAPPING_STATE_UNMAPPED)
		journal->logical_blocks_used++;

	if (data_vio->mapped.state != VDO_MAPPING_STATE_UNMAPPED)
		journal->logical_blocks_used--;
}

/**
 * assign_entry() - Assign an entry waiter to the active block.
 *
 * Implements waiter_callback_fn.
 */
static void assign_entry(struct vdo_waiter *waiter, void *context)
{
	struct data_vio *data_vio = vdo_waiter_as_data_vio(waiter);
	struct recovery_journal_block *block = context;
	struct recovery_journal *journal = block->journal;

	/* Record the point at which we will make the journal entry. */
	data_vio->recovery_journal_point = (struct journal_point) {
		.sequence_number = block->sequence_number,
		.entry_count = block->entry_count,
	};

	update_usages(journal, data_vio);
	journal->available_space--;

	if (!vdo_waitq_has_waiters(&block->entry_waiters))
		journal->events.blocks.started++;

	vdo_waitq_enqueue_waiter(&block->entry_waiters, &data_vio->waiter);
	block->entry_count++;
	block->uncommitted_entry_count++;
	journal->events.entries.started++;

	if (is_block_full(block)) {
		/*
		 * The block is full, so we can write it anytime henceforth. If it is already
		 * committing, we'll queue it for writing when it comes back.
		 */
		schedule_block_write(journal, block);
	}

	/* Force out slab journal tail blocks when threshold is reached. */
	check_slab_journal_commit_threshold(journal);
}

static void assign_entries(struct recovery_journal *journal)
{
	if (journal->adding_entries) {
		/* Protect against re-entrancy. */
		return;
	}

	journal->adding_entries = true;
	while (vdo_waitq_has_waiters(&journal->entry_waiters) &&
	       prepare_to_assign_entry(journal)) {
		vdo_waitq_notify_next_waiter(&journal->entry_waiters,
					     assign_entry, journal->active_block);
	}

	/* Now that we've finished with entries, see if we have a batch of blocks to write. */
	write_blocks(journal);
	journal->adding_entries = false;
}

/**
 * recycle_journal_block() - Prepare an in-memory journal block to be reused now that it has been
 *                           fully committed.
 * @block: The block to be recycled.
 */
static void recycle_journal_block(struct recovery_journal_block *block)
{
	struct recovery_journal *journal = block->journal;
	block_count_t i;

	list_move_tail(&block->list_node, &journal->free_tail_blocks);

	/* Release any unused entry locks. */
	for (i = block->entry_count; i < journal->entries_per_block; i++)
		release_journal_block_reference(block);

	/*
	 * Release our own lock against reaping now that the block is completely committed, or
	 * we're giving up because we're in read-only mode.
	 */
	if (block->entry_count > 0)
		release_journal_block_reference(block);

	if (block == journal->active_block)
		journal->active_block = NULL;
}

/**
 * continue_committed_waiter() - invoked whenever a VIO is to be released from the journal because
 *                               its entry was committed to disk.
 *
 * Implements waiter_callback_fn.
 */
static void continue_committed_waiter(struct vdo_waiter *waiter, void *context)
{
	struct data_vio *data_vio = vdo_waiter_as_data_vio(waiter);
	struct recovery_journal *journal = context;
	int result = (is_read_only(journal) ? VDO_READ_ONLY : VDO_SUCCESS);
	bool has_decrement;

	VDO_ASSERT_LOG_ONLY(vdo_before_journal_point(&journal->commit_point,
						     &data_vio->recovery_journal_point),
			    "DataVIOs released from recovery journal in order. Recovery journal point is (%llu, %u), but commit waiter point is (%llu, %u)",
			    (unsigned long long) journal->commit_point.sequence_number,
			    journal->commit_point.entry_count,
			    (unsigned long long) data_vio->recovery_journal_point.sequence_number,
			    data_vio->recovery_journal_point.entry_count);

	journal->commit_point = data_vio->recovery_journal_point;
	data_vio->last_async_operation = VIO_ASYNC_OP_UPDATE_REFERENCE_COUNTS;
	if (result != VDO_SUCCESS) {
		continue_data_vio_with_error(data_vio, result);
		return;
	}

	/*
	 * The increment must be launched first since it must come before the
	 * decrement if they are in the same slab.
	 */
	has_decrement = (data_vio->decrement_updater.zpbn.pbn != VDO_ZERO_BLOCK);
	if ((data_vio->increment_updater.zpbn.pbn != VDO_ZERO_BLOCK) || !has_decrement)
		continue_data_vio(data_vio);

	if (has_decrement)
		vdo_launch_completion(&data_vio->decrement_completion);
}

/**
 * notify_commit_waiters() - Notify any VIOs whose entries have now committed.
 * @journal: The recovery journal to update.
 */
static void notify_commit_waiters(struct recovery_journal *journal)
{
	struct recovery_journal_block *block;

	list_for_each_entry(block, &journal->active_tail_blocks, list_node) {
		if (block->committing)
			return;

		vdo_waitq_notify_all_waiters(&block->commit_waiters,
					     continue_committed_waiter, journal);
		if (is_read_only(journal)) {
			vdo_waitq_notify_all_waiters(&block->entry_waiters,
						     continue_committed_waiter,
						     journal);
		} else if (is_block_dirty(block) || !is_block_full(block)) {
			/* Stop at partially-committed or partially-filled blocks. */
			return;
		}
	}
}

/**
 * recycle_journal_blocks() - Recycle any journal blocks which have been fully committed.
 * @journal: The recovery journal to update.
 */
static void recycle_journal_blocks(struct recovery_journal *journal)
{
	struct recovery_journal_block *block, *tmp;

	list_for_each_entry_safe(block, tmp, &journal->active_tail_blocks, list_node) {
		if (block->committing) {
			/* Don't recycle committing blocks. */
			return;
		}

		if (!is_read_only(journal) &&
		    (is_block_dirty(block) || !is_block_full(block))) {
			/*
			 * Don't recycle partially written or partially full blocks, except in
			 * read-only mode.
			 */
			return;
		}

		recycle_journal_block(block);
	}
}

/**
 * complete_write() - Handle post-commit processing.
 * @completion: The completion of the VIO writing this block.
 *
 * This is the callback registered by write_block(). If more entries accumulated in the block being
 * committed while the commit was in progress, another commit will be initiated.
 */
static void complete_write(struct vdo_completion *completion)
{
	struct recovery_journal_block *block = completion->parent;
	struct recovery_journal *journal = block->journal;
	struct recovery_journal_block *last_active_block;

	assert_on_journal_thread(journal, __func__);

	journal->pending_write_count -= 1;
	journal->events.blocks.committed += 1;
	journal->events.entries.committed += block->entries_in_commit;
	block->uncommitted_entry_count -= block->entries_in_commit;
	block->entries_in_commit = 0;
	block->committing = false;

	/* If this block is the latest block to be acknowledged, record that fact. */
	if (block->sequence_number > journal->last_write_acknowledged)
		journal->last_write_acknowledged = block->sequence_number;

	last_active_block = get_journal_block(&journal->active_tail_blocks);
	VDO_ASSERT_LOG_ONLY((block->sequence_number >= last_active_block->sequence_number),
			    "completed journal write is still active");

	notify_commit_waiters(journal);

	/*
	 * Is this block now full? Reaping, and adding entries, might have already sent it off for
	 * rewriting; else, queue it for rewrite.
	 */
	if (is_block_dirty(block) && is_block_full(block))
		schedule_block_write(journal, block);

	recycle_journal_blocks(journal);
	write_blocks(journal);

	check_for_drain_complete(journal);
}

static void handle_write_error(struct vdo_completion *completion)
{
	struct recovery_journal_block *block = completion->parent;
	struct recovery_journal *journal = block->journal;

	vio_record_metadata_io_error(as_vio(completion));
	vdo_log_error_strerror(completion->result,
			       "cannot write recovery journal block %llu",
			       (unsigned long long) block->sequence_number);
	enter_journal_read_only_mode(journal, completion->result);
	complete_write(completion);
}

static void complete_write_endio(struct bio *bio)
{
	struct vio *vio = bio->bi_private;
	struct recovery_journal_block *block = vio->completion.parent;
	struct recovery_journal *journal = block->journal;

	continue_vio_after_io(vio, complete_write, journal->thread_id);
}

/**
 * add_queued_recovery_entries() - Actually add entries from the queue to the given block.
 * @block: The journal block.
 */
static void add_queued_recovery_entries(struct recovery_journal_block *block)
{
	while (vdo_waitq_has_waiters(&block->entry_waiters)) {
		struct data_vio *data_vio =
			vdo_waiter_as_data_vio(vdo_waitq_dequeue_waiter(&block->entry_waiters));
		struct tree_lock *lock = &data_vio->tree_lock;
		struct packed_recovery_journal_entry *packed_entry;
		struct recovery_journal_entry new_entry;

		if (block->sector->entry_count == RECOVERY_JOURNAL_ENTRIES_PER_SECTOR)
			set_active_sector(block,
					  (char *) block->sector + VDO_SECTOR_SIZE);

		/* Compose and encode the entry. */
		packed_entry = &block->sector->entries[block->sector->entry_count++];
		new_entry = (struct recovery_journal_entry) {
			.mapping = {
				.pbn = data_vio->increment_updater.zpbn.pbn,
				.state = data_vio->increment_updater.zpbn.state,
			},
			.unmapping = {
				.pbn = data_vio->decrement_updater.zpbn.pbn,
				.state = data_vio->decrement_updater.zpbn.state,
			},
			.operation = data_vio->increment_updater.operation,
			.slot = lock->tree_slots[lock->height].block_map_slot,
		};
		*packed_entry = vdo_pack_recovery_journal_entry(&new_entry);
		data_vio->recovery_sequence_number = block->sequence_number;

		/* Enqueue the data_vio to wait for its entry to commit. */
		vdo_waitq_enqueue_waiter(&block->commit_waiters, &data_vio->waiter);
	}
}

/**
 * write_block() - Issue a block for writing.
 *
 * Implements waiter_callback_fn.
 */
static void write_block(struct vdo_waiter *waiter, void *context __always_unused)
{
	struct recovery_journal_block *block =
		container_of(waiter, struct recovery_journal_block, write_waiter);
	struct recovery_journal *journal = block->journal;
	struct packed_journal_header *header = get_block_header(block);

	if (block->committing || !vdo_waitq_has_waiters(&block->entry_waiters) ||
	    is_read_only(journal))
		return;

	block->entries_in_commit = vdo_waitq_num_waiters(&block->entry_waiters);
	add_queued_recovery_entries(block);

	journal->pending_write_count += 1;
	journal->events.blocks.written += 1;
	journal->events.entries.written += block->entries_in_commit;

	header->block_map_head = __cpu_to_le64(journal->block_map_head);
	header->slab_journal_head = __cpu_to_le64(journal->slab_journal_head);
	header->entry_count = __cpu_to_le16(block->entry_count);

	block->committing = true;

	/*
	 * We must issue a flush and a FUA for every commit. The flush is necessary to ensure that
	 * the data being referenced is stable. The FUA is necessary to ensure that the journal
	 * block itself is stable before allowing overwrites of the lbn's previous data.
	 */
	vdo_submit_metadata_vio(&block->vio, journal->origin + block->block_number,
				complete_write_endio, handle_write_error,
				REQ_OP_WRITE | REQ_PRIO | REQ_PREFLUSH | REQ_SYNC | REQ_FUA);
}


/**
 * write_blocks() - Attempt to commit blocks, according to write policy.
 * @journal: The recovery journal.
 */
static void write_blocks(struct recovery_journal *journal)
{
	assert_on_journal_thread(journal, __func__);
	/*
	 * We call this function after adding entries to the journal and after finishing a block
	 * write. Thus, when this function terminates we must either have no VIOs waiting in the
	 * journal or have some outstanding IO to provide a future wakeup.
	 *
	 * We want to only issue full blocks if there are no pending writes. However, if there are
	 * no outstanding writes and some unwritten entries, we must issue a block, even if it's
	 * the active block and it isn't full.
	 */
	if (journal->pending_write_count > 0)
		return;

	/* Write all the full blocks. */
	vdo_waitq_notify_all_waiters(&journal->pending_writes, write_block, NULL);

	/*
	 * Do we need to write the active block? Only if we have no outstanding writes, even after
	 * issuing all of the full writes.
	 */
	if ((journal->pending_write_count == 0) && (journal->active_block != NULL))
		write_block(&journal->active_block->write_waiter, NULL);
}

/**
 * vdo_add_recovery_journal_entry() - Add an entry to a recovery journal.
 * @journal: The journal in which to make an entry.
 * @data_vio: The data_vio for which to add the entry. The entry will be taken
 *	      from the logical and new_mapped fields of the data_vio. The
 *	      data_vio's recovery_sequence_number field will be set to the
 *	      sequence number of the journal block in which the entry was
 *	      made.
 *
 * This method is asynchronous. The data_vio will not be called back until the entry is committed
 * to the on-disk journal.
 */
void vdo_add_recovery_journal_entry(struct recovery_journal *journal,
				    struct data_vio *data_vio)
{
	assert_on_journal_thread(journal, __func__);
	if (!vdo_is_state_normal(&journal->state)) {
		continue_data_vio_with_error(data_vio, VDO_INVALID_ADMIN_STATE);
		return;
	}

	if (is_read_only(journal)) {
		continue_data_vio_with_error(data_vio, VDO_READ_ONLY);
		return;
	}

	VDO_ASSERT_LOG_ONLY(data_vio->recovery_sequence_number == 0,
			    "journal lock not held for new entry");

	vdo_advance_journal_point(&journal->append_point, journal->entries_per_block);
	vdo_waitq_enqueue_waiter(&journal->entry_waiters, &data_vio->waiter);
	assign_entries(journal);
}

/**
 * is_lock_locked() - Check whether a lock is locked for a zone type.
 * @journal: The recovery journal.
 * @lock_number: The lock to check.
 * @zone_type: The type of the zone.
 *
 * If the recovery journal has a lock on the lock number, both logical and physical zones are
 * considered locked.
 *
 * Return: true if the specified lock has references (is locked).
 */
static bool is_lock_locked(struct recovery_journal *journal, block_count_t lock_number,
			   enum vdo_zone_type zone_type)
{
	atomic_t *zone_count;
	bool locked;

	if (is_journal_zone_locked(journal, lock_number))
		return true;

	zone_count = get_zone_count_ptr(journal, lock_number, zone_type);
	locked = (atomic_read(zone_count) != 0);
	/* Pairs with implicit barrier in vdo_release_recovery_journal_block_reference() */
	smp_rmb();
	return locked;
}

/**
 * reap_recovery_journal() - Conduct a sweep on a recovery journal to reclaim unreferenced blocks.
 * @journal: The recovery journal.
 */
static void reap_recovery_journal(struct recovery_journal *journal)
{
	if (journal->reaping) {
		/*
		 * We already have an outstanding reap in progress. We need to wait for it to
		 * finish.
		 */
		return;
	}

	if (vdo_is_state_quiescent(&journal->state)) {
		/* We are supposed to not do IO. Don't botch it by reaping. */
		return;
	}

	/*
	 * Start reclaiming blocks only when the journal head has no references. Then stop when a
	 * block is referenced.
	 */
	while ((journal->block_map_reap_head < journal->last_write_acknowledged) &&
		!is_lock_locked(journal, journal->block_map_head_block_number,
				VDO_ZONE_TYPE_LOGICAL)) {
		journal->block_map_reap_head++;
		if (++journal->block_map_head_block_number == journal->size)
			journal->block_map_head_block_number = 0;
	}

	while ((journal->slab_journal_reap_head < journal->last_write_acknowledged) &&
		!is_lock_locked(journal, journal->slab_journal_head_block_number,
				VDO_ZONE_TYPE_PHYSICAL)) {
		journal->slab_journal_reap_head++;
		if (++journal->slab_journal_head_block_number == journal->size)
			journal->slab_journal_head_block_number = 0;
	}

	if ((journal->block_map_reap_head == journal->block_map_head) &&
	    (journal->slab_journal_reap_head == journal->slab_journal_head)) {
		/* Nothing happened. */
		return;
	}

	/*
	 * If the block map head will advance, we must flush any block map page modified by the
	 * entries we are reaping. If the slab journal head will advance, we must flush the slab
	 * summary update covering the slab journal that just released some lock.
	 */
	journal->reaping = true;
	vdo_submit_flush_vio(journal->flush_vio, flush_endio, handle_flush_error);
}

/**
 * vdo_acquire_recovery_journal_block_reference() - Acquire a reference to a recovery journal block
 *                                                  from somewhere other than the journal itself.
 * @journal: The recovery journal.
 * @sequence_number: The journal sequence number of the referenced block.
 * @zone_type: The type of the zone making the adjustment.
 * @zone_id: The ID of the zone making the adjustment.
 */
void vdo_acquire_recovery_journal_block_reference(struct recovery_journal *journal,
						  sequence_number_t sequence_number,
						  enum vdo_zone_type zone_type,
						  zone_count_t zone_id)
{
	block_count_t lock_number;
	u16 *current_value;

	if (sequence_number == 0)
		return;

	VDO_ASSERT_LOG_ONLY((zone_type != VDO_ZONE_TYPE_JOURNAL),
			    "invalid lock count increment from journal zone");

	lock_number = vdo_get_recovery_journal_block_number(journal, sequence_number);
	current_value = get_counter(journal, lock_number, zone_type, zone_id);
	VDO_ASSERT_LOG_ONLY(*current_value < U16_MAX,
			    "increment of lock counter must not overflow");

	if (*current_value == 0) {
		/*
		 * This zone is acquiring this lock for the first time. Extra barriers because this
		 * was original developed using an atomic add operation that implicitly had them.
		 */
		smp_mb__before_atomic();
		atomic_inc(get_zone_count_ptr(journal, lock_number, zone_type));
		/* same as before_atomic */
		smp_mb__after_atomic();
	}

	*current_value += 1;
}

/**
 * vdo_release_journal_entry_lock() - Release a single per-entry reference count for a recovery
 *                                    journal block.
 * @journal: The recovery journal.
 * @sequence_number: The journal sequence number of the referenced block.
 */
void vdo_release_journal_entry_lock(struct recovery_journal *journal,
				    sequence_number_t sequence_number)
{
	block_count_t lock_number;

	if (sequence_number == 0)
		return;

	lock_number = vdo_get_recovery_journal_block_number(journal, sequence_number);
	/*
	 * Extra barriers because this was originally developed using an atomic add operation that
	 * implicitly had them.
	 */
	smp_mb__before_atomic();
	atomic_inc(get_decrement_counter(journal, lock_number));
	/* same as before_atomic */
	smp_mb__after_atomic();
}

/**
 * initiate_drain() - Initiate a drain.
 *
 * Implements vdo_admin_initiator_fn.
 */
static void initiate_drain(struct admin_state *state)
{
	check_for_drain_complete(container_of(state, struct recovery_journal, state));
}

/**
 * vdo_drain_recovery_journal() - Drain recovery journal I/O.
 * @journal: The journal to drain.
 * @operation: The drain operation (suspend or save).
 * @parent: The completion to notify once the journal is drained.
 *
 * All uncommitted entries will be written out.
 */
void vdo_drain_recovery_journal(struct recovery_journal *journal,
				const struct admin_state_code *operation,
				struct vdo_completion *parent)
{
	assert_on_journal_thread(journal, __func__);
	vdo_start_draining(&journal->state, operation, parent, initiate_drain);
}

/**
 * resume_lock_counter() - Re-allow notifications from a suspended lock counter.
 * @counter: The counter.
 *
 * Return: true if the lock counter was suspended.
 */
static bool resume_lock_counter(struct lock_counter *counter)
{
	int prior_state;

	/*
	 * Extra barriers because this was original developed using a CAS operation that implicitly
	 * had them.
	 */
	smp_mb__before_atomic();
	prior_state = atomic_cmpxchg(&counter->state, LOCK_COUNTER_STATE_SUSPENDED,
				     LOCK_COUNTER_STATE_NOT_NOTIFYING);
	/* same as before_atomic */
	smp_mb__after_atomic();

	return (prior_state == LOCK_COUNTER_STATE_SUSPENDED);
}

/**
 * vdo_resume_recovery_journal() - Resume a recovery journal which has been drained.
 * @journal: The journal to resume.
 * @parent: The completion to finish once the journal is resumed.
 */
void vdo_resume_recovery_journal(struct recovery_journal *journal,
				 struct vdo_completion *parent)
{
	bool saved;

	assert_on_journal_thread(journal, __func__);
	saved = vdo_is_state_saved(&journal->state);
	vdo_set_completion_result(parent, vdo_resume_if_quiescent(&journal->state));
	if (is_read_only(journal)) {
		vdo_continue_completion(parent, VDO_READ_ONLY);
		return;
	}

	if (saved)
		initialize_journal_state(journal);

	if (resume_lock_counter(&journal->lock_counter)) {
		/* We might have missed a notification. */
		reap_recovery_journal(journal);
	}

	vdo_launch_completion(parent);
}

/**
 * vdo_get_recovery_journal_logical_blocks_used() - Get the number of logical blocks in use by the
 *                                                  VDO.
 * @journal: The journal.
 *
 * Return: The number of logical blocks in use by the VDO.
 */
block_count_t vdo_get_recovery_journal_logical_blocks_used(const struct recovery_journal *journal)
{
	return journal->logical_blocks_used;
}

/**
 * vdo_get_recovery_journal_statistics() - Get the current statistics from the recovery journal.
 * @journal: The recovery journal to query.
 *
 * Return: A copy of the current statistics for the journal.
 */
struct recovery_journal_statistics
vdo_get_recovery_journal_statistics(const struct recovery_journal *journal)
{
	return journal->events;
}

/**
 * dump_recovery_block() - Dump the contents of the recovery block to the log.
 * @block: The block to dump.
 */
static void dump_recovery_block(const struct recovery_journal_block *block)
{
	vdo_log_info("    sequence number %llu; entries %u; %s; %zu entry waiters; %zu commit waiters",
		     (unsigned long long) block->sequence_number, block->entry_count,
		     (block->committing ? "committing" : "waiting"),
		     vdo_waitq_num_waiters(&block->entry_waiters),
		     vdo_waitq_num_waiters(&block->commit_waiters));
}

/**
 * vdo_dump_recovery_journal_statistics() - Dump some current statistics and other debug info from
 *                                          the recovery journal.
 * @journal: The recovery journal to dump.
 */
void vdo_dump_recovery_journal_statistics(const struct recovery_journal *journal)
{
	const struct recovery_journal_block *block;
	struct recovery_journal_statistics stats = vdo_get_recovery_journal_statistics(journal);

	vdo_log_info("Recovery Journal");
	vdo_log_info("	block_map_head=%llu slab_journal_head=%llu last_write_acknowledged=%llu tail=%llu block_map_reap_head=%llu slab_journal_reap_head=%llu disk_full=%llu slab_journal_commits_requested=%llu entry_waiters=%zu",
		     (unsigned long long) journal->block_map_head,
		     (unsigned long long) journal->slab_journal_head,
		     (unsigned long long) journal->last_write_acknowledged,
		     (unsigned long long) journal->tail,
		     (unsigned long long) journal->block_map_reap_head,
		     (unsigned long long) journal->slab_journal_reap_head,
		     (unsigned long long) stats.disk_full,
		     (unsigned long long) stats.slab_journal_commits_requested,
		     vdo_waitq_num_waiters(&journal->entry_waiters));
	vdo_log_info("	entries: started=%llu written=%llu committed=%llu",
		     (unsigned long long) stats.entries.started,
		     (unsigned long long) stats.entries.written,
		     (unsigned long long) stats.entries.committed);
	vdo_log_info("	blocks: started=%llu written=%llu committed=%llu",
		     (unsigned long long) stats.blocks.started,
		     (unsigned long long) stats.blocks.written,
		     (unsigned long long) stats.blocks.committed);

	vdo_log_info("	active blocks:");
	list_for_each_entry(block, &journal->active_tail_blocks, list_node)
		dump_recovery_block(block);
}