block/rnbd/rnbd-clt.c

// SPDX-License-Identifier: GPL-2.0-or-later
/*
 * RDMA Network Block Driver
 *
 * Copyright (c) 2014 - 2018 ProfitBricks GmbH. All rights reserved.
 * Copyright (c) 2018 - 2019 1&1 IONOS Cloud GmbH. All rights reserved.
 * Copyright (c) 2019 - 2020 1&1 IONOS SE. All rights reserved.
 */

#undef pr_fmt
#define pr_fmt(fmt) KBUILD_MODNAME " L" __stringify(__LINE__) ": " fmt

#include <linux/module.h>
#include <linux/blkdev.h>
#include <linux/hdreg.h>
#include <linux/scatterlist.h>
#include <linux/idr.h>

#include "rnbd-clt.h"

MODULE_DESCRIPTION("RDMA Network Block Device Client");
MODULE_LICENSE("GPL");

static int rnbd_client_major;
static DEFINE_IDA(index_ida);
static DEFINE_MUTEX(ida_lock);
static DEFINE_MUTEX(sess_lock);
static LIST_HEAD(sess_list);

/*
 * Maximum number of partitions an instance can have.
 * 6 bits = 64 minors = 63 partitions (one minor is used for the device itself)
 */
#define RNBD_PART_BITS		6

static inline bool rnbd_clt_get_sess(struct rnbd_clt_session *sess)
{
	return refcount_inc_not_zero(&sess->refcount);
}

static void free_sess(struct rnbd_clt_session *sess);

static void rnbd_clt_put_sess(struct rnbd_clt_session *sess)
{
	might_sleep();

	if (refcount_dec_and_test(&sess->refcount))
		free_sess(sess);
}

static void rnbd_clt_put_dev(struct rnbd_clt_dev *dev)
{
	might_sleep();

	if (!refcount_dec_and_test(&dev->refcount))
		return;

	mutex_lock(&ida_lock);
	ida_simple_remove(&index_ida, dev->clt_device_id);
	mutex_unlock(&ida_lock);
	kfree(dev->hw_queues);
	kfree(dev->pathname);
	rnbd_clt_put_sess(dev->sess);
	mutex_destroy(&dev->lock);
	kfree(dev);
}

static inline bool rnbd_clt_get_dev(struct rnbd_clt_dev *dev)
{
	return refcount_inc_not_zero(&dev->refcount);
}

static int rnbd_clt_set_dev_attr(struct rnbd_clt_dev *dev,
				 const struct rnbd_msg_open_rsp *rsp)
{
	struct rnbd_clt_session *sess = dev->sess;

	if (!rsp->logical_block_size)
		return -EINVAL;

	dev->device_id		    = le32_to_cpu(rsp->device_id);
	dev->nsectors		    = le64_to_cpu(rsp->nsectors);
	dev->logical_block_size	    = le16_to_cpu(rsp->logical_block_size);
	dev->physical_block_size    = le16_to_cpu(rsp->physical_block_size);
	dev->max_write_same_sectors = le32_to_cpu(rsp->max_write_same_sectors);
	dev->max_discard_sectors    = le32_to_cpu(rsp->max_discard_sectors);
	dev->discard_granularity    = le32_to_cpu(rsp->discard_granularity);
	dev->discard_alignment	    = le32_to_cpu(rsp->discard_alignment);
	dev->secure_discard	    = le16_to_cpu(rsp->secure_discard);
	dev->rotational		    = rsp->rotational;
	dev->wc			    = !!(rsp->cache_policy & RNBD_WRITEBACK);
	dev->fua		    = !!(rsp->cache_policy & RNBD_FUA);

	dev->max_hw_sectors = sess->max_io_size / SECTOR_SIZE;
	dev->max_segments = BMAX_SEGMENTS;

	return 0;
}

static int rnbd_clt_change_capacity(struct rnbd_clt_dev *dev,
				    size_t new_nsectors)
{
	rnbd_clt_info(dev, "Device size changed from %zu to %zu sectors\n",
		       dev->nsectors, new_nsectors);
	dev->nsectors = new_nsectors;
	set_capacity_and_notify(dev->gd, dev->nsectors);
	return 0;
}

static int process_msg_open_rsp(struct rnbd_clt_dev *dev,
				struct rnbd_msg_open_rsp *rsp)
{
	struct kobject *gd_kobj;
	int err = 0;

	mutex_lock(&dev->lock);
	if (dev->dev_state == DEV_STATE_UNMAPPED) {
		rnbd_clt_info(dev,
			       "Ignoring Open-Response message from server for  unmapped device\n");
		err = -ENOENT;
		goto out;
	}
	if (dev->dev_state == DEV_STATE_MAPPED_DISCONNECTED) {
		u64 nsectors = le64_to_cpu(rsp->nsectors);

		/*
		 * If the device was remapped and the size changed in the
		 * meantime we need to revalidate it
		 */
		if (dev->nsectors != nsectors)
			rnbd_clt_change_capacity(dev, nsectors);
		gd_kobj = &disk_to_dev(dev->gd)->kobj;
		kobject_uevent(gd_kobj, KOBJ_ONLINE);
		rnbd_clt_info(dev, "Device online, device remapped successfully\n");
	}
	err = rnbd_clt_set_dev_attr(dev, rsp);
	if (err)
		goto out;
	dev->dev_state = DEV_STATE_MAPPED;

out:
	mutex_unlock(&dev->lock);

	return err;
}

int rnbd_clt_resize_disk(struct rnbd_clt_dev *dev, size_t newsize)
{
	int ret = 0;

	mutex_lock(&dev->lock);
	if (dev->dev_state != DEV_STATE_MAPPED) {
		pr_err("Failed to set new size of the device, device is not opened\n");
		ret = -ENOENT;
		goto out;
	}
	ret = rnbd_clt_change_capacity(dev, newsize);

out:
	mutex_unlock(&dev->lock);

	return ret;
}

static inline void rnbd_clt_dev_requeue(struct rnbd_queue *q)
{
	if (WARN_ON(!q->hctx))
		return;

	/* We can come here from interrupt, thus async=true */
	blk_mq_run_hw_queue(q->hctx, true);
}

enum {
	RNBD_DELAY_IFBUSY = -1,
};

/**
 * rnbd_get_cpu_qlist() - finds a list with HW queues to be rerun
 * @sess:	Session to find a queue for
 * @cpu:	Cpu to start the search from
 *
 * Description:
 *     Each CPU has a list of HW queues, which needs to be rerun.  If a list
 *     is not empty - it is marked with a bit.  This function finds first
 *     set bit in a bitmap and returns corresponding CPU list.
 */
static struct rnbd_cpu_qlist *
rnbd_get_cpu_qlist(struct rnbd_clt_session *sess, int cpu)
{
	int bit;

	/* Search from cpu to nr_cpu_ids */
	bit = find_next_bit(sess->cpu_queues_bm, nr_cpu_ids, cpu);
	if (bit < nr_cpu_ids) {
		return per_cpu_ptr(sess->cpu_queues, bit);
	} else if (cpu != 0) {
		/* Search from 0 to cpu */
		bit = find_next_bit(sess->cpu_queues_bm, cpu, 0);
		if (bit < cpu)
			return per_cpu_ptr(sess->cpu_queues, bit);
	}

	return NULL;
}

static inline int nxt_cpu(int cpu)
{
	return (cpu + 1) % nr_cpu_ids;
}

/**
 * rnbd_rerun_if_needed() - rerun next queue marked as stopped
 * @sess:	Session to rerun a queue on
 *
 * Description:
 *     Each CPU has it's own list of HW queues, which should be rerun.
 *     Function finds such list with HW queues, takes a list lock, picks up
 *     the first HW queue out of the list and requeues it.
 *
 * Return:
 *     True if the queue was requeued, false otherwise.
 *
 * Context:
 *     Does not matter.
 */
static bool rnbd_rerun_if_needed(struct rnbd_clt_session *sess)
{
	struct rnbd_queue *q = NULL;
	struct rnbd_cpu_qlist *cpu_q;
	unsigned long flags;
	int *cpup;

	/*
	 * To keep fairness and not to let other queues starve we always
	 * try to wake up someone else in round-robin manner.  That of course
	 * increases latency but queues always have a chance to be executed.
	 */
	cpup = get_cpu_ptr(sess->cpu_rr);
	for (cpu_q = rnbd_get_cpu_qlist(sess, nxt_cpu(*cpup)); cpu_q;
	     cpu_q = rnbd_get_cpu_qlist(sess, nxt_cpu(cpu_q->cpu))) {
		if (!spin_trylock_irqsave(&cpu_q->requeue_lock, flags))
			continue;
		if (!test_bit(cpu_q->cpu, sess->cpu_queues_bm))
			goto unlock;
		q = list_first_entry_or_null(&cpu_q->requeue_list,
					     typeof(*q), requeue_list);
		if (WARN_ON(!q))
			goto clear_bit;
		list_del_init(&q->requeue_list);
		clear_bit_unlock(0, &q->in_list);

		if (list_empty(&cpu_q->requeue_list)) {
			/* Clear bit if nothing is left */
clear_bit:
			clear_bit(cpu_q->cpu, sess->cpu_queues_bm);
		}
unlock:
		spin_unlock_irqrestore(&cpu_q->requeue_lock, flags);

		if (q)
			break;
	}

	/**
	 * Saves the CPU that is going to be requeued on the per-cpu var. Just
	 * incrementing it doesn't work because rnbd_get_cpu_qlist() will
	 * always return the first CPU with something on the queue list when the
	 * value stored on the var is greater than the last CPU with something
	 * on the list.
	 */
	if (cpu_q)
		*cpup = cpu_q->cpu;
	put_cpu_var(sess->cpu_rr);

	if (q)
		rnbd_clt_dev_requeue(q);

	return q;
}

/**
 * rnbd_rerun_all_if_idle() - rerun all queues left in the list if
 *				 session is idling (there are no requests
 *				 in-flight).
 * @sess:	Session to rerun the queues on
 *
 * Description:
 *     This function tries to rerun all stopped queues if there are no
 *     requests in-flight anymore.  This function tries to solve an obvious
 *     problem, when number of tags < than number of queues (hctx), which
 *     are stopped and put to sleep.  If last permit, which has been just put,
 *     does not wake up all left queues (hctxs), IO requests hang forever.
 *
 *     That can happen when all number of permits, say N, have been exhausted
 *     from one CPU, and we have many block devices per session, say M.
 *     Each block device has it's own queue (hctx) for each CPU, so eventually
 *     we can put that number of queues (hctxs) to sleep: M x nr_cpu_ids.
 *     If number of permits N < M x nr_cpu_ids finally we will get an IO hang.
 *
 *     To avoid this hang last caller of rnbd_put_permit() (last caller is the
 *     one who observes sess->busy == 0) must wake up all remaining queues.
 *
 * Context:
 *     Does not matter.
 */
static void rnbd_rerun_all_if_idle(struct rnbd_clt_session *sess)
{
	bool requeued;

	do {
		requeued = rnbd_rerun_if_needed(sess);
	} while (atomic_read(&sess->busy) == 0 && requeued);
}

static struct rtrs_permit *rnbd_get_permit(struct rnbd_clt_session *sess,
					     enum rtrs_clt_con_type con_type,
					     enum wait_type wait)
{
	struct rtrs_permit *permit;

	permit = rtrs_clt_get_permit(sess->rtrs, con_type, wait);
	if (permit)
		/* We have a subtle rare case here, when all permits can be
		 * consumed before busy counter increased.  This is safe,
		 * because loser will get NULL as a permit, observe 0 busy
		 * counter and immediately restart the queue himself.
		 */
		atomic_inc(&sess->busy);

	return permit;
}

static void rnbd_put_permit(struct rnbd_clt_session *sess,
			     struct rtrs_permit *permit)
{
	rtrs_clt_put_permit(sess->rtrs, permit);
	atomic_dec(&sess->busy);
	/* Paired with rnbd_clt_dev_add_to_requeue().  Decrement first
	 * and then check queue bits.
	 */
	smp_mb__after_atomic();
	rnbd_rerun_all_if_idle(sess);
}

static struct rnbd_iu *rnbd_get_iu(struct rnbd_clt_session *sess,
				     enum rtrs_clt_con_type con_type,
				     enum wait_type wait)
{
	struct rnbd_iu *iu;
	struct rtrs_permit *permit;

	iu = kzalloc(sizeof(*iu), GFP_KERNEL);
	if (!iu)
		return NULL;

	permit = rnbd_get_permit(sess, con_type, wait);
	if (!permit) {
		kfree(iu);
		return NULL;
	}

	iu->permit = permit;
	/*
	 * 1st reference is dropped after finishing sending a "user" message,
	 * 2nd reference is dropped after confirmation with the response is
	 * returned.
	 * 1st and 2nd can happen in any order, so the rnbd_iu should be
	 * released (rtrs_permit returned to rtrs) only after both
	 * are finished.
	 */
	atomic_set(&iu->refcount, 2);
	init_waitqueue_head(&iu->comp.wait);
	iu->comp.errno = INT_MAX;

	if (sg_alloc_table(&iu->sgt, 1, GFP_KERNEL)) {
		rnbd_put_permit(sess, permit);
		kfree(iu);
		return NULL;
	}

	return iu;
}

static void rnbd_put_iu(struct rnbd_clt_session *sess, struct rnbd_iu *iu)
{
	if (atomic_dec_and_test(&iu->refcount)) {
		sg_free_table(&iu->sgt);
		rnbd_put_permit(sess, iu->permit);
		kfree(iu);
	}
}

static void rnbd_softirq_done_fn(struct request *rq)
{
	struct rnbd_clt_dev *dev	= rq->rq_disk->private_data;
	struct rnbd_clt_session *sess	= dev->sess;
	struct rnbd_iu *iu;

	iu = blk_mq_rq_to_pdu(rq);
	sg_free_table_chained(&iu->sgt, RNBD_INLINE_SG_CNT);
	rnbd_put_permit(sess, iu->permit);
	blk_mq_end_request(rq, errno_to_blk_status(iu->errno));
}

static void msg_io_conf(void *priv, int errno)
{
	struct rnbd_iu *iu = priv;
	struct rnbd_clt_dev *dev = iu->dev;
	struct request *rq = iu->rq;
	int rw = rq_data_dir(rq);

	iu->errno = errno;

	blk_mq_complete_request(rq);

	if (errno)
		rnbd_clt_info_rl(dev, "%s I/O failed with err: %d\n",
				 rw == READ ? "read" : "write", errno);
}

static void wake_up_iu_comp(struct rnbd_iu *iu, int errno)
{
	iu->comp.errno = errno;
	wake_up(&iu->comp.wait);
}

static void msg_conf(void *priv, int errno)
{
	struct rnbd_iu *iu = priv;

	iu->errno = errno;
	schedule_work(&iu->work);
}

static int send_usr_msg(struct rtrs_clt *rtrs, int dir,
			struct rnbd_iu *iu, struct kvec *vec,
			size_t len, struct scatterlist *sg, unsigned int sg_len,
			void (*conf)(struct work_struct *work),
			int *errno, int wait)
{
	int err;
	struct rtrs_clt_req_ops req_ops;

	INIT_WORK(&iu->work, conf);
	req_ops = (struct rtrs_clt_req_ops) {
		.priv = iu,
		.conf_fn = msg_conf,
	};
	err = rtrs_clt_request(dir, &req_ops, rtrs, iu->permit,
				vec, 1, len, sg, sg_len);
	if (!err && wait) {
		wait_event(iu->comp.wait, iu->comp.errno != INT_MAX);
		*errno = iu->comp.errno;
	} else {
		*errno = 0;
	}

	return err;
}

static void msg_close_conf(struct work_struct *work)
{
	struct rnbd_iu *iu = container_of(work, struct rnbd_iu, work);
	struct rnbd_clt_dev *dev = iu->dev;

	wake_up_iu_comp(iu, iu->errno);
	rnbd_put_iu(dev->sess, iu);
	rnbd_clt_put_dev(dev);
}

static int send_msg_close(struct rnbd_clt_dev *dev, u32 device_id,
			  enum wait_type wait)
{
	struct rnbd_clt_session *sess = dev->sess;
	struct rnbd_msg_close msg;
	struct rnbd_iu *iu;
	struct kvec vec = {
		.iov_base = &msg,
		.iov_len  = sizeof(msg)
	};
	int err, errno;

	iu = rnbd_get_iu(sess, RTRS_ADMIN_CON, RTRS_PERMIT_WAIT);
	if (!iu)
		return -ENOMEM;

	iu->buf = NULL;
	iu->dev = dev;

	msg.hdr.type	= cpu_to_le16(RNBD_MSG_CLOSE);
	msg.device_id	= cpu_to_le32(device_id);

	WARN_ON(!rnbd_clt_get_dev(dev));
	err = send_usr_msg(sess->rtrs, WRITE, iu, &vec, 0, NULL, 0,
			   msg_close_conf, &errno, wait);
	if (err) {
		rnbd_clt_put_dev(dev);
		rnbd_put_iu(sess, iu);
	} else {
		err = errno;
	}

	rnbd_put_iu(sess, iu);
	return err;
}

static void msg_open_conf(struct work_struct *work)
{
	struct rnbd_iu *iu = container_of(work, struct rnbd_iu, work);
	struct rnbd_msg_open_rsp *rsp = iu->buf;
	struct rnbd_clt_dev *dev = iu->dev;
	int errno = iu->errno;

	if (errno) {
		rnbd_clt_err(dev,
			      "Opening failed, server responded: %d\n",
			      errno);
	} else {
		errno = process_msg_open_rsp(dev, rsp);
		if (errno) {
			u32 device_id = le32_to_cpu(rsp->device_id);
			/*
			 * If server thinks its fine, but we fail to process
			 * then be nice and send a close to server.
			 */
			send_msg_close(dev, device_id, RTRS_PERMIT_NOWAIT);
		}
	}
	kfree(rsp);
	wake_up_iu_comp(iu, errno);
	rnbd_put_iu(dev->sess, iu);
	rnbd_clt_put_dev(dev);
}

static void msg_sess_info_conf(struct work_struct *work)
{
	struct rnbd_iu *iu = container_of(work, struct rnbd_iu, work);
	struct rnbd_msg_sess_info_rsp *rsp = iu->buf;
	struct rnbd_clt_session *sess = iu->sess;

	if (!iu->errno)
		sess->ver = min_t(u8, rsp->ver, RNBD_PROTO_VER_MAJOR);

	kfree(rsp);
	wake_up_iu_comp(iu, iu->errno);
	rnbd_put_iu(sess, iu);
	rnbd_clt_put_sess(sess);
}

static int send_msg_open(struct rnbd_clt_dev *dev, enum wait_type wait)
{
	struct rnbd_clt_session *sess = dev->sess;
	struct rnbd_msg_open_rsp *rsp;
	struct rnbd_msg_open msg;
	struct rnbd_iu *iu;
	struct kvec vec = {
		.iov_base = &msg,
		.iov_len  = sizeof(msg)
	};
	int err, errno;

	rsp = kzalloc(sizeof(*rsp), GFP_KERNEL);
	if (!rsp)
		return -ENOMEM;

	iu = rnbd_get_iu(sess, RTRS_ADMIN_CON, RTRS_PERMIT_WAIT);
	if (!iu) {
		kfree(rsp);
		return -ENOMEM;
	}

	iu->buf = rsp;
	iu->dev = dev;

	sg_init_one(iu->sgt.sgl, rsp, sizeof(*rsp));

	msg.hdr.type	= cpu_to_le16(RNBD_MSG_OPEN);
	msg.access_mode	= dev->access_mode;
	strscpy(msg.dev_name, dev->pathname, sizeof(msg.dev_name));

	WARN_ON(!rnbd_clt_get_dev(dev));
	err = send_usr_msg(sess->rtrs, READ, iu,
			   &vec, sizeof(*rsp), iu->sgt.sgl, 1,
			   msg_open_conf, &errno, wait);
	if (err) {
		rnbd_clt_put_dev(dev);
		rnbd_put_iu(sess, iu);
		kfree(rsp);
	} else {
		err = errno;
	}

	rnbd_put_iu(sess, iu);
	return err;
}

static int send_msg_sess_info(struct rnbd_clt_session *sess, enum wait_type wait)
{
	struct rnbd_msg_sess_info_rsp *rsp;
	struct rnbd_msg_sess_info msg;
	struct rnbd_iu *iu;
	struct kvec vec = {
		.iov_base = &msg,
		.iov_len  = sizeof(msg)
	};
	int err, errno;

	rsp = kzalloc(sizeof(*rsp), GFP_KERNEL);
	if (!rsp)
		return -ENOMEM;

	iu = rnbd_get_iu(sess, RTRS_ADMIN_CON, RTRS_PERMIT_WAIT);
	if (!iu) {
		kfree(rsp);
		return -ENOMEM;
	}

	iu->buf = rsp;
	iu->sess = sess;
	sg_init_one(iu->sgt.sgl, rsp, sizeof(*rsp));

	msg.hdr.type = cpu_to_le16(RNBD_MSG_SESS_INFO);
	msg.ver      = RNBD_PROTO_VER_MAJOR;

	if (!rnbd_clt_get_sess(sess)) {
		/*
		 * That can happen only in one case, when RTRS has restablished
		 * the connection and link_ev() is called, but session is almost
		 * dead, last reference on session is put and caller is waiting
		 * for RTRS to close everything.
		 */
		err = -ENODEV;
		goto put_iu;
	}
	err = send_usr_msg(sess->rtrs, READ, iu,
			   &vec, sizeof(*rsp), iu->sgt.sgl, 1,
			   msg_sess_info_conf, &errno, wait);
	if (err) {
		rnbd_clt_put_sess(sess);
put_iu:
		rnbd_put_iu(sess, iu);
		kfree(rsp);
	} else {
		err = errno;
	}
	rnbd_put_iu(sess, iu);
	return err;
}

static void set_dev_states_to_disconnected(struct rnbd_clt_session *sess)
{
	struct rnbd_clt_dev *dev;
	struct kobject *gd_kobj;

	mutex_lock(&sess->lock);
	list_for_each_entry(dev, &sess->devs_list, list) {
		rnbd_clt_err(dev, "Device disconnected.\n");

		mutex_lock(&dev->lock);
		if (dev->dev_state == DEV_STATE_MAPPED) {
			dev->dev_state = DEV_STATE_MAPPED_DISCONNECTED;
			gd_kobj = &disk_to_dev(dev->gd)->kobj;
			kobject_uevent(gd_kobj, KOBJ_OFFLINE);
		}
		mutex_unlock(&dev->lock);
	}
	mutex_unlock(&sess->lock);
}

static void remap_devs(struct rnbd_clt_session *sess)
{
	struct rnbd_clt_dev *dev;
	struct rtrs_attrs attrs;
	int err;

	/*
	 * Careful here: we are called from RTRS link event directly,
	 * thus we can't send any RTRS request and wait for response
	 * or RTRS will not be able to complete request with failure
	 * if something goes wrong (failing of outstanding requests
	 * happens exactly from the context where we are blocking now).
	 *
	 * So to avoid deadlocks each usr message sent from here must
	 * be asynchronous.
	 */

	err = send_msg_sess_info(sess, RTRS_PERMIT_NOWAIT);
	if (err) {
		pr_err("send_msg_sess_info(\"%s\"): %d\n", sess->sessname, err);
		return;
	}

	err = rtrs_clt_query(sess->rtrs, &attrs);
	if (err) {
		pr_err("rtrs_clt_query(\"%s\"): %d\n", sess->sessname, err);
		return;
	}
	mutex_lock(&sess->lock);
	sess->max_io_size = attrs.max_io_size;

	list_for_each_entry(dev, &sess->devs_list, list) {
		bool skip;

		mutex_lock(&dev->lock);
		skip = (dev->dev_state == DEV_STATE_INIT);
		mutex_unlock(&dev->lock);
		if (skip)
			/*
			 * When device is establishing connection for the first
			 * time - do not remap, it will be closed soon.
			 */
			continue;

		rnbd_clt_info(dev, "session reconnected, remapping device\n");
		err = send_msg_open(dev, RTRS_PERMIT_NOWAIT);
		if (err) {
			rnbd_clt_err(dev, "send_msg_open(): %d\n", err);
			break;
		}
	}
	mutex_unlock(&sess->lock);
}

static void rnbd_clt_link_ev(void *priv, enum rtrs_clt_link_ev ev)
{
	struct rnbd_clt_session *sess = priv;

	switch (ev) {
	case RTRS_CLT_LINK_EV_DISCONNECTED:
		set_dev_states_to_disconnected(sess);
		break;
	case RTRS_CLT_LINK_EV_RECONNECTED:
		remap_devs(sess);
		break;
	default:
		pr_err("Unknown session event received (%d), session: %s\n",
		       ev, sess->sessname);
	}
}

static void rnbd_init_cpu_qlists(struct rnbd_cpu_qlist __percpu *cpu_queues)
{
	unsigned int cpu;
	struct rnbd_cpu_qlist *cpu_q;

	for_each_possible_cpu(cpu) {
		cpu_q = per_cpu_ptr(cpu_queues, cpu);

		cpu_q->cpu = cpu;
		INIT_LIST_HEAD(&cpu_q->requeue_list);
		spin_lock_init(&cpu_q->requeue_lock);
	}
}

static void destroy_mq_tags(struct rnbd_clt_session *sess)
{
	if (sess->tag_set.tags)
		blk_mq_free_tag_set(&sess->tag_set);
}

static inline void wake_up_rtrs_waiters(struct rnbd_clt_session *sess)
{
	sess->rtrs_ready = true;
	wake_up_all(&sess->rtrs_waitq);
}

static void close_rtrs(struct rnbd_clt_session *sess)
{
	might_sleep();

	if (!IS_ERR_OR_NULL(sess->rtrs)) {
		rtrs_clt_close(sess->rtrs);
		sess->rtrs = NULL;
		wake_up_rtrs_waiters(sess);
	}
}

static void free_sess(struct rnbd_clt_session *sess)
{
	WARN_ON(!list_empty(&sess->devs_list));

	might_sleep();

	close_rtrs(sess);
	destroy_mq_tags(sess);
	if (!list_empty(&sess->list)) {
		mutex_lock(&sess_lock);
		list_del(&sess->list);
		mutex_unlock(&sess_lock);
	}
	free_percpu(sess->cpu_queues);
	free_percpu(sess->cpu_rr);
	mutex_destroy(&sess->lock);
	kfree(sess);
}

static struct rnbd_clt_session *alloc_sess(const char *sessname)
{
	struct rnbd_clt_session *sess;
	int err, cpu;

	sess = kzalloc_node(sizeof(*sess), GFP_KERNEL, NUMA_NO_NODE);
	if (!sess)
		return ERR_PTR(-ENOMEM);
	strscpy(sess->sessname, sessname, sizeof(sess->sessname));
	atomic_set(&sess->busy, 0);
	mutex_init(&sess->lock);
	INIT_LIST_HEAD(&sess->devs_list);
	INIT_LIST_HEAD(&sess->list);
	bitmap_zero(sess->cpu_queues_bm, num_possible_cpus());
	init_waitqueue_head(&sess->rtrs_waitq);
	refcount_set(&sess->refcount, 1);

	sess->cpu_queues = alloc_percpu(struct rnbd_cpu_qlist);
	if (!sess->cpu_queues) {
		err = -ENOMEM;
		goto err;
	}
	rnbd_init_cpu_qlists(sess->cpu_queues);

	/*
	 * That is simple percpu variable which stores cpu indices, which are
	 * incremented on each access.  We need that for the sake of fairness
	 * to wake up queues in a round-robin manner.
	 */
	sess->cpu_rr = alloc_percpu(int);
	if (!sess->cpu_rr) {
		err = -ENOMEM;
		goto err;
	}
	for_each_possible_cpu(cpu)
		* per_cpu_ptr(sess->cpu_rr, cpu) = cpu;

	return sess;

err:
	free_sess(sess);

	return ERR_PTR(err);
}

static int wait_for_rtrs_connection(struct rnbd_clt_session *sess)
{
	wait_event(sess->rtrs_waitq, sess->rtrs_ready);
	if (IS_ERR_OR_NULL(sess->rtrs))
		return -ECONNRESET;

	return 0;
}

static void wait_for_rtrs_disconnection(struct rnbd_clt_session *sess)
	__releases(&sess_lock)
	__acquires(&sess_lock)
{
	DEFINE_WAIT(wait);

	prepare_to_wait(&sess->rtrs_waitq, &wait, TASK_UNINTERRUPTIBLE);
	if (IS_ERR_OR_NULL(sess->rtrs)) {
		finish_wait(&sess->rtrs_waitq, &wait);
		return;
	}
	mutex_unlock(&sess_lock);
	/* loop in caller, see __find_and_get_sess().
	 * You can't leave mutex locked and call schedule(), you will catch a
	 * deadlock with a caller of free_sess(), which has just put the last
	 * reference and is about to take the sess_lock in order to delete
	 * the session from the list.
	 */
	schedule();
	mutex_lock(&sess_lock);
}

static struct rnbd_clt_session *__find_and_get_sess(const char *sessname)
	__releases(&sess_lock)
	__acquires(&sess_lock)
{
	struct rnbd_clt_session *sess, *sn;
	int err;

again:
	list_for_each_entry_safe(sess, sn, &sess_list, list) {
		if (strcmp(sessname, sess->sessname))
			continue;

		if (sess->rtrs_ready && IS_ERR_OR_NULL(sess->rtrs))
			/*
			 * No RTRS connection, session is dying.
			 */
			continue;

		if (rnbd_clt_get_sess(sess)) {
			/*
			 * Alive session is found, wait for RTRS connection.
			 */
			mutex_unlock(&sess_lock);
			err = wait_for_rtrs_connection(sess);
			if (err)
				rnbd_clt_put_sess(sess);
			mutex_lock(&sess_lock);

			if (err)
				/* Session is dying, repeat the loop */
				goto again;

			return sess;
		}
		/*
		 * Ref is 0, session is dying, wait for RTRS disconnect
		 * in order to avoid session names clashes.
		 */
		wait_for_rtrs_disconnection(sess);
		/*
		 * RTRS is disconnected and soon session will be freed,
		 * so repeat a loop.
		 */
		goto again;
	}

	return NULL;
}

/* caller is responsible for initializing 'first' to false */
static struct
rnbd_clt_session *find_or_create_sess(const char *sessname, bool *first)
{
	struct rnbd_clt_session *sess = NULL;

	mutex_lock(&sess_lock);
	sess = __find_and_get_sess(sessname);
	if (!sess) {
		sess = alloc_sess(sessname);
		if (IS_ERR(sess)) {
			mutex_unlock(&sess_lock);
			return sess;
		}
		list_add(&sess->list, &sess_list);
		*first = true;
	}
	mutex_unlock(&sess_lock);

	return sess;
}

static int rnbd_client_open(struct block_device *block_device, fmode_t mode)
{
	struct rnbd_clt_dev *dev = block_device->bd_disk->private_data;

	if (dev->read_only && (mode & FMODE_WRITE))
		return -EPERM;

	if (dev->dev_state == DEV_STATE_UNMAPPED ||
	    !rnbd_clt_get_dev(dev))
		return -EIO;

	return 0;
}

static void rnbd_client_release(struct gendisk *gen, fmode_t mode)
{
	struct rnbd_clt_dev *dev = gen->private_data;

	rnbd_clt_put_dev(dev);
}

static int rnbd_client_getgeo(struct block_device *block_device,
			      struct hd_geometry *geo)
{
	u64 size;
	struct rnbd_clt_dev *dev;

	dev = block_device->bd_disk->private_data;
	size = dev->size * (dev->logical_block_size / SECTOR_SIZE);
	geo->cylinders	= size >> 6;	/* size/64 */
	geo->heads	= 4;
	geo->sectors	= 16;
	geo->start	= 0;

	return 0;
}

static const struct block_device_operations rnbd_client_ops = {
	.owner		= THIS_MODULE,
	.open		= rnbd_client_open,
	.release	= rnbd_client_release,
	.getgeo		= rnbd_client_getgeo
};

/* The amount of data that belongs to an I/O and the amount of data that
 * should be read or written to the disk (bi_size) can differ.
 *
 * E.g. When WRITE_SAME is used, only a small amount of data is
 * transferred that is then written repeatedly over a lot of sectors.
 *
 * Get the size of data to be transferred via RTRS by summing up the size
 * of the scather-gather list entries.
 */
static size_t rnbd_clt_get_sg_size(struct scatterlist *sglist, u32 len)
{
	struct scatterlist *sg;
	size_t tsize = 0;
	int i;

	for_each_sg(sglist, sg, len, i)
		tsize += sg->length;
	return tsize;
}

static int rnbd_client_xfer_request(struct rnbd_clt_dev *dev,
				     struct request *rq,
				     struct rnbd_iu *iu)
{
	struct rtrs_clt *rtrs = dev->sess->rtrs;
	struct rtrs_permit *permit = iu->permit;
	struct rnbd_msg_io msg;
	struct rtrs_clt_req_ops req_ops;
	unsigned int sg_cnt = 0;
	struct kvec vec;
	size_t size;
	int err;

	iu->rq		= rq;
	iu->dev		= dev;
	msg.sector	= cpu_to_le64(blk_rq_pos(rq));
	msg.bi_size	= cpu_to_le32(blk_rq_bytes(rq));
	msg.rw		= cpu_to_le32(rq_to_rnbd_flags(rq));
	msg.prio	= cpu_to_le16(req_get_ioprio(rq));

	/*
	 * We only support discards with single segment for now.
	 * See queue limits.
	 */
	if (req_op(rq) != REQ_OP_DISCARD)
		sg_cnt = blk_rq_map_sg(dev->queue, rq, iu->sgt.sgl);

	if (sg_cnt == 0)
		sg_mark_end(&iu->sgt.sgl[0]);

	msg.hdr.type	= cpu_to_le16(RNBD_MSG_IO);
	msg.device_id	= cpu_to_le32(dev->device_id);

	vec = (struct kvec) {
		.iov_base = &msg,
		.iov_len  = sizeof(msg)
	};
	size = rnbd_clt_get_sg_size(iu->sgt.sgl, sg_cnt);
	req_ops = (struct rtrs_clt_req_ops) {
		.priv = iu,
		.conf_fn = msg_io_conf,
	};
	err = rtrs_clt_request(rq_data_dir(rq), &req_ops, rtrs, permit,
			       &vec, 1, size, iu->sgt.sgl, sg_cnt);
	if (err) {
		rnbd_clt_err_rl(dev, "RTRS failed to transfer IO, err: %d\n",
				 err);
		return err;
	}

	return 0;
}

/**
 * rnbd_clt_dev_add_to_requeue() - add device to requeue if session is busy
 * @dev:	Device to be checked
 * @q:		Queue to be added to the requeue list if required
 *
 * Description:
 *     If session is busy, that means someone will requeue us when resources
 *     are freed.  If session is not doing anything - device is not added to
 *     the list and @false is returned.
 */
static bool rnbd_clt_dev_add_to_requeue(struct rnbd_clt_dev *dev,
						struct rnbd_queue *q)
{
	struct rnbd_clt_session *sess = dev->sess;
	struct rnbd_cpu_qlist *cpu_q;
	unsigned long flags;
	bool added = true;
	bool need_set;

	cpu_q = get_cpu_ptr(sess->cpu_queues);
	spin_lock_irqsave(&cpu_q->requeue_lock, flags);

	if (!test_and_set_bit_lock(0, &q->in_list)) {
		if (WARN_ON(!list_empty(&q->requeue_list)))
			goto unlock;

		need_set = !test_bit(cpu_q->cpu, sess->cpu_queues_bm);
		if (need_set) {
			set_bit(cpu_q->cpu, sess->cpu_queues_bm);
			/* Paired with rnbd_put_permit(). Set a bit first
			 * and then observe the busy counter.
			 */
			smp_mb__before_atomic();
		}
		if (atomic_read(&sess->busy)) {
			list_add_tail(&q->requeue_list, &cpu_q->requeue_list);
		} else {
			/* Very unlikely, but possible: busy counter was
			 * observed as zero.  Drop all bits and return
			 * false to restart the queue by ourselves.
			 */
			if (need_set)
				clear_bit(cpu_q->cpu, sess->cpu_queues_bm);
			clear_bit_unlock(0, &q->in_list);
			added = false;
		}
	}
unlock:
	spin_unlock_irqrestore(&cpu_q->requeue_lock, flags);
	put_cpu_ptr(sess->cpu_queues);

	return added;
}

static void rnbd_clt_dev_kick_mq_queue(struct rnbd_clt_dev *dev,
					struct blk_mq_hw_ctx *hctx,
					int delay)
{
	struct rnbd_queue *q = hctx->driver_data;

	if (delay != RNBD_DELAY_IFBUSY)
		blk_mq_delay_run_hw_queue(hctx, delay);
	else if (!rnbd_clt_dev_add_to_requeue(dev, q))
		/*
		 * If session is not busy we have to restart
		 * the queue ourselves.
		 */
		blk_mq_delay_run_hw_queue(hctx, 10/*ms*/);
}

static blk_status_t rnbd_queue_rq(struct blk_mq_hw_ctx *hctx,
				   const struct blk_mq_queue_data *bd)
{
	struct request *rq = bd->rq;
	struct rnbd_clt_dev *dev = rq->rq_disk->private_data;
	struct rnbd_iu *iu = blk_mq_rq_to_pdu(rq);
	int err;
	blk_status_t ret = BLK_STS_IOERR;

	if (dev->dev_state != DEV_STATE_MAPPED)
		return BLK_STS_IOERR;

	iu->permit = rnbd_get_permit(dev->sess, RTRS_IO_CON,
				      RTRS_PERMIT_NOWAIT);
	if (!iu->permit) {
		rnbd_clt_dev_kick_mq_queue(dev, hctx, RNBD_DELAY_IFBUSY);
		return BLK_STS_RESOURCE;
	}

	iu->sgt.sgl = iu->first_sgl;
	err = sg_alloc_table_chained(&iu->sgt,
				     /* Even-if the request has no segment,
				      * sglist must have one entry at least.
				      */
				     blk_rq_nr_phys_segments(rq) ? : 1,
				     iu->sgt.sgl,
				     RNBD_INLINE_SG_CNT);
	if (err) {
		rnbd_clt_err_rl(dev, "sg_alloc_table_chained ret=%d\n", err);
		rnbd_clt_dev_kick_mq_queue(dev, hctx, 10/*ms*/);
		rnbd_put_permit(dev->sess, iu->permit);
		return BLK_STS_RESOURCE;
	}

	blk_mq_start_request(rq);
	err = rnbd_client_xfer_request(dev, rq, iu);
	if (err == 0)
		return BLK_STS_OK;
	if (err == -EAGAIN || err == -ENOMEM) {
		rnbd_clt_dev_kick_mq_queue(dev, hctx, 10/*ms*/);
		ret = BLK_STS_RESOURCE;
	}
	sg_free_table_chained(&iu->sgt, RNBD_INLINE_SG_CNT);
	rnbd_put_permit(dev->sess, iu->permit);
	return ret;
}

static int rnbd_rdma_poll(struct blk_mq_hw_ctx *hctx)
{
	struct rnbd_queue *q = hctx->driver_data;
	struct rnbd_clt_dev *dev = q->dev;
	int cnt;

	cnt = rtrs_clt_rdma_cq_direct(dev->sess->rtrs, hctx->queue_num);
	return cnt;
}

static int rnbd_rdma_map_queues(struct blk_mq_tag_set *set)
{
	struct rnbd_clt_session *sess = set->driver_data;

	/* shared read/write queues */
	set->map[HCTX_TYPE_DEFAULT].nr_queues = num_online_cpus();
	set->map[HCTX_TYPE_DEFAULT].queue_offset = 0;
	set->map[HCTX_TYPE_READ].nr_queues = num_online_cpus();
	set->map[HCTX_TYPE_READ].queue_offset = 0;
	blk_mq_map_queues(&set->map[HCTX_TYPE_DEFAULT]);
	blk_mq_map_queues(&set->map[HCTX_TYPE_READ]);

	if (sess->nr_poll_queues) {
		/* dedicated queue for poll */
		set->map[HCTX_TYPE_POLL].nr_queues = sess->nr_poll_queues;
		set->map[HCTX_TYPE_POLL].queue_offset = set->map[HCTX_TYPE_READ].queue_offset +
			set->map[HCTX_TYPE_READ].nr_queues;
		blk_mq_map_queues(&set->map[HCTX_TYPE_POLL]);
		pr_info("[session=%s] mapped %d/%d/%d default/read/poll queues.\n",
			sess->sessname,
			set->map[HCTX_TYPE_DEFAULT].nr_queues,
			set->map[HCTX_TYPE_READ].nr_queues,
			set->map[HCTX_TYPE_POLL].nr_queues);
	} else {
		pr_info("[session=%s] mapped %d/%d default/read queues.\n",
			sess->sessname,
			set->map[HCTX_TYPE_DEFAULT].nr_queues,
			set->map[HCTX_TYPE_READ].nr_queues);
	}

	return 0;
}

static struct blk_mq_ops rnbd_mq_ops = {
	.queue_rq	= rnbd_queue_rq,
	.complete	= rnbd_softirq_done_fn,
	.map_queues     = rnbd_rdma_map_queues,
	.poll           = rnbd_rdma_poll,
};

static int setup_mq_tags(struct rnbd_clt_session *sess)
{
	struct blk_mq_tag_set *tag_set = &sess->tag_set;

	memset(tag_set, 0, sizeof(*tag_set));
	tag_set->ops		= &rnbd_mq_ops;
	tag_set->queue_depth	= sess->queue_depth;
	tag_set->numa_node		= NUMA_NO_NODE;
	tag_set->flags		= BLK_MQ_F_SHOULD_MERGE |
				  BLK_MQ_F_TAG_QUEUE_SHARED;
	tag_set->cmd_size	= sizeof(struct rnbd_iu) + RNBD_RDMA_SGL_SIZE;

	/* for HCTX_TYPE_DEFAULT, HCTX_TYPE_READ, HCTX_TYPE_POLL */
	tag_set->nr_maps        = sess->nr_poll_queues ? HCTX_MAX_TYPES : 2;
	/*
	 * HCTX_TYPE_DEFAULT and HCTX_TYPE_READ share one set of queues
	 * others are for HCTX_TYPE_POLL
	 */
	tag_set->nr_hw_queues	= num_online_cpus() + sess->nr_poll_queues;
	tag_set->driver_data    = sess;

	return blk_mq_alloc_tag_set(tag_set);
}

static struct rnbd_clt_session *
find_and_get_or_create_sess(const char *sessname,
			    const struct rtrs_addr *paths,
			    size_t path_cnt, u16 port_nr, u32 nr_poll_queues)
{
	struct rnbd_clt_session *sess;
	struct rtrs_attrs attrs;
	int err;
	bool first = false;
	struct rtrs_clt_ops rtrs_ops;

	sess = find_or_create_sess(sessname, &first);
	if (sess == ERR_PTR(-ENOMEM))
		return ERR_PTR(-ENOMEM);
	else if ((nr_poll_queues && !first) ||  (!nr_poll_queues && sess->nr_poll_queues)) {
		/*
		 * A device MUST have its own session to use the polling-mode.
		 * It must fail to map new device with the same session.
		 */
		err = -EINVAL;
		goto put_sess;
	}

	if (!first)
		return sess;

	if (!path_cnt) {
		pr_err("Session %s not found, and path parameter not given", sessname);
		err = -ENXIO;
		goto put_sess;
	}

	rtrs_ops = (struct rtrs_clt_ops) {
		.priv = sess,
		.link_ev = rnbd_clt_link_ev,
	};
	/*
	 * Nothing was found, establish rtrs connection and proceed further.
	 */
	sess->rtrs = rtrs_clt_open(&rtrs_ops, sessname,
				   paths, path_cnt, port_nr,
				   0, /* Do not use pdu of rtrs */
				   RECONNECT_DELAY, BMAX_SEGMENTS,
				   MAX_RECONNECTS, nr_poll_queues);
	if (IS_ERR(sess->rtrs)) {
		err = PTR_ERR(sess->rtrs);
		goto wake_up_and_put;
	}

	err = rtrs_clt_query(sess->rtrs, &attrs);
	if (err)
		goto close_rtrs;

	sess->max_io_size = attrs.max_io_size;
	sess->queue_depth = attrs.queue_depth;
	sess->nr_poll_queues = nr_poll_queues;

	err = setup_mq_tags(sess);
	if (err)
		goto close_rtrs;

	err = send_msg_sess_info(sess, RTRS_PERMIT_WAIT);
	if (err)
		goto close_rtrs;

	wake_up_rtrs_waiters(sess);

	return sess;

close_rtrs:
	close_rtrs(sess);
put_sess:
	rnbd_clt_put_sess(sess);

	return ERR_PTR(err);

wake_up_and_put:
	wake_up_rtrs_waiters(sess);
	goto put_sess;
}

static inline void rnbd_init_hw_queue(struct rnbd_clt_dev *dev,
				       struct rnbd_queue *q,
				       struct blk_mq_hw_ctx *hctx)
{
	INIT_LIST_HEAD(&q->requeue_list);
	q->dev  = dev;
	q->hctx = hctx;
}

static void rnbd_init_mq_hw_queues(struct rnbd_clt_dev *dev)
{
	int i;
	struct blk_mq_hw_ctx *hctx;
	struct rnbd_queue *q;

	queue_for_each_hw_ctx(dev->queue, hctx, i) {
		q = &dev->hw_queues[i];
		rnbd_init_hw_queue(dev, q, hctx);
		hctx->driver_data = q;
	}
}

static int setup_mq_dev(struct rnbd_clt_dev *dev)
{
	dev->queue = blk_mq_init_queue(&dev->sess->tag_set);
	if (IS_ERR(dev->queue)) {
		rnbd_clt_err(dev, "Initializing multiqueue queue failed, err: %ld\n",
			      PTR_ERR(dev->queue));
		return PTR_ERR(dev->queue);
	}
	rnbd_init_mq_hw_queues(dev);
	return 0;
}

static void setup_request_queue(struct rnbd_clt_dev *dev)
{
	blk_queue_logical_block_size(dev->queue, dev->logical_block_size);
	blk_queue_physical_block_size(dev->queue, dev->physical_block_size);
	blk_queue_max_hw_sectors(dev->queue, dev->max_hw_sectors);
	blk_queue_max_write_same_sectors(dev->queue,
					 dev->max_write_same_sectors);

	/*
	 * we don't support discards to "discontiguous" segments
	 * in on request
	 */
	blk_queue_max_discard_segments(dev->queue, 1);

	blk_queue_max_discard_sectors(dev->queue, dev->max_discard_sectors);
	dev->queue->limits.discard_granularity	= dev->discard_granularity;
	dev->queue->limits.discard_alignment	= dev->discard_alignment;
	if (dev->max_discard_sectors)
		blk_queue_flag_set(QUEUE_FLAG_DISCARD, dev->queue);
	if (dev->secure_discard)
		blk_queue_flag_set(QUEUE_FLAG_SECERASE, dev->queue);

	blk_queue_flag_set(QUEUE_FLAG_SAME_COMP, dev->queue);
	blk_queue_flag_set(QUEUE_FLAG_SAME_FORCE, dev->queue);
	blk_queue_max_segments(dev->queue, dev->max_segments);
	blk_queue_io_opt(dev->queue, dev->sess->max_io_size);
	blk_queue_virt_boundary(dev->queue, SZ_4K - 1);
	blk_queue_write_cache(dev->queue, dev->wc, dev->fua);
	dev->queue->queuedata = dev;
}

static void rnbd_clt_setup_gen_disk(struct rnbd_clt_dev *dev, int idx)
{
	dev->gd->major		= rnbd_client_major;
	dev->gd->first_minor	= idx << RNBD_PART_BITS;
	dev->gd->fops		= &rnbd_client_ops;
	dev->gd->queue		= dev->queue;
	dev->gd->private_data	= dev;
	snprintf(dev->gd->disk_name, sizeof(dev->gd->disk_name), "rnbd%d",
		 idx);
	pr_debug("disk_name=%s, capacity=%zu\n",
		 dev->gd->disk_name,
		 dev->nsectors * (dev->logical_block_size / SECTOR_SIZE)
		 );

	set_capacity(dev->gd, dev->nsectors);

	if (dev->access_mode == RNBD_ACCESS_RO) {
		dev->read_only = true;
		set_disk_ro(dev->gd, true);
	} else {
		dev->read_only = false;
	}

	if (!dev->rotational)
		blk_queue_flag_set(QUEUE_FLAG_NONROT, dev->queue);
	add_disk(dev->gd);
}

static int rnbd_client_setup_device(struct rnbd_clt_dev *dev)
{
	int err, idx = dev->clt_device_id;

	dev->size = dev->nsectors * dev->logical_block_size;

	err = setup_mq_dev(dev);
	if (err)
		return err;

	setup_request_queue(dev);

	dev->gd = alloc_disk_node(1 << RNBD_PART_BITS,	NUMA_NO_NODE);
	if (!dev->gd) {
		blk_cleanup_queue(dev->queue);
		return -ENOMEM;
	}

	rnbd_clt_setup_gen_disk(dev, idx);

	return 0;
}

static struct rnbd_clt_dev *init_dev(struct rnbd_clt_session *sess,
				      enum rnbd_access_mode access_mode,
				      const char *pathname,
				      u32 nr_poll_queues)
{
	struct rnbd_clt_dev *dev;
	int ret;

	dev = kzalloc_node(sizeof(*dev), GFP_KERNEL, NUMA_NO_NODE);
	if (!dev)
		return ERR_PTR(-ENOMEM);

	/*
	 * nr_cpu_ids: the number of softirq queues
	 * nr_poll_queues: the number of polling queues
	 */
	dev->hw_queues = kcalloc(nr_cpu_ids + nr_poll_queues,
				 sizeof(*dev->hw_queues),
				 GFP_KERNEL);
	if (!dev->hw_queues) {
		ret = -ENOMEM;
		goto out_alloc;
	}

	mutex_lock(&ida_lock);
	ret = ida_simple_get(&index_ida, 0, 1 << (MINORBITS - RNBD_PART_BITS),
			     GFP_KERNEL);
	mutex_unlock(&ida_lock);
	if (ret < 0) {
		pr_err("Failed to initialize device '%s' from session %s, allocating idr failed, err: %d\n",
		       pathname, sess->sessname, ret);
		goto out_queues;
	}

	dev->pathname = kstrdup(pathname, GFP_KERNEL);
	if (!dev->pathname) {
		ret = -ENOMEM;
		goto out_queues;
	}

	dev->clt_device_id	= ret;
	dev->sess		= sess;
	dev->access_mode	= access_mode;
	dev->nr_poll_queues	= nr_poll_queues;
	mutex_init(&dev->lock);
	refcount_set(&dev->refcount, 1);
	dev->dev_state = DEV_STATE_INIT;

	/*
	 * Here we called from sysfs entry, thus clt-sysfs is
	 * responsible that session will not disappear.
	 */
	WARN_ON(!rnbd_clt_get_sess(sess));

	return dev;

out_queues:
	kfree(dev->hw_queues);
out_alloc:
	kfree(dev);
	return ERR_PTR(ret);
}

static bool __exists_dev(const char *pathname, const char *sessname)
{
	struct rnbd_clt_session *sess;
	struct rnbd_clt_dev *dev;
	bool found = false;

	list_for_each_entry(sess, &sess_list, list) {
		if (sessname && strncmp(sess->sessname, sessname,
					sizeof(sess->sessname)))
			continue;
		mutex_lock(&sess->lock);
		list_for_each_entry(dev, &sess->devs_list, list) {
			if (strlen(dev->pathname) == strlen(pathname) &&
			    !strcmp(dev->pathname, pathname)) {
				found = true;
				break;
			}
		}
		mutex_unlock(&sess->lock);
		if (found)
			break;
	}

	return found;
}

static bool exists_devpath(const char *pathname, const char *sessname)
{
	bool found;

	mutex_lock(&sess_lock);
	found = __exists_dev(pathname, sessname);
	mutex_unlock(&sess_lock);

	return found;
}

static bool insert_dev_if_not_exists_devpath(struct rnbd_clt_dev *dev)
{
	bool found;
	struct rnbd_clt_session *sess = dev->sess;

	mutex_lock(&sess_lock);
	found = __exists_dev(dev->pathname, sess->sessname);
	if (!found) {
		mutex_lock(&sess->lock);
		list_add_tail(&dev->list, &sess->devs_list);
		mutex_unlock(&sess->lock);
	}
	mutex_unlock(&sess_lock);

	return found;
}

static void delete_dev(struct rnbd_clt_dev *dev)
{
	struct rnbd_clt_session *sess = dev->sess;

	mutex_lock(&sess->lock);
	list_del(&dev->list);
	mutex_unlock(&sess->lock);
}

struct rnbd_clt_dev *rnbd_clt_map_device(const char *sessname,
					   struct rtrs_addr *paths,
					   size_t path_cnt, u16 port_nr,
					   const char *pathname,
					   enum rnbd_access_mode access_mode,
					   u32 nr_poll_queues)
{
	struct rnbd_clt_session *sess;
	struct rnbd_clt_dev *dev;
	int ret;

	if (exists_devpath(pathname, sessname))
		return ERR_PTR(-EEXIST);

	sess = find_and_get_or_create_sess(sessname, paths, path_cnt, port_nr, nr_poll_queues);
	if (IS_ERR(sess))
		return ERR_CAST(sess);

	dev = init_dev(sess, access_mode, pathname, nr_poll_queues);
	if (IS_ERR(dev)) {
		pr_err("map_device: failed to map device '%s' from session %s, can't initialize device, err: %ld\n",
		       pathname, sess->sessname, PTR_ERR(dev));
		ret = PTR_ERR(dev);
		goto put_sess;
	}
	if (insert_dev_if_not_exists_devpath(dev)) {
		ret = -EEXIST;
		goto put_dev;
	}
	ret = send_msg_open(dev, RTRS_PERMIT_WAIT);
	if (ret) {
		rnbd_clt_err(dev,
			      "map_device: failed, can't open remote device, err: %d\n",
			      ret);
		goto del_dev;
	}
	mutex_lock(&dev->lock);
	pr_debug("Opened remote device: session=%s, path='%s'\n",
		 sess->sessname, pathname);
	ret = rnbd_client_setup_device(dev);
	if (ret) {
		rnbd_clt_err(dev,
			      "map_device: Failed to configure device, err: %d\n",
			      ret);
		mutex_unlock(&dev->lock);
		goto send_close;
	}

	rnbd_clt_info(dev,
		       "map_device: Device mapped as %s (nsectors: %zu, logical_block_size: %d, physical_block_size: %d, max_write_same_sectors: %d, max_discard_sectors: %d, discard_granularity: %d, discard_alignment: %d, secure_discard: %d, max_segments: %d, max_hw_sectors: %d, rotational: %d, wc: %d, fua: %d)\n",
		       dev->gd->disk_name, dev->nsectors,
		       dev->logical_block_size, dev->physical_block_size,
		       dev->max_write_same_sectors, dev->max_discard_sectors,
		       dev->discard_granularity, dev->discard_alignment,
		       dev->secure_discard, dev->max_segments,
		       dev->max_hw_sectors, dev->rotational, dev->wc, dev->fua);

	mutex_unlock(&dev->lock);
	rnbd_clt_put_sess(sess);

	return dev;

send_close:
	send_msg_close(dev, dev->device_id, RTRS_PERMIT_WAIT);
del_dev:
	delete_dev(dev);
put_dev:
	rnbd_clt_put_dev(dev);
put_sess:
	rnbd_clt_put_sess(sess);

	return ERR_PTR(ret);
}

static void destroy_gen_disk(struct rnbd_clt_dev *dev)
{
	del_gendisk(dev->gd);
	blk_cleanup_queue(dev->queue);
	put_disk(dev->gd);
}

static void destroy_sysfs(struct rnbd_clt_dev *dev,
			  const struct attribute *sysfs_self)
{
	rnbd_clt_remove_dev_symlink(dev);
	if (dev->kobj.state_initialized) {
		if (sysfs_self)
			/* To avoid deadlock firstly remove itself */
			sysfs_remove_file_self(&dev->kobj, sysfs_self);
		kobject_del(&dev->kobj);
		kobject_put(&dev->kobj);
	}
}

int rnbd_clt_unmap_device(struct rnbd_clt_dev *dev, bool force,
			   const struct attribute *sysfs_self)
{
	struct rnbd_clt_session *sess = dev->sess;
	int refcount, ret = 0;
	bool was_mapped;

	mutex_lock(&dev->lock);
	if (dev->dev_state == DEV_STATE_UNMAPPED) {
		rnbd_clt_info(dev, "Device is already being unmapped\n");
		ret = -EALREADY;
		goto err;
	}
	refcount = refcount_read(&dev->refcount);
	if (!force && refcount > 1) {
		rnbd_clt_err(dev,
			      "Closing device failed, device is in use, (%d device users)\n",
			      refcount - 1);
		ret = -EBUSY;
		goto err;
	}
	was_mapped = (dev->dev_state == DEV_STATE_MAPPED);
	dev->dev_state = DEV_STATE_UNMAPPED;
	mutex_unlock(&dev->lock);

	delete_dev(dev);
	destroy_sysfs(dev, sysfs_self);
	destroy_gen_disk(dev);
	if (was_mapped && sess->rtrs)
		send_msg_close(dev, dev->device_id, RTRS_PERMIT_WAIT);

	rnbd_clt_info(dev, "Device is unmapped\n");

	/* Likely last reference put */
	rnbd_clt_put_dev(dev);

	/*
	 * Here device and session can be vanished!
	 */

	return 0;
err:
	mutex_unlock(&dev->lock);

	return ret;
}

int rnbd_clt_remap_device(struct rnbd_clt_dev *dev)
{
	int err;

	mutex_lock(&dev->lock);
	if (dev->dev_state == DEV_STATE_MAPPED_DISCONNECTED)
		err = 0;
	else if (dev->dev_state == DEV_STATE_UNMAPPED)
		err = -ENODEV;
	else if (dev->dev_state == DEV_STATE_MAPPED)
		err = -EALREADY;
	else
		err = -EBUSY;
	mutex_unlock(&dev->lock);
	if (!err) {
		rnbd_clt_info(dev, "Remapping device.\n");
		err = send_msg_open(dev, RTRS_PERMIT_WAIT);
		if (err)
			rnbd_clt_err(dev, "remap_device: %d\n", err);
	}

	return err;
}

static void unmap_device_work(struct work_struct *work)
{
	struct rnbd_clt_dev *dev;

	dev = container_of(work, typeof(*dev), unmap_on_rmmod_work);
	rnbd_clt_unmap_device(dev, true, NULL);
}

static void rnbd_destroy_sessions(void)
{
	struct rnbd_clt_session *sess, *sn;
	struct rnbd_clt_dev *dev, *tn;

	/* Firstly forbid access through sysfs interface */
	rnbd_clt_destroy_sysfs_files();

	/*
	 * Here at this point there is no any concurrent access to sessions
	 * list and devices list:
	 *   1. New session or device can't be created - session sysfs files
	 *      are removed.
	 *   2. Device or session can't be removed - module reference is taken
	 *      into account in unmap device sysfs callback.
	 *   3. No IO requests inflight - each file open of block_dev increases
	 *      module reference in get_disk().
	 *
	 * But still there can be user requests inflights, which are sent by
	 * asynchronous send_msg_*() functions, thus before unmapping devices
	 * RTRS session must be explicitly closed.
	 */

	list_for_each_entry_safe(sess, sn, &sess_list, list) {
		if (!rnbd_clt_get_sess(sess))
			continue;
		close_rtrs(sess);
		list_for_each_entry_safe(dev, tn, &sess->devs_list, list) {
			/*
			 * Here unmap happens in parallel for only one reason:
			 * blk_cleanup_queue() takes around half a second, so
			 * on huge amount of devices the whole module unload
			 * procedure takes minutes.
			 */
			INIT_WORK(&dev->unmap_on_rmmod_work, unmap_device_work);
			queue_work(system_long_wq, &dev->unmap_on_rmmod_work);
		}
		rnbd_clt_put_sess(sess);
	}
	/* Wait for all scheduled unmap works */
	flush_workqueue(system_long_wq);
	WARN_ON(!list_empty(&sess_list));
}

static int __init rnbd_client_init(void)
{
	int err = 0;

	BUILD_BUG_ON(sizeof(struct rnbd_msg_hdr) != 4);
	BUILD_BUG_ON(sizeof(struct rnbd_msg_sess_info) != 36);
	BUILD_BUG_ON(sizeof(struct rnbd_msg_sess_info_rsp) != 36);
	BUILD_BUG_ON(sizeof(struct rnbd_msg_open) != 264);
	BUILD_BUG_ON(sizeof(struct rnbd_msg_close) != 8);
	BUILD_BUG_ON(sizeof(struct rnbd_msg_open_rsp) != 56);
	rnbd_client_major = register_blkdev(rnbd_client_major, "rnbd");
	if (rnbd_client_major <= 0) {
		pr_err("Failed to load module, block device registration failed\n");
		return -EBUSY;
	}

	err = rnbd_clt_create_sysfs_files();
	if (err) {
		pr_err("Failed to load module, creating sysfs device files failed, err: %d\n",
		       err);
		unregister_blkdev(rnbd_client_major, "rnbd");
	}

	return err;
}

static void __exit rnbd_client_exit(void)
{
	rnbd_destroy_sessions();
	unregister_blkdev(rnbd_client_major, "rnbd");
	ida_destroy(&index_ida);
}

module_init(rnbd_client_init);
module_exit(rnbd_client_exit);