soc/ti/k3-navss-ringacc.c

// SPDX-License-Identifier: GPL-2.0+
/*
 * TI K3 AM65x NAVSS Ring accelerator Manager (RA) subsystem driver
 *
 * Copyright (C) 2018 Texas Instruments Incorporated - http://www.ti.com
 */

#include <common.h>
#include <cpu_func.h>
#include <log.h>
#include <asm/cache.h>
#include <asm/io.h>
#include <malloc.h>
#include <asm/bitops.h>
#include <dm.h>
#include <dm/device_compat.h>
#include <dm/devres.h>
#include <dm/read.h>
#include <dm/uclass.h>
#include <linux/bitops.h>
#include <linux/compat.h>
#include <linux/dma-mapping.h>
#include <linux/err.h>
#include <linux/soc/ti/k3-navss-ringacc.h>
#include <linux/soc/ti/ti_sci_protocol.h>
#include <linux/soc/ti/cppi5.h>

#define set_bit(bit, bitmap)	__set_bit(bit, bitmap)
#define clear_bit(bit, bitmap)	__clear_bit(bit, bitmap)
#define dma_free_coherent(dev, size, cpu_addr, dma_handle) \
	dma_free_coherent(cpu_addr)
#define dma_zalloc_coherent(dev, size, dma_handle, flag) \
({ \
	void	*ring_mem_virt; \
	ring_mem_virt = dma_alloc_coherent((size), \
					   (unsigned long *)(dma_handle)); \
	if (ring_mem_virt) \
		memset(ring_mem_virt, 0, (size)); \
	ring_mem_virt; \
})

static LIST_HEAD(k3_nav_ringacc_list);

static	void ringacc_writel(u32 v, void __iomem *reg)
{
	pr_debug("WRITEL(32): v(%08X)-->reg(%p)\n", v, reg);
	writel(v, reg);
}

static	u32 ringacc_readl(void __iomem *reg)
{
	u32 v;

	v = readl(reg);
	pr_debug("READL(32): v(%08X)<--reg(%p)\n", v, reg);
	return v;
}

#define KNAV_RINGACC_CFG_RING_SIZE_ELCNT_MASK		GENMASK(19, 0)
#define K3_DMARING_RING_CFG_RING_SIZE_ELCNT_MASK		GENMASK(15, 0)

/**
 * struct k3_nav_ring_rt_regs -  The RA Control/Status Registers region
 */
struct k3_nav_ring_rt_regs {
	u32	resv_16[4];
	u32	db;		/* RT Ring N Doorbell Register */
	u32	resv_4[1];
	u32	occ;		/* RT Ring N Occupancy Register */
	u32	indx;		/* RT Ring N Current Index Register */
	u32	hwocc;		/* RT Ring N Hardware Occupancy Register */
	u32	hwindx;		/* RT Ring N Current Index Register */
};

#define KNAV_RINGACC_RT_REGS_STEP	0x1000
#define K3_DMARING_RING_RT_REGS_STEP			0x2000
#define K3_DMARING_RING_RT_REGS_REVERSE_OFS		0x1000
#define KNAV_RINGACC_RT_OCC_MASK		GENMASK(20, 0)
#define K3_DMARING_RING_RT_OCC_TDOWN_COMPLETE		BIT(31)
#define K3_DMARING_RING_RT_DB_ENTRY_MASK		GENMASK(7, 0)
#define K3_DMARING_RING_RT_DB_TDOWN_ACK		BIT(31)


/**
 * struct k3_nav_ring_fifo_regs -  The Ring Accelerator Queues Registers region
 */
struct k3_nav_ring_fifo_regs {
	u32	head_data[128];		/* Ring Head Entry Data Registers */
	u32	tail_data[128];		/* Ring Tail Entry Data Registers */
	u32	peek_head_data[128];	/* Ring Peek Head Entry Data Regs */
	u32	peek_tail_data[128];	/* Ring Peek Tail Entry Data Regs */
};

#define KNAV_RINGACC_FIFO_WINDOW_SIZE_BYTES  (512U)
#define KNAV_RINGACC_FIFO_REGS_STEP	0x1000
#define KNAV_RINGACC_MAX_DB_RING_CNT    (127U)

/**
 * struct k3_nav_ring_ops -  Ring operations
 */
struct k3_nav_ring_ops {
	int (*push_tail)(struct k3_nav_ring *ring, void *elm);
	int (*push_head)(struct k3_nav_ring *ring, void *elm);
	int (*pop_tail)(struct k3_nav_ring *ring, void *elm);
	int (*pop_head)(struct k3_nav_ring *ring, void *elm);
};

/**
 * struct k3_nav_ring_state - Internal state tracking structure
 *
 * @free: Number of free entries
 * @occ: Occupancy
 * @windex: Write index
 * @rindex: Read index
 */
struct k3_nav_ring_state {
	u32 free;
	u32 occ;
	u32 windex;
	u32 rindex;
	u32 tdown_complete:1;
};

/**
 * struct k3_nav_ring - RA Ring descriptor
 *
 * @rt - Ring control/status registers
 * @fifos - Ring queues registers
 * @ring_mem_dma - Ring buffer dma address
 * @ring_mem_virt - Ring buffer virt address
 * @ops - Ring operations
 * @size - Ring size in elements
 * @elm_size - Size of the ring element
 * @mode - Ring mode
 * @flags - flags
 * @ring_id - Ring Id
 * @parent - Pointer on struct @k3_nav_ringacc
 * @use_count - Use count for shared rings
 */
struct k3_nav_ring {
	struct k3_nav_ring_rt_regs __iomem *rt;
	struct k3_nav_ring_fifo_regs __iomem *fifos;
	dma_addr_t	ring_mem_dma;
	void		*ring_mem_virt;
	struct k3_nav_ring_ops *ops;
	u32		size;
	enum k3_nav_ring_size elm_size;
	enum k3_nav_ring_mode mode;
	u32		flags;
#define KNAV_RING_FLAG_BUSY	BIT(1)
#define K3_NAV_RING_FLAG_SHARED	BIT(2)
#define K3_NAV_RING_FLAG_REVERSE BIT(3)
	struct k3_nav_ring_state state;
	u32		ring_id;
	struct k3_nav_ringacc	*parent;
	u32		use_count;
};

struct k3_nav_ringacc_ops {
	int (*init)(struct udevice *dev, struct k3_nav_ringacc *ringacc);
};

/**
 * struct k3_nav_ringacc - Rings accelerator descriptor
 *
 * @dev - pointer on RA device
 * @num_rings - number of ring in RA
 * @rm_gp_range - general purpose rings range from tisci
 * @dma_ring_reset_quirk - DMA reset w/a enable
 * @num_proxies - number of RA proxies
 * @rings - array of rings descriptors (struct @k3_nav_ring)
 * @list - list of RAs in the system
 * @tisci - pointer ti-sci handle
 * @tisci_ring_ops - ti-sci rings ops
 * @tisci_dev_id - ti-sci device id
 * @ops: SoC specific ringacc operation
 * @dual_ring: indicate k3_dmaring dual ring support
 */
struct k3_nav_ringacc {
	struct udevice *dev;
	u32 num_rings; /* number of rings in Ringacc module */
	unsigned long *rings_inuse;
	struct ti_sci_resource *rm_gp_range;
	bool dma_ring_reset_quirk;
	u32 num_proxies;

	struct k3_nav_ring *rings;
	struct list_head list;

	const struct ti_sci_handle *tisci;
	const struct ti_sci_rm_ringacc_ops *tisci_ring_ops;
	u32  tisci_dev_id;

	const struct k3_nav_ringacc_ops *ops;
	bool dual_ring;
};

static int k3_nav_ringacc_ring_read_occ(struct k3_nav_ring *ring)
{
	return readl(&ring->rt->occ) & KNAV_RINGACC_RT_OCC_MASK;
}

static void k3_nav_ringacc_ring_update_occ(struct k3_nav_ring *ring)
{
	u32 val;

	val = readl(&ring->rt->occ);

	ring->state.occ = val & KNAV_RINGACC_RT_OCC_MASK;
	ring->state.tdown_complete = !!(val & K3_DMARING_RING_RT_OCC_TDOWN_COMPLETE);
}

static void *k3_nav_ringacc_get_elm_addr(struct k3_nav_ring *ring, u32 idx)
{
	return (idx * (4 << ring->elm_size) + ring->ring_mem_virt);
}

static int k3_nav_ringacc_ring_push_mem(struct k3_nav_ring *ring, void *elem);
static int k3_nav_ringacc_ring_pop_mem(struct k3_nav_ring *ring, void *elem);
static int k3_dmaring_ring_fwd_pop_mem(struct k3_nav_ring *ring, void *elem);
static int k3_dmaring_ring_reverse_pop_mem(struct k3_nav_ring *ring, void *elem);

static struct k3_nav_ring_ops k3_nav_mode_ring_ops = {
		.push_tail = k3_nav_ringacc_ring_push_mem,
		.pop_head = k3_nav_ringacc_ring_pop_mem,
};

static struct k3_nav_ring_ops k3_dmaring_fwd_ring_ops = {
		.push_tail = k3_nav_ringacc_ring_push_mem,
		.pop_head = k3_dmaring_ring_fwd_pop_mem,
};

static struct k3_nav_ring_ops k3_dmaring_reverse_ring_ops = {
		.pop_head = k3_dmaring_ring_reverse_pop_mem,
};

struct udevice *k3_nav_ringacc_get_dev(struct k3_nav_ringacc *ringacc)
{
	return ringacc->dev;
}

struct k3_nav_ring *k3_nav_ringacc_request_ring(struct k3_nav_ringacc *ringacc,
						int id)
{
	if (id == K3_NAV_RINGACC_RING_ID_ANY) {
		/* Request for any general purpose ring */
		struct ti_sci_resource_desc *gp_rings =
					&ringacc->rm_gp_range->desc[0];
		unsigned long size;

		size = gp_rings->start + gp_rings->num;
		id = find_next_zero_bit(ringacc->rings_inuse,
					size, gp_rings->start);
		if (id == size)
			goto error;
	} else if (id < 0) {
		goto error;
	}

	if (test_bit(id, ringacc->rings_inuse) &&
	    !(ringacc->rings[id].flags & K3_NAV_RING_FLAG_SHARED))
		goto error;
	else if (ringacc->rings[id].flags & K3_NAV_RING_FLAG_SHARED)
		goto out;

	if (!try_module_get(ringacc->dev->driver->owner))
		goto error;

	pr_debug("Giving ring#%d\n", id);

	set_bit(id, ringacc->rings_inuse);
out:
	ringacc->rings[id].use_count++;
	return &ringacc->rings[id];

error:
	return NULL;
}

static int k3_dmaring_ring_request_rings_pair(struct k3_nav_ringacc *ringacc,
					      int fwd_id, int compl_id,
					      struct k3_nav_ring **fwd_ring,
					      struct k3_nav_ring **compl_ring)
{
	/* k3_dmaring: fwd_id == compl_id, so we ignore compl_id */
	if (fwd_id < 0)
		return -EINVAL;

	if (test_bit(fwd_id, ringacc->rings_inuse))
		return -EBUSY;

	*fwd_ring = &ringacc->rings[fwd_id];
	*compl_ring = &ringacc->rings[fwd_id + ringacc->num_rings];
	set_bit(fwd_id, ringacc->rings_inuse);
	ringacc->rings[fwd_id].use_count++;
	dev_dbg(ringacc->dev, "Giving ring#%d\n", fwd_id);

	return 0;
}

int k3_nav_ringacc_request_rings_pair(struct k3_nav_ringacc *ringacc,
				      int fwd_id, int compl_id,
				      struct k3_nav_ring **fwd_ring,
				      struct k3_nav_ring **compl_ring)
{
	int ret = 0;

	if (!fwd_ring || !compl_ring)
		return -EINVAL;

	if (ringacc->dual_ring)
		return k3_dmaring_ring_request_rings_pair(ringacc, fwd_id, compl_id,
						    fwd_ring, compl_ring);

	*fwd_ring = k3_nav_ringacc_request_ring(ringacc, fwd_id);
	if (!(*fwd_ring))
		return -ENODEV;

	*compl_ring = k3_nav_ringacc_request_ring(ringacc, compl_id);
	if (!(*compl_ring)) {
		k3_nav_ringacc_ring_free(*fwd_ring);
		ret = -ENODEV;
	}

	return ret;
}

static void k3_ringacc_ring_reset_sci(struct k3_nav_ring *ring)
{
	struct k3_nav_ringacc *ringacc = ring->parent;
	int ret;

	ret = ringacc->tisci_ring_ops->config(
			ringacc->tisci,
			TI_SCI_MSG_VALUE_RM_RING_COUNT_VALID,
			ringacc->tisci_dev_id,
			ring->ring_id,
			0,
			0,
			ring->size,
			0,
			0,
			0);
	if (ret)
		dev_err(ringacc->dev, "TISCI reset ring fail (%d) ring_idx %d\n",
			ret, ring->ring_id);
}

void k3_nav_ringacc_ring_reset(struct k3_nav_ring *ring)
{
	if (!ring || !(ring->flags & KNAV_RING_FLAG_BUSY))
		return;

	memset(&ring->state, 0, sizeof(ring->state));

	k3_ringacc_ring_reset_sci(ring);
}

static void k3_ringacc_ring_reconfig_qmode_sci(struct k3_nav_ring *ring,
					       enum k3_nav_ring_mode mode)
{
	struct k3_nav_ringacc *ringacc = ring->parent;
	int ret;

	ret = ringacc->tisci_ring_ops->config(
			ringacc->tisci,
			TI_SCI_MSG_VALUE_RM_RING_MODE_VALID,
			ringacc->tisci_dev_id,
			ring->ring_id,
			0,
			0,
			0,
			mode,
			0,
			0);
	if (ret)
		dev_err(ringacc->dev, "TISCI reconf qmode fail (%d) ring_idx %d\n",
			ret, ring->ring_id);
}

void k3_nav_ringacc_ring_reset_dma(struct k3_nav_ring *ring, u32 occ)
{
	if (!ring || !(ring->flags & KNAV_RING_FLAG_BUSY))
		return;

	if (!ring->parent->dma_ring_reset_quirk) {
		k3_nav_ringacc_ring_reset(ring);
		return;
	}

	if (!occ)
		occ = ringacc_readl(&ring->rt->occ);

	if (occ) {
		u32 db_ring_cnt, db_ring_cnt_cur;

		pr_debug("%s %u occ: %u\n", __func__,
			 ring->ring_id, occ);
		/* 2. Reset the ring */
		k3_ringacc_ring_reset_sci(ring);

		/*
		 * 3. Setup the ring in ring/doorbell mode
		 * (if not already in this mode)
		 */
		if (ring->mode != K3_NAV_RINGACC_RING_MODE_RING)
			k3_ringacc_ring_reconfig_qmode_sci(
					ring, K3_NAV_RINGACC_RING_MODE_RING);
		/*
		 * 4. Ring the doorbell 2**22 – ringOcc times.
		 * This will wrap the internal UDMAP ring state occupancy
		 * counter (which is 21-bits wide) to 0.
		 */
		db_ring_cnt = (1U << 22) - occ;

		while (db_ring_cnt != 0) {
			/*
			 * Ring the doorbell with the maximum count each
			 * iteration if possible to minimize the total
			 * of writes
			 */
			if (db_ring_cnt > KNAV_RINGACC_MAX_DB_RING_CNT)
				db_ring_cnt_cur = KNAV_RINGACC_MAX_DB_RING_CNT;
			else
				db_ring_cnt_cur = db_ring_cnt;

			writel(db_ring_cnt_cur, &ring->rt->db);
			db_ring_cnt -= db_ring_cnt_cur;
		}

		/* 5. Restore the original ring mode (if not ring mode) */
		if (ring->mode != K3_NAV_RINGACC_RING_MODE_RING)
			k3_ringacc_ring_reconfig_qmode_sci(ring, ring->mode);
	}

	/* 2. Reset the ring */
	k3_nav_ringacc_ring_reset(ring);
}

static void k3_ringacc_ring_free_sci(struct k3_nav_ring *ring)
{
	struct k3_nav_ringacc *ringacc = ring->parent;
	int ret;

	ret = ringacc->tisci_ring_ops->config(
			ringacc->tisci,
			TI_SCI_MSG_VALUE_RM_ALL_NO_ORDER,
			ringacc->tisci_dev_id,
			ring->ring_id,
			0,
			0,
			0,
			0,
			0,
			0);
	if (ret)
		dev_err(ringacc->dev, "TISCI ring free fail (%d) ring_idx %d\n",
			ret, ring->ring_id);
}

int k3_nav_ringacc_ring_free(struct k3_nav_ring *ring)
{
	struct k3_nav_ringacc *ringacc;

	if (!ring)
		return -EINVAL;

	ringacc = ring->parent;

	/*
	 * k3_dmaring: rings shared memory and configuration, only forward ring is
	 * configured and reverse ring considered as slave.
	 */
	if (ringacc->dual_ring && (ring->flags & K3_NAV_RING_FLAG_REVERSE))
		return 0;

	pr_debug("%s flags: 0x%08x\n", __func__, ring->flags);

	if (!test_bit(ring->ring_id, ringacc->rings_inuse))
		return -EINVAL;

	if (--ring->use_count)
		goto out;

	if (!(ring->flags & KNAV_RING_FLAG_BUSY))
		goto no_init;

	k3_ringacc_ring_free_sci(ring);

	dma_free_coherent(ringacc->dev,
			  ring->size * (4 << ring->elm_size),
			  ring->ring_mem_virt, ring->ring_mem_dma);
	ring->flags &= ~KNAV_RING_FLAG_BUSY;
	ring->ops = NULL;

no_init:
	clear_bit(ring->ring_id, ringacc->rings_inuse);

	module_put(ringacc->dev->driver->owner);

out:
	return 0;
}

u32 k3_nav_ringacc_get_ring_id(struct k3_nav_ring *ring)
{
	if (!ring)
		return -EINVAL;

	return ring->ring_id;
}

static int k3_nav_ringacc_ring_cfg_sci(struct k3_nav_ring *ring)
{
	struct k3_nav_ringacc *ringacc = ring->parent;
	u32 ring_idx;
	int ret;

	if (!ringacc->tisci)
		return -EINVAL;

	ring_idx = ring->ring_id;
	ret = ringacc->tisci_ring_ops->config(
			ringacc->tisci,
			TI_SCI_MSG_VALUE_RM_ALL_NO_ORDER,
			ringacc->tisci_dev_id,
			ring_idx,
			lower_32_bits(ring->ring_mem_dma),
			upper_32_bits(ring->ring_mem_dma),
			ring->size,
			ring->mode,
			ring->elm_size,
			0);
	if (ret)
		dev_err(ringacc->dev, "TISCI config ring fail (%d) ring_idx %d\n",
			ret, ring_idx);

	return ret;
}

static int k3_dmaring_ring_cfg(struct k3_nav_ring *ring, struct k3_nav_ring_cfg *cfg)
{
	struct k3_nav_ringacc *ringacc;
	struct k3_nav_ring *reverse_ring;
	int ret = 0;

	if (cfg->elm_size != K3_NAV_RINGACC_RING_ELSIZE_8 ||
	    cfg->mode != K3_NAV_RINGACC_RING_MODE_RING ||
	    cfg->size & ~K3_DMARING_RING_CFG_RING_SIZE_ELCNT_MASK)
		return -EINVAL;

	ringacc = ring->parent;

	/*
	 * k3_dmaring: rings shared memory and configuration, only forward ring is
	 * configured and reverse ring considered as slave.
	 */
	if (ringacc->dual_ring && (ring->flags & K3_NAV_RING_FLAG_REVERSE))
		return 0;

	if (!test_bit(ring->ring_id, ringacc->rings_inuse))
		return -EINVAL;

	ring->size = cfg->size;
	ring->elm_size = cfg->elm_size;
	ring->mode = cfg->mode;
	memset(&ring->state, 0, sizeof(ring->state));

	ring->ops = &k3_dmaring_fwd_ring_ops;

	ring->ring_mem_virt =
		dma_alloc_coherent(ring->size * (4 << ring->elm_size),
				   (unsigned long *)&ring->ring_mem_dma);
	if (!ring->ring_mem_virt) {
		dev_err(ringacc->dev, "Failed to alloc ring mem\n");
		ret = -ENOMEM;
		goto err_free_ops;
	}

	ret = k3_nav_ringacc_ring_cfg_sci(ring);
	if (ret)
		goto err_free_mem;

	ring->flags |= KNAV_RING_FLAG_BUSY;

	/* k3_dmaring: configure reverse ring */
	reverse_ring = &ringacc->rings[ring->ring_id + ringacc->num_rings];
	reverse_ring->size = cfg->size;
	reverse_ring->elm_size = cfg->elm_size;
	reverse_ring->mode = cfg->mode;
	memset(&reverse_ring->state, 0, sizeof(reverse_ring->state));
	reverse_ring->ops = &k3_dmaring_reverse_ring_ops;

	reverse_ring->ring_mem_virt = ring->ring_mem_virt;
	reverse_ring->ring_mem_dma = ring->ring_mem_dma;
	reverse_ring->flags |= KNAV_RING_FLAG_BUSY;

	return 0;

err_free_mem:
	dma_free_coherent(ringacc->dev,
			  ring->size * (4 << ring->elm_size),
			  ring->ring_mem_virt,
			  ring->ring_mem_dma);
err_free_ops:
	ring->ops = NULL;
	return ret;
}

int k3_nav_ringacc_ring_cfg(struct k3_nav_ring *ring,
			    struct k3_nav_ring_cfg *cfg)
{
	struct k3_nav_ringacc *ringacc = ring->parent;
	int ret = 0;

	if (!ring || !cfg)
		return -EINVAL;

	if (ringacc->dual_ring)
		return k3_dmaring_ring_cfg(ring, cfg);

	if (cfg->elm_size > K3_NAV_RINGACC_RING_ELSIZE_256 ||
	    cfg->mode > K3_NAV_RINGACC_RING_MODE_QM ||
	    cfg->size & ~KNAV_RINGACC_CFG_RING_SIZE_ELCNT_MASK ||
	    !test_bit(ring->ring_id, ringacc->rings_inuse))
		return -EINVAL;

	if (ring->use_count != 1)
		return 0;

	ring->size = cfg->size;
	ring->elm_size = cfg->elm_size;
	ring->mode = cfg->mode;
	memset(&ring->state, 0, sizeof(ring->state));

	switch (ring->mode) {
	case K3_NAV_RINGACC_RING_MODE_RING:
		ring->ops = &k3_nav_mode_ring_ops;
		break;
	default:
		ring->ops = NULL;
		ret = -EINVAL;
		goto err_free_ops;
	};

	ring->ring_mem_virt =
			dma_zalloc_coherent(ringacc->dev,
					    ring->size * (4 << ring->elm_size),
					    &ring->ring_mem_dma, GFP_KERNEL);
	if (!ring->ring_mem_virt) {
		dev_err(ringacc->dev, "Failed to alloc ring mem\n");
		ret = -ENOMEM;
		goto err_free_ops;
	}

	ret = k3_nav_ringacc_ring_cfg_sci(ring);

	if (ret)
		goto err_free_mem;

	ring->flags |= KNAV_RING_FLAG_BUSY;
	ring->flags |= (cfg->flags & K3_NAV_RINGACC_RING_SHARED) ?
			K3_NAV_RING_FLAG_SHARED : 0;

	return 0;

err_free_mem:
	dma_free_coherent(ringacc->dev,
			  ring->size * (4 << ring->elm_size),
			  ring->ring_mem_virt,
			  ring->ring_mem_dma);
err_free_ops:
	ring->ops = NULL;
	return ret;
}

u32 k3_nav_ringacc_ring_get_size(struct k3_nav_ring *ring)
{
	if (!ring || !(ring->flags & KNAV_RING_FLAG_BUSY))
		return -EINVAL;

	return ring->size;
}

u32 k3_nav_ringacc_ring_get_free(struct k3_nav_ring *ring)
{
	if (!ring || !(ring->flags & KNAV_RING_FLAG_BUSY))
		return -EINVAL;

	if (!ring->state.free)
		ring->state.free = ring->size - ringacc_readl(&ring->rt->occ);

	return ring->state.free;
}

u32 k3_nav_ringacc_ring_get_occ(struct k3_nav_ring *ring)
{
	if (!ring || !(ring->flags & KNAV_RING_FLAG_BUSY))
		return -EINVAL;

	return ringacc_readl(&ring->rt->occ);
}

u32 k3_nav_ringacc_ring_is_full(struct k3_nav_ring *ring)
{
	return !k3_nav_ringacc_ring_get_free(ring);
}

enum k3_ringacc_access_mode {
	K3_RINGACC_ACCESS_MODE_PUSH_HEAD,
	K3_RINGACC_ACCESS_MODE_POP_HEAD,
	K3_RINGACC_ACCESS_MODE_PUSH_TAIL,
	K3_RINGACC_ACCESS_MODE_POP_TAIL,
	K3_RINGACC_ACCESS_MODE_PEEK_HEAD,
	K3_RINGACC_ACCESS_MODE_PEEK_TAIL,
};

static int k3_dmaring_ring_fwd_pop_mem(struct k3_nav_ring *ring, void *elem)
{
	void *elem_ptr;
	u32 elem_idx;

	/*
	 * k3_dmaring: forward ring is always tied DMA channel and HW does not
	 * maintain any state data required for POP operation and its unknown
	 * how much elements were consumed by HW. So, to actually
	 * do POP, the read pointer has to be recalculated every time.
	 */
	ring->state.occ = k3_nav_ringacc_ring_read_occ(ring);
	if (ring->state.windex >= ring->state.occ)
		elem_idx = ring->state.windex - ring->state.occ;
	else
		elem_idx = ring->size - (ring->state.occ - ring->state.windex);

	elem_ptr = k3_nav_ringacc_get_elm_addr(ring, elem_idx);
	invalidate_dcache_range((unsigned long)ring->ring_mem_virt,
				ALIGN((unsigned long)ring->ring_mem_virt +
				      ring->size * (4 << ring->elm_size),
				      ARCH_DMA_MINALIGN));

	memcpy(elem, elem_ptr, (4 << ring->elm_size));

	ring->state.occ--;
	writel(-1, &ring->rt->db);

	dev_dbg(ring->parent->dev, "%s: occ%d Windex%d Rindex%d pos_ptr%px\n",
		__func__, ring->state.occ, ring->state.windex, elem_idx,
		elem_ptr);
	return 0;
}

static int k3_dmaring_ring_reverse_pop_mem(struct k3_nav_ring *ring, void *elem)
{
	void *elem_ptr;

	elem_ptr = k3_nav_ringacc_get_elm_addr(ring, ring->state.rindex);

	if (ring->state.occ) {
		invalidate_dcache_range((unsigned long)ring->ring_mem_virt,
					ALIGN((unsigned long)ring->ring_mem_virt +
					ring->size * (4 << ring->elm_size),
					ARCH_DMA_MINALIGN));

		memcpy(elem, elem_ptr, (4 << ring->elm_size));
		ring->state.rindex = (ring->state.rindex + 1) % ring->size;
		ring->state.occ--;
		writel(-1 & K3_DMARING_RING_RT_DB_ENTRY_MASK, &ring->rt->db);
	}

	dev_dbg(ring->parent->dev, "%s: occ%d index%d pos_ptr%px\n",
		__func__, ring->state.occ, ring->state.rindex, elem_ptr);
	return 0;
}

static int k3_nav_ringacc_ring_push_mem(struct k3_nav_ring *ring, void *elem)
{
	void *elem_ptr;

	elem_ptr = k3_nav_ringacc_get_elm_addr(ring, ring->state.windex);

	memcpy(elem_ptr, elem, (4 << ring->elm_size));

	flush_dcache_range((unsigned long)ring->ring_mem_virt,
			   ALIGN((unsigned long)ring->ring_mem_virt +
				 ring->size * (4 << ring->elm_size),
				 ARCH_DMA_MINALIGN));

	ring->state.windex = (ring->state.windex + 1) % ring->size;
	ring->state.free--;
	ringacc_writel(1, &ring->rt->db);

	pr_debug("ring_push_mem: free%d index%d\n",
		 ring->state.free, ring->state.windex);

	return 0;
}

static int k3_nav_ringacc_ring_pop_mem(struct k3_nav_ring *ring, void *elem)
{
	void *elem_ptr;

	elem_ptr = k3_nav_ringacc_get_elm_addr(ring, ring->state.rindex);

	invalidate_dcache_range((unsigned long)ring->ring_mem_virt,
				ALIGN((unsigned long)ring->ring_mem_virt +
				      ring->size * (4 << ring->elm_size),
				      ARCH_DMA_MINALIGN));

	memcpy(elem, elem_ptr, (4 << ring->elm_size));

	ring->state.rindex = (ring->state.rindex + 1) % ring->size;
	ring->state.occ--;
	ringacc_writel(-1, &ring->rt->db);

	pr_debug("ring_pop_mem: occ%d index%d pos_ptr%p\n",
		 ring->state.occ, ring->state.rindex, elem_ptr);
	return 0;
}

int k3_nav_ringacc_ring_push(struct k3_nav_ring *ring, void *elem)
{
	int ret = -EOPNOTSUPP;

	if (!ring || !(ring->flags & KNAV_RING_FLAG_BUSY))
		return -EINVAL;

	pr_debug("ring_push%d: free%d index%d\n",
		 ring->ring_id, ring->state.free, ring->state.windex);

	if (k3_nav_ringacc_ring_is_full(ring))
		return -ENOMEM;

	if (ring->ops && ring->ops->push_tail)
		ret = ring->ops->push_tail(ring, elem);

	return ret;
}

int k3_nav_ringacc_ring_push_head(struct k3_nav_ring *ring, void *elem)
{
	int ret = -EOPNOTSUPP;

	if (!ring || !(ring->flags & KNAV_RING_FLAG_BUSY))
		return -EINVAL;

	pr_debug("ring_push_head: free%d index%d\n",
		 ring->state.free, ring->state.windex);

	if (k3_nav_ringacc_ring_is_full(ring))
		return -ENOMEM;

	if (ring->ops && ring->ops->push_head)
		ret = ring->ops->push_head(ring, elem);

	return ret;
}

int k3_nav_ringacc_ring_pop(struct k3_nav_ring *ring, void *elem)
{
	int ret = -EOPNOTSUPP;

	if (!ring || !(ring->flags & KNAV_RING_FLAG_BUSY))
		return -EINVAL;

	if (!ring->state.occ)
		k3_nav_ringacc_ring_update_occ(ring);

	pr_debug("ring_pop%d: occ%d index%d\n",
		 ring->ring_id, ring->state.occ, ring->state.rindex);

	if (!ring->state.occ && !ring->state.tdown_complete)
		return -ENODATA;

	if (ring->ops && ring->ops->pop_head)
		ret = ring->ops->pop_head(ring, elem);

	return ret;
}

int k3_nav_ringacc_ring_pop_tail(struct k3_nav_ring *ring, void *elem)
{
	int ret = -EOPNOTSUPP;

	if (!ring || !(ring->flags & KNAV_RING_FLAG_BUSY))
		return -EINVAL;

	if (!ring->state.occ)
		k3_nav_ringacc_ring_update_occ(ring);

	pr_debug("ring_pop_tail: occ%d index%d\n",
		 ring->state.occ, ring->state.rindex);

	if (!ring->state.occ)
		return -ENODATA;

	if (ring->ops && ring->ops->pop_tail)
		ret = ring->ops->pop_tail(ring, elem);

	return ret;
}

static int k3_nav_ringacc_probe_dt(struct k3_nav_ringacc *ringacc)
{
	struct udevice *dev = ringacc->dev;
	struct udevice *devp = dev;
	struct udevice *tisci_dev = NULL;
	int ret;

	ringacc->num_rings = dev_read_u32_default(dev, "ti,num-rings", 0);
	if (!ringacc->num_rings) {
		dev_err(dev, "ti,num-rings read failure %d\n", ret);
		return -EINVAL;
	}

	ringacc->dma_ring_reset_quirk =
			dev_read_bool(dev, "ti,dma-ring-reset-quirk");

	ret = uclass_get_device_by_phandle(UCLASS_FIRMWARE, devp,
					   "ti,sci", &tisci_dev);
	if (ret) {
		pr_debug("TISCI RA RM get failed (%d)\n", ret);
		ringacc->tisci = NULL;
		return -ENODEV;
	}
	ringacc->tisci = (struct ti_sci_handle *)
			 (ti_sci_get_handle_from_sysfw(tisci_dev));

	ret = dev_read_u32_default(devp, "ti,sci", 0);
	if (!ret) {
		dev_err(dev, "TISCI RA RM disabled\n");
		ringacc->tisci = NULL;
		return ret;
	}

	ret = dev_read_u32(devp, "ti,sci-dev-id", &ringacc->tisci_dev_id);
	if (ret) {
		dev_err(dev, "ti,sci-dev-id read failure %d\n", ret);
		ringacc->tisci = NULL;
		return ret;
	}

	ringacc->rm_gp_range = devm_ti_sci_get_of_resource(
					ringacc->tisci, dev,
					ringacc->tisci_dev_id,
					"ti,sci-rm-range-gp-rings");
	if (IS_ERR(ringacc->rm_gp_range))
		ret = PTR_ERR(ringacc->rm_gp_range);

	return 0;
}

static int k3_nav_ringacc_init(struct udevice *dev, struct k3_nav_ringacc *ringacc)
{
	void __iomem *base_rt;
	int ret, i;

	ret = k3_nav_ringacc_probe_dt(ringacc);
	if (ret)
		return ret;

	base_rt = (uint32_t *)devfdt_get_addr_name(dev, "rt");
	pr_debug("rt %p\n", base_rt);
	if (IS_ERR(base_rt))
		return PTR_ERR(base_rt);

	ringacc->rings = devm_kzalloc(dev,
				      sizeof(*ringacc->rings) *
				      ringacc->num_rings,
				      GFP_KERNEL);
	ringacc->rings_inuse = devm_kcalloc(dev,
					    BITS_TO_LONGS(ringacc->num_rings),
					    sizeof(unsigned long), GFP_KERNEL);

	if (!ringacc->rings || !ringacc->rings_inuse)
		return -ENOMEM;

	for (i = 0; i < ringacc->num_rings; i++) {
		ringacc->rings[i].rt = base_rt +
				       KNAV_RINGACC_RT_REGS_STEP * i;
		ringacc->rings[i].parent = ringacc;
		ringacc->rings[i].ring_id = i;
	}
	dev_set_drvdata(dev, ringacc);

	ringacc->tisci_ring_ops = &ringacc->tisci->ops.rm_ring_ops;

	list_add_tail(&ringacc->list, &k3_nav_ringacc_list);

	dev_info(dev, "Ring Accelerator probed rings:%u, gp-rings[%u,%u] sci-dev-id:%u\n",
		 ringacc->num_rings,
		 ringacc->rm_gp_range->desc[0].start,
		 ringacc->rm_gp_range->desc[0].num,
		 ringacc->tisci_dev_id);
	dev_info(dev, "dma-ring-reset-quirk: %s\n",
		 ringacc->dma_ring_reset_quirk ? "enabled" : "disabled");
	return 0;
}

struct k3_nav_ringacc *k3_ringacc_dmarings_init(struct udevice *dev,
						struct k3_ringacc_init_data *data)
{
	struct k3_nav_ringacc *ringacc;
	void __iomem *base_rt;
	int i;

	ringacc = devm_kzalloc(dev, sizeof(*ringacc), GFP_KERNEL);
	if (!ringacc)
		return ERR_PTR(-ENOMEM);

	ringacc->dual_ring = true;

	ringacc->dev = dev;
	ringacc->num_rings = data->num_rings;
	ringacc->tisci = data->tisci;
	ringacc->tisci_dev_id = data->tisci_dev_id;

	base_rt = (uint32_t *)devfdt_get_addr_name(dev, "ringrt");
	if (IS_ERR(base_rt))
		return base_rt;

	ringacc->rings = devm_kzalloc(dev,
				      sizeof(*ringacc->rings) *
				      ringacc->num_rings * 2,
				      GFP_KERNEL);
	ringacc->rings_inuse = devm_kcalloc(dev,
					    BITS_TO_LONGS(ringacc->num_rings),
					    sizeof(unsigned long), GFP_KERNEL);

	if (!ringacc->rings || !ringacc->rings_inuse)
		return ERR_PTR(-ENOMEM);

	for (i = 0; i < ringacc->num_rings; i++) {
		struct k3_nav_ring *ring = &ringacc->rings[i];

		ring->rt = base_rt + K3_DMARING_RING_RT_REGS_STEP * i;
		ring->parent = ringacc;
		ring->ring_id = i;

		ring = &ringacc->rings[ringacc->num_rings + i];
		ring->rt = base_rt + K3_DMARING_RING_RT_REGS_STEP * i +
			   K3_DMARING_RING_RT_REGS_REVERSE_OFS;
		ring->parent = ringacc;
		ring->ring_id = i;
		ring->flags = K3_NAV_RING_FLAG_REVERSE;
	}

	ringacc->tisci_ring_ops = &ringacc->tisci->ops.rm_ring_ops;

	dev_info(dev, "k3_dmaring Ring probed rings:%u, sci-dev-id:%u\n",
		 ringacc->num_rings,
		 ringacc->tisci_dev_id);
	dev_info(dev, "dma-ring-reset-quirk: %s\n",
		 ringacc->dma_ring_reset_quirk ? "enabled" : "disabled");

	return ringacc;
}

struct ringacc_match_data {
	struct k3_nav_ringacc_ops ops;
};

static struct ringacc_match_data k3_nav_ringacc_data = {
	.ops = {
		.init = k3_nav_ringacc_init,
	},
};

static const struct udevice_id knav_ringacc_ids[] = {
	{ .compatible = "ti,am654-navss-ringacc", .data = (ulong)&k3_nav_ringacc_data, },
	{},
};

static int k3_nav_ringacc_probe(struct udevice *dev)
{
	struct k3_nav_ringacc *ringacc;
	int ret;
	const struct ringacc_match_data *match_data;

	match_data = (struct ringacc_match_data *)dev_get_driver_data(dev);

	ringacc = dev_get_priv(dev);
	if (!ringacc)
		return -ENOMEM;

	ringacc->dev = dev;
	ringacc->ops = &match_data->ops;
	ret = ringacc->ops->init(dev, ringacc);
	if (ret)
		return ret;

	return 0;
}

U_BOOT_DRIVER(k3_navss_ringacc) = {
	.name	= "k3-navss-ringacc",
	.id	= UCLASS_MISC,
	.of_match = knav_ringacc_ids,
	.probe = k3_nav_ringacc_probe,
	.priv_auto	= sizeof(struct k3_nav_ringacc),
};