1 // SPDX-License-Identifier: GPL-2.0-only
2 /* Marvell RVU Admin Function driver
3  *
4  * Copyright (C) 2020 Marvell.
5  *
6  */
7 
8 #include <linux/bitfield.h>
9 #include <linux/pci.h>
10 #include "rvu_struct.h"
11 #include "rvu_reg.h"
12 #include "mbox.h"
13 #include "rvu.h"
14 
15 /* CPT PF device id */
16 #define	PCI_DEVID_OTX2_CPT_PF	0xA0FD
17 #define	PCI_DEVID_OTX2_CPT10K_PF 0xA0F2
18 
19 /* Length of initial context fetch in 128 byte words */
20 #define CPT_CTX_ILEN    1ULL
21 
22 #define cpt_get_eng_sts(e_min, e_max, rsp, etype)                   \
23 ({                                                                  \
24 	u64 free_sts = 0, busy_sts = 0;                             \
25 	typeof(rsp) _rsp = rsp;                                     \
26 	u32 e, i;                                                   \
27 								    \
28 	for (e = (e_min), i = 0; e < (e_max); e++, i++) {           \
29 		reg = rvu_read64(rvu, blkaddr, CPT_AF_EXEX_STS(e)); \
30 		if (reg & 0x1)                                      \
31 			busy_sts |= 1ULL << i;                      \
32 								    \
33 		if (reg & 0x2)                                      \
34 			free_sts |= 1ULL << i;                      \
35 	}                                                           \
36 	(_rsp)->busy_sts_##etype = busy_sts;                        \
37 	(_rsp)->free_sts_##etype = free_sts;                        \
38 })
39 
cpt_af_flt_intr_handler(int vec,void * ptr)40 static irqreturn_t cpt_af_flt_intr_handler(int vec, void *ptr)
41 {
42 	struct rvu_block *block = ptr;
43 	struct rvu *rvu = block->rvu;
44 	int blkaddr = block->addr;
45 	u64 reg, val;
46 	int i, eng;
47 	u8 grp;
48 
49 	reg = rvu_read64(rvu, blkaddr, CPT_AF_FLTX_INT(vec));
50 	dev_err_ratelimited(rvu->dev, "Received CPTAF FLT%d irq : 0x%llx", vec, reg);
51 
52 	i = -1;
53 	while ((i = find_next_bit((unsigned long *)&reg, 64, i + 1)) < 64) {
54 		switch (vec) {
55 		case 0:
56 			eng = i;
57 			break;
58 		case 1:
59 			eng = i + 64;
60 			break;
61 		case 2:
62 			eng = i + 128;
63 			break;
64 		}
65 		grp = rvu_read64(rvu, blkaddr, CPT_AF_EXEX_CTL2(eng)) & 0xFF;
66 		/* Disable and enable the engine which triggers fault */
67 		rvu_write64(rvu, blkaddr, CPT_AF_EXEX_CTL2(eng), 0x0);
68 		val = rvu_read64(rvu, blkaddr, CPT_AF_EXEX_CTL(eng));
69 		rvu_write64(rvu, blkaddr, CPT_AF_EXEX_CTL(eng), val & ~1ULL);
70 
71 		rvu_write64(rvu, blkaddr, CPT_AF_EXEX_CTL2(eng), grp);
72 		rvu_write64(rvu, blkaddr, CPT_AF_EXEX_CTL(eng), val | 1ULL);
73 
74 		spin_lock(&rvu->cpt_intr_lock);
75 		block->cpt_flt_eng_map[vec] |= BIT_ULL(i);
76 		val = rvu_read64(rvu, blkaddr, CPT_AF_EXEX_STS(eng));
77 		val = val & 0x3;
78 		if (val == 0x1 || val == 0x2)
79 			block->cpt_rcvrd_eng_map[vec] |= BIT_ULL(i);
80 		spin_unlock(&rvu->cpt_intr_lock);
81 	}
82 	rvu_write64(rvu, blkaddr, CPT_AF_FLTX_INT(vec), reg);
83 
84 	return IRQ_HANDLED;
85 }
86 
rvu_cpt_af_flt0_intr_handler(int irq,void * ptr)87 static irqreturn_t rvu_cpt_af_flt0_intr_handler(int irq, void *ptr)
88 {
89 	return cpt_af_flt_intr_handler(CPT_AF_INT_VEC_FLT0, ptr);
90 }
91 
rvu_cpt_af_flt1_intr_handler(int irq,void * ptr)92 static irqreturn_t rvu_cpt_af_flt1_intr_handler(int irq, void *ptr)
93 {
94 	return cpt_af_flt_intr_handler(CPT_AF_INT_VEC_FLT1, ptr);
95 }
96 
rvu_cpt_af_flt2_intr_handler(int irq,void * ptr)97 static irqreturn_t rvu_cpt_af_flt2_intr_handler(int irq, void *ptr)
98 {
99 	return cpt_af_flt_intr_handler(CPT_10K_AF_INT_VEC_FLT2, ptr);
100 }
101 
rvu_cpt_af_rvu_intr_handler(int irq,void * ptr)102 static irqreturn_t rvu_cpt_af_rvu_intr_handler(int irq, void *ptr)
103 {
104 	struct rvu_block *block = ptr;
105 	struct rvu *rvu = block->rvu;
106 	int blkaddr = block->addr;
107 	u64 reg;
108 
109 	reg = rvu_read64(rvu, blkaddr, CPT_AF_RVU_INT);
110 	dev_err_ratelimited(rvu->dev, "Received CPTAF RVU irq : 0x%llx", reg);
111 
112 	rvu_write64(rvu, blkaddr, CPT_AF_RVU_INT, reg);
113 	return IRQ_HANDLED;
114 }
115 
rvu_cpt_af_ras_intr_handler(int irq,void * ptr)116 static irqreturn_t rvu_cpt_af_ras_intr_handler(int irq, void *ptr)
117 {
118 	struct rvu_block *block = ptr;
119 	struct rvu *rvu = block->rvu;
120 	int blkaddr = block->addr;
121 	u64 reg;
122 
123 	reg = rvu_read64(rvu, blkaddr, CPT_AF_RAS_INT);
124 	dev_err_ratelimited(rvu->dev, "Received CPTAF RAS irq : 0x%llx", reg);
125 
126 	rvu_write64(rvu, blkaddr, CPT_AF_RAS_INT, reg);
127 	return IRQ_HANDLED;
128 }
129 
rvu_cpt_do_register_interrupt(struct rvu_block * block,int irq_offs,irq_handler_t handler,const char * name)130 static int rvu_cpt_do_register_interrupt(struct rvu_block *block, int irq_offs,
131 					 irq_handler_t handler,
132 					 const char *name)
133 {
134 	struct rvu *rvu = block->rvu;
135 	int ret;
136 
137 	ret = request_irq(pci_irq_vector(rvu->pdev, irq_offs), handler, 0,
138 			  name, block);
139 	if (ret) {
140 		dev_err(rvu->dev, "RVUAF: %s irq registration failed", name);
141 		return ret;
142 	}
143 
144 	WARN_ON(rvu->irq_allocated[irq_offs]);
145 	rvu->irq_allocated[irq_offs] = true;
146 	return 0;
147 }
148 
cpt_10k_unregister_interrupts(struct rvu_block * block,int off)149 static void cpt_10k_unregister_interrupts(struct rvu_block *block, int off)
150 {
151 	struct rvu *rvu = block->rvu;
152 	int blkaddr = block->addr;
153 	int i;
154 
155 	/* Disable all CPT AF interrupts */
156 	rvu_write64(rvu, blkaddr, CPT_AF_FLTX_INT_ENA_W1C(0), ~0ULL);
157 	rvu_write64(rvu, blkaddr, CPT_AF_FLTX_INT_ENA_W1C(1), ~0ULL);
158 	rvu_write64(rvu, blkaddr, CPT_AF_FLTX_INT_ENA_W1C(2), 0xFFFF);
159 
160 	rvu_write64(rvu, blkaddr, CPT_AF_RVU_INT_ENA_W1C, 0x1);
161 	rvu_write64(rvu, blkaddr, CPT_AF_RAS_INT_ENA_W1C, 0x1);
162 
163 	for (i = 0; i < CPT_10K_AF_INT_VEC_CNT; i++)
164 		if (rvu->irq_allocated[off + i]) {
165 			free_irq(pci_irq_vector(rvu->pdev, off + i), block);
166 			rvu->irq_allocated[off + i] = false;
167 		}
168 }
169 
cpt_unregister_interrupts(struct rvu * rvu,int blkaddr)170 static void cpt_unregister_interrupts(struct rvu *rvu, int blkaddr)
171 {
172 	struct rvu_hwinfo *hw = rvu->hw;
173 	struct rvu_block *block;
174 	int i, offs;
175 
176 	if (!is_block_implemented(rvu->hw, blkaddr))
177 		return;
178 	offs = rvu_read64(rvu, blkaddr, CPT_PRIV_AF_INT_CFG) & 0x7FF;
179 	if (!offs) {
180 		dev_warn(rvu->dev,
181 			 "Failed to get CPT_AF_INT vector offsets\n");
182 		return;
183 	}
184 	block = &hw->block[blkaddr];
185 	if (!is_rvu_otx2(rvu))
186 		return cpt_10k_unregister_interrupts(block, offs);
187 
188 	/* Disable all CPT AF interrupts */
189 	for (i = 0; i < CPT_AF_INT_VEC_RVU; i++)
190 		rvu_write64(rvu, blkaddr, CPT_AF_FLTX_INT_ENA_W1C(i), ~0ULL);
191 	rvu_write64(rvu, blkaddr, CPT_AF_RVU_INT_ENA_W1C, 0x1);
192 	rvu_write64(rvu, blkaddr, CPT_AF_RAS_INT_ENA_W1C, 0x1);
193 
194 	for (i = 0; i < CPT_AF_INT_VEC_CNT; i++)
195 		if (rvu->irq_allocated[offs + i]) {
196 			free_irq(pci_irq_vector(rvu->pdev, offs + i), block);
197 			rvu->irq_allocated[offs + i] = false;
198 		}
199 }
200 
rvu_cpt_unregister_interrupts(struct rvu * rvu)201 void rvu_cpt_unregister_interrupts(struct rvu *rvu)
202 {
203 	cpt_unregister_interrupts(rvu, BLKADDR_CPT0);
204 	cpt_unregister_interrupts(rvu, BLKADDR_CPT1);
205 }
206 
cpt_10k_register_interrupts(struct rvu_block * block,int off)207 static int cpt_10k_register_interrupts(struct rvu_block *block, int off)
208 {
209 	struct rvu *rvu = block->rvu;
210 	int blkaddr = block->addr;
211 	irq_handler_t flt_fn;
212 	int i, ret;
213 
214 	for (i = CPT_10K_AF_INT_VEC_FLT0; i < CPT_10K_AF_INT_VEC_RVU; i++) {
215 		sprintf(&rvu->irq_name[(off + i) * NAME_SIZE], "CPTAF FLT%d", i);
216 
217 		switch (i) {
218 		case CPT_10K_AF_INT_VEC_FLT0:
219 			flt_fn = rvu_cpt_af_flt0_intr_handler;
220 			break;
221 		case CPT_10K_AF_INT_VEC_FLT1:
222 			flt_fn = rvu_cpt_af_flt1_intr_handler;
223 			break;
224 		case CPT_10K_AF_INT_VEC_FLT2:
225 			flt_fn = rvu_cpt_af_flt2_intr_handler;
226 			break;
227 		}
228 		ret = rvu_cpt_do_register_interrupt(block, off + i,
229 						    flt_fn, &rvu->irq_name[(off + i) * NAME_SIZE]);
230 		if (ret)
231 			goto err;
232 		if (i == CPT_10K_AF_INT_VEC_FLT2)
233 			rvu_write64(rvu, blkaddr, CPT_AF_FLTX_INT_ENA_W1S(i), 0xFFFF);
234 		else
235 			rvu_write64(rvu, blkaddr, CPT_AF_FLTX_INT_ENA_W1S(i), ~0ULL);
236 	}
237 
238 	ret = rvu_cpt_do_register_interrupt(block, off + CPT_10K_AF_INT_VEC_RVU,
239 					    rvu_cpt_af_rvu_intr_handler,
240 					    "CPTAF RVU");
241 	if (ret)
242 		goto err;
243 	rvu_write64(rvu, blkaddr, CPT_AF_RVU_INT_ENA_W1S, 0x1);
244 
245 	ret = rvu_cpt_do_register_interrupt(block, off + CPT_10K_AF_INT_VEC_RAS,
246 					    rvu_cpt_af_ras_intr_handler,
247 					    "CPTAF RAS");
248 	if (ret)
249 		goto err;
250 	rvu_write64(rvu, blkaddr, CPT_AF_RAS_INT_ENA_W1S, 0x1);
251 
252 	return 0;
253 err:
254 	rvu_cpt_unregister_interrupts(rvu);
255 	return ret;
256 }
257 
cpt_register_interrupts(struct rvu * rvu,int blkaddr)258 static int cpt_register_interrupts(struct rvu *rvu, int blkaddr)
259 {
260 	struct rvu_hwinfo *hw = rvu->hw;
261 	struct rvu_block *block;
262 	irq_handler_t flt_fn;
263 	int i, offs, ret = 0;
264 
265 	if (!is_block_implemented(rvu->hw, blkaddr))
266 		return 0;
267 
268 	block = &hw->block[blkaddr];
269 	offs = rvu_read64(rvu, blkaddr, CPT_PRIV_AF_INT_CFG) & 0x7FF;
270 	if (!offs) {
271 		dev_warn(rvu->dev,
272 			 "Failed to get CPT_AF_INT vector offsets\n");
273 		return 0;
274 	}
275 
276 	if (!is_rvu_otx2(rvu))
277 		return cpt_10k_register_interrupts(block, offs);
278 
279 	for (i = CPT_AF_INT_VEC_FLT0; i < CPT_AF_INT_VEC_RVU; i++) {
280 		sprintf(&rvu->irq_name[(offs + i) * NAME_SIZE], "CPTAF FLT%d", i);
281 		switch (i) {
282 		case CPT_AF_INT_VEC_FLT0:
283 			flt_fn = rvu_cpt_af_flt0_intr_handler;
284 			break;
285 		case CPT_AF_INT_VEC_FLT1:
286 			flt_fn = rvu_cpt_af_flt1_intr_handler;
287 			break;
288 		}
289 		ret = rvu_cpt_do_register_interrupt(block, offs + i,
290 						    flt_fn, &rvu->irq_name[(offs + i) * NAME_SIZE]);
291 		if (ret)
292 			goto err;
293 		rvu_write64(rvu, blkaddr, CPT_AF_FLTX_INT_ENA_W1S(i), ~0ULL);
294 	}
295 
296 	ret = rvu_cpt_do_register_interrupt(block, offs + CPT_AF_INT_VEC_RVU,
297 					    rvu_cpt_af_rvu_intr_handler,
298 					    "CPTAF RVU");
299 	if (ret)
300 		goto err;
301 	rvu_write64(rvu, blkaddr, CPT_AF_RVU_INT_ENA_W1S, 0x1);
302 
303 	ret = rvu_cpt_do_register_interrupt(block, offs + CPT_AF_INT_VEC_RAS,
304 					    rvu_cpt_af_ras_intr_handler,
305 					    "CPTAF RAS");
306 	if (ret)
307 		goto err;
308 	rvu_write64(rvu, blkaddr, CPT_AF_RAS_INT_ENA_W1S, 0x1);
309 
310 	return 0;
311 err:
312 	rvu_cpt_unregister_interrupts(rvu);
313 	return ret;
314 }
315 
rvu_cpt_register_interrupts(struct rvu * rvu)316 int rvu_cpt_register_interrupts(struct rvu *rvu)
317 {
318 	int ret;
319 
320 	ret = cpt_register_interrupts(rvu, BLKADDR_CPT0);
321 	if (ret)
322 		return ret;
323 
324 	return cpt_register_interrupts(rvu, BLKADDR_CPT1);
325 }
326 
get_cpt_pf_num(struct rvu * rvu)327 static int get_cpt_pf_num(struct rvu *rvu)
328 {
329 	int i, domain_nr, cpt_pf_num = -1;
330 	struct pci_dev *pdev;
331 
332 	domain_nr = pci_domain_nr(rvu->pdev->bus);
333 	for (i = 0; i < rvu->hw->total_pfs; i++) {
334 		pdev = pci_get_domain_bus_and_slot(domain_nr, i + 1, 0);
335 		if (!pdev)
336 			continue;
337 
338 		if (pdev->device == PCI_DEVID_OTX2_CPT_PF ||
339 		    pdev->device == PCI_DEVID_OTX2_CPT10K_PF) {
340 			cpt_pf_num = i;
341 			put_device(&pdev->dev);
342 			break;
343 		}
344 		put_device(&pdev->dev);
345 	}
346 	return cpt_pf_num;
347 }
348 
is_cpt_pf(struct rvu * rvu,u16 pcifunc)349 static bool is_cpt_pf(struct rvu *rvu, u16 pcifunc)
350 {
351 	int cpt_pf_num = rvu->cpt_pf_num;
352 
353 	if (rvu_get_pf(pcifunc) != cpt_pf_num)
354 		return false;
355 	if (pcifunc & RVU_PFVF_FUNC_MASK)
356 		return false;
357 
358 	return true;
359 }
360 
is_cpt_vf(struct rvu * rvu,u16 pcifunc)361 static bool is_cpt_vf(struct rvu *rvu, u16 pcifunc)
362 {
363 	int cpt_pf_num = rvu->cpt_pf_num;
364 
365 	if (rvu_get_pf(pcifunc) != cpt_pf_num)
366 		return false;
367 	if (!(pcifunc & RVU_PFVF_FUNC_MASK))
368 		return false;
369 
370 	return true;
371 }
372 
validate_and_get_cpt_blkaddr(int req_blkaddr)373 static int validate_and_get_cpt_blkaddr(int req_blkaddr)
374 {
375 	int blkaddr;
376 
377 	blkaddr = req_blkaddr ? req_blkaddr : BLKADDR_CPT0;
378 	if (blkaddr != BLKADDR_CPT0 && blkaddr != BLKADDR_CPT1)
379 		return -EINVAL;
380 
381 	return blkaddr;
382 }
383 
rvu_mbox_handler_cpt_lf_alloc(struct rvu * rvu,struct cpt_lf_alloc_req_msg * req,struct msg_rsp * rsp)384 int rvu_mbox_handler_cpt_lf_alloc(struct rvu *rvu,
385 				  struct cpt_lf_alloc_req_msg *req,
386 				  struct msg_rsp *rsp)
387 {
388 	u16 pcifunc = req->hdr.pcifunc;
389 	struct rvu_block *block;
390 	int cptlf, blkaddr;
391 	int num_lfs, slot;
392 	u64 val;
393 
394 	blkaddr = validate_and_get_cpt_blkaddr(req->blkaddr);
395 	if (blkaddr < 0)
396 		return blkaddr;
397 
398 	if (req->eng_grpmsk == 0x0)
399 		return CPT_AF_ERR_GRP_INVALID;
400 
401 	block = &rvu->hw->block[blkaddr];
402 	num_lfs = rvu_get_rsrc_mapcount(rvu_get_pfvf(rvu, pcifunc),
403 					block->addr);
404 	if (!num_lfs)
405 		return CPT_AF_ERR_LF_INVALID;
406 
407 	/* Check if requested 'CPTLF <=> NIXLF' mapping is valid */
408 	if (req->nix_pf_func) {
409 		/* If default, use 'this' CPTLF's PFFUNC */
410 		if (req->nix_pf_func == RVU_DEFAULT_PF_FUNC)
411 			req->nix_pf_func = pcifunc;
412 		if (!is_pffunc_map_valid(rvu, req->nix_pf_func, BLKTYPE_NIX))
413 			return CPT_AF_ERR_NIX_PF_FUNC_INVALID;
414 	}
415 
416 	/* Check if requested 'CPTLF <=> SSOLF' mapping is valid */
417 	if (req->sso_pf_func) {
418 		/* If default, use 'this' CPTLF's PFFUNC */
419 		if (req->sso_pf_func == RVU_DEFAULT_PF_FUNC)
420 			req->sso_pf_func = pcifunc;
421 		if (!is_pffunc_map_valid(rvu, req->sso_pf_func, BLKTYPE_SSO))
422 			return CPT_AF_ERR_SSO_PF_FUNC_INVALID;
423 	}
424 
425 	for (slot = 0; slot < num_lfs; slot++) {
426 		cptlf = rvu_get_lf(rvu, block, pcifunc, slot);
427 		if (cptlf < 0)
428 			return CPT_AF_ERR_LF_INVALID;
429 
430 		/* Set CPT LF group and priority */
431 		val = (u64)req->eng_grpmsk << 48 | 1;
432 		if (!is_rvu_otx2(rvu)) {
433 			if (req->ctx_ilen_valid)
434 				val |= (req->ctx_ilen << 17);
435 			else
436 				val |= (CPT_CTX_ILEN << 17);
437 		}
438 
439 		rvu_write64(rvu, blkaddr, CPT_AF_LFX_CTL(cptlf), val);
440 
441 		/* Set CPT LF NIX_PF_FUNC and SSO_PF_FUNC. EXE_LDWB is set
442 		 * on reset.
443 		 */
444 		val = rvu_read64(rvu, blkaddr, CPT_AF_LFX_CTL2(cptlf));
445 		val &= ~(GENMASK_ULL(63, 48) | GENMASK_ULL(47, 32));
446 		val |= ((u64)req->nix_pf_func << 48 |
447 			(u64)req->sso_pf_func << 32);
448 		rvu_write64(rvu, blkaddr, CPT_AF_LFX_CTL2(cptlf), val);
449 	}
450 
451 	return 0;
452 }
453 
cpt_lf_free(struct rvu * rvu,struct msg_req * req,int blkaddr)454 static int cpt_lf_free(struct rvu *rvu, struct msg_req *req, int blkaddr)
455 {
456 	u16 pcifunc = req->hdr.pcifunc;
457 	int num_lfs, cptlf, slot, err;
458 	struct rvu_block *block;
459 
460 	block = &rvu->hw->block[blkaddr];
461 	num_lfs = rvu_get_rsrc_mapcount(rvu_get_pfvf(rvu, pcifunc),
462 					block->addr);
463 	if (!num_lfs)
464 		return 0;
465 
466 	for (slot = 0; slot < num_lfs; slot++) {
467 		cptlf = rvu_get_lf(rvu, block, pcifunc, slot);
468 		if (cptlf < 0)
469 			return CPT_AF_ERR_LF_INVALID;
470 
471 		/* Perform teardown */
472 		rvu_cpt_lf_teardown(rvu, pcifunc, blkaddr, cptlf, slot);
473 
474 		/* Reset LF */
475 		err = rvu_lf_reset(rvu, block, cptlf);
476 		if (err) {
477 			dev_err(rvu->dev, "Failed to reset blkaddr %d LF%d\n",
478 				block->addr, cptlf);
479 		}
480 	}
481 
482 	return 0;
483 }
484 
rvu_mbox_handler_cpt_lf_free(struct rvu * rvu,struct msg_req * req,struct msg_rsp * rsp)485 int rvu_mbox_handler_cpt_lf_free(struct rvu *rvu, struct msg_req *req,
486 				 struct msg_rsp *rsp)
487 {
488 	int ret;
489 
490 	ret = cpt_lf_free(rvu, req, BLKADDR_CPT0);
491 	if (ret)
492 		return ret;
493 
494 	if (is_block_implemented(rvu->hw, BLKADDR_CPT1))
495 		ret = cpt_lf_free(rvu, req, BLKADDR_CPT1);
496 
497 	return ret;
498 }
499 
cpt_inline_ipsec_cfg_inbound(struct rvu * rvu,int blkaddr,u8 cptlf,struct cpt_inline_ipsec_cfg_msg * req)500 static int cpt_inline_ipsec_cfg_inbound(struct rvu *rvu, int blkaddr, u8 cptlf,
501 					struct cpt_inline_ipsec_cfg_msg *req)
502 {
503 	u16 sso_pf_func = req->sso_pf_func;
504 	u8 nix_sel;
505 	u64 val;
506 
507 	val = rvu_read64(rvu, blkaddr, CPT_AF_LFX_CTL(cptlf));
508 	if (req->enable && (val & BIT_ULL(16))) {
509 		/* IPSec inline outbound path is already enabled for a given
510 		 * CPT LF, HRM states that inline inbound & outbound paths
511 		 * must not be enabled at the same time for a given CPT LF
512 		 */
513 		return CPT_AF_ERR_INLINE_IPSEC_INB_ENA;
514 	}
515 	/* Check if requested 'CPTLF <=> SSOLF' mapping is valid */
516 	if (sso_pf_func && !is_pffunc_map_valid(rvu, sso_pf_func, BLKTYPE_SSO))
517 		return CPT_AF_ERR_SSO_PF_FUNC_INVALID;
518 
519 	nix_sel = (blkaddr == BLKADDR_CPT1) ? 1 : 0;
520 	/* Enable CPT LF for IPsec inline inbound operations */
521 	if (req->enable)
522 		val |= BIT_ULL(9);
523 	else
524 		val &= ~BIT_ULL(9);
525 
526 	val |= (u64)nix_sel << 8;
527 	rvu_write64(rvu, blkaddr, CPT_AF_LFX_CTL(cptlf), val);
528 
529 	if (sso_pf_func) {
530 		/* Set SSO_PF_FUNC */
531 		val = rvu_read64(rvu, blkaddr, CPT_AF_LFX_CTL2(cptlf));
532 		val |= (u64)sso_pf_func << 32;
533 		val |= (u64)req->nix_pf_func << 48;
534 		rvu_write64(rvu, blkaddr, CPT_AF_LFX_CTL2(cptlf), val);
535 	}
536 	if (req->sso_pf_func_ovrd)
537 		/* Set SSO_PF_FUNC_OVRD for inline IPSec */
538 		rvu_write64(rvu, blkaddr, CPT_AF_ECO, 0x1);
539 
540 	/* Configure the X2P Link register with the cpt base channel number and
541 	 * range of channels it should propagate to X2P
542 	 */
543 	if (!is_rvu_otx2(rvu)) {
544 		val = (ilog2(NIX_CHAN_CPT_X2P_MASK + 1) << 16);
545 		val |= (u64)rvu->hw->cpt_chan_base;
546 
547 		rvu_write64(rvu, blkaddr, CPT_AF_X2PX_LINK_CFG(0), val);
548 		rvu_write64(rvu, blkaddr, CPT_AF_X2PX_LINK_CFG(1), val);
549 	}
550 
551 	return 0;
552 }
553 
cpt_inline_ipsec_cfg_outbound(struct rvu * rvu,int blkaddr,u8 cptlf,struct cpt_inline_ipsec_cfg_msg * req)554 static int cpt_inline_ipsec_cfg_outbound(struct rvu *rvu, int blkaddr, u8 cptlf,
555 					 struct cpt_inline_ipsec_cfg_msg *req)
556 {
557 	u16 nix_pf_func = req->nix_pf_func;
558 	int nix_blkaddr;
559 	u8 nix_sel;
560 	u64 val;
561 
562 	val = rvu_read64(rvu, blkaddr, CPT_AF_LFX_CTL(cptlf));
563 	if (req->enable && (val & BIT_ULL(9))) {
564 		/* IPSec inline inbound path is already enabled for a given
565 		 * CPT LF, HRM states that inline inbound & outbound paths
566 		 * must not be enabled at the same time for a given CPT LF
567 		 */
568 		return CPT_AF_ERR_INLINE_IPSEC_OUT_ENA;
569 	}
570 
571 	/* Check if requested 'CPTLF <=> NIXLF' mapping is valid */
572 	if (nix_pf_func && !is_pffunc_map_valid(rvu, nix_pf_func, BLKTYPE_NIX))
573 		return CPT_AF_ERR_NIX_PF_FUNC_INVALID;
574 
575 	/* Enable CPT LF for IPsec inline outbound operations */
576 	if (req->enable)
577 		val |= BIT_ULL(16);
578 	else
579 		val &= ~BIT_ULL(16);
580 	rvu_write64(rvu, blkaddr, CPT_AF_LFX_CTL(cptlf), val);
581 
582 	if (nix_pf_func) {
583 		/* Set NIX_PF_FUNC */
584 		val = rvu_read64(rvu, blkaddr, CPT_AF_LFX_CTL2(cptlf));
585 		val |= (u64)nix_pf_func << 48;
586 		rvu_write64(rvu, blkaddr, CPT_AF_LFX_CTL2(cptlf), val);
587 
588 		nix_blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NIX, nix_pf_func);
589 		nix_sel = (nix_blkaddr == BLKADDR_NIX0) ? 0 : 1;
590 
591 		val = rvu_read64(rvu, blkaddr, CPT_AF_LFX_CTL(cptlf));
592 		val |= (u64)nix_sel << 8;
593 		rvu_write64(rvu, blkaddr, CPT_AF_LFX_CTL(cptlf), val);
594 	}
595 
596 	return 0;
597 }
598 
rvu_mbox_handler_cpt_inline_ipsec_cfg(struct rvu * rvu,struct cpt_inline_ipsec_cfg_msg * req,struct msg_rsp * rsp)599 int rvu_mbox_handler_cpt_inline_ipsec_cfg(struct rvu *rvu,
600 					  struct cpt_inline_ipsec_cfg_msg *req,
601 					  struct msg_rsp *rsp)
602 {
603 	u16 pcifunc = req->hdr.pcifunc;
604 	struct rvu_block *block;
605 	int cptlf, blkaddr, ret;
606 	u16 actual_slot;
607 
608 	blkaddr = rvu_get_blkaddr_from_slot(rvu, BLKTYPE_CPT, pcifunc,
609 					    req->slot, &actual_slot);
610 	if (blkaddr < 0)
611 		return CPT_AF_ERR_LF_INVALID;
612 
613 	block = &rvu->hw->block[blkaddr];
614 
615 	cptlf = rvu_get_lf(rvu, block, pcifunc, actual_slot);
616 	if (cptlf < 0)
617 		return CPT_AF_ERR_LF_INVALID;
618 
619 	switch (req->dir) {
620 	case CPT_INLINE_INBOUND:
621 		ret = cpt_inline_ipsec_cfg_inbound(rvu, blkaddr, cptlf, req);
622 		break;
623 
624 	case CPT_INLINE_OUTBOUND:
625 		ret = cpt_inline_ipsec_cfg_outbound(rvu, blkaddr, cptlf, req);
626 		break;
627 
628 	default:
629 		return CPT_AF_ERR_PARAM;
630 	}
631 
632 	return ret;
633 }
634 
is_valid_offset(struct rvu * rvu,struct cpt_rd_wr_reg_msg * req)635 static bool is_valid_offset(struct rvu *rvu, struct cpt_rd_wr_reg_msg *req)
636 {
637 	u64 offset = req->reg_offset;
638 	int blkaddr, num_lfs, lf;
639 	struct rvu_block *block;
640 	struct rvu_pfvf *pfvf;
641 
642 	blkaddr = validate_and_get_cpt_blkaddr(req->blkaddr);
643 	if (blkaddr < 0)
644 		return false;
645 
646 	/* Registers that can be accessed from PF/VF */
647 	if ((offset & 0xFF000) ==  CPT_AF_LFX_CTL(0) ||
648 	    (offset & 0xFF000) ==  CPT_AF_LFX_CTL2(0)) {
649 		if (offset & 7)
650 			return false;
651 
652 		lf = (offset & 0xFFF) >> 3;
653 		block = &rvu->hw->block[blkaddr];
654 		pfvf = rvu_get_pfvf(rvu, req->hdr.pcifunc);
655 		num_lfs = rvu_get_rsrc_mapcount(pfvf, block->addr);
656 		if (lf >= num_lfs)
657 			/* Slot is not valid for that PF/VF */
658 			return false;
659 
660 		/* Translate local LF used by VFs to global CPT LF */
661 		lf = rvu_get_lf(rvu, &rvu->hw->block[blkaddr],
662 				req->hdr.pcifunc, lf);
663 		if (lf < 0)
664 			return false;
665 
666 		return true;
667 	} else if (!(req->hdr.pcifunc & RVU_PFVF_FUNC_MASK)) {
668 		/* Registers that can be accessed from PF */
669 		switch (offset) {
670 		case CPT_AF_DIAG:
671 		case CPT_AF_CTL:
672 		case CPT_AF_PF_FUNC:
673 		case CPT_AF_BLK_RST:
674 		case CPT_AF_CONSTANTS1:
675 		case CPT_AF_CTX_FLUSH_TIMER:
676 			return true;
677 		}
678 
679 		switch (offset & 0xFF000) {
680 		case CPT_AF_EXEX_STS(0):
681 		case CPT_AF_EXEX_CTL(0):
682 		case CPT_AF_EXEX_CTL2(0):
683 		case CPT_AF_EXEX_UCODE_BASE(0):
684 			if (offset & 7)
685 				return false;
686 			break;
687 		default:
688 			return false;
689 		}
690 		return true;
691 	}
692 	return false;
693 }
694 
rvu_mbox_handler_cpt_rd_wr_register(struct rvu * rvu,struct cpt_rd_wr_reg_msg * req,struct cpt_rd_wr_reg_msg * rsp)695 int rvu_mbox_handler_cpt_rd_wr_register(struct rvu *rvu,
696 					struct cpt_rd_wr_reg_msg *req,
697 					struct cpt_rd_wr_reg_msg *rsp)
698 {
699 	int blkaddr;
700 
701 	blkaddr = validate_and_get_cpt_blkaddr(req->blkaddr);
702 	if (blkaddr < 0)
703 		return blkaddr;
704 
705 	/* This message is accepted only if sent from CPT PF/VF */
706 	if (!is_cpt_pf(rvu, req->hdr.pcifunc) &&
707 	    !is_cpt_vf(rvu, req->hdr.pcifunc))
708 		return CPT_AF_ERR_ACCESS_DENIED;
709 
710 	rsp->reg_offset = req->reg_offset;
711 	rsp->ret_val = req->ret_val;
712 	rsp->is_write = req->is_write;
713 
714 	if (!is_valid_offset(rvu, req))
715 		return CPT_AF_ERR_ACCESS_DENIED;
716 
717 	if (req->is_write)
718 		rvu_write64(rvu, blkaddr, req->reg_offset, req->val);
719 	else
720 		rsp->val = rvu_read64(rvu, blkaddr, req->reg_offset);
721 
722 	return 0;
723 }
724 
get_ctx_pc(struct rvu * rvu,struct cpt_sts_rsp * rsp,int blkaddr)725 static void get_ctx_pc(struct rvu *rvu, struct cpt_sts_rsp *rsp, int blkaddr)
726 {
727 	if (is_rvu_otx2(rvu))
728 		return;
729 
730 	rsp->ctx_mis_pc = rvu_read64(rvu, blkaddr, CPT_AF_CTX_MIS_PC);
731 	rsp->ctx_hit_pc = rvu_read64(rvu, blkaddr, CPT_AF_CTX_HIT_PC);
732 	rsp->ctx_aop_pc = rvu_read64(rvu, blkaddr, CPT_AF_CTX_AOP_PC);
733 	rsp->ctx_aop_lat_pc = rvu_read64(rvu, blkaddr,
734 					 CPT_AF_CTX_AOP_LATENCY_PC);
735 	rsp->ctx_ifetch_pc = rvu_read64(rvu, blkaddr, CPT_AF_CTX_IFETCH_PC);
736 	rsp->ctx_ifetch_lat_pc = rvu_read64(rvu, blkaddr,
737 					    CPT_AF_CTX_IFETCH_LATENCY_PC);
738 	rsp->ctx_ffetch_pc = rvu_read64(rvu, blkaddr, CPT_AF_CTX_FFETCH_PC);
739 	rsp->ctx_ffetch_lat_pc = rvu_read64(rvu, blkaddr,
740 					    CPT_AF_CTX_FFETCH_LATENCY_PC);
741 	rsp->ctx_wback_pc = rvu_read64(rvu, blkaddr, CPT_AF_CTX_FFETCH_PC);
742 	rsp->ctx_wback_lat_pc = rvu_read64(rvu, blkaddr,
743 					   CPT_AF_CTX_FFETCH_LATENCY_PC);
744 	rsp->ctx_psh_pc = rvu_read64(rvu, blkaddr, CPT_AF_CTX_FFETCH_PC);
745 	rsp->ctx_psh_lat_pc = rvu_read64(rvu, blkaddr,
746 					 CPT_AF_CTX_FFETCH_LATENCY_PC);
747 	rsp->ctx_err = rvu_read64(rvu, blkaddr, CPT_AF_CTX_ERR);
748 	rsp->ctx_enc_id = rvu_read64(rvu, blkaddr, CPT_AF_CTX_ENC_ID);
749 	rsp->ctx_flush_timer = rvu_read64(rvu, blkaddr, CPT_AF_CTX_FLUSH_TIMER);
750 
751 	rsp->rxc_time = rvu_read64(rvu, blkaddr, CPT_AF_RXC_TIME);
752 	rsp->rxc_time_cfg = rvu_read64(rvu, blkaddr, CPT_AF_RXC_TIME_CFG);
753 	rsp->rxc_active_sts = rvu_read64(rvu, blkaddr, CPT_AF_RXC_ACTIVE_STS);
754 	rsp->rxc_zombie_sts = rvu_read64(rvu, blkaddr, CPT_AF_RXC_ZOMBIE_STS);
755 	rsp->rxc_dfrg = rvu_read64(rvu, blkaddr, CPT_AF_RXC_DFRG);
756 	rsp->x2p_link_cfg0 = rvu_read64(rvu, blkaddr, CPT_AF_X2PX_LINK_CFG(0));
757 	rsp->x2p_link_cfg1 = rvu_read64(rvu, blkaddr, CPT_AF_X2PX_LINK_CFG(1));
758 }
759 
get_eng_sts(struct rvu * rvu,struct cpt_sts_rsp * rsp,int blkaddr)760 static void get_eng_sts(struct rvu *rvu, struct cpt_sts_rsp *rsp, int blkaddr)
761 {
762 	u16 max_ses, max_ies, max_aes;
763 	u32 e_min = 0, e_max = 0;
764 	u64 reg;
765 
766 	reg = rvu_read64(rvu, blkaddr, CPT_AF_CONSTANTS1);
767 	max_ses = reg & 0xffff;
768 	max_ies = (reg >> 16) & 0xffff;
769 	max_aes = (reg >> 32) & 0xffff;
770 
771 	/* Get AE status */
772 	e_min = max_ses + max_ies;
773 	e_max = max_ses + max_ies + max_aes;
774 	cpt_get_eng_sts(e_min, e_max, rsp, ae);
775 	/* Get SE status */
776 	e_min = 0;
777 	e_max = max_ses;
778 	cpt_get_eng_sts(e_min, e_max, rsp, se);
779 	/* Get IE status */
780 	e_min = max_ses;
781 	e_max = max_ses + max_ies;
782 	cpt_get_eng_sts(e_min, e_max, rsp, ie);
783 }
784 
rvu_mbox_handler_cpt_sts(struct rvu * rvu,struct cpt_sts_req * req,struct cpt_sts_rsp * rsp)785 int rvu_mbox_handler_cpt_sts(struct rvu *rvu, struct cpt_sts_req *req,
786 			     struct cpt_sts_rsp *rsp)
787 {
788 	int blkaddr;
789 
790 	blkaddr = validate_and_get_cpt_blkaddr(req->blkaddr);
791 	if (blkaddr < 0)
792 		return blkaddr;
793 
794 	/* This message is accepted only if sent from CPT PF/VF */
795 	if (!is_cpt_pf(rvu, req->hdr.pcifunc) &&
796 	    !is_cpt_vf(rvu, req->hdr.pcifunc))
797 		return CPT_AF_ERR_ACCESS_DENIED;
798 
799 	get_ctx_pc(rvu, rsp, blkaddr);
800 
801 	/* Get CPT engines status */
802 	get_eng_sts(rvu, rsp, blkaddr);
803 
804 	/* Read CPT instruction PC registers */
805 	rsp->inst_req_pc = rvu_read64(rvu, blkaddr, CPT_AF_INST_REQ_PC);
806 	rsp->inst_lat_pc = rvu_read64(rvu, blkaddr, CPT_AF_INST_LATENCY_PC);
807 	rsp->rd_req_pc = rvu_read64(rvu, blkaddr, CPT_AF_RD_REQ_PC);
808 	rsp->rd_lat_pc = rvu_read64(rvu, blkaddr, CPT_AF_RD_LATENCY_PC);
809 	rsp->rd_uc_pc = rvu_read64(rvu, blkaddr, CPT_AF_RD_UC_PC);
810 	rsp->active_cycles_pc = rvu_read64(rvu, blkaddr,
811 					   CPT_AF_ACTIVE_CYCLES_PC);
812 	rsp->exe_err_info = rvu_read64(rvu, blkaddr, CPT_AF_EXE_ERR_INFO);
813 	rsp->cptclk_cnt = rvu_read64(rvu, blkaddr, CPT_AF_CPTCLK_CNT);
814 	rsp->diag = rvu_read64(rvu, blkaddr, CPT_AF_DIAG);
815 
816 	return 0;
817 }
818 
819 #define RXC_ZOMBIE_THRES  GENMASK_ULL(59, 48)
820 #define RXC_ZOMBIE_LIMIT  GENMASK_ULL(43, 32)
821 #define RXC_ACTIVE_THRES  GENMASK_ULL(27, 16)
822 #define RXC_ACTIVE_LIMIT  GENMASK_ULL(11, 0)
823 #define RXC_ACTIVE_COUNT  GENMASK_ULL(60, 48)
824 #define RXC_ZOMBIE_COUNT  GENMASK_ULL(60, 48)
825 
cpt_rxc_time_cfg(struct rvu * rvu,struct cpt_rxc_time_cfg_req * req,int blkaddr,struct cpt_rxc_time_cfg_req * save)826 static void cpt_rxc_time_cfg(struct rvu *rvu, struct cpt_rxc_time_cfg_req *req,
827 			     int blkaddr, struct cpt_rxc_time_cfg_req *save)
828 {
829 	u64 dfrg_reg;
830 
831 	if (save) {
832 		/* Save older config */
833 		dfrg_reg = rvu_read64(rvu, blkaddr, CPT_AF_RXC_DFRG);
834 		save->zombie_thres = FIELD_GET(RXC_ZOMBIE_THRES, dfrg_reg);
835 		save->zombie_limit = FIELD_GET(RXC_ZOMBIE_LIMIT, dfrg_reg);
836 		save->active_thres = FIELD_GET(RXC_ACTIVE_THRES, dfrg_reg);
837 		save->active_limit = FIELD_GET(RXC_ACTIVE_LIMIT, dfrg_reg);
838 
839 		save->step = rvu_read64(rvu, blkaddr, CPT_AF_RXC_TIME_CFG);
840 	}
841 
842 	dfrg_reg = FIELD_PREP(RXC_ZOMBIE_THRES, req->zombie_thres);
843 	dfrg_reg |= FIELD_PREP(RXC_ZOMBIE_LIMIT, req->zombie_limit);
844 	dfrg_reg |= FIELD_PREP(RXC_ACTIVE_THRES, req->active_thres);
845 	dfrg_reg |= FIELD_PREP(RXC_ACTIVE_LIMIT, req->active_limit);
846 
847 	rvu_write64(rvu, blkaddr, CPT_AF_RXC_TIME_CFG, req->step);
848 	rvu_write64(rvu, blkaddr, CPT_AF_RXC_DFRG, dfrg_reg);
849 }
850 
rvu_mbox_handler_cpt_rxc_time_cfg(struct rvu * rvu,struct cpt_rxc_time_cfg_req * req,struct msg_rsp * rsp)851 int rvu_mbox_handler_cpt_rxc_time_cfg(struct rvu *rvu,
852 				      struct cpt_rxc_time_cfg_req *req,
853 				      struct msg_rsp *rsp)
854 {
855 	int blkaddr;
856 
857 	blkaddr = validate_and_get_cpt_blkaddr(req->blkaddr);
858 	if (blkaddr < 0)
859 		return blkaddr;
860 
861 	/* This message is accepted only if sent from CPT PF/VF */
862 	if (!is_cpt_pf(rvu, req->hdr.pcifunc) &&
863 	    !is_cpt_vf(rvu, req->hdr.pcifunc))
864 		return CPT_AF_ERR_ACCESS_DENIED;
865 
866 	cpt_rxc_time_cfg(rvu, req, blkaddr, NULL);
867 
868 	return 0;
869 }
870 
rvu_mbox_handler_cpt_ctx_cache_sync(struct rvu * rvu,struct msg_req * req,struct msg_rsp * rsp)871 int rvu_mbox_handler_cpt_ctx_cache_sync(struct rvu *rvu, struct msg_req *req,
872 					struct msg_rsp *rsp)
873 {
874 	return rvu_cpt_ctx_flush(rvu, req->hdr.pcifunc);
875 }
876 
rvu_mbox_handler_cpt_lf_reset(struct rvu * rvu,struct cpt_lf_rst_req * req,struct msg_rsp * rsp)877 int rvu_mbox_handler_cpt_lf_reset(struct rvu *rvu, struct cpt_lf_rst_req *req,
878 				  struct msg_rsp *rsp)
879 {
880 	u16 pcifunc = req->hdr.pcifunc;
881 	struct rvu_block *block;
882 	int cptlf, blkaddr, ret;
883 	u16 actual_slot;
884 	u64 ctl, ctl2;
885 
886 	blkaddr = rvu_get_blkaddr_from_slot(rvu, BLKTYPE_CPT, pcifunc,
887 					    req->slot, &actual_slot);
888 	if (blkaddr < 0)
889 		return CPT_AF_ERR_LF_INVALID;
890 
891 	block = &rvu->hw->block[blkaddr];
892 
893 	cptlf = rvu_get_lf(rvu, block, pcifunc, actual_slot);
894 	if (cptlf < 0)
895 		return CPT_AF_ERR_LF_INVALID;
896 	ctl = rvu_read64(rvu, blkaddr, CPT_AF_LFX_CTL(cptlf));
897 	ctl2 = rvu_read64(rvu, blkaddr, CPT_AF_LFX_CTL2(cptlf));
898 
899 	ret = rvu_lf_reset(rvu, block, cptlf);
900 	if (ret)
901 		dev_err(rvu->dev, "Failed to reset blkaddr %d LF%d\n",
902 			block->addr, cptlf);
903 
904 	rvu_write64(rvu, blkaddr, CPT_AF_LFX_CTL(cptlf), ctl);
905 	rvu_write64(rvu, blkaddr, CPT_AF_LFX_CTL2(cptlf), ctl2);
906 
907 	return 0;
908 }
909 
rvu_mbox_handler_cpt_flt_eng_info(struct rvu * rvu,struct cpt_flt_eng_info_req * req,struct cpt_flt_eng_info_rsp * rsp)910 int rvu_mbox_handler_cpt_flt_eng_info(struct rvu *rvu, struct cpt_flt_eng_info_req *req,
911 				      struct cpt_flt_eng_info_rsp *rsp)
912 {
913 	struct rvu_block *block;
914 	unsigned long flags;
915 	int blkaddr, vec;
916 
917 	blkaddr = validate_and_get_cpt_blkaddr(req->blkaddr);
918 	if (blkaddr < 0)
919 		return blkaddr;
920 
921 	block = &rvu->hw->block[blkaddr];
922 	for (vec = 0; vec < CPT_10K_AF_INT_VEC_RVU; vec++) {
923 		spin_lock_irqsave(&rvu->cpt_intr_lock, flags);
924 		rsp->flt_eng_map[vec] = block->cpt_flt_eng_map[vec];
925 		rsp->rcvrd_eng_map[vec] = block->cpt_rcvrd_eng_map[vec];
926 		if (req->reset) {
927 			block->cpt_flt_eng_map[vec] = 0x0;
928 			block->cpt_rcvrd_eng_map[vec] = 0x0;
929 		}
930 		spin_unlock_irqrestore(&rvu->cpt_intr_lock, flags);
931 	}
932 	return 0;
933 }
934 
cpt_rxc_teardown(struct rvu * rvu,int blkaddr)935 static void cpt_rxc_teardown(struct rvu *rvu, int blkaddr)
936 {
937 	struct cpt_rxc_time_cfg_req req, prev;
938 	int timeout = 2000;
939 	u64 reg;
940 
941 	if (is_rvu_otx2(rvu))
942 		return;
943 
944 	/* Set time limit to minimum values, so that rxc entries will be
945 	 * flushed out quickly.
946 	 */
947 	req.step = 1;
948 	req.zombie_thres = 1;
949 	req.zombie_limit = 1;
950 	req.active_thres = 1;
951 	req.active_limit = 1;
952 
953 	cpt_rxc_time_cfg(rvu, &req, blkaddr, &prev);
954 
955 	do {
956 		reg = rvu_read64(rvu, blkaddr, CPT_AF_RXC_ACTIVE_STS);
957 		udelay(1);
958 		if (FIELD_GET(RXC_ACTIVE_COUNT, reg))
959 			timeout--;
960 		else
961 			break;
962 	} while (timeout);
963 
964 	if (timeout == 0)
965 		dev_warn(rvu->dev, "Poll for RXC active count hits hard loop counter\n");
966 
967 	timeout = 2000;
968 	do {
969 		reg = rvu_read64(rvu, blkaddr, CPT_AF_RXC_ZOMBIE_STS);
970 		udelay(1);
971 		if (FIELD_GET(RXC_ZOMBIE_COUNT, reg))
972 			timeout--;
973 		else
974 			break;
975 	} while (timeout);
976 
977 	if (timeout == 0)
978 		dev_warn(rvu->dev, "Poll for RXC zombie count hits hard loop counter\n");
979 
980 	/* Restore config */
981 	cpt_rxc_time_cfg(rvu, &prev, blkaddr, NULL);
982 }
983 
984 #define INFLIGHT   GENMASK_ULL(8, 0)
985 #define GRB_CNT    GENMASK_ULL(39, 32)
986 #define GWB_CNT    GENMASK_ULL(47, 40)
987 #define XQ_XOR     GENMASK_ULL(63, 63)
988 #define DQPTR      GENMASK_ULL(19, 0)
989 #define NQPTR      GENMASK_ULL(51, 32)
990 
cpt_lf_disable_iqueue(struct rvu * rvu,int blkaddr,int slot)991 static void cpt_lf_disable_iqueue(struct rvu *rvu, int blkaddr, int slot)
992 {
993 	int timeout = 1000000;
994 	u64 inprog, inst_ptr;
995 	u64 qsize, pending;
996 	int i = 0;
997 
998 	/* Disable instructions enqueuing */
999 	rvu_write64(rvu, blkaddr, CPT_AF_BAR2_ALIASX(slot, CPT_LF_CTL), 0x0);
1000 
1001 	inprog = rvu_read64(rvu, blkaddr,
1002 			    CPT_AF_BAR2_ALIASX(slot, CPT_LF_INPROG));
1003 	inprog |= BIT_ULL(16);
1004 	rvu_write64(rvu, blkaddr,
1005 		    CPT_AF_BAR2_ALIASX(slot, CPT_LF_INPROG), inprog);
1006 
1007 	qsize = rvu_read64(rvu, blkaddr,
1008 			   CPT_AF_BAR2_ALIASX(slot, CPT_LF_Q_SIZE)) & 0x7FFF;
1009 	do {
1010 		inst_ptr = rvu_read64(rvu, blkaddr,
1011 				      CPT_AF_BAR2_ALIASX(slot, CPT_LF_Q_INST_PTR));
1012 		pending = (FIELD_GET(XQ_XOR, inst_ptr) * qsize * 40) +
1013 			  FIELD_GET(NQPTR, inst_ptr) -
1014 			  FIELD_GET(DQPTR, inst_ptr);
1015 		udelay(1);
1016 		timeout--;
1017 	} while ((pending != 0) && (timeout != 0));
1018 
1019 	if (timeout == 0)
1020 		dev_warn(rvu->dev, "TIMEOUT: CPT poll on pending instructions\n");
1021 
1022 	timeout = 1000000;
1023 	/* Wait for CPT queue to become execution-quiescent */
1024 	do {
1025 		inprog = rvu_read64(rvu, blkaddr,
1026 				    CPT_AF_BAR2_ALIASX(slot, CPT_LF_INPROG));
1027 
1028 		if ((FIELD_GET(INFLIGHT, inprog) == 0) &&
1029 		    (FIELD_GET(GRB_CNT, inprog) == 0)) {
1030 			i++;
1031 		} else {
1032 			i = 0;
1033 			timeout--;
1034 		}
1035 	} while ((timeout != 0) && (i < 10));
1036 
1037 	if (timeout == 0)
1038 		dev_warn(rvu->dev, "TIMEOUT: CPT poll on inflight count\n");
1039 	/* Wait for 2 us to flush all queue writes to memory */
1040 	udelay(2);
1041 }
1042 
rvu_cpt_lf_teardown(struct rvu * rvu,u16 pcifunc,int blkaddr,int lf,int slot)1043 int rvu_cpt_lf_teardown(struct rvu *rvu, u16 pcifunc, int blkaddr, int lf, int slot)
1044 {
1045 	u64 reg;
1046 
1047 	if (is_cpt_pf(rvu, pcifunc) || is_cpt_vf(rvu, pcifunc))
1048 		cpt_rxc_teardown(rvu, blkaddr);
1049 
1050 	mutex_lock(&rvu->alias_lock);
1051 	/* Enable BAR2 ALIAS for this pcifunc. */
1052 	reg = BIT_ULL(16) | pcifunc;
1053 	rvu_bar2_sel_write64(rvu, blkaddr, CPT_AF_BAR2_SEL, reg);
1054 
1055 	cpt_lf_disable_iqueue(rvu, blkaddr, slot);
1056 
1057 	rvu_bar2_sel_write64(rvu, blkaddr, CPT_AF_BAR2_SEL, 0);
1058 	mutex_unlock(&rvu->alias_lock);
1059 
1060 	return 0;
1061 }
1062 
1063 #define CPT_RES_LEN    16
1064 #define CPT_SE_IE_EGRP 1ULL
1065 
cpt_inline_inb_lf_cmd_send(struct rvu * rvu,int blkaddr,int nix_blkaddr)1066 static int cpt_inline_inb_lf_cmd_send(struct rvu *rvu, int blkaddr,
1067 				      int nix_blkaddr)
1068 {
1069 	int cpt_pf_num = rvu->cpt_pf_num;
1070 	struct cpt_inst_lmtst_req *req;
1071 	dma_addr_t res_daddr;
1072 	int timeout = 3000;
1073 	u8 cpt_idx;
1074 	u64 *inst;
1075 	u16 *res;
1076 	int rc;
1077 
1078 	res = kzalloc(CPT_RES_LEN, GFP_KERNEL);
1079 	if (!res)
1080 		return -ENOMEM;
1081 
1082 	res_daddr = dma_map_single(rvu->dev, res, CPT_RES_LEN,
1083 				   DMA_BIDIRECTIONAL);
1084 	if (dma_mapping_error(rvu->dev, res_daddr)) {
1085 		dev_err(rvu->dev, "DMA mapping failed for CPT result\n");
1086 		rc = -EFAULT;
1087 		goto res_free;
1088 	}
1089 	*res = 0xFFFF;
1090 
1091 	/* Send mbox message to CPT PF */
1092 	req = (struct cpt_inst_lmtst_req *)
1093 	       otx2_mbox_alloc_msg_rsp(&rvu->afpf_wq_info.mbox_up,
1094 				       cpt_pf_num, sizeof(*req),
1095 				       sizeof(struct msg_rsp));
1096 	if (!req) {
1097 		rc = -ENOMEM;
1098 		goto res_daddr_unmap;
1099 	}
1100 	req->hdr.sig = OTX2_MBOX_REQ_SIG;
1101 	req->hdr.id = MBOX_MSG_CPT_INST_LMTST;
1102 
1103 	inst = req->inst;
1104 	/* Prepare CPT_INST_S */
1105 	inst[0] = 0;
1106 	inst[1] = res_daddr;
1107 	/* AF PF FUNC */
1108 	inst[2] = 0;
1109 	/* Set QORD */
1110 	inst[3] = 1;
1111 	inst[4] = 0;
1112 	inst[5] = 0;
1113 	inst[6] = 0;
1114 	/* Set EGRP */
1115 	inst[7] = CPT_SE_IE_EGRP << 61;
1116 
1117 	/* Subtract 1 from the NIX-CPT credit count to preserve
1118 	 * credit counts.
1119 	 */
1120 	cpt_idx = (blkaddr == BLKADDR_CPT0) ? 0 : 1;
1121 	rvu_write64(rvu, nix_blkaddr, NIX_AF_RX_CPTX_CREDIT(cpt_idx),
1122 		    BIT_ULL(22) - 1);
1123 
1124 	otx2_mbox_msg_send(&rvu->afpf_wq_info.mbox_up, cpt_pf_num);
1125 	rc = otx2_mbox_wait_for_rsp(&rvu->afpf_wq_info.mbox_up, cpt_pf_num);
1126 	if (rc)
1127 		dev_warn(rvu->dev, "notification to pf %d failed\n",
1128 			 cpt_pf_num);
1129 	/* Wait for CPT instruction to be completed */
1130 	do {
1131 		mdelay(1);
1132 		if (*res == 0xFFFF)
1133 			timeout--;
1134 		else
1135 			break;
1136 	} while (timeout);
1137 
1138 	if (timeout == 0)
1139 		dev_warn(rvu->dev, "Poll for result hits hard loop counter\n");
1140 
1141 res_daddr_unmap:
1142 	dma_unmap_single(rvu->dev, res_daddr, CPT_RES_LEN, DMA_BIDIRECTIONAL);
1143 res_free:
1144 	kfree(res);
1145 
1146 	return 0;
1147 }
1148 
1149 #define CTX_CAM_PF_FUNC   GENMASK_ULL(61, 46)
1150 #define CTX_CAM_CPTR      GENMASK_ULL(45, 0)
1151 
rvu_cpt_ctx_flush(struct rvu * rvu,u16 pcifunc)1152 int rvu_cpt_ctx_flush(struct rvu *rvu, u16 pcifunc)
1153 {
1154 	int nix_blkaddr, blkaddr;
1155 	u16 max_ctx_entries, i;
1156 	int slot = 0, num_lfs;
1157 	u64 reg, cam_data;
1158 	int rc;
1159 
1160 	nix_blkaddr = rvu_get_blkaddr(rvu, BLKTYPE_NIX, pcifunc);
1161 	if (nix_blkaddr < 0)
1162 		return -EINVAL;
1163 
1164 	if (is_rvu_otx2(rvu))
1165 		return 0;
1166 
1167 	blkaddr = (nix_blkaddr == BLKADDR_NIX1) ? BLKADDR_CPT1 : BLKADDR_CPT0;
1168 
1169 	/* Submit CPT_INST_S to track when all packets have been
1170 	 * flushed through for the NIX PF FUNC in inline inbound case.
1171 	 */
1172 	rc = cpt_inline_inb_lf_cmd_send(rvu, blkaddr, nix_blkaddr);
1173 	if (rc)
1174 		return rc;
1175 
1176 	/* Wait for rxc entries to be flushed out */
1177 	cpt_rxc_teardown(rvu, blkaddr);
1178 
1179 	reg = rvu_read64(rvu, blkaddr, CPT_AF_CONSTANTS0);
1180 	max_ctx_entries = (reg >> 48) & 0xFFF;
1181 
1182 	mutex_lock(&rvu->rsrc_lock);
1183 
1184 	num_lfs = rvu_get_rsrc_mapcount(rvu_get_pfvf(rvu, pcifunc),
1185 					blkaddr);
1186 	if (num_lfs == 0) {
1187 		dev_warn(rvu->dev, "CPT LF is not configured\n");
1188 		goto unlock;
1189 	}
1190 
1191 	/* Enable BAR2 ALIAS for this pcifunc. */
1192 	reg = BIT_ULL(16) | pcifunc;
1193 	rvu_bar2_sel_write64(rvu, blkaddr, CPT_AF_BAR2_SEL, reg);
1194 
1195 	for (i = 0; i < max_ctx_entries; i++) {
1196 		cam_data = rvu_read64(rvu, blkaddr, CPT_AF_CTX_CAM_DATA(i));
1197 
1198 		if ((FIELD_GET(CTX_CAM_PF_FUNC, cam_data) == pcifunc) &&
1199 		    FIELD_GET(CTX_CAM_CPTR, cam_data)) {
1200 			reg = BIT_ULL(46) | FIELD_GET(CTX_CAM_CPTR, cam_data);
1201 			rvu_write64(rvu, blkaddr,
1202 				    CPT_AF_BAR2_ALIASX(slot, CPT_LF_CTX_FLUSH),
1203 				    reg);
1204 		}
1205 	}
1206 	rvu_bar2_sel_write64(rvu, blkaddr, CPT_AF_BAR2_SEL, 0);
1207 
1208 unlock:
1209 	mutex_unlock(&rvu->rsrc_lock);
1210 
1211 	return 0;
1212 }
1213 
rvu_cpt_init(struct rvu * rvu)1214 int rvu_cpt_init(struct rvu *rvu)
1215 {
1216 	/* Retrieve CPT PF number */
1217 	rvu->cpt_pf_num = get_cpt_pf_num(rvu);
1218 	spin_lock_init(&rvu->cpt_intr_lock);
1219 
1220 	return 0;
1221 }
1222