1 /*-
2 * SPDX-License-Identifier: BSD-2-Clause
3 *
4 * Copyright © 2023 Dmitry Salychev
5 *
6 * Redistribution and use in source and binary forms, with or without
7 * modification, are permitted provided that the following conditions
8 * are met:
9 * 1. Redistributions of source code must retain the above copyright
10 * notice, this list of conditions and the following disclaimer.
11 * 2. Redistributions in binary form must reproduce the above copyright
12 * notice, this list of conditions and the following disclaimer in the
13 * documentation and/or other materials provided with the distribution.
14 *
15 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
16 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
17 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
18 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
19 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
20 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
21 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
22 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
23 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
24 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
25 * SUCH DAMAGE.
26 */
27
28 /*
29 * QBMan channel to process ingress traffic (Rx, Tx confirmation, Rx error).
30 *
31 * NOTE: Several WQs are organized into a single channel.
32 */
33
34 #include <sys/param.h>
35 #include <sys/systm.h>
36 #include <sys/kernel.h>
37 #include <sys/bus.h>
38 #include <sys/rman.h>
39 #include <sys/module.h>
40 #include <sys/malloc.h>
41 #include <sys/mutex.h>
42 #include <sys/socket.h>
43 #include <sys/sockio.h>
44 #include <sys/sysctl.h>
45 #include <sys/mbuf.h>
46 #include <sys/taskqueue.h>
47 #include <sys/sysctl.h>
48 #include <sys/buf_ring.h>
49 #include <sys/smp.h>
50 #include <sys/proc.h>
51
52 #include <machine/bus.h>
53 #include <machine/resource.h>
54 #include <machine/atomic.h>
55 #include <machine/vmparam.h>
56
57 #include <net/ethernet.h>
58 #include <net/bpf.h>
59 #include <net/if.h>
60 #include <net/if_dl.h>
61 #include <net/if_media.h>
62 #include <net/if_types.h>
63 #include <net/if_var.h>
64
65 #include "dpaa2_types.h"
66 #include "dpaa2_channel.h"
67 #include "dpaa2_ni.h"
68 #include "dpaa2_mc.h"
69 #include "dpaa2_mc_if.h"
70 #include "dpaa2_mcp.h"
71 #include "dpaa2_io.h"
72 #include "dpaa2_con.h"
73 #include "dpaa2_buf.h"
74 #include "dpaa2_swp.h"
75 #include "dpaa2_swp_if.h"
76 #include "dpaa2_bp.h"
77 #include "dpaa2_cmd_if.h"
78
79 MALLOC_DEFINE(M_DPAA2_CH, "dpaa2_ch", "DPAA2 QBMan Channel");
80
81 #define RX_SEG_N (1u)
82 #define RX_SEG_SZ (((MJUM9BYTES - 1) / PAGE_SIZE + 1) * PAGE_SIZE)
83 #define RX_SEG_MAXSZ (((MJUM9BYTES - 1) / PAGE_SIZE + 1) * PAGE_SIZE)
84 CTASSERT(RX_SEG_SZ % PAGE_SIZE == 0);
85 CTASSERT(RX_SEG_MAXSZ % PAGE_SIZE == 0);
86
87 #define TX_SEG_N (16u) /* XXX-DSL: does DPAA2 limit exist? */
88 #define TX_SEG_SZ (PAGE_SIZE)
89 #define TX_SEG_MAXSZ (TX_SEG_N * TX_SEG_SZ)
90 CTASSERT(TX_SEG_SZ % PAGE_SIZE == 0);
91 CTASSERT(TX_SEG_MAXSZ % PAGE_SIZE == 0);
92
93 #define SGT_SEG_N (1u)
94 #define SGT_SEG_SZ (PAGE_SIZE)
95 #define SGT_SEG_MAXSZ (PAGE_SIZE)
96 CTASSERT(SGT_SEG_SZ % PAGE_SIZE == 0);
97 CTASSERT(SGT_SEG_MAXSZ % PAGE_SIZE == 0);
98
99 static int dpaa2_chan_setup_dma(device_t, struct dpaa2_channel *, bus_size_t);
100 static int dpaa2_chan_alloc_storage(device_t, struct dpaa2_channel *, bus_size_t,
101 int, bus_size_t);
102 static void dpaa2_chan_bp_task(void *, int);
103
104 /**
105 * @brief Сonfigures QBMan channel and registers data availability notifications.
106 */
107 int
dpaa2_chan_setup(device_t dev,device_t iodev,device_t condev,device_t bpdev,struct dpaa2_channel ** channel,uint32_t flowid,task_fn_t cleanup_task_fn)108 dpaa2_chan_setup(device_t dev, device_t iodev, device_t condev, device_t bpdev,
109 struct dpaa2_channel **channel, uint32_t flowid, task_fn_t cleanup_task_fn)
110 {
111 device_t pdev = device_get_parent(dev);
112 device_t child = dev;
113 struct dpaa2_ni_softc *sc = device_get_softc(dev);
114 struct dpaa2_io_softc *iosc = device_get_softc(iodev);
115 struct dpaa2_con_softc *consc = device_get_softc(condev);
116 struct dpaa2_devinfo *rcinfo = device_get_ivars(pdev);
117 struct dpaa2_devinfo *ioinfo = device_get_ivars(iodev);
118 struct dpaa2_devinfo *coninfo = device_get_ivars(condev);
119 struct dpaa2_con_notif_cfg notif_cfg;
120 struct dpaa2_io_notif_ctx *ctx;
121 struct dpaa2_channel *ch = NULL;
122 struct dpaa2_cmd cmd;
123 uint16_t rctk, contk;
124 int error;
125
126 DPAA2_CMD_INIT(&cmd);
127
128 error = DPAA2_CMD_RC_OPEN(dev, child, &cmd, rcinfo->id, &rctk);
129 if (error) {
130 device_printf(dev, "%s: failed to open DPRC: id=%d, error=%d\n",
131 __func__, rcinfo->id, error);
132 goto fail_rc_open;
133 }
134 error = DPAA2_CMD_CON_OPEN(dev, child, &cmd, coninfo->id, &contk);
135 if (error) {
136 device_printf(dev, "%s: failed to open DPCON: id=%d, error=%d\n",
137 __func__, coninfo->id, error);
138 goto fail_con_open;
139 }
140
141 error = DPAA2_CMD_CON_ENABLE(dev, child, &cmd);
142 if (error) {
143 device_printf(dev, "%s: failed to enable channel: dpcon_id=%d, "
144 "chan_id=%d\n", __func__, coninfo->id, consc->attr.chan_id);
145 goto fail_con_enable;
146 }
147
148 ch = malloc(sizeof(struct dpaa2_channel), M_DPAA2_CH, M_WAITOK | M_ZERO);
149 if (ch == NULL) {
150 device_printf(dev, "%s: malloc() failed\n", __func__);
151 error = ENOMEM;
152 goto fail_malloc;
153 }
154
155 ch->ni_dev = dev;
156 ch->io_dev = iodev;
157 ch->con_dev = condev;
158 ch->id = consc->attr.chan_id;
159 ch->flowid = flowid;
160 ch->tx_frames = 0; /* for debug purposes */
161 ch->tx_dropped = 0; /* for debug purposes */
162 ch->store_sz = 0;
163 ch->store_idx = 0;
164 ch->recycled_n = 0;
165 ch->rxq_n = 0;
166
167 NET_TASK_INIT(&ch->cleanup_task, 0, cleanup_task_fn, ch);
168 NET_TASK_INIT(&ch->bp_task, 0, dpaa2_chan_bp_task, ch);
169
170 ch->cleanup_tq = taskqueue_create("dpaa2_ch cleanup", M_WAITOK,
171 taskqueue_thread_enqueue, &ch->cleanup_tq);
172 taskqueue_start_threads_cpuset(&ch->cleanup_tq, 1, PI_NET,
173 &iosc->cpu_mask, "dpaa2_ch%d cleanup", ch->id);
174
175 error = dpaa2_chan_setup_dma(dev, ch, sc->buf_align);
176 if (error != 0) {
177 device_printf(dev, "%s: failed to setup DMA\n", __func__);
178 goto fail_dma_setup;
179 }
180
181 mtx_init(&ch->xmit_mtx, "dpaa2_ch_xmit", NULL, MTX_DEF);
182
183 ch->xmit_br = buf_ring_alloc(DPAA2_TX_BUFRING_SZ, M_DEVBUF, M_NOWAIT,
184 &ch->xmit_mtx);
185 if (ch->xmit_br == NULL) {
186 device_printf(dev, "%s: buf_ring_alloc() failed\n", __func__);
187 error = ENOMEM;
188 goto fail_buf_ring;
189 }
190
191 DPAA2_BUF_INIT(&ch->store);
192
193 /* Register the new notification context */
194 ctx = &ch->ctx;
195 ctx->qman_ctx = (uint64_t)ctx;
196 ctx->cdan_en = true;
197 ctx->fq_chan_id = ch->id;
198 ctx->io_dev = ch->io_dev;
199 ctx->channel = ch;
200 error = DPAA2_SWP_CONF_WQ_CHANNEL(ch->io_dev, ctx);
201 if (error) {
202 device_printf(dev, "%s: failed to register CDAN context\n",
203 __func__);
204 goto fail_dpcon_notif;
205 }
206
207 /* Register DPCON notification within Management Complex */
208 notif_cfg.dpio_id = ioinfo->id;
209 notif_cfg.prior = 0;
210 notif_cfg.qman_ctx = ctx->qman_ctx;
211 error = DPAA2_CMD_CON_SET_NOTIF(dev, child, &cmd, ¬if_cfg);
212 if (error) {
213 device_printf(dev, "%s: failed to register DPCON "
214 "notifications: dpcon_id=%d, chan_id=%d\n", __func__,
215 coninfo->id, consc->attr.chan_id);
216 goto fail_dpcon_notif;
217 }
218
219 /* Allocate initial # of Rx buffers and a channel storage */
220 error = dpaa2_buf_seed_pool(dev, bpdev, ch, DPAA2_NI_BUFS_INIT,
221 DPAA2_RX_BUF_SIZE, NULL);
222 if (error) {
223 device_printf(dev, "%s: failed to seed buffer pool\n",
224 __func__);
225 goto fail_dpcon_notif;
226 }
227 error = dpaa2_chan_alloc_storage(dev, ch, DPAA2_ETH_STORE_SIZE,
228 BUS_DMA_NOWAIT, sc->buf_align);
229 if (error != 0) {
230 device_printf(dev, "%s: failed to allocate channel storage\n",
231 __func__);
232 goto fail_dpcon_notif;
233 } else {
234 ch->store_sz = DPAA2_ETH_STORE_FRAMES;
235 }
236
237 /* Prepare queues for the channel */
238 error = dpaa2_chan_setup_fq(dev, ch, DPAA2_NI_QUEUE_TX_CONF);
239 if (error) {
240 device_printf(dev, "%s: failed to prepare TxConf queue: "
241 "error=%d\n", __func__, error);
242 goto fail_fq_setup;
243 }
244 error = dpaa2_chan_setup_fq(dev, ch, DPAA2_NI_QUEUE_RX);
245 if (error) {
246 device_printf(dev, "%s: failed to prepare Rx queue: error=%d\n",
247 __func__, error);
248 goto fail_fq_setup;
249 }
250
251 if (bootverbose) {
252 device_printf(dev, "channel: dpio_id=%d dpcon_id=%d chan_id=%d, "
253 "priorities=%d\n", ioinfo->id, coninfo->id, ch->id,
254 consc->attr.prior_num);
255 }
256
257 *channel = ch;
258
259 (void)DPAA2_CMD_CON_CLOSE(dev, child, &cmd);
260 (void)DPAA2_CMD_RC_CLOSE(dev, child, DPAA2_CMD_TK(&cmd, rctk));
261
262 return (0);
263
264 fail_fq_setup:
265 if (ch->store.vaddr != NULL) {
266 bus_dmamem_free(ch->store.dmat, ch->store.vaddr, ch->store.dmap);
267 }
268 if (ch->store.dmat != NULL) {
269 bus_dma_tag_destroy(ch->store.dmat);
270 }
271 ch->store.dmat = NULL;
272 ch->store.vaddr = NULL;
273 ch->store.paddr = 0;
274 ch->store.nseg = 0;
275 fail_dpcon_notif:
276 buf_ring_free(ch->xmit_br, M_DEVBUF);
277 fail_buf_ring:
278 mtx_destroy(&ch->xmit_mtx);
279 fail_dma_setup:
280 /* while (taskqueue_cancel(ch->cleanup_tq, &ch->cleanup_task, NULL)) { */
281 /* taskqueue_drain(ch->cleanup_tq, &ch->cleanup_task); */
282 /* } */
283 /* taskqueue_free(ch->cleanup_tq); */
284 fail_malloc:
285 (void)DPAA2_CMD_CON_DISABLE(dev, child, DPAA2_CMD_TK(&cmd, contk));
286 fail_con_enable:
287 (void)DPAA2_CMD_CON_CLOSE(dev, child, DPAA2_CMD_TK(&cmd, contk));
288 fail_con_open:
289 (void)DPAA2_CMD_RC_CLOSE(dev, child, DPAA2_CMD_TK(&cmd, rctk));
290 fail_rc_open:
291 return (error);
292 }
293
294 /**
295 * @brief Performs an initial configuration of the frame queue.
296 */
297 int
dpaa2_chan_setup_fq(device_t dev,struct dpaa2_channel * ch,enum dpaa2_ni_queue_type queue_type)298 dpaa2_chan_setup_fq(device_t dev, struct dpaa2_channel *ch,
299 enum dpaa2_ni_queue_type queue_type)
300 {
301 struct dpaa2_ni_softc *sc = device_get_softc(dev);
302 struct dpaa2_ni_fq *fq;
303
304 switch (queue_type) {
305 case DPAA2_NI_QUEUE_TX_CONF:
306 /* One queue per channel */
307 fq = &ch->txc_queue;
308 fq->chan = ch;
309 fq->flowid = ch->flowid;
310 fq->tc = 0; /* ignored */
311 fq->type = queue_type;
312 break;
313 case DPAA2_NI_QUEUE_RX:
314 KASSERT(sc->attr.num.rx_tcs <= DPAA2_MAX_TCS,
315 ("too many Rx traffic classes: rx_tcs=%d\n",
316 sc->attr.num.rx_tcs));
317
318 /* One queue per Rx traffic class within a channel */
319 for (int i = 0; i < sc->attr.num.rx_tcs; i++) {
320 fq = &ch->rx_queues[i];
321 fq->chan = ch;
322 fq->flowid = ch->flowid;
323 fq->tc = (uint8_t) i;
324 fq->type = queue_type;
325
326 ch->rxq_n++;
327 }
328 break;
329 case DPAA2_NI_QUEUE_RX_ERR:
330 /* One queue per network interface */
331 fq = &sc->rxe_queue;
332 fq->chan = ch;
333 fq->flowid = 0; /* ignored */
334 fq->tc = 0; /* ignored */
335 fq->type = queue_type;
336 break;
337 default:
338 device_printf(dev, "%s: unexpected frame queue type: %d\n",
339 __func__, queue_type);
340 return (EINVAL);
341 }
342
343 return (0);
344 }
345
346 /**
347 * @brief Obtain the next dequeue response from the channel storage.
348 */
349 int
dpaa2_chan_next_frame(struct dpaa2_channel * ch,struct dpaa2_dq ** dq)350 dpaa2_chan_next_frame(struct dpaa2_channel *ch, struct dpaa2_dq **dq)
351 {
352 struct dpaa2_buf *buf = &ch->store;
353 struct dpaa2_dq *msgs = (struct dpaa2_dq *)buf->vaddr;
354 struct dpaa2_dq *msg = &msgs[ch->store_idx];
355 int rc = EINPROGRESS;
356
357 ch->store_idx++;
358
359 if (msg->fdr.desc.stat & DPAA2_DQ_STAT_EXPIRED) {
360 rc = EALREADY; /* VDQ command is expired */
361 ch->store_idx = 0;
362 if (!(msg->fdr.desc.stat & DPAA2_DQ_STAT_VALIDFRAME)) {
363 msg = NULL; /* Null response, FD is invalid */
364 }
365 }
366 if (msg != NULL && (msg->fdr.desc.stat & DPAA2_DQ_STAT_FQEMPTY)) {
367 rc = ENOENT; /* FQ is empty */
368 ch->store_idx = 0;
369 }
370
371 if (dq != NULL) {
372 *dq = msg;
373 }
374
375 return (rc);
376 }
377
378 static int
dpaa2_chan_setup_dma(device_t dev,struct dpaa2_channel * ch,bus_size_t alignment)379 dpaa2_chan_setup_dma(device_t dev, struct dpaa2_channel *ch,
380 bus_size_t alignment)
381 {
382 int error;
383
384 mtx_init(&ch->dma_mtx, "dpaa2_ch_dma_mtx", NULL, MTX_DEF);
385
386 error = bus_dma_tag_create(
387 bus_get_dma_tag(dev), /* parent */
388 alignment, 0, /* alignment, boundary */
389 BUS_SPACE_MAXADDR, /* low restricted addr */
390 BUS_SPACE_MAXADDR, /* high restricted addr */
391 NULL, NULL, /* filter, filterarg */
392 RX_SEG_MAXSZ, /* maxsize */
393 RX_SEG_N, /* nsegments */
394 RX_SEG_SZ, /* maxsegsize */
395 0, /* flags */
396 NULL, /* lockfunc */
397 NULL, /* lockarg */
398 &ch->rx_dmat);
399 if (error) {
400 device_printf(dev, "%s: failed to create rx_dmat\n", __func__);
401 goto fail_rx_tag;
402 }
403
404 error = bus_dma_tag_create(
405 bus_get_dma_tag(dev), /* parent */
406 alignment, 0, /* alignment, boundary */
407 BUS_SPACE_MAXADDR, /* low restricted addr */
408 BUS_SPACE_MAXADDR, /* high restricted addr */
409 NULL, NULL, /* filter, filterarg */
410 TX_SEG_MAXSZ, /* maxsize */
411 TX_SEG_N, /* nsegments */
412 TX_SEG_SZ, /* maxsegsize */
413 0, /* flags */
414 NULL, /* lockfunc */
415 NULL, /* lockarg */
416 &ch->tx_dmat);
417 if (error) {
418 device_printf(dev, "%s: failed to create tx_dmat\n", __func__);
419 goto fail_tx_tag;
420 }
421
422 error = bus_dma_tag_create(
423 bus_get_dma_tag(dev), /* parent */
424 alignment, 0, /* alignment, boundary */
425 BUS_SPACE_MAXADDR, /* low restricted addr */
426 BUS_SPACE_MAXADDR, /* high restricted addr */
427 NULL, NULL, /* filter, filterarg */
428 SGT_SEG_MAXSZ, /* maxsize */
429 SGT_SEG_N, /* nsegments */
430 SGT_SEG_SZ, /* maxsegsize */
431 0, /* flags */
432 NULL, /* lockfunc */
433 NULL, /* lockarg */
434 &ch->sgt_dmat);
435 if (error) {
436 device_printf(dev, "%s: failed to create sgt_dmat\n", __func__);
437 goto fail_sgt_tag;
438 }
439
440 return (0);
441
442 fail_sgt_tag:
443 bus_dma_tag_destroy(ch->tx_dmat);
444 fail_tx_tag:
445 bus_dma_tag_destroy(ch->rx_dmat);
446 fail_rx_tag:
447 mtx_destroy(&ch->dma_mtx);
448 ch->rx_dmat = NULL;
449 ch->tx_dmat = NULL;
450 ch->sgt_dmat = NULL;
451
452 return (error);
453 }
454
455 /**
456 * @brief Allocate a DMA-mapped storage to keep responses from VDQ command.
457 */
458 static int
dpaa2_chan_alloc_storage(device_t dev,struct dpaa2_channel * ch,bus_size_t size,int mapflags,bus_size_t alignment)459 dpaa2_chan_alloc_storage(device_t dev, struct dpaa2_channel *ch, bus_size_t size,
460 int mapflags, bus_size_t alignment)
461 {
462 struct dpaa2_buf *buf = &ch->store;
463 uint32_t maxsize = ((size - 1) / PAGE_SIZE + 1) * PAGE_SIZE;
464 int error;
465
466 error = bus_dma_tag_create(
467 bus_get_dma_tag(dev), /* parent */
468 alignment, 0, /* alignment, boundary */
469 BUS_SPACE_MAXADDR, /* low restricted addr */
470 BUS_SPACE_MAXADDR, /* high restricted addr */
471 NULL, NULL, /* filter, filterarg */
472 maxsize, /* maxsize */
473 1, /* nsegments */
474 maxsize, /* maxsegsize */
475 BUS_DMA_ALLOCNOW, /* flags */
476 NULL, /* lockfunc */
477 NULL, /* lockarg */
478 &buf->dmat);
479 if (error != 0) {
480 device_printf(dev, "%s: failed to create DMA tag\n", __func__);
481 goto fail_tag;
482 }
483
484 error = bus_dmamem_alloc(buf->dmat, (void **)&buf->vaddr,
485 BUS_DMA_ZERO | BUS_DMA_COHERENT, &buf->dmap);
486 if (error != 0) {
487 device_printf(dev, "%s: failed to allocate storage memory\n",
488 __func__);
489 goto fail_map_create;
490 }
491
492 buf->paddr = 0;
493 error = bus_dmamap_load(buf->dmat, buf->dmap, buf->vaddr, size,
494 dpaa2_dmamap_oneseg_cb, &buf->paddr, mapflags);
495 if (error != 0) {
496 device_printf(dev, "%s: failed to map storage memory\n",
497 __func__);
498 goto fail_map_load;
499 }
500
501 bus_dmamap_sync(buf->dmat, buf->dmap,
502 BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
503
504 buf->nseg = 1;
505
506 return (0);
507
508 fail_map_load:
509 bus_dmamem_free(buf->dmat, buf->vaddr, buf->dmap);
510 fail_map_create:
511 bus_dma_tag_destroy(buf->dmat);
512 fail_tag:
513 buf->dmat = NULL;
514 buf->vaddr = NULL;
515 buf->paddr = 0;
516 buf->nseg = 0;
517
518 return (error);
519 }
520
521 /**
522 * @brief Release new buffers to the buffer pool if necessary.
523 */
524 static void
dpaa2_chan_bp_task(void * arg,int count)525 dpaa2_chan_bp_task(void *arg, int count)
526 {
527 struct dpaa2_channel *ch = (struct dpaa2_channel *)arg;
528 struct dpaa2_ni_softc *sc = device_get_softc(ch->ni_dev);
529 struct dpaa2_bp_softc *bpsc;
530 struct dpaa2_bp_conf bpconf;
531 const int buf_num = DPAA2_ATOMIC_READ(&sc->buf_num);
532 device_t bpdev;
533 int error;
534
535 /* There's only one buffer pool for now */
536 bpdev = (device_t)rman_get_start(sc->res[DPAA2_NI_BP_RID(0)]);
537 bpsc = device_get_softc(bpdev);
538
539 /* Get state of the buffer pool */
540 error = DPAA2_SWP_QUERY_BP(ch->io_dev, bpsc->attr.bpid, &bpconf);
541 if (error) {
542 device_printf(sc->dev, "%s: DPAA2_SWP_QUERY_BP() failed: "
543 "error=%d\n", __func__, error);
544 return;
545 }
546
547 /* Double allocated Rx buffers if amount of free buffers is < 25% */
548 if (bpconf.free_bufn < (buf_num >> 2)) {
549 mtx_assert(&ch->dma_mtx, MA_NOTOWNED);
550 mtx_lock(&ch->dma_mtx);
551 (void)dpaa2_buf_seed_pool(ch->ni_dev, bpdev, ch, buf_num,
552 DPAA2_RX_BUF_SIZE, &ch->dma_mtx);
553 mtx_unlock(&ch->dma_mtx);
554
555 DPAA2_ATOMIC_XCHG(&sc->buf_free, bpconf.free_bufn);
556 }
557 }
558