1 /*-
2 * Broadcom NetXtreme-C/E network driver.
3 *
4 * Copyright (c) 2024 Broadcom, All Rights Reserved.
5 * The term Broadcom refers to Broadcom Limited and/or its subsidiaries
6 *
7 * Redistribution and use in source and binary forms, with or without
8 * modification, are permitted provided that the following conditions
9 * are met:
10 * 1. Redistributions of source code must retain the above copyright
11 * notice, this list of conditions and the following disclaimer.
12 * 2. Redistributions in binary form must reproduce the above copyright
13 * notice, this list of conditions and the following disclaimer in the
14 * documentation and/or other materials provided with the distribution.
15 *
16 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS'
17 * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
18 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
19 * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS
20 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
21 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
22 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
23 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
24 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
25 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF
26 * THE POSSIBILITY OF SUCH DAMAGE.
27 */
28
29 #include <linux/module.h>
30 #include <linux/kernel.h>
31 #include <linux/errno.h>
32 #include <linux/interrupt.h>
33 #include <linux/pci.h>
34 #include <linux/netdevice.h>
35 #include <linux/bitops.h>
36 #include <linux/delay.h>
37 #include <asm/byteorder.h>
38 #include <linux/bitmap.h>
39 #include <linux/rcupdate.h>
40 #include <net/if.h>
41 #include <net/if_dl.h>
42 #include <net/if_media.h>
43 #include <net/if_var.h>
44 #include <net/ethernet.h>
45 #include <net/iflib.h>
46
47 #include "hsi_struct_def.h"
48 #include "bnxt.h"
49 #include "bnxt_hwrm.h"
50 #include "bnxt_ulp.h"
51
52 void bnxt_destroy_irq(struct bnxt_softc *softc);
53
bnxt_register_dev(struct bnxt_en_dev * edev,int ulp_id,struct bnxt_ulp_ops * ulp_ops,void * handle)54 static int bnxt_register_dev(struct bnxt_en_dev *edev, int ulp_id,
55 struct bnxt_ulp_ops *ulp_ops, void *handle)
56 {
57 struct bnxt_softc *bp = edev->softc;
58 struct bnxt_ulp *ulp;
59 int rc = 0;
60
61 if (ulp_id >= BNXT_MAX_ULP)
62 return -EINVAL;
63
64 mtx_lock(&bp->en_ops_lock);
65 ulp = &edev->ulp_tbl[ulp_id];
66 if (rcu_access_pointer(ulp->ulp_ops)) {
67 device_printf(bp->dev, "ulp id %d already registered\n", ulp_id);
68 rc = -EBUSY;
69 goto exit;
70 }
71
72 edev->flags &= ~BNXT_EN_FLAG_ULP_STOPPED;
73 atomic_set(&ulp->ref_count, 0);
74 ulp->handle = handle;
75 rcu_assign_pointer(ulp->ulp_ops, ulp_ops);
76
77 if (ulp_id == BNXT_ROCE_ULP) {
78 if (test_bit(BNXT_STATE_OPEN, &bp->state) && bp->is_dev_init)
79 bnxt_hwrm_vnic_cfg(bp, &bp->vnic_info);
80 }
81
82 exit:
83 mtx_unlock(&bp->en_ops_lock);
84 return rc;
85 }
86
bnxt_unregister_dev(struct bnxt_en_dev * edev,int ulp_id)87 static int bnxt_unregister_dev(struct bnxt_en_dev *edev, int ulp_id)
88 {
89 struct bnxt_softc *bp = edev->softc;
90 struct bnxt_ulp *ulp;
91 int i = 0;
92
93 if (ulp_id >= BNXT_MAX_ULP)
94 return -EINVAL;
95
96 ulp = &edev->ulp_tbl[ulp_id];
97 if (!rcu_access_pointer(ulp->ulp_ops)) {
98 device_printf(bp->dev, "ulp id %d not registered\n", ulp_id);
99 return -EINVAL;
100 }
101 if (ulp_id == BNXT_ROCE_ULP && ulp->msix_requested)
102 edev->en_ops->bnxt_free_msix(edev, ulp_id);
103
104 mtx_lock(&bp->en_ops_lock);
105 RCU_INIT_POINTER(ulp->ulp_ops, NULL);
106 synchronize_rcu();
107 ulp->max_async_event_id = 0;
108 ulp->async_events_bmap = NULL;
109 while (atomic_read(&ulp->ref_count) != 0 && i < 10) {
110 msleep(100);
111 i++;
112 }
113 mtx_unlock(&bp->en_ops_lock);
114 return 0;
115 }
116
bnxt_fill_msix_vecs(struct bnxt_softc * bp,struct bnxt_msix_entry * ent)117 static void bnxt_fill_msix_vecs(struct bnxt_softc *bp, struct bnxt_msix_entry *ent)
118 {
119 struct bnxt_en_dev *edev = bp->edev;
120 int num_msix, idx, i;
121
122 num_msix = edev->ulp_tbl[BNXT_ROCE_ULP].msix_requested;
123 idx = edev->ulp_tbl[BNXT_ROCE_ULP].msix_base;
124 for (i = 0; i < num_msix; i++) {
125 ent[i].vector = bp->irq_tbl[idx + i].vector;
126 ent[i].ring_idx = idx + i;
127 if (BNXT_CHIP_P5(bp))
128 ent[i].db_offset = DB_PF_OFFSET_P5;
129 else
130 ent[i].db_offset = (idx + i) * 0x80;
131
132 }
133 }
134
bnxt_req_msix_vecs(struct bnxt_en_dev * edev,int ulp_id,struct bnxt_msix_entry * ent,int num_msix)135 static int bnxt_req_msix_vecs(struct bnxt_en_dev *edev, int ulp_id,
136 struct bnxt_msix_entry *ent, int num_msix)
137 {
138 struct bnxt_softc *bp = edev->softc;
139 int avail_msix, idx;
140
141 if (ulp_id != BNXT_ROCE_ULP)
142 return -EINVAL;
143
144 if (edev->ulp_tbl[ulp_id].msix_requested)
145 return -EAGAIN;
146
147 idx = bp->total_irqs - BNXT_ROCE_IRQ_COUNT;
148 avail_msix = BNXT_ROCE_IRQ_COUNT;
149
150 mtx_lock(&bp->en_ops_lock);
151 edev->ulp_tbl[ulp_id].msix_base = idx;
152 edev->ulp_tbl[ulp_id].msix_requested = avail_msix;
153
154 bnxt_fill_msix_vecs(bp, ent);
155 edev->flags |= BNXT_EN_FLAG_MSIX_REQUESTED;
156 mtx_unlock(&bp->en_ops_lock);
157 return avail_msix;
158 }
159
bnxt_free_msix_vecs(struct bnxt_en_dev * edev,int ulp_id)160 static int bnxt_free_msix_vecs(struct bnxt_en_dev *edev, int ulp_id)
161 {
162 struct bnxt_softc *bp = edev->softc;
163
164 if (ulp_id != BNXT_ROCE_ULP)
165 return -EINVAL;
166
167 if (!(edev->flags & BNXT_EN_FLAG_MSIX_REQUESTED))
168 return 0;
169
170 mtx_lock(&bp->en_ops_lock);
171 edev->ulp_tbl[ulp_id].msix_requested = 0;
172 edev->flags &= ~BNXT_EN_FLAG_MSIX_REQUESTED;
173 if (edev->flags & BNXT_EN_FLAG_ULP_STOPPED)
174 goto stopped;
175
176 stopped:
177 mtx_unlock(&bp->en_ops_lock);
178
179 return 0;
180 }
181
bnxt_get_ulp_msix_num(struct bnxt_softc * bp)182 int bnxt_get_ulp_msix_num(struct bnxt_softc *bp)
183 {
184 if (bnxt_ulp_registered(bp->edev, BNXT_ROCE_ULP)) {
185 struct bnxt_en_dev *edev = bp->edev;
186
187 return edev->ulp_tbl[BNXT_ROCE_ULP].msix_requested;
188 }
189 return 0;
190 }
191
bnxt_get_ulp_msix_base(struct bnxt_softc * bp)192 int bnxt_get_ulp_msix_base(struct bnxt_softc *bp)
193 {
194 if (bnxt_ulp_registered(bp->edev, BNXT_ROCE_ULP)) {
195 struct bnxt_en_dev *edev = bp->edev;
196
197 if (edev->ulp_tbl[BNXT_ROCE_ULP].msix_requested)
198 return edev->ulp_tbl[BNXT_ROCE_ULP].msix_base;
199 }
200 return 0;
201 }
202
bnxt_send_msg(struct bnxt_en_dev * edev,int ulp_id,struct bnxt_fw_msg * fw_msg)203 static int bnxt_send_msg(struct bnxt_en_dev *edev, int ulp_id,
204 struct bnxt_fw_msg *fw_msg)
205 {
206 struct bnxt_softc *softc = edev->softc;
207 int rc;
208
209 if ((ulp_id != BNXT_ROCE_ULP) && softc->fw_reset_state)
210 return -EBUSY;
211
212 rc = bnxt_hwrm_passthrough(softc, fw_msg->msg, fw_msg->msg_len, fw_msg->resp,
213 fw_msg->resp_max_len, fw_msg->timeout);
214 return rc;
215 }
216
bnxt_ulp_get(struct bnxt_ulp * ulp)217 static void bnxt_ulp_get(struct bnxt_ulp *ulp)
218 {
219 atomic_inc(&ulp->ref_count);
220 }
221
bnxt_ulp_put(struct bnxt_ulp * ulp)222 static void bnxt_ulp_put(struct bnxt_ulp *ulp)
223 {
224 atomic_dec(&ulp->ref_count);
225 }
226
bnxt_ulp_stop(struct bnxt_softc * bp)227 void bnxt_ulp_stop(struct bnxt_softc *bp)
228 {
229 struct bnxt_en_dev *edev = bp->edev;
230 struct bnxt_ulp_ops *ops;
231 int i;
232
233 if (!edev)
234 return;
235
236 edev->flags |= BNXT_EN_FLAG_ULP_STOPPED;
237 edev->en_state = bp->state;
238 for (i = 0; i < BNXT_MAX_ULP; i++) {
239 struct bnxt_ulp *ulp = &edev->ulp_tbl[i];
240
241 ops = ulp->ulp_ops;
242 if (!ops || !ops->ulp_stop)
243 continue;
244 ops->ulp_stop(ulp->handle);
245 }
246 }
247
bnxt_ulp_start(struct bnxt_softc * bp,int err)248 void bnxt_ulp_start(struct bnxt_softc *bp, int err)
249 {
250 struct bnxt_en_dev *edev = bp->edev;
251 struct bnxt_ulp_ops *ops;
252 int i;
253
254 if (!edev)
255 return;
256
257 edev->flags &= ~BNXT_EN_FLAG_ULP_STOPPED;
258 edev->en_state = bp->state;
259
260 if (err)
261 return;
262
263 for (i = 0; i < BNXT_MAX_ULP; i++) {
264 struct bnxt_ulp *ulp = &edev->ulp_tbl[i];
265
266 ops = ulp->ulp_ops;
267 if (!ops || !ops->ulp_start)
268 continue;
269 ops->ulp_start(ulp->handle);
270 }
271 }
272
bnxt_ulp_sriov_cfg(struct bnxt_softc * bp,int num_vfs)273 void bnxt_ulp_sriov_cfg(struct bnxt_softc *bp, int num_vfs)
274 {
275 struct bnxt_en_dev *edev = bp->edev;
276 struct bnxt_ulp_ops *ops;
277 int i;
278
279 if (!edev)
280 return;
281
282 for (i = 0; i < BNXT_MAX_ULP; i++) {
283 struct bnxt_ulp *ulp = &edev->ulp_tbl[i];
284
285 rcu_read_lock();
286 ops = rcu_dereference(ulp->ulp_ops);
287 if (!ops || !ops->ulp_sriov_config) {
288 rcu_read_unlock();
289 continue;
290 }
291 bnxt_ulp_get(ulp);
292 rcu_read_unlock();
293 ops->ulp_sriov_config(ulp->handle, num_vfs);
294 bnxt_ulp_put(ulp);
295 }
296 }
297
bnxt_ulp_shutdown(struct bnxt_softc * bp)298 void bnxt_ulp_shutdown(struct bnxt_softc *bp)
299 {
300 struct bnxt_en_dev *edev = bp->edev;
301 struct bnxt_ulp_ops *ops;
302 int i;
303
304 if (!edev)
305 return;
306
307 for (i = 0; i < BNXT_MAX_ULP; i++) {
308 struct bnxt_ulp *ulp = &edev->ulp_tbl[i];
309
310 ops = ulp->ulp_ops;
311 if (!ops || !ops->ulp_shutdown)
312 continue;
313 ops->ulp_shutdown(ulp->handle);
314 }
315 }
316
bnxt_ulp_irq_stop(struct bnxt_softc * bp)317 void bnxt_ulp_irq_stop(struct bnxt_softc *bp)
318 {
319 struct bnxt_en_dev *edev = bp->edev;
320 struct bnxt_ulp_ops *ops;
321
322 if (!edev || !(edev->flags & BNXT_EN_FLAG_MSIX_REQUESTED))
323 return;
324
325 if (bnxt_ulp_registered(bp->edev, BNXT_ROCE_ULP)) {
326 struct bnxt_ulp *ulp = &edev->ulp_tbl[BNXT_ROCE_ULP];
327
328 if (!ulp->msix_requested)
329 return;
330
331 ops = ulp->ulp_ops;
332 if (!ops || !ops->ulp_irq_stop)
333 return;
334 ops->ulp_irq_stop(ulp->handle);
335 }
336 }
337
bnxt_ulp_async_events(struct bnxt_softc * bp,struct hwrm_async_event_cmpl * cmpl)338 void bnxt_ulp_async_events(struct bnxt_softc *bp, struct hwrm_async_event_cmpl *cmpl)
339 {
340 u16 event_id = le16_to_cpu(cmpl->event_id);
341 struct bnxt_en_dev *edev = bp->edev;
342 struct bnxt_ulp_ops *ops;
343 int i;
344
345 if (!edev)
346 return;
347
348 rcu_read_lock();
349 for (i = 0; i < BNXT_MAX_ULP; i++) {
350 struct bnxt_ulp *ulp = &edev->ulp_tbl[i];
351
352 ops = rcu_dereference(ulp->ulp_ops);
353 if (!ops || !ops->ulp_async_notifier)
354 continue;
355 if (!ulp->async_events_bmap ||
356 event_id > ulp->max_async_event_id)
357 continue;
358
359 /* Read max_async_event_id first before testing the bitmap. */
360 rmb();
361 if (edev->flags & BNXT_EN_FLAG_ULP_STOPPED)
362 continue;
363
364 if (test_bit(event_id, ulp->async_events_bmap))
365 ops->ulp_async_notifier(ulp->handle, cmpl);
366 }
367 rcu_read_unlock();
368 }
369
bnxt_register_async_events(struct bnxt_en_dev * edev,int ulp_id,unsigned long * events_bmap,u16 max_id)370 static int bnxt_register_async_events(struct bnxt_en_dev *edev, int ulp_id,
371 unsigned long *events_bmap, u16 max_id)
372 {
373 struct bnxt_softc *bp = edev->softc;
374 struct bnxt_ulp *ulp;
375
376 if (ulp_id >= BNXT_MAX_ULP)
377 return -EINVAL;
378
379 mtx_lock(&bp->en_ops_lock);
380 ulp = &edev->ulp_tbl[ulp_id];
381 ulp->async_events_bmap = events_bmap;
382 wmb();
383 ulp->max_async_event_id = max_id;
384 bnxt_hwrm_func_drv_rgtr(bp, events_bmap, max_id + 1, true);
385 mtx_unlock(&bp->en_ops_lock);
386 return 0;
387 }
388
bnxt_destroy_irq(struct bnxt_softc * softc)389 void bnxt_destroy_irq(struct bnxt_softc *softc)
390 {
391 kfree(softc->irq_tbl);
392 }
393
bnxt_populate_irq(struct bnxt_softc * softc)394 static int bnxt_populate_irq(struct bnxt_softc *softc)
395 {
396 struct resource_list *rl = NULL;
397 struct resource_list_entry *rle = NULL;
398 struct bnxt_msix_tbl *irq_tbl = NULL;
399 struct pci_devinfo *dinfo = NULL;
400 int i;
401
402 softc->total_irqs = softc->scctx->isc_nrxqsets + BNXT_ROCE_IRQ_COUNT;
403 irq_tbl = kzalloc(softc->total_irqs * sizeof(*softc->irq_tbl), GFP_KERNEL);
404
405 if (!irq_tbl) {
406 device_printf(softc->dev, "Failed to allocate IRQ table\n");
407 return -1;
408 }
409 dinfo = device_get_ivars(softc->pdev->dev.bsddev);
410 rl = &dinfo->resources;
411 rle = resource_list_find(rl, SYS_RES_IRQ, 1);
412 softc->pdev->dev.irq_start = rle->start;
413 softc->pdev->dev.irq_end = rle->start + softc->total_irqs;
414
415 for (i = 0; i < softc->total_irqs; i++) {
416 irq_tbl[i].entry = i;
417 irq_tbl[i].vector = softc->pdev->dev.irq_start + i;
418 }
419
420 softc->irq_tbl = irq_tbl;
421
422 return 0;
423 }
424
425 static const struct bnxt_en_ops bnxt_en_ops_tbl = {
426 .bnxt_register_device = bnxt_register_dev,
427 .bnxt_unregister_device = bnxt_unregister_dev,
428 .bnxt_request_msix = bnxt_req_msix_vecs,
429 .bnxt_free_msix = bnxt_free_msix_vecs,
430 .bnxt_send_fw_msg = bnxt_send_msg,
431 .bnxt_register_fw_async_events = bnxt_register_async_events,
432 };
433
bnxt_aux_dev_release(struct device * dev)434 void bnxt_aux_dev_release(struct device *dev)
435 {
436 struct bnxt_aux_dev *bnxt_adev =
437 container_of(dev, struct bnxt_aux_dev, aux_dev.dev);
438 struct bnxt_softc *bp = bnxt_adev->edev->softc;
439
440 kfree(bnxt_adev->edev);
441 bnxt_adev->edev = NULL;
442 bp->edev = NULL;
443 }
444
bnxt_set_edev_info(struct bnxt_en_dev * edev,struct bnxt_softc * bp)445 static inline void bnxt_set_edev_info(struct bnxt_en_dev *edev, struct bnxt_softc *bp)
446 {
447 edev->en_ops = &bnxt_en_ops_tbl;
448 edev->net = bp->ifp;
449 edev->pdev = bp->pdev;
450 edev->softc = bp;
451 edev->l2_db_size = bp->db_size;
452 mtx_init(&bp->en_ops_lock, "Ethernet ops lock", NULL, MTX_DEF);
453
454 if (bp->flags & BNXT_FLAG_ROCEV1_CAP)
455 edev->flags |= BNXT_EN_FLAG_ROCEV1_CAP;
456 if (bp->flags & BNXT_FLAG_ROCEV2_CAP)
457 edev->flags |= BNXT_EN_FLAG_ROCEV2_CAP;
458 if (bp->is_asym_q)
459 edev->flags |= BNXT_EN_FLAG_ASYM_Q;
460 edev->hwrm_bar = bp->hwrm_bar;
461 edev->port_partition_type = bp->port_partition_type;
462 edev->ulp_version = BNXT_ULP_VERSION;
463 }
464
bnxt_rdma_aux_device_del(struct bnxt_softc * softc)465 int bnxt_rdma_aux_device_del(struct bnxt_softc *softc)
466 {
467 struct bnxt_aux_dev *bnxt_adev = softc->aux_dev;
468 struct auxiliary_device *adev;
469
470 adev = &bnxt_adev->aux_dev;
471 auxiliary_device_delete(adev);
472 auxiliary_device_uninit(adev);
473 bnxt_destroy_irq(softc);
474
475 return 0;
476 }
477
bnxt_rdma_aux_device_add(struct bnxt_softc * bp)478 int bnxt_rdma_aux_device_add(struct bnxt_softc *bp)
479 {
480 struct bnxt_aux_dev *bnxt_adev = bp->aux_dev;
481 struct bnxt_en_dev *edev = bnxt_adev->edev;
482 struct auxiliary_device *aux_dev;
483 int ret = -1;
484
485 if (bnxt_populate_irq(bp))
486 return ret;
487
488 device_printf(bp->dev, "V:D:SV:SD %x:%x:%x:%x, irq 0x%x, "
489 "devfn 0x%x, cla 0x%x, rev 0x%x, msi_en 0x%x\n",
490 bp->pdev->vendor, bp->pdev->device, bp->pdev->subsystem_vendor,
491 bp->pdev->subsystem_device, bp->pdev->irq, bp->pdev->devfn,
492 bp->pdev->class, bp->pdev->revision, bp->pdev->msi_enabled);
493
494 aux_dev = &bnxt_adev->aux_dev;
495 aux_dev->id = bnxt_adev->id;
496 aux_dev->name = "rdma";
497 aux_dev->dev.parent = &bp->pdev->dev;
498 aux_dev->dev.release = bnxt_aux_dev_release;
499
500 if (!edev) {
501 edev = kzalloc(sizeof(*edev), GFP_KERNEL);
502 if (!edev)
503 return -ENOMEM;
504 }
505
506 bnxt_set_edev_info(edev, bp);
507 bnxt_adev->edev = edev;
508 bp->edev = edev;
509
510 ret = auxiliary_device_init(aux_dev);
511 if (ret)
512 goto err_free_edev;
513
514 ret = auxiliary_device_add(aux_dev);
515 if (ret)
516 goto err_dev_uninit;
517
518 return 0;
519 err_dev_uninit:
520 auxiliary_device_uninit(aux_dev);
521 err_free_edev:
522 kfree(edev);
523 bnxt_adev->edev = NULL;
524 bp->edev = NULL;
525 return ret;
526 }
527