1 // SPDX-License-Identifier: GPL-2.0
2 /* Copyright(c) 2017 - 2019 Pensando Systems, Inc */
3
4 #include <linux/ethtool.h>
5 #include <linux/printk.h>
6 #include <linux/dynamic_debug.h>
7 #include <linux/netdevice.h>
8 #include <linux/etherdevice.h>
9 #include <linux/if_vlan.h>
10 #include <linux/rtnetlink.h>
11 #include <linux/interrupt.h>
12 #include <linux/pci.h>
13 #include <linux/cpumask.h>
14
15 #include "ionic.h"
16 #include "ionic_bus.h"
17 #include "ionic_lif.h"
18 #include "ionic_txrx.h"
19 #include "ionic_ethtool.h"
20 #include "ionic_debugfs.h"
21
22 /* queuetype support level */
23 static const u8 ionic_qtype_versions[IONIC_QTYPE_MAX] = {
24 [IONIC_QTYPE_ADMINQ] = 0, /* 0 = Base version with CQ support */
25 [IONIC_QTYPE_NOTIFYQ] = 0, /* 0 = Base version */
26 [IONIC_QTYPE_RXQ] = 0, /* 0 = Base version with CQ+SG support */
27 [IONIC_QTYPE_TXQ] = 1, /* 0 = Base version with CQ+SG support
28 * 1 = ... with Tx SG version 1
29 */
30 };
31
32 static void ionic_lif_rx_mode(struct ionic_lif *lif, unsigned int rx_mode);
33 static int ionic_lif_addr_add(struct ionic_lif *lif, const u8 *addr);
34 static int ionic_lif_addr_del(struct ionic_lif *lif, const u8 *addr);
35 static void ionic_link_status_check(struct ionic_lif *lif);
36 static void ionic_lif_handle_fw_down(struct ionic_lif *lif);
37 static void ionic_lif_handle_fw_up(struct ionic_lif *lif);
38 static void ionic_lif_set_netdev_info(struct ionic_lif *lif);
39
40 static void ionic_txrx_deinit(struct ionic_lif *lif);
41 static int ionic_txrx_init(struct ionic_lif *lif);
42 static int ionic_start_queues(struct ionic_lif *lif);
43 static void ionic_stop_queues(struct ionic_lif *lif);
44 static void ionic_lif_queue_identify(struct ionic_lif *lif);
45
ionic_dim_work(struct work_struct * work)46 static void ionic_dim_work(struct work_struct *work)
47 {
48 struct dim *dim = container_of(work, struct dim, work);
49 struct dim_cq_moder cur_moder;
50 struct ionic_qcq *qcq;
51 u32 new_coal;
52
53 cur_moder = net_dim_get_rx_moderation(dim->mode, dim->profile_ix);
54 qcq = container_of(dim, struct ionic_qcq, dim);
55 new_coal = ionic_coal_usec_to_hw(qcq->q.lif->ionic, cur_moder.usec);
56 qcq->intr.dim_coal_hw = new_coal ? new_coal : 1;
57 dim->state = DIM_START_MEASURE;
58 }
59
ionic_lif_deferred_work(struct work_struct * work)60 static void ionic_lif_deferred_work(struct work_struct *work)
61 {
62 struct ionic_lif *lif = container_of(work, struct ionic_lif, deferred.work);
63 struct ionic_deferred *def = &lif->deferred;
64 struct ionic_deferred_work *w = NULL;
65
66 do {
67 spin_lock_bh(&def->lock);
68 if (!list_empty(&def->list)) {
69 w = list_first_entry(&def->list,
70 struct ionic_deferred_work, list);
71 list_del(&w->list);
72 }
73 spin_unlock_bh(&def->lock);
74
75 if (!w)
76 break;
77
78 switch (w->type) {
79 case IONIC_DW_TYPE_RX_MODE:
80 ionic_lif_rx_mode(lif, w->rx_mode);
81 break;
82 case IONIC_DW_TYPE_RX_ADDR_ADD:
83 ionic_lif_addr_add(lif, w->addr);
84 break;
85 case IONIC_DW_TYPE_RX_ADDR_DEL:
86 ionic_lif_addr_del(lif, w->addr);
87 break;
88 case IONIC_DW_TYPE_LINK_STATUS:
89 ionic_link_status_check(lif);
90 break;
91 case IONIC_DW_TYPE_LIF_RESET:
92 if (w->fw_status)
93 ionic_lif_handle_fw_up(lif);
94 else
95 ionic_lif_handle_fw_down(lif);
96 break;
97 default:
98 break;
99 }
100 kfree(w);
101 w = NULL;
102 } while (true);
103 }
104
ionic_lif_deferred_enqueue(struct ionic_deferred * def,struct ionic_deferred_work * work)105 void ionic_lif_deferred_enqueue(struct ionic_deferred *def,
106 struct ionic_deferred_work *work)
107 {
108 spin_lock_bh(&def->lock);
109 list_add_tail(&work->list, &def->list);
110 spin_unlock_bh(&def->lock);
111 schedule_work(&def->work);
112 }
113
ionic_link_status_check(struct ionic_lif * lif)114 static void ionic_link_status_check(struct ionic_lif *lif)
115 {
116 struct net_device *netdev = lif->netdev;
117 u16 link_status;
118 bool link_up;
119
120 if (!test_bit(IONIC_LIF_F_LINK_CHECK_REQUESTED, lif->state))
121 return;
122
123 /* Don't put carrier back up if we're in a broken state */
124 if (test_bit(IONIC_LIF_F_BROKEN, lif->state)) {
125 clear_bit(IONIC_LIF_F_LINK_CHECK_REQUESTED, lif->state);
126 return;
127 }
128
129 link_status = le16_to_cpu(lif->info->status.link_status);
130 link_up = link_status == IONIC_PORT_OPER_STATUS_UP;
131
132 if (link_up) {
133 int err = 0;
134
135 if (netdev->flags & IFF_UP && netif_running(netdev)) {
136 mutex_lock(&lif->queue_lock);
137 err = ionic_start_queues(lif);
138 if (err && err != -EBUSY) {
139 netdev_err(lif->netdev,
140 "Failed to start queues: %d\n", err);
141 set_bit(IONIC_LIF_F_BROKEN, lif->state);
142 netif_carrier_off(lif->netdev);
143 }
144 mutex_unlock(&lif->queue_lock);
145 }
146
147 if (!err && !netif_carrier_ok(netdev)) {
148 ionic_port_identify(lif->ionic);
149 netdev_info(netdev, "Link up - %d Gbps\n",
150 le32_to_cpu(lif->info->status.link_speed) / 1000);
151 netif_carrier_on(netdev);
152 }
153 } else {
154 if (netif_carrier_ok(netdev)) {
155 netdev_info(netdev, "Link down\n");
156 netif_carrier_off(netdev);
157 }
158
159 if (netdev->flags & IFF_UP && netif_running(netdev)) {
160 mutex_lock(&lif->queue_lock);
161 ionic_stop_queues(lif);
162 mutex_unlock(&lif->queue_lock);
163 }
164 }
165
166 clear_bit(IONIC_LIF_F_LINK_CHECK_REQUESTED, lif->state);
167 }
168
ionic_link_status_check_request(struct ionic_lif * lif,bool can_sleep)169 void ionic_link_status_check_request(struct ionic_lif *lif, bool can_sleep)
170 {
171 struct ionic_deferred_work *work;
172
173 /* we only need one request outstanding at a time */
174 if (test_and_set_bit(IONIC_LIF_F_LINK_CHECK_REQUESTED, lif->state))
175 return;
176
177 if (!can_sleep) {
178 work = kzalloc(sizeof(*work), GFP_ATOMIC);
179 if (!work) {
180 clear_bit(IONIC_LIF_F_LINK_CHECK_REQUESTED, lif->state);
181 return;
182 }
183
184 work->type = IONIC_DW_TYPE_LINK_STATUS;
185 ionic_lif_deferred_enqueue(&lif->deferred, work);
186 } else {
187 ionic_link_status_check(lif);
188 }
189 }
190
ionic_isr(int irq,void * data)191 static irqreturn_t ionic_isr(int irq, void *data)
192 {
193 struct napi_struct *napi = data;
194
195 napi_schedule_irqoff(napi);
196
197 return IRQ_HANDLED;
198 }
199
ionic_request_irq(struct ionic_lif * lif,struct ionic_qcq * qcq)200 static int ionic_request_irq(struct ionic_lif *lif, struct ionic_qcq *qcq)
201 {
202 struct ionic_intr_info *intr = &qcq->intr;
203 struct device *dev = lif->ionic->dev;
204 struct ionic_queue *q = &qcq->q;
205 const char *name;
206
207 if (lif->registered)
208 name = lif->netdev->name;
209 else
210 name = dev_name(dev);
211
212 snprintf(intr->name, sizeof(intr->name),
213 "%s-%s-%s", IONIC_DRV_NAME, name, q->name);
214
215 return devm_request_irq(dev, intr->vector, ionic_isr,
216 0, intr->name, &qcq->napi);
217 }
218
ionic_intr_alloc(struct ionic_lif * lif,struct ionic_intr_info * intr)219 static int ionic_intr_alloc(struct ionic_lif *lif, struct ionic_intr_info *intr)
220 {
221 struct ionic *ionic = lif->ionic;
222 int index;
223
224 index = find_first_zero_bit(ionic->intrs, ionic->nintrs);
225 if (index == ionic->nintrs) {
226 netdev_warn(lif->netdev, "%s: no intr, index=%d nintrs=%d\n",
227 __func__, index, ionic->nintrs);
228 return -ENOSPC;
229 }
230
231 set_bit(index, ionic->intrs);
232 ionic_intr_init(&ionic->idev, intr, index);
233
234 return 0;
235 }
236
ionic_intr_free(struct ionic * ionic,int index)237 static void ionic_intr_free(struct ionic *ionic, int index)
238 {
239 if (index != IONIC_INTR_INDEX_NOT_ASSIGNED && index < ionic->nintrs)
240 clear_bit(index, ionic->intrs);
241 }
242
ionic_qcq_enable(struct ionic_qcq * qcq)243 static int ionic_qcq_enable(struct ionic_qcq *qcq)
244 {
245 struct ionic_queue *q = &qcq->q;
246 struct ionic_lif *lif = q->lif;
247 struct ionic_dev *idev;
248 struct device *dev;
249
250 struct ionic_admin_ctx ctx = {
251 .work = COMPLETION_INITIALIZER_ONSTACK(ctx.work),
252 .cmd.q_control = {
253 .opcode = IONIC_CMD_Q_CONTROL,
254 .lif_index = cpu_to_le16(lif->index),
255 .type = q->type,
256 .index = cpu_to_le32(q->index),
257 .oper = IONIC_Q_ENABLE,
258 },
259 };
260
261 idev = &lif->ionic->idev;
262 dev = lif->ionic->dev;
263
264 dev_dbg(dev, "q_enable.index %d q_enable.qtype %d\n",
265 ctx.cmd.q_control.index, ctx.cmd.q_control.type);
266
267 if (qcq->flags & IONIC_QCQ_F_INTR) {
268 irq_set_affinity_hint(qcq->intr.vector,
269 &qcq->intr.affinity_mask);
270 napi_enable(&qcq->napi);
271 ionic_intr_clean(idev->intr_ctrl, qcq->intr.index);
272 ionic_intr_mask(idev->intr_ctrl, qcq->intr.index,
273 IONIC_INTR_MASK_CLEAR);
274 }
275
276 return ionic_adminq_post_wait(lif, &ctx);
277 }
278
ionic_qcq_disable(struct ionic_qcq * qcq,bool send_to_hw)279 static int ionic_qcq_disable(struct ionic_qcq *qcq, bool send_to_hw)
280 {
281 struct ionic_queue *q;
282 struct ionic_lif *lif;
283 int err = 0;
284
285 struct ionic_admin_ctx ctx = {
286 .work = COMPLETION_INITIALIZER_ONSTACK(ctx.work),
287 .cmd.q_control = {
288 .opcode = IONIC_CMD_Q_CONTROL,
289 .oper = IONIC_Q_DISABLE,
290 },
291 };
292
293 if (!qcq)
294 return -ENXIO;
295
296 q = &qcq->q;
297 lif = q->lif;
298
299 if (qcq->flags & IONIC_QCQ_F_INTR) {
300 struct ionic_dev *idev = &lif->ionic->idev;
301
302 cancel_work_sync(&qcq->dim.work);
303 ionic_intr_mask(idev->intr_ctrl, qcq->intr.index,
304 IONIC_INTR_MASK_SET);
305 synchronize_irq(qcq->intr.vector);
306 irq_set_affinity_hint(qcq->intr.vector, NULL);
307 napi_disable(&qcq->napi);
308 }
309
310 if (send_to_hw) {
311 ctx.cmd.q_control.lif_index = cpu_to_le16(lif->index);
312 ctx.cmd.q_control.type = q->type;
313 ctx.cmd.q_control.index = cpu_to_le32(q->index);
314 dev_dbg(lif->ionic->dev, "q_disable.index %d q_disable.qtype %d\n",
315 ctx.cmd.q_control.index, ctx.cmd.q_control.type);
316
317 err = ionic_adminq_post_wait(lif, &ctx);
318 }
319
320 return err;
321 }
322
ionic_lif_qcq_deinit(struct ionic_lif * lif,struct ionic_qcq * qcq)323 static void ionic_lif_qcq_deinit(struct ionic_lif *lif, struct ionic_qcq *qcq)
324 {
325 struct ionic_dev *idev = &lif->ionic->idev;
326
327 if (!qcq)
328 return;
329
330 if (!(qcq->flags & IONIC_QCQ_F_INITED))
331 return;
332
333 if (qcq->flags & IONIC_QCQ_F_INTR) {
334 ionic_intr_mask(idev->intr_ctrl, qcq->intr.index,
335 IONIC_INTR_MASK_SET);
336 netif_napi_del(&qcq->napi);
337 }
338
339 qcq->flags &= ~IONIC_QCQ_F_INITED;
340 }
341
ionic_qcq_intr_free(struct ionic_lif * lif,struct ionic_qcq * qcq)342 static void ionic_qcq_intr_free(struct ionic_lif *lif, struct ionic_qcq *qcq)
343 {
344 if (!(qcq->flags & IONIC_QCQ_F_INTR) || qcq->intr.vector == 0)
345 return;
346
347 irq_set_affinity_hint(qcq->intr.vector, NULL);
348 devm_free_irq(lif->ionic->dev, qcq->intr.vector, &qcq->napi);
349 qcq->intr.vector = 0;
350 ionic_intr_free(lif->ionic, qcq->intr.index);
351 qcq->intr.index = IONIC_INTR_INDEX_NOT_ASSIGNED;
352 }
353
ionic_qcq_free(struct ionic_lif * lif,struct ionic_qcq * qcq)354 static void ionic_qcq_free(struct ionic_lif *lif, struct ionic_qcq *qcq)
355 {
356 struct device *dev = lif->ionic->dev;
357
358 if (!qcq)
359 return;
360
361 ionic_debugfs_del_qcq(qcq);
362
363 if (qcq->q_base) {
364 dma_free_coherent(dev, qcq->q_size, qcq->q_base, qcq->q_base_pa);
365 qcq->q_base = NULL;
366 qcq->q_base_pa = 0;
367 }
368
369 if (qcq->cq_base) {
370 dma_free_coherent(dev, qcq->cq_size, qcq->cq_base, qcq->cq_base_pa);
371 qcq->cq_base = NULL;
372 qcq->cq_base_pa = 0;
373 }
374
375 if (qcq->sg_base) {
376 dma_free_coherent(dev, qcq->sg_size, qcq->sg_base, qcq->sg_base_pa);
377 qcq->sg_base = NULL;
378 qcq->sg_base_pa = 0;
379 }
380
381 ionic_qcq_intr_free(lif, qcq);
382
383 if (qcq->cq.info) {
384 devm_kfree(dev, qcq->cq.info);
385 qcq->cq.info = NULL;
386 }
387 if (qcq->q.info) {
388 devm_kfree(dev, qcq->q.info);
389 qcq->q.info = NULL;
390 }
391 }
392
ionic_qcqs_free(struct ionic_lif * lif)393 static void ionic_qcqs_free(struct ionic_lif *lif)
394 {
395 struct device *dev = lif->ionic->dev;
396 struct ionic_qcq *adminqcq;
397 unsigned long irqflags;
398
399 if (lif->notifyqcq) {
400 ionic_qcq_free(lif, lif->notifyqcq);
401 devm_kfree(dev, lif->notifyqcq);
402 lif->notifyqcq = NULL;
403 }
404
405 if (lif->adminqcq) {
406 spin_lock_irqsave(&lif->adminq_lock, irqflags);
407 adminqcq = READ_ONCE(lif->adminqcq);
408 lif->adminqcq = NULL;
409 spin_unlock_irqrestore(&lif->adminq_lock, irqflags);
410 if (adminqcq) {
411 ionic_qcq_free(lif, adminqcq);
412 devm_kfree(dev, adminqcq);
413 }
414 }
415
416 if (lif->rxqcqs) {
417 devm_kfree(dev, lif->rxqstats);
418 lif->rxqstats = NULL;
419 devm_kfree(dev, lif->rxqcqs);
420 lif->rxqcqs = NULL;
421 }
422
423 if (lif->txqcqs) {
424 devm_kfree(dev, lif->txqstats);
425 lif->txqstats = NULL;
426 devm_kfree(dev, lif->txqcqs);
427 lif->txqcqs = NULL;
428 }
429 }
430
ionic_link_qcq_interrupts(struct ionic_qcq * src_qcq,struct ionic_qcq * n_qcq)431 static void ionic_link_qcq_interrupts(struct ionic_qcq *src_qcq,
432 struct ionic_qcq *n_qcq)
433 {
434 if (WARN_ON(n_qcq->flags & IONIC_QCQ_F_INTR)) {
435 ionic_intr_free(n_qcq->cq.lif->ionic, n_qcq->intr.index);
436 n_qcq->flags &= ~IONIC_QCQ_F_INTR;
437 }
438
439 n_qcq->intr.vector = src_qcq->intr.vector;
440 n_qcq->intr.index = src_qcq->intr.index;
441 }
442
ionic_alloc_qcq_interrupt(struct ionic_lif * lif,struct ionic_qcq * qcq)443 static int ionic_alloc_qcq_interrupt(struct ionic_lif *lif, struct ionic_qcq *qcq)
444 {
445 int err;
446
447 if (!(qcq->flags & IONIC_QCQ_F_INTR)) {
448 qcq->intr.index = IONIC_INTR_INDEX_NOT_ASSIGNED;
449 return 0;
450 }
451
452 err = ionic_intr_alloc(lif, &qcq->intr);
453 if (err) {
454 netdev_warn(lif->netdev, "no intr for %s: %d\n",
455 qcq->q.name, err);
456 goto err_out;
457 }
458
459 err = ionic_bus_get_irq(lif->ionic, qcq->intr.index);
460 if (err < 0) {
461 netdev_warn(lif->netdev, "no vector for %s: %d\n",
462 qcq->q.name, err);
463 goto err_out_free_intr;
464 }
465 qcq->intr.vector = err;
466 ionic_intr_mask_assert(lif->ionic->idev.intr_ctrl, qcq->intr.index,
467 IONIC_INTR_MASK_SET);
468
469 err = ionic_request_irq(lif, qcq);
470 if (err) {
471 netdev_warn(lif->netdev, "irq request failed %d\n", err);
472 goto err_out_free_intr;
473 }
474
475 /* try to get the irq on the local numa node first */
476 qcq->intr.cpu = cpumask_local_spread(qcq->intr.index,
477 dev_to_node(lif->ionic->dev));
478 if (qcq->intr.cpu != -1)
479 cpumask_set_cpu(qcq->intr.cpu, &qcq->intr.affinity_mask);
480
481 netdev_dbg(lif->netdev, "%s: Interrupt index %d\n", qcq->q.name, qcq->intr.index);
482 return 0;
483
484 err_out_free_intr:
485 ionic_intr_free(lif->ionic, qcq->intr.index);
486 err_out:
487 return err;
488 }
489
ionic_qcq_alloc(struct ionic_lif * lif,unsigned int type,unsigned int index,const char * name,unsigned int flags,unsigned int num_descs,unsigned int desc_size,unsigned int cq_desc_size,unsigned int sg_desc_size,unsigned int pid,struct ionic_qcq ** qcq)490 static int ionic_qcq_alloc(struct ionic_lif *lif, unsigned int type,
491 unsigned int index,
492 const char *name, unsigned int flags,
493 unsigned int num_descs, unsigned int desc_size,
494 unsigned int cq_desc_size,
495 unsigned int sg_desc_size,
496 unsigned int pid, struct ionic_qcq **qcq)
497 {
498 struct ionic_dev *idev = &lif->ionic->idev;
499 struct device *dev = lif->ionic->dev;
500 void *q_base, *cq_base, *sg_base;
501 dma_addr_t cq_base_pa = 0;
502 dma_addr_t sg_base_pa = 0;
503 dma_addr_t q_base_pa = 0;
504 struct ionic_qcq *new;
505 int err;
506
507 *qcq = NULL;
508
509 new = devm_kzalloc(dev, sizeof(*new), GFP_KERNEL);
510 if (!new) {
511 netdev_err(lif->netdev, "Cannot allocate queue structure\n");
512 err = -ENOMEM;
513 goto err_out;
514 }
515
516 new->q.dev = dev;
517 new->flags = flags;
518
519 new->q.info = devm_kcalloc(dev, num_descs, sizeof(*new->q.info),
520 GFP_KERNEL);
521 if (!new->q.info) {
522 netdev_err(lif->netdev, "Cannot allocate queue info\n");
523 err = -ENOMEM;
524 goto err_out_free_qcq;
525 }
526
527 new->q.type = type;
528 new->q.max_sg_elems = lif->qtype_info[type].max_sg_elems;
529
530 err = ionic_q_init(lif, idev, &new->q, index, name, num_descs,
531 desc_size, sg_desc_size, pid);
532 if (err) {
533 netdev_err(lif->netdev, "Cannot initialize queue\n");
534 goto err_out_free_q_info;
535 }
536
537 err = ionic_alloc_qcq_interrupt(lif, new);
538 if (err)
539 goto err_out;
540
541 new->cq.info = devm_kcalloc(dev, num_descs, sizeof(*new->cq.info),
542 GFP_KERNEL);
543 if (!new->cq.info) {
544 netdev_err(lif->netdev, "Cannot allocate completion queue info\n");
545 err = -ENOMEM;
546 goto err_out_free_irq;
547 }
548
549 err = ionic_cq_init(lif, &new->cq, &new->intr, num_descs, cq_desc_size);
550 if (err) {
551 netdev_err(lif->netdev, "Cannot initialize completion queue\n");
552 goto err_out_free_cq_info;
553 }
554
555 if (flags & IONIC_QCQ_F_NOTIFYQ) {
556 int q_size, cq_size;
557
558 /* q & cq need to be contiguous in case of notifyq */
559 q_size = ALIGN(num_descs * desc_size, PAGE_SIZE);
560 cq_size = ALIGN(num_descs * cq_desc_size, PAGE_SIZE);
561
562 new->q_size = PAGE_SIZE + q_size + cq_size;
563 new->q_base = dma_alloc_coherent(dev, new->q_size,
564 &new->q_base_pa, GFP_KERNEL);
565 if (!new->q_base) {
566 netdev_err(lif->netdev, "Cannot allocate qcq DMA memory\n");
567 err = -ENOMEM;
568 goto err_out_free_cq_info;
569 }
570 q_base = PTR_ALIGN(new->q_base, PAGE_SIZE);
571 q_base_pa = ALIGN(new->q_base_pa, PAGE_SIZE);
572 ionic_q_map(&new->q, q_base, q_base_pa);
573
574 cq_base = PTR_ALIGN(q_base + q_size, PAGE_SIZE);
575 cq_base_pa = ALIGN(new->q_base_pa + q_size, PAGE_SIZE);
576 ionic_cq_map(&new->cq, cq_base, cq_base_pa);
577 ionic_cq_bind(&new->cq, &new->q);
578 } else {
579 new->q_size = PAGE_SIZE + (num_descs * desc_size);
580 new->q_base = dma_alloc_coherent(dev, new->q_size, &new->q_base_pa,
581 GFP_KERNEL);
582 if (!new->q_base) {
583 netdev_err(lif->netdev, "Cannot allocate queue DMA memory\n");
584 err = -ENOMEM;
585 goto err_out_free_cq_info;
586 }
587 q_base = PTR_ALIGN(new->q_base, PAGE_SIZE);
588 q_base_pa = ALIGN(new->q_base_pa, PAGE_SIZE);
589 ionic_q_map(&new->q, q_base, q_base_pa);
590
591 new->cq_size = PAGE_SIZE + (num_descs * cq_desc_size);
592 new->cq_base = dma_alloc_coherent(dev, new->cq_size, &new->cq_base_pa,
593 GFP_KERNEL);
594 if (!new->cq_base) {
595 netdev_err(lif->netdev, "Cannot allocate cq DMA memory\n");
596 err = -ENOMEM;
597 goto err_out_free_q;
598 }
599 cq_base = PTR_ALIGN(new->cq_base, PAGE_SIZE);
600 cq_base_pa = ALIGN(new->cq_base_pa, PAGE_SIZE);
601 ionic_cq_map(&new->cq, cq_base, cq_base_pa);
602 ionic_cq_bind(&new->cq, &new->q);
603 }
604
605 if (flags & IONIC_QCQ_F_SG) {
606 new->sg_size = PAGE_SIZE + (num_descs * sg_desc_size);
607 new->sg_base = dma_alloc_coherent(dev, new->sg_size, &new->sg_base_pa,
608 GFP_KERNEL);
609 if (!new->sg_base) {
610 netdev_err(lif->netdev, "Cannot allocate sg DMA memory\n");
611 err = -ENOMEM;
612 goto err_out_free_cq;
613 }
614 sg_base = PTR_ALIGN(new->sg_base, PAGE_SIZE);
615 sg_base_pa = ALIGN(new->sg_base_pa, PAGE_SIZE);
616 ionic_q_sg_map(&new->q, sg_base, sg_base_pa);
617 }
618
619 INIT_WORK(&new->dim.work, ionic_dim_work);
620 new->dim.mode = DIM_CQ_PERIOD_MODE_START_FROM_EQE;
621
622 *qcq = new;
623
624 return 0;
625
626 err_out_free_cq:
627 dma_free_coherent(dev, new->cq_size, new->cq_base, new->cq_base_pa);
628 err_out_free_q:
629 dma_free_coherent(dev, new->q_size, new->q_base, new->q_base_pa);
630 err_out_free_cq_info:
631 devm_kfree(dev, new->cq.info);
632 err_out_free_irq:
633 if (flags & IONIC_QCQ_F_INTR) {
634 devm_free_irq(dev, new->intr.vector, &new->napi);
635 ionic_intr_free(lif->ionic, new->intr.index);
636 }
637 err_out_free_q_info:
638 devm_kfree(dev, new->q.info);
639 err_out_free_qcq:
640 devm_kfree(dev, new);
641 err_out:
642 dev_err(dev, "qcq alloc of %s%d failed %d\n", name, index, err);
643 return err;
644 }
645
ionic_qcqs_alloc(struct ionic_lif * lif)646 static int ionic_qcqs_alloc(struct ionic_lif *lif)
647 {
648 struct device *dev = lif->ionic->dev;
649 unsigned int flags;
650 int err;
651
652 flags = IONIC_QCQ_F_INTR;
653 err = ionic_qcq_alloc(lif, IONIC_QTYPE_ADMINQ, 0, "admin", flags,
654 IONIC_ADMINQ_LENGTH,
655 sizeof(struct ionic_admin_cmd),
656 sizeof(struct ionic_admin_comp),
657 0, lif->kern_pid, &lif->adminqcq);
658 if (err)
659 return err;
660 ionic_debugfs_add_qcq(lif, lif->adminqcq);
661
662 if (lif->ionic->nnqs_per_lif) {
663 flags = IONIC_QCQ_F_NOTIFYQ;
664 err = ionic_qcq_alloc(lif, IONIC_QTYPE_NOTIFYQ, 0, "notifyq",
665 flags, IONIC_NOTIFYQ_LENGTH,
666 sizeof(struct ionic_notifyq_cmd),
667 sizeof(union ionic_notifyq_comp),
668 0, lif->kern_pid, &lif->notifyqcq);
669 if (err)
670 goto err_out;
671 ionic_debugfs_add_qcq(lif, lif->notifyqcq);
672
673 /* Let the notifyq ride on the adminq interrupt */
674 ionic_link_qcq_interrupts(lif->adminqcq, lif->notifyqcq);
675 }
676
677 err = -ENOMEM;
678 lif->txqcqs = devm_kcalloc(dev, lif->ionic->ntxqs_per_lif,
679 sizeof(*lif->txqcqs), GFP_KERNEL);
680 if (!lif->txqcqs)
681 goto err_out;
682 lif->rxqcqs = devm_kcalloc(dev, lif->ionic->nrxqs_per_lif,
683 sizeof(*lif->rxqcqs), GFP_KERNEL);
684 if (!lif->rxqcqs)
685 goto err_out;
686
687 lif->txqstats = devm_kcalloc(dev, lif->ionic->ntxqs_per_lif + 1,
688 sizeof(*lif->txqstats), GFP_KERNEL);
689 if (!lif->txqstats)
690 goto err_out;
691 lif->rxqstats = devm_kcalloc(dev, lif->ionic->nrxqs_per_lif + 1,
692 sizeof(*lif->rxqstats), GFP_KERNEL);
693 if (!lif->rxqstats)
694 goto err_out;
695
696 return 0;
697
698 err_out:
699 ionic_qcqs_free(lif);
700 return err;
701 }
702
ionic_qcq_sanitize(struct ionic_qcq * qcq)703 static void ionic_qcq_sanitize(struct ionic_qcq *qcq)
704 {
705 qcq->q.tail_idx = 0;
706 qcq->q.head_idx = 0;
707 qcq->cq.tail_idx = 0;
708 qcq->cq.done_color = 1;
709 memset(qcq->q_base, 0, qcq->q_size);
710 memset(qcq->cq_base, 0, qcq->cq_size);
711 memset(qcq->sg_base, 0, qcq->sg_size);
712 }
713
ionic_lif_txq_init(struct ionic_lif * lif,struct ionic_qcq * qcq)714 static int ionic_lif_txq_init(struct ionic_lif *lif, struct ionic_qcq *qcq)
715 {
716 struct device *dev = lif->ionic->dev;
717 struct ionic_queue *q = &qcq->q;
718 struct ionic_cq *cq = &qcq->cq;
719 struct ionic_admin_ctx ctx = {
720 .work = COMPLETION_INITIALIZER_ONSTACK(ctx.work),
721 .cmd.q_init = {
722 .opcode = IONIC_CMD_Q_INIT,
723 .lif_index = cpu_to_le16(lif->index),
724 .type = q->type,
725 .ver = lif->qtype_info[q->type].version,
726 .index = cpu_to_le32(q->index),
727 .flags = cpu_to_le16(IONIC_QINIT_F_IRQ |
728 IONIC_QINIT_F_SG),
729 .pid = cpu_to_le16(q->pid),
730 .ring_size = ilog2(q->num_descs),
731 .ring_base = cpu_to_le64(q->base_pa),
732 .cq_ring_base = cpu_to_le64(cq->base_pa),
733 .sg_ring_base = cpu_to_le64(q->sg_base_pa),
734 .features = cpu_to_le64(q->features),
735 },
736 };
737 unsigned int intr_index;
738 int err;
739
740 intr_index = qcq->intr.index;
741
742 ctx.cmd.q_init.intr_index = cpu_to_le16(intr_index);
743
744 dev_dbg(dev, "txq_init.pid %d\n", ctx.cmd.q_init.pid);
745 dev_dbg(dev, "txq_init.index %d\n", ctx.cmd.q_init.index);
746 dev_dbg(dev, "txq_init.ring_base 0x%llx\n", ctx.cmd.q_init.ring_base);
747 dev_dbg(dev, "txq_init.ring_size %d\n", ctx.cmd.q_init.ring_size);
748 dev_dbg(dev, "txq_init.flags 0x%x\n", ctx.cmd.q_init.flags);
749 dev_dbg(dev, "txq_init.ver %d\n", ctx.cmd.q_init.ver);
750 dev_dbg(dev, "txq_init.intr_index %d\n", ctx.cmd.q_init.intr_index);
751
752 ionic_qcq_sanitize(qcq);
753
754 err = ionic_adminq_post_wait(lif, &ctx);
755 if (err)
756 return err;
757
758 q->hw_type = ctx.comp.q_init.hw_type;
759 q->hw_index = le32_to_cpu(ctx.comp.q_init.hw_index);
760 q->dbval = IONIC_DBELL_QID(q->hw_index);
761
762 dev_dbg(dev, "txq->hw_type %d\n", q->hw_type);
763 dev_dbg(dev, "txq->hw_index %d\n", q->hw_index);
764
765 if (test_bit(IONIC_LIF_F_SPLIT_INTR, lif->state))
766 netif_napi_add(lif->netdev, &qcq->napi, ionic_tx_napi,
767 NAPI_POLL_WEIGHT);
768
769 qcq->flags |= IONIC_QCQ_F_INITED;
770
771 return 0;
772 }
773
ionic_lif_rxq_init(struct ionic_lif * lif,struct ionic_qcq * qcq)774 static int ionic_lif_rxq_init(struct ionic_lif *lif, struct ionic_qcq *qcq)
775 {
776 struct device *dev = lif->ionic->dev;
777 struct ionic_queue *q = &qcq->q;
778 struct ionic_cq *cq = &qcq->cq;
779 struct ionic_admin_ctx ctx = {
780 .work = COMPLETION_INITIALIZER_ONSTACK(ctx.work),
781 .cmd.q_init = {
782 .opcode = IONIC_CMD_Q_INIT,
783 .lif_index = cpu_to_le16(lif->index),
784 .type = q->type,
785 .ver = lif->qtype_info[q->type].version,
786 .index = cpu_to_le32(q->index),
787 .flags = cpu_to_le16(IONIC_QINIT_F_IRQ |
788 IONIC_QINIT_F_SG),
789 .intr_index = cpu_to_le16(cq->bound_intr->index),
790 .pid = cpu_to_le16(q->pid),
791 .ring_size = ilog2(q->num_descs),
792 .ring_base = cpu_to_le64(q->base_pa),
793 .cq_ring_base = cpu_to_le64(cq->base_pa),
794 .sg_ring_base = cpu_to_le64(q->sg_base_pa),
795 .features = cpu_to_le64(q->features),
796 },
797 };
798 int err;
799
800 dev_dbg(dev, "rxq_init.pid %d\n", ctx.cmd.q_init.pid);
801 dev_dbg(dev, "rxq_init.index %d\n", ctx.cmd.q_init.index);
802 dev_dbg(dev, "rxq_init.ring_base 0x%llx\n", ctx.cmd.q_init.ring_base);
803 dev_dbg(dev, "rxq_init.ring_size %d\n", ctx.cmd.q_init.ring_size);
804 dev_dbg(dev, "rxq_init.flags 0x%x\n", ctx.cmd.q_init.flags);
805 dev_dbg(dev, "rxq_init.ver %d\n", ctx.cmd.q_init.ver);
806 dev_dbg(dev, "rxq_init.intr_index %d\n", ctx.cmd.q_init.intr_index);
807
808 ionic_qcq_sanitize(qcq);
809
810 err = ionic_adminq_post_wait(lif, &ctx);
811 if (err)
812 return err;
813
814 q->hw_type = ctx.comp.q_init.hw_type;
815 q->hw_index = le32_to_cpu(ctx.comp.q_init.hw_index);
816 q->dbval = IONIC_DBELL_QID(q->hw_index);
817
818 dev_dbg(dev, "rxq->hw_type %d\n", q->hw_type);
819 dev_dbg(dev, "rxq->hw_index %d\n", q->hw_index);
820
821 if (test_bit(IONIC_LIF_F_SPLIT_INTR, lif->state))
822 netif_napi_add(lif->netdev, &qcq->napi, ionic_rx_napi,
823 NAPI_POLL_WEIGHT);
824 else
825 netif_napi_add(lif->netdev, &qcq->napi, ionic_txrx_napi,
826 NAPI_POLL_WEIGHT);
827
828 qcq->flags |= IONIC_QCQ_F_INITED;
829
830 return 0;
831 }
832
ionic_lif_create_hwstamp_txq(struct ionic_lif * lif)833 int ionic_lif_create_hwstamp_txq(struct ionic_lif *lif)
834 {
835 unsigned int num_desc, desc_sz, comp_sz, sg_desc_sz;
836 unsigned int txq_i, flags;
837 struct ionic_qcq *txq;
838 u64 features;
839 int err;
840
841 mutex_lock(&lif->queue_lock);
842
843 if (lif->hwstamp_txq)
844 goto out;
845
846 features = IONIC_Q_F_2X_CQ_DESC | IONIC_TXQ_F_HWSTAMP;
847
848 num_desc = IONIC_MIN_TXRX_DESC;
849 desc_sz = sizeof(struct ionic_txq_desc);
850 comp_sz = 2 * sizeof(struct ionic_txq_comp);
851
852 if (lif->qtype_info[IONIC_QTYPE_TXQ].version >= 1 &&
853 lif->qtype_info[IONIC_QTYPE_TXQ].sg_desc_sz == sizeof(struct ionic_txq_sg_desc_v1))
854 sg_desc_sz = sizeof(struct ionic_txq_sg_desc_v1);
855 else
856 sg_desc_sz = sizeof(struct ionic_txq_sg_desc);
857
858 txq_i = lif->ionic->ntxqs_per_lif;
859 flags = IONIC_QCQ_F_TX_STATS | IONIC_QCQ_F_SG;
860
861 err = ionic_qcq_alloc(lif, IONIC_QTYPE_TXQ, txq_i, "hwstamp_tx", flags,
862 num_desc, desc_sz, comp_sz, sg_desc_sz,
863 lif->kern_pid, &txq);
864 if (err)
865 goto err_qcq_alloc;
866
867 txq->q.features = features;
868
869 ionic_link_qcq_interrupts(lif->adminqcq, txq);
870 ionic_debugfs_add_qcq(lif, txq);
871
872 lif->hwstamp_txq = txq;
873
874 if (netif_running(lif->netdev)) {
875 err = ionic_lif_txq_init(lif, txq);
876 if (err)
877 goto err_qcq_init;
878
879 if (test_bit(IONIC_LIF_F_UP, lif->state)) {
880 err = ionic_qcq_enable(txq);
881 if (err)
882 goto err_qcq_enable;
883 }
884 }
885
886 out:
887 mutex_unlock(&lif->queue_lock);
888
889 return 0;
890
891 err_qcq_enable:
892 ionic_lif_qcq_deinit(lif, txq);
893 err_qcq_init:
894 lif->hwstamp_txq = NULL;
895 ionic_debugfs_del_qcq(txq);
896 ionic_qcq_free(lif, txq);
897 devm_kfree(lif->ionic->dev, txq);
898 err_qcq_alloc:
899 mutex_unlock(&lif->queue_lock);
900 return err;
901 }
902
ionic_lif_create_hwstamp_rxq(struct ionic_lif * lif)903 int ionic_lif_create_hwstamp_rxq(struct ionic_lif *lif)
904 {
905 unsigned int num_desc, desc_sz, comp_sz, sg_desc_sz;
906 unsigned int rxq_i, flags;
907 struct ionic_qcq *rxq;
908 u64 features;
909 int err;
910
911 mutex_lock(&lif->queue_lock);
912
913 if (lif->hwstamp_rxq)
914 goto out;
915
916 features = IONIC_Q_F_2X_CQ_DESC | IONIC_RXQ_F_HWSTAMP;
917
918 num_desc = IONIC_MIN_TXRX_DESC;
919 desc_sz = sizeof(struct ionic_rxq_desc);
920 comp_sz = 2 * sizeof(struct ionic_rxq_comp);
921 sg_desc_sz = sizeof(struct ionic_rxq_sg_desc);
922
923 rxq_i = lif->ionic->nrxqs_per_lif;
924 flags = IONIC_QCQ_F_RX_STATS | IONIC_QCQ_F_SG;
925
926 err = ionic_qcq_alloc(lif, IONIC_QTYPE_RXQ, rxq_i, "hwstamp_rx", flags,
927 num_desc, desc_sz, comp_sz, sg_desc_sz,
928 lif->kern_pid, &rxq);
929 if (err)
930 goto err_qcq_alloc;
931
932 rxq->q.features = features;
933
934 ionic_link_qcq_interrupts(lif->adminqcq, rxq);
935 ionic_debugfs_add_qcq(lif, rxq);
936
937 lif->hwstamp_rxq = rxq;
938
939 if (netif_running(lif->netdev)) {
940 err = ionic_lif_rxq_init(lif, rxq);
941 if (err)
942 goto err_qcq_init;
943
944 if (test_bit(IONIC_LIF_F_UP, lif->state)) {
945 ionic_rx_fill(&rxq->q);
946 err = ionic_qcq_enable(rxq);
947 if (err)
948 goto err_qcq_enable;
949 }
950 }
951
952 out:
953 mutex_unlock(&lif->queue_lock);
954
955 return 0;
956
957 err_qcq_enable:
958 ionic_lif_qcq_deinit(lif, rxq);
959 err_qcq_init:
960 lif->hwstamp_rxq = NULL;
961 ionic_debugfs_del_qcq(rxq);
962 ionic_qcq_free(lif, rxq);
963 devm_kfree(lif->ionic->dev, rxq);
964 err_qcq_alloc:
965 mutex_unlock(&lif->queue_lock);
966 return err;
967 }
968
ionic_lif_config_hwstamp_rxq_all(struct ionic_lif * lif,bool rx_all)969 int ionic_lif_config_hwstamp_rxq_all(struct ionic_lif *lif, bool rx_all)
970 {
971 struct ionic_queue_params qparam;
972
973 ionic_init_queue_params(lif, &qparam);
974
975 if (rx_all)
976 qparam.rxq_features = IONIC_Q_F_2X_CQ_DESC | IONIC_RXQ_F_HWSTAMP;
977 else
978 qparam.rxq_features = 0;
979
980 /* if we're not running, just set the values and return */
981 if (!netif_running(lif->netdev)) {
982 lif->rxq_features = qparam.rxq_features;
983 return 0;
984 }
985
986 return ionic_reconfigure_queues(lif, &qparam);
987 }
988
ionic_lif_set_hwstamp_txmode(struct ionic_lif * lif,u16 txstamp_mode)989 int ionic_lif_set_hwstamp_txmode(struct ionic_lif *lif, u16 txstamp_mode)
990 {
991 struct ionic_admin_ctx ctx = {
992 .work = COMPLETION_INITIALIZER_ONSTACK(ctx.work),
993 .cmd.lif_setattr = {
994 .opcode = IONIC_CMD_LIF_SETATTR,
995 .index = cpu_to_le16(lif->index),
996 .attr = IONIC_LIF_ATTR_TXSTAMP,
997 .txstamp_mode = cpu_to_le16(txstamp_mode),
998 },
999 };
1000
1001 return ionic_adminq_post_wait(lif, &ctx);
1002 }
1003
ionic_lif_del_hwstamp_rxfilt(struct ionic_lif * lif)1004 static void ionic_lif_del_hwstamp_rxfilt(struct ionic_lif *lif)
1005 {
1006 struct ionic_admin_ctx ctx = {
1007 .work = COMPLETION_INITIALIZER_ONSTACK(ctx.work),
1008 .cmd.rx_filter_del = {
1009 .opcode = IONIC_CMD_RX_FILTER_DEL,
1010 .lif_index = cpu_to_le16(lif->index),
1011 },
1012 };
1013 struct ionic_rx_filter *f;
1014 u32 filter_id;
1015 int err;
1016
1017 spin_lock_bh(&lif->rx_filters.lock);
1018
1019 f = ionic_rx_filter_rxsteer(lif);
1020 if (!f) {
1021 spin_unlock_bh(&lif->rx_filters.lock);
1022 return;
1023 }
1024
1025 filter_id = f->filter_id;
1026 ionic_rx_filter_free(lif, f);
1027
1028 spin_unlock_bh(&lif->rx_filters.lock);
1029
1030 netdev_dbg(lif->netdev, "rx_filter del RXSTEER (id %d)\n", filter_id);
1031
1032 ctx.cmd.rx_filter_del.filter_id = cpu_to_le32(filter_id);
1033
1034 err = ionic_adminq_post_wait(lif, &ctx);
1035 if (err && err != -EEXIST)
1036 netdev_dbg(lif->netdev, "failed to delete rx_filter RXSTEER (id %d)\n", filter_id);
1037 }
1038
ionic_lif_add_hwstamp_rxfilt(struct ionic_lif * lif,u64 pkt_class)1039 static int ionic_lif_add_hwstamp_rxfilt(struct ionic_lif *lif, u64 pkt_class)
1040 {
1041 struct ionic_admin_ctx ctx = {
1042 .work = COMPLETION_INITIALIZER_ONSTACK(ctx.work),
1043 .cmd.rx_filter_add = {
1044 .opcode = IONIC_CMD_RX_FILTER_ADD,
1045 .lif_index = cpu_to_le16(lif->index),
1046 .match = cpu_to_le16(IONIC_RX_FILTER_STEER_PKTCLASS),
1047 .pkt_class = cpu_to_le64(pkt_class),
1048 },
1049 };
1050 u8 qtype;
1051 u32 qid;
1052 int err;
1053
1054 if (!lif->hwstamp_rxq)
1055 return -EINVAL;
1056
1057 qtype = lif->hwstamp_rxq->q.type;
1058 ctx.cmd.rx_filter_add.qtype = qtype;
1059
1060 qid = lif->hwstamp_rxq->q.index;
1061 ctx.cmd.rx_filter_add.qid = cpu_to_le32(qid);
1062
1063 netdev_dbg(lif->netdev, "rx_filter add RXSTEER\n");
1064 err = ionic_adminq_post_wait(lif, &ctx);
1065 if (err && err != -EEXIST)
1066 return err;
1067
1068 return ionic_rx_filter_save(lif, 0, qid, 0, &ctx);
1069 }
1070
ionic_lif_set_hwstamp_rxfilt(struct ionic_lif * lif,u64 pkt_class)1071 int ionic_lif_set_hwstamp_rxfilt(struct ionic_lif *lif, u64 pkt_class)
1072 {
1073 ionic_lif_del_hwstamp_rxfilt(lif);
1074
1075 if (!pkt_class)
1076 return 0;
1077
1078 return ionic_lif_add_hwstamp_rxfilt(lif, pkt_class);
1079 }
1080
ionic_notifyq_service(struct ionic_cq * cq,struct ionic_cq_info * cq_info)1081 static bool ionic_notifyq_service(struct ionic_cq *cq,
1082 struct ionic_cq_info *cq_info)
1083 {
1084 union ionic_notifyq_comp *comp = cq_info->cq_desc;
1085 struct ionic_deferred_work *work;
1086 struct net_device *netdev;
1087 struct ionic_queue *q;
1088 struct ionic_lif *lif;
1089 u64 eid;
1090
1091 q = cq->bound_q;
1092 lif = q->info[0].cb_arg;
1093 netdev = lif->netdev;
1094 eid = le64_to_cpu(comp->event.eid);
1095
1096 /* Have we run out of new completions to process? */
1097 if ((s64)(eid - lif->last_eid) <= 0)
1098 return false;
1099
1100 lif->last_eid = eid;
1101
1102 dev_dbg(lif->ionic->dev, "notifyq event:\n");
1103 dynamic_hex_dump("event ", DUMP_PREFIX_OFFSET, 16, 1,
1104 comp, sizeof(*comp), true);
1105
1106 switch (le16_to_cpu(comp->event.ecode)) {
1107 case IONIC_EVENT_LINK_CHANGE:
1108 ionic_link_status_check_request(lif, CAN_NOT_SLEEP);
1109 break;
1110 case IONIC_EVENT_RESET:
1111 work = kzalloc(sizeof(*work), GFP_ATOMIC);
1112 if (!work) {
1113 netdev_err(lif->netdev, "Reset event dropped\n");
1114 } else {
1115 work->type = IONIC_DW_TYPE_LIF_RESET;
1116 ionic_lif_deferred_enqueue(&lif->deferred, work);
1117 }
1118 break;
1119 default:
1120 netdev_warn(netdev, "Notifyq event ecode=%d eid=%lld\n",
1121 comp->event.ecode, eid);
1122 break;
1123 }
1124
1125 return true;
1126 }
1127
ionic_adminq_service(struct ionic_cq * cq,struct ionic_cq_info * cq_info)1128 static bool ionic_adminq_service(struct ionic_cq *cq,
1129 struct ionic_cq_info *cq_info)
1130 {
1131 struct ionic_admin_comp *comp = cq_info->cq_desc;
1132
1133 if (!color_match(comp->color, cq->done_color))
1134 return false;
1135
1136 ionic_q_service(cq->bound_q, cq_info, le16_to_cpu(comp->comp_index));
1137
1138 return true;
1139 }
1140
ionic_adminq_napi(struct napi_struct * napi,int budget)1141 static int ionic_adminq_napi(struct napi_struct *napi, int budget)
1142 {
1143 struct ionic_intr_info *intr = napi_to_cq(napi)->bound_intr;
1144 struct ionic_lif *lif = napi_to_cq(napi)->lif;
1145 struct ionic_dev *idev = &lif->ionic->idev;
1146 unsigned long irqflags;
1147 unsigned int flags = 0;
1148 int rx_work = 0;
1149 int tx_work = 0;
1150 int n_work = 0;
1151 int a_work = 0;
1152 int work_done;
1153 int credits;
1154
1155 if (lif->notifyqcq && lif->notifyqcq->flags & IONIC_QCQ_F_INITED)
1156 n_work = ionic_cq_service(&lif->notifyqcq->cq, budget,
1157 ionic_notifyq_service, NULL, NULL);
1158
1159 spin_lock_irqsave(&lif->adminq_lock, irqflags);
1160 if (lif->adminqcq && lif->adminqcq->flags & IONIC_QCQ_F_INITED)
1161 a_work = ionic_cq_service(&lif->adminqcq->cq, budget,
1162 ionic_adminq_service, NULL, NULL);
1163 spin_unlock_irqrestore(&lif->adminq_lock, irqflags);
1164
1165 if (lif->hwstamp_rxq)
1166 rx_work = ionic_cq_service(&lif->hwstamp_rxq->cq, budget,
1167 ionic_rx_service, NULL, NULL);
1168
1169 if (lif->hwstamp_txq)
1170 tx_work = ionic_cq_service(&lif->hwstamp_txq->cq, budget,
1171 ionic_tx_service, NULL, NULL);
1172
1173 work_done = max(max(n_work, a_work), max(rx_work, tx_work));
1174 if (work_done < budget && napi_complete_done(napi, work_done)) {
1175 flags |= IONIC_INTR_CRED_UNMASK;
1176 intr->rearm_count++;
1177 }
1178
1179 if (work_done || flags) {
1180 flags |= IONIC_INTR_CRED_RESET_COALESCE;
1181 credits = n_work + a_work + rx_work + tx_work;
1182 ionic_intr_credits(idev->intr_ctrl, intr->index, credits, flags);
1183 }
1184
1185 return work_done;
1186 }
1187
ionic_get_stats64(struct net_device * netdev,struct rtnl_link_stats64 * ns)1188 void ionic_get_stats64(struct net_device *netdev,
1189 struct rtnl_link_stats64 *ns)
1190 {
1191 struct ionic_lif *lif = netdev_priv(netdev);
1192 struct ionic_lif_stats *ls;
1193
1194 memset(ns, 0, sizeof(*ns));
1195 ls = &lif->info->stats;
1196
1197 ns->rx_packets = le64_to_cpu(ls->rx_ucast_packets) +
1198 le64_to_cpu(ls->rx_mcast_packets) +
1199 le64_to_cpu(ls->rx_bcast_packets);
1200
1201 ns->tx_packets = le64_to_cpu(ls->tx_ucast_packets) +
1202 le64_to_cpu(ls->tx_mcast_packets) +
1203 le64_to_cpu(ls->tx_bcast_packets);
1204
1205 ns->rx_bytes = le64_to_cpu(ls->rx_ucast_bytes) +
1206 le64_to_cpu(ls->rx_mcast_bytes) +
1207 le64_to_cpu(ls->rx_bcast_bytes);
1208
1209 ns->tx_bytes = le64_to_cpu(ls->tx_ucast_bytes) +
1210 le64_to_cpu(ls->tx_mcast_bytes) +
1211 le64_to_cpu(ls->tx_bcast_bytes);
1212
1213 ns->rx_dropped = le64_to_cpu(ls->rx_ucast_drop_packets) +
1214 le64_to_cpu(ls->rx_mcast_drop_packets) +
1215 le64_to_cpu(ls->rx_bcast_drop_packets);
1216
1217 ns->tx_dropped = le64_to_cpu(ls->tx_ucast_drop_packets) +
1218 le64_to_cpu(ls->tx_mcast_drop_packets) +
1219 le64_to_cpu(ls->tx_bcast_drop_packets);
1220
1221 ns->multicast = le64_to_cpu(ls->rx_mcast_packets);
1222
1223 ns->rx_over_errors = le64_to_cpu(ls->rx_queue_empty);
1224
1225 ns->rx_missed_errors = le64_to_cpu(ls->rx_dma_error) +
1226 le64_to_cpu(ls->rx_queue_disabled) +
1227 le64_to_cpu(ls->rx_desc_fetch_error) +
1228 le64_to_cpu(ls->rx_desc_data_error);
1229
1230 ns->tx_aborted_errors = le64_to_cpu(ls->tx_dma_error) +
1231 le64_to_cpu(ls->tx_queue_disabled) +
1232 le64_to_cpu(ls->tx_desc_fetch_error) +
1233 le64_to_cpu(ls->tx_desc_data_error);
1234
1235 ns->rx_errors = ns->rx_over_errors +
1236 ns->rx_missed_errors;
1237
1238 ns->tx_errors = ns->tx_aborted_errors;
1239 }
1240
ionic_lif_addr_add(struct ionic_lif * lif,const u8 * addr)1241 static int ionic_lif_addr_add(struct ionic_lif *lif, const u8 *addr)
1242 {
1243 struct ionic_admin_ctx ctx = {
1244 .work = COMPLETION_INITIALIZER_ONSTACK(ctx.work),
1245 .cmd.rx_filter_add = {
1246 .opcode = IONIC_CMD_RX_FILTER_ADD,
1247 .lif_index = cpu_to_le16(lif->index),
1248 .match = cpu_to_le16(IONIC_RX_FILTER_MATCH_MAC),
1249 },
1250 };
1251 struct ionic_rx_filter *f;
1252 int err;
1253
1254 /* don't bother if we already have it */
1255 spin_lock_bh(&lif->rx_filters.lock);
1256 f = ionic_rx_filter_by_addr(lif, addr);
1257 spin_unlock_bh(&lif->rx_filters.lock);
1258 if (f)
1259 return 0;
1260
1261 netdev_dbg(lif->netdev, "rx_filter add ADDR %pM\n", addr);
1262
1263 memcpy(ctx.cmd.rx_filter_add.mac.addr, addr, ETH_ALEN);
1264 err = ionic_adminq_post_wait(lif, &ctx);
1265 if (err && err != -EEXIST)
1266 return err;
1267
1268 return ionic_rx_filter_save(lif, 0, IONIC_RXQ_INDEX_ANY, 0, &ctx);
1269 }
1270
ionic_lif_addr_del(struct ionic_lif * lif,const u8 * addr)1271 static int ionic_lif_addr_del(struct ionic_lif *lif, const u8 *addr)
1272 {
1273 struct ionic_admin_ctx ctx = {
1274 .work = COMPLETION_INITIALIZER_ONSTACK(ctx.work),
1275 .cmd.rx_filter_del = {
1276 .opcode = IONIC_CMD_RX_FILTER_DEL,
1277 .lif_index = cpu_to_le16(lif->index),
1278 },
1279 };
1280 struct ionic_rx_filter *f;
1281 int err;
1282
1283 spin_lock_bh(&lif->rx_filters.lock);
1284 f = ionic_rx_filter_by_addr(lif, addr);
1285 if (!f) {
1286 spin_unlock_bh(&lif->rx_filters.lock);
1287 return -ENOENT;
1288 }
1289
1290 netdev_dbg(lif->netdev, "rx_filter del ADDR %pM (id %d)\n",
1291 addr, f->filter_id);
1292
1293 ctx.cmd.rx_filter_del.filter_id = cpu_to_le32(f->filter_id);
1294 ionic_rx_filter_free(lif, f);
1295 spin_unlock_bh(&lif->rx_filters.lock);
1296
1297 err = ionic_adminq_post_wait(lif, &ctx);
1298 if (err && err != -EEXIST)
1299 return err;
1300
1301 return 0;
1302 }
1303
ionic_lif_addr(struct ionic_lif * lif,const u8 * addr,bool add,bool can_sleep)1304 static int ionic_lif_addr(struct ionic_lif *lif, const u8 *addr, bool add,
1305 bool can_sleep)
1306 {
1307 struct ionic_deferred_work *work;
1308 unsigned int nmfilters;
1309 unsigned int nufilters;
1310
1311 if (add) {
1312 /* Do we have space for this filter? We test the counters
1313 * here before checking the need for deferral so that we
1314 * can return an overflow error to the stack.
1315 */
1316 nmfilters = le32_to_cpu(lif->identity->eth.max_mcast_filters);
1317 nufilters = le32_to_cpu(lif->identity->eth.max_ucast_filters);
1318
1319 if ((is_multicast_ether_addr(addr) && lif->nmcast < nmfilters))
1320 lif->nmcast++;
1321 else if (!is_multicast_ether_addr(addr) &&
1322 lif->nucast < nufilters)
1323 lif->nucast++;
1324 else
1325 return -ENOSPC;
1326 } else {
1327 if (is_multicast_ether_addr(addr) && lif->nmcast)
1328 lif->nmcast--;
1329 else if (!is_multicast_ether_addr(addr) && lif->nucast)
1330 lif->nucast--;
1331 }
1332
1333 if (!can_sleep) {
1334 work = kzalloc(sizeof(*work), GFP_ATOMIC);
1335 if (!work)
1336 return -ENOMEM;
1337 work->type = add ? IONIC_DW_TYPE_RX_ADDR_ADD :
1338 IONIC_DW_TYPE_RX_ADDR_DEL;
1339 memcpy(work->addr, addr, ETH_ALEN);
1340 netdev_dbg(lif->netdev, "deferred: rx_filter %s %pM\n",
1341 add ? "add" : "del", addr);
1342 ionic_lif_deferred_enqueue(&lif->deferred, work);
1343 } else {
1344 netdev_dbg(lif->netdev, "rx_filter %s %pM\n",
1345 add ? "add" : "del", addr);
1346 if (add)
1347 return ionic_lif_addr_add(lif, addr);
1348 else
1349 return ionic_lif_addr_del(lif, addr);
1350 }
1351
1352 return 0;
1353 }
1354
ionic_addr_add(struct net_device * netdev,const u8 * addr)1355 static int ionic_addr_add(struct net_device *netdev, const u8 *addr)
1356 {
1357 return ionic_lif_addr(netdev_priv(netdev), addr, ADD_ADDR, CAN_SLEEP);
1358 }
1359
ionic_ndo_addr_add(struct net_device * netdev,const u8 * addr)1360 static int ionic_ndo_addr_add(struct net_device *netdev, const u8 *addr)
1361 {
1362 return ionic_lif_addr(netdev_priv(netdev), addr, ADD_ADDR, CAN_NOT_SLEEP);
1363 }
1364
ionic_addr_del(struct net_device * netdev,const u8 * addr)1365 static int ionic_addr_del(struct net_device *netdev, const u8 *addr)
1366 {
1367 return ionic_lif_addr(netdev_priv(netdev), addr, DEL_ADDR, CAN_SLEEP);
1368 }
1369
ionic_ndo_addr_del(struct net_device * netdev,const u8 * addr)1370 static int ionic_ndo_addr_del(struct net_device *netdev, const u8 *addr)
1371 {
1372 return ionic_lif_addr(netdev_priv(netdev), addr, DEL_ADDR, CAN_NOT_SLEEP);
1373 }
1374
ionic_lif_rx_mode(struct ionic_lif * lif,unsigned int rx_mode)1375 static void ionic_lif_rx_mode(struct ionic_lif *lif, unsigned int rx_mode)
1376 {
1377 struct ionic_admin_ctx ctx = {
1378 .work = COMPLETION_INITIALIZER_ONSTACK(ctx.work),
1379 .cmd.rx_mode_set = {
1380 .opcode = IONIC_CMD_RX_MODE_SET,
1381 .lif_index = cpu_to_le16(lif->index),
1382 .rx_mode = cpu_to_le16(rx_mode),
1383 },
1384 };
1385 char buf[128];
1386 int err;
1387 int i;
1388 #define REMAIN(__x) (sizeof(buf) - (__x))
1389
1390 i = scnprintf(buf, sizeof(buf), "rx_mode 0x%04x -> 0x%04x:",
1391 lif->rx_mode, rx_mode);
1392 if (rx_mode & IONIC_RX_MODE_F_UNICAST)
1393 i += scnprintf(&buf[i], REMAIN(i), " RX_MODE_F_UNICAST");
1394 if (rx_mode & IONIC_RX_MODE_F_MULTICAST)
1395 i += scnprintf(&buf[i], REMAIN(i), " RX_MODE_F_MULTICAST");
1396 if (rx_mode & IONIC_RX_MODE_F_BROADCAST)
1397 i += scnprintf(&buf[i], REMAIN(i), " RX_MODE_F_BROADCAST");
1398 if (rx_mode & IONIC_RX_MODE_F_PROMISC)
1399 i += scnprintf(&buf[i], REMAIN(i), " RX_MODE_F_PROMISC");
1400 if (rx_mode & IONIC_RX_MODE_F_ALLMULTI)
1401 i += scnprintf(&buf[i], REMAIN(i), " RX_MODE_F_ALLMULTI");
1402 netdev_dbg(lif->netdev, "lif%d %s\n", lif->index, buf);
1403
1404 err = ionic_adminq_post_wait(lif, &ctx);
1405 if (err)
1406 netdev_warn(lif->netdev, "set rx_mode 0x%04x failed: %d\n",
1407 rx_mode, err);
1408 else
1409 lif->rx_mode = rx_mode;
1410 }
1411
ionic_set_rx_mode(struct net_device * netdev,bool can_sleep)1412 static void ionic_set_rx_mode(struct net_device *netdev, bool can_sleep)
1413 {
1414 struct ionic_lif *lif = netdev_priv(netdev);
1415 struct ionic_deferred_work *work;
1416 unsigned int nfilters;
1417 unsigned int rx_mode;
1418
1419 rx_mode = IONIC_RX_MODE_F_UNICAST;
1420 rx_mode |= (netdev->flags & IFF_MULTICAST) ? IONIC_RX_MODE_F_MULTICAST : 0;
1421 rx_mode |= (netdev->flags & IFF_BROADCAST) ? IONIC_RX_MODE_F_BROADCAST : 0;
1422 rx_mode |= (netdev->flags & IFF_PROMISC) ? IONIC_RX_MODE_F_PROMISC : 0;
1423 rx_mode |= (netdev->flags & IFF_ALLMULTI) ? IONIC_RX_MODE_F_ALLMULTI : 0;
1424
1425 /* sync unicast addresses
1426 * next check to see if we're in an overflow state
1427 * if so, we track that we overflowed and enable NIC PROMISC
1428 * else if the overflow is set and not needed
1429 * we remove our overflow flag and check the netdev flags
1430 * to see if we can disable NIC PROMISC
1431 */
1432 if (can_sleep)
1433 __dev_uc_sync(netdev, ionic_addr_add, ionic_addr_del);
1434 else
1435 __dev_uc_sync(netdev, ionic_ndo_addr_add, ionic_ndo_addr_del);
1436 nfilters = le32_to_cpu(lif->identity->eth.max_ucast_filters);
1437 if (netdev_uc_count(netdev) + 1 > nfilters) {
1438 rx_mode |= IONIC_RX_MODE_F_PROMISC;
1439 lif->uc_overflow = true;
1440 } else if (lif->uc_overflow) {
1441 lif->uc_overflow = false;
1442 if (!(netdev->flags & IFF_PROMISC))
1443 rx_mode &= ~IONIC_RX_MODE_F_PROMISC;
1444 }
1445
1446 /* same for multicast */
1447 if (can_sleep)
1448 __dev_mc_sync(netdev, ionic_addr_add, ionic_addr_del);
1449 else
1450 __dev_mc_sync(netdev, ionic_ndo_addr_add, ionic_ndo_addr_del);
1451 nfilters = le32_to_cpu(lif->identity->eth.max_mcast_filters);
1452 if (netdev_mc_count(netdev) > nfilters) {
1453 rx_mode |= IONIC_RX_MODE_F_ALLMULTI;
1454 lif->mc_overflow = true;
1455 } else if (lif->mc_overflow) {
1456 lif->mc_overflow = false;
1457 if (!(netdev->flags & IFF_ALLMULTI))
1458 rx_mode &= ~IONIC_RX_MODE_F_ALLMULTI;
1459 }
1460
1461 if (lif->rx_mode != rx_mode) {
1462 if (!can_sleep) {
1463 work = kzalloc(sizeof(*work), GFP_ATOMIC);
1464 if (!work) {
1465 netdev_err(lif->netdev, "rxmode change dropped\n");
1466 return;
1467 }
1468 work->type = IONIC_DW_TYPE_RX_MODE;
1469 work->rx_mode = rx_mode;
1470 netdev_dbg(lif->netdev, "deferred: rx_mode\n");
1471 ionic_lif_deferred_enqueue(&lif->deferred, work);
1472 } else {
1473 ionic_lif_rx_mode(lif, rx_mode);
1474 }
1475 }
1476 }
1477
ionic_ndo_set_rx_mode(struct net_device * netdev)1478 static void ionic_ndo_set_rx_mode(struct net_device *netdev)
1479 {
1480 ionic_set_rx_mode(netdev, CAN_NOT_SLEEP);
1481 }
1482
ionic_netdev_features_to_nic(netdev_features_t features)1483 static __le64 ionic_netdev_features_to_nic(netdev_features_t features)
1484 {
1485 u64 wanted = 0;
1486
1487 if (features & NETIF_F_HW_VLAN_CTAG_TX)
1488 wanted |= IONIC_ETH_HW_VLAN_TX_TAG;
1489 if (features & NETIF_F_HW_VLAN_CTAG_RX)
1490 wanted |= IONIC_ETH_HW_VLAN_RX_STRIP;
1491 if (features & NETIF_F_HW_VLAN_CTAG_FILTER)
1492 wanted |= IONIC_ETH_HW_VLAN_RX_FILTER;
1493 if (features & NETIF_F_RXHASH)
1494 wanted |= IONIC_ETH_HW_RX_HASH;
1495 if (features & NETIF_F_RXCSUM)
1496 wanted |= IONIC_ETH_HW_RX_CSUM;
1497 if (features & NETIF_F_SG)
1498 wanted |= IONIC_ETH_HW_TX_SG;
1499 if (features & NETIF_F_HW_CSUM)
1500 wanted |= IONIC_ETH_HW_TX_CSUM;
1501 if (features & NETIF_F_TSO)
1502 wanted |= IONIC_ETH_HW_TSO;
1503 if (features & NETIF_F_TSO6)
1504 wanted |= IONIC_ETH_HW_TSO_IPV6;
1505 if (features & NETIF_F_TSO_ECN)
1506 wanted |= IONIC_ETH_HW_TSO_ECN;
1507 if (features & NETIF_F_GSO_GRE)
1508 wanted |= IONIC_ETH_HW_TSO_GRE;
1509 if (features & NETIF_F_GSO_GRE_CSUM)
1510 wanted |= IONIC_ETH_HW_TSO_GRE_CSUM;
1511 if (features & NETIF_F_GSO_IPXIP4)
1512 wanted |= IONIC_ETH_HW_TSO_IPXIP4;
1513 if (features & NETIF_F_GSO_IPXIP6)
1514 wanted |= IONIC_ETH_HW_TSO_IPXIP6;
1515 if (features & NETIF_F_GSO_UDP_TUNNEL)
1516 wanted |= IONIC_ETH_HW_TSO_UDP;
1517 if (features & NETIF_F_GSO_UDP_TUNNEL_CSUM)
1518 wanted |= IONIC_ETH_HW_TSO_UDP_CSUM;
1519
1520 return cpu_to_le64(wanted);
1521 }
1522
ionic_set_nic_features(struct ionic_lif * lif,netdev_features_t features)1523 static int ionic_set_nic_features(struct ionic_lif *lif,
1524 netdev_features_t features)
1525 {
1526 struct device *dev = lif->ionic->dev;
1527 struct ionic_admin_ctx ctx = {
1528 .work = COMPLETION_INITIALIZER_ONSTACK(ctx.work),
1529 .cmd.lif_setattr = {
1530 .opcode = IONIC_CMD_LIF_SETATTR,
1531 .index = cpu_to_le16(lif->index),
1532 .attr = IONIC_LIF_ATTR_FEATURES,
1533 },
1534 };
1535 u64 vlan_flags = IONIC_ETH_HW_VLAN_TX_TAG |
1536 IONIC_ETH_HW_VLAN_RX_STRIP |
1537 IONIC_ETH_HW_VLAN_RX_FILTER;
1538 u64 old_hw_features;
1539 int err;
1540
1541 ctx.cmd.lif_setattr.features = ionic_netdev_features_to_nic(features);
1542
1543 if (lif->phc)
1544 ctx.cmd.lif_setattr.features |= cpu_to_le64(IONIC_ETH_HW_TIMESTAMP);
1545
1546 err = ionic_adminq_post_wait(lif, &ctx);
1547 if (err)
1548 return err;
1549
1550 old_hw_features = lif->hw_features;
1551 lif->hw_features = le64_to_cpu(ctx.cmd.lif_setattr.features &
1552 ctx.comp.lif_setattr.features);
1553
1554 if ((old_hw_features ^ lif->hw_features) & IONIC_ETH_HW_RX_HASH)
1555 ionic_lif_rss_config(lif, lif->rss_types, NULL, NULL);
1556
1557 if ((vlan_flags & features) &&
1558 !(vlan_flags & le64_to_cpu(ctx.comp.lif_setattr.features)))
1559 dev_info_once(lif->ionic->dev, "NIC is not supporting vlan offload, likely in SmartNIC mode\n");
1560
1561 if (lif->hw_features & IONIC_ETH_HW_VLAN_TX_TAG)
1562 dev_dbg(dev, "feature ETH_HW_VLAN_TX_TAG\n");
1563 if (lif->hw_features & IONIC_ETH_HW_VLAN_RX_STRIP)
1564 dev_dbg(dev, "feature ETH_HW_VLAN_RX_STRIP\n");
1565 if (lif->hw_features & IONIC_ETH_HW_VLAN_RX_FILTER)
1566 dev_dbg(dev, "feature ETH_HW_VLAN_RX_FILTER\n");
1567 if (lif->hw_features & IONIC_ETH_HW_RX_HASH)
1568 dev_dbg(dev, "feature ETH_HW_RX_HASH\n");
1569 if (lif->hw_features & IONIC_ETH_HW_TX_SG)
1570 dev_dbg(dev, "feature ETH_HW_TX_SG\n");
1571 if (lif->hw_features & IONIC_ETH_HW_TX_CSUM)
1572 dev_dbg(dev, "feature ETH_HW_TX_CSUM\n");
1573 if (lif->hw_features & IONIC_ETH_HW_RX_CSUM)
1574 dev_dbg(dev, "feature ETH_HW_RX_CSUM\n");
1575 if (lif->hw_features & IONIC_ETH_HW_TSO)
1576 dev_dbg(dev, "feature ETH_HW_TSO\n");
1577 if (lif->hw_features & IONIC_ETH_HW_TSO_IPV6)
1578 dev_dbg(dev, "feature ETH_HW_TSO_IPV6\n");
1579 if (lif->hw_features & IONIC_ETH_HW_TSO_ECN)
1580 dev_dbg(dev, "feature ETH_HW_TSO_ECN\n");
1581 if (lif->hw_features & IONIC_ETH_HW_TSO_GRE)
1582 dev_dbg(dev, "feature ETH_HW_TSO_GRE\n");
1583 if (lif->hw_features & IONIC_ETH_HW_TSO_GRE_CSUM)
1584 dev_dbg(dev, "feature ETH_HW_TSO_GRE_CSUM\n");
1585 if (lif->hw_features & IONIC_ETH_HW_TSO_IPXIP4)
1586 dev_dbg(dev, "feature ETH_HW_TSO_IPXIP4\n");
1587 if (lif->hw_features & IONIC_ETH_HW_TSO_IPXIP6)
1588 dev_dbg(dev, "feature ETH_HW_TSO_IPXIP6\n");
1589 if (lif->hw_features & IONIC_ETH_HW_TSO_UDP)
1590 dev_dbg(dev, "feature ETH_HW_TSO_UDP\n");
1591 if (lif->hw_features & IONIC_ETH_HW_TSO_UDP_CSUM)
1592 dev_dbg(dev, "feature ETH_HW_TSO_UDP_CSUM\n");
1593 if (lif->hw_features & IONIC_ETH_HW_TIMESTAMP)
1594 dev_dbg(dev, "feature ETH_HW_TIMESTAMP\n");
1595
1596 return 0;
1597 }
1598
ionic_init_nic_features(struct ionic_lif * lif)1599 static int ionic_init_nic_features(struct ionic_lif *lif)
1600 {
1601 struct net_device *netdev = lif->netdev;
1602 netdev_features_t features;
1603 int err;
1604
1605 /* set up what we expect to support by default */
1606 features = NETIF_F_HW_VLAN_CTAG_TX |
1607 NETIF_F_HW_VLAN_CTAG_RX |
1608 NETIF_F_HW_VLAN_CTAG_FILTER |
1609 NETIF_F_RXHASH |
1610 NETIF_F_SG |
1611 NETIF_F_HW_CSUM |
1612 NETIF_F_RXCSUM |
1613 NETIF_F_TSO |
1614 NETIF_F_TSO6 |
1615 NETIF_F_TSO_ECN;
1616
1617 err = ionic_set_nic_features(lif, features);
1618 if (err)
1619 return err;
1620
1621 /* tell the netdev what we actually can support */
1622 netdev->features |= NETIF_F_HIGHDMA;
1623
1624 if (lif->hw_features & IONIC_ETH_HW_VLAN_TX_TAG)
1625 netdev->hw_features |= NETIF_F_HW_VLAN_CTAG_TX;
1626 if (lif->hw_features & IONIC_ETH_HW_VLAN_RX_STRIP)
1627 netdev->hw_features |= NETIF_F_HW_VLAN_CTAG_RX;
1628 if (lif->hw_features & IONIC_ETH_HW_VLAN_RX_FILTER)
1629 netdev->hw_features |= NETIF_F_HW_VLAN_CTAG_FILTER;
1630 if (lif->hw_features & IONIC_ETH_HW_RX_HASH)
1631 netdev->hw_features |= NETIF_F_RXHASH;
1632 if (lif->hw_features & IONIC_ETH_HW_TX_SG)
1633 netdev->hw_features |= NETIF_F_SG;
1634
1635 if (lif->hw_features & IONIC_ETH_HW_TX_CSUM)
1636 netdev->hw_enc_features |= NETIF_F_HW_CSUM;
1637 if (lif->hw_features & IONIC_ETH_HW_RX_CSUM)
1638 netdev->hw_enc_features |= NETIF_F_RXCSUM;
1639 if (lif->hw_features & IONIC_ETH_HW_TSO)
1640 netdev->hw_enc_features |= NETIF_F_TSO;
1641 if (lif->hw_features & IONIC_ETH_HW_TSO_IPV6)
1642 netdev->hw_enc_features |= NETIF_F_TSO6;
1643 if (lif->hw_features & IONIC_ETH_HW_TSO_ECN)
1644 netdev->hw_enc_features |= NETIF_F_TSO_ECN;
1645 if (lif->hw_features & IONIC_ETH_HW_TSO_GRE)
1646 netdev->hw_enc_features |= NETIF_F_GSO_GRE;
1647 if (lif->hw_features & IONIC_ETH_HW_TSO_GRE_CSUM)
1648 netdev->hw_enc_features |= NETIF_F_GSO_GRE_CSUM;
1649 if (lif->hw_features & IONIC_ETH_HW_TSO_IPXIP4)
1650 netdev->hw_enc_features |= NETIF_F_GSO_IPXIP4;
1651 if (lif->hw_features & IONIC_ETH_HW_TSO_IPXIP6)
1652 netdev->hw_enc_features |= NETIF_F_GSO_IPXIP6;
1653 if (lif->hw_features & IONIC_ETH_HW_TSO_UDP)
1654 netdev->hw_enc_features |= NETIF_F_GSO_UDP_TUNNEL;
1655 if (lif->hw_features & IONIC_ETH_HW_TSO_UDP_CSUM)
1656 netdev->hw_enc_features |= NETIF_F_GSO_UDP_TUNNEL_CSUM;
1657
1658 netdev->hw_features |= netdev->hw_enc_features;
1659 netdev->features |= netdev->hw_features;
1660 netdev->vlan_features |= netdev->features & ~NETIF_F_VLAN_FEATURES;
1661
1662 netdev->priv_flags |= IFF_UNICAST_FLT |
1663 IFF_LIVE_ADDR_CHANGE;
1664
1665 return 0;
1666 }
1667
ionic_set_features(struct net_device * netdev,netdev_features_t features)1668 static int ionic_set_features(struct net_device *netdev,
1669 netdev_features_t features)
1670 {
1671 struct ionic_lif *lif = netdev_priv(netdev);
1672 int err;
1673
1674 netdev_dbg(netdev, "%s: lif->features=0x%08llx new_features=0x%08llx\n",
1675 __func__, (u64)lif->netdev->features, (u64)features);
1676
1677 err = ionic_set_nic_features(lif, features);
1678
1679 return err;
1680 }
1681
ionic_set_mac_address(struct net_device * netdev,void * sa)1682 static int ionic_set_mac_address(struct net_device *netdev, void *sa)
1683 {
1684 struct sockaddr *addr = sa;
1685 u8 *mac;
1686 int err;
1687
1688 mac = (u8 *)addr->sa_data;
1689 if (ether_addr_equal(netdev->dev_addr, mac))
1690 return 0;
1691
1692 err = eth_prepare_mac_addr_change(netdev, addr);
1693 if (err)
1694 return err;
1695
1696 if (!is_zero_ether_addr(netdev->dev_addr)) {
1697 netdev_info(netdev, "deleting mac addr %pM\n",
1698 netdev->dev_addr);
1699 ionic_addr_del(netdev, netdev->dev_addr);
1700 }
1701
1702 eth_commit_mac_addr_change(netdev, addr);
1703 netdev_info(netdev, "updating mac addr %pM\n", mac);
1704
1705 return ionic_addr_add(netdev, mac);
1706 }
1707
ionic_stop_queues_reconfig(struct ionic_lif * lif)1708 static void ionic_stop_queues_reconfig(struct ionic_lif *lif)
1709 {
1710 /* Stop and clean the queues before reconfiguration */
1711 mutex_lock(&lif->queue_lock);
1712 netif_device_detach(lif->netdev);
1713 ionic_stop_queues(lif);
1714 ionic_txrx_deinit(lif);
1715 }
1716
ionic_start_queues_reconfig(struct ionic_lif * lif)1717 static int ionic_start_queues_reconfig(struct ionic_lif *lif)
1718 {
1719 int err;
1720
1721 /* Re-init the queues after reconfiguration */
1722
1723 /* The only way txrx_init can fail here is if communication
1724 * with FW is suddenly broken. There's not much we can do
1725 * at this point - error messages have already been printed,
1726 * so we can continue on and the user can eventually do a
1727 * DOWN and UP to try to reset and clear the issue.
1728 */
1729 err = ionic_txrx_init(lif);
1730 mutex_unlock(&lif->queue_lock);
1731 ionic_link_status_check_request(lif, CAN_SLEEP);
1732 netif_device_attach(lif->netdev);
1733
1734 return err;
1735 }
1736
ionic_change_mtu(struct net_device * netdev,int new_mtu)1737 static int ionic_change_mtu(struct net_device *netdev, int new_mtu)
1738 {
1739 struct ionic_lif *lif = netdev_priv(netdev);
1740 struct ionic_admin_ctx ctx = {
1741 .work = COMPLETION_INITIALIZER_ONSTACK(ctx.work),
1742 .cmd.lif_setattr = {
1743 .opcode = IONIC_CMD_LIF_SETATTR,
1744 .index = cpu_to_le16(lif->index),
1745 .attr = IONIC_LIF_ATTR_MTU,
1746 .mtu = cpu_to_le32(new_mtu),
1747 },
1748 };
1749 int err;
1750
1751 err = ionic_adminq_post_wait(lif, &ctx);
1752 if (err)
1753 return err;
1754
1755 /* if we're not running, nothing more to do */
1756 if (!netif_running(netdev)) {
1757 netdev->mtu = new_mtu;
1758 return 0;
1759 }
1760
1761 ionic_stop_queues_reconfig(lif);
1762 netdev->mtu = new_mtu;
1763 return ionic_start_queues_reconfig(lif);
1764 }
1765
ionic_tx_timeout_work(struct work_struct * ws)1766 static void ionic_tx_timeout_work(struct work_struct *ws)
1767 {
1768 struct ionic_lif *lif = container_of(ws, struct ionic_lif, tx_timeout_work);
1769
1770 if (test_bit(IONIC_LIF_F_FW_RESET, lif->state))
1771 return;
1772
1773 /* if we were stopped before this scheduled job was launched,
1774 * don't bother the queues as they are already stopped.
1775 */
1776 if (!netif_running(lif->netdev))
1777 return;
1778
1779 ionic_stop_queues_reconfig(lif);
1780 ionic_start_queues_reconfig(lif);
1781 }
1782
ionic_tx_timeout(struct net_device * netdev,unsigned int txqueue)1783 static void ionic_tx_timeout(struct net_device *netdev, unsigned int txqueue)
1784 {
1785 struct ionic_lif *lif = netdev_priv(netdev);
1786
1787 netdev_info(lif->netdev, "Tx Timeout triggered - txq %d\n", txqueue);
1788 schedule_work(&lif->tx_timeout_work);
1789 }
1790
ionic_vlan_rx_add_vid(struct net_device * netdev,__be16 proto,u16 vid)1791 static int ionic_vlan_rx_add_vid(struct net_device *netdev, __be16 proto,
1792 u16 vid)
1793 {
1794 struct ionic_lif *lif = netdev_priv(netdev);
1795 struct ionic_admin_ctx ctx = {
1796 .work = COMPLETION_INITIALIZER_ONSTACK(ctx.work),
1797 .cmd.rx_filter_add = {
1798 .opcode = IONIC_CMD_RX_FILTER_ADD,
1799 .lif_index = cpu_to_le16(lif->index),
1800 .match = cpu_to_le16(IONIC_RX_FILTER_MATCH_VLAN),
1801 .vlan.vlan = cpu_to_le16(vid),
1802 },
1803 };
1804 int err;
1805
1806 netdev_dbg(netdev, "rx_filter add VLAN %d\n", vid);
1807 err = ionic_adminq_post_wait(lif, &ctx);
1808 if (err)
1809 return err;
1810
1811 return ionic_rx_filter_save(lif, 0, IONIC_RXQ_INDEX_ANY, 0, &ctx);
1812 }
1813
ionic_vlan_rx_kill_vid(struct net_device * netdev,__be16 proto,u16 vid)1814 static int ionic_vlan_rx_kill_vid(struct net_device *netdev, __be16 proto,
1815 u16 vid)
1816 {
1817 struct ionic_lif *lif = netdev_priv(netdev);
1818 struct ionic_admin_ctx ctx = {
1819 .work = COMPLETION_INITIALIZER_ONSTACK(ctx.work),
1820 .cmd.rx_filter_del = {
1821 .opcode = IONIC_CMD_RX_FILTER_DEL,
1822 .lif_index = cpu_to_le16(lif->index),
1823 },
1824 };
1825 struct ionic_rx_filter *f;
1826
1827 spin_lock_bh(&lif->rx_filters.lock);
1828
1829 f = ionic_rx_filter_by_vlan(lif, vid);
1830 if (!f) {
1831 spin_unlock_bh(&lif->rx_filters.lock);
1832 return -ENOENT;
1833 }
1834
1835 netdev_dbg(netdev, "rx_filter del VLAN %d (id %d)\n",
1836 vid, f->filter_id);
1837
1838 ctx.cmd.rx_filter_del.filter_id = cpu_to_le32(f->filter_id);
1839 ionic_rx_filter_free(lif, f);
1840 spin_unlock_bh(&lif->rx_filters.lock);
1841
1842 return ionic_adminq_post_wait(lif, &ctx);
1843 }
1844
ionic_lif_rss_config(struct ionic_lif * lif,const u16 types,const u8 * key,const u32 * indir)1845 int ionic_lif_rss_config(struct ionic_lif *lif, const u16 types,
1846 const u8 *key, const u32 *indir)
1847 {
1848 struct ionic_admin_ctx ctx = {
1849 .work = COMPLETION_INITIALIZER_ONSTACK(ctx.work),
1850 .cmd.lif_setattr = {
1851 .opcode = IONIC_CMD_LIF_SETATTR,
1852 .attr = IONIC_LIF_ATTR_RSS,
1853 .rss.addr = cpu_to_le64(lif->rss_ind_tbl_pa),
1854 },
1855 };
1856 unsigned int i, tbl_sz;
1857
1858 if (lif->hw_features & IONIC_ETH_HW_RX_HASH) {
1859 lif->rss_types = types;
1860 ctx.cmd.lif_setattr.rss.types = cpu_to_le16(types);
1861 }
1862
1863 if (key)
1864 memcpy(lif->rss_hash_key, key, IONIC_RSS_HASH_KEY_SIZE);
1865
1866 if (indir) {
1867 tbl_sz = le16_to_cpu(lif->ionic->ident.lif.eth.rss_ind_tbl_sz);
1868 for (i = 0; i < tbl_sz; i++)
1869 lif->rss_ind_tbl[i] = indir[i];
1870 }
1871
1872 memcpy(ctx.cmd.lif_setattr.rss.key, lif->rss_hash_key,
1873 IONIC_RSS_HASH_KEY_SIZE);
1874
1875 return ionic_adminq_post_wait(lif, &ctx);
1876 }
1877
ionic_lif_rss_init(struct ionic_lif * lif)1878 static int ionic_lif_rss_init(struct ionic_lif *lif)
1879 {
1880 unsigned int tbl_sz;
1881 unsigned int i;
1882
1883 lif->rss_types = IONIC_RSS_TYPE_IPV4 |
1884 IONIC_RSS_TYPE_IPV4_TCP |
1885 IONIC_RSS_TYPE_IPV4_UDP |
1886 IONIC_RSS_TYPE_IPV6 |
1887 IONIC_RSS_TYPE_IPV6_TCP |
1888 IONIC_RSS_TYPE_IPV6_UDP;
1889
1890 /* Fill indirection table with 'default' values */
1891 tbl_sz = le16_to_cpu(lif->ionic->ident.lif.eth.rss_ind_tbl_sz);
1892 for (i = 0; i < tbl_sz; i++)
1893 lif->rss_ind_tbl[i] = ethtool_rxfh_indir_default(i, lif->nxqs);
1894
1895 return ionic_lif_rss_config(lif, lif->rss_types, NULL, NULL);
1896 }
1897
ionic_lif_rss_deinit(struct ionic_lif * lif)1898 static void ionic_lif_rss_deinit(struct ionic_lif *lif)
1899 {
1900 int tbl_sz;
1901
1902 tbl_sz = le16_to_cpu(lif->ionic->ident.lif.eth.rss_ind_tbl_sz);
1903 memset(lif->rss_ind_tbl, 0, tbl_sz);
1904 memset(lif->rss_hash_key, 0, IONIC_RSS_HASH_KEY_SIZE);
1905
1906 ionic_lif_rss_config(lif, 0x0, NULL, NULL);
1907 }
1908
ionic_lif_quiesce(struct ionic_lif * lif)1909 static void ionic_lif_quiesce(struct ionic_lif *lif)
1910 {
1911 struct ionic_admin_ctx ctx = {
1912 .work = COMPLETION_INITIALIZER_ONSTACK(ctx.work),
1913 .cmd.lif_setattr = {
1914 .opcode = IONIC_CMD_LIF_SETATTR,
1915 .index = cpu_to_le16(lif->index),
1916 .attr = IONIC_LIF_ATTR_STATE,
1917 .state = IONIC_LIF_QUIESCE,
1918 },
1919 };
1920 int err;
1921
1922 err = ionic_adminq_post_wait(lif, &ctx);
1923 if (err)
1924 netdev_err(lif->netdev, "lif quiesce failed %d\n", err);
1925 }
1926
ionic_txrx_disable(struct ionic_lif * lif)1927 static void ionic_txrx_disable(struct ionic_lif *lif)
1928 {
1929 unsigned int i;
1930 int err = 0;
1931
1932 if (lif->txqcqs) {
1933 for (i = 0; i < lif->nxqs; i++)
1934 err = ionic_qcq_disable(lif->txqcqs[i], (err != -ETIMEDOUT));
1935 }
1936
1937 if (lif->hwstamp_txq)
1938 err = ionic_qcq_disable(lif->hwstamp_txq, (err != -ETIMEDOUT));
1939
1940 if (lif->rxqcqs) {
1941 for (i = 0; i < lif->nxqs; i++)
1942 err = ionic_qcq_disable(lif->rxqcqs[i], (err != -ETIMEDOUT));
1943 }
1944
1945 if (lif->hwstamp_rxq)
1946 err = ionic_qcq_disable(lif->hwstamp_rxq, (err != -ETIMEDOUT));
1947
1948 ionic_lif_quiesce(lif);
1949 }
1950
ionic_txrx_deinit(struct ionic_lif * lif)1951 static void ionic_txrx_deinit(struct ionic_lif *lif)
1952 {
1953 unsigned int i;
1954
1955 if (lif->txqcqs) {
1956 for (i = 0; i < lif->nxqs && lif->txqcqs[i]; i++) {
1957 ionic_lif_qcq_deinit(lif, lif->txqcqs[i]);
1958 ionic_tx_flush(&lif->txqcqs[i]->cq);
1959 ionic_tx_empty(&lif->txqcqs[i]->q);
1960 }
1961 }
1962
1963 if (lif->rxqcqs) {
1964 for (i = 0; i < lif->nxqs && lif->rxqcqs[i]; i++) {
1965 ionic_lif_qcq_deinit(lif, lif->rxqcqs[i]);
1966 ionic_rx_empty(&lif->rxqcqs[i]->q);
1967 }
1968 }
1969 lif->rx_mode = 0;
1970
1971 if (lif->hwstamp_txq) {
1972 ionic_lif_qcq_deinit(lif, lif->hwstamp_txq);
1973 ionic_tx_flush(&lif->hwstamp_txq->cq);
1974 ionic_tx_empty(&lif->hwstamp_txq->q);
1975 }
1976
1977 if (lif->hwstamp_rxq) {
1978 ionic_lif_qcq_deinit(lif, lif->hwstamp_rxq);
1979 ionic_rx_empty(&lif->hwstamp_rxq->q);
1980 }
1981 }
1982
ionic_txrx_free(struct ionic_lif * lif)1983 static void ionic_txrx_free(struct ionic_lif *lif)
1984 {
1985 unsigned int i;
1986
1987 if (lif->txqcqs) {
1988 for (i = 0; i < lif->ionic->ntxqs_per_lif && lif->txqcqs[i]; i++) {
1989 ionic_qcq_free(lif, lif->txqcqs[i]);
1990 devm_kfree(lif->ionic->dev, lif->txqcqs[i]);
1991 lif->txqcqs[i] = NULL;
1992 }
1993 }
1994
1995 if (lif->rxqcqs) {
1996 for (i = 0; i < lif->ionic->nrxqs_per_lif && lif->rxqcqs[i]; i++) {
1997 ionic_qcq_free(lif, lif->rxqcqs[i]);
1998 devm_kfree(lif->ionic->dev, lif->rxqcqs[i]);
1999 lif->rxqcqs[i] = NULL;
2000 }
2001 }
2002
2003 if (lif->hwstamp_txq) {
2004 ionic_qcq_free(lif, lif->hwstamp_txq);
2005 devm_kfree(lif->ionic->dev, lif->hwstamp_txq);
2006 lif->hwstamp_txq = NULL;
2007 }
2008
2009 if (lif->hwstamp_rxq) {
2010 ionic_qcq_free(lif, lif->hwstamp_rxq);
2011 devm_kfree(lif->ionic->dev, lif->hwstamp_rxq);
2012 lif->hwstamp_rxq = NULL;
2013 }
2014 }
2015
ionic_txrx_alloc(struct ionic_lif * lif)2016 static int ionic_txrx_alloc(struct ionic_lif *lif)
2017 {
2018 unsigned int comp_sz, desc_sz, num_desc, sg_desc_sz;
2019 unsigned int flags, i;
2020 int err = 0;
2021
2022 num_desc = lif->ntxq_descs;
2023 desc_sz = sizeof(struct ionic_txq_desc);
2024 comp_sz = sizeof(struct ionic_txq_comp);
2025
2026 if (lif->qtype_info[IONIC_QTYPE_TXQ].version >= 1 &&
2027 lif->qtype_info[IONIC_QTYPE_TXQ].sg_desc_sz ==
2028 sizeof(struct ionic_txq_sg_desc_v1))
2029 sg_desc_sz = sizeof(struct ionic_txq_sg_desc_v1);
2030 else
2031 sg_desc_sz = sizeof(struct ionic_txq_sg_desc);
2032
2033 flags = IONIC_QCQ_F_TX_STATS | IONIC_QCQ_F_SG;
2034 if (test_bit(IONIC_LIF_F_SPLIT_INTR, lif->state))
2035 flags |= IONIC_QCQ_F_INTR;
2036 for (i = 0; i < lif->nxqs; i++) {
2037 err = ionic_qcq_alloc(lif, IONIC_QTYPE_TXQ, i, "tx", flags,
2038 num_desc, desc_sz, comp_sz, sg_desc_sz,
2039 lif->kern_pid, &lif->txqcqs[i]);
2040 if (err)
2041 goto err_out;
2042
2043 if (flags & IONIC_QCQ_F_INTR) {
2044 ionic_intr_coal_init(lif->ionic->idev.intr_ctrl,
2045 lif->txqcqs[i]->intr.index,
2046 lif->tx_coalesce_hw);
2047 if (test_bit(IONIC_LIF_F_TX_DIM_INTR, lif->state))
2048 lif->txqcqs[i]->intr.dim_coal_hw = lif->tx_coalesce_hw;
2049 }
2050
2051 ionic_debugfs_add_qcq(lif, lif->txqcqs[i]);
2052 }
2053
2054 flags = IONIC_QCQ_F_RX_STATS | IONIC_QCQ_F_SG | IONIC_QCQ_F_INTR;
2055
2056 num_desc = lif->nrxq_descs;
2057 desc_sz = sizeof(struct ionic_rxq_desc);
2058 comp_sz = sizeof(struct ionic_rxq_comp);
2059 sg_desc_sz = sizeof(struct ionic_rxq_sg_desc);
2060
2061 if (lif->rxq_features & IONIC_Q_F_2X_CQ_DESC)
2062 comp_sz *= 2;
2063
2064 for (i = 0; i < lif->nxqs; i++) {
2065 err = ionic_qcq_alloc(lif, IONIC_QTYPE_RXQ, i, "rx", flags,
2066 num_desc, desc_sz, comp_sz, sg_desc_sz,
2067 lif->kern_pid, &lif->rxqcqs[i]);
2068 if (err)
2069 goto err_out;
2070
2071 lif->rxqcqs[i]->q.features = lif->rxq_features;
2072
2073 ionic_intr_coal_init(lif->ionic->idev.intr_ctrl,
2074 lif->rxqcqs[i]->intr.index,
2075 lif->rx_coalesce_hw);
2076 if (test_bit(IONIC_LIF_F_RX_DIM_INTR, lif->state))
2077 lif->rxqcqs[i]->intr.dim_coal_hw = lif->rx_coalesce_hw;
2078
2079 if (!test_bit(IONIC_LIF_F_SPLIT_INTR, lif->state))
2080 ionic_link_qcq_interrupts(lif->rxqcqs[i],
2081 lif->txqcqs[i]);
2082
2083 ionic_debugfs_add_qcq(lif, lif->rxqcqs[i]);
2084 }
2085
2086 return 0;
2087
2088 err_out:
2089 ionic_txrx_free(lif);
2090
2091 return err;
2092 }
2093
ionic_txrx_init(struct ionic_lif * lif)2094 static int ionic_txrx_init(struct ionic_lif *lif)
2095 {
2096 unsigned int i;
2097 int err;
2098
2099 for (i = 0; i < lif->nxqs; i++) {
2100 err = ionic_lif_txq_init(lif, lif->txqcqs[i]);
2101 if (err)
2102 goto err_out;
2103
2104 err = ionic_lif_rxq_init(lif, lif->rxqcqs[i]);
2105 if (err) {
2106 ionic_lif_qcq_deinit(lif, lif->txqcqs[i]);
2107 goto err_out;
2108 }
2109 }
2110
2111 if (lif->netdev->features & NETIF_F_RXHASH)
2112 ionic_lif_rss_init(lif);
2113
2114 ionic_set_rx_mode(lif->netdev, CAN_SLEEP);
2115
2116 return 0;
2117
2118 err_out:
2119 while (i--) {
2120 ionic_lif_qcq_deinit(lif, lif->txqcqs[i]);
2121 ionic_lif_qcq_deinit(lif, lif->rxqcqs[i]);
2122 }
2123
2124 return err;
2125 }
2126
ionic_txrx_enable(struct ionic_lif * lif)2127 static int ionic_txrx_enable(struct ionic_lif *lif)
2128 {
2129 int derr = 0;
2130 int i, err;
2131
2132 for (i = 0; i < lif->nxqs; i++) {
2133 if (!(lif->rxqcqs[i] && lif->txqcqs[i])) {
2134 dev_err(lif->ionic->dev, "%s: bad qcq %d\n", __func__, i);
2135 err = -ENXIO;
2136 goto err_out;
2137 }
2138
2139 ionic_rx_fill(&lif->rxqcqs[i]->q);
2140 err = ionic_qcq_enable(lif->rxqcqs[i]);
2141 if (err)
2142 goto err_out;
2143
2144 err = ionic_qcq_enable(lif->txqcqs[i]);
2145 if (err) {
2146 derr = ionic_qcq_disable(lif->rxqcqs[i], (err != -ETIMEDOUT));
2147 goto err_out;
2148 }
2149 }
2150
2151 if (lif->hwstamp_rxq) {
2152 ionic_rx_fill(&lif->hwstamp_rxq->q);
2153 err = ionic_qcq_enable(lif->hwstamp_rxq);
2154 if (err)
2155 goto err_out_hwstamp_rx;
2156 }
2157
2158 if (lif->hwstamp_txq) {
2159 err = ionic_qcq_enable(lif->hwstamp_txq);
2160 if (err)
2161 goto err_out_hwstamp_tx;
2162 }
2163
2164 return 0;
2165
2166 err_out_hwstamp_tx:
2167 if (lif->hwstamp_rxq)
2168 derr = ionic_qcq_disable(lif->hwstamp_rxq, (derr != -ETIMEDOUT));
2169 err_out_hwstamp_rx:
2170 i = lif->nxqs;
2171 err_out:
2172 while (i--) {
2173 derr = ionic_qcq_disable(lif->txqcqs[i], (derr != -ETIMEDOUT));
2174 derr = ionic_qcq_disable(lif->rxqcqs[i], (derr != -ETIMEDOUT));
2175 }
2176
2177 return err;
2178 }
2179
ionic_start_queues(struct ionic_lif * lif)2180 static int ionic_start_queues(struct ionic_lif *lif)
2181 {
2182 int err;
2183
2184 if (test_bit(IONIC_LIF_F_BROKEN, lif->state))
2185 return -EIO;
2186
2187 if (test_bit(IONIC_LIF_F_FW_RESET, lif->state))
2188 return -EBUSY;
2189
2190 if (test_and_set_bit(IONIC_LIF_F_UP, lif->state))
2191 return 0;
2192
2193 err = ionic_txrx_enable(lif);
2194 if (err) {
2195 clear_bit(IONIC_LIF_F_UP, lif->state);
2196 return err;
2197 }
2198 netif_tx_wake_all_queues(lif->netdev);
2199
2200 return 0;
2201 }
2202
ionic_open(struct net_device * netdev)2203 static int ionic_open(struct net_device *netdev)
2204 {
2205 struct ionic_lif *lif = netdev_priv(netdev);
2206 int err;
2207
2208 /* If recovering from a broken state, clear the bit and we'll try again */
2209 if (test_and_clear_bit(IONIC_LIF_F_BROKEN, lif->state))
2210 netdev_info(netdev, "clearing broken state\n");
2211
2212 err = ionic_txrx_alloc(lif);
2213 if (err)
2214 return err;
2215
2216 err = ionic_txrx_init(lif);
2217 if (err)
2218 goto err_txrx_free;
2219
2220 err = netif_set_real_num_tx_queues(netdev, lif->nxqs);
2221 if (err)
2222 goto err_txrx_deinit;
2223
2224 err = netif_set_real_num_rx_queues(netdev, lif->nxqs);
2225 if (err)
2226 goto err_txrx_deinit;
2227
2228 /* don't start the queues until we have link */
2229 if (netif_carrier_ok(netdev)) {
2230 err = ionic_start_queues(lif);
2231 if (err)
2232 goto err_txrx_deinit;
2233 }
2234
2235 return 0;
2236
2237 err_txrx_deinit:
2238 ionic_txrx_deinit(lif);
2239 err_txrx_free:
2240 ionic_txrx_free(lif);
2241 return err;
2242 }
2243
ionic_stop_queues(struct ionic_lif * lif)2244 static void ionic_stop_queues(struct ionic_lif *lif)
2245 {
2246 if (!test_and_clear_bit(IONIC_LIF_F_UP, lif->state))
2247 return;
2248
2249 netif_tx_disable(lif->netdev);
2250 ionic_txrx_disable(lif);
2251 }
2252
ionic_stop(struct net_device * netdev)2253 static int ionic_stop(struct net_device *netdev)
2254 {
2255 struct ionic_lif *lif = netdev_priv(netdev);
2256
2257 if (test_bit(IONIC_LIF_F_FW_RESET, lif->state))
2258 return 0;
2259
2260 ionic_stop_queues(lif);
2261 ionic_txrx_deinit(lif);
2262 ionic_txrx_free(lif);
2263
2264 return 0;
2265 }
2266
ionic_do_ioctl(struct net_device * netdev,struct ifreq * ifr,int cmd)2267 static int ionic_do_ioctl(struct net_device *netdev, struct ifreq *ifr, int cmd)
2268 {
2269 struct ionic_lif *lif = netdev_priv(netdev);
2270
2271 switch (cmd) {
2272 case SIOCSHWTSTAMP:
2273 return ionic_lif_hwstamp_set(lif, ifr);
2274 case SIOCGHWTSTAMP:
2275 return ionic_lif_hwstamp_get(lif, ifr);
2276 default:
2277 return -EOPNOTSUPP;
2278 }
2279 }
2280
ionic_get_vf_config(struct net_device * netdev,int vf,struct ifla_vf_info * ivf)2281 static int ionic_get_vf_config(struct net_device *netdev,
2282 int vf, struct ifla_vf_info *ivf)
2283 {
2284 struct ionic_lif *lif = netdev_priv(netdev);
2285 struct ionic *ionic = lif->ionic;
2286 int ret = 0;
2287
2288 if (!netif_device_present(netdev))
2289 return -EBUSY;
2290
2291 down_read(&ionic->vf_op_lock);
2292
2293 if (vf >= pci_num_vf(ionic->pdev) || !ionic->vfs) {
2294 ret = -EINVAL;
2295 } else {
2296 ivf->vf = vf;
2297 ivf->vlan = le16_to_cpu(ionic->vfs[vf].vlanid);
2298 ivf->qos = 0;
2299 ivf->spoofchk = ionic->vfs[vf].spoofchk;
2300 ivf->linkstate = ionic->vfs[vf].linkstate;
2301 ivf->max_tx_rate = le32_to_cpu(ionic->vfs[vf].maxrate);
2302 ivf->trusted = ionic->vfs[vf].trusted;
2303 ether_addr_copy(ivf->mac, ionic->vfs[vf].macaddr);
2304 }
2305
2306 up_read(&ionic->vf_op_lock);
2307 return ret;
2308 }
2309
ionic_get_vf_stats(struct net_device * netdev,int vf,struct ifla_vf_stats * vf_stats)2310 static int ionic_get_vf_stats(struct net_device *netdev, int vf,
2311 struct ifla_vf_stats *vf_stats)
2312 {
2313 struct ionic_lif *lif = netdev_priv(netdev);
2314 struct ionic *ionic = lif->ionic;
2315 struct ionic_lif_stats *vs;
2316 int ret = 0;
2317
2318 if (!netif_device_present(netdev))
2319 return -EBUSY;
2320
2321 down_read(&ionic->vf_op_lock);
2322
2323 if (vf >= pci_num_vf(ionic->pdev) || !ionic->vfs) {
2324 ret = -EINVAL;
2325 } else {
2326 memset(vf_stats, 0, sizeof(*vf_stats));
2327 vs = &ionic->vfs[vf].stats;
2328
2329 vf_stats->rx_packets = le64_to_cpu(vs->rx_ucast_packets);
2330 vf_stats->tx_packets = le64_to_cpu(vs->tx_ucast_packets);
2331 vf_stats->rx_bytes = le64_to_cpu(vs->rx_ucast_bytes);
2332 vf_stats->tx_bytes = le64_to_cpu(vs->tx_ucast_bytes);
2333 vf_stats->broadcast = le64_to_cpu(vs->rx_bcast_packets);
2334 vf_stats->multicast = le64_to_cpu(vs->rx_mcast_packets);
2335 vf_stats->rx_dropped = le64_to_cpu(vs->rx_ucast_drop_packets) +
2336 le64_to_cpu(vs->rx_mcast_drop_packets) +
2337 le64_to_cpu(vs->rx_bcast_drop_packets);
2338 vf_stats->tx_dropped = le64_to_cpu(vs->tx_ucast_drop_packets) +
2339 le64_to_cpu(vs->tx_mcast_drop_packets) +
2340 le64_to_cpu(vs->tx_bcast_drop_packets);
2341 }
2342
2343 up_read(&ionic->vf_op_lock);
2344 return ret;
2345 }
2346
ionic_set_vf_mac(struct net_device * netdev,int vf,u8 * mac)2347 static int ionic_set_vf_mac(struct net_device *netdev, int vf, u8 *mac)
2348 {
2349 struct ionic_lif *lif = netdev_priv(netdev);
2350 struct ionic *ionic = lif->ionic;
2351 int ret;
2352
2353 if (!(is_zero_ether_addr(mac) || is_valid_ether_addr(mac)))
2354 return -EINVAL;
2355
2356 if (!netif_device_present(netdev))
2357 return -EBUSY;
2358
2359 down_write(&ionic->vf_op_lock);
2360
2361 if (vf >= pci_num_vf(ionic->pdev) || !ionic->vfs) {
2362 ret = -EINVAL;
2363 } else {
2364 ret = ionic_set_vf_config(ionic, vf, IONIC_VF_ATTR_MAC, mac);
2365 if (!ret)
2366 ether_addr_copy(ionic->vfs[vf].macaddr, mac);
2367 }
2368
2369 up_write(&ionic->vf_op_lock);
2370 return ret;
2371 }
2372
ionic_set_vf_vlan(struct net_device * netdev,int vf,u16 vlan,u8 qos,__be16 proto)2373 static int ionic_set_vf_vlan(struct net_device *netdev, int vf, u16 vlan,
2374 u8 qos, __be16 proto)
2375 {
2376 struct ionic_lif *lif = netdev_priv(netdev);
2377 struct ionic *ionic = lif->ionic;
2378 int ret;
2379
2380 /* until someday when we support qos */
2381 if (qos)
2382 return -EINVAL;
2383
2384 if (vlan > 4095)
2385 return -EINVAL;
2386
2387 if (proto != htons(ETH_P_8021Q))
2388 return -EPROTONOSUPPORT;
2389
2390 if (!netif_device_present(netdev))
2391 return -EBUSY;
2392
2393 down_write(&ionic->vf_op_lock);
2394
2395 if (vf >= pci_num_vf(ionic->pdev) || !ionic->vfs) {
2396 ret = -EINVAL;
2397 } else {
2398 ret = ionic_set_vf_config(ionic, vf,
2399 IONIC_VF_ATTR_VLAN, (u8 *)&vlan);
2400 if (!ret)
2401 ionic->vfs[vf].vlanid = cpu_to_le16(vlan);
2402 }
2403
2404 up_write(&ionic->vf_op_lock);
2405 return ret;
2406 }
2407
ionic_set_vf_rate(struct net_device * netdev,int vf,int tx_min,int tx_max)2408 static int ionic_set_vf_rate(struct net_device *netdev, int vf,
2409 int tx_min, int tx_max)
2410 {
2411 struct ionic_lif *lif = netdev_priv(netdev);
2412 struct ionic *ionic = lif->ionic;
2413 int ret;
2414
2415 /* setting the min just seems silly */
2416 if (tx_min)
2417 return -EINVAL;
2418
2419 if (!netif_device_present(netdev))
2420 return -EBUSY;
2421
2422 down_write(&ionic->vf_op_lock);
2423
2424 if (vf >= pci_num_vf(ionic->pdev) || !ionic->vfs) {
2425 ret = -EINVAL;
2426 } else {
2427 ret = ionic_set_vf_config(ionic, vf,
2428 IONIC_VF_ATTR_RATE, (u8 *)&tx_max);
2429 if (!ret)
2430 lif->ionic->vfs[vf].maxrate = cpu_to_le32(tx_max);
2431 }
2432
2433 up_write(&ionic->vf_op_lock);
2434 return ret;
2435 }
2436
ionic_set_vf_spoofchk(struct net_device * netdev,int vf,bool set)2437 static int ionic_set_vf_spoofchk(struct net_device *netdev, int vf, bool set)
2438 {
2439 struct ionic_lif *lif = netdev_priv(netdev);
2440 struct ionic *ionic = lif->ionic;
2441 u8 data = set; /* convert to u8 for config */
2442 int ret;
2443
2444 if (!netif_device_present(netdev))
2445 return -EBUSY;
2446
2447 down_write(&ionic->vf_op_lock);
2448
2449 if (vf >= pci_num_vf(ionic->pdev) || !ionic->vfs) {
2450 ret = -EINVAL;
2451 } else {
2452 ret = ionic_set_vf_config(ionic, vf,
2453 IONIC_VF_ATTR_SPOOFCHK, &data);
2454 if (!ret)
2455 ionic->vfs[vf].spoofchk = data;
2456 }
2457
2458 up_write(&ionic->vf_op_lock);
2459 return ret;
2460 }
2461
ionic_set_vf_trust(struct net_device * netdev,int vf,bool set)2462 static int ionic_set_vf_trust(struct net_device *netdev, int vf, bool set)
2463 {
2464 struct ionic_lif *lif = netdev_priv(netdev);
2465 struct ionic *ionic = lif->ionic;
2466 u8 data = set; /* convert to u8 for config */
2467 int ret;
2468
2469 if (!netif_device_present(netdev))
2470 return -EBUSY;
2471
2472 down_write(&ionic->vf_op_lock);
2473
2474 if (vf >= pci_num_vf(ionic->pdev) || !ionic->vfs) {
2475 ret = -EINVAL;
2476 } else {
2477 ret = ionic_set_vf_config(ionic, vf,
2478 IONIC_VF_ATTR_TRUST, &data);
2479 if (!ret)
2480 ionic->vfs[vf].trusted = data;
2481 }
2482
2483 up_write(&ionic->vf_op_lock);
2484 return ret;
2485 }
2486
ionic_set_vf_link_state(struct net_device * netdev,int vf,int set)2487 static int ionic_set_vf_link_state(struct net_device *netdev, int vf, int set)
2488 {
2489 struct ionic_lif *lif = netdev_priv(netdev);
2490 struct ionic *ionic = lif->ionic;
2491 u8 data;
2492 int ret;
2493
2494 switch (set) {
2495 case IFLA_VF_LINK_STATE_ENABLE:
2496 data = IONIC_VF_LINK_STATUS_UP;
2497 break;
2498 case IFLA_VF_LINK_STATE_DISABLE:
2499 data = IONIC_VF_LINK_STATUS_DOWN;
2500 break;
2501 case IFLA_VF_LINK_STATE_AUTO:
2502 data = IONIC_VF_LINK_STATUS_AUTO;
2503 break;
2504 default:
2505 return -EINVAL;
2506 }
2507
2508 if (!netif_device_present(netdev))
2509 return -EBUSY;
2510
2511 down_write(&ionic->vf_op_lock);
2512
2513 if (vf >= pci_num_vf(ionic->pdev) || !ionic->vfs) {
2514 ret = -EINVAL;
2515 } else {
2516 ret = ionic_set_vf_config(ionic, vf,
2517 IONIC_VF_ATTR_LINKSTATE, &data);
2518 if (!ret)
2519 ionic->vfs[vf].linkstate = set;
2520 }
2521
2522 up_write(&ionic->vf_op_lock);
2523 return ret;
2524 }
2525
2526 static const struct net_device_ops ionic_netdev_ops = {
2527 .ndo_open = ionic_open,
2528 .ndo_stop = ionic_stop,
2529 .ndo_do_ioctl = ionic_do_ioctl,
2530 .ndo_start_xmit = ionic_start_xmit,
2531 .ndo_get_stats64 = ionic_get_stats64,
2532 .ndo_set_rx_mode = ionic_ndo_set_rx_mode,
2533 .ndo_set_features = ionic_set_features,
2534 .ndo_set_mac_address = ionic_set_mac_address,
2535 .ndo_validate_addr = eth_validate_addr,
2536 .ndo_tx_timeout = ionic_tx_timeout,
2537 .ndo_change_mtu = ionic_change_mtu,
2538 .ndo_vlan_rx_add_vid = ionic_vlan_rx_add_vid,
2539 .ndo_vlan_rx_kill_vid = ionic_vlan_rx_kill_vid,
2540 .ndo_set_vf_vlan = ionic_set_vf_vlan,
2541 .ndo_set_vf_trust = ionic_set_vf_trust,
2542 .ndo_set_vf_mac = ionic_set_vf_mac,
2543 .ndo_set_vf_rate = ionic_set_vf_rate,
2544 .ndo_set_vf_spoofchk = ionic_set_vf_spoofchk,
2545 .ndo_get_vf_config = ionic_get_vf_config,
2546 .ndo_set_vf_link_state = ionic_set_vf_link_state,
2547 .ndo_get_vf_stats = ionic_get_vf_stats,
2548 };
2549
ionic_swap_queues(struct ionic_qcq * a,struct ionic_qcq * b)2550 static void ionic_swap_queues(struct ionic_qcq *a, struct ionic_qcq *b)
2551 {
2552 /* only swapping the queues, not the napi, flags, or other stuff */
2553 swap(a->q.features, b->q.features);
2554 swap(a->q.num_descs, b->q.num_descs);
2555 swap(a->q.desc_size, b->q.desc_size);
2556 swap(a->q.base, b->q.base);
2557 swap(a->q.base_pa, b->q.base_pa);
2558 swap(a->q.info, b->q.info);
2559 swap(a->q_base, b->q_base);
2560 swap(a->q_base_pa, b->q_base_pa);
2561 swap(a->q_size, b->q_size);
2562
2563 swap(a->q.sg_desc_size, b->q.sg_desc_size);
2564 swap(a->q.sg_base, b->q.sg_base);
2565 swap(a->q.sg_base_pa, b->q.sg_base_pa);
2566 swap(a->sg_base, b->sg_base);
2567 swap(a->sg_base_pa, b->sg_base_pa);
2568 swap(a->sg_size, b->sg_size);
2569
2570 swap(a->cq.num_descs, b->cq.num_descs);
2571 swap(a->cq.desc_size, b->cq.desc_size);
2572 swap(a->cq.base, b->cq.base);
2573 swap(a->cq.base_pa, b->cq.base_pa);
2574 swap(a->cq.info, b->cq.info);
2575 swap(a->cq_base, b->cq_base);
2576 swap(a->cq_base_pa, b->cq_base_pa);
2577 swap(a->cq_size, b->cq_size);
2578
2579 ionic_debugfs_del_qcq(a);
2580 ionic_debugfs_add_qcq(a->q.lif, a);
2581 }
2582
ionic_reconfigure_queues(struct ionic_lif * lif,struct ionic_queue_params * qparam)2583 int ionic_reconfigure_queues(struct ionic_lif *lif,
2584 struct ionic_queue_params *qparam)
2585 {
2586 unsigned int comp_sz, desc_sz, num_desc, sg_desc_sz;
2587 struct ionic_qcq **tx_qcqs = NULL;
2588 struct ionic_qcq **rx_qcqs = NULL;
2589 unsigned int flags, i;
2590 int err = -ENOMEM;
2591
2592 /* allocate temporary qcq arrays to hold new queue structs */
2593 if (qparam->nxqs != lif->nxqs || qparam->ntxq_descs != lif->ntxq_descs) {
2594 tx_qcqs = devm_kcalloc(lif->ionic->dev, lif->ionic->ntxqs_per_lif,
2595 sizeof(struct ionic_qcq *), GFP_KERNEL);
2596 if (!tx_qcqs)
2597 goto err_out;
2598 }
2599 if (qparam->nxqs != lif->nxqs ||
2600 qparam->nrxq_descs != lif->nrxq_descs ||
2601 qparam->rxq_features != lif->rxq_features) {
2602 rx_qcqs = devm_kcalloc(lif->ionic->dev, lif->ionic->nrxqs_per_lif,
2603 sizeof(struct ionic_qcq *), GFP_KERNEL);
2604 if (!rx_qcqs)
2605 goto err_out;
2606 }
2607
2608 /* allocate new desc_info and rings, but leave the interrupt setup
2609 * until later so as to not mess with the still-running queues
2610 */
2611 if (tx_qcqs) {
2612 num_desc = qparam->ntxq_descs;
2613 desc_sz = sizeof(struct ionic_txq_desc);
2614 comp_sz = sizeof(struct ionic_txq_comp);
2615
2616 if (lif->qtype_info[IONIC_QTYPE_TXQ].version >= 1 &&
2617 lif->qtype_info[IONIC_QTYPE_TXQ].sg_desc_sz ==
2618 sizeof(struct ionic_txq_sg_desc_v1))
2619 sg_desc_sz = sizeof(struct ionic_txq_sg_desc_v1);
2620 else
2621 sg_desc_sz = sizeof(struct ionic_txq_sg_desc);
2622
2623 for (i = 0; i < qparam->nxqs; i++) {
2624 flags = lif->txqcqs[i]->flags & ~IONIC_QCQ_F_INTR;
2625 err = ionic_qcq_alloc(lif, IONIC_QTYPE_TXQ, i, "tx", flags,
2626 num_desc, desc_sz, comp_sz, sg_desc_sz,
2627 lif->kern_pid, &tx_qcqs[i]);
2628 if (err)
2629 goto err_out;
2630 }
2631 }
2632
2633 if (rx_qcqs) {
2634 num_desc = qparam->nrxq_descs;
2635 desc_sz = sizeof(struct ionic_rxq_desc);
2636 comp_sz = sizeof(struct ionic_rxq_comp);
2637 sg_desc_sz = sizeof(struct ionic_rxq_sg_desc);
2638
2639 if (qparam->rxq_features & IONIC_Q_F_2X_CQ_DESC)
2640 comp_sz *= 2;
2641
2642 for (i = 0; i < qparam->nxqs; i++) {
2643 flags = lif->rxqcqs[i]->flags & ~IONIC_QCQ_F_INTR;
2644 err = ionic_qcq_alloc(lif, IONIC_QTYPE_RXQ, i, "rx", flags,
2645 num_desc, desc_sz, comp_sz, sg_desc_sz,
2646 lif->kern_pid, &rx_qcqs[i]);
2647 if (err)
2648 goto err_out;
2649
2650 rx_qcqs[i]->q.features = qparam->rxq_features;
2651 }
2652 }
2653
2654 /* stop and clean the queues */
2655 ionic_stop_queues_reconfig(lif);
2656
2657 if (qparam->nxqs != lif->nxqs) {
2658 err = netif_set_real_num_tx_queues(lif->netdev, qparam->nxqs);
2659 if (err)
2660 goto err_out_reinit_unlock;
2661 err = netif_set_real_num_rx_queues(lif->netdev, qparam->nxqs);
2662 if (err) {
2663 netif_set_real_num_tx_queues(lif->netdev, lif->nxqs);
2664 goto err_out_reinit_unlock;
2665 }
2666 }
2667
2668 /* swap new desc_info and rings, keeping existing interrupt config */
2669 if (tx_qcqs) {
2670 lif->ntxq_descs = qparam->ntxq_descs;
2671 for (i = 0; i < qparam->nxqs; i++)
2672 ionic_swap_queues(lif->txqcqs[i], tx_qcqs[i]);
2673 }
2674
2675 if (rx_qcqs) {
2676 lif->nrxq_descs = qparam->nrxq_descs;
2677 for (i = 0; i < qparam->nxqs; i++)
2678 ionic_swap_queues(lif->rxqcqs[i], rx_qcqs[i]);
2679 }
2680
2681 /* if we need to change the interrupt layout, this is the time */
2682 if (qparam->intr_split != test_bit(IONIC_LIF_F_SPLIT_INTR, lif->state) ||
2683 qparam->nxqs != lif->nxqs) {
2684 if (qparam->intr_split) {
2685 set_bit(IONIC_LIF_F_SPLIT_INTR, lif->state);
2686 } else {
2687 clear_bit(IONIC_LIF_F_SPLIT_INTR, lif->state);
2688 lif->tx_coalesce_usecs = lif->rx_coalesce_usecs;
2689 lif->tx_coalesce_hw = lif->rx_coalesce_hw;
2690 }
2691
2692 /* clear existing interrupt assignments */
2693 for (i = 0; i < lif->ionic->ntxqs_per_lif; i++) {
2694 ionic_qcq_intr_free(lif, lif->txqcqs[i]);
2695 ionic_qcq_intr_free(lif, lif->rxqcqs[i]);
2696 }
2697
2698 /* re-assign the interrupts */
2699 for (i = 0; i < qparam->nxqs; i++) {
2700 lif->rxqcqs[i]->flags |= IONIC_QCQ_F_INTR;
2701 err = ionic_alloc_qcq_interrupt(lif, lif->rxqcqs[i]);
2702 ionic_intr_coal_init(lif->ionic->idev.intr_ctrl,
2703 lif->rxqcqs[i]->intr.index,
2704 lif->rx_coalesce_hw);
2705
2706 if (qparam->intr_split) {
2707 lif->txqcqs[i]->flags |= IONIC_QCQ_F_INTR;
2708 err = ionic_alloc_qcq_interrupt(lif, lif->txqcqs[i]);
2709 ionic_intr_coal_init(lif->ionic->idev.intr_ctrl,
2710 lif->txqcqs[i]->intr.index,
2711 lif->tx_coalesce_hw);
2712 if (test_bit(IONIC_LIF_F_TX_DIM_INTR, lif->state))
2713 lif->txqcqs[i]->intr.dim_coal_hw = lif->tx_coalesce_hw;
2714 } else {
2715 lif->txqcqs[i]->flags &= ~IONIC_QCQ_F_INTR;
2716 ionic_link_qcq_interrupts(lif->rxqcqs[i], lif->txqcqs[i]);
2717 }
2718 }
2719 }
2720
2721 /* now we can rework the debugfs mappings */
2722 if (tx_qcqs) {
2723 for (i = 0; i < qparam->nxqs; i++) {
2724 ionic_debugfs_del_qcq(lif->txqcqs[i]);
2725 ionic_debugfs_add_qcq(lif, lif->txqcqs[i]);
2726 }
2727 }
2728
2729 if (rx_qcqs) {
2730 for (i = 0; i < qparam->nxqs; i++) {
2731 ionic_debugfs_del_qcq(lif->rxqcqs[i]);
2732 ionic_debugfs_add_qcq(lif, lif->rxqcqs[i]);
2733 }
2734 }
2735
2736 swap(lif->nxqs, qparam->nxqs);
2737 swap(lif->rxq_features, qparam->rxq_features);
2738
2739 err_out_reinit_unlock:
2740 /* re-init the queues, but don't lose an error code */
2741 if (err)
2742 ionic_start_queues_reconfig(lif);
2743 else
2744 err = ionic_start_queues_reconfig(lif);
2745
2746 err_out:
2747 /* free old allocs without cleaning intr */
2748 for (i = 0; i < qparam->nxqs; i++) {
2749 if (tx_qcqs && tx_qcqs[i]) {
2750 tx_qcqs[i]->flags &= ~IONIC_QCQ_F_INTR;
2751 ionic_qcq_free(lif, tx_qcqs[i]);
2752 devm_kfree(lif->ionic->dev, tx_qcqs[i]);
2753 tx_qcqs[i] = NULL;
2754 }
2755 if (rx_qcqs && rx_qcqs[i]) {
2756 rx_qcqs[i]->flags &= ~IONIC_QCQ_F_INTR;
2757 ionic_qcq_free(lif, rx_qcqs[i]);
2758 devm_kfree(lif->ionic->dev, rx_qcqs[i]);
2759 rx_qcqs[i] = NULL;
2760 }
2761 }
2762
2763 /* free q array */
2764 if (rx_qcqs) {
2765 devm_kfree(lif->ionic->dev, rx_qcqs);
2766 rx_qcqs = NULL;
2767 }
2768 if (tx_qcqs) {
2769 devm_kfree(lif->ionic->dev, tx_qcqs);
2770 tx_qcqs = NULL;
2771 }
2772
2773 /* clean the unused dma and info allocations when new set is smaller
2774 * than the full array, but leave the qcq shells in place
2775 */
2776 for (i = lif->nxqs; i < lif->ionic->ntxqs_per_lif; i++) {
2777 lif->txqcqs[i]->flags &= ~IONIC_QCQ_F_INTR;
2778 ionic_qcq_free(lif, lif->txqcqs[i]);
2779
2780 lif->rxqcqs[i]->flags &= ~IONIC_QCQ_F_INTR;
2781 ionic_qcq_free(lif, lif->rxqcqs[i]);
2782 }
2783
2784 return err;
2785 }
2786
ionic_lif_alloc(struct ionic * ionic)2787 int ionic_lif_alloc(struct ionic *ionic)
2788 {
2789 struct device *dev = ionic->dev;
2790 union ionic_lif_identity *lid;
2791 struct net_device *netdev;
2792 struct ionic_lif *lif;
2793 int tbl_sz;
2794 int err;
2795
2796 lid = kzalloc(sizeof(*lid), GFP_KERNEL);
2797 if (!lid)
2798 return -ENOMEM;
2799
2800 netdev = alloc_etherdev_mqs(sizeof(*lif),
2801 ionic->ntxqs_per_lif, ionic->ntxqs_per_lif);
2802 if (!netdev) {
2803 dev_err(dev, "Cannot allocate netdev, aborting\n");
2804 err = -ENOMEM;
2805 goto err_out_free_lid;
2806 }
2807
2808 SET_NETDEV_DEV(netdev, dev);
2809
2810 lif = netdev_priv(netdev);
2811 lif->netdev = netdev;
2812 ionic->lif = lif;
2813 netdev->netdev_ops = &ionic_netdev_ops;
2814 ionic_ethtool_set_ops(netdev);
2815
2816 netdev->watchdog_timeo = 2 * HZ;
2817 netif_carrier_off(netdev);
2818
2819 lif->identity = lid;
2820 lif->lif_type = IONIC_LIF_TYPE_CLASSIC;
2821 err = ionic_lif_identify(ionic, lif->lif_type, lif->identity);
2822 if (err) {
2823 dev_err(ionic->dev, "Cannot identify type %d: %d\n",
2824 lif->lif_type, err);
2825 goto err_out_free_netdev;
2826 }
2827 lif->netdev->min_mtu = max_t(unsigned int, ETH_MIN_MTU,
2828 le32_to_cpu(lif->identity->eth.min_frame_size));
2829 lif->netdev->max_mtu =
2830 le32_to_cpu(lif->identity->eth.max_frame_size) - ETH_HLEN - VLAN_HLEN;
2831
2832 lif->neqs = ionic->neqs_per_lif;
2833 lif->nxqs = ionic->ntxqs_per_lif;
2834
2835 lif->ionic = ionic;
2836 lif->index = 0;
2837 lif->ntxq_descs = IONIC_DEF_TXRX_DESC;
2838 lif->nrxq_descs = IONIC_DEF_TXRX_DESC;
2839
2840 /* Convert the default coalesce value to actual hw resolution */
2841 lif->rx_coalesce_usecs = IONIC_ITR_COAL_USEC_DEFAULT;
2842 lif->rx_coalesce_hw = ionic_coal_usec_to_hw(lif->ionic,
2843 lif->rx_coalesce_usecs);
2844 lif->tx_coalesce_usecs = lif->rx_coalesce_usecs;
2845 lif->tx_coalesce_hw = lif->rx_coalesce_hw;
2846 set_bit(IONIC_LIF_F_RX_DIM_INTR, lif->state);
2847 set_bit(IONIC_LIF_F_TX_DIM_INTR, lif->state);
2848
2849 snprintf(lif->name, sizeof(lif->name), "lif%u", lif->index);
2850
2851 spin_lock_init(&lif->adminq_lock);
2852
2853 spin_lock_init(&lif->deferred.lock);
2854 INIT_LIST_HEAD(&lif->deferred.list);
2855 INIT_WORK(&lif->deferred.work, ionic_lif_deferred_work);
2856
2857 /* allocate lif info */
2858 lif->info_sz = ALIGN(sizeof(*lif->info), PAGE_SIZE);
2859 lif->info = dma_alloc_coherent(dev, lif->info_sz,
2860 &lif->info_pa, GFP_KERNEL);
2861 if (!lif->info) {
2862 dev_err(dev, "Failed to allocate lif info, aborting\n");
2863 err = -ENOMEM;
2864 goto err_out_free_netdev;
2865 }
2866
2867 ionic_debugfs_add_lif(lif);
2868
2869 /* allocate control queues and txrx queue arrays */
2870 ionic_lif_queue_identify(lif);
2871 err = ionic_qcqs_alloc(lif);
2872 if (err)
2873 goto err_out_free_lif_info;
2874
2875 /* allocate rss indirection table */
2876 tbl_sz = le16_to_cpu(lif->ionic->ident.lif.eth.rss_ind_tbl_sz);
2877 lif->rss_ind_tbl_sz = sizeof(*lif->rss_ind_tbl) * tbl_sz;
2878 lif->rss_ind_tbl = dma_alloc_coherent(dev, lif->rss_ind_tbl_sz,
2879 &lif->rss_ind_tbl_pa,
2880 GFP_KERNEL);
2881
2882 if (!lif->rss_ind_tbl) {
2883 err = -ENOMEM;
2884 dev_err(dev, "Failed to allocate rss indirection table, aborting\n");
2885 goto err_out_free_qcqs;
2886 }
2887 netdev_rss_key_fill(lif->rss_hash_key, IONIC_RSS_HASH_KEY_SIZE);
2888
2889 ionic_lif_alloc_phc(lif);
2890
2891 return 0;
2892
2893 err_out_free_qcqs:
2894 ionic_qcqs_free(lif);
2895 err_out_free_lif_info:
2896 dma_free_coherent(dev, lif->info_sz, lif->info, lif->info_pa);
2897 lif->info = NULL;
2898 lif->info_pa = 0;
2899 err_out_free_netdev:
2900 free_netdev(lif->netdev);
2901 lif = NULL;
2902 err_out_free_lid:
2903 kfree(lid);
2904
2905 return err;
2906 }
2907
ionic_lif_reset(struct ionic_lif * lif)2908 static void ionic_lif_reset(struct ionic_lif *lif)
2909 {
2910 struct ionic_dev *idev = &lif->ionic->idev;
2911
2912 mutex_lock(&lif->ionic->dev_cmd_lock);
2913 ionic_dev_cmd_lif_reset(idev, lif->index);
2914 ionic_dev_cmd_wait(lif->ionic, DEVCMD_TIMEOUT);
2915 mutex_unlock(&lif->ionic->dev_cmd_lock);
2916 }
2917
ionic_lif_handle_fw_down(struct ionic_lif * lif)2918 static void ionic_lif_handle_fw_down(struct ionic_lif *lif)
2919 {
2920 struct ionic *ionic = lif->ionic;
2921
2922 if (test_and_set_bit(IONIC_LIF_F_FW_RESET, lif->state))
2923 return;
2924
2925 dev_info(ionic->dev, "FW Down: Stopping LIFs\n");
2926
2927 netif_device_detach(lif->netdev);
2928
2929 if (test_bit(IONIC_LIF_F_UP, lif->state)) {
2930 dev_info(ionic->dev, "Surprise FW stop, stopping queues\n");
2931 mutex_lock(&lif->queue_lock);
2932 ionic_stop_queues(lif);
2933 mutex_unlock(&lif->queue_lock);
2934 }
2935
2936 if (netif_running(lif->netdev)) {
2937 ionic_txrx_deinit(lif);
2938 ionic_txrx_free(lif);
2939 }
2940 ionic_lif_deinit(lif);
2941 ionic_reset(ionic);
2942 ionic_qcqs_free(lif);
2943
2944 dev_info(ionic->dev, "FW Down: LIFs stopped\n");
2945 }
2946
ionic_lif_handle_fw_up(struct ionic_lif * lif)2947 static void ionic_lif_handle_fw_up(struct ionic_lif *lif)
2948 {
2949 struct ionic *ionic = lif->ionic;
2950 int err;
2951
2952 if (!test_bit(IONIC_LIF_F_FW_RESET, lif->state))
2953 return;
2954
2955 dev_info(ionic->dev, "FW Up: restarting LIFs\n");
2956
2957 ionic_init_devinfo(ionic);
2958 err = ionic_identify(ionic);
2959 if (err)
2960 goto err_out;
2961 err = ionic_port_identify(ionic);
2962 if (err)
2963 goto err_out;
2964 err = ionic_port_init(ionic);
2965 if (err)
2966 goto err_out;
2967 err = ionic_qcqs_alloc(lif);
2968 if (err)
2969 goto err_out;
2970
2971 err = ionic_lif_init(lif);
2972 if (err)
2973 goto err_qcqs_free;
2974
2975 if (lif->registered)
2976 ionic_lif_set_netdev_info(lif);
2977
2978 ionic_rx_filter_replay(lif);
2979
2980 if (netif_running(lif->netdev)) {
2981 err = ionic_txrx_alloc(lif);
2982 if (err)
2983 goto err_lifs_deinit;
2984
2985 err = ionic_txrx_init(lif);
2986 if (err)
2987 goto err_txrx_free;
2988 }
2989
2990 clear_bit(IONIC_LIF_F_FW_RESET, lif->state);
2991 ionic_link_status_check_request(lif, CAN_SLEEP);
2992 netif_device_attach(lif->netdev);
2993 dev_info(ionic->dev, "FW Up: LIFs restarted\n");
2994
2995 /* restore the hardware timestamping queues */
2996 ionic_lif_hwstamp_replay(lif);
2997
2998 return;
2999
3000 err_txrx_free:
3001 ionic_txrx_free(lif);
3002 err_lifs_deinit:
3003 ionic_lif_deinit(lif);
3004 err_qcqs_free:
3005 ionic_qcqs_free(lif);
3006 err_out:
3007 dev_err(ionic->dev, "FW Up: LIFs restart failed - err %d\n", err);
3008 }
3009
ionic_lif_free(struct ionic_lif * lif)3010 void ionic_lif_free(struct ionic_lif *lif)
3011 {
3012 struct device *dev = lif->ionic->dev;
3013
3014 ionic_lif_free_phc(lif);
3015
3016 /* free rss indirection table */
3017 dma_free_coherent(dev, lif->rss_ind_tbl_sz, lif->rss_ind_tbl,
3018 lif->rss_ind_tbl_pa);
3019 lif->rss_ind_tbl = NULL;
3020 lif->rss_ind_tbl_pa = 0;
3021
3022 /* free queues */
3023 ionic_qcqs_free(lif);
3024 if (!test_bit(IONIC_LIF_F_FW_RESET, lif->state))
3025 ionic_lif_reset(lif);
3026
3027 /* free lif info */
3028 kfree(lif->identity);
3029 dma_free_coherent(dev, lif->info_sz, lif->info, lif->info_pa);
3030 lif->info = NULL;
3031 lif->info_pa = 0;
3032
3033 /* unmap doorbell page */
3034 ionic_bus_unmap_dbpage(lif->ionic, lif->kern_dbpage);
3035 lif->kern_dbpage = NULL;
3036 kfree(lif->dbid_inuse);
3037 lif->dbid_inuse = NULL;
3038
3039 /* free netdev & lif */
3040 ionic_debugfs_del_lif(lif);
3041 free_netdev(lif->netdev);
3042 }
3043
ionic_lif_deinit(struct ionic_lif * lif)3044 void ionic_lif_deinit(struct ionic_lif *lif)
3045 {
3046 if (!test_and_clear_bit(IONIC_LIF_F_INITED, lif->state))
3047 return;
3048
3049 if (!test_bit(IONIC_LIF_F_FW_RESET, lif->state)) {
3050 cancel_work_sync(&lif->deferred.work);
3051 cancel_work_sync(&lif->tx_timeout_work);
3052 ionic_rx_filters_deinit(lif);
3053 if (lif->netdev->features & NETIF_F_RXHASH)
3054 ionic_lif_rss_deinit(lif);
3055 }
3056
3057 napi_disable(&lif->adminqcq->napi);
3058 ionic_lif_qcq_deinit(lif, lif->notifyqcq);
3059 ionic_lif_qcq_deinit(lif, lif->adminqcq);
3060
3061 mutex_destroy(&lif->queue_lock);
3062 ionic_lif_reset(lif);
3063 }
3064
ionic_lif_adminq_init(struct ionic_lif * lif)3065 static int ionic_lif_adminq_init(struct ionic_lif *lif)
3066 {
3067 struct device *dev = lif->ionic->dev;
3068 struct ionic_q_init_comp comp;
3069 struct ionic_dev *idev;
3070 struct ionic_qcq *qcq;
3071 struct ionic_queue *q;
3072 int err;
3073
3074 idev = &lif->ionic->idev;
3075 qcq = lif->adminqcq;
3076 q = &qcq->q;
3077
3078 mutex_lock(&lif->ionic->dev_cmd_lock);
3079 ionic_dev_cmd_adminq_init(idev, qcq, lif->index, qcq->intr.index);
3080 err = ionic_dev_cmd_wait(lif->ionic, DEVCMD_TIMEOUT);
3081 ionic_dev_cmd_comp(idev, (union ionic_dev_cmd_comp *)&comp);
3082 mutex_unlock(&lif->ionic->dev_cmd_lock);
3083 if (err) {
3084 netdev_err(lif->netdev, "adminq init failed %d\n", err);
3085 return err;
3086 }
3087
3088 q->hw_type = comp.hw_type;
3089 q->hw_index = le32_to_cpu(comp.hw_index);
3090 q->dbval = IONIC_DBELL_QID(q->hw_index);
3091
3092 dev_dbg(dev, "adminq->hw_type %d\n", q->hw_type);
3093 dev_dbg(dev, "adminq->hw_index %d\n", q->hw_index);
3094
3095 netif_napi_add(lif->netdev, &qcq->napi, ionic_adminq_napi,
3096 NAPI_POLL_WEIGHT);
3097
3098 napi_enable(&qcq->napi);
3099
3100 if (qcq->flags & IONIC_QCQ_F_INTR)
3101 ionic_intr_mask(idev->intr_ctrl, qcq->intr.index,
3102 IONIC_INTR_MASK_CLEAR);
3103
3104 qcq->flags |= IONIC_QCQ_F_INITED;
3105
3106 return 0;
3107 }
3108
ionic_lif_notifyq_init(struct ionic_lif * lif)3109 static int ionic_lif_notifyq_init(struct ionic_lif *lif)
3110 {
3111 struct ionic_qcq *qcq = lif->notifyqcq;
3112 struct device *dev = lif->ionic->dev;
3113 struct ionic_queue *q = &qcq->q;
3114 int err;
3115
3116 struct ionic_admin_ctx ctx = {
3117 .work = COMPLETION_INITIALIZER_ONSTACK(ctx.work),
3118 .cmd.q_init = {
3119 .opcode = IONIC_CMD_Q_INIT,
3120 .lif_index = cpu_to_le16(lif->index),
3121 .type = q->type,
3122 .ver = lif->qtype_info[q->type].version,
3123 .index = cpu_to_le32(q->index),
3124 .flags = cpu_to_le16(IONIC_QINIT_F_IRQ |
3125 IONIC_QINIT_F_ENA),
3126 .intr_index = cpu_to_le16(lif->adminqcq->intr.index),
3127 .pid = cpu_to_le16(q->pid),
3128 .ring_size = ilog2(q->num_descs),
3129 .ring_base = cpu_to_le64(q->base_pa),
3130 }
3131 };
3132
3133 dev_dbg(dev, "notifyq_init.pid %d\n", ctx.cmd.q_init.pid);
3134 dev_dbg(dev, "notifyq_init.index %d\n", ctx.cmd.q_init.index);
3135 dev_dbg(dev, "notifyq_init.ring_base 0x%llx\n", ctx.cmd.q_init.ring_base);
3136 dev_dbg(dev, "notifyq_init.ring_size %d\n", ctx.cmd.q_init.ring_size);
3137
3138 err = ionic_adminq_post_wait(lif, &ctx);
3139 if (err)
3140 return err;
3141
3142 lif->last_eid = 0;
3143 q->hw_type = ctx.comp.q_init.hw_type;
3144 q->hw_index = le32_to_cpu(ctx.comp.q_init.hw_index);
3145 q->dbval = IONIC_DBELL_QID(q->hw_index);
3146
3147 dev_dbg(dev, "notifyq->hw_type %d\n", q->hw_type);
3148 dev_dbg(dev, "notifyq->hw_index %d\n", q->hw_index);
3149
3150 /* preset the callback info */
3151 q->info[0].cb_arg = lif;
3152
3153 qcq->flags |= IONIC_QCQ_F_INITED;
3154
3155 return 0;
3156 }
3157
ionic_station_set(struct ionic_lif * lif)3158 static int ionic_station_set(struct ionic_lif *lif)
3159 {
3160 struct net_device *netdev = lif->netdev;
3161 struct ionic_admin_ctx ctx = {
3162 .work = COMPLETION_INITIALIZER_ONSTACK(ctx.work),
3163 .cmd.lif_getattr = {
3164 .opcode = IONIC_CMD_LIF_GETATTR,
3165 .index = cpu_to_le16(lif->index),
3166 .attr = IONIC_LIF_ATTR_MAC,
3167 },
3168 };
3169 struct sockaddr addr;
3170 int err;
3171
3172 err = ionic_adminq_post_wait(lif, &ctx);
3173 if (err)
3174 return err;
3175 netdev_dbg(lif->netdev, "found initial MAC addr %pM\n",
3176 ctx.comp.lif_getattr.mac);
3177 if (is_zero_ether_addr(ctx.comp.lif_getattr.mac))
3178 return 0;
3179
3180 if (!is_zero_ether_addr(netdev->dev_addr)) {
3181 /* If the netdev mac is non-zero and doesn't match the default
3182 * device address, it was set by something earlier and we're
3183 * likely here again after a fw-upgrade reset. We need to be
3184 * sure the netdev mac is in our filter list.
3185 */
3186 if (!ether_addr_equal(ctx.comp.lif_getattr.mac,
3187 netdev->dev_addr))
3188 ionic_lif_addr(lif, netdev->dev_addr, ADD_ADDR, CAN_SLEEP);
3189 } else {
3190 /* Update the netdev mac with the device's mac */
3191 memcpy(addr.sa_data, ctx.comp.lif_getattr.mac, netdev->addr_len);
3192 addr.sa_family = AF_INET;
3193 err = eth_prepare_mac_addr_change(netdev, &addr);
3194 if (err) {
3195 netdev_warn(lif->netdev, "ignoring bad MAC addr from NIC %pM - err %d\n",
3196 addr.sa_data, err);
3197 return 0;
3198 }
3199
3200 eth_commit_mac_addr_change(netdev, &addr);
3201 }
3202
3203 netdev_dbg(lif->netdev, "adding station MAC addr %pM\n",
3204 netdev->dev_addr);
3205 ionic_lif_addr(lif, netdev->dev_addr, ADD_ADDR, CAN_SLEEP);
3206
3207 return 0;
3208 }
3209
ionic_lif_init(struct ionic_lif * lif)3210 int ionic_lif_init(struct ionic_lif *lif)
3211 {
3212 struct ionic_dev *idev = &lif->ionic->idev;
3213 struct device *dev = lif->ionic->dev;
3214 struct ionic_lif_init_comp comp;
3215 int dbpage_num;
3216 int err;
3217
3218 mutex_lock(&lif->ionic->dev_cmd_lock);
3219 ionic_dev_cmd_lif_init(idev, lif->index, lif->info_pa);
3220 err = ionic_dev_cmd_wait(lif->ionic, DEVCMD_TIMEOUT);
3221 ionic_dev_cmd_comp(idev, (union ionic_dev_cmd_comp *)&comp);
3222 mutex_unlock(&lif->ionic->dev_cmd_lock);
3223 if (err)
3224 return err;
3225
3226 lif->hw_index = le16_to_cpu(comp.hw_index);
3227 mutex_init(&lif->queue_lock);
3228
3229 /* now that we have the hw_index we can figure out our doorbell page */
3230 lif->dbid_count = le32_to_cpu(lif->ionic->ident.dev.ndbpgs_per_lif);
3231 if (!lif->dbid_count) {
3232 dev_err(dev, "No doorbell pages, aborting\n");
3233 return -EINVAL;
3234 }
3235
3236 lif->dbid_inuse = bitmap_alloc(lif->dbid_count, GFP_KERNEL);
3237 if (!lif->dbid_inuse) {
3238 dev_err(dev, "Failed alloc doorbell id bitmap, aborting\n");
3239 return -ENOMEM;
3240 }
3241
3242 /* first doorbell id reserved for kernel (dbid aka pid == zero) */
3243 set_bit(0, lif->dbid_inuse);
3244 lif->kern_pid = 0;
3245
3246 dbpage_num = ionic_db_page_num(lif, lif->kern_pid);
3247 lif->kern_dbpage = ionic_bus_map_dbpage(lif->ionic, dbpage_num);
3248 if (!lif->kern_dbpage) {
3249 dev_err(dev, "Cannot map dbpage, aborting\n");
3250 err = -ENOMEM;
3251 goto err_out_free_dbid;
3252 }
3253
3254 err = ionic_lif_adminq_init(lif);
3255 if (err)
3256 goto err_out_adminq_deinit;
3257
3258 if (lif->ionic->nnqs_per_lif) {
3259 err = ionic_lif_notifyq_init(lif);
3260 if (err)
3261 goto err_out_notifyq_deinit;
3262 }
3263
3264 err = ionic_init_nic_features(lif);
3265 if (err)
3266 goto err_out_notifyq_deinit;
3267
3268 if (!test_bit(IONIC_LIF_F_FW_RESET, lif->state)) {
3269 err = ionic_rx_filters_init(lif);
3270 if (err)
3271 goto err_out_notifyq_deinit;
3272 }
3273
3274 err = ionic_station_set(lif);
3275 if (err)
3276 goto err_out_notifyq_deinit;
3277
3278 lif->rx_copybreak = IONIC_RX_COPYBREAK_DEFAULT;
3279
3280 set_bit(IONIC_LIF_F_INITED, lif->state);
3281
3282 INIT_WORK(&lif->tx_timeout_work, ionic_tx_timeout_work);
3283
3284 return 0;
3285
3286 err_out_notifyq_deinit:
3287 ionic_lif_qcq_deinit(lif, lif->notifyqcq);
3288 err_out_adminq_deinit:
3289 ionic_lif_qcq_deinit(lif, lif->adminqcq);
3290 ionic_lif_reset(lif);
3291 ionic_bus_unmap_dbpage(lif->ionic, lif->kern_dbpage);
3292 lif->kern_dbpage = NULL;
3293 err_out_free_dbid:
3294 kfree(lif->dbid_inuse);
3295 lif->dbid_inuse = NULL;
3296
3297 return err;
3298 }
3299
ionic_lif_notify_work(struct work_struct * ws)3300 static void ionic_lif_notify_work(struct work_struct *ws)
3301 {
3302 }
3303
ionic_lif_set_netdev_info(struct ionic_lif * lif)3304 static void ionic_lif_set_netdev_info(struct ionic_lif *lif)
3305 {
3306 struct ionic_admin_ctx ctx = {
3307 .work = COMPLETION_INITIALIZER_ONSTACK(ctx.work),
3308 .cmd.lif_setattr = {
3309 .opcode = IONIC_CMD_LIF_SETATTR,
3310 .index = cpu_to_le16(lif->index),
3311 .attr = IONIC_LIF_ATTR_NAME,
3312 },
3313 };
3314
3315 strlcpy(ctx.cmd.lif_setattr.name, lif->netdev->name,
3316 sizeof(ctx.cmd.lif_setattr.name));
3317
3318 ionic_adminq_post_wait(lif, &ctx);
3319 }
3320
ionic_netdev_lif(struct net_device * netdev)3321 static struct ionic_lif *ionic_netdev_lif(struct net_device *netdev)
3322 {
3323 if (!netdev || netdev->netdev_ops->ndo_start_xmit != ionic_start_xmit)
3324 return NULL;
3325
3326 return netdev_priv(netdev);
3327 }
3328
ionic_lif_notify(struct notifier_block * nb,unsigned long event,void * info)3329 static int ionic_lif_notify(struct notifier_block *nb,
3330 unsigned long event, void *info)
3331 {
3332 struct net_device *ndev = netdev_notifier_info_to_dev(info);
3333 struct ionic *ionic = container_of(nb, struct ionic, nb);
3334 struct ionic_lif *lif = ionic_netdev_lif(ndev);
3335
3336 if (!lif || lif->ionic != ionic)
3337 return NOTIFY_DONE;
3338
3339 switch (event) {
3340 case NETDEV_CHANGENAME:
3341 ionic_lif_set_netdev_info(lif);
3342 break;
3343 }
3344
3345 return NOTIFY_DONE;
3346 }
3347
ionic_lif_register(struct ionic_lif * lif)3348 int ionic_lif_register(struct ionic_lif *lif)
3349 {
3350 int err;
3351
3352 ionic_lif_register_phc(lif);
3353
3354 INIT_WORK(&lif->ionic->nb_work, ionic_lif_notify_work);
3355
3356 lif->ionic->nb.notifier_call = ionic_lif_notify;
3357
3358 err = register_netdevice_notifier(&lif->ionic->nb);
3359 if (err)
3360 lif->ionic->nb.notifier_call = NULL;
3361
3362 /* only register LIF0 for now */
3363 err = register_netdev(lif->netdev);
3364 if (err) {
3365 dev_err(lif->ionic->dev, "Cannot register net device, aborting\n");
3366 ionic_lif_unregister_phc(lif);
3367 return err;
3368 }
3369
3370 ionic_link_status_check_request(lif, CAN_SLEEP);
3371 lif->registered = true;
3372 ionic_lif_set_netdev_info(lif);
3373
3374 return 0;
3375 }
3376
ionic_lif_unregister(struct ionic_lif * lif)3377 void ionic_lif_unregister(struct ionic_lif *lif)
3378 {
3379 if (lif->ionic->nb.notifier_call) {
3380 unregister_netdevice_notifier(&lif->ionic->nb);
3381 cancel_work_sync(&lif->ionic->nb_work);
3382 lif->ionic->nb.notifier_call = NULL;
3383 }
3384
3385 if (lif->netdev->reg_state == NETREG_REGISTERED)
3386 unregister_netdev(lif->netdev);
3387
3388 ionic_lif_unregister_phc(lif);
3389
3390 lif->registered = false;
3391 }
3392
ionic_lif_queue_identify(struct ionic_lif * lif)3393 static void ionic_lif_queue_identify(struct ionic_lif *lif)
3394 {
3395 union ionic_q_identity __iomem *q_ident;
3396 struct ionic *ionic = lif->ionic;
3397 struct ionic_dev *idev;
3398 int qtype;
3399 int err;
3400
3401 idev = &lif->ionic->idev;
3402 q_ident = (union ionic_q_identity __iomem *)&idev->dev_cmd_regs->data;
3403
3404 for (qtype = 0; qtype < ARRAY_SIZE(ionic_qtype_versions); qtype++) {
3405 struct ionic_qtype_info *qti = &lif->qtype_info[qtype];
3406
3407 /* filter out the ones we know about */
3408 switch (qtype) {
3409 case IONIC_QTYPE_ADMINQ:
3410 case IONIC_QTYPE_NOTIFYQ:
3411 case IONIC_QTYPE_RXQ:
3412 case IONIC_QTYPE_TXQ:
3413 break;
3414 default:
3415 continue;
3416 }
3417
3418 memset(qti, 0, sizeof(*qti));
3419
3420 mutex_lock(&ionic->dev_cmd_lock);
3421 ionic_dev_cmd_queue_identify(idev, lif->lif_type, qtype,
3422 ionic_qtype_versions[qtype]);
3423 err = ionic_dev_cmd_wait(ionic, DEVCMD_TIMEOUT);
3424 if (!err) {
3425 qti->version = readb(&q_ident->version);
3426 qti->supported = readb(&q_ident->supported);
3427 qti->features = readq(&q_ident->features);
3428 qti->desc_sz = readw(&q_ident->desc_sz);
3429 qti->comp_sz = readw(&q_ident->comp_sz);
3430 qti->sg_desc_sz = readw(&q_ident->sg_desc_sz);
3431 qti->max_sg_elems = readw(&q_ident->max_sg_elems);
3432 qti->sg_desc_stride = readw(&q_ident->sg_desc_stride);
3433 }
3434 mutex_unlock(&ionic->dev_cmd_lock);
3435
3436 if (err == -EINVAL) {
3437 dev_err(ionic->dev, "qtype %d not supported\n", qtype);
3438 continue;
3439 } else if (err == -EIO) {
3440 dev_err(ionic->dev, "q_ident failed, not supported on older FW\n");
3441 return;
3442 } else if (err) {
3443 dev_err(ionic->dev, "q_ident failed, qtype %d: %d\n",
3444 qtype, err);
3445 return;
3446 }
3447
3448 dev_dbg(ionic->dev, " qtype[%d].version = %d\n",
3449 qtype, qti->version);
3450 dev_dbg(ionic->dev, " qtype[%d].supported = 0x%02x\n",
3451 qtype, qti->supported);
3452 dev_dbg(ionic->dev, " qtype[%d].features = 0x%04llx\n",
3453 qtype, qti->features);
3454 dev_dbg(ionic->dev, " qtype[%d].desc_sz = %d\n",
3455 qtype, qti->desc_sz);
3456 dev_dbg(ionic->dev, " qtype[%d].comp_sz = %d\n",
3457 qtype, qti->comp_sz);
3458 dev_dbg(ionic->dev, " qtype[%d].sg_desc_sz = %d\n",
3459 qtype, qti->sg_desc_sz);
3460 dev_dbg(ionic->dev, " qtype[%d].max_sg_elems = %d\n",
3461 qtype, qti->max_sg_elems);
3462 dev_dbg(ionic->dev, " qtype[%d].sg_desc_stride = %d\n",
3463 qtype, qti->sg_desc_stride);
3464 }
3465 }
3466
ionic_lif_identify(struct ionic * ionic,u8 lif_type,union ionic_lif_identity * lid)3467 int ionic_lif_identify(struct ionic *ionic, u8 lif_type,
3468 union ionic_lif_identity *lid)
3469 {
3470 struct ionic_dev *idev = &ionic->idev;
3471 size_t sz;
3472 int err;
3473
3474 sz = min(sizeof(*lid), sizeof(idev->dev_cmd_regs->data));
3475
3476 mutex_lock(&ionic->dev_cmd_lock);
3477 ionic_dev_cmd_lif_identify(idev, lif_type, IONIC_IDENTITY_VERSION_1);
3478 err = ionic_dev_cmd_wait(ionic, DEVCMD_TIMEOUT);
3479 memcpy_fromio(lid, &idev->dev_cmd_regs->data, sz);
3480 mutex_unlock(&ionic->dev_cmd_lock);
3481 if (err)
3482 return (err);
3483
3484 dev_dbg(ionic->dev, "capabilities 0x%llx\n",
3485 le64_to_cpu(lid->capabilities));
3486
3487 dev_dbg(ionic->dev, "eth.max_ucast_filters %d\n",
3488 le32_to_cpu(lid->eth.max_ucast_filters));
3489 dev_dbg(ionic->dev, "eth.max_mcast_filters %d\n",
3490 le32_to_cpu(lid->eth.max_mcast_filters));
3491 dev_dbg(ionic->dev, "eth.features 0x%llx\n",
3492 le64_to_cpu(lid->eth.config.features));
3493 dev_dbg(ionic->dev, "eth.queue_count[IONIC_QTYPE_ADMINQ] %d\n",
3494 le32_to_cpu(lid->eth.config.queue_count[IONIC_QTYPE_ADMINQ]));
3495 dev_dbg(ionic->dev, "eth.queue_count[IONIC_QTYPE_NOTIFYQ] %d\n",
3496 le32_to_cpu(lid->eth.config.queue_count[IONIC_QTYPE_NOTIFYQ]));
3497 dev_dbg(ionic->dev, "eth.queue_count[IONIC_QTYPE_RXQ] %d\n",
3498 le32_to_cpu(lid->eth.config.queue_count[IONIC_QTYPE_RXQ]));
3499 dev_dbg(ionic->dev, "eth.queue_count[IONIC_QTYPE_TXQ] %d\n",
3500 le32_to_cpu(lid->eth.config.queue_count[IONIC_QTYPE_TXQ]));
3501 dev_dbg(ionic->dev, "eth.config.name %s\n", lid->eth.config.name);
3502 dev_dbg(ionic->dev, "eth.config.mac %pM\n", lid->eth.config.mac);
3503 dev_dbg(ionic->dev, "eth.config.mtu %d\n",
3504 le32_to_cpu(lid->eth.config.mtu));
3505
3506 return 0;
3507 }
3508
ionic_lif_size(struct ionic * ionic)3509 int ionic_lif_size(struct ionic *ionic)
3510 {
3511 struct ionic_identity *ident = &ionic->ident;
3512 unsigned int nintrs, dev_nintrs;
3513 union ionic_lif_config *lc;
3514 unsigned int ntxqs_per_lif;
3515 unsigned int nrxqs_per_lif;
3516 unsigned int neqs_per_lif;
3517 unsigned int nnqs_per_lif;
3518 unsigned int nxqs, neqs;
3519 unsigned int min_intrs;
3520 int err;
3521
3522 lc = &ident->lif.eth.config;
3523 dev_nintrs = le32_to_cpu(ident->dev.nintrs);
3524 neqs_per_lif = le32_to_cpu(ident->lif.rdma.eq_qtype.qid_count);
3525 nnqs_per_lif = le32_to_cpu(lc->queue_count[IONIC_QTYPE_NOTIFYQ]);
3526 ntxqs_per_lif = le32_to_cpu(lc->queue_count[IONIC_QTYPE_TXQ]);
3527 nrxqs_per_lif = le32_to_cpu(lc->queue_count[IONIC_QTYPE_RXQ]);
3528
3529 /* reserve last queue id for hardware timestamping */
3530 if (lc->features & cpu_to_le64(IONIC_ETH_HW_TIMESTAMP)) {
3531 if (ntxqs_per_lif <= 1 || nrxqs_per_lif <= 1) {
3532 lc->features &= cpu_to_le64(~IONIC_ETH_HW_TIMESTAMP);
3533 } else {
3534 ntxqs_per_lif -= 1;
3535 nrxqs_per_lif -= 1;
3536 }
3537 }
3538
3539 nxqs = min(ntxqs_per_lif, nrxqs_per_lif);
3540 nxqs = min(nxqs, num_online_cpus());
3541 neqs = min(neqs_per_lif, num_online_cpus());
3542
3543 try_again:
3544 /* interrupt usage:
3545 * 1 for master lif adminq/notifyq
3546 * 1 for each CPU for master lif TxRx queue pairs
3547 * whatever's left is for RDMA queues
3548 */
3549 nintrs = 1 + nxqs + neqs;
3550 min_intrs = 2; /* adminq + 1 TxRx queue pair */
3551
3552 if (nintrs > dev_nintrs)
3553 goto try_fewer;
3554
3555 err = ionic_bus_alloc_irq_vectors(ionic, nintrs);
3556 if (err < 0 && err != -ENOSPC) {
3557 dev_err(ionic->dev, "Can't get intrs from OS: %d\n", err);
3558 return err;
3559 }
3560 if (err == -ENOSPC)
3561 goto try_fewer;
3562
3563 if (err != nintrs) {
3564 ionic_bus_free_irq_vectors(ionic);
3565 goto try_fewer;
3566 }
3567
3568 ionic->nnqs_per_lif = nnqs_per_lif;
3569 ionic->neqs_per_lif = neqs;
3570 ionic->ntxqs_per_lif = nxqs;
3571 ionic->nrxqs_per_lif = nxqs;
3572 ionic->nintrs = nintrs;
3573
3574 ionic_debugfs_add_sizes(ionic);
3575
3576 return 0;
3577
3578 try_fewer:
3579 if (nnqs_per_lif > 1) {
3580 nnqs_per_lif >>= 1;
3581 goto try_again;
3582 }
3583 if (neqs > 1) {
3584 neqs >>= 1;
3585 goto try_again;
3586 }
3587 if (nxqs > 1) {
3588 nxqs >>= 1;
3589 goto try_again;
3590 }
3591 dev_err(ionic->dev, "Can't get minimum %d intrs from OS\n", min_intrs);
3592 return -ENOSPC;
3593 }
3594