1 /**********************************************************************
2  * Author: Cavium, Inc.
3  *
4  * Contact: support@cavium.com
5  *          Please include "LiquidIO" in the subject.
6  *
7  * Copyright (c) 2003-2016 Cavium, Inc.
8  *
9  * This file is free software; you can redistribute it and/or modify
10  * it under the terms of the GNU General Public License, Version 2, as
11  * published by the Free Software Foundation.
12  *
13  * This file is distributed in the hope that it will be useful, but
14  * AS-IS and WITHOUT ANY WARRANTY; without even the implied warranty
15  * of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE, TITLE, or
16  * NONINFRINGEMENT.  See the GNU General Public License for more details.
17  ***********************************************************************/
18 #include <linux/module.h>
19 #include <linux/interrupt.h>
20 #include <linux/pci.h>
21 #include <linux/firmware.h>
22 #include <net/vxlan.h>
23 #include <linux/kthread.h>
24 #include "liquidio_common.h"
25 #include "octeon_droq.h"
26 #include "octeon_iq.h"
27 #include "response_manager.h"
28 #include "octeon_device.h"
29 #include "octeon_nic.h"
30 #include "octeon_main.h"
31 #include "octeon_network.h"
32 #include "cn66xx_regs.h"
33 #include "cn66xx_device.h"
34 #include "cn68xx_device.h"
35 #include "cn23xx_pf_device.h"
36 #include "liquidio_image.h"
37 #include "lio_vf_rep.h"
38 
39 MODULE_AUTHOR("Cavium Networks, <support@cavium.com>");
40 MODULE_DESCRIPTION("Cavium LiquidIO Intelligent Server Adapter Driver");
41 MODULE_LICENSE("GPL");
42 MODULE_VERSION(LIQUIDIO_VERSION);
43 MODULE_FIRMWARE(LIO_FW_DIR LIO_FW_BASE_NAME LIO_210SV_NAME
44 		"_" LIO_FW_NAME_TYPE_NIC LIO_FW_NAME_SUFFIX);
45 MODULE_FIRMWARE(LIO_FW_DIR LIO_FW_BASE_NAME LIO_210NV_NAME
46 		"_" LIO_FW_NAME_TYPE_NIC LIO_FW_NAME_SUFFIX);
47 MODULE_FIRMWARE(LIO_FW_DIR LIO_FW_BASE_NAME LIO_410NV_NAME
48 		"_" LIO_FW_NAME_TYPE_NIC LIO_FW_NAME_SUFFIX);
49 MODULE_FIRMWARE(LIO_FW_DIR LIO_FW_BASE_NAME LIO_23XX_NAME
50 		"_" LIO_FW_NAME_TYPE_NIC LIO_FW_NAME_SUFFIX);
51 
52 static int ddr_timeout = 10000;
53 module_param(ddr_timeout, int, 0644);
54 MODULE_PARM_DESC(ddr_timeout,
55 		 "Number of milliseconds to wait for DDR initialization. 0 waits for ddr_timeout to be set to non-zero value before starting to check");
56 
57 #define DEFAULT_MSG_ENABLE (NETIF_MSG_DRV | NETIF_MSG_PROBE | NETIF_MSG_LINK)
58 
59 static int debug = -1;
60 module_param(debug, int, 0644);
61 MODULE_PARM_DESC(debug, "NETIF_MSG debug bits");
62 
63 static char fw_type[LIO_MAX_FW_TYPE_LEN] = LIO_FW_NAME_TYPE_AUTO;
64 module_param_string(fw_type, fw_type, sizeof(fw_type), 0444);
65 MODULE_PARM_DESC(fw_type, "Type of firmware to be loaded (default is \"auto\"), which uses firmware in flash, if present, else loads \"nic\".");
66 
67 static u32 console_bitmask;
68 module_param(console_bitmask, int, 0644);
69 MODULE_PARM_DESC(console_bitmask,
70 		 "Bitmask indicating which consoles have debug output redirected to syslog.");
71 
72 /**
73  * \brief determines if a given console has debug enabled.
74  * @param console console to check
75  * @returns  1 = enabled. 0 otherwise
76  */
77 static int octeon_console_debug_enabled(u32 console)
78 {
79 	return (console_bitmask >> (console)) & 0x1;
80 }
81 
82 /* Polling interval for determining when NIC application is alive */
83 #define LIQUIDIO_STARTER_POLL_INTERVAL_MS 100
84 
85 /* runtime link query interval */
86 #define LIQUIDIO_LINK_QUERY_INTERVAL_MS         1000
87 /* update localtime to octeon firmware every 60 seconds.
88  * make firmware to use same time reference, so that it will be easy to
89  * correlate firmware logged events/errors with host events, for debugging.
90  */
91 #define LIO_SYNC_OCTEON_TIME_INTERVAL_MS 60000
92 
93 /* time to wait for possible in-flight requests in milliseconds */
94 #define WAIT_INFLIGHT_REQUEST	msecs_to_jiffies(1000)
95 
96 struct lio_trusted_vf_ctx {
97 	struct completion complete;
98 	int status;
99 };
100 
101 struct oct_link_status_resp {
102 	u64 rh;
103 	struct oct_link_info link_info;
104 	u64 status;
105 };
106 
107 struct oct_timestamp_resp {
108 	u64 rh;
109 	u64 timestamp;
110 	u64 status;
111 };
112 
113 #define OCT_TIMESTAMP_RESP_SIZE (sizeof(struct oct_timestamp_resp))
114 
115 union tx_info {
116 	u64 u64;
117 	struct {
118 #ifdef __BIG_ENDIAN_BITFIELD
119 		u16 gso_size;
120 		u16 gso_segs;
121 		u32 reserved;
122 #else
123 		u32 reserved;
124 		u16 gso_segs;
125 		u16 gso_size;
126 #endif
127 	} s;
128 };
129 
130 /** Octeon device properties to be used by the NIC module.
131  * Each octeon device in the system will be represented
132  * by this structure in the NIC module.
133  */
134 
135 #define OCTNIC_GSO_MAX_HEADER_SIZE 128
136 #define OCTNIC_GSO_MAX_SIZE                                                    \
137 	(CN23XX_DEFAULT_INPUT_JABBER - OCTNIC_GSO_MAX_HEADER_SIZE)
138 
139 struct handshake {
140 	struct completion init;
141 	struct completion started;
142 	struct pci_dev *pci_dev;
143 	int init_ok;
144 	int started_ok;
145 };
146 
147 #ifdef CONFIG_PCI_IOV
148 static int liquidio_enable_sriov(struct pci_dev *dev, int num_vfs);
149 #endif
150 
151 static int octeon_dbg_console_print(struct octeon_device *oct, u32 console_num,
152 				    char *prefix, char *suffix);
153 
154 static int octeon_device_init(struct octeon_device *);
155 static int liquidio_stop(struct net_device *netdev);
156 static void liquidio_remove(struct pci_dev *pdev);
157 static int liquidio_probe(struct pci_dev *pdev,
158 			  const struct pci_device_id *ent);
159 static int liquidio_set_vf_link_state(struct net_device *netdev, int vfidx,
160 				      int linkstate);
161 
162 static struct handshake handshake[MAX_OCTEON_DEVICES];
163 static struct completion first_stage;
164 
165 static void octeon_droq_bh(unsigned long pdev)
166 {
167 	int q_no;
168 	int reschedule = 0;
169 	struct octeon_device *oct = (struct octeon_device *)pdev;
170 	struct octeon_device_priv *oct_priv =
171 		(struct octeon_device_priv *)oct->priv;
172 
173 	for (q_no = 0; q_no < MAX_OCTEON_OUTPUT_QUEUES(oct); q_no++) {
174 		if (!(oct->io_qmask.oq & BIT_ULL(q_no)))
175 			continue;
176 		reschedule |= octeon_droq_process_packets(oct, oct->droq[q_no],
177 							  MAX_PACKET_BUDGET);
178 		lio_enable_irq(oct->droq[q_no], NULL);
179 
180 		if (OCTEON_CN23XX_PF(oct) && oct->msix_on) {
181 			/* set time and cnt interrupt thresholds for this DROQ
182 			 * for NAPI
183 			 */
184 			int adjusted_q_no = q_no + oct->sriov_info.pf_srn;
185 
186 			octeon_write_csr64(
187 			    oct, CN23XX_SLI_OQ_PKT_INT_LEVELS(adjusted_q_no),
188 			    0x5700000040ULL);
189 			octeon_write_csr64(
190 			    oct, CN23XX_SLI_OQ_PKTS_SENT(adjusted_q_no), 0);
191 		}
192 	}
193 
194 	if (reschedule)
195 		tasklet_schedule(&oct_priv->droq_tasklet);
196 }
197 
198 static int lio_wait_for_oq_pkts(struct octeon_device *oct)
199 {
200 	struct octeon_device_priv *oct_priv =
201 		(struct octeon_device_priv *)oct->priv;
202 	int retry = 100, pkt_cnt = 0, pending_pkts = 0;
203 	int i;
204 
205 	do {
206 		pending_pkts = 0;
207 
208 		for (i = 0; i < MAX_OCTEON_OUTPUT_QUEUES(oct); i++) {
209 			if (!(oct->io_qmask.oq & BIT_ULL(i)))
210 				continue;
211 			pkt_cnt += octeon_droq_check_hw_for_pkts(oct->droq[i]);
212 		}
213 		if (pkt_cnt > 0) {
214 			pending_pkts += pkt_cnt;
215 			tasklet_schedule(&oct_priv->droq_tasklet);
216 		}
217 		pkt_cnt = 0;
218 		schedule_timeout_uninterruptible(1);
219 
220 	} while (retry-- && pending_pkts);
221 
222 	return pkt_cnt;
223 }
224 
225 /**
226  * \brief Forces all IO queues off on a given device
227  * @param oct Pointer to Octeon device
228  */
229 static void force_io_queues_off(struct octeon_device *oct)
230 {
231 	if ((oct->chip_id == OCTEON_CN66XX) ||
232 	    (oct->chip_id == OCTEON_CN68XX)) {
233 		/* Reset the Enable bits for Input Queues. */
234 		octeon_write_csr(oct, CN6XXX_SLI_PKT_INSTR_ENB, 0);
235 
236 		/* Reset the Enable bits for Output Queues. */
237 		octeon_write_csr(oct, CN6XXX_SLI_PKT_OUT_ENB, 0);
238 	}
239 }
240 
241 /**
242  * \brief Cause device to go quiet so it can be safely removed/reset/etc
243  * @param oct Pointer to Octeon device
244  */
245 static inline void pcierror_quiesce_device(struct octeon_device *oct)
246 {
247 	int i;
248 
249 	/* Disable the input and output queues now. No more packets will
250 	 * arrive from Octeon, but we should wait for all packet processing
251 	 * to finish.
252 	 */
253 	force_io_queues_off(oct);
254 
255 	/* To allow for in-flight requests */
256 	schedule_timeout_uninterruptible(WAIT_INFLIGHT_REQUEST);
257 
258 	if (wait_for_pending_requests(oct))
259 		dev_err(&oct->pci_dev->dev, "There were pending requests\n");
260 
261 	/* Force all requests waiting to be fetched by OCTEON to complete. */
262 	for (i = 0; i < MAX_OCTEON_INSTR_QUEUES(oct); i++) {
263 		struct octeon_instr_queue *iq;
264 
265 		if (!(oct->io_qmask.iq & BIT_ULL(i)))
266 			continue;
267 		iq = oct->instr_queue[i];
268 
269 		if (atomic_read(&iq->instr_pending)) {
270 			spin_lock_bh(&iq->lock);
271 			iq->fill_cnt = 0;
272 			iq->octeon_read_index = iq->host_write_index;
273 			iq->stats.instr_processed +=
274 				atomic_read(&iq->instr_pending);
275 			lio_process_iq_request_list(oct, iq, 0);
276 			spin_unlock_bh(&iq->lock);
277 		}
278 	}
279 
280 	/* Force all pending ordered list requests to time out. */
281 	lio_process_ordered_list(oct, 1);
282 
283 	/* We do not need to wait for output queue packets to be processed. */
284 }
285 
286 /**
287  * \brief Cleanup PCI AER uncorrectable error status
288  * @param dev Pointer to PCI device
289  */
290 static void cleanup_aer_uncorrect_error_status(struct pci_dev *dev)
291 {
292 	int pos = 0x100;
293 	u32 status, mask;
294 
295 	pr_info("%s :\n", __func__);
296 
297 	pci_read_config_dword(dev, pos + PCI_ERR_UNCOR_STATUS, &status);
298 	pci_read_config_dword(dev, pos + PCI_ERR_UNCOR_SEVER, &mask);
299 	if (dev->error_state == pci_channel_io_normal)
300 		status &= ~mask;        /* Clear corresponding nonfatal bits */
301 	else
302 		status &= mask;         /* Clear corresponding fatal bits */
303 	pci_write_config_dword(dev, pos + PCI_ERR_UNCOR_STATUS, status);
304 }
305 
306 /**
307  * \brief Stop all PCI IO to a given device
308  * @param dev Pointer to Octeon device
309  */
310 static void stop_pci_io(struct octeon_device *oct)
311 {
312 	/* No more instructions will be forwarded. */
313 	atomic_set(&oct->status, OCT_DEV_IN_RESET);
314 
315 	pci_disable_device(oct->pci_dev);
316 
317 	/* Disable interrupts  */
318 	oct->fn_list.disable_interrupt(oct, OCTEON_ALL_INTR);
319 
320 	pcierror_quiesce_device(oct);
321 
322 	/* Release the interrupt line */
323 	free_irq(oct->pci_dev->irq, oct);
324 
325 	if (oct->flags & LIO_FLAG_MSI_ENABLED)
326 		pci_disable_msi(oct->pci_dev);
327 
328 	dev_dbg(&oct->pci_dev->dev, "Device state is now %s\n",
329 		lio_get_state_string(&oct->status));
330 
331 	/* making it a common function for all OCTEON models */
332 	cleanup_aer_uncorrect_error_status(oct->pci_dev);
333 }
334 
335 /**
336  * \brief called when PCI error is detected
337  * @param pdev Pointer to PCI device
338  * @param state The current pci connection state
339  *
340  * This function is called after a PCI bus error affecting
341  * this device has been detected.
342  */
343 static pci_ers_result_t liquidio_pcie_error_detected(struct pci_dev *pdev,
344 						     pci_channel_state_t state)
345 {
346 	struct octeon_device *oct = pci_get_drvdata(pdev);
347 
348 	/* Non-correctable Non-fatal errors */
349 	if (state == pci_channel_io_normal) {
350 		dev_err(&oct->pci_dev->dev, "Non-correctable non-fatal error reported:\n");
351 		cleanup_aer_uncorrect_error_status(oct->pci_dev);
352 		return PCI_ERS_RESULT_CAN_RECOVER;
353 	}
354 
355 	/* Non-correctable Fatal errors */
356 	dev_err(&oct->pci_dev->dev, "Non-correctable FATAL reported by PCI AER driver\n");
357 	stop_pci_io(oct);
358 
359 	/* Always return a DISCONNECT. There is no support for recovery but only
360 	 * for a clean shutdown.
361 	 */
362 	return PCI_ERS_RESULT_DISCONNECT;
363 }
364 
365 /**
366  * \brief mmio handler
367  * @param pdev Pointer to PCI device
368  */
369 static pci_ers_result_t liquidio_pcie_mmio_enabled(
370 				struct pci_dev *pdev __attribute__((unused)))
371 {
372 	/* We should never hit this since we never ask for a reset for a Fatal
373 	 * Error. We always return DISCONNECT in io_error above.
374 	 * But play safe and return RECOVERED for now.
375 	 */
376 	return PCI_ERS_RESULT_RECOVERED;
377 }
378 
379 /**
380  * \brief called after the pci bus has been reset.
381  * @param pdev Pointer to PCI device
382  *
383  * Restart the card from scratch, as if from a cold-boot. Implementation
384  * resembles the first-half of the octeon_resume routine.
385  */
386 static pci_ers_result_t liquidio_pcie_slot_reset(
387 				struct pci_dev *pdev __attribute__((unused)))
388 {
389 	/* We should never hit this since we never ask for a reset for a Fatal
390 	 * Error. We always return DISCONNECT in io_error above.
391 	 * But play safe and return RECOVERED for now.
392 	 */
393 	return PCI_ERS_RESULT_RECOVERED;
394 }
395 
396 /**
397  * \brief called when traffic can start flowing again.
398  * @param pdev Pointer to PCI device
399  *
400  * This callback is called when the error recovery driver tells us that
401  * its OK to resume normal operation. Implementation resembles the
402  * second-half of the octeon_resume routine.
403  */
404 static void liquidio_pcie_resume(struct pci_dev *pdev __attribute__((unused)))
405 {
406 	/* Nothing to be done here. */
407 }
408 
409 #ifdef CONFIG_PM
410 /**
411  * \brief called when suspending
412  * @param pdev Pointer to PCI device
413  * @param state state to suspend to
414  */
415 static int liquidio_suspend(struct pci_dev *pdev __attribute__((unused)),
416 			    pm_message_t state __attribute__((unused)))
417 {
418 	return 0;
419 }
420 
421 /**
422  * \brief called when resuming
423  * @param pdev Pointer to PCI device
424  */
425 static int liquidio_resume(struct pci_dev *pdev __attribute__((unused)))
426 {
427 	return 0;
428 }
429 #endif
430 
431 /* For PCI-E Advanced Error Recovery (AER) Interface */
432 static const struct pci_error_handlers liquidio_err_handler = {
433 	.error_detected = liquidio_pcie_error_detected,
434 	.mmio_enabled	= liquidio_pcie_mmio_enabled,
435 	.slot_reset	= liquidio_pcie_slot_reset,
436 	.resume		= liquidio_pcie_resume,
437 };
438 
439 static const struct pci_device_id liquidio_pci_tbl[] = {
440 	{       /* 68xx */
441 		PCI_VENDOR_ID_CAVIUM, 0x91, PCI_ANY_ID, PCI_ANY_ID, 0, 0, 0
442 	},
443 	{       /* 66xx */
444 		PCI_VENDOR_ID_CAVIUM, 0x92, PCI_ANY_ID, PCI_ANY_ID, 0, 0, 0
445 	},
446 	{       /* 23xx pf */
447 		PCI_VENDOR_ID_CAVIUM, 0x9702, PCI_ANY_ID, PCI_ANY_ID, 0, 0, 0
448 	},
449 	{
450 		0, 0, 0, 0, 0, 0, 0
451 	}
452 };
453 MODULE_DEVICE_TABLE(pci, liquidio_pci_tbl);
454 
455 static struct pci_driver liquidio_pci_driver = {
456 	.name		= "LiquidIO",
457 	.id_table	= liquidio_pci_tbl,
458 	.probe		= liquidio_probe,
459 	.remove		= liquidio_remove,
460 	.err_handler	= &liquidio_err_handler,    /* For AER */
461 
462 #ifdef CONFIG_PM
463 	.suspend	= liquidio_suspend,
464 	.resume		= liquidio_resume,
465 #endif
466 #ifdef CONFIG_PCI_IOV
467 	.sriov_configure = liquidio_enable_sriov,
468 #endif
469 };
470 
471 /**
472  * \brief register PCI driver
473  */
474 static int liquidio_init_pci(void)
475 {
476 	return pci_register_driver(&liquidio_pci_driver);
477 }
478 
479 /**
480  * \brief unregister PCI driver
481  */
482 static void liquidio_deinit_pci(void)
483 {
484 	pci_unregister_driver(&liquidio_pci_driver);
485 }
486 
487 /**
488  * \brief Check Tx queue status, and take appropriate action
489  * @param lio per-network private data
490  * @returns 0 if full, number of queues woken up otherwise
491  */
492 static inline int check_txq_status(struct lio *lio)
493 {
494 	int numqs = lio->netdev->real_num_tx_queues;
495 	int ret_val = 0;
496 	int q, iq;
497 
498 	/* check each sub-queue state */
499 	for (q = 0; q < numqs; q++) {
500 		iq = lio->linfo.txpciq[q %
501 			lio->oct_dev->num_iqs].s.q_no;
502 		if (octnet_iq_is_full(lio->oct_dev, iq))
503 			continue;
504 		if (__netif_subqueue_stopped(lio->netdev, q)) {
505 			netif_wake_subqueue(lio->netdev, q);
506 			INCR_INSTRQUEUE_PKT_COUNT(lio->oct_dev, iq,
507 						  tx_restart, 1);
508 			ret_val++;
509 		}
510 	}
511 
512 	return ret_val;
513 }
514 
515 /**
516  * \brief Print link information
517  * @param netdev network device
518  */
519 static void print_link_info(struct net_device *netdev)
520 {
521 	struct lio *lio = GET_LIO(netdev);
522 
523 	if (!ifstate_check(lio, LIO_IFSTATE_RESETTING) &&
524 	    ifstate_check(lio, LIO_IFSTATE_REGISTERED)) {
525 		struct oct_link_info *linfo = &lio->linfo;
526 
527 		if (linfo->link.s.link_up) {
528 			netif_info(lio, link, lio->netdev, "%d Mbps %s Duplex UP\n",
529 				   linfo->link.s.speed,
530 				   (linfo->link.s.duplex) ? "Full" : "Half");
531 		} else {
532 			netif_info(lio, link, lio->netdev, "Link Down\n");
533 		}
534 	}
535 }
536 
537 /**
538  * \brief Routine to notify MTU change
539  * @param work work_struct data structure
540  */
541 static void octnet_link_status_change(struct work_struct *work)
542 {
543 	struct cavium_wk *wk = (struct cavium_wk *)work;
544 	struct lio *lio = (struct lio *)wk->ctxptr;
545 
546 	/* lio->linfo.link.s.mtu always contains max MTU of the lio interface.
547 	 * this API is invoked only when new max-MTU of the interface is
548 	 * less than current MTU.
549 	 */
550 	rtnl_lock();
551 	dev_set_mtu(lio->netdev, lio->linfo.link.s.mtu);
552 	rtnl_unlock();
553 }
554 
555 /**
556  * \brief Sets up the mtu status change work
557  * @param netdev network device
558  */
559 static inline int setup_link_status_change_wq(struct net_device *netdev)
560 {
561 	struct lio *lio = GET_LIO(netdev);
562 	struct octeon_device *oct = lio->oct_dev;
563 
564 	lio->link_status_wq.wq = alloc_workqueue("link-status",
565 						 WQ_MEM_RECLAIM, 0);
566 	if (!lio->link_status_wq.wq) {
567 		dev_err(&oct->pci_dev->dev, "unable to create cavium link status wq\n");
568 		return -1;
569 	}
570 	INIT_DELAYED_WORK(&lio->link_status_wq.wk.work,
571 			  octnet_link_status_change);
572 	lio->link_status_wq.wk.ctxptr = lio;
573 
574 	return 0;
575 }
576 
577 static inline void cleanup_link_status_change_wq(struct net_device *netdev)
578 {
579 	struct lio *lio = GET_LIO(netdev);
580 
581 	if (lio->link_status_wq.wq) {
582 		cancel_delayed_work_sync(&lio->link_status_wq.wk.work);
583 		destroy_workqueue(lio->link_status_wq.wq);
584 	}
585 }
586 
587 /**
588  * \brief Update link status
589  * @param netdev network device
590  * @param ls link status structure
591  *
592  * Called on receipt of a link status response from the core application to
593  * update each interface's link status.
594  */
595 static inline void update_link_status(struct net_device *netdev,
596 				      union oct_link_status *ls)
597 {
598 	struct lio *lio = GET_LIO(netdev);
599 	int changed = (lio->linfo.link.u64 != ls->u64);
600 	int current_max_mtu = lio->linfo.link.s.mtu;
601 	struct octeon_device *oct = lio->oct_dev;
602 
603 	dev_dbg(&oct->pci_dev->dev, "%s: lio->linfo.link.u64=%llx, ls->u64=%llx\n",
604 		__func__, lio->linfo.link.u64, ls->u64);
605 	lio->linfo.link.u64 = ls->u64;
606 
607 	if ((lio->intf_open) && (changed)) {
608 		print_link_info(netdev);
609 		lio->link_changes++;
610 
611 		if (lio->linfo.link.s.link_up) {
612 			dev_dbg(&oct->pci_dev->dev, "%s: link_up", __func__);
613 			netif_carrier_on(netdev);
614 			wake_txqs(netdev);
615 		} else {
616 			dev_dbg(&oct->pci_dev->dev, "%s: link_off", __func__);
617 			netif_carrier_off(netdev);
618 			stop_txqs(netdev);
619 		}
620 		if (lio->linfo.link.s.mtu != current_max_mtu) {
621 			netif_info(lio, probe, lio->netdev, "Max MTU changed from %d to %d\n",
622 				   current_max_mtu, lio->linfo.link.s.mtu);
623 			netdev->max_mtu = lio->linfo.link.s.mtu;
624 		}
625 		if (lio->linfo.link.s.mtu < netdev->mtu) {
626 			dev_warn(&oct->pci_dev->dev,
627 				 "Current MTU is higher than new max MTU; Reducing the current mtu from %d to %d\n",
628 				     netdev->mtu, lio->linfo.link.s.mtu);
629 			queue_delayed_work(lio->link_status_wq.wq,
630 					   &lio->link_status_wq.wk.work, 0);
631 		}
632 	}
633 }
634 
635 /**
636  * lio_sync_octeon_time - send latest localtime to octeon firmware so that
637  * firmware will correct it's time, in case there is a time skew
638  *
639  * @work: work scheduled to send time update to octeon firmware
640  **/
641 static void lio_sync_octeon_time(struct work_struct *work)
642 {
643 	struct cavium_wk *wk = (struct cavium_wk *)work;
644 	struct lio *lio = (struct lio *)wk->ctxptr;
645 	struct octeon_device *oct = lio->oct_dev;
646 	struct octeon_soft_command *sc;
647 	struct timespec64 ts;
648 	struct lio_time *lt;
649 	int ret;
650 
651 	sc = octeon_alloc_soft_command(oct, sizeof(struct lio_time), 16, 0);
652 	if (!sc) {
653 		dev_err(&oct->pci_dev->dev,
654 			"Failed to sync time to octeon: soft command allocation failed\n");
655 		return;
656 	}
657 
658 	lt = (struct lio_time *)sc->virtdptr;
659 
660 	/* Get time of the day */
661 	ktime_get_real_ts64(&ts);
662 	lt->sec = ts.tv_sec;
663 	lt->nsec = ts.tv_nsec;
664 	octeon_swap_8B_data((u64 *)lt, (sizeof(struct lio_time)) / 8);
665 
666 	sc->iq_no = lio->linfo.txpciq[0].s.q_no;
667 	octeon_prepare_soft_command(oct, sc, OPCODE_NIC,
668 				    OPCODE_NIC_SYNC_OCTEON_TIME, 0, 0, 0);
669 
670 	init_completion(&sc->complete);
671 	sc->sc_status = OCTEON_REQUEST_PENDING;
672 
673 	ret = octeon_send_soft_command(oct, sc);
674 	if (ret == IQ_SEND_FAILED) {
675 		dev_err(&oct->pci_dev->dev,
676 			"Failed to sync time to octeon: failed to send soft command\n");
677 		octeon_free_soft_command(oct, sc);
678 	} else {
679 		WRITE_ONCE(sc->caller_is_done, true);
680 	}
681 
682 	queue_delayed_work(lio->sync_octeon_time_wq.wq,
683 			   &lio->sync_octeon_time_wq.wk.work,
684 			   msecs_to_jiffies(LIO_SYNC_OCTEON_TIME_INTERVAL_MS));
685 }
686 
687 /**
688  * setup_sync_octeon_time_wq - Sets up the work to periodically update
689  * local time to octeon firmware
690  *
691  * @netdev - network device which should send time update to firmware
692  **/
693 static inline int setup_sync_octeon_time_wq(struct net_device *netdev)
694 {
695 	struct lio *lio = GET_LIO(netdev);
696 	struct octeon_device *oct = lio->oct_dev;
697 
698 	lio->sync_octeon_time_wq.wq =
699 		alloc_workqueue("update-octeon-time", WQ_MEM_RECLAIM, 0);
700 	if (!lio->sync_octeon_time_wq.wq) {
701 		dev_err(&oct->pci_dev->dev, "Unable to create wq to update octeon time\n");
702 		return -1;
703 	}
704 	INIT_DELAYED_WORK(&lio->sync_octeon_time_wq.wk.work,
705 			  lio_sync_octeon_time);
706 	lio->sync_octeon_time_wq.wk.ctxptr = lio;
707 	queue_delayed_work(lio->sync_octeon_time_wq.wq,
708 			   &lio->sync_octeon_time_wq.wk.work,
709 			   msecs_to_jiffies(LIO_SYNC_OCTEON_TIME_INTERVAL_MS));
710 
711 	return 0;
712 }
713 
714 /**
715  * cleanup_sync_octeon_time_wq - stop scheduling and destroy the work created
716  * to periodically update local time to octeon firmware
717  *
718  * @netdev - network device which should send time update to firmware
719  **/
720 static inline void cleanup_sync_octeon_time_wq(struct net_device *netdev)
721 {
722 	struct lio *lio = GET_LIO(netdev);
723 	struct cavium_wq *time_wq = &lio->sync_octeon_time_wq;
724 
725 	if (time_wq->wq) {
726 		cancel_delayed_work_sync(&time_wq->wk.work);
727 		destroy_workqueue(time_wq->wq);
728 	}
729 }
730 
731 static struct octeon_device *get_other_octeon_device(struct octeon_device *oct)
732 {
733 	struct octeon_device *other_oct;
734 
735 	other_oct = lio_get_device(oct->octeon_id + 1);
736 
737 	if (other_oct && other_oct->pci_dev) {
738 		int oct_busnum, other_oct_busnum;
739 
740 		oct_busnum = oct->pci_dev->bus->number;
741 		other_oct_busnum = other_oct->pci_dev->bus->number;
742 
743 		if (oct_busnum == other_oct_busnum) {
744 			int oct_slot, other_oct_slot;
745 
746 			oct_slot = PCI_SLOT(oct->pci_dev->devfn);
747 			other_oct_slot = PCI_SLOT(other_oct->pci_dev->devfn);
748 
749 			if (oct_slot == other_oct_slot)
750 				return other_oct;
751 		}
752 	}
753 
754 	return NULL;
755 }
756 
757 static void disable_all_vf_links(struct octeon_device *oct)
758 {
759 	struct net_device *netdev;
760 	int max_vfs, vf, i;
761 
762 	if (!oct)
763 		return;
764 
765 	max_vfs = oct->sriov_info.max_vfs;
766 
767 	for (i = 0; i < oct->ifcount; i++) {
768 		netdev = oct->props[i].netdev;
769 		if (!netdev)
770 			continue;
771 
772 		for (vf = 0; vf < max_vfs; vf++)
773 			liquidio_set_vf_link_state(netdev, vf,
774 						   IFLA_VF_LINK_STATE_DISABLE);
775 	}
776 }
777 
778 static int liquidio_watchdog(void *param)
779 {
780 	bool err_msg_was_printed[LIO_MAX_CORES];
781 	u16 mask_of_crashed_or_stuck_cores = 0;
782 	bool all_vf_links_are_disabled = false;
783 	struct octeon_device *oct = param;
784 	struct octeon_device *other_oct;
785 #ifdef CONFIG_MODULE_UNLOAD
786 	long refcount, vfs_referencing_pf;
787 	u64 vfs_mask1, vfs_mask2;
788 #endif
789 	int core;
790 
791 	memset(err_msg_was_printed, 0, sizeof(err_msg_was_printed));
792 
793 	while (!kthread_should_stop()) {
794 		/* sleep for a couple of seconds so that we don't hog the CPU */
795 		set_current_state(TASK_INTERRUPTIBLE);
796 		schedule_timeout(msecs_to_jiffies(2000));
797 
798 		mask_of_crashed_or_stuck_cores =
799 		    (u16)octeon_read_csr64(oct, CN23XX_SLI_SCRATCH2);
800 
801 		if (!mask_of_crashed_or_stuck_cores)
802 			continue;
803 
804 		WRITE_ONCE(oct->cores_crashed, true);
805 		other_oct = get_other_octeon_device(oct);
806 		if (other_oct)
807 			WRITE_ONCE(other_oct->cores_crashed, true);
808 
809 		for (core = 0; core < LIO_MAX_CORES; core++) {
810 			bool core_crashed_or_got_stuck;
811 
812 			core_crashed_or_got_stuck =
813 						(mask_of_crashed_or_stuck_cores
814 						 >> core) & 1;
815 
816 			if (core_crashed_or_got_stuck &&
817 			    !err_msg_was_printed[core]) {
818 				dev_err(&oct->pci_dev->dev,
819 					"ERROR: Octeon core %d crashed or got stuck!  See oct-fwdump for details.\n",
820 					core);
821 				err_msg_was_printed[core] = true;
822 			}
823 		}
824 
825 		if (all_vf_links_are_disabled)
826 			continue;
827 
828 		disable_all_vf_links(oct);
829 		disable_all_vf_links(other_oct);
830 		all_vf_links_are_disabled = true;
831 
832 #ifdef CONFIG_MODULE_UNLOAD
833 		vfs_mask1 = READ_ONCE(oct->sriov_info.vf_drv_loaded_mask);
834 		vfs_mask2 = READ_ONCE(other_oct->sriov_info.vf_drv_loaded_mask);
835 
836 		vfs_referencing_pf  = hweight64(vfs_mask1);
837 		vfs_referencing_pf += hweight64(vfs_mask2);
838 
839 		refcount = module_refcount(THIS_MODULE);
840 		if (refcount >= vfs_referencing_pf) {
841 			while (vfs_referencing_pf) {
842 				module_put(THIS_MODULE);
843 				vfs_referencing_pf--;
844 			}
845 		}
846 #endif
847 	}
848 
849 	return 0;
850 }
851 
852 /**
853  * \brief PCI probe handler
854  * @param pdev PCI device structure
855  * @param ent unused
856  */
857 static int
858 liquidio_probe(struct pci_dev *pdev,
859 	       const struct pci_device_id *ent __attribute__((unused)))
860 {
861 	struct octeon_device *oct_dev = NULL;
862 	struct handshake *hs;
863 
864 	oct_dev = octeon_allocate_device(pdev->device,
865 					 sizeof(struct octeon_device_priv));
866 	if (!oct_dev) {
867 		dev_err(&pdev->dev, "Unable to allocate device\n");
868 		return -ENOMEM;
869 	}
870 
871 	if (pdev->device == OCTEON_CN23XX_PF_VID)
872 		oct_dev->msix_on = LIO_FLAG_MSIX_ENABLED;
873 
874 	/* Enable PTP for 6XXX Device */
875 	if (((pdev->device == OCTEON_CN66XX) ||
876 	     (pdev->device == OCTEON_CN68XX)))
877 		oct_dev->ptp_enable = true;
878 	else
879 		oct_dev->ptp_enable = false;
880 
881 	dev_info(&pdev->dev, "Initializing device %x:%x.\n",
882 		 (u32)pdev->vendor, (u32)pdev->device);
883 
884 	/* Assign octeon_device for this device to the private data area. */
885 	pci_set_drvdata(pdev, oct_dev);
886 
887 	/* set linux specific device pointer */
888 	oct_dev->pci_dev = (void *)pdev;
889 
890 	oct_dev->subsystem_id = pdev->subsystem_vendor |
891 		(pdev->subsystem_device << 16);
892 
893 	hs = &handshake[oct_dev->octeon_id];
894 	init_completion(&hs->init);
895 	init_completion(&hs->started);
896 	hs->pci_dev = pdev;
897 
898 	if (oct_dev->octeon_id == 0)
899 		/* first LiquidIO NIC is detected */
900 		complete(&first_stage);
901 
902 	if (octeon_device_init(oct_dev)) {
903 		complete(&hs->init);
904 		liquidio_remove(pdev);
905 		return -ENOMEM;
906 	}
907 
908 	if (OCTEON_CN23XX_PF(oct_dev)) {
909 		u8 bus, device, function;
910 
911 		if (atomic_read(oct_dev->adapter_refcount) == 1) {
912 			/* Each NIC gets one watchdog kernel thread.  The first
913 			 * PF (of each NIC) that gets pci_driver->probe()'d
914 			 * creates that thread.
915 			 */
916 			bus = pdev->bus->number;
917 			device = PCI_SLOT(pdev->devfn);
918 			function = PCI_FUNC(pdev->devfn);
919 			oct_dev->watchdog_task = kthread_create(
920 			    liquidio_watchdog, oct_dev,
921 			    "liowd/%02hhx:%02hhx.%hhx", bus, device, function);
922 			if (!IS_ERR(oct_dev->watchdog_task)) {
923 				wake_up_process(oct_dev->watchdog_task);
924 			} else {
925 				oct_dev->watchdog_task = NULL;
926 				dev_err(&oct_dev->pci_dev->dev,
927 					"failed to create kernel_thread\n");
928 				liquidio_remove(pdev);
929 				return -1;
930 			}
931 		}
932 	}
933 
934 	oct_dev->rx_pause = 1;
935 	oct_dev->tx_pause = 1;
936 
937 	dev_dbg(&oct_dev->pci_dev->dev, "Device is ready\n");
938 
939 	return 0;
940 }
941 
942 static bool fw_type_is_auto(void)
943 {
944 	return strncmp(fw_type, LIO_FW_NAME_TYPE_AUTO,
945 		       sizeof(LIO_FW_NAME_TYPE_AUTO)) == 0;
946 }
947 
948 /**
949  * \brief PCI FLR for each Octeon device.
950  * @param oct octeon device
951  */
952 static void octeon_pci_flr(struct octeon_device *oct)
953 {
954 	int rc;
955 
956 	pci_save_state(oct->pci_dev);
957 
958 	pci_cfg_access_lock(oct->pci_dev);
959 
960 	/* Quiesce the device completely */
961 	pci_write_config_word(oct->pci_dev, PCI_COMMAND,
962 			      PCI_COMMAND_INTX_DISABLE);
963 
964 	rc = __pci_reset_function_locked(oct->pci_dev);
965 
966 	if (rc != 0)
967 		dev_err(&oct->pci_dev->dev, "Error %d resetting PCI function %d\n",
968 			rc, oct->pf_num);
969 
970 	pci_cfg_access_unlock(oct->pci_dev);
971 
972 	pci_restore_state(oct->pci_dev);
973 }
974 
975 /**
976  *\brief Destroy resources associated with octeon device
977  * @param pdev PCI device structure
978  * @param ent unused
979  */
980 static void octeon_destroy_resources(struct octeon_device *oct)
981 {
982 	int i, refcount;
983 	struct msix_entry *msix_entries;
984 	struct octeon_device_priv *oct_priv =
985 		(struct octeon_device_priv *)oct->priv;
986 
987 	struct handshake *hs;
988 
989 	switch (atomic_read(&oct->status)) {
990 	case OCT_DEV_RUNNING:
991 	case OCT_DEV_CORE_OK:
992 
993 		/* No more instructions will be forwarded. */
994 		atomic_set(&oct->status, OCT_DEV_IN_RESET);
995 
996 		oct->app_mode = CVM_DRV_INVALID_APP;
997 		dev_dbg(&oct->pci_dev->dev, "Device state is now %s\n",
998 			lio_get_state_string(&oct->status));
999 
1000 		schedule_timeout_uninterruptible(HZ / 10);
1001 
1002 		/* fallthrough */
1003 	case OCT_DEV_HOST_OK:
1004 
1005 		/* fallthrough */
1006 	case OCT_DEV_CONSOLE_INIT_DONE:
1007 		/* Remove any consoles */
1008 		octeon_remove_consoles(oct);
1009 
1010 		/* fallthrough */
1011 	case OCT_DEV_IO_QUEUES_DONE:
1012 		if (lio_wait_for_instr_fetch(oct))
1013 			dev_err(&oct->pci_dev->dev, "IQ had pending instructions\n");
1014 
1015 		if (wait_for_pending_requests(oct))
1016 			dev_err(&oct->pci_dev->dev, "There were pending requests\n");
1017 
1018 		/* Disable the input and output queues now. No more packets will
1019 		 * arrive from Octeon, but we should wait for all packet
1020 		 * processing to finish.
1021 		 */
1022 		oct->fn_list.disable_io_queues(oct);
1023 
1024 		if (lio_wait_for_oq_pkts(oct))
1025 			dev_err(&oct->pci_dev->dev, "OQ had pending packets\n");
1026 
1027 		/* Force all requests waiting to be fetched by OCTEON to
1028 		 * complete.
1029 		 */
1030 		for (i = 0; i < MAX_OCTEON_INSTR_QUEUES(oct); i++) {
1031 			struct octeon_instr_queue *iq;
1032 
1033 			if (!(oct->io_qmask.iq & BIT_ULL(i)))
1034 				continue;
1035 			iq = oct->instr_queue[i];
1036 
1037 			if (atomic_read(&iq->instr_pending)) {
1038 				spin_lock_bh(&iq->lock);
1039 				iq->fill_cnt = 0;
1040 				iq->octeon_read_index = iq->host_write_index;
1041 				iq->stats.instr_processed +=
1042 					atomic_read(&iq->instr_pending);
1043 				lio_process_iq_request_list(oct, iq, 0);
1044 				spin_unlock_bh(&iq->lock);
1045 			}
1046 		}
1047 
1048 		lio_process_ordered_list(oct, 1);
1049 		octeon_free_sc_done_list(oct);
1050 		octeon_free_sc_zombie_list(oct);
1051 
1052 	/* fallthrough */
1053 	case OCT_DEV_INTR_SET_DONE:
1054 		/* Disable interrupts  */
1055 		oct->fn_list.disable_interrupt(oct, OCTEON_ALL_INTR);
1056 
1057 		if (oct->msix_on) {
1058 			msix_entries = (struct msix_entry *)oct->msix_entries;
1059 			for (i = 0; i < oct->num_msix_irqs - 1; i++) {
1060 				if (oct->ioq_vector[i].vector) {
1061 					/* clear the affinity_cpumask */
1062 					irq_set_affinity_hint(
1063 							msix_entries[i].vector,
1064 							NULL);
1065 					free_irq(msix_entries[i].vector,
1066 						 &oct->ioq_vector[i]);
1067 					oct->ioq_vector[i].vector = 0;
1068 				}
1069 			}
1070 			/* non-iov vector's argument is oct struct */
1071 			free_irq(msix_entries[i].vector, oct);
1072 
1073 			pci_disable_msix(oct->pci_dev);
1074 			kfree(oct->msix_entries);
1075 			oct->msix_entries = NULL;
1076 		} else {
1077 			/* Release the interrupt line */
1078 			free_irq(oct->pci_dev->irq, oct);
1079 
1080 			if (oct->flags & LIO_FLAG_MSI_ENABLED)
1081 				pci_disable_msi(oct->pci_dev);
1082 		}
1083 
1084 		kfree(oct->irq_name_storage);
1085 		oct->irq_name_storage = NULL;
1086 
1087 	/* fallthrough */
1088 	case OCT_DEV_MSIX_ALLOC_VECTOR_DONE:
1089 		if (OCTEON_CN23XX_PF(oct))
1090 			octeon_free_ioq_vector(oct);
1091 
1092 	/* fallthrough */
1093 	case OCT_DEV_MBOX_SETUP_DONE:
1094 		if (OCTEON_CN23XX_PF(oct))
1095 			oct->fn_list.free_mbox(oct);
1096 
1097 	/* fallthrough */
1098 	case OCT_DEV_IN_RESET:
1099 	case OCT_DEV_DROQ_INIT_DONE:
1100 		/* Wait for any pending operations */
1101 		mdelay(100);
1102 		for (i = 0; i < MAX_OCTEON_OUTPUT_QUEUES(oct); i++) {
1103 			if (!(oct->io_qmask.oq & BIT_ULL(i)))
1104 				continue;
1105 			octeon_delete_droq(oct, i);
1106 		}
1107 
1108 		/* Force any pending handshakes to complete */
1109 		for (i = 0; i < MAX_OCTEON_DEVICES; i++) {
1110 			hs = &handshake[i];
1111 
1112 			if (hs->pci_dev) {
1113 				handshake[oct->octeon_id].init_ok = 0;
1114 				complete(&handshake[oct->octeon_id].init);
1115 				handshake[oct->octeon_id].started_ok = 0;
1116 				complete(&handshake[oct->octeon_id].started);
1117 			}
1118 		}
1119 
1120 		/* fallthrough */
1121 	case OCT_DEV_RESP_LIST_INIT_DONE:
1122 		octeon_delete_response_list(oct);
1123 
1124 		/* fallthrough */
1125 	case OCT_DEV_INSTR_QUEUE_INIT_DONE:
1126 		for (i = 0; i < MAX_OCTEON_INSTR_QUEUES(oct); i++) {
1127 			if (!(oct->io_qmask.iq & BIT_ULL(i)))
1128 				continue;
1129 			octeon_delete_instr_queue(oct, i);
1130 		}
1131 #ifdef CONFIG_PCI_IOV
1132 		if (oct->sriov_info.sriov_enabled)
1133 			pci_disable_sriov(oct->pci_dev);
1134 #endif
1135 		/* fallthrough */
1136 	case OCT_DEV_SC_BUFF_POOL_INIT_DONE:
1137 		octeon_free_sc_buffer_pool(oct);
1138 
1139 		/* fallthrough */
1140 	case OCT_DEV_DISPATCH_INIT_DONE:
1141 		octeon_delete_dispatch_list(oct);
1142 		cancel_delayed_work_sync(&oct->nic_poll_work.work);
1143 
1144 		/* fallthrough */
1145 	case OCT_DEV_PCI_MAP_DONE:
1146 		refcount = octeon_deregister_device(oct);
1147 
1148 		/* Soft reset the octeon device before exiting.
1149 		 * However, if fw was loaded from card (i.e. autoboot),
1150 		 * perform an FLR instead.
1151 		 * Implementation note: only soft-reset the device
1152 		 * if it is a CN6XXX OR the LAST CN23XX device.
1153 		 */
1154 		if (atomic_read(oct->adapter_fw_state) == FW_IS_PRELOADED)
1155 			octeon_pci_flr(oct);
1156 		else if (OCTEON_CN6XXX(oct) || !refcount)
1157 			oct->fn_list.soft_reset(oct);
1158 
1159 		octeon_unmap_pci_barx(oct, 0);
1160 		octeon_unmap_pci_barx(oct, 1);
1161 
1162 		/* fallthrough */
1163 	case OCT_DEV_PCI_ENABLE_DONE:
1164 		pci_clear_master(oct->pci_dev);
1165 		/* Disable the device, releasing the PCI INT */
1166 		pci_disable_device(oct->pci_dev);
1167 
1168 		/* fallthrough */
1169 	case OCT_DEV_BEGIN_STATE:
1170 		/* Nothing to be done here either */
1171 		break;
1172 	}                       /* end switch (oct->status) */
1173 
1174 	tasklet_kill(&oct_priv->droq_tasklet);
1175 }
1176 
1177 /**
1178  * \brief Send Rx control command
1179  * @param lio per-network private data
1180  * @param start_stop whether to start or stop
1181  */
1182 static void send_rx_ctrl_cmd(struct lio *lio, int start_stop)
1183 {
1184 	struct octeon_soft_command *sc;
1185 	union octnet_cmd *ncmd;
1186 	struct octeon_device *oct = (struct octeon_device *)lio->oct_dev;
1187 	int retval;
1188 
1189 	if (oct->props[lio->ifidx].rx_on == start_stop)
1190 		return;
1191 
1192 	sc = (struct octeon_soft_command *)
1193 		octeon_alloc_soft_command(oct, OCTNET_CMD_SIZE,
1194 					  16, 0);
1195 	if (!sc) {
1196 		netif_info(lio, rx_err, lio->netdev,
1197 			   "Failed to allocate octeon_soft_command\n");
1198 		return;
1199 	}
1200 
1201 	ncmd = (union octnet_cmd *)sc->virtdptr;
1202 
1203 	ncmd->u64 = 0;
1204 	ncmd->s.cmd = OCTNET_CMD_RX_CTL;
1205 	ncmd->s.param1 = start_stop;
1206 
1207 	octeon_swap_8B_data((u64 *)ncmd, (OCTNET_CMD_SIZE >> 3));
1208 
1209 	sc->iq_no = lio->linfo.txpciq[0].s.q_no;
1210 
1211 	octeon_prepare_soft_command(oct, sc, OPCODE_NIC,
1212 				    OPCODE_NIC_CMD, 0, 0, 0);
1213 
1214 	init_completion(&sc->complete);
1215 	sc->sc_status = OCTEON_REQUEST_PENDING;
1216 
1217 	retval = octeon_send_soft_command(oct, sc);
1218 	if (retval == IQ_SEND_FAILED) {
1219 		netif_info(lio, rx_err, lio->netdev, "Failed to send RX Control message\n");
1220 		octeon_free_soft_command(oct, sc);
1221 		return;
1222 	} else {
1223 		/* Sleep on a wait queue till the cond flag indicates that the
1224 		 * response arrived or timed-out.
1225 		 */
1226 		retval = wait_for_sc_completion_timeout(oct, sc, 0);
1227 		if (retval)
1228 			return;
1229 
1230 		oct->props[lio->ifidx].rx_on = start_stop;
1231 		WRITE_ONCE(sc->caller_is_done, true);
1232 	}
1233 }
1234 
1235 /**
1236  * \brief Destroy NIC device interface
1237  * @param oct octeon device
1238  * @param ifidx which interface to destroy
1239  *
1240  * Cleanup associated with each interface for an Octeon device  when NIC
1241  * module is being unloaded or if initialization fails during load.
1242  */
1243 static void liquidio_destroy_nic_device(struct octeon_device *oct, int ifidx)
1244 {
1245 	struct net_device *netdev = oct->props[ifidx].netdev;
1246 	struct octeon_device_priv *oct_priv =
1247 		(struct octeon_device_priv *)oct->priv;
1248 	struct napi_struct *napi, *n;
1249 	struct lio *lio;
1250 
1251 	if (!netdev) {
1252 		dev_err(&oct->pci_dev->dev, "%s No netdevice ptr for index %d\n",
1253 			__func__, ifidx);
1254 		return;
1255 	}
1256 
1257 	lio = GET_LIO(netdev);
1258 
1259 	dev_dbg(&oct->pci_dev->dev, "NIC device cleanup\n");
1260 
1261 	if (atomic_read(&lio->ifstate) & LIO_IFSTATE_RUNNING)
1262 		liquidio_stop(netdev);
1263 
1264 	if (oct->props[lio->ifidx].napi_enabled == 1) {
1265 		list_for_each_entry_safe(napi, n, &netdev->napi_list, dev_list)
1266 			napi_disable(napi);
1267 
1268 		oct->props[lio->ifidx].napi_enabled = 0;
1269 
1270 		if (OCTEON_CN23XX_PF(oct))
1271 			oct->droq[0]->ops.poll_mode = 0;
1272 	}
1273 
1274 	/* Delete NAPI */
1275 	list_for_each_entry_safe(napi, n, &netdev->napi_list, dev_list)
1276 		netif_napi_del(napi);
1277 
1278 	tasklet_enable(&oct_priv->droq_tasklet);
1279 
1280 	if (atomic_read(&lio->ifstate) & LIO_IFSTATE_REGISTERED)
1281 		unregister_netdev(netdev);
1282 
1283 	cleanup_sync_octeon_time_wq(netdev);
1284 	cleanup_link_status_change_wq(netdev);
1285 
1286 	cleanup_rx_oom_poll_fn(netdev);
1287 
1288 	lio_delete_glists(lio);
1289 
1290 	free_netdev(netdev);
1291 
1292 	oct->props[ifidx].gmxport = -1;
1293 
1294 	oct->props[ifidx].netdev = NULL;
1295 }
1296 
1297 /**
1298  * \brief Stop complete NIC functionality
1299  * @param oct octeon device
1300  */
1301 static int liquidio_stop_nic_module(struct octeon_device *oct)
1302 {
1303 	int i, j;
1304 	struct lio *lio;
1305 
1306 	dev_dbg(&oct->pci_dev->dev, "Stopping network interfaces\n");
1307 	if (!oct->ifcount) {
1308 		dev_err(&oct->pci_dev->dev, "Init for Octeon was not completed\n");
1309 		return 1;
1310 	}
1311 
1312 	spin_lock_bh(&oct->cmd_resp_wqlock);
1313 	oct->cmd_resp_state = OCT_DRV_OFFLINE;
1314 	spin_unlock_bh(&oct->cmd_resp_wqlock);
1315 
1316 	lio_vf_rep_destroy(oct);
1317 
1318 	for (i = 0; i < oct->ifcount; i++) {
1319 		lio = GET_LIO(oct->props[i].netdev);
1320 		for (j = 0; j < oct->num_oqs; j++)
1321 			octeon_unregister_droq_ops(oct,
1322 						   lio->linfo.rxpciq[j].s.q_no);
1323 	}
1324 
1325 	for (i = 0; i < oct->ifcount; i++)
1326 		liquidio_destroy_nic_device(oct, i);
1327 
1328 	if (oct->devlink) {
1329 		devlink_unregister(oct->devlink);
1330 		devlink_free(oct->devlink);
1331 		oct->devlink = NULL;
1332 	}
1333 
1334 	dev_dbg(&oct->pci_dev->dev, "Network interfaces stopped\n");
1335 	return 0;
1336 }
1337 
1338 /**
1339  * \brief Cleans up resources at unload time
1340  * @param pdev PCI device structure
1341  */
1342 static void liquidio_remove(struct pci_dev *pdev)
1343 {
1344 	struct octeon_device *oct_dev = pci_get_drvdata(pdev);
1345 
1346 	dev_dbg(&oct_dev->pci_dev->dev, "Stopping device\n");
1347 
1348 	if (oct_dev->watchdog_task)
1349 		kthread_stop(oct_dev->watchdog_task);
1350 
1351 	if (!oct_dev->octeon_id &&
1352 	    oct_dev->fw_info.app_cap_flags & LIQUIDIO_SWITCHDEV_CAP)
1353 		lio_vf_rep_modexit();
1354 
1355 	if (oct_dev->app_mode && (oct_dev->app_mode == CVM_DRV_NIC_APP))
1356 		liquidio_stop_nic_module(oct_dev);
1357 
1358 	/* Reset the octeon device and cleanup all memory allocated for
1359 	 * the octeon device by driver.
1360 	 */
1361 	octeon_destroy_resources(oct_dev);
1362 
1363 	dev_info(&oct_dev->pci_dev->dev, "Device removed\n");
1364 
1365 	/* This octeon device has been removed. Update the global
1366 	 * data structure to reflect this. Free the device structure.
1367 	 */
1368 	octeon_free_device_mem(oct_dev);
1369 }
1370 
1371 /**
1372  * \brief Identify the Octeon device and to map the BAR address space
1373  * @param oct octeon device
1374  */
1375 static int octeon_chip_specific_setup(struct octeon_device *oct)
1376 {
1377 	u32 dev_id, rev_id;
1378 	int ret = 1;
1379 	char *s;
1380 
1381 	pci_read_config_dword(oct->pci_dev, 0, &dev_id);
1382 	pci_read_config_dword(oct->pci_dev, 8, &rev_id);
1383 	oct->rev_id = rev_id & 0xff;
1384 
1385 	switch (dev_id) {
1386 	case OCTEON_CN68XX_PCIID:
1387 		oct->chip_id = OCTEON_CN68XX;
1388 		ret = lio_setup_cn68xx_octeon_device(oct);
1389 		s = "CN68XX";
1390 		break;
1391 
1392 	case OCTEON_CN66XX_PCIID:
1393 		oct->chip_id = OCTEON_CN66XX;
1394 		ret = lio_setup_cn66xx_octeon_device(oct);
1395 		s = "CN66XX";
1396 		break;
1397 
1398 	case OCTEON_CN23XX_PCIID_PF:
1399 		oct->chip_id = OCTEON_CN23XX_PF_VID;
1400 		ret = setup_cn23xx_octeon_pf_device(oct);
1401 		if (ret)
1402 			break;
1403 #ifdef CONFIG_PCI_IOV
1404 		if (!ret)
1405 			pci_sriov_set_totalvfs(oct->pci_dev,
1406 					       oct->sriov_info.max_vfs);
1407 #endif
1408 		s = "CN23XX";
1409 		break;
1410 
1411 	default:
1412 		s = "?";
1413 		dev_err(&oct->pci_dev->dev, "Unknown device found (dev_id: %x)\n",
1414 			dev_id);
1415 	}
1416 
1417 	if (!ret)
1418 		dev_info(&oct->pci_dev->dev, "%s PASS%d.%d %s Version: %s\n", s,
1419 			 OCTEON_MAJOR_REV(oct),
1420 			 OCTEON_MINOR_REV(oct),
1421 			 octeon_get_conf(oct)->card_name,
1422 			 LIQUIDIO_VERSION);
1423 
1424 	return ret;
1425 }
1426 
1427 /**
1428  * \brief PCI initialization for each Octeon device.
1429  * @param oct octeon device
1430  */
1431 static int octeon_pci_os_setup(struct octeon_device *oct)
1432 {
1433 	/* setup PCI stuff first */
1434 	if (pci_enable_device(oct->pci_dev)) {
1435 		dev_err(&oct->pci_dev->dev, "pci_enable_device failed\n");
1436 		return 1;
1437 	}
1438 
1439 	if (dma_set_mask_and_coherent(&oct->pci_dev->dev, DMA_BIT_MASK(64))) {
1440 		dev_err(&oct->pci_dev->dev, "Unexpected DMA device capability\n");
1441 		pci_disable_device(oct->pci_dev);
1442 		return 1;
1443 	}
1444 
1445 	/* Enable PCI DMA Master. */
1446 	pci_set_master(oct->pci_dev);
1447 
1448 	return 0;
1449 }
1450 
1451 /**
1452  * \brief Unmap and free network buffer
1453  * @param buf buffer
1454  */
1455 static void free_netbuf(void *buf)
1456 {
1457 	struct sk_buff *skb;
1458 	struct octnet_buf_free_info *finfo;
1459 	struct lio *lio;
1460 
1461 	finfo = (struct octnet_buf_free_info *)buf;
1462 	skb = finfo->skb;
1463 	lio = finfo->lio;
1464 
1465 	dma_unmap_single(&lio->oct_dev->pci_dev->dev, finfo->dptr, skb->len,
1466 			 DMA_TO_DEVICE);
1467 
1468 	tx_buffer_free(skb);
1469 }
1470 
1471 /**
1472  * \brief Unmap and free gather buffer
1473  * @param buf buffer
1474  */
1475 static void free_netsgbuf(void *buf)
1476 {
1477 	struct octnet_buf_free_info *finfo;
1478 	struct sk_buff *skb;
1479 	struct lio *lio;
1480 	struct octnic_gather *g;
1481 	int i, frags, iq;
1482 
1483 	finfo = (struct octnet_buf_free_info *)buf;
1484 	skb = finfo->skb;
1485 	lio = finfo->lio;
1486 	g = finfo->g;
1487 	frags = skb_shinfo(skb)->nr_frags;
1488 
1489 	dma_unmap_single(&lio->oct_dev->pci_dev->dev,
1490 			 g->sg[0].ptr[0], (skb->len - skb->data_len),
1491 			 DMA_TO_DEVICE);
1492 
1493 	i = 1;
1494 	while (frags--) {
1495 		skb_frag_t *frag = &skb_shinfo(skb)->frags[i - 1];
1496 
1497 		pci_unmap_page((lio->oct_dev)->pci_dev,
1498 			       g->sg[(i >> 2)].ptr[(i & 3)],
1499 			       skb_frag_size(frag), DMA_TO_DEVICE);
1500 		i++;
1501 	}
1502 
1503 	iq = skb_iq(lio->oct_dev, skb);
1504 	spin_lock(&lio->glist_lock[iq]);
1505 	list_add_tail(&g->list, &lio->glist[iq]);
1506 	spin_unlock(&lio->glist_lock[iq]);
1507 
1508 	tx_buffer_free(skb);
1509 }
1510 
1511 /**
1512  * \brief Unmap and free gather buffer with response
1513  * @param buf buffer
1514  */
1515 static void free_netsgbuf_with_resp(void *buf)
1516 {
1517 	struct octeon_soft_command *sc;
1518 	struct octnet_buf_free_info *finfo;
1519 	struct sk_buff *skb;
1520 	struct lio *lio;
1521 	struct octnic_gather *g;
1522 	int i, frags, iq;
1523 
1524 	sc = (struct octeon_soft_command *)buf;
1525 	skb = (struct sk_buff *)sc->callback_arg;
1526 	finfo = (struct octnet_buf_free_info *)&skb->cb;
1527 
1528 	lio = finfo->lio;
1529 	g = finfo->g;
1530 	frags = skb_shinfo(skb)->nr_frags;
1531 
1532 	dma_unmap_single(&lio->oct_dev->pci_dev->dev,
1533 			 g->sg[0].ptr[0], (skb->len - skb->data_len),
1534 			 DMA_TO_DEVICE);
1535 
1536 	i = 1;
1537 	while (frags--) {
1538 		skb_frag_t *frag = &skb_shinfo(skb)->frags[i - 1];
1539 
1540 		pci_unmap_page((lio->oct_dev)->pci_dev,
1541 			       g->sg[(i >> 2)].ptr[(i & 3)],
1542 			       skb_frag_size(frag), DMA_TO_DEVICE);
1543 		i++;
1544 	}
1545 
1546 	iq = skb_iq(lio->oct_dev, skb);
1547 
1548 	spin_lock(&lio->glist_lock[iq]);
1549 	list_add_tail(&g->list, &lio->glist[iq]);
1550 	spin_unlock(&lio->glist_lock[iq]);
1551 
1552 	/* Don't free the skb yet */
1553 }
1554 
1555 /**
1556  * \brief Adjust ptp frequency
1557  * @param ptp PTP clock info
1558  * @param ppb how much to adjust by, in parts-per-billion
1559  */
1560 static int liquidio_ptp_adjfreq(struct ptp_clock_info *ptp, s32 ppb)
1561 {
1562 	struct lio *lio = container_of(ptp, struct lio, ptp_info);
1563 	struct octeon_device *oct = (struct octeon_device *)lio->oct_dev;
1564 	u64 comp, delta;
1565 	unsigned long flags;
1566 	bool neg_adj = false;
1567 
1568 	if (ppb < 0) {
1569 		neg_adj = true;
1570 		ppb = -ppb;
1571 	}
1572 
1573 	/* The hardware adds the clock compensation value to the
1574 	 * PTP clock on every coprocessor clock cycle, so we
1575 	 * compute the delta in terms of coprocessor clocks.
1576 	 */
1577 	delta = (u64)ppb << 32;
1578 	do_div(delta, oct->coproc_clock_rate);
1579 
1580 	spin_lock_irqsave(&lio->ptp_lock, flags);
1581 	comp = lio_pci_readq(oct, CN6XXX_MIO_PTP_CLOCK_COMP);
1582 	if (neg_adj)
1583 		comp -= delta;
1584 	else
1585 		comp += delta;
1586 	lio_pci_writeq(oct, comp, CN6XXX_MIO_PTP_CLOCK_COMP);
1587 	spin_unlock_irqrestore(&lio->ptp_lock, flags);
1588 
1589 	return 0;
1590 }
1591 
1592 /**
1593  * \brief Adjust ptp time
1594  * @param ptp PTP clock info
1595  * @param delta how much to adjust by, in nanosecs
1596  */
1597 static int liquidio_ptp_adjtime(struct ptp_clock_info *ptp, s64 delta)
1598 {
1599 	unsigned long flags;
1600 	struct lio *lio = container_of(ptp, struct lio, ptp_info);
1601 
1602 	spin_lock_irqsave(&lio->ptp_lock, flags);
1603 	lio->ptp_adjust += delta;
1604 	spin_unlock_irqrestore(&lio->ptp_lock, flags);
1605 
1606 	return 0;
1607 }
1608 
1609 /**
1610  * \brief Get hardware clock time, including any adjustment
1611  * @param ptp PTP clock info
1612  * @param ts timespec
1613  */
1614 static int liquidio_ptp_gettime(struct ptp_clock_info *ptp,
1615 				struct timespec64 *ts)
1616 {
1617 	u64 ns;
1618 	unsigned long flags;
1619 	struct lio *lio = container_of(ptp, struct lio, ptp_info);
1620 	struct octeon_device *oct = (struct octeon_device *)lio->oct_dev;
1621 
1622 	spin_lock_irqsave(&lio->ptp_lock, flags);
1623 	ns = lio_pci_readq(oct, CN6XXX_MIO_PTP_CLOCK_HI);
1624 	ns += lio->ptp_adjust;
1625 	spin_unlock_irqrestore(&lio->ptp_lock, flags);
1626 
1627 	*ts = ns_to_timespec64(ns);
1628 
1629 	return 0;
1630 }
1631 
1632 /**
1633  * \brief Set hardware clock time. Reset adjustment
1634  * @param ptp PTP clock info
1635  * @param ts timespec
1636  */
1637 static int liquidio_ptp_settime(struct ptp_clock_info *ptp,
1638 				const struct timespec64 *ts)
1639 {
1640 	u64 ns;
1641 	unsigned long flags;
1642 	struct lio *lio = container_of(ptp, struct lio, ptp_info);
1643 	struct octeon_device *oct = (struct octeon_device *)lio->oct_dev;
1644 
1645 	ns = timespec64_to_ns(ts);
1646 
1647 	spin_lock_irqsave(&lio->ptp_lock, flags);
1648 	lio_pci_writeq(oct, ns, CN6XXX_MIO_PTP_CLOCK_HI);
1649 	lio->ptp_adjust = 0;
1650 	spin_unlock_irqrestore(&lio->ptp_lock, flags);
1651 
1652 	return 0;
1653 }
1654 
1655 /**
1656  * \brief Check if PTP is enabled
1657  * @param ptp PTP clock info
1658  * @param rq request
1659  * @param on is it on
1660  */
1661 static int
1662 liquidio_ptp_enable(struct ptp_clock_info *ptp __attribute__((unused)),
1663 		    struct ptp_clock_request *rq __attribute__((unused)),
1664 		    int on __attribute__((unused)))
1665 {
1666 	return -EOPNOTSUPP;
1667 }
1668 
1669 /**
1670  * \brief Open PTP clock source
1671  * @param netdev network device
1672  */
1673 static void oct_ptp_open(struct net_device *netdev)
1674 {
1675 	struct lio *lio = GET_LIO(netdev);
1676 	struct octeon_device *oct = (struct octeon_device *)lio->oct_dev;
1677 
1678 	spin_lock_init(&lio->ptp_lock);
1679 
1680 	snprintf(lio->ptp_info.name, 16, "%s", netdev->name);
1681 	lio->ptp_info.owner = THIS_MODULE;
1682 	lio->ptp_info.max_adj = 250000000;
1683 	lio->ptp_info.n_alarm = 0;
1684 	lio->ptp_info.n_ext_ts = 0;
1685 	lio->ptp_info.n_per_out = 0;
1686 	lio->ptp_info.pps = 0;
1687 	lio->ptp_info.adjfreq = liquidio_ptp_adjfreq;
1688 	lio->ptp_info.adjtime = liquidio_ptp_adjtime;
1689 	lio->ptp_info.gettime64 = liquidio_ptp_gettime;
1690 	lio->ptp_info.settime64 = liquidio_ptp_settime;
1691 	lio->ptp_info.enable = liquidio_ptp_enable;
1692 
1693 	lio->ptp_adjust = 0;
1694 
1695 	lio->ptp_clock = ptp_clock_register(&lio->ptp_info,
1696 					     &oct->pci_dev->dev);
1697 
1698 	if (IS_ERR(lio->ptp_clock))
1699 		lio->ptp_clock = NULL;
1700 }
1701 
1702 /**
1703  * \brief Init PTP clock
1704  * @param oct octeon device
1705  */
1706 static void liquidio_ptp_init(struct octeon_device *oct)
1707 {
1708 	u64 clock_comp, cfg;
1709 
1710 	clock_comp = (u64)NSEC_PER_SEC << 32;
1711 	do_div(clock_comp, oct->coproc_clock_rate);
1712 	lio_pci_writeq(oct, clock_comp, CN6XXX_MIO_PTP_CLOCK_COMP);
1713 
1714 	/* Enable */
1715 	cfg = lio_pci_readq(oct, CN6XXX_MIO_PTP_CLOCK_CFG);
1716 	lio_pci_writeq(oct, cfg | 0x01, CN6XXX_MIO_PTP_CLOCK_CFG);
1717 }
1718 
1719 /**
1720  * \brief Load firmware to device
1721  * @param oct octeon device
1722  *
1723  * Maps device to firmware filename, requests firmware, and downloads it
1724  */
1725 static int load_firmware(struct octeon_device *oct)
1726 {
1727 	int ret = 0;
1728 	const struct firmware *fw;
1729 	char fw_name[LIO_MAX_FW_FILENAME_LEN];
1730 	char *tmp_fw_type;
1731 
1732 	if (fw_type_is_auto()) {
1733 		tmp_fw_type = LIO_FW_NAME_TYPE_NIC;
1734 		strncpy(fw_type, tmp_fw_type, sizeof(fw_type));
1735 	} else {
1736 		tmp_fw_type = fw_type;
1737 	}
1738 
1739 	sprintf(fw_name, "%s%s%s_%s%s", LIO_FW_DIR, LIO_FW_BASE_NAME,
1740 		octeon_get_conf(oct)->card_name, tmp_fw_type,
1741 		LIO_FW_NAME_SUFFIX);
1742 
1743 	ret = request_firmware(&fw, fw_name, &oct->pci_dev->dev);
1744 	if (ret) {
1745 		dev_err(&oct->pci_dev->dev, "Request firmware failed. Could not find file %s.\n",
1746 			fw_name);
1747 		release_firmware(fw);
1748 		return ret;
1749 	}
1750 
1751 	ret = octeon_download_firmware(oct, fw->data, fw->size);
1752 
1753 	release_firmware(fw);
1754 
1755 	return ret;
1756 }
1757 
1758 /**
1759  * \brief Poll routine for checking transmit queue status
1760  * @param work work_struct data structure
1761  */
1762 static void octnet_poll_check_txq_status(struct work_struct *work)
1763 {
1764 	struct cavium_wk *wk = (struct cavium_wk *)work;
1765 	struct lio *lio = (struct lio *)wk->ctxptr;
1766 
1767 	if (!ifstate_check(lio, LIO_IFSTATE_RUNNING))
1768 		return;
1769 
1770 	check_txq_status(lio);
1771 	queue_delayed_work(lio->txq_status_wq.wq,
1772 			   &lio->txq_status_wq.wk.work, msecs_to_jiffies(1));
1773 }
1774 
1775 /**
1776  * \brief Sets up the txq poll check
1777  * @param netdev network device
1778  */
1779 static inline int setup_tx_poll_fn(struct net_device *netdev)
1780 {
1781 	struct lio *lio = GET_LIO(netdev);
1782 	struct octeon_device *oct = lio->oct_dev;
1783 
1784 	lio->txq_status_wq.wq = alloc_workqueue("txq-status",
1785 						WQ_MEM_RECLAIM, 0);
1786 	if (!lio->txq_status_wq.wq) {
1787 		dev_err(&oct->pci_dev->dev, "unable to create cavium txq status wq\n");
1788 		return -1;
1789 	}
1790 	INIT_DELAYED_WORK(&lio->txq_status_wq.wk.work,
1791 			  octnet_poll_check_txq_status);
1792 	lio->txq_status_wq.wk.ctxptr = lio;
1793 	queue_delayed_work(lio->txq_status_wq.wq,
1794 			   &lio->txq_status_wq.wk.work, msecs_to_jiffies(1));
1795 	return 0;
1796 }
1797 
1798 static inline void cleanup_tx_poll_fn(struct net_device *netdev)
1799 {
1800 	struct lio *lio = GET_LIO(netdev);
1801 
1802 	if (lio->txq_status_wq.wq) {
1803 		cancel_delayed_work_sync(&lio->txq_status_wq.wk.work);
1804 		destroy_workqueue(lio->txq_status_wq.wq);
1805 	}
1806 }
1807 
1808 /**
1809  * \brief Net device open for LiquidIO
1810  * @param netdev network device
1811  */
1812 static int liquidio_open(struct net_device *netdev)
1813 {
1814 	struct lio *lio = GET_LIO(netdev);
1815 	struct octeon_device *oct = lio->oct_dev;
1816 	struct octeon_device_priv *oct_priv =
1817 		(struct octeon_device_priv *)oct->priv;
1818 	struct napi_struct *napi, *n;
1819 
1820 	if (oct->props[lio->ifidx].napi_enabled == 0) {
1821 		tasklet_disable(&oct_priv->droq_tasklet);
1822 
1823 		list_for_each_entry_safe(napi, n, &netdev->napi_list, dev_list)
1824 			napi_enable(napi);
1825 
1826 		oct->props[lio->ifidx].napi_enabled = 1;
1827 
1828 		if (OCTEON_CN23XX_PF(oct))
1829 			oct->droq[0]->ops.poll_mode = 1;
1830 	}
1831 
1832 	if (oct->ptp_enable)
1833 		oct_ptp_open(netdev);
1834 
1835 	ifstate_set(lio, LIO_IFSTATE_RUNNING);
1836 
1837 	if (OCTEON_CN23XX_PF(oct)) {
1838 		if (!oct->msix_on)
1839 			if (setup_tx_poll_fn(netdev))
1840 				return -1;
1841 	} else {
1842 		if (setup_tx_poll_fn(netdev))
1843 			return -1;
1844 	}
1845 
1846 	netif_tx_start_all_queues(netdev);
1847 
1848 	/* Ready for link status updates */
1849 	lio->intf_open = 1;
1850 
1851 	netif_info(lio, ifup, lio->netdev, "Interface Open, ready for traffic\n");
1852 
1853 	/* tell Octeon to start forwarding packets to host */
1854 	send_rx_ctrl_cmd(lio, 1);
1855 
1856 	/* start periodical statistics fetch */
1857 	INIT_DELAYED_WORK(&lio->stats_wk.work, lio_fetch_stats);
1858 	lio->stats_wk.ctxptr = lio;
1859 	schedule_delayed_work(&lio->stats_wk.work, msecs_to_jiffies
1860 					(LIQUIDIO_NDEV_STATS_POLL_TIME_MS));
1861 
1862 	dev_info(&oct->pci_dev->dev, "%s interface is opened\n",
1863 		 netdev->name);
1864 
1865 	return 0;
1866 }
1867 
1868 /**
1869  * \brief Net device stop for LiquidIO
1870  * @param netdev network device
1871  */
1872 static int liquidio_stop(struct net_device *netdev)
1873 {
1874 	struct lio *lio = GET_LIO(netdev);
1875 	struct octeon_device *oct = lio->oct_dev;
1876 	struct octeon_device_priv *oct_priv =
1877 		(struct octeon_device_priv *)oct->priv;
1878 	struct napi_struct *napi, *n;
1879 
1880 	ifstate_reset(lio, LIO_IFSTATE_RUNNING);
1881 
1882 	/* Stop any link updates */
1883 	lio->intf_open = 0;
1884 
1885 	stop_txqs(netdev);
1886 
1887 	/* Inform that netif carrier is down */
1888 	netif_carrier_off(netdev);
1889 	netif_tx_disable(netdev);
1890 
1891 	lio->linfo.link.s.link_up = 0;
1892 	lio->link_changes++;
1893 
1894 	/* Tell Octeon that nic interface is down. */
1895 	send_rx_ctrl_cmd(lio, 0);
1896 
1897 	if (OCTEON_CN23XX_PF(oct)) {
1898 		if (!oct->msix_on)
1899 			cleanup_tx_poll_fn(netdev);
1900 	} else {
1901 		cleanup_tx_poll_fn(netdev);
1902 	}
1903 
1904 	cancel_delayed_work_sync(&lio->stats_wk.work);
1905 
1906 	if (lio->ptp_clock) {
1907 		ptp_clock_unregister(lio->ptp_clock);
1908 		lio->ptp_clock = NULL;
1909 	}
1910 
1911 	/* Wait for any pending Rx descriptors */
1912 	if (lio_wait_for_clean_oq(oct))
1913 		netif_info(lio, rx_err, lio->netdev,
1914 			   "Proceeding with stop interface after partial RX desc processing\n");
1915 
1916 	if (oct->props[lio->ifidx].napi_enabled == 1) {
1917 		list_for_each_entry_safe(napi, n, &netdev->napi_list, dev_list)
1918 			napi_disable(napi);
1919 
1920 		oct->props[lio->ifidx].napi_enabled = 0;
1921 
1922 		if (OCTEON_CN23XX_PF(oct))
1923 			oct->droq[0]->ops.poll_mode = 0;
1924 
1925 		tasklet_enable(&oct_priv->droq_tasklet);
1926 	}
1927 
1928 	dev_info(&oct->pci_dev->dev, "%s interface is stopped\n", netdev->name);
1929 
1930 	return 0;
1931 }
1932 
1933 /**
1934  * \brief Converts a mask based on net device flags
1935  * @param netdev network device
1936  *
1937  * This routine generates a octnet_ifflags mask from the net device flags
1938  * received from the OS.
1939  */
1940 static inline enum octnet_ifflags get_new_flags(struct net_device *netdev)
1941 {
1942 	enum octnet_ifflags f = OCTNET_IFFLAG_UNICAST;
1943 
1944 	if (netdev->flags & IFF_PROMISC)
1945 		f |= OCTNET_IFFLAG_PROMISC;
1946 
1947 	if (netdev->flags & IFF_ALLMULTI)
1948 		f |= OCTNET_IFFLAG_ALLMULTI;
1949 
1950 	if (netdev->flags & IFF_MULTICAST) {
1951 		f |= OCTNET_IFFLAG_MULTICAST;
1952 
1953 		/* Accept all multicast addresses if there are more than we
1954 		 * can handle
1955 		 */
1956 		if (netdev_mc_count(netdev) > MAX_OCTEON_MULTICAST_ADDR)
1957 			f |= OCTNET_IFFLAG_ALLMULTI;
1958 	}
1959 
1960 	if (netdev->flags & IFF_BROADCAST)
1961 		f |= OCTNET_IFFLAG_BROADCAST;
1962 
1963 	return f;
1964 }
1965 
1966 /**
1967  * \brief Net device set_multicast_list
1968  * @param netdev network device
1969  */
1970 static void liquidio_set_mcast_list(struct net_device *netdev)
1971 {
1972 	struct lio *lio = GET_LIO(netdev);
1973 	struct octeon_device *oct = lio->oct_dev;
1974 	struct octnic_ctrl_pkt nctrl;
1975 	struct netdev_hw_addr *ha;
1976 	u64 *mc;
1977 	int ret;
1978 	int mc_count = min(netdev_mc_count(netdev), MAX_OCTEON_MULTICAST_ADDR);
1979 
1980 	memset(&nctrl, 0, sizeof(struct octnic_ctrl_pkt));
1981 
1982 	/* Create a ctrl pkt command to be sent to core app. */
1983 	nctrl.ncmd.u64 = 0;
1984 	nctrl.ncmd.s.cmd = OCTNET_CMD_SET_MULTI_LIST;
1985 	nctrl.ncmd.s.param1 = get_new_flags(netdev);
1986 	nctrl.ncmd.s.param2 = mc_count;
1987 	nctrl.ncmd.s.more = mc_count;
1988 	nctrl.iq_no = lio->linfo.txpciq[0].s.q_no;
1989 	nctrl.netpndev = (u64)netdev;
1990 	nctrl.cb_fn = liquidio_link_ctrl_cmd_completion;
1991 
1992 	/* copy all the addresses into the udd */
1993 	mc = &nctrl.udd[0];
1994 	netdev_for_each_mc_addr(ha, netdev) {
1995 		*mc = 0;
1996 		memcpy(((u8 *)mc) + 2, ha->addr, ETH_ALEN);
1997 		/* no need to swap bytes */
1998 
1999 		if (++mc > &nctrl.udd[mc_count])
2000 			break;
2001 	}
2002 
2003 	/* Apparently, any activity in this call from the kernel has to
2004 	 * be atomic. So we won't wait for response.
2005 	 */
2006 
2007 	ret = octnet_send_nic_ctrl_pkt(lio->oct_dev, &nctrl);
2008 	if (ret) {
2009 		dev_err(&oct->pci_dev->dev, "DEVFLAGS change failed in core (ret: 0x%x)\n",
2010 			ret);
2011 	}
2012 }
2013 
2014 /**
2015  * \brief Net device set_mac_address
2016  * @param netdev network device
2017  */
2018 static int liquidio_set_mac(struct net_device *netdev, void *p)
2019 {
2020 	int ret = 0;
2021 	struct lio *lio = GET_LIO(netdev);
2022 	struct octeon_device *oct = lio->oct_dev;
2023 	struct sockaddr *addr = (struct sockaddr *)p;
2024 	struct octnic_ctrl_pkt nctrl;
2025 
2026 	if (!is_valid_ether_addr(addr->sa_data))
2027 		return -EADDRNOTAVAIL;
2028 
2029 	memset(&nctrl, 0, sizeof(struct octnic_ctrl_pkt));
2030 
2031 	nctrl.ncmd.u64 = 0;
2032 	nctrl.ncmd.s.cmd = OCTNET_CMD_CHANGE_MACADDR;
2033 	nctrl.ncmd.s.param1 = 0;
2034 	nctrl.ncmd.s.more = 1;
2035 	nctrl.iq_no = lio->linfo.txpciq[0].s.q_no;
2036 	nctrl.netpndev = (u64)netdev;
2037 
2038 	nctrl.udd[0] = 0;
2039 	/* The MAC Address is presented in network byte order. */
2040 	memcpy((u8 *)&nctrl.udd[0] + 2, addr->sa_data, ETH_ALEN);
2041 
2042 	ret = octnet_send_nic_ctrl_pkt(lio->oct_dev, &nctrl);
2043 	if (ret < 0) {
2044 		dev_err(&oct->pci_dev->dev, "MAC Address change failed\n");
2045 		return -ENOMEM;
2046 	}
2047 
2048 	if (nctrl.sc_status) {
2049 		dev_err(&oct->pci_dev->dev,
2050 			"%s: MAC Address change failed. sc return=%x\n",
2051 			 __func__, nctrl.sc_status);
2052 		return -EIO;
2053 	}
2054 
2055 	memcpy(netdev->dev_addr, addr->sa_data, netdev->addr_len);
2056 	memcpy(((u8 *)&lio->linfo.hw_addr) + 2, addr->sa_data, ETH_ALEN);
2057 
2058 	return 0;
2059 }
2060 
2061 static void
2062 liquidio_get_stats64(struct net_device *netdev,
2063 		     struct rtnl_link_stats64 *lstats)
2064 {
2065 	struct lio *lio = GET_LIO(netdev);
2066 	struct octeon_device *oct;
2067 	u64 pkts = 0, drop = 0, bytes = 0;
2068 	struct oct_droq_stats *oq_stats;
2069 	struct oct_iq_stats *iq_stats;
2070 	int i, iq_no, oq_no;
2071 
2072 	oct = lio->oct_dev;
2073 
2074 	if (ifstate_check(lio, LIO_IFSTATE_RESETTING))
2075 		return;
2076 
2077 	for (i = 0; i < oct->num_iqs; i++) {
2078 		iq_no = lio->linfo.txpciq[i].s.q_no;
2079 		iq_stats = &oct->instr_queue[iq_no]->stats;
2080 		pkts += iq_stats->tx_done;
2081 		drop += iq_stats->tx_dropped;
2082 		bytes += iq_stats->tx_tot_bytes;
2083 	}
2084 
2085 	lstats->tx_packets = pkts;
2086 	lstats->tx_bytes = bytes;
2087 	lstats->tx_dropped = drop;
2088 
2089 	pkts = 0;
2090 	drop = 0;
2091 	bytes = 0;
2092 
2093 	for (i = 0; i < oct->num_oqs; i++) {
2094 		oq_no = lio->linfo.rxpciq[i].s.q_no;
2095 		oq_stats = &oct->droq[oq_no]->stats;
2096 		pkts += oq_stats->rx_pkts_received;
2097 		drop += (oq_stats->rx_dropped +
2098 			 oq_stats->dropped_nodispatch +
2099 			 oq_stats->dropped_toomany +
2100 			 oq_stats->dropped_nomem);
2101 		bytes += oq_stats->rx_bytes_received;
2102 	}
2103 
2104 	lstats->rx_bytes = bytes;
2105 	lstats->rx_packets = pkts;
2106 	lstats->rx_dropped = drop;
2107 
2108 	lstats->multicast = oct->link_stats.fromwire.fw_total_mcast;
2109 	lstats->collisions = oct->link_stats.fromhost.total_collisions;
2110 
2111 	/* detailed rx_errors: */
2112 	lstats->rx_length_errors = oct->link_stats.fromwire.l2_err;
2113 	/* recved pkt with crc error    */
2114 	lstats->rx_crc_errors = oct->link_stats.fromwire.fcs_err;
2115 	/* recv'd frame alignment error */
2116 	lstats->rx_frame_errors = oct->link_stats.fromwire.frame_err;
2117 	/* recv'r fifo overrun */
2118 	lstats->rx_fifo_errors = oct->link_stats.fromwire.fifo_err;
2119 
2120 	lstats->rx_errors = lstats->rx_length_errors + lstats->rx_crc_errors +
2121 		lstats->rx_frame_errors + lstats->rx_fifo_errors;
2122 
2123 	/* detailed tx_errors */
2124 	lstats->tx_aborted_errors = oct->link_stats.fromhost.fw_err_pko;
2125 	lstats->tx_carrier_errors = oct->link_stats.fromhost.fw_err_link;
2126 	lstats->tx_fifo_errors = oct->link_stats.fromhost.fifo_err;
2127 
2128 	lstats->tx_errors = lstats->tx_aborted_errors +
2129 		lstats->tx_carrier_errors +
2130 		lstats->tx_fifo_errors;
2131 }
2132 
2133 /**
2134  * \brief Handler for SIOCSHWTSTAMP ioctl
2135  * @param netdev network device
2136  * @param ifr interface request
2137  * @param cmd command
2138  */
2139 static int hwtstamp_ioctl(struct net_device *netdev, struct ifreq *ifr)
2140 {
2141 	struct hwtstamp_config conf;
2142 	struct lio *lio = GET_LIO(netdev);
2143 
2144 	if (copy_from_user(&conf, ifr->ifr_data, sizeof(conf)))
2145 		return -EFAULT;
2146 
2147 	if (conf.flags)
2148 		return -EINVAL;
2149 
2150 	switch (conf.tx_type) {
2151 	case HWTSTAMP_TX_ON:
2152 	case HWTSTAMP_TX_OFF:
2153 		break;
2154 	default:
2155 		return -ERANGE;
2156 	}
2157 
2158 	switch (conf.rx_filter) {
2159 	case HWTSTAMP_FILTER_NONE:
2160 		break;
2161 	case HWTSTAMP_FILTER_ALL:
2162 	case HWTSTAMP_FILTER_SOME:
2163 	case HWTSTAMP_FILTER_PTP_V1_L4_EVENT:
2164 	case HWTSTAMP_FILTER_PTP_V1_L4_SYNC:
2165 	case HWTSTAMP_FILTER_PTP_V1_L4_DELAY_REQ:
2166 	case HWTSTAMP_FILTER_PTP_V2_L4_EVENT:
2167 	case HWTSTAMP_FILTER_PTP_V2_L4_SYNC:
2168 	case HWTSTAMP_FILTER_PTP_V2_L4_DELAY_REQ:
2169 	case HWTSTAMP_FILTER_PTP_V2_L2_EVENT:
2170 	case HWTSTAMP_FILTER_PTP_V2_L2_SYNC:
2171 	case HWTSTAMP_FILTER_PTP_V2_L2_DELAY_REQ:
2172 	case HWTSTAMP_FILTER_PTP_V2_EVENT:
2173 	case HWTSTAMP_FILTER_PTP_V2_SYNC:
2174 	case HWTSTAMP_FILTER_PTP_V2_DELAY_REQ:
2175 	case HWTSTAMP_FILTER_NTP_ALL:
2176 		conf.rx_filter = HWTSTAMP_FILTER_ALL;
2177 		break;
2178 	default:
2179 		return -ERANGE;
2180 	}
2181 
2182 	if (conf.rx_filter == HWTSTAMP_FILTER_ALL)
2183 		ifstate_set(lio, LIO_IFSTATE_RX_TIMESTAMP_ENABLED);
2184 
2185 	else
2186 		ifstate_reset(lio, LIO_IFSTATE_RX_TIMESTAMP_ENABLED);
2187 
2188 	return copy_to_user(ifr->ifr_data, &conf, sizeof(conf)) ? -EFAULT : 0;
2189 }
2190 
2191 /**
2192  * \brief ioctl handler
2193  * @param netdev network device
2194  * @param ifr interface request
2195  * @param cmd command
2196  */
2197 static int liquidio_ioctl(struct net_device *netdev, struct ifreq *ifr, int cmd)
2198 {
2199 	struct lio *lio = GET_LIO(netdev);
2200 
2201 	switch (cmd) {
2202 	case SIOCSHWTSTAMP:
2203 		if (lio->oct_dev->ptp_enable)
2204 			return hwtstamp_ioctl(netdev, ifr);
2205 		/* fall through */
2206 	default:
2207 		return -EOPNOTSUPP;
2208 	}
2209 }
2210 
2211 /**
2212  * \brief handle a Tx timestamp response
2213  * @param status response status
2214  * @param buf pointer to skb
2215  */
2216 static void handle_timestamp(struct octeon_device *oct,
2217 			     u32 status,
2218 			     void *buf)
2219 {
2220 	struct octnet_buf_free_info *finfo;
2221 	struct octeon_soft_command *sc;
2222 	struct oct_timestamp_resp *resp;
2223 	struct lio *lio;
2224 	struct sk_buff *skb = (struct sk_buff *)buf;
2225 
2226 	finfo = (struct octnet_buf_free_info *)skb->cb;
2227 	lio = finfo->lio;
2228 	sc = finfo->sc;
2229 	oct = lio->oct_dev;
2230 	resp = (struct oct_timestamp_resp *)sc->virtrptr;
2231 
2232 	if (status != OCTEON_REQUEST_DONE) {
2233 		dev_err(&oct->pci_dev->dev, "Tx timestamp instruction failed. Status: %llx\n",
2234 			CVM_CAST64(status));
2235 		resp->timestamp = 0;
2236 	}
2237 
2238 	octeon_swap_8B_data(&resp->timestamp, 1);
2239 
2240 	if (unlikely((skb_shinfo(skb)->tx_flags & SKBTX_IN_PROGRESS) != 0)) {
2241 		struct skb_shared_hwtstamps ts;
2242 		u64 ns = resp->timestamp;
2243 
2244 		netif_info(lio, tx_done, lio->netdev,
2245 			   "Got resulting SKBTX_HW_TSTAMP skb=%p ns=%016llu\n",
2246 			   skb, (unsigned long long)ns);
2247 		ts.hwtstamp = ns_to_ktime(ns + lio->ptp_adjust);
2248 		skb_tstamp_tx(skb, &ts);
2249 	}
2250 
2251 	octeon_free_soft_command(oct, sc);
2252 	tx_buffer_free(skb);
2253 }
2254 
2255 /* \brief Send a data packet that will be timestamped
2256  * @param oct octeon device
2257  * @param ndata pointer to network data
2258  * @param finfo pointer to private network data
2259  */
2260 static inline int send_nic_timestamp_pkt(struct octeon_device *oct,
2261 					 struct octnic_data_pkt *ndata,
2262 					 struct octnet_buf_free_info *finfo,
2263 					 int xmit_more)
2264 {
2265 	int retval;
2266 	struct octeon_soft_command *sc;
2267 	struct lio *lio;
2268 	int ring_doorbell;
2269 	u32 len;
2270 
2271 	lio = finfo->lio;
2272 
2273 	sc = octeon_alloc_soft_command_resp(oct, &ndata->cmd,
2274 					    sizeof(struct oct_timestamp_resp));
2275 	finfo->sc = sc;
2276 
2277 	if (!sc) {
2278 		dev_err(&oct->pci_dev->dev, "No memory for timestamped data packet\n");
2279 		return IQ_SEND_FAILED;
2280 	}
2281 
2282 	if (ndata->reqtype == REQTYPE_NORESP_NET)
2283 		ndata->reqtype = REQTYPE_RESP_NET;
2284 	else if (ndata->reqtype == REQTYPE_NORESP_NET_SG)
2285 		ndata->reqtype = REQTYPE_RESP_NET_SG;
2286 
2287 	sc->callback = handle_timestamp;
2288 	sc->callback_arg = finfo->skb;
2289 	sc->iq_no = ndata->q_no;
2290 
2291 	if (OCTEON_CN23XX_PF(oct))
2292 		len = (u32)((struct octeon_instr_ih3 *)
2293 			    (&sc->cmd.cmd3.ih3))->dlengsz;
2294 	else
2295 		len = (u32)((struct octeon_instr_ih2 *)
2296 			    (&sc->cmd.cmd2.ih2))->dlengsz;
2297 
2298 	ring_doorbell = !xmit_more;
2299 
2300 	retval = octeon_send_command(oct, sc->iq_no, ring_doorbell, &sc->cmd,
2301 				     sc, len, ndata->reqtype);
2302 
2303 	if (retval == IQ_SEND_FAILED) {
2304 		dev_err(&oct->pci_dev->dev, "timestamp data packet failed status: %x\n",
2305 			retval);
2306 		octeon_free_soft_command(oct, sc);
2307 	} else {
2308 		netif_info(lio, tx_queued, lio->netdev, "Queued timestamp packet\n");
2309 	}
2310 
2311 	return retval;
2312 }
2313 
2314 /** \brief Transmit networks packets to the Octeon interface
2315  * @param skbuff   skbuff struct to be passed to network layer.
2316  * @param netdev    pointer to network device
2317  * @returns whether the packet was transmitted to the device okay or not
2318  *             (NETDEV_TX_OK or NETDEV_TX_BUSY)
2319  */
2320 static netdev_tx_t liquidio_xmit(struct sk_buff *skb, struct net_device *netdev)
2321 {
2322 	struct lio *lio;
2323 	struct octnet_buf_free_info *finfo;
2324 	union octnic_cmd_setup cmdsetup;
2325 	struct octnic_data_pkt ndata;
2326 	struct octeon_device *oct;
2327 	struct oct_iq_stats *stats;
2328 	struct octeon_instr_irh *irh;
2329 	union tx_info *tx_info;
2330 	int status = 0;
2331 	int q_idx = 0, iq_no = 0;
2332 	int j, xmit_more = 0;
2333 	u64 dptr = 0;
2334 	u32 tag = 0;
2335 
2336 	lio = GET_LIO(netdev);
2337 	oct = lio->oct_dev;
2338 
2339 	q_idx = skb_iq(oct, skb);
2340 	tag = q_idx;
2341 	iq_no = lio->linfo.txpciq[q_idx].s.q_no;
2342 
2343 	stats = &oct->instr_queue[iq_no]->stats;
2344 
2345 	/* Check for all conditions in which the current packet cannot be
2346 	 * transmitted.
2347 	 */
2348 	if (!(atomic_read(&lio->ifstate) & LIO_IFSTATE_RUNNING) ||
2349 	    (!lio->linfo.link.s.link_up) ||
2350 	    (skb->len <= 0)) {
2351 		netif_info(lio, tx_err, lio->netdev,
2352 			   "Transmit failed link_status : %d\n",
2353 			   lio->linfo.link.s.link_up);
2354 		goto lio_xmit_failed;
2355 	}
2356 
2357 	/* Use space in skb->cb to store info used to unmap and
2358 	 * free the buffers.
2359 	 */
2360 	finfo = (struct octnet_buf_free_info *)skb->cb;
2361 	finfo->lio = lio;
2362 	finfo->skb = skb;
2363 	finfo->sc = NULL;
2364 
2365 	/* Prepare the attributes for the data to be passed to OSI. */
2366 	memset(&ndata, 0, sizeof(struct octnic_data_pkt));
2367 
2368 	ndata.buf = (void *)finfo;
2369 
2370 	ndata.q_no = iq_no;
2371 
2372 	if (octnet_iq_is_full(oct, ndata.q_no)) {
2373 		/* defer sending if queue is full */
2374 		netif_info(lio, tx_err, lio->netdev, "Transmit failed iq:%d full\n",
2375 			   ndata.q_no);
2376 		stats->tx_iq_busy++;
2377 		return NETDEV_TX_BUSY;
2378 	}
2379 
2380 	/* pr_info(" XMIT - valid Qs: %d, 1st Q no: %d, cpu:  %d, q_no:%d\n",
2381 	 *	lio->linfo.num_txpciq, lio->txq, cpu, ndata.q_no);
2382 	 */
2383 
2384 	ndata.datasize = skb->len;
2385 
2386 	cmdsetup.u64 = 0;
2387 	cmdsetup.s.iq_no = iq_no;
2388 
2389 	if (skb->ip_summed == CHECKSUM_PARTIAL) {
2390 		if (skb->encapsulation) {
2391 			cmdsetup.s.tnl_csum = 1;
2392 			stats->tx_vxlan++;
2393 		} else {
2394 			cmdsetup.s.transport_csum = 1;
2395 		}
2396 	}
2397 	if (unlikely(skb_shinfo(skb)->tx_flags & SKBTX_HW_TSTAMP)) {
2398 		skb_shinfo(skb)->tx_flags |= SKBTX_IN_PROGRESS;
2399 		cmdsetup.s.timestamp = 1;
2400 	}
2401 
2402 	if (skb_shinfo(skb)->nr_frags == 0) {
2403 		cmdsetup.s.u.datasize = skb->len;
2404 		octnet_prepare_pci_cmd(oct, &ndata.cmd, &cmdsetup, tag);
2405 
2406 		/* Offload checksum calculation for TCP/UDP packets */
2407 		dptr = dma_map_single(&oct->pci_dev->dev,
2408 				      skb->data,
2409 				      skb->len,
2410 				      DMA_TO_DEVICE);
2411 		if (dma_mapping_error(&oct->pci_dev->dev, dptr)) {
2412 			dev_err(&oct->pci_dev->dev, "%s DMA mapping error 1\n",
2413 				__func__);
2414 			stats->tx_dmamap_fail++;
2415 			return NETDEV_TX_BUSY;
2416 		}
2417 
2418 		if (OCTEON_CN23XX_PF(oct))
2419 			ndata.cmd.cmd3.dptr = dptr;
2420 		else
2421 			ndata.cmd.cmd2.dptr = dptr;
2422 		finfo->dptr = dptr;
2423 		ndata.reqtype = REQTYPE_NORESP_NET;
2424 
2425 	} else {
2426 		int i, frags;
2427 		skb_frag_t *frag;
2428 		struct octnic_gather *g;
2429 
2430 		spin_lock(&lio->glist_lock[q_idx]);
2431 		g = (struct octnic_gather *)
2432 			lio_list_delete_head(&lio->glist[q_idx]);
2433 		spin_unlock(&lio->glist_lock[q_idx]);
2434 
2435 		if (!g) {
2436 			netif_info(lio, tx_err, lio->netdev,
2437 				   "Transmit scatter gather: glist null!\n");
2438 			goto lio_xmit_failed;
2439 		}
2440 
2441 		cmdsetup.s.gather = 1;
2442 		cmdsetup.s.u.gatherptrs = (skb_shinfo(skb)->nr_frags + 1);
2443 		octnet_prepare_pci_cmd(oct, &ndata.cmd, &cmdsetup, tag);
2444 
2445 		memset(g->sg, 0, g->sg_size);
2446 
2447 		g->sg[0].ptr[0] = dma_map_single(&oct->pci_dev->dev,
2448 						 skb->data,
2449 						 (skb->len - skb->data_len),
2450 						 DMA_TO_DEVICE);
2451 		if (dma_mapping_error(&oct->pci_dev->dev, g->sg[0].ptr[0])) {
2452 			dev_err(&oct->pci_dev->dev, "%s DMA mapping error 2\n",
2453 				__func__);
2454 			stats->tx_dmamap_fail++;
2455 			return NETDEV_TX_BUSY;
2456 		}
2457 		add_sg_size(&g->sg[0], (skb->len - skb->data_len), 0);
2458 
2459 		frags = skb_shinfo(skb)->nr_frags;
2460 		i = 1;
2461 		while (frags--) {
2462 			frag = &skb_shinfo(skb)->frags[i - 1];
2463 
2464 			g->sg[(i >> 2)].ptr[(i & 3)] =
2465 				skb_frag_dma_map(&oct->pci_dev->dev,
2466 					         frag, 0, skb_frag_size(frag),
2467 						 DMA_TO_DEVICE);
2468 
2469 			if (dma_mapping_error(&oct->pci_dev->dev,
2470 					      g->sg[i >> 2].ptr[i & 3])) {
2471 				dma_unmap_single(&oct->pci_dev->dev,
2472 						 g->sg[0].ptr[0],
2473 						 skb->len - skb->data_len,
2474 						 DMA_TO_DEVICE);
2475 				for (j = 1; j < i; j++) {
2476 					frag = &skb_shinfo(skb)->frags[j - 1];
2477 					dma_unmap_page(&oct->pci_dev->dev,
2478 						       g->sg[j >> 2].ptr[j & 3],
2479 						       skb_frag_size(frag),
2480 						       DMA_TO_DEVICE);
2481 				}
2482 				dev_err(&oct->pci_dev->dev, "%s DMA mapping error 3\n",
2483 					__func__);
2484 				return NETDEV_TX_BUSY;
2485 			}
2486 
2487 			add_sg_size(&g->sg[(i >> 2)], skb_frag_size(frag),
2488 				    (i & 3));
2489 			i++;
2490 		}
2491 
2492 		dptr = g->sg_dma_ptr;
2493 
2494 		if (OCTEON_CN23XX_PF(oct))
2495 			ndata.cmd.cmd3.dptr = dptr;
2496 		else
2497 			ndata.cmd.cmd2.dptr = dptr;
2498 		finfo->dptr = dptr;
2499 		finfo->g = g;
2500 
2501 		ndata.reqtype = REQTYPE_NORESP_NET_SG;
2502 	}
2503 
2504 	if (OCTEON_CN23XX_PF(oct)) {
2505 		irh = (struct octeon_instr_irh *)&ndata.cmd.cmd3.irh;
2506 		tx_info = (union tx_info *)&ndata.cmd.cmd3.ossp[0];
2507 	} else {
2508 		irh = (struct octeon_instr_irh *)&ndata.cmd.cmd2.irh;
2509 		tx_info = (union tx_info *)&ndata.cmd.cmd2.ossp[0];
2510 	}
2511 
2512 	if (skb_shinfo(skb)->gso_size) {
2513 		tx_info->s.gso_size = skb_shinfo(skb)->gso_size;
2514 		tx_info->s.gso_segs = skb_shinfo(skb)->gso_segs;
2515 		stats->tx_gso++;
2516 	}
2517 
2518 	/* HW insert VLAN tag */
2519 	if (skb_vlan_tag_present(skb)) {
2520 		irh->priority = skb_vlan_tag_get(skb) >> 13;
2521 		irh->vlan = skb_vlan_tag_get(skb) & 0xfff;
2522 	}
2523 
2524 	xmit_more = netdev_xmit_more();
2525 
2526 	if (unlikely(cmdsetup.s.timestamp))
2527 		status = send_nic_timestamp_pkt(oct, &ndata, finfo, xmit_more);
2528 	else
2529 		status = octnet_send_nic_data_pkt(oct, &ndata, xmit_more);
2530 	if (status == IQ_SEND_FAILED)
2531 		goto lio_xmit_failed;
2532 
2533 	netif_info(lio, tx_queued, lio->netdev, "Transmit queued successfully\n");
2534 
2535 	if (status == IQ_SEND_STOP)
2536 		netif_stop_subqueue(netdev, q_idx);
2537 
2538 	netif_trans_update(netdev);
2539 
2540 	if (tx_info->s.gso_segs)
2541 		stats->tx_done += tx_info->s.gso_segs;
2542 	else
2543 		stats->tx_done++;
2544 	stats->tx_tot_bytes += ndata.datasize;
2545 
2546 	return NETDEV_TX_OK;
2547 
2548 lio_xmit_failed:
2549 	stats->tx_dropped++;
2550 	netif_info(lio, tx_err, lio->netdev, "IQ%d Transmit dropped:%llu\n",
2551 		   iq_no, stats->tx_dropped);
2552 	if (dptr)
2553 		dma_unmap_single(&oct->pci_dev->dev, dptr,
2554 				 ndata.datasize, DMA_TO_DEVICE);
2555 
2556 	octeon_ring_doorbell_locked(oct, iq_no);
2557 
2558 	tx_buffer_free(skb);
2559 	return NETDEV_TX_OK;
2560 }
2561 
2562 /** \brief Network device Tx timeout
2563  * @param netdev    pointer to network device
2564  */
2565 static void liquidio_tx_timeout(struct net_device *netdev)
2566 {
2567 	struct lio *lio;
2568 
2569 	lio = GET_LIO(netdev);
2570 
2571 	netif_info(lio, tx_err, lio->netdev,
2572 		   "Transmit timeout tx_dropped:%ld, waking up queues now!!\n",
2573 		   netdev->stats.tx_dropped);
2574 	netif_trans_update(netdev);
2575 	wake_txqs(netdev);
2576 }
2577 
2578 static int liquidio_vlan_rx_add_vid(struct net_device *netdev,
2579 				    __be16 proto __attribute__((unused)),
2580 				    u16 vid)
2581 {
2582 	struct lio *lio = GET_LIO(netdev);
2583 	struct octeon_device *oct = lio->oct_dev;
2584 	struct octnic_ctrl_pkt nctrl;
2585 	int ret = 0;
2586 
2587 	memset(&nctrl, 0, sizeof(struct octnic_ctrl_pkt));
2588 
2589 	nctrl.ncmd.u64 = 0;
2590 	nctrl.ncmd.s.cmd = OCTNET_CMD_ADD_VLAN_FILTER;
2591 	nctrl.ncmd.s.param1 = vid;
2592 	nctrl.iq_no = lio->linfo.txpciq[0].s.q_no;
2593 	nctrl.netpndev = (u64)netdev;
2594 	nctrl.cb_fn = liquidio_link_ctrl_cmd_completion;
2595 
2596 	ret = octnet_send_nic_ctrl_pkt(lio->oct_dev, &nctrl);
2597 	if (ret) {
2598 		dev_err(&oct->pci_dev->dev, "Add VLAN filter failed in core (ret: 0x%x)\n",
2599 			ret);
2600 		if (ret > 0)
2601 			ret = -EIO;
2602 	}
2603 
2604 	return ret;
2605 }
2606 
2607 static int liquidio_vlan_rx_kill_vid(struct net_device *netdev,
2608 				     __be16 proto __attribute__((unused)),
2609 				     u16 vid)
2610 {
2611 	struct lio *lio = GET_LIO(netdev);
2612 	struct octeon_device *oct = lio->oct_dev;
2613 	struct octnic_ctrl_pkt nctrl;
2614 	int ret = 0;
2615 
2616 	memset(&nctrl, 0, sizeof(struct octnic_ctrl_pkt));
2617 
2618 	nctrl.ncmd.u64 = 0;
2619 	nctrl.ncmd.s.cmd = OCTNET_CMD_DEL_VLAN_FILTER;
2620 	nctrl.ncmd.s.param1 = vid;
2621 	nctrl.iq_no = lio->linfo.txpciq[0].s.q_no;
2622 	nctrl.netpndev = (u64)netdev;
2623 	nctrl.cb_fn = liquidio_link_ctrl_cmd_completion;
2624 
2625 	ret = octnet_send_nic_ctrl_pkt(lio->oct_dev, &nctrl);
2626 	if (ret) {
2627 		dev_err(&oct->pci_dev->dev, "Del VLAN filter failed in core (ret: 0x%x)\n",
2628 			ret);
2629 		if (ret > 0)
2630 			ret = -EIO;
2631 	}
2632 	return ret;
2633 }
2634 
2635 /** Sending command to enable/disable RX checksum offload
2636  * @param netdev                pointer to network device
2637  * @param command               OCTNET_CMD_TNL_RX_CSUM_CTL
2638  * @param rx_cmd_bit            OCTNET_CMD_RXCSUM_ENABLE/
2639  *                              OCTNET_CMD_RXCSUM_DISABLE
2640  * @returns                     SUCCESS or FAILURE
2641  */
2642 static int liquidio_set_rxcsum_command(struct net_device *netdev, int command,
2643 				       u8 rx_cmd)
2644 {
2645 	struct lio *lio = GET_LIO(netdev);
2646 	struct octeon_device *oct = lio->oct_dev;
2647 	struct octnic_ctrl_pkt nctrl;
2648 	int ret = 0;
2649 
2650 	memset(&nctrl, 0, sizeof(struct octnic_ctrl_pkt));
2651 
2652 	nctrl.ncmd.u64 = 0;
2653 	nctrl.ncmd.s.cmd = command;
2654 	nctrl.ncmd.s.param1 = rx_cmd;
2655 	nctrl.iq_no = lio->linfo.txpciq[0].s.q_no;
2656 	nctrl.netpndev = (u64)netdev;
2657 	nctrl.cb_fn = liquidio_link_ctrl_cmd_completion;
2658 
2659 	ret = octnet_send_nic_ctrl_pkt(lio->oct_dev, &nctrl);
2660 	if (ret) {
2661 		dev_err(&oct->pci_dev->dev,
2662 			"DEVFLAGS RXCSUM change failed in core(ret:0x%x)\n",
2663 			ret);
2664 		if (ret > 0)
2665 			ret = -EIO;
2666 	}
2667 	return ret;
2668 }
2669 
2670 /** Sending command to add/delete VxLAN UDP port to firmware
2671  * @param netdev                pointer to network device
2672  * @param command               OCTNET_CMD_VXLAN_PORT_CONFIG
2673  * @param vxlan_port            VxLAN port to be added or deleted
2674  * @param vxlan_cmd_bit         OCTNET_CMD_VXLAN_PORT_ADD,
2675  *                              OCTNET_CMD_VXLAN_PORT_DEL
2676  * @returns                     SUCCESS or FAILURE
2677  */
2678 static int liquidio_vxlan_port_command(struct net_device *netdev, int command,
2679 				       u16 vxlan_port, u8 vxlan_cmd_bit)
2680 {
2681 	struct lio *lio = GET_LIO(netdev);
2682 	struct octeon_device *oct = lio->oct_dev;
2683 	struct octnic_ctrl_pkt nctrl;
2684 	int ret = 0;
2685 
2686 	memset(&nctrl, 0, sizeof(struct octnic_ctrl_pkt));
2687 
2688 	nctrl.ncmd.u64 = 0;
2689 	nctrl.ncmd.s.cmd = command;
2690 	nctrl.ncmd.s.more = vxlan_cmd_bit;
2691 	nctrl.ncmd.s.param1 = vxlan_port;
2692 	nctrl.iq_no = lio->linfo.txpciq[0].s.q_no;
2693 	nctrl.netpndev = (u64)netdev;
2694 	nctrl.cb_fn = liquidio_link_ctrl_cmd_completion;
2695 
2696 	ret = octnet_send_nic_ctrl_pkt(lio->oct_dev, &nctrl);
2697 	if (ret) {
2698 		dev_err(&oct->pci_dev->dev,
2699 			"VxLAN port add/delete failed in core (ret:0x%x)\n",
2700 			ret);
2701 		if (ret > 0)
2702 			ret = -EIO;
2703 	}
2704 	return ret;
2705 }
2706 
2707 /** \brief Net device fix features
2708  * @param netdev  pointer to network device
2709  * @param request features requested
2710  * @returns updated features list
2711  */
2712 static netdev_features_t liquidio_fix_features(struct net_device *netdev,
2713 					       netdev_features_t request)
2714 {
2715 	struct lio *lio = netdev_priv(netdev);
2716 
2717 	if ((request & NETIF_F_RXCSUM) &&
2718 	    !(lio->dev_capability & NETIF_F_RXCSUM))
2719 		request &= ~NETIF_F_RXCSUM;
2720 
2721 	if ((request & NETIF_F_HW_CSUM) &&
2722 	    !(lio->dev_capability & NETIF_F_HW_CSUM))
2723 		request &= ~NETIF_F_HW_CSUM;
2724 
2725 	if ((request & NETIF_F_TSO) && !(lio->dev_capability & NETIF_F_TSO))
2726 		request &= ~NETIF_F_TSO;
2727 
2728 	if ((request & NETIF_F_TSO6) && !(lio->dev_capability & NETIF_F_TSO6))
2729 		request &= ~NETIF_F_TSO6;
2730 
2731 	if ((request & NETIF_F_LRO) && !(lio->dev_capability & NETIF_F_LRO))
2732 		request &= ~NETIF_F_LRO;
2733 
2734 	/*Disable LRO if RXCSUM is off */
2735 	if (!(request & NETIF_F_RXCSUM) && (netdev->features & NETIF_F_LRO) &&
2736 	    (lio->dev_capability & NETIF_F_LRO))
2737 		request &= ~NETIF_F_LRO;
2738 
2739 	if ((request & NETIF_F_HW_VLAN_CTAG_FILTER) &&
2740 	    !(lio->dev_capability & NETIF_F_HW_VLAN_CTAG_FILTER))
2741 		request &= ~NETIF_F_HW_VLAN_CTAG_FILTER;
2742 
2743 	return request;
2744 }
2745 
2746 /** \brief Net device set features
2747  * @param netdev  pointer to network device
2748  * @param features features to enable/disable
2749  */
2750 static int liquidio_set_features(struct net_device *netdev,
2751 				 netdev_features_t features)
2752 {
2753 	struct lio *lio = netdev_priv(netdev);
2754 
2755 	if ((features & NETIF_F_LRO) &&
2756 	    (lio->dev_capability & NETIF_F_LRO) &&
2757 	    !(netdev->features & NETIF_F_LRO))
2758 		liquidio_set_feature(netdev, OCTNET_CMD_LRO_ENABLE,
2759 				     OCTNIC_LROIPV4 | OCTNIC_LROIPV6);
2760 	else if (!(features & NETIF_F_LRO) &&
2761 		 (lio->dev_capability & NETIF_F_LRO) &&
2762 		 (netdev->features & NETIF_F_LRO))
2763 		liquidio_set_feature(netdev, OCTNET_CMD_LRO_DISABLE,
2764 				     OCTNIC_LROIPV4 | OCTNIC_LROIPV6);
2765 
2766 	/* Sending command to firmware to enable/disable RX checksum
2767 	 * offload settings using ethtool
2768 	 */
2769 	if (!(netdev->features & NETIF_F_RXCSUM) &&
2770 	    (lio->enc_dev_capability & NETIF_F_RXCSUM) &&
2771 	    (features & NETIF_F_RXCSUM))
2772 		liquidio_set_rxcsum_command(netdev,
2773 					    OCTNET_CMD_TNL_RX_CSUM_CTL,
2774 					    OCTNET_CMD_RXCSUM_ENABLE);
2775 	else if ((netdev->features & NETIF_F_RXCSUM) &&
2776 		 (lio->enc_dev_capability & NETIF_F_RXCSUM) &&
2777 		 !(features & NETIF_F_RXCSUM))
2778 		liquidio_set_rxcsum_command(netdev, OCTNET_CMD_TNL_RX_CSUM_CTL,
2779 					    OCTNET_CMD_RXCSUM_DISABLE);
2780 
2781 	if ((features & NETIF_F_HW_VLAN_CTAG_FILTER) &&
2782 	    (lio->dev_capability & NETIF_F_HW_VLAN_CTAG_FILTER) &&
2783 	    !(netdev->features & NETIF_F_HW_VLAN_CTAG_FILTER))
2784 		liquidio_set_feature(netdev, OCTNET_CMD_VLAN_FILTER_CTL,
2785 				     OCTNET_CMD_VLAN_FILTER_ENABLE);
2786 	else if (!(features & NETIF_F_HW_VLAN_CTAG_FILTER) &&
2787 		 (lio->dev_capability & NETIF_F_HW_VLAN_CTAG_FILTER) &&
2788 		 (netdev->features & NETIF_F_HW_VLAN_CTAG_FILTER))
2789 		liquidio_set_feature(netdev, OCTNET_CMD_VLAN_FILTER_CTL,
2790 				     OCTNET_CMD_VLAN_FILTER_DISABLE);
2791 
2792 	return 0;
2793 }
2794 
2795 static void liquidio_add_vxlan_port(struct net_device *netdev,
2796 				    struct udp_tunnel_info *ti)
2797 {
2798 	if (ti->type != UDP_TUNNEL_TYPE_VXLAN)
2799 		return;
2800 
2801 	liquidio_vxlan_port_command(netdev,
2802 				    OCTNET_CMD_VXLAN_PORT_CONFIG,
2803 				    htons(ti->port),
2804 				    OCTNET_CMD_VXLAN_PORT_ADD);
2805 }
2806 
2807 static void liquidio_del_vxlan_port(struct net_device *netdev,
2808 				    struct udp_tunnel_info *ti)
2809 {
2810 	if (ti->type != UDP_TUNNEL_TYPE_VXLAN)
2811 		return;
2812 
2813 	liquidio_vxlan_port_command(netdev,
2814 				    OCTNET_CMD_VXLAN_PORT_CONFIG,
2815 				    htons(ti->port),
2816 				    OCTNET_CMD_VXLAN_PORT_DEL);
2817 }
2818 
2819 static int __liquidio_set_vf_mac(struct net_device *netdev, int vfidx,
2820 				 u8 *mac, bool is_admin_assigned)
2821 {
2822 	struct lio *lio = GET_LIO(netdev);
2823 	struct octeon_device *oct = lio->oct_dev;
2824 	struct octnic_ctrl_pkt nctrl;
2825 	int ret = 0;
2826 
2827 	if (!is_valid_ether_addr(mac))
2828 		return -EINVAL;
2829 
2830 	if (vfidx < 0 || vfidx >= oct->sriov_info.max_vfs)
2831 		return -EINVAL;
2832 
2833 	memset(&nctrl, 0, sizeof(struct octnic_ctrl_pkt));
2834 
2835 	nctrl.ncmd.u64 = 0;
2836 	nctrl.ncmd.s.cmd = OCTNET_CMD_CHANGE_MACADDR;
2837 	/* vfidx is 0 based, but vf_num (param1) is 1 based */
2838 	nctrl.ncmd.s.param1 = vfidx + 1;
2839 	nctrl.ncmd.s.more = 1;
2840 	nctrl.iq_no = lio->linfo.txpciq[0].s.q_no;
2841 	nctrl.netpndev = (u64)netdev;
2842 	if (is_admin_assigned) {
2843 		nctrl.ncmd.s.param2 = true;
2844 		nctrl.cb_fn = liquidio_link_ctrl_cmd_completion;
2845 	}
2846 
2847 	nctrl.udd[0] = 0;
2848 	/* The MAC Address is presented in network byte order. */
2849 	ether_addr_copy((u8 *)&nctrl.udd[0] + 2, mac);
2850 
2851 	oct->sriov_info.vf_macaddr[vfidx] = nctrl.udd[0];
2852 
2853 	ret = octnet_send_nic_ctrl_pkt(oct, &nctrl);
2854 	if (ret > 0)
2855 		ret = -EIO;
2856 
2857 	return ret;
2858 }
2859 
2860 static int liquidio_set_vf_mac(struct net_device *netdev, int vfidx, u8 *mac)
2861 {
2862 	struct lio *lio = GET_LIO(netdev);
2863 	struct octeon_device *oct = lio->oct_dev;
2864 	int retval;
2865 
2866 	if (vfidx < 0 || vfidx >= oct->sriov_info.num_vfs_alloced)
2867 		return -EINVAL;
2868 
2869 	retval = __liquidio_set_vf_mac(netdev, vfidx, mac, true);
2870 	if (!retval)
2871 		cn23xx_tell_vf_its_macaddr_changed(oct, vfidx, mac);
2872 
2873 	return retval;
2874 }
2875 
2876 static int liquidio_set_vf_spoofchk(struct net_device *netdev, int vfidx,
2877 				    bool enable)
2878 {
2879 	struct lio *lio = GET_LIO(netdev);
2880 	struct octeon_device *oct = lio->oct_dev;
2881 	struct octnic_ctrl_pkt nctrl;
2882 	int retval;
2883 
2884 	if (!(oct->fw_info.app_cap_flags & LIQUIDIO_SPOOFCHK_CAP)) {
2885 		netif_info(lio, drv, lio->netdev,
2886 			   "firmware does not support spoofchk\n");
2887 		return -EOPNOTSUPP;
2888 	}
2889 
2890 	if (vfidx < 0 || vfidx >= oct->sriov_info.num_vfs_alloced) {
2891 		netif_info(lio, drv, lio->netdev, "Invalid vfidx %d\n", vfidx);
2892 		return -EINVAL;
2893 	}
2894 
2895 	if (enable) {
2896 		if (oct->sriov_info.vf_spoofchk[vfidx])
2897 			return 0;
2898 	} else {
2899 		/* Clear */
2900 		if (!oct->sriov_info.vf_spoofchk[vfidx])
2901 			return 0;
2902 	}
2903 
2904 	memset(&nctrl, 0, sizeof(struct octnic_ctrl_pkt));
2905 	nctrl.ncmd.s.cmdgroup = OCTNET_CMD_GROUP1;
2906 	nctrl.ncmd.s.cmd = OCTNET_CMD_SET_VF_SPOOFCHK;
2907 	nctrl.ncmd.s.param1 =
2908 		vfidx + 1; /* vfidx is 0 based,
2909 			    * but vf_num (param1) is 1 based
2910 			    */
2911 	nctrl.ncmd.s.param2 = enable;
2912 	nctrl.ncmd.s.more = 0;
2913 	nctrl.iq_no = lio->linfo.txpciq[0].s.q_no;
2914 	nctrl.cb_fn = NULL;
2915 
2916 	retval = octnet_send_nic_ctrl_pkt(oct, &nctrl);
2917 
2918 	if (retval) {
2919 		netif_info(lio, drv, lio->netdev,
2920 			   "Failed to set VF %d spoofchk %s\n", vfidx,
2921 			enable ? "on" : "off");
2922 		return -1;
2923 	}
2924 
2925 	oct->sriov_info.vf_spoofchk[vfidx] = enable;
2926 	netif_info(lio, drv, lio->netdev, "VF %u spoofchk is %s\n", vfidx,
2927 		   enable ? "on" : "off");
2928 
2929 	return 0;
2930 }
2931 
2932 static int liquidio_set_vf_vlan(struct net_device *netdev, int vfidx,
2933 				u16 vlan, u8 qos, __be16 vlan_proto)
2934 {
2935 	struct lio *lio = GET_LIO(netdev);
2936 	struct octeon_device *oct = lio->oct_dev;
2937 	struct octnic_ctrl_pkt nctrl;
2938 	u16 vlantci;
2939 	int ret = 0;
2940 
2941 	if (vfidx < 0 || vfidx >= oct->sriov_info.num_vfs_alloced)
2942 		return -EINVAL;
2943 
2944 	if (vlan_proto != htons(ETH_P_8021Q))
2945 		return -EPROTONOSUPPORT;
2946 
2947 	if (vlan >= VLAN_N_VID || qos > 7)
2948 		return -EINVAL;
2949 
2950 	if (vlan)
2951 		vlantci = vlan | (u16)qos << VLAN_PRIO_SHIFT;
2952 	else
2953 		vlantci = 0;
2954 
2955 	if (oct->sriov_info.vf_vlantci[vfidx] == vlantci)
2956 		return 0;
2957 
2958 	memset(&nctrl, 0, sizeof(struct octnic_ctrl_pkt));
2959 
2960 	if (vlan)
2961 		nctrl.ncmd.s.cmd = OCTNET_CMD_ADD_VLAN_FILTER;
2962 	else
2963 		nctrl.ncmd.s.cmd = OCTNET_CMD_DEL_VLAN_FILTER;
2964 
2965 	nctrl.ncmd.s.param1 = vlantci;
2966 	nctrl.ncmd.s.param2 =
2967 	    vfidx + 1; /* vfidx is 0 based, but vf_num (param2) is 1 based */
2968 	nctrl.ncmd.s.more = 0;
2969 	nctrl.iq_no = lio->linfo.txpciq[0].s.q_no;
2970 	nctrl.cb_fn = NULL;
2971 
2972 	ret = octnet_send_nic_ctrl_pkt(oct, &nctrl);
2973 	if (ret) {
2974 		if (ret > 0)
2975 			ret = -EIO;
2976 		return ret;
2977 	}
2978 
2979 	oct->sriov_info.vf_vlantci[vfidx] = vlantci;
2980 
2981 	return ret;
2982 }
2983 
2984 static int liquidio_get_vf_config(struct net_device *netdev, int vfidx,
2985 				  struct ifla_vf_info *ivi)
2986 {
2987 	struct lio *lio = GET_LIO(netdev);
2988 	struct octeon_device *oct = lio->oct_dev;
2989 	u8 *macaddr;
2990 
2991 	if (vfidx < 0 || vfidx >= oct->sriov_info.num_vfs_alloced)
2992 		return -EINVAL;
2993 
2994 	memset(ivi, 0, sizeof(struct ifla_vf_info));
2995 
2996 	ivi->vf = vfidx;
2997 	macaddr = 2 + (u8 *)&oct->sriov_info.vf_macaddr[vfidx];
2998 	ether_addr_copy(&ivi->mac[0], macaddr);
2999 	ivi->vlan = oct->sriov_info.vf_vlantci[vfidx] & VLAN_VID_MASK;
3000 	ivi->qos = oct->sriov_info.vf_vlantci[vfidx] >> VLAN_PRIO_SHIFT;
3001 	if (oct->sriov_info.trusted_vf.active &&
3002 	    oct->sriov_info.trusted_vf.id == vfidx)
3003 		ivi->trusted = true;
3004 	else
3005 		ivi->trusted = false;
3006 	ivi->linkstate = oct->sriov_info.vf_linkstate[vfidx];
3007 	ivi->spoofchk = oct->sriov_info.vf_spoofchk[vfidx];
3008 	ivi->max_tx_rate = lio->linfo.link.s.speed;
3009 	ivi->min_tx_rate = 0;
3010 
3011 	return 0;
3012 }
3013 
3014 static int liquidio_send_vf_trust_cmd(struct lio *lio, int vfidx, bool trusted)
3015 {
3016 	struct octeon_device *oct = lio->oct_dev;
3017 	struct octeon_soft_command *sc;
3018 	int retval;
3019 
3020 	sc = octeon_alloc_soft_command(oct, 0, 16, 0);
3021 	if (!sc)
3022 		return -ENOMEM;
3023 
3024 	sc->iq_no = lio->linfo.txpciq[0].s.q_no;
3025 
3026 	/* vfidx is 0 based, but vf_num (param1) is 1 based */
3027 	octeon_prepare_soft_command(oct, sc, OPCODE_NIC,
3028 				    OPCODE_NIC_SET_TRUSTED_VF, 0, vfidx + 1,
3029 				    trusted);
3030 
3031 	init_completion(&sc->complete);
3032 	sc->sc_status = OCTEON_REQUEST_PENDING;
3033 
3034 	retval = octeon_send_soft_command(oct, sc);
3035 	if (retval == IQ_SEND_FAILED) {
3036 		octeon_free_soft_command(oct, sc);
3037 		retval = -1;
3038 	} else {
3039 		/* Wait for response or timeout */
3040 		retval = wait_for_sc_completion_timeout(oct, sc, 0);
3041 		if (retval)
3042 			return (retval);
3043 
3044 		WRITE_ONCE(sc->caller_is_done, true);
3045 	}
3046 
3047 	return retval;
3048 }
3049 
3050 static int liquidio_set_vf_trust(struct net_device *netdev, int vfidx,
3051 				 bool setting)
3052 {
3053 	struct lio *lio = GET_LIO(netdev);
3054 	struct octeon_device *oct = lio->oct_dev;
3055 
3056 	if (strcmp(oct->fw_info.liquidio_firmware_version, "1.7.1") < 0) {
3057 		/* trusted vf is not supported by firmware older than 1.7.1 */
3058 		return -EOPNOTSUPP;
3059 	}
3060 
3061 	if (vfidx < 0 || vfidx >= oct->sriov_info.num_vfs_alloced) {
3062 		netif_info(lio, drv, lio->netdev, "Invalid vfidx %d\n", vfidx);
3063 		return -EINVAL;
3064 	}
3065 
3066 	if (setting) {
3067 		/* Set */
3068 
3069 		if (oct->sriov_info.trusted_vf.active &&
3070 		    oct->sriov_info.trusted_vf.id == vfidx)
3071 			return 0;
3072 
3073 		if (oct->sriov_info.trusted_vf.active) {
3074 			netif_info(lio, drv, lio->netdev, "More than one trusted VF is not allowed\n");
3075 			return -EPERM;
3076 		}
3077 	} else {
3078 		/* Clear */
3079 
3080 		if (!oct->sriov_info.trusted_vf.active)
3081 			return 0;
3082 	}
3083 
3084 	if (!liquidio_send_vf_trust_cmd(lio, vfidx, setting)) {
3085 		if (setting) {
3086 			oct->sriov_info.trusted_vf.id = vfidx;
3087 			oct->sriov_info.trusted_vf.active = true;
3088 		} else {
3089 			oct->sriov_info.trusted_vf.active = false;
3090 		}
3091 
3092 		netif_info(lio, drv, lio->netdev, "VF %u is %strusted\n", vfidx,
3093 			   setting ? "" : "not ");
3094 	} else {
3095 		netif_info(lio, drv, lio->netdev, "Failed to set VF trusted\n");
3096 		return -1;
3097 	}
3098 
3099 	return 0;
3100 }
3101 
3102 static int liquidio_set_vf_link_state(struct net_device *netdev, int vfidx,
3103 				      int linkstate)
3104 {
3105 	struct lio *lio = GET_LIO(netdev);
3106 	struct octeon_device *oct = lio->oct_dev;
3107 	struct octnic_ctrl_pkt nctrl;
3108 	int ret = 0;
3109 
3110 	if (vfidx < 0 || vfidx >= oct->sriov_info.num_vfs_alloced)
3111 		return -EINVAL;
3112 
3113 	if (oct->sriov_info.vf_linkstate[vfidx] == linkstate)
3114 		return 0;
3115 
3116 	memset(&nctrl, 0, sizeof(struct octnic_ctrl_pkt));
3117 	nctrl.ncmd.s.cmd = OCTNET_CMD_SET_VF_LINKSTATE;
3118 	nctrl.ncmd.s.param1 =
3119 	    vfidx + 1; /* vfidx is 0 based, but vf_num (param1) is 1 based */
3120 	nctrl.ncmd.s.param2 = linkstate;
3121 	nctrl.ncmd.s.more = 0;
3122 	nctrl.iq_no = lio->linfo.txpciq[0].s.q_no;
3123 	nctrl.cb_fn = NULL;
3124 
3125 	ret = octnet_send_nic_ctrl_pkt(oct, &nctrl);
3126 
3127 	if (!ret)
3128 		oct->sriov_info.vf_linkstate[vfidx] = linkstate;
3129 	else if (ret > 0)
3130 		ret = -EIO;
3131 
3132 	return ret;
3133 }
3134 
3135 static int
3136 liquidio_eswitch_mode_get(struct devlink *devlink, u16 *mode)
3137 {
3138 	struct lio_devlink_priv *priv;
3139 	struct octeon_device *oct;
3140 
3141 	priv = devlink_priv(devlink);
3142 	oct = priv->oct;
3143 
3144 	*mode = oct->eswitch_mode;
3145 
3146 	return 0;
3147 }
3148 
3149 static int
3150 liquidio_eswitch_mode_set(struct devlink *devlink, u16 mode,
3151 			  struct netlink_ext_ack *extack)
3152 {
3153 	struct lio_devlink_priv *priv;
3154 	struct octeon_device *oct;
3155 	int ret = 0;
3156 
3157 	priv = devlink_priv(devlink);
3158 	oct = priv->oct;
3159 
3160 	if (!(oct->fw_info.app_cap_flags & LIQUIDIO_SWITCHDEV_CAP))
3161 		return -EINVAL;
3162 
3163 	if (oct->eswitch_mode == mode)
3164 		return 0;
3165 
3166 	switch (mode) {
3167 	case DEVLINK_ESWITCH_MODE_SWITCHDEV:
3168 		oct->eswitch_mode = mode;
3169 		ret = lio_vf_rep_create(oct);
3170 		break;
3171 
3172 	case DEVLINK_ESWITCH_MODE_LEGACY:
3173 		lio_vf_rep_destroy(oct);
3174 		oct->eswitch_mode = mode;
3175 		break;
3176 
3177 	default:
3178 		ret = -EINVAL;
3179 	}
3180 
3181 	return ret;
3182 }
3183 
3184 static const struct devlink_ops liquidio_devlink_ops = {
3185 	.eswitch_mode_get = liquidio_eswitch_mode_get,
3186 	.eswitch_mode_set = liquidio_eswitch_mode_set,
3187 };
3188 
3189 static int
3190 liquidio_get_port_parent_id(struct net_device *dev,
3191 			    struct netdev_phys_item_id *ppid)
3192 {
3193 	struct lio *lio = GET_LIO(dev);
3194 	struct octeon_device *oct = lio->oct_dev;
3195 
3196 	if (oct->eswitch_mode != DEVLINK_ESWITCH_MODE_SWITCHDEV)
3197 		return -EOPNOTSUPP;
3198 
3199 	ppid->id_len = ETH_ALEN;
3200 	ether_addr_copy(ppid->id, (void *)&lio->linfo.hw_addr + 2);
3201 
3202 	return 0;
3203 }
3204 
3205 static int liquidio_get_vf_stats(struct net_device *netdev, int vfidx,
3206 				 struct ifla_vf_stats *vf_stats)
3207 {
3208 	struct lio *lio = GET_LIO(netdev);
3209 	struct octeon_device *oct = lio->oct_dev;
3210 	struct oct_vf_stats stats;
3211 	int ret;
3212 
3213 	if (vfidx < 0 || vfidx >= oct->sriov_info.num_vfs_alloced)
3214 		return -EINVAL;
3215 
3216 	memset(&stats, 0, sizeof(struct oct_vf_stats));
3217 	ret = cn23xx_get_vf_stats(oct, vfidx, &stats);
3218 	if (!ret) {
3219 		vf_stats->rx_packets = stats.rx_packets;
3220 		vf_stats->tx_packets = stats.tx_packets;
3221 		vf_stats->rx_bytes = stats.rx_bytes;
3222 		vf_stats->tx_bytes = stats.tx_bytes;
3223 		vf_stats->broadcast = stats.broadcast;
3224 		vf_stats->multicast = stats.multicast;
3225 	}
3226 
3227 	return ret;
3228 }
3229 
3230 static const struct net_device_ops lionetdevops = {
3231 	.ndo_open		= liquidio_open,
3232 	.ndo_stop		= liquidio_stop,
3233 	.ndo_start_xmit		= liquidio_xmit,
3234 	.ndo_get_stats64	= liquidio_get_stats64,
3235 	.ndo_set_mac_address	= liquidio_set_mac,
3236 	.ndo_set_rx_mode	= liquidio_set_mcast_list,
3237 	.ndo_tx_timeout		= liquidio_tx_timeout,
3238 
3239 	.ndo_vlan_rx_add_vid    = liquidio_vlan_rx_add_vid,
3240 	.ndo_vlan_rx_kill_vid   = liquidio_vlan_rx_kill_vid,
3241 	.ndo_change_mtu		= liquidio_change_mtu,
3242 	.ndo_do_ioctl		= liquidio_ioctl,
3243 	.ndo_fix_features	= liquidio_fix_features,
3244 	.ndo_set_features	= liquidio_set_features,
3245 	.ndo_udp_tunnel_add	= liquidio_add_vxlan_port,
3246 	.ndo_udp_tunnel_del	= liquidio_del_vxlan_port,
3247 	.ndo_set_vf_mac		= liquidio_set_vf_mac,
3248 	.ndo_set_vf_vlan	= liquidio_set_vf_vlan,
3249 	.ndo_get_vf_config	= liquidio_get_vf_config,
3250 	.ndo_set_vf_spoofchk	= liquidio_set_vf_spoofchk,
3251 	.ndo_set_vf_trust	= liquidio_set_vf_trust,
3252 	.ndo_set_vf_link_state  = liquidio_set_vf_link_state,
3253 	.ndo_get_vf_stats	= liquidio_get_vf_stats,
3254 	.ndo_get_port_parent_id	= liquidio_get_port_parent_id,
3255 };
3256 
3257 /** \brief Entry point for the liquidio module
3258  */
3259 static int __init liquidio_init(void)
3260 {
3261 	int i;
3262 	struct handshake *hs;
3263 
3264 	init_completion(&first_stage);
3265 
3266 	octeon_init_device_list(OCTEON_CONFIG_TYPE_DEFAULT);
3267 
3268 	if (liquidio_init_pci())
3269 		return -EINVAL;
3270 
3271 	wait_for_completion_timeout(&first_stage, msecs_to_jiffies(1000));
3272 
3273 	for (i = 0; i < MAX_OCTEON_DEVICES; i++) {
3274 		hs = &handshake[i];
3275 		if (hs->pci_dev) {
3276 			wait_for_completion(&hs->init);
3277 			if (!hs->init_ok) {
3278 				/* init handshake failed */
3279 				dev_err(&hs->pci_dev->dev,
3280 					"Failed to init device\n");
3281 				liquidio_deinit_pci();
3282 				return -EIO;
3283 			}
3284 		}
3285 	}
3286 
3287 	for (i = 0; i < MAX_OCTEON_DEVICES; i++) {
3288 		hs = &handshake[i];
3289 		if (hs->pci_dev) {
3290 			wait_for_completion_timeout(&hs->started,
3291 						    msecs_to_jiffies(30000));
3292 			if (!hs->started_ok) {
3293 				/* starter handshake failed */
3294 				dev_err(&hs->pci_dev->dev,
3295 					"Firmware failed to start\n");
3296 				liquidio_deinit_pci();
3297 				return -EIO;
3298 			}
3299 		}
3300 	}
3301 
3302 	return 0;
3303 }
3304 
3305 static int lio_nic_info(struct octeon_recv_info *recv_info, void *buf)
3306 {
3307 	struct octeon_device *oct = (struct octeon_device *)buf;
3308 	struct octeon_recv_pkt *recv_pkt = recv_info->recv_pkt;
3309 	int gmxport = 0;
3310 	union oct_link_status *ls;
3311 	int i;
3312 
3313 	if (recv_pkt->buffer_size[0] != (sizeof(*ls) + OCT_DROQ_INFO_SIZE)) {
3314 		dev_err(&oct->pci_dev->dev, "Malformed NIC_INFO, len=%d, ifidx=%d\n",
3315 			recv_pkt->buffer_size[0],
3316 			recv_pkt->rh.r_nic_info.gmxport);
3317 		goto nic_info_err;
3318 	}
3319 
3320 	gmxport = recv_pkt->rh.r_nic_info.gmxport;
3321 	ls = (union oct_link_status *)(get_rbd(recv_pkt->buffer_ptr[0]) +
3322 		OCT_DROQ_INFO_SIZE);
3323 
3324 	octeon_swap_8B_data((u64 *)ls, (sizeof(union oct_link_status)) >> 3);
3325 	for (i = 0; i < oct->ifcount; i++) {
3326 		if (oct->props[i].gmxport == gmxport) {
3327 			update_link_status(oct->props[i].netdev, ls);
3328 			break;
3329 		}
3330 	}
3331 
3332 nic_info_err:
3333 	for (i = 0; i < recv_pkt->buffer_count; i++)
3334 		recv_buffer_free(recv_pkt->buffer_ptr[i]);
3335 	octeon_free_recv_info(recv_info);
3336 	return 0;
3337 }
3338 
3339 /**
3340  * \brief Setup network interfaces
3341  * @param octeon_dev  octeon device
3342  *
3343  * Called during init time for each device. It assumes the NIC
3344  * is already up and running.  The link information for each
3345  * interface is passed in link_info.
3346  */
3347 static int setup_nic_devices(struct octeon_device *octeon_dev)
3348 {
3349 	struct lio *lio = NULL;
3350 	struct net_device *netdev;
3351 	u8 mac[6], i, j, *fw_ver, *micro_ver;
3352 	unsigned long micro;
3353 	u32 cur_ver;
3354 	struct octeon_soft_command *sc;
3355 	struct liquidio_if_cfg_resp *resp;
3356 	struct octdev_props *props;
3357 	int retval, num_iqueues, num_oqueues;
3358 	int max_num_queues = 0;
3359 	union oct_nic_if_cfg if_cfg;
3360 	unsigned int base_queue;
3361 	unsigned int gmx_port_id;
3362 	u32 resp_size, data_size;
3363 	u32 ifidx_or_pfnum;
3364 	struct lio_version *vdata;
3365 	struct devlink *devlink;
3366 	struct lio_devlink_priv *lio_devlink;
3367 
3368 	/* This is to handle link status changes */
3369 	octeon_register_dispatch_fn(octeon_dev, OPCODE_NIC,
3370 				    OPCODE_NIC_INFO,
3371 				    lio_nic_info, octeon_dev);
3372 
3373 	/* REQTYPE_RESP_NET and REQTYPE_SOFT_COMMAND do not have free functions.
3374 	 * They are handled directly.
3375 	 */
3376 	octeon_register_reqtype_free_fn(octeon_dev, REQTYPE_NORESP_NET,
3377 					free_netbuf);
3378 
3379 	octeon_register_reqtype_free_fn(octeon_dev, REQTYPE_NORESP_NET_SG,
3380 					free_netsgbuf);
3381 
3382 	octeon_register_reqtype_free_fn(octeon_dev, REQTYPE_RESP_NET_SG,
3383 					free_netsgbuf_with_resp);
3384 
3385 	for (i = 0; i < octeon_dev->ifcount; i++) {
3386 		resp_size = sizeof(struct liquidio_if_cfg_resp);
3387 		data_size = sizeof(struct lio_version);
3388 		sc = (struct octeon_soft_command *)
3389 			octeon_alloc_soft_command(octeon_dev, data_size,
3390 						  resp_size, 0);
3391 		resp = (struct liquidio_if_cfg_resp *)sc->virtrptr;
3392 		vdata = (struct lio_version *)sc->virtdptr;
3393 
3394 		*((u64 *)vdata) = 0;
3395 		vdata->major = cpu_to_be16(LIQUIDIO_BASE_MAJOR_VERSION);
3396 		vdata->minor = cpu_to_be16(LIQUIDIO_BASE_MINOR_VERSION);
3397 		vdata->micro = cpu_to_be16(LIQUIDIO_BASE_MICRO_VERSION);
3398 
3399 		if (OCTEON_CN23XX_PF(octeon_dev)) {
3400 			num_iqueues = octeon_dev->sriov_info.num_pf_rings;
3401 			num_oqueues = octeon_dev->sriov_info.num_pf_rings;
3402 			base_queue = octeon_dev->sriov_info.pf_srn;
3403 
3404 			gmx_port_id = octeon_dev->pf_num;
3405 			ifidx_or_pfnum = octeon_dev->pf_num;
3406 		} else {
3407 			num_iqueues = CFG_GET_NUM_TXQS_NIC_IF(
3408 						octeon_get_conf(octeon_dev), i);
3409 			num_oqueues = CFG_GET_NUM_RXQS_NIC_IF(
3410 						octeon_get_conf(octeon_dev), i);
3411 			base_queue = CFG_GET_BASE_QUE_NIC_IF(
3412 						octeon_get_conf(octeon_dev), i);
3413 			gmx_port_id = CFG_GET_GMXID_NIC_IF(
3414 						octeon_get_conf(octeon_dev), i);
3415 			ifidx_or_pfnum = i;
3416 		}
3417 
3418 		dev_dbg(&octeon_dev->pci_dev->dev,
3419 			"requesting config for interface %d, iqs %d, oqs %d\n",
3420 			ifidx_or_pfnum, num_iqueues, num_oqueues);
3421 
3422 		if_cfg.u64 = 0;
3423 		if_cfg.s.num_iqueues = num_iqueues;
3424 		if_cfg.s.num_oqueues = num_oqueues;
3425 		if_cfg.s.base_queue = base_queue;
3426 		if_cfg.s.gmx_port_id = gmx_port_id;
3427 
3428 		sc->iq_no = 0;
3429 
3430 		octeon_prepare_soft_command(octeon_dev, sc, OPCODE_NIC,
3431 					    OPCODE_NIC_IF_CFG, 0,
3432 					    if_cfg.u64, 0);
3433 
3434 		init_completion(&sc->complete);
3435 		sc->sc_status = OCTEON_REQUEST_PENDING;
3436 
3437 		retval = octeon_send_soft_command(octeon_dev, sc);
3438 		if (retval == IQ_SEND_FAILED) {
3439 			dev_err(&octeon_dev->pci_dev->dev,
3440 				"iq/oq config failed status: %x\n",
3441 				retval);
3442 			/* Soft instr is freed by driver in case of failure. */
3443 			octeon_free_soft_command(octeon_dev, sc);
3444 			return(-EIO);
3445 		}
3446 
3447 		/* Sleep on a wait queue till the cond flag indicates that the
3448 		 * response arrived or timed-out.
3449 		 */
3450 		retval = wait_for_sc_completion_timeout(octeon_dev, sc, 0);
3451 		if (retval)
3452 			return retval;
3453 
3454 		retval = resp->status;
3455 		if (retval) {
3456 			dev_err(&octeon_dev->pci_dev->dev, "iq/oq config failed\n");
3457 			WRITE_ONCE(sc->caller_is_done, true);
3458 			goto setup_nic_dev_done;
3459 		}
3460 		snprintf(octeon_dev->fw_info.liquidio_firmware_version,
3461 			 32, "%s",
3462 			 resp->cfg_info.liquidio_firmware_version);
3463 
3464 		/* Verify f/w version (in case of 'auto' loading from flash) */
3465 		fw_ver = octeon_dev->fw_info.liquidio_firmware_version;
3466 		if (memcmp(LIQUIDIO_BASE_VERSION,
3467 			   fw_ver,
3468 			   strlen(LIQUIDIO_BASE_VERSION))) {
3469 			dev_err(&octeon_dev->pci_dev->dev,
3470 				"Unmatched firmware version. Expected %s.x, got %s.\n",
3471 				LIQUIDIO_BASE_VERSION, fw_ver);
3472 			WRITE_ONCE(sc->caller_is_done, true);
3473 			goto setup_nic_dev_done;
3474 		} else if (atomic_read(octeon_dev->adapter_fw_state) ==
3475 			   FW_IS_PRELOADED) {
3476 			dev_info(&octeon_dev->pci_dev->dev,
3477 				 "Using auto-loaded firmware version %s.\n",
3478 				 fw_ver);
3479 		}
3480 
3481 		/* extract micro version field; point past '<maj>.<min>.' */
3482 		micro_ver = fw_ver + strlen(LIQUIDIO_BASE_VERSION) + 1;
3483 		if (kstrtoul(micro_ver, 10, &micro) != 0)
3484 			micro = 0;
3485 		octeon_dev->fw_info.ver.maj = LIQUIDIO_BASE_MAJOR_VERSION;
3486 		octeon_dev->fw_info.ver.min = LIQUIDIO_BASE_MINOR_VERSION;
3487 		octeon_dev->fw_info.ver.rev = micro;
3488 
3489 		octeon_swap_8B_data((u64 *)(&resp->cfg_info),
3490 				    (sizeof(struct liquidio_if_cfg_info)) >> 3);
3491 
3492 		num_iqueues = hweight64(resp->cfg_info.iqmask);
3493 		num_oqueues = hweight64(resp->cfg_info.oqmask);
3494 
3495 		if (!(num_iqueues) || !(num_oqueues)) {
3496 			dev_err(&octeon_dev->pci_dev->dev,
3497 				"Got bad iqueues (%016llx) or oqueues (%016llx) from firmware.\n",
3498 				resp->cfg_info.iqmask,
3499 				resp->cfg_info.oqmask);
3500 			WRITE_ONCE(sc->caller_is_done, true);
3501 			goto setup_nic_dev_done;
3502 		}
3503 
3504 		if (OCTEON_CN6XXX(octeon_dev)) {
3505 			max_num_queues = CFG_GET_IQ_MAX_Q(CHIP_CONF(octeon_dev,
3506 								    cn6xxx));
3507 		} else if (OCTEON_CN23XX_PF(octeon_dev)) {
3508 			max_num_queues = CFG_GET_IQ_MAX_Q(CHIP_CONF(octeon_dev,
3509 								    cn23xx_pf));
3510 		}
3511 
3512 		dev_dbg(&octeon_dev->pci_dev->dev,
3513 			"interface %d, iqmask %016llx, oqmask %016llx, numiqueues %d, numoqueues %d max_num_queues: %d\n",
3514 			i, resp->cfg_info.iqmask, resp->cfg_info.oqmask,
3515 			num_iqueues, num_oqueues, max_num_queues);
3516 		netdev = alloc_etherdev_mq(LIO_SIZE, max_num_queues);
3517 
3518 		if (!netdev) {
3519 			dev_err(&octeon_dev->pci_dev->dev, "Device allocation failed\n");
3520 			WRITE_ONCE(sc->caller_is_done, true);
3521 			goto setup_nic_dev_done;
3522 		}
3523 
3524 		SET_NETDEV_DEV(netdev, &octeon_dev->pci_dev->dev);
3525 
3526 		/* Associate the routines that will handle different
3527 		 * netdev tasks.
3528 		 */
3529 		netdev->netdev_ops = &lionetdevops;
3530 
3531 		retval = netif_set_real_num_rx_queues(netdev, num_oqueues);
3532 		if (retval) {
3533 			dev_err(&octeon_dev->pci_dev->dev,
3534 				"setting real number rx failed\n");
3535 			WRITE_ONCE(sc->caller_is_done, true);
3536 			goto setup_nic_dev_free;
3537 		}
3538 
3539 		retval = netif_set_real_num_tx_queues(netdev, num_iqueues);
3540 		if (retval) {
3541 			dev_err(&octeon_dev->pci_dev->dev,
3542 				"setting real number tx failed\n");
3543 			WRITE_ONCE(sc->caller_is_done, true);
3544 			goto setup_nic_dev_free;
3545 		}
3546 
3547 		lio = GET_LIO(netdev);
3548 
3549 		memset(lio, 0, sizeof(struct lio));
3550 
3551 		lio->ifidx = ifidx_or_pfnum;
3552 
3553 		props = &octeon_dev->props[i];
3554 		props->gmxport = resp->cfg_info.linfo.gmxport;
3555 		props->netdev = netdev;
3556 
3557 		lio->linfo.num_rxpciq = num_oqueues;
3558 		lio->linfo.num_txpciq = num_iqueues;
3559 		for (j = 0; j < num_oqueues; j++) {
3560 			lio->linfo.rxpciq[j].u64 =
3561 				resp->cfg_info.linfo.rxpciq[j].u64;
3562 		}
3563 		for (j = 0; j < num_iqueues; j++) {
3564 			lio->linfo.txpciq[j].u64 =
3565 				resp->cfg_info.linfo.txpciq[j].u64;
3566 		}
3567 		lio->linfo.hw_addr = resp->cfg_info.linfo.hw_addr;
3568 		lio->linfo.gmxport = resp->cfg_info.linfo.gmxport;
3569 		lio->linfo.link.u64 = resp->cfg_info.linfo.link.u64;
3570 
3571 		WRITE_ONCE(sc->caller_is_done, true);
3572 
3573 		lio->msg_enable = netif_msg_init(debug, DEFAULT_MSG_ENABLE);
3574 
3575 		if (OCTEON_CN23XX_PF(octeon_dev) ||
3576 		    OCTEON_CN6XXX(octeon_dev)) {
3577 			lio->dev_capability = NETIF_F_HIGHDMA
3578 					      | NETIF_F_IP_CSUM
3579 					      | NETIF_F_IPV6_CSUM
3580 					      | NETIF_F_SG | NETIF_F_RXCSUM
3581 					      | NETIF_F_GRO
3582 					      | NETIF_F_TSO | NETIF_F_TSO6
3583 					      | NETIF_F_LRO;
3584 		}
3585 		netif_set_gso_max_size(netdev, OCTNIC_GSO_MAX_SIZE);
3586 
3587 		/*  Copy of transmit encapsulation capabilities:
3588 		 *  TSO, TSO6, Checksums for this device
3589 		 */
3590 		lio->enc_dev_capability = NETIF_F_IP_CSUM
3591 					  | NETIF_F_IPV6_CSUM
3592 					  | NETIF_F_GSO_UDP_TUNNEL
3593 					  | NETIF_F_HW_CSUM | NETIF_F_SG
3594 					  | NETIF_F_RXCSUM
3595 					  | NETIF_F_TSO | NETIF_F_TSO6
3596 					  | NETIF_F_LRO;
3597 
3598 		netdev->hw_enc_features = (lio->enc_dev_capability &
3599 					   ~NETIF_F_LRO);
3600 
3601 		lio->dev_capability |= NETIF_F_GSO_UDP_TUNNEL;
3602 
3603 		netdev->vlan_features = lio->dev_capability;
3604 		/* Add any unchangeable hw features */
3605 		lio->dev_capability |=  NETIF_F_HW_VLAN_CTAG_FILTER |
3606 					NETIF_F_HW_VLAN_CTAG_RX |
3607 					NETIF_F_HW_VLAN_CTAG_TX;
3608 
3609 		netdev->features = (lio->dev_capability & ~NETIF_F_LRO);
3610 
3611 		netdev->hw_features = lio->dev_capability;
3612 		/*HW_VLAN_RX and HW_VLAN_FILTER is always on*/
3613 		netdev->hw_features = netdev->hw_features &
3614 			~NETIF_F_HW_VLAN_CTAG_RX;
3615 
3616 		/* MTU range: 68 - 16000 */
3617 		netdev->min_mtu = LIO_MIN_MTU_SIZE;
3618 		netdev->max_mtu = LIO_MAX_MTU_SIZE;
3619 
3620 		/* Point to the  properties for octeon device to which this
3621 		 * interface belongs.
3622 		 */
3623 		lio->oct_dev = octeon_dev;
3624 		lio->octprops = props;
3625 		lio->netdev = netdev;
3626 
3627 		dev_dbg(&octeon_dev->pci_dev->dev,
3628 			"if%d gmx: %d hw_addr: 0x%llx\n", i,
3629 			lio->linfo.gmxport, CVM_CAST64(lio->linfo.hw_addr));
3630 
3631 		for (j = 0; j < octeon_dev->sriov_info.max_vfs; j++) {
3632 			u8 vfmac[ETH_ALEN];
3633 
3634 			eth_random_addr(vfmac);
3635 			if (__liquidio_set_vf_mac(netdev, j, vfmac, false)) {
3636 				dev_err(&octeon_dev->pci_dev->dev,
3637 					"Error setting VF%d MAC address\n",
3638 					j);
3639 				goto setup_nic_dev_free;
3640 			}
3641 		}
3642 
3643 		/* 64-bit swap required on LE machines */
3644 		octeon_swap_8B_data(&lio->linfo.hw_addr, 1);
3645 		for (j = 0; j < 6; j++)
3646 			mac[j] = *((u8 *)(((u8 *)&lio->linfo.hw_addr) + 2 + j));
3647 
3648 		/* Copy MAC Address to OS network device structure */
3649 
3650 		ether_addr_copy(netdev->dev_addr, mac);
3651 
3652 		/* By default all interfaces on a single Octeon uses the same
3653 		 * tx and rx queues
3654 		 */
3655 		lio->txq = lio->linfo.txpciq[0].s.q_no;
3656 		lio->rxq = lio->linfo.rxpciq[0].s.q_no;
3657 		if (liquidio_setup_io_queues(octeon_dev, i,
3658 					     lio->linfo.num_txpciq,
3659 					     lio->linfo.num_rxpciq)) {
3660 			dev_err(&octeon_dev->pci_dev->dev, "I/O queues creation failed\n");
3661 			goto setup_nic_dev_free;
3662 		}
3663 
3664 		ifstate_set(lio, LIO_IFSTATE_DROQ_OPS);
3665 
3666 		lio->tx_qsize = octeon_get_tx_qsize(octeon_dev, lio->txq);
3667 		lio->rx_qsize = octeon_get_rx_qsize(octeon_dev, lio->rxq);
3668 
3669 		if (lio_setup_glists(octeon_dev, lio, num_iqueues)) {
3670 			dev_err(&octeon_dev->pci_dev->dev,
3671 				"Gather list allocation failed\n");
3672 			goto setup_nic_dev_free;
3673 		}
3674 
3675 		/* Register ethtool support */
3676 		liquidio_set_ethtool_ops(netdev);
3677 		if (lio->oct_dev->chip_id == OCTEON_CN23XX_PF_VID)
3678 			octeon_dev->priv_flags = OCT_PRIV_FLAG_DEFAULT;
3679 		else
3680 			octeon_dev->priv_flags = 0x0;
3681 
3682 		if (netdev->features & NETIF_F_LRO)
3683 			liquidio_set_feature(netdev, OCTNET_CMD_LRO_ENABLE,
3684 					     OCTNIC_LROIPV4 | OCTNIC_LROIPV6);
3685 
3686 		liquidio_set_feature(netdev, OCTNET_CMD_VLAN_FILTER_CTL,
3687 				     OCTNET_CMD_VLAN_FILTER_ENABLE);
3688 
3689 		if ((debug != -1) && (debug & NETIF_MSG_HW))
3690 			liquidio_set_feature(netdev,
3691 					     OCTNET_CMD_VERBOSE_ENABLE, 0);
3692 
3693 		if (setup_link_status_change_wq(netdev))
3694 			goto setup_nic_dev_free;
3695 
3696 		if ((octeon_dev->fw_info.app_cap_flags &
3697 		     LIQUIDIO_TIME_SYNC_CAP) &&
3698 		    setup_sync_octeon_time_wq(netdev))
3699 			goto setup_nic_dev_free;
3700 
3701 		if (setup_rx_oom_poll_fn(netdev))
3702 			goto setup_nic_dev_free;
3703 
3704 		/* Register the network device with the OS */
3705 		if (register_netdev(netdev)) {
3706 			dev_err(&octeon_dev->pci_dev->dev, "Device registration failed\n");
3707 			goto setup_nic_dev_free;
3708 		}
3709 
3710 		dev_dbg(&octeon_dev->pci_dev->dev,
3711 			"Setup NIC ifidx:%d mac:%02x%02x%02x%02x%02x%02x\n",
3712 			i, mac[0], mac[1], mac[2], mac[3], mac[4], mac[5]);
3713 		netif_carrier_off(netdev);
3714 		lio->link_changes++;
3715 
3716 		ifstate_set(lio, LIO_IFSTATE_REGISTERED);
3717 
3718 		/* Sending command to firmware to enable Rx checksum offload
3719 		 * by default at the time of setup of Liquidio driver for
3720 		 * this device
3721 		 */
3722 		liquidio_set_rxcsum_command(netdev, OCTNET_CMD_TNL_RX_CSUM_CTL,
3723 					    OCTNET_CMD_RXCSUM_ENABLE);
3724 		liquidio_set_feature(netdev, OCTNET_CMD_TNL_TX_CSUM_CTL,
3725 				     OCTNET_CMD_TXCSUM_ENABLE);
3726 
3727 		dev_dbg(&octeon_dev->pci_dev->dev,
3728 			"NIC ifidx:%d Setup successful\n", i);
3729 
3730 		if (octeon_dev->subsystem_id ==
3731 			OCTEON_CN2350_25GB_SUBSYS_ID ||
3732 		    octeon_dev->subsystem_id ==
3733 			OCTEON_CN2360_25GB_SUBSYS_ID) {
3734 			cur_ver = OCT_FW_VER(octeon_dev->fw_info.ver.maj,
3735 					     octeon_dev->fw_info.ver.min,
3736 					     octeon_dev->fw_info.ver.rev);
3737 
3738 			/* speed control unsupported in f/w older than 1.7.2 */
3739 			if (cur_ver < OCT_FW_VER(1, 7, 2)) {
3740 				dev_info(&octeon_dev->pci_dev->dev,
3741 					 "speed setting not supported by f/w.");
3742 				octeon_dev->speed_setting = 25;
3743 				octeon_dev->no_speed_setting = 1;
3744 			} else {
3745 				liquidio_get_speed(lio);
3746 			}
3747 
3748 			if (octeon_dev->speed_setting == 0) {
3749 				octeon_dev->speed_setting = 25;
3750 				octeon_dev->no_speed_setting = 1;
3751 			}
3752 		} else {
3753 			octeon_dev->no_speed_setting = 1;
3754 			octeon_dev->speed_setting = 10;
3755 		}
3756 		octeon_dev->speed_boot = octeon_dev->speed_setting;
3757 
3758 		/* don't read FEC setting if unsupported by f/w (see above) */
3759 		if (octeon_dev->speed_boot == 25 &&
3760 		    !octeon_dev->no_speed_setting) {
3761 			liquidio_get_fec(lio);
3762 			octeon_dev->props[lio->ifidx].fec_boot =
3763 				octeon_dev->props[lio->ifidx].fec;
3764 		}
3765 	}
3766 
3767 	devlink = devlink_alloc(&liquidio_devlink_ops,
3768 				sizeof(struct lio_devlink_priv));
3769 	if (!devlink) {
3770 		dev_err(&octeon_dev->pci_dev->dev, "devlink alloc failed\n");
3771 		goto setup_nic_dev_free;
3772 	}
3773 
3774 	lio_devlink = devlink_priv(devlink);
3775 	lio_devlink->oct = octeon_dev;
3776 
3777 	if (devlink_register(devlink, &octeon_dev->pci_dev->dev)) {
3778 		devlink_free(devlink);
3779 		dev_err(&octeon_dev->pci_dev->dev,
3780 			"devlink registration failed\n");
3781 		goto setup_nic_dev_free;
3782 	}
3783 
3784 	octeon_dev->devlink = devlink;
3785 	octeon_dev->eswitch_mode = DEVLINK_ESWITCH_MODE_LEGACY;
3786 
3787 	return 0;
3788 
3789 setup_nic_dev_free:
3790 
3791 	while (i--) {
3792 		dev_err(&octeon_dev->pci_dev->dev,
3793 			"NIC ifidx:%d Setup failed\n", i);
3794 		liquidio_destroy_nic_device(octeon_dev, i);
3795 	}
3796 
3797 setup_nic_dev_done:
3798 
3799 	return -ENODEV;
3800 }
3801 
3802 #ifdef CONFIG_PCI_IOV
3803 static int octeon_enable_sriov(struct octeon_device *oct)
3804 {
3805 	unsigned int num_vfs_alloced = oct->sriov_info.num_vfs_alloced;
3806 	struct pci_dev *vfdev;
3807 	int err;
3808 	u32 u;
3809 
3810 	if (OCTEON_CN23XX_PF(oct) && num_vfs_alloced) {
3811 		err = pci_enable_sriov(oct->pci_dev,
3812 				       oct->sriov_info.num_vfs_alloced);
3813 		if (err) {
3814 			dev_err(&oct->pci_dev->dev,
3815 				"OCTEON: Failed to enable PCI sriov: %d\n",
3816 				err);
3817 			oct->sriov_info.num_vfs_alloced = 0;
3818 			return err;
3819 		}
3820 		oct->sriov_info.sriov_enabled = 1;
3821 
3822 		/* init lookup table that maps DPI ring number to VF pci_dev
3823 		 * struct pointer
3824 		 */
3825 		u = 0;
3826 		vfdev = pci_get_device(PCI_VENDOR_ID_CAVIUM,
3827 				       OCTEON_CN23XX_VF_VID, NULL);
3828 		while (vfdev) {
3829 			if (vfdev->is_virtfn &&
3830 			    (vfdev->physfn == oct->pci_dev)) {
3831 				oct->sriov_info.dpiring_to_vfpcidev_lut[u] =
3832 					vfdev;
3833 				u += oct->sriov_info.rings_per_vf;
3834 			}
3835 			vfdev = pci_get_device(PCI_VENDOR_ID_CAVIUM,
3836 					       OCTEON_CN23XX_VF_VID, vfdev);
3837 		}
3838 	}
3839 
3840 	return num_vfs_alloced;
3841 }
3842 
3843 static int lio_pci_sriov_disable(struct octeon_device *oct)
3844 {
3845 	int u;
3846 
3847 	if (pci_vfs_assigned(oct->pci_dev)) {
3848 		dev_err(&oct->pci_dev->dev, "VFs are still assigned to VMs.\n");
3849 		return -EPERM;
3850 	}
3851 
3852 	pci_disable_sriov(oct->pci_dev);
3853 
3854 	u = 0;
3855 	while (u < MAX_POSSIBLE_VFS) {
3856 		oct->sriov_info.dpiring_to_vfpcidev_lut[u] = NULL;
3857 		u += oct->sriov_info.rings_per_vf;
3858 	}
3859 
3860 	oct->sriov_info.num_vfs_alloced = 0;
3861 	dev_info(&oct->pci_dev->dev, "oct->pf_num:%d disabled VFs\n",
3862 		 oct->pf_num);
3863 
3864 	return 0;
3865 }
3866 
3867 static int liquidio_enable_sriov(struct pci_dev *dev, int num_vfs)
3868 {
3869 	struct octeon_device *oct = pci_get_drvdata(dev);
3870 	int ret = 0;
3871 
3872 	if ((num_vfs == oct->sriov_info.num_vfs_alloced) &&
3873 	    (oct->sriov_info.sriov_enabled)) {
3874 		dev_info(&oct->pci_dev->dev, "oct->pf_num:%d already enabled num_vfs:%d\n",
3875 			 oct->pf_num, num_vfs);
3876 		return 0;
3877 	}
3878 
3879 	if (!num_vfs) {
3880 		lio_vf_rep_destroy(oct);
3881 		ret = lio_pci_sriov_disable(oct);
3882 	} else if (num_vfs > oct->sriov_info.max_vfs) {
3883 		dev_err(&oct->pci_dev->dev,
3884 			"OCTEON: Max allowed VFs:%d user requested:%d",
3885 			oct->sriov_info.max_vfs, num_vfs);
3886 		ret = -EPERM;
3887 	} else {
3888 		oct->sriov_info.num_vfs_alloced = num_vfs;
3889 		ret = octeon_enable_sriov(oct);
3890 		dev_info(&oct->pci_dev->dev, "oct->pf_num:%d num_vfs:%d\n",
3891 			 oct->pf_num, num_vfs);
3892 		ret = lio_vf_rep_create(oct);
3893 		if (ret)
3894 			dev_info(&oct->pci_dev->dev,
3895 				 "vf representor create failed");
3896 	}
3897 
3898 	return ret;
3899 }
3900 #endif
3901 
3902 /**
3903  * \brief initialize the NIC
3904  * @param oct octeon device
3905  *
3906  * This initialization routine is called once the Octeon device application is
3907  * up and running
3908  */
3909 static int liquidio_init_nic_module(struct octeon_device *oct)
3910 {
3911 	int i, retval = 0;
3912 	int num_nic_ports = CFG_GET_NUM_NIC_PORTS(octeon_get_conf(oct));
3913 
3914 	dev_dbg(&oct->pci_dev->dev, "Initializing network interfaces\n");
3915 
3916 	/* only default iq and oq were initialized
3917 	 * initialize the rest as well
3918 	 */
3919 	/* run port_config command for each port */
3920 	oct->ifcount = num_nic_ports;
3921 
3922 	memset(oct->props, 0, sizeof(struct octdev_props) * num_nic_ports);
3923 
3924 	for (i = 0; i < MAX_OCTEON_LINKS; i++)
3925 		oct->props[i].gmxport = -1;
3926 
3927 	retval = setup_nic_devices(oct);
3928 	if (retval) {
3929 		dev_err(&oct->pci_dev->dev, "Setup NIC devices failed\n");
3930 		goto octnet_init_failure;
3931 	}
3932 
3933 	/* Call vf_rep_modinit if the firmware is switchdev capable
3934 	 * and do it from the first liquidio function probed.
3935 	 */
3936 	if (!oct->octeon_id &&
3937 	    oct->fw_info.app_cap_flags & LIQUIDIO_SWITCHDEV_CAP) {
3938 		retval = lio_vf_rep_modinit();
3939 		if (retval) {
3940 			liquidio_stop_nic_module(oct);
3941 			goto octnet_init_failure;
3942 		}
3943 	}
3944 
3945 	liquidio_ptp_init(oct);
3946 
3947 	dev_dbg(&oct->pci_dev->dev, "Network interfaces ready\n");
3948 
3949 	return retval;
3950 
3951 octnet_init_failure:
3952 
3953 	oct->ifcount = 0;
3954 
3955 	return retval;
3956 }
3957 
3958 /**
3959  * \brief starter callback that invokes the remaining initialization work after
3960  * the NIC is up and running.
3961  * @param octptr  work struct work_struct
3962  */
3963 static void nic_starter(struct work_struct *work)
3964 {
3965 	struct octeon_device *oct;
3966 	struct cavium_wk *wk = (struct cavium_wk *)work;
3967 
3968 	oct = (struct octeon_device *)wk->ctxptr;
3969 
3970 	if (atomic_read(&oct->status) == OCT_DEV_RUNNING)
3971 		return;
3972 
3973 	/* If the status of the device is CORE_OK, the core
3974 	 * application has reported its application type. Call
3975 	 * any registered handlers now and move to the RUNNING
3976 	 * state.
3977 	 */
3978 	if (atomic_read(&oct->status) != OCT_DEV_CORE_OK) {
3979 		schedule_delayed_work(&oct->nic_poll_work.work,
3980 				      LIQUIDIO_STARTER_POLL_INTERVAL_MS);
3981 		return;
3982 	}
3983 
3984 	atomic_set(&oct->status, OCT_DEV_RUNNING);
3985 
3986 	if (oct->app_mode && oct->app_mode == CVM_DRV_NIC_APP) {
3987 		dev_dbg(&oct->pci_dev->dev, "Starting NIC module\n");
3988 
3989 		if (liquidio_init_nic_module(oct))
3990 			dev_err(&oct->pci_dev->dev, "NIC initialization failed\n");
3991 		else
3992 			handshake[oct->octeon_id].started_ok = 1;
3993 	} else {
3994 		dev_err(&oct->pci_dev->dev,
3995 			"Unexpected application running on NIC (%d). Check firmware.\n",
3996 			oct->app_mode);
3997 	}
3998 
3999 	complete(&handshake[oct->octeon_id].started);
4000 }
4001 
4002 static int
4003 octeon_recv_vf_drv_notice(struct octeon_recv_info *recv_info, void *buf)
4004 {
4005 	struct octeon_device *oct = (struct octeon_device *)buf;
4006 	struct octeon_recv_pkt *recv_pkt = recv_info->recv_pkt;
4007 	int i, notice, vf_idx;
4008 	bool cores_crashed;
4009 	u64 *data, vf_num;
4010 
4011 	notice = recv_pkt->rh.r.ossp;
4012 	data = (u64 *)(get_rbd(recv_pkt->buffer_ptr[0]) + OCT_DROQ_INFO_SIZE);
4013 
4014 	/* the first 64-bit word of data is the vf_num */
4015 	vf_num = data[0];
4016 	octeon_swap_8B_data(&vf_num, 1);
4017 	vf_idx = (int)vf_num - 1;
4018 
4019 	cores_crashed = READ_ONCE(oct->cores_crashed);
4020 
4021 	if (notice == VF_DRV_LOADED) {
4022 		if (!(oct->sriov_info.vf_drv_loaded_mask & BIT_ULL(vf_idx))) {
4023 			oct->sriov_info.vf_drv_loaded_mask |= BIT_ULL(vf_idx);
4024 			dev_info(&oct->pci_dev->dev,
4025 				 "driver for VF%d was loaded\n", vf_idx);
4026 			if (!cores_crashed)
4027 				try_module_get(THIS_MODULE);
4028 		}
4029 	} else if (notice == VF_DRV_REMOVED) {
4030 		if (oct->sriov_info.vf_drv_loaded_mask & BIT_ULL(vf_idx)) {
4031 			oct->sriov_info.vf_drv_loaded_mask &= ~BIT_ULL(vf_idx);
4032 			dev_info(&oct->pci_dev->dev,
4033 				 "driver for VF%d was removed\n", vf_idx);
4034 			if (!cores_crashed)
4035 				module_put(THIS_MODULE);
4036 		}
4037 	} else if (notice == VF_DRV_MACADDR_CHANGED) {
4038 		u8 *b = (u8 *)&data[1];
4039 
4040 		oct->sriov_info.vf_macaddr[vf_idx] = data[1];
4041 		dev_info(&oct->pci_dev->dev,
4042 			 "VF driver changed VF%d's MAC address to %pM\n",
4043 			 vf_idx, b + 2);
4044 	}
4045 
4046 	for (i = 0; i < recv_pkt->buffer_count; i++)
4047 		recv_buffer_free(recv_pkt->buffer_ptr[i]);
4048 	octeon_free_recv_info(recv_info);
4049 
4050 	return 0;
4051 }
4052 
4053 /**
4054  * \brief Device initialization for each Octeon device that is probed
4055  * @param octeon_dev  octeon device
4056  */
4057 static int octeon_device_init(struct octeon_device *octeon_dev)
4058 {
4059 	int j, ret;
4060 	char bootcmd[] = "\n";
4061 	char *dbg_enb = NULL;
4062 	enum lio_fw_state fw_state;
4063 	struct octeon_device_priv *oct_priv =
4064 		(struct octeon_device_priv *)octeon_dev->priv;
4065 	atomic_set(&octeon_dev->status, OCT_DEV_BEGIN_STATE);
4066 
4067 	/* Enable access to the octeon device and make its DMA capability
4068 	 * known to the OS.
4069 	 */
4070 	if (octeon_pci_os_setup(octeon_dev))
4071 		return 1;
4072 
4073 	atomic_set(&octeon_dev->status, OCT_DEV_PCI_ENABLE_DONE);
4074 
4075 	/* Identify the Octeon type and map the BAR address space. */
4076 	if (octeon_chip_specific_setup(octeon_dev)) {
4077 		dev_err(&octeon_dev->pci_dev->dev, "Chip specific setup failed\n");
4078 		return 1;
4079 	}
4080 
4081 	atomic_set(&octeon_dev->status, OCT_DEV_PCI_MAP_DONE);
4082 
4083 	/* Only add a reference after setting status 'OCT_DEV_PCI_MAP_DONE',
4084 	 * since that is what is required for the reference to be removed
4085 	 * during de-initialization (see 'octeon_destroy_resources').
4086 	 */
4087 	octeon_register_device(octeon_dev, octeon_dev->pci_dev->bus->number,
4088 			       PCI_SLOT(octeon_dev->pci_dev->devfn),
4089 			       PCI_FUNC(octeon_dev->pci_dev->devfn),
4090 			       true);
4091 
4092 	octeon_dev->app_mode = CVM_DRV_INVALID_APP;
4093 
4094 	/* CN23XX supports preloaded firmware if the following is true:
4095 	 *
4096 	 * The adapter indicates that firmware is currently running AND
4097 	 * 'fw_type' is 'auto'.
4098 	 *
4099 	 * (default state is NEEDS_TO_BE_LOADED, override it if appropriate).
4100 	 */
4101 	if (OCTEON_CN23XX_PF(octeon_dev) &&
4102 	    cn23xx_fw_loaded(octeon_dev) && fw_type_is_auto()) {
4103 		atomic_cmpxchg(octeon_dev->adapter_fw_state,
4104 			       FW_NEEDS_TO_BE_LOADED, FW_IS_PRELOADED);
4105 	}
4106 
4107 	/* If loading firmware, only first device of adapter needs to do so. */
4108 	fw_state = atomic_cmpxchg(octeon_dev->adapter_fw_state,
4109 				  FW_NEEDS_TO_BE_LOADED,
4110 				  FW_IS_BEING_LOADED);
4111 
4112 	/* Here, [local variable] 'fw_state' is set to one of:
4113 	 *
4114 	 *   FW_IS_PRELOADED:       No firmware is to be loaded (see above)
4115 	 *   FW_NEEDS_TO_BE_LOADED: The driver's first instance will load
4116 	 *                          firmware to the adapter.
4117 	 *   FW_IS_BEING_LOADED:    The driver's second instance will not load
4118 	 *                          firmware to the adapter.
4119 	 */
4120 
4121 	/* Prior to f/w load, perform a soft reset of the Octeon device;
4122 	 * if error resetting, return w/error.
4123 	 */
4124 	if (fw_state == FW_NEEDS_TO_BE_LOADED)
4125 		if (octeon_dev->fn_list.soft_reset(octeon_dev))
4126 			return 1;
4127 
4128 	/* Initialize the dispatch mechanism used to push packets arriving on
4129 	 * Octeon Output queues.
4130 	 */
4131 	if (octeon_init_dispatch_list(octeon_dev))
4132 		return 1;
4133 
4134 	octeon_register_dispatch_fn(octeon_dev, OPCODE_NIC,
4135 				    OPCODE_NIC_CORE_DRV_ACTIVE,
4136 				    octeon_core_drv_init,
4137 				    octeon_dev);
4138 
4139 	octeon_register_dispatch_fn(octeon_dev, OPCODE_NIC,
4140 				    OPCODE_NIC_VF_DRV_NOTICE,
4141 				    octeon_recv_vf_drv_notice, octeon_dev);
4142 	INIT_DELAYED_WORK(&octeon_dev->nic_poll_work.work, nic_starter);
4143 	octeon_dev->nic_poll_work.ctxptr = (void *)octeon_dev;
4144 	schedule_delayed_work(&octeon_dev->nic_poll_work.work,
4145 			      LIQUIDIO_STARTER_POLL_INTERVAL_MS);
4146 
4147 	atomic_set(&octeon_dev->status, OCT_DEV_DISPATCH_INIT_DONE);
4148 
4149 	if (octeon_set_io_queues_off(octeon_dev)) {
4150 		dev_err(&octeon_dev->pci_dev->dev, "setting io queues off failed\n");
4151 		return 1;
4152 	}
4153 
4154 	if (OCTEON_CN23XX_PF(octeon_dev)) {
4155 		ret = octeon_dev->fn_list.setup_device_regs(octeon_dev);
4156 		if (ret) {
4157 			dev_err(&octeon_dev->pci_dev->dev, "OCTEON: Failed to configure device registers\n");
4158 			return ret;
4159 		}
4160 	}
4161 
4162 	/* Initialize soft command buffer pool
4163 	 */
4164 	if (octeon_setup_sc_buffer_pool(octeon_dev)) {
4165 		dev_err(&octeon_dev->pci_dev->dev, "sc buffer pool allocation failed\n");
4166 		return 1;
4167 	}
4168 	atomic_set(&octeon_dev->status, OCT_DEV_SC_BUFF_POOL_INIT_DONE);
4169 
4170 	/*  Setup the data structures that manage this Octeon's Input queues. */
4171 	if (octeon_setup_instr_queues(octeon_dev)) {
4172 		dev_err(&octeon_dev->pci_dev->dev,
4173 			"instruction queue initialization failed\n");
4174 		return 1;
4175 	}
4176 	atomic_set(&octeon_dev->status, OCT_DEV_INSTR_QUEUE_INIT_DONE);
4177 
4178 	/* Initialize lists to manage the requests of different types that
4179 	 * arrive from user & kernel applications for this octeon device.
4180 	 */
4181 	if (octeon_setup_response_list(octeon_dev)) {
4182 		dev_err(&octeon_dev->pci_dev->dev, "Response list allocation failed\n");
4183 		return 1;
4184 	}
4185 	atomic_set(&octeon_dev->status, OCT_DEV_RESP_LIST_INIT_DONE);
4186 
4187 	if (octeon_setup_output_queues(octeon_dev)) {
4188 		dev_err(&octeon_dev->pci_dev->dev, "Output queue initialization failed\n");
4189 		return 1;
4190 	}
4191 
4192 	atomic_set(&octeon_dev->status, OCT_DEV_DROQ_INIT_DONE);
4193 
4194 	if (OCTEON_CN23XX_PF(octeon_dev)) {
4195 		if (octeon_dev->fn_list.setup_mbox(octeon_dev)) {
4196 			dev_err(&octeon_dev->pci_dev->dev, "OCTEON: Mailbox setup failed\n");
4197 			return 1;
4198 		}
4199 		atomic_set(&octeon_dev->status, OCT_DEV_MBOX_SETUP_DONE);
4200 
4201 		if (octeon_allocate_ioq_vector
4202 				(octeon_dev,
4203 				 octeon_dev->sriov_info.num_pf_rings)) {
4204 			dev_err(&octeon_dev->pci_dev->dev, "OCTEON: ioq vector allocation failed\n");
4205 			return 1;
4206 		}
4207 		atomic_set(&octeon_dev->status, OCT_DEV_MSIX_ALLOC_VECTOR_DONE);
4208 
4209 	} else {
4210 		/* The input and output queue registers were setup earlier (the
4211 		 * queues were not enabled). Any additional registers
4212 		 * that need to be programmed should be done now.
4213 		 */
4214 		ret = octeon_dev->fn_list.setup_device_regs(octeon_dev);
4215 		if (ret) {
4216 			dev_err(&octeon_dev->pci_dev->dev,
4217 				"Failed to configure device registers\n");
4218 			return ret;
4219 		}
4220 	}
4221 
4222 	/* Initialize the tasklet that handles output queue packet processing.*/
4223 	dev_dbg(&octeon_dev->pci_dev->dev, "Initializing droq tasklet\n");
4224 	tasklet_init(&oct_priv->droq_tasklet, octeon_droq_bh,
4225 		     (unsigned long)octeon_dev);
4226 
4227 	/* Setup the interrupt handler and record the INT SUM register address
4228 	 */
4229 	if (octeon_setup_interrupt(octeon_dev,
4230 				   octeon_dev->sriov_info.num_pf_rings))
4231 		return 1;
4232 
4233 	/* Enable Octeon device interrupts */
4234 	octeon_dev->fn_list.enable_interrupt(octeon_dev, OCTEON_ALL_INTR);
4235 
4236 	atomic_set(&octeon_dev->status, OCT_DEV_INTR_SET_DONE);
4237 
4238 	/* Send Credit for Octeon Output queues. Credits are always sent BEFORE
4239 	 * the output queue is enabled.
4240 	 * This ensures that we'll receive the f/w CORE DRV_ACTIVE message in
4241 	 * case we've configured CN23XX_SLI_GBL_CONTROL[NOPTR_D] = 0.
4242 	 * Otherwise, it is possible that the DRV_ACTIVE message will be sent
4243 	 * before any credits have been issued, causing the ring to be reset
4244 	 * (and the f/w appear to never have started).
4245 	 */
4246 	for (j = 0; j < octeon_dev->num_oqs; j++)
4247 		writel(octeon_dev->droq[j]->max_count,
4248 		       octeon_dev->droq[j]->pkts_credit_reg);
4249 
4250 	/* Enable the input and output queues for this Octeon device */
4251 	ret = octeon_dev->fn_list.enable_io_queues(octeon_dev);
4252 	if (ret) {
4253 		dev_err(&octeon_dev->pci_dev->dev, "Failed to enable input/output queues");
4254 		return ret;
4255 	}
4256 
4257 	atomic_set(&octeon_dev->status, OCT_DEV_IO_QUEUES_DONE);
4258 
4259 	if (fw_state == FW_NEEDS_TO_BE_LOADED) {
4260 		dev_dbg(&octeon_dev->pci_dev->dev, "Waiting for DDR initialization...\n");
4261 		if (!ddr_timeout) {
4262 			dev_info(&octeon_dev->pci_dev->dev,
4263 				 "WAITING. Set ddr_timeout to non-zero value to proceed with initialization.\n");
4264 		}
4265 
4266 		schedule_timeout_uninterruptible(HZ * LIO_RESET_SECS);
4267 
4268 		/* Wait for the octeon to initialize DDR after the soft-reset.*/
4269 		while (!ddr_timeout) {
4270 			set_current_state(TASK_INTERRUPTIBLE);
4271 			if (schedule_timeout(HZ / 10)) {
4272 				/* user probably pressed Control-C */
4273 				return 1;
4274 			}
4275 		}
4276 		ret = octeon_wait_for_ddr_init(octeon_dev, &ddr_timeout);
4277 		if (ret) {
4278 			dev_err(&octeon_dev->pci_dev->dev,
4279 				"DDR not initialized. Please confirm that board is configured to boot from Flash, ret: %d\n",
4280 				ret);
4281 			return 1;
4282 		}
4283 
4284 		if (octeon_wait_for_bootloader(octeon_dev, 1000)) {
4285 			dev_err(&octeon_dev->pci_dev->dev, "Board not responding\n");
4286 			return 1;
4287 		}
4288 
4289 		/* Divert uboot to take commands from host instead. */
4290 		ret = octeon_console_send_cmd(octeon_dev, bootcmd, 50);
4291 
4292 		dev_dbg(&octeon_dev->pci_dev->dev, "Initializing consoles\n");
4293 		ret = octeon_init_consoles(octeon_dev);
4294 		if (ret) {
4295 			dev_err(&octeon_dev->pci_dev->dev, "Could not access board consoles\n");
4296 			return 1;
4297 		}
4298 		/* If console debug enabled, specify empty string to use default
4299 		 * enablement ELSE specify NULL string for 'disabled'.
4300 		 */
4301 		dbg_enb = octeon_console_debug_enabled(0) ? "" : NULL;
4302 		ret = octeon_add_console(octeon_dev, 0, dbg_enb);
4303 		if (ret) {
4304 			dev_err(&octeon_dev->pci_dev->dev, "Could not access board console\n");
4305 			return 1;
4306 		} else if (octeon_console_debug_enabled(0)) {
4307 			/* If console was added AND we're logging console output
4308 			 * then set our console print function.
4309 			 */
4310 			octeon_dev->console[0].print = octeon_dbg_console_print;
4311 		}
4312 
4313 		atomic_set(&octeon_dev->status, OCT_DEV_CONSOLE_INIT_DONE);
4314 
4315 		dev_dbg(&octeon_dev->pci_dev->dev, "Loading firmware\n");
4316 		ret = load_firmware(octeon_dev);
4317 		if (ret) {
4318 			dev_err(&octeon_dev->pci_dev->dev, "Could not load firmware to board\n");
4319 			return 1;
4320 		}
4321 
4322 		atomic_set(octeon_dev->adapter_fw_state, FW_HAS_BEEN_LOADED);
4323 	}
4324 
4325 	handshake[octeon_dev->octeon_id].init_ok = 1;
4326 	complete(&handshake[octeon_dev->octeon_id].init);
4327 
4328 	atomic_set(&octeon_dev->status, OCT_DEV_HOST_OK);
4329 
4330 	return 0;
4331 }
4332 
4333 /**
4334  * \brief Debug console print function
4335  * @param octeon_dev  octeon device
4336  * @param console_num console number
4337  * @param prefix      first portion of line to display
4338  * @param suffix      second portion of line to display
4339  *
4340  * The OCTEON debug console outputs entire lines (excluding '\n').
4341  * Normally, the line will be passed in the 'prefix' parameter.
4342  * However, due to buffering, it is possible for a line to be split into two
4343  * parts, in which case they will be passed as the 'prefix' parameter and
4344  * 'suffix' parameter.
4345  */
4346 static int octeon_dbg_console_print(struct octeon_device *oct, u32 console_num,
4347 				    char *prefix, char *suffix)
4348 {
4349 	if (prefix && suffix)
4350 		dev_info(&oct->pci_dev->dev, "%u: %s%s\n", console_num, prefix,
4351 			 suffix);
4352 	else if (prefix)
4353 		dev_info(&oct->pci_dev->dev, "%u: %s\n", console_num, prefix);
4354 	else if (suffix)
4355 		dev_info(&oct->pci_dev->dev, "%u: %s\n", console_num, suffix);
4356 
4357 	return 0;
4358 }
4359 
4360 /**
4361  * \brief Exits the module
4362  */
4363 static void __exit liquidio_exit(void)
4364 {
4365 	liquidio_deinit_pci();
4366 
4367 	pr_info("LiquidIO network module is now unloaded\n");
4368 }
4369 
4370 module_init(liquidio_init);
4371 module_exit(liquidio_exit);
4372