1 /* 2 * BSD LICENSE 3 * 4 * Copyright(c) 2017 Cavium, Inc.. All rights reserved. 5 * All rights reserved. 6 * 7 * Redistribution and use in source and binary forms, with or without 8 * modification, are permitted provided that the following conditions 9 * are met: 10 * 11 * * Redistributions of source code must retain the above copyright 12 * notice, this list of conditions and the following disclaimer. 13 * * Redistributions in binary form must reproduce the above copyright 14 * notice, this list of conditions and the following disclaimer in 15 * the documentation and/or other materials provided with the 16 * distribution. 17 * * Neither the name of Cavium, Inc. nor the names of its 18 * contributors may be used to endorse or promote products derived 19 * from this software without specific prior written permission. 20 * 21 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 22 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 23 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 24 * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 25 * OWNER(S) OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 26 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 27 * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 28 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 29 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 30 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 31 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 32 */ 33 /*$FreeBSD$*/ 34 35 #include "lio_bsd.h" 36 #include "lio_common.h" 37 #include "lio_droq.h" 38 #include "lio_iq.h" 39 #include "lio_response_manager.h" 40 #include "lio_device.h" 41 #include "lio_main.h" 42 #include "lio_network.h" 43 #include "cn23xx_pf_device.h" 44 #include "lio_image.h" 45 #include "lio_mem_ops.h" 46 47 static struct lio_config default_cn23xx_conf = { 48 .card_type = LIO_23XX, 49 .card_name = LIO_23XX_NAME, 50 /* IQ attributes */ 51 .iq = { 52 .max_iqs = LIO_CN23XX_CFG_IO_QUEUES, 53 .pending_list_size = (LIO_CN23XX_DEFAULT_IQ_DESCRIPTORS * 54 LIO_CN23XX_CFG_IO_QUEUES), 55 .instr_type = LIO_64BYTE_INSTR, 56 .db_min = LIO_CN23XX_DB_MIN, 57 .db_timeout = LIO_CN23XX_DB_TIMEOUT, 58 .iq_intr_pkt = LIO_CN23XX_DEF_IQ_INTR_THRESHOLD, 59 }, 60 61 /* OQ attributes */ 62 .oq = { 63 .max_oqs = LIO_CN23XX_CFG_IO_QUEUES, 64 .pkts_per_intr = LIO_CN23XX_OQ_PKTS_PER_INTR, 65 .refill_threshold = LIO_CN23XX_OQ_REFIL_THRESHOLD, 66 .oq_intr_pkt = LIO_CN23XX_OQ_INTR_PKT, 67 .oq_intr_time = LIO_CN23XX_OQ_INTR_TIME, 68 }, 69 70 .num_nic_ports = LIO_CN23XX_DEFAULT_NUM_PORTS, 71 .num_def_rx_descs = LIO_CN23XX_DEFAULT_OQ_DESCRIPTORS, 72 .num_def_tx_descs = LIO_CN23XX_DEFAULT_IQ_DESCRIPTORS, 73 .def_rx_buf_size = LIO_CN23XX_OQ_BUF_SIZE, 74 75 /* For ethernet interface 0: Port cfg Attributes */ 76 .nic_if_cfg[0] = { 77 /* Max Txqs: Half for each of the two ports :max_iq/2 */ 78 .max_txqs = LIO_MAX_TXQS_PER_INTF, 79 80 /* Actual configured value. Range could be: 1...max_txqs */ 81 .num_txqs = LIO_DEF_TXQS_PER_INTF, 82 83 /* Max Rxqs: Half for each of the two ports :max_oq/2 */ 84 .max_rxqs = LIO_MAX_RXQS_PER_INTF, 85 86 /* Actual configured value. Range could be: 1...max_rxqs */ 87 .num_rxqs = LIO_DEF_RXQS_PER_INTF, 88 89 /* Num of desc for rx rings */ 90 .num_rx_descs = LIO_CN23XX_DEFAULT_OQ_DESCRIPTORS, 91 92 /* Num of desc for tx rings */ 93 .num_tx_descs = LIO_CN23XX_DEFAULT_IQ_DESCRIPTORS, 94 95 /* 96 * Mbuf size, We need not change buf size even for Jumbo frames. 97 * Octeon can send jumbo frames in 4 consecutive descriptors, 98 */ 99 .rx_buf_size = LIO_CN23XX_OQ_BUF_SIZE, 100 101 .base_queue = LIO_BASE_QUEUE_NOT_REQUESTED, 102 103 .gmx_port_id = 0, 104 }, 105 106 .nic_if_cfg[1] = { 107 /* Max Txqs: Half for each of the two ports :max_iq/2 */ 108 .max_txqs = LIO_MAX_TXQS_PER_INTF, 109 110 /* Actual configured value. Range could be: 1...max_txqs */ 111 .num_txqs = LIO_DEF_TXQS_PER_INTF, 112 113 /* Max Rxqs: Half for each of the two ports :max_oq/2 */ 114 .max_rxqs = LIO_MAX_RXQS_PER_INTF, 115 116 /* Actual configured value. Range could be: 1...max_rxqs */ 117 .num_rxqs = LIO_DEF_RXQS_PER_INTF, 118 119 /* Num of desc for rx rings */ 120 .num_rx_descs = LIO_CN23XX_DEFAULT_OQ_DESCRIPTORS, 121 122 /* Num of desc for tx rings */ 123 .num_tx_descs = LIO_CN23XX_DEFAULT_IQ_DESCRIPTORS, 124 125 /* 126 * Mbuf size, We need not change buf size even for Jumbo frames. 127 * Octeon can send jumbo frames in 4 consecutive descriptors, 128 */ 129 .rx_buf_size = LIO_CN23XX_OQ_BUF_SIZE, 130 131 .base_queue = LIO_BASE_QUEUE_NOT_REQUESTED, 132 133 .gmx_port_id = 1, 134 }, 135 136 .misc = { 137 /* Host driver link query interval */ 138 .oct_link_query_interval = 100, 139 140 /* Octeon link query interval */ 141 .host_link_query_interval = 500, 142 143 .enable_sli_oq_bp = 0, 144 145 /* Control queue group */ 146 .ctrlq_grp = 1, 147 } 148 }; 149 150 static struct lio_config_ptr { 151 uint32_t conf_type; 152 } oct_conf_info[LIO_MAX_DEVICES] = { 153 154 { 155 LIO_CFG_TYPE_DEFAULT, 156 }, { 157 LIO_CFG_TYPE_DEFAULT, 158 }, { 159 LIO_CFG_TYPE_DEFAULT, 160 }, { 161 LIO_CFG_TYPE_DEFAULT, 162 }, 163 }; 164 165 static char lio_state_str[LIO_DEV_STATES + 1][32] = { 166 "BEGIN", "PCI-ENABLE-DONE", "PCI-MAP-DONE", "DISPATCH-INIT-DONE", 167 "IQ-INIT-DONE", "SCBUFF-POOL-INIT-DONE", "RESPLIST-INIT-DONE", 168 "DROQ-INIT-DONE", "MBOX-SETUP-DONE", "MSIX-ALLOC-VECTOR-DONE", 169 "INTR-SET-DONE", "IO-QUEUES-INIT-DONE", "CONSOLE-INIT-DONE", 170 "HOST-READY", "CORE-READY", "RUNNING", "IN-RESET", 171 "INVALID" 172 }; 173 174 static char lio_app_str[LIO_DRV_APP_COUNT + 1][32] = {"BASE", "NIC", "UNKNOWN"}; 175 176 static struct octeon_device *octeon_device[LIO_MAX_DEVICES]; 177 static volatile int lio_adapter_refcounts[LIO_MAX_DEVICES]; 178 179 static uint32_t octeon_device_count; 180 /* locks device array (i.e. octeon_device[]) */ 181 struct mtx octeon_devices_lock; 182 183 static struct lio_core_setup core_setup[LIO_MAX_DEVICES]; 184 185 static void 186 oct_set_config_info(int oct_id, int conf_type) 187 { 188 189 if (conf_type < 0 || conf_type > (LIO_NUM_CFGS - 1)) 190 conf_type = LIO_CFG_TYPE_DEFAULT; 191 oct_conf_info[oct_id].conf_type = conf_type; 192 } 193 194 void 195 lio_init_device_list(int conf_type) 196 { 197 int i; 198 199 bzero(octeon_device, (sizeof(void *) * LIO_MAX_DEVICES)); 200 for (i = 0; i < LIO_MAX_DEVICES; i++) 201 oct_set_config_info(i, conf_type); 202 mtx_init(&octeon_devices_lock, "octeon_devices_lock", NULL, MTX_DEF); 203 } 204 205 static void * 206 __lio_retrieve_config_info(struct octeon_device *oct, uint16_t card_type) 207 { 208 void *ret = NULL; 209 uint32_t oct_id = oct->octeon_id; 210 211 switch (oct_conf_info[oct_id].conf_type) { 212 case LIO_CFG_TYPE_DEFAULT: 213 if (oct->chip_id == LIO_CN23XX_PF_VID) { 214 ret = &default_cn23xx_conf; 215 } 216 217 break; 218 default: 219 break; 220 } 221 return (ret); 222 } 223 224 void * 225 lio_get_config_info(struct octeon_device *oct, uint16_t card_type) 226 { 227 void *conf = NULL; 228 229 conf = __lio_retrieve_config_info(oct, card_type); 230 if (conf == NULL) 231 return (NULL); 232 233 return (conf); 234 } 235 236 char * 237 lio_get_state_string(volatile int *state_ptr) 238 { 239 int32_t istate = (int32_t)atomic_load_acq_int(state_ptr); 240 241 if (istate > LIO_DEV_STATES || istate < 0) 242 return (lio_state_str[LIO_DEV_STATE_INVALID]); 243 244 return (lio_state_str[istate]); 245 } 246 247 static char * 248 lio_get_app_string(uint32_t app_mode) 249 { 250 251 if (app_mode <= LIO_DRV_APP_END) 252 return (lio_app_str[app_mode - LIO_DRV_APP_START]); 253 254 return (lio_app_str[LIO_DRV_INVALID_APP - LIO_DRV_APP_START]); 255 } 256 257 void 258 lio_free_device_mem(struct octeon_device *oct) 259 { 260 int i; 261 262 for (i = 0; i < LIO_MAX_OUTPUT_QUEUES(oct); i++) { 263 if ((oct->io_qmask.oq & BIT_ULL(i)) && (oct->droq[i])) 264 free(oct->droq[i], M_DEVBUF); 265 } 266 267 for (i = 0; i < LIO_MAX_INSTR_QUEUES(oct); i++) { 268 if ((oct->io_qmask.iq & BIT_ULL(i)) && (oct->instr_queue[i])) 269 free(oct->instr_queue[i], M_DEVBUF); 270 } 271 272 i = oct->octeon_id; 273 free(oct->chip, M_DEVBUF); 274 275 octeon_device[i] = NULL; 276 octeon_device_count--; 277 } 278 279 static struct octeon_device * 280 lio_allocate_device_mem(device_t device) 281 { 282 struct octeon_device *oct; 283 uint32_t configsize = 0, pci_id = 0, size; 284 uint8_t *buf = NULL; 285 286 pci_id = pci_get_device(device); 287 switch (pci_id) { 288 case LIO_CN23XX_PF_VID: 289 configsize = sizeof(struct lio_cn23xx_pf); 290 break; 291 default: 292 device_printf(device, "Error: Unknown PCI Device: 0x%x\n", 293 pci_id); 294 return (NULL); 295 } 296 297 if (configsize & 0x7) 298 configsize += (8 - (configsize & 0x7)); 299 300 size = configsize + 301 (sizeof(struct lio_dispatch) * LIO_DISPATCH_LIST_SIZE); 302 303 buf = malloc(size, M_DEVBUF, M_NOWAIT | M_ZERO); 304 if (buf == NULL) 305 return (NULL); 306 307 oct = (struct octeon_device *)device_get_softc(device); 308 oct->chip = (void *)(buf); 309 oct->dispatch.dlist = (struct lio_dispatch *)(buf + configsize); 310 311 return (oct); 312 } 313 314 struct octeon_device * 315 lio_allocate_device(device_t device) 316 { 317 struct octeon_device *oct = NULL; 318 uint32_t oct_idx = 0; 319 320 mtx_lock(&octeon_devices_lock); 321 322 for (oct_idx = 0; oct_idx < LIO_MAX_DEVICES; oct_idx++) 323 if (!octeon_device[oct_idx]) 324 break; 325 326 if (oct_idx < LIO_MAX_DEVICES) { 327 oct = lio_allocate_device_mem(device); 328 if (oct != NULL) { 329 octeon_device_count++; 330 octeon_device[oct_idx] = oct; 331 } 332 } 333 334 mtx_unlock(&octeon_devices_lock); 335 336 if (oct == NULL) 337 return (NULL); 338 339 mtx_init(&oct->pci_win_lock, "pci_win_lock", NULL, MTX_DEF); 340 mtx_init(&oct->mem_access_lock, "mem_access_lock", NULL, MTX_DEF); 341 342 oct->octeon_id = oct_idx; 343 snprintf(oct->device_name, sizeof(oct->device_name), "%s%d", 344 LIO_DRV_NAME, oct->octeon_id); 345 346 return (oct); 347 } 348 349 /* 350 * Register a device's bus location at initialization time. 351 * @param oct - pointer to the octeon device structure. 352 * @param bus - PCIe bus # 353 * @param dev - PCIe device # 354 * @param func - PCIe function # 355 * @param is_pf - TRUE for PF, FALSE for VF 356 * @return reference count of device's adapter 357 */ 358 int 359 lio_register_device(struct octeon_device *oct, int bus, int dev, int func, 360 int is_pf) 361 { 362 int idx, refcount; 363 364 oct->loc.bus = bus; 365 oct->loc.dev = dev; 366 oct->loc.func = func; 367 368 oct->adapter_refcount = &lio_adapter_refcounts[oct->octeon_id]; 369 atomic_store_rel_int(oct->adapter_refcount, 0); 370 371 mtx_lock(&octeon_devices_lock); 372 for (idx = (int)oct->octeon_id - 1; idx >= 0; idx--) { 373 if (octeon_device[idx] == NULL) { 374 lio_dev_err(oct, "%s: Internal driver error, missing dev\n", 375 __func__); 376 mtx_unlock(&octeon_devices_lock); 377 atomic_add_int(oct->adapter_refcount, 1); 378 return (1); /* here, refcount is guaranteed to be 1 */ 379 } 380 381 /* if another device is at same bus/dev, use its refcounter */ 382 if ((octeon_device[idx]->loc.bus == bus) && 383 (octeon_device[idx]->loc.dev == dev)) { 384 oct->adapter_refcount = 385 octeon_device[idx]->adapter_refcount; 386 break; 387 } 388 } 389 390 mtx_unlock(&octeon_devices_lock); 391 392 atomic_add_int(oct->adapter_refcount, 1); 393 refcount = atomic_load_acq_int(oct->adapter_refcount); 394 395 lio_dev_dbg(oct, "%s: %02x:%02x:%d refcount %u\n", __func__, 396 oct->loc.bus, oct->loc.dev, oct->loc.func, refcount); 397 398 return (refcount); 399 } 400 401 /* 402 * Deregister a device at de-initialization time. 403 * @param oct - pointer to the octeon device structure. 404 * @return reference count of device's adapter 405 */ 406 int 407 lio_deregister_device(struct octeon_device *oct) 408 { 409 int refcount; 410 411 atomic_subtract_int(oct->adapter_refcount, 1); 412 refcount = atomic_load_acq_int(oct->adapter_refcount); 413 414 lio_dev_dbg(oct, "%s: %04d:%02d:%d refcount %u\n", __func__, 415 oct->loc.bus, oct->loc.dev, oct->loc.func, refcount); 416 417 return (refcount); 418 } 419 420 int 421 lio_allocate_ioq_vector(struct octeon_device *oct) 422 { 423 struct lio_ioq_vector *ioq_vector; 424 int i, cpu_num, num_ioqs = 0, size; 425 426 if (LIO_CN23XX_PF(oct)) 427 num_ioqs = oct->sriov_info.num_pf_rings; 428 429 size = sizeof(struct lio_ioq_vector) * num_ioqs; 430 431 oct->ioq_vector = malloc(size, M_DEVBUF, M_NOWAIT | M_ZERO); 432 if (oct->ioq_vector == NULL) 433 return (1); 434 435 for (i = 0; i < num_ioqs; i++) { 436 ioq_vector = &oct->ioq_vector[i]; 437 ioq_vector->oct_dev = oct; 438 ioq_vector->droq_index = i; 439 cpu_num = i % mp_ncpus; 440 CPU_SETOF(cpu_num, &ioq_vector->affinity_mask); 441 442 if (oct->chip_id == LIO_CN23XX_PF_VID) 443 ioq_vector->ioq_num = i + oct->sriov_info.pf_srn; 444 else 445 ioq_vector->ioq_num = i; 446 } 447 return (0); 448 } 449 450 void 451 lio_free_ioq_vector(struct octeon_device *oct) 452 { 453 454 free(oct->ioq_vector, M_DEVBUF); 455 oct->ioq_vector = NULL; 456 } 457 458 /* this function is only for setting up the first queue */ 459 int 460 lio_setup_instr_queue0(struct octeon_device *oct) 461 { 462 union octeon_txpciq txpciq; 463 uint32_t iq_no = 0; 464 uint32_t num_descs = 0; 465 466 if (LIO_CN23XX_PF(oct)) 467 num_descs = 468 LIO_GET_NUM_DEF_TX_DESCS_CFG(LIO_CHIP_CONF(oct, 469 cn23xx_pf)); 470 471 oct->num_iqs = 0; 472 473 oct->instr_queue[0]->q_index = 0; 474 oct->instr_queue[0]->app_ctx = (void *)(size_t)0; 475 oct->instr_queue[0]->ifidx = 0; 476 txpciq.txpciq64 = 0; 477 txpciq.s.q_no = iq_no; 478 txpciq.s.pkind = oct->pfvf_hsword.pkind; 479 txpciq.s.use_qpg = 0; 480 txpciq.s.qpg = 0; 481 if (lio_init_instr_queue(oct, txpciq, num_descs)) { 482 /* prevent memory leak */ 483 lio_delete_instr_queue(oct, 0); 484 return (1); 485 } 486 487 oct->num_iqs++; 488 return (0); 489 } 490 491 int 492 lio_setup_output_queue0(struct octeon_device *oct) 493 { 494 uint32_t desc_size = 0, num_descs = 0, oq_no = 0; 495 496 if (LIO_CN23XX_PF(oct)) { 497 num_descs = 498 LIO_GET_NUM_DEF_RX_DESCS_CFG(LIO_CHIP_CONF(oct, 499 cn23xx_pf)); 500 desc_size = 501 LIO_GET_DEF_RX_BUF_SIZE_CFG(LIO_CHIP_CONF(oct, 502 cn23xx_pf)); 503 } 504 505 oct->num_oqs = 0; 506 507 if (lio_init_droq(oct, oq_no, num_descs, desc_size, NULL)) { 508 return (1); 509 } 510 511 oct->num_oqs++; 512 513 return (0); 514 } 515 516 int 517 lio_init_dispatch_list(struct octeon_device *oct) 518 { 519 uint32_t i; 520 521 oct->dispatch.count = 0; 522 523 for (i = 0; i < LIO_DISPATCH_LIST_SIZE; i++) { 524 oct->dispatch.dlist[i].opcode = 0; 525 STAILQ_INIT(&oct->dispatch.dlist[i].head); 526 } 527 528 mtx_init(&oct->dispatch.lock, "dispatch_lock", NULL, MTX_DEF); 529 530 return (0); 531 } 532 533 void 534 lio_delete_dispatch_list(struct octeon_device *oct) 535 { 536 struct lio_stailq_head freelist; 537 struct lio_stailq_node *temp, *tmp2; 538 uint32_t i; 539 540 STAILQ_INIT(&freelist); 541 542 mtx_lock(&oct->dispatch.lock); 543 544 for (i = 0; i < LIO_DISPATCH_LIST_SIZE; i++) { 545 struct lio_stailq_head *dispatch; 546 547 dispatch = &oct->dispatch.dlist[i].head; 548 while (!STAILQ_EMPTY(dispatch)) { 549 temp = STAILQ_FIRST(dispatch); 550 STAILQ_REMOVE_HEAD(&oct->dispatch.dlist[i].head, 551 entries); 552 STAILQ_INSERT_TAIL(&freelist, temp, entries); 553 } 554 555 oct->dispatch.dlist[i].opcode = 0; 556 } 557 558 oct->dispatch.count = 0; 559 560 mtx_unlock(&oct->dispatch.lock); 561 562 STAILQ_FOREACH_SAFE(temp, &freelist, entries, tmp2) { 563 STAILQ_REMOVE_HEAD(&freelist, entries); 564 free(temp, M_DEVBUF); 565 } 566 } 567 568 lio_dispatch_fn_t 569 lio_get_dispatch(struct octeon_device *octeon_dev, uint16_t opcode, 570 uint16_t subcode) 571 { 572 struct lio_stailq_node *dispatch; 573 lio_dispatch_fn_t fn = NULL; 574 uint32_t idx; 575 uint16_t combined_opcode = LIO_OPCODE_SUBCODE(opcode, subcode); 576 577 idx = combined_opcode & LIO_OPCODE_MASK; 578 579 mtx_lock(&octeon_dev->dispatch.lock); 580 581 if (octeon_dev->dispatch.count == 0) { 582 mtx_unlock(&octeon_dev->dispatch.lock); 583 return (NULL); 584 } 585 586 if (!(octeon_dev->dispatch.dlist[idx].opcode)) { 587 mtx_unlock(&octeon_dev->dispatch.lock); 588 return (NULL); 589 } 590 591 if (octeon_dev->dispatch.dlist[idx].opcode == combined_opcode) { 592 fn = octeon_dev->dispatch.dlist[idx].dispatch_fn; 593 } else { 594 STAILQ_FOREACH(dispatch, &octeon_dev->dispatch.dlist[idx].head, 595 entries) { 596 if (((struct lio_dispatch *)dispatch)->opcode == 597 combined_opcode) { 598 fn = ((struct lio_dispatch *) 599 dispatch)->dispatch_fn; 600 break; 601 } 602 } 603 } 604 605 mtx_unlock(&octeon_dev->dispatch.lock); 606 return (fn); 607 } 608 609 /* 610 * lio_register_dispatch_fn 611 * Parameters: 612 * octeon_id - id of the octeon device. 613 * opcode - opcode for which driver should call the registered function 614 * subcode - subcode for which driver should call the registered function 615 * fn - The function to call when a packet with "opcode" arrives in 616 * octeon output queues. 617 * fn_arg - The argument to be passed when calling function "fn". 618 * Description: 619 * Registers a function and its argument to be called when a packet 620 * arrives in Octeon output queues with "opcode". 621 * Returns: 622 * Success: 0 623 * Failure: 1 624 * Locks: 625 * No locks are held. 626 */ 627 int 628 lio_register_dispatch_fn(struct octeon_device *oct, uint16_t opcode, 629 uint16_t subcode, lio_dispatch_fn_t fn, void *fn_arg) 630 { 631 lio_dispatch_fn_t pfn; 632 uint32_t idx; 633 uint16_t combined_opcode = LIO_OPCODE_SUBCODE(opcode, subcode); 634 635 idx = combined_opcode & LIO_OPCODE_MASK; 636 637 mtx_lock(&oct->dispatch.lock); 638 /* Add dispatch function to first level of lookup table */ 639 if (oct->dispatch.dlist[idx].opcode == 0) { 640 oct->dispatch.dlist[idx].opcode = combined_opcode; 641 oct->dispatch.dlist[idx].dispatch_fn = fn; 642 oct->dispatch.dlist[idx].arg = fn_arg; 643 oct->dispatch.count++; 644 mtx_unlock(&oct->dispatch.lock); 645 return (0); 646 } 647 648 mtx_unlock(&oct->dispatch.lock); 649 650 /* 651 * Check if there was a function already registered for this 652 * opcode/subcode. 653 */ 654 pfn = lio_get_dispatch(oct, opcode, subcode); 655 if (!pfn) { 656 struct lio_dispatch *dispatch; 657 658 lio_dev_dbg(oct, 659 "Adding opcode to dispatch list linked list\n"); 660 dispatch = (struct lio_dispatch *) 661 malloc(sizeof(struct lio_dispatch), 662 M_DEVBUF, M_NOWAIT | M_ZERO); 663 if (dispatch == NULL) { 664 lio_dev_err(oct, 665 "No memory to add dispatch function\n"); 666 return (1); 667 } 668 669 dispatch->opcode = combined_opcode; 670 dispatch->dispatch_fn = fn; 671 dispatch->arg = fn_arg; 672 673 /* 674 * Add dispatch function to linked list of fn ptrs 675 * at the hashed index. 676 */ 677 mtx_lock(&oct->dispatch.lock); 678 STAILQ_INSERT_HEAD(&oct->dispatch.dlist[idx].head, 679 &dispatch->node, entries); 680 oct->dispatch.count++; 681 mtx_unlock(&oct->dispatch.lock); 682 683 } else { 684 lio_dev_err(oct, "Found previously registered dispatch fn for opcode/subcode: %x/%x\n", 685 opcode, subcode); 686 return (1); 687 } 688 689 return (0); 690 } 691 692 /* 693 * lio_unregister_dispatch_fn 694 * Parameters: 695 * oct - octeon device 696 * opcode - driver should unregister the function for this opcode 697 * subcode - driver should unregister the function for this subcode 698 * Description: 699 * Unregister the function set for this opcode+subcode. 700 * Returns: 701 * Success: 0 702 * Failure: 1 703 * Locks: 704 * No locks are held. 705 */ 706 int 707 lio_unregister_dispatch_fn(struct octeon_device *oct, uint16_t opcode, 708 uint16_t subcode) 709 { 710 struct lio_stailq_head *dispatch_head; 711 struct lio_stailq_node *dispatch, *dfree = NULL, *tmp2; 712 int retval = 0; 713 uint32_t idx; 714 uint16_t combined_opcode = LIO_OPCODE_SUBCODE(opcode, subcode); 715 716 idx = combined_opcode & LIO_OPCODE_MASK; 717 718 mtx_lock(&oct->dispatch.lock); 719 720 if (oct->dispatch.count == 0) { 721 mtx_unlock(&oct->dispatch.lock); 722 lio_dev_err(oct, "No dispatch functions registered for this device\n"); 723 return (1); 724 } 725 if (oct->dispatch.dlist[idx].opcode == combined_opcode) { 726 dispatch_head = &oct->dispatch.dlist[idx].head; 727 if (!STAILQ_EMPTY(dispatch_head)) { 728 dispatch = STAILQ_FIRST(dispatch_head); 729 oct->dispatch.dlist[idx].opcode = 730 ((struct lio_dispatch *)dispatch)->opcode; 731 oct->dispatch.dlist[idx].dispatch_fn = 732 ((struct lio_dispatch *)dispatch)->dispatch_fn; 733 oct->dispatch.dlist[idx].arg = 734 ((struct lio_dispatch *)dispatch)->arg; 735 STAILQ_REMOVE_HEAD(dispatch_head, entries); 736 dfree = dispatch; 737 } else { 738 oct->dispatch.dlist[idx].opcode = 0; 739 oct->dispatch.dlist[idx].dispatch_fn = NULL; 740 oct->dispatch.dlist[idx].arg = NULL; 741 } 742 } else { 743 retval = 1; 744 STAILQ_FOREACH_SAFE(dispatch, 745 &oct->dispatch.dlist[idx].head, 746 entries, tmp2) { 747 if (((struct lio_dispatch *)dispatch)->opcode == 748 combined_opcode) { 749 STAILQ_REMOVE(&oct->dispatch.dlist[idx].head, 750 dispatch, 751 lio_stailq_node, entries); 752 dfree = dispatch; 753 retval = 0; 754 } 755 } 756 } 757 758 if (!retval) 759 oct->dispatch.count--; 760 761 mtx_unlock(&oct->dispatch.lock); 762 free(dfree, M_DEVBUF); 763 764 return (retval); 765 } 766 767 int 768 lio_core_drv_init(struct lio_recv_info *recv_info, void *buf) 769 { 770 struct octeon_device *oct = (struct octeon_device *)buf; 771 struct lio_recv_pkt *recv_pkt = recv_info->recv_pkt; 772 struct lio_core_setup *cs = NULL; 773 uint32_t i; 774 uint32_t num_nic_ports = 0; 775 char app_name[16]; 776 777 if (LIO_CN23XX_PF(oct)) 778 num_nic_ports = LIO_GET_NUM_NIC_PORTS_CFG( 779 LIO_CHIP_CONF(oct, cn23xx_pf)); 780 781 if (atomic_load_acq_int(&oct->status) >= LIO_DEV_RUNNING) { 782 lio_dev_err(oct, "Received CORE OK when device state is 0x%x\n", 783 atomic_load_acq_int(&oct->status)); 784 goto core_drv_init_err; 785 } 786 787 strncpy(app_name, 788 lio_get_app_string((uint32_t) 789 recv_pkt->rh.r_core_drv_init.app_mode), 790 sizeof(app_name) - 1); 791 oct->app_mode = (uint32_t)recv_pkt->rh.r_core_drv_init.app_mode; 792 if (recv_pkt->rh.r_core_drv_init.app_mode == LIO_DRV_NIC_APP) { 793 oct->fw_info.max_nic_ports = 794 (uint32_t)recv_pkt->rh.r_core_drv_init.max_nic_ports; 795 oct->fw_info.num_gmx_ports = 796 (uint32_t)recv_pkt->rh.r_core_drv_init.num_gmx_ports; 797 } 798 799 if (oct->fw_info.max_nic_ports < num_nic_ports) { 800 lio_dev_err(oct, "Config has more ports than firmware allows (%d > %d).\n", 801 num_nic_ports, oct->fw_info.max_nic_ports); 802 goto core_drv_init_err; 803 } 804 805 oct->fw_info.app_cap_flags = recv_pkt->rh.r_core_drv_init.app_cap_flags; 806 oct->fw_info.app_mode = (uint32_t)recv_pkt->rh.r_core_drv_init.app_mode; 807 oct->pfvf_hsword.app_mode = 808 (uint32_t)recv_pkt->rh.r_core_drv_init.app_mode; 809 810 oct->pfvf_hsword.pkind = recv_pkt->rh.r_core_drv_init.pkind; 811 812 for (i = 0; i < oct->num_iqs; i++) 813 oct->instr_queue[i]->txpciq.s.pkind = oct->pfvf_hsword.pkind; 814 815 atomic_store_rel_int(&oct->status, LIO_DEV_CORE_OK); 816 817 cs = &core_setup[oct->octeon_id]; 818 819 if (recv_pkt->buffer_size[0] != (sizeof(*cs) + LIO_DROQ_INFO_SIZE)) { 820 lio_dev_dbg(oct, "Core setup bytes expected %llu found %d\n", 821 LIO_CAST64(sizeof(*cs) + LIO_DROQ_INFO_SIZE), 822 recv_pkt->buffer_size[0]); 823 } 824 825 memcpy(cs, recv_pkt->buffer_ptr[0]->m_data + LIO_DROQ_INFO_SIZE, 826 sizeof(*cs)); 827 strncpy(oct->boardinfo.name, cs->boardname, LIO_BOARD_NAME); 828 strncpy(oct->boardinfo.serial_number, cs->board_serial_number, 829 LIO_SERIAL_NUM_LEN); 830 831 lio_swap_8B_data((uint64_t *)cs, (sizeof(*cs) >> 3)); 832 833 oct->boardinfo.major = cs->board_rev_major; 834 oct->boardinfo.minor = cs->board_rev_minor; 835 836 lio_dev_info(oct, "Running %s (%llu Hz)\n", app_name, 837 LIO_CAST64(cs->corefreq)); 838 839 core_drv_init_err: 840 for (i = 0; i < recv_pkt->buffer_count; i++) 841 lio_recv_buffer_free(recv_pkt->buffer_ptr[i]); 842 843 lio_free_recv_info(recv_info); 844 return (0); 845 } 846 847 int 848 lio_get_tx_qsize(struct octeon_device *oct, uint32_t q_no) 849 { 850 851 if ((oct != NULL) && (q_no < (uint32_t)LIO_MAX_INSTR_QUEUES(oct)) && 852 (oct->io_qmask.iq & BIT_ULL(q_no))) 853 return (oct->instr_queue[q_no]->max_count); 854 855 856 return (-1); 857 } 858 859 int 860 lio_get_rx_qsize(struct octeon_device *oct, uint32_t q_no) 861 { 862 863 if ((oct != NULL) && (q_no < (uint32_t)LIO_MAX_OUTPUT_QUEUES(oct)) && 864 (oct->io_qmask.oq & BIT_ULL(q_no))) 865 return (oct->droq[q_no]->max_count); 866 867 return (-1); 868 } 869 870 /* Returns the host firmware handshake OCTEON specific configuration */ 871 struct lio_config * 872 lio_get_conf(struct octeon_device *oct) 873 { 874 struct lio_config *default_oct_conf = NULL; 875 876 /* 877 * check the OCTEON Device model & return the corresponding octeon 878 * configuration 879 */ 880 if (LIO_CN23XX_PF(oct)) { 881 default_oct_conf = (struct lio_config *)( 882 LIO_CHIP_CONF(oct, cn23xx_pf)); 883 } 884 885 return (default_oct_conf); 886 } 887 888 /* 889 * Get the octeon device pointer. 890 * @param octeon_id - The id for which the octeon device pointer is required. 891 * @return Success: Octeon device pointer. 892 * @return Failure: NULL. 893 */ 894 struct octeon_device * 895 lio_get_device(uint32_t octeon_id) 896 { 897 898 if (octeon_id >= LIO_MAX_DEVICES) 899 return (NULL); 900 else 901 return (octeon_device[octeon_id]); 902 } 903 904 uint64_t 905 lio_pci_readq(struct octeon_device *oct, uint64_t addr) 906 { 907 uint64_t val64; 908 volatile uint32_t addrhi; 909 910 mtx_lock(&oct->pci_win_lock); 911 912 /* 913 * The windowed read happens when the LSB of the addr is written. 914 * So write MSB first 915 */ 916 addrhi = (addr >> 32); 917 if (oct->chip_id == LIO_CN23XX_PF_VID) 918 addrhi |= 0x00060000; 919 lio_write_csr32(oct, oct->reg_list.pci_win_rd_addr_hi, addrhi); 920 921 /* Read back to preserve ordering of writes */ 922 (void)lio_read_csr32(oct, oct->reg_list.pci_win_rd_addr_hi); 923 924 lio_write_csr32(oct, oct->reg_list.pci_win_rd_addr_lo, 925 addr & 0xffffffff); 926 (void)lio_read_csr32(oct, oct->reg_list.pci_win_rd_addr_lo); 927 928 val64 = lio_read_csr64(oct, oct->reg_list.pci_win_rd_data); 929 930 mtx_unlock(&oct->pci_win_lock); 931 932 return (val64); 933 } 934 935 void 936 lio_pci_writeq(struct octeon_device *oct, uint64_t val, uint64_t addr) 937 { 938 939 mtx_lock(&oct->pci_win_lock); 940 941 lio_write_csr64(oct, oct->reg_list.pci_win_wr_addr, addr); 942 943 /* The write happens when the LSB is written. So write MSB first. */ 944 lio_write_csr32(oct, oct->reg_list.pci_win_wr_data_hi, val >> 32); 945 /* Read the MSB to ensure ordering of writes. */ 946 (void)lio_read_csr32(oct, oct->reg_list.pci_win_wr_data_hi); 947 948 lio_write_csr32(oct, oct->reg_list.pci_win_wr_data_lo, 949 val & 0xffffffff); 950 951 mtx_unlock(&oct->pci_win_lock); 952 } 953 954 int 955 lio_mem_access_ok(struct octeon_device *oct) 956 { 957 uint64_t access_okay = 0; 958 uint64_t lmc0_reset_ctl; 959 960 /* Check to make sure a DDR interface is enabled */ 961 if (LIO_CN23XX_PF(oct)) { 962 lmc0_reset_ctl = lio_pci_readq(oct, LIO_CN23XX_LMC0_RESET_CTL); 963 access_okay = 964 (lmc0_reset_ctl & LIO_CN23XX_LMC0_RESET_CTL_DDR3RST_MASK); 965 } 966 967 return (access_okay ? 0 : 1); 968 } 969 970 int 971 lio_wait_for_ddr_init(struct octeon_device *oct, unsigned long *timeout) 972 { 973 int ret = 1; 974 uint32_t ms; 975 976 if (timeout == NULL) 977 return (ret); 978 979 for (ms = 0; ret && ((*timeout == 0) || (ms <= *timeout)); ms += 100) { 980 ret = lio_mem_access_ok(oct); 981 982 /* wait 100 ms */ 983 if (ret) 984 lio_sleep_timeout(100); 985 } 986 987 return (ret); 988 } 989 990 /* 991 * Get the octeon id assigned to the octeon device passed as argument. 992 * This function is exported to other modules. 993 * @param dev - octeon device pointer passed as a void *. 994 * @return octeon device id 995 */ 996 int 997 lio_get_device_id(void *dev) 998 { 999 struct octeon_device *octeon_dev = (struct octeon_device *)dev; 1000 uint32_t i; 1001 1002 for (i = 0; i < LIO_MAX_DEVICES; i++) 1003 if (octeon_device[i] == octeon_dev) 1004 return (octeon_dev->octeon_id); 1005 1006 return (-1); 1007 } 1008 1009 void 1010 lio_enable_irq(struct lio_droq *droq, struct lio_instr_queue *iq) 1011 { 1012 struct octeon_device *oct = NULL; 1013 uint64_t instr_cnt; 1014 uint32_t pkts_pend; 1015 1016 /* the whole thing needs to be atomic, ideally */ 1017 if (droq != NULL) { 1018 oct = droq->oct_dev; 1019 pkts_pend = atomic_load_acq_int(&droq->pkts_pending); 1020 mtx_lock(&droq->lock); 1021 lio_write_csr32(oct, droq->pkts_sent_reg, 1022 droq->pkt_count - pkts_pend); 1023 droq->pkt_count = pkts_pend; 1024 /* this write needs to be flushed before we release the lock */ 1025 __compiler_membar(); 1026 mtx_unlock(&droq->lock); 1027 } 1028 1029 if (iq != NULL) { 1030 oct = iq->oct_dev; 1031 mtx_lock(&iq->lock); 1032 lio_write_csr32(oct, iq->inst_cnt_reg, iq->pkt_in_done); 1033 iq->pkt_in_done = 0; 1034 /* this write needs to be flushed before we release the lock */ 1035 __compiler_membar(); 1036 mtx_unlock(&iq->lock); 1037 } 1038 1039 /* 1040 * Implementation note: 1041 * 1042 * SLI_PKT(x)_CNTS[RESEND] is written separately so that if an interrupt 1043 * DOES occur as a result of RESEND, the DROQ lock will NOT be held. 1044 * 1045 * Write resend. Writing RESEND in SLI_PKTX_CNTS should be enough 1046 * to trigger tx interrupts as well, if they are pending. 1047 */ 1048 if ((oct != NULL) && (LIO_CN23XX_PF(oct))) { 1049 if (droq != NULL) 1050 lio_write_csr64(oct, droq->pkts_sent_reg, 1051 LIO_CN23XX_INTR_RESEND); 1052 /* we race with firmrware here. */ 1053 /* read and write the IN_DONE_CNTS */ 1054 else if (iq != NULL) { 1055 instr_cnt = lio_read_csr64(oct, iq->inst_cnt_reg); 1056 lio_write_csr64(oct, iq->inst_cnt_reg, 1057 ((instr_cnt & 0xFFFFFFFF00000000ULL) | 1058 LIO_CN23XX_INTR_RESEND)); 1059 } 1060 } 1061 } 1062