1 /****************************************************************************** 2 3 Copyright (c) 2013-2018, Intel Corporation 4 All rights reserved. 5 6 Redistribution and use in source and binary forms, with or without 7 modification, are permitted provided that the following conditions are met: 8 9 1. Redistributions of source code must retain the above copyright notice, 10 this list of conditions and the following disclaimer. 11 12 2. Redistributions in binary form must reproduce the above copyright 13 notice, this list of conditions and the following disclaimer in the 14 documentation and/or other materials provided with the distribution. 15 16 3. Neither the name of the Intel Corporation nor the names of its 17 contributors may be used to endorse or promote products derived from 18 this software without specific prior written permission. 19 20 THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 21 AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 22 IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 23 ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE 24 LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 25 CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 26 SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 27 INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 28 CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 29 ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 30 POSSIBILITY OF SUCH DAMAGE. 31 32 ******************************************************************************/ 33 /*$FreeBSD$*/ 34 35 #include "i40e_status.h" 36 #include "i40e_type.h" 37 #include "i40e_register.h" 38 #include "i40e_adminq.h" 39 #include "i40e_prototype.h" 40 41 /** 42 * i40e_adminq_init_regs - Initialize AdminQ registers 43 * @hw: pointer to the hardware structure 44 * 45 * This assumes the alloc_asq and alloc_arq functions have already been called 46 **/ 47 static void i40e_adminq_init_regs(struct i40e_hw *hw) 48 { 49 /* set head and tail registers in our local struct */ 50 if (i40e_is_vf(hw)) { 51 hw->aq.asq.tail = I40E_VF_ATQT1; 52 hw->aq.asq.head = I40E_VF_ATQH1; 53 hw->aq.asq.len = I40E_VF_ATQLEN1; 54 hw->aq.asq.bal = I40E_VF_ATQBAL1; 55 hw->aq.asq.bah = I40E_VF_ATQBAH1; 56 hw->aq.arq.tail = I40E_VF_ARQT1; 57 hw->aq.arq.head = I40E_VF_ARQH1; 58 hw->aq.arq.len = I40E_VF_ARQLEN1; 59 hw->aq.arq.bal = I40E_VF_ARQBAL1; 60 hw->aq.arq.bah = I40E_VF_ARQBAH1; 61 } else { 62 hw->aq.asq.tail = I40E_PF_ATQT; 63 hw->aq.asq.head = I40E_PF_ATQH; 64 hw->aq.asq.len = I40E_PF_ATQLEN; 65 hw->aq.asq.bal = I40E_PF_ATQBAL; 66 hw->aq.asq.bah = I40E_PF_ATQBAH; 67 hw->aq.arq.tail = I40E_PF_ARQT; 68 hw->aq.arq.head = I40E_PF_ARQH; 69 hw->aq.arq.len = I40E_PF_ARQLEN; 70 hw->aq.arq.bal = I40E_PF_ARQBAL; 71 hw->aq.arq.bah = I40E_PF_ARQBAH; 72 } 73 } 74 75 /** 76 * i40e_alloc_adminq_asq_ring - Allocate Admin Queue send rings 77 * @hw: pointer to the hardware structure 78 **/ 79 enum i40e_status_code i40e_alloc_adminq_asq_ring(struct i40e_hw *hw) 80 { 81 enum i40e_status_code ret_code; 82 83 ret_code = i40e_allocate_dma_mem(hw, &hw->aq.asq.desc_buf, 84 i40e_mem_atq_ring, 85 (hw->aq.num_asq_entries * 86 sizeof(struct i40e_aq_desc)), 87 I40E_ADMINQ_DESC_ALIGNMENT); 88 if (ret_code) 89 return ret_code; 90 91 ret_code = i40e_allocate_virt_mem(hw, &hw->aq.asq.cmd_buf, 92 (hw->aq.num_asq_entries * 93 sizeof(struct i40e_asq_cmd_details))); 94 if (ret_code) { 95 i40e_free_dma_mem(hw, &hw->aq.asq.desc_buf); 96 return ret_code; 97 } 98 99 return ret_code; 100 } 101 102 /** 103 * i40e_alloc_adminq_arq_ring - Allocate Admin Queue receive rings 104 * @hw: pointer to the hardware structure 105 **/ 106 enum i40e_status_code i40e_alloc_adminq_arq_ring(struct i40e_hw *hw) 107 { 108 enum i40e_status_code ret_code; 109 110 ret_code = i40e_allocate_dma_mem(hw, &hw->aq.arq.desc_buf, 111 i40e_mem_arq_ring, 112 (hw->aq.num_arq_entries * 113 sizeof(struct i40e_aq_desc)), 114 I40E_ADMINQ_DESC_ALIGNMENT); 115 116 return ret_code; 117 } 118 119 /** 120 * i40e_free_adminq_asq - Free Admin Queue send rings 121 * @hw: pointer to the hardware structure 122 * 123 * This assumes the posted send buffers have already been cleaned 124 * and de-allocated 125 **/ 126 void i40e_free_adminq_asq(struct i40e_hw *hw) 127 { 128 i40e_free_dma_mem(hw, &hw->aq.asq.desc_buf); 129 } 130 131 /** 132 * i40e_free_adminq_arq - Free Admin Queue receive rings 133 * @hw: pointer to the hardware structure 134 * 135 * This assumes the posted receive buffers have already been cleaned 136 * and de-allocated 137 **/ 138 void i40e_free_adminq_arq(struct i40e_hw *hw) 139 { 140 i40e_free_dma_mem(hw, &hw->aq.arq.desc_buf); 141 } 142 143 /** 144 * i40e_alloc_arq_bufs - Allocate pre-posted buffers for the receive queue 145 * @hw: pointer to the hardware structure 146 **/ 147 static enum i40e_status_code i40e_alloc_arq_bufs(struct i40e_hw *hw) 148 { 149 enum i40e_status_code ret_code; 150 struct i40e_aq_desc *desc; 151 struct i40e_dma_mem *bi; 152 int i; 153 154 /* We'll be allocating the buffer info memory first, then we can 155 * allocate the mapped buffers for the event processing 156 */ 157 158 /* buffer_info structures do not need alignment */ 159 ret_code = i40e_allocate_virt_mem(hw, &hw->aq.arq.dma_head, 160 (hw->aq.num_arq_entries * sizeof(struct i40e_dma_mem))); 161 if (ret_code) 162 goto alloc_arq_bufs; 163 hw->aq.arq.r.arq_bi = (struct i40e_dma_mem *)hw->aq.arq.dma_head.va; 164 165 /* allocate the mapped buffers */ 166 for (i = 0; i < hw->aq.num_arq_entries; i++) { 167 bi = &hw->aq.arq.r.arq_bi[i]; 168 ret_code = i40e_allocate_dma_mem(hw, bi, 169 i40e_mem_arq_buf, 170 hw->aq.arq_buf_size, 171 I40E_ADMINQ_DESC_ALIGNMENT); 172 if (ret_code) 173 goto unwind_alloc_arq_bufs; 174 175 /* now configure the descriptors for use */ 176 desc = I40E_ADMINQ_DESC(hw->aq.arq, i); 177 178 desc->flags = CPU_TO_LE16(I40E_AQ_FLAG_BUF); 179 if (hw->aq.arq_buf_size > I40E_AQ_LARGE_BUF) 180 desc->flags |= CPU_TO_LE16(I40E_AQ_FLAG_LB); 181 desc->opcode = 0; 182 /* This is in accordance with Admin queue design, there is no 183 * register for buffer size configuration 184 */ 185 desc->datalen = CPU_TO_LE16((u16)bi->size); 186 desc->retval = 0; 187 desc->cookie_high = 0; 188 desc->cookie_low = 0; 189 desc->params.external.addr_high = 190 CPU_TO_LE32(I40E_HI_DWORD(bi->pa)); 191 desc->params.external.addr_low = 192 CPU_TO_LE32(I40E_LO_DWORD(bi->pa)); 193 desc->params.external.param0 = 0; 194 desc->params.external.param1 = 0; 195 } 196 197 alloc_arq_bufs: 198 return ret_code; 199 200 unwind_alloc_arq_bufs: 201 /* don't try to free the one that failed... */ 202 i--; 203 for (; i >= 0; i--) 204 i40e_free_dma_mem(hw, &hw->aq.arq.r.arq_bi[i]); 205 i40e_free_virt_mem(hw, &hw->aq.arq.dma_head); 206 207 return ret_code; 208 } 209 210 /** 211 * i40e_alloc_asq_bufs - Allocate empty buffer structs for the send queue 212 * @hw: pointer to the hardware structure 213 **/ 214 static enum i40e_status_code i40e_alloc_asq_bufs(struct i40e_hw *hw) 215 { 216 enum i40e_status_code ret_code; 217 struct i40e_dma_mem *bi; 218 int i; 219 220 /* No mapped memory needed yet, just the buffer info structures */ 221 ret_code = i40e_allocate_virt_mem(hw, &hw->aq.asq.dma_head, 222 (hw->aq.num_asq_entries * sizeof(struct i40e_dma_mem))); 223 if (ret_code) 224 goto alloc_asq_bufs; 225 hw->aq.asq.r.asq_bi = (struct i40e_dma_mem *)hw->aq.asq.dma_head.va; 226 227 /* allocate the mapped buffers */ 228 for (i = 0; i < hw->aq.num_asq_entries; i++) { 229 bi = &hw->aq.asq.r.asq_bi[i]; 230 ret_code = i40e_allocate_dma_mem(hw, bi, 231 i40e_mem_asq_buf, 232 hw->aq.asq_buf_size, 233 I40E_ADMINQ_DESC_ALIGNMENT); 234 if (ret_code) 235 goto unwind_alloc_asq_bufs; 236 } 237 alloc_asq_bufs: 238 return ret_code; 239 240 unwind_alloc_asq_bufs: 241 /* don't try to free the one that failed... */ 242 i--; 243 for (; i >= 0; i--) 244 i40e_free_dma_mem(hw, &hw->aq.asq.r.asq_bi[i]); 245 i40e_free_virt_mem(hw, &hw->aq.asq.dma_head); 246 247 return ret_code; 248 } 249 250 /** 251 * i40e_free_arq_bufs - Free receive queue buffer info elements 252 * @hw: pointer to the hardware structure 253 **/ 254 static void i40e_free_arq_bufs(struct i40e_hw *hw) 255 { 256 int i; 257 258 /* free descriptors */ 259 for (i = 0; i < hw->aq.num_arq_entries; i++) 260 i40e_free_dma_mem(hw, &hw->aq.arq.r.arq_bi[i]); 261 262 /* free the descriptor memory */ 263 i40e_free_dma_mem(hw, &hw->aq.arq.desc_buf); 264 265 /* free the dma header */ 266 i40e_free_virt_mem(hw, &hw->aq.arq.dma_head); 267 } 268 269 /** 270 * i40e_free_asq_bufs - Free send queue buffer info elements 271 * @hw: pointer to the hardware structure 272 **/ 273 static void i40e_free_asq_bufs(struct i40e_hw *hw) 274 { 275 int i; 276 277 /* only unmap if the address is non-NULL */ 278 for (i = 0; i < hw->aq.num_asq_entries; i++) 279 if (hw->aq.asq.r.asq_bi[i].pa) 280 i40e_free_dma_mem(hw, &hw->aq.asq.r.asq_bi[i]); 281 282 /* free the buffer info list */ 283 i40e_free_virt_mem(hw, &hw->aq.asq.cmd_buf); 284 285 /* free the descriptor memory */ 286 i40e_free_dma_mem(hw, &hw->aq.asq.desc_buf); 287 288 /* free the dma header */ 289 i40e_free_virt_mem(hw, &hw->aq.asq.dma_head); 290 } 291 292 /** 293 * i40e_config_asq_regs - configure ASQ registers 294 * @hw: pointer to the hardware structure 295 * 296 * Configure base address and length registers for the transmit queue 297 **/ 298 static enum i40e_status_code i40e_config_asq_regs(struct i40e_hw *hw) 299 { 300 enum i40e_status_code ret_code = I40E_SUCCESS; 301 u32 reg = 0; 302 303 /* Clear Head and Tail */ 304 wr32(hw, hw->aq.asq.head, 0); 305 wr32(hw, hw->aq.asq.tail, 0); 306 307 /* set starting point */ 308 if (!i40e_is_vf(hw)) 309 wr32(hw, hw->aq.asq.len, (hw->aq.num_asq_entries | 310 I40E_PF_ATQLEN_ATQENABLE_MASK)); 311 if (i40e_is_vf(hw)) 312 wr32(hw, hw->aq.asq.len, (hw->aq.num_asq_entries | 313 I40E_VF_ATQLEN1_ATQENABLE_MASK)); 314 wr32(hw, hw->aq.asq.bal, I40E_LO_DWORD(hw->aq.asq.desc_buf.pa)); 315 wr32(hw, hw->aq.asq.bah, I40E_HI_DWORD(hw->aq.asq.desc_buf.pa)); 316 317 /* Check one register to verify that config was applied */ 318 reg = rd32(hw, hw->aq.asq.bal); 319 if (reg != I40E_LO_DWORD(hw->aq.asq.desc_buf.pa)) 320 ret_code = I40E_ERR_ADMIN_QUEUE_ERROR; 321 322 return ret_code; 323 } 324 325 /** 326 * i40e_config_arq_regs - ARQ register configuration 327 * @hw: pointer to the hardware structure 328 * 329 * Configure base address and length registers for the receive (event queue) 330 **/ 331 static enum i40e_status_code i40e_config_arq_regs(struct i40e_hw *hw) 332 { 333 enum i40e_status_code ret_code = I40E_SUCCESS; 334 u32 reg = 0; 335 336 /* Clear Head and Tail */ 337 wr32(hw, hw->aq.arq.head, 0); 338 wr32(hw, hw->aq.arq.tail, 0); 339 340 /* set starting point */ 341 if (!i40e_is_vf(hw)) 342 wr32(hw, hw->aq.arq.len, (hw->aq.num_arq_entries | 343 I40E_PF_ARQLEN_ARQENABLE_MASK)); 344 if (i40e_is_vf(hw)) 345 wr32(hw, hw->aq.arq.len, (hw->aq.num_arq_entries | 346 I40E_VF_ARQLEN1_ARQENABLE_MASK)); 347 wr32(hw, hw->aq.arq.bal, I40E_LO_DWORD(hw->aq.arq.desc_buf.pa)); 348 wr32(hw, hw->aq.arq.bah, I40E_HI_DWORD(hw->aq.arq.desc_buf.pa)); 349 350 /* Update tail in the HW to post pre-allocated buffers */ 351 wr32(hw, hw->aq.arq.tail, hw->aq.num_arq_entries - 1); 352 353 /* Check one register to verify that config was applied */ 354 reg = rd32(hw, hw->aq.arq.bal); 355 if (reg != I40E_LO_DWORD(hw->aq.arq.desc_buf.pa)) 356 ret_code = I40E_ERR_ADMIN_QUEUE_ERROR; 357 358 return ret_code; 359 } 360 361 /** 362 * i40e_init_asq - main initialization routine for ASQ 363 * @hw: pointer to the hardware structure 364 * 365 * This is the main initialization routine for the Admin Send Queue 366 * Prior to calling this function, drivers *MUST* set the following fields 367 * in the hw->aq structure: 368 * - hw->aq.num_asq_entries 369 * - hw->aq.arq_buf_size 370 * 371 * Do *NOT* hold the lock when calling this as the memory allocation routines 372 * called are not going to be atomic context safe 373 **/ 374 enum i40e_status_code i40e_init_asq(struct i40e_hw *hw) 375 { 376 enum i40e_status_code ret_code = I40E_SUCCESS; 377 378 if (hw->aq.asq.count > 0) { 379 /* queue already initialized */ 380 ret_code = I40E_ERR_NOT_READY; 381 goto init_adminq_exit; 382 } 383 384 /* verify input for valid configuration */ 385 if ((hw->aq.num_asq_entries == 0) || 386 (hw->aq.asq_buf_size == 0)) { 387 ret_code = I40E_ERR_CONFIG; 388 goto init_adminq_exit; 389 } 390 391 hw->aq.asq.next_to_use = 0; 392 hw->aq.asq.next_to_clean = 0; 393 394 /* allocate the ring memory */ 395 ret_code = i40e_alloc_adminq_asq_ring(hw); 396 if (ret_code != I40E_SUCCESS) 397 goto init_adminq_exit; 398 399 /* allocate buffers in the rings */ 400 ret_code = i40e_alloc_asq_bufs(hw); 401 if (ret_code != I40E_SUCCESS) 402 goto init_adminq_free_rings; 403 404 /* initialize base registers */ 405 ret_code = i40e_config_asq_regs(hw); 406 if (ret_code != I40E_SUCCESS) 407 goto init_adminq_free_rings; 408 409 /* success! */ 410 hw->aq.asq.count = hw->aq.num_asq_entries; 411 goto init_adminq_exit; 412 413 init_adminq_free_rings: 414 i40e_free_adminq_asq(hw); 415 416 init_adminq_exit: 417 return ret_code; 418 } 419 420 /** 421 * i40e_init_arq - initialize ARQ 422 * @hw: pointer to the hardware structure 423 * 424 * The main initialization routine for the Admin Receive (Event) Queue. 425 * Prior to calling this function, drivers *MUST* set the following fields 426 * in the hw->aq structure: 427 * - hw->aq.num_asq_entries 428 * - hw->aq.arq_buf_size 429 * 430 * Do *NOT* hold the lock when calling this as the memory allocation routines 431 * called are not going to be atomic context safe 432 **/ 433 enum i40e_status_code i40e_init_arq(struct i40e_hw *hw) 434 { 435 enum i40e_status_code ret_code = I40E_SUCCESS; 436 437 if (hw->aq.arq.count > 0) { 438 /* queue already initialized */ 439 ret_code = I40E_ERR_NOT_READY; 440 goto init_adminq_exit; 441 } 442 443 /* verify input for valid configuration */ 444 if ((hw->aq.num_arq_entries == 0) || 445 (hw->aq.arq_buf_size == 0)) { 446 ret_code = I40E_ERR_CONFIG; 447 goto init_adminq_exit; 448 } 449 450 hw->aq.arq.next_to_use = 0; 451 hw->aq.arq.next_to_clean = 0; 452 453 /* allocate the ring memory */ 454 ret_code = i40e_alloc_adminq_arq_ring(hw); 455 if (ret_code != I40E_SUCCESS) 456 goto init_adminq_exit; 457 458 /* allocate buffers in the rings */ 459 ret_code = i40e_alloc_arq_bufs(hw); 460 if (ret_code != I40E_SUCCESS) 461 goto init_adminq_free_rings; 462 463 /* initialize base registers */ 464 ret_code = i40e_config_arq_regs(hw); 465 if (ret_code != I40E_SUCCESS) 466 goto init_adminq_free_rings; 467 468 /* success! */ 469 hw->aq.arq.count = hw->aq.num_arq_entries; 470 goto init_adminq_exit; 471 472 init_adminq_free_rings: 473 i40e_free_adminq_arq(hw); 474 475 init_adminq_exit: 476 return ret_code; 477 } 478 479 /** 480 * i40e_shutdown_asq - shutdown the ASQ 481 * @hw: pointer to the hardware structure 482 * 483 * The main shutdown routine for the Admin Send Queue 484 **/ 485 enum i40e_status_code i40e_shutdown_asq(struct i40e_hw *hw) 486 { 487 enum i40e_status_code ret_code = I40E_SUCCESS; 488 489 i40e_acquire_spinlock(&hw->aq.asq_spinlock); 490 491 if (hw->aq.asq.count == 0) { 492 ret_code = I40E_ERR_NOT_READY; 493 goto shutdown_asq_out; 494 } 495 496 /* Stop firmware AdminQ processing */ 497 wr32(hw, hw->aq.asq.head, 0); 498 wr32(hw, hw->aq.asq.tail, 0); 499 wr32(hw, hw->aq.asq.len, 0); 500 wr32(hw, hw->aq.asq.bal, 0); 501 wr32(hw, hw->aq.asq.bah, 0); 502 503 hw->aq.asq.count = 0; /* to indicate uninitialized queue */ 504 505 /* free ring buffers */ 506 i40e_free_asq_bufs(hw); 507 508 shutdown_asq_out: 509 i40e_release_spinlock(&hw->aq.asq_spinlock); 510 return ret_code; 511 } 512 513 /** 514 * i40e_shutdown_arq - shutdown ARQ 515 * @hw: pointer to the hardware structure 516 * 517 * The main shutdown routine for the Admin Receive Queue 518 **/ 519 enum i40e_status_code i40e_shutdown_arq(struct i40e_hw *hw) 520 { 521 enum i40e_status_code ret_code = I40E_SUCCESS; 522 523 i40e_acquire_spinlock(&hw->aq.arq_spinlock); 524 525 if (hw->aq.arq.count == 0) { 526 ret_code = I40E_ERR_NOT_READY; 527 goto shutdown_arq_out; 528 } 529 530 /* Stop firmware AdminQ processing */ 531 wr32(hw, hw->aq.arq.head, 0); 532 wr32(hw, hw->aq.arq.tail, 0); 533 wr32(hw, hw->aq.arq.len, 0); 534 wr32(hw, hw->aq.arq.bal, 0); 535 wr32(hw, hw->aq.arq.bah, 0); 536 537 hw->aq.arq.count = 0; /* to indicate uninitialized queue */ 538 539 /* free ring buffers */ 540 i40e_free_arq_bufs(hw); 541 542 shutdown_arq_out: 543 i40e_release_spinlock(&hw->aq.arq_spinlock); 544 return ret_code; 545 } 546 547 /** 548 * i40e_resume_aq - resume AQ processing from 0 549 * @hw: pointer to the hardware structure 550 **/ 551 static void i40e_resume_aq(struct i40e_hw *hw) 552 { 553 /* Registers are reset after PF reset */ 554 hw->aq.asq.next_to_use = 0; 555 hw->aq.asq.next_to_clean = 0; 556 557 i40e_config_asq_regs(hw); 558 559 hw->aq.arq.next_to_use = 0; 560 hw->aq.arq.next_to_clean = 0; 561 562 i40e_config_arq_regs(hw); 563 } 564 565 /** 566 * i40e_init_adminq - main initialization routine for Admin Queue 567 * @hw: pointer to the hardware structure 568 * 569 * Prior to calling this function, drivers *MUST* set the following fields 570 * in the hw->aq structure: 571 * - hw->aq.num_asq_entries 572 * - hw->aq.num_arq_entries 573 * - hw->aq.arq_buf_size 574 * - hw->aq.asq_buf_size 575 **/ 576 enum i40e_status_code i40e_init_adminq(struct i40e_hw *hw) 577 { 578 u16 cfg_ptr, oem_hi, oem_lo; 579 u16 eetrack_lo, eetrack_hi; 580 enum i40e_status_code ret_code; 581 int retry = 0; 582 583 /* verify input for valid configuration */ 584 if ((hw->aq.num_arq_entries == 0) || 585 (hw->aq.num_asq_entries == 0) || 586 (hw->aq.arq_buf_size == 0) || 587 (hw->aq.asq_buf_size == 0)) { 588 ret_code = I40E_ERR_CONFIG; 589 goto init_adminq_exit; 590 } 591 i40e_init_spinlock(&hw->aq.asq_spinlock); 592 i40e_init_spinlock(&hw->aq.arq_spinlock); 593 594 /* Set up register offsets */ 595 i40e_adminq_init_regs(hw); 596 597 /* setup ASQ command write back timeout */ 598 hw->aq.asq_cmd_timeout = I40E_ASQ_CMD_TIMEOUT; 599 600 /* allocate the ASQ */ 601 ret_code = i40e_init_asq(hw); 602 if (ret_code != I40E_SUCCESS) 603 goto init_adminq_destroy_spinlocks; 604 605 /* allocate the ARQ */ 606 ret_code = i40e_init_arq(hw); 607 if (ret_code != I40E_SUCCESS) 608 goto init_adminq_free_asq; 609 610 /* VF has no need of firmware */ 611 if (i40e_is_vf(hw)) 612 goto init_adminq_exit; 613 /* There are some cases where the firmware may not be quite ready 614 * for AdminQ operations, so we retry the AdminQ setup a few times 615 * if we see timeouts in this first AQ call. 616 */ 617 do { 618 ret_code = i40e_aq_get_firmware_version(hw, 619 &hw->aq.fw_maj_ver, 620 &hw->aq.fw_min_ver, 621 &hw->aq.fw_build, 622 &hw->aq.api_maj_ver, 623 &hw->aq.api_min_ver, 624 NULL); 625 if (ret_code != I40E_ERR_ADMIN_QUEUE_TIMEOUT) 626 break; 627 retry++; 628 i40e_msec_delay(100); 629 i40e_resume_aq(hw); 630 } while (retry < 10); 631 if (ret_code != I40E_SUCCESS) 632 goto init_adminq_free_arq; 633 634 /* get the NVM version info */ 635 i40e_read_nvm_word(hw, I40E_SR_NVM_DEV_STARTER_VERSION, 636 &hw->nvm.version); 637 i40e_read_nvm_word(hw, I40E_SR_NVM_EETRACK_LO, &eetrack_lo); 638 i40e_read_nvm_word(hw, I40E_SR_NVM_EETRACK_HI, &eetrack_hi); 639 hw->nvm.eetrack = (eetrack_hi << 16) | eetrack_lo; 640 i40e_read_nvm_word(hw, I40E_SR_BOOT_CONFIG_PTR, &cfg_ptr); 641 i40e_read_nvm_word(hw, (cfg_ptr + I40E_NVM_OEM_VER_OFF), 642 &oem_hi); 643 i40e_read_nvm_word(hw, (cfg_ptr + (I40E_NVM_OEM_VER_OFF + 1)), 644 &oem_lo); 645 hw->nvm.oem_ver = ((u32)oem_hi << 16) | oem_lo; 646 647 /* The ability to RX (not drop) 802.1ad frames was added in API 1.7 */ 648 if ((hw->aq.api_maj_ver > 1) || 649 ((hw->aq.api_maj_ver == 1) && 650 (hw->aq.api_min_ver >= 7))) 651 hw->flags |= I40E_HW_FLAG_802_1AD_CAPABLE; 652 653 if (hw->mac.type == I40E_MAC_XL710 && 654 hw->aq.api_maj_ver == I40E_FW_API_VERSION_MAJOR && 655 hw->aq.api_min_ver >= I40E_MINOR_VER_GET_LINK_INFO_XL710) { 656 hw->flags |= I40E_HW_FLAG_AQ_PHY_ACCESS_CAPABLE; 657 } 658 659 /* Newer versions of firmware require lock when reading the NVM */ 660 if ((hw->aq.api_maj_ver > 1) || 661 ((hw->aq.api_maj_ver == 1) && 662 (hw->aq.api_min_ver >= 5))) 663 hw->flags |= I40E_HW_FLAG_NVM_READ_REQUIRES_LOCK; 664 665 if (hw->aq.api_maj_ver > I40E_FW_API_VERSION_MAJOR) { 666 ret_code = I40E_ERR_FIRMWARE_API_VERSION; 667 goto init_adminq_free_arq; 668 } 669 670 /* pre-emptive resource lock release */ 671 i40e_aq_release_resource(hw, I40E_NVM_RESOURCE_ID, 0, NULL); 672 hw->nvm_release_on_done = FALSE; 673 hw->nvmupd_state = I40E_NVMUPD_STATE_INIT; 674 675 ret_code = I40E_SUCCESS; 676 677 /* success! */ 678 goto init_adminq_exit; 679 680 init_adminq_free_arq: 681 i40e_shutdown_arq(hw); 682 init_adminq_free_asq: 683 i40e_shutdown_asq(hw); 684 init_adminq_destroy_spinlocks: 685 i40e_destroy_spinlock(&hw->aq.asq_spinlock); 686 i40e_destroy_spinlock(&hw->aq.arq_spinlock); 687 688 init_adminq_exit: 689 return ret_code; 690 } 691 692 /** 693 * i40e_shutdown_adminq - shutdown routine for the Admin Queue 694 * @hw: pointer to the hardware structure 695 **/ 696 enum i40e_status_code i40e_shutdown_adminq(struct i40e_hw *hw) 697 { 698 enum i40e_status_code ret_code = I40E_SUCCESS; 699 700 if (i40e_check_asq_alive(hw)) 701 i40e_aq_queue_shutdown(hw, TRUE); 702 703 i40e_shutdown_asq(hw); 704 i40e_shutdown_arq(hw); 705 i40e_destroy_spinlock(&hw->aq.asq_spinlock); 706 i40e_destroy_spinlock(&hw->aq.arq_spinlock); 707 708 if (hw->nvm_buff.va) 709 i40e_free_virt_mem(hw, &hw->nvm_buff); 710 711 return ret_code; 712 } 713 714 /** 715 * i40e_clean_asq - cleans Admin send queue 716 * @hw: pointer to the hardware structure 717 * 718 * returns the number of free desc 719 **/ 720 u16 i40e_clean_asq(struct i40e_hw *hw) 721 { 722 struct i40e_adminq_ring *asq = &(hw->aq.asq); 723 struct i40e_asq_cmd_details *details; 724 u16 ntc = asq->next_to_clean; 725 struct i40e_aq_desc desc_cb; 726 struct i40e_aq_desc *desc; 727 728 desc = I40E_ADMINQ_DESC(*asq, ntc); 729 details = I40E_ADMINQ_DETAILS(*asq, ntc); 730 while (rd32(hw, hw->aq.asq.head) != ntc) { 731 i40e_debug(hw, I40E_DEBUG_AQ_MESSAGE, 732 "ntc %d head %d.\n", ntc, rd32(hw, hw->aq.asq.head)); 733 734 if (details->callback) { 735 I40E_ADMINQ_CALLBACK cb_func = 736 (I40E_ADMINQ_CALLBACK)details->callback; 737 i40e_memcpy(&desc_cb, desc, sizeof(struct i40e_aq_desc), 738 I40E_DMA_TO_DMA); 739 cb_func(hw, &desc_cb); 740 } 741 i40e_memset(desc, 0, sizeof(*desc), I40E_DMA_MEM); 742 i40e_memset(details, 0, sizeof(*details), I40E_NONDMA_MEM); 743 ntc++; 744 if (ntc == asq->count) 745 ntc = 0; 746 desc = I40E_ADMINQ_DESC(*asq, ntc); 747 details = I40E_ADMINQ_DETAILS(*asq, ntc); 748 } 749 750 asq->next_to_clean = ntc; 751 752 return I40E_DESC_UNUSED(asq); 753 } 754 755 /** 756 * i40e_asq_done - check if FW has processed the Admin Send Queue 757 * @hw: pointer to the hw struct 758 * 759 * Returns TRUE if the firmware has processed all descriptors on the 760 * admin send queue. Returns FALSE if there are still requests pending. 761 **/ 762 bool i40e_asq_done(struct i40e_hw *hw) 763 { 764 /* AQ designers suggest use of head for better 765 * timing reliability than DD bit 766 */ 767 return rd32(hw, hw->aq.asq.head) == hw->aq.asq.next_to_use; 768 769 } 770 771 /** 772 * i40e_asq_send_command - send command to Admin Queue 773 * @hw: pointer to the hw struct 774 * @desc: prefilled descriptor describing the command (non DMA mem) 775 * @buff: buffer to use for indirect commands 776 * @buff_size: size of buffer for indirect commands 777 * @cmd_details: pointer to command details structure 778 * 779 * This is the main send command driver routine for the Admin Queue send 780 * queue. It runs the queue, cleans the queue, etc 781 **/ 782 enum i40e_status_code i40e_asq_send_command(struct i40e_hw *hw, 783 struct i40e_aq_desc *desc, 784 void *buff, /* can be NULL */ 785 u16 buff_size, 786 struct i40e_asq_cmd_details *cmd_details) 787 { 788 enum i40e_status_code status = I40E_SUCCESS; 789 struct i40e_dma_mem *dma_buff = NULL; 790 struct i40e_asq_cmd_details *details; 791 struct i40e_aq_desc *desc_on_ring; 792 bool cmd_completed = FALSE; 793 u16 retval = 0; 794 u32 val = 0; 795 796 i40e_acquire_spinlock(&hw->aq.asq_spinlock); 797 798 hw->aq.asq_last_status = I40E_AQ_RC_OK; 799 800 if (hw->aq.asq.count == 0) { 801 i40e_debug(hw, I40E_DEBUG_AQ_MESSAGE, 802 "AQTX: Admin queue not initialized.\n"); 803 status = I40E_ERR_QUEUE_EMPTY; 804 goto asq_send_command_error; 805 } 806 807 val = rd32(hw, hw->aq.asq.head); 808 if (val >= hw->aq.num_asq_entries) { 809 i40e_debug(hw, I40E_DEBUG_AQ_MESSAGE, 810 "AQTX: head overrun at %d\n", val); 811 status = I40E_ERR_QUEUE_EMPTY; 812 goto asq_send_command_error; 813 } 814 815 details = I40E_ADMINQ_DETAILS(hw->aq.asq, hw->aq.asq.next_to_use); 816 if (cmd_details) { 817 i40e_memcpy(details, 818 cmd_details, 819 sizeof(struct i40e_asq_cmd_details), 820 I40E_NONDMA_TO_NONDMA); 821 822 /* If the cmd_details are defined copy the cookie. The 823 * CPU_TO_LE32 is not needed here because the data is ignored 824 * by the FW, only used by the driver 825 */ 826 if (details->cookie) { 827 desc->cookie_high = 828 CPU_TO_LE32(I40E_HI_DWORD(details->cookie)); 829 desc->cookie_low = 830 CPU_TO_LE32(I40E_LO_DWORD(details->cookie)); 831 } 832 } else { 833 i40e_memset(details, 0, 834 sizeof(struct i40e_asq_cmd_details), 835 I40E_NONDMA_MEM); 836 } 837 838 /* clear requested flags and then set additional flags if defined */ 839 desc->flags &= ~CPU_TO_LE16(details->flags_dis); 840 desc->flags |= CPU_TO_LE16(details->flags_ena); 841 842 if (buff_size > hw->aq.asq_buf_size) { 843 i40e_debug(hw, 844 I40E_DEBUG_AQ_MESSAGE, 845 "AQTX: Invalid buffer size: %d.\n", 846 buff_size); 847 status = I40E_ERR_INVALID_SIZE; 848 goto asq_send_command_error; 849 } 850 851 if (details->postpone && !details->async) { 852 i40e_debug(hw, 853 I40E_DEBUG_AQ_MESSAGE, 854 "AQTX: Async flag not set along with postpone flag"); 855 status = I40E_ERR_PARAM; 856 goto asq_send_command_error; 857 } 858 859 /* call clean and check queue available function to reclaim the 860 * descriptors that were processed by FW, the function returns the 861 * number of desc available 862 */ 863 /* the clean function called here could be called in a separate thread 864 * in case of asynchronous completions 865 */ 866 if (i40e_clean_asq(hw) == 0) { 867 i40e_debug(hw, 868 I40E_DEBUG_AQ_MESSAGE, 869 "AQTX: Error queue is full.\n"); 870 status = I40E_ERR_ADMIN_QUEUE_FULL; 871 goto asq_send_command_error; 872 } 873 874 /* initialize the temp desc pointer with the right desc */ 875 desc_on_ring = I40E_ADMINQ_DESC(hw->aq.asq, hw->aq.asq.next_to_use); 876 877 /* if the desc is available copy the temp desc to the right place */ 878 i40e_memcpy(desc_on_ring, desc, sizeof(struct i40e_aq_desc), 879 I40E_NONDMA_TO_DMA); 880 881 /* if buff is not NULL assume indirect command */ 882 if (buff != NULL) { 883 dma_buff = &(hw->aq.asq.r.asq_bi[hw->aq.asq.next_to_use]); 884 /* copy the user buff into the respective DMA buff */ 885 i40e_memcpy(dma_buff->va, buff, buff_size, 886 I40E_NONDMA_TO_DMA); 887 desc_on_ring->datalen = CPU_TO_LE16(buff_size); 888 889 /* Update the address values in the desc with the pa value 890 * for respective buffer 891 */ 892 desc_on_ring->params.external.addr_high = 893 CPU_TO_LE32(I40E_HI_DWORD(dma_buff->pa)); 894 desc_on_ring->params.external.addr_low = 895 CPU_TO_LE32(I40E_LO_DWORD(dma_buff->pa)); 896 } 897 898 /* bump the tail */ 899 i40e_debug(hw, I40E_DEBUG_AQ_MESSAGE, "AQTX: desc and buffer:\n"); 900 i40e_debug_aq(hw, I40E_DEBUG_AQ_COMMAND, (void *)desc_on_ring, 901 buff, buff_size); 902 (hw->aq.asq.next_to_use)++; 903 if (hw->aq.asq.next_to_use == hw->aq.asq.count) 904 hw->aq.asq.next_to_use = 0; 905 if (!details->postpone) 906 wr32(hw, hw->aq.asq.tail, hw->aq.asq.next_to_use); 907 908 /* if cmd_details are not defined or async flag is not set, 909 * we need to wait for desc write back 910 */ 911 if (!details->async && !details->postpone) { 912 u32 total_delay = 0; 913 914 do { 915 /* AQ designers suggest use of head for better 916 * timing reliability than DD bit 917 */ 918 if (i40e_asq_done(hw)) 919 break; 920 i40e_usec_delay(50); 921 total_delay += 50; 922 } while (total_delay < hw->aq.asq_cmd_timeout); 923 } 924 925 /* if ready, copy the desc back to temp */ 926 if (i40e_asq_done(hw)) { 927 i40e_memcpy(desc, desc_on_ring, sizeof(struct i40e_aq_desc), 928 I40E_DMA_TO_NONDMA); 929 if (buff != NULL) 930 i40e_memcpy(buff, dma_buff->va, buff_size, 931 I40E_DMA_TO_NONDMA); 932 retval = LE16_TO_CPU(desc->retval); 933 if (retval != 0) { 934 i40e_debug(hw, 935 I40E_DEBUG_AQ_MESSAGE, 936 "AQTX: Command completed with error 0x%X.\n", 937 retval); 938 939 /* strip off FW internal code */ 940 retval &= 0xff; 941 } 942 cmd_completed = TRUE; 943 if ((enum i40e_admin_queue_err)retval == I40E_AQ_RC_OK) 944 status = I40E_SUCCESS; 945 else 946 status = I40E_ERR_ADMIN_QUEUE_ERROR; 947 hw->aq.asq_last_status = (enum i40e_admin_queue_err)retval; 948 } 949 950 i40e_debug(hw, I40E_DEBUG_AQ_MESSAGE, 951 "AQTX: desc and buffer writeback:\n"); 952 i40e_debug_aq(hw, I40E_DEBUG_AQ_COMMAND, (void *)desc, buff, buff_size); 953 954 /* save writeback aq if requested */ 955 if (details->wb_desc) 956 i40e_memcpy(details->wb_desc, desc_on_ring, 957 sizeof(struct i40e_aq_desc), I40E_DMA_TO_NONDMA); 958 959 /* update the error if time out occurred */ 960 if ((!cmd_completed) && 961 (!details->async && !details->postpone)) { 962 if (rd32(hw, hw->aq.asq.len) & I40E_GL_ATQLEN_ATQCRIT_MASK) { 963 i40e_debug(hw, I40E_DEBUG_AQ_MESSAGE, 964 "AQTX: AQ Critical error.\n"); 965 status = I40E_ERR_ADMIN_QUEUE_CRITICAL_ERROR; 966 } else { 967 i40e_debug(hw, I40E_DEBUG_AQ_MESSAGE, 968 "AQTX: Writeback timeout.\n"); 969 status = I40E_ERR_ADMIN_QUEUE_TIMEOUT; 970 } 971 } 972 973 asq_send_command_error: 974 i40e_release_spinlock(&hw->aq.asq_spinlock); 975 return status; 976 } 977 978 /** 979 * i40e_fill_default_direct_cmd_desc - AQ descriptor helper function 980 * @desc: pointer to the temp descriptor (non DMA mem) 981 * @opcode: the opcode can be used to decide which flags to turn off or on 982 * 983 * Fill the desc with default values 984 **/ 985 void i40e_fill_default_direct_cmd_desc(struct i40e_aq_desc *desc, 986 u16 opcode) 987 { 988 /* zero out the desc */ 989 i40e_memset((void *)desc, 0, sizeof(struct i40e_aq_desc), 990 I40E_NONDMA_MEM); 991 desc->opcode = CPU_TO_LE16(opcode); 992 desc->flags = CPU_TO_LE16(I40E_AQ_FLAG_SI); 993 } 994 995 /** 996 * i40e_clean_arq_element 997 * @hw: pointer to the hw struct 998 * @e: event info from the receive descriptor, includes any buffers 999 * @pending: number of events that could be left to process 1000 * 1001 * This function cleans one Admin Receive Queue element and returns 1002 * the contents through e. It can also return how many events are 1003 * left to process through 'pending' 1004 **/ 1005 enum i40e_status_code i40e_clean_arq_element(struct i40e_hw *hw, 1006 struct i40e_arq_event_info *e, 1007 u16 *pending) 1008 { 1009 enum i40e_status_code ret_code = I40E_SUCCESS; 1010 u16 ntc = hw->aq.arq.next_to_clean; 1011 struct i40e_aq_desc *desc; 1012 struct i40e_dma_mem *bi; 1013 u16 desc_idx; 1014 u16 datalen; 1015 u16 flags; 1016 u16 ntu; 1017 1018 /* pre-clean the event info */ 1019 i40e_memset(&e->desc, 0, sizeof(e->desc), I40E_NONDMA_MEM); 1020 1021 /* take the lock before we start messing with the ring */ 1022 i40e_acquire_spinlock(&hw->aq.arq_spinlock); 1023 1024 if (hw->aq.arq.count == 0) { 1025 i40e_debug(hw, I40E_DEBUG_AQ_MESSAGE, 1026 "AQRX: Admin queue not initialized.\n"); 1027 ret_code = I40E_ERR_QUEUE_EMPTY; 1028 goto clean_arq_element_err; 1029 } 1030 1031 /* set next_to_use to head */ 1032 if (!i40e_is_vf(hw)) 1033 ntu = rd32(hw, hw->aq.arq.head) & I40E_PF_ARQH_ARQH_MASK; 1034 else 1035 ntu = rd32(hw, hw->aq.arq.head) & I40E_VF_ARQH1_ARQH_MASK; 1036 if (ntu == ntc) { 1037 /* nothing to do - shouldn't need to update ring's values */ 1038 ret_code = I40E_ERR_ADMIN_QUEUE_NO_WORK; 1039 goto clean_arq_element_out; 1040 } 1041 1042 /* now clean the next descriptor */ 1043 desc = I40E_ADMINQ_DESC(hw->aq.arq, ntc); 1044 desc_idx = ntc; 1045 1046 hw->aq.arq_last_status = 1047 (enum i40e_admin_queue_err)LE16_TO_CPU(desc->retval); 1048 flags = LE16_TO_CPU(desc->flags); 1049 if (flags & I40E_AQ_FLAG_ERR) { 1050 ret_code = I40E_ERR_ADMIN_QUEUE_ERROR; 1051 i40e_debug(hw, 1052 I40E_DEBUG_AQ_MESSAGE, 1053 "AQRX: Event received with error 0x%X.\n", 1054 hw->aq.arq_last_status); 1055 } 1056 1057 i40e_memcpy(&e->desc, desc, sizeof(struct i40e_aq_desc), 1058 I40E_DMA_TO_NONDMA); 1059 datalen = LE16_TO_CPU(desc->datalen); 1060 e->msg_len = min(datalen, e->buf_len); 1061 if (e->msg_buf != NULL && (e->msg_len != 0)) 1062 i40e_memcpy(e->msg_buf, 1063 hw->aq.arq.r.arq_bi[desc_idx].va, 1064 e->msg_len, I40E_DMA_TO_NONDMA); 1065 1066 i40e_debug(hw, I40E_DEBUG_AQ_MESSAGE, "AQRX: desc and buffer:\n"); 1067 i40e_debug_aq(hw, I40E_DEBUG_AQ_COMMAND, (void *)desc, e->msg_buf, 1068 hw->aq.arq_buf_size); 1069 1070 /* Restore the original datalen and buffer address in the desc, 1071 * FW updates datalen to indicate the event message 1072 * size 1073 */ 1074 bi = &hw->aq.arq.r.arq_bi[ntc]; 1075 i40e_memset((void *)desc, 0, sizeof(struct i40e_aq_desc), I40E_DMA_MEM); 1076 1077 desc->flags = CPU_TO_LE16(I40E_AQ_FLAG_BUF); 1078 if (hw->aq.arq_buf_size > I40E_AQ_LARGE_BUF) 1079 desc->flags |= CPU_TO_LE16(I40E_AQ_FLAG_LB); 1080 desc->datalen = CPU_TO_LE16((u16)bi->size); 1081 desc->params.external.addr_high = CPU_TO_LE32(I40E_HI_DWORD(bi->pa)); 1082 desc->params.external.addr_low = CPU_TO_LE32(I40E_LO_DWORD(bi->pa)); 1083 1084 /* set tail = the last cleaned desc index. */ 1085 wr32(hw, hw->aq.arq.tail, ntc); 1086 /* ntc is updated to tail + 1 */ 1087 ntc++; 1088 if (ntc == hw->aq.num_arq_entries) 1089 ntc = 0; 1090 hw->aq.arq.next_to_clean = ntc; 1091 hw->aq.arq.next_to_use = ntu; 1092 1093 i40e_nvmupd_check_wait_event(hw, LE16_TO_CPU(e->desc.opcode), &e->desc); 1094 clean_arq_element_out: 1095 /* Set pending if needed, unlock and return */ 1096 if (pending != NULL) 1097 *pending = (ntc > ntu ? hw->aq.arq.count : 0) + (ntu - ntc); 1098 clean_arq_element_err: 1099 i40e_release_spinlock(&hw->aq.arq_spinlock); 1100 1101 return ret_code; 1102 } 1103 1104