1 /* QLogic qed NIC Driver 2 * Copyright (c) 2015-2017 QLogic Corporation 3 * 4 * This software is available to you under a choice of one of two 5 * licenses. You may choose to be licensed under the terms of the GNU 6 * General Public License (GPL) Version 2, available from the file 7 * COPYING in the main directory of this source tree, or the 8 * OpenIB.org BSD license below: 9 * 10 * Redistribution and use in source and binary forms, with or 11 * without modification, are permitted provided that the following 12 * conditions are met: 13 * 14 * - Redistributions of source code must retain the above 15 * copyright notice, this list of conditions and the following 16 * disclaimer. 17 * 18 * - Redistributions in binary form must reproduce the above 19 * copyright notice, this list of conditions and the following 20 * disclaimer in the documentation and /or other materials 21 * provided with the distribution. 22 * 23 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 24 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 25 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 26 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS 27 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN 28 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN 29 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 30 * SOFTWARE. 31 */ 32 33 #include <linux/types.h> 34 #include <linux/io.h> 35 #include <linux/delay.h> 36 #include <linux/dma-mapping.h> 37 #include <linux/errno.h> 38 #include <linux/kernel.h> 39 #include <linux/list.h> 40 #include <linux/mutex.h> 41 #include <linux/pci.h> 42 #include <linux/slab.h> 43 #include <linux/spinlock.h> 44 #include <linux/string.h> 45 #include <linux/qed/qed_chain.h> 46 #include "qed.h" 47 #include "qed_hsi.h" 48 #include "qed_hw.h" 49 #include "qed_reg_addr.h" 50 #include "qed_sriov.h" 51 52 #define QED_BAR_ACQUIRE_TIMEOUT 1000 53 54 /* Invalid values */ 55 #define QED_BAR_INVALID_OFFSET (cpu_to_le32(-1)) 56 57 struct qed_ptt { 58 struct list_head list_entry; 59 unsigned int idx; 60 struct pxp_ptt_entry pxp; 61 u8 hwfn_id; 62 }; 63 64 struct qed_ptt_pool { 65 struct list_head free_list; 66 spinlock_t lock; /* ptt synchronized access */ 67 struct qed_ptt ptts[PXP_EXTERNAL_BAR_PF_WINDOW_NUM]; 68 }; 69 70 int qed_ptt_pool_alloc(struct qed_hwfn *p_hwfn) 71 { 72 struct qed_ptt_pool *p_pool = kmalloc(sizeof(*p_pool), GFP_KERNEL); 73 int i; 74 75 if (!p_pool) 76 return -ENOMEM; 77 78 INIT_LIST_HEAD(&p_pool->free_list); 79 for (i = 0; i < PXP_EXTERNAL_BAR_PF_WINDOW_NUM; i++) { 80 p_pool->ptts[i].idx = i; 81 p_pool->ptts[i].pxp.offset = QED_BAR_INVALID_OFFSET; 82 p_pool->ptts[i].pxp.pretend.control = 0; 83 p_pool->ptts[i].hwfn_id = p_hwfn->my_id; 84 if (i >= RESERVED_PTT_MAX) 85 list_add(&p_pool->ptts[i].list_entry, 86 &p_pool->free_list); 87 } 88 89 p_hwfn->p_ptt_pool = p_pool; 90 spin_lock_init(&p_pool->lock); 91 92 return 0; 93 } 94 95 void qed_ptt_invalidate(struct qed_hwfn *p_hwfn) 96 { 97 struct qed_ptt *p_ptt; 98 int i; 99 100 for (i = 0; i < PXP_EXTERNAL_BAR_PF_WINDOW_NUM; i++) { 101 p_ptt = &p_hwfn->p_ptt_pool->ptts[i]; 102 p_ptt->pxp.offset = QED_BAR_INVALID_OFFSET; 103 } 104 } 105 106 void qed_ptt_pool_free(struct qed_hwfn *p_hwfn) 107 { 108 kfree(p_hwfn->p_ptt_pool); 109 p_hwfn->p_ptt_pool = NULL; 110 } 111 112 struct qed_ptt *qed_ptt_acquire(struct qed_hwfn *p_hwfn) 113 { 114 struct qed_ptt *p_ptt; 115 unsigned int i; 116 117 /* Take the free PTT from the list */ 118 for (i = 0; i < QED_BAR_ACQUIRE_TIMEOUT; i++) { 119 spin_lock_bh(&p_hwfn->p_ptt_pool->lock); 120 121 if (!list_empty(&p_hwfn->p_ptt_pool->free_list)) { 122 p_ptt = list_first_entry(&p_hwfn->p_ptt_pool->free_list, 123 struct qed_ptt, list_entry); 124 list_del(&p_ptt->list_entry); 125 126 spin_unlock_bh(&p_hwfn->p_ptt_pool->lock); 127 128 DP_VERBOSE(p_hwfn, NETIF_MSG_HW, 129 "allocated ptt %d\n", p_ptt->idx); 130 return p_ptt; 131 } 132 133 spin_unlock_bh(&p_hwfn->p_ptt_pool->lock); 134 usleep_range(1000, 2000); 135 } 136 137 DP_NOTICE(p_hwfn, "PTT acquire timeout - failed to allocate PTT\n"); 138 return NULL; 139 } 140 141 void qed_ptt_release(struct qed_hwfn *p_hwfn, struct qed_ptt *p_ptt) 142 { 143 spin_lock_bh(&p_hwfn->p_ptt_pool->lock); 144 list_add(&p_ptt->list_entry, &p_hwfn->p_ptt_pool->free_list); 145 spin_unlock_bh(&p_hwfn->p_ptt_pool->lock); 146 } 147 148 u32 qed_ptt_get_hw_addr(struct qed_hwfn *p_hwfn, struct qed_ptt *p_ptt) 149 { 150 /* The HW is using DWORDS and we need to translate it to Bytes */ 151 return le32_to_cpu(p_ptt->pxp.offset) << 2; 152 } 153 154 static u32 qed_ptt_config_addr(struct qed_ptt *p_ptt) 155 { 156 return PXP_PF_WINDOW_ADMIN_PER_PF_START + 157 p_ptt->idx * sizeof(struct pxp_ptt_entry); 158 } 159 160 u32 qed_ptt_get_bar_addr(struct qed_ptt *p_ptt) 161 { 162 return PXP_EXTERNAL_BAR_PF_WINDOW_START + 163 p_ptt->idx * PXP_EXTERNAL_BAR_PF_WINDOW_SINGLE_SIZE; 164 } 165 166 void qed_ptt_set_win(struct qed_hwfn *p_hwfn, 167 struct qed_ptt *p_ptt, u32 new_hw_addr) 168 { 169 u32 prev_hw_addr; 170 171 prev_hw_addr = qed_ptt_get_hw_addr(p_hwfn, p_ptt); 172 173 if (new_hw_addr == prev_hw_addr) 174 return; 175 176 /* Update PTT entery in admin window */ 177 DP_VERBOSE(p_hwfn, NETIF_MSG_HW, 178 "Updating PTT entry %d to offset 0x%x\n", 179 p_ptt->idx, new_hw_addr); 180 181 /* The HW is using DWORDS and the address is in Bytes */ 182 p_ptt->pxp.offset = cpu_to_le32(new_hw_addr >> 2); 183 184 REG_WR(p_hwfn, 185 qed_ptt_config_addr(p_ptt) + 186 offsetof(struct pxp_ptt_entry, offset), 187 le32_to_cpu(p_ptt->pxp.offset)); 188 } 189 190 static u32 qed_set_ptt(struct qed_hwfn *p_hwfn, 191 struct qed_ptt *p_ptt, u32 hw_addr) 192 { 193 u32 win_hw_addr = qed_ptt_get_hw_addr(p_hwfn, p_ptt); 194 u32 offset; 195 196 offset = hw_addr - win_hw_addr; 197 198 if (p_ptt->hwfn_id != p_hwfn->my_id) 199 DP_NOTICE(p_hwfn, 200 "ptt[%d] of hwfn[%02x] is used by hwfn[%02x]!\n", 201 p_ptt->idx, p_ptt->hwfn_id, p_hwfn->my_id); 202 203 /* Verify the address is within the window */ 204 if (hw_addr < win_hw_addr || 205 offset >= PXP_EXTERNAL_BAR_PF_WINDOW_SINGLE_SIZE) { 206 qed_ptt_set_win(p_hwfn, p_ptt, hw_addr); 207 offset = 0; 208 } 209 210 return qed_ptt_get_bar_addr(p_ptt) + offset; 211 } 212 213 struct qed_ptt *qed_get_reserved_ptt(struct qed_hwfn *p_hwfn, 214 enum reserved_ptts ptt_idx) 215 { 216 if (ptt_idx >= RESERVED_PTT_MAX) { 217 DP_NOTICE(p_hwfn, 218 "Requested PTT %d is out of range\n", ptt_idx); 219 return NULL; 220 } 221 222 return &p_hwfn->p_ptt_pool->ptts[ptt_idx]; 223 } 224 225 void qed_wr(struct qed_hwfn *p_hwfn, 226 struct qed_ptt *p_ptt, 227 u32 hw_addr, u32 val) 228 { 229 u32 bar_addr = qed_set_ptt(p_hwfn, p_ptt, hw_addr); 230 231 REG_WR(p_hwfn, bar_addr, val); 232 DP_VERBOSE(p_hwfn, NETIF_MSG_HW, 233 "bar_addr 0x%x, hw_addr 0x%x, val 0x%x\n", 234 bar_addr, hw_addr, val); 235 } 236 237 u32 qed_rd(struct qed_hwfn *p_hwfn, 238 struct qed_ptt *p_ptt, 239 u32 hw_addr) 240 { 241 u32 bar_addr = qed_set_ptt(p_hwfn, p_ptt, hw_addr); 242 u32 val = REG_RD(p_hwfn, bar_addr); 243 244 DP_VERBOSE(p_hwfn, NETIF_MSG_HW, 245 "bar_addr 0x%x, hw_addr 0x%x, val 0x%x\n", 246 bar_addr, hw_addr, val); 247 248 return val; 249 } 250 251 static void qed_memcpy_hw(struct qed_hwfn *p_hwfn, 252 struct qed_ptt *p_ptt, 253 void *addr, u32 hw_addr, size_t n, bool to_device) 254 { 255 u32 dw_count, *host_addr, hw_offset; 256 size_t quota, done = 0; 257 u32 __iomem *reg_addr; 258 259 while (done < n) { 260 quota = min_t(size_t, n - done, 261 PXP_EXTERNAL_BAR_PF_WINDOW_SINGLE_SIZE); 262 263 if (IS_PF(p_hwfn->cdev)) { 264 qed_ptt_set_win(p_hwfn, p_ptt, hw_addr + done); 265 hw_offset = qed_ptt_get_bar_addr(p_ptt); 266 } else { 267 hw_offset = hw_addr + done; 268 } 269 270 dw_count = quota / 4; 271 host_addr = (u32 *)((u8 *)addr + done); 272 reg_addr = (u32 __iomem *)REG_ADDR(p_hwfn, hw_offset); 273 if (to_device) 274 while (dw_count--) 275 DIRECT_REG_WR(reg_addr++, *host_addr++); 276 else 277 while (dw_count--) 278 *host_addr++ = DIRECT_REG_RD(reg_addr++); 279 280 done += quota; 281 } 282 } 283 284 void qed_memcpy_from(struct qed_hwfn *p_hwfn, 285 struct qed_ptt *p_ptt, void *dest, u32 hw_addr, size_t n) 286 { 287 DP_VERBOSE(p_hwfn, NETIF_MSG_HW, 288 "hw_addr 0x%x, dest %p hw_addr 0x%x, size %lu\n", 289 hw_addr, dest, hw_addr, (unsigned long)n); 290 291 qed_memcpy_hw(p_hwfn, p_ptt, dest, hw_addr, n, false); 292 } 293 294 void qed_memcpy_to(struct qed_hwfn *p_hwfn, 295 struct qed_ptt *p_ptt, u32 hw_addr, void *src, size_t n) 296 { 297 DP_VERBOSE(p_hwfn, NETIF_MSG_HW, 298 "hw_addr 0x%x, hw_addr 0x%x, src %p size %lu\n", 299 hw_addr, hw_addr, src, (unsigned long)n); 300 301 qed_memcpy_hw(p_hwfn, p_ptt, src, hw_addr, n, true); 302 } 303 304 void qed_fid_pretend(struct qed_hwfn *p_hwfn, struct qed_ptt *p_ptt, u16 fid) 305 { 306 u16 control = 0; 307 308 SET_FIELD(control, PXP_PRETEND_CMD_IS_CONCRETE, 1); 309 SET_FIELD(control, PXP_PRETEND_CMD_PRETEND_FUNCTION, 1); 310 311 /* Every pretend undos previous pretends, including 312 * previous port pretend. 313 */ 314 SET_FIELD(control, PXP_PRETEND_CMD_PORT, 0); 315 SET_FIELD(control, PXP_PRETEND_CMD_USE_PORT, 0); 316 SET_FIELD(control, PXP_PRETEND_CMD_PRETEND_PORT, 1); 317 318 if (!GET_FIELD(fid, PXP_CONCRETE_FID_VFVALID)) 319 fid = GET_FIELD(fid, PXP_CONCRETE_FID_PFID); 320 321 p_ptt->pxp.pretend.control = cpu_to_le16(control); 322 p_ptt->pxp.pretend.fid.concrete_fid.fid = cpu_to_le16(fid); 323 324 REG_WR(p_hwfn, 325 qed_ptt_config_addr(p_ptt) + 326 offsetof(struct pxp_ptt_entry, pretend), 327 *(u32 *)&p_ptt->pxp.pretend); 328 } 329 330 void qed_port_pretend(struct qed_hwfn *p_hwfn, 331 struct qed_ptt *p_ptt, u8 port_id) 332 { 333 u16 control = 0; 334 335 SET_FIELD(control, PXP_PRETEND_CMD_PORT, port_id); 336 SET_FIELD(control, PXP_PRETEND_CMD_USE_PORT, 1); 337 SET_FIELD(control, PXP_PRETEND_CMD_PRETEND_PORT, 1); 338 339 p_ptt->pxp.pretend.control = cpu_to_le16(control); 340 341 REG_WR(p_hwfn, 342 qed_ptt_config_addr(p_ptt) + 343 offsetof(struct pxp_ptt_entry, pretend), 344 *(u32 *)&p_ptt->pxp.pretend); 345 } 346 347 void qed_port_unpretend(struct qed_hwfn *p_hwfn, struct qed_ptt *p_ptt) 348 { 349 u16 control = 0; 350 351 SET_FIELD(control, PXP_PRETEND_CMD_PORT, 0); 352 SET_FIELD(control, PXP_PRETEND_CMD_USE_PORT, 0); 353 SET_FIELD(control, PXP_PRETEND_CMD_PRETEND_PORT, 1); 354 355 p_ptt->pxp.pretend.control = cpu_to_le16(control); 356 357 REG_WR(p_hwfn, 358 qed_ptt_config_addr(p_ptt) + 359 offsetof(struct pxp_ptt_entry, pretend), 360 *(u32 *)&p_ptt->pxp.pretend); 361 } 362 363 void qed_port_fid_pretend(struct qed_hwfn *p_hwfn, 364 struct qed_ptt *p_ptt, u8 port_id, u16 fid) 365 { 366 u16 control = 0; 367 368 SET_FIELD(control, PXP_PRETEND_CMD_PORT, port_id); 369 SET_FIELD(control, PXP_PRETEND_CMD_USE_PORT, 1); 370 SET_FIELD(control, PXP_PRETEND_CMD_PRETEND_PORT, 1); 371 SET_FIELD(control, PXP_PRETEND_CMD_IS_CONCRETE, 1); 372 SET_FIELD(control, PXP_PRETEND_CMD_PRETEND_FUNCTION, 1); 373 if (!GET_FIELD(fid, PXP_CONCRETE_FID_VFVALID)) 374 fid = GET_FIELD(fid, PXP_CONCRETE_FID_PFID); 375 p_ptt->pxp.pretend.control = cpu_to_le16(control); 376 p_ptt->pxp.pretend.fid.concrete_fid.fid = cpu_to_le16(fid); 377 REG_WR(p_hwfn, 378 qed_ptt_config_addr(p_ptt) + 379 offsetof(struct pxp_ptt_entry, pretend), 380 *(u32 *)&p_ptt->pxp.pretend); 381 } 382 383 u32 qed_vfid_to_concrete(struct qed_hwfn *p_hwfn, u8 vfid) 384 { 385 u32 concrete_fid = 0; 386 387 SET_FIELD(concrete_fid, PXP_CONCRETE_FID_PFID, p_hwfn->rel_pf_id); 388 SET_FIELD(concrete_fid, PXP_CONCRETE_FID_VFID, vfid); 389 SET_FIELD(concrete_fid, PXP_CONCRETE_FID_VFVALID, 1); 390 391 return concrete_fid; 392 } 393 394 /* DMAE */ 395 static void qed_dmae_opcode(struct qed_hwfn *p_hwfn, 396 const u8 is_src_type_grc, 397 const u8 is_dst_type_grc, 398 struct qed_dmae_params *p_params) 399 { 400 u16 opcode_b = 0; 401 u32 opcode = 0; 402 403 /* Whether the source is the PCIe or the GRC. 404 * 0- The source is the PCIe 405 * 1- The source is the GRC. 406 */ 407 opcode |= (is_src_type_grc ? DMAE_CMD_SRC_MASK_GRC 408 : DMAE_CMD_SRC_MASK_PCIE) << 409 DMAE_CMD_SRC_SHIFT; 410 opcode |= ((p_hwfn->rel_pf_id & DMAE_CMD_SRC_PF_ID_MASK) << 411 DMAE_CMD_SRC_PF_ID_SHIFT); 412 413 /* The destination of the DMA can be: 0-None 1-PCIe 2-GRC 3-None */ 414 opcode |= (is_dst_type_grc ? DMAE_CMD_DST_MASK_GRC 415 : DMAE_CMD_DST_MASK_PCIE) << 416 DMAE_CMD_DST_SHIFT; 417 opcode |= ((p_hwfn->rel_pf_id & DMAE_CMD_DST_PF_ID_MASK) << 418 DMAE_CMD_DST_PF_ID_SHIFT); 419 420 /* Whether to write a completion word to the completion destination: 421 * 0-Do not write a completion word 422 * 1-Write the completion word 423 */ 424 opcode |= (DMAE_CMD_COMP_WORD_EN_MASK << DMAE_CMD_COMP_WORD_EN_SHIFT); 425 opcode |= (DMAE_CMD_SRC_ADDR_RESET_MASK << 426 DMAE_CMD_SRC_ADDR_RESET_SHIFT); 427 428 if (p_params->flags & QED_DMAE_FLAG_COMPLETION_DST) 429 opcode |= (1 << DMAE_CMD_COMP_FUNC_SHIFT); 430 431 opcode |= (DMAE_CMD_ENDIANITY << DMAE_CMD_ENDIANITY_MODE_SHIFT); 432 433 opcode |= ((p_hwfn->port_id) << DMAE_CMD_PORT_ID_SHIFT); 434 435 /* reset source address in next go */ 436 opcode |= (DMAE_CMD_SRC_ADDR_RESET_MASK << 437 DMAE_CMD_SRC_ADDR_RESET_SHIFT); 438 439 /* reset dest address in next go */ 440 opcode |= (DMAE_CMD_DST_ADDR_RESET_MASK << 441 DMAE_CMD_DST_ADDR_RESET_SHIFT); 442 443 /* SRC/DST VFID: all 1's - pf, otherwise VF id */ 444 if (p_params->flags & QED_DMAE_FLAG_VF_SRC) { 445 opcode |= 1 << DMAE_CMD_SRC_VF_ID_VALID_SHIFT; 446 opcode_b |= p_params->src_vfid << DMAE_CMD_SRC_VF_ID_SHIFT; 447 } else { 448 opcode_b |= DMAE_CMD_SRC_VF_ID_MASK << 449 DMAE_CMD_SRC_VF_ID_SHIFT; 450 } 451 452 if (p_params->flags & QED_DMAE_FLAG_VF_DST) { 453 opcode |= 1 << DMAE_CMD_DST_VF_ID_VALID_SHIFT; 454 opcode_b |= p_params->dst_vfid << DMAE_CMD_DST_VF_ID_SHIFT; 455 } else { 456 opcode_b |= DMAE_CMD_DST_VF_ID_MASK << DMAE_CMD_DST_VF_ID_SHIFT; 457 } 458 459 p_hwfn->dmae_info.p_dmae_cmd->opcode = cpu_to_le32(opcode); 460 p_hwfn->dmae_info.p_dmae_cmd->opcode_b = cpu_to_le16(opcode_b); 461 } 462 463 u32 qed_dmae_idx_to_go_cmd(u8 idx) 464 { 465 /* All the DMAE 'go' registers form an array in internal memory */ 466 return DMAE_REG_GO_C0 + (idx << 2); 467 } 468 469 static int qed_dmae_post_command(struct qed_hwfn *p_hwfn, 470 struct qed_ptt *p_ptt) 471 { 472 struct dmae_cmd *p_command = p_hwfn->dmae_info.p_dmae_cmd; 473 u8 idx_cmd = p_hwfn->dmae_info.channel, i; 474 int qed_status = 0; 475 476 /* verify address is not NULL */ 477 if ((((!p_command->dst_addr_lo) && (!p_command->dst_addr_hi)) || 478 ((!p_command->src_addr_lo) && (!p_command->src_addr_hi)))) { 479 DP_NOTICE(p_hwfn, 480 "source or destination address 0 idx_cmd=%d\n" 481 "opcode = [0x%08x,0x%04x] len=0x%x src=0x%x:%x dst=0x%x:%x\n", 482 idx_cmd, 483 le32_to_cpu(p_command->opcode), 484 le16_to_cpu(p_command->opcode_b), 485 le16_to_cpu(p_command->length_dw), 486 le32_to_cpu(p_command->src_addr_hi), 487 le32_to_cpu(p_command->src_addr_lo), 488 le32_to_cpu(p_command->dst_addr_hi), 489 le32_to_cpu(p_command->dst_addr_lo)); 490 491 return -EINVAL; 492 } 493 494 DP_VERBOSE(p_hwfn, 495 NETIF_MSG_HW, 496 "Posting DMAE command [idx %d]: opcode = [0x%08x,0x%04x] len=0x%x src=0x%x:%x dst=0x%x:%x\n", 497 idx_cmd, 498 le32_to_cpu(p_command->opcode), 499 le16_to_cpu(p_command->opcode_b), 500 le16_to_cpu(p_command->length_dw), 501 le32_to_cpu(p_command->src_addr_hi), 502 le32_to_cpu(p_command->src_addr_lo), 503 le32_to_cpu(p_command->dst_addr_hi), 504 le32_to_cpu(p_command->dst_addr_lo)); 505 506 /* Copy the command to DMAE - need to do it before every call 507 * for source/dest address no reset. 508 * The first 9 DWs are the command registers, the 10 DW is the 509 * GO register, and the rest are result registers 510 * (which are read only by the client). 511 */ 512 for (i = 0; i < DMAE_CMD_SIZE; i++) { 513 u32 data = (i < DMAE_CMD_SIZE_TO_FILL) ? 514 *(((u32 *)p_command) + i) : 0; 515 516 qed_wr(p_hwfn, p_ptt, 517 DMAE_REG_CMD_MEM + 518 (idx_cmd * DMAE_CMD_SIZE * sizeof(u32)) + 519 (i * sizeof(u32)), data); 520 } 521 522 qed_wr(p_hwfn, p_ptt, qed_dmae_idx_to_go_cmd(idx_cmd), DMAE_GO_VALUE); 523 524 return qed_status; 525 } 526 527 int qed_dmae_info_alloc(struct qed_hwfn *p_hwfn) 528 { 529 dma_addr_t *p_addr = &p_hwfn->dmae_info.completion_word_phys_addr; 530 struct dmae_cmd **p_cmd = &p_hwfn->dmae_info.p_dmae_cmd; 531 u32 **p_buff = &p_hwfn->dmae_info.p_intermediate_buffer; 532 u32 **p_comp = &p_hwfn->dmae_info.p_completion_word; 533 534 *p_comp = dma_alloc_coherent(&p_hwfn->cdev->pdev->dev, 535 sizeof(u32), p_addr, GFP_KERNEL); 536 if (!*p_comp) 537 goto err; 538 539 p_addr = &p_hwfn->dmae_info.dmae_cmd_phys_addr; 540 *p_cmd = dma_alloc_coherent(&p_hwfn->cdev->pdev->dev, 541 sizeof(struct dmae_cmd), 542 p_addr, GFP_KERNEL); 543 if (!*p_cmd) 544 goto err; 545 546 p_addr = &p_hwfn->dmae_info.intermediate_buffer_phys_addr; 547 *p_buff = dma_alloc_coherent(&p_hwfn->cdev->pdev->dev, 548 sizeof(u32) * DMAE_MAX_RW_SIZE, 549 p_addr, GFP_KERNEL); 550 if (!*p_buff) 551 goto err; 552 553 p_hwfn->dmae_info.channel = p_hwfn->rel_pf_id; 554 555 return 0; 556 err: 557 qed_dmae_info_free(p_hwfn); 558 return -ENOMEM; 559 } 560 561 void qed_dmae_info_free(struct qed_hwfn *p_hwfn) 562 { 563 dma_addr_t p_phys; 564 565 /* Just make sure no one is in the middle */ 566 mutex_lock(&p_hwfn->dmae_info.mutex); 567 568 if (p_hwfn->dmae_info.p_completion_word) { 569 p_phys = p_hwfn->dmae_info.completion_word_phys_addr; 570 dma_free_coherent(&p_hwfn->cdev->pdev->dev, 571 sizeof(u32), 572 p_hwfn->dmae_info.p_completion_word, p_phys); 573 p_hwfn->dmae_info.p_completion_word = NULL; 574 } 575 576 if (p_hwfn->dmae_info.p_dmae_cmd) { 577 p_phys = p_hwfn->dmae_info.dmae_cmd_phys_addr; 578 dma_free_coherent(&p_hwfn->cdev->pdev->dev, 579 sizeof(struct dmae_cmd), 580 p_hwfn->dmae_info.p_dmae_cmd, p_phys); 581 p_hwfn->dmae_info.p_dmae_cmd = NULL; 582 } 583 584 if (p_hwfn->dmae_info.p_intermediate_buffer) { 585 p_phys = p_hwfn->dmae_info.intermediate_buffer_phys_addr; 586 dma_free_coherent(&p_hwfn->cdev->pdev->dev, 587 sizeof(u32) * DMAE_MAX_RW_SIZE, 588 p_hwfn->dmae_info.p_intermediate_buffer, 589 p_phys); 590 p_hwfn->dmae_info.p_intermediate_buffer = NULL; 591 } 592 593 mutex_unlock(&p_hwfn->dmae_info.mutex); 594 } 595 596 static int qed_dmae_operation_wait(struct qed_hwfn *p_hwfn) 597 { 598 u32 wait_cnt_limit = 10000, wait_cnt = 0; 599 int qed_status = 0; 600 601 barrier(); 602 while (*p_hwfn->dmae_info.p_completion_word != DMAE_COMPLETION_VAL) { 603 udelay(DMAE_MIN_WAIT_TIME); 604 if (++wait_cnt > wait_cnt_limit) { 605 DP_NOTICE(p_hwfn->cdev, 606 "Timed-out waiting for operation to complete. Completion word is 0x%08x expected 0x%08x.\n", 607 *p_hwfn->dmae_info.p_completion_word, 608 DMAE_COMPLETION_VAL); 609 qed_status = -EBUSY; 610 break; 611 } 612 613 /* to sync the completion_word since we are not 614 * using the volatile keyword for p_completion_word 615 */ 616 barrier(); 617 } 618 619 if (qed_status == 0) 620 *p_hwfn->dmae_info.p_completion_word = 0; 621 622 return qed_status; 623 } 624 625 static int qed_dmae_execute_sub_operation(struct qed_hwfn *p_hwfn, 626 struct qed_ptt *p_ptt, 627 u64 src_addr, 628 u64 dst_addr, 629 u8 src_type, 630 u8 dst_type, 631 u32 length_dw) 632 { 633 dma_addr_t phys = p_hwfn->dmae_info.intermediate_buffer_phys_addr; 634 struct dmae_cmd *cmd = p_hwfn->dmae_info.p_dmae_cmd; 635 int qed_status = 0; 636 637 switch (src_type) { 638 case QED_DMAE_ADDRESS_GRC: 639 case QED_DMAE_ADDRESS_HOST_PHYS: 640 cmd->src_addr_hi = cpu_to_le32(upper_32_bits(src_addr)); 641 cmd->src_addr_lo = cpu_to_le32(lower_32_bits(src_addr)); 642 break; 643 /* for virtual source addresses we use the intermediate buffer. */ 644 case QED_DMAE_ADDRESS_HOST_VIRT: 645 cmd->src_addr_hi = cpu_to_le32(upper_32_bits(phys)); 646 cmd->src_addr_lo = cpu_to_le32(lower_32_bits(phys)); 647 memcpy(&p_hwfn->dmae_info.p_intermediate_buffer[0], 648 (void *)(uintptr_t)src_addr, 649 length_dw * sizeof(u32)); 650 break; 651 default: 652 return -EINVAL; 653 } 654 655 switch (dst_type) { 656 case QED_DMAE_ADDRESS_GRC: 657 case QED_DMAE_ADDRESS_HOST_PHYS: 658 cmd->dst_addr_hi = cpu_to_le32(upper_32_bits(dst_addr)); 659 cmd->dst_addr_lo = cpu_to_le32(lower_32_bits(dst_addr)); 660 break; 661 /* for virtual source addresses we use the intermediate buffer. */ 662 case QED_DMAE_ADDRESS_HOST_VIRT: 663 cmd->dst_addr_hi = cpu_to_le32(upper_32_bits(phys)); 664 cmd->dst_addr_lo = cpu_to_le32(lower_32_bits(phys)); 665 break; 666 default: 667 return -EINVAL; 668 } 669 670 cmd->length_dw = cpu_to_le16((u16)length_dw); 671 672 qed_dmae_post_command(p_hwfn, p_ptt); 673 674 qed_status = qed_dmae_operation_wait(p_hwfn); 675 676 if (qed_status) { 677 DP_NOTICE(p_hwfn, 678 "qed_dmae_host2grc: Wait Failed. source_addr 0x%llx, grc_addr 0x%llx, size_in_dwords 0x%x\n", 679 src_addr, dst_addr, length_dw); 680 return qed_status; 681 } 682 683 if (dst_type == QED_DMAE_ADDRESS_HOST_VIRT) 684 memcpy((void *)(uintptr_t)(dst_addr), 685 &p_hwfn->dmae_info.p_intermediate_buffer[0], 686 length_dw * sizeof(u32)); 687 688 return 0; 689 } 690 691 static int qed_dmae_execute_command(struct qed_hwfn *p_hwfn, 692 struct qed_ptt *p_ptt, 693 u64 src_addr, u64 dst_addr, 694 u8 src_type, u8 dst_type, 695 u32 size_in_dwords, 696 struct qed_dmae_params *p_params) 697 { 698 dma_addr_t phys = p_hwfn->dmae_info.completion_word_phys_addr; 699 u16 length_cur = 0, i = 0, cnt_split = 0, length_mod = 0; 700 struct dmae_cmd *cmd = p_hwfn->dmae_info.p_dmae_cmd; 701 u64 src_addr_split = 0, dst_addr_split = 0; 702 u16 length_limit = DMAE_MAX_RW_SIZE; 703 int qed_status = 0; 704 u32 offset = 0; 705 706 if (p_hwfn->cdev->recov_in_prog) { 707 DP_VERBOSE(p_hwfn, 708 NETIF_MSG_HW, 709 "Recovery is in progress. Avoid DMAE transaction [{src: addr 0x%llx, type %d}, {dst: addr 0x%llx, type %d}, size %d].\n", 710 src_addr, src_type, dst_addr, dst_type, 711 size_in_dwords); 712 713 /* Let the flow complete w/o any error handling */ 714 return 0; 715 } 716 717 qed_dmae_opcode(p_hwfn, 718 (src_type == QED_DMAE_ADDRESS_GRC), 719 (dst_type == QED_DMAE_ADDRESS_GRC), 720 p_params); 721 722 cmd->comp_addr_lo = cpu_to_le32(lower_32_bits(phys)); 723 cmd->comp_addr_hi = cpu_to_le32(upper_32_bits(phys)); 724 cmd->comp_val = cpu_to_le32(DMAE_COMPLETION_VAL); 725 726 /* Check if the grc_addr is valid like < MAX_GRC_OFFSET */ 727 cnt_split = size_in_dwords / length_limit; 728 length_mod = size_in_dwords % length_limit; 729 730 src_addr_split = src_addr; 731 dst_addr_split = dst_addr; 732 733 for (i = 0; i <= cnt_split; i++) { 734 offset = length_limit * i; 735 736 if (!(p_params->flags & QED_DMAE_FLAG_RW_REPL_SRC)) { 737 if (src_type == QED_DMAE_ADDRESS_GRC) 738 src_addr_split = src_addr + offset; 739 else 740 src_addr_split = src_addr + (offset * 4); 741 } 742 743 if (dst_type == QED_DMAE_ADDRESS_GRC) 744 dst_addr_split = dst_addr + offset; 745 else 746 dst_addr_split = dst_addr + (offset * 4); 747 748 length_cur = (cnt_split == i) ? length_mod : length_limit; 749 750 /* might be zero on last iteration */ 751 if (!length_cur) 752 continue; 753 754 qed_status = qed_dmae_execute_sub_operation(p_hwfn, 755 p_ptt, 756 src_addr_split, 757 dst_addr_split, 758 src_type, 759 dst_type, 760 length_cur); 761 if (qed_status) { 762 DP_NOTICE(p_hwfn, 763 "qed_dmae_execute_sub_operation Failed with error 0x%x. source_addr 0x%llx, destination addr 0x%llx, size_in_dwords 0x%x\n", 764 qed_status, src_addr, dst_addr, length_cur); 765 break; 766 } 767 } 768 769 return qed_status; 770 } 771 772 int qed_dmae_host2grc(struct qed_hwfn *p_hwfn, 773 struct qed_ptt *p_ptt, 774 u64 source_addr, u32 grc_addr, u32 size_in_dwords, u32 flags) 775 { 776 u32 grc_addr_in_dw = grc_addr / sizeof(u32); 777 struct qed_dmae_params params; 778 int rc; 779 780 memset(¶ms, 0, sizeof(struct qed_dmae_params)); 781 params.flags = flags; 782 783 mutex_lock(&p_hwfn->dmae_info.mutex); 784 785 rc = qed_dmae_execute_command(p_hwfn, p_ptt, source_addr, 786 grc_addr_in_dw, 787 QED_DMAE_ADDRESS_HOST_VIRT, 788 QED_DMAE_ADDRESS_GRC, 789 size_in_dwords, ¶ms); 790 791 mutex_unlock(&p_hwfn->dmae_info.mutex); 792 793 return rc; 794 } 795 796 int qed_dmae_grc2host(struct qed_hwfn *p_hwfn, 797 struct qed_ptt *p_ptt, 798 u32 grc_addr, 799 dma_addr_t dest_addr, u32 size_in_dwords, u32 flags) 800 { 801 u32 grc_addr_in_dw = grc_addr / sizeof(u32); 802 struct qed_dmae_params params; 803 int rc; 804 805 memset(¶ms, 0, sizeof(struct qed_dmae_params)); 806 params.flags = flags; 807 808 mutex_lock(&p_hwfn->dmae_info.mutex); 809 810 rc = qed_dmae_execute_command(p_hwfn, p_ptt, grc_addr_in_dw, 811 dest_addr, QED_DMAE_ADDRESS_GRC, 812 QED_DMAE_ADDRESS_HOST_VIRT, 813 size_in_dwords, ¶ms); 814 815 mutex_unlock(&p_hwfn->dmae_info.mutex); 816 817 return rc; 818 } 819 820 int qed_dmae_host2host(struct qed_hwfn *p_hwfn, 821 struct qed_ptt *p_ptt, 822 dma_addr_t source_addr, 823 dma_addr_t dest_addr, 824 u32 size_in_dwords, struct qed_dmae_params *p_params) 825 { 826 int rc; 827 828 mutex_lock(&(p_hwfn->dmae_info.mutex)); 829 830 rc = qed_dmae_execute_command(p_hwfn, p_ptt, source_addr, 831 dest_addr, 832 QED_DMAE_ADDRESS_HOST_PHYS, 833 QED_DMAE_ADDRESS_HOST_PHYS, 834 size_in_dwords, p_params); 835 836 mutex_unlock(&(p_hwfn->dmae_info.mutex)); 837 838 return rc; 839 } 840 841 int qed_dmae_sanity(struct qed_hwfn *p_hwfn, 842 struct qed_ptt *p_ptt, const char *phase) 843 { 844 u32 size = PAGE_SIZE / 2, val; 845 struct qed_dmae_params params; 846 int rc = 0; 847 dma_addr_t p_phys; 848 void *p_virt; 849 u32 *p_tmp; 850 851 p_virt = dma_alloc_coherent(&p_hwfn->cdev->pdev->dev, 852 2 * size, &p_phys, GFP_KERNEL); 853 if (!p_virt) { 854 DP_NOTICE(p_hwfn, 855 "DMAE sanity [%s]: failed to allocate memory\n", 856 phase); 857 return -ENOMEM; 858 } 859 860 /* Fill the bottom half of the allocated memory with a known pattern */ 861 for (p_tmp = (u32 *)p_virt; 862 p_tmp < (u32 *)((u8 *)p_virt + size); p_tmp++) { 863 /* Save the address itself as the value */ 864 val = (u32)(uintptr_t)p_tmp; 865 *p_tmp = val; 866 } 867 868 /* Zero the top half of the allocated memory */ 869 memset((u8 *)p_virt + size, 0, size); 870 871 DP_VERBOSE(p_hwfn, 872 QED_MSG_SP, 873 "DMAE sanity [%s]: src_addr={phys 0x%llx, virt %p}, dst_addr={phys 0x%llx, virt %p}, size 0x%x\n", 874 phase, 875 (u64)p_phys, 876 p_virt, (u64)(p_phys + size), (u8 *)p_virt + size, size); 877 878 memset(¶ms, 0, sizeof(params)); 879 rc = qed_dmae_host2host(p_hwfn, p_ptt, p_phys, p_phys + size, 880 size / 4 /* size_in_dwords */, ¶ms); 881 if (rc) { 882 DP_NOTICE(p_hwfn, 883 "DMAE sanity [%s]: qed_dmae_host2host() failed. rc = %d.\n", 884 phase, rc); 885 goto out; 886 } 887 888 /* Verify that the top half of the allocated memory has the pattern */ 889 for (p_tmp = (u32 *)((u8 *)p_virt + size); 890 p_tmp < (u32 *)((u8 *)p_virt + (2 * size)); p_tmp++) { 891 /* The corresponding address in the bottom half */ 892 val = (u32)(uintptr_t)p_tmp - size; 893 894 if (*p_tmp != val) { 895 DP_NOTICE(p_hwfn, 896 "DMAE sanity [%s]: addr={phys 0x%llx, virt %p}, read_val 0x%08x, expected_val 0x%08x\n", 897 phase, 898 (u64)p_phys + ((u8 *)p_tmp - (u8 *)p_virt), 899 p_tmp, *p_tmp, val); 900 rc = -EINVAL; 901 goto out; 902 } 903 } 904 905 out: 906 dma_free_coherent(&p_hwfn->cdev->pdev->dev, 2 * size, p_virt, p_phys); 907 return rc; 908 } 909