1 // SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause 2 /* Copyright(c) 2018-2019 Realtek Corporation 3 */ 4 5 #if defined(__FreeBSD__) 6 #define LINUXKPI_PARAM_PREFIX rtw88_pci_ 7 #endif 8 9 #include <linux/module.h> 10 #include <linux/pci.h> 11 #include "main.h" 12 #include "pci.h" 13 #include "reg.h" 14 #include "tx.h" 15 #include "rx.h" 16 #include "fw.h" 17 #include "ps.h" 18 #include "debug.h" 19 #if defined(__FreeBSD__) 20 #include <linux/pm.h> 21 #endif 22 23 static bool rtw_disable_msi; 24 static bool rtw_pci_disable_aspm; 25 module_param_named(disable_msi, rtw_disable_msi, bool, 0644); 26 module_param_named(disable_aspm, rtw_pci_disable_aspm, bool, 0644); 27 MODULE_PARM_DESC(disable_msi, "Set Y to disable MSI interrupt support"); 28 MODULE_PARM_DESC(disable_aspm, "Set Y to disable PCI ASPM support"); 29 30 static u32 rtw_pci_tx_queue_idx_addr[] = { 31 [RTW_TX_QUEUE_BK] = RTK_PCI_TXBD_IDX_BKQ, 32 [RTW_TX_QUEUE_BE] = RTK_PCI_TXBD_IDX_BEQ, 33 [RTW_TX_QUEUE_VI] = RTK_PCI_TXBD_IDX_VIQ, 34 [RTW_TX_QUEUE_VO] = RTK_PCI_TXBD_IDX_VOQ, 35 [RTW_TX_QUEUE_MGMT] = RTK_PCI_TXBD_IDX_MGMTQ, 36 [RTW_TX_QUEUE_HI0] = RTK_PCI_TXBD_IDX_HI0Q, 37 [RTW_TX_QUEUE_H2C] = RTK_PCI_TXBD_IDX_H2CQ, 38 }; 39 40 static u8 rtw_pci_get_tx_qsel(struct sk_buff *skb, 41 enum rtw_tx_queue_type queue) 42 { 43 switch (queue) { 44 case RTW_TX_QUEUE_BCN: 45 return TX_DESC_QSEL_BEACON; 46 case RTW_TX_QUEUE_H2C: 47 return TX_DESC_QSEL_H2C; 48 case RTW_TX_QUEUE_MGMT: 49 return TX_DESC_QSEL_MGMT; 50 case RTW_TX_QUEUE_HI0: 51 return TX_DESC_QSEL_HIGH; 52 default: 53 return skb->priority; 54 } 55 }; 56 57 static u8 rtw_pci_read8(struct rtw_dev *rtwdev, u32 addr) 58 { 59 struct rtw_pci *rtwpci = (struct rtw_pci *)rtwdev->priv; 60 61 #if defined(__linux__) 62 return readb(rtwpci->mmap + addr); 63 #elif defined(__FreeBSD__) 64 u8 val; 65 66 val = bus_read_1((struct resource *)rtwpci->mmap, addr); 67 rtw_dbg(rtwdev, RTW_DBG_IO_RW, "R08 (%#010x) -> %#04x\n", addr, val); 68 return (val); 69 #endif 70 } 71 72 static u16 rtw_pci_read16(struct rtw_dev *rtwdev, u32 addr) 73 { 74 struct rtw_pci *rtwpci = (struct rtw_pci *)rtwdev->priv; 75 76 #if defined(__linux__) 77 return readw(rtwpci->mmap + addr); 78 #elif defined(__FreeBSD__) 79 u16 val; 80 81 val = bus_read_2((struct resource *)rtwpci->mmap, addr); 82 rtw_dbg(rtwdev, RTW_DBG_IO_RW, "R16 (%#010x) -> %#06x\n", addr, val); 83 return (val); 84 #endif 85 } 86 87 static u32 rtw_pci_read32(struct rtw_dev *rtwdev, u32 addr) 88 { 89 struct rtw_pci *rtwpci = (struct rtw_pci *)rtwdev->priv; 90 91 #if defined(__linux__) 92 return readl(rtwpci->mmap + addr); 93 #elif defined(__FreeBSD__) 94 u32 val; 95 96 val = bus_read_4((struct resource *)rtwpci->mmap, addr); 97 rtw_dbg(rtwdev, RTW_DBG_IO_RW, "R32 (%#010x) -> %#010x\n", addr, val); 98 return (val); 99 #endif 100 } 101 102 static void rtw_pci_write8(struct rtw_dev *rtwdev, u32 addr, u8 val) 103 { 104 struct rtw_pci *rtwpci = (struct rtw_pci *)rtwdev->priv; 105 106 #if defined(__linux__) 107 writeb(val, rtwpci->mmap + addr); 108 #elif defined(__FreeBSD__) 109 rtw_dbg(rtwdev, RTW_DBG_IO_RW, "W08 (%#010x) <- %#04x\n", addr, val); 110 return (bus_write_1((struct resource *)rtwpci->mmap, addr, val)); 111 #endif 112 } 113 114 static void rtw_pci_write16(struct rtw_dev *rtwdev, u32 addr, u16 val) 115 { 116 struct rtw_pci *rtwpci = (struct rtw_pci *)rtwdev->priv; 117 118 #if defined(__linux__) 119 writew(val, rtwpci->mmap + addr); 120 #elif defined(__FreeBSD__) 121 rtw_dbg(rtwdev, RTW_DBG_IO_RW, "W16 (%#010x) <- %#06x\n", addr, val); 122 return (bus_write_2((struct resource *)rtwpci->mmap, addr, val)); 123 #endif 124 } 125 126 static void rtw_pci_write32(struct rtw_dev *rtwdev, u32 addr, u32 val) 127 { 128 struct rtw_pci *rtwpci = (struct rtw_pci *)rtwdev->priv; 129 130 #if defined(__linux__) 131 writel(val, rtwpci->mmap + addr); 132 #elif defined(__FreeBSD__) 133 rtw_dbg(rtwdev, RTW_DBG_IO_RW, "W32 (%#010x) <- %#010x\n", addr, val); 134 return (bus_write_4((struct resource *)rtwpci->mmap, addr, val)); 135 #endif 136 } 137 138 static void rtw_pci_free_tx_ring_skbs(struct rtw_dev *rtwdev, 139 struct rtw_pci_tx_ring *tx_ring) 140 { 141 struct pci_dev *pdev = to_pci_dev(rtwdev->dev); 142 struct rtw_pci_tx_data *tx_data; 143 struct sk_buff *skb, *tmp; 144 dma_addr_t dma; 145 146 /* free every skb remained in tx list */ 147 skb_queue_walk_safe(&tx_ring->queue, skb, tmp) { 148 __skb_unlink(skb, &tx_ring->queue); 149 tx_data = rtw_pci_get_tx_data(skb); 150 dma = tx_data->dma; 151 152 dma_unmap_single(&pdev->dev, dma, skb->len, DMA_TO_DEVICE); 153 dev_kfree_skb_any(skb); 154 } 155 } 156 157 static void rtw_pci_free_tx_ring(struct rtw_dev *rtwdev, 158 struct rtw_pci_tx_ring *tx_ring) 159 { 160 struct pci_dev *pdev = to_pci_dev(rtwdev->dev); 161 u8 *head = tx_ring->r.head; 162 u32 len = tx_ring->r.len; 163 int ring_sz = len * tx_ring->r.desc_size; 164 165 rtw_pci_free_tx_ring_skbs(rtwdev, tx_ring); 166 167 /* free the ring itself */ 168 dma_free_coherent(&pdev->dev, ring_sz, head, tx_ring->r.dma); 169 tx_ring->r.head = NULL; 170 } 171 172 static void rtw_pci_free_rx_ring_skbs(struct rtw_dev *rtwdev, 173 struct rtw_pci_rx_ring *rx_ring) 174 { 175 struct pci_dev *pdev = to_pci_dev(rtwdev->dev); 176 struct sk_buff *skb; 177 int buf_sz = RTK_PCI_RX_BUF_SIZE; 178 dma_addr_t dma; 179 int i; 180 181 for (i = 0; i < rx_ring->r.len; i++) { 182 skb = rx_ring->buf[i]; 183 if (!skb) 184 continue; 185 186 dma = *((dma_addr_t *)skb->cb); 187 dma_unmap_single(&pdev->dev, dma, buf_sz, DMA_FROM_DEVICE); 188 dev_kfree_skb(skb); 189 rx_ring->buf[i] = NULL; 190 } 191 } 192 193 static void rtw_pci_free_rx_ring(struct rtw_dev *rtwdev, 194 struct rtw_pci_rx_ring *rx_ring) 195 { 196 struct pci_dev *pdev = to_pci_dev(rtwdev->dev); 197 u8 *head = rx_ring->r.head; 198 int ring_sz = rx_ring->r.desc_size * rx_ring->r.len; 199 200 rtw_pci_free_rx_ring_skbs(rtwdev, rx_ring); 201 202 dma_free_coherent(&pdev->dev, ring_sz, head, rx_ring->r.dma); 203 } 204 205 static void rtw_pci_free_trx_ring(struct rtw_dev *rtwdev) 206 { 207 struct rtw_pci *rtwpci = (struct rtw_pci *)rtwdev->priv; 208 struct rtw_pci_tx_ring *tx_ring; 209 struct rtw_pci_rx_ring *rx_ring; 210 int i; 211 212 for (i = 0; i < RTK_MAX_TX_QUEUE_NUM; i++) { 213 tx_ring = &rtwpci->tx_rings[i]; 214 rtw_pci_free_tx_ring(rtwdev, tx_ring); 215 } 216 217 for (i = 0; i < RTK_MAX_RX_QUEUE_NUM; i++) { 218 rx_ring = &rtwpci->rx_rings[i]; 219 rtw_pci_free_rx_ring(rtwdev, rx_ring); 220 } 221 } 222 223 static int rtw_pci_init_tx_ring(struct rtw_dev *rtwdev, 224 struct rtw_pci_tx_ring *tx_ring, 225 u8 desc_size, u32 len) 226 { 227 struct pci_dev *pdev = to_pci_dev(rtwdev->dev); 228 int ring_sz = desc_size * len; 229 dma_addr_t dma; 230 u8 *head; 231 232 if (len > TRX_BD_IDX_MASK) { 233 rtw_err(rtwdev, "len %d exceeds maximum TX entries\n", len); 234 return -EINVAL; 235 } 236 237 head = dma_alloc_coherent(&pdev->dev, ring_sz, &dma, GFP_KERNEL); 238 if (!head) { 239 rtw_err(rtwdev, "failed to allocate tx ring\n"); 240 return -ENOMEM; 241 } 242 243 skb_queue_head_init(&tx_ring->queue); 244 tx_ring->r.head = head; 245 tx_ring->r.dma = dma; 246 tx_ring->r.len = len; 247 tx_ring->r.desc_size = desc_size; 248 tx_ring->r.wp = 0; 249 tx_ring->r.rp = 0; 250 251 return 0; 252 } 253 254 static int rtw_pci_reset_rx_desc(struct rtw_dev *rtwdev, struct sk_buff *skb, 255 struct rtw_pci_rx_ring *rx_ring, 256 u32 idx, u32 desc_sz) 257 { 258 struct pci_dev *pdev = to_pci_dev(rtwdev->dev); 259 struct rtw_pci_rx_buffer_desc *buf_desc; 260 int buf_sz = RTK_PCI_RX_BUF_SIZE; 261 dma_addr_t dma; 262 263 if (!skb) 264 return -EINVAL; 265 266 dma = dma_map_single(&pdev->dev, skb->data, buf_sz, DMA_FROM_DEVICE); 267 if (dma_mapping_error(&pdev->dev, dma)) 268 return -EBUSY; 269 270 *((dma_addr_t *)skb->cb) = dma; 271 buf_desc = (struct rtw_pci_rx_buffer_desc *)(rx_ring->r.head + 272 idx * desc_sz); 273 memset(buf_desc, 0, sizeof(*buf_desc)); 274 buf_desc->buf_size = cpu_to_le16(RTK_PCI_RX_BUF_SIZE); 275 buf_desc->dma = cpu_to_le32(dma); 276 277 return 0; 278 } 279 280 static void rtw_pci_sync_rx_desc_device(struct rtw_dev *rtwdev, dma_addr_t dma, 281 struct rtw_pci_rx_ring *rx_ring, 282 u32 idx, u32 desc_sz) 283 { 284 struct device *dev = rtwdev->dev; 285 struct rtw_pci_rx_buffer_desc *buf_desc; 286 int buf_sz = RTK_PCI_RX_BUF_SIZE; 287 288 dma_sync_single_for_device(dev, dma, buf_sz, DMA_FROM_DEVICE); 289 290 buf_desc = (struct rtw_pci_rx_buffer_desc *)(rx_ring->r.head + 291 idx * desc_sz); 292 memset(buf_desc, 0, sizeof(*buf_desc)); 293 buf_desc->buf_size = cpu_to_le16(RTK_PCI_RX_BUF_SIZE); 294 buf_desc->dma = cpu_to_le32(dma); 295 } 296 297 static int rtw_pci_init_rx_ring(struct rtw_dev *rtwdev, 298 struct rtw_pci_rx_ring *rx_ring, 299 u8 desc_size, u32 len) 300 { 301 struct pci_dev *pdev = to_pci_dev(rtwdev->dev); 302 struct sk_buff *skb = NULL; 303 dma_addr_t dma; 304 u8 *head; 305 int ring_sz = desc_size * len; 306 int buf_sz = RTK_PCI_RX_BUF_SIZE; 307 int i, allocated; 308 int ret = 0; 309 310 head = dma_alloc_coherent(&pdev->dev, ring_sz, &dma, GFP_KERNEL); 311 if (!head) { 312 rtw_err(rtwdev, "failed to allocate rx ring\n"); 313 return -ENOMEM; 314 } 315 rx_ring->r.head = head; 316 317 for (i = 0; i < len; i++) { 318 skb = dev_alloc_skb(buf_sz); 319 if (!skb) { 320 allocated = i; 321 ret = -ENOMEM; 322 goto err_out; 323 } 324 325 memset(skb->data, 0, buf_sz); 326 rx_ring->buf[i] = skb; 327 ret = rtw_pci_reset_rx_desc(rtwdev, skb, rx_ring, i, desc_size); 328 if (ret) { 329 allocated = i; 330 dev_kfree_skb_any(skb); 331 goto err_out; 332 } 333 } 334 335 rx_ring->r.dma = dma; 336 rx_ring->r.len = len; 337 rx_ring->r.desc_size = desc_size; 338 rx_ring->r.wp = 0; 339 rx_ring->r.rp = 0; 340 341 return 0; 342 343 err_out: 344 for (i = 0; i < allocated; i++) { 345 skb = rx_ring->buf[i]; 346 if (!skb) 347 continue; 348 dma = *((dma_addr_t *)skb->cb); 349 dma_unmap_single(&pdev->dev, dma, buf_sz, DMA_FROM_DEVICE); 350 dev_kfree_skb_any(skb); 351 rx_ring->buf[i] = NULL; 352 } 353 dma_free_coherent(&pdev->dev, ring_sz, head, dma); 354 355 rtw_err(rtwdev, "failed to init rx buffer\n"); 356 357 return ret; 358 } 359 360 static int rtw_pci_init_trx_ring(struct rtw_dev *rtwdev) 361 { 362 struct rtw_pci *rtwpci = (struct rtw_pci *)rtwdev->priv; 363 struct rtw_pci_tx_ring *tx_ring; 364 struct rtw_pci_rx_ring *rx_ring; 365 const struct rtw_chip_info *chip = rtwdev->chip; 366 int i = 0, j = 0, tx_alloced = 0, rx_alloced = 0; 367 int tx_desc_size, rx_desc_size; 368 u32 len; 369 int ret; 370 371 tx_desc_size = chip->tx_buf_desc_sz; 372 373 for (i = 0; i < RTK_MAX_TX_QUEUE_NUM; i++) { 374 tx_ring = &rtwpci->tx_rings[i]; 375 len = max_num_of_tx_queue(i); 376 ret = rtw_pci_init_tx_ring(rtwdev, tx_ring, tx_desc_size, len); 377 if (ret) 378 goto out; 379 } 380 381 rx_desc_size = chip->rx_buf_desc_sz; 382 383 for (j = 0; j < RTK_MAX_RX_QUEUE_NUM; j++) { 384 rx_ring = &rtwpci->rx_rings[j]; 385 ret = rtw_pci_init_rx_ring(rtwdev, rx_ring, rx_desc_size, 386 RTK_MAX_RX_DESC_NUM); 387 if (ret) 388 goto out; 389 } 390 391 return 0; 392 393 out: 394 tx_alloced = i; 395 for (i = 0; i < tx_alloced; i++) { 396 tx_ring = &rtwpci->tx_rings[i]; 397 rtw_pci_free_tx_ring(rtwdev, tx_ring); 398 } 399 400 rx_alloced = j; 401 for (j = 0; j < rx_alloced; j++) { 402 rx_ring = &rtwpci->rx_rings[j]; 403 rtw_pci_free_rx_ring(rtwdev, rx_ring); 404 } 405 406 return ret; 407 } 408 409 static void rtw_pci_deinit(struct rtw_dev *rtwdev) 410 { 411 rtw_pci_free_trx_ring(rtwdev); 412 } 413 414 static int rtw_pci_init(struct rtw_dev *rtwdev) 415 { 416 struct rtw_pci *rtwpci = (struct rtw_pci *)rtwdev->priv; 417 int ret = 0; 418 419 rtwpci->irq_mask[0] = IMR_HIGHDOK | 420 IMR_MGNTDOK | 421 IMR_BKDOK | 422 IMR_BEDOK | 423 IMR_VIDOK | 424 IMR_VODOK | 425 IMR_ROK | 426 IMR_BCNDMAINT_E | 427 IMR_C2HCMD | 428 0; 429 rtwpci->irq_mask[1] = IMR_TXFOVW | 430 0; 431 rtwpci->irq_mask[3] = IMR_H2CDOK | 432 0; 433 spin_lock_init(&rtwpci->irq_lock); 434 spin_lock_init(&rtwpci->hwirq_lock); 435 ret = rtw_pci_init_trx_ring(rtwdev); 436 437 return ret; 438 } 439 440 static void rtw_pci_reset_buf_desc(struct rtw_dev *rtwdev) 441 { 442 struct rtw_pci *rtwpci = (struct rtw_pci *)rtwdev->priv; 443 u32 len; 444 u8 tmp; 445 dma_addr_t dma; 446 447 tmp = rtw_read8(rtwdev, RTK_PCI_CTRL + 3); 448 rtw_write8(rtwdev, RTK_PCI_CTRL + 3, tmp | 0xf7); 449 450 dma = rtwpci->tx_rings[RTW_TX_QUEUE_BCN].r.dma; 451 rtw_write32(rtwdev, RTK_PCI_TXBD_DESA_BCNQ, dma); 452 453 if (!rtw_chip_wcpu_11n(rtwdev)) { 454 len = rtwpci->tx_rings[RTW_TX_QUEUE_H2C].r.len; 455 dma = rtwpci->tx_rings[RTW_TX_QUEUE_H2C].r.dma; 456 rtwpci->tx_rings[RTW_TX_QUEUE_H2C].r.rp = 0; 457 rtwpci->tx_rings[RTW_TX_QUEUE_H2C].r.wp = 0; 458 rtw_write16(rtwdev, RTK_PCI_TXBD_NUM_H2CQ, len & TRX_BD_IDX_MASK); 459 rtw_write32(rtwdev, RTK_PCI_TXBD_DESA_H2CQ, dma); 460 } 461 462 len = rtwpci->tx_rings[RTW_TX_QUEUE_BK].r.len; 463 dma = rtwpci->tx_rings[RTW_TX_QUEUE_BK].r.dma; 464 rtwpci->tx_rings[RTW_TX_QUEUE_BK].r.rp = 0; 465 rtwpci->tx_rings[RTW_TX_QUEUE_BK].r.wp = 0; 466 rtw_write16(rtwdev, RTK_PCI_TXBD_NUM_BKQ, len & TRX_BD_IDX_MASK); 467 rtw_write32(rtwdev, RTK_PCI_TXBD_DESA_BKQ, dma); 468 469 len = rtwpci->tx_rings[RTW_TX_QUEUE_BE].r.len; 470 dma = rtwpci->tx_rings[RTW_TX_QUEUE_BE].r.dma; 471 rtwpci->tx_rings[RTW_TX_QUEUE_BE].r.rp = 0; 472 rtwpci->tx_rings[RTW_TX_QUEUE_BE].r.wp = 0; 473 rtw_write16(rtwdev, RTK_PCI_TXBD_NUM_BEQ, len & TRX_BD_IDX_MASK); 474 rtw_write32(rtwdev, RTK_PCI_TXBD_DESA_BEQ, dma); 475 476 len = rtwpci->tx_rings[RTW_TX_QUEUE_VO].r.len; 477 dma = rtwpci->tx_rings[RTW_TX_QUEUE_VO].r.dma; 478 rtwpci->tx_rings[RTW_TX_QUEUE_VO].r.rp = 0; 479 rtwpci->tx_rings[RTW_TX_QUEUE_VO].r.wp = 0; 480 rtw_write16(rtwdev, RTK_PCI_TXBD_NUM_VOQ, len & TRX_BD_IDX_MASK); 481 rtw_write32(rtwdev, RTK_PCI_TXBD_DESA_VOQ, dma); 482 483 len = rtwpci->tx_rings[RTW_TX_QUEUE_VI].r.len; 484 dma = rtwpci->tx_rings[RTW_TX_QUEUE_VI].r.dma; 485 rtwpci->tx_rings[RTW_TX_QUEUE_VI].r.rp = 0; 486 rtwpci->tx_rings[RTW_TX_QUEUE_VI].r.wp = 0; 487 rtw_write16(rtwdev, RTK_PCI_TXBD_NUM_VIQ, len & TRX_BD_IDX_MASK); 488 rtw_write32(rtwdev, RTK_PCI_TXBD_DESA_VIQ, dma); 489 490 len = rtwpci->tx_rings[RTW_TX_QUEUE_MGMT].r.len; 491 dma = rtwpci->tx_rings[RTW_TX_QUEUE_MGMT].r.dma; 492 rtwpci->tx_rings[RTW_TX_QUEUE_MGMT].r.rp = 0; 493 rtwpci->tx_rings[RTW_TX_QUEUE_MGMT].r.wp = 0; 494 rtw_write16(rtwdev, RTK_PCI_TXBD_NUM_MGMTQ, len & TRX_BD_IDX_MASK); 495 rtw_write32(rtwdev, RTK_PCI_TXBD_DESA_MGMTQ, dma); 496 497 len = rtwpci->tx_rings[RTW_TX_QUEUE_HI0].r.len; 498 dma = rtwpci->tx_rings[RTW_TX_QUEUE_HI0].r.dma; 499 rtwpci->tx_rings[RTW_TX_QUEUE_HI0].r.rp = 0; 500 rtwpci->tx_rings[RTW_TX_QUEUE_HI0].r.wp = 0; 501 rtw_write16(rtwdev, RTK_PCI_TXBD_NUM_HI0Q, len & TRX_BD_IDX_MASK); 502 rtw_write32(rtwdev, RTK_PCI_TXBD_DESA_HI0Q, dma); 503 504 len = rtwpci->rx_rings[RTW_RX_QUEUE_MPDU].r.len; 505 dma = rtwpci->rx_rings[RTW_RX_QUEUE_MPDU].r.dma; 506 rtwpci->rx_rings[RTW_RX_QUEUE_MPDU].r.rp = 0; 507 rtwpci->rx_rings[RTW_RX_QUEUE_MPDU].r.wp = 0; 508 rtw_write16(rtwdev, RTK_PCI_RXBD_NUM_MPDUQ, len & TRX_BD_IDX_MASK); 509 rtw_write32(rtwdev, RTK_PCI_RXBD_DESA_MPDUQ, dma); 510 511 /* reset read/write point */ 512 rtw_write32(rtwdev, RTK_PCI_TXBD_RWPTR_CLR, 0xffffffff); 513 514 /* reset H2C Queue index in a single write */ 515 if (rtw_chip_wcpu_11ac(rtwdev)) 516 rtw_write32_set(rtwdev, RTK_PCI_TXBD_H2CQ_CSR, 517 BIT_CLR_H2CQ_HOST_IDX | BIT_CLR_H2CQ_HW_IDX); 518 } 519 520 static void rtw_pci_reset_trx_ring(struct rtw_dev *rtwdev) 521 { 522 rtw_pci_reset_buf_desc(rtwdev); 523 } 524 525 static void rtw_pci_enable_interrupt(struct rtw_dev *rtwdev, 526 struct rtw_pci *rtwpci, bool exclude_rx) 527 { 528 unsigned long flags; 529 u32 imr0_unmask = exclude_rx ? IMR_ROK : 0; 530 531 spin_lock_irqsave(&rtwpci->hwirq_lock, flags); 532 533 rtw_write32(rtwdev, RTK_PCI_HIMR0, rtwpci->irq_mask[0] & ~imr0_unmask); 534 rtw_write32(rtwdev, RTK_PCI_HIMR1, rtwpci->irq_mask[1]); 535 if (rtw_chip_wcpu_11ac(rtwdev)) 536 rtw_write32(rtwdev, RTK_PCI_HIMR3, rtwpci->irq_mask[3]); 537 538 rtwpci->irq_enabled = true; 539 540 spin_unlock_irqrestore(&rtwpci->hwirq_lock, flags); 541 } 542 543 static void rtw_pci_disable_interrupt(struct rtw_dev *rtwdev, 544 struct rtw_pci *rtwpci) 545 { 546 unsigned long flags; 547 548 spin_lock_irqsave(&rtwpci->hwirq_lock, flags); 549 550 if (!rtwpci->irq_enabled) 551 goto out; 552 553 rtw_write32(rtwdev, RTK_PCI_HIMR0, 0); 554 rtw_write32(rtwdev, RTK_PCI_HIMR1, 0); 555 if (rtw_chip_wcpu_11ac(rtwdev)) 556 rtw_write32(rtwdev, RTK_PCI_HIMR3, 0); 557 558 rtwpci->irq_enabled = false; 559 560 out: 561 spin_unlock_irqrestore(&rtwpci->hwirq_lock, flags); 562 } 563 564 static void rtw_pci_dma_reset(struct rtw_dev *rtwdev, struct rtw_pci *rtwpci) 565 { 566 /* reset dma and rx tag */ 567 rtw_write32_set(rtwdev, RTK_PCI_CTRL, 568 BIT_RST_TRXDMA_INTF | BIT_RX_TAG_EN); 569 rtwpci->rx_tag = 0; 570 } 571 572 static int rtw_pci_setup(struct rtw_dev *rtwdev) 573 { 574 struct rtw_pci *rtwpci = (struct rtw_pci *)rtwdev->priv; 575 576 rtw_pci_reset_trx_ring(rtwdev); 577 rtw_pci_dma_reset(rtwdev, rtwpci); 578 579 return 0; 580 } 581 582 static void rtw_pci_dma_release(struct rtw_dev *rtwdev, struct rtw_pci *rtwpci) 583 { 584 struct rtw_pci_tx_ring *tx_ring; 585 enum rtw_tx_queue_type queue; 586 587 rtw_pci_reset_trx_ring(rtwdev); 588 for (queue = 0; queue < RTK_MAX_TX_QUEUE_NUM; queue++) { 589 tx_ring = &rtwpci->tx_rings[queue]; 590 rtw_pci_free_tx_ring_skbs(rtwdev, tx_ring); 591 } 592 } 593 594 static void rtw_pci_napi_start(struct rtw_dev *rtwdev) 595 { 596 struct rtw_pci *rtwpci = (struct rtw_pci *)rtwdev->priv; 597 598 if (test_and_set_bit(RTW_PCI_FLAG_NAPI_RUNNING, rtwpci->flags)) 599 return; 600 601 napi_enable(&rtwpci->napi); 602 } 603 604 static void rtw_pci_napi_stop(struct rtw_dev *rtwdev) 605 { 606 struct rtw_pci *rtwpci = (struct rtw_pci *)rtwdev->priv; 607 608 if (!test_and_clear_bit(RTW_PCI_FLAG_NAPI_RUNNING, rtwpci->flags)) 609 return; 610 611 napi_synchronize(&rtwpci->napi); 612 napi_disable(&rtwpci->napi); 613 } 614 615 static int rtw_pci_start(struct rtw_dev *rtwdev) 616 { 617 struct rtw_pci *rtwpci = (struct rtw_pci *)rtwdev->priv; 618 619 rtw_pci_napi_start(rtwdev); 620 621 spin_lock_bh(&rtwpci->irq_lock); 622 rtwpci->running = true; 623 rtw_pci_enable_interrupt(rtwdev, rtwpci, false); 624 spin_unlock_bh(&rtwpci->irq_lock); 625 626 return 0; 627 } 628 629 static void rtw_pci_stop(struct rtw_dev *rtwdev) 630 { 631 struct rtw_pci *rtwpci = (struct rtw_pci *)rtwdev->priv; 632 struct pci_dev *pdev = rtwpci->pdev; 633 634 spin_lock_bh(&rtwpci->irq_lock); 635 rtwpci->running = false; 636 rtw_pci_disable_interrupt(rtwdev, rtwpci); 637 spin_unlock_bh(&rtwpci->irq_lock); 638 639 synchronize_irq(pdev->irq); 640 rtw_pci_napi_stop(rtwdev); 641 642 spin_lock_bh(&rtwpci->irq_lock); 643 rtw_pci_dma_release(rtwdev, rtwpci); 644 spin_unlock_bh(&rtwpci->irq_lock); 645 } 646 647 static void rtw_pci_deep_ps_enter(struct rtw_dev *rtwdev) 648 { 649 struct rtw_pci *rtwpci = (struct rtw_pci *)rtwdev->priv; 650 struct rtw_pci_tx_ring *tx_ring; 651 enum rtw_tx_queue_type queue; 652 bool tx_empty = true; 653 654 if (rtw_fw_feature_check(&rtwdev->fw, FW_FEATURE_TX_WAKE)) 655 goto enter_deep_ps; 656 657 lockdep_assert_held(&rtwpci->irq_lock); 658 659 /* Deep PS state is not allowed to TX-DMA */ 660 for (queue = 0; queue < RTK_MAX_TX_QUEUE_NUM; queue++) { 661 /* BCN queue is rsvd page, does not have DMA interrupt 662 * H2C queue is managed by firmware 663 */ 664 if (queue == RTW_TX_QUEUE_BCN || 665 queue == RTW_TX_QUEUE_H2C) 666 continue; 667 668 tx_ring = &rtwpci->tx_rings[queue]; 669 670 /* check if there is any skb DMAing */ 671 if (skb_queue_len(&tx_ring->queue)) { 672 tx_empty = false; 673 break; 674 } 675 } 676 677 if (!tx_empty) { 678 rtw_dbg(rtwdev, RTW_DBG_PS, 679 "TX path not empty, cannot enter deep power save state\n"); 680 return; 681 } 682 enter_deep_ps: 683 set_bit(RTW_FLAG_LEISURE_PS_DEEP, rtwdev->flags); 684 rtw_power_mode_change(rtwdev, true); 685 } 686 687 static void rtw_pci_deep_ps_leave(struct rtw_dev *rtwdev) 688 { 689 #if defined(__linux__) 690 struct rtw_pci *rtwpci = (struct rtw_pci *)rtwdev->priv; 691 692 lockdep_assert_held(&rtwpci->irq_lock); 693 #elif defined(__FreeBSD__) 694 lockdep_assert_held(&((struct rtw_pci *)rtwdev->priv)->irq_lock); 695 #endif 696 697 if (test_and_clear_bit(RTW_FLAG_LEISURE_PS_DEEP, rtwdev->flags)) 698 rtw_power_mode_change(rtwdev, false); 699 } 700 701 static void rtw_pci_deep_ps(struct rtw_dev *rtwdev, bool enter) 702 { 703 struct rtw_pci *rtwpci = (struct rtw_pci *)rtwdev->priv; 704 705 spin_lock_bh(&rtwpci->irq_lock); 706 707 if (enter && !test_bit(RTW_FLAG_LEISURE_PS_DEEP, rtwdev->flags)) 708 rtw_pci_deep_ps_enter(rtwdev); 709 710 if (!enter && test_bit(RTW_FLAG_LEISURE_PS_DEEP, rtwdev->flags)) 711 rtw_pci_deep_ps_leave(rtwdev); 712 713 spin_unlock_bh(&rtwpci->irq_lock); 714 } 715 716 static void rtw_pci_release_rsvd_page(struct rtw_pci *rtwpci, 717 struct rtw_pci_tx_ring *ring) 718 { 719 struct sk_buff *prev = skb_dequeue(&ring->queue); 720 struct rtw_pci_tx_data *tx_data; 721 dma_addr_t dma; 722 723 if (!prev) 724 return; 725 726 tx_data = rtw_pci_get_tx_data(prev); 727 dma = tx_data->dma; 728 dma_unmap_single(&rtwpci->pdev->dev, dma, prev->len, DMA_TO_DEVICE); 729 dev_kfree_skb_any(prev); 730 } 731 732 static void rtw_pci_dma_check(struct rtw_dev *rtwdev, 733 struct rtw_pci_rx_ring *rx_ring, 734 u32 idx) 735 { 736 struct rtw_pci *rtwpci = (struct rtw_pci *)rtwdev->priv; 737 const struct rtw_chip_info *chip = rtwdev->chip; 738 struct rtw_pci_rx_buffer_desc *buf_desc; 739 u32 desc_sz = chip->rx_buf_desc_sz; 740 u16 total_pkt_size; 741 742 buf_desc = (struct rtw_pci_rx_buffer_desc *)(rx_ring->r.head + 743 idx * desc_sz); 744 total_pkt_size = le16_to_cpu(buf_desc->total_pkt_size); 745 746 /* rx tag mismatch, throw a warning */ 747 if (total_pkt_size != rtwpci->rx_tag) 748 rtw_warn(rtwdev, "pci bus timeout, check dma status\n"); 749 750 rtwpci->rx_tag = (rtwpci->rx_tag + 1) % RX_TAG_MAX; 751 } 752 753 static u32 __pci_get_hw_tx_ring_rp(struct rtw_dev *rtwdev, u8 pci_q) 754 { 755 u32 bd_idx_addr = rtw_pci_tx_queue_idx_addr[pci_q]; 756 u32 bd_idx = rtw_read16(rtwdev, bd_idx_addr + 2); 757 758 return FIELD_GET(TRX_BD_IDX_MASK, bd_idx); 759 } 760 761 static void __pci_flush_queue(struct rtw_dev *rtwdev, u8 pci_q, bool drop) 762 { 763 struct rtw_pci *rtwpci = (struct rtw_pci *)rtwdev->priv; 764 struct rtw_pci_tx_ring *ring = &rtwpci->tx_rings[pci_q]; 765 u32 cur_rp; 766 u8 i; 767 768 /* Because the time taked by the I/O in __pci_get_hw_tx_ring_rp is a 769 * bit dynamic, it's hard to define a reasonable fixed total timeout to 770 * use read_poll_timeout* helper. Instead, we can ensure a reasonable 771 * polling times, so we just use for loop with udelay here. 772 */ 773 for (i = 0; i < 30; i++) { 774 cur_rp = __pci_get_hw_tx_ring_rp(rtwdev, pci_q); 775 if (cur_rp == ring->r.wp) 776 return; 777 778 udelay(1); 779 } 780 781 if (!drop) 782 rtw_warn(rtwdev, "timed out to flush pci tx ring[%d]\n", pci_q); 783 } 784 785 static void __rtw_pci_flush_queues(struct rtw_dev *rtwdev, u32 pci_queues, 786 bool drop) 787 { 788 u8 q; 789 790 for (q = 0; q < RTK_MAX_TX_QUEUE_NUM; q++) { 791 /* Unnecessary to flush BCN, H2C and HI tx queues. */ 792 if (q == RTW_TX_QUEUE_BCN || q == RTW_TX_QUEUE_H2C || 793 q == RTW_TX_QUEUE_HI0) 794 continue; 795 796 if (pci_queues & BIT(q)) 797 __pci_flush_queue(rtwdev, q, drop); 798 } 799 } 800 801 static void rtw_pci_flush_queues(struct rtw_dev *rtwdev, u32 queues, bool drop) 802 { 803 u32 pci_queues = 0; 804 u8 i; 805 806 /* If all of the hardware queues are requested to flush, 807 * flush all of the pci queues. 808 */ 809 if (queues == BIT(rtwdev->hw->queues) - 1) { 810 pci_queues = BIT(RTK_MAX_TX_QUEUE_NUM) - 1; 811 } else { 812 for (i = 0; i < rtwdev->hw->queues; i++) 813 if (queues & BIT(i)) 814 pci_queues |= BIT(rtw_tx_ac_to_hwq(i)); 815 } 816 817 __rtw_pci_flush_queues(rtwdev, pci_queues, drop); 818 } 819 820 static void rtw_pci_tx_kick_off_queue(struct rtw_dev *rtwdev, 821 enum rtw_tx_queue_type queue) 822 { 823 struct rtw_pci *rtwpci = (struct rtw_pci *)rtwdev->priv; 824 struct rtw_pci_tx_ring *ring; 825 u32 bd_idx; 826 827 ring = &rtwpci->tx_rings[queue]; 828 bd_idx = rtw_pci_tx_queue_idx_addr[queue]; 829 830 spin_lock_bh(&rtwpci->irq_lock); 831 if (!rtw_fw_feature_check(&rtwdev->fw, FW_FEATURE_TX_WAKE)) 832 rtw_pci_deep_ps_leave(rtwdev); 833 rtw_write16(rtwdev, bd_idx, ring->r.wp & TRX_BD_IDX_MASK); 834 spin_unlock_bh(&rtwpci->irq_lock); 835 } 836 837 static void rtw_pci_tx_kick_off(struct rtw_dev *rtwdev) 838 { 839 struct rtw_pci *rtwpci = (struct rtw_pci *)rtwdev->priv; 840 enum rtw_tx_queue_type queue; 841 842 for (queue = 0; queue < RTK_MAX_TX_QUEUE_NUM; queue++) 843 if (test_and_clear_bit(queue, rtwpci->tx_queued)) 844 rtw_pci_tx_kick_off_queue(rtwdev, queue); 845 } 846 847 static int rtw_pci_tx_write_data(struct rtw_dev *rtwdev, 848 struct rtw_tx_pkt_info *pkt_info, 849 struct sk_buff *skb, 850 enum rtw_tx_queue_type queue) 851 { 852 struct rtw_pci *rtwpci = (struct rtw_pci *)rtwdev->priv; 853 const struct rtw_chip_info *chip = rtwdev->chip; 854 struct rtw_pci_tx_ring *ring; 855 struct rtw_pci_tx_data *tx_data; 856 dma_addr_t dma; 857 u32 tx_pkt_desc_sz = chip->tx_pkt_desc_sz; 858 u32 tx_buf_desc_sz = chip->tx_buf_desc_sz; 859 u32 size; 860 u32 psb_len; 861 u8 *pkt_desc; 862 struct rtw_pci_tx_buffer_desc *buf_desc; 863 864 ring = &rtwpci->tx_rings[queue]; 865 866 size = skb->len; 867 868 if (queue == RTW_TX_QUEUE_BCN) 869 rtw_pci_release_rsvd_page(rtwpci, ring); 870 else if (!avail_desc(ring->r.wp, ring->r.rp, ring->r.len)) 871 return -ENOSPC; 872 873 pkt_desc = skb_push(skb, chip->tx_pkt_desc_sz); 874 memset(pkt_desc, 0, tx_pkt_desc_sz); 875 pkt_info->qsel = rtw_pci_get_tx_qsel(skb, queue); 876 rtw_tx_fill_tx_desc(pkt_info, skb); 877 dma = dma_map_single(&rtwpci->pdev->dev, skb->data, skb->len, 878 DMA_TO_DEVICE); 879 if (dma_mapping_error(&rtwpci->pdev->dev, dma)) 880 return -EBUSY; 881 882 /* after this we got dma mapped, there is no way back */ 883 buf_desc = get_tx_buffer_desc(ring, tx_buf_desc_sz); 884 memset(buf_desc, 0, tx_buf_desc_sz); 885 psb_len = (skb->len - 1) / 128 + 1; 886 if (queue == RTW_TX_QUEUE_BCN) 887 psb_len |= 1 << RTK_PCI_TXBD_OWN_OFFSET; 888 889 buf_desc[0].psb_len = cpu_to_le16(psb_len); 890 buf_desc[0].buf_size = cpu_to_le16(tx_pkt_desc_sz); 891 buf_desc[0].dma = cpu_to_le32(dma); 892 buf_desc[1].buf_size = cpu_to_le16(size); 893 buf_desc[1].dma = cpu_to_le32(dma + tx_pkt_desc_sz); 894 895 tx_data = rtw_pci_get_tx_data(skb); 896 tx_data->dma = dma; 897 tx_data->sn = pkt_info->sn; 898 899 spin_lock_bh(&rtwpci->irq_lock); 900 901 skb_queue_tail(&ring->queue, skb); 902 903 if (queue == RTW_TX_QUEUE_BCN) 904 goto out_unlock; 905 906 /* update write-index, and kick it off later */ 907 set_bit(queue, rtwpci->tx_queued); 908 if (++ring->r.wp >= ring->r.len) 909 ring->r.wp = 0; 910 911 out_unlock: 912 spin_unlock_bh(&rtwpci->irq_lock); 913 914 return 0; 915 } 916 917 static int rtw_pci_write_data_rsvd_page(struct rtw_dev *rtwdev, u8 *buf, 918 u32 size) 919 { 920 struct sk_buff *skb; 921 struct rtw_tx_pkt_info pkt_info = {0}; 922 u8 reg_bcn_work; 923 int ret; 924 925 skb = rtw_tx_write_data_rsvd_page_get(rtwdev, &pkt_info, buf, size); 926 if (!skb) 927 return -ENOMEM; 928 929 ret = rtw_pci_tx_write_data(rtwdev, &pkt_info, skb, RTW_TX_QUEUE_BCN); 930 if (ret) { 931 #if defined(__FreeBSD__) 932 dev_kfree_skb_any(skb); 933 #endif 934 rtw_err(rtwdev, "failed to write rsvd page data\n"); 935 return ret; 936 } 937 938 /* reserved pages go through beacon queue */ 939 reg_bcn_work = rtw_read8(rtwdev, RTK_PCI_TXBD_BCN_WORK); 940 reg_bcn_work |= BIT_PCI_BCNQ_FLAG; 941 rtw_write8(rtwdev, RTK_PCI_TXBD_BCN_WORK, reg_bcn_work); 942 943 return 0; 944 } 945 946 static int rtw_pci_write_data_h2c(struct rtw_dev *rtwdev, u8 *buf, u32 size) 947 { 948 struct sk_buff *skb; 949 struct rtw_tx_pkt_info pkt_info = {0}; 950 int ret; 951 952 skb = rtw_tx_write_data_h2c_get(rtwdev, &pkt_info, buf, size); 953 if (!skb) 954 return -ENOMEM; 955 956 ret = rtw_pci_tx_write_data(rtwdev, &pkt_info, skb, RTW_TX_QUEUE_H2C); 957 if (ret) { 958 #if defined(__FreeBSD__) 959 dev_kfree_skb_any(skb); 960 #endif 961 rtw_err(rtwdev, "failed to write h2c data\n"); 962 return ret; 963 } 964 965 rtw_pci_tx_kick_off_queue(rtwdev, RTW_TX_QUEUE_H2C); 966 967 return 0; 968 } 969 970 static int rtw_pci_tx_write(struct rtw_dev *rtwdev, 971 struct rtw_tx_pkt_info *pkt_info, 972 struct sk_buff *skb) 973 { 974 enum rtw_tx_queue_type queue = rtw_tx_queue_mapping(skb); 975 struct rtw_pci *rtwpci = (struct rtw_pci *)rtwdev->priv; 976 struct rtw_pci_tx_ring *ring; 977 int ret; 978 979 ret = rtw_pci_tx_write_data(rtwdev, pkt_info, skb, queue); 980 if (ret) 981 return ret; 982 983 ring = &rtwpci->tx_rings[queue]; 984 spin_lock_bh(&rtwpci->irq_lock); 985 if (avail_desc(ring->r.wp, ring->r.rp, ring->r.len) < 2) { 986 ieee80211_stop_queue(rtwdev->hw, skb_get_queue_mapping(skb)); 987 ring->queue_stopped = true; 988 } 989 spin_unlock_bh(&rtwpci->irq_lock); 990 991 return 0; 992 } 993 994 static void rtw_pci_tx_isr(struct rtw_dev *rtwdev, struct rtw_pci *rtwpci, 995 u8 hw_queue) 996 { 997 struct ieee80211_hw *hw = rtwdev->hw; 998 struct ieee80211_tx_info *info; 999 struct rtw_pci_tx_ring *ring; 1000 struct rtw_pci_tx_data *tx_data; 1001 struct sk_buff *skb; 1002 u32 count; 1003 u32 bd_idx_addr; 1004 u32 bd_idx, cur_rp, rp_idx; 1005 u16 q_map; 1006 1007 ring = &rtwpci->tx_rings[hw_queue]; 1008 1009 bd_idx_addr = rtw_pci_tx_queue_idx_addr[hw_queue]; 1010 bd_idx = rtw_read32(rtwdev, bd_idx_addr); 1011 cur_rp = bd_idx >> 16; 1012 cur_rp &= TRX_BD_IDX_MASK; 1013 rp_idx = ring->r.rp; 1014 if (cur_rp >= ring->r.rp) 1015 count = cur_rp - ring->r.rp; 1016 else 1017 count = ring->r.len - (ring->r.rp - cur_rp); 1018 1019 while (count--) { 1020 skb = skb_dequeue(&ring->queue); 1021 if (!skb) { 1022 rtw_err(rtwdev, "failed to dequeue %d skb TX queue %d, BD=0x%08x, rp %d -> %d\n", 1023 count, hw_queue, bd_idx, ring->r.rp, cur_rp); 1024 break; 1025 } 1026 tx_data = rtw_pci_get_tx_data(skb); 1027 dma_unmap_single(&rtwpci->pdev->dev, tx_data->dma, skb->len, 1028 DMA_TO_DEVICE); 1029 1030 /* just free command packets from host to card */ 1031 if (hw_queue == RTW_TX_QUEUE_H2C) { 1032 dev_kfree_skb_irq(skb); 1033 continue; 1034 } 1035 1036 if (ring->queue_stopped && 1037 avail_desc(ring->r.wp, rp_idx, ring->r.len) > 4) { 1038 q_map = skb_get_queue_mapping(skb); 1039 ieee80211_wake_queue(hw, q_map); 1040 ring->queue_stopped = false; 1041 } 1042 1043 if (++rp_idx >= ring->r.len) 1044 rp_idx = 0; 1045 1046 skb_pull(skb, rtwdev->chip->tx_pkt_desc_sz); 1047 1048 info = IEEE80211_SKB_CB(skb); 1049 1050 /* enqueue to wait for tx report */ 1051 if (info->flags & IEEE80211_TX_CTL_REQ_TX_STATUS) { 1052 rtw_tx_report_enqueue(rtwdev, skb, tx_data->sn); 1053 continue; 1054 } 1055 1056 /* always ACK for others, then they won't be marked as drop */ 1057 if (info->flags & IEEE80211_TX_CTL_NO_ACK) 1058 info->flags |= IEEE80211_TX_STAT_NOACK_TRANSMITTED; 1059 else 1060 info->flags |= IEEE80211_TX_STAT_ACK; 1061 1062 ieee80211_tx_info_clear_status(info); 1063 ieee80211_tx_status_irqsafe(hw, skb); 1064 } 1065 1066 ring->r.rp = cur_rp; 1067 } 1068 1069 static void rtw_pci_rx_isr(struct rtw_dev *rtwdev) 1070 { 1071 struct rtw_pci *rtwpci = (struct rtw_pci *)rtwdev->priv; 1072 struct napi_struct *napi = &rtwpci->napi; 1073 1074 napi_schedule(napi); 1075 } 1076 1077 static int rtw_pci_get_hw_rx_ring_nr(struct rtw_dev *rtwdev, 1078 struct rtw_pci *rtwpci) 1079 { 1080 struct rtw_pci_rx_ring *ring; 1081 int count = 0; 1082 u32 tmp, cur_wp; 1083 1084 ring = &rtwpci->rx_rings[RTW_RX_QUEUE_MPDU]; 1085 tmp = rtw_read32(rtwdev, RTK_PCI_RXBD_IDX_MPDUQ); 1086 cur_wp = u32_get_bits(tmp, TRX_BD_HW_IDX_MASK); 1087 if (cur_wp >= ring->r.wp) 1088 count = cur_wp - ring->r.wp; 1089 else 1090 count = ring->r.len - (ring->r.wp - cur_wp); 1091 1092 return count; 1093 } 1094 1095 static u32 rtw_pci_rx_napi(struct rtw_dev *rtwdev, struct rtw_pci *rtwpci, 1096 u8 hw_queue, u32 limit) 1097 { 1098 const struct rtw_chip_info *chip = rtwdev->chip; 1099 struct napi_struct *napi = &rtwpci->napi; 1100 struct rtw_pci_rx_ring *ring = &rtwpci->rx_rings[RTW_RX_QUEUE_MPDU]; 1101 struct rtw_rx_pkt_stat pkt_stat; 1102 struct ieee80211_rx_status rx_status; 1103 struct sk_buff *skb, *new; 1104 u32 cur_rp = ring->r.rp; 1105 u32 count, rx_done = 0; 1106 u32 pkt_offset; 1107 u32 pkt_desc_sz = chip->rx_pkt_desc_sz; 1108 u32 buf_desc_sz = chip->rx_buf_desc_sz; 1109 u32 new_len; 1110 u8 *rx_desc; 1111 dma_addr_t dma; 1112 1113 count = rtw_pci_get_hw_rx_ring_nr(rtwdev, rtwpci); 1114 count = min(count, limit); 1115 1116 while (count--) { 1117 rtw_pci_dma_check(rtwdev, ring, cur_rp); 1118 skb = ring->buf[cur_rp]; 1119 dma = *((dma_addr_t *)skb->cb); 1120 dma_sync_single_for_cpu(rtwdev->dev, dma, RTK_PCI_RX_BUF_SIZE, 1121 DMA_FROM_DEVICE); 1122 rx_desc = skb->data; 1123 chip->ops->query_rx_desc(rtwdev, rx_desc, &pkt_stat, &rx_status); 1124 1125 /* offset from rx_desc to payload */ 1126 pkt_offset = pkt_desc_sz + pkt_stat.drv_info_sz + 1127 pkt_stat.shift; 1128 1129 /* allocate a new skb for this frame, 1130 * discard the frame if none available 1131 */ 1132 new_len = pkt_stat.pkt_len + pkt_offset; 1133 new = dev_alloc_skb(new_len); 1134 if (WARN_ONCE(!new, "rx routine starvation\n")) 1135 goto next_rp; 1136 1137 /* put the DMA data including rx_desc from phy to new skb */ 1138 skb_put_data(new, skb->data, new_len); 1139 1140 if (pkt_stat.is_c2h) { 1141 rtw_fw_c2h_cmd_rx_irqsafe(rtwdev, pkt_offset, new); 1142 } else { 1143 /* remove rx_desc */ 1144 skb_pull(new, pkt_offset); 1145 1146 rtw_rx_stats(rtwdev, pkt_stat.vif, new); 1147 memcpy(new->cb, &rx_status, sizeof(rx_status)); 1148 ieee80211_rx_napi(rtwdev->hw, NULL, new, napi); 1149 rx_done++; 1150 } 1151 1152 next_rp: 1153 /* new skb delivered to mac80211, re-enable original skb DMA */ 1154 rtw_pci_sync_rx_desc_device(rtwdev, dma, ring, cur_rp, 1155 buf_desc_sz); 1156 1157 /* host read next element in ring */ 1158 if (++cur_rp >= ring->r.len) 1159 cur_rp = 0; 1160 } 1161 1162 ring->r.rp = cur_rp; 1163 /* 'rp', the last position we have read, is seen as previous posistion 1164 * of 'wp' that is used to calculate 'count' next time. 1165 */ 1166 ring->r.wp = cur_rp; 1167 rtw_write16(rtwdev, RTK_PCI_RXBD_IDX_MPDUQ, ring->r.rp); 1168 1169 return rx_done; 1170 } 1171 1172 static void rtw_pci_irq_recognized(struct rtw_dev *rtwdev, 1173 struct rtw_pci *rtwpci, u32 *irq_status) 1174 { 1175 unsigned long flags; 1176 1177 spin_lock_irqsave(&rtwpci->hwirq_lock, flags); 1178 1179 irq_status[0] = rtw_read32(rtwdev, RTK_PCI_HISR0); 1180 irq_status[1] = rtw_read32(rtwdev, RTK_PCI_HISR1); 1181 if (rtw_chip_wcpu_11ac(rtwdev)) 1182 irq_status[3] = rtw_read32(rtwdev, RTK_PCI_HISR3); 1183 else 1184 irq_status[3] = 0; 1185 irq_status[0] &= rtwpci->irq_mask[0]; 1186 irq_status[1] &= rtwpci->irq_mask[1]; 1187 irq_status[3] &= rtwpci->irq_mask[3]; 1188 rtw_write32(rtwdev, RTK_PCI_HISR0, irq_status[0]); 1189 rtw_write32(rtwdev, RTK_PCI_HISR1, irq_status[1]); 1190 if (rtw_chip_wcpu_11ac(rtwdev)) 1191 rtw_write32(rtwdev, RTK_PCI_HISR3, irq_status[3]); 1192 1193 spin_unlock_irqrestore(&rtwpci->hwirq_lock, flags); 1194 } 1195 1196 static irqreturn_t rtw_pci_interrupt_handler(int irq, void *dev) 1197 { 1198 struct rtw_dev *rtwdev = dev; 1199 struct rtw_pci *rtwpci = (struct rtw_pci *)rtwdev->priv; 1200 1201 /* disable RTW PCI interrupt to avoid more interrupts before the end of 1202 * thread function 1203 * 1204 * disable HIMR here to also avoid new HISR flag being raised before 1205 * the HISRs have been Write-1-cleared for MSI. If not all of the HISRs 1206 * are cleared, the edge-triggered interrupt will not be generated when 1207 * a new HISR flag is set. 1208 */ 1209 rtw_pci_disable_interrupt(rtwdev, rtwpci); 1210 1211 return IRQ_WAKE_THREAD; 1212 } 1213 1214 static irqreturn_t rtw_pci_interrupt_threadfn(int irq, void *dev) 1215 { 1216 struct rtw_dev *rtwdev = dev; 1217 struct rtw_pci *rtwpci = (struct rtw_pci *)rtwdev->priv; 1218 u32 irq_status[4]; 1219 bool rx = false; 1220 1221 spin_lock_bh(&rtwpci->irq_lock); 1222 rtw_pci_irq_recognized(rtwdev, rtwpci, irq_status); 1223 1224 if (irq_status[0] & IMR_MGNTDOK) 1225 rtw_pci_tx_isr(rtwdev, rtwpci, RTW_TX_QUEUE_MGMT); 1226 if (irq_status[0] & IMR_HIGHDOK) 1227 rtw_pci_tx_isr(rtwdev, rtwpci, RTW_TX_QUEUE_HI0); 1228 if (irq_status[0] & IMR_BEDOK) 1229 rtw_pci_tx_isr(rtwdev, rtwpci, RTW_TX_QUEUE_BE); 1230 if (irq_status[0] & IMR_BKDOK) 1231 rtw_pci_tx_isr(rtwdev, rtwpci, RTW_TX_QUEUE_BK); 1232 if (irq_status[0] & IMR_VODOK) 1233 rtw_pci_tx_isr(rtwdev, rtwpci, RTW_TX_QUEUE_VO); 1234 if (irq_status[0] & IMR_VIDOK) 1235 rtw_pci_tx_isr(rtwdev, rtwpci, RTW_TX_QUEUE_VI); 1236 if (irq_status[3] & IMR_H2CDOK) 1237 rtw_pci_tx_isr(rtwdev, rtwpci, RTW_TX_QUEUE_H2C); 1238 if (irq_status[0] & IMR_ROK) { 1239 rtw_pci_rx_isr(rtwdev); 1240 rx = true; 1241 } 1242 if (unlikely(irq_status[0] & IMR_C2HCMD)) 1243 rtw_fw_c2h_cmd_isr(rtwdev); 1244 1245 /* all of the jobs for this interrupt have been done */ 1246 if (rtwpci->running) 1247 rtw_pci_enable_interrupt(rtwdev, rtwpci, rx); 1248 spin_unlock_bh(&rtwpci->irq_lock); 1249 1250 return IRQ_HANDLED; 1251 } 1252 1253 static int rtw_pci_io_mapping(struct rtw_dev *rtwdev, 1254 struct pci_dev *pdev) 1255 { 1256 struct rtw_pci *rtwpci = (struct rtw_pci *)rtwdev->priv; 1257 unsigned long len; 1258 u8 bar_id = 2; 1259 int ret; 1260 1261 ret = pci_request_regions(pdev, KBUILD_MODNAME); 1262 if (ret) { 1263 rtw_err(rtwdev, "failed to request pci regions\n"); 1264 return ret; 1265 } 1266 1267 #if defined(__FreeBSD__) 1268 ret = dma_set_mask(&pdev->dev, DMA_BIT_MASK(32)); 1269 if (ret) { 1270 rtw_err(rtwdev, "failed to set dma mask to 32-bit\n"); 1271 goto err_release_regions; 1272 } 1273 1274 ret = dma_set_coherent_mask(&pdev->dev, DMA_BIT_MASK(32)); 1275 if (ret) { 1276 rtw_err(rtwdev, "failed to set consistent dma mask to 32-bit\n"); 1277 goto err_release_regions; 1278 } 1279 #endif 1280 1281 len = pci_resource_len(pdev, bar_id); 1282 #if defined(__FreeBSD__) 1283 linuxkpi_pcim_want_to_use_bus_functions(pdev); 1284 #endif 1285 rtwpci->mmap = pci_iomap(pdev, bar_id, len); 1286 if (!rtwpci->mmap) { 1287 pci_release_regions(pdev); 1288 rtw_err(rtwdev, "failed to map pci memory\n"); 1289 return -ENOMEM; 1290 } 1291 1292 return 0; 1293 #if defined(__FreeBSD__) 1294 err_release_regions: 1295 pci_release_regions(pdev); 1296 return ret; 1297 #endif 1298 } 1299 1300 static void rtw_pci_io_unmapping(struct rtw_dev *rtwdev, 1301 struct pci_dev *pdev) 1302 { 1303 struct rtw_pci *rtwpci = (struct rtw_pci *)rtwdev->priv; 1304 1305 if (rtwpci->mmap) { 1306 pci_iounmap(pdev, rtwpci->mmap); 1307 pci_release_regions(pdev); 1308 } 1309 } 1310 1311 static void rtw_dbi_write8(struct rtw_dev *rtwdev, u16 addr, u8 data) 1312 { 1313 u16 write_addr; 1314 u16 remainder = addr & ~(BITS_DBI_WREN | BITS_DBI_ADDR_MASK); 1315 u8 flag; 1316 u8 cnt; 1317 1318 write_addr = addr & BITS_DBI_ADDR_MASK; 1319 write_addr |= u16_encode_bits(BIT(remainder), BITS_DBI_WREN); 1320 rtw_write8(rtwdev, REG_DBI_WDATA_V1 + remainder, data); 1321 rtw_write16(rtwdev, REG_DBI_FLAG_V1, write_addr); 1322 rtw_write8(rtwdev, REG_DBI_FLAG_V1 + 2, BIT_DBI_WFLAG >> 16); 1323 1324 for (cnt = 0; cnt < RTW_PCI_WR_RETRY_CNT; cnt++) { 1325 flag = rtw_read8(rtwdev, REG_DBI_FLAG_V1 + 2); 1326 if (flag == 0) 1327 return; 1328 1329 udelay(10); 1330 } 1331 1332 WARN(flag, "failed to write to DBI register, addr=0x%04x\n", addr); 1333 } 1334 1335 static int rtw_dbi_read8(struct rtw_dev *rtwdev, u16 addr, u8 *value) 1336 { 1337 u16 read_addr = addr & BITS_DBI_ADDR_MASK; 1338 u8 flag; 1339 u8 cnt; 1340 1341 rtw_write16(rtwdev, REG_DBI_FLAG_V1, read_addr); 1342 rtw_write8(rtwdev, REG_DBI_FLAG_V1 + 2, BIT_DBI_RFLAG >> 16); 1343 1344 for (cnt = 0; cnt < RTW_PCI_WR_RETRY_CNT; cnt++) { 1345 flag = rtw_read8(rtwdev, REG_DBI_FLAG_V1 + 2); 1346 if (flag == 0) { 1347 read_addr = REG_DBI_RDATA_V1 + (addr & 3); 1348 *value = rtw_read8(rtwdev, read_addr); 1349 return 0; 1350 } 1351 1352 udelay(10); 1353 } 1354 1355 WARN(1, "failed to read DBI register, addr=0x%04x\n", addr); 1356 return -EIO; 1357 } 1358 1359 static void rtw_mdio_write(struct rtw_dev *rtwdev, u8 addr, u16 data, bool g1) 1360 { 1361 u8 page; 1362 u8 wflag; 1363 u8 cnt; 1364 1365 rtw_write16(rtwdev, REG_MDIO_V1, data); 1366 1367 page = addr < RTW_PCI_MDIO_PG_SZ ? 0 : 1; 1368 page += g1 ? RTW_PCI_MDIO_PG_OFFS_G1 : RTW_PCI_MDIO_PG_OFFS_G2; 1369 rtw_write8(rtwdev, REG_PCIE_MIX_CFG, addr & BITS_MDIO_ADDR_MASK); 1370 rtw_write8(rtwdev, REG_PCIE_MIX_CFG + 3, page); 1371 rtw_write32_mask(rtwdev, REG_PCIE_MIX_CFG, BIT_MDIO_WFLAG_V1, 1); 1372 1373 for (cnt = 0; cnt < RTW_PCI_WR_RETRY_CNT; cnt++) { 1374 wflag = rtw_read32_mask(rtwdev, REG_PCIE_MIX_CFG, 1375 BIT_MDIO_WFLAG_V1); 1376 if (wflag == 0) 1377 return; 1378 1379 udelay(10); 1380 } 1381 1382 WARN(wflag, "failed to write to MDIO register, addr=0x%02x\n", addr); 1383 } 1384 1385 static void rtw_pci_clkreq_set(struct rtw_dev *rtwdev, bool enable) 1386 { 1387 u8 value; 1388 int ret; 1389 1390 if (rtw_pci_disable_aspm) 1391 return; 1392 1393 ret = rtw_dbi_read8(rtwdev, RTK_PCIE_LINK_CFG, &value); 1394 if (ret) { 1395 rtw_err(rtwdev, "failed to read CLKREQ_L1, ret=%d", ret); 1396 return; 1397 } 1398 1399 if (enable) 1400 value |= BIT_CLKREQ_SW_EN; 1401 else 1402 value &= ~BIT_CLKREQ_SW_EN; 1403 1404 rtw_dbi_write8(rtwdev, RTK_PCIE_LINK_CFG, value); 1405 } 1406 1407 static void rtw_pci_clkreq_pad_low(struct rtw_dev *rtwdev, bool enable) 1408 { 1409 u8 value; 1410 int ret; 1411 1412 ret = rtw_dbi_read8(rtwdev, RTK_PCIE_LINK_CFG, &value); 1413 if (ret) { 1414 rtw_err(rtwdev, "failed to read CLKREQ_L1, ret=%d", ret); 1415 return; 1416 } 1417 1418 if (enable) 1419 value &= ~BIT_CLKREQ_N_PAD; 1420 else 1421 value |= BIT_CLKREQ_N_PAD; 1422 1423 rtw_dbi_write8(rtwdev, RTK_PCIE_LINK_CFG, value); 1424 } 1425 1426 static void rtw_pci_aspm_set(struct rtw_dev *rtwdev, bool enable) 1427 { 1428 u8 value; 1429 int ret; 1430 1431 if (rtw_pci_disable_aspm) 1432 return; 1433 1434 ret = rtw_dbi_read8(rtwdev, RTK_PCIE_LINK_CFG, &value); 1435 if (ret) { 1436 rtw_err(rtwdev, "failed to read ASPM, ret=%d", ret); 1437 return; 1438 } 1439 1440 if (enable) 1441 value |= BIT_L1_SW_EN; 1442 else 1443 value &= ~BIT_L1_SW_EN; 1444 1445 rtw_dbi_write8(rtwdev, RTK_PCIE_LINK_CFG, value); 1446 } 1447 1448 static void rtw_pci_link_ps(struct rtw_dev *rtwdev, bool enter) 1449 { 1450 struct rtw_pci *rtwpci = (struct rtw_pci *)rtwdev->priv; 1451 1452 /* Like CLKREQ, ASPM is also implemented by two HW modules, and can 1453 * only be enabled when host supports it. 1454 * 1455 * And ASPM mechanism should be enabled when driver/firmware enters 1456 * power save mode, without having heavy traffic. Because we've 1457 * experienced some inter-operability issues that the link tends 1458 * to enter L1 state on the fly even when driver is having high 1459 * throughput. This is probably because the ASPM behavior slightly 1460 * varies from different SOC. 1461 */ 1462 if (!(rtwpci->link_ctrl & PCI_EXP_LNKCTL_ASPM_L1)) 1463 return; 1464 1465 if ((enter && atomic_dec_if_positive(&rtwpci->link_usage) == 0) || 1466 (!enter && atomic_inc_return(&rtwpci->link_usage) == 1)) 1467 rtw_pci_aspm_set(rtwdev, enter); 1468 } 1469 1470 static void rtw_pci_link_cfg(struct rtw_dev *rtwdev) 1471 { 1472 const struct rtw_chip_info *chip = rtwdev->chip; 1473 struct rtw_pci *rtwpci = (struct rtw_pci *)rtwdev->priv; 1474 struct pci_dev *pdev = rtwpci->pdev; 1475 u16 link_ctrl; 1476 int ret; 1477 1478 /* RTL8822CE has enabled REFCLK auto calibration, it does not need 1479 * to add clock delay to cover the REFCLK timing gap. 1480 */ 1481 if (chip->id == RTW_CHIP_TYPE_8822C) 1482 rtw_dbi_write8(rtwdev, RTK_PCIE_CLKDLY_CTRL, 0); 1483 1484 /* Though there is standard PCIE configuration space to set the 1485 * link control register, but by Realtek's design, driver should 1486 * check if host supports CLKREQ/ASPM to enable the HW module. 1487 * 1488 * These functions are implemented by two HW modules associated, 1489 * one is responsible to access PCIE configuration space to 1490 * follow the host settings, and another is in charge of doing 1491 * CLKREQ/ASPM mechanisms, it is default disabled. Because sometimes 1492 * the host does not support it, and due to some reasons or wrong 1493 * settings (ex. CLKREQ# not Bi-Direction), it could lead to device 1494 * loss if HW misbehaves on the link. 1495 * 1496 * Hence it's designed that driver should first check the PCIE 1497 * configuration space is sync'ed and enabled, then driver can turn 1498 * on the other module that is actually working on the mechanism. 1499 */ 1500 ret = pcie_capability_read_word(pdev, PCI_EXP_LNKCTL, &link_ctrl); 1501 if (ret) { 1502 rtw_err(rtwdev, "failed to read PCI cap, ret=%d\n", ret); 1503 return; 1504 } 1505 1506 if (link_ctrl & PCI_EXP_LNKCTL_CLKREQ_EN) 1507 rtw_pci_clkreq_set(rtwdev, true); 1508 1509 rtwpci->link_ctrl = link_ctrl; 1510 } 1511 1512 static void rtw_pci_interface_cfg(struct rtw_dev *rtwdev) 1513 { 1514 const struct rtw_chip_info *chip = rtwdev->chip; 1515 1516 switch (chip->id) { 1517 case RTW_CHIP_TYPE_8822C: 1518 if (rtwdev->hal.cut_version >= RTW_CHIP_VER_CUT_D) 1519 rtw_write32_mask(rtwdev, REG_HCI_MIX_CFG, 1520 BIT_PCIE_EMAC_PDN_AUX_TO_FAST_CLK, 1); 1521 break; 1522 default: 1523 break; 1524 } 1525 } 1526 1527 static void rtw_pci_phy_cfg(struct rtw_dev *rtwdev) 1528 { 1529 struct rtw_pci *rtwpci = (struct rtw_pci *)rtwdev->priv; 1530 const struct rtw_chip_info *chip = rtwdev->chip; 1531 struct pci_dev *pdev = rtwpci->pdev; 1532 const struct rtw_intf_phy_para *para; 1533 u16 cut; 1534 u16 value; 1535 u16 offset; 1536 int i; 1537 int ret; 1538 1539 cut = BIT(0) << rtwdev->hal.cut_version; 1540 1541 for (i = 0; i < chip->intf_table->n_gen1_para; i++) { 1542 para = &chip->intf_table->gen1_para[i]; 1543 if (!(para->cut_mask & cut)) 1544 continue; 1545 if (para->offset == 0xffff) 1546 break; 1547 offset = para->offset; 1548 value = para->value; 1549 if (para->ip_sel == RTW_IP_SEL_PHY) 1550 rtw_mdio_write(rtwdev, offset, value, true); 1551 else 1552 rtw_dbi_write8(rtwdev, offset, value); 1553 } 1554 1555 for (i = 0; i < chip->intf_table->n_gen2_para; i++) { 1556 para = &chip->intf_table->gen2_para[i]; 1557 if (!(para->cut_mask & cut)) 1558 continue; 1559 if (para->offset == 0xffff) 1560 break; 1561 offset = para->offset; 1562 value = para->value; 1563 if (para->ip_sel == RTW_IP_SEL_PHY) 1564 rtw_mdio_write(rtwdev, offset, value, false); 1565 else 1566 rtw_dbi_write8(rtwdev, offset, value); 1567 } 1568 1569 rtw_pci_link_cfg(rtwdev); 1570 1571 /* Disable 8821ce completion timeout by default */ 1572 if (chip->id == RTW_CHIP_TYPE_8821C) { 1573 ret = pcie_capability_set_word(pdev, PCI_EXP_DEVCTL2, 1574 PCI_EXP_DEVCTL2_COMP_TMOUT_DIS); 1575 if (ret) 1576 rtw_err(rtwdev, "failed to set PCI cap, ret = %d\n", 1577 ret); 1578 } 1579 } 1580 1581 static int __maybe_unused rtw_pci_suspend(struct device *dev) 1582 { 1583 struct ieee80211_hw *hw = dev_get_drvdata(dev); 1584 struct rtw_dev *rtwdev = hw->priv; 1585 const struct rtw_chip_info *chip = rtwdev->chip; 1586 struct rtw_efuse *efuse = &rtwdev->efuse; 1587 1588 if (chip->id == RTW_CHIP_TYPE_8822C && efuse->rfe_option == 6) 1589 rtw_pci_clkreq_pad_low(rtwdev, true); 1590 return 0; 1591 } 1592 1593 static int __maybe_unused rtw_pci_resume(struct device *dev) 1594 { 1595 struct ieee80211_hw *hw = dev_get_drvdata(dev); 1596 struct rtw_dev *rtwdev = hw->priv; 1597 const struct rtw_chip_info *chip = rtwdev->chip; 1598 struct rtw_efuse *efuse = &rtwdev->efuse; 1599 1600 if (chip->id == RTW_CHIP_TYPE_8822C && efuse->rfe_option == 6) 1601 rtw_pci_clkreq_pad_low(rtwdev, false); 1602 return 0; 1603 } 1604 1605 SIMPLE_DEV_PM_OPS(rtw_pm_ops, rtw_pci_suspend, rtw_pci_resume); 1606 EXPORT_SYMBOL(rtw_pm_ops); 1607 1608 static int rtw_pci_claim(struct rtw_dev *rtwdev, struct pci_dev *pdev) 1609 { 1610 int ret; 1611 1612 ret = pci_enable_device(pdev); 1613 if (ret) { 1614 rtw_err(rtwdev, "failed to enable pci device\n"); 1615 return ret; 1616 } 1617 1618 pci_set_master(pdev); 1619 pci_set_drvdata(pdev, rtwdev->hw); 1620 SET_IEEE80211_DEV(rtwdev->hw, &pdev->dev); 1621 1622 return 0; 1623 } 1624 1625 static void rtw_pci_declaim(struct rtw_dev *rtwdev, struct pci_dev *pdev) 1626 { 1627 pci_disable_device(pdev); 1628 } 1629 1630 static int rtw_pci_setup_resource(struct rtw_dev *rtwdev, struct pci_dev *pdev) 1631 { 1632 struct rtw_pci *rtwpci; 1633 int ret; 1634 1635 rtwpci = (struct rtw_pci *)rtwdev->priv; 1636 rtwpci->pdev = pdev; 1637 1638 /* after this driver can access to hw registers */ 1639 ret = rtw_pci_io_mapping(rtwdev, pdev); 1640 if (ret) { 1641 rtw_err(rtwdev, "failed to request pci io region\n"); 1642 goto err_out; 1643 } 1644 1645 ret = rtw_pci_init(rtwdev); 1646 if (ret) { 1647 rtw_err(rtwdev, "failed to allocate pci resources\n"); 1648 goto err_io_unmap; 1649 } 1650 1651 return 0; 1652 1653 err_io_unmap: 1654 rtw_pci_io_unmapping(rtwdev, pdev); 1655 1656 err_out: 1657 return ret; 1658 } 1659 1660 static void rtw_pci_destroy(struct rtw_dev *rtwdev, struct pci_dev *pdev) 1661 { 1662 rtw_pci_deinit(rtwdev); 1663 rtw_pci_io_unmapping(rtwdev, pdev); 1664 } 1665 1666 static struct rtw_hci_ops rtw_pci_ops = { 1667 .tx_write = rtw_pci_tx_write, 1668 .tx_kick_off = rtw_pci_tx_kick_off, 1669 .flush_queues = rtw_pci_flush_queues, 1670 .setup = rtw_pci_setup, 1671 .start = rtw_pci_start, 1672 .stop = rtw_pci_stop, 1673 .deep_ps = rtw_pci_deep_ps, 1674 .link_ps = rtw_pci_link_ps, 1675 .interface_cfg = rtw_pci_interface_cfg, 1676 1677 .read8 = rtw_pci_read8, 1678 .read16 = rtw_pci_read16, 1679 .read32 = rtw_pci_read32, 1680 .write8 = rtw_pci_write8, 1681 .write16 = rtw_pci_write16, 1682 .write32 = rtw_pci_write32, 1683 .write_data_rsvd_page = rtw_pci_write_data_rsvd_page, 1684 .write_data_h2c = rtw_pci_write_data_h2c, 1685 }; 1686 1687 static int rtw_pci_request_irq(struct rtw_dev *rtwdev, struct pci_dev *pdev) 1688 { 1689 unsigned int flags = PCI_IRQ_LEGACY; 1690 int ret; 1691 1692 if (!rtw_disable_msi) 1693 flags |= PCI_IRQ_MSI; 1694 1695 ret = pci_alloc_irq_vectors(pdev, 1, 1, flags); 1696 if (ret < 0) { 1697 rtw_err(rtwdev, "failed to alloc PCI irq vectors\n"); 1698 return ret; 1699 } 1700 1701 ret = devm_request_threaded_irq(rtwdev->dev, pdev->irq, 1702 rtw_pci_interrupt_handler, 1703 rtw_pci_interrupt_threadfn, 1704 IRQF_SHARED, KBUILD_MODNAME, rtwdev); 1705 if (ret) { 1706 rtw_err(rtwdev, "failed to request irq %d\n", ret); 1707 pci_free_irq_vectors(pdev); 1708 } 1709 1710 return ret; 1711 } 1712 1713 static void rtw_pci_free_irq(struct rtw_dev *rtwdev, struct pci_dev *pdev) 1714 { 1715 devm_free_irq(rtwdev->dev, pdev->irq, rtwdev); 1716 pci_free_irq_vectors(pdev); 1717 } 1718 1719 static int rtw_pci_napi_poll(struct napi_struct *napi, int budget) 1720 { 1721 struct rtw_pci *rtwpci = container_of(napi, struct rtw_pci, napi); 1722 struct rtw_dev *rtwdev = container_of((void *)rtwpci, struct rtw_dev, 1723 priv); 1724 int work_done = 0; 1725 1726 if (rtwpci->rx_no_aspm) 1727 rtw_pci_link_ps(rtwdev, false); 1728 1729 while (work_done < budget) { 1730 u32 work_done_once; 1731 1732 work_done_once = rtw_pci_rx_napi(rtwdev, rtwpci, RTW_RX_QUEUE_MPDU, 1733 budget - work_done); 1734 if (work_done_once == 0) 1735 break; 1736 work_done += work_done_once; 1737 } 1738 if (work_done < budget) { 1739 napi_complete_done(napi, work_done); 1740 spin_lock_bh(&rtwpci->irq_lock); 1741 if (rtwpci->running) 1742 rtw_pci_enable_interrupt(rtwdev, rtwpci, false); 1743 spin_unlock_bh(&rtwpci->irq_lock); 1744 /* When ISR happens during polling and before napi_complete 1745 * while no further data is received. Data on the dma_ring will 1746 * not be processed immediately. Check whether dma ring is 1747 * empty and perform napi_schedule accordingly. 1748 */ 1749 if (rtw_pci_get_hw_rx_ring_nr(rtwdev, rtwpci)) 1750 napi_schedule(napi); 1751 } 1752 if (rtwpci->rx_no_aspm) 1753 rtw_pci_link_ps(rtwdev, true); 1754 1755 return work_done; 1756 } 1757 1758 static void rtw_pci_napi_init(struct rtw_dev *rtwdev) 1759 { 1760 struct rtw_pci *rtwpci = (struct rtw_pci *)rtwdev->priv; 1761 1762 init_dummy_netdev(&rtwpci->netdev); 1763 netif_napi_add(&rtwpci->netdev, &rtwpci->napi, rtw_pci_napi_poll); 1764 } 1765 1766 static void rtw_pci_napi_deinit(struct rtw_dev *rtwdev) 1767 { 1768 struct rtw_pci *rtwpci = (struct rtw_pci *)rtwdev->priv; 1769 1770 rtw_pci_napi_stop(rtwdev); 1771 netif_napi_del(&rtwpci->napi); 1772 } 1773 1774 int rtw_pci_probe(struct pci_dev *pdev, 1775 const struct pci_device_id *id) 1776 { 1777 struct pci_dev *bridge = pci_upstream_bridge(pdev); 1778 struct ieee80211_hw *hw; 1779 struct rtw_dev *rtwdev; 1780 struct rtw_pci *rtwpci; 1781 int drv_data_size; 1782 int ret; 1783 1784 drv_data_size = sizeof(struct rtw_dev) + sizeof(struct rtw_pci); 1785 hw = ieee80211_alloc_hw(drv_data_size, &rtw_ops); 1786 if (!hw) { 1787 dev_err(&pdev->dev, "failed to allocate hw\n"); 1788 return -ENOMEM; 1789 } 1790 1791 rtwdev = hw->priv; 1792 rtwdev->hw = hw; 1793 rtwdev->dev = &pdev->dev; 1794 rtwdev->chip = (struct rtw_chip_info *)id->driver_data; 1795 rtwdev->hci.ops = &rtw_pci_ops; 1796 rtwdev->hci.type = RTW_HCI_TYPE_PCIE; 1797 1798 rtwpci = (struct rtw_pci *)rtwdev->priv; 1799 atomic_set(&rtwpci->link_usage, 1); 1800 1801 ret = rtw_core_init(rtwdev); 1802 if (ret) 1803 goto err_release_hw; 1804 1805 rtw_dbg(rtwdev, RTW_DBG_PCI, 1806 "rtw88 pci probe: vendor=0x%4.04X device=0x%4.04X rev=%d\n", 1807 pdev->vendor, pdev->device, pdev->revision); 1808 1809 ret = rtw_pci_claim(rtwdev, pdev); 1810 if (ret) { 1811 rtw_err(rtwdev, "failed to claim pci device\n"); 1812 goto err_deinit_core; 1813 } 1814 1815 ret = rtw_pci_setup_resource(rtwdev, pdev); 1816 if (ret) { 1817 rtw_err(rtwdev, "failed to setup pci resources\n"); 1818 goto err_pci_declaim; 1819 } 1820 1821 rtw_pci_napi_init(rtwdev); 1822 1823 ret = rtw_chip_info_setup(rtwdev); 1824 if (ret) { 1825 rtw_err(rtwdev, "failed to setup chip information\n"); 1826 goto err_destroy_pci; 1827 } 1828 1829 /* Disable PCIe ASPM L1 while doing NAPI poll for 8821CE */ 1830 if (rtwdev->chip->id == RTW_CHIP_TYPE_8821C && bridge->vendor == PCI_VENDOR_ID_INTEL) 1831 rtwpci->rx_no_aspm = true; 1832 1833 rtw_pci_phy_cfg(rtwdev); 1834 1835 ret = rtw_register_hw(rtwdev, hw); 1836 if (ret) { 1837 rtw_err(rtwdev, "failed to register hw\n"); 1838 goto err_destroy_pci; 1839 } 1840 1841 ret = rtw_pci_request_irq(rtwdev, pdev); 1842 if (ret) { 1843 ieee80211_unregister_hw(hw); 1844 goto err_destroy_pci; 1845 } 1846 1847 return 0; 1848 1849 err_destroy_pci: 1850 rtw_pci_napi_deinit(rtwdev); 1851 rtw_pci_destroy(rtwdev, pdev); 1852 1853 err_pci_declaim: 1854 rtw_pci_declaim(rtwdev, pdev); 1855 1856 err_deinit_core: 1857 rtw_core_deinit(rtwdev); 1858 1859 err_release_hw: 1860 ieee80211_free_hw(hw); 1861 1862 return ret; 1863 } 1864 EXPORT_SYMBOL(rtw_pci_probe); 1865 1866 void rtw_pci_remove(struct pci_dev *pdev) 1867 { 1868 struct ieee80211_hw *hw = pci_get_drvdata(pdev); 1869 struct rtw_dev *rtwdev; 1870 struct rtw_pci *rtwpci; 1871 1872 if (!hw) 1873 return; 1874 1875 rtwdev = hw->priv; 1876 rtwpci = (struct rtw_pci *)rtwdev->priv; 1877 1878 rtw_unregister_hw(rtwdev, hw); 1879 rtw_pci_disable_interrupt(rtwdev, rtwpci); 1880 rtw_pci_napi_deinit(rtwdev); 1881 rtw_pci_destroy(rtwdev, pdev); 1882 rtw_pci_declaim(rtwdev, pdev); 1883 rtw_pci_free_irq(rtwdev, pdev); 1884 rtw_core_deinit(rtwdev); 1885 ieee80211_free_hw(hw); 1886 } 1887 EXPORT_SYMBOL(rtw_pci_remove); 1888 1889 void rtw_pci_shutdown(struct pci_dev *pdev) 1890 { 1891 struct ieee80211_hw *hw = pci_get_drvdata(pdev); 1892 struct rtw_dev *rtwdev; 1893 const struct rtw_chip_info *chip; 1894 1895 if (!hw) 1896 return; 1897 1898 rtwdev = hw->priv; 1899 chip = rtwdev->chip; 1900 1901 if (chip->ops->shutdown) 1902 chip->ops->shutdown(rtwdev); 1903 1904 pci_set_power_state(pdev, PCI_D3hot); 1905 } 1906 EXPORT_SYMBOL(rtw_pci_shutdown); 1907 1908 MODULE_AUTHOR("Realtek Corporation"); 1909 MODULE_DESCRIPTION("Realtek 802.11ac wireless PCI driver"); 1910 MODULE_LICENSE("Dual BSD/GPL"); 1911 #if defined(__FreeBSD__) 1912 MODULE_VERSION(rtw_pci, 1); 1913 MODULE_DEPEND(rtw_pci, linuxkpi, 1, 1, 1); 1914 MODULE_DEPEND(rtw_pci, linuxkpi_wlan, 1, 1, 1); 1915 #ifdef CONFIG_RTW88_DEBUGFS 1916 MODULE_DEPEND(rtw_pci, lindebugfs, 1, 1, 1); 1917 #endif 1918 #endif 1919