1 // SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause 2 /* Copyright (c) 2021, Microsoft Corporation. */ 3 4 #include <uapi/linux/bpf.h> 5 6 #include <linux/inetdevice.h> 7 #include <linux/etherdevice.h> 8 #include <linux/ethtool.h> 9 #include <linux/filter.h> 10 #include <linux/mm.h> 11 12 #include <net/checksum.h> 13 #include <net/ip6_checksum.h> 14 15 #include "mana.h" 16 17 /* Microsoft Azure Network Adapter (MANA) functions */ 18 19 static int mana_open(struct net_device *ndev) 20 { 21 struct mana_port_context *apc = netdev_priv(ndev); 22 int err; 23 24 err = mana_alloc_queues(ndev); 25 if (err) 26 return err; 27 28 apc->port_is_up = true; 29 30 /* Ensure port state updated before txq state */ 31 smp_wmb(); 32 33 netif_carrier_on(ndev); 34 netif_tx_wake_all_queues(ndev); 35 36 return 0; 37 } 38 39 static int mana_close(struct net_device *ndev) 40 { 41 struct mana_port_context *apc = netdev_priv(ndev); 42 43 if (!apc->port_is_up) 44 return 0; 45 46 return mana_detach(ndev, true); 47 } 48 49 static bool mana_can_tx(struct gdma_queue *wq) 50 { 51 return mana_gd_wq_avail_space(wq) >= MAX_TX_WQE_SIZE; 52 } 53 54 static unsigned int mana_checksum_info(struct sk_buff *skb) 55 { 56 if (skb->protocol == htons(ETH_P_IP)) { 57 struct iphdr *ip = ip_hdr(skb); 58 59 if (ip->protocol == IPPROTO_TCP) 60 return IPPROTO_TCP; 61 62 if (ip->protocol == IPPROTO_UDP) 63 return IPPROTO_UDP; 64 } else if (skb->protocol == htons(ETH_P_IPV6)) { 65 struct ipv6hdr *ip6 = ipv6_hdr(skb); 66 67 if (ip6->nexthdr == IPPROTO_TCP) 68 return IPPROTO_TCP; 69 70 if (ip6->nexthdr == IPPROTO_UDP) 71 return IPPROTO_UDP; 72 } 73 74 /* No csum offloading */ 75 return 0; 76 } 77 78 static int mana_map_skb(struct sk_buff *skb, struct mana_port_context *apc, 79 struct mana_tx_package *tp) 80 { 81 struct mana_skb_head *ash = (struct mana_skb_head *)skb->head; 82 struct gdma_dev *gd = apc->ac->gdma_dev; 83 struct gdma_context *gc; 84 struct device *dev; 85 skb_frag_t *frag; 86 dma_addr_t da; 87 int i; 88 89 gc = gd->gdma_context; 90 dev = gc->dev; 91 da = dma_map_single(dev, skb->data, skb_headlen(skb), DMA_TO_DEVICE); 92 93 if (dma_mapping_error(dev, da)) 94 return -ENOMEM; 95 96 ash->dma_handle[0] = da; 97 ash->size[0] = skb_headlen(skb); 98 99 tp->wqe_req.sgl[0].address = ash->dma_handle[0]; 100 tp->wqe_req.sgl[0].mem_key = gd->gpa_mkey; 101 tp->wqe_req.sgl[0].size = ash->size[0]; 102 103 for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) { 104 frag = &skb_shinfo(skb)->frags[i]; 105 da = skb_frag_dma_map(dev, frag, 0, skb_frag_size(frag), 106 DMA_TO_DEVICE); 107 108 if (dma_mapping_error(dev, da)) 109 goto frag_err; 110 111 ash->dma_handle[i + 1] = da; 112 ash->size[i + 1] = skb_frag_size(frag); 113 114 tp->wqe_req.sgl[i + 1].address = ash->dma_handle[i + 1]; 115 tp->wqe_req.sgl[i + 1].mem_key = gd->gpa_mkey; 116 tp->wqe_req.sgl[i + 1].size = ash->size[i + 1]; 117 } 118 119 return 0; 120 121 frag_err: 122 for (i = i - 1; i >= 0; i--) 123 dma_unmap_page(dev, ash->dma_handle[i + 1], ash->size[i + 1], 124 DMA_TO_DEVICE); 125 126 dma_unmap_single(dev, ash->dma_handle[0], ash->size[0], DMA_TO_DEVICE); 127 128 return -ENOMEM; 129 } 130 131 int mana_start_xmit(struct sk_buff *skb, struct net_device *ndev) 132 { 133 enum mana_tx_pkt_format pkt_fmt = MANA_SHORT_PKT_FMT; 134 struct mana_port_context *apc = netdev_priv(ndev); 135 u16 txq_idx = skb_get_queue_mapping(skb); 136 struct gdma_dev *gd = apc->ac->gdma_dev; 137 bool ipv4 = false, ipv6 = false; 138 struct mana_tx_package pkg = {}; 139 struct netdev_queue *net_txq; 140 struct mana_stats_tx *tx_stats; 141 struct gdma_queue *gdma_sq; 142 unsigned int csum_type; 143 struct mana_txq *txq; 144 struct mana_cq *cq; 145 int err, len; 146 147 if (unlikely(!apc->port_is_up)) 148 goto tx_drop; 149 150 if (skb_cow_head(skb, MANA_HEADROOM)) 151 goto tx_drop_count; 152 153 txq = &apc->tx_qp[txq_idx].txq; 154 gdma_sq = txq->gdma_sq; 155 cq = &apc->tx_qp[txq_idx].tx_cq; 156 157 pkg.tx_oob.s_oob.vcq_num = cq->gdma_id; 158 pkg.tx_oob.s_oob.vsq_frame = txq->vsq_frame; 159 160 if (txq->vp_offset > MANA_SHORT_VPORT_OFFSET_MAX) { 161 pkg.tx_oob.l_oob.long_vp_offset = txq->vp_offset; 162 pkt_fmt = MANA_LONG_PKT_FMT; 163 } else { 164 pkg.tx_oob.s_oob.short_vp_offset = txq->vp_offset; 165 } 166 167 pkg.tx_oob.s_oob.pkt_fmt = pkt_fmt; 168 169 if (pkt_fmt == MANA_SHORT_PKT_FMT) 170 pkg.wqe_req.inline_oob_size = sizeof(struct mana_tx_short_oob); 171 else 172 pkg.wqe_req.inline_oob_size = sizeof(struct mana_tx_oob); 173 174 pkg.wqe_req.inline_oob_data = &pkg.tx_oob; 175 pkg.wqe_req.flags = 0; 176 pkg.wqe_req.client_data_unit = 0; 177 178 pkg.wqe_req.num_sge = 1 + skb_shinfo(skb)->nr_frags; 179 WARN_ON_ONCE(pkg.wqe_req.num_sge > 30); 180 181 if (pkg.wqe_req.num_sge <= ARRAY_SIZE(pkg.sgl_array)) { 182 pkg.wqe_req.sgl = pkg.sgl_array; 183 } else { 184 pkg.sgl_ptr = kmalloc_array(pkg.wqe_req.num_sge, 185 sizeof(struct gdma_sge), 186 GFP_ATOMIC); 187 if (!pkg.sgl_ptr) 188 goto tx_drop_count; 189 190 pkg.wqe_req.sgl = pkg.sgl_ptr; 191 } 192 193 if (skb->protocol == htons(ETH_P_IP)) 194 ipv4 = true; 195 else if (skb->protocol == htons(ETH_P_IPV6)) 196 ipv6 = true; 197 198 if (skb_is_gso(skb)) { 199 pkg.tx_oob.s_oob.is_outer_ipv4 = ipv4; 200 pkg.tx_oob.s_oob.is_outer_ipv6 = ipv6; 201 202 pkg.tx_oob.s_oob.comp_iphdr_csum = 1; 203 pkg.tx_oob.s_oob.comp_tcp_csum = 1; 204 pkg.tx_oob.s_oob.trans_off = skb_transport_offset(skb); 205 206 pkg.wqe_req.client_data_unit = skb_shinfo(skb)->gso_size; 207 pkg.wqe_req.flags = GDMA_WR_OOB_IN_SGL | GDMA_WR_PAD_BY_SGE0; 208 if (ipv4) { 209 ip_hdr(skb)->tot_len = 0; 210 ip_hdr(skb)->check = 0; 211 tcp_hdr(skb)->check = 212 ~csum_tcpudp_magic(ip_hdr(skb)->saddr, 213 ip_hdr(skb)->daddr, 0, 214 IPPROTO_TCP, 0); 215 } else { 216 ipv6_hdr(skb)->payload_len = 0; 217 tcp_hdr(skb)->check = 218 ~csum_ipv6_magic(&ipv6_hdr(skb)->saddr, 219 &ipv6_hdr(skb)->daddr, 0, 220 IPPROTO_TCP, 0); 221 } 222 } else if (skb->ip_summed == CHECKSUM_PARTIAL) { 223 csum_type = mana_checksum_info(skb); 224 225 if (csum_type == IPPROTO_TCP) { 226 pkg.tx_oob.s_oob.is_outer_ipv4 = ipv4; 227 pkg.tx_oob.s_oob.is_outer_ipv6 = ipv6; 228 229 pkg.tx_oob.s_oob.comp_tcp_csum = 1; 230 pkg.tx_oob.s_oob.trans_off = skb_transport_offset(skb); 231 232 } else if (csum_type == IPPROTO_UDP) { 233 pkg.tx_oob.s_oob.is_outer_ipv4 = ipv4; 234 pkg.tx_oob.s_oob.is_outer_ipv6 = ipv6; 235 236 pkg.tx_oob.s_oob.comp_udp_csum = 1; 237 } else { 238 /* Can't do offload of this type of checksum */ 239 if (skb_checksum_help(skb)) 240 goto free_sgl_ptr; 241 } 242 } 243 244 if (mana_map_skb(skb, apc, &pkg)) 245 goto free_sgl_ptr; 246 247 skb_queue_tail(&txq->pending_skbs, skb); 248 249 len = skb->len; 250 net_txq = netdev_get_tx_queue(ndev, txq_idx); 251 252 err = mana_gd_post_work_request(gdma_sq, &pkg.wqe_req, 253 (struct gdma_posted_wqe_info *)skb->cb); 254 if (!mana_can_tx(gdma_sq)) { 255 netif_tx_stop_queue(net_txq); 256 apc->eth_stats.stop_queue++; 257 } 258 259 if (err) { 260 (void)skb_dequeue_tail(&txq->pending_skbs); 261 netdev_warn(ndev, "Failed to post TX OOB: %d\n", err); 262 err = NETDEV_TX_BUSY; 263 goto tx_busy; 264 } 265 266 err = NETDEV_TX_OK; 267 atomic_inc(&txq->pending_sends); 268 269 mana_gd_wq_ring_doorbell(gd->gdma_context, gdma_sq); 270 271 /* skb may be freed after mana_gd_post_work_request. Do not use it. */ 272 skb = NULL; 273 274 tx_stats = &txq->stats; 275 u64_stats_update_begin(&tx_stats->syncp); 276 tx_stats->packets++; 277 tx_stats->bytes += len; 278 u64_stats_update_end(&tx_stats->syncp); 279 280 tx_busy: 281 if (netif_tx_queue_stopped(net_txq) && mana_can_tx(gdma_sq)) { 282 netif_tx_wake_queue(net_txq); 283 apc->eth_stats.wake_queue++; 284 } 285 286 kfree(pkg.sgl_ptr); 287 return err; 288 289 free_sgl_ptr: 290 kfree(pkg.sgl_ptr); 291 tx_drop_count: 292 ndev->stats.tx_dropped++; 293 tx_drop: 294 dev_kfree_skb_any(skb); 295 return NETDEV_TX_OK; 296 } 297 298 static void mana_get_stats64(struct net_device *ndev, 299 struct rtnl_link_stats64 *st) 300 { 301 struct mana_port_context *apc = netdev_priv(ndev); 302 unsigned int num_queues = apc->num_queues; 303 struct mana_stats_rx *rx_stats; 304 struct mana_stats_tx *tx_stats; 305 unsigned int start; 306 u64 packets, bytes; 307 int q; 308 309 if (!apc->port_is_up) 310 return; 311 312 netdev_stats_to_stats64(st, &ndev->stats); 313 314 for (q = 0; q < num_queues; q++) { 315 rx_stats = &apc->rxqs[q]->stats; 316 317 do { 318 start = u64_stats_fetch_begin_irq(&rx_stats->syncp); 319 packets = rx_stats->packets; 320 bytes = rx_stats->bytes; 321 } while (u64_stats_fetch_retry_irq(&rx_stats->syncp, start)); 322 323 st->rx_packets += packets; 324 st->rx_bytes += bytes; 325 } 326 327 for (q = 0; q < num_queues; q++) { 328 tx_stats = &apc->tx_qp[q].txq.stats; 329 330 do { 331 start = u64_stats_fetch_begin_irq(&tx_stats->syncp); 332 packets = tx_stats->packets; 333 bytes = tx_stats->bytes; 334 } while (u64_stats_fetch_retry_irq(&tx_stats->syncp, start)); 335 336 st->tx_packets += packets; 337 st->tx_bytes += bytes; 338 } 339 } 340 341 static int mana_get_tx_queue(struct net_device *ndev, struct sk_buff *skb, 342 int old_q) 343 { 344 struct mana_port_context *apc = netdev_priv(ndev); 345 u32 hash = skb_get_hash(skb); 346 struct sock *sk = skb->sk; 347 int txq; 348 349 txq = apc->indir_table[hash & MANA_INDIRECT_TABLE_MASK]; 350 351 if (txq != old_q && sk && sk_fullsock(sk) && 352 rcu_access_pointer(sk->sk_dst_cache)) 353 sk_tx_queue_set(sk, txq); 354 355 return txq; 356 } 357 358 static u16 mana_select_queue(struct net_device *ndev, struct sk_buff *skb, 359 struct net_device *sb_dev) 360 { 361 int txq; 362 363 if (ndev->real_num_tx_queues == 1) 364 return 0; 365 366 txq = sk_tx_queue_get(skb->sk); 367 368 if (txq < 0 || skb->ooo_okay || txq >= ndev->real_num_tx_queues) { 369 if (skb_rx_queue_recorded(skb)) 370 txq = skb_get_rx_queue(skb); 371 else 372 txq = mana_get_tx_queue(ndev, skb, txq); 373 } 374 375 return txq; 376 } 377 378 static const struct net_device_ops mana_devops = { 379 .ndo_open = mana_open, 380 .ndo_stop = mana_close, 381 .ndo_select_queue = mana_select_queue, 382 .ndo_start_xmit = mana_start_xmit, 383 .ndo_validate_addr = eth_validate_addr, 384 .ndo_get_stats64 = mana_get_stats64, 385 .ndo_bpf = mana_bpf, 386 .ndo_xdp_xmit = mana_xdp_xmit, 387 }; 388 389 static void mana_cleanup_port_context(struct mana_port_context *apc) 390 { 391 kfree(apc->rxqs); 392 apc->rxqs = NULL; 393 } 394 395 static int mana_init_port_context(struct mana_port_context *apc) 396 { 397 apc->rxqs = kcalloc(apc->num_queues, sizeof(struct mana_rxq *), 398 GFP_KERNEL); 399 400 return !apc->rxqs ? -ENOMEM : 0; 401 } 402 403 static int mana_send_request(struct mana_context *ac, void *in_buf, 404 u32 in_len, void *out_buf, u32 out_len) 405 { 406 struct gdma_context *gc = ac->gdma_dev->gdma_context; 407 struct gdma_resp_hdr *resp = out_buf; 408 struct gdma_req_hdr *req = in_buf; 409 struct device *dev = gc->dev; 410 static atomic_t activity_id; 411 int err; 412 413 req->dev_id = gc->mana.dev_id; 414 req->activity_id = atomic_inc_return(&activity_id); 415 416 err = mana_gd_send_request(gc, in_len, in_buf, out_len, 417 out_buf); 418 if (err || resp->status) { 419 dev_err(dev, "Failed to send mana message: %d, 0x%x\n", 420 err, resp->status); 421 return err ? err : -EPROTO; 422 } 423 424 if (req->dev_id.as_uint32 != resp->dev_id.as_uint32 || 425 req->activity_id != resp->activity_id) { 426 dev_err(dev, "Unexpected mana message response: %x,%x,%x,%x\n", 427 req->dev_id.as_uint32, resp->dev_id.as_uint32, 428 req->activity_id, resp->activity_id); 429 return -EPROTO; 430 } 431 432 return 0; 433 } 434 435 static int mana_verify_resp_hdr(const struct gdma_resp_hdr *resp_hdr, 436 const enum mana_command_code expected_code, 437 const u32 min_size) 438 { 439 if (resp_hdr->response.msg_type != expected_code) 440 return -EPROTO; 441 442 if (resp_hdr->response.msg_version < GDMA_MESSAGE_V1) 443 return -EPROTO; 444 445 if (resp_hdr->response.msg_size < min_size) 446 return -EPROTO; 447 448 return 0; 449 } 450 451 static int mana_pf_register_hw_vport(struct mana_port_context *apc) 452 { 453 struct mana_register_hw_vport_resp resp = {}; 454 struct mana_register_hw_vport_req req = {}; 455 int err; 456 457 mana_gd_init_req_hdr(&req.hdr, MANA_REGISTER_HW_PORT, 458 sizeof(req), sizeof(resp)); 459 req.attached_gfid = 1; 460 req.is_pf_default_vport = 1; 461 req.allow_all_ether_types = 1; 462 463 err = mana_send_request(apc->ac, &req, sizeof(req), &resp, 464 sizeof(resp)); 465 if (err) { 466 netdev_err(apc->ndev, "Failed to register hw vPort: %d\n", err); 467 return err; 468 } 469 470 err = mana_verify_resp_hdr(&resp.hdr, MANA_REGISTER_HW_PORT, 471 sizeof(resp)); 472 if (err || resp.hdr.status) { 473 netdev_err(apc->ndev, "Failed to register hw vPort: %d, 0x%x\n", 474 err, resp.hdr.status); 475 return err ? err : -EPROTO; 476 } 477 478 apc->port_handle = resp.hw_vport_handle; 479 return 0; 480 } 481 482 static void mana_pf_deregister_hw_vport(struct mana_port_context *apc) 483 { 484 struct mana_deregister_hw_vport_resp resp = {}; 485 struct mana_deregister_hw_vport_req req = {}; 486 int err; 487 488 mana_gd_init_req_hdr(&req.hdr, MANA_DEREGISTER_HW_PORT, 489 sizeof(req), sizeof(resp)); 490 req.hw_vport_handle = apc->port_handle; 491 492 err = mana_send_request(apc->ac, &req, sizeof(req), &resp, 493 sizeof(resp)); 494 if (err) { 495 netdev_err(apc->ndev, "Failed to unregister hw vPort: %d\n", 496 err); 497 return; 498 } 499 500 err = mana_verify_resp_hdr(&resp.hdr, MANA_DEREGISTER_HW_PORT, 501 sizeof(resp)); 502 if (err || resp.hdr.status) 503 netdev_err(apc->ndev, 504 "Failed to deregister hw vPort: %d, 0x%x\n", 505 err, resp.hdr.status); 506 } 507 508 static int mana_pf_register_filter(struct mana_port_context *apc) 509 { 510 struct mana_register_filter_resp resp = {}; 511 struct mana_register_filter_req req = {}; 512 int err; 513 514 mana_gd_init_req_hdr(&req.hdr, MANA_REGISTER_FILTER, 515 sizeof(req), sizeof(resp)); 516 req.vport = apc->port_handle; 517 memcpy(req.mac_addr, apc->mac_addr, ETH_ALEN); 518 519 err = mana_send_request(apc->ac, &req, sizeof(req), &resp, 520 sizeof(resp)); 521 if (err) { 522 netdev_err(apc->ndev, "Failed to register filter: %d\n", err); 523 return err; 524 } 525 526 err = mana_verify_resp_hdr(&resp.hdr, MANA_REGISTER_FILTER, 527 sizeof(resp)); 528 if (err || resp.hdr.status) { 529 netdev_err(apc->ndev, "Failed to register filter: %d, 0x%x\n", 530 err, resp.hdr.status); 531 return err ? err : -EPROTO; 532 } 533 534 apc->pf_filter_handle = resp.filter_handle; 535 return 0; 536 } 537 538 static void mana_pf_deregister_filter(struct mana_port_context *apc) 539 { 540 struct mana_deregister_filter_resp resp = {}; 541 struct mana_deregister_filter_req req = {}; 542 int err; 543 544 mana_gd_init_req_hdr(&req.hdr, MANA_DEREGISTER_FILTER, 545 sizeof(req), sizeof(resp)); 546 req.filter_handle = apc->pf_filter_handle; 547 548 err = mana_send_request(apc->ac, &req, sizeof(req), &resp, 549 sizeof(resp)); 550 if (err) { 551 netdev_err(apc->ndev, "Failed to unregister filter: %d\n", 552 err); 553 return; 554 } 555 556 err = mana_verify_resp_hdr(&resp.hdr, MANA_DEREGISTER_FILTER, 557 sizeof(resp)); 558 if (err || resp.hdr.status) 559 netdev_err(apc->ndev, 560 "Failed to deregister filter: %d, 0x%x\n", 561 err, resp.hdr.status); 562 } 563 564 static int mana_query_device_cfg(struct mana_context *ac, u32 proto_major_ver, 565 u32 proto_minor_ver, u32 proto_micro_ver, 566 u16 *max_num_vports) 567 { 568 struct gdma_context *gc = ac->gdma_dev->gdma_context; 569 struct mana_query_device_cfg_resp resp = {}; 570 struct mana_query_device_cfg_req req = {}; 571 struct device *dev = gc->dev; 572 int err = 0; 573 574 mana_gd_init_req_hdr(&req.hdr, MANA_QUERY_DEV_CONFIG, 575 sizeof(req), sizeof(resp)); 576 req.proto_major_ver = proto_major_ver; 577 req.proto_minor_ver = proto_minor_ver; 578 req.proto_micro_ver = proto_micro_ver; 579 580 err = mana_send_request(ac, &req, sizeof(req), &resp, sizeof(resp)); 581 if (err) { 582 dev_err(dev, "Failed to query config: %d", err); 583 return err; 584 } 585 586 err = mana_verify_resp_hdr(&resp.hdr, MANA_QUERY_DEV_CONFIG, 587 sizeof(resp)); 588 if (err || resp.hdr.status) { 589 dev_err(dev, "Invalid query result: %d, 0x%x\n", err, 590 resp.hdr.status); 591 if (!err) 592 err = -EPROTO; 593 return err; 594 } 595 596 *max_num_vports = resp.max_num_vports; 597 598 return 0; 599 } 600 601 static int mana_query_vport_cfg(struct mana_port_context *apc, u32 vport_index, 602 u32 *max_sq, u32 *max_rq, u32 *num_indir_entry) 603 { 604 struct mana_query_vport_cfg_resp resp = {}; 605 struct mana_query_vport_cfg_req req = {}; 606 int err; 607 608 mana_gd_init_req_hdr(&req.hdr, MANA_QUERY_VPORT_CONFIG, 609 sizeof(req), sizeof(resp)); 610 611 req.vport_index = vport_index; 612 613 err = mana_send_request(apc->ac, &req, sizeof(req), &resp, 614 sizeof(resp)); 615 if (err) 616 return err; 617 618 err = mana_verify_resp_hdr(&resp.hdr, MANA_QUERY_VPORT_CONFIG, 619 sizeof(resp)); 620 if (err) 621 return err; 622 623 if (resp.hdr.status) 624 return -EPROTO; 625 626 *max_sq = resp.max_num_sq; 627 *max_rq = resp.max_num_rq; 628 *num_indir_entry = resp.num_indirection_ent; 629 630 apc->port_handle = resp.vport; 631 ether_addr_copy(apc->mac_addr, resp.mac_addr); 632 633 return 0; 634 } 635 636 static int mana_cfg_vport(struct mana_port_context *apc, u32 protection_dom_id, 637 u32 doorbell_pg_id) 638 { 639 struct mana_config_vport_resp resp = {}; 640 struct mana_config_vport_req req = {}; 641 int err; 642 643 mana_gd_init_req_hdr(&req.hdr, MANA_CONFIG_VPORT_TX, 644 sizeof(req), sizeof(resp)); 645 req.vport = apc->port_handle; 646 req.pdid = protection_dom_id; 647 req.doorbell_pageid = doorbell_pg_id; 648 649 err = mana_send_request(apc->ac, &req, sizeof(req), &resp, 650 sizeof(resp)); 651 if (err) { 652 netdev_err(apc->ndev, "Failed to configure vPort: %d\n", err); 653 goto out; 654 } 655 656 err = mana_verify_resp_hdr(&resp.hdr, MANA_CONFIG_VPORT_TX, 657 sizeof(resp)); 658 if (err || resp.hdr.status) { 659 netdev_err(apc->ndev, "Failed to configure vPort: %d, 0x%x\n", 660 err, resp.hdr.status); 661 if (!err) 662 err = -EPROTO; 663 664 goto out; 665 } 666 667 apc->tx_shortform_allowed = resp.short_form_allowed; 668 apc->tx_vp_offset = resp.tx_vport_offset; 669 out: 670 return err; 671 } 672 673 static int mana_cfg_vport_steering(struct mana_port_context *apc, 674 enum TRI_STATE rx, 675 bool update_default_rxobj, bool update_key, 676 bool update_tab) 677 { 678 u16 num_entries = MANA_INDIRECT_TABLE_SIZE; 679 struct mana_cfg_rx_steer_req *req = NULL; 680 struct mana_cfg_rx_steer_resp resp = {}; 681 struct net_device *ndev = apc->ndev; 682 mana_handle_t *req_indir_tab; 683 u32 req_buf_size; 684 int err; 685 686 req_buf_size = sizeof(*req) + sizeof(mana_handle_t) * num_entries; 687 req = kzalloc(req_buf_size, GFP_KERNEL); 688 if (!req) 689 return -ENOMEM; 690 691 mana_gd_init_req_hdr(&req->hdr, MANA_CONFIG_VPORT_RX, req_buf_size, 692 sizeof(resp)); 693 694 req->vport = apc->port_handle; 695 req->num_indir_entries = num_entries; 696 req->indir_tab_offset = sizeof(*req); 697 req->rx_enable = rx; 698 req->rss_enable = apc->rss_state; 699 req->update_default_rxobj = update_default_rxobj; 700 req->update_hashkey = update_key; 701 req->update_indir_tab = update_tab; 702 req->default_rxobj = apc->default_rxobj; 703 704 if (update_key) 705 memcpy(&req->hashkey, apc->hashkey, MANA_HASH_KEY_SIZE); 706 707 if (update_tab) { 708 req_indir_tab = (mana_handle_t *)(req + 1); 709 memcpy(req_indir_tab, apc->rxobj_table, 710 req->num_indir_entries * sizeof(mana_handle_t)); 711 } 712 713 err = mana_send_request(apc->ac, req, req_buf_size, &resp, 714 sizeof(resp)); 715 if (err) { 716 netdev_err(ndev, "Failed to configure vPort RX: %d\n", err); 717 goto out; 718 } 719 720 err = mana_verify_resp_hdr(&resp.hdr, MANA_CONFIG_VPORT_RX, 721 sizeof(resp)); 722 if (err) { 723 netdev_err(ndev, "vPort RX configuration failed: %d\n", err); 724 goto out; 725 } 726 727 if (resp.hdr.status) { 728 netdev_err(ndev, "vPort RX configuration failed: 0x%x\n", 729 resp.hdr.status); 730 err = -EPROTO; 731 } 732 out: 733 kfree(req); 734 return err; 735 } 736 737 static int mana_create_wq_obj(struct mana_port_context *apc, 738 mana_handle_t vport, 739 u32 wq_type, struct mana_obj_spec *wq_spec, 740 struct mana_obj_spec *cq_spec, 741 mana_handle_t *wq_obj) 742 { 743 struct mana_create_wqobj_resp resp = {}; 744 struct mana_create_wqobj_req req = {}; 745 struct net_device *ndev = apc->ndev; 746 int err; 747 748 mana_gd_init_req_hdr(&req.hdr, MANA_CREATE_WQ_OBJ, 749 sizeof(req), sizeof(resp)); 750 req.vport = vport; 751 req.wq_type = wq_type; 752 req.wq_gdma_region = wq_spec->gdma_region; 753 req.cq_gdma_region = cq_spec->gdma_region; 754 req.wq_size = wq_spec->queue_size; 755 req.cq_size = cq_spec->queue_size; 756 req.cq_moderation_ctx_id = cq_spec->modr_ctx_id; 757 req.cq_parent_qid = cq_spec->attached_eq; 758 759 err = mana_send_request(apc->ac, &req, sizeof(req), &resp, 760 sizeof(resp)); 761 if (err) { 762 netdev_err(ndev, "Failed to create WQ object: %d\n", err); 763 goto out; 764 } 765 766 err = mana_verify_resp_hdr(&resp.hdr, MANA_CREATE_WQ_OBJ, 767 sizeof(resp)); 768 if (err || resp.hdr.status) { 769 netdev_err(ndev, "Failed to create WQ object: %d, 0x%x\n", err, 770 resp.hdr.status); 771 if (!err) 772 err = -EPROTO; 773 goto out; 774 } 775 776 if (resp.wq_obj == INVALID_MANA_HANDLE) { 777 netdev_err(ndev, "Got an invalid WQ object handle\n"); 778 err = -EPROTO; 779 goto out; 780 } 781 782 *wq_obj = resp.wq_obj; 783 wq_spec->queue_index = resp.wq_id; 784 cq_spec->queue_index = resp.cq_id; 785 786 return 0; 787 out: 788 return err; 789 } 790 791 static void mana_destroy_wq_obj(struct mana_port_context *apc, u32 wq_type, 792 mana_handle_t wq_obj) 793 { 794 struct mana_destroy_wqobj_resp resp = {}; 795 struct mana_destroy_wqobj_req req = {}; 796 struct net_device *ndev = apc->ndev; 797 int err; 798 799 mana_gd_init_req_hdr(&req.hdr, MANA_DESTROY_WQ_OBJ, 800 sizeof(req), sizeof(resp)); 801 req.wq_type = wq_type; 802 req.wq_obj_handle = wq_obj; 803 804 err = mana_send_request(apc->ac, &req, sizeof(req), &resp, 805 sizeof(resp)); 806 if (err) { 807 netdev_err(ndev, "Failed to destroy WQ object: %d\n", err); 808 return; 809 } 810 811 err = mana_verify_resp_hdr(&resp.hdr, MANA_DESTROY_WQ_OBJ, 812 sizeof(resp)); 813 if (err || resp.hdr.status) 814 netdev_err(ndev, "Failed to destroy WQ object: %d, 0x%x\n", err, 815 resp.hdr.status); 816 } 817 818 static void mana_destroy_eq(struct mana_context *ac) 819 { 820 struct gdma_context *gc = ac->gdma_dev->gdma_context; 821 struct gdma_queue *eq; 822 int i; 823 824 if (!ac->eqs) 825 return; 826 827 for (i = 0; i < gc->max_num_queues; i++) { 828 eq = ac->eqs[i].eq; 829 if (!eq) 830 continue; 831 832 mana_gd_destroy_queue(gc, eq); 833 } 834 835 kfree(ac->eqs); 836 ac->eqs = NULL; 837 } 838 839 static int mana_create_eq(struct mana_context *ac) 840 { 841 struct gdma_dev *gd = ac->gdma_dev; 842 struct gdma_context *gc = gd->gdma_context; 843 struct gdma_queue_spec spec = {}; 844 int err; 845 int i; 846 847 ac->eqs = kcalloc(gc->max_num_queues, sizeof(struct mana_eq), 848 GFP_KERNEL); 849 if (!ac->eqs) 850 return -ENOMEM; 851 852 spec.type = GDMA_EQ; 853 spec.monitor_avl_buf = false; 854 spec.queue_size = EQ_SIZE; 855 spec.eq.callback = NULL; 856 spec.eq.context = ac->eqs; 857 spec.eq.log2_throttle_limit = LOG2_EQ_THROTTLE; 858 859 for (i = 0; i < gc->max_num_queues; i++) { 860 err = mana_gd_create_mana_eq(gd, &spec, &ac->eqs[i].eq); 861 if (err) 862 goto out; 863 } 864 865 return 0; 866 out: 867 mana_destroy_eq(ac); 868 return err; 869 } 870 871 static int mana_fence_rq(struct mana_port_context *apc, struct mana_rxq *rxq) 872 { 873 struct mana_fence_rq_resp resp = {}; 874 struct mana_fence_rq_req req = {}; 875 int err; 876 877 init_completion(&rxq->fence_event); 878 879 mana_gd_init_req_hdr(&req.hdr, MANA_FENCE_RQ, 880 sizeof(req), sizeof(resp)); 881 req.wq_obj_handle = rxq->rxobj; 882 883 err = mana_send_request(apc->ac, &req, sizeof(req), &resp, 884 sizeof(resp)); 885 if (err) { 886 netdev_err(apc->ndev, "Failed to fence RQ %u: %d\n", 887 rxq->rxq_idx, err); 888 return err; 889 } 890 891 err = mana_verify_resp_hdr(&resp.hdr, MANA_FENCE_RQ, sizeof(resp)); 892 if (err || resp.hdr.status) { 893 netdev_err(apc->ndev, "Failed to fence RQ %u: %d, 0x%x\n", 894 rxq->rxq_idx, err, resp.hdr.status); 895 if (!err) 896 err = -EPROTO; 897 898 return err; 899 } 900 901 if (wait_for_completion_timeout(&rxq->fence_event, 10 * HZ) == 0) { 902 netdev_err(apc->ndev, "Failed to fence RQ %u: timed out\n", 903 rxq->rxq_idx); 904 return -ETIMEDOUT; 905 } 906 907 return 0; 908 } 909 910 static void mana_fence_rqs(struct mana_port_context *apc) 911 { 912 unsigned int rxq_idx; 913 struct mana_rxq *rxq; 914 int err; 915 916 for (rxq_idx = 0; rxq_idx < apc->num_queues; rxq_idx++) { 917 rxq = apc->rxqs[rxq_idx]; 918 err = mana_fence_rq(apc, rxq); 919 920 /* In case of any error, use sleep instead. */ 921 if (err) 922 msleep(100); 923 } 924 } 925 926 static int mana_move_wq_tail(struct gdma_queue *wq, u32 num_units) 927 { 928 u32 used_space_old; 929 u32 used_space_new; 930 931 used_space_old = wq->head - wq->tail; 932 used_space_new = wq->head - (wq->tail + num_units); 933 934 if (WARN_ON_ONCE(used_space_new > used_space_old)) 935 return -ERANGE; 936 937 wq->tail += num_units; 938 return 0; 939 } 940 941 static void mana_unmap_skb(struct sk_buff *skb, struct mana_port_context *apc) 942 { 943 struct mana_skb_head *ash = (struct mana_skb_head *)skb->head; 944 struct gdma_context *gc = apc->ac->gdma_dev->gdma_context; 945 struct device *dev = gc->dev; 946 int i; 947 948 dma_unmap_single(dev, ash->dma_handle[0], ash->size[0], DMA_TO_DEVICE); 949 950 for (i = 1; i < skb_shinfo(skb)->nr_frags + 1; i++) 951 dma_unmap_page(dev, ash->dma_handle[i], ash->size[i], 952 DMA_TO_DEVICE); 953 } 954 955 static void mana_poll_tx_cq(struct mana_cq *cq) 956 { 957 struct gdma_comp *completions = cq->gdma_comp_buf; 958 struct gdma_posted_wqe_info *wqe_info; 959 unsigned int pkt_transmitted = 0; 960 unsigned int wqe_unit_cnt = 0; 961 struct mana_txq *txq = cq->txq; 962 struct mana_port_context *apc; 963 struct netdev_queue *net_txq; 964 struct gdma_queue *gdma_wq; 965 unsigned int avail_space; 966 struct net_device *ndev; 967 struct sk_buff *skb; 968 bool txq_stopped; 969 int comp_read; 970 int i; 971 972 ndev = txq->ndev; 973 apc = netdev_priv(ndev); 974 975 comp_read = mana_gd_poll_cq(cq->gdma_cq, completions, 976 CQE_POLLING_BUFFER); 977 978 if (comp_read < 1) 979 return; 980 981 for (i = 0; i < comp_read; i++) { 982 struct mana_tx_comp_oob *cqe_oob; 983 984 if (WARN_ON_ONCE(!completions[i].is_sq)) 985 return; 986 987 cqe_oob = (struct mana_tx_comp_oob *)completions[i].cqe_data; 988 if (WARN_ON_ONCE(cqe_oob->cqe_hdr.client_type != 989 MANA_CQE_COMPLETION)) 990 return; 991 992 switch (cqe_oob->cqe_hdr.cqe_type) { 993 case CQE_TX_OKAY: 994 break; 995 996 case CQE_TX_SA_DROP: 997 case CQE_TX_MTU_DROP: 998 case CQE_TX_INVALID_OOB: 999 case CQE_TX_INVALID_ETH_TYPE: 1000 case CQE_TX_HDR_PROCESSING_ERROR: 1001 case CQE_TX_VF_DISABLED: 1002 case CQE_TX_VPORT_IDX_OUT_OF_RANGE: 1003 case CQE_TX_VPORT_DISABLED: 1004 case CQE_TX_VLAN_TAGGING_VIOLATION: 1005 WARN_ONCE(1, "TX: CQE error %d: ignored.\n", 1006 cqe_oob->cqe_hdr.cqe_type); 1007 break; 1008 1009 default: 1010 /* If the CQE type is unexpected, log an error, assert, 1011 * and go through the error path. 1012 */ 1013 WARN_ONCE(1, "TX: Unexpected CQE type %d: HW BUG?\n", 1014 cqe_oob->cqe_hdr.cqe_type); 1015 return; 1016 } 1017 1018 if (WARN_ON_ONCE(txq->gdma_txq_id != completions[i].wq_num)) 1019 return; 1020 1021 skb = skb_dequeue(&txq->pending_skbs); 1022 if (WARN_ON_ONCE(!skb)) 1023 return; 1024 1025 wqe_info = (struct gdma_posted_wqe_info *)skb->cb; 1026 wqe_unit_cnt += wqe_info->wqe_size_in_bu; 1027 1028 mana_unmap_skb(skb, apc); 1029 1030 napi_consume_skb(skb, cq->budget); 1031 1032 pkt_transmitted++; 1033 } 1034 1035 if (WARN_ON_ONCE(wqe_unit_cnt == 0)) 1036 return; 1037 1038 mana_move_wq_tail(txq->gdma_sq, wqe_unit_cnt); 1039 1040 gdma_wq = txq->gdma_sq; 1041 avail_space = mana_gd_wq_avail_space(gdma_wq); 1042 1043 /* Ensure tail updated before checking q stop */ 1044 smp_mb(); 1045 1046 net_txq = txq->net_txq; 1047 txq_stopped = netif_tx_queue_stopped(net_txq); 1048 1049 /* Ensure checking txq_stopped before apc->port_is_up. */ 1050 smp_rmb(); 1051 1052 if (txq_stopped && apc->port_is_up && avail_space >= MAX_TX_WQE_SIZE) { 1053 netif_tx_wake_queue(net_txq); 1054 apc->eth_stats.wake_queue++; 1055 } 1056 1057 if (atomic_sub_return(pkt_transmitted, &txq->pending_sends) < 0) 1058 WARN_ON_ONCE(1); 1059 1060 cq->work_done = pkt_transmitted; 1061 } 1062 1063 static void mana_post_pkt_rxq(struct mana_rxq *rxq) 1064 { 1065 struct mana_recv_buf_oob *recv_buf_oob; 1066 u32 curr_index; 1067 int err; 1068 1069 curr_index = rxq->buf_index++; 1070 if (rxq->buf_index == rxq->num_rx_buf) 1071 rxq->buf_index = 0; 1072 1073 recv_buf_oob = &rxq->rx_oobs[curr_index]; 1074 1075 err = mana_gd_post_and_ring(rxq->gdma_rq, &recv_buf_oob->wqe_req, 1076 &recv_buf_oob->wqe_inf); 1077 if (WARN_ON_ONCE(err)) 1078 return; 1079 1080 WARN_ON_ONCE(recv_buf_oob->wqe_inf.wqe_size_in_bu != 1); 1081 } 1082 1083 static struct sk_buff *mana_build_skb(void *buf_va, uint pkt_len, 1084 struct xdp_buff *xdp) 1085 { 1086 struct sk_buff *skb = build_skb(buf_va, PAGE_SIZE); 1087 1088 if (!skb) 1089 return NULL; 1090 1091 if (xdp->data_hard_start) { 1092 skb_reserve(skb, xdp->data - xdp->data_hard_start); 1093 skb_put(skb, xdp->data_end - xdp->data); 1094 } else { 1095 skb_reserve(skb, XDP_PACKET_HEADROOM); 1096 skb_put(skb, pkt_len); 1097 } 1098 1099 return skb; 1100 } 1101 1102 static void mana_rx_skb(void *buf_va, struct mana_rxcomp_oob *cqe, 1103 struct mana_rxq *rxq) 1104 { 1105 struct mana_stats_rx *rx_stats = &rxq->stats; 1106 struct net_device *ndev = rxq->ndev; 1107 uint pkt_len = cqe->ppi[0].pkt_len; 1108 u16 rxq_idx = rxq->rxq_idx; 1109 struct napi_struct *napi; 1110 struct xdp_buff xdp = {}; 1111 struct sk_buff *skb; 1112 u32 hash_value; 1113 u32 act; 1114 1115 rxq->rx_cq.work_done++; 1116 napi = &rxq->rx_cq.napi; 1117 1118 if (!buf_va) { 1119 ++ndev->stats.rx_dropped; 1120 return; 1121 } 1122 1123 act = mana_run_xdp(ndev, rxq, &xdp, buf_va, pkt_len); 1124 1125 if (act == XDP_REDIRECT && !rxq->xdp_rc) 1126 return; 1127 1128 if (act != XDP_PASS && act != XDP_TX) 1129 goto drop_xdp; 1130 1131 skb = mana_build_skb(buf_va, pkt_len, &xdp); 1132 1133 if (!skb) 1134 goto drop; 1135 1136 skb->dev = napi->dev; 1137 1138 skb->protocol = eth_type_trans(skb, ndev); 1139 skb_checksum_none_assert(skb); 1140 skb_record_rx_queue(skb, rxq_idx); 1141 1142 if ((ndev->features & NETIF_F_RXCSUM) && cqe->rx_iphdr_csum_succeed) { 1143 if (cqe->rx_tcp_csum_succeed || cqe->rx_udp_csum_succeed) 1144 skb->ip_summed = CHECKSUM_UNNECESSARY; 1145 } 1146 1147 if (cqe->rx_hashtype != 0 && (ndev->features & NETIF_F_RXHASH)) { 1148 hash_value = cqe->ppi[0].pkt_hash; 1149 1150 if (cqe->rx_hashtype & MANA_HASH_L4) 1151 skb_set_hash(skb, hash_value, PKT_HASH_TYPE_L4); 1152 else 1153 skb_set_hash(skb, hash_value, PKT_HASH_TYPE_L3); 1154 } 1155 1156 u64_stats_update_begin(&rx_stats->syncp); 1157 rx_stats->packets++; 1158 rx_stats->bytes += pkt_len; 1159 1160 if (act == XDP_TX) 1161 rx_stats->xdp_tx++; 1162 u64_stats_update_end(&rx_stats->syncp); 1163 1164 if (act == XDP_TX) { 1165 skb_set_queue_mapping(skb, rxq_idx); 1166 mana_xdp_tx(skb, ndev); 1167 return; 1168 } 1169 1170 napi_gro_receive(napi, skb); 1171 1172 return; 1173 1174 drop_xdp: 1175 u64_stats_update_begin(&rx_stats->syncp); 1176 rx_stats->xdp_drop++; 1177 u64_stats_update_end(&rx_stats->syncp); 1178 1179 drop: 1180 WARN_ON_ONCE(rxq->xdp_save_page); 1181 rxq->xdp_save_page = virt_to_page(buf_va); 1182 1183 ++ndev->stats.rx_dropped; 1184 1185 return; 1186 } 1187 1188 static void mana_process_rx_cqe(struct mana_rxq *rxq, struct mana_cq *cq, 1189 struct gdma_comp *cqe) 1190 { 1191 struct mana_rxcomp_oob *oob = (struct mana_rxcomp_oob *)cqe->cqe_data; 1192 struct gdma_context *gc = rxq->gdma_rq->gdma_dev->gdma_context; 1193 struct net_device *ndev = rxq->ndev; 1194 struct mana_recv_buf_oob *rxbuf_oob; 1195 struct device *dev = gc->dev; 1196 void *new_buf, *old_buf; 1197 struct page *new_page; 1198 u32 curr, pktlen; 1199 dma_addr_t da; 1200 1201 switch (oob->cqe_hdr.cqe_type) { 1202 case CQE_RX_OKAY: 1203 break; 1204 1205 case CQE_RX_TRUNCATED: 1206 ++ndev->stats.rx_dropped; 1207 rxbuf_oob = &rxq->rx_oobs[rxq->buf_index]; 1208 netdev_warn_once(ndev, "Dropped a truncated packet\n"); 1209 goto drop; 1210 1211 case CQE_RX_COALESCED_4: 1212 netdev_err(ndev, "RX coalescing is unsupported\n"); 1213 return; 1214 1215 case CQE_RX_OBJECT_FENCE: 1216 complete(&rxq->fence_event); 1217 return; 1218 1219 default: 1220 netdev_err(ndev, "Unknown RX CQE type = %d\n", 1221 oob->cqe_hdr.cqe_type); 1222 return; 1223 } 1224 1225 pktlen = oob->ppi[0].pkt_len; 1226 1227 if (pktlen == 0) { 1228 /* data packets should never have packetlength of zero */ 1229 netdev_err(ndev, "RX pkt len=0, rq=%u, cq=%u, rxobj=0x%llx\n", 1230 rxq->gdma_id, cq->gdma_id, rxq->rxobj); 1231 return; 1232 } 1233 1234 curr = rxq->buf_index; 1235 rxbuf_oob = &rxq->rx_oobs[curr]; 1236 WARN_ON_ONCE(rxbuf_oob->wqe_inf.wqe_size_in_bu != 1); 1237 1238 /* Reuse XDP dropped page if available */ 1239 if (rxq->xdp_save_page) { 1240 new_page = rxq->xdp_save_page; 1241 rxq->xdp_save_page = NULL; 1242 } else { 1243 new_page = alloc_page(GFP_ATOMIC); 1244 } 1245 1246 if (new_page) { 1247 da = dma_map_page(dev, new_page, XDP_PACKET_HEADROOM, rxq->datasize, 1248 DMA_FROM_DEVICE); 1249 1250 if (dma_mapping_error(dev, da)) { 1251 __free_page(new_page); 1252 new_page = NULL; 1253 } 1254 } 1255 1256 new_buf = new_page ? page_to_virt(new_page) : NULL; 1257 1258 if (new_buf) { 1259 dma_unmap_page(dev, rxbuf_oob->buf_dma_addr, rxq->datasize, 1260 DMA_FROM_DEVICE); 1261 1262 old_buf = rxbuf_oob->buf_va; 1263 1264 /* refresh the rxbuf_oob with the new page */ 1265 rxbuf_oob->buf_va = new_buf; 1266 rxbuf_oob->buf_dma_addr = da; 1267 rxbuf_oob->sgl[0].address = rxbuf_oob->buf_dma_addr; 1268 } else { 1269 old_buf = NULL; /* drop the packet if no memory */ 1270 } 1271 1272 mana_rx_skb(old_buf, oob, rxq); 1273 1274 drop: 1275 mana_move_wq_tail(rxq->gdma_rq, rxbuf_oob->wqe_inf.wqe_size_in_bu); 1276 1277 mana_post_pkt_rxq(rxq); 1278 } 1279 1280 static void mana_poll_rx_cq(struct mana_cq *cq) 1281 { 1282 struct gdma_comp *comp = cq->gdma_comp_buf; 1283 struct mana_rxq *rxq = cq->rxq; 1284 int comp_read, i; 1285 1286 comp_read = mana_gd_poll_cq(cq->gdma_cq, comp, CQE_POLLING_BUFFER); 1287 WARN_ON_ONCE(comp_read > CQE_POLLING_BUFFER); 1288 1289 rxq->xdp_flush = false; 1290 1291 for (i = 0; i < comp_read; i++) { 1292 if (WARN_ON_ONCE(comp[i].is_sq)) 1293 return; 1294 1295 /* verify recv cqe references the right rxq */ 1296 if (WARN_ON_ONCE(comp[i].wq_num != cq->rxq->gdma_id)) 1297 return; 1298 1299 mana_process_rx_cqe(rxq, cq, &comp[i]); 1300 } 1301 1302 if (rxq->xdp_flush) 1303 xdp_do_flush(); 1304 } 1305 1306 static void mana_cq_handler(void *context, struct gdma_queue *gdma_queue) 1307 { 1308 struct mana_cq *cq = context; 1309 u8 arm_bit; 1310 1311 WARN_ON_ONCE(cq->gdma_cq != gdma_queue); 1312 1313 if (cq->type == MANA_CQ_TYPE_RX) 1314 mana_poll_rx_cq(cq); 1315 else 1316 mana_poll_tx_cq(cq); 1317 1318 if (cq->work_done < cq->budget && 1319 napi_complete_done(&cq->napi, cq->work_done)) { 1320 arm_bit = SET_ARM_BIT; 1321 } else { 1322 arm_bit = 0; 1323 } 1324 1325 mana_gd_ring_cq(gdma_queue, arm_bit); 1326 } 1327 1328 static int mana_poll(struct napi_struct *napi, int budget) 1329 { 1330 struct mana_cq *cq = container_of(napi, struct mana_cq, napi); 1331 1332 cq->work_done = 0; 1333 cq->budget = budget; 1334 1335 mana_cq_handler(cq, cq->gdma_cq); 1336 1337 return min(cq->work_done, budget); 1338 } 1339 1340 static void mana_schedule_napi(void *context, struct gdma_queue *gdma_queue) 1341 { 1342 struct mana_cq *cq = context; 1343 1344 napi_schedule_irqoff(&cq->napi); 1345 } 1346 1347 static void mana_deinit_cq(struct mana_port_context *apc, struct mana_cq *cq) 1348 { 1349 struct gdma_dev *gd = apc->ac->gdma_dev; 1350 1351 if (!cq->gdma_cq) 1352 return; 1353 1354 mana_gd_destroy_queue(gd->gdma_context, cq->gdma_cq); 1355 } 1356 1357 static void mana_deinit_txq(struct mana_port_context *apc, struct mana_txq *txq) 1358 { 1359 struct gdma_dev *gd = apc->ac->gdma_dev; 1360 1361 if (!txq->gdma_sq) 1362 return; 1363 1364 mana_gd_destroy_queue(gd->gdma_context, txq->gdma_sq); 1365 } 1366 1367 static void mana_destroy_txq(struct mana_port_context *apc) 1368 { 1369 struct napi_struct *napi; 1370 int i; 1371 1372 if (!apc->tx_qp) 1373 return; 1374 1375 for (i = 0; i < apc->num_queues; i++) { 1376 napi = &apc->tx_qp[i].tx_cq.napi; 1377 napi_synchronize(napi); 1378 napi_disable(napi); 1379 netif_napi_del(napi); 1380 1381 mana_destroy_wq_obj(apc, GDMA_SQ, apc->tx_qp[i].tx_object); 1382 1383 mana_deinit_cq(apc, &apc->tx_qp[i].tx_cq); 1384 1385 mana_deinit_txq(apc, &apc->tx_qp[i].txq); 1386 } 1387 1388 kfree(apc->tx_qp); 1389 apc->tx_qp = NULL; 1390 } 1391 1392 static int mana_create_txq(struct mana_port_context *apc, 1393 struct net_device *net) 1394 { 1395 struct mana_context *ac = apc->ac; 1396 struct gdma_dev *gd = ac->gdma_dev; 1397 struct mana_obj_spec wq_spec; 1398 struct mana_obj_spec cq_spec; 1399 struct gdma_queue_spec spec; 1400 struct gdma_context *gc; 1401 struct mana_txq *txq; 1402 struct mana_cq *cq; 1403 u32 txq_size; 1404 u32 cq_size; 1405 int err; 1406 int i; 1407 1408 apc->tx_qp = kcalloc(apc->num_queues, sizeof(struct mana_tx_qp), 1409 GFP_KERNEL); 1410 if (!apc->tx_qp) 1411 return -ENOMEM; 1412 1413 /* The minimum size of the WQE is 32 bytes, hence 1414 * MAX_SEND_BUFFERS_PER_QUEUE represents the maximum number of WQEs 1415 * the SQ can store. This value is then used to size other queues 1416 * to prevent overflow. 1417 */ 1418 txq_size = MAX_SEND_BUFFERS_PER_QUEUE * 32; 1419 BUILD_BUG_ON(!PAGE_ALIGNED(txq_size)); 1420 1421 cq_size = MAX_SEND_BUFFERS_PER_QUEUE * COMP_ENTRY_SIZE; 1422 cq_size = PAGE_ALIGN(cq_size); 1423 1424 gc = gd->gdma_context; 1425 1426 for (i = 0; i < apc->num_queues; i++) { 1427 apc->tx_qp[i].tx_object = INVALID_MANA_HANDLE; 1428 1429 /* Create SQ */ 1430 txq = &apc->tx_qp[i].txq; 1431 1432 u64_stats_init(&txq->stats.syncp); 1433 txq->ndev = net; 1434 txq->net_txq = netdev_get_tx_queue(net, i); 1435 txq->vp_offset = apc->tx_vp_offset; 1436 skb_queue_head_init(&txq->pending_skbs); 1437 1438 memset(&spec, 0, sizeof(spec)); 1439 spec.type = GDMA_SQ; 1440 spec.monitor_avl_buf = true; 1441 spec.queue_size = txq_size; 1442 err = mana_gd_create_mana_wq_cq(gd, &spec, &txq->gdma_sq); 1443 if (err) 1444 goto out; 1445 1446 /* Create SQ's CQ */ 1447 cq = &apc->tx_qp[i].tx_cq; 1448 cq->type = MANA_CQ_TYPE_TX; 1449 1450 cq->txq = txq; 1451 1452 memset(&spec, 0, sizeof(spec)); 1453 spec.type = GDMA_CQ; 1454 spec.monitor_avl_buf = false; 1455 spec.queue_size = cq_size; 1456 spec.cq.callback = mana_schedule_napi; 1457 spec.cq.parent_eq = ac->eqs[i].eq; 1458 spec.cq.context = cq; 1459 err = mana_gd_create_mana_wq_cq(gd, &spec, &cq->gdma_cq); 1460 if (err) 1461 goto out; 1462 1463 memset(&wq_spec, 0, sizeof(wq_spec)); 1464 memset(&cq_spec, 0, sizeof(cq_spec)); 1465 1466 wq_spec.gdma_region = txq->gdma_sq->mem_info.gdma_region; 1467 wq_spec.queue_size = txq->gdma_sq->queue_size; 1468 1469 cq_spec.gdma_region = cq->gdma_cq->mem_info.gdma_region; 1470 cq_spec.queue_size = cq->gdma_cq->queue_size; 1471 cq_spec.modr_ctx_id = 0; 1472 cq_spec.attached_eq = cq->gdma_cq->cq.parent->id; 1473 1474 err = mana_create_wq_obj(apc, apc->port_handle, GDMA_SQ, 1475 &wq_spec, &cq_spec, 1476 &apc->tx_qp[i].tx_object); 1477 1478 if (err) 1479 goto out; 1480 1481 txq->gdma_sq->id = wq_spec.queue_index; 1482 cq->gdma_cq->id = cq_spec.queue_index; 1483 1484 txq->gdma_sq->mem_info.gdma_region = GDMA_INVALID_DMA_REGION; 1485 cq->gdma_cq->mem_info.gdma_region = GDMA_INVALID_DMA_REGION; 1486 1487 txq->gdma_txq_id = txq->gdma_sq->id; 1488 1489 cq->gdma_id = cq->gdma_cq->id; 1490 1491 if (WARN_ON(cq->gdma_id >= gc->max_num_cqs)) { 1492 err = -EINVAL; 1493 goto out; 1494 } 1495 1496 gc->cq_table[cq->gdma_id] = cq->gdma_cq; 1497 1498 netif_napi_add_tx(net, &cq->napi, mana_poll); 1499 napi_enable(&cq->napi); 1500 1501 mana_gd_ring_cq(cq->gdma_cq, SET_ARM_BIT); 1502 } 1503 1504 return 0; 1505 out: 1506 mana_destroy_txq(apc); 1507 return err; 1508 } 1509 1510 static void mana_destroy_rxq(struct mana_port_context *apc, 1511 struct mana_rxq *rxq, bool validate_state) 1512 1513 { 1514 struct gdma_context *gc = apc->ac->gdma_dev->gdma_context; 1515 struct mana_recv_buf_oob *rx_oob; 1516 struct device *dev = gc->dev; 1517 struct napi_struct *napi; 1518 int i; 1519 1520 if (!rxq) 1521 return; 1522 1523 napi = &rxq->rx_cq.napi; 1524 1525 if (validate_state) 1526 napi_synchronize(napi); 1527 1528 napi_disable(napi); 1529 1530 xdp_rxq_info_unreg(&rxq->xdp_rxq); 1531 1532 netif_napi_del(napi); 1533 1534 mana_destroy_wq_obj(apc, GDMA_RQ, rxq->rxobj); 1535 1536 mana_deinit_cq(apc, &rxq->rx_cq); 1537 1538 if (rxq->xdp_save_page) 1539 __free_page(rxq->xdp_save_page); 1540 1541 for (i = 0; i < rxq->num_rx_buf; i++) { 1542 rx_oob = &rxq->rx_oobs[i]; 1543 1544 if (!rx_oob->buf_va) 1545 continue; 1546 1547 dma_unmap_page(dev, rx_oob->buf_dma_addr, rxq->datasize, 1548 DMA_FROM_DEVICE); 1549 1550 free_page((unsigned long)rx_oob->buf_va); 1551 rx_oob->buf_va = NULL; 1552 } 1553 1554 if (rxq->gdma_rq) 1555 mana_gd_destroy_queue(gc, rxq->gdma_rq); 1556 1557 kfree(rxq); 1558 } 1559 1560 #define MANA_WQE_HEADER_SIZE 16 1561 #define MANA_WQE_SGE_SIZE 16 1562 1563 static int mana_alloc_rx_wqe(struct mana_port_context *apc, 1564 struct mana_rxq *rxq, u32 *rxq_size, u32 *cq_size) 1565 { 1566 struct gdma_context *gc = apc->ac->gdma_dev->gdma_context; 1567 struct mana_recv_buf_oob *rx_oob; 1568 struct device *dev = gc->dev; 1569 struct page *page; 1570 dma_addr_t da; 1571 u32 buf_idx; 1572 1573 WARN_ON(rxq->datasize == 0 || rxq->datasize > PAGE_SIZE); 1574 1575 *rxq_size = 0; 1576 *cq_size = 0; 1577 1578 for (buf_idx = 0; buf_idx < rxq->num_rx_buf; buf_idx++) { 1579 rx_oob = &rxq->rx_oobs[buf_idx]; 1580 memset(rx_oob, 0, sizeof(*rx_oob)); 1581 1582 page = alloc_page(GFP_KERNEL); 1583 if (!page) 1584 return -ENOMEM; 1585 1586 da = dma_map_page(dev, page, XDP_PACKET_HEADROOM, rxq->datasize, 1587 DMA_FROM_DEVICE); 1588 1589 if (dma_mapping_error(dev, da)) { 1590 __free_page(page); 1591 return -ENOMEM; 1592 } 1593 1594 rx_oob->buf_va = page_to_virt(page); 1595 rx_oob->buf_dma_addr = da; 1596 1597 rx_oob->num_sge = 1; 1598 rx_oob->sgl[0].address = rx_oob->buf_dma_addr; 1599 rx_oob->sgl[0].size = rxq->datasize; 1600 rx_oob->sgl[0].mem_key = apc->ac->gdma_dev->gpa_mkey; 1601 1602 rx_oob->wqe_req.sgl = rx_oob->sgl; 1603 rx_oob->wqe_req.num_sge = rx_oob->num_sge; 1604 rx_oob->wqe_req.inline_oob_size = 0; 1605 rx_oob->wqe_req.inline_oob_data = NULL; 1606 rx_oob->wqe_req.flags = 0; 1607 rx_oob->wqe_req.client_data_unit = 0; 1608 1609 *rxq_size += ALIGN(MANA_WQE_HEADER_SIZE + 1610 MANA_WQE_SGE_SIZE * rx_oob->num_sge, 32); 1611 *cq_size += COMP_ENTRY_SIZE; 1612 } 1613 1614 return 0; 1615 } 1616 1617 static int mana_push_wqe(struct mana_rxq *rxq) 1618 { 1619 struct mana_recv_buf_oob *rx_oob; 1620 u32 buf_idx; 1621 int err; 1622 1623 for (buf_idx = 0; buf_idx < rxq->num_rx_buf; buf_idx++) { 1624 rx_oob = &rxq->rx_oobs[buf_idx]; 1625 1626 err = mana_gd_post_and_ring(rxq->gdma_rq, &rx_oob->wqe_req, 1627 &rx_oob->wqe_inf); 1628 if (err) 1629 return -ENOSPC; 1630 } 1631 1632 return 0; 1633 } 1634 1635 static struct mana_rxq *mana_create_rxq(struct mana_port_context *apc, 1636 u32 rxq_idx, struct mana_eq *eq, 1637 struct net_device *ndev) 1638 { 1639 struct gdma_dev *gd = apc->ac->gdma_dev; 1640 struct mana_obj_spec wq_spec; 1641 struct mana_obj_spec cq_spec; 1642 struct gdma_queue_spec spec; 1643 struct mana_cq *cq = NULL; 1644 struct gdma_context *gc; 1645 u32 cq_size, rq_size; 1646 struct mana_rxq *rxq; 1647 int err; 1648 1649 gc = gd->gdma_context; 1650 1651 rxq = kzalloc(struct_size(rxq, rx_oobs, RX_BUFFERS_PER_QUEUE), 1652 GFP_KERNEL); 1653 if (!rxq) 1654 return NULL; 1655 1656 rxq->ndev = ndev; 1657 rxq->num_rx_buf = RX_BUFFERS_PER_QUEUE; 1658 rxq->rxq_idx = rxq_idx; 1659 rxq->datasize = ALIGN(MAX_FRAME_SIZE, 64); 1660 rxq->rxobj = INVALID_MANA_HANDLE; 1661 1662 err = mana_alloc_rx_wqe(apc, rxq, &rq_size, &cq_size); 1663 if (err) 1664 goto out; 1665 1666 rq_size = PAGE_ALIGN(rq_size); 1667 cq_size = PAGE_ALIGN(cq_size); 1668 1669 /* Create RQ */ 1670 memset(&spec, 0, sizeof(spec)); 1671 spec.type = GDMA_RQ; 1672 spec.monitor_avl_buf = true; 1673 spec.queue_size = rq_size; 1674 err = mana_gd_create_mana_wq_cq(gd, &spec, &rxq->gdma_rq); 1675 if (err) 1676 goto out; 1677 1678 /* Create RQ's CQ */ 1679 cq = &rxq->rx_cq; 1680 cq->type = MANA_CQ_TYPE_RX; 1681 cq->rxq = rxq; 1682 1683 memset(&spec, 0, sizeof(spec)); 1684 spec.type = GDMA_CQ; 1685 spec.monitor_avl_buf = false; 1686 spec.queue_size = cq_size; 1687 spec.cq.callback = mana_schedule_napi; 1688 spec.cq.parent_eq = eq->eq; 1689 spec.cq.context = cq; 1690 err = mana_gd_create_mana_wq_cq(gd, &spec, &cq->gdma_cq); 1691 if (err) 1692 goto out; 1693 1694 memset(&wq_spec, 0, sizeof(wq_spec)); 1695 memset(&cq_spec, 0, sizeof(cq_spec)); 1696 wq_spec.gdma_region = rxq->gdma_rq->mem_info.gdma_region; 1697 wq_spec.queue_size = rxq->gdma_rq->queue_size; 1698 1699 cq_spec.gdma_region = cq->gdma_cq->mem_info.gdma_region; 1700 cq_spec.queue_size = cq->gdma_cq->queue_size; 1701 cq_spec.modr_ctx_id = 0; 1702 cq_spec.attached_eq = cq->gdma_cq->cq.parent->id; 1703 1704 err = mana_create_wq_obj(apc, apc->port_handle, GDMA_RQ, 1705 &wq_spec, &cq_spec, &rxq->rxobj); 1706 if (err) 1707 goto out; 1708 1709 rxq->gdma_rq->id = wq_spec.queue_index; 1710 cq->gdma_cq->id = cq_spec.queue_index; 1711 1712 rxq->gdma_rq->mem_info.gdma_region = GDMA_INVALID_DMA_REGION; 1713 cq->gdma_cq->mem_info.gdma_region = GDMA_INVALID_DMA_REGION; 1714 1715 rxq->gdma_id = rxq->gdma_rq->id; 1716 cq->gdma_id = cq->gdma_cq->id; 1717 1718 err = mana_push_wqe(rxq); 1719 if (err) 1720 goto out; 1721 1722 if (WARN_ON(cq->gdma_id >= gc->max_num_cqs)) { 1723 err = -EINVAL; 1724 goto out; 1725 } 1726 1727 gc->cq_table[cq->gdma_id] = cq->gdma_cq; 1728 1729 netif_napi_add_weight(ndev, &cq->napi, mana_poll, 1); 1730 1731 WARN_ON(xdp_rxq_info_reg(&rxq->xdp_rxq, ndev, rxq_idx, 1732 cq->napi.napi_id)); 1733 WARN_ON(xdp_rxq_info_reg_mem_model(&rxq->xdp_rxq, 1734 MEM_TYPE_PAGE_SHARED, NULL)); 1735 1736 napi_enable(&cq->napi); 1737 1738 mana_gd_ring_cq(cq->gdma_cq, SET_ARM_BIT); 1739 out: 1740 if (!err) 1741 return rxq; 1742 1743 netdev_err(ndev, "Failed to create RXQ: err = %d\n", err); 1744 1745 mana_destroy_rxq(apc, rxq, false); 1746 1747 if (cq) 1748 mana_deinit_cq(apc, cq); 1749 1750 return NULL; 1751 } 1752 1753 static int mana_add_rx_queues(struct mana_port_context *apc, 1754 struct net_device *ndev) 1755 { 1756 struct mana_context *ac = apc->ac; 1757 struct mana_rxq *rxq; 1758 int err = 0; 1759 int i; 1760 1761 for (i = 0; i < apc->num_queues; i++) { 1762 rxq = mana_create_rxq(apc, i, &ac->eqs[i], ndev); 1763 if (!rxq) { 1764 err = -ENOMEM; 1765 goto out; 1766 } 1767 1768 u64_stats_init(&rxq->stats.syncp); 1769 1770 apc->rxqs[i] = rxq; 1771 } 1772 1773 apc->default_rxobj = apc->rxqs[0]->rxobj; 1774 out: 1775 return err; 1776 } 1777 1778 static void mana_destroy_vport(struct mana_port_context *apc) 1779 { 1780 struct gdma_dev *gd = apc->ac->gdma_dev; 1781 struct mana_rxq *rxq; 1782 u32 rxq_idx; 1783 1784 for (rxq_idx = 0; rxq_idx < apc->num_queues; rxq_idx++) { 1785 rxq = apc->rxqs[rxq_idx]; 1786 if (!rxq) 1787 continue; 1788 1789 mana_destroy_rxq(apc, rxq, true); 1790 apc->rxqs[rxq_idx] = NULL; 1791 } 1792 1793 mana_destroy_txq(apc); 1794 1795 if (gd->gdma_context->is_pf) 1796 mana_pf_deregister_hw_vport(apc); 1797 } 1798 1799 static int mana_create_vport(struct mana_port_context *apc, 1800 struct net_device *net) 1801 { 1802 struct gdma_dev *gd = apc->ac->gdma_dev; 1803 int err; 1804 1805 apc->default_rxobj = INVALID_MANA_HANDLE; 1806 1807 if (gd->gdma_context->is_pf) { 1808 err = mana_pf_register_hw_vport(apc); 1809 if (err) 1810 return err; 1811 } 1812 1813 err = mana_cfg_vport(apc, gd->pdid, gd->doorbell); 1814 if (err) 1815 return err; 1816 1817 return mana_create_txq(apc, net); 1818 } 1819 1820 static void mana_rss_table_init(struct mana_port_context *apc) 1821 { 1822 int i; 1823 1824 for (i = 0; i < MANA_INDIRECT_TABLE_SIZE; i++) 1825 apc->indir_table[i] = 1826 ethtool_rxfh_indir_default(i, apc->num_queues); 1827 } 1828 1829 int mana_config_rss(struct mana_port_context *apc, enum TRI_STATE rx, 1830 bool update_hash, bool update_tab) 1831 { 1832 u32 queue_idx; 1833 int err; 1834 int i; 1835 1836 if (update_tab) { 1837 for (i = 0; i < MANA_INDIRECT_TABLE_SIZE; i++) { 1838 queue_idx = apc->indir_table[i]; 1839 apc->rxobj_table[i] = apc->rxqs[queue_idx]->rxobj; 1840 } 1841 } 1842 1843 err = mana_cfg_vport_steering(apc, rx, true, update_hash, update_tab); 1844 if (err) 1845 return err; 1846 1847 mana_fence_rqs(apc); 1848 1849 return 0; 1850 } 1851 1852 static int mana_init_port(struct net_device *ndev) 1853 { 1854 struct mana_port_context *apc = netdev_priv(ndev); 1855 u32 max_txq, max_rxq, max_queues; 1856 int port_idx = apc->port_idx; 1857 u32 num_indirect_entries; 1858 int err; 1859 1860 err = mana_init_port_context(apc); 1861 if (err) 1862 return err; 1863 1864 err = mana_query_vport_cfg(apc, port_idx, &max_txq, &max_rxq, 1865 &num_indirect_entries); 1866 if (err) { 1867 netdev_err(ndev, "Failed to query info for vPort %d\n", 1868 port_idx); 1869 goto reset_apc; 1870 } 1871 1872 max_queues = min_t(u32, max_txq, max_rxq); 1873 if (apc->max_queues > max_queues) 1874 apc->max_queues = max_queues; 1875 1876 if (apc->num_queues > apc->max_queues) 1877 apc->num_queues = apc->max_queues; 1878 1879 eth_hw_addr_set(ndev, apc->mac_addr); 1880 1881 return 0; 1882 1883 reset_apc: 1884 kfree(apc->rxqs); 1885 apc->rxqs = NULL; 1886 return err; 1887 } 1888 1889 int mana_alloc_queues(struct net_device *ndev) 1890 { 1891 struct mana_port_context *apc = netdev_priv(ndev); 1892 struct gdma_dev *gd = apc->ac->gdma_dev; 1893 int err; 1894 1895 err = mana_create_vport(apc, ndev); 1896 if (err) 1897 return err; 1898 1899 err = netif_set_real_num_tx_queues(ndev, apc->num_queues); 1900 if (err) 1901 goto destroy_vport; 1902 1903 err = mana_add_rx_queues(apc, ndev); 1904 if (err) 1905 goto destroy_vport; 1906 1907 apc->rss_state = apc->num_queues > 1 ? TRI_STATE_TRUE : TRI_STATE_FALSE; 1908 1909 err = netif_set_real_num_rx_queues(ndev, apc->num_queues); 1910 if (err) 1911 goto destroy_vport; 1912 1913 mana_rss_table_init(apc); 1914 1915 err = mana_config_rss(apc, TRI_STATE_TRUE, true, true); 1916 if (err) 1917 goto destroy_vport; 1918 1919 if (gd->gdma_context->is_pf) { 1920 err = mana_pf_register_filter(apc); 1921 if (err) 1922 goto destroy_vport; 1923 } 1924 1925 mana_chn_setxdp(apc, mana_xdp_get(apc)); 1926 1927 return 0; 1928 1929 destroy_vport: 1930 mana_destroy_vport(apc); 1931 return err; 1932 } 1933 1934 int mana_attach(struct net_device *ndev) 1935 { 1936 struct mana_port_context *apc = netdev_priv(ndev); 1937 int err; 1938 1939 ASSERT_RTNL(); 1940 1941 err = mana_init_port(ndev); 1942 if (err) 1943 return err; 1944 1945 if (apc->port_st_save) { 1946 err = mana_alloc_queues(ndev); 1947 if (err) { 1948 mana_cleanup_port_context(apc); 1949 return err; 1950 } 1951 } 1952 1953 apc->port_is_up = apc->port_st_save; 1954 1955 /* Ensure port state updated before txq state */ 1956 smp_wmb(); 1957 1958 if (apc->port_is_up) 1959 netif_carrier_on(ndev); 1960 1961 netif_device_attach(ndev); 1962 1963 return 0; 1964 } 1965 1966 static int mana_dealloc_queues(struct net_device *ndev) 1967 { 1968 struct mana_port_context *apc = netdev_priv(ndev); 1969 struct gdma_dev *gd = apc->ac->gdma_dev; 1970 struct mana_txq *txq; 1971 int i, err; 1972 1973 if (apc->port_is_up) 1974 return -EINVAL; 1975 1976 mana_chn_setxdp(apc, NULL); 1977 1978 if (gd->gdma_context->is_pf) 1979 mana_pf_deregister_filter(apc); 1980 1981 /* No packet can be transmitted now since apc->port_is_up is false. 1982 * There is still a tiny chance that mana_poll_tx_cq() can re-enable 1983 * a txq because it may not timely see apc->port_is_up being cleared 1984 * to false, but it doesn't matter since mana_start_xmit() drops any 1985 * new packets due to apc->port_is_up being false. 1986 * 1987 * Drain all the in-flight TX packets 1988 */ 1989 for (i = 0; i < apc->num_queues; i++) { 1990 txq = &apc->tx_qp[i].txq; 1991 1992 while (atomic_read(&txq->pending_sends) > 0) 1993 usleep_range(1000, 2000); 1994 } 1995 1996 /* We're 100% sure the queues can no longer be woken up, because 1997 * we're sure now mana_poll_tx_cq() can't be running. 1998 */ 1999 2000 apc->rss_state = TRI_STATE_FALSE; 2001 err = mana_config_rss(apc, TRI_STATE_FALSE, false, false); 2002 if (err) { 2003 netdev_err(ndev, "Failed to disable vPort: %d\n", err); 2004 return err; 2005 } 2006 2007 mana_destroy_vport(apc); 2008 2009 return 0; 2010 } 2011 2012 int mana_detach(struct net_device *ndev, bool from_close) 2013 { 2014 struct mana_port_context *apc = netdev_priv(ndev); 2015 int err; 2016 2017 ASSERT_RTNL(); 2018 2019 apc->port_st_save = apc->port_is_up; 2020 apc->port_is_up = false; 2021 2022 /* Ensure port state updated before txq state */ 2023 smp_wmb(); 2024 2025 netif_tx_disable(ndev); 2026 netif_carrier_off(ndev); 2027 2028 if (apc->port_st_save) { 2029 err = mana_dealloc_queues(ndev); 2030 if (err) 2031 return err; 2032 } 2033 2034 if (!from_close) { 2035 netif_device_detach(ndev); 2036 mana_cleanup_port_context(apc); 2037 } 2038 2039 return 0; 2040 } 2041 2042 static int mana_probe_port(struct mana_context *ac, int port_idx, 2043 struct net_device **ndev_storage) 2044 { 2045 struct gdma_context *gc = ac->gdma_dev->gdma_context; 2046 struct mana_port_context *apc; 2047 struct net_device *ndev; 2048 int err; 2049 2050 ndev = alloc_etherdev_mq(sizeof(struct mana_port_context), 2051 gc->max_num_queues); 2052 if (!ndev) 2053 return -ENOMEM; 2054 2055 *ndev_storage = ndev; 2056 2057 apc = netdev_priv(ndev); 2058 apc->ac = ac; 2059 apc->ndev = ndev; 2060 apc->max_queues = gc->max_num_queues; 2061 apc->num_queues = gc->max_num_queues; 2062 apc->port_handle = INVALID_MANA_HANDLE; 2063 apc->pf_filter_handle = INVALID_MANA_HANDLE; 2064 apc->port_idx = port_idx; 2065 2066 ndev->netdev_ops = &mana_devops; 2067 ndev->ethtool_ops = &mana_ethtool_ops; 2068 ndev->mtu = ETH_DATA_LEN; 2069 ndev->max_mtu = ndev->mtu; 2070 ndev->min_mtu = ndev->mtu; 2071 ndev->needed_headroom = MANA_HEADROOM; 2072 SET_NETDEV_DEV(ndev, gc->dev); 2073 2074 netif_carrier_off(ndev); 2075 2076 netdev_rss_key_fill(apc->hashkey, MANA_HASH_KEY_SIZE); 2077 2078 err = mana_init_port(ndev); 2079 if (err) 2080 goto free_net; 2081 2082 netdev_lockdep_set_classes(ndev); 2083 2084 ndev->hw_features = NETIF_F_SG | NETIF_F_IP_CSUM | NETIF_F_IPV6_CSUM; 2085 ndev->hw_features |= NETIF_F_RXCSUM; 2086 ndev->hw_features |= NETIF_F_TSO | NETIF_F_TSO6; 2087 ndev->hw_features |= NETIF_F_RXHASH; 2088 ndev->features = ndev->hw_features; 2089 ndev->vlan_features = 0; 2090 2091 err = register_netdev(ndev); 2092 if (err) { 2093 netdev_err(ndev, "Unable to register netdev.\n"); 2094 goto reset_apc; 2095 } 2096 2097 return 0; 2098 2099 reset_apc: 2100 kfree(apc->rxqs); 2101 apc->rxqs = NULL; 2102 free_net: 2103 *ndev_storage = NULL; 2104 netdev_err(ndev, "Failed to probe vPort %d: %d\n", port_idx, err); 2105 free_netdev(ndev); 2106 return err; 2107 } 2108 2109 int mana_probe(struct gdma_dev *gd, bool resuming) 2110 { 2111 struct gdma_context *gc = gd->gdma_context; 2112 struct mana_context *ac = gd->driver_data; 2113 struct device *dev = gc->dev; 2114 u16 num_ports = 0; 2115 int err; 2116 int i; 2117 2118 dev_info(dev, 2119 "Microsoft Azure Network Adapter protocol version: %d.%d.%d\n", 2120 MANA_MAJOR_VERSION, MANA_MINOR_VERSION, MANA_MICRO_VERSION); 2121 2122 err = mana_gd_register_device(gd); 2123 if (err) 2124 return err; 2125 2126 if (!resuming) { 2127 ac = kzalloc(sizeof(*ac), GFP_KERNEL); 2128 if (!ac) 2129 return -ENOMEM; 2130 2131 ac->gdma_dev = gd; 2132 gd->driver_data = ac; 2133 } 2134 2135 err = mana_create_eq(ac); 2136 if (err) 2137 goto out; 2138 2139 err = mana_query_device_cfg(ac, MANA_MAJOR_VERSION, MANA_MINOR_VERSION, 2140 MANA_MICRO_VERSION, &num_ports); 2141 if (err) 2142 goto out; 2143 2144 if (!resuming) { 2145 ac->num_ports = num_ports; 2146 } else { 2147 if (ac->num_ports != num_ports) { 2148 dev_err(dev, "The number of vPorts changed: %d->%d\n", 2149 ac->num_ports, num_ports); 2150 err = -EPROTO; 2151 goto out; 2152 } 2153 } 2154 2155 if (ac->num_ports == 0) 2156 dev_err(dev, "Failed to detect any vPort\n"); 2157 2158 if (ac->num_ports > MAX_PORTS_IN_MANA_DEV) 2159 ac->num_ports = MAX_PORTS_IN_MANA_DEV; 2160 2161 if (!resuming) { 2162 for (i = 0; i < ac->num_ports; i++) { 2163 err = mana_probe_port(ac, i, &ac->ports[i]); 2164 if (err) 2165 break; 2166 } 2167 } else { 2168 for (i = 0; i < ac->num_ports; i++) { 2169 rtnl_lock(); 2170 err = mana_attach(ac->ports[i]); 2171 rtnl_unlock(); 2172 if (err) 2173 break; 2174 } 2175 } 2176 out: 2177 if (err) 2178 mana_remove(gd, false); 2179 2180 return err; 2181 } 2182 2183 void mana_remove(struct gdma_dev *gd, bool suspending) 2184 { 2185 struct gdma_context *gc = gd->gdma_context; 2186 struct mana_context *ac = gd->driver_data; 2187 struct device *dev = gc->dev; 2188 struct net_device *ndev; 2189 int err; 2190 int i; 2191 2192 for (i = 0; i < ac->num_ports; i++) { 2193 ndev = ac->ports[i]; 2194 if (!ndev) { 2195 if (i == 0) 2196 dev_err(dev, "No net device to remove\n"); 2197 goto out; 2198 } 2199 2200 /* All cleanup actions should stay after rtnl_lock(), otherwise 2201 * other functions may access partially cleaned up data. 2202 */ 2203 rtnl_lock(); 2204 2205 err = mana_detach(ndev, false); 2206 if (err) 2207 netdev_err(ndev, "Failed to detach vPort %d: %d\n", 2208 i, err); 2209 2210 if (suspending) { 2211 /* No need to unregister the ndev. */ 2212 rtnl_unlock(); 2213 continue; 2214 } 2215 2216 unregister_netdevice(ndev); 2217 2218 rtnl_unlock(); 2219 2220 free_netdev(ndev); 2221 } 2222 2223 mana_destroy_eq(ac); 2224 2225 out: 2226 mana_gd_deregister_device(gd); 2227 2228 if (suspending) 2229 return; 2230 2231 gd->driver_data = NULL; 2232 gd->gdma_context = NULL; 2233 kfree(ac); 2234 } 2235