1 // SPDX-License-Identifier: ISC 2 /* 3 * Copyright (C) 2016 Felix Fietkau <nbd@nbd.name> 4 */ 5 6 #include "mt76.h" 7 8 static int 9 mt76_txq_get_qid(struct ieee80211_txq *txq) 10 { 11 if (!txq->sta) 12 return MT_TXQ_BE; 13 14 return txq->ac; 15 } 16 17 void 18 mt76_tx_check_agg_ssn(struct ieee80211_sta *sta, struct sk_buff *skb) 19 { 20 struct ieee80211_hdr *hdr = (struct ieee80211_hdr *)skb->data; 21 struct ieee80211_txq *txq; 22 struct mt76_txq *mtxq; 23 u8 tid; 24 25 if (!sta || !ieee80211_is_data_qos(hdr->frame_control) || 26 !ieee80211_is_data_present(hdr->frame_control)) 27 return; 28 29 tid = skb->priority & IEEE80211_QOS_CTL_TAG1D_MASK; 30 txq = sta->txq[tid]; 31 mtxq = (struct mt76_txq *)txq->drv_priv; 32 if (!mtxq->aggr) 33 return; 34 35 mtxq->agg_ssn = le16_to_cpu(hdr->seq_ctrl) + 0x10; 36 } 37 EXPORT_SYMBOL_GPL(mt76_tx_check_agg_ssn); 38 39 void 40 mt76_tx_status_lock(struct mt76_dev *dev, struct sk_buff_head *list) 41 __acquires(&dev->status_lock) 42 { 43 __skb_queue_head_init(list); 44 spin_lock_bh(&dev->status_lock); 45 } 46 EXPORT_SYMBOL_GPL(mt76_tx_status_lock); 47 48 void 49 mt76_tx_status_unlock(struct mt76_dev *dev, struct sk_buff_head *list) 50 __releases(&dev->status_lock) 51 { 52 struct ieee80211_hw *hw; 53 struct sk_buff *skb; 54 55 spin_unlock_bh(&dev->status_lock); 56 57 rcu_read_lock(); 58 while ((skb = __skb_dequeue(list)) != NULL) { 59 struct ieee80211_tx_status status = { 60 .skb = skb, 61 .info = IEEE80211_SKB_CB(skb), 62 }; 63 struct ieee80211_rate_status rs = {}; 64 struct mt76_tx_cb *cb = mt76_tx_skb_cb(skb); 65 struct mt76_wcid *wcid; 66 67 wcid = rcu_dereference(dev->wcid[cb->wcid]); 68 if (wcid) { 69 status.sta = wcid_to_sta(wcid); 70 if (status.sta && (wcid->rate.flags || wcid->rate.legacy)) { 71 rs.rate_idx = wcid->rate; 72 status.rates = &rs; 73 status.n_rates = 1; 74 } else { 75 status.n_rates = 0; 76 } 77 } 78 79 hw = mt76_tx_status_get_hw(dev, skb); 80 ieee80211_tx_status_ext(hw, &status); 81 } 82 rcu_read_unlock(); 83 } 84 EXPORT_SYMBOL_GPL(mt76_tx_status_unlock); 85 86 static void 87 __mt76_tx_status_skb_done(struct mt76_dev *dev, struct sk_buff *skb, u8 flags, 88 struct sk_buff_head *list) 89 { 90 struct ieee80211_tx_info *info = IEEE80211_SKB_CB(skb); 91 struct mt76_tx_cb *cb = mt76_tx_skb_cb(skb); 92 u8 done = MT_TX_CB_DMA_DONE | MT_TX_CB_TXS_DONE; 93 94 flags |= cb->flags; 95 cb->flags = flags; 96 97 if ((flags & done) != done) 98 return; 99 100 /* Tx status can be unreliable. if it fails, mark the frame as ACKed */ 101 if (flags & MT_TX_CB_TXS_FAILED) { 102 info->status.rates[0].count = 0; 103 info->status.rates[0].idx = -1; 104 info->flags |= IEEE80211_TX_STAT_ACK; 105 } 106 107 __skb_queue_tail(list, skb); 108 } 109 110 void 111 mt76_tx_status_skb_done(struct mt76_dev *dev, struct sk_buff *skb, 112 struct sk_buff_head *list) 113 { 114 __mt76_tx_status_skb_done(dev, skb, MT_TX_CB_TXS_DONE, list); 115 } 116 EXPORT_SYMBOL_GPL(mt76_tx_status_skb_done); 117 118 int 119 mt76_tx_status_skb_add(struct mt76_dev *dev, struct mt76_wcid *wcid, 120 struct sk_buff *skb) 121 { 122 struct ieee80211_tx_info *info = IEEE80211_SKB_CB(skb); 123 struct mt76_tx_cb *cb = mt76_tx_skb_cb(skb); 124 int pid; 125 126 memset(cb, 0, sizeof(*cb)); 127 128 if (!wcid || !rcu_access_pointer(dev->wcid[wcid->idx])) 129 return MT_PACKET_ID_NO_ACK; 130 131 if (info->flags & IEEE80211_TX_CTL_NO_ACK) 132 return MT_PACKET_ID_NO_ACK; 133 134 if (!(info->flags & (IEEE80211_TX_CTL_REQ_TX_STATUS | 135 IEEE80211_TX_CTL_RATE_CTRL_PROBE))) 136 return MT_PACKET_ID_NO_SKB; 137 138 spin_lock_bh(&dev->status_lock); 139 140 pid = idr_alloc(&wcid->pktid, skb, MT_PACKET_ID_FIRST, 141 MT_PACKET_ID_MASK, GFP_ATOMIC); 142 if (pid < 0) { 143 pid = MT_PACKET_ID_NO_SKB; 144 goto out; 145 } 146 147 cb->wcid = wcid->idx; 148 cb->pktid = pid; 149 150 if (list_empty(&wcid->list)) 151 list_add_tail(&wcid->list, &dev->wcid_list); 152 153 out: 154 spin_unlock_bh(&dev->status_lock); 155 156 return pid; 157 } 158 EXPORT_SYMBOL_GPL(mt76_tx_status_skb_add); 159 160 struct sk_buff * 161 mt76_tx_status_skb_get(struct mt76_dev *dev, struct mt76_wcid *wcid, int pktid, 162 struct sk_buff_head *list) 163 { 164 struct sk_buff *skb; 165 int id; 166 167 lockdep_assert_held(&dev->status_lock); 168 169 skb = idr_remove(&wcid->pktid, pktid); 170 if (skb) 171 goto out; 172 173 /* look for stale entries in the wcid idr queue */ 174 idr_for_each_entry(&wcid->pktid, skb, id) { 175 struct mt76_tx_cb *cb = mt76_tx_skb_cb(skb); 176 177 if (pktid >= 0) { 178 if (!(cb->flags & MT_TX_CB_DMA_DONE)) 179 continue; 180 181 if (time_is_after_jiffies(cb->jiffies + 182 MT_TX_STATUS_SKB_TIMEOUT)) 183 continue; 184 } 185 186 /* It has been too long since DMA_DONE, time out this packet 187 * and stop waiting for TXS callback. 188 */ 189 idr_remove(&wcid->pktid, cb->pktid); 190 __mt76_tx_status_skb_done(dev, skb, MT_TX_CB_TXS_FAILED | 191 MT_TX_CB_TXS_DONE, list); 192 } 193 194 out: 195 if (idr_is_empty(&wcid->pktid)) 196 list_del_init(&wcid->list); 197 198 return skb; 199 } 200 EXPORT_SYMBOL_GPL(mt76_tx_status_skb_get); 201 202 void 203 mt76_tx_status_check(struct mt76_dev *dev, bool flush) 204 { 205 struct mt76_wcid *wcid, *tmp; 206 struct sk_buff_head list; 207 208 mt76_tx_status_lock(dev, &list); 209 list_for_each_entry_safe(wcid, tmp, &dev->wcid_list, list) 210 mt76_tx_status_skb_get(dev, wcid, flush ? -1 : 0, &list); 211 mt76_tx_status_unlock(dev, &list); 212 } 213 EXPORT_SYMBOL_GPL(mt76_tx_status_check); 214 215 static void 216 mt76_tx_check_non_aql(struct mt76_dev *dev, struct mt76_wcid *wcid, 217 struct sk_buff *skb) 218 { 219 struct ieee80211_tx_info *info = IEEE80211_SKB_CB(skb); 220 int pending; 221 222 if (!wcid || info->tx_time_est) 223 return; 224 225 pending = atomic_dec_return(&wcid->non_aql_packets); 226 if (pending < 0) 227 atomic_cmpxchg(&wcid->non_aql_packets, pending, 0); 228 } 229 230 void __mt76_tx_complete_skb(struct mt76_dev *dev, u16 wcid_idx, struct sk_buff *skb, 231 struct list_head *free_list) 232 { 233 struct mt76_tx_cb *cb = mt76_tx_skb_cb(skb); 234 struct ieee80211_tx_status status = { 235 .skb = skb, 236 .free_list = free_list, 237 }; 238 struct mt76_wcid *wcid = NULL; 239 struct ieee80211_hw *hw; 240 struct sk_buff_head list; 241 242 rcu_read_lock(); 243 244 if (wcid_idx < ARRAY_SIZE(dev->wcid)) 245 wcid = rcu_dereference(dev->wcid[wcid_idx]); 246 247 mt76_tx_check_non_aql(dev, wcid, skb); 248 249 #ifdef CONFIG_NL80211_TESTMODE 250 if (mt76_is_testmode_skb(dev, skb, &hw)) { 251 struct mt76_phy *phy = hw->priv; 252 253 if (skb == phy->test.tx_skb) 254 phy->test.tx_done++; 255 if (phy->test.tx_queued == phy->test.tx_done) 256 wake_up(&dev->tx_wait); 257 258 dev_kfree_skb_any(skb); 259 goto out; 260 } 261 #endif 262 263 if (cb->pktid < MT_PACKET_ID_FIRST) { 264 hw = mt76_tx_status_get_hw(dev, skb); 265 status.sta = wcid_to_sta(wcid); 266 ieee80211_tx_status_ext(hw, &status); 267 goto out; 268 } 269 270 mt76_tx_status_lock(dev, &list); 271 cb->jiffies = jiffies; 272 __mt76_tx_status_skb_done(dev, skb, MT_TX_CB_DMA_DONE, &list); 273 mt76_tx_status_unlock(dev, &list); 274 275 out: 276 rcu_read_unlock(); 277 } 278 EXPORT_SYMBOL_GPL(__mt76_tx_complete_skb); 279 280 static int 281 __mt76_tx_queue_skb(struct mt76_phy *phy, int qid, struct sk_buff *skb, 282 struct mt76_wcid *wcid, struct ieee80211_sta *sta, 283 bool *stop) 284 { 285 struct ieee80211_tx_info *info = IEEE80211_SKB_CB(skb); 286 struct mt76_queue *q = phy->q_tx[qid]; 287 struct mt76_dev *dev = phy->dev; 288 bool non_aql; 289 int pending; 290 int idx; 291 292 non_aql = !info->tx_time_est; 293 idx = dev->queue_ops->tx_queue_skb(dev, q, qid, skb, wcid, sta); 294 if (idx < 0 || !sta) 295 return idx; 296 297 wcid = (struct mt76_wcid *)sta->drv_priv; 298 q->entry[idx].wcid = wcid->idx; 299 300 if (!non_aql) 301 return idx; 302 303 pending = atomic_inc_return(&wcid->non_aql_packets); 304 if (stop && pending >= MT_MAX_NON_AQL_PKT) 305 *stop = true; 306 307 return idx; 308 } 309 310 void 311 mt76_tx(struct mt76_phy *phy, struct ieee80211_sta *sta, 312 struct mt76_wcid *wcid, struct sk_buff *skb) 313 { 314 struct mt76_dev *dev = phy->dev; 315 struct ieee80211_tx_info *info = IEEE80211_SKB_CB(skb); 316 struct ieee80211_hdr *hdr = (struct ieee80211_hdr *)skb->data; 317 struct mt76_queue *q; 318 int qid = skb_get_queue_mapping(skb); 319 320 if (mt76_testmode_enabled(phy)) { 321 ieee80211_free_txskb(phy->hw, skb); 322 return; 323 } 324 325 if (WARN_ON(qid >= MT_TXQ_PSD)) { 326 qid = MT_TXQ_BE; 327 skb_set_queue_mapping(skb, qid); 328 } 329 330 if ((dev->drv->drv_flags & MT_DRV_HW_MGMT_TXQ) && 331 !(info->flags & IEEE80211_TX_CTL_HW_80211_ENCAP) && 332 !ieee80211_is_data(hdr->frame_control) && 333 !ieee80211_is_bufferable_mmpdu(hdr->frame_control)) { 334 qid = MT_TXQ_PSD; 335 } 336 337 if (wcid && !(wcid->tx_info & MT_WCID_TX_INFO_SET)) 338 ieee80211_get_tx_rates(info->control.vif, sta, skb, 339 info->control.rates, 1); 340 341 info->hw_queue |= FIELD_PREP(MT_TX_HW_QUEUE_PHY, phy->band_idx); 342 q = phy->q_tx[qid]; 343 344 spin_lock_bh(&q->lock); 345 __mt76_tx_queue_skb(phy, qid, skb, wcid, sta, NULL); 346 dev->queue_ops->kick(dev, q); 347 spin_unlock_bh(&q->lock); 348 } 349 EXPORT_SYMBOL_GPL(mt76_tx); 350 351 static struct sk_buff * 352 mt76_txq_dequeue(struct mt76_phy *phy, struct mt76_txq *mtxq) 353 { 354 struct ieee80211_txq *txq = mtxq_to_txq(mtxq); 355 struct ieee80211_tx_info *info; 356 struct sk_buff *skb; 357 358 skb = ieee80211_tx_dequeue(phy->hw, txq); 359 if (!skb) 360 return NULL; 361 362 info = IEEE80211_SKB_CB(skb); 363 info->hw_queue |= FIELD_PREP(MT_TX_HW_QUEUE_PHY, phy->band_idx); 364 365 return skb; 366 } 367 368 static void 369 mt76_queue_ps_skb(struct mt76_phy *phy, struct ieee80211_sta *sta, 370 struct sk_buff *skb, bool last) 371 { 372 struct mt76_wcid *wcid = (struct mt76_wcid *)sta->drv_priv; 373 struct ieee80211_tx_info *info = IEEE80211_SKB_CB(skb); 374 375 info->control.flags |= IEEE80211_TX_CTRL_PS_RESPONSE; 376 if (last) 377 info->flags |= IEEE80211_TX_STATUS_EOSP | 378 IEEE80211_TX_CTL_REQ_TX_STATUS; 379 380 mt76_skb_set_moredata(skb, !last); 381 __mt76_tx_queue_skb(phy, MT_TXQ_PSD, skb, wcid, sta, NULL); 382 } 383 384 void 385 mt76_release_buffered_frames(struct ieee80211_hw *hw, struct ieee80211_sta *sta, 386 u16 tids, int nframes, 387 enum ieee80211_frame_release_type reason, 388 bool more_data) 389 { 390 struct mt76_phy *phy = hw->priv; 391 struct mt76_dev *dev = phy->dev; 392 struct sk_buff *last_skb = NULL; 393 struct mt76_queue *hwq = phy->q_tx[MT_TXQ_PSD]; 394 int i; 395 396 spin_lock_bh(&hwq->lock); 397 for (i = 0; tids && nframes; i++, tids >>= 1) { 398 struct ieee80211_txq *txq = sta->txq[i]; 399 struct mt76_txq *mtxq = (struct mt76_txq *)txq->drv_priv; 400 struct sk_buff *skb; 401 402 if (!(tids & 1)) 403 continue; 404 405 do { 406 skb = mt76_txq_dequeue(phy, mtxq); 407 if (!skb) 408 break; 409 410 nframes--; 411 if (last_skb) 412 mt76_queue_ps_skb(phy, sta, last_skb, false); 413 414 last_skb = skb; 415 } while (nframes); 416 } 417 418 if (last_skb) { 419 mt76_queue_ps_skb(phy, sta, last_skb, true); 420 dev->queue_ops->kick(dev, hwq); 421 } else { 422 ieee80211_sta_eosp(sta); 423 } 424 425 spin_unlock_bh(&hwq->lock); 426 } 427 EXPORT_SYMBOL_GPL(mt76_release_buffered_frames); 428 429 static bool 430 mt76_txq_stopped(struct mt76_queue *q) 431 { 432 return q->stopped || q->blocked || 433 q->queued + MT_TXQ_FREE_THR >= q->ndesc; 434 } 435 436 static int 437 mt76_txq_send_burst(struct mt76_phy *phy, struct mt76_queue *q, 438 struct mt76_txq *mtxq, struct mt76_wcid *wcid) 439 { 440 struct mt76_dev *dev = phy->dev; 441 struct ieee80211_txq *txq = mtxq_to_txq(mtxq); 442 enum mt76_txq_id qid = mt76_txq_get_qid(txq); 443 struct ieee80211_tx_info *info; 444 struct sk_buff *skb; 445 int n_frames = 1; 446 bool stop = false; 447 int idx; 448 449 if (test_bit(MT_WCID_FLAG_PS, &wcid->flags)) 450 return 0; 451 452 if (atomic_read(&wcid->non_aql_packets) >= MT_MAX_NON_AQL_PKT) 453 return 0; 454 455 skb = mt76_txq_dequeue(phy, mtxq); 456 if (!skb) 457 return 0; 458 459 info = IEEE80211_SKB_CB(skb); 460 if (!(wcid->tx_info & MT_WCID_TX_INFO_SET)) 461 ieee80211_get_tx_rates(txq->vif, txq->sta, skb, 462 info->control.rates, 1); 463 464 spin_lock(&q->lock); 465 idx = __mt76_tx_queue_skb(phy, qid, skb, wcid, txq->sta, &stop); 466 spin_unlock(&q->lock); 467 if (idx < 0) 468 return idx; 469 470 do { 471 if (test_bit(MT76_RESET, &phy->state)) 472 return -EBUSY; 473 474 if (stop || mt76_txq_stopped(q)) 475 break; 476 477 skb = mt76_txq_dequeue(phy, mtxq); 478 if (!skb) 479 break; 480 481 info = IEEE80211_SKB_CB(skb); 482 if (!(wcid->tx_info & MT_WCID_TX_INFO_SET)) 483 ieee80211_get_tx_rates(txq->vif, txq->sta, skb, 484 info->control.rates, 1); 485 486 spin_lock(&q->lock); 487 idx = __mt76_tx_queue_skb(phy, qid, skb, wcid, txq->sta, &stop); 488 spin_unlock(&q->lock); 489 if (idx < 0) 490 break; 491 492 n_frames++; 493 } while (1); 494 495 spin_lock(&q->lock); 496 dev->queue_ops->kick(dev, q); 497 spin_unlock(&q->lock); 498 499 return n_frames; 500 } 501 502 static int 503 mt76_txq_schedule_list(struct mt76_phy *phy, enum mt76_txq_id qid) 504 { 505 struct mt76_queue *q = phy->q_tx[qid]; 506 struct mt76_dev *dev = phy->dev; 507 struct ieee80211_txq *txq; 508 struct mt76_txq *mtxq; 509 struct mt76_wcid *wcid; 510 int ret = 0; 511 512 while (1) { 513 int n_frames = 0; 514 515 if (test_bit(MT76_RESET, &phy->state)) 516 return -EBUSY; 517 518 if (dev->queue_ops->tx_cleanup && 519 q->queued + 2 * MT_TXQ_FREE_THR >= q->ndesc) { 520 dev->queue_ops->tx_cleanup(dev, q, false); 521 } 522 523 txq = ieee80211_next_txq(phy->hw, qid); 524 if (!txq) 525 break; 526 527 mtxq = (struct mt76_txq *)txq->drv_priv; 528 wcid = rcu_dereference(dev->wcid[mtxq->wcid]); 529 if (!wcid || test_bit(MT_WCID_FLAG_PS, &wcid->flags)) 530 continue; 531 532 if (mtxq->send_bar && mtxq->aggr) { 533 struct ieee80211_txq *txq = mtxq_to_txq(mtxq); 534 struct ieee80211_sta *sta = txq->sta; 535 struct ieee80211_vif *vif = txq->vif; 536 u16 agg_ssn = mtxq->agg_ssn; 537 u8 tid = txq->tid; 538 539 mtxq->send_bar = false; 540 ieee80211_send_bar(vif, sta->addr, tid, agg_ssn); 541 } 542 543 if (!mt76_txq_stopped(q)) 544 n_frames = mt76_txq_send_burst(phy, q, mtxq, wcid); 545 546 ieee80211_return_txq(phy->hw, txq, false); 547 548 if (unlikely(n_frames < 0)) 549 return n_frames; 550 551 ret += n_frames; 552 } 553 554 return ret; 555 } 556 557 void mt76_txq_schedule(struct mt76_phy *phy, enum mt76_txq_id qid) 558 { 559 int len; 560 561 if (qid >= 4) 562 return; 563 564 local_bh_disable(); 565 rcu_read_lock(); 566 567 do { 568 ieee80211_txq_schedule_start(phy->hw, qid); 569 len = mt76_txq_schedule_list(phy, qid); 570 ieee80211_txq_schedule_end(phy->hw, qid); 571 } while (len > 0); 572 573 rcu_read_unlock(); 574 local_bh_enable(); 575 } 576 EXPORT_SYMBOL_GPL(mt76_txq_schedule); 577 578 void mt76_txq_schedule_all(struct mt76_phy *phy) 579 { 580 int i; 581 582 for (i = 0; i <= MT_TXQ_BK; i++) 583 mt76_txq_schedule(phy, i); 584 } 585 EXPORT_SYMBOL_GPL(mt76_txq_schedule_all); 586 587 void mt76_tx_worker_run(struct mt76_dev *dev) 588 { 589 struct mt76_phy *phy; 590 int i; 591 592 for (i = 0; i < ARRAY_SIZE(dev->phys); i++) { 593 phy = dev->phys[i]; 594 if (!phy) 595 continue; 596 597 mt76_txq_schedule_all(phy); 598 } 599 600 #ifdef CONFIG_NL80211_TESTMODE 601 for (i = 0; i < ARRAY_SIZE(dev->phys); i++) { 602 phy = dev->phys[i]; 603 if (!phy || !phy->test.tx_pending) 604 continue; 605 606 mt76_testmode_tx_pending(phy); 607 } 608 #endif 609 } 610 EXPORT_SYMBOL_GPL(mt76_tx_worker_run); 611 612 void mt76_tx_worker(struct mt76_worker *w) 613 { 614 struct mt76_dev *dev = container_of(w, struct mt76_dev, tx_worker); 615 616 mt76_tx_worker_run(dev); 617 } 618 619 void mt76_stop_tx_queues(struct mt76_phy *phy, struct ieee80211_sta *sta, 620 bool send_bar) 621 { 622 int i; 623 624 for (i = 0; i < ARRAY_SIZE(sta->txq); i++) { 625 struct ieee80211_txq *txq = sta->txq[i]; 626 struct mt76_queue *hwq; 627 struct mt76_txq *mtxq; 628 629 if (!txq) 630 continue; 631 632 hwq = phy->q_tx[mt76_txq_get_qid(txq)]; 633 mtxq = (struct mt76_txq *)txq->drv_priv; 634 635 spin_lock_bh(&hwq->lock); 636 mtxq->send_bar = mtxq->aggr && send_bar; 637 spin_unlock_bh(&hwq->lock); 638 } 639 } 640 EXPORT_SYMBOL_GPL(mt76_stop_tx_queues); 641 642 void mt76_wake_tx_queue(struct ieee80211_hw *hw, struct ieee80211_txq *txq) 643 { 644 struct mt76_phy *phy = hw->priv; 645 struct mt76_dev *dev = phy->dev; 646 647 if (!test_bit(MT76_STATE_RUNNING, &phy->state)) 648 return; 649 650 mt76_worker_schedule(&dev->tx_worker); 651 } 652 EXPORT_SYMBOL_GPL(mt76_wake_tx_queue); 653 654 u8 mt76_ac_to_hwq(u8 ac) 655 { 656 static const u8 wmm_queue_map[] = { 657 [IEEE80211_AC_BE] = 0, 658 [IEEE80211_AC_BK] = 1, 659 [IEEE80211_AC_VI] = 2, 660 [IEEE80211_AC_VO] = 3, 661 }; 662 663 if (WARN_ON(ac >= IEEE80211_NUM_ACS)) 664 return 0; 665 666 return wmm_queue_map[ac]; 667 } 668 EXPORT_SYMBOL_GPL(mt76_ac_to_hwq); 669 670 int mt76_skb_adjust_pad(struct sk_buff *skb, int pad) 671 { 672 struct sk_buff *iter, *last = skb; 673 674 /* First packet of a A-MSDU burst keeps track of the whole burst 675 * length, need to update length of it and the last packet. 676 */ 677 skb_walk_frags(skb, iter) { 678 last = iter; 679 if (!iter->next) { 680 skb->data_len += pad; 681 skb->len += pad; 682 break; 683 } 684 } 685 686 if (skb_pad(last, pad)) 687 return -ENOMEM; 688 689 __skb_put(last, pad); 690 691 return 0; 692 } 693 EXPORT_SYMBOL_GPL(mt76_skb_adjust_pad); 694 695 void mt76_queue_tx_complete(struct mt76_dev *dev, struct mt76_queue *q, 696 struct mt76_queue_entry *e) 697 { 698 if (e->skb) 699 dev->drv->tx_complete_skb(dev, e); 700 701 spin_lock_bh(&q->lock); 702 q->tail = (q->tail + 1) % q->ndesc; 703 q->queued--; 704 spin_unlock_bh(&q->lock); 705 } 706 EXPORT_SYMBOL_GPL(mt76_queue_tx_complete); 707 708 void __mt76_set_tx_blocked(struct mt76_dev *dev, bool blocked) 709 { 710 struct mt76_phy *phy = &dev->phy; 711 struct mt76_queue *q = phy->q_tx[0]; 712 713 if (blocked == q->blocked) 714 return; 715 716 q->blocked = blocked; 717 718 phy = dev->phys[MT_BAND1]; 719 if (phy) { 720 q = phy->q_tx[0]; 721 q->blocked = blocked; 722 } 723 phy = dev->phys[MT_BAND2]; 724 if (phy) { 725 q = phy->q_tx[0]; 726 q->blocked = blocked; 727 } 728 729 if (!blocked) 730 mt76_worker_schedule(&dev->tx_worker); 731 } 732 EXPORT_SYMBOL_GPL(__mt76_set_tx_blocked); 733 734 int mt76_token_consume(struct mt76_dev *dev, struct mt76_txwi_cache **ptxwi) 735 { 736 int token; 737 738 spin_lock_bh(&dev->token_lock); 739 740 token = idr_alloc(&dev->token, *ptxwi, 0, dev->token_size, GFP_ATOMIC); 741 if (token >= 0) 742 dev->token_count++; 743 744 #ifdef CONFIG_NET_MEDIATEK_SOC_WED 745 if (mtk_wed_device_active(&dev->mmio.wed) && 746 token >= dev->mmio.wed.wlan.token_start) 747 dev->wed_token_count++; 748 #endif 749 750 if (dev->token_count >= dev->token_size - MT76_TOKEN_FREE_THR) 751 __mt76_set_tx_blocked(dev, true); 752 753 spin_unlock_bh(&dev->token_lock); 754 755 return token; 756 } 757 EXPORT_SYMBOL_GPL(mt76_token_consume); 758 759 struct mt76_txwi_cache * 760 mt76_token_release(struct mt76_dev *dev, int token, bool *wake) 761 { 762 struct mt76_txwi_cache *txwi; 763 764 spin_lock_bh(&dev->token_lock); 765 766 txwi = idr_remove(&dev->token, token); 767 if (txwi) { 768 dev->token_count--; 769 770 #ifdef CONFIG_NET_MEDIATEK_SOC_WED 771 if (mtk_wed_device_active(&dev->mmio.wed) && 772 token >= dev->mmio.wed.wlan.token_start && 773 --dev->wed_token_count == 0) 774 wake_up(&dev->tx_wait); 775 #endif 776 } 777 778 if (dev->token_count < dev->token_size - MT76_TOKEN_FREE_THR && 779 dev->phy.q_tx[0]->blocked) 780 *wake = true; 781 782 spin_unlock_bh(&dev->token_lock); 783 784 return txwi; 785 } 786 EXPORT_SYMBOL_GPL(mt76_token_release); 787