Lines Matching refs:aq
118 struct efa_com_admin_queue *aq = &edev->aq; in efa_com_admin_init_sq() local
119 struct efa_com_admin_sq *sq = &aq->sq; in efa_com_admin_init_sq()
120 u16 size = aq->depth * sizeof(*sq->entries); in efa_com_admin_init_sq()
126 dma_alloc_coherent(aq->dmadev, size, &sq->dma_addr, GFP_KERNEL); in efa_com_admin_init_sq()
144 EFA_SET(&aq_caps, EFA_REGS_AQ_CAPS_AQ_DEPTH, aq->depth); in efa_com_admin_init_sq()
155 struct efa_com_admin_queue *aq = &edev->aq; in efa_com_admin_init_cq() local
156 struct efa_com_admin_cq *cq = &aq->cq; in efa_com_admin_init_cq()
157 u16 size = aq->depth * sizeof(*cq->entries); in efa_com_admin_init_cq()
163 dma_alloc_coherent(aq->dmadev, size, &cq->dma_addr, GFP_KERNEL); in efa_com_admin_init_cq()
178 EFA_SET(&acq_caps, EFA_REGS_ACQ_CAPS_ACQ_DEPTH, aq->depth); in efa_com_admin_init_cq()
182 aq->msix_vector_idx); in efa_com_admin_init_cq()
236 static u16 efa_com_alloc_ctx_id(struct efa_com_admin_queue *aq) in efa_com_alloc_ctx_id() argument
240 spin_lock(&aq->comp_ctx_lock); in efa_com_alloc_ctx_id()
241 ctx_id = aq->comp_ctx_pool[aq->comp_ctx_pool_next]; in efa_com_alloc_ctx_id()
242 aq->comp_ctx_pool_next++; in efa_com_alloc_ctx_id()
243 spin_unlock(&aq->comp_ctx_lock); in efa_com_alloc_ctx_id()
248 static void efa_com_dealloc_ctx_id(struct efa_com_admin_queue *aq, in efa_com_dealloc_ctx_id() argument
251 spin_lock(&aq->comp_ctx_lock); in efa_com_dealloc_ctx_id()
252 aq->comp_ctx_pool_next--; in efa_com_dealloc_ctx_id()
253 aq->comp_ctx_pool[aq->comp_ctx_pool_next] = ctx_id; in efa_com_dealloc_ctx_id()
254 spin_unlock(&aq->comp_ctx_lock); in efa_com_dealloc_ctx_id()
257 static inline void efa_com_put_comp_ctx(struct efa_com_admin_queue *aq, in efa_com_put_comp_ctx() argument
262 u16 ctx_id = cmd_id & (aq->depth - 1); in efa_com_put_comp_ctx()
264 ibdev_dbg(aq->efa_dev, "Put completion command_id %#x\n", cmd_id); in efa_com_put_comp_ctx()
266 efa_com_dealloc_ctx_id(aq, ctx_id); in efa_com_put_comp_ctx()
269 static struct efa_comp_ctx *efa_com_get_comp_ctx(struct efa_com_admin_queue *aq, in efa_com_get_comp_ctx() argument
272 u16 ctx_id = cmd_id & (aq->depth - 1); in efa_com_get_comp_ctx()
274 if (aq->comp_ctx[ctx_id].occupied && capture) { in efa_com_get_comp_ctx()
276 aq->efa_dev, in efa_com_get_comp_ctx()
283 aq->comp_ctx[ctx_id].occupied = 1; in efa_com_get_comp_ctx()
284 ibdev_dbg(aq->efa_dev, in efa_com_get_comp_ctx()
288 return &aq->comp_ctx[ctx_id]; in efa_com_get_comp_ctx()
291 static struct efa_comp_ctx *__efa_com_submit_admin_cmd(struct efa_com_admin_queue *aq, in __efa_com_submit_admin_cmd() argument
304 queue_size_mask = aq->depth - 1; in __efa_com_submit_admin_cmd()
305 pi = aq->sq.pc & queue_size_mask; in __efa_com_submit_admin_cmd()
307 ctx_id = efa_com_alloc_ctx_id(aq); in __efa_com_submit_admin_cmd()
311 cmd_id |= aq->sq.pc & ~queue_size_mask; in __efa_com_submit_admin_cmd()
316 EFA_ADMIN_AQ_COMMON_DESC_PHASE, aq->sq.phase); in __efa_com_submit_admin_cmd()
318 comp_ctx = efa_com_get_comp_ctx(aq, cmd_id, true); in __efa_com_submit_admin_cmd()
320 efa_com_dealloc_ctx_id(aq, ctx_id); in __efa_com_submit_admin_cmd()
331 aqe = &aq->sq.entries[pi]; in __efa_com_submit_admin_cmd()
335 aq->sq.pc++; in __efa_com_submit_admin_cmd()
336 atomic64_inc(&aq->stats.submitted_cmd); in __efa_com_submit_admin_cmd()
338 if ((aq->sq.pc & queue_size_mask) == 0) in __efa_com_submit_admin_cmd()
339 aq->sq.phase = !aq->sq.phase; in __efa_com_submit_admin_cmd()
342 writel(aq->sq.pc, aq->sq.db_addr); in __efa_com_submit_admin_cmd()
347 static inline int efa_com_init_comp_ctxt(struct efa_com_admin_queue *aq) in efa_com_init_comp_ctxt() argument
349 size_t pool_size = aq->depth * sizeof(*aq->comp_ctx_pool); in efa_com_init_comp_ctxt()
350 size_t size = aq->depth * sizeof(struct efa_comp_ctx); in efa_com_init_comp_ctxt()
354 aq->comp_ctx = devm_kzalloc(aq->dmadev, size, GFP_KERNEL); in efa_com_init_comp_ctxt()
355 aq->comp_ctx_pool = devm_kzalloc(aq->dmadev, pool_size, GFP_KERNEL); in efa_com_init_comp_ctxt()
356 if (!aq->comp_ctx || !aq->comp_ctx_pool) { in efa_com_init_comp_ctxt()
357 devm_kfree(aq->dmadev, aq->comp_ctx_pool); in efa_com_init_comp_ctxt()
358 devm_kfree(aq->dmadev, aq->comp_ctx); in efa_com_init_comp_ctxt()
362 for (i = 0; i < aq->depth; i++) { in efa_com_init_comp_ctxt()
363 comp_ctx = efa_com_get_comp_ctx(aq, i, false); in efa_com_init_comp_ctxt()
367 aq->comp_ctx_pool[i] = i; in efa_com_init_comp_ctxt()
370 spin_lock_init(&aq->comp_ctx_lock); in efa_com_init_comp_ctxt()
372 aq->comp_ctx_pool_next = 0; in efa_com_init_comp_ctxt()
377 static struct efa_comp_ctx *efa_com_submit_admin_cmd(struct efa_com_admin_queue *aq, in efa_com_submit_admin_cmd() argument
385 spin_lock(&aq->sq.lock); in efa_com_submit_admin_cmd()
386 if (!test_bit(EFA_AQ_STATE_RUNNING_BIT, &aq->state)) { in efa_com_submit_admin_cmd()
387 ibdev_err_ratelimited(aq->efa_dev, "Admin queue is closed\n"); in efa_com_submit_admin_cmd()
388 spin_unlock(&aq->sq.lock); in efa_com_submit_admin_cmd()
392 comp_ctx = __efa_com_submit_admin_cmd(aq, cmd, cmd_size_in_bytes, comp, in efa_com_submit_admin_cmd()
394 spin_unlock(&aq->sq.lock); in efa_com_submit_admin_cmd()
396 clear_bit(EFA_AQ_STATE_RUNNING_BIT, &aq->state); in efa_com_submit_admin_cmd()
401 static void efa_com_handle_single_admin_completion(struct efa_com_admin_queue *aq, in efa_com_handle_single_admin_completion() argument
410 comp_ctx = efa_com_get_comp_ctx(aq, cmd_id, false); in efa_com_handle_single_admin_completion()
412 ibdev_err(aq->efa_dev, in efa_com_handle_single_admin_completion()
414 clear_bit(EFA_AQ_STATE_RUNNING_BIT, &aq->state); in efa_com_handle_single_admin_completion()
421 if (!test_bit(EFA_AQ_STATE_POLLING_BIT, &aq->state)) in efa_com_handle_single_admin_completion()
425 static void efa_com_handle_admin_completion(struct efa_com_admin_queue *aq) in efa_com_handle_admin_completion() argument
433 queue_size_mask = aq->depth - 1; in efa_com_handle_admin_completion()
435 ci = aq->cq.cc & queue_size_mask; in efa_com_handle_admin_completion()
436 phase = aq->cq.phase; in efa_com_handle_admin_completion()
438 cqe = &aq->cq.entries[ci]; in efa_com_handle_admin_completion()
448 efa_com_handle_single_admin_completion(aq, cqe); in efa_com_handle_admin_completion()
452 if (ci == aq->depth) { in efa_com_handle_admin_completion()
457 cqe = &aq->cq.entries[ci]; in efa_com_handle_admin_completion()
460 aq->cq.cc += comp_num; in efa_com_handle_admin_completion()
461 aq->cq.phase = phase; in efa_com_handle_admin_completion()
462 aq->sq.cc += comp_num; in efa_com_handle_admin_completion()
463 atomic64_add(comp_num, &aq->stats.completed_cmd); in efa_com_handle_admin_completion()
486 struct efa_com_admin_queue *aq) in efa_com_wait_and_process_admin_cq_polling() argument
492 timeout = jiffies + usecs_to_jiffies(aq->completion_timeout); in efa_com_wait_and_process_admin_cq_polling()
495 spin_lock_irqsave(&aq->cq.lock, flags); in efa_com_wait_and_process_admin_cq_polling()
496 efa_com_handle_admin_completion(aq); in efa_com_wait_and_process_admin_cq_polling()
497 spin_unlock_irqrestore(&aq->cq.lock, flags); in efa_com_wait_and_process_admin_cq_polling()
504 aq->efa_dev, in efa_com_wait_and_process_admin_cq_polling()
507 atomic64_inc(&aq->stats.no_completion); in efa_com_wait_and_process_admin_cq_polling()
509 clear_bit(EFA_AQ_STATE_RUNNING_BIT, &aq->state); in efa_com_wait_and_process_admin_cq_polling()
514 msleep(aq->poll_interval); in efa_com_wait_and_process_admin_cq_polling()
519 efa_com_put_comp_ctx(aq, comp_ctx); in efa_com_wait_and_process_admin_cq_polling()
524 struct efa_com_admin_queue *aq) in efa_com_wait_and_process_admin_cq_interrupts() argument
530 usecs_to_jiffies(aq->completion_timeout)); in efa_com_wait_and_process_admin_cq_interrupts()
539 spin_lock_irqsave(&aq->cq.lock, flags); in efa_com_wait_and_process_admin_cq_interrupts()
540 efa_com_handle_admin_completion(aq); in efa_com_wait_and_process_admin_cq_interrupts()
541 spin_unlock_irqrestore(&aq->cq.lock, flags); in efa_com_wait_and_process_admin_cq_interrupts()
543 atomic64_inc(&aq->stats.no_completion); in efa_com_wait_and_process_admin_cq_interrupts()
547 aq->efa_dev, in efa_com_wait_and_process_admin_cq_interrupts()
551 comp_ctx, aq->sq.pc, aq->sq.cc, aq->cq.cc); in efa_com_wait_and_process_admin_cq_interrupts()
554 aq->efa_dev, in efa_com_wait_and_process_admin_cq_interrupts()
558 comp_ctx, aq->sq.pc, aq->sq.cc, aq->cq.cc); in efa_com_wait_and_process_admin_cq_interrupts()
560 clear_bit(EFA_AQ_STATE_RUNNING_BIT, &aq->state); in efa_com_wait_and_process_admin_cq_interrupts()
567 efa_com_put_comp_ctx(aq, comp_ctx); in efa_com_wait_and_process_admin_cq_interrupts()
580 struct efa_com_admin_queue *aq) in efa_com_wait_and_process_admin_cq() argument
582 if (test_bit(EFA_AQ_STATE_POLLING_BIT, &aq->state)) in efa_com_wait_and_process_admin_cq()
583 return efa_com_wait_and_process_admin_cq_polling(comp_ctx, aq); in efa_com_wait_and_process_admin_cq()
585 return efa_com_wait_and_process_admin_cq_interrupts(comp_ctx, aq); in efa_com_wait_and_process_admin_cq()
601 int efa_com_cmd_exec(struct efa_com_admin_queue *aq, in efa_com_cmd_exec() argument
613 down(&aq->avail_cmds); in efa_com_cmd_exec()
615 ibdev_dbg(aq->efa_dev, "%s (opcode %d)\n", in efa_com_cmd_exec()
618 comp_ctx = efa_com_submit_admin_cmd(aq, cmd, cmd_size, comp, comp_size); in efa_com_cmd_exec()
621 aq->efa_dev, in efa_com_cmd_exec()
626 up(&aq->avail_cmds); in efa_com_cmd_exec()
627 atomic64_inc(&aq->stats.cmd_err); in efa_com_cmd_exec()
631 err = efa_com_wait_and_process_admin_cq(comp_ctx, aq); in efa_com_cmd_exec()
634 aq->efa_dev, in efa_com_cmd_exec()
639 atomic64_inc(&aq->stats.cmd_err); in efa_com_cmd_exec()
642 up(&aq->avail_cmds); in efa_com_cmd_exec()
653 struct efa_com_admin_queue *aq = &edev->aq; in efa_com_admin_destroy() local
655 struct efa_com_admin_cq *cq = &aq->cq; in efa_com_admin_destroy()
656 struct efa_com_admin_sq *sq = &aq->sq; in efa_com_admin_destroy()
659 clear_bit(EFA_AQ_STATE_RUNNING_BIT, &aq->state); in efa_com_admin_destroy()
661 devm_kfree(edev->dmadev, aq->comp_ctx_pool); in efa_com_admin_destroy()
662 devm_kfree(edev->dmadev, aq->comp_ctx); in efa_com_admin_destroy()
664 size = aq->depth * sizeof(*sq->entries); in efa_com_admin_destroy()
667 size = aq->depth * sizeof(*cq->entries); in efa_com_admin_destroy()
690 set_bit(EFA_AQ_STATE_POLLING_BIT, &edev->aq.state); in efa_com_set_admin_polling_mode()
692 clear_bit(EFA_AQ_STATE_POLLING_BIT, &edev->aq.state); in efa_com_set_admin_polling_mode()
697 atomic64_t *s = (atomic64_t *)&edev->aq.stats; in efa_com_stats_init()
700 for (i = 0; i < sizeof(edev->aq.stats) / sizeof(*s); i++, s++) in efa_com_stats_init()
717 struct efa_com_admin_queue *aq = &edev->aq; in efa_com_admin_init() local
730 aq->depth = EFA_ADMIN_QUEUE_DEPTH; in efa_com_admin_init()
732 aq->dmadev = edev->dmadev; in efa_com_admin_init()
733 aq->efa_dev = edev->efa_dev; in efa_com_admin_init()
734 set_bit(EFA_AQ_STATE_POLLING_BIT, &aq->state); in efa_com_admin_init()
736 sema_init(&aq->avail_cmds, aq->depth); in efa_com_admin_init()
740 err = efa_com_init_comp_ctxt(aq); in efa_com_admin_init()
762 aq->completion_timeout = timeout * 100000; in efa_com_admin_init()
764 aq->completion_timeout = ADMIN_CMD_TIMEOUT_US; in efa_com_admin_init()
766 aq->poll_interval = EFA_POLL_INTERVAL_MS; in efa_com_admin_init()
768 set_bit(EFA_AQ_STATE_RUNNING_BIT, &aq->state); in efa_com_admin_init()
773 dma_free_coherent(edev->dmadev, aq->depth * sizeof(*aq->cq.entries), in efa_com_admin_init()
774 aq->cq.entries, aq->cq.dma_addr); in efa_com_admin_init()
776 dma_free_coherent(edev->dmadev, aq->depth * sizeof(*aq->sq.entries), in efa_com_admin_init()
777 aq->sq.entries, aq->sq.dma_addr); in efa_com_admin_init()
779 devm_kfree(edev->dmadev, aq->comp_ctx); in efa_com_admin_init()
797 spin_lock_irqsave(&edev->aq.cq.lock, flags); in efa_com_admin_q_comp_intr_handler()
798 efa_com_handle_admin_completion(&edev->aq); in efa_com_admin_q_comp_intr_handler()
799 spin_unlock_irqrestore(&edev->aq.cq.lock, flags); in efa_com_admin_q_comp_intr_handler()
1078 edev->aq.completion_timeout = timeout * 100000; in efa_com_dev_reset()
1080 edev->aq.completion_timeout = ADMIN_CMD_TIMEOUT_US; in efa_com_dev_reset()