1 // SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause 2 3 /* Authors: Cheng Xu <chengyou@linux.alibaba.com> */ 4 /* Kai Shen <kaishen@linux.alibaba.com> */ 5 /* Copyright (c) 2020-2022, Alibaba Group. */ 6 7 #include "erdma_verbs.h" 8 9 #define MAX_POLL_CHUNK_SIZE 16 10 11 void notify_eq(struct erdma_eq *eq) 12 { 13 u64 db_data = FIELD_PREP(ERDMA_EQDB_CI_MASK, eq->ci) | 14 FIELD_PREP(ERDMA_EQDB_ARM_MASK, 1); 15 16 *eq->db_record = db_data; 17 writeq(db_data, eq->db_addr); 18 19 atomic64_inc(&eq->notify_num); 20 } 21 22 void *get_next_valid_eqe(struct erdma_eq *eq) 23 { 24 u64 *eqe = get_queue_entry(eq->qbuf, eq->ci, eq->depth, EQE_SHIFT); 25 u32 owner = FIELD_GET(ERDMA_CEQE_HDR_O_MASK, READ_ONCE(*eqe)); 26 27 return owner ^ !!(eq->ci & eq->depth) ? eqe : NULL; 28 } 29 30 void erdma_aeq_event_handler(struct erdma_dev *dev) 31 { 32 struct erdma_aeqe *aeqe; 33 u32 cqn, qpn; 34 struct erdma_qp *qp; 35 struct erdma_cq *cq; 36 struct ib_event event; 37 u32 poll_cnt = 0; 38 39 memset(&event, 0, sizeof(event)); 40 41 while (poll_cnt < MAX_POLL_CHUNK_SIZE) { 42 aeqe = get_next_valid_eqe(&dev->aeq); 43 if (!aeqe) 44 break; 45 46 dma_rmb(); 47 48 dev->aeq.ci++; 49 atomic64_inc(&dev->aeq.event_num); 50 poll_cnt++; 51 52 if (FIELD_GET(ERDMA_AEQE_HDR_TYPE_MASK, 53 le32_to_cpu(aeqe->hdr)) == ERDMA_AE_TYPE_CQ_ERR) { 54 cqn = le32_to_cpu(aeqe->event_data0); 55 cq = find_cq_by_cqn(dev, cqn); 56 if (!cq) 57 continue; 58 59 event.device = cq->ibcq.device; 60 event.element.cq = &cq->ibcq; 61 event.event = IB_EVENT_CQ_ERR; 62 if (cq->ibcq.event_handler) 63 cq->ibcq.event_handler(&event, 64 cq->ibcq.cq_context); 65 } else { 66 qpn = le32_to_cpu(aeqe->event_data0); 67 qp = find_qp_by_qpn(dev, qpn); 68 if (!qp) 69 continue; 70 71 event.device = qp->ibqp.device; 72 event.element.qp = &qp->ibqp; 73 event.event = IB_EVENT_QP_FATAL; 74 if (qp->ibqp.event_handler) 75 qp->ibqp.event_handler(&event, 76 qp->ibqp.qp_context); 77 } 78 } 79 80 notify_eq(&dev->aeq); 81 } 82 83 int erdma_aeq_init(struct erdma_dev *dev) 84 { 85 struct erdma_eq *eq = &dev->aeq; 86 u32 buf_size; 87 88 eq->depth = ERDMA_DEFAULT_EQ_DEPTH; 89 buf_size = eq->depth << EQE_SHIFT; 90 91 eq->qbuf = 92 dma_alloc_coherent(&dev->pdev->dev, WARPPED_BUFSIZE(buf_size), 93 &eq->qbuf_dma_addr, GFP_KERNEL | __GFP_ZERO); 94 if (!eq->qbuf) 95 return -ENOMEM; 96 97 spin_lock_init(&eq->lock); 98 atomic64_set(&eq->event_num, 0); 99 atomic64_set(&eq->notify_num, 0); 100 101 eq->db_addr = (u64 __iomem *)(dev->func_bar + ERDMA_REGS_AEQ_DB_REG); 102 eq->db_record = (u64 *)(eq->qbuf + buf_size); 103 104 erdma_reg_write32(dev, ERDMA_REGS_AEQ_ADDR_H_REG, 105 upper_32_bits(eq->qbuf_dma_addr)); 106 erdma_reg_write32(dev, ERDMA_REGS_AEQ_ADDR_L_REG, 107 lower_32_bits(eq->qbuf_dma_addr)); 108 erdma_reg_write32(dev, ERDMA_REGS_AEQ_DEPTH_REG, eq->depth); 109 erdma_reg_write64(dev, ERDMA_AEQ_DB_HOST_ADDR_REG, 110 eq->qbuf_dma_addr + buf_size); 111 112 return 0; 113 } 114 115 void erdma_aeq_destroy(struct erdma_dev *dev) 116 { 117 struct erdma_eq *eq = &dev->aeq; 118 119 dma_free_coherent(&dev->pdev->dev, 120 WARPPED_BUFSIZE(eq->depth << EQE_SHIFT), eq->qbuf, 121 eq->qbuf_dma_addr); 122 } 123 124 void erdma_ceq_completion_handler(struct erdma_eq_cb *ceq_cb) 125 { 126 struct erdma_dev *dev = ceq_cb->dev; 127 struct erdma_cq *cq; 128 u32 poll_cnt = 0; 129 u64 *ceqe; 130 int cqn; 131 132 if (!ceq_cb->ready) 133 return; 134 135 while (poll_cnt < MAX_POLL_CHUNK_SIZE) { 136 ceqe = get_next_valid_eqe(&ceq_cb->eq); 137 if (!ceqe) 138 break; 139 140 dma_rmb(); 141 ceq_cb->eq.ci++; 142 poll_cnt++; 143 cqn = FIELD_GET(ERDMA_CEQE_HDR_CQN_MASK, READ_ONCE(*ceqe)); 144 145 cq = find_cq_by_cqn(dev, cqn); 146 if (!cq) 147 continue; 148 149 if (rdma_is_kernel_res(&cq->ibcq.res)) 150 cq->kern_cq.cmdsn++; 151 152 if (cq->ibcq.comp_handler) 153 cq->ibcq.comp_handler(&cq->ibcq, cq->ibcq.cq_context); 154 } 155 156 notify_eq(&ceq_cb->eq); 157 } 158 159 static irqreturn_t erdma_intr_ceq_handler(int irq, void *data) 160 { 161 struct erdma_eq_cb *ceq_cb = data; 162 163 tasklet_schedule(&ceq_cb->tasklet); 164 165 return IRQ_HANDLED; 166 } 167 168 static void erdma_intr_ceq_task(unsigned long data) 169 { 170 erdma_ceq_completion_handler((struct erdma_eq_cb *)data); 171 } 172 173 static int erdma_set_ceq_irq(struct erdma_dev *dev, u16 ceqn) 174 { 175 struct erdma_eq_cb *eqc = &dev->ceqs[ceqn]; 176 int err; 177 178 snprintf(eqc->irq.name, ERDMA_IRQNAME_SIZE, "erdma-ceq%u@pci:%s", ceqn, 179 pci_name(dev->pdev)); 180 eqc->irq.msix_vector = pci_irq_vector(dev->pdev, ceqn + 1); 181 182 tasklet_init(&dev->ceqs[ceqn].tasklet, erdma_intr_ceq_task, 183 (unsigned long)&dev->ceqs[ceqn]); 184 185 cpumask_set_cpu(cpumask_local_spread(ceqn + 1, dev->attrs.numa_node), 186 &eqc->irq.affinity_hint_mask); 187 188 err = request_irq(eqc->irq.msix_vector, erdma_intr_ceq_handler, 0, 189 eqc->irq.name, eqc); 190 if (err) { 191 dev_err(&dev->pdev->dev, "failed to request_irq(%d)\n", err); 192 return err; 193 } 194 195 irq_set_affinity_hint(eqc->irq.msix_vector, 196 &eqc->irq.affinity_hint_mask); 197 198 return 0; 199 } 200 201 static void erdma_free_ceq_irq(struct erdma_dev *dev, u16 ceqn) 202 { 203 struct erdma_eq_cb *eqc = &dev->ceqs[ceqn]; 204 205 irq_set_affinity_hint(eqc->irq.msix_vector, NULL); 206 free_irq(eqc->irq.msix_vector, eqc); 207 } 208 209 static int create_eq_cmd(struct erdma_dev *dev, u32 eqn, struct erdma_eq *eq) 210 { 211 struct erdma_cmdq_create_eq_req req; 212 dma_addr_t db_info_dma_addr; 213 214 erdma_cmdq_build_reqhdr(&req.hdr, CMDQ_SUBMOD_COMMON, 215 CMDQ_OPCODE_CREATE_EQ); 216 req.eqn = eqn; 217 req.depth = ilog2(eq->depth); 218 req.qbuf_addr = eq->qbuf_dma_addr; 219 req.qtype = ERDMA_EQ_TYPE_CEQ; 220 /* Vector index is the same as EQN. */ 221 req.vector_idx = eqn; 222 db_info_dma_addr = eq->qbuf_dma_addr + (eq->depth << EQE_SHIFT); 223 req.db_dma_addr_l = lower_32_bits(db_info_dma_addr); 224 req.db_dma_addr_h = upper_32_bits(db_info_dma_addr); 225 226 return erdma_post_cmd_wait(&dev->cmdq, &req, sizeof(req), NULL, NULL); 227 } 228 229 static int erdma_ceq_init_one(struct erdma_dev *dev, u16 ceqn) 230 { 231 struct erdma_eq *eq = &dev->ceqs[ceqn].eq; 232 u32 buf_size = ERDMA_DEFAULT_EQ_DEPTH << EQE_SHIFT; 233 int ret; 234 235 eq->qbuf = 236 dma_alloc_coherent(&dev->pdev->dev, WARPPED_BUFSIZE(buf_size), 237 &eq->qbuf_dma_addr, GFP_KERNEL | __GFP_ZERO); 238 if (!eq->qbuf) 239 return -ENOMEM; 240 241 spin_lock_init(&eq->lock); 242 atomic64_set(&eq->event_num, 0); 243 atomic64_set(&eq->notify_num, 0); 244 245 eq->depth = ERDMA_DEFAULT_EQ_DEPTH; 246 eq->db_addr = 247 (u64 __iomem *)(dev->func_bar + ERDMA_REGS_CEQ_DB_BASE_REG + 248 (ceqn + 1) * ERDMA_DB_SIZE); 249 eq->db_record = (u64 *)(eq->qbuf + buf_size); 250 eq->ci = 0; 251 dev->ceqs[ceqn].dev = dev; 252 253 /* CEQ indexed from 1, 0 rsvd for CMDQ-EQ. */ 254 ret = create_eq_cmd(dev, ceqn + 1, eq); 255 dev->ceqs[ceqn].ready = ret ? false : true; 256 257 return ret; 258 } 259 260 static void erdma_ceq_uninit_one(struct erdma_dev *dev, u16 ceqn) 261 { 262 struct erdma_eq *eq = &dev->ceqs[ceqn].eq; 263 u32 buf_size = ERDMA_DEFAULT_EQ_DEPTH << EQE_SHIFT; 264 struct erdma_cmdq_destroy_eq_req req; 265 int err; 266 267 dev->ceqs[ceqn].ready = 0; 268 269 erdma_cmdq_build_reqhdr(&req.hdr, CMDQ_SUBMOD_COMMON, 270 CMDQ_OPCODE_DESTROY_EQ); 271 /* CEQ indexed from 1, 0 rsvd for CMDQ-EQ. */ 272 req.eqn = ceqn + 1; 273 req.qtype = ERDMA_EQ_TYPE_CEQ; 274 req.vector_idx = ceqn + 1; 275 276 err = erdma_post_cmd_wait(&dev->cmdq, &req, sizeof(req), NULL, NULL); 277 if (err) 278 return; 279 280 dma_free_coherent(&dev->pdev->dev, WARPPED_BUFSIZE(buf_size), eq->qbuf, 281 eq->qbuf_dma_addr); 282 } 283 284 int erdma_ceqs_init(struct erdma_dev *dev) 285 { 286 u32 i, j; 287 int err; 288 289 for (i = 0; i < dev->attrs.irq_num - 1; i++) { 290 err = erdma_ceq_init_one(dev, i); 291 if (err) 292 goto out_err; 293 294 err = erdma_set_ceq_irq(dev, i); 295 if (err) { 296 erdma_ceq_uninit_one(dev, i); 297 goto out_err; 298 } 299 } 300 301 return 0; 302 303 out_err: 304 for (j = 0; j < i; j++) { 305 erdma_free_ceq_irq(dev, j); 306 erdma_ceq_uninit_one(dev, j); 307 } 308 309 return err; 310 } 311 312 void erdma_ceqs_uninit(struct erdma_dev *dev) 313 { 314 u32 i; 315 316 for (i = 0; i < dev->attrs.irq_num - 1; i++) { 317 erdma_free_ceq_irq(dev, i); 318 erdma_ceq_uninit_one(dev, i); 319 } 320 } 321