1 // SPDX-License-Identifier: GPL-2.0 2 /* Copyright(c) 2023 Advanced Micro Devices, Inc */ 3 4 #include <linux/dynamic_debug.h> 5 6 #include "core.h" 7 8 struct pdsc_wait_context { 9 struct pdsc_qcq *qcq; 10 struct completion wait_completion; 11 }; 12 13 static int pdsc_process_notifyq(struct pdsc_qcq *qcq) 14 { 15 union pds_core_notifyq_comp *comp; 16 struct pdsc *pdsc = qcq->pdsc; 17 struct pdsc_cq *cq = &qcq->cq; 18 struct pdsc_cq_info *cq_info; 19 int nq_work = 0; 20 u64 eid; 21 22 cq_info = &cq->info[cq->tail_idx]; 23 comp = cq_info->comp; 24 eid = le64_to_cpu(comp->event.eid); 25 while (eid > pdsc->last_eid) { 26 u16 ecode = le16_to_cpu(comp->event.ecode); 27 28 switch (ecode) { 29 case PDS_EVENT_LINK_CHANGE: 30 dev_info(pdsc->dev, "NotifyQ LINK_CHANGE ecode %d eid %lld\n", 31 ecode, eid); 32 pdsc_notify(PDS_EVENT_LINK_CHANGE, comp); 33 break; 34 35 case PDS_EVENT_RESET: 36 dev_info(pdsc->dev, "NotifyQ RESET ecode %d eid %lld\n", 37 ecode, eid); 38 pdsc_notify(PDS_EVENT_RESET, comp); 39 break; 40 41 case PDS_EVENT_XCVR: 42 dev_info(pdsc->dev, "NotifyQ XCVR ecode %d eid %lld\n", 43 ecode, eid); 44 break; 45 46 default: 47 dev_info(pdsc->dev, "NotifyQ ecode %d eid %lld\n", 48 ecode, eid); 49 break; 50 } 51 52 pdsc->last_eid = eid; 53 cq->tail_idx = (cq->tail_idx + 1) & (cq->num_descs - 1); 54 cq_info = &cq->info[cq->tail_idx]; 55 comp = cq_info->comp; 56 eid = le64_to_cpu(comp->event.eid); 57 58 nq_work++; 59 } 60 61 qcq->accum_work += nq_work; 62 63 return nq_work; 64 } 65 66 void pdsc_process_adminq(struct pdsc_qcq *qcq) 67 { 68 union pds_core_adminq_comp *comp; 69 struct pdsc_queue *q = &qcq->q; 70 struct pdsc *pdsc = qcq->pdsc; 71 struct pdsc_cq *cq = &qcq->cq; 72 struct pdsc_q_info *q_info; 73 unsigned long irqflags; 74 int nq_work = 0; 75 int aq_work = 0; 76 int credits; 77 78 /* Don't process AdminQ when shutting down */ 79 if (pdsc->state & BIT_ULL(PDSC_S_STOPPING_DRIVER)) { 80 dev_err(pdsc->dev, "%s: called while PDSC_S_STOPPING_DRIVER\n", 81 __func__); 82 return; 83 } 84 85 /* Check for NotifyQ event */ 86 nq_work = pdsc_process_notifyq(&pdsc->notifyqcq); 87 88 /* Check for empty queue, which can happen if the interrupt was 89 * for a NotifyQ event and there are no new AdminQ completions. 90 */ 91 if (q->tail_idx == q->head_idx) 92 goto credits; 93 94 /* Find the first completion to clean, 95 * run the callback in the related q_info, 96 * and continue while we still match done color 97 */ 98 spin_lock_irqsave(&pdsc->adminq_lock, irqflags); 99 comp = cq->info[cq->tail_idx].comp; 100 while (pdsc_color_match(comp->color, cq->done_color)) { 101 q_info = &q->info[q->tail_idx]; 102 q->tail_idx = (q->tail_idx + 1) & (q->num_descs - 1); 103 104 /* Copy out the completion data */ 105 memcpy(q_info->dest, comp, sizeof(*comp)); 106 107 complete_all(&q_info->wc->wait_completion); 108 109 if (cq->tail_idx == cq->num_descs - 1) 110 cq->done_color = !cq->done_color; 111 cq->tail_idx = (cq->tail_idx + 1) & (cq->num_descs - 1); 112 comp = cq->info[cq->tail_idx].comp; 113 114 aq_work++; 115 } 116 spin_unlock_irqrestore(&pdsc->adminq_lock, irqflags); 117 118 qcq->accum_work += aq_work; 119 120 credits: 121 /* Return the interrupt credits, one for each completion */ 122 credits = nq_work + aq_work; 123 if (credits) 124 pds_core_intr_credits(&pdsc->intr_ctrl[qcq->intx], 125 credits, 126 PDS_CORE_INTR_CRED_REARM); 127 } 128 129 void pdsc_work_thread(struct work_struct *work) 130 { 131 struct pdsc_qcq *qcq = container_of(work, struct pdsc_qcq, work); 132 133 pdsc_process_adminq(qcq); 134 } 135 136 irqreturn_t pdsc_adminq_isr(int irq, void *data) 137 { 138 struct pdsc_qcq *qcq = data; 139 struct pdsc *pdsc = qcq->pdsc; 140 141 /* Don't process AdminQ when shutting down */ 142 if (pdsc->state & BIT_ULL(PDSC_S_STOPPING_DRIVER)) { 143 dev_err(pdsc->dev, "%s: called while PDSC_S_STOPPING_DRIVER\n", 144 __func__); 145 return IRQ_HANDLED; 146 } 147 148 queue_work(pdsc->wq, &qcq->work); 149 pds_core_intr_mask(&pdsc->intr_ctrl[qcq->intx], PDS_CORE_INTR_MASK_CLEAR); 150 151 return IRQ_HANDLED; 152 } 153 154 static int __pdsc_adminq_post(struct pdsc *pdsc, 155 struct pdsc_qcq *qcq, 156 union pds_core_adminq_cmd *cmd, 157 union pds_core_adminq_comp *comp, 158 struct pdsc_wait_context *wc) 159 { 160 struct pdsc_queue *q = &qcq->q; 161 struct pdsc_q_info *q_info; 162 unsigned long irqflags; 163 unsigned int avail; 164 int index; 165 int ret; 166 167 spin_lock_irqsave(&pdsc->adminq_lock, irqflags); 168 169 /* Check for space in the queue */ 170 avail = q->tail_idx; 171 if (q->head_idx >= avail) 172 avail += q->num_descs - q->head_idx - 1; 173 else 174 avail -= q->head_idx + 1; 175 if (!avail) { 176 ret = -ENOSPC; 177 goto err_out_unlock; 178 } 179 180 /* Check that the FW is running */ 181 if (!pdsc_is_fw_running(pdsc)) { 182 u8 fw_status = ioread8(&pdsc->info_regs->fw_status); 183 184 dev_info(pdsc->dev, "%s: post failed - fw not running %#02x:\n", 185 __func__, fw_status); 186 ret = -ENXIO; 187 188 goto err_out_unlock; 189 } 190 191 /* Post the request */ 192 index = q->head_idx; 193 q_info = &q->info[index]; 194 q_info->wc = wc; 195 q_info->dest = comp; 196 memcpy(q_info->desc, cmd, sizeof(*cmd)); 197 198 dev_dbg(pdsc->dev, "head_idx %d tail_idx %d\n", 199 q->head_idx, q->tail_idx); 200 dev_dbg(pdsc->dev, "post admin queue command:\n"); 201 dynamic_hex_dump("cmd ", DUMP_PREFIX_OFFSET, 16, 1, 202 cmd, sizeof(*cmd), true); 203 204 q->head_idx = (q->head_idx + 1) & (q->num_descs - 1); 205 206 pds_core_dbell_ring(pdsc->kern_dbpage, 207 q->hw_type, q->dbval | q->head_idx); 208 ret = index; 209 210 err_out_unlock: 211 spin_unlock_irqrestore(&pdsc->adminq_lock, irqflags); 212 return ret; 213 } 214 215 int pdsc_adminq_post(struct pdsc *pdsc, 216 union pds_core_adminq_cmd *cmd, 217 union pds_core_adminq_comp *comp, 218 bool fast_poll) 219 { 220 struct pdsc_wait_context wc = { 221 .wait_completion = 222 COMPLETION_INITIALIZER_ONSTACK(wc.wait_completion), 223 }; 224 unsigned long poll_interval = 1; 225 unsigned long poll_jiffies; 226 unsigned long time_limit; 227 unsigned long time_start; 228 unsigned long time_done; 229 unsigned long remaining; 230 int err = 0; 231 int index; 232 233 wc.qcq = &pdsc->adminqcq; 234 index = __pdsc_adminq_post(pdsc, &pdsc->adminqcq, cmd, comp, &wc); 235 if (index < 0) { 236 err = index; 237 goto err_out; 238 } 239 240 time_start = jiffies; 241 time_limit = time_start + HZ * pdsc->devcmd_timeout; 242 do { 243 /* Timeslice the actual wait to catch IO errors etc early */ 244 poll_jiffies = msecs_to_jiffies(poll_interval); 245 remaining = wait_for_completion_timeout(&wc.wait_completion, 246 poll_jiffies); 247 if (remaining) 248 break; 249 250 if (!pdsc_is_fw_running(pdsc)) { 251 u8 fw_status = ioread8(&pdsc->info_regs->fw_status); 252 253 dev_dbg(pdsc->dev, "%s: post wait failed - fw not running %#02x:\n", 254 __func__, fw_status); 255 err = -ENXIO; 256 break; 257 } 258 259 /* When fast_poll is not requested, prevent aggressive polling 260 * on failures due to timeouts by doing exponential back off. 261 */ 262 if (!fast_poll && poll_interval < PDSC_ADMINQ_MAX_POLL_INTERVAL) 263 poll_interval <<= 1; 264 } while (time_before(jiffies, time_limit)); 265 time_done = jiffies; 266 dev_dbg(pdsc->dev, "%s: elapsed %d msecs\n", 267 __func__, jiffies_to_msecs(time_done - time_start)); 268 269 /* Check the results */ 270 if (time_after_eq(time_done, time_limit)) 271 err = -ETIMEDOUT; 272 273 dev_dbg(pdsc->dev, "read admin queue completion idx %d:\n", index); 274 dynamic_hex_dump("comp ", DUMP_PREFIX_OFFSET, 16, 1, 275 comp, sizeof(*comp), true); 276 277 if (remaining && comp->status) 278 err = pdsc_err_to_errno(comp->status); 279 280 err_out: 281 if (err) { 282 dev_dbg(pdsc->dev, "%s: opcode %d status %d err %pe\n", 283 __func__, cmd->opcode, comp->status, ERR_PTR(err)); 284 if (err == -ENXIO || err == -ETIMEDOUT) 285 queue_work(pdsc->wq, &pdsc->health_work); 286 } 287 288 return err; 289 } 290 EXPORT_SYMBOL_GPL(pdsc_adminq_post); 291