1 // SPDX-License-Identifier: (GPL-2.0 OR BSD-3-Clause) 2 /* 3 * Copyright(c) 2018 Intel Corporation. 4 * 5 * This file is provided under a dual BSD/GPLv2 license. When using or 6 * redistributing this file, you may do so under either license. 7 * 8 * GPL LICENSE SUMMARY 9 * 10 * This program is free software; you can redistribute it and/or modify 11 * it under the terms of version 2 of the GNU General Public License as 12 * published by the Free Software Foundation. 13 * 14 * This program is distributed in the hope that it will be useful, but 15 * WITHOUT ANY WARRANTY; without even the implied warranty of 16 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU 17 * General Public License for more details. 18 * 19 * BSD LICENSE 20 * 21 * Redistribution and use in source and binary forms, with or without 22 * modification, are permitted provided that the following conditions 23 * are met: 24 * 25 * - Redistributions of source code must retain the above copyright 26 * notice, this list of conditions and the following disclaimer. 27 * - Redistributions in binary form must reproduce the above copyright 28 * notice, this list of conditions and the following disclaimer in 29 * the documentation and/or other materials provided with the 30 * distribution. 31 * - Neither the name of Intel Corporation nor the names of its 32 * contributors may be used to endorse or promote products derived 33 * from this software without specific prior written permission. 34 * 35 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 36 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 37 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 38 * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 39 * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 40 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 41 * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 42 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 43 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 44 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 45 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 46 * 47 */ 48 49 #include "hfi.h" 50 #include "affinity.h" 51 #include "sdma.h" 52 53 /** 54 * msix_initialize() - Calculate, request and configure MSIx IRQs 55 * @dd: valid hfi1 devdata 56 * 57 */ 58 int msix_initialize(struct hfi1_devdata *dd) 59 { 60 u32 total; 61 int ret; 62 struct hfi1_msix_entry *entries; 63 64 /* 65 * MSIx interrupt count: 66 * one for the general, "slow path" interrupt 67 * one per used SDMA engine 68 * one per kernel receive context 69 * one for each VNIC context 70 * ...any new IRQs should be added here. 71 */ 72 total = 1 + dd->num_sdma + dd->n_krcv_queues + dd->num_vnic_contexts; 73 74 if (total >= CCE_NUM_MSIX_VECTORS) 75 return -EINVAL; 76 77 ret = pci_alloc_irq_vectors(dd->pcidev, total, total, PCI_IRQ_MSIX); 78 if (ret < 0) { 79 dd_dev_err(dd, "pci_alloc_irq_vectors() failed: %d\n", ret); 80 return ret; 81 } 82 83 entries = kcalloc(total, sizeof(*dd->msix_info.msix_entries), 84 GFP_KERNEL); 85 if (!entries) { 86 pci_free_irq_vectors(dd->pcidev); 87 return -ENOMEM; 88 } 89 90 dd->msix_info.msix_entries = entries; 91 spin_lock_init(&dd->msix_info.msix_lock); 92 bitmap_zero(dd->msix_info.in_use_msix, total); 93 dd->msix_info.max_requested = total; 94 dd_dev_info(dd, "%u MSI-X interrupts allocated\n", total); 95 96 return 0; 97 } 98 99 /** 100 * msix_request_irq() - Allocate a free MSIx IRQ 101 * @dd: valid devdata 102 * @arg: context information for the IRQ 103 * @handler: IRQ handler 104 * @thread: IRQ thread handler (could be NULL) 105 * @idx: zero base idx if multiple devices are needed 106 * @type: affinty IRQ type 107 * 108 * Allocated an MSIx vector if available, and then create the appropriate 109 * meta data needed to keep track of the pci IRQ request. 110 * 111 * Return: 112 * < 0 Error 113 * >= 0 MSIx vector 114 * 115 */ 116 static int msix_request_irq(struct hfi1_devdata *dd, void *arg, 117 irq_handler_t handler, irq_handler_t thread, 118 enum irq_type type, const char *name) 119 { 120 unsigned long nr; 121 int irq; 122 int ret; 123 struct hfi1_msix_entry *me; 124 125 /* Allocate an MSIx vector */ 126 spin_lock(&dd->msix_info.msix_lock); 127 nr = find_first_zero_bit(dd->msix_info.in_use_msix, 128 dd->msix_info.max_requested); 129 if (nr < dd->msix_info.max_requested) 130 __set_bit(nr, dd->msix_info.in_use_msix); 131 spin_unlock(&dd->msix_info.msix_lock); 132 133 if (nr == dd->msix_info.max_requested) 134 return -ENOSPC; 135 136 if (type < IRQ_SDMA || type >= IRQ_OTHER) 137 return -EINVAL; 138 139 irq = pci_irq_vector(dd->pcidev, nr); 140 ret = pci_request_irq(dd->pcidev, nr, handler, thread, arg, name); 141 if (ret) { 142 dd_dev_err(dd, 143 "%s: request for IRQ %d failed, MSIx %lu, err %d\n", 144 name, irq, nr, ret); 145 spin_lock(&dd->msix_info.msix_lock); 146 __clear_bit(nr, dd->msix_info.in_use_msix); 147 spin_unlock(&dd->msix_info.msix_lock); 148 return ret; 149 } 150 151 /* 152 * assign arg after pci_request_irq call, so it will be 153 * cleaned up 154 */ 155 me = &dd->msix_info.msix_entries[nr]; 156 me->irq = irq; 157 me->arg = arg; 158 me->type = type; 159 160 /* This is a request, so a failure is not fatal */ 161 ret = hfi1_get_irq_affinity(dd, me); 162 if (ret) 163 dd_dev_err(dd, "unable to pin IRQ %d\n", ret); 164 165 return nr; 166 } 167 168 static int msix_request_rcd_irq_common(struct hfi1_ctxtdata *rcd, 169 irq_handler_t handler, 170 irq_handler_t thread, 171 const char *name) 172 { 173 int nr = msix_request_irq(rcd->dd, rcd, handler, thread, 174 IRQ_RCVCTXT, name); 175 if (nr < 0) 176 return nr; 177 178 /* 179 * Set the interrupt register and mask for this 180 * context's interrupt. 181 */ 182 rcd->ireg = (IS_RCVAVAIL_START + rcd->ctxt) / 64; 183 rcd->imask = ((u64)1) << ((IS_RCVAVAIL_START + rcd->ctxt) % 64); 184 rcd->msix_intr = nr; 185 remap_intr(rcd->dd, IS_RCVAVAIL_START + rcd->ctxt, nr); 186 187 return 0; 188 } 189 190 /** 191 * msix_request_rcd_irq() - Helper function for RCVAVAIL IRQs 192 * @rcd: valid rcd context 193 * 194 */ 195 int msix_request_rcd_irq(struct hfi1_ctxtdata *rcd) 196 { 197 char name[MAX_NAME_SIZE]; 198 199 snprintf(name, sizeof(name), DRIVER_NAME "_%d kctxt%d", 200 rcd->dd->unit, rcd->ctxt); 201 202 return msix_request_rcd_irq_common(rcd, receive_context_interrupt, 203 receive_context_thread, name); 204 } 205 206 /** 207 * msix_request_smda_ira() - Helper for getting SDMA IRQ resources 208 * @sde: valid sdma engine 209 * 210 */ 211 int msix_request_sdma_irq(struct sdma_engine *sde) 212 { 213 int nr; 214 char name[MAX_NAME_SIZE]; 215 216 snprintf(name, sizeof(name), DRIVER_NAME "_%d sdma%d", 217 sde->dd->unit, sde->this_idx); 218 nr = msix_request_irq(sde->dd, sde, sdma_interrupt, NULL, 219 IRQ_SDMA, name); 220 if (nr < 0) 221 return nr; 222 sde->msix_intr = nr; 223 remap_sdma_interrupts(sde->dd, sde->this_idx, nr); 224 225 return 0; 226 } 227 228 /** 229 * msix_request_general_irq(void) - Helper for getting general IRQ 230 * resources 231 * @dd: valid device data 232 */ 233 int msix_request_general_irq(struct hfi1_devdata *dd) 234 { 235 int nr; 236 char name[MAX_NAME_SIZE]; 237 238 snprintf(name, sizeof(name), DRIVER_NAME "_%d", dd->unit); 239 nr = msix_request_irq(dd, dd, general_interrupt, NULL, IRQ_GENERAL, 240 name); 241 if (nr < 0) 242 return nr; 243 244 /* general interrupt must be MSIx vector 0 */ 245 if (nr) { 246 msix_free_irq(dd, (u8)nr); 247 dd_dev_err(dd, "Invalid index %d for GENERAL IRQ\n", nr); 248 return -EINVAL; 249 } 250 251 return 0; 252 } 253 254 /** 255 * enable_sdma_src() - Helper to enable SDMA IRQ srcs 256 * @dd: valid devdata structure 257 * @i: index of SDMA engine 258 */ 259 static void enable_sdma_srcs(struct hfi1_devdata *dd, int i) 260 { 261 set_intr_bits(dd, IS_SDMA_START + i, IS_SDMA_START + i, true); 262 set_intr_bits(dd, IS_SDMA_PROGRESS_START + i, 263 IS_SDMA_PROGRESS_START + i, true); 264 set_intr_bits(dd, IS_SDMA_IDLE_START + i, IS_SDMA_IDLE_START + i, true); 265 set_intr_bits(dd, IS_SDMAENG_ERR_START + i, IS_SDMAENG_ERR_START + i, 266 true); 267 } 268 269 /** 270 * msix_request_irqs() - Allocate all MSIx IRQs 271 * @dd: valid devdata structure 272 * 273 * Helper function to request the used MSIx IRQs. 274 * 275 */ 276 int msix_request_irqs(struct hfi1_devdata *dd) 277 { 278 int i; 279 int ret = msix_request_general_irq(dd); 280 281 if (ret) 282 return ret; 283 284 for (i = 0; i < dd->num_sdma; i++) { 285 struct sdma_engine *sde = &dd->per_sdma[i]; 286 287 ret = msix_request_sdma_irq(sde); 288 if (ret) 289 return ret; 290 enable_sdma_srcs(sde->dd, i); 291 } 292 293 for (i = 0; i < dd->n_krcv_queues; i++) { 294 struct hfi1_ctxtdata *rcd = hfi1_rcd_get_by_index_safe(dd, i); 295 296 if (rcd) 297 ret = msix_request_rcd_irq(rcd); 298 hfi1_rcd_put(rcd); 299 if (ret) 300 return ret; 301 } 302 303 return 0; 304 } 305 306 /** 307 * msix_free_irq() - Free the specified MSIx resources and IRQ 308 * @dd: valid devdata 309 * @msix_intr: MSIx vector to free. 310 * 311 */ 312 void msix_free_irq(struct hfi1_devdata *dd, u8 msix_intr) 313 { 314 struct hfi1_msix_entry *me; 315 316 if (msix_intr >= dd->msix_info.max_requested) 317 return; 318 319 me = &dd->msix_info.msix_entries[msix_intr]; 320 321 if (!me->arg) /* => no irq, no affinity */ 322 return; 323 324 hfi1_put_irq_affinity(dd, me); 325 pci_free_irq(dd->pcidev, msix_intr, me->arg); 326 327 me->arg = NULL; 328 329 spin_lock(&dd->msix_info.msix_lock); 330 __clear_bit(msix_intr, dd->msix_info.in_use_msix); 331 spin_unlock(&dd->msix_info.msix_lock); 332 } 333 334 /** 335 * hfi1_clean_up_msix_interrupts() - Free all MSIx IRQ resources 336 * @dd: valid device data data structure 337 * 338 * Free the MSIx and associated PCI resources, if they have been allocated. 339 */ 340 void msix_clean_up_interrupts(struct hfi1_devdata *dd) 341 { 342 int i; 343 struct hfi1_msix_entry *me = dd->msix_info.msix_entries; 344 345 /* remove irqs - must happen before disabling/turning off */ 346 for (i = 0; i < dd->msix_info.max_requested; i++, me++) 347 msix_free_irq(dd, i); 348 349 /* clean structures */ 350 kfree(dd->msix_info.msix_entries); 351 dd->msix_info.msix_entries = NULL; 352 dd->msix_info.max_requested = 0; 353 354 pci_free_irq_vectors(dd->pcidev); 355 } 356 357 /** 358 * msix_vnic_syncrhonize_irq() - Vnic IRQ synchronize 359 * @dd: valid devdata 360 */ 361 void msix_vnic_synchronize_irq(struct hfi1_devdata *dd) 362 { 363 int i; 364 365 for (i = 0; i < dd->vnic.num_ctxt; i++) { 366 struct hfi1_ctxtdata *rcd = dd->vnic.ctxt[i]; 367 struct hfi1_msix_entry *me; 368 369 me = &dd->msix_info.msix_entries[rcd->msix_intr]; 370 371 synchronize_irq(me->irq); 372 } 373 } 374