1 // SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB
2 /* Copyright (c) 2019 Mellanox Technologies. */
3
4 #include <linux/interrupt.h>
5 #include <linux/notifier.h>
6 #include <linux/module.h>
7 #include <linux/mlx5/driver.h>
8 #include "mlx5_core.h"
9 #ifdef CONFIG_RFS_ACCEL
10 #include <linux/cpu_rmap.h>
11 #endif
12
13 #define MLX5_MAX_IRQ_NAME (32)
14
15 struct mlx5_irq {
16 struct atomic_notifier_head nh;
17 cpumask_var_t mask;
18 char name[MLX5_MAX_IRQ_NAME];
19 };
20
21 struct mlx5_irq_table {
22 struct mlx5_irq *irq;
23 int nvec;
24 #ifdef CONFIG_RFS_ACCEL
25 struct cpu_rmap *rmap;
26 #endif
27 };
28
mlx5_irq_table_init(struct mlx5_core_dev * dev)29 int mlx5_irq_table_init(struct mlx5_core_dev *dev)
30 {
31 struct mlx5_irq_table *irq_table;
32
33 if (mlx5_core_is_sf(dev))
34 return 0;
35
36 irq_table = kvzalloc(sizeof(*irq_table), GFP_KERNEL);
37 if (!irq_table)
38 return -ENOMEM;
39
40 dev->priv.irq_table = irq_table;
41 return 0;
42 }
43
mlx5_irq_table_cleanup(struct mlx5_core_dev * dev)44 void mlx5_irq_table_cleanup(struct mlx5_core_dev *dev)
45 {
46 if (mlx5_core_is_sf(dev))
47 return;
48
49 kvfree(dev->priv.irq_table);
50 }
51
mlx5_irq_get_num_comp(struct mlx5_irq_table * table)52 int mlx5_irq_get_num_comp(struct mlx5_irq_table *table)
53 {
54 return table->nvec - MLX5_IRQ_VEC_COMP_BASE;
55 }
56
mlx5_irq_get(struct mlx5_core_dev * dev,int vecidx)57 static struct mlx5_irq *mlx5_irq_get(struct mlx5_core_dev *dev, int vecidx)
58 {
59 struct mlx5_irq_table *irq_table = dev->priv.irq_table;
60
61 return &irq_table->irq[vecidx];
62 }
63
64 /**
65 * mlx5_get_default_msix_vec_count - Get the default number of MSI-X vectors
66 * to be ssigned to each VF.
67 * @dev: PF to work on
68 * @num_vfs: Number of enabled VFs
69 */
mlx5_get_default_msix_vec_count(struct mlx5_core_dev * dev,int num_vfs)70 int mlx5_get_default_msix_vec_count(struct mlx5_core_dev *dev, int num_vfs)
71 {
72 int num_vf_msix, min_msix, max_msix;
73
74 num_vf_msix = MLX5_CAP_GEN_MAX(dev, num_total_dynamic_vf_msix);
75 if (!num_vf_msix)
76 return 0;
77
78 min_msix = MLX5_CAP_GEN(dev, min_dynamic_vf_msix_table_size);
79 max_msix = MLX5_CAP_GEN(dev, max_dynamic_vf_msix_table_size);
80
81 /* Limit maximum number of MSI-X vectors so the default configuration
82 * has some available in the pool. This will allow the user to increase
83 * the number of vectors in a VF without having to first size-down other
84 * VFs.
85 */
86 return max(min(num_vf_msix / num_vfs, max_msix / 2), min_msix);
87 }
88
89 /**
90 * mlx5_set_msix_vec_count - Set dynamically allocated MSI-X on the VF
91 * @dev: PF to work on
92 * @function_id: Internal PCI VF function IDd
93 * @msix_vec_count: Number of MSI-X vectors to set
94 */
mlx5_set_msix_vec_count(struct mlx5_core_dev * dev,int function_id,int msix_vec_count)95 int mlx5_set_msix_vec_count(struct mlx5_core_dev *dev, int function_id,
96 int msix_vec_count)
97 {
98 int sz = MLX5_ST_SZ_BYTES(set_hca_cap_in);
99 int num_vf_msix, min_msix, max_msix;
100 void *hca_cap, *cap;
101 int ret;
102
103 num_vf_msix = MLX5_CAP_GEN_MAX(dev, num_total_dynamic_vf_msix);
104 if (!num_vf_msix)
105 return 0;
106
107 if (!MLX5_CAP_GEN(dev, vport_group_manager) || !mlx5_core_is_pf(dev))
108 return -EOPNOTSUPP;
109
110 min_msix = MLX5_CAP_GEN(dev, min_dynamic_vf_msix_table_size);
111 max_msix = MLX5_CAP_GEN(dev, max_dynamic_vf_msix_table_size);
112
113 if (msix_vec_count < min_msix)
114 return -EINVAL;
115
116 if (msix_vec_count > max_msix)
117 return -EOVERFLOW;
118
119 hca_cap = kzalloc(sz, GFP_KERNEL);
120 if (!hca_cap)
121 return -ENOMEM;
122
123 cap = MLX5_ADDR_OF(set_hca_cap_in, hca_cap, capability);
124 MLX5_SET(cmd_hca_cap, cap, dynamic_msix_table_size, msix_vec_count);
125
126 MLX5_SET(set_hca_cap_in, hca_cap, opcode, MLX5_CMD_OP_SET_HCA_CAP);
127 MLX5_SET(set_hca_cap_in, hca_cap, other_function, 1);
128 MLX5_SET(set_hca_cap_in, hca_cap, function_id, function_id);
129
130 MLX5_SET(set_hca_cap_in, hca_cap, op_mod,
131 MLX5_SET_HCA_CAP_OP_MOD_GENERAL_DEVICE << 1);
132 ret = mlx5_cmd_exec_in(dev, set_hca_cap, hca_cap);
133 kfree(hca_cap);
134 return ret;
135 }
136
mlx5_irq_attach_nb(struct mlx5_irq_table * irq_table,int vecidx,struct notifier_block * nb)137 int mlx5_irq_attach_nb(struct mlx5_irq_table *irq_table, int vecidx,
138 struct notifier_block *nb)
139 {
140 struct mlx5_irq *irq;
141
142 irq = &irq_table->irq[vecidx];
143 return atomic_notifier_chain_register(&irq->nh, nb);
144 }
145
mlx5_irq_detach_nb(struct mlx5_irq_table * irq_table,int vecidx,struct notifier_block * nb)146 int mlx5_irq_detach_nb(struct mlx5_irq_table *irq_table, int vecidx,
147 struct notifier_block *nb)
148 {
149 struct mlx5_irq *irq;
150
151 irq = &irq_table->irq[vecidx];
152 return atomic_notifier_chain_unregister(&irq->nh, nb);
153 }
154
mlx5_irq_int_handler(int irq,void * nh)155 static irqreturn_t mlx5_irq_int_handler(int irq, void *nh)
156 {
157 atomic_notifier_call_chain(nh, 0, NULL);
158 return IRQ_HANDLED;
159 }
160
irq_set_name(char * name,int vecidx)161 static void irq_set_name(char *name, int vecidx)
162 {
163 if (vecidx == 0) {
164 snprintf(name, MLX5_MAX_IRQ_NAME, "mlx5_async");
165 return;
166 }
167
168 snprintf(name, MLX5_MAX_IRQ_NAME, "mlx5_comp%d",
169 vecidx - MLX5_IRQ_VEC_COMP_BASE);
170 }
171
request_irqs(struct mlx5_core_dev * dev,int nvec)172 static int request_irqs(struct mlx5_core_dev *dev, int nvec)
173 {
174 char name[MLX5_MAX_IRQ_NAME];
175 int err;
176 int i;
177
178 for (i = 0; i < nvec; i++) {
179 struct mlx5_irq *irq = mlx5_irq_get(dev, i);
180 int irqn = pci_irq_vector(dev->pdev, i);
181
182 irq_set_name(name, i);
183 ATOMIC_INIT_NOTIFIER_HEAD(&irq->nh);
184 snprintf(irq->name, MLX5_MAX_IRQ_NAME,
185 "%s@pci:%s", name, pci_name(dev->pdev));
186 err = request_irq(irqn, mlx5_irq_int_handler, 0, irq->name,
187 &irq->nh);
188 if (err) {
189 mlx5_core_err(dev, "Failed to request irq\n");
190 goto err_request_irq;
191 }
192 }
193 return 0;
194
195 err_request_irq:
196 while (i--) {
197 struct mlx5_irq *irq = mlx5_irq_get(dev, i);
198 int irqn = pci_irq_vector(dev->pdev, i);
199
200 free_irq(irqn, &irq->nh);
201 }
202 return err;
203 }
204
irq_clear_rmap(struct mlx5_core_dev * dev)205 static void irq_clear_rmap(struct mlx5_core_dev *dev)
206 {
207 #ifdef CONFIG_RFS_ACCEL
208 struct mlx5_irq_table *irq_table = dev->priv.irq_table;
209
210 free_irq_cpu_rmap(irq_table->rmap);
211 #endif
212 }
213
irq_set_rmap(struct mlx5_core_dev * mdev)214 static int irq_set_rmap(struct mlx5_core_dev *mdev)
215 {
216 int err = 0;
217 #ifdef CONFIG_RFS_ACCEL
218 struct mlx5_irq_table *irq_table = mdev->priv.irq_table;
219 int num_affinity_vec;
220 int vecidx;
221
222 num_affinity_vec = mlx5_irq_get_num_comp(irq_table);
223 irq_table->rmap = alloc_irq_cpu_rmap(num_affinity_vec);
224 if (!irq_table->rmap) {
225 err = -ENOMEM;
226 mlx5_core_err(mdev, "Failed to allocate cpu_rmap. err %d", err);
227 goto err_out;
228 }
229
230 vecidx = MLX5_IRQ_VEC_COMP_BASE;
231 for (; vecidx < irq_table->nvec; vecidx++) {
232 err = irq_cpu_rmap_add(irq_table->rmap,
233 pci_irq_vector(mdev->pdev, vecidx));
234 if (err) {
235 mlx5_core_err(mdev, "irq_cpu_rmap_add failed. err %d",
236 err);
237 goto err_irq_cpu_rmap_add;
238 }
239 }
240 return 0;
241
242 err_irq_cpu_rmap_add:
243 irq_clear_rmap(mdev);
244 err_out:
245 #endif
246 return err;
247 }
248
249 /* Completion IRQ vectors */
250
set_comp_irq_affinity_hint(struct mlx5_core_dev * mdev,int i)251 static int set_comp_irq_affinity_hint(struct mlx5_core_dev *mdev, int i)
252 {
253 int vecidx = MLX5_IRQ_VEC_COMP_BASE + i;
254 struct mlx5_irq *irq;
255 int irqn;
256
257 irq = mlx5_irq_get(mdev, vecidx);
258 irqn = pci_irq_vector(mdev->pdev, vecidx);
259 if (!zalloc_cpumask_var(&irq->mask, GFP_KERNEL)) {
260 mlx5_core_warn(mdev, "zalloc_cpumask_var failed");
261 return -ENOMEM;
262 }
263
264 cpumask_set_cpu(cpumask_local_spread(i, mdev->priv.numa_node),
265 irq->mask);
266 if (IS_ENABLED(CONFIG_SMP) &&
267 irq_set_affinity_hint(irqn, irq->mask))
268 mlx5_core_warn(mdev, "irq_set_affinity_hint failed, irq 0x%.4x",
269 irqn);
270
271 return 0;
272 }
273
clear_comp_irq_affinity_hint(struct mlx5_core_dev * mdev,int i)274 static void clear_comp_irq_affinity_hint(struct mlx5_core_dev *mdev, int i)
275 {
276 int vecidx = MLX5_IRQ_VEC_COMP_BASE + i;
277 struct mlx5_irq *irq;
278 int irqn;
279
280 irq = mlx5_irq_get(mdev, vecidx);
281 irqn = pci_irq_vector(mdev->pdev, vecidx);
282 irq_set_affinity_hint(irqn, NULL);
283 free_cpumask_var(irq->mask);
284 }
285
set_comp_irq_affinity_hints(struct mlx5_core_dev * mdev)286 static int set_comp_irq_affinity_hints(struct mlx5_core_dev *mdev)
287 {
288 int nvec = mlx5_irq_get_num_comp(mdev->priv.irq_table);
289 int err;
290 int i;
291
292 for (i = 0; i < nvec; i++) {
293 err = set_comp_irq_affinity_hint(mdev, i);
294 if (err)
295 goto err_out;
296 }
297
298 return 0;
299
300 err_out:
301 for (i--; i >= 0; i--)
302 clear_comp_irq_affinity_hint(mdev, i);
303
304 return err;
305 }
306
clear_comp_irqs_affinity_hints(struct mlx5_core_dev * mdev)307 static void clear_comp_irqs_affinity_hints(struct mlx5_core_dev *mdev)
308 {
309 int nvec = mlx5_irq_get_num_comp(mdev->priv.irq_table);
310 int i;
311
312 for (i = 0; i < nvec; i++)
313 clear_comp_irq_affinity_hint(mdev, i);
314 }
315
316 struct cpumask *
mlx5_irq_get_affinity_mask(struct mlx5_irq_table * irq_table,int vecidx)317 mlx5_irq_get_affinity_mask(struct mlx5_irq_table *irq_table, int vecidx)
318 {
319 return irq_table->irq[vecidx].mask;
320 }
321
322 #ifdef CONFIG_RFS_ACCEL
mlx5_irq_get_rmap(struct mlx5_irq_table * irq_table)323 struct cpu_rmap *mlx5_irq_get_rmap(struct mlx5_irq_table *irq_table)
324 {
325 return irq_table->rmap;
326 }
327 #endif
328
unrequest_irqs(struct mlx5_core_dev * dev)329 static void unrequest_irqs(struct mlx5_core_dev *dev)
330 {
331 struct mlx5_irq_table *table = dev->priv.irq_table;
332 int i;
333
334 for (i = 0; i < table->nvec; i++)
335 free_irq(pci_irq_vector(dev->pdev, i),
336 &mlx5_irq_get(dev, i)->nh);
337 }
338
mlx5_irq_table_create(struct mlx5_core_dev * dev)339 int mlx5_irq_table_create(struct mlx5_core_dev *dev)
340 {
341 struct mlx5_priv *priv = &dev->priv;
342 struct mlx5_irq_table *table = priv->irq_table;
343 int num_eqs = MLX5_CAP_GEN(dev, max_num_eqs) ?
344 MLX5_CAP_GEN(dev, max_num_eqs) :
345 1 << MLX5_CAP_GEN(dev, log_max_eq);
346 int nvec;
347 int err;
348
349 if (mlx5_core_is_sf(dev))
350 return 0;
351
352 nvec = MLX5_CAP_GEN(dev, num_ports) * num_online_cpus() +
353 MLX5_IRQ_VEC_COMP_BASE;
354 nvec = min_t(int, nvec, num_eqs);
355 if (nvec <= MLX5_IRQ_VEC_COMP_BASE)
356 return -ENOMEM;
357
358 table->irq = kcalloc(nvec, sizeof(*table->irq), GFP_KERNEL);
359 if (!table->irq)
360 return -ENOMEM;
361
362 nvec = pci_alloc_irq_vectors(dev->pdev, MLX5_IRQ_VEC_COMP_BASE + 1,
363 nvec, PCI_IRQ_MSIX);
364 if (nvec < 0) {
365 err = nvec;
366 goto err_free_irq;
367 }
368
369 table->nvec = nvec;
370
371 err = irq_set_rmap(dev);
372 if (err)
373 goto err_set_rmap;
374
375 err = request_irqs(dev, nvec);
376 if (err)
377 goto err_request_irqs;
378
379 err = set_comp_irq_affinity_hints(dev);
380 if (err) {
381 mlx5_core_err(dev, "Failed to alloc affinity hint cpumask\n");
382 goto err_set_affinity;
383 }
384
385 return 0;
386
387 err_set_affinity:
388 unrequest_irqs(dev);
389 err_request_irqs:
390 irq_clear_rmap(dev);
391 err_set_rmap:
392 pci_free_irq_vectors(dev->pdev);
393 err_free_irq:
394 kfree(table->irq);
395 return err;
396 }
397
mlx5_irq_table_destroy(struct mlx5_core_dev * dev)398 void mlx5_irq_table_destroy(struct mlx5_core_dev *dev)
399 {
400 struct mlx5_irq_table *table = dev->priv.irq_table;
401 int i;
402
403 if (mlx5_core_is_sf(dev))
404 return;
405
406 /* free_irq requires that affinity and rmap will be cleared
407 * before calling it. This is why there is asymmetry with set_rmap
408 * which should be called after alloc_irq but before request_irq.
409 */
410 irq_clear_rmap(dev);
411 clear_comp_irqs_affinity_hints(dev);
412 for (i = 0; i < table->nvec; i++)
413 free_irq(pci_irq_vector(dev->pdev, i),
414 &mlx5_irq_get(dev, i)->nh);
415 pci_free_irq_vectors(dev->pdev);
416 kfree(table->irq);
417 }
418
mlx5_irq_table_get(struct mlx5_core_dev * dev)419 struct mlx5_irq_table *mlx5_irq_table_get(struct mlx5_core_dev *dev)
420 {
421 #ifdef CONFIG_MLX5_SF
422 if (mlx5_core_is_sf(dev))
423 return dev->priv.parent_mdev->priv.irq_table;
424 #endif
425 return dev->priv.irq_table;
426 }
427