1 /*-
2 * SPDX-License-Identifier: BSD-2-Clause
3 *
4 * Copyright (c) 2017, Bryan Venteicher <bryanv@FreeBSD.org>
5 * All rights reserved.
6 *
7 * Redistribution and use in source and binary forms, with or without
8 * modification, are permitted provided that the following conditions
9 * are met:
10 * 1. Redistributions of source code must retain the above copyright
11 * notice unmodified, this list of conditions, and the following
12 * disclaimer.
13 * 2. Redistributions in binary form must reproduce the above copyright
14 * notice, this list of conditions and the following disclaimer in the
15 * documentation and/or other materials provided with the distribution.
16 *
17 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
18 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
19 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
20 * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
21 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
22 * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
23 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
24 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
25 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
26 * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
27 */
28
29 #include <sys/param.h>
30 #include <sys/systm.h>
31 #include <sys/bus.h>
32 #include <sys/kernel.h>
33 #include <sys/sbuf.h>
34 #include <sys/sysctl.h>
35 #include <sys/module.h>
36 #include <sys/malloc.h>
37
38 #include <machine/bus.h>
39 #include <machine/resource.h>
40 #include <sys/bus.h>
41 #include <sys/rman.h>
42
43 #include <dev/pci/pcivar.h>
44 #include <dev/pci/pcireg.h>
45
46 #include <dev/virtio/virtio.h>
47 #include <dev/virtio/virtqueue.h>
48 #include <dev/virtio/pci/virtio_pci.h>
49 #include <dev/virtio/pci/virtio_pci_var.h>
50
51 #include "virtio_pci_if.h"
52 #include "virtio_if.h"
53
54 static void vtpci_describe_features(struct vtpci_common *, const char *,
55 uint64_t);
56 static int vtpci_alloc_msix(struct vtpci_common *, int);
57 static int vtpci_alloc_msi(struct vtpci_common *);
58 static int vtpci_alloc_intr_msix_pervq(struct vtpci_common *);
59 static int vtpci_alloc_intr_msix_shared(struct vtpci_common *);
60 static int vtpci_alloc_intr_msi(struct vtpci_common *);
61 static int vtpci_alloc_intr_intx(struct vtpci_common *);
62 static int vtpci_alloc_interrupt(struct vtpci_common *, int, int,
63 struct vtpci_interrupt *);
64 static void vtpci_free_interrupt(struct vtpci_common *,
65 struct vtpci_interrupt *);
66
67 static void vtpci_free_interrupts(struct vtpci_common *);
68 static void vtpci_free_virtqueues(struct vtpci_common *);
69 static void vtpci_cleanup_setup_intr_attempt(struct vtpci_common *);
70 static int vtpci_alloc_intr_resources(struct vtpci_common *);
71 static int vtpci_setup_intx_interrupt(struct vtpci_common *,
72 enum intr_type);
73 static int vtpci_setup_pervq_msix_interrupts(struct vtpci_common *,
74 enum intr_type);
75 static int vtpci_set_host_msix_vectors(struct vtpci_common *);
76 static int vtpci_setup_msix_interrupts(struct vtpci_common *,
77 enum intr_type);
78 static int vtpci_setup_intrs(struct vtpci_common *, enum intr_type);
79 static int vtpci_reinit_virtqueue(struct vtpci_common *, int);
80 static void vtpci_intx_intr(void *);
81 static int vtpci_vq_shared_intr_filter(void *);
82 static void vtpci_vq_shared_intr(void *);
83 static int vtpci_vq_intr_filter(void *);
84 static void vtpci_vq_intr(void *);
85 static void vtpci_config_intr(void *);
86
87 static void vtpci_setup_sysctl(struct vtpci_common *);
88
89 #define vtpci_setup_msi_interrupt vtpci_setup_intx_interrupt
90
91 /*
92 * This module contains two drivers:
93 * - virtio_pci_legacy for pre-V1 support
94 * - virtio_pci_modern for V1 support
95 */
96 MODULE_VERSION(virtio_pci, 1);
97 MODULE_DEPEND(virtio_pci, pci, 1, 1, 1);
98 MODULE_DEPEND(virtio_pci, virtio, 1, 1, 1);
99
100 int vtpci_disable_msix = 0;
101 TUNABLE_INT("hw.virtio.pci.disable_msix", &vtpci_disable_msix);
102
103 static uint8_t
vtpci_read_isr(struct vtpci_common * cn)104 vtpci_read_isr(struct vtpci_common *cn)
105 {
106 return (VIRTIO_PCI_READ_ISR(cn->vtpci_dev));
107 }
108
109 static uint16_t
vtpci_get_vq_size(struct vtpci_common * cn,int idx)110 vtpci_get_vq_size(struct vtpci_common *cn, int idx)
111 {
112 return (VIRTIO_PCI_GET_VQ_SIZE(cn->vtpci_dev, idx));
113 }
114
115 static bus_size_t
vtpci_get_vq_notify_off(struct vtpci_common * cn,int idx)116 vtpci_get_vq_notify_off(struct vtpci_common *cn, int idx)
117 {
118 return (VIRTIO_PCI_GET_VQ_NOTIFY_OFF(cn->vtpci_dev, idx));
119 }
120
121 static void
vtpci_set_vq(struct vtpci_common * cn,struct virtqueue * vq)122 vtpci_set_vq(struct vtpci_common *cn, struct virtqueue *vq)
123 {
124 VIRTIO_PCI_SET_VQ(cn->vtpci_dev, vq);
125 }
126
127 static void
vtpci_disable_vq(struct vtpci_common * cn,int idx)128 vtpci_disable_vq(struct vtpci_common *cn, int idx)
129 {
130 VIRTIO_PCI_DISABLE_VQ(cn->vtpci_dev, idx);
131 }
132
133 static int
vtpci_register_cfg_msix(struct vtpci_common * cn,struct vtpci_interrupt * intr)134 vtpci_register_cfg_msix(struct vtpci_common *cn, struct vtpci_interrupt *intr)
135 {
136 return (VIRTIO_PCI_REGISTER_CFG_MSIX(cn->vtpci_dev, intr));
137 }
138
139 static int
vtpci_register_vq_msix(struct vtpci_common * cn,int idx,struct vtpci_interrupt * intr)140 vtpci_register_vq_msix(struct vtpci_common *cn, int idx,
141 struct vtpci_interrupt *intr)
142 {
143 return (VIRTIO_PCI_REGISTER_VQ_MSIX(cn->vtpci_dev, idx, intr));
144 }
145
146 void
vtpci_init(struct vtpci_common * cn,device_t dev,bool modern)147 vtpci_init(struct vtpci_common *cn, device_t dev, bool modern)
148 {
149
150 cn->vtpci_dev = dev;
151
152 pci_enable_busmaster(dev);
153
154 if (modern)
155 cn->vtpci_flags |= VTPCI_FLAG_MODERN;
156 if (pci_find_cap(dev, PCIY_MSI, NULL) != 0)
157 cn->vtpci_flags |= VTPCI_FLAG_NO_MSI;
158 if (pci_find_cap(dev, PCIY_MSIX, NULL) != 0)
159 cn->vtpci_flags |= VTPCI_FLAG_NO_MSIX;
160
161 vtpci_setup_sysctl(cn);
162 }
163
164 int
vtpci_add_child(struct vtpci_common * cn)165 vtpci_add_child(struct vtpci_common *cn)
166 {
167 device_t dev, child;
168
169 dev = cn->vtpci_dev;
170
171 child = device_add_child(dev, NULL, DEVICE_UNIT_ANY);
172 if (child == NULL) {
173 device_printf(dev, "cannot create child device\n");
174 return (ENOMEM);
175 }
176
177 cn->vtpci_child_dev = child;
178
179 return (0);
180 }
181
182 int
vtpci_delete_child(struct vtpci_common * cn)183 vtpci_delete_child(struct vtpci_common *cn)
184 {
185 device_t dev, child;
186 int error;
187
188 dev = cn->vtpci_dev;
189
190 child = cn->vtpci_child_dev;
191 if (child != NULL) {
192 error = device_delete_child(dev, child);
193 if (error)
194 return (error);
195 cn->vtpci_child_dev = NULL;
196 }
197
198 return (0);
199 }
200
201 void
vtpci_child_detached(struct vtpci_common * cn)202 vtpci_child_detached(struct vtpci_common *cn)
203 {
204
205 vtpci_release_child_resources(cn);
206
207 cn->vtpci_child_feat_desc = NULL;
208 cn->vtpci_host_features = 0;
209 cn->vtpci_features = 0;
210 }
211
212 int
vtpci_reinit(struct vtpci_common * cn)213 vtpci_reinit(struct vtpci_common *cn)
214 {
215 int idx, error;
216
217 for (idx = 0; idx < cn->vtpci_nvqs; idx++) {
218 error = vtpci_reinit_virtqueue(cn, idx);
219 if (error)
220 return (error);
221 }
222
223 if (vtpci_is_msix_enabled(cn)) {
224 error = vtpci_set_host_msix_vectors(cn);
225 if (error)
226 return (error);
227 }
228
229 return (0);
230 }
231
232 static void
vtpci_describe_features(struct vtpci_common * cn,const char * msg,uint64_t features)233 vtpci_describe_features(struct vtpci_common *cn, const char *msg,
234 uint64_t features)
235 {
236 device_t dev, child;
237
238 dev = cn->vtpci_dev;
239 child = cn->vtpci_child_dev;
240
241 if (device_is_attached(child) || bootverbose == 0)
242 return;
243
244 virtio_describe(dev, msg, features, cn->vtpci_child_feat_desc);
245 }
246
247 uint64_t
vtpci_negotiate_features(struct vtpci_common * cn,uint64_t child_features,uint64_t host_features)248 vtpci_negotiate_features(struct vtpci_common *cn,
249 uint64_t child_features, uint64_t host_features)
250 {
251 uint64_t features;
252
253 cn->vtpci_host_features = host_features;
254 vtpci_describe_features(cn, "host", host_features);
255
256 /*
257 * Limit negotiated features to what the driver, virtqueue, and
258 * host all support.
259 */
260 features = host_features & child_features;
261 features = virtio_filter_transport_features(features);
262
263 cn->vtpci_features = features;
264 vtpci_describe_features(cn, "negotiated", features);
265
266 return (features);
267 }
268
269 bool
vtpci_with_feature(struct vtpci_common * cn,uint64_t feature)270 vtpci_with_feature(struct vtpci_common *cn, uint64_t feature)
271 {
272 return ((cn->vtpci_features & feature) != 0);
273 }
274
275 int
vtpci_read_ivar(struct vtpci_common * cn,int index,uintptr_t * result)276 vtpci_read_ivar(struct vtpci_common *cn, int index, uintptr_t *result)
277 {
278 device_t dev;
279 int error;
280
281 dev = cn->vtpci_dev;
282 error = 0;
283
284 switch (index) {
285 case VIRTIO_IVAR_SUBDEVICE:
286 *result = pci_get_subdevice(dev);
287 break;
288 case VIRTIO_IVAR_VENDOR:
289 *result = pci_get_vendor(dev);
290 break;
291 case VIRTIO_IVAR_DEVICE:
292 *result = pci_get_device(dev);
293 break;
294 case VIRTIO_IVAR_SUBVENDOR:
295 *result = pci_get_subvendor(dev);
296 break;
297 case VIRTIO_IVAR_MODERN:
298 *result = vtpci_is_modern(cn);
299 break;
300 default:
301 error = ENOENT;
302 }
303
304 return (error);
305 }
306
307 int
vtpci_write_ivar(struct vtpci_common * cn,int index,uintptr_t value)308 vtpci_write_ivar(struct vtpci_common *cn, int index, uintptr_t value)
309 {
310 int error;
311
312 error = 0;
313
314 switch (index) {
315 case VIRTIO_IVAR_FEATURE_DESC:
316 cn->vtpci_child_feat_desc = (void *) value;
317 break;
318 default:
319 error = ENOENT;
320 }
321
322 return (error);
323 }
324
325 int
vtpci_alloc_virtqueues(struct vtpci_common * cn,int nvqs,struct vq_alloc_info * vq_info)326 vtpci_alloc_virtqueues(struct vtpci_common *cn, int nvqs,
327 struct vq_alloc_info *vq_info)
328 {
329 device_t dev;
330 int idx, align, error;
331
332 dev = cn->vtpci_dev;
333
334 /*
335 * This is VIRTIO_PCI_VRING_ALIGN from legacy VirtIO. In modern VirtIO,
336 * the tables do not have to be allocated contiguously, but we do so
337 * anyways.
338 */
339 align = 4096;
340
341 if (cn->vtpci_nvqs != 0)
342 return (EALREADY);
343 if (nvqs <= 0)
344 return (EINVAL);
345
346 cn->vtpci_vqs = malloc(nvqs * sizeof(struct vtpci_virtqueue),
347 M_DEVBUF, M_NOWAIT | M_ZERO);
348 if (cn->vtpci_vqs == NULL)
349 return (ENOMEM);
350
351 for (idx = 0; idx < nvqs; idx++) {
352 struct vtpci_virtqueue *vqx;
353 struct vq_alloc_info *info;
354 struct virtqueue *vq;
355 bus_size_t notify_offset;
356 uint16_t size;
357
358 vqx = &cn->vtpci_vqs[idx];
359 info = &vq_info[idx];
360
361 size = vtpci_get_vq_size(cn, idx);
362 notify_offset = vtpci_get_vq_notify_off(cn, idx);
363
364 error = virtqueue_alloc(dev, idx, size, notify_offset, align,
365 ~(vm_paddr_t)0, info, &vq);
366 if (error) {
367 device_printf(dev,
368 "cannot allocate virtqueue %d: %d\n", idx, error);
369 break;
370 }
371
372 vtpci_set_vq(cn, vq);
373
374 vqx->vtv_vq = *info->vqai_vq = vq;
375 vqx->vtv_no_intr = info->vqai_intr == NULL;
376
377 cn->vtpci_nvqs++;
378 }
379
380 if (error)
381 vtpci_free_virtqueues(cn);
382
383 return (error);
384 }
385
386 static int
vtpci_alloc_msix(struct vtpci_common * cn,int nvectors)387 vtpci_alloc_msix(struct vtpci_common *cn, int nvectors)
388 {
389 device_t dev;
390 int nmsix, cnt, required;
391
392 dev = cn->vtpci_dev;
393
394 /* Allocate an additional vector for the config changes. */
395 required = nvectors + 1;
396
397 nmsix = pci_msix_count(dev);
398 if (nmsix < required)
399 return (1);
400
401 cnt = required;
402 if (pci_alloc_msix(dev, &cnt) == 0 && cnt >= required) {
403 cn->vtpci_nmsix_resources = required;
404 return (0);
405 }
406
407 pci_release_msi(dev);
408
409 return (1);
410 }
411
412 static int
vtpci_alloc_msi(struct vtpci_common * cn)413 vtpci_alloc_msi(struct vtpci_common *cn)
414 {
415 device_t dev;
416 int nmsi, cnt, required;
417
418 dev = cn->vtpci_dev;
419 required = 1;
420
421 nmsi = pci_msi_count(dev);
422 if (nmsi < required)
423 return (1);
424
425 cnt = required;
426 if (pci_alloc_msi(dev, &cnt) == 0 && cnt >= required)
427 return (0);
428
429 pci_release_msi(dev);
430
431 return (1);
432 }
433
434 static int
vtpci_alloc_intr_msix_pervq(struct vtpci_common * cn)435 vtpci_alloc_intr_msix_pervq(struct vtpci_common *cn)
436 {
437 int i, nvectors, error;
438
439 if (vtpci_disable_msix != 0 || cn->vtpci_flags & VTPCI_FLAG_NO_MSIX)
440 return (ENOTSUP);
441
442 for (nvectors = 0, i = 0; i < cn->vtpci_nvqs; i++) {
443 if (cn->vtpci_vqs[i].vtv_no_intr == 0)
444 nvectors++;
445 }
446
447 error = vtpci_alloc_msix(cn, nvectors);
448 if (error)
449 return (error);
450
451 cn->vtpci_flags |= VTPCI_FLAG_MSIX;
452
453 return (0);
454 }
455
456 static int
vtpci_alloc_intr_msix_shared(struct vtpci_common * cn)457 vtpci_alloc_intr_msix_shared(struct vtpci_common *cn)
458 {
459 int error;
460
461 if (vtpci_disable_msix != 0 || cn->vtpci_flags & VTPCI_FLAG_NO_MSIX)
462 return (ENOTSUP);
463
464 error = vtpci_alloc_msix(cn, 1);
465 if (error)
466 return (error);
467
468 cn->vtpci_flags |= VTPCI_FLAG_MSIX | VTPCI_FLAG_SHARED_MSIX;
469
470 return (0);
471 }
472
473 static int
vtpci_alloc_intr_msi(struct vtpci_common * cn)474 vtpci_alloc_intr_msi(struct vtpci_common *cn)
475 {
476 int error;
477
478 /* Only BHyVe supports MSI. */
479 if (cn->vtpci_flags & VTPCI_FLAG_NO_MSI)
480 return (ENOTSUP);
481
482 error = vtpci_alloc_msi(cn);
483 if (error)
484 return (error);
485
486 cn->vtpci_flags |= VTPCI_FLAG_MSI;
487
488 return (0);
489 }
490
491 static int
vtpci_alloc_intr_intx(struct vtpci_common * cn)492 vtpci_alloc_intr_intx(struct vtpci_common *cn)
493 {
494
495 cn->vtpci_flags |= VTPCI_FLAG_INTX;
496
497 return (0);
498 }
499
500 static int
vtpci_alloc_interrupt(struct vtpci_common * cn,int rid,int flags,struct vtpci_interrupt * intr)501 vtpci_alloc_interrupt(struct vtpci_common *cn, int rid, int flags,
502 struct vtpci_interrupt *intr)
503 {
504 struct resource *irq;
505
506 irq = bus_alloc_resource_any(cn->vtpci_dev, SYS_RES_IRQ, &rid, flags);
507 if (irq == NULL)
508 return (ENXIO);
509
510 intr->vti_irq = irq;
511 intr->vti_rid = rid;
512
513 return (0);
514 }
515
516 static void
vtpci_free_interrupt(struct vtpci_common * cn,struct vtpci_interrupt * intr)517 vtpci_free_interrupt(struct vtpci_common *cn, struct vtpci_interrupt *intr)
518 {
519 device_t dev;
520
521 dev = cn->vtpci_dev;
522
523 if (intr->vti_handler != NULL) {
524 bus_teardown_intr(dev, intr->vti_irq, intr->vti_handler);
525 intr->vti_handler = NULL;
526 }
527
528 if (intr->vti_irq != NULL) {
529 bus_release_resource(dev, SYS_RES_IRQ, intr->vti_rid,
530 intr->vti_irq);
531 intr->vti_irq = NULL;
532 intr->vti_rid = -1;
533 }
534 }
535
536 static void
vtpci_free_interrupts(struct vtpci_common * cn)537 vtpci_free_interrupts(struct vtpci_common *cn)
538 {
539 struct vtpci_interrupt *intr;
540 int i, nvq_intrs;
541
542 vtpci_free_interrupt(cn, &cn->vtpci_device_interrupt);
543
544 if (cn->vtpci_nmsix_resources != 0) {
545 nvq_intrs = cn->vtpci_nmsix_resources - 1;
546 cn->vtpci_nmsix_resources = 0;
547
548 if ((intr = cn->vtpci_msix_vq_interrupts) != NULL) {
549 for (i = 0; i < nvq_intrs; i++, intr++)
550 vtpci_free_interrupt(cn, intr);
551
552 free(cn->vtpci_msix_vq_interrupts, M_DEVBUF);
553 cn->vtpci_msix_vq_interrupts = NULL;
554 }
555 }
556
557 if (cn->vtpci_flags & (VTPCI_FLAG_MSI | VTPCI_FLAG_MSIX))
558 pci_release_msi(cn->vtpci_dev);
559
560 cn->vtpci_flags &= ~VTPCI_FLAG_ITYPE_MASK;
561 }
562
563 static void
vtpci_free_virtqueues(struct vtpci_common * cn)564 vtpci_free_virtqueues(struct vtpci_common *cn)
565 {
566 struct vtpci_virtqueue *vqx;
567 int idx;
568
569 for (idx = 0; idx < cn->vtpci_nvqs; idx++) {
570 vtpci_disable_vq(cn, idx);
571
572 vqx = &cn->vtpci_vqs[idx];
573 virtqueue_free(vqx->vtv_vq);
574 vqx->vtv_vq = NULL;
575 }
576
577 free(cn->vtpci_vqs, M_DEVBUF);
578 cn->vtpci_vqs = NULL;
579 cn->vtpci_nvqs = 0;
580 }
581
582 void
vtpci_release_child_resources(struct vtpci_common * cn)583 vtpci_release_child_resources(struct vtpci_common *cn)
584 {
585
586 vtpci_free_interrupts(cn);
587 vtpci_free_virtqueues(cn);
588 }
589
590 static void
vtpci_cleanup_setup_intr_attempt(struct vtpci_common * cn)591 vtpci_cleanup_setup_intr_attempt(struct vtpci_common *cn)
592 {
593 int idx;
594
595 if (cn->vtpci_flags & VTPCI_FLAG_MSIX) {
596 vtpci_register_cfg_msix(cn, NULL);
597
598 for (idx = 0; idx < cn->vtpci_nvqs; idx++)
599 vtpci_register_vq_msix(cn, idx, NULL);
600 }
601
602 vtpci_free_interrupts(cn);
603 }
604
605 static int
vtpci_alloc_intr_resources(struct vtpci_common * cn)606 vtpci_alloc_intr_resources(struct vtpci_common *cn)
607 {
608 struct vtpci_interrupt *intr;
609 int i, rid, flags, nvq_intrs, error;
610
611 flags = RF_ACTIVE;
612
613 if (cn->vtpci_flags & VTPCI_FLAG_INTX) {
614 rid = 0;
615 flags |= RF_SHAREABLE;
616 } else
617 rid = 1;
618
619 /*
620 * When using INTX or MSI interrupts, this resource handles all
621 * interrupts. When using MSIX, this resource handles just the
622 * configuration changed interrupt.
623 */
624 intr = &cn->vtpci_device_interrupt;
625
626 error = vtpci_alloc_interrupt(cn, rid, flags, intr);
627 if (error || cn->vtpci_flags & (VTPCI_FLAG_INTX | VTPCI_FLAG_MSI))
628 return (error);
629
630 /*
631 * Now allocate the interrupts for the virtqueues. This may be one
632 * for all the virtqueues, or one for each virtqueue. Subtract one
633 * below for because of the configuration changed interrupt.
634 */
635 nvq_intrs = cn->vtpci_nmsix_resources - 1;
636
637 cn->vtpci_msix_vq_interrupts = malloc(nvq_intrs *
638 sizeof(struct vtpci_interrupt), M_DEVBUF, M_NOWAIT | M_ZERO);
639 if (cn->vtpci_msix_vq_interrupts == NULL)
640 return (ENOMEM);
641
642 intr = cn->vtpci_msix_vq_interrupts;
643
644 for (i = 0, rid++; i < nvq_intrs; i++, rid++, intr++) {
645 error = vtpci_alloc_interrupt(cn, rid, flags, intr);
646 if (error)
647 return (error);
648 }
649
650 return (0);
651 }
652
653 static int
vtpci_setup_intx_interrupt(struct vtpci_common * cn,enum intr_type type)654 vtpci_setup_intx_interrupt(struct vtpci_common *cn, enum intr_type type)
655 {
656 struct vtpci_interrupt *intr;
657 int error;
658
659 intr = &cn->vtpci_device_interrupt;
660
661 error = bus_setup_intr(cn->vtpci_dev, intr->vti_irq, type, NULL,
662 vtpci_intx_intr, cn, &intr->vti_handler);
663
664 return (error);
665 }
666
667 static int
vtpci_setup_pervq_msix_interrupts(struct vtpci_common * cn,enum intr_type type)668 vtpci_setup_pervq_msix_interrupts(struct vtpci_common *cn, enum intr_type type)
669 {
670 struct vtpci_virtqueue *vqx;
671 struct vtpci_interrupt *intr;
672 int i, error;
673
674 intr = cn->vtpci_msix_vq_interrupts;
675
676 for (i = 0; i < cn->vtpci_nvqs; i++) {
677 vqx = &cn->vtpci_vqs[i];
678
679 if (vqx->vtv_no_intr)
680 continue;
681
682 error = bus_setup_intr(cn->vtpci_dev, intr->vti_irq, type,
683 vtpci_vq_intr_filter, vtpci_vq_intr, vqx->vtv_vq,
684 &intr->vti_handler);
685 if (error)
686 return (error);
687
688 intr++;
689 }
690
691 return (0);
692 }
693
694 static int
vtpci_set_host_msix_vectors(struct vtpci_common * cn)695 vtpci_set_host_msix_vectors(struct vtpci_common *cn)
696 {
697 struct vtpci_interrupt *intr, *tintr;
698 int idx, error;
699
700 intr = &cn->vtpci_device_interrupt;
701 error = vtpci_register_cfg_msix(cn, intr);
702 if (error)
703 return (error);
704
705 intr = cn->vtpci_msix_vq_interrupts;
706 for (idx = 0; idx < cn->vtpci_nvqs; idx++) {
707 if (cn->vtpci_vqs[idx].vtv_no_intr)
708 tintr = NULL;
709 else
710 tintr = intr;
711
712 error = vtpci_register_vq_msix(cn, idx, tintr);
713 if (error)
714 break;
715
716 /*
717 * For shared MSIX, all the virtqueues share the first
718 * interrupt.
719 */
720 if (!cn->vtpci_vqs[idx].vtv_no_intr &&
721 (cn->vtpci_flags & VTPCI_FLAG_SHARED_MSIX) == 0)
722 intr++;
723 }
724
725 return (error);
726 }
727
728 static int
vtpci_setup_msix_interrupts(struct vtpci_common * cn,enum intr_type type)729 vtpci_setup_msix_interrupts(struct vtpci_common *cn, enum intr_type type)
730 {
731 struct vtpci_interrupt *intr;
732 int error;
733
734 intr = &cn->vtpci_device_interrupt;
735
736 error = bus_setup_intr(cn->vtpci_dev, intr->vti_irq, type, NULL,
737 vtpci_config_intr, cn, &intr->vti_handler);
738 if (error)
739 return (error);
740
741 if (cn->vtpci_flags & VTPCI_FLAG_SHARED_MSIX) {
742 intr = &cn->vtpci_msix_vq_interrupts[0];
743
744 error = bus_setup_intr(cn->vtpci_dev, intr->vti_irq, type,
745 vtpci_vq_shared_intr_filter, vtpci_vq_shared_intr, cn,
746 &intr->vti_handler);
747 } else
748 error = vtpci_setup_pervq_msix_interrupts(cn, type);
749
750 return (error ? error : vtpci_set_host_msix_vectors(cn));
751 }
752
753 static int
vtpci_setup_intrs(struct vtpci_common * cn,enum intr_type type)754 vtpci_setup_intrs(struct vtpci_common *cn, enum intr_type type)
755 {
756 int error;
757
758 type |= INTR_MPSAFE;
759 KASSERT(cn->vtpci_flags & VTPCI_FLAG_ITYPE_MASK,
760 ("%s: no interrupt type selected %#x", __func__, cn->vtpci_flags));
761
762 error = vtpci_alloc_intr_resources(cn);
763 if (error)
764 return (error);
765
766 if (cn->vtpci_flags & VTPCI_FLAG_INTX)
767 error = vtpci_setup_intx_interrupt(cn, type);
768 else if (cn->vtpci_flags & VTPCI_FLAG_MSI)
769 error = vtpci_setup_msi_interrupt(cn, type);
770 else
771 error = vtpci_setup_msix_interrupts(cn, type);
772
773 return (error);
774 }
775
776 int
vtpci_setup_interrupts(struct vtpci_common * cn,enum intr_type type)777 vtpci_setup_interrupts(struct vtpci_common *cn, enum intr_type type)
778 {
779 device_t dev;
780 int attempt, error;
781
782 dev = cn->vtpci_dev;
783
784 for (attempt = 0; attempt < 5; attempt++) {
785 /*
786 * Start with the most desirable interrupt configuration and
787 * fallback towards less desirable ones.
788 */
789 switch (attempt) {
790 case 0:
791 error = vtpci_alloc_intr_msix_pervq(cn);
792 break;
793 case 1:
794 error = vtpci_alloc_intr_msix_shared(cn);
795 break;
796 case 2:
797 error = vtpci_alloc_intr_msi(cn);
798 break;
799 case 3:
800 error = vtpci_alloc_intr_intx(cn);
801 break;
802 default:
803 device_printf(dev,
804 "exhausted all interrupt allocation attempts\n");
805 return (ENXIO);
806 }
807
808 if (error == 0 && vtpci_setup_intrs(cn, type) == 0)
809 break;
810
811 vtpci_cleanup_setup_intr_attempt(cn);
812 }
813
814 if (bootverbose) {
815 if (cn->vtpci_flags & VTPCI_FLAG_INTX)
816 device_printf(dev, "using legacy interrupt\n");
817 else if (cn->vtpci_flags & VTPCI_FLAG_MSI)
818 device_printf(dev, "using MSI interrupt\n");
819 else if (cn->vtpci_flags & VTPCI_FLAG_SHARED_MSIX)
820 device_printf(dev, "using shared MSIX interrupts\n");
821 else
822 device_printf(dev, "using per VQ MSIX interrupts\n");
823 }
824
825 return (0);
826 }
827
828 static int
vtpci_reinit_virtqueue(struct vtpci_common * cn,int idx)829 vtpci_reinit_virtqueue(struct vtpci_common *cn, int idx)
830 {
831 struct vtpci_virtqueue *vqx;
832 struct virtqueue *vq;
833 int error;
834
835 vqx = &cn->vtpci_vqs[idx];
836 vq = vqx->vtv_vq;
837
838 KASSERT(vq != NULL, ("%s: vq %d not allocated", __func__, idx));
839
840 error = virtqueue_reinit(vq, vtpci_get_vq_size(cn, idx));
841 if (error == 0)
842 vtpci_set_vq(cn, vq);
843
844 return (error);
845 }
846
847 static void
vtpci_intx_intr(void * xcn)848 vtpci_intx_intr(void *xcn)
849 {
850 struct vtpci_common *cn;
851 struct vtpci_virtqueue *vqx;
852 int i;
853 uint8_t isr;
854
855 cn = xcn;
856 isr = vtpci_read_isr(cn);
857
858 if (isr & VIRTIO_PCI_ISR_CONFIG)
859 vtpci_config_intr(cn);
860
861 if (isr & VIRTIO_PCI_ISR_INTR) {
862 vqx = &cn->vtpci_vqs[0];
863 for (i = 0; i < cn->vtpci_nvqs; i++, vqx++) {
864 if (vqx->vtv_no_intr == 0)
865 virtqueue_intr(vqx->vtv_vq);
866 }
867 }
868 }
869
870 static int
vtpci_vq_shared_intr_filter(void * xcn)871 vtpci_vq_shared_intr_filter(void *xcn)
872 {
873 struct vtpci_common *cn;
874 struct vtpci_virtqueue *vqx;
875 int i, rc;
876
877 cn = xcn;
878 vqx = &cn->vtpci_vqs[0];
879 rc = 0;
880
881 for (i = 0; i < cn->vtpci_nvqs; i++, vqx++) {
882 if (vqx->vtv_no_intr == 0)
883 rc |= virtqueue_intr_filter(vqx->vtv_vq);
884 }
885
886 return (rc ? FILTER_SCHEDULE_THREAD : FILTER_STRAY);
887 }
888
889 static void
vtpci_vq_shared_intr(void * xcn)890 vtpci_vq_shared_intr(void *xcn)
891 {
892 struct vtpci_common *cn;
893 struct vtpci_virtqueue *vqx;
894 int i;
895
896 cn = xcn;
897 vqx = &cn->vtpci_vqs[0];
898
899 for (i = 0; i < cn->vtpci_nvqs; i++, vqx++) {
900 if (vqx->vtv_no_intr == 0)
901 virtqueue_intr(vqx->vtv_vq);
902 }
903 }
904
905 static int
vtpci_vq_intr_filter(void * xvq)906 vtpci_vq_intr_filter(void *xvq)
907 {
908 struct virtqueue *vq;
909 int rc;
910
911 vq = xvq;
912 rc = virtqueue_intr_filter(vq);
913
914 return (rc ? FILTER_SCHEDULE_THREAD : FILTER_STRAY);
915 }
916
917 static void
vtpci_vq_intr(void * xvq)918 vtpci_vq_intr(void *xvq)
919 {
920 struct virtqueue *vq;
921
922 vq = xvq;
923 virtqueue_intr(vq);
924 }
925
926 static void
vtpci_config_intr(void * xcn)927 vtpci_config_intr(void *xcn)
928 {
929 struct vtpci_common *cn;
930 device_t child;
931
932 cn = xcn;
933 child = cn->vtpci_child_dev;
934
935 if (child != NULL)
936 VIRTIO_CONFIG_CHANGE(child);
937 }
938
939 static int
vtpci_feature_sysctl(struct sysctl_req * req,struct vtpci_common * cn,uint64_t features)940 vtpci_feature_sysctl(struct sysctl_req *req, struct vtpci_common *cn,
941 uint64_t features)
942 {
943 struct sbuf *sb;
944 int error;
945
946 sb = sbuf_new_for_sysctl(NULL, NULL, 256, req);
947 if (sb == NULL)
948 return (ENOMEM);
949
950 error = virtio_describe_sbuf(sb, features, cn->vtpci_child_feat_desc);
951 sbuf_delete(sb);
952
953 return (error);
954 }
955
956 static int
vtpci_host_features_sysctl(SYSCTL_HANDLER_ARGS)957 vtpci_host_features_sysctl(SYSCTL_HANDLER_ARGS)
958 {
959 struct vtpci_common *cn;
960
961 cn = arg1;
962
963 return (vtpci_feature_sysctl(req, cn, cn->vtpci_host_features));
964 }
965
966 static int
vtpci_negotiated_features_sysctl(SYSCTL_HANDLER_ARGS)967 vtpci_negotiated_features_sysctl(SYSCTL_HANDLER_ARGS)
968 {
969 struct vtpci_common *cn;
970
971 cn = arg1;
972
973 return (vtpci_feature_sysctl(req, cn, cn->vtpci_features));
974 }
975
976 static void
vtpci_setup_sysctl(struct vtpci_common * cn)977 vtpci_setup_sysctl(struct vtpci_common *cn)
978 {
979 device_t dev;
980 struct sysctl_ctx_list *ctx;
981 struct sysctl_oid *tree;
982 struct sysctl_oid_list *child;
983
984 dev = cn->vtpci_dev;
985 ctx = device_get_sysctl_ctx(dev);
986 tree = device_get_sysctl_tree(dev);
987 child = SYSCTL_CHILDREN(tree);
988
989 SYSCTL_ADD_INT(ctx, child, OID_AUTO, "nvqs",
990 CTLFLAG_RD, &cn->vtpci_nvqs, 0, "Number of virtqueues");
991
992 SYSCTL_ADD_PROC(ctx, child, OID_AUTO, "host_features",
993 CTLTYPE_STRING | CTLFLAG_RD | CTLFLAG_MPSAFE, cn, 0,
994 vtpci_host_features_sysctl, "A", "Features supported by the host");
995 SYSCTL_ADD_PROC(ctx, child, OID_AUTO, "negotiated_features",
996 CTLTYPE_STRING | CTLFLAG_RD | CTLFLAG_MPSAFE, cn, 0,
997 vtpci_negotiated_features_sysctl, "A", "Features negotiated");
998 }
999