1 /*-
2 * SPDX-License-Identifier: BSD-2-Clause
3 *
4 * Copyright (c) 2017, Bryan Venteicher <bryanv@FreeBSD.org>
5 * All rights reserved.
6 *
7 * Redistribution and use in source and binary forms, with or without
8 * modification, are permitted provided that the following conditions
9 * are met:
10 * 1. Redistributions of source code must retain the above copyright
11 * notice unmodified, this list of conditions, and the following
12 * disclaimer.
13 * 2. Redistributions in binary form must reproduce the above copyright
14 * notice, this list of conditions and the following disclaimer in the
15 * documentation and/or other materials provided with the distribution.
16 *
17 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
18 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
19 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
20 * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
21 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
22 * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
23 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
24 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
25 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
26 * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
27 */
28
29 /* Driver for the modern VirtIO PCI interface. */
30
31 #include <sys/param.h>
32 #include <sys/systm.h>
33 #include <sys/bus.h>
34 #include <sys/lock.h>
35 #include <sys/kernel.h>
36 #include <sys/module.h>
37
38 #include <machine/bus.h>
39 #include <machine/cpu.h>
40 #include <machine/resource.h>
41 #include <sys/bus.h>
42 #include <sys/rman.h>
43
44 #include <dev/pci/pcivar.h>
45 #include <dev/pci/pcireg.h>
46
47 #include <dev/virtio/virtio.h>
48 #include <dev/virtio/virtqueue.h>
49 #include <dev/virtio/pci/virtio_pci.h>
50 #include <dev/virtio/pci/virtio_pci_modern_var.h>
51
52 #include "virtio_bus_if.h"
53 #include "virtio_pci_if.h"
54 #include "virtio_if.h"
55
56 struct vtpci_modern_resource_map {
57 struct resource_map vtrm_map;
58 int vtrm_cap_offset;
59 int vtrm_bar;
60 int vtrm_offset;
61 int vtrm_length;
62 int vtrm_type; /* SYS_RES_{MEMORY, IOPORT} */
63 };
64
65 struct vtpci_modern_bar_resource {
66 struct resource *vtbr_res;
67 int vtbr_type;
68 };
69
70 struct vtpci_modern_softc {
71 device_t vtpci_dev;
72 struct vtpci_common vtpci_common;
73 uint32_t vtpci_notify_offset_multiplier;
74 uint16_t vtpci_devid;
75 int vtpci_msix_bar;
76 struct resource *vtpci_msix_res;
77
78 struct vtpci_modern_resource_map vtpci_common_res_map;
79 struct vtpci_modern_resource_map vtpci_notify_res_map;
80 struct vtpci_modern_resource_map vtpci_isr_res_map;
81 struct vtpci_modern_resource_map vtpci_device_res_map;
82
83 #define VTPCI_MODERN_MAX_BARS 6
84 struct vtpci_modern_bar_resource vtpci_bar_res[VTPCI_MODERN_MAX_BARS];
85 };
86
87 static int vtpci_modern_probe(device_t);
88 static int vtpci_modern_attach(device_t);
89 static int vtpci_modern_detach(device_t);
90 static int vtpci_modern_suspend(device_t);
91 static int vtpci_modern_resume(device_t);
92 static int vtpci_modern_shutdown(device_t);
93
94 static void vtpci_modern_driver_added(device_t, driver_t *);
95 static void vtpci_modern_child_detached(device_t, device_t);
96 static int vtpci_modern_read_ivar(device_t, device_t, int, uintptr_t *);
97 static int vtpci_modern_write_ivar(device_t, device_t, int, uintptr_t);
98
99 static uint8_t vtpci_modern_read_isr(device_t);
100 static uint16_t vtpci_modern_get_vq_size(device_t, int);
101 static bus_size_t vtpci_modern_get_vq_notify_off(device_t, int);
102 static void vtpci_modern_set_vq(device_t, struct virtqueue *);
103 static void vtpci_modern_disable_vq(device_t, int);
104 static int vtpci_modern_register_msix(struct vtpci_modern_softc *, int,
105 struct vtpci_interrupt *);
106 static int vtpci_modern_register_cfg_msix(device_t,
107 struct vtpci_interrupt *);
108 static int vtpci_modern_register_vq_msix(device_t, int idx,
109 struct vtpci_interrupt *);
110
111 static uint64_t vtpci_modern_negotiate_features(device_t, uint64_t);
112 static int vtpci_modern_finalize_features(device_t);
113 static bool vtpci_modern_with_feature(device_t, uint64_t);
114 static int vtpci_modern_alloc_virtqueues(device_t, int,
115 struct vq_alloc_info *);
116 static int vtpci_modern_setup_interrupts(device_t, enum intr_type);
117 static void vtpci_modern_stop(device_t);
118 static int vtpci_modern_reinit(device_t, uint64_t);
119 static void vtpci_modern_reinit_complete(device_t);
120 static void vtpci_modern_notify_vq(device_t, uint16_t, bus_size_t);
121 static int vtpci_modern_config_generation(device_t);
122 static void vtpci_modern_read_dev_config(device_t, bus_size_t, void *, int);
123 static void vtpci_modern_write_dev_config(device_t, bus_size_t, const void *, int);
124
125 static int vtpci_modern_probe_configs(device_t);
126 static int vtpci_modern_find_cap(device_t, uint8_t, int *);
127 static int vtpci_modern_map_configs(struct vtpci_modern_softc *);
128 static void vtpci_modern_unmap_configs(struct vtpci_modern_softc *);
129 static int vtpci_modern_find_cap_resource(struct vtpci_modern_softc *,
130 uint8_t, int, int, struct vtpci_modern_resource_map *);
131 static int vtpci_modern_bar_type(struct vtpci_modern_softc *, int);
132 static struct resource *vtpci_modern_get_bar_resource(
133 struct vtpci_modern_softc *, int, int);
134 static struct resource *vtpci_modern_alloc_bar_resource(
135 struct vtpci_modern_softc *, int, int);
136 static void vtpci_modern_free_bar_resources(struct vtpci_modern_softc *);
137 static int vtpci_modern_alloc_resource_map(struct vtpci_modern_softc *,
138 struct vtpci_modern_resource_map *);
139 static void vtpci_modern_free_resource_map(struct vtpci_modern_softc *,
140 struct vtpci_modern_resource_map *);
141 static void vtpci_modern_alloc_msix_resource(struct vtpci_modern_softc *);
142 static void vtpci_modern_free_msix_resource(struct vtpci_modern_softc *);
143
144 static void vtpci_modern_probe_and_attach_child(struct vtpci_modern_softc *);
145
146 static uint64_t vtpci_modern_read_features(struct vtpci_modern_softc *);
147 static void vtpci_modern_write_features(struct vtpci_modern_softc *,
148 uint64_t);
149 static void vtpci_modern_select_virtqueue(struct vtpci_modern_softc *, int);
150 static uint8_t vtpci_modern_get_status(struct vtpci_modern_softc *);
151 static void vtpci_modern_set_status(struct vtpci_modern_softc *, uint8_t);
152 static void vtpci_modern_reset(struct vtpci_modern_softc *);
153 static void vtpci_modern_enable_virtqueues(struct vtpci_modern_softc *);
154
155 static uint8_t vtpci_modern_read_common_1(struct vtpci_modern_softc *,
156 bus_size_t);
157 static uint16_t vtpci_modern_read_common_2(struct vtpci_modern_softc *,
158 bus_size_t);
159 static uint32_t vtpci_modern_read_common_4(struct vtpci_modern_softc *,
160 bus_size_t);
161 static void vtpci_modern_write_common_1(struct vtpci_modern_softc *,
162 bus_size_t, uint8_t);
163 static void vtpci_modern_write_common_2(struct vtpci_modern_softc *,
164 bus_size_t, uint16_t);
165 static void vtpci_modern_write_common_4(struct vtpci_modern_softc *,
166 bus_size_t, uint32_t);
167 static void vtpci_modern_write_common_8(struct vtpci_modern_softc *,
168 bus_size_t, uint64_t);
169 static void vtpci_modern_write_notify_2(struct vtpci_modern_softc *,
170 bus_size_t, uint16_t);
171 static uint8_t vtpci_modern_read_isr_1(struct vtpci_modern_softc *,
172 bus_size_t);
173 static uint8_t vtpci_modern_read_device_1(struct vtpci_modern_softc *,
174 bus_size_t);
175 static uint16_t vtpci_modern_read_device_2(struct vtpci_modern_softc *,
176 bus_size_t);
177 static uint32_t vtpci_modern_read_device_4(struct vtpci_modern_softc *,
178 bus_size_t);
179 static uint64_t vtpci_modern_read_device_8(struct vtpci_modern_softc *,
180 bus_size_t);
181 static void vtpci_modern_write_device_1(struct vtpci_modern_softc *,
182 bus_size_t, uint8_t);
183 static void vtpci_modern_write_device_2(struct vtpci_modern_softc *,
184 bus_size_t, uint16_t);
185 static void vtpci_modern_write_device_4(struct vtpci_modern_softc *,
186 bus_size_t, uint32_t);
187 static void vtpci_modern_write_device_8(struct vtpci_modern_softc *,
188 bus_size_t, uint64_t);
189
190 /* Tunables. */
191 static int vtpci_modern_transitional = 0;
192 TUNABLE_INT("hw.virtio.pci.transitional", &vtpci_modern_transitional);
193
194 static device_method_t vtpci_modern_methods[] = {
195 /* Device interface. */
196 DEVMETHOD(device_probe, vtpci_modern_probe),
197 DEVMETHOD(device_attach, vtpci_modern_attach),
198 DEVMETHOD(device_detach, vtpci_modern_detach),
199 DEVMETHOD(device_suspend, vtpci_modern_suspend),
200 DEVMETHOD(device_resume, vtpci_modern_resume),
201 DEVMETHOD(device_shutdown, vtpci_modern_shutdown),
202
203 /* Bus interface. */
204 DEVMETHOD(bus_driver_added, vtpci_modern_driver_added),
205 DEVMETHOD(bus_child_detached, vtpci_modern_child_detached),
206 DEVMETHOD(bus_child_pnpinfo, virtio_child_pnpinfo),
207 DEVMETHOD(bus_read_ivar, vtpci_modern_read_ivar),
208 DEVMETHOD(bus_write_ivar, vtpci_modern_write_ivar),
209
210 /* VirtIO PCI interface. */
211 DEVMETHOD(virtio_pci_read_isr, vtpci_modern_read_isr),
212 DEVMETHOD(virtio_pci_get_vq_size, vtpci_modern_get_vq_size),
213 DEVMETHOD(virtio_pci_get_vq_notify_off, vtpci_modern_get_vq_notify_off),
214 DEVMETHOD(virtio_pci_set_vq, vtpci_modern_set_vq),
215 DEVMETHOD(virtio_pci_disable_vq, vtpci_modern_disable_vq),
216 DEVMETHOD(virtio_pci_register_cfg_msix, vtpci_modern_register_cfg_msix),
217 DEVMETHOD(virtio_pci_register_vq_msix, vtpci_modern_register_vq_msix),
218
219 /* VirtIO bus interface. */
220 DEVMETHOD(virtio_bus_negotiate_features, vtpci_modern_negotiate_features),
221 DEVMETHOD(virtio_bus_finalize_features, vtpci_modern_finalize_features),
222 DEVMETHOD(virtio_bus_with_feature, vtpci_modern_with_feature),
223 DEVMETHOD(virtio_bus_alloc_virtqueues, vtpci_modern_alloc_virtqueues),
224 DEVMETHOD(virtio_bus_setup_intr, vtpci_modern_setup_interrupts),
225 DEVMETHOD(virtio_bus_stop, vtpci_modern_stop),
226 DEVMETHOD(virtio_bus_reinit, vtpci_modern_reinit),
227 DEVMETHOD(virtio_bus_reinit_complete, vtpci_modern_reinit_complete),
228 DEVMETHOD(virtio_bus_notify_vq, vtpci_modern_notify_vq),
229 DEVMETHOD(virtio_bus_config_generation, vtpci_modern_config_generation),
230 DEVMETHOD(virtio_bus_read_device_config, vtpci_modern_read_dev_config),
231 DEVMETHOD(virtio_bus_write_device_config, vtpci_modern_write_dev_config),
232
233 DEVMETHOD_END
234 };
235
236 static driver_t vtpci_modern_driver = {
237 .name = "virtio_pci",
238 .methods = vtpci_modern_methods,
239 .size = sizeof(struct vtpci_modern_softc)
240 };
241
242 DRIVER_MODULE(virtio_pci_modern, pci, vtpci_modern_driver, 0, 0);
243
244 static int
vtpci_modern_probe(device_t dev)245 vtpci_modern_probe(device_t dev)
246 {
247 const char *name;
248 uint16_t devid;
249
250 if (pci_get_vendor(dev) != VIRTIO_PCI_VENDORID)
251 return (ENXIO);
252
253 if (pci_get_device(dev) < VIRTIO_PCI_DEVICEID_MIN ||
254 pci_get_device(dev) > VIRTIO_PCI_DEVICEID_MODERN_MAX)
255 return (ENXIO);
256
257 if (pci_get_device(dev) < VIRTIO_PCI_DEVICEID_MODERN_MIN) {
258 if (!vtpci_modern_transitional)
259 return (ENXIO);
260 devid = pci_get_subdevice(dev);
261 } else
262 devid = pci_get_device(dev) - VIRTIO_PCI_DEVICEID_MODERN_MIN;
263
264 if (vtpci_modern_probe_configs(dev) != 0)
265 return (ENXIO);
266
267 name = virtio_device_name(devid);
268 if (name == NULL)
269 name = "Unknown";
270
271 device_set_descf(dev, "VirtIO PCI (modern) %s adapter", name);
272
273 return (BUS_PROBE_DEFAULT);
274 }
275
276 static int
vtpci_modern_attach(device_t dev)277 vtpci_modern_attach(device_t dev)
278 {
279 struct vtpci_modern_softc *sc;
280 int error;
281
282 sc = device_get_softc(dev);
283 sc->vtpci_dev = dev;
284 vtpci_init(&sc->vtpci_common, dev, true);
285
286 if (pci_get_device(dev) < VIRTIO_PCI_DEVICEID_MODERN_MIN)
287 sc->vtpci_devid = pci_get_subdevice(dev);
288 else
289 sc->vtpci_devid = pci_get_device(dev) -
290 VIRTIO_PCI_DEVICEID_MODERN_MIN;
291
292 error = vtpci_modern_map_configs(sc);
293 if (error) {
294 device_printf(dev, "cannot map configs\n");
295 vtpci_modern_unmap_configs(sc);
296 return (error);
297 }
298
299 vtpci_modern_reset(sc);
300
301 /* Tell the host we've noticed this device. */
302 vtpci_modern_set_status(sc, VIRTIO_CONFIG_STATUS_ACK);
303
304 error = vtpci_add_child(&sc->vtpci_common);
305 if (error)
306 goto fail;
307
308 vtpci_modern_probe_and_attach_child(sc);
309
310 return (0);
311
312 fail:
313 vtpci_modern_set_status(sc, VIRTIO_CONFIG_STATUS_FAILED);
314 vtpci_modern_detach(dev);
315
316 return (error);
317 }
318
319 static int
vtpci_modern_detach(device_t dev)320 vtpci_modern_detach(device_t dev)
321 {
322 struct vtpci_modern_softc *sc;
323 int error;
324
325 sc = device_get_softc(dev);
326
327 error = vtpci_delete_child(&sc->vtpci_common);
328 if (error)
329 return (error);
330
331 vtpci_modern_reset(sc);
332 vtpci_modern_unmap_configs(sc);
333
334 return (0);
335 }
336
337 static int
vtpci_modern_suspend(device_t dev)338 vtpci_modern_suspend(device_t dev)
339 {
340 return (bus_generic_suspend(dev));
341 }
342
343 static int
vtpci_modern_resume(device_t dev)344 vtpci_modern_resume(device_t dev)
345 {
346 return (bus_generic_resume(dev));
347 }
348
349 static int
vtpci_modern_shutdown(device_t dev)350 vtpci_modern_shutdown(device_t dev)
351 {
352 (void) bus_generic_shutdown(dev);
353 /* Forcibly stop the host device. */
354 vtpci_modern_stop(dev);
355
356 return (0);
357 }
358
359 static void
vtpci_modern_driver_added(device_t dev,driver_t * driver)360 vtpci_modern_driver_added(device_t dev, driver_t *driver)
361 {
362 vtpci_modern_probe_and_attach_child(device_get_softc(dev));
363 }
364
365 static void
vtpci_modern_child_detached(device_t dev,device_t child)366 vtpci_modern_child_detached(device_t dev, device_t child)
367 {
368 struct vtpci_modern_softc *sc;
369
370 sc = device_get_softc(dev);
371
372 vtpci_modern_reset(sc);
373 vtpci_child_detached(&sc->vtpci_common);
374
375 /* After the reset, retell the host we've noticed this device. */
376 vtpci_modern_set_status(sc, VIRTIO_CONFIG_STATUS_ACK);
377 }
378
379 static int
vtpci_modern_read_ivar(device_t dev,device_t child,int index,uintptr_t * result)380 vtpci_modern_read_ivar(device_t dev, device_t child, int index,
381 uintptr_t *result)
382 {
383 struct vtpci_modern_softc *sc;
384 struct vtpci_common *cn;
385
386 sc = device_get_softc(dev);
387 cn = &sc->vtpci_common;
388
389 if (vtpci_child_device(cn) != child)
390 return (ENOENT);
391
392 switch (index) {
393 case VIRTIO_IVAR_DEVTYPE:
394 *result = sc->vtpci_devid;
395 break;
396 default:
397 return (vtpci_read_ivar(cn, index, result));
398 }
399
400 return (0);
401 }
402
403 static int
vtpci_modern_write_ivar(device_t dev,device_t child,int index,uintptr_t value)404 vtpci_modern_write_ivar(device_t dev, device_t child, int index,
405 uintptr_t value)
406 {
407 struct vtpci_modern_softc *sc;
408 struct vtpci_common *cn;
409
410 sc = device_get_softc(dev);
411 cn = &sc->vtpci_common;
412
413 if (vtpci_child_device(cn) != child)
414 return (ENOENT);
415
416 switch (index) {
417 default:
418 return (vtpci_write_ivar(cn, index, value));
419 }
420
421 return (0);
422 }
423
424 static uint64_t
vtpci_modern_negotiate_features(device_t dev,uint64_t child_features)425 vtpci_modern_negotiate_features(device_t dev, uint64_t child_features)
426 {
427 struct vtpci_modern_softc *sc;
428 uint64_t host_features, features;
429
430 sc = device_get_softc(dev);
431 host_features = vtpci_modern_read_features(sc);
432
433 /*
434 * Since the driver was added as a child of the modern PCI bus,
435 * always add the V1 flag.
436 */
437 child_features |= VIRTIO_F_VERSION_1;
438
439 features = vtpci_negotiate_features(&sc->vtpci_common,
440 child_features, host_features);
441 vtpci_modern_write_features(sc, features);
442
443 return (features);
444 }
445
446 static int
vtpci_modern_finalize_features(device_t dev)447 vtpci_modern_finalize_features(device_t dev)
448 {
449 struct vtpci_modern_softc *sc;
450 uint8_t status;
451
452 sc = device_get_softc(dev);
453
454 /*
455 * Must re-read the status after setting it to verify the negotiated
456 * features were accepted by the device.
457 */
458 vtpci_modern_set_status(sc, VIRTIO_CONFIG_S_FEATURES_OK);
459
460 status = vtpci_modern_get_status(sc);
461 if ((status & VIRTIO_CONFIG_S_FEATURES_OK) == 0) {
462 device_printf(dev, "desired features were not accepted\n");
463 return (ENOTSUP);
464 }
465
466 return (0);
467 }
468
469 static bool
vtpci_modern_with_feature(device_t dev,uint64_t feature)470 vtpci_modern_with_feature(device_t dev, uint64_t feature)
471 {
472 struct vtpci_modern_softc *sc;
473
474 sc = device_get_softc(dev);
475
476 return (vtpci_with_feature(&sc->vtpci_common, feature));
477 }
478
479 static uint64_t
vtpci_modern_read_features(struct vtpci_modern_softc * sc)480 vtpci_modern_read_features(struct vtpci_modern_softc *sc)
481 {
482 uint32_t features0, features1;
483
484 vtpci_modern_write_common_4(sc, VIRTIO_PCI_COMMON_DFSELECT, 0);
485 features0 = vtpci_modern_read_common_4(sc, VIRTIO_PCI_COMMON_DF);
486 vtpci_modern_write_common_4(sc, VIRTIO_PCI_COMMON_DFSELECT, 1);
487 features1 = vtpci_modern_read_common_4(sc, VIRTIO_PCI_COMMON_DF);
488
489 return (((uint64_t) features1 << 32) | features0);
490 }
491
492 static void
vtpci_modern_write_features(struct vtpci_modern_softc * sc,uint64_t features)493 vtpci_modern_write_features(struct vtpci_modern_softc *sc, uint64_t features)
494 {
495 uint32_t features0, features1;
496
497 features0 = features;
498 features1 = features >> 32;
499
500 vtpci_modern_write_common_4(sc, VIRTIO_PCI_COMMON_GFSELECT, 0);
501 vtpci_modern_write_common_4(sc, VIRTIO_PCI_COMMON_GF, features0);
502 vtpci_modern_write_common_4(sc, VIRTIO_PCI_COMMON_GFSELECT, 1);
503 vtpci_modern_write_common_4(sc, VIRTIO_PCI_COMMON_GF, features1);
504 }
505
506 static int
vtpci_modern_alloc_virtqueues(device_t dev,int nvqs,struct vq_alloc_info * vq_info)507 vtpci_modern_alloc_virtqueues(device_t dev, int nvqs,
508 struct vq_alloc_info *vq_info)
509 {
510 struct vtpci_modern_softc *sc;
511 struct vtpci_common *cn;
512 uint16_t max_nvqs;
513
514 sc = device_get_softc(dev);
515 cn = &sc->vtpci_common;
516
517 max_nvqs = vtpci_modern_read_common_2(sc, VIRTIO_PCI_COMMON_NUMQ);
518 if (nvqs > max_nvqs) {
519 device_printf(sc->vtpci_dev, "requested virtqueue count %d "
520 "exceeds max %d\n", nvqs, max_nvqs);
521 return (E2BIG);
522 }
523
524 return (vtpci_alloc_virtqueues(cn, nvqs, vq_info));
525 }
526
527 static int
vtpci_modern_setup_interrupts(device_t dev,enum intr_type type)528 vtpci_modern_setup_interrupts(device_t dev, enum intr_type type)
529 {
530 struct vtpci_modern_softc *sc;
531 int error;
532
533 sc = device_get_softc(dev);
534
535 error = vtpci_setup_interrupts(&sc->vtpci_common, type);
536 if (error == 0)
537 vtpci_modern_enable_virtqueues(sc);
538
539 return (error);
540 }
541
542 static void
vtpci_modern_stop(device_t dev)543 vtpci_modern_stop(device_t dev)
544 {
545 vtpci_modern_reset(device_get_softc(dev));
546 }
547
548 static int
vtpci_modern_reinit(device_t dev,uint64_t features)549 vtpci_modern_reinit(device_t dev, uint64_t features)
550 {
551 struct vtpci_modern_softc *sc;
552 struct vtpci_common *cn;
553 int error;
554
555 sc = device_get_softc(dev);
556 cn = &sc->vtpci_common;
557
558 /*
559 * Redrive the device initialization. This is a bit of an abuse of
560 * the specification, but VirtualBox, QEMU/KVM, and BHyVe seem to
561 * play nice.
562 *
563 * We do not allow the host device to change from what was originally
564 * negotiated beyond what the guest driver changed. MSIX state should
565 * not change, number of virtqueues and their size remain the same, etc.
566 * This will need to be rethought when we want to support migration.
567 */
568
569 if (vtpci_modern_get_status(sc) != VIRTIO_CONFIG_STATUS_RESET)
570 vtpci_modern_stop(dev);
571
572 /*
573 * Quickly drive the status through ACK and DRIVER. The device does
574 * not become usable again until DRIVER_OK in reinit complete.
575 */
576 vtpci_modern_set_status(sc, VIRTIO_CONFIG_STATUS_ACK);
577 vtpci_modern_set_status(sc, VIRTIO_CONFIG_STATUS_DRIVER);
578
579 /*
580 * TODO: Check that features are not added as to what was
581 * originally negotiated.
582 */
583 vtpci_modern_negotiate_features(dev, features);
584 error = vtpci_modern_finalize_features(dev);
585 if (error) {
586 device_printf(dev, "cannot finalize features during reinit\n");
587 return (error);
588 }
589
590 error = vtpci_reinit(cn);
591 if (error)
592 return (error);
593
594 return (0);
595 }
596
597 static void
vtpci_modern_reinit_complete(device_t dev)598 vtpci_modern_reinit_complete(device_t dev)
599 {
600 struct vtpci_modern_softc *sc;
601
602 sc = device_get_softc(dev);
603
604 vtpci_modern_enable_virtqueues(sc);
605 vtpci_modern_set_status(sc, VIRTIO_CONFIG_STATUS_DRIVER_OK);
606 }
607
608 static void
vtpci_modern_notify_vq(device_t dev,uint16_t queue,bus_size_t offset)609 vtpci_modern_notify_vq(device_t dev, uint16_t queue, bus_size_t offset)
610 {
611 struct vtpci_modern_softc *sc;
612
613 sc = device_get_softc(dev);
614
615 vtpci_modern_write_notify_2(sc, offset, queue);
616 }
617
618 static uint8_t
vtpci_modern_get_status(struct vtpci_modern_softc * sc)619 vtpci_modern_get_status(struct vtpci_modern_softc *sc)
620 {
621 return (vtpci_modern_read_common_1(sc, VIRTIO_PCI_COMMON_STATUS));
622 }
623
624 static void
vtpci_modern_set_status(struct vtpci_modern_softc * sc,uint8_t status)625 vtpci_modern_set_status(struct vtpci_modern_softc *sc, uint8_t status)
626 {
627 if (status != VIRTIO_CONFIG_STATUS_RESET)
628 status |= vtpci_modern_get_status(sc);
629
630 vtpci_modern_write_common_1(sc, VIRTIO_PCI_COMMON_STATUS, status);
631 }
632
633 static int
vtpci_modern_config_generation(device_t dev)634 vtpci_modern_config_generation(device_t dev)
635 {
636 struct vtpci_modern_softc *sc;
637 uint8_t gen;
638
639 sc = device_get_softc(dev);
640 gen = vtpci_modern_read_common_1(sc, VIRTIO_PCI_COMMON_CFGGENERATION);
641
642 return (gen);
643 }
644
645 static void
vtpci_modern_read_dev_config(device_t dev,bus_size_t offset,void * dst,int length)646 vtpci_modern_read_dev_config(device_t dev, bus_size_t offset, void *dst,
647 int length)
648 {
649 struct vtpci_modern_softc *sc;
650
651 sc = device_get_softc(dev);
652
653 if (sc->vtpci_device_res_map.vtrm_map.r_size == 0) {
654 panic("%s: attempt to read dev config but not present",
655 __func__);
656 }
657
658 switch (length) {
659 case 1:
660 *(uint8_t *) dst = vtpci_modern_read_device_1(sc, offset);
661 break;
662 case 2:
663 *(uint16_t *) dst = virtio_htog16(true,
664 vtpci_modern_read_device_2(sc, offset));
665 break;
666 case 4:
667 *(uint32_t *) dst = virtio_htog32(true,
668 vtpci_modern_read_device_4(sc, offset));
669 break;
670 case 8:
671 *(uint64_t *) dst = virtio_htog64(true,
672 vtpci_modern_read_device_8(sc, offset));
673 break;
674 default:
675 panic("%s: device %s invalid device read length %d offset %d",
676 __func__, device_get_nameunit(dev), length, (int) offset);
677 }
678 }
679
680 static void
vtpci_modern_write_dev_config(device_t dev,bus_size_t offset,const void * src,int length)681 vtpci_modern_write_dev_config(device_t dev, bus_size_t offset, const void *src,
682 int length)
683 {
684 struct vtpci_modern_softc *sc;
685
686 sc = device_get_softc(dev);
687
688 if (sc->vtpci_device_res_map.vtrm_map.r_size == 0) {
689 panic("%s: attempt to write dev config but not present",
690 __func__);
691 }
692
693 switch (length) {
694 case 1:
695 vtpci_modern_write_device_1(sc, offset, *(const uint8_t *) src);
696 break;
697 case 2: {
698 uint16_t val = virtio_gtoh16(true, *(const uint16_t *) src);
699 vtpci_modern_write_device_2(sc, offset, val);
700 break;
701 }
702 case 4: {
703 uint32_t val = virtio_gtoh32(true, *(const uint32_t *) src);
704 vtpci_modern_write_device_4(sc, offset, val);
705 break;
706 }
707 case 8: {
708 uint64_t val = virtio_gtoh64(true, *(const uint64_t *) src);
709 vtpci_modern_write_device_8(sc, offset, val);
710 break;
711 }
712 default:
713 panic("%s: device %s invalid device write length %d offset %d",
714 __func__, device_get_nameunit(dev), length, (int) offset);
715 }
716 }
717
718 static int
vtpci_modern_probe_configs(device_t dev)719 vtpci_modern_probe_configs(device_t dev)
720 {
721 int error;
722
723 /*
724 * These config capabilities must be present. The DEVICE_CFG
725 * capability is only present if the device requires it.
726 */
727
728 error = vtpci_modern_find_cap(dev, VIRTIO_PCI_CAP_COMMON_CFG, NULL);
729 if (error) {
730 device_printf(dev, "cannot find COMMON_CFG capability\n");
731 return (error);
732 }
733
734 error = vtpci_modern_find_cap(dev, VIRTIO_PCI_CAP_NOTIFY_CFG, NULL);
735 if (error) {
736 device_printf(dev, "cannot find NOTIFY_CFG capability\n");
737 return (error);
738 }
739
740 error = vtpci_modern_find_cap(dev, VIRTIO_PCI_CAP_ISR_CFG, NULL);
741 if (error) {
742 device_printf(dev, "cannot find ISR_CFG capability\n");
743 return (error);
744 }
745
746 return (0);
747 }
748
749 static int
vtpci_modern_find_cap(device_t dev,uint8_t cfg_type,int * cap_offset)750 vtpci_modern_find_cap(device_t dev, uint8_t cfg_type, int *cap_offset)
751 {
752 uint32_t type, bar;
753 int capreg, error;
754
755 for (error = pci_find_cap(dev, PCIY_VENDOR, &capreg);
756 error == 0;
757 error = pci_find_next_cap(dev, PCIY_VENDOR, capreg, &capreg)) {
758
759 type = pci_read_config(dev, capreg +
760 offsetof(struct virtio_pci_cap, cfg_type), 1);
761 bar = pci_read_config(dev, capreg +
762 offsetof(struct virtio_pci_cap, bar), 1);
763
764 /* Must ignore reserved BARs. */
765 if (bar >= VTPCI_MODERN_MAX_BARS)
766 continue;
767
768 if (type == cfg_type) {
769 if (cap_offset != NULL)
770 *cap_offset = capreg;
771 break;
772 }
773 }
774
775 return (error);
776 }
777
778 static int
vtpci_modern_map_common_config(struct vtpci_modern_softc * sc)779 vtpci_modern_map_common_config(struct vtpci_modern_softc *sc)
780 {
781 device_t dev;
782 int error;
783
784 dev = sc->vtpci_dev;
785
786 error = vtpci_modern_find_cap_resource(sc, VIRTIO_PCI_CAP_COMMON_CFG,
787 sizeof(struct virtio_pci_common_cfg), 4, &sc->vtpci_common_res_map);
788 if (error) {
789 device_printf(dev, "cannot find cap COMMON_CFG resource\n");
790 return (error);
791 }
792
793 error = vtpci_modern_alloc_resource_map(sc, &sc->vtpci_common_res_map);
794 if (error) {
795 device_printf(dev, "cannot alloc resource for COMMON_CFG\n");
796 return (error);
797 }
798
799 return (0);
800 }
801
802 static int
vtpci_modern_map_notify_config(struct vtpci_modern_softc * sc)803 vtpci_modern_map_notify_config(struct vtpci_modern_softc *sc)
804 {
805 device_t dev;
806 int cap_offset, error;
807
808 dev = sc->vtpci_dev;
809
810 error = vtpci_modern_find_cap_resource(sc, VIRTIO_PCI_CAP_NOTIFY_CFG,
811 -1, 2, &sc->vtpci_notify_res_map);
812 if (error) {
813 device_printf(dev, "cannot find cap NOTIFY_CFG resource\n");
814 return (error);
815 }
816
817 cap_offset = sc->vtpci_notify_res_map.vtrm_cap_offset;
818
819 sc->vtpci_notify_offset_multiplier = pci_read_config(dev, cap_offset +
820 offsetof(struct virtio_pci_notify_cap, notify_off_multiplier), 4);
821
822 error = vtpci_modern_alloc_resource_map(sc, &sc->vtpci_notify_res_map);
823 if (error) {
824 device_printf(dev, "cannot alloc resource for NOTIFY_CFG\n");
825 return (error);
826 }
827
828 return (0);
829 }
830
831 static int
vtpci_modern_map_isr_config(struct vtpci_modern_softc * sc)832 vtpci_modern_map_isr_config(struct vtpci_modern_softc *sc)
833 {
834 device_t dev;
835 int error;
836
837 dev = sc->vtpci_dev;
838
839 error = vtpci_modern_find_cap_resource(sc, VIRTIO_PCI_CAP_ISR_CFG,
840 sizeof(uint8_t), 1, &sc->vtpci_isr_res_map);
841 if (error) {
842 device_printf(dev, "cannot find cap ISR_CFG resource\n");
843 return (error);
844 }
845
846 error = vtpci_modern_alloc_resource_map(sc, &sc->vtpci_isr_res_map);
847 if (error) {
848 device_printf(dev, "cannot alloc resource for ISR_CFG\n");
849 return (error);
850 }
851
852 return (0);
853 }
854
855 static int
vtpci_modern_map_device_config(struct vtpci_modern_softc * sc)856 vtpci_modern_map_device_config(struct vtpci_modern_softc *sc)
857 {
858 device_t dev;
859 int error;
860
861 dev = sc->vtpci_dev;
862
863 error = vtpci_modern_find_cap_resource(sc, VIRTIO_PCI_CAP_DEVICE_CFG,
864 -1, 4, &sc->vtpci_device_res_map);
865 if (error == ENOENT) {
866 /* Device configuration is optional depending on device. */
867 return (0);
868 } else if (error) {
869 device_printf(dev, "cannot find cap DEVICE_CFG resource\n");
870 return (error);
871 }
872
873 error = vtpci_modern_alloc_resource_map(sc, &sc->vtpci_device_res_map);
874 if (error) {
875 device_printf(dev, "cannot alloc resource for DEVICE_CFG\n");
876 return (error);
877 }
878
879 return (0);
880 }
881
882 static int
vtpci_modern_map_configs(struct vtpci_modern_softc * sc)883 vtpci_modern_map_configs(struct vtpci_modern_softc *sc)
884 {
885 int error;
886
887 error = vtpci_modern_map_common_config(sc);
888 if (error)
889 return (error);
890
891 error = vtpci_modern_map_notify_config(sc);
892 if (error)
893 return (error);
894
895 error = vtpci_modern_map_isr_config(sc);
896 if (error)
897 return (error);
898
899 error = vtpci_modern_map_device_config(sc);
900 if (error)
901 return (error);
902
903 vtpci_modern_alloc_msix_resource(sc);
904
905 return (0);
906 }
907
908 static void
vtpci_modern_unmap_configs(struct vtpci_modern_softc * sc)909 vtpci_modern_unmap_configs(struct vtpci_modern_softc *sc)
910 {
911
912 vtpci_modern_free_resource_map(sc, &sc->vtpci_common_res_map);
913 vtpci_modern_free_resource_map(sc, &sc->vtpci_notify_res_map);
914 vtpci_modern_free_resource_map(sc, &sc->vtpci_isr_res_map);
915 vtpci_modern_free_resource_map(sc, &sc->vtpci_device_res_map);
916
917 vtpci_modern_free_bar_resources(sc);
918 vtpci_modern_free_msix_resource(sc);
919
920 sc->vtpci_notify_offset_multiplier = 0;
921 }
922
923 static int
vtpci_modern_find_cap_resource(struct vtpci_modern_softc * sc,uint8_t cfg_type,int min_size,int alignment,struct vtpci_modern_resource_map * res)924 vtpci_modern_find_cap_resource(struct vtpci_modern_softc *sc, uint8_t cfg_type,
925 int min_size, int alignment, struct vtpci_modern_resource_map *res)
926 {
927 device_t dev;
928 int cap_offset, offset, length, error;
929 uint8_t bar, cap_length;
930
931 dev = sc->vtpci_dev;
932
933 error = vtpci_modern_find_cap(dev, cfg_type, &cap_offset);
934 if (error)
935 return (error);
936
937 cap_length = pci_read_config(dev,
938 cap_offset + offsetof(struct virtio_pci_cap, cap_len), 1);
939
940 if (cap_length < sizeof(struct virtio_pci_cap)) {
941 device_printf(dev, "cap %u length %d less than expected\n",
942 cfg_type, cap_length);
943 return (ENXIO);
944 }
945
946 bar = pci_read_config(dev,
947 cap_offset + offsetof(struct virtio_pci_cap, bar), 1);
948 offset = pci_read_config(dev,
949 cap_offset + offsetof(struct virtio_pci_cap, offset), 4);
950 length = pci_read_config(dev,
951 cap_offset + offsetof(struct virtio_pci_cap, length), 4);
952
953 if (min_size != -1 && length < min_size) {
954 device_printf(dev, "cap %u struct length %d less than min %d\n",
955 cfg_type, length, min_size);
956 return (ENXIO);
957 }
958
959 if (offset % alignment) {
960 device_printf(dev, "cap %u struct offset %d not aligned to %d\n",
961 cfg_type, offset, alignment);
962 return (ENXIO);
963 }
964
965 /* BMV: TODO Can we determine the size of the BAR here? */
966
967 res->vtrm_cap_offset = cap_offset;
968 res->vtrm_bar = bar;
969 res->vtrm_offset = offset;
970 res->vtrm_length = length;
971 res->vtrm_type = vtpci_modern_bar_type(sc, bar);
972
973 return (0);
974 }
975
976 static int
vtpci_modern_bar_type(struct vtpci_modern_softc * sc,int bar)977 vtpci_modern_bar_type(struct vtpci_modern_softc *sc, int bar)
978 {
979 uint32_t val;
980
981 /*
982 * The BAR described by a config capability may be either an IOPORT or
983 * MEM, but we must know the type when calling bus_alloc_resource().
984 */
985 val = pci_read_config(sc->vtpci_dev, PCIR_BAR(bar), 4);
986 if (PCI_BAR_IO(val))
987 return (SYS_RES_IOPORT);
988 else
989 return (SYS_RES_MEMORY);
990 }
991
992 static struct resource *
vtpci_modern_get_bar_resource(struct vtpci_modern_softc * sc,int bar,int type)993 vtpci_modern_get_bar_resource(struct vtpci_modern_softc *sc, int bar, int type)
994 {
995 struct resource *res;
996
997 MPASS(bar >= 0 && bar < VTPCI_MODERN_MAX_BARS);
998 res = sc->vtpci_bar_res[bar].vtbr_res;
999 MPASS(res == NULL || sc->vtpci_bar_res[bar].vtbr_type == type);
1000
1001 return (res);
1002 }
1003
1004 static struct resource *
vtpci_modern_alloc_bar_resource(struct vtpci_modern_softc * sc,int bar,int type)1005 vtpci_modern_alloc_bar_resource(struct vtpci_modern_softc *sc, int bar,
1006 int type)
1007 {
1008 struct resource *res;
1009 int rid;
1010
1011 MPASS(bar >= 0 && bar < VTPCI_MODERN_MAX_BARS);
1012 MPASS(type == SYS_RES_MEMORY || type == SYS_RES_IOPORT);
1013
1014 res = sc->vtpci_bar_res[bar].vtbr_res;
1015 if (res != NULL) {
1016 MPASS(sc->vtpci_bar_res[bar].vtbr_type == type);
1017 return (res);
1018 }
1019
1020 rid = PCIR_BAR(bar);
1021 res = bus_alloc_resource_any(sc->vtpci_dev, type, &rid,
1022 RF_ACTIVE | RF_UNMAPPED);
1023 if (res != NULL) {
1024 sc->vtpci_bar_res[bar].vtbr_res = res;
1025 sc->vtpci_bar_res[bar].vtbr_type = type;
1026 }
1027
1028 return (res);
1029 }
1030
1031 static void
vtpci_modern_free_bar_resources(struct vtpci_modern_softc * sc)1032 vtpci_modern_free_bar_resources(struct vtpci_modern_softc *sc)
1033 {
1034 device_t dev;
1035 struct resource *res;
1036 int bar, rid, type;
1037
1038 dev = sc->vtpci_dev;
1039
1040 for (bar = 0; bar < VTPCI_MODERN_MAX_BARS; bar++) {
1041 res = sc->vtpci_bar_res[bar].vtbr_res;
1042 type = sc->vtpci_bar_res[bar].vtbr_type;
1043
1044 if (res != NULL) {
1045 rid = PCIR_BAR(bar);
1046 bus_release_resource(dev, type, rid, res);
1047 sc->vtpci_bar_res[bar].vtbr_res = NULL;
1048 sc->vtpci_bar_res[bar].vtbr_type = 0;
1049 }
1050 }
1051 }
1052
1053 static int
vtpci_modern_alloc_resource_map(struct vtpci_modern_softc * sc,struct vtpci_modern_resource_map * map)1054 vtpci_modern_alloc_resource_map(struct vtpci_modern_softc *sc,
1055 struct vtpci_modern_resource_map *map)
1056 {
1057 struct resource_map_request req;
1058 struct resource *res;
1059 int type;
1060
1061 type = map->vtrm_type;
1062
1063 res = vtpci_modern_alloc_bar_resource(sc, map->vtrm_bar, type);
1064 if (res == NULL)
1065 return (ENXIO);
1066
1067 resource_init_map_request(&req);
1068 req.offset = map->vtrm_offset;
1069 req.length = map->vtrm_length;
1070
1071 return (bus_map_resource(sc->vtpci_dev, type, res, &req,
1072 &map->vtrm_map));
1073 }
1074
1075 static void
vtpci_modern_free_resource_map(struct vtpci_modern_softc * sc,struct vtpci_modern_resource_map * map)1076 vtpci_modern_free_resource_map(struct vtpci_modern_softc *sc,
1077 struct vtpci_modern_resource_map *map)
1078 {
1079 struct resource *res;
1080 int type;
1081
1082 type = map->vtrm_type;
1083 res = vtpci_modern_get_bar_resource(sc, map->vtrm_bar, type);
1084
1085 if (res != NULL && map->vtrm_map.r_size != 0) {
1086 bus_unmap_resource(sc->vtpci_dev, type, res, &map->vtrm_map);
1087 bzero(map, sizeof(struct vtpci_modern_resource_map));
1088 }
1089 }
1090
1091 static void
vtpci_modern_alloc_msix_resource(struct vtpci_modern_softc * sc)1092 vtpci_modern_alloc_msix_resource(struct vtpci_modern_softc *sc)
1093 {
1094 device_t dev;
1095 int bar;
1096
1097 dev = sc->vtpci_dev;
1098
1099 if (!vtpci_is_msix_available(&sc->vtpci_common) ||
1100 (bar = pci_msix_table_bar(dev)) == -1)
1101 return;
1102
1103 /* TODO: Can this BAR be in the 0-5 range? */
1104 sc->vtpci_msix_bar = bar;
1105 if ((sc->vtpci_msix_res = bus_alloc_resource_any(dev, SYS_RES_MEMORY,
1106 &bar, RF_ACTIVE)) == NULL)
1107 device_printf(dev, "Unable to map MSIX table\n");
1108 }
1109
1110 static void
vtpci_modern_free_msix_resource(struct vtpci_modern_softc * sc)1111 vtpci_modern_free_msix_resource(struct vtpci_modern_softc *sc)
1112 {
1113 device_t dev;
1114
1115 dev = sc->vtpci_dev;
1116
1117 if (sc->vtpci_msix_res != NULL) {
1118 bus_release_resource(dev, SYS_RES_MEMORY, sc->vtpci_msix_bar,
1119 sc->vtpci_msix_res);
1120 sc->vtpci_msix_bar = 0;
1121 sc->vtpci_msix_res = NULL;
1122 }
1123 }
1124
1125 static void
vtpci_modern_probe_and_attach_child(struct vtpci_modern_softc * sc)1126 vtpci_modern_probe_and_attach_child(struct vtpci_modern_softc *sc)
1127 {
1128 device_t dev, child;
1129
1130 dev = sc->vtpci_dev;
1131 child = vtpci_child_device(&sc->vtpci_common);
1132
1133 if (child == NULL || device_get_state(child) != DS_NOTPRESENT)
1134 return;
1135
1136 if (device_probe(child) != 0)
1137 return;
1138
1139 vtpci_modern_set_status(sc, VIRTIO_CONFIG_STATUS_DRIVER);
1140
1141 if (device_attach(child) != 0) {
1142 vtpci_modern_set_status(sc, VIRTIO_CONFIG_STATUS_FAILED);
1143 /* Reset state for later attempt. */
1144 vtpci_modern_child_detached(dev, child);
1145 } else {
1146 vtpci_modern_set_status(sc, VIRTIO_CONFIG_STATUS_DRIVER_OK);
1147 VIRTIO_ATTACH_COMPLETED(child);
1148 }
1149 }
1150
1151 static int
vtpci_modern_register_msix(struct vtpci_modern_softc * sc,int offset,struct vtpci_interrupt * intr)1152 vtpci_modern_register_msix(struct vtpci_modern_softc *sc, int offset,
1153 struct vtpci_interrupt *intr)
1154 {
1155 uint16_t vector;
1156
1157 if (intr != NULL) {
1158 /* Map from guest rid to host vector. */
1159 vector = intr->vti_rid - 1;
1160 } else
1161 vector = VIRTIO_MSI_NO_VECTOR;
1162
1163 vtpci_modern_write_common_2(sc, offset, vector);
1164 return (vtpci_modern_read_common_2(sc, offset) == vector ? 0 : ENODEV);
1165 }
1166
1167 static int
vtpci_modern_register_cfg_msix(device_t dev,struct vtpci_interrupt * intr)1168 vtpci_modern_register_cfg_msix(device_t dev, struct vtpci_interrupt *intr)
1169 {
1170 struct vtpci_modern_softc *sc;
1171 int error;
1172
1173 sc = device_get_softc(dev);
1174
1175 error = vtpci_modern_register_msix(sc, VIRTIO_PCI_COMMON_MSIX, intr);
1176 if (error) {
1177 device_printf(dev,
1178 "unable to register config MSIX interrupt\n");
1179 return (error);
1180 }
1181
1182 return (0);
1183 }
1184
1185 static int
vtpci_modern_register_vq_msix(device_t dev,int idx,struct vtpci_interrupt * intr)1186 vtpci_modern_register_vq_msix(device_t dev, int idx,
1187 struct vtpci_interrupt *intr)
1188 {
1189 struct vtpci_modern_softc *sc;
1190 int error;
1191
1192 sc = device_get_softc(dev);
1193
1194 vtpci_modern_select_virtqueue(sc, idx);
1195 error = vtpci_modern_register_msix(sc, VIRTIO_PCI_COMMON_Q_MSIX, intr);
1196 if (error) {
1197 device_printf(dev,
1198 "unable to register virtqueue MSIX interrupt\n");
1199 return (error);
1200 }
1201
1202 return (0);
1203 }
1204
1205 static void
vtpci_modern_reset(struct vtpci_modern_softc * sc)1206 vtpci_modern_reset(struct vtpci_modern_softc *sc)
1207 {
1208 /*
1209 * Setting the status to RESET sets the host device to the
1210 * original, uninitialized state. Must poll the status until
1211 * the reset is complete.
1212 */
1213 vtpci_modern_set_status(sc, VIRTIO_CONFIG_STATUS_RESET);
1214
1215 while (vtpci_modern_get_status(sc) != VIRTIO_CONFIG_STATUS_RESET)
1216 cpu_spinwait();
1217 }
1218
1219 static void
vtpci_modern_select_virtqueue(struct vtpci_modern_softc * sc,int idx)1220 vtpci_modern_select_virtqueue(struct vtpci_modern_softc *sc, int idx)
1221 {
1222 vtpci_modern_write_common_2(sc, VIRTIO_PCI_COMMON_Q_SELECT, idx);
1223 }
1224
1225 static uint8_t
vtpci_modern_read_isr(device_t dev)1226 vtpci_modern_read_isr(device_t dev)
1227 {
1228 return (vtpci_modern_read_isr_1(device_get_softc(dev), 0));
1229 }
1230
1231 static uint16_t
vtpci_modern_get_vq_size(device_t dev,int idx)1232 vtpci_modern_get_vq_size(device_t dev, int idx)
1233 {
1234 struct vtpci_modern_softc *sc;
1235
1236 sc = device_get_softc(dev);
1237
1238 vtpci_modern_select_virtqueue(sc, idx);
1239 return (vtpci_modern_read_common_2(sc, VIRTIO_PCI_COMMON_Q_SIZE));
1240 }
1241
1242 static bus_size_t
vtpci_modern_get_vq_notify_off(device_t dev,int idx)1243 vtpci_modern_get_vq_notify_off(device_t dev, int idx)
1244 {
1245 struct vtpci_modern_softc *sc;
1246 uint16_t q_notify_off;
1247
1248 sc = device_get_softc(dev);
1249
1250 vtpci_modern_select_virtqueue(sc, idx);
1251 q_notify_off = vtpci_modern_read_common_2(sc, VIRTIO_PCI_COMMON_Q_NOFF);
1252
1253 return (q_notify_off * sc->vtpci_notify_offset_multiplier);
1254 }
1255
1256 static void
vtpci_modern_set_vq(device_t dev,struct virtqueue * vq)1257 vtpci_modern_set_vq(device_t dev, struct virtqueue *vq)
1258 {
1259 struct vtpci_modern_softc *sc;
1260
1261 sc = device_get_softc(dev);
1262
1263 vtpci_modern_select_virtqueue(sc, virtqueue_index(vq));
1264
1265 /* BMV: Currently we never adjust the device's proposed VQ size. */
1266 vtpci_modern_write_common_2(sc,
1267 VIRTIO_PCI_COMMON_Q_SIZE, virtqueue_size(vq));
1268
1269 vtpci_modern_write_common_8(sc,
1270 VIRTIO_PCI_COMMON_Q_DESCLO, virtqueue_desc_paddr(vq));
1271 vtpci_modern_write_common_8(sc,
1272 VIRTIO_PCI_COMMON_Q_AVAILLO, virtqueue_avail_paddr(vq));
1273 vtpci_modern_write_common_8(sc,
1274 VIRTIO_PCI_COMMON_Q_USEDLO, virtqueue_used_paddr(vq));
1275 }
1276
1277 static void
vtpci_modern_disable_vq(device_t dev,int idx)1278 vtpci_modern_disable_vq(device_t dev, int idx)
1279 {
1280 struct vtpci_modern_softc *sc;
1281
1282 sc = device_get_softc(dev);
1283
1284 vtpci_modern_select_virtqueue(sc, idx);
1285 vtpci_modern_write_common_8(sc, VIRTIO_PCI_COMMON_Q_DESCLO, 0ULL);
1286 vtpci_modern_write_common_8(sc, VIRTIO_PCI_COMMON_Q_AVAILLO, 0ULL);
1287 vtpci_modern_write_common_8(sc, VIRTIO_PCI_COMMON_Q_USEDLO, 0ULL);
1288 }
1289
1290 static void
vtpci_modern_enable_virtqueues(struct vtpci_modern_softc * sc)1291 vtpci_modern_enable_virtqueues(struct vtpci_modern_softc *sc)
1292 {
1293 int idx;
1294
1295 for (idx = 0; idx < sc->vtpci_common.vtpci_nvqs; idx++) {
1296 vtpci_modern_select_virtqueue(sc, idx);
1297 vtpci_modern_write_common_2(sc, VIRTIO_PCI_COMMON_Q_ENABLE, 1);
1298 }
1299 }
1300
1301 static uint8_t
vtpci_modern_read_common_1(struct vtpci_modern_softc * sc,bus_size_t off)1302 vtpci_modern_read_common_1(struct vtpci_modern_softc *sc, bus_size_t off)
1303 {
1304 return (bus_read_1(&sc->vtpci_common_res_map.vtrm_map, off));
1305 }
1306
1307 static uint16_t
vtpci_modern_read_common_2(struct vtpci_modern_softc * sc,bus_size_t off)1308 vtpci_modern_read_common_2(struct vtpci_modern_softc *sc, bus_size_t off)
1309 {
1310 return virtio_htog16(true,
1311 bus_read_2(&sc->vtpci_common_res_map.vtrm_map, off));
1312 }
1313
1314 static uint32_t
vtpci_modern_read_common_4(struct vtpci_modern_softc * sc,bus_size_t off)1315 vtpci_modern_read_common_4(struct vtpci_modern_softc *sc, bus_size_t off)
1316 {
1317 return virtio_htog32(true,
1318 bus_read_4(&sc->vtpci_common_res_map.vtrm_map, off));
1319 }
1320
1321 static void
vtpci_modern_write_common_1(struct vtpci_modern_softc * sc,bus_size_t off,uint8_t val)1322 vtpci_modern_write_common_1(struct vtpci_modern_softc *sc, bus_size_t off,
1323 uint8_t val)
1324 {
1325 bus_write_1(&sc->vtpci_common_res_map.vtrm_map, off, val);
1326 }
1327
1328 static void
vtpci_modern_write_common_2(struct vtpci_modern_softc * sc,bus_size_t off,uint16_t val)1329 vtpci_modern_write_common_2(struct vtpci_modern_softc *sc, bus_size_t off,
1330 uint16_t val)
1331 {
1332 bus_write_2(&sc->vtpci_common_res_map.vtrm_map,
1333 off, virtio_gtoh16(true, val));
1334 }
1335
1336 static void
vtpci_modern_write_common_4(struct vtpci_modern_softc * sc,bus_size_t off,uint32_t val)1337 vtpci_modern_write_common_4(struct vtpci_modern_softc *sc, bus_size_t off,
1338 uint32_t val)
1339 {
1340 bus_write_4(&sc->vtpci_common_res_map.vtrm_map,
1341 off, virtio_gtoh32(true, val));
1342 }
1343
1344 static void
vtpci_modern_write_common_8(struct vtpci_modern_softc * sc,bus_size_t off,uint64_t val)1345 vtpci_modern_write_common_8(struct vtpci_modern_softc *sc, bus_size_t off,
1346 uint64_t val)
1347 {
1348 uint32_t val0, val1;
1349
1350 val0 = (uint32_t) val;
1351 val1 = val >> 32;
1352
1353 vtpci_modern_write_common_4(sc, off, val0);
1354 vtpci_modern_write_common_4(sc, off + 4, val1);
1355 }
1356
1357 static void
vtpci_modern_write_notify_2(struct vtpci_modern_softc * sc,bus_size_t off,uint16_t val)1358 vtpci_modern_write_notify_2(struct vtpci_modern_softc *sc, bus_size_t off,
1359 uint16_t val)
1360 {
1361 bus_write_2(&sc->vtpci_notify_res_map.vtrm_map, off, val);
1362 }
1363
1364 static uint8_t
vtpci_modern_read_isr_1(struct vtpci_modern_softc * sc,bus_size_t off)1365 vtpci_modern_read_isr_1(struct vtpci_modern_softc *sc, bus_size_t off)
1366 {
1367 return (bus_read_1(&sc->vtpci_isr_res_map.vtrm_map, off));
1368 }
1369
1370 static uint8_t
vtpci_modern_read_device_1(struct vtpci_modern_softc * sc,bus_size_t off)1371 vtpci_modern_read_device_1(struct vtpci_modern_softc *sc, bus_size_t off)
1372 {
1373 return (bus_read_1(&sc->vtpci_device_res_map.vtrm_map, off));
1374 }
1375
1376 static uint16_t
vtpci_modern_read_device_2(struct vtpci_modern_softc * sc,bus_size_t off)1377 vtpci_modern_read_device_2(struct vtpci_modern_softc *sc, bus_size_t off)
1378 {
1379 return (bus_read_2(&sc->vtpci_device_res_map.vtrm_map, off));
1380 }
1381
1382 static uint32_t
vtpci_modern_read_device_4(struct vtpci_modern_softc * sc,bus_size_t off)1383 vtpci_modern_read_device_4(struct vtpci_modern_softc *sc, bus_size_t off)
1384 {
1385 return (bus_read_4(&sc->vtpci_device_res_map.vtrm_map, off));
1386 }
1387
1388 static uint64_t
vtpci_modern_read_device_8(struct vtpci_modern_softc * sc,bus_size_t off)1389 vtpci_modern_read_device_8(struct vtpci_modern_softc *sc, bus_size_t off)
1390 {
1391 device_t dev;
1392 int gen;
1393 uint32_t val0, val1;
1394
1395 dev = sc->vtpci_dev;
1396
1397 /*
1398 * Treat the 64-bit field as two 32-bit fields. Use the generation
1399 * to ensure a consistent read.
1400 */
1401 do {
1402 gen = vtpci_modern_config_generation(dev);
1403 val0 = vtpci_modern_read_device_4(sc, off);
1404 val1 = vtpci_modern_read_device_4(sc, off + 4);
1405 } while (gen != vtpci_modern_config_generation(dev));
1406
1407 return (((uint64_t) val1 << 32) | val0);
1408 }
1409
1410 static void
vtpci_modern_write_device_1(struct vtpci_modern_softc * sc,bus_size_t off,uint8_t val)1411 vtpci_modern_write_device_1(struct vtpci_modern_softc *sc, bus_size_t off,
1412 uint8_t val)
1413 {
1414 bus_write_1(&sc->vtpci_device_res_map.vtrm_map, off, val);
1415 }
1416
1417 static void
vtpci_modern_write_device_2(struct vtpci_modern_softc * sc,bus_size_t off,uint16_t val)1418 vtpci_modern_write_device_2(struct vtpci_modern_softc *sc, bus_size_t off,
1419 uint16_t val)
1420 {
1421 bus_write_2(&sc->vtpci_device_res_map.vtrm_map, off, val);
1422 }
1423
1424 static void
vtpci_modern_write_device_4(struct vtpci_modern_softc * sc,bus_size_t off,uint32_t val)1425 vtpci_modern_write_device_4(struct vtpci_modern_softc *sc, bus_size_t off,
1426 uint32_t val)
1427 {
1428 bus_write_4(&sc->vtpci_device_res_map.vtrm_map, off, val);
1429 }
1430
1431 static void
vtpci_modern_write_device_8(struct vtpci_modern_softc * sc,bus_size_t off,uint64_t val)1432 vtpci_modern_write_device_8(struct vtpci_modern_softc *sc, bus_size_t off,
1433 uint64_t val)
1434 {
1435 uint32_t val0, val1;
1436
1437 val0 = (uint32_t) val;
1438 val1 = val >> 32;
1439
1440 vtpci_modern_write_device_4(sc, off, val0);
1441 vtpci_modern_write_device_4(sc, off + 4, val1);
1442 }
1443