1 /*-
2 * SPDX-License-Identifier: BSD-2-Clause
3 *
4 * Copyright (c) 2017, Bryan Venteicher <bryanv@FreeBSD.org>
5 * All rights reserved.
6 *
7 * Redistribution and use in source and binary forms, with or without
8 * modification, are permitted provided that the following conditions
9 * are met:
10 * 1. Redistributions of source code must retain the above copyright
11 * notice unmodified, this list of conditions, and the following
12 * disclaimer.
13 * 2. Redistributions in binary form must reproduce the above copyright
14 * notice, this list of conditions and the following disclaimer in the
15 * documentation and/or other materials provided with the distribution.
16 *
17 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
18 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
19 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
20 * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
21 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
22 * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
23 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
24 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
25 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
26 * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
27 */
28
29 /* Driver for the modern VirtIO PCI interface. */
30
31 #include <sys/param.h>
32 #include <sys/systm.h>
33 #include <sys/bus.h>
34 #include <sys/lock.h>
35 #include <sys/kernel.h>
36 #include <sys/module.h>
37
38 #include <machine/bus.h>
39 #include <machine/cpu.h>
40 #include <machine/resource.h>
41 #include <sys/bus.h>
42 #include <sys/rman.h>
43
44 #include <dev/pci/pcivar.h>
45 #include <dev/pci/pcireg.h>
46
47 #include <dev/virtio/virtio.h>
48 #include <dev/virtio/virtqueue.h>
49 #include <dev/virtio/pci/virtio_pci.h>
50 #include <dev/virtio/pci/virtio_pci_modern_var.h>
51
52 #include "virtio_bus_if.h"
53 #include "virtio_pci_if.h"
54 #include "virtio_if.h"
55
56 struct vtpci_modern_resource_map {
57 struct resource_map vtrm_map;
58 int vtrm_cap_offset;
59 int vtrm_bar;
60 int vtrm_offset;
61 int vtrm_length;
62 int vtrm_type; /* SYS_RES_{MEMORY, IOPORT} */
63 };
64
65 struct vtpci_modern_bar_resource {
66 struct resource *vtbr_res;
67 int vtbr_type;
68 };
69
70 struct vtpci_modern_softc {
71 device_t vtpci_dev;
72 struct vtpci_common vtpci_common;
73 uint32_t vtpci_notify_offset_multiplier;
74 uint16_t vtpci_devid;
75 int vtpci_msix_bar;
76 struct resource *vtpci_msix_res;
77
78 struct vtpci_modern_resource_map vtpci_common_res_map;
79 struct vtpci_modern_resource_map vtpci_notify_res_map;
80 struct vtpci_modern_resource_map vtpci_isr_res_map;
81 struct vtpci_modern_resource_map vtpci_device_res_map;
82
83 #define VTPCI_MODERN_MAX_BARS 6
84 struct vtpci_modern_bar_resource vtpci_bar_res[VTPCI_MODERN_MAX_BARS];
85 };
86
87 static int vtpci_modern_probe(device_t);
88 static int vtpci_modern_attach(device_t);
89 static int vtpci_modern_detach(device_t);
90 static int vtpci_modern_suspend(device_t);
91 static int vtpci_modern_resume(device_t);
92 static int vtpci_modern_shutdown(device_t);
93
94 static void vtpci_modern_driver_added(device_t, driver_t *);
95 static void vtpci_modern_child_detached(device_t, device_t);
96 static int vtpci_modern_read_ivar(device_t, device_t, int, uintptr_t *);
97 static int vtpci_modern_write_ivar(device_t, device_t, int, uintptr_t);
98
99 static uint8_t vtpci_modern_read_isr(device_t);
100 static uint16_t vtpci_modern_get_vq_size(device_t, int);
101 static bus_size_t vtpci_modern_get_vq_notify_off(device_t, int);
102 static void vtpci_modern_set_vq(device_t, struct virtqueue *);
103 static void vtpci_modern_disable_vq(device_t, int);
104 static int vtpci_modern_register_msix(struct vtpci_modern_softc *, int,
105 struct vtpci_interrupt *);
106 static int vtpci_modern_register_cfg_msix(device_t,
107 struct vtpci_interrupt *);
108 static int vtpci_modern_register_vq_msix(device_t, int idx,
109 struct vtpci_interrupt *);
110
111 static uint64_t vtpci_modern_negotiate_features(device_t, uint64_t);
112 static int vtpci_modern_finalize_features(device_t);
113 static bool vtpci_modern_with_feature(device_t, uint64_t);
114 static int vtpci_modern_alloc_virtqueues(device_t, int,
115 struct vq_alloc_info *);
116 static int vtpci_modern_setup_interrupts(device_t, enum intr_type);
117 static void vtpci_modern_stop(device_t);
118 static int vtpci_modern_reinit(device_t, uint64_t);
119 static void vtpci_modern_reinit_complete(device_t);
120 static void vtpci_modern_notify_vq(device_t, uint16_t, bus_size_t);
121 static int vtpci_modern_config_generation(device_t);
122 static void vtpci_modern_read_dev_config(device_t, bus_size_t, void *, int);
123 static void vtpci_modern_write_dev_config(device_t, bus_size_t, const void *, int);
124
125 static int vtpci_modern_probe_configs(device_t);
126 static int vtpci_modern_find_cap(device_t, uint8_t, int *);
127 static int vtpci_modern_map_configs(struct vtpci_modern_softc *);
128 static void vtpci_modern_unmap_configs(struct vtpci_modern_softc *);
129 static int vtpci_modern_find_cap_resource(struct vtpci_modern_softc *,
130 uint8_t, int, int, struct vtpci_modern_resource_map *);
131 static int vtpci_modern_bar_type(struct vtpci_modern_softc *, int);
132 static struct resource *vtpci_modern_get_bar_resource(
133 struct vtpci_modern_softc *, int, int);
134 static struct resource *vtpci_modern_alloc_bar_resource(
135 struct vtpci_modern_softc *, int, int);
136 static void vtpci_modern_free_bar_resources(struct vtpci_modern_softc *);
137 static int vtpci_modern_alloc_resource_map(struct vtpci_modern_softc *,
138 struct vtpci_modern_resource_map *);
139 static void vtpci_modern_free_resource_map(struct vtpci_modern_softc *,
140 struct vtpci_modern_resource_map *);
141 static void vtpci_modern_alloc_msix_resource(struct vtpci_modern_softc *);
142 static void vtpci_modern_free_msix_resource(struct vtpci_modern_softc *);
143
144 static void vtpci_modern_probe_and_attach_child(struct vtpci_modern_softc *);
145
146 static uint64_t vtpci_modern_read_features(struct vtpci_modern_softc *);
147 static void vtpci_modern_write_features(struct vtpci_modern_softc *,
148 uint64_t);
149 static void vtpci_modern_select_virtqueue(struct vtpci_modern_softc *, int);
150 static uint8_t vtpci_modern_get_status(struct vtpci_modern_softc *);
151 static void vtpci_modern_set_status(struct vtpci_modern_softc *, uint8_t);
152 static void vtpci_modern_reset(struct vtpci_modern_softc *);
153 static void vtpci_modern_enable_virtqueues(struct vtpci_modern_softc *);
154
155 static uint8_t vtpci_modern_read_common_1(struct vtpci_modern_softc *,
156 bus_size_t);
157 static uint16_t vtpci_modern_read_common_2(struct vtpci_modern_softc *,
158 bus_size_t);
159 static uint32_t vtpci_modern_read_common_4(struct vtpci_modern_softc *,
160 bus_size_t);
161 static void vtpci_modern_write_common_1(struct vtpci_modern_softc *,
162 bus_size_t, uint8_t);
163 static void vtpci_modern_write_common_2(struct vtpci_modern_softc *,
164 bus_size_t, uint16_t);
165 static void vtpci_modern_write_common_4(struct vtpci_modern_softc *,
166 bus_size_t, uint32_t);
167 static void vtpci_modern_write_common_8(struct vtpci_modern_softc *,
168 bus_size_t, uint64_t);
169 static void vtpci_modern_write_notify_2(struct vtpci_modern_softc *,
170 bus_size_t, uint16_t);
171 static uint8_t vtpci_modern_read_isr_1(struct vtpci_modern_softc *,
172 bus_size_t);
173 static uint8_t vtpci_modern_read_device_1(struct vtpci_modern_softc *,
174 bus_size_t);
175 static uint16_t vtpci_modern_read_device_2(struct vtpci_modern_softc *,
176 bus_size_t);
177 static uint32_t vtpci_modern_read_device_4(struct vtpci_modern_softc *,
178 bus_size_t);
179 static uint64_t vtpci_modern_read_device_8(struct vtpci_modern_softc *,
180 bus_size_t);
181 static void vtpci_modern_write_device_1(struct vtpci_modern_softc *,
182 bus_size_t, uint8_t);
183 static void vtpci_modern_write_device_2(struct vtpci_modern_softc *,
184 bus_size_t, uint16_t);
185 static void vtpci_modern_write_device_4(struct vtpci_modern_softc *,
186 bus_size_t, uint32_t);
187 static void vtpci_modern_write_device_8(struct vtpci_modern_softc *,
188 bus_size_t, uint64_t);
189
190 /* Tunables. */
191 static int vtpci_modern_transitional = 0;
192 TUNABLE_INT("hw.virtio.pci.transitional", &vtpci_modern_transitional);
193
194 static device_method_t vtpci_modern_methods[] = {
195 /* Device interface. */
196 DEVMETHOD(device_probe, vtpci_modern_probe),
197 DEVMETHOD(device_attach, vtpci_modern_attach),
198 DEVMETHOD(device_detach, vtpci_modern_detach),
199 DEVMETHOD(device_suspend, vtpci_modern_suspend),
200 DEVMETHOD(device_resume, vtpci_modern_resume),
201 DEVMETHOD(device_shutdown, vtpci_modern_shutdown),
202
203 /* Bus interface. */
204 DEVMETHOD(bus_driver_added, vtpci_modern_driver_added),
205 DEVMETHOD(bus_child_detached, vtpci_modern_child_detached),
206 DEVMETHOD(bus_child_pnpinfo, virtio_child_pnpinfo),
207 DEVMETHOD(bus_read_ivar, vtpci_modern_read_ivar),
208 DEVMETHOD(bus_write_ivar, vtpci_modern_write_ivar),
209
210 /* VirtIO PCI interface. */
211 DEVMETHOD(virtio_pci_read_isr, vtpci_modern_read_isr),
212 DEVMETHOD(virtio_pci_get_vq_size, vtpci_modern_get_vq_size),
213 DEVMETHOD(virtio_pci_get_vq_notify_off, vtpci_modern_get_vq_notify_off),
214 DEVMETHOD(virtio_pci_set_vq, vtpci_modern_set_vq),
215 DEVMETHOD(virtio_pci_disable_vq, vtpci_modern_disable_vq),
216 DEVMETHOD(virtio_pci_register_cfg_msix, vtpci_modern_register_cfg_msix),
217 DEVMETHOD(virtio_pci_register_vq_msix, vtpci_modern_register_vq_msix),
218
219 /* VirtIO bus interface. */
220 DEVMETHOD(virtio_bus_negotiate_features, vtpci_modern_negotiate_features),
221 DEVMETHOD(virtio_bus_finalize_features, vtpci_modern_finalize_features),
222 DEVMETHOD(virtio_bus_with_feature, vtpci_modern_with_feature),
223 DEVMETHOD(virtio_bus_alloc_virtqueues, vtpci_modern_alloc_virtqueues),
224 DEVMETHOD(virtio_bus_setup_intr, vtpci_modern_setup_interrupts),
225 DEVMETHOD(virtio_bus_stop, vtpci_modern_stop),
226 DEVMETHOD(virtio_bus_reinit, vtpci_modern_reinit),
227 DEVMETHOD(virtio_bus_reinit_complete, vtpci_modern_reinit_complete),
228 DEVMETHOD(virtio_bus_notify_vq, vtpci_modern_notify_vq),
229 DEVMETHOD(virtio_bus_config_generation, vtpci_modern_config_generation),
230 DEVMETHOD(virtio_bus_read_device_config, vtpci_modern_read_dev_config),
231 DEVMETHOD(virtio_bus_write_device_config, vtpci_modern_write_dev_config),
232
233 DEVMETHOD_END
234 };
235
236 static driver_t vtpci_modern_driver = {
237 .name = "virtio_pci",
238 .methods = vtpci_modern_methods,
239 .size = sizeof(struct vtpci_modern_softc)
240 };
241
242 DRIVER_MODULE(virtio_pci_modern, pci, vtpci_modern_driver, 0, 0);
243
244 static int
vtpci_modern_probe(device_t dev)245 vtpci_modern_probe(device_t dev)
246 {
247 char desc[64];
248 const char *name;
249 uint16_t devid;
250
251 if (pci_get_vendor(dev) != VIRTIO_PCI_VENDORID)
252 return (ENXIO);
253
254 if (pci_get_device(dev) < VIRTIO_PCI_DEVICEID_MIN ||
255 pci_get_device(dev) > VIRTIO_PCI_DEVICEID_MODERN_MAX)
256 return (ENXIO);
257
258 if (pci_get_device(dev) < VIRTIO_PCI_DEVICEID_MODERN_MIN) {
259 if (!vtpci_modern_transitional)
260 return (ENXIO);
261 devid = pci_get_subdevice(dev);
262 } else
263 devid = pci_get_device(dev) - VIRTIO_PCI_DEVICEID_MODERN_MIN;
264
265 if (vtpci_modern_probe_configs(dev) != 0)
266 return (ENXIO);
267
268 name = virtio_device_name(devid);
269 if (name == NULL)
270 name = "Unknown";
271
272 snprintf(desc, sizeof(desc), "VirtIO PCI (modern) %s adapter", name);
273 device_set_desc_copy(dev, desc);
274
275 return (BUS_PROBE_DEFAULT);
276 }
277
278 static int
vtpci_modern_attach(device_t dev)279 vtpci_modern_attach(device_t dev)
280 {
281 struct vtpci_modern_softc *sc;
282 int error;
283
284 sc = device_get_softc(dev);
285 sc->vtpci_dev = dev;
286 vtpci_init(&sc->vtpci_common, dev, true);
287
288 if (pci_get_device(dev) < VIRTIO_PCI_DEVICEID_MODERN_MIN)
289 sc->vtpci_devid = pci_get_subdevice(dev);
290 else
291 sc->vtpci_devid = pci_get_device(dev) -
292 VIRTIO_PCI_DEVICEID_MODERN_MIN;
293
294 error = vtpci_modern_map_configs(sc);
295 if (error) {
296 device_printf(dev, "cannot map configs\n");
297 vtpci_modern_unmap_configs(sc);
298 return (error);
299 }
300
301 vtpci_modern_reset(sc);
302
303 /* Tell the host we've noticed this device. */
304 vtpci_modern_set_status(sc, VIRTIO_CONFIG_STATUS_ACK);
305
306 error = vtpci_add_child(&sc->vtpci_common);
307 if (error)
308 goto fail;
309
310 vtpci_modern_probe_and_attach_child(sc);
311
312 return (0);
313
314 fail:
315 vtpci_modern_set_status(sc, VIRTIO_CONFIG_STATUS_FAILED);
316 vtpci_modern_detach(dev);
317
318 return (error);
319 }
320
321 static int
vtpci_modern_detach(device_t dev)322 vtpci_modern_detach(device_t dev)
323 {
324 struct vtpci_modern_softc *sc;
325 int error;
326
327 sc = device_get_softc(dev);
328
329 error = vtpci_delete_child(&sc->vtpci_common);
330 if (error)
331 return (error);
332
333 vtpci_modern_reset(sc);
334 vtpci_modern_unmap_configs(sc);
335
336 return (0);
337 }
338
339 static int
vtpci_modern_suspend(device_t dev)340 vtpci_modern_suspend(device_t dev)
341 {
342 return (bus_generic_suspend(dev));
343 }
344
345 static int
vtpci_modern_resume(device_t dev)346 vtpci_modern_resume(device_t dev)
347 {
348 return (bus_generic_resume(dev));
349 }
350
351 static int
vtpci_modern_shutdown(device_t dev)352 vtpci_modern_shutdown(device_t dev)
353 {
354 (void) bus_generic_shutdown(dev);
355 /* Forcibly stop the host device. */
356 vtpci_modern_stop(dev);
357
358 return (0);
359 }
360
361 static void
vtpci_modern_driver_added(device_t dev,driver_t * driver)362 vtpci_modern_driver_added(device_t dev, driver_t *driver)
363 {
364 vtpci_modern_probe_and_attach_child(device_get_softc(dev));
365 }
366
367 static void
vtpci_modern_child_detached(device_t dev,device_t child)368 vtpci_modern_child_detached(device_t dev, device_t child)
369 {
370 struct vtpci_modern_softc *sc;
371
372 sc = device_get_softc(dev);
373
374 vtpci_modern_reset(sc);
375 vtpci_child_detached(&sc->vtpci_common);
376
377 /* After the reset, retell the host we've noticed this device. */
378 vtpci_modern_set_status(sc, VIRTIO_CONFIG_STATUS_ACK);
379 }
380
381 static int
vtpci_modern_read_ivar(device_t dev,device_t child,int index,uintptr_t * result)382 vtpci_modern_read_ivar(device_t dev, device_t child, int index,
383 uintptr_t *result)
384 {
385 struct vtpci_modern_softc *sc;
386 struct vtpci_common *cn;
387
388 sc = device_get_softc(dev);
389 cn = &sc->vtpci_common;
390
391 if (vtpci_child_device(cn) != child)
392 return (ENOENT);
393
394 switch (index) {
395 case VIRTIO_IVAR_DEVTYPE:
396 *result = sc->vtpci_devid;
397 break;
398 default:
399 return (vtpci_read_ivar(cn, index, result));
400 }
401
402 return (0);
403 }
404
405 static int
vtpci_modern_write_ivar(device_t dev,device_t child,int index,uintptr_t value)406 vtpci_modern_write_ivar(device_t dev, device_t child, int index,
407 uintptr_t value)
408 {
409 struct vtpci_modern_softc *sc;
410 struct vtpci_common *cn;
411
412 sc = device_get_softc(dev);
413 cn = &sc->vtpci_common;
414
415 if (vtpci_child_device(cn) != child)
416 return (ENOENT);
417
418 switch (index) {
419 default:
420 return (vtpci_write_ivar(cn, index, value));
421 }
422
423 return (0);
424 }
425
426 static uint64_t
vtpci_modern_negotiate_features(device_t dev,uint64_t child_features)427 vtpci_modern_negotiate_features(device_t dev, uint64_t child_features)
428 {
429 struct vtpci_modern_softc *sc;
430 uint64_t host_features, features;
431
432 sc = device_get_softc(dev);
433 host_features = vtpci_modern_read_features(sc);
434
435 /*
436 * Since the driver was added as a child of the modern PCI bus,
437 * always add the V1 flag.
438 */
439 child_features |= VIRTIO_F_VERSION_1;
440
441 features = vtpci_negotiate_features(&sc->vtpci_common,
442 child_features, host_features);
443 vtpci_modern_write_features(sc, features);
444
445 return (features);
446 }
447
448 static int
vtpci_modern_finalize_features(device_t dev)449 vtpci_modern_finalize_features(device_t dev)
450 {
451 struct vtpci_modern_softc *sc;
452 uint8_t status;
453
454 sc = device_get_softc(dev);
455
456 /*
457 * Must re-read the status after setting it to verify the negotiated
458 * features were accepted by the device.
459 */
460 vtpci_modern_set_status(sc, VIRTIO_CONFIG_S_FEATURES_OK);
461
462 status = vtpci_modern_get_status(sc);
463 if ((status & VIRTIO_CONFIG_S_FEATURES_OK) == 0) {
464 device_printf(dev, "desired features were not accepted\n");
465 return (ENOTSUP);
466 }
467
468 return (0);
469 }
470
471 static bool
vtpci_modern_with_feature(device_t dev,uint64_t feature)472 vtpci_modern_with_feature(device_t dev, uint64_t feature)
473 {
474 struct vtpci_modern_softc *sc;
475
476 sc = device_get_softc(dev);
477
478 return (vtpci_with_feature(&sc->vtpci_common, feature));
479 }
480
481 static uint64_t
vtpci_modern_read_features(struct vtpci_modern_softc * sc)482 vtpci_modern_read_features(struct vtpci_modern_softc *sc)
483 {
484 uint32_t features0, features1;
485
486 vtpci_modern_write_common_4(sc, VIRTIO_PCI_COMMON_DFSELECT, 0);
487 features0 = vtpci_modern_read_common_4(sc, VIRTIO_PCI_COMMON_DF);
488 vtpci_modern_write_common_4(sc, VIRTIO_PCI_COMMON_DFSELECT, 1);
489 features1 = vtpci_modern_read_common_4(sc, VIRTIO_PCI_COMMON_DF);
490
491 return (((uint64_t) features1 << 32) | features0);
492 }
493
494 static void
vtpci_modern_write_features(struct vtpci_modern_softc * sc,uint64_t features)495 vtpci_modern_write_features(struct vtpci_modern_softc *sc, uint64_t features)
496 {
497 uint32_t features0, features1;
498
499 features0 = features;
500 features1 = features >> 32;
501
502 vtpci_modern_write_common_4(sc, VIRTIO_PCI_COMMON_GFSELECT, 0);
503 vtpci_modern_write_common_4(sc, VIRTIO_PCI_COMMON_GF, features0);
504 vtpci_modern_write_common_4(sc, VIRTIO_PCI_COMMON_GFSELECT, 1);
505 vtpci_modern_write_common_4(sc, VIRTIO_PCI_COMMON_GF, features1);
506 }
507
508 static int
vtpci_modern_alloc_virtqueues(device_t dev,int nvqs,struct vq_alloc_info * vq_info)509 vtpci_modern_alloc_virtqueues(device_t dev, int nvqs,
510 struct vq_alloc_info *vq_info)
511 {
512 struct vtpci_modern_softc *sc;
513 struct vtpci_common *cn;
514 uint16_t max_nvqs;
515
516 sc = device_get_softc(dev);
517 cn = &sc->vtpci_common;
518
519 max_nvqs = vtpci_modern_read_common_2(sc, VIRTIO_PCI_COMMON_NUMQ);
520 if (nvqs > max_nvqs) {
521 device_printf(sc->vtpci_dev, "requested virtqueue count %d "
522 "exceeds max %d\n", nvqs, max_nvqs);
523 return (E2BIG);
524 }
525
526 return (vtpci_alloc_virtqueues(cn, nvqs, vq_info));
527 }
528
529 static int
vtpci_modern_setup_interrupts(device_t dev,enum intr_type type)530 vtpci_modern_setup_interrupts(device_t dev, enum intr_type type)
531 {
532 struct vtpci_modern_softc *sc;
533 int error;
534
535 sc = device_get_softc(dev);
536
537 error = vtpci_setup_interrupts(&sc->vtpci_common, type);
538 if (error == 0)
539 vtpci_modern_enable_virtqueues(sc);
540
541 return (error);
542 }
543
544 static void
vtpci_modern_stop(device_t dev)545 vtpci_modern_stop(device_t dev)
546 {
547 vtpci_modern_reset(device_get_softc(dev));
548 }
549
550 static int
vtpci_modern_reinit(device_t dev,uint64_t features)551 vtpci_modern_reinit(device_t dev, uint64_t features)
552 {
553 struct vtpci_modern_softc *sc;
554 struct vtpci_common *cn;
555 int error;
556
557 sc = device_get_softc(dev);
558 cn = &sc->vtpci_common;
559
560 /*
561 * Redrive the device initialization. This is a bit of an abuse of
562 * the specification, but VirtualBox, QEMU/KVM, and BHyVe seem to
563 * play nice.
564 *
565 * We do not allow the host device to change from what was originally
566 * negotiated beyond what the guest driver changed. MSIX state should
567 * not change, number of virtqueues and their size remain the same, etc.
568 * This will need to be rethought when we want to support migration.
569 */
570
571 if (vtpci_modern_get_status(sc) != VIRTIO_CONFIG_STATUS_RESET)
572 vtpci_modern_stop(dev);
573
574 /*
575 * Quickly drive the status through ACK and DRIVER. The device does
576 * not become usable again until DRIVER_OK in reinit complete.
577 */
578 vtpci_modern_set_status(sc, VIRTIO_CONFIG_STATUS_ACK);
579 vtpci_modern_set_status(sc, VIRTIO_CONFIG_STATUS_DRIVER);
580
581 /*
582 * TODO: Check that features are not added as to what was
583 * originally negotiated.
584 */
585 vtpci_modern_negotiate_features(dev, features);
586 error = vtpci_modern_finalize_features(dev);
587 if (error) {
588 device_printf(dev, "cannot finalize features during reinit\n");
589 return (error);
590 }
591
592 error = vtpci_reinit(cn);
593 if (error)
594 return (error);
595
596 return (0);
597 }
598
599 static void
vtpci_modern_reinit_complete(device_t dev)600 vtpci_modern_reinit_complete(device_t dev)
601 {
602 struct vtpci_modern_softc *sc;
603
604 sc = device_get_softc(dev);
605
606 vtpci_modern_enable_virtqueues(sc);
607 vtpci_modern_set_status(sc, VIRTIO_CONFIG_STATUS_DRIVER_OK);
608 }
609
610 static void
vtpci_modern_notify_vq(device_t dev,uint16_t queue,bus_size_t offset)611 vtpci_modern_notify_vq(device_t dev, uint16_t queue, bus_size_t offset)
612 {
613 struct vtpci_modern_softc *sc;
614
615 sc = device_get_softc(dev);
616
617 vtpci_modern_write_notify_2(sc, offset, queue);
618 }
619
620 static uint8_t
vtpci_modern_get_status(struct vtpci_modern_softc * sc)621 vtpci_modern_get_status(struct vtpci_modern_softc *sc)
622 {
623 return (vtpci_modern_read_common_1(sc, VIRTIO_PCI_COMMON_STATUS));
624 }
625
626 static void
vtpci_modern_set_status(struct vtpci_modern_softc * sc,uint8_t status)627 vtpci_modern_set_status(struct vtpci_modern_softc *sc, uint8_t status)
628 {
629 if (status != VIRTIO_CONFIG_STATUS_RESET)
630 status |= vtpci_modern_get_status(sc);
631
632 vtpci_modern_write_common_1(sc, VIRTIO_PCI_COMMON_STATUS, status);
633 }
634
635 static int
vtpci_modern_config_generation(device_t dev)636 vtpci_modern_config_generation(device_t dev)
637 {
638 struct vtpci_modern_softc *sc;
639 uint8_t gen;
640
641 sc = device_get_softc(dev);
642 gen = vtpci_modern_read_common_1(sc, VIRTIO_PCI_COMMON_CFGGENERATION);
643
644 return (gen);
645 }
646
647 static void
vtpci_modern_read_dev_config(device_t dev,bus_size_t offset,void * dst,int length)648 vtpci_modern_read_dev_config(device_t dev, bus_size_t offset, void *dst,
649 int length)
650 {
651 struct vtpci_modern_softc *sc;
652
653 sc = device_get_softc(dev);
654
655 if (sc->vtpci_device_res_map.vtrm_map.r_size == 0) {
656 panic("%s: attempt to read dev config but not present",
657 __func__);
658 }
659
660 switch (length) {
661 case 1:
662 *(uint8_t *) dst = vtpci_modern_read_device_1(sc, offset);
663 break;
664 case 2:
665 *(uint16_t *) dst = virtio_htog16(true,
666 vtpci_modern_read_device_2(sc, offset));
667 break;
668 case 4:
669 *(uint32_t *) dst = virtio_htog32(true,
670 vtpci_modern_read_device_4(sc, offset));
671 break;
672 case 8:
673 *(uint64_t *) dst = virtio_htog64(true,
674 vtpci_modern_read_device_8(sc, offset));
675 break;
676 default:
677 panic("%s: device %s invalid device read length %d offset %d",
678 __func__, device_get_nameunit(dev), length, (int) offset);
679 }
680 }
681
682 static void
vtpci_modern_write_dev_config(device_t dev,bus_size_t offset,const void * src,int length)683 vtpci_modern_write_dev_config(device_t dev, bus_size_t offset, const void *src,
684 int length)
685 {
686 struct vtpci_modern_softc *sc;
687
688 sc = device_get_softc(dev);
689
690 if (sc->vtpci_device_res_map.vtrm_map.r_size == 0) {
691 panic("%s: attempt to write dev config but not present",
692 __func__);
693 }
694
695 switch (length) {
696 case 1:
697 vtpci_modern_write_device_1(sc, offset, *(const uint8_t *) src);
698 break;
699 case 2: {
700 uint16_t val = virtio_gtoh16(true, *(const uint16_t *) src);
701 vtpci_modern_write_device_2(sc, offset, val);
702 break;
703 }
704 case 4: {
705 uint32_t val = virtio_gtoh32(true, *(const uint32_t *) src);
706 vtpci_modern_write_device_4(sc, offset, val);
707 break;
708 }
709 case 8: {
710 uint64_t val = virtio_gtoh64(true, *(const uint64_t *) src);
711 vtpci_modern_write_device_8(sc, offset, val);
712 break;
713 }
714 default:
715 panic("%s: device %s invalid device write length %d offset %d",
716 __func__, device_get_nameunit(dev), length, (int) offset);
717 }
718 }
719
720 static int
vtpci_modern_probe_configs(device_t dev)721 vtpci_modern_probe_configs(device_t dev)
722 {
723 int error;
724
725 /*
726 * These config capabilities must be present. The DEVICE_CFG
727 * capability is only present if the device requires it.
728 */
729
730 error = vtpci_modern_find_cap(dev, VIRTIO_PCI_CAP_COMMON_CFG, NULL);
731 if (error) {
732 device_printf(dev, "cannot find COMMON_CFG capability\n");
733 return (error);
734 }
735
736 error = vtpci_modern_find_cap(dev, VIRTIO_PCI_CAP_NOTIFY_CFG, NULL);
737 if (error) {
738 device_printf(dev, "cannot find NOTIFY_CFG capability\n");
739 return (error);
740 }
741
742 error = vtpci_modern_find_cap(dev, VIRTIO_PCI_CAP_ISR_CFG, NULL);
743 if (error) {
744 device_printf(dev, "cannot find ISR_CFG capability\n");
745 return (error);
746 }
747
748 return (0);
749 }
750
751 static int
vtpci_modern_find_cap(device_t dev,uint8_t cfg_type,int * cap_offset)752 vtpci_modern_find_cap(device_t dev, uint8_t cfg_type, int *cap_offset)
753 {
754 uint32_t type, bar;
755 int capreg, error;
756
757 for (error = pci_find_cap(dev, PCIY_VENDOR, &capreg);
758 error == 0;
759 error = pci_find_next_cap(dev, PCIY_VENDOR, capreg, &capreg)) {
760
761 type = pci_read_config(dev, capreg +
762 offsetof(struct virtio_pci_cap, cfg_type), 1);
763 bar = pci_read_config(dev, capreg +
764 offsetof(struct virtio_pci_cap, bar), 1);
765
766 /* Must ignore reserved BARs. */
767 if (bar >= VTPCI_MODERN_MAX_BARS)
768 continue;
769
770 if (type == cfg_type) {
771 if (cap_offset != NULL)
772 *cap_offset = capreg;
773 break;
774 }
775 }
776
777 return (error);
778 }
779
780 static int
vtpci_modern_map_common_config(struct vtpci_modern_softc * sc)781 vtpci_modern_map_common_config(struct vtpci_modern_softc *sc)
782 {
783 device_t dev;
784 int error;
785
786 dev = sc->vtpci_dev;
787
788 error = vtpci_modern_find_cap_resource(sc, VIRTIO_PCI_CAP_COMMON_CFG,
789 sizeof(struct virtio_pci_common_cfg), 4, &sc->vtpci_common_res_map);
790 if (error) {
791 device_printf(dev, "cannot find cap COMMON_CFG resource\n");
792 return (error);
793 }
794
795 error = vtpci_modern_alloc_resource_map(sc, &sc->vtpci_common_res_map);
796 if (error) {
797 device_printf(dev, "cannot alloc resource for COMMON_CFG\n");
798 return (error);
799 }
800
801 return (0);
802 }
803
804 static int
vtpci_modern_map_notify_config(struct vtpci_modern_softc * sc)805 vtpci_modern_map_notify_config(struct vtpci_modern_softc *sc)
806 {
807 device_t dev;
808 int cap_offset, error;
809
810 dev = sc->vtpci_dev;
811
812 error = vtpci_modern_find_cap_resource(sc, VIRTIO_PCI_CAP_NOTIFY_CFG,
813 -1, 2, &sc->vtpci_notify_res_map);
814 if (error) {
815 device_printf(dev, "cannot find cap NOTIFY_CFG resource\n");
816 return (error);
817 }
818
819 cap_offset = sc->vtpci_notify_res_map.vtrm_cap_offset;
820
821 sc->vtpci_notify_offset_multiplier = pci_read_config(dev, cap_offset +
822 offsetof(struct virtio_pci_notify_cap, notify_off_multiplier), 4);
823
824 error = vtpci_modern_alloc_resource_map(sc, &sc->vtpci_notify_res_map);
825 if (error) {
826 device_printf(dev, "cannot alloc resource for NOTIFY_CFG\n");
827 return (error);
828 }
829
830 return (0);
831 }
832
833 static int
vtpci_modern_map_isr_config(struct vtpci_modern_softc * sc)834 vtpci_modern_map_isr_config(struct vtpci_modern_softc *sc)
835 {
836 device_t dev;
837 int error;
838
839 dev = sc->vtpci_dev;
840
841 error = vtpci_modern_find_cap_resource(sc, VIRTIO_PCI_CAP_ISR_CFG,
842 sizeof(uint8_t), 1, &sc->vtpci_isr_res_map);
843 if (error) {
844 device_printf(dev, "cannot find cap ISR_CFG resource\n");
845 return (error);
846 }
847
848 error = vtpci_modern_alloc_resource_map(sc, &sc->vtpci_isr_res_map);
849 if (error) {
850 device_printf(dev, "cannot alloc resource for ISR_CFG\n");
851 return (error);
852 }
853
854 return (0);
855 }
856
857 static int
vtpci_modern_map_device_config(struct vtpci_modern_softc * sc)858 vtpci_modern_map_device_config(struct vtpci_modern_softc *sc)
859 {
860 device_t dev;
861 int error;
862
863 dev = sc->vtpci_dev;
864
865 error = vtpci_modern_find_cap_resource(sc, VIRTIO_PCI_CAP_DEVICE_CFG,
866 -1, 4, &sc->vtpci_device_res_map);
867 if (error == ENOENT) {
868 /* Device configuration is optional depending on device. */
869 return (0);
870 } else if (error) {
871 device_printf(dev, "cannot find cap DEVICE_CFG resource\n");
872 return (error);
873 }
874
875 error = vtpci_modern_alloc_resource_map(sc, &sc->vtpci_device_res_map);
876 if (error) {
877 device_printf(dev, "cannot alloc resource for DEVICE_CFG\n");
878 return (error);
879 }
880
881 return (0);
882 }
883
884 static int
vtpci_modern_map_configs(struct vtpci_modern_softc * sc)885 vtpci_modern_map_configs(struct vtpci_modern_softc *sc)
886 {
887 int error;
888
889 error = vtpci_modern_map_common_config(sc);
890 if (error)
891 return (error);
892
893 error = vtpci_modern_map_notify_config(sc);
894 if (error)
895 return (error);
896
897 error = vtpci_modern_map_isr_config(sc);
898 if (error)
899 return (error);
900
901 error = vtpci_modern_map_device_config(sc);
902 if (error)
903 return (error);
904
905 vtpci_modern_alloc_msix_resource(sc);
906
907 return (0);
908 }
909
910 static void
vtpci_modern_unmap_configs(struct vtpci_modern_softc * sc)911 vtpci_modern_unmap_configs(struct vtpci_modern_softc *sc)
912 {
913
914 vtpci_modern_free_resource_map(sc, &sc->vtpci_common_res_map);
915 vtpci_modern_free_resource_map(sc, &sc->vtpci_notify_res_map);
916 vtpci_modern_free_resource_map(sc, &sc->vtpci_isr_res_map);
917 vtpci_modern_free_resource_map(sc, &sc->vtpci_device_res_map);
918
919 vtpci_modern_free_bar_resources(sc);
920 vtpci_modern_free_msix_resource(sc);
921
922 sc->vtpci_notify_offset_multiplier = 0;
923 }
924
925 static int
vtpci_modern_find_cap_resource(struct vtpci_modern_softc * sc,uint8_t cfg_type,int min_size,int alignment,struct vtpci_modern_resource_map * res)926 vtpci_modern_find_cap_resource(struct vtpci_modern_softc *sc, uint8_t cfg_type,
927 int min_size, int alignment, struct vtpci_modern_resource_map *res)
928 {
929 device_t dev;
930 int cap_offset, offset, length, error;
931 uint8_t bar, cap_length;
932
933 dev = sc->vtpci_dev;
934
935 error = vtpci_modern_find_cap(dev, cfg_type, &cap_offset);
936 if (error)
937 return (error);
938
939 cap_length = pci_read_config(dev,
940 cap_offset + offsetof(struct virtio_pci_cap, cap_len), 1);
941
942 if (cap_length < sizeof(struct virtio_pci_cap)) {
943 device_printf(dev, "cap %u length %d less than expected\n",
944 cfg_type, cap_length);
945 return (ENXIO);
946 }
947
948 bar = pci_read_config(dev,
949 cap_offset + offsetof(struct virtio_pci_cap, bar), 1);
950 offset = pci_read_config(dev,
951 cap_offset + offsetof(struct virtio_pci_cap, offset), 4);
952 length = pci_read_config(dev,
953 cap_offset + offsetof(struct virtio_pci_cap, length), 4);
954
955 if (min_size != -1 && length < min_size) {
956 device_printf(dev, "cap %u struct length %d less than min %d\n",
957 cfg_type, length, min_size);
958 return (ENXIO);
959 }
960
961 if (offset % alignment) {
962 device_printf(dev, "cap %u struct offset %d not aligned to %d\n",
963 cfg_type, offset, alignment);
964 return (ENXIO);
965 }
966
967 /* BMV: TODO Can we determine the size of the BAR here? */
968
969 res->vtrm_cap_offset = cap_offset;
970 res->vtrm_bar = bar;
971 res->vtrm_offset = offset;
972 res->vtrm_length = length;
973 res->vtrm_type = vtpci_modern_bar_type(sc, bar);
974
975 return (0);
976 }
977
978 static int
vtpci_modern_bar_type(struct vtpci_modern_softc * sc,int bar)979 vtpci_modern_bar_type(struct vtpci_modern_softc *sc, int bar)
980 {
981 uint32_t val;
982
983 /*
984 * The BAR described by a config capability may be either an IOPORT or
985 * MEM, but we must know the type when calling bus_alloc_resource().
986 */
987 val = pci_read_config(sc->vtpci_dev, PCIR_BAR(bar), 4);
988 if (PCI_BAR_IO(val))
989 return (SYS_RES_IOPORT);
990 else
991 return (SYS_RES_MEMORY);
992 }
993
994 static struct resource *
vtpci_modern_get_bar_resource(struct vtpci_modern_softc * sc,int bar,int type)995 vtpci_modern_get_bar_resource(struct vtpci_modern_softc *sc, int bar, int type)
996 {
997 struct resource *res;
998
999 MPASS(bar >= 0 && bar < VTPCI_MODERN_MAX_BARS);
1000 res = sc->vtpci_bar_res[bar].vtbr_res;
1001 MPASS(res == NULL || sc->vtpci_bar_res[bar].vtbr_type == type);
1002
1003 return (res);
1004 }
1005
1006 static struct resource *
vtpci_modern_alloc_bar_resource(struct vtpci_modern_softc * sc,int bar,int type)1007 vtpci_modern_alloc_bar_resource(struct vtpci_modern_softc *sc, int bar,
1008 int type)
1009 {
1010 struct resource *res;
1011 int rid;
1012
1013 MPASS(bar >= 0 && bar < VTPCI_MODERN_MAX_BARS);
1014 MPASS(type == SYS_RES_MEMORY || type == SYS_RES_IOPORT);
1015
1016 res = sc->vtpci_bar_res[bar].vtbr_res;
1017 if (res != NULL) {
1018 MPASS(sc->vtpci_bar_res[bar].vtbr_type == type);
1019 return (res);
1020 }
1021
1022 rid = PCIR_BAR(bar);
1023 res = bus_alloc_resource_any(sc->vtpci_dev, type, &rid,
1024 RF_ACTIVE | RF_UNMAPPED);
1025 if (res != NULL) {
1026 sc->vtpci_bar_res[bar].vtbr_res = res;
1027 sc->vtpci_bar_res[bar].vtbr_type = type;
1028 }
1029
1030 return (res);
1031 }
1032
1033 static void
vtpci_modern_free_bar_resources(struct vtpci_modern_softc * sc)1034 vtpci_modern_free_bar_resources(struct vtpci_modern_softc *sc)
1035 {
1036 device_t dev;
1037 struct resource *res;
1038 int bar, rid, type;
1039
1040 dev = sc->vtpci_dev;
1041
1042 for (bar = 0; bar < VTPCI_MODERN_MAX_BARS; bar++) {
1043 res = sc->vtpci_bar_res[bar].vtbr_res;
1044 type = sc->vtpci_bar_res[bar].vtbr_type;
1045
1046 if (res != NULL) {
1047 rid = PCIR_BAR(bar);
1048 bus_release_resource(dev, type, rid, res);
1049 sc->vtpci_bar_res[bar].vtbr_res = NULL;
1050 sc->vtpci_bar_res[bar].vtbr_type = 0;
1051 }
1052 }
1053 }
1054
1055 static int
vtpci_modern_alloc_resource_map(struct vtpci_modern_softc * sc,struct vtpci_modern_resource_map * map)1056 vtpci_modern_alloc_resource_map(struct vtpci_modern_softc *sc,
1057 struct vtpci_modern_resource_map *map)
1058 {
1059 struct resource_map_request req;
1060 struct resource *res;
1061 int type;
1062
1063 type = map->vtrm_type;
1064
1065 res = vtpci_modern_alloc_bar_resource(sc, map->vtrm_bar, type);
1066 if (res == NULL)
1067 return (ENXIO);
1068
1069 resource_init_map_request(&req);
1070 req.offset = map->vtrm_offset;
1071 req.length = map->vtrm_length;
1072
1073 return (bus_map_resource(sc->vtpci_dev, type, res, &req,
1074 &map->vtrm_map));
1075 }
1076
1077 static void
vtpci_modern_free_resource_map(struct vtpci_modern_softc * sc,struct vtpci_modern_resource_map * map)1078 vtpci_modern_free_resource_map(struct vtpci_modern_softc *sc,
1079 struct vtpci_modern_resource_map *map)
1080 {
1081 struct resource *res;
1082 int type;
1083
1084 type = map->vtrm_type;
1085 res = vtpci_modern_get_bar_resource(sc, map->vtrm_bar, type);
1086
1087 if (res != NULL && map->vtrm_map.r_size != 0) {
1088 bus_unmap_resource(sc->vtpci_dev, type, res, &map->vtrm_map);
1089 bzero(map, sizeof(struct vtpci_modern_resource_map));
1090 }
1091 }
1092
1093 static void
vtpci_modern_alloc_msix_resource(struct vtpci_modern_softc * sc)1094 vtpci_modern_alloc_msix_resource(struct vtpci_modern_softc *sc)
1095 {
1096 device_t dev;
1097 int bar;
1098
1099 dev = sc->vtpci_dev;
1100
1101 if (!vtpci_is_msix_available(&sc->vtpci_common) ||
1102 (bar = pci_msix_table_bar(dev)) == -1)
1103 return;
1104
1105 /* TODO: Can this BAR be in the 0-5 range? */
1106 sc->vtpci_msix_bar = bar;
1107 if ((sc->vtpci_msix_res = bus_alloc_resource_any(dev, SYS_RES_MEMORY,
1108 &bar, RF_ACTIVE)) == NULL)
1109 device_printf(dev, "Unable to map MSIX table\n");
1110 }
1111
1112 static void
vtpci_modern_free_msix_resource(struct vtpci_modern_softc * sc)1113 vtpci_modern_free_msix_resource(struct vtpci_modern_softc *sc)
1114 {
1115 device_t dev;
1116
1117 dev = sc->vtpci_dev;
1118
1119 if (sc->vtpci_msix_res != NULL) {
1120 bus_release_resource(dev, SYS_RES_MEMORY, sc->vtpci_msix_bar,
1121 sc->vtpci_msix_res);
1122 sc->vtpci_msix_bar = 0;
1123 sc->vtpci_msix_res = NULL;
1124 }
1125 }
1126
1127 static void
vtpci_modern_probe_and_attach_child(struct vtpci_modern_softc * sc)1128 vtpci_modern_probe_and_attach_child(struct vtpci_modern_softc *sc)
1129 {
1130 device_t dev, child;
1131
1132 dev = sc->vtpci_dev;
1133 child = vtpci_child_device(&sc->vtpci_common);
1134
1135 if (child == NULL || device_get_state(child) != DS_NOTPRESENT)
1136 return;
1137
1138 if (device_probe(child) != 0)
1139 return;
1140
1141 vtpci_modern_set_status(sc, VIRTIO_CONFIG_STATUS_DRIVER);
1142
1143 if (device_attach(child) != 0) {
1144 vtpci_modern_set_status(sc, VIRTIO_CONFIG_STATUS_FAILED);
1145 /* Reset state for later attempt. */
1146 vtpci_modern_child_detached(dev, child);
1147 } else {
1148 vtpci_modern_set_status(sc, VIRTIO_CONFIG_STATUS_DRIVER_OK);
1149 VIRTIO_ATTACH_COMPLETED(child);
1150 }
1151 }
1152
1153 static int
vtpci_modern_register_msix(struct vtpci_modern_softc * sc,int offset,struct vtpci_interrupt * intr)1154 vtpci_modern_register_msix(struct vtpci_modern_softc *sc, int offset,
1155 struct vtpci_interrupt *intr)
1156 {
1157 uint16_t vector;
1158
1159 if (intr != NULL) {
1160 /* Map from guest rid to host vector. */
1161 vector = intr->vti_rid - 1;
1162 } else
1163 vector = VIRTIO_MSI_NO_VECTOR;
1164
1165 vtpci_modern_write_common_2(sc, offset, vector);
1166 return (vtpci_modern_read_common_2(sc, offset) == vector ? 0 : ENODEV);
1167 }
1168
1169 static int
vtpci_modern_register_cfg_msix(device_t dev,struct vtpci_interrupt * intr)1170 vtpci_modern_register_cfg_msix(device_t dev, struct vtpci_interrupt *intr)
1171 {
1172 struct vtpci_modern_softc *sc;
1173 int error;
1174
1175 sc = device_get_softc(dev);
1176
1177 error = vtpci_modern_register_msix(sc, VIRTIO_PCI_COMMON_MSIX, intr);
1178 if (error) {
1179 device_printf(dev,
1180 "unable to register config MSIX interrupt\n");
1181 return (error);
1182 }
1183
1184 return (0);
1185 }
1186
1187 static int
vtpci_modern_register_vq_msix(device_t dev,int idx,struct vtpci_interrupt * intr)1188 vtpci_modern_register_vq_msix(device_t dev, int idx,
1189 struct vtpci_interrupt *intr)
1190 {
1191 struct vtpci_modern_softc *sc;
1192 int error;
1193
1194 sc = device_get_softc(dev);
1195
1196 vtpci_modern_select_virtqueue(sc, idx);
1197 error = vtpci_modern_register_msix(sc, VIRTIO_PCI_COMMON_Q_MSIX, intr);
1198 if (error) {
1199 device_printf(dev,
1200 "unable to register virtqueue MSIX interrupt\n");
1201 return (error);
1202 }
1203
1204 return (0);
1205 }
1206
1207 static void
vtpci_modern_reset(struct vtpci_modern_softc * sc)1208 vtpci_modern_reset(struct vtpci_modern_softc *sc)
1209 {
1210 /*
1211 * Setting the status to RESET sets the host device to the
1212 * original, uninitialized state. Must poll the status until
1213 * the reset is complete.
1214 */
1215 vtpci_modern_set_status(sc, VIRTIO_CONFIG_STATUS_RESET);
1216
1217 while (vtpci_modern_get_status(sc) != VIRTIO_CONFIG_STATUS_RESET)
1218 cpu_spinwait();
1219 }
1220
1221 static void
vtpci_modern_select_virtqueue(struct vtpci_modern_softc * sc,int idx)1222 vtpci_modern_select_virtqueue(struct vtpci_modern_softc *sc, int idx)
1223 {
1224 vtpci_modern_write_common_2(sc, VIRTIO_PCI_COMMON_Q_SELECT, idx);
1225 }
1226
1227 static uint8_t
vtpci_modern_read_isr(device_t dev)1228 vtpci_modern_read_isr(device_t dev)
1229 {
1230 return (vtpci_modern_read_isr_1(device_get_softc(dev), 0));
1231 }
1232
1233 static uint16_t
vtpci_modern_get_vq_size(device_t dev,int idx)1234 vtpci_modern_get_vq_size(device_t dev, int idx)
1235 {
1236 struct vtpci_modern_softc *sc;
1237
1238 sc = device_get_softc(dev);
1239
1240 vtpci_modern_select_virtqueue(sc, idx);
1241 return (vtpci_modern_read_common_2(sc, VIRTIO_PCI_COMMON_Q_SIZE));
1242 }
1243
1244 static bus_size_t
vtpci_modern_get_vq_notify_off(device_t dev,int idx)1245 vtpci_modern_get_vq_notify_off(device_t dev, int idx)
1246 {
1247 struct vtpci_modern_softc *sc;
1248 uint16_t q_notify_off;
1249
1250 sc = device_get_softc(dev);
1251
1252 vtpci_modern_select_virtqueue(sc, idx);
1253 q_notify_off = vtpci_modern_read_common_2(sc, VIRTIO_PCI_COMMON_Q_NOFF);
1254
1255 return (q_notify_off * sc->vtpci_notify_offset_multiplier);
1256 }
1257
1258 static void
vtpci_modern_set_vq(device_t dev,struct virtqueue * vq)1259 vtpci_modern_set_vq(device_t dev, struct virtqueue *vq)
1260 {
1261 struct vtpci_modern_softc *sc;
1262
1263 sc = device_get_softc(dev);
1264
1265 vtpci_modern_select_virtqueue(sc, virtqueue_index(vq));
1266
1267 /* BMV: Currently we never adjust the device's proposed VQ size. */
1268 vtpci_modern_write_common_2(sc,
1269 VIRTIO_PCI_COMMON_Q_SIZE, virtqueue_size(vq));
1270
1271 vtpci_modern_write_common_8(sc,
1272 VIRTIO_PCI_COMMON_Q_DESCLO, virtqueue_desc_paddr(vq));
1273 vtpci_modern_write_common_8(sc,
1274 VIRTIO_PCI_COMMON_Q_AVAILLO, virtqueue_avail_paddr(vq));
1275 vtpci_modern_write_common_8(sc,
1276 VIRTIO_PCI_COMMON_Q_USEDLO, virtqueue_used_paddr(vq));
1277 }
1278
1279 static void
vtpci_modern_disable_vq(device_t dev,int idx)1280 vtpci_modern_disable_vq(device_t dev, int idx)
1281 {
1282 struct vtpci_modern_softc *sc;
1283
1284 sc = device_get_softc(dev);
1285
1286 vtpci_modern_select_virtqueue(sc, idx);
1287 vtpci_modern_write_common_8(sc, VIRTIO_PCI_COMMON_Q_DESCLO, 0ULL);
1288 vtpci_modern_write_common_8(sc, VIRTIO_PCI_COMMON_Q_AVAILLO, 0ULL);
1289 vtpci_modern_write_common_8(sc, VIRTIO_PCI_COMMON_Q_USEDLO, 0ULL);
1290 }
1291
1292 static void
vtpci_modern_enable_virtqueues(struct vtpci_modern_softc * sc)1293 vtpci_modern_enable_virtqueues(struct vtpci_modern_softc *sc)
1294 {
1295 int idx;
1296
1297 for (idx = 0; idx < sc->vtpci_common.vtpci_nvqs; idx++) {
1298 vtpci_modern_select_virtqueue(sc, idx);
1299 vtpci_modern_write_common_2(sc, VIRTIO_PCI_COMMON_Q_ENABLE, 1);
1300 }
1301 }
1302
1303 static uint8_t
vtpci_modern_read_common_1(struct vtpci_modern_softc * sc,bus_size_t off)1304 vtpci_modern_read_common_1(struct vtpci_modern_softc *sc, bus_size_t off)
1305 {
1306 return (bus_read_1(&sc->vtpci_common_res_map.vtrm_map, off));
1307 }
1308
1309 static uint16_t
vtpci_modern_read_common_2(struct vtpci_modern_softc * sc,bus_size_t off)1310 vtpci_modern_read_common_2(struct vtpci_modern_softc *sc, bus_size_t off)
1311 {
1312 return virtio_htog16(true,
1313 bus_read_2(&sc->vtpci_common_res_map.vtrm_map, off));
1314 }
1315
1316 static uint32_t
vtpci_modern_read_common_4(struct vtpci_modern_softc * sc,bus_size_t off)1317 vtpci_modern_read_common_4(struct vtpci_modern_softc *sc, bus_size_t off)
1318 {
1319 return virtio_htog32(true,
1320 bus_read_4(&sc->vtpci_common_res_map.vtrm_map, off));
1321 }
1322
1323 static void
vtpci_modern_write_common_1(struct vtpci_modern_softc * sc,bus_size_t off,uint8_t val)1324 vtpci_modern_write_common_1(struct vtpci_modern_softc *sc, bus_size_t off,
1325 uint8_t val)
1326 {
1327 bus_write_1(&sc->vtpci_common_res_map.vtrm_map, off, val);
1328 }
1329
1330 static void
vtpci_modern_write_common_2(struct vtpci_modern_softc * sc,bus_size_t off,uint16_t val)1331 vtpci_modern_write_common_2(struct vtpci_modern_softc *sc, bus_size_t off,
1332 uint16_t val)
1333 {
1334 bus_write_2(&sc->vtpci_common_res_map.vtrm_map,
1335 off, virtio_gtoh16(true, val));
1336 }
1337
1338 static void
vtpci_modern_write_common_4(struct vtpci_modern_softc * sc,bus_size_t off,uint32_t val)1339 vtpci_modern_write_common_4(struct vtpci_modern_softc *sc, bus_size_t off,
1340 uint32_t val)
1341 {
1342 bus_write_4(&sc->vtpci_common_res_map.vtrm_map,
1343 off, virtio_gtoh32(true, val));
1344 }
1345
1346 static void
vtpci_modern_write_common_8(struct vtpci_modern_softc * sc,bus_size_t off,uint64_t val)1347 vtpci_modern_write_common_8(struct vtpci_modern_softc *sc, bus_size_t off,
1348 uint64_t val)
1349 {
1350 uint32_t val0, val1;
1351
1352 val0 = (uint32_t) val;
1353 val1 = val >> 32;
1354
1355 vtpci_modern_write_common_4(sc, off, val0);
1356 vtpci_modern_write_common_4(sc, off + 4, val1);
1357 }
1358
1359 static void
vtpci_modern_write_notify_2(struct vtpci_modern_softc * sc,bus_size_t off,uint16_t val)1360 vtpci_modern_write_notify_2(struct vtpci_modern_softc *sc, bus_size_t off,
1361 uint16_t val)
1362 {
1363 bus_write_2(&sc->vtpci_notify_res_map.vtrm_map, off, val);
1364 }
1365
1366 static uint8_t
vtpci_modern_read_isr_1(struct vtpci_modern_softc * sc,bus_size_t off)1367 vtpci_modern_read_isr_1(struct vtpci_modern_softc *sc, bus_size_t off)
1368 {
1369 return (bus_read_1(&sc->vtpci_isr_res_map.vtrm_map, off));
1370 }
1371
1372 static uint8_t
vtpci_modern_read_device_1(struct vtpci_modern_softc * sc,bus_size_t off)1373 vtpci_modern_read_device_1(struct vtpci_modern_softc *sc, bus_size_t off)
1374 {
1375 return (bus_read_1(&sc->vtpci_device_res_map.vtrm_map, off));
1376 }
1377
1378 static uint16_t
vtpci_modern_read_device_2(struct vtpci_modern_softc * sc,bus_size_t off)1379 vtpci_modern_read_device_2(struct vtpci_modern_softc *sc, bus_size_t off)
1380 {
1381 return (bus_read_2(&sc->vtpci_device_res_map.vtrm_map, off));
1382 }
1383
1384 static uint32_t
vtpci_modern_read_device_4(struct vtpci_modern_softc * sc,bus_size_t off)1385 vtpci_modern_read_device_4(struct vtpci_modern_softc *sc, bus_size_t off)
1386 {
1387 return (bus_read_4(&sc->vtpci_device_res_map.vtrm_map, off));
1388 }
1389
1390 static uint64_t
vtpci_modern_read_device_8(struct vtpci_modern_softc * sc,bus_size_t off)1391 vtpci_modern_read_device_8(struct vtpci_modern_softc *sc, bus_size_t off)
1392 {
1393 device_t dev;
1394 int gen;
1395 uint32_t val0, val1;
1396
1397 dev = sc->vtpci_dev;
1398
1399 /*
1400 * Treat the 64-bit field as two 32-bit fields. Use the generation
1401 * to ensure a consistent read.
1402 */
1403 do {
1404 gen = vtpci_modern_config_generation(dev);
1405 val0 = vtpci_modern_read_device_4(sc, off);
1406 val1 = vtpci_modern_read_device_4(sc, off + 4);
1407 } while (gen != vtpci_modern_config_generation(dev));
1408
1409 return (((uint64_t) val1 << 32) | val0);
1410 }
1411
1412 static void
vtpci_modern_write_device_1(struct vtpci_modern_softc * sc,bus_size_t off,uint8_t val)1413 vtpci_modern_write_device_1(struct vtpci_modern_softc *sc, bus_size_t off,
1414 uint8_t val)
1415 {
1416 bus_write_1(&sc->vtpci_device_res_map.vtrm_map, off, val);
1417 }
1418
1419 static void
vtpci_modern_write_device_2(struct vtpci_modern_softc * sc,bus_size_t off,uint16_t val)1420 vtpci_modern_write_device_2(struct vtpci_modern_softc *sc, bus_size_t off,
1421 uint16_t val)
1422 {
1423 bus_write_2(&sc->vtpci_device_res_map.vtrm_map, off, val);
1424 }
1425
1426 static void
vtpci_modern_write_device_4(struct vtpci_modern_softc * sc,bus_size_t off,uint32_t val)1427 vtpci_modern_write_device_4(struct vtpci_modern_softc *sc, bus_size_t off,
1428 uint32_t val)
1429 {
1430 bus_write_4(&sc->vtpci_device_res_map.vtrm_map, off, val);
1431 }
1432
1433 static void
vtpci_modern_write_device_8(struct vtpci_modern_softc * sc,bus_size_t off,uint64_t val)1434 vtpci_modern_write_device_8(struct vtpci_modern_softc *sc, bus_size_t off,
1435 uint64_t val)
1436 {
1437 uint32_t val0, val1;
1438
1439 val0 = (uint32_t) val;
1440 val1 = val >> 32;
1441
1442 vtpci_modern_write_device_4(sc, off, val0);
1443 vtpci_modern_write_device_4(sc, off + 4, val1);
1444 }
1445