1 // SPDX-License-Identifier: GPL-2.0-or-later 2 /* 3 * Virtio PCI driver - modern (virtio 1.0) device support 4 * 5 * This module allows virtio devices to be used over a virtual PCI device. 6 * This can be used with QEMU based VMMs like KVM or Xen. 7 * 8 * Copyright IBM Corp. 2007 9 * Copyright Red Hat, Inc. 2014 10 * 11 * Authors: 12 * Anthony Liguori <aliguori@us.ibm.com> 13 * Rusty Russell <rusty@rustcorp.com.au> 14 * Michael S. Tsirkin <mst@redhat.com> 15 */ 16 17 #include <linux/delay.h> 18 #define VIRTIO_PCI_NO_LEGACY 19 #define VIRTIO_RING_NO_LEGACY 20 #include "virtio_pci_common.h" 21 22 static u64 vp_get_features(struct virtio_device *vdev) 23 { 24 struct virtio_pci_device *vp_dev = to_vp_device(vdev); 25 26 return vp_modern_get_features(&vp_dev->mdev); 27 } 28 29 static void vp_transport_features(struct virtio_device *vdev, u64 features) 30 { 31 struct virtio_pci_device *vp_dev = to_vp_device(vdev); 32 struct pci_dev *pci_dev = vp_dev->pci_dev; 33 34 if ((features & BIT_ULL(VIRTIO_F_SR_IOV)) && 35 pci_find_ext_capability(pci_dev, PCI_EXT_CAP_ID_SRIOV)) 36 __virtio_set_bit(vdev, VIRTIO_F_SR_IOV); 37 38 if (features & BIT_ULL(VIRTIO_F_RING_RESET)) 39 __virtio_set_bit(vdev, VIRTIO_F_RING_RESET); 40 } 41 42 /* virtio config->finalize_features() implementation */ 43 static int vp_finalize_features(struct virtio_device *vdev) 44 { 45 struct virtio_pci_device *vp_dev = to_vp_device(vdev); 46 u64 features = vdev->features; 47 48 /* Give virtio_ring a chance to accept features. */ 49 vring_transport_features(vdev); 50 51 /* Give virtio_pci a chance to accept features. */ 52 vp_transport_features(vdev, features); 53 54 if (!__virtio_test_bit(vdev, VIRTIO_F_VERSION_1)) { 55 dev_err(&vdev->dev, "virtio: device uses modern interface " 56 "but does not have VIRTIO_F_VERSION_1\n"); 57 return -EINVAL; 58 } 59 60 vp_modern_set_features(&vp_dev->mdev, vdev->features); 61 62 return 0; 63 } 64 65 /* virtio config->get() implementation */ 66 static void vp_get(struct virtio_device *vdev, unsigned int offset, 67 void *buf, unsigned int len) 68 { 69 struct virtio_pci_device *vp_dev = to_vp_device(vdev); 70 struct virtio_pci_modern_device *mdev = &vp_dev->mdev; 71 void __iomem *device = mdev->device; 72 u8 b; 73 __le16 w; 74 __le32 l; 75 76 BUG_ON(offset + len > mdev->device_len); 77 78 switch (len) { 79 case 1: 80 b = ioread8(device + offset); 81 memcpy(buf, &b, sizeof b); 82 break; 83 case 2: 84 w = cpu_to_le16(ioread16(device + offset)); 85 memcpy(buf, &w, sizeof w); 86 break; 87 case 4: 88 l = cpu_to_le32(ioread32(device + offset)); 89 memcpy(buf, &l, sizeof l); 90 break; 91 case 8: 92 l = cpu_to_le32(ioread32(device + offset)); 93 memcpy(buf, &l, sizeof l); 94 l = cpu_to_le32(ioread32(device + offset + sizeof l)); 95 memcpy(buf + sizeof l, &l, sizeof l); 96 break; 97 default: 98 BUG(); 99 } 100 } 101 102 /* the config->set() implementation. it's symmetric to the config->get() 103 * implementation */ 104 static void vp_set(struct virtio_device *vdev, unsigned int offset, 105 const void *buf, unsigned int len) 106 { 107 struct virtio_pci_device *vp_dev = to_vp_device(vdev); 108 struct virtio_pci_modern_device *mdev = &vp_dev->mdev; 109 void __iomem *device = mdev->device; 110 u8 b; 111 __le16 w; 112 __le32 l; 113 114 BUG_ON(offset + len > mdev->device_len); 115 116 switch (len) { 117 case 1: 118 memcpy(&b, buf, sizeof b); 119 iowrite8(b, device + offset); 120 break; 121 case 2: 122 memcpy(&w, buf, sizeof w); 123 iowrite16(le16_to_cpu(w), device + offset); 124 break; 125 case 4: 126 memcpy(&l, buf, sizeof l); 127 iowrite32(le32_to_cpu(l), device + offset); 128 break; 129 case 8: 130 memcpy(&l, buf, sizeof l); 131 iowrite32(le32_to_cpu(l), device + offset); 132 memcpy(&l, buf + sizeof l, sizeof l); 133 iowrite32(le32_to_cpu(l), device + offset + sizeof l); 134 break; 135 default: 136 BUG(); 137 } 138 } 139 140 static u32 vp_generation(struct virtio_device *vdev) 141 { 142 struct virtio_pci_device *vp_dev = to_vp_device(vdev); 143 144 return vp_modern_generation(&vp_dev->mdev); 145 } 146 147 /* config->{get,set}_status() implementations */ 148 static u8 vp_get_status(struct virtio_device *vdev) 149 { 150 struct virtio_pci_device *vp_dev = to_vp_device(vdev); 151 152 return vp_modern_get_status(&vp_dev->mdev); 153 } 154 155 static void vp_set_status(struct virtio_device *vdev, u8 status) 156 { 157 struct virtio_pci_device *vp_dev = to_vp_device(vdev); 158 159 /* We should never be setting status to 0. */ 160 BUG_ON(status == 0); 161 vp_modern_set_status(&vp_dev->mdev, status); 162 } 163 164 static void vp_reset(struct virtio_device *vdev) 165 { 166 struct virtio_pci_device *vp_dev = to_vp_device(vdev); 167 struct virtio_pci_modern_device *mdev = &vp_dev->mdev; 168 169 /* 0 status means a reset. */ 170 vp_modern_set_status(mdev, 0); 171 /* After writing 0 to device_status, the driver MUST wait for a read of 172 * device_status to return 0 before reinitializing the device. 173 * This will flush out the status write, and flush in device writes, 174 * including MSI-X interrupts, if any. 175 */ 176 while (vp_modern_get_status(mdev)) 177 msleep(1); 178 /* Flush pending VQ/configuration callbacks. */ 179 vp_synchronize_vectors(vdev); 180 } 181 182 static int vp_active_vq(struct virtqueue *vq, u16 msix_vec) 183 { 184 struct virtio_pci_device *vp_dev = to_vp_device(vq->vdev); 185 struct virtio_pci_modern_device *mdev = &vp_dev->mdev; 186 unsigned long index; 187 188 index = vq->index; 189 190 /* activate the queue */ 191 vp_modern_set_queue_size(mdev, index, virtqueue_get_vring_size(vq)); 192 vp_modern_queue_address(mdev, index, virtqueue_get_desc_addr(vq), 193 virtqueue_get_avail_addr(vq), 194 virtqueue_get_used_addr(vq)); 195 196 if (msix_vec != VIRTIO_MSI_NO_VECTOR) { 197 msix_vec = vp_modern_queue_vector(mdev, index, msix_vec); 198 if (msix_vec == VIRTIO_MSI_NO_VECTOR) 199 return -EBUSY; 200 } 201 202 return 0; 203 } 204 205 static int vp_modern_disable_vq_and_reset(struct virtqueue *vq) 206 { 207 struct virtio_pci_device *vp_dev = to_vp_device(vq->vdev); 208 struct virtio_pci_modern_device *mdev = &vp_dev->mdev; 209 struct virtio_pci_vq_info *info; 210 unsigned long flags; 211 212 if (!virtio_has_feature(vq->vdev, VIRTIO_F_RING_RESET)) 213 return -ENOENT; 214 215 vp_modern_set_queue_reset(mdev, vq->index); 216 217 info = vp_dev->vqs[vq->index]; 218 219 /* delete vq from irq handler */ 220 spin_lock_irqsave(&vp_dev->lock, flags); 221 list_del(&info->node); 222 spin_unlock_irqrestore(&vp_dev->lock, flags); 223 224 INIT_LIST_HEAD(&info->node); 225 226 #ifdef CONFIG_VIRTIO_HARDEN_NOTIFICATION 227 __virtqueue_break(vq); 228 #endif 229 230 /* For the case where vq has an exclusive irq, call synchronize_irq() to 231 * wait for completion. 232 * 233 * note: We can't use disable_irq() since it conflicts with the affinity 234 * managed IRQ that is used by some drivers. 235 */ 236 if (vp_dev->per_vq_vectors && info->msix_vector != VIRTIO_MSI_NO_VECTOR) 237 synchronize_irq(pci_irq_vector(vp_dev->pci_dev, info->msix_vector)); 238 239 vq->reset = true; 240 241 return 0; 242 } 243 244 static int vp_modern_enable_vq_after_reset(struct virtqueue *vq) 245 { 246 struct virtio_pci_device *vp_dev = to_vp_device(vq->vdev); 247 struct virtio_pci_modern_device *mdev = &vp_dev->mdev; 248 struct virtio_pci_vq_info *info; 249 unsigned long flags, index; 250 int err; 251 252 if (!vq->reset) 253 return -EBUSY; 254 255 index = vq->index; 256 info = vp_dev->vqs[index]; 257 258 if (vp_modern_get_queue_reset(mdev, index)) 259 return -EBUSY; 260 261 if (vp_modern_get_queue_enable(mdev, index)) 262 return -EBUSY; 263 264 err = vp_active_vq(vq, info->msix_vector); 265 if (err) 266 return err; 267 268 if (vq->callback) { 269 spin_lock_irqsave(&vp_dev->lock, flags); 270 list_add(&info->node, &vp_dev->virtqueues); 271 spin_unlock_irqrestore(&vp_dev->lock, flags); 272 } else { 273 INIT_LIST_HEAD(&info->node); 274 } 275 276 #ifdef CONFIG_VIRTIO_HARDEN_NOTIFICATION 277 __virtqueue_unbreak(vq); 278 #endif 279 280 vp_modern_set_queue_enable(&vp_dev->mdev, index, true); 281 vq->reset = false; 282 283 return 0; 284 } 285 286 static u16 vp_config_vector(struct virtio_pci_device *vp_dev, u16 vector) 287 { 288 return vp_modern_config_vector(&vp_dev->mdev, vector); 289 } 290 291 static bool vp_notify_with_data(struct virtqueue *vq) 292 { 293 u32 data = vring_notification_data(vq); 294 295 iowrite32(data, (void __iomem *)vq->priv); 296 297 return true; 298 } 299 300 static struct virtqueue *setup_vq(struct virtio_pci_device *vp_dev, 301 struct virtio_pci_vq_info *info, 302 unsigned int index, 303 void (*callback)(struct virtqueue *vq), 304 const char *name, 305 bool ctx, 306 u16 msix_vec) 307 { 308 309 struct virtio_pci_modern_device *mdev = &vp_dev->mdev; 310 bool (*notify)(struct virtqueue *vq); 311 struct virtqueue *vq; 312 u16 num; 313 int err; 314 315 if (__virtio_test_bit(&vp_dev->vdev, VIRTIO_F_NOTIFICATION_DATA)) 316 notify = vp_notify_with_data; 317 else 318 notify = vp_notify; 319 320 if (index >= vp_modern_get_num_queues(mdev)) 321 return ERR_PTR(-EINVAL); 322 323 /* Check if queue is either not available or already active. */ 324 num = vp_modern_get_queue_size(mdev, index); 325 if (!num || vp_modern_get_queue_enable(mdev, index)) 326 return ERR_PTR(-ENOENT); 327 328 info->msix_vector = msix_vec; 329 330 /* create the vring */ 331 vq = vring_create_virtqueue(index, num, 332 SMP_CACHE_BYTES, &vp_dev->vdev, 333 true, true, ctx, 334 notify, callback, name); 335 if (!vq) 336 return ERR_PTR(-ENOMEM); 337 338 vq->num_max = num; 339 340 err = vp_active_vq(vq, msix_vec); 341 if (err) 342 goto err; 343 344 vq->priv = (void __force *)vp_modern_map_vq_notify(mdev, index, NULL); 345 if (!vq->priv) { 346 err = -ENOMEM; 347 goto err; 348 } 349 350 return vq; 351 352 err: 353 vring_del_virtqueue(vq); 354 return ERR_PTR(err); 355 } 356 357 static int vp_modern_find_vqs(struct virtio_device *vdev, unsigned int nvqs, 358 struct virtqueue *vqs[], 359 vq_callback_t *callbacks[], 360 const char * const names[], const bool *ctx, 361 struct irq_affinity *desc) 362 { 363 struct virtio_pci_device *vp_dev = to_vp_device(vdev); 364 struct virtqueue *vq; 365 int rc = vp_find_vqs(vdev, nvqs, vqs, callbacks, names, ctx, desc); 366 367 if (rc) 368 return rc; 369 370 /* Select and activate all queues. Has to be done last: once we do 371 * this, there's no way to go back except reset. 372 */ 373 list_for_each_entry(vq, &vdev->vqs, list) 374 vp_modern_set_queue_enable(&vp_dev->mdev, vq->index, true); 375 376 return 0; 377 } 378 379 static void del_vq(struct virtio_pci_vq_info *info) 380 { 381 struct virtqueue *vq = info->vq; 382 struct virtio_pci_device *vp_dev = to_vp_device(vq->vdev); 383 struct virtio_pci_modern_device *mdev = &vp_dev->mdev; 384 385 if (vp_dev->msix_enabled) 386 vp_modern_queue_vector(mdev, vq->index, 387 VIRTIO_MSI_NO_VECTOR); 388 389 if (!mdev->notify_base) 390 pci_iounmap(mdev->pci_dev, (void __force __iomem *)vq->priv); 391 392 vring_del_virtqueue(vq); 393 } 394 395 static int virtio_pci_find_shm_cap(struct pci_dev *dev, u8 required_id, 396 u8 *bar, u64 *offset, u64 *len) 397 { 398 int pos; 399 400 for (pos = pci_find_capability(dev, PCI_CAP_ID_VNDR); pos > 0; 401 pos = pci_find_next_capability(dev, pos, PCI_CAP_ID_VNDR)) { 402 u8 type, cap_len, id, res_bar; 403 u32 tmp32; 404 u64 res_offset, res_length; 405 406 pci_read_config_byte(dev, pos + offsetof(struct virtio_pci_cap, 407 cfg_type), &type); 408 if (type != VIRTIO_PCI_CAP_SHARED_MEMORY_CFG) 409 continue; 410 411 pci_read_config_byte(dev, pos + offsetof(struct virtio_pci_cap, 412 cap_len), &cap_len); 413 if (cap_len != sizeof(struct virtio_pci_cap64)) { 414 dev_err(&dev->dev, "%s: shm cap with bad size offset:" 415 " %d size: %d\n", __func__, pos, cap_len); 416 continue; 417 } 418 419 pci_read_config_byte(dev, pos + offsetof(struct virtio_pci_cap, 420 id), &id); 421 if (id != required_id) 422 continue; 423 424 pci_read_config_byte(dev, pos + offsetof(struct virtio_pci_cap, 425 bar), &res_bar); 426 if (res_bar >= PCI_STD_NUM_BARS) 427 continue; 428 429 /* Type and ID match, and the BAR value isn't reserved. 430 * Looks good. 431 */ 432 433 /* Read the lower 32bit of length and offset */ 434 pci_read_config_dword(dev, pos + offsetof(struct virtio_pci_cap, 435 offset), &tmp32); 436 res_offset = tmp32; 437 pci_read_config_dword(dev, pos + offsetof(struct virtio_pci_cap, 438 length), &tmp32); 439 res_length = tmp32; 440 441 /* and now the top half */ 442 pci_read_config_dword(dev, 443 pos + offsetof(struct virtio_pci_cap64, 444 offset_hi), &tmp32); 445 res_offset |= ((u64)tmp32) << 32; 446 pci_read_config_dword(dev, 447 pos + offsetof(struct virtio_pci_cap64, 448 length_hi), &tmp32); 449 res_length |= ((u64)tmp32) << 32; 450 451 *bar = res_bar; 452 *offset = res_offset; 453 *len = res_length; 454 455 return pos; 456 } 457 return 0; 458 } 459 460 static bool vp_get_shm_region(struct virtio_device *vdev, 461 struct virtio_shm_region *region, u8 id) 462 { 463 struct virtio_pci_device *vp_dev = to_vp_device(vdev); 464 struct pci_dev *pci_dev = vp_dev->pci_dev; 465 u8 bar; 466 u64 offset, len; 467 phys_addr_t phys_addr; 468 size_t bar_len; 469 470 if (!virtio_pci_find_shm_cap(pci_dev, id, &bar, &offset, &len)) 471 return false; 472 473 phys_addr = pci_resource_start(pci_dev, bar); 474 bar_len = pci_resource_len(pci_dev, bar); 475 476 if ((offset + len) < offset) { 477 dev_err(&pci_dev->dev, "%s: cap offset+len overflow detected\n", 478 __func__); 479 return false; 480 } 481 482 if (offset + len > bar_len) { 483 dev_err(&pci_dev->dev, "%s: bar shorter than cap offset+len\n", 484 __func__); 485 return false; 486 } 487 488 region->len = len; 489 region->addr = (u64) phys_addr + offset; 490 491 return true; 492 } 493 494 static const struct virtio_config_ops virtio_pci_config_nodev_ops = { 495 .get = NULL, 496 .set = NULL, 497 .generation = vp_generation, 498 .get_status = vp_get_status, 499 .set_status = vp_set_status, 500 .reset = vp_reset, 501 .find_vqs = vp_modern_find_vqs, 502 .del_vqs = vp_del_vqs, 503 .synchronize_cbs = vp_synchronize_vectors, 504 .get_features = vp_get_features, 505 .finalize_features = vp_finalize_features, 506 .bus_name = vp_bus_name, 507 .set_vq_affinity = vp_set_vq_affinity, 508 .get_vq_affinity = vp_get_vq_affinity, 509 .get_shm_region = vp_get_shm_region, 510 .disable_vq_and_reset = vp_modern_disable_vq_and_reset, 511 .enable_vq_after_reset = vp_modern_enable_vq_after_reset, 512 }; 513 514 static const struct virtio_config_ops virtio_pci_config_ops = { 515 .get = vp_get, 516 .set = vp_set, 517 .generation = vp_generation, 518 .get_status = vp_get_status, 519 .set_status = vp_set_status, 520 .reset = vp_reset, 521 .find_vqs = vp_modern_find_vqs, 522 .del_vqs = vp_del_vqs, 523 .synchronize_cbs = vp_synchronize_vectors, 524 .get_features = vp_get_features, 525 .finalize_features = vp_finalize_features, 526 .bus_name = vp_bus_name, 527 .set_vq_affinity = vp_set_vq_affinity, 528 .get_vq_affinity = vp_get_vq_affinity, 529 .get_shm_region = vp_get_shm_region, 530 .disable_vq_and_reset = vp_modern_disable_vq_and_reset, 531 .enable_vq_after_reset = vp_modern_enable_vq_after_reset, 532 }; 533 534 /* the PCI probing function */ 535 int virtio_pci_modern_probe(struct virtio_pci_device *vp_dev) 536 { 537 struct virtio_pci_modern_device *mdev = &vp_dev->mdev; 538 struct pci_dev *pci_dev = vp_dev->pci_dev; 539 int err; 540 541 mdev->pci_dev = pci_dev; 542 543 err = vp_modern_probe(mdev); 544 if (err) 545 return err; 546 547 if (mdev->device) 548 vp_dev->vdev.config = &virtio_pci_config_ops; 549 else 550 vp_dev->vdev.config = &virtio_pci_config_nodev_ops; 551 552 vp_dev->config_vector = vp_config_vector; 553 vp_dev->setup_vq = setup_vq; 554 vp_dev->del_vq = del_vq; 555 vp_dev->isr = mdev->isr; 556 vp_dev->vdev.id = mdev->id; 557 558 return 0; 559 } 560 561 void virtio_pci_modern_remove(struct virtio_pci_device *vp_dev) 562 { 563 struct virtio_pci_modern_device *mdev = &vp_dev->mdev; 564 565 vp_modern_remove(mdev); 566 } 567