1 /*- 2 * SPDX-License-Identifier: BSD-2-Clause-FreeBSD 3 * 4 * Copyright (c) 2011, Bryan Venteicher <bryanv@FreeBSD.org> 5 * All rights reserved. 6 * 7 * Redistribution and use in source and binary forms, with or without 8 * modification, are permitted provided that the following conditions 9 * are met: 10 * 1. Redistributions of source code must retain the above copyright 11 * notice unmodified, this list of conditions, and the following 12 * disclaimer. 13 * 2. Redistributions in binary form must reproduce the above copyright 14 * notice, this list of conditions and the following disclaimer in the 15 * documentation and/or other materials provided with the distribution. 16 * 17 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR 18 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES 19 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. 20 * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, 21 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT 22 * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 23 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 24 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 25 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF 26 * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 27 */ 28 29 /* 30 * Implements the virtqueue interface as basically described 31 * in the original VirtIO paper. 32 */ 33 34 #include <sys/cdefs.h> 35 __FBSDID("$FreeBSD$"); 36 37 #include <sys/param.h> 38 #include <sys/systm.h> 39 #include <sys/kernel.h> 40 #include <sys/malloc.h> 41 #include <sys/sglist.h> 42 #include <vm/vm.h> 43 #include <vm/pmap.h> 44 45 #include <machine/cpu.h> 46 #include <machine/bus.h> 47 #include <machine/atomic.h> 48 #include <machine/resource.h> 49 #include <sys/bus.h> 50 #include <sys/rman.h> 51 52 #include <dev/virtio/virtio.h> 53 #include <dev/virtio/virtqueue.h> 54 #include <dev/virtio/virtio_ring.h> 55 56 #include "virtio_bus_if.h" 57 58 struct virtqueue { 59 device_t vq_dev; 60 char vq_name[VIRTQUEUE_MAX_NAME_SZ]; 61 uint16_t vq_queue_index; 62 uint16_t vq_nentries; 63 uint32_t vq_flags; 64 #define VIRTQUEUE_FLAG_INDIRECT 0x0001 65 #define VIRTQUEUE_FLAG_EVENT_IDX 0x0002 66 67 int vq_alignment; 68 int vq_ring_size; 69 void *vq_ring_mem; 70 int vq_max_indirect_size; 71 int vq_indirect_mem_size; 72 virtqueue_intr_t *vq_intrhand; 73 void *vq_intrhand_arg; 74 75 struct vring vq_ring; 76 uint16_t vq_free_cnt; 77 uint16_t vq_queued_cnt; 78 /* 79 * Head of the free chain in the descriptor table. If 80 * there are no free descriptors, this will be set to 81 * VQ_RING_DESC_CHAIN_END. 82 */ 83 uint16_t vq_desc_head_idx; 84 /* 85 * Last consumed descriptor in the used table, 86 * trails vq_ring.used->idx. 87 */ 88 uint16_t vq_used_cons_idx; 89 90 struct vq_desc_extra { 91 void *cookie; 92 struct vring_desc *indirect; 93 vm_paddr_t indirect_paddr; 94 uint16_t ndescs; 95 } vq_descx[0]; 96 }; 97 98 /* 99 * The maximum virtqueue size is 2^15. Use that value as the end of 100 * descriptor chain terminator since it will never be a valid index 101 * in the descriptor table. This is used to verify we are correctly 102 * handling vq_free_cnt. 103 */ 104 #define VQ_RING_DESC_CHAIN_END 32768 105 106 #define VQASSERT(_vq, _exp, _msg, ...) \ 107 KASSERT((_exp),("%s: %s - "_msg, __func__, (_vq)->vq_name, \ 108 ##__VA_ARGS__)) 109 110 #define VQ_RING_ASSERT_VALID_IDX(_vq, _idx) \ 111 VQASSERT((_vq), (_idx) < (_vq)->vq_nentries, \ 112 "invalid ring index: %d, max: %d", (_idx), \ 113 (_vq)->vq_nentries) 114 115 #define VQ_RING_ASSERT_CHAIN_TERM(_vq) \ 116 VQASSERT((_vq), (_vq)->vq_desc_head_idx == \ 117 VQ_RING_DESC_CHAIN_END, "full ring terminated " \ 118 "incorrectly: head idx: %d", (_vq)->vq_desc_head_idx) 119 120 static int virtqueue_init_indirect(struct virtqueue *vq, int); 121 static void virtqueue_free_indirect(struct virtqueue *vq); 122 static void virtqueue_init_indirect_list(struct virtqueue *, 123 struct vring_desc *); 124 125 static void vq_ring_init(struct virtqueue *); 126 static void vq_ring_update_avail(struct virtqueue *, uint16_t); 127 static uint16_t vq_ring_enqueue_segments(struct virtqueue *, 128 struct vring_desc *, uint16_t, struct sglist *, int, int); 129 static int vq_ring_use_indirect(struct virtqueue *, int); 130 static void vq_ring_enqueue_indirect(struct virtqueue *, void *, 131 struct sglist *, int, int); 132 static int vq_ring_enable_interrupt(struct virtqueue *, uint16_t); 133 static int vq_ring_must_notify_host(struct virtqueue *); 134 static void vq_ring_notify_host(struct virtqueue *); 135 static void vq_ring_free_chain(struct virtqueue *, uint16_t); 136 137 uint64_t 138 virtqueue_filter_features(uint64_t features) 139 { 140 uint64_t mask; 141 142 mask = (1 << VIRTIO_TRANSPORT_F_START) - 1; 143 mask |= VIRTIO_RING_F_INDIRECT_DESC; 144 mask |= VIRTIO_RING_F_EVENT_IDX; 145 146 return (features & mask); 147 } 148 149 int 150 virtqueue_alloc(device_t dev, uint16_t queue, uint16_t size, int align, 151 vm_paddr_t highaddr, struct vq_alloc_info *info, struct virtqueue **vqp) 152 { 153 struct virtqueue *vq; 154 int error; 155 156 *vqp = NULL; 157 error = 0; 158 159 if (size == 0) { 160 device_printf(dev, 161 "virtqueue %d (%s) does not exist (size is zero)\n", 162 queue, info->vqai_name); 163 return (ENODEV); 164 } else if (!powerof2(size)) { 165 device_printf(dev, 166 "virtqueue %d (%s) size is not a power of 2: %d\n", 167 queue, info->vqai_name, size); 168 return (ENXIO); 169 } else if (info->vqai_maxindirsz > VIRTIO_MAX_INDIRECT) { 170 device_printf(dev, "virtqueue %d (%s) requested too many " 171 "indirect descriptors: %d, max %d\n", 172 queue, info->vqai_name, info->vqai_maxindirsz, 173 VIRTIO_MAX_INDIRECT); 174 return (EINVAL); 175 } 176 177 vq = malloc(sizeof(struct virtqueue) + 178 size * sizeof(struct vq_desc_extra), M_DEVBUF, M_NOWAIT | M_ZERO); 179 if (vq == NULL) { 180 device_printf(dev, "cannot allocate virtqueue\n"); 181 return (ENOMEM); 182 } 183 184 vq->vq_dev = dev; 185 strlcpy(vq->vq_name, info->vqai_name, sizeof(vq->vq_name)); 186 vq->vq_queue_index = queue; 187 vq->vq_alignment = align; 188 vq->vq_nentries = size; 189 vq->vq_free_cnt = size; 190 vq->vq_intrhand = info->vqai_intr; 191 vq->vq_intrhand_arg = info->vqai_intr_arg; 192 193 if (VIRTIO_BUS_WITH_FEATURE(dev, VIRTIO_RING_F_EVENT_IDX) != 0) 194 vq->vq_flags |= VIRTQUEUE_FLAG_EVENT_IDX; 195 196 if (info->vqai_maxindirsz > 1) { 197 error = virtqueue_init_indirect(vq, info->vqai_maxindirsz); 198 if (error) 199 goto fail; 200 } 201 202 vq->vq_ring_size = round_page(vring_size(size, align)); 203 vq->vq_ring_mem = contigmalloc(vq->vq_ring_size, M_DEVBUF, 204 M_NOWAIT | M_ZERO, 0, highaddr, PAGE_SIZE, 0); 205 if (vq->vq_ring_mem == NULL) { 206 device_printf(dev, 207 "cannot allocate memory for virtqueue ring\n"); 208 error = ENOMEM; 209 goto fail; 210 } 211 212 vq_ring_init(vq); 213 virtqueue_disable_intr(vq); 214 215 *vqp = vq; 216 217 fail: 218 if (error) 219 virtqueue_free(vq); 220 221 return (error); 222 } 223 224 static int 225 virtqueue_init_indirect(struct virtqueue *vq, int indirect_size) 226 { 227 device_t dev; 228 struct vq_desc_extra *dxp; 229 int i, size; 230 231 dev = vq->vq_dev; 232 233 if (VIRTIO_BUS_WITH_FEATURE(dev, VIRTIO_RING_F_INDIRECT_DESC) == 0) { 234 /* 235 * Indirect descriptors requested by the driver but not 236 * negotiated. Return zero to keep the initialization 237 * going: we'll run fine without. 238 */ 239 if (bootverbose) 240 device_printf(dev, "virtqueue %d (%s) requested " 241 "indirect descriptors but not negotiated\n", 242 vq->vq_queue_index, vq->vq_name); 243 return (0); 244 } 245 246 size = indirect_size * sizeof(struct vring_desc); 247 vq->vq_max_indirect_size = indirect_size; 248 vq->vq_indirect_mem_size = size; 249 vq->vq_flags |= VIRTQUEUE_FLAG_INDIRECT; 250 251 for (i = 0; i < vq->vq_nentries; i++) { 252 dxp = &vq->vq_descx[i]; 253 254 dxp->indirect = malloc(size, M_DEVBUF, M_NOWAIT); 255 if (dxp->indirect == NULL) { 256 device_printf(dev, "cannot allocate indirect list\n"); 257 return (ENOMEM); 258 } 259 260 dxp->indirect_paddr = vtophys(dxp->indirect); 261 virtqueue_init_indirect_list(vq, dxp->indirect); 262 } 263 264 return (0); 265 } 266 267 static void 268 virtqueue_free_indirect(struct virtqueue *vq) 269 { 270 struct vq_desc_extra *dxp; 271 int i; 272 273 for (i = 0; i < vq->vq_nentries; i++) { 274 dxp = &vq->vq_descx[i]; 275 276 if (dxp->indirect == NULL) 277 break; 278 279 free(dxp->indirect, M_DEVBUF); 280 dxp->indirect = NULL; 281 dxp->indirect_paddr = 0; 282 } 283 284 vq->vq_flags &= ~VIRTQUEUE_FLAG_INDIRECT; 285 vq->vq_indirect_mem_size = 0; 286 } 287 288 static void 289 virtqueue_init_indirect_list(struct virtqueue *vq, 290 struct vring_desc *indirect) 291 { 292 int i; 293 294 bzero(indirect, vq->vq_indirect_mem_size); 295 296 for (i = 0; i < vq->vq_max_indirect_size - 1; i++) 297 indirect[i].next = i + 1; 298 indirect[i].next = VQ_RING_DESC_CHAIN_END; 299 } 300 301 int 302 virtqueue_reinit(struct virtqueue *vq, uint16_t size) 303 { 304 struct vq_desc_extra *dxp; 305 int i; 306 307 if (vq->vq_nentries != size) { 308 device_printf(vq->vq_dev, 309 "%s: '%s' changed size; old=%hu, new=%hu\n", 310 __func__, vq->vq_name, vq->vq_nentries, size); 311 return (EINVAL); 312 } 313 314 /* Warn if the virtqueue was not properly cleaned up. */ 315 if (vq->vq_free_cnt != vq->vq_nentries) { 316 device_printf(vq->vq_dev, 317 "%s: warning '%s' virtqueue not empty, " 318 "leaking %d entries\n", __func__, vq->vq_name, 319 vq->vq_nentries - vq->vq_free_cnt); 320 } 321 322 vq->vq_desc_head_idx = 0; 323 vq->vq_used_cons_idx = 0; 324 vq->vq_queued_cnt = 0; 325 vq->vq_free_cnt = vq->vq_nentries; 326 327 /* To be safe, reset all our allocated memory. */ 328 bzero(vq->vq_ring_mem, vq->vq_ring_size); 329 for (i = 0; i < vq->vq_nentries; i++) { 330 dxp = &vq->vq_descx[i]; 331 dxp->cookie = NULL; 332 dxp->ndescs = 0; 333 if (vq->vq_flags & VIRTQUEUE_FLAG_INDIRECT) 334 virtqueue_init_indirect_list(vq, dxp->indirect); 335 } 336 337 vq_ring_init(vq); 338 virtqueue_disable_intr(vq); 339 340 return (0); 341 } 342 343 void 344 virtqueue_free(struct virtqueue *vq) 345 { 346 347 if (vq->vq_free_cnt != vq->vq_nentries) { 348 device_printf(vq->vq_dev, "%s: freeing non-empty virtqueue, " 349 "leaking %d entries\n", vq->vq_name, 350 vq->vq_nentries - vq->vq_free_cnt); 351 } 352 353 if (vq->vq_flags & VIRTQUEUE_FLAG_INDIRECT) 354 virtqueue_free_indirect(vq); 355 356 if (vq->vq_ring_mem != NULL) { 357 contigfree(vq->vq_ring_mem, vq->vq_ring_size, M_DEVBUF); 358 vq->vq_ring_size = 0; 359 vq->vq_ring_mem = NULL; 360 } 361 362 free(vq, M_DEVBUF); 363 } 364 365 vm_paddr_t 366 virtqueue_paddr(struct virtqueue *vq) 367 { 368 369 return (vtophys(vq->vq_ring_mem)); 370 } 371 372 int 373 virtqueue_size(struct virtqueue *vq) 374 { 375 376 return (vq->vq_nentries); 377 } 378 379 int 380 virtqueue_nfree(struct virtqueue *vq) 381 { 382 383 return (vq->vq_free_cnt); 384 } 385 386 int 387 virtqueue_empty(struct virtqueue *vq) 388 { 389 390 return (vq->vq_nentries == vq->vq_free_cnt); 391 } 392 393 int 394 virtqueue_full(struct virtqueue *vq) 395 { 396 397 return (vq->vq_free_cnt == 0); 398 } 399 400 void 401 virtqueue_notify(struct virtqueue *vq) 402 { 403 404 /* Ensure updated avail->idx is visible to host. */ 405 mb(); 406 407 if (vq_ring_must_notify_host(vq)) 408 vq_ring_notify_host(vq); 409 vq->vq_queued_cnt = 0; 410 } 411 412 int 413 virtqueue_nused(struct virtqueue *vq) 414 { 415 uint16_t used_idx, nused; 416 417 used_idx = vq->vq_ring.used->idx; 418 419 nused = (uint16_t)(used_idx - vq->vq_used_cons_idx); 420 VQASSERT(vq, nused <= vq->vq_nentries, "used more than available"); 421 422 return (nused); 423 } 424 425 int 426 virtqueue_intr_filter(struct virtqueue *vq) 427 { 428 429 if (vq->vq_used_cons_idx == vq->vq_ring.used->idx) 430 return (0); 431 432 virtqueue_disable_intr(vq); 433 434 return (1); 435 } 436 437 void 438 virtqueue_intr(struct virtqueue *vq) 439 { 440 441 vq->vq_intrhand(vq->vq_intrhand_arg); 442 } 443 444 int 445 virtqueue_enable_intr(struct virtqueue *vq) 446 { 447 448 return (vq_ring_enable_interrupt(vq, 0)); 449 } 450 451 int 452 virtqueue_postpone_intr(struct virtqueue *vq, vq_postpone_t hint) 453 { 454 uint16_t ndesc, avail_idx; 455 456 avail_idx = vq->vq_ring.avail->idx; 457 ndesc = (uint16_t)(avail_idx - vq->vq_used_cons_idx); 458 459 switch (hint) { 460 case VQ_POSTPONE_SHORT: 461 ndesc = ndesc / 4; 462 break; 463 case VQ_POSTPONE_LONG: 464 ndesc = (ndesc * 3) / 4; 465 break; 466 case VQ_POSTPONE_EMPTIED: 467 break; 468 } 469 470 return (vq_ring_enable_interrupt(vq, ndesc)); 471 } 472 473 /* 474 * Note this is only considered a hint to the host. 475 */ 476 void 477 virtqueue_disable_intr(struct virtqueue *vq) 478 { 479 480 if (vq->vq_flags & VIRTQUEUE_FLAG_EVENT_IDX) { 481 vring_used_event(&vq->vq_ring) = vq->vq_used_cons_idx - 482 vq->vq_nentries - 1; 483 } else 484 vq->vq_ring.avail->flags |= VRING_AVAIL_F_NO_INTERRUPT; 485 } 486 487 int 488 virtqueue_enqueue(struct virtqueue *vq, void *cookie, struct sglist *sg, 489 int readable, int writable) 490 { 491 struct vq_desc_extra *dxp; 492 int needed; 493 uint16_t head_idx, idx; 494 495 needed = readable + writable; 496 497 VQASSERT(vq, cookie != NULL, "enqueuing with no cookie"); 498 VQASSERT(vq, needed == sg->sg_nseg, 499 "segment count mismatch, %d, %d", needed, sg->sg_nseg); 500 VQASSERT(vq, 501 needed <= vq->vq_nentries || needed <= vq->vq_max_indirect_size, 502 "too many segments to enqueue: %d, %d/%d", needed, 503 vq->vq_nentries, vq->vq_max_indirect_size); 504 505 if (needed < 1) 506 return (EINVAL); 507 if (vq->vq_free_cnt == 0) 508 return (ENOSPC); 509 510 if (vq_ring_use_indirect(vq, needed)) { 511 vq_ring_enqueue_indirect(vq, cookie, sg, readable, writable); 512 return (0); 513 } else if (vq->vq_free_cnt < needed) 514 return (EMSGSIZE); 515 516 head_idx = vq->vq_desc_head_idx; 517 VQ_RING_ASSERT_VALID_IDX(vq, head_idx); 518 dxp = &vq->vq_descx[head_idx]; 519 520 VQASSERT(vq, dxp->cookie == NULL, 521 "cookie already exists for index %d", head_idx); 522 dxp->cookie = cookie; 523 dxp->ndescs = needed; 524 525 idx = vq_ring_enqueue_segments(vq, vq->vq_ring.desc, head_idx, 526 sg, readable, writable); 527 528 vq->vq_desc_head_idx = idx; 529 vq->vq_free_cnt -= needed; 530 if (vq->vq_free_cnt == 0) 531 VQ_RING_ASSERT_CHAIN_TERM(vq); 532 else 533 VQ_RING_ASSERT_VALID_IDX(vq, idx); 534 535 vq_ring_update_avail(vq, head_idx); 536 537 return (0); 538 } 539 540 void * 541 virtqueue_dequeue(struct virtqueue *vq, uint32_t *len) 542 { 543 struct vring_used_elem *uep; 544 void *cookie; 545 uint16_t used_idx, desc_idx; 546 547 if (vq->vq_used_cons_idx == vq->vq_ring.used->idx) 548 return (NULL); 549 550 used_idx = vq->vq_used_cons_idx++ & (vq->vq_nentries - 1); 551 uep = &vq->vq_ring.used->ring[used_idx]; 552 553 rmb(); 554 desc_idx = (uint16_t) uep->id; 555 if (len != NULL) 556 *len = uep->len; 557 558 vq_ring_free_chain(vq, desc_idx); 559 560 cookie = vq->vq_descx[desc_idx].cookie; 561 VQASSERT(vq, cookie != NULL, "no cookie for index %d", desc_idx); 562 vq->vq_descx[desc_idx].cookie = NULL; 563 564 return (cookie); 565 } 566 567 void * 568 virtqueue_poll(struct virtqueue *vq, uint32_t *len) 569 { 570 void *cookie; 571 572 VIRTIO_BUS_POLL(vq->vq_dev); 573 while ((cookie = virtqueue_dequeue(vq, len)) == NULL) { 574 cpu_spinwait(); 575 VIRTIO_BUS_POLL(vq->vq_dev); 576 } 577 578 return (cookie); 579 } 580 581 void * 582 virtqueue_drain(struct virtqueue *vq, int *last) 583 { 584 void *cookie; 585 int idx; 586 587 cookie = NULL; 588 idx = *last; 589 590 while (idx < vq->vq_nentries && cookie == NULL) { 591 if ((cookie = vq->vq_descx[idx].cookie) != NULL) { 592 vq->vq_descx[idx].cookie = NULL; 593 /* Free chain to keep free count consistent. */ 594 vq_ring_free_chain(vq, idx); 595 } 596 idx++; 597 } 598 599 *last = idx; 600 601 return (cookie); 602 } 603 604 void 605 virtqueue_dump(struct virtqueue *vq) 606 { 607 608 if (vq == NULL) 609 return; 610 611 printf("VQ: %s - size=%d; free=%d; used=%d; queued=%d; " 612 "desc_head_idx=%d; avail.idx=%d; used_cons_idx=%d; " 613 "used.idx=%d; used_event_idx=%d; avail.flags=0x%x; used.flags=0x%x\n", 614 vq->vq_name, vq->vq_nentries, vq->vq_free_cnt, 615 virtqueue_nused(vq), vq->vq_queued_cnt, vq->vq_desc_head_idx, 616 vq->vq_ring.avail->idx, vq->vq_used_cons_idx, 617 vq->vq_ring.used->idx, 618 vring_used_event(&vq->vq_ring), 619 vq->vq_ring.avail->flags, 620 vq->vq_ring.used->flags); 621 } 622 623 static void 624 vq_ring_init(struct virtqueue *vq) 625 { 626 struct vring *vr; 627 char *ring_mem; 628 int i, size; 629 630 ring_mem = vq->vq_ring_mem; 631 size = vq->vq_nentries; 632 vr = &vq->vq_ring; 633 634 vring_init(vr, size, ring_mem, vq->vq_alignment); 635 636 for (i = 0; i < size - 1; i++) 637 vr->desc[i].next = i + 1; 638 vr->desc[i].next = VQ_RING_DESC_CHAIN_END; 639 } 640 641 static void 642 vq_ring_update_avail(struct virtqueue *vq, uint16_t desc_idx) 643 { 644 uint16_t avail_idx; 645 646 /* 647 * Place the head of the descriptor chain into the next slot and make 648 * it usable to the host. The chain is made available now rather than 649 * deferring to virtqueue_notify() in the hopes that if the host is 650 * currently running on another CPU, we can keep it processing the new 651 * descriptor. 652 */ 653 avail_idx = vq->vq_ring.avail->idx & (vq->vq_nentries - 1); 654 vq->vq_ring.avail->ring[avail_idx] = desc_idx; 655 656 wmb(); 657 vq->vq_ring.avail->idx++; 658 659 /* Keep pending count until virtqueue_notify(). */ 660 vq->vq_queued_cnt++; 661 } 662 663 static uint16_t 664 vq_ring_enqueue_segments(struct virtqueue *vq, struct vring_desc *desc, 665 uint16_t head_idx, struct sglist *sg, int readable, int writable) 666 { 667 struct sglist_seg *seg; 668 struct vring_desc *dp; 669 int i, needed; 670 uint16_t idx; 671 672 needed = readable + writable; 673 674 for (i = 0, idx = head_idx, seg = sg->sg_segs; 675 i < needed; 676 i++, idx = dp->next, seg++) { 677 VQASSERT(vq, idx != VQ_RING_DESC_CHAIN_END, 678 "premature end of free desc chain"); 679 680 dp = &desc[idx]; 681 dp->addr = seg->ss_paddr; 682 dp->len = seg->ss_len; 683 dp->flags = 0; 684 685 if (i < needed - 1) 686 dp->flags |= VRING_DESC_F_NEXT; 687 if (i >= readable) 688 dp->flags |= VRING_DESC_F_WRITE; 689 } 690 691 return (idx); 692 } 693 694 static int 695 vq_ring_use_indirect(struct virtqueue *vq, int needed) 696 { 697 698 if ((vq->vq_flags & VIRTQUEUE_FLAG_INDIRECT) == 0) 699 return (0); 700 701 if (vq->vq_max_indirect_size < needed) 702 return (0); 703 704 if (needed < 2) 705 return (0); 706 707 return (1); 708 } 709 710 static void 711 vq_ring_enqueue_indirect(struct virtqueue *vq, void *cookie, 712 struct sglist *sg, int readable, int writable) 713 { 714 struct vring_desc *dp; 715 struct vq_desc_extra *dxp; 716 int needed; 717 uint16_t head_idx; 718 719 needed = readable + writable; 720 VQASSERT(vq, needed <= vq->vq_max_indirect_size, 721 "enqueuing too many indirect descriptors"); 722 723 head_idx = vq->vq_desc_head_idx; 724 VQ_RING_ASSERT_VALID_IDX(vq, head_idx); 725 dp = &vq->vq_ring.desc[head_idx]; 726 dxp = &vq->vq_descx[head_idx]; 727 728 VQASSERT(vq, dxp->cookie == NULL, 729 "cookie already exists for index %d", head_idx); 730 dxp->cookie = cookie; 731 dxp->ndescs = 1; 732 733 dp->addr = dxp->indirect_paddr; 734 dp->len = needed * sizeof(struct vring_desc); 735 dp->flags = VRING_DESC_F_INDIRECT; 736 737 vq_ring_enqueue_segments(vq, dxp->indirect, 0, 738 sg, readable, writable); 739 740 vq->vq_desc_head_idx = dp->next; 741 vq->vq_free_cnt--; 742 if (vq->vq_free_cnt == 0) 743 VQ_RING_ASSERT_CHAIN_TERM(vq); 744 else 745 VQ_RING_ASSERT_VALID_IDX(vq, vq->vq_desc_head_idx); 746 747 vq_ring_update_avail(vq, head_idx); 748 } 749 750 static int 751 vq_ring_enable_interrupt(struct virtqueue *vq, uint16_t ndesc) 752 { 753 754 /* 755 * Enable interrupts, making sure we get the latest index of 756 * what's already been consumed. 757 */ 758 if (vq->vq_flags & VIRTQUEUE_FLAG_EVENT_IDX) 759 vring_used_event(&vq->vq_ring) = vq->vq_used_cons_idx + ndesc; 760 else 761 vq->vq_ring.avail->flags &= ~VRING_AVAIL_F_NO_INTERRUPT; 762 763 mb(); 764 765 /* 766 * Enough items may have already been consumed to meet our threshold 767 * since we last checked. Let our caller know so it processes the new 768 * entries. 769 */ 770 if (virtqueue_nused(vq) > ndesc) 771 return (1); 772 773 return (0); 774 } 775 776 static int 777 vq_ring_must_notify_host(struct virtqueue *vq) 778 { 779 uint16_t new_idx, prev_idx, event_idx; 780 781 if (vq->vq_flags & VIRTQUEUE_FLAG_EVENT_IDX) { 782 new_idx = vq->vq_ring.avail->idx; 783 prev_idx = new_idx - vq->vq_queued_cnt; 784 event_idx = vring_avail_event(&vq->vq_ring); 785 786 return (vring_need_event(event_idx, new_idx, prev_idx) != 0); 787 } 788 789 return ((vq->vq_ring.used->flags & VRING_USED_F_NO_NOTIFY) == 0); 790 } 791 792 static void 793 vq_ring_notify_host(struct virtqueue *vq) 794 { 795 796 VIRTIO_BUS_NOTIFY_VQ(vq->vq_dev, vq->vq_queue_index); 797 } 798 799 static void 800 vq_ring_free_chain(struct virtqueue *vq, uint16_t desc_idx) 801 { 802 struct vring_desc *dp; 803 struct vq_desc_extra *dxp; 804 805 VQ_RING_ASSERT_VALID_IDX(vq, desc_idx); 806 dp = &vq->vq_ring.desc[desc_idx]; 807 dxp = &vq->vq_descx[desc_idx]; 808 809 if (vq->vq_free_cnt == 0) 810 VQ_RING_ASSERT_CHAIN_TERM(vq); 811 812 vq->vq_free_cnt += dxp->ndescs; 813 dxp->ndescs--; 814 815 if ((dp->flags & VRING_DESC_F_INDIRECT) == 0) { 816 while (dp->flags & VRING_DESC_F_NEXT) { 817 VQ_RING_ASSERT_VALID_IDX(vq, dp->next); 818 dp = &vq->vq_ring.desc[dp->next]; 819 dxp->ndescs--; 820 } 821 } 822 823 VQASSERT(vq, dxp->ndescs == 0, 824 "failed to free entire desc chain, remaining: %d", dxp->ndescs); 825 826 /* 827 * We must append the existing free chain, if any, to the end of 828 * newly freed chain. If the virtqueue was completely used, then 829 * head would be VQ_RING_DESC_CHAIN_END (ASSERTed above). 830 */ 831 dp->next = vq->vq_desc_head_idx; 832 vq->vq_desc_head_idx = desc_idx; 833 } 834