1 /*- 2 * SPDX-License-Identifier: BSD-2-Clause-FreeBSD 3 * 4 * Copyright (c) 2011, Bryan Venteicher <bryanv@FreeBSD.org> 5 * All rights reserved. 6 * 7 * Redistribution and use in source and binary forms, with or without 8 * modification, are permitted provided that the following conditions 9 * are met: 10 * 1. Redistributions of source code must retain the above copyright 11 * notice unmodified, this list of conditions, and the following 12 * disclaimer. 13 * 2. Redistributions in binary form must reproduce the above copyright 14 * notice, this list of conditions and the following disclaimer in the 15 * documentation and/or other materials provided with the distribution. 16 * 17 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR 18 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES 19 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. 20 * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, 21 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT 22 * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 23 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 24 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 25 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF 26 * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 27 * 28 * $FreeBSD: head/sys/dev/virtio/balloon/virtio_balloon.c 326255 2017-11-27 14:52:40Z pfg $ 29 */ 30 31 /* 32 * Copyright (c) 2018 The DragonFly Project. All rights reserved. 33 * 34 * This code is derived from software contributed to The DragonFly Project 35 * by Diederik de Groot <info@talon.nl> 36 * 37 * Redistribution and use in source and binary forms, with or without 38 * modification, are permitted provided that the following conditions 39 * are met: 40 * 41 * 1. Redistributions of source code must retain the above copyright 42 * notice, this list of conditions and the following disclaimer. 43 * 2. Redistributions in binary form must reproduce the above copyright 44 * notice, this list of conditions and the following disclaimer in 45 * the documentation and/or other materials provided with the 46 * distribution. 47 * 3. Neither the name of The DragonFly Project nor the names of its 48 * contributors may be used to endorse or promote products derived 49 * from this software without specific, prior written permission. 50 * 51 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 52 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 53 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS 54 * FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE 55 * COPYRIGHT HOLDERS OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, 56 * INCIDENTAL, SPECIAL, EXEMPLARY OR CONSEQUENTIAL DAMAGES (INCLUDING, 57 * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; 58 * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED 59 * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, 60 * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT 61 * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 62 * SUCH DAMAGE. 63 */ 64 65 /* Driver for VirtIO memory balloon devices. */ 66 67 #include <sys/cdefs.h> 68 #include <sys/param.h> 69 #include <sys/systm.h> 70 #include <sys/kernel.h> 71 #include <sys/endian.h> 72 #include <sys/kthread.h> 73 #include <sys/malloc.h> 74 #include <sys/module.h> 75 #include <sys/sglist.h> 76 #include <sys/sysctl.h> 77 #include <sys/lock.h> 78 #include <sys/queue.h> 79 80 #include <vm/vm.h> 81 #include <vm/vm_page.h> 82 #include <sys/bus.h> 83 #include <sys/rman.h> 84 85 #include <dev/virtual/virtio/virtio/virtio.h> 86 #include <dev/virtual/virtio/virtio/virtqueue.h> 87 #include <dev/virtual/virtio/balloon/virtio_balloon.h> 88 89 struct vtballoon_softc { 90 device_t vtballoon_dev; 91 struct lwkt_serialize vtballoon_slz; 92 uint64_t vtballoon_features; 93 uint32_t vtballoon_flags; 94 #define VTBALLOON_FLAG_DETACH 0x01 95 96 struct virtqueue *vtballoon_inflate_vq; 97 struct virtqueue *vtballoon_deflate_vq; 98 99 uint32_t vtballoon_desired_npages; 100 uint32_t vtballoon_current_npages; 101 TAILQ_HEAD(,vm_page) vtballoon_pages; 102 103 struct thread *vtballoon_td; 104 uint32_t *vtballoon_page_frames; 105 int vtballoon_pagereq; 106 int vtballoon_timeout; 107 int vtballoon_nintr; 108 int vtballoon_debug; 109 #define VTBALLOON_INFO 0x01 110 #define VTBALLOON_ERROR 0x02 111 #define VTBALLOON_DEBUG 0x04 112 #define VTBALLOON_TRACE 0x08 113 114 struct virtqueue *vtballoon_stats_vq; 115 struct vtballoon_stat vtballoon_stats[VTBALLOON_S_NR]; 116 bool vtballoon_update_stats; 117 }; 118 119 static struct virtio_feature_desc vtballoon_feature_desc[] = { 120 { VIRTIO_BALLOON_F_MUST_TELL_HOST, "MustTellHost" }, 121 { VIRTIO_BALLOON_F_STATS_VQ, "StatsVq" }, 122 { VIRTIO_BALLOON_F_DEFLATE_ON_OOM, "DeflateOnOutOfMemory" }, 123 { 0, NULL } 124 }; 125 126 #define vtballoon_dprintf(_sc, _level, _msg, _args ...) do { \ 127 if ((_sc)->vtballoon_debug & (_level)) \ 128 device_printf((_sc)->vtballoon_dev, "%s:%d: "_msg, \ 129 __FUNCTION__, __LINE__, ##_args); \ 130 } while (0) 131 132 static int vtballoon_probe(device_t); 133 static int vtballoon_attach(device_t); 134 static int vtballoon_detach(device_t); 135 136 static int vtballoon_alloc_intrs(struct vtballoon_softc *sc); 137 138 static void vtballoon_negotiate_features(struct vtballoon_softc *); 139 static int vtballoon_alloc_virtqueues(struct vtballoon_softc *); 140 141 static void vtballoon_config_change_intr(void *); 142 143 static void vtballoon_update_stats(struct vtballoon_softc *sc); 144 static void vtballoon_stats_vq_intr(void *); 145 146 static void vtballoon_inflate_vq_intr(void *); 147 static void vtballoon_deflate_vq_intr(void *); 148 static void vtballoon_inflate(struct vtballoon_softc *, int); 149 static void vtballoon_deflate(struct vtballoon_softc *, int); 150 151 static void vtballoon_send_page_frames(struct vtballoon_softc *, 152 struct virtqueue *, int); 153 154 static void vtballoon_pop(struct vtballoon_softc *); 155 static void vtballoon_stop(struct vtballoon_softc *); 156 157 static vm_page_t vtballoon_alloc_page(struct vtballoon_softc *); 158 static void vtballoon_free_page(struct vtballoon_softc *, vm_page_t); 159 160 static int vtballoon_sleep(struct vtballoon_softc *); 161 static void vtballoon_thread(void *); 162 static void vtballoon_get_tunables(struct vtballoon_softc *); 163 static void vtballoon_add_sysctl(struct vtballoon_softc *); 164 165 /* 166 * Features desired/implemented by this driver. 167 * VIRTIO_BALLOON_F_STATS_VQ | VIRTIO_BALLOON_F_MUST_TELL_HOST 168 */ 169 #define VTBALLOON_FEATURES VIRTIO_BALLOON_F_STATS_VQ 170 171 /* Timeout between retries when the balloon needs inflating. */ 172 #define VTBALLOON_LOWMEM_TIMEOUT hz * 100 173 174 /* vm_page_alloc flags */ 175 #define VTBALLOON_REGULAR_ALLOC VM_ALLOC_NORMAL 176 #define VTBALLOON_LOWMEM_ALLOC VM_ALLOC_SYSTEM 177 178 /* 179 * Maximum number of pages we'll request to inflate or deflate 180 * the balloon in one virtqueue request. Both Linux and NetBSD 181 * have settled on 256, doing up to 1MB at a time. 182 */ 183 #define VTBALLOON_PAGES_PER_REQUEST 256 184 185 /* 186 * Default Debug Level 187 * VTBALLOON_INFO | VTBALLOON_ERROR | VTBALLOON_DEBUG | VTBALLOON_TRACE 188 */ 189 #define VTBALLOON_DEFAULT_DEBUG_LEVEL VTBALLOON_INFO | VTBALLOON_ERROR 190 191 /* 192 * Maximum number of interrupts to request 193 */ 194 #define VTBALLOON_MAX_INTERRUPTS 4 195 196 /* Must be able to fix all pages frames in one page (segment). */ 197 CTASSERT(VTBALLOON_PAGES_PER_REQUEST * sizeof(uint32_t) <= PAGE_SIZE); 198 199 #define VTBALLOON_SLZ(_sc) &(_sc)->vtballoon_slz 200 #define VTBALLOON_ENTER_SLZ(_sc) lwkt_serialize_enter(VTBALLOON_SLZ(sc)); 201 #define VTBALLOON_EXIT_SLZ(_sc) lwkt_serialize_exit(VTBALLOON_SLZ(sc)); 202 203 static device_method_t vtballoon_methods[] = { 204 /* Device methods. */ 205 DEVMETHOD(device_probe, vtballoon_probe), 206 DEVMETHOD(device_attach, vtballoon_attach), 207 DEVMETHOD(device_detach, vtballoon_detach), 208 209 DEVMETHOD_END 210 }; 211 212 static driver_t vtballoon_driver = { 213 "vtballoon", 214 vtballoon_methods, 215 sizeof(struct vtballoon_softc) 216 }; 217 static devclass_t vtballoon_devclass; 218 219 DRIVER_MODULE(virtio_balloon, virtio_pci, vtballoon_driver, 220 vtballoon_devclass, NULL, NULL); 221 MODULE_VERSION(virtio_balloon, 1); 222 MODULE_DEPEND(virtio_balloon, virtio, 1, 1, 1); 223 224 static int 225 vtballoon_probe(device_t dev) 226 { 227 struct vtballoon_softc *sc = device_get_softc(dev); 228 vtballoon_dprintf(sc, VTBALLOON_TRACE, "\n"); 229 if (virtio_get_device_type(dev) != VIRTIO_ID_BALLOON) 230 return (ENXIO); 231 232 device_set_desc(dev, "VirtIO Balloon Adapter"); 233 234 return (BUS_PROBE_DEFAULT); 235 } 236 237 struct irqmap { 238 int irq; 239 int idx; 240 driver_intr_t *handler; 241 const char * handler_name; 242 }; 243 244 static int 245 vtballoon_attach(device_t dev) 246 { 247 struct vtballoon_softc *sc; 248 int error, i; 249 250 sc = device_get_softc(dev); 251 sc->vtballoon_dev = dev; 252 sc->vtballoon_debug = VTBALLOON_DEFAULT_DEBUG_LEVEL; 253 254 vtballoon_dprintf(sc, VTBALLOON_TRACE, "\n"); 255 256 lwkt_serialize_init(VTBALLOON_SLZ(sc)); 257 TAILQ_INIT(&sc->vtballoon_pages); 258 259 vtballoon_get_tunables(sc); 260 vtballoon_add_sysctl(sc); 261 262 virtio_set_feature_desc(dev, vtballoon_feature_desc); 263 vtballoon_negotiate_features(sc); 264 265 sc->vtballoon_page_frames = contigmalloc(VTBALLOON_PAGES_PER_REQUEST * 266 sizeof(uint32_t), M_DEVBUF, M_NOWAIT | M_ZERO, 0, BUS_SPACE_MAXADDR, 16, 0); 267 if (sc->vtballoon_page_frames == NULL) { 268 error = ENOMEM; 269 vtballoon_dprintf(sc, VTBALLOON_ERROR, "cannot allocate page frame request array (error:%d)\n", error); 270 goto fail; 271 } 272 error = vtballoon_alloc_intrs(sc); 273 if (error) { 274 vtballoon_dprintf(sc, VTBALLOON_ERROR, "cannot allocate interrupts (error:%d)\n", error); 275 goto fail; 276 } 277 278 error = vtballoon_alloc_virtqueues(sc); 279 if (error) { 280 vtballoon_dprintf(sc, VTBALLOON_ERROR, "cannot allocate virtqueues (error:%d)\n", error); 281 goto fail; 282 } 283 284 int nrhandlers = virtio_with_feature(sc->vtballoon_dev, VIRTIO_BALLOON_F_STATS_VQ) ? 4 : 3; 285 struct irqmap info[4]; 286 287 /* Possible "Virtqueue <-> IRQ" configurations */ 288 switch (sc->vtballoon_nintr) { 289 case 1: 290 info[2] = (struct irqmap){0, -1, vtballoon_config_change_intr, "config"}; 291 info[0] = (struct irqmap){0, 0, vtballoon_inflate_vq_intr, "inflate"}; 292 info[1] = (struct irqmap){0, 1, vtballoon_deflate_vq_intr, "deflate"}; 293 info[3] = (struct irqmap){0, 2, vtballoon_stats_vq_intr, "stats"}; 294 break; 295 case 2: 296 info[2] = (struct irqmap){1, -1, vtballoon_config_change_intr, "config"}; 297 info[0] = (struct irqmap){0, 0, vtballoon_inflate_vq_intr, "inflate"}; 298 info[1] = (struct irqmap){0, 1, vtballoon_deflate_vq_intr, "deflate"}; 299 info[3] = (struct irqmap){0, 2, vtballoon_stats_vq_intr, "stats"}; 300 break; 301 case 3: 302 info[2] = (struct irqmap){2, -1, vtballoon_config_change_intr, "config"}; 303 info[0] = (struct irqmap){0, 0, vtballoon_inflate_vq_intr, "inflate"}; 304 info[1] = (struct irqmap){1, 1, vtballoon_deflate_vq_intr, "deflate"}; 305 info[3] = (struct irqmap){2, 2, vtballoon_stats_vq_intr, "stats"}; 306 break; 307 case 4: 308 info[2] = (struct irqmap){3, -1, vtballoon_config_change_intr, "config"}; 309 info[0] = (struct irqmap){0, 0, vtballoon_inflate_vq_intr, "inflate"}; 310 info[1] = (struct irqmap){1, 1, vtballoon_deflate_vq_intr, "deflate"}; 311 info[3] = (struct irqmap){2, 2, vtballoon_stats_vq_intr, "stats"}; 312 break; 313 default: 314 vtballoon_dprintf(sc, VTBALLOON_ERROR, "Invalid interrupt vector count: %d\n", sc->vtballoon_nintr); 315 goto fail; 316 } 317 for (i = 0; i < nrhandlers; i++) { 318 error = virtio_bind_intr(sc->vtballoon_dev, info[i].irq, info[i].idx, 319 info[i].handler, sc); 320 if (error) { 321 vtballoon_dprintf(sc, VTBALLOON_ERROR, "cannot bind virtqueue '%s' handler to IRQ:%d/%d\n", 322 info[i].handler_name, info[i].irq, sc->vtballoon_nintr); 323 goto fail; 324 } 325 } 326 327 for (i = 0; i < sc->vtballoon_nintr; i++) { 328 error = virtio_setup_intr(dev, i, VTBALLOON_SLZ(sc)); 329 if (error) { 330 vtballoon_dprintf(sc, VTBALLOON_ERROR, "cannot setup virtqueue interrupt:%d (error:%d)\n", i, error); 331 goto fail; 332 } 333 } 334 335 error = kthread_create(vtballoon_thread, sc, &sc->vtballoon_td, "virtio_balloon"); 336 if (error) { 337 vtballoon_dprintf(sc, VTBALLOON_ERROR, "cannot create balloon kthread (error:%d)\n", error); 338 goto fail; 339 } 340 341 virtqueue_enable_intr(sc->vtballoon_inflate_vq); 342 virtqueue_enable_intr(sc->vtballoon_deflate_vq); 343 344 if (virtio_with_feature(sc->vtballoon_dev, VIRTIO_BALLOON_F_STATS_VQ)) { 345 virtqueue_enable_intr(sc->vtballoon_stats_vq); 346 #if 0 /* enabling this causes a panic, on asserting ASSERT_SERIALIZED(sc) in vtballoon_update_stats */ 347 /* 348 * Prime this stats virtqueue with one buffer so the hypervisor can 349 * use it to signal us later. 350 */ 351 VTBALLOON_ENTER_SLZ(sc); 352 vtballoon_update_stats(sc); 353 VTBALLOON_EXIT_SLZ(sc); 354 #endif 355 } 356 357 fail: 358 if (error) 359 vtballoon_detach(dev); 360 361 return (error); 362 } 363 364 static int 365 vtballoon_detach(device_t dev) 366 { 367 struct vtballoon_softc *sc; 368 int i; 369 370 sc = device_get_softc(dev); 371 vtballoon_dprintf(sc, VTBALLOON_TRACE, "\n"); 372 373 if (sc->vtballoon_td != NULL) { 374 VTBALLOON_ENTER_SLZ(sc); 375 sc->vtballoon_flags |= VTBALLOON_FLAG_DETACH; 376 377 /* drain */ 378 wakeup_one(sc); 379 zsleep(sc->vtballoon_td, VTBALLOON_SLZ(sc), 0, "vtbdth", 0); 380 VTBALLOON_EXIT_SLZ(sc); 381 sc->vtballoon_td = NULL; 382 } 383 384 lwkt_serialize_handler_disable(VTBALLOON_SLZ(sc)); 385 386 for (i = 0; i < sc->vtballoon_nintr; i++) 387 virtio_teardown_intr(dev, i); 388 389 if (device_is_attached(dev)) { 390 vtballoon_pop(sc); 391 vtballoon_stop(sc); 392 } 393 394 if (sc->vtballoon_page_frames != NULL) { 395 contigfree(sc->vtballoon_page_frames, VTBALLOON_PAGES_PER_REQUEST * 396 sizeof(uint32_t), M_DEVBUF); 397 sc->vtballoon_page_frames = NULL; 398 } 399 return (0); 400 } 401 402 static void 403 vtballoon_negotiate_features(struct vtballoon_softc *sc) 404 { 405 device_t dev; 406 uint64_t features; 407 408 dev = sc->vtballoon_dev; 409 vtballoon_dprintf(sc, VTBALLOON_TRACE, "\n"); 410 features = virtio_negotiate_features(dev, VTBALLOON_FEATURES); 411 sc->vtballoon_features = features; 412 } 413 414 static int vtballoon_alloc_intrs(struct vtballoon_softc *sc) 415 { 416 vtballoon_dprintf(sc, VTBALLOON_TRACE, "\n"); 417 int cnt, error; 418 int intrcount = virtio_intr_count(sc->vtballoon_dev); 419 int use_config = 1; 420 421 intrcount = imin(intrcount, VTBALLOON_MAX_INTERRUPTS); 422 if (intrcount < 1) 423 return (ENXIO); 424 425 cnt = intrcount; 426 error = virtio_intr_alloc(sc->vtballoon_dev, &cnt, use_config, NULL); 427 if (error != 0) { 428 virtio_intr_release(sc->vtballoon_dev); 429 return (error); 430 } 431 sc->vtballoon_nintr = cnt; 432 vtballoon_dprintf(sc, VTBALLOON_TRACE, "%d Interrupts Allocated\n", sc->vtballoon_nintr); 433 return (0); 434 } 435 436 static int 437 vtballoon_alloc_virtqueues(struct vtballoon_softc *sc) 438 { 439 device_t dev; 440 struct vq_alloc_info vq_info[3]; 441 int nvqs; 442 443 dev = sc->vtballoon_dev; 444 vtballoon_dprintf(sc, VTBALLOON_TRACE, "\n"); 445 nvqs = 2; 446 447 VQ_ALLOC_INFO_INIT(&vq_info[0], 0, &sc->vtballoon_inflate_vq, 448 "%s inflate", device_get_nameunit(dev)); 449 450 VQ_ALLOC_INFO_INIT(&vq_info[1], 0, &sc->vtballoon_deflate_vq, 451 "%s deflate", device_get_nameunit(dev)); 452 453 if (virtio_with_feature(sc->vtballoon_dev, VIRTIO_BALLOON_F_STATS_VQ)) { 454 VQ_ALLOC_INFO_INIT(&vq_info[2], 0, &sc->vtballoon_stats_vq, 455 "%s stats", device_get_nameunit(dev)); 456 nvqs = 3; 457 } 458 return (virtio_alloc_virtqueues(dev, nvqs, vq_info)); 459 } 460 461 static void 462 vtballoon_config_change_intr(void *arg) 463 { 464 struct vtballoon_softc *sc = arg; 465 vtballoon_dprintf(sc, VTBALLOON_TRACE, "\n"); 466 ASSERT_SERIALIZED(VTBALLOON_SLZ(sc)); 467 wakeup_one(sc); 468 } 469 470 static inline void 471 vtballoon_update_stat(struct vtballoon_softc *sc, int idx, 472 uint16_t tag, uint64_t val) 473 { 474 KASSERT(idx >= VTBALLOON_S_NR, ("Stats index out of bounds")); 475 /* 476 * XXX: Required for endianess in the future 477 * sc->vtballoon_stats[idx].tag = virtio_is_little_endian(sc->vtballoon_dev) ? le16toh(tag) : tag; 478 * sc->vtballoon_stats[idx].val = virtio_is_little_endian(sc->vtballoon_dev) ? le64toh(val) : val; 479 * at the moment virtio balloon is always little endian. 480 * 481 */ 482 sc->vtballoon_stats[idx].tag = le16toh(tag); 483 sc->vtballoon_stats[idx].val = le64toh(val); 484 485 } 486 487 /* 488 * collect guest side statistics 489 * 490 * XXX: am i using the correct memory and pagefault values 491 */ 492 static unsigned int collect_balloon_stats(struct vtballoon_softc *sc) 493 { 494 #define pages_to_bytes(x) ((uint64_t)(x) << PAGE_SHIFT) 495 unsigned int idx = 0; 496 struct vmtotal total; 497 struct vmmeter vmm; 498 struct vmstats vms; 499 size_t vmt_size = sizeof(total); 500 size_t vmm_size = sizeof(vmm); 501 size_t vms_size = sizeof(vms); 502 503 vtballoon_dprintf(sc, VTBALLOON_TRACE, "Updating Stats Buffer\n"); 504 if (!kernel_sysctlbyname("vm.vmtotal", &total, &vmt_size, NULL, 0, NULL)) { 505 /* Total amount of free memory )*/ 506 vtballoon_update_stat(sc, idx++, VTBALLOON_S_MEMFREE, 507 pages_to_bytes(total.t_rm - total.t_arm)); 508 /* Total amount of memory */ 509 vtballoon_update_stat(sc, idx++, VTBALLOON_S_MEMTOT, 510 pages_to_bytes(total.t_rm)); 511 /* Available memory as in /proc */ 512 vtballoon_update_stat(sc, idx++, VTBALLOON_S_AVAIL, 513 pages_to_bytes(total.t_arm)); 514 } 515 if (!kernel_sysctlbyname("vm.vmstats", &vms, &vms_size, NULL, 0, NULL)) { 516 /* Disk caches */ 517 vtballoon_update_stat(sc, idx++, VTBALLOON_S_CACHES, 518 pages_to_bytes(vms.v_cache_count)); 519 } 520 if (!kernel_sysctlbyname("vm.vmmeter", &vmm, &vmm_size, NULL, 0, NULL)) { 521 /* Amount of memory swapped in */ 522 vtballoon_update_stat(sc, idx++, VTBALLOON_S_SWAP_IN, 523 pages_to_bytes(vmm.v_swappgsin)); 524 /* Amount of memory swapped out */ 525 vtballoon_update_stat(sc, idx++, VTBALLOON_S_SWAP_OUT, 526 pages_to_bytes(vmm.v_swappgsout)); 527 /* Number of major faults */ 528 vtballoon_update_stat(sc, idx++, VTBALLOON_S_MAJFLT, 529 vmm.v_vm_faults); 530 /* Number of minor faults */ 531 vtballoon_update_stat(sc, idx++, VTBALLOON_S_MINFLT, 532 vmm.v_intrans); 533 } 534 535 if (sc->vtballoon_debug & VTBALLOON_TRACE) { 536 static const char *vt_balloon_names[]=VTBALLOON_S_NAMES; 537 int i; 538 for (i=0; i < idx; i++) { 539 kprintf("\t%s = %lu\n", vt_balloon_names[sc->vtballoon_stats[i].tag], sc->vtballoon_stats[i].val); 540 } 541 } 542 543 return idx; 544 } 545 546 static void 547 vtballoon_update_stats(struct vtballoon_softc *sc) 548 { 549 struct virtqueue *vq = sc->vtballoon_stats_vq; 550 551 ASSERT_SERIALIZED(VTBALLOON_SLZ(sc)); 552 553 vtballoon_dprintf(sc, VTBALLOON_TRACE, "Stats Requested\n"); 554 555 struct sglist sg; 556 struct sglist_seg segs[1]; 557 unsigned int num_stats; 558 int error; 559 560 num_stats = collect_balloon_stats(sc); 561 562 sglist_init(&sg, 1, segs); 563 error = sglist_append(&sg, sc->vtballoon_stats, sizeof(sc->vtballoon_stats[0]) * num_stats); 564 KASSERT(error == 0, ("error adding page frames to sglist")); 565 566 error = virtqueue_enqueue(vq, vq, &sg, 1, 0); 567 KASSERT(error == 0, ("error enqueuing page frames to virtqueue")); 568 virtqueue_notify(sc->vtballoon_stats_vq, NULL); 569 } 570 571 /* 572 * While most virtqueues communicate guest-initiated requests to the hypervisor, 573 * the stats queue operates in reverse. The driver(host) initializes the virtqueue 574 * with a single buffer. From that point forward, all conversations consist of 575 * a hypervisor request (a call to this function) which directs us to refill 576 * the virtqueue with a fresh stats buffer. Since stats collection can sleep, 577 * we delegate the job to the vtballoon_thread which will do the actual stats 578 * collecting work. 579 */ 580 static void 581 vtballoon_stats_vq_intr(void *arg) 582 { 583 struct vtballoon_softc *sc = arg; 584 struct virtqueue *vq = sc->vtballoon_stats_vq; 585 586 ASSERT_SERIALIZED(VTBALLOON_SLZ(sc)); 587 if (sc->vtballoon_update_stats || !virtqueue_pending(vq)) 588 return; 589 590 vtballoon_dprintf(sc, VTBALLOON_TRACE, "Ballooon Stats Requested\n"); 591 sc->vtballoon_update_stats = true; 592 wakeup_one(sc); 593 virtqueue_dequeue(vq, NULL); 594 } 595 596 static void 597 vtballoon_inflate_vq_intr(void *arg) 598 { 599 struct vtballoon_softc *sc = arg; 600 struct virtqueue *vq = sc->vtballoon_inflate_vq; 601 ASSERT_SERIALIZED(VTBALLOON_SLZ(sc)); 602 if (!virtqueue_pending(vq)) 603 return; 604 wakeup_one(sc); 605 } 606 607 static void 608 vtballoon_deflate_vq_intr(void *arg) 609 { 610 struct vtballoon_softc *sc = arg; 611 struct virtqueue *vq = sc->vtballoon_deflate_vq; 612 ASSERT_SERIALIZED(VTBALLOON_SLZ(sc)); 613 if (!virtqueue_pending(vq)) 614 return; 615 wakeup_one(sc); 616 } 617 618 static void 619 vtballoon_inflate(struct vtballoon_softc *sc, int npages) 620 { 621 struct virtqueue *vq; 622 623 vm_page_t m; 624 int i; 625 626 vq = sc->vtballoon_inflate_vq; 627 628 if (npages > VTBALLOON_PAGES_PER_REQUEST) 629 npages = VTBALLOON_PAGES_PER_REQUEST; 630 631 for (i = 0; i < npages; i++) { 632 if ((m = vtballoon_alloc_page(sc)) == NULL) { 633 /* First allocate usign VTBALLOON_REGULAR_ALLOC and fall back to VTBALLOON_LOWMEM_ALLOC 634 * when the guest is under severe memory pressure. Quickly decrease the 635 * allocation rate, allowing the system to swap out pages. 636 */ 637 sc->vtballoon_pagereq = VM_ALLOC_SYSTEM | VM_ALLOC_INTERRUPT; 638 sc->vtballoon_timeout = VTBALLOON_LOWMEM_TIMEOUT; 639 break; 640 } 641 642 sc->vtballoon_page_frames[i] = 643 VM_PAGE_TO_PHYS(m) >> VIRTIO_BALLOON_PFN_SHIFT; 644 645 KASSERT(m->queue == PQ_NONE, 646 ("%s: allocated page %p on queue", __func__, m)); 647 TAILQ_INSERT_TAIL(&sc->vtballoon_pages, m, pageq); 648 } 649 650 if (i > 0) 651 vtballoon_send_page_frames(sc, vq, i); 652 } 653 654 static void 655 vtballoon_deflate(struct vtballoon_softc *sc, int npages) 656 { 657 TAILQ_HEAD(, vm_page) free_pages; 658 struct virtqueue *vq; 659 vm_page_t m; 660 int i; 661 662 vq = sc->vtballoon_deflate_vq; 663 TAILQ_INIT(&free_pages); 664 665 if (npages > VTBALLOON_PAGES_PER_REQUEST) 666 npages = VTBALLOON_PAGES_PER_REQUEST; 667 668 for (i = 0; i < npages; i++) { 669 m = TAILQ_FIRST(&sc->vtballoon_pages); 670 KASSERT(m != NULL, ("%s: no more pages to deflate", __func__)); 671 672 sc->vtballoon_page_frames[i] = 673 VM_PAGE_TO_PHYS(m) >> VIRTIO_BALLOON_PFN_SHIFT; 674 675 TAILQ_REMOVE(&sc->vtballoon_pages, m, pageq); 676 TAILQ_INSERT_TAIL(&free_pages, m, pageq); 677 } 678 679 if (i > 0) { 680 /* 681 * Note that if virtio VIRTIO_BALLOON_F_MUST_TELL_HOST 682 * feature is true, we *have* to tell host first 683 * before freeing the pages. 684 */ 685 vtballoon_send_page_frames(sc, vq, i); 686 687 while ((m = TAILQ_FIRST(&free_pages)) != NULL) { 688 TAILQ_REMOVE(&free_pages, m, pageq); 689 vtballoon_free_page(sc, m); 690 } 691 } 692 693 KASSERT((TAILQ_EMPTY(&sc->vtballoon_pages) && 694 sc->vtballoon_current_npages == 0) || 695 (!TAILQ_EMPTY(&sc->vtballoon_pages) && 696 sc->vtballoon_current_npages != 0), 697 ("%s: bogus page count %d", __func__, 698 sc->vtballoon_current_npages)); 699 } 700 701 static void 702 vtballoon_send_page_frames(struct vtballoon_softc *sc, struct virtqueue *vq, 703 int npages) 704 { 705 struct sglist sg; 706 struct sglist_seg segs[1]; 707 void *c; 708 int error; 709 710 sglist_init(&sg, 1, segs); 711 712 error = sglist_append(&sg, sc->vtballoon_page_frames, 713 npages * sizeof(uint32_t)); 714 KASSERT(error == 0, ("error adding page frames to sglist")); 715 716 error = virtqueue_enqueue(vq, vq, &sg, 1, 0); 717 KASSERT(error == 0, ("error enqueuing page frames to virtqueue")); 718 virtqueue_notify(vq, NULL); 719 720 /* 721 * Inflate and deflate operations are done synchronously. The 722 * interrupt handler will wake us up. 723 */ 724 VTBALLOON_ENTER_SLZ(sc); 725 while ((c = virtqueue_dequeue(vq, NULL)) == NULL) { 726 zsleep(sc, VTBALLOON_SLZ(sc), 0, "vtbspf", 0); 727 } 728 VTBALLOON_EXIT_SLZ(sc); 729 730 KASSERT(c == vq, ("unexpected balloon operation response")); 731 } 732 733 static void 734 vtballoon_pop(struct vtballoon_softc *sc) 735 { 736 vtballoon_dprintf(sc, VTBALLOON_TRACE, "Popping\n"); 737 738 while (!TAILQ_EMPTY(&sc->vtballoon_pages)) 739 vtballoon_deflate(sc, sc->vtballoon_current_npages); 740 } 741 742 static void 743 vtballoon_stop(struct vtballoon_softc *sc) 744 { 745 vtballoon_dprintf(sc, VTBALLOON_TRACE, "Stopping\n"); 746 747 virtqueue_disable_intr(sc->vtballoon_inflate_vq); 748 virtqueue_disable_intr(sc->vtballoon_deflate_vq); 749 /* 750 if (virtio_with_feature(sc->vtballoon_dev, VIRTIO_BALLOON_F_STATS_VQ)) { 751 virtqueue_disable_intr(sc->vtballoon_stats_vq); 752 } 753 */ 754 virtio_stop(sc->vtballoon_dev); 755 } 756 757 static vm_page_t 758 vtballoon_alloc_page(struct vtballoon_softc *sc) 759 { 760 vm_page_t m; 761 762 m = vm_page_alloc(NULL, 0, sc->vtballoon_pagereq); 763 if (m != NULL) 764 sc->vtballoon_current_npages++; 765 766 return (m); 767 } 768 769 static void 770 vtballoon_free_page(struct vtballoon_softc *sc, vm_page_t m) 771 { 772 vm_page_free_toq(m); 773 sc->vtballoon_current_npages--; 774 } 775 776 static uint32_t 777 vtballoon_desired_size(struct vtballoon_softc *sc) 778 { 779 uint32_t desired; 780 781 desired = virtio_read_dev_config_4(sc->vtballoon_dev, 782 offsetof(struct virtio_balloon_config, num_pages)); 783 784 return (le32toh(desired)); 785 } 786 787 static void 788 vtballoon_update_size(struct vtballoon_softc *sc) 789 { 790 virtio_write_dev_config_4(sc->vtballoon_dev, 791 offsetof(struct virtio_balloon_config, actual), 792 htole32(sc->vtballoon_current_npages)); 793 } 794 795 static int 796 vtballoon_sleep(struct vtballoon_softc *sc) 797 { 798 int rc, timeout; 799 uint32_t current, desired; 800 801 rc = 0; 802 current = sc->vtballoon_current_npages; 803 sc->vtballoon_pagereq = VM_ALLOC_NORMAL | VM_ALLOC_INTERRUPT; 804 805 VTBALLOON_ENTER_SLZ(sc); 806 for (;;) { 807 if (sc->vtballoon_flags & VTBALLOON_FLAG_DETACH) { 808 rc = 1; 809 break; 810 } 811 812 desired = vtballoon_desired_size(sc); 813 if (desired != sc->vtballoon_desired_npages) 814 vtballoon_dprintf(sc, VTBALLOON_DEBUG, "balloon %s %d -> %d (4K pages)\n", 815 desired < sc->vtballoon_desired_npages ? "deflating" : "inflating", 816 current, desired); 817 818 sc->vtballoon_desired_npages = desired; 819 820 /* 821 * If given, use non-zero timeout on the first time through 822 * the loop. On subsequent times, timeout will be zero so 823 * we will reevaluate the desired size of the balloon and 824 * break out to retry if needed. 825 */ 826 timeout = sc->vtballoon_timeout; 827 sc->vtballoon_timeout = 0; 828 829 if (current > desired) 830 break; 831 else if (current < desired && timeout == 0) 832 break; 833 else if (sc->vtballoon_update_stats) 834 break; 835 else if (!timeout) 836 vtballoon_dprintf(sc, VTBALLOON_TRACE, "balloon %d (4K pages) reached\n", current); 837 838 zsleep(sc, VTBALLOON_SLZ(sc), 0, "vtbslp", timeout); 839 } 840 VTBALLOON_EXIT_SLZ(sc); 841 842 return (rc); 843 } 844 845 static void 846 vtballoon_thread(void *arg) 847 { 848 struct vtballoon_softc *sc = arg; 849 vtballoon_dprintf(sc, VTBALLOON_TRACE, "Thread started.\n"); 850 851 uint32_t current, desired; 852 for (;;) { 853 if (vtballoon_sleep(sc) != 0) 854 break; 855 856 current = sc->vtballoon_current_npages; 857 desired = sc->vtballoon_desired_npages; 858 859 if (desired != current) { 860 if (desired > current) 861 vtballoon_inflate(sc, desired - current); 862 else 863 vtballoon_deflate(sc, current - desired); 864 865 vtballoon_update_size(sc); 866 } 867 if (sc->vtballoon_update_stats) { 868 vtballoon_update_stats(sc); 869 sc->vtballoon_update_stats = false; 870 } 871 } 872 873 kthread_exit(); 874 } 875 876 static void 877 vtballoon_get_tunables(struct vtballoon_softc *sc) 878 { 879 char tmpstr[64]; 880 vtballoon_dprintf(sc, VTBALLOON_TRACE, "\n"); 881 882 TUNABLE_INT_FETCH("hw.vtballoon.debug_level", &sc->vtballoon_debug); 883 884 ksnprintf(tmpstr, sizeof(tmpstr), "dev.vtballoon.%d.debug_level", 885 device_get_unit(sc->vtballoon_dev)); 886 TUNABLE_INT_FETCH(tmpstr, &sc->vtballoon_debug); 887 } 888 889 static void 890 vtballoon_add_sysctl(struct vtballoon_softc *sc) 891 { 892 device_t dev; 893 struct sysctl_ctx_list *ctx; 894 struct sysctl_oid *tree; 895 struct sysctl_oid_list *child; 896 897 dev = sc->vtballoon_dev; 898 vtballoon_dprintf(sc, VTBALLOON_TRACE, "\n"); 899 900 ctx = device_get_sysctl_ctx(dev); 901 tree = device_get_sysctl_tree(dev); 902 child = SYSCTL_CHILDREN(tree); 903 904 SYSCTL_ADD_INT(ctx, child, OID_AUTO, "debug_level", 905 CTLFLAG_RW, &sc->vtballoon_debug, 0, 906 "Debug level"); 907 908 SYSCTL_ADD_UINT(ctx, child, OID_AUTO, "desired", 909 CTLFLAG_RD, &sc->vtballoon_desired_npages, sizeof(uint32_t), 910 "Desired balloon size in pages"); 911 912 SYSCTL_ADD_UINT(ctx, child, OID_AUTO, "current", 913 CTLFLAG_RD, &sc->vtballoon_current_npages, sizeof(uint32_t), 914 "Current balloon size in pages"); 915 } 916