1 /*-
2  * SPDX-License-Identifier: BSD-2-Clause
3  *
4  * Copyright (c) 2011, Bryan Venteicher <bryanv@FreeBSD.org>
5  * All rights reserved.
6  *
7  * Redistribution and use in source and binary forms, with or without
8  * modification, are permitted provided that the following conditions
9  * are met:
10  * 1. Redistributions of source code must retain the above copyright
11  *    notice unmodified, this list of conditions, and the following
12  *    disclaimer.
13  * 2. Redistributions in binary form must reproduce the above copyright
14  *    notice, this list of conditions and the following disclaimer in the
15  *    documentation and/or other materials provided with the distribution.
16  *
17  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
18  * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
19  * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
20  * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
21  * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
22  * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
23  * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
24  * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
25  * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
26  * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
27  */
28 
29 /* Driver for VirtIO memory balloon devices. */
30 
31 #include <sys/cdefs.h>
32 #include <sys/param.h>
33 #include <sys/systm.h>
34 #include <sys/kernel.h>
35 #include <sys/endian.h>
36 #include <sys/kthread.h>
37 #include <sys/malloc.h>
38 #include <sys/module.h>
39 #include <sys/sglist.h>
40 #include <sys/sysctl.h>
41 #include <sys/lock.h>
42 #include <sys/mutex.h>
43 #include <sys/queue.h>
44 
45 #include <vm/vm.h>
46 #include <vm/vm_page.h>
47 
48 #include <machine/bus.h>
49 #include <machine/resource.h>
50 #include <sys/bus.h>
51 #include <sys/rman.h>
52 
53 #include <dev/virtio/virtio.h>
54 #include <dev/virtio/virtqueue.h>
55 #include <dev/virtio/balloon/virtio_balloon.h>
56 
57 #include "virtio_if.h"
58 
59 struct vtballoon_softc {
60 	device_t		 vtballoon_dev;
61 	struct mtx		 vtballoon_mtx;
62 	uint64_t		 vtballoon_features;
63 	uint32_t		 vtballoon_flags;
64 #define VTBALLOON_FLAG_DETACH	 0x01
65 
66 	struct virtqueue	*vtballoon_inflate_vq;
67 	struct virtqueue	*vtballoon_deflate_vq;
68 
69 	uint32_t		 vtballoon_desired_npages;
70 	uint32_t		 vtballoon_current_npages;
71 	TAILQ_HEAD(,vm_page)	 vtballoon_pages;
72 
73 	struct thread		*vtballoon_td;
74 	uint32_t		*vtballoon_page_frames;
75 	int			 vtballoon_timeout;
76 };
77 
78 static struct virtio_feature_desc vtballoon_feature_desc[] = {
79 	{ VIRTIO_BALLOON_F_MUST_TELL_HOST,	"MustTellHost"	},
80 	{ VIRTIO_BALLOON_F_STATS_VQ,		"StatsVq"	},
81 	{ VIRTIO_BALLOON_F_DEFLATE_ON_OOM,	"DeflateOnOOM"	},
82 
83 	{ 0, NULL }
84 };
85 
86 static int	vtballoon_probe(device_t);
87 static int	vtballoon_attach(device_t);
88 static int	vtballoon_detach(device_t);
89 static int	vtballoon_config_change(device_t);
90 
91 static int	vtballoon_negotiate_features(struct vtballoon_softc *);
92 static int	vtballoon_setup_features(struct vtballoon_softc *);
93 static int	vtballoon_alloc_virtqueues(struct vtballoon_softc *);
94 
95 static void	vtballoon_vq_intr(void *);
96 
97 static void	vtballoon_inflate(struct vtballoon_softc *, int);
98 static void	vtballoon_deflate(struct vtballoon_softc *, int);
99 
100 static void	vtballoon_send_page_frames(struct vtballoon_softc *,
101 		    struct virtqueue *, int);
102 
103 static void	vtballoon_pop(struct vtballoon_softc *);
104 static void	vtballoon_stop(struct vtballoon_softc *);
105 
106 static vm_page_t
107 		vtballoon_alloc_page(struct vtballoon_softc *);
108 static void	vtballoon_free_page(struct vtballoon_softc *, vm_page_t);
109 
110 static int	vtballoon_sleep(struct vtballoon_softc *);
111 static void	vtballoon_thread(void *);
112 static void	vtballoon_setup_sysctl(struct vtballoon_softc *);
113 
114 #define vtballoon_modern(_sc) \
115     (((_sc)->vtballoon_features & VIRTIO_F_VERSION_1) != 0)
116 
117 /* Features desired/implemented by this driver. */
118 #define VTBALLOON_FEATURES		VIRTIO_BALLOON_F_MUST_TELL_HOST
119 
120 /* Timeout between retries when the balloon needs inflating. */
121 #define VTBALLOON_LOWMEM_TIMEOUT	hz
122 
123 /*
124  * Maximum number of pages we'll request to inflate or deflate
125  * the balloon in one virtqueue request. Both Linux and NetBSD
126  * have settled on 256, doing up to 1MB at a time.
127  */
128 #define VTBALLOON_PAGES_PER_REQUEST	256
129 
130 /* Must be able to fix all pages frames in one page (segment). */
131 CTASSERT(VTBALLOON_PAGES_PER_REQUEST * sizeof(uint32_t) <= PAGE_SIZE);
132 
133 #define VTBALLOON_MTX(_sc)		&(_sc)->vtballoon_mtx
134 #define VTBALLOON_LOCK_INIT(_sc, _name)	mtx_init(VTBALLOON_MTX((_sc)), _name, \
135 					    "VirtIO Balloon Lock", MTX_DEF)
136 #define VTBALLOON_LOCK(_sc)		mtx_lock(VTBALLOON_MTX((_sc)))
137 #define VTBALLOON_UNLOCK(_sc)		mtx_unlock(VTBALLOON_MTX((_sc)))
138 #define VTBALLOON_LOCK_DESTROY(_sc)	mtx_destroy(VTBALLOON_MTX((_sc)))
139 
140 static device_method_t vtballoon_methods[] = {
141 	/* Device methods. */
142 	DEVMETHOD(device_probe,		vtballoon_probe),
143 	DEVMETHOD(device_attach,	vtballoon_attach),
144 	DEVMETHOD(device_detach,	vtballoon_detach),
145 
146 	/* VirtIO methods. */
147 	DEVMETHOD(virtio_config_change, vtballoon_config_change),
148 
149 	DEVMETHOD_END
150 };
151 
152 static driver_t vtballoon_driver = {
153 	"vtballoon",
154 	vtballoon_methods,
155 	sizeof(struct vtballoon_softc)
156 };
157 
158 VIRTIO_DRIVER_MODULE(virtio_balloon, vtballoon_driver, 0, 0);
159 MODULE_VERSION(virtio_balloon, 1);
160 MODULE_DEPEND(virtio_balloon, virtio, 1, 1, 1);
161 
162 VIRTIO_SIMPLE_PNPINFO(virtio_balloon, VIRTIO_ID_BALLOON,
163     "VirtIO Balloon Adapter");
164 
165 static int
166 vtballoon_probe(device_t dev)
167 {
168 	return (VIRTIO_SIMPLE_PROBE(dev, virtio_balloon));
169 }
170 
171 static int
172 vtballoon_attach(device_t dev)
173 {
174 	struct vtballoon_softc *sc;
175 	int error;
176 
177 	sc = device_get_softc(dev);
178 	sc->vtballoon_dev = dev;
179 	virtio_set_feature_desc(dev, vtballoon_feature_desc);
180 
181 	VTBALLOON_LOCK_INIT(sc, device_get_nameunit(dev));
182 	TAILQ_INIT(&sc->vtballoon_pages);
183 
184 	vtballoon_setup_sysctl(sc);
185 
186 	error = vtballoon_setup_features(sc);
187 	if (error) {
188 		device_printf(dev, "cannot setup features\n");
189 		goto fail;
190 	}
191 
192 	sc->vtballoon_page_frames = malloc(VTBALLOON_PAGES_PER_REQUEST *
193 	    sizeof(uint32_t), M_DEVBUF, M_NOWAIT | M_ZERO);
194 	if (sc->vtballoon_page_frames == NULL) {
195 		error = ENOMEM;
196 		device_printf(dev,
197 		    "cannot allocate page frame request array\n");
198 		goto fail;
199 	}
200 
201 	error = vtballoon_alloc_virtqueues(sc);
202 	if (error) {
203 		device_printf(dev, "cannot allocate virtqueues\n");
204 		goto fail;
205 	}
206 
207 	error = virtio_setup_intr(dev, INTR_TYPE_MISC);
208 	if (error) {
209 		device_printf(dev, "cannot setup virtqueue interrupts\n");
210 		goto fail;
211 	}
212 
213 	error = kthread_add(vtballoon_thread, sc, NULL, &sc->vtballoon_td,
214 	    0, 0, "virtio_balloon");
215 	if (error) {
216 		device_printf(dev, "cannot create balloon kthread\n");
217 		goto fail;
218 	}
219 
220 	virtqueue_enable_intr(sc->vtballoon_inflate_vq);
221 	virtqueue_enable_intr(sc->vtballoon_deflate_vq);
222 
223 fail:
224 	if (error)
225 		vtballoon_detach(dev);
226 
227 	return (error);
228 }
229 
230 static int
231 vtballoon_detach(device_t dev)
232 {
233 	struct vtballoon_softc *sc;
234 
235 	sc = device_get_softc(dev);
236 
237 	if (sc->vtballoon_td != NULL) {
238 		VTBALLOON_LOCK(sc);
239 		sc->vtballoon_flags |= VTBALLOON_FLAG_DETACH;
240 		wakeup_one(sc);
241 		msleep(sc->vtballoon_td, VTBALLOON_MTX(sc), 0, "vtbdth", 0);
242 		VTBALLOON_UNLOCK(sc);
243 
244 		sc->vtballoon_td = NULL;
245 	}
246 
247 	if (device_is_attached(dev)) {
248 		vtballoon_pop(sc);
249 		vtballoon_stop(sc);
250 	}
251 
252 	if (sc->vtballoon_page_frames != NULL) {
253 		free(sc->vtballoon_page_frames, M_DEVBUF);
254 		sc->vtballoon_page_frames = NULL;
255 	}
256 
257 	VTBALLOON_LOCK_DESTROY(sc);
258 
259 	return (0);
260 }
261 
262 static int
263 vtballoon_config_change(device_t dev)
264 {
265 	struct vtballoon_softc *sc;
266 
267 	sc = device_get_softc(dev);
268 
269 	VTBALLOON_LOCK(sc);
270 	wakeup_one(sc);
271 	VTBALLOON_UNLOCK(sc);
272 
273 	return (1);
274 }
275 
276 static int
277 vtballoon_negotiate_features(struct vtballoon_softc *sc)
278 {
279 	device_t dev;
280 	uint64_t features;
281 
282 	dev = sc->vtballoon_dev;
283 	features = VTBALLOON_FEATURES;
284 
285 	sc->vtballoon_features = virtio_negotiate_features(dev, features);
286 	return (virtio_finalize_features(dev));
287 }
288 
289 static int
290 vtballoon_setup_features(struct vtballoon_softc *sc)
291 {
292 	int error;
293 
294 	error = vtballoon_negotiate_features(sc);
295 	if (error)
296 		return (error);
297 
298 	return (0);
299 }
300 
301 static int
302 vtballoon_alloc_virtqueues(struct vtballoon_softc *sc)
303 {
304 	device_t dev;
305 	struct vq_alloc_info vq_info[2];
306 	int nvqs;
307 
308 	dev = sc->vtballoon_dev;
309 	nvqs = 2;
310 
311 	VQ_ALLOC_INFO_INIT(&vq_info[0], 0, vtballoon_vq_intr, sc,
312 	    &sc->vtballoon_inflate_vq, "%s inflate", device_get_nameunit(dev));
313 
314 	VQ_ALLOC_INFO_INIT(&vq_info[1], 0, vtballoon_vq_intr, sc,
315 	    &sc->vtballoon_deflate_vq, "%s deflate", device_get_nameunit(dev));
316 
317 	return (virtio_alloc_virtqueues(dev, 0, nvqs, vq_info));
318 }
319 
320 static void
321 vtballoon_vq_intr(void *xsc)
322 {
323 	struct vtballoon_softc *sc;
324 
325 	sc = xsc;
326 
327 	VTBALLOON_LOCK(sc);
328 	wakeup_one(sc);
329 	VTBALLOON_UNLOCK(sc);
330 }
331 
332 static void
333 vtballoon_inflate(struct vtballoon_softc *sc, int npages)
334 {
335 	struct virtqueue *vq;
336 	vm_page_t m;
337 	int i;
338 
339 	vq = sc->vtballoon_inflate_vq;
340 
341 	if (npages > VTBALLOON_PAGES_PER_REQUEST)
342 		npages = VTBALLOON_PAGES_PER_REQUEST;
343 
344 	for (i = 0; i < npages; i++) {
345 		if ((m = vtballoon_alloc_page(sc)) == NULL) {
346 			sc->vtballoon_timeout = VTBALLOON_LOWMEM_TIMEOUT;
347 			break;
348 		}
349 
350 		sc->vtballoon_page_frames[i] =
351 		    VM_PAGE_TO_PHYS(m) >> VIRTIO_BALLOON_PFN_SHIFT;
352 
353 		KASSERT(m->a.queue == PQ_NONE,
354 		    ("%s: allocated page %p on queue", __func__, m));
355 		TAILQ_INSERT_TAIL(&sc->vtballoon_pages, m, plinks.q);
356 	}
357 
358 	if (i > 0)
359 		vtballoon_send_page_frames(sc, vq, i);
360 }
361 
362 static void
363 vtballoon_deflate(struct vtballoon_softc *sc, int npages)
364 {
365 	TAILQ_HEAD(, vm_page) free_pages;
366 	struct virtqueue *vq;
367 	vm_page_t m;
368 	int i;
369 
370 	vq = sc->vtballoon_deflate_vq;
371 	TAILQ_INIT(&free_pages);
372 
373 	if (npages > VTBALLOON_PAGES_PER_REQUEST)
374 		npages = VTBALLOON_PAGES_PER_REQUEST;
375 
376 	for (i = 0; i < npages; i++) {
377 		m = TAILQ_FIRST(&sc->vtballoon_pages);
378 		KASSERT(m != NULL, ("%s: no more pages to deflate", __func__));
379 
380 		sc->vtballoon_page_frames[i] =
381 		    VM_PAGE_TO_PHYS(m) >> VIRTIO_BALLOON_PFN_SHIFT;
382 
383 		TAILQ_REMOVE(&sc->vtballoon_pages, m, plinks.q);
384 		TAILQ_INSERT_TAIL(&free_pages, m, plinks.q);
385 	}
386 
387 	if (i > 0) {
388 		/* Always tell host first before freeing the pages. */
389 		vtballoon_send_page_frames(sc, vq, i);
390 
391 		while ((m = TAILQ_FIRST(&free_pages)) != NULL) {
392 			TAILQ_REMOVE(&free_pages, m, plinks.q);
393 			vtballoon_free_page(sc, m);
394 		}
395 	}
396 
397 	KASSERT((TAILQ_EMPTY(&sc->vtballoon_pages) &&
398 	    sc->vtballoon_current_npages == 0) ||
399 	    (!TAILQ_EMPTY(&sc->vtballoon_pages) &&
400 	    sc->vtballoon_current_npages != 0),
401 	    ("%s: bogus page count %d", __func__,
402 	    sc->vtballoon_current_npages));
403 }
404 
405 static void
406 vtballoon_send_page_frames(struct vtballoon_softc *sc, struct virtqueue *vq,
407     int npages)
408 {
409 	struct sglist sg;
410 	struct sglist_seg segs[1];
411 	void *c;
412 	int error __diagused;
413 
414 	sglist_init(&sg, 1, segs);
415 
416 	error = sglist_append(&sg, sc->vtballoon_page_frames,
417 	    npages * sizeof(uint32_t));
418 	KASSERT(error == 0, ("error adding page frames to sglist"));
419 
420 	error = virtqueue_enqueue(vq, vq, &sg, 1, 0);
421 	KASSERT(error == 0, ("error enqueuing page frames to virtqueue"));
422 	virtqueue_notify(vq);
423 
424 	/*
425 	 * Inflate and deflate operations are done synchronously. The
426 	 * interrupt handler will wake us up.
427 	 */
428 	VTBALLOON_LOCK(sc);
429 	while ((c = virtqueue_dequeue(vq, NULL)) == NULL)
430 		msleep(sc, VTBALLOON_MTX(sc), 0, "vtbspf", 0);
431 	VTBALLOON_UNLOCK(sc);
432 
433 	KASSERT(c == vq, ("unexpected balloon operation response"));
434 }
435 
436 static void
437 vtballoon_pop(struct vtballoon_softc *sc)
438 {
439 
440 	while (!TAILQ_EMPTY(&sc->vtballoon_pages))
441 		vtballoon_deflate(sc, sc->vtballoon_current_npages);
442 }
443 
444 static void
445 vtballoon_stop(struct vtballoon_softc *sc)
446 {
447 
448 	virtqueue_disable_intr(sc->vtballoon_inflate_vq);
449 	virtqueue_disable_intr(sc->vtballoon_deflate_vq);
450 
451 	virtio_stop(sc->vtballoon_dev);
452 }
453 
454 static vm_page_t
455 vtballoon_alloc_page(struct vtballoon_softc *sc)
456 {
457 	vm_page_t m;
458 
459 	m = vm_page_alloc_noobj(VM_ALLOC_NODUMP);
460 	if (m != NULL)
461 		sc->vtballoon_current_npages++;
462 
463 	return (m);
464 }
465 
466 static void
467 vtballoon_free_page(struct vtballoon_softc *sc, vm_page_t m)
468 {
469 
470 	vm_page_free(m);
471 	sc->vtballoon_current_npages--;
472 }
473 
474 static uint32_t
475 vtballoon_desired_size(struct vtballoon_softc *sc)
476 {
477 	uint32_t desired;
478 
479 	desired = virtio_read_dev_config_4(sc->vtballoon_dev,
480 	    offsetof(struct virtio_balloon_config, num_pages));
481 
482 	if (vtballoon_modern(sc))
483 		return (desired);
484 	else
485 		return (le32toh(desired));
486 }
487 
488 static void
489 vtballoon_update_size(struct vtballoon_softc *sc)
490 {
491 	uint32_t npages;
492 
493 	npages = sc->vtballoon_current_npages;
494 	if (!vtballoon_modern(sc))
495 		npages = htole32(npages);
496 
497 	virtio_write_dev_config_4(sc->vtballoon_dev,
498 	    offsetof(struct virtio_balloon_config, actual), npages);
499 }
500 
501 static int
502 vtballoon_sleep(struct vtballoon_softc *sc)
503 {
504 	int rc, timeout;
505 	uint32_t current, desired;
506 
507 	rc = 0;
508 	current = sc->vtballoon_current_npages;
509 
510 	VTBALLOON_LOCK(sc);
511 	for (;;) {
512 		if (sc->vtballoon_flags & VTBALLOON_FLAG_DETACH) {
513 			rc = 1;
514 			break;
515 		}
516 
517 		desired = vtballoon_desired_size(sc);
518 		sc->vtballoon_desired_npages = desired;
519 
520 		/*
521 		 * If given, use non-zero timeout on the first time through
522 		 * the loop. On subsequent times, timeout will be zero so
523 		 * we will reevaluate the desired size of the balloon and
524 		 * break out to retry if needed.
525 		 */
526 		timeout = sc->vtballoon_timeout;
527 		sc->vtballoon_timeout = 0;
528 
529 		if (current > desired)
530 			break;
531 		if (current < desired && timeout == 0)
532 			break;
533 
534 		msleep(sc, VTBALLOON_MTX(sc), 0, "vtbslp", timeout);
535 	}
536 	VTBALLOON_UNLOCK(sc);
537 
538 	return (rc);
539 }
540 
541 static void
542 vtballoon_thread(void *xsc)
543 {
544 	struct vtballoon_softc *sc;
545 	uint32_t current, desired;
546 
547 	sc = xsc;
548 
549 	for (;;) {
550 		if (vtballoon_sleep(sc) != 0)
551 			break;
552 
553 		current = sc->vtballoon_current_npages;
554 		desired = sc->vtballoon_desired_npages;
555 
556 		if (desired != current) {
557 			if (desired > current)
558 				vtballoon_inflate(sc, desired - current);
559 			else
560 				vtballoon_deflate(sc, current - desired);
561 
562 			vtballoon_update_size(sc);
563 		}
564 	}
565 
566 	kthread_exit();
567 }
568 
569 static void
570 vtballoon_setup_sysctl(struct vtballoon_softc *sc)
571 {
572 	device_t dev;
573 	struct sysctl_ctx_list *ctx;
574 	struct sysctl_oid *tree;
575 	struct sysctl_oid_list *child;
576 
577 	dev = sc->vtballoon_dev;
578 	ctx = device_get_sysctl_ctx(dev);
579 	tree = device_get_sysctl_tree(dev);
580 	child = SYSCTL_CHILDREN(tree);
581 
582 	SYSCTL_ADD_UINT(ctx, child, OID_AUTO, "desired",
583 	    CTLFLAG_RD, &sc->vtballoon_desired_npages, sizeof(uint32_t),
584 	    "Desired balloon size in pages");
585 
586 	SYSCTL_ADD_UINT(ctx, child, OID_AUTO, "current",
587 	    CTLFLAG_RD, &sc->vtballoon_current_npages, sizeof(uint32_t),
588 	    "Current balloon size in pages");
589 }
590