1 /*-
2  * SPDX-License-Identifier: BSD-2-Clause-FreeBSD
3  *
4  * Copyright (c) 2011, Bryan Venteicher <bryanv@FreeBSD.org>
5  * All rights reserved.
6  *
7  * Redistribution and use in source and binary forms, with or without
8  * modification, are permitted provided that the following conditions
9  * are met:
10  * 1. Redistributions of source code must retain the above copyright
11  *    notice unmodified, this list of conditions, and the following
12  *    disclaimer.
13  * 2. Redistributions in binary form must reproduce the above copyright
14  *    notice, this list of conditions and the following disclaimer in the
15  *    documentation and/or other materials provided with the distribution.
16  *
17  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
18  * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
19  * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
20  * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
21  * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
22  * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
23  * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
24  * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
25  * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
26  * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
27  */
28 
29 /* Driver for VirtIO memory balloon devices. */
30 
31 #include <sys/cdefs.h>
32 __FBSDID("$FreeBSD$");
33 
34 #include <sys/param.h>
35 #include <sys/systm.h>
36 #include <sys/kernel.h>
37 #include <sys/endian.h>
38 #include <sys/kthread.h>
39 #include <sys/malloc.h>
40 #include <sys/module.h>
41 #include <sys/sglist.h>
42 #include <sys/sysctl.h>
43 #include <sys/lock.h>
44 #include <sys/mutex.h>
45 #include <sys/queue.h>
46 
47 #include <vm/vm.h>
48 #include <vm/vm_page.h>
49 
50 #include <machine/bus.h>
51 #include <machine/resource.h>
52 #include <sys/bus.h>
53 #include <sys/rman.h>
54 
55 #include <dev/virtio/virtio.h>
56 #include <dev/virtio/virtqueue.h>
57 #include <dev/virtio/balloon/virtio_balloon.h>
58 
59 #include "virtio_if.h"
60 
61 struct vtballoon_softc {
62 	device_t		 vtballoon_dev;
63 	struct mtx		 vtballoon_mtx;
64 	uint64_t		 vtballoon_features;
65 	uint32_t		 vtballoon_flags;
66 #define VTBALLOON_FLAG_DETACH	 0x01
67 
68 	struct virtqueue	*vtballoon_inflate_vq;
69 	struct virtqueue	*vtballoon_deflate_vq;
70 
71 	uint32_t		 vtballoon_desired_npages;
72 	uint32_t		 vtballoon_current_npages;
73 	TAILQ_HEAD(,vm_page)	 vtballoon_pages;
74 
75 	struct thread		*vtballoon_td;
76 	uint32_t		*vtballoon_page_frames;
77 	int			 vtballoon_timeout;
78 };
79 
80 static struct virtio_feature_desc vtballoon_feature_desc[] = {
81 	{ VIRTIO_BALLOON_F_MUST_TELL_HOST,	"MustTellHost"	},
82 	{ VIRTIO_BALLOON_F_STATS_VQ,		"StatsVq"	},
83 	{ VIRTIO_BALLOON_F_DEFLATE_ON_OOM,	"DeflateOnOOM"	},
84 
85 	{ 0, NULL }
86 };
87 
88 static int	vtballoon_probe(device_t);
89 static int	vtballoon_attach(device_t);
90 static int	vtballoon_detach(device_t);
91 static int	vtballoon_config_change(device_t);
92 
93 static int	vtballoon_negotiate_features(struct vtballoon_softc *);
94 static int	vtballoon_setup_features(struct vtballoon_softc *);
95 static int	vtballoon_alloc_virtqueues(struct vtballoon_softc *);
96 
97 static void	vtballoon_vq_intr(void *);
98 
99 static void	vtballoon_inflate(struct vtballoon_softc *, int);
100 static void	vtballoon_deflate(struct vtballoon_softc *, int);
101 
102 static void	vtballoon_send_page_frames(struct vtballoon_softc *,
103 		    struct virtqueue *, int);
104 
105 static void	vtballoon_pop(struct vtballoon_softc *);
106 static void	vtballoon_stop(struct vtballoon_softc *);
107 
108 static vm_page_t
109 		vtballoon_alloc_page(struct vtballoon_softc *);
110 static void	vtballoon_free_page(struct vtballoon_softc *, vm_page_t);
111 
112 static int	vtballoon_sleep(struct vtballoon_softc *);
113 static void	vtballoon_thread(void *);
114 static void	vtballoon_setup_sysctl(struct vtballoon_softc *);
115 
116 #define vtballoon_modern(_sc) \
117     (((_sc)->vtballoon_features & VIRTIO_F_VERSION_1) != 0)
118 
119 /* Features desired/implemented by this driver. */
120 #define VTBALLOON_FEATURES		VIRTIO_BALLOON_F_MUST_TELL_HOST
121 
122 /* Timeout between retries when the balloon needs inflating. */
123 #define VTBALLOON_LOWMEM_TIMEOUT	hz
124 
125 /*
126  * Maximum number of pages we'll request to inflate or deflate
127  * the balloon in one virtqueue request. Both Linux and NetBSD
128  * have settled on 256, doing up to 1MB at a time.
129  */
130 #define VTBALLOON_PAGES_PER_REQUEST	256
131 
132 /* Must be able to fix all pages frames in one page (segment). */
133 CTASSERT(VTBALLOON_PAGES_PER_REQUEST * sizeof(uint32_t) <= PAGE_SIZE);
134 
135 #define VTBALLOON_MTX(_sc)		&(_sc)->vtballoon_mtx
136 #define VTBALLOON_LOCK_INIT(_sc, _name)	mtx_init(VTBALLOON_MTX((_sc)), _name, \
137 					    "VirtIO Balloon Lock", MTX_DEF)
138 #define VTBALLOON_LOCK(_sc)		mtx_lock(VTBALLOON_MTX((_sc)))
139 #define VTBALLOON_UNLOCK(_sc)		mtx_unlock(VTBALLOON_MTX((_sc)))
140 #define VTBALLOON_LOCK_DESTROY(_sc)	mtx_destroy(VTBALLOON_MTX((_sc)))
141 
142 static device_method_t vtballoon_methods[] = {
143 	/* Device methods. */
144 	DEVMETHOD(device_probe,		vtballoon_probe),
145 	DEVMETHOD(device_attach,	vtballoon_attach),
146 	DEVMETHOD(device_detach,	vtballoon_detach),
147 
148 	/* VirtIO methods. */
149 	DEVMETHOD(virtio_config_change, vtballoon_config_change),
150 
151 	DEVMETHOD_END
152 };
153 
154 static driver_t vtballoon_driver = {
155 	"vtballoon",
156 	vtballoon_methods,
157 	sizeof(struct vtballoon_softc)
158 };
159 
160 VIRTIO_DRIVER_MODULE(virtio_balloon, vtballoon_driver, 0, 0);
161 MODULE_VERSION(virtio_balloon, 1);
162 MODULE_DEPEND(virtio_balloon, virtio, 1, 1, 1);
163 
164 VIRTIO_SIMPLE_PNPINFO(virtio_balloon, VIRTIO_ID_BALLOON,
165     "VirtIO Balloon Adapter");
166 
167 static int
168 vtballoon_probe(device_t dev)
169 {
170 	return (VIRTIO_SIMPLE_PROBE(dev, virtio_balloon));
171 }
172 
173 static int
174 vtballoon_attach(device_t dev)
175 {
176 	struct vtballoon_softc *sc;
177 	int error;
178 
179 	sc = device_get_softc(dev);
180 	sc->vtballoon_dev = dev;
181 	virtio_set_feature_desc(dev, vtballoon_feature_desc);
182 
183 	VTBALLOON_LOCK_INIT(sc, device_get_nameunit(dev));
184 	TAILQ_INIT(&sc->vtballoon_pages);
185 
186 	vtballoon_setup_sysctl(sc);
187 
188 	error = vtballoon_setup_features(sc);
189 	if (error) {
190 		device_printf(dev, "cannot setup features\n");
191 		goto fail;
192 	}
193 
194 	sc->vtballoon_page_frames = malloc(VTBALLOON_PAGES_PER_REQUEST *
195 	    sizeof(uint32_t), M_DEVBUF, M_NOWAIT | M_ZERO);
196 	if (sc->vtballoon_page_frames == NULL) {
197 		error = ENOMEM;
198 		device_printf(dev,
199 		    "cannot allocate page frame request array\n");
200 		goto fail;
201 	}
202 
203 	error = vtballoon_alloc_virtqueues(sc);
204 	if (error) {
205 		device_printf(dev, "cannot allocate virtqueues\n");
206 		goto fail;
207 	}
208 
209 	error = virtio_setup_intr(dev, INTR_TYPE_MISC);
210 	if (error) {
211 		device_printf(dev, "cannot setup virtqueue interrupts\n");
212 		goto fail;
213 	}
214 
215 	error = kthread_add(vtballoon_thread, sc, NULL, &sc->vtballoon_td,
216 	    0, 0, "virtio_balloon");
217 	if (error) {
218 		device_printf(dev, "cannot create balloon kthread\n");
219 		goto fail;
220 	}
221 
222 	virtqueue_enable_intr(sc->vtballoon_inflate_vq);
223 	virtqueue_enable_intr(sc->vtballoon_deflate_vq);
224 
225 fail:
226 	if (error)
227 		vtballoon_detach(dev);
228 
229 	return (error);
230 }
231 
232 static int
233 vtballoon_detach(device_t dev)
234 {
235 	struct vtballoon_softc *sc;
236 
237 	sc = device_get_softc(dev);
238 
239 	if (sc->vtballoon_td != NULL) {
240 		VTBALLOON_LOCK(sc);
241 		sc->vtballoon_flags |= VTBALLOON_FLAG_DETACH;
242 		wakeup_one(sc);
243 		msleep(sc->vtballoon_td, VTBALLOON_MTX(sc), 0, "vtbdth", 0);
244 		VTBALLOON_UNLOCK(sc);
245 
246 		sc->vtballoon_td = NULL;
247 	}
248 
249 	if (device_is_attached(dev)) {
250 		vtballoon_pop(sc);
251 		vtballoon_stop(sc);
252 	}
253 
254 	if (sc->vtballoon_page_frames != NULL) {
255 		free(sc->vtballoon_page_frames, M_DEVBUF);
256 		sc->vtballoon_page_frames = NULL;
257 	}
258 
259 	VTBALLOON_LOCK_DESTROY(sc);
260 
261 	return (0);
262 }
263 
264 static int
265 vtballoon_config_change(device_t dev)
266 {
267 	struct vtballoon_softc *sc;
268 
269 	sc = device_get_softc(dev);
270 
271 	VTBALLOON_LOCK(sc);
272 	wakeup_one(sc);
273 	VTBALLOON_UNLOCK(sc);
274 
275 	return (1);
276 }
277 
278 static int
279 vtballoon_negotiate_features(struct vtballoon_softc *sc)
280 {
281 	device_t dev;
282 	uint64_t features;
283 
284 	dev = sc->vtballoon_dev;
285 	features = VTBALLOON_FEATURES;
286 
287 	sc->vtballoon_features = virtio_negotiate_features(dev, features);
288 	return (virtio_finalize_features(dev));
289 }
290 
291 static int
292 vtballoon_setup_features(struct vtballoon_softc *sc)
293 {
294 	int error;
295 
296 	error = vtballoon_negotiate_features(sc);
297 	if (error)
298 		return (error);
299 
300 	return (0);
301 }
302 
303 static int
304 vtballoon_alloc_virtqueues(struct vtballoon_softc *sc)
305 {
306 	device_t dev;
307 	struct vq_alloc_info vq_info[2];
308 	int nvqs;
309 
310 	dev = sc->vtballoon_dev;
311 	nvqs = 2;
312 
313 	VQ_ALLOC_INFO_INIT(&vq_info[0], 0, vtballoon_vq_intr, sc,
314 	    &sc->vtballoon_inflate_vq, "%s inflate", device_get_nameunit(dev));
315 
316 	VQ_ALLOC_INFO_INIT(&vq_info[1], 0, vtballoon_vq_intr, sc,
317 	    &sc->vtballoon_deflate_vq, "%s deflate", device_get_nameunit(dev));
318 
319 	return (virtio_alloc_virtqueues(dev, 0, nvqs, vq_info));
320 }
321 
322 static void
323 vtballoon_vq_intr(void *xsc)
324 {
325 	struct vtballoon_softc *sc;
326 
327 	sc = xsc;
328 
329 	VTBALLOON_LOCK(sc);
330 	wakeup_one(sc);
331 	VTBALLOON_UNLOCK(sc);
332 }
333 
334 static void
335 vtballoon_inflate(struct vtballoon_softc *sc, int npages)
336 {
337 	struct virtqueue *vq;
338 	vm_page_t m;
339 	int i;
340 
341 	vq = sc->vtballoon_inflate_vq;
342 
343 	if (npages > VTBALLOON_PAGES_PER_REQUEST)
344 		npages = VTBALLOON_PAGES_PER_REQUEST;
345 
346 	for (i = 0; i < npages; i++) {
347 		if ((m = vtballoon_alloc_page(sc)) == NULL) {
348 			sc->vtballoon_timeout = VTBALLOON_LOWMEM_TIMEOUT;
349 			break;
350 		}
351 
352 		sc->vtballoon_page_frames[i] =
353 		    VM_PAGE_TO_PHYS(m) >> VIRTIO_BALLOON_PFN_SHIFT;
354 
355 		KASSERT(m->a.queue == PQ_NONE,
356 		    ("%s: allocated page %p on queue", __func__, m));
357 		TAILQ_INSERT_TAIL(&sc->vtballoon_pages, m, plinks.q);
358 	}
359 
360 	if (i > 0)
361 		vtballoon_send_page_frames(sc, vq, i);
362 }
363 
364 static void
365 vtballoon_deflate(struct vtballoon_softc *sc, int npages)
366 {
367 	TAILQ_HEAD(, vm_page) free_pages;
368 	struct virtqueue *vq;
369 	vm_page_t m;
370 	int i;
371 
372 	vq = sc->vtballoon_deflate_vq;
373 	TAILQ_INIT(&free_pages);
374 
375 	if (npages > VTBALLOON_PAGES_PER_REQUEST)
376 		npages = VTBALLOON_PAGES_PER_REQUEST;
377 
378 	for (i = 0; i < npages; i++) {
379 		m = TAILQ_FIRST(&sc->vtballoon_pages);
380 		KASSERT(m != NULL, ("%s: no more pages to deflate", __func__));
381 
382 		sc->vtballoon_page_frames[i] =
383 		    VM_PAGE_TO_PHYS(m) >> VIRTIO_BALLOON_PFN_SHIFT;
384 
385 		TAILQ_REMOVE(&sc->vtballoon_pages, m, plinks.q);
386 		TAILQ_INSERT_TAIL(&free_pages, m, plinks.q);
387 	}
388 
389 	if (i > 0) {
390 		/* Always tell host first before freeing the pages. */
391 		vtballoon_send_page_frames(sc, vq, i);
392 
393 		while ((m = TAILQ_FIRST(&free_pages)) != NULL) {
394 			TAILQ_REMOVE(&free_pages, m, plinks.q);
395 			vtballoon_free_page(sc, m);
396 		}
397 	}
398 
399 	KASSERT((TAILQ_EMPTY(&sc->vtballoon_pages) &&
400 	    sc->vtballoon_current_npages == 0) ||
401 	    (!TAILQ_EMPTY(&sc->vtballoon_pages) &&
402 	    sc->vtballoon_current_npages != 0),
403 	    ("%s: bogus page count %d", __func__,
404 	    sc->vtballoon_current_npages));
405 }
406 
407 static void
408 vtballoon_send_page_frames(struct vtballoon_softc *sc, struct virtqueue *vq,
409     int npages)
410 {
411 	struct sglist sg;
412 	struct sglist_seg segs[1];
413 	void *c;
414 	int error __diagused;
415 
416 	sglist_init(&sg, 1, segs);
417 
418 	error = sglist_append(&sg, sc->vtballoon_page_frames,
419 	    npages * sizeof(uint32_t));
420 	KASSERT(error == 0, ("error adding page frames to sglist"));
421 
422 	error = virtqueue_enqueue(vq, vq, &sg, 1, 0);
423 	KASSERT(error == 0, ("error enqueuing page frames to virtqueue"));
424 	virtqueue_notify(vq);
425 
426 	/*
427 	 * Inflate and deflate operations are done synchronously. The
428 	 * interrupt handler will wake us up.
429 	 */
430 	VTBALLOON_LOCK(sc);
431 	while ((c = virtqueue_dequeue(vq, NULL)) == NULL)
432 		msleep(sc, VTBALLOON_MTX(sc), 0, "vtbspf", 0);
433 	VTBALLOON_UNLOCK(sc);
434 
435 	KASSERT(c == vq, ("unexpected balloon operation response"));
436 }
437 
438 static void
439 vtballoon_pop(struct vtballoon_softc *sc)
440 {
441 
442 	while (!TAILQ_EMPTY(&sc->vtballoon_pages))
443 		vtballoon_deflate(sc, sc->vtballoon_current_npages);
444 }
445 
446 static void
447 vtballoon_stop(struct vtballoon_softc *sc)
448 {
449 
450 	virtqueue_disable_intr(sc->vtballoon_inflate_vq);
451 	virtqueue_disable_intr(sc->vtballoon_deflate_vq);
452 
453 	virtio_stop(sc->vtballoon_dev);
454 }
455 
456 static vm_page_t
457 vtballoon_alloc_page(struct vtballoon_softc *sc)
458 {
459 	vm_page_t m;
460 
461 	m = vm_page_alloc_noobj(VM_ALLOC_NODUMP);
462 	if (m != NULL)
463 		sc->vtballoon_current_npages++;
464 
465 	return (m);
466 }
467 
468 static void
469 vtballoon_free_page(struct vtballoon_softc *sc, vm_page_t m)
470 {
471 
472 	vm_page_free(m);
473 	sc->vtballoon_current_npages--;
474 }
475 
476 static uint32_t
477 vtballoon_desired_size(struct vtballoon_softc *sc)
478 {
479 	uint32_t desired;
480 
481 	desired = virtio_read_dev_config_4(sc->vtballoon_dev,
482 	    offsetof(struct virtio_balloon_config, num_pages));
483 
484 	if (vtballoon_modern(sc))
485 		return (desired);
486 	else
487 		return (le32toh(desired));
488 }
489 
490 static void
491 vtballoon_update_size(struct vtballoon_softc *sc)
492 {
493 	uint32_t npages;
494 
495 	npages = sc->vtballoon_current_npages;
496 	if (!vtballoon_modern(sc))
497 		npages = htole32(npages);
498 
499 	virtio_write_dev_config_4(sc->vtballoon_dev,
500 	    offsetof(struct virtio_balloon_config, actual), npages);
501 }
502 
503 static int
504 vtballoon_sleep(struct vtballoon_softc *sc)
505 {
506 	int rc, timeout;
507 	uint32_t current, desired;
508 
509 	rc = 0;
510 	current = sc->vtballoon_current_npages;
511 
512 	VTBALLOON_LOCK(sc);
513 	for (;;) {
514 		if (sc->vtballoon_flags & VTBALLOON_FLAG_DETACH) {
515 			rc = 1;
516 			break;
517 		}
518 
519 		desired = vtballoon_desired_size(sc);
520 		sc->vtballoon_desired_npages = desired;
521 
522 		/*
523 		 * If given, use non-zero timeout on the first time through
524 		 * the loop. On subsequent times, timeout will be zero so
525 		 * we will reevaluate the desired size of the balloon and
526 		 * break out to retry if needed.
527 		 */
528 		timeout = sc->vtballoon_timeout;
529 		sc->vtballoon_timeout = 0;
530 
531 		if (current > desired)
532 			break;
533 		if (current < desired && timeout == 0)
534 			break;
535 
536 		msleep(sc, VTBALLOON_MTX(sc), 0, "vtbslp", timeout);
537 	}
538 	VTBALLOON_UNLOCK(sc);
539 
540 	return (rc);
541 }
542 
543 static void
544 vtballoon_thread(void *xsc)
545 {
546 	struct vtballoon_softc *sc;
547 	uint32_t current, desired;
548 
549 	sc = xsc;
550 
551 	for (;;) {
552 		if (vtballoon_sleep(sc) != 0)
553 			break;
554 
555 		current = sc->vtballoon_current_npages;
556 		desired = sc->vtballoon_desired_npages;
557 
558 		if (desired != current) {
559 			if (desired > current)
560 				vtballoon_inflate(sc, desired - current);
561 			else
562 				vtballoon_deflate(sc, current - desired);
563 
564 			vtballoon_update_size(sc);
565 		}
566 	}
567 
568 	kthread_exit();
569 }
570 
571 static void
572 vtballoon_setup_sysctl(struct vtballoon_softc *sc)
573 {
574 	device_t dev;
575 	struct sysctl_ctx_list *ctx;
576 	struct sysctl_oid *tree;
577 	struct sysctl_oid_list *child;
578 
579 	dev = sc->vtballoon_dev;
580 	ctx = device_get_sysctl_ctx(dev);
581 	tree = device_get_sysctl_tree(dev);
582 	child = SYSCTL_CHILDREN(tree);
583 
584 	SYSCTL_ADD_UINT(ctx, child, OID_AUTO, "desired",
585 	    CTLFLAG_RD, &sc->vtballoon_desired_npages, sizeof(uint32_t),
586 	    "Desired balloon size in pages");
587 
588 	SYSCTL_ADD_UINT(ctx, child, OID_AUTO, "current",
589 	    CTLFLAG_RD, &sc->vtballoon_current_npages, sizeof(uint32_t),
590 	    "Current balloon size in pages");
591 }
592