1 /*-
2  * SPDX-License-Identifier: BSD-2-Clause-FreeBSD
3  *
4  * Copyright (c) 2011, Bryan Venteicher <bryanv@FreeBSD.org>
5  * All rights reserved.
6  *
7  * Redistribution and use in source and binary forms, with or without
8  * modification, are permitted provided that the following conditions
9  * are met:
10  * 1. Redistributions of source code must retain the above copyright
11  *    notice unmodified, this list of conditions, and the following
12  *    disclaimer.
13  * 2. Redistributions in binary form must reproduce the above copyright
14  *    notice, this list of conditions and the following disclaimer in the
15  *    documentation and/or other materials provided with the distribution.
16  *
17  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
18  * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
19  * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
20  * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
21  * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
22  * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
23  * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
24  * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
25  * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
26  * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
27  */
28 
29 /* Driver for VirtIO memory balloon devices. */
30 
31 #include <sys/cdefs.h>
32 __FBSDID("$FreeBSD$");
33 
34 #include <sys/param.h>
35 #include <sys/systm.h>
36 #include <sys/kernel.h>
37 #include <sys/endian.h>
38 #include <sys/kthread.h>
39 #include <sys/malloc.h>
40 #include <sys/module.h>
41 #include <sys/sglist.h>
42 #include <sys/sysctl.h>
43 #include <sys/lock.h>
44 #include <sys/mutex.h>
45 #include <sys/queue.h>
46 
47 #include <vm/vm.h>
48 #include <vm/vm_page.h>
49 
50 #include <machine/bus.h>
51 #include <machine/resource.h>
52 #include <sys/bus.h>
53 #include <sys/rman.h>
54 
55 #include <dev/virtio/virtio.h>
56 #include <dev/virtio/virtqueue.h>
57 #include <dev/virtio/balloon/virtio_balloon.h>
58 
59 #include "virtio_if.h"
60 
61 struct vtballoon_softc {
62 	device_t		 vtballoon_dev;
63 	struct mtx		 vtballoon_mtx;
64 	uint64_t		 vtballoon_features;
65 	uint32_t		 vtballoon_flags;
66 #define VTBALLOON_FLAG_DETACH	 0x01
67 
68 	struct virtqueue	*vtballoon_inflate_vq;
69 	struct virtqueue	*vtballoon_deflate_vq;
70 
71 	uint32_t		 vtballoon_desired_npages;
72 	uint32_t		 vtballoon_current_npages;
73 	TAILQ_HEAD(,vm_page)	 vtballoon_pages;
74 
75 	struct thread		*vtballoon_td;
76 	uint32_t		*vtballoon_page_frames;
77 	int			 vtballoon_timeout;
78 };
79 
80 static struct virtio_feature_desc vtballoon_feature_desc[] = {
81 	{ VIRTIO_BALLOON_F_MUST_TELL_HOST,	"MustTellHost"	},
82 	{ VIRTIO_BALLOON_F_STATS_VQ,		"StatsVq"	},
83 	{ 0, NULL }
84 };
85 
86 static int	vtballoon_probe(device_t);
87 static int	vtballoon_attach(device_t);
88 static int	vtballoon_detach(device_t);
89 static int	vtballoon_config_change(device_t);
90 
91 static void	vtballoon_negotiate_features(struct vtballoon_softc *);
92 static int	vtballoon_alloc_virtqueues(struct vtballoon_softc *);
93 
94 static void	vtballoon_vq_intr(void *);
95 
96 static void	vtballoon_inflate(struct vtballoon_softc *, int);
97 static void	vtballoon_deflate(struct vtballoon_softc *, int);
98 
99 static void	vtballoon_send_page_frames(struct vtballoon_softc *,
100 		    struct virtqueue *, int);
101 
102 static void	vtballoon_pop(struct vtballoon_softc *);
103 static void	vtballoon_stop(struct vtballoon_softc *);
104 
105 static vm_page_t
106 		vtballoon_alloc_page(struct vtballoon_softc *);
107 static void	vtballoon_free_page(struct vtballoon_softc *, vm_page_t);
108 
109 static int	vtballoon_sleep(struct vtballoon_softc *);
110 static void	vtballoon_thread(void *);
111 static void	vtballoon_add_sysctl(struct vtballoon_softc *);
112 
113 /* Features desired/implemented by this driver. */
114 #define VTBALLOON_FEATURES		0
115 
116 /* Timeout between retries when the balloon needs inflating. */
117 #define VTBALLOON_LOWMEM_TIMEOUT	hz
118 
119 /*
120  * Maximum number of pages we'll request to inflate or deflate
121  * the balloon in one virtqueue request. Both Linux and NetBSD
122  * have settled on 256, doing up to 1MB at a time.
123  */
124 #define VTBALLOON_PAGES_PER_REQUEST	256
125 
126 /* Must be able to fix all pages frames in one page (segment). */
127 CTASSERT(VTBALLOON_PAGES_PER_REQUEST * sizeof(uint32_t) <= PAGE_SIZE);
128 
129 #define VTBALLOON_MTX(_sc)		&(_sc)->vtballoon_mtx
130 #define VTBALLOON_LOCK_INIT(_sc, _name)	mtx_init(VTBALLOON_MTX((_sc)), _name, \
131 					    "VirtIO Balloon Lock", MTX_DEF)
132 #define VTBALLOON_LOCK(_sc)		mtx_lock(VTBALLOON_MTX((_sc)))
133 #define VTBALLOON_UNLOCK(_sc)		mtx_unlock(VTBALLOON_MTX((_sc)))
134 #define VTBALLOON_LOCK_DESTROY(_sc)	mtx_destroy(VTBALLOON_MTX((_sc)))
135 
136 static device_method_t vtballoon_methods[] = {
137 	/* Device methods. */
138 	DEVMETHOD(device_probe,		vtballoon_probe),
139 	DEVMETHOD(device_attach,	vtballoon_attach),
140 	DEVMETHOD(device_detach,	vtballoon_detach),
141 
142 	/* VirtIO methods. */
143 	DEVMETHOD(virtio_config_change, vtballoon_config_change),
144 
145 	DEVMETHOD_END
146 };
147 
148 static driver_t vtballoon_driver = {
149 	"vtballoon",
150 	vtballoon_methods,
151 	sizeof(struct vtballoon_softc)
152 };
153 static devclass_t vtballoon_devclass;
154 
155 DRIVER_MODULE(virtio_balloon, virtio_mmio, vtballoon_driver,
156     vtballoon_devclass, 0, 0);
157 DRIVER_MODULE(virtio_balloon, virtio_pci, vtballoon_driver,
158     vtballoon_devclass, 0, 0);
159 MODULE_VERSION(virtio_balloon, 1);
160 MODULE_DEPEND(virtio_balloon, virtio, 1, 1, 1);
161 
162 VIRTIO_SIMPLE_PNPTABLE(virtio_balloon, VIRTIO_ID_BALLOON,
163     "VirtIO Balloon Adapter");
164 VIRTIO_SIMPLE_PNPINFO(virtio_mmio, virtio_balloon);
165 VIRTIO_SIMPLE_PNPINFO(virtio_pci, virtio_balloon);
166 
167 static int
168 vtballoon_probe(device_t dev)
169 {
170 	return (VIRTIO_SIMPLE_PROBE(dev, virtio_balloon));
171 }
172 
173 static int
174 vtballoon_attach(device_t dev)
175 {
176 	struct vtballoon_softc *sc;
177 	int error;
178 
179 	sc = device_get_softc(dev);
180 	sc->vtballoon_dev = dev;
181 
182 	VTBALLOON_LOCK_INIT(sc, device_get_nameunit(dev));
183 	TAILQ_INIT(&sc->vtballoon_pages);
184 
185 	vtballoon_add_sysctl(sc);
186 
187 	virtio_set_feature_desc(dev, vtballoon_feature_desc);
188 	vtballoon_negotiate_features(sc);
189 
190 	sc->vtballoon_page_frames = malloc(VTBALLOON_PAGES_PER_REQUEST *
191 	    sizeof(uint32_t), M_DEVBUF, M_NOWAIT | M_ZERO);
192 	if (sc->vtballoon_page_frames == NULL) {
193 		error = ENOMEM;
194 		device_printf(dev,
195 		    "cannot allocate page frame request array\n");
196 		goto fail;
197 	}
198 
199 	error = vtballoon_alloc_virtqueues(sc);
200 	if (error) {
201 		device_printf(dev, "cannot allocate virtqueues\n");
202 		goto fail;
203 	}
204 
205 	error = virtio_setup_intr(dev, INTR_TYPE_MISC);
206 	if (error) {
207 		device_printf(dev, "cannot setup virtqueue interrupts\n");
208 		goto fail;
209 	}
210 
211 	error = kthread_add(vtballoon_thread, sc, NULL, &sc->vtballoon_td,
212 	    0, 0, "virtio_balloon");
213 	if (error) {
214 		device_printf(dev, "cannot create balloon kthread\n");
215 		goto fail;
216 	}
217 
218 	virtqueue_enable_intr(sc->vtballoon_inflate_vq);
219 	virtqueue_enable_intr(sc->vtballoon_deflate_vq);
220 
221 fail:
222 	if (error)
223 		vtballoon_detach(dev);
224 
225 	return (error);
226 }
227 
228 static int
229 vtballoon_detach(device_t dev)
230 {
231 	struct vtballoon_softc *sc;
232 
233 	sc = device_get_softc(dev);
234 
235 	if (sc->vtballoon_td != NULL) {
236 		VTBALLOON_LOCK(sc);
237 		sc->vtballoon_flags |= VTBALLOON_FLAG_DETACH;
238 		wakeup_one(sc);
239 		msleep(sc->vtballoon_td, VTBALLOON_MTX(sc), 0, "vtbdth", 0);
240 		VTBALLOON_UNLOCK(sc);
241 
242 		sc->vtballoon_td = NULL;
243 	}
244 
245 	if (device_is_attached(dev)) {
246 		vtballoon_pop(sc);
247 		vtballoon_stop(sc);
248 	}
249 
250 	if (sc->vtballoon_page_frames != NULL) {
251 		free(sc->vtballoon_page_frames, M_DEVBUF);
252 		sc->vtballoon_page_frames = NULL;
253 	}
254 
255 	VTBALLOON_LOCK_DESTROY(sc);
256 
257 	return (0);
258 }
259 
260 static int
261 vtballoon_config_change(device_t dev)
262 {
263 	struct vtballoon_softc *sc;
264 
265 	sc = device_get_softc(dev);
266 
267 	VTBALLOON_LOCK(sc);
268 	wakeup_one(sc);
269 	VTBALLOON_UNLOCK(sc);
270 
271 	return (1);
272 }
273 
274 static void
275 vtballoon_negotiate_features(struct vtballoon_softc *sc)
276 {
277 	device_t dev;
278 	uint64_t features;
279 
280 	dev = sc->vtballoon_dev;
281 	features = virtio_negotiate_features(dev, VTBALLOON_FEATURES);
282 	sc->vtballoon_features = features;
283 }
284 
285 static int
286 vtballoon_alloc_virtqueues(struct vtballoon_softc *sc)
287 {
288 	device_t dev;
289 	struct vq_alloc_info vq_info[2];
290 	int nvqs;
291 
292 	dev = sc->vtballoon_dev;
293 	nvqs = 2;
294 
295 	VQ_ALLOC_INFO_INIT(&vq_info[0], 0, vtballoon_vq_intr, sc,
296 	    &sc->vtballoon_inflate_vq, "%s inflate", device_get_nameunit(dev));
297 
298 	VQ_ALLOC_INFO_INIT(&vq_info[1], 0, vtballoon_vq_intr, sc,
299 	    &sc->vtballoon_deflate_vq, "%s deflate", device_get_nameunit(dev));
300 
301 	return (virtio_alloc_virtqueues(dev, 0, nvqs, vq_info));
302 }
303 
304 static void
305 vtballoon_vq_intr(void *xsc)
306 {
307 	struct vtballoon_softc *sc;
308 
309 	sc = xsc;
310 
311 	VTBALLOON_LOCK(sc);
312 	wakeup_one(sc);
313 	VTBALLOON_UNLOCK(sc);
314 }
315 
316 static void
317 vtballoon_inflate(struct vtballoon_softc *sc, int npages)
318 {
319 	struct virtqueue *vq;
320 	vm_page_t m;
321 	int i;
322 
323 	vq = sc->vtballoon_inflate_vq;
324 
325 	if (npages > VTBALLOON_PAGES_PER_REQUEST)
326 		npages = VTBALLOON_PAGES_PER_REQUEST;
327 
328 	for (i = 0; i < npages; i++) {
329 		if ((m = vtballoon_alloc_page(sc)) == NULL) {
330 			sc->vtballoon_timeout = VTBALLOON_LOWMEM_TIMEOUT;
331 			break;
332 		}
333 
334 		sc->vtballoon_page_frames[i] =
335 		    VM_PAGE_TO_PHYS(m) >> VIRTIO_BALLOON_PFN_SHIFT;
336 
337 		KASSERT(m->a.queue == PQ_NONE,
338 		    ("%s: allocated page %p on queue", __func__, m));
339 		TAILQ_INSERT_TAIL(&sc->vtballoon_pages, m, plinks.q);
340 	}
341 
342 	if (i > 0)
343 		vtballoon_send_page_frames(sc, vq, i);
344 }
345 
346 static void
347 vtballoon_deflate(struct vtballoon_softc *sc, int npages)
348 {
349 	TAILQ_HEAD(, vm_page) free_pages;
350 	struct virtqueue *vq;
351 	vm_page_t m;
352 	int i;
353 
354 	vq = sc->vtballoon_deflate_vq;
355 	TAILQ_INIT(&free_pages);
356 
357 	if (npages > VTBALLOON_PAGES_PER_REQUEST)
358 		npages = VTBALLOON_PAGES_PER_REQUEST;
359 
360 	for (i = 0; i < npages; i++) {
361 		m = TAILQ_FIRST(&sc->vtballoon_pages);
362 		KASSERT(m != NULL, ("%s: no more pages to deflate", __func__));
363 
364 		sc->vtballoon_page_frames[i] =
365 		    VM_PAGE_TO_PHYS(m) >> VIRTIO_BALLOON_PFN_SHIFT;
366 
367 		TAILQ_REMOVE(&sc->vtballoon_pages, m, plinks.q);
368 		TAILQ_INSERT_TAIL(&free_pages, m, plinks.q);
369 	}
370 
371 	if (i > 0) {
372 		/* Always tell host first before freeing the pages. */
373 		vtballoon_send_page_frames(sc, vq, i);
374 
375 		while ((m = TAILQ_FIRST(&free_pages)) != NULL) {
376 			TAILQ_REMOVE(&free_pages, m, plinks.q);
377 			vtballoon_free_page(sc, m);
378 		}
379 	}
380 
381 	KASSERT((TAILQ_EMPTY(&sc->vtballoon_pages) &&
382 	    sc->vtballoon_current_npages == 0) ||
383 	    (!TAILQ_EMPTY(&sc->vtballoon_pages) &&
384 	    sc->vtballoon_current_npages != 0),
385 	    ("%s: bogus page count %d", __func__,
386 	    sc->vtballoon_current_npages));
387 }
388 
389 static void
390 vtballoon_send_page_frames(struct vtballoon_softc *sc, struct virtqueue *vq,
391     int npages)
392 {
393 	struct sglist sg;
394 	struct sglist_seg segs[1];
395 	void *c;
396 	int error;
397 
398 	sglist_init(&sg, 1, segs);
399 
400 	error = sglist_append(&sg, sc->vtballoon_page_frames,
401 	    npages * sizeof(uint32_t));
402 	KASSERT(error == 0, ("error adding page frames to sglist"));
403 
404 	error = virtqueue_enqueue(vq, vq, &sg, 1, 0);
405 	KASSERT(error == 0, ("error enqueuing page frames to virtqueue"));
406 	virtqueue_notify(vq);
407 
408 	/*
409 	 * Inflate and deflate operations are done synchronously. The
410 	 * interrupt handler will wake us up.
411 	 */
412 	VTBALLOON_LOCK(sc);
413 	while ((c = virtqueue_dequeue(vq, NULL)) == NULL)
414 		msleep(sc, VTBALLOON_MTX(sc), 0, "vtbspf", 0);
415 	VTBALLOON_UNLOCK(sc);
416 
417 	KASSERT(c == vq, ("unexpected balloon operation response"));
418 }
419 
420 static void
421 vtballoon_pop(struct vtballoon_softc *sc)
422 {
423 
424 	while (!TAILQ_EMPTY(&sc->vtballoon_pages))
425 		vtballoon_deflate(sc, sc->vtballoon_current_npages);
426 }
427 
428 static void
429 vtballoon_stop(struct vtballoon_softc *sc)
430 {
431 
432 	virtqueue_disable_intr(sc->vtballoon_inflate_vq);
433 	virtqueue_disable_intr(sc->vtballoon_deflate_vq);
434 
435 	virtio_stop(sc->vtballoon_dev);
436 }
437 
438 static vm_page_t
439 vtballoon_alloc_page(struct vtballoon_softc *sc)
440 {
441 	vm_page_t m;
442 
443 	m = vm_page_alloc(NULL, 0, VM_ALLOC_NORMAL | VM_ALLOC_NOOBJ);
444 	if (m != NULL)
445 		sc->vtballoon_current_npages++;
446 
447 	return (m);
448 }
449 
450 static void
451 vtballoon_free_page(struct vtballoon_softc *sc, vm_page_t m)
452 {
453 
454 	vm_page_free(m);
455 	sc->vtballoon_current_npages--;
456 }
457 
458 static uint32_t
459 vtballoon_desired_size(struct vtballoon_softc *sc)
460 {
461 	uint32_t desired;
462 
463 	desired = virtio_read_dev_config_4(sc->vtballoon_dev,
464 	    offsetof(struct virtio_balloon_config, num_pages));
465 
466 	return (le32toh(desired));
467 }
468 
469 static void
470 vtballoon_update_size(struct vtballoon_softc *sc)
471 {
472 
473 	virtio_write_dev_config_4(sc->vtballoon_dev,
474 	    offsetof(struct virtio_balloon_config, actual),
475 	    htole32(sc->vtballoon_current_npages));
476 }
477 
478 static int
479 vtballoon_sleep(struct vtballoon_softc *sc)
480 {
481 	int rc, timeout;
482 	uint32_t current, desired;
483 
484 	rc = 0;
485 	current = sc->vtballoon_current_npages;
486 
487 	VTBALLOON_LOCK(sc);
488 	for (;;) {
489 		if (sc->vtballoon_flags & VTBALLOON_FLAG_DETACH) {
490 			rc = 1;
491 			break;
492 		}
493 
494 		desired = vtballoon_desired_size(sc);
495 		sc->vtballoon_desired_npages = desired;
496 
497 		/*
498 		 * If given, use non-zero timeout on the first time through
499 		 * the loop. On subsequent times, timeout will be zero so
500 		 * we will reevaluate the desired size of the balloon and
501 		 * break out to retry if needed.
502 		 */
503 		timeout = sc->vtballoon_timeout;
504 		sc->vtballoon_timeout = 0;
505 
506 		if (current > desired)
507 			break;
508 		if (current < desired && timeout == 0)
509 			break;
510 
511 		msleep(sc, VTBALLOON_MTX(sc), 0, "vtbslp", timeout);
512 	}
513 	VTBALLOON_UNLOCK(sc);
514 
515 	return (rc);
516 }
517 
518 static void
519 vtballoon_thread(void *xsc)
520 {
521 	struct vtballoon_softc *sc;
522 	uint32_t current, desired;
523 
524 	sc = xsc;
525 
526 	for (;;) {
527 		if (vtballoon_sleep(sc) != 0)
528 			break;
529 
530 		current = sc->vtballoon_current_npages;
531 		desired = sc->vtballoon_desired_npages;
532 
533 		if (desired != current) {
534 			if (desired > current)
535 				vtballoon_inflate(sc, desired - current);
536 			else
537 				vtballoon_deflate(sc, current - desired);
538 
539 			vtballoon_update_size(sc);
540 		}
541 	}
542 
543 	kthread_exit();
544 }
545 
546 static void
547 vtballoon_add_sysctl(struct vtballoon_softc *sc)
548 {
549 	device_t dev;
550 	struct sysctl_ctx_list *ctx;
551 	struct sysctl_oid *tree;
552 	struct sysctl_oid_list *child;
553 
554 	dev = sc->vtballoon_dev;
555 	ctx = device_get_sysctl_ctx(dev);
556 	tree = device_get_sysctl_tree(dev);
557 	child = SYSCTL_CHILDREN(tree);
558 
559 	SYSCTL_ADD_UINT(ctx, child, OID_AUTO, "desired",
560 	    CTLFLAG_RD, &sc->vtballoon_desired_npages, sizeof(uint32_t),
561 	    "Desired balloon size in pages");
562 
563 	SYSCTL_ADD_UINT(ctx, child, OID_AUTO, "current",
564 	    CTLFLAG_RD, &sc->vtballoon_current_npages, sizeof(uint32_t),
565 	    "Current balloon size in pages");
566 }
567