1 /*-
2  * Copyright (c) 2011, Bryan Venteicher <bryanv@daemoninthecloset.org>
3  * All rights reserved.
4  *
5  * Redistribution and use in source and binary forms, with or without
6  * modification, are permitted provided that the following conditions
7  * are met:
8  * 1. Redistributions of source code must retain the above copyright
9  *    notice unmodified, this list of conditions, and the following
10  *    disclaimer.
11  * 2. Redistributions in binary form must reproduce the above copyright
12  *    notice, this list of conditions and the following disclaimer in the
13  *    documentation and/or other materials provided with the distribution.
14  *
15  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
16  * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
17  * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
18  * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
19  * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
20  * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
21  * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
22  * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
23  * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
24  * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
25  */
26 
27 /* Driver for VirtIO memory balloon devices. */
28 
29 #include <sys/cdefs.h>
30 __FBSDID("$FreeBSD$");
31 
32 #include <sys/param.h>
33 #include <sys/systm.h>
34 #include <sys/kernel.h>
35 #include <sys/endian.h>
36 #include <sys/kthread.h>
37 #include <sys/malloc.h>
38 #include <sys/module.h>
39 #include <sys/sglist.h>
40 #include <sys/sysctl.h>
41 #include <sys/lock.h>
42 #include <sys/mutex.h>
43 #include <sys/queue.h>
44 
45 #include <vm/vm.h>
46 #include <vm/vm_page.h>
47 
48 #include <machine/bus.h>
49 #include <machine/resource.h>
50 #include <sys/bus.h>
51 #include <sys/rman.h>
52 
53 #include <dev/virtio/virtio.h>
54 #include <dev/virtio/virtqueue.h>
55 #include <dev/virtio/balloon/virtio_balloon.h>
56 
57 #include "virtio_if.h"
58 
59 struct vtballoon_softc {
60 	device_t		 vtballoon_dev;
61 	struct mtx		 vtballoon_mtx;
62 	uint64_t		 vtballoon_features;
63 	uint32_t		 vtballoon_flags;
64 #define VTBALLOON_FLAG_DETACH	 0x01
65 
66 	struct virtqueue	*vtballoon_inflate_vq;
67 	struct virtqueue	*vtballoon_deflate_vq;
68 
69 	uint32_t		 vtballoon_desired_npages;
70 	uint32_t		 vtballoon_current_npages;
71 	TAILQ_HEAD(,vm_page)	 vtballoon_pages;
72 
73 	struct proc		*vtballoon_kproc;
74 	uint32_t		*vtballoon_page_frames;
75 	int			 vtballoon_timeout;
76 };
77 
78 static struct virtio_feature_desc vtballoon_feature_desc[] = {
79 	{ VIRTIO_BALLOON_F_MUST_TELL_HOST,	"MustTellHost"	},
80 	{ VIRTIO_BALLOON_F_STATS_VQ,		"StatsVq"	},
81 
82 	{ 0, NULL }
83 };
84 
85 static int	vtballoon_probe(device_t);
86 static int	vtballoon_attach(device_t);
87 static int	vtballoon_detach(device_t);
88 static int	vtballoon_config_change(device_t);
89 
90 static void	vtballoon_negotiate_features(struct vtballoon_softc *);
91 static int	vtballoon_alloc_virtqueues(struct vtballoon_softc *);
92 
93 static int	vtballoon_vq_intr(void *);
94 
95 static void	vtballoon_inflate(struct vtballoon_softc *, int);
96 static void	vtballoon_deflate(struct vtballoon_softc *, int);
97 
98 static void	vtballoon_send_page_frames(struct vtballoon_softc *,
99 		    struct virtqueue *, int);
100 
101 static void	vtballoon_pop(struct vtballoon_softc *);
102 static void	vtballoon_stop(struct vtballoon_softc *);
103 
104 static vm_page_t
105 		vtballoon_alloc_page(struct vtballoon_softc *);
106 static void	vtballoon_free_page(struct vtballoon_softc *, vm_page_t);
107 
108 static int	vtballoon_sleep(struct vtballoon_softc *);
109 static void	vtballoon_thread(void *);
110 static void	vtballoon_add_sysctl(struct vtballoon_softc *);
111 
112 /* Features desired/implemented by this driver. */
113 #define VTBALLOON_FEATURES		0
114 
115 /* Timeout between retries when the balloon needs inflating. */
116 #define VTBALLOON_LOWMEM_TIMEOUT	hz
117 
118 /*
119  * Maximum number of pages we'll request to inflate or deflate
120  * the balloon in one virtqueue request. Both Linux and NetBSD
121  * have settled on 256, doing up to 1MB at a time.
122  */
123 #define VTBALLOON_PAGES_PER_REQUEST	256
124 
125 /* Must be able to fix all pages frames in one page (segment). */
126 CTASSERT(VTBALLOON_PAGES_PER_REQUEST * sizeof(uint32_t) <= PAGE_SIZE);
127 
128 #define VTBALLOON_MTX(_sc)		&(_sc)->vtballoon_mtx
129 #define VTBALLOON_LOCK_INIT(_sc, _name)	mtx_init(VTBALLOON_MTX((_sc)), _name, \
130 					    "VirtIO Balloon Lock", MTX_SPIN)
131 #define VTBALLOON_LOCK(_sc)		mtx_lock_spin(VTBALLOON_MTX((_sc)))
132 #define VTBALLOON_UNLOCK(_sc)		mtx_unlock_spin(VTBALLOON_MTX((_sc)))
133 #define VTBALLOON_LOCK_DESTROY(_sc)	mtx_destroy(VTBALLOON_MTX((_sc)))
134 
135 static device_method_t vtballoon_methods[] = {
136 	/* Device methods. */
137 	DEVMETHOD(device_probe,		vtballoon_probe),
138 	DEVMETHOD(device_attach,	vtballoon_attach),
139 	DEVMETHOD(device_detach,	vtballoon_detach),
140 
141 	/* VirtIO methods. */
142 	DEVMETHOD(virtio_config_change, vtballoon_config_change),
143 
144 	DEVMETHOD_END
145 };
146 
147 static driver_t vtballoon_driver = {
148 	"vtballoon",
149 	vtballoon_methods,
150 	sizeof(struct vtballoon_softc)
151 };
152 static devclass_t vtballoon_devclass;
153 
154 DRIVER_MODULE(virtio_balloon, virtio_pci, vtballoon_driver,
155     vtballoon_devclass, 0, 0);
156 MODULE_VERSION(virtio_balloon, 1);
157 MODULE_DEPEND(virtio_balloon, virtio, 1, 1, 1);
158 
159 static int
160 vtballoon_probe(device_t dev)
161 {
162 
163 	if (virtio_get_device_type(dev) != VIRTIO_ID_BALLOON)
164 		return (ENXIO);
165 
166 	device_set_desc(dev, "VirtIO Balloon Adapter");
167 
168 	return (BUS_PROBE_DEFAULT);
169 }
170 
171 static int
172 vtballoon_attach(device_t dev)
173 {
174 	struct vtballoon_softc *sc;
175 	int error;
176 
177 	sc = device_get_softc(dev);
178 	sc->vtballoon_dev = dev;
179 
180 	VTBALLOON_LOCK_INIT(sc, device_get_nameunit(dev));
181 	TAILQ_INIT(&sc->vtballoon_pages);
182 
183 	vtballoon_add_sysctl(sc);
184 
185 	virtio_set_feature_desc(dev, vtballoon_feature_desc);
186 	vtballoon_negotiate_features(sc);
187 
188 	sc->vtballoon_page_frames = malloc(VTBALLOON_PAGES_PER_REQUEST *
189 	    sizeof(uint32_t), M_DEVBUF, M_NOWAIT | M_ZERO);
190 	if (sc->vtballoon_page_frames == NULL) {
191 		error = ENOMEM;
192 		device_printf(dev,
193 		    "cannot allocate page frame request array\n");
194 		goto fail;
195 	}
196 
197 	error = vtballoon_alloc_virtqueues(sc);
198 	if (error) {
199 		device_printf(dev, "cannot allocate virtqueues\n");
200 		goto fail;
201 	}
202 
203 	error = virtio_setup_intr(dev, INTR_TYPE_MISC);
204 	if (error) {
205 		device_printf(dev, "cannot setup virtqueue interrupts\n");
206 		goto fail;
207 	}
208 
209 	error = kproc_create(vtballoon_thread, sc, &sc->vtballoon_kproc,
210 	    0, 0, "virtio_balloon");
211 	if (error) {
212 		device_printf(dev, "cannot create balloon kproc\n");
213 		goto fail;
214 	}
215 
216 	virtqueue_enable_intr(sc->vtballoon_inflate_vq);
217 	virtqueue_enable_intr(sc->vtballoon_deflate_vq);
218 
219 fail:
220 	if (error)
221 		vtballoon_detach(dev);
222 
223 	return (error);
224 }
225 
226 static int
227 vtballoon_detach(device_t dev)
228 {
229 	struct vtballoon_softc *sc;
230 
231 	sc = device_get_softc(dev);
232 
233 	if (sc->vtballoon_kproc != NULL) {
234 		VTBALLOON_LOCK(sc);
235 		sc->vtballoon_flags |= VTBALLOON_FLAG_DETACH;
236 		wakeup_one(sc);
237 		msleep_spin(sc->vtballoon_kproc, VTBALLOON_MTX(sc),
238 		    "vtbdth", 0);
239 		VTBALLOON_UNLOCK(sc);
240 
241 		sc->vtballoon_kproc = NULL;
242 	}
243 
244 	if (device_is_attached(dev)) {
245 		vtballoon_pop(sc);
246 		vtballoon_stop(sc);
247 	}
248 
249 	if (sc->vtballoon_page_frames != NULL) {
250 		free(sc->vtballoon_page_frames, M_DEVBUF);
251 		sc->vtballoon_page_frames = NULL;
252 	}
253 
254 	VTBALLOON_LOCK_DESTROY(sc);
255 
256 	return (0);
257 }
258 
259 static int
260 vtballoon_config_change(device_t dev)
261 {
262 	struct vtballoon_softc *sc;
263 
264 	sc = device_get_softc(dev);
265 
266 	VTBALLOON_LOCK(sc);
267 	wakeup_one(sc);
268 	VTBALLOON_UNLOCK(sc);
269 
270 	return (1);
271 }
272 
273 static void
274 vtballoon_negotiate_features(struct vtballoon_softc *sc)
275 {
276 	device_t dev;
277 	uint64_t features;
278 
279 	dev = sc->vtballoon_dev;
280 	features = virtio_negotiate_features(dev, VTBALLOON_FEATURES);
281 	sc->vtballoon_features = features;
282 }
283 
284 static int
285 vtballoon_alloc_virtqueues(struct vtballoon_softc *sc)
286 {
287 	device_t dev;
288 	struct vq_alloc_info vq_info[2];
289 	int nvqs;
290 
291 	dev = sc->vtballoon_dev;
292 	nvqs = 2;
293 
294 	VQ_ALLOC_INFO_INIT(&vq_info[0], 0, vtballoon_vq_intr, sc,
295 	    &sc->vtballoon_inflate_vq, "%s inflate", device_get_nameunit(dev));
296 
297 	VQ_ALLOC_INFO_INIT(&vq_info[1], 0, vtballoon_vq_intr, sc,
298 	    &sc->vtballoon_deflate_vq, "%s deflate", device_get_nameunit(dev));
299 
300 	return (virtio_alloc_virtqueues(dev, 0, nvqs, vq_info));
301 }
302 
303 static int
304 vtballoon_vq_intr(void *xsc)
305 {
306 	struct vtballoon_softc *sc;
307 
308 	sc = xsc;
309 
310 	VTBALLOON_LOCK(sc);
311 	wakeup_one(sc);
312 	VTBALLOON_UNLOCK(sc);
313 
314 	return (1);
315 }
316 
317 static void
318 vtballoon_inflate(struct vtballoon_softc *sc, int npages)
319 {
320 	struct virtqueue *vq;
321 	vm_page_t m;
322 	int i;
323 
324 	vq = sc->vtballoon_inflate_vq;
325 	m = NULL;
326 
327 	if (npages > VTBALLOON_PAGES_PER_REQUEST)
328 		npages = VTBALLOON_PAGES_PER_REQUEST;
329 	KASSERT(npages > 0, ("balloon doesn't need inflating?"));
330 
331 	for (i = 0; i < npages; i++) {
332 		if ((m = vtballoon_alloc_page(sc)) == NULL)
333 			break;
334 
335 		sc->vtballoon_page_frames[i] =
336 		    VM_PAGE_TO_PHYS(m) >> VIRTIO_BALLOON_PFN_SHIFT;
337 
338 		KASSERT(m->queue == PQ_NONE, ("allocated page on queue"));
339 		TAILQ_INSERT_TAIL(&sc->vtballoon_pages, m, pageq);
340 	}
341 
342 	if (i > 0)
343 		vtballoon_send_page_frames(sc, vq, i);
344 
345 	if (m == NULL)
346 		sc->vtballoon_timeout = VTBALLOON_LOWMEM_TIMEOUT;
347 }
348 
349 static void
350 vtballoon_deflate(struct vtballoon_softc *sc, int npages)
351 {
352 	TAILQ_HEAD(, vm_page) free_pages;
353 	struct virtqueue *vq;
354 	vm_page_t m;
355 	int i;
356 
357 	vq = sc->vtballoon_deflate_vq;
358 	TAILQ_INIT(&free_pages);
359 
360 	if (npages > VTBALLOON_PAGES_PER_REQUEST)
361 		npages = VTBALLOON_PAGES_PER_REQUEST;
362 	KASSERT(npages > 0, ("balloon doesn't need deflating?"));
363 
364 	for (i = 0; i < npages; i++) {
365 		m = TAILQ_FIRST(&sc->vtballoon_pages);
366 		KASSERT(m != NULL, ("no more pages to deflate"));
367 
368 		sc->vtballoon_page_frames[i] =
369 		    VM_PAGE_TO_PHYS(m) >> VIRTIO_BALLOON_PFN_SHIFT;
370 
371 		TAILQ_REMOVE(&sc->vtballoon_pages, m, pageq);
372 		TAILQ_INSERT_TAIL(&free_pages, m, pageq);
373 	}
374 
375 	if (i > 0) {
376 		/* Always tell host first before freeing the pages. */
377 		vtballoon_send_page_frames(sc, vq, i);
378 
379 		while ((m = TAILQ_FIRST(&free_pages)) != NULL) {
380 			TAILQ_REMOVE(&free_pages, m, pageq);
381 			vtballoon_free_page(sc, m);
382 		}
383 	}
384 
385 	KASSERT((TAILQ_EMPTY(&sc->vtballoon_pages) &&
386 	    sc->vtballoon_current_npages == 0) ||
387 	    (!TAILQ_EMPTY(&sc->vtballoon_pages) &&
388 	    sc->vtballoon_current_npages != 0), ("balloon empty?"));
389 }
390 
391 static void
392 vtballoon_send_page_frames(struct vtballoon_softc *sc, struct virtqueue *vq,
393     int npages)
394 {
395 	struct sglist sg;
396 	struct sglist_seg segs[1];
397 	void *c;
398 	int error;
399 
400 	sglist_init(&sg, 1, segs);
401 
402 	error = sglist_append(&sg, sc->vtballoon_page_frames,
403 	    npages * sizeof(uint32_t));
404 	KASSERT(error == 0, ("error adding page frames to sglist"));
405 
406 	error = virtqueue_enqueue(vq, vq, &sg, 1, 0);
407 	KASSERT(error == 0, ("error enqueuing page frames to virtqueue"));
408 	virtqueue_notify(vq);
409 
410 	/*
411 	 * Inflate and deflate operations are done synchronously. The
412 	 * interrupt handler will wake us up.
413 	 */
414 	VTBALLOON_LOCK(sc);
415 	while ((c = virtqueue_dequeue(vq, NULL)) == NULL)
416 		msleep_spin(sc, VTBALLOON_MTX(sc), "vtbspf", 0);
417 	VTBALLOON_UNLOCK(sc);
418 
419 	KASSERT(c == vq, ("unexpected balloon operation response"));
420 }
421 
422 static void
423 vtballoon_pop(struct vtballoon_softc *sc)
424 {
425 
426 	while (!TAILQ_EMPTY(&sc->vtballoon_pages))
427 		vtballoon_deflate(sc, sc->vtballoon_current_npages);
428 }
429 
430 static void
431 vtballoon_stop(struct vtballoon_softc *sc)
432 {
433 
434 	virtqueue_disable_intr(sc->vtballoon_inflate_vq);
435 	virtqueue_disable_intr(sc->vtballoon_deflate_vq);
436 
437 	virtio_stop(sc->vtballoon_dev);
438 }
439 
440 static vm_page_t
441 vtballoon_alloc_page(struct vtballoon_softc *sc)
442 {
443 	vm_page_t m;
444 
445 	m = vm_page_alloc(NULL, 0, VM_ALLOC_NORMAL | VM_ALLOC_WIRED |
446 	    VM_ALLOC_NOOBJ);
447 	if (m != NULL)
448 		sc->vtballoon_current_npages++;
449 
450 	return (m);
451 }
452 
453 static void
454 vtballoon_free_page(struct vtballoon_softc *sc, vm_page_t m)
455 {
456 
457 	vm_page_unwire(m, 0);
458 	vm_page_free(m);
459 	sc->vtballoon_current_npages--;
460 }
461 
462 static uint32_t
463 vtballoon_desired_size(struct vtballoon_softc *sc)
464 {
465 	uint32_t desired;
466 
467 	desired = virtio_read_dev_config_4(sc->vtballoon_dev,
468 	    offsetof(struct virtio_balloon_config, num_pages));
469 
470 	return (le32toh(desired));
471 }
472 
473 static void
474 vtballoon_update_size(struct vtballoon_softc *sc)
475 {
476 
477 	virtio_write_dev_config_4(sc->vtballoon_dev,
478 	    offsetof(struct virtio_balloon_config, actual),
479 	    htole32(sc->vtballoon_current_npages));
480 }
481 
482 static int
483 vtballoon_sleep(struct vtballoon_softc *sc)
484 {
485 	int rc, timeout;
486 	uint32_t current, desired;
487 
488 	rc = 0;
489 	current = sc->vtballoon_current_npages;
490 
491 	VTBALLOON_LOCK(sc);
492 	for (;;) {
493 		if (sc->vtballoon_flags & VTBALLOON_FLAG_DETACH) {
494 			rc = 1;
495 			break;
496 		}
497 
498 		desired = vtballoon_desired_size(sc);
499 		sc->vtballoon_desired_npages = desired;
500 
501 		/*
502 		 * If given, use non-zero timeout on the first time through
503 		 * the loop. On subsequent times, timeout will be zero so
504 		 * we will reevaluate the desired size of the balloon and
505 		 * break out to retry if needed.
506 		 */
507 		timeout = sc->vtballoon_timeout;
508 		sc->vtballoon_timeout = 0;
509 
510 		if (current > desired)
511 			break;
512 		if (current < desired && timeout == 0)
513 			break;
514 
515 		msleep_spin(sc, VTBALLOON_MTX(sc), "vtbslp", timeout);
516 	}
517 	VTBALLOON_UNLOCK(sc);
518 
519 	return (rc);
520 }
521 
522 static void
523 vtballoon_thread(void *xsc)
524 {
525 	struct vtballoon_softc *sc;
526 	uint32_t current, desired;
527 
528 	sc = xsc;
529 
530 	for (;;) {
531 		if (vtballoon_sleep(sc) != 0)
532 			break;
533 
534 		current = sc->vtballoon_current_npages;
535 		desired = sc->vtballoon_desired_npages;
536 
537 		if (desired != current) {
538 			if (desired > current)
539 				vtballoon_inflate(sc, desired - current);
540 			else
541 				vtballoon_deflate(sc, current - desired);
542 
543 			vtballoon_update_size(sc);
544 		}
545 	}
546 
547 	kproc_exit(0);
548 }
549 
550 static void
551 vtballoon_add_sysctl(struct vtballoon_softc *sc)
552 {
553 	device_t dev;
554 	struct sysctl_ctx_list *ctx;
555 	struct sysctl_oid *tree;
556 	struct sysctl_oid_list *child;
557 
558 	dev = sc->vtballoon_dev;
559 	ctx = device_get_sysctl_ctx(dev);
560 	tree = device_get_sysctl_tree(dev);
561 	child = SYSCTL_CHILDREN(tree);
562 
563 	SYSCTL_ADD_UINT(ctx, child, OID_AUTO, "desired",
564 	    CTLFLAG_RD, &sc->vtballoon_desired_npages, sizeof(uint32_t),
565 	    "Desired balloon size in pages");
566 
567 	SYSCTL_ADD_UINT(ctx, child, OID_AUTO, "current",
568 	    CTLFLAG_RD, &sc->vtballoon_current_npages, sizeof(uint32_t),
569 	    "Current balloon size in pages");
570 }
571