1 // SPDX-License-Identifier: GPL-2.0
2 /* Simple test of virtio code, entirely in userpsace. */
3 #define _GNU_SOURCE
4 #include <sched.h>
5 #include <err.h>
6 #include <linux/kernel.h>
7 #include <linux/err.h>
8 #include <linux/virtio.h>
9 #include <linux/vringh.h>
10 #include <linux/virtio_ring.h>
11 #include <linux/virtio_config.h>
12 #include <linux/uaccess.h>
13 #include <sys/types.h>
14 #include <sys/stat.h>
15 #include <sys/mman.h>
16 #include <sys/wait.h>
17 #include <fcntl.h>
18 
19 #define USER_MEM (1024*1024)
20 void *__user_addr_min, *__user_addr_max;
21 void *__kmalloc_fake, *__kfree_ignore_start, *__kfree_ignore_end;
22 static u64 user_addr_offset;
23 
24 #define RINGSIZE 256
25 #define ALIGN 4096
26 
never_notify_host(struct virtqueue * vq)27 static bool never_notify_host(struct virtqueue *vq)
28 {
29 	abort();
30 }
31 
never_callback_guest(struct virtqueue * vq)32 static void never_callback_guest(struct virtqueue *vq)
33 {
34 	abort();
35 }
36 
getrange_iov(struct vringh * vrh,u64 addr,struct vringh_range * r)37 static bool getrange_iov(struct vringh *vrh, u64 addr, struct vringh_range *r)
38 {
39 	if (addr < (u64)(unsigned long)__user_addr_min - user_addr_offset)
40 		return false;
41 	if (addr >= (u64)(unsigned long)__user_addr_max - user_addr_offset)
42 		return false;
43 
44 	r->start = (u64)(unsigned long)__user_addr_min - user_addr_offset;
45 	r->end_incl = (u64)(unsigned long)__user_addr_max - 1 - user_addr_offset;
46 	r->offset = user_addr_offset;
47 	return true;
48 }
49 
50 /* We return single byte ranges. */
getrange_slow(struct vringh * vrh,u64 addr,struct vringh_range * r)51 static bool getrange_slow(struct vringh *vrh, u64 addr, struct vringh_range *r)
52 {
53 	if (addr < (u64)(unsigned long)__user_addr_min - user_addr_offset)
54 		return false;
55 	if (addr >= (u64)(unsigned long)__user_addr_max - user_addr_offset)
56 		return false;
57 
58 	r->start = addr;
59 	r->end_incl = r->start;
60 	r->offset = user_addr_offset;
61 	return true;
62 }
63 
64 struct guest_virtio_device {
65 	struct virtio_device vdev;
66 	int to_host_fd;
67 	unsigned long notifies;
68 };
69 
parallel_notify_host(struct virtqueue * vq)70 static bool parallel_notify_host(struct virtqueue *vq)
71 {
72 	int rc;
73 	struct guest_virtio_device *gvdev;
74 
75 	gvdev = container_of(vq->vdev, struct guest_virtio_device, vdev);
76 	rc = write(gvdev->to_host_fd, "", 1);
77 	if (rc < 0)
78 		return false;
79 	gvdev->notifies++;
80 	return true;
81 }
82 
no_notify_host(struct virtqueue * vq)83 static bool no_notify_host(struct virtqueue *vq)
84 {
85 	return true;
86 }
87 
88 #define NUM_XFERS (10000000)
89 
90 /* We aim for two "distant" cpus. */
find_cpus(unsigned int * first,unsigned int * last)91 static void find_cpus(unsigned int *first, unsigned int *last)
92 {
93 	unsigned int i;
94 
95 	*first = -1U;
96 	*last = 0;
97 	for (i = 0; i < 4096; i++) {
98 		cpu_set_t set;
99 		CPU_ZERO(&set);
100 		CPU_SET(i, &set);
101 		if (sched_setaffinity(getpid(), sizeof(set), &set) == 0) {
102 			if (i < *first)
103 				*first = i;
104 			if (i > *last)
105 				*last = i;
106 		}
107 	}
108 }
109 
110 /* Opencoded version for fast mode */
vringh_get_head(struct vringh * vrh,u16 * head)111 static inline int vringh_get_head(struct vringh *vrh, u16 *head)
112 {
113 	u16 avail_idx, i;
114 	int err;
115 
116 	err = get_user(avail_idx, &vrh->vring.avail->idx);
117 	if (err)
118 		return err;
119 
120 	if (vrh->last_avail_idx == avail_idx)
121 		return 0;
122 
123 	/* Only get avail ring entries after they have been exposed by guest. */
124 	virtio_rmb(vrh->weak_barriers);
125 
126 	i = vrh->last_avail_idx & (vrh->vring.num - 1);
127 
128 	err = get_user(*head, &vrh->vring.avail->ring[i]);
129 	if (err)
130 		return err;
131 
132 	vrh->last_avail_idx++;
133 	return 1;
134 }
135 
parallel_test(u64 features,bool (* getrange)(struct vringh * vrh,u64 addr,struct vringh_range * r),bool fast_vringh)136 static int parallel_test(u64 features,
137 			 bool (*getrange)(struct vringh *vrh,
138 					  u64 addr, struct vringh_range *r),
139 			 bool fast_vringh)
140 {
141 	void *host_map, *guest_map;
142 	int fd, mapsize, to_guest[2], to_host[2];
143 	unsigned long xfers = 0, notifies = 0, receives = 0;
144 	unsigned int first_cpu, last_cpu;
145 	cpu_set_t cpu_set;
146 	char buf[128];
147 
148 	/* Create real file to mmap. */
149 	fd = open("/tmp/vringh_test-file", O_RDWR|O_CREAT|O_TRUNC, 0600);
150 	if (fd < 0)
151 		err(1, "Opening /tmp/vringh_test-file");
152 
153 	/* Extra room at the end for some data, and indirects */
154 	mapsize = vring_size(RINGSIZE, ALIGN)
155 		+ RINGSIZE * 2 * sizeof(int)
156 		+ RINGSIZE * 6 * sizeof(struct vring_desc);
157 	mapsize = (mapsize + getpagesize() - 1) & ~(getpagesize() - 1);
158 	ftruncate(fd, mapsize);
159 
160 	/* Parent and child use separate addresses, to check our mapping logic! */
161 	host_map = mmap(NULL, mapsize, PROT_READ|PROT_WRITE, MAP_SHARED, fd, 0);
162 	guest_map = mmap(NULL, mapsize, PROT_READ|PROT_WRITE, MAP_SHARED, fd, 0);
163 
164 	pipe(to_guest);
165 	pipe(to_host);
166 
167 	CPU_ZERO(&cpu_set);
168 	find_cpus(&first_cpu, &last_cpu);
169 	printf("Using CPUS %u and %u\n", first_cpu, last_cpu);
170 	fflush(stdout);
171 
172 	if (fork() != 0) {
173 		struct vringh vrh;
174 		int status, err, rlen = 0;
175 		char rbuf[5];
176 
177 		/* We are the host: never access guest addresses! */
178 		munmap(guest_map, mapsize);
179 
180 		__user_addr_min = host_map;
181 		__user_addr_max = __user_addr_min + mapsize;
182 		user_addr_offset = host_map - guest_map;
183 		assert(user_addr_offset);
184 
185 		close(to_guest[0]);
186 		close(to_host[1]);
187 
188 		vring_init(&vrh.vring, RINGSIZE, host_map, ALIGN);
189 		vringh_init_user(&vrh, features, RINGSIZE, true,
190 				 vrh.vring.desc, vrh.vring.avail, vrh.vring.used);
191 		CPU_SET(first_cpu, &cpu_set);
192 		if (sched_setaffinity(getpid(), sizeof(cpu_set), &cpu_set))
193 			errx(1, "Could not set affinity to cpu %u", first_cpu);
194 
195 		while (xfers < NUM_XFERS) {
196 			struct iovec host_riov[2], host_wiov[2];
197 			struct vringh_iov riov, wiov;
198 			u16 head, written;
199 
200 			if (fast_vringh) {
201 				for (;;) {
202 					err = vringh_get_head(&vrh, &head);
203 					if (err != 0)
204 						break;
205 					err = vringh_need_notify_user(&vrh);
206 					if (err < 0)
207 						errx(1, "vringh_need_notify_user: %i",
208 						     err);
209 					if (err) {
210 						write(to_guest[1], "", 1);
211 						notifies++;
212 					}
213 				}
214 				if (err != 1)
215 					errx(1, "vringh_get_head");
216 				written = 0;
217 				goto complete;
218 			} else {
219 				vringh_iov_init(&riov,
220 						host_riov,
221 						ARRAY_SIZE(host_riov));
222 				vringh_iov_init(&wiov,
223 						host_wiov,
224 						ARRAY_SIZE(host_wiov));
225 
226 				err = vringh_getdesc_user(&vrh, &riov, &wiov,
227 							  getrange, &head);
228 			}
229 			if (err == 0) {
230 				err = vringh_need_notify_user(&vrh);
231 				if (err < 0)
232 					errx(1, "vringh_need_notify_user: %i",
233 					     err);
234 				if (err) {
235 					write(to_guest[1], "", 1);
236 					notifies++;
237 				}
238 
239 				if (!vringh_notify_enable_user(&vrh))
240 					continue;
241 
242 				/* Swallow all notifies at once. */
243 				if (read(to_host[0], buf, sizeof(buf)) < 1)
244 					break;
245 
246 				vringh_notify_disable_user(&vrh);
247 				receives++;
248 				continue;
249 			}
250 			if (err != 1)
251 				errx(1, "vringh_getdesc_user: %i", err);
252 
253 			/* We simply copy bytes. */
254 			if (riov.used) {
255 				rlen = vringh_iov_pull_user(&riov, rbuf,
256 							    sizeof(rbuf));
257 				if (rlen != 4)
258 					errx(1, "vringh_iov_pull_user: %i",
259 					     rlen);
260 				assert(riov.i == riov.used);
261 				written = 0;
262 			} else {
263 				err = vringh_iov_push_user(&wiov, rbuf, rlen);
264 				if (err != rlen)
265 					errx(1, "vringh_iov_push_user: %i",
266 					     err);
267 				assert(wiov.i == wiov.used);
268 				written = err;
269 			}
270 		complete:
271 			xfers++;
272 
273 			err = vringh_complete_user(&vrh, head, written);
274 			if (err != 0)
275 				errx(1, "vringh_complete_user: %i", err);
276 		}
277 
278 		err = vringh_need_notify_user(&vrh);
279 		if (err < 0)
280 			errx(1, "vringh_need_notify_user: %i", err);
281 		if (err) {
282 			write(to_guest[1], "", 1);
283 			notifies++;
284 		}
285 		wait(&status);
286 		if (!WIFEXITED(status))
287 			errx(1, "Child died with signal %i?", WTERMSIG(status));
288 		if (WEXITSTATUS(status) != 0)
289 			errx(1, "Child exited %i?", WEXITSTATUS(status));
290 		printf("Host: notified %lu, pinged %lu\n", notifies, receives);
291 		return 0;
292 	} else {
293 		struct guest_virtio_device gvdev;
294 		struct virtqueue *vq;
295 		unsigned int *data;
296 		struct vring_desc *indirects;
297 		unsigned int finished = 0;
298 
299 		/* We pass sg[]s pointing into here, but we need RINGSIZE+1 */
300 		data = guest_map + vring_size(RINGSIZE, ALIGN);
301 		indirects = (void *)data + (RINGSIZE + 1) * 2 * sizeof(int);
302 
303 		/* We are the guest. */
304 		munmap(host_map, mapsize);
305 
306 		close(to_guest[1]);
307 		close(to_host[0]);
308 
309 		gvdev.vdev.features = features;
310 		INIT_LIST_HEAD(&gvdev.vdev.vqs);
311 		gvdev.to_host_fd = to_host[1];
312 		gvdev.notifies = 0;
313 
314 		CPU_SET(first_cpu, &cpu_set);
315 		if (sched_setaffinity(getpid(), sizeof(cpu_set), &cpu_set))
316 			err(1, "Could not set affinity to cpu %u", first_cpu);
317 
318 		vq = vring_new_virtqueue(0, RINGSIZE, ALIGN, &gvdev.vdev, true,
319 					 false, guest_map,
320 					 fast_vringh ? no_notify_host
321 					 : parallel_notify_host,
322 					 never_callback_guest, "guest vq");
323 
324 		/* Don't kfree indirects. */
325 		__kfree_ignore_start = indirects;
326 		__kfree_ignore_end = indirects + RINGSIZE * 6;
327 
328 		while (xfers < NUM_XFERS) {
329 			struct scatterlist sg[4];
330 			unsigned int num_sg, len;
331 			int *dbuf, err;
332 			bool output = !(xfers % 2);
333 
334 			/* Consume bufs. */
335 			while ((dbuf = virtqueue_get_buf(vq, &len)) != NULL) {
336 				if (len == 4)
337 					assert(*dbuf == finished - 1);
338 				else if (!fast_vringh)
339 					assert(*dbuf == finished);
340 				finished++;
341 			}
342 
343 			/* Produce a buffer. */
344 			dbuf = data + (xfers % (RINGSIZE + 1));
345 
346 			if (output)
347 				*dbuf = xfers;
348 			else
349 				*dbuf = -1;
350 
351 			switch ((xfers / sizeof(*dbuf)) % 4) {
352 			case 0:
353 				/* Nasty three-element sg list. */
354 				sg_init_table(sg, num_sg = 3);
355 				sg_set_buf(&sg[0], (void *)dbuf, 1);
356 				sg_set_buf(&sg[1], (void *)dbuf + 1, 2);
357 				sg_set_buf(&sg[2], (void *)dbuf + 3, 1);
358 				break;
359 			case 1:
360 				sg_init_table(sg, num_sg = 2);
361 				sg_set_buf(&sg[0], (void *)dbuf, 1);
362 				sg_set_buf(&sg[1], (void *)dbuf + 1, 3);
363 				break;
364 			case 2:
365 				sg_init_table(sg, num_sg = 1);
366 				sg_set_buf(&sg[0], (void *)dbuf, 4);
367 				break;
368 			case 3:
369 				sg_init_table(sg, num_sg = 4);
370 				sg_set_buf(&sg[0], (void *)dbuf, 1);
371 				sg_set_buf(&sg[1], (void *)dbuf + 1, 1);
372 				sg_set_buf(&sg[2], (void *)dbuf + 2, 1);
373 				sg_set_buf(&sg[3], (void *)dbuf + 3, 1);
374 				break;
375 			}
376 
377 			/* May allocate an indirect, so force it to allocate
378 			 * user addr */
379 			__kmalloc_fake = indirects + (xfers % RINGSIZE) * 4;
380 			if (output)
381 				err = virtqueue_add_outbuf(vq, sg, num_sg, dbuf,
382 							   GFP_KERNEL);
383 			else
384 				err = virtqueue_add_inbuf(vq, sg, num_sg,
385 							  dbuf, GFP_KERNEL);
386 
387 			if (err == -ENOSPC) {
388 				if (!virtqueue_enable_cb_delayed(vq))
389 					continue;
390 				/* Swallow all notifies at once. */
391 				if (read(to_guest[0], buf, sizeof(buf)) < 1)
392 					break;
393 
394 				receives++;
395 				virtqueue_disable_cb(vq);
396 				continue;
397 			}
398 
399 			if (err)
400 				errx(1, "virtqueue_add_in/outbuf: %i", err);
401 
402 			xfers++;
403 			virtqueue_kick(vq);
404 		}
405 
406 		/* Any extra? */
407 		while (finished != xfers) {
408 			int *dbuf;
409 			unsigned int len;
410 
411 			/* Consume bufs. */
412 			dbuf = virtqueue_get_buf(vq, &len);
413 			if (dbuf) {
414 				if (len == 4)
415 					assert(*dbuf == finished - 1);
416 				else
417 					assert(len == 0);
418 				finished++;
419 				continue;
420 			}
421 
422 			if (!virtqueue_enable_cb_delayed(vq))
423 				continue;
424 			if (read(to_guest[0], buf, sizeof(buf)) < 1)
425 				break;
426 
427 			receives++;
428 			virtqueue_disable_cb(vq);
429 		}
430 
431 		printf("Guest: notified %lu, pinged %lu\n",
432 		       gvdev.notifies, receives);
433 		vring_del_virtqueue(vq);
434 		return 0;
435 	}
436 }
437 
main(int argc,char * argv[])438 int main(int argc, char *argv[])
439 {
440 	struct virtio_device vdev;
441 	struct virtqueue *vq;
442 	struct vringh vrh;
443 	struct scatterlist guest_sg[RINGSIZE], *sgs[2];
444 	struct iovec host_riov[2], host_wiov[2];
445 	struct vringh_iov riov, wiov;
446 	struct vring_used_elem used[RINGSIZE];
447 	char buf[28];
448 	u16 head;
449 	int err;
450 	unsigned i;
451 	void *ret;
452 	bool (*getrange)(struct vringh *vrh, u64 addr, struct vringh_range *r);
453 	bool fast_vringh = false, parallel = false;
454 
455 	getrange = getrange_iov;
456 	vdev.features = 0;
457 	INIT_LIST_HEAD(&vdev.vqs);
458 
459 	while (argv[1]) {
460 		if (strcmp(argv[1], "--indirect") == 0)
461 			__virtio_set_bit(&vdev, VIRTIO_RING_F_INDIRECT_DESC);
462 		else if (strcmp(argv[1], "--eventidx") == 0)
463 			__virtio_set_bit(&vdev, VIRTIO_RING_F_EVENT_IDX);
464 		else if (strcmp(argv[1], "--virtio-1") == 0)
465 			__virtio_set_bit(&vdev, VIRTIO_F_VERSION_1);
466 		else if (strcmp(argv[1], "--slow-range") == 0)
467 			getrange = getrange_slow;
468 		else if (strcmp(argv[1], "--fast-vringh") == 0)
469 			fast_vringh = true;
470 		else if (strcmp(argv[1], "--parallel") == 0)
471 			parallel = true;
472 		else
473 			errx(1, "Unknown arg %s", argv[1]);
474 		argv++;
475 	}
476 
477 	if (parallel)
478 		return parallel_test(vdev.features, getrange, fast_vringh);
479 
480 	if (posix_memalign(&__user_addr_min, PAGE_SIZE, USER_MEM) != 0)
481 		abort();
482 	__user_addr_max = __user_addr_min + USER_MEM;
483 	memset(__user_addr_min, 0, vring_size(RINGSIZE, ALIGN));
484 
485 	/* Set up guest side. */
486 	vq = vring_new_virtqueue(0, RINGSIZE, ALIGN, &vdev, true, false,
487 				 __user_addr_min,
488 				 never_notify_host, never_callback_guest,
489 				 "guest vq");
490 
491 	/* Set up host side. */
492 	vring_init(&vrh.vring, RINGSIZE, __user_addr_min, ALIGN);
493 	vringh_init_user(&vrh, vdev.features, RINGSIZE, true,
494 			 vrh.vring.desc, vrh.vring.avail, vrh.vring.used);
495 
496 	/* No descriptor to get yet... */
497 	err = vringh_getdesc_user(&vrh, &riov, &wiov, getrange, &head);
498 	if (err != 0)
499 		errx(1, "vringh_getdesc_user: %i", err);
500 
501 	/* Guest puts in a descriptor. */
502 	memcpy(__user_addr_max - 1, "a", 1);
503 	sg_init_table(guest_sg, 1);
504 	sg_set_buf(&guest_sg[0], __user_addr_max - 1, 1);
505 	sg_init_table(guest_sg+1, 1);
506 	sg_set_buf(&guest_sg[1], __user_addr_max - 3, 2);
507 	sgs[0] = &guest_sg[0];
508 	sgs[1] = &guest_sg[1];
509 
510 	/* May allocate an indirect, so force it to allocate user addr */
511 	__kmalloc_fake = __user_addr_min + vring_size(RINGSIZE, ALIGN);
512 	err = virtqueue_add_sgs(vq, sgs, 1, 1, &err, GFP_KERNEL);
513 	if (err)
514 		errx(1, "virtqueue_add_sgs: %i", err);
515 	__kmalloc_fake = NULL;
516 
517 	/* Host retreives it. */
518 	vringh_iov_init(&riov, host_riov, ARRAY_SIZE(host_riov));
519 	vringh_iov_init(&wiov, host_wiov, ARRAY_SIZE(host_wiov));
520 
521 	err = vringh_getdesc_user(&vrh, &riov, &wiov, getrange, &head);
522 	if (err != 1)
523 		errx(1, "vringh_getdesc_user: %i", err);
524 
525 	assert(riov.used == 1);
526 	assert(riov.iov[0].iov_base == __user_addr_max - 1);
527 	assert(riov.iov[0].iov_len == 1);
528 	if (getrange != getrange_slow) {
529 		assert(wiov.used == 1);
530 		assert(wiov.iov[0].iov_base == __user_addr_max - 3);
531 		assert(wiov.iov[0].iov_len == 2);
532 	} else {
533 		assert(wiov.used == 2);
534 		assert(wiov.iov[0].iov_base == __user_addr_max - 3);
535 		assert(wiov.iov[0].iov_len == 1);
536 		assert(wiov.iov[1].iov_base == __user_addr_max - 2);
537 		assert(wiov.iov[1].iov_len == 1);
538 	}
539 
540 	err = vringh_iov_pull_user(&riov, buf, 5);
541 	if (err != 1)
542 		errx(1, "vringh_iov_pull_user: %i", err);
543 	assert(buf[0] == 'a');
544 	assert(riov.i == 1);
545 	assert(vringh_iov_pull_user(&riov, buf, 5) == 0);
546 
547 	memcpy(buf, "bcdef", 5);
548 	err = vringh_iov_push_user(&wiov, buf, 5);
549 	if (err != 2)
550 		errx(1, "vringh_iov_push_user: %i", err);
551 	assert(memcmp(__user_addr_max - 3, "bc", 2) == 0);
552 	assert(wiov.i == wiov.used);
553 	assert(vringh_iov_push_user(&wiov, buf, 5) == 0);
554 
555 	/* Host is done. */
556 	err = vringh_complete_user(&vrh, head, err);
557 	if (err != 0)
558 		errx(1, "vringh_complete_user: %i", err);
559 
560 	/* Guest should see used token now. */
561 	__kfree_ignore_start = __user_addr_min + vring_size(RINGSIZE, ALIGN);
562 	__kfree_ignore_end = __kfree_ignore_start + 1;
563 	ret = virtqueue_get_buf(vq, &i);
564 	if (ret != &err)
565 		errx(1, "virtqueue_get_buf: %p", ret);
566 	assert(i == 2);
567 
568 	/* Guest puts in a huge descriptor. */
569 	sg_init_table(guest_sg, RINGSIZE);
570 	for (i = 0; i < RINGSIZE; i++) {
571 		sg_set_buf(&guest_sg[i],
572 			   __user_addr_max - USER_MEM/4, USER_MEM/4);
573 	}
574 
575 	/* Fill contents with recognisable garbage. */
576 	for (i = 0; i < USER_MEM/4; i++)
577 		((char *)__user_addr_max - USER_MEM/4)[i] = i;
578 
579 	/* This will allocate an indirect, so force it to allocate user addr */
580 	__kmalloc_fake = __user_addr_min + vring_size(RINGSIZE, ALIGN);
581 	err = virtqueue_add_outbuf(vq, guest_sg, RINGSIZE, &err, GFP_KERNEL);
582 	if (err)
583 		errx(1, "virtqueue_add_outbuf (large): %i", err);
584 	__kmalloc_fake = NULL;
585 
586 	/* Host picks it up (allocates new iov). */
587 	vringh_iov_init(&riov, host_riov, ARRAY_SIZE(host_riov));
588 	vringh_iov_init(&wiov, host_wiov, ARRAY_SIZE(host_wiov));
589 
590 	err = vringh_getdesc_user(&vrh, &riov, &wiov, getrange, &head);
591 	if (err != 1)
592 		errx(1, "vringh_getdesc_user: %i", err);
593 
594 	assert(riov.max_num & VRINGH_IOV_ALLOCATED);
595 	assert(riov.iov != host_riov);
596 	if (getrange != getrange_slow)
597 		assert(riov.used == RINGSIZE);
598 	else
599 		assert(riov.used == RINGSIZE * USER_MEM/4);
600 
601 	assert(!(wiov.max_num & VRINGH_IOV_ALLOCATED));
602 	assert(wiov.used == 0);
603 
604 	/* Pull data back out (in odd chunks), should be as expected. */
605 	for (i = 0; i < RINGSIZE * USER_MEM/4; i += 3) {
606 		err = vringh_iov_pull_user(&riov, buf, 3);
607 		if (err != 3 && i + err != RINGSIZE * USER_MEM/4)
608 			errx(1, "vringh_iov_pull_user large: %i", err);
609 		assert(buf[0] == (char)i);
610 		assert(err < 2 || buf[1] == (char)(i + 1));
611 		assert(err < 3 || buf[2] == (char)(i + 2));
612 	}
613 	assert(riov.i == riov.used);
614 	vringh_iov_cleanup(&riov);
615 	vringh_iov_cleanup(&wiov);
616 
617 	/* Complete using multi interface, just because we can. */
618 	used[0].id = head;
619 	used[0].len = 0;
620 	err = vringh_complete_multi_user(&vrh, used, 1);
621 	if (err)
622 		errx(1, "vringh_complete_multi_user(1): %i", err);
623 
624 	/* Free up those descriptors. */
625 	ret = virtqueue_get_buf(vq, &i);
626 	if (ret != &err)
627 		errx(1, "virtqueue_get_buf: %p", ret);
628 
629 	/* Add lots of descriptors. */
630 	sg_init_table(guest_sg, 1);
631 	sg_set_buf(&guest_sg[0], __user_addr_max - 1, 1);
632 	for (i = 0; i < RINGSIZE; i++) {
633 		err = virtqueue_add_outbuf(vq, guest_sg, 1, &err, GFP_KERNEL);
634 		if (err)
635 			errx(1, "virtqueue_add_outbuf (multiple): %i", err);
636 	}
637 
638 	/* Now get many, and consume them all at once. */
639 	vringh_iov_init(&riov, host_riov, ARRAY_SIZE(host_riov));
640 	vringh_iov_init(&wiov, host_wiov, ARRAY_SIZE(host_wiov));
641 
642 	for (i = 0; i < RINGSIZE; i++) {
643 		err = vringh_getdesc_user(&vrh, &riov, &wiov, getrange, &head);
644 		if (err != 1)
645 			errx(1, "vringh_getdesc_user: %i", err);
646 		used[i].id = head;
647 		used[i].len = 0;
648 	}
649 	/* Make sure it wraps around ring, to test! */
650 	assert(vrh.vring.used->idx % RINGSIZE != 0);
651 	err = vringh_complete_multi_user(&vrh, used, RINGSIZE);
652 	if (err)
653 		errx(1, "vringh_complete_multi_user: %i", err);
654 
655 	/* Free those buffers. */
656 	for (i = 0; i < RINGSIZE; i++) {
657 		unsigned len;
658 		assert(virtqueue_get_buf(vq, &len) != NULL);
659 	}
660 
661 	/* Test weird (but legal!) indirect. */
662 	if (__virtio_test_bit(&vdev, VIRTIO_RING_F_INDIRECT_DESC)) {
663 		char *data = __user_addr_max - USER_MEM/4;
664 		struct vring_desc *d = __user_addr_max - USER_MEM/2;
665 		struct vring vring;
666 
667 		/* Force creation of direct, which we modify. */
668 		__virtio_clear_bit(&vdev, VIRTIO_RING_F_INDIRECT_DESC);
669 		vq = vring_new_virtqueue(0, RINGSIZE, ALIGN, &vdev, true,
670 					 false, __user_addr_min,
671 					 never_notify_host,
672 					 never_callback_guest,
673 					 "guest vq");
674 
675 		sg_init_table(guest_sg, 4);
676 		sg_set_buf(&guest_sg[0], d, sizeof(*d)*2);
677 		sg_set_buf(&guest_sg[1], d + 2, sizeof(*d)*1);
678 		sg_set_buf(&guest_sg[2], data + 6, 4);
679 		sg_set_buf(&guest_sg[3], d + 3, sizeof(*d)*3);
680 
681 		err = virtqueue_add_outbuf(vq, guest_sg, 4, &err, GFP_KERNEL);
682 		if (err)
683 			errx(1, "virtqueue_add_outbuf (indirect): %i", err);
684 
685 		vring_init(&vring, RINGSIZE, __user_addr_min, ALIGN);
686 
687 		/* They're used in order, but double-check... */
688 		assert(vring.desc[0].addr == (unsigned long)d);
689 		assert(vring.desc[1].addr == (unsigned long)(d+2));
690 		assert(vring.desc[2].addr == (unsigned long)data + 6);
691 		assert(vring.desc[3].addr == (unsigned long)(d+3));
692 		vring.desc[0].flags |= VRING_DESC_F_INDIRECT;
693 		vring.desc[1].flags |= VRING_DESC_F_INDIRECT;
694 		vring.desc[3].flags |= VRING_DESC_F_INDIRECT;
695 
696 		/* First indirect */
697 		d[0].addr = (unsigned long)data;
698 		d[0].len = 1;
699 		d[0].flags = VRING_DESC_F_NEXT;
700 		d[0].next = 1;
701 		d[1].addr = (unsigned long)data + 1;
702 		d[1].len = 2;
703 		d[1].flags = 0;
704 
705 		/* Second indirect */
706 		d[2].addr = (unsigned long)data + 3;
707 		d[2].len = 3;
708 		d[2].flags = 0;
709 
710 		/* Third indirect */
711 		d[3].addr = (unsigned long)data + 10;
712 		d[3].len = 5;
713 		d[3].flags = VRING_DESC_F_NEXT;
714 		d[3].next = 1;
715 		d[4].addr = (unsigned long)data + 15;
716 		d[4].len = 6;
717 		d[4].flags = VRING_DESC_F_NEXT;
718 		d[4].next = 2;
719 		d[5].addr = (unsigned long)data + 21;
720 		d[5].len = 7;
721 		d[5].flags = 0;
722 
723 		/* Host picks it up (allocates new iov). */
724 		vringh_iov_init(&riov, host_riov, ARRAY_SIZE(host_riov));
725 		vringh_iov_init(&wiov, host_wiov, ARRAY_SIZE(host_wiov));
726 
727 		err = vringh_getdesc_user(&vrh, &riov, &wiov, getrange, &head);
728 		if (err != 1)
729 			errx(1, "vringh_getdesc_user: %i", err);
730 
731 		if (head != 0)
732 			errx(1, "vringh_getdesc_user: head %i not 0", head);
733 
734 		assert(riov.max_num & VRINGH_IOV_ALLOCATED);
735 		if (getrange != getrange_slow)
736 			assert(riov.used == 7);
737 		else
738 			assert(riov.used == 28);
739 		err = vringh_iov_pull_user(&riov, buf, 29);
740 		assert(err == 28);
741 
742 		/* Data should be linear. */
743 		for (i = 0; i < err; i++)
744 			assert(buf[i] == i);
745 		vringh_iov_cleanup(&riov);
746 	}
747 
748 	/* Don't leak memory... */
749 	vring_del_virtqueue(vq);
750 	free(__user_addr_min);
751 
752 	return 0;
753 }
754