xref: /freebsd/sys/dev/vmware/vmci/vmci_qpair.c (revision c697fb7f)
1 /*-
2  * Copyright (c) 2018 VMware, Inc.
3  *
4  * SPDX-License-Identifier: (BSD-2-Clause OR GPL-2.0)
5  */
6 
7 /* This file implements Queue accessor methods. */
8 
9 /*
10  * vmci_qpair is an interface that hides the queue pair internals. Rather than
11  * access each queue in a pair directly, operations are performed on the queue
12  * as a whole. This is simpler and less error-prone, and allows for future
13  * queue pair features to be added under the hood with no change to the client
14  * code.
15  */
16 
17 #include <sys/cdefs.h>
18 __FBSDID("$FreeBSD$");
19 
20 #include "vmci_kernel_api.h"
21 #include "vmci_kernel_defs.h"
22 #include "vmci_kernel_if.h"
23 #include "vmci_queue.h"
24 #include "vmci_queue_pair.h"
25 
26 /* This structure is opaque to the clients. */
27 struct vmci_qpair {
28 	struct vmci_handle	handle;
29 	struct vmci_queue	*produce_q;
30 	struct vmci_queue	*consume_q;
31 	uint64_t		produce_q_size;
32 	uint64_t		consume_q_size;
33 	vmci_id			peer;
34 	uint32_t		flags;
35 	vmci_privilege_flags	priv_flags;
36 	uint32_t		blocked;
37 	vmci_event		event;
38 };
39 
40 static void	vmci_qpair_get_queue_headers(const struct vmci_qpair *qpair,
41 		    struct vmci_queue_header **produce_q_header,
42 		    struct vmci_queue_header **consume_q_header);
43 
44 /*
45  *------------------------------------------------------------------------------
46  *
47  * vmci_queue_add_producer_tail --
48  *
49  *     Helper routine to increment the Producer Tail.
50  *
51  * Results:
52  *     VMCI_ERROR_NOT_FOUND if the vmm_world registered with the queue cannot
53  *     be found. Otherwise VMCI_SUCCESS.
54  *
55  * Side effects:
56  *     None.
57  *
58  *------------------------------------------------------------------------------
59  */
60 
61 static inline int
62 vmci_queue_add_producer_tail(struct vmci_queue *queue,
63     size_t add, uint64_t queue_size)
64 {
65 
66 	vmci_queue_header_add_producer_tail(queue->q_header, add, queue_size);
67 	return (VMCI_SUCCESS);
68 }
69 
70 /*
71  *------------------------------------------------------------------------------
72  *
73  * vmci_queue_add_consumer_head --
74  *
75  *     Helper routine to increment the Consumer Head.
76  *
77  * Results:
78  *     VMCI_ERROR_NOT_FOUND if the vmm_world registered with the queue cannot
79  *     be found. Otherwise VMCI_SUCCESS.
80  *
81  * Side effects:
82  *     None.
83  *
84  *------------------------------------------------------------------------------
85  */
86 
87 static inline int
88 vmci_queue_add_consumer_head(struct vmci_queue *queue,
89     size_t add, uint64_t queue_size)
90 {
91 
92 	vmci_queue_header_add_consumer_head(queue->q_header, add, queue_size);
93 	return (VMCI_SUCCESS);
94 }
95 
96 /*
97  *------------------------------------------------------------------------------
98  *
99  * vmci_qpair_get_queue_headers --
100  *
101  *     Helper routine that will retrieve the produce and consume headers of a
102  *     given queue pair.
103  *
104  * Results:
105  *     VMCI_SUCCESS if either current or saved queue headers are found.
106  *     Appropriate error code otherwise.
107  *
108  * Side effects:
109  *     None.
110  *
111  *------------------------------------------------------------------------------
112  */
113 
114 static void
115 vmci_qpair_get_queue_headers(const struct vmci_qpair *qpair,
116     struct vmci_queue_header **produce_q_header,
117     struct vmci_queue_header **consume_q_header)
118 {
119 
120 	ASSERT((qpair->produce_q != NULL) && (qpair->consume_q != NULL));
121 	*produce_q_header = qpair->produce_q->q_header;
122 	*consume_q_header = qpair->consume_q->q_header;
123 }
124 
125 /*
126  *------------------------------------------------------------------------------
127  *
128  * vmci_qpair_alloc --
129  *
130  *     This is the client interface for allocating the memory for a vmci_qpair
131  *     structure and then attaching to the underlying queue. If an error occurs
132  *     allocating the memory for the vmci_qpair structure, no attempt is made to
133  *     attach. If an error occurs attaching, then there's the vmci_qpair
134  *     structure is freed.
135  *
136  * Results:
137  *     An err, if < 0.
138  *
139  * Side effects:
140  *     None.
141  *
142  *------------------------------------------------------------------------------
143  */
144 
145 int
146 vmci_qpair_alloc(struct vmci_qpair **qpair, struct vmci_handle *handle,
147     uint64_t produce_q_size, uint64_t consume_q_size, vmci_id peer,
148     uint32_t flags, vmci_privilege_flags priv_flags)
149 {
150 	struct vmci_qpair *my_qpair;
151 	vmci_event_release_cb wakeup_cb;
152 	void *client_data;
153 	int retval;
154 
155 	/*
156 	 * Restrict the size of a queuepair. Though the device enforces a limit
157 	 * on the total amount of memory that can be allocated to queuepairs for
158 	 * a guest, we avoid unnecessarily allocating a lot of memory. Also, we
159 	 * try to allocate this memory before we make the queuepair allocation
160 	 * hypercall.
161 	 *
162 	 * (Note that this doesn't prevent all cases; a user with only this much
163 	 * physical memory could still get into trouble.) The error used by the
164 	 * device is NO_RESOURCES, so use that here too.
165 	 */
166 
167 	if (produce_q_size + consume_q_size <
168 	    MAX(produce_q_size, consume_q_size) ||
169 	    produce_q_size + consume_q_size > VMCI_MAX_GUEST_QP_MEMORY)
170 		return (VMCI_ERROR_NO_RESOURCES);
171 
172 	if (flags & VMCI_QPFLAG_NONBLOCK)
173 		return (VMCI_ERROR_INVALID_ARGS);
174 
175 	my_qpair = vmci_alloc_kernel_mem(sizeof(*my_qpair), VMCI_MEMORY_NORMAL);
176 	if (!my_qpair)
177 		return (VMCI_ERROR_NO_MEM);
178 
179 	my_qpair->produce_q_size = produce_q_size;
180 	my_qpair->consume_q_size = consume_q_size;
181 	my_qpair->peer = peer;
182 	my_qpair->flags = flags;
183 	my_qpair->priv_flags = priv_flags;
184 
185 	client_data = NULL;
186 	wakeup_cb = NULL;
187 
188 	retval = vmci_queue_pair_alloc(handle, &my_qpair->produce_q,
189 	    my_qpair->produce_q_size, &my_qpair->consume_q,
190 	    my_qpair->consume_q_size, my_qpair->peer, my_qpair->flags,
191 	    my_qpair->priv_flags);
192 
193 	if (retval < VMCI_SUCCESS) {
194 		vmci_free_kernel_mem(my_qpair, sizeof(*my_qpair));
195 		return (retval);
196 	}
197 
198 	*qpair = my_qpair;
199 	my_qpair->handle = *handle;
200 
201 	return (retval);
202 }
203 
204 /*
205  *------------------------------------------------------------------------------
206  *
207  * vmci_qpair_detach --
208  *
209  *     This is the client interface for detaching from a vmci_qpair. Note that
210  *     this routine will free the memory allocated for the vmci_qpair structure,
211  *     too.
212  *
213  * Results:
214  *     An error, if < 0.
215  *
216  * Side effects:
217  *     Will clear the caller's pointer to the vmci_qpair structure.
218  *
219  *------------------------------------------------------------------------------
220  */
221 
222 int
223 vmci_qpair_detach(struct vmci_qpair **qpair)
224 {
225 	struct vmci_qpair *old_qpair;
226 	int result;
227 
228 	if (!qpair || !(*qpair))
229 		return (VMCI_ERROR_INVALID_ARGS);
230 
231 	old_qpair = *qpair;
232 	result = vmci_queue_pair_detach(old_qpair->handle);
233 
234 	/*
235 	 * The guest can fail to detach for a number of reasons, and if it does
236 	 * so, it will cleanup the entry (if there is one). We need to release
237 	 * the qpair struct here; there isn't much the caller can do, and we
238 	 * don't want to leak.
239 	 */
240 
241 	if (old_qpair->flags & VMCI_QPFLAG_LOCAL)
242 		vmci_destroy_event(&old_qpair->event);
243 
244 	vmci_free_kernel_mem(old_qpair, sizeof(*old_qpair));
245 	*qpair = NULL;
246 
247 	return (result);
248 }
249 
250 /*
251  *------------------------------------------------------------------------------
252  *
253  * vmci_qpair_get_produce_indexes --
254  *
255  *     This is the client interface for getting the current indexes of the
256  *     qpair from the point of the view of the caller as the producer.
257  *
258  * Results:
259  *     err, if < 0
260  *     Success otherwise.
261  *
262  * Side effects:
263  *     None.
264  *
265  *------------------------------------------------------------------------------
266  */
267 
268 int
269 vmci_qpair_get_produce_indexes(const struct vmci_qpair *qpair,
270     uint64_t *producer_tail, uint64_t *consumer_head)
271 {
272 	struct vmci_queue_header *consume_q_header;
273 	struct vmci_queue_header *produce_q_header;
274 
275 	if (!qpair)
276 		return (VMCI_ERROR_INVALID_ARGS);
277 
278 	vmci_qpair_get_queue_headers(qpair, &produce_q_header,
279 	    &consume_q_header);
280 	vmci_queue_header_get_pointers(produce_q_header, consume_q_header,
281 	    producer_tail, consumer_head);
282 
283 	if ((producer_tail && *producer_tail >= qpair->produce_q_size) ||
284 	    (consumer_head && *consumer_head >= qpair->produce_q_size))
285 		return (VMCI_ERROR_INVALID_SIZE);
286 
287 	return (VMCI_SUCCESS);
288 }
289 
290 /*
291  *------------------------------------------------------------------------------
292  *
293  * vmci_qpair_get_consume_indexes --
294  *
295  *     This is the client interface for getting the current indexes of the
296  *     QPair from the point of the view of the caller as the consumer.
297  *
298  * Results:
299  *     err, if < 0
300  *     Success otherwise.
301  *
302  * Side effects:
303  *     None.
304  *
305  *------------------------------------------------------------------------------
306  */
307 
308 int
309 vmci_qpair_get_consume_indexes(const struct vmci_qpair *qpair,
310     uint64_t *consumer_tail, uint64_t *producer_head)
311 {
312 	struct vmci_queue_header *consume_q_header;
313 	struct vmci_queue_header *produce_q_header;
314 
315 	if (!qpair)
316 		return (VMCI_ERROR_INVALID_ARGS);
317 
318 	vmci_qpair_get_queue_headers(qpair, &produce_q_header,
319 	    &consume_q_header);
320 	vmci_queue_header_get_pointers(consume_q_header, produce_q_header,
321 	    consumer_tail, producer_head);
322 
323 	if ((consumer_tail && *consumer_tail >= qpair->consume_q_size) ||
324 	    (producer_head && *producer_head >= qpair->consume_q_size))
325 		return (VMCI_ERROR_INVALID_SIZE);
326 
327 	return (VMCI_SUCCESS);
328 }
329 
330 /*
331  *------------------------------------------------------------------------------
332  *
333  * vmci_qpair_produce_free_space --
334  *
335  *     This is the client interface for getting the amount of free space in the
336  *     QPair from the point of the view of the caller as the producer which is
337  *     the common case.
338  *
339  * Results:
340  *     Err, if < 0.
341  *     Full queue if = 0.
342  *     Number of available bytes into which data can be enqueued if > 0.
343  *
344  * Side effects:
345  *     None.
346  *
347  *------------------------------------------------------------------------------
348  */
349 
350 int64_t
351 vmci_qpair_produce_free_space(const struct vmci_qpair *qpair)
352 {
353 	struct vmci_queue_header *consume_q_header;
354 	struct vmci_queue_header *produce_q_header;
355 	int64_t result;
356 
357 	if (!qpair)
358 		return (VMCI_ERROR_INVALID_ARGS);
359 
360 	vmci_qpair_get_queue_headers(qpair, &produce_q_header,
361 	    &consume_q_header);
362 	result = vmci_queue_header_free_space(produce_q_header, consume_q_header,
363 	    qpair->produce_q_size);
364 
365 	return (result);
366 }
367 
368 /*
369  *------------------------------------------------------------------------------
370  *
371  * vmci_qpair_consume_free_space --
372  *
373  *     This is the client interface for getting the amount of free space in the
374  *     QPair from the point of the view of the caller as the consumer which is
375  *     not the common case (see vmci_qpair_Produce_free_space(), above).
376  *
377  * Results:
378  *     Err, if < 0.
379  *     Full queue if = 0.
380  *     Number of available bytes into which data can be enqueued if > 0.
381  *
382  * Side effects:
383  *     None.
384  *
385  *------------------------------------------------------------------------------
386  */
387 
388 int64_t
389 vmci_qpair_consume_free_space(const struct vmci_qpair *qpair)
390 {
391 	struct vmci_queue_header *consume_q_header;
392 	struct vmci_queue_header *produce_q_header;
393 	int64_t result;
394 
395 	if (!qpair)
396 		return (VMCI_ERROR_INVALID_ARGS);
397 
398 	vmci_qpair_get_queue_headers(qpair, &produce_q_header,
399 	    &consume_q_header);
400 	result = vmci_queue_header_free_space(consume_q_header, produce_q_header,
401 	    qpair->consume_q_size);
402 
403 	return (result);
404 }
405 
406 /*
407  *------------------------------------------------------------------------------
408  *
409  * vmci_qpair_produce_buf_ready --
410  *
411  *     This is the client interface for getting the amount of enqueued data in
412  *     the QPair from the point of the view of the caller as the producer which
413  *     is not the common case (see vmci_qpair_Consume_buf_ready(), above).
414  *
415  * Results:
416  *     Err, if < 0.
417  *     Empty queue if = 0.
418  *     Number of bytes ready to be dequeued if > 0.
419  *
420  * Side effects:
421  *     None.
422  *
423  *------------------------------------------------------------------------------
424  */
425 
426 int64_t
427 vmci_qpair_produce_buf_ready(const struct vmci_qpair *qpair)
428 {
429 	struct vmci_queue_header *consume_q_header;
430 	struct vmci_queue_header *produce_q_header;
431 	int64_t result;
432 
433 	if (!qpair)
434 		return (VMCI_ERROR_INVALID_ARGS);
435 
436 	vmci_qpair_get_queue_headers(qpair, &produce_q_header,
437 	    &consume_q_header);
438 	result = vmci_queue_header_buf_ready(produce_q_header, consume_q_header,
439 	    qpair->produce_q_size);
440 
441 	return (result);
442 }
443 
444 /*
445  *------------------------------------------------------------------------------
446  *
447  * vmci_qpair_consume_buf_ready --
448  *
449  *     This is the client interface for getting the amount of enqueued data in
450  *     the QPair from the point of the view of the caller as the consumer which
451  *     is the normal case.
452  *
453  * Results:
454  *     Err, if < 0.
455  *     Empty queue if = 0.
456  *     Number of bytes ready to be dequeued if > 0.
457  *
458  * Side effects:
459  *     None.
460  *
461  *------------------------------------------------------------------------------
462  */
463 
464 int64_t
465 vmci_qpair_consume_buf_ready(const struct vmci_qpair *qpair)
466 {
467 	struct vmci_queue_header *consume_q_header;
468 	struct vmci_queue_header *produce_q_header;
469 	int64_t result;
470 
471 	if (!qpair)
472 		return (VMCI_ERROR_INVALID_ARGS);
473 
474 	vmci_qpair_get_queue_headers(qpair, &produce_q_header,
475 	    &consume_q_header);
476 	result = vmci_queue_header_buf_ready(consume_q_header, produce_q_header,
477 	    qpair->consume_q_size);
478 
479 	return (result);
480 }
481 
482 /*
483  *------------------------------------------------------------------------------
484  *
485  * enqueue --
486  *
487  *     Enqueues a given buffer to the produce queue using the provided function.
488  *     As many bytes as possible (space available in the queue) are enqueued.
489  *
490  * Results:
491  *     VMCI_ERROR_QUEUEPAIR_NOSPACE if no space was available to enqueue data.
492  *     VMCI_ERROR_INVALID_SIZE, if any queue pointer is outside the queue
493  *     (as defined by the queue size).
494  *     VMCI_ERROR_INVALID_ARGS, if an error occured when accessing the buffer.
495  *     VMCI_ERROR_QUEUEPAIR_NOTATTACHED, if the queue pair pages aren't
496  *     available.
497  *     Otherwise, the number of bytes written to the queue is returned.
498  *
499  * Side effects:
500  *     Updates the tail pointer of the produce queue.
501  *
502  *------------------------------------------------------------------------------
503  */
504 
505 static ssize_t
506 enqueue(struct vmci_queue *produce_q, struct vmci_queue *consume_q,
507     const uint64_t produce_q_size, const void *buf, size_t buf_size,
508     int buf_type, vmci_memcpy_to_queue_func memcpy_to_queue, bool can_block)
509 {
510 	ssize_t result;
511 	size_t written;
512 	int64_t free_space;
513 	uint64_t tail;
514 
515 	ASSERT((produce_q != NULL) && (consume_q != NULL));
516 
517 	free_space = vmci_queue_header_free_space(produce_q->q_header,
518 	    consume_q->q_header,
519 	    produce_q_size);
520 	if (free_space == 0)
521 		return (VMCI_ERROR_QUEUEPAIR_NOSPACE);
522 
523 	if (free_space < VMCI_SUCCESS)
524 		return ((ssize_t)free_space);
525 
526 	written = (size_t)(free_space > buf_size ? buf_size : free_space);
527 	tail = vmci_queue_header_producer_tail(produce_q->q_header);
528 	if (LIKELY(tail + written < produce_q_size))
529 		result = memcpy_to_queue(produce_q, tail, buf, 0, written,
530 		    buf_type, can_block);
531 	else {
532 		/* Tail pointer wraps around. */
533 
534 		const size_t tmp = (size_t)(produce_q_size - tail);
535 
536 		result = memcpy_to_queue(produce_q, tail, buf, 0, tmp, buf_type,
537 		    can_block);
538 		if (result >= VMCI_SUCCESS)
539 			result = memcpy_to_queue(produce_q, 0, buf, tmp,
540 			    written - tmp, buf_type, can_block);
541 	}
542 
543 	if (result < VMCI_SUCCESS)
544 		return (result);
545 
546 	result = vmci_queue_add_producer_tail(produce_q, written,
547 	    produce_q_size);
548 	if (result < VMCI_SUCCESS)
549 		return (result);
550 	return (written);
551 }
552 
553 /*
554  *------------------------------------------------------------------------------
555  *
556  * dequeue --
557  *
558  *     Dequeues data (if available) from the given consume queue. Writes data
559  *     to the user provided buffer using the provided function.
560  *
561  * Results:
562  *     VMCI_ERROR_QUEUEPAIR_NODATA if no data was available to dequeue.
563  *     VMCI_ERROR_INVALID_SIZE, if any queue pointer is outside the queue
564  *     (as defined by the queue size).
565  *     VMCI_ERROR_INVALID_ARGS, if an error occured when accessing the buffer.
566  *     VMCI_ERROR_NOT_FOUND, if the vmm_world registered with the queue pair
567  *     cannot be found.
568  *     Otherwise the number of bytes dequeued is returned.
569  *
570  * Side effects:
571  *     Updates the head pointer of the consume queue.
572  *
573  *------------------------------------------------------------------------------
574  */
575 
576 static ssize_t
577 dequeue(struct vmci_queue *produce_q,
578     struct vmci_queue *consume_q, const uint64_t consume_q_size, void *buf,
579     size_t buf_size, int buf_type,
580     vmci_memcpy_from_queue_func memcpy_from_queue, bool update_consumer,
581     bool can_block)
582 {
583 	ssize_t result;
584 	size_t read;
585 	int64_t buf_ready;
586 	uint64_t head;
587 
588 	ASSERT((produce_q != NULL) && (consume_q != NULL));
589 
590 	buf_ready = vmci_queue_header_buf_ready(consume_q->q_header,
591 	    produce_q->q_header, consume_q_size);
592 	if (buf_ready == 0)
593 		return (VMCI_ERROR_QUEUEPAIR_NODATA);
594 	if (buf_ready < VMCI_SUCCESS)
595 		return ((ssize_t)buf_ready);
596 
597 	read = (size_t)(buf_ready > buf_size ? buf_size : buf_ready);
598 	head = vmci_queue_header_consumer_head(produce_q->q_header);
599 	if (LIKELY(head + read < consume_q_size))
600 		result = memcpy_from_queue(buf, 0, consume_q, head, read,
601 		    buf_type, can_block);
602 	else {
603 		/* Head pointer wraps around. */
604 
605 		const size_t tmp = (size_t)(consume_q_size - head);
606 
607 		result = memcpy_from_queue(buf, 0, consume_q, head, tmp,
608 		    buf_type, can_block);
609 		if (result >= VMCI_SUCCESS)
610 			result = memcpy_from_queue(buf, tmp, consume_q, 0,
611 			    read - tmp, buf_type, can_block);
612 	}
613 
614 	if (result < VMCI_SUCCESS)
615 		return (result);
616 
617 	if (update_consumer) {
618 		result = vmci_queue_add_consumer_head(produce_q, read,
619 		    consume_q_size);
620 		if (result < VMCI_SUCCESS)
621 			return (result);
622 	}
623 
624 	return (read);
625 }
626 
627 /*
628  *------------------------------------------------------------------------------
629  *
630  * vmci_qpair_enqueue --
631  *
632  *     This is the client interface for enqueueing data into the queue.
633  *
634  * Results:
635  *     Err, if < 0.
636  *     Number of bytes enqueued if >= 0.
637  *
638  * Side effects:
639  *     None.
640  *
641  *------------------------------------------------------------------------------
642  */
643 
644 ssize_t
645 vmci_qpair_enqueue(struct vmci_qpair *qpair, const void *buf, size_t buf_size,
646     int buf_type)
647 {
648 	ssize_t result;
649 
650 	if (!qpair || !buf)
651 		return (VMCI_ERROR_INVALID_ARGS);
652 
653 	result = enqueue(qpair->produce_q, qpair->consume_q,
654 	    qpair->produce_q_size, buf, buf_size, buf_type,
655 	    qpair->flags & VMCI_QPFLAG_LOCAL?
656 	    vmci_memcpy_to_queue_local : vmci_memcpy_to_queue,
657 	    !(qpair->flags & VMCI_QPFLAG_NONBLOCK));
658 
659 	return (result);
660 }
661 
662 /*
663  *------------------------------------------------------------------------------
664  *
665  * vmci_qpair_dequeue --
666  *
667  *     This is the client interface for dequeueing data from the queue.
668  *
669  * Results:
670  *     Err, if < 0.
671  *     Number of bytes dequeued if >= 0.
672  *
673  * Side effects:
674  *     None.
675  *
676  *------------------------------------------------------------------------------
677  */
678 
679 ssize_t
680 vmci_qpair_dequeue(struct vmci_qpair *qpair, void *buf, size_t buf_size,
681     int buf_type)
682 {
683 	ssize_t result;
684 
685 	if (!qpair || !buf)
686 		return (VMCI_ERROR_INVALID_ARGS);
687 
688 	result = dequeue(qpair->produce_q, qpair->consume_q,
689 	    qpair->consume_q_size, buf, buf_size, buf_type,
690 	    qpair->flags & VMCI_QPFLAG_LOCAL?
691 	    vmci_memcpy_from_queue_local : vmci_memcpy_from_queue, true,
692 	    !(qpair->flags & VMCI_QPFLAG_NONBLOCK));
693 
694 	return (result);
695 }
696 
697 /*
698  *------------------------------------------------------------------------------
699  *
700  * vmci_qpair_peek --
701  *
702  *     This is the client interface for peeking into a queue.  (I.e., copy
703  *     data from the queue without updating the head pointer.)
704  *
705  * Results:
706  *     Err, if < 0.
707  *     Number of bytes peeked, if >= 0.
708  *
709  * Side effects:
710  *     None.
711  *
712  *------------------------------------------------------------------------------
713  */
714 
715 ssize_t
716 vmci_qpair_peek(struct vmci_qpair *qpair, void *buf, size_t buf_size,
717     int buf_type)
718 {
719 	ssize_t result;
720 
721 	if (!qpair || !buf)
722 		return (VMCI_ERROR_INVALID_ARGS);
723 
724 	result = dequeue(qpair->produce_q, qpair->consume_q,
725 	    qpair->consume_q_size, buf, buf_size, buf_type,
726 	    qpair->flags & VMCI_QPFLAG_LOCAL?
727 	    vmci_memcpy_from_queue_local : vmci_memcpy_from_queue, false,
728 	    !(qpair->flags & VMCI_QPFLAG_NONBLOCK));
729 
730 	return (result);
731 }
732 
733 /*
734  *------------------------------------------------------------------------------
735  *
736  * vmci_qpair_enquev --
737  *
738  *     This is the client interface for enqueueing data into the queue.
739  *
740  * Results:
741  *     Err, if < 0.
742  *     Number of bytes enqueued if >= 0.
743  *
744  * Side effects:
745  *     None.
746  *
747  *------------------------------------------------------------------------------
748  */
749 
750 ssize_t
751 vmci_qpair_enquev(struct vmci_qpair *qpair, void *iov, size_t iov_size,
752     int buf_type)
753 {
754 	ssize_t result;
755 
756 	if (!qpair || !iov)
757 		return (VMCI_ERROR_INVALID_ARGS);
758 
759 	result = enqueue(qpair->produce_q, qpair->consume_q,
760 	    qpair->produce_q_size, iov, iov_size, buf_type,
761 	    qpair->flags & VMCI_QPFLAG_LOCAL?
762 	    vmci_memcpy_to_queue_v_local : vmci_memcpy_to_queue_v,
763 	    !(qpair->flags & VMCI_QPFLAG_NONBLOCK));
764 
765 	return (result);
766 }
767 
768 /*
769  *------------------------------------------------------------------------------
770  *
771  * vmci_qpair_dequev --
772  *
773  *     This is the client interface for dequeueing data from the queue.
774  *
775  * Results:
776  *     Err, if < 0.
777  *     Number of bytes dequeued if >= 0.
778  *
779  * Side effects:
780  *     None.
781  *
782  *------------------------------------------------------------------------------
783  */
784 
785 ssize_t
786 vmci_qpair_dequev(struct vmci_qpair *qpair, void *iov, size_t iov_size,
787     int buf_type)
788 {
789 	ssize_t result;
790 
791 	if (!qpair || !iov)
792 		return (VMCI_ERROR_INVALID_ARGS);
793 
794 	result = dequeue(qpair->produce_q, qpair->consume_q,
795 	    qpair->consume_q_size, iov, iov_size, buf_type,
796 	    qpair->flags & VMCI_QPFLAG_LOCAL?
797 	    vmci_memcpy_from_queue_v_local : vmci_memcpy_from_queue_v, true,
798 	    !(qpair->flags & VMCI_QPFLAG_NONBLOCK));
799 
800 	return (result);
801 }
802 
803 /*
804  *------------------------------------------------------------------------------
805  *
806  * vmci_qpair_peekv --
807  *
808  *     This is the client interface for peeking into a queue.  (I.e., copy
809  *     data from the queue without updating the head pointer.)
810  *
811  * Results:
812  *     Err, if < 0.
813  *     Number of bytes peeked, if >= 0.
814  *
815  * Side effects:
816  *     None.
817  *
818  *------------------------------------------------------------------------------
819  */
820 
821 ssize_t
822 vmci_qpair_peekv(struct vmci_qpair *qpair, void *iov, size_t iov_size,
823     int buf_type)
824 {
825 	ssize_t result;
826 
827 	if (!qpair || !iov)
828 		return (VMCI_ERROR_INVALID_ARGS);
829 
830 	result = dequeue(qpair->produce_q, qpair->consume_q,
831 	    qpair->consume_q_size, iov, iov_size, buf_type,
832 	    qpair->flags & VMCI_QPFLAG_LOCAL?
833 	    vmci_memcpy_from_queue_v_local : vmci_memcpy_from_queue_v, false,
834 	    !(qpair->flags & VMCI_QPFLAG_NONBLOCK));
835 
836 	return (result);
837 }
838