163a93856SMark Peek /*-
23eeb7511SMark Peek * Copyright (c) 2018 VMware, Inc.
363a93856SMark Peek *
48c302b2eSMark Peek * SPDX-License-Identifier: (BSD-2-Clause OR GPL-2.0)
563a93856SMark Peek */
663a93856SMark Peek
763a93856SMark Peek /* This file implements defines and helper functions. */
863a93856SMark Peek
963a93856SMark Peek #include <sys/cdefs.h>
1063a93856SMark Peek #include <sys/malloc.h>
1163a93856SMark Peek #include <sys/proc.h>
1263a93856SMark Peek #include <sys/uio.h>
1363a93856SMark Peek
1463a93856SMark Peek #include <machine/bus.h>
1563a93856SMark Peek
1663a93856SMark Peek #include "vmci.h"
1763a93856SMark Peek #include "vmci_defs.h"
1863a93856SMark Peek #include "vmci_kernel_defs.h"
1963a93856SMark Peek #include "vmci_kernel_if.h"
2063a93856SMark Peek #include "vmci_queue.h"
2163a93856SMark Peek
2263a93856SMark Peek struct vmci_queue_kernel_if {
2363a93856SMark Peek size_t num_pages; /* Num pages incl. header. */
2463a93856SMark Peek struct vmci_dma_alloc *dmas; /* For dma alloc. */
2563a93856SMark Peek };
2663a93856SMark Peek
2763a93856SMark Peek /*
2863a93856SMark Peek *------------------------------------------------------------------------------
2963a93856SMark Peek *
3063a93856SMark Peek * vmci_init_lock
3163a93856SMark Peek *
3263a93856SMark Peek * Initializes the lock. Must be called before use.
3363a93856SMark Peek *
3463a93856SMark Peek * Results:
3563a93856SMark Peek * Always VMCI_SUCCESS.
3663a93856SMark Peek *
3763a93856SMark Peek * Side effects:
3863a93856SMark Peek * Thread can block.
3963a93856SMark Peek *
4063a93856SMark Peek *------------------------------------------------------------------------------
4163a93856SMark Peek */
4263a93856SMark Peek
4363a93856SMark Peek int
vmci_init_lock(vmci_lock * lock,char * name)4463a93856SMark Peek vmci_init_lock(vmci_lock *lock, char *name)
4563a93856SMark Peek {
4663a93856SMark Peek
4763a93856SMark Peek mtx_init(lock, name, NULL, MTX_DEF | MTX_NOWITNESS);
4863a93856SMark Peek return (VMCI_SUCCESS);
4963a93856SMark Peek }
5063a93856SMark Peek
5163a93856SMark Peek /*
5263a93856SMark Peek *------------------------------------------------------------------------------
5363a93856SMark Peek *
5463a93856SMark Peek * vmci_cleanup_lock
5563a93856SMark Peek *
5663a93856SMark Peek * Cleanup the lock. Must be called before deallocating lock.
5763a93856SMark Peek *
5863a93856SMark Peek * Results:
5963a93856SMark Peek * None
6063a93856SMark Peek *
6163a93856SMark Peek * Side effects:
6263a93856SMark Peek * Deletes kernel lock state
6363a93856SMark Peek *
6463a93856SMark Peek *------------------------------------------------------------------------------
6563a93856SMark Peek */
6663a93856SMark Peek
6763a93856SMark Peek void
vmci_cleanup_lock(vmci_lock * lock)6863a93856SMark Peek vmci_cleanup_lock(vmci_lock *lock)
6963a93856SMark Peek {
7063a93856SMark Peek
71c75c1d2dSMark Johnston if (mtx_initialized(lock))
7263a93856SMark Peek mtx_destroy(lock);
7363a93856SMark Peek }
7463a93856SMark Peek
7563a93856SMark Peek /*
7663a93856SMark Peek *------------------------------------------------------------------------------
7763a93856SMark Peek *
7863a93856SMark Peek * vmci_grab_lock
7963a93856SMark Peek *
8063a93856SMark Peek * Grabs the given lock.
8163a93856SMark Peek *
8263a93856SMark Peek * Results:
8363a93856SMark Peek * None
8463a93856SMark Peek *
8563a93856SMark Peek * Side effects:
8663a93856SMark Peek * Thread can block.
8763a93856SMark Peek *
8863a93856SMark Peek *------------------------------------------------------------------------------
8963a93856SMark Peek */
9063a93856SMark Peek
9163a93856SMark Peek void
vmci_grab_lock(vmci_lock * lock)9263a93856SMark Peek vmci_grab_lock(vmci_lock *lock)
9363a93856SMark Peek {
9463a93856SMark Peek
9563a93856SMark Peek mtx_lock(lock);
9663a93856SMark Peek }
9763a93856SMark Peek
9863a93856SMark Peek /*
9963a93856SMark Peek *------------------------------------------------------------------------------
10063a93856SMark Peek *
10163a93856SMark Peek * vmci_release_lock
10263a93856SMark Peek *
10363a93856SMark Peek * Releases the given lock.
10463a93856SMark Peek *
10563a93856SMark Peek * Results:
10663a93856SMark Peek * None
10763a93856SMark Peek *
10863a93856SMark Peek * Side effects:
10963a93856SMark Peek * A thread blocked on this lock may wake up.
11063a93856SMark Peek *
11163a93856SMark Peek *------------------------------------------------------------------------------
11263a93856SMark Peek */
11363a93856SMark Peek
11463a93856SMark Peek void
vmci_release_lock(vmci_lock * lock)11563a93856SMark Peek vmci_release_lock(vmci_lock *lock)
11663a93856SMark Peek {
11763a93856SMark Peek
11863a93856SMark Peek mtx_unlock(lock);
11963a93856SMark Peek }
12063a93856SMark Peek
12163a93856SMark Peek /*
12263a93856SMark Peek *------------------------------------------------------------------------------
12363a93856SMark Peek *
12463a93856SMark Peek * vmci_grab_lock_bh
12563a93856SMark Peek *
12663a93856SMark Peek * Grabs the given lock.
12763a93856SMark Peek *
12863a93856SMark Peek * Results:
12963a93856SMark Peek * None
13063a93856SMark Peek *
13163a93856SMark Peek * Side effects:
13263a93856SMark Peek * None.
13363a93856SMark Peek *
13463a93856SMark Peek *------------------------------------------------------------------------------
13563a93856SMark Peek */
13663a93856SMark Peek
13763a93856SMark Peek void
vmci_grab_lock_bh(vmci_lock * lock)13863a93856SMark Peek vmci_grab_lock_bh(vmci_lock *lock)
13963a93856SMark Peek {
14063a93856SMark Peek
14163a93856SMark Peek mtx_lock(lock);
14263a93856SMark Peek }
14363a93856SMark Peek
14463a93856SMark Peek /*
14563a93856SMark Peek *------------------------------------------------------------------------------
14663a93856SMark Peek *
14763a93856SMark Peek * vmci_release_lock_bh
14863a93856SMark Peek *
14963a93856SMark Peek * Releases the given lock.
15063a93856SMark Peek *
15163a93856SMark Peek * Results:
15263a93856SMark Peek * None
15363a93856SMark Peek *
15463a93856SMark Peek * Side effects:
15563a93856SMark Peek * None.
15663a93856SMark Peek *
15763a93856SMark Peek *------------------------------------------------------------------------------
15863a93856SMark Peek */
15963a93856SMark Peek
16063a93856SMark Peek void
vmci_release_lock_bh(vmci_lock * lock)16163a93856SMark Peek vmci_release_lock_bh(vmci_lock *lock)
16263a93856SMark Peek {
16363a93856SMark Peek
16463a93856SMark Peek mtx_unlock(lock);
16563a93856SMark Peek }
16663a93856SMark Peek
16763a93856SMark Peek /*
16863a93856SMark Peek *------------------------------------------------------------------------------
16963a93856SMark Peek *
1700f14bcbeSMark Peek * vmci_initialized_lock
1710f14bcbeSMark Peek *
1720f14bcbeSMark Peek * Returns whether a lock has been initialized.
1730f14bcbeSMark Peek *
1740f14bcbeSMark Peek * Results:
1750f14bcbeSMark Peek * Return 1 if initialized or 0 if unininitialized.
1760f14bcbeSMark Peek *
1770f14bcbeSMark Peek * Side effects:
1780f14bcbeSMark Peek * None
1790f14bcbeSMark Peek *
1800f14bcbeSMark Peek *------------------------------------------------------------------------------
1810f14bcbeSMark Peek */
1820f14bcbeSMark Peek
1830f14bcbeSMark Peek int
vmci_initialized_lock(vmci_lock * lock)1840f14bcbeSMark Peek vmci_initialized_lock(vmci_lock *lock)
1850f14bcbeSMark Peek {
1860f14bcbeSMark Peek
1870f14bcbeSMark Peek return mtx_initialized(lock);
1880f14bcbeSMark Peek }
1890f14bcbeSMark Peek
1900f14bcbeSMark Peek /*
1910f14bcbeSMark Peek *------------------------------------------------------------------------------
1920f14bcbeSMark Peek *
19363a93856SMark Peek * vmci_alloc_kernel_mem
19463a93856SMark Peek *
19563a93856SMark Peek * Allocate physically contiguous memory for the VMCI driver.
19663a93856SMark Peek *
19763a93856SMark Peek * Results:
19863a93856SMark Peek * The address allocated or NULL on error.
19963a93856SMark Peek *
20063a93856SMark Peek *
20163a93856SMark Peek * Side effects:
20263a93856SMark Peek * Memory may be allocated.
20363a93856SMark Peek *
20463a93856SMark Peek *------------------------------------------------------------------------------
20563a93856SMark Peek */
20663a93856SMark Peek
20763a93856SMark Peek void *
vmci_alloc_kernel_mem(size_t size,int flags)20863a93856SMark Peek vmci_alloc_kernel_mem(size_t size, int flags)
20963a93856SMark Peek {
21063a93856SMark Peek void *ptr;
21163a93856SMark Peek
21263a93856SMark Peek if ((flags & VMCI_MEMORY_ATOMIC) != 0)
21363a93856SMark Peek ptr = contigmalloc(size, M_DEVBUF, M_NOWAIT, 0, 0xFFFFFFFF,
21463a93856SMark Peek 8, 1024 * 1024);
21563a93856SMark Peek else
21663a93856SMark Peek ptr = contigmalloc(size, M_DEVBUF, M_WAITOK, 0, 0xFFFFFFFF,
21763a93856SMark Peek 8, 1024 * 1024);
21863a93856SMark Peek
21963a93856SMark Peek return (ptr);
22063a93856SMark Peek }
22163a93856SMark Peek
22263a93856SMark Peek /*
22363a93856SMark Peek *------------------------------------------------------------------------------
22463a93856SMark Peek *
22563a93856SMark Peek * vmci_free_kernel_mem
22663a93856SMark Peek *
22763a93856SMark Peek * Free kernel memory allocated for the VMCI driver.
22863a93856SMark Peek *
22963a93856SMark Peek * Results:
23063a93856SMark Peek * None.
23163a93856SMark Peek *
23263a93856SMark Peek * Side effects:
23363a93856SMark Peek * Memory is freed.
23463a93856SMark Peek *
23563a93856SMark Peek *------------------------------------------------------------------------------
23663a93856SMark Peek */
23763a93856SMark Peek
23863a93856SMark Peek void
vmci_free_kernel_mem(void * ptr,size_t size)23963a93856SMark Peek vmci_free_kernel_mem(void *ptr, size_t size)
24063a93856SMark Peek {
24163a93856SMark Peek
24263a93856SMark Peek contigfree(ptr, size, M_DEVBUF);
24363a93856SMark Peek }
24463a93856SMark Peek
24563a93856SMark Peek /*
24663a93856SMark Peek *------------------------------------------------------------------------------
24763a93856SMark Peek *
24863a93856SMark Peek * vmci_can_schedule_delayed_work --
24963a93856SMark Peek *
25063a93856SMark Peek * Checks to see if the given platform supports delayed work callbacks.
25163a93856SMark Peek *
25263a93856SMark Peek * Results:
25363a93856SMark Peek * true if it does. false otherwise.
25463a93856SMark Peek *
25563a93856SMark Peek * Side effects:
25663a93856SMark Peek * None.
25763a93856SMark Peek *
25863a93856SMark Peek *------------------------------------------------------------------------------
25963a93856SMark Peek */
26063a93856SMark Peek
26163a93856SMark Peek bool
vmci_can_schedule_delayed_work(void)26263a93856SMark Peek vmci_can_schedule_delayed_work(void)
26363a93856SMark Peek {
26463a93856SMark Peek
26563a93856SMark Peek return (true);
26663a93856SMark Peek }
26763a93856SMark Peek
26863a93856SMark Peek /*
26963a93856SMark Peek *------------------------------------------------------------------------------
27063a93856SMark Peek *
27163a93856SMark Peek * vmci_schedule_delayed_work --
27263a93856SMark Peek *
27363a93856SMark Peek * Schedule the specified callback.
27463a93856SMark Peek *
27563a93856SMark Peek * Results:
27663a93856SMark Peek * Zero on success, error code otherwise.
27763a93856SMark Peek *
27863a93856SMark Peek * Side effects:
27963a93856SMark Peek * None.
28063a93856SMark Peek *
28163a93856SMark Peek *------------------------------------------------------------------------------
28263a93856SMark Peek */
28363a93856SMark Peek
28463a93856SMark Peek int
vmci_schedule_delayed_work(vmci_work_fn * work_fn,void * data)28563a93856SMark Peek vmci_schedule_delayed_work(vmci_work_fn *work_fn, void *data)
28663a93856SMark Peek {
28763a93856SMark Peek
28863a93856SMark Peek return (vmci_schedule_delayed_work_fn(work_fn, data));
28963a93856SMark Peek }
29063a93856SMark Peek
29163a93856SMark Peek /*
29263a93856SMark Peek *------------------------------------------------------------------------------
29363a93856SMark Peek *
29463a93856SMark Peek * vmci_create_event --
29563a93856SMark Peek *
29663a93856SMark Peek * Results:
29763a93856SMark Peek * None.
29863a93856SMark Peek *
29963a93856SMark Peek * Side effects:
30063a93856SMark Peek * None.
30163a93856SMark Peek *
30263a93856SMark Peek *------------------------------------------------------------------------------
30363a93856SMark Peek */
30463a93856SMark Peek
30563a93856SMark Peek void
vmci_create_event(vmci_event * event)30663a93856SMark Peek vmci_create_event(vmci_event *event)
30763a93856SMark Peek {
30863a93856SMark Peek
30963a93856SMark Peek sema_init(event, 0, "vmci_event");
31063a93856SMark Peek }
31163a93856SMark Peek
31263a93856SMark Peek /*
31363a93856SMark Peek *------------------------------------------------------------------------------
31463a93856SMark Peek *
31563a93856SMark Peek * vmci_destroy_event --
31663a93856SMark Peek *
31763a93856SMark Peek * Results:
31863a93856SMark Peek * None.
31963a93856SMark Peek *
32063a93856SMark Peek * Side effects:
32163a93856SMark Peek * None.
32263a93856SMark Peek *
32363a93856SMark Peek *------------------------------------------------------------------------------
32463a93856SMark Peek */
32563a93856SMark Peek
32663a93856SMark Peek void
vmci_destroy_event(vmci_event * event)32763a93856SMark Peek vmci_destroy_event(vmci_event *event)
32863a93856SMark Peek {
32963a93856SMark Peek
33063a93856SMark Peek if (mtx_owned(&event->sema_mtx))
33163a93856SMark Peek sema_destroy(event);
33263a93856SMark Peek }
33363a93856SMark Peek
33463a93856SMark Peek /*
33563a93856SMark Peek *------------------------------------------------------------------------------
33663a93856SMark Peek *
33763a93856SMark Peek * vmci_signal_event --
33863a93856SMark Peek *
33963a93856SMark Peek * Results:
34063a93856SMark Peek * None.
34163a93856SMark Peek *
34263a93856SMark Peek * Side effects:
34363a93856SMark Peek * None.
34463a93856SMark Peek *
34563a93856SMark Peek *------------------------------------------------------------------------------
34663a93856SMark Peek */
34763a93856SMark Peek
34863a93856SMark Peek void
vmci_signal_event(vmci_event * event)34963a93856SMark Peek vmci_signal_event(vmci_event *event)
35063a93856SMark Peek {
35163a93856SMark Peek
35263a93856SMark Peek sema_post(event);
35363a93856SMark Peek }
35463a93856SMark Peek
35563a93856SMark Peek /*
35663a93856SMark Peek *------------------------------------------------------------------------------
35763a93856SMark Peek *
35863a93856SMark Peek * vmci_wait_on_event --
35963a93856SMark Peek *
36063a93856SMark Peek * Results:
36163a93856SMark Peek * None.
36263a93856SMark Peek *
36363a93856SMark Peek * Side effects:
36463a93856SMark Peek * None.
36563a93856SMark Peek *
36663a93856SMark Peek *------------------------------------------------------------------------------
36763a93856SMark Peek */
36863a93856SMark Peek
36963a93856SMark Peek void
vmci_wait_on_event(vmci_event * event,vmci_event_release_cb release_cb,void * client_data)37063a93856SMark Peek vmci_wait_on_event(vmci_event *event, vmci_event_release_cb release_cb,
37163a93856SMark Peek void *client_data)
37263a93856SMark Peek {
37363a93856SMark Peek
37463a93856SMark Peek release_cb(client_data);
37563a93856SMark Peek sema_wait(event);
37663a93856SMark Peek }
37763a93856SMark Peek
37863a93856SMark Peek /*
37963a93856SMark Peek *------------------------------------------------------------------------------
38063a93856SMark Peek *
38163a93856SMark Peek * vmci_mutex_init --
38263a93856SMark Peek *
38363a93856SMark Peek * Initializes the mutex. Must be called before use.
38463a93856SMark Peek *
38563a93856SMark Peek * Results:
38663a93856SMark Peek * Success.
38763a93856SMark Peek *
38863a93856SMark Peek * Side effects:
38963a93856SMark Peek * None.
39063a93856SMark Peek *
39163a93856SMark Peek *------------------------------------------------------------------------------
39263a93856SMark Peek */
39363a93856SMark Peek
39463a93856SMark Peek int
vmci_mutex_init(vmci_mutex * mutex,char * name)39563a93856SMark Peek vmci_mutex_init(vmci_mutex *mutex, char *name)
39663a93856SMark Peek {
39763a93856SMark Peek
39863a93856SMark Peek mtx_init(mutex, name, NULL, MTX_DEF | MTX_NOWITNESS);
39963a93856SMark Peek return (VMCI_SUCCESS);
40063a93856SMark Peek }
40163a93856SMark Peek
40263a93856SMark Peek /*
40363a93856SMark Peek *------------------------------------------------------------------------------
40463a93856SMark Peek *
40563a93856SMark Peek * vmci_mutex_destroy --
40663a93856SMark Peek *
40763a93856SMark Peek * Destroys the mutex.
40863a93856SMark Peek *
40963a93856SMark Peek * Results:
41063a93856SMark Peek * None.
41163a93856SMark Peek *
41263a93856SMark Peek * Side effects:
41363a93856SMark Peek * None.
41463a93856SMark Peek *
41563a93856SMark Peek *------------------------------------------------------------------------------
41663a93856SMark Peek */
41763a93856SMark Peek
41863a93856SMark Peek void
vmci_mutex_destroy(vmci_mutex * mutex)41963a93856SMark Peek vmci_mutex_destroy(vmci_mutex *mutex)
42063a93856SMark Peek {
42163a93856SMark Peek
42263a93856SMark Peek mtx_destroy(mutex);
42363a93856SMark Peek }
42463a93856SMark Peek
42563a93856SMark Peek /*
42663a93856SMark Peek *------------------------------------------------------------------------------
42763a93856SMark Peek *
42863a93856SMark Peek * vmci_mutex_acquire --
42963a93856SMark Peek *
43063a93856SMark Peek * Acquires the mutex.
43163a93856SMark Peek *
43263a93856SMark Peek * Results:
43363a93856SMark Peek * None.
43463a93856SMark Peek *
43563a93856SMark Peek * Side effects:
43663a93856SMark Peek * Thread may block.
43763a93856SMark Peek *
43863a93856SMark Peek *------------------------------------------------------------------------------
43963a93856SMark Peek */
44063a93856SMark Peek
44163a93856SMark Peek void
vmci_mutex_acquire(vmci_mutex * mutex)44263a93856SMark Peek vmci_mutex_acquire(vmci_mutex *mutex)
44363a93856SMark Peek {
44463a93856SMark Peek
44563a93856SMark Peek mtx_lock(mutex);
44663a93856SMark Peek }
44763a93856SMark Peek
44863a93856SMark Peek /*
44963a93856SMark Peek *------------------------------------------------------------------------------
45063a93856SMark Peek *
45163a93856SMark Peek * vmci_mutex_release --
45263a93856SMark Peek *
45363a93856SMark Peek * Releases the mutex.
45463a93856SMark Peek *
45563a93856SMark Peek * Results:
45663a93856SMark Peek * None.
45763a93856SMark Peek *
45863a93856SMark Peek * Side effects:
45963a93856SMark Peek * May wake up the thread blocking on this mutex.
46063a93856SMark Peek *
46163a93856SMark Peek *------------------------------------------------------------------------------
46263a93856SMark Peek */
46363a93856SMark Peek
46463a93856SMark Peek void
vmci_mutex_release(vmci_mutex * mutex)46563a93856SMark Peek vmci_mutex_release(vmci_mutex *mutex)
46663a93856SMark Peek {
46763a93856SMark Peek
46863a93856SMark Peek mtx_unlock(mutex);
46963a93856SMark Peek }
47063a93856SMark Peek
47163a93856SMark Peek /*
47263a93856SMark Peek *------------------------------------------------------------------------------
47363a93856SMark Peek *
4740f14bcbeSMark Peek * vmci_mutex_initialized
4750f14bcbeSMark Peek *
4760f14bcbeSMark Peek * Returns whether a mutex has been initialized.
4770f14bcbeSMark Peek *
4780f14bcbeSMark Peek * Results:
4790f14bcbeSMark Peek * Return 1 if initialized or 0 if unininitialized.
4800f14bcbeSMark Peek *
4810f14bcbeSMark Peek * Side effects:
4820f14bcbeSMark Peek * None
4830f14bcbeSMark Peek *
4840f14bcbeSMark Peek *------------------------------------------------------------------------------
4850f14bcbeSMark Peek */
4860f14bcbeSMark Peek
4870f14bcbeSMark Peek int
vmci_mutex_initialized(vmci_mutex * mutex)4880f14bcbeSMark Peek vmci_mutex_initialized(vmci_mutex *mutex)
4890f14bcbeSMark Peek {
4900f14bcbeSMark Peek
4910f14bcbeSMark Peek return mtx_initialized(mutex);
4920f14bcbeSMark Peek }
4930f14bcbeSMark Peek /*
4940f14bcbeSMark Peek *------------------------------------------------------------------------------
4950f14bcbeSMark Peek *
49663a93856SMark Peek * vmci_alloc_queue --
49763a93856SMark Peek *
49863a93856SMark Peek * Allocates kernel queue pages of specified size with IOMMU mappings, plus
49963a93856SMark Peek * space for the queue structure/kernel interface and the queue header.
50063a93856SMark Peek *
50163a93856SMark Peek * Results:
50263a93856SMark Peek * Pointer to the queue on success, NULL otherwise.
50363a93856SMark Peek *
50463a93856SMark Peek * Side effects:
50563a93856SMark Peek * Memory is allocated.
50663a93856SMark Peek *
50763a93856SMark Peek *------------------------------------------------------------------------------
50863a93856SMark Peek */
50963a93856SMark Peek
51063a93856SMark Peek void *
vmci_alloc_queue(uint64_t size,uint32_t flags)51163a93856SMark Peek vmci_alloc_queue(uint64_t size, uint32_t flags)
51263a93856SMark Peek {
51363a93856SMark Peek struct vmci_queue *queue;
51463a93856SMark Peek size_t i;
51563a93856SMark Peek const size_t num_pages = CEILING(size, PAGE_SIZE) + 1;
51663a93856SMark Peek const size_t dmas_size = num_pages * sizeof(struct vmci_dma_alloc);
51763a93856SMark Peek const size_t queue_size =
51863a93856SMark Peek sizeof(*queue) + sizeof(*(queue->kernel_if)) + dmas_size;
51963a93856SMark Peek
52063a93856SMark Peek /* Size should be enforced by vmci_qpair_alloc(), double-check here. */
52163a93856SMark Peek if (size > VMCI_MAX_GUEST_QP_MEMORY) {
52263a93856SMark Peek ASSERT(false);
52363a93856SMark Peek return (NULL);
52463a93856SMark Peek }
52563a93856SMark Peek
52663a93856SMark Peek queue = malloc(queue_size, M_DEVBUF, M_NOWAIT);
52763a93856SMark Peek if (!queue)
52863a93856SMark Peek return (NULL);
52963a93856SMark Peek
53063a93856SMark Peek queue->q_header = NULL;
53163a93856SMark Peek queue->saved_header = NULL;
53263a93856SMark Peek queue->kernel_if = (struct vmci_queue_kernel_if *)(queue + 1);
53363a93856SMark Peek queue->kernel_if->num_pages = num_pages;
53463a93856SMark Peek queue->kernel_if->dmas = (struct vmci_dma_alloc *)(queue->kernel_if +
53563a93856SMark Peek 1);
53663a93856SMark Peek for (i = 0; i < num_pages; i++) {
53763a93856SMark Peek vmci_dma_malloc(PAGE_SIZE, 1, &queue->kernel_if->dmas[i]);
53863a93856SMark Peek if (!queue->kernel_if->dmas[i].dma_vaddr) {
53963a93856SMark Peek /* Size excl. the header. */
54063a93856SMark Peek vmci_free_queue(queue, i * PAGE_SIZE);
54163a93856SMark Peek return (NULL);
54263a93856SMark Peek }
54363a93856SMark Peek }
54463a93856SMark Peek
54563a93856SMark Peek /* Queue header is the first page. */
54663a93856SMark Peek queue->q_header = (void *)queue->kernel_if->dmas[0].dma_vaddr;
54763a93856SMark Peek
54863a93856SMark Peek return ((void *)queue);
54963a93856SMark Peek }
55063a93856SMark Peek
55163a93856SMark Peek /*
55263a93856SMark Peek *------------------------------------------------------------------------------
55363a93856SMark Peek *
55463a93856SMark Peek * vmci_free_queue --
55563a93856SMark Peek *
55663a93856SMark Peek * Frees kernel VA space for a given queue and its queue header, and frees
55763a93856SMark Peek * physical data pages.
55863a93856SMark Peek *
55963a93856SMark Peek * Results:
56063a93856SMark Peek * None.
56163a93856SMark Peek *
56263a93856SMark Peek * Side effects:
56363a93856SMark Peek * Memory is freed.
56463a93856SMark Peek *
56563a93856SMark Peek *------------------------------------------------------------------------------
56663a93856SMark Peek */
56763a93856SMark Peek
56863a93856SMark Peek void
vmci_free_queue(void * q,uint64_t size)56963a93856SMark Peek vmci_free_queue(void *q, uint64_t size)
57063a93856SMark Peek {
57163a93856SMark Peek struct vmci_queue *queue = q;
57263a93856SMark Peek
57363a93856SMark Peek if (queue) {
57463a93856SMark Peek const size_t num_pages = CEILING(size, PAGE_SIZE) + 1;
57563a93856SMark Peek uint64_t i;
57663a93856SMark Peek
57763a93856SMark Peek /* Given size doesn't include header, so add in a page here. */
57863a93856SMark Peek for (i = 0; i < num_pages; i++)
57963a93856SMark Peek vmci_dma_free(&queue->kernel_if->dmas[i]);
58063a93856SMark Peek free(queue, M_DEVBUF);
58163a93856SMark Peek }
58263a93856SMark Peek }
58363a93856SMark Peek
58463a93856SMark Peek /*
58563a93856SMark Peek *------------------------------------------------------------------------------
58663a93856SMark Peek *
58763a93856SMark Peek * vmci_alloc_ppn_set --
58863a93856SMark Peek *
58963a93856SMark Peek * Allocates two list of PPNs --- one for the pages in the produce queue,
59063a93856SMark Peek * and the other for the pages in the consume queue. Intializes the list of
59163a93856SMark Peek * PPNs with the page frame numbers of the KVA for the two queues (and the
59263a93856SMark Peek * queue headers).
59363a93856SMark Peek *
59463a93856SMark Peek * Results:
59563a93856SMark Peek * Success or failure.
59663a93856SMark Peek *
59763a93856SMark Peek * Side effects:
59863a93856SMark Peek * Memory may be allocated.
59963a93856SMark Peek *
60063a93856SMark Peek *-----------------------------------------------------------------------------
60163a93856SMark Peek */
60263a93856SMark Peek
60363a93856SMark Peek int
vmci_alloc_ppn_set(void * prod_q,uint64_t num_produce_pages,void * cons_q,uint64_t num_consume_pages,struct ppn_set * ppn_set)60463a93856SMark Peek vmci_alloc_ppn_set(void *prod_q, uint64_t num_produce_pages, void *cons_q,
60563a93856SMark Peek uint64_t num_consume_pages, struct ppn_set *ppn_set)
60663a93856SMark Peek {
60763a93856SMark Peek struct vmci_queue *consume_q = cons_q;
60863a93856SMark Peek struct vmci_queue *produce_q = prod_q;
60963a93856SMark Peek vmci_ppn_list consume_ppns;
61063a93856SMark Peek vmci_ppn_list produce_ppns;
61163a93856SMark Peek uint64_t i;
61263a93856SMark Peek
61363a93856SMark Peek if (!produce_q || !num_produce_pages || !consume_q ||
61463a93856SMark Peek !num_consume_pages || !ppn_set)
61563a93856SMark Peek return (VMCI_ERROR_INVALID_ARGS);
61663a93856SMark Peek
61763a93856SMark Peek if (ppn_set->initialized)
61863a93856SMark Peek return (VMCI_ERROR_ALREADY_EXISTS);
61963a93856SMark Peek
62063a93856SMark Peek produce_ppns =
62163a93856SMark Peek vmci_alloc_kernel_mem(num_produce_pages * sizeof(*produce_ppns),
62263a93856SMark Peek VMCI_MEMORY_NORMAL);
62363a93856SMark Peek if (!produce_ppns)
62463a93856SMark Peek return (VMCI_ERROR_NO_MEM);
62563a93856SMark Peek
62663a93856SMark Peek consume_ppns =
62763a93856SMark Peek vmci_alloc_kernel_mem(num_consume_pages * sizeof(*consume_ppns),
62863a93856SMark Peek VMCI_MEMORY_NORMAL);
62963a93856SMark Peek if (!consume_ppns) {
63063a93856SMark Peek vmci_free_kernel_mem(produce_ppns,
63163a93856SMark Peek num_produce_pages * sizeof(*produce_ppns));
63263a93856SMark Peek return (VMCI_ERROR_NO_MEM);
63363a93856SMark Peek }
63463a93856SMark Peek
63563a93856SMark Peek for (i = 0; i < num_produce_pages; i++) {
63663a93856SMark Peek unsigned long pfn;
63763a93856SMark Peek
63863a93856SMark Peek produce_ppns[i] =
63963a93856SMark Peek pfn = produce_q->kernel_if->dmas[i].dma_paddr >> PAGE_SHIFT;
64063a93856SMark Peek
64163a93856SMark Peek /*
64263a93856SMark Peek * Fail allocation if PFN isn't supported by hypervisor.
64363a93856SMark Peek */
64463a93856SMark Peek
64563a93856SMark Peek if (sizeof(pfn) >
64663a93856SMark Peek sizeof(*produce_ppns) && pfn != produce_ppns[i])
64763a93856SMark Peek goto ppn_error;
64863a93856SMark Peek }
64963a93856SMark Peek for (i = 0; i < num_consume_pages; i++) {
65063a93856SMark Peek unsigned long pfn;
65163a93856SMark Peek
65263a93856SMark Peek consume_ppns[i] =
65363a93856SMark Peek pfn = consume_q->kernel_if->dmas[i].dma_paddr >> PAGE_SHIFT;
65463a93856SMark Peek
65563a93856SMark Peek /*
65663a93856SMark Peek * Fail allocation if PFN isn't supported by hypervisor.
65763a93856SMark Peek */
65863a93856SMark Peek
65963a93856SMark Peek if (sizeof(pfn) >
66063a93856SMark Peek sizeof(*consume_ppns) && pfn != consume_ppns[i])
66163a93856SMark Peek goto ppn_error;
66263a93856SMark Peek }
66363a93856SMark Peek
66463a93856SMark Peek ppn_set->num_produce_pages = num_produce_pages;
66563a93856SMark Peek ppn_set->num_consume_pages = num_consume_pages;
66663a93856SMark Peek ppn_set->produce_ppns = produce_ppns;
66763a93856SMark Peek ppn_set->consume_ppns = consume_ppns;
66863a93856SMark Peek ppn_set->initialized = true;
66963a93856SMark Peek return (VMCI_SUCCESS);
67063a93856SMark Peek
67163a93856SMark Peek ppn_error:
67263a93856SMark Peek vmci_free_kernel_mem(produce_ppns, num_produce_pages *
67363a93856SMark Peek sizeof(*produce_ppns));
67463a93856SMark Peek vmci_free_kernel_mem(consume_ppns, num_consume_pages *
67563a93856SMark Peek sizeof(*consume_ppns));
67663a93856SMark Peek return (VMCI_ERROR_INVALID_ARGS);
67763a93856SMark Peek }
67863a93856SMark Peek
67963a93856SMark Peek /*
68063a93856SMark Peek *------------------------------------------------------------------------------
68163a93856SMark Peek *
68263a93856SMark Peek * vmci_free_ppn_set --
68363a93856SMark Peek *
68463a93856SMark Peek * Frees the two list of PPNs for a queue pair.
68563a93856SMark Peek *
68663a93856SMark Peek * Results:
68763a93856SMark Peek * None.
68863a93856SMark Peek *
68963a93856SMark Peek * Side effects:
69063a93856SMark Peek * None.
69163a93856SMark Peek *
69263a93856SMark Peek *------------------------------------------------------------------------------
69363a93856SMark Peek */
69463a93856SMark Peek
69563a93856SMark Peek void
vmci_free_ppn_set(struct ppn_set * ppn_set)69663a93856SMark Peek vmci_free_ppn_set(struct ppn_set *ppn_set)
69763a93856SMark Peek {
69863a93856SMark Peek
69963a93856SMark Peek ASSERT(ppn_set);
70063a93856SMark Peek if (ppn_set->initialized) {
70163a93856SMark Peek /* Do not call these functions on NULL inputs. */
70263a93856SMark Peek ASSERT(ppn_set->produce_ppns && ppn_set->consume_ppns);
70363a93856SMark Peek vmci_free_kernel_mem(ppn_set->produce_ppns,
70463a93856SMark Peek ppn_set->num_produce_pages *
70563a93856SMark Peek sizeof(*ppn_set->produce_ppns));
70663a93856SMark Peek vmci_free_kernel_mem(ppn_set->consume_ppns,
70763a93856SMark Peek ppn_set->num_consume_pages *
70863a93856SMark Peek sizeof(*ppn_set->consume_ppns));
70963a93856SMark Peek }
71063a93856SMark Peek memset(ppn_set, 0, sizeof(*ppn_set));
71163a93856SMark Peek }
71263a93856SMark Peek
71363a93856SMark Peek /*
71463a93856SMark Peek *------------------------------------------------------------------------------
71563a93856SMark Peek *
71663a93856SMark Peek * vmci_populate_ppn_list --
71763a93856SMark Peek *
71863a93856SMark Peek * Populates the list of PPNs in the hypercall structure with the PPNS
71963a93856SMark Peek * of the produce queue and the consume queue.
72063a93856SMark Peek *
72163a93856SMark Peek * Results:
72263a93856SMark Peek * VMCI_SUCCESS.
72363a93856SMark Peek *
72463a93856SMark Peek * Side effects:
72563a93856SMark Peek * None.
72663a93856SMark Peek *
72763a93856SMark Peek *------------------------------------------------------------------------------
72863a93856SMark Peek */
72963a93856SMark Peek
73063a93856SMark Peek int
vmci_populate_ppn_list(uint8_t * call_buf,const struct ppn_set * ppn_set)73163a93856SMark Peek vmci_populate_ppn_list(uint8_t *call_buf, const struct ppn_set *ppn_set)
73263a93856SMark Peek {
73363a93856SMark Peek
73463a93856SMark Peek ASSERT(call_buf && ppn_set && ppn_set->initialized);
73563a93856SMark Peek memcpy(call_buf, ppn_set->produce_ppns,
73663a93856SMark Peek ppn_set->num_produce_pages * sizeof(*ppn_set->produce_ppns));
73763a93856SMark Peek memcpy(call_buf + ppn_set->num_produce_pages *
73863a93856SMark Peek sizeof(*ppn_set->produce_ppns), ppn_set->consume_ppns,
73963a93856SMark Peek ppn_set->num_consume_pages * sizeof(*ppn_set->consume_ppns));
74063a93856SMark Peek
74163a93856SMark Peek return (VMCI_SUCCESS);
74263a93856SMark Peek }
74363a93856SMark Peek
74463a93856SMark Peek /*
74563a93856SMark Peek *------------------------------------------------------------------------------
74663a93856SMark Peek *
74763a93856SMark Peek * vmci_memcpy_{to,from}iovec --
74863a93856SMark Peek *
74963a93856SMark Peek * These helper routines will copy the specified bytes to/from memory that's
75063a93856SMark Peek * specified as a struct iovec. The routines can not verify the correctness
75163a93856SMark Peek * of the struct iovec's contents.
75263a93856SMark Peek *
75363a93856SMark Peek * Results:
75463a93856SMark Peek * None.
75563a93856SMark Peek *
75663a93856SMark Peek * Side effects:
75763a93856SMark Peek * None.
75863a93856SMark Peek *
75963a93856SMark Peek *------------------------------------------------------------------------------
76063a93856SMark Peek */
76163a93856SMark Peek
76263a93856SMark Peek static inline void
vmci_memcpy_toiovec(struct iovec * iov,uint8_t * src,size_t len)76363a93856SMark Peek vmci_memcpy_toiovec(struct iovec *iov, uint8_t *src, size_t len)
76463a93856SMark Peek {
76563a93856SMark Peek
76663a93856SMark Peek while (len > 0) {
76763a93856SMark Peek if (iov->iov_len) {
76863a93856SMark Peek size_t to_copy = MIN(iov->iov_len, len);
76963a93856SMark Peek memcpy(iov->iov_base, src, to_copy);
77063a93856SMark Peek src += to_copy;
77163a93856SMark Peek len -= to_copy;
77263a93856SMark Peek iov->iov_base = (void *)((uintptr_t) iov->iov_base +
77363a93856SMark Peek to_copy);
77463a93856SMark Peek iov->iov_len -= to_copy;
77563a93856SMark Peek }
77663a93856SMark Peek iov++;
77763a93856SMark Peek }
77863a93856SMark Peek }
77963a93856SMark Peek
78063a93856SMark Peek static inline void
vmci_memcpy_fromiovec(uint8_t * dst,struct iovec * iov,size_t len)78163a93856SMark Peek vmci_memcpy_fromiovec(uint8_t *dst, struct iovec *iov, size_t len)
78263a93856SMark Peek {
78363a93856SMark Peek
78463a93856SMark Peek while (len > 0) {
78563a93856SMark Peek if (iov->iov_len) {
78663a93856SMark Peek size_t to_copy = MIN(iov->iov_len, len);
78763a93856SMark Peek memcpy(dst, iov->iov_base, to_copy);
78863a93856SMark Peek dst += to_copy;
78963a93856SMark Peek len -= to_copy;
79063a93856SMark Peek iov->iov_base = (void *)((uintptr_t) iov->iov_base +
79163a93856SMark Peek to_copy);
79263a93856SMark Peek iov->iov_len -= to_copy;
79363a93856SMark Peek }
79463a93856SMark Peek iov++;
79563a93856SMark Peek }
79663a93856SMark Peek }
79763a93856SMark Peek
79863a93856SMark Peek /*
79963a93856SMark Peek *------------------------------------------------------------------------------
80063a93856SMark Peek *
80163a93856SMark Peek * __vmci_memcpy_to_queue --
80263a93856SMark Peek *
80363a93856SMark Peek * Copies from a given buffer or iovector to a VMCI Queue. Assumes that
80463a93856SMark Peek * offset + size does not wrap around in the queue.
80563a93856SMark Peek *
80663a93856SMark Peek * Results:
80763a93856SMark Peek * Zero on success, negative error code on failure.
80863a93856SMark Peek *
80963a93856SMark Peek * Side effects:
81063a93856SMark Peek * None.
81163a93856SMark Peek *
81263a93856SMark Peek *------------------------------------------------------------------------------
81363a93856SMark Peek */
81463a93856SMark Peek
81563a93856SMark Peek #pragma GCC diagnostic ignored "-Wcast-qual"
81663a93856SMark Peek static int
__vmci_memcpy_to_queue(struct vmci_queue * queue,uint64_t queue_offset,const void * src,size_t size,bool is_iovec)81763a93856SMark Peek __vmci_memcpy_to_queue(struct vmci_queue *queue, uint64_t queue_offset,
81863a93856SMark Peek const void *src, size_t size, bool is_iovec)
81963a93856SMark Peek {
82063a93856SMark Peek struct vmci_queue_kernel_if *kernel_if = queue->kernel_if;
82163a93856SMark Peek size_t bytes_copied = 0;
82263a93856SMark Peek
82363a93856SMark Peek while (bytes_copied < size) {
82463a93856SMark Peek const uint64_t page_index =
82563a93856SMark Peek (queue_offset + bytes_copied) / PAGE_SIZE;
82663a93856SMark Peek const size_t page_offset =
82763a93856SMark Peek (queue_offset + bytes_copied) & (PAGE_SIZE - 1);
82863a93856SMark Peek void *va;
82963a93856SMark Peek size_t to_copy;
83063a93856SMark Peek
83163a93856SMark Peek /* Skip header. */
83263a93856SMark Peek va = (void *)kernel_if->dmas[page_index + 1].dma_vaddr;
83363a93856SMark Peek
83463a93856SMark Peek ASSERT(va);
83563a93856SMark Peek /*
83663a93856SMark Peek * Fill up the page if we have enough payload, or else
83763a93856SMark Peek * copy the remaining bytes.
83863a93856SMark Peek */
83963a93856SMark Peek to_copy = MIN(PAGE_SIZE - page_offset, size - bytes_copied);
84063a93856SMark Peek
84163a93856SMark Peek if (is_iovec) {
84263a93856SMark Peek struct iovec *iov = (struct iovec *)src;
84363a93856SMark Peek
84463a93856SMark Peek /* The iovec will track bytes_copied internally. */
84563a93856SMark Peek vmci_memcpy_fromiovec((uint8_t *)va + page_offset,
84663a93856SMark Peek iov, to_copy);
84763a93856SMark Peek } else
84863a93856SMark Peek memcpy((uint8_t *)va + page_offset,
84963a93856SMark Peek (uint8_t *)src + bytes_copied, to_copy);
85063a93856SMark Peek bytes_copied += to_copy;
85163a93856SMark Peek }
85263a93856SMark Peek
85363a93856SMark Peek return (VMCI_SUCCESS);
85463a93856SMark Peek }
85563a93856SMark Peek
85663a93856SMark Peek /*
85763a93856SMark Peek *------------------------------------------------------------------------------
85863a93856SMark Peek *
85963a93856SMark Peek * __vmci_memcpy_from_queue --
86063a93856SMark Peek *
86163a93856SMark Peek * Copies to a given buffer or iovector from a VMCI Queue. Assumes that
86263a93856SMark Peek * offset + size does not wrap around in the queue.
86363a93856SMark Peek *
86463a93856SMark Peek * Results:
86563a93856SMark Peek * Zero on success, negative error code on failure.
86663a93856SMark Peek *
86763a93856SMark Peek * Side effects:
86863a93856SMark Peek * None.
86963a93856SMark Peek *
87063a93856SMark Peek *------------------------------------------------------------------------------
87163a93856SMark Peek */
87263a93856SMark Peek
87363a93856SMark Peek static int
__vmci_memcpy_from_queue(void * dest,const struct vmci_queue * queue,uint64_t queue_offset,size_t size,bool is_iovec)87463a93856SMark Peek __vmci_memcpy_from_queue(void *dest, const struct vmci_queue *queue,
87563a93856SMark Peek uint64_t queue_offset, size_t size, bool is_iovec)
87663a93856SMark Peek {
87763a93856SMark Peek struct vmci_queue_kernel_if *kernel_if = queue->kernel_if;
87863a93856SMark Peek size_t bytes_copied = 0;
87963a93856SMark Peek
88063a93856SMark Peek while (bytes_copied < size) {
88163a93856SMark Peek const uint64_t page_index =
88263a93856SMark Peek (queue_offset + bytes_copied) / PAGE_SIZE;
88363a93856SMark Peek const size_t page_offset =
88463a93856SMark Peek (queue_offset + bytes_copied) & (PAGE_SIZE - 1);
88563a93856SMark Peek void *va;
88663a93856SMark Peek size_t to_copy;
88763a93856SMark Peek
88863a93856SMark Peek /* Skip header. */
88963a93856SMark Peek va = (void *)kernel_if->dmas[page_index + 1].dma_vaddr;
89063a93856SMark Peek
89163a93856SMark Peek ASSERT(va);
89263a93856SMark Peek /*
89363a93856SMark Peek * Fill up the page if we have enough payload, or else
89463a93856SMark Peek * copy the remaining bytes.
89563a93856SMark Peek */
89663a93856SMark Peek to_copy = MIN(PAGE_SIZE - page_offset, size - bytes_copied);
89763a93856SMark Peek
89863a93856SMark Peek if (is_iovec) {
89963a93856SMark Peek struct iovec *iov = (struct iovec *)dest;
90063a93856SMark Peek
90163a93856SMark Peek /* The iovec will track bytesCopied internally. */
90263a93856SMark Peek vmci_memcpy_toiovec(iov, (uint8_t *)va +
90363a93856SMark Peek page_offset, to_copy);
90463a93856SMark Peek } else
90563a93856SMark Peek memcpy((uint8_t *)dest + bytes_copied,
90663a93856SMark Peek (uint8_t *)va + page_offset, to_copy);
90763a93856SMark Peek
90863a93856SMark Peek bytes_copied += to_copy;
90963a93856SMark Peek }
91063a93856SMark Peek
91163a93856SMark Peek return (VMCI_SUCCESS);
91263a93856SMark Peek }
91363a93856SMark Peek
91463a93856SMark Peek /*
91563a93856SMark Peek *------------------------------------------------------------------------------
91663a93856SMark Peek *
91763a93856SMark Peek * vmci_memcpy_to_queue --
91863a93856SMark Peek *
91963a93856SMark Peek * Copies from a given buffer to a VMCI Queue.
92063a93856SMark Peek *
92163a93856SMark Peek * Results:
92263a93856SMark Peek * Zero on success, negative error code on failure.
92363a93856SMark Peek *
92463a93856SMark Peek * Side effects:
92563a93856SMark Peek * None.
92663a93856SMark Peek *
92763a93856SMark Peek *------------------------------------------------------------------------------
92863a93856SMark Peek */
92963a93856SMark Peek
93063a93856SMark Peek int
vmci_memcpy_to_queue(struct vmci_queue * queue,uint64_t queue_offset,const void * src,size_t src_offset,size_t size,int buf_type,bool can_block)93163a93856SMark Peek vmci_memcpy_to_queue(struct vmci_queue *queue, uint64_t queue_offset,
93263a93856SMark Peek const void *src, size_t src_offset, size_t size, int buf_type,
93363a93856SMark Peek bool can_block)
93463a93856SMark Peek {
93563a93856SMark Peek
93663a93856SMark Peek ASSERT(can_block);
93763a93856SMark Peek
93863a93856SMark Peek return (__vmci_memcpy_to_queue(queue, queue_offset,
93963a93856SMark Peek (uint8_t *)src + src_offset, size, false));
94063a93856SMark Peek }
94163a93856SMark Peek
94263a93856SMark Peek /*
94363a93856SMark Peek *------------------------------------------------------------------------------
94463a93856SMark Peek *
94563a93856SMark Peek * vmci_memcpy_from_queue --
94663a93856SMark Peek *
94763a93856SMark Peek * Copies to a given buffer from a VMCI Queue.
94863a93856SMark Peek *
94963a93856SMark Peek * Results:
95063a93856SMark Peek * Zero on success, negative error code on failure.
95163a93856SMark Peek *
95263a93856SMark Peek * Side effects:
95363a93856SMark Peek * None.
95463a93856SMark Peek *
95563a93856SMark Peek *------------------------------------------------------------------------------
95663a93856SMark Peek */
95763a93856SMark Peek
95863a93856SMark Peek int
vmci_memcpy_from_queue(void * dest,size_t dest_offset,const struct vmci_queue * queue,uint64_t queue_offset,size_t size,int buf_type,bool can_block)95963a93856SMark Peek vmci_memcpy_from_queue(void *dest, size_t dest_offset,
96063a93856SMark Peek const struct vmci_queue *queue, uint64_t queue_offset, size_t size,
96163a93856SMark Peek int buf_type, bool can_block)
96263a93856SMark Peek {
96363a93856SMark Peek
96463a93856SMark Peek ASSERT(can_block);
96563a93856SMark Peek
96663a93856SMark Peek return (__vmci_memcpy_from_queue((uint8_t *)dest + dest_offset,
96763a93856SMark Peek queue, queue_offset, size, false));
96863a93856SMark Peek }
96963a93856SMark Peek
97063a93856SMark Peek /*
97163a93856SMark Peek *------------------------------------------------------------------------------
97263a93856SMark Peek *
97363a93856SMark Peek * vmci_memcpy_to_queue_local --
97463a93856SMark Peek *
97563a93856SMark Peek * Copies from a given buffer to a local VMCI queue. This is the
97663a93856SMark Peek * same as a regular copy.
97763a93856SMark Peek *
97863a93856SMark Peek * Results:
97963a93856SMark Peek * Zero on success, negative error code on failure.
98063a93856SMark Peek *
98163a93856SMark Peek * Side effects:
98263a93856SMark Peek * None.
98363a93856SMark Peek *
98463a93856SMark Peek *------------------------------------------------------------------------------
98563a93856SMark Peek */
98663a93856SMark Peek
98763a93856SMark Peek int
vmci_memcpy_to_queue_local(struct vmci_queue * queue,uint64_t queue_offset,const void * src,size_t src_offset,size_t size,int buf_type,bool can_block)98863a93856SMark Peek vmci_memcpy_to_queue_local(struct vmci_queue *queue, uint64_t queue_offset,
98963a93856SMark Peek const void *src, size_t src_offset, size_t size, int buf_type,
99063a93856SMark Peek bool can_block)
99163a93856SMark Peek {
99263a93856SMark Peek
99363a93856SMark Peek ASSERT(can_block);
99463a93856SMark Peek
99563a93856SMark Peek return (__vmci_memcpy_to_queue(queue, queue_offset,
99663a93856SMark Peek (uint8_t *)src + src_offset, size, false));
99763a93856SMark Peek }
99863a93856SMark Peek
99963a93856SMark Peek /*
100063a93856SMark Peek *------------------------------------------------------------------------------
100163a93856SMark Peek *
100263a93856SMark Peek * vmci_memcpy_from_queue_local --
100363a93856SMark Peek *
100463a93856SMark Peek * Copies to a given buffer from a VMCI Queue.
100563a93856SMark Peek *
100663a93856SMark Peek * Results:
100763a93856SMark Peek * Zero on success, negative error code on failure.
100863a93856SMark Peek *
100963a93856SMark Peek * Side effects:
101063a93856SMark Peek * None.
101163a93856SMark Peek *
101263a93856SMark Peek *------------------------------------------------------------------------------
101363a93856SMark Peek */
101463a93856SMark Peek
101563a93856SMark Peek int
vmci_memcpy_from_queue_local(void * dest,size_t dest_offset,const struct vmci_queue * queue,uint64_t queue_offset,size_t size,int buf_type,bool can_block)101663a93856SMark Peek vmci_memcpy_from_queue_local(void *dest, size_t dest_offset,
101763a93856SMark Peek const struct vmci_queue *queue, uint64_t queue_offset, size_t size,
101863a93856SMark Peek int buf_type, bool can_block)
101963a93856SMark Peek {
102063a93856SMark Peek
102163a93856SMark Peek ASSERT(can_block);
102263a93856SMark Peek
102363a93856SMark Peek return (__vmci_memcpy_from_queue((uint8_t *)dest + dest_offset,
102463a93856SMark Peek queue, queue_offset, size, false));
102563a93856SMark Peek }
102663a93856SMark Peek
102763a93856SMark Peek /*------------------------------------------------------------------------------
102863a93856SMark Peek *
102963a93856SMark Peek * vmci_memcpy_to_queue_v --
103063a93856SMark Peek *
103163a93856SMark Peek * Copies from a given iovec from a VMCI Queue.
103263a93856SMark Peek *
103363a93856SMark Peek * Results:
103463a93856SMark Peek * Zero on success, negative error code on failure.
103563a93856SMark Peek *
103663a93856SMark Peek * Side effects:
103763a93856SMark Peek * None.
103863a93856SMark Peek *
103963a93856SMark Peek *------------------------------------------------------------------------------
104063a93856SMark Peek */
104163a93856SMark Peek
104263a93856SMark Peek int
vmci_memcpy_to_queue_v(struct vmci_queue * queue,uint64_t queue_offset,const void * src,size_t src_offset,size_t size,int buf_type,bool can_block)104363a93856SMark Peek vmci_memcpy_to_queue_v(struct vmci_queue *queue, uint64_t queue_offset,
104463a93856SMark Peek const void *src, size_t src_offset, size_t size, int buf_type,
104563a93856SMark Peek bool can_block)
104663a93856SMark Peek {
104763a93856SMark Peek
104863a93856SMark Peek ASSERT(can_block);
104963a93856SMark Peek
105063a93856SMark Peek /*
105163a93856SMark Peek * We ignore src_offset because src is really a struct iovec * and will
105263a93856SMark Peek * maintain offset internally.
105363a93856SMark Peek */
105463a93856SMark Peek return (__vmci_memcpy_to_queue(queue, queue_offset, src, size,
105563a93856SMark Peek true));
105663a93856SMark Peek }
105763a93856SMark Peek
105863a93856SMark Peek /*
105963a93856SMark Peek *------------------------------------------------------------------------------
106063a93856SMark Peek *
106163a93856SMark Peek * vmci_memcpy_from_queue_v --
106263a93856SMark Peek *
106363a93856SMark Peek * Copies to a given iovec from a VMCI Queue.
106463a93856SMark Peek *
106563a93856SMark Peek * Results:
106663a93856SMark Peek * Zero on success, negative error code on failure.
106763a93856SMark Peek *
106863a93856SMark Peek * Side effects:
106963a93856SMark Peek * None.
107063a93856SMark Peek *
107163a93856SMark Peek *------------------------------------------------------------------------------
107263a93856SMark Peek */
107363a93856SMark Peek
107463a93856SMark Peek int
vmci_memcpy_from_queue_v(void * dest,size_t dest_offset,const struct vmci_queue * queue,uint64_t queue_offset,size_t size,int buf_type,bool can_block)107563a93856SMark Peek vmci_memcpy_from_queue_v(void *dest, size_t dest_offset,
107663a93856SMark Peek const struct vmci_queue *queue, uint64_t queue_offset, size_t size,
107763a93856SMark Peek int buf_type, bool can_block)
107863a93856SMark Peek {
107963a93856SMark Peek
108063a93856SMark Peek ASSERT(can_block);
108163a93856SMark Peek
108263a93856SMark Peek /*
108363a93856SMark Peek * We ignore dest_offset because dest is really a struct iovec * and
108463a93856SMark Peek * will maintain offset internally.
108563a93856SMark Peek */
108663a93856SMark Peek return (__vmci_memcpy_from_queue(dest, queue, queue_offset, size,
108763a93856SMark Peek true));
108863a93856SMark Peek }
108963a93856SMark Peek
109063a93856SMark Peek /*
109163a93856SMark Peek *------------------------------------------------------------------------------
109263a93856SMark Peek *
109363a93856SMark Peek * vmci_read_port_bytes --
109463a93856SMark Peek *
109563a93856SMark Peek * Copy memory from an I/O port to kernel memory.
109663a93856SMark Peek *
109763a93856SMark Peek * Results:
109863a93856SMark Peek * No results.
109963a93856SMark Peek *
110063a93856SMark Peek * Side effects:
110163a93856SMark Peek * None.
110263a93856SMark Peek *
110363a93856SMark Peek *------------------------------------------------------------------------------
110463a93856SMark Peek */
110563a93856SMark Peek
110663a93856SMark Peek void
vmci_read_port_bytes(vmci_io_handle handle,vmci_io_port port,uint8_t * buffer,size_t buffer_length)110763a93856SMark Peek vmci_read_port_bytes(vmci_io_handle handle, vmci_io_port port, uint8_t *buffer,
110863a93856SMark Peek size_t buffer_length)
110963a93856SMark Peek {
111063a93856SMark Peek
111163a93856SMark Peek insb(port, buffer, buffer_length);
111263a93856SMark Peek }
1113