1 /*-
2 * Copyright (c) 2018 VMware, Inc.
3 *
4 * SPDX-License-Identifier: (BSD-2-Clause OR GPL-2.0)
5 */
6
7 /* This file implements the VMCI doorbell API. */
8
9 #include <sys/types.h>
10
11 #include "vmci_doorbell.h"
12 #include "vmci_driver.h"
13 #include "vmci_kernel_api.h"
14 #include "vmci_kernel_defs.h"
15 #include "vmci_resource.h"
16 #include "vmci_utils.h"
17
18 #define LGPFX "vmci_doorbell: "
19
20 #define VMCI_DOORBELL_INDEX_TABLE_SIZE 64
21 #define VMCI_DOORBELL_HASH(_idx) \
22 vmci_hash_id((_idx), VMCI_DOORBELL_INDEX_TABLE_SIZE)
23
24 /* Describes a doorbell notification handle allocated by the host. */
25 struct vmci_doorbell_entry {
26 struct vmci_resource resource;
27 uint32_t idx;
28 vmci_list_item(vmci_doorbell_entry) idx_list_item;
29 vmci_privilege_flags priv_flags;
30 bool is_doorbell;
31 bool run_delayed;
32 vmci_callback notify_cb;
33 void *client_data;
34 vmci_event destroy_event;
35 volatile int active;
36 };
37
38 struct vmci_doorbell_index_table {
39 vmci_lock lock;
40 vmci_list(vmci_doorbell_entry) entries[VMCI_DOORBELL_INDEX_TABLE_SIZE];
41 };
42
43 /* The VMCI index table keeps track of currently registered doorbells. */
44 static struct vmci_doorbell_index_table vmci_doorbell_it;
45
46 /*
47 * The max_notify_idx is one larger than the currently known bitmap index in
48 * use, and is used to determine how much of the bitmap needs to be scanned.
49 */
50 static uint32_t max_notify_idx;
51
52 /*
53 * The notify_idx_count is used for determining whether there are free entries
54 * within the bitmap (if notify_idx_count + 1 < max_notify_idx).
55 */
56 static uint32_t notify_idx_count;
57
58 /*
59 * The last_notify_idx_reserved is used to track the last index handed out - in
60 * the case where multiple handles share a notification index, we hand out
61 * indexes round robin based on last_notify_idx_reserved.
62 */
63 static uint32_t last_notify_idx_reserved;
64
65 /* This is a one entry cache used to by the index allocation. */
66 static uint32_t last_notify_idx_released = PAGE_SIZE;
67
68 static void vmci_doorbell_free_cb(void *client_data);
69 static int vmci_doorbell_release_cb(void *client_data);
70 static void vmci_doorbell_delayed_dispatch_cb(void *data);
71
72 /*
73 *------------------------------------------------------------------------------
74 *
75 * vmci_doorbell_init --
76 *
77 * General init code.
78 *
79 * Result:
80 * VMCI_SUCCESS on success, lock allocation error otherwise.
81 *
82 * Side effects:
83 * None.
84 *
85 *------------------------------------------------------------------------------
86 */
87
88 int
vmci_doorbell_init(void)89 vmci_doorbell_init(void)
90 {
91 uint32_t bucket;
92
93 for (bucket = 0; bucket < ARRAYSIZE(vmci_doorbell_it.entries);
94 ++bucket)
95 vmci_list_init(&vmci_doorbell_it.entries[bucket]);
96
97 return (vmci_init_lock(&vmci_doorbell_it.lock,
98 "VMCI Doorbell index table lock"));
99 }
100
101 /*
102 *------------------------------------------------------------------------------
103 *
104 * vmci_doorbell_exit --
105 *
106 * General exit code.
107 *
108 * Result:
109 * None.
110 *
111 * Side effects:
112 * None.
113 *
114 *------------------------------------------------------------------------------
115 */
116
117 void
vmci_doorbell_exit(void)118 vmci_doorbell_exit(void)
119 {
120
121 vmci_cleanup_lock(&vmci_doorbell_it.lock);
122 }
123
124 /*
125 *------------------------------------------------------------------------------
126 *
127 * vmci_doorbell_free_cb --
128 *
129 * Callback to free doorbell entry structure when resource is no longer used,
130 * i.e. the reference count reached 0. The entry is freed in
131 * vmci_doorbell_destroy(), which is waiting on the signal that gets fired
132 * here.
133 *
134 * Result:
135 * None.
136 *
137 * Side effects:
138 * Signals VMCI event.
139 *
140 *------------------------------------------------------------------------------
141 */
142
143 static void
vmci_doorbell_free_cb(void * client_data)144 vmci_doorbell_free_cb(void *client_data)
145 {
146 struct vmci_doorbell_entry *entry;
147
148 entry = (struct vmci_doorbell_entry *)client_data;
149 ASSERT(entry);
150 vmci_signal_event(&entry->destroy_event);
151 }
152
153 /*
154 *------------------------------------------------------------------------------
155 *
156 * vmci_doorbell_release_cb --
157 *
158 * Callback to release the resource reference. It is called by the
159 * vmci_wait_on_event function before it blocks.
160 *
161 * Result:
162 * Always 0.
163 *
164 * Side effects:
165 * None.
166 *
167 *------------------------------------------------------------------------------
168 */
169
170 static int
vmci_doorbell_release_cb(void * client_data)171 vmci_doorbell_release_cb(void *client_data)
172 {
173 struct vmci_doorbell_entry *entry;
174
175 entry = (struct vmci_doorbell_entry *)client_data;
176 ASSERT(entry);
177 vmci_resource_release(&entry->resource);
178 return (0);
179 }
180
181 /*
182 *------------------------------------------------------------------------------
183 *
184 * vmci_doorbell_get_priv_flags --
185 *
186 * Utility function that retrieves the privilege flags associated with a
187 * given doorbell handle. For guest endpoints, the privileges are determined
188 * by the context ID, but for host endpoints privileges are associated with
189 * the complete handle. Hypervisor endpoints are not yet supported.
190 *
191 * Result:
192 * VMCI_SUCCESS on success,
193 * VMCI_ERROR_NOT_FOUND if handle isn't found,
194 * VMCI_ERROR_INVALID_ARGS if handle is invalid.
195 *
196 * Side effects:
197 * None.
198 *
199 *------------------------------------------------------------------------------
200 */
201
202 int
vmci_doorbell_get_priv_flags(struct vmci_handle handle,vmci_privilege_flags * priv_flags)203 vmci_doorbell_get_priv_flags(struct vmci_handle handle,
204 vmci_privilege_flags *priv_flags)
205 {
206
207 if (priv_flags == NULL || handle.context == VMCI_INVALID_ID)
208 return (VMCI_ERROR_INVALID_ARGS);
209
210 if (handle.context == VMCI_HOST_CONTEXT_ID) {
211 struct vmci_doorbell_entry *entry;
212 struct vmci_resource *resource;
213
214 resource = vmci_resource_get(handle,
215 VMCI_RESOURCE_TYPE_DOORBELL);
216 if (resource == NULL)
217 return (VMCI_ERROR_NOT_FOUND);
218 entry = RESOURCE_CONTAINER(
219 resource, struct vmci_doorbell_entry, resource);
220 *priv_flags = entry->priv_flags;
221 vmci_resource_release(resource);
222 } else if (handle.context == VMCI_HYPERVISOR_CONTEXT_ID) {
223 /* Hypervisor endpoints for notifications are not supported. */
224 return (VMCI_ERROR_INVALID_ARGS);
225 } else
226 *priv_flags = VMCI_NO_PRIVILEGE_FLAGS;
227
228 return (VMCI_SUCCESS);
229 }
230
231 /*
232 *------------------------------------------------------------------------------
233 *
234 * vmci_doorbell_index_table_find --
235 *
236 * Find doorbell entry by bitmap index.
237 *
238 * Results:
239 * Entry if found, NULL if not.
240 *
241 * Side effects:
242 * None.
243 *
244 *------------------------------------------------------------------------------
245 */
246
247 static struct vmci_doorbell_entry *
vmci_doorbell_index_table_find(uint32_t idx)248 vmci_doorbell_index_table_find(uint32_t idx)
249 {
250 struct vmci_doorbell_entry *iter;
251 uint32_t bucket;
252
253 bucket = VMCI_DOORBELL_HASH(idx);
254
255 vmci_list_scan(iter, &vmci_doorbell_it.entries[bucket], idx_list_item) {
256 if (idx == iter->idx)
257 return (iter);
258 }
259
260 return (NULL);
261 }
262
263 /*
264 *------------------------------------------------------------------------------
265 *
266 * vmci_doorbell_index_table_add --
267 *
268 * Add the given entry to the index table. This will hold() the entry's
269 * resource so that the entry is not deleted before it is removed from the
270 * table.
271 *
272 * Results:
273 * None.
274 *
275 * Side effects:
276 * None.
277 *
278 *------------------------------------------------------------------------------
279 */
280
281 static void
vmci_doorbell_index_table_add(struct vmci_doorbell_entry * entry)282 vmci_doorbell_index_table_add(struct vmci_doorbell_entry *entry)
283 {
284 uint32_t bucket;
285 uint32_t new_notify_idx;
286
287 ASSERT(entry);
288
289 vmci_resource_hold(&entry->resource);
290
291 vmci_grab_lock_bh(&vmci_doorbell_it.lock);
292
293 /*
294 * Below we try to allocate an index in the notification bitmap with
295 * "not too much" sharing between resources. If we use less that the
296 * full bitmap, we either add to the end if there are no unused flags
297 * within the currently used area, or we search for unused ones. If we
298 * use the full bitmap, we allocate the index round robin.
299 */
300
301 if (max_notify_idx < PAGE_SIZE || notify_idx_count < PAGE_SIZE) {
302 if (last_notify_idx_released < max_notify_idx &&
303 !vmci_doorbell_index_table_find(last_notify_idx_released)) {
304 new_notify_idx = last_notify_idx_released;
305 last_notify_idx_released = PAGE_SIZE;
306 } else {
307 bool reused = false;
308 new_notify_idx = last_notify_idx_reserved;
309 if (notify_idx_count + 1 < max_notify_idx) {
310 do {
311 if (!vmci_doorbell_index_table_find(
312 new_notify_idx)) {
313 reused = true;
314 break;
315 }
316 new_notify_idx = (new_notify_idx + 1) %
317 max_notify_idx;
318 } while (new_notify_idx !=
319 last_notify_idx_released);
320 }
321 if (!reused) {
322 new_notify_idx = max_notify_idx;
323 max_notify_idx++;
324 }
325 }
326 } else {
327 new_notify_idx = (last_notify_idx_reserved + 1) % PAGE_SIZE;
328 }
329 last_notify_idx_reserved = new_notify_idx;
330 notify_idx_count++;
331
332 entry->idx = new_notify_idx;
333 bucket = VMCI_DOORBELL_HASH(entry->idx);
334 vmci_list_insert(&vmci_doorbell_it.entries[bucket], entry,
335 idx_list_item);
336
337 vmci_release_lock_bh(&vmci_doorbell_it.lock);
338 }
339
340 /*
341 *------------------------------------------------------------------------------
342 *
343 * vmci_doorbell_index_table_remove --
344 *
345 * Remove the given entry from the index table. This will release() the
346 * entry's resource.
347 *
348 * Results:
349 * None.
350 *
351 * Side effects:
352 * None.
353 *
354 *------------------------------------------------------------------------------
355 */
356
357 static void
vmci_doorbell_index_table_remove(struct vmci_doorbell_entry * entry)358 vmci_doorbell_index_table_remove(struct vmci_doorbell_entry *entry)
359 {
360 ASSERT(entry);
361
362 vmci_grab_lock_bh(&vmci_doorbell_it.lock);
363
364 vmci_list_remove(entry, idx_list_item);
365
366 notify_idx_count--;
367 if (entry->idx == max_notify_idx - 1) {
368 /*
369 * If we delete an entry with the maximum known notification
370 * index, we take the opportunity to prune the current max. As
371 * there might be other unused indices immediately below, we
372 * lower the maximum until we hit an index in use
373 */
374
375 while (max_notify_idx > 0 &&
376 !vmci_doorbell_index_table_find(max_notify_idx - 1))
377 max_notify_idx--;
378 }
379 last_notify_idx_released = entry->idx;
380
381 vmci_release_lock_bh(&vmci_doorbell_it.lock);
382
383 vmci_resource_release(&entry->resource);
384 }
385
386 /*
387 *------------------------------------------------------------------------------
388 *
389 * vmci_doorbell_link --
390 *
391 * Creates a link between the given doorbell handle and the given index in
392 * the bitmap in the device backend.
393 *
394 * Results:
395 * VMCI_SUCCESS if success, appropriate error code otherwise.
396 *
397 * Side effects:
398 * Notification state is created in hypervisor.
399 *
400 *------------------------------------------------------------------------------
401 */
402
403 static int
vmci_doorbell_link(struct vmci_handle handle,bool is_doorbell,uint32_t notify_idx)404 vmci_doorbell_link(struct vmci_handle handle, bool is_doorbell,
405 uint32_t notify_idx)
406 {
407 struct vmci_doorbell_link_msg link_msg;
408 vmci_id resource_id;
409
410 ASSERT(!VMCI_HANDLE_INVALID(handle));
411
412 if (is_doorbell)
413 resource_id = VMCI_DOORBELL_LINK;
414 else {
415 ASSERT(false);
416 return (VMCI_ERROR_UNAVAILABLE);
417 }
418
419 link_msg.hdr.dst = VMCI_MAKE_HANDLE(VMCI_HYPERVISOR_CONTEXT_ID,
420 resource_id);
421 link_msg.hdr.src = VMCI_ANON_SRC_HANDLE;
422 link_msg.hdr.payload_size = sizeof(link_msg) - VMCI_DG_HEADERSIZE;
423 link_msg.handle = handle;
424 link_msg.notify_idx = notify_idx;
425
426 return (vmci_send_datagram((struct vmci_datagram *)&link_msg));
427 }
428
429 /*
430 *------------------------------------------------------------------------------
431 *
432 * vmci_doorbell_unlink --
433 *
434 * Unlinks the given doorbell handle from an index in the bitmap in the
435 * device backend.
436 *
437 * Results:
438 * VMCI_SUCCESS if success, appropriate error code otherwise.
439 *
440 * Side effects:
441 * Notification state is destroyed in hypervisor.
442 *
443 *------------------------------------------------------------------------------
444 */
445
446 static int
vmci_doorbell_unlink(struct vmci_handle handle,bool is_doorbell)447 vmci_doorbell_unlink(struct vmci_handle handle, bool is_doorbell)
448 {
449 struct vmci_doorbell_unlink_msg unlink_msg;
450 vmci_id resource_id;
451
452 ASSERT(!VMCI_HANDLE_INVALID(handle));
453
454 if (is_doorbell)
455 resource_id = VMCI_DOORBELL_UNLINK;
456 else {
457 ASSERT(false);
458 return (VMCI_ERROR_UNAVAILABLE);
459 }
460
461 unlink_msg.hdr.dst = VMCI_MAKE_HANDLE(VMCI_HYPERVISOR_CONTEXT_ID,
462 resource_id);
463 unlink_msg.hdr.src = VMCI_ANON_SRC_HANDLE;
464 unlink_msg.hdr.payload_size = sizeof(unlink_msg) - VMCI_DG_HEADERSIZE;
465 unlink_msg.handle = handle;
466
467 return (vmci_send_datagram((struct vmci_datagram *)&unlink_msg));
468 }
469
470 /*
471 *------------------------------------------------------------------------------
472 *
473 * vmci_doorbell_create --
474 *
475 * Creates a doorbell with the given callback. If the handle is
476 * VMCI_INVALID_HANDLE, a free handle will be assigned, if possible. The
477 * callback can be run immediately (potentially with locks held - the
478 * default) or delayed (in a kernel thread) by specifying the flag
479 * VMCI_FLAG_DELAYED_CB. If delayed execution is selected, a given callback
480 * may not be run if the kernel is unable to allocate memory for the delayed
481 * execution (highly unlikely).
482 *
483 * Results:
484 * VMCI_SUCCESS on success, appropriate error code otherwise.
485 *
486 * Side effects:
487 * None.
488 *
489 *------------------------------------------------------------------------------
490 */
491
492 int
vmci_doorbell_create(struct vmci_handle * handle,uint32_t flags,vmci_privilege_flags priv_flags,vmci_callback notify_cb,void * client_data)493 vmci_doorbell_create(struct vmci_handle *handle, uint32_t flags,
494 vmci_privilege_flags priv_flags, vmci_callback notify_cb, void *client_data)
495 {
496 struct vmci_doorbell_entry *entry;
497 struct vmci_handle new_handle;
498 int result;
499
500 if (!handle || !notify_cb || flags & ~VMCI_FLAG_DELAYED_CB ||
501 priv_flags & ~VMCI_PRIVILEGE_ALL_FLAGS)
502 return (VMCI_ERROR_INVALID_ARGS);
503
504 entry = vmci_alloc_kernel_mem(sizeof(*entry), VMCI_MEMORY_NORMAL);
505 if (entry == NULL) {
506 VMCI_LOG_WARNING(LGPFX"Failed allocating memory for datagram "
507 "entry.\n");
508 return (VMCI_ERROR_NO_MEM);
509 }
510
511 if (!vmci_can_schedule_delayed_work() &&
512 (flags & VMCI_FLAG_DELAYED_CB)) {
513 result = VMCI_ERROR_INVALID_ARGS;
514 goto free_mem;
515 }
516
517 if (VMCI_HANDLE_INVALID(*handle)) {
518 vmci_id context_id;
519
520 context_id = vmci_get_context_id();
521 vmci_id resource_id = vmci_resource_get_id(context_id);
522 if (resource_id == VMCI_INVALID_ID) {
523 result = VMCI_ERROR_NO_HANDLE;
524 goto free_mem;
525 }
526 new_handle = VMCI_MAKE_HANDLE(context_id, resource_id);
527 } else {
528 if (VMCI_INVALID_ID == handle->resource) {
529 VMCI_LOG_DEBUG(LGPFX"Invalid argument "
530 "(handle=0x%x:0x%x).\n", handle->context,
531 handle->resource);
532 result = VMCI_ERROR_INVALID_ARGS;
533 goto free_mem;
534 }
535 new_handle = *handle;
536 }
537
538 entry->idx = 0;
539 entry->priv_flags = priv_flags;
540 entry->is_doorbell = true;
541 entry->run_delayed = (flags & VMCI_FLAG_DELAYED_CB) ? true : false;
542 entry->notify_cb = notify_cb;
543 entry->client_data = client_data;
544 atomic_store_int(&entry->active, 0);
545 vmci_create_event(&entry->destroy_event);
546
547 result = vmci_resource_add(&entry->resource,
548 VMCI_RESOURCE_TYPE_DOORBELL, new_handle, vmci_doorbell_free_cb,
549 entry);
550 if (result != VMCI_SUCCESS) {
551 VMCI_LOG_WARNING(LGPFX"Failed to add new resource "
552 "(handle=0x%x:0x%x).\n", new_handle.context,
553 new_handle.resource);
554 if (result == VMCI_ERROR_DUPLICATE_ENTRY)
555 result = VMCI_ERROR_ALREADY_EXISTS;
556
557 goto destroy;
558 }
559
560 vmci_doorbell_index_table_add(entry);
561 result = vmci_doorbell_link(new_handle, entry->is_doorbell, entry->idx);
562 if (VMCI_SUCCESS != result)
563 goto destroy_resource;
564 atomic_store_int(&entry->active, 1);
565
566 if (VMCI_HANDLE_INVALID(*handle))
567 *handle = new_handle;
568
569 return (result);
570
571 destroy_resource:
572 vmci_doorbell_index_table_remove(entry);
573 vmci_resource_remove(new_handle, VMCI_RESOURCE_TYPE_DOORBELL);
574 destroy:
575 vmci_destroy_event(&entry->destroy_event);
576 free_mem:
577 vmci_free_kernel_mem(entry, sizeof(*entry));
578 return (result);
579 }
580
581 /*
582 *------------------------------------------------------------------------------
583 *
584 * vmci_doorbell_destroy --
585 *
586 * Destroys a doorbell previously created with vmci_doorbell_create. This
587 * operation may block waiting for a callback to finish.
588 *
589 * Results:
590 * VMCI_SUCCESS on success, appropriate error code otherwise.
591 *
592 * Side effects:
593 * May block.
594 *
595 *------------------------------------------------------------------------------
596 */
597
598 int
vmci_doorbell_destroy(struct vmci_handle handle)599 vmci_doorbell_destroy(struct vmci_handle handle)
600 {
601 struct vmci_doorbell_entry *entry;
602 struct vmci_resource *resource;
603 int result;
604
605 if (VMCI_HANDLE_INVALID(handle))
606 return (VMCI_ERROR_INVALID_ARGS);
607
608 resource = vmci_resource_get(handle, VMCI_RESOURCE_TYPE_DOORBELL);
609 if (resource == NULL) {
610 VMCI_LOG_DEBUG(LGPFX"Failed to destroy doorbell "
611 "(handle=0x%x:0x%x).\n", handle.context, handle.resource);
612 return (VMCI_ERROR_NOT_FOUND);
613 }
614 entry = RESOURCE_CONTAINER(resource, struct vmci_doorbell_entry,
615 resource);
616
617 vmci_doorbell_index_table_remove(entry);
618
619 result = vmci_doorbell_unlink(handle, entry->is_doorbell);
620 if (VMCI_SUCCESS != result) {
621 /*
622 * The only reason this should fail would be an inconsistency
623 * between guest and hypervisor state, where the guest believes
624 * it has an active registration whereas the hypervisor doesn't.
625 * One case where this may happen is if a doorbell is
626 * unregistered following a hibernation at a time where the
627 * doorbell state hasn't been restored on the hypervisor side
628 * yet. Since the handle has now been removed in the guest,
629 * we just print a warning and return success.
630 */
631
632 VMCI_LOG_DEBUG(LGPFX"Unlink of %s (handle=0x%x:0x%x) unknown "
633 "by hypervisor (error=%d).\n",
634 entry->is_doorbell ? "doorbell" : "queuepair",
635 handle.context, handle.resource, result);
636 }
637
638 /*
639 * Now remove the resource from the table. It might still be in use
640 * after this, in a callback or still on the delayed work queue.
641 */
642
643 vmci_resource_remove(handle, VMCI_RESOURCE_TYPE_DOORBELL);
644
645 /*
646 * We now wait on the destroyEvent and release the reference we got
647 * above.
648 */
649
650 vmci_wait_on_event(&entry->destroy_event, vmci_doorbell_release_cb,
651 entry);
652
653 /*
654 * We know that we are now the only reference to the above entry so
655 * can safely free it.
656 */
657
658 vmci_destroy_event(&entry->destroy_event);
659 vmci_free_kernel_mem(entry, sizeof(*entry));
660
661 return (VMCI_SUCCESS);
662 }
663
664 /*
665 *------------------------------------------------------------------------------
666 *
667 * vmci_doorbell_notify_as_guest --
668 *
669 * Notify another guest or the host. We send a datagram down to the host
670 * via the hypervisor with the notification info.
671 *
672 * Results:
673 * VMCI_SUCCESS on success, appropriate error code otherwise.
674 *
675 * Side effects:
676 * May do a hypercall.
677 *
678 *------------------------------------------------------------------------------
679 */
680
681 static int
vmci_doorbell_notify_as_guest(struct vmci_handle handle,vmci_privilege_flags priv_flags)682 vmci_doorbell_notify_as_guest(struct vmci_handle handle,
683 vmci_privilege_flags priv_flags)
684 {
685 struct vmci_doorbell_notify_msg notify_msg;
686
687 notify_msg.hdr.dst = VMCI_MAKE_HANDLE(VMCI_HYPERVISOR_CONTEXT_ID,
688 VMCI_DOORBELL_NOTIFY);
689 notify_msg.hdr.src = VMCI_ANON_SRC_HANDLE;
690 notify_msg.hdr.payload_size = sizeof(notify_msg) - VMCI_DG_HEADERSIZE;
691 notify_msg.handle = handle;
692
693 return (vmci_send_datagram((struct vmci_datagram *)¬ify_msg));
694 }
695
696 /*
697 *------------------------------------------------------------------------------
698 *
699 * vmci_doorbell_notify --
700 *
701 * Generates a notification on the doorbell identified by the handle. For
702 * host side generation of notifications, the caller can specify what the
703 * privilege of the calling side is.
704 *
705 * Results:
706 * VMCI_SUCCESS on success, appropriate error code otherwise.
707 *
708 * Side effects:
709 * May do a hypercall.
710 *
711 *------------------------------------------------------------------------------
712 */
713
714 int
vmci_doorbell_notify(struct vmci_handle dst,vmci_privilege_flags priv_flags)715 vmci_doorbell_notify(struct vmci_handle dst, vmci_privilege_flags priv_flags)
716 {
717 if (VMCI_HANDLE_INVALID(dst) ||
718 (priv_flags & ~VMCI_PRIVILEGE_ALL_FLAGS))
719 return (VMCI_ERROR_INVALID_ARGS);
720
721 return (vmci_doorbell_notify_as_guest(dst, priv_flags));
722 }
723
724 /*
725 *------------------------------------------------------------------------------
726 *
727 * vmci_doorbell_delayed_dispatch_cb --
728 *
729 * Calls the specified callback in a delayed context.
730 *
731 * Results:
732 * None.
733 *
734 * Side effects:
735 * None.
736 *
737 *------------------------------------------------------------------------------
738 */
739
740 static void
vmci_doorbell_delayed_dispatch_cb(void * data)741 vmci_doorbell_delayed_dispatch_cb(void *data)
742 {
743 struct vmci_doorbell_entry *entry = (struct vmci_doorbell_entry *)data;
744
745 ASSERT(data);
746
747 entry->notify_cb(entry->client_data);
748
749 vmci_resource_release(&entry->resource);
750 }
751
752 /*
753 *------------------------------------------------------------------------------
754 *
755 * vmci_doorbell_sync --
756 *
757 * Use this as a synchronization point when setting globals, for example,
758 * during device shutdown.
759 *
760 * Results:
761 * None.
762 *
763 * Side effects:
764 * None.
765 *
766 *------------------------------------------------------------------------------
767 */
768
769 void
vmci_doorbell_sync(void)770 vmci_doorbell_sync(void)
771 {
772
773 vmci_grab_lock_bh(&vmci_doorbell_it.lock);
774 vmci_release_lock_bh(&vmci_doorbell_it.lock);
775 vmci_resource_sync();
776 }
777
778 /*
779 *------------------------------------------------------------------------------
780 *
781 * vmci_register_notification_bitmap --
782 *
783 * Register the notification bitmap with the host.
784 *
785 * Results:
786 * true if the bitmap is registered successfully with the device, false
787 * otherwise.
788 *
789 * Side effects:
790 * None.
791 *
792 *------------------------------------------------------------------------------
793 */
794
795 bool
vmci_register_notification_bitmap(PPN bitmap_ppn)796 vmci_register_notification_bitmap(PPN bitmap_ppn)
797 {
798 struct vmci_notify_bitmap_set_msg bitmap_set_msg;
799 int result;
800
801 /*
802 * Do not ASSERT() on the guest device here. This function can get
803 * called during device initialization, so the ASSERT() will fail even
804 * though the device is (almost) up.
805 */
806
807 bitmap_set_msg.hdr.dst = VMCI_MAKE_HANDLE(VMCI_HYPERVISOR_CONTEXT_ID,
808 VMCI_SET_NOTIFY_BITMAP);
809 bitmap_set_msg.hdr.src = VMCI_ANON_SRC_HANDLE;
810 bitmap_set_msg.hdr.payload_size =
811 sizeof(bitmap_set_msg) - VMCI_DG_HEADERSIZE;
812 bitmap_set_msg.bitmap_ppn = bitmap_ppn;
813
814 result = vmci_send_datagram((struct vmci_datagram *)&bitmap_set_msg);
815 if (result != VMCI_SUCCESS) {
816 VMCI_LOG_DEBUG(LGPFX"Failed to register (PPN=%u) as "
817 "notification bitmap (error=%d).\n",
818 bitmap_ppn, result);
819 return (false);
820 }
821 return (true);
822 }
823
824 /*
825 *------------------------------------------------------------------------------
826 *
827 * vmci_doorbell_fire_entries --
828 *
829 * Executes or schedules the handlers for a given notify index.
830 *
831 * Result:
832 * Notification hash entry if found. NULL otherwise.
833 *
834 * Side effects:
835 * Whatever the side effects of the handlers are.
836 *
837 *------------------------------------------------------------------------------
838 */
839
840 static void
vmci_doorbell_fire_entries(uint32_t notify_idx)841 vmci_doorbell_fire_entries(uint32_t notify_idx)
842 {
843 struct vmci_doorbell_entry *iter;
844 uint32_t bucket = VMCI_DOORBELL_HASH(notify_idx);
845
846 vmci_grab_lock_bh(&vmci_doorbell_it.lock);
847
848 vmci_list_scan(iter, &vmci_doorbell_it.entries[bucket], idx_list_item) {
849 if (iter->idx == notify_idx &&
850 atomic_load_int(&iter->active) == 1) {
851 ASSERT(iter->notify_cb);
852 if (iter->run_delayed) {
853 int err;
854
855 vmci_resource_hold(&iter->resource);
856 err = vmci_schedule_delayed_work(
857 vmci_doorbell_delayed_dispatch_cb, iter);
858 if (err != VMCI_SUCCESS) {
859 vmci_resource_release(&iter->resource);
860 goto out;
861 }
862 } else
863 iter->notify_cb(iter->client_data);
864 }
865 }
866
867 out:
868 vmci_release_lock_bh(&vmci_doorbell_it.lock);
869 }
870
871 /*
872 *------------------------------------------------------------------------------
873 *
874 * vmci_scan_notification_bitmap --
875 *
876 * Scans the notification bitmap, collects pending notifications, resets
877 * the bitmap and invokes appropriate callbacks.
878 *
879 * Results:
880 * None.
881 *
882 * Side effects:
883 * May schedule tasks, allocate memory and run callbacks.
884 *
885 *------------------------------------------------------------------------------
886 */
887
888 void
vmci_scan_notification_bitmap(uint8_t * bitmap)889 vmci_scan_notification_bitmap(uint8_t *bitmap)
890 {
891 uint32_t idx;
892
893 ASSERT(bitmap);
894
895 for (idx = 0; idx < max_notify_idx; idx++) {
896 if (bitmap[idx] & 0x1) {
897 bitmap[idx] &= ~1;
898 vmci_doorbell_fire_entries(idx);
899 }
900 }
901 }
902