1 // SPDX-License-Identifier: GPL-2.0-only
5 * Copyright (C) 2012 VMware, Inc. All rights reserved.
8 #include <linux/vmw_vmci_defs.h>
9 #include <linux/vmw_vmci_api.h>
10 #include <linux/highmem.h>
11 #include <linux/kernel.h>
13 #include <linux/module.h>
14 #include <linux/mutex.h>
15 #include <linux/pagemap.h>
16 #include <linux/pci.h>
17 #include <linux/sched.h>
18 #include <linux/slab.h>
19 #include <linux/uio.h>
20 #include <linux/wait.h>
21 #include <linux/vmalloc.h>
22 #include <linux/skbuff.h>
24 #include "vmci_handle_array.h"
25 #include "vmci_queue_pair.h"
26 #include "vmci_datagram.h"
27 #include "vmci_resource.h"
28 #include "vmci_context.h"
29 #include "vmci_driver.h"
30 #include "vmci_event.h"
31 #include "vmci_route.h"
34 * In the following, we will distinguish between two kinds of VMX processes -
35 * the ones with versions lower than VMCI_VERSION_NOVMVM that use specialized
36 * VMCI page files in the VMX and supporting VM to VM communication and the
37 * newer ones that use the guest memory directly. We will in the following
38 * refer to the older VMX versions as old-style VMX'en, and the newer ones as
41 * The state transition datagram is as follows (the VMCIQPB_ prefix has been
42 * removed for readability) - see below for more details on the transtions:
44 * -------------- NEW -------------
47 * CREATED_NO_MEM <-----------------> CREATED_MEM
49 * | o-----------------------o |
52 * ATTACHED_NO_MEM <----------------> ATTACHED_MEM
54 * | o----------------------o |
57 * SHUTDOWN_NO_MEM <----------------> SHUTDOWN_MEM
60 * -------------> gone <-------------
62 * In more detail. When a VMCI queue pair is first created, it will be in the
63 * VMCIQPB_NEW state. It will then move into one of the following states:
65 * - VMCIQPB_CREATED_NO_MEM: this state indicates that either:
67 * - the created was performed by a host endpoint, in which case there is
68 * no backing memory yet.
70 * - the create was initiated by an old-style VMX, that uses
71 * vmci_qp_broker_set_page_store to specify the UVAs of the queue pair at
72 * a later point in time. This state can be distinguished from the one
73 * above by the context ID of the creator. A host side is not allowed to
74 * attach until the page store has been set.
76 * - VMCIQPB_CREATED_MEM: this state is the result when the queue pair
77 * is created by a VMX using the queue pair device backend that
78 * sets the UVAs of the queue pair immediately and stores the
79 * information for later attachers. At this point, it is ready for
80 * the host side to attach to it.
82 * Once the queue pair is in one of the created states (with the exception of
83 * the case mentioned for older VMX'en above), it is possible to attach to the
84 * queue pair. Again we have two new states possible:
86 * - VMCIQPB_ATTACHED_MEM: this state can be reached through the following
89 * - from VMCIQPB_CREATED_NO_MEM when a new-style VMX allocates a queue
90 * pair, and attaches to a queue pair previously created by the host side.
92 * - from VMCIQPB_CREATED_MEM when the host side attaches to a queue pair
93 * already created by a guest.
95 * - from VMCIQPB_ATTACHED_NO_MEM, when an old-style VMX calls
96 * vmci_qp_broker_set_page_store (see below).
98 * - VMCIQPB_ATTACHED_NO_MEM: If the queue pair already was in the
99 * VMCIQPB_CREATED_NO_MEM due to a host side create, an old-style VMX will
100 * bring the queue pair into this state. Once vmci_qp_broker_set_page_store
101 * is called to register the user memory, the VMCIQPB_ATTACH_MEM state
104 * From the attached queue pair, the queue pair can enter the shutdown states
105 * when either side of the queue pair detaches. If the guest side detaches
106 * first, the queue pair will enter the VMCIQPB_SHUTDOWN_NO_MEM state, where
107 * the content of the queue pair will no longer be available. If the host
108 * side detaches first, the queue pair will either enter the
109 * VMCIQPB_SHUTDOWN_MEM, if the guest memory is currently mapped, or
110 * VMCIQPB_SHUTDOWN_NO_MEM, if the guest memory is not mapped
111 * (e.g., the host detaches while a guest is stunned).
113 * New-style VMX'en will also unmap guest memory, if the guest is
114 * quiesced, e.g., during a snapshot operation. In that case, the guest
115 * memory will no longer be available, and the queue pair will transition from
116 * *_MEM state to a *_NO_MEM state. The VMX may later map the memory once more,
117 * in which case the queue pair will transition from the *_NO_MEM state at that
118 * point back to the *_MEM state. Note that the *_NO_MEM state may have changed,
119 * since the peer may have either attached or detached in the meantime. The
120 * values are laid out such that ++ on a state will move from a *_NO_MEM to a
121 * *_MEM state, and vice versa.
124 /* The Kernel specific component of the struct vmci_queue structure. */
125 struct vmci_queue_kern_if {
126 struct mutex __mutex; /* Protects the queue. */
127 struct mutex *mutex; /* Shared by producer and consumer queues. */
128 size_t num_pages; /* Number of pages incl. header. */
129 bool host; /* Host or guest? */
134 } g; /* Used by the guest. */
137 struct page **header_page;
138 } h; /* Used by the host. */
143 * This structure is opaque to the clients.
146 struct vmci_handle handle;
147 struct vmci_queue *produce_q;
148 struct vmci_queue *consume_q;
155 unsigned int blocked;
156 unsigned int generation;
157 wait_queue_head_t event;
160 enum qp_broker_state {
162 VMCIQPB_CREATED_NO_MEM,
164 VMCIQPB_ATTACHED_NO_MEM,
165 VMCIQPB_ATTACHED_MEM,
166 VMCIQPB_SHUTDOWN_NO_MEM,
167 VMCIQPB_SHUTDOWN_MEM,
171 #define QPBROKERSTATE_HAS_MEM(_qpb) (_qpb->state == VMCIQPB_CREATED_MEM || \
172 _qpb->state == VMCIQPB_ATTACHED_MEM || \
173 _qpb->state == VMCIQPB_SHUTDOWN_MEM)
176 * In the queue pair broker, we always use the guest point of view for
177 * the produce and consume queue values and references, e.g., the
178 * produce queue size stored is the guests produce queue size. The
179 * host endpoint will need to swap these around. The only exception is
180 * the local queue pairs on the host, in which case the host endpoint
181 * that creates the queue pair will have the right orientation, and
182 * the attaching host endpoint will need to swap.
185 struct list_head list_item;
186 struct vmci_handle handle;
194 struct qp_broker_entry {
195 struct vmci_resource resource;
199 enum qp_broker_state state;
200 bool require_trusted_attach;
201 bool created_by_trusted;
202 bool vmci_page_files; /* Created by VMX using VMCI page files */
203 struct vmci_queue *produce_q;
204 struct vmci_queue *consume_q;
205 struct vmci_queue_header saved_produce_q;
206 struct vmci_queue_header saved_consume_q;
207 vmci_event_release_cb wakeup_cb;
209 void *local_mem; /* Kernel memory for local queue pair */
212 struct qp_guest_endpoint {
213 struct vmci_resource resource;
218 struct ppn_set ppn_set;
222 struct list_head head;
223 struct mutex mutex; /* Protect queue list. */
226 static struct qp_list qp_broker_list = {
227 .head = LIST_HEAD_INIT(qp_broker_list.head),
228 .mutex = __MUTEX_INITIALIZER(qp_broker_list.mutex),
231 static struct qp_list qp_guest_endpoints = {
232 .head = LIST_HEAD_INIT(qp_guest_endpoints.head),
233 .mutex = __MUTEX_INITIALIZER(qp_guest_endpoints.mutex),
236 #define INVALID_VMCI_GUEST_MEM_ID 0
237 #define QPE_NUM_PAGES(_QPE) ((u32) \
238 (DIV_ROUND_UP(_QPE.produce_size, PAGE_SIZE) + \
239 DIV_ROUND_UP(_QPE.consume_size, PAGE_SIZE) + 2))
240 #define QP_SIZES_ARE_VALID(_prod_qsize, _cons_qsize) \
241 ((_prod_qsize) + (_cons_qsize) >= max(_prod_qsize, _cons_qsize) && \
242 (_prod_qsize) + (_cons_qsize) <= VMCI_MAX_GUEST_QP_MEMORY)
245 * Frees kernel VA space for a given queue and its queue header, and
246 * frees physical data pages.
248 static void qp_free_queue(void *q, u64 size)
250 struct vmci_queue *queue = q;
255 /* Given size does not include header, so add in a page here. */
256 for (i = 0; i < DIV_ROUND_UP(size, PAGE_SIZE) + 1; i++) {
257 dma_free_coherent(&vmci_pdev->dev, PAGE_SIZE,
258 queue->kernel_if->u.g.vas[i],
259 queue->kernel_if->u.g.pas[i]);
267 * Allocates kernel queue pages of specified size with IOMMU mappings,
268 * plus space for the queue structure/kernel interface and the queue
271 static void *qp_alloc_queue(u64 size, u32 flags)
274 struct vmci_queue *queue;
277 size_t queue_size = sizeof(*queue) + sizeof(*queue->kernel_if);
280 if (size > SIZE_MAX - PAGE_SIZE)
282 num_pages = DIV_ROUND_UP(size, PAGE_SIZE) + 1;
284 (SIZE_MAX - queue_size) /
285 (sizeof(*queue->kernel_if->u.g.pas) +
286 sizeof(*queue->kernel_if->u.g.vas)))
289 pas_size = num_pages * sizeof(*queue->kernel_if->u.g.pas);
290 vas_size = num_pages * sizeof(*queue->kernel_if->u.g.vas);
291 queue_size += pas_size + vas_size;
293 queue = vmalloc(queue_size);
297 queue->q_header = NULL;
298 queue->saved_header = NULL;
299 queue->kernel_if = (struct vmci_queue_kern_if *)(queue + 1);
300 queue->kernel_if->mutex = NULL;
301 queue->kernel_if->num_pages = num_pages;
302 queue->kernel_if->u.g.pas = (dma_addr_t *)(queue->kernel_if + 1);
303 queue->kernel_if->u.g.vas =
304 (void **)((u8 *)queue->kernel_if->u.g.pas + pas_size);
305 queue->kernel_if->host = false;
307 for (i = 0; i < num_pages; i++) {
308 queue->kernel_if->u.g.vas[i] =
309 dma_alloc_coherent(&vmci_pdev->dev, PAGE_SIZE,
310 &queue->kernel_if->u.g.pas[i],
312 if (!queue->kernel_if->u.g.vas[i]) {
313 /* Size excl. the header. */
314 qp_free_queue(queue, i * PAGE_SIZE);
319 /* Queue header is the first page. */
320 queue->q_header = queue->kernel_if->u.g.vas[0];
326 * Copies from a given buffer or iovector to a VMCI Queue. Uses
327 * kmap()/kunmap() to dynamically map/unmap required portions of the queue
328 * by traversing the offset -> page translation structure for the queue.
329 * Assumes that offset + size does not wrap around in the queue.
331 static int qp_memcpy_to_queue_iter(struct vmci_queue *queue,
333 struct iov_iter *from,
336 struct vmci_queue_kern_if *kernel_if = queue->kernel_if;
337 size_t bytes_copied = 0;
339 while (bytes_copied < size) {
340 const u64 page_index =
341 (queue_offset + bytes_copied) / PAGE_SIZE;
342 const size_t page_offset =
343 (queue_offset + bytes_copied) & (PAGE_SIZE - 1);
348 va = kmap(kernel_if->u.h.page[page_index]);
350 va = kernel_if->u.g.vas[page_index + 1];
353 if (size - bytes_copied > PAGE_SIZE - page_offset)
354 /* Enough payload to fill up from this page. */
355 to_copy = PAGE_SIZE - page_offset;
357 to_copy = size - bytes_copied;
359 if (!copy_from_iter_full((u8 *)va + page_offset, to_copy,
362 kunmap(kernel_if->u.h.page[page_index]);
363 return VMCI_ERROR_INVALID_ARGS;
365 bytes_copied += to_copy;
367 kunmap(kernel_if->u.h.page[page_index]);
374 * Copies to a given buffer or iovector from a VMCI Queue. Uses
375 * kmap()/kunmap() to dynamically map/unmap required portions of the queue
376 * by traversing the offset -> page translation structure for the queue.
377 * Assumes that offset + size does not wrap around in the queue.
379 static int qp_memcpy_from_queue_iter(struct iov_iter *to,
380 const struct vmci_queue *queue,
381 u64 queue_offset, size_t size)
383 struct vmci_queue_kern_if *kernel_if = queue->kernel_if;
384 size_t bytes_copied = 0;
386 while (bytes_copied < size) {
387 const u64 page_index =
388 (queue_offset + bytes_copied) / PAGE_SIZE;
389 const size_t page_offset =
390 (queue_offset + bytes_copied) & (PAGE_SIZE - 1);
396 va = kmap(kernel_if->u.h.page[page_index]);
398 va = kernel_if->u.g.vas[page_index + 1];
401 if (size - bytes_copied > PAGE_SIZE - page_offset)
402 /* Enough payload to fill up this page. */
403 to_copy = PAGE_SIZE - page_offset;
405 to_copy = size - bytes_copied;
407 err = copy_to_iter((u8 *)va + page_offset, to_copy, to);
408 if (err != to_copy) {
410 kunmap(kernel_if->u.h.page[page_index]);
411 return VMCI_ERROR_INVALID_ARGS;
413 bytes_copied += to_copy;
415 kunmap(kernel_if->u.h.page[page_index]);
422 * Allocates two list of PPNs --- one for the pages in the produce queue,
423 * and the other for the pages in the consume queue. Intializes the list
424 * of PPNs with the page frame numbers of the KVA for the two queues (and
425 * the queue headers).
427 static int qp_alloc_ppn_set(void *prod_q,
428 u64 num_produce_pages,
430 u64 num_consume_pages, struct ppn_set *ppn_set)
434 struct vmci_queue *produce_q = prod_q;
435 struct vmci_queue *consume_q = cons_q;
438 if (!produce_q || !num_produce_pages || !consume_q ||
439 !num_consume_pages || !ppn_set)
440 return VMCI_ERROR_INVALID_ARGS;
442 if (ppn_set->initialized)
443 return VMCI_ERROR_ALREADY_EXISTS;
446 kmalloc_array(num_produce_pages, sizeof(*produce_ppns),
449 return VMCI_ERROR_NO_MEM;
452 kmalloc_array(num_consume_pages, sizeof(*consume_ppns),
456 return VMCI_ERROR_NO_MEM;
459 for (i = 0; i < num_produce_pages; i++)
461 produce_q->kernel_if->u.g.pas[i] >> PAGE_SHIFT;
463 for (i = 0; i < num_consume_pages; i++)
465 consume_q->kernel_if->u.g.pas[i] >> PAGE_SHIFT;
467 ppn_set->num_produce_pages = num_produce_pages;
468 ppn_set->num_consume_pages = num_consume_pages;
469 ppn_set->produce_ppns = produce_ppns;
470 ppn_set->consume_ppns = consume_ppns;
471 ppn_set->initialized = true;
476 * Frees the two list of PPNs for a queue pair.
478 static void qp_free_ppn_set(struct ppn_set *ppn_set)
480 if (ppn_set->initialized) {
481 /* Do not call these functions on NULL inputs. */
482 kfree(ppn_set->produce_ppns);
483 kfree(ppn_set->consume_ppns);
485 memset(ppn_set, 0, sizeof(*ppn_set));
489 * Populates the list of PPNs in the hypercall structure with the PPNS
490 * of the produce queue and the consume queue.
492 static int qp_populate_ppn_set(u8 *call_buf, const struct ppn_set *ppn_set)
494 if (vmci_use_ppn64()) {
495 memcpy(call_buf, ppn_set->produce_ppns,
496 ppn_set->num_produce_pages *
497 sizeof(*ppn_set->produce_ppns));
499 ppn_set->num_produce_pages *
500 sizeof(*ppn_set->produce_ppns),
501 ppn_set->consume_ppns,
502 ppn_set->num_consume_pages *
503 sizeof(*ppn_set->consume_ppns));
506 u32 *ppns = (u32 *) call_buf;
508 for (i = 0; i < ppn_set->num_produce_pages; i++)
509 ppns[i] = (u32) ppn_set->produce_ppns[i];
511 ppns = &ppns[ppn_set->num_produce_pages];
513 for (i = 0; i < ppn_set->num_consume_pages; i++)
514 ppns[i] = (u32) ppn_set->consume_ppns[i];
521 * Allocates kernel VA space of specified size plus space for the queue
522 * and kernel interface. This is different from the guest queue allocator,
523 * because we do not allocate our own queue header/data pages here but
524 * share those of the guest.
526 static struct vmci_queue *qp_host_alloc_queue(u64 size)
528 struct vmci_queue *queue;
529 size_t queue_page_size;
531 const size_t queue_size = sizeof(*queue) + sizeof(*(queue->kernel_if));
533 if (size > min_t(size_t, VMCI_MAX_GUEST_QP_MEMORY, SIZE_MAX - PAGE_SIZE))
535 num_pages = DIV_ROUND_UP(size, PAGE_SIZE) + 1;
536 if (num_pages > (SIZE_MAX - queue_size) /
537 sizeof(*queue->kernel_if->u.h.page))
540 queue_page_size = num_pages * sizeof(*queue->kernel_if->u.h.page);
542 if (queue_size + queue_page_size > KMALLOC_MAX_SIZE)
545 queue = kzalloc(queue_size + queue_page_size, GFP_KERNEL);
547 queue->q_header = NULL;
548 queue->saved_header = NULL;
549 queue->kernel_if = (struct vmci_queue_kern_if *)(queue + 1);
550 queue->kernel_if->host = true;
551 queue->kernel_if->mutex = NULL;
552 queue->kernel_if->num_pages = num_pages;
553 queue->kernel_if->u.h.header_page =
554 (struct page **)((u8 *)queue + queue_size);
555 queue->kernel_if->u.h.page =
556 &queue->kernel_if->u.h.header_page[1];
563 * Frees kernel memory for a given queue (header plus translation
566 static void qp_host_free_queue(struct vmci_queue *queue, u64 queue_size)
572 * Initialize the mutex for the pair of queues. This mutex is used to
573 * protect the q_header and the buffer from changing out from under any
574 * users of either queue. Of course, it's only any good if the mutexes
575 * are actually acquired. Queue structure must lie on non-paged memory
576 * or we cannot guarantee access to the mutex.
578 static void qp_init_queue_mutex(struct vmci_queue *produce_q,
579 struct vmci_queue *consume_q)
582 * Only the host queue has shared state - the guest queues do not
583 * need to synchronize access using a queue mutex.
586 if (produce_q->kernel_if->host) {
587 produce_q->kernel_if->mutex = &produce_q->kernel_if->__mutex;
588 consume_q->kernel_if->mutex = &produce_q->kernel_if->__mutex;
589 mutex_init(produce_q->kernel_if->mutex);
594 * Cleans up the mutex for the pair of queues.
596 static void qp_cleanup_queue_mutex(struct vmci_queue *produce_q,
597 struct vmci_queue *consume_q)
599 if (produce_q->kernel_if->host) {
600 produce_q->kernel_if->mutex = NULL;
601 consume_q->kernel_if->mutex = NULL;
606 * Acquire the mutex for the queue. Note that the produce_q and
607 * the consume_q share a mutex. So, only one of the two need to
608 * be passed in to this routine. Either will work just fine.
610 static void qp_acquire_queue_mutex(struct vmci_queue *queue)
612 if (queue->kernel_if->host)
613 mutex_lock(queue->kernel_if->mutex);
617 * Release the mutex for the queue. Note that the produce_q and
618 * the consume_q share a mutex. So, only one of the two need to
619 * be passed in to this routine. Either will work just fine.
621 static void qp_release_queue_mutex(struct vmci_queue *queue)
623 if (queue->kernel_if->host)
624 mutex_unlock(queue->kernel_if->mutex);
628 * Helper function to release pages in the PageStoreAttachInfo
629 * previously obtained using get_user_pages.
631 static void qp_release_pages(struct page **pages,
632 u64 num_pages, bool dirty)
636 for (i = 0; i < num_pages; i++) {
638 set_page_dirty_lock(pages[i]);
646 * Lock the user pages referenced by the {produce,consume}Buffer
647 * struct into memory and populate the {produce,consume}Pages
648 * arrays in the attach structure with them.
650 static int qp_host_get_user_memory(u64 produce_uva,
652 struct vmci_queue *produce_q,
653 struct vmci_queue *consume_q)
656 int err = VMCI_SUCCESS;
658 retval = get_user_pages_fast((uintptr_t) produce_uva,
659 produce_q->kernel_if->num_pages,
661 produce_q->kernel_if->u.h.header_page);
662 if (retval < (int)produce_q->kernel_if->num_pages) {
663 pr_debug("get_user_pages_fast(produce) failed (retval=%d)",
666 qp_release_pages(produce_q->kernel_if->u.h.header_page,
668 err = VMCI_ERROR_NO_MEM;
672 retval = get_user_pages_fast((uintptr_t) consume_uva,
673 consume_q->kernel_if->num_pages,
675 consume_q->kernel_if->u.h.header_page);
676 if (retval < (int)consume_q->kernel_if->num_pages) {
677 pr_debug("get_user_pages_fast(consume) failed (retval=%d)",
680 qp_release_pages(consume_q->kernel_if->u.h.header_page,
682 qp_release_pages(produce_q->kernel_if->u.h.header_page,
683 produce_q->kernel_if->num_pages, false);
684 err = VMCI_ERROR_NO_MEM;
692 * Registers the specification of the user pages used for backing a queue
693 * pair. Enough information to map in pages is stored in the OS specific
694 * part of the struct vmci_queue structure.
696 static int qp_host_register_user_memory(struct vmci_qp_page_store *page_store,
697 struct vmci_queue *produce_q,
698 struct vmci_queue *consume_q)
704 * The new style and the old style mapping only differs in
705 * that we either get a single or two UVAs, so we split the
706 * single UVA range at the appropriate spot.
708 produce_uva = page_store->pages;
709 consume_uva = page_store->pages +
710 produce_q->kernel_if->num_pages * PAGE_SIZE;
711 return qp_host_get_user_memory(produce_uva, consume_uva, produce_q,
716 * Releases and removes the references to user pages stored in the attach
717 * struct. Pages are released from the page cache and may become
720 static void qp_host_unregister_user_memory(struct vmci_queue *produce_q,
721 struct vmci_queue *consume_q)
723 qp_release_pages(produce_q->kernel_if->u.h.header_page,
724 produce_q->kernel_if->num_pages, true);
725 memset(produce_q->kernel_if->u.h.header_page, 0,
726 sizeof(*produce_q->kernel_if->u.h.header_page) *
727 produce_q->kernel_if->num_pages);
728 qp_release_pages(consume_q->kernel_if->u.h.header_page,
729 consume_q->kernel_if->num_pages, true);
730 memset(consume_q->kernel_if->u.h.header_page, 0,
731 sizeof(*consume_q->kernel_if->u.h.header_page) *
732 consume_q->kernel_if->num_pages);
736 * Once qp_host_register_user_memory has been performed on a
737 * queue, the queue pair headers can be mapped into the
738 * kernel. Once mapped, they must be unmapped with
739 * qp_host_unmap_queues prior to calling
740 * qp_host_unregister_user_memory.
743 static int qp_host_map_queues(struct vmci_queue *produce_q,
744 struct vmci_queue *consume_q)
748 if (!produce_q->q_header || !consume_q->q_header) {
749 struct page *headers[2];
751 if (produce_q->q_header != consume_q->q_header)
752 return VMCI_ERROR_QUEUEPAIR_MISMATCH;
754 if (produce_q->kernel_if->u.h.header_page == NULL ||
755 *produce_q->kernel_if->u.h.header_page == NULL)
756 return VMCI_ERROR_UNAVAILABLE;
758 headers[0] = *produce_q->kernel_if->u.h.header_page;
759 headers[1] = *consume_q->kernel_if->u.h.header_page;
761 produce_q->q_header = vmap(headers, 2, VM_MAP, PAGE_KERNEL);
762 if (produce_q->q_header != NULL) {
763 consume_q->q_header =
764 (struct vmci_queue_header *)((u8 *)
765 produce_q->q_header +
767 result = VMCI_SUCCESS;
769 pr_warn("vmap failed\n");
770 result = VMCI_ERROR_NO_MEM;
773 result = VMCI_SUCCESS;
780 * Unmaps previously mapped queue pair headers from the kernel.
781 * Pages are unpinned.
783 static int qp_host_unmap_queues(u32 gid,
784 struct vmci_queue *produce_q,
785 struct vmci_queue *consume_q)
787 if (produce_q->q_header) {
788 if (produce_q->q_header < consume_q->q_header)
789 vunmap(produce_q->q_header);
791 vunmap(consume_q->q_header);
793 produce_q->q_header = NULL;
794 consume_q->q_header = NULL;
801 * Finds the entry in the list corresponding to a given handle. Assumes
802 * that the list is locked.
804 static struct qp_entry *qp_list_find(struct qp_list *qp_list,
805 struct vmci_handle handle)
807 struct qp_entry *entry;
809 if (vmci_handle_is_invalid(handle))
812 list_for_each_entry(entry, &qp_list->head, list_item) {
813 if (vmci_handle_is_equal(entry->handle, handle))
821 * Finds the entry in the list corresponding to a given handle.
823 static struct qp_guest_endpoint *
824 qp_guest_handle_to_entry(struct vmci_handle handle)
826 struct qp_guest_endpoint *entry;
827 struct qp_entry *qp = qp_list_find(&qp_guest_endpoints, handle);
829 entry = qp ? container_of(
830 qp, struct qp_guest_endpoint, qp) : NULL;
835 * Finds the entry in the list corresponding to a given handle.
837 static struct qp_broker_entry *
838 qp_broker_handle_to_entry(struct vmci_handle handle)
840 struct qp_broker_entry *entry;
841 struct qp_entry *qp = qp_list_find(&qp_broker_list, handle);
843 entry = qp ? container_of(
844 qp, struct qp_broker_entry, qp) : NULL;
849 * Dispatches a queue pair event message directly into the local event
852 static int qp_notify_peer_local(bool attach, struct vmci_handle handle)
854 u32 context_id = vmci_get_context_id();
855 struct vmci_event_qp ev;
857 ev.msg.hdr.dst = vmci_make_handle(context_id, VMCI_EVENT_HANDLER);
858 ev.msg.hdr.src = vmci_make_handle(VMCI_HYPERVISOR_CONTEXT_ID,
859 VMCI_CONTEXT_RESOURCE_ID);
860 ev.msg.hdr.payload_size = sizeof(ev) - sizeof(ev.msg.hdr);
861 ev.msg.event_data.event =
862 attach ? VMCI_EVENT_QP_PEER_ATTACH : VMCI_EVENT_QP_PEER_DETACH;
863 ev.payload.peer_id = context_id;
864 ev.payload.handle = handle;
866 return vmci_event_dispatch(&ev.msg.hdr);
870 * Allocates and initializes a qp_guest_endpoint structure.
871 * Allocates a queue_pair rid (and handle) iff the given entry has
872 * an invalid handle. 0 through VMCI_RESERVED_RESOURCE_ID_MAX
873 * are reserved handles. Assumes that the QP list mutex is held
876 static struct qp_guest_endpoint *
877 qp_guest_endpoint_create(struct vmci_handle handle,
886 struct qp_guest_endpoint *entry;
887 /* One page each for the queue headers. */
888 const u64 num_ppns = DIV_ROUND_UP(produce_size, PAGE_SIZE) +
889 DIV_ROUND_UP(consume_size, PAGE_SIZE) + 2;
891 if (vmci_handle_is_invalid(handle)) {
892 u32 context_id = vmci_get_context_id();
894 handle = vmci_make_handle(context_id, VMCI_INVALID_ID);
897 entry = kzalloc(sizeof(*entry), GFP_KERNEL);
899 entry->qp.peer = peer;
900 entry->qp.flags = flags;
901 entry->qp.produce_size = produce_size;
902 entry->qp.consume_size = consume_size;
903 entry->qp.ref_count = 0;
904 entry->num_ppns = num_ppns;
905 entry->produce_q = produce_q;
906 entry->consume_q = consume_q;
907 INIT_LIST_HEAD(&entry->qp.list_item);
909 /* Add resource obj */
910 result = vmci_resource_add(&entry->resource,
911 VMCI_RESOURCE_TYPE_QPAIR_GUEST,
913 entry->qp.handle = vmci_resource_handle(&entry->resource);
914 if ((result != VMCI_SUCCESS) ||
915 qp_list_find(&qp_guest_endpoints, entry->qp.handle)) {
916 pr_warn("Failed to add new resource (handle=0x%x:0x%x), error: %d",
917 handle.context, handle.resource, result);
926 * Frees a qp_guest_endpoint structure.
928 static void qp_guest_endpoint_destroy(struct qp_guest_endpoint *entry)
930 qp_free_ppn_set(&entry->ppn_set);
931 qp_cleanup_queue_mutex(entry->produce_q, entry->consume_q);
932 qp_free_queue(entry->produce_q, entry->qp.produce_size);
933 qp_free_queue(entry->consume_q, entry->qp.consume_size);
934 /* Unlink from resource hash table and free callback */
935 vmci_resource_remove(&entry->resource);
941 * Helper to make a queue_pairAlloc hypercall when the driver is
942 * supporting a guest device.
944 static int qp_alloc_hypercall(const struct qp_guest_endpoint *entry)
946 struct vmci_qp_alloc_msg *alloc_msg;
951 if (!entry || entry->num_ppns <= 2)
952 return VMCI_ERROR_INVALID_ARGS;
954 ppn_size = vmci_use_ppn64() ? sizeof(u64) : sizeof(u32);
955 msg_size = sizeof(*alloc_msg) +
956 (size_t) entry->num_ppns * ppn_size;
957 alloc_msg = kmalloc(msg_size, GFP_KERNEL);
959 return VMCI_ERROR_NO_MEM;
961 alloc_msg->hdr.dst = vmci_make_handle(VMCI_HYPERVISOR_CONTEXT_ID,
962 VMCI_QUEUEPAIR_ALLOC);
963 alloc_msg->hdr.src = VMCI_ANON_SRC_HANDLE;
964 alloc_msg->hdr.payload_size = msg_size - VMCI_DG_HEADERSIZE;
965 alloc_msg->handle = entry->qp.handle;
966 alloc_msg->peer = entry->qp.peer;
967 alloc_msg->flags = entry->qp.flags;
968 alloc_msg->produce_size = entry->qp.produce_size;
969 alloc_msg->consume_size = entry->qp.consume_size;
970 alloc_msg->num_ppns = entry->num_ppns;
972 result = qp_populate_ppn_set((u8 *)alloc_msg + sizeof(*alloc_msg),
974 if (result == VMCI_SUCCESS)
975 result = vmci_send_datagram(&alloc_msg->hdr);
983 * Helper to make a queue_pairDetach hypercall when the driver is
984 * supporting a guest device.
986 static int qp_detatch_hypercall(struct vmci_handle handle)
988 struct vmci_qp_detach_msg detach_msg;
990 detach_msg.hdr.dst = vmci_make_handle(VMCI_HYPERVISOR_CONTEXT_ID,
991 VMCI_QUEUEPAIR_DETACH);
992 detach_msg.hdr.src = VMCI_ANON_SRC_HANDLE;
993 detach_msg.hdr.payload_size = sizeof(handle);
994 detach_msg.handle = handle;
996 return vmci_send_datagram(&detach_msg.hdr);
1000 * Adds the given entry to the list. Assumes that the list is locked.
1002 static void qp_list_add_entry(struct qp_list *qp_list, struct qp_entry *entry)
1005 list_add(&entry->list_item, &qp_list->head);
1009 * Removes the given entry from the list. Assumes that the list is locked.
1011 static void qp_list_remove_entry(struct qp_list *qp_list,
1012 struct qp_entry *entry)
1015 list_del(&entry->list_item);
1019 * Helper for VMCI queue_pair detach interface. Frees the physical
1020 * pages for the queue pair.
1022 static int qp_detatch_guest_work(struct vmci_handle handle)
1025 struct qp_guest_endpoint *entry;
1026 u32 ref_count = ~0; /* To avoid compiler warning below */
1028 mutex_lock(&qp_guest_endpoints.mutex);
1030 entry = qp_guest_handle_to_entry(handle);
1032 mutex_unlock(&qp_guest_endpoints.mutex);
1033 return VMCI_ERROR_NOT_FOUND;
1036 if (entry->qp.flags & VMCI_QPFLAG_LOCAL) {
1037 result = VMCI_SUCCESS;
1039 if (entry->qp.ref_count > 1) {
1040 result = qp_notify_peer_local(false, handle);
1042 * We can fail to notify a local queuepair
1043 * because we can't allocate. We still want
1044 * to release the entry if that happens, so
1045 * don't bail out yet.
1049 result = qp_detatch_hypercall(handle);
1050 if (result < VMCI_SUCCESS) {
1052 * We failed to notify a non-local queuepair.
1053 * That other queuepair might still be
1054 * accessing the shared memory, so don't
1055 * release the entry yet. It will get cleaned
1056 * up by VMCIqueue_pair_Exit() if necessary
1057 * (assuming we are going away, otherwise why
1061 mutex_unlock(&qp_guest_endpoints.mutex);
1067 * If we get here then we either failed to notify a local queuepair, or
1068 * we succeeded in all cases. Release the entry if required.
1071 entry->qp.ref_count--;
1072 if (entry->qp.ref_count == 0)
1073 qp_list_remove_entry(&qp_guest_endpoints, &entry->qp);
1075 /* If we didn't remove the entry, this could change once we unlock. */
1077 ref_count = entry->qp.ref_count;
1079 mutex_unlock(&qp_guest_endpoints.mutex);
1082 qp_guest_endpoint_destroy(entry);
1088 * This functions handles the actual allocation of a VMCI queue
1089 * pair guest endpoint. Allocates physical pages for the queue
1090 * pair. It makes OS dependent calls through generic wrappers.
1092 static int qp_alloc_guest_work(struct vmci_handle *handle,
1093 struct vmci_queue **produce_q,
1095 struct vmci_queue **consume_q,
1101 const u64 num_produce_pages =
1102 DIV_ROUND_UP(produce_size, PAGE_SIZE) + 1;
1103 const u64 num_consume_pages =
1104 DIV_ROUND_UP(consume_size, PAGE_SIZE) + 1;
1105 void *my_produce_q = NULL;
1106 void *my_consume_q = NULL;
1108 struct qp_guest_endpoint *queue_pair_entry = NULL;
1110 if (priv_flags != VMCI_NO_PRIVILEGE_FLAGS)
1111 return VMCI_ERROR_NO_ACCESS;
1113 mutex_lock(&qp_guest_endpoints.mutex);
1115 queue_pair_entry = qp_guest_handle_to_entry(*handle);
1116 if (queue_pair_entry) {
1117 if (queue_pair_entry->qp.flags & VMCI_QPFLAG_LOCAL) {
1118 /* Local attach case. */
1119 if (queue_pair_entry->qp.ref_count > 1) {
1120 pr_devel("Error attempting to attach more than once\n");
1121 result = VMCI_ERROR_UNAVAILABLE;
1122 goto error_keep_entry;
1125 if (queue_pair_entry->qp.produce_size != consume_size ||
1126 queue_pair_entry->qp.consume_size !=
1128 queue_pair_entry->qp.flags !=
1129 (flags & ~VMCI_QPFLAG_ATTACH_ONLY)) {
1130 pr_devel("Error mismatched queue pair in local attach\n");
1131 result = VMCI_ERROR_QUEUEPAIR_MISMATCH;
1132 goto error_keep_entry;
1136 * Do a local attach. We swap the consume and
1137 * produce queues for the attacher and deliver
1140 result = qp_notify_peer_local(true, *handle);
1141 if (result < VMCI_SUCCESS)
1142 goto error_keep_entry;
1144 my_produce_q = queue_pair_entry->consume_q;
1145 my_consume_q = queue_pair_entry->produce_q;
1149 result = VMCI_ERROR_ALREADY_EXISTS;
1150 goto error_keep_entry;
1153 my_produce_q = qp_alloc_queue(produce_size, flags);
1154 if (!my_produce_q) {
1155 pr_warn("Error allocating pages for produce queue\n");
1156 result = VMCI_ERROR_NO_MEM;
1160 my_consume_q = qp_alloc_queue(consume_size, flags);
1161 if (!my_consume_q) {
1162 pr_warn("Error allocating pages for consume queue\n");
1163 result = VMCI_ERROR_NO_MEM;
1167 queue_pair_entry = qp_guest_endpoint_create(*handle, peer, flags,
1168 produce_size, consume_size,
1169 my_produce_q, my_consume_q);
1170 if (!queue_pair_entry) {
1171 pr_warn("Error allocating memory in %s\n", __func__);
1172 result = VMCI_ERROR_NO_MEM;
1176 result = qp_alloc_ppn_set(my_produce_q, num_produce_pages, my_consume_q,
1178 &queue_pair_entry->ppn_set);
1179 if (result < VMCI_SUCCESS) {
1180 pr_warn("qp_alloc_ppn_set failed\n");
1185 * It's only necessary to notify the host if this queue pair will be
1186 * attached to from another context.
1188 if (queue_pair_entry->qp.flags & VMCI_QPFLAG_LOCAL) {
1189 /* Local create case. */
1190 u32 context_id = vmci_get_context_id();
1193 * Enforce similar checks on local queue pairs as we
1194 * do for regular ones. The handle's context must
1195 * match the creator or attacher context id (here they
1196 * are both the current context id) and the
1197 * attach-only flag cannot exist during create. We
1198 * also ensure specified peer is this context or an
1201 if (queue_pair_entry->qp.handle.context != context_id ||
1202 (queue_pair_entry->qp.peer != VMCI_INVALID_ID &&
1203 queue_pair_entry->qp.peer != context_id)) {
1204 result = VMCI_ERROR_NO_ACCESS;
1208 if (queue_pair_entry->qp.flags & VMCI_QPFLAG_ATTACH_ONLY) {
1209 result = VMCI_ERROR_NOT_FOUND;
1213 result = qp_alloc_hypercall(queue_pair_entry);
1214 if (result < VMCI_SUCCESS) {
1215 pr_devel("qp_alloc_hypercall result = %d\n", result);
1220 qp_init_queue_mutex((struct vmci_queue *)my_produce_q,
1221 (struct vmci_queue *)my_consume_q);
1223 qp_list_add_entry(&qp_guest_endpoints, &queue_pair_entry->qp);
1226 queue_pair_entry->qp.ref_count++;
1227 *handle = queue_pair_entry->qp.handle;
1228 *produce_q = (struct vmci_queue *)my_produce_q;
1229 *consume_q = (struct vmci_queue *)my_consume_q;
1232 * We should initialize the queue pair header pages on a local
1233 * queue pair create. For non-local queue pairs, the
1234 * hypervisor initializes the header pages in the create step.
1236 if ((queue_pair_entry->qp.flags & VMCI_QPFLAG_LOCAL) &&
1237 queue_pair_entry->qp.ref_count == 1) {
1238 vmci_q_header_init((*produce_q)->q_header, *handle);
1239 vmci_q_header_init((*consume_q)->q_header, *handle);
1242 mutex_unlock(&qp_guest_endpoints.mutex);
1244 return VMCI_SUCCESS;
1247 mutex_unlock(&qp_guest_endpoints.mutex);
1248 if (queue_pair_entry) {
1249 /* The queues will be freed inside the destroy routine. */
1250 qp_guest_endpoint_destroy(queue_pair_entry);
1252 qp_free_queue(my_produce_q, produce_size);
1253 qp_free_queue(my_consume_q, consume_size);
1258 /* This path should only be used when an existing entry was found. */
1259 mutex_unlock(&qp_guest_endpoints.mutex);
1264 * The first endpoint issuing a queue pair allocation will create the state
1265 * of the queue pair in the queue pair broker.
1267 * If the creator is a guest, it will associate a VMX virtual address range
1268 * with the queue pair as specified by the page_store. For compatibility with
1269 * older VMX'en, that would use a separate step to set the VMX virtual
1270 * address range, the virtual address range can be registered later using
1271 * vmci_qp_broker_set_page_store. In that case, a page_store of NULL should be
1274 * If the creator is the host, a page_store of NULL should be used as well,
1275 * since the host is not able to supply a page store for the queue pair.
1277 * For older VMX and host callers, the queue pair will be created in the
1278 * VMCIQPB_CREATED_NO_MEM state, and for current VMX callers, it will be
1279 * created in VMCOQPB_CREATED_MEM state.
1281 static int qp_broker_create(struct vmci_handle handle,
1287 struct vmci_qp_page_store *page_store,
1288 struct vmci_ctx *context,
1289 vmci_event_release_cb wakeup_cb,
1290 void *client_data, struct qp_broker_entry **ent)
1292 struct qp_broker_entry *entry = NULL;
1293 const u32 context_id = vmci_ctx_get_id(context);
1294 bool is_local = flags & VMCI_QPFLAG_LOCAL;
1296 u64 guest_produce_size;
1297 u64 guest_consume_size;
1299 /* Do not create if the caller asked not to. */
1300 if (flags & VMCI_QPFLAG_ATTACH_ONLY)
1301 return VMCI_ERROR_NOT_FOUND;
1304 * Creator's context ID should match handle's context ID or the creator
1305 * must allow the context in handle's context ID as the "peer".
1307 if (handle.context != context_id && handle.context != peer)
1308 return VMCI_ERROR_NO_ACCESS;
1310 if (VMCI_CONTEXT_IS_VM(context_id) && VMCI_CONTEXT_IS_VM(peer))
1311 return VMCI_ERROR_DST_UNREACHABLE;
1314 * Creator's context ID for local queue pairs should match the
1315 * peer, if a peer is specified.
1317 if (is_local && peer != VMCI_INVALID_ID && context_id != peer)
1318 return VMCI_ERROR_NO_ACCESS;
1320 entry = kzalloc(sizeof(*entry), GFP_ATOMIC);
1322 return VMCI_ERROR_NO_MEM;
1324 if (vmci_ctx_get_id(context) == VMCI_HOST_CONTEXT_ID && !is_local) {
1326 * The queue pair broker entry stores values from the guest
1327 * point of view, so a creating host side endpoint should swap
1328 * produce and consume values -- unless it is a local queue
1329 * pair, in which case no swapping is necessary, since the local
1330 * attacher will swap queues.
1333 guest_produce_size = consume_size;
1334 guest_consume_size = produce_size;
1336 guest_produce_size = produce_size;
1337 guest_consume_size = consume_size;
1340 entry->qp.handle = handle;
1341 entry->qp.peer = peer;
1342 entry->qp.flags = flags;
1343 entry->qp.produce_size = guest_produce_size;
1344 entry->qp.consume_size = guest_consume_size;
1345 entry->qp.ref_count = 1;
1346 entry->create_id = context_id;
1347 entry->attach_id = VMCI_INVALID_ID;
1348 entry->state = VMCIQPB_NEW;
1349 entry->require_trusted_attach =
1350 !!(context->priv_flags & VMCI_PRIVILEGE_FLAG_RESTRICTED);
1351 entry->created_by_trusted =
1352 !!(priv_flags & VMCI_PRIVILEGE_FLAG_TRUSTED);
1353 entry->vmci_page_files = false;
1354 entry->wakeup_cb = wakeup_cb;
1355 entry->client_data = client_data;
1356 entry->produce_q = qp_host_alloc_queue(guest_produce_size);
1357 if (entry->produce_q == NULL) {
1358 result = VMCI_ERROR_NO_MEM;
1361 entry->consume_q = qp_host_alloc_queue(guest_consume_size);
1362 if (entry->consume_q == NULL) {
1363 result = VMCI_ERROR_NO_MEM;
1367 qp_init_queue_mutex(entry->produce_q, entry->consume_q);
1369 INIT_LIST_HEAD(&entry->qp.list_item);
1374 entry->local_mem = kcalloc(QPE_NUM_PAGES(entry->qp),
1375 PAGE_SIZE, GFP_KERNEL);
1376 if (entry->local_mem == NULL) {
1377 result = VMCI_ERROR_NO_MEM;
1380 entry->state = VMCIQPB_CREATED_MEM;
1381 entry->produce_q->q_header = entry->local_mem;
1382 tmp = (u8 *)entry->local_mem + PAGE_SIZE *
1383 (DIV_ROUND_UP(entry->qp.produce_size, PAGE_SIZE) + 1);
1384 entry->consume_q->q_header = (struct vmci_queue_header *)tmp;
1385 } else if (page_store) {
1387 * The VMX already initialized the queue pair headers, so no
1388 * need for the kernel side to do that.
1390 result = qp_host_register_user_memory(page_store,
1393 if (result < VMCI_SUCCESS)
1396 entry->state = VMCIQPB_CREATED_MEM;
1399 * A create without a page_store may be either a host
1400 * side create (in which case we are waiting for the
1401 * guest side to supply the memory) or an old style
1402 * queue pair create (in which case we will expect a
1403 * set page store call as the next step).
1405 entry->state = VMCIQPB_CREATED_NO_MEM;
1408 qp_list_add_entry(&qp_broker_list, &entry->qp);
1412 /* Add to resource obj */
1413 result = vmci_resource_add(&entry->resource,
1414 VMCI_RESOURCE_TYPE_QPAIR_HOST,
1416 if (result != VMCI_SUCCESS) {
1417 pr_warn("Failed to add new resource (handle=0x%x:0x%x), error: %d",
1418 handle.context, handle.resource, result);
1422 entry->qp.handle = vmci_resource_handle(&entry->resource);
1424 vmci_q_header_init(entry->produce_q->q_header,
1426 vmci_q_header_init(entry->consume_q->q_header,
1430 vmci_ctx_qp_create(context, entry->qp.handle);
1432 return VMCI_SUCCESS;
1435 if (entry != NULL) {
1436 qp_host_free_queue(entry->produce_q, guest_produce_size);
1437 qp_host_free_queue(entry->consume_q, guest_consume_size);
1445 * Enqueues an event datagram to notify the peer VM attached to
1446 * the given queue pair handle about attach/detach event by the
1447 * given VM. Returns Payload size of datagram enqueued on
1448 * success, error code otherwise.
1450 static int qp_notify_peer(bool attach,
1451 struct vmci_handle handle,
1456 struct vmci_event_qp ev;
1458 if (vmci_handle_is_invalid(handle) || my_id == VMCI_INVALID_ID ||
1459 peer_id == VMCI_INVALID_ID)
1460 return VMCI_ERROR_INVALID_ARGS;
1463 * In vmci_ctx_enqueue_datagram() we enforce the upper limit on
1464 * number of pending events from the hypervisor to a given VM
1465 * otherwise a rogue VM could do an arbitrary number of attach
1466 * and detach operations causing memory pressure in the host
1470 ev.msg.hdr.dst = vmci_make_handle(peer_id, VMCI_EVENT_HANDLER);
1471 ev.msg.hdr.src = vmci_make_handle(VMCI_HYPERVISOR_CONTEXT_ID,
1472 VMCI_CONTEXT_RESOURCE_ID);
1473 ev.msg.hdr.payload_size = sizeof(ev) - sizeof(ev.msg.hdr);
1474 ev.msg.event_data.event = attach ?
1475 VMCI_EVENT_QP_PEER_ATTACH : VMCI_EVENT_QP_PEER_DETACH;
1476 ev.payload.handle = handle;
1477 ev.payload.peer_id = my_id;
1479 rv = vmci_datagram_dispatch(VMCI_HYPERVISOR_CONTEXT_ID,
1480 &ev.msg.hdr, false);
1481 if (rv < VMCI_SUCCESS)
1482 pr_warn("Failed to enqueue queue_pair %s event datagram for context (ID=0x%x)\n",
1483 attach ? "ATTACH" : "DETACH", peer_id);
1489 * The second endpoint issuing a queue pair allocation will attach to
1490 * the queue pair registered with the queue pair broker.
1492 * If the attacher is a guest, it will associate a VMX virtual address
1493 * range with the queue pair as specified by the page_store. At this
1494 * point, the already attach host endpoint may start using the queue
1495 * pair, and an attach event is sent to it. For compatibility with
1496 * older VMX'en, that used a separate step to set the VMX virtual
1497 * address range, the virtual address range can be registered later
1498 * using vmci_qp_broker_set_page_store. In that case, a page_store of
1499 * NULL should be used, and the attach event will be generated once
1500 * the actual page store has been set.
1502 * If the attacher is the host, a page_store of NULL should be used as
1503 * well, since the page store information is already set by the guest.
1505 * For new VMX and host callers, the queue pair will be moved to the
1506 * VMCIQPB_ATTACHED_MEM state, and for older VMX callers, it will be
1507 * moved to the VMCOQPB_ATTACHED_NO_MEM state.
1509 static int qp_broker_attach(struct qp_broker_entry *entry,
1515 struct vmci_qp_page_store *page_store,
1516 struct vmci_ctx *context,
1517 vmci_event_release_cb wakeup_cb,
1519 struct qp_broker_entry **ent)
1521 const u32 context_id = vmci_ctx_get_id(context);
1522 bool is_local = flags & VMCI_QPFLAG_LOCAL;
1525 if (entry->state != VMCIQPB_CREATED_NO_MEM &&
1526 entry->state != VMCIQPB_CREATED_MEM)
1527 return VMCI_ERROR_UNAVAILABLE;
1530 if (!(entry->qp.flags & VMCI_QPFLAG_LOCAL) ||
1531 context_id != entry->create_id) {
1532 return VMCI_ERROR_INVALID_ARGS;
1534 } else if (context_id == entry->create_id ||
1535 context_id == entry->attach_id) {
1536 return VMCI_ERROR_ALREADY_EXISTS;
1539 if (VMCI_CONTEXT_IS_VM(context_id) &&
1540 VMCI_CONTEXT_IS_VM(entry->create_id))
1541 return VMCI_ERROR_DST_UNREACHABLE;
1544 * If we are attaching from a restricted context then the queuepair
1545 * must have been created by a trusted endpoint.
1547 if ((context->priv_flags & VMCI_PRIVILEGE_FLAG_RESTRICTED) &&
1548 !entry->created_by_trusted)
1549 return VMCI_ERROR_NO_ACCESS;
1552 * If we are attaching to a queuepair that was created by a restricted
1553 * context then we must be trusted.
1555 if (entry->require_trusted_attach &&
1556 (!(priv_flags & VMCI_PRIVILEGE_FLAG_TRUSTED)))
1557 return VMCI_ERROR_NO_ACCESS;
1560 * If the creator specifies VMCI_INVALID_ID in "peer" field, access
1561 * control check is not performed.
1563 if (entry->qp.peer != VMCI_INVALID_ID && entry->qp.peer != context_id)
1564 return VMCI_ERROR_NO_ACCESS;
1566 if (entry->create_id == VMCI_HOST_CONTEXT_ID) {
1568 * Do not attach if the caller doesn't support Host Queue Pairs
1569 * and a host created this queue pair.
1572 if (!vmci_ctx_supports_host_qp(context))
1573 return VMCI_ERROR_INVALID_RESOURCE;
1575 } else if (context_id == VMCI_HOST_CONTEXT_ID) {
1576 struct vmci_ctx *create_context;
1577 bool supports_host_qp;
1580 * Do not attach a host to a user created queue pair if that
1581 * user doesn't support host queue pair end points.
1584 create_context = vmci_ctx_get(entry->create_id);
1585 supports_host_qp = vmci_ctx_supports_host_qp(create_context);
1586 vmci_ctx_put(create_context);
1588 if (!supports_host_qp)
1589 return VMCI_ERROR_INVALID_RESOURCE;
1592 if ((entry->qp.flags & ~VMCI_QP_ASYMM) != (flags & ~VMCI_QP_ASYMM_PEER))
1593 return VMCI_ERROR_QUEUEPAIR_MISMATCH;
1595 if (context_id != VMCI_HOST_CONTEXT_ID) {
1597 * The queue pair broker entry stores values from the guest
1598 * point of view, so an attaching guest should match the values
1599 * stored in the entry.
1602 if (entry->qp.produce_size != produce_size ||
1603 entry->qp.consume_size != consume_size) {
1604 return VMCI_ERROR_QUEUEPAIR_MISMATCH;
1606 } else if (entry->qp.produce_size != consume_size ||
1607 entry->qp.consume_size != produce_size) {
1608 return VMCI_ERROR_QUEUEPAIR_MISMATCH;
1611 if (context_id != VMCI_HOST_CONTEXT_ID) {
1613 * If a guest attached to a queue pair, it will supply
1614 * the backing memory. If this is a pre NOVMVM vmx,
1615 * the backing memory will be supplied by calling
1616 * vmci_qp_broker_set_page_store() following the
1617 * return of the vmci_qp_broker_alloc() call. If it is
1618 * a vmx of version NOVMVM or later, the page store
1619 * must be supplied as part of the
1620 * vmci_qp_broker_alloc call. Under all circumstances
1621 * must the initially created queue pair not have any
1622 * memory associated with it already.
1625 if (entry->state != VMCIQPB_CREATED_NO_MEM)
1626 return VMCI_ERROR_INVALID_ARGS;
1628 if (page_store != NULL) {
1630 * Patch up host state to point to guest
1631 * supplied memory. The VMX already
1632 * initialized the queue pair headers, so no
1633 * need for the kernel side to do that.
1636 result = qp_host_register_user_memory(page_store,
1639 if (result < VMCI_SUCCESS)
1642 entry->state = VMCIQPB_ATTACHED_MEM;
1644 entry->state = VMCIQPB_ATTACHED_NO_MEM;
1646 } else if (entry->state == VMCIQPB_CREATED_NO_MEM) {
1648 * The host side is attempting to attach to a queue
1649 * pair that doesn't have any memory associated with
1650 * it. This must be a pre NOVMVM vmx that hasn't set
1651 * the page store information yet, or a quiesced VM.
1654 return VMCI_ERROR_UNAVAILABLE;
1656 /* The host side has successfully attached to a queue pair. */
1657 entry->state = VMCIQPB_ATTACHED_MEM;
1660 if (entry->state == VMCIQPB_ATTACHED_MEM) {
1662 qp_notify_peer(true, entry->qp.handle, context_id,
1664 if (result < VMCI_SUCCESS)
1665 pr_warn("Failed to notify peer (ID=0x%x) of attach to queue pair (handle=0x%x:0x%x)\n",
1666 entry->create_id, entry->qp.handle.context,
1667 entry->qp.handle.resource);
1670 entry->attach_id = context_id;
1671 entry->qp.ref_count++;
1673 entry->wakeup_cb = wakeup_cb;
1674 entry->client_data = client_data;
1678 * When attaching to local queue pairs, the context already has
1679 * an entry tracking the queue pair, so don't add another one.
1682 vmci_ctx_qp_create(context, entry->qp.handle);
1687 return VMCI_SUCCESS;
1691 * queue_pair_Alloc for use when setting up queue pair endpoints
1694 static int qp_broker_alloc(struct vmci_handle handle,
1700 struct vmci_qp_page_store *page_store,
1701 struct vmci_ctx *context,
1702 vmci_event_release_cb wakeup_cb,
1704 struct qp_broker_entry **ent,
1707 const u32 context_id = vmci_ctx_get_id(context);
1709 struct qp_broker_entry *entry = NULL;
1710 bool is_local = flags & VMCI_QPFLAG_LOCAL;
1713 if (vmci_handle_is_invalid(handle) ||
1714 (flags & ~VMCI_QP_ALL_FLAGS) || is_local ||
1715 !(produce_size || consume_size) ||
1716 !context || context_id == VMCI_INVALID_ID ||
1717 handle.context == VMCI_INVALID_ID) {
1718 return VMCI_ERROR_INVALID_ARGS;
1721 if (page_store && !VMCI_QP_PAGESTORE_IS_WELLFORMED(page_store))
1722 return VMCI_ERROR_INVALID_ARGS;
1725 * In the initial argument check, we ensure that non-vmkernel hosts
1726 * are not allowed to create local queue pairs.
1729 mutex_lock(&qp_broker_list.mutex);
1731 if (!is_local && vmci_ctx_qp_exists(context, handle)) {
1732 pr_devel("Context (ID=0x%x) already attached to queue pair (handle=0x%x:0x%x)\n",
1733 context_id, handle.context, handle.resource);
1734 mutex_unlock(&qp_broker_list.mutex);
1735 return VMCI_ERROR_ALREADY_EXISTS;
1738 if (handle.resource != VMCI_INVALID_ID)
1739 entry = qp_broker_handle_to_entry(handle);
1744 qp_broker_create(handle, peer, flags, priv_flags,
1745 produce_size, consume_size, page_store,
1746 context, wakeup_cb, client_data, ent);
1750 qp_broker_attach(entry, peer, flags, priv_flags,
1751 produce_size, consume_size, page_store,
1752 context, wakeup_cb, client_data, ent);
1755 mutex_unlock(&qp_broker_list.mutex);
1758 *swap = (context_id == VMCI_HOST_CONTEXT_ID) &&
1759 !(create && is_local);
1765 * This function implements the kernel API for allocating a queue
1768 static int qp_alloc_host_work(struct vmci_handle *handle,
1769 struct vmci_queue **produce_q,
1771 struct vmci_queue **consume_q,
1776 vmci_event_release_cb wakeup_cb,
1779 struct vmci_handle new_handle;
1780 struct vmci_ctx *context;
1781 struct qp_broker_entry *entry;
1785 if (vmci_handle_is_invalid(*handle)) {
1786 new_handle = vmci_make_handle(
1787 VMCI_HOST_CONTEXT_ID, VMCI_INVALID_ID);
1789 new_handle = *handle;
1791 context = vmci_ctx_get(VMCI_HOST_CONTEXT_ID);
1794 qp_broker_alloc(new_handle, peer, flags, priv_flags,
1795 produce_size, consume_size, NULL, context,
1796 wakeup_cb, client_data, &entry, &swap);
1797 if (result == VMCI_SUCCESS) {
1800 * If this is a local queue pair, the attacher
1801 * will swap around produce and consume
1805 *produce_q = entry->consume_q;
1806 *consume_q = entry->produce_q;
1808 *produce_q = entry->produce_q;
1809 *consume_q = entry->consume_q;
1812 *handle = vmci_resource_handle(&entry->resource);
1814 *handle = VMCI_INVALID_HANDLE;
1815 pr_devel("queue pair broker failed to alloc (result=%d)\n",
1818 vmci_ctx_put(context);
1823 * Allocates a VMCI queue_pair. Only checks validity of input
1824 * arguments. The real work is done in the host or guest
1825 * specific function.
1827 int vmci_qp_alloc(struct vmci_handle *handle,
1828 struct vmci_queue **produce_q,
1830 struct vmci_queue **consume_q,
1835 bool guest_endpoint,
1836 vmci_event_release_cb wakeup_cb,
1839 if (!handle || !produce_q || !consume_q ||
1840 (!produce_size && !consume_size) || (flags & ~VMCI_QP_ALL_FLAGS))
1841 return VMCI_ERROR_INVALID_ARGS;
1843 if (guest_endpoint) {
1844 return qp_alloc_guest_work(handle, produce_q,
1845 produce_size, consume_q,
1849 return qp_alloc_host_work(handle, produce_q,
1850 produce_size, consume_q,
1851 consume_size, peer, flags,
1852 priv_flags, wakeup_cb, client_data);
1857 * This function implements the host kernel API for detaching from
1860 static int qp_detatch_host_work(struct vmci_handle handle)
1863 struct vmci_ctx *context;
1865 context = vmci_ctx_get(VMCI_HOST_CONTEXT_ID);
1867 result = vmci_qp_broker_detach(handle, context);
1869 vmci_ctx_put(context);
1874 * Detaches from a VMCI queue_pair. Only checks validity of input argument.
1875 * Real work is done in the host or guest specific function.
1877 static int qp_detatch(struct vmci_handle handle, bool guest_endpoint)
1879 if (vmci_handle_is_invalid(handle))
1880 return VMCI_ERROR_INVALID_ARGS;
1883 return qp_detatch_guest_work(handle);
1885 return qp_detatch_host_work(handle);
1889 * Returns the entry from the head of the list. Assumes that the list is
1892 static struct qp_entry *qp_list_get_head(struct qp_list *qp_list)
1894 if (!list_empty(&qp_list->head)) {
1895 struct qp_entry *entry =
1896 list_first_entry(&qp_list->head, struct qp_entry,
1904 void vmci_qp_broker_exit(void)
1906 struct qp_entry *entry;
1907 struct qp_broker_entry *be;
1909 mutex_lock(&qp_broker_list.mutex);
1911 while ((entry = qp_list_get_head(&qp_broker_list))) {
1912 be = (struct qp_broker_entry *)entry;
1914 qp_list_remove_entry(&qp_broker_list, entry);
1918 mutex_unlock(&qp_broker_list.mutex);
1922 * Requests that a queue pair be allocated with the VMCI queue
1923 * pair broker. Allocates a queue pair entry if one does not
1924 * exist. Attaches to one if it exists, and retrieves the page
1925 * files backing that queue_pair. Assumes that the queue pair
1926 * broker lock is held.
1928 int vmci_qp_broker_alloc(struct vmci_handle handle,
1934 struct vmci_qp_page_store *page_store,
1935 struct vmci_ctx *context)
1937 if (!QP_SIZES_ARE_VALID(produce_size, consume_size))
1938 return VMCI_ERROR_NO_RESOURCES;
1940 return qp_broker_alloc(handle, peer, flags, priv_flags,
1941 produce_size, consume_size,
1942 page_store, context, NULL, NULL, NULL, NULL);
1946 * VMX'en with versions lower than VMCI_VERSION_NOVMVM use a separate
1947 * step to add the UVAs of the VMX mapping of the queue pair. This function
1948 * provides backwards compatibility with such VMX'en, and takes care of
1949 * registering the page store for a queue pair previously allocated by the
1950 * VMX during create or attach. This function will move the queue pair state
1951 * to either from VMCIQBP_CREATED_NO_MEM to VMCIQBP_CREATED_MEM or
1952 * VMCIQBP_ATTACHED_NO_MEM to VMCIQBP_ATTACHED_MEM. If moving to the
1953 * attached state with memory, the queue pair is ready to be used by the
1954 * host peer, and an attached event will be generated.
1956 * Assumes that the queue pair broker lock is held.
1958 * This function is only used by the hosted platform, since there is no
1959 * issue with backwards compatibility for vmkernel.
1961 int vmci_qp_broker_set_page_store(struct vmci_handle handle,
1964 struct vmci_ctx *context)
1966 struct qp_broker_entry *entry;
1968 const u32 context_id = vmci_ctx_get_id(context);
1970 if (vmci_handle_is_invalid(handle) || !context ||
1971 context_id == VMCI_INVALID_ID)
1972 return VMCI_ERROR_INVALID_ARGS;
1975 * We only support guest to host queue pairs, so the VMX must
1976 * supply UVAs for the mapped page files.
1979 if (produce_uva == 0 || consume_uva == 0)
1980 return VMCI_ERROR_INVALID_ARGS;
1982 mutex_lock(&qp_broker_list.mutex);
1984 if (!vmci_ctx_qp_exists(context, handle)) {
1985 pr_warn("Context (ID=0x%x) not attached to queue pair (handle=0x%x:0x%x)\n",
1986 context_id, handle.context, handle.resource);
1987 result = VMCI_ERROR_NOT_FOUND;
1991 entry = qp_broker_handle_to_entry(handle);
1993 result = VMCI_ERROR_NOT_FOUND;
1998 * If I'm the owner then I can set the page store.
2000 * Or, if a host created the queue_pair and I'm the attached peer
2001 * then I can set the page store.
2003 if (entry->create_id != context_id &&
2004 (entry->create_id != VMCI_HOST_CONTEXT_ID ||
2005 entry->attach_id != context_id)) {
2006 result = VMCI_ERROR_QUEUEPAIR_NOTOWNER;
2010 if (entry->state != VMCIQPB_CREATED_NO_MEM &&
2011 entry->state != VMCIQPB_ATTACHED_NO_MEM) {
2012 result = VMCI_ERROR_UNAVAILABLE;
2016 result = qp_host_get_user_memory(produce_uva, consume_uva,
2017 entry->produce_q, entry->consume_q);
2018 if (result < VMCI_SUCCESS)
2021 result = qp_host_map_queues(entry->produce_q, entry->consume_q);
2022 if (result < VMCI_SUCCESS) {
2023 qp_host_unregister_user_memory(entry->produce_q,
2028 if (entry->state == VMCIQPB_CREATED_NO_MEM)
2029 entry->state = VMCIQPB_CREATED_MEM;
2031 entry->state = VMCIQPB_ATTACHED_MEM;
2033 entry->vmci_page_files = true;
2035 if (entry->state == VMCIQPB_ATTACHED_MEM) {
2037 qp_notify_peer(true, handle, context_id, entry->create_id);
2038 if (result < VMCI_SUCCESS) {
2039 pr_warn("Failed to notify peer (ID=0x%x) of attach to queue pair (handle=0x%x:0x%x)\n",
2040 entry->create_id, entry->qp.handle.context,
2041 entry->qp.handle.resource);
2045 result = VMCI_SUCCESS;
2047 mutex_unlock(&qp_broker_list.mutex);
2052 * Resets saved queue headers for the given QP broker
2053 * entry. Should be used when guest memory becomes available
2054 * again, or the guest detaches.
2056 static void qp_reset_saved_headers(struct qp_broker_entry *entry)
2058 entry->produce_q->saved_header = NULL;
2059 entry->consume_q->saved_header = NULL;
2063 * The main entry point for detaching from a queue pair registered with the
2064 * queue pair broker. If more than one endpoint is attached to the queue
2065 * pair, the first endpoint will mainly decrement a reference count and
2066 * generate a notification to its peer. The last endpoint will clean up
2067 * the queue pair state registered with the broker.
2069 * When a guest endpoint detaches, it will unmap and unregister the guest
2070 * memory backing the queue pair. If the host is still attached, it will
2071 * no longer be able to access the queue pair content.
2073 * If the queue pair is already in a state where there is no memory
2074 * registered for the queue pair (any *_NO_MEM state), it will transition to
2075 * the VMCIQPB_SHUTDOWN_NO_MEM state. This will also happen, if a guest
2076 * endpoint is the first of two endpoints to detach. If the host endpoint is
2077 * the first out of two to detach, the queue pair will move to the
2078 * VMCIQPB_SHUTDOWN_MEM state.
2080 int vmci_qp_broker_detach(struct vmci_handle handle, struct vmci_ctx *context)
2082 struct qp_broker_entry *entry;
2083 const u32 context_id = vmci_ctx_get_id(context);
2085 bool is_local = false;
2088 if (vmci_handle_is_invalid(handle) || !context ||
2089 context_id == VMCI_INVALID_ID) {
2090 return VMCI_ERROR_INVALID_ARGS;
2093 mutex_lock(&qp_broker_list.mutex);
2095 if (!vmci_ctx_qp_exists(context, handle)) {
2096 pr_devel("Context (ID=0x%x) not attached to queue pair (handle=0x%x:0x%x)\n",
2097 context_id, handle.context, handle.resource);
2098 result = VMCI_ERROR_NOT_FOUND;
2102 entry = qp_broker_handle_to_entry(handle);
2104 pr_devel("Context (ID=0x%x) reports being attached to queue pair(handle=0x%x:0x%x) that isn't present in broker\n",
2105 context_id, handle.context, handle.resource);
2106 result = VMCI_ERROR_NOT_FOUND;
2110 if (context_id != entry->create_id && context_id != entry->attach_id) {
2111 result = VMCI_ERROR_QUEUEPAIR_NOTATTACHED;
2115 if (context_id == entry->create_id) {
2116 peer_id = entry->attach_id;
2117 entry->create_id = VMCI_INVALID_ID;
2119 peer_id = entry->create_id;
2120 entry->attach_id = VMCI_INVALID_ID;
2122 entry->qp.ref_count--;
2124 is_local = entry->qp.flags & VMCI_QPFLAG_LOCAL;
2126 if (context_id != VMCI_HOST_CONTEXT_ID) {
2127 bool headers_mapped;
2130 * Pre NOVMVM vmx'en may detach from a queue pair
2131 * before setting the page store, and in that case
2132 * there is no user memory to detach from. Also, more
2133 * recent VMX'en may detach from a queue pair in the
2137 qp_acquire_queue_mutex(entry->produce_q);
2138 headers_mapped = entry->produce_q->q_header ||
2139 entry->consume_q->q_header;
2140 if (QPBROKERSTATE_HAS_MEM(entry)) {
2142 qp_host_unmap_queues(INVALID_VMCI_GUEST_MEM_ID,
2145 if (result < VMCI_SUCCESS)
2146 pr_warn("Failed to unmap queue headers for queue pair (handle=0x%x:0x%x,result=%d)\n",
2147 handle.context, handle.resource,
2150 qp_host_unregister_user_memory(entry->produce_q,
2155 if (!headers_mapped)
2156 qp_reset_saved_headers(entry);
2158 qp_release_queue_mutex(entry->produce_q);
2160 if (!headers_mapped && entry->wakeup_cb)
2161 entry->wakeup_cb(entry->client_data);
2164 if (entry->wakeup_cb) {
2165 entry->wakeup_cb = NULL;
2166 entry->client_data = NULL;
2170 if (entry->qp.ref_count == 0) {
2171 qp_list_remove_entry(&qp_broker_list, &entry->qp);
2174 kfree(entry->local_mem);
2176 qp_cleanup_queue_mutex(entry->produce_q, entry->consume_q);
2177 qp_host_free_queue(entry->produce_q, entry->qp.produce_size);
2178 qp_host_free_queue(entry->consume_q, entry->qp.consume_size);
2179 /* Unlink from resource hash table and free callback */
2180 vmci_resource_remove(&entry->resource);
2184 vmci_ctx_qp_destroy(context, handle);
2186 qp_notify_peer(false, handle, context_id, peer_id);
2187 if (context_id == VMCI_HOST_CONTEXT_ID &&
2188 QPBROKERSTATE_HAS_MEM(entry)) {
2189 entry->state = VMCIQPB_SHUTDOWN_MEM;
2191 entry->state = VMCIQPB_SHUTDOWN_NO_MEM;
2195 vmci_ctx_qp_destroy(context, handle);
2198 result = VMCI_SUCCESS;
2200 mutex_unlock(&qp_broker_list.mutex);
2205 * Establishes the necessary mappings for a queue pair given a
2206 * reference to the queue pair guest memory. This is usually
2207 * called when a guest is unquiesced and the VMX is allowed to
2208 * map guest memory once again.
2210 int vmci_qp_broker_map(struct vmci_handle handle,
2211 struct vmci_ctx *context,
2214 struct qp_broker_entry *entry;
2215 const u32 context_id = vmci_ctx_get_id(context);
2218 if (vmci_handle_is_invalid(handle) || !context ||
2219 context_id == VMCI_INVALID_ID)
2220 return VMCI_ERROR_INVALID_ARGS;
2222 mutex_lock(&qp_broker_list.mutex);
2224 if (!vmci_ctx_qp_exists(context, handle)) {
2225 pr_devel("Context (ID=0x%x) not attached to queue pair (handle=0x%x:0x%x)\n",
2226 context_id, handle.context, handle.resource);
2227 result = VMCI_ERROR_NOT_FOUND;
2231 entry = qp_broker_handle_to_entry(handle);
2233 pr_devel("Context (ID=0x%x) reports being attached to queue pair (handle=0x%x:0x%x) that isn't present in broker\n",
2234 context_id, handle.context, handle.resource);
2235 result = VMCI_ERROR_NOT_FOUND;
2239 if (context_id != entry->create_id && context_id != entry->attach_id) {
2240 result = VMCI_ERROR_QUEUEPAIR_NOTATTACHED;
2244 result = VMCI_SUCCESS;
2246 if (context_id != VMCI_HOST_CONTEXT_ID &&
2247 !QPBROKERSTATE_HAS_MEM(entry)) {
2248 struct vmci_qp_page_store page_store;
2250 page_store.pages = guest_mem;
2251 page_store.len = QPE_NUM_PAGES(entry->qp);
2253 qp_acquire_queue_mutex(entry->produce_q);
2254 qp_reset_saved_headers(entry);
2256 qp_host_register_user_memory(&page_store,
2259 qp_release_queue_mutex(entry->produce_q);
2260 if (result == VMCI_SUCCESS) {
2261 /* Move state from *_NO_MEM to *_MEM */
2265 if (entry->wakeup_cb)
2266 entry->wakeup_cb(entry->client_data);
2271 mutex_unlock(&qp_broker_list.mutex);
2276 * Saves a snapshot of the queue headers for the given QP broker
2277 * entry. Should be used when guest memory is unmapped.
2279 * VMCI_SUCCESS on success, appropriate error code if guest memory
2280 * can't be accessed..
2282 static int qp_save_headers(struct qp_broker_entry *entry)
2286 if (entry->produce_q->saved_header != NULL &&
2287 entry->consume_q->saved_header != NULL) {
2289 * If the headers have already been saved, we don't need to do
2290 * it again, and we don't want to map in the headers
2294 return VMCI_SUCCESS;
2297 if (NULL == entry->produce_q->q_header ||
2298 NULL == entry->consume_q->q_header) {
2299 result = qp_host_map_queues(entry->produce_q, entry->consume_q);
2300 if (result < VMCI_SUCCESS)
2304 memcpy(&entry->saved_produce_q, entry->produce_q->q_header,
2305 sizeof(entry->saved_produce_q));
2306 entry->produce_q->saved_header = &entry->saved_produce_q;
2307 memcpy(&entry->saved_consume_q, entry->consume_q->q_header,
2308 sizeof(entry->saved_consume_q));
2309 entry->consume_q->saved_header = &entry->saved_consume_q;
2311 return VMCI_SUCCESS;
2315 * Removes all references to the guest memory of a given queue pair, and
2316 * will move the queue pair from state *_MEM to *_NO_MEM. It is usually
2317 * called when a VM is being quiesced where access to guest memory should
2320 int vmci_qp_broker_unmap(struct vmci_handle handle,
2321 struct vmci_ctx *context,
2324 struct qp_broker_entry *entry;
2325 const u32 context_id = vmci_ctx_get_id(context);
2328 if (vmci_handle_is_invalid(handle) || !context ||
2329 context_id == VMCI_INVALID_ID)
2330 return VMCI_ERROR_INVALID_ARGS;
2332 mutex_lock(&qp_broker_list.mutex);
2334 if (!vmci_ctx_qp_exists(context, handle)) {
2335 pr_devel("Context (ID=0x%x) not attached to queue pair (handle=0x%x:0x%x)\n",
2336 context_id, handle.context, handle.resource);
2337 result = VMCI_ERROR_NOT_FOUND;
2341 entry = qp_broker_handle_to_entry(handle);
2343 pr_devel("Context (ID=0x%x) reports being attached to queue pair (handle=0x%x:0x%x) that isn't present in broker\n",
2344 context_id, handle.context, handle.resource);
2345 result = VMCI_ERROR_NOT_FOUND;
2349 if (context_id != entry->create_id && context_id != entry->attach_id) {
2350 result = VMCI_ERROR_QUEUEPAIR_NOTATTACHED;
2354 if (context_id != VMCI_HOST_CONTEXT_ID &&
2355 QPBROKERSTATE_HAS_MEM(entry)) {
2356 qp_acquire_queue_mutex(entry->produce_q);
2357 result = qp_save_headers(entry);
2358 if (result < VMCI_SUCCESS)
2359 pr_warn("Failed to save queue headers for queue pair (handle=0x%x:0x%x,result=%d)\n",
2360 handle.context, handle.resource, result);
2362 qp_host_unmap_queues(gid, entry->produce_q, entry->consume_q);
2365 * On hosted, when we unmap queue pairs, the VMX will also
2366 * unmap the guest memory, so we invalidate the previously
2367 * registered memory. If the queue pair is mapped again at a
2368 * later point in time, we will need to reregister the user
2369 * memory with a possibly new user VA.
2371 qp_host_unregister_user_memory(entry->produce_q,
2375 * Move state from *_MEM to *_NO_MEM.
2379 qp_release_queue_mutex(entry->produce_q);
2382 result = VMCI_SUCCESS;
2385 mutex_unlock(&qp_broker_list.mutex);
2390 * Destroys all guest queue pair endpoints. If active guest queue
2391 * pairs still exist, hypercalls to attempt detach from these
2392 * queue pairs will be made. Any failure to detach is silently
2395 void vmci_qp_guest_endpoints_exit(void)
2397 struct qp_entry *entry;
2398 struct qp_guest_endpoint *ep;
2400 mutex_lock(&qp_guest_endpoints.mutex);
2402 while ((entry = qp_list_get_head(&qp_guest_endpoints))) {
2403 ep = (struct qp_guest_endpoint *)entry;
2405 /* Don't make a hypercall for local queue_pairs. */
2406 if (!(entry->flags & VMCI_QPFLAG_LOCAL))
2407 qp_detatch_hypercall(entry->handle);
2409 /* We cannot fail the exit, so let's reset ref_count. */
2410 entry->ref_count = 0;
2411 qp_list_remove_entry(&qp_guest_endpoints, entry);
2413 qp_guest_endpoint_destroy(ep);
2416 mutex_unlock(&qp_guest_endpoints.mutex);
2420 * Helper routine that will lock the queue pair before subsequent
2422 * Note: Non-blocking on the host side is currently only implemented in ESX.
2423 * Since non-blocking isn't yet implemented on the host personality we
2424 * have no reason to acquire a spin lock. So to avoid the use of an
2425 * unnecessary lock only acquire the mutex if we can block.
2427 static void qp_lock(const struct vmci_qp *qpair)
2429 qp_acquire_queue_mutex(qpair->produce_q);
2433 * Helper routine that unlocks the queue pair after calling
2436 static void qp_unlock(const struct vmci_qp *qpair)
2438 qp_release_queue_mutex(qpair->produce_q);
2442 * The queue headers may not be mapped at all times. If a queue is
2443 * currently not mapped, it will be attempted to do so.
2445 static int qp_map_queue_headers(struct vmci_queue *produce_q,
2446 struct vmci_queue *consume_q)
2450 if (NULL == produce_q->q_header || NULL == consume_q->q_header) {
2451 result = qp_host_map_queues(produce_q, consume_q);
2452 if (result < VMCI_SUCCESS)
2453 return (produce_q->saved_header &&
2454 consume_q->saved_header) ?
2455 VMCI_ERROR_QUEUEPAIR_NOT_READY :
2456 VMCI_ERROR_QUEUEPAIR_NOTATTACHED;
2459 return VMCI_SUCCESS;
2463 * Helper routine that will retrieve the produce and consume
2464 * headers of a given queue pair. If the guest memory of the
2465 * queue pair is currently not available, the saved queue headers
2466 * will be returned, if these are available.
2468 static int qp_get_queue_headers(const struct vmci_qp *qpair,
2469 struct vmci_queue_header **produce_q_header,
2470 struct vmci_queue_header **consume_q_header)
2474 result = qp_map_queue_headers(qpair->produce_q, qpair->consume_q);
2475 if (result == VMCI_SUCCESS) {
2476 *produce_q_header = qpair->produce_q->q_header;
2477 *consume_q_header = qpair->consume_q->q_header;
2478 } else if (qpair->produce_q->saved_header &&
2479 qpair->consume_q->saved_header) {
2480 *produce_q_header = qpair->produce_q->saved_header;
2481 *consume_q_header = qpair->consume_q->saved_header;
2482 result = VMCI_SUCCESS;
2489 * Callback from VMCI queue pair broker indicating that a queue
2490 * pair that was previously not ready, now either is ready or
2493 static int qp_wakeup_cb(void *client_data)
2495 struct vmci_qp *qpair = (struct vmci_qp *)client_data;
2498 while (qpair->blocked > 0) {
2500 qpair->generation++;
2501 wake_up(&qpair->event);
2505 return VMCI_SUCCESS;
2509 * Makes the calling thread wait for the queue pair to become
2510 * ready for host side access. Returns true when thread is
2511 * woken up after queue pair state change, false otherwise.
2513 static bool qp_wait_for_ready_queue(struct vmci_qp *qpair)
2515 unsigned int generation;
2518 generation = qpair->generation;
2520 wait_event(qpair->event, generation != qpair->generation);
2527 * Enqueues a given buffer to the produce queue using the provided
2528 * function. As many bytes as possible (space available in the queue)
2529 * are enqueued. Assumes the queue->mutex has been acquired. Returns
2530 * VMCI_ERROR_QUEUEPAIR_NOSPACE if no space was available to enqueue
2531 * data, VMCI_ERROR_INVALID_SIZE, if any queue pointer is outside the
2532 * queue (as defined by the queue size), VMCI_ERROR_INVALID_ARGS, if
2533 * an error occured when accessing the buffer,
2534 * VMCI_ERROR_QUEUEPAIR_NOTATTACHED, if the queue pair pages aren't
2535 * available. Otherwise, the number of bytes written to the queue is
2536 * returned. Updates the tail pointer of the produce queue.
2538 static ssize_t qp_enqueue_locked(struct vmci_queue *produce_q,
2539 struct vmci_queue *consume_q,
2540 const u64 produce_q_size,
2541 struct iov_iter *from)
2545 size_t buf_size = iov_iter_count(from);
2549 result = qp_map_queue_headers(produce_q, consume_q);
2550 if (unlikely(result != VMCI_SUCCESS))
2553 free_space = vmci_q_header_free_space(produce_q->q_header,
2554 consume_q->q_header,
2556 if (free_space == 0)
2557 return VMCI_ERROR_QUEUEPAIR_NOSPACE;
2559 if (free_space < VMCI_SUCCESS)
2560 return (ssize_t) free_space;
2562 written = (size_t) (free_space > buf_size ? buf_size : free_space);
2563 tail = vmci_q_header_producer_tail(produce_q->q_header);
2564 if (likely(tail + written < produce_q_size)) {
2565 result = qp_memcpy_to_queue_iter(produce_q, tail, from, written);
2567 /* Tail pointer wraps around. */
2569 const size_t tmp = (size_t) (produce_q_size - tail);
2571 result = qp_memcpy_to_queue_iter(produce_q, tail, from, tmp);
2572 if (result >= VMCI_SUCCESS)
2573 result = qp_memcpy_to_queue_iter(produce_q, 0, from,
2577 if (result < VMCI_SUCCESS)
2581 * This virt_wmb() ensures that data written to the queue
2582 * is observable before the new producer_tail is.
2586 vmci_q_header_add_producer_tail(produce_q->q_header, written,
2592 * Dequeues data (if available) from the given consume queue. Writes data
2593 * to the user provided buffer using the provided function.
2594 * Assumes the queue->mutex has been acquired.
2596 * VMCI_ERROR_QUEUEPAIR_NODATA if no data was available to dequeue.
2597 * VMCI_ERROR_INVALID_SIZE, if any queue pointer is outside the queue
2598 * (as defined by the queue size).
2599 * VMCI_ERROR_INVALID_ARGS, if an error occured when accessing the buffer.
2600 * Otherwise the number of bytes dequeued is returned.
2602 * Updates the head pointer of the consume queue.
2604 static ssize_t qp_dequeue_locked(struct vmci_queue *produce_q,
2605 struct vmci_queue *consume_q,
2606 const u64 consume_q_size,
2607 struct iov_iter *to,
2608 bool update_consumer)
2610 size_t buf_size = iov_iter_count(to);
2616 result = qp_map_queue_headers(produce_q, consume_q);
2617 if (unlikely(result != VMCI_SUCCESS))
2620 buf_ready = vmci_q_header_buf_ready(consume_q->q_header,
2621 produce_q->q_header,
2624 return VMCI_ERROR_QUEUEPAIR_NODATA;
2626 if (buf_ready < VMCI_SUCCESS)
2627 return (ssize_t) buf_ready;
2630 * This virt_rmb() ensures that data from the queue will be read
2631 * after we have determined how much is ready to be consumed.
2635 read = (size_t) (buf_ready > buf_size ? buf_size : buf_ready);
2636 head = vmci_q_header_consumer_head(produce_q->q_header);
2637 if (likely(head + read < consume_q_size)) {
2638 result = qp_memcpy_from_queue_iter(to, consume_q, head, read);
2640 /* Head pointer wraps around. */
2642 const size_t tmp = (size_t) (consume_q_size - head);
2644 result = qp_memcpy_from_queue_iter(to, consume_q, head, tmp);
2645 if (result >= VMCI_SUCCESS)
2646 result = qp_memcpy_from_queue_iter(to, consume_q, 0,
2651 if (result < VMCI_SUCCESS)
2654 if (update_consumer)
2655 vmci_q_header_add_consumer_head(produce_q->q_header,
2656 read, consume_q_size);
2662 * vmci_qpair_alloc() - Allocates a queue pair.
2663 * @qpair: Pointer for the new vmci_qp struct.
2664 * @handle: Handle to track the resource.
2665 * @produce_qsize: Desired size of the producer queue.
2666 * @consume_qsize: Desired size of the consumer queue.
2667 * @peer: ContextID of the peer.
2668 * @flags: VMCI flags.
2669 * @priv_flags: VMCI priviledge flags.
2671 * This is the client interface for allocating the memory for a
2672 * vmci_qp structure and then attaching to the underlying
2673 * queue. If an error occurs allocating the memory for the
2674 * vmci_qp structure no attempt is made to attach. If an
2675 * error occurs attaching, then the structure is freed.
2677 int vmci_qpair_alloc(struct vmci_qp **qpair,
2678 struct vmci_handle *handle,
2685 struct vmci_qp *my_qpair;
2687 struct vmci_handle src = VMCI_INVALID_HANDLE;
2688 struct vmci_handle dst = vmci_make_handle(peer, VMCI_INVALID_ID);
2689 enum vmci_route route;
2690 vmci_event_release_cb wakeup_cb;
2694 * Restrict the size of a queuepair. The device already
2695 * enforces a limit on the total amount of memory that can be
2696 * allocated to queuepairs for a guest. However, we try to
2697 * allocate this memory before we make the queuepair
2698 * allocation hypercall. On Linux, we allocate each page
2699 * separately, which means rather than fail, the guest will
2700 * thrash while it tries to allocate, and will become
2701 * increasingly unresponsive to the point where it appears to
2702 * be hung. So we place a limit on the size of an individual
2703 * queuepair here, and leave the device to enforce the
2704 * restriction on total queuepair memory. (Note that this
2705 * doesn't prevent all cases; a user with only this much
2706 * physical memory could still get into trouble.) The error
2707 * used by the device is NO_RESOURCES, so use that here too.
2710 if (!QP_SIZES_ARE_VALID(produce_qsize, consume_qsize))
2711 return VMCI_ERROR_NO_RESOURCES;
2713 retval = vmci_route(&src, &dst, false, &route);
2714 if (retval < VMCI_SUCCESS)
2715 route = vmci_guest_code_active() ?
2716 VMCI_ROUTE_AS_GUEST : VMCI_ROUTE_AS_HOST;
2718 if (flags & (VMCI_QPFLAG_NONBLOCK | VMCI_QPFLAG_PINNED)) {
2719 pr_devel("NONBLOCK OR PINNED set");
2720 return VMCI_ERROR_INVALID_ARGS;
2723 my_qpair = kzalloc(sizeof(*my_qpair), GFP_KERNEL);
2725 return VMCI_ERROR_NO_MEM;
2727 my_qpair->produce_q_size = produce_qsize;
2728 my_qpair->consume_q_size = consume_qsize;
2729 my_qpair->peer = peer;
2730 my_qpair->flags = flags;
2731 my_qpair->priv_flags = priv_flags;
2736 if (VMCI_ROUTE_AS_HOST == route) {
2737 my_qpair->guest_endpoint = false;
2738 if (!(flags & VMCI_QPFLAG_LOCAL)) {
2739 my_qpair->blocked = 0;
2740 my_qpair->generation = 0;
2741 init_waitqueue_head(&my_qpair->event);
2742 wakeup_cb = qp_wakeup_cb;
2743 client_data = (void *)my_qpair;
2746 my_qpair->guest_endpoint = true;
2749 retval = vmci_qp_alloc(handle,
2750 &my_qpair->produce_q,
2751 my_qpair->produce_q_size,
2752 &my_qpair->consume_q,
2753 my_qpair->consume_q_size,
2756 my_qpair->priv_flags,
2757 my_qpair->guest_endpoint,
2758 wakeup_cb, client_data);
2760 if (retval < VMCI_SUCCESS) {
2766 my_qpair->handle = *handle;
2770 EXPORT_SYMBOL_GPL(vmci_qpair_alloc);
2773 * vmci_qpair_detach() - Detatches the client from a queue pair.
2774 * @qpair: Reference of a pointer to the qpair struct.
2776 * This is the client interface for detaching from a VMCIQPair.
2777 * Note that this routine will free the memory allocated for the
2778 * vmci_qp structure too.
2780 int vmci_qpair_detach(struct vmci_qp **qpair)
2783 struct vmci_qp *old_qpair;
2785 if (!qpair || !(*qpair))
2786 return VMCI_ERROR_INVALID_ARGS;
2789 result = qp_detatch(old_qpair->handle, old_qpair->guest_endpoint);
2792 * The guest can fail to detach for a number of reasons, and
2793 * if it does so, it will cleanup the entry (if there is one).
2794 * The host can fail too, but it won't cleanup the entry
2795 * immediately, it will do that later when the context is
2796 * freed. Either way, we need to release the qpair struct
2797 * here; there isn't much the caller can do, and we don't want
2801 memset(old_qpair, 0, sizeof(*old_qpair));
2802 old_qpair->handle = VMCI_INVALID_HANDLE;
2803 old_qpair->peer = VMCI_INVALID_ID;
2809 EXPORT_SYMBOL_GPL(vmci_qpair_detach);
2812 * vmci_qpair_get_produce_indexes() - Retrieves the indexes of the producer.
2813 * @qpair: Pointer to the queue pair struct.
2814 * @producer_tail: Reference used for storing producer tail index.
2815 * @consumer_head: Reference used for storing the consumer head index.
2817 * This is the client interface for getting the current indexes of the
2818 * QPair from the point of the view of the caller as the producer.
2820 int vmci_qpair_get_produce_indexes(const struct vmci_qp *qpair,
2824 struct vmci_queue_header *produce_q_header;
2825 struct vmci_queue_header *consume_q_header;
2829 return VMCI_ERROR_INVALID_ARGS;
2833 qp_get_queue_headers(qpair, &produce_q_header, &consume_q_header);
2834 if (result == VMCI_SUCCESS)
2835 vmci_q_header_get_pointers(produce_q_header, consume_q_header,
2836 producer_tail, consumer_head);
2839 if (result == VMCI_SUCCESS &&
2840 ((producer_tail && *producer_tail >= qpair->produce_q_size) ||
2841 (consumer_head && *consumer_head >= qpair->produce_q_size)))
2842 return VMCI_ERROR_INVALID_SIZE;
2846 EXPORT_SYMBOL_GPL(vmci_qpair_get_produce_indexes);
2849 * vmci_qpair_get_consume_indexes() - Retrieves the indexes of the consumer.
2850 * @qpair: Pointer to the queue pair struct.
2851 * @consumer_tail: Reference used for storing consumer tail index.
2852 * @producer_head: Reference used for storing the producer head index.
2854 * This is the client interface for getting the current indexes of the
2855 * QPair from the point of the view of the caller as the consumer.
2857 int vmci_qpair_get_consume_indexes(const struct vmci_qp *qpair,
2861 struct vmci_queue_header *produce_q_header;
2862 struct vmci_queue_header *consume_q_header;
2866 return VMCI_ERROR_INVALID_ARGS;
2870 qp_get_queue_headers(qpair, &produce_q_header, &consume_q_header);
2871 if (result == VMCI_SUCCESS)
2872 vmci_q_header_get_pointers(consume_q_header, produce_q_header,
2873 consumer_tail, producer_head);
2876 if (result == VMCI_SUCCESS &&
2877 ((consumer_tail && *consumer_tail >= qpair->consume_q_size) ||
2878 (producer_head && *producer_head >= qpair->consume_q_size)))
2879 return VMCI_ERROR_INVALID_SIZE;
2883 EXPORT_SYMBOL_GPL(vmci_qpair_get_consume_indexes);
2886 * vmci_qpair_produce_free_space() - Retrieves free space in producer queue.
2887 * @qpair: Pointer to the queue pair struct.
2889 * This is the client interface for getting the amount of free
2890 * space in the QPair from the point of the view of the caller as
2891 * the producer which is the common case. Returns < 0 if err, else
2892 * available bytes into which data can be enqueued if > 0.
2894 s64 vmci_qpair_produce_free_space(const struct vmci_qp *qpair)
2896 struct vmci_queue_header *produce_q_header;
2897 struct vmci_queue_header *consume_q_header;
2901 return VMCI_ERROR_INVALID_ARGS;
2905 qp_get_queue_headers(qpair, &produce_q_header, &consume_q_header);
2906 if (result == VMCI_SUCCESS)
2907 result = vmci_q_header_free_space(produce_q_header,
2909 qpair->produce_q_size);
2917 EXPORT_SYMBOL_GPL(vmci_qpair_produce_free_space);
2920 * vmci_qpair_consume_free_space() - Retrieves free space in consumer queue.
2921 * @qpair: Pointer to the queue pair struct.
2923 * This is the client interface for getting the amount of free
2924 * space in the QPair from the point of the view of the caller as
2925 * the consumer which is not the common case. Returns < 0 if err, else
2926 * available bytes into which data can be enqueued if > 0.
2928 s64 vmci_qpair_consume_free_space(const struct vmci_qp *qpair)
2930 struct vmci_queue_header *produce_q_header;
2931 struct vmci_queue_header *consume_q_header;
2935 return VMCI_ERROR_INVALID_ARGS;
2939 qp_get_queue_headers(qpair, &produce_q_header, &consume_q_header);
2940 if (result == VMCI_SUCCESS)
2941 result = vmci_q_header_free_space(consume_q_header,
2943 qpair->consume_q_size);
2951 EXPORT_SYMBOL_GPL(vmci_qpair_consume_free_space);
2954 * vmci_qpair_produce_buf_ready() - Gets bytes ready to read from
2956 * @qpair: Pointer to the queue pair struct.
2958 * This is the client interface for getting the amount of
2959 * enqueued data in the QPair from the point of the view of the
2960 * caller as the producer which is not the common case. Returns < 0 if err,
2961 * else available bytes that may be read.
2963 s64 vmci_qpair_produce_buf_ready(const struct vmci_qp *qpair)
2965 struct vmci_queue_header *produce_q_header;
2966 struct vmci_queue_header *consume_q_header;
2970 return VMCI_ERROR_INVALID_ARGS;
2974 qp_get_queue_headers(qpair, &produce_q_header, &consume_q_header);
2975 if (result == VMCI_SUCCESS)
2976 result = vmci_q_header_buf_ready(produce_q_header,
2978 qpair->produce_q_size);
2986 EXPORT_SYMBOL_GPL(vmci_qpair_produce_buf_ready);
2989 * vmci_qpair_consume_buf_ready() - Gets bytes ready to read from
2991 * @qpair: Pointer to the queue pair struct.
2993 * This is the client interface for getting the amount of
2994 * enqueued data in the QPair from the point of the view of the
2995 * caller as the consumer which is the normal case. Returns < 0 if err,
2996 * else available bytes that may be read.
2998 s64 vmci_qpair_consume_buf_ready(const struct vmci_qp *qpair)
3000 struct vmci_queue_header *produce_q_header;
3001 struct vmci_queue_header *consume_q_header;
3005 return VMCI_ERROR_INVALID_ARGS;
3009 qp_get_queue_headers(qpair, &produce_q_header, &consume_q_header);
3010 if (result == VMCI_SUCCESS)
3011 result = vmci_q_header_buf_ready(consume_q_header,
3013 qpair->consume_q_size);
3021 EXPORT_SYMBOL_GPL(vmci_qpair_consume_buf_ready);
3024 * vmci_qpair_enqueue() - Throw data on the queue.
3025 * @qpair: Pointer to the queue pair struct.
3026 * @buf: Pointer to buffer containing data
3027 * @buf_size: Length of buffer.
3028 * @buf_type: Buffer type (Unused).
3030 * This is the client interface for enqueueing data into the queue.
3031 * Returns number of bytes enqueued or < 0 on error.
3033 ssize_t vmci_qpair_enqueue(struct vmci_qp *qpair,
3039 struct iov_iter from;
3040 struct kvec v = {.iov_base = (void *)buf, .iov_len = buf_size};
3043 return VMCI_ERROR_INVALID_ARGS;
3045 iov_iter_kvec(&from, WRITE, &v, 1, buf_size);
3050 result = qp_enqueue_locked(qpair->produce_q,
3052 qpair->produce_q_size,
3055 if (result == VMCI_ERROR_QUEUEPAIR_NOT_READY &&
3056 !qp_wait_for_ready_queue(qpair))
3057 result = VMCI_ERROR_WOULD_BLOCK;
3059 } while (result == VMCI_ERROR_QUEUEPAIR_NOT_READY);
3065 EXPORT_SYMBOL_GPL(vmci_qpair_enqueue);
3068 * vmci_qpair_dequeue() - Get data from the queue.
3069 * @qpair: Pointer to the queue pair struct.
3070 * @buf: Pointer to buffer for the data
3071 * @buf_size: Length of buffer.
3072 * @buf_type: Buffer type (Unused).
3074 * This is the client interface for dequeueing data from the queue.
3075 * Returns number of bytes dequeued or < 0 on error.
3077 ssize_t vmci_qpair_dequeue(struct vmci_qp *qpair,
3084 struct kvec v = {.iov_base = buf, .iov_len = buf_size};
3087 return VMCI_ERROR_INVALID_ARGS;
3089 iov_iter_kvec(&to, READ, &v, 1, buf_size);
3094 result = qp_dequeue_locked(qpair->produce_q,
3096 qpair->consume_q_size,
3099 if (result == VMCI_ERROR_QUEUEPAIR_NOT_READY &&
3100 !qp_wait_for_ready_queue(qpair))
3101 result = VMCI_ERROR_WOULD_BLOCK;
3103 } while (result == VMCI_ERROR_QUEUEPAIR_NOT_READY);
3109 EXPORT_SYMBOL_GPL(vmci_qpair_dequeue);
3112 * vmci_qpair_peek() - Peek at the data in the queue.
3113 * @qpair: Pointer to the queue pair struct.
3114 * @buf: Pointer to buffer for the data
3115 * @buf_size: Length of buffer.
3116 * @buf_type: Buffer type (Unused on Linux).
3118 * This is the client interface for peeking into a queue. (I.e.,
3119 * copy data from the queue without updating the head pointer.)
3120 * Returns number of bytes dequeued or < 0 on error.
3122 ssize_t vmci_qpair_peek(struct vmci_qp *qpair,
3128 struct kvec v = {.iov_base = buf, .iov_len = buf_size};
3132 return VMCI_ERROR_INVALID_ARGS;
3134 iov_iter_kvec(&to, READ, &v, 1, buf_size);
3139 result = qp_dequeue_locked(qpair->produce_q,
3141 qpair->consume_q_size,
3144 if (result == VMCI_ERROR_QUEUEPAIR_NOT_READY &&
3145 !qp_wait_for_ready_queue(qpair))
3146 result = VMCI_ERROR_WOULD_BLOCK;
3148 } while (result == VMCI_ERROR_QUEUEPAIR_NOT_READY);
3154 EXPORT_SYMBOL_GPL(vmci_qpair_peek);
3157 * vmci_qpair_enquev() - Throw data on the queue using iov.
3158 * @qpair: Pointer to the queue pair struct.
3159 * @iov: Pointer to buffer containing data
3160 * @iov_size: Length of buffer.
3161 * @buf_type: Buffer type (Unused).
3163 * This is the client interface for enqueueing data into the queue.
3164 * This function uses IO vectors to handle the work. Returns number
3165 * of bytes enqueued or < 0 on error.
3167 ssize_t vmci_qpair_enquev(struct vmci_qp *qpair,
3175 return VMCI_ERROR_INVALID_ARGS;
3180 result = qp_enqueue_locked(qpair->produce_q,
3182 qpair->produce_q_size,
3185 if (result == VMCI_ERROR_QUEUEPAIR_NOT_READY &&
3186 !qp_wait_for_ready_queue(qpair))
3187 result = VMCI_ERROR_WOULD_BLOCK;
3189 } while (result == VMCI_ERROR_QUEUEPAIR_NOT_READY);
3195 EXPORT_SYMBOL_GPL(vmci_qpair_enquev);
3198 * vmci_qpair_dequev() - Get data from the queue using iov.
3199 * @qpair: Pointer to the queue pair struct.
3200 * @iov: Pointer to buffer for the data
3201 * @iov_size: Length of buffer.
3202 * @buf_type: Buffer type (Unused).
3204 * This is the client interface for dequeueing data from the queue.
3205 * This function uses IO vectors to handle the work. Returns number
3206 * of bytes dequeued or < 0 on error.
3208 ssize_t vmci_qpair_dequev(struct vmci_qp *qpair,
3216 return VMCI_ERROR_INVALID_ARGS;
3221 result = qp_dequeue_locked(qpair->produce_q,
3223 qpair->consume_q_size,
3224 &msg->msg_iter, true);
3226 if (result == VMCI_ERROR_QUEUEPAIR_NOT_READY &&
3227 !qp_wait_for_ready_queue(qpair))
3228 result = VMCI_ERROR_WOULD_BLOCK;
3230 } while (result == VMCI_ERROR_QUEUEPAIR_NOT_READY);
3236 EXPORT_SYMBOL_GPL(vmci_qpair_dequev);
3239 * vmci_qpair_peekv() - Peek at the data in the queue using iov.
3240 * @qpair: Pointer to the queue pair struct.
3241 * @iov: Pointer to buffer for the data
3242 * @iov_size: Length of buffer.
3243 * @buf_type: Buffer type (Unused on Linux).
3245 * This is the client interface for peeking into a queue. (I.e.,
3246 * copy data from the queue without updating the head pointer.)
3247 * This function uses IO vectors to handle the work. Returns number
3248 * of bytes peeked or < 0 on error.
3250 ssize_t vmci_qpair_peekv(struct vmci_qp *qpair,
3258 return VMCI_ERROR_INVALID_ARGS;
3263 result = qp_dequeue_locked(qpair->produce_q,
3265 qpair->consume_q_size,
3266 &msg->msg_iter, false);
3268 if (result == VMCI_ERROR_QUEUEPAIR_NOT_READY &&
3269 !qp_wait_for_ready_queue(qpair))
3270 result = VMCI_ERROR_WOULD_BLOCK;
3272 } while (result == VMCI_ERROR_QUEUEPAIR_NOT_READY);
3277 EXPORT_SYMBOL_GPL(vmci_qpair_peekv);