1 // SPDX-License-Identifier: GPL-2.0+
3 * Adjunct processor matrix VFIO device driver callbacks.
5 * Copyright IBM Corp. 2018
7 * Author(s): Tony Krowiak <akrowiak@linux.ibm.com>
8 * Halil Pasic <pasic@linux.ibm.com>
9 * Pierre Morel <pmorel@linux.ibm.com>
11 #include <linux/string.h>
12 #include <linux/vfio.h>
13 #include <linux/device.h>
14 #include <linux/list.h>
15 #include <linux/ctype.h>
16 #include <linux/bitops.h>
17 #include <linux/kvm_host.h>
18 #include <linux/module.h>
20 #include <asm/zcrypt.h>
22 #include "vfio_ap_private.h"
24 #define VFIO_AP_MDEV_TYPE_HWVIRT "passthrough"
25 #define VFIO_AP_MDEV_NAME_HWVIRT "VFIO AP Passthrough Device"
27 static int vfio_ap_mdev_reset_queues(struct mdev_device *mdev);
28 static struct vfio_ap_queue *vfio_ap_find_queue(int apqn);
30 static int match_apqn(struct device *dev, const void *data)
32 struct vfio_ap_queue *q = dev_get_drvdata(dev);
34 return (q->apqn == *(int *)(data)) ? 1 : 0;
38 * vfio_ap_get_queue: Retrieve a queue with a specific APQN from a list
39 * @matrix_mdev: the associated mediated matrix
40 * @apqn: The queue APQN
42 * Retrieve a queue with a specific APQN from the list of the
43 * devices of the vfio_ap_drv.
44 * Verify that the APID and the APQI are set in the matrix.
46 * Returns the pointer to the associated vfio_ap_queue
48 static struct vfio_ap_queue *vfio_ap_get_queue(
49 struct ap_matrix_mdev *matrix_mdev,
52 struct vfio_ap_queue *q;
54 if (!test_bit_inv(AP_QID_CARD(apqn), matrix_mdev->matrix.apm))
56 if (!test_bit_inv(AP_QID_QUEUE(apqn), matrix_mdev->matrix.aqm))
59 q = vfio_ap_find_queue(apqn);
61 q->matrix_mdev = matrix_mdev;
67 * vfio_ap_wait_for_irqclear
68 * @apqn: The AP Queue number
70 * Checks the IRQ bit for the status of this APQN using ap_tapq.
71 * Returns if the ap_tapq function succeeded and the bit is clear.
72 * Returns if ap_tapq function failed with invalid, deconfigured or
74 * Otherwise retries up to 5 times after waiting 20ms.
77 static void vfio_ap_wait_for_irqclear(int apqn)
79 struct ap_queue_status status;
83 status = ap_tapq(apqn, NULL);
84 switch (status.response_code) {
85 case AP_RESPONSE_NORMAL:
86 case AP_RESPONSE_RESET_IN_PROGRESS:
87 if (!status.irq_enabled)
90 case AP_RESPONSE_BUSY:
93 case AP_RESPONSE_Q_NOT_AVAIL:
94 case AP_RESPONSE_DECONFIGURED:
95 case AP_RESPONSE_CHECKSTOPPED:
97 WARN_ONCE(1, "%s: tapq rc %02x: %04x\n", __func__,
98 status.response_code, apqn);
103 WARN_ONCE(1, "%s: tapq rc %02x: %04x could not clear IR bit\n",
104 __func__, status.response_code, apqn);
108 * vfio_ap_free_aqic_resources
109 * @q: The vfio_ap_queue
111 * Unregisters the ISC in the GIB when the saved ISC not invalid.
112 * Unpin the guest's page holding the NIB when it exist.
113 * Reset the saved_pfn and saved_isc to invalid values.
116 static void vfio_ap_free_aqic_resources(struct vfio_ap_queue *q)
120 if (q->saved_isc != VFIO_AP_ISC_INVALID &&
121 !WARN_ON(!(q->matrix_mdev && q->matrix_mdev->kvm))) {
122 kvm_s390_gisc_unregister(q->matrix_mdev->kvm, q->saved_isc);
123 q->saved_isc = VFIO_AP_ISC_INVALID;
125 if (q->saved_pfn && !WARN_ON(!q->matrix_mdev)) {
126 vfio_unpin_pages(mdev_dev(q->matrix_mdev->mdev),
133 * vfio_ap_irq_disable
134 * @q: The vfio_ap_queue
136 * Uses ap_aqic to disable the interruption and in case of success, reset
137 * in progress or IRQ disable command already proceeded: calls
138 * vfio_ap_wait_for_irqclear() to check for the IRQ bit to be clear
139 * and calls vfio_ap_free_aqic_resources() to free the resources associated
140 * with the AP interrupt handling.
142 * In the case the AP is busy, or a reset is in progress,
143 * retries after 20ms, up to 5 times.
145 * Returns if ap_aqic function failed with invalid, deconfigured or
148 static struct ap_queue_status vfio_ap_irq_disable(struct vfio_ap_queue *q)
150 struct ap_qirq_ctrl aqic_gisa = {};
151 struct ap_queue_status status;
155 status = ap_aqic(q->apqn, aqic_gisa, NULL);
156 switch (status.response_code) {
157 case AP_RESPONSE_OTHERWISE_CHANGED:
158 case AP_RESPONSE_NORMAL:
159 vfio_ap_wait_for_irqclear(q->apqn);
161 case AP_RESPONSE_RESET_IN_PROGRESS:
162 case AP_RESPONSE_BUSY:
165 case AP_RESPONSE_Q_NOT_AVAIL:
166 case AP_RESPONSE_DECONFIGURED:
167 case AP_RESPONSE_CHECKSTOPPED:
168 case AP_RESPONSE_INVALID_ADDRESS:
170 /* All cases in default means AP not operational */
171 WARN_ONCE(1, "%s: ap_aqic status %d\n", __func__,
172 status.response_code);
177 WARN_ONCE(1, "%s: ap_aqic status %d\n", __func__,
178 status.response_code);
180 vfio_ap_free_aqic_resources(q);
181 q->matrix_mdev = NULL;
186 * vfio_ap_setirq: Enable Interruption for a APQN
188 * @dev: the device associated with the ap_queue
189 * @q: the vfio_ap_queue holding AQIC parameters
191 * Pin the NIB saved in *q
192 * Register the guest ISC to GIB interface and retrieve the
193 * host ISC to issue the host side PQAP/AQIC
195 * Response.status may be set to AP_RESPONSE_INVALID_ADDRESS in case the
196 * vfio_pin_pages failed.
198 * Otherwise return the ap_queue_status returned by the ap_aqic(),
199 * all retry handling will be done by the guest.
201 static struct ap_queue_status vfio_ap_irq_enable(struct vfio_ap_queue *q,
205 struct ap_qirq_ctrl aqic_gisa = {};
206 struct ap_queue_status status = {};
207 struct kvm_s390_gisa *gisa;
209 unsigned long h_nib, g_pfn, h_pfn;
212 g_pfn = nib >> PAGE_SHIFT;
213 ret = vfio_pin_pages(mdev_dev(q->matrix_mdev->mdev), &g_pfn, 1,
214 IOMMU_READ | IOMMU_WRITE, &h_pfn);
219 status.response_code = AP_RESPONSE_INVALID_ADDRESS;
223 kvm = q->matrix_mdev->kvm;
224 gisa = kvm->arch.gisa_int.origin;
226 h_nib = (h_pfn << PAGE_SHIFT) | (nib & ~PAGE_MASK);
227 aqic_gisa.gisc = isc;
228 aqic_gisa.isc = kvm_s390_gisc_register(kvm, isc);
230 aqic_gisa.gisa = (uint64_t)gisa >> 4;
232 status = ap_aqic(q->apqn, aqic_gisa, (void *)h_nib);
233 switch (status.response_code) {
234 case AP_RESPONSE_NORMAL:
235 /* See if we did clear older IRQ configuration */
236 vfio_ap_free_aqic_resources(q);
237 q->saved_pfn = g_pfn;
240 case AP_RESPONSE_OTHERWISE_CHANGED:
241 /* We could not modify IRQ setings: clear new configuration */
242 vfio_unpin_pages(mdev_dev(q->matrix_mdev->mdev), &g_pfn, 1);
243 kvm_s390_gisc_unregister(kvm, isc);
246 pr_warn("%s: apqn %04x: response: %02x\n", __func__, q->apqn,
247 status.response_code);
248 vfio_ap_irq_disable(q);
256 * handle_pqap: PQAP instruction callback
258 * @vcpu: The vcpu on which we received the PQAP instruction
260 * Get the general register contents to initialize internal variables.
265 * Response.status may be set to following Response Code:
266 * - AP_RESPONSE_Q_NOT_AVAIL: if the queue is not available
267 * - AP_RESPONSE_DECONFIGURED: if the queue is not configured
268 * - AP_RESPONSE_NORMAL (0) : in case of successs
269 * Check vfio_ap_setirq() and vfio_ap_clrirq() for other possible RC.
270 * We take the matrix_dev lock to ensure serialization on queues and
271 * mediated device access.
273 * Return 0 if we could handle the request inside KVM.
274 * otherwise, returns -EOPNOTSUPP to let QEMU handle the fault.
276 static int handle_pqap(struct kvm_vcpu *vcpu)
280 struct vfio_ap_queue *q;
281 struct ap_queue_status qstatus = {
282 .response_code = AP_RESPONSE_Q_NOT_AVAIL, };
283 struct ap_matrix_mdev *matrix_mdev;
285 /* If we do not use the AIV facility just go to userland */
286 if (!(vcpu->arch.sie_block->eca & ECA_AIV))
289 apqn = vcpu->run->s.regs.gprs[0] & 0xffff;
290 mutex_lock(&matrix_dev->lock);
292 if (!vcpu->kvm->arch.crypto.pqap_hook)
294 matrix_mdev = container_of(vcpu->kvm->arch.crypto.pqap_hook,
295 struct ap_matrix_mdev, pqap_hook);
298 * If the KVM pointer is in the process of being set, wait until the
299 * process has completed.
301 wait_event_cmd(matrix_mdev->wait_for_kvm,
302 !matrix_mdev->kvm_busy,
303 mutex_unlock(&matrix_dev->lock),
304 mutex_lock(&matrix_dev->lock));
306 /* If the there is no guest using the mdev, there is nothing to do */
307 if (!matrix_mdev->kvm)
310 q = vfio_ap_get_queue(matrix_mdev, apqn);
314 status = vcpu->run->s.regs.gprs[1];
316 /* If IR bit(16) is set we enable the interrupt */
317 if ((status >> (63 - 16)) & 0x01)
318 qstatus = vfio_ap_irq_enable(q, status & 0x07,
319 vcpu->run->s.regs.gprs[2]);
321 qstatus = vfio_ap_irq_disable(q);
324 memcpy(&vcpu->run->s.regs.gprs[1], &qstatus, sizeof(qstatus));
325 vcpu->run->s.regs.gprs[1] >>= 32;
326 mutex_unlock(&matrix_dev->lock);
330 static void vfio_ap_matrix_init(struct ap_config_info *info,
331 struct ap_matrix *matrix)
333 matrix->apm_max = info->apxa ? info->Na : 63;
334 matrix->aqm_max = info->apxa ? info->Nd : 15;
335 matrix->adm_max = info->apxa ? info->Nd : 15;
338 static int vfio_ap_mdev_create(struct mdev_device *mdev)
340 struct ap_matrix_mdev *matrix_mdev;
342 if ((atomic_dec_if_positive(&matrix_dev->available_instances) < 0))
345 matrix_mdev = kzalloc(sizeof(*matrix_mdev), GFP_KERNEL);
347 atomic_inc(&matrix_dev->available_instances);
351 matrix_mdev->mdev = mdev;
352 vfio_ap_matrix_init(&matrix_dev->info, &matrix_mdev->matrix);
353 init_waitqueue_head(&matrix_mdev->wait_for_kvm);
354 mdev_set_drvdata(mdev, matrix_mdev);
355 matrix_mdev->pqap_hook.hook = handle_pqap;
356 matrix_mdev->pqap_hook.owner = THIS_MODULE;
357 mutex_lock(&matrix_dev->lock);
358 list_add(&matrix_mdev->node, &matrix_dev->mdev_list);
359 mutex_unlock(&matrix_dev->lock);
364 static int vfio_ap_mdev_remove(struct mdev_device *mdev)
366 struct ap_matrix_mdev *matrix_mdev = mdev_get_drvdata(mdev);
368 mutex_lock(&matrix_dev->lock);
369 vfio_ap_mdev_reset_queues(mdev);
370 list_del(&matrix_mdev->node);
372 mdev_set_drvdata(mdev, NULL);
373 atomic_inc(&matrix_dev->available_instances);
374 mutex_unlock(&matrix_dev->lock);
379 static ssize_t name_show(struct mdev_type *mtype,
380 struct mdev_type_attribute *attr, char *buf)
382 return sprintf(buf, "%s\n", VFIO_AP_MDEV_NAME_HWVIRT);
385 static MDEV_TYPE_ATTR_RO(name);
387 static ssize_t available_instances_show(struct mdev_type *mtype,
388 struct mdev_type_attribute *attr,
391 return sprintf(buf, "%d\n",
392 atomic_read(&matrix_dev->available_instances));
395 static MDEV_TYPE_ATTR_RO(available_instances);
397 static ssize_t device_api_show(struct mdev_type *mtype,
398 struct mdev_type_attribute *attr, char *buf)
400 return sprintf(buf, "%s\n", VFIO_DEVICE_API_AP_STRING);
403 static MDEV_TYPE_ATTR_RO(device_api);
405 static struct attribute *vfio_ap_mdev_type_attrs[] = {
406 &mdev_type_attr_name.attr,
407 &mdev_type_attr_device_api.attr,
408 &mdev_type_attr_available_instances.attr,
412 static struct attribute_group vfio_ap_mdev_hwvirt_type_group = {
413 .name = VFIO_AP_MDEV_TYPE_HWVIRT,
414 .attrs = vfio_ap_mdev_type_attrs,
417 static struct attribute_group *vfio_ap_mdev_type_groups[] = {
418 &vfio_ap_mdev_hwvirt_type_group,
422 struct vfio_ap_queue_reserved {
431 * @dev: an AP queue device
432 * @data: a struct vfio_ap_queue_reserved reference
434 * Flags whether the AP queue device (@dev) has a queue ID containing the APQN,
435 * apid or apqi specified in @data:
437 * - If @data contains both an apid and apqi value, then @data will be flagged
438 * as reserved if the APID and APQI fields for the AP queue device matches
440 * - If @data contains only an apid value, @data will be flagged as
441 * reserved if the APID field in the AP queue device matches
443 * - If @data contains only an apqi value, @data will be flagged as
444 * reserved if the APQI field in the AP queue device matches
446 * Returns 0 to indicate the input to function succeeded. Returns -EINVAL if
447 * @data does not contain either an apid or apqi.
449 static int vfio_ap_has_queue(struct device *dev, void *data)
451 struct vfio_ap_queue_reserved *qres = data;
452 struct ap_queue *ap_queue = to_ap_queue(dev);
456 if (qres->apid && qres->apqi) {
457 qid = AP_MKQID(*qres->apid, *qres->apqi);
458 if (qid == ap_queue->qid)
459 qres->reserved = true;
460 } else if (qres->apid && !qres->apqi) {
461 id = AP_QID_CARD(ap_queue->qid);
462 if (id == *qres->apid)
463 qres->reserved = true;
464 } else if (!qres->apid && qres->apqi) {
465 id = AP_QID_QUEUE(ap_queue->qid);
466 if (id == *qres->apqi)
467 qres->reserved = true;
476 * vfio_ap_verify_queue_reserved
478 * @matrix_dev: a mediated matrix device
479 * @apid: an AP adapter ID
480 * @apqi: an AP queue index
482 * Verifies that the AP queue with @apid/@apqi is reserved by the VFIO AP device
483 * driver according to the following rules:
485 * - If both @apid and @apqi are not NULL, then there must be an AP queue
486 * device bound to the vfio_ap driver with the APQN identified by @apid and
489 * - If only @apid is not NULL, then there must be an AP queue device bound
490 * to the vfio_ap driver with an APQN containing @apid
492 * - If only @apqi is not NULL, then there must be an AP queue device bound
493 * to the vfio_ap driver with an APQN containing @apqi
495 * Returns 0 if the AP queue is reserved; otherwise, returns -EADDRNOTAVAIL.
497 static int vfio_ap_verify_queue_reserved(unsigned long *apid,
501 struct vfio_ap_queue_reserved qres;
505 qres.reserved = false;
507 ret = driver_for_each_device(&matrix_dev->vfio_ap_drv->driver, NULL,
508 &qres, vfio_ap_has_queue);
515 return -EADDRNOTAVAIL;
519 vfio_ap_mdev_verify_queues_reserved_for_apid(struct ap_matrix_mdev *matrix_mdev,
524 unsigned long nbits = matrix_mdev->matrix.aqm_max + 1;
526 if (find_first_bit_inv(matrix_mdev->matrix.aqm, nbits) >= nbits)
527 return vfio_ap_verify_queue_reserved(&apid, NULL);
529 for_each_set_bit_inv(apqi, matrix_mdev->matrix.aqm, nbits) {
530 ret = vfio_ap_verify_queue_reserved(&apid, &apqi);
539 * vfio_ap_mdev_verify_no_sharing
541 * Verifies that the APQNs derived from the cross product of the AP adapter IDs
542 * and AP queue indexes comprising the AP matrix are not configured for another
543 * mediated device. AP queue sharing is not allowed.
545 * @matrix_mdev: the mediated matrix device
547 * Returns 0 if the APQNs are not shared, otherwise; returns -EADDRINUSE.
549 static int vfio_ap_mdev_verify_no_sharing(struct ap_matrix_mdev *matrix_mdev)
551 struct ap_matrix_mdev *lstdev;
552 DECLARE_BITMAP(apm, AP_DEVICES);
553 DECLARE_BITMAP(aqm, AP_DOMAINS);
555 list_for_each_entry(lstdev, &matrix_dev->mdev_list, node) {
556 if (matrix_mdev == lstdev)
559 memset(apm, 0, sizeof(apm));
560 memset(aqm, 0, sizeof(aqm));
563 * We work on full longs, as we can only exclude the leftover
564 * bits in non-inverse order. The leftover is all zeros.
566 if (!bitmap_and(apm, matrix_mdev->matrix.apm,
567 lstdev->matrix.apm, AP_DEVICES))
570 if (!bitmap_and(aqm, matrix_mdev->matrix.aqm,
571 lstdev->matrix.aqm, AP_DOMAINS))
581 * assign_adapter_store
583 * @dev: the matrix device
584 * @attr: the mediated matrix device's assign_adapter attribute
585 * @buf: a buffer containing the AP adapter number (APID) to
587 * @count: the number of bytes in @buf
589 * Parses the APID from @buf and sets the corresponding bit in the mediated
590 * matrix device's APM.
592 * Returns the number of bytes processed if the APID is valid; otherwise,
593 * returns one of the following errors:
596 * The APID is not a valid number
599 * The APID exceeds the maximum value configured for the system
602 * An APQN derived from the cross product of the APID being assigned
603 * and the APQIs previously assigned is not bound to the vfio_ap device
604 * driver; or, if no APQIs have yet been assigned, the APID is not
605 * contained in an APQN bound to the vfio_ap device driver.
608 * An APQN derived from the cross product of the APID being assigned
609 * and the APQIs previously assigned is being used by another mediated
612 static ssize_t assign_adapter_store(struct device *dev,
613 struct device_attribute *attr,
614 const char *buf, size_t count)
618 struct mdev_device *mdev = mdev_from_dev(dev);
619 struct ap_matrix_mdev *matrix_mdev = mdev_get_drvdata(mdev);
621 mutex_lock(&matrix_dev->lock);
624 * If the KVM pointer is in flux or the guest is running, disallow
625 * un-assignment of adapter
627 if (matrix_mdev->kvm_busy || matrix_mdev->kvm) {
632 ret = kstrtoul(buf, 0, &apid);
636 if (apid > matrix_mdev->matrix.apm_max) {
642 * Set the bit in the AP mask (APM) corresponding to the AP adapter
643 * number (APID). The bits in the mask, from most significant to least
644 * significant bit, correspond to APIDs 0-255.
646 ret = vfio_ap_mdev_verify_queues_reserved_for_apid(matrix_mdev, apid);
650 set_bit_inv(apid, matrix_mdev->matrix.apm);
652 ret = vfio_ap_mdev_verify_no_sharing(matrix_mdev);
660 clear_bit_inv(apid, matrix_mdev->matrix.apm);
662 mutex_unlock(&matrix_dev->lock);
666 static DEVICE_ATTR_WO(assign_adapter);
669 * unassign_adapter_store
671 * @dev: the matrix device
672 * @attr: the mediated matrix device's unassign_adapter attribute
673 * @buf: a buffer containing the adapter number (APID) to be unassigned
674 * @count: the number of bytes in @buf
676 * Parses the APID from @buf and clears the corresponding bit in the mediated
677 * matrix device's APM.
679 * Returns the number of bytes processed if the APID is valid; otherwise,
680 * returns one of the following errors:
681 * -EINVAL if the APID is not a number
682 * -ENODEV if the APID it exceeds the maximum value configured for the
685 static ssize_t unassign_adapter_store(struct device *dev,
686 struct device_attribute *attr,
687 const char *buf, size_t count)
691 struct mdev_device *mdev = mdev_from_dev(dev);
692 struct ap_matrix_mdev *matrix_mdev = mdev_get_drvdata(mdev);
694 mutex_lock(&matrix_dev->lock);
697 * If the KVM pointer is in flux or the guest is running, disallow
698 * un-assignment of adapter
700 if (matrix_mdev->kvm_busy || matrix_mdev->kvm) {
705 ret = kstrtoul(buf, 0, &apid);
709 if (apid > matrix_mdev->matrix.apm_max) {
714 clear_bit_inv((unsigned long)apid, matrix_mdev->matrix.apm);
717 mutex_unlock(&matrix_dev->lock);
720 static DEVICE_ATTR_WO(unassign_adapter);
723 vfio_ap_mdev_verify_queues_reserved_for_apqi(struct ap_matrix_mdev *matrix_mdev,
728 unsigned long nbits = matrix_mdev->matrix.apm_max + 1;
730 if (find_first_bit_inv(matrix_mdev->matrix.apm, nbits) >= nbits)
731 return vfio_ap_verify_queue_reserved(NULL, &apqi);
733 for_each_set_bit_inv(apid, matrix_mdev->matrix.apm, nbits) {
734 ret = vfio_ap_verify_queue_reserved(&apid, &apqi);
743 * assign_domain_store
745 * @dev: the matrix device
746 * @attr: the mediated matrix device's assign_domain attribute
747 * @buf: a buffer containing the AP queue index (APQI) of the domain to
749 * @count: the number of bytes in @buf
751 * Parses the APQI from @buf and sets the corresponding bit in the mediated
752 * matrix device's AQM.
754 * Returns the number of bytes processed if the APQI is valid; otherwise returns
755 * one of the following errors:
758 * The APQI is not a valid number
761 * The APQI exceeds the maximum value configured for the system
764 * An APQN derived from the cross product of the APQI being assigned
765 * and the APIDs previously assigned is not bound to the vfio_ap device
766 * driver; or, if no APIDs have yet been assigned, the APQI is not
767 * contained in an APQN bound to the vfio_ap device driver.
770 * An APQN derived from the cross product of the APQI being assigned
771 * and the APIDs previously assigned is being used by another mediated
774 static ssize_t assign_domain_store(struct device *dev,
775 struct device_attribute *attr,
776 const char *buf, size_t count)
780 struct mdev_device *mdev = mdev_from_dev(dev);
781 struct ap_matrix_mdev *matrix_mdev = mdev_get_drvdata(mdev);
782 unsigned long max_apqi = matrix_mdev->matrix.aqm_max;
784 mutex_lock(&matrix_dev->lock);
787 * If the KVM pointer is in flux or the guest is running, disallow
788 * assignment of domain
790 if (matrix_mdev->kvm_busy || matrix_mdev->kvm) {
795 ret = kstrtoul(buf, 0, &apqi);
798 if (apqi > max_apqi) {
803 ret = vfio_ap_mdev_verify_queues_reserved_for_apqi(matrix_mdev, apqi);
807 set_bit_inv(apqi, matrix_mdev->matrix.aqm);
809 ret = vfio_ap_mdev_verify_no_sharing(matrix_mdev);
817 clear_bit_inv(apqi, matrix_mdev->matrix.aqm);
819 mutex_unlock(&matrix_dev->lock);
823 static DEVICE_ATTR_WO(assign_domain);
827 * unassign_domain_store
829 * @dev: the matrix device
830 * @attr: the mediated matrix device's unassign_domain attribute
831 * @buf: a buffer containing the AP queue index (APQI) of the domain to
833 * @count: the number of bytes in @buf
835 * Parses the APQI from @buf and clears the corresponding bit in the
836 * mediated matrix device's AQM.
838 * Returns the number of bytes processed if the APQI is valid; otherwise,
839 * returns one of the following errors:
840 * -EINVAL if the APQI is not a number
841 * -ENODEV if the APQI exceeds the maximum value configured for the system
843 static ssize_t unassign_domain_store(struct device *dev,
844 struct device_attribute *attr,
845 const char *buf, size_t count)
849 struct mdev_device *mdev = mdev_from_dev(dev);
850 struct ap_matrix_mdev *matrix_mdev = mdev_get_drvdata(mdev);
852 mutex_lock(&matrix_dev->lock);
855 * If the KVM pointer is in flux or the guest is running, disallow
856 * un-assignment of domain
858 if (matrix_mdev->kvm_busy || matrix_mdev->kvm) {
863 ret = kstrtoul(buf, 0, &apqi);
867 if (apqi > matrix_mdev->matrix.aqm_max) {
872 clear_bit_inv((unsigned long)apqi, matrix_mdev->matrix.aqm);
876 mutex_unlock(&matrix_dev->lock);
879 static DEVICE_ATTR_WO(unassign_domain);
882 * assign_control_domain_store
884 * @dev: the matrix device
885 * @attr: the mediated matrix device's assign_control_domain attribute
886 * @buf: a buffer containing the domain ID to be assigned
887 * @count: the number of bytes in @buf
889 * Parses the domain ID from @buf and sets the corresponding bit in the mediated
890 * matrix device's ADM.
892 * Returns the number of bytes processed if the domain ID is valid; otherwise,
893 * returns one of the following errors:
894 * -EINVAL if the ID is not a number
895 * -ENODEV if the ID exceeds the maximum value configured for the system
897 static ssize_t assign_control_domain_store(struct device *dev,
898 struct device_attribute *attr,
899 const char *buf, size_t count)
903 struct mdev_device *mdev = mdev_from_dev(dev);
904 struct ap_matrix_mdev *matrix_mdev = mdev_get_drvdata(mdev);
906 mutex_lock(&matrix_dev->lock);
909 * If the KVM pointer is in flux or the guest is running, disallow
910 * assignment of control domain.
912 if (matrix_mdev->kvm_busy || matrix_mdev->kvm) {
917 ret = kstrtoul(buf, 0, &id);
921 if (id > matrix_mdev->matrix.adm_max) {
926 /* Set the bit in the ADM (bitmask) corresponding to the AP control
927 * domain number (id). The bits in the mask, from most significant to
928 * least significant, correspond to IDs 0 up to the one less than the
929 * number of control domains that can be assigned.
931 set_bit_inv(id, matrix_mdev->matrix.adm);
934 mutex_unlock(&matrix_dev->lock);
937 static DEVICE_ATTR_WO(assign_control_domain);
940 * unassign_control_domain_store
942 * @dev: the matrix device
943 * @attr: the mediated matrix device's unassign_control_domain attribute
944 * @buf: a buffer containing the domain ID to be unassigned
945 * @count: the number of bytes in @buf
947 * Parses the domain ID from @buf and clears the corresponding bit in the
948 * mediated matrix device's ADM.
950 * Returns the number of bytes processed if the domain ID is valid; otherwise,
951 * returns one of the following errors:
952 * -EINVAL if the ID is not a number
953 * -ENODEV if the ID exceeds the maximum value configured for the system
955 static ssize_t unassign_control_domain_store(struct device *dev,
956 struct device_attribute *attr,
957 const char *buf, size_t count)
961 struct mdev_device *mdev = mdev_from_dev(dev);
962 struct ap_matrix_mdev *matrix_mdev = mdev_get_drvdata(mdev);
963 unsigned long max_domid = matrix_mdev->matrix.adm_max;
965 mutex_lock(&matrix_dev->lock);
968 * If the KVM pointer is in flux or the guest is running, disallow
969 * un-assignment of control domain.
971 if (matrix_mdev->kvm_busy || matrix_mdev->kvm) {
976 ret = kstrtoul(buf, 0, &domid);
979 if (domid > max_domid) {
984 clear_bit_inv(domid, matrix_mdev->matrix.adm);
987 mutex_unlock(&matrix_dev->lock);
990 static DEVICE_ATTR_WO(unassign_control_domain);
992 static ssize_t control_domains_show(struct device *dev,
993 struct device_attribute *dev_attr,
1000 struct mdev_device *mdev = mdev_from_dev(dev);
1001 struct ap_matrix_mdev *matrix_mdev = mdev_get_drvdata(mdev);
1002 unsigned long max_domid = matrix_mdev->matrix.adm_max;
1004 mutex_lock(&matrix_dev->lock);
1005 for_each_set_bit_inv(id, matrix_mdev->matrix.adm, max_domid + 1) {
1006 n = sprintf(bufpos, "%04lx\n", id);
1010 mutex_unlock(&matrix_dev->lock);
1014 static DEVICE_ATTR_RO(control_domains);
1016 static ssize_t matrix_show(struct device *dev, struct device_attribute *attr,
1019 struct mdev_device *mdev = mdev_from_dev(dev);
1020 struct ap_matrix_mdev *matrix_mdev = mdev_get_drvdata(mdev);
1024 unsigned long apid1;
1025 unsigned long apqi1;
1026 unsigned long napm_bits = matrix_mdev->matrix.apm_max + 1;
1027 unsigned long naqm_bits = matrix_mdev->matrix.aqm_max + 1;
1031 apid1 = find_first_bit_inv(matrix_mdev->matrix.apm, napm_bits);
1032 apqi1 = find_first_bit_inv(matrix_mdev->matrix.aqm, naqm_bits);
1034 mutex_lock(&matrix_dev->lock);
1036 if ((apid1 < napm_bits) && (apqi1 < naqm_bits)) {
1037 for_each_set_bit_inv(apid, matrix_mdev->matrix.apm, napm_bits) {
1038 for_each_set_bit_inv(apqi, matrix_mdev->matrix.aqm,
1040 n = sprintf(bufpos, "%02lx.%04lx\n", apid,
1046 } else if (apid1 < napm_bits) {
1047 for_each_set_bit_inv(apid, matrix_mdev->matrix.apm, napm_bits) {
1048 n = sprintf(bufpos, "%02lx.\n", apid);
1052 } else if (apqi1 < naqm_bits) {
1053 for_each_set_bit_inv(apqi, matrix_mdev->matrix.aqm, naqm_bits) {
1054 n = sprintf(bufpos, ".%04lx\n", apqi);
1060 mutex_unlock(&matrix_dev->lock);
1064 static DEVICE_ATTR_RO(matrix);
1066 static struct attribute *vfio_ap_mdev_attrs[] = {
1067 &dev_attr_assign_adapter.attr,
1068 &dev_attr_unassign_adapter.attr,
1069 &dev_attr_assign_domain.attr,
1070 &dev_attr_unassign_domain.attr,
1071 &dev_attr_assign_control_domain.attr,
1072 &dev_attr_unassign_control_domain.attr,
1073 &dev_attr_control_domains.attr,
1074 &dev_attr_matrix.attr,
1078 static struct attribute_group vfio_ap_mdev_attr_group = {
1079 .attrs = vfio_ap_mdev_attrs
1082 static const struct attribute_group *vfio_ap_mdev_attr_groups[] = {
1083 &vfio_ap_mdev_attr_group,
1088 * vfio_ap_mdev_set_kvm
1090 * @matrix_mdev: a mediated matrix device
1091 * @kvm: reference to KVM instance
1093 * Sets all data for @matrix_mdev that are needed to manage AP resources
1094 * for the guest whose state is represented by @kvm.
1096 * Note: The matrix_dev->lock must be taken prior to calling
1097 * this function; however, the lock will be temporarily released while the
1098 * guest's AP configuration is set to avoid a potential lockdep splat.
1099 * The kvm->lock is taken to set the guest's AP configuration which, under
1100 * certain circumstances, will result in a circular lock dependency if this is
1101 * done under the @matrix_mdev->lock.
1103 * Return 0 if no other mediated matrix device has a reference to @kvm;
1104 * otherwise, returns an -EPERM.
1106 static int vfio_ap_mdev_set_kvm(struct ap_matrix_mdev *matrix_mdev,
1109 struct ap_matrix_mdev *m;
1111 if (kvm->arch.crypto.crycbd) {
1112 list_for_each_entry(m, &matrix_dev->mdev_list, node) {
1113 if (m != matrix_mdev && m->kvm == kvm)
1118 matrix_mdev->kvm_busy = true;
1119 mutex_unlock(&matrix_dev->lock);
1120 kvm_arch_crypto_set_masks(kvm,
1121 matrix_mdev->matrix.apm,
1122 matrix_mdev->matrix.aqm,
1123 matrix_mdev->matrix.adm);
1124 mutex_lock(&matrix_dev->lock);
1125 kvm->arch.crypto.pqap_hook = &matrix_mdev->pqap_hook;
1126 matrix_mdev->kvm = kvm;
1127 matrix_mdev->kvm_busy = false;
1128 wake_up_all(&matrix_mdev->wait_for_kvm);
1135 * vfio_ap_mdev_iommu_notifier: IOMMU notifier callback
1137 * @nb: The notifier block
1138 * @action: Action to be taken
1139 * @data: data associated with the request
1141 * For an UNMAP request, unpin the guest IOVA (the NIB guest address we
1142 * pinned before). Other requests are ignored.
1145 static int vfio_ap_mdev_iommu_notifier(struct notifier_block *nb,
1146 unsigned long action, void *data)
1148 struct ap_matrix_mdev *matrix_mdev;
1150 matrix_mdev = container_of(nb, struct ap_matrix_mdev, iommu_notifier);
1152 if (action == VFIO_IOMMU_NOTIFY_DMA_UNMAP) {
1153 struct vfio_iommu_type1_dma_unmap *unmap = data;
1154 unsigned long g_pfn = unmap->iova >> PAGE_SHIFT;
1156 vfio_unpin_pages(mdev_dev(matrix_mdev->mdev), &g_pfn, 1);
1164 * vfio_ap_mdev_unset_kvm
1166 * @matrix_mdev: a matrix mediated device
1168 * Performs clean-up of resources no longer needed by @matrix_mdev.
1170 * Note: The matrix_dev->lock must be taken prior to calling
1171 * this function; however, the lock will be temporarily released while the
1172 * guest's AP configuration is cleared to avoid a potential lockdep splat.
1173 * The kvm->lock is taken to clear the guest's AP configuration which, under
1174 * certain circumstances, will result in a circular lock dependency if this is
1175 * done under the @matrix_mdev->lock.
1178 static void vfio_ap_mdev_unset_kvm(struct ap_matrix_mdev *matrix_mdev)
1181 * If the KVM pointer is in the process of being set, wait until the
1182 * process has completed.
1184 wait_event_cmd(matrix_mdev->wait_for_kvm,
1185 !matrix_mdev->kvm_busy,
1186 mutex_unlock(&matrix_dev->lock),
1187 mutex_lock(&matrix_dev->lock));
1189 if (matrix_mdev->kvm) {
1190 matrix_mdev->kvm_busy = true;
1191 mutex_unlock(&matrix_dev->lock);
1192 kvm_arch_crypto_clear_masks(matrix_mdev->kvm);
1193 mutex_lock(&matrix_dev->lock);
1194 vfio_ap_mdev_reset_queues(matrix_mdev->mdev);
1195 matrix_mdev->kvm->arch.crypto.pqap_hook = NULL;
1196 kvm_put_kvm(matrix_mdev->kvm);
1197 matrix_mdev->kvm = NULL;
1198 matrix_mdev->kvm_busy = false;
1199 wake_up_all(&matrix_mdev->wait_for_kvm);
1203 static int vfio_ap_mdev_group_notifier(struct notifier_block *nb,
1204 unsigned long action, void *data)
1206 int notify_rc = NOTIFY_OK;
1207 struct ap_matrix_mdev *matrix_mdev;
1209 if (action != VFIO_GROUP_NOTIFY_SET_KVM)
1212 mutex_lock(&matrix_dev->lock);
1213 matrix_mdev = container_of(nb, struct ap_matrix_mdev, group_notifier);
1216 vfio_ap_mdev_unset_kvm(matrix_mdev);
1217 else if (vfio_ap_mdev_set_kvm(matrix_mdev, data))
1218 notify_rc = NOTIFY_DONE;
1220 mutex_unlock(&matrix_dev->lock);
1225 static struct vfio_ap_queue *vfio_ap_find_queue(int apqn)
1228 struct vfio_ap_queue *q = NULL;
1230 dev = driver_find_device(&matrix_dev->vfio_ap_drv->driver, NULL,
1233 q = dev_get_drvdata(dev);
1240 int vfio_ap_mdev_reset_queue(struct vfio_ap_queue *q,
1243 struct ap_queue_status status;
1251 status = ap_zapq(q->apqn);
1252 switch (status.response_code) {
1253 case AP_RESPONSE_NORMAL:
1256 case AP_RESPONSE_RESET_IN_PROGRESS:
1263 case AP_RESPONSE_Q_NOT_AVAIL:
1264 case AP_RESPONSE_DECONFIGURED:
1265 case AP_RESPONSE_CHECKSTOPPED:
1266 WARN_ON_ONCE(status.irq_enabled);
1268 goto free_resources;
1270 /* things are really broken, give up */
1271 WARN(true, "PQAP/ZAPQ completed with invalid rc (%x)\n",
1272 status.response_code);
1276 /* wait for the reset to take effect */
1278 if (status.queue_empty && !status.irq_enabled)
1281 status = ap_tapq(q->apqn, NULL);
1283 WARN_ON_ONCE(retry2 <= 0);
1286 vfio_ap_free_aqic_resources(q);
1291 static int vfio_ap_mdev_reset_queues(struct mdev_device *mdev)
1295 unsigned long apid, apqi;
1296 struct vfio_ap_queue *q;
1297 struct ap_matrix_mdev *matrix_mdev = mdev_get_drvdata(mdev);
1299 for_each_set_bit_inv(apid, matrix_mdev->matrix.apm,
1300 matrix_mdev->matrix.apm_max + 1) {
1301 for_each_set_bit_inv(apqi, matrix_mdev->matrix.aqm,
1302 matrix_mdev->matrix.aqm_max + 1) {
1303 q = vfio_ap_find_queue(AP_MKQID(apid, apqi));
1304 ret = vfio_ap_mdev_reset_queue(q, 1);
1306 * Regardless whether a queue turns out to be busy, or
1307 * is not operational, we need to continue resetting
1308 * the remaining queues.
1318 static int vfio_ap_mdev_open(struct mdev_device *mdev)
1320 struct ap_matrix_mdev *matrix_mdev = mdev_get_drvdata(mdev);
1321 unsigned long events;
1325 if (!try_module_get(THIS_MODULE))
1328 matrix_mdev->group_notifier.notifier_call = vfio_ap_mdev_group_notifier;
1329 events = VFIO_GROUP_NOTIFY_SET_KVM;
1331 ret = vfio_register_notifier(mdev_dev(mdev), VFIO_GROUP_NOTIFY,
1332 &events, &matrix_mdev->group_notifier);
1334 module_put(THIS_MODULE);
1338 matrix_mdev->iommu_notifier.notifier_call = vfio_ap_mdev_iommu_notifier;
1339 events = VFIO_IOMMU_NOTIFY_DMA_UNMAP;
1340 ret = vfio_register_notifier(mdev_dev(mdev), VFIO_IOMMU_NOTIFY,
1341 &events, &matrix_mdev->iommu_notifier);
1345 vfio_unregister_notifier(mdev_dev(mdev), VFIO_GROUP_NOTIFY,
1346 &matrix_mdev->group_notifier);
1347 module_put(THIS_MODULE);
1351 static void vfio_ap_mdev_release(struct mdev_device *mdev)
1353 struct ap_matrix_mdev *matrix_mdev = mdev_get_drvdata(mdev);
1355 mutex_lock(&matrix_dev->lock);
1356 vfio_ap_mdev_unset_kvm(matrix_mdev);
1357 mutex_unlock(&matrix_dev->lock);
1359 vfio_unregister_notifier(mdev_dev(mdev), VFIO_IOMMU_NOTIFY,
1360 &matrix_mdev->iommu_notifier);
1361 vfio_unregister_notifier(mdev_dev(mdev), VFIO_GROUP_NOTIFY,
1362 &matrix_mdev->group_notifier);
1363 module_put(THIS_MODULE);
1366 static int vfio_ap_mdev_get_device_info(unsigned long arg)
1368 unsigned long minsz;
1369 struct vfio_device_info info;
1371 minsz = offsetofend(struct vfio_device_info, num_irqs);
1373 if (copy_from_user(&info, (void __user *)arg, minsz))
1376 if (info.argsz < minsz)
1379 info.flags = VFIO_DEVICE_FLAGS_AP | VFIO_DEVICE_FLAGS_RESET;
1380 info.num_regions = 0;
1383 return copy_to_user((void __user *)arg, &info, minsz) ? -EFAULT : 0;
1386 static ssize_t vfio_ap_mdev_ioctl(struct mdev_device *mdev,
1387 unsigned int cmd, unsigned long arg)
1390 struct ap_matrix_mdev *matrix_mdev;
1392 mutex_lock(&matrix_dev->lock);
1394 case VFIO_DEVICE_GET_INFO:
1395 ret = vfio_ap_mdev_get_device_info(arg);
1397 case VFIO_DEVICE_RESET:
1398 matrix_mdev = mdev_get_drvdata(mdev);
1399 if (WARN(!matrix_mdev, "Driver data missing from mdev!!")) {
1405 * If the KVM pointer is in the process of being set, wait until
1406 * the process has completed.
1408 wait_event_cmd(matrix_mdev->wait_for_kvm,
1409 !matrix_mdev->kvm_busy,
1410 mutex_unlock(&matrix_dev->lock),
1411 mutex_lock(&matrix_dev->lock));
1413 ret = vfio_ap_mdev_reset_queues(mdev);
1419 mutex_unlock(&matrix_dev->lock);
1424 static const struct mdev_parent_ops vfio_ap_matrix_ops = {
1425 .owner = THIS_MODULE,
1426 .supported_type_groups = vfio_ap_mdev_type_groups,
1427 .mdev_attr_groups = vfio_ap_mdev_attr_groups,
1428 .create = vfio_ap_mdev_create,
1429 .remove = vfio_ap_mdev_remove,
1430 .open = vfio_ap_mdev_open,
1431 .release = vfio_ap_mdev_release,
1432 .ioctl = vfio_ap_mdev_ioctl,
1435 int vfio_ap_mdev_register(void)
1437 atomic_set(&matrix_dev->available_instances, MAX_ZDEV_ENTRIES_EXT);
1439 return mdev_register_device(&matrix_dev->device, &vfio_ap_matrix_ops);
1442 void vfio_ap_mdev_unregister(void)
1444 mdev_unregister_device(&matrix_dev->device);