#include <linux/uaccess.h>
#include <linux/vfio.h>
#include <linux/workqueue.h>
-#include <linux/mdev.h>
#include <linux/notifier.h>
#include <linux/dma-iommu.h>
#include <linux/irqdomain.h>
+#include "vfio.h"
#define DRIVER_VERSION "0.2"
#define DRIVER_AUTHOR "Alex Williamson <alex.williamson@redhat.com>"
struct vfio_iommu {
struct list_head domain_list;
struct list_head iova_list;
- struct vfio_domain *external_domain; /* domain for external user */
struct mutex lock;
struct rb_root dma_list;
struct blocking_notifier_head notifier;
bool nesting;
bool dirty_page_tracking;
bool container_open;
+ struct list_head emulated_iommu_groups;
};
struct vfio_domain {
struct vfio_iommu_group {
struct iommu_group *iommu_group;
struct list_head next;
- bool mdev_group; /* An mdev group */
bool pinned_page_dirty_scope;
};
size_t len;
};
-#define IS_IOMMU_CAP_DOMAIN_IN_CONTAINER(iommu) \
- (!list_empty(&iommu->domain_list))
-
#define DIRTY_BITMAP_BYTES(n) (ALIGN(n, BITS_PER_TYPE(u64)) / BITS_PER_BYTE)
/*
static void vfio_dma_bitmap_free(struct vfio_dma *dma)
{
- kfree(dma->bitmap);
+ kvfree(dma->bitmap);
dma->bitmap = NULL;
}
* already pinned and accounted. Accounting should be done if there is no
* iommu capable domain in the container.
*/
- do_accounting = !IS_IOMMU_CAP_DOMAIN_IN_CONTAINER(iommu);
+ do_accounting = list_empty(&iommu->domain_list);
for (i = 0; i < npage; i++) {
struct vfio_pfn *vpfn;
mutex_lock(&iommu->lock);
- do_accounting = !IS_IOMMU_CAP_DOMAIN_IN_CONTAINER(iommu);
+ do_accounting = list_empty(&iommu->domain_list);
for (i = 0; i < npage; i++) {
struct vfio_dma *dma;
dma_addr_t iova;
if (!dma->size)
return 0;
- if (!IS_IOMMU_CAP_DOMAIN_IN_CONTAINER(iommu))
+ if (list_empty(&iommu->domain_list))
return 0;
/*
vfio_link_dma(iommu, dma);
/* Don't pin and map if container doesn't contain IOMMU capable domain*/
- if (!IS_IOMMU_CAP_DOMAIN_IN_CONTAINER(iommu))
+ if (list_empty(&iommu->domain_list))
dma->size = size;
else
ret = vfio_pin_map_dma(iommu, dma, size);
vfio_iommu_find_iommu_group(struct vfio_iommu *iommu,
struct iommu_group *iommu_group)
{
+ struct vfio_iommu_group *group;
struct vfio_domain *domain;
- struct vfio_iommu_group *group = NULL;
list_for_each_entry(domain, &iommu->domain_list, next) {
group = find_iommu_group(domain, iommu_group);
return group;
}
- if (iommu->external_domain)
- group = find_iommu_group(iommu->external_domain, iommu_group);
-
- return group;
+ list_for_each_entry(group, &iommu->emulated_iommu_groups, next)
+ if (group->iommu_group == iommu_group)
+ return group;
+ return NULL;
}
static bool vfio_iommu_has_sw_msi(struct list_head *group_resv_regions,
return ret;
}
-static int vfio_mdev_attach_domain(struct device *dev, void *data)
-{
- struct mdev_device *mdev = to_mdev_device(dev);
- struct iommu_domain *domain = data;
- struct device *iommu_device;
-
- iommu_device = mdev_get_iommu_device(mdev);
- if (iommu_device) {
- if (iommu_dev_feature_enabled(iommu_device, IOMMU_DEV_FEAT_AUX))
- return iommu_aux_attach_device(domain, iommu_device);
- else
- return iommu_attach_device(domain, iommu_device);
- }
-
- return -EINVAL;
-}
-
-static int vfio_mdev_detach_domain(struct device *dev, void *data)
-{
- struct mdev_device *mdev = to_mdev_device(dev);
- struct iommu_domain *domain = data;
- struct device *iommu_device;
-
- iommu_device = mdev_get_iommu_device(mdev);
- if (iommu_device) {
- if (iommu_dev_feature_enabled(iommu_device, IOMMU_DEV_FEAT_AUX))
- iommu_aux_detach_device(domain, iommu_device);
- else
- iommu_detach_device(domain, iommu_device);
- }
-
- return 0;
-}
-
-static int vfio_iommu_attach_group(struct vfio_domain *domain,
- struct vfio_iommu_group *group)
-{
- if (group->mdev_group)
- return iommu_group_for_each_dev(group->iommu_group,
- domain->domain,
- vfio_mdev_attach_domain);
- else
- return iommu_attach_group(domain->domain, group->iommu_group);
-}
-
-static void vfio_iommu_detach_group(struct vfio_domain *domain,
- struct vfio_iommu_group *group)
-{
- if (group->mdev_group)
- iommu_group_for_each_dev(group->iommu_group, domain->domain,
- vfio_mdev_detach_domain);
- else
- iommu_detach_group(domain->domain, group->iommu_group);
-}
-
-static bool vfio_bus_is_mdev(struct bus_type *bus)
-{
- struct bus_type *mdev_bus;
- bool ret = false;
-
- mdev_bus = symbol_get(mdev_bus_type);
- if (mdev_bus) {
- ret = (bus == mdev_bus);
- symbol_put(mdev_bus_type);
- }
-
- return ret;
-}
-
-static int vfio_mdev_iommu_device(struct device *dev, void *data)
-{
- struct mdev_device *mdev = to_mdev_device(dev);
- struct device **old = data, *new;
-
- new = mdev_get_iommu_device(mdev);
- if (!new || (*old && *old != new))
- return -EINVAL;
-
- *old = new;
-
- return 0;
-}
-
/*
* This is a helper function to insert an address range to iova list.
* The list is initially created with a single entry corresponding to
}
static int vfio_iommu_type1_attach_group(void *iommu_data,
- struct iommu_group *iommu_group)
+ struct iommu_group *iommu_group, enum vfio_group_type type)
{
struct vfio_iommu *iommu = iommu_data;
struct vfio_iommu_group *group;
struct vfio_domain *domain, *d;
struct bus_type *bus = NULL;
- int ret;
bool resv_msi, msi_remap;
phys_addr_t resv_msi_base = 0;
struct iommu_domain_geometry *geo;
LIST_HEAD(iova_copy);
LIST_HEAD(group_resv_regions);
+ int ret = -EINVAL;
mutex_lock(&iommu->lock);
/* Check for duplicates */
- if (vfio_iommu_find_iommu_group(iommu, iommu_group)) {
- mutex_unlock(&iommu->lock);
- return -EINVAL;
- }
+ if (vfio_iommu_find_iommu_group(iommu, iommu_group))
+ goto out_unlock;
+ ret = -ENOMEM;
group = kzalloc(sizeof(*group), GFP_KERNEL);
- domain = kzalloc(sizeof(*domain), GFP_KERNEL);
- if (!group || !domain) {
- ret = -ENOMEM;
- goto out_free;
- }
-
+ if (!group)
+ goto out_unlock;
group->iommu_group = iommu_group;
+ if (type == VFIO_EMULATED_IOMMU) {
+ list_add(&group->next, &iommu->emulated_iommu_groups);
+ /*
+ * An emulated IOMMU group cannot dirty memory directly, it can
+ * only use interfaces that provide dirty tracking.
+ * The iommu scope can only be promoted with the addition of a
+ * dirty tracking group.
+ */
+ group->pinned_page_dirty_scope = true;
+ ret = 0;
+ goto out_unlock;
+ }
+
/* Determine bus_type in order to allocate a domain */
ret = iommu_group_for_each_dev(iommu_group, &bus, vfio_bus_type);
if (ret)
- goto out_free;
-
- if (vfio_bus_is_mdev(bus)) {
- struct device *iommu_device = NULL;
-
- group->mdev_group = true;
-
- /* Determine the isolation type */
- ret = iommu_group_for_each_dev(iommu_group, &iommu_device,
- vfio_mdev_iommu_device);
- if (ret || !iommu_device) {
- if (!iommu->external_domain) {
- INIT_LIST_HEAD(&domain->group_list);
- iommu->external_domain = domain;
- vfio_update_pgsize_bitmap(iommu);
- } else {
- kfree(domain);
- }
-
- list_add(&group->next,
- &iommu->external_domain->group_list);
- /*
- * Non-iommu backed group cannot dirty memory directly,
- * it can only use interfaces that provide dirty
- * tracking.
- * The iommu scope can only be promoted with the
- * addition of a dirty tracking group.
- */
- group->pinned_page_dirty_scope = true;
- mutex_unlock(&iommu->lock);
+ goto out_free_group;
- return 0;
- }
-
- bus = iommu_device->bus;
- }
+ ret = -ENOMEM;
+ domain = kzalloc(sizeof(*domain), GFP_KERNEL);
+ if (!domain)
+ goto out_free_group;
+ ret = -EIO;
domain->domain = iommu_domain_alloc(bus);
- if (!domain->domain) {
- ret = -EIO;
- goto out_free;
- }
+ if (!domain->domain)
+ goto out_free_domain;
if (iommu->nesting) {
ret = iommu_enable_nesting(domain->domain);
goto out_domain;
}
- ret = vfio_iommu_attach_group(domain, group);
+ ret = iommu_attach_group(domain->domain, group->iommu_group);
if (ret)
goto out_domain;
list_for_each_entry(d, &iommu->domain_list, next) {
if (d->domain->ops == domain->domain->ops &&
d->prot == domain->prot) {
- vfio_iommu_detach_group(domain, group);
- if (!vfio_iommu_attach_group(d, group)) {
+ iommu_detach_group(domain->domain, group->iommu_group);
+ if (!iommu_attach_group(d->domain,
+ group->iommu_group)) {
list_add(&group->next, &d->group_list);
iommu_domain_free(domain->domain);
kfree(domain);
goto done;
}
- ret = vfio_iommu_attach_group(domain, group);
+ ret = iommu_attach_group(domain->domain,
+ group->iommu_group);
if (ret)
goto out_domain;
}
return 0;
out_detach:
- vfio_iommu_detach_group(domain, group);
+ iommu_detach_group(domain->domain, group->iommu_group);
out_domain:
iommu_domain_free(domain->domain);
vfio_iommu_iova_free(&iova_copy);
vfio_iommu_resv_free(&group_resv_regions);
-out_free:
+out_free_domain:
kfree(domain);
+out_free_group:
kfree(group);
+out_unlock:
mutex_unlock(&iommu->lock);
return ret;
}
LIST_HEAD(iova_copy);
mutex_lock(&iommu->lock);
+ list_for_each_entry(group, &iommu->emulated_iommu_groups, next) {
+ if (group->iommu_group != iommu_group)
+ continue;
+ update_dirty_scope = !group->pinned_page_dirty_scope;
+ list_del(&group->next);
+ kfree(group);
- if (iommu->external_domain) {
- group = find_iommu_group(iommu->external_domain, iommu_group);
- if (group) {
- update_dirty_scope = !group->pinned_page_dirty_scope;
- list_del(&group->next);
- kfree(group);
-
- if (list_empty(&iommu->external_domain->group_list)) {
- if (!IS_IOMMU_CAP_DOMAIN_IN_CONTAINER(iommu)) {
- WARN_ON(iommu->notifier.head);
- vfio_iommu_unmap_unpin_all(iommu);
- }
-
- kfree(iommu->external_domain);
- iommu->external_domain = NULL;
- }
- goto detach_group_done;
+ if (list_empty(&iommu->emulated_iommu_groups) &&
+ list_empty(&iommu->domain_list)) {
+ WARN_ON(iommu->notifier.head);
+ vfio_iommu_unmap_unpin_all(iommu);
}
+ goto detach_group_done;
}
/*
if (!group)
continue;
- vfio_iommu_detach_group(domain, group);
+ iommu_detach_group(domain->domain, group->iommu_group);
update_dirty_scope = !group->pinned_page_dirty_scope;
list_del(&group->next);
kfree(group);
*/
if (list_empty(&domain->group_list)) {
if (list_is_singular(&iommu->domain_list)) {
- if (!iommu->external_domain) {
+ if (list_empty(&iommu->emulated_iommu_groups)) {
WARN_ON(iommu->notifier.head);
vfio_iommu_unmap_unpin_all(iommu);
} else {
mutex_init(&iommu->lock);
BLOCKING_INIT_NOTIFIER_HEAD(&iommu->notifier);
init_waitqueue_head(&iommu->vaddr_wait);
+ iommu->pgsize_bitmap = PAGE_MASK;
+ INIT_LIST_HEAD(&iommu->emulated_iommu_groups);
return iommu;
}
-static void vfio_release_domain(struct vfio_domain *domain, bool external)
+static void vfio_release_domain(struct vfio_domain *domain)
{
struct vfio_iommu_group *group, *group_tmp;
list_for_each_entry_safe(group, group_tmp,
&domain->group_list, next) {
- if (!external)
- vfio_iommu_detach_group(domain, group);
+ iommu_detach_group(domain->domain, group->iommu_group);
list_del(&group->next);
kfree(group);
}
- if (!external)
- iommu_domain_free(domain->domain);
+ iommu_domain_free(domain->domain);
}
static void vfio_iommu_type1_release(void *iommu_data)
{
struct vfio_iommu *iommu = iommu_data;
struct vfio_domain *domain, *domain_tmp;
+ struct vfio_iommu_group *group, *next_group;
- if (iommu->external_domain) {
- vfio_release_domain(iommu->external_domain, true);
- kfree(iommu->external_domain);
+ list_for_each_entry_safe(group, next_group,
+ &iommu->emulated_iommu_groups, next) {
+ list_del(&group->next);
+ kfree(group);
}
vfio_iommu_unmap_unpin_all(iommu);
list_for_each_entry_safe(domain, domain_tmp,
&iommu->domain_list, next) {
- vfio_release_domain(domain, false);
+ vfio_release_domain(domain);
list_del(&domain->next);
kfree(domain);
}
projects / linux-2.6-microblaze.git / blobdiff