Merge tag 'iommu-updates-v5.15' of git://git.kernel.org/pub/scm/linux/kernel/git...

[linux-2.6-microblaze.git] / drivers / iommu / dma-iommu.c

diff --git a/drivers/iommu/dma-iommu.c b/drivers/iommu/dma-iommu.c
index 174b2b3..896bea0 100644 (file)
--- a/drivers/iommu/dma-iommu.c
+++ b/drivers/iommu/dma-iommu.c
@@ -317,6 +317,30 @@ static bool dev_is_untrusted(struct device *dev)
        return dev_is_pci(dev) && to_pci_dev(dev)->untrusted;
 }
 
+/* sysfs updates are serialised by the mutex of the group owning @domain */
+int iommu_dma_init_fq(struct iommu_domain *domain)
+{
+       struct iommu_dma_cookie *cookie = domain->iova_cookie;
+       int ret;
+
+       if (cookie->fq_domain)
+               return 0;
+
+       ret = init_iova_flush_queue(&cookie->iovad, iommu_dma_flush_iotlb_all,
+                                   iommu_dma_entry_dtor);
+       if (ret) {
+               pr_warn("iova flush queue initialization failed\n");
+               return ret;
+       }
+       /*
+        * Prevent incomplete iovad->fq being observable. Pairs with path from
+        * __iommu_dma_unmap() through iommu_dma_free_iova() to queue_iova()
+        */
+       smp_wmb();
+       WRITE_ONCE(cookie->fq_domain, domain);
+       return 0;
+}
+
 /**
  * iommu_dma_init_domain - Initialise a DMA mapping domain
  * @domain: IOMMU domain previously prepared by iommu_get_dma_cookie()
@@ -370,17 +394,9 @@ static int iommu_dma_init_domain(struct iommu_domain *domain, dma_addr_t base,
 
        init_iova_domain(iovad, 1UL << order, base_pfn);
 
-       if (!cookie->fq_domain && (!dev || !dev_is_untrusted(dev)) &&
-           domain->ops->flush_iotlb_all && !iommu_get_dma_strict(domain)) {
-               if (init_iova_flush_queue(iovad, iommu_dma_flush_iotlb_all,
-                                         iommu_dma_entry_dtor))
-                       pr_warn("iova flush queue initialization failed\n");
-               else
-                       cookie->fq_domain = domain;
-       }
-
-       if (!dev)
-               return 0;
+       /* If the FQ fails we can simply fall back to strict mode */
+       if (domain->type == IOMMU_DOMAIN_DMA_FQ && iommu_dma_init_fq(domain))
+               domain->type = IOMMU_DOMAIN_DMA;
 
        return iova_reserve_iommu_regions(dev, domain);
 }
@@ -455,17 +471,17 @@ static dma_addr_t iommu_dma_alloc_iova(struct iommu_domain *domain,
 }
 
 static void iommu_dma_free_iova(struct iommu_dma_cookie *cookie,
-               dma_addr_t iova, size_t size, struct page *freelist)
+               dma_addr_t iova, size_t size, struct iommu_iotlb_gather *gather)
 {
        struct iova_domain *iovad = &cookie->iovad;
 
        /* The MSI case is only ever cleaning up its most recent allocation */
        if (cookie->type == IOMMU_DMA_MSI_COOKIE)
                cookie->msi_iova -= size;
-       else if (cookie->fq_domain)     /* non-strict mode */
+       else if (gather && gather->queued)
                queue_iova(iovad, iova_pfn(iovad, iova),
                                size >> iova_shift(iovad),
-                               (unsigned long)freelist);
+                               (unsigned long)gather->freelist);
        else
                free_iova_fast(iovad, iova_pfn(iovad, iova),
                                size >> iova_shift(iovad));
@@ -484,13 +500,14 @@ static void __iommu_dma_unmap(struct device *dev, dma_addr_t dma_addr,
        dma_addr -= iova_off;
        size = iova_align(iovad, size + iova_off);
        iommu_iotlb_gather_init(&iotlb_gather);
+       iotlb_gather.queued = READ_ONCE(cookie->fq_domain);
 
        unmapped = iommu_unmap_fast(domain, dma_addr, size, &iotlb_gather);
        WARN_ON(unmapped != size);
 
-       if (!cookie->fq_domain)
+       if (!iotlb_gather.queued)
                iommu_iotlb_sync(domain, &iotlb_gather);
-       iommu_dma_free_iova(cookie, dma_addr, size, iotlb_gather.freelist);
+       iommu_dma_free_iova(cookie, dma_addr, size, &iotlb_gather);
 }
 
 static void __iommu_dma_unmap_swiotlb(struct device *dev, dma_addr_t dma_addr,
@@ -1330,7 +1347,7 @@ void iommu_setup_dma_ops(struct device *dev, u64 dma_base, u64 dma_limit)
         * The IOMMU core code allocates the default DMA domain, which the
         * underlying IOMMU driver needs to support via the dma-iommu layer.
         */
-       if (domain->type == IOMMU_DOMAIN_DMA) {
+       if (iommu_is_dma_domain(domain)) {
                if (iommu_dma_init_domain(domain, dma_base, dma_limit, dev))
                        goto out_err;
                dev->dma_ops = &iommu_dma_ops;