Merge tag 'dma-mapping-6.9-2024-03-24' of git://git.infradead.org/users/hch/dma-mapping

[linux-2.6-microblaze.git] / kernel / dma / swiotlb.c

diff --git a/kernel/dma/swiotlb.c b/kernel/dma/swiotlb.c
index 77974ce..86fe172 100644 (file)
--- a/kernel/dma/swiotlb.c
+++ b/kernel/dma/swiotlb.c
@@ -1003,8 +1003,7 @@ static int swiotlb_search_pool_area(struct device *dev, struct io_tlb_pool *pool
        dma_addr_t tbl_dma_addr =
                phys_to_dma_unencrypted(dev, pool->start) & boundary_mask;
        unsigned long max_slots = get_max_slots(boundary_mask);
-       unsigned int iotlb_align_mask =
-               dma_get_min_align_mask(dev) | alloc_align_mask;
+       unsigned int iotlb_align_mask = dma_get_min_align_mask(dev);
        unsigned int nslots = nr_slots(alloc_size), stride;
        unsigned int offset = swiotlb_align_offset(dev, orig_addr);
        unsigned int index, slots_checked, count = 0, i;
@@ -1016,18 +1015,29 @@ static int swiotlb_search_pool_area(struct device *dev, struct io_tlb_pool *pool
        BUG_ON(area_index >= pool->nareas);
 
        /*
-        * For allocations of PAGE_SIZE or larger only look for page aligned
-        * allocations.
+        * Historically, swiotlb allocations >= PAGE_SIZE were guaranteed to be
+        * page-aligned in the absence of any other alignment requirements.
+        * 'alloc_align_mask' was later introduced to specify the alignment
+        * explicitly, however this is passed as zero for streaming mappings
+        * and so we preserve the old behaviour there in case any drivers are
+        * relying on it.
         */
-       if (alloc_size >= PAGE_SIZE)
-               iotlb_align_mask |= ~PAGE_MASK;
-       iotlb_align_mask &= ~(IO_TLB_SIZE - 1);
+       if (!alloc_align_mask && !iotlb_align_mask && alloc_size >= PAGE_SIZE)
+               alloc_align_mask = PAGE_SIZE - 1;
+
+       /*
+        * Ensure that the allocation is at least slot-aligned and update
+        * 'iotlb_align_mask' to ignore bits that will be preserved when
+        * offsetting into the allocation.
+        */
+       alloc_align_mask |= (IO_TLB_SIZE - 1);
+       iotlb_align_mask &= ~alloc_align_mask;
 
        /*
         * For mappings with an alignment requirement don't bother looping to
         * unaligned slots once we found an aligned one.
         */
-       stride = (iotlb_align_mask >> IO_TLB_SHIFT) + 1;
+       stride = get_max_slots(max(alloc_align_mask, iotlb_align_mask));
 
        spin_lock_irqsave(&area->lock, flags);
        if (unlikely(nslots > pool->area_nslabs - area->used))
@@ -1037,11 +1047,14 @@ static int swiotlb_search_pool_area(struct device *dev, struct io_tlb_pool *pool
        index = area->index;
 
        for (slots_checked = 0; slots_checked < pool->area_nslabs; ) {
+               phys_addr_t tlb_addr;
+
                slot_index = slot_base + index;
+               tlb_addr = slot_addr(tbl_dma_addr, slot_index);
 
-               if (orig_addr &&
-                   (slot_addr(tbl_dma_addr, slot_index) &
-                    iotlb_align_mask) != (orig_addr & iotlb_align_mask)) {
+               if ((tlb_addr & alloc_align_mask) ||
+                   (orig_addr && (tlb_addr & iotlb_align_mask) !=
+                                 (orig_addr & iotlb_align_mask))) {
                        index = wrap_area_index(pool, index + 1);
                        slots_checked++;
                        continue;
@@ -1677,16 +1690,24 @@ struct page *swiotlb_alloc(struct device *dev, size_t size)
        struct io_tlb_mem *mem = dev->dma_io_tlb_mem;
        struct io_tlb_pool *pool;
        phys_addr_t tlb_addr;
+       unsigned int align;
        int index;
 
        if (!mem)
                return NULL;
 
-       index = swiotlb_find_slots(dev, 0, size, 0, &pool);
+       align = (1 << (get_order(size) + PAGE_SHIFT)) - 1;
+       index = swiotlb_find_slots(dev, 0, size, align, &pool);
        if (index == -1)
                return NULL;
 
        tlb_addr = slot_addr(pool->start, index);
+       if (unlikely(!PAGE_ALIGNED(tlb_addr))) {
+               dev_WARN_ONCE(dev, 1, "Cannot allocate pages from non page-aligned swiotlb addr 0x%pa.\n",
+                             &tlb_addr);
+               swiotlb_release_slots(dev, tlb_addr);
+               return NULL;
+       }
 
        return pfn_to_page(PFN_DOWN(tlb_addr));
 }