dma-mapping: replace zone_dma_bits by zone_dma_limit

The hardware DMA limit might not be power of 2. When RAM range starts
above 0, say 4GB, DMA limit of 30 bits should end at 5GB.  A single high
bit can not encode this limit.

Use a plain  address for the DMA zone limit instead.

Since the DMA zone can now potentially span beyond 4GB physical limit of
DMA32, make sure to use DMA zone for GFP_DMA32 allocations in that case.

Signed-off-by: Catalin Marinas <catalin.marinas@arm.com>
Co-developed-by: Baruch Siach <baruch@tkos.co.il>
Signed-off-by: Baruch Siach <baruch@tkos.co.il>
Reviewed-by: Catalin Marinas <catalin.marinas@arm.com>
Reviewed-by: Petr Tesarik <ptesarik@suse.com>
Signed-off-by: Christoph Hellwig <hch@lst.de>

diff --git a/arch/arm64/mm/init.c b/arch/arm64/mm/init.c
index 9b5ab6818f7f39c4740b464e119f569430b8fc57..c45e2152ca9e91e40709cf139ffc8008ffd1d729 100644 (file)
--- a/arch/arm64/mm/init.c
+++ b/arch/arm64/mm/init.c
@@ -115,35 +115,35 @@ static void __init arch_reserve_crashkernel(void)
 }
 
 /*
- * Return the maximum physical address for a zone accessible by the given bits
- * limit. If DRAM starts above 32-bit, expand the zone to the maximum
+ * Return the maximum physical address for a zone given its limit.
+ * If DRAM starts above 32-bit, expand the zone to the maximum
  * available memory, otherwise cap it at 32-bit.
  */
-static phys_addr_t __init max_zone_phys(unsigned int zone_bits)
+static phys_addr_t __init max_zone_phys(phys_addr_t zone_limit)
 {
-       phys_addr_t zone_mask = DMA_BIT_MASK(zone_bits);
        phys_addr_t phys_start = memblock_start_of_DRAM();
 
        if (phys_start > U32_MAX)
-               zone_mask = PHYS_ADDR_MAX;
-       else if (phys_start > zone_mask)
-               zone_mask = U32_MAX;
+               zone_limit = PHYS_ADDR_MAX;
+       else if (phys_start > zone_limit)
+               zone_limit = U32_MAX;
 
-       return min(zone_mask, memblock_end_of_DRAM() - 1) + 1;
+       return min(zone_limit, memblock_end_of_DRAM() - 1) + 1;
 }
 
 static void __init zone_sizes_init(void)
 {
        unsigned long max_zone_pfns[MAX_NR_ZONES]  = {0};
-       unsigned int __maybe_unused acpi_zone_dma_bits;
-       unsigned int __maybe_unused dt_zone_dma_bits;
-       phys_addr_t __maybe_unused dma32_phys_limit = max_zone_phys(32);
+       phys_addr_t __maybe_unused acpi_zone_dma_limit;
+       phys_addr_t __maybe_unused dt_zone_dma_limit;
+       phys_addr_t __maybe_unused dma32_phys_limit =
+               max_zone_phys(DMA_BIT_MASK(32));
 
 #ifdef CONFIG_ZONE_DMA
-       acpi_zone_dma_bits = fls64(acpi_iort_dma_get_max_cpu_address());
-       dt_zone_dma_bits = fls64(of_dma_get_max_cpu_address(NULL));
-       zone_dma_bits = min3(32U, dt_zone_dma_bits, acpi_zone_dma_bits);
-       arm64_dma_phys_limit = max_zone_phys(zone_dma_bits);
+       acpi_zone_dma_limit = acpi_iort_dma_get_max_cpu_address();
+       dt_zone_dma_limit = of_dma_get_max_cpu_address(NULL);
+       zone_dma_limit = min(dt_zone_dma_limit, acpi_zone_dma_limit);
+       arm64_dma_phys_limit = max_zone_phys(zone_dma_limit);
        max_zone_pfns[ZONE_DMA] = PFN_DOWN(arm64_dma_phys_limit);
 #endif
 #ifdef CONFIG_ZONE_DMA32

diff --git a/arch/powerpc/mm/mem.c b/arch/powerpc/mm/mem.c
index da21cb018984eb84f39e22a280e918dcd955c8ac..7e217aa4a274ee4f50cda776b7b743fe38b681fe 100644 (file)
--- a/arch/powerpc/mm/mem.c
+++ b/arch/powerpc/mm/mem.c
@@ -216,7 +216,7 @@ static int __init mark_nonram_nosave(void)
  * everything else. GFP_DMA32 page allocations automatically fall back to
  * ZONE_DMA.
  *
- * By using 31-bit unconditionally, we can exploit zone_dma_bits to inform the
+ * By using 31-bit unconditionally, we can exploit zone_dma_limit to inform the
  * generic DMA mapping code.  32-bit only devices (if not handled by an IOMMU
  * anyway) will take a first dip into ZONE_NORMAL and get otherwise served by
  * ZONE_DMA.
@@ -230,6 +230,7 @@ void __init paging_init(void)
 {
        unsigned long long total_ram = memblock_phys_mem_size();
        phys_addr_t top_of_ram = memblock_end_of_DRAM();
+       int zone_dma_bits;
 
 #ifdef CONFIG_HIGHMEM
        unsigned long v = __fix_to_virt(FIX_KMAP_END);
@@ -256,6 +257,8 @@ void __init paging_init(void)
        else
                zone_dma_bits = 31;
 
+       zone_dma_limit = DMA_BIT_MASK(zone_dma_bits);
+
 #ifdef CONFIG_ZONE_DMA
        max_zone_pfns[ZONE_DMA] = min(max_low_pfn,
                                      1UL << (zone_dma_bits - PAGE_SHIFT));

diff --git a/arch/s390/mm/init.c b/arch/s390/mm/init.c
index e3d258f9e72610a9d28bc77bdb580daddfcd7db6..688abc65c79ea9574800f280f08f57c9f6381db6 100644 (file)
--- a/arch/s390/mm/init.c
+++ b/arch/s390/mm/init.c
@@ -97,7 +97,7 @@ void __init paging_init(void)
 
        vmem_map_init();
        sparse_init();
-       zone_dma_bits = 31;
+       zone_dma_limit = DMA_BIT_MASK(31);
        memset(max_zone_pfns, 0, sizeof(max_zone_pfns));
        max_zone_pfns[ZONE_DMA] = virt_to_pfn(MAX_DMA_ADDRESS);
        max_zone_pfns[ZONE_NORMAL] = max_low_pfn;

diff --git a/include/linux/dma-direct.h b/include/linux/dma-direct.h
index edbe13d007762b7850886ad1ad6a9a812aec7176..d7e30d4f7503a898a456df8eedf6a2cd284c35ff 100644 (file)
--- a/include/linux/dma-direct.h
+++ b/include/linux/dma-direct.h
@@ -12,7 +12,7 @@
 #include <linux/mem_encrypt.h>
 #include <linux/swiotlb.h>
 
-extern unsigned int zone_dma_bits;
+extern u64 zone_dma_limit;
 
 /*
  * Record the mapping of CPU physical to DMA addresses for a given region.

diff --git a/kernel/dma/direct.c b/kernel/dma/direct.c
index 4480a3cd92e0878f7eeb82a226100e23e6476d81..f2ba074a6a5425bce72b41f9db43a5bbe21b8102 100644 (file)
--- a/kernel/dma/direct.c
+++ b/kernel/dma/direct.c
@@ -20,7 +20,7 @@
  * it for entirely different regions. In that case the arch code needs to
  * override the variable below for dma-direct to work properly.
  */
-unsigned int zone_dma_bits __ro_after_init = 24;
+u64 zone_dma_limit __ro_after_init = DMA_BIT_MASK(24);
 
 static inline dma_addr_t phys_to_dma_direct(struct device *dev,
                phys_addr_t phys)
@@ -59,7 +59,7 @@ static gfp_t dma_direct_optimal_gfp_mask(struct device *dev, u64 *phys_limit)
         * zones.
         */
        *phys_limit = dma_to_phys(dev, dma_limit);
-       if (*phys_limit <= DMA_BIT_MASK(zone_dma_bits))
+       if (*phys_limit <= zone_dma_limit)
                return GFP_DMA;
        if (*phys_limit <= DMA_BIT_MASK(32))
                return GFP_DMA32;
@@ -580,7 +580,7 @@ int dma_direct_supported(struct device *dev, u64 mask)
         * part of the check.
         */
        if (IS_ENABLED(CONFIG_ZONE_DMA))
-               min_mask = min_t(u64, min_mask, DMA_BIT_MASK(zone_dma_bits));
+               min_mask = min_t(u64, min_mask, zone_dma_limit);
        return mask >= phys_to_dma_unencrypted(dev, min_mask);
 }
 

diff --git a/kernel/dma/pool.c b/kernel/dma/pool.c
index d10613eb0f639c26862b5782154c1227b5d4f9ae..7b04f7575796b8f3a4bd29ac39f56f1e2f07168f 100644 (file)
--- a/kernel/dma/pool.c
+++ b/kernel/dma/pool.c
@@ -70,9 +70,9 @@ static bool cma_in_zone(gfp_t gfp)
        /* CMA can't cross zone boundaries, see cma_activate_area() */
        end = cma_get_base(cma) + size - 1;
        if (IS_ENABLED(CONFIG_ZONE_DMA) && (gfp & GFP_DMA))
-               return end <= DMA_BIT_MASK(zone_dma_bits);
+               return end <= zone_dma_limit;
        if (IS_ENABLED(CONFIG_ZONE_DMA32) && (gfp & GFP_DMA32))
-               return end <= DMA_BIT_MASK(32);
+               return end <= max(DMA_BIT_MASK(32), zone_dma_limit);
        return true;
 }
 

diff --git a/kernel/dma/swiotlb.c b/kernel/dma/swiotlb.c
index df68d29740a035e5583c9529b9a481f394fba50d..abcf3fa63a56931c8cbdefea8b6b4b56fd7b2b51 100644 (file)
--- a/kernel/dma/swiotlb.c
+++ b/kernel/dma/swiotlb.c
@@ -450,9 +450,9 @@ int swiotlb_init_late(size_t size, gfp_t gfp_mask,
        if (!remap)
                io_tlb_default_mem.can_grow = true;
        if (IS_ENABLED(CONFIG_ZONE_DMA) && (gfp_mask & __GFP_DMA))
-               io_tlb_default_mem.phys_limit = DMA_BIT_MASK(zone_dma_bits);
+               io_tlb_default_mem.phys_limit = zone_dma_limit;
        else if (IS_ENABLED(CONFIG_ZONE_DMA32) && (gfp_mask & __GFP_DMA32))
-               io_tlb_default_mem.phys_limit = DMA_BIT_MASK(32);
+               io_tlb_default_mem.phys_limit = max(DMA_BIT_MASK(32), zone_dma_limit);
        else
                io_tlb_default_mem.phys_limit = virt_to_phys(high_memory - 1);
 #endif
@@ -629,7 +629,7 @@ static struct page *swiotlb_alloc_tlb(struct device *dev, size_t bytes,
        }
 
        gfp &= ~GFP_ZONEMASK;
-       if (phys_limit <= DMA_BIT_MASK(zone_dma_bits))
+       if (phys_limit <= zone_dma_limit)
                gfp |= __GFP_DMA;
        else if (phys_limit <= DMA_BIT_MASK(32))
                gfp |= __GFP_DMA32;