dma-debug: Convert to XArray

The dma-debug code was using the radix tree tags in a tremendously
inefficient way.  Instead, chain dma entries which hit the same
cacheline just like the s390 and btrfs code do.

Signed-off-by: Matthew Wilcox <willy@infradead.org>

diff --git a/kernel/dma/debug.c b/kernel/dma/debug.c
index 099002d84f466af0a8e421d035ac8eb119e08bd8..61eff138d74c07946f9707421994acae1bfde47f 100644 (file)
--- a/kernel/dma/debug.c
+++ b/kernel/dma/debug.c
@@ -19,6 +19,7 @@
 #include <linux/uaccess.h>
 #include <linux/export.h>
 #include <linux/device.h>
+#include <linux/xarray.h>
 #include <linux/types.h>
 #include <linux/sched.h>
 #include <linux/ctype.h>
@@ -53,6 +54,7 @@ enum map_err_types {
 /**
  * struct dma_debug_entry - track a dma_map* or dma_alloc_coherent mapping
  * @list: node on pre-allocated free_entries list
+ * @next: chain of colliding entries
  * @dev: 'dev' argument to dma_map_{page|single|sg} or dma_alloc_coherent
  * @type: single, page, sg, coherent
  * @pfn: page frame of the start address
@@ -66,6 +68,7 @@ enum map_err_types {
  */
 struct dma_debug_entry {
        struct list_head list;
+       struct dma_debug_entry *next;
        struct device    *dev;
        int              type;
        unsigned long    pfn;
@@ -446,9 +449,7 @@ void debug_dma_dump_mappings(struct device *dev)
  * At any time debug_dma_assert_idle() can be called to trigger a
  * warning if any cachelines in the given page are in the active set.
  */
-static RADIX_TREE(dma_active_cacheline, GFP_NOWAIT);
-static DEFINE_SPINLOCK(radix_lock);
-#define ACTIVE_CACHELINE_MAX_OVERLAP ((1 << RADIX_TREE_MAX_TAGS) - 1)
+static DEFINE_XARRAY_FLAGS(dma_active_cacheline, XA_FLAGS_LOCK_IRQ);
 #define CACHELINE_PER_PAGE_SHIFT (PAGE_SHIFT - L1_CACHE_SHIFT)
 #define CACHELINES_PER_PAGE (1 << CACHELINE_PER_PAGE_SHIFT)
 
@@ -458,61 +459,11 @@ static phys_addr_t to_cacheline_number(struct dma_debug_entry *entry)
                (entry->offset >> L1_CACHE_SHIFT);
 }
 
-static int active_cacheline_read_overlap(phys_addr_t cln)
-{
-       int overlap = 0, i;
-
-       for (i = RADIX_TREE_MAX_TAGS - 1; i >= 0; i--)
-               if (radix_tree_tag_get(&dma_active_cacheline, cln, i))
-                       overlap |= 1 << i;
-       return overlap;
-}
-
-static int active_cacheline_set_overlap(phys_addr_t cln, int overlap)
-{
-       int i;
-
-       if (overlap > ACTIVE_CACHELINE_MAX_OVERLAP || overlap < 0)
-               return overlap;
-
-       for (i = RADIX_TREE_MAX_TAGS - 1; i >= 0; i--)
-               if (overlap & 1 << i)
-                       radix_tree_tag_set(&dma_active_cacheline, cln, i);
-               else
-                       radix_tree_tag_clear(&dma_active_cacheline, cln, i);
-
-       return overlap;
-}
-
-static void active_cacheline_inc_overlap(phys_addr_t cln)
-{
-       int overlap = active_cacheline_read_overlap(cln);
-
-       overlap = active_cacheline_set_overlap(cln, ++overlap);
-
-       /* If we overflowed the overlap counter then we're potentially
-        * leaking dma-mappings.  Otherwise, if maps and unmaps are
-        * balanced then this overflow may cause false negatives in
-        * debug_dma_assert_idle() as the cacheline may be marked idle
-        * prematurely.
-        */
-       WARN_ONCE(overlap > ACTIVE_CACHELINE_MAX_OVERLAP,
-                 pr_fmt("exceeded %d overlapping mappings of cacheline %pa\n"),
-                 ACTIVE_CACHELINE_MAX_OVERLAP, &cln);
-}
-
-static int active_cacheline_dec_overlap(phys_addr_t cln)
-{
-       int overlap = active_cacheline_read_overlap(cln);
-
-       return active_cacheline_set_overlap(cln, --overlap);
-}
-
 static int active_cacheline_insert(struct dma_debug_entry *entry)
 {
        phys_addr_t cln = to_cacheline_number(entry);
+       XA_STATE(xas, &dma_active_cacheline, cln);
        unsigned long flags;
-       int rc;
 
        /* If the device is not writing memory then we don't have any
         * concerns about the cpu consuming stale data.  This mitigates
@@ -521,37 +472,47 @@ static int active_cacheline_insert(struct dma_debug_entry *entry)
        if (entry->direction == DMA_TO_DEVICE)
                return 0;
 
-       spin_lock_irqsave(&radix_lock, flags);
-       rc = radix_tree_insert(&dma_active_cacheline, cln, entry);
-       if (rc == -EEXIST)
-               active_cacheline_inc_overlap(cln);
-       spin_unlock_irqrestore(&radix_lock, flags);
+       xas_lock_irqsave(&xas, flags);
+       entry->next = xas_load(&xas);
+       xas_store(&xas, entry);
+       xas_unlock_irqrestore(&xas, flags);
 
-       return rc;
+       /* We cannot sleep to allocate memory; just return ENOMEM */
+       return xas_error(&xas);
 }
 
 static void active_cacheline_remove(struct dma_debug_entry *entry)
 {
        phys_addr_t cln = to_cacheline_number(entry);
+       XA_STATE(xas, &dma_active_cacheline, cln);
+       struct dma_debug_entry *next, *prev;
        unsigned long flags;
 
        /* ...mirror the insert case */
        if (entry->direction == DMA_TO_DEVICE)
                return;
 
-       spin_lock_irqsave(&radix_lock, flags);
-       /* since we are counting overlaps the final put of the
-        * cacheline will occur when the overlap count is 0.
-        * active_cacheline_dec_overlap() returns -1 in that case
-        */
-       if (active_cacheline_dec_overlap(cln) < 0)
-               radix_tree_delete(&dma_active_cacheline, cln);
-       spin_unlock_irqrestore(&radix_lock, flags);
+       xas_lock_irqsave(&xas, flags);
+       next = xas_load(&xas);
+       if (next == entry) {
+               xas_store(&xas, entry->next);
+       } else {
+               do {
+                       prev = next;
+                       next = next->next;
+                       if (!next)
+                               goto out;
+               } while (next != entry);
+
+               prev->next = entry->next;
+       }
+out:
+       xas_unlock_irqrestore(&xas, flags);
 }
 
 /**
  * debug_dma_assert_idle() - assert that a page is not undergoing dma
- * @page: page to lookup in the dma_active_cacheline tree
+ * @page: page to lookup in the dma_active_cacheline array
  *
  * Place a call to this routine in cases where the cpu touching the page
  * before the dma completes (page is dma_unmapped) will lead to data
@@ -559,12 +520,8 @@ static void active_cacheline_remove(struct dma_debug_entry *entry)
  */
 void debug_dma_assert_idle(struct page *page)
 {
-       static struct dma_debug_entry *ents[CACHELINES_PER_PAGE];
-       struct dma_debug_entry *entry = NULL;
-       void **results = (void **) &ents;
-       unsigned int nents, i;
-       unsigned long flags;
-       phys_addr_t cln;
+       struct dma_debug_entry *entry;
+       unsigned long cln;
 
        if (dma_debug_disabled())
                return;
@@ -572,21 +529,9 @@ void debug_dma_assert_idle(struct page *page)
        if (!page)
                return;
 
-       cln = (phys_addr_t) page_to_pfn(page) << CACHELINE_PER_PAGE_SHIFT;
-       spin_lock_irqsave(&radix_lock, flags);
-       nents = radix_tree_gang_lookup(&dma_active_cacheline, results, cln,
-                                      CACHELINES_PER_PAGE);
-       for (i = 0; i < nents; i++) {
-               phys_addr_t ent_cln = to_cacheline_number(ents[i]);
-
-               if (ent_cln == cln) {
-                       entry = ents[i];
-                       break;
-               } else if (ent_cln >= cln + CACHELINES_PER_PAGE)
-                       break;
-       }
-       spin_unlock_irqrestore(&radix_lock, flags);
-
+       cln = page_to_pfn(page) << CACHELINE_PER_PAGE_SHIFT;
+       entry = xa_find(&dma_active_cacheline, &cln,
+                       cln + CACHELINES_PER_PAGE - 1, XA_PRESENT);
        if (!entry)
                return;