s390: Convert gmap to XArray

The three radix trees in gmap are all converted to the XArray.
This is another case where the multiple locks held mandates the use
of the xa_reserve() API.  The gmap_insert_rmap() function is
considerably simplified by using the advanced API;
gmap_radix_tree_free() turns out to just be xa_destroy(), and
gmap_rmap_radix_tree_free() is a nice little iteration followed
by xa_destroy().

Signed-off-by: Matthew Wilcox <willy@infradead.org>

diff --git a/arch/s390/include/asm/gmap.h b/arch/s390/include/asm/gmap.h
index fcbd638fb9f4c353523c7a3f1f7dd1b56ecfd0fe..dd2a984c7b4e51cc9728456f1d61d83b47f5e426 100644 (file)
--- a/arch/s390/include/asm/gmap.h
+++ b/arch/s390/include/asm/gmap.h
@@ -9,6 +9,8 @@
 #ifndef _ASM_S390_GMAP_H
 #define _ASM_S390_GMAP_H
 
+#include <linux/xarray.h>
+
 /* Generic bits for GMAP notification on DAT table entry changes. */
 #define GMAP_NOTIFY_SHADOW     0x2
 #define GMAP_NOTIFY_MPROT      0x1
@@ -22,14 +24,14 @@
  * @list: list head for the mm->context gmap list
  * @crst_list: list of all crst tables used in the guest address space
  * @mm: pointer to the parent mm_struct
- * @guest_to_host: radix tree with guest to host address translation
- * @host_to_guest: radix tree with pointer to segment table entries
+ * @guest_to_host: guest to host address translation
+ * @host_to_guest: pointers to segment table entries
  * @guest_table_lock: spinlock to protect all entries in the guest page table
  * @ref_count: reference counter for the gmap structure
  * @table: pointer to the page directory
  * @asce: address space control element for gmap page table
  * @pfault_enabled: defines if pfaults are applicable for the guest
- * @host_to_rmap: radix tree with gmap_rmap lists
+ * @host_to_rmap: gmap_rmap lists
  * @children: list of shadow gmap structures
  * @pt_list: list of all page tables used in the shadow guest address space
  * @shadow_lock: spinlock to protect the shadow gmap list
@@ -43,8 +45,8 @@ struct gmap {
        struct list_head list;
        struct list_head crst_list;
        struct mm_struct *mm;
-       struct radix_tree_root guest_to_host;
-       struct radix_tree_root host_to_guest;
+       struct xarray guest_to_host;
+       struct xarray host_to_guest;
        spinlock_t guest_table_lock;
        atomic_t ref_count;
        unsigned long *table;
@@ -53,7 +55,7 @@ struct gmap {
        void *private;
        bool pfault_enabled;
        /* Additional data for shadow guest address spaces */
-       struct radix_tree_root host_to_rmap;
+       struct xarray host_to_rmap;
        struct list_head children;
        struct list_head pt_list;
        spinlock_t shadow_lock;

diff --git a/arch/s390/include/asm/pgtable.h b/arch/s390/include/asm/pgtable.h
index 9b274fcaacb680b1d2cc63c2ac7c3fe57ec1cde1..b670a2e90b0b74cb72b77dc75d28bde6714c1bee 100644 (file)
--- a/arch/s390/include/asm/pgtable.h
+++ b/arch/s390/include/asm/pgtable.h
@@ -15,7 +15,6 @@
 #include <linux/sched.h>
 #include <linux/mm_types.h>
 #include <linux/page-flags.h>
-#include <linux/radix-tree.h>
 #include <linux/atomic.h>
 #include <asm/bug.h>
 #include <asm/page.h>

diff --git a/arch/s390/mm/gmap.c b/arch/s390/mm/gmap.c
index 39c3a6e3d26218161bd63e5746d862b9af335edd..d59cd5944c5737cf7e38a8f24326e98c1657d571 100644 (file)
--- a/arch/s390/mm/gmap.c
+++ b/arch/s390/mm/gmap.c
@@ -62,9 +62,9 @@ static struct gmap *gmap_alloc(unsigned long limit)
        INIT_LIST_HEAD(&gmap->crst_list);
        INIT_LIST_HEAD(&gmap->children);
        INIT_LIST_HEAD(&gmap->pt_list);
-       INIT_RADIX_TREE(&gmap->guest_to_host, GFP_KERNEL);
-       INIT_RADIX_TREE(&gmap->host_to_guest, GFP_ATOMIC);
-       INIT_RADIX_TREE(&gmap->host_to_rmap, GFP_ATOMIC);
+       xa_init(&gmap->guest_to_host);
+       xa_init(&gmap->host_to_guest);
+       xa_init(&gmap->host_to_rmap);
        spin_lock_init(&gmap->guest_table_lock);
        spin_lock_init(&gmap->shadow_lock);
        atomic_set(&gmap->ref_count, 1);
@@ -123,55 +123,16 @@ static void gmap_flush_tlb(struct gmap *gmap)
                __tlb_flush_global();
 }
 
-static void gmap_radix_tree_free(struct radix_tree_root *root)
-{
-       struct radix_tree_iter iter;
-       unsigned long indices[16];
-       unsigned long index;
-       void __rcu **slot;
-       int i, nr;
-
-       /* A radix tree is freed by deleting all of its entries */
-       index = 0;
-       do {
-               nr = 0;
-               radix_tree_for_each_slot(slot, root, &iter, index) {
-                       indices[nr] = iter.index;
-                       if (++nr == 16)
-                               break;
-               }
-               for (i = 0; i < nr; i++) {
-                       index = indices[i];
-                       radix_tree_delete(root, index);
-               }
-       } while (nr > 0);
-}
-
-static void gmap_rmap_radix_tree_free(struct radix_tree_root *root)
+static void gmap_rmap_free(struct xarray *xa)
 {
        struct gmap_rmap *rmap, *rnext, *head;
-       struct radix_tree_iter iter;
-       unsigned long indices[16];
        unsigned long index;
-       void __rcu **slot;
-       int i, nr;
-
-       /* A radix tree is freed by deleting all of its entries */
-       index = 0;
-       do {
-               nr = 0;
-               radix_tree_for_each_slot(slot, root, &iter, index) {
-                       indices[nr] = iter.index;
-                       if (++nr == 16)
-                               break;
-               }
-               for (i = 0; i < nr; i++) {
-                       index = indices[i];
-                       head = radix_tree_delete(root, index);
-                       gmap_for_each_rmap_safe(rmap, rnext, head)
-                               kfree(rmap);
-               }
-       } while (nr > 0);
+
+       xa_for_each(xa, index, head) {
+               gmap_for_each_rmap_safe(rmap, rnext, head)
+                       kfree(rmap);
+       }
+       xa_destroy(xa);
 }
 
 /**
@@ -190,15 +151,15 @@ static void gmap_free(struct gmap *gmap)
        /* Free all segment & region tables. */
        list_for_each_entry_safe(page, next, &gmap->crst_list, lru)
                __free_pages(page, CRST_ALLOC_ORDER);
-       gmap_radix_tree_free(&gmap->guest_to_host);
-       gmap_radix_tree_free(&gmap->host_to_guest);
+       xa_destroy(&gmap->guest_to_host);
+       xa_destroy(&gmap->host_to_guest);
 
        /* Free additional data for a shadow gmap */
        if (gmap_is_shadow(gmap)) {
                /* Free all page tables. */
                list_for_each_entry_safe(page, next, &gmap->pt_list, lru)
                        page_table_free_pgste(page);
-               gmap_rmap_radix_tree_free(&gmap->host_to_rmap);
+               gmap_rmap_free(&gmap->host_to_rmap);
                /* Release reference to the parent */
                gmap_put(gmap->parent);
        }
@@ -360,7 +321,7 @@ static int __gmap_unlink_by_vmaddr(struct gmap *gmap, unsigned long vmaddr)
 
        BUG_ON(gmap_is_shadow(gmap));
        spin_lock(&gmap->guest_table_lock);
-       entry = radix_tree_delete(&gmap->host_to_guest, vmaddr >> PMD_SHIFT);
+       entry = xa_erase(&gmap->host_to_guest, vmaddr >> PMD_SHIFT);
        if (entry) {
                flush = (*entry != _SEGMENT_ENTRY_EMPTY);
                *entry = _SEGMENT_ENTRY_EMPTY;
@@ -380,7 +341,7 @@ static int __gmap_unmap_by_gaddr(struct gmap *gmap, unsigned long gaddr)
 {
        unsigned long vmaddr;
 
-       vmaddr = (unsigned long) radix_tree_delete(&gmap->guest_to_host,
+       vmaddr = (unsigned long) xa_erase(&gmap->guest_to_host,
                                                   gaddr >> PMD_SHIFT);
        return vmaddr ? __gmap_unlink_by_vmaddr(gmap, vmaddr) : 0;
 }
@@ -443,9 +404,9 @@ int gmap_map_segment(struct gmap *gmap, unsigned long from,
                /* Remove old translation */
                flush |= __gmap_unmap_by_gaddr(gmap, to + off);
                /* Store new translation */
-               if (radix_tree_insert(&gmap->guest_to_host,
+               if (xa_is_err(xa_store(&gmap->guest_to_host,
                                      (to + off) >> PMD_SHIFT,
-                                     (void *) from + off))
+                                     (void *) from + off, GFP_KERNEL)))
                        break;
        }
        up_write(&gmap->mm->mmap_sem);
@@ -476,7 +437,7 @@ unsigned long __gmap_translate(struct gmap *gmap, unsigned long gaddr)
        unsigned long vmaddr;
 
        vmaddr = (unsigned long)
-               radix_tree_lookup(&gmap->guest_to_host, gaddr >> PMD_SHIFT);
+               xa_load(&gmap->guest_to_host, gaddr >> PMD_SHIFT);
        /* Note: guest_to_host is empty for a shadow gmap */
        return vmaddr ? (vmaddr | (gaddr & ~PMD_MASK)) : -EFAULT;
 }
@@ -593,15 +554,17 @@ int __gmap_link(struct gmap *gmap, unsigned long gaddr, unsigned long vmaddr)
        /* Are we allowed to use huge pages? */
        if (pmd_large(*pmd) && !gmap->mm->context.allow_gmap_hpage_1m)
                return -EFAULT;
-       /* Link gmap segment table entry location to page table. */
-       rc = radix_tree_preload(GFP_KERNEL);
-       if (rc)
+       /* Reserve an entry for this address before we acquire locks */
+       rc = xa_insert(&gmap->host_to_guest, vmaddr >> PMD_SHIFT, NULL,
+                       GFP_KERNEL);
+       if (rc == -ENOMEM)
                return rc;
+       /* Link gmap segment table entry location to page table. */
        ptl = pmd_lock(mm, pmd);
        spin_lock(&gmap->guest_table_lock);
        if (*table == _SEGMENT_ENTRY_EMPTY) {
-               rc = radix_tree_insert(&gmap->host_to_guest,
-                                      vmaddr >> PMD_SHIFT, table);
+               rc = xa_err(xa_store(&gmap->host_to_guest, vmaddr >> PMD_SHIFT,
+                                       table, GFP_ATOMIC));
                if (!rc) {
                        if (pmd_large(*pmd)) {
                                *table = (pmd_val(*pmd) &
@@ -620,7 +583,6 @@ int __gmap_link(struct gmap *gmap, unsigned long gaddr, unsigned long vmaddr)
        }
        spin_unlock(&gmap->guest_table_lock);
        spin_unlock(ptl);
-       radix_tree_preload_end();
        return rc;
 }
 
@@ -678,7 +640,7 @@ void __gmap_zap(struct gmap *gmap, unsigned long gaddr)
        pte_t *ptep;
 
        /* Find the vm address for the guest address */
-       vmaddr = (unsigned long) radix_tree_lookup(&gmap->guest_to_host,
+       vmaddr = (unsigned long) xa_load(&gmap->guest_to_host,
                                                   gaddr >> PMD_SHIFT);
        if (vmaddr) {
                vmaddr |= gaddr & ~PMD_MASK;
@@ -700,8 +662,7 @@ void gmap_discard(struct gmap *gmap, unsigned long from, unsigned long to)
        for (gaddr = from; gaddr < to;
             gaddr = (gaddr + PMD_SIZE) & PMD_MASK) {
                /* Find the vm address for the guest address */
-               vmaddr = (unsigned long)
-                       radix_tree_lookup(&gmap->guest_to_host,
+               vmaddr = (unsigned long) xa_load(&gmap->guest_to_host,
                                          gaddr >> PMD_SHIFT);
                if (!vmaddr)
                        continue;
@@ -1162,30 +1123,50 @@ int gmap_read_table(struct gmap *gmap, unsigned long gaddr, unsigned long *val)
 }
 EXPORT_SYMBOL_GPL(gmap_read_table);
 
+/*
+ * If there is an entry for this address already, returns -EBUSY.  If
+ * we run out of memory, returns -ENOMEM.  If we inserted a reserved entry,
+ * returns 0
+ */
+static inline int gmap_reserve_rmap(struct gmap *sg, unsigned long vmaddr)
+{
+       return xa_insert(&sg->host_to_rmap, vmaddr >> PAGE_SHIFT, NULL,
+                       GFP_KERNEL);
+}
+
+/*
+ * If there is a reserved entry for this address, free the entry (and
+ * potentially other memory in the XArray).
+ */
+static inline void gmap_release_rmap(struct gmap *sg, unsigned long vmaddr)
+{
+       xa_cmpxchg(&sg->host_to_rmap, vmaddr >> PAGE_SHIFT, XA_ZERO_ENTRY,
+                       NULL, GFP_KERNEL);
+}
+
 /**
- * gmap_insert_rmap - add a rmap to the host_to_rmap radix tree
+ * gmap_insert_rmap - add a rmap to the host_to_rmap
  * @sg: pointer to the shadow guest address space structure
  * @vmaddr: vm address associated with the rmap
  * @rmap: pointer to the rmap structure
  *
- * Called with the sg->guest_table_lock
+ * Called with the sg->guest_table_lock and page table lock held
  */
 static inline void gmap_insert_rmap(struct gmap *sg, unsigned long vmaddr,
                                    struct gmap_rmap *rmap)
 {
-       void __rcu **slot;
+       XA_STATE(xas, &sg->host_to_rmap, vmaddr >> PAGE_SHIFT);
+       void *entry;
 
        BUG_ON(!gmap_is_shadow(sg));
-       slot = radix_tree_lookup_slot(&sg->host_to_rmap, vmaddr >> PAGE_SHIFT);
-       if (slot) {
-               rmap->next = radix_tree_deref_slot_protected(slot,
-                                                       &sg->guest_table_lock);
-               radix_tree_replace_slot(&sg->host_to_rmap, slot, rmap);
-       } else {
-               rmap->next = NULL;
-               radix_tree_insert(&sg->host_to_rmap, vmaddr >> PAGE_SHIFT,
-                                 rmap);
-       }
+
+       xas_lock(&xas);
+       entry = xas_load(&xas);
+       if (entry == XA_ZERO_ENTRY)
+               entry = NULL;
+       rmap->next = entry;
+       xas_store(&xas, rmap);
+       xas_unlock(&xas);
 }
 
 /**
@@ -1218,8 +1199,8 @@ static int gmap_protect_rmap(struct gmap *sg, unsigned long raddr,
                if (!rmap)
                        return -ENOMEM;
                rmap->raddr = raddr;
-               rc = radix_tree_preload(GFP_KERNEL);
-               if (rc) {
+               rc = gmap_reserve_rmap(sg, vmaddr);
+               if (rc == -ENOMEM) {
                        kfree(rmap);
                        return rc;
                }
@@ -1234,8 +1215,8 @@ static int gmap_protect_rmap(struct gmap *sg, unsigned long raddr,
                        spin_unlock(&sg->guest_table_lock);
                        gmap_pte_op_end(ptl);
                }
-               radix_tree_preload_end();
                if (rc) {
+                       gmap_release_rmap(sg, vmaddr);
                        kfree(rmap);
                        rc = gmap_pte_op_fixup(parent, paddr, vmaddr, PROT_READ);
                        if (rc)
@@ -2123,8 +2104,8 @@ int gmap_shadow_page(struct gmap *sg, unsigned long saddr, pte_t pte)
                        rc = vmaddr;
                        break;
                }
-               rc = radix_tree_preload(GFP_KERNEL);
-               if (rc)
+               rc = gmap_reserve_rmap(sg, vmaddr);
+               if (rc == -ENOMEM)
                        break;
                rc = -EAGAIN;
                sptep = gmap_pte_op_walk(parent, paddr, &ptl);
@@ -2135,7 +2116,7 @@ int gmap_shadow_page(struct gmap *sg, unsigned long saddr, pte_t pte)
                        if (!tptep) {
                                spin_unlock(&sg->guest_table_lock);
                                gmap_pte_op_end(ptl);
-                               radix_tree_preload_end();
+                               gmap_release_rmap(sg, vmaddr);
                                break;
                        }
                        rc = ptep_shadow_pte(sg->mm, saddr, sptep, tptep, pte);
@@ -2147,8 +2128,9 @@ int gmap_shadow_page(struct gmap *sg, unsigned long saddr, pte_t pte)
                        }
                        gmap_pte_op_end(ptl);
                        spin_unlock(&sg->guest_table_lock);
+                       if (rc < 0)
+                               gmap_release_rmap(sg, vmaddr);
                }
-               radix_tree_preload_end();
                if (!rc)
                        break;
                rc = gmap_pte_op_fixup(parent, paddr, vmaddr, prot);
@@ -2191,7 +2173,7 @@ static void gmap_shadow_notify(struct gmap *sg, unsigned long vmaddr,
                return;
        }
        /* Remove the page table tree from on specific entry */
-       head = radix_tree_delete(&sg->host_to_rmap, vmaddr >> PAGE_SHIFT);
+       head = xa_erase(&sg->host_to_rmap, vmaddr >> PAGE_SHIFT);
        gmap_for_each_rmap_safe(rmap, rnext, head) {
                bits = rmap->raddr & _SHADOW_RMAP_MASK;
                raddr = rmap->raddr ^ bits;
@@ -2239,8 +2221,7 @@ void ptep_notify(struct mm_struct *mm, unsigned long vmaddr,
        rcu_read_lock();
        list_for_each_entry_rcu(gmap, &mm->context.gmap_list, list) {
                spin_lock(&gmap->guest_table_lock);
-               table = radix_tree_lookup(&gmap->host_to_guest,
-                                         vmaddr >> PMD_SHIFT);
+               table = xa_load(&gmap->host_to_guest, vmaddr >> PMD_SHIFT);
                if (table)
                        gaddr = __gmap_segment_gaddr(table) + offset;
                spin_unlock(&gmap->guest_table_lock);
@@ -2304,7 +2285,7 @@ static void gmap_pmdp_clear(struct mm_struct *mm, unsigned long vmaddr,
        rcu_read_lock();
        list_for_each_entry_rcu(gmap, &mm->context.gmap_list, list) {
                spin_lock(&gmap->guest_table_lock);
-               pmdp = (pmd_t *)radix_tree_delete(&gmap->host_to_guest,
+               pmdp = (pmd_t *)xa_erase(&gmap->host_to_guest,
                                                  vmaddr >> PMD_SHIFT);
                if (pmdp) {
                        gaddr = __gmap_segment_gaddr((unsigned long *)pmdp);
@@ -2357,8 +2338,7 @@ void gmap_pmdp_idte_local(struct mm_struct *mm, unsigned long vmaddr)
        rcu_read_lock();
        list_for_each_entry_rcu(gmap, &mm->context.gmap_list, list) {
                spin_lock(&gmap->guest_table_lock);
-               entry = radix_tree_delete(&gmap->host_to_guest,
-                                         vmaddr >> PMD_SHIFT);
+               entry = xa_erase(&gmap->host_to_guest, vmaddr >> PMD_SHIFT);
                if (entry) {
                        pmdp = (pmd_t *)entry;
                        gaddr = __gmap_segment_gaddr(entry);
@@ -2392,8 +2372,7 @@ void gmap_pmdp_idte_global(struct mm_struct *mm, unsigned long vmaddr)
        rcu_read_lock();
        list_for_each_entry_rcu(gmap, &mm->context.gmap_list, list) {
                spin_lock(&gmap->guest_table_lock);
-               entry = radix_tree_delete(&gmap->host_to_guest,
-                                         vmaddr >> PMD_SHIFT);
+               entry = xa_erase(&gmap->host_to_guest, vmaddr >> PMD_SHIFT);
                if (entry) {
                        pmdp = (pmd_t *)entry;
                        gaddr = __gmap_segment_gaddr(entry);