mm/hmm: let users to tag specific PFN with DMA mapped bit

Introduce new sticky flag (HMM_PFN_DMA_MAPPED), which isn't overwritten
by HMM range fault. Such flag allows users to tag specific PFNs with information
if this specific PFN was already DMA mapped.

Signed-off-by: Leon Romanovsky <leonro@nvidia.com>

diff --git a/include/linux/hmm.h b/include/linux/hmm.h
index 126a365716676a0eecb4b1e6cb97f7f8df1bce9a..2999697db83a9726f6e44e6df777a753bf558b01 100644 (file)
--- a/include/linux/hmm.h
+++ b/include/linux/hmm.h
@@ -23,6 +23,8 @@ struct mmu_interval_notifier;
  * HMM_PFN_WRITE - if the page memory can be written to (requires HMM_PFN_VALID)
  * HMM_PFN_ERROR - accessing the pfn is impossible and the device should
  *                 fail. ie poisoned memory, special pages, no vma, etc
+ * HMM_PFN_DMA_MAPPED - Flag preserved on input-to-output transformation
+ *                     to mark that page is already DMA mapped
  *
  * On input:
  * 0                 - Return the current state of the page, do not fault it.
@@ -36,6 +38,8 @@ enum hmm_pfn_flags {
        HMM_PFN_VALID = 1UL << (BITS_PER_LONG - 1),
        HMM_PFN_WRITE = 1UL << (BITS_PER_LONG - 2),
        HMM_PFN_ERROR = 1UL << (BITS_PER_LONG - 3),
+       /* Sticky lag, carried from Input to Output */
+       HMM_PFN_DMA_MAPPED = 1UL << (BITS_PER_LONG - 7),
        HMM_PFN_ORDER_SHIFT = (BITS_PER_LONG - 8),
 
        /* Input flags */

diff --git a/mm/hmm.c b/mm/hmm.c
index 7e0229ae4a5a6bfc06bc8fde06abf31f6198093c..2a0c34d7cb2bd141eaf8b89192490bdac3aca8ea 100644 (file)
--- a/mm/hmm.c
+++ b/mm/hmm.c
@@ -44,8 +44,10 @@ static int hmm_pfns_fill(unsigned long addr, unsigned long end,
 {
        unsigned long i = (addr - range->start) >> PAGE_SHIFT;
 
-       for (; addr < end; addr += PAGE_SIZE, i++)
-               range->hmm_pfns[i] = cpu_flags;
+       for (; addr < end; addr += PAGE_SIZE, i++) {
+               range->hmm_pfns[i] &= HMM_PFN_DMA_MAPPED;
+               range->hmm_pfns[i] |= cpu_flags;
+       }
        return 0;
 }
 
@@ -202,8 +204,10 @@ static int hmm_vma_handle_pmd(struct mm_walk *walk, unsigned long addr,
                return hmm_vma_fault(addr, end, required_fault, walk);
 
        pfn = pmd_pfn(pmd) + ((addr & ~PMD_MASK) >> PAGE_SHIFT);
-       for (i = 0; addr < end; addr += PAGE_SIZE, i++, pfn++)
-               hmm_pfns[i] = pfn | cpu_flags;
+       for (i = 0; addr < end; addr += PAGE_SIZE, i++, pfn++) {
+               hmm_pfns[i] &= HMM_PFN_DMA_MAPPED;
+               hmm_pfns[i] |= pfn | cpu_flags;
+       }
        return 0;
 }
 #else /* CONFIG_TRANSPARENT_HUGEPAGE */
@@ -236,7 +240,7 @@ static int hmm_vma_handle_pte(struct mm_walk *walk, unsigned long addr,
                        hmm_pte_need_fault(hmm_vma_walk, pfn_req_flags, 0);
                if (required_fault)
                        goto fault;
-               *hmm_pfn = 0;
+               *hmm_pfn = *hmm_pfn & HMM_PFN_DMA_MAPPED;
                return 0;
        }
 
@@ -253,14 +257,14 @@ static int hmm_vma_handle_pte(struct mm_walk *walk, unsigned long addr,
                        cpu_flags = HMM_PFN_VALID;
                        if (is_writable_device_private_entry(entry))
                                cpu_flags |= HMM_PFN_WRITE;
-                       *hmm_pfn = swp_offset_pfn(entry) | cpu_flags;
+                       *hmm_pfn = (*hmm_pfn & HMM_PFN_DMA_MAPPED) | swp_offset_pfn(entry) | cpu_flags;
                        return 0;
                }
 
                required_fault =
                        hmm_pte_need_fault(hmm_vma_walk, pfn_req_flags, 0);
                if (!required_fault) {
-                       *hmm_pfn = 0;
+                       *hmm_pfn = *hmm_pfn & HMM_PFN_DMA_MAPPED;
                        return 0;
                }
 
@@ -304,11 +308,11 @@ static int hmm_vma_handle_pte(struct mm_walk *walk, unsigned long addr,
                        pte_unmap(ptep);
                        return -EFAULT;
                }
-               *hmm_pfn = HMM_PFN_ERROR;
+               *hmm_pfn = (*hmm_pfn & HMM_PFN_DMA_MAPPED) | HMM_PFN_ERROR;
                return 0;
        }
 
-       *hmm_pfn = pte_pfn(pte) | cpu_flags;
+       *hmm_pfn = (*hmm_pfn & HMM_PFN_DMA_MAPPED) | pte_pfn(pte) | cpu_flags;
        return 0;
 
 fault:
@@ -448,8 +452,10 @@ static int hmm_vma_walk_pud(pud_t *pudp, unsigned long start, unsigned long end,
                }
 
                pfn = pud_pfn(pud) + ((addr & ~PUD_MASK) >> PAGE_SHIFT);
-               for (i = 0; i < npages; ++i, ++pfn)
-                       hmm_pfns[i] = pfn | cpu_flags;
+               for (i = 0; i < npages; ++i, ++pfn) {
+                       hmm_pfns[i] &= HMM_PFN_DMA_MAPPED;
+                       hmm_pfns[i] |= pfn | cpu_flags;
+               }
                goto out_unlock;
        }
 
@@ -507,8 +513,10 @@ static int hmm_vma_walk_hugetlb_entry(pte_t *pte, unsigned long hmask,
        }
 
        pfn = pte_pfn(entry) + ((start & ~hmask) >> PAGE_SHIFT);
-       for (; addr < end; addr += PAGE_SIZE, i++, pfn++)
-               range->hmm_pfns[i] = pfn | cpu_flags;
+       for (; addr < end; addr += PAGE_SIZE, i++, pfn++) {
+               range->hmm_pfns[i] &= HMM_PFN_DMA_MAPPED;
+               range->hmm_pfns[i] |= pfn | cpu_flags;
+       }
 
        spin_unlock(ptl);
        return 0;