x86/mm: Return correct level from lookup_address() if pte is none

Currently, lookup_address() returns two things:

  1. A "pte_t" (which might be a p[g4um]d_t)
  2. The 'level' of the page tables where the "pte_t" was found
     (returned via a pointer)

If no pte_t is found, 'level' is essentially garbage.

Always fill out the level.  For NULL "pte_t"s, fill in the level where
the p*d_none() entry was found mirroring the "found" behavior.

Always filling out the level allows using lookup_address() to precisely skip
over holes when walking kernel page tables.

Add one more entry into enum pg_level to indicate the size of the VA
covered by one PGD entry in 5-level paging mode.

Update comments for lookup_address() and lookup_address_in_pgd() to
reflect changes in the interface.

Signed-off-by: Kirill A. Shutemov <kirill.shutemov@linux.intel.com>
Signed-off-by: Borislav Petkov (AMD) <bp@alien8.de>
Reviewed-by: Rick Edgecombe <rick.p.edgecombe@intel.com>
Reviewed-by: Baoquan He <bhe@redhat.com>
Reviewed-by: Dave Hansen <dave.hansen@intel.com>
Tested-by: Tao Liu <ltao@redhat.com>
Link: https://lore.kernel.org/r/20240614095904.1345461-9-kirill.shutemov@linux.intel.com

diff --git a/arch/x86/include/asm/pgtable_types.h b/arch/x86/include/asm/pgtable_types.h
index b786449626267e0dd84be5fd4297dc2cd81f2b6d..2f321137736c17ab07a3a04d49d290d20823b467 100644 (file)
--- a/arch/x86/include/asm/pgtable_types.h
+++ b/arch/x86/include/asm/pgtable_types.h
@@ -549,6 +549,7 @@ enum pg_level {
        PG_LEVEL_2M,
        PG_LEVEL_1G,
        PG_LEVEL_512G,
+       PG_LEVEL_256T,
        PG_LEVEL_NUM
 };
 

diff --git a/arch/x86/mm/pat/set_memory.c b/arch/x86/mm/pat/set_memory.c
index 498812f067cd597e3c75871be1e4f148c70daa43..a7a7a6c6a3fba886e6bad6141397b7d05435c8bb 100644 (file)
--- a/arch/x86/mm/pat/set_memory.c
+++ b/arch/x86/mm/pat/set_memory.c
@@ -662,8 +662,9 @@ static inline pgprot_t verify_rwx(pgprot_t old, pgprot_t new, unsigned long star
 
 /*
  * Lookup the page table entry for a virtual address in a specific pgd.
- * Return a pointer to the entry, the level of the mapping, and the effective
- * NX and RW bits of all page table levels.
+ * Return a pointer to the entry (or NULL if the entry does not exist),
+ * the level of the entry, and the effective NX and RW bits of all
+ * page table levels.
  */
 pte_t *lookup_address_in_pgd_attr(pgd_t *pgd, unsigned long address,
                                  unsigned int *level, bool *nx, bool *rw)
@@ -672,13 +673,14 @@ pte_t *lookup_address_in_pgd_attr(pgd_t *pgd, unsigned long address,
        pud_t *pud;
        pmd_t *pmd;
 
-       *level = PG_LEVEL_NONE;
+       *level = PG_LEVEL_256T;
        *nx = false;
        *rw = true;
 
        if (pgd_none(*pgd))
                return NULL;
 
+       *level = PG_LEVEL_512G;
        *nx |= pgd_flags(*pgd) & _PAGE_NX;
        *rw &= pgd_flags(*pgd) & _PAGE_RW;
 
@@ -686,10 +688,10 @@ pte_t *lookup_address_in_pgd_attr(pgd_t *pgd, unsigned long address,
        if (p4d_none(*p4d))
                return NULL;
 
-       *level = PG_LEVEL_512G;
        if (p4d_leaf(*p4d) || !p4d_present(*p4d))
                return (pte_t *)p4d;
 
+       *level = PG_LEVEL_1G;
        *nx |= p4d_flags(*p4d) & _PAGE_NX;
        *rw &= p4d_flags(*p4d) & _PAGE_RW;
 
@@ -697,10 +699,10 @@ pte_t *lookup_address_in_pgd_attr(pgd_t *pgd, unsigned long address,
        if (pud_none(*pud))
                return NULL;
 
-       *level = PG_LEVEL_1G;
        if (pud_leaf(*pud) || !pud_present(*pud))
                return (pte_t *)pud;
 
+       *level = PG_LEVEL_2M;
        *nx |= pud_flags(*pud) & _PAGE_NX;
        *rw &= pud_flags(*pud) & _PAGE_RW;
 
@@ -708,15 +710,13 @@ pte_t *lookup_address_in_pgd_attr(pgd_t *pgd, unsigned long address,
        if (pmd_none(*pmd))
                return NULL;
 
-       *level = PG_LEVEL_2M;
        if (pmd_leaf(*pmd) || !pmd_present(*pmd))
                return (pte_t *)pmd;
 
+       *level = PG_LEVEL_4K;
        *nx |= pmd_flags(*pmd) & _PAGE_NX;
        *rw &= pmd_flags(*pmd) & _PAGE_RW;
 
-       *level = PG_LEVEL_4K;
-
        return pte_offset_kernel(pmd, address);
 }
 
@@ -736,9 +736,8 @@ pte_t *lookup_address_in_pgd(pgd_t *pgd, unsigned long address,
  * Lookup the page table entry for a virtual address. Return a pointer
  * to the entry and the level of the mapping.
  *
- * Note: We return pud and pmd either when the entry is marked large
- * or when the present bit is not set. Otherwise we would return a
- * pointer to a nonexisting mapping.
+ * Note: the function returns p4d, pud or pmd either when the entry is marked
+ * large or when the present bit is not set. Otherwise it returns NULL.
  */
 pte_t *lookup_address(unsigned long address, unsigned int *level)
 {