arm64: Add support for FEAT_HAFT

Armv8.9/v9.4 introduces the feature Hardware managed Access Flag
for Table descriptors (FEAT_HAFT). The feature is indicated by
ID_AA64MMFR1_EL1.HAFDBS == 0b0011 and can be enabled by
TCR2_EL1.HAFT so it has a dependency on FEAT_TCR2.

Adds the Kconfig for FEAT_HAFT and support detecting and enabling
the feature. The feature is enabled in __cpu_setup() before MMU on
just like HA. A CPU capability is added to notify the user of the
feature.

Add definition of P{G,4,U,M}D_TABLE_AF bit and set the AF bit
when creating the page table, which will save the hardware
from having to update them at runtime. This will be ignored if
FEAT_HAFT is not enabled.

The AF bit of table descriptors cannot be managed by the software
per spec, unlike the HA. So this should be used only if it's supported
system wide by system_supports_haft().

Signed-off-by: Yicong Yang <yangyicong@hisilicon.com>
Link: https://lore.kernel.org/r/20241102104235.62560-4-yangyicong@huawei.com
Reviewed-by: Catalin Marinas <catalin.marinas@arm.com>
[catalin.marinas@arm.com: added the ID check back to __cpu_setup in case of future CPU errata]
Signed-off-by: Catalin Marinas <catalin.marinas@arm.com>

diff --git a/arch/arm64/Kconfig b/arch/arm64/Kconfig
index 3e29b44d2d7bd693c6209383a67b8d0d14345406..b90f92f89d540ed8fb7cf6a3112dc99beacdfaf9 100644 (file)
--- a/arch/arm64/Kconfig
+++ b/arch/arm64/Kconfig
@@ -2176,6 +2176,21 @@ config ARCH_PKEY_BITS
        int
        default 3
 
+config ARM64_HAFT
+       bool "Support for Hardware managed Access Flag for Table Descriptors"
+       depends on ARM64_HW_AFDBM
+       default y
+       help
+         The ARMv8.9/ARMv9.5 introduces the feature Hardware managed Access
+         Flag for Table descriptors. When enabled an architectural executed
+         memory access will update the Access Flag in each Table descriptor
+         which is accessed during the translation table walk and for which
+         the Access Flag is 0. The Access Flag of the Table descriptor use
+         the same bit of PTE_AF.
+
+         The feature will only be enabled if all the CPUs in the system
+         support this feature. If unsure, say Y.
+
 endmenu # "ARMv8.9 architectural features"
 
 config ARM64_SVE

diff --git a/arch/arm64/include/asm/cpufeature.h b/arch/arm64/include/asm/cpufeature.h
index 3d261cc123c1e22ac7bc9cfcde463624c76b2084..ed8c784ca082c2795b1223b652431785bd41864d 100644 (file)
--- a/arch/arm64/include/asm/cpufeature.h
+++ b/arch/arm64/include/asm/cpufeature.h
@@ -838,6 +838,12 @@ static inline bool system_supports_poe(void)
                alternative_has_cap_unlikely(ARM64_HAS_S1POE);
 }
 
+static inline bool system_supports_haft(void)
+{
+       return IS_ENABLED(CONFIG_ARM64_HAFT) &&
+               cpus_have_final_cap(ARM64_HAFT);
+}
+
 int do_emulate_mrs(struct pt_regs *regs, u32 sys_reg, u32 rt);
 bool try_emulate_mrs(struct pt_regs *regs, u32 isn);
 

diff --git a/arch/arm64/include/asm/pgalloc.h b/arch/arm64/include/asm/pgalloc.h
index 8ff5f2a2579e4d60a9883793ae3576897fd94261..e75422864d1bd68e7769021721067a05a857dc1c 100644 (file)
--- a/arch/arm64/include/asm/pgalloc.h
+++ b/arch/arm64/include/asm/pgalloc.h
@@ -28,7 +28,7 @@ static inline void __pud_populate(pud_t *pudp, phys_addr_t pmdp, pudval_t prot)
 
 static inline void pud_populate(struct mm_struct *mm, pud_t *pudp, pmd_t *pmdp)
 {
-       pudval_t pudval = PUD_TYPE_TABLE;
+       pudval_t pudval = PUD_TYPE_TABLE | PUD_TABLE_AF;
 
        pudval |= (mm == &init_mm) ? PUD_TABLE_UXN : PUD_TABLE_PXN;
        __pud_populate(pudp, __pa(pmdp), pudval);
@@ -50,7 +50,7 @@ static inline void __p4d_populate(p4d_t *p4dp, phys_addr_t pudp, p4dval_t prot)
 
 static inline void p4d_populate(struct mm_struct *mm, p4d_t *p4dp, pud_t *pudp)
 {
-       p4dval_t p4dval = P4D_TYPE_TABLE;
+       p4dval_t p4dval = P4D_TYPE_TABLE | P4D_TABLE_AF;
 
        p4dval |= (mm == &init_mm) ? P4D_TABLE_UXN : P4D_TABLE_PXN;
        __p4d_populate(p4dp, __pa(pudp), p4dval);
@@ -79,7 +79,7 @@ static inline void __pgd_populate(pgd_t *pgdp, phys_addr_t p4dp, pgdval_t prot)
 
 static inline void pgd_populate(struct mm_struct *mm, pgd_t *pgdp, p4d_t *p4dp)
 {
-       pgdval_t pgdval = PGD_TYPE_TABLE;
+       pgdval_t pgdval = PGD_TYPE_TABLE | PGD_TABLE_AF;
 
        pgdval |= (mm == &init_mm) ? PGD_TABLE_UXN : PGD_TABLE_PXN;
        __pgd_populate(pgdp, __pa(p4dp), pgdval);
@@ -127,14 +127,16 @@ static inline void
 pmd_populate_kernel(struct mm_struct *mm, pmd_t *pmdp, pte_t *ptep)
 {
        VM_BUG_ON(mm && mm != &init_mm);
-       __pmd_populate(pmdp, __pa(ptep), PMD_TYPE_TABLE | PMD_TABLE_UXN);
+       __pmd_populate(pmdp, __pa(ptep),
+                      PMD_TYPE_TABLE | PMD_TABLE_AF | PMD_TABLE_UXN);
 }
 
 static inline void
 pmd_populate(struct mm_struct *mm, pmd_t *pmdp, pgtable_t ptep)
 {
        VM_BUG_ON(mm == &init_mm);
-       __pmd_populate(pmdp, page_to_phys(ptep), PMD_TYPE_TABLE | PMD_TABLE_PXN);
+       __pmd_populate(pmdp, page_to_phys(ptep),
+                      PMD_TYPE_TABLE | PMD_TABLE_AF | PMD_TABLE_PXN);
 }
 
 #endif

diff --git a/arch/arm64/include/asm/pgtable-hwdef.h b/arch/arm64/include/asm/pgtable-hwdef.h
index fd330c1db289a60168b35942fa2667bc3171f39a..c78a988cca93a57914c7c1a5a108221f88aefaee 100644 (file)
--- a/arch/arm64/include/asm/pgtable-hwdef.h
+++ b/arch/arm64/include/asm/pgtable-hwdef.h
@@ -99,6 +99,7 @@
 #define PGD_TYPE_TABLE         (_AT(pgdval_t, 3) << 0)
 #define PGD_TABLE_BIT          (_AT(pgdval_t, 1) << 1)
 #define PGD_TYPE_MASK          (_AT(pgdval_t, 3) << 0)
+#define PGD_TABLE_AF           (_AT(pgdval_t, 1) << 10)        /* Ignored if no FEAT_HAFT */
 #define PGD_TABLE_PXN          (_AT(pgdval_t, 1) << 59)
 #define PGD_TABLE_UXN          (_AT(pgdval_t, 1) << 60)
 
@@ -110,6 +111,7 @@
 #define P4D_TYPE_MASK          (_AT(p4dval_t, 3) << 0)
 #define P4D_TYPE_SECT          (_AT(p4dval_t, 1) << 0)
 #define P4D_SECT_RDONLY                (_AT(p4dval_t, 1) << 7)         /* AP[2] */
+#define P4D_TABLE_AF           (_AT(p4dval_t, 1) << 10)        /* Ignored if no FEAT_HAFT */
 #define P4D_TABLE_PXN          (_AT(p4dval_t, 1) << 59)
 #define P4D_TABLE_UXN          (_AT(p4dval_t, 1) << 60)
 
@@ -121,6 +123,7 @@
 #define PUD_TYPE_MASK          (_AT(pudval_t, 3) << 0)
 #define PUD_TYPE_SECT          (_AT(pudval_t, 1) << 0)
 #define PUD_SECT_RDONLY                (_AT(pudval_t, 1) << 7)         /* AP[2] */
+#define PUD_TABLE_AF           (_AT(pudval_t, 1) << 10)        /* Ignored if no FEAT_HAFT */
 #define PUD_TABLE_PXN          (_AT(pudval_t, 1) << 59)
 #define PUD_TABLE_UXN          (_AT(pudval_t, 1) << 60)
 
@@ -131,6 +134,7 @@
 #define PMD_TYPE_TABLE         (_AT(pmdval_t, 3) << 0)
 #define PMD_TYPE_SECT          (_AT(pmdval_t, 1) << 0)
 #define PMD_TABLE_BIT          (_AT(pmdval_t, 1) << 1)
+#define PMD_TABLE_AF           (_AT(pmdval_t, 1) << 10)        /* Ignored if no FEAT_HAFT */
 
 /*
  * Section

diff --git a/arch/arm64/kernel/cpufeature.c b/arch/arm64/kernel/cpufeature.c
index 718728a85430fad5151b73fa213a510efac3f834..878712fa0d3b139d5b5271d4ce88364660a57850 100644 (file)
--- a/arch/arm64/kernel/cpufeature.c
+++ b/arch/arm64/kernel/cpufeature.c
@@ -2590,6 +2590,21 @@ static const struct arm64_cpu_capabilities arm64_features[] = {
                .cpus = &dbm_cpus,
                ARM64_CPUID_FIELDS(ID_AA64MMFR1_EL1, HAFDBS, DBM)
        },
+#endif
+#ifdef CONFIG_ARM64_HAFT
+       {
+               .desc = "Hardware managed Access Flag for Table Descriptors",
+               /*
+                * Contrary to the page/block access flag, the table access flag
+                * cannot be emulated in software (no access fault will occur).
+                * Therefore this should be used only if it's supported system
+                * wide.
+                */
+               .type = ARM64_CPUCAP_SYSTEM_FEATURE,
+               .capability = ARM64_HAFT,
+               .matches = has_cpuid_feature,
+               ARM64_CPUID_FIELDS(ID_AA64MMFR1_EL1, HAFDBS, HAFT)
+       },
 #endif
        {
                .desc = "CRC32 instructions",

diff --git a/arch/arm64/mm/fixmap.c b/arch/arm64/mm/fixmap.c
index de1e09d986ad230f22eff8def3416c5d94d82bb3..c5c5425791da72a01ef7ae789abd7c901b4f27b0 100644 (file)
--- a/arch/arm64/mm/fixmap.c
+++ b/arch/arm64/mm/fixmap.c
@@ -47,7 +47,8 @@ static void __init early_fixmap_init_pte(pmd_t *pmdp, unsigned long addr)
 
        if (pmd_none(pmd)) {
                ptep = bm_pte[BM_PTE_TABLE_IDX(addr)];
-               __pmd_populate(pmdp, __pa_symbol(ptep), PMD_TYPE_TABLE);
+               __pmd_populate(pmdp, __pa_symbol(ptep),
+                              PMD_TYPE_TABLE | PMD_TABLE_AF);
        }
 }
 
@@ -59,7 +60,8 @@ static void __init early_fixmap_init_pmd(pud_t *pudp, unsigned long addr,
        pmd_t *pmdp;
 
        if (pud_none(pud))
-               __pud_populate(pudp, __pa_symbol(bm_pmd), PUD_TYPE_TABLE);
+               __pud_populate(pudp, __pa_symbol(bm_pmd),
+                              PUD_TYPE_TABLE | PUD_TABLE_AF);
 
        pmdp = pmd_offset_kimg(pudp, addr);
        do {
@@ -86,7 +88,8 @@ static void __init early_fixmap_init_pud(p4d_t *p4dp, unsigned long addr,
        }
 
        if (p4d_none(p4d))
-               __p4d_populate(p4dp, __pa_symbol(bm_pud), P4D_TYPE_TABLE);
+               __p4d_populate(p4dp, __pa_symbol(bm_pud),
+                              P4D_TYPE_TABLE | P4D_TABLE_AF);
 
        pudp = pud_offset_kimg(p4dp, addr);
        early_fixmap_init_pmd(pudp, addr, end);

diff --git a/arch/arm64/mm/mmu.c b/arch/arm64/mm/mmu.c
index e55b02fbddc8f38fcef2def8249c4008285fce2e..6441a45eaeda9fe031c427608c11e5c0b59a7e29 100644 (file)
--- a/arch/arm64/mm/mmu.c
+++ b/arch/arm64/mm/mmu.c
@@ -201,7 +201,7 @@ static void alloc_init_cont_pte(pmd_t *pmdp, unsigned long addr,
 
        BUG_ON(pmd_sect(pmd));
        if (pmd_none(pmd)) {
-               pmdval_t pmdval = PMD_TYPE_TABLE | PMD_TABLE_UXN;
+               pmdval_t pmdval = PMD_TYPE_TABLE | PMD_TABLE_UXN | PMD_TABLE_AF;
                phys_addr_t pte_phys;
 
                if (flags & NO_EXEC_MAPPINGS)
@@ -288,7 +288,7 @@ static void alloc_init_cont_pmd(pud_t *pudp, unsigned long addr,
         */
        BUG_ON(pud_sect(pud));
        if (pud_none(pud)) {
-               pudval_t pudval = PUD_TYPE_TABLE | PUD_TABLE_UXN;
+               pudval_t pudval = PUD_TYPE_TABLE | PUD_TABLE_UXN | PUD_TABLE_AF;
                phys_addr_t pmd_phys;
 
                if (flags & NO_EXEC_MAPPINGS)
@@ -333,7 +333,7 @@ static void alloc_init_pud(p4d_t *p4dp, unsigned long addr, unsigned long end,
        pud_t *pudp;
 
        if (p4d_none(p4d)) {
-               p4dval_t p4dval = P4D_TYPE_TABLE | P4D_TABLE_UXN;
+               p4dval_t p4dval = P4D_TYPE_TABLE | P4D_TABLE_UXN | P4D_TABLE_AF;
                phys_addr_t pud_phys;
 
                if (flags & NO_EXEC_MAPPINGS)
@@ -391,7 +391,7 @@ static void alloc_init_p4d(pgd_t *pgdp, unsigned long addr, unsigned long end,
        p4d_t *p4dp;
 
        if (pgd_none(pgd)) {
-               pgdval_t pgdval = PGD_TYPE_TABLE | PGD_TABLE_UXN;
+               pgdval_t pgdval = PGD_TYPE_TABLE | PGD_TABLE_UXN | PGD_TABLE_AF;
                phys_addr_t p4d_phys;
 
                if (flags & NO_EXEC_MAPPINGS)

diff --git a/arch/arm64/mm/proc.S b/arch/arm64/mm/proc.S
index ccbae4525891e386aab90c030d7d716b80ccd21f..b8edc5765441e60f63ff08b4cf88a224374539b7 100644 (file)
--- a/arch/arm64/mm/proc.S
+++ b/arch/arm64/mm/proc.S
@@ -495,9 +495,14 @@ alternative_else_nop_endif
         * via capabilities.
         */
        mrs     x9, ID_AA64MMFR1_EL1
-       and     x9, x9, ID_AA64MMFR1_EL1_HAFDBS_MASK
+       ubfx    x9, x9, ID_AA64MMFR1_EL1_HAFDBS_SHIFT, #4
        cbz     x9, 1f
        orr     tcr, tcr, #TCR_HA               // hardware Access flag update
+#ifdef CONFIG_ARM64_HAFT
+       cmp     x9, ID_AA64MMFR1_EL1_HAFDBS_HAFT
+       b.lt    1f
+       orr     tcr2, tcr2, TCR2_EL1x_HAFT
+#endif /* CONFIG_ARM64_HAFT */
 1:
 #endif /* CONFIG_ARM64_HW_AFDBM */
        msr     mair_el1, mair

diff --git a/arch/arm64/tools/cpucaps b/arch/arm64/tools/cpucaps
index eedb5acc21ed98bfaaaaa6badcd9a266308ace8e..b35004fa8313e9aa321994664b72e6dbd57a877c 100644 (file)
--- a/arch/arm64/tools/cpucaps
+++ b/arch/arm64/tools/cpucaps
@@ -56,6 +56,7 @@ HAS_TLB_RANGE
 HAS_VA52
 HAS_VIRT_HOST_EXTN
 HAS_WFXT
+HAFT
 HW_DBM
 KVM_HVHE
 KVM_PROTECTED_MODE