arm64: assembler: add conditional cache fixups

It would be helpful if we could use both `dcache_by_line_op` and
`invalidate_icache_by_line` for user memory without accidentally fixing
up unexpected faults when performing maintenance on kernel addresses.

Let's make this possible by having both macros take an optional fixup
label, and only generating an extable entry if a label is provided.

At the same time, let's clean up the labels used to be globally unique
using \@ as we do for other macros.

There should be no functional change as a result of this patch.

Signed-off-by: Mark Rutland <mark.rutland@arm.com>
Signed-off-by: Fuad Tabba <tabba@google.com>
Cc: Ard Biesheuvel <aedb@kernel.org>
Cc: Catalin Marinas <catalin.marinas@arm.com>
Cc: Fuad Tabba <tabba@google.com>
Cc: Will Deacon <will@kernel.org>
Reviewed-by: Ard Biesheuvel <ardb@kernel.org>
Link: https://lore.kernel.org/r/20210524083001.2586635-3-tabba@google.com
Signed-off-by: Will Deacon <will@kernel.org>

diff --git a/arch/arm64/include/asm/assembler.h b/arch/arm64/include/asm/assembler.h
index 6a0fbc599196ee58b792d3ceff5adecfbe3ca336..0a276b46ef501dc7064a9627a4265ba04e63f76a 100644 (file)
--- a/arch/arm64/include/asm/assembler.h
+++ b/arch/arm64/include/asm/assembler.h
@@ -130,15 +130,27 @@ alternative_endif
        .endm
 
 /*
- * Emit an entry into the exception table
+ * Create an exception table entry for `insn`, which will branch to `fixup`
+ * when an unhandled fault is taken.
  */
-       .macro          _asm_extable, from, to
+       .macro          _asm_extable, insn, fixup
        .pushsection    __ex_table, "a"
        .align          3
-       .long           (\from - .), (\to - .)
+       .long           (\insn - .), (\fixup - .)
        .popsection
        .endm
 
+/*
+ * Create an exception table entry for `insn` if `fixup` is provided. Otherwise
+ * do nothing.
+ */
+       .macro          _cond_extable, insn, fixup
+       .ifnc           \fixup,
+       _asm_extable    \insn, \fixup
+       .endif
+       .endm
+
+
 #define USER(l, x...)                          \
 9999:  x;                                      \
        _asm_extable    9999b, l
@@ -383,6 +395,7 @@ alternative_cb_end
  *     domain:         domain used in dsb instruciton
  *     addr:           starting virtual address of the region
  *     size:           size of the region
+ *     fixup:          optional label to branch to on user fault
  *     Corrupts:       addr, size, tmp1, tmp2
  */
        .macro __dcache_op_workaround_clean_cache, op, addr
@@ -393,12 +406,12 @@ alternative_else
 alternative_endif
        .endm
 
-       .macro dcache_by_line_op op, domain, addr, size, tmp1, tmp2
+       .macro dcache_by_line_op op, domain, addr, size, tmp1, tmp2, fixup
        dcache_line_size \tmp1, \tmp2
        add     \size, \addr, \size
        sub     \tmp2, \tmp1, #1
        bic     \addr, \addr, \tmp2
-9998:
+.Ldcache_op\@:
        .ifc    \op, cvau
        __dcache_op_workaround_clean_cache \op, \addr
        .else
@@ -418,8 +431,10 @@ alternative_endif
        .endif
        add     \addr, \addr, \tmp1
        cmp     \addr, \size
-       b.lo    9998b
+       b.lo    .Ldcache_op\@
        dsb     \domain
+
+       _cond_extable .Ldcache_op\@, \fixup
        .endm
 
 /*
@@ -427,20 +442,22 @@ alternative_endif
  * [start, end)
  *
  *     start, end:     virtual addresses describing the region
- *     label:          A label to branch to on user fault.
+ *     fixup:          optional label to branch to on user fault
  *     Corrupts:       tmp1, tmp2
  */
-       .macro invalidate_icache_by_line start, end, tmp1, tmp2, label
+       .macro invalidate_icache_by_line start, end, tmp1, tmp2, fixup
        icache_line_size \tmp1, \tmp2
        sub     \tmp2, \tmp1, #1
        bic     \tmp2, \start, \tmp2
-9997:
-USER(\label, ic        ivau, \tmp2)                    // invalidate I line PoU
+.Licache_op\@:
+       ic      ivau, \tmp2                     // invalidate I line PoU
        add     \tmp2, \tmp2, \tmp1
        cmp     \tmp2, \end
-       b.lo    9997b
+       b.lo    .Licache_op\@
        dsb     ish
        isb
+
+       _cond_extable .Licache_op\@, \fixup
        .endm
 
 /*