arm64/bpf: use movn/movk/movk sequence to generate kernel addresses

On arm64, all executable code is guaranteed to reside in the vmalloc
space (or the module space), and so jump targets will only use 48
bits at most, and the remaining bits are guaranteed to be 0x1.

This means we can generate an immediate jump address using a sequence
of one MOVN (move wide negated) and two MOVK instructions, where the
first one sets the lower 16 bits but also sets all top bits to 0x1.

Signed-off-by: Ard Biesheuvel <ard.biesheuvel@linaro.org>
Acked-by: Will Deacon <will.deacon@arm.com>
Acked-by: Daniel Borkmann <daniel@iogearbox.net>
Signed-off-by: Daniel Borkmann <daniel@iogearbox.net>

diff --git a/arch/arm64/net/bpf_jit_comp.c b/arch/arm64/net/bpf_jit_comp.c
index 89198017e8e681268504235331471b38b7e945b9..b872859240234166b8634703ca83996bb4603e39 100644 (file)
--- a/arch/arm64/net/bpf_jit_comp.c
+++ b/arch/arm64/net/bpf_jit_comp.c
@@ -134,10 +134,9 @@ static inline void emit_a64_mov_i64(const int reg, const u64 val,
 }
 
 /*
- * This is an unoptimized 64 immediate emission used for BPF to BPF call
- * addresses. It will always do a full 64 bit decomposition as otherwise
- * more complexity in the last extra pass is required since we previously
- * reserved 4 instructions for the address.
+ * Kernel addresses in the vmalloc space use at most 48 bits, and the
+ * remaining bits are guaranteed to be 0x1. So we can compose the address
+ * with a fixed length movn/movk/movk sequence.
  */
 static inline void emit_addr_mov_i64(const int reg, const u64 val,
                                     struct jit_ctx *ctx)
@@ -145,8 +144,8 @@ static inline void emit_addr_mov_i64(const int reg, const u64 val,
        u64 tmp = val;
        int shift = 0;
 
-       emit(A64_MOVZ(1, reg, tmp & 0xffff, shift), ctx);
-       for (;shift < 48;) {
+       emit(A64_MOVN(1, reg, ~tmp & 0xffff, shift), ctx);
+       while (shift < 32) {
                tmp >>= 16;
                shift += 16;
                emit(A64_MOVK(1, reg, tmp & 0xffff, shift), ctx);
@@ -634,11 +633,7 @@ emit_cond_jmp:
                                            &func_addr, &func_addr_fixed);
                if (ret < 0)
                        return ret;
-               if (func_addr_fixed)
-                       /* We can use optimized emission here. */
-                       emit_a64_mov_i64(tmp, func_addr, ctx);
-               else
-                       emit_addr_mov_i64(tmp, func_addr, ctx);
+               emit_addr_mov_i64(tmp, func_addr, ctx);
                emit(A64_BLR(tmp), ctx);
                emit(A64_MOV(1, r0, A64_R(0)), ctx);
                break;