x86/cfi,bpf: Fix bpf_callback_t CFI

Where the main BPF program is expected to match bpf_func_t,
sub-programs are expected to match bpf_callback_t.

This fixes things like:

tools/testing/selftests/bpf/progs/bloom_filter_bench.c:

           bpf_for_each_map_elem(&array_map, bloom_callback, &data, 0);

Signed-off-by: Peter Zijlstra (Intel) <peterz@infradead.org>
Link: https://lore.kernel.org/r/20231215092707.451956710@infradead.org
Signed-off-by: Alexei Starovoitov <ast@kernel.org>

diff --git a/arch/x86/include/asm/cfi.h b/arch/x86/include/asm/cfi.h
index 7a7b0b823a98d9876fac9181fe33d45f3af4cf76..8779abd217b7a10c652da1c794a343f95db854b2 100644 (file)
--- a/arch/x86/include/asm/cfi.h
+++ b/arch/x86/include/asm/cfi.h
@@ -106,6 +106,7 @@ struct pt_regs;
 enum bug_trap_type handle_cfi_failure(struct pt_regs *regs);
 #define __bpfcall
 extern u32 cfi_bpf_hash;
+extern u32 cfi_bpf_subprog_hash;
 
 static inline int cfi_get_offset(void)
 {
@@ -128,6 +129,7 @@ static inline enum bug_trap_type handle_cfi_failure(struct pt_regs *regs)
        return BUG_TRAP_TYPE_NONE;
 }
 #define cfi_bpf_hash 0U
+#define cfi_bpf_subprog_hash 0U
 #endif /* CONFIG_CFI_CLANG */
 
 #endif /* _ASM_X86_CFI_H */

diff --git a/arch/x86/kernel/alternative.c b/arch/x86/kernel/alternative.c
index d808d3aaec7e2c770c26c44270c568f3416a13b9..cb393efd5ccd0ded37eb5a01631b722bb4692ad4 100644 (file)
--- a/arch/x86/kernel/alternative.c
+++ b/arch/x86/kernel/alternative.c
@@ -866,6 +866,23 @@ asm (
 "      .size   cfi_bpf_hash, 4                                 \n"
 "      .popsection                                             \n"
 );
+
+/* Must match bpf_callback_t */
+extern u64 __bpf_callback_fn(u64, u64, u64, u64, u64);
+
+__ADDRESSABLE(__bpf_callback_fn);
+
+/* u32 __ro_after_init cfi_bpf_subprog_hash = __kcfi_typeid___bpf_callback_fn; */
+asm (
+"      .pushsection    .data..ro_after_init,\"aw\",@progbits   \n"
+"      .type   cfi_bpf_subprog_hash,@object                    \n"
+"      .globl  cfi_bpf_subprog_hash                            \n"
+"      .p2align        2, 0x0                                  \n"
+"cfi_bpf_subprog_hash:                                         \n"
+"      .long   __kcfi_typeid___bpf_callback_fn                 \n"
+"      .size   cfi_bpf_subprog_hash, 4                         \n"
+"      .popsection                                             \n"
+);
 #endif
 
 #ifdef CONFIG_FINEIBT
@@ -1181,6 +1198,7 @@ static void __apply_fineibt(s32 *start_retpoline, s32 *end_retpoline,
                if (builtin) {
                        cfi_seed = get_random_u32();
                        cfi_bpf_hash = cfi_rehash(cfi_bpf_hash);
+                       cfi_bpf_subprog_hash = cfi_rehash(cfi_bpf_subprog_hash);
                }
 
                ret = cfi_rand_preamble(start_cfi, end_cfi);

diff --git a/arch/x86/net/bpf_jit_comp.c b/arch/x86/net/bpf_jit_comp.c
index 5d5b967b111d3713951270803e21c3118a41f223..43b8c08fdf8d71f20e633ab1e81bee781a5df14a 100644 (file)
--- a/arch/x86/net/bpf_jit_comp.c
+++ b/arch/x86/net/bpf_jit_comp.c
@@ -312,12 +312,13 @@ static void pop_callee_regs(u8 **pprog, bool *callee_regs_used)
  * in arch/x86/kernel/alternative.c
  */
 
-static void emit_fineibt(u8 **pprog)
+static void emit_fineibt(u8 **pprog, bool is_subprog)
 {
+       u32 hash = is_subprog ? cfi_bpf_subprog_hash : cfi_bpf_hash;
        u8 *prog = *pprog;
 
        EMIT_ENDBR();
-       EMIT3_off32(0x41, 0x81, 0xea, cfi_bpf_hash);    /* subl $hash, %r10d    */
+       EMIT3_off32(0x41, 0x81, 0xea, hash);            /* subl $hash, %r10d    */
        EMIT2(0x74, 0x07);                              /* jz.d8 +7             */
        EMIT2(0x0f, 0x0b);                              /* ud2                  */
        EMIT1(0x90);                                    /* nop                  */
@@ -326,11 +327,12 @@ static void emit_fineibt(u8 **pprog)
        *pprog = prog;
 }
 
-static void emit_kcfi(u8 **pprog)
+static void emit_kcfi(u8 **pprog, bool is_subprog)
 {
+       u32 hash = is_subprog ? cfi_bpf_subprog_hash : cfi_bpf_hash;
        u8 *prog = *pprog;
 
-       EMIT1_off32(0xb8, cfi_bpf_hash);                /* movl $hash, %eax     */
+       EMIT1_off32(0xb8, hash);                        /* movl $hash, %eax     */
 #ifdef CONFIG_CALL_PADDING
        EMIT1(0x90);
        EMIT1(0x90);
@@ -349,17 +351,17 @@ static void emit_kcfi(u8 **pprog)
        *pprog = prog;
 }
 
-static void emit_cfi(u8 **pprog)
+static void emit_cfi(u8 **pprog, bool is_subprog)
 {
        u8 *prog = *pprog;
 
        switch (cfi_mode) {
        case CFI_FINEIBT:
-               emit_fineibt(&prog);
+               emit_fineibt(&prog, is_subprog);
                break;
 
        case CFI_KCFI:
-               emit_kcfi(&prog);
+               emit_kcfi(&prog, is_subprog);
                break;
 
        default:
@@ -381,7 +383,7 @@ static void emit_prologue(u8 **pprog, u32 stack_depth, bool ebpf_from_cbpf,
 {
        u8 *prog = *pprog;
 
-       emit_cfi(&prog);
+       emit_cfi(&prog, is_subprog);
        /* BPF trampoline can be made to work without these nops,
         * but let's waste 5 bytes for now and optimize later
         */