arm64: lto: Strengthen READ_ONCE() to acquire when CONFIG_LTO=y

When building with LTO, there is an increased risk of the compiler
converting an address dependency headed by a READ_ONCE() invocation
into a control dependency and consequently allowing for harmful
reordering by the CPU.

Ensure that such transformations are harmless by overriding the generic
READ_ONCE() definition with one that provides acquire semantics when
building with LTO.

Acked-by: Peter Zijlstra (Intel) <peterz@infradead.org>
Acked-by: Mark Rutland <mark.rutland@arm.com>
Signed-off-by: Will Deacon <will@kernel.org>

diff --git a/arch/arm64/include/asm/rwonce.h b/arch/arm64/include/asm/rwonce.h
new file mode 100644 (file)
index 0000000..1bce62f
--- /dev/null
+++ b/arch/arm64/include/asm/rwonce.h
@@ -0,0 +1,73 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+/*
+ * Copyright (C) 2020 Google LLC.
+ */
+#ifndef __ASM_RWONCE_H
+#define __ASM_RWONCE_H
+
+#ifdef CONFIG_LTO
+
+#include <linux/compiler_types.h>
+#include <asm/alternative-macros.h>
+
+#ifndef BUILD_VDSO
+
+#ifdef CONFIG_AS_HAS_LDAPR
+#define __LOAD_RCPC(sfx, regs...)                                      \
+       ALTERNATIVE(                                                    \
+               "ldar"  #sfx "\t" #regs,                                \
+               ".arch_extension rcpc\n"                                \
+               "ldapr" #sfx "\t" #regs,                                \
+       ARM64_HAS_LDAPR)
+#else
+#define __LOAD_RCPC(sfx, regs...)      "ldar" #sfx "\t" #regs
+#endif /* CONFIG_AS_HAS_LDAPR */
+
+/*
+ * When building with LTO, there is an increased risk of the compiler
+ * converting an address dependency headed by a READ_ONCE() invocation
+ * into a control dependency and consequently allowing for harmful
+ * reordering by the CPU.
+ *
+ * Ensure that such transformations are harmless by overriding the generic
+ * READ_ONCE() definition with one that provides RCpc acquire semantics
+ * when building with LTO.
+ */
+#define __READ_ONCE(x)                                                 \
+({                                                                     \
+       typeof(&(x)) __x = &(x);                                        \
+       int atomic = 1;                                                 \
+       union { __unqual_scalar_typeof(*__x) __val; char __c[1]; } __u; \
+       switch (sizeof(x)) {                                            \
+       case 1:                                                         \
+               asm volatile(__LOAD_RCPC(b, %w0, %1)                    \
+                       : "=r" (*(__u8 *)__u.__c)                       \
+                       : "Q" (*__x) : "memory");                       \
+               break;                                                  \
+       case 2:                                                         \
+               asm volatile(__LOAD_RCPC(h, %w0, %1)                    \
+                       : "=r" (*(__u16 *)__u.__c)                      \
+                       : "Q" (*__x) : "memory");                       \
+               break;                                                  \
+       case 4:                                                         \
+               asm volatile(__LOAD_RCPC(, %w0, %1)                     \
+                       : "=r" (*(__u32 *)__u.__c)                      \
+                       : "Q" (*__x) : "memory");                       \
+               break;                                                  \
+       case 8:                                                         \
+               asm volatile(__LOAD_RCPC(, %0, %1)                      \
+                       : "=r" (*(__u64 *)__u.__c)                      \
+                       : "Q" (*__x) : "memory");                       \
+               break;                                                  \
+       default:                                                        \
+               atomic = 0;                                             \
+       }                                                               \
+       atomic ? (typeof(*__x))__u.__val : (*(volatile typeof(__x))__x);\
+})
+
+#endif /* !BUILD_VDSO */
+#endif /* CONFIG_LTO */
+
+#include <asm-generic/rwonce.h>
+
+#endif /* __ASM_RWONCE_H */

diff --git a/arch/arm64/kernel/vdso/Makefile b/arch/arm64/kernel/vdso/Makefile
index d65f52264abaeb9edb0566037665a0e970f49ade..a8f8e409e2bfbcdd2a9c0318eb41465570a83882 100644 (file)
--- a/arch/arm64/kernel/vdso/Makefile
+++ b/arch/arm64/kernel/vdso/Makefile
@@ -28,7 +28,7 @@ ldflags-y := -shared -nostdlib -soname=linux-vdso.so.1 --hash-style=sysv      \
             $(btildflags-y) -T
 
 ccflags-y := -fno-common -fno-builtin -fno-stack-protector -ffixed-x18
-ccflags-y += -DDISABLE_BRANCH_PROFILING
+ccflags-y += -DDISABLE_BRANCH_PROFILING -DBUILD_VDSO
 
 CFLAGS_REMOVE_vgettimeofday.o = $(CC_FLAGS_FTRACE) -Os $(CC_FLAGS_SCS) $(GCC_PLUGINS_CFLAGS)
 KASAN_SANITIZE                 := n

diff --git a/arch/arm64/kernel/vdso32/Makefile b/arch/arm64/kernel/vdso32/Makefile
index 79280c53b9a616953eead419df59948c5f7e13a5..a1e0f91e6cea649a6527b367b16618c356905918 100644 (file)
--- a/arch/arm64/kernel/vdso32/Makefile
+++ b/arch/arm64/kernel/vdso32/Makefile
@@ -48,7 +48,7 @@ cc32-as-instr = $(call try-run,\
 # As a result we set our own flags here.
 
 # KBUILD_CPPFLAGS and NOSTDINC_FLAGS from top-level Makefile
-VDSO_CPPFLAGS := -D__KERNEL__ -nostdinc -isystem $(shell $(CC_COMPAT) -print-file-name=include)
+VDSO_CPPFLAGS := -DBUILD_VDSO -D__KERNEL__ -nostdinc -isystem $(shell $(CC_COMPAT) -print-file-name=include)
 VDSO_CPPFLAGS += $(LINUXINCLUDE)
 
 # Common C and assembly flags

diff --git a/arch/arm64/kernel/vmlinux.lds.S b/arch/arm64/kernel/vmlinux.lds.S
index 1bda604f4c704bd22f810c02fcacd60d720c43e4..d6cdcf4aa6a54d11e717d96387de229ea0c93aee 100644 (file)
--- a/arch/arm64/kernel/vmlinux.lds.S
+++ b/arch/arm64/kernel/vmlinux.lds.S
@@ -201,7 +201,7 @@ SECTIONS
                INIT_CALLS
                CON_INITCALL
                INIT_RAM_FS
-               *(.init.rodata.* .init.bss)     /* from the EFI stub */
+               *(.init.altinstructions .init.rodata.* .init.bss)       /* from the EFI stub */
        }
        .exit.data : {
                EXIT_DATA