arm64: Avoid aligning normal memory pointers in __memcpy_{to,from}io

__memcpy_{to,from}io fall back to byte-at-a-time copying if both the
source and destination pointers are not 8-byte aligned. Since one of the
pointers always points at normal memory, this is unnecessary and
detrimental to performance, so only do byte copying until we hit an 8-byte
boundary for the device pointer.

This change was motivated by performance issues in the pstore driver.
On a test platform, measuring probe time for pstore, console buffer
size of 1/4MB and pmsg of 1/2MB, was in the 90-107ms region. Change
managed to reduce it to 10-25ms, an improvement in boot time.

Cc: Kees Cook <keescook@chromium.org>
Cc: Anton Vorontsov <anton@enomsg.org>
Cc: Tony Luck <tony.luck@intel.com>
Cc: Catalin Marinas <catalin.marinas@arm.com>
Cc: Will Deacon <will.deacon@arm.com>
Cc: Anton Vorontsov <anton@enomsg.org>
Cc: Robin Murphy <robin.murphy@arm.com>
Signed-off-by: Mark Salyzyn <salyzyn@android.com>
Signed-off-by: Will Deacon <will.deacon@arm.com>

diff --git a/arch/arm64/kernel/io.c b/arch/arm64/kernel/io.c
index 354be2a872ae81bd58db3506cd0c7f77a62cd402..79b17384efface023d0331594df8554de67efaf8 100644 (file)
--- a/arch/arm64/kernel/io.c
+++ b/arch/arm64/kernel/io.c
@@ -25,8 +25,7 @@
  */
 void __memcpy_fromio(void *to, const volatile void __iomem *from, size_t count)
 {
-       while (count && (!IS_ALIGNED((unsigned long)from, 8) ||
-                        !IS_ALIGNED((unsigned long)to, 8))) {
+       while (count && !IS_ALIGNED((unsigned long)from, 8)) {
                *(u8 *)to = __raw_readb(from);
                from++;
                to++;
@@ -54,23 +53,22 @@ EXPORT_SYMBOL(__memcpy_fromio);
  */
 void __memcpy_toio(volatile void __iomem *to, const void *from, size_t count)
 {
-       while (count && (!IS_ALIGNED((unsigned long)to, 8) ||
-                        !IS_ALIGNED((unsigned long)from, 8))) {
-               __raw_writeb(*(volatile u8 *)from, to);
+       while (count && !IS_ALIGNED((unsigned long)to, 8)) {
+               __raw_writeb(*(u8 *)from, to);
                from++;
                to++;
                count--;
        }
 
        while (count >= 8) {
-               __raw_writeq(*(volatile u64 *)from, to);
+               __raw_writeq(*(u64 *)from, to);
                from += 8;
                to += 8;
                count -= 8;
        }
 
        while (count) {
-               __raw_writeb(*(volatile u8 *)from, to);
+               __raw_writeb(*(u8 *)from, to);
                from++;
                to++;
                count--;