From: Linus Torvalds <torvalds@linux-foundation.org>
Date: Mon, 29 Sep 2025 22:55:20 +0000 (-0700)
Subject: Merge tag 'fsverity-for-linus' of git://git.kernel.org/pub/scm/fs/fsverity/linux
X-Git-Url: https://www.infradead.org/git/?a=commitdiff_plain;h=1896ce8eb6c61824f6c1125d69d8fda1f44a22f8;p=users%2Fhch%2Fmisc.git

Merge tag 'fsverity-for-linus' of git://git.kernel.org/pub/scm/fs/fsverity/linux

Pull interleaved SHA-256 hashing support from Eric Biggers:
 "Optimize fsverity with 2-way interleaved hashing

  Add support for 2-way interleaved SHA-256 hashing to lib/crypto/, and
  make fsverity use it for faster file data verification. This improves
  fsverity performance on many x86_64 and arm64 processors.

  Later, I plan to make dm-verity use this too"

* tag 'fsverity-for-linus' of git://git.kernel.org/pub/scm/fs/fsverity/linux:
  fsverity: Use 2-way interleaved SHA-256 hashing when supported
  fsverity: Remove inode parameter from fsverity_hash_block()
  lib/crypto: tests: Add tests and benchmark for sha256_finup_2x()
  lib/crypto: x86/sha256: Add support for 2-way interleaved hashing
  lib/crypto: arm64/sha256: Add support for 2-way interleaved hashing
  lib/crypto: sha256: Add support for 2-way interleaved hashing
---

1896ce8eb6c61824f6c1125d69d8fda1f44a22f8
diff --cc lib/crypto/arm64/sha256.h
index be4aeda9d0e6,a7bc3a90ada6..80d06df27d3a
--- a/lib/crypto/arm64/sha256.h
+++ b/lib/crypto/arm64/sha256.h
@@@ -44,9 -44,46 +44,46 @@@ static void sha256_blocks(struct sha256
  	}
  }
  
+ static_assert(offsetof(struct __sha256_ctx, state) == 0);
+ static_assert(offsetof(struct __sha256_ctx, bytecount) == 32);
+ static_assert(offsetof(struct __sha256_ctx, buf) == 40);
+ asmlinkage void sha256_ce_finup2x(const struct __sha256_ctx *ctx,
+ 				  const u8 *data1, const u8 *data2, int len,
+ 				  u8 out1[SHA256_DIGEST_SIZE],
+ 				  u8 out2[SHA256_DIGEST_SIZE]);
+ 
+ #define sha256_finup_2x_arch sha256_finup_2x_arch
+ static bool sha256_finup_2x_arch(const struct __sha256_ctx *ctx,
+ 				 const u8 *data1, const u8 *data2, size_t len,
+ 				 u8 out1[SHA256_DIGEST_SIZE],
+ 				 u8 out2[SHA256_DIGEST_SIZE])
+ {
+ 	/*
+ 	 * The assembly requires len >= SHA256_BLOCK_SIZE && len <= INT_MAX.
+ 	 * Further limit len to 65536 to avoid spending too long with preemption
+ 	 * disabled.  (Of course, in practice len is nearly always 4096 anyway.)
+ 	 */
+ 	if (IS_ENABLED(CONFIG_KERNEL_MODE_NEON) &&
+ 	    static_branch_likely(&have_ce) && len >= SHA256_BLOCK_SIZE &&
+ 	    len <= 65536 && likely(may_use_simd())) {
+ 		kernel_neon_begin();
+ 		sha256_ce_finup2x(ctx, data1, data2, len, out1, out2);
+ 		kernel_neon_end();
+ 		kmsan_unpoison_memory(out1, SHA256_DIGEST_SIZE);
+ 		kmsan_unpoison_memory(out2, SHA256_DIGEST_SIZE);
+ 		return true;
+ 	}
+ 	return false;
+ }
+ 
+ static bool sha256_finup_2x_is_optimized_arch(void)
+ {
+ 	return static_key_enabled(&have_ce);
+ }
+ 
  #ifdef CONFIG_KERNEL_MODE_NEON
  #define sha256_mod_init_arch sha256_mod_init_arch
 -static inline void sha256_mod_init_arch(void)
 +static void sha256_mod_init_arch(void)
  {
  	if (cpu_have_named_feature(ASIMD)) {
  		static_branch_enable(&have_neon);
diff --cc lib/crypto/x86/sha256.h
index 41fa95fbc3bf,ecea65b608db..38e33b22a092
--- a/lib/crypto/x86/sha256.h
+++ b/lib/crypto/x86/sha256.h
@@@ -5,8 -5,11 +5,10 @@@
   * Copyright 2025 Google LLC
   */
  #include <asm/fpu/api.h>
 -#include <crypto/internal/simd.h>
  #include <linux/static_call.h>
  
+ static __ro_after_init DEFINE_STATIC_KEY_FALSE(have_sha_ni);
+ 
  DEFINE_STATIC_CALL(sha256_blocks_x86, sha256_blocks_generic);
  
  #define DEFINE_X86_SHA256_FN(c_fn, asm_fn)                                 \
@@@ -35,8 -38,44 +37,44 @@@ static void sha256_blocks(struct sha256
  	static_call(sha256_blocks_x86)(state, data, nblocks);
  }
  
+ static_assert(offsetof(struct __sha256_ctx, state) == 0);
+ static_assert(offsetof(struct __sha256_ctx, bytecount) == 32);
+ static_assert(offsetof(struct __sha256_ctx, buf) == 40);
+ asmlinkage void sha256_ni_finup2x(const struct __sha256_ctx *ctx,
+ 				  const u8 *data1, const u8 *data2, int len,
+ 				  u8 out1[SHA256_DIGEST_SIZE],
+ 				  u8 out2[SHA256_DIGEST_SIZE]);
+ 
+ #define sha256_finup_2x_arch sha256_finup_2x_arch
+ static bool sha256_finup_2x_arch(const struct __sha256_ctx *ctx,
+ 				 const u8 *data1, const u8 *data2, size_t len,
+ 				 u8 out1[SHA256_DIGEST_SIZE],
+ 				 u8 out2[SHA256_DIGEST_SIZE])
+ {
+ 	/*
+ 	 * The assembly requires len >= SHA256_BLOCK_SIZE && len <= INT_MAX.
+ 	 * Further limit len to 65536 to avoid spending too long with preemption
+ 	 * disabled.  (Of course, in practice len is nearly always 4096 anyway.)
+ 	 */
+ 	if (static_branch_likely(&have_sha_ni) && len >= SHA256_BLOCK_SIZE &&
+ 	    len <= 65536 && likely(irq_fpu_usable())) {
+ 		kernel_fpu_begin();
+ 		sha256_ni_finup2x(ctx, data1, data2, len, out1, out2);
+ 		kernel_fpu_end();
+ 		kmsan_unpoison_memory(out1, SHA256_DIGEST_SIZE);
+ 		kmsan_unpoison_memory(out2, SHA256_DIGEST_SIZE);
+ 		return true;
+ 	}
+ 	return false;
+ }
+ 
+ static bool sha256_finup_2x_is_optimized_arch(void)
+ {
+ 	return static_key_enabled(&have_sha_ni);
+ }
+ 
  #define sha256_mod_init_arch sha256_mod_init_arch
 -static inline void sha256_mod_init_arch(void)
 +static void sha256_mod_init_arch(void)
  {
  	if (boot_cpu_has(X86_FEATURE_SHA_NI)) {
  		static_call_update(sha256_blocks_x86, sha256_blocks_ni);