From 699618d422759fe096a28ad2778d79807a5384ce Mon Sep 17 00:00:00 2001 From: Eric Biggers Date: Mon, 28 Apr 2025 10:00:35 -0700 Subject: [PATCH 01/16] crypto: sparc/sha256 - implement library instead of shash Instead of providing crypto_shash algorithms for the arch-optimized SHA-256 code, instead implement the SHA-256 library. This is much simpler, it makes the SHA-256 library functions be arch-optimized, and it fixes the longstanding issue where the arch-optimized SHA-256 was disabled by default. SHA-256 still remains available through crypto_shash, but individual architectures no longer need to handle it. Signed-off-by: Eric Biggers Signed-off-by: Herbert Xu --- arch/sparc/crypto/Kconfig | 10 -- arch/sparc/crypto/Makefile | 2 - arch/sparc/crypto/sha256_glue.c | 128 ----------------------- arch/sparc/lib/Makefile | 1 + arch/sparc/lib/crypto/Kconfig | 8 ++ arch/sparc/lib/crypto/Makefile | 4 + arch/sparc/lib/crypto/sha256.c | 64 ++++++++++++ arch/sparc/{ => lib}/crypto/sha256_asm.S | 2 +- lib/crypto/Kconfig | 3 + 9 files changed, 81 insertions(+), 141 deletions(-) delete mode 100644 arch/sparc/crypto/sha256_glue.c create mode 100644 arch/sparc/lib/crypto/Kconfig create mode 100644 arch/sparc/lib/crypto/Makefile create mode 100644 arch/sparc/lib/crypto/sha256.c rename arch/sparc/{ => lib}/crypto/sha256_asm.S (96%) diff --git a/arch/sparc/crypto/Kconfig b/arch/sparc/crypto/Kconfig index e858597de89d..a6ba319c42dc 100644 --- a/arch/sparc/crypto/Kconfig +++ b/arch/sparc/crypto/Kconfig @@ -36,16 +36,6 @@ config CRYPTO_SHA1_SPARC64 Architecture: sparc64 -config CRYPTO_SHA256_SPARC64 - tristate "Hash functions: SHA-224 and SHA-256" - depends on SPARC64 - select CRYPTO_SHA256 - select CRYPTO_HASH - help - SHA-224 and SHA-256 secure hash algorithms (FIPS 180) - - Architecture: sparc64 using crypto instructions, when available - config CRYPTO_SHA512_SPARC64 tristate "Hash functions: SHA-384 and SHA-512" depends on SPARC64 diff --git a/arch/sparc/crypto/Makefile b/arch/sparc/crypto/Makefile index a2d7fca40cb4..701c39edb0d7 100644 --- a/arch/sparc/crypto/Makefile +++ b/arch/sparc/crypto/Makefile @@ -4,7 +4,6 @@ # obj-$(CONFIG_CRYPTO_SHA1_SPARC64) += sha1-sparc64.o -obj-$(CONFIG_CRYPTO_SHA256_SPARC64) += sha256-sparc64.o obj-$(CONFIG_CRYPTO_SHA512_SPARC64) += sha512-sparc64.o obj-$(CONFIG_CRYPTO_MD5_SPARC64) += md5-sparc64.o @@ -13,7 +12,6 @@ obj-$(CONFIG_CRYPTO_DES_SPARC64) += des-sparc64.o obj-$(CONFIG_CRYPTO_CAMELLIA_SPARC64) += camellia-sparc64.o sha1-sparc64-y := sha1_asm.o sha1_glue.o -sha256-sparc64-y := sha256_asm.o sha256_glue.o sha512-sparc64-y := sha512_asm.o sha512_glue.o md5-sparc64-y := md5_asm.o md5_glue.o diff --git a/arch/sparc/crypto/sha256_glue.c b/arch/sparc/crypto/sha256_glue.c deleted file mode 100644 index 25008603a986..000000000000 --- a/arch/sparc/crypto/sha256_glue.c +++ /dev/null @@ -1,128 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-only -/* Glue code for SHA256 hashing optimized for sparc64 crypto opcodes. - * - * This is based largely upon crypto/sha256_generic.c - * - * Copyright (c) Jean-Luc Cooke - * Copyright (c) Andrew McDonald - * Copyright (c) 2002 James Morris - * SHA224 Support Copyright 2007 Intel Corporation - */ - -#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt - -#include -#include -#include -#include -#include -#include -#include -#include - -asmlinkage void sha256_sparc64_transform(u32 *digest, const char *data, - unsigned int rounds); - -static void sha256_block(struct crypto_sha256_state *sctx, const u8 *src, - int blocks) -{ - sha256_sparc64_transform(sctx->state, src, blocks); -} - -static int sha256_sparc64_update(struct shash_desc *desc, const u8 *data, - unsigned int len) -{ - return sha256_base_do_update_blocks(desc, data, len, sha256_block); -} - -static int sha256_sparc64_finup(struct shash_desc *desc, const u8 *src, - unsigned int len, u8 *out) -{ - sha256_base_do_finup(desc, src, len, sha256_block); - return sha256_base_finish(desc, out); -} - -static struct shash_alg sha256_alg = { - .digestsize = SHA256_DIGEST_SIZE, - .init = sha256_base_init, - .update = sha256_sparc64_update, - .finup = sha256_sparc64_finup, - .descsize = sizeof(struct crypto_sha256_state), - .base = { - .cra_name = "sha256", - .cra_driver_name= "sha256-sparc64", - .cra_priority = SPARC_CR_OPCODE_PRIORITY, - .cra_flags = CRYPTO_AHASH_ALG_BLOCK_ONLY | - CRYPTO_AHASH_ALG_FINUP_MAX, - .cra_blocksize = SHA256_BLOCK_SIZE, - .cra_module = THIS_MODULE, - } -}; - -static struct shash_alg sha224_alg = { - .digestsize = SHA224_DIGEST_SIZE, - .init = sha224_base_init, - .update = sha256_sparc64_update, - .finup = sha256_sparc64_finup, - .descsize = sizeof(struct crypto_sha256_state), - .base = { - .cra_name = "sha224", - .cra_driver_name= "sha224-sparc64", - .cra_priority = SPARC_CR_OPCODE_PRIORITY, - .cra_flags = CRYPTO_AHASH_ALG_BLOCK_ONLY | - CRYPTO_AHASH_ALG_FINUP_MAX, - .cra_blocksize = SHA224_BLOCK_SIZE, - .cra_module = THIS_MODULE, - } -}; - -static bool __init sparc64_has_sha256_opcode(void) -{ - unsigned long cfr; - - if (!(sparc64_elf_hwcap & HWCAP_SPARC_CRYPTO)) - return false; - - __asm__ __volatile__("rd %%asr26, %0" : "=r" (cfr)); - if (!(cfr & CFR_SHA256)) - return false; - - return true; -} - -static int __init sha256_sparc64_mod_init(void) -{ - if (sparc64_has_sha256_opcode()) { - int ret = crypto_register_shash(&sha224_alg); - if (ret < 0) - return ret; - - ret = crypto_register_shash(&sha256_alg); - if (ret < 0) { - crypto_unregister_shash(&sha224_alg); - return ret; - } - - pr_info("Using sparc64 sha256 opcode optimized SHA-256/SHA-224 implementation\n"); - return 0; - } - pr_info("sparc64 sha256 opcode not available.\n"); - return -ENODEV; -} - -static void __exit sha256_sparc64_mod_fini(void) -{ - crypto_unregister_shash(&sha224_alg); - crypto_unregister_shash(&sha256_alg); -} - -module_init(sha256_sparc64_mod_init); -module_exit(sha256_sparc64_mod_fini); - -MODULE_LICENSE("GPL"); -MODULE_DESCRIPTION("SHA-224 and SHA-256 Secure Hash Algorithm, sparc64 sha256 opcode accelerated"); - -MODULE_ALIAS_CRYPTO("sha224"); -MODULE_ALIAS_CRYPTO("sha256"); - -#include "crop_devid.c" diff --git a/arch/sparc/lib/Makefile b/arch/sparc/lib/Makefile index 5724d0f356eb..98887dc295a1 100644 --- a/arch/sparc/lib/Makefile +++ b/arch/sparc/lib/Makefile @@ -4,6 +4,7 @@ asflags-y := -ansi -DST_DIV0=0x02 +obj-y += crypto/ lib-$(CONFIG_SPARC32) += ashrdi3.o lib-$(CONFIG_SPARC32) += memcpy.o memset.o lib-y += strlen.o diff --git a/arch/sparc/lib/crypto/Kconfig b/arch/sparc/lib/crypto/Kconfig new file mode 100644 index 000000000000..e5c3e4d3dba6 --- /dev/null +++ b/arch/sparc/lib/crypto/Kconfig @@ -0,0 +1,8 @@ +# SPDX-License-Identifier: GPL-2.0-only + +config CRYPTO_SHA256_SPARC64 + tristate + depends on SPARC64 + default CRYPTO_LIB_SHA256 + select CRYPTO_ARCH_HAVE_LIB_SHA256 + select CRYPTO_LIB_SHA256_GENERIC diff --git a/arch/sparc/lib/crypto/Makefile b/arch/sparc/lib/crypto/Makefile new file mode 100644 index 000000000000..75ee244ad6f7 --- /dev/null +++ b/arch/sparc/lib/crypto/Makefile @@ -0,0 +1,4 @@ +# SPDX-License-Identifier: GPL-2.0-only + +obj-$(CONFIG_CRYPTO_SHA256_SPARC64) += sha256-sparc64.o +sha256-sparc64-y := sha256.o sha256_asm.o diff --git a/arch/sparc/lib/crypto/sha256.c b/arch/sparc/lib/crypto/sha256.c new file mode 100644 index 000000000000..6f118a23d210 --- /dev/null +++ b/arch/sparc/lib/crypto/sha256.c @@ -0,0 +1,64 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * SHA-256 accelerated using the sparc64 sha256 opcodes + * + * Copyright (c) Jean-Luc Cooke + * Copyright (c) Andrew McDonald + * Copyright (c) 2002 James Morris + * SHA224 Support Copyright 2007 Intel Corporation + */ + +#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt + +#include +#include +#include +#include +#include +#include + +static __ro_after_init DEFINE_STATIC_KEY_FALSE(have_sha256_opcodes); + +asmlinkage void sha256_sparc64_transform(u32 state[SHA256_STATE_WORDS], + const u8 *data, size_t nblocks); + +void sha256_blocks_arch(u32 state[SHA256_STATE_WORDS], + const u8 *data, size_t nblocks) +{ + if (static_branch_likely(&have_sha256_opcodes)) + sha256_sparc64_transform(state, data, nblocks); + else + sha256_blocks_generic(state, data, nblocks); +} +EXPORT_SYMBOL(sha256_blocks_arch); + +bool sha256_is_arch_optimized(void) +{ + return static_key_enabled(&have_sha256_opcodes); +} +EXPORT_SYMBOL(sha256_is_arch_optimized); + +static int __init sha256_sparc64_mod_init(void) +{ + unsigned long cfr; + + if (!(sparc64_elf_hwcap & HWCAP_SPARC_CRYPTO)) + return 0; + + __asm__ __volatile__("rd %%asr26, %0" : "=r" (cfr)); + if (!(cfr & CFR_SHA256)) + return 0; + + static_branch_enable(&have_sha256_opcodes); + pr_info("Using sparc64 sha256 opcode optimized SHA-256/SHA-224 implementation\n"); + return 0; +} +arch_initcall(sha256_sparc64_mod_init); + +static void __exit sha256_sparc64_mod_exit(void) +{ +} +module_exit(sha256_sparc64_mod_exit); + +MODULE_LICENSE("GPL"); +MODULE_DESCRIPTION("SHA-256 accelerated using the sparc64 sha256 opcodes"); diff --git a/arch/sparc/crypto/sha256_asm.S b/arch/sparc/lib/crypto/sha256_asm.S similarity index 96% rename from arch/sparc/crypto/sha256_asm.S rename to arch/sparc/lib/crypto/sha256_asm.S index 8ce88611e98a..ddcdd3daf31e 100644 --- a/arch/sparc/crypto/sha256_asm.S +++ b/arch/sparc/lib/crypto/sha256_asm.S @@ -4,7 +4,7 @@ #include ENTRY(sha256_sparc64_transform) - /* %o0 = digest, %o1 = data, %o2 = rounds */ + /* %o0 = state, %o1 = data, %o2 = nblocks */ VISEntryHalf ld [%o0 + 0x00], %f0 ld [%o0 + 0x04], %f1 diff --git a/lib/crypto/Kconfig b/lib/crypto/Kconfig index 7fe678047939..6319358b38c2 100644 --- a/lib/crypto/Kconfig +++ b/lib/crypto/Kconfig @@ -181,6 +181,9 @@ endif if S390 source "arch/s390/lib/crypto/Kconfig" endif +if SPARC +source "arch/sparc/lib/crypto/Kconfig" +endif if X86 source "arch/x86/lib/crypto/Kconfig" endif -- 2.50.1 From 11d7956d526f72da82b575389a2d8f9eb15dea6c Mon Sep 17 00:00:00 2001 From: Eric Biggers Date: Mon, 28 Apr 2025 10:00:36 -0700 Subject: [PATCH 02/16] crypto: x86/sha256 - implement library instead of shash Instead of providing crypto_shash algorithms for the arch-optimized SHA-256 code, instead implement the SHA-256 library. This is much simpler, it makes the SHA-256 library functions be arch-optimized, and it fixes the longstanding issue where the arch-optimized SHA-256 was disabled by default. SHA-256 still remains available through crypto_shash, but individual architectures no longer need to handle it. To match sha256_blocks_arch(), change the type of the nblocks parameter of the assembly functions from int to size_t. The assembly functions actually already treated it as size_t. Signed-off-by: Eric Biggers Signed-off-by: Herbert Xu --- arch/x86/crypto/Kconfig | 14 - arch/x86/crypto/Makefile | 3 - arch/x86/crypto/sha256_ssse3_glue.c | 432 ------------------ arch/x86/lib/crypto/Kconfig | 7 + arch/x86/lib/crypto/Makefile | 3 + arch/x86/{ => lib}/crypto/sha256-avx-asm.S | 12 +- arch/x86/{ => lib}/crypto/sha256-avx2-asm.S | 12 +- .../crypto/sha256-ni-asm.S} | 36 +- arch/x86/{ => lib}/crypto/sha256-ssse3-asm.S | 14 +- arch/x86/lib/crypto/sha256.c | 74 +++ 10 files changed, 118 insertions(+), 489 deletions(-) delete mode 100644 arch/x86/crypto/sha256_ssse3_glue.c rename arch/x86/{ => lib}/crypto/sha256-avx-asm.S (98%) rename arch/x86/{ => lib}/crypto/sha256-avx2-asm.S (98%) rename arch/x86/{crypto/sha256_ni_asm.S => lib/crypto/sha256-ni-asm.S} (85%) rename arch/x86/{ => lib}/crypto/sha256-ssse3-asm.S (98%) create mode 100644 arch/x86/lib/crypto/sha256.c diff --git a/arch/x86/crypto/Kconfig b/arch/x86/crypto/Kconfig index 9e941362e4cd..56cfdc79e2c6 100644 --- a/arch/x86/crypto/Kconfig +++ b/arch/x86/crypto/Kconfig @@ -390,20 +390,6 @@ config CRYPTO_SHA1_SSSE3 - AVX2 (Advanced Vector Extensions 2) - SHA-NI (SHA Extensions New Instructions) -config CRYPTO_SHA256_SSSE3 - tristate "Hash functions: SHA-224 and SHA-256 (SSSE3/AVX/AVX2/SHA-NI)" - depends on 64BIT - select CRYPTO_SHA256 - select CRYPTO_HASH - help - SHA-224 and SHA-256 secure hash algorithms (FIPS 180) - - Architecture: x86_64 using: - - SSSE3 (Supplemental SSE3) - - AVX (Advanced Vector Extensions) - - AVX2 (Advanced Vector Extensions 2) - - SHA-NI (SHA Extensions New Instructions) - config CRYPTO_SHA512_SSSE3 tristate "Hash functions: SHA-384 and SHA-512 (SSSE3/AVX/AVX2)" depends on 64BIT diff --git a/arch/x86/crypto/Makefile b/arch/x86/crypto/Makefile index fad59a6c6c26..aa289a9e0153 100644 --- a/arch/x86/crypto/Makefile +++ b/arch/x86/crypto/Makefile @@ -54,9 +54,6 @@ endif obj-$(CONFIG_CRYPTO_SHA1_SSSE3) += sha1-ssse3.o sha1-ssse3-y := sha1_avx2_x86_64_asm.o sha1_ssse3_asm.o sha1_ni_asm.o sha1_ssse3_glue.o -obj-$(CONFIG_CRYPTO_SHA256_SSSE3) += sha256-ssse3.o -sha256-ssse3-y := sha256-ssse3-asm.o sha256-avx-asm.o sha256-avx2-asm.o sha256_ni_asm.o sha256_ssse3_glue.o - obj-$(CONFIG_CRYPTO_SHA512_SSSE3) += sha512-ssse3.o sha512-ssse3-y := sha512-ssse3-asm.o sha512-avx-asm.o sha512-avx2-asm.o sha512_ssse3_glue.o diff --git a/arch/x86/crypto/sha256_ssse3_glue.c b/arch/x86/crypto/sha256_ssse3_glue.c deleted file mode 100644 index a5d3be00550b..000000000000 --- a/arch/x86/crypto/sha256_ssse3_glue.c +++ /dev/null @@ -1,432 +0,0 @@ -/* - * Cryptographic API. - * - * Glue code for the SHA256 Secure Hash Algorithm assembler implementations - * using SSSE3, AVX, AVX2, and SHA-NI instructions. - * - * This file is based on sha256_generic.c - * - * Copyright (C) 2013 Intel Corporation. - * - * Author: - * Tim Chen - * - * This program is free software; you can redistribute it and/or modify it - * under the terms of the GNU General Public License as published by the Free - * Software Foundation; either version 2 of the License, or (at your option) - * any later version. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, - * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF - * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND - * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS - * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN - * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN - * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - * SOFTWARE. - */ - - -#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt - -#include -#include -#include -#include -#include -#include -#include - -asmlinkage void sha256_transform_ssse3(struct crypto_sha256_state *state, - const u8 *data, int blocks); - -static const struct x86_cpu_id module_cpu_ids[] = { - X86_MATCH_FEATURE(X86_FEATURE_SHA_NI, NULL), - X86_MATCH_FEATURE(X86_FEATURE_AVX2, NULL), - X86_MATCH_FEATURE(X86_FEATURE_AVX, NULL), - X86_MATCH_FEATURE(X86_FEATURE_SSSE3, NULL), - {} -}; -MODULE_DEVICE_TABLE(x86cpu, module_cpu_ids); - -static int _sha256_update(struct shash_desc *desc, const u8 *data, - unsigned int len, - sha256_block_fn *sha256_xform) -{ - int remain; - - /* - * Make sure struct crypto_sha256_state begins directly with the SHA256 - * 256-bit internal state, as this is what the asm functions expect. - */ - BUILD_BUG_ON(offsetof(struct crypto_sha256_state, state) != 0); - - kernel_fpu_begin(); - remain = sha256_base_do_update_blocks(desc, data, len, sha256_xform); - kernel_fpu_end(); - - return remain; -} - -static int sha256_finup(struct shash_desc *desc, const u8 *data, - unsigned int len, u8 *out, sha256_block_fn *sha256_xform) -{ - kernel_fpu_begin(); - sha256_base_do_finup(desc, data, len, sha256_xform); - kernel_fpu_end(); - - return sha256_base_finish(desc, out); -} - -static int sha256_ssse3_update(struct shash_desc *desc, const u8 *data, - unsigned int len) -{ - return _sha256_update(desc, data, len, sha256_transform_ssse3); -} - -static int sha256_ssse3_finup(struct shash_desc *desc, const u8 *data, - unsigned int len, u8 *out) -{ - return sha256_finup(desc, data, len, out, sha256_transform_ssse3); -} - -static int sha256_ssse3_digest(struct shash_desc *desc, const u8 *data, - unsigned int len, u8 *out) -{ - return sha256_base_init(desc) ?: - sha256_ssse3_finup(desc, data, len, out); -} - -static struct shash_alg sha256_ssse3_algs[] = { { - .digestsize = SHA256_DIGEST_SIZE, - .init = sha256_base_init, - .update = sha256_ssse3_update, - .finup = sha256_ssse3_finup, - .digest = sha256_ssse3_digest, - .descsize = sizeof(struct crypto_sha256_state), - .base = { - .cra_name = "sha256", - .cra_driver_name = "sha256-ssse3", - .cra_priority = 150, - .cra_flags = CRYPTO_AHASH_ALG_BLOCK_ONLY | - CRYPTO_AHASH_ALG_FINUP_MAX, - .cra_blocksize = SHA256_BLOCK_SIZE, - .cra_module = THIS_MODULE, - } -}, { - .digestsize = SHA224_DIGEST_SIZE, - .init = sha224_base_init, - .update = sha256_ssse3_update, - .finup = sha256_ssse3_finup, - .descsize = sizeof(struct crypto_sha256_state), - .base = { - .cra_name = "sha224", - .cra_driver_name = "sha224-ssse3", - .cra_priority = 150, - .cra_flags = CRYPTO_AHASH_ALG_BLOCK_ONLY | - CRYPTO_AHASH_ALG_FINUP_MAX, - .cra_blocksize = SHA224_BLOCK_SIZE, - .cra_module = THIS_MODULE, - } -} }; - -static int register_sha256_ssse3(void) -{ - if (boot_cpu_has(X86_FEATURE_SSSE3)) - return crypto_register_shashes(sha256_ssse3_algs, - ARRAY_SIZE(sha256_ssse3_algs)); - return 0; -} - -static void unregister_sha256_ssse3(void) -{ - if (boot_cpu_has(X86_FEATURE_SSSE3)) - crypto_unregister_shashes(sha256_ssse3_algs, - ARRAY_SIZE(sha256_ssse3_algs)); -} - -asmlinkage void sha256_transform_avx(struct crypto_sha256_state *state, - const u8 *data, int blocks); - -static int sha256_avx_update(struct shash_desc *desc, const u8 *data, - unsigned int len) -{ - return _sha256_update(desc, data, len, sha256_transform_avx); -} - -static int sha256_avx_finup(struct shash_desc *desc, const u8 *data, - unsigned int len, u8 *out) -{ - return sha256_finup(desc, data, len, out, sha256_transform_avx); -} - -static int sha256_avx_digest(struct shash_desc *desc, const u8 *data, - unsigned int len, u8 *out) -{ - return sha256_base_init(desc) ?: - sha256_avx_finup(desc, data, len, out); -} - -static struct shash_alg sha256_avx_algs[] = { { - .digestsize = SHA256_DIGEST_SIZE, - .init = sha256_base_init, - .update = sha256_avx_update, - .finup = sha256_avx_finup, - .digest = sha256_avx_digest, - .descsize = sizeof(struct crypto_sha256_state), - .base = { - .cra_name = "sha256", - .cra_driver_name = "sha256-avx", - .cra_priority = 160, - .cra_flags = CRYPTO_AHASH_ALG_BLOCK_ONLY | - CRYPTO_AHASH_ALG_FINUP_MAX, - .cra_blocksize = SHA256_BLOCK_SIZE, - .cra_module = THIS_MODULE, - } -}, { - .digestsize = SHA224_DIGEST_SIZE, - .init = sha224_base_init, - .update = sha256_avx_update, - .finup = sha256_avx_finup, - .descsize = sizeof(struct crypto_sha256_state), - .base = { - .cra_name = "sha224", - .cra_driver_name = "sha224-avx", - .cra_priority = 160, - .cra_flags = CRYPTO_AHASH_ALG_BLOCK_ONLY | - CRYPTO_AHASH_ALG_FINUP_MAX, - .cra_blocksize = SHA224_BLOCK_SIZE, - .cra_module = THIS_MODULE, - } -} }; - -static bool avx_usable(void) -{ - if (!cpu_has_xfeatures(XFEATURE_MASK_SSE | XFEATURE_MASK_YMM, NULL)) { - if (boot_cpu_has(X86_FEATURE_AVX)) - pr_info("AVX detected but unusable.\n"); - return false; - } - - return true; -} - -static int register_sha256_avx(void) -{ - if (avx_usable()) - return crypto_register_shashes(sha256_avx_algs, - ARRAY_SIZE(sha256_avx_algs)); - return 0; -} - -static void unregister_sha256_avx(void) -{ - if (avx_usable()) - crypto_unregister_shashes(sha256_avx_algs, - ARRAY_SIZE(sha256_avx_algs)); -} - -asmlinkage void sha256_transform_rorx(struct crypto_sha256_state *state, - const u8 *data, int blocks); - -static int sha256_avx2_update(struct shash_desc *desc, const u8 *data, - unsigned int len) -{ - return _sha256_update(desc, data, len, sha256_transform_rorx); -} - -static int sha256_avx2_finup(struct shash_desc *desc, const u8 *data, - unsigned int len, u8 *out) -{ - return sha256_finup(desc, data, len, out, sha256_transform_rorx); -} - -static int sha256_avx2_digest(struct shash_desc *desc, const u8 *data, - unsigned int len, u8 *out) -{ - return sha256_base_init(desc) ?: - sha256_avx2_finup(desc, data, len, out); -} - -static struct shash_alg sha256_avx2_algs[] = { { - .digestsize = SHA256_DIGEST_SIZE, - .init = sha256_base_init, - .update = sha256_avx2_update, - .finup = sha256_avx2_finup, - .digest = sha256_avx2_digest, - .descsize = sizeof(struct crypto_sha256_state), - .base = { - .cra_name = "sha256", - .cra_driver_name = "sha256-avx2", - .cra_priority = 170, - .cra_flags = CRYPTO_AHASH_ALG_BLOCK_ONLY | - CRYPTO_AHASH_ALG_FINUP_MAX, - .cra_blocksize = SHA256_BLOCK_SIZE, - .cra_module = THIS_MODULE, - } -}, { - .digestsize = SHA224_DIGEST_SIZE, - .init = sha224_base_init, - .update = sha256_avx2_update, - .finup = sha256_avx2_finup, - .descsize = sizeof(struct crypto_sha256_state), - .base = { - .cra_name = "sha224", - .cra_driver_name = "sha224-avx2", - .cra_priority = 170, - .cra_flags = CRYPTO_AHASH_ALG_BLOCK_ONLY | - CRYPTO_AHASH_ALG_FINUP_MAX, - .cra_blocksize = SHA224_BLOCK_SIZE, - .cra_module = THIS_MODULE, - } -} }; - -static bool avx2_usable(void) -{ - if (avx_usable() && boot_cpu_has(X86_FEATURE_AVX2) && - boot_cpu_has(X86_FEATURE_BMI2)) - return true; - - return false; -} - -static int register_sha256_avx2(void) -{ - if (avx2_usable()) - return crypto_register_shashes(sha256_avx2_algs, - ARRAY_SIZE(sha256_avx2_algs)); - return 0; -} - -static void unregister_sha256_avx2(void) -{ - if (avx2_usable()) - crypto_unregister_shashes(sha256_avx2_algs, - ARRAY_SIZE(sha256_avx2_algs)); -} - -asmlinkage void sha256_ni_transform(struct crypto_sha256_state *digest, - const u8 *data, int rounds); - -static int sha256_ni_update(struct shash_desc *desc, const u8 *data, - unsigned int len) -{ - return _sha256_update(desc, data, len, sha256_ni_transform); -} - -static int sha256_ni_finup(struct shash_desc *desc, const u8 *data, - unsigned int len, u8 *out) -{ - return sha256_finup(desc, data, len, out, sha256_ni_transform); -} - -static int sha256_ni_digest(struct shash_desc *desc, const u8 *data, - unsigned int len, u8 *out) -{ - return sha256_base_init(desc) ?: - sha256_ni_finup(desc, data, len, out); -} - -static struct shash_alg sha256_ni_algs[] = { { - .digestsize = SHA256_DIGEST_SIZE, - .init = sha256_base_init, - .update = sha256_ni_update, - .finup = sha256_ni_finup, - .digest = sha256_ni_digest, - .descsize = sizeof(struct crypto_sha256_state), - .base = { - .cra_name = "sha256", - .cra_driver_name = "sha256-ni", - .cra_priority = 250, - .cra_flags = CRYPTO_AHASH_ALG_BLOCK_ONLY | - CRYPTO_AHASH_ALG_FINUP_MAX, - .cra_blocksize = SHA256_BLOCK_SIZE, - .cra_module = THIS_MODULE, - } -}, { - .digestsize = SHA224_DIGEST_SIZE, - .init = sha224_base_init, - .update = sha256_ni_update, - .finup = sha256_ni_finup, - .descsize = sizeof(struct crypto_sha256_state), - .base = { - .cra_name = "sha224", - .cra_driver_name = "sha224-ni", - .cra_priority = 250, - .cra_flags = CRYPTO_AHASH_ALG_BLOCK_ONLY | - CRYPTO_AHASH_ALG_FINUP_MAX, - .cra_blocksize = SHA224_BLOCK_SIZE, - .cra_module = THIS_MODULE, - } -} }; - -static int register_sha256_ni(void) -{ - if (boot_cpu_has(X86_FEATURE_SHA_NI)) - return crypto_register_shashes(sha256_ni_algs, - ARRAY_SIZE(sha256_ni_algs)); - return 0; -} - -static void unregister_sha256_ni(void) -{ - if (boot_cpu_has(X86_FEATURE_SHA_NI)) - crypto_unregister_shashes(sha256_ni_algs, - ARRAY_SIZE(sha256_ni_algs)); -} - -static int __init sha256_ssse3_mod_init(void) -{ - if (!x86_match_cpu(module_cpu_ids)) - return -ENODEV; - - if (register_sha256_ssse3()) - goto fail; - - if (register_sha256_avx()) { - unregister_sha256_ssse3(); - goto fail; - } - - if (register_sha256_avx2()) { - unregister_sha256_avx(); - unregister_sha256_ssse3(); - goto fail; - } - - if (register_sha256_ni()) { - unregister_sha256_avx2(); - unregister_sha256_avx(); - unregister_sha256_ssse3(); - goto fail; - } - - return 0; -fail: - return -ENODEV; -} - -static void __exit sha256_ssse3_mod_fini(void) -{ - unregister_sha256_ni(); - unregister_sha256_avx2(); - unregister_sha256_avx(); - unregister_sha256_ssse3(); -} - -module_init(sha256_ssse3_mod_init); -module_exit(sha256_ssse3_mod_fini); - -MODULE_LICENSE("GPL"); -MODULE_DESCRIPTION("SHA256 Secure Hash Algorithm, Supplemental SSE3 accelerated"); - -MODULE_ALIAS_CRYPTO("sha256"); -MODULE_ALIAS_CRYPTO("sha256-ssse3"); -MODULE_ALIAS_CRYPTO("sha256-avx"); -MODULE_ALIAS_CRYPTO("sha256-avx2"); -MODULE_ALIAS_CRYPTO("sha224"); -MODULE_ALIAS_CRYPTO("sha224-ssse3"); -MODULE_ALIAS_CRYPTO("sha224-avx"); -MODULE_ALIAS_CRYPTO("sha224-avx2"); -MODULE_ALIAS_CRYPTO("sha256-ni"); -MODULE_ALIAS_CRYPTO("sha224-ni"); diff --git a/arch/x86/lib/crypto/Kconfig b/arch/x86/lib/crypto/Kconfig index 546fe2afe0b5..e344579db3d8 100644 --- a/arch/x86/lib/crypto/Kconfig +++ b/arch/x86/lib/crypto/Kconfig @@ -24,3 +24,10 @@ config CRYPTO_POLY1305_X86_64 depends on 64BIT default CRYPTO_LIB_POLY1305 select CRYPTO_ARCH_HAVE_LIB_POLY1305 + +config CRYPTO_SHA256_X86_64 + tristate + depends on 64BIT + default CRYPTO_LIB_SHA256 + select CRYPTO_ARCH_HAVE_LIB_SHA256 + select CRYPTO_LIB_SHA256_GENERIC diff --git a/arch/x86/lib/crypto/Makefile b/arch/x86/lib/crypto/Makefile index c2ff8c5f1046..abceca3d31c0 100644 --- a/arch/x86/lib/crypto/Makefile +++ b/arch/x86/lib/crypto/Makefile @@ -10,6 +10,9 @@ obj-$(CONFIG_CRYPTO_POLY1305_X86_64) += poly1305-x86_64.o poly1305-x86_64-y := poly1305-x86_64-cryptogams.o poly1305_glue.o targets += poly1305-x86_64-cryptogams.S +obj-$(CONFIG_CRYPTO_SHA256_X86_64) += sha256-x86_64.o +sha256-x86_64-y := sha256.o sha256-ssse3-asm.o sha256-avx-asm.o sha256-avx2-asm.o sha256-ni-asm.o + quiet_cmd_perlasm = PERLASM $@ cmd_perlasm = $(PERL) $< > $@ diff --git a/arch/x86/crypto/sha256-avx-asm.S b/arch/x86/lib/crypto/sha256-avx-asm.S similarity index 98% rename from arch/x86/crypto/sha256-avx-asm.S rename to arch/x86/lib/crypto/sha256-avx-asm.S index 53de72bdd851..0d7b2c3e45d9 100644 --- a/arch/x86/crypto/sha256-avx-asm.S +++ b/arch/x86/lib/crypto/sha256-avx-asm.S @@ -48,7 +48,7 @@ ######################################################################## #include -#include +#include ## assume buffers not aligned #define VMOVDQ vmovdqu @@ -341,13 +341,13 @@ a = TMP_ .endm ######################################################################## -## void sha256_transform_avx(state sha256_state *state, const u8 *data, int blocks) -## arg 1 : pointer to state -## arg 2 : pointer to input data -## arg 3 : Num blocks +## void sha256_transform_avx(u32 state[SHA256_STATE_WORDS], +## const u8 *data, size_t nblocks); ######################################################################## .text -SYM_TYPED_FUNC_START(sha256_transform_avx) +SYM_FUNC_START(sha256_transform_avx) + ANNOTATE_NOENDBR # since this is called only via static_call + pushq %rbx pushq %r12 pushq %r13 diff --git a/arch/x86/crypto/sha256-avx2-asm.S b/arch/x86/lib/crypto/sha256-avx2-asm.S similarity index 98% rename from arch/x86/crypto/sha256-avx2-asm.S rename to arch/x86/lib/crypto/sha256-avx2-asm.S index 0bbec1c75cd0..25d3380321ec 100644 --- a/arch/x86/crypto/sha256-avx2-asm.S +++ b/arch/x86/lib/crypto/sha256-avx2-asm.S @@ -49,7 +49,7 @@ ######################################################################## #include -#include +#include ## assume buffers not aligned #define VMOVDQ vmovdqu @@ -518,13 +518,13 @@ STACK_SIZE = _CTX + _CTX_SIZE .endm ######################################################################## -## void sha256_transform_rorx(struct sha256_state *state, const u8 *data, int blocks) -## arg 1 : pointer to state -## arg 2 : pointer to input data -## arg 3 : Num blocks +## void sha256_transform_rorx(u32 state[SHA256_STATE_WORDS], +## const u8 *data, size_t nblocks); ######################################################################## .text -SYM_TYPED_FUNC_START(sha256_transform_rorx) +SYM_FUNC_START(sha256_transform_rorx) + ANNOTATE_NOENDBR # since this is called only via static_call + pushq %rbx pushq %r12 pushq %r13 diff --git a/arch/x86/crypto/sha256_ni_asm.S b/arch/x86/lib/crypto/sha256-ni-asm.S similarity index 85% rename from arch/x86/crypto/sha256_ni_asm.S rename to arch/x86/lib/crypto/sha256-ni-asm.S index d515a55a3bc1..d3548206cf3d 100644 --- a/arch/x86/crypto/sha256_ni_asm.S +++ b/arch/x86/lib/crypto/sha256-ni-asm.S @@ -54,9 +54,9 @@ */ #include -#include +#include -#define DIGEST_PTR %rdi /* 1st arg */ +#define STATE_PTR %rdi /* 1st arg */ #define DATA_PTR %rsi /* 2nd arg */ #define NUM_BLKS %rdx /* 3rd arg */ @@ -98,24 +98,20 @@ .endm /* - * Intel SHA Extensions optimized implementation of a SHA-256 update function + * Intel SHA Extensions optimized implementation of a SHA-256 block function * - * The function takes a pointer to the current hash values, a pointer to the - * input data, and a number of 64 byte blocks to process. Once all blocks have - * been processed, the digest pointer is updated with the resulting hash value. - * The function only processes complete blocks, there is no functionality to - * store partial blocks. All message padding and hash value initialization must - * be done outside the update function. + * This function takes a pointer to the current SHA-256 state, a pointer to the + * input data, and the number of 64-byte blocks to process. Once all blocks + * have been processed, the state is updated with the new state. This function + * only processes complete blocks. State initialization, buffering of partial + * blocks, and digest finalization is expected to be handled elsewhere. * - * void sha256_ni_transform(uint32_t *digest, const void *data, - uint32_t numBlocks); - * digest : pointer to digest - * data: pointer to input data - * numBlocks: Number of blocks to process + * void sha256_ni_transform(u32 state[SHA256_STATE_WORDS], + * const u8 *data, size_t nblocks); */ - .text -SYM_TYPED_FUNC_START(sha256_ni_transform) +SYM_FUNC_START(sha256_ni_transform) + ANNOTATE_NOENDBR # since this is called only via static_call shl $6, NUM_BLKS /* convert to bytes */ jz .Ldone_hash @@ -126,8 +122,8 @@ SYM_TYPED_FUNC_START(sha256_ni_transform) * Need to reorder these appropriately * DCBA, HGFE -> ABEF, CDGH */ - movdqu 0*16(DIGEST_PTR), STATE0 /* DCBA */ - movdqu 1*16(DIGEST_PTR), STATE1 /* HGFE */ + movdqu 0*16(STATE_PTR), STATE0 /* DCBA */ + movdqu 1*16(STATE_PTR), STATE1 /* HGFE */ movdqa STATE0, TMP punpcklqdq STATE1, STATE0 /* FEBA */ @@ -166,8 +162,8 @@ SYM_TYPED_FUNC_START(sha256_ni_transform) pshufd $0xB1, STATE0, STATE0 /* HGFE */ pshufd $0x1B, STATE1, STATE1 /* DCBA */ - movdqu STATE1, 0*16(DIGEST_PTR) - movdqu STATE0, 1*16(DIGEST_PTR) + movdqu STATE1, 0*16(STATE_PTR) + movdqu STATE0, 1*16(STATE_PTR) .Ldone_hash: diff --git a/arch/x86/crypto/sha256-ssse3-asm.S b/arch/x86/lib/crypto/sha256-ssse3-asm.S similarity index 98% rename from arch/x86/crypto/sha256-ssse3-asm.S rename to arch/x86/lib/crypto/sha256-ssse3-asm.S index 93264ee44543..7f24a4cdcb25 100644 --- a/arch/x86/crypto/sha256-ssse3-asm.S +++ b/arch/x86/lib/crypto/sha256-ssse3-asm.S @@ -47,7 +47,7 @@ ######################################################################## #include -#include +#include ## assume buffers not aligned #define MOVDQ movdqu @@ -348,15 +348,13 @@ a = TMP_ .endm ######################################################################## -## void sha256_transform_ssse3(struct sha256_state *state, const u8 *data, -## int blocks); -## arg 1 : pointer to state -## (struct sha256_state is assumed to begin with u32 state[8]) -## arg 2 : pointer to input data -## arg 3 : Num blocks +## void sha256_transform_ssse3(u32 state[SHA256_STATE_WORDS], +## const u8 *data, size_t nblocks); ######################################################################## .text -SYM_TYPED_FUNC_START(sha256_transform_ssse3) +SYM_FUNC_START(sha256_transform_ssse3) + ANNOTATE_NOENDBR # since this is called only via static_call + pushq %rbx pushq %r12 pushq %r13 diff --git a/arch/x86/lib/crypto/sha256.c b/arch/x86/lib/crypto/sha256.c new file mode 100644 index 000000000000..47865b5cd94b --- /dev/null +++ b/arch/x86/lib/crypto/sha256.c @@ -0,0 +1,74 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* + * SHA-256 optimized for x86_64 + * + * Copyright 2025 Google LLC + */ +#include +#include +#include +#include +#include +#include + +asmlinkage void sha256_transform_ssse3(u32 state[SHA256_STATE_WORDS], + const u8 *data, size_t nblocks); +asmlinkage void sha256_transform_avx(u32 state[SHA256_STATE_WORDS], + const u8 *data, size_t nblocks); +asmlinkage void sha256_transform_rorx(u32 state[SHA256_STATE_WORDS], + const u8 *data, size_t nblocks); +asmlinkage void sha256_ni_transform(u32 state[SHA256_STATE_WORDS], + const u8 *data, size_t nblocks); + +static __ro_after_init DEFINE_STATIC_KEY_FALSE(have_sha256_x86); + +DEFINE_STATIC_CALL(sha256_blocks_x86, sha256_transform_ssse3); + +void sha256_blocks_arch(u32 state[SHA256_STATE_WORDS], + const u8 *data, size_t nblocks) +{ + if (static_branch_likely(&have_sha256_x86) && crypto_simd_usable()) { + kernel_fpu_begin(); + static_call(sha256_blocks_x86)(state, data, nblocks); + kernel_fpu_end(); + } else { + sha256_blocks_generic(state, data, nblocks); + } +} +EXPORT_SYMBOL(sha256_blocks_arch); + +bool sha256_is_arch_optimized(void) +{ + return static_key_enabled(&have_sha256_x86); +} +EXPORT_SYMBOL(sha256_is_arch_optimized); + +static int __init sha256_x86_mod_init(void) +{ + if (boot_cpu_has(X86_FEATURE_SHA_NI)) { + static_call_update(sha256_blocks_x86, sha256_ni_transform); + } else if (cpu_has_xfeatures(XFEATURE_MASK_SSE | + XFEATURE_MASK_YMM, NULL) && + boot_cpu_has(X86_FEATURE_AVX)) { + if (boot_cpu_has(X86_FEATURE_AVX2) && + boot_cpu_has(X86_FEATURE_BMI2)) + static_call_update(sha256_blocks_x86, + sha256_transform_rorx); + else + static_call_update(sha256_blocks_x86, + sha256_transform_avx); + } else if (!boot_cpu_has(X86_FEATURE_SSSE3)) { + return 0; + } + static_branch_enable(&have_sha256_x86); + return 0; +} +arch_initcall(sha256_x86_mod_init); + +static void __exit sha256_x86_mod_exit(void) +{ +} +module_exit(sha256_x86_mod_exit); + +MODULE_LICENSE("GPL"); +MODULE_DESCRIPTION("SHA-256 optimized for x86_64"); -- 2.50.1 From 5aab01777fd38ff820bd7ae6cbc15d03b85de92f Mon Sep 17 00:00:00 2001 From: Eric Biggers Date: Mon, 28 Apr 2025 10:00:37 -0700 Subject: [PATCH 03/16] crypto: sha256 - remove sha256_base.h sha256_base.h is no longer used, so remove it. Signed-off-by: Eric Biggers Signed-off-by: Herbert Xu --- include/crypto/sha256_base.h | 151 ----------------------------------- 1 file changed, 151 deletions(-) delete mode 100644 include/crypto/sha256_base.h diff --git a/include/crypto/sha256_base.h b/include/crypto/sha256_base.h deleted file mode 100644 index 804361731a7a..000000000000 --- a/include/crypto/sha256_base.h +++ /dev/null @@ -1,151 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0-only */ -/* - * sha256_base.h - core logic for SHA-256 implementations - * - * Copyright (C) 2015 Linaro Ltd - */ - -#ifndef _CRYPTO_SHA256_BASE_H -#define _CRYPTO_SHA256_BASE_H - -#include -#include -#include -#include -#include -#include -#include - -typedef void (sha256_block_fn)(struct crypto_sha256_state *sst, u8 const *src, - int blocks); - -static inline int sha224_base_init(struct shash_desc *desc) -{ - struct sha256_state *sctx = shash_desc_ctx(desc); - - sha224_init(sctx); - return 0; -} - -static inline int sha256_base_init(struct shash_desc *desc) -{ - struct sha256_state *sctx = shash_desc_ctx(desc); - - sha256_init(sctx); - return 0; -} - -static inline int lib_sha256_base_do_update(struct sha256_state *sctx, - const u8 *data, - unsigned int len, - sha256_block_fn *block_fn) -{ - unsigned int partial = sctx->count % SHA256_BLOCK_SIZE; - - sctx->count += len; - BLOCK_HASH_UPDATE_BLOCKS(block_fn, &sctx->ctx, data, len, - SHA256_BLOCK_SIZE, sctx->buf, partial); - return 0; -} - -static inline int lib_sha256_base_do_update_blocks( - struct crypto_sha256_state *sctx, const u8 *data, unsigned int len, - sha256_block_fn *block_fn) -{ - unsigned int remain = len - round_down(len, SHA256_BLOCK_SIZE); - - sctx->count += len - remain; - block_fn(sctx, data, len / SHA256_BLOCK_SIZE); - return remain; -} - -static inline int sha256_base_do_update_blocks( - struct shash_desc *desc, const u8 *data, unsigned int len, - sha256_block_fn *block_fn) -{ - return lib_sha256_base_do_update_blocks(shash_desc_ctx(desc), data, - len, block_fn); -} - -static inline int lib_sha256_base_do_finup(struct crypto_sha256_state *sctx, - const u8 *src, unsigned int len, - sha256_block_fn *block_fn) -{ - unsigned int bit_offset = SHA256_BLOCK_SIZE / 8 - 1; - union { - __be64 b64[SHA256_BLOCK_SIZE / 4]; - u8 u8[SHA256_BLOCK_SIZE * 2]; - } block = {}; - - if (len >= bit_offset * 8) - bit_offset += SHA256_BLOCK_SIZE / 8; - memcpy(&block, src, len); - block.u8[len] = 0x80; - sctx->count += len; - block.b64[bit_offset] = cpu_to_be64(sctx->count << 3); - block_fn(sctx, block.u8, (bit_offset + 1) * 8 / SHA256_BLOCK_SIZE); - memzero_explicit(&block, sizeof(block)); - - return 0; -} - -static inline int sha256_base_do_finup(struct shash_desc *desc, - const u8 *src, unsigned int len, - sha256_block_fn *block_fn) -{ - struct crypto_sha256_state *sctx = shash_desc_ctx(desc); - - if (len >= SHA256_BLOCK_SIZE) { - int remain; - - remain = lib_sha256_base_do_update_blocks(sctx, src, len, - block_fn); - src += len - remain; - len = remain; - } - return lib_sha256_base_do_finup(sctx, src, len, block_fn); -} - -static inline int lib_sha256_base_do_finalize(struct sha256_state *sctx, - sha256_block_fn *block_fn) -{ - unsigned int partial = sctx->count % SHA256_BLOCK_SIZE; - struct crypto_sha256_state *state = (void *)sctx; - - sctx->count -= partial; - return lib_sha256_base_do_finup(state, sctx->buf, partial, block_fn); -} - -static inline int __sha256_base_finish(u32 state[SHA256_DIGEST_SIZE / 4], - u8 *out, unsigned int digest_size) -{ - __be32 *digest = (__be32 *)out; - int i; - - for (i = 0; digest_size > 0; i++, digest_size -= sizeof(__be32)) - put_unaligned_be32(state[i], digest++); - return 0; -} - -static inline void lib_sha256_base_finish(struct sha256_state *sctx, u8 *out, - unsigned int digest_size) -{ - __sha256_base_finish(sctx->state, out, digest_size); - memzero_explicit(sctx, sizeof(*sctx)); -} - -static inline int sha256_base_finish(struct shash_desc *desc, u8 *out) -{ - unsigned int digest_size = crypto_shash_digestsize(desc->tfm); - struct crypto_sha256_state *sctx = shash_desc_ctx(desc); - - return __sha256_base_finish(sctx->state, out, digest_size); -} - -static inline void sha256_transform_blocks(struct crypto_sha256_state *sst, - const u8 *input, int blocks) -{ - sha256_blocks_generic(sst->state, input, blocks); -} - -#endif /* _CRYPTO_SHA256_BASE_H */ -- 2.50.1 From 7350fef56b7629a539897c332d7324629580671d Mon Sep 17 00:00:00 2001 From: Eric Biggers Date: Mon, 28 Apr 2025 10:00:38 -0700 Subject: [PATCH 04/16] crypto: lib/sha256 - improve function prototypes Follow best practices by changing the length parameters to size_t and explicitly specifying the length of the output digest arrays. Signed-off-by: Eric Biggers Signed-off-by: Herbert Xu --- include/crypto/sha2.h | 8 ++++---- lib/crypto/sha256.c | 8 ++++---- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/include/crypto/sha2.h b/include/crypto/sha2.h index 9a56286d736d..9853cd2d1291 100644 --- a/include/crypto/sha2.h +++ b/include/crypto/sha2.h @@ -100,9 +100,9 @@ static inline void sha256_init(struct sha256_state *sctx) sctx->state[7] = SHA256_H7; sctx->count = 0; } -void sha256_update(struct sha256_state *sctx, const u8 *data, unsigned int len); -void sha256_final(struct sha256_state *sctx, u8 *out); -void sha256(const u8 *data, unsigned int len, u8 *out); +void sha256_update(struct sha256_state *sctx, const u8 *data, size_t len); +void sha256_final(struct sha256_state *sctx, u8 out[SHA256_DIGEST_SIZE]); +void sha256(const u8 *data, size_t len, u8 out[SHA256_DIGEST_SIZE]); static inline void sha224_init(struct sha256_state *sctx) { @@ -117,6 +117,6 @@ static inline void sha224_init(struct sha256_state *sctx) sctx->count = 0; } /* Simply use sha256_update as it is equivalent to sha224_update. */ -void sha224_final(struct sha256_state *sctx, u8 *out); +void sha224_final(struct sha256_state *sctx, u8 out[SHA224_DIGEST_SIZE]); #endif /* _CRYPTO_SHA2_H */ diff --git a/lib/crypto/sha256.c b/lib/crypto/sha256.c index 4b19cf977ef1..563f09c9f381 100644 --- a/lib/crypto/sha256.c +++ b/lib/crypto/sha256.c @@ -70,7 +70,7 @@ static inline void __sha256_update(struct sha256_state *sctx, const u8 *data, memcpy(&sctx->buf[partial], data, len); } -void sha256_update(struct sha256_state *sctx, const u8 *data, unsigned int len) +void sha256_update(struct sha256_state *sctx, const u8 *data, size_t len) { __sha256_update(sctx, data, len, false); } @@ -101,19 +101,19 @@ static inline void __sha256_final(struct sha256_state *sctx, u8 *out, memzero_explicit(sctx, sizeof(*sctx)); } -void sha256_final(struct sha256_state *sctx, u8 *out) +void sha256_final(struct sha256_state *sctx, u8 out[SHA256_DIGEST_SIZE]) { __sha256_final(sctx, out, SHA256_DIGEST_SIZE, false); } EXPORT_SYMBOL(sha256_final); -void sha224_final(struct sha256_state *sctx, u8 *out) +void sha224_final(struct sha256_state *sctx, u8 out[SHA224_DIGEST_SIZE]) { __sha256_final(sctx, out, SHA224_DIGEST_SIZE, false); } EXPORT_SYMBOL(sha224_final); -void sha256(const u8 *data, unsigned int len, u8 *out) +void sha256(const u8 *data, size_t len, u8 out[SHA256_DIGEST_SIZE]) { struct sha256_state sctx; -- 2.50.1 From 04c535ab156bf6c08aeb04d39bbcff0f53b7a0a7 Mon Sep 17 00:00:00 2001 From: Suman Kumar Chakraborty Date: Mon, 28 Apr 2025 18:24:26 +0100 Subject: [PATCH 05/16] crypto: qat - include qat_common in top Makefile To ensure proper functionality, each specific driver needs to access functions located in the qat_common folder. Move the include path for qat_common to the top-level Makefile. This eliminates the need for redundant include directives in the Makefiles of individual drivers. Suggested-by: Andy Shevchenko Signed-off-by: Suman Kumar Chakraborty Reviewed-by: Giovanni Cabiddu Signed-off-by: Herbert Xu --- drivers/crypto/intel/qat/Makefile | 1 + drivers/crypto/intel/qat/qat_420xx/Makefile | 1 - drivers/crypto/intel/qat/qat_4xxx/Makefile | 1 - drivers/crypto/intel/qat/qat_c3xxx/Makefile | 1 - drivers/crypto/intel/qat/qat_c3xxxvf/Makefile | 1 - drivers/crypto/intel/qat/qat_c62x/Makefile | 1 - drivers/crypto/intel/qat/qat_c62xvf/Makefile | 1 - drivers/crypto/intel/qat/qat_dh895xcc/Makefile | 1 - drivers/crypto/intel/qat/qat_dh895xccvf/Makefile | 1 - 9 files changed, 1 insertion(+), 8 deletions(-) diff --git a/drivers/crypto/intel/qat/Makefile b/drivers/crypto/intel/qat/Makefile index 235b69f4f3f7..1eda8dc18515 100644 --- a/drivers/crypto/intel/qat/Makefile +++ b/drivers/crypto/intel/qat/Makefile @@ -1,4 +1,5 @@ # SPDX-License-Identifier: GPL-2.0 +subdir-ccflags-y := -I$(src)/qat_common obj-$(CONFIG_CRYPTO_DEV_QAT) += qat_common/ obj-$(CONFIG_CRYPTO_DEV_QAT_DH895xCC) += qat_dh895xcc/ obj-$(CONFIG_CRYPTO_DEV_QAT_C3XXX) += qat_c3xxx/ diff --git a/drivers/crypto/intel/qat/qat_420xx/Makefile b/drivers/crypto/intel/qat/qat_420xx/Makefile index 72b24b1804cf..f6df54d2993e 100644 --- a/drivers/crypto/intel/qat/qat_420xx/Makefile +++ b/drivers/crypto/intel/qat/qat_420xx/Makefile @@ -1,4 +1,3 @@ # SPDX-License-Identifier: GPL-2.0-only -ccflags-y := -I $(src)/../qat_common obj-$(CONFIG_CRYPTO_DEV_QAT_420XX) += qat_420xx.o qat_420xx-y := adf_drv.o adf_420xx_hw_data.o diff --git a/drivers/crypto/intel/qat/qat_4xxx/Makefile b/drivers/crypto/intel/qat/qat_4xxx/Makefile index e8480bb80dee..188b611445e6 100644 --- a/drivers/crypto/intel/qat/qat_4xxx/Makefile +++ b/drivers/crypto/intel/qat/qat_4xxx/Makefile @@ -1,4 +1,3 @@ # SPDX-License-Identifier: (BSD-3-Clause OR GPL-2.0-only) -ccflags-y := -I $(src)/../qat_common obj-$(CONFIG_CRYPTO_DEV_QAT_4XXX) += qat_4xxx.o qat_4xxx-y := adf_drv.o adf_4xxx_hw_data.o diff --git a/drivers/crypto/intel/qat/qat_c3xxx/Makefile b/drivers/crypto/intel/qat/qat_c3xxx/Makefile index d9e568572da8..43604c025f0c 100644 --- a/drivers/crypto/intel/qat/qat_c3xxx/Makefile +++ b/drivers/crypto/intel/qat/qat_c3xxx/Makefile @@ -1,4 +1,3 @@ # SPDX-License-Identifier: GPL-2.0-only -ccflags-y := -I $(src)/../qat_common obj-$(CONFIG_CRYPTO_DEV_QAT_C3XXX) += qat_c3xxx.o qat_c3xxx-y := adf_drv.o adf_c3xxx_hw_data.o diff --git a/drivers/crypto/intel/qat/qat_c3xxxvf/Makefile b/drivers/crypto/intel/qat/qat_c3xxxvf/Makefile index 31a908a211ac..03f6745b4aa2 100644 --- a/drivers/crypto/intel/qat/qat_c3xxxvf/Makefile +++ b/drivers/crypto/intel/qat/qat_c3xxxvf/Makefile @@ -1,4 +1,3 @@ # SPDX-License-Identifier: GPL-2.0-only -ccflags-y := -I $(src)/../qat_common obj-$(CONFIG_CRYPTO_DEV_QAT_C3XXXVF) += qat_c3xxxvf.o qat_c3xxxvf-y := adf_drv.o adf_c3xxxvf_hw_data.o diff --git a/drivers/crypto/intel/qat/qat_c62x/Makefile b/drivers/crypto/intel/qat/qat_c62x/Makefile index cbdaaa135e84..f3d722bef088 100644 --- a/drivers/crypto/intel/qat/qat_c62x/Makefile +++ b/drivers/crypto/intel/qat/qat_c62x/Makefile @@ -1,4 +1,3 @@ # SPDX-License-Identifier: GPL-2.0-only -ccflags-y := -I $(src)/../qat_common obj-$(CONFIG_CRYPTO_DEV_QAT_C62X) += qat_c62x.o qat_c62x-y := adf_drv.o adf_c62x_hw_data.o diff --git a/drivers/crypto/intel/qat/qat_c62xvf/Makefile b/drivers/crypto/intel/qat/qat_c62xvf/Makefile index 60e499b041ec..ed7f3f722d99 100644 --- a/drivers/crypto/intel/qat/qat_c62xvf/Makefile +++ b/drivers/crypto/intel/qat/qat_c62xvf/Makefile @@ -1,4 +1,3 @@ # SPDX-License-Identifier: GPL-2.0-only -ccflags-y := -I $(src)/../qat_common obj-$(CONFIG_CRYPTO_DEV_QAT_C62XVF) += qat_c62xvf.o qat_c62xvf-y := adf_drv.o adf_c62xvf_hw_data.o diff --git a/drivers/crypto/intel/qat/qat_dh895xcc/Makefile b/drivers/crypto/intel/qat/qat_dh895xcc/Makefile index 5bf5c890c362..1427fe76f171 100644 --- a/drivers/crypto/intel/qat/qat_dh895xcc/Makefile +++ b/drivers/crypto/intel/qat/qat_dh895xcc/Makefile @@ -1,4 +1,3 @@ # SPDX-License-Identifier: GPL-2.0-only -ccflags-y := -I $(src)/../qat_common obj-$(CONFIG_CRYPTO_DEV_QAT_DH895xCC) += qat_dh895xcc.o qat_dh895xcc-y := adf_drv.o adf_dh895xcc_hw_data.o diff --git a/drivers/crypto/intel/qat/qat_dh895xccvf/Makefile b/drivers/crypto/intel/qat/qat_dh895xccvf/Makefile index 93f9c81edf09..c2fdb6e0f68f 100644 --- a/drivers/crypto/intel/qat/qat_dh895xccvf/Makefile +++ b/drivers/crypto/intel/qat/qat_dh895xccvf/Makefile @@ -1,4 +1,3 @@ # SPDX-License-Identifier: GPL-2.0-only -ccflags-y := -I $(src)/../qat_common obj-$(CONFIG_CRYPTO_DEV_QAT_DH895xCCVF) += qat_dh895xccvf.o qat_dh895xccvf-y := adf_drv.o adf_dh895xccvf_hw_data.o -- 2.50.1 From 4a7e1e815c5de24339254eafb6f17cfe0d885a60 Mon Sep 17 00:00:00 2001 From: Herbert Xu Date: Tue, 29 Apr 2025 16:42:20 +0800 Subject: [PATCH 06/16] crypto: rmd160 - Use API partial block handling Use the Crypto API partial block handling. Signed-off-by: Herbert Xu --- crypto/rmd160.c | 88 ++++++++++++++++++++----------------------------- 1 file changed, 36 insertions(+), 52 deletions(-) diff --git a/crypto/rmd160.c b/crypto/rmd160.c index c5fe4034b153..890d29e46a76 100644 --- a/crypto/rmd160.c +++ b/crypto/rmd160.c @@ -9,18 +9,14 @@ * Copyright (c) 2008 Adrian-Ken Rueegsegger */ #include -#include +#include #include -#include -#include -#include - +#include #include "ripemd.h" struct rmd160_ctx { u64 byte_count; u32 state[5]; - __le32 buffer[16]; }; #define K1 RMD_K1 @@ -265,72 +261,59 @@ static int rmd160_init(struct shash_desc *desc) rctx->state[3] = RMD_H3; rctx->state[4] = RMD_H4; - memset(rctx->buffer, 0, sizeof(rctx->buffer)); - return 0; } static int rmd160_update(struct shash_desc *desc, const u8 *data, unsigned int len) { + int remain = len - round_down(len, RMD160_BLOCK_SIZE); struct rmd160_ctx *rctx = shash_desc_ctx(desc); - const u32 avail = sizeof(rctx->buffer) - (rctx->byte_count & 0x3f); - - rctx->byte_count += len; + __le32 buffer[RMD160_BLOCK_SIZE / 4]; - /* Enough space in buffer? If so copy and we're done */ - if (avail > len) { - memcpy((char *)rctx->buffer + (sizeof(rctx->buffer) - avail), - data, len); - goto out; - } - - memcpy((char *)rctx->buffer + (sizeof(rctx->buffer) - avail), - data, avail); + rctx->byte_count += len - remain; - rmd160_transform(rctx->state, rctx->buffer); - data += avail; - len -= avail; - - while (len >= sizeof(rctx->buffer)) { - memcpy(rctx->buffer, data, sizeof(rctx->buffer)); - rmd160_transform(rctx->state, rctx->buffer); - data += sizeof(rctx->buffer); - len -= sizeof(rctx->buffer); - } + do { + memcpy(buffer, data, sizeof(buffer)); + rmd160_transform(rctx->state, buffer); + data += sizeof(buffer); + len -= sizeof(buffer); + } while (len >= sizeof(buffer)); - memcpy(rctx->buffer, data, len); - -out: - return 0; + memzero_explicit(buffer, sizeof(buffer)); + return remain; } /* Add padding and return the message digest. */ -static int rmd160_final(struct shash_desc *desc, u8 *out) +static int rmd160_finup(struct shash_desc *desc, const u8 *src, + unsigned int len, u8 *out) { + unsigned int bit_offset = RMD160_BLOCK_SIZE / 8 - 1; struct rmd160_ctx *rctx = shash_desc_ctx(desc); - u32 i, index, padlen; - __le64 bits; + union { + __le64 l64[RMD160_BLOCK_SIZE / 4]; + __le32 l32[RMD160_BLOCK_SIZE / 2]; + u8 u8[RMD160_BLOCK_SIZE * 2]; + } block = {}; __le32 *dst = (__le32 *)out; - static const u8 padding[64] = { 0x80, }; - - bits = cpu_to_le64(rctx->byte_count << 3); - - /* Pad out to 56 mod 64 */ - index = rctx->byte_count & 0x3f; - padlen = (index < 56) ? (56 - index) : ((64+56) - index); - rmd160_update(desc, padding, padlen); + u32 i; - /* Append length */ - rmd160_update(desc, (const u8 *)&bits, sizeof(bits)); + rctx->byte_count += len; + if (len >= bit_offset * 8) + bit_offset += RMD160_BLOCK_SIZE / 8; + memcpy(&block, src, len); + block.u8[len] = 0x80; + block.l64[bit_offset] = cpu_to_le64(rctx->byte_count << 3); + + rmd160_transform(rctx->state, block.l32); + if (bit_offset > RMD160_BLOCK_SIZE / 8) + rmd160_transform(rctx->state, + block.l32 + RMD160_BLOCK_SIZE / 4); + memzero_explicit(&block, sizeof(block)); /* Store state in digest */ for (i = 0; i < 5; i++) dst[i] = cpu_to_le32p(&rctx->state[i]); - - /* Wipe context */ - memset(rctx, 0, sizeof(*rctx)); - return 0; } @@ -338,11 +321,12 @@ static struct shash_alg alg = { .digestsize = RMD160_DIGEST_SIZE, .init = rmd160_init, .update = rmd160_update, - .final = rmd160_final, + .finup = rmd160_finup, .descsize = sizeof(struct rmd160_ctx), .base = { .cra_name = "rmd160", .cra_driver_name = "rmd160-generic", + .cra_flags = CRYPTO_AHASH_ALG_BLOCK_ONLY, .cra_blocksize = RMD160_BLOCK_SIZE, .cra_module = THIS_MODULE, } -- 2.50.1 From 0e2392b6513cb138e2915ad08e33ac029358e957 Mon Sep 17 00:00:00 2001 From: Herbert Xu Date: Tue, 29 Apr 2025 16:43:22 +0800 Subject: [PATCH 07/16] crypto: streebog - Use API partial block handling Use the Crypto API partial block handling. Signed-off-by: Herbert Xu --- crypto/streebog_generic.c | 71 ++++++++++++++------------------------- include/crypto/streebog.h | 5 --- 2 files changed, 25 insertions(+), 51 deletions(-) diff --git a/crypto/streebog_generic.c b/crypto/streebog_generic.c index dc625ffc54ad..99c07e0c10ba 100644 --- a/crypto/streebog_generic.c +++ b/crypto/streebog_generic.c @@ -13,9 +13,10 @@ */ #include -#include -#include #include +#include +#include +#include static const struct streebog_uint512 buffer0 = { { 0, 0, 0, 0, 0, 0, 0, 0 @@ -919,17 +920,6 @@ static int streebog_init(struct shash_desc *desc) return 0; } -static void streebog_pad(struct streebog_state *ctx) -{ - if (ctx->fillsize >= STREEBOG_BLOCK_SIZE) - return; - - memset(ctx->buffer + ctx->fillsize, 0, - sizeof(ctx->buffer) - ctx->fillsize); - - ctx->buffer[ctx->fillsize] = 1; -} - static void streebog_add512(const struct streebog_uint512 *x, const struct streebog_uint512 *y, struct streebog_uint512 *r) @@ -984,16 +974,23 @@ static void streebog_stage2(struct streebog_state *ctx, const u8 *data) streebog_add512(&ctx->Sigma, &m, &ctx->Sigma); } -static void streebog_stage3(struct streebog_state *ctx) +static void streebog_stage3(struct streebog_state *ctx, const u8 *src, + unsigned int len) { struct streebog_uint512 buf = { { 0 } }; + union { + u8 buffer[STREEBOG_BLOCK_SIZE]; + struct streebog_uint512 m; + } u = {}; - buf.qword[0] = cpu_to_le64(ctx->fillsize << 3); - streebog_pad(ctx); + buf.qword[0] = cpu_to_le64(len << 3); + memcpy(u.buffer, src, len); + u.buffer[len] = 1; - streebog_g(&ctx->h, &ctx->N, &ctx->m); + streebog_g(&ctx->h, &ctx->N, &u.m); streebog_add512(&ctx->N, &buf, &ctx->N); - streebog_add512(&ctx->Sigma, &ctx->m, &ctx->Sigma); + streebog_add512(&ctx->Sigma, &u.m, &ctx->Sigma); + memzero_explicit(&u, sizeof(u)); streebog_g(&ctx->h, &buffer0, &ctx->N); streebog_g(&ctx->h, &buffer0, &ctx->Sigma); memcpy(&ctx->hash, &ctx->h, sizeof(struct streebog_uint512)); @@ -1003,42 +1000,22 @@ static int streebog_update(struct shash_desc *desc, const u8 *data, unsigned int len) { struct streebog_state *ctx = shash_desc_ctx(desc); - size_t chunksize; - if (ctx->fillsize) { - chunksize = STREEBOG_BLOCK_SIZE - ctx->fillsize; - if (chunksize > len) - chunksize = len; - memcpy(&ctx->buffer[ctx->fillsize], data, chunksize); - ctx->fillsize += chunksize; - len -= chunksize; - data += chunksize; - - if (ctx->fillsize == STREEBOG_BLOCK_SIZE) { - streebog_stage2(ctx, ctx->buffer); - ctx->fillsize = 0; - } - } - - while (len >= STREEBOG_BLOCK_SIZE) { + do { streebog_stage2(ctx, data); data += STREEBOG_BLOCK_SIZE; len -= STREEBOG_BLOCK_SIZE; - } + } while (len >= STREEBOG_BLOCK_SIZE); - if (len) { - memcpy(&ctx->buffer, data, len); - ctx->fillsize = len; - } - return 0; + return len; } -static int streebog_final(struct shash_desc *desc, u8 *digest) +static int streebog_finup(struct shash_desc *desc, const u8 *src, + unsigned int len, u8 *digest) { struct streebog_state *ctx = shash_desc_ctx(desc); - streebog_stage3(ctx); - ctx->fillsize = 0; + streebog_stage3(ctx, src, len); if (crypto_shash_digestsize(desc->tfm) == STREEBOG256_DIGEST_SIZE) memcpy(digest, &ctx->hash.qword[4], STREEBOG256_DIGEST_SIZE); else @@ -1050,11 +1027,12 @@ static struct shash_alg algs[2] = { { .digestsize = STREEBOG256_DIGEST_SIZE, .init = streebog_init, .update = streebog_update, - .final = streebog_final, + .finup = streebog_finup, .descsize = sizeof(struct streebog_state), .base = { .cra_name = "streebog256", .cra_driver_name = "streebog256-generic", + .cra_flags = CRYPTO_AHASH_ALG_BLOCK_ONLY, .cra_blocksize = STREEBOG_BLOCK_SIZE, .cra_module = THIS_MODULE, }, @@ -1062,11 +1040,12 @@ static struct shash_alg algs[2] = { { .digestsize = STREEBOG512_DIGEST_SIZE, .init = streebog_init, .update = streebog_update, - .final = streebog_final, + .finup = streebog_finup, .descsize = sizeof(struct streebog_state), .base = { .cra_name = "streebog512", .cra_driver_name = "streebog512-generic", + .cra_flags = CRYPTO_AHASH_ALG_BLOCK_ONLY, .cra_blocksize = STREEBOG_BLOCK_SIZE, .cra_module = THIS_MODULE, } diff --git a/include/crypto/streebog.h b/include/crypto/streebog.h index cae1b4a01971..570f720a113b 100644 --- a/include/crypto/streebog.h +++ b/include/crypto/streebog.h @@ -23,15 +23,10 @@ struct streebog_uint512 { }; struct streebog_state { - union { - u8 buffer[STREEBOG_BLOCK_SIZE]; - struct streebog_uint512 m; - }; struct streebog_uint512 hash; struct streebog_uint512 h; struct streebog_uint512 N; struct streebog_uint512 Sigma; - size_t fillsize; }; #endif /* !_CRYPTO_STREEBOG_H_ */ -- 2.50.1 From 89490e6b80c53bf7783fe183a2fda8d0944f52d2 Mon Sep 17 00:00:00 2001 From: Herbert Xu Date: Tue, 29 Apr 2025 16:49:32 +0800 Subject: [PATCH 08/16] crypto: s390/hmac - Extend hash length counters to 128 bits As sha512 requires 128-bit counters, extend the hash length counters to that length. Previously they were just 32 bits which means that a >4G sha256 hash would be incorrect. Signed-off-by: Herbert Xu --- arch/s390/crypto/hmac_s390.c | 25 ++++++++++++++----------- 1 file changed, 14 insertions(+), 11 deletions(-) diff --git a/arch/s390/crypto/hmac_s390.c b/arch/s390/crypto/hmac_s390.c index bba9a818dfdc..e6edf1013228 100644 --- a/arch/s390/crypto/hmac_s390.c +++ b/arch/s390/crypto/hmac_s390.c @@ -72,23 +72,23 @@ struct s390_kmac_sha2_ctx { u8 param[MAX_DIGEST_SIZE + MAX_IMBL_SIZE + MAX_BLOCK_SIZE]; union s390_kmac_gr0 gr0; u8 buf[MAX_BLOCK_SIZE]; - unsigned int buflen; + u64 buflen[2]; }; /* * kmac_sha2_set_imbl - sets the input message bit-length based on the blocksize */ -static inline void kmac_sha2_set_imbl(u8 *param, unsigned int buflen, - unsigned int blocksize) +static inline void kmac_sha2_set_imbl(u8 *param, u64 buflen_lo, + u64 buflen_hi, unsigned int blocksize) { u8 *imbl = param + SHA2_IMBL_OFFSET(blocksize); switch (blocksize) { case SHA256_BLOCK_SIZE: - *(u64 *)imbl = (u64)buflen * BITS_PER_BYTE; + *(u64 *)imbl = buflen_lo * BITS_PER_BYTE; break; case SHA512_BLOCK_SIZE: - *(u128 *)imbl = (u128)buflen * BITS_PER_BYTE; + *(u128 *)imbl = (((u128)buflen_hi << 64) + buflen_lo) << 3; break; default: break; @@ -176,7 +176,8 @@ static int s390_hmac_sha2_init(struct shash_desc *desc) memcpy(ctx->param + SHA2_KEY_OFFSET(bs), tfm_ctx->key, bs); - ctx->buflen = 0; + ctx->buflen[0] = 0; + ctx->buflen[1] = 0; ctx->gr0.reg = 0; switch (crypto_shash_digestsize(desc->tfm)) { case SHA224_DIGEST_SIZE: @@ -206,8 +207,10 @@ static int s390_hmac_sha2_update(struct shash_desc *desc, unsigned int offset, n; /* check current buffer */ - offset = ctx->buflen % bs; - ctx->buflen += len; + offset = ctx->buflen[0] % bs; + ctx->buflen[0] += len; + if (ctx->buflen[0] < len) + ctx->buflen[1]++; if (offset + len < bs) goto store; @@ -243,8 +246,8 @@ static int s390_hmac_sha2_final(struct shash_desc *desc, u8 *out) unsigned int bs = crypto_shash_blocksize(desc->tfm); ctx->gr0.iimp = 0; - kmac_sha2_set_imbl(ctx->param, ctx->buflen, bs); - _cpacf_kmac(&ctx->gr0.reg, ctx->param, ctx->buf, ctx->buflen % bs); + kmac_sha2_set_imbl(ctx->param, ctx->buflen[0], ctx->buflen[1], bs); + _cpacf_kmac(&ctx->gr0.reg, ctx->param, ctx->buf, ctx->buflen[0] % bs); memcpy(out, ctx->param, crypto_shash_digestsize(desc->tfm)); return 0; @@ -262,7 +265,7 @@ static int s390_hmac_sha2_digest(struct shash_desc *desc, return rc; ctx->gr0.iimp = 0; - kmac_sha2_set_imbl(ctx->param, len, + kmac_sha2_set_imbl(ctx->param, len, 0, crypto_shash_blocksize(desc->tfm)); _cpacf_kmac(&ctx->gr0.reg, ctx->param, data, len); memcpy(out, ctx->param, ds); -- 2.50.1 From ef93f1562803cd7bb8159e3abedaf7f47dce4e35 Mon Sep 17 00:00:00 2001 From: Herbert Xu Date: Wed, 30 Apr 2025 16:17:02 +0800 Subject: [PATCH 09/16] Revert "crypto: run initcalls for generic implementations earlier" This reverts commit c4741b23059794bd99beef0f700103b0d983b3fd. Crypto API self-tests no longer run at registration time and now occur either at late_initcall or upon the first use. Therefore the premise of the above commit no longer exists. Revert it and subsequent additions of subsys_initcall and arch_initcall. Note that lib/crypto calls will stay at subsys_initcall (or rather downgraded from arch_initcall) because they may need to occur before Crypto API registration. Signed-off-by: Herbert Xu --- arch/arm/crypto/aes-neonbs-glue.c | 2 +- arch/arm/lib/crypto/chacha-glue.c | 2 +- arch/arm/lib/crypto/poly1305-glue.c | 2 +- arch/arm/lib/crypto/sha256.c | 2 +- arch/arm64/lib/crypto/chacha-neon-glue.c | 2 +- arch/arm64/lib/crypto/poly1305-glue.c | 2 +- arch/arm64/lib/crypto/sha256.c | 2 +- arch/powerpc/lib/crypto/chacha-p10-glue.c | 2 +- arch/powerpc/lib/crypto/poly1305-p10-glue.c | 2 +- arch/riscv/lib/crypto/chacha-riscv64-glue.c | 2 +- arch/riscv/lib/crypto/sha256.c | 2 +- arch/s390/lib/crypto/sha256.c | 2 +- arch/sparc/lib/crypto/sha256.c | 2 +- arch/x86/lib/crypto/chacha_glue.c | 2 +- arch/x86/lib/crypto/poly1305_glue.c | 2 +- arch/x86/lib/crypto/sha256.c | 2 +- crypto/842.c | 2 +- crypto/adiantum.c | 2 +- crypto/aegis128-core.c | 2 +- crypto/aes_generic.c | 2 +- crypto/algboss.c | 8 +------- crypto/ansi_cprng.c | 2 +- crypto/anubis.c | 2 +- crypto/arc4.c | 2 +- crypto/aria_generic.c | 2 +- crypto/authenc.c | 2 +- crypto/authencesn.c | 2 +- crypto/blake2b_generic.c | 2 +- crypto/blowfish_generic.c | 2 +- crypto/camellia_generic.c | 2 +- crypto/cast5_generic.c | 2 +- crypto/cast6_generic.c | 2 +- crypto/cbc.c | 2 +- crypto/ccm.c | 2 +- crypto/chacha.c | 2 +- crypto/chacha20poly1305.c | 2 +- crypto/cmac.c | 2 +- crypto/crc32_generic.c | 2 +- crypto/crc32c_generic.c | 2 +- crypto/cryptd.c | 2 +- crypto/crypto_null.c | 2 +- crypto/ctr.c | 2 +- crypto/cts.c | 2 +- crypto/curve25519-generic.c | 2 +- crypto/deflate.c | 2 +- crypto/des_generic.c | 2 +- crypto/dh.c | 2 +- crypto/drbg.c | 2 +- crypto/ecb.c | 2 +- crypto/ecdh.c | 2 +- crypto/ecdsa.c | 2 +- crypto/echainiv.c | 2 +- crypto/essiv.c | 2 +- crypto/fcrypt.c | 2 +- crypto/fips.c | 2 +- crypto/gcm.c | 2 +- crypto/ghash-generic.c | 2 +- crypto/hctr2.c | 2 +- crypto/hmac.c | 2 +- crypto/khazad.c | 2 +- crypto/krb5enc.c | 2 +- crypto/lrw.c | 2 +- crypto/lz4.c | 2 +- crypto/lz4hc.c | 2 +- crypto/lzo-rle.c | 2 +- crypto/lzo.c | 2 +- crypto/md4.c | 2 +- crypto/md5.c | 2 +- crypto/michael_mic.c | 2 +- crypto/nhpoly1305.c | 2 +- crypto/pcbc.c | 2 +- crypto/pcrypt.c | 2 +- crypto/polyval-generic.c | 2 +- crypto/rmd160.c | 2 +- crypto/rsa.c | 2 +- crypto/seed.c | 2 +- crypto/seqiv.c | 2 +- crypto/serpent_generic.c | 2 +- crypto/sha1_generic.c | 2 +- crypto/sha256.c | 2 +- crypto/sha512_generic.c | 2 +- crypto/sm3_generic.c | 2 +- crypto/sm4_generic.c | 2 +- crypto/streebog_generic.c | 2 +- crypto/tea.c | 2 +- crypto/twofish_generic.c | 2 +- crypto/wp512.c | 2 +- crypto/xcbc.c | 2 +- crypto/xctr.c | 2 +- crypto/xts.c | 2 +- crypto/xxhash_generic.c | 2 +- crypto/zstd.c | 2 +- 92 files changed, 92 insertions(+), 98 deletions(-) diff --git a/arch/arm/crypto/aes-neonbs-glue.c b/arch/arm/crypto/aes-neonbs-glue.c index 95418df97fb4..c60104dc1585 100644 --- a/arch/arm/crypto/aes-neonbs-glue.c +++ b/arch/arm/crypto/aes-neonbs-glue.c @@ -407,5 +407,5 @@ static int __init aes_init(void) return crypto_register_skciphers(aes_algs, ARRAY_SIZE(aes_algs)); } -late_initcall(aes_init); +module_init(aes_init); module_exit(aes_exit); diff --git a/arch/arm/lib/crypto/chacha-glue.c b/arch/arm/lib/crypto/chacha-glue.c index 12afb40cf1ff..1e28736834a0 100644 --- a/arch/arm/lib/crypto/chacha-glue.c +++ b/arch/arm/lib/crypto/chacha-glue.c @@ -122,7 +122,7 @@ static int __init chacha_arm_mod_init(void) } return 0; } -arch_initcall(chacha_arm_mod_init); +subsys_initcall(chacha_arm_mod_init); static void __exit chacha_arm_mod_exit(void) { diff --git a/arch/arm/lib/crypto/poly1305-glue.c b/arch/arm/lib/crypto/poly1305-glue.c index 91da42b26d9c..2603b0771f2c 100644 --- a/arch/arm/lib/crypto/poly1305-glue.c +++ b/arch/arm/lib/crypto/poly1305-glue.c @@ -69,7 +69,7 @@ static int __init arm_poly1305_mod_init(void) static_branch_enable(&have_neon); return 0; } -arch_initcall(arm_poly1305_mod_init); +subsys_initcall(arm_poly1305_mod_init); static void __exit arm_poly1305_mod_exit(void) { diff --git a/arch/arm/lib/crypto/sha256.c b/arch/arm/lib/crypto/sha256.c index 3a8dfc304807..81612b7b9043 100644 --- a/arch/arm/lib/crypto/sha256.c +++ b/arch/arm/lib/crypto/sha256.c @@ -53,7 +53,7 @@ static int __init sha256_arm_mod_init(void) } return 0; } -arch_initcall(sha256_arm_mod_init); +subsys_initcall(sha256_arm_mod_init); static void __exit sha256_arm_mod_exit(void) { diff --git a/arch/arm64/lib/crypto/chacha-neon-glue.c b/arch/arm64/lib/crypto/chacha-neon-glue.c index 14a2836eff61..2b0de97a6daf 100644 --- a/arch/arm64/lib/crypto/chacha-neon-glue.c +++ b/arch/arm64/lib/crypto/chacha-neon-glue.c @@ -104,7 +104,7 @@ static int __init chacha_simd_mod_init(void) static_branch_enable(&have_neon); return 0; } -arch_initcall(chacha_simd_mod_init); +subsys_initcall(chacha_simd_mod_init); static void __exit chacha_simd_mod_exit(void) { diff --git a/arch/arm64/lib/crypto/poly1305-glue.c b/arch/arm64/lib/crypto/poly1305-glue.c index 681c26557336..6a661cf04821 100644 --- a/arch/arm64/lib/crypto/poly1305-glue.c +++ b/arch/arm64/lib/crypto/poly1305-glue.c @@ -62,7 +62,7 @@ static int __init neon_poly1305_mod_init(void) static_branch_enable(&have_neon); return 0; } -arch_initcall(neon_poly1305_mod_init); +subsys_initcall(neon_poly1305_mod_init); static void __exit neon_poly1305_mod_exit(void) { diff --git a/arch/arm64/lib/crypto/sha256.c b/arch/arm64/lib/crypto/sha256.c index 2bd413c586d2..354fc402c6d3 100644 --- a/arch/arm64/lib/crypto/sha256.c +++ b/arch/arm64/lib/crypto/sha256.c @@ -64,7 +64,7 @@ static int __init sha256_arm64_mod_init(void) } return 0; } -arch_initcall(sha256_arm64_mod_init); +subsys_initcall(sha256_arm64_mod_init); static void __exit sha256_arm64_mod_exit(void) { diff --git a/arch/powerpc/lib/crypto/chacha-p10-glue.c b/arch/powerpc/lib/crypto/chacha-p10-glue.c index 351ed409f9b2..51daeaf5d26e 100644 --- a/arch/powerpc/lib/crypto/chacha-p10-glue.c +++ b/arch/powerpc/lib/crypto/chacha-p10-glue.c @@ -87,7 +87,7 @@ static int __init chacha_p10_init(void) static_branch_enable(&have_p10); return 0; } -arch_initcall(chacha_p10_init); +subsys_initcall(chacha_p10_init); static void __exit chacha_p10_exit(void) { diff --git a/arch/powerpc/lib/crypto/poly1305-p10-glue.c b/arch/powerpc/lib/crypto/poly1305-p10-glue.c index 50ac802220e0..16c2a8316696 100644 --- a/arch/powerpc/lib/crypto/poly1305-p10-glue.c +++ b/arch/powerpc/lib/crypto/poly1305-p10-glue.c @@ -76,7 +76,7 @@ static int __init poly1305_p10_init(void) static_branch_enable(&have_p10); return 0; } -arch_initcall(poly1305_p10_init); +subsys_initcall(poly1305_p10_init); static void __exit poly1305_p10_exit(void) { diff --git a/arch/riscv/lib/crypto/chacha-riscv64-glue.c b/arch/riscv/lib/crypto/chacha-riscv64-glue.c index afc4e3be3cac..1740e1ca3a94 100644 --- a/arch/riscv/lib/crypto/chacha-riscv64-glue.c +++ b/arch/riscv/lib/crypto/chacha-riscv64-glue.c @@ -62,7 +62,7 @@ static int __init riscv64_chacha_mod_init(void) static_branch_enable(&use_zvkb); return 0; } -arch_initcall(riscv64_chacha_mod_init); +subsys_initcall(riscv64_chacha_mod_init); static void __exit riscv64_chacha_mod_exit(void) { diff --git a/arch/riscv/lib/crypto/sha256.c b/arch/riscv/lib/crypto/sha256.c index 18b84030f0b3..c8065c70337b 100644 --- a/arch/riscv/lib/crypto/sha256.c +++ b/arch/riscv/lib/crypto/sha256.c @@ -50,7 +50,7 @@ static int __init riscv64_sha256_mod_init(void) static_branch_enable(&have_extensions); return 0; } -arch_initcall(riscv64_sha256_mod_init); +subsys_initcall(riscv64_sha256_mod_init); static void __exit riscv64_sha256_mod_exit(void) { diff --git a/arch/s390/lib/crypto/sha256.c b/arch/s390/lib/crypto/sha256.c index 50c592ce7a5d..f1d73ea1eef6 100644 --- a/arch/s390/lib/crypto/sha256.c +++ b/arch/s390/lib/crypto/sha256.c @@ -36,7 +36,7 @@ static int __init sha256_s390_mod_init(void) static_branch_enable(&have_cpacf_sha256); return 0; } -arch_initcall(sha256_s390_mod_init); +subsys_initcall(sha256_s390_mod_init); static void __exit sha256_s390_mod_exit(void) { diff --git a/arch/sparc/lib/crypto/sha256.c b/arch/sparc/lib/crypto/sha256.c index 6f118a23d210..0394c2b72f9e 100644 --- a/arch/sparc/lib/crypto/sha256.c +++ b/arch/sparc/lib/crypto/sha256.c @@ -53,7 +53,7 @@ static int __init sha256_sparc64_mod_init(void) pr_info("Using sparc64 sha256 opcode optimized SHA-256/SHA-224 implementation\n"); return 0; } -arch_initcall(sha256_sparc64_mod_init); +subsys_initcall(sha256_sparc64_mod_init); static void __exit sha256_sparc64_mod_exit(void) { diff --git a/arch/x86/lib/crypto/chacha_glue.c b/arch/x86/lib/crypto/chacha_glue.c index 59bf63c00072..94fcefbc8827 100644 --- a/arch/x86/lib/crypto/chacha_glue.c +++ b/arch/x86/lib/crypto/chacha_glue.c @@ -174,7 +174,7 @@ static int __init chacha_simd_mod_init(void) } return 0; } -arch_initcall(chacha_simd_mod_init); +subsys_initcall(chacha_simd_mod_init); static void __exit chacha_simd_mod_exit(void) { diff --git a/arch/x86/lib/crypto/poly1305_glue.c b/arch/x86/lib/crypto/poly1305_glue.c index f799828c5809..b7e78a583e07 100644 --- a/arch/x86/lib/crypto/poly1305_glue.c +++ b/arch/x86/lib/crypto/poly1305_glue.c @@ -117,7 +117,7 @@ static int __init poly1305_simd_mod_init(void) static_branch_enable(&poly1305_use_avx512); return 0; } -arch_initcall(poly1305_simd_mod_init); +subsys_initcall(poly1305_simd_mod_init); static void __exit poly1305_simd_mod_exit(void) { diff --git a/arch/x86/lib/crypto/sha256.c b/arch/x86/lib/crypto/sha256.c index 47865b5cd94b..ea17514cb0f3 100644 --- a/arch/x86/lib/crypto/sha256.c +++ b/arch/x86/lib/crypto/sha256.c @@ -63,7 +63,7 @@ static int __init sha256_x86_mod_init(void) static_branch_enable(&have_sha256_x86); return 0; } -arch_initcall(sha256_x86_mod_init); +subsys_initcall(sha256_x86_mod_init); static void __exit sha256_x86_mod_exit(void) { diff --git a/crypto/842.c b/crypto/842.c index 881945d44328..8c257c40e2b9 100644 --- a/crypto/842.c +++ b/crypto/842.c @@ -70,7 +70,7 @@ static int __init crypto842_mod_init(void) { return crypto_register_scomp(&scomp); } -subsys_initcall(crypto842_mod_init); +module_init(crypto842_mod_init); static void __exit crypto842_mod_exit(void) { diff --git a/crypto/adiantum.c b/crypto/adiantum.c index c3ef583598b4..a6bca877c3c7 100644 --- a/crypto/adiantum.c +++ b/crypto/adiantum.c @@ -639,7 +639,7 @@ static void __exit adiantum_module_exit(void) crypto_unregister_template(&adiantum_tmpl); } -subsys_initcall(adiantum_module_init); +module_init(adiantum_module_init); module_exit(adiantum_module_exit); MODULE_DESCRIPTION("Adiantum length-preserving encryption mode"); diff --git a/crypto/aegis128-core.c b/crypto/aegis128-core.c index 72f6ee1345ef..ca80d861345d 100644 --- a/crypto/aegis128-core.c +++ b/crypto/aegis128-core.c @@ -566,7 +566,7 @@ static void __exit crypto_aegis128_module_exit(void) crypto_unregister_aead(&crypto_aegis128_alg_generic); } -subsys_initcall(crypto_aegis128_module_init); +module_init(crypto_aegis128_module_init); module_exit(crypto_aegis128_module_exit); MODULE_LICENSE("GPL"); diff --git a/crypto/aes_generic.c b/crypto/aes_generic.c index 3c66d425c97b..85d2e78c8ef2 100644 --- a/crypto/aes_generic.c +++ b/crypto/aes_generic.c @@ -1311,7 +1311,7 @@ static void __exit aes_fini(void) crypto_unregister_alg(&aes_alg); } -subsys_initcall(aes_init); +module_init(aes_init); module_exit(aes_fini); MODULE_DESCRIPTION("Rijndael (AES) Cipher Algorithm"); diff --git a/crypto/algboss.c b/crypto/algboss.c index a20926bfd34e..ef5c73780fc7 100644 --- a/crypto/algboss.c +++ b/crypto/algboss.c @@ -247,13 +247,7 @@ static void __exit cryptomgr_exit(void) BUG_ON(err); } -/* - * This is arch_initcall() so that the crypto self-tests are run on algorithms - * registered early by subsys_initcall(). subsys_initcall() is needed for - * generic implementations so that they're available for comparison tests when - * other implementations are registered later by module_init(). - */ -arch_initcall(cryptomgr_init); +module_init(cryptomgr_init); module_exit(cryptomgr_exit); MODULE_LICENSE("GPL"); diff --git a/crypto/ansi_cprng.c b/crypto/ansi_cprng.c index 64f57c4c4b06..153523ce6076 100644 --- a/crypto/ansi_cprng.c +++ b/crypto/ansi_cprng.c @@ -467,7 +467,7 @@ MODULE_DESCRIPTION("Software Pseudo Random Number Generator"); MODULE_AUTHOR("Neil Horman "); module_param(dbg, int, 0); MODULE_PARM_DESC(dbg, "Boolean to enable debugging (0/1 == off/on)"); -subsys_initcall(prng_mod_init); +module_init(prng_mod_init); module_exit(prng_mod_fini); MODULE_ALIAS_CRYPTO("stdrng"); MODULE_ALIAS_CRYPTO("ansi_cprng"); diff --git a/crypto/anubis.c b/crypto/anubis.c index 886e7c913688..4268c3833baa 100644 --- a/crypto/anubis.c +++ b/crypto/anubis.c @@ -694,7 +694,7 @@ static void __exit anubis_mod_fini(void) crypto_unregister_alg(&anubis_alg); } -subsys_initcall(anubis_mod_init); +module_init(anubis_mod_init); module_exit(anubis_mod_fini); MODULE_LICENSE("GPL"); diff --git a/crypto/arc4.c b/crypto/arc4.c index 1a4825c97c5a..1608018111d0 100644 --- a/crypto/arc4.c +++ b/crypto/arc4.c @@ -73,7 +73,7 @@ static void __exit arc4_exit(void) crypto_unregister_lskcipher(&arc4_alg); } -subsys_initcall(arc4_init); +module_init(arc4_init); module_exit(arc4_exit); MODULE_LICENSE("GPL"); diff --git a/crypto/aria_generic.c b/crypto/aria_generic.c index bd359d3313c2..faa7900383f6 100644 --- a/crypto/aria_generic.c +++ b/crypto/aria_generic.c @@ -304,7 +304,7 @@ static void __exit aria_fini(void) crypto_unregister_alg(&aria_alg); } -subsys_initcall(aria_init); +module_init(aria_init); module_exit(aria_fini); MODULE_DESCRIPTION("ARIA Cipher Algorithm"); diff --git a/crypto/authenc.c b/crypto/authenc.c index 3aaf3ab4e360..9521ae2f112e 100644 --- a/crypto/authenc.c +++ b/crypto/authenc.c @@ -451,7 +451,7 @@ static void __exit crypto_authenc_module_exit(void) crypto_unregister_template(&crypto_authenc_tmpl); } -subsys_initcall(crypto_authenc_module_init); +module_init(crypto_authenc_module_init); module_exit(crypto_authenc_module_exit); MODULE_LICENSE("GPL"); diff --git a/crypto/authencesn.c b/crypto/authencesn.c index 2cc933e2f790..b1c78313cbc1 100644 --- a/crypto/authencesn.c +++ b/crypto/authencesn.c @@ -465,7 +465,7 @@ static void __exit crypto_authenc_esn_module_exit(void) crypto_unregister_template(&crypto_authenc_esn_tmpl); } -subsys_initcall(crypto_authenc_esn_module_init); +module_init(crypto_authenc_esn_module_init); module_exit(crypto_authenc_esn_module_exit); MODULE_LICENSE("GPL"); diff --git a/crypto/blake2b_generic.c b/crypto/blake2b_generic.c index 6fa38965a493..60f056217510 100644 --- a/crypto/blake2b_generic.c +++ b/crypto/blake2b_generic.c @@ -176,7 +176,7 @@ static void __exit blake2b_mod_fini(void) crypto_unregister_shashes(blake2b_algs, ARRAY_SIZE(blake2b_algs)); } -subsys_initcall(blake2b_mod_init); +module_init(blake2b_mod_init); module_exit(blake2b_mod_fini); MODULE_AUTHOR("David Sterba "); diff --git a/crypto/blowfish_generic.c b/crypto/blowfish_generic.c index 0146bc762c09..f3c5f9b09850 100644 --- a/crypto/blowfish_generic.c +++ b/crypto/blowfish_generic.c @@ -124,7 +124,7 @@ static void __exit blowfish_mod_fini(void) crypto_unregister_alg(&alg); } -subsys_initcall(blowfish_mod_init); +module_init(blowfish_mod_init); module_exit(blowfish_mod_fini); MODULE_LICENSE("GPL"); diff --git a/crypto/camellia_generic.c b/crypto/camellia_generic.c index 197fcf3abc89..ee4336a04b93 100644 --- a/crypto/camellia_generic.c +++ b/crypto/camellia_generic.c @@ -1064,7 +1064,7 @@ static void __exit camellia_fini(void) crypto_unregister_alg(&camellia_alg); } -subsys_initcall(camellia_init); +module_init(camellia_init); module_exit(camellia_fini); MODULE_DESCRIPTION("Camellia Cipher Algorithm"); diff --git a/crypto/cast5_generic.c b/crypto/cast5_generic.c index f3e57775fa02..f68330793e0c 100644 --- a/crypto/cast5_generic.c +++ b/crypto/cast5_generic.c @@ -531,7 +531,7 @@ static void __exit cast5_mod_fini(void) crypto_unregister_alg(&alg); } -subsys_initcall(cast5_mod_init); +module_init(cast5_mod_init); module_exit(cast5_mod_fini); MODULE_LICENSE("GPL"); diff --git a/crypto/cast6_generic.c b/crypto/cast6_generic.c index 11b725b12f27..4c08c42646f0 100644 --- a/crypto/cast6_generic.c +++ b/crypto/cast6_generic.c @@ -271,7 +271,7 @@ static void __exit cast6_mod_fini(void) crypto_unregister_alg(&alg); } -subsys_initcall(cast6_mod_init); +module_init(cast6_mod_init); module_exit(cast6_mod_fini); MODULE_LICENSE("GPL"); diff --git a/crypto/cbc.c b/crypto/cbc.c index e81918ca68b7..ed3df6246765 100644 --- a/crypto/cbc.c +++ b/crypto/cbc.c @@ -179,7 +179,7 @@ static void __exit crypto_cbc_module_exit(void) crypto_unregister_template(&crypto_cbc_tmpl); } -subsys_initcall(crypto_cbc_module_init); +module_init(crypto_cbc_module_init); module_exit(crypto_cbc_module_exit); MODULE_LICENSE("GPL"); diff --git a/crypto/ccm.c b/crypto/ccm.c index f3f455e4908b..2ae929ffdef8 100644 --- a/crypto/ccm.c +++ b/crypto/ccm.c @@ -929,7 +929,7 @@ static void __exit crypto_ccm_module_exit(void) ARRAY_SIZE(crypto_ccm_tmpls)); } -subsys_initcall(crypto_ccm_module_init); +module_init(crypto_ccm_module_init); module_exit(crypto_ccm_module_exit); MODULE_LICENSE("GPL"); diff --git a/crypto/chacha.c b/crypto/chacha.c index 5103bc0b2881..28a8ad6197ab 100644 --- a/crypto/chacha.c +++ b/crypto/chacha.c @@ -243,7 +243,7 @@ static void __exit crypto_chacha_mod_fini(void) crypto_unregister_skciphers(algs, num_algs); } -subsys_initcall(crypto_chacha_mod_init); +module_init(crypto_chacha_mod_init); module_exit(crypto_chacha_mod_fini); MODULE_LICENSE("GPL"); diff --git a/crypto/chacha20poly1305.c b/crypto/chacha20poly1305.c index b29f66ba1e2f..b4b5a7198d84 100644 --- a/crypto/chacha20poly1305.c +++ b/crypto/chacha20poly1305.c @@ -476,7 +476,7 @@ static void __exit chacha20poly1305_module_exit(void) ARRAY_SIZE(rfc7539_tmpls)); } -subsys_initcall(chacha20poly1305_module_init); +module_init(chacha20poly1305_module_init); module_exit(chacha20poly1305_module_exit); MODULE_LICENSE("GPL"); diff --git a/crypto/cmac.c b/crypto/cmac.c index f297042a324b..1b03964abe00 100644 --- a/crypto/cmac.c +++ b/crypto/cmac.c @@ -251,7 +251,7 @@ static void __exit crypto_cmac_module_exit(void) crypto_unregister_template(&crypto_cmac_tmpl); } -subsys_initcall(crypto_cmac_module_init); +module_init(crypto_cmac_module_init); module_exit(crypto_cmac_module_exit); MODULE_LICENSE("GPL"); diff --git a/crypto/crc32_generic.c b/crypto/crc32_generic.c index 783a30b27398..cc371d42601f 100644 --- a/crypto/crc32_generic.c +++ b/crypto/crc32_generic.c @@ -172,7 +172,7 @@ static void __exit crc32_mod_fini(void) crypto_unregister_shashes(algs, num_algs); } -subsys_initcall(crc32_mod_init); +module_init(crc32_mod_init); module_exit(crc32_mod_fini); MODULE_AUTHOR("Alexander Boyko "); diff --git a/crypto/crc32c_generic.c b/crypto/crc32c_generic.c index b1a36d32dc50..e5377898414a 100644 --- a/crypto/crc32c_generic.c +++ b/crypto/crc32c_generic.c @@ -212,7 +212,7 @@ static void __exit crc32c_mod_fini(void) crypto_unregister_shashes(algs, num_algs); } -subsys_initcall(crc32c_mod_init); +module_init(crc32c_mod_init); module_exit(crc32c_mod_fini); MODULE_AUTHOR("Clay Haapala "); diff --git a/crypto/cryptd.c b/crypto/cryptd.c index 31d022d47f7a..5bb6f8d88cc2 100644 --- a/crypto/cryptd.c +++ b/crypto/cryptd.c @@ -1138,7 +1138,7 @@ static void __exit cryptd_exit(void) crypto_unregister_template(&cryptd_tmpl); } -subsys_initcall(cryptd_init); +module_init(cryptd_init); module_exit(cryptd_exit); MODULE_LICENSE("GPL"); diff --git a/crypto/crypto_null.c b/crypto/crypto_null.c index ced90f88ee07..5822753b0995 100644 --- a/crypto/crypto_null.c +++ b/crypto/crypto_null.c @@ -210,7 +210,7 @@ static void __exit crypto_null_mod_fini(void) crypto_unregister_skcipher(&skcipher_null); } -subsys_initcall(crypto_null_mod_init); +module_init(crypto_null_mod_init); module_exit(crypto_null_mod_fini); MODULE_LICENSE("GPL"); diff --git a/crypto/ctr.c b/crypto/ctr.c index 97a947b0a876..a388f0ceb3a0 100644 --- a/crypto/ctr.c +++ b/crypto/ctr.c @@ -350,7 +350,7 @@ static void __exit crypto_ctr_module_exit(void) ARRAY_SIZE(crypto_ctr_tmpls)); } -subsys_initcall(crypto_ctr_module_init); +module_init(crypto_ctr_module_init); module_exit(crypto_ctr_module_exit); MODULE_LICENSE("GPL"); diff --git a/crypto/cts.c b/crypto/cts.c index f5b42156b6c7..48898d5e24ff 100644 --- a/crypto/cts.c +++ b/crypto/cts.c @@ -402,7 +402,7 @@ static void __exit crypto_cts_module_exit(void) crypto_unregister_template(&crypto_cts_tmpl); } -subsys_initcall(crypto_cts_module_init); +module_init(crypto_cts_module_init); module_exit(crypto_cts_module_exit); MODULE_LICENSE("Dual BSD/GPL"); diff --git a/crypto/curve25519-generic.c b/crypto/curve25519-generic.c index 68a673262e04..f3e56e73c66c 100644 --- a/crypto/curve25519-generic.c +++ b/crypto/curve25519-generic.c @@ -82,7 +82,7 @@ static void __exit curve25519_exit(void) crypto_unregister_kpp(&curve25519_alg); } -subsys_initcall(curve25519_init); +module_init(curve25519_init); module_exit(curve25519_exit); MODULE_ALIAS_CRYPTO("curve25519"); diff --git a/crypto/deflate.c b/crypto/deflate.c index 0d2b64d96d6e..7eb1a5c44ee0 100644 --- a/crypto/deflate.c +++ b/crypto/deflate.c @@ -243,7 +243,7 @@ static void __exit deflate_mod_fini(void) crypto_acomp_free_streams(&deflate_streams); } -subsys_initcall(deflate_mod_init); +module_init(deflate_mod_init); module_exit(deflate_mod_fini); MODULE_LICENSE("GPL"); diff --git a/crypto/des_generic.c b/crypto/des_generic.c index 1274e18d3eb9..fce341400914 100644 --- a/crypto/des_generic.c +++ b/crypto/des_generic.c @@ -122,7 +122,7 @@ static void __exit des_generic_mod_fini(void) crypto_unregister_algs(des_algs, ARRAY_SIZE(des_algs)); } -subsys_initcall(des_generic_mod_init); +module_init(des_generic_mod_init); module_exit(des_generic_mod_fini); MODULE_LICENSE("GPL"); diff --git a/crypto/dh.c b/crypto/dh.c index afc0fd847761..8250eeeebd0f 100644 --- a/crypto/dh.c +++ b/crypto/dh.c @@ -920,7 +920,7 @@ static void __exit dh_exit(void) crypto_unregister_kpp(&dh); } -subsys_initcall(dh_init); +module_init(dh_init); module_exit(dh_exit); MODULE_ALIAS_CRYPTO("dh"); MODULE_LICENSE("GPL"); diff --git a/crypto/drbg.c b/crypto/drbg.c index f28dfc2511a2..dbe4c8bb5ceb 100644 --- a/crypto/drbg.c +++ b/crypto/drbg.c @@ -2132,7 +2132,7 @@ static void __exit drbg_exit(void) crypto_unregister_rngs(drbg_algs, (ARRAY_SIZE(drbg_cores) * 2)); } -subsys_initcall(drbg_init); +module_init(drbg_init); module_exit(drbg_exit); #ifndef CRYPTO_DRBG_HASH_STRING #define CRYPTO_DRBG_HASH_STRING "" diff --git a/crypto/ecb.c b/crypto/ecb.c index 95d7e972865a..cd1b20456dad 100644 --- a/crypto/ecb.c +++ b/crypto/ecb.c @@ -219,7 +219,7 @@ static void __exit crypto_ecb_module_exit(void) crypto_unregister_template(&crypto_ecb_tmpl); } -subsys_initcall(crypto_ecb_module_init); +module_init(crypto_ecb_module_init); module_exit(crypto_ecb_module_exit); MODULE_LICENSE("GPL"); diff --git a/crypto/ecdh.c b/crypto/ecdh.c index 72cfd1590156..9f0b93b3166d 100644 --- a/crypto/ecdh.c +++ b/crypto/ecdh.c @@ -240,7 +240,7 @@ static void __exit ecdh_exit(void) crypto_unregister_kpp(&ecdh_nist_p384); } -subsys_initcall(ecdh_init); +module_init(ecdh_init); module_exit(ecdh_exit); MODULE_ALIAS_CRYPTO("ecdh"); MODULE_LICENSE("GPL"); diff --git a/crypto/ecdsa.c b/crypto/ecdsa.c index a70b60a90a3c..ce8e4364842f 100644 --- a/crypto/ecdsa.c +++ b/crypto/ecdsa.c @@ -334,7 +334,7 @@ static void __exit ecdsa_exit(void) crypto_unregister_sig(&ecdsa_nist_p521); } -subsys_initcall(ecdsa_init); +module_init(ecdsa_init); module_exit(ecdsa_exit); MODULE_LICENSE("GPL"); diff --git a/crypto/echainiv.c b/crypto/echainiv.c index 69686668625e..1913be8dfbba 100644 --- a/crypto/echainiv.c +++ b/crypto/echainiv.c @@ -157,7 +157,7 @@ static void __exit echainiv_module_exit(void) crypto_unregister_template(&echainiv_tmpl); } -subsys_initcall(echainiv_module_init); +module_init(echainiv_module_init); module_exit(echainiv_module_exit); MODULE_LICENSE("GPL"); diff --git a/crypto/essiv.c b/crypto/essiv.c index cfce8ef7ab1f..d003b78fcd85 100644 --- a/crypto/essiv.c +++ b/crypto/essiv.c @@ -641,7 +641,7 @@ static void __exit essiv_module_exit(void) crypto_unregister_template(&essiv_tmpl); } -subsys_initcall(essiv_module_init); +module_init(essiv_module_init); module_exit(essiv_module_exit); MODULE_DESCRIPTION("ESSIV skcipher/aead wrapper for block encryption"); diff --git a/crypto/fcrypt.c b/crypto/fcrypt.c index 95a16e88899b..80036835cec5 100644 --- a/crypto/fcrypt.c +++ b/crypto/fcrypt.c @@ -411,7 +411,7 @@ static void __exit fcrypt_mod_fini(void) crypto_unregister_alg(&fcrypt_alg); } -subsys_initcall(fcrypt_mod_init); +module_init(fcrypt_mod_init); module_exit(fcrypt_mod_fini); MODULE_LICENSE("Dual BSD/GPL"); diff --git a/crypto/fips.c b/crypto/fips.c index 2fa3a9ee61a1..e88a604cb42b 100644 --- a/crypto/fips.c +++ b/crypto/fips.c @@ -95,5 +95,5 @@ static void __exit fips_exit(void) crypto_proc_fips_exit(); } -subsys_initcall(fips_init); +module_init(fips_init); module_exit(fips_exit); diff --git a/crypto/gcm.c b/crypto/gcm.c index 84f7c23d14e4..54ca9faf0e0c 100644 --- a/crypto/gcm.c +++ b/crypto/gcm.c @@ -1152,7 +1152,7 @@ static void __exit crypto_gcm_module_exit(void) ARRAY_SIZE(crypto_gcm_tmpls)); } -subsys_initcall(crypto_gcm_module_init); +module_init(crypto_gcm_module_init); module_exit(crypto_gcm_module_exit); MODULE_LICENSE("GPL"); diff --git a/crypto/ghash-generic.c b/crypto/ghash-generic.c index b5fc20a0dafc..e5803c249c12 100644 --- a/crypto/ghash-generic.c +++ b/crypto/ghash-generic.c @@ -153,7 +153,7 @@ static void __exit ghash_mod_exit(void) crypto_unregister_shash(&ghash_alg); } -subsys_initcall(ghash_mod_init); +module_init(ghash_mod_init); module_exit(ghash_mod_exit); MODULE_LICENSE("GPL"); diff --git a/crypto/hctr2.c b/crypto/hctr2.c index cbcd673be481..c8932777bba8 100644 --- a/crypto/hctr2.c +++ b/crypto/hctr2.c @@ -570,7 +570,7 @@ static void __exit hctr2_module_exit(void) ARRAY_SIZE(hctr2_tmpls)); } -subsys_initcall(hctr2_module_init); +module_init(hctr2_module_init); module_exit(hctr2_module_exit); MODULE_DESCRIPTION("HCTR2 length-preserving encryption mode"); diff --git a/crypto/hmac.c b/crypto/hmac.c index dfb153511865..ba36ddf50037 100644 --- a/crypto/hmac.c +++ b/crypto/hmac.c @@ -257,7 +257,7 @@ static void __exit hmac_module_exit(void) crypto_unregister_template(&hmac_tmpl); } -subsys_initcall(hmac_module_init); +module_init(hmac_module_init); module_exit(hmac_module_exit); MODULE_LICENSE("GPL"); diff --git a/crypto/khazad.c b/crypto/khazad.c index 7ad338ca2c18..024264ee9cd1 100644 --- a/crypto/khazad.c +++ b/crypto/khazad.c @@ -871,7 +871,7 @@ static void __exit khazad_mod_fini(void) } -subsys_initcall(khazad_mod_init); +module_init(khazad_mod_init); module_exit(khazad_mod_fini); MODULE_LICENSE("GPL"); diff --git a/crypto/krb5enc.c b/crypto/krb5enc.c index d07769bf149e..a1de55994d92 100644 --- a/crypto/krb5enc.c +++ b/crypto/krb5enc.c @@ -496,7 +496,7 @@ static void __exit crypto_krb5enc_module_exit(void) crypto_unregister_template(&crypto_krb5enc_tmpl); } -subsys_initcall(crypto_krb5enc_module_init); +module_init(crypto_krb5enc_module_init); module_exit(crypto_krb5enc_module_exit); MODULE_LICENSE("GPL"); diff --git a/crypto/lrw.c b/crypto/lrw.c index 391ae0f7641f..e7f0368f8c97 100644 --- a/crypto/lrw.c +++ b/crypto/lrw.c @@ -420,7 +420,7 @@ static void __exit lrw_module_exit(void) crypto_unregister_template(&lrw_tmpl); } -subsys_initcall(lrw_module_init); +module_init(lrw_module_init); module_exit(lrw_module_exit); MODULE_LICENSE("GPL"); diff --git a/crypto/lz4.c b/crypto/lz4.c index 9661ed01692f..7a984ae5ae52 100644 --- a/crypto/lz4.c +++ b/crypto/lz4.c @@ -89,7 +89,7 @@ static void __exit lz4_mod_fini(void) crypto_unregister_scomp(&scomp); } -subsys_initcall(lz4_mod_init); +module_init(lz4_mod_init); module_exit(lz4_mod_fini); MODULE_LICENSE("GPL"); diff --git a/crypto/lz4hc.c b/crypto/lz4hc.c index a637fddc1ccd..9c61d05b6214 100644 --- a/crypto/lz4hc.c +++ b/crypto/lz4hc.c @@ -87,7 +87,7 @@ static void __exit lz4hc_mod_fini(void) crypto_unregister_scomp(&scomp); } -subsys_initcall(lz4hc_mod_init); +module_init(lz4hc_mod_init); module_exit(lz4hc_mod_fini); MODULE_LICENSE("GPL"); diff --git a/crypto/lzo-rle.c b/crypto/lzo-rle.c index e7efcf107179..ba013f2d5090 100644 --- a/crypto/lzo-rle.c +++ b/crypto/lzo-rle.c @@ -91,7 +91,7 @@ static void __exit lzorle_mod_fini(void) crypto_unregister_scomp(&scomp); } -subsys_initcall(lzorle_mod_init); +module_init(lzorle_mod_init); module_exit(lzorle_mod_fini); MODULE_LICENSE("GPL"); diff --git a/crypto/lzo.c b/crypto/lzo.c index f1b36a1ca6f6..7867e2c67c4e 100644 --- a/crypto/lzo.c +++ b/crypto/lzo.c @@ -91,7 +91,7 @@ static void __exit lzo_mod_fini(void) crypto_unregister_scomp(&scomp); } -subsys_initcall(lzo_mod_init); +module_init(lzo_mod_init); module_exit(lzo_mod_fini); MODULE_LICENSE("GPL"); diff --git a/crypto/md4.c b/crypto/md4.c index 2e7f2f319f95..55bf47e23c13 100644 --- a/crypto/md4.c +++ b/crypto/md4.c @@ -233,7 +233,7 @@ static void __exit md4_mod_fini(void) crypto_unregister_shash(&alg); } -subsys_initcall(md4_mod_init); +module_init(md4_mod_init); module_exit(md4_mod_fini); MODULE_LICENSE("GPL"); diff --git a/crypto/md5.c b/crypto/md5.c index 994005cd977d..32c0819f5118 100644 --- a/crypto/md5.c +++ b/crypto/md5.c @@ -216,7 +216,7 @@ static void __exit md5_mod_fini(void) crypto_unregister_shash(&alg); } -subsys_initcall(md5_mod_init); +module_init(md5_mod_init); module_exit(md5_mod_fini); MODULE_LICENSE("GPL"); diff --git a/crypto/michael_mic.c b/crypto/michael_mic.c index 0d14e980d4d6..69ad35f524d7 100644 --- a/crypto/michael_mic.c +++ b/crypto/michael_mic.c @@ -167,7 +167,7 @@ static void __exit michael_mic_exit(void) } -subsys_initcall(michael_mic_init); +module_init(michael_mic_init); module_exit(michael_mic_exit); MODULE_LICENSE("GPL v2"); diff --git a/crypto/nhpoly1305.c b/crypto/nhpoly1305.c index a661d4f667cd..2b648615b5ec 100644 --- a/crypto/nhpoly1305.c +++ b/crypto/nhpoly1305.c @@ -245,7 +245,7 @@ static void __exit nhpoly1305_mod_exit(void) crypto_unregister_shash(&nhpoly1305_alg); } -subsys_initcall(nhpoly1305_mod_init); +module_init(nhpoly1305_mod_init); module_exit(nhpoly1305_mod_exit); MODULE_DESCRIPTION("NHPoly1305 ε-almost-∆-universal hash function"); diff --git a/crypto/pcbc.c b/crypto/pcbc.c index 9d2e56d6744a..d092717ea4fc 100644 --- a/crypto/pcbc.c +++ b/crypto/pcbc.c @@ -186,7 +186,7 @@ static void __exit crypto_pcbc_module_exit(void) crypto_unregister_template(&crypto_pcbc_tmpl); } -subsys_initcall(crypto_pcbc_module_init); +module_init(crypto_pcbc_module_init); module_exit(crypto_pcbc_module_exit); MODULE_LICENSE("GPL"); diff --git a/crypto/pcrypt.c b/crypto/pcrypt.c index 7fc79e7dce44..c33d29a523e0 100644 --- a/crypto/pcrypt.c +++ b/crypto/pcrypt.c @@ -381,7 +381,7 @@ static void __exit pcrypt_exit(void) kset_unregister(pcrypt_kset); } -subsys_initcall(pcrypt_init); +module_init(pcrypt_init); module_exit(pcrypt_exit); MODULE_LICENSE("GPL"); diff --git a/crypto/polyval-generic.c b/crypto/polyval-generic.c index ffd174e75420..db8adb56e4ca 100644 --- a/crypto/polyval-generic.c +++ b/crypto/polyval-generic.c @@ -196,7 +196,7 @@ static void __exit polyval_mod_exit(void) crypto_unregister_shash(&polyval_alg); } -subsys_initcall(polyval_mod_init); +module_init(polyval_mod_init); module_exit(polyval_mod_exit); MODULE_LICENSE("GPL"); diff --git a/crypto/rmd160.c b/crypto/rmd160.c index 890d29e46a76..9860b60c9be4 100644 --- a/crypto/rmd160.c +++ b/crypto/rmd160.c @@ -342,7 +342,7 @@ static void __exit rmd160_mod_fini(void) crypto_unregister_shash(&alg); } -subsys_initcall(rmd160_mod_init); +module_init(rmd160_mod_init); module_exit(rmd160_mod_fini); MODULE_LICENSE("GPL"); diff --git a/crypto/rsa.c b/crypto/rsa.c index b7d21529c552..6c7734083c98 100644 --- a/crypto/rsa.c +++ b/crypto/rsa.c @@ -430,7 +430,7 @@ static void __exit rsa_exit(void) crypto_unregister_akcipher(&rsa); } -subsys_initcall(rsa_init); +module_init(rsa_init); module_exit(rsa_exit); MODULE_ALIAS_CRYPTO("rsa"); MODULE_LICENSE("GPL"); diff --git a/crypto/seed.c b/crypto/seed.c index d05d8ed909fa..815391f213de 100644 --- a/crypto/seed.c +++ b/crypto/seed.c @@ -460,7 +460,7 @@ static void __exit seed_fini(void) crypto_unregister_alg(&seed_alg); } -subsys_initcall(seed_init); +module_init(seed_init); module_exit(seed_fini); MODULE_DESCRIPTION("SEED Cipher Algorithm"); diff --git a/crypto/seqiv.c b/crypto/seqiv.c index 17e11d51ddc3..a17ef5184398 100644 --- a/crypto/seqiv.c +++ b/crypto/seqiv.c @@ -179,7 +179,7 @@ static void __exit seqiv_module_exit(void) crypto_unregister_template(&seqiv_tmpl); } -subsys_initcall(seqiv_module_init); +module_init(seqiv_module_init); module_exit(seqiv_module_exit); MODULE_LICENSE("GPL"); diff --git a/crypto/serpent_generic.c b/crypto/serpent_generic.c index f6ef187be6fe..b21e7606c652 100644 --- a/crypto/serpent_generic.c +++ b/crypto/serpent_generic.c @@ -599,7 +599,7 @@ static void __exit serpent_mod_fini(void) crypto_unregister_alg(&srp_alg); } -subsys_initcall(serpent_mod_init); +module_init(serpent_mod_init); module_exit(serpent_mod_fini); MODULE_LICENSE("GPL"); diff --git a/crypto/sha1_generic.c b/crypto/sha1_generic.c index 7a3c837923b5..024e8043bab0 100644 --- a/crypto/sha1_generic.c +++ b/crypto/sha1_generic.c @@ -77,7 +77,7 @@ static void __exit sha1_generic_mod_fini(void) crypto_unregister_shash(&alg); } -subsys_initcall(sha1_generic_mod_init); +module_init(sha1_generic_mod_init); module_exit(sha1_generic_mod_fini); MODULE_LICENSE("GPL"); diff --git a/crypto/sha256.c b/crypto/sha256.c index c2588d08ee3e..47ad7e4cc55f 100644 --- a/crypto/sha256.c +++ b/crypto/sha256.c @@ -224,7 +224,7 @@ static int __init crypto_sha256_mod_init(void) num_algs /= 2; return crypto_register_shashes(algs, ARRAY_SIZE(algs)); } -subsys_initcall(crypto_sha256_mod_init); +module_init(crypto_sha256_mod_init); static void __exit crypto_sha256_mod_exit(void) { diff --git a/crypto/sha512_generic.c b/crypto/sha512_generic.c index bfea65f4181c..7368173f545e 100644 --- a/crypto/sha512_generic.c +++ b/crypto/sha512_generic.c @@ -205,7 +205,7 @@ static void __exit sha512_generic_mod_fini(void) crypto_unregister_shashes(sha512_algs, ARRAY_SIZE(sha512_algs)); } -subsys_initcall(sha512_generic_mod_init); +module_init(sha512_generic_mod_init); module_exit(sha512_generic_mod_fini); MODULE_LICENSE("GPL"); diff --git a/crypto/sm3_generic.c b/crypto/sm3_generic.c index 4fb6957c2f0c..7529139fcc96 100644 --- a/crypto/sm3_generic.c +++ b/crypto/sm3_generic.c @@ -62,7 +62,7 @@ static void __exit sm3_generic_mod_fini(void) crypto_unregister_shash(&sm3_alg); } -subsys_initcall(sm3_generic_mod_init); +module_init(sm3_generic_mod_init); module_exit(sm3_generic_mod_fini); MODULE_LICENSE("GPL v2"); diff --git a/crypto/sm4_generic.c b/crypto/sm4_generic.c index 7df86369ac00..d57444e8428c 100644 --- a/crypto/sm4_generic.c +++ b/crypto/sm4_generic.c @@ -83,7 +83,7 @@ static void __exit sm4_fini(void) crypto_unregister_alg(&sm4_alg); } -subsys_initcall(sm4_init); +module_init(sm4_init); module_exit(sm4_fini); MODULE_DESCRIPTION("SM4 Cipher Algorithm"); diff --git a/crypto/streebog_generic.c b/crypto/streebog_generic.c index 99c07e0c10ba..57bbf70f4c22 100644 --- a/crypto/streebog_generic.c +++ b/crypto/streebog_generic.c @@ -1061,7 +1061,7 @@ static void __exit streebog_mod_fini(void) crypto_unregister_shashes(algs, ARRAY_SIZE(algs)); } -subsys_initcall(streebog_mod_init); +module_init(streebog_mod_init); module_exit(streebog_mod_fini); MODULE_LICENSE("GPL"); diff --git a/crypto/tea.c b/crypto/tea.c index b315da8c89eb..cb05140e3470 100644 --- a/crypto/tea.c +++ b/crypto/tea.c @@ -255,7 +255,7 @@ MODULE_ALIAS_CRYPTO("tea"); MODULE_ALIAS_CRYPTO("xtea"); MODULE_ALIAS_CRYPTO("xeta"); -subsys_initcall(tea_mod_init); +module_init(tea_mod_init); module_exit(tea_mod_fini); MODULE_LICENSE("GPL"); diff --git a/crypto/twofish_generic.c b/crypto/twofish_generic.c index 19f2b365e140..368018cfa9bf 100644 --- a/crypto/twofish_generic.c +++ b/crypto/twofish_generic.c @@ -187,7 +187,7 @@ static void __exit twofish_mod_fini(void) crypto_unregister_alg(&alg); } -subsys_initcall(twofish_mod_init); +module_init(twofish_mod_init); module_exit(twofish_mod_fini); MODULE_LICENSE("GPL"); diff --git a/crypto/wp512.c b/crypto/wp512.c index 07994e5ebf4e..41f13d490333 100644 --- a/crypto/wp512.c +++ b/crypto/wp512.c @@ -1169,7 +1169,7 @@ MODULE_ALIAS_CRYPTO("wp512"); MODULE_ALIAS_CRYPTO("wp384"); MODULE_ALIAS_CRYPTO("wp256"); -subsys_initcall(wp512_mod_init); +module_init(wp512_mod_init); module_exit(wp512_mod_fini); MODULE_LICENSE("GPL"); diff --git a/crypto/xcbc.c b/crypto/xcbc.c index 970ff581dc58..6c5f6766fdd6 100644 --- a/crypto/xcbc.c +++ b/crypto/xcbc.c @@ -199,7 +199,7 @@ static void __exit crypto_xcbc_module_exit(void) crypto_unregister_template(&crypto_xcbc_tmpl); } -subsys_initcall(crypto_xcbc_module_init); +module_init(crypto_xcbc_module_init); module_exit(crypto_xcbc_module_exit); MODULE_LICENSE("GPL"); diff --git a/crypto/xctr.c b/crypto/xctr.c index 9c536ab6d2e5..607ab82cb19b 100644 --- a/crypto/xctr.c +++ b/crypto/xctr.c @@ -182,7 +182,7 @@ static void __exit crypto_xctr_module_exit(void) crypto_unregister_template(&crypto_xctr_tmpl); } -subsys_initcall(crypto_xctr_module_init); +module_init(crypto_xctr_module_init); module_exit(crypto_xctr_module_exit); MODULE_LICENSE("GPL"); diff --git a/crypto/xts.c b/crypto/xts.c index 31529c9ef08f..1a9edd55a3a2 100644 --- a/crypto/xts.c +++ b/crypto/xts.c @@ -466,7 +466,7 @@ static void __exit xts_module_exit(void) crypto_unregister_template(&xts_tmpl); } -subsys_initcall(xts_module_init); +module_init(xts_module_init); module_exit(xts_module_exit); MODULE_LICENSE("GPL"); diff --git a/crypto/xxhash_generic.c b/crypto/xxhash_generic.c index ac206ad4184d..175bb7ae0fcd 100644 --- a/crypto/xxhash_generic.c +++ b/crypto/xxhash_generic.c @@ -96,7 +96,7 @@ static void __exit xxhash_mod_fini(void) crypto_unregister_shash(&alg); } -subsys_initcall(xxhash_mod_init); +module_init(xxhash_mod_init); module_exit(xxhash_mod_fini); MODULE_AUTHOR("Nikolay Borisov "); diff --git a/crypto/zstd.c b/crypto/zstd.c index 90bb4f36f846..7570e11b4ee6 100644 --- a/crypto/zstd.c +++ b/crypto/zstd.c @@ -196,7 +196,7 @@ static void __exit zstd_mod_fini(void) crypto_unregister_scomp(&scomp); } -subsys_initcall(zstd_mod_init); +module_init(zstd_mod_init); module_exit(zstd_mod_fini); MODULE_LICENSE("GPL"); -- 2.50.1 From ce026b35b7b62094b640d3781d09bbd5b132b326 Mon Sep 17 00:00:00 2001 From: Herbert Xu Date: Wed, 30 Apr 2025 16:25:55 +0800 Subject: [PATCH 10/16] crypto: x86/blake2s - Include linux/init.h Explicitly include linux/init.h rather than pulling it through potluck. Signed-off-by: Herbert Xu --- arch/x86/lib/crypto/blake2s-glue.c | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/arch/x86/lib/crypto/blake2s-glue.c b/arch/x86/lib/crypto/blake2s-glue.c index 00f84f29cc8c..adc296cd17c9 100644 --- a/arch/x86/lib/crypto/blake2s-glue.c +++ b/arch/x86/lib/crypto/blake2s-glue.c @@ -3,17 +3,15 @@ * Copyright (C) 2015-2019 Jason A. Donenfeld . All Rights Reserved. */ -#include - -#include -#include -#include -#include - #include #include #include #include +#include +#include +#include +#include +#include asmlinkage void blake2s_compress_ssse3(struct blake2s_state *state, const u8 *block, const size_t nblocks, -- 2.50.1 From b8ca43806292d7094103c2af44fbb9824874b19e Mon Sep 17 00:00:00 2001 From: George Abraham P Date: Wed, 30 Apr 2025 12:34:43 +0100 Subject: [PATCH 11/16] crypto: qat - rename and relocate timer logic Rename adf_gen4_timer.c to adf_timer.c and adf_gen4_timer.h to adf_timer.h to make the files generation-agnostic. This includes renaming the start() and stop() timer APIs and macro definitions to be generic, allowing for reuse across different device generations. This does not introduce any functional changes. Signed-off-by: George Abraham P Reviewed-by: Giovanni Cabiddu Reviewed-by: Andy Shevchenko Signed-off-by: Herbert Xu --- .../intel/qat/qat_420xx/adf_420xx_hw_data.c | 6 +++--- .../intel/qat/qat_4xxx/adf_4xxx_hw_data.c | 6 +++--- drivers/crypto/intel/qat/qat_common/Makefile | 2 +- .../{adf_gen4_timer.c => adf_timer.c} | 18 +++++++++--------- .../{adf_gen4_timer.h => adf_timer.h} | 10 +++++----- 5 files changed, 21 insertions(+), 21 deletions(-) rename drivers/crypto/intel/qat/qat_common/{adf_gen4_timer.c => adf_timer.c} (78%) rename drivers/crypto/intel/qat/qat_common/{adf_gen4_timer.h => adf_timer.h} (58%) diff --git a/drivers/crypto/intel/qat/qat_420xx/adf_420xx_hw_data.c b/drivers/crypto/intel/qat/qat_420xx/adf_420xx_hw_data.c index 795f4598400b..5817b3164185 100644 --- a/drivers/crypto/intel/qat/qat_420xx/adf_420xx_hw_data.c +++ b/drivers/crypto/intel/qat/qat_420xx/adf_420xx_hw_data.c @@ -15,9 +15,9 @@ #include #include #include -#include #include #include +#include #include "adf_420xx_hw_data.h" #include "icp_qat_hw.h" @@ -468,8 +468,8 @@ void adf_init_hw_data_420xx(struct adf_hw_device_data *hw_data, u32 dev_id) hw_data->enable_pm = adf_gen4_enable_pm; hw_data->handle_pm_interrupt = adf_gen4_handle_pm_interrupt; hw_data->dev_config = adf_gen4_dev_config; - hw_data->start_timer = adf_gen4_timer_start; - hw_data->stop_timer = adf_gen4_timer_stop; + hw_data->start_timer = adf_timer_start; + hw_data->stop_timer = adf_timer_stop; hw_data->get_hb_clock = adf_gen4_get_heartbeat_clock; hw_data->num_hb_ctrs = ADF_NUM_HB_CNT_PER_AE; hw_data->clock_frequency = ADF_420XX_AE_FREQ; diff --git a/drivers/crypto/intel/qat/qat_4xxx/adf_4xxx_hw_data.c b/drivers/crypto/intel/qat/qat_4xxx/adf_4xxx_hw_data.c index 7d4c366aa8b2..2d89d4a3a7b9 100644 --- a/drivers/crypto/intel/qat/qat_4xxx/adf_4xxx_hw_data.c +++ b/drivers/crypto/intel/qat/qat_4xxx/adf_4xxx_hw_data.c @@ -15,9 +15,9 @@ #include #include #include "adf_gen4_ras.h" -#include #include #include +#include #include "adf_4xxx_hw_data.h" #include "icp_qat_hw.h" @@ -454,8 +454,8 @@ void adf_init_hw_data_4xxx(struct adf_hw_device_data *hw_data, u32 dev_id) hw_data->enable_pm = adf_gen4_enable_pm; hw_data->handle_pm_interrupt = adf_gen4_handle_pm_interrupt; hw_data->dev_config = adf_gen4_dev_config; - hw_data->start_timer = adf_gen4_timer_start; - hw_data->stop_timer = adf_gen4_timer_stop; + hw_data->start_timer = adf_timer_start; + hw_data->stop_timer = adf_timer_stop; hw_data->get_hb_clock = adf_gen4_get_heartbeat_clock; hw_data->num_hb_ctrs = ADF_NUM_HB_CNT_PER_AE; hw_data->clock_frequency = ADF_4XXX_AE_FREQ; diff --git a/drivers/crypto/intel/qat/qat_common/Makefile b/drivers/crypto/intel/qat/qat_common/Makefile index af5df29fd2e3..0370eaad42b1 100644 --- a/drivers/crypto/intel/qat/qat_common/Makefile +++ b/drivers/crypto/intel/qat/qat_common/Makefile @@ -19,7 +19,6 @@ intel_qat-y := adf_accel_engine.o \ adf_gen4_hw_data.o \ adf_gen4_pm.o \ adf_gen4_ras.o \ - adf_gen4_timer.o \ adf_gen4_vf_mig.o \ adf_hw_arbiter.o \ adf_init.o \ @@ -30,6 +29,7 @@ intel_qat-y := adf_accel_engine.o \ adf_sysfs.o \ adf_sysfs_ras_counters.o \ adf_sysfs_rl.o \ + adf_timer.o \ adf_transport.o \ qat_algs.o \ qat_algs_send.o \ diff --git a/drivers/crypto/intel/qat/qat_common/adf_gen4_timer.c b/drivers/crypto/intel/qat/qat_common/adf_timer.c similarity index 78% rename from drivers/crypto/intel/qat/qat_common/adf_gen4_timer.c rename to drivers/crypto/intel/qat/qat_common/adf_timer.c index 35ccb91d6ec1..8962a49f145a 100644 --- a/drivers/crypto/intel/qat/qat_common/adf_gen4_timer.c +++ b/drivers/crypto/intel/qat/qat_common/adf_timer.c @@ -12,9 +12,9 @@ #include "adf_admin.h" #include "adf_accel_devices.h" #include "adf_common_drv.h" -#include "adf_gen4_timer.h" +#include "adf_timer.h" -#define ADF_GEN4_TIMER_PERIOD_MS 200 +#define ADF_DEFAULT_TIMER_PERIOD_MS 200 /* This periodic update is used to trigger HB, RL & TL fw events */ static void work_handler(struct work_struct *work) @@ -27,16 +27,16 @@ static void work_handler(struct work_struct *work) accel_dev = timer_ctx->accel_dev; adf_misc_wq_queue_delayed_work(&timer_ctx->work_ctx, - msecs_to_jiffies(ADF_GEN4_TIMER_PERIOD_MS)); + msecs_to_jiffies(ADF_DEFAULT_TIMER_PERIOD_MS)); time_periods = div_u64(ktime_ms_delta(ktime_get_real(), timer_ctx->initial_ktime), - ADF_GEN4_TIMER_PERIOD_MS); + ADF_DEFAULT_TIMER_PERIOD_MS); if (adf_send_admin_tim_sync(accel_dev, time_periods)) dev_err(&GET_DEV(accel_dev), "Failed to synchronize qat timer\n"); } -int adf_gen4_timer_start(struct adf_accel_dev *accel_dev) +int adf_timer_start(struct adf_accel_dev *accel_dev) { struct adf_timer *timer_ctx; @@ -50,13 +50,13 @@ int adf_gen4_timer_start(struct adf_accel_dev *accel_dev) INIT_DELAYED_WORK(&timer_ctx->work_ctx, work_handler); adf_misc_wq_queue_delayed_work(&timer_ctx->work_ctx, - msecs_to_jiffies(ADF_GEN4_TIMER_PERIOD_MS)); + msecs_to_jiffies(ADF_DEFAULT_TIMER_PERIOD_MS)); return 0; } -EXPORT_SYMBOL_GPL(adf_gen4_timer_start); +EXPORT_SYMBOL_GPL(adf_timer_start); -void adf_gen4_timer_stop(struct adf_accel_dev *accel_dev) +void adf_timer_stop(struct adf_accel_dev *accel_dev) { struct adf_timer *timer_ctx = accel_dev->timer; @@ -68,4 +68,4 @@ void adf_gen4_timer_stop(struct adf_accel_dev *accel_dev) kfree(timer_ctx); accel_dev->timer = NULL; } -EXPORT_SYMBOL_GPL(adf_gen4_timer_stop); +EXPORT_SYMBOL_GPL(adf_timer_stop); diff --git a/drivers/crypto/intel/qat/qat_common/adf_gen4_timer.h b/drivers/crypto/intel/qat/qat_common/adf_timer.h similarity index 58% rename from drivers/crypto/intel/qat/qat_common/adf_gen4_timer.h rename to drivers/crypto/intel/qat/qat_common/adf_timer.h index 66a709e7b358..68e5136d6ba1 100644 --- a/drivers/crypto/intel/qat/qat_common/adf_gen4_timer.h +++ b/drivers/crypto/intel/qat/qat_common/adf_timer.h @@ -1,8 +1,8 @@ /* SPDX-License-Identifier: GPL-2.0-only */ /* Copyright(c) 2023 Intel Corporation */ -#ifndef ADF_GEN4_TIMER_H_ -#define ADF_GEN4_TIMER_H_ +#ifndef ADF_TIMER_H_ +#define ADF_TIMER_H_ #include #include @@ -15,7 +15,7 @@ struct adf_timer { ktime_t initial_ktime; }; -int adf_gen4_timer_start(struct adf_accel_dev *accel_dev); -void adf_gen4_timer_stop(struct adf_accel_dev *accel_dev); +int adf_timer_start(struct adf_accel_dev *accel_dev); +void adf_timer_stop(struct adf_accel_dev *accel_dev); -#endif /* ADF_GEN4_TIMER_H_ */ +#endif /* ADF_TIMER_H_ */ -- 2.50.1 From cd0e7160f80f5fd3ebb6481ddf4f562838cad4bf Mon Sep 17 00:00:00 2001 From: Suman Kumar Chakraborty Date: Wed, 30 Apr 2025 12:34:44 +0100 Subject: [PATCH 12/16] crypto: qat - refactor compression template logic The logic that generates the compression templates, which are used by to submit compression requests to the QAT device, is very similar between QAT devices and diverges mainly on the HW generation-specific configuration word. This makes the logic that generates the compression and decompression templates common between GEN2 and GEN4 devices and abstracts the generation-specific logic to the generation-specific implementations. The adf_gen2_dc.c and adf_gen4_dc.c have been replaced by adf_dc.c, and the generation-specific logic has been reduced and moved to adf_gen2_hw_data.c and adf_gen4_hw_data.c. This does not introduce any functional change. Co-developed-by: Vijay Sundar Selvamani Signed-off-by: Vijay Sundar Selvamani Signed-off-by: Suman Kumar Chakraborty Reviewed-by: Giovanni Cabiddu Reviewed-by: Andy Shevchenko Signed-off-by: Herbert Xu --- .../intel/qat/qat_420xx/adf_420xx_hw_data.c | 1 - .../intel/qat/qat_4xxx/adf_4xxx_hw_data.c | 1 - .../intel/qat/qat_c3xxx/adf_c3xxx_hw_data.c | 1 - .../qat/qat_c3xxxvf/adf_c3xxxvf_hw_data.c | 1 - .../intel/qat/qat_c62x/adf_c62x_hw_data.c | 1 - .../intel/qat/qat_c62xvf/adf_c62xvf_hw_data.c | 1 - drivers/crypto/intel/qat/qat_common/Makefile | 3 +- .../intel/qat/qat_common/adf_accel_devices.h | 4 +- .../qat_common/{adf_gen2_dc.c => adf_dc.c} | 47 +++++------ drivers/crypto/intel/qat/qat_common/adf_dc.h | 17 ++++ .../crypto/intel/qat/qat_common/adf_gen2_dc.h | 10 --- .../intel/qat/qat_common/adf_gen2_hw_data.c | 57 +++++++++++++ .../intel/qat/qat_common/adf_gen2_hw_data.h | 1 + .../crypto/intel/qat/qat_common/adf_gen4_dc.c | 83 ------------------- .../crypto/intel/qat/qat_common/adf_gen4_dc.h | 10 --- .../intel/qat/qat_common/adf_gen4_hw_data.c | 70 ++++++++++++++++ .../intel/qat/qat_common/adf_gen4_hw_data.h | 2 + .../intel/qat/qat_common/qat_comp_algs.c | 5 +- .../intel/qat/qat_common/qat_compression.c | 1 - .../intel/qat/qat_common/qat_compression.h | 1 - .../qat/qat_dh895xcc/adf_dh895xcc_hw_data.c | 1 - .../qat_dh895xccvf/adf_dh895xccvf_hw_data.c | 1 - 22 files changed, 173 insertions(+), 146 deletions(-) rename drivers/crypto/intel/qat/qat_common/{adf_gen2_dc.c => adf_dc.c} (61%) create mode 100644 drivers/crypto/intel/qat/qat_common/adf_dc.h delete mode 100644 drivers/crypto/intel/qat/qat_common/adf_gen2_dc.h delete mode 100644 drivers/crypto/intel/qat/qat_common/adf_gen4_dc.c delete mode 100644 drivers/crypto/intel/qat/qat_common/adf_gen4_dc.h diff --git a/drivers/crypto/intel/qat/qat_420xx/adf_420xx_hw_data.c b/drivers/crypto/intel/qat/qat_420xx/adf_420xx_hw_data.c index 5817b3164185..7c3c0f561c95 100644 --- a/drivers/crypto/intel/qat/qat_420xx/adf_420xx_hw_data.c +++ b/drivers/crypto/intel/qat/qat_420xx/adf_420xx_hw_data.c @@ -9,7 +9,6 @@ #include #include #include -#include #include #include #include diff --git a/drivers/crypto/intel/qat/qat_4xxx/adf_4xxx_hw_data.c b/drivers/crypto/intel/qat/qat_4xxx/adf_4xxx_hw_data.c index 2d89d4a3a7b9..bd0b1b1015c0 100644 --- a/drivers/crypto/intel/qat/qat_4xxx/adf_4xxx_hw_data.c +++ b/drivers/crypto/intel/qat/qat_4xxx/adf_4xxx_hw_data.c @@ -9,7 +9,6 @@ #include #include #include -#include #include #include #include diff --git a/drivers/crypto/intel/qat/qat_c3xxx/adf_c3xxx_hw_data.c b/drivers/crypto/intel/qat/qat_c3xxx/adf_c3xxx_hw_data.c index 9425af26d34c..07f2c42a68f5 100644 --- a/drivers/crypto/intel/qat/qat_c3xxx/adf_c3xxx_hw_data.c +++ b/drivers/crypto/intel/qat/qat_c3xxx/adf_c3xxx_hw_data.c @@ -5,7 +5,6 @@ #include #include #include -#include #include #include #include diff --git a/drivers/crypto/intel/qat/qat_c3xxxvf/adf_c3xxxvf_hw_data.c b/drivers/crypto/intel/qat/qat_c3xxxvf/adf_c3xxxvf_hw_data.c index f73d9a4a9ab7..db3c33fa1881 100644 --- a/drivers/crypto/intel/qat/qat_c3xxxvf/adf_c3xxxvf_hw_data.c +++ b/drivers/crypto/intel/qat/qat_c3xxxvf/adf_c3xxxvf_hw_data.c @@ -3,7 +3,6 @@ #include #include #include -#include #include #include #include diff --git a/drivers/crypto/intel/qat/qat_c62x/adf_c62x_hw_data.c b/drivers/crypto/intel/qat/qat_c62x/adf_c62x_hw_data.c index 1a2f36b603fb..0b410b41474d 100644 --- a/drivers/crypto/intel/qat/qat_c62x/adf_c62x_hw_data.c +++ b/drivers/crypto/intel/qat/qat_c62x/adf_c62x_hw_data.c @@ -5,7 +5,6 @@ #include #include #include -#include #include #include #include diff --git a/drivers/crypto/intel/qat/qat_c62xvf/adf_c62xvf_hw_data.c b/drivers/crypto/intel/qat/qat_c62xvf/adf_c62xvf_hw_data.c index 29e53b41a895..7f00035d3661 100644 --- a/drivers/crypto/intel/qat/qat_c62xvf/adf_c62xvf_hw_data.c +++ b/drivers/crypto/intel/qat/qat_c62xvf/adf_c62xvf_hw_data.c @@ -3,7 +3,6 @@ #include #include #include -#include #include #include #include diff --git a/drivers/crypto/intel/qat/qat_common/Makefile b/drivers/crypto/intel/qat/qat_common/Makefile index 0370eaad42b1..0a9da7398a78 100644 --- a/drivers/crypto/intel/qat/qat_common/Makefile +++ b/drivers/crypto/intel/qat/qat_common/Makefile @@ -8,13 +8,12 @@ intel_qat-y := adf_accel_engine.o \ adf_cfg_services.o \ adf_clock.o \ adf_ctl_drv.o \ + adf_dc.o \ adf_dev_mgr.o \ adf_gen2_config.o \ - adf_gen2_dc.o \ adf_gen2_hw_csr_data.o \ adf_gen2_hw_data.o \ adf_gen4_config.o \ - adf_gen4_dc.o \ adf_gen4_hw_csr_data.o \ adf_gen4_hw_data.o \ adf_gen4_pm.o \ diff --git a/drivers/crypto/intel/qat/qat_common/adf_accel_devices.h b/drivers/crypto/intel/qat/qat_common/adf_accel_devices.h index 1e301a20c244..a39f506322f6 100644 --- a/drivers/crypto/intel/qat/qat_common/adf_accel_devices.h +++ b/drivers/crypto/intel/qat/qat_common/adf_accel_devices.h @@ -12,6 +12,7 @@ #include #include #include "adf_cfg_common.h" +#include "adf_dc.h" #include "adf_rl.h" #include "adf_telemetry.h" #include "adf_pfvf_msg.h" @@ -267,7 +268,8 @@ struct adf_pfvf_ops { }; struct adf_dc_ops { - void (*build_deflate_ctx)(void *ctx); + int (*build_comp_block)(void *ctx, enum adf_dc_algo algo); + int (*build_decomp_block)(void *ctx, enum adf_dc_algo algo); }; struct qat_migdev_ops { diff --git a/drivers/crypto/intel/qat/qat_common/adf_gen2_dc.c b/drivers/crypto/intel/qat/qat_common/adf_dc.c similarity index 61% rename from drivers/crypto/intel/qat/qat_common/adf_gen2_dc.c rename to drivers/crypto/intel/qat/qat_common/adf_dc.c index 47261b1c1da6..4beb4b7dbf0e 100644 --- a/drivers/crypto/intel/qat/qat_common/adf_gen2_dc.c +++ b/drivers/crypto/intel/qat/qat_common/adf_dc.c @@ -1,22 +1,21 @@ // SPDX-License-Identifier: GPL-2.0-only /* Copyright(c) 2022 Intel Corporation */ #include "adf_accel_devices.h" -#include "adf_gen2_dc.h" +#include "adf_dc.h" #include "icp_qat_fw_comp.h" -static void qat_comp_build_deflate_ctx(void *ctx) +int qat_comp_build_ctx(struct adf_accel_dev *accel_dev, void *ctx, enum adf_dc_algo algo) { - struct icp_qat_fw_comp_req *req_tmpl = (struct icp_qat_fw_comp_req *)ctx; - struct icp_qat_fw_comn_req_hdr *header = &req_tmpl->comn_hdr; - struct icp_qat_fw_comp_req_hdr_cd_pars *cd_pars = &req_tmpl->cd_pars; - struct icp_qat_fw_comp_req_params *req_pars = &req_tmpl->comp_pars; + struct icp_qat_fw_comp_req *req_tmpl = ctx; struct icp_qat_fw_comp_cd_hdr *comp_cd_ctrl = &req_tmpl->comp_cd_ctrl; + struct icp_qat_fw_comp_req_params *req_pars = &req_tmpl->comp_pars; + struct icp_qat_fw_comn_req_hdr *header = &req_tmpl->comn_hdr; + int ret; memset(req_tmpl, 0, sizeof(*req_tmpl)); header->hdr_flags = ICP_QAT_FW_COMN_HDR_FLAGS_BUILD(ICP_QAT_FW_COMN_REQ_FLAG_SET); header->service_type = ICP_QAT_FW_COMN_REQ_CPM_FW_COMP; - header->service_cmd_id = ICP_QAT_FW_COMP_CMD_STATIC; header->comn_req_flags = ICP_QAT_FW_COMN_FLAGS_BUILD(QAT_COMN_CD_FLD_TYPE_16BYTE_DATA, QAT_COMN_PTR_TYPE_SGL); @@ -26,12 +25,14 @@ static void qat_comp_build_deflate_ctx(void *ctx) ICP_QAT_FW_COMP_NOT_ENH_AUTO_SELECT_BEST, ICP_QAT_FW_COMP_NOT_DISABLE_TYPE0_ENH_AUTO_SELECT_BEST, ICP_QAT_FW_COMP_ENABLE_SECURE_RAM_USED_AS_INTMD_BUF); - cd_pars->u.sl.comp_slice_cfg_word[0] = - ICP_QAT_HW_COMPRESSION_CONFIG_BUILD(ICP_QAT_HW_COMPRESSION_DIR_COMPRESS, - ICP_QAT_HW_COMPRESSION_DELAYED_MATCH_DISABLED, - ICP_QAT_HW_COMPRESSION_ALGO_DEFLATE, - ICP_QAT_HW_COMPRESSION_DEPTH_1, - ICP_QAT_HW_COMPRESSION_FILE_TYPE_0); + + /* Build HW config block for compression */ + ret = GET_DC_OPS(accel_dev)->build_comp_block(ctx, algo); + if (ret) { + dev_err(&GET_DEV(accel_dev), "Failed to build compression block\n"); + return ret; + } + req_pars->crc.legacy.initial_adler = COMP_CPR_INITIAL_ADLER; req_pars->crc.legacy.initial_crc32 = COMP_CPR_INITIAL_CRC; req_pars->req_par_flags = @@ -52,19 +53,11 @@ static void qat_comp_build_deflate_ctx(void *ctx) /* Fill second half of the template for decompression */ memcpy(req_tmpl + 1, req_tmpl, sizeof(*req_tmpl)); req_tmpl++; - header = &req_tmpl->comn_hdr; - header->service_cmd_id = ICP_QAT_FW_COMP_CMD_DECOMPRESS; - cd_pars = &req_tmpl->cd_pars; - cd_pars->u.sl.comp_slice_cfg_word[0] = - ICP_QAT_HW_COMPRESSION_CONFIG_BUILD(ICP_QAT_HW_COMPRESSION_DIR_DECOMPRESS, - ICP_QAT_HW_COMPRESSION_DELAYED_MATCH_DISABLED, - ICP_QAT_HW_COMPRESSION_ALGO_DEFLATE, - ICP_QAT_HW_COMPRESSION_DEPTH_1, - ICP_QAT_HW_COMPRESSION_FILE_TYPE_0); -} -void adf_gen2_init_dc_ops(struct adf_dc_ops *dc_ops) -{ - dc_ops->build_deflate_ctx = qat_comp_build_deflate_ctx; + /* Build HW config block for decompression */ + ret = GET_DC_OPS(accel_dev)->build_decomp_block(req_tmpl, algo); + if (ret) + dev_err(&GET_DEV(accel_dev), "Failed to build decompression block\n"); + + return ret; } -EXPORT_SYMBOL_GPL(adf_gen2_init_dc_ops); diff --git a/drivers/crypto/intel/qat/qat_common/adf_dc.h b/drivers/crypto/intel/qat/qat_common/adf_dc.h new file mode 100644 index 000000000000..6cb5e09054a6 --- /dev/null +++ b/drivers/crypto/intel/qat/qat_common/adf_dc.h @@ -0,0 +1,17 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* Copyright(c) 2025 Intel Corporation */ +#ifndef ADF_DC_H +#define ADF_DC_H + +struct adf_accel_dev; + +enum adf_dc_algo { + QAT_DEFLATE, + QAT_LZ4, + QAT_LZ4S, + QAT_ZSTD, +}; + +int qat_comp_build_ctx(struct adf_accel_dev *accel_dev, void *ctx, enum adf_dc_algo algo); + +#endif /* ADF_DC_H */ diff --git a/drivers/crypto/intel/qat/qat_common/adf_gen2_dc.h b/drivers/crypto/intel/qat/qat_common/adf_gen2_dc.h deleted file mode 100644 index 6eae023354d7..000000000000 --- a/drivers/crypto/intel/qat/qat_common/adf_gen2_dc.h +++ /dev/null @@ -1,10 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0-only */ -/* Copyright(c) 2022 Intel Corporation */ -#ifndef ADF_GEN2_DC_H -#define ADF_GEN2_DC_H - -#include "adf_accel_devices.h" - -void adf_gen2_init_dc_ops(struct adf_dc_ops *dc_ops); - -#endif /* ADF_GEN2_DC_H */ diff --git a/drivers/crypto/intel/qat/qat_common/adf_gen2_hw_data.c b/drivers/crypto/intel/qat/qat_common/adf_gen2_hw_data.c index 2b263442c856..6a505e9a5cf9 100644 --- a/drivers/crypto/intel/qat/qat_common/adf_gen2_hw_data.c +++ b/drivers/crypto/intel/qat/qat_common/adf_gen2_hw_data.c @@ -1,7 +1,9 @@ // SPDX-License-Identifier: (BSD-3-Clause OR GPL-2.0-only) /* Copyright(c) 2020 Intel Corporation */ #include "adf_common_drv.h" +#include "adf_dc.h" #include "adf_gen2_hw_data.h" +#include "icp_qat_fw_comp.h" #include "icp_qat_hw.h" #include @@ -169,3 +171,58 @@ void adf_gen2_set_ssm_wdtimer(struct adf_accel_dev *accel_dev) } } EXPORT_SYMBOL_GPL(adf_gen2_set_ssm_wdtimer); + +static int adf_gen2_build_comp_block(void *ctx, enum adf_dc_algo algo) +{ + struct icp_qat_fw_comp_req *req_tmpl = ctx; + struct icp_qat_fw_comp_req_hdr_cd_pars *cd_pars = &req_tmpl->cd_pars; + struct icp_qat_fw_comn_req_hdr *header = &req_tmpl->comn_hdr; + + switch (algo) { + case QAT_DEFLATE: + header->service_cmd_id = ICP_QAT_FW_COMP_CMD_STATIC; + break; + default: + return -EINVAL; + } + + cd_pars->u.sl.comp_slice_cfg_word[0] = + ICP_QAT_HW_COMPRESSION_CONFIG_BUILD(ICP_QAT_HW_COMPRESSION_DIR_COMPRESS, + ICP_QAT_HW_COMPRESSION_DELAYED_MATCH_DISABLED, + ICP_QAT_HW_COMPRESSION_ALGO_DEFLATE, + ICP_QAT_HW_COMPRESSION_DEPTH_1, + ICP_QAT_HW_COMPRESSION_FILE_TYPE_0); + + return 0; +} + +static int adf_gen2_build_decomp_block(void *ctx, enum adf_dc_algo algo) +{ + struct icp_qat_fw_comp_req *req_tmpl = ctx; + struct icp_qat_fw_comp_req_hdr_cd_pars *cd_pars = &req_tmpl->cd_pars; + struct icp_qat_fw_comn_req_hdr *header = &req_tmpl->comn_hdr; + + switch (algo) { + case QAT_DEFLATE: + header->service_cmd_id = ICP_QAT_FW_COMP_CMD_DECOMPRESS; + break; + default: + return -EINVAL; + } + + cd_pars->u.sl.comp_slice_cfg_word[0] = + ICP_QAT_HW_COMPRESSION_CONFIG_BUILD(ICP_QAT_HW_COMPRESSION_DIR_DECOMPRESS, + ICP_QAT_HW_COMPRESSION_DELAYED_MATCH_DISABLED, + ICP_QAT_HW_COMPRESSION_ALGO_DEFLATE, + ICP_QAT_HW_COMPRESSION_DEPTH_1, + ICP_QAT_HW_COMPRESSION_FILE_TYPE_0); + + return 0; +} + +void adf_gen2_init_dc_ops(struct adf_dc_ops *dc_ops) +{ + dc_ops->build_comp_block = adf_gen2_build_comp_block; + dc_ops->build_decomp_block = adf_gen2_build_decomp_block; +} +EXPORT_SYMBOL_GPL(adf_gen2_init_dc_ops); diff --git a/drivers/crypto/intel/qat/qat_common/adf_gen2_hw_data.h b/drivers/crypto/intel/qat/qat_common/adf_gen2_hw_data.h index 708e9186127b..59bad368a921 100644 --- a/drivers/crypto/intel/qat/qat_common/adf_gen2_hw_data.h +++ b/drivers/crypto/intel/qat/qat_common/adf_gen2_hw_data.h @@ -88,5 +88,6 @@ void adf_gen2_get_arb_info(struct arb_info *arb_info); void adf_gen2_enable_ints(struct adf_accel_dev *accel_dev); u32 adf_gen2_get_accel_cap(struct adf_accel_dev *accel_dev); void adf_gen2_set_ssm_wdtimer(struct adf_accel_dev *accel_dev); +void adf_gen2_init_dc_ops(struct adf_dc_ops *dc_ops); #endif diff --git a/drivers/crypto/intel/qat/qat_common/adf_gen4_dc.c b/drivers/crypto/intel/qat/qat_common/adf_gen4_dc.c deleted file mode 100644 index 5859238e37de..000000000000 --- a/drivers/crypto/intel/qat/qat_common/adf_gen4_dc.c +++ /dev/null @@ -1,83 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-only -/* Copyright(c) 2022 Intel Corporation */ -#include "adf_accel_devices.h" -#include "icp_qat_fw_comp.h" -#include "icp_qat_hw_20_comp.h" -#include "adf_gen4_dc.h" - -static void qat_comp_build_deflate(void *ctx) -{ - struct icp_qat_fw_comp_req *req_tmpl = - (struct icp_qat_fw_comp_req *)ctx; - struct icp_qat_fw_comn_req_hdr *header = &req_tmpl->comn_hdr; - struct icp_qat_fw_comp_req_hdr_cd_pars *cd_pars = &req_tmpl->cd_pars; - struct icp_qat_fw_comp_req_params *req_pars = &req_tmpl->comp_pars; - struct icp_qat_hw_comp_20_config_csr_upper hw_comp_upper_csr = {0}; - struct icp_qat_hw_comp_20_config_csr_lower hw_comp_lower_csr = {0}; - struct icp_qat_hw_decomp_20_config_csr_lower hw_decomp_lower_csr = {0}; - u32 upper_val; - u32 lower_val; - - memset(req_tmpl, 0, sizeof(*req_tmpl)); - header->hdr_flags = - ICP_QAT_FW_COMN_HDR_FLAGS_BUILD(ICP_QAT_FW_COMN_REQ_FLAG_SET); - header->service_type = ICP_QAT_FW_COMN_REQ_CPM_FW_COMP; - header->service_cmd_id = ICP_QAT_FW_COMP_CMD_STATIC; - header->comn_req_flags = - ICP_QAT_FW_COMN_FLAGS_BUILD(QAT_COMN_CD_FLD_TYPE_16BYTE_DATA, - QAT_COMN_PTR_TYPE_SGL); - header->serv_specif_flags = - ICP_QAT_FW_COMP_FLAGS_BUILD(ICP_QAT_FW_COMP_STATELESS_SESSION, - ICP_QAT_FW_COMP_AUTO_SELECT_BEST, - ICP_QAT_FW_COMP_NOT_ENH_AUTO_SELECT_BEST, - ICP_QAT_FW_COMP_NOT_DISABLE_TYPE0_ENH_AUTO_SELECT_BEST, - ICP_QAT_FW_COMP_ENABLE_SECURE_RAM_USED_AS_INTMD_BUF); - hw_comp_lower_csr.skip_ctrl = ICP_QAT_HW_COMP_20_BYTE_SKIP_3BYTE_LITERAL; - hw_comp_lower_csr.algo = ICP_QAT_HW_COMP_20_HW_COMP_FORMAT_ILZ77; - hw_comp_lower_csr.lllbd = ICP_QAT_HW_COMP_20_LLLBD_CTRL_LLLBD_ENABLED; - hw_comp_lower_csr.sd = ICP_QAT_HW_COMP_20_SEARCH_DEPTH_LEVEL_1; - hw_comp_lower_csr.hash_update = ICP_QAT_HW_COMP_20_SKIP_HASH_UPDATE_DONT_ALLOW; - hw_comp_lower_csr.edmm = ICP_QAT_HW_COMP_20_EXTENDED_DELAY_MATCH_MODE_EDMM_ENABLED; - hw_comp_upper_csr.nice = ICP_QAT_HW_COMP_20_CONFIG_CSR_NICE_PARAM_DEFAULT_VAL; - hw_comp_upper_csr.lazy = ICP_QAT_HW_COMP_20_CONFIG_CSR_LAZY_PARAM_DEFAULT_VAL; - - upper_val = ICP_QAT_FW_COMP_20_BUILD_CONFIG_UPPER(hw_comp_upper_csr); - lower_val = ICP_QAT_FW_COMP_20_BUILD_CONFIG_LOWER(hw_comp_lower_csr); - - cd_pars->u.sl.comp_slice_cfg_word[0] = lower_val; - cd_pars->u.sl.comp_slice_cfg_word[1] = upper_val; - - req_pars->crc.legacy.initial_adler = COMP_CPR_INITIAL_ADLER; - req_pars->crc.legacy.initial_crc32 = COMP_CPR_INITIAL_CRC; - req_pars->req_par_flags = - ICP_QAT_FW_COMP_REQ_PARAM_FLAGS_BUILD(ICP_QAT_FW_COMP_SOP, - ICP_QAT_FW_COMP_EOP, - ICP_QAT_FW_COMP_BFINAL, - ICP_QAT_FW_COMP_CNV, - ICP_QAT_FW_COMP_CNV_RECOVERY, - ICP_QAT_FW_COMP_NO_CNV_DFX, - ICP_QAT_FW_COMP_CRC_MODE_LEGACY, - ICP_QAT_FW_COMP_NO_XXHASH_ACC, - ICP_QAT_FW_COMP_CNV_ERROR_NONE, - ICP_QAT_FW_COMP_NO_APPEND_CRC, - ICP_QAT_FW_COMP_NO_DROP_DATA); - - /* Fill second half of the template for decompression */ - memcpy(req_tmpl + 1, req_tmpl, sizeof(*req_tmpl)); - req_tmpl++; - header = &req_tmpl->comn_hdr; - header->service_cmd_id = ICP_QAT_FW_COMP_CMD_DECOMPRESS; - cd_pars = &req_tmpl->cd_pars; - - hw_decomp_lower_csr.algo = ICP_QAT_HW_DECOMP_20_HW_DECOMP_FORMAT_DEFLATE; - lower_val = ICP_QAT_FW_DECOMP_20_BUILD_CONFIG_LOWER(hw_decomp_lower_csr); - - cd_pars->u.sl.comp_slice_cfg_word[0] = lower_val; - cd_pars->u.sl.comp_slice_cfg_word[1] = 0; -} - -void adf_gen4_init_dc_ops(struct adf_dc_ops *dc_ops) -{ - dc_ops->build_deflate_ctx = qat_comp_build_deflate; -} -EXPORT_SYMBOL_GPL(adf_gen4_init_dc_ops); diff --git a/drivers/crypto/intel/qat/qat_common/adf_gen4_dc.h b/drivers/crypto/intel/qat/qat_common/adf_gen4_dc.h deleted file mode 100644 index 0b1a6774412e..000000000000 --- a/drivers/crypto/intel/qat/qat_common/adf_gen4_dc.h +++ /dev/null @@ -1,10 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0-only */ -/* Copyright(c) 2022 Intel Corporation */ -#ifndef ADF_GEN4_DC_H -#define ADF_GEN4_DC_H - -#include "adf_accel_devices.h" - -void adf_gen4_init_dc_ops(struct adf_dc_ops *dc_ops); - -#endif /* ADF_GEN4_DC_H */ diff --git a/drivers/crypto/intel/qat/qat_common/adf_gen4_hw_data.c b/drivers/crypto/intel/qat/qat_common/adf_gen4_hw_data.c index 099949a2421c..0406cb09c5bb 100644 --- a/drivers/crypto/intel/qat/qat_common/adf_gen4_hw_data.c +++ b/drivers/crypto/intel/qat/qat_common/adf_gen4_hw_data.c @@ -9,6 +9,8 @@ #include "adf_fw_config.h" #include "adf_gen4_hw_data.h" #include "adf_gen4_pm.h" +#include "icp_qat_fw_comp.h" +#include "icp_qat_hw_20_comp.h" u32 adf_gen4_get_accel_mask(struct adf_hw_device_data *self) { @@ -663,3 +665,71 @@ int adf_gen4_bank_state_restore(struct adf_accel_dev *accel_dev, u32 bank_number return ret; } EXPORT_SYMBOL_GPL(adf_gen4_bank_state_restore); + +static int adf_gen4_build_comp_block(void *ctx, enum adf_dc_algo algo) +{ + struct icp_qat_fw_comp_req *req_tmpl = ctx; + struct icp_qat_fw_comp_req_hdr_cd_pars *cd_pars = &req_tmpl->cd_pars; + struct icp_qat_hw_comp_20_config_csr_upper hw_comp_upper_csr = { }; + struct icp_qat_hw_comp_20_config_csr_lower hw_comp_lower_csr = { }; + struct icp_qat_fw_comn_req_hdr *header = &req_tmpl->comn_hdr; + u32 upper_val; + u32 lower_val; + + switch (algo) { + case QAT_DEFLATE: + header->service_cmd_id = ICP_QAT_FW_COMP_CMD_DYNAMIC; + break; + default: + return -EINVAL; + } + + hw_comp_lower_csr.skip_ctrl = ICP_QAT_HW_COMP_20_BYTE_SKIP_3BYTE_LITERAL; + hw_comp_lower_csr.algo = ICP_QAT_HW_COMP_20_HW_COMP_FORMAT_ILZ77; + hw_comp_lower_csr.lllbd = ICP_QAT_HW_COMP_20_LLLBD_CTRL_LLLBD_ENABLED; + hw_comp_lower_csr.sd = ICP_QAT_HW_COMP_20_SEARCH_DEPTH_LEVEL_1; + hw_comp_lower_csr.hash_update = ICP_QAT_HW_COMP_20_SKIP_HASH_UPDATE_DONT_ALLOW; + hw_comp_lower_csr.edmm = ICP_QAT_HW_COMP_20_EXTENDED_DELAY_MATCH_MODE_EDMM_ENABLED; + hw_comp_upper_csr.nice = ICP_QAT_HW_COMP_20_CONFIG_CSR_NICE_PARAM_DEFAULT_VAL; + hw_comp_upper_csr.lazy = ICP_QAT_HW_COMP_20_CONFIG_CSR_LAZY_PARAM_DEFAULT_VAL; + + upper_val = ICP_QAT_FW_COMP_20_BUILD_CONFIG_UPPER(hw_comp_upper_csr); + lower_val = ICP_QAT_FW_COMP_20_BUILD_CONFIG_LOWER(hw_comp_lower_csr); + + cd_pars->u.sl.comp_slice_cfg_word[0] = lower_val; + cd_pars->u.sl.comp_slice_cfg_word[1] = upper_val; + + return 0; +} + +static int adf_gen4_build_decomp_block(void *ctx, enum adf_dc_algo algo) +{ + struct icp_qat_fw_comp_req *req_tmpl = ctx; + struct icp_qat_hw_decomp_20_config_csr_lower hw_decomp_lower_csr = { }; + struct icp_qat_fw_comp_req_hdr_cd_pars *cd_pars = &req_tmpl->cd_pars; + struct icp_qat_fw_comn_req_hdr *header = &req_tmpl->comn_hdr; + u32 lower_val; + + switch (algo) { + case QAT_DEFLATE: + header->service_cmd_id = ICP_QAT_FW_COMP_CMD_DECOMPRESS; + break; + default: + return -EINVAL; + } + + hw_decomp_lower_csr.algo = ICP_QAT_HW_DECOMP_20_HW_DECOMP_FORMAT_DEFLATE; + lower_val = ICP_QAT_FW_DECOMP_20_BUILD_CONFIG_LOWER(hw_decomp_lower_csr); + + cd_pars->u.sl.comp_slice_cfg_word[0] = lower_val; + cd_pars->u.sl.comp_slice_cfg_word[1] = 0; + + return 0; +} + +void adf_gen4_init_dc_ops(struct adf_dc_ops *dc_ops) +{ + dc_ops->build_comp_block = adf_gen4_build_comp_block; + dc_ops->build_decomp_block = adf_gen4_build_decomp_block; +} +EXPORT_SYMBOL_GPL(adf_gen4_init_dc_ops); diff --git a/drivers/crypto/intel/qat/qat_common/adf_gen4_hw_data.h b/drivers/crypto/intel/qat/qat_common/adf_gen4_hw_data.h index 51fc2eaa263e..e4f4d5fa616d 100644 --- a/drivers/crypto/intel/qat/qat_common/adf_gen4_hw_data.h +++ b/drivers/crypto/intel/qat/qat_common/adf_gen4_hw_data.h @@ -7,6 +7,7 @@ #include "adf_accel_devices.h" #include "adf_cfg_common.h" +#include "adf_dc.h" /* PCIe configuration space */ #define ADF_GEN4_BAR_MASK (BIT(0) | BIT(2) | BIT(4)) @@ -180,5 +181,6 @@ int adf_gen4_bank_state_save(struct adf_accel_dev *accel_dev, u32 bank_number, int adf_gen4_bank_state_restore(struct adf_accel_dev *accel_dev, u32 bank_number, struct bank_state *state); bool adf_gen4_services_supported(unsigned long service_mask); +void adf_gen4_init_dc_ops(struct adf_dc_ops *dc_ops); #endif diff --git a/drivers/crypto/intel/qat/qat_common/qat_comp_algs.c b/drivers/crypto/intel/qat/qat_common/qat_comp_algs.c index a0a29b97a749..8b123472b71c 100644 --- a/drivers/crypto/intel/qat/qat_common/qat_comp_algs.c +++ b/drivers/crypto/intel/qat/qat_common/qat_comp_algs.c @@ -8,6 +8,7 @@ #include #include "adf_accel_devices.h" #include "adf_common_drv.h" +#include "adf_dc.h" #include "qat_bl.h" #include "qat_comp_req.h" #include "qat_compression.h" @@ -145,9 +146,7 @@ static int qat_comp_alg_init_tfm(struct crypto_acomp *acomp_tfm) return -EINVAL; ctx->inst = inst; - ctx->inst->build_deflate_ctx(ctx->comp_ctx); - - return 0; + return qat_comp_build_ctx(inst->accel_dev, ctx->comp_ctx, QAT_DEFLATE); } static void qat_comp_alg_exit_tfm(struct crypto_acomp *acomp_tfm) diff --git a/drivers/crypto/intel/qat/qat_common/qat_compression.c b/drivers/crypto/intel/qat/qat_common/qat_compression.c index 7842a9f22178..c285b45b8679 100644 --- a/drivers/crypto/intel/qat/qat_common/qat_compression.c +++ b/drivers/crypto/intel/qat/qat_common/qat_compression.c @@ -144,7 +144,6 @@ static int qat_compression_create_instances(struct adf_accel_dev *accel_dev) inst->id = i; atomic_set(&inst->refctr, 0); inst->accel_dev = accel_dev; - inst->build_deflate_ctx = GET_DC_OPS(accel_dev)->build_deflate_ctx; snprintf(key, sizeof(key), ADF_DC "%d" ADF_RING_DC_BANK_NUM, i); ret = adf_cfg_get_param_value(accel_dev, SEC, key, val); diff --git a/drivers/crypto/intel/qat/qat_common/qat_compression.h b/drivers/crypto/intel/qat/qat_common/qat_compression.h index aebac2302dcf..5ced3ed0e5ea 100644 --- a/drivers/crypto/intel/qat/qat_common/qat_compression.h +++ b/drivers/crypto/intel/qat/qat_common/qat_compression.h @@ -20,7 +20,6 @@ struct qat_compression_instance { atomic_t refctr; struct qat_instance_backlog backlog; struct adf_dc_data *dc_data; - void (*build_deflate_ctx)(void *ctx); }; static inline bool adf_hw_dev_has_compression(struct adf_accel_dev *accel_dev) diff --git a/drivers/crypto/intel/qat/qat_dh895xcc/adf_dh895xcc_hw_data.c b/drivers/crypto/intel/qat/qat_dh895xcc/adf_dh895xcc_hw_data.c index bf9e8f34f451..5b4bd0ba1ccb 100644 --- a/drivers/crypto/intel/qat/qat_dh895xcc/adf_dh895xcc_hw_data.c +++ b/drivers/crypto/intel/qat/qat_dh895xcc/adf_dh895xcc_hw_data.c @@ -4,7 +4,6 @@ #include #include #include -#include #include #include #include diff --git a/drivers/crypto/intel/qat/qat_dh895xccvf/adf_dh895xccvf_hw_data.c b/drivers/crypto/intel/qat/qat_dh895xccvf/adf_dh895xccvf_hw_data.c index bc59c1473eef..828456c43b76 100644 --- a/drivers/crypto/intel/qat/qat_dh895xccvf/adf_dh895xccvf_hw_data.c +++ b/drivers/crypto/intel/qat/qat_dh895xccvf/adf_dh895xccvf_hw_data.c @@ -3,7 +3,6 @@ #include #include #include -#include #include #include #include -- 2.50.1 From 7e673b80f420eb00660e01bf82befa8eaf652b28 Mon Sep 17 00:00:00 2001 From: Suman Kumar Chakraborty Date: Wed, 30 Apr 2025 12:34:45 +0100 Subject: [PATCH 13/16] crypto: qat - use pr_fmt() in qat uclo.c Add pr_fmt() to qat uclo.c logging and update the debug and error messages to utilize it accordingly. This does not introduce any functional changes. Signed-off-by: Suman Kumar Chakraborty Reviewed-by: Giovanni Cabiddu Reviewed-by: Andy Shevchenko Signed-off-by: Herbert Xu --- .../crypto/intel/qat/qat_common/qat_uclo.c | 135 +++++++++--------- 1 file changed, 65 insertions(+), 70 deletions(-) diff --git a/drivers/crypto/intel/qat/qat_common/qat_uclo.c b/drivers/crypto/intel/qat/qat_common/qat_uclo.c index 620300e70238..9dc26f5d2d01 100644 --- a/drivers/crypto/intel/qat/qat_common/qat_uclo.c +++ b/drivers/crypto/intel/qat/qat_common/qat_uclo.c @@ -1,5 +1,8 @@ // SPDX-License-Identifier: (BSD-3-Clause OR GPL-2.0-only) /* Copyright(c) 2014 - 2020 Intel Corporation */ + +#define pr_fmt(fmt) "QAT: " fmt + #include #include #include @@ -60,7 +63,7 @@ static int qat_uclo_free_ae_data(struct icp_qat_uclo_aedata *ae_data) unsigned int i; if (!ae_data) { - pr_err("QAT: bad argument, ae_data is NULL\n"); + pr_err("bad argument, ae_data is NULL\n"); return -EINVAL; } @@ -87,12 +90,11 @@ static int qat_uclo_check_uof_format(struct icp_qat_uof_filehdr *hdr) int min = hdr->min_ver & 0xff; if (hdr->file_id != ICP_QAT_UOF_FID) { - pr_err("QAT: Invalid header 0x%x\n", hdr->file_id); + pr_err("Invalid header 0x%x\n", hdr->file_id); return -EINVAL; } if (min != ICP_QAT_UOF_MINVER || maj != ICP_QAT_UOF_MAJVER) { - pr_err("QAT: bad UOF version, major 0x%x, minor 0x%x\n", - maj, min); + pr_err("bad UOF version, major 0x%x, minor 0x%x\n", maj, min); return -EINVAL; } return 0; @@ -104,20 +106,19 @@ static int qat_uclo_check_suof_format(struct icp_qat_suof_filehdr *suof_hdr) int min = suof_hdr->min_ver & 0xff; if (suof_hdr->file_id != ICP_QAT_SUOF_FID) { - pr_err("QAT: invalid header 0x%x\n", suof_hdr->file_id); + pr_err("invalid header 0x%x\n", suof_hdr->file_id); return -EINVAL; } if (suof_hdr->fw_type != 0) { - pr_err("QAT: unsupported firmware type\n"); + pr_err("unsupported firmware type\n"); return -EINVAL; } if (suof_hdr->num_chunks <= 0x1) { - pr_err("QAT: SUOF chunk amount is incorrect\n"); + pr_err("SUOF chunk amount is incorrect\n"); return -EINVAL; } if (maj != ICP_QAT_SUOF_MAJVER || min != ICP_QAT_SUOF_MINVER) { - pr_err("QAT: bad SUOF version, major 0x%x, minor 0x%x\n", - maj, min); + pr_err("bad SUOF version, major 0x%x, minor 0x%x\n", maj, min); return -EINVAL; } return 0; @@ -224,24 +225,24 @@ static int qat_uclo_fetch_initmem_ae(struct icp_qat_fw_loader_handle *handle, char *str; if ((init_mem->addr + init_mem->num_in_bytes) > (size_range << 0x2)) { - pr_err("QAT: initmem is out of range"); + pr_err("initmem is out of range"); return -EINVAL; } if (init_mem->scope != ICP_QAT_UOF_LOCAL_SCOPE) { - pr_err("QAT: Memory scope for init_mem error\n"); + pr_err("Memory scope for init_mem error\n"); return -EINVAL; } str = qat_uclo_get_string(&obj_handle->str_table, init_mem->sym_name); if (!str) { - pr_err("QAT: AE name assigned in UOF init table is NULL\n"); + pr_err("AE name assigned in UOF init table is NULL\n"); return -EINVAL; } if (qat_uclo_parse_num(str, ae)) { - pr_err("QAT: Parse num for AE number failed\n"); + pr_err("Parse num for AE number failed\n"); return -EINVAL; } if (*ae >= ICP_QAT_UCLO_MAX_AE) { - pr_err("QAT: ae %d out of range\n", *ae); + pr_err("ae %d out of range\n", *ae); return -EINVAL; } return 0; @@ -357,8 +358,7 @@ static int qat_uclo_init_ae_memory(struct icp_qat_fw_loader_handle *handle, return -EINVAL; break; default: - pr_err("QAT: initmem region error. region type=0x%x\n", - init_mem->region); + pr_err("initmem region error. region type=0x%x\n", init_mem->region); return -EINVAL; } return 0; @@ -432,7 +432,7 @@ static int qat_uclo_init_memory(struct icp_qat_fw_loader_handle *handle) for_each_set_bit(ae, &ae_mask, handle->hal_handle->ae_max_num) { if (qat_hal_batch_wr_lm(handle, ae, obj_handle->lm_init_tab[ae])) { - pr_err("QAT: fail to batch init lmem for AE %d\n", ae); + pr_err("fail to batch init lmem for AE %d\n", ae); return -EINVAL; } qat_uclo_cleanup_batch_init_list(handle, @@ -540,26 +540,26 @@ qat_uclo_check_image_compat(struct icp_qat_uof_encap_obj *encap_uof_obj, code_page->imp_expr_tab_offset); if (uc_var_tab->entry_num || imp_var_tab->entry_num || imp_expr_tab->entry_num) { - pr_err("QAT: UOF can't contain imported variable to be parsed\n"); + pr_err("UOF can't contain imported variable to be parsed\n"); return -EINVAL; } neigh_reg_tab = (struct icp_qat_uof_objtable *) (encap_uof_obj->beg_uof + code_page->neigh_reg_tab_offset); if (neigh_reg_tab->entry_num) { - pr_err("QAT: UOF can't contain neighbor register table\n"); + pr_err("UOF can't contain neighbor register table\n"); return -EINVAL; } if (image->numpages > 1) { - pr_err("QAT: UOF can't contain multiple pages\n"); + pr_err("UOF can't contain multiple pages\n"); return -EINVAL; } if (ICP_QAT_SHARED_USTORE_MODE(image->ae_mode)) { - pr_err("QAT: UOF can't use shared control store feature\n"); + pr_err("UOF can't use shared control store feature\n"); return -EFAULT; } if (RELOADABLE_CTX_SHARED_MODE(image->ae_mode)) { - pr_err("QAT: UOF can't use reloadable feature\n"); + pr_err("UOF can't use reloadable feature\n"); return -EFAULT; } return 0; @@ -678,7 +678,7 @@ static int qat_uclo_map_ae(struct icp_qat_fw_loader_handle *handle, int max_ae) } } if (!mflag) { - pr_err("QAT: uimage uses AE not set\n"); + pr_err("uimage uses AE not set\n"); return -EINVAL; } return 0; @@ -738,8 +738,7 @@ qat_uclo_get_dev_type(struct icp_qat_fw_loader_handle *handle) case PCI_DEVICE_ID_INTEL_QAT_420XX: return ICP_QAT_AC_4XXX_A_DEV_TYPE; default: - pr_err("QAT: unsupported device 0x%x\n", - handle->pci_dev->device); + pr_err("unsupported device 0x%x\n", handle->pci_dev->device); return 0; } } @@ -749,7 +748,7 @@ static int qat_uclo_check_uof_compat(struct icp_qat_uclo_objhandle *obj_handle) unsigned int maj_ver, prod_type = obj_handle->prod_type; if (!(prod_type & obj_handle->encap_uof_obj.obj_hdr->ac_dev_type)) { - pr_err("QAT: UOF type 0x%x doesn't match with platform 0x%x\n", + pr_err("UOF type 0x%x doesn't match with platform 0x%x\n", obj_handle->encap_uof_obj.obj_hdr->ac_dev_type, prod_type); return -EINVAL; @@ -757,7 +756,7 @@ static int qat_uclo_check_uof_compat(struct icp_qat_uclo_objhandle *obj_handle) maj_ver = obj_handle->prod_rev & 0xff; if (obj_handle->encap_uof_obj.obj_hdr->max_cpu_ver < maj_ver || obj_handle->encap_uof_obj.obj_hdr->min_cpu_ver > maj_ver) { - pr_err("QAT: UOF majVer 0x%x out of range\n", maj_ver); + pr_err("UOF majVer 0x%x out of range\n", maj_ver); return -EINVAL; } return 0; @@ -800,7 +799,7 @@ static int qat_uclo_init_reg(struct icp_qat_fw_loader_handle *handle, case ICP_NEIGH_REL: return qat_hal_init_nn(handle, ae, ctx_mask, reg_addr, value); default: - pr_err("QAT: UOF uses not supported reg type 0x%x\n", reg_type); + pr_err("UOF uses not supported reg type 0x%x\n", reg_type); return -EFAULT; } return 0; @@ -836,8 +835,7 @@ static int qat_uclo_init_reg_sym(struct icp_qat_fw_loader_handle *handle, case ICP_QAT_UOF_INIT_REG_CTX: /* check if ctx is appropriate for the ctxMode */ if (!((1 << init_regsym->ctx) & ctx_mask)) { - pr_err("QAT: invalid ctx num = 0x%x\n", - init_regsym->ctx); + pr_err("invalid ctx num = 0x%x\n", init_regsym->ctx); return -EINVAL; } qat_uclo_init_reg(handle, ae, @@ -849,10 +847,10 @@ static int qat_uclo_init_reg_sym(struct icp_qat_fw_loader_handle *handle, exp_res); break; case ICP_QAT_UOF_INIT_EXPR: - pr_err("QAT: INIT_EXPR feature not supported\n"); + pr_err("INIT_EXPR feature not supported\n"); return -EINVAL; case ICP_QAT_UOF_INIT_EXPR_ENDIAN_SWAP: - pr_err("QAT: INIT_EXPR_ENDIAN_SWAP feature not supported\n"); + pr_err("INIT_EXPR_ENDIAN_SWAP feature not supported\n"); return -EINVAL; default: break; @@ -872,7 +870,7 @@ static int qat_uclo_init_globals(struct icp_qat_fw_loader_handle *handle) return 0; if (obj_handle->init_mem_tab.entry_num) { if (qat_uclo_init_memory(handle)) { - pr_err("QAT: initialize memory failed\n"); + pr_err("initialize memory failed\n"); return -EINVAL; } } @@ -901,40 +899,40 @@ static int qat_hal_set_modes(struct icp_qat_fw_loader_handle *handle, mode = ICP_QAT_CTX_MODE(uof_image->ae_mode); ret = qat_hal_set_ae_ctx_mode(handle, ae, mode); if (ret) { - pr_err("QAT: qat_hal_set_ae_ctx_mode error\n"); + pr_err("qat_hal_set_ae_ctx_mode error\n"); return ret; } if (handle->chip_info->nn) { mode = ICP_QAT_NN_MODE(uof_image->ae_mode); ret = qat_hal_set_ae_nn_mode(handle, ae, mode); if (ret) { - pr_err("QAT: qat_hal_set_ae_nn_mode error\n"); + pr_err("qat_hal_set_ae_nn_mode error\n"); return ret; } } mode = ICP_QAT_LOC_MEM0_MODE(uof_image->ae_mode); ret = qat_hal_set_ae_lm_mode(handle, ae, ICP_LMEM0, mode); if (ret) { - pr_err("QAT: qat_hal_set_ae_lm_mode LMEM0 error\n"); + pr_err("qat_hal_set_ae_lm_mode LMEM0 error\n"); return ret; } mode = ICP_QAT_LOC_MEM1_MODE(uof_image->ae_mode); ret = qat_hal_set_ae_lm_mode(handle, ae, ICP_LMEM1, mode); if (ret) { - pr_err("QAT: qat_hal_set_ae_lm_mode LMEM1 error\n"); + pr_err("qat_hal_set_ae_lm_mode LMEM1 error\n"); return ret; } if (handle->chip_info->lm2lm3) { mode = ICP_QAT_LOC_MEM2_MODE(uof_image->ae_mode); ret = qat_hal_set_ae_lm_mode(handle, ae, ICP_LMEM2, mode); if (ret) { - pr_err("QAT: qat_hal_set_ae_lm_mode LMEM2 error\n"); + pr_err("qat_hal_set_ae_lm_mode LMEM2 error\n"); return ret; } mode = ICP_QAT_LOC_MEM3_MODE(uof_image->ae_mode); ret = qat_hal_set_ae_lm_mode(handle, ae, ICP_LMEM3, mode); if (ret) { - pr_err("QAT: qat_hal_set_ae_lm_mode LMEM3 error\n"); + pr_err("qat_hal_set_ae_lm_mode LMEM3 error\n"); return ret; } mode = ICP_QAT_LOC_TINDEX_MODE(uof_image->ae_mode); @@ -998,7 +996,7 @@ static int qat_uclo_parse_uof_obj(struct icp_qat_fw_loader_handle *handle) obj_handle->prod_rev = PID_MAJOR_REV | (PID_MINOR_REV & handle->hal_handle->revision_id); if (qat_uclo_check_uof_compat(obj_handle)) { - pr_err("QAT: UOF incompatible\n"); + pr_err("UOF incompatible\n"); return -EINVAL; } obj_handle->uword_buf = kcalloc(UWORD_CPYBUF_SIZE, sizeof(u64), @@ -1009,7 +1007,7 @@ static int qat_uclo_parse_uof_obj(struct icp_qat_fw_loader_handle *handle) if (!obj_handle->obj_hdr->file_buff || !qat_uclo_map_str_table(obj_handle->obj_hdr, ICP_QAT_UOF_STRT, &obj_handle->str_table)) { - pr_err("QAT: UOF doesn't have effective images\n"); + pr_err("UOF doesn't have effective images\n"); goto out_err; } obj_handle->uimage_num = @@ -1018,7 +1016,7 @@ static int qat_uclo_parse_uof_obj(struct icp_qat_fw_loader_handle *handle) if (!obj_handle->uimage_num) goto out_err; if (qat_uclo_map_ae(handle, handle->hal_handle->ae_max_num)) { - pr_err("QAT: Bad object\n"); + pr_err("Bad object\n"); goto out_check_uof_aemask_err; } qat_uclo_init_uword_num(handle); @@ -1051,7 +1049,7 @@ static int qat_uclo_map_suof_file_hdr(struct icp_qat_fw_loader_handle *handle, check_sum = qat_uclo_calc_str_checksum((char *)&suof_ptr->min_ver, min_ver_offset); if (check_sum != suof_ptr->check_sum) { - pr_err("QAT: incorrect SUOF checksum\n"); + pr_err("incorrect SUOF checksum\n"); return -EINVAL; } suof_handle->check_sum = suof_ptr->check_sum; @@ -1113,14 +1111,13 @@ static int qat_uclo_check_simg_compat(struct icp_qat_fw_loader_handle *handle, prod_rev = PID_MAJOR_REV | (PID_MINOR_REV & handle->hal_handle->revision_id); if (img_ae_mode->dev_type != prod_type) { - pr_err("QAT: incompatible product type %x\n", - img_ae_mode->dev_type); + pr_err("incompatible product type %x\n", img_ae_mode->dev_type); return -EINVAL; } maj_ver = prod_rev & 0xff; if (maj_ver > img_ae_mode->devmax_ver || maj_ver < img_ae_mode->devmin_ver) { - pr_err("QAT: incompatible device majver 0x%x\n", maj_ver); + pr_err("incompatible device majver 0x%x\n", maj_ver); return -EINVAL; } return 0; @@ -1163,7 +1160,7 @@ static int qat_uclo_map_suof(struct icp_qat_fw_loader_handle *handle, struct icp_qat_suof_img_hdr img_header; if (!suof_ptr || suof_size == 0) { - pr_err("QAT: input parameter SUOF pointer/size is NULL\n"); + pr_err("input parameter SUOF pointer/size is NULL\n"); return -EINVAL; } if (qat_uclo_check_suof_format(suof_ptr)) @@ -1237,7 +1234,7 @@ static int qat_uclo_auth_fw(struct icp_qat_fw_loader_handle *handle, return 0; } while (retry++ < FW_AUTH_MAX_RETRY); auth_fail: - pr_err("QAT: authentication error (FCU_STATUS = 0x%x),retry = %d\n", + pr_err("authentication error (FCU_STATUS = 0x%x),retry = %d\n", fcu_sts & FCU_AUTH_STS_MASK, retry); return -EINVAL; } @@ -1273,14 +1270,13 @@ static int qat_uclo_broadcast_load_fw(struct icp_qat_fw_loader_handle *handle, fcu_sts_csr = handle->chip_info->fcu_sts_csr; fcu_loaded_csr = handle->chip_info->fcu_loaded_ae_csr; } else { - pr_err("Chip 0x%x doesn't support broadcast load\n", - handle->pci_dev->device); + pr_err("Chip 0x%x doesn't support broadcast load\n", handle->pci_dev->device); return -EINVAL; } for_each_set_bit(ae, &ae_mask, handle->hal_handle->ae_max_num) { if (qat_hal_check_ae_active(handle, (unsigned char)ae)) { - pr_err("QAT: Broadcast load failed. AE is not enabled or active.\n"); + pr_err("Broadcast load failed. AE is not enabled or active.\n"); return -EINVAL; } @@ -1312,7 +1308,7 @@ static int qat_uclo_broadcast_load_fw(struct icp_qat_fw_loader_handle *handle, } while (retry++ < FW_AUTH_MAX_RETRY); if (retry > FW_AUTH_MAX_RETRY) { - pr_err("QAT: broadcast load failed timeout %d\n", retry); + pr_err("broadcast load failed timeout %d\n", retry); return -EINVAL; } } @@ -1397,13 +1393,13 @@ static int qat_uclo_check_image(struct icp_qat_fw_loader_handle *handle, if (size > ICP_QAT_CSS_RSA3K_MAX_IMAGE_LEN) goto err; } else { - pr_err("QAT: Unsupported firmware type\n"); + pr_err("Unsupported firmware type\n"); return -EINVAL; } return 0; err: - pr_err("QAT: Invalid %s firmware image\n", fw_type_name); + pr_err("Invalid %s firmware image\n", fw_type_name); return -EINVAL; } @@ -1422,12 +1418,12 @@ static int qat_uclo_map_auth_fw(struct icp_qat_fw_loader_handle *handle, ret = qat_uclo_simg_alloc(handle, &img_desc, ICP_QAT_CSS_RSA4K_MAX_IMAGE_LEN); if (ret) { - pr_err("QAT: error, allocate continuous dram fail\n"); + pr_err("error, allocate continuous dram fail\n"); return ret; } if (!IS_ALIGNED(img_desc.dram_size, 8) || !img_desc.dram_bus_addr) { - pr_debug("QAT: invalid address\n"); + pr_debug("invalid address\n"); qat_uclo_simg_free(handle, &img_desc); return -EINVAL; } @@ -1489,7 +1485,7 @@ static int qat_uclo_map_auth_fw(struct icp_qat_fw_loader_handle *handle, auth_desc->img_len = size - ICP_QAT_AE_IMG_OFFSET(handle); if (bus_addr + auth_desc->img_len > img_desc.dram_bus_addr + ICP_QAT_CSS_RSA4K_MAX_IMAGE_LEN) { - pr_err("QAT: insufficient memory size for authentication data\n"); + pr_err("insufficient memory size for authentication data\n"); qat_uclo_simg_free(handle, &img_desc); return -ENOMEM; } @@ -1546,7 +1542,7 @@ static int qat_uclo_load_fw(struct icp_qat_fw_loader_handle *handle, if (!((desc->ae_mask >> i) & 0x1)) continue; if (qat_hal_check_ae_active(handle, i)) { - pr_err("QAT: AE %d is active\n", i); + pr_err("AE %d is active\n", i); return -EINVAL; } SET_CAP_CSR(handle, fcu_ctl_csr, @@ -1566,7 +1562,7 @@ static int qat_uclo_load_fw(struct icp_qat_fw_loader_handle *handle, } } while (retry++ < FW_AUTH_MAX_RETRY); if (retry > FW_AUTH_MAX_RETRY) { - pr_err("QAT: firmware load failed timeout %x\n", retry); + pr_err("firmware load failed timeout %x\n", retry); return -EINVAL; } } @@ -1584,7 +1580,7 @@ static int qat_uclo_map_suof_obj(struct icp_qat_fw_loader_handle *handle, handle->sobj_handle = suof_handle; if (qat_uclo_map_suof(handle, addr_ptr, mem_size)) { qat_uclo_del_suof(handle); - pr_err("QAT: map SUOF failed\n"); + pr_err("map SUOF failed\n"); return -EINVAL; } return 0; @@ -1608,7 +1604,7 @@ int qat_uclo_wr_mimage(struct icp_qat_fw_loader_handle *handle, qat_uclo_ummap_auth_fw(handle, &desc); } else { if (handle->chip_info->mmp_sram_size < mem_size) { - pr_err("QAT: MMP size is too large: 0x%x\n", mem_size); + pr_err("MMP size is too large: 0x%x\n", mem_size); return -EFBIG; } qat_uclo_wr_sram_by_words(handle, 0, addr_ptr, mem_size); @@ -1634,7 +1630,7 @@ static int qat_uclo_map_uof_obj(struct icp_qat_fw_loader_handle *handle, objhdl->obj_hdr = qat_uclo_map_chunk((char *)objhdl->obj_buf, filehdr, ICP_QAT_UOF_OBJS); if (!objhdl->obj_hdr) { - pr_err("QAT: object file chunk is null\n"); + pr_err("object file chunk is null\n"); goto out_objhdr_err; } handle->obj_handle = objhdl; @@ -1669,7 +1665,7 @@ static int qat_uclo_map_mof_file_hdr(struct icp_qat_fw_loader_handle *handle, checksum = qat_uclo_calc_str_checksum(&mof_ptr->min_ver, min_ver_offset); if (checksum != mof_ptr->checksum) { - pr_err("QAT: incorrect MOF checksum\n"); + pr_err("incorrect MOF checksum\n"); return -EINVAL; } @@ -1705,7 +1701,7 @@ static int qat_uclo_seek_obj_inside_mof(struct icp_qat_mof_handle *mobj_handle, } } - pr_err("QAT: object %s is not found inside MOF\n", obj_name); + pr_err("object %s is not found inside MOF\n", obj_name); return -EINVAL; } @@ -1722,7 +1718,7 @@ static int qat_uclo_map_obj_from_mof(struct icp_qat_mof_handle *mobj_handle, ICP_QAT_MOF_OBJ_CHUNKID_LEN)) { obj = mobj_handle->sobjs_hdr + obj_chunkhdr->offset; } else { - pr_err("QAT: unsupported chunk id\n"); + pr_err("unsupported chunk id\n"); return -EINVAL; } mobj_hdr->obj_buf = obj; @@ -1783,7 +1779,7 @@ static int qat_uclo_map_objs_from_mof(struct icp_qat_mof_handle *mobj_handle) } if ((uobj_chunk_num + sobj_chunk_num) != *valid_chunk) { - pr_err("QAT: inconsistent UOF/SUOF chunk amount\n"); + pr_err("inconsistent UOF/SUOF chunk amount\n"); return -EINVAL; } return 0; @@ -1824,17 +1820,16 @@ static int qat_uclo_check_mof_format(struct icp_qat_mof_file_hdr *mof_hdr) int min = mof_hdr->min_ver & 0xff; if (mof_hdr->file_id != ICP_QAT_MOF_FID) { - pr_err("QAT: invalid header 0x%x\n", mof_hdr->file_id); + pr_err("invalid header 0x%x\n", mof_hdr->file_id); return -EINVAL; } if (mof_hdr->num_chunks <= 0x1) { - pr_err("QAT: MOF chunk amount is incorrect\n"); + pr_err("MOF chunk amount is incorrect\n"); return -EINVAL; } if (maj != ICP_QAT_MOF_MAJVER || min != ICP_QAT_MOF_MINVER) { - pr_err("QAT: bad MOF version, major 0x%x, minor 0x%x\n", - maj, min); + pr_err("bad MOF version, major 0x%x, minor 0x%x\n", maj, min); return -EINVAL; } return 0; -- 2.50.1 From 98943958a4895299c4f54719f4ef34b950a2ca4c Mon Sep 17 00:00:00 2001 From: Jack Xu Date: Wed, 30 Apr 2025 12:34:46 +0100 Subject: [PATCH 14/16] crypto: qat - refactor FW signing algorithm The current implementation is designed to support single FW signing authentication only. Refactor the implementation to support other FW signing methods. This does not include any functional change. Co-developed-by: Suman Kumar Chakraborty Signed-off-by: Suman Kumar Chakraborty Signed-off-by: Jack Xu Reviewed-by: Giovanni Cabiddu Signed-off-by: Herbert Xu --- .../crypto/intel/qat/qat_common/qat_uclo.c | 154 ++++++++++-------- 1 file changed, 84 insertions(+), 70 deletions(-) diff --git a/drivers/crypto/intel/qat/qat_common/qat_uclo.c b/drivers/crypto/intel/qat/qat_common/qat_uclo.c index 9dc26f5d2d01..d7f2ceb81f1f 100644 --- a/drivers/crypto/intel/qat/qat_common/qat_uclo.c +++ b/drivers/crypto/intel/qat/qat_common/qat_uclo.c @@ -1033,6 +1033,23 @@ out_err: return -EFAULT; } +static unsigned int qat_uclo_simg_hdr2sign_len(struct icp_qat_fw_loader_handle *handle) +{ + return ICP_QAT_AE_IMG_OFFSET(handle); +} + +static unsigned int qat_uclo_simg_hdr2cont_len(struct icp_qat_fw_loader_handle *handle) +{ + return ICP_QAT_AE_IMG_OFFSET(handle); +} + +static unsigned int qat_uclo_simg_fw_type(struct icp_qat_fw_loader_handle *handle, void *img_ptr) +{ + struct icp_qat_css_hdr *hdr = img_ptr; + + return hdr->fw_type; +} + static int qat_uclo_map_suof_file_hdr(struct icp_qat_fw_loader_handle *handle, struct icp_qat_suof_filehdr *suof_ptr, int suof_size) @@ -1064,9 +1081,9 @@ static void qat_uclo_map_simg(struct icp_qat_fw_loader_handle *handle, struct icp_qat_suof_chunk_hdr *suof_chunk_hdr) { struct icp_qat_suof_handle *suof_handle = handle->sobj_handle; - unsigned int offset = ICP_QAT_AE_IMG_OFFSET(handle); - struct icp_qat_simg_ae_mode *ae_mode; + unsigned int offset = qat_uclo_simg_hdr2cont_len(handle); struct icp_qat_suof_objhdr *suof_objhdr; + struct icp_qat_simg_ae_mode *ae_mode; suof_img_hdr->simg_buf = (suof_handle->suof_buf + suof_chunk_hdr->offset + @@ -1362,21 +1379,24 @@ static void qat_uclo_ummap_auth_fw(struct icp_qat_fw_loader_handle *handle, } static int qat_uclo_check_image(struct icp_qat_fw_loader_handle *handle, - char *image, unsigned int size, + void *image, unsigned int size, unsigned int fw_type) { char *fw_type_name = fw_type ? "MMP" : "AE"; unsigned int css_dword_size = sizeof(u32); + unsigned int header_len, simg_type; + struct icp_qat_css_hdr *css_hdr; if (handle->chip_info->fw_auth) { - struct icp_qat_css_hdr *css_hdr = (struct icp_qat_css_hdr *)image; - unsigned int header_len = ICP_QAT_AE_IMG_OFFSET(handle); + header_len = qat_uclo_simg_hdr2sign_len(handle); + simg_type = qat_uclo_simg_fw_type(handle, image); + css_hdr = image; if ((css_hdr->header_len * css_dword_size) != header_len) goto err; if ((css_hdr->size * css_dword_size) != size) goto err; - if (fw_type != css_hdr->fw_type) + if (fw_type != simg_type) goto err; if (size <= header_len) goto err; @@ -1403,113 +1423,85 @@ err: return -EINVAL; } -static int qat_uclo_map_auth_fw(struct icp_qat_fw_loader_handle *handle, - char *image, unsigned int size, - struct icp_qat_fw_auth_desc **desc) +static int qat_uclo_build_auth_desc_RSA(struct icp_qat_fw_loader_handle *handle, + char *image, unsigned int size, + struct icp_firml_dram_desc *dram_desc, + unsigned int fw_type, struct icp_qat_fw_auth_desc **desc) { struct icp_qat_css_hdr *css_hdr = (struct icp_qat_css_hdr *)image; - struct icp_qat_fw_auth_desc *auth_desc; - struct icp_qat_auth_chunk *auth_chunk; - u64 virt_addr, bus_addr, virt_base; - unsigned int simg_offset = sizeof(*auth_chunk); struct icp_qat_simg_ae_mode *simg_ae_mode; - struct icp_firml_dram_desc img_desc; - int ret; - - ret = qat_uclo_simg_alloc(handle, &img_desc, ICP_QAT_CSS_RSA4K_MAX_IMAGE_LEN); - if (ret) { - pr_err("error, allocate continuous dram fail\n"); - return ret; - } - - if (!IS_ALIGNED(img_desc.dram_size, 8) || !img_desc.dram_bus_addr) { - pr_debug("invalid address\n"); - qat_uclo_simg_free(handle, &img_desc); - return -EINVAL; - } + struct icp_qat_fw_auth_desc *auth_desc; + char *virt_addr, *virt_base; + u64 bus_addr; - auth_chunk = img_desc.dram_base_addr_v; - auth_chunk->chunk_size = img_desc.dram_size; - auth_chunk->chunk_bus_addr = img_desc.dram_bus_addr; - virt_base = (uintptr_t)img_desc.dram_base_addr_v + simg_offset; - bus_addr = img_desc.dram_bus_addr + simg_offset; - auth_desc = img_desc.dram_base_addr_v; - auth_desc->css_hdr_high = (unsigned int)(bus_addr >> BITS_PER_TYPE(u32)); - auth_desc->css_hdr_low = (unsigned int)bus_addr; + virt_base = dram_desc->dram_base_addr_v; + virt_base += sizeof(struct icp_qat_auth_chunk); + bus_addr = dram_desc->dram_bus_addr + sizeof(struct icp_qat_auth_chunk); + auth_desc = dram_desc->dram_base_addr_v; + auth_desc->css_hdr_high = upper_32_bits(bus_addr); + auth_desc->css_hdr_low = lower_32_bits(bus_addr); virt_addr = virt_base; - memcpy((void *)(uintptr_t)virt_addr, image, sizeof(*css_hdr)); + memcpy(virt_addr, image, sizeof(*css_hdr)); /* pub key */ bus_addr = ADD_ADDR(auth_desc->css_hdr_high, auth_desc->css_hdr_low) + sizeof(*css_hdr); virt_addr = virt_addr + sizeof(*css_hdr); - auth_desc->fwsk_pub_high = (unsigned int)(bus_addr >> BITS_PER_TYPE(u32)); - auth_desc->fwsk_pub_low = (unsigned int)bus_addr; + auth_desc->fwsk_pub_high = upper_32_bits(bus_addr); + auth_desc->fwsk_pub_low = lower_32_bits(bus_addr); - memcpy((void *)(uintptr_t)virt_addr, - (void *)(image + sizeof(*css_hdr)), - ICP_QAT_CSS_FWSK_MODULUS_LEN(handle)); + memcpy(virt_addr, image + sizeof(*css_hdr), ICP_QAT_CSS_FWSK_MODULUS_LEN(handle)); /* padding */ memset((void *)(uintptr_t)(virt_addr + ICP_QAT_CSS_FWSK_MODULUS_LEN(handle)), 0, ICP_QAT_CSS_FWSK_PAD_LEN(handle)); /* exponent */ - memcpy((void *)(uintptr_t)(virt_addr + ICP_QAT_CSS_FWSK_MODULUS_LEN(handle) + - ICP_QAT_CSS_FWSK_PAD_LEN(handle)), - (void *)(image + sizeof(*css_hdr) + - ICP_QAT_CSS_FWSK_MODULUS_LEN(handle)), - sizeof(unsigned int)); + memcpy(virt_addr + ICP_QAT_CSS_FWSK_MODULUS_LEN(handle) + + ICP_QAT_CSS_FWSK_PAD_LEN(handle), image + sizeof(*css_hdr) + + ICP_QAT_CSS_FWSK_MODULUS_LEN(handle), sizeof(unsigned int)); /* signature */ bus_addr = ADD_ADDR(auth_desc->fwsk_pub_high, auth_desc->fwsk_pub_low) + ICP_QAT_CSS_FWSK_PUB_LEN(handle); virt_addr = virt_addr + ICP_QAT_CSS_FWSK_PUB_LEN(handle); - auth_desc->signature_high = (unsigned int)(bus_addr >> BITS_PER_TYPE(u32)); - auth_desc->signature_low = (unsigned int)bus_addr; + auth_desc->signature_high = upper_32_bits(bus_addr); + auth_desc->signature_low = lower_32_bits(bus_addr); - memcpy((void *)(uintptr_t)virt_addr, - (void *)(image + sizeof(*css_hdr) + - ICP_QAT_CSS_FWSK_MODULUS_LEN(handle) + - ICP_QAT_CSS_FWSK_EXPONENT_LEN(handle)), - ICP_QAT_CSS_SIGNATURE_LEN(handle)); + memcpy(virt_addr, image + sizeof(*css_hdr) + ICP_QAT_CSS_FWSK_MODULUS_LEN(handle) + + ICP_QAT_CSS_FWSK_EXPONENT_LEN(handle), ICP_QAT_CSS_SIGNATURE_LEN(handle)); bus_addr = ADD_ADDR(auth_desc->signature_high, auth_desc->signature_low) + ICP_QAT_CSS_SIGNATURE_LEN(handle); virt_addr += ICP_QAT_CSS_SIGNATURE_LEN(handle); - auth_desc->img_high = (unsigned int)(bus_addr >> BITS_PER_TYPE(u32)); - auth_desc->img_low = (unsigned int)bus_addr; - auth_desc->img_len = size - ICP_QAT_AE_IMG_OFFSET(handle); - if (bus_addr + auth_desc->img_len > img_desc.dram_bus_addr + - ICP_QAT_CSS_RSA4K_MAX_IMAGE_LEN) { + auth_desc->img_high = upper_32_bits(bus_addr); + auth_desc->img_low = lower_32_bits(bus_addr); + auth_desc->img_len = size - qat_uclo_simg_hdr2sign_len(handle); + if (bus_addr + auth_desc->img_len > + dram_desc->dram_bus_addr + ICP_QAT_CSS_RSA4K_MAX_IMAGE_LEN) { pr_err("insufficient memory size for authentication data\n"); - qat_uclo_simg_free(handle, &img_desc); + qat_uclo_simg_free(handle, dram_desc); return -ENOMEM; } - memcpy((void *)(uintptr_t)virt_addr, - (void *)(image + ICP_QAT_AE_IMG_OFFSET(handle)), - auth_desc->img_len); + memcpy(virt_addr, image + qat_uclo_simg_hdr2sign_len(handle), auth_desc->img_len); virt_addr = virt_base; /* AE firmware */ - if (((struct icp_qat_css_hdr *)(uintptr_t)virt_addr)->fw_type == - CSS_AE_FIRMWARE) { + if (fw_type == CSS_AE_FIRMWARE) { auth_desc->img_ae_mode_data_high = auth_desc->img_high; auth_desc->img_ae_mode_data_low = auth_desc->img_low; bus_addr = ADD_ADDR(auth_desc->img_ae_mode_data_high, auth_desc->img_ae_mode_data_low) + sizeof(struct icp_qat_simg_ae_mode); - auth_desc->img_ae_init_data_high = - (unsigned int)(bus_addr >> BITS_PER_TYPE(u32)); - auth_desc->img_ae_init_data_low = (unsigned int)bus_addr; + auth_desc->img_ae_init_data_high = upper_32_bits(bus_addr); + auth_desc->img_ae_init_data_low = lower_32_bits(bus_addr); bus_addr += ICP_QAT_SIMG_AE_INIT_SEQ_LEN; - auth_desc->img_ae_insts_high = - (unsigned int)(bus_addr >> BITS_PER_TYPE(u32)); - auth_desc->img_ae_insts_low = (unsigned int)bus_addr; + auth_desc->img_ae_insts_high = upper_32_bits(bus_addr); + auth_desc->img_ae_insts_low = lower_32_bits(bus_addr); virt_addr += sizeof(struct icp_qat_css_hdr); virt_addr += ICP_QAT_CSS_FWSK_PUB_LEN(handle); virt_addr += ICP_QAT_CSS_SIGNATURE_LEN(handle); @@ -1523,6 +1515,28 @@ static int qat_uclo_map_auth_fw(struct icp_qat_fw_loader_handle *handle, return 0; } +static int qat_uclo_map_auth_fw(struct icp_qat_fw_loader_handle *handle, + char *image, unsigned int size, + struct icp_qat_fw_auth_desc **desc) +{ + struct icp_qat_auth_chunk *auth_chunk; + struct icp_firml_dram_desc img_desc; + unsigned int simg_fw_type; + int ret; + + ret = qat_uclo_simg_alloc(handle, &img_desc, ICP_QAT_CSS_RSA4K_MAX_IMAGE_LEN); + if (ret) + return ret; + + simg_fw_type = qat_uclo_simg_fw_type(handle, image); + auth_chunk = img_desc.dram_base_addr_v; + auth_chunk->chunk_size = img_desc.dram_size; + auth_chunk->chunk_bus_addr = img_desc.dram_bus_addr; + + return qat_uclo_build_auth_desc_RSA(handle, image, size, &img_desc, + simg_fw_type, desc); +} + static int qat_uclo_load_fw(struct icp_qat_fw_loader_handle *handle, struct icp_qat_fw_auth_desc *desc) { -- 2.50.1 From e7b73261096749d5fa3320528f86daecd08a0556 Mon Sep 17 00:00:00 2001 From: Jack Xu Date: Wed, 30 Apr 2025 12:34:47 +0100 Subject: [PATCH 15/16] crypto: qat - add GEN6 firmware loader Add support for the QAT GEN6 devices in the firmware loader. This includes handling firmware images signed with the RSA 3K and the XMSS algorithms. Co-developed-by: Suman Kumar Chakraborty Signed-off-by: Suman Kumar Chakraborty Signed-off-by: Jack Xu Signed-off-by: Herbert Xu --- .../intel/qat/qat_common/adf_accel_devices.h | 2 + .../qat/qat_common/icp_qat_fw_loader_handle.h | 1 + .../intel/qat/qat_common/icp_qat_uclo.h | 23 +++ drivers/crypto/intel/qat/qat_common/qat_hal.c | 3 + .../crypto/intel/qat/qat_common/qat_uclo.c | 154 +++++++++++++++++- 5 files changed, 176 insertions(+), 7 deletions(-) diff --git a/drivers/crypto/intel/qat/qat_common/adf_accel_devices.h b/drivers/crypto/intel/qat/qat_common/adf_accel_devices.h index a39f506322f6..ed8b85360573 100644 --- a/drivers/crypto/intel/qat/qat_common/adf_accel_devices.h +++ b/drivers/crypto/intel/qat/qat_common/adf_accel_devices.h @@ -34,6 +34,8 @@ #define PCI_DEVICE_ID_INTEL_QAT_402XXIOV 0x4945 #define PCI_DEVICE_ID_INTEL_QAT_420XX 0x4946 #define PCI_DEVICE_ID_INTEL_QAT_420XXIOV 0x4947 +#define PCI_DEVICE_ID_INTEL_QAT_6XXX 0x4948 + #define ADF_DEVICE_FUSECTL_OFFSET 0x40 #define ADF_DEVICE_LEGFUSE_OFFSET 0x4C #define ADF_DEVICE_FUSECTL_MASK 0x80000000 diff --git a/drivers/crypto/intel/qat/qat_common/icp_qat_fw_loader_handle.h b/drivers/crypto/intel/qat/qat_common/icp_qat_fw_loader_handle.h index 7eb5daef4f88..6887930c7995 100644 --- a/drivers/crypto/intel/qat/qat_common/icp_qat_fw_loader_handle.h +++ b/drivers/crypto/intel/qat/qat_common/icp_qat_fw_loader_handle.h @@ -35,6 +35,7 @@ struct icp_qat_fw_loader_chip_info { u32 wakeup_event_val; bool fw_auth; bool css_3k; + bool dual_sign; bool tgroup_share_ustore; u32 fcu_ctl_csr; u32 fcu_sts_csr; diff --git a/drivers/crypto/intel/qat/qat_common/icp_qat_uclo.h b/drivers/crypto/intel/qat/qat_common/icp_qat_uclo.h index 1c7bcd8e4055..6313c35eff0c 100644 --- a/drivers/crypto/intel/qat/qat_common/icp_qat_uclo.h +++ b/drivers/crypto/intel/qat/qat_common/icp_qat_uclo.h @@ -7,6 +7,7 @@ #define ICP_QAT_AC_C62X_DEV_TYPE 0x01000000 #define ICP_QAT_AC_C3XXX_DEV_TYPE 0x02000000 #define ICP_QAT_AC_4XXX_A_DEV_TYPE 0x08000000 +#define ICP_QAT_AC_6XXX_DEV_TYPE 0x80000000 #define ICP_QAT_UCLO_MAX_AE 17 #define ICP_QAT_UCLO_MAX_CTX 8 #define ICP_QAT_UCLO_MAX_UIMAGE (ICP_QAT_UCLO_MAX_AE * ICP_QAT_UCLO_MAX_CTX) @@ -81,6 +82,21 @@ #define ICP_QAT_CSS_RSA4K_MAX_IMAGE_LEN 0x40000 #define ICP_QAT_CSS_RSA3K_MAX_IMAGE_LEN 0x30000 +/* All lengths below are in bytes */ +#define ICP_QAT_DUALSIGN_OPAQUE_HDR_LEN 12 +#define ICP_QAT_DUALSIGN_OPAQUE_HDR_ALIGN_LEN 16 +#define ICP_QAT_DUALSIGN_OPAQUE_DATA_LEN 3540 +#define ICP_QAT_DUALSIGN_XMSS_PUBKEY_LEN 64 +#define ICP_QAT_DUALSIGN_XMSS_SIG_LEN 2692 +#define ICP_QAT_DUALSIGN_XMSS_SIG_ALIGN_LEN 2696 +#define ICP_QAT_DUALSIGN_MISC_INFO_LEN 16 +#define ICP_QAT_DUALSIGN_FW_TYPE_LEN 7 +#define ICP_QAT_DUALSIGN_MODULE_TYPE 0x14 +#define ICP_QAT_DUALSIGN_HDR_LEN 0x375 +#define ICP_QAT_DUALSIGN_HDR_VER 0x40001 +#define ICP_QAT_DUALSIGN_HDR_LEN_OFFSET 4 +#define ICP_QAT_DUALSIGN_HDR_VER_OFFSET 8 + #define ICP_QAT_CTX_MODE(ae_mode) ((ae_mode) & 0xf) #define ICP_QAT_NN_MODE(ae_mode) (((ae_mode) >> 0x4) & 0xf) #define ICP_QAT_SHARED_USTORE_MODE(ae_mode) (((ae_mode) >> 0xb) & 0x1) @@ -440,6 +456,13 @@ struct icp_qat_fw_auth_desc { unsigned int img_ae_init_data_low; unsigned int img_ae_insts_high; unsigned int img_ae_insts_low; + unsigned int cpp_mask; + unsigned int reserved; + unsigned int xmss_pubkey_high; + unsigned int xmss_pubkey_low; + unsigned int xmss_sig_high; + unsigned int xmss_sig_low; + unsigned int reserved2[2]; }; struct icp_qat_auth_chunk { diff --git a/drivers/crypto/intel/qat/qat_common/qat_hal.c b/drivers/crypto/intel/qat/qat_common/qat_hal.c index 841c1d7d3ffe..da4eca6e1633 100644 --- a/drivers/crypto/intel/qat/qat_common/qat_hal.c +++ b/drivers/crypto/intel/qat/qat_common/qat_hal.c @@ -698,6 +698,7 @@ static int qat_hal_chip_init(struct icp_qat_fw_loader_handle *handle, case PCI_DEVICE_ID_INTEL_QAT_401XX: case PCI_DEVICE_ID_INTEL_QAT_402XX: case PCI_DEVICE_ID_INTEL_QAT_420XX: + case PCI_DEVICE_ID_INTEL_QAT_6XXX: handle->chip_info->mmp_sram_size = 0; handle->chip_info->nn = false; handle->chip_info->lm2lm3 = true; @@ -712,6 +713,8 @@ static int qat_hal_chip_init(struct icp_qat_fw_loader_handle *handle, handle->chip_info->wakeup_event_val = 0x80000000; handle->chip_info->fw_auth = true; handle->chip_info->css_3k = true; + if (handle->pci_dev->device == PCI_DEVICE_ID_INTEL_QAT_6XXX) + handle->chip_info->dual_sign = true; handle->chip_info->tgroup_share_ustore = true; handle->chip_info->fcu_ctl_csr = FCU_CONTROL_4XXX; handle->chip_info->fcu_sts_csr = FCU_STATUS_4XXX; diff --git a/drivers/crypto/intel/qat/qat_common/qat_uclo.c b/drivers/crypto/intel/qat/qat_common/qat_uclo.c index d7f2ceb81f1f..21d652a1c8ef 100644 --- a/drivers/crypto/intel/qat/qat_common/qat_uclo.c +++ b/drivers/crypto/intel/qat/qat_common/qat_uclo.c @@ -10,6 +10,7 @@ #include #include #include +#include #include "adf_accel_devices.h" #include "adf_common_drv.h" #include "icp_qat_uclo.h" @@ -737,6 +738,8 @@ qat_uclo_get_dev_type(struct icp_qat_fw_loader_handle *handle) case PCI_DEVICE_ID_INTEL_QAT_402XX: case PCI_DEVICE_ID_INTEL_QAT_420XX: return ICP_QAT_AC_4XXX_A_DEV_TYPE; + case PCI_DEVICE_ID_INTEL_QAT_6XXX: + return ICP_QAT_AC_6XXX_DEV_TYPE; default: pr_err("unsupported device 0x%x\n", handle->pci_dev->device); return 0; @@ -1035,17 +1038,30 @@ out_err: static unsigned int qat_uclo_simg_hdr2sign_len(struct icp_qat_fw_loader_handle *handle) { + if (handle->chip_info->dual_sign) + return ICP_QAT_DUALSIGN_OPAQUE_DATA_LEN; + return ICP_QAT_AE_IMG_OFFSET(handle); } static unsigned int qat_uclo_simg_hdr2cont_len(struct icp_qat_fw_loader_handle *handle) { + if (handle->chip_info->dual_sign) + return ICP_QAT_DUALSIGN_OPAQUE_DATA_LEN + ICP_QAT_DUALSIGN_MISC_INFO_LEN; + return ICP_QAT_AE_IMG_OFFSET(handle); } static unsigned int qat_uclo_simg_fw_type(struct icp_qat_fw_loader_handle *handle, void *img_ptr) { struct icp_qat_css_hdr *hdr = img_ptr; + char *fw_hdr = img_ptr; + unsigned int offset; + + if (handle->chip_info->dual_sign) { + offset = qat_uclo_simg_hdr2sign_len(handle) + ICP_QAT_DUALSIGN_FW_TYPE_LEN; + return *(fw_hdr + offset); + } return hdr->fw_type; } @@ -1390,16 +1406,27 @@ static int qat_uclo_check_image(struct icp_qat_fw_loader_handle *handle, if (handle->chip_info->fw_auth) { header_len = qat_uclo_simg_hdr2sign_len(handle); simg_type = qat_uclo_simg_fw_type(handle, image); - css_hdr = image; - if ((css_hdr->header_len * css_dword_size) != header_len) - goto err; - if ((css_hdr->size * css_dword_size) != size) - goto err; + + if (handle->chip_info->dual_sign) { + if (css_hdr->module_type != ICP_QAT_DUALSIGN_MODULE_TYPE) + goto err; + if (css_hdr->header_len != ICP_QAT_DUALSIGN_HDR_LEN) + goto err; + if (css_hdr->header_ver != ICP_QAT_DUALSIGN_HDR_VER) + goto err; + } else { + if (css_hdr->header_len * css_dword_size != header_len) + goto err; + if (css_hdr->size * css_dword_size != size) + goto err; + if (size <= header_len) + goto err; + } + if (fw_type != simg_type) goto err; - if (size <= header_len) - goto err; + size -= header_len; } @@ -1515,6 +1542,115 @@ static int qat_uclo_build_auth_desc_RSA(struct icp_qat_fw_loader_handle *handle, return 0; } +static int qat_uclo_build_auth_desc_dualsign(struct icp_qat_fw_loader_handle *handle, + char *image, unsigned int size, + struct icp_firml_dram_desc *dram_desc, + unsigned int fw_type, + struct icp_qat_fw_auth_desc **desc) +{ + struct icp_qat_simg_ae_mode *simg_ae_mode; + struct icp_qat_fw_auth_desc *auth_desc; + unsigned int chunk_offset, img_offset; + u64 bus_addr, addr; + char *virt_addr; + + virt_addr = dram_desc->dram_base_addr_v; + virt_addr += sizeof(struct icp_qat_auth_chunk); + bus_addr = dram_desc->dram_bus_addr + sizeof(struct icp_qat_auth_chunk); + + auth_desc = dram_desc->dram_base_addr_v; + auth_desc->img_len = size - qat_uclo_simg_hdr2sign_len(handle); + auth_desc->css_hdr_high = upper_32_bits(bus_addr); + auth_desc->css_hdr_low = lower_32_bits(bus_addr); + memcpy(virt_addr, image, ICP_QAT_DUALSIGN_OPAQUE_HDR_LEN); + + img_offset = ICP_QAT_DUALSIGN_OPAQUE_HDR_LEN; + chunk_offset = ICP_QAT_DUALSIGN_OPAQUE_HDR_ALIGN_LEN; + + /* RSA pub key */ + addr = bus_addr + chunk_offset; + auth_desc->fwsk_pub_high = upper_32_bits(addr); + auth_desc->fwsk_pub_low = lower_32_bits(addr); + memcpy(virt_addr + chunk_offset, image + img_offset, ICP_QAT_CSS_FWSK_MODULUS_LEN(handle)); + + img_offset += ICP_QAT_CSS_FWSK_MODULUS_LEN(handle); + chunk_offset += ICP_QAT_CSS_FWSK_MODULUS_LEN(handle); + /* RSA padding */ + memset(virt_addr + chunk_offset, 0, ICP_QAT_CSS_FWSK_PAD_LEN(handle)); + + chunk_offset += ICP_QAT_CSS_FWSK_PAD_LEN(handle); + /* RSA exponent */ + memcpy(virt_addr + chunk_offset, image + img_offset, ICP_QAT_CSS_FWSK_EXPONENT_LEN(handle)); + + img_offset += ICP_QAT_CSS_FWSK_EXPONENT_LEN(handle); + chunk_offset += ICP_QAT_CSS_FWSK_EXPONENT_LEN(handle); + /* RSA signature */ + addr = bus_addr + chunk_offset; + auth_desc->signature_high = upper_32_bits(addr); + auth_desc->signature_low = lower_32_bits(addr); + memcpy(virt_addr + chunk_offset, image + img_offset, ICP_QAT_CSS_SIGNATURE_LEN(handle)); + + img_offset += ICP_QAT_CSS_SIGNATURE_LEN(handle); + chunk_offset += ICP_QAT_CSS_SIGNATURE_LEN(handle); + /* XMSS pubkey */ + addr = bus_addr + chunk_offset; + auth_desc->xmss_pubkey_high = upper_32_bits(addr); + auth_desc->xmss_pubkey_low = lower_32_bits(addr); + memcpy(virt_addr + chunk_offset, image + img_offset, ICP_QAT_DUALSIGN_XMSS_PUBKEY_LEN); + + img_offset += ICP_QAT_DUALSIGN_XMSS_PUBKEY_LEN; + chunk_offset += ICP_QAT_DUALSIGN_XMSS_PUBKEY_LEN; + /* XMSS signature */ + addr = bus_addr + chunk_offset; + auth_desc->xmss_sig_high = upper_32_bits(addr); + auth_desc->xmss_sig_low = lower_32_bits(addr); + memcpy(virt_addr + chunk_offset, image + img_offset, ICP_QAT_DUALSIGN_XMSS_SIG_LEN); + + img_offset += ICP_QAT_DUALSIGN_XMSS_SIG_LEN; + chunk_offset += ICP_QAT_DUALSIGN_XMSS_SIG_ALIGN_LEN; + + if (dram_desc->dram_size < (chunk_offset + auth_desc->img_len)) { + pr_err("auth chunk memory size is not enough to store data\n"); + return -ENOMEM; + } + + /* Signed data */ + addr = bus_addr + chunk_offset; + auth_desc->img_high = upper_32_bits(addr); + auth_desc->img_low = lower_32_bits(addr); + memcpy(virt_addr + chunk_offset, image + img_offset, auth_desc->img_len); + + chunk_offset += ICP_QAT_DUALSIGN_MISC_INFO_LEN; + /* AE firmware */ + if (fw_type == CSS_AE_FIRMWARE) { + /* AE mode data */ + addr = bus_addr + chunk_offset; + auth_desc->img_ae_mode_data_high = upper_32_bits(addr); + auth_desc->img_ae_mode_data_low = lower_32_bits(addr); + simg_ae_mode = + (struct icp_qat_simg_ae_mode *)(virt_addr + chunk_offset); + auth_desc->ae_mask = simg_ae_mode->ae_mask & handle->cfg_ae_mask; + + chunk_offset += sizeof(struct icp_qat_simg_ae_mode); + /* AE init seq */ + addr = bus_addr + chunk_offset; + auth_desc->img_ae_init_data_high = upper_32_bits(addr); + auth_desc->img_ae_init_data_low = lower_32_bits(addr); + + chunk_offset += ICP_QAT_SIMG_AE_INIT_SEQ_LEN; + /* AE instructions */ + addr = bus_addr + chunk_offset; + auth_desc->img_ae_insts_high = upper_32_bits(addr); + auth_desc->img_ae_insts_low = lower_32_bits(addr); + } else { + addr = bus_addr + chunk_offset; + auth_desc->img_ae_insts_high = upper_32_bits(addr); + auth_desc->img_ae_insts_low = lower_32_bits(addr); + } + *desc = auth_desc; + return 0; +} + static int qat_uclo_map_auth_fw(struct icp_qat_fw_loader_handle *handle, char *image, unsigned int size, struct icp_qat_fw_auth_desc **desc) @@ -1533,6 +1669,10 @@ static int qat_uclo_map_auth_fw(struct icp_qat_fw_loader_handle *handle, auth_chunk->chunk_size = img_desc.dram_size; auth_chunk->chunk_bus_addr = img_desc.dram_bus_addr; + if (handle->chip_info->dual_sign) + return qat_uclo_build_auth_desc_dualsign(handle, image, size, &img_desc, + simg_fw_type, desc); + return qat_uclo_build_auth_desc_RSA(handle, image, size, &img_desc, simg_fw_type, desc); } -- 2.50.1 From 61e152873adb00c32f6d87c04ab3b8ac6e14495f Mon Sep 17 00:00:00 2001 From: Giovanni Cabiddu Date: Wed, 30 Apr 2025 12:34:48 +0100 Subject: [PATCH 16/16] crypto: qat - export adf_get_service_mask() Export the function adf_get_service_mask() as it will be used by the qat_6xxx driver to configure the device. Signed-off-by: Giovanni Cabiddu Reviewed-by: Andy Shevchenko Signed-off-by: Herbert Xu --- drivers/crypto/intel/qat/qat_common/adf_cfg_services.c | 3 ++- drivers/crypto/intel/qat/qat_common/adf_cfg_services.h | 1 + 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/drivers/crypto/intel/qat/qat_common/adf_cfg_services.c b/drivers/crypto/intel/qat/qat_common/adf_cfg_services.c index 30abcd9e1283..c39871291da7 100644 --- a/drivers/crypto/intel/qat/qat_common/adf_cfg_services.c +++ b/drivers/crypto/intel/qat/qat_common/adf_cfg_services.c @@ -116,7 +116,7 @@ int adf_parse_service_string(struct adf_accel_dev *accel_dev, const char *in, return adf_service_mask_to_string(mask, out, out_len); } -static int adf_get_service_mask(struct adf_accel_dev *accel_dev, unsigned long *mask) +int adf_get_service_mask(struct adf_accel_dev *accel_dev, unsigned long *mask) { char services[ADF_CFG_MAX_VAL_LEN_IN_BYTES] = { }; size_t len; @@ -138,6 +138,7 @@ static int adf_get_service_mask(struct adf_accel_dev *accel_dev, unsigned long * return ret; } +EXPORT_SYMBOL_GPL(adf_get_service_mask); int adf_get_service_enabled(struct adf_accel_dev *accel_dev) { diff --git a/drivers/crypto/intel/qat/qat_common/adf_cfg_services.h b/drivers/crypto/intel/qat/qat_common/adf_cfg_services.h index f6bafc15cbc6..3742c450878f 100644 --- a/drivers/crypto/intel/qat/qat_common/adf_cfg_services.h +++ b/drivers/crypto/intel/qat/qat_common/adf_cfg_services.h @@ -32,5 +32,6 @@ enum { int adf_parse_service_string(struct adf_accel_dev *accel_dev, const char *in, size_t in_len, char *out, size_t out_len); int adf_get_service_enabled(struct adf_accel_dev *accel_dev); +int adf_get_service_mask(struct adf_accel_dev *accel_dev, unsigned long *mask); #endif -- 2.50.1