From aeaad5bfb18890cc73ca32d63f1f02feb5f3f651 Mon Sep 17 00:00:00 2001 From: Eric Biggers Date: Mon, 5 May 2025 12:10:45 -0700 Subject: [PATCH 01/16] crypto: null - use memcpy_sglist() Make null_skcipher_crypt() use memcpy_sglist() instead of the skcipher_walk API, as this is simpler. Signed-off-by: Eric Biggers Signed-off-by: Herbert Xu --- crypto/crypto_null.c | 17 ++++------------- 1 file changed, 4 insertions(+), 13 deletions(-) diff --git a/crypto/crypto_null.c b/crypto/crypto_null.c index 48c71b925f37..34588f39fdfc 100644 --- a/crypto/crypto_null.c +++ b/crypto/crypto_null.c @@ -15,6 +15,7 @@ #include #include #include +#include #include #include #include @@ -60,19 +61,9 @@ static void null_crypt(struct crypto_tfm *tfm, u8 *dst, const u8 *src) static int null_skcipher_crypt(struct skcipher_request *req) { - struct skcipher_walk walk; - int err; - - err = skcipher_walk_virt(&walk, req, false); - - while (walk.nbytes) { - if (walk.src.virt.addr != walk.dst.virt.addr) - memcpy(walk.dst.virt.addr, walk.src.virt.addr, - walk.nbytes); - err = skcipher_walk_done(&walk, 0); - } - - return err; + if (req->src != req->dst) + memcpy_sglist(req->dst, req->src, req->cryptlen); + return 0; } static struct shash_alg digest_null = { -- 2.51.0 From 3357b6c94569095f87a350bffa5a0a6e0c19c962 Mon Sep 17 00:00:00 2001 From: Eric Biggers Date: Mon, 5 May 2025 13:33:39 -0700 Subject: [PATCH 02/16] crypto: tcrypt - rename CRYPTO_TEST to CRYPTO_BENCHMARK tcrypt is actually a benchmarking module and not the actual tests. This regularly causes confusion. Update the kconfig option name and help text accordingly. Signed-off-by: Eric Biggers Acked-by: Geert Uytterhoeven # m68k Signed-off-by: Herbert Xu --- arch/arm/configs/exynos_defconfig | 2 +- arch/arm/configs/pxa_defconfig | 2 +- arch/arm/configs/spitz_defconfig | 2 +- arch/arm64/configs/defconfig | 2 +- arch/m68k/configs/amiga_defconfig | 2 +- arch/m68k/configs/apollo_defconfig | 2 +- arch/m68k/configs/atari_defconfig | 2 +- arch/m68k/configs/bvme6000_defconfig | 2 +- arch/m68k/configs/hp300_defconfig | 2 +- arch/m68k/configs/mac_defconfig | 2 +- arch/m68k/configs/multi_defconfig | 2 +- arch/m68k/configs/mvme147_defconfig | 2 +- arch/m68k/configs/mvme16x_defconfig | 2 +- arch/m68k/configs/q40_defconfig | 2 +- arch/m68k/configs/sun3_defconfig | 2 +- arch/m68k/configs/sun3x_defconfig | 2 +- arch/mips/configs/gpr_defconfig | 2 +- arch/mips/configs/lemote2f_defconfig | 2 +- arch/mips/configs/mtx1_defconfig | 2 +- arch/mips/configs/rb532_defconfig | 2 +- arch/parisc/configs/generic-32bit_defconfig | 2 +- arch/powerpc/configs/g5_defconfig | 2 +- arch/powerpc/configs/powernv_defconfig | 2 +- arch/powerpc/configs/ppc64_defconfig | 2 +- arch/powerpc/configs/ppc64e_defconfig | 2 +- arch/powerpc/configs/ppc6xx_defconfig | 2 +- arch/s390/configs/debug_defconfig | 2 +- arch/s390/configs/defconfig | 2 +- arch/sparc/configs/sparc64_defconfig | 2 +- crypto/Kconfig | 10 +++++++--- crypto/Makefile | 2 +- crypto/tcrypt.c | 8 ++++---- crypto/tcrypt.h | 4 ++-- 33 files changed, 43 insertions(+), 39 deletions(-) diff --git a/arch/arm/configs/exynos_defconfig b/arch/arm/configs/exynos_defconfig index c6792c0256a6..e81964cce516 100644 --- a/arch/arm/configs/exynos_defconfig +++ b/arch/arm/configs/exynos_defconfig @@ -349,7 +349,7 @@ CONFIG_NLS_ASCII=y CONFIG_NLS_ISO8859_1=y CONFIG_NLS_UTF8=y CONFIG_CRYPTO_USER=m -CONFIG_CRYPTO_TEST=m +CONFIG_CRYPTO_BENCHMARK=m CONFIG_CRYPTO_DH=m CONFIG_CRYPTO_LRW=m CONFIG_CRYPTO_XTS=m diff --git a/arch/arm/configs/pxa_defconfig b/arch/arm/configs/pxa_defconfig index 56be85752909..edc5b4d17f77 100644 --- a/arch/arm/configs/pxa_defconfig +++ b/arch/arm/configs/pxa_defconfig @@ -639,7 +639,7 @@ CONFIG_SECURITY=y CONFIG_CRYPTO_MANAGER=y CONFIG_CRYPTO_CRYPTD=m CONFIG_CRYPTO_AUTHENC=m -CONFIG_CRYPTO_TEST=m +CONFIG_CRYPTO_BENCHMARK=m CONFIG_CRYPTO_BLOWFISH=m CONFIG_CRYPTO_CAST5=m CONFIG_CRYPTO_CAST6=m diff --git a/arch/arm/configs/spitz_defconfig b/arch/arm/configs/spitz_defconfig index ffec59e3f49c..ac2a0f998c73 100644 --- a/arch/arm/configs/spitz_defconfig +++ b/arch/arm/configs/spitz_defconfig @@ -215,7 +215,7 @@ CONFIG_NLS_ISO8859_1=y CONFIG_NLS_UTF8=y CONFIG_DEBUG_KERNEL=y CONFIG_CRYPTO_NULL=m -CONFIG_CRYPTO_TEST=m +CONFIG_CRYPTO_BENCHMARK=m CONFIG_CRYPTO_ECB=m CONFIG_CRYPTO_SHA256=m CONFIG_CRYPTO_AES=m diff --git a/arch/arm64/configs/defconfig b/arch/arm64/configs/defconfig index b0d4c7d173ea..4166bc4d3cec 100644 --- a/arch/arm64/configs/defconfig +++ b/arch/arm64/configs/defconfig @@ -1729,7 +1729,7 @@ CONFIG_NLS_CODEPAGE_437=y CONFIG_NLS_ISO8859_1=y CONFIG_SECURITY=y CONFIG_CRYPTO_USER=y -CONFIG_CRYPTO_TEST=m +CONFIG_CRYPTO_BENCHMARK=m CONFIG_CRYPTO_ECHAINIV=y CONFIG_CRYPTO_MICHAEL_MIC=m CONFIG_CRYPTO_ANSI_CPRNG=y diff --git a/arch/m68k/configs/amiga_defconfig b/arch/m68k/configs/amiga_defconfig index 31ecb8b7b9f1..77f78d326a32 100644 --- a/arch/m68k/configs/amiga_defconfig +++ b/arch/m68k/configs/amiga_defconfig @@ -551,7 +551,7 @@ CONFIG_ENCRYPTED_KEYS=m CONFIG_HARDENED_USERCOPY=y CONFIG_CRYPTO_USER=m CONFIG_CRYPTO_CRYPTD=m -CONFIG_CRYPTO_TEST=m +CONFIG_CRYPTO_BENCHMARK=m CONFIG_CRYPTO_RSA=m CONFIG_CRYPTO_DH=m CONFIG_CRYPTO_ECDH=m diff --git a/arch/m68k/configs/apollo_defconfig b/arch/m68k/configs/apollo_defconfig index 1f57514624d5..f4031aa5d37f 100644 --- a/arch/m68k/configs/apollo_defconfig +++ b/arch/m68k/configs/apollo_defconfig @@ -508,7 +508,7 @@ CONFIG_ENCRYPTED_KEYS=m CONFIG_HARDENED_USERCOPY=y CONFIG_CRYPTO_USER=m CONFIG_CRYPTO_CRYPTD=m -CONFIG_CRYPTO_TEST=m +CONFIG_CRYPTO_BENCHMARK=m CONFIG_CRYPTO_RSA=m CONFIG_CRYPTO_DH=m CONFIG_CRYPTO_ECDH=m diff --git a/arch/m68k/configs/atari_defconfig b/arch/m68k/configs/atari_defconfig index 02db7a48e57e..fa92131cf4b3 100644 --- a/arch/m68k/configs/atari_defconfig +++ b/arch/m68k/configs/atari_defconfig @@ -528,7 +528,7 @@ CONFIG_ENCRYPTED_KEYS=m CONFIG_HARDENED_USERCOPY=y CONFIG_CRYPTO_USER=m CONFIG_CRYPTO_CRYPTD=m -CONFIG_CRYPTO_TEST=m +CONFIG_CRYPTO_BENCHMARK=m CONFIG_CRYPTO_RSA=m CONFIG_CRYPTO_DH=m CONFIG_CRYPTO_ECDH=m diff --git a/arch/m68k/configs/bvme6000_defconfig b/arch/m68k/configs/bvme6000_defconfig index f0e673cb17eb..9c2afc477061 100644 --- a/arch/m68k/configs/bvme6000_defconfig +++ b/arch/m68k/configs/bvme6000_defconfig @@ -500,7 +500,7 @@ CONFIG_ENCRYPTED_KEYS=m CONFIG_HARDENED_USERCOPY=y CONFIG_CRYPTO_USER=m CONFIG_CRYPTO_CRYPTD=m -CONFIG_CRYPTO_TEST=m +CONFIG_CRYPTO_BENCHMARK=m CONFIG_CRYPTO_RSA=m CONFIG_CRYPTO_DH=m CONFIG_CRYPTO_ECDH=m diff --git a/arch/m68k/configs/hp300_defconfig b/arch/m68k/configs/hp300_defconfig index e8ca5a50b86d..e7cdab059d96 100644 --- a/arch/m68k/configs/hp300_defconfig +++ b/arch/m68k/configs/hp300_defconfig @@ -510,7 +510,7 @@ CONFIG_ENCRYPTED_KEYS=m CONFIG_HARDENED_USERCOPY=y CONFIG_CRYPTO_USER=m CONFIG_CRYPTO_CRYPTD=m -CONFIG_CRYPTO_TEST=m +CONFIG_CRYPTO_BENCHMARK=m CONFIG_CRYPTO_RSA=m CONFIG_CRYPTO_DH=m CONFIG_CRYPTO_ECDH=m diff --git a/arch/m68k/configs/mac_defconfig b/arch/m68k/configs/mac_defconfig index b3a270441bb1..0a79751c20a5 100644 --- a/arch/m68k/configs/mac_defconfig +++ b/arch/m68k/configs/mac_defconfig @@ -527,7 +527,7 @@ CONFIG_ENCRYPTED_KEYS=m CONFIG_HARDENED_USERCOPY=y CONFIG_CRYPTO_USER=m CONFIG_CRYPTO_CRYPTD=m -CONFIG_CRYPTO_TEST=m +CONFIG_CRYPTO_BENCHMARK=m CONFIG_CRYPTO_RSA=m CONFIG_CRYPTO_DH=m CONFIG_CRYPTO_ECDH=m diff --git a/arch/m68k/configs/multi_defconfig b/arch/m68k/configs/multi_defconfig index d215dba006ce..f8ca490ee65a 100644 --- a/arch/m68k/configs/multi_defconfig +++ b/arch/m68k/configs/multi_defconfig @@ -614,7 +614,7 @@ CONFIG_ENCRYPTED_KEYS=m CONFIG_HARDENED_USERCOPY=y CONFIG_CRYPTO_USER=m CONFIG_CRYPTO_CRYPTD=m -CONFIG_CRYPTO_TEST=m +CONFIG_CRYPTO_BENCHMARK=m CONFIG_CRYPTO_RSA=m CONFIG_CRYPTO_DH=m CONFIG_CRYPTO_ECDH=m diff --git a/arch/m68k/configs/mvme147_defconfig b/arch/m68k/configs/mvme147_defconfig index a888ed93ff82..88fdcea906f3 100644 --- a/arch/m68k/configs/mvme147_defconfig +++ b/arch/m68k/configs/mvme147_defconfig @@ -500,7 +500,7 @@ CONFIG_ENCRYPTED_KEYS=m CONFIG_HARDENED_USERCOPY=y CONFIG_CRYPTO_USER=m CONFIG_CRYPTO_CRYPTD=m -CONFIG_CRYPTO_TEST=m +CONFIG_CRYPTO_BENCHMARK=m CONFIG_CRYPTO_RSA=m CONFIG_CRYPTO_DH=m CONFIG_CRYPTO_ECDH=m diff --git a/arch/m68k/configs/mvme16x_defconfig b/arch/m68k/configs/mvme16x_defconfig index b481782375f6..8acbe83dac72 100644 --- a/arch/m68k/configs/mvme16x_defconfig +++ b/arch/m68k/configs/mvme16x_defconfig @@ -501,7 +501,7 @@ CONFIG_ENCRYPTED_KEYS=m CONFIG_HARDENED_USERCOPY=y CONFIG_CRYPTO_USER=m CONFIG_CRYPTO_CRYPTD=m -CONFIG_CRYPTO_TEST=m +CONFIG_CRYPTO_BENCHMARK=m CONFIG_CRYPTO_RSA=m CONFIG_CRYPTO_DH=m CONFIG_CRYPTO_ECDH=m diff --git a/arch/m68k/configs/q40_defconfig b/arch/m68k/configs/q40_defconfig index 6eba743d8eb5..e3095301f3c5 100644 --- a/arch/m68k/configs/q40_defconfig +++ b/arch/m68k/configs/q40_defconfig @@ -517,7 +517,7 @@ CONFIG_ENCRYPTED_KEYS=m CONFIG_HARDENED_USERCOPY=y CONFIG_CRYPTO_USER=m CONFIG_CRYPTO_CRYPTD=m -CONFIG_CRYPTO_TEST=m +CONFIG_CRYPTO_BENCHMARK=m CONFIG_CRYPTO_RSA=m CONFIG_CRYPTO_DH=m CONFIG_CRYPTO_ECDH=m diff --git a/arch/m68k/configs/sun3_defconfig b/arch/m68k/configs/sun3_defconfig index 9bdbb418ffa8..948e48ddd128 100644 --- a/arch/m68k/configs/sun3_defconfig +++ b/arch/m68k/configs/sun3_defconfig @@ -498,7 +498,7 @@ CONFIG_ENCRYPTED_KEYS=m CONFIG_HARDENED_USERCOPY=y CONFIG_CRYPTO_USER=m CONFIG_CRYPTO_CRYPTD=m -CONFIG_CRYPTO_TEST=m +CONFIG_CRYPTO_BENCHMARK=m CONFIG_CRYPTO_RSA=m CONFIG_CRYPTO_DH=m CONFIG_CRYPTO_ECDH=m diff --git a/arch/m68k/configs/sun3x_defconfig b/arch/m68k/configs/sun3x_defconfig index e1cf20fa5343..5bcf9181c37c 100644 --- a/arch/m68k/configs/sun3x_defconfig +++ b/arch/m68k/configs/sun3x_defconfig @@ -498,7 +498,7 @@ CONFIG_ENCRYPTED_KEYS=m CONFIG_HARDENED_USERCOPY=y CONFIG_CRYPTO_USER=m CONFIG_CRYPTO_CRYPTD=m -CONFIG_CRYPTO_TEST=m +CONFIG_CRYPTO_BENCHMARK=m CONFIG_CRYPTO_RSA=m CONFIG_CRYPTO_DH=m CONFIG_CRYPTO_ECDH=m diff --git a/arch/mips/configs/gpr_defconfig b/arch/mips/configs/gpr_defconfig index 12f3eed8a946..48c8feec958f 100644 --- a/arch/mips/configs/gpr_defconfig +++ b/arch/mips/configs/gpr_defconfig @@ -273,7 +273,7 @@ CONFIG_NLS_CODEPAGE_437=y CONFIG_NLS_CODEPAGE_850=y CONFIG_NLS_ISO8859_1=y CONFIG_CRYPTO_AUTHENC=m -CONFIG_CRYPTO_TEST=m +CONFIG_CRYPTO_BENCHMARK=m CONFIG_CRYPTO_PCBC=m CONFIG_CRYPTO_MD4=m CONFIG_CRYPTO_MICHAEL_MIC=m diff --git a/arch/mips/configs/lemote2f_defconfig b/arch/mips/configs/lemote2f_defconfig index 71d6340497c9..5038a27d035f 100644 --- a/arch/mips/configs/lemote2f_defconfig +++ b/arch/mips/configs/lemote2f_defconfig @@ -297,7 +297,7 @@ CONFIG_NLS_KOI8_R=m CONFIG_NLS_KOI8_U=m CONFIG_NLS_UTF8=y CONFIG_CRYPTO_CRYPTD=m -CONFIG_CRYPTO_TEST=m +CONFIG_CRYPTO_BENCHMARK=m CONFIG_CRYPTO_BLOWFISH=m CONFIG_CRYPTO_CAMELLIA=m CONFIG_CRYPTO_CAST5=m diff --git a/arch/mips/configs/mtx1_defconfig b/arch/mips/configs/mtx1_defconfig index 06b7a0b97eca..cbf9c35a6177 100644 --- a/arch/mips/configs/mtx1_defconfig +++ b/arch/mips/configs/mtx1_defconfig @@ -662,7 +662,7 @@ CONFIG_NLS_ISO8859_15=m CONFIG_NLS_KOI8_R=m CONFIG_NLS_KOI8_U=m CONFIG_NLS_UTF8=m -CONFIG_CRYPTO_TEST=m +CONFIG_CRYPTO_BENCHMARK=m CONFIG_CRYPTO_PCBC=m CONFIG_CRYPTO_HMAC=y CONFIG_CRYPTO_MD5=y diff --git a/arch/mips/configs/rb532_defconfig b/arch/mips/configs/rb532_defconfig index 42b161d587c7..9fb114ef5e2d 100644 --- a/arch/mips/configs/rb532_defconfig +++ b/arch/mips/configs/rb532_defconfig @@ -153,6 +153,6 @@ CONFIG_JFFS2_FS=y CONFIG_JFFS2_SUMMARY=y CONFIG_JFFS2_COMPRESSION_OPTIONS=y CONFIG_SQUASHFS=y -CONFIG_CRYPTO_TEST=m +CONFIG_CRYPTO_BENCHMARK=m # CONFIG_CRYPTO_HW is not set CONFIG_STRIP_ASM_SYMS=y diff --git a/arch/parisc/configs/generic-32bit_defconfig b/arch/parisc/configs/generic-32bit_defconfig index 5b65c9859613..94928d114d4c 100644 --- a/arch/parisc/configs/generic-32bit_defconfig +++ b/arch/parisc/configs/generic-32bit_defconfig @@ -251,7 +251,7 @@ CONFIG_CIFS=m CONFIG_CIFS_XATTR=y CONFIG_CIFS_POSIX=y # CONFIG_CIFS_DEBUG is not set -CONFIG_CRYPTO_TEST=m +CONFIG_CRYPTO_BENCHMARK=m CONFIG_CRYPTO_BLOWFISH=m CONFIG_CRYPTO_CAST5=m CONFIG_CRYPTO_CAST6=m diff --git a/arch/powerpc/configs/g5_defconfig b/arch/powerpc/configs/g5_defconfig index 7e58f3e6c987..428f17b45513 100644 --- a/arch/powerpc/configs/g5_defconfig +++ b/arch/powerpc/configs/g5_defconfig @@ -235,7 +235,7 @@ CONFIG_MAGIC_SYSRQ=y CONFIG_DEBUG_KERNEL=y CONFIG_DEBUG_MUTEXES=y CONFIG_BOOTX_TEXT=y -CONFIG_CRYPTO_TEST=m +CONFIG_CRYPTO_BENCHMARK=m CONFIG_CRYPTO_PCBC=m CONFIG_CRYPTO_HMAC=y CONFIG_CRYPTO_MICHAEL_MIC=m diff --git a/arch/powerpc/configs/powernv_defconfig b/arch/powerpc/configs/powernv_defconfig index 6b6d7467fecf..379229c982a4 100644 --- a/arch/powerpc/configs/powernv_defconfig +++ b/arch/powerpc/configs/powernv_defconfig @@ -317,7 +317,7 @@ CONFIG_CODE_PATCHING_SELFTEST=y CONFIG_FTR_FIXUP_SELFTEST=y CONFIG_MSI_BITMAP_SELFTEST=y CONFIG_XMON=y -CONFIG_CRYPTO_TEST=m +CONFIG_CRYPTO_BENCHMARK=m CONFIG_CRYPTO_PCBC=m CONFIG_CRYPTO_HMAC=y CONFIG_CRYPTO_MD5_PPC=m diff --git a/arch/powerpc/configs/ppc64_defconfig b/arch/powerpc/configs/ppc64_defconfig index 5fa154185efa..3423c405cad4 100644 --- a/arch/powerpc/configs/ppc64_defconfig +++ b/arch/powerpc/configs/ppc64_defconfig @@ -377,7 +377,7 @@ CONFIG_IMA_WRITE_POLICY=y CONFIG_IMA_APPRAISE=y CONFIG_IMA_ARCH_POLICY=y CONFIG_IMA_APPRAISE_MODSIG=y -CONFIG_CRYPTO_TEST=m +CONFIG_CRYPTO_BENCHMARK=m CONFIG_CRYPTO_BLOWFISH=m CONFIG_CRYPTO_CAST6=m CONFIG_CRYPTO_SERPENT=m diff --git a/arch/powerpc/configs/ppc64e_defconfig b/arch/powerpc/configs/ppc64e_defconfig index d2e659a2d8cb..90247b2a0ab0 100644 --- a/arch/powerpc/configs/ppc64e_defconfig +++ b/arch/powerpc/configs/ppc64e_defconfig @@ -220,7 +220,7 @@ CONFIG_CODE_PATCHING_SELFTEST=y CONFIG_FTR_FIXUP_SELFTEST=y CONFIG_MSI_BITMAP_SELFTEST=y CONFIG_XMON=y -CONFIG_CRYPTO_TEST=m +CONFIG_CRYPTO_BENCHMARK=m CONFIG_CRYPTO_CCM=m CONFIG_CRYPTO_GCM=m CONFIG_CRYPTO_PCBC=m diff --git a/arch/powerpc/configs/ppc6xx_defconfig b/arch/powerpc/configs/ppc6xx_defconfig index a91a766b71a4..242c1fab9d46 100644 --- a/arch/powerpc/configs/ppc6xx_defconfig +++ b/arch/powerpc/configs/ppc6xx_defconfig @@ -1073,7 +1073,7 @@ CONFIG_SECURITY_NETWORK_XFRM=y CONFIG_SECURITY_SELINUX=y CONFIG_SECURITY_SELINUX_BOOTPARAM=y CONFIG_SECURITY_SELINUX_DISABLE=y -CONFIG_CRYPTO_TEST=m +CONFIG_CRYPTO_BENCHMARK=m CONFIG_CRYPTO_CTS=m CONFIG_CRYPTO_LRW=m CONFIG_CRYPTO_PCBC=m diff --git a/arch/s390/configs/debug_defconfig b/arch/s390/configs/debug_defconfig index de69faa4d94f..e6f7c8f4ee8b 100644 --- a/arch/s390/configs/debug_defconfig +++ b/arch/s390/configs/debug_defconfig @@ -746,7 +746,7 @@ CONFIG_CRYPTO_USER=m # CONFIG_CRYPTO_MANAGER_DISABLE_TESTS is not set CONFIG_CRYPTO_PCRYPT=m CONFIG_CRYPTO_CRYPTD=m -CONFIG_CRYPTO_TEST=m +CONFIG_CRYPTO_BENCHMARK=m CONFIG_CRYPTO_DH=m CONFIG_CRYPTO_ECDH=m CONFIG_CRYPTO_ECDSA=m diff --git a/arch/s390/configs/defconfig b/arch/s390/configs/defconfig index f12679448e97..7495bf8e483c 100644 --- a/arch/s390/configs/defconfig +++ b/arch/s390/configs/defconfig @@ -732,7 +732,7 @@ CONFIG_CRYPTO_USER=m # CONFIG_CRYPTO_MANAGER_DISABLE_TESTS is not set CONFIG_CRYPTO_PCRYPT=m CONFIG_CRYPTO_CRYPTD=m -CONFIG_CRYPTO_TEST=m +CONFIG_CRYPTO_BENCHMARK=m CONFIG_CRYPTO_DH=m CONFIG_CRYPTO_ECDH=m CONFIG_CRYPTO_ECDSA=m diff --git a/arch/sparc/configs/sparc64_defconfig b/arch/sparc/configs/sparc64_defconfig index f1ba0fefe1f9..7a7c4dec2925 100644 --- a/arch/sparc/configs/sparc64_defconfig +++ b/arch/sparc/configs/sparc64_defconfig @@ -205,7 +205,7 @@ CONFIG_BLK_DEV_IO_TRACE=y CONFIG_UPROBE_EVENTS=y CONFIG_KEYS=y CONFIG_CRYPTO_NULL=m -CONFIG_CRYPTO_TEST=m +CONFIG_CRYPTO_BENCHMARK=m CONFIG_CRYPTO_LRW=m CONFIG_CRYPTO_PCBC=m CONFIG_CRYPTO_XTS=m diff --git a/crypto/Kconfig b/crypto/Kconfig index 7347277bedf3..08af9de30ff0 100644 --- a/crypto/Kconfig +++ b/crypto/Kconfig @@ -240,12 +240,16 @@ config CRYPTO_KRB5ENC profile. This is required for Kerberos 5-style encryption, used by sunrpc/NFS and rxrpc/AFS. -config CRYPTO_TEST - tristate "Testing module" +config CRYPTO_BENCHMARK + tristate "Crypto benchmarking module" depends on m || EXPERT select CRYPTO_MANAGER help - Quick & dirty crypto test module. + Quick & dirty crypto benchmarking module. + + This is mainly intended for use by people developing cryptographic + algorithms in the kernel. It should not be enabled in production + kernels. config CRYPTO_SIMD tristate diff --git a/crypto/Makefile b/crypto/Makefile index 0f77e093512c..0b4641c5bbe1 100644 --- a/crypto/Makefile +++ b/crypto/Makefile @@ -172,7 +172,7 @@ KASAN_SANITIZE_jitterentropy.o = n UBSAN_SANITIZE_jitterentropy.o = n jitterentropy_rng-y := jitterentropy.o jitterentropy-kcapi.o obj-$(CONFIG_CRYPTO_JITTERENTROPY_TESTINTERFACE) += jitterentropy-testing.o -obj-$(CONFIG_CRYPTO_TEST) += tcrypt.o +obj-$(CONFIG_CRYPTO_BENCHMARK) += tcrypt.o obj-$(CONFIG_CRYPTO_GHASH) += ghash-generic.o obj-$(CONFIG_CRYPTO_POLYVAL) += polyval-generic.o obj-$(CONFIG_CRYPTO_USER_API) += af_alg.o diff --git a/crypto/tcrypt.c b/crypto/tcrypt.c index 879fc21dcc16..d1d88debbd71 100644 --- a/crypto/tcrypt.c +++ b/crypto/tcrypt.c @@ -1,8 +1,8 @@ // SPDX-License-Identifier: GPL-2.0-or-later /* - * Quick & dirty crypto testing module. + * Quick & dirty crypto benchmarking module. * - * This will only exist until we have a better testing mechanism + * This will only exist until we have a better benchmarking mechanism * (e.g. a char device). * * Copyright (c) 2002 James Morris @@ -39,7 +39,7 @@ #include "tcrypt.h" /* - * Need slab memory for testing (size in number of pages). + * Need slab memory for benchmarking (size in number of pages). */ #define TVMEMSIZE 4 @@ -2868,5 +2868,5 @@ module_param(klen, uint, 0); MODULE_PARM_DESC(klen, "Key length (defaults to 0)"); MODULE_LICENSE("GPL"); -MODULE_DESCRIPTION("Quick & dirty crypto testing module"); +MODULE_DESCRIPTION("Quick & dirty crypto benchmarking module"); MODULE_AUTHOR("James Morris "); diff --git a/crypto/tcrypt.h b/crypto/tcrypt.h index 96c843a24607..7f938ac93e58 100644 --- a/crypto/tcrypt.h +++ b/crypto/tcrypt.h @@ -1,8 +1,8 @@ /* SPDX-License-Identifier: GPL-2.0-or-later */ /* - * Quick & dirty crypto testing module. + * Quick & dirty crypto benchmarking module. * - * This will only exist until we have a better testing mechanism + * This will only exist until we have a better benchmarking mechanism * (e.g. a char device). * * Copyright (c) 2002 James Morris -- 2.51.0 From d469eaed223fa485eabebd3bcd05ddd3c891f54e Mon Sep 17 00:00:00 2001 From: Eric Biggers Date: Mon, 5 May 2025 13:33:40 -0700 Subject: [PATCH 03/16] crypto: testmgr - remove panic_on_fail The cryptomgr.panic_on_fail=1 kernel command-line parameter is not very useful now that the tests have been fixed to WARN on failure, since developers can just use panic_on_warn=1 instead. There's no need for a special option just for the crypto self-tests. Remove it. Signed-off-by: Eric Biggers Signed-off-by: Herbert Xu --- crypto/testmgr.c | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) diff --git a/crypto/testmgr.c b/crypto/testmgr.c index f100be516f52..1c71616ae2cf 100644 --- a/crypto/testmgr.c +++ b/crypto/testmgr.c @@ -45,9 +45,6 @@ static bool notests; module_param(notests, bool, 0644); MODULE_PARM_DESC(notests, "disable crypto self-tests"); -static bool panic_on_fail; -module_param(panic_on_fail, bool, 0444); - #ifdef CONFIG_CRYPTO_MANAGER_EXTRA_TESTS static bool noextratests; module_param(noextratests, bool, 0644); @@ -5854,11 +5851,10 @@ int alg_test(const char *driver, const char *alg, u32 type, u32 mask) test_done: if (rc) { - if (fips_enabled || panic_on_fail) { + if (fips_enabled) { fips_fail_notify(); - panic("alg: self-tests for %s (%s) failed in %s mode!\n", - driver, alg, - fips_enabled ? "fips" : "panic_on_fail"); + panic("alg: self-tests for %s (%s) failed in fips mode!\n", + driver, alg); } pr_warn("alg: self-tests for %s using %s failed (rc=%d)", alg, driver, rc); -- 2.51.0 From 40b9969796bfa49ed1b0f7ddc254f48cb2ac6d2c Mon Sep 17 00:00:00 2001 From: Eric Biggers Date: Mon, 5 May 2025 13:33:41 -0700 Subject: [PATCH 04/16] crypto: testmgr - replace CRYPTO_MANAGER_DISABLE_TESTS with CRYPTO_SELFTESTS The negative-sense of CRYPTO_MANAGER_DISABLE_TESTS is a longstanding mistake that regularly causes confusion. Especially bad is that you can have CRYPTO=n && CRYPTO_MANAGER_DISABLE_TESTS=n, which is ambiguous. Replace CRYPTO_MANAGER_DISABLE_TESTS with CRYPTO_SELFTESTS which has the expected behavior. The tests continue to be disabled by default. Signed-off-by: Eric Biggers Signed-off-by: Herbert Xu --- arch/arm/configs/milbeaut_m10v_defconfig | 2 +- arch/loongarch/configs/loongson3_defconfig | 2 +- arch/s390/configs/debug_defconfig | 2 +- arch/s390/configs/defconfig | 2 +- crypto/Kconfig | 24 ++++++++++++++-------- crypto/algapi.c | 4 ++-- crypto/algboss.c | 2 +- crypto/api.c | 3 +-- crypto/hkdf.c | 2 +- crypto/internal.h | 5 ++--- crypto/kdf_sp800108.c | 2 +- crypto/testmgr.c | 12 +++++------ lib/crypto/Makefile | 9 +++----- lib/crypto/aescfb.c | 2 +- lib/crypto/aesgcm.c | 2 +- lib/crypto/blake2s.c | 2 +- lib/crypto/chacha20poly1305.c | 2 +- lib/crypto/curve25519.c | 2 +- 18 files changed, 41 insertions(+), 40 deletions(-) diff --git a/arch/arm/configs/milbeaut_m10v_defconfig b/arch/arm/configs/milbeaut_m10v_defconfig index 4ec21f477c63..9dd47e1d90bb 100644 --- a/arch/arm/configs/milbeaut_m10v_defconfig +++ b/arch/arm/configs/milbeaut_m10v_defconfig @@ -94,7 +94,7 @@ CONFIG_NLS_ISO8859_1=y CONFIG_NLS_UTF8=y CONFIG_KEYS=y CONFIG_CRYPTO_MANAGER=y -# CONFIG_CRYPTO_MANAGER_DISABLE_TESTS is not set +CONFIG_CRYPTO_SELFTESTS=y # CONFIG_CRYPTO_ECHAINIV is not set CONFIG_CRYPTO_AES=y CONFIG_CRYPTO_SEQIV=m diff --git a/arch/loongarch/configs/loongson3_defconfig b/arch/loongarch/configs/loongson3_defconfig index 90f21dfe22b1..0d59af6007b7 100644 --- a/arch/loongarch/configs/loongson3_defconfig +++ b/arch/loongarch/configs/loongson3_defconfig @@ -1026,7 +1026,7 @@ CONFIG_SECURITY_APPARMOR=y CONFIG_SECURITY_YAMA=y CONFIG_DEFAULT_SECURITY_DAC=y CONFIG_CRYPTO_USER=m -# CONFIG_CRYPTO_MANAGER_DISABLE_TESTS is not set +CONFIG_CRYPTO_SELFTESTS=y CONFIG_CRYPTO_PCRYPT=m CONFIG_CRYPTO_CRYPTD=m CONFIG_CRYPTO_ANUBIS=m diff --git a/arch/s390/configs/debug_defconfig b/arch/s390/configs/debug_defconfig index e6f7c8f4ee8b..bc960b1eabae 100644 --- a/arch/s390/configs/debug_defconfig +++ b/arch/s390/configs/debug_defconfig @@ -743,7 +743,7 @@ CONFIG_IMA_WRITE_POLICY=y CONFIG_IMA_APPRAISE=y CONFIG_BUG_ON_DATA_CORRUPTION=y CONFIG_CRYPTO_USER=m -# CONFIG_CRYPTO_MANAGER_DISABLE_TESTS is not set +CONFIG_CRYPTO_SELFTESTS=y CONFIG_CRYPTO_PCRYPT=m CONFIG_CRYPTO_CRYPTD=m CONFIG_CRYPTO_BENCHMARK=m diff --git a/arch/s390/configs/defconfig b/arch/s390/configs/defconfig index 7495bf8e483c..fda2f264e7d3 100644 --- a/arch/s390/configs/defconfig +++ b/arch/s390/configs/defconfig @@ -729,7 +729,7 @@ CONFIG_IMA_APPRAISE=y CONFIG_BUG_ON_DATA_CORRUPTION=y CONFIG_CRYPTO_FIPS=y CONFIG_CRYPTO_USER=m -# CONFIG_CRYPTO_MANAGER_DISABLE_TESTS is not set +CONFIG_CRYPTO_SELFTESTS=y CONFIG_CRYPTO_PCRYPT=m CONFIG_CRYPTO_CRYPTD=m CONFIG_CRYPTO_BENCHMARK=m diff --git a/crypto/Kconfig b/crypto/Kconfig index 08af9de30ff0..d326d3c5544f 100644 --- a/crypto/Kconfig +++ b/crypto/Kconfig @@ -25,7 +25,7 @@ menu "Crypto core or helper" config CRYPTO_FIPS bool "FIPS 200 compliance" - depends on (CRYPTO_ANSI_CPRNG || CRYPTO_DRBG) && !CRYPTO_MANAGER_DISABLE_TESTS + depends on (CRYPTO_ANSI_CPRNG || CRYPTO_DRBG) && CRYPTO_SELFTESTS depends on (MODULE_SIG || !MODULES) help This option enables the fips boot option which is @@ -143,8 +143,8 @@ config CRYPTO_ACOMP config CRYPTO_HKDF tristate - select CRYPTO_SHA256 if !CONFIG_CRYPTO_MANAGER_DISABLE_TESTS - select CRYPTO_SHA512 if !CONFIG_CRYPTO_MANAGER_DISABLE_TESTS + select CRYPTO_SHA256 if CRYPTO_SELFTESTS + select CRYPTO_SHA512 if CRYPTO_SELFTESTS select CRYPTO_HASH2 config CRYPTO_MANAGER @@ -173,16 +173,22 @@ config CRYPTO_USER Userspace configuration for cryptographic instantiations such as cbc(aes). -config CRYPTO_MANAGER_DISABLE_TESTS - bool "Disable run-time self tests" - default y +config CRYPTO_SELFTESTS + bool "Enable cryptographic self-tests" + depends on DEBUG_KERNEL help - Disable run-time self tests that normally take place at - algorithm registration. + Enable the cryptographic self-tests. + + The cryptographic self-tests run at boot time, or at algorithm + registration time if algorithms are dynamically loaded later. + + This is primarily intended for developer use. It should not be + enabled in production kernels, unless you are trying to use these + tests to fulfill a FIPS testing requirement. config CRYPTO_MANAGER_EXTRA_TESTS bool "Enable extra run-time crypto self tests" - depends on DEBUG_KERNEL && !CRYPTO_MANAGER_DISABLE_TESTS && CRYPTO_MANAGER + depends on DEBUG_KERNEL && CRYPTO_SELFTESTS && CRYPTO_MANAGER help Enable extra run-time self tests of registered crypto algorithms, including randomized fuzz tests. diff --git a/crypto/algapi.c b/crypto/algapi.c index 532d3efc3c7d..9030a30c89e8 100644 --- a/crypto/algapi.c +++ b/crypto/algapi.c @@ -275,7 +275,7 @@ static struct crypto_larval *crypto_alloc_test_larval(struct crypto_alg *alg) struct crypto_larval *larval; if (!IS_ENABLED(CONFIG_CRYPTO_MANAGER) || - IS_ENABLED(CONFIG_CRYPTO_MANAGER_DISABLE_TESTS) || + !IS_ENABLED(CONFIG_CRYPTO_SELFTESTS) || (alg->cra_flags & CRYPTO_ALG_INTERNAL)) return NULL; /* No self-test needed */ @@ -1059,7 +1059,7 @@ static void __init crypto_start_tests(void) if (!IS_BUILTIN(CONFIG_CRYPTO_ALGAPI)) return; - if (IS_ENABLED(CONFIG_CRYPTO_MANAGER_DISABLE_TESTS)) + if (!IS_ENABLED(CONFIG_CRYPTO_SELFTESTS)) return; set_crypto_boot_test_finished(); diff --git a/crypto/algboss.c b/crypto/algboss.c index ef5c73780fc7..846f586889ee 100644 --- a/crypto/algboss.c +++ b/crypto/algboss.c @@ -189,7 +189,7 @@ static int cryptomgr_schedule_test(struct crypto_alg *alg) struct task_struct *thread; struct crypto_test_param *param; - if (IS_ENABLED(CONFIG_CRYPTO_MANAGER_DISABLE_TESTS)) + if (!IS_ENABLED(CONFIG_CRYPTO_SELFTESTS)) return NOTIFY_DONE; if (!try_module_get(THIS_MODULE)) diff --git a/crypto/api.c b/crypto/api.c index 5cd5ec105bb1..133d9b626922 100644 --- a/crypto/api.c +++ b/crypto/api.c @@ -31,8 +31,7 @@ EXPORT_SYMBOL_GPL(crypto_alg_sem); BLOCKING_NOTIFIER_HEAD(crypto_chain); EXPORT_SYMBOL_GPL(crypto_chain); -#if IS_BUILTIN(CONFIG_CRYPTO_ALGAPI) && \ - !IS_ENABLED(CONFIG_CRYPTO_MANAGER_DISABLE_TESTS) +#if IS_BUILTIN(CONFIG_CRYPTO_ALGAPI) && IS_ENABLED(CONFIG_CRYPTO_SELFTESTS) DEFINE_STATIC_KEY_FALSE(__crypto_boot_test_finished); #endif diff --git a/crypto/hkdf.c b/crypto/hkdf.c index 2434c5c42545..f24c2a8d4df9 100644 --- a/crypto/hkdf.c +++ b/crypto/hkdf.c @@ -543,7 +543,7 @@ static int __init crypto_hkdf_module_init(void) { int ret = 0, i; - if (IS_ENABLED(CONFIG_CRYPTO_MANAGER_DISABLE_TESTS)) + if (!IS_ENABLED(CONFIG_CRYPTO_SELFTESTS)) return 0; for (i = 0; i < ARRAY_SIZE(hkdf_sha256_tv); i++) { diff --git a/crypto/internal.h b/crypto/internal.h index 2ed79bf208ca..b9afd68767c1 100644 --- a/crypto/internal.h +++ b/crypto/internal.h @@ -67,8 +67,7 @@ extern struct blocking_notifier_head crypto_chain; int alg_test(const char *driver, const char *alg, u32 type, u32 mask); -#if !IS_BUILTIN(CONFIG_CRYPTO_ALGAPI) || \ - IS_ENABLED(CONFIG_CRYPTO_MANAGER_DISABLE_TESTS) +#if !IS_BUILTIN(CONFIG_CRYPTO_ALGAPI) || !IS_ENABLED(CONFIG_CRYPTO_SELFTESTS) static inline bool crypto_boot_test_finished(void) { return true; @@ -87,7 +86,7 @@ static inline void set_crypto_boot_test_finished(void) static_branch_enable(&__crypto_boot_test_finished); } #endif /* !IS_BUILTIN(CONFIG_CRYPTO_ALGAPI) || - * IS_ENABLED(CONFIG_CRYPTO_MANAGER_DISABLE_TESTS) + * !IS_ENABLED(CONFIG_CRYPTO_SELFTESTS) */ #ifdef CONFIG_PROC_FS diff --git a/crypto/kdf_sp800108.c b/crypto/kdf_sp800108.c index c3f9938e1ad2..b7a6bf9da773 100644 --- a/crypto/kdf_sp800108.c +++ b/crypto/kdf_sp800108.c @@ -127,7 +127,7 @@ static int __init crypto_kdf108_init(void) { int ret; - if (IS_ENABLED(CONFIG_CRYPTO_MANAGER_DISABLE_TESTS)) + if (!IS_ENABLED(CONFIG_CRYPTO_SELFTESTS)) return 0; ret = kdf_test(&kdf_ctr_hmac_sha256_tv_template[0], "hmac(sha256)", diff --git a/crypto/testmgr.c b/crypto/testmgr.c index 1c71616ae2cf..3a624c9b5525 100644 --- a/crypto/testmgr.c +++ b/crypto/testmgr.c @@ -55,7 +55,7 @@ module_param(fuzz_iterations, uint, 0644); MODULE_PARM_DESC(fuzz_iterations, "number of fuzz test iterations"); #endif -#ifdef CONFIG_CRYPTO_MANAGER_DISABLE_TESTS +#ifndef CONFIG_CRYPTO_SELFTESTS /* a perfect nop */ int alg_test(const char *driver, const char *alg, u32 type, u32 mask) @@ -321,10 +321,10 @@ struct testvec_config { /* * The following are the lists of testvec_configs to test for each algorithm - * type when the basic crypto self-tests are enabled, i.e. when - * CONFIG_CRYPTO_MANAGER_DISABLE_TESTS is unset. They aim to provide good test - * coverage, while keeping the test time much shorter than the full fuzz tests - * so that the basic tests can be enabled in a wider range of circumstances. + * type when the basic crypto self-tests are enabled. They aim to provide good + * test coverage, while keeping the test time much shorter than the full fuzz + * tests so that the basic tests can be enabled in a wider range of + * circumstances. */ /* Configs for skciphers and aeads */ @@ -5899,6 +5899,6 @@ non_fips_alg: return alg_fips_disabled(driver, alg); } -#endif /* CONFIG_CRYPTO_MANAGER_DISABLE_TESTS */ +#endif /* CONFIG_CRYPTO_SELFTESTS */ EXPORT_SYMBOL_GPL(alg_test); diff --git a/lib/crypto/Makefile b/lib/crypto/Makefile index ff4aa22e5ccc..599e61375938 100644 --- a/lib/crypto/Makefile +++ b/lib/crypto/Makefile @@ -25,9 +25,11 @@ obj-$(CONFIG_CRYPTO_LIB_GF128MUL) += gf128mul.o obj-y += libblake2s.o libblake2s-y := blake2s.o libblake2s-$(CONFIG_CRYPTO_LIB_BLAKE2S_GENERIC) += blake2s-generic.o +libblake2s-$(CONFIG_CRYPTO_SELFTESTS) += blake2s-selftest.o obj-$(CONFIG_CRYPTO_LIB_CHACHA20POLY1305) += libchacha20poly1305.o libchacha20poly1305-y += chacha20poly1305.o +libchacha20poly1305-$(CONFIG_CRYPTO_SELFTESTS) += chacha20poly1305-selftest.o obj-$(CONFIG_CRYPTO_LIB_CURVE25519_GENERIC) += libcurve25519-generic.o libcurve25519-generic-y := curve25519-fiat32.o @@ -36,6 +38,7 @@ libcurve25519-generic-y += curve25519-generic.o obj-$(CONFIG_CRYPTO_LIB_CURVE25519) += libcurve25519.o libcurve25519-y += curve25519.o +libcurve25519-$(CONFIG_CRYPTO_SELFTESTS) += curve25519-selftest.o obj-$(CONFIG_CRYPTO_LIB_DES) += libdes.o libdes-y := des.o @@ -57,12 +60,6 @@ libsha256-y := sha256.o obj-$(CONFIG_CRYPTO_LIB_SHA256_GENERIC) += libsha256-generic.o libsha256-generic-y := sha256-generic.o -ifneq ($(CONFIG_CRYPTO_MANAGER_DISABLE_TESTS),y) -libblake2s-y += blake2s-selftest.o -libchacha20poly1305-y += chacha20poly1305-selftest.o -libcurve25519-y += curve25519-selftest.o -endif - obj-$(CONFIG_MPILIB) += mpi/ obj-$(CONFIG_CRYPTO_MANAGER_EXTRA_TESTS) += simd.o diff --git a/lib/crypto/aescfb.c b/lib/crypto/aescfb.c index 749dc1258a44..437613265e14 100644 --- a/lib/crypto/aescfb.c +++ b/lib/crypto/aescfb.c @@ -99,7 +99,7 @@ MODULE_DESCRIPTION("Generic AES-CFB library"); MODULE_AUTHOR("Ard Biesheuvel "); MODULE_LICENSE("GPL"); -#ifndef CONFIG_CRYPTO_MANAGER_DISABLE_TESTS +#ifdef CONFIG_CRYPTO_SELFTESTS /* * Test code below. Vectors taken from crypto/testmgr.h diff --git a/lib/crypto/aesgcm.c b/lib/crypto/aesgcm.c index 902e49410aaf..277824d6b4af 100644 --- a/lib/crypto/aesgcm.c +++ b/lib/crypto/aesgcm.c @@ -199,7 +199,7 @@ MODULE_DESCRIPTION("Generic AES-GCM library"); MODULE_AUTHOR("Ard Biesheuvel "); MODULE_LICENSE("GPL"); -#ifndef CONFIG_CRYPTO_MANAGER_DISABLE_TESTS +#ifdef CONFIG_CRYPTO_SELFTESTS /* * Test code below. Vectors taken from crypto/testmgr.h diff --git a/lib/crypto/blake2s.c b/lib/crypto/blake2s.c index 71a316552cc5..b0f9a678300b 100644 --- a/lib/crypto/blake2s.c +++ b/lib/crypto/blake2s.c @@ -60,7 +60,7 @@ EXPORT_SYMBOL(blake2s_final); static int __init blake2s_mod_init(void) { - if (!IS_ENABLED(CONFIG_CRYPTO_MANAGER_DISABLE_TESTS) && + if (IS_ENABLED(CONFIG_CRYPTO_SELFTESTS) && WARN_ON(!blake2s_selftest())) return -ENODEV; return 0; diff --git a/lib/crypto/chacha20poly1305.c b/lib/crypto/chacha20poly1305.c index fbd3690e2531..e29eed49a5a1 100644 --- a/lib/crypto/chacha20poly1305.c +++ b/lib/crypto/chacha20poly1305.c @@ -358,7 +358,7 @@ EXPORT_SYMBOL(chacha20poly1305_decrypt_sg_inplace); static int __init chacha20poly1305_init(void) { - if (!IS_ENABLED(CONFIG_CRYPTO_MANAGER_DISABLE_TESTS) && + if (IS_ENABLED(CONFIG_CRYPTO_SELFTESTS) && WARN_ON(!chacha20poly1305_selftest())) return -ENODEV; return 0; diff --git a/lib/crypto/curve25519.c b/lib/crypto/curve25519.c index 064b352c6907..6850b76a80c9 100644 --- a/lib/crypto/curve25519.c +++ b/lib/crypto/curve25519.c @@ -15,7 +15,7 @@ static int __init curve25519_init(void) { - if (!IS_ENABLED(CONFIG_CRYPTO_MANAGER_DISABLE_TESTS) && + if (IS_ENABLED(CONFIG_CRYPTO_SELFTESTS) && WARN_ON(!curve25519_selftest())) return -ENODEV; return 0; -- 2.51.0 From 698de822780fbae79b11e5d749863c1aa64a0a55 Mon Sep 17 00:00:00 2001 From: Eric Biggers Date: Mon, 5 May 2025 13:33:42 -0700 Subject: [PATCH 05/16] crypto: testmgr - make it easier to enable the full set of tests Currently the full set of crypto self-tests requires CONFIG_CRYPTO_MANAGER_EXTRA_TESTS=y. This is problematic in two ways. First, developers regularly overlook this option. Second, the description of the tests as "extra" sometimes gives the impression that it is not required that all algorithms pass these tests. Given that the main use case for the crypto self-tests is for developers, make enabling CONFIG_CRYPTO_SELFTESTS=y just enable the full set of crypto self-tests by default. The slow tests can still be disabled by adding the command-line parameter cryptomgr.noextratests=1, soon to be renamed to cryptomgr.noslowtests=1. The only known use case for doing this is for people trying to use the crypto self-tests to satisfy the FIPS 140-3 pre-operational self-testing requirements when the kernel is being validated as a FIPS 140-3 cryptographic module. Signed-off-by: Eric Biggers Signed-off-by: Herbert Xu --- crypto/Kconfig | 10 ------ crypto/testmgr.c | 62 +++------------------------------- include/crypto/internal/simd.h | 6 ++-- lib/crypto/Makefile | 2 +- 4 files changed, 8 insertions(+), 72 deletions(-) diff --git a/crypto/Kconfig b/crypto/Kconfig index d326d3c5544f..277fc9335767 100644 --- a/crypto/Kconfig +++ b/crypto/Kconfig @@ -186,16 +186,6 @@ config CRYPTO_SELFTESTS enabled in production kernels, unless you are trying to use these tests to fulfill a FIPS testing requirement. -config CRYPTO_MANAGER_EXTRA_TESTS - bool "Enable extra run-time crypto self tests" - depends on DEBUG_KERNEL && CRYPTO_SELFTESTS && CRYPTO_MANAGER - help - Enable extra run-time self tests of registered crypto algorithms, - including randomized fuzz tests. - - This is intended for developer use only, as these tests take much - longer to run than the normal self tests. - config CRYPTO_NULL tristate "Null algorithms" select CRYPTO_ALGAPI diff --git a/crypto/testmgr.c b/crypto/testmgr.c index 3a624c9b5525..a6707cda8328 100644 --- a/crypto/testmgr.c +++ b/crypto/testmgr.c @@ -45,7 +45,6 @@ static bool notests; module_param(notests, bool, 0644); MODULE_PARM_DESC(notests, "disable crypto self-tests"); -#ifdef CONFIG_CRYPTO_MANAGER_EXTRA_TESTS static bool noextratests; module_param(noextratests, bool, 0644); MODULE_PARM_DESC(noextratests, "disable expensive crypto self-tests"); @@ -53,7 +52,6 @@ MODULE_PARM_DESC(noextratests, "disable expensive crypto self-tests"); static unsigned int fuzz_iterations = 100; module_param(fuzz_iterations, uint, 0644); MODULE_PARM_DESC(fuzz_iterations, "number of fuzz test iterations"); -#endif #ifndef CONFIG_CRYPTO_SELFTESTS @@ -321,10 +319,9 @@ struct testvec_config { /* * The following are the lists of testvec_configs to test for each algorithm - * type when the basic crypto self-tests are enabled. They aim to provide good - * test coverage, while keeping the test time much shorter than the full fuzz - * tests so that the basic tests can be enabled in a wider range of - * circumstances. + * type when the fast crypto self-tests are enabled. They aim to provide good + * test coverage, while keeping the test time much shorter than the full tests + * so that the fast tests can be used to fulfill FIPS 140 testing requirements. */ /* Configs for skciphers and aeads */ @@ -873,8 +870,6 @@ static int prepare_keybuf(const u8 *key, unsigned int ksize, err; \ }) -#ifdef CONFIG_CRYPTO_MANAGER_EXTRA_TESTS - /* * The fuzz tests use prandom instead of the normal Linux RNG since they don't * need cryptographically secure random numbers. This greatly improves the @@ -1239,15 +1234,6 @@ too_long: algname); return -ENAMETOOLONG; } -#else /* !CONFIG_CRYPTO_MANAGER_EXTRA_TESTS */ -static void crypto_disable_simd_for_test(void) -{ -} - -static void crypto_reenable_simd_for_test(void) -{ -} -#endif /* !CONFIG_CRYPTO_MANAGER_EXTRA_TESTS */ static int build_hash_sglist(struct test_sglist *tsgl, const struct hash_testvec *vec, @@ -1688,7 +1674,6 @@ static int test_hash_vec(const struct hash_testvec *vec, unsigned int vec_num, return err; } -#ifdef CONFIG_CRYPTO_MANAGER_EXTRA_TESTS if (!noextratests) { struct rnd_state rng; struct testvec_config cfg; @@ -1706,11 +1691,9 @@ static int test_hash_vec(const struct hash_testvec *vec, unsigned int vec_num, cond_resched(); } } -#endif return 0; } -#ifdef CONFIG_CRYPTO_MANAGER_EXTRA_TESTS /* * Generate a hash test vector from the given implementation. * Assumes the buffers in 'vec' were already allocated. @@ -1876,17 +1859,6 @@ out: kfree_sensitive(generic_desc); return err; } -#else /* !CONFIG_CRYPTO_MANAGER_EXTRA_TESTS */ -static int test_hash_vs_generic_impl(const char *generic_driver, - unsigned int maxkeysize, - struct ahash_request *req, - struct shash_desc *desc, - struct test_sglist *tsgl, - u8 *hashstate) -{ - return 0; -} -#endif /* !CONFIG_CRYPTO_MANAGER_EXTRA_TESTS */ static int alloc_shash(const char *driver, u32 type, u32 mask, struct crypto_shash **tfm_ret, @@ -2260,7 +2232,6 @@ static int test_aead_vec(int enc, const struct aead_testvec *vec, return err; } -#ifdef CONFIG_CRYPTO_MANAGER_EXTRA_TESTS if (!noextratests) { struct rnd_state rng; struct testvec_config cfg; @@ -2278,12 +2249,9 @@ static int test_aead_vec(int enc, const struct aead_testvec *vec, cond_resched(); } } -#endif return 0; } -#ifdef CONFIG_CRYPTO_MANAGER_EXTRA_TESTS - struct aead_extra_tests_ctx { struct rnd_state rng; struct aead_request *req; @@ -2668,14 +2636,6 @@ out: kfree(ctx); return err; } -#else /* !CONFIG_CRYPTO_MANAGER_EXTRA_TESTS */ -static int test_aead_extra(const struct alg_test_desc *test_desc, - struct aead_request *req, - struct cipher_test_sglists *tsgls) -{ - return 0; -} -#endif /* !CONFIG_CRYPTO_MANAGER_EXTRA_TESTS */ static int test_aead(int enc, const struct aead_test_suite *suite, struct aead_request *req, @@ -3015,7 +2975,6 @@ static int test_skcipher_vec(int enc, const struct cipher_testvec *vec, return err; } -#ifdef CONFIG_CRYPTO_MANAGER_EXTRA_TESTS if (!noextratests) { struct rnd_state rng; struct testvec_config cfg; @@ -3033,11 +2992,9 @@ static int test_skcipher_vec(int enc, const struct cipher_testvec *vec, cond_resched(); } } -#endif return 0; } -#ifdef CONFIG_CRYPTO_MANAGER_EXTRA_TESTS /* * Generate a symmetric cipher test vector from the given implementation. * Assumes the buffers in 'vec' were already allocated. @@ -3236,14 +3193,6 @@ out: skcipher_request_free(generic_req); return err; } -#else /* !CONFIG_CRYPTO_MANAGER_EXTRA_TESTS */ -static int test_skcipher_vs_generic_impl(const char *generic_driver, - struct skcipher_request *req, - struct cipher_test_sglists *tsgls) -{ - return 0; -} -#endif /* !CONFIG_CRYPTO_MANAGER_EXTRA_TESTS */ static int test_skcipher(int enc, const struct cipher_test_suite *suite, struct skcipher_request *req, @@ -5760,9 +5709,8 @@ static void testmgr_onetime_init(void) alg_check_test_descs_order(); alg_check_testvec_configs(); -#ifdef CONFIG_CRYPTO_MANAGER_EXTRA_TESTS - pr_warn("alg: extra crypto tests enabled. This is intended for developer use only.\n"); -#endif + if (!noextratests) + pr_warn("alg: full crypto tests enabled. This is intended for developer use only.\n"); } static int alg_find_test(const char *alg) diff --git a/include/crypto/internal/simd.h b/include/crypto/internal/simd.h index f56049bd1660..7e7f1ac3b7fd 100644 --- a/include/crypto/internal/simd.h +++ b/include/crypto/internal/simd.h @@ -44,11 +44,9 @@ void simd_unregister_aeads(struct aead_alg *algs, int count, * * This delegates to may_use_simd(), except that this also returns false if SIMD * in crypto code has been temporarily disabled on this CPU by the crypto - * self-tests, in order to test the no-SIMD fallback code. This override is - * currently limited to configurations where the extra self-tests are enabled, - * because it might be a bit too invasive to be part of the regular self-tests. + * self-tests, in order to test the no-SIMD fallback code. */ -#ifdef CONFIG_CRYPTO_MANAGER_EXTRA_TESTS +#ifdef CONFIG_CRYPTO_SELFTESTS DECLARE_PER_CPU(bool, crypto_simd_disabled_for_test); #define crypto_simd_usable() \ (may_use_simd() && !this_cpu_read(crypto_simd_disabled_for_test)) diff --git a/lib/crypto/Makefile b/lib/crypto/Makefile index 599e61375938..3e79283b617d 100644 --- a/lib/crypto/Makefile +++ b/lib/crypto/Makefile @@ -62,7 +62,7 @@ libsha256-generic-y := sha256-generic.o obj-$(CONFIG_MPILIB) += mpi/ -obj-$(CONFIG_CRYPTO_MANAGER_EXTRA_TESTS) += simd.o +obj-$(CONFIG_CRYPTO_SELFTESTS) += simd.o obj-$(CONFIG_CRYPTO_LIB_SM3) += libsm3.o libsm3-y := sm3.o -- 2.51.0 From 42e45119efba92461839d1473abda5e31b0e71ed Mon Sep 17 00:00:00 2001 From: Eric Biggers Date: Mon, 5 May 2025 13:33:43 -0700 Subject: [PATCH 06/16] crypto: testmgr - rename noextratests to noslowtests Rename the noextratests module parameter to noslowtests, and replace other remaining mentions of "extra" in the code with "slow". This addresses confusion regarding the word "extra" like that seen at https://lore.kernel.org/r/6cecf2de-9aa0-f6ea-0c2d-8e974a1a820b@huawei.com/. Signed-off-by: Eric Biggers Signed-off-by: Herbert Xu --- crypto/testmgr.c | 41 ++++++++++++++++++++--------------------- 1 file changed, 20 insertions(+), 21 deletions(-) diff --git a/crypto/testmgr.c b/crypto/testmgr.c index a6707cda8328..fc28000c27f5 100644 --- a/crypto/testmgr.c +++ b/crypto/testmgr.c @@ -43,11 +43,11 @@ MODULE_IMPORT_NS("CRYPTO_INTERNAL"); static bool notests; module_param(notests, bool, 0644); -MODULE_PARM_DESC(notests, "disable crypto self-tests"); +MODULE_PARM_DESC(notests, "disable all crypto self-tests"); -static bool noextratests; -module_param(noextratests, bool, 0644); -MODULE_PARM_DESC(noextratests, "disable expensive crypto self-tests"); +static bool noslowtests; +module_param(noslowtests, bool, 0644); +MODULE_PARM_DESC(noslowtests, "disable slow crypto self-tests"); static unsigned int fuzz_iterations = 100; module_param(fuzz_iterations, uint, 0644); @@ -1674,7 +1674,7 @@ static int test_hash_vec(const struct hash_testvec *vec, unsigned int vec_num, return err; } - if (!noextratests) { + if (!noslowtests) { struct rnd_state rng; struct testvec_config cfg; char cfgname[TESTVEC_CONFIG_NAMELEN]; @@ -1764,7 +1764,7 @@ static int test_hash_vs_generic_impl(const char *generic_driver, char cfgname[TESTVEC_CONFIG_NAMELEN]; int err; - if (noextratests) + if (noslowtests) return 0; init_rnd_state(&rng); @@ -2232,7 +2232,7 @@ static int test_aead_vec(int enc, const struct aead_testvec *vec, return err; } - if (!noextratests) { + if (!noslowtests) { struct rnd_state rng; struct testvec_config cfg; char cfgname[TESTVEC_CONFIG_NAMELEN]; @@ -2252,7 +2252,7 @@ static int test_aead_vec(int enc, const struct aead_testvec *vec, return 0; } -struct aead_extra_tests_ctx { +struct aead_slow_tests_ctx { struct rnd_state rng; struct aead_request *req; struct crypto_aead *tfm; @@ -2427,8 +2427,7 @@ static void generate_random_aead_testvec(struct rnd_state *rng, vec->alen, vec->plen, authsize, vec->klen, vec->novrfy); } -static void try_to_generate_inauthentic_testvec( - struct aead_extra_tests_ctx *ctx) +static void try_to_generate_inauthentic_testvec(struct aead_slow_tests_ctx *ctx) { int i; @@ -2447,7 +2446,7 @@ static void try_to_generate_inauthentic_testvec( * Generate inauthentic test vectors (i.e. ciphertext, AAD pairs that aren't the * result of an encryption with the key) and verify that decryption fails. */ -static int test_aead_inauthentic_inputs(struct aead_extra_tests_ctx *ctx) +static int test_aead_inauthentic_inputs(struct aead_slow_tests_ctx *ctx) { unsigned int i; int err; @@ -2482,7 +2481,7 @@ static int test_aead_inauthentic_inputs(struct aead_extra_tests_ctx *ctx) * Test the AEAD algorithm against the corresponding generic implementation, if * one is available. */ -static int test_aead_vs_generic_impl(struct aead_extra_tests_ctx *ctx) +static int test_aead_vs_generic_impl(struct aead_slow_tests_ctx *ctx) { struct crypto_aead *tfm = ctx->tfm; const char *algname = crypto_aead_alg(tfm)->base.cra_name; @@ -2586,15 +2585,15 @@ out: return err; } -static int test_aead_extra(const struct alg_test_desc *test_desc, - struct aead_request *req, - struct cipher_test_sglists *tsgls) +static int test_aead_slow(const struct alg_test_desc *test_desc, + struct aead_request *req, + struct cipher_test_sglists *tsgls) { - struct aead_extra_tests_ctx *ctx; + struct aead_slow_tests_ctx *ctx; unsigned int i; int err; - if (noextratests) + if (noslowtests) return 0; ctx = kzalloc(sizeof(*ctx), GFP_KERNEL); @@ -2701,7 +2700,7 @@ static int alg_test_aead(const struct alg_test_desc *desc, const char *driver, if (err) goto out; - err = test_aead_extra(desc, req, tsgls); + err = test_aead_slow(desc, req, tsgls); out: free_cipher_test_sglists(tsgls); aead_request_free(req); @@ -2975,7 +2974,7 @@ static int test_skcipher_vec(int enc, const struct cipher_testvec *vec, return err; } - if (!noextratests) { + if (!noslowtests) { struct rnd_state rng; struct testvec_config cfg; char cfgname[TESTVEC_CONFIG_NAMELEN]; @@ -3077,7 +3076,7 @@ static int test_skcipher_vs_generic_impl(const char *generic_driver, char cfgname[TESTVEC_CONFIG_NAMELEN]; int err; - if (noextratests) + if (noslowtests) return 0; init_rnd_state(&rng); @@ -5709,7 +5708,7 @@ static void testmgr_onetime_init(void) alg_check_test_descs_order(); alg_check_testvec_configs(); - if (!noextratests) + if (!noslowtests) pr_warn("alg: full crypto tests enabled. This is intended for developer use only.\n"); } -- 2.51.0 From 6f9d0f53f50405fcf4abf7ff45cec7baed71151c Mon Sep 17 00:00:00 2001 From: Eric Biggers Date: Mon, 5 May 2025 13:33:44 -0700 Subject: [PATCH 07/16] crypto: Kconfig - make CRYPTO_MANAGER a hidden symbol There is no reason for people configuring the kernel to be asked about CRYPTO_MANAGER, so make it a hidden symbol. Signed-off-by: Eric Biggers Signed-off-by: Herbert Xu --- arch/arm/configs/milbeaut_m10v_defconfig | 1 - arch/arm/configs/pxa_defconfig | 1 - arch/mips/configs/decstation_64_defconfig | 1 - arch/mips/configs/decstation_defconfig | 1 - arch/mips/configs/decstation_r4k_defconfig | 1 - arch/mips/configs/ip28_defconfig | 1 - arch/parisc/configs/generic-64bit_defconfig | 1 - arch/sh/configs/migor_defconfig | 1 - crypto/Kconfig | 2 +- 9 files changed, 1 insertion(+), 9 deletions(-) diff --git a/arch/arm/configs/milbeaut_m10v_defconfig b/arch/arm/configs/milbeaut_m10v_defconfig index 9dd47e1d90bb..242e7d5a3f68 100644 --- a/arch/arm/configs/milbeaut_m10v_defconfig +++ b/arch/arm/configs/milbeaut_m10v_defconfig @@ -93,7 +93,6 @@ CONFIG_NLS_CODEPAGE_437=y CONFIG_NLS_ISO8859_1=y CONFIG_NLS_UTF8=y CONFIG_KEYS=y -CONFIG_CRYPTO_MANAGER=y CONFIG_CRYPTO_SELFTESTS=y # CONFIG_CRYPTO_ECHAINIV is not set CONFIG_CRYPTO_AES=y diff --git a/arch/arm/configs/pxa_defconfig b/arch/arm/configs/pxa_defconfig index edc5b4d17f77..ded4b9a5accf 100644 --- a/arch/arm/configs/pxa_defconfig +++ b/arch/arm/configs/pxa_defconfig @@ -636,7 +636,6 @@ CONFIG_NLS_ISO8859_15=m CONFIG_NLS_UTF8=m CONFIG_TIMER_STATS=y CONFIG_SECURITY=y -CONFIG_CRYPTO_MANAGER=y CONFIG_CRYPTO_CRYPTD=m CONFIG_CRYPTO_AUTHENC=m CONFIG_CRYPTO_BENCHMARK=m diff --git a/arch/mips/configs/decstation_64_defconfig b/arch/mips/configs/decstation_64_defconfig index 9655567614aa..85a4472cb058 100644 --- a/arch/mips/configs/decstation_64_defconfig +++ b/arch/mips/configs/decstation_64_defconfig @@ -168,7 +168,6 @@ CONFIG_NLS_ISO8859_14=m CONFIG_NLS_ISO8859_15=m CONFIG_NLS_UTF8=m CONFIG_CRYPTO_RSA=m -CONFIG_CRYPTO_MANAGER=y CONFIG_CRYPTO_CCM=m CONFIG_CRYPTO_GCM=m CONFIG_CRYPTO_CHACHA20POLY1305=m diff --git a/arch/mips/configs/decstation_defconfig b/arch/mips/configs/decstation_defconfig index 1539fe8eb34d..a3b2c8da2dde 100644 --- a/arch/mips/configs/decstation_defconfig +++ b/arch/mips/configs/decstation_defconfig @@ -163,7 +163,6 @@ CONFIG_NLS_ISO8859_14=m CONFIG_NLS_ISO8859_15=m CONFIG_NLS_UTF8=m CONFIG_CRYPTO_RSA=m -CONFIG_CRYPTO_MANAGER=y CONFIG_CRYPTO_CCM=m CONFIG_CRYPTO_GCM=m CONFIG_CRYPTO_CHACHA20POLY1305=m diff --git a/arch/mips/configs/decstation_r4k_defconfig b/arch/mips/configs/decstation_r4k_defconfig index 58c36720c94a..a476717b8a6a 100644 --- a/arch/mips/configs/decstation_r4k_defconfig +++ b/arch/mips/configs/decstation_r4k_defconfig @@ -163,7 +163,6 @@ CONFIG_NLS_ISO8859_14=m CONFIG_NLS_ISO8859_15=m CONFIG_NLS_UTF8=m CONFIG_CRYPTO_RSA=m -CONFIG_CRYPTO_MANAGER=y CONFIG_CRYPTO_CCM=m CONFIG_CRYPTO_GCM=m CONFIG_CRYPTO_CHACHA20POLY1305=m diff --git a/arch/mips/configs/ip28_defconfig b/arch/mips/configs/ip28_defconfig index e0040110a3ee..6db21e498faa 100644 --- a/arch/mips/configs/ip28_defconfig +++ b/arch/mips/configs/ip28_defconfig @@ -60,6 +60,5 @@ CONFIG_TMPFS_POSIX_ACL=y CONFIG_NFS_FS=y CONFIG_NFS_V3_ACL=y CONFIG_ROOT_NFS=y -CONFIG_CRYPTO_MANAGER=y # CONFIG_CRYPTO_HW is not set CONFIG_MAGIC_SYSRQ=y diff --git a/arch/parisc/configs/generic-64bit_defconfig b/arch/parisc/configs/generic-64bit_defconfig index ecc9ffcc11cd..d8cd7f858b2a 100644 --- a/arch/parisc/configs/generic-64bit_defconfig +++ b/arch/parisc/configs/generic-64bit_defconfig @@ -283,7 +283,6 @@ CONFIG_NLS_ASCII=m CONFIG_NLS_ISO8859_1=m CONFIG_NLS_ISO8859_2=m CONFIG_NLS_UTF8=m -CONFIG_CRYPTO_MANAGER=y CONFIG_CRYPTO_FCRYPT=m CONFIG_CRYPTO_ECB=m CONFIG_CRYPTO_PCBC=m diff --git a/arch/sh/configs/migor_defconfig b/arch/sh/configs/migor_defconfig index fc2010c241fb..31dbd8888aaa 100644 --- a/arch/sh/configs/migor_defconfig +++ b/arch/sh/configs/migor_defconfig @@ -87,6 +87,5 @@ CONFIG_TMPFS=y CONFIG_NFS_FS=y CONFIG_ROOT_NFS=y CONFIG_DEBUG_FS=y -CONFIG_CRYPTO_MANAGER=y # CONFIG_CRYPTO_ANSI_CPRNG is not set # CONFIG_CRYPTO_HW is not set diff --git a/crypto/Kconfig b/crypto/Kconfig index 277fc9335767..50f6e285b92a 100644 --- a/crypto/Kconfig +++ b/crypto/Kconfig @@ -148,7 +148,7 @@ config CRYPTO_HKDF select CRYPTO_HASH2 config CRYPTO_MANAGER - tristate "Cryptographic algorithm manager" + tristate select CRYPTO_MANAGER2 help Create default cryptographic template instantiations such as -- 2.51.0 From 57999ed153ed7e651afecbabe0e998e75cf2d798 Mon Sep 17 00:00:00 2001 From: Eric Biggers Date: Mon, 5 May 2025 13:33:45 -0700 Subject: [PATCH 08/16] crypto: testmgr - enable CRYPTO_MANAGER when CRYPTO_SELFTESTS crypto/testmgr.c is compiled only when CRYPTO_MANAGER is enabled. To make CRYPTO_SELFTESTS work as expected when CRYPTO_MANAGER doesn't get enabled for another reason, automatically set CRYPTO_MANAGER to the value of CRYPTO_ALGAPI when CRYPTO_SELFTESTS is enabled. Signed-off-by: Eric Biggers Signed-off-by: Herbert Xu --- crypto/Kconfig | 5 +++-- crypto/algapi.c | 3 +-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/crypto/Kconfig b/crypto/Kconfig index 50f6e285b92a..e9fee7818e27 100644 --- a/crypto/Kconfig +++ b/crypto/Kconfig @@ -149,10 +149,11 @@ config CRYPTO_HKDF config CRYPTO_MANAGER tristate + default CRYPTO_ALGAPI if CRYPTO_SELFTESTS select CRYPTO_MANAGER2 help - Create default cryptographic template instantiations such as - cbc(aes). + This provides the support for instantiating templates such as + cbc(aes), and the support for the crypto self-tests. config CRYPTO_MANAGER2 def_tristate CRYPTO_MANAGER || (CRYPTO_MANAGER!=n && CRYPTO_ALGAPI=y) diff --git a/crypto/algapi.c b/crypto/algapi.c index 9030a30c89e8..25b5519e3b71 100644 --- a/crypto/algapi.c +++ b/crypto/algapi.c @@ -274,8 +274,7 @@ static struct crypto_larval *crypto_alloc_test_larval(struct crypto_alg *alg) { struct crypto_larval *larval; - if (!IS_ENABLED(CONFIG_CRYPTO_MANAGER) || - !IS_ENABLED(CONFIG_CRYPTO_SELFTESTS) || + if (!IS_ENABLED(CONFIG_CRYPTO_SELFTESTS) || (alg->cra_flags & CRYPTO_ALG_INTERNAL)) return NULL; /* No self-test needed */ -- 2.51.0 From 33cd93435cea665b24ca3f9b3d6af42afb3ba7bc Mon Sep 17 00:00:00 2001 From: Dan Carpenter Date: Thu, 8 May 2025 09:26:25 +0300 Subject: [PATCH 09/16] crypto: qat/qat_6xxx - Fix NULL vs IS_ERR() check in adf_probe() The pcim_iomap_region() returns error pointers. It doesn't return NULL pointers. Update the check to match. Fixes: 17fd7514ae68 ("crypto: qat - add qat_6xxx driver") Signed-off-by: Dan Carpenter Acked-by: Giovanni Cabiddu Signed-off-by: Herbert Xu --- drivers/crypto/intel/qat/qat_6xxx/adf_drv.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/drivers/crypto/intel/qat/qat_6xxx/adf_drv.c b/drivers/crypto/intel/qat/qat_6xxx/adf_drv.c index 2531c337e0dd..132e26501621 100644 --- a/drivers/crypto/intel/qat/qat_6xxx/adf_drv.c +++ b/drivers/crypto/intel/qat/qat_6xxx/adf_drv.c @@ -156,8 +156,8 @@ static int adf_probe(struct pci_dev *pdev, const struct pci_device_id *ent) /* Map 64-bit PCIe BAR */ bar->virt_addr = pcim_iomap_region(pdev, bar_map[i], pci_name(pdev)); - if (!bar->virt_addr) { - ret = -ENOMEM; + if (IS_ERR(bar->virt_addr)) { + ret = PTR_ERR(bar->virt_addr); return dev_err_probe(dev, ret, "Failed to ioremap PCI region.\n"); } } -- 2.51.0 From 714ca27e9bf4608fcb1f627cd5599441f448771e Mon Sep 17 00:00:00 2001 From: Yury Norov Date: Thu, 8 May 2025 15:59:50 -0400 Subject: [PATCH 10/16] crypto: iaa - Optimize rebalance_wq_table() The function opencodes for_each_cpu() by using a plain for-loop. The loop calls cpumask_weight() inside the conditional section. Because cpumask_weight() is O(1), the overall complexity of the function is O(node * node_cpus^2). Also, cpumask_nth() internally calls hweight(), which, if not hardware accelerated, is slower than cpumask_next() in for_each_cpu(). If switched to the dedicated for_each_cpu(), the rebalance_wq_table() can drop calling cpumask_weight(), together with some housekeeping code. This makes the overall complexity O(node * node_cpus), or simply speaking O(nr_cpu_ids). While there, fix opencoded for_each_possible_cpu() too. Signed-off-by: Yury Norov Signed-off-by: Herbert Xu --- drivers/crypto/intel/iaa/iaa_crypto_main.c | 35 +++++++++------------- 1 file changed, 14 insertions(+), 21 deletions(-) diff --git a/drivers/crypto/intel/iaa/iaa_crypto_main.c b/drivers/crypto/intel/iaa/iaa_crypto_main.c index e9c762124f11..23f585219fb4 100644 --- a/drivers/crypto/intel/iaa/iaa_crypto_main.c +++ b/drivers/crypto/intel/iaa/iaa_crypto_main.c @@ -894,7 +894,7 @@ out: static void rebalance_wq_table(void) { const struct cpumask *node_cpus; - int node, cpu, iaa = -1; + int node_cpu, node, cpu, iaa = 0; if (nr_iaa == 0) return; @@ -905,36 +905,29 @@ static void rebalance_wq_table(void) clear_wq_table(); if (nr_iaa == 1) { - for (cpu = 0; cpu < nr_cpus; cpu++) { - if (WARN_ON(wq_table_add_wqs(0, cpu))) { - pr_debug("could not add any wqs for iaa 0 to cpu %d!\n", cpu); - return; - } + for_each_possible_cpu(cpu) { + if (WARN_ON(wq_table_add_wqs(0, cpu))) + goto err; } return; } for_each_node_with_cpus(node) { + cpu = 0; node_cpus = cpumask_of_node(node); - for (cpu = 0; cpu < cpumask_weight(node_cpus); cpu++) { - int node_cpu = cpumask_nth(cpu, node_cpus); - - if (WARN_ON(node_cpu >= nr_cpu_ids)) { - pr_debug("node_cpu %d doesn't exist!\n", node_cpu); - return; - } - - if ((cpu % cpus_per_iaa) == 0) - iaa++; - - if (WARN_ON(wq_table_add_wqs(iaa, node_cpu))) { - pr_debug("could not add any wqs for iaa %d to cpu %d!\n", iaa, cpu); - return; - } + for_each_cpu(node_cpu, node_cpus) { + iaa = cpu / cpus_per_iaa; + if (WARN_ON(wq_table_add_wqs(iaa, node_cpu))) + goto err; + cpu++; } } + + return; +err: + pr_debug("could not add any wqs for iaa %d to cpu %d!\n", iaa, cpu); } static inline int check_completion(struct device *dev, -- 2.51.0 From c66d7ebbe2fa14e41913adb421090a7426f59786 Mon Sep 17 00:00:00 2001 From: Herbert Xu Date: Sat, 10 May 2025 17:13:47 +0800 Subject: [PATCH 11/16] crypto: powerpc/poly1305 - Add SIMD fallback Add a SIMD fallback path for poly1305-p10 by converting the 2^64 based hash state into a 2^44 base. In order to ensure that the generic fallback is actually 2^44, add ARCH_SUPPORTS_INT128 to powerpc and make poly1305-p10 depend on it. Fixes: ba8f8624fde2 ("crypto: poly1305-p10 - Glue code for optmized Poly1305 implementation for ppc64le") Signed-off-by: Herbert Xu --- arch/powerpc/Kconfig | 1 + arch/powerpc/lib/crypto/Kconfig | 2 +- arch/powerpc/lib/crypto/poly1305-p10-glue.c | 41 +++++++++++++++++++-- 3 files changed, 40 insertions(+), 4 deletions(-) diff --git a/arch/powerpc/Kconfig b/arch/powerpc/Kconfig index 6722625a406a..651e0c32957a 100644 --- a/arch/powerpc/Kconfig +++ b/arch/powerpc/Kconfig @@ -173,6 +173,7 @@ config PPC select ARCH_STACKWALK select ARCH_SUPPORTS_ATOMIC_RMW select ARCH_SUPPORTS_DEBUG_PAGEALLOC if PPC_BOOK3S || PPC_8xx + select ARCH_SUPPORTS_INT128 if PPC64 && CC_HAS_INT128 select ARCH_USE_BUILTIN_BSWAP select ARCH_USE_CMPXCHG_LOCKREF if PPC64 select ARCH_USE_MEMTEST diff --git a/arch/powerpc/lib/crypto/Kconfig b/arch/powerpc/lib/crypto/Kconfig index ffa541ad6d5d..6761fdb6193c 100644 --- a/arch/powerpc/lib/crypto/Kconfig +++ b/arch/powerpc/lib/crypto/Kconfig @@ -9,7 +9,7 @@ config CRYPTO_CHACHA20_P10 config CRYPTO_POLY1305_P10 tristate - depends on PPC64 && CPU_LITTLE_ENDIAN && VSX + depends on PPC64 && CPU_LITTLE_ENDIAN && VSX && ARCH_SUPPORTS_INT128 default CRYPTO_LIB_POLY1305 select CRYPTO_ARCH_HAVE_LIB_POLY1305 select CRYPTO_LIB_POLY1305_GENERIC diff --git a/arch/powerpc/lib/crypto/poly1305-p10-glue.c b/arch/powerpc/lib/crypto/poly1305-p10-glue.c index 3f1664a724b6..280c10c48c53 100644 --- a/arch/powerpc/lib/crypto/poly1305-p10-glue.c +++ b/arch/powerpc/lib/crypto/poly1305-p10-glue.c @@ -6,6 +6,7 @@ */ #include #include +#include #include #include #include @@ -18,6 +19,11 @@ asmlinkage void poly1305_emit_64(const struct poly1305_state *state, const u32 n static __ro_after_init DEFINE_STATIC_KEY_FALSE(have_p10); +static inline bool is_state_base64(struct poly1305_block_state *state) +{ + return state->core_r.precomputed_s.r64[2]; +} + static void vsx_begin(void) { preempt_disable(); @@ -30,12 +36,35 @@ static void vsx_end(void) preempt_enable(); } +static void convert_to_base2_44(struct poly1305_block_state *state) +{ + u8 raw_key[POLY1305_BLOCK_SIZE]; + u64 h0, h1, h2; + + if (!is_state_base64(state)) + return; + + state->core_r.precomputed_s.r64[2] = 0; + put_unaligned_le64(state->core_r.key.r64[0], raw_key + 0); + put_unaligned_le64(state->core_r.key.r64[1], raw_key + 8); + poly1305_core_setkey(&state->core_r, raw_key); + + h0 = state->h.h64[0]; + h1 = state->h.h64[1]; + h2 = state->h.h64[2]; + state->h.h64[0] = h0 & 0xfffffffffffULL; + state->h.h64[1] = h0 >> 44 | (h1 & 0xffffffULL) << 20; + state->h.h64[2] = h1 >> 24 | h2 << 40; +} + void poly1305_block_init_arch(struct poly1305_block_state *dctx, const u8 raw_key[POLY1305_BLOCK_SIZE]) { - if (!static_key_enabled(&have_p10)) + dctx->core_r.precomputed_s.r64[2] = 0; + if (!static_key_enabled(&have_p10) || !crypto_simd_usable()) return poly1305_block_init_generic(dctx, raw_key); + dctx->core_r.precomputed_s.r64[2] = 1; dctx->h = (struct poly1305_state){}; dctx->core_r.key.r64[0] = get_unaligned_le64(raw_key + 0); dctx->core_r.key.r64[1] = get_unaligned_le64(raw_key + 8); @@ -45,8 +74,11 @@ EXPORT_SYMBOL_GPL(poly1305_block_init_arch); void poly1305_blocks_arch(struct poly1305_block_state *state, const u8 *src, unsigned int len, u32 padbit) { - if (!static_key_enabled(&have_p10)) + if (!static_key_enabled(&have_p10) || !is_state_base64(state) || + !crypto_simd_usable()) { + convert_to_base2_44(state); return poly1305_blocks_generic(state, src, len, padbit); + } vsx_begin(); if (len >= POLY1305_BLOCK_SIZE * 4) { poly1305_p10le_4blocks(state, src, len); @@ -66,7 +98,10 @@ void poly1305_emit_arch(const struct poly1305_state *state, u8 digest[POLY1305_DIGEST_SIZE], const u32 nonce[4]) { - if (!static_key_enabled(&have_p10)) + struct poly1305_block_state *dctx = + container_of(state, struct poly1305_block_state, h); + + if (!static_key_enabled(&have_p10) || !is_state_base64(dctx)) return poly1305_emit_generic(state, digest, nonce); poly1305_emit_64(state, nonce, digest); } -- 2.51.0 From 8a4e047c6cc07676f637608a9dd675349b5de0a7 Mon Sep 17 00:00:00 2001 From: Herbert Xu Date: Sat, 10 May 2025 18:41:31 +0800 Subject: [PATCH 12/16] crypto: marvell/cesa - Handle zero-length skcipher requests Do not access random memory for zero-length skcipher requests. Just return 0. Fixes: f63601fd616a ("crypto: marvell/cesa - add a new driver for Marvell's CESA") Signed-off-by: Herbert Xu --- drivers/crypto/marvell/cesa/cipher.c | 3 +++ 1 file changed, 3 insertions(+) diff --git a/drivers/crypto/marvell/cesa/cipher.c b/drivers/crypto/marvell/cesa/cipher.c index cf62db50f958..48c5c8ea8c43 100644 --- a/drivers/crypto/marvell/cesa/cipher.c +++ b/drivers/crypto/marvell/cesa/cipher.c @@ -459,6 +459,9 @@ static int mv_cesa_skcipher_queue_req(struct skcipher_request *req, struct mv_cesa_skcipher_req *creq = skcipher_request_ctx(req); struct mv_cesa_engine *engine; + if (!req->cryptlen) + return 0; + ret = mv_cesa_skcipher_req_init(req, tmpl); if (ret) return ret; -- 2.51.0 From 1bafd82d9a40cf09c6c40f1c09cc35b7050b1a9f Mon Sep 17 00:00:00 2001 From: Herbert Xu Date: Sat, 10 May 2025 18:43:33 +0800 Subject: [PATCH 13/16] crypto: marvell/cesa - Avoid empty transfer descriptor The user may set req->src even if req->nbytes == 0. If there is no data to hash from req->src, do not generate an empty TDMA descriptor. Fixes: db509a45339f ("crypto: marvell/cesa - add TDMA support") Signed-off-by: Herbert Xu --- drivers/crypto/marvell/cesa/hash.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/crypto/marvell/cesa/hash.c b/drivers/crypto/marvell/cesa/hash.c index f150861ceaf6..6815eddc9068 100644 --- a/drivers/crypto/marvell/cesa/hash.c +++ b/drivers/crypto/marvell/cesa/hash.c @@ -663,7 +663,7 @@ static int mv_cesa_ahash_dma_req_init(struct ahash_request *req) if (ret) goto err_free_tdma; - if (iter.src.sg) { + if (iter.base.len > iter.src.op_offset) { /* * Add all the new data, inserting an operation block and * launch command between each full SRAM block-worth of -- 2.51.0 From 63935e2ee1f2a371e511f853737b9efebc238bc7 Mon Sep 17 00:00:00 2001 From: John Allen Date: Mon, 12 May 2025 18:17:05 +0000 Subject: [PATCH 14/16] crypto: ccp - Add support for PCI device 0x17D8 Add a new CCP/PSP PCI device ID. Signed-off-by: John Allen Signed-off-by: Herbert Xu --- drivers/crypto/ccp/sp-pci.c | 1 + 1 file changed, 1 insertion(+) diff --git a/drivers/crypto/ccp/sp-pci.c b/drivers/crypto/ccp/sp-pci.c index 2ebc878da160..54ab3279f521 100644 --- a/drivers/crypto/ccp/sp-pci.c +++ b/drivers/crypto/ccp/sp-pci.c @@ -535,6 +535,7 @@ static const struct pci_device_id sp_pci_table[] = { { PCI_VDEVICE(AMD, 0x1134), (kernel_ulong_t)&dev_vdata[7] }, { PCI_VDEVICE(AMD, 0x17E0), (kernel_ulong_t)&dev_vdata[7] }, { PCI_VDEVICE(AMD, 0x156E), (kernel_ulong_t)&dev_vdata[8] }, + { PCI_VDEVICE(AMD, 0x17D8), (kernel_ulong_t)&dev_vdata[8] }, /* Last entry must be zero */ { 0, } }; -- 2.51.0 From 7f3401d0a506dac688948a62ef8ef7b18cc791e1 Mon Sep 17 00:00:00 2001 From: Suman Kumar Chakraborty Date: Tue, 13 May 2025 11:25:26 +0100 Subject: [PATCH 15/16] crypto: qat - enable RAS support for GEN6 devices Enable the reporting and handling of errors for QAT GEN6 devices. Errors are categorized as correctable, non-fatal, or fatal. Error handling involves reading the error source registers (ERRSOU0 to ERRSOU3) to determine the source of the error and then decoding the actual source reading specific registers. The action taken depends on the error type: - Correctable and Non-Fatal errors. These error are logged, cleared and the corresponding counter is incremented. - Fatal errors. These errors are logged, cleared and a Function Level Reset (FLR) is scheduled. This reports and handles the following errors: - Accelerator engine (AE) correctable errors - Accelerator engine (AE) uncorrectable errors - Chassis push-pull (CPP) errors - Host interface (HI) parity errors - Internal memory parity errors - Receive interface (RI) errors - Transmit interface (TI) errors - Interface for system-on-chip (SoC) fabric (IOSF) primary command parity errors - Shared RAM and slice module (SSM) errors Signed-off-by: Suman Kumar Chakraborty Reviewed-by: Giovanni Cabiddu Signed-off-by: Herbert Xu --- .../intel/qat/qat_6xxx/adf_6xxx_hw_data.c | 2 + drivers/crypto/intel/qat/qat_common/Makefile | 1 + .../intel/qat/qat_common/adf_gen6_ras.c | 818 ++++++++++++++++++ .../intel/qat/qat_common/adf_gen6_ras.h | 504 +++++++++++ 4 files changed, 1325 insertions(+) create mode 100644 drivers/crypto/intel/qat/qat_common/adf_gen6_ras.c create mode 100644 drivers/crypto/intel/qat/qat_common/adf_gen6_ras.h diff --git a/drivers/crypto/intel/qat/qat_6xxx/adf_6xxx_hw_data.c b/drivers/crypto/intel/qat/qat_6xxx/adf_6xxx_hw_data.c index 73d479383b1f..359a6447ccb8 100644 --- a/drivers/crypto/intel/qat/qat_6xxx/adf_6xxx_hw_data.c +++ b/drivers/crypto/intel/qat/qat_6xxx/adf_6xxx_hw_data.c @@ -16,6 +16,7 @@ #include #include #include +#include #include #include #include "adf_6xxx_hw_data.h" @@ -834,6 +835,7 @@ void adf_init_hw_data_6xxx(struct adf_hw_device_data *hw_data) adf_gen6_init_hw_csr_ops(&hw_data->csr_ops); adf_gen6_init_pf_pfvf_ops(&hw_data->pfvf_ops); adf_gen6_init_dc_ops(&hw_data->dc_ops); + adf_gen6_init_ras_ops(&hw_data->ras_ops); } void adf_clean_hw_data_6xxx(struct adf_hw_device_data *hw_data) diff --git a/drivers/crypto/intel/qat/qat_common/Makefile b/drivers/crypto/intel/qat/qat_common/Makefile index 958f9a8ac6b3..66bb295ace28 100644 --- a/drivers/crypto/intel/qat/qat_common/Makefile +++ b/drivers/crypto/intel/qat/qat_common/Makefile @@ -19,6 +19,7 @@ intel_qat-y := adf_accel_engine.o \ adf_gen4_pm.o \ adf_gen4_ras.o \ adf_gen4_vf_mig.o \ + adf_gen6_ras.o \ adf_gen6_shared.o \ adf_hw_arbiter.o \ adf_init.o \ diff --git a/drivers/crypto/intel/qat/qat_common/adf_gen6_ras.c b/drivers/crypto/intel/qat/qat_common/adf_gen6_ras.c new file mode 100644 index 000000000000..967253082a98 --- /dev/null +++ b/drivers/crypto/intel/qat/qat_common/adf_gen6_ras.c @@ -0,0 +1,818 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* Copyright(c) 2025 Intel Corporation */ +#include +#include + +#include "adf_common_drv.h" +#include "adf_gen6_ras.h" +#include "adf_sysfs_ras_counters.h" + +static void enable_errsou_reporting(void __iomem *csr) +{ + /* Enable correctable error reporting in ERRSOU0 */ + ADF_CSR_WR(csr, ADF_GEN6_ERRMSK0, 0); + + /* Enable uncorrectable error reporting in ERRSOU1 */ + ADF_CSR_WR(csr, ADF_GEN6_ERRMSK1, 0); + + /* + * Enable uncorrectable error reporting in ERRSOU2 + * but disable PM interrupt by default + */ + ADF_CSR_WR(csr, ADF_GEN6_ERRMSK2, ADF_GEN6_ERRSOU2_PM_INT_BIT); + + /* Enable uncorrectable error reporting in ERRSOU3 */ + ADF_CSR_WR(csr, ADF_GEN6_ERRMSK3, 0); +} + +static void enable_ae_error_reporting(struct adf_accel_dev *accel_dev, void __iomem *csr) +{ + u32 ae_mask = GET_HW_DATA(accel_dev)->ae_mask; + + /* Enable acceleration engine correctable error reporting */ + ADF_CSR_WR(csr, ADF_GEN6_HIAECORERRLOGENABLE_CPP0, ae_mask); + + /* Enable acceleration engine uncorrectable error reporting */ + ADF_CSR_WR(csr, ADF_GEN6_HIAEUNCERRLOGENABLE_CPP0, ae_mask); +} + +static void enable_cpp_error_reporting(struct adf_accel_dev *accel_dev, void __iomem *csr) +{ + /* Enable HI CPP agents command parity error reporting */ + ADF_CSR_WR(csr, ADF_GEN6_HICPPAGENTCMDPARERRLOGENABLE, + ADF_6XXX_HICPPAGENTCMDPARERRLOG_MASK); + + ADF_CSR_WR(csr, ADF_GEN6_CPP_CFC_ERR_CTRL, ADF_GEN6_CPP_CFC_ERR_CTRL_MASK); +} + +static void enable_ti_ri_error_reporting(void __iomem *csr) +{ + u32 reg, mask; + + /* Enable RI memory error reporting */ + mask = ADF_GEN6_RIMEM_PARERR_FATAL_MASK | ADF_GEN6_RIMEM_PARERR_CERR_MASK; + ADF_CSR_WR(csr, ADF_GEN6_RI_MEM_PAR_ERR_EN0, mask); + + /* Enable IOSF primary command parity error reporting */ + ADF_CSR_WR(csr, ADF_GEN6_RIMISCCTL, ADF_GEN6_RIMISCSTS_BIT); + + /* Enable TI internal memory parity error reporting */ + reg = ADF_CSR_RD(csr, ADF_GEN6_TI_CI_PAR_ERR_MASK); + reg &= ~ADF_GEN6_TI_CI_PAR_STS_MASK; + ADF_CSR_WR(csr, ADF_GEN6_TI_CI_PAR_ERR_MASK, reg); + + reg = ADF_CSR_RD(csr, ADF_GEN6_TI_PULL0FUB_PAR_ERR_MASK); + reg &= ~ADF_GEN6_TI_PULL0FUB_PAR_STS_MASK; + ADF_CSR_WR(csr, ADF_GEN6_TI_PULL0FUB_PAR_ERR_MASK, reg); + + reg = ADF_CSR_RD(csr, ADF_GEN6_TI_PUSHFUB_PAR_ERR_MASK); + reg &= ~ADF_GEN6_TI_PUSHFUB_PAR_STS_MASK; + ADF_CSR_WR(csr, ADF_GEN6_TI_PUSHFUB_PAR_ERR_MASK, reg); + + reg = ADF_CSR_RD(csr, ADF_GEN6_TI_CD_PAR_ERR_MASK); + reg &= ~ADF_GEN6_TI_CD_PAR_STS_MASK; + ADF_CSR_WR(csr, ADF_GEN6_TI_CD_PAR_ERR_MASK, reg); + + reg = ADF_CSR_RD(csr, ADF_GEN6_TI_TRNSB_PAR_ERR_MASK); + reg &= ~ADF_GEN6_TI_TRNSB_PAR_STS_MASK; + ADF_CSR_WR(csr, ADF_GEN6_TI_TRNSB_PAR_ERR_MASK, reg); + + /* Enable error handling in RI, TI CPP interface control registers */ + ADF_CSR_WR(csr, ADF_GEN6_RICPPINTCTL, ADF_GEN6_RICPPINTCTL_MASK); + ADF_CSR_WR(csr, ADF_GEN6_TICPPINTCTL, ADF_GEN6_TICPPINTCTL_MASK); + + /* + * Enable error detection and reporting in TIMISCSTS + * with bits 1, 2 and 30 value preserved + */ + reg = ADF_CSR_RD(csr, ADF_GEN6_TIMISCCTL); + reg &= ADF_GEN6_TIMSCCTL_RELAY_MASK; + reg |= ADF_GEN6_TIMISCCTL_BIT; + ADF_CSR_WR(csr, ADF_GEN6_TIMISCCTL, reg); +} + +static void enable_ssm_error_reporting(struct adf_accel_dev *accel_dev, + void __iomem *csr) +{ + /* Enable SSM interrupts */ + ADF_CSR_WR(csr, ADF_GEN6_INTMASKSSM, 0); +} + +static void adf_gen6_enable_ras(struct adf_accel_dev *accel_dev) +{ + void __iomem *csr = adf_get_pmisc_base(accel_dev); + + enable_errsou_reporting(csr); + enable_ae_error_reporting(accel_dev, csr); + enable_cpp_error_reporting(accel_dev, csr); + enable_ti_ri_error_reporting(csr); + enable_ssm_error_reporting(accel_dev, csr); +} + +static void disable_errsou_reporting(void __iomem *csr) +{ + u32 val; + + /* Disable correctable error reporting in ERRSOU0 */ + ADF_CSR_WR(csr, ADF_GEN6_ERRMSK0, ADF_GEN6_ERRSOU0_MASK); + + /* Disable uncorrectable error reporting in ERRSOU1 */ + ADF_CSR_WR(csr, ADF_GEN6_ERRMSK1, ADF_GEN6_ERRMSK1_MASK); + + /* Disable uncorrectable error reporting in ERRSOU2 */ + val = ADF_CSR_RD(csr, ADF_GEN6_ERRMSK2); + val |= ADF_GEN6_ERRSOU2_DIS_MASK; + ADF_CSR_WR(csr, ADF_GEN6_ERRMSK2, val); + + /* Disable uncorrectable error reporting in ERRSOU3 */ + ADF_CSR_WR(csr, ADF_GEN6_ERRMSK3, ADF_GEN6_ERRSOU3_DIS_MASK); +} + +static void disable_ae_error_reporting(void __iomem *csr) +{ + /* Disable acceleration engine correctable error reporting */ + ADF_CSR_WR(csr, ADF_GEN6_HIAECORERRLOGENABLE_CPP0, 0); + + /* Disable acceleration engine uncorrectable error reporting */ + ADF_CSR_WR(csr, ADF_GEN6_HIAEUNCERRLOGENABLE_CPP0, 0); +} + +static void disable_cpp_error_reporting(void __iomem *csr) +{ + /* Disable HI CPP agents command parity error reporting */ + ADF_CSR_WR(csr, ADF_GEN6_HICPPAGENTCMDPARERRLOGENABLE, 0); + + ADF_CSR_WR(csr, ADF_GEN6_CPP_CFC_ERR_CTRL, ADF_GEN6_CPP_CFC_ERR_CTRL_DIS_MASK); +} + +static void disable_ti_ri_error_reporting(void __iomem *csr) +{ + u32 reg; + + /* Disable RI memory error reporting */ + ADF_CSR_WR(csr, ADF_GEN6_RI_MEM_PAR_ERR_EN0, 0); + + /* Disable IOSF primary command parity error reporting */ + reg = ADF_CSR_RD(csr, ADF_GEN6_RIMISCCTL); + reg &= ~ADF_GEN6_RIMISCSTS_BIT; + ADF_CSR_WR(csr, ADF_GEN6_RIMISCCTL, reg); + + /* Disable TI internal memory parity error reporting */ + ADF_CSR_WR(csr, ADF_GEN6_TI_CI_PAR_ERR_MASK, ADF_GEN6_TI_CI_PAR_STS_MASK); + ADF_CSR_WR(csr, ADF_GEN6_TI_PULL0FUB_PAR_ERR_MASK, ADF_GEN6_TI_PULL0FUB_PAR_STS_MASK); + ADF_CSR_WR(csr, ADF_GEN6_TI_PUSHFUB_PAR_ERR_MASK, ADF_GEN6_TI_PUSHFUB_PAR_STS_MASK); + ADF_CSR_WR(csr, ADF_GEN6_TI_CD_PAR_ERR_MASK, ADF_GEN6_TI_CD_PAR_STS_MASK); + ADF_CSR_WR(csr, ADF_GEN6_TI_TRNSB_PAR_ERR_MASK, ADF_GEN6_TI_TRNSB_PAR_STS_MASK); + + /* Disable error handling in RI, TI CPP interface control registers */ + reg = ADF_CSR_RD(csr, ADF_GEN6_RICPPINTCTL); + reg &= ~ADF_GEN6_RICPPINTCTL_MASK; + ADF_CSR_WR(csr, ADF_GEN6_RICPPINTCTL, reg); + + reg = ADF_CSR_RD(csr, ADF_GEN6_TICPPINTCTL); + reg &= ~ADF_GEN6_TICPPINTCTL_MASK; + ADF_CSR_WR(csr, ADF_GEN6_TICPPINTCTL, reg); + + /* + * Disable error detection and reporting in TIMISCSTS + * with bits 1, 2 and 30 value preserved + */ + reg = ADF_CSR_RD(csr, ADF_GEN6_TIMISCCTL); + reg &= ADF_GEN6_TIMSCCTL_RELAY_MASK; + ADF_CSR_WR(csr, ADF_GEN6_TIMISCCTL, reg); +} + +static void disable_ssm_error_reporting(void __iomem *csr) +{ + /* Disable SSM interrupts */ + ADF_CSR_WR(csr, ADF_GEN6_INTMASKSSM, ADF_GEN6_INTMASKSSM_MASK); +} + +static void adf_gen6_disable_ras(struct adf_accel_dev *accel_dev) +{ + void __iomem *csr = adf_get_pmisc_base(accel_dev); + + disable_errsou_reporting(csr); + disable_ae_error_reporting(csr); + disable_cpp_error_reporting(csr); + disable_ti_ri_error_reporting(csr); + disable_ssm_error_reporting(csr); +} + +static void adf_gen6_process_errsou0(struct adf_accel_dev *accel_dev, void __iomem *csr) +{ + u32 ae, errsou; + + ae = ADF_CSR_RD(csr, ADF_GEN6_HIAECORERRLOG_CPP0); + ae &= GET_HW_DATA(accel_dev)->ae_mask; + + dev_warn(&GET_DEV(accel_dev), "Correctable error detected: %#x\n", ae); + + ADF_RAS_ERR_CTR_INC(accel_dev->ras_errors, ADF_RAS_CORR); + + /* Clear interrupt from ERRSOU0 */ + ADF_CSR_WR(csr, ADF_GEN6_HIAECORERRLOG_CPP0, ae); + + errsou = ADF_CSR_RD(csr, ADF_GEN6_ERRSOU0); + if (errsou & ADF_GEN6_ERRSOU0_MASK) + dev_warn(&GET_DEV(accel_dev), "errsou0 still set: %#x\n", errsou); +} + +static void adf_handle_cpp_ae_unc(struct adf_accel_dev *accel_dev, void __iomem *csr, + u32 errsou) +{ + u32 ae; + + if (!(errsou & ADF_GEN6_ERRSOU1_CPP0_MEUNC_BIT)) + return; + + ae = ADF_CSR_RD(csr, ADF_GEN6_HIAEUNCERRLOG_CPP0); + ae &= GET_HW_DATA(accel_dev)->ae_mask; + if (ae) { + dev_err(&GET_DEV(accel_dev), "Uncorrectable error detected: %#x\n", ae); + ADF_RAS_ERR_CTR_INC(accel_dev->ras_errors, ADF_RAS_UNCORR); + ADF_CSR_WR(csr, ADF_GEN6_HIAEUNCERRLOG_CPP0, ae); + } +} + +static void adf_handle_cpp_cmd_par_err(struct adf_accel_dev *accel_dev, void __iomem *csr, + u32 errsou) +{ + u32 cmd_par_err; + + if (!(errsou & ADF_GEN6_ERRSOU1_CPP_CMDPARERR_BIT)) + return; + + cmd_par_err = ADF_CSR_RD(csr, ADF_GEN6_HICPPAGENTCMDPARERRLOG); + cmd_par_err &= ADF_6XXX_HICPPAGENTCMDPARERRLOG_MASK; + if (cmd_par_err) { + dev_err(&GET_DEV(accel_dev), "HI CPP agent command parity error: %#x\n", + cmd_par_err); + ADF_RAS_ERR_CTR_INC(accel_dev->ras_errors, ADF_RAS_FATAL); + ADF_CSR_WR(csr, ADF_GEN6_HICPPAGENTCMDPARERRLOG, cmd_par_err); + } +} + +static void adf_handle_ri_mem_par_err(struct adf_accel_dev *accel_dev, void __iomem *csr, + u32 errsou) +{ + u32 rimem_parerr_sts; + + if (!(errsou & ADF_GEN6_ERRSOU1_RIMEM_PARERR_STS_BIT)) + return; + + rimem_parerr_sts = ADF_CSR_RD(csr, ADF_GEN6_RIMEM_PARERR_STS); + rimem_parerr_sts &= ADF_GEN6_RIMEM_PARERR_CERR_MASK | + ADF_GEN6_RIMEM_PARERR_FATAL_MASK; + if (rimem_parerr_sts & ADF_GEN6_RIMEM_PARERR_CERR_MASK) { + dev_err(&GET_DEV(accel_dev), "RI memory parity correctable error: %#x\n", + rimem_parerr_sts); + ADF_RAS_ERR_CTR_INC(accel_dev->ras_errors, ADF_RAS_CORR); + } + + if (rimem_parerr_sts & ADF_GEN6_RIMEM_PARERR_FATAL_MASK) { + dev_err(&GET_DEV(accel_dev), "RI memory parity fatal error: %#x\n", + rimem_parerr_sts); + ADF_RAS_ERR_CTR_INC(accel_dev->ras_errors, ADF_RAS_FATAL); + } + + ADF_CSR_WR(csr, ADF_GEN6_RIMEM_PARERR_STS, rimem_parerr_sts); +} + +static void adf_handle_ti_ci_par_sts(struct adf_accel_dev *accel_dev, void __iomem *csr) +{ + u32 ti_ci_par_sts; + + ti_ci_par_sts = ADF_CSR_RD(csr, ADF_GEN6_TI_CI_PAR_STS); + ti_ci_par_sts &= ADF_GEN6_TI_CI_PAR_STS_MASK; + if (ti_ci_par_sts) { + dev_err(&GET_DEV(accel_dev), "TI memory parity error: %#x\n", ti_ci_par_sts); + ADF_CSR_WR(csr, ADF_GEN6_TI_CI_PAR_STS, ti_ci_par_sts); + ADF_RAS_ERR_CTR_INC(accel_dev->ras_errors, ADF_RAS_UNCORR); + } +} + +static void adf_handle_ti_pullfub_par_sts(struct adf_accel_dev *accel_dev, void __iomem *csr) +{ + u32 ti_pullfub_par_sts; + + ti_pullfub_par_sts = ADF_CSR_RD(csr, ADF_GEN6_TI_PULL0FUB_PAR_STS); + ti_pullfub_par_sts &= ADF_GEN6_TI_PULL0FUB_PAR_STS_MASK; + if (ti_pullfub_par_sts) { + dev_err(&GET_DEV(accel_dev), "TI pull parity error: %#x\n", ti_pullfub_par_sts); + ADF_CSR_WR(csr, ADF_GEN6_TI_PULL0FUB_PAR_STS, ti_pullfub_par_sts); + ADF_RAS_ERR_CTR_INC(accel_dev->ras_errors, ADF_RAS_UNCORR); + } +} + +static void adf_handle_ti_pushfub_par_sts(struct adf_accel_dev *accel_dev, void __iomem *csr) +{ + u32 ti_pushfub_par_sts; + + ti_pushfub_par_sts = ADF_CSR_RD(csr, ADF_GEN6_TI_PUSHFUB_PAR_STS); + ti_pushfub_par_sts &= ADF_GEN6_TI_PUSHFUB_PAR_STS_MASK; + if (ti_pushfub_par_sts) { + dev_err(&GET_DEV(accel_dev), "TI push parity error: %#x\n", ti_pushfub_par_sts); + ADF_RAS_ERR_CTR_INC(accel_dev->ras_errors, ADF_RAS_UNCORR); + ADF_CSR_WR(csr, ADF_GEN6_TI_PUSHFUB_PAR_STS, ti_pushfub_par_sts); + } +} + +static void adf_handle_ti_cd_par_sts(struct adf_accel_dev *accel_dev, void __iomem *csr) +{ + u32 ti_cd_par_sts; + + ti_cd_par_sts = ADF_CSR_RD(csr, ADF_GEN6_TI_CD_PAR_STS); + ti_cd_par_sts &= ADF_GEN6_TI_CD_PAR_STS_MASK; + if (ti_cd_par_sts) { + dev_err(&GET_DEV(accel_dev), "TI CD parity error: %#x\n", ti_cd_par_sts); + ADF_RAS_ERR_CTR_INC(accel_dev->ras_errors, ADF_RAS_UNCORR); + ADF_CSR_WR(csr, ADF_GEN6_TI_CD_PAR_STS, ti_cd_par_sts); + } +} + +static void adf_handle_ti_trnsb_par_sts(struct adf_accel_dev *accel_dev, void __iomem *csr) +{ + u32 ti_trnsb_par_sts; + + ti_trnsb_par_sts = ADF_CSR_RD(csr, ADF_GEN6_TI_TRNSB_PAR_STS); + ti_trnsb_par_sts &= ADF_GEN6_TI_TRNSB_PAR_STS_MASK; + if (ti_trnsb_par_sts) { + dev_err(&GET_DEV(accel_dev), "TI TRNSB parity error: %#x\n", ti_trnsb_par_sts); + ADF_RAS_ERR_CTR_INC(accel_dev->ras_errors, ADF_RAS_UNCORR); + ADF_CSR_WR(csr, ADF_GEN6_TI_TRNSB_PAR_STS, ti_trnsb_par_sts); + } +} + +static void adf_handle_iosfp_cmd_parerr(struct adf_accel_dev *accel_dev, void __iomem *csr) +{ + u32 rimiscsts; + + rimiscsts = ADF_CSR_RD(csr, ADF_GEN6_RIMISCSTS); + rimiscsts &= ADF_GEN6_RIMISCSTS_BIT; + if (rimiscsts) { + dev_err(&GET_DEV(accel_dev), "Command parity error detected on IOSFP: %#x\n", + rimiscsts); + ADF_RAS_ERR_CTR_INC(accel_dev->ras_errors, ADF_RAS_FATAL); + ADF_CSR_WR(csr, ADF_GEN6_RIMISCSTS, rimiscsts); + } +} + +static void adf_handle_ti_err(struct adf_accel_dev *accel_dev, void __iomem *csr, + u32 errsou) +{ + if (!(errsou & ADF_GEN6_ERRSOU1_TIMEM_PARERR_STS_BIT)) + return; + + adf_handle_ti_ci_par_sts(accel_dev, csr); + adf_handle_ti_pullfub_par_sts(accel_dev, csr); + adf_handle_ti_pushfub_par_sts(accel_dev, csr); + adf_handle_ti_cd_par_sts(accel_dev, csr); + adf_handle_ti_trnsb_par_sts(accel_dev, csr); + adf_handle_iosfp_cmd_parerr(accel_dev, csr); +} + +static void adf_handle_sfi_cmd_parerr(struct adf_accel_dev *accel_dev, void __iomem *csr, + u32 errsou) +{ + if (!(errsou & ADF_GEN6_ERRSOU1_SFICMD_PARERR_BIT)) + return; + + dev_err(&GET_DEV(accel_dev), + "Command parity error detected on streaming fabric interface\n"); + + ADF_RAS_ERR_CTR_INC(accel_dev->ras_errors, ADF_RAS_FATAL); +} + +static void adf_gen6_process_errsou1(struct adf_accel_dev *accel_dev, void __iomem *csr, + u32 errsou) +{ + adf_handle_cpp_ae_unc(accel_dev, csr, errsou); + adf_handle_cpp_cmd_par_err(accel_dev, csr, errsou); + adf_handle_ri_mem_par_err(accel_dev, csr, errsou); + adf_handle_ti_err(accel_dev, csr, errsou); + adf_handle_sfi_cmd_parerr(accel_dev, csr, errsou); + + errsou = ADF_CSR_RD(csr, ADF_GEN6_ERRSOU1); + if (errsou & ADF_GEN6_ERRSOU1_MASK) + dev_warn(&GET_DEV(accel_dev), "errsou1 still set: %#x\n", errsou); +} + +static void adf_handle_cerrssmsh(struct adf_accel_dev *accel_dev, void __iomem *csr) +{ + u32 reg; + + reg = ADF_CSR_RD(csr, ADF_GEN6_CERRSSMSH); + reg &= ADF_GEN6_CERRSSMSH_ERROR_BIT; + if (reg) { + dev_warn(&GET_DEV(accel_dev), + "Correctable error on ssm shared memory: %#x\n", reg); + ADF_RAS_ERR_CTR_INC(accel_dev->ras_errors, ADF_RAS_CORR); + ADF_CSR_WR(csr, ADF_GEN6_CERRSSMSH, reg); + } +} + +static void adf_handle_uerrssmsh(struct adf_accel_dev *accel_dev, void __iomem *csr, + u32 iastatssm) +{ + u32 reg; + + if (!(iastatssm & ADF_GEN6_IAINTSTATSSM_SH_ERR_BIT)) + return; + + reg = ADF_CSR_RD(csr, ADF_GEN6_UERRSSMSH); + reg &= ADF_GEN6_UERRSSMSH_MASK; + if (reg) { + dev_err(&GET_DEV(accel_dev), + "Fatal error on ssm shared memory: %#x\n", reg); + ADF_RAS_ERR_CTR_INC(accel_dev->ras_errors, ADF_RAS_FATAL); + ADF_CSR_WR(csr, ADF_GEN6_UERRSSMSH, reg); + } +} + +static void adf_handle_pperr_err(struct adf_accel_dev *accel_dev, void __iomem *csr, + u32 iastatssm) +{ + u32 reg; + + if (!(iastatssm & ADF_GEN6_IAINTSTATSSM_PPERR_BIT)) + return; + + reg = ADF_CSR_RD(csr, ADF_GEN6_PPERR); + reg &= ADF_GEN6_PPERR_MASK; + if (reg) { + dev_err(&GET_DEV(accel_dev), + "Fatal push or pull data error: %#x\n", reg); + ADF_RAS_ERR_CTR_INC(accel_dev->ras_errors, ADF_RAS_FATAL); + ADF_CSR_WR(csr, ADF_GEN6_PPERR, reg); + } +} + +static void adf_handle_scmpar_err(struct adf_accel_dev *accel_dev, void __iomem *csr, + u32 iastatssm) +{ + u32 reg; + + if (!(iastatssm & ADF_GEN6_IAINTSTATSSM_SCMPAR_ERR_BIT)) + return; + + reg = ADF_CSR_RD(csr, ADF_GEN6_SSM_FERR_STATUS); + reg &= ADF_GEN6_SCM_PAR_ERR_MASK; + if (reg) { + dev_err(&GET_DEV(accel_dev), "Fatal error on SCM: %#x\n", reg); + ADF_RAS_ERR_CTR_INC(accel_dev->ras_errors, ADF_RAS_FATAL); + ADF_CSR_WR(csr, ADF_GEN6_SSM_FERR_STATUS, reg); + } +} + +static void adf_handle_cpppar_err(struct adf_accel_dev *accel_dev, void __iomem *csr, + u32 iastatssm) +{ + u32 reg; + + if (!(iastatssm & ADF_GEN6_IAINTSTATSSM_CPPPAR_ERR_BIT)) + return; + + reg = ADF_CSR_RD(csr, ADF_GEN6_SSM_FERR_STATUS); + reg &= ADF_GEN6_CPP_PAR_ERR_MASK; + if (reg) { + dev_err(&GET_DEV(accel_dev), "Fatal error on CPP: %#x\n", reg); + ADF_RAS_ERR_CTR_INC(accel_dev->ras_errors, ADF_RAS_FATAL); + ADF_CSR_WR(csr, ADF_GEN6_SSM_FERR_STATUS, reg); + } +} + +static void adf_handle_rfpar_err(struct adf_accel_dev *accel_dev, void __iomem *csr, + u32 iastatssm) +{ + u32 reg; + + if (!(iastatssm & ADF_GEN6_IAINTSTATSSM_RFPAR_ERR_BIT)) + return; + + reg = ADF_CSR_RD(csr, ADF_GEN6_SSM_FERR_STATUS); + reg &= ADF_GEN6_RF_PAR_ERR_MASK; + if (reg) { + dev_err(&GET_DEV(accel_dev), "Fatal error on RF Parity: %#x\n", reg); + ADF_RAS_ERR_CTR_INC(accel_dev->ras_errors, ADF_RAS_FATAL); + ADF_CSR_WR(csr, ADF_GEN6_SSM_FERR_STATUS, reg); + } +} + +static void adf_handle_unexp_cpl_err(struct adf_accel_dev *accel_dev, void __iomem *csr, + u32 iastatssm) +{ + u32 reg; + + if (!(iastatssm & ADF_GEN6_IAINTSTATSSM_UNEXP_CPL_ERR_BIT)) + return; + + reg = ADF_CSR_RD(csr, ADF_GEN6_SSM_FERR_STATUS); + reg &= ADF_GEN6_UNEXP_CPL_ERR_MASK; + if (reg) { + dev_err(&GET_DEV(accel_dev), + "Fatal error for AXI unexpected tag/length: %#x\n", reg); + ADF_RAS_ERR_CTR_INC(accel_dev->ras_errors, ADF_RAS_FATAL); + ADF_CSR_WR(csr, ADF_GEN6_SSM_FERR_STATUS, reg); + } +} + +static void adf_handle_iaintstatssm(struct adf_accel_dev *accel_dev, void __iomem *csr) +{ + u32 iastatssm = ADF_CSR_RD(csr, ADF_GEN6_IAINTSTATSSM); + + iastatssm &= ADF_GEN6_IAINTSTATSSM_MASK; + if (!iastatssm) + return; + + adf_handle_uerrssmsh(accel_dev, csr, iastatssm); + adf_handle_pperr_err(accel_dev, csr, iastatssm); + adf_handle_scmpar_err(accel_dev, csr, iastatssm); + adf_handle_cpppar_err(accel_dev, csr, iastatssm); + adf_handle_rfpar_err(accel_dev, csr, iastatssm); + adf_handle_unexp_cpl_err(accel_dev, csr, iastatssm); + + ADF_CSR_WR(csr, ADF_GEN6_IAINTSTATSSM, iastatssm); +} + +static void adf_handle_ssm(struct adf_accel_dev *accel_dev, void __iomem *csr, u32 errsou) +{ + if (!(errsou & ADF_GEN6_ERRSOU2_SSM_ERR_BIT)) + return; + + adf_handle_cerrssmsh(accel_dev, csr); + adf_handle_iaintstatssm(accel_dev, csr); +} + +static void adf_handle_cpp_cfc_err(struct adf_accel_dev *accel_dev, void __iomem *csr, + u32 errsou) +{ + u32 reg; + + if (!(errsou & ADF_GEN6_ERRSOU2_CPP_CFC_ERR_STATUS_BIT)) + return; + + reg = ADF_CSR_RD(csr, ADF_GEN6_CPP_CFC_ERR_STATUS); + if (reg & ADF_GEN6_CPP_CFC_ERR_STATUS_DATAPAR_BIT) { + dev_err(&GET_DEV(accel_dev), "CPP_CFC_ERR: data parity: %#x", reg); + ADF_RAS_ERR_CTR_INC(accel_dev->ras_errors, ADF_RAS_UNCORR); + } + + if (reg & ADF_GEN6_CPP_CFC_ERR_STATUS_CMDPAR_BIT) { + dev_err(&GET_DEV(accel_dev), "CPP_CFC_ERR: command parity: %#x", reg); + ADF_RAS_ERR_CTR_INC(accel_dev->ras_errors, ADF_RAS_FATAL); + } + + if (reg & ADF_GEN6_CPP_CFC_FATAL_ERR_BIT) { + dev_err(&GET_DEV(accel_dev), "CPP_CFC_ERR: errors: %#x", reg); + ADF_RAS_ERR_CTR_INC(accel_dev->ras_errors, ADF_RAS_FATAL); + } + + ADF_CSR_WR(csr, ADF_GEN6_CPP_CFC_ERR_STATUS_CLR, + ADF_GEN6_CPP_CFC_ERR_STATUS_CLR_MASK); +} + +static void adf_gen6_process_errsou2(struct adf_accel_dev *accel_dev, void __iomem *csr, + u32 errsou) +{ + adf_handle_ssm(accel_dev, csr, errsou); + adf_handle_cpp_cfc_err(accel_dev, csr, errsou); + + errsou = ADF_CSR_RD(csr, ADF_GEN6_ERRSOU2); + if (errsou & ADF_GEN6_ERRSOU2_MASK) + dev_warn(&GET_DEV(accel_dev), "errsou2 still set: %#x\n", errsou); +} + +static void adf_handle_timiscsts(struct adf_accel_dev *accel_dev, void __iomem *csr, + u32 errsou) +{ + u32 timiscsts; + + if (!(errsou & ADF_GEN6_ERRSOU3_TIMISCSTS_BIT)) + return; + + timiscsts = ADF_CSR_RD(csr, ADF_GEN6_TIMISCSTS); + if (timiscsts) { + dev_err(&GET_DEV(accel_dev), "Fatal error in transmit interface: %#x\n", + timiscsts); + ADF_RAS_ERR_CTR_INC(accel_dev->ras_errors, ADF_RAS_FATAL); + } +} + +static void adf_handle_ricppintsts(struct adf_accel_dev *accel_dev, void __iomem *csr, + u32 errsou) +{ + u32 ricppintsts; + + if (!(errsou & ADF_GEN6_ERRSOU3_RICPPINTSTS_MASK)) + return; + + ricppintsts = ADF_CSR_RD(csr, ADF_GEN6_RICPPINTSTS); + ricppintsts &= ADF_GEN6_RICPPINTSTS_MASK; + if (ricppintsts) { + dev_err(&GET_DEV(accel_dev), "RI push pull error: %#x\n", ricppintsts); + ADF_RAS_ERR_CTR_INC(accel_dev->ras_errors, ADF_RAS_UNCORR); + ADF_CSR_WR(csr, ADF_GEN6_RICPPINTSTS, ricppintsts); + } +} + +static void adf_handle_ticppintsts(struct adf_accel_dev *accel_dev, void __iomem *csr, + u32 errsou) +{ + u32 ticppintsts; + + if (!(errsou & ADF_GEN6_ERRSOU3_TICPPINTSTS_MASK)) + return; + + ticppintsts = ADF_CSR_RD(csr, ADF_GEN6_TICPPINTSTS); + ticppintsts &= ADF_GEN6_TICPPINTSTS_MASK; + if (ticppintsts) { + dev_err(&GET_DEV(accel_dev), "TI push pull error: %#x\n", ticppintsts); + ADF_RAS_ERR_CTR_INC(accel_dev->ras_errors, ADF_RAS_FATAL); + ADF_CSR_WR(csr, ADF_GEN6_TICPPINTSTS, ticppintsts); + } +} + +static void adf_handle_atufaultstatus(struct adf_accel_dev *accel_dev, void __iomem *csr, + u32 errsou) +{ + u32 max_rp_num = GET_HW_DATA(accel_dev)->num_banks; + u32 atufaultstatus; + u32 i; + + if (!(errsou & ADF_GEN6_ERRSOU3_ATUFAULTSTATUS_BIT)) + return; + + for (i = 0; i < max_rp_num; i++) { + atufaultstatus = ADF_CSR_RD(csr, ADF_GEN6_ATUFAULTSTATUS(i)); + + atufaultstatus &= ADF_GEN6_ATUFAULTSTATUS_BIT; + if (atufaultstatus) { + dev_err(&GET_DEV(accel_dev), "Ring pair (%u) ATU detected fault: %#x\n", i, + atufaultstatus); + ADF_RAS_ERR_CTR_INC(accel_dev->ras_errors, ADF_RAS_UNCORR); + ADF_CSR_WR(csr, ADF_GEN6_ATUFAULTSTATUS(i), atufaultstatus); + } + } +} + +static void adf_handle_rlterror(struct adf_accel_dev *accel_dev, void __iomem *csr, + u32 errsou) +{ + u32 rlterror; + + if (!(errsou & ADF_GEN6_ERRSOU3_RLTERROR_BIT)) + return; + + rlterror = ADF_CSR_RD(csr, ADF_GEN6_RLT_ERRLOG); + rlterror &= ADF_GEN6_RLT_ERRLOG_MASK; + if (rlterror) { + dev_err(&GET_DEV(accel_dev), "Error in rate limiting block: %#x\n", rlterror); + ADF_RAS_ERR_CTR_INC(accel_dev->ras_errors, ADF_RAS_UNCORR); + ADF_CSR_WR(csr, ADF_GEN6_RLT_ERRLOG, rlterror); + } +} + +static void adf_handle_vflr(struct adf_accel_dev *accel_dev, void __iomem *csr, u32 errsou) +{ + if (!(errsou & ADF_GEN6_ERRSOU3_VFLRNOTIFY_BIT)) + return; + + dev_err(&GET_DEV(accel_dev), "Uncorrectable error in VF\n"); + ADF_RAS_ERR_CTR_INC(accel_dev->ras_errors, ADF_RAS_UNCORR); +} + +static void adf_handle_tc_vc_map_error(struct adf_accel_dev *accel_dev, void __iomem *csr, + u32 errsou) +{ + if (!(errsou & ADF_GEN6_ERRSOU3_TC_VC_MAP_ERROR_BIT)) + return; + + dev_err(&GET_DEV(accel_dev), "Violation of PCIe TC VC mapping\n"); + ADF_RAS_ERR_CTR_INC(accel_dev->ras_errors, ADF_RAS_FATAL); +} + +static void adf_handle_pcie_devhalt(struct adf_accel_dev *accel_dev, void __iomem *csr, + u32 errsou) +{ + if (!(errsou & ADF_GEN6_ERRSOU3_PCIE_DEVHALT_BIT)) + return; + + dev_err(&GET_DEV(accel_dev), + "DEVHALT due to an error in an incoming transaction\n"); + ADF_RAS_ERR_CTR_INC(accel_dev->ras_errors, ADF_RAS_FATAL); +} + +static void adf_handle_pg_req_devhalt(struct adf_accel_dev *accel_dev, void __iomem *csr, + u32 errsou) +{ + if (!(errsou & ADF_GEN6_ERRSOU3_PG_REQ_DEVHALT_BIT)) + return; + + dev_err(&GET_DEV(accel_dev), + "Error due to response failure in response to a page request\n"); + ADF_RAS_ERR_CTR_INC(accel_dev->ras_errors, ADF_RAS_FATAL); +} + +static void adf_handle_xlt_cpl_devhalt(struct adf_accel_dev *accel_dev, void __iomem *csr, + u32 errsou) +{ + if (!(errsou & ADF_GEN6_ERRSOU3_XLT_CPL_DEVHALT_BIT)) + return; + + dev_err(&GET_DEV(accel_dev), "Error status for a address translation request\n"); + ADF_RAS_ERR_CTR_INC(accel_dev->ras_errors, ADF_RAS_FATAL); +} + +static void adf_handle_ti_int_err_devhalt(struct adf_accel_dev *accel_dev, void __iomem *csr, + u32 errsou) +{ + if (!(errsou & ADF_GEN6_ERRSOU3_TI_INT_ERR_DEVHALT_BIT)) + return; + + dev_err(&GET_DEV(accel_dev), "DEVHALT due to a TI internal memory error\n"); + ADF_RAS_ERR_CTR_INC(accel_dev->ras_errors, ADF_RAS_FATAL); +} + +static void adf_gen6_process_errsou3(struct adf_accel_dev *accel_dev, void __iomem *csr, + u32 errsou) +{ + adf_handle_timiscsts(accel_dev, csr, errsou); + adf_handle_ricppintsts(accel_dev, csr, errsou); + adf_handle_ticppintsts(accel_dev, csr, errsou); + adf_handle_atufaultstatus(accel_dev, csr, errsou); + adf_handle_rlterror(accel_dev, csr, errsou); + adf_handle_vflr(accel_dev, csr, errsou); + adf_handle_tc_vc_map_error(accel_dev, csr, errsou); + adf_handle_pcie_devhalt(accel_dev, csr, errsou); + adf_handle_pg_req_devhalt(accel_dev, csr, errsou); + adf_handle_xlt_cpl_devhalt(accel_dev, csr, errsou); + adf_handle_ti_int_err_devhalt(accel_dev, csr, errsou); + + errsou = ADF_CSR_RD(csr, ADF_GEN6_ERRSOU3); + if (errsou & ADF_GEN6_ERRSOU3_MASK) + dev_warn(&GET_DEV(accel_dev), "errsou3 still set: %#x\n", errsou); +} + +static void adf_gen6_is_reset_required(struct adf_accel_dev *accel_dev, void __iomem *csr, + bool *reset_required) +{ + u8 reset, dev_state; + u32 gensts; + + gensts = ADF_CSR_RD(csr, ADF_GEN6_GENSTS); + dev_state = FIELD_GET(ADF_GEN6_GENSTS_DEVICE_STATE_MASK, gensts); + reset = FIELD_GET(ADF_GEN6_GENSTS_RESET_TYPE_MASK, gensts); + if (dev_state == ADF_GEN6_GENSTS_DEVHALT && reset == ADF_GEN6_GENSTS_PFLR) { + *reset_required = true; + return; + } + + if (reset == ADF_GEN6_GENSTS_COLD_RESET) + dev_err(&GET_DEV(accel_dev), "Fatal error, cold reset required\n"); + + *reset_required = false; +} + +static bool adf_gen6_handle_interrupt(struct adf_accel_dev *accel_dev, bool *reset_required) +{ + void __iomem *csr = adf_get_pmisc_base(accel_dev); + bool handled = false; + u32 errsou; + + errsou = ADF_CSR_RD(csr, ADF_GEN6_ERRSOU0); + if (errsou & ADF_GEN6_ERRSOU0_MASK) { + adf_gen6_process_errsou0(accel_dev, csr); + handled = true; + } + + errsou = ADF_CSR_RD(csr, ADF_GEN6_ERRSOU1); + if (errsou & ADF_GEN6_ERRSOU1_MASK) { + adf_gen6_process_errsou1(accel_dev, csr, errsou); + handled = true; + } + + errsou = ADF_CSR_RD(csr, ADF_GEN6_ERRSOU2); + if (errsou & ADF_GEN6_ERRSOU2_MASK) { + adf_gen6_process_errsou2(accel_dev, csr, errsou); + handled = true; + } + + errsou = ADF_CSR_RD(csr, ADF_GEN6_ERRSOU3); + if (errsou & ADF_GEN6_ERRSOU3_MASK) { + adf_gen6_process_errsou3(accel_dev, csr, errsou); + handled = true; + } + + adf_gen6_is_reset_required(accel_dev, csr, reset_required); + + return handled; +} + +void adf_gen6_init_ras_ops(struct adf_ras_ops *ras_ops) +{ + ras_ops->enable_ras_errors = adf_gen6_enable_ras; + ras_ops->disable_ras_errors = adf_gen6_disable_ras; + ras_ops->handle_interrupt = adf_gen6_handle_interrupt; +} +EXPORT_SYMBOL_GPL(adf_gen6_init_ras_ops); diff --git a/drivers/crypto/intel/qat/qat_common/adf_gen6_ras.h b/drivers/crypto/intel/qat/qat_common/adf_gen6_ras.h new file mode 100644 index 000000000000..66ced271d173 --- /dev/null +++ b/drivers/crypto/intel/qat/qat_common/adf_gen6_ras.h @@ -0,0 +1,504 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* Copyright(c) 2025 Intel Corporation */ +#ifndef ADF_GEN6_RAS_H_ +#define ADF_GEN6_RAS_H_ + +#include + +struct adf_ras_ops; + +/* Error source registers */ +#define ADF_GEN6_ERRSOU0 0x41A200 +#define ADF_GEN6_ERRSOU1 0x41A204 +#define ADF_GEN6_ERRSOU2 0x41A208 +#define ADF_GEN6_ERRSOU3 0x41A20C + +/* Error source mask registers */ +#define ADF_GEN6_ERRMSK0 0x41A210 +#define ADF_GEN6_ERRMSK1 0x41A214 +#define ADF_GEN6_ERRMSK2 0x41A218 +#define ADF_GEN6_ERRMSK3 0x41A21C + +/* ERRSOU0 Correctable error mask */ +#define ADF_GEN6_ERRSOU0_MASK BIT(0) + +#define ADF_GEN6_ERRSOU1_CPP0_MEUNC_BIT BIT(0) +#define ADF_GEN6_ERRSOU1_CPP_CMDPARERR_BIT BIT(1) +#define ADF_GEN6_ERRSOU1_RIMEM_PARERR_STS_BIT BIT(2) +#define ADF_GEN6_ERRSOU1_TIMEM_PARERR_STS_BIT BIT(3) +#define ADF_GEN6_ERRSOU1_SFICMD_PARERR_BIT BIT(4) + +#define ADF_GEN6_ERRSOU1_MASK ( \ + (ADF_GEN6_ERRSOU1_CPP0_MEUNC_BIT) | \ + (ADF_GEN6_ERRSOU1_CPP_CMDPARERR_BIT) | \ + (ADF_GEN6_ERRSOU1_RIMEM_PARERR_STS_BIT) | \ + (ADF_GEN6_ERRSOU1_TIMEM_PARERR_STS_BIT) | \ + (ADF_GEN6_ERRSOU1_SFICMD_PARERR_BIT)) + +#define ADF_GEN6_ERRMSK1_CPP0_MEUNC_BIT BIT(0) +#define ADF_GEN6_ERRMSK1_CPP_CMDPARERR_BIT BIT(1) +#define ADF_GEN6_ERRMSK1_RIMEM_PARERR_STS_BIT BIT(2) +#define ADF_GEN6_ERRMSK1_TIMEM_PARERR_STS_BIT BIT(3) +#define ADF_GEN6_ERRMSK1_IOSFCMD_PARERR_BIT BIT(4) + +#define ADF_GEN6_ERRMSK1_MASK ( \ + (ADF_GEN6_ERRMSK1_CPP0_MEUNC_BIT) | \ + (ADF_GEN6_ERRMSK1_CPP_CMDPARERR_BIT) | \ + (ADF_GEN6_ERRMSK1_RIMEM_PARERR_STS_BIT) | \ + (ADF_GEN6_ERRMSK1_TIMEM_PARERR_STS_BIT) | \ + (ADF_GEN6_ERRMSK1_IOSFCMD_PARERR_BIT)) + +/* HI AE Uncorrectable error log */ +#define ADF_GEN6_HIAEUNCERRLOG_CPP0 0x41A300 + +/* HI AE Uncorrectable error log enable */ +#define ADF_GEN6_HIAEUNCERRLOGENABLE_CPP0 0x41A320 + +/* HI AE Correctable error log */ +#define ADF_GEN6_HIAECORERRLOG_CPP0 0x41A308 + +/* HI AE Correctable error log enable */ +#define ADF_GEN6_HIAECORERRLOGENABLE_CPP0 0x41A318 + +/* HI CPP Agent Command parity error log */ +#define ADF_GEN6_HICPPAGENTCMDPARERRLOG 0x41A310 + +/* HI CPP Agent command parity error logging enable */ +#define ADF_GEN6_HICPPAGENTCMDPARERRLOGENABLE 0x41A314 + +#define ADF_6XXX_HICPPAGENTCMDPARERRLOG_MASK 0x1B + +/* RI Memory parity error status register */ +#define ADF_GEN6_RIMEM_PARERR_STS 0x41B128 + +/* RI Memory parity error reporting enable */ +#define ADF_GEN6_RI_MEM_PAR_ERR_EN0 0x41B12C + +/* + * RI Memory parity error mask + * BIT(4) - ri_tlq_phdr parity error + * BIT(5) - ri_tlq_pdata parity error + * BIT(6) - ri_tlq_nphdr parity error + * BIT(7) - ri_tlq_npdata parity error + * BIT(8) - ri_tlq_cplhdr parity error + * BIT(10) - BIT(13) - ri_tlq_cpldata[0:3] parity error + * BIT(19) - ri_cds_cmd_fifo parity error + * BIT(20) - ri_obc_ricpl_fifo parity error + * BIT(21) - ri_obc_tiricpl_fifo parity error + * BIT(22) - ri_obc_cppcpl_fifo parity error + * BIT(23) - ri_obc_pendcpl_fifo parity error + * BIT(24) - ri_cpp_cmd_fifo parity error + * BIT(25) - ri_cds_ticmd_fifo parity error + * BIT(26) - riti_cmd_fifo parity error + * BIT(27) - ri_int_msixtbl parity error + * BIT(28) - ri_int_imstbl parity error + * BIT(30) - ri_kpt_fuses parity error + */ +#define ADF_GEN6_RIMEM_PARERR_FATAL_MASK \ + (BIT(0) | BIT(1) | BIT(2) | BIT(4) | BIT(5) | BIT(6) | \ + BIT(7) | BIT(8) | BIT(18) | BIT(19) | BIT(20) | BIT(21) | \ + BIT(22) | BIT(23) | BIT(24) | BIT(25) | BIT(26) | BIT(27) | \ + BIT(28) | BIT(30)) + +#define ADF_GEN6_RIMEM_PARERR_CERR_MASK \ + (BIT(10) | BIT(11) | BIT(12) | BIT(13)) + +/* TI CI parity status */ +#define ADF_GEN6_TI_CI_PAR_STS 0x50060C + +/* TI CI parity reporting mask */ +#define ADF_GEN6_TI_CI_PAR_ERR_MASK 0x500608 + +/* + * TI CI parity status mask + * BIT(0) - CdCmdQ_sts patiry error status + * BIT(1) - CdDataQ_sts parity error status + * BIT(3) - CPP_SkidQ_sts parity error status + */ +#define ADF_GEN6_TI_CI_PAR_STS_MASK \ + (BIT(0) | BIT(1) | BIT(3)) + +/* TI PULLFUB parity status */ +#define ADF_GEN6_TI_PULL0FUB_PAR_STS 0x500618 + +/* TI PULLFUB parity error reporting mask */ +#define ADF_GEN6_TI_PULL0FUB_PAR_ERR_MASK 0x500614 + +/* + * TI PULLFUB parity status mask + * BIT(0) - TrnPullReqQ_sts parity status + * BIT(1) - TrnSharedDataQ_sts parity status + * BIT(2) - TrnPullReqDataQ_sts parity status + * BIT(4) - CPP_CiPullReqQ_sts parity status + * BIT(5) - CPP_TrnPullReqQ_sts parity status + * BIT(6) - CPP_PullidQ_sts parity status + * BIT(7) - CPP_WaitDataQ_sts parity status + * BIT(8) - CPP_CdDataQ_sts parity status + * BIT(9) - CPP_TrnDataQP0_sts parity status + * BIT(10) - BIT(11) - CPP_TrnDataQRF[00:01]_sts parity status + * BIT(12) - CPP_TrnDataQP1_sts parity status + * BIT(13) - BIT(14) - CPP_TrnDataQRF[10:11]_sts parity status + */ +#define ADF_GEN6_TI_PULL0FUB_PAR_STS_MASK \ + (BIT(0) | BIT(1) | BIT(2) | BIT(4) | BIT(5) | BIT(6) | BIT(7) | \ + BIT(8) | BIT(9) | BIT(10) | BIT(11) | BIT(12) | BIT(13) | BIT(14)) + +/* TI PUSHUB parity status */ +#define ADF_GEN6_TI_PUSHFUB_PAR_STS 0x500630 + +/* TI PUSHFUB parity error reporting mask */ +#define ADF_GEN6_TI_PUSHFUB_PAR_ERR_MASK 0x50062C + +/* + * TI PUSHUB parity status mask + * BIT(0) - SbPushReqQ_sts parity status + * BIT(1) - BIT(2) - SbPushDataQ[0:1]_sts parity status + * BIT(4) - CPP_CdPushReqQ_sts parity status + * BIT(5) - BIT(6) - CPP_CdPushDataQ[0:1]_sts parity status + * BIT(7) - CPP_SbPushReqQ_sts parity status + * BIT(8) - CPP_SbPushDataQP_sts parity status + * BIT(9) - BIT(10) - CPP_SbPushDataQRF[0:1]_sts parity status + */ +#define ADF_GEN6_TI_PUSHFUB_PAR_STS_MASK \ + (BIT(0) | BIT(1) | BIT(2) | BIT(4) | BIT(5) | \ + BIT(6) | BIT(7) | BIT(8) | BIT(9) | BIT(10)) + +/* TI CD parity status */ +#define ADF_GEN6_TI_CD_PAR_STS 0x50063C + +/* TI CD parity error mask */ +#define ADF_GEN6_TI_CD_PAR_ERR_MASK 0x500638 + +/* + * TI CD parity status mask + * BIT(0) - BIT(15) - CtxMdRam[0:15]_sts parity status + * BIT(16) - Leaf2ClusterRam_sts parity status + * BIT(17) - BIT(18) - Ring2LeafRam[0:1]_sts parity status + * BIT(19) - VirtualQ_sts parity status + * BIT(20) - DtRdQ_sts parity status + * BIT(21) - DtWrQ_sts parity status + * BIT(22) - RiCmdQ_sts parity status + * BIT(23) - BypassQ_sts parity status + * BIT(24) - DtRdQ_sc_sts parity status + * BIT(25) - DtWrQ_sc_sts parity status + */ +#define ADF_GEN6_TI_CD_PAR_STS_MASK \ + (BIT(0) | BIT(1) | BIT(2) | BIT(3) | BIT(4) | BIT(5) | BIT(6) | \ + BIT(7) | BIT(8) | BIT(9) | BIT(10) | BIT(11) | BIT(12) | BIT(13) | \ + BIT(14) | BIT(15) | BIT(16) | BIT(17) | BIT(18) | BIT(19) | BIT(20) | \ + BIT(21) | BIT(22) | BIT(23) | BIT(24) | BIT(25)) + +/* TI TRNSB parity status */ +#define ADF_GEN6_TI_TRNSB_PAR_STS 0x500648 + +/* TI TRNSB parity error reporting mask */ +#define ADF_GEN6_TI_TRNSB_PAR_ERR_MASK 0x500644 + +/* + * TI TRNSB parity status mask + * BIT(0) - TrnPHdrQP_sts parity status + * BIT(1) - TrnPHdrQRF_sts parity status + * BIT(2) - TrnPDataQP_sts parity status + * BIT(3) - BIT(6) - TrnPDataQRF[0:3]_sts parity status + * BIT(7) - TrnNpHdrQP_sts parity status + * BIT(8) - BIT(9) - TrnNpHdrQRF[0:1]_sts parity status + * BIT(10) - TrnCplHdrQ_sts parity status + * BIT(11) - TrnPutObsReqQ_sts parity status + * BIT(12) - TrnPushReqQ_sts parity status + * BIT(13) - SbSplitIdRam_sts parity status + * BIT(14) - SbReqCountQ_sts parity status + * BIT(15) - SbCplTrkRam_sts parity status + * BIT(16) - SbGetObsReqQ_sts parity status + * BIT(17) - SbEpochIdQ_sts parity status + * BIT(18) - SbAtCplHdrQ_sts parity status + * BIT(19) - SbAtCplDataQ_sts parity status + * BIT(20) - SbReqCountRam_sts parity status + * BIT(21) - SbAtCplHdrQ_sc_sts parity status + */ +#define ADF_GEN6_TI_TRNSB_PAR_STS_MASK \ + (BIT(0) | BIT(1) | BIT(2) | BIT(3) | BIT(4) | BIT(5) | BIT(6) | \ + BIT(7) | BIT(8) | BIT(9) | BIT(10) | BIT(11) | BIT(12) | \ + BIT(13) | BIT(14) | BIT(15) | BIT(16) | BIT(17) | BIT(18) | \ + BIT(19) | BIT(20) | BIT(21)) + +/* Status register to log misc error on RI */ +#define ADF_GEN6_RIMISCSTS 0x41B1B8 + +/* Status control register to log misc RI error */ +#define ADF_GEN6_RIMISCCTL 0x41B1BC + +/* + * ERRSOU2 bit mask + * BIT(0) - SSM Interrupt Mask + * BIT(1) - CFC on CPP. ORed of CFC Push error and Pull error + * BIT(2) - BIT(4) - CPP attention interrupts + * BIT(18) - PM interrupt + */ +#define ADF_GEN6_ERRSOU2_SSM_ERR_BIT BIT(0) +#define ADF_GEN6_ERRSOU2_CPP_CFC_ERR_STATUS_BIT BIT(1) +#define ADF_GEN6_ERRSOU2_CPP_CFC_ATT_INT_MASK \ + (BIT(2) | BIT(3) | BIT(4)) + +#define ADF_GEN6_ERRSOU2_PM_INT_BIT BIT(18) + +#define ADF_GEN6_ERRSOU2_MASK \ + (ADF_GEN6_ERRSOU2_SSM_ERR_BIT | \ + ADF_GEN6_ERRSOU2_CPP_CFC_ERR_STATUS_BIT) + +#define ADF_GEN6_ERRSOU2_DIS_MASK \ + (ADF_GEN6_ERRSOU2_SSM_ERR_BIT | \ + ADF_GEN6_ERRSOU2_CPP_CFC_ERR_STATUS_BIT | \ + ADF_GEN6_ERRSOU2_CPP_CFC_ATT_INT_MASK) + +#define ADF_GEN6_IAINTSTATSSM 0x28 + +/* IAINTSTATSSM error bit mask definitions */ +#define ADF_GEN6_IAINTSTATSSM_SH_ERR_BIT BIT(0) +#define ADF_GEN6_IAINTSTATSSM_PPERR_BIT BIT(2) +#define ADF_GEN6_IAINTSTATSSM_SCMPAR_ERR_BIT BIT(4) +#define ADF_GEN6_IAINTSTATSSM_CPPPAR_ERR_BIT BIT(5) +#define ADF_GEN6_IAINTSTATSSM_RFPAR_ERR_BIT BIT(6) +#define ADF_GEN6_IAINTSTATSSM_UNEXP_CPL_ERR_BIT BIT(7) + +#define ADF_GEN6_IAINTSTATSSM_MASK \ + (ADF_GEN6_IAINTSTATSSM_SH_ERR_BIT | \ + ADF_GEN6_IAINTSTATSSM_PPERR_BIT | \ + ADF_GEN6_IAINTSTATSSM_SCMPAR_ERR_BIT | \ + ADF_GEN6_IAINTSTATSSM_CPPPAR_ERR_BIT | \ + ADF_GEN6_IAINTSTATSSM_RFPAR_ERR_BIT | \ + ADF_GEN6_IAINTSTATSSM_UNEXP_CPL_ERR_BIT) + +#define ADF_GEN6_UERRSSMSH 0x18 + +/* + * UERRSSMSH error bit mask definitions + * + * BIT(0) - Indicates one uncorrectable error + * BIT(15) - Indicates multiple uncorrectable errors + * in device shared memory + */ +#define ADF_GEN6_UERRSSMSH_MASK (BIT(0) | BIT(15)) + +/* + * CERRSSMSH error bit + * BIT(0) - Indicates one correctable error + */ +#define ADF_GEN6_CERRSSMSH_ERROR_BIT (BIT(0) | BIT(15) | BIT(24)) +#define ADF_GEN6_CERRSSMSH 0x10 + +#define ADF_GEN6_INTMASKSSM 0x0 + +/* + * Error reporting mask in INTMASKSSM + * BIT(0) - Shared memory uncorrectable interrupt mask + * BIT(2) - PPERR interrupt mask + * BIT(4) - SCM parity error interrupt mask + * BIT(5) - CPP parity error interrupt mask + * BIT(6) - SHRAM RF parity error interrupt mask + * BIT(7) - AXI unexpected completion error mask + */ +#define ADF_GEN6_INTMASKSSM_MASK \ + (BIT(0) | BIT(2) | BIT(4) | BIT(5) | BIT(6) | BIT(7)) + +/* CPP push or pull error */ +#define ADF_GEN6_PPERR 0x8 + +#define ADF_GEN6_PPERR_MASK (BIT(0) | BIT(1)) + +/* + * SSM_FERR_STATUS error bit mask definitions + */ +#define ADF_GEN6_SCM_PAR_ERR_MASK BIT(5) +#define ADF_GEN6_CPP_PAR_ERR_MASK (BIT(0) | BIT(1) | BIT(2)) +#define ADF_GEN6_UNEXP_CPL_ERR_MASK (BIT(3) | BIT(4) | BIT(10) | BIT(11)) +#define ADF_GEN6_RF_PAR_ERR_MASK BIT(16) + +#define ADF_GEN6_SSM_FERR_STATUS 0x9C + +#define ADF_GEN6_CPP_CFC_ERR_STATUS 0x640C04 + +/* + * BIT(0) - Indicates one or more CPP CFC errors + * BIT(1) - Indicates multiple CPP CFC errors + * BIT(7) - Indicates CPP CFC command parity error type + * BIT(8) - Indicates CPP CFC data parity error type + */ +#define ADF_GEN6_CPP_CFC_ERR_STATUS_ERR_BIT BIT(0) +#define ADF_GEN6_CPP_CFC_ERR_STATUS_MERR_BIT BIT(1) +#define ADF_GEN6_CPP_CFC_ERR_STATUS_CMDPAR_BIT BIT(7) +#define ADF_GEN6_CPP_CFC_ERR_STATUS_DATAPAR_BIT BIT(8) +#define ADF_GEN6_CPP_CFC_FATAL_ERR_BIT \ + (ADF_GEN6_CPP_CFC_ERR_STATUS_ERR_BIT | \ + ADF_GEN6_CPP_CFC_ERR_STATUS_MERR_BIT) + +/* + * BIT(0) - Enables CFC to detect and log a push/pull data error + * BIT(1) - Enables CFC to generate interrupt to PCIEP for a CPP error + * BIT(4) - When 1 parity detection is disabled + * BIT(5) - When 1 parity detection is disabled on CPP command bus + * BIT(6) - When 1 parity detection is disabled on CPP push/pull bus + * BIT(9) - When 1 RF parity error detection is disabled + */ +#define ADF_GEN6_CPP_CFC_ERR_CTRL_MASK (BIT(0) | BIT(1)) + +#define ADF_GEN6_CPP_CFC_ERR_CTRL_DIS_MASK \ + (BIT(4) | BIT(5) | BIT(6) | BIT(9) | BIT(10)) + +#define ADF_GEN6_CPP_CFC_ERR_CTRL 0x640C00 + +/* + * BIT(0) - Clears bit(0) of ADF_GEN6_CPP_CFC_ERR_STATUS + * when an error is reported on CPP + * BIT(1) - Clears bit(1) of ADF_GEN6_CPP_CFC_ERR_STATUS + * when multiple errors are reported on CPP + * BIT(2) - Clears bit(2) of ADF_GEN6_CPP_CFC_ERR_STATUS + * when attention interrupt is reported + */ +#define ADF_GEN6_CPP_CFC_ERR_STATUS_CLR_MASK (BIT(0) | BIT(1) | BIT(2)) +#define ADF_GEN6_CPP_CFC_ERR_STATUS_CLR 0x640C08 + +/* + * ERRSOU3 bit masks + * BIT(0) - indicates error response order overflow and/or BME error + * BIT(1) - indicates RI push/pull error + * BIT(2) - indicates TI push/pull error + * BIT(5) - indicates TI pull parity error + * BIT(6) - indicates RI push parity error + * BIT(7) - indicates VFLR interrupt + * BIT(8) - indicates ring pair interrupts for ATU detected fault + * BIT(9) - indicates rate limiting error + */ +#define ADF_GEN6_ERRSOU3_TIMISCSTS_BIT BIT(0) +#define ADF_GEN6_ERRSOU3_RICPPINTSTS_MASK (BIT(1) | BIT(6)) +#define ADF_GEN6_ERRSOU3_TICPPINTSTS_MASK (BIT(2) | BIT(5)) +#define ADF_GEN6_ERRSOU3_VFLRNOTIFY_BIT BIT(7) +#define ADF_GEN6_ERRSOU3_ATUFAULTSTATUS_BIT BIT(8) +#define ADF_GEN6_ERRSOU3_RLTERROR_BIT BIT(9) +#define ADF_GEN6_ERRSOU3_TC_VC_MAP_ERROR_BIT BIT(16) +#define ADF_GEN6_ERRSOU3_PCIE_DEVHALT_BIT BIT(17) +#define ADF_GEN6_ERRSOU3_PG_REQ_DEVHALT_BIT BIT(18) +#define ADF_GEN6_ERRSOU3_XLT_CPL_DEVHALT_BIT BIT(19) +#define ADF_GEN6_ERRSOU3_TI_INT_ERR_DEVHALT_BIT BIT(20) + +#define ADF_GEN6_ERRSOU3_MASK ( \ + (ADF_GEN6_ERRSOU3_TIMISCSTS_BIT) | \ + (ADF_GEN6_ERRSOU3_RICPPINTSTS_MASK) | \ + (ADF_GEN6_ERRSOU3_TICPPINTSTS_MASK) | \ + (ADF_GEN6_ERRSOU3_VFLRNOTIFY_BIT) | \ + (ADF_GEN6_ERRSOU3_ATUFAULTSTATUS_BIT) | \ + (ADF_GEN6_ERRSOU3_RLTERROR_BIT) | \ + (ADF_GEN6_ERRSOU3_TC_VC_MAP_ERROR_BIT) | \ + (ADF_GEN6_ERRSOU3_PCIE_DEVHALT_BIT) | \ + (ADF_GEN6_ERRSOU3_PG_REQ_DEVHALT_BIT) | \ + (ADF_GEN6_ERRSOU3_XLT_CPL_DEVHALT_BIT) | \ + (ADF_GEN6_ERRSOU3_TI_INT_ERR_DEVHALT_BIT)) + +#define ADF_GEN6_ERRSOU3_DIS_MASK ( \ + (ADF_GEN6_ERRSOU3_TIMISCSTS_BIT) | \ + (ADF_GEN6_ERRSOU3_RICPPINTSTS_MASK) | \ + (ADF_GEN6_ERRSOU3_TICPPINTSTS_MASK) | \ + (ADF_GEN6_ERRSOU3_VFLRNOTIFY_BIT) | \ + (ADF_GEN6_ERRSOU3_ATUFAULTSTATUS_BIT) | \ + (ADF_GEN6_ERRSOU3_RLTERROR_BIT) | \ + (ADF_GEN6_ERRSOU3_TC_VC_MAP_ERROR_BIT)) + +/* Rate limiting error log register */ +#define ADF_GEN6_RLT_ERRLOG 0x508814 + +#define ADF_GEN6_RLT_ERRLOG_MASK (BIT(0) | BIT(1) | BIT(2) | BIT(3)) + +/* TI misc status register */ +#define ADF_GEN6_TIMISCSTS 0x50054C + +/* TI misc error reporting mask */ +#define ADF_GEN6_TIMISCCTL 0x500548 + +/* + * TI Misc error reporting control mask + * BIT(0) - Enables error detection and logging in TIMISCSTS register + * BIT(1) - It has effect only when SRIOV enabled, this bit is 0 by default + * BIT(2) - Enables the D-F-x counter within the dispatch arbiter + * to start based on the command triggered from + * BIT(30) - Disables VFLR functionality + * bits 1, 2 and 30 value should be preserved and not meant to be changed + * within RAS. + */ +#define ADF_GEN6_TIMISCCTL_BIT BIT(0) +#define ADF_GEN6_TIMSCCTL_RELAY_MASK (BIT(1) | BIT(2) | BIT(30)) + +/* RI CPP interface status register */ +#define ADF_GEN6_RICPPINTSTS 0x41A330 + +/* + * Uncorrectable error mask in RICPPINTSTS register + * BIT(0) - RI asserted the CPP error signal during a push + * BIT(1) - RI detected the CPP error signal asserted during a pull + * BIT(2) - RI detected a push data parity error + * BIT(3) - RI detected a push valid parity error + */ +#define ADF_GEN6_RICPPINTSTS_MASK (BIT(0) | BIT(1) | BIT(2) | BIT(3)) + +/* RI CPP interface register control */ +#define ADF_GEN6_RICPPINTCTL 0x41A32C + +/* + * Control bit mask for RICPPINTCTL register + * BIT(0) - value of 1 enables error detection and reporting + * on the RI CPP Push interface + * BIT(1) - value of 1 enables error detection and reporting + * on the RI CPP Pull interface + * BIT(2) - value of 1 enables error detection and reporting + * on the RI Parity + * BIT(3) - value of 1 enable checking parity on CPP + */ +#define ADF_GEN6_RICPPINTCTL_MASK \ + (BIT(0) | BIT(1) | BIT(2) | BIT(3) | BIT(4)) + +/* TI CPP interface status register */ +#define ADF_GEN6_TICPPINTSTS 0x50053C + +/* + * Uncorrectable error mask in TICPPINTSTS register + * BIT(0) - value of 1 indicates that the TI asserted + * the CPP error signal during a push + * BIT(1) - value of 1 indicates that the TI detected + * the CPP error signal asserted during a pull + * BIT(2) - value of 1 indicates that the TI detected + * a pull data parity error + */ +#define ADF_GEN6_TICPPINTSTS_MASK (BIT(0) | BIT(1) | BIT(2)) + +/* TI CPP interface status register control */ +#define ADF_GEN6_TICPPINTCTL 0x500538 + +/* + * Control bit mask for TICPPINTCTL register + * BIT(0) - value of 1 enables error detection and reporting on + * the TI CPP Push interface + * BIT(1) - value of 1 enables error detection and reporting on + * the TI CPP Push interface + * BIT(2) - value of 1 enables parity error detection and logging on + * the TI CPP Pull interface + * BIT(3) - value of 1 enables CPP CMD and Pull Data parity checking + */ +#define ADF_GEN6_TICPPINTCTL_MASK \ + (BIT(0) | BIT(1) | BIT(2) | BIT(3) | BIT(4)) + +/* ATU fault status register */ +#define ADF_GEN6_ATUFAULTSTATUS(i) (0x506000 + ((i) * 0x4)) + +#define ADF_GEN6_ATUFAULTSTATUS_BIT BIT(0) + +/* Command parity error detected on IOSFP command to QAT */ +#define ADF_GEN6_RIMISCSTS_BIT BIT(0) + +#define ADF_GEN6_GENSTS 0x41A220 +#define ADF_GEN6_GENSTS_DEVICE_STATE_MASK GENMASK(1, 0) +#define ADF_GEN6_GENSTS_RESET_TYPE_MASK GENMASK(3, 2) +#define ADF_GEN6_GENSTS_PFLR 0x1 +#define ADF_GEN6_GENSTS_COLD_RESET 0x3 +#define ADF_GEN6_GENSTS_DEVHALT 0x1 + +void adf_gen6_init_ras_ops(struct adf_ras_ops *ras_ops); + +#endif /* ADF_GEN6_RAS_H_ */ -- 2.51.0 From 662c62bb83a9780c25e2d1d8bae6c2cc9a7c6663 Mon Sep 17 00:00:00 2001 From: Suman Kumar Chakraborty Date: Tue, 13 May 2025 11:25:27 +0100 Subject: [PATCH 16/16] crypto: qat - enable reporting of error counters for GEN6 devices Enable the reporting of error counters through sysfs for QAT GEN6 devices and update the ABI documentation. This enables the reporting of the following: - errors_correctable - hardware correctable errors that allow the system to recover without data loss. - errors_nonfatal: errors that can be isolated to specific in-flight requests. - errors_fatal: errors that cannot be contained to a request, requiring a Function Level Reset (FLR) upon occurrence. Signed-off-by: Suman Kumar Chakraborty Reviewed-by: Giovanni Cabiddu Signed-off-by: Herbert Xu --- Documentation/ABI/testing/sysfs-driver-qat_ras | 8 ++++---- drivers/crypto/intel/qat/qat_6xxx/adf_drv.c | 2 ++ 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/Documentation/ABI/testing/sysfs-driver-qat_ras b/Documentation/ABI/testing/sysfs-driver-qat_ras index 176dea1e9c0a..82ceb04445ec 100644 --- a/Documentation/ABI/testing/sysfs-driver-qat_ras +++ b/Documentation/ABI/testing/sysfs-driver-qat_ras @@ -4,7 +4,7 @@ KernelVersion: 6.7 Contact: qat-linux@intel.com Description: (RO) Reports the number of correctable errors detected by the device. - This attribute is only available for qat_4xxx devices. + This attribute is only available for qat_4xxx and qat_6xxx devices. What: /sys/bus/pci/devices//qat_ras/errors_nonfatal Date: January 2024 @@ -12,7 +12,7 @@ KernelVersion: 6.7 Contact: qat-linux@intel.com Description: (RO) Reports the number of non fatal errors detected by the device. - This attribute is only available for qat_4xxx devices. + This attribute is only available for qat_4xxx and qat_6xxx devices. What: /sys/bus/pci/devices//qat_ras/errors_fatal Date: January 2024 @@ -20,7 +20,7 @@ KernelVersion: 6.7 Contact: qat-linux@intel.com Description: (RO) Reports the number of fatal errors detected by the device. - This attribute is only available for qat_4xxx devices. + This attribute is only available for qat_4xxx and qat_6xxx devices. What: /sys/bus/pci/devices//qat_ras/reset_error_counters Date: January 2024 @@ -38,4 +38,4 @@ Description: (WO) Write to resets all error counters of a device. # cat /sys/bus/pci/devices//qat_ras/errors_fatal 0 - This attribute is only available for qat_4xxx devices. + This attribute is only available for qat_4xxx and qat_6xxx devices. diff --git a/drivers/crypto/intel/qat/qat_6xxx/adf_drv.c b/drivers/crypto/intel/qat/qat_6xxx/adf_drv.c index 132e26501621..c1dc9c56fdf5 100644 --- a/drivers/crypto/intel/qat/qat_6xxx/adf_drv.c +++ b/drivers/crypto/intel/qat/qat_6xxx/adf_drv.c @@ -173,6 +173,8 @@ static int adf_probe(struct pci_dev *pdev, const struct pci_device_id *ent) if (ret) return dev_err_probe(dev, ret, "Failed to save pci state.\n"); + accel_dev->ras_errors.enabled = true; + adf_dbgfs_init(accel_dev); ret = devm_add_action_or_reset(dev, adf_dbgfs_cleanup, accel_dev); -- 2.51.0