From 52b3b329d8e589575d16d8d9adbca9e08041ee82 Mon Sep 17 00:00:00 2001 From: Krzysztof Kozlowski Date: Fri, 7 Mar 2025 10:33:09 +0100 Subject: [PATCH 01/16] dt-bindings: rng: rockchip,rk3588-rng: Drop unnecessary status from example Device nodes are enabled by default, so no need for 'status = "okay"' in the DTS example. Reviewed-by: Heiko Stuebner Signed-off-by: Krzysztof Kozlowski Acked-by: Rob Herring (Arm) Signed-off-by: Herbert Xu --- Documentation/devicetree/bindings/rng/rockchip,rk3588-rng.yaml | 1 - 1 file changed, 1 deletion(-) diff --git a/Documentation/devicetree/bindings/rng/rockchip,rk3588-rng.yaml b/Documentation/devicetree/bindings/rng/rockchip,rk3588-rng.yaml index 757967212f55..ca71b400bcae 100644 --- a/Documentation/devicetree/bindings/rng/rockchip,rk3588-rng.yaml +++ b/Documentation/devicetree/bindings/rng/rockchip,rk3588-rng.yaml @@ -53,7 +53,6 @@ examples: interrupts = ; clocks = <&scmi_clk SCMI_HCLK_SECURE_NS>; resets = <&scmi_reset SCMI_SRST_H_TRNG_NS>; - status = "okay"; }; }; -- 2.51.0 From b949f55644a6d1645c0a71f78afabf12aec7c33b Mon Sep 17 00:00:00 2001 From: Dionna Glaze Date: Sat, 8 Mar 2025 12:10:28 +1100 Subject: [PATCH 02/16] crypto: ccp - Fix uAPI definitions of PSP errors Additions to the error enum after explicit 0x27 setting for SEV_RET_INVALID_KEY leads to incorrect value assignments. Use explicit values to match the manufacturer specifications more clearly. Fixes: 3a45dc2b419e ("crypto: ccp: Define the SEV-SNP commands") CC: stable@vger.kernel.org Signed-off-by: Dionna Glaze Reviewed-by: Tom Lendacky Signed-off-by: Alexey Kardashevskiy Signed-off-by: Herbert Xu --- include/uapi/linux/psp-sev.h | 21 ++++++++++++++------- 1 file changed, 14 insertions(+), 7 deletions(-) diff --git a/include/uapi/linux/psp-sev.h b/include/uapi/linux/psp-sev.h index 832c15d9155b..eeb20dfb1fda 100644 --- a/include/uapi/linux/psp-sev.h +++ b/include/uapi/linux/psp-sev.h @@ -73,13 +73,20 @@ typedef enum { SEV_RET_INVALID_PARAM, SEV_RET_RESOURCE_LIMIT, SEV_RET_SECURE_DATA_INVALID, - SEV_RET_INVALID_KEY = 0x27, - SEV_RET_INVALID_PAGE_SIZE, - SEV_RET_INVALID_PAGE_STATE, - SEV_RET_INVALID_MDATA_ENTRY, - SEV_RET_INVALID_PAGE_OWNER, - SEV_RET_INVALID_PAGE_AEAD_OFLOW, - SEV_RET_RMP_INIT_REQUIRED, + SEV_RET_INVALID_PAGE_SIZE = 0x0019, + SEV_RET_INVALID_PAGE_STATE = 0x001A, + SEV_RET_INVALID_MDATA_ENTRY = 0x001B, + SEV_RET_INVALID_PAGE_OWNER = 0x001C, + SEV_RET_AEAD_OFLOW = 0x001D, + SEV_RET_EXIT_RING_BUFFER = 0x001F, + SEV_RET_RMP_INIT_REQUIRED = 0x0020, + SEV_RET_BAD_SVN = 0x0021, + SEV_RET_BAD_VERSION = 0x0022, + SEV_RET_SHUTDOWN_REQUIRED = 0x0023, + SEV_RET_UPDATE_FAILED = 0x0024, + SEV_RET_RESTORE_REQUIRED = 0x0025, + SEV_RET_RMP_INITIALIZATION_FAILED = 0x0026, + SEV_RET_INVALID_KEY = 0x0027, SEV_RET_MAX, } sev_ret_code; -- 2.51.0 From 65775cf313987926e9746b0ca7f5519d297af2da Mon Sep 17 00:00:00 2001 From: Herbert Xu Date: Sat, 8 Mar 2025 20:45:21 +0800 Subject: [PATCH 03/16] crypto: scatterwalk - Change scatterwalk_next calling convention Rather than returning the address and storing the length into an argument pointer, add an address field to the walk struct and use that to store the address. The length is returned directly. Change the done functions to use this stored address instead of getting them from the caller. Split the address into two using a union. The user should only access the const version so that it is never changed. Signed-off-by: Herbert Xu --- arch/arm/crypto/ghash-ce-glue.c | 7 +++--- arch/arm64/crypto/aes-ce-ccm-glue.c | 9 +++---- arch/arm64/crypto/ghash-ce-glue.c | 7 +++--- arch/arm64/crypto/sm4-ce-ccm-glue.c | 8 +++--- arch/arm64/crypto/sm4-ce-gcm-glue.c | 8 +++--- arch/s390/crypto/aes_s390.c | 21 +++++++--------- arch/x86/crypto/aegis128-aesni-glue.c | 7 +++--- arch/x86/crypto/aesni-intel_glue.c | 9 +++---- crypto/aegis128-core.c | 7 +++--- crypto/scatterwalk.c | 14 +++++------ crypto/skcipher.c | 10 +++++--- drivers/crypto/nx/nx.c | 7 +++--- include/crypto/algapi.h | 7 ++++++ include/crypto/scatterwalk.h | 35 ++++++++++++++------------- 14 files changed, 78 insertions(+), 78 deletions(-) diff --git a/arch/arm/crypto/ghash-ce-glue.c b/arch/arm/crypto/ghash-ce-glue.c index 9613ffed84f9..dab66b520b6e 100644 --- a/arch/arm/crypto/ghash-ce-glue.c +++ b/arch/arm/crypto/ghash-ce-glue.c @@ -460,11 +460,10 @@ static void gcm_calculate_auth_mac(struct aead_request *req, u64 dg[], u32 len) do { unsigned int n; - const u8 *p; - p = scatterwalk_next(&walk, len, &n); - gcm_update_mac(dg, p, n, buf, &buf_count, ctx); - scatterwalk_done_src(&walk, p, n); + n = scatterwalk_next(&walk, len); + gcm_update_mac(dg, walk.addr, n, buf, &buf_count, ctx); + scatterwalk_done_src(&walk, n); if (unlikely(len / SZ_4K > (len - n) / SZ_4K)) { kernel_neon_end(); diff --git a/arch/arm64/crypto/aes-ce-ccm-glue.c b/arch/arm64/crypto/aes-ce-ccm-glue.c index 1c29546983bf..2d791d51891b 100644 --- a/arch/arm64/crypto/aes-ce-ccm-glue.c +++ b/arch/arm64/crypto/aes-ce-ccm-glue.c @@ -157,12 +157,11 @@ static void ccm_calculate_auth_mac(struct aead_request *req, u8 mac[]) do { unsigned int n; - const u8 *p; - p = scatterwalk_next(&walk, len, &n); - macp = ce_aes_ccm_auth_data(mac, p, n, macp, ctx->key_enc, - num_rounds(ctx)); - scatterwalk_done_src(&walk, p, n); + n = scatterwalk_next(&walk, len); + macp = ce_aes_ccm_auth_data(mac, walk.addr, n, macp, + ctx->key_enc, num_rounds(ctx)); + scatterwalk_done_src(&walk, n); len -= n; } while (len); } diff --git a/arch/arm64/crypto/ghash-ce-glue.c b/arch/arm64/crypto/ghash-ce-glue.c index 69d4fb78c30d..071e122f9c37 100644 --- a/arch/arm64/crypto/ghash-ce-glue.c +++ b/arch/arm64/crypto/ghash-ce-glue.c @@ -309,11 +309,10 @@ static void gcm_calculate_auth_mac(struct aead_request *req, u64 dg[], u32 len) do { unsigned int n; - const u8 *p; - p = scatterwalk_next(&walk, len, &n); - gcm_update_mac(dg, p, n, buf, &buf_count, ctx); - scatterwalk_done_src(&walk, p, n); + n = scatterwalk_next(&walk, len); + gcm_update_mac(dg, walk.addr, n, buf, &buf_count, ctx); + scatterwalk_done_src(&walk, n); len -= n; } while (len); diff --git a/arch/arm64/crypto/sm4-ce-ccm-glue.c b/arch/arm64/crypto/sm4-ce-ccm-glue.c index 119f86eb7cc9..e9cc1c1364ec 100644 --- a/arch/arm64/crypto/sm4-ce-ccm-glue.c +++ b/arch/arm64/crypto/sm4-ce-ccm-glue.c @@ -113,10 +113,10 @@ static void ccm_calculate_auth_mac(struct aead_request *req, u8 mac[]) do { unsigned int n, orig_n; - const u8 *p, *orig_p; + const u8 *p; - orig_p = scatterwalk_next(&walk, assoclen, &orig_n); - p = orig_p; + orig_n = scatterwalk_next(&walk, assoclen); + p = walk.addr; n = orig_n; while (n > 0) { @@ -149,7 +149,7 @@ static void ccm_calculate_auth_mac(struct aead_request *req, u8 mac[]) } } - scatterwalk_done_src(&walk, orig_p, orig_n); + scatterwalk_done_src(&walk, orig_n); assoclen -= orig_n; } while (assoclen); } diff --git a/arch/arm64/crypto/sm4-ce-gcm-glue.c b/arch/arm64/crypto/sm4-ce-gcm-glue.c index 2e27d7752d4f..c2ea3d5f690b 100644 --- a/arch/arm64/crypto/sm4-ce-gcm-glue.c +++ b/arch/arm64/crypto/sm4-ce-gcm-glue.c @@ -83,10 +83,10 @@ static void gcm_calculate_auth_mac(struct aead_request *req, u8 ghash[]) do { unsigned int n, orig_n; - const u8 *p, *orig_p; + const u8 *p; - orig_p = scatterwalk_next(&walk, assoclen, &orig_n); - p = orig_p; + orig_n = scatterwalk_next(&walk, assoclen); + p = walk.addr; n = orig_n; if (n + buflen < GHASH_BLOCK_SIZE) { @@ -118,7 +118,7 @@ static void gcm_calculate_auth_mac(struct aead_request *req, u8 ghash[]) memcpy(&buffer[0], p, buflen); } - scatterwalk_done_src(&walk, orig_p, orig_n); + scatterwalk_done_src(&walk, orig_n); assoclen -= orig_n; } while (assoclen); diff --git a/arch/s390/crypto/aes_s390.c b/arch/s390/crypto/aes_s390.c index 7fd303df05ab..ed85bd6e298f 100644 --- a/arch/s390/crypto/aes_s390.c +++ b/arch/s390/crypto/aes_s390.c @@ -66,7 +66,6 @@ struct s390_xts_ctx { struct gcm_sg_walk { struct scatter_walk walk; unsigned int walk_bytes; - u8 *walk_ptr; unsigned int walk_bytes_remain; u8 buf[AES_BLOCK_SIZE]; unsigned int buf_bytes; @@ -789,8 +788,7 @@ static inline unsigned int _gcm_sg_clamp_and_map(struct gcm_sg_walk *gw) { if (gw->walk_bytes_remain == 0) return 0; - gw->walk_ptr = scatterwalk_next(&gw->walk, gw->walk_bytes_remain, - &gw->walk_bytes); + gw->walk_bytes = scatterwalk_next(&gw->walk, gw->walk_bytes_remain); return gw->walk_bytes; } @@ -799,10 +797,9 @@ static inline void _gcm_sg_unmap_and_advance(struct gcm_sg_walk *gw, { gw->walk_bytes_remain -= nbytes; if (out) - scatterwalk_done_dst(&gw->walk, gw->walk_ptr, nbytes); + scatterwalk_done_dst(&gw->walk, nbytes); else - scatterwalk_done_src(&gw->walk, gw->walk_ptr, nbytes); - gw->walk_ptr = NULL; + scatterwalk_done_src(&gw->walk, nbytes); } static int gcm_in_walk_go(struct gcm_sg_walk *gw, unsigned int minbytesneeded) @@ -828,14 +825,14 @@ static int gcm_in_walk_go(struct gcm_sg_walk *gw, unsigned int minbytesneeded) } if (!gw->buf_bytes && gw->walk_bytes >= minbytesneeded) { - gw->ptr = gw->walk_ptr; + gw->ptr = gw->walk.addr; gw->nbytes = gw->walk_bytes; goto out; } while (1) { n = min(gw->walk_bytes, AES_BLOCK_SIZE - gw->buf_bytes); - memcpy(gw->buf + gw->buf_bytes, gw->walk_ptr, n); + memcpy(gw->buf + gw->buf_bytes, gw->walk.addr, n); gw->buf_bytes += n; _gcm_sg_unmap_and_advance(gw, n, false); if (gw->buf_bytes >= minbytesneeded) { @@ -869,13 +866,13 @@ static int gcm_out_walk_go(struct gcm_sg_walk *gw, unsigned int minbytesneeded) } if (gw->walk_bytes >= minbytesneeded) { - gw->ptr = gw->walk_ptr; + gw->ptr = gw->walk.addr; gw->nbytes = gw->walk_bytes; goto out; } - scatterwalk_unmap(gw->walk_ptr); - gw->walk_ptr = NULL; + /* XXX */ + scatterwalk_unmap(gw->walk.addr); gw->ptr = gw->buf; gw->nbytes = sizeof(gw->buf); @@ -914,7 +911,7 @@ static int gcm_out_walk_done(struct gcm_sg_walk *gw, unsigned int bytesdone) if (!_gcm_sg_clamp_and_map(gw)) return i; n = min(gw->walk_bytes, bytesdone - i); - memcpy(gw->walk_ptr, gw->buf + i, n); + memcpy(gw->walk.addr, gw->buf + i, n); _gcm_sg_unmap_and_advance(gw, n, true); } } else diff --git a/arch/x86/crypto/aegis128-aesni-glue.c b/arch/x86/crypto/aegis128-aesni-glue.c index 1bd093d073ed..26786e15abac 100644 --- a/arch/x86/crypto/aegis128-aesni-glue.c +++ b/arch/x86/crypto/aegis128-aesni-glue.c @@ -71,10 +71,9 @@ static void crypto_aegis128_aesni_process_ad( scatterwalk_start(&walk, sg_src); while (assoclen != 0) { - unsigned int size; - const u8 *mapped = scatterwalk_next(&walk, assoclen, &size); + unsigned int size = scatterwalk_next(&walk, assoclen); + const u8 *src = walk.addr; unsigned int left = size; - const u8 *src = mapped; if (pos + size >= AEGIS128_BLOCK_SIZE) { if (pos > 0) { @@ -97,7 +96,7 @@ static void crypto_aegis128_aesni_process_ad( pos += left; assoclen -= size; - scatterwalk_done_src(&walk, mapped, size); + scatterwalk_done_src(&walk, size); } if (pos > 0) { diff --git a/arch/x86/crypto/aesni-intel_glue.c b/arch/x86/crypto/aesni-intel_glue.c index c4bd05688b55..e141b7995304 100644 --- a/arch/x86/crypto/aesni-intel_glue.c +++ b/arch/x86/crypto/aesni-intel_glue.c @@ -1306,12 +1306,11 @@ static void gcm_process_assoc(const struct aes_gcm_key *key, u8 ghash_acc[16], scatterwalk_start(&walk, sg_src); while (assoclen) { - unsigned int orig_len_this_step; - const u8 *orig_src = scatterwalk_next(&walk, assoclen, - &orig_len_this_step); + unsigned int orig_len_this_step = scatterwalk_next( + &walk, assoclen); unsigned int len_this_step = orig_len_this_step; unsigned int len; - const u8 *src = orig_src; + const u8 *src = walk.addr; if (unlikely(pos)) { len = min(len_this_step, 16 - pos); @@ -1335,7 +1334,7 @@ static void gcm_process_assoc(const struct aes_gcm_key *key, u8 ghash_acc[16], pos = len_this_step; } next: - scatterwalk_done_src(&walk, orig_src, orig_len_this_step); + scatterwalk_done_src(&walk, orig_len_this_step); if (need_resched()) { kernel_fpu_end(); kernel_fpu_begin(); diff --git a/crypto/aegis128-core.c b/crypto/aegis128-core.c index 15d64d836356..72f6ee1345ef 100644 --- a/crypto/aegis128-core.c +++ b/crypto/aegis128-core.c @@ -284,10 +284,9 @@ static void crypto_aegis128_process_ad(struct aegis_state *state, scatterwalk_start(&walk, sg_src); while (assoclen != 0) { - unsigned int size; - const u8 *mapped = scatterwalk_next(&walk, assoclen, &size); + unsigned int size = scatterwalk_next(&walk, assoclen); + const u8 *src = walk.addr; unsigned int left = size; - const u8 *src = mapped; if (pos + size >= AEGIS_BLOCK_SIZE) { if (pos > 0) { @@ -308,7 +307,7 @@ static void crypto_aegis128_process_ad(struct aegis_state *state, pos += left; assoclen -= size; - scatterwalk_done_src(&walk, mapped, size); + scatterwalk_done_src(&walk, size); } if (pos > 0) { diff --git a/crypto/scatterwalk.c b/crypto/scatterwalk.c index 87c080f565d4..20a28c6d94da 100644 --- a/crypto/scatterwalk.c +++ b/crypto/scatterwalk.c @@ -34,12 +34,11 @@ inline void memcpy_from_scatterwalk(void *buf, struct scatter_walk *walk, unsigned int nbytes) { do { - const void *src_addr; unsigned int to_copy; - src_addr = scatterwalk_next(walk, nbytes, &to_copy); - memcpy(buf, src_addr, to_copy); - scatterwalk_done_src(walk, src_addr, to_copy); + to_copy = scatterwalk_next(walk, nbytes); + memcpy(buf, walk->addr, to_copy); + scatterwalk_done_src(walk, to_copy); buf += to_copy; nbytes -= to_copy; } while (nbytes); @@ -50,12 +49,11 @@ inline void memcpy_to_scatterwalk(struct scatter_walk *walk, const void *buf, unsigned int nbytes) { do { - void *dst_addr; unsigned int to_copy; - dst_addr = scatterwalk_next(walk, nbytes, &to_copy); - memcpy(dst_addr, buf, to_copy); - scatterwalk_done_dst(walk, dst_addr, to_copy); + to_copy = scatterwalk_next(walk, nbytes); + memcpy(walk->addr, buf, to_copy); + scatterwalk_done_dst(walk, to_copy); buf += to_copy; nbytes -= to_copy; } while (nbytes); diff --git a/crypto/skcipher.c b/crypto/skcipher.c index 66d19c360dd8..0c6911154241 100644 --- a/crypto/skcipher.c +++ b/crypto/skcipher.c @@ -41,12 +41,16 @@ static int skcipher_walk_next(struct skcipher_walk *walk); static inline void skcipher_map_src(struct skcipher_walk *walk) { - walk->src.virt.addr = scatterwalk_map(&walk->in); + /* XXX */ + walk->in.__addr = scatterwalk_map(&walk->in); + walk->src.virt.addr = walk->in.addr; } static inline void skcipher_map_dst(struct skcipher_walk *walk) { - walk->dst.virt.addr = scatterwalk_map(&walk->out); + /* XXX */ + walk->out.__addr = scatterwalk_map(&walk->out); + walk->dst.virt.addr = walk->out.addr; } static inline gfp_t skcipher_walk_gfp(struct skcipher_walk *walk) @@ -120,7 +124,7 @@ int skcipher_walk_done(struct skcipher_walk *walk, int res) goto dst_done; } - scatterwalk_done_dst(&walk->out, walk->dst.virt.addr, n); + scatterwalk_done_dst(&walk->out, n); dst_done: if (res > 0) diff --git a/drivers/crypto/nx/nx.c b/drivers/crypto/nx/nx.c index dd95e5361d88..a3b979193d9b 100644 --- a/drivers/crypto/nx/nx.c +++ b/drivers/crypto/nx/nx.c @@ -154,17 +154,16 @@ struct nx_sg *nx_walk_and_build(struct nx_sg *nx_dst, struct scatter_walk walk; struct nx_sg *nx_sg = nx_dst; unsigned int n, len = *src_len; - char *dst; /* we need to fast forward through @start bytes first */ scatterwalk_start_at_pos(&walk, sg_src, start); while (len && (nx_sg - nx_dst) < sglen) { - dst = scatterwalk_next(&walk, len, &n); + n = scatterwalk_next(&walk, len); - nx_sg = nx_build_sg_list(nx_sg, dst, &n, sglen - (nx_sg - nx_dst)); + nx_sg = nx_build_sg_list(nx_sg, walk.addr, &n, sglen - (nx_sg - nx_dst)); - scatterwalk_done_src(&walk, dst, n); + scatterwalk_done_src(&walk, n); len -= n; } /* update to_process */ diff --git a/include/crypto/algapi.h b/include/crypto/algapi.h index 94989b2e1350..f92e22686a68 100644 --- a/include/crypto/algapi.h +++ b/include/crypto/algapi.h @@ -54,6 +54,7 @@ struct rtattr; struct scatterlist; struct seq_file; struct sk_buff; +union crypto_no_such_thing; struct crypto_instance { struct crypto_alg alg; @@ -108,6 +109,12 @@ struct crypto_queue { struct scatter_walk { struct scatterlist *sg; unsigned int offset; + union { + void *const addr; + + /* Private API field, do not touch. */ + union crypto_no_such_thing *__addr; + }; }; struct crypto_attr_alg { diff --git a/include/crypto/scatterwalk.h b/include/crypto/scatterwalk.h index 3024adbdd443..8523c7591d95 100644 --- a/include/crypto/scatterwalk.h +++ b/include/crypto/scatterwalk.h @@ -120,18 +120,20 @@ static inline void *scatterwalk_map(struct scatter_walk *walk) * scatterwalk_next() - Get the next data buffer in a scatterlist walk * @walk: the scatter_walk * @total: the total number of bytes remaining, > 0 - * @nbytes_ret: (out) the next number of bytes available, <= @total * - * Return: A virtual address for the next segment of data from the scatterlist. - * The caller must call scatterwalk_done_src() or scatterwalk_done_dst() - * when it is done using this virtual address. + * A virtual address for the next segment of data from the scatterlist will + * be placed into @walk->addr. The caller must call scatterwalk_done_src() + * or scatterwalk_done_dst() when it is done using this virtual address. + * + * Returns: the next number of bytes available, <= @total */ -static inline void *scatterwalk_next(struct scatter_walk *walk, - unsigned int total, - unsigned int *nbytes_ret) +static inline unsigned int scatterwalk_next(struct scatter_walk *walk, + unsigned int total) { - *nbytes_ret = scatterwalk_clamp(walk, total); - return scatterwalk_map(walk); + unsigned int nbytes = scatterwalk_clamp(walk, total); + + walk->__addr = scatterwalk_map(walk); + return nbytes; } static inline void scatterwalk_unmap(const void *vaddr) @@ -149,32 +151,31 @@ static inline void scatterwalk_advance(struct scatter_walk *walk, /** * scatterwalk_done_src() - Finish one step of a walk of source scatterlist * @walk: the scatter_walk - * @vaddr: the address returned by scatterwalk_next() * @nbytes: the number of bytes processed this step, less than or equal to the * number of bytes that scatterwalk_next() returned. * - * Use this if the @vaddr was not written to, i.e. it is source data. + * Use this if the mapped address was not written to, i.e. it is source data. */ static inline void scatterwalk_done_src(struct scatter_walk *walk, - const void *vaddr, unsigned int nbytes) + unsigned int nbytes) { - scatterwalk_unmap(vaddr); + scatterwalk_unmap(walk->addr); scatterwalk_advance(walk, nbytes); } /** * scatterwalk_done_dst() - Finish one step of a walk of destination scatterlist * @walk: the scatter_walk - * @vaddr: the address returned by scatterwalk_next() * @nbytes: the number of bytes processed this step, less than or equal to the * number of bytes that scatterwalk_next() returned. * - * Use this if the @vaddr may have been written to, i.e. it is destination data. + * Use this if the mapped address may have been written to, i.e. it is + * destination data. */ static inline void scatterwalk_done_dst(struct scatter_walk *walk, - void *vaddr, unsigned int nbytes) + unsigned int nbytes) { - scatterwalk_unmap(vaddr); + scatterwalk_unmap(walk->addr); /* * Explicitly check ARCH_IMPLEMENTS_FLUSH_DCACHE_PAGE instead of just * relying on flush_dcache_page() being a no-op when not implemented, -- 2.51.0 From 131bdceca1f0a2d9381270dc40f898458e5e184b Mon Sep 17 00:00:00 2001 From: Herbert Xu Date: Sat, 8 Mar 2025 20:45:23 +0800 Subject: [PATCH 04/16] crypto: scatterwalk - Add memcpy_sglist Add memcpy_sglist which copies one SG list to another. Signed-off-by: Herbert Xu --- crypto/scatterwalk.c | 27 +++++++++++++++++++++++++++ include/crypto/scatterwalk.h | 3 +++ 2 files changed, 30 insertions(+) diff --git a/crypto/scatterwalk.c b/crypto/scatterwalk.c index 20a28c6d94da..8225801488d5 100644 --- a/crypto/scatterwalk.c +++ b/crypto/scatterwalk.c @@ -86,6 +86,33 @@ void memcpy_to_sglist(struct scatterlist *sg, unsigned int start, } EXPORT_SYMBOL_GPL(memcpy_to_sglist); +void memcpy_sglist(struct scatterlist *dst, struct scatterlist *src, + unsigned int nbytes) +{ + struct scatter_walk swalk; + struct scatter_walk dwalk; + + if (unlikely(nbytes == 0)) /* in case sg == NULL */ + return; + + scatterwalk_start(&swalk, src); + scatterwalk_start(&dwalk, dst); + + do { + unsigned int slen, dlen; + unsigned int len; + + slen = scatterwalk_next(&swalk, nbytes); + dlen = scatterwalk_next(&dwalk, nbytes); + len = min(slen, dlen); + memcpy(dwalk.addr, swalk.addr, len); + scatterwalk_done_dst(&dwalk, len); + scatterwalk_done_src(&swalk, len); + nbytes -= len; + } while (nbytes); +} +EXPORT_SYMBOL_GPL(memcpy_sglist); + struct scatterlist *scatterwalk_ffwd(struct scatterlist dst[2], struct scatterlist *src, unsigned int len) diff --git a/include/crypto/scatterwalk.h b/include/crypto/scatterwalk.h index 8523c7591d95..c62f47d04eb1 100644 --- a/include/crypto/scatterwalk.h +++ b/include/crypto/scatterwalk.h @@ -210,6 +210,9 @@ void memcpy_from_sglist(void *buf, struct scatterlist *sg, void memcpy_to_sglist(struct scatterlist *sg, unsigned int start, const void *buf, unsigned int nbytes); +void memcpy_sglist(struct scatterlist *dst, struct scatterlist *src, + unsigned int nbytes); + /* In new code, please use memcpy_{from,to}_sglist() directly instead. */ static inline void scatterwalk_map_and_copy(void *buf, struct scatterlist *sg, unsigned int start, -- 2.51.0 From db873be6f0549597f92c72986b1939643a7f9a75 Mon Sep 17 00:00:00 2001 From: Herbert Xu Date: Sat, 8 Mar 2025 20:45:25 +0800 Subject: [PATCH 05/16] crypto: skcipher - Eliminate duplicate virt.addr field Reuse the addr field from struct scatter_walk for skcipher_walk. Keep the existing virt.addr fields but make them const for the user to access the mapped address. Signed-off-by: Herbert Xu --- crypto/skcipher.c | 29 ++++++++++++----------------- include/crypto/algapi.h | 5 +++-- include/crypto/internal/skcipher.h | 26 +++++++++++++++++++++----- 3 files changed, 36 insertions(+), 24 deletions(-) diff --git a/crypto/skcipher.c b/crypto/skcipher.c index 0c6911154241..ab5d852febcd 100644 --- a/crypto/skcipher.c +++ b/crypto/skcipher.c @@ -43,14 +43,12 @@ static inline void skcipher_map_src(struct skcipher_walk *walk) { /* XXX */ walk->in.__addr = scatterwalk_map(&walk->in); - walk->src.virt.addr = walk->in.addr; } static inline void skcipher_map_dst(struct skcipher_walk *walk) { /* XXX */ walk->out.__addr = scatterwalk_map(&walk->out); - walk->dst.virt.addr = walk->out.addr; } static inline gfp_t skcipher_walk_gfp(struct skcipher_walk *walk) @@ -100,8 +98,7 @@ int skcipher_walk_done(struct skcipher_walk *walk, int res) SKCIPHER_WALK_DIFF)))) { scatterwalk_advance(&walk->in, n); } else if (walk->flags & SKCIPHER_WALK_DIFF) { - scatterwalk_unmap(walk->src.virt.addr); - scatterwalk_advance(&walk->in, n); + scatterwalk_done_src(&walk->in, n); } else if (walk->flags & SKCIPHER_WALK_COPY) { scatterwalk_advance(&walk->in, n); skcipher_map_dst(walk); @@ -116,11 +113,8 @@ int skcipher_walk_done(struct skcipher_walk *walk, int res) */ res = -EINVAL; total = 0; - } else { - u8 *buf = PTR_ALIGN(walk->buffer, walk->alignmask + 1); - - memcpy_to_scatterwalk(&walk->out, buf, n); - } + } else + memcpy_to_scatterwalk(&walk->out, walk->out.addr, n); goto dst_done; } @@ -162,7 +156,7 @@ static int skcipher_next_slow(struct skcipher_walk *walk, unsigned int bsize) { unsigned alignmask = walk->alignmask; unsigned n; - u8 *buffer; + void *buffer; if (!walk->buffer) walk->buffer = walk->page; @@ -176,10 +170,11 @@ static int skcipher_next_slow(struct skcipher_walk *walk, unsigned int bsize) return skcipher_walk_done(walk, -ENOMEM); walk->buffer = buffer; } - walk->dst.virt.addr = PTR_ALIGN(buffer, alignmask + 1); - walk->src.virt.addr = walk->dst.virt.addr; - memcpy_from_scatterwalk(walk->src.virt.addr, &walk->in, bsize); + buffer = PTR_ALIGN(buffer, alignmask + 1); + memcpy_from_scatterwalk(buffer, &walk->in, bsize); + walk->out.__addr = buffer; + walk->in.__addr = walk->out.addr; walk->nbytes = bsize; walk->flags |= SKCIPHER_WALK_SLOW; @@ -189,7 +184,7 @@ static int skcipher_next_slow(struct skcipher_walk *walk, unsigned int bsize) static int skcipher_next_copy(struct skcipher_walk *walk) { - u8 *tmp = walk->page; + void *tmp = walk->page; skcipher_map_src(walk); memcpy(tmp, walk->src.virt.addr, walk->nbytes); @@ -199,8 +194,8 @@ static int skcipher_next_copy(struct skcipher_walk *walk) * processed (which might be less than walk->nbytes) is known. */ - walk->src.virt.addr = tmp; - walk->dst.virt.addr = tmp; + walk->in.__addr = tmp; + walk->out.__addr = tmp; return 0; } @@ -214,7 +209,7 @@ static int skcipher_next_fast(struct skcipher_walk *walk) (u8 *)(sg_page(walk->out.sg) + (walk->out.offset >> PAGE_SHIFT)); skcipher_map_dst(walk); - walk->src.virt.addr = walk->dst.virt.addr; + walk->in.__addr = walk->dst.virt.addr; if (diff) { walk->flags |= SKCIPHER_WALK_DIFF; diff --git a/include/crypto/algapi.h b/include/crypto/algapi.h index f92e22686a68..6e07bbc04089 100644 --- a/include/crypto/algapi.h +++ b/include/crypto/algapi.h @@ -107,14 +107,15 @@ struct crypto_queue { }; struct scatter_walk { - struct scatterlist *sg; - unsigned int offset; + /* Must be the first member, see struct skcipher_walk. */ union { void *const addr; /* Private API field, do not touch. */ union crypto_no_such_thing *__addr; }; + struct scatterlist *sg; + unsigned int offset; }; struct crypto_attr_alg { diff --git a/include/crypto/internal/skcipher.h b/include/crypto/internal/skcipher.h index d6ae7a86fed2..c705124432c5 100644 --- a/include/crypto/internal/skcipher.h +++ b/include/crypto/internal/skcipher.h @@ -56,15 +56,31 @@ struct crypto_lskcipher_spawn { struct skcipher_walk { union { + /* Virtual address of the source. */ struct { - void *addr; - } virt; - } src, dst; + struct { + void *const addr; + } virt; + } src; + + /* Private field for the API, do not use. */ + struct scatter_walk in; + }; - struct scatter_walk in; unsigned int nbytes; - struct scatter_walk out; + union { + /* Virtual address of the destination. */ + struct { + struct { + void *const addr; + } virt; + } dst; + + /* Private field for the API, do not use. */ + struct scatter_walk out; + }; + unsigned int total; u8 *page; -- 2.51.0 From 37d451809f572ec197d3c18d9638c8715274255f Mon Sep 17 00:00:00 2001 From: Herbert Xu Date: Sat, 8 Mar 2025 20:53:13 +0800 Subject: [PATCH 06/16] crypto: skcipher - Make skcipher_walk src.virt.addr const Mark the src.virt.addr field in struct skcipher_walk as a pointer to const data. This guarantees that the user won't modify the data which should be done through dst.virt.addr to ensure that flushing is done when necessary. Signed-off-by: Herbert Xu --- arch/arm/crypto/aes-ce-glue.c | 2 +- arch/arm64/crypto/aes-neonbs-glue.c | 3 ++- arch/powerpc/crypto/aes-gcm-p10-glue.c | 6 +++--- arch/powerpc/crypto/aes_ctr.c | 2 +- arch/sparc/crypto/aes_glue.c | 2 +- arch/x86/crypto/des3_ede_glue.c | 2 +- crypto/ctr.c | 10 ++++----- crypto/lrw.c | 2 +- crypto/pcbc.c | 28 +++++++++++++------------- crypto/xctr.c | 2 +- crypto/xts.c | 2 +- include/crypto/ctr.h | 2 +- include/crypto/internal/skcipher.h | 2 +- 13 files changed, 33 insertions(+), 32 deletions(-) diff --git a/arch/arm/crypto/aes-ce-glue.c b/arch/arm/crypto/aes-ce-glue.c index 21df5e7f51f9..1cf61f51e766 100644 --- a/arch/arm/crypto/aes-ce-glue.c +++ b/arch/arm/crypto/aes-ce-glue.c @@ -399,9 +399,9 @@ static int ctr_encrypt(struct skcipher_request *req) } if (walk.nbytes) { u8 __aligned(8) tail[AES_BLOCK_SIZE]; + const u8 *tsrc = walk.src.virt.addr; unsigned int nbytes = walk.nbytes; u8 *tdst = walk.dst.virt.addr; - u8 *tsrc = walk.src.virt.addr; /* * Tell aes_ctr_encrypt() to process a tail block. diff --git a/arch/arm64/crypto/aes-neonbs-glue.c b/arch/arm64/crypto/aes-neonbs-glue.c index 46425e7b9755..c4a623e86593 100644 --- a/arch/arm64/crypto/aes-neonbs-glue.c +++ b/arch/arm64/crypto/aes-neonbs-glue.c @@ -287,7 +287,8 @@ static int __xts_crypt(struct skcipher_request *req, bool encrypt, struct skcipher_walk walk; int nbytes, err; int first = 1; - u8 *out, *in; + const u8 *in; + u8 *out; if (req->cryptlen < AES_BLOCK_SIZE) return -EINVAL; diff --git a/arch/powerpc/crypto/aes-gcm-p10-glue.c b/arch/powerpc/crypto/aes-gcm-p10-glue.c index 679f52794baf..85f4fd4b1bdc 100644 --- a/arch/powerpc/crypto/aes-gcm-p10-glue.c +++ b/arch/powerpc/crypto/aes-gcm-p10-glue.c @@ -35,9 +35,9 @@ MODULE_ALIAS_CRYPTO("aes"); asmlinkage int aes_p10_set_encrypt_key(const u8 *userKey, const int bits, void *key); asmlinkage void aes_p10_encrypt(const u8 *in, u8 *out, const void *key); -asmlinkage void aes_p10_gcm_encrypt(u8 *in, u8 *out, size_t len, +asmlinkage void aes_p10_gcm_encrypt(const u8 *in, u8 *out, size_t len, void *rkey, u8 *iv, void *Xi); -asmlinkage void aes_p10_gcm_decrypt(u8 *in, u8 *out, size_t len, +asmlinkage void aes_p10_gcm_decrypt(const u8 *in, u8 *out, size_t len, void *rkey, u8 *iv, void *Xi); asmlinkage void gcm_init_htable(unsigned char htable[], unsigned char Xi[]); asmlinkage void gcm_ghash_p10(unsigned char *Xi, unsigned char *Htable, @@ -261,7 +261,7 @@ static int p10_aes_gcm_crypt(struct aead_request *req, u8 *riv, return ret; while ((nbytes = walk.nbytes) > 0 && ret == 0) { - u8 *src = walk.src.virt.addr; + const u8 *src = walk.src.virt.addr; u8 *dst = walk.dst.virt.addr; u8 buf[AES_BLOCK_SIZE]; diff --git a/arch/powerpc/crypto/aes_ctr.c b/arch/powerpc/crypto/aes_ctr.c index 9a3da8cd62f3..3da75f42529a 100644 --- a/arch/powerpc/crypto/aes_ctr.c +++ b/arch/powerpc/crypto/aes_ctr.c @@ -69,9 +69,9 @@ static int p8_aes_ctr_setkey(struct crypto_skcipher *tfm, const u8 *key, static void p8_aes_ctr_final(const struct p8_aes_ctr_ctx *ctx, struct skcipher_walk *walk) { + const u8 *src = walk->src.virt.addr; u8 *ctrblk = walk->iv; u8 keystream[AES_BLOCK_SIZE]; - u8 *src = walk->src.virt.addr; u8 *dst = walk->dst.virt.addr; unsigned int nbytes = walk->nbytes; diff --git a/arch/sparc/crypto/aes_glue.c b/arch/sparc/crypto/aes_glue.c index e3d2138ff9e2..683150830356 100644 --- a/arch/sparc/crypto/aes_glue.c +++ b/arch/sparc/crypto/aes_glue.c @@ -321,7 +321,7 @@ static void ctr_crypt_final(const struct crypto_sparc64_aes_ctx *ctx, { u8 *ctrblk = walk->iv; u64 keystream[AES_BLOCK_SIZE / sizeof(u64)]; - u8 *src = walk->src.virt.addr; + const u8 *src = walk->src.virt.addr; u8 *dst = walk->dst.virt.addr; unsigned int nbytes = walk->nbytes; diff --git a/arch/x86/crypto/des3_ede_glue.c b/arch/x86/crypto/des3_ede_glue.c index e88439d3828e..34600f90d8a6 100644 --- a/arch/x86/crypto/des3_ede_glue.c +++ b/arch/x86/crypto/des3_ede_glue.c @@ -73,7 +73,7 @@ static int ecb_crypt(struct skcipher_request *req, const u32 *expkey) err = skcipher_walk_virt(&walk, req, false); while ((nbytes = walk.nbytes)) { - u8 *wsrc = walk.src.virt.addr; + const u8 *wsrc = walk.src.virt.addr; u8 *wdst = walk.dst.virt.addr; /* Process four block batch */ diff --git a/crypto/ctr.c b/crypto/ctr.c index 73c0d6e53b2f..97a947b0a876 100644 --- a/crypto/ctr.c +++ b/crypto/ctr.c @@ -33,7 +33,7 @@ static void crypto_ctr_crypt_final(struct skcipher_walk *walk, u8 *ctrblk = walk->iv; u8 tmp[MAX_CIPHER_BLOCKSIZE + MAX_CIPHER_ALIGNMASK]; u8 *keystream = PTR_ALIGN(tmp + 0, alignmask + 1); - u8 *src = walk->src.virt.addr; + const u8 *src = walk->src.virt.addr; u8 *dst = walk->dst.virt.addr; unsigned int nbytes = walk->nbytes; @@ -50,7 +50,7 @@ static int crypto_ctr_crypt_segment(struct skcipher_walk *walk, crypto_cipher_alg(tfm)->cia_encrypt; unsigned int bsize = crypto_cipher_blocksize(tfm); u8 *ctrblk = walk->iv; - u8 *src = walk->src.virt.addr; + const u8 *src = walk->src.virt.addr; u8 *dst = walk->dst.virt.addr; unsigned int nbytes = walk->nbytes; @@ -77,20 +77,20 @@ static int crypto_ctr_crypt_inplace(struct skcipher_walk *walk, unsigned int bsize = crypto_cipher_blocksize(tfm); unsigned long alignmask = crypto_cipher_alignmask(tfm); unsigned int nbytes = walk->nbytes; + u8 *dst = walk->dst.virt.addr; u8 *ctrblk = walk->iv; - u8 *src = walk->src.virt.addr; u8 tmp[MAX_CIPHER_BLOCKSIZE + MAX_CIPHER_ALIGNMASK]; u8 *keystream = PTR_ALIGN(tmp + 0, alignmask + 1); do { /* create keystream */ fn(crypto_cipher_tfm(tfm), keystream, ctrblk); - crypto_xor(src, keystream, bsize); + crypto_xor(dst, keystream, bsize); /* increment counter in counterblock */ crypto_inc(ctrblk, bsize); - src += bsize; + dst += bsize; } while ((nbytes -= bsize) >= bsize); return nbytes; diff --git a/crypto/lrw.c b/crypto/lrw.c index e216fbf2b786..391ae0f7641f 100644 --- a/crypto/lrw.c +++ b/crypto/lrw.c @@ -167,7 +167,7 @@ static int lrw_xor_tweak(struct skcipher_request *req, bool second_pass) while (w.nbytes) { unsigned int avail = w.nbytes; - be128 *wsrc; + const be128 *wsrc; be128 *wdst; wsrc = w.src.virt.addr; diff --git a/crypto/pcbc.c b/crypto/pcbc.c index cbfb3ac14b3a..9d2e56d6744a 100644 --- a/crypto/pcbc.c +++ b/crypto/pcbc.c @@ -22,8 +22,8 @@ static int crypto_pcbc_encrypt_segment(struct skcipher_request *req, struct crypto_cipher *tfm) { int bsize = crypto_cipher_blocksize(tfm); + const u8 *src = walk->src.virt.addr; unsigned int nbytes = walk->nbytes; - u8 *src = walk->src.virt.addr; u8 *dst = walk->dst.virt.addr; u8 * const iv = walk->iv; @@ -45,17 +45,17 @@ static int crypto_pcbc_encrypt_inplace(struct skcipher_request *req, { int bsize = crypto_cipher_blocksize(tfm); unsigned int nbytes = walk->nbytes; - u8 *src = walk->src.virt.addr; + u8 *dst = walk->dst.virt.addr; u8 * const iv = walk->iv; u8 tmpbuf[MAX_CIPHER_BLOCKSIZE]; do { - memcpy(tmpbuf, src, bsize); - crypto_xor(iv, src, bsize); - crypto_cipher_encrypt_one(tfm, src, iv); - crypto_xor_cpy(iv, tmpbuf, src, bsize); + memcpy(tmpbuf, dst, bsize); + crypto_xor(iv, dst, bsize); + crypto_cipher_encrypt_one(tfm, dst, iv); + crypto_xor_cpy(iv, tmpbuf, dst, bsize); - src += bsize; + dst += bsize; } while ((nbytes -= bsize) >= bsize); return nbytes; @@ -89,8 +89,8 @@ static int crypto_pcbc_decrypt_segment(struct skcipher_request *req, struct crypto_cipher *tfm) { int bsize = crypto_cipher_blocksize(tfm); + const u8 *src = walk->src.virt.addr; unsigned int nbytes = walk->nbytes; - u8 *src = walk->src.virt.addr; u8 *dst = walk->dst.virt.addr; u8 * const iv = walk->iv; @@ -112,17 +112,17 @@ static int crypto_pcbc_decrypt_inplace(struct skcipher_request *req, { int bsize = crypto_cipher_blocksize(tfm); unsigned int nbytes = walk->nbytes; - u8 *src = walk->src.virt.addr; + u8 *dst = walk->dst.virt.addr; u8 * const iv = walk->iv; u8 tmpbuf[MAX_CIPHER_BLOCKSIZE] __aligned(__alignof__(u32)); do { - memcpy(tmpbuf, src, bsize); - crypto_cipher_decrypt_one(tfm, src, src); - crypto_xor(src, iv, bsize); - crypto_xor_cpy(iv, src, tmpbuf, bsize); + memcpy(tmpbuf, dst, bsize); + crypto_cipher_decrypt_one(tfm, dst, dst); + crypto_xor(dst, iv, bsize); + crypto_xor_cpy(iv, dst, tmpbuf, bsize); - src += bsize; + dst += bsize; } while ((nbytes -= bsize) >= bsize); return nbytes; diff --git a/crypto/xctr.c b/crypto/xctr.c index 6ed9c85ededa..9c536ab6d2e5 100644 --- a/crypto/xctr.c +++ b/crypto/xctr.c @@ -78,7 +78,7 @@ static int crypto_xctr_crypt_inplace(struct skcipher_walk *walk, crypto_cipher_alg(tfm)->cia_encrypt; unsigned long alignmask = crypto_cipher_alignmask(tfm); unsigned int nbytes = walk->nbytes; - u8 *data = walk->src.virt.addr; + u8 *data = walk->dst.virt.addr; u8 tmp[XCTR_BLOCKSIZE + MAX_CIPHER_ALIGNMASK]; u8 *keystream = PTR_ALIGN(tmp + 0, alignmask + 1); __le32 ctr32 = cpu_to_le32(byte_ctr / XCTR_BLOCKSIZE + 1); diff --git a/crypto/xts.c b/crypto/xts.c index 821060ede2cf..31529c9ef08f 100644 --- a/crypto/xts.c +++ b/crypto/xts.c @@ -99,7 +99,7 @@ static int xts_xor_tweak(struct skcipher_request *req, bool second_pass, while (w.nbytes) { unsigned int avail = w.nbytes; - le128 *wsrc; + const le128 *wsrc; le128 *wdst; wsrc = w.src.virt.addr; diff --git a/include/crypto/ctr.h b/include/crypto/ctr.h index a1c66d1001af..da1ee73e9ce9 100644 --- a/include/crypto/ctr.h +++ b/include/crypto/ctr.h @@ -34,8 +34,8 @@ static inline int crypto_ctr_encrypt_walk(struct skcipher_request *req, err = skcipher_walk_virt(&walk, req, false); while (walk.nbytes > 0) { + const u8 *src = walk.src.virt.addr; u8 *dst = walk.dst.virt.addr; - u8 *src = walk.src.virt.addr; int nbytes = walk.nbytes; int tail = 0; diff --git a/include/crypto/internal/skcipher.h b/include/crypto/internal/skcipher.h index c705124432c5..a958ab0636ad 100644 --- a/include/crypto/internal/skcipher.h +++ b/include/crypto/internal/skcipher.h @@ -59,7 +59,7 @@ struct skcipher_walk { /* Virtual address of the source. */ struct { struct { - void *const addr; + const void *const addr; } virt; } src; -- 2.51.0 From 01894c8488d84138bd79311200ee5d9dd088b9d7 Mon Sep 17 00:00:00 2001 From: Ethan Carter Edwards Date: Sat, 8 Mar 2025 19:30:52 -0500 Subject: [PATCH 07/16] crypto: artpec6 - change from kzalloc to kcalloc in artpec6_crypto_probe() We are trying to get rid of all multiplications from allocation functions to prevent potential integer overflows. Here the multiplication is probably safe, but using kcalloc() is more appropriate and improves readability. This patch has no effect on runtime behavior. Link: https://github.com/KSPP/linux/issues/162 [1] Link: https://www.kernel.org/doc/html/next/process/deprecated.html#open-coded-arithmetic-in-allocator-arguments Signed-off-by: Ethan Carter Edwards Acked-by: Jesper Nilsson Signed-off-by: Herbert Xu --- drivers/crypto/axis/artpec6_crypto.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/drivers/crypto/axis/artpec6_crypto.c b/drivers/crypto/axis/artpec6_crypto.c index 1c1f57baef0e..500b08e42282 100644 --- a/drivers/crypto/axis/artpec6_crypto.c +++ b/drivers/crypto/axis/artpec6_crypto.c @@ -2897,13 +2897,13 @@ static int artpec6_crypto_probe(struct platform_device *pdev) tasklet_init(&ac->task, artpec6_crypto_task, (unsigned long)ac); - ac->pad_buffer = devm_kzalloc(&pdev->dev, 2 * ARTPEC_CACHE_LINE_MAX, + ac->pad_buffer = devm_kcalloc(&pdev->dev, 2, ARTPEC_CACHE_LINE_MAX, GFP_KERNEL); if (!ac->pad_buffer) return -ENOMEM; ac->pad_buffer = PTR_ALIGN(ac->pad_buffer, ARTPEC_CACHE_LINE_MAX); - ac->zero_buffer = devm_kzalloc(&pdev->dev, 2 * ARTPEC_CACHE_LINE_MAX, + ac->zero_buffer = devm_kcalloc(&pdev->dev, 2, ARTPEC_CACHE_LINE_MAX, GFP_KERNEL); if (!ac->zero_buffer) return -ENOMEM; -- 2.51.0 From 3d6979bf3bd51f47e889327e10e4653d55168c21 Mon Sep 17 00:00:00 2001 From: Herbert Xu Date: Sun, 9 Mar 2025 10:43:12 +0800 Subject: [PATCH 08/16] crypto: api - Add cra_type->destroy hook Add a cra_type->destroy hook so that resources can be freed after the last user of a registered algorithm is gone. Signed-off-by: Herbert Xu --- crypto/api.c | 10 ++++++++++ crypto/internal.h | 6 ++++-- 2 files changed, 14 insertions(+), 2 deletions(-) diff --git a/crypto/api.c b/crypto/api.c index c2c4eb14ef95..91957bb52f3f 100644 --- a/crypto/api.c +++ b/crypto/api.c @@ -707,5 +707,15 @@ void crypto_req_done(void *data, int err) } EXPORT_SYMBOL_GPL(crypto_req_done); +void crypto_destroy_alg(struct crypto_alg *alg) +{ + if (alg->cra_type && alg->cra_type->destroy) + alg->cra_type->destroy(alg); + + if (alg->cra_destroy) + alg->cra_destroy(alg); +} +EXPORT_SYMBOL_GPL(crypto_destroy_alg); + MODULE_DESCRIPTION("Cryptographic core API"); MODULE_LICENSE("GPL"); diff --git a/crypto/internal.h b/crypto/internal.h index 08d43b40e7db..11567ea24fc3 100644 --- a/crypto/internal.h +++ b/crypto/internal.h @@ -40,6 +40,7 @@ struct crypto_type { void (*show)(struct seq_file *m, struct crypto_alg *alg); int (*report)(struct sk_buff *skb, struct crypto_alg *alg); void (*free)(struct crypto_instance *inst); + void (*destroy)(struct crypto_alg *alg); unsigned int type; unsigned int maskclear; @@ -127,6 +128,7 @@ void *crypto_create_tfm_node(struct crypto_alg *alg, const struct crypto_type *frontend, int node); void *crypto_clone_tfm(const struct crypto_type *frontend, struct crypto_tfm *otfm); +void crypto_destroy_alg(struct crypto_alg *alg); static inline void *crypto_create_tfm(struct crypto_alg *alg, const struct crypto_type *frontend) @@ -163,8 +165,8 @@ static inline struct crypto_alg *crypto_alg_get(struct crypto_alg *alg) static inline void crypto_alg_put(struct crypto_alg *alg) { - if (refcount_dec_and_test(&alg->cra_refcnt) && alg->cra_destroy) - alg->cra_destroy(alg); + if (refcount_dec_and_test(&alg->cra_refcnt)) + crypto_destroy_alg(alg); } static inline int crypto_tmpl_get(struct crypto_template *tmpl) -- 2.51.0 From 0af7304c0696ec5b3589af6973b7f27e014c2903 Mon Sep 17 00:00:00 2001 From: Herbert Xu Date: Sun, 9 Mar 2025 10:43:14 +0800 Subject: [PATCH 09/16] crypto: scomp - Remove tfm argument from alloc/free_ctx The tfm argument is completely unused and meaningless as the same stream object is identical over all transforms of a given algorithm. Remove it. Signed-off-by: Herbert Xu --- crypto/842.c | 8 ++++---- crypto/deflate.c | 4 ++-- crypto/lz4.c | 8 ++++---- crypto/lz4hc.c | 8 ++++---- crypto/lzo-rle.c | 8 ++++---- crypto/lzo.c | 8 ++++---- crypto/zstd.c | 4 ++-- drivers/crypto/cavium/zip/zip_crypto.c | 6 +++--- drivers/crypto/cavium/zip/zip_crypto.h | 6 +++--- include/crypto/internal/scompress.h | 8 ++++---- 10 files changed, 34 insertions(+), 34 deletions(-) diff --git a/crypto/842.c b/crypto/842.c index e59e54d76960..2238478c3493 100644 --- a/crypto/842.c +++ b/crypto/842.c @@ -28,7 +28,7 @@ struct crypto842_ctx { void *wmem; /* working memory for compress */ }; -static void *crypto842_alloc_ctx(struct crypto_scomp *tfm) +static void *crypto842_alloc_ctx(void) { void *ctx; @@ -43,14 +43,14 @@ static int crypto842_init(struct crypto_tfm *tfm) { struct crypto842_ctx *ctx = crypto_tfm_ctx(tfm); - ctx->wmem = crypto842_alloc_ctx(NULL); + ctx->wmem = crypto842_alloc_ctx(); if (IS_ERR(ctx->wmem)) return -ENOMEM; return 0; } -static void crypto842_free_ctx(struct crypto_scomp *tfm, void *ctx) +static void crypto842_free_ctx(void *ctx) { kfree(ctx); } @@ -59,7 +59,7 @@ static void crypto842_exit(struct crypto_tfm *tfm) { struct crypto842_ctx *ctx = crypto_tfm_ctx(tfm); - crypto842_free_ctx(NULL, ctx->wmem); + crypto842_free_ctx(ctx->wmem); } static int crypto842_compress(struct crypto_tfm *tfm, diff --git a/crypto/deflate.c b/crypto/deflate.c index 98e8bcb81a6a..1bf7184ad670 100644 --- a/crypto/deflate.c +++ b/crypto/deflate.c @@ -112,7 +112,7 @@ out: return ret; } -static void *deflate_alloc_ctx(struct crypto_scomp *tfm) +static void *deflate_alloc_ctx(void) { struct deflate_ctx *ctx; int ret; @@ -143,7 +143,7 @@ static void __deflate_exit(void *ctx) deflate_decomp_exit(ctx); } -static void deflate_free_ctx(struct crypto_scomp *tfm, void *ctx) +static void deflate_free_ctx(void *ctx) { __deflate_exit(ctx); kfree_sensitive(ctx); diff --git a/crypto/lz4.c b/crypto/lz4.c index 0606f8862e78..e66c6d1ba34f 100644 --- a/crypto/lz4.c +++ b/crypto/lz4.c @@ -16,7 +16,7 @@ struct lz4_ctx { void *lz4_comp_mem; }; -static void *lz4_alloc_ctx(struct crypto_scomp *tfm) +static void *lz4_alloc_ctx(void) { void *ctx; @@ -31,14 +31,14 @@ static int lz4_init(struct crypto_tfm *tfm) { struct lz4_ctx *ctx = crypto_tfm_ctx(tfm); - ctx->lz4_comp_mem = lz4_alloc_ctx(NULL); + ctx->lz4_comp_mem = lz4_alloc_ctx(); if (IS_ERR(ctx->lz4_comp_mem)) return -ENOMEM; return 0; } -static void lz4_free_ctx(struct crypto_scomp *tfm, void *ctx) +static void lz4_free_ctx(void *ctx) { vfree(ctx); } @@ -47,7 +47,7 @@ static void lz4_exit(struct crypto_tfm *tfm) { struct lz4_ctx *ctx = crypto_tfm_ctx(tfm); - lz4_free_ctx(NULL, ctx->lz4_comp_mem); + lz4_free_ctx(ctx->lz4_comp_mem); } static int __lz4_compress_crypto(const u8 *src, unsigned int slen, diff --git a/crypto/lz4hc.c b/crypto/lz4hc.c index d7cc94aa2fcf..25a95b65aca5 100644 --- a/crypto/lz4hc.c +++ b/crypto/lz4hc.c @@ -15,7 +15,7 @@ struct lz4hc_ctx { void *lz4hc_comp_mem; }; -static void *lz4hc_alloc_ctx(struct crypto_scomp *tfm) +static void *lz4hc_alloc_ctx(void) { void *ctx; @@ -30,14 +30,14 @@ static int lz4hc_init(struct crypto_tfm *tfm) { struct lz4hc_ctx *ctx = crypto_tfm_ctx(tfm); - ctx->lz4hc_comp_mem = lz4hc_alloc_ctx(NULL); + ctx->lz4hc_comp_mem = lz4hc_alloc_ctx(); if (IS_ERR(ctx->lz4hc_comp_mem)) return -ENOMEM; return 0; } -static void lz4hc_free_ctx(struct crypto_scomp *tfm, void *ctx) +static void lz4hc_free_ctx(void *ctx) { vfree(ctx); } @@ -46,7 +46,7 @@ static void lz4hc_exit(struct crypto_tfm *tfm) { struct lz4hc_ctx *ctx = crypto_tfm_ctx(tfm); - lz4hc_free_ctx(NULL, ctx->lz4hc_comp_mem); + lz4hc_free_ctx(ctx->lz4hc_comp_mem); } static int __lz4hc_compress_crypto(const u8 *src, unsigned int slen, diff --git a/crypto/lzo-rle.c b/crypto/lzo-rle.c index 0abc2d87f042..6c845e7d32f5 100644 --- a/crypto/lzo-rle.c +++ b/crypto/lzo-rle.c @@ -15,7 +15,7 @@ struct lzorle_ctx { void *lzorle_comp_mem; }; -static void *lzorle_alloc_ctx(struct crypto_scomp *tfm) +static void *lzorle_alloc_ctx(void) { void *ctx; @@ -30,14 +30,14 @@ static int lzorle_init(struct crypto_tfm *tfm) { struct lzorle_ctx *ctx = crypto_tfm_ctx(tfm); - ctx->lzorle_comp_mem = lzorle_alloc_ctx(NULL); + ctx->lzorle_comp_mem = lzorle_alloc_ctx(); if (IS_ERR(ctx->lzorle_comp_mem)) return -ENOMEM; return 0; } -static void lzorle_free_ctx(struct crypto_scomp *tfm, void *ctx) +static void lzorle_free_ctx(void *ctx) { kvfree(ctx); } @@ -46,7 +46,7 @@ static void lzorle_exit(struct crypto_tfm *tfm) { struct lzorle_ctx *ctx = crypto_tfm_ctx(tfm); - lzorle_free_ctx(NULL, ctx->lzorle_comp_mem); + lzorle_free_ctx(ctx->lzorle_comp_mem); } static int __lzorle_compress(const u8 *src, unsigned int slen, diff --git a/crypto/lzo.c b/crypto/lzo.c index 8338851c7406..035d62e2afe0 100644 --- a/crypto/lzo.c +++ b/crypto/lzo.c @@ -15,7 +15,7 @@ struct lzo_ctx { void *lzo_comp_mem; }; -static void *lzo_alloc_ctx(struct crypto_scomp *tfm) +static void *lzo_alloc_ctx(void) { void *ctx; @@ -30,14 +30,14 @@ static int lzo_init(struct crypto_tfm *tfm) { struct lzo_ctx *ctx = crypto_tfm_ctx(tfm); - ctx->lzo_comp_mem = lzo_alloc_ctx(NULL); + ctx->lzo_comp_mem = lzo_alloc_ctx(); if (IS_ERR(ctx->lzo_comp_mem)) return -ENOMEM; return 0; } -static void lzo_free_ctx(struct crypto_scomp *tfm, void *ctx) +static void lzo_free_ctx(void *ctx) { kvfree(ctx); } @@ -46,7 +46,7 @@ static void lzo_exit(struct crypto_tfm *tfm) { struct lzo_ctx *ctx = crypto_tfm_ctx(tfm); - lzo_free_ctx(NULL, ctx->lzo_comp_mem); + lzo_free_ctx(ctx->lzo_comp_mem); } static int __lzo_compress(const u8 *src, unsigned int slen, diff --git a/crypto/zstd.c b/crypto/zstd.c index 154a969c83a8..68a093427944 100644 --- a/crypto/zstd.c +++ b/crypto/zstd.c @@ -103,7 +103,7 @@ static int __zstd_init(void *ctx) return ret; } -static void *zstd_alloc_ctx(struct crypto_scomp *tfm) +static void *zstd_alloc_ctx(void) { int ret; struct zstd_ctx *ctx; @@ -134,7 +134,7 @@ static void __zstd_exit(void *ctx) zstd_decomp_exit(ctx); } -static void zstd_free_ctx(struct crypto_scomp *tfm, void *ctx) +static void zstd_free_ctx(void *ctx) { __zstd_exit(ctx); kfree_sensitive(ctx); diff --git a/drivers/crypto/cavium/zip/zip_crypto.c b/drivers/crypto/cavium/zip/zip_crypto.c index 1046a746d36f..a9c3efce8f2d 100644 --- a/drivers/crypto/cavium/zip/zip_crypto.c +++ b/drivers/crypto/cavium/zip/zip_crypto.c @@ -236,7 +236,7 @@ int zip_comp_decompress(struct crypto_tfm *tfm, } /* Legacy compress framework end */ /* SCOMP framework start */ -void *zip_alloc_scomp_ctx_deflate(struct crypto_scomp *tfm) +void *zip_alloc_scomp_ctx_deflate(void) { int ret; struct zip_kernel_ctx *zip_ctx; @@ -255,7 +255,7 @@ void *zip_alloc_scomp_ctx_deflate(struct crypto_scomp *tfm) return zip_ctx; } -void *zip_alloc_scomp_ctx_lzs(struct crypto_scomp *tfm) +void *zip_alloc_scomp_ctx_lzs(void) { int ret; struct zip_kernel_ctx *zip_ctx; @@ -274,7 +274,7 @@ void *zip_alloc_scomp_ctx_lzs(struct crypto_scomp *tfm) return zip_ctx; } -void zip_free_scomp_ctx(struct crypto_scomp *tfm, void *ctx) +void zip_free_scomp_ctx(void *ctx) { struct zip_kernel_ctx *zip_ctx = ctx; diff --git a/drivers/crypto/cavium/zip/zip_crypto.h b/drivers/crypto/cavium/zip/zip_crypto.h index b59ddfcacd34..dbe20bfeb3e9 100644 --- a/drivers/crypto/cavium/zip/zip_crypto.h +++ b/drivers/crypto/cavium/zip/zip_crypto.h @@ -67,9 +67,9 @@ int zip_comp_decompress(struct crypto_tfm *tfm, const u8 *src, unsigned int slen, u8 *dst, unsigned int *dlen); -void *zip_alloc_scomp_ctx_deflate(struct crypto_scomp *tfm); -void *zip_alloc_scomp_ctx_lzs(struct crypto_scomp *tfm); -void zip_free_scomp_ctx(struct crypto_scomp *tfm, void *zip_ctx); +void *zip_alloc_scomp_ctx_deflate(void); +void *zip_alloc_scomp_ctx_lzs(void); +void zip_free_scomp_ctx(void *zip_ctx); int zip_scomp_compress(struct crypto_scomp *tfm, const u8 *src, unsigned int slen, u8 *dst, unsigned int *dlen, void *ctx); diff --git a/include/crypto/internal/scompress.h b/include/crypto/internal/scompress.h index 07a10fd2d321..6ba9974df7d3 100644 --- a/include/crypto/internal/scompress.h +++ b/include/crypto/internal/scompress.h @@ -31,8 +31,8 @@ struct crypto_scomp { * @calg: Cmonn algorithm data structure shared with acomp */ struct scomp_alg { - void *(*alloc_ctx)(struct crypto_scomp *tfm); - void (*free_ctx)(struct crypto_scomp *tfm, void *ctx); + void *(*alloc_ctx)(void); + void (*free_ctx)(void *ctx); int (*compress)(struct crypto_scomp *tfm, const u8 *src, unsigned int slen, u8 *dst, unsigned int *dlen, void *ctx); @@ -73,13 +73,13 @@ static inline struct scomp_alg *crypto_scomp_alg(struct crypto_scomp *tfm) static inline void *crypto_scomp_alloc_ctx(struct crypto_scomp *tfm) { - return crypto_scomp_alg(tfm)->alloc_ctx(tfm); + return crypto_scomp_alg(tfm)->alloc_ctx(); } static inline void crypto_scomp_free_ctx(struct crypto_scomp *tfm, void *ctx) { - return crypto_scomp_alg(tfm)->free_ctx(tfm, ctx); + return crypto_scomp_alg(tfm)->free_ctx(ctx); } static inline int crypto_scomp_compress(struct crypto_scomp *tfm, -- 2.51.0 From 3d72ad46a23ae42450d1f475bb472151dede5b93 Mon Sep 17 00:00:00 2001 From: Herbert Xu Date: Sun, 9 Mar 2025 10:43:17 +0800 Subject: [PATCH 10/16] crypto: acomp - Move stream management into scomp layer Rather than allocating the stream memory in the request object, move it into a per-cpu buffer managed by scomp. This takes the stress off the user from having to manage large request objects and setting up their own per-cpu buffers in order to do so. Signed-off-by: Herbert Xu --- crypto/acompress.c | 30 ---------- crypto/compress.h | 2 - crypto/scompress.c | 90 +++++++++++++++++++---------- include/crypto/acompress.h | 26 ++++++++- include/crypto/internal/acompress.h | 17 +----- include/crypto/internal/scompress.h | 12 +--- 6 files changed, 84 insertions(+), 93 deletions(-) diff --git a/crypto/acompress.c b/crypto/acompress.c index 30176316140a..ef36ec31d73d 100644 --- a/crypto/acompress.c +++ b/crypto/acompress.c @@ -123,36 +123,6 @@ struct crypto_acomp *crypto_alloc_acomp_node(const char *alg_name, u32 type, } EXPORT_SYMBOL_GPL(crypto_alloc_acomp_node); -struct acomp_req *acomp_request_alloc(struct crypto_acomp *acomp) -{ - struct crypto_tfm *tfm = crypto_acomp_tfm(acomp); - struct acomp_req *req; - - req = __acomp_request_alloc(acomp); - if (req && (tfm->__crt_alg->cra_type != &crypto_acomp_type)) - return crypto_acomp_scomp_alloc_ctx(req); - - return req; -} -EXPORT_SYMBOL_GPL(acomp_request_alloc); - -void acomp_request_free(struct acomp_req *req) -{ - struct crypto_acomp *acomp = crypto_acomp_reqtfm(req); - struct crypto_tfm *tfm = crypto_acomp_tfm(acomp); - - if (tfm->__crt_alg->cra_type != &crypto_acomp_type) - crypto_acomp_scomp_free_ctx(req); - - if (req->base.flags & CRYPTO_ACOMP_ALLOC_OUTPUT) { - acomp->dst_free(req->dst); - req->dst = NULL; - } - - __acomp_request_free(req); -} -EXPORT_SYMBOL_GPL(acomp_request_free); - void comp_prepare_alg(struct comp_alg_common *alg) { struct crypto_alg *base = &alg->base; diff --git a/crypto/compress.h b/crypto/compress.h index c3cedfb5e606..f7737a1fcbbd 100644 --- a/crypto/compress.h +++ b/crypto/compress.h @@ -15,8 +15,6 @@ struct acomp_req; struct comp_alg_common; int crypto_init_scomp_ops_async(struct crypto_tfm *tfm); -struct acomp_req *crypto_acomp_scomp_alloc_ctx(struct acomp_req *req); -void crypto_acomp_scomp_free_ctx(struct acomp_req *req); void comp_prepare_alg(struct comp_alg_common *alg); diff --git a/crypto/scompress.c b/crypto/scompress.c index 1cef6bb06a81..9b6d9bbbc73a 100644 --- a/crypto/scompress.c +++ b/crypto/scompress.c @@ -98,13 +98,62 @@ error: return -ENOMEM; } +static void scomp_free_streams(struct scomp_alg *alg) +{ + struct crypto_acomp_stream __percpu *stream = alg->stream; + int i; + + for_each_possible_cpu(i) { + struct crypto_acomp_stream *ps = per_cpu_ptr(stream, i); + + if (!ps->ctx) + break; + + alg->free_ctx(ps); + } + + free_percpu(stream); +} + +static int scomp_alloc_streams(struct scomp_alg *alg) +{ + struct crypto_acomp_stream __percpu *stream; + int i; + + stream = alloc_percpu(struct crypto_acomp_stream); + if (!stream) + return -ENOMEM; + + for_each_possible_cpu(i) { + struct crypto_acomp_stream *ps = per_cpu_ptr(stream, i); + + ps->ctx = alg->alloc_ctx(); + if (IS_ERR(ps->ctx)) { + scomp_free_streams(alg); + return PTR_ERR(ps->ctx); + } + + spin_lock_init(&ps->lock); + } + + alg->stream = stream; + return 0; +} + static int crypto_scomp_init_tfm(struct crypto_tfm *tfm) { + struct scomp_alg *alg = crypto_scomp_alg(__crypto_scomp_tfm(tfm)); int ret = 0; mutex_lock(&scomp_lock); + if (!alg->stream) { + ret = scomp_alloc_streams(alg); + if (ret) + goto unlock; + } if (!scomp_scratch_users++) ret = crypto_scomp_alloc_scratches(); +unlock: mutex_unlock(&scomp_lock); return ret; @@ -115,7 +164,7 @@ static int scomp_acomp_comp_decomp(struct acomp_req *req, int dir) struct crypto_acomp *tfm = crypto_acomp_reqtfm(req); void **tfm_ctx = acomp_tfm_ctx(tfm); struct crypto_scomp *scomp = *tfm_ctx; - void **ctx = acomp_request_ctx(req); + struct crypto_acomp_stream *stream; struct scomp_scratch *scratch; void *src, *dst; unsigned int dlen; @@ -148,12 +197,15 @@ static int scomp_acomp_comp_decomp(struct acomp_req *req, int dir) else dst = scratch->dst; + stream = raw_cpu_ptr(crypto_scomp_alg(scomp)->stream); + spin_lock(&stream->lock); if (dir) ret = crypto_scomp_compress(scomp, src, req->slen, - dst, &req->dlen, *ctx); + dst, &req->dlen, stream->ctx); else ret = crypto_scomp_decompress(scomp, src, req->slen, - dst, &req->dlen, *ctx); + dst, &req->dlen, stream->ctx); + spin_unlock(&stream->lock); if (!ret) { if (!req->dst) { req->dst = sgl_alloc(req->dlen, GFP_ATOMIC, NULL); @@ -226,45 +278,19 @@ int crypto_init_scomp_ops_async(struct crypto_tfm *tfm) crt->compress = scomp_acomp_compress; crt->decompress = scomp_acomp_decompress; crt->dst_free = sgl_free; - crt->reqsize = sizeof(void *); return 0; } -struct acomp_req *crypto_acomp_scomp_alloc_ctx(struct acomp_req *req) +static void crypto_scomp_destroy(struct crypto_alg *alg) { - struct crypto_acomp *acomp = crypto_acomp_reqtfm(req); - struct crypto_tfm *tfm = crypto_acomp_tfm(acomp); - struct crypto_scomp **tfm_ctx = crypto_tfm_ctx(tfm); - struct crypto_scomp *scomp = *tfm_ctx; - void *ctx; - - ctx = crypto_scomp_alloc_ctx(scomp); - if (IS_ERR(ctx)) { - kfree(req); - return NULL; - } - - *req->__ctx = ctx; - - return req; -} - -void crypto_acomp_scomp_free_ctx(struct acomp_req *req) -{ - struct crypto_acomp *acomp = crypto_acomp_reqtfm(req); - struct crypto_tfm *tfm = crypto_acomp_tfm(acomp); - struct crypto_scomp **tfm_ctx = crypto_tfm_ctx(tfm); - struct crypto_scomp *scomp = *tfm_ctx; - void *ctx = *req->__ctx; - - if (ctx) - crypto_scomp_free_ctx(scomp, ctx); + scomp_free_streams(__crypto_scomp_alg(alg)); } static const struct crypto_type crypto_scomp_type = { .extsize = crypto_alg_extsize, .init_tfm = crypto_scomp_init_tfm, + .destroy = crypto_scomp_destroy, #ifdef CONFIG_PROC_FS .show = crypto_scomp_show, #endif diff --git a/include/crypto/acompress.h b/include/crypto/acompress.h index b6d5136e689d..c4937709ad0e 100644 --- a/include/crypto/acompress.h +++ b/include/crypto/acompress.h @@ -10,8 +10,12 @@ #define _CRYPTO_ACOMP_H #include +#include #include #include +#include +#include +#include #define CRYPTO_ACOMP_ALLOC_OUTPUT 0x00000001 #define CRYPTO_ACOMP_DST_MAX 131072 @@ -54,8 +58,14 @@ struct crypto_acomp { struct crypto_tfm base; }; +struct crypto_acomp_stream { + spinlock_t lock; + void *ctx; +}; + #define COMP_ALG_COMMON { \ struct crypto_alg base; \ + struct crypto_acomp_stream __percpu *stream; \ } struct comp_alg_common COMP_ALG_COMMON; @@ -173,7 +183,16 @@ static inline int crypto_has_acomp(const char *alg_name, u32 type, u32 mask) * * Return: allocated handle in case of success or NULL in case of an error */ -struct acomp_req *acomp_request_alloc(struct crypto_acomp *tfm); +static inline struct acomp_req *acomp_request_alloc_noprof(struct crypto_acomp *tfm) +{ + struct acomp_req *req; + + req = kzalloc_noprof(sizeof(*req) + crypto_acomp_reqsize(tfm), GFP_KERNEL); + if (likely(req)) + acomp_request_set_tfm(req, tfm); + return req; +} +#define acomp_request_alloc(...) alloc_hooks(acomp_request_alloc_noprof(__VA_ARGS__)) /** * acomp_request_free() -- zeroize and free asynchronous (de)compression @@ -182,7 +201,10 @@ struct acomp_req *acomp_request_alloc(struct crypto_acomp *tfm); * * @req: request to free */ -void acomp_request_free(struct acomp_req *req); +static inline void acomp_request_free(struct acomp_req *req) +{ + kfree_sensitive(req); +} /** * acomp_request_set_callback() -- Sets an asynchronous callback diff --git a/include/crypto/internal/acompress.h b/include/crypto/internal/acompress.h index 8831edaafc05..4a8f7e3beaa1 100644 --- a/include/crypto/internal/acompress.h +++ b/include/crypto/internal/acompress.h @@ -32,6 +32,7 @@ * * @reqsize: Context size for (de)compression requests * @base: Common crypto API algorithm data structure + * @stream: Per-cpu memory for algorithm * @calg: Cmonn algorithm data structure shared with scomp */ struct acomp_alg { @@ -68,22 +69,6 @@ static inline void acomp_request_complete(struct acomp_req *req, crypto_request_complete(&req->base, err); } -static inline struct acomp_req *__acomp_request_alloc_noprof(struct crypto_acomp *tfm) -{ - struct acomp_req *req; - - req = kzalloc_noprof(sizeof(*req) + crypto_acomp_reqsize(tfm), GFP_KERNEL); - if (likely(req)) - acomp_request_set_tfm(req, tfm); - return req; -} -#define __acomp_request_alloc(...) alloc_hooks(__acomp_request_alloc_noprof(__VA_ARGS__)) - -static inline void __acomp_request_free(struct acomp_req *req) -{ - kfree_sensitive(req); -} - /** * crypto_register_acomp() -- Register asynchronous compression algorithm * diff --git a/include/crypto/internal/scompress.h b/include/crypto/internal/scompress.h index 6ba9974df7d3..88986ab8ce15 100644 --- a/include/crypto/internal/scompress.h +++ b/include/crypto/internal/scompress.h @@ -28,6 +28,7 @@ struct crypto_scomp { * @compress: Function performs a compress operation * @decompress: Function performs a de-compress operation * @base: Common crypto API algorithm data structure + * @stream: Per-cpu memory for algorithm * @calg: Cmonn algorithm data structure shared with acomp */ struct scomp_alg { @@ -71,17 +72,6 @@ static inline struct scomp_alg *crypto_scomp_alg(struct crypto_scomp *tfm) return __crypto_scomp_alg(crypto_scomp_tfm(tfm)->__crt_alg); } -static inline void *crypto_scomp_alloc_ctx(struct crypto_scomp *tfm) -{ - return crypto_scomp_alg(tfm)->alloc_ctx(); -} - -static inline void crypto_scomp_free_ctx(struct crypto_scomp *tfm, - void *ctx) -{ - return crypto_scomp_alg(tfm)->free_ctx(ctx); -} - static inline int crypto_scomp_compress(struct crypto_scomp *tfm, const u8 *src, unsigned int slen, u8 *dst, unsigned int *dlen, void *ctx) -- 2.51.0 From cff12830e2cb2044067167f95a537074240347b7 Mon Sep 17 00:00:00 2001 From: Herbert Xu Date: Sun, 9 Mar 2025 10:43:19 +0800 Subject: [PATCH 11/16] crypto: scomp - Disable BH when taking per-cpu spin lock Disable BH when taking per-cpu spin locks. This isn't an issue right now because the only user zswap calls scomp from process context. However, if scomp is called from softirq context the spin lock may dead-lock. Signed-off-by: Herbert Xu --- crypto/scompress.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/crypto/scompress.c b/crypto/scompress.c index 9b6d9bbbc73a..a2ce481a10bb 100644 --- a/crypto/scompress.c +++ b/crypto/scompress.c @@ -182,7 +182,7 @@ static int scomp_acomp_comp_decomp(struct acomp_req *req, int dir) dlen = req->dlen; scratch = raw_cpu_ptr(&scomp_scratch); - spin_lock(&scratch->lock); + spin_lock_bh(&scratch->lock); if (sg_nents(req->src) == 1 && !PageHighMem(sg_page(req->src))) { src = page_to_virt(sg_page(req->src)) + req->src->offset; @@ -230,7 +230,7 @@ static int scomp_acomp_comp_decomp(struct acomp_req *req, int dir) } } out: - spin_unlock(&scratch->lock); + spin_unlock_bh(&scratch->lock); return ret; } -- 2.51.0 From b67a026003725a5d2496eba691c293694ab4847a Mon Sep 17 00:00:00 2001 From: Herbert Xu Date: Sun, 9 Mar 2025 10:43:21 +0800 Subject: [PATCH 12/16] crypto: acomp - Add request chaining and virtual addresses This adds request chaining and virtual address support to the acomp interface. It is identical to the ahash interface, except that a new flag CRYPTO_ACOMP_REQ_NONDMA has been added to indicate that the virtual addresses are not suitable for DMA. This is because all existing and potential acomp users can provide memory that is suitable for DMA so there is no need for a fall-back copy path. Signed-off-by: Herbert Xu --- crypto/acompress.c | 197 +++++++++++++++++++++++++++ include/crypto/acompress.h | 198 ++++++++++++++++++++++++++-- include/crypto/internal/acompress.h | 42 ++++++ 3 files changed, 424 insertions(+), 13 deletions(-) diff --git a/crypto/acompress.c b/crypto/acompress.c index ef36ec31d73d..45444e99a9db 100644 --- a/crypto/acompress.c +++ b/crypto/acompress.c @@ -23,6 +23,8 @@ struct crypto_scomp; static const struct crypto_type crypto_acomp_type; +static void acomp_reqchain_done(void *data, int err); + static inline struct acomp_alg *__crypto_acomp_alg(struct crypto_alg *alg) { return container_of(alg, struct acomp_alg, calg.base); @@ -123,6 +125,201 @@ struct crypto_acomp *crypto_alloc_acomp_node(const char *alg_name, u32 type, } EXPORT_SYMBOL_GPL(crypto_alloc_acomp_node); +static bool acomp_request_has_nondma(struct acomp_req *req) +{ + struct acomp_req *r2; + + if (acomp_request_isnondma(req)) + return true; + + list_for_each_entry(r2, &req->base.list, base.list) + if (acomp_request_isnondma(r2)) + return true; + + return false; +} + +static void acomp_save_req(struct acomp_req *req, crypto_completion_t cplt) +{ + struct crypto_acomp *tfm = crypto_acomp_reqtfm(req); + struct acomp_req_chain *state = &req->chain; + + if (!acomp_is_async(tfm)) + return; + + state->compl = req->base.complete; + state->data = req->base.data; + req->base.complete = cplt; + req->base.data = state; + state->req0 = req; +} + +static void acomp_restore_req(struct acomp_req_chain *state) +{ + struct acomp_req *req = state->req0; + struct crypto_acomp *tfm; + + tfm = crypto_acomp_reqtfm(req); + if (!acomp_is_async(tfm)) + return; + + req->base.complete = state->compl; + req->base.data = state->data; +} + +static void acomp_reqchain_virt(struct acomp_req_chain *state, int err) +{ + struct acomp_req *req = state->cur; + unsigned int slen = req->slen; + unsigned int dlen = req->dlen; + + req->base.err = err; + state = &req->chain; + + if (state->src) + acomp_request_set_src_dma(req, state->src, slen); + if (state->dst) + acomp_request_set_dst_dma(req, state->dst, dlen); + state->src = NULL; + state->dst = NULL; +} + +static void acomp_virt_to_sg(struct acomp_req *req) +{ + struct acomp_req_chain *state = &req->chain; + + if (acomp_request_src_isvirt(req)) { + unsigned int slen = req->slen; + const u8 *svirt = req->svirt; + + state->src = svirt; + sg_init_one(&state->ssg, svirt, slen); + acomp_request_set_src_sg(req, &state->ssg, slen); + } + + if (acomp_request_dst_isvirt(req)) { + unsigned int dlen = req->dlen; + u8 *dvirt = req->dvirt; + + state->dst = dvirt; + sg_init_one(&state->dsg, dvirt, dlen); + acomp_request_set_dst_sg(req, &state->dsg, dlen); + } +} + +static int acomp_reqchain_finish(struct acomp_req_chain *state, + int err, u32 mask) +{ + struct acomp_req *req0 = state->req0; + struct acomp_req *req = state->cur; + struct acomp_req *n; + + acomp_reqchain_virt(state, err); + + if (req != req0) + list_add_tail(&req->base.list, &req0->base.list); + + list_for_each_entry_safe(req, n, &state->head, base.list) { + list_del_init(&req->base.list); + + req->base.flags &= mask; + req->base.complete = acomp_reqchain_done; + req->base.data = state; + state->cur = req; + + acomp_virt_to_sg(req); + err = state->op(req); + + if (err == -EINPROGRESS) { + if (!list_empty(&state->head)) + err = -EBUSY; + goto out; + } + + if (err == -EBUSY) + goto out; + + acomp_reqchain_virt(state, err); + list_add_tail(&req->base.list, &req0->base.list); + } + + acomp_restore_req(state); + +out: + return err; +} + +static void acomp_reqchain_done(void *data, int err) +{ + struct acomp_req_chain *state = data; + crypto_completion_t compl = state->compl; + + data = state->data; + + if (err == -EINPROGRESS) { + if (!list_empty(&state->head)) + return; + goto notify; + } + + err = acomp_reqchain_finish(state, err, CRYPTO_TFM_REQ_MAY_BACKLOG); + if (err == -EBUSY) + return; + +notify: + compl(data, err); +} + +static int acomp_do_req_chain(struct acomp_req *req, + int (*op)(struct acomp_req *req)) +{ + struct crypto_acomp *tfm = crypto_acomp_reqtfm(req); + struct acomp_req_chain *state = &req->chain; + int err; + + if (crypto_acomp_req_chain(tfm) || + (!acomp_request_chained(req) && !acomp_request_isvirt(req))) + return op(req); + + /* + * There are no in-kernel users that do this. If and ever + * such users come into being then we could add a fall-back + * path. + */ + if (acomp_request_has_nondma(req)) + return -EINVAL; + + if (acomp_is_async(tfm)) { + acomp_save_req(req, acomp_reqchain_done); + state = req->base.data; + } + + state->op = op; + state->cur = req; + state->src = NULL; + INIT_LIST_HEAD(&state->head); + list_splice_init(&req->base.list, &state->head); + + acomp_virt_to_sg(req); + err = op(req); + if (err == -EBUSY || err == -EINPROGRESS) + return -EBUSY; + + return acomp_reqchain_finish(state, err, ~0); +} + +int crypto_acomp_compress(struct acomp_req *req) +{ + return acomp_do_req_chain(req, crypto_acomp_reqtfm(req)->compress); +} +EXPORT_SYMBOL_GPL(crypto_acomp_compress); + +int crypto_acomp_decompress(struct acomp_req *req) +{ + return acomp_do_req_chain(req, crypto_acomp_reqtfm(req)->decompress); +} +EXPORT_SYMBOL_GPL(crypto_acomp_decompress); + void comp_prepare_alg(struct comp_alg_common *alg) { struct crypto_alg *base = &alg->base; diff --git a/include/crypto/acompress.h b/include/crypto/acompress.h index c4937709ad0e..c4d8a29274c6 100644 --- a/include/crypto/acompress.h +++ b/include/crypto/acompress.h @@ -13,13 +13,42 @@ #include #include #include +#include #include #include #include #define CRYPTO_ACOMP_ALLOC_OUTPUT 0x00000001 + +/* Set this bit if source is virtual address instead of SG list. */ +#define CRYPTO_ACOMP_REQ_SRC_VIRT 0x00000002 + +/* Set this bit for if virtual address source cannot be used for DMA. */ +#define CRYPTO_ACOMP_REQ_SRC_NONDMA 0x00000004 + +/* Set this bit if destination is virtual address instead of SG list. */ +#define CRYPTO_ACOMP_REQ_DST_VIRT 0x00000008 + +/* Set this bit for if virtual address destination cannot be used for DMA. */ +#define CRYPTO_ACOMP_REQ_DST_NONDMA 0x00000010 + #define CRYPTO_ACOMP_DST_MAX 131072 +struct acomp_req; + +struct acomp_req_chain { + struct list_head head; + struct acomp_req *req0; + struct acomp_req *cur; + int (*op)(struct acomp_req *req); + crypto_completion_t compl; + void *data; + struct scatterlist ssg; + struct scatterlist dsg; + const u8 *src; + u8 *dst; +}; + /** * struct acomp_req - asynchronous (de)compression request * @@ -28,14 +57,24 @@ * @dst: Destination data * @slen: Size of the input buffer * @dlen: Size of the output buffer and number of bytes produced + * @chain: Private API code data, do not use * @__ctx: Start of private context data */ struct acomp_req { struct crypto_async_request base; - struct scatterlist *src; - struct scatterlist *dst; + union { + struct scatterlist *src; + const u8 *svirt; + }; + union { + struct scatterlist *dst; + u8 *dvirt; + }; unsigned int slen; unsigned int dlen; + + struct acomp_req_chain chain; + void *__ctx[] CRYPTO_MINALIGN_ATTR; }; @@ -222,10 +261,16 @@ static inline void acomp_request_set_callback(struct acomp_req *req, crypto_completion_t cmpl, void *data) { + u32 keep = CRYPTO_ACOMP_ALLOC_OUTPUT | CRYPTO_ACOMP_REQ_SRC_VIRT | + CRYPTO_ACOMP_REQ_SRC_NONDMA | CRYPTO_ACOMP_REQ_DST_VIRT | + CRYPTO_ACOMP_REQ_DST_NONDMA; + req->base.complete = cmpl; req->base.data = data; - req->base.flags &= CRYPTO_ACOMP_ALLOC_OUTPUT; - req->base.flags |= flgs & ~CRYPTO_ACOMP_ALLOC_OUTPUT; + req->base.flags &= keep; + req->base.flags |= flgs & ~keep; + + crypto_reqchain_init(&req->base); } /** @@ -252,11 +297,144 @@ static inline void acomp_request_set_params(struct acomp_req *req, req->slen = slen; req->dlen = dlen; - req->base.flags &= ~CRYPTO_ACOMP_ALLOC_OUTPUT; + req->base.flags &= ~(CRYPTO_ACOMP_ALLOC_OUTPUT | + CRYPTO_ACOMP_REQ_SRC_VIRT | + CRYPTO_ACOMP_REQ_SRC_NONDMA | + CRYPTO_ACOMP_REQ_DST_VIRT | + CRYPTO_ACOMP_REQ_DST_NONDMA); if (!req->dst) req->base.flags |= CRYPTO_ACOMP_ALLOC_OUTPUT; } +/** + * acomp_request_set_src_sg() -- Sets source scatterlist + * + * Sets source scatterlist required by an acomp operation. + * + * @req: asynchronous compress request + * @src: pointer to input buffer scatterlist + * @slen: size of the input buffer + */ +static inline void acomp_request_set_src_sg(struct acomp_req *req, + struct scatterlist *src, + unsigned int slen) +{ + req->src = src; + req->slen = slen; + + req->base.flags &= ~CRYPTO_ACOMP_REQ_SRC_NONDMA; + req->base.flags &= ~CRYPTO_ACOMP_REQ_SRC_VIRT; +} + +/** + * acomp_request_set_src_dma() -- Sets DMA source virtual address + * + * Sets source virtual address required by an acomp operation. + * The address must be usable for DMA. + * + * @req: asynchronous compress request + * @src: virtual address pointer to input buffer + * @slen: size of the input buffer + */ +static inline void acomp_request_set_src_dma(struct acomp_req *req, + const u8 *src, unsigned int slen) +{ + req->svirt = src; + req->slen = slen; + + req->base.flags &= ~CRYPTO_ACOMP_REQ_SRC_NONDMA; + req->base.flags |= CRYPTO_ACOMP_REQ_SRC_VIRT; +} + +/** + * acomp_request_set_src_nondma() -- Sets non-DMA source virtual address + * + * Sets source virtual address required by an acomp operation. + * The address can not be used for DMA. + * + * @req: asynchronous compress request + * @src: virtual address pointer to input buffer + * @slen: size of the input buffer + */ +static inline void acomp_request_set_src_nondma(struct acomp_req *req, + const u8 *src, + unsigned int slen) +{ + req->svirt = src; + req->slen = slen; + + req->base.flags |= CRYPTO_ACOMP_REQ_SRC_NONDMA; + req->base.flags |= CRYPTO_ACOMP_REQ_SRC_VIRT; +} + +/** + * acomp_request_set_dst_sg() -- Sets destination scatterlist + * + * Sets destination scatterlist required by an acomp operation. + * + * @req: asynchronous compress request + * @dst: pointer to output buffer scatterlist + * @dlen: size of the output buffer + */ +static inline void acomp_request_set_dst_sg(struct acomp_req *req, + struct scatterlist *dst, + unsigned int dlen) +{ + req->dst = dst; + req->dlen = dlen; + + req->base.flags &= ~CRYPTO_ACOMP_REQ_DST_NONDMA; + req->base.flags &= ~CRYPTO_ACOMP_REQ_DST_VIRT; +} + +/** + * acomp_request_set_dst_dma() -- Sets DMA destination virtual address + * + * Sets destination virtual address required by an acomp operation. + * The address must be usable for DMA. + * + * @req: asynchronous compress request + * @dst: virtual address pointer to output buffer + * @dlen: size of the output buffer + */ +static inline void acomp_request_set_dst_dma(struct acomp_req *req, + u8 *dst, unsigned int dlen) +{ + req->dvirt = dst; + req->dlen = dlen; + + req->base.flags &= ~CRYPTO_ACOMP_ALLOC_OUTPUT; + req->base.flags &= ~CRYPTO_ACOMP_REQ_DST_NONDMA; + req->base.flags |= CRYPTO_ACOMP_REQ_DST_VIRT; +} + +/** + * acomp_request_set_dst_nondma() -- Sets non-DMA destination virtual address + * + * Sets destination virtual address required by an acomp operation. + * The address can not be used for DMA. + * + * @req: asynchronous compress request + * @dst: virtual address pointer to output buffer + * @dlen: size of the output buffer + */ +static inline void acomp_request_set_dst_nondma(struct acomp_req *req, + u8 *dst, unsigned int dlen) +{ + req->dvirt = dst; + req->dlen = dlen; + + req->base.flags &= ~CRYPTO_ACOMP_ALLOC_OUTPUT; + req->base.flags |= CRYPTO_ACOMP_REQ_DST_NONDMA; + req->base.flags |= CRYPTO_ACOMP_REQ_DST_VIRT; +} + +static inline void acomp_request_chain(struct acomp_req *req, + struct acomp_req *head) +{ + crypto_request_chain(&req->base, &head->base); +} + /** * crypto_acomp_compress() -- Invoke asynchronous compress operation * @@ -266,10 +444,7 @@ static inline void acomp_request_set_params(struct acomp_req *req, * * Return: zero on success; error code in case of error */ -static inline int crypto_acomp_compress(struct acomp_req *req) -{ - return crypto_acomp_reqtfm(req)->compress(req); -} +int crypto_acomp_compress(struct acomp_req *req); /** * crypto_acomp_decompress() -- Invoke asynchronous decompress operation @@ -280,9 +455,6 @@ static inline int crypto_acomp_compress(struct acomp_req *req) * * Return: zero on success; error code in case of error */ -static inline int crypto_acomp_decompress(struct acomp_req *req) -{ - return crypto_acomp_reqtfm(req)->decompress(req); -} +int crypto_acomp_decompress(struct acomp_req *req); #endif diff --git a/include/crypto/internal/acompress.h b/include/crypto/internal/acompress.h index 4a8f7e3beaa1..957a5ed7c7f1 100644 --- a/include/crypto/internal/acompress.h +++ b/include/crypto/internal/acompress.h @@ -94,4 +94,46 @@ void crypto_unregister_acomp(struct acomp_alg *alg); int crypto_register_acomps(struct acomp_alg *algs, int count); void crypto_unregister_acomps(struct acomp_alg *algs, int count); +static inline bool acomp_request_chained(struct acomp_req *req) +{ + return crypto_request_chained(&req->base); +} + +static inline bool acomp_request_src_isvirt(struct acomp_req *req) +{ + return req->base.flags & CRYPTO_ACOMP_REQ_SRC_VIRT; +} + +static inline bool acomp_request_dst_isvirt(struct acomp_req *req) +{ + return req->base.flags & CRYPTO_ACOMP_REQ_DST_VIRT; +} + +static inline bool acomp_request_isvirt(struct acomp_req *req) +{ + return req->base.flags & (CRYPTO_ACOMP_REQ_SRC_VIRT | + CRYPTO_ACOMP_REQ_DST_VIRT); +} + +static inline bool acomp_request_src_isnondma(struct acomp_req *req) +{ + return req->base.flags & CRYPTO_ACOMP_REQ_SRC_NONDMA; +} + +static inline bool acomp_request_dst_isnondma(struct acomp_req *req) +{ + return req->base.flags & CRYPTO_ACOMP_REQ_DST_NONDMA; +} + +static inline bool acomp_request_isnondma(struct acomp_req *req) +{ + return req->base.flags & (CRYPTO_ACOMP_REQ_SRC_NONDMA | + CRYPTO_ACOMP_REQ_DST_NONDMA); +} + +static inline bool crypto_acomp_req_chain(struct crypto_acomp *tfm) +{ + return crypto_tfm_req_chain(&tfm->base); +} + #endif -- 2.51.0 From d2d072a313c1817a0d72d7b8301eaf29ce7f83fc Mon Sep 17 00:00:00 2001 From: Herbert Xu Date: Sun, 9 Mar 2025 10:43:24 +0800 Subject: [PATCH 13/16] crypto: testmgr - Remove NULL dst acomp tests In preparation for the partial removal of NULL dst acomp support, remove the tests for them. Signed-off-by: Herbert Xu --- crypto/testmgr.c | 29 ----------------------------- 1 file changed, 29 deletions(-) diff --git a/crypto/testmgr.c b/crypto/testmgr.c index cbce769b16ef..140872765dcd 100644 --- a/crypto/testmgr.c +++ b/crypto/testmgr.c @@ -3522,21 +3522,6 @@ static int test_acomp(struct crypto_acomp *tfm, goto out; } -#ifdef CONFIG_CRYPTO_MANAGER_EXTRA_TESTS - crypto_init_wait(&wait); - sg_init_one(&src, input_vec, ilen); - acomp_request_set_params(req, &src, NULL, ilen, 0); - - ret = crypto_wait_req(crypto_acomp_compress(req), &wait); - if (ret) { - pr_err("alg: acomp: compression failed on NULL dst buffer test %d for %s: ret=%d\n", - i + 1, algo, -ret); - kfree(input_vec); - acomp_request_free(req); - goto out; - } -#endif - kfree(input_vec); acomp_request_free(req); } @@ -3598,20 +3583,6 @@ static int test_acomp(struct crypto_acomp *tfm, goto out; } -#ifdef CONFIG_CRYPTO_MANAGER_EXTRA_TESTS - crypto_init_wait(&wait); - acomp_request_set_params(req, &src, NULL, ilen, 0); - - ret = crypto_wait_req(crypto_acomp_decompress(req), &wait); - if (ret) { - pr_err("alg: acomp: decompression failed on NULL dst buffer test %d for %s: ret=%d\n", - i + 1, algo, -ret); - kfree(input_vec); - acomp_request_free(req); - goto out; - } -#endif - kfree(input_vec); acomp_request_free(req); } -- 2.51.0 From ea3d35467ba474768013365ce5f2c6eb454fed3a Mon Sep 17 00:00:00 2001 From: Suman Kumar Chakraborty Date: Mon, 10 Mar 2025 16:15:40 +0000 Subject: [PATCH 14/16] crypto: qat - add macro to write 64-bit values to registers Introduce the ADF_CSR_WR_LO_HI macro to simplify writing a 64-bit values to hardware registers. This macro works by splitting the 64-bit value into two 32-bit segments, which are then written separately to the specified lower and upper register offsets. Update the adf_gen4_set_ssm_wdtimer() function to utilize this newly introduced macro. Signed-off-by: Suman Kumar Chakraborty Reviewed-by: Giovanni Cabiddu Reviewed-by: Andy Shevchenko Signed-off-by: Herbert Xu --- .../intel/qat/qat_common/adf_accel_devices.h | 10 +++++++ .../intel/qat/qat_common/adf_gen4_hw_data.c | 30 ++++--------------- 2 files changed, 16 insertions(+), 24 deletions(-) diff --git a/drivers/crypto/intel/qat/qat_common/adf_accel_devices.h b/drivers/crypto/intel/qat/qat_common/adf_accel_devices.h index 9c15c31ad134..0509174d6030 100644 --- a/drivers/crypto/intel/qat/qat_common/adf_accel_devices.h +++ b/drivers/crypto/intel/qat/qat_common/adf_accel_devices.h @@ -10,6 +10,7 @@ #include #include #include +#include #include "adf_cfg_common.h" #include "adf_rl.h" #include "adf_telemetry.h" @@ -371,6 +372,15 @@ struct adf_hw_device_data { /* CSR write macro */ #define ADF_CSR_WR(csr_base, csr_offset, val) \ __raw_writel(val, csr_base + csr_offset) +/* + * CSR write macro to handle cases where the high and low + * offsets are sparsely located. + */ +#define ADF_CSR_WR64_LO_HI(csr_base, csr_low_offset, csr_high_offset, val) \ +do { \ + ADF_CSR_WR(csr_base, csr_low_offset, lower_32_bits(val)); \ + ADF_CSR_WR(csr_base, csr_high_offset, upper_32_bits(val)); \ +} while (0) /* CSR read macro */ #define ADF_CSR_RD(csr_base, csr_offset) __raw_readl(csr_base + csr_offset) diff --git a/drivers/crypto/intel/qat/qat_common/adf_gen4_hw_data.c b/drivers/crypto/intel/qat/qat_common/adf_gen4_hw_data.c index ade279e48010..099949a2421c 100644 --- a/drivers/crypto/intel/qat/qat_common/adf_gen4_hw_data.c +++ b/drivers/crypto/intel/qat/qat_common/adf_gen4_hw_data.c @@ -135,36 +135,18 @@ int adf_gen4_init_device(struct adf_accel_dev *accel_dev) } EXPORT_SYMBOL_GPL(adf_gen4_init_device); -static inline void adf_gen4_unpack_ssm_wdtimer(u64 value, u32 *upper, - u32 *lower) -{ - *lower = lower_32_bits(value); - *upper = upper_32_bits(value); -} - void adf_gen4_set_ssm_wdtimer(struct adf_accel_dev *accel_dev) { void __iomem *pmisc_addr = adf_get_pmisc_base(accel_dev); u64 timer_val_pke = ADF_SSM_WDT_PKE_DEFAULT_VALUE; u64 timer_val = ADF_SSM_WDT_DEFAULT_VALUE; - u32 ssm_wdt_pke_high = 0; - u32 ssm_wdt_pke_low = 0; - u32 ssm_wdt_high = 0; - u32 ssm_wdt_low = 0; - /* Convert 64bit WDT timer value into 32bit values for - * mmio write to 32bit CSRs. - */ - adf_gen4_unpack_ssm_wdtimer(timer_val, &ssm_wdt_high, &ssm_wdt_low); - adf_gen4_unpack_ssm_wdtimer(timer_val_pke, &ssm_wdt_pke_high, - &ssm_wdt_pke_low); - - /* Enable WDT for sym and dc */ - ADF_CSR_WR(pmisc_addr, ADF_SSMWDTL_OFFSET, ssm_wdt_low); - ADF_CSR_WR(pmisc_addr, ADF_SSMWDTH_OFFSET, ssm_wdt_high); - /* Enable WDT for pke */ - ADF_CSR_WR(pmisc_addr, ADF_SSMWDTPKEL_OFFSET, ssm_wdt_pke_low); - ADF_CSR_WR(pmisc_addr, ADF_SSMWDTPKEH_OFFSET, ssm_wdt_pke_high); + /* Enable watchdog timer for sym and dc */ + ADF_CSR_WR64_LO_HI(pmisc_addr, ADF_SSMWDTL_OFFSET, ADF_SSMWDTH_OFFSET, timer_val); + + /* Enable watchdog timer for pke */ + ADF_CSR_WR64_LO_HI(pmisc_addr, ADF_SSMWDTPKEL_OFFSET, ADF_SSMWDTPKEH_OFFSET, + timer_val_pke); } EXPORT_SYMBOL_GPL(adf_gen4_set_ssm_wdtimer); -- 2.51.0 From 7450ebd29cd9b9745f005f2609badacea15fbe30 Mon Sep 17 00:00:00 2001 From: Eric Biggers Date: Mon, 10 Mar 2025 10:20:16 -0700 Subject: [PATCH 15/16] crypto: scatterwalk - simplify map and unmap calling convention Now that the address returned by scatterwalk_map() is always being stored into the same struct scatter_walk that is passed in, make scatterwalk_map() do so itself and return void. Similarly, now that scatterwalk_unmap() is always being passed the address field within a struct scatter_walk, make scatterwalk_unmap() take a pointer to struct scatter_walk instead of the address directly. Signed-off-by: Eric Biggers Signed-off-by: Herbert Xu --- arch/s390/crypto/aes_s390.c | 3 +-- crypto/skcipher.c | 28 +++++++---------------- include/crypto/scatterwalk.h | 43 ++++++++++++++++++++---------------- 3 files changed, 33 insertions(+), 41 deletions(-) diff --git a/arch/s390/crypto/aes_s390.c b/arch/s390/crypto/aes_s390.c index ed85bd6e298f..5d36f4020dfa 100644 --- a/arch/s390/crypto/aes_s390.c +++ b/arch/s390/crypto/aes_s390.c @@ -871,8 +871,7 @@ static int gcm_out_walk_go(struct gcm_sg_walk *gw, unsigned int minbytesneeded) goto out; } - /* XXX */ - scatterwalk_unmap(gw->walk.addr); + scatterwalk_unmap(&gw->walk); gw->ptr = gw->buf; gw->nbytes = sizeof(gw->buf); diff --git a/crypto/skcipher.c b/crypto/skcipher.c index ab5d852febcd..132075a905d9 100644 --- a/crypto/skcipher.c +++ b/crypto/skcipher.c @@ -39,18 +39,6 @@ static const struct crypto_type crypto_skcipher_type; static int skcipher_walk_next(struct skcipher_walk *walk); -static inline void skcipher_map_src(struct skcipher_walk *walk) -{ - /* XXX */ - walk->in.__addr = scatterwalk_map(&walk->in); -} - -static inline void skcipher_map_dst(struct skcipher_walk *walk) -{ - /* XXX */ - walk->out.__addr = scatterwalk_map(&walk->out); -} - static inline gfp_t skcipher_walk_gfp(struct skcipher_walk *walk) { return walk->flags & SKCIPHER_WALK_SLEEP ? GFP_KERNEL : GFP_ATOMIC; @@ -101,8 +89,8 @@ int skcipher_walk_done(struct skcipher_walk *walk, int res) scatterwalk_done_src(&walk->in, n); } else if (walk->flags & SKCIPHER_WALK_COPY) { scatterwalk_advance(&walk->in, n); - skcipher_map_dst(walk); - memcpy(walk->dst.virt.addr, walk->page, n); + scatterwalk_map(&walk->out); + memcpy(walk->out.addr, walk->page, n); } else { /* SKCIPHER_WALK_SLOW */ if (res > 0) { /* @@ -186,9 +174,9 @@ static int skcipher_next_copy(struct skcipher_walk *walk) { void *tmp = walk->page; - skcipher_map_src(walk); - memcpy(tmp, walk->src.virt.addr, walk->nbytes); - scatterwalk_unmap(walk->src.virt.addr); + scatterwalk_map(&walk->in); + memcpy(tmp, walk->in.addr, walk->nbytes); + scatterwalk_unmap(&walk->in); /* * walk->in is advanced later when the number of bytes actually * processed (which might be less than walk->nbytes) is known. @@ -208,12 +196,12 @@ static int skcipher_next_fast(struct skcipher_walk *walk) diff |= (u8 *)(sg_page(walk->in.sg) + (walk->in.offset >> PAGE_SHIFT)) - (u8 *)(sg_page(walk->out.sg) + (walk->out.offset >> PAGE_SHIFT)); - skcipher_map_dst(walk); - walk->in.__addr = walk->dst.virt.addr; + scatterwalk_map(&walk->out); + walk->in.__addr = walk->out.__addr; if (diff) { walk->flags |= SKCIPHER_WALK_DIFF; - skcipher_map_src(walk); + scatterwalk_map(&walk->in); } return 0; diff --git a/include/crypto/scatterwalk.h b/include/crypto/scatterwalk.h index c62f47d04eb1..b7e617ae4442 100644 --- a/include/crypto/scatterwalk.h +++ b/include/crypto/scatterwalk.h @@ -97,23 +97,28 @@ static inline void scatterwalk_get_sglist(struct scatter_walk *walk, scatterwalk_crypto_chain(sg_out, sg_next(walk->sg), 2); } -static inline void *scatterwalk_map(struct scatter_walk *walk) +static inline void scatterwalk_map(struct scatter_walk *walk) { struct page *base_page = sg_page(walk->sg); - if (IS_ENABLED(CONFIG_HIGHMEM)) - return kmap_local_page(base_page + (walk->offset >> PAGE_SHIFT)) + - offset_in_page(walk->offset); - /* - * When !HIGHMEM we allow the walker to return segments that span a page - * boundary; see scatterwalk_clamp(). To make it clear that in this - * case we're working in the linear buffer of the whole sg entry in the - * kernel's direct map rather than within the mapped buffer of a single - * page, compute the address as an offset from the page_address() of the - * first page of the sg entry. Either way the result is the address in - * the direct map, but this makes it clearer what is really going on. - */ - return page_address(base_page) + walk->offset; + if (IS_ENABLED(CONFIG_HIGHMEM)) { + walk->__addr = kmap_local_page(base_page + + (walk->offset >> PAGE_SHIFT)) + + offset_in_page(walk->offset); + } else { + /* + * When !HIGHMEM we allow the walker to return segments that + * span a page boundary; see scatterwalk_clamp(). To make it + * clear that in this case we're working in the linear buffer of + * the whole sg entry in the kernel's direct map rather than + * within the mapped buffer of a single page, compute the + * address as an offset from the page_address() of the first + * page of the sg entry. Either way the result is the address + * in the direct map, but this makes it clearer what is really + * going on. + */ + walk->__addr = page_address(base_page) + walk->offset; + } } /** @@ -132,14 +137,14 @@ static inline unsigned int scatterwalk_next(struct scatter_walk *walk, { unsigned int nbytes = scatterwalk_clamp(walk, total); - walk->__addr = scatterwalk_map(walk); + scatterwalk_map(walk); return nbytes; } -static inline void scatterwalk_unmap(const void *vaddr) +static inline void scatterwalk_unmap(struct scatter_walk *walk) { if (IS_ENABLED(CONFIG_HIGHMEM)) - kunmap_local(vaddr); + kunmap_local(walk->__addr); } static inline void scatterwalk_advance(struct scatter_walk *walk, @@ -159,7 +164,7 @@ static inline void scatterwalk_advance(struct scatter_walk *walk, static inline void scatterwalk_done_src(struct scatter_walk *walk, unsigned int nbytes) { - scatterwalk_unmap(walk->addr); + scatterwalk_unmap(walk); scatterwalk_advance(walk, nbytes); } @@ -175,7 +180,7 @@ static inline void scatterwalk_done_src(struct scatter_walk *walk, static inline void scatterwalk_done_dst(struct scatter_walk *walk, unsigned int nbytes) { - scatterwalk_unmap(walk->addr); + scatterwalk_unmap(walk); /* * Explicitly check ARCH_IMPLEMENTS_FLUSH_DCACHE_PAGE instead of just * relying on flush_dcache_page() being a no-op when not implemented, -- 2.51.0 From f3bda3b9b69cb193968ba781446ff18c734f0276 Mon Sep 17 00:00:00 2001 From: Suman Kumar Chakraborty Date: Wed, 12 Mar 2025 11:39:38 +0000 Subject: [PATCH 16/16] crypto: qat - introduce fuse array Change the representation of fuses in the accelerator device structure from a single value to an array. This allows the structure to accommodate additional fuses that are required for future generations of QAT hardware. This does not introduce any functional changes. Signed-off-by: Suman Kumar Chakraborty Reviewed-by: Giovanni Cabiddu Reviewed-by: Andy Shevchenko Signed-off-by: Herbert Xu --- .../crypto/intel/qat/qat_420xx/adf_420xx_hw_data.c | 2 +- drivers/crypto/intel/qat/qat_420xx/adf_drv.c | 2 +- drivers/crypto/intel/qat/qat_4xxx/adf_4xxx_hw_data.c | 2 +- drivers/crypto/intel/qat/qat_4xxx/adf_drv.c | 2 +- .../crypto/intel/qat/qat_c3xxx/adf_c3xxx_hw_data.c | 4 ++-- drivers/crypto/intel/qat/qat_c3xxx/adf_drv.c | 2 +- drivers/crypto/intel/qat/qat_c62x/adf_c62x_hw_data.c | 4 ++-- drivers/crypto/intel/qat/qat_c62x/adf_drv.c | 4 ++-- .../crypto/intel/qat/qat_common/adf_accel_devices.h | 12 +++++++++++- .../crypto/intel/qat/qat_common/adf_gen2_hw_data.c | 2 +- .../intel/qat/qat_dh895xcc/adf_dh895xcc_hw_data.c | 6 +++--- drivers/crypto/intel/qat/qat_dh895xcc/adf_drv.c | 2 +- 12 files changed, 27 insertions(+), 17 deletions(-) diff --git a/drivers/crypto/intel/qat/qat_420xx/adf_420xx_hw_data.c b/drivers/crypto/intel/qat/qat_420xx/adf_420xx_hw_data.c index 87c71914ae87..98f3c1210666 100644 --- a/drivers/crypto/intel/qat/qat_420xx/adf_420xx_hw_data.c +++ b/drivers/crypto/intel/qat/qat_420xx/adf_420xx_hw_data.c @@ -98,7 +98,7 @@ static struct adf_hw_device_class adf_420xx_class = { static u32 get_ae_mask(struct adf_hw_device_data *self) { - u32 me_disable = self->fuses; + u32 me_disable = self->fuses[ADF_FUSECTL4]; return ~me_disable & ADF_420XX_ACCELENGINES_MASK; } diff --git a/drivers/crypto/intel/qat/qat_420xx/adf_drv.c b/drivers/crypto/intel/qat/qat_420xx/adf_drv.c index 9589d60fb281..8084aa0f7f41 100644 --- a/drivers/crypto/intel/qat/qat_420xx/adf_drv.c +++ b/drivers/crypto/intel/qat/qat_420xx/adf_drv.c @@ -79,7 +79,7 @@ static int adf_probe(struct pci_dev *pdev, const struct pci_device_id *ent) adf_init_hw_data_420xx(accel_dev->hw_device, ent->device); pci_read_config_byte(pdev, PCI_REVISION_ID, &accel_pci_dev->revid); - pci_read_config_dword(pdev, ADF_GEN4_FUSECTL4_OFFSET, &hw_data->fuses); + pci_read_config_dword(pdev, ADF_GEN4_FUSECTL4_OFFSET, &hw_data->fuses[ADF_FUSECTL4]); /* Get Accelerators and Accelerators Engines masks */ hw_data->accel_mask = hw_data->get_accel_mask(hw_data); diff --git a/drivers/crypto/intel/qat/qat_4xxx/adf_4xxx_hw_data.c b/drivers/crypto/intel/qat/qat_4xxx/adf_4xxx_hw_data.c index 36eebda3a028..4eb6ef99efdd 100644 --- a/drivers/crypto/intel/qat/qat_4xxx/adf_4xxx_hw_data.c +++ b/drivers/crypto/intel/qat/qat_4xxx/adf_4xxx_hw_data.c @@ -101,7 +101,7 @@ static struct adf_hw_device_class adf_4xxx_class = { static u32 get_ae_mask(struct adf_hw_device_data *self) { - u32 me_disable = self->fuses; + u32 me_disable = self->fuses[ADF_FUSECTL4]; return ~me_disable & ADF_4XXX_ACCELENGINES_MASK; } diff --git a/drivers/crypto/intel/qat/qat_4xxx/adf_drv.c b/drivers/crypto/intel/qat/qat_4xxx/adf_drv.c index d7de1cad1335..5537a9991e4e 100644 --- a/drivers/crypto/intel/qat/qat_4xxx/adf_drv.c +++ b/drivers/crypto/intel/qat/qat_4xxx/adf_drv.c @@ -81,7 +81,7 @@ static int adf_probe(struct pci_dev *pdev, const struct pci_device_id *ent) adf_init_hw_data_4xxx(accel_dev->hw_device, ent->device); pci_read_config_byte(pdev, PCI_REVISION_ID, &accel_pci_dev->revid); - pci_read_config_dword(pdev, ADF_GEN4_FUSECTL4_OFFSET, &hw_data->fuses); + pci_read_config_dword(pdev, ADF_GEN4_FUSECTL4_OFFSET, &hw_data->fuses[ADF_FUSECTL4]); /* Get Accelerators and Accelerators Engines masks */ hw_data->accel_mask = hw_data->get_accel_mask(hw_data); diff --git a/drivers/crypto/intel/qat/qat_c3xxx/adf_c3xxx_hw_data.c b/drivers/crypto/intel/qat/qat_c3xxx/adf_c3xxx_hw_data.c index 201f9412c582..e78f7bfd30b8 100644 --- a/drivers/crypto/intel/qat/qat_c3xxx/adf_c3xxx_hw_data.c +++ b/drivers/crypto/intel/qat/qat_c3xxx/adf_c3xxx_hw_data.c @@ -27,8 +27,8 @@ static struct adf_hw_device_class c3xxx_class = { static u32 get_accel_mask(struct adf_hw_device_data *self) { + u32 fuses = self->fuses[ADF_FUSECTL0]; u32 straps = self->straps; - u32 fuses = self->fuses; u32 accel; accel = ~(fuses | straps) >> ADF_C3XXX_ACCELERATORS_REG_OFFSET; @@ -39,8 +39,8 @@ static u32 get_accel_mask(struct adf_hw_device_data *self) static u32 get_ae_mask(struct adf_hw_device_data *self) { + u32 fuses = self->fuses[ADF_FUSECTL0]; u32 straps = self->straps; - u32 fuses = self->fuses; unsigned long disabled; u32 ae_disable; int accel; diff --git a/drivers/crypto/intel/qat/qat_c3xxx/adf_drv.c b/drivers/crypto/intel/qat/qat_c3xxx/adf_drv.c index caa53882fda6..b825b35ab4bf 100644 --- a/drivers/crypto/intel/qat/qat_c3xxx/adf_drv.c +++ b/drivers/crypto/intel/qat/qat_c3xxx/adf_drv.c @@ -126,7 +126,7 @@ static int adf_probe(struct pci_dev *pdev, const struct pci_device_id *ent) adf_init_hw_data_c3xxx(accel_dev->hw_device); pci_read_config_byte(pdev, PCI_REVISION_ID, &accel_pci_dev->revid); pci_read_config_dword(pdev, ADF_DEVICE_FUSECTL_OFFSET, - &hw_data->fuses); + &hw_data->fuses[ADF_FUSECTL0]); pci_read_config_dword(pdev, ADF_C3XXX_SOFTSTRAP_CSR_OFFSET, &hw_data->straps); diff --git a/drivers/crypto/intel/qat/qat_c62x/adf_c62x_hw_data.c b/drivers/crypto/intel/qat/qat_c62x/adf_c62x_hw_data.c index 6b5b0cf9c7c7..32ebe09477a8 100644 --- a/drivers/crypto/intel/qat/qat_c62x/adf_c62x_hw_data.c +++ b/drivers/crypto/intel/qat/qat_c62x/adf_c62x_hw_data.c @@ -27,8 +27,8 @@ static struct adf_hw_device_class c62x_class = { static u32 get_accel_mask(struct adf_hw_device_data *self) { + u32 fuses = self->fuses[ADF_FUSECTL0]; u32 straps = self->straps; - u32 fuses = self->fuses; u32 accel; accel = ~(fuses | straps) >> ADF_C62X_ACCELERATORS_REG_OFFSET; @@ -39,8 +39,8 @@ static u32 get_accel_mask(struct adf_hw_device_data *self) static u32 get_ae_mask(struct adf_hw_device_data *self) { + u32 fuses = self->fuses[ADF_FUSECTL0]; u32 straps = self->straps; - u32 fuses = self->fuses; unsigned long disabled; u32 ae_disable; int accel; diff --git a/drivers/crypto/intel/qat/qat_c62x/adf_drv.c b/drivers/crypto/intel/qat/qat_c62x/adf_drv.c index b7398fee19ed..8a7bdec358d6 100644 --- a/drivers/crypto/intel/qat/qat_c62x/adf_drv.c +++ b/drivers/crypto/intel/qat/qat_c62x/adf_drv.c @@ -126,7 +126,7 @@ static int adf_probe(struct pci_dev *pdev, const struct pci_device_id *ent) adf_init_hw_data_c62x(accel_dev->hw_device); pci_read_config_byte(pdev, PCI_REVISION_ID, &accel_pci_dev->revid); pci_read_config_dword(pdev, ADF_DEVICE_FUSECTL_OFFSET, - &hw_data->fuses); + &hw_data->fuses[ADF_FUSECTL0]); pci_read_config_dword(pdev, ADF_C62X_SOFTSTRAP_CSR_OFFSET, &hw_data->straps); @@ -169,7 +169,7 @@ static int adf_probe(struct pci_dev *pdev, const struct pci_device_id *ent) hw_data->accel_capabilities_mask = hw_data->get_accel_cap(accel_dev); /* Find and map all the device's BARS */ - i = (hw_data->fuses & ADF_DEVICE_FUSECTL_MASK) ? 1 : 0; + i = (hw_data->fuses[ADF_FUSECTL0] & ADF_DEVICE_FUSECTL_MASK) ? 1 : 0; bar_mask = pci_select_bars(pdev, IORESOURCE_MEM); for_each_set_bit(bar_nr, &bar_mask, ADF_PCI_MAX_BARS * 2) { struct adf_bar *bar = &accel_pci_dev->pci_bars[i++]; diff --git a/drivers/crypto/intel/qat/qat_common/adf_accel_devices.h b/drivers/crypto/intel/qat/qat_common/adf_accel_devices.h index 0509174d6030..dc21551153cb 100644 --- a/drivers/crypto/intel/qat/qat_common/adf_accel_devices.h +++ b/drivers/crypto/intel/qat/qat_common/adf_accel_devices.h @@ -53,6 +53,16 @@ enum adf_accel_capabilities { ADF_ACCEL_CAPABILITIES_RANDOM_NUMBER = 128 }; +enum adf_fuses { + ADF_FUSECTL0, + ADF_FUSECTL1, + ADF_FUSECTL2, + ADF_FUSECTL3, + ADF_FUSECTL4, + ADF_FUSECTL5, + ADF_MAX_FUSES +}; + struct adf_bar { resource_size_t base_addr; void __iomem *virt_addr; @@ -345,7 +355,7 @@ struct adf_hw_device_data { struct qat_migdev_ops vfmig_ops; const char *fw_name; const char *fw_mmp_name; - u32 fuses; + u32 fuses[ADF_MAX_FUSES]; u32 straps; u32 accel_capabilities_mask; u32 extended_dc_capabilities; diff --git a/drivers/crypto/intel/qat/qat_common/adf_gen2_hw_data.c b/drivers/crypto/intel/qat/qat_common/adf_gen2_hw_data.c index 1f64bf49b221..2b263442c856 100644 --- a/drivers/crypto/intel/qat/qat_common/adf_gen2_hw_data.c +++ b/drivers/crypto/intel/qat/qat_common/adf_gen2_hw_data.c @@ -115,8 +115,8 @@ u32 adf_gen2_get_accel_cap(struct adf_accel_dev *accel_dev) { struct adf_hw_device_data *hw_data = accel_dev->hw_device; struct pci_dev *pdev = accel_dev->accel_pci_dev.pci_dev; + u32 fuses = hw_data->fuses[ADF_FUSECTL0]; u32 straps = hw_data->straps; - u32 fuses = hw_data->fuses; u32 legfuses; u32 capabilities = ICP_ACCEL_CAPABILITIES_CRYPTO_SYMMETRIC | ICP_ACCEL_CAPABILITIES_CRYPTO_ASYMMETRIC | diff --git a/drivers/crypto/intel/qat/qat_dh895xcc/adf_dh895xcc_hw_data.c b/drivers/crypto/intel/qat/qat_dh895xcc/adf_dh895xcc_hw_data.c index c0661ff5e929..e48bcf1818cd 100644 --- a/drivers/crypto/intel/qat/qat_dh895xcc/adf_dh895xcc_hw_data.c +++ b/drivers/crypto/intel/qat/qat_dh895xcc/adf_dh895xcc_hw_data.c @@ -29,7 +29,7 @@ static struct adf_hw_device_class dh895xcc_class = { static u32 get_accel_mask(struct adf_hw_device_data *self) { - u32 fuses = self->fuses; + u32 fuses = self->fuses[ADF_FUSECTL0]; return ~fuses >> ADF_DH895XCC_ACCELERATORS_REG_OFFSET & ADF_DH895XCC_ACCELERATORS_MASK; @@ -37,7 +37,7 @@ static u32 get_accel_mask(struct adf_hw_device_data *self) static u32 get_ae_mask(struct adf_hw_device_data *self) { - u32 fuses = self->fuses; + u32 fuses = self->fuses[ADF_FUSECTL0]; return ~fuses & ADF_DH895XCC_ACCELENGINES_MASK; } @@ -99,7 +99,7 @@ static u32 get_accel_cap(struct adf_accel_dev *accel_dev) static enum dev_sku_info get_sku(struct adf_hw_device_data *self) { - int sku = (self->fuses & ADF_DH895XCC_FUSECTL_SKU_MASK) + int sku = (self->fuses[ADF_FUSECTL0] & ADF_DH895XCC_FUSECTL_SKU_MASK) >> ADF_DH895XCC_FUSECTL_SKU_SHIFT; switch (sku) { diff --git a/drivers/crypto/intel/qat/qat_dh895xcc/adf_drv.c b/drivers/crypto/intel/qat/qat_dh895xcc/adf_drv.c index 3137fc3b5cf6..07e9d7e52861 100644 --- a/drivers/crypto/intel/qat/qat_dh895xcc/adf_drv.c +++ b/drivers/crypto/intel/qat/qat_dh895xcc/adf_drv.c @@ -126,7 +126,7 @@ static int adf_probe(struct pci_dev *pdev, const struct pci_device_id *ent) adf_init_hw_data_dh895xcc(accel_dev->hw_device); pci_read_config_byte(pdev, PCI_REVISION_ID, &accel_pci_dev->revid); pci_read_config_dword(pdev, ADF_DEVICE_FUSECTL_OFFSET, - &hw_data->fuses); + &hw_data->fuses[ADF_FUSECTL0]); /* Get Accelerators and Accelerators Engines masks */ hw_data->accel_mask = hw_data->get_accel_mask(hw_data); -- 2.51.0