From 9ae0c92fec69374c6db8dddb0df00d86b9afa5da Mon Sep 17 00:00:00 2001 From: Herbert Xu Date: Fri, 11 Apr 2025 09:26:47 +0800 Subject: [PATCH 01/16] crypto: scomp - Fix wild memory accesses in scomp_free_streams In order to use scomp_free_streams to free the partially allocted streams in the allocation error path, move the alg->stream assignment to the beginning. Also check for error pointers in scomp_free_streams before freeing the ctx. Finally set alg->stream to NULL to not break subsequent attempts to allocate the streams. Fixes: 3d72ad46a23a ("crypto: acomp - Move stream management into scomp layer") Reported-by: syzkaller Co-developed-by: Kuniyuki Iwashima Signed-off-by: Kuniyuki Iwashima Co-developed-by: Herbert Xu Signed-off-by: Herbert Xu --- crypto/scompress.c | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/crypto/scompress.c b/crypto/scompress.c index f67ce38d203d..5762fcc63b51 100644 --- a/crypto/scompress.c +++ b/crypto/scompress.c @@ -111,13 +111,14 @@ static void scomp_free_streams(struct scomp_alg *alg) struct crypto_acomp_stream __percpu *stream = alg->stream; int i; + alg->stream = NULL; if (!stream) return; for_each_possible_cpu(i) { struct crypto_acomp_stream *ps = per_cpu_ptr(stream, i); - if (!ps->ctx) + if (IS_ERR_OR_NULL(ps->ctx)) break; alg->free_ctx(ps->ctx); @@ -135,6 +136,8 @@ static int scomp_alloc_streams(struct scomp_alg *alg) if (!stream) return -ENOMEM; + alg->stream = stream; + for_each_possible_cpu(i) { struct crypto_acomp_stream *ps = per_cpu_ptr(stream, i); @@ -146,8 +149,6 @@ static int scomp_alloc_streams(struct scomp_alg *alg) spin_lock_init(&ps->lock); } - - alg->stream = stream; return 0; } -- 2.51.0 From b2e689baf220408aff8ee5dfb4edb0817e1632bb Mon Sep 17 00:00:00 2001 From: Herbert Xu Date: Fri, 11 Apr 2025 15:14:18 +0800 Subject: [PATCH 02/16] crypto: ahash - Disable request chaining Disable hash request chaining in case a driver that copies an ahash_request object by hand accidentally triggers chaining. Reported-by: Manorit Chawdhry Fixes: f2ffe5a9183d ("crypto: hash - Add request chaining API") Signed-off-by: Herbert Xu Tested-by: Manorit Chawdhry Signed-off-by: Herbert Xu --- crypto/ahash.c | 76 +--------------------------------- include/crypto/hash.h | 6 ++- include/crypto/internal/hash.h | 2 +- 3 files changed, 7 insertions(+), 77 deletions(-) diff --git a/crypto/ahash.c b/crypto/ahash.c index 9f57b925b116..2d9eec2b2b1c 100644 --- a/crypto/ahash.c +++ b/crypto/ahash.c @@ -315,16 +315,7 @@ EXPORT_SYMBOL_GPL(crypto_ahash_setkey); static bool ahash_request_hasvirt(struct ahash_request *req) { - struct ahash_request *r2; - - if (ahash_request_isvirt(req)) - return true; - - list_for_each_entry(r2, &req->base.list, base.list) - if (ahash_request_isvirt(r2)) - return true; - - return false; + return ahash_request_isvirt(req); } static int ahash_reqchain_virt(struct ahash_save_req_state *state, @@ -472,7 +463,6 @@ static int ahash_do_req_chain(struct ahash_request *req, bool update = op == crypto_ahash_alg(tfm)->update; struct ahash_save_req_state *state; struct ahash_save_req_state state0; - struct ahash_request *r2; u8 *page = NULL; int err; @@ -509,7 +499,6 @@ static int ahash_do_req_chain(struct ahash_request *req, state->offset = 0; state->nbytes = 0; INIT_LIST_HEAD(&state->head); - list_splice_init(&req->base.list, &state->head); if (page) sg_init_one(&state->sg, page, PAGE_SIZE); @@ -540,9 +529,6 @@ out_free_page: out_set_chain: req->base.err = err; - list_for_each_entry(r2, &req->base.list, base.list) - r2->base.err = err; - return err; } @@ -551,19 +537,10 @@ int crypto_ahash_init(struct ahash_request *req) struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); if (likely(tfm->using_shash)) { - struct ahash_request *r2; int err; err = crypto_shash_init(prepare_shash_desc(req, tfm)); req->base.err = err; - - list_for_each_entry(r2, &req->base.list, base.list) { - struct shash_desc *desc; - - desc = prepare_shash_desc(r2, tfm); - r2->base.err = crypto_shash_init(desc); - } - return err; } @@ -620,19 +597,10 @@ int crypto_ahash_update(struct ahash_request *req) struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); if (likely(tfm->using_shash)) { - struct ahash_request *r2; int err; err = shash_ahash_update(req, ahash_request_ctx(req)); req->base.err = err; - - list_for_each_entry(r2, &req->base.list, base.list) { - struct shash_desc *desc; - - desc = ahash_request_ctx(r2); - r2->base.err = shash_ahash_update(r2, desc); - } - return err; } @@ -645,19 +613,10 @@ int crypto_ahash_final(struct ahash_request *req) struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); if (likely(tfm->using_shash)) { - struct ahash_request *r2; int err; err = crypto_shash_final(ahash_request_ctx(req), req->result); req->base.err = err; - - list_for_each_entry(r2, &req->base.list, base.list) { - struct shash_desc *desc; - - desc = ahash_request_ctx(r2); - r2->base.err = crypto_shash_final(desc, r2->result); - } - return err; } @@ -670,19 +629,10 @@ int crypto_ahash_finup(struct ahash_request *req) struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); if (likely(tfm->using_shash)) { - struct ahash_request *r2; int err; err = shash_ahash_finup(req, ahash_request_ctx(req)); req->base.err = err; - - list_for_each_entry(r2, &req->base.list, base.list) { - struct shash_desc *desc; - - desc = ahash_request_ctx(r2); - r2->base.err = shash_ahash_finup(r2, desc); - } - return err; } @@ -757,19 +707,10 @@ int crypto_ahash_digest(struct ahash_request *req) struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); if (likely(tfm->using_shash)) { - struct ahash_request *r2; int err; err = shash_ahash_digest(req, prepare_shash_desc(req, tfm)); req->base.err = err; - - list_for_each_entry(r2, &req->base.list, base.list) { - struct shash_desc *desc; - - desc = prepare_shash_desc(r2, tfm); - r2->base.err = shash_ahash_digest(r2, desc); - } - return err; } @@ -1133,20 +1074,5 @@ int ahash_register_instance(struct crypto_template *tmpl, } EXPORT_SYMBOL_GPL(ahash_register_instance); -void ahash_request_free(struct ahash_request *req) -{ - struct ahash_request *tmp; - struct ahash_request *r2; - - if (unlikely(!req)) - return; - - list_for_each_entry_safe(r2, tmp, &req->base.list, base.list) - kfree_sensitive(r2); - - kfree_sensitive(req); -} -EXPORT_SYMBOL_GPL(ahash_request_free); - MODULE_LICENSE("GPL"); MODULE_DESCRIPTION("Asynchronous cryptographic hash type"); diff --git a/include/crypto/hash.h b/include/crypto/hash.h index 2aa83ee0ec98..a67988316d06 100644 --- a/include/crypto/hash.h +++ b/include/crypto/hash.h @@ -10,6 +10,7 @@ #include #include +#include #include /* Set this bit for virtual address instead of SG list. */ @@ -581,7 +582,10 @@ static inline struct ahash_request *ahash_request_alloc_noprof( * ahash_request_free() - zeroize and free the request data structure * @req: request data structure cipher handle to be freed */ -void ahash_request_free(struct ahash_request *req); +static inline void ahash_request_free(struct ahash_request *req) +{ + kfree_sensitive(req); +} static inline struct ahash_request *ahash_request_cast( struct crypto_async_request *req) diff --git a/include/crypto/internal/hash.h b/include/crypto/internal/hash.h index 485e22cf517e..052ac7924af3 100644 --- a/include/crypto/internal/hash.h +++ b/include/crypto/internal/hash.h @@ -249,7 +249,7 @@ static inline struct crypto_shash *__crypto_shash_cast(struct crypto_tfm *tfm) static inline bool ahash_request_chained(struct ahash_request *req) { - return crypto_request_chained(&req->base); + return false; } static inline bool ahash_request_isvirt(struct ahash_request *req) -- 2.51.0 From 5976fe19e240a681b5dd13362c8507e4c66b466b Mon Sep 17 00:00:00 2001 From: Herbert Xu Date: Sat, 12 Apr 2025 13:36:51 +0800 Subject: [PATCH 03/16] Revert "crypto: testmgr - Add multibuffer acomp testing" This reverts commit 99585c2192cb1ce212876e82ef01d1c98c7f4699. Remove the acomp multibuffer tests so that the interface can be redesigned. Signed-off-by: Herbert Xu --- crypto/testmgr.c | 147 +++++++++++++++++++++-------------------------- 1 file changed, 64 insertions(+), 83 deletions(-) diff --git a/crypto/testmgr.c b/crypto/testmgr.c index abd609d4c8ef..82977ea25db3 100644 --- a/crypto/testmgr.c +++ b/crypto/testmgr.c @@ -58,9 +58,6 @@ module_param(fuzz_iterations, uint, 0644); MODULE_PARM_DESC(fuzz_iterations, "number of fuzz test iterations"); #endif -/* Multibuffer is unlimited. Set arbitrary limit for testing. */ -#define MAX_MB_MSGS 16 - #ifdef CONFIG_CRYPTO_MANAGER_DISABLE_TESTS /* a perfect nop */ @@ -3329,48 +3326,27 @@ static int test_acomp(struct crypto_acomp *tfm, int ctcount, int dtcount) { const char *algo = crypto_tfm_alg_driver_name(crypto_acomp_tfm(tfm)); - struct scatterlist *src = NULL, *dst = NULL; - struct acomp_req *reqs[MAX_MB_MSGS] = {}; - char *decomp_out[MAX_MB_MSGS] = {}; - char *output[MAX_MB_MSGS] = {}; - struct crypto_wait wait; - struct acomp_req *req; - int ret = -ENOMEM; unsigned int i; + char *output, *decomp_out; + int ret; + struct scatterlist src, dst; + struct acomp_req *req; + struct crypto_wait wait; - src = kmalloc_array(MAX_MB_MSGS, sizeof(*src), GFP_KERNEL); - if (!src) - goto out; - dst = kmalloc_array(MAX_MB_MSGS, sizeof(*dst), GFP_KERNEL); - if (!dst) - goto out; - - for (i = 0; i < MAX_MB_MSGS; i++) { - reqs[i] = acomp_request_alloc(tfm); - if (!reqs[i]) - goto out; - - acomp_request_set_callback(reqs[i], - CRYPTO_TFM_REQ_MAY_SLEEP | - CRYPTO_TFM_REQ_MAY_BACKLOG, - crypto_req_done, &wait); - if (i) - acomp_request_chain(reqs[i], reqs[0]); - - output[i] = kmalloc(COMP_BUF_SIZE, GFP_KERNEL); - if (!output[i]) - goto out; + output = kmalloc(COMP_BUF_SIZE, GFP_KERNEL); + if (!output) + return -ENOMEM; - decomp_out[i] = kmalloc(COMP_BUF_SIZE, GFP_KERNEL); - if (!decomp_out[i]) - goto out; + decomp_out = kmalloc(COMP_BUF_SIZE, GFP_KERNEL); + if (!decomp_out) { + kfree(output); + return -ENOMEM; } for (i = 0; i < ctcount; i++) { unsigned int dlen = COMP_BUF_SIZE; int ilen = ctemplate[i].inlen; void *input_vec; - int j; input_vec = kmemdup(ctemplate[i].input, ilen, GFP_KERNEL); if (!input_vec) { @@ -3378,61 +3354,70 @@ static int test_acomp(struct crypto_acomp *tfm, goto out; } + memset(output, 0, dlen); crypto_init_wait(&wait); - sg_init_one(src, input_vec, ilen); + sg_init_one(&src, input_vec, ilen); + sg_init_one(&dst, output, dlen); - for (j = 0; j < MAX_MB_MSGS; j++) { - sg_init_one(dst + j, output[j], dlen); - acomp_request_set_params(reqs[j], src, dst + j, ilen, dlen); + req = acomp_request_alloc(tfm); + if (!req) { + pr_err("alg: acomp: request alloc failed for %s\n", + algo); + kfree(input_vec); + ret = -ENOMEM; + goto out; } - req = reqs[0]; + acomp_request_set_params(req, &src, &dst, ilen, dlen); + acomp_request_set_callback(req, CRYPTO_TFM_REQ_MAY_BACKLOG, + crypto_req_done, &wait); + ret = crypto_wait_req(crypto_acomp_compress(req), &wait); if (ret) { pr_err("alg: acomp: compression failed on test %d for %s: ret=%d\n", i + 1, algo, -ret); kfree(input_vec); + acomp_request_free(req); goto out; } ilen = req->dlen; dlen = COMP_BUF_SIZE; + sg_init_one(&src, output, ilen); + sg_init_one(&dst, decomp_out, dlen); crypto_init_wait(&wait); - for (j = 0; j < MAX_MB_MSGS; j++) { - sg_init_one(src + j, output[j], ilen); - sg_init_one(dst + j, decomp_out[j], dlen); - acomp_request_set_params(reqs[j], src + j, dst + j, ilen, dlen); - } - - crypto_wait_req(crypto_acomp_decompress(req), &wait); - for (j = 0; j < MAX_MB_MSGS; j++) { - ret = reqs[j]->base.err; - if (ret) { - pr_err("alg: acomp: compression failed on test %d (%d) for %s: ret=%d\n", - i + 1, j, algo, -ret); - kfree(input_vec); - goto out; - } + acomp_request_set_params(req, &src, &dst, ilen, dlen); - if (reqs[j]->dlen != ctemplate[i].inlen) { - pr_err("alg: acomp: Compression test %d (%d) failed for %s: output len = %d\n", - i + 1, j, algo, reqs[j]->dlen); - ret = -EINVAL; - kfree(input_vec); - goto out; - } + ret = crypto_wait_req(crypto_acomp_decompress(req), &wait); + if (ret) { + pr_err("alg: acomp: compression failed on test %d for %s: ret=%d\n", + i + 1, algo, -ret); + kfree(input_vec); + acomp_request_free(req); + goto out; + } - if (memcmp(input_vec, decomp_out[j], reqs[j]->dlen)) { - pr_err("alg: acomp: Compression test %d (%d) failed for %s\n", - i + 1, j, algo); - hexdump(output[j], reqs[j]->dlen); - ret = -EINVAL; - kfree(input_vec); - goto out; - } + if (req->dlen != ctemplate[i].inlen) { + pr_err("alg: acomp: Compression test %d failed for %s: output len = %d\n", + i + 1, algo, req->dlen); + ret = -EINVAL; + kfree(input_vec); + acomp_request_free(req); + goto out; + } + + if (memcmp(input_vec, decomp_out, req->dlen)) { + pr_err("alg: acomp: Compression test %d failed for %s\n", + i + 1, algo); + hexdump(output, req->dlen); + ret = -EINVAL; + kfree(input_vec); + acomp_request_free(req); + goto out; } kfree(input_vec); + acomp_request_free(req); } for (i = 0; i < dtcount; i++) { @@ -3446,9 +3431,10 @@ static int test_acomp(struct crypto_acomp *tfm, goto out; } + memset(output, 0, dlen); crypto_init_wait(&wait); - sg_init_one(src, input_vec, ilen); - sg_init_one(dst, output[0], dlen); + sg_init_one(&src, input_vec, ilen); + sg_init_one(&dst, output, dlen); req = acomp_request_alloc(tfm); if (!req) { @@ -3459,7 +3445,7 @@ static int test_acomp(struct crypto_acomp *tfm, goto out; } - acomp_request_set_params(req, src, dst, ilen, dlen); + acomp_request_set_params(req, &src, &dst, ilen, dlen); acomp_request_set_callback(req, CRYPTO_TFM_REQ_MAY_BACKLOG, crypto_req_done, &wait); @@ -3481,10 +3467,10 @@ static int test_acomp(struct crypto_acomp *tfm, goto out; } - if (memcmp(output[0], dtemplate[i].output, req->dlen)) { + if (memcmp(output, dtemplate[i].output, req->dlen)) { pr_err("alg: acomp: Decompression test %d failed for %s\n", i + 1, algo); - hexdump(output[0], req->dlen); + hexdump(output, req->dlen); ret = -EINVAL; kfree(input_vec); acomp_request_free(req); @@ -3498,13 +3484,8 @@ static int test_acomp(struct crypto_acomp *tfm, ret = 0; out: - acomp_request_free(reqs[0]); - for (i = 0; i < MAX_MB_MSGS; i++) { - kfree(output[i]); - kfree(decomp_out[i]); - } - kfree(dst); - kfree(src); + kfree(decomp_out); + kfree(output); return ret; } -- 2.51.0 From 78e2846aa4b29b44368377842d2369e0198022b7 Mon Sep 17 00:00:00 2001 From: Herbert Xu Date: Sat, 12 Apr 2025 13:36:53 +0800 Subject: [PATCH 04/16] crypto: deflate - Remove request chaining Remove request chaining support from deflate. Signed-off-by: Herbert Xu --- crypto/deflate.c | 14 -------------- 1 file changed, 14 deletions(-) diff --git a/crypto/deflate.c b/crypto/deflate.c index bc76c343a0cf..57d7af4dfdfb 100644 --- a/crypto/deflate.c +++ b/crypto/deflate.c @@ -111,7 +111,6 @@ static int deflate_compress(struct acomp_req *req) { struct crypto_acomp_stream *s; struct deflate_stream *ds; - struct acomp_req *r2; int err; s = crypto_acomp_lock_stream_bh(&deflate_streams); @@ -126,12 +125,6 @@ static int deflate_compress(struct acomp_req *req) } err = deflate_compress_one(req, ds); - req->base.err = err; - - list_for_each_entry(r2, &req->base.list, base.list) { - zlib_deflateReset(&ds->stream); - r2->base.err = deflate_compress_one(r2, ds); - } out: crypto_acomp_unlock_stream_bh(s); @@ -199,7 +192,6 @@ static int deflate_decompress(struct acomp_req *req) { struct crypto_acomp_stream *s; struct deflate_stream *ds; - struct acomp_req *r2; int err; s = crypto_acomp_lock_stream_bh(&deflate_streams); @@ -212,12 +204,6 @@ static int deflate_decompress(struct acomp_req *req) } err = deflate_decompress_one(req, ds); - req->base.err = err; - - list_for_each_entry(r2, &req->base.list, base.list) { - zlib_inflateReset(&ds->stream); - r2->base.err = deflate_decompress_one(r2, ds); - } out: crypto_acomp_unlock_stream_bh(s); -- 2.51.0 From 64929fe8c0a43508eee952cf57903a61c52601e7 Mon Sep 17 00:00:00 2001 From: Herbert Xu Date: Sat, 12 Apr 2025 13:36:55 +0800 Subject: [PATCH 05/16] crypto: acomp - Remove request chaining Request chaining requires the user to do too much book keeping. Remove it from acomp. Signed-off-by: Herbert Xu --- crypto/acompress.c | 117 ++++++++-------------------- crypto/scompress.c | 18 +---- include/crypto/acompress.h | 14 ---- include/crypto/internal/acompress.h | 5 -- 4 files changed, 35 insertions(+), 119 deletions(-) diff --git a/crypto/acompress.c b/crypto/acompress.c index 5d0b8b8b84f6..b682a88781f0 100644 --- a/crypto/acompress.c +++ b/crypto/acompress.c @@ -177,7 +177,6 @@ static void acomp_save_req(struct acomp_req *req, crypto_completion_t cplt) state->data = req->base.data; req->base.complete = cplt; req->base.data = state; - state->req0 = req; } static void acomp_restore_req(struct acomp_req *req) @@ -188,23 +187,20 @@ static void acomp_restore_req(struct acomp_req *req) req->base.data = state->data; } -static void acomp_reqchain_virt(struct acomp_req_chain *state, int err) +static void acomp_reqchain_virt(struct acomp_req *req) { - struct acomp_req *req = state->cur; + struct acomp_req_chain *state = &req->chain; unsigned int slen = req->slen; unsigned int dlen = req->dlen; - req->base.err = err; - state = &req->chain; - if (state->flags & CRYPTO_ACOMP_REQ_SRC_VIRT) acomp_request_set_src_dma(req, state->src, slen); else if (state->flags & CRYPTO_ACOMP_REQ_SRC_FOLIO) - acomp_request_set_src_folio(req, state->sfolio, state->soff, slen); + acomp_request_set_src_folio(req, state->sfolio, req->soff, slen); if (state->flags & CRYPTO_ACOMP_REQ_DST_VIRT) acomp_request_set_dst_dma(req, state->dst, dlen); else if (state->flags & CRYPTO_ACOMP_REQ_DST_FOLIO) - acomp_request_set_dst_folio(req, state->dfolio, state->doff, dlen); + acomp_request_set_dst_folio(req, state->dfolio, req->doff, dlen); } static void acomp_virt_to_sg(struct acomp_req *req) @@ -229,7 +225,6 @@ static void acomp_virt_to_sg(struct acomp_req *req) size_t off = req->soff; state->sfolio = folio; - state->soff = off; sg_init_table(&state->ssg, 1); sg_set_page(&state->ssg, folio_page(folio, off / PAGE_SIZE), slen, off % PAGE_SIZE); @@ -249,7 +244,6 @@ static void acomp_virt_to_sg(struct acomp_req *req) size_t off = req->doff; state->dfolio = folio; - state->doff = off; sg_init_table(&state->dsg, 1); sg_set_page(&state->dsg, folio_page(folio, off / PAGE_SIZE), dlen, off % PAGE_SIZE); @@ -257,8 +251,7 @@ static void acomp_virt_to_sg(struct acomp_req *req) } } -static int acomp_do_nondma(struct acomp_req_chain *state, - struct acomp_req *req) +static int acomp_do_nondma(struct acomp_req *req, bool comp) { u32 keep = CRYPTO_ACOMP_REQ_SRC_VIRT | CRYPTO_ACOMP_REQ_SRC_NONDMA | @@ -275,7 +268,7 @@ static int acomp_do_nondma(struct acomp_req_chain *state, fbreq->slen = req->slen; fbreq->dlen = req->dlen; - if (state->op == crypto_acomp_reqtfm(req)->compress) + if (comp) err = crypto_acomp_compress(fbreq); else err = crypto_acomp_decompress(fbreq); @@ -284,114 +277,70 @@ static int acomp_do_nondma(struct acomp_req_chain *state, return err; } -static int acomp_do_one_req(struct acomp_req_chain *state, - struct acomp_req *req) +static int acomp_do_one_req(struct acomp_req *req, bool comp) { - state->cur = req; - if (acomp_request_isnondma(req)) - return acomp_do_nondma(state, req); + return acomp_do_nondma(req, comp); acomp_virt_to_sg(req); - return state->op(req); + return comp ? crypto_acomp_reqtfm(req)->compress(req) : + crypto_acomp_reqtfm(req)->decompress(req); } -static int acomp_reqchain_finish(struct acomp_req *req0, int err, u32 mask) +static int acomp_reqchain_finish(struct acomp_req *req, int err) { - struct acomp_req_chain *state = req0->base.data; - struct acomp_req *req = state->cur; - struct acomp_req *n; - - acomp_reqchain_virt(state, err); - - if (req != req0) - list_add_tail(&req->base.list, &req0->base.list); - - list_for_each_entry_safe(req, n, &state->head, base.list) { - list_del_init(&req->base.list); - - req->base.flags &= mask; - req->base.complete = acomp_reqchain_done; - req->base.data = state; - - err = acomp_do_one_req(state, req); - - if (err == -EINPROGRESS) { - if (!list_empty(&state->head)) - err = -EBUSY; - goto out; - } - - if (err == -EBUSY) - goto out; - - acomp_reqchain_virt(state, err); - list_add_tail(&req->base.list, &req0->base.list); - } - - acomp_restore_req(req0); - -out: + acomp_reqchain_virt(req); + acomp_restore_req(req); return err; } static void acomp_reqchain_done(void *data, int err) { - struct acomp_req_chain *state = data; - crypto_completion_t compl = state->compl; + struct acomp_req *req = data; + crypto_completion_t compl; - data = state->data; + compl = req->chain.compl; + data = req->chain.data; - if (err == -EINPROGRESS) { - if (!list_empty(&state->head)) - return; + if (err == -EINPROGRESS) goto notify; - } - err = acomp_reqchain_finish(state->req0, err, - CRYPTO_TFM_REQ_MAY_BACKLOG); - if (err == -EBUSY) - return; + err = acomp_reqchain_finish(req, err); notify: compl(data, err); } -static int acomp_do_req_chain(struct acomp_req *req, - int (*op)(struct acomp_req *req)) +static int acomp_do_req_chain(struct acomp_req *req, bool comp) { - struct crypto_acomp *tfm = crypto_acomp_reqtfm(req); - struct acomp_req_chain *state; int err; - if (crypto_acomp_req_chain(tfm) || - (!acomp_request_chained(req) && acomp_request_issg(req))) - return op(req); - acomp_save_req(req, acomp_reqchain_done); - state = req->base.data; - state->op = op; - state->src = NULL; - INIT_LIST_HEAD(&state->head); - list_splice_init(&req->base.list, &state->head); - - err = acomp_do_one_req(state, req); + err = acomp_do_one_req(req, comp); if (err == -EBUSY || err == -EINPROGRESS) - return -EBUSY; + return err; - return acomp_reqchain_finish(req, err, ~0); + return acomp_reqchain_finish(req, err); } int crypto_acomp_compress(struct acomp_req *req) { - return acomp_do_req_chain(req, crypto_acomp_reqtfm(req)->compress); + struct crypto_acomp *tfm = crypto_acomp_reqtfm(req); + + if (crypto_acomp_req_chain(tfm) || acomp_request_issg(req)) + crypto_acomp_reqtfm(req)->compress(req); + return acomp_do_req_chain(req, true); } EXPORT_SYMBOL_GPL(crypto_acomp_compress); int crypto_acomp_decompress(struct acomp_req *req) { - return acomp_do_req_chain(req, crypto_acomp_reqtfm(req)->decompress); + struct crypto_acomp *tfm = crypto_acomp_reqtfm(req); + + if (crypto_acomp_req_chain(tfm) || acomp_request_issg(req)) + crypto_acomp_reqtfm(req)->decompress(req); + return acomp_do_req_chain(req, false); } EXPORT_SYMBOL_GPL(crypto_acomp_decompress); diff --git a/crypto/scompress.c b/crypto/scompress.c index 4a2b3933aa95..7ade3f2fee7e 100644 --- a/crypto/scompress.c +++ b/crypto/scompress.c @@ -284,28 +284,14 @@ static int scomp_acomp_comp_decomp(struct acomp_req *req, int dir) return ret; } -static int scomp_acomp_chain(struct acomp_req *req, int dir) -{ - struct acomp_req *r2; - int err; - - err = scomp_acomp_comp_decomp(req, dir); - req->base.err = err; - - list_for_each_entry(r2, &req->base.list, base.list) - r2->base.err = scomp_acomp_comp_decomp(r2, dir); - - return err; -} - static int scomp_acomp_compress(struct acomp_req *req) { - return scomp_acomp_chain(req, 1); + return scomp_acomp_comp_decomp(req, 1); } static int scomp_acomp_decompress(struct acomp_req *req) { - return scomp_acomp_chain(req, 0); + return scomp_acomp_comp_decomp(req, 0); } static void crypto_exit_scomp_ops_async(struct crypto_tfm *tfm) diff --git a/include/crypto/acompress.h b/include/crypto/acompress.h index f0e01ff77d92..080e134df35c 100644 --- a/include/crypto/acompress.h +++ b/include/crypto/acompress.h @@ -52,10 +52,6 @@ struct acomp_req; struct folio; struct acomp_req_chain { - struct list_head head; - struct acomp_req *req0; - struct acomp_req *cur; - int (*op)(struct acomp_req *req); crypto_completion_t compl; void *data; struct scatterlist ssg; @@ -68,8 +64,6 @@ struct acomp_req_chain { u8 *dst; struct folio *dfolio; }; - size_t soff; - size_t doff; u32 flags; }; @@ -343,8 +337,6 @@ static inline void acomp_request_set_callback(struct acomp_req *req, req->base.data = data; req->base.flags &= keep; req->base.flags |= flgs & ~keep; - - crypto_reqchain_init(&req->base); } /** @@ -552,12 +544,6 @@ static inline void acomp_request_set_dst_folio(struct acomp_req *req, req->base.flags |= CRYPTO_ACOMP_REQ_DST_FOLIO; } -static inline void acomp_request_chain(struct acomp_req *req, - struct acomp_req *head) -{ - crypto_request_chain(&req->base, &head->base); -} - /** * crypto_acomp_compress() -- Invoke asynchronous compress operation * diff --git a/include/crypto/internal/acompress.h b/include/crypto/internal/acompress.h index fbbff9a8a2d9..960cdd1f3a57 100644 --- a/include/crypto/internal/acompress.h +++ b/include/crypto/internal/acompress.h @@ -151,11 +151,6 @@ void crypto_unregister_acomp(struct acomp_alg *alg); int crypto_register_acomps(struct acomp_alg *algs, int count); void crypto_unregister_acomps(struct acomp_alg *algs, int count); -static inline bool acomp_request_chained(struct acomp_req *req) -{ - return crypto_request_chained(&req->base); -} - static inline bool acomp_request_issg(struct acomp_req *req) { return !(req->base.flags & (CRYPTO_ACOMP_REQ_SRC_VIRT | -- 2.51.0 From 69e5a1228d43f76f7a6a2e73ddd55c543fac8826 Mon Sep 17 00:00:00 2001 From: Herbert Xu Date: Sat, 12 Apr 2025 13:36:57 +0800 Subject: [PATCH 06/16] Revert "crypto: tcrypt - Restore multibuffer ahash tests" This reverts commit c664f034172705a75f3f8a0c409b9bf95b633093. Remove the multibuffer ahash speed tests again. Signed-off-by: Herbert Xu --- crypto/tcrypt.c | 231 ------------------------------------------------ 1 file changed, 231 deletions(-) diff --git a/crypto/tcrypt.c b/crypto/tcrypt.c index 96f4a66be14c..879fc21dcc16 100644 --- a/crypto/tcrypt.c +++ b/crypto/tcrypt.c @@ -716,207 +716,6 @@ static inline int do_one_ahash_op(struct ahash_request *req, int ret) return crypto_wait_req(ret, wait); } -struct test_mb_ahash_data { - struct scatterlist sg[XBUFSIZE]; - char result[64]; - struct ahash_request *req; - struct crypto_wait wait; - char *xbuf[XBUFSIZE]; -}; - -static inline int do_mult_ahash_op(struct test_mb_ahash_data *data, u32 num_mb, - int *rc) -{ - int i, err; - - /* Fire up a bunch of concurrent requests */ - err = crypto_ahash_digest(data[0].req); - - /* Wait for all requests to finish */ - err = crypto_wait_req(err, &data[0].wait); - if (num_mb < 2) - return err; - - for (i = 0; i < num_mb; i++) { - rc[i] = ahash_request_err(data[i].req); - if (rc[i]) { - pr_info("concurrent request %d error %d\n", i, rc[i]); - err = rc[i]; - } - } - - return err; -} - -static int test_mb_ahash_jiffies(struct test_mb_ahash_data *data, int blen, - int secs, u32 num_mb) -{ - unsigned long start, end; - int bcount; - int ret = 0; - int *rc; - - rc = kcalloc(num_mb, sizeof(*rc), GFP_KERNEL); - if (!rc) - return -ENOMEM; - - for (start = jiffies, end = start + secs * HZ, bcount = 0; - time_before(jiffies, end); bcount++) { - ret = do_mult_ahash_op(data, num_mb, rc); - if (ret) - goto out; - } - - pr_cont("%d operations in %d seconds (%llu bytes)\n", - bcount * num_mb, secs, (u64)bcount * blen * num_mb); - -out: - kfree(rc); - return ret; -} - -static int test_mb_ahash_cycles(struct test_mb_ahash_data *data, int blen, - u32 num_mb) -{ - unsigned long cycles = 0; - int ret = 0; - int i; - int *rc; - - rc = kcalloc(num_mb, sizeof(*rc), GFP_KERNEL); - if (!rc) - return -ENOMEM; - - /* Warm-up run. */ - for (i = 0; i < 4; i++) { - ret = do_mult_ahash_op(data, num_mb, rc); - if (ret) - goto out; - } - - /* The real thing. */ - for (i = 0; i < 8; i++) { - cycles_t start, end; - - start = get_cycles(); - ret = do_mult_ahash_op(data, num_mb, rc); - end = get_cycles(); - - if (ret) - goto out; - - cycles += end - start; - } - - pr_cont("1 operation in %lu cycles (%d bytes)\n", - (cycles + 4) / (8 * num_mb), blen); - -out: - kfree(rc); - return ret; -} - -static void test_mb_ahash_speed(const char *algo, unsigned int secs, - struct hash_speed *speed, u32 num_mb) -{ - struct test_mb_ahash_data *data; - struct crypto_ahash *tfm; - unsigned int i, j, k; - int ret; - - data = kcalloc(num_mb, sizeof(*data), GFP_KERNEL); - if (!data) - return; - - tfm = crypto_alloc_ahash(algo, 0, 0); - if (IS_ERR(tfm)) { - pr_err("failed to load transform for %s: %ld\n", - algo, PTR_ERR(tfm)); - goto free_data; - } - - for (i = 0; i < num_mb; ++i) { - if (testmgr_alloc_buf(data[i].xbuf)) - goto out; - - crypto_init_wait(&data[i].wait); - - data[i].req = ahash_request_alloc(tfm, GFP_KERNEL); - if (!data[i].req) { - pr_err("alg: hash: Failed to allocate request for %s\n", - algo); - goto out; - } - - - if (i) { - ahash_request_set_callback(data[i].req, 0, NULL, NULL); - ahash_request_chain(data[i].req, data[0].req); - } else - ahash_request_set_callback(data[0].req, 0, - crypto_req_done, - &data[0].wait); - - sg_init_table(data[i].sg, XBUFSIZE); - for (j = 0; j < XBUFSIZE; j++) { - sg_set_buf(data[i].sg + j, data[i].xbuf[j], PAGE_SIZE); - memset(data[i].xbuf[j], 0xff, PAGE_SIZE); - } - } - - pr_info("\ntesting speed of multibuffer %s (%s)\n", algo, - get_driver_name(crypto_ahash, tfm)); - - for (i = 0; speed[i].blen != 0; i++) { - /* For some reason this only tests digests. */ - if (speed[i].blen != speed[i].plen) - continue; - - if (speed[i].blen > XBUFSIZE * PAGE_SIZE) { - pr_err("template (%u) too big for tvmem (%lu)\n", - speed[i].blen, XBUFSIZE * PAGE_SIZE); - goto out; - } - - if (klen) - crypto_ahash_setkey(tfm, tvmem[0], klen); - - for (k = 0; k < num_mb; k++) - ahash_request_set_crypt(data[k].req, data[k].sg, - data[k].result, speed[i].blen); - - pr_info("test%3u " - "(%5u byte blocks,%5u bytes per update,%4u updates): ", - i, speed[i].blen, speed[i].plen, - speed[i].blen / speed[i].plen); - - if (secs) { - ret = test_mb_ahash_jiffies(data, speed[i].blen, secs, - num_mb); - cond_resched(); - } else { - ret = test_mb_ahash_cycles(data, speed[i].blen, num_mb); - } - - - if (ret) { - pr_err("At least one hashing failed ret=%d\n", ret); - break; - } - } - -out: - ahash_request_free(data[0].req); - - for (k = 0; k < num_mb; ++k) - testmgr_free_buf(data[k].xbuf); - - crypto_free_ahash(tfm); - -free_data: - kfree(data); -} - static int test_ahash_jiffies_digest(struct ahash_request *req, int blen, char *out, int secs) { @@ -2584,36 +2383,6 @@ static int do_test(const char *alg, u32 type, u32 mask, int m, u32 num_mb) test_ahash_speed("sm3", sec, generic_hash_speed_template); if (mode > 400 && mode < 500) break; fallthrough; - case 450: - test_mb_ahash_speed("sha1", sec, generic_hash_speed_template, - num_mb); - if (mode > 400 && mode < 500) break; - fallthrough; - case 451: - test_mb_ahash_speed("sha256", sec, generic_hash_speed_template, - num_mb); - if (mode > 400 && mode < 500) break; - fallthrough; - case 452: - test_mb_ahash_speed("sha512", sec, generic_hash_speed_template, - num_mb); - if (mode > 400 && mode < 500) break; - fallthrough; - case 453: - test_mb_ahash_speed("sm3", sec, generic_hash_speed_template, - num_mb); - if (mode > 400 && mode < 500) break; - fallthrough; - case 454: - test_mb_ahash_speed("streebog256", sec, - generic_hash_speed_template, num_mb); - if (mode > 400 && mode < 500) break; - fallthrough; - case 455: - test_mb_ahash_speed("streebog512", sec, - generic_hash_speed_template, num_mb); - if (mode > 400 && mode < 500) break; - fallthrough; case 499: break; -- 2.51.0 From 5bb61dc76d11a661c323dee1505b408d18c31565 Mon Sep 17 00:00:00 2001 From: Herbert Xu Date: Sat, 12 Apr 2025 13:37:00 +0800 Subject: [PATCH 07/16] crypto: ahash - Remove request chaining Request chaining requires the user to do too much book keeping. Remove it from ahash. Signed-off-by: Herbert Xu --- crypto/ahash.c | 169 +++++++-------------------------- include/crypto/algapi.h | 5 - include/crypto/hash.h | 12 --- include/crypto/internal/hash.h | 5 - include/linux/crypto.h | 15 --- 5 files changed, 32 insertions(+), 174 deletions(-) diff --git a/crypto/ahash.c b/crypto/ahash.c index 2d9eec2b2b1c..fc59897c234c 100644 --- a/crypto/ahash.c +++ b/crypto/ahash.c @@ -43,10 +43,7 @@ struct crypto_hash_walk { }; struct ahash_save_req_state { - struct list_head head; struct ahash_request *req0; - struct ahash_request *cur; - int (*op)(struct ahash_request *req); crypto_completion_t compl; void *data; struct scatterlist sg; @@ -54,9 +51,9 @@ struct ahash_save_req_state { u8 *page; unsigned int offset; unsigned int nbytes; + bool update; }; -static void ahash_reqchain_done(void *data, int err); static int ahash_save_req(struct ahash_request *req, crypto_completion_t cplt); static void ahash_restore_req(struct ahash_request *req); static void ahash_def_finup_done1(void *data, int err); @@ -313,21 +310,17 @@ int crypto_ahash_setkey(struct crypto_ahash *tfm, const u8 *key, } EXPORT_SYMBOL_GPL(crypto_ahash_setkey); -static bool ahash_request_hasvirt(struct ahash_request *req) -{ - return ahash_request_isvirt(req); -} - static int ahash_reqchain_virt(struct ahash_save_req_state *state, int err, u32 mask) { - struct ahash_request *req = state->cur; + struct ahash_request *req = state->req0; + struct crypto_ahash *tfm; + + tfm = crypto_ahash_reqtfm(req); for (;;) { unsigned len = state->nbytes; - req->base.err = err; - if (!state->offset) break; @@ -346,10 +339,9 @@ static int ahash_reqchain_virt(struct ahash_save_req_state *state, state->offset += len; req->nbytes = len; - err = state->op(req); + err = crypto_ahash_alg(tfm)->update(req); if (err == -EINPROGRESS) { - if (!list_empty(&state->head) || - state->offset < state->nbytes) + if (state->offset < state->nbytes) err = -EBUSY; break; } @@ -365,64 +357,12 @@ static int ahash_reqchain_finish(struct ahash_request *req0, struct ahash_save_req_state *state, int err, u32 mask) { - struct ahash_request *req = state->cur; - struct crypto_ahash *tfm; - struct ahash_request *n; - bool update; u8 *page; err = ahash_reqchain_virt(state, err, mask); if (err == -EINPROGRESS || err == -EBUSY) goto out; - if (req != req0) - list_add_tail(&req->base.list, &req0->base.list); - - tfm = crypto_ahash_reqtfm(req); - update = state->op == crypto_ahash_alg(tfm)->update; - - list_for_each_entry_safe(req, n, &state->head, base.list) { - list_del_init(&req->base.list); - - req->base.flags &= mask; - req->base.complete = ahash_reqchain_done; - req->base.data = state; - state->cur = req; - - if (update && ahash_request_isvirt(req) && req->nbytes) { - unsigned len = req->nbytes; - u8 *result = req->result; - - state->src = req->svirt; - state->nbytes = len; - - len = min(PAGE_SIZE, len); - - memcpy(state->page, req->svirt, len); - state->offset = len; - - ahash_request_set_crypt(req, &state->sg, result, len); - } - - err = state->op(req); - - if (err == -EINPROGRESS) { - if (!list_empty(&state->head) || - state->offset < state->nbytes) - err = -EBUSY; - goto out; - } - - if (err == -EBUSY) - goto out; - - err = ahash_reqchain_virt(state, err, mask); - if (err == -EINPROGRESS || err == -EBUSY) - goto out; - - list_add_tail(&req->base.list, &req0->base.list); - } - page = state->page; if (page) { memset(page, 0, PAGE_SIZE); @@ -442,7 +382,7 @@ static void ahash_reqchain_done(void *data, int err) data = state->data; if (err == -EINPROGRESS) { - if (!list_empty(&state->head) || state->offset < state->nbytes) + if (state->offset < state->nbytes) return; goto notify; } @@ -467,21 +407,14 @@ static int ahash_do_req_chain(struct ahash_request *req, int err; if (crypto_ahash_req_chain(tfm) || - (!ahash_request_chained(req) && - (!update || !ahash_request_isvirt(req)))) + !update || !ahash_request_isvirt(req)) return op(req); - if (update && ahash_request_hasvirt(req)) { - gfp_t gfp; - u32 flags; - - flags = ahash_request_flags(req); - gfp = (flags & CRYPTO_TFM_REQ_MAY_SLEEP) ? - GFP_KERNEL : GFP_ATOMIC; - page = (void *)__get_free_page(gfp); + if (update && ahash_request_isvirt(req)) { + page = (void *)__get_free_page(GFP_ATOMIC); err = -ENOMEM; if (!page) - goto out_set_chain; + goto out; } state = &state0; @@ -493,12 +426,10 @@ static int ahash_do_req_chain(struct ahash_request *req, state = req->base.data; } - state->op = op; - state->cur = req; + state->update = update; state->page = page; state->offset = 0; state->nbytes = 0; - INIT_LIST_HEAD(&state->head); if (page) sg_init_one(&state->sg, page, PAGE_SIZE); @@ -519,16 +450,18 @@ static int ahash_do_req_chain(struct ahash_request *req, } err = op(req); - if (err == -EBUSY || err == -EINPROGRESS) - return -EBUSY; + if (err == -EINPROGRESS || err == -EBUSY) { + if (state->offset < state->nbytes) + err = -EBUSY; + return err; + } return ahash_reqchain_finish(req, state, err, ~0); out_free_page: free_page((unsigned long)page); -out_set_chain: - req->base.err = err; +out: return err; } @@ -536,17 +469,10 @@ int crypto_ahash_init(struct ahash_request *req) { struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); - if (likely(tfm->using_shash)) { - int err; - - err = crypto_shash_init(prepare_shash_desc(req, tfm)); - req->base.err = err; - return err; - } - + if (likely(tfm->using_shash)) + return crypto_shash_init(prepare_shash_desc(req, tfm)); if (crypto_ahash_get_flags(tfm) & CRYPTO_TFM_NEED_KEY) return -ENOKEY; - return ahash_do_req_chain(req, crypto_ahash_alg(tfm)->init); } EXPORT_SYMBOL_GPL(crypto_ahash_init); @@ -555,15 +481,11 @@ static int ahash_save_req(struct ahash_request *req, crypto_completion_t cplt) { struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); struct ahash_save_req_state *state; - gfp_t gfp; - u32 flags; if (!ahash_is_async(tfm)) return 0; - flags = ahash_request_flags(req); - gfp = (flags & CRYPTO_TFM_REQ_MAY_SLEEP) ? GFP_KERNEL : GFP_ATOMIC; - state = kmalloc(sizeof(*state), gfp); + state = kmalloc(sizeof(*state), GFP_ATOMIC); if (!state) return -ENOMEM; @@ -596,14 +518,8 @@ int crypto_ahash_update(struct ahash_request *req) { struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); - if (likely(tfm->using_shash)) { - int err; - - err = shash_ahash_update(req, ahash_request_ctx(req)); - req->base.err = err; - return err; - } - + if (likely(tfm->using_shash)) + return shash_ahash_update(req, ahash_request_ctx(req)); return ahash_do_req_chain(req, crypto_ahash_alg(tfm)->update); } EXPORT_SYMBOL_GPL(crypto_ahash_update); @@ -612,14 +528,8 @@ int crypto_ahash_final(struct ahash_request *req) { struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); - if (likely(tfm->using_shash)) { - int err; - - err = crypto_shash_final(ahash_request_ctx(req), req->result); - req->base.err = err; - return err; - } - + if (likely(tfm->using_shash)) + return crypto_shash_final(ahash_request_ctx(req), req->result); return ahash_do_req_chain(req, crypto_ahash_alg(tfm)->final); } EXPORT_SYMBOL_GPL(crypto_ahash_final); @@ -628,18 +538,11 @@ int crypto_ahash_finup(struct ahash_request *req) { struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); - if (likely(tfm->using_shash)) { - int err; - - err = shash_ahash_finup(req, ahash_request_ctx(req)); - req->base.err = err; - return err; - } - + if (likely(tfm->using_shash)) + return shash_ahash_finup(req, ahash_request_ctx(req)); if (!crypto_ahash_alg(tfm)->finup || - (!crypto_ahash_req_chain(tfm) && ahash_request_hasvirt(req))) + (!crypto_ahash_req_chain(tfm) && ahash_request_isvirt(req))) return ahash_def_finup(req); - return ahash_do_req_chain(req, crypto_ahash_alg(tfm)->finup); } EXPORT_SYMBOL_GPL(crypto_ahash_finup); @@ -706,20 +609,12 @@ int crypto_ahash_digest(struct ahash_request *req) { struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); - if (likely(tfm->using_shash)) { - int err; - - err = shash_ahash_digest(req, prepare_shash_desc(req, tfm)); - req->base.err = err; - return err; - } - - if (!crypto_ahash_req_chain(tfm) && ahash_request_hasvirt(req)) + if (likely(tfm->using_shash)) + return shash_ahash_digest(req, prepare_shash_desc(req, tfm)); + if (!crypto_ahash_req_chain(tfm) && ahash_request_isvirt(req)) return ahash_def_digest(req); - if (crypto_ahash_get_flags(tfm) & CRYPTO_TFM_NEED_KEY) return -ENOKEY; - return ahash_do_req_chain(req, crypto_ahash_alg(tfm)->digest); } EXPORT_SYMBOL_GPL(crypto_ahash_digest); diff --git a/include/crypto/algapi.h b/include/crypto/algapi.h index 03b7eca8af9a..ede622ecefa8 100644 --- a/include/crypto/algapi.h +++ b/include/crypto/algapi.h @@ -267,11 +267,6 @@ static inline u32 crypto_tfm_alg_type(struct crypto_tfm *tfm) return tfm->__crt_alg->cra_flags & CRYPTO_ALG_TYPE_MASK; } -static inline bool crypto_request_chained(struct crypto_async_request *req) -{ - return !list_empty(&req->list); -} - static inline bool crypto_tfm_req_chain(struct crypto_tfm *tfm) { return tfm->__crt_alg->cra_flags & CRYPTO_ALG_REQ_CHAIN; diff --git a/include/crypto/hash.h b/include/crypto/hash.h index a13e2de3c9b4..113a5835e586 100644 --- a/include/crypto/hash.h +++ b/include/crypto/hash.h @@ -630,7 +630,6 @@ static inline void ahash_request_set_callback(struct ahash_request *req, flags &= ~keep; req->base.flags &= keep; req->base.flags |= flags; - crypto_reqchain_init(&req->base); } /** @@ -679,12 +678,6 @@ static inline void ahash_request_set_virt(struct ahash_request *req, req->base.flags |= CRYPTO_AHASH_REQ_VIRT; } -static inline void ahash_request_chain(struct ahash_request *req, - struct ahash_request *head) -{ - crypto_request_chain(&req->base, &head->base); -} - /** * DOC: Synchronous Message Digest API * @@ -986,11 +979,6 @@ static inline void shash_desc_zero(struct shash_desc *desc) sizeof(*desc) + crypto_shash_descsize(desc->tfm)); } -static inline int ahash_request_err(struct ahash_request *req) -{ - return req->base.err; -} - static inline bool ahash_is_async(struct crypto_ahash *tfm) { return crypto_tfm_is_async(&tfm->base); diff --git a/include/crypto/internal/hash.h b/include/crypto/internal/hash.h index 052ac7924af3..e2a1fac38610 100644 --- a/include/crypto/internal/hash.h +++ b/include/crypto/internal/hash.h @@ -247,11 +247,6 @@ static inline struct crypto_shash *__crypto_shash_cast(struct crypto_tfm *tfm) return container_of(tfm, struct crypto_shash, base); } -static inline bool ahash_request_chained(struct ahash_request *req) -{ - return false; -} - static inline bool ahash_request_isvirt(struct ahash_request *req) { return req->base.flags & CRYPTO_AHASH_REQ_VIRT; diff --git a/include/linux/crypto.h b/include/linux/crypto.h index 1e3809d28abd..dd817f56ff0c 100644 --- a/include/linux/crypto.h +++ b/include/linux/crypto.h @@ -14,7 +14,6 @@ #include #include -#include #include #include #include @@ -179,7 +178,6 @@ struct crypto_async_request { struct crypto_tfm *tfm; u32 flags; - int err; }; /** @@ -473,19 +471,6 @@ static inline unsigned int crypto_tfm_ctx_alignment(void) return __alignof__(tfm->__crt_ctx); } -static inline void crypto_reqchain_init(struct crypto_async_request *req) -{ - req->err = -EINPROGRESS; - INIT_LIST_HEAD(&req->list); -} - -static inline void crypto_request_chain(struct crypto_async_request *req, - struct crypto_async_request *head) -{ - req->err = -EINPROGRESS; - list_add_tail(&req->list, &head->list); -} - static inline bool crypto_tfm_is_async(struct crypto_tfm *tfm) { return tfm->__crt_alg->cra_flags & CRYPTO_ALG_ASYNC; -- 2.51.0 From b93336cd767fc266dcccfa034a1fb32ae1a23564 Mon Sep 17 00:00:00 2001 From: Thorsten Blum Date: Mon, 7 Apr 2025 10:22:47 +0200 Subject: [PATCH 08/16] crypto: x509 - Replace kmalloc() + NUL-termination with kzalloc() Use kzalloc() to zero out the one-element array instead of using kmalloc() followed by a manual NUL-termination. No functional changes intended. Signed-off-by: Thorsten Blum Reviewed-by: Lukas Wunner Signed-off-by: Herbert Xu --- crypto/asymmetric_keys/x509_cert_parser.c | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/crypto/asymmetric_keys/x509_cert_parser.c b/crypto/asymmetric_keys/x509_cert_parser.c index ee2fdab42334..2ffe4ae90bea 100644 --- a/crypto/asymmetric_keys/x509_cert_parser.c +++ b/crypto/asymmetric_keys/x509_cert_parser.c @@ -372,10 +372,9 @@ static int x509_fabricate_name(struct x509_parse_context *ctx, size_t hdrlen, /* Empty name string if no material */ if (!ctx->cn_size && !ctx->o_size && !ctx->email_size) { - buffer = kmalloc(1, GFP_KERNEL); + buffer = kzalloc(1, GFP_KERNEL); if (!buffer) return -ENOMEM; - buffer[0] = 0; goto done; } -- 2.51.0 From 1451e3e561be9ff4e86b911b9367a2223275d16f Mon Sep 17 00:00:00 2001 From: Herbert Xu Date: Mon, 7 Apr 2025 18:02:51 +0800 Subject: [PATCH 09/16] crypto: api - Add helpers to manage request flags Add helpers so that the ON_STACK request flag management is not duplicated all over the place. Signed-off-by: Herbert Xu --- include/crypto/algapi.h | 5 +++++ include/linux/crypto.h | 24 ++++++++++++++++++++++++ 2 files changed, 29 insertions(+) diff --git a/include/crypto/algapi.h b/include/crypto/algapi.h index ede622ecefa8..6999e10ea09e 100644 --- a/include/crypto/algapi.h +++ b/include/crypto/algapi.h @@ -272,4 +272,9 @@ static inline bool crypto_tfm_req_chain(struct crypto_tfm *tfm) return tfm->__crt_alg->cra_flags & CRYPTO_ALG_REQ_CHAIN; } +static inline u32 crypto_request_flags(struct crypto_async_request *req) +{ + return req->flags & ~CRYPTO_TFM_REQ_ON_STACK; +} + #endif /* _CRYPTO_ALGAPI_H */ diff --git a/include/linux/crypto.h b/include/linux/crypto.h index dd817f56ff0c..a387f1547ea0 100644 --- a/include/linux/crypto.h +++ b/include/linux/crypto.h @@ -476,5 +476,29 @@ static inline bool crypto_tfm_is_async(struct crypto_tfm *tfm) return tfm->__crt_alg->cra_flags & CRYPTO_ALG_ASYNC; } +static inline bool crypto_req_on_stack(struct crypto_async_request *req) +{ + return req->flags & CRYPTO_TFM_REQ_ON_STACK; +} + +static inline void crypto_request_set_callback( + struct crypto_async_request *req, u32 flags, + crypto_completion_t compl, void *data) +{ + u32 keep = CRYPTO_TFM_REQ_ON_STACK; + + req->complete = compl; + req->data = data; + req->flags &= keep; + req->flags |= flags & ~keep; +} + +static inline void crypto_request_set_tfm(struct crypto_async_request *req, + struct crypto_tfm *tfm) +{ + req->tfm = tfm; + req->flags &= ~CRYPTO_TFM_REQ_ON_STACK; +} + #endif /* _LINUX_CRYPTO_H */ -- 2.51.0 From b04b395f7a29ed28d3cb27a7b39ac67dfb959fa0 Mon Sep 17 00:00:00 2001 From: Herbert Xu Date: Mon, 7 Apr 2025 18:02:53 +0800 Subject: [PATCH 10/16] crypto: acomp - Use request flag helpers and add acomp_request_flags Use the newly added request flag helpers to manage the request flags. Also add acomp_request_flags which lets bottom-level users to access the request flags without the bits private to the acomp API. Signed-off-by: Herbert Xu --- include/crypto/acompress.h | 27 ++++++++++++++++----------- include/crypto/internal/acompress.h | 6 ++++++ 2 files changed, 22 insertions(+), 11 deletions(-) diff --git a/include/crypto/acompress.h b/include/crypto/acompress.h index 080e134df35c..f383a4008854 100644 --- a/include/crypto/acompress.h +++ b/include/crypto/acompress.h @@ -38,6 +38,12 @@ /* Set this bit if destination is a folio. */ #define CRYPTO_ACOMP_REQ_DST_FOLIO 0x00000040 +/* Private flags that should not be touched by the user. */ +#define CRYPTO_ACOMP_REQ_PRIVATE \ + (CRYPTO_ACOMP_REQ_SRC_VIRT | CRYPTO_ACOMP_REQ_SRC_NONDMA | \ + CRYPTO_ACOMP_REQ_DST_VIRT | CRYPTO_ACOMP_REQ_DST_NONDMA | \ + CRYPTO_ACOMP_REQ_SRC_FOLIO | CRYPTO_ACOMP_REQ_DST_FOLIO) + #define CRYPTO_ACOMP_DST_MAX 131072 #define MAX_SYNC_COMP_REQSIZE 0 @@ -201,7 +207,7 @@ static inline unsigned int crypto_acomp_reqsize(struct crypto_acomp *tfm) static inline void acomp_request_set_tfm(struct acomp_req *req, struct crypto_acomp *tfm) { - req->base.tfm = crypto_acomp_tfm(tfm); + crypto_request_set_tfm(&req->base, crypto_acomp_tfm(tfm)); } static inline bool acomp_is_async(struct crypto_acomp *tfm) @@ -298,6 +304,11 @@ static inline void *acomp_request_extra(struct acomp_req *req) return (void *)((char *)req + len); } +static inline bool acomp_req_on_stack(struct acomp_req *req) +{ + return crypto_req_on_stack(&req->base); +} + /** * acomp_request_free() -- zeroize and free asynchronous (de)compression * request as well as the output buffer if allocated @@ -307,7 +318,7 @@ static inline void *acomp_request_extra(struct acomp_req *req) */ static inline void acomp_request_free(struct acomp_req *req) { - if (!req || (req->base.flags & CRYPTO_TFM_REQ_ON_STACK)) + if (!req || acomp_req_on_stack(req)) return; kfree_sensitive(req); } @@ -328,15 +339,9 @@ static inline void acomp_request_set_callback(struct acomp_req *req, crypto_completion_t cmpl, void *data) { - u32 keep = CRYPTO_ACOMP_REQ_SRC_VIRT | CRYPTO_ACOMP_REQ_SRC_NONDMA | - CRYPTO_ACOMP_REQ_DST_VIRT | CRYPTO_ACOMP_REQ_DST_NONDMA | - CRYPTO_ACOMP_REQ_SRC_FOLIO | CRYPTO_ACOMP_REQ_DST_FOLIO | - CRYPTO_TFM_REQ_ON_STACK; - - req->base.complete = cmpl; - req->base.data = data; - req->base.flags &= keep; - req->base.flags |= flgs & ~keep; + flgs &= ~CRYPTO_ACOMP_REQ_PRIVATE; + flgs |= req->base.flags & CRYPTO_ACOMP_REQ_PRIVATE; + crypto_request_set_callback(&req->base, flgs, cmpl, data); } /** diff --git a/include/crypto/internal/acompress.h b/include/crypto/internal/acompress.h index 960cdd1f3a57..5483ca5b46ad 100644 --- a/include/crypto/internal/acompress.h +++ b/include/crypto/internal/acompress.h @@ -229,4 +229,10 @@ static inline bool acomp_walk_more_src(const struct acomp_walk *walk, int cur) { return walk->slen != cur; } + +static inline u32 acomp_request_flags(struct acomp_req *req) +{ + return crypto_request_flags(&req->base) & ~CRYPTO_ACOMP_REQ_PRIVATE; +} + #endif -- 2.51.0 From 05fa2c6e87da31eab150cdaca6697cd1de122ec7 Mon Sep 17 00:00:00 2001 From: Herbert Xu Date: Mon, 7 Apr 2025 18:02:55 +0800 Subject: [PATCH 11/16] crypto: acomp - Add ACOMP_FBREQ_ON_STACK Add a helper to create an on-stack fallback request from a given request. Use this helper in acomp_do_nondma. Signed-off-by: Herbert Xu --- crypto/acompress.c | 14 +------------- include/crypto/internal/acompress.h | 26 ++++++++++++++++++++++++++ 2 files changed, 27 insertions(+), 13 deletions(-) diff --git a/crypto/acompress.c b/crypto/acompress.c index b682a88781f0..f343b1a4b1d1 100644 --- a/crypto/acompress.c +++ b/crypto/acompress.c @@ -253,21 +253,9 @@ static void acomp_virt_to_sg(struct acomp_req *req) static int acomp_do_nondma(struct acomp_req *req, bool comp) { - u32 keep = CRYPTO_ACOMP_REQ_SRC_VIRT | - CRYPTO_ACOMP_REQ_SRC_NONDMA | - CRYPTO_ACOMP_REQ_DST_VIRT | - CRYPTO_ACOMP_REQ_DST_NONDMA; - ACOMP_REQUEST_ON_STACK(fbreq, crypto_acomp_reqtfm(req)); + ACOMP_FBREQ_ON_STACK(fbreq, req); int err; - acomp_request_set_callback(fbreq, req->base.flags, NULL, NULL); - fbreq->base.flags &= ~keep; - fbreq->base.flags |= req->base.flags & keep; - fbreq->src = req->src; - fbreq->dst = req->dst; - fbreq->slen = req->slen; - fbreq->dlen = req->dlen; - if (comp) err = crypto_acomp_compress(fbreq); else diff --git a/include/crypto/internal/acompress.h b/include/crypto/internal/acompress.h index 5483ca5b46ad..8840fd2c1db5 100644 --- a/include/crypto/internal/acompress.h +++ b/include/crypto/internal/acompress.h @@ -23,6 +23,12 @@ struct acomp_req *name = acomp_request_on_stack_init( \ __##name##_req, (tfm), 0, true) +#define ACOMP_FBREQ_ON_STACK(name, req) \ + char __##name##_req[sizeof(struct acomp_req) + \ + MAX_SYNC_COMP_REQSIZE] CRYPTO_MINALIGN_ATTR; \ + struct acomp_req *name = acomp_fbreq_on_stack_init( \ + __##name##_req, (req)) + /** * struct acomp_alg - asynchronous compression algorithm * @@ -235,4 +241,24 @@ static inline u32 acomp_request_flags(struct acomp_req *req) return crypto_request_flags(&req->base) & ~CRYPTO_ACOMP_REQ_PRIVATE; } +static inline struct acomp_req *acomp_fbreq_on_stack_init( + char *buf, struct acomp_req *old) +{ + struct crypto_acomp *tfm = crypto_acomp_reqtfm(old); + struct acomp_req *req; + + req = acomp_request_on_stack_init(buf, tfm, 0, true); + acomp_request_set_callback(req, acomp_request_flags(old), NULL, NULL); + req->base.flags &= ~CRYPTO_ACOMP_REQ_PRIVATE; + req->base.flags |= old->base.flags & CRYPTO_ACOMP_REQ_PRIVATE; + req->src = old->src; + req->dst = old->dst; + req->slen = old->slen; + req->dlen = old->dlen; + req->soff = old->soff; + req->doff = old->doff; + + return req; +} + #endif -- 2.51.0 From d0a5c9d079decad95a77638c19bc5b3a6a0ffe21 Mon Sep 17 00:00:00 2001 From: Herbert Xu Date: Mon, 7 Apr 2025 18:02:58 +0800 Subject: [PATCH 12/16] crypto: iaa - Switch to ACOMP_FBREQ_ON_STACK Rather than copying the request by hand, use the ACOMP_FBREQ_ON_STACK helper to do it. Signed-off-by: Herbert Xu --- drivers/crypto/intel/iaa/iaa_crypto_main.c | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/drivers/crypto/intel/iaa/iaa_crypto_main.c b/drivers/crypto/intel/iaa/iaa_crypto_main.c index aa63df16f20b..b4f15e738cee 100644 --- a/drivers/crypto/intel/iaa/iaa_crypto_main.c +++ b/drivers/crypto/intel/iaa/iaa_crypto_main.c @@ -999,12 +999,9 @@ out: static int deflate_generic_decompress(struct acomp_req *req) { - ACOMP_REQUEST_ON_STACK(fbreq, crypto_acomp_reqtfm(req)); + ACOMP_FBREQ_ON_STACK(fbreq, req); int ret; - acomp_request_set_callback(fbreq, 0, NULL, NULL); - acomp_request_set_params(fbreq, req->src, req->dst, req->slen, - req->dlen); ret = crypto_acomp_decompress(fbreq); req->dlen = fbreq->dlen; -- 2.51.0 From 097c432caaa6d91f87732fe991cb08139e31101a Mon Sep 17 00:00:00 2001 From: Herbert Xu Date: Mon, 7 Apr 2025 18:03:00 +0800 Subject: [PATCH 13/16] crypto: acomp - Add ACOMP_REQUEST_CLONE Add a new helper ACOMP_REQUEST_CLONE that will transform a stack request into a dynamically allocated one if possible, and otherwise switch it over to the sycnrhonous fallback transform. The intended usage is: ACOMP_STACK_ON_REQUEST(req, tfm); ... err = crypto_acomp_compress(req); /* The request cannot complete synchronously. */ if (err == -EAGAIN) { /* This will not fail. */ req = ACOMP_REQUEST_CLONE(req, gfp); /* Redo operation. */ err = crypto_acomp_compress(req); } Signed-off-by: Herbert Xu --- crypto/acompress.c | 23 ++++++++++++++++++++++ include/crypto/acompress.h | 30 +++++++++++++++++++++++++---- include/crypto/internal/acompress.h | 11 +++-------- 3 files changed, 52 insertions(+), 12 deletions(-) diff --git a/crypto/acompress.c b/crypto/acompress.c index f343b1a4b1d1..530b9bfd03a5 100644 --- a/crypto/acompress.c +++ b/crypto/acompress.c @@ -316,6 +316,8 @@ int crypto_acomp_compress(struct acomp_req *req) { struct crypto_acomp *tfm = crypto_acomp_reqtfm(req); + if (acomp_req_on_stack(req) && acomp_is_async(tfm)) + return -EAGAIN; if (crypto_acomp_req_chain(tfm) || acomp_request_issg(req)) crypto_acomp_reqtfm(req)->compress(req); return acomp_do_req_chain(req, true); @@ -326,6 +328,8 @@ int crypto_acomp_decompress(struct acomp_req *req) { struct crypto_acomp *tfm = crypto_acomp_reqtfm(req); + if (acomp_req_on_stack(req) && acomp_is_async(tfm)) + return -EAGAIN; if (crypto_acomp_req_chain(tfm) || acomp_request_issg(req)) crypto_acomp_reqtfm(req)->decompress(req); return acomp_do_req_chain(req, false); @@ -603,5 +607,24 @@ int acomp_walk_virt(struct acomp_walk *__restrict walk, } EXPORT_SYMBOL_GPL(acomp_walk_virt); +struct acomp_req *acomp_request_clone(struct acomp_req *req, + size_t total, gfp_t gfp) +{ + struct crypto_acomp *tfm = crypto_acomp_reqtfm(req); + struct acomp_req *nreq; + + nreq = kmalloc(total, gfp); + if (!nreq) { + acomp_request_set_tfm(req, tfm->fb); + req->base.flags = CRYPTO_TFM_REQ_ON_STACK; + return req; + } + + memcpy(nreq, req, total); + acomp_request_set_tfm(req, tfm); + return req; +} +EXPORT_SYMBOL_GPL(acomp_request_clone); + MODULE_LICENSE("GPL"); MODULE_DESCRIPTION("Asynchronous compression type"); diff --git a/include/crypto/acompress.h b/include/crypto/acompress.h index f383a4008854..93cee67c27c0 100644 --- a/include/crypto/acompress.h +++ b/include/crypto/acompress.h @@ -49,10 +49,19 @@ #define MAX_SYNC_COMP_REQSIZE 0 #define ACOMP_REQUEST_ALLOC(name, tfm, gfp) \ + char __##name##_req[sizeof(struct acomp_req) + \ + MAX_SYNC_COMP_REQSIZE] CRYPTO_MINALIGN_ATTR; \ + struct acomp_req *name = acomp_request_alloc_init( \ + __##name##_req, (tfm), (gfp)) + +#define ACOMP_REQUEST_ON_STACK(name, tfm) \ char __##name##_req[sizeof(struct acomp_req) + \ MAX_SYNC_COMP_REQSIZE] CRYPTO_MINALIGN_ATTR; \ struct acomp_req *name = acomp_request_on_stack_init( \ - __##name##_req, (tfm), (gfp), false) + __##name##_req, (tfm)) + +#define ACOMP_REQUEST_CLONE(name, gfp) \ + acomp_request_clone(name, sizeof(__##name##_req), gfp) struct acomp_req; struct folio; @@ -571,12 +580,12 @@ int crypto_acomp_compress(struct acomp_req *req); */ int crypto_acomp_decompress(struct acomp_req *req); -static inline struct acomp_req *acomp_request_on_stack_init( - char *buf, struct crypto_acomp *tfm, gfp_t gfp, bool stackonly) +static inline struct acomp_req *acomp_request_alloc_init( + char *buf, struct crypto_acomp *tfm, gfp_t gfp) { struct acomp_req *req; - if (!stackonly && (req = acomp_request_alloc(tfm, gfp))) + if ((req = acomp_request_alloc(tfm, gfp))) return req; req = (void *)buf; @@ -586,4 +595,17 @@ static inline struct acomp_req *acomp_request_on_stack_init( return req; } +static inline struct acomp_req *acomp_request_on_stack_init( + char *buf, struct crypto_acomp *tfm) +{ + struct acomp_req *req = (void *)buf; + + acomp_request_set_tfm(req, tfm); + req->base.flags = CRYPTO_TFM_REQ_ON_STACK; + return req; +} + +struct acomp_req *acomp_request_clone(struct acomp_req *req, + size_t total, gfp_t gfp); + #endif diff --git a/include/crypto/internal/acompress.h b/include/crypto/internal/acompress.h index 8840fd2c1db5..b51d66633935 100644 --- a/include/crypto/internal/acompress.h +++ b/include/crypto/internal/acompress.h @@ -17,12 +17,6 @@ #include #include -#define ACOMP_REQUEST_ON_STACK(name, tfm) \ - char __##name##_req[sizeof(struct acomp_req) + \ - MAX_SYNC_COMP_REQSIZE] CRYPTO_MINALIGN_ATTR; \ - struct acomp_req *name = acomp_request_on_stack_init( \ - __##name##_req, (tfm), 0, true) - #define ACOMP_FBREQ_ON_STACK(name, req) \ char __##name##_req[sizeof(struct acomp_req) + \ MAX_SYNC_COMP_REQSIZE] CRYPTO_MINALIGN_ATTR; \ @@ -245,9 +239,10 @@ static inline struct acomp_req *acomp_fbreq_on_stack_init( char *buf, struct acomp_req *old) { struct crypto_acomp *tfm = crypto_acomp_reqtfm(old); - struct acomp_req *req; + struct acomp_req *req = (void *)buf; - req = acomp_request_on_stack_init(buf, tfm, 0, true); + acomp_request_set_tfm(req, tfm->fb); + req->base.flags = CRYPTO_TFM_REQ_ON_STACK; acomp_request_set_callback(req, acomp_request_flags(old), NULL, NULL); req->base.flags &= ~CRYPTO_ACOMP_REQ_PRIVATE; req->base.flags |= old->base.flags & CRYPTO_ACOMP_REQ_PRIVATE; -- 2.51.0 From e87e95d8dde623f199afbb364f89e6b20ffa61ec Mon Sep 17 00:00:00 2001 From: Herbert Xu Date: Mon, 7 Apr 2025 18:03:02 +0800 Subject: [PATCH 14/16] ubifs: Use ACOMP_REQUEST_CLONE Switch to the new acomp API where stacks requests are used by default and a dynamic request is only allocted when necessary. Signed-off-by: Herbert Xu --- fs/ubifs/compress.c | 247 ++++++++++++++++++++------------------------ 1 file changed, 114 insertions(+), 133 deletions(-) diff --git a/fs/ubifs/compress.c b/fs/ubifs/compress.c index ea6f06adcd43..059a02691edd 100644 --- a/fs/ubifs/compress.c +++ b/fs/ubifs/compress.c @@ -19,6 +19,11 @@ #include #include "ubifs.h" +union ubifs_in_ptr { + const void *buf; + struct folio *folio; +}; + /* Fake description object for the "none" compressor */ static struct ubifs_compressor none_compr = { .compr_type = UBIFS_COMPR_NONE, @@ -68,28 +73,61 @@ static struct ubifs_compressor zstd_compr = { /* All UBIFS compressors */ struct ubifs_compressor *ubifs_compressors[UBIFS_COMPR_TYPES_CNT]; -static int ubifs_compress_req(const struct ubifs_info *c, - struct acomp_req *req, - void *out_buf, int *out_len, - const char *compr_name) +static void ubifs_compress_common(int *compr_type, union ubifs_in_ptr in_ptr, + size_t in_offset, int in_len, bool in_folio, + void *out_buf, int *out_len) { - struct crypto_wait wait; - int in_len = req->slen; + struct ubifs_compressor *compr = ubifs_compressors[*compr_type]; int dlen = *out_len; int err; + if (*compr_type == UBIFS_COMPR_NONE) + goto no_compr; + + /* If the input data is small, do not even try to compress it */ + if (in_len < UBIFS_MIN_COMPR_LEN) + goto no_compr; + dlen = min(dlen, in_len - UBIFS_MIN_COMPRESS_DIFF); - crypto_init_wait(&wait); - acomp_request_set_callback(req, CRYPTO_TFM_REQ_MAY_BACKLOG, - crypto_req_done, &wait); - acomp_request_set_dst_dma(req, out_buf, dlen); - err = crypto_acomp_compress(req); - err = crypto_wait_req(err, &wait); - *out_len = req->dlen; - acomp_request_free(req); + do { + ACOMP_REQUEST_ON_STACK(req, compr->cc); + DECLARE_CRYPTO_WAIT(wait); + + acomp_request_set_callback(req, 0, NULL, NULL); + if (in_folio) + acomp_request_set_src_folio(req, in_ptr.folio, + in_offset, in_len); + else + acomp_request_set_src_dma(req, in_ptr.buf, in_len); + acomp_request_set_dst_dma(req, out_buf, dlen); + err = crypto_acomp_compress(req); + dlen = req->dlen; + if (err != -EAGAIN) + break; + + req = ACOMP_REQUEST_CLONE(req, GFP_NOFS | __GFP_NOWARN); + acomp_request_set_callback(req, CRYPTO_TFM_REQ_MAY_BACKLOG, + crypto_req_done, &wait); + err = crypto_acomp_compress(req); + err = crypto_wait_req(err, &wait); + dlen = req->dlen; + acomp_request_free(req); + } while (0); + + *out_len = dlen; + if (err) + goto no_compr; - return err; + return; + +no_compr: + if (in_folio) + memcpy_from_folio(out_buf, in_ptr.folio, in_offset, in_len); + else + memcpy(out_buf, in_ptr.buf, in_len); + *out_len = in_len; + *compr_type = UBIFS_COMPR_NONE; } /** @@ -114,32 +152,10 @@ static int ubifs_compress_req(const struct ubifs_info *c, void ubifs_compress(const struct ubifs_info *c, const void *in_buf, int in_len, void *out_buf, int *out_len, int *compr_type) { - int err; - struct ubifs_compressor *compr = ubifs_compressors[*compr_type]; - - if (*compr_type == UBIFS_COMPR_NONE) - goto no_compr; + union ubifs_in_ptr in_ptr = { .buf = in_buf }; - /* If the input data is small, do not even try to compress it */ - if (in_len < UBIFS_MIN_COMPR_LEN) - goto no_compr; - - { - ACOMP_REQUEST_ALLOC(req, compr->cc, GFP_NOFS | __GFP_NOWARN); - - acomp_request_set_src_dma(req, in_buf, in_len); - err = ubifs_compress_req(c, req, out_buf, out_len, compr->name); - } - - if (err) - goto no_compr; - - return; - -no_compr: - memcpy(out_buf, in_buf, in_len); - *out_len = in_len; - *compr_type = UBIFS_COMPR_NONE; + ubifs_compress_common(compr_type, in_ptr, 0, in_len, false, + out_buf, out_len); } /** @@ -166,55 +182,71 @@ void ubifs_compress_folio(const struct ubifs_info *c, struct folio *in_folio, size_t in_offset, int in_len, void *out_buf, int *out_len, int *compr_type) { - int err; - struct ubifs_compressor *compr = ubifs_compressors[*compr_type]; + union ubifs_in_ptr in_ptr = { .folio = in_folio }; - if (*compr_type == UBIFS_COMPR_NONE) - goto no_compr; - - /* If the input data is small, do not even try to compress it */ - if (in_len < UBIFS_MIN_COMPR_LEN) - goto no_compr; + ubifs_compress_common(compr_type, in_ptr, in_offset, in_len, true, + out_buf, out_len); +} - { - ACOMP_REQUEST_ALLOC(req, compr->cc, GFP_NOFS | __GFP_NOWARN); +static int ubifs_decompress_common(const struct ubifs_info *c, + const void *in_buf, int in_len, + void *out_ptr, size_t out_offset, + int *out_len, bool out_folio, + int compr_type) +{ + struct ubifs_compressor *compr; + int dlen = *out_len; + int err; - acomp_request_set_src_folio(req, in_folio, in_offset, in_len); - err = ubifs_compress_req(c, req, out_buf, out_len, compr->name); + if (unlikely(compr_type < 0 || compr_type >= UBIFS_COMPR_TYPES_CNT)) { + ubifs_err(c, "invalid compression type %d", compr_type); + return -EINVAL; } - if (err) - goto no_compr; - - return; + compr = ubifs_compressors[compr_type]; -no_compr: - memcpy_from_folio(out_buf, in_folio, in_offset, in_len); - *out_len = in_len; - *compr_type = UBIFS_COMPR_NONE; -} + if (unlikely(!compr->capi_name)) { + ubifs_err(c, "%s compression is not compiled in", compr->name); + return -EINVAL; + } -static int ubifs_decompress_req(const struct ubifs_info *c, - struct acomp_req *req, - const void *in_buf, int in_len, int *out_len, - const char *compr_name) -{ - struct crypto_wait wait; - int err; + if (compr_type == UBIFS_COMPR_NONE) { + if (out_folio) + memcpy_to_folio(out_ptr, out_offset, in_buf, in_len); + else + memcpy(out_ptr, in_buf, in_len); + *out_len = in_len; + return 0; + } - crypto_init_wait(&wait); - acomp_request_set_callback(req, CRYPTO_TFM_REQ_MAY_BACKLOG, - crypto_req_done, &wait); - acomp_request_set_src_dma(req, in_buf, in_len); - err = crypto_acomp_decompress(req); - err = crypto_wait_req(err, &wait); - *out_len = req->dlen; + do { + ACOMP_REQUEST_ON_STACK(req, compr->cc); + DECLARE_CRYPTO_WAIT(wait); + acomp_request_set_callback(req, CRYPTO_TFM_REQ_MAY_BACKLOG, + crypto_req_done, &wait); + acomp_request_set_src_dma(req, in_buf, in_len); + if (out_folio) + acomp_request_set_dst_folio(req, out_ptr, out_offset, + dlen); + else + acomp_request_set_dst_dma(req, out_ptr, dlen); + err = crypto_acomp_decompress(req); + dlen = req->dlen; + if (err != -EAGAIN) + break; + + req = ACOMP_REQUEST_CLONE(req, GFP_NOFS | __GFP_NOWARN); + err = crypto_acomp_decompress(req); + err = crypto_wait_req(err, &wait); + dlen = req->dlen; + acomp_request_free(req); + } while (0); + + *out_len = dlen; if (err) ubifs_err(c, "cannot decompress %d bytes, compressor %s, error %d", - in_len, compr_name, err); - - acomp_request_free(req); + in_len, compr->name, err); return err; } @@ -235,33 +267,8 @@ static int ubifs_decompress_req(const struct ubifs_info *c, int ubifs_decompress(const struct ubifs_info *c, const void *in_buf, int in_len, void *out_buf, int *out_len, int compr_type) { - struct ubifs_compressor *compr; - - if (unlikely(compr_type < 0 || compr_type >= UBIFS_COMPR_TYPES_CNT)) { - ubifs_err(c, "invalid compression type %d", compr_type); - return -EINVAL; - } - - compr = ubifs_compressors[compr_type]; - - if (unlikely(!compr->capi_name)) { - ubifs_err(c, "%s compression is not compiled in", compr->name); - return -EINVAL; - } - - if (compr_type == UBIFS_COMPR_NONE) { - memcpy(out_buf, in_buf, in_len); - *out_len = in_len; - return 0; - } - - { - ACOMP_REQUEST_ALLOC(req, compr->cc, GFP_NOFS | __GFP_NOWARN); - - acomp_request_set_dst_dma(req, out_buf, *out_len); - return ubifs_decompress_req(c, req, in_buf, in_len, out_len, - compr->name); - } + return ubifs_decompress_common(c, in_buf, in_len, out_buf, 0, out_len, + false, compr_type); } /** @@ -283,34 +290,8 @@ int ubifs_decompress_folio(const struct ubifs_info *c, const void *in_buf, int in_len, struct folio *out_folio, size_t out_offset, int *out_len, int compr_type) { - struct ubifs_compressor *compr; - - if (unlikely(compr_type < 0 || compr_type >= UBIFS_COMPR_TYPES_CNT)) { - ubifs_err(c, "invalid compression type %d", compr_type); - return -EINVAL; - } - - compr = ubifs_compressors[compr_type]; - - if (unlikely(!compr->capi_name)) { - ubifs_err(c, "%s compression is not compiled in", compr->name); - return -EINVAL; - } - - if (compr_type == UBIFS_COMPR_NONE) { - memcpy_to_folio(out_folio, out_offset, in_buf, in_len); - *out_len = in_len; - return 0; - } - - { - ACOMP_REQUEST_ALLOC(req, compr->cc, GFP_NOFS | __GFP_NOWARN); - - acomp_request_set_dst_folio(req, out_folio, out_offset, - *out_len); - return ubifs_decompress_req(c, req, in_buf, in_len, out_len, - compr->name); - } + return ubifs_decompress_common(c, in_buf, in_len, out_folio, + out_offset, out_len, true, compr_type); } /** -- 2.51.0 From 018cba2ecc3bb97d3cb24470d2e1245cd90d98c3 Mon Sep 17 00:00:00 2001 From: Herbert Xu Date: Mon, 7 Apr 2025 18:03:05 +0800 Subject: [PATCH 15/16] crypto: acomp - Remove ACOMP_REQUEST_ALLOC Remove ACOMP_REQUEST_ALLOC in favour of ACOMP_REQUEST_ON_STACK with ACOMP_REQUEST_CLONE. Signed-off-by: Herbert Xu --- include/crypto/acompress.h | 21 --------------------- 1 file changed, 21 deletions(-) diff --git a/include/crypto/acompress.h b/include/crypto/acompress.h index 93cee67c27c0..96ec0090a855 100644 --- a/include/crypto/acompress.h +++ b/include/crypto/acompress.h @@ -48,12 +48,6 @@ #define MAX_SYNC_COMP_REQSIZE 0 -#define ACOMP_REQUEST_ALLOC(name, tfm, gfp) \ - char __##name##_req[sizeof(struct acomp_req) + \ - MAX_SYNC_COMP_REQSIZE] CRYPTO_MINALIGN_ATTR; \ - struct acomp_req *name = acomp_request_alloc_init( \ - __##name##_req, (tfm), (gfp)) - #define ACOMP_REQUEST_ON_STACK(name, tfm) \ char __##name##_req[sizeof(struct acomp_req) + \ MAX_SYNC_COMP_REQSIZE] CRYPTO_MINALIGN_ATTR; \ @@ -580,21 +574,6 @@ int crypto_acomp_compress(struct acomp_req *req); */ int crypto_acomp_decompress(struct acomp_req *req); -static inline struct acomp_req *acomp_request_alloc_init( - char *buf, struct crypto_acomp *tfm, gfp_t gfp) -{ - struct acomp_req *req; - - if ((req = acomp_request_alloc(tfm, gfp))) - return req; - - req = (void *)buf; - acomp_request_set_tfm(req, tfm->fb); - req->base.flags = CRYPTO_TFM_REQ_ON_STACK; - - return req; -} - static inline struct acomp_req *acomp_request_on_stack_init( char *buf, struct crypto_acomp *tfm) { -- 2.51.0 From 5f3437e9c89eec7bbf0ee5f582894d41f57528bc Mon Sep 17 00:00:00 2001 From: Herbert Xu Date: Mon, 7 Apr 2025 18:05:27 +0800 Subject: [PATCH 16/16] crypto: acomp - Simplify folio handling Rather than storing the folio as is and handling it later, convert it to a scatterlist right away. Signed-off-by: Herbert Xu --- crypto/acompress.c | 42 ++------------------------- crypto/scompress.c | 10 ++----- include/crypto/acompress.h | 45 +++++------------------------ include/crypto/internal/acompress.h | 16 +--------- 4 files changed, 12 insertions(+), 101 deletions(-) diff --git a/crypto/acompress.c b/crypto/acompress.c index 530b9bfd03a5..674325ecefbc 100644 --- a/crypto/acompress.c +++ b/crypto/acompress.c @@ -33,9 +33,7 @@ struct crypto_scomp; enum { ACOMP_WALK_SLEEP = 1 << 0, ACOMP_WALK_SRC_LINEAR = 1 << 1, - ACOMP_WALK_SRC_FOLIO = 1 << 2, - ACOMP_WALK_DST_LINEAR = 1 << 3, - ACOMP_WALK_DST_FOLIO = 1 << 4, + ACOMP_WALK_DST_LINEAR = 1 << 2, }; static const struct crypto_type crypto_acomp_type; @@ -195,12 +193,8 @@ static void acomp_reqchain_virt(struct acomp_req *req) if (state->flags & CRYPTO_ACOMP_REQ_SRC_VIRT) acomp_request_set_src_dma(req, state->src, slen); - else if (state->flags & CRYPTO_ACOMP_REQ_SRC_FOLIO) - acomp_request_set_src_folio(req, state->sfolio, req->soff, slen); if (state->flags & CRYPTO_ACOMP_REQ_DST_VIRT) acomp_request_set_dst_dma(req, state->dst, dlen); - else if (state->flags & CRYPTO_ACOMP_REQ_DST_FOLIO) - acomp_request_set_dst_folio(req, state->dfolio, req->doff, dlen); } static void acomp_virt_to_sg(struct acomp_req *req) @@ -208,9 +202,7 @@ static void acomp_virt_to_sg(struct acomp_req *req) struct acomp_req_chain *state = &req->chain; state->flags = req->base.flags & (CRYPTO_ACOMP_REQ_SRC_VIRT | - CRYPTO_ACOMP_REQ_DST_VIRT | - CRYPTO_ACOMP_REQ_SRC_FOLIO | - CRYPTO_ACOMP_REQ_DST_FOLIO); + CRYPTO_ACOMP_REQ_DST_VIRT); if (acomp_request_src_isvirt(req)) { unsigned int slen = req->slen; @@ -219,16 +211,6 @@ static void acomp_virt_to_sg(struct acomp_req *req) state->src = svirt; sg_init_one(&state->ssg, svirt, slen); acomp_request_set_src_sg(req, &state->ssg, slen); - } else if (acomp_request_src_isfolio(req)) { - struct folio *folio = req->sfolio; - unsigned int slen = req->slen; - size_t off = req->soff; - - state->sfolio = folio; - sg_init_table(&state->ssg, 1); - sg_set_page(&state->ssg, folio_page(folio, off / PAGE_SIZE), - slen, off % PAGE_SIZE); - acomp_request_set_src_sg(req, &state->ssg, slen); } if (acomp_request_dst_isvirt(req)) { @@ -238,16 +220,6 @@ static void acomp_virt_to_sg(struct acomp_req *req) state->dst = dvirt; sg_init_one(&state->dsg, dvirt, dlen); acomp_request_set_dst_sg(req, &state->dsg, dlen); - } else if (acomp_request_dst_isfolio(req)) { - struct folio *folio = req->dfolio; - unsigned int dlen = req->dlen; - size_t off = req->doff; - - state->dfolio = folio; - sg_init_table(&state->dsg, 1); - sg_set_page(&state->dsg, folio_page(folio, off / PAGE_SIZE), - dlen, off % PAGE_SIZE); - acomp_request_set_src_sg(req, &state->dsg, dlen); } } @@ -579,18 +551,8 @@ int acomp_walk_virt(struct acomp_walk *__restrict walk, walk->flags |= ACOMP_WALK_SLEEP; if ((req->base.flags & CRYPTO_ACOMP_REQ_SRC_VIRT)) walk->flags |= ACOMP_WALK_SRC_LINEAR; - else if ((req->base.flags & CRYPTO_ACOMP_REQ_SRC_FOLIO)) { - src = &req->chain.ssg; - sg_init_table(src, 1); - sg_set_folio(src, req->sfolio, walk->slen, req->soff); - } if ((req->base.flags & CRYPTO_ACOMP_REQ_DST_VIRT)) walk->flags |= ACOMP_WALK_DST_LINEAR; - else if ((req->base.flags & CRYPTO_ACOMP_REQ_DST_FOLIO)) { - dst = &req->chain.dsg; - sg_init_table(dst, 1); - sg_set_folio(dst, req->dfolio, walk->dlen, req->doff); - } if ((walk->flags & ACOMP_WALK_SRC_LINEAR)) { walk->in.sg = (void *)req->svirt; diff --git a/crypto/scompress.c b/crypto/scompress.c index 7ade3f2fee7e..c330b81bc5a6 100644 --- a/crypto/scompress.c +++ b/crypto/scompress.c @@ -193,10 +193,7 @@ static int scomp_acomp_comp_decomp(struct acomp_req *req, int dir) if (dst_isvirt) dst = req->dvirt; else { - if (acomp_request_dst_isfolio(req)) { - dpage = folio_page(req->dfolio, 0); - doff = req->doff; - } else if (dlen <= req->dst->length) { + if (dlen <= req->dst->length) { dpage = sg_page(req->dst); doff = req->dst->offset; } else @@ -218,10 +215,7 @@ static int scomp_acomp_comp_decomp(struct acomp_req *req, int dir) else { src = NULL; do { - if (acomp_request_src_isfolio(req)) { - spage = folio_page(req->sfolio, 0); - soff = req->soff; - } else if (slen <= req->src->length) { + if (slen <= req->src->length) { spage = sg_page(req->src); soff = req->src->offset; } else diff --git a/include/crypto/acompress.h b/include/crypto/acompress.h index 96ec0090a855..1b30290d6380 100644 --- a/include/crypto/acompress.h +++ b/include/crypto/acompress.h @@ -32,17 +32,10 @@ /* Set this bit for if virtual address destination cannot be used for DMA. */ #define CRYPTO_ACOMP_REQ_DST_NONDMA 0x00000010 -/* Set this bit if source is a folio. */ -#define CRYPTO_ACOMP_REQ_SRC_FOLIO 0x00000020 - -/* Set this bit if destination is a folio. */ -#define CRYPTO_ACOMP_REQ_DST_FOLIO 0x00000040 - /* Private flags that should not be touched by the user. */ #define CRYPTO_ACOMP_REQ_PRIVATE \ (CRYPTO_ACOMP_REQ_SRC_VIRT | CRYPTO_ACOMP_REQ_SRC_NONDMA | \ - CRYPTO_ACOMP_REQ_DST_VIRT | CRYPTO_ACOMP_REQ_DST_NONDMA | \ - CRYPTO_ACOMP_REQ_SRC_FOLIO | CRYPTO_ACOMP_REQ_DST_FOLIO) + CRYPTO_ACOMP_REQ_DST_VIRT | CRYPTO_ACOMP_REQ_DST_NONDMA) #define CRYPTO_ACOMP_DST_MAX 131072 @@ -84,10 +77,6 @@ struct acomp_req_chain { * @dst: Destination scatterlist * @svirt: Source virtual address * @dvirt: Destination virtual address - * @sfolio: Source folio - * @soff: Source folio offset - * @dfolio: Destination folio - * @doff: Destination folio offset * @slen: Size of the input buffer * @dlen: Size of the output buffer and number of bytes produced * @chain: Private API code data, do not use @@ -98,15 +87,11 @@ struct acomp_req { union { struct scatterlist *src; const u8 *svirt; - struct folio *sfolio; }; union { struct scatterlist *dst; u8 *dvirt; - struct folio *dfolio; }; - size_t soff; - size_t doff; unsigned int slen; unsigned int dlen; @@ -373,8 +358,6 @@ static inline void acomp_request_set_params(struct acomp_req *req, req->base.flags &= ~(CRYPTO_ACOMP_REQ_SRC_VIRT | CRYPTO_ACOMP_REQ_SRC_NONDMA | - CRYPTO_ACOMP_REQ_SRC_FOLIO | - CRYPTO_ACOMP_REQ_DST_FOLIO | CRYPTO_ACOMP_REQ_DST_VIRT | CRYPTO_ACOMP_REQ_DST_NONDMA); } @@ -397,7 +380,6 @@ static inline void acomp_request_set_src_sg(struct acomp_req *req, req->base.flags &= ~CRYPTO_ACOMP_REQ_SRC_NONDMA; req->base.flags &= ~CRYPTO_ACOMP_REQ_SRC_VIRT; - req->base.flags &= ~CRYPTO_ACOMP_REQ_SRC_FOLIO; } /** @@ -417,7 +399,6 @@ static inline void acomp_request_set_src_dma(struct acomp_req *req, req->slen = slen; req->base.flags &= ~CRYPTO_ACOMP_REQ_SRC_NONDMA; - req->base.flags &= ~CRYPTO_ACOMP_REQ_SRC_FOLIO; req->base.flags |= CRYPTO_ACOMP_REQ_SRC_VIRT; } @@ -438,7 +419,6 @@ static inline void acomp_request_set_src_nondma(struct acomp_req *req, req->svirt = src; req->slen = slen; - req->base.flags &= ~CRYPTO_ACOMP_REQ_SRC_FOLIO; req->base.flags |= CRYPTO_ACOMP_REQ_SRC_NONDMA; req->base.flags |= CRYPTO_ACOMP_REQ_SRC_VIRT; } @@ -457,13 +437,9 @@ static inline void acomp_request_set_src_folio(struct acomp_req *req, struct folio *folio, size_t off, unsigned int len) { - req->sfolio = folio; - req->soff = off; - req->slen = len; - - req->base.flags &= ~CRYPTO_ACOMP_REQ_SRC_NONDMA; - req->base.flags &= ~CRYPTO_ACOMP_REQ_SRC_VIRT; - req->base.flags |= CRYPTO_ACOMP_REQ_SRC_FOLIO; + sg_init_table(&req->chain.ssg, 1); + sg_set_folio(&req->chain.ssg, folio, len, off); + acomp_request_set_src_sg(req, &req->chain.ssg, len); } /** @@ -484,7 +460,6 @@ static inline void acomp_request_set_dst_sg(struct acomp_req *req, req->base.flags &= ~CRYPTO_ACOMP_REQ_DST_NONDMA; req->base.flags &= ~CRYPTO_ACOMP_REQ_DST_VIRT; - req->base.flags &= ~CRYPTO_ACOMP_REQ_DST_FOLIO; } /** @@ -504,7 +479,6 @@ static inline void acomp_request_set_dst_dma(struct acomp_req *req, req->dlen = dlen; req->base.flags &= ~CRYPTO_ACOMP_REQ_DST_NONDMA; - req->base.flags &= ~CRYPTO_ACOMP_REQ_DST_FOLIO; req->base.flags |= CRYPTO_ACOMP_REQ_DST_VIRT; } @@ -524,7 +498,6 @@ static inline void acomp_request_set_dst_nondma(struct acomp_req *req, req->dvirt = dst; req->dlen = dlen; - req->base.flags &= ~CRYPTO_ACOMP_REQ_DST_FOLIO; req->base.flags |= CRYPTO_ACOMP_REQ_DST_NONDMA; req->base.flags |= CRYPTO_ACOMP_REQ_DST_VIRT; } @@ -543,13 +516,9 @@ static inline void acomp_request_set_dst_folio(struct acomp_req *req, struct folio *folio, size_t off, unsigned int len) { - req->dfolio = folio; - req->doff = off; - req->dlen = len; - - req->base.flags &= ~CRYPTO_ACOMP_REQ_DST_NONDMA; - req->base.flags &= ~CRYPTO_ACOMP_REQ_DST_VIRT; - req->base.flags |= CRYPTO_ACOMP_REQ_DST_FOLIO; + sg_init_table(&req->chain.dsg, 1); + sg_set_folio(&req->chain.dsg, folio, len, off); + acomp_request_set_dst_sg(req, &req->chain.dsg, len); } /** diff --git a/include/crypto/internal/acompress.h b/include/crypto/internal/acompress.h index b51d66633935..d6d53c7696fd 100644 --- a/include/crypto/internal/acompress.h +++ b/include/crypto/internal/acompress.h @@ -154,9 +154,7 @@ void crypto_unregister_acomps(struct acomp_alg *algs, int count); static inline bool acomp_request_issg(struct acomp_req *req) { return !(req->base.flags & (CRYPTO_ACOMP_REQ_SRC_VIRT | - CRYPTO_ACOMP_REQ_DST_VIRT | - CRYPTO_ACOMP_REQ_SRC_FOLIO | - CRYPTO_ACOMP_REQ_DST_FOLIO)); + CRYPTO_ACOMP_REQ_DST_VIRT)); } static inline bool acomp_request_src_isvirt(struct acomp_req *req) @@ -191,16 +189,6 @@ static inline bool acomp_request_isnondma(struct acomp_req *req) CRYPTO_ACOMP_REQ_DST_NONDMA); } -static inline bool acomp_request_src_isfolio(struct acomp_req *req) -{ - return req->base.flags & CRYPTO_ACOMP_REQ_SRC_FOLIO; -} - -static inline bool acomp_request_dst_isfolio(struct acomp_req *req) -{ - return req->base.flags & CRYPTO_ACOMP_REQ_DST_FOLIO; -} - static inline bool crypto_acomp_req_chain(struct crypto_acomp *tfm) { return crypto_tfm_req_chain(&tfm->base); @@ -250,8 +238,6 @@ static inline struct acomp_req *acomp_fbreq_on_stack_init( req->dst = old->dst; req->slen = old->slen; req->dlen = old->dlen; - req->soff = old->soff; - req->doff = old->doff; return req; } -- 2.51.0