nvme-tcp: use crc32c() and skb_copy_and_crc32c_datagram_iter()

Now that the crc32c() library function directly takes advantage of
architecture-specific optimizations and there also now exists a function
skb_copy_and_crc32c_datagram_iter(), it is unnecessary to go through the
crypto_ahash API.  Just use those functions.  This is much simpler, and
it also improves performance due to eliminating the crypto API overhead.

Reviewed-by: Christoph Hellwig <hch@lst.de>
Reviewed-by: Sagi Grimberg <sagi@grimberg.me>
Signed-off-by: Eric Biggers <ebiggers@google.com>
Reviewed-by: Hannes Reinecke <hare@suse.de>
Link: https://patch.msgid.link/20250519175012.36581-10-ebiggers@kernel.org
Signed-off-by: Jakub Kicinski <kuba@kernel.org>

diff --git a/drivers/nvme/host/Kconfig b/drivers/nvme/host/Kconfig
index 4d64b6935bb91571139a53a4f6925f90ee54555f..7dca58f0a237bc91a1ec04d66c5856b668a2b997 100644 (file)
--- a/drivers/nvme/host/Kconfig
+++ b/drivers/nvme/host/Kconfig
@@ -84,9 +84,9 @@ config NVME_TCP
        tristate "NVM Express over Fabrics TCP host driver"
        depends on INET
        depends on BLOCK
+       select CRC32
+       select NET_CRC32C
        select NVME_FABRICS
-       select CRYPTO
-       select CRYPTO_CRC32C
        help
          This provides support for the NVMe over Fabrics protocol using
          the TCP transport.  This allows you to use remote block devices

diff --git a/drivers/nvme/host/tcp.c b/drivers/nvme/host/tcp.c
index aba365f97cf6b42ba3c93e666708c8c315d121f2..8ae6cc2280caaace98cdcfec0037e846b58a6d94 100644 (file)
--- a/drivers/nvme/host/tcp.c
+++ b/drivers/nvme/host/tcp.c
@@ -8,6 +8,7 @@
 #include <linux/init.h>
 #include <linux/slab.h>
 #include <linux/err.h>
+#include <linux/crc32.h>
 #include <linux/nvme-tcp.h>
 #include <linux/nvme-keyring.h>
 #include <net/sock.h>
@@ -16,7 +17,6 @@
 #include <net/tls_prot.h>
 #include <net/handshake.h>
 #include <linux/blk-mq.h>
-#include <crypto/hash.h>
 #include <net/busy_poll.h>
 #include <trace/events/sock.h>
 
@@ -168,8 +168,8 @@ struct nvme_tcp_queue {
        bool                    hdr_digest;
        bool                    data_digest;
        bool                    tls_enabled;
-       struct ahash_request    *rcv_hash;
-       struct ahash_request    *snd_hash;
+       u32                     rcv_crc;
+       u32                     snd_crc;
        __le32                  exp_ddgst;
        __le32                  recv_ddgst;
        struct completion       tls_complete;
@@ -456,32 +456,38 @@ nvme_tcp_fetch_request(struct nvme_tcp_queue *queue)
        return req;
 }
 
-static inline void nvme_tcp_ddgst_final(struct ahash_request *hash,
-               __le32 *dgst)
+#define NVME_TCP_CRC_SEED (~0)
+
+static inline void nvme_tcp_ddgst_update(u32 *crcp,
+               struct page *page, size_t off, size_t len)
 {
-       ahash_request_set_crypt(hash, NULL, (u8 *)dgst, 0);
-       crypto_ahash_final(hash);
+       page += off / PAGE_SIZE;
+       off %= PAGE_SIZE;
+       while (len) {
+               const void *vaddr = kmap_local_page(page);
+               size_t n = min(len, (size_t)PAGE_SIZE - off);
+
+               *crcp = crc32c(*crcp, vaddr + off, n);
+               kunmap_local(vaddr);
+               page++;
+               off = 0;
+               len -= n;
+       }
 }
 
-static inline void nvme_tcp_ddgst_update(struct ahash_request *hash,
-               struct page *page, off_t off, size_t len)
+static inline __le32 nvme_tcp_ddgst_final(u32 crc)
 {
-       struct scatterlist sg;
-
-       sg_init_table(&sg, 1);
-       sg_set_page(&sg, page, len, off);
-       ahash_request_set_crypt(hash, &sg, NULL, len);
-       crypto_ahash_update(hash);
+       return cpu_to_le32(~crc);
 }
 
-static inline void nvme_tcp_hdgst(struct ahash_request *hash,
-               void *pdu, size_t len)
+static inline __le32 nvme_tcp_hdgst(const void *pdu, size_t len)
 {
-       struct scatterlist sg;
+       return cpu_to_le32(~crc32c(NVME_TCP_CRC_SEED, pdu, len));
+}
 
-       sg_init_one(&sg, pdu, len);
-       ahash_request_set_crypt(hash, &sg, pdu + len, len);
-       crypto_ahash_digest(hash);
+static inline void nvme_tcp_set_hdgst(void *pdu, size_t len)
+{
+       *(__le32 *)(pdu + len) = nvme_tcp_hdgst(pdu, len);
 }
 
 static int nvme_tcp_verify_hdgst(struct nvme_tcp_queue *queue,
@@ -499,8 +505,7 @@ static int nvme_tcp_verify_hdgst(struct nvme_tcp_queue *queue,
        }
 
        recv_digest = *(__le32 *)(pdu + hdr->hlen);
-       nvme_tcp_hdgst(queue->rcv_hash, pdu, pdu_len);
-       exp_digest = *(__le32 *)(pdu + hdr->hlen);
+       exp_digest = nvme_tcp_hdgst(pdu, pdu_len);
        if (recv_digest != exp_digest) {
                dev_err(queue->ctrl->ctrl.device,
                        "header digest error: recv %#x expected %#x\n",
@@ -526,7 +531,7 @@ static int nvme_tcp_check_ddgst(struct nvme_tcp_queue *queue, void *pdu)
                nvme_tcp_queue_id(queue));
                return -EPROTO;
        }
-       crypto_ahash_init(queue->rcv_hash);
+       queue->rcv_crc = NVME_TCP_CRC_SEED;
 
        return 0;
 }
@@ -926,8 +931,8 @@ static int nvme_tcp_recv_data(struct nvme_tcp_queue *queue, struct sk_buff *skb,
                                iov_iter_count(&req->iter));
 
                if (queue->data_digest)
-                       ret = skb_copy_and_hash_datagram_iter(skb, *offset,
-                               &req->iter, recv_len, queue->rcv_hash);
+                       ret = skb_copy_and_crc32c_datagram_iter(skb, *offset,
+                               &req->iter, recv_len, &queue->rcv_crc);
                else
                        ret = skb_copy_datagram_iter(skb, *offset,
                                        &req->iter, recv_len);
@@ -945,7 +950,7 @@ static int nvme_tcp_recv_data(struct nvme_tcp_queue *queue, struct sk_buff *skb,
 
        if (!queue->data_remaining) {
                if (queue->data_digest) {
-                       nvme_tcp_ddgst_final(queue->rcv_hash, &queue->exp_ddgst);
+                       queue->exp_ddgst = nvme_tcp_ddgst_final(queue->rcv_crc);
                        queue->ddgst_remaining = NVME_TCP_DIGEST_LENGTH;
                } else {
                        if (pdu->hdr.flags & NVME_TCP_F_DATA_SUCCESS) {
@@ -1147,7 +1152,7 @@ static int nvme_tcp_try_send_data(struct nvme_tcp_request *req)
                        return ret;
 
                if (queue->data_digest)
-                       nvme_tcp_ddgst_update(queue->snd_hash, page,
+                       nvme_tcp_ddgst_update(&queue->snd_crc, page,
                                        offset, ret);
 
                /*
@@ -1161,8 +1166,8 @@ static int nvme_tcp_try_send_data(struct nvme_tcp_request *req)
                /* fully successful last send in current PDU */
                if (last && ret == len) {
                        if (queue->data_digest) {
-                               nvme_tcp_ddgst_final(queue->snd_hash,
-                                       &req->ddgst);
+                               req->ddgst =
+                                       nvme_tcp_ddgst_final(queue->snd_crc);
                                req->state = NVME_TCP_SEND_DDGST;
                                req->offset = 0;
                        } else {
@@ -1194,7 +1199,7 @@ static int nvme_tcp_try_send_cmd_pdu(struct nvme_tcp_request *req)
                msg.msg_flags |= MSG_EOR;
 
        if (queue->hdr_digest && !req->offset)
-               nvme_tcp_hdgst(queue->snd_hash, pdu, sizeof(*pdu));
+               nvme_tcp_set_hdgst(pdu, sizeof(*pdu));
 
        bvec_set_virt(&bvec, (void *)pdu + req->offset, len);
        iov_iter_bvec(&msg.msg_iter, ITER_SOURCE, &bvec, 1, len);
@@ -1207,7 +1212,7 @@ static int nvme_tcp_try_send_cmd_pdu(struct nvme_tcp_request *req)
                if (inline_data) {
                        req->state = NVME_TCP_SEND_DATA;
                        if (queue->data_digest)
-                               crypto_ahash_init(queue->snd_hash);
+                               queue->snd_crc = NVME_TCP_CRC_SEED;
                } else {
                        nvme_tcp_done_send_req(queue);
                }
@@ -1229,7 +1234,7 @@ static int nvme_tcp_try_send_data_pdu(struct nvme_tcp_request *req)
        int ret;
 
        if (queue->hdr_digest && !req->offset)
-               nvme_tcp_hdgst(queue->snd_hash, pdu, sizeof(*pdu));
+               nvme_tcp_set_hdgst(pdu, sizeof(*pdu));
 
        if (!req->h2cdata_left)
                msg.msg_flags |= MSG_SPLICE_PAGES;
@@ -1244,7 +1249,7 @@ static int nvme_tcp_try_send_data_pdu(struct nvme_tcp_request *req)
        if (!len) {
                req->state = NVME_TCP_SEND_DATA;
                if (queue->data_digest)
-                       crypto_ahash_init(queue->snd_hash);
+                       queue->snd_crc = NVME_TCP_CRC_SEED;
                return 1;
        }
        req->offset += ret;
@@ -1384,41 +1389,6 @@ static void nvme_tcp_io_work(struct work_struct *w)
        queue_work_on(queue->io_cpu, nvme_tcp_wq, &queue->io_work);
 }
 
-static void nvme_tcp_free_crypto(struct nvme_tcp_queue *queue)
-{
-       struct crypto_ahash *tfm = crypto_ahash_reqtfm(queue->rcv_hash);
-
-       ahash_request_free(queue->rcv_hash);
-       ahash_request_free(queue->snd_hash);
-       crypto_free_ahash(tfm);
-}
-
-static int nvme_tcp_alloc_crypto(struct nvme_tcp_queue *queue)
-{
-       struct crypto_ahash *tfm;
-
-       tfm = crypto_alloc_ahash("crc32c", 0, CRYPTO_ALG_ASYNC);
-       if (IS_ERR(tfm))
-               return PTR_ERR(tfm);
-
-       queue->snd_hash = ahash_request_alloc(tfm, GFP_KERNEL);
-       if (!queue->snd_hash)
-               goto free_tfm;
-       ahash_request_set_callback(queue->snd_hash, 0, NULL, NULL);
-
-       queue->rcv_hash = ahash_request_alloc(tfm, GFP_KERNEL);
-       if (!queue->rcv_hash)
-               goto free_snd_hash;
-       ahash_request_set_callback(queue->rcv_hash, 0, NULL, NULL);
-
-       return 0;
-free_snd_hash:
-       ahash_request_free(queue->snd_hash);
-free_tfm:
-       crypto_free_ahash(tfm);
-       return -ENOMEM;
-}
-
 static void nvme_tcp_free_async_req(struct nvme_tcp_ctrl *ctrl)
 {
        struct nvme_tcp_request *async = &ctrl->async_req;
@@ -1451,9 +1421,6 @@ static void nvme_tcp_free_queue(struct nvme_ctrl *nctrl, int qid)
        if (!test_and_clear_bit(NVME_TCP_Q_ALLOCATED, &queue->flags))
                return;
 
-       if (queue->hdr_digest || queue->data_digest)
-               nvme_tcp_free_crypto(queue);
-
        page_frag_cache_drain(&queue->pf_cache);
 
        noreclaim_flag = memalloc_noreclaim_save();
@@ -1867,21 +1834,13 @@ static int nvme_tcp_alloc_queue(struct nvme_ctrl *nctrl, int qid,
 
        queue->hdr_digest = nctrl->opts->hdr_digest;
        queue->data_digest = nctrl->opts->data_digest;
-       if (queue->hdr_digest || queue->data_digest) {
-               ret = nvme_tcp_alloc_crypto(queue);
-               if (ret) {
-                       dev_err(nctrl->device,
-                               "failed to allocate queue %d crypto\n", qid);
-                       goto err_sock;
-               }
-       }
 
        rcv_pdu_size = sizeof(struct nvme_tcp_rsp_pdu) +
                        nvme_tcp_hdgst_len(queue);
        queue->pdu = kmalloc(rcv_pdu_size, GFP_KERNEL);
        if (!queue->pdu) {
                ret = -ENOMEM;
-               goto err_crypto;
+               goto err_sock;
        }
 
        dev_dbg(nctrl->device, "connecting queue %d\n",
@@ -1914,9 +1873,6 @@ err_init_connect:
        kernel_sock_shutdown(queue->sock, SHUT_RDWR);
 err_rcv_pdu:
        kfree(queue->pdu);
-err_crypto:
-       if (queue->hdr_digest || queue->data_digest)
-               nvme_tcp_free_crypto(queue);
 err_sock:
        /* ->sock will be released by fput() */
        fput(queue->sock->file);