s390/crypto: Add hardware acceleration for full AES-XTS mode

Add new cipher exploiting the full AES-XTS hardware acceleration
introduced with message-security assist extension 10.

The full AES-XTS cipher is registered as preferred cipher in addition
to the discrete AES-XTS variant.

Reviewed-by: Harald Freudenberger <freude@linux.ibm.com>
Signed-off-by: Holger Dengler <dengler@linux.ibm.com>
Signed-off-by: Vasily Gorbik <gor@linux.ibm.com>

diff --git a/arch/s390/crypto/aes_s390.c b/arch/s390/crypto/aes_s390.c
index c6fe5405de4a4c46eae0db78c11b6fa9a41afa46..8cc02d6e0d0fecbd3606bbf687ff050f2cae3688 100644 (file)
--- a/arch/s390/crypto/aes_s390.c
+++ b/arch/s390/crypto/aes_s390.c
@@ -51,8 +51,13 @@ struct s390_aes_ctx {
 };
 
 struct s390_xts_ctx {
-       u8 key[32];
-       u8 pcc_key[32];
+       union {
+               u8 keys[64];
+               struct {
+                       u8 key[32];
+                       u8 pcc_key[32];
+               };
+       };
        int key_len;
        unsigned long fc;
        struct crypto_skcipher *fallback;
@@ -526,6 +531,108 @@ static struct skcipher_alg xts_aes_alg = {
        .decrypt                =       xts_aes_decrypt,
 };
 
+static int fullxts_aes_set_key(struct crypto_skcipher *tfm, const u8 *in_key,
+                              unsigned int key_len)
+{
+       struct s390_xts_ctx *xts_ctx = crypto_skcipher_ctx(tfm);
+       unsigned long fc;
+       int err;
+
+       err = xts_fallback_setkey(tfm, in_key, key_len);
+       if (err)
+               return err;
+
+       /* Pick the correct function code based on the key length */
+       fc = (key_len == 32) ? CPACF_KM_XTS_128_FULL :
+            (key_len == 64) ? CPACF_KM_XTS_256_FULL : 0;
+
+       /* Check if the function code is available */
+       xts_ctx->fc = (fc && cpacf_test_func(&km_functions, fc)) ? fc : 0;
+       if (!xts_ctx->fc)
+               return 0;
+
+       /* Store double-key */
+       memcpy(xts_ctx->keys, in_key, key_len);
+       xts_ctx->key_len = key_len;
+       return 0;
+}
+
+static int fullxts_aes_crypt(struct skcipher_request *req,  unsigned long modifier)
+{
+       struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
+       struct s390_xts_ctx *xts_ctx = crypto_skcipher_ctx(tfm);
+       unsigned int offset, nbytes, n;
+       struct skcipher_walk walk;
+       int ret;
+       struct {
+               __u8 key[64];
+               __u8 tweak[16];
+               __u8 nap[16];
+       } fxts_param = {
+               .nap = {0},
+       };
+
+       if (req->cryptlen < AES_BLOCK_SIZE)
+               return -EINVAL;
+
+       if (unlikely(!xts_ctx->fc || (req->cryptlen % AES_BLOCK_SIZE) != 0)) {
+               struct skcipher_request *subreq = skcipher_request_ctx(req);
+
+               *subreq = *req;
+               skcipher_request_set_tfm(subreq, xts_ctx->fallback);
+               return (modifier & CPACF_DECRYPT) ?
+                       crypto_skcipher_decrypt(subreq) :
+                       crypto_skcipher_encrypt(subreq);
+       }
+
+       ret = skcipher_walk_virt(&walk, req, false);
+       if (ret)
+               return ret;
+
+       offset = xts_ctx->key_len & 0x20;
+       memcpy(fxts_param.key + offset, xts_ctx->keys, xts_ctx->key_len);
+       memcpy(fxts_param.tweak, req->iv, AES_BLOCK_SIZE);
+       fxts_param.nap[0] = 0x01; /* initial alpha power (1, little-endian) */
+
+       while ((nbytes = walk.nbytes) != 0) {
+               /* only use complete blocks */
+               n = nbytes & ~(AES_BLOCK_SIZE - 1);
+               cpacf_km(xts_ctx->fc | modifier, fxts_param.key + offset,
+                        walk.dst.virt.addr, walk.src.virt.addr, n);
+               ret = skcipher_walk_done(&walk, nbytes - n);
+       }
+       memzero_explicit(&fxts_param, sizeof(fxts_param));
+       return ret;
+}
+
+static int fullxts_aes_encrypt(struct skcipher_request *req)
+{
+       return fullxts_aes_crypt(req, 0);
+}
+
+static int fullxts_aes_decrypt(struct skcipher_request *req)
+{
+       return fullxts_aes_crypt(req, CPACF_DECRYPT);
+}
+
+static struct skcipher_alg fullxts_aes_alg = {
+       .base.cra_name          =       "xts(aes)",
+       .base.cra_driver_name   =       "full-xts-aes-s390",
+       .base.cra_priority      =       403,    /* aes-xts-s390 + 1 */
+       .base.cra_flags         =       CRYPTO_ALG_NEED_FALLBACK,
+       .base.cra_blocksize     =       AES_BLOCK_SIZE,
+       .base.cra_ctxsize       =       sizeof(struct s390_xts_ctx),
+       .base.cra_module        =       THIS_MODULE,
+       .init                   =       xts_fallback_init,
+       .exit                   =       xts_fallback_exit,
+       .min_keysize            =       2 * AES_MIN_KEY_SIZE,
+       .max_keysize            =       2 * AES_MAX_KEY_SIZE,
+       .ivsize                 =       AES_BLOCK_SIZE,
+       .setkey                 =       fullxts_aes_set_key,
+       .encrypt                =       fullxts_aes_encrypt,
+       .decrypt                =       fullxts_aes_decrypt,
+};
+
 static int ctr_aes_set_key(struct crypto_skcipher *tfm, const u8 *in_key,
                           unsigned int key_len)
 {
@@ -955,7 +1062,7 @@ static struct aead_alg gcm_aes_aead = {
 };
 
 static struct crypto_alg *aes_s390_alg;
-static struct skcipher_alg *aes_s390_skcipher_algs[4];
+static struct skcipher_alg *aes_s390_skcipher_algs[5];
 static int aes_s390_skciphers_num;
 static struct aead_alg *aes_s390_aead_alg;
 
@@ -1012,6 +1119,13 @@ static int __init aes_s390_init(void)
                        goto out_err;
        }
 
+       if (cpacf_test_func(&km_functions, CPACF_KM_XTS_128_FULL) ||
+           cpacf_test_func(&km_functions, CPACF_KM_XTS_256_FULL)) {
+               ret = aes_s390_register_skcipher(&fullxts_aes_alg);
+               if (ret)
+                       goto out_err;
+       }
+
        if (cpacf_test_func(&km_functions, CPACF_KM_XTS_128) ||
            cpacf_test_func(&km_functions, CPACF_KM_XTS_256)) {
                ret = aes_s390_register_skcipher(&xts_aes_alg);

diff --git a/arch/s390/include/asm/cpacf.h b/arch/s390/include/asm/cpacf.h
index dae8843b164f82f532971c067e1bf6e063cac92b..b2f355751bf6b7346f7305c4ff7327d7461e203c 100644 (file)
--- a/arch/s390/include/asm/cpacf.h
+++ b/arch/s390/include/asm/cpacf.h
@@ -54,6 +54,8 @@
 #define CPACF_KM_XTS_256       0x34
 #define CPACF_KM_PXTS_128      0x3a
 #define CPACF_KM_PXTS_256      0x3c
+#define CPACF_KM_XTS_128_FULL  0x52
+#define CPACF_KM_XTS_256_FULL  0x54
 
 /*
  * Function codes for the KMC (CIPHER MESSAGE WITH CHAINING)