crypto: shash - eliminate indirect call for default import and export

Most shash algorithms don't have custom ->import and ->export functions,
resulting in the memcpy() based default being used.  Yet,
crypto_shash_import() and crypto_shash_export() still make an indirect
call, which is expensive.  Therefore, change how the default import and
export are called to make it so that crypto_shash_import() and
crypto_shash_export() don't do an indirect call in this case.

Signed-off-by: Eric Biggers <ebiggers@google.com>
Signed-off-by: Herbert Xu <herbert@gondor.apana.org.au>

diff --git a/crypto/shash.c b/crypto/shash.c
index 15fee57cca8efd8b227de8f467c89d8db32ab9f5..52420c41db44ae549a341fdd9b440583f97300da 100644 (file)
--- a/crypto/shash.c
+++ b/crypto/shash.c
@@ -277,17 +277,34 @@ int crypto_shash_tfm_digest(struct crypto_shash *tfm, const u8 *data,
 }
 EXPORT_SYMBOL_GPL(crypto_shash_tfm_digest);
 
-static int shash_default_export(struct shash_desc *desc, void *out)
+int crypto_shash_export(struct shash_desc *desc, void *out)
 {
-       memcpy(out, shash_desc_ctx(desc), crypto_shash_descsize(desc->tfm));
+       struct crypto_shash *tfm = desc->tfm;
+       struct shash_alg *shash = crypto_shash_alg(tfm);
+
+       if (shash->export)
+               return shash->export(desc, out);
+
+       memcpy(out, shash_desc_ctx(desc), crypto_shash_descsize(tfm));
        return 0;
 }
+EXPORT_SYMBOL_GPL(crypto_shash_export);
 
-static int shash_default_import(struct shash_desc *desc, const void *in)
+int crypto_shash_import(struct shash_desc *desc, const void *in)
 {
-       memcpy(shash_desc_ctx(desc), in, crypto_shash_descsize(desc->tfm));
+       struct crypto_shash *tfm = desc->tfm;
+       struct shash_alg *shash = crypto_shash_alg(tfm);
+
+       if (crypto_shash_get_flags(tfm) & CRYPTO_TFM_NEED_KEY)
+               return -ENOKEY;
+
+       if (shash->import)
+               return shash->import(desc, in);
+
+       memcpy(shash_desc_ctx(desc), in, crypto_shash_descsize(tfm));
        return 0;
 }
+EXPORT_SYMBOL_GPL(crypto_shash_import);
 
 static int shash_async_setkey(struct crypto_ahash *tfm, const u8 *key,
                              unsigned int keylen)
@@ -666,15 +683,23 @@ static int shash_prepare_alg(struct shash_alg *alg)
        base->cra_type = &crypto_shash_type;
        base->cra_flags |= CRYPTO_ALG_TYPE_SHASH;
 
+       /*
+        * Handle missing optional functions.  For each one we can either
+        * install a default here, or we can leave the pointer as NULL and check
+        * the pointer for NULL in crypto_shash_*(), avoiding an indirect call
+        * when the default behavior is desired.  For ->finup and ->digest we
+        * install defaults, since for optimal performance algorithms should
+        * implement these anyway.  On the other hand, for ->import and
+        * ->export the common case and best performance comes from the simple
+        * memcpy of the shash_desc_ctx, so when those pointers are NULL we
+        * leave them NULL and provide the memcpy with no indirect call.
+        */
        if (!alg->finup)
                alg->finup = shash_default_finup;
        if (!alg->digest)
                alg->digest = shash_default_digest;
-       if (!alg->export) {
-               alg->export = shash_default_export;
-               alg->import = shash_default_import;
+       if (!alg->export)
                alg->halg.statesize = alg->descsize;
-       }
        if (!alg->setkey)
                alg->setkey = shash_no_setkey;
 

diff --git a/include/crypto/hash.h b/include/crypto/hash.h
index f7c2a22cd776daa3eb42c6310ddd8bd69ee175be..52e57e93b2f591b253eb11ef7c962aa85e8de2df 100644 (file)
--- a/include/crypto/hash.h
+++ b/include/crypto/hash.h
@@ -952,10 +952,7 @@ int crypto_shash_tfm_digest(struct crypto_shash *tfm, const u8 *data,
  * Context: Any context.
  * Return: 0 if the export creation was successful; < 0 if an error occurred
  */
-static inline int crypto_shash_export(struct shash_desc *desc, void *out)
-{
-       return crypto_shash_alg(desc->tfm)->export(desc, out);
-}
+int crypto_shash_export(struct shash_desc *desc, void *out);
 
 /**
  * crypto_shash_import() - import operational state
@@ -969,15 +966,7 @@ static inline int crypto_shash_export(struct shash_desc *desc, void *out)
  * Context: Any context.
  * Return: 0 if the import was successful; < 0 if an error occurred
  */
-static inline int crypto_shash_import(struct shash_desc *desc, const void *in)
-{
-       struct crypto_shash *tfm = desc->tfm;
-
-       if (crypto_shash_get_flags(tfm) & CRYPTO_TFM_NEED_KEY)
-               return -ENOKEY;
-
-       return crypto_shash_alg(tfm)->import(desc, in);
-}
+int crypto_shash_import(struct shash_desc *desc, const void *in);
 
 /**
  * crypto_shash_init() - (re)initialize message digest