Commit 2698bce1 authored by Ard Biesheuvel's avatar Ard Biesheuvel Committed by Herbert Xu

crypto: aegis128 - avoid function pointers for parameterization

Instead of passing around an ops structure with function pointers,
which forces indirect calls to be used, refactor the code slightly
so we can use ordinary function calls. At the same time, switch to
a static key to decide whether or not the SIMD code path may be used.
Signed-off-by: default avatarArd Biesheuvel <ard.biesheuvel@linaro.org>
Signed-off-by: default avatarHerbert Xu <herbert@gondor.apana.org.au>
parent be867f98
...@@ -13,6 +13,7 @@ ...@@ -13,6 +13,7 @@
#include <crypto/scatterwalk.h> #include <crypto/scatterwalk.h>
#include <linux/err.h> #include <linux/err.h>
#include <linux/init.h> #include <linux/init.h>
#include <linux/jump_label.h>
#include <linux/kernel.h> #include <linux/kernel.h>
#include <linux/module.h> #include <linux/module.h>
#include <linux/scatterlist.h> #include <linux/scatterlist.h>
...@@ -35,15 +36,7 @@ struct aegis_ctx { ...@@ -35,15 +36,7 @@ struct aegis_ctx {
union aegis_block key; union aegis_block key;
}; };
struct aegis128_ops { static __ro_after_init DEFINE_STATIC_KEY_FALSE(have_simd);
int (*skcipher_walk_init)(struct skcipher_walk *walk,
struct aead_request *req, bool atomic);
void (*crypt_chunk)(struct aegis_state *state, u8 *dst,
const u8 *src, unsigned int size);
};
static bool have_simd;
static const union aegis_block crypto_aegis_const[2] = { static const union aegis_block crypto_aegis_const[2] = {
{ .words64 = { { .words64 = {
...@@ -59,7 +52,7 @@ static const union aegis_block crypto_aegis_const[2] = { ...@@ -59,7 +52,7 @@ static const union aegis_block crypto_aegis_const[2] = {
static bool aegis128_do_simd(void) static bool aegis128_do_simd(void)
{ {
#ifdef CONFIG_CRYPTO_AEGIS128_SIMD #ifdef CONFIG_CRYPTO_AEGIS128_SIMD
if (have_simd) if (static_branch_likely(&have_simd))
return crypto_simd_usable(); return crypto_simd_usable();
#endif #endif
return false; return false;
...@@ -323,25 +316,27 @@ static void crypto_aegis128_process_ad(struct aegis_state *state, ...@@ -323,25 +316,27 @@ static void crypto_aegis128_process_ad(struct aegis_state *state,
} }
} }
static void crypto_aegis128_process_crypt(struct aegis_state *state, static __always_inline
struct aead_request *req, int crypto_aegis128_process_crypt(struct aegis_state *state,
const struct aegis128_ops *ops) struct aead_request *req,
struct skcipher_walk *walk,
void (*crypt)(struct aegis_state *state,
u8 *dst, const u8 *src,
unsigned int size))
{ {
struct skcipher_walk walk; int err = 0;
ops->skcipher_walk_init(&walk, req, false); while (walk->nbytes) {
unsigned int nbytes = walk->nbytes;
while (walk.nbytes) { if (nbytes < walk->total)
unsigned int nbytes = walk.nbytes; nbytes = round_down(nbytes, walk->stride);
if (nbytes < walk.total) crypt(state, walk->dst.virt.addr, walk->src.virt.addr, nbytes);
nbytes = round_down(nbytes, walk.stride);
ops->crypt_chunk(state, walk.dst.virt.addr, walk.src.virt.addr, err = skcipher_walk_done(walk, walk->nbytes - nbytes);
nbytes);
skcipher_walk_done(&walk, walk.nbytes - nbytes);
} }
return err;
} }
static void crypto_aegis128_final(struct aegis_state *state, static void crypto_aegis128_final(struct aegis_state *state,
...@@ -390,39 +385,27 @@ static int crypto_aegis128_setauthsize(struct crypto_aead *tfm, ...@@ -390,39 +385,27 @@ static int crypto_aegis128_setauthsize(struct crypto_aead *tfm,
return 0; return 0;
} }
static void crypto_aegis128_crypt(struct aead_request *req, static int crypto_aegis128_encrypt(struct aead_request *req)
union aegis_block *tag_xor,
unsigned int cryptlen,
const struct aegis128_ops *ops)
{ {
struct crypto_aead *tfm = crypto_aead_reqtfm(req); struct crypto_aead *tfm = crypto_aead_reqtfm(req);
union aegis_block tag = {};
unsigned int authsize = crypto_aead_authsize(tfm);
struct aegis_ctx *ctx = crypto_aead_ctx(tfm); struct aegis_ctx *ctx = crypto_aead_ctx(tfm);
unsigned int cryptlen = req->cryptlen;
struct skcipher_walk walk;
struct aegis_state state; struct aegis_state state;
crypto_aegis128_init(&state, &ctx->key, req->iv); crypto_aegis128_init(&state, &ctx->key, req->iv);
crypto_aegis128_process_ad(&state, req->src, req->assoclen); crypto_aegis128_process_ad(&state, req->src, req->assoclen);
crypto_aegis128_process_crypt(&state, req, ops);
crypto_aegis128_final(&state, tag_xor, req->assoclen, cryptlen);
}
static int crypto_aegis128_encrypt(struct aead_request *req)
{
const struct aegis128_ops *ops = &(struct aegis128_ops){
.skcipher_walk_init = skcipher_walk_aead_encrypt,
.crypt_chunk = crypto_aegis128_encrypt_chunk,
};
struct crypto_aead *tfm = crypto_aead_reqtfm(req);
union aegis_block tag = {};
unsigned int authsize = crypto_aead_authsize(tfm);
unsigned int cryptlen = req->cryptlen;
skcipher_walk_aead_encrypt(&walk, req, false);
if (aegis128_do_simd()) if (aegis128_do_simd())
ops = &(struct aegis128_ops){ crypto_aegis128_process_crypt(&state, req, &walk,
.skcipher_walk_init = skcipher_walk_aead_encrypt, crypto_aegis128_encrypt_chunk_simd);
.crypt_chunk = crypto_aegis128_encrypt_chunk_simd }; else
crypto_aegis128_process_crypt(&state, req, &walk,
crypto_aegis128_crypt(req, &tag, cryptlen, ops); crypto_aegis128_encrypt_chunk);
crypto_aegis128_final(&state, &tag, req->assoclen, cryptlen);
scatterwalk_map_and_copy(tag.bytes, req->dst, req->assoclen + cryptlen, scatterwalk_map_and_copy(tag.bytes, req->dst, req->assoclen + cryptlen,
authsize, 1); authsize, 1);
...@@ -431,26 +414,29 @@ static int crypto_aegis128_encrypt(struct aead_request *req) ...@@ -431,26 +414,29 @@ static int crypto_aegis128_encrypt(struct aead_request *req)
static int crypto_aegis128_decrypt(struct aead_request *req) static int crypto_aegis128_decrypt(struct aead_request *req)
{ {
const struct aegis128_ops *ops = &(struct aegis128_ops){
.skcipher_walk_init = skcipher_walk_aead_decrypt,
.crypt_chunk = crypto_aegis128_decrypt_chunk,
};
static const u8 zeros[AEGIS128_MAX_AUTH_SIZE] = {}; static const u8 zeros[AEGIS128_MAX_AUTH_SIZE] = {};
struct crypto_aead *tfm = crypto_aead_reqtfm(req); struct crypto_aead *tfm = crypto_aead_reqtfm(req);
union aegis_block tag; union aegis_block tag;
unsigned int authsize = crypto_aead_authsize(tfm); unsigned int authsize = crypto_aead_authsize(tfm);
unsigned int cryptlen = req->cryptlen - authsize; unsigned int cryptlen = req->cryptlen - authsize;
struct aegis_ctx *ctx = crypto_aead_ctx(tfm);
struct skcipher_walk walk;
struct aegis_state state;
scatterwalk_map_and_copy(tag.bytes, req->src, req->assoclen + cryptlen, scatterwalk_map_and_copy(tag.bytes, req->src, req->assoclen + cryptlen,
authsize, 0); authsize, 0);
if (aegis128_do_simd()) crypto_aegis128_init(&state, &ctx->key, req->iv);
ops = &(struct aegis128_ops){ crypto_aegis128_process_ad(&state, req->src, req->assoclen);
.skcipher_walk_init = skcipher_walk_aead_decrypt,
.crypt_chunk = crypto_aegis128_decrypt_chunk_simd };
crypto_aegis128_crypt(req, &tag, cryptlen, ops); skcipher_walk_aead_decrypt(&walk, req, false);
if (aegis128_do_simd())
crypto_aegis128_process_crypt(&state, req, &walk,
crypto_aegis128_decrypt_chunk_simd);
else
crypto_aegis128_process_crypt(&state, req, &walk,
crypto_aegis128_decrypt_chunk);
crypto_aegis128_final(&state, &tag, req->assoclen, cryptlen);
return crypto_memneq(tag.bytes, zeros, authsize) ? -EBADMSG : 0; return crypto_memneq(tag.bytes, zeros, authsize) ? -EBADMSG : 0;
} }
...@@ -481,8 +467,9 @@ static struct aead_alg crypto_aegis128_alg = { ...@@ -481,8 +467,9 @@ static struct aead_alg crypto_aegis128_alg = {
static int __init crypto_aegis128_module_init(void) static int __init crypto_aegis128_module_init(void)
{ {
if (IS_ENABLED(CONFIG_CRYPTO_AEGIS128_SIMD)) if (IS_ENABLED(CONFIG_CRYPTO_AEGIS128_SIMD) &&
have_simd = crypto_aegis128_have_simd(); crypto_aegis128_have_simd())
static_branch_enable(&have_simd);
return crypto_register_aead(&crypto_aegis128_alg); return crypto_register_aead(&crypto_aegis128_alg);
} }
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment