diff --git a/arch/arm/crypto/poly1305-glue.c b/arch/arm/crypto/poly1305-glue.c index 4464ffbf8fd1..238dbf50d05d 100644 --- a/arch/arm/crypto/poly1305-glue.c +++ b/arch/arm/crypto/poly1305-glue.c @@ -218,7 +218,7 @@ static struct shash_alg arm_poly1305_algs[] = {{ .descsize = sizeof(struct poly1305_desc_ctx), .base.cra_name = "poly1305", - .base.cra_driver_name = "poly1305-arm", + .base.cra_driver_name = "poly1305-arm-old", .base.cra_priority = 150, .base.cra_blocksize = POLY1305_BLOCK_SIZE, .base.cra_module = THIS_MODULE, @@ -238,6 +238,13 @@ static struct shash_alg arm_poly1305_algs[] = {{ #endif }}; +bool poly1305_is_arch_optimized(void) +{ + /* We always can use at least the ARM scalar implementation. */ + return true; +} +EXPORT_SYMBOL(poly1305_is_arch_optimized); + static int __init arm_poly1305_mod_init(void) { if (IS_ENABLED(CONFIG_KERNEL_MODE_NEON) && @@ -264,7 +271,7 @@ static void __exit arm_poly1305_mod_exit(void) ARRAY_SIZE(arm_poly1305_algs)); } -module_init(arm_poly1305_mod_init); +arch_initcall(arm_poly1305_mod_init); module_exit(arm_poly1305_mod_exit); MODULE_DESCRIPTION("Accelerated Poly1305 transform for ARM"); diff --git a/arch/arm64/crypto/poly1305-glue.c b/arch/arm64/crypto/poly1305-glue.c index 18883ea438f3..89415731d9be 100644 --- a/arch/arm64/crypto/poly1305-glue.c +++ b/arch/arm64/crypto/poly1305-glue.c @@ -206,6 +206,13 @@ static struct shash_alg neon_poly1305_alg = { .base.cra_module = THIS_MODULE, }; +bool poly1305_is_arch_optimized(void) +{ + /* We always can use at least the ARM64 scalar implementation. */ + return true; +} +EXPORT_SYMBOL(poly1305_is_arch_optimized); + static int __init neon_poly1305_mod_init(void) { if (!cpu_have_named_feature(ASIMD)) @@ -223,7 +230,7 @@ static void __exit neon_poly1305_mod_exit(void) crypto_unregister_shash(&neon_poly1305_alg); } -module_init(neon_poly1305_mod_init); +arch_initcall(neon_poly1305_mod_init); module_exit(neon_poly1305_mod_exit); MODULE_DESCRIPTION("Poly1305 transform using NEON instructions"); diff --git a/arch/mips/crypto/poly1305-glue.c b/arch/mips/crypto/poly1305-glue.c index c03ad0bbe69c..9eee71dc0b99 100644 --- a/arch/mips/crypto/poly1305-glue.c +++ b/arch/mips/crypto/poly1305-glue.c @@ -165,12 +165,18 @@ static struct shash_alg mips_poly1305_alg = { .descsize = sizeof(struct poly1305_desc_ctx), .base.cra_name = "poly1305", - .base.cra_driver_name = "poly1305-mips", + .base.cra_driver_name = "poly1305-mips-old", .base.cra_priority = 200, .base.cra_blocksize = POLY1305_BLOCK_SIZE, .base.cra_module = THIS_MODULE, }; +bool poly1305_is_arch_optimized(void) +{ + return true; +} +EXPORT_SYMBOL(poly1305_is_arch_optimized); + static int __init mips_poly1305_mod_init(void) { return IS_REACHABLE(CONFIG_CRYPTO_HASH) ? @@ -183,7 +189,7 @@ static void __exit mips_poly1305_mod_exit(void) crypto_unregister_shash(&mips_poly1305_alg); } -module_init(mips_poly1305_mod_init); +arch_initcall(mips_poly1305_mod_init); module_exit(mips_poly1305_mod_exit); MODULE_DESCRIPTION("Poly1305 transform (MIPS accelerated"); diff --git a/arch/powerpc/crypto/poly1305-p10-glue.c b/arch/powerpc/crypto/poly1305-p10-glue.c index 87bd5eba4497..40d296d52c23 100644 --- a/arch/powerpc/crypto/poly1305-p10-glue.c +++ b/arch/powerpc/crypto/poly1305-p10-glue.c @@ -113,6 +113,12 @@ void poly1305_final_arch(struct poly1305_desc_ctx *dctx, u8 *dst) } EXPORT_SYMBOL(poly1305_final_arch); +bool poly1305_is_arch_optimized(void) +{ + return static_key_enabled(&have_p10); +} +EXPORT_SYMBOL(poly1305_is_arch_optimized); + static int __init poly1305_p10_init(void) { if (cpu_has_feature(CPU_FTR_ARCH_31)) diff --git a/arch/x86/crypto/poly1305_glue.c b/arch/x86/crypto/poly1305_glue.c index 8b5593c46da7..40d03a9a4fb5 100644 --- a/arch/x86/crypto/poly1305_glue.c +++ b/arch/x86/crypto/poly1305_glue.c @@ -257,6 +257,12 @@ static struct shash_alg alg = { }, }; +bool poly1305_is_arch_optimized(void) +{ + return static_key_enabled(&poly1305_use_avx); +} +EXPORT_SYMBOL(poly1305_is_arch_optimized); + static int __init poly1305_simd_mod_init(void) { if (boot_cpu_has(X86_FEATURE_AVX) && @@ -280,7 +286,7 @@ static void __exit poly1305_simd_mod_exit(void) crypto_unregister_shash(&alg); } -module_init(poly1305_simd_mod_init); +arch_initcall(poly1305_simd_mod_init); module_exit(poly1305_simd_mod_exit); MODULE_LICENSE("GPL"); diff --git a/crypto/Makefile b/crypto/Makefile index baf5040ca661..5d2f2a28d8a0 100644 --- a/crypto/Makefile +++ b/crypto/Makefile @@ -149,7 +149,8 @@ obj-$(CONFIG_CRYPTO_SEED) += seed.o obj-$(CONFIG_CRYPTO_ARIA) += aria_generic.o obj-$(CONFIG_CRYPTO_CHACHA20) += chacha.o CFLAGS_chacha.o += -DARCH=$(ARCH) -obj-$(CONFIG_CRYPTO_POLY1305) += poly1305_generic.o +obj-$(CONFIG_CRYPTO_POLY1305) += poly1305.o +CFLAGS_poly1305.o += -DARCH=$(ARCH) obj-$(CONFIG_CRYPTO_DEFLATE) += deflate.o obj-$(CONFIG_CRYPTO_MICHAEL_MIC) += michael_mic.o obj-$(CONFIG_CRYPTO_CRC32C) += crc32c_generic.o diff --git a/crypto/poly1305.c b/crypto/poly1305.c new file mode 100644 index 000000000000..e0436bdc462b --- /dev/null +++ b/crypto/poly1305.c @@ -0,0 +1,152 @@ +/* + * Crypto API wrapper for the Poly1305 library functions + * + * Copyright (C) 2015 Martin Willi + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + */ + +#include +#include +#include +#include +#include +#include + +struct crypto_poly1305_desc_ctx { + struct poly1305_desc_ctx base; + u8 key[POLY1305_KEY_SIZE]; + unsigned int keysize; +}; + +static int crypto_poly1305_init(struct shash_desc *desc) +{ + struct crypto_poly1305_desc_ctx *dctx = shash_desc_ctx(desc); + + dctx->keysize = 0; + return 0; +} + +static int crypto_poly1305_update(struct shash_desc *desc, + const u8 *src, unsigned int srclen, bool arch) +{ + struct crypto_poly1305_desc_ctx *dctx = shash_desc_ctx(desc); + unsigned int bytes; + + /* + * The key is passed as the first 32 "data" bytes. The actual + * poly1305_init() can be called only once the full key is available. + */ + if (dctx->keysize < POLY1305_KEY_SIZE) { + bytes = min(srclen, POLY1305_KEY_SIZE - dctx->keysize); + memcpy(&dctx->key[dctx->keysize], src, bytes); + dctx->keysize += bytes; + if (dctx->keysize < POLY1305_KEY_SIZE) + return 0; + if (arch) + poly1305_init(&dctx->base, dctx->key); + else + poly1305_init_generic(&dctx->base, dctx->key); + src += bytes; + srclen -= bytes; + } + + if (arch) + poly1305_update(&dctx->base, src, srclen); + else + poly1305_update_generic(&dctx->base, src, srclen); + + return 0; +} + +static int crypto_poly1305_update_generic(struct shash_desc *desc, + const u8 *src, unsigned int srclen) +{ + return crypto_poly1305_update(desc, src, srclen, false); +} + +static int crypto_poly1305_update_arch(struct shash_desc *desc, + const u8 *src, unsigned int srclen) +{ + return crypto_poly1305_update(desc, src, srclen, true); +} + +static int crypto_poly1305_final(struct shash_desc *desc, u8 *dst, bool arch) +{ + struct crypto_poly1305_desc_ctx *dctx = shash_desc_ctx(desc); + + if (unlikely(dctx->keysize != POLY1305_KEY_SIZE)) + return -ENOKEY; + + if (arch) + poly1305_final(&dctx->base, dst); + else + poly1305_final_generic(&dctx->base, dst); + memzero_explicit(&dctx->key, sizeof(dctx->key)); + return 0; +} + +static int crypto_poly1305_final_generic(struct shash_desc *desc, u8 *dst) +{ + return crypto_poly1305_final(desc, dst, false); +} + +static int crypto_poly1305_final_arch(struct shash_desc *desc, u8 *dst) +{ + return crypto_poly1305_final(desc, dst, true); +} + +static struct shash_alg poly1305_algs[] = { + { + .base.cra_name = "poly1305", + .base.cra_driver_name = "poly1305-generic", + .base.cra_priority = 100, + .base.cra_blocksize = POLY1305_BLOCK_SIZE, + .base.cra_module = THIS_MODULE, + .digestsize = POLY1305_DIGEST_SIZE, + .init = crypto_poly1305_init, + .update = crypto_poly1305_update_generic, + .final = crypto_poly1305_final_generic, + .descsize = sizeof(struct crypto_poly1305_desc_ctx), + }, + { + .base.cra_name = "poly1305", + .base.cra_driver_name = "poly1305-" __stringify(ARCH), + .base.cra_priority = 300, + .base.cra_blocksize = POLY1305_BLOCK_SIZE, + .base.cra_module = THIS_MODULE, + .digestsize = POLY1305_DIGEST_SIZE, + .init = crypto_poly1305_init, + .update = crypto_poly1305_update_arch, + .final = crypto_poly1305_final_arch, + .descsize = sizeof(struct crypto_poly1305_desc_ctx), + }, +}; + +static int num_algs; + +static int __init poly1305_mod_init(void) +{ + /* register the arch flavours only if they differ from generic */ + num_algs = poly1305_is_arch_optimized() ? 2 : 1; + + return crypto_register_shashes(poly1305_algs, num_algs); +} + +static void __exit poly1305_mod_exit(void) +{ + crypto_unregister_shashes(poly1305_algs, num_algs); +} + +subsys_initcall(poly1305_mod_init); +module_exit(poly1305_mod_exit); + +MODULE_LICENSE("GPL"); +MODULE_AUTHOR("Martin Willi "); +MODULE_DESCRIPTION("Crypto API wrapper for the Poly1305 library functions"); +MODULE_ALIAS_CRYPTO("poly1305"); +MODULE_ALIAS_CRYPTO("poly1305-generic"); +MODULE_ALIAS_CRYPTO("poly1305-" __stringify(ARCH)); diff --git a/crypto/poly1305_generic.c b/crypto/poly1305_generic.c deleted file mode 100644 index e6f29a98725a..000000000000 --- a/crypto/poly1305_generic.c +++ /dev/null @@ -1,149 +0,0 @@ -/* - * Poly1305 authenticator algorithm, RFC7539 - * - * Copyright (C) 2015 Martin Willi - * - * Based on public domain code by Andrew Moon and Daniel J. Bernstein. - * - * This program is free software; you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation; either version 2 of the License, or - * (at your option) any later version. - */ - -#include -#include -#include -#include -#include -#include -#include - -static int crypto_poly1305_init(struct shash_desc *desc) -{ - struct poly1305_desc_ctx *dctx = shash_desc_ctx(desc); - - poly1305_core_init(&dctx->h); - dctx->buflen = 0; - dctx->rset = 0; - dctx->sset = false; - - return 0; -} - -static unsigned int crypto_poly1305_setdesckey(struct poly1305_desc_ctx *dctx, - const u8 *src, unsigned int srclen) -{ - if (!dctx->sset) { - if (!dctx->rset && srclen >= POLY1305_BLOCK_SIZE) { - poly1305_core_setkey(&dctx->core_r, src); - src += POLY1305_BLOCK_SIZE; - srclen -= POLY1305_BLOCK_SIZE; - dctx->rset = 2; - } - if (srclen >= POLY1305_BLOCK_SIZE) { - dctx->s[0] = get_unaligned_le32(src + 0); - dctx->s[1] = get_unaligned_le32(src + 4); - dctx->s[2] = get_unaligned_le32(src + 8); - dctx->s[3] = get_unaligned_le32(src + 12); - src += POLY1305_BLOCK_SIZE; - srclen -= POLY1305_BLOCK_SIZE; - dctx->sset = true; - } - } - return srclen; -} - -static void poly1305_blocks(struct poly1305_desc_ctx *dctx, const u8 *src, - unsigned int srclen) -{ - unsigned int datalen; - - if (unlikely(!dctx->sset)) { - datalen = crypto_poly1305_setdesckey(dctx, src, srclen); - src += srclen - datalen; - srclen = datalen; - } - - poly1305_core_blocks(&dctx->h, &dctx->core_r, src, - srclen / POLY1305_BLOCK_SIZE, 1); -} - -static int crypto_poly1305_update(struct shash_desc *desc, - const u8 *src, unsigned int srclen) -{ - struct poly1305_desc_ctx *dctx = shash_desc_ctx(desc); - unsigned int bytes; - - if (unlikely(dctx->buflen)) { - bytes = min(srclen, POLY1305_BLOCK_SIZE - dctx->buflen); - memcpy(dctx->buf + dctx->buflen, src, bytes); - src += bytes; - srclen -= bytes; - dctx->buflen += bytes; - - if (dctx->buflen == POLY1305_BLOCK_SIZE) { - poly1305_blocks(dctx, dctx->buf, - POLY1305_BLOCK_SIZE); - dctx->buflen = 0; - } - } - - if (likely(srclen >= POLY1305_BLOCK_SIZE)) { - poly1305_blocks(dctx, src, srclen); - src += srclen - (srclen % POLY1305_BLOCK_SIZE); - srclen %= POLY1305_BLOCK_SIZE; - } - - if (unlikely(srclen)) { - dctx->buflen = srclen; - memcpy(dctx->buf, src, srclen); - } - - return 0; -} - -static int crypto_poly1305_final(struct shash_desc *desc, u8 *dst) -{ - struct poly1305_desc_ctx *dctx = shash_desc_ctx(desc); - - if (unlikely(!dctx->sset)) - return -ENOKEY; - - poly1305_final_generic(dctx, dst); - return 0; -} - -static struct shash_alg poly1305_alg = { - .digestsize = POLY1305_DIGEST_SIZE, - .init = crypto_poly1305_init, - .update = crypto_poly1305_update, - .final = crypto_poly1305_final, - .descsize = sizeof(struct poly1305_desc_ctx), - .base = { - .cra_name = "poly1305", - .cra_driver_name = "poly1305-generic", - .cra_priority = 100, - .cra_blocksize = POLY1305_BLOCK_SIZE, - .cra_module = THIS_MODULE, - }, -}; - -static int __init poly1305_mod_init(void) -{ - return crypto_register_shash(&poly1305_alg); -} - -static void __exit poly1305_mod_exit(void) -{ - crypto_unregister_shash(&poly1305_alg); -} - -subsys_initcall(poly1305_mod_init); -module_exit(poly1305_mod_exit); - -MODULE_LICENSE("GPL"); -MODULE_AUTHOR("Martin Willi "); -MODULE_DESCRIPTION("Poly1305 authenticator"); -MODULE_ALIAS_CRYPTO("poly1305"); -MODULE_ALIAS_CRYPTO("poly1305-generic"); diff --git a/include/crypto/poly1305.h b/include/crypto/poly1305.h index 090692ec3bc7..91444965772a 100644 --- a/include/crypto/poly1305.h +++ b/include/crypto/poly1305.h @@ -96,4 +96,13 @@ static inline void poly1305_final(struct poly1305_desc_ctx *desc, u8 *digest) poly1305_final_generic(desc, digest); } +#if IS_ENABLED(CONFIG_CRYPTO_ARCH_HAVE_LIB_POLY1305) +bool poly1305_is_arch_optimized(void); +#else +static inline bool poly1305_is_arch_optimized(void) +{ + return false; +} +#endif + #endif