diff --git a/crypto/fipsmodule/bn/add.c b/crypto/fipsmodule/bn/add.c index 38a845062..1a8785e52 100644 --- a/crypto/fipsmodule/bn/add.c +++ b/crypto/fipsmodule/bn/add.c @@ -117,10 +117,7 @@ int bn_uadd_consttime(BIGNUM *r, const BIGNUM *a, const BIGNUM *b) { BN_ULONG carry = bn_add_words(r->d, a->d, b->d, min); for (int i = min; i < max; i++) { - // |r| and |a| may alias, so use a temporary. - BN_ULONG tmp = carry + a->d[i]; - carry = tmp < a->d[i]; - r->d[i] = tmp; + r->d[i] = CRYPTO_addc_w(a->d[i], 0, carry, &carry); } r->d[max] = carry; @@ -241,10 +238,7 @@ int bn_usub_consttime(BIGNUM *r, const BIGNUM *a, const BIGNUM *b) { BN_ULONG borrow = bn_sub_words(r->d, a->d, b->d, b_width); for (int i = b_width; i < a->width; i++) { - // |r| and |a| may alias, so use a temporary. - BN_ULONG tmp = a->d[i]; - r->d[i] = a->d[i] - borrow; - borrow = tmp < r->d[i]; + r->d[i] = CRYPTO_subc_w(a->d[i], 0, borrow, &borrow); } if (borrow) { diff --git a/crypto/fipsmodule/bn/generic.c b/crypto/fipsmodule/bn/generic.c index e4f4518e9..247398fdd 100644 --- a/crypto/fipsmodule/bn/generic.c +++ b/crypto/fipsmodule/bn/generic.c @@ -567,37 +567,6 @@ void bn_sqr_comba4(BN_ULONG r[8], const BN_ULONG a[4]) { #if !defined(BN_ADD_ASM) -// bn_add_with_carry returns |x + y + carry|, and sets |*out_carry| to the -// carry bit. |carry| must be zero or one. -static inline BN_ULONG bn_add_with_carry(BN_ULONG x, BN_ULONG y, BN_ULONG carry, - BN_ULONG *out_carry) { - assert(carry == 0 || carry == 1); -#if defined(BN_ULLONG) - BN_ULLONG ret = carry; - ret += (BN_ULLONG)x + y; - *out_carry = (BN_ULONG)(ret >> BN_BITS2); - return (BN_ULONG)ret; -#else - x += carry; - carry = x < carry; - BN_ULONG ret = x + y; - carry += ret < x; - *out_carry = carry; - return ret; -#endif -} - -// bn_sub_with_borrow returns |x - y - borrow|, and sets |*out_borrow| to the -// borrow bit. |borrow| must be zero or one. -static inline BN_ULONG bn_sub_with_borrow(BN_ULONG x, BN_ULONG y, - BN_ULONG borrow, - BN_ULONG *out_borrow) { - assert(borrow == 0 || borrow == 1); - BN_ULONG ret = x - y - borrow; - *out_borrow = (x < y) | ((x == y) & borrow); - return ret; -} - BN_ULONG bn_add_words(BN_ULONG *r, const BN_ULONG *a, const BN_ULONG *b, size_t n) { if (n == 0) { @@ -606,17 +575,17 @@ BN_ULONG bn_add_words(BN_ULONG *r, const BN_ULONG *a, const BN_ULONG *b, BN_ULONG carry = 0; while (n & ~3) { - r[0] = bn_add_with_carry(a[0], b[0], carry, &carry); - r[1] = bn_add_with_carry(a[1], b[1], carry, &carry); - r[2] = bn_add_with_carry(a[2], b[2], carry, &carry); - r[3] = bn_add_with_carry(a[3], b[3], carry, &carry); + r[0] = CRYPTO_addc_w(a[0], b[0], carry, &carry); + r[1] = CRYPTO_addc_w(a[1], b[1], carry, &carry); + r[2] = CRYPTO_addc_w(a[2], b[2], carry, &carry); + r[3] = CRYPTO_addc_w(a[3], b[3], carry, &carry); a += 4; b += 4; r += 4; n -= 4; } while (n) { - r[0] = bn_add_with_carry(a[0], b[0], carry, &carry); + r[0] = CRYPTO_addc_w(a[0], b[0], carry, &carry); a++; b++; r++; @@ -633,17 +602,17 @@ BN_ULONG bn_sub_words(BN_ULONG *r, const BN_ULONG *a, const BN_ULONG *b, BN_ULONG borrow = 0; while (n & ~3) { - r[0] = bn_sub_with_borrow(a[0], b[0], borrow, &borrow); - r[1] = bn_sub_with_borrow(a[1], b[1], borrow, &borrow); - r[2] = bn_sub_with_borrow(a[2], b[2], borrow, &borrow); - r[3] = bn_sub_with_borrow(a[3], b[3], borrow, &borrow); + r[0] = CRYPTO_subc_w(a[0], b[0], borrow, &borrow); + r[1] = CRYPTO_subc_w(a[1], b[1], borrow, &borrow); + r[2] = CRYPTO_subc_w(a[2], b[2], borrow, &borrow); + r[3] = CRYPTO_subc_w(a[3], b[3], borrow, &borrow); a += 4; b += 4; r += 4; n -= 4; } while (n) { - r[0] = bn_sub_with_borrow(a[0], b[0], borrow, &borrow); + r[0] = CRYPTO_subc_w(a[0], b[0], borrow, &borrow); a++; b++; r++; diff --git a/crypto/fipsmodule/bn/mul.c b/crypto/fipsmodule/bn/mul.c index fe4e4d7aa..7537899c0 100644 --- a/crypto/fipsmodule/bn/mul.c +++ b/crypto/fipsmodule/bn/mul.c @@ -143,17 +143,13 @@ static BN_ULONG bn_sub_part_words(BN_ULONG *r, const BN_ULONG *a, // in |a| were zeros. dl = -dl; for (int i = 0; i < dl; i++) { - r[i] = 0u - b[i] - borrow; - borrow |= r[i] != 0; + r[i] = CRYPTO_subc_w(0, b[i], borrow, &borrow); } } else { // |b| is shorter than |a|. Complete the subtraction as if the excess words // in |b| were zeros. for (int i = 0; i < dl; i++) { - // |r| and |a| may alias, so use a temporary. - BN_ULONG tmp = a[i]; - r[i] = a[i] - borrow; - borrow = tmp < r[i]; + r[i] = CRYPTO_subc_w(a[i], 0, borrow, &borrow); } } diff --git a/crypto/fipsmodule/sha/sha1.c b/crypto/fipsmodule/sha/sha1.c index 35dda2fa2..4baeed669 100644 --- a/crypto/fipsmodule/sha/sha1.c +++ b/crypto/fipsmodule/sha/sha1.c @@ -134,12 +134,11 @@ void CRYPTO_fips_186_2_prf(uint8_t *out, size_t out_len, SHA1_Transform(&ctx, block); // XKEY = (1 + XKEY + w_i) mod 2^b - uint64_t carry = 1; + uint32_t carry = 1; for (int i = 4; i >= 0; i--) { - carry += CRYPTO_load_u32_be(block + i * 4); - carry += ctx.h[i]; - CRYPTO_store_u32_be(block + i * 4, (uint32_t)carry); - carry >>= 32; + uint32_t tmp = CRYPTO_load_u32_be(block + i * 4); + tmp = CRYPTO_addc_u32(tmp, ctx.h[i], carry, &carry); + CRYPTO_store_u32_be(block + i * 4, tmp); } // Output w_i. diff --git a/crypto/internal.h b/crypto/internal.h index 4de4597de..f2db41c41 100644 --- a/crypto/internal.h +++ b/crypto/internal.h @@ -259,6 +259,12 @@ OPENSSL_EXPORT void OPENSSL_reset_malloc_counter_for_testing(void); OPENSSL_INLINE void OPENSSL_reset_malloc_counter_for_testing(void) {} #endif +#if defined(__has_builtin) +#define OPENSSL_HAS_BUILTIN(x) __has_builtin(x) +#else +#define OPENSSL_HAS_BUILTIN(x) 0 +#endif + // Pointer utility functions. @@ -1134,6 +1140,110 @@ static inline uint64_t CRYPTO_rotr_u64(uint64_t value, int shift) { } +// Arithmetic functions. + +// CRYPTO_addc_* returns |x + y + carry|, and sets |*out_carry| to the carry +// bit. |carry| must be zero or one. +#if OPENSSL_HAS_BUILTIN(__builtin_addc) + +#define CRYPTO_GENERIC_ADDC(x, y, carry, out_carry) \ + (_Generic((x), \ + unsigned: __builtin_addc, \ + unsigned long: __builtin_addcl, \ + unsigned long long: __builtin_addcll))((x), (y), (carry), (out_carry)) + +static inline uint32_t CRYPTO_addc_u32(uint32_t x, uint32_t y, uint32_t carry, + uint32_t *out_carry) { + assert(carry <= 1); + return CRYPTO_GENERIC_ADDC(x, y, carry, out_carry); +} + +static inline uint64_t CRYPTO_addc_u64(uint64_t x, uint64_t y, uint64_t carry, + uint64_t *out_carry) { + assert(carry <= 1); + return CRYPTO_GENERIC_ADDC(x, y, carry, out_carry); +} + +#else + +static inline uint32_t CRYPTO_addc_u32(uint32_t x, uint32_t y, uint32_t carry, + uint32_t *out_carry) { + assert(carry <= 1); + uint64_t ret = carry; + ret += (uint64_t)x + y; + *out_carry = (uint32_t)(ret >> 32); + return (uint32_t)ret; +} + +static inline uint64_t CRYPTO_addc_u64(uint64_t x, uint64_t y, uint64_t carry, + uint64_t *out_carry) { + assert(carry <= 1); +#if defined(BORINGSSL_HAS_UINT128) + uint128_t ret = carry; + ret += (uint128_t)x + y; + *out_carry = (uint64_t)(ret >> 64); + return (uint64_t)ret; +#else + x += carry; + carry = x < carry; + uint64_t ret = x + y; + carry += ret < x; + *out_carry = carry; + return ret; +#endif +} +#endif + +// CRYPTO_subc_* returns |x - y - borrow|, and sets |*out_borrow| to the borrow +// bit. |borrow| must be zero or one. +#if OPENSSL_HAS_BUILTIN(__builtin_subc) + +#define CRYPTO_GENERIC_SUBC(x, y, borrow, out_borrow) \ + (_Generic((x), \ + unsigned: __builtin_subc, \ + unsigned long: __builtin_subcl, \ + unsigned long long: __builtin_subcll))((x), (y), (borrow), (out_borrow)) + +static inline uint32_t CRYPTO_subc_u32(uint32_t x, uint32_t y, uint32_t borrow, + uint32_t *out_borrow) { + assert(borrow <= 1); + return CRYPTO_GENERIC_SUBC(x, y, borrow, out_borrow); +} + +static inline uint64_t CRYPTO_subc_u64(uint64_t x, uint64_t y, uint64_t borrow, + uint64_t *out_borrow) { + assert(borrow <= 1); + return CRYPTO_GENERIC_SUBC(x, y, borrow, out_borrow); +} + +#else + +static inline uint32_t CRYPTO_subc_u32(uint32_t x, uint32_t y, uint32_t borrow, + uint32_t *out_borrow) { + assert(borrow <= 1); + uint32_t ret = x - y - borrow; + *out_borrow = (x < y) | ((x == y) & borrow); + return ret; +} + +static inline uint64_t CRYPTO_subc_u64(uint64_t x, uint64_t y, uint64_t borrow, + uint64_t *out_borrow) { + assert(borrow <= 1); + uint64_t ret = x - y - borrow; + *out_borrow = (x < y) | ((x == y) & borrow); + return ret; +} +#endif + +#if defined(OPENSSL_64_BIT) +#define CRYPTO_addc_w CRYPTO_addc_u64 +#define CRYPTO_subc_w CRYPTO_subc_u64 +#else +#define CRYPTO_addc_w CRYPTO_addc_u32 +#define CRYPTO_subc_w CRYPTO_subc_u32 +#endif + + // FIPS functions. #if defined(BORINGSSL_FIPS)