diff --git a/BUILD.generated.bzl b/BUILD.generated.bzl index 259d4f53f..1cf34cd92 100644 --- a/BUILD.generated.bzl +++ b/BUILD.generated.bzl @@ -267,6 +267,7 @@ crypto_internal_headers = [ "src/crypto/x509v3/internal.h", "src/third_party/fiat/curve25519_32.h", "src/third_party/fiat/curve25519_64.h", + "src/third_party/fiat/curve25519_64_adx.h", "src/third_party/fiat/curve25519_64_msvc.h", "src/third_party/fiat/p256_32.h", "src/third_party/fiat/p256_64.h", @@ -348,6 +349,7 @@ crypto_sources = [ "src/crypto/cpu_intel.c", "src/crypto/crypto.c", "src/crypto/curve25519/curve25519.c", + "src/crypto/curve25519/curve25519_64_adx.c", "src/crypto/curve25519/spake25519.c", "src/crypto/des/des.c", "src/crypto/dh_extra/dh_asn1.c", diff --git a/BUILD.generated_tests.bzl b/BUILD.generated_tests.bzl index 92dec1e01..af0ec106e 100644 --- a/BUILD.generated_tests.bzl +++ b/BUILD.generated_tests.bzl @@ -72,6 +72,7 @@ test_support_sources = [ "src/ssl/test/test_state.h", "src/third_party/fiat/curve25519_32.h", "src/third_party/fiat/curve25519_64.h", + "src/third_party/fiat/curve25519_64_adx.h", "src/third_party/fiat/curve25519_64_msvc.h", "src/third_party/fiat/p256_32.h", "src/third_party/fiat/p256_64.h", diff --git a/CMakeLists.txt b/CMakeLists.txt index 402b9b7d9..5b833f824 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -341,6 +341,7 @@ add_library( src/crypto/cpu_intel.c src/crypto/crypto.c src/crypto/curve25519/curve25519.c + src/crypto/curve25519/curve25519_64_adx.c src/crypto/curve25519/spake25519.c src/crypto/des/des.c src/crypto/dh_extra/dh_asn1.c diff --git a/sources.json b/sources.json index 510e3070c..7f20c10cd 100644 --- a/sources.json +++ b/sources.json @@ -77,6 +77,7 @@ "src/crypto/cpu_intel.c", "src/crypto/crypto.c", "src/crypto/curve25519/curve25519.c", + "src/crypto/curve25519/curve25519_64_adx.c", "src/crypto/curve25519/spake25519.c", "src/crypto/des/des.c", "src/crypto/dh_extra/dh_asn1.c", @@ -430,6 +431,7 @@ "src/crypto/x509v3/internal.h", "src/third_party/fiat/curve25519_32.h", "src/third_party/fiat/curve25519_64.h", + "src/third_party/fiat/curve25519_64_adx.h", "src/third_party/fiat/curve25519_64_msvc.h", "src/third_party/fiat/p256_32.h", "src/third_party/fiat/p256_64.h", diff --git a/src/crypto/CMakeLists.txt b/src/crypto/CMakeLists.txt index ef47623cc..5f0c160aa 100644 --- a/src/crypto/CMakeLists.txt +++ b/src/crypto/CMakeLists.txt @@ -15,6 +15,7 @@ endif() set( CRYPTO_SOURCES_ASM curve25519/asm/x25519-asm-arm.S + curve25519/asm/fiat_curve25519_adx.S hrss/asm/poly_rq_mul.S poly1305/poly1305_arm_asm.S ) @@ -137,6 +138,7 @@ add_library( cpu_intel.c crypto.c curve25519/curve25519.c + curve25519/curve25519_64_adx.c curve25519/spake25519.c des/des.c dh_extra/params.c diff --git a/src/crypto/curve25519/asm/fiat_curve25519_adx.S b/src/crypto/curve25519/asm/fiat_curve25519_adx.S new file mode 100644 index 000000000..0c4218117 --- /dev/null +++ b/src/crypto/curve25519/asm/fiat_curve25519_adx.S @@ -0,0 +1,11 @@ +#if !defined(OPENSSL_NO_ASM) && !defined(OPENSSL_SMALL) && defined(__x86_64__) + +#if defined(BORINGSSL_PREFIX) +#include +#endif + +.intel_syntax noprefix +#include "../../../third_party/fiat/asm/fiat_curve25519_adx_mul.S" +#include "../../../third_party/fiat/asm/fiat_curve25519_adx_square.S" + +#endif diff --git a/src/crypto/curve25519/curve25519.c b/src/crypto/curve25519/curve25519.c index d4a3f21d6..ec3067b24 100644 --- a/src/crypto/curve25519/curve25519.c +++ b/src/crypto/curve25519/curve25519.c @@ -19,8 +19,6 @@ // // The field functions are shared by Ed25519 and X25519 where possible. -#include - #include #include @@ -31,7 +29,6 @@ #include "internal.h" #include "../internal.h" - // Various pre-computed constants. #include "./curve25519_tables.h" @@ -2069,6 +2066,12 @@ static void x25519_scalar_mult(uint8_t out[32], const uint8_t scalar[32], x25519_NEON(out, scalar, point); return; } +#elif defined(BORINGSSL_FE25519_ADX) + if (CRYPTO_is_BMI1_capable() && CRYPTO_is_BMI2_capable() && + CRYPTO_is_ADX_capable()) { + x25519_scalar_mult_adx(out, scalar, point); + return; + } #endif x25519_scalar_mult_generic(out, scalar, point); diff --git a/src/crypto/curve25519/curve25519_64_adx.c b/src/crypto/curve25519/curve25519_64_adx.c new file mode 100644 index 000000000..276898964 --- /dev/null +++ b/src/crypto/curve25519/curve25519_64_adx.c @@ -0,0 +1,18 @@ +/* Copyright (c) 2023, Google Inc. + * + * Permission to use, copy, modify, and/or distribute this software for any + * purpose with or without fee is hereby granted, provided that the above + * copyright notice and this permission notice appear in all copies. + * + * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES + * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF + * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY + * SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES + * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION + * OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN + * CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. */ + +#include "internal.h" +#if defined(BORINGSSL_FE25519_ADX) +#include "../../third_party/fiat/curve25519_64_adx.h" +#endif diff --git a/src/crypto/curve25519/internal.h b/src/crypto/curve25519/internal.h index 1420601bb..4de134419 100644 --- a/src/crypto/curve25519/internal.h +++ b/src/crypto/curve25519/internal.h @@ -15,14 +15,13 @@ #ifndef OPENSSL_HEADER_CURVE25519_INTERNAL_H #define OPENSSL_HEADER_CURVE25519_INTERNAL_H -#if defined(__cplusplus) -extern "C" { -#endif - -#include +#include #include "../internal.h" +#if defined(__cplusplus) +extern "C" { +#endif #if defined(OPENSSL_ARM) && !defined(OPENSSL_NO_ASM) && !defined(OPENSSL_APPLE) #define BORINGSSL_X25519_NEON @@ -32,6 +31,26 @@ void x25519_NEON(uint8_t out[32], const uint8_t scalar[32], const uint8_t point[32]); #endif +#if !defined(OPENSSL_NO_ASM) && !defined(OPENSSL_SMALL) && \ + defined(__GNUC__) && defined(__x86_64__) +#define BORINGSSL_FE25519_ADX + +// fiat_curve25519_adx_mul is defined in +// third_party/fiat/asm/fiat_curve25519_adx_mul.S +void __attribute__((sysv_abi)) +fiat_curve25519_adx_mul(uint64_t out[4], const uint64_t in1[4], + const uint64_t in2[4]); + +// fiat_curve25519_adx_square is defined in +// third_party/fiat/asm/fiat_curve25519_adx_square.S +void __attribute__((sysv_abi)) +fiat_curve25519_adx_square(uint64_t out[4], const uint64_t in[4]); + +// x25519_scalar_mult_adx is defined in third_party/fiat/curve25519_64_adx.h +void x25519_scalar_mult_adx(uint8_t out[32], const uint8_t scalar[32], + const uint8_t point[32]); +#endif + #if defined(OPENSSL_64_BIT) // fe means field element. Here the field is \Z/(2^255-19). An element t, // entries t[0]...t[4], represents the integer t[0]+2^51 t[1]+2^102 t[2]+2^153 diff --git a/src/crypto/curve25519/x25519_test.cc b/src/crypto/curve25519/x25519_test.cc index f512d01ab..8c08ee22c 100644 --- a/src/crypto/curve25519/x25519_test.cc +++ b/src/crypto/curve25519/x25519_test.cc @@ -20,6 +20,7 @@ #include +#include "internal.h" #include "../internal.h" #include "../test/abi_test.h" #include "../test/file_test.h" @@ -231,3 +232,27 @@ TEST(X25519Test, NeonABI) { CHECK_ABI(x25519_NEON, secret, kScalar, kPoint); } #endif // BORINGSSL_X25519_NEON && SUPPORTS_ABI_TEST + +#if defined(BORINGSSL_FE25519_ADX) && defined(SUPPORTS_ABI_TEST) +TEST(X25519Test, AdxMulABI) { + static const uint64_t in1[4] = {0}, in2[4] = {0}; + uint64_t out[4]; + if (CRYPTO_is_BMI1_capable() && CRYPTO_is_BMI2_capable() && + CRYPTO_is_ADX_capable()) { + CHECK_ABI(fiat_curve25519_adx_mul, out, in1, in2); + } else { + GTEST_SKIP() << "Can't test ABI of ADX code without ADX"; + } +} + +TEST(X25519Test, AdxSquareABI) { + static const uint64_t in[4] = {0}; + uint64_t out[4]; + if (CRYPTO_is_BMI1_capable() && CRYPTO_is_BMI2_capable() && + CRYPTO_is_ADX_capable()) { + CHECK_ABI(fiat_curve25519_adx_square, out, in); + } else { + GTEST_SKIP() << "Can't test ABI of ADX code without ADX"; + } +} +#endif // BORINGSSL_FE25519_ADX && SUPPORTS_ABI_TEST diff --git a/src/crypto/dsa/dsa_test.cc b/src/crypto/dsa/dsa_test.cc index 611b00347..22e9e13bb 100644 --- a/src/crypto/dsa/dsa_test.cc +++ b/src/crypto/dsa/dsa_test.cc @@ -257,7 +257,10 @@ TEST(DSATest, Verify) { TEST(DSATest, InvalidGroup) { bssl::UniquePtr dsa = GetFIPSDSA(); ASSERT_TRUE(dsa); - BN_zero(dsa->g); + bssl::UniquePtr zero(BN_new()); + ASSERT_TRUE(zero); + ASSERT_TRUE(DSA_set0_pqg(dsa.get(), /*p=*/nullptr, /*q=*/nullptr, + /*g=*/zero.release())); std::vector sig(DSA_size(dsa.get())); unsigned sig_len; @@ -305,7 +308,10 @@ TEST(DSATest, MissingPrivate) { TEST(DSATest, ZeroPrivateKey) { bssl::UniquePtr dsa = GetFIPSDSA(); ASSERT_TRUE(dsa); - BN_zero(dsa->priv_key); + bssl::UniquePtr zero(BN_new()); + ASSERT_TRUE(zero); + ASSERT_TRUE(DSA_set0_key(dsa.get(), /*pub_key=*/nullptr, + /*priv_key=*/zero.release())); static const uint8_t kZeroDigest[32] = {0}; std::vector sig(DSA_size(dsa.get())); diff --git a/src/crypto/dsa/internal.h b/src/crypto/dsa/internal.h index 560ae7f33..aae32637c 100644 --- a/src/crypto/dsa/internal.h +++ b/src/crypto/dsa/internal.h @@ -17,11 +17,29 @@ #include +#include + #if defined(__cplusplus) extern "C" { #endif +struct dsa_st { + BIGNUM *p; + BIGNUM *q; + BIGNUM *g; + + BIGNUM *pub_key; + BIGNUM *priv_key; + + // Normally used to cache montgomery values + CRYPTO_MUTEX method_mont_lock; + BN_MONT_CTX *method_mont_p; + BN_MONT_CTX *method_mont_q; + CRYPTO_refcount_t references; + CRYPTO_EX_DATA ex_data; +}; + // dsa_check_key performs cheap self-checks on |dsa|, and ensures it is within // DoS bounds. It returns one on success and zero on error. int dsa_check_key(const DSA *dsa); diff --git a/src/crypto/evp/print.c b/src/crypto/evp/print.c index 925074e5a..05383b72b 100644 --- a/src/crypto/evp/print.c +++ b/src/crypto/evp/print.c @@ -194,12 +194,12 @@ static int rsa_priv_print(BIO *bp, const EVP_PKEY *pkey, int indent) { static int do_dsa_print(BIO *bp, const DSA *x, int off, int ptype) { const BIGNUM *priv_key = NULL; if (ptype == 2) { - priv_key = x->priv_key; + priv_key = DSA_get0_priv_key(x); } const BIGNUM *pub_key = NULL; if (ptype > 0) { - pub_key = x->pub_key; + pub_key = DSA_get0_pub_key(x); } const char *ktype = "DSA-Parameters"; @@ -210,14 +210,15 @@ static int do_dsa_print(BIO *bp, const DSA *x, int off, int ptype) { } if (!BIO_indent(bp, off, 128) || - BIO_printf(bp, "%s: (%u bit)\n", ktype, BN_num_bits(x->p)) <= 0 || + BIO_printf(bp, "%s: (%u bit)\n", ktype, BN_num_bits(DSA_get0_p(x))) <= + 0 || // |priv_key| and |pub_key| may be NULL, in which case |bn_print| will // silently skip them. !bn_print(bp, "priv:", priv_key, off) || !bn_print(bp, "pub:", pub_key, off) || - !bn_print(bp, "P:", x->p, off) || - !bn_print(bp, "Q:", x->q, off) || - !bn_print(bp, "G:", x->g, off)) { + !bn_print(bp, "P:", DSA_get0_p(x), off) || + !bn_print(bp, "Q:", DSA_get0_q(x), off) || + !bn_print(bp, "G:", DSA_get0_g(x), off)) { return 0; } diff --git a/src/include/openssl/dsa.h b/src/include/openssl/dsa.h index 30afd4383..4075001af 100644 --- a/src/include/openssl/dsa.h +++ b/src/include/openssl/dsa.h @@ -62,9 +62,7 @@ #include -#include #include -#include #if defined(__cplusplus) extern "C" { @@ -398,25 +396,6 @@ OPENSSL_EXPORT DSA *DSA_generate_parameters(int bits, unsigned char *seed, void *cb_arg); -struct dsa_st { - long version; - BIGNUM *p; - BIGNUM *q; // == 20 - BIGNUM *g; - - BIGNUM *pub_key; // y public key - BIGNUM *priv_key; // x private key - - int flags; - // Normally used to cache montgomery values - CRYPTO_MUTEX method_mont_lock; - BN_MONT_CTX *method_mont_p; - BN_MONT_CTX *method_mont_q; - CRYPTO_refcount_t references; - CRYPTO_EX_DATA ex_data; -}; - - #if defined(__cplusplus) } // extern C diff --git a/src/third_party/fiat/asm/fiat_curve25519_adx_mul.S b/src/third_party/fiat/asm/fiat_curve25519_adx_mul.S new file mode 100644 index 000000000..acbac97b8 --- /dev/null +++ b/src/third_party/fiat/asm/fiat_curve25519_adx_mul.S @@ -0,0 +1,155 @@ +.text +#if defined(__APPLE__) +.global _fiat_curve25519_adx_mul +_fiat_curve25519_adx_mul: +#else +.global fiat_curve25519_adx_mul +fiat_curve25519_adx_mul: +#endif + +.cfi_startproc +mov [rsp - 0x08], rbp +.cfi_offset rbp, -8-0x08 +mov rbp, rsp + +mov rax, rdx +mov rdx, [ rsi + 0x18 ] +mulx r11, r10, [ rax + 0x8 ] +mov rdx, [ rax + 0x0 ] +mov [ rsp - 0x58 ], r15 +.cfi_offset r15, -8-0x58 +mulx r8, rcx, [ rsi + 0x18 ] +mov rdx, [ rsi + 0x8 ] +mov [ rsp - 0x80 ], rbx +.cfi_offset rbx, -8-0x80 +mulx rbx, r9, [ rax + 0x18 ] +mov rdx, [ rsi + 0x8 ] +mov [ rsp - 0x70 ], r12 +.cfi_offset r12, -8-0x70 +mulx r15, r12, [ rax + 0x8 ] +mov rdx, [ rsi + 0x0 ] +mov [ rsp - 0x68 ], r13 +.cfi_offset r13, -8-0x68 +mov [ rsp - 0x60 ], r14 +.cfi_offset r14, -8-0x60 +mulx r14, r13, [ rax + 0x0 ] +mov rdx, [ rax + 0x10 ] +mov [ rsp - 0x18 ], r15 +mov [ rsp - 0x50 ], rdi +mulx rdi, r15, [ rsi + 0x0 ] +mov rdx, [ rax + 0x18 ] +mov [ rsp - 0x48 ], r13 +mov [ rsp - 0x40 ], r9 +mulx r9, r13, [ rsi + 0x0 ] +test al, al +adox rcx, rdi +mov rdx, [ rsi + 0x10 ] +mov [ rsp - 0x38 ], r13 +mulx r13, rdi, [ rax + 0x8 ] +adox r10, r9 +mov rdx, 0x0 +adox rbx, rdx +adcx rdi, rcx +adcx r8, r10 +mov r9, rdx +adcx r9, rbx +mov rdx, [ rsi + 0x10 ] +mulx r10, rcx, [ rax + 0x0 ] +mov rdx, [ rsi + 0x0 ] +mov [ rsp - 0x30 ], r15 +mulx r15, rbx, [ rax + 0x8 ] +mov rdx, -0x2 +inc rdx +adox rcx, r15 +setc r15b +clc +adcx rcx, r12 +adox r10, rdi +mov rdx, [ rax + 0x10 ] +mov [ rsp - 0x78 ], rcx +mulx rcx, rdi, [ rsi + 0x10 ] +adox rdi, r8 +mov rdx, [ rax + 0x18 ] +mov [ rsp - 0x28 ], rcx +mulx rcx, r8, [ rsi + 0x10 ] +mov rdx, [ rax + 0x10 ] +mov [ rsp - 0x20 ], r8 +mulx r12, r8, [ rsi + 0x18 ] +adox r8, r9 +mov rdx, [ rsi + 0x8 ] +mov [ rsp - 0x10 ], r12 +mulx r12, r9, [ rax + 0x10 ] +movzx rdx, r15b +lea rdx, [ rdx + rcx ] +adcx r9, r10 +adcx r13, rdi +mov r15, 0x0 +mov r10, r15 +adox r10, rdx +mov rdx, [ rax + 0x18 ] +mulx rcx, rdi, [ rsi + 0x18 ] +adox rcx, r15 +adcx r11, r8 +mov rdx, r15 +adcx rdx, r10 +adcx rcx, r15 +mov r8, rdx +mov rdx, [ rax + 0x0 ] +mulx r15, r10, [ rsi + 0x8 ] +test al, al +adox r10, r14 +adcx rbx, r10 +adox r15, [ rsp - 0x78 ] +adcx r15, [ rsp - 0x30 ] +adox r9, [ rsp - 0x18 ] +adcx r9, [ rsp - 0x38 ] +adox r13, [ rsp - 0x40 ] +adcx r12, r13 +adox r11, [ rsp - 0x20 ] +adcx r11, [ rsp - 0x28 ] +mov rdx, 0x26 +mulx rsi, r14, r12 +adox rdi, r8 +adcx rdi, [ rsp - 0x10 ] +mulx r10, r8, r11 +mov r13, 0x0 +adox rcx, r13 +adcx rcx, r13 +mulx r11, r12, rdi +xor rdi, rdi +adox r8, rbx +adox r12, r15 +mulx rbx, r13, rcx +adcx r14, [ rsp - 0x48 ] +adox r13, r9 +adox rbx, rdi +adcx rsi, r8 +adcx r10, r12 +adcx r11, r13 +adc rbx, 0x0 +mulx r9, r15, rbx +xor r9, r9 +adox r15, r14 +mov rdi, r9 +adox rdi, rsi +mov rcx, r9 +adox rcx, r10 +mov r8, [ rsp - 0x50 ] +mov [ r8 + 0x8 ], rdi +mov r12, r9 +adox r12, r11 +mov r14, r9 +cmovo r14, rdx +mov [ r8 + 0x18 ], r12 +adcx r15, r14 +mov [ r8 + 0x0 ], r15 +mov [ r8 + 0x10 ], rcx +mov rbx, [ rsp - 0x80 ] +mov r12, [ rsp - 0x70 ] +mov r13, [ rsp - 0x68 ] +mov r14, [ rsp - 0x60 ] +mov r15, [ rsp - 0x58 ] + +mov rbp, [rsp - 0x08] +ret +.cfi_endproc diff --git a/src/third_party/fiat/asm/fiat_curve25519_adx_square.S b/src/third_party/fiat/asm/fiat_curve25519_adx_square.S new file mode 100644 index 000000000..2ba3fd413 --- /dev/null +++ b/src/third_party/fiat/asm/fiat_curve25519_adx_square.S @@ -0,0 +1,123 @@ +.text +#if defined(__APPLE__) +.global _fiat_curve25519_adx_square +_fiat_curve25519_adx_square: +#else +.global fiat_curve25519_adx_square +fiat_curve25519_adx_square: +#endif + +.cfi_startproc +mov [rsp - 0x08], rbp +.cfi_offset rbp, -8-0x08 +mov rbp, rsp + +mov rdx, [ rsi + 0x0 ] +mulx r10, rax, [ rsi + 0x8 ] +mov rdx, [ rsi + 0x0 ] +mulx rcx, r11, [ rsi + 0x10 ] +xor rdx, rdx +adox r11, r10 +mov rdx, [ rsi + 0x0 ] +mulx r9, r8, [ rsi + 0x18 ] +mov rdx, [ rsi + 0x8 ] +mov [ rsp - 0x80 ], rbx +.cfi_offset rbx, -8-0x80 +mulx rbx, r10, [ rsi + 0x18 ] +adox r8, rcx +mov [rsp - 0x48 ], rdi +adox r10, r9 +adcx rax, rax +mov rdx, [ rsi + 0x10 ] +mulx r9, rcx, [ rsi + 0x18 ] +adox rcx, rbx +mov rdx, [ rsi + 0x10 ] +mulx rdi, rbx, [ rsi + 0x8 ] +mov rdx, 0x0 +adox r9, rdx +mov [ rsp - 0x70 ], r12 +.cfi_offset r12, -8-0x70 +mov r12, -0x3 +inc r12 +adox rbx, r8 +adox rdi, r10 +adcx r11, r11 +mov r8, rdx +adox r8, rcx +mov r10, rdx +adox r10, r9 +adcx rbx, rbx +mov rdx, [ rsi + 0x0 ] +mulx r9, rcx, rdx +mov rdx, [ rsi + 0x8 ] +mov [ rsp - 0x68 ], r13 +.cfi_offset r13, -8-0x68 +mov [ rsp - 0x60 ], r14 +.cfi_offset r14, -8-0x60 +mulx r14, r13, rdx +seto dl +inc r12 +adox r9, rax +adox r13, r11 +adox r14, rbx +adcx rdi, rdi +mov al, dl +mov rdx, [ rsi + 0x10 ] +mulx rbx, r11, rdx +adox r11, rdi +adcx r8, r8 +adox rbx, r8 +adcx r10, r10 +movzx rdx, al +mov rdi, 0x0 +adcx rdx, rdi +movzx r8, al +lea r8, [ r8 + rdx ] +mov rdx, [ rsi + 0x18 ] +mulx rdi, rax, rdx +adox rax, r10 +mov rdx, 0x26 +mov [ rsp - 0x58 ], r15 +.cfi_offset r15, -8-0x58 +mulx r15, r10, r11 +clc +adcx r10, rcx +mulx r11, rcx, rbx +adox r8, rdi +mulx rdi, rbx, r8 +inc r12 +adox rcx, r9 +mulx r8, r9, rax +adcx r15, rcx +adox r9, r13 +adcx r11, r9 +adox rbx, r14 +adox rdi, r12 +adcx r8, rbx +adc rdi, 0x0 +mulx r14, r13, rdi +test al, al +mov rdi, [ rsp - 0x48 ] +adox r13, r10 +mov r14, r12 +adox r14, r15 +mov [ rdi + 0x8 ], r14 +mov rax, r12 +adox rax, r11 +mov r10, r12 +adox r10, r8 +mov [ rdi + 0x10 ], rax +mov rcx, r12 +cmovo rcx, rdx +adcx r13, rcx +mov [ rdi + 0x0 ], r13 +mov [ rdi + 0x18 ], r10 +mov rbx, [ rsp - 0x80 ] +mov r12, [ rsp - 0x70 ] +mov r13, [ rsp - 0x68 ] +mov r14, [ rsp - 0x60 ] +mov r15, [ rsp - 0x58 ] + +mov rbp, [rsp - 0x08] +ret +.cfi_endproc diff --git a/src/third_party/fiat/curve25519_64_adx.h b/src/third_party/fiat/curve25519_64_adx.h new file mode 100644 index 000000000..af9a9f8c0 --- /dev/null +++ b/src/third_party/fiat/curve25519_64_adx.h @@ -0,0 +1,528 @@ +#include +#include +#include + +typedef uint64_t fe4[4]; +typedef uint8_t fiat_uint1; +typedef int8_t fiat_int1; + +static __inline__ uint64_t fiat_value_barrier_u64(uint64_t a) { + __asm__("" : "+r"(a) : /* no inputs */); + return a; +} +static inline void fe4_mul(fe4 out, const fe4 x, const fe4 y) { fiat_curve25519_adx_mul(out, x, y); } +static inline void fe4_sq(fe4 out, const fe4 x) { fiat_curve25519_adx_square(out, x); } + +/* + * The function fiat_mulx_u64 is a multiplication, returning the full double-width result. + * + * Postconditions: + * out1 = (arg1 * arg2) mod 2^64 + * out2 = ⌊arg1 * arg2 / 2^64⌋ + * + * Input Bounds: + * arg1: [0x0 ~> 0xffffffffffffffff] + * arg2: [0x0 ~> 0xffffffffffffffff] + * Output Bounds: + * out1: [0x0 ~> 0xffffffffffffffff] + * out2: [0x0 ~> 0xffffffffffffffff] + */ +static inline void fiat_mulx_u64(uint64_t* out1, uint64_t* out2, uint64_t arg1, uint64_t arg2) { +// NOTE: edited after generation +#if defined(_M_X64) + unsigned long long t; + *out1 = _umul128(arg1, arg2, &t); + *out2 = t; +#elif defined(_M_ARM64) + *out1 = arg1 * arg2; + *out2 = __umulh(arg1, arg2); +#else + unsigned __int128 t = (unsigned __int128)arg1 * arg2; + *out1 = t; + *out2 = (t >> 64); +#endif +} + +/* + * The function fiat_addcarryx_u64 is an addition with carry. + * + * Postconditions: + * out1 = (arg1 + arg2 + arg3) mod 2^64 + * out2 = ⌊(arg1 + arg2 + arg3) / 2^64⌋ + * + * Input Bounds: + * arg1: [0x0 ~> 0x1] + * arg2: [0x0 ~> 0xffffffffffffffff] + * arg3: [0x0 ~> 0xffffffffffffffff] + * Output Bounds: + * out1: [0x0 ~> 0xffffffffffffffff] + * out2: [0x0 ~> 0x1] + */ +static inline void fiat_addcarryx_u64(uint64_t* out1, fiat_uint1* out2, fiat_uint1 arg1, uint64_t arg2, uint64_t arg3) { +// NOTE: edited after generation +#if defined(__has_builtin) +# if __has_builtin(__builtin_ia32_addcarryx_u64) +# define addcarry64 __builtin_ia32_addcarryx_u64 +# endif +#endif +#if defined(addcarry64) + long long unsigned int t; + *out2 = addcarry64(arg1, arg2, arg3, &t); + *out1 = t; +#elif defined(_M_X64) + long long unsigned int t; + *out2 = _addcarry_u64(arg1, arg2, arg3, out1); + *out1 = t; +#else + arg2 += arg1; + arg1 = arg2 < arg1; + uint64_t ret = arg2 + arg3; + arg1 += ret < arg2; + *out1 = ret; + *out2 = arg1; +#endif +#undef addcarry64 +} + +/* + * The function fiat_subborrowx_u64 is a subtraction with borrow. + * + * Postconditions: + * out1 = (-arg1 + arg2 + -arg3) mod 2^64 + * out2 = -⌊(-arg1 + arg2 + -arg3) / 2^64⌋ + * + * Input Bounds: + * arg1: [0x0 ~> 0x1] + * arg2: [0x0 ~> 0xffffffffffffffff] + * arg3: [0x0 ~> 0xffffffffffffffff] + * Output Bounds: + * out1: [0x0 ~> 0xffffffffffffffff] + * out2: [0x0 ~> 0x1] + */ +static inline void fiat_subborrowx_u64(uint64_t* out1, fiat_uint1* out2, fiat_uint1 arg1, uint64_t arg2, uint64_t arg3) { +#if defined(__has_builtin) +# if __has_builtin(__builtin_ia32_subborrow_u64) +# define subborrow64 __builtin_ia32_subborrow_u64 +# endif +#endif +#if defined(subborrow64) + long long unsigned int t; + *out2 = subborrow64(arg1, arg2, arg3, &t); + *out1 = t; +#elif defined(_M_X64) + long long unsigned int t; + *out2 = _subborrow_u64(arg1, arg2, arg3, &t); // NOTE: edited after generation + *out1 = t; +#else + *out1 = arg2 - arg3 - arg1; + *out2 = (arg2 < arg3) | ((arg2 == arg3) & arg1); +#endif +#undef subborrow64 +} + +/* + * The function fiat_cmovznz_u64 is a single-word conditional move. + * + * Postconditions: + * out1 = (if arg1 = 0 then arg2 else arg3) + * + * Input Bounds: + * arg1: [0x0 ~> 0x1] + * arg2: [0x0 ~> 0xffffffffffffffff] + * arg3: [0x0 ~> 0xffffffffffffffff] + * Output Bounds: + * out1: [0x0 ~> 0xffffffffffffffff] + */ +static inline void fiat_cmovznz_u64(uint64_t* out1, fiat_uint1 arg1, uint64_t arg2, uint64_t arg3) { + fiat_uint1 x1; + uint64_t x2; + uint64_t x3; + x1 = (!(!arg1)); + x2 = ((fiat_int1)(0x0 - x1) & UINT64_C(0xffffffffffffffff)); + x3 = ((fiat_value_barrier_u64(x2) & arg3) | (fiat_value_barrier_u64((~x2)) & arg2)); + *out1 = x3; +} + +/* + * Input Bounds: + * arg1: [[0x0 ~> 0xffffffffffffffff], [0x0 ~> 0xffffffffffffffff], [0x0 ~> 0xffffffffffffffff], [0x0 ~> 0xffffffffffffffff]] + * arg2: [[0x0 ~> 0xffffffffffffffff], [0x0 ~> 0xffffffffffffffff], [0x0 ~> 0xffffffffffffffff], [0x0 ~> 0xffffffffffffffff]] + * Output Bounds: + * out1: [[0x0 ~> 0xffffffffffffffff], [0x0 ~> 0xffffffffffffffff], [0x0 ~> 0xffffffffffffffff], [0x0 ~> 0xffffffffffffffff]] + */ +static void fe4_add(uint64_t out1[4], const uint64_t arg1[4], const uint64_t arg2[4]) { + uint64_t x1; + fiat_uint1 x2; + uint64_t x3; + fiat_uint1 x4; + uint64_t x5; + fiat_uint1 x6; + uint64_t x7; + fiat_uint1 x8; + uint64_t x9; + uint64_t x10; + fiat_uint1 x11; + uint64_t x12; + fiat_uint1 x13; + uint64_t x14; + fiat_uint1 x15; + uint64_t x16; + fiat_uint1 x17; + uint64_t x18; + uint64_t x19; + fiat_uint1 x20; + fiat_addcarryx_u64(&x1, &x2, 0x0, (arg1[0]), (arg2[0])); + fiat_addcarryx_u64(&x3, &x4, x2, (arg1[1]), (arg2[1])); + fiat_addcarryx_u64(&x5, &x6, x4, (arg1[2]), (arg2[2])); + fiat_addcarryx_u64(&x7, &x8, x6, (arg1[3]), (arg2[3])); + fiat_cmovznz_u64(&x9, x8, 0x0, UINT8_C(0x26)); // NOTE: clang 14 for Zen 2 uses sbb, and + fiat_addcarryx_u64(&x10, &x11, 0x0, x1, x9); + fiat_addcarryx_u64(&x12, &x13, x11, x3, 0x0); + fiat_addcarryx_u64(&x14, &x15, x13, x5, 0x0); + fiat_addcarryx_u64(&x16, &x17, x15, x7, 0x0); + fiat_cmovznz_u64(&x18, x17, 0x0, UINT8_C(0x26)); // NOTE: clang 14 for Zen 2 uses sbb, and + fiat_addcarryx_u64(&x19, &x20, 0x0, x10, x18); + out1[0] = x19; + out1[1] = x12; + out1[2] = x14; + out1[3] = x16; +} + +/* + * Input Bounds: + * arg1: [[0x0 ~> 0xffffffffffffffff], [0x0 ~> 0xffffffffffffffff], [0x0 ~> 0xffffffffffffffff], [0x0 ~> 0xffffffffffffffff]] + * arg2: [[0x0 ~> 0xffffffffffffffff], [0x0 ~> 0xffffffffffffffff], [0x0 ~> 0xffffffffffffffff], [0x0 ~> 0xffffffffffffffff]] + * Output Bounds: + * out1: [[0x0 ~> 0xffffffffffffffff], [0x0 ~> 0xffffffffffffffff], [0x0 ~> 0xffffffffffffffff], [0x0 ~> 0xffffffffffffffff]] + */ +static void fe4_sub(uint64_t out1[4], const uint64_t arg1[4], const uint64_t arg2[4]) { + uint64_t x1; + uint64_t x2; + fiat_uint1 x3; + uint64_t x4; + uint64_t x5; + fiat_uint1 x6; + uint64_t x7; + uint64_t x8; + fiat_uint1 x9; + uint64_t x10; + uint64_t x11; + fiat_uint1 x12; + uint64_t x13; + uint64_t x14; + fiat_uint1 x15; + uint64_t x16; + fiat_uint1 x17; + uint64_t x18; + fiat_uint1 x19; + uint64_t x20; + fiat_uint1 x21; + uint64_t x22; + uint64_t x23; + fiat_uint1 x24; + x1 = (arg2[0]); + fiat_subborrowx_u64(&x2, &x3, 0x0, (arg1[0]), x1); + x4 = (arg2[1]); + fiat_subborrowx_u64(&x5, &x6, x3, (arg1[1]), x4); + x7 = (arg2[2]); + fiat_subborrowx_u64(&x8, &x9, x6, (arg1[2]), x7); + x10 = (arg2[3]); + fiat_subborrowx_u64(&x11, &x12, x9, (arg1[3]), x10); + fiat_cmovznz_u64(&x13, x12, 0x0, UINT8_C(0x26)); // NOTE: clang 14 for Zen 2 uses sbb, and + fiat_subborrowx_u64(&x14, &x15, 0x0, x2, x13); + fiat_subborrowx_u64(&x16, &x17, x15, x5, 0x0); + fiat_subborrowx_u64(&x18, &x19, x17, x8, 0x0); + fiat_subborrowx_u64(&x20, &x21, x19, x11, 0x0); + fiat_cmovznz_u64(&x22, x21, 0x0, UINT8_C(0x26)); // NOTE: clang 14 for Zen 2 uses sbb, and + fiat_subborrowx_u64(&x23, &x24, 0x0, x14, x22); + out1[0] = x23; + out1[1] = x16; + out1[2] = x18; + out1[3] = x20; +} + +/* + * Input Bounds: + * arg1: [[0x0 ~> 0xffffffffffffffff], [0x0 ~> 0xffffffffffffffff], [0x0 ~> 0xffffffffffffffff], [0x0 ~> 0xffffffffffffffff]] + * arg2: [0x0 ~> 0x3ffffffffffffff] // NOTE: this is not any uint64! + * Output Bounds: + * out1: [[0x0 ~> 0xffffffffffffffff], [0x0 ~> 0xffffffffffffffff], [0x0 ~> 0xffffffffffffffff], [0x0 ~> 0xffffffffffffffff]] + */ +static void fe4_scmul(uint64_t out1[4], const uint64_t arg1[4], uint64_t arg2) { + uint64_t x1; + uint64_t x2; + uint64_t x3; + uint64_t x4; + uint64_t x5; + fiat_uint1 x6; + uint64_t x7; + uint64_t x8; + uint64_t x9; + fiat_uint1 x10; + uint64_t x11; + uint64_t x12; + uint64_t x13; + fiat_uint1 x14; + uint64_t x15; + uint64_t x16; + uint64_t x17; + fiat_uint1 x18; + uint64_t x19; + fiat_uint1 x20; + uint64_t x21; + fiat_uint1 x22; + uint64_t x23; + fiat_uint1 x24; + uint64_t x25; + uint64_t x26; + fiat_uint1 x27; + fiat_mulx_u64(&x1, &x2, (arg1[0]), arg2); + fiat_mulx_u64(&x3, &x4, (arg1[1]), arg2); + fiat_addcarryx_u64(&x5, &x6, 0x0, x2, x3); + fiat_mulx_u64(&x7, &x8, (arg1[2]), arg2); + fiat_addcarryx_u64(&x9, &x10, x6, x4, x7); + fiat_mulx_u64(&x11, &x12, (arg1[3]), arg2); + fiat_addcarryx_u64(&x13, &x14, x10, x8, x11); + fiat_mulx_u64(&x15, &x16, (x12 + (uint64_t)x14), UINT8_C(0x26)); + fiat_addcarryx_u64(&x17, &x18, 0x0, x1, x15); + fiat_addcarryx_u64(&x19, &x20, x18, x5, 0x0); + fiat_addcarryx_u64(&x21, &x22, x20, x9, 0x0); + fiat_addcarryx_u64(&x23, &x24, x22, x13, 0x0); + fiat_cmovznz_u64(&x25, x24, 0x0, UINT8_C(0x26)); // NOTE: clang 14 for Zen 2 uses sbb, and + fiat_addcarryx_u64(&x26, &x27, 0x0, x17, x25); + out1[0] = x26; + out1[1] = x19; + out1[2] = x21; + out1[3] = x23; +} + +/* + * Input Bounds: + * arg1: [[0x0 ~> 0xffffffffffffffff], [0x0 ~> 0xffffffffffffffff], [0x0 ~> 0xffffffffffffffff], [0x0 ~> 0xffffffffffffffff]] + * Output Bounds: + * out1: [[0x0 ~> 0xffffffffffffffff], [0x0 ~> 0xffffffffffffffff], [0x0 ~> 0xffffffffffffffff], [0x0 ~> 0xffffffffffffffff]] + */ +static void fe4_canon(uint64_t out1[4], const uint64_t arg1[4]) { + uint64_t x1; + fiat_uint1 x2; + uint64_t x3; + fiat_uint1 x4; + uint64_t x5; + fiat_uint1 x6; + uint64_t x7; + fiat_uint1 x8; + uint64_t x9; + uint64_t x10; + uint64_t x11; + uint64_t x12; + uint64_t x13; + fiat_uint1 x14; + uint64_t x15; + fiat_uint1 x16; + uint64_t x17; + fiat_uint1 x18; + uint64_t x19; + fiat_uint1 x20; + uint64_t x21; + uint64_t x22; + uint64_t x23; + uint64_t x24; + fiat_subborrowx_u64(&x1, &x2, 0x0, (arg1[0]), UINT64_C(0xffffffffffffffed)); + fiat_subborrowx_u64(&x3, &x4, x2, (arg1[1]), UINT64_C(0xffffffffffffffff)); + fiat_subborrowx_u64(&x5, &x6, x4, (arg1[2]), UINT64_C(0xffffffffffffffff)); + fiat_subborrowx_u64(&x7, &x8, x6, (arg1[3]), UINT64_C(0x7fffffffffffffff)); + fiat_cmovznz_u64(&x9, x8, x1, (arg1[0])); + fiat_cmovznz_u64(&x10, x8, x3, (arg1[1])); + fiat_cmovznz_u64(&x11, x8, x5, (arg1[2])); + fiat_cmovznz_u64(&x12, x8, x7, (arg1[3])); + fiat_subborrowx_u64(&x13, &x14, 0x0, x9, UINT64_C(0xffffffffffffffed)); + fiat_subborrowx_u64(&x15, &x16, x14, x10, UINT64_C(0xffffffffffffffff)); + fiat_subborrowx_u64(&x17, &x18, x16, x11, UINT64_C(0xffffffffffffffff)); + fiat_subborrowx_u64(&x19, &x20, x18, x12, UINT64_C(0x7fffffffffffffff)); + fiat_cmovznz_u64(&x21, x20, x13, x9); + fiat_cmovznz_u64(&x22, x20, x15, x10); + fiat_cmovznz_u64(&x23, x20, x17, x11); + fiat_cmovznz_u64(&x24, x20, x19, x12); + out1[0] = x21; + out1[1] = x22; + out1[2] = x23; + out1[3] = x24; +} + +/* + * Input Bounds: + * arg1: [0x0 ~> 0x1] + * arg2: [[0x0 ~> 0xffffffffffffffff], [0x0 ~> 0xffffffffffffffff], [0x0 ~> 0xffffffffffffffff], [0x0 ~> 0xffffffffffffffff]] + * arg3: [[0x0 ~> 0xffffffffffffffff], [0x0 ~> 0xffffffffffffffff], [0x0 ~> 0xffffffffffffffff], [0x0 ~> 0xffffffffffffffff]] + * Output Bounds: + * out1: [[0x0 ~> 0xffffffffffffffff], [0x0 ~> 0xffffffffffffffff], [0x0 ~> 0xffffffffffffffff], [0x0 ~> 0xffffffffffffffff]] + * out2: [[0x0 ~> 0xffffffffffffffff], [0x0 ~> 0xffffffffffffffff], [0x0 ~> 0xffffffffffffffff], [0x0 ~> 0xffffffffffffffff]] + */ +static void fe4_cswap(uint64_t out1[4], uint64_t out2[4], fiat_uint1 arg1, const uint64_t arg2[4], const uint64_t arg3[4]) { + uint64_t x1; + uint64_t x2; + uint64_t x3; + uint64_t x4; + uint64_t x5; + uint64_t x6; + uint64_t x7; + uint64_t x8; + // NOTE: clang 14 for Zen 2 uses YMM registers + fiat_cmovznz_u64(&x1, arg1, (arg2[0]), (arg3[0])); + fiat_cmovznz_u64(&x2, arg1, (arg2[1]), (arg3[1])); + fiat_cmovznz_u64(&x3, arg1, (arg2[2]), (arg3[2])); + fiat_cmovznz_u64(&x4, arg1, (arg2[3]), (arg3[3])); + fiat_cmovznz_u64(&x5, arg1, (arg3[0]), (arg2[0])); + fiat_cmovznz_u64(&x6, arg1, (arg3[1]), (arg2[1])); + fiat_cmovznz_u64(&x7, arg1, (arg3[2]), (arg2[2])); + fiat_cmovznz_u64(&x8, arg1, (arg3[3]), (arg2[3])); + out1[0] = x1; + out1[1] = x2; + out1[2] = x3; + out1[3] = x4; + out2[0] = x5; + out2[1] = x6; + out2[2] = x7; + out2[3] = x8; +} + +// The following functions are adaped from crypto/curve25519/curve25519.c +// It would be desirable to share the code, but with the current field +// implementations both 4-limb and 5-limb versions of the curve-level code need +// to be included in builds targetting an unknown variant of x86_64. + +static void fe4_invert(fe4 out, const fe4 z) { + fe4 t0; + fe4 t1; + fe4 t2; + fe4 t3; + int i; + + fe4_sq(t0, z); + fe4_sq(t1, t0); + for (i = 1; i < 2; ++i) { + fe4_sq(t1, t1); + } + fe4_mul(t1, z, t1); + fe4_mul(t0, t0, t1); + fe4_sq(t2, t0); + fe4_mul(t1, t1, t2); + fe4_sq(t2, t1); + for (i = 1; i < 5; ++i) { + fe4_sq(t2, t2); + } + fe4_mul(t1, t2, t1); + fe4_sq(t2, t1); + for (i = 1; i < 10; ++i) { + fe4_sq(t2, t2); + } + fe4_mul(t2, t2, t1); + fe4_sq(t3, t2); + for (i = 1; i < 20; ++i) { + fe4_sq(t3, t3); + } + fe4_mul(t2, t3, t2); + fe4_sq(t2, t2); + for (i = 1; i < 10; ++i) { + fe4_sq(t2, t2); + } + fe4_mul(t1, t2, t1); + fe4_sq(t2, t1); + for (i = 1; i < 50; ++i) { + fe4_sq(t2, t2); + } + fe4_mul(t2, t2, t1); + fe4_sq(t3, t2); + for (i = 1; i < 100; ++i) { + fe4_sq(t3, t3); + } + fe4_mul(t2, t3, t2); + fe4_sq(t2, t2); + for (i = 1; i < 50; ++i) { + fe4_sq(t2, t2); + } + fe4_mul(t1, t2, t1); + fe4_sq(t1, t1); + for (i = 1; i < 5; ++i) { + fe4_sq(t1, t1); + } + fe4_mul(out, t1, t0); +} + +void x25519_scalar_mult_adx(uint8_t out[32], const uint8_t scalar[32], + const uint8_t point[32]) { + uint8_t e[32]; + memcpy(e, scalar, 32); + e[0] &= 248; + e[31] &= 127; + e[31] |= 64; + + // The following implementation was transcribed to Coq and proven to + // correspond to unary scalar multiplication in affine coordinates given that + // x1 != 0 is the x coordinate of some point on the curve. It was also checked + // in Coq that doing a ladderstep with x1 = x3 = 0 gives z2' = z3' = 0, and z2 + // = z3 = 0 gives z2' = z3' = 0. The statement was quantified over the + // underlying field, so it applies to Curve25519 itself and the quadratic + // twist of Curve25519. It was not proven in Coq that prime-field arithmetic + // correctly simulates extension-field arithmetic on prime-field values. + // The decoding of the byte array representation of e was not considered. + // Specification of Montgomery curves in affine coordinates: + // + // Proof that these form a group that is isomorphic to a Weierstrass curve: + // + // Coq transcription and correctness proof of the loop (where scalarbits=255): + // + // + // preconditions: 0 <= e < 2^255 (not necessarily e < order), fe_invert(0) = 0 + fe4 x1, x2 = {1}, z2 = {0}, x3, z3 = {1}, tmp0, tmp1; + OPENSSL_memcpy(x1, point, sizeof(fe4)); + x1[3] &= (uint64_t)(-1)>>1; + OPENSSL_memcpy(x3, x1, sizeof(fe4)); + + unsigned swap = 0; + int pos; + for (pos = 254; pos >= 0; --pos) { + // loop invariant as of right before the test, for the case where x1 != 0: + // pos >= -1; if z2 = 0 then x2 is nonzero; if z3 = 0 then x3 is nonzero + // let r := e >> (pos+1) in the following equalities of projective points: + // to_xz (r*P) === if swap then (x3, z3) else (x2, z2) + // to_xz ((r+1)*P) === if swap then (x2, z2) else (x3, z3) + // x1 is the nonzero x coordinate of the nonzero point (r*P-(r+1)*P) + unsigned b = 1 & (e[pos / 8] >> (pos & 7)); + swap ^= b; + fe4_cswap(x2, x3, swap, x2, x3); + fe4_cswap(z2, z3, swap, z2, z3); + swap = b; + // Coq transcription of ladderstep formula (called from transcribed loop): + // + // + // x1 != 0 + // x1 = 0 + fe4_sub(tmp0, x3, z3); + fe4_sub(tmp1, x2, z2); + fe4_add(x2, x2, z2); + fe4_add(z2, x3, z3); + fe4_mul(z3, tmp0, x2); + fe4_mul(z2, z2, tmp1); + fe4_sq(tmp0, tmp1); + fe4_sq(tmp1, x2); + fe4_add(x3, z3, z2); + fe4_sub(z2, z3, z2); + fe4_mul(x2, tmp1, tmp0); + fe4_sub(tmp1, tmp1, tmp0); + fe4_sq(z2, z2); + fe4_scmul(z3, tmp1, 121666); + fe4_sq(x3, x3); + fe4_add(tmp0, tmp0, z3); + fe4_mul(z3, x1, z2); + fe4_mul(z2, tmp1, tmp0); + } + // here pos=-1, so r=e, so to_xz (e*P) === if swap then (x3, z3) else (x2, z2) + fe4_cswap(x2, x3, swap, x2, x3); + fe4_cswap(z2, z3, swap, z2, z3); + + fe4_invert(z2, z2); + fe4_mul(x2, x2, z2); + fe4_canon(x2, x2); + OPENSSL_memcpy(out, x2, sizeof(fe4)); +}