diff --git a/crypto/ecdsa/ecdsa.c b/crypto/ecdsa/ecdsa.c index 0e8e42f88..067fd6ccb 100644 --- a/crypto/ecdsa/ecdsa.c +++ b/crypto/ecdsa/ecdsa.c @@ -73,7 +73,7 @@ int ECDSA_sign(int type, const uint8_t *digest, size_t digest_len, uint8_t *sig, int ECDSA_verify(int type, const uint8_t *digest, size_t digest_len, const uint8_t *sig, size_t sig_len, EC_KEY *eckey) { ECDSA_SIG *s; - int ret = -1; + int ret = 0; if (eckey->ecdsa_meth && eckey->ecdsa_meth->verify) { return eckey->ecdsa_meth->verify(digest, digest_len, sig, sig_len, eckey); @@ -127,7 +127,7 @@ ECDSA_SIG *ECDSA_do_sign(const uint8_t *digest, size_t digest_len, int ECDSA_do_verify(const uint8_t *digest, size_t digest_len, const ECDSA_SIG *sig, EC_KEY *eckey) { - int ret = -1; + int ret = 0; BN_CTX *ctx; BIGNUM *order, *u1, *u2, *m, *X; EC_POINT *point = NULL; @@ -136,20 +136,20 @@ int ECDSA_do_verify(const uint8_t *digest, size_t digest_len, if (eckey->ecdsa_meth && eckey->ecdsa_meth->verify) { OPENSSL_PUT_ERROR(ECDSA, ECDSA_do_verify, ECDSA_R_NOT_IMPLEMENTED); - return -1; + return 0; } /* check input values */ if (eckey == NULL || (group = EC_KEY_get0_group(eckey)) == NULL || (pub_key = EC_KEY_get0_public_key(eckey)) == NULL || sig == NULL) { OPENSSL_PUT_ERROR(ECDSA, ECDSA_do_verify, ECDSA_R_MISSING_PARAMETERS); - return -1; + return 0; } ctx = BN_CTX_new(); if (!ctx) { OPENSSL_PUT_ERROR(ECDSA, ECDSA_do_verify, ERR_R_MALLOC_FAILURE); - return -1; + return 0; } BN_CTX_start(ctx); order = BN_CTX_get(ctx); diff --git a/crypto/evp/evp_ctx.c b/crypto/evp/evp_ctx.c index 1365dae77..1416d3a2d 100644 --- a/crypto/evp/evp_ctx.c +++ b/crypto/evp/evp_ctx.c @@ -232,7 +232,6 @@ int EVP_PKEY_CTX_ctrl(EVP_PKEY_CTX *ctx, int keytype, int optype, int cmd, } int EVP_PKEY_sign_init(EVP_PKEY_CTX *ctx) { - int ret; if (!ctx || !ctx->pmeth || !ctx->pmeth->sign) { OPENSSL_PUT_ERROR(EVP, EVP_PKEY_sign_init, EVP_R_OPERATION_NOT_SUPPORTED_FOR_THIS_KEYTYPE); @@ -244,14 +243,12 @@ int EVP_PKEY_sign_init(EVP_PKEY_CTX *ctx) { return 1; } - ret = ctx->pmeth->sign_init(ctx); - /* TODO(fork): normalise the return value convention. */ - if (ret <= 0) { + if (!ctx->pmeth->sign_init(ctx)) { ctx->operation = EVP_PKEY_OP_UNDEFINED; return 0; } - return ret; + return 1; } int EVP_PKEY_sign(EVP_PKEY_CTX *ctx, uint8_t *sig, size_t *sig_len, @@ -269,22 +266,21 @@ int EVP_PKEY_sign(EVP_PKEY_CTX *ctx, uint8_t *sig, size_t *sig_len, } int EVP_PKEY_verify_init(EVP_PKEY_CTX *ctx) { - int ret; - if (!ctx || !ctx->pmeth || !ctx->pmeth->verify) { OPENSSL_PUT_ERROR(EVP, EVP_PKEY_verify_init, EVP_R_OPERATION_NOT_SUPPORTED_FOR_THIS_KEYTYPE); - return -2; + return 0; } ctx->operation = EVP_PKEY_OP_VERIFY; if (!ctx->pmeth->verify_init) { return 1; } - ret = ctx->pmeth->verify_init(ctx); - if (ret <= 0) { + if (!ctx->pmeth->verify_init(ctx)) { ctx->operation = EVP_PKEY_OP_UNDEFINED; + return 0; } - return ret; + + return 1; } int EVP_PKEY_verify(EVP_PKEY_CTX *ctx, const uint8_t *sig, size_t sig_len, @@ -292,31 +288,30 @@ int EVP_PKEY_verify(EVP_PKEY_CTX *ctx, const uint8_t *sig, size_t sig_len, if (!ctx || !ctx->pmeth || !ctx->pmeth->verify) { OPENSSL_PUT_ERROR(EVP, EVP_PKEY_verify, EVP_R_OPERATION_NOT_SUPPORTED_FOR_THIS_KEYTYPE); - return -2; + return 0; } if (ctx->operation != EVP_PKEY_OP_VERIFY) { OPENSSL_PUT_ERROR(EVP, EVP_PKEY_verify, EVP_R_OPERATON_NOT_INITIALIZED); - return -1; + return 0; } return ctx->pmeth->verify(ctx, sig, sig_len, data, data_len); } int EVP_PKEY_encrypt_init(EVP_PKEY_CTX *ctx) { - int ret; if (!ctx || !ctx->pmeth || !ctx->pmeth->encrypt) { OPENSSL_PUT_ERROR(EVP, EVP_PKEY_encrypt_init, EVP_R_OPERATION_NOT_SUPPORTED_FOR_THIS_KEYTYPE); - return -2; + return 0; } ctx->operation = EVP_PKEY_OP_ENCRYPT; if (!ctx->pmeth->encrypt_init) { return 1; } - ret = ctx->pmeth->encrypt_init(ctx); - if (ret <= 0) { + if (!ctx->pmeth->encrypt_init(ctx)) { ctx->operation = EVP_PKEY_OP_UNDEFINED; + return 0; } - return ret; + return 1; } int EVP_PKEY_encrypt(EVP_PKEY_CTX *ctx, uint8_t *out, size_t *outlen, @@ -324,32 +319,30 @@ int EVP_PKEY_encrypt(EVP_PKEY_CTX *ctx, uint8_t *out, size_t *outlen, if (!ctx || !ctx->pmeth || !ctx->pmeth->encrypt) { OPENSSL_PUT_ERROR(EVP, EVP_PKEY_encrypt, EVP_R_OPERATION_NOT_SUPPORTED_FOR_THIS_KEYTYPE); - return -2; + return 0; } if (ctx->operation != EVP_PKEY_OP_ENCRYPT) { OPENSSL_PUT_ERROR(EVP, EVP_PKEY_encrypt, EVP_R_OPERATON_NOT_INITIALIZED); - return -1; + return 0; } return ctx->pmeth->encrypt(ctx, out, outlen, in, inlen); } int EVP_PKEY_decrypt_init(EVP_PKEY_CTX *ctx) { - int ret; - if (!ctx || !ctx->pmeth || !ctx->pmeth->decrypt) { OPENSSL_PUT_ERROR(EVP, EVP_PKEY_decrypt_init, EVP_R_OPERATION_NOT_SUPPORTED_FOR_THIS_KEYTYPE); - return -2; + return 0; } ctx->operation = EVP_PKEY_OP_DECRYPT; if (!ctx->pmeth->decrypt_init) { return 1; } - ret = ctx->pmeth->decrypt_init(ctx); - if (ret <= 0) { + if (!ctx->pmeth->decrypt_init(ctx)) { ctx->operation = EVP_PKEY_OP_UNDEFINED; + return 0; } - return ret; + return 1; } int EVP_PKEY_decrypt(EVP_PKEY_CTX *ctx, uint8_t *out, size_t *outlen, @@ -357,31 +350,30 @@ int EVP_PKEY_decrypt(EVP_PKEY_CTX *ctx, uint8_t *out, size_t *outlen, if (!ctx || !ctx->pmeth || !ctx->pmeth->decrypt) { OPENSSL_PUT_ERROR(EVP, EVP_PKEY_decrypt, EVP_R_OPERATION_NOT_SUPPORTED_FOR_THIS_KEYTYPE); - return -2; + return 0; } if (ctx->operation != EVP_PKEY_OP_DECRYPT) { OPENSSL_PUT_ERROR(EVP, EVP_PKEY_decrypt, EVP_R_OPERATON_NOT_INITIALIZED); - return -1; + return 0; } return ctx->pmeth->decrypt(ctx, out, outlen, in, inlen); } int EVP_PKEY_derive_init(EVP_PKEY_CTX *ctx) { - int ret; if (!ctx || !ctx->pmeth || !ctx->pmeth->derive) { OPENSSL_PUT_ERROR(EVP, EVP_PKEY_derive_init, EVP_R_OPERATION_NOT_SUPPORTED_FOR_THIS_KEYTYPE); - return -2; + return 0; } ctx->operation = EVP_PKEY_OP_DERIVE; if (!ctx->pmeth->derive_init) { return 1; } - ret = ctx->pmeth->derive_init(ctx); - if (ret <= 0) { + if (!ctx->pmeth->derive_init(ctx)) { ctx->operation = EVP_PKEY_OP_UNDEFINED; + return 0; } - return ret; + return 1; } int EVP_PKEY_derive_set_peer(EVP_PKEY_CTX *ctx, EVP_PKEY *peer) { @@ -391,20 +383,20 @@ int EVP_PKEY_derive_set_peer(EVP_PKEY_CTX *ctx, EVP_PKEY *peer) { !ctx->pmeth->ctrl) { OPENSSL_PUT_ERROR(EVP, EVP_PKEY_derive_set_peer, EVP_R_OPERATION_NOT_SUPPORTED_FOR_THIS_KEYTYPE); - return -2; + return 0; } if (ctx->operation != EVP_PKEY_OP_DERIVE && ctx->operation != EVP_PKEY_OP_ENCRYPT && ctx->operation != EVP_PKEY_OP_DECRYPT) { OPENSSL_PUT_ERROR(EVP, EVP_PKEY_derive_set_peer, EVP_R_OPERATON_NOT_INITIALIZED); - return -1; + return 0; } ret = ctx->pmeth->ctrl(ctx, EVP_PKEY_CTRL_PEER_KEY, 0, peer); if (ret <= 0) { - return ret; + return 0; } if (ret == 2) { @@ -413,12 +405,12 @@ int EVP_PKEY_derive_set_peer(EVP_PKEY_CTX *ctx, EVP_PKEY *peer) { if (!ctx->pkey) { OPENSSL_PUT_ERROR(EVP, EVP_PKEY_derive_set_peer, EVP_R_NO_KEY_SET); - return -1; + return 0; } if (ctx->pkey->type != peer->type) { OPENSSL_PUT_ERROR(EVP, EVP_PKEY_derive_set_peer, EVP_R_DIFFERENT_KEY_TYPES); - return -1; + return 0; } /* ran@cryptocom.ru: For clarity. The error is if parameters in peer are @@ -430,7 +422,7 @@ int EVP_PKEY_derive_set_peer(EVP_PKEY_CTX *ctx, EVP_PKEY *peer) { !EVP_PKEY_cmp_parameters(ctx->pkey, peer)) { OPENSSL_PUT_ERROR(EVP, EVP_PKEY_derive_set_peer, EVP_R_DIFFERENT_PARAMETERS); - return -1; + return 0; } if (ctx->peerkey) { @@ -442,7 +434,7 @@ int EVP_PKEY_derive_set_peer(EVP_PKEY_CTX *ctx, EVP_PKEY *peer) { if (ret <= 0) { ctx->peerkey = NULL; - return ret; + return 0; } CRYPTO_add(&peer->references, 1, CRYPTO_LOCK_EVP_PKEY); @@ -453,58 +445,55 @@ int EVP_PKEY_derive(EVP_PKEY_CTX *ctx, uint8_t *key, size_t *out_key_len) { if (!ctx || !ctx->pmeth || !ctx->pmeth->derive) { OPENSSL_PUT_ERROR(EVP, EVP_PKEY_derive, EVP_R_OPERATION_NOT_SUPPORTED_FOR_THIS_KEYTYPE); - return -2; + return 0; } if (ctx->operation != EVP_PKEY_OP_DERIVE) { OPENSSL_PUT_ERROR(EVP, EVP_PKEY_derive, EVP_R_OPERATON_NOT_INITIALIZED); - return -1; + return 0; } return ctx->pmeth->derive(ctx, key, out_key_len); } int EVP_PKEY_keygen_init(EVP_PKEY_CTX *ctx) { - int ret; if (!ctx || !ctx->pmeth || !ctx->pmeth->keygen) { OPENSSL_PUT_ERROR(EVP, EVP_PKEY_keygen_init, EVP_R_OPERATION_NOT_SUPPORTED_FOR_THIS_KEYTYPE); - return -2; + return 0; } ctx->operation = EVP_PKEY_OP_KEYGEN; if (!ctx->pmeth->keygen_init) { return 1; } - ret = ctx->pmeth->keygen_init(ctx); - if (ret <= 0) { + if (!ctx->pmeth->keygen_init(ctx)) { ctx->operation = EVP_PKEY_OP_UNDEFINED; + return 0; } - return ret; + return 1; } int EVP_PKEY_keygen(EVP_PKEY_CTX *ctx, EVP_PKEY **ppkey) { - int ret; - if (!ctx || !ctx->pmeth || !ctx->pmeth->keygen) { OPENSSL_PUT_ERROR(EVP, EVP_PKEY_keygen, EVP_R_OPERATION_NOT_SUPPORTED_FOR_THIS_KEYTYPE); - return -2; + return 0; } if (ctx->operation != EVP_PKEY_OP_KEYGEN) { OPENSSL_PUT_ERROR(EVP, EVP_PKEY_keygen, EVP_R_OPERATON_NOT_INITIALIZED); - return -1; + return 0; } if (!ppkey) { - return -1; + return 0; } if (!*ppkey) { *ppkey = EVP_PKEY_new(); } - ret = ctx->pmeth->keygen(ctx, *ppkey); - if (ret <= 0) { + if (!ctx->pmeth->keygen(ctx, *ppkey)) { EVP_PKEY_free(*ppkey); *ppkey = NULL; + return 0; } - return ret; + return 1; } diff --git a/crypto/evp/p_ec.c b/crypto/evp/p_ec.c index 2305e6e8f..e429f77a5 100644 --- a/crypto/evp/p_ec.c +++ b/crypto/evp/p_ec.c @@ -164,7 +164,7 @@ static void pkey_ec_cleanup(EVP_PKEY_CTX *ctx) { static int pkey_ec_sign(EVP_PKEY_CTX *ctx, uint8_t *sig, size_t *siglen, const uint8_t *tbs, size_t tbslen) { - int ret, type; + int type; unsigned int sltmp; EC_PKEY_CTX *dctx = ctx->data; EC_KEY *ec = ctx->pkey->pkey.ec; @@ -182,10 +182,8 @@ static int pkey_ec_sign(EVP_PKEY_CTX *ctx, uint8_t *sig, size_t *siglen, type = EVP_MD_type(dctx->md); } - ret = ECDSA_sign(type, tbs, tbslen, sig, &sltmp, ec); - - if (ret <= 0) { - return ret; + if (!ECDSA_sign(type, tbs, tbslen, sig, &sltmp, ec)) { + return 0; } *siglen = (size_t)sltmp; return 1; @@ -235,7 +233,7 @@ static int pkey_ec_derive(EVP_PKEY_CTX *ctx, uint8_t *key, ret = ECDH_compute_key(key, outlen, pubkey, eckey, 0); if (ret < 0) { - return ret; + return 0; } *keylen = ret; return 1; diff --git a/crypto/evp/p_rsa.c b/crypto/evp/p_rsa.c index 458b606e1..bcaca4b54 100644 --- a/crypto/evp/p_rsa.c +++ b/crypto/evp/p_rsa.c @@ -168,58 +168,57 @@ static int setup_tbuf(RSA_PKEY_CTX *ctx, EVP_PKEY_CTX *pk) { static int pkey_rsa_sign(EVP_PKEY_CTX *ctx, uint8_t *sig, size_t *siglen, const uint8_t *tbs, size_t tbslen) { - int ret; RSA_PKEY_CTX *rctx = ctx->data; RSA *rsa = ctx->pkey->pkey.rsa; + const size_t key_len = EVP_PKEY_size(ctx->pkey); if (!sig) { - *siglen = RSA_size(rsa); + *siglen = key_len; return 1; - } else if (*siglen < (size_t)RSA_size(rsa)) { + } + + if (*siglen < key_len) { OPENSSL_PUT_ERROR(EVP, pkey_rsa_sign, EVP_R_BUFFER_TOO_SMALL); return 0; } if (rctx->md) { + unsigned int out_len; + if (tbslen != EVP_MD_size(rctx->md)) { OPENSSL_PUT_ERROR(EVP, pkey_rsa_sign, EVP_R_INVALID_DIGEST_LENGTH); - return -1; + return 0; } if (EVP_MD_type(rctx->md) == NID_mdc2) { OPENSSL_PUT_ERROR(EVP, pkey_rsa_sign, EVP_R_NO_MDC2_SUPPORT); - ret = -1; - } else if (rctx->pad_mode == RSA_PKCS1_PADDING) { - unsigned int sltmp; - ret = RSA_sign(EVP_MD_type(rctx->md), tbs, tbslen, sig, &sltmp, rsa); - if (ret <= 0) { - return ret; - } - ret = sltmp; - } else if (rctx->pad_mode == RSA_PKCS1_PSS_PADDING) { - if (!setup_tbuf(rctx, ctx) || - !RSA_padding_add_PKCS1_PSS_mgf1(rsa, rctx->tbuf, tbs, rctx->md, - rctx->mgf1md, rctx->saltlen)) { - return -1; - } - /* TODO(fork): don't use private_encrypt. */ - ret = RSA_private_encrypt(RSA_size(rsa), rctx->tbuf, sig, rsa, - RSA_NO_PADDING); - } else { - return -1; + return 0; + } + + switch (rctx->pad_mode) { + case RSA_PKCS1_PADDING: + if (!RSA_sign(EVP_MD_type(rctx->md), tbs, tbslen, sig, &out_len, rsa)) { + return 0; + } + *siglen = out_len; + return 1; + + case RSA_PKCS1_PSS_PADDING: + if (!setup_tbuf(rctx, ctx) || + !RSA_padding_add_PKCS1_PSS_mgf1(rsa, rctx->tbuf, tbs, rctx->md, + rctx->mgf1md, rctx->saltlen) || + !RSA_encrypt(rsa, siglen, sig, *siglen, rctx->tbuf, key_len, + RSA_NO_PADDING)) { + return 0; + } + return 1; + + default: + return 0; } - } else { - /* TODO(fork): don't use private_encrypt. */ - ret = RSA_private_encrypt(tbslen, tbs, sig, ctx->pkey->pkey.rsa, - rctx->pad_mode); } - if (ret < 0) { - return ret; - } - *siglen = ret; - - return 1; + return RSA_sign_raw(rsa, siglen, sig, *siglen, tbs, tbslen, rctx->pad_mode); } static int pkey_rsa_verify(EVP_PKEY_CTX *ctx, const uint8_t *sig, @@ -228,42 +227,33 @@ static int pkey_rsa_verify(EVP_PKEY_CTX *ctx, const uint8_t *sig, RSA_PKEY_CTX *rctx = ctx->data; RSA *rsa = ctx->pkey->pkey.rsa; size_t rslen; + const size_t key_len = EVP_PKEY_size(ctx->pkey); if (rctx->md) { - if (rctx->pad_mode == RSA_PKCS1_PADDING) { - return RSA_verify(EVP_MD_type(rctx->md), tbs, tbslen, sig, siglen, rsa); - } + switch (rctx->pad_mode) { + case RSA_PKCS1_PADDING: + return RSA_verify(EVP_MD_type(rctx->md), tbs, tbslen, sig, siglen, rsa); - if (rctx->pad_mode == RSA_PKCS1_PSS_PADDING) { - int ret; - if (!setup_tbuf(rctx, ctx)) { - return -1; - } - /* TODO(fork): don't use public_decrypt. */ - ret = RSA_public_decrypt(siglen, sig, rctx->tbuf, rsa, RSA_NO_PADDING); - if (ret <= 0) { + case RSA_PKCS1_PSS_PADDING: + if (!setup_tbuf(rctx, ctx) || + !RSA_verify_raw(rsa, &rslen, rctx->tbuf, key_len, sig, siglen, + RSA_NO_PADDING) || + !RSA_verify_PKCS1_PSS_mgf1(rsa, tbs, rctx->md, rctx->mgf1md, + rctx->tbuf, rctx->saltlen)) { + return 0; + } + return 1; + + default: return 0; - } - ret = RSA_verify_PKCS1_PSS_mgf1(rsa, tbs, rctx->md, rctx->mgf1md, - rctx->tbuf, rctx->saltlen); - if (ret <= 0) { - return 0; - } - return 1; - } else { - return -1; - } - } else { - if (!setup_tbuf(rctx, ctx)) { - return -1; - } - rslen = RSA_public_decrypt(siglen, sig, rctx->tbuf, rsa, rctx->pad_mode); - if (rslen == 0) { - return 0; } } - if (rslen != tbslen || CRYPTO_memcmp(tbs, rctx->tbuf, rslen) != 0) { + if (!setup_tbuf(rctx, ctx) || + !RSA_verify_raw(rsa, &rslen, rctx->tbuf, key_len, sig, siglen, + rctx->pad_mode) || + rslen != tbslen || + CRYPTO_memcmp(tbs, rctx->tbuf, rslen) != 0) { return 0; } @@ -272,77 +262,73 @@ static int pkey_rsa_verify(EVP_PKEY_CTX *ctx, const uint8_t *sig, static int pkey_rsa_encrypt(EVP_PKEY_CTX *ctx, uint8_t *out, size_t *outlen, const uint8_t *in, size_t inlen) { - int ret; RSA_PKEY_CTX *rctx = ctx->data; RSA *rsa = ctx->pkey->pkey.rsa; + const size_t key_len = EVP_PKEY_size(ctx->pkey); if (!out) { - *outlen = RSA_size(rsa); + *outlen = key_len; return 1; - } else if (*outlen < (size_t)RSA_size(rsa)) { + } + + if (*outlen < key_len) { OPENSSL_PUT_ERROR(EVP, pkey_rsa_encrypt, EVP_R_BUFFER_TOO_SMALL); return 0; } if (rctx->pad_mode == RSA_PKCS1_OAEP_PADDING) { - int klen = RSA_size(rsa); - if (!setup_tbuf(rctx, ctx)) { - return -1; - } - if (!RSA_padding_add_PKCS1_OAEP_mgf1(rctx->tbuf, klen, in, inlen, + if (!setup_tbuf(rctx, ctx) || + !RSA_padding_add_PKCS1_OAEP_mgf1(rctx->tbuf, key_len, in, inlen, rctx->oaep_label, rctx->oaep_labellen, - rctx->md, rctx->mgf1md)) { - return -1; + rctx->md, rctx->mgf1md) || + !RSA_encrypt(rsa, outlen, out, *outlen, rctx->tbuf, key_len, + RSA_NO_PADDING)) { + return 0; } - ret = RSA_public_encrypt(klen, rctx->tbuf, out, rsa, RSA_NO_PADDING); - } else { - ret = RSA_public_encrypt(inlen, in, out, rsa, rctx->pad_mode); + return 1; } - if (ret < 0) { - return ret; - } - *outlen = ret; - - return 1; + return RSA_encrypt(rsa, outlen, out, *outlen, in, inlen, rctx->pad_mode); } static int pkey_rsa_decrypt(EVP_PKEY_CTX *ctx, uint8_t *out, size_t *outlen, const uint8_t *in, size_t inlen) { - int ret; RSA_PKEY_CTX *rctx = ctx->data; RSA *rsa = ctx->pkey->pkey.rsa; + const size_t key_len = EVP_PKEY_size(ctx->pkey); if (!out) { - *outlen = RSA_size(rsa); + *outlen = key_len; return 1; - } else if (*outlen < (size_t)RSA_size(rsa)) { + } + + if (*outlen < key_len) { OPENSSL_PUT_ERROR(EVP, pkey_rsa_decrypt, EVP_R_BUFFER_TOO_SMALL); return 0; } if (rctx->pad_mode == RSA_PKCS1_OAEP_PADDING) { - if (!setup_tbuf(rctx, ctx)) { - return -1; + size_t plaintext_len; + int message_len; + + if (!setup_tbuf(rctx, ctx) || + !RSA_decrypt(rsa, &plaintext_len, rctx->tbuf, key_len, in, inlen, + RSA_NO_PADDING)) { + return 0; } - ret = RSA_private_decrypt(inlen, in, rctx->tbuf, rsa, RSA_NO_PADDING); - if (ret <= 0) { - return ret; - } - ret = RSA_padding_check_PKCS1_OAEP_mgf1( - out, ret, rctx->tbuf, ret, rctx->oaep_label, + + message_len = RSA_padding_check_PKCS1_OAEP_mgf1( + out, key_len, rctx->tbuf, plaintext_len, rctx->oaep_label, rctx->oaep_labellen, rctx->md, rctx->mgf1md); - } else { - ret = RSA_private_decrypt(inlen, in, out, rsa, rctx->pad_mode); + if (message_len < 0) { + return 0; + } + *outlen = message_len; + return 1; } - if (ret < 0) { - return ret; - } - *outlen = ret; - - return 1; + return RSA_decrypt(rsa, outlen, out, key_len, in, inlen, rctx->pad_mode); } static int check_padding_md(const EVP_MD *md, int padding) { @@ -507,7 +493,6 @@ static int pkey_rsa_keygen(EVP_PKEY_CTX *ctx, EVP_PKEY *pkey) { RSA *rsa = NULL; RSA_PKEY_CTX *rctx = ctx->data; - int ret; if (!rctx->pub_exp) { rctx->pub_exp = BN_new(); if (!rctx->pub_exp || !BN_set_word(rctx->pub_exp, RSA_F4)) @@ -518,13 +503,13 @@ static int pkey_rsa_keygen(EVP_PKEY_CTX *ctx, EVP_PKEY *pkey) { return 0; } - ret = RSA_generate_key_ex(rsa, rctx->nbits, rctx->pub_exp, NULL); - if (ret > 0) { - EVP_PKEY_assign_RSA(pkey, rsa); - } else { + if (!RSA_generate_key_ex(rsa, rctx->nbits, rctx->pub_exp, NULL)) { RSA_free(rsa); + return 0; } - return ret; + + EVP_PKEY_assign_RSA(pkey, rsa); + return 1; } const EVP_PKEY_METHOD rsa_pkey_meth = { diff --git a/include/openssl/ecdsa.h b/include/openssl/ecdsa.h index 8a74ecb61..a2436f1b3 100644 --- a/include/openssl/ecdsa.h +++ b/include/openssl/ecdsa.h @@ -78,12 +78,8 @@ int ECDSA_sign(int type, const uint8_t *digest, size_t digest_len, uint8_t *sig, /* ECDSA_verify verifies that |sig_len| bytes from |sig| constitute a valid * signature by |key| of |digest|. (The |type| argument should be zero.) It - * returns one on success, zero if the signature is invalid and -1 on error. - * - * DANGER: this function differs from the usual OpenSSL return value - * convention. - * - * TODO(fork): change this to return the usual 0/1. */ + * returns one on success or zero if the signature is invalid or an error + * occured. */ int ECDSA_verify(int type, const uint8_t *digest, size_t digest_len, const uint8_t *sig, size_t sig_len, EC_KEY *key); @@ -116,12 +112,9 @@ ECDSA_SIG *ECDSA_do_sign(const uint8_t *digest, size_t digest_len, EC_KEY *key); /* ECDSA_verify verifies that |sig| constitutes a valid signature by |key| of - * |digest|. It returns one on success, zero if the signature is invalid and -1 + * |digest|. It returns one on success or zero if the signature is invalid or * on error. * - * DANGER: this function differs from the usual OpenSSL return value - * convention. - * * TODO(fork): remove this function. */ int ECDSA_do_verify(const uint8_t *digest, size_t digest_len, const ECDSA_SIG *sig, EC_KEY *key); diff --git a/include/openssl/evp.h b/include/openssl/evp.h index b522e8f57..d47d51fff 100644 --- a/include/openssl/evp.h +++ b/include/openssl/evp.h @@ -222,8 +222,7 @@ int i2d_PublicKey(EVP_PKEY *key, uint8_t **outp); * operation will be written to |*pctx|; this can be used to set alternative * signing options. * - * It returns one on success, or <= 0 on error. WARNING: this differs from the - * usual OpenSSL return convention. */ + * It returns one on success, or zero on error. */ int EVP_DigestSignInit(EVP_MD_CTX *ctx, EVP_PKEY_CTX **pctx, const EVP_MD *type, ENGINE *e, EVP_PKEY *pkey); @@ -239,8 +238,7 @@ int EVP_DigestSignUpdate(EVP_MD_CTX *ctx, const void *data, size_t len); * is successful, the signature is written to |out_sig| and |*out_sig_len| is * set to its length. * - * It returns one on success and <= 0 on error. WARNING: this differs from the - * usual, OpenSSL return value convention. */ + * It returns one on success, or zero on error. */ int EVP_DigestSignFinal(EVP_MD_CTX *ctx, uint8_t *out_sig, size_t *out_sig_len); @@ -252,8 +250,7 @@ int EVP_DigestSignFinal(EVP_MD_CTX *ctx, uint8_t *out_sig, size_t *out_sig_len); * operation will be written to |*pctx|; this can be used to set alternative * signing options. * - * It returns one on success, or <= 0 on error. WARNING: this differs from the - * usual OpenSSL return convention. */ + * It returns one on success, or zero on error. */ int EVP_DigestVerifyInit(EVP_MD_CTX *ctx, EVP_PKEY_CTX **pctx, const EVP_MD *type, ENGINE *e, EVP_PKEY *pkey); @@ -422,8 +419,7 @@ int EVP_PKEY_CTX_ctrl(EVP_PKEY_CTX *ctx, int keytype, int optype, int cmd, /* EVP_PKEY_sign_init initialises an |EVP_PKEY_CTX| for a signing operation. It * should be called before |EVP_PKEY_sign|. * - * It returns one on success or <= 0 on error. WARNING: this differs from the - * usual return value convention. */ + * It returns one on success or zero on error. */ int EVP_PKEY_sign_init(EVP_PKEY_CTX *ctx); /* EVP_PKEY_sign signs |data_len| bytes from |data| using |ctx|. If |sig| is @@ -443,23 +439,20 @@ int EVP_PKEY_sign(EVP_PKEY_CTX *ctx, uint8_t *sig, size_t *sig_len, /* EVP_PKEY_verify_init initialises an |EVP_PKEY_CTX| for a signature * verification operation. It should be called before |EVP_PKEY_verify|. * - * It returns one on success or <= 0 on error. WARNING: this differs from the - * usual return value convention. */ + * It returns one on success or zero on error. */ int EVP_PKEY_verify_init(EVP_PKEY_CTX *ctx); /* EVP_PKEY_verify verifies that |sig_len| bytes from |sig| are a valid signature * for |data|. * - * It returns one on success or zero on error. (Note: this differs from - * OpenSSL, which can also return negative values to indicate an error. ) */ + * It returns one on success or zero on error. */ int EVP_PKEY_verify(EVP_PKEY_CTX *ctx, const uint8_t *sig, size_t sig_len, const uint8_t *data, size_t data_len); /* EVP_PKEY_encrypt_init initialises an |EVP_PKEY_CTX| for an encryption * operation. It should be called before |EVP_PKEY_encrypt|. * - * It returns one on success or <= 0 on error. WARNING: this differs from the - * usual return value convention. */ + * It returns one on success or zero on error. */ int EVP_PKEY_encrypt_init(EVP_PKEY_CTX *ctx); /* EVP_PKEY_encrypt encrypts |in_len| bytes from |in|. If |out| is NULL, the @@ -471,16 +464,14 @@ int EVP_PKEY_encrypt_init(EVP_PKEY_CTX *ctx); * WARNING: Setting |out| to NULL only gives the maximum size of the * ciphertext. The actual ciphertext may be smaller. * - * It returns one on success or <= 0 on error. (Note: this differs from - * OpenSSL, which can also return negative values to indicate an error. ) */ + * It returns one on success or zero on error. */ int EVP_PKEY_encrypt(EVP_PKEY_CTX *ctx, uint8_t *out, size_t *out_len, const uint8_t *in, size_t in_len); /* EVP_PKEY_decrypt_init initialises an |EVP_PKEY_CTX| for a decryption * operation. It should be called before |EVP_PKEY_decrypt|. * - * It returns one on success or <= 0 on error. WARNING: this differs from the - * usual return value convention. */ + * It returns one on success or zero on error. */ int EVP_PKEY_decrypt_init(EVP_PKEY_CTX *ctx); /* EVP_PKEY_decrypt decrypts |in_len| bytes from |in|. If |out| is NULL, the @@ -492,8 +483,7 @@ int EVP_PKEY_decrypt_init(EVP_PKEY_CTX *ctx); * WARNING: Setting |out| to NULL only gives the maximum size of the * plaintext. The actual plaintext may be smaller. * - * It returns one on success or <= 0 on error. (Note: this differs from - * OpenSSL, which can also return negative values to indicate an error. ) */ + * It returns one on success or zero on error. */ int EVP_PKEY_decrypt(EVP_PKEY_CTX *ctx, uint8_t *out, size_t *out_len, const uint8_t *in, size_t in_len); @@ -501,15 +491,13 @@ int EVP_PKEY_decrypt(EVP_PKEY_CTX *ctx, uint8_t *out, size_t *out_len, * operation. It should be called before |EVP_PKEY_derive_set_peer| and * |EVP_PKEY_derive|. * - * It returns one on success or <= 0 on error. WARNING: this differs from the - * usual return value convention. */ + * It returns one on success or zero on error. */ int EVP_PKEY_derive_init(EVP_PKEY_CTX *ctx); /* EVP_PKEY_derive_set_peer sets the peer's key to be used for key derivation * by |ctx| to |peer|. It should be called after |EVP_PKEY_derive_init|. (For * example, this is used to set the peer's key in (EC)DH.) It returns one on - * success and <= 0 on error. WARNING: this differs from the usual return value - * convention. */ + * success and zero on error. */ int EVP_PKEY_derive_set_peer(EVP_PKEY_CTX *ctx, EVP_PKEY *peer); /* EVP_PKEY_derive derives a shared key between the two keys configured in @@ -521,21 +509,18 @@ int EVP_PKEY_derive_set_peer(EVP_PKEY_CTX *ctx, EVP_PKEY *peer); * WARNING: Setting |out| to NULL only gives the maximum size of the key. The * actual key may be smaller. * - * It returns one on success and <= 0 on error. WARNING: this differs from the - * usual return convention. */ + * It returns one on success and zero on error. */ int EVP_PKEY_derive(EVP_PKEY_CTX *ctx, uint8_t *key, size_t *out_key_len); /* EVP_PKEY_keygen_init initialises an |EVP_PKEY_CTX| for a key generation * operation. It should be called before |EVP_PKEY_keygen|. * - * It returns one on success or <= 0 on error. WARNING: this differs from the - * usual return value convention. */ + * It returns one on success or zero on error. */ int EVP_PKEY_keygen_init(EVP_PKEY_CTX *ctx); /* EVP_PKEY_keygen performs a key generation operation using the values from * |ctx| and sets |*ppkey| to a fresh |EVP_PKEY| containing the resulting key. - * It returns one on success or <= 0 on error. WARNING: this differs from the - * normal return value convention. */ + * It returns one on success or zero on error. */ int EVP_PKEY_keygen(EVP_PKEY_CTX *ctx, EVP_PKEY **ppkey); diff --git a/include/openssl/rsa.h b/include/openssl/rsa.h index 821bb1bab..9cc465d76 100644 --- a/include/openssl/rsa.h +++ b/include/openssl/rsa.h @@ -93,7 +93,9 @@ int RSA_up_ref(RSA *rsa); * for |e|. If |cb| is not NULL then it is called during the key generation * process. In addition to the calls documented for |BN_generate_prime_ex|, it * is called with event=2 when the n'th prime is rejected as unsuitable and - * with event=3 when a suitable value for |p| is found. */ + * with event=3 when a suitable value for |p| is found. + * + * It returns one on success or zero on error. */ int RSA_generate_key_ex(RSA *rsa, int bits, BIGNUM *e, BN_GENCB *cb);