Skip to content

Fix RSA PSS saltlen handling #308

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
padelsbach merged 3 commits into from
Oct 17, 2025
Merged

Fix RSA PSS saltlen handling #308

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
476cd36
Fix RSA PSS saltlen handling
ColtonWilley Oct 14, 2025
56c841f
Only use RSA_PSS_SALTLEN_AUTO_DIGEST_MAX if defined, dont run tests f…
ColtonWilley Oct 14, 2025
3481baf
Fix auto saltlen when WOLFSSL_PSS_SALT_LEN_DISCOVER is not defined
ColtonWilley Oct 14, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
8 changes: 6 additions & 2 deletions src/wp_rsa_kmgmt.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -415,8 +415,12 @@ static int wp_rsagen_check_key_size(wp_RsaGenCtx* rsagen)
*/
void wp_rsa_get_pss_mds(wp_Rsa* rsa, char** mdName, char** mgfMdName)
{
*mdName = rsa->pssParams.mdName;
*mgfMdName = rsa->pssParams.mgfMdName;
if (mdName != NULL) {
*mdName = rsa->pssParams.mdName;
}
if (mgfMdName != NULL) {
*mgfMdName = rsa->pssParams.mgfMdName;
}
}

/**
Expand Down
30 changes: 28 additions & 2 deletions src/wp_rsa_sig.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -118,14 +118,24 @@ static int wp_rsa_setup_md(wp_RsaSigCtx* ctx, const char* mdName,
const char* mdProps, int op)
{
int ok = 1;
char *localMdName = NULL;

WOLFPROV_ENTER(WP_LOG_RSA, "wp_rsa_setup_md");

if (mdProps == NULL) {
mdProps = ctx->propQuery;
}

if (mdName != NULL) {
/* For PSS restricted mode, not allowed to set new MD */
if (ctx->padMode == RSA_PKCS1_PSS_PADDING && ctx->minSaltLen != -1) {
wp_rsa_get_pss_mds(ctx->rsa, &localMdName, NULL);
if (mdName != NULL &&
XSTRNCASECMP(localMdName, mdName, XSTRLEN(localMdName)) != 0) {
ok = 0;
}
}

if (ok && mdName != NULL) {
int rc;
enum wc_HashType hashType;

Expand Down Expand Up @@ -381,7 +391,10 @@ static int wp_pss_salt_len_to_wc(int saltLen, enum wc_HashType hashType,
}
else if (saltLen == RSA_PSS_SALTLEN_AUTO) {
#ifndef WOLFSSL_PSS_SALT_LEN_DISCOVER
saltLen = wc_HashGetDigestSize(hashType);
saltLen = wc_RsaEncryptSize(key) - wc_HashGetDigestSize(hashType) - 2;
Comment thread
padelsbach marked this conversation as resolved.
if (((mp_count_bits(&key->n) - 1) & 0x7) == 0) {
saltLen--;
}
#else
saltLen = RSA_PSS_SALT_LEN_DISCOVER;
#endif
Expand Down Expand Up @@ -462,6 +475,7 @@ static int wp_rsa_signverify_init(wp_RsaSigCtx* ctx, wp_Rsa* rsa,
const OSSL_PARAM params[], int op)
{
int ok = 1;
int saltLen = 0;

WOLFPROV_ENTER(WP_LOG_RSA, "wp_rsa_signverify_init");

Expand Down Expand Up @@ -498,6 +512,13 @@ static int wp_rsa_signverify_init(wp_RsaSigCtx* ctx, wp_Rsa* rsa,
ctx->saltLen = WP_RSA_DEFAULT_SALT_LEN;
#endif
ctx->minSaltLen = 0;

/* If we have already set PSS salt len, use that */
saltLen = wp_rsa_get_pss_salt_len(ctx->rsa);
if (saltLen != 0) {
ctx->saltLen = saltLen;
ctx->minSaltLen = saltLen;
}
}
else {
char* mdName;
Expand Down Expand Up @@ -2142,8 +2163,13 @@ static int wp_rsa_set_salt_len(wp_RsaSigCtx* ctx, const OSSL_PARAM* p)
else {
ok = 0;
}
#ifdef RSA_PSS_SALTLEN_AUTO_DIGEST_MAX
/* RSA_PSS_SALTLEN_AUTO_DIGEST_MAX is the smallest negative value supported. */
if (ok && (ctx->saltLen < RSA_PSS_SALTLEN_AUTO_DIGEST_MAX)) {
#else
/* RSA_PSS_SALTLEN_MAX is the smallest negative value supported. */
if (ok && (ctx->saltLen < RSA_PSS_SALTLEN_MAX)) {
#endif
ok = 0;
}
if (ok && (ctx->saltLen >= 0) && (ctx->saltLen < ctx->minSaltLen)) {
Expand Down
90 changes: 45 additions & 45 deletions test/test_ecc.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1253,28 +1253,28 @@ int test_ecdsa_p192(void *data)
if (err == 0) {
PRINT_MSG("Sign with OpenSSL");
ecdsaSigLen = sizeof(ecdsaSig);
err = test_digest_sign(pkey, osslLibCtx, buf, sizeof(buf), md,
ecdsaSig, &ecdsaSigLen, 0);
err = test_digest_sign(pkey, osslLibCtx, buf, sizeof(buf), md, NULL,
ecdsaSig, &ecdsaSigLen, 0, 0);
}
if (err == 0) {
PRINT_MSG("Verify with wolfprovider");
err = test_digest_verify(pkey, wpLibCtx, buf, sizeof(buf), md,
ecdsaSig, ecdsaSigLen, 0);
err = test_digest_verify(pkey, wpLibCtx, buf, sizeof(buf), md, NULL,
ecdsaSig, ecdsaSigLen, 0, 0);
}
if (err == 0) {
PRINT_MSG("Verify bad signature with wolfprovider");
ecdsaSig[1] ^= 0x80;
res = test_digest_verify(pkey, wpLibCtx, buf, sizeof(buf), md,
ecdsaSig, ecdsaSigLen, 0);
res = test_digest_verify(pkey, wpLibCtx, buf, sizeof(buf), md, NULL,
ecdsaSig, ecdsaSigLen, 0, 0);
if (res != 1)
err = 1;
}

if (err == 0) {
PRINT_MSG("Sign with wolfprovider");
ecdsaSigLen = sizeof(ecdsaSig);
err = test_digest_sign(pkey, wpLibCtx, buf, sizeof(buf), md,
ecdsaSig, &ecdsaSigLen, 0);
err = test_digest_sign(pkey, wpLibCtx, buf, sizeof(buf), md, NULL,
ecdsaSig, &ecdsaSigLen, 0, 0);
#if defined(HAVE_FIPS) || defined(HAVE_FIPS_VERSION)
err = err != 1;
if (err == 0) {
Expand All @@ -1289,8 +1289,8 @@ int test_ecdsa_p192(void *data)
}
if (err == 0) {
PRINT_MSG("Verify with OpenSSL");
err = test_digest_verify(pkey, osslLibCtx, buf, sizeof(buf), md,
ecdsaSig, ecdsaSigLen, 0);
err = test_digest_verify(pkey, osslLibCtx, buf, sizeof(buf), md, NULL,
ecdsaSig, ecdsaSigLen, 0, 0);
}
#endif /* HAVE_FIPS || HAVE_FIPS_VERSION */

Expand Down Expand Up @@ -1326,32 +1326,32 @@ int test_ecdsa_p224(void *data)
if (err == 0) {
PRINT_MSG("Sign with OpenSSL");
ecdsaSigLen = sizeof(ecdsaSig);
err = test_digest_sign(pkey, osslLibCtx, buf, sizeof(buf), md,
ecdsaSig, &ecdsaSigLen, 0);
err = test_digest_sign(pkey, osslLibCtx, buf, sizeof(buf), md, NULL,
ecdsaSig, &ecdsaSigLen, 0, 0);
}
if (err == 0) {
PRINT_MSG("Verify with wolfprovider");
err = test_digest_verify(pkey, wpLibCtx, buf, sizeof(buf), md,
ecdsaSig, ecdsaSigLen, 0);
err = test_digest_verify(pkey, wpLibCtx, buf, sizeof(buf), md, NULL,
ecdsaSig, ecdsaSigLen, 0, 0);
}
if (err == 0) {
PRINT_MSG("Verify bad signature with wolfprovider");
ecdsaSig[1] ^= 0x80;
res = test_digest_verify(pkey, wpLibCtx, buf, sizeof(buf), md,
ecdsaSig, ecdsaSigLen, 0);
res = test_digest_verify(pkey, wpLibCtx, buf, sizeof(buf), md, NULL,
ecdsaSig, ecdsaSigLen, 0, 0);
if (res != 1)
err = 1;
}
if (err == 0) {
PRINT_MSG("Sign with wolfprovider");
ecdsaSigLen = sizeof(ecdsaSig);
err = test_digest_sign(pkey, wpLibCtx, buf, sizeof(buf), md,
ecdsaSig, &ecdsaSigLen, 0);
err = test_digest_sign(pkey, wpLibCtx, buf, sizeof(buf), md, NULL,
ecdsaSig, &ecdsaSigLen, 0, 0);
}
if (err == 0) {
PRINT_MSG("Verify with OpenSSL");
err = test_digest_verify(pkey, osslLibCtx, buf, sizeof(buf), md,
ecdsaSig, ecdsaSigLen, 0);
err = test_digest_verify(pkey, osslLibCtx, buf, sizeof(buf), md, NULL,
ecdsaSig, ecdsaSigLen, 0, 0);
}

EVP_PKEY_free(pkey);
Expand Down Expand Up @@ -1382,31 +1382,31 @@ int test_ecdsa_p256(void *data)
PRINT_MSG("Sign with OpenSSL");
ecdsaSigLen = sizeof(ecdsaSig);
err = test_digest_sign(pkey, osslLibCtx, buf, sizeof(buf), "SHA-256",
ecdsaSig, &ecdsaSigLen, 0);
NULL, ecdsaSig, &ecdsaSigLen, 0, 0);
}
if (err == 0) {
PRINT_MSG("Verify with wolfprovider");
err = test_digest_verify(pkey, wpLibCtx, buf, sizeof(buf), "SHA-256",
ecdsaSig, ecdsaSigLen, 0);
NULL, ecdsaSig, ecdsaSigLen, 0, 0);
}
if (err == 0) {
PRINT_MSG("Verify bad signature with wolfprovider");
ecdsaSig[1] ^= 0x80;
res = test_digest_verify(pkey, wpLibCtx, buf, sizeof(buf), "SHA-256",
ecdsaSig, ecdsaSigLen, 0);
NULL, ecdsaSig, ecdsaSigLen, 0, 0);
if (res != 1)
err = 1;
}
if (err == 0) {
PRINT_MSG("Sign with wolfprovider");
ecdsaSigLen = sizeof(ecdsaSig);
err = test_digest_sign(pkey, wpLibCtx, buf, sizeof(buf), "SHA-256",
ecdsaSig, &ecdsaSigLen, 0);
NULL, ecdsaSig, &ecdsaSigLen, 0, 0);
}
if (err == 0) {
PRINT_MSG("Verify with OpenSSL");
err = test_digest_verify(pkey, osslLibCtx, buf, sizeof(buf), "SHA-256",
ecdsaSig, ecdsaSigLen, 0);
NULL, ecdsaSig, ecdsaSigLen, 0, 0);
}

EVP_PKEY_free(pkey);
Expand Down Expand Up @@ -1441,32 +1441,32 @@ int test_ecdsa_p384(void *data)
if (err == 0) {
PRINT_MSG("Sign with OpenSSL");
ecdsaSigLen = sizeof(ecdsaSig);
err = test_digest_sign(pkey, osslLibCtx, buf, sizeof(buf), md,
ecdsaSig, &ecdsaSigLen, 0);
err = test_digest_sign(pkey, osslLibCtx, buf, sizeof(buf), md, NULL,
ecdsaSig, &ecdsaSigLen, 0, 0);
}
if (err == 0) {
PRINT_MSG("Verify with wolfprovider");
err = test_digest_verify(pkey, wpLibCtx, buf, sizeof(buf), md,
ecdsaSig, ecdsaSigLen, 0);
err = test_digest_verify(pkey, wpLibCtx, buf, sizeof(buf), md, NULL,
ecdsaSig, ecdsaSigLen, 0, 0);
}
if (err == 0) {
PRINT_MSG("Verify bad signature with wolfprovider");
ecdsaSig[1] ^= 0x80;
res = test_digest_verify(pkey, wpLibCtx, buf, sizeof(buf), md,
ecdsaSig, ecdsaSigLen, 0);
res = test_digest_verify(pkey, wpLibCtx, buf, sizeof(buf), md, NULL,
ecdsaSig, ecdsaSigLen, 0, 0);
if (res != 1)
err = 1;
}
if (err == 0) {
PRINT_MSG("Sign with wolfprovider");
ecdsaSigLen = sizeof(ecdsaSig);
err = test_digest_sign(pkey, wpLibCtx, buf, sizeof(buf), md,
ecdsaSig, &ecdsaSigLen, 0);
err = test_digest_sign(pkey, wpLibCtx, buf, sizeof(buf), md, NULL,
ecdsaSig, &ecdsaSigLen, 0, 0);
}
if (err == 0) {
PRINT_MSG("Verify with OpenSSL");
err = test_digest_verify(pkey, osslLibCtx, buf, sizeof(buf), md,
ecdsaSig, ecdsaSigLen, 0);
err = test_digest_verify(pkey, osslLibCtx, buf, sizeof(buf), md, NULL,
ecdsaSig, ecdsaSigLen, 0, 0);
}

EVP_PKEY_free(pkey);
Expand Down Expand Up @@ -1501,32 +1501,32 @@ int test_ecdsa_p521(void *data)
if (err == 0) {
PRINT_MSG("Sign with OpenSSL");
ecdsaSigLen = sizeof(ecdsaSig);
err = test_digest_sign(pkey, osslLibCtx, buf, sizeof(buf), md,
ecdsaSig, &ecdsaSigLen, 0);
err = test_digest_sign(pkey, osslLibCtx, buf, sizeof(buf), md, NULL,
ecdsaSig, &ecdsaSigLen, 0, 0);
}
if (err == 0) {
PRINT_MSG("Verify with wolfprovider");
err = test_digest_verify(pkey, wpLibCtx, buf, sizeof(buf), md,
ecdsaSig, ecdsaSigLen, 0);
err = test_digest_verify(pkey, wpLibCtx, buf, sizeof(buf), md, NULL,
ecdsaSig, ecdsaSigLen, 0, 0);
}
if (err == 0) {
PRINT_MSG("Verify bad signature with wolfprovider");
ecdsaSig[1] ^= 0x80;
res = test_digest_verify(pkey, wpLibCtx, buf, sizeof(buf), md,
ecdsaSig, ecdsaSigLen, 0);
res = test_digest_verify(pkey, wpLibCtx, buf, sizeof(buf), md, NULL,
ecdsaSig, ecdsaSigLen, 0, 0);
if (res != 1)
err = 1;
}
if (err == 0) {
PRINT_MSG("Sign with wolfprovider");
ecdsaSigLen = sizeof(ecdsaSig);
err = test_digest_sign(pkey, wpLibCtx, buf, sizeof(buf), md,
ecdsaSig, &ecdsaSigLen, 0);
err = test_digest_sign(pkey, wpLibCtx, buf, sizeof(buf), md, NULL,
ecdsaSig, &ecdsaSigLen, 0, 0);
}
if (err == 0) {
PRINT_MSG("Verify with OpenSSL");
err = test_digest_verify(pkey, osslLibCtx, buf, sizeof(buf), md,
ecdsaSig, ecdsaSigLen, 0);
err = test_digest_verify(pkey, osslLibCtx, buf, sizeof(buf), md, NULL,
ecdsaSig, ecdsaSigLen, 0, 0);
}

EVP_PKEY_free(pkey);
Expand Down
32 changes: 17 additions & 15 deletions test/test_ecx.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -107,29 +107,29 @@ static int sign_verify(unsigned char* sig, size_t sigLen,

if (err == 0) {
PRINT_MSG("Sign with OpenSSL (%s)", name);
err = test_digest_sign(pkey, osslLibCtx, buf, bufLen, NULL,
sig, &sigLen, 0);
err = test_digest_sign(pkey, osslLibCtx, buf, bufLen, NULL, NULL, sig,
&sigLen, 0, 0);
}
if (err == 0) {
PRINT_MSG("Verify with WolfProvider (%s)", name);
err = test_digest_verify(pkey, wpLibCtx, buf, bufLen, NULL,
sig, sigLen, 0);
err = test_digest_verify(pkey, wpLibCtx, buf, bufLen, NULL, NULL, sig,
sigLen, 0, 0);
}
if (err == 0) {
PRINT_MSG("Verify bad signature with WolfProvider (%s)", name);
sig[1] ^= 0x80;
err = test_digest_verify(pkey, wpLibCtx, buf, bufLen, NULL,
sig, sigLen, 0) != 1;
err = test_digest_verify(pkey, wpLibCtx, buf, bufLen, NULL, NULL, sig,
sigLen, 0, 0) != 1;
}
if (err == 0) {
PRINT_MSG("Sign with WolfProvider (%s)", name);
err = test_digest_sign(pkey, wpLibCtx, buf, bufLen, NULL,
sig, &sigLen, 0);
err = test_digest_sign(pkey, wpLibCtx, buf, bufLen, NULL, NULL, sig,
&sigLen, 0, 0);
}
if (err == 0) {
PRINT_MSG("Verify with OpenSSL (%s)", name);
err = test_digest_verify(pkey, osslLibCtx, buf, bufLen, NULL,
sig, sigLen, 0);
err = test_digest_verify(pkey, osslLibCtx, buf, bufLen, NULL, NULL, sig,
sigLen, 0, 0);
}

return err;
Expand Down Expand Up @@ -409,7 +409,7 @@ int test_ecx_sign_verify_raw_pub(void *data)
if (err == 0) {
PRINT_MSG("Sign with OpenSSL (%s)", types[i].name);
err = test_digest_sign(pkey_der, osslLibCtx, buf, bufLen, NULL,
types[i].sig, &types[i].sigLen, 0);
NULL, types[i].sig, &types[i].sigLen, 0, 0);
}

/* Create keys from the public byte arrays */
Expand Down Expand Up @@ -489,25 +489,27 @@ int test_ecx_sign_verify_raw_pub(void *data)
if (err == 0) {
PRINT_MSG("Verify with OpenSSL (%s)", types[i].name);
err = test_digest_verify(pkey_ossl, osslLibCtx, buf, bufLen, NULL,
types[i].sig, types[i].sigLen, 0);
NULL, types[i].sig, types[i].sigLen, 0, 0);
}
if (err == 0) {
PRINT_MSG("Verify with WolfProvider (%s)", types[i].name);
err = test_digest_verify(pkey_wolf, wpLibCtx, buf, bufLen, NULL,
types[i].sig, types[i].sigLen, 0);
NULL, types[i].sig, types[i].sigLen, 0, 0);
}

/* Verify bad signature with the public keys */
types[i].sig[1] ^= 0x80;
if (err == 0) {
PRINT_MSG("Verify bad signature with OpenSSL (%s)", types[i].name);
err = test_digest_verify(pkey_ossl, osslLibCtx, buf, bufLen, NULL,
types[i].sig, types[i].sigLen, 0) != 1;
NULL, types[i].sig, types[i].sigLen,
0, 0) != 1;
}
if (err == 0) {
PRINT_MSG("Verify bad signature with WolfProvider (%s)", types[i].name);
err = test_digest_verify(pkey_wolf, wpLibCtx, buf, bufLen, NULL,
types[i].sig, types[i].sigLen, 0) != 1;
NULL, types[i].sig, types[i].sigLen,
0, 0) != 1;
}

EVP_PKEY_free(pkey_der);
Expand Down
Loading
Loading