From a68176c61ee85164d80827f4b7b72f190dbb5746 Mon Sep 17 00:00:00 2001
From: Takashi Kojo <kojo@wolfssl.com>
Date: Thu, 9 Jul 2026 06:17:27 +0900
Subject: [PATCH 1/2] check if the RSA key is big enough to actually produce an
 RSA-PSS/SHA-256

---
 src/internal.c | 11 +++++++++--
 1 file changed, 9 insertions(+), 2 deletions(-)

diff --git a/src/internal.c b/src/internal.c
index 3765426a327..8cd36407f90 100644
--- a/src/internal.c
+++ b/src/internal.c
@@ -31062,6 +31062,8 @@ int SetSuitesHashSigAlgo(Suites* suites, const char* list)
 #endif /* OPENSSL_EXTRA */
 
 #if !defined(NO_TLS) && (!defined(NO_WOLFSSL_SERVER) || !defined(NO_CERTS))
+/* Smallest RSA key able to make an RSA-PSS/SHA-256 signature: emLen >= 2*hLen+2 = 66 bytes (RFC 8017 9.1.1, salt len = hash len). */
+#define RSA_PSS_SHA256_MIN_SZ  66   /* bytes: 2*WC_SHA256_DIGEST_SIZE + 2 */
 static int MatchSigAlgo(WOLFSSL* ssl, int sigAlgo)
 {
 #ifdef HAVE_ED25519
@@ -31124,9 +31126,14 @@ static int MatchSigAlgo(WOLFSSL* ssl, int sigAlgo)
         if (IsAtLeastTLSv1_3(ssl->version))
             return sigAlgo == rsa_pss_sa_algo;
     #endif
-        /* TLS 1.2 and below - RSA-PSS allowed. */
-        if (sigAlgo == rsa_pss_sa_algo)
+        /* TLS 1.2 and below - RSA-PSS allowed, but only if the RSA key is big
+         * enough to actually produce an RSA-PSS/SHA-256 signature. */
+        if (sigAlgo == rsa_pss_sa_algo) {
+            if (ssl->buffers.keySz != 0 &&
+                    ssl->buffers.keySz < RSA_PSS_SHA256_MIN_SZ)
+                return 0;
             return 1;
+        }
     }
 #endif
 #ifdef HAVE_ECC_BRAINPOOL

From 14d939b8011f473bd3c9fdb2d030a02edce41194 Mon Sep 17 00:00:00 2001
From: Takashi Kojo <kojo@wolfssl.com>
Date: Wed, 15 Jul 2026 06:53:13 +0900
Subject: [PATCH 2/2] Key-size check for RSA-PSS/SHA-256/384/512

---
 src/internal.c | 48 +++++++++++++++++++++++++++++++++++++++++-------
 1 file changed, 41 insertions(+), 7 deletions(-)

diff --git a/src/internal.c b/src/internal.c
index 8cd36407f90..0ea1b04f2b7 100644
--- a/src/internal.c
+++ b/src/internal.c
@@ -31062,10 +31062,42 @@ int SetSuitesHashSigAlgo(Suites* suites, const char* list)
 #endif /* OPENSSL_EXTRA */
 
 #if !defined(NO_TLS) && (!defined(NO_WOLFSSL_SERVER) || !defined(NO_CERTS))
-/* Smallest RSA key able to make an RSA-PSS/SHA-256 signature: emLen >= 2*hLen+2 = 66 bytes (RFC 8017 9.1.1, salt len = hash len). */
-#define RSA_PSS_SHA256_MIN_SZ  66   /* bytes: 2*WC_SHA256_DIGEST_SIZE + 2 */
-static int MatchSigAlgo(WOLFSSL* ssl, int sigAlgo)
+#ifdef WC_RSA_PSS
+/* Smallest RSA key able to make an RSA-PSS signature with the given hash:
+ * emLen >= 2*hLen + 2 (RFC 8017 9.1.1, salt length = hash length).
+ * Returns 0 when the hash isn't one used with RSA-PSS - no restriction. */
+static word32 MinRsaPssKeySz(int hashAlgo)
+{
+    word32 hLen;
+
+    switch (hashAlgo) {
+    #ifndef NO_SHA256
+        case sha256_mac:
+            hLen = WC_SHA256_DIGEST_SIZE;
+            break;
+    #endif
+    #ifdef WOLFSSL_SHA384
+        case sha384_mac:
+            hLen = WC_SHA384_DIGEST_SIZE;
+            break;
+    #endif
+    #ifdef WOLFSSL_SHA512
+        case sha512_mac:
+            hLen = WC_SHA512_DIGEST_SIZE;
+            break;
+    #endif
+        default:
+            return 0;
+    }
+
+    return 2 * hLen + 2;
+}
+#endif /* WC_RSA_PSS */
+
+static int MatchSigAlgo(WOLFSSL* ssl, int hashAlgo, int sigAlgo)
 {
+    (void)hashAlgo;
+
 #ifdef HAVE_ED25519
     if (ssl->pkCurveOID == ECC_ED25519_OID) {
         /* Certificate has Ed25519 key, only match with Ed25519 sig alg  */
@@ -31127,10 +31159,12 @@ static int MatchSigAlgo(WOLFSSL* ssl, int sigAlgo)
             return sigAlgo == rsa_pss_sa_algo;
     #endif
         /* TLS 1.2 and below - RSA-PSS allowed, but only if the RSA key is big
-         * enough to actually produce an RSA-PSS/SHA-256 signature. */
+         * enough to actually produce an RSA-PSS signature with this hash. */
         if (sigAlgo == rsa_pss_sa_algo) {
-            if (ssl->buffers.keySz != 0 &&
-                    ssl->buffers.keySz < RSA_PSS_SHA256_MIN_SZ)
+            word32 minSz = MinRsaPssKeySz(hashAlgo);
+
+            if (minSz != 0 && ssl->buffers.keySz != 0 &&
+                    (word32)ssl->buffers.keySz < minSz)
                 return 0;
             return 1;
         }
@@ -31280,7 +31314,7 @@ int PickHashSigAlgo(WOLFSSL* ssl, const byte* hashSigAlgo, word32 hashSigAlgoSz,
         if (hashAlgo < minHash)
             continue;
         /* Keep looking if signature algorithm isn't supported by cert. */
-        if (!MatchSigAlgo(ssl, sigAlgo))
+        if (!MatchSigAlgo(ssl, hashAlgo, sigAlgo))
             continue;
 
         if (matchSuites) {