tls: exclude PKCS#1 v1.5 RSA from TLS 1.3 signature_algorithms extension

src/internal.c

@@ -3611,6 +3611,15 @@ void InitSuitesHashSigAlgo(byte* hashSigAlgo, int haveSig, int tls1_2,
         #endif
         }
     #endif
+    /* PKCS#1 v1.5 RSA is not valid in TLS 1.3 (RFC 8446). In builds that
+     * support only TLS 1.3 (WOLFSSL_NO_TLS12), never add rsa_sa_algo.
+     * In mixed TLS 1.3 + TLS 1.2 builds, keep rsa_sa_algo in the list so
+     * that a ClientHello sent as TLS 1.3 can still negotiate TLS 1.2 after
+     * a server-initiated version downgrade. */
+#if defined(WOLFSSL_NO_TLS12)
+    if (!tls1_3)
+#endif
+    {
     #ifdef WOLFSSL_SHA512
         AddSuiteHashSigAlgo(hashSigAlgo, sha512_mac, rsa_sa_algo, keySz, &idx);
     #endif
@@ -3628,6 +3637,7 @@ void InitSuitesHashSigAlgo(byte* hashSigAlgo, int haveSig, int tls1_2,
         AddSuiteHashSigAlgo(hashSigAlgo, sha_mac, rsa_sa_algo, keySz, &idx);
     #endif
     }
+    }
 
 #ifdef HAVE_ANON
     if (haveSig & SIG_ANON) {

src/tls.c

@@ -7838,11 +7838,37 @@ static int TLSX_CA_Names_Parse(WOLFSSL *ssl, const byte* input,
 static word16 TLSX_SignatureAlgorithms_GetSize(void* data)
 {
     SignatureAlgorithms* sa = (SignatureAlgorithms*)data;
+    const byte* list;
+    word16      listSz;
 
-    if (sa->hashSigAlgoSz == 0)
-        return OPAQUE16_LEN + WOLFSSL_SUITES(sa->ssl)->hashSigAlgoSz;
-    else
-        return OPAQUE16_LEN + sa->hashSigAlgoSz;
+    if (sa->hashSigAlgoSz == 0) {
+        list   = WOLFSSL_SUITES(sa->ssl)->hashSigAlgo;
+        listSz = WOLFSSL_SUITES(sa->ssl)->hashSigAlgoSz;
+    }
+    else {
+        list   = sa->hashSigAlgo;
+        listSz = sa->hashSigAlgoSz;
+    }
+
+#if defined(WOLFSSL_TLS13) && !defined(NO_RSA)
+    /* PKCS#1 v1.5 RSA (rsa_sa_algo) is not a valid TLS 1.3 signature
+     * algorithm (RFC 8446 section 4.2.3). Filter it out when sending a
+     * TLS 1.3 ClientHello on a connection where downgrade to TLS 1.2 has
+     * been disabled, so the extension only advertises algorithms that are
+     * actually usable in the negotiated version. */
+    if (IsAtLeastTLSv1_3(sa->ssl->version) && !sa->ssl->options.downgrade) {
+        word16 sz = 0, i;
+        for (i = 0; i + 1 < listSz; i += 2) {
+            if (list[i + 1] != rsa_sa_algo)
+                sz += 2;
+        }
+        return OPAQUE16_LEN + sz;
+    }
+#else
+    (void)list;
+#endif
+
+    return OPAQUE16_LEN + listSz;
 }
 
 /* Creates a bit string of supported hash algorithms with RSA PSS.
@@ -7888,19 +7914,40 @@ static word16 TLSX_SignatureAlgorithms_Write(void* data, byte* output)
 {
     SignatureAlgorithms* sa = (SignatureAlgorithms*)data;
     const Suites* suites = WOLFSSL_SUITES(sa->ssl);
+    const byte* list;
+    word16 listSz;
     word16 hashSigAlgoSz;
 
     if (sa->hashSigAlgoSz == 0) {
-        c16toa(suites->hashSigAlgoSz, output);
-        XMEMCPY(output + OPAQUE16_LEN, suites->hashSigAlgo,
-                suites->hashSigAlgoSz);
-        hashSigAlgoSz = suites->hashSigAlgoSz;
+        list   = suites->hashSigAlgo;
+        listSz = suites->hashSigAlgoSz;
     }
     else {
-        c16toa(sa->hashSigAlgoSz, output);
-        XMEMCPY(output + OPAQUE16_LEN, sa->hashSigAlgo,
-                sa->hashSigAlgoSz);
-        hashSigAlgoSz = sa->hashSigAlgoSz;
+        list   = sa->hashSigAlgo;
+        listSz = sa->hashSigAlgoSz;
+    }
+
+#if defined(WOLFSSL_TLS13) && !defined(NO_RSA)
+    /* Filter PKCS#1 v1.5 RSA sig algs from a TLS 1.3 ClientHello when
+     * downgrade to TLS 1.2 is not allowed (matches GetSize filtering). */
+    if (IsAtLeastTLSv1_3(sa->ssl->version) && !sa->ssl->options.downgrade) {
+        word16 sz = 0, i;
+        for (i = 0; i + 1 < listSz; i += 2) {
+            if (list[i + 1] != rsa_sa_algo) {
+                output[OPAQUE16_LEN + sz]     = list[i];
+                output[OPAQUE16_LEN + sz + 1] = list[i + 1];
+                sz += 2;
+            }
+        }
+        c16toa(sz, output);
+        hashSigAlgoSz = sz;
+    }
+    else
+#endif
+    {
+        c16toa(listSz, output);
+        XMEMCPY(output + OPAQUE16_LEN, list, listSz);
+        hashSigAlgoSz = listSz;
     }
 
 #ifndef NO_RSA

src/tls13.c

@@ -5701,6 +5701,34 @@ int DoTls13ServerHello(WOLFSSL* ssl, const byte* input, word32* inOutIdx,
             ssl->version.minor = args->pv.minor;
             ssl->options.tls1_3 = 0;
 
+#if !defined(WOLFSSL_NO_TLS12) && !defined(NO_RSA)
+            /* After downgrading from TLS 1.3 to TLS 1.2, ensure PKCS#1
+             * v1.5 RSA signature algorithms are present in the hashSigAlgo
+             * list so that TLS 1.2 algorithm selection (PickHashSigAlgo,
+             * CertificateVerify) works correctly. They may be absent when
+             * the connection was initialised in TLS 1.3-only mode. */
+            {
+                Suites* ds = ssl->suites != NULL ? ssl->suites :
+                             (ssl->ctx != NULL ? ssl->ctx->suites : NULL);
+                if (ds != NULL) {
+                    word16 i;
+                    int hasRsaSa = 0;
+                    for (i = 0; i + 1 < ds->hashSigAlgoSz; i += 2) {
+                        if (ds->hashSigAlgo[i + 1] == rsa_sa_algo) {
+                            hasRsaSa = 1;
+                            break;
+                        }
+                    }
+                    if (!hasRsaSa) {
+                        /* Rebuild the list without TLS 1.3 restrictions so
+                         * PKCS#1 v1.5 RSA entries are included. */
+                        InitSuitesHashSigAlgo(ds->hashSigAlgo, SIG_ALL, 1, 0,
+                            ssl->buffers.keySz, &ds->hashSigAlgoSz);
+                    }
+                }
+            }
+#endif /* !WOLFSSL_NO_TLS12 && !NO_RSA */
+
 #ifdef WOLFSSL_DTLS13
             if (ssl->options.dtls) {
                 ret = Dtls13ClientDoDowngrade(ssl);