Fix DTLS handshake retransmission edge cases

doc/api_ref/tls.rst

@@ -257,6 +257,13 @@ available:
      this connection and the connection has not been subsequently
      closed.
 
+   .. cpp:function:: std::optional<std::chrono::milliseconds> next_retransmission_timeout()
+
+      Returns the remaining time until a call to ``timeout_check`` might
+      retransmit DTLS handshake data. A zero duration means that a
+      retransmission is due. If ``std::nullopt`` is returned, no timeout check
+      is currently needed.
+
    .. cpp:function:: bool is_closed()
 
       Returns true if and only if either a close notification or a
@@ -277,10 +284,8 @@ available:
    .. cpp:function:: bool timeout_check()
 
       This function does nothing unless the channel represents a DTLS
-      connection and a handshake is actively in progress. In this case
-      it will check the current timeout state and potentially initiate
-      retransmission of handshake packets. Returns true if a timeout
-      condition occurred.
+      connection with a handshake in progress. Returns true if a timeout
+      condition occurred and handshake packets were retransmitted.
 
    .. cpp:function:: void renegotiate(bool force_full_renegotiation = false)
 

news.rst

@@ -102,6 +102,11 @@ Version 3.13.0, Not Yet Released
 
 * Upgrade to TLS-Anvil 1.5 (GH #5630)
 
+* Fix DTLS 1.2 handshake retransmission edge cases, including pacing of
+  repeated timeout checks, replay of final flights after local activation,
+  partial server-flight delivery, and delayed server-side flight handling.
+  (GH #2310 #2498 #4022 #4036)
+
 Version 3.12.0, 2026-05-06
 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
 

src/bogo_shim/bogo_shim.cpp

@@ -130,6 +130,7 @@ std::string map_to_bogo_error(const std::string& e) noexcept {
       {"Certificate usage constraints do not allow signing", ":KEY_USAGE_BIT_INCORRECT:"},
       {"Can't agree on a ciphersuite with client", ":NO_SHARED_CIPHER:"},
       {"Can't interleave application and handshake data", ":UNEXPECTED_RECORD:"},
+      {"Unexpected new DTLS handshake message", ":UNEXPECTED_RECORD:"},
       {"Certificate chain exceeds policy specified maximum size", ":EXCESSIVE_MESSAGE_SIZE:"},
       {"Certificate key type did not match ciphersuite", ":WRONG_CERTIFICATE_TYPE:"},
       {"Certificate usage constraints do not allow this ciphersuite", ":KEY_USAGE_BIT_INCORRECT:"},

src/bogo_shim/config.json

@@ -265,7 +265,6 @@
         "KeyChangeWithBufferedMessages-DTLS": "No DTLS 1.3",
         "Renegotiate-DTLS-Server-Forbidden": "Botan tolerates DTLS renegotiation",
         "Renegotiate-DTLS-Client-Forbidden": "Botan tolerates DTLS renegotiation",
-        "DTLS12-SendExtraFinished-*": "Botan tolerates extra Finished messages in DTLS",
         "MixCompleteMessageWithFragments-DTLS-TLS12": "DTLS fragment handling difference",
         "RejectPSSKeyType-*": "Botan does not reject RSA-PSS key type",
         "CertificateCipherMismatch-PSS": "Botan does not reject PSS cipher mismatch",

src/bogo_shim/config_no_tls13.json

@@ -274,7 +274,6 @@
         "KeyChangeWithBufferedMessages-DTLS": "No DTLS 1.3",
         "Renegotiate-DTLS-Server-Forbidden": "Botan tolerates DTLS renegotiation",
         "Renegotiate-DTLS-Client-Forbidden": "Botan tolerates DTLS renegotiation",
-        "DTLS12-SendExtraFinished-*": "Botan tolerates extra Finished messages in DTLS",
         "MixCompleteMessageWithFragments-DTLS-TLS12": "DTLS fragment handling difference",
         "RejectPSSKeyType-*": "Botan does not reject RSA-PSS key type",
         "CertificateCipherMismatch-PSS": "Botan does not reject PSS cipher mismatch",

src/lib/tls/tls12/tls_channel_impl_12.cpp

@@ -21,9 +21,25 @@
 #include <botan/internal/tls_handshake_state.h>
 #include <botan/internal/tls_record.h>
 #include <botan/internal/tls_seq_numbers.h>
+#include <utility>
 
 namespace Botan::TLS {
 
+namespace {
+
+bool is_new_dtls_association_client_hello(std::span<const uint8_t> msg_and_header, Record_Type record_type) {
+   constexpr size_t DTLS_HANDSHAKE_HEADER_SIZE = 12;
+
+   // Every new DTLS handshake starts at message_seq 0. A cookie-bearing
+   // ClientHello uses message_seq 1, so one arriving late belongs to the
+   // previous handshake and must not replace the active association's state.
+   return record_type == Record_Type::Handshake && msg_and_header.size() >= DTLS_HANDSHAKE_HEADER_SIZE &&
+          static_cast<Handshake_Type>(msg_and_header[0]) == Handshake_Type::ClientHello &&
+          load_be(msg_and_header.subspan<4, 2>()) == 0;
+}
+
+}  // namespace
+
 Channel_Impl_12::Channel_Impl_12(const std::shared_ptr<Callbacks>& callbacks,
                                  const std::shared_ptr<Session_Manager>& session_manager,
                                  const std::shared_ptr<RandomNumberGenerator>& rng,
@@ -171,7 +187,7 @@ Handshake_State& Channel_Impl_12::create_handshake_state(Protocol_Version versio
 }
 
 bool Channel_Impl_12::timeout_check() {
-   if(m_pending_state) {
+   if(m_is_datagram && !m_has_been_closed && m_pending_state) {
       return m_pending_state->handshake_io().timeout_check();
    }
 
@@ -259,6 +275,14 @@ bool Channel_Impl_12::is_active() const {
    return !is_closed() && is_handshake_complete();
 }
 
+std::optional<std::chrono::milliseconds> Channel_Impl_12::next_retransmission_timeout() const {
+   if(m_is_datagram && !m_has_been_closed && m_pending_state) {
+      return m_pending_state->handshake_io().next_retransmission_timeout();
+   }
+
+   return std::nullopt;
+}
+
 bool Channel_Impl_12::is_closed() const {
    return m_has_been_closed;
 }
@@ -278,9 +302,19 @@ void Channel_Impl_12::activate_session() {
       map_remove_if(not_current_epoch, m_read_cipher_states);
    }
 
-   // For DTLS, keep the handshake IO for last-flight retransmission.
+   // In a full handshake the server sends the terminal flight; in an
+   // abbreviated handshake the client does. Both endpoints retain handshake
+   // sequence state, but only the terminal sender replays its outgoing flight.
+   const bool sent_terminal_dtls_flight = m_is_datagram && (m_is_server == (state.server_hello_done() != nullptr));
+
    if(m_is_datagram) {
       m_active_state = Active_Connection_State_12(state, application_protocol(), m_pending_state->take_handshake_io());
+      if(auto* dtls_io = m_active_state->dtls_handshake_io()) {
+         // Retain receive sequence state on both endpoints to distinguish a
+         // retransmission from an unexpected new handshake message. Only the
+         // terminal-flight sender responds by replaying its final flight.
+         dtls_io->finalize_handshake(sent_terminal_dtls_flight);
+      }
    } else {
       m_active_state = Active_Connection_State_12(state, application_protocol());
    }
@@ -332,11 +366,26 @@ size_t Channel_Impl_12::from_peer(std::span<const uint8_t> data) {
             return 0;
          }
 
+         const bool old_unprotected_record = m_is_datagram && record.epoch() == 0 && m_active_state.has_value() &&
+                                             sequence_numbers().current_read_epoch() > 0;
+
+         // Once encrypted traffic is expected, epoch-zero records are
+         // unauthenticated. Only handshake records can be useful as part of a
+         // retransmitted flight or an explicitly allowed association restart.
+         if(old_unprotected_record && record.type() != Record_Type::Handshake &&
+            record.type() != Record_Type::ChangeCipherSpec) {
+            continue;
+         }
+
          if(m_record_buf.size() > MAX_PLAINTEXT_SIZE) {
+            if(old_unprotected_record) {
+               continue;
+            }
+
             throw TLS_Exception(Alert::RecordOverflow, "TLS plaintext record is larger than allowed maximum");
          }
 
-         const bool epoch0_restart = m_is_datagram && record.epoch() == 0 && m_active_state.has_value();
+         const bool epoch0_restart = allow_epoch0_restart && record.epoch() == 0 && m_active_state.has_value();
          BOTAN_ASSERT_IMPLICATION(epoch0_restart, allow_epoch0_restart, "Allowed state");
 
          const bool initial_record = epoch0_restart || (pending_state() == nullptr && !m_active_state.has_value());
@@ -346,7 +395,7 @@ size_t Channel_Impl_12::from_peer(std::span<const uint8_t> data) {
             initial_handshake_message = (type == Handshake_Type::ClientHello);
          }
 
-         if(record.type() != Record_Type::Alert) {
+         if(record.type() != Record_Type::Alert && !old_unprotected_record) {
             if(initial_record) {
                // For initial records just check for basic sanity
                if(record.version().major_version() != 3 && record.version().major_version() != 0xFE) {
@@ -406,26 +455,54 @@ void Channel_Impl_12::process_handshake_ccs(const secure_vector<uint8_t>& record
                                             Record_Type record_type,
                                             Protocol_Version record_version,
                                             bool epoch0_restart) {
+   const auto process_retransmitted_record = [&] {
+      BOTAN_ASSERT(m_active_state.has_value(), "Have active DTLS association for retransmission");
+      BOTAN_ASSERT_NONNULL(m_active_state->dtls_handshake_io());
+      try {
+         m_active_state->dtls_handshake_io()->add_retransmitted_record(
+            record.data(), record.size(), record_type, record_sequence);
+      } catch(...) {
+         // Epoch-zero records are unauthenticated and may be spoofed. Invalid
+         // retransmissions must not tear down an established association.
+         if((record_sequence >> 48) > 0) {
+            throw;
+         }
+      }
+   };
+
    if(!m_pending_state) {
-      // No pending handshake, possibly new:
-      if(record_version.is_datagram_protocol() && !epoch0_restart) {
-         if(m_sequence_numbers) {
-            /*
-            * Might be a peer retransmit under epoch - 1 in which
-            * case we must retransmit last flight
-            */
-            sequence_numbers().read_accept(record_sequence);
+      // With no pending handshake this is either a new handshake attempt or a
+      // DTLS retransmission from the previous handshake. The latter must not
+      // create fresh pending state; it only asks us to replay our last flight.
+      if(epoch0_restart && m_sequence_numbers && m_active_state.has_value()) {
+         const bool starts_new_handshake = is_new_dtls_association_client_hello(record, record_type);
+
+         if(!starts_new_handshake) {
+            process_retransmitted_record();
+            return;
+         }
+      }
 
+      if(m_is_datagram && !epoch0_restart) {
+         if(m_sequence_numbers) {
             const uint16_t epoch = record_sequence >> 48;
 
             const uint16_t current_epoch = sequence_numbers().current_read_epoch();
             if(epoch == current_epoch) {
-               create_handshake_state(record_version);
+               // Either endpoint can initiate renegotiation from FINISHED:
+               // clients send ClientHello, servers send HelloRequest.
+               const bool starts_new_handshake =
+                  (record_type == Record_Type::Handshake && !record.empty() &&
+                   (static_cast<Handshake_Type>(record[0]) == Handshake_Type::ClientHello ||
+                    static_cast<Handshake_Type>(record[0]) == Handshake_Type::HelloRequest));
+
+               if(m_active_state.has_value() && !starts_new_handshake) {
+                  process_retransmitted_record();
+               } else {
+                  create_handshake_state(record_version);
+               }
             } else if(current_epoch > 0 && epoch == current_epoch - 1) {
-               BOTAN_ASSERT(m_active_state.has_value() && m_active_state->dtls_handshake_io(),
-                            "Have DTLS handshake IO for retransmission");
-               m_active_state->dtls_handshake_io()->add_record(
-                  record.data(), record.size(), record_type, record_sequence);
+               process_retransmitted_record();
             }
          } else {
             create_handshake_state(record_version);

src/lib/tls/tls12/tls_channel_impl_12.h

@@ -27,6 +27,7 @@ namespace TLS {
 class Callbacks;
 class Connection_Cipher_State;
 class Connection_Sequence_Numbers;
+class Handshake_IO;
 class Handshake_State;
 class Handshake_Message;
 class Client_Hello_12;
@@ -87,6 +88,8 @@ class Channel_Impl_12 : public Channel_Impl {
       */
       bool is_active() const override;
 
+      std::optional<std::chrono::milliseconds> next_retransmission_timeout() const override;
+
       /**
       * @return true iff the connection has been definitely closed
       */
@@ -142,10 +145,8 @@ class Channel_Impl_12 : public Channel_Impl {
       bool secure_renegotiation_supported() const override;
 
       /**
-      * Perform a handshake timeout check. This does nothing unless
-      * this is a DTLS channel with a pending handshake state, in
-      * which case we check for timeout and potentially retransmit
-      * handshake packets.
+      * Perform a handshake timeout check. This does nothing unless this is a
+      * DTLS channel with a handshake in progress.
       */
       bool timeout_check() override;
 
@@ -158,7 +159,7 @@ class Channel_Impl_12 : public Channel_Impl {
                                          bool epoch0_restart) = 0;
 
       Handshake_State& create_handshake_state(Protocol_Version version);
-      virtual std::unique_ptr<Handshake_State> new_handshake_state(std::unique_ptr<class Handshake_IO> io) = 0;
+      virtual std::unique_ptr<Handshake_State> new_handshake_state(std::unique_ptr<Handshake_IO> io) = 0;
 
       void inspect_handshake_message(const Handshake_Message& msg);
 

src/lib/tls/tls12/tls_connection_state_12.h

@@ -39,7 +39,8 @@ class Active_Connection_State_12 final {
 
       Active_Connection_State_12(const Handshake_State& state, std::string application_protocol);
 
-      // DTLS variant: takes the handshake IO for replay of final flight
+      // DTLS variant: retains handshake IO for retransmission validation and
+      // reactive replay of the terminal flight when appropriate.
       Active_Connection_State_12(const Handshake_State& state,
                                  std::string application_protocol,
                                  std::unique_ptr<Handshake_IO> io);
@@ -75,12 +76,14 @@ class Active_Connection_State_12 final {
       bool supports_extended_master_secret() const { return m_supports_extended_master_secret; }
 
       /**
-       * For DTLS: the handshake IO from the completed handshake, needed
-       * to retransmit the last flight when records arrive under the
-       * previous epoch. Null for stream TLS.
+       * For DTLS: the handshake IO from the completed handshake, needed to
+       * validate retransmissions and, for the terminal-flight sender, replay
+       * the final flight. Null for stream TLS.
        */
       Datagram_Handshake_IO* dtls_handshake_io() { return m_dtls_handshake_io.get(); }
 
+      const Datagram_Handshake_IO* dtls_handshake_io() const { return m_dtls_handshake_io.get(); }
+
    private:
       Protocol_Version m_version;
       uint16_t m_ciphersuite_code = 0;