diff --git a/daemon/connect/Connection.cpp b/daemon/connect/Connection.cpp index 5ade5746..795dcc3e 100644 --- a/daemon/connect/Connection.cpp +++ b/daemon/connect/Connection.cpp @@ -3,7 +3,7 @@ * * Copyright (C) 2004 Sven Henkel * Copyright (C) 2007-2019 Andrey Prygunkov - * Copyright (C) 2024-2025 Denis + * Copyright (C) 2024-2026 Denis * * This program is free software; you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by @@ -172,27 +172,26 @@ bool Connection::Connect() else { DoDisconnect(); + m_status = csDisconnected; } return res; } -bool Connection::Disconnect() +void Connection::Disconnect() { debug("Disconnecting"); if (m_status == csDisconnected) { - return true; + return; } - bool res = DoDisconnect(); + DoDisconnect(); m_status = csDisconnected; m_socket = INVALID_SOCKET; m_bufAvail = 0; - - return res; } bool Connection::Bind() @@ -853,7 +852,7 @@ bool Connection::ConnectWithTimeout(void* address, int address_len) return true; } -bool Connection::DoDisconnect() +void Connection::DoDisconnect() { debug("Do disconnecting"); @@ -883,7 +882,6 @@ bool Connection::DoDisconnect() } m_status = csDisconnected; - return true; } void Connection::ReadBuffer(char** buffer, int *bufLen) diff --git a/daemon/connect/Connection.h b/daemon/connect/Connection.h index b6c91ee7..53e9a083 100644 --- a/daemon/connect/Connection.h +++ b/daemon/connect/Connection.h @@ -3,7 +3,7 @@ * * Copyright (C) 2004 Sven Henkel * Copyright (C) 2007-2017 Andrey Prygunkov - * Copyright (C) 2024-2025 Denis + * Copyright (C) 2024-2026 Denis * * This program is free software; you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by @@ -55,7 +55,7 @@ class Connection static void Init(); static void Final(); virtual bool Connect(); - virtual bool Disconnect(); + virtual void Disconnect(); bool Bind(); bool Send(const char* buffer, int size); bool Recv(char* buffer, int size); @@ -130,7 +130,7 @@ class Connection virtual void PrintError(const char* errMsg); int GetLastNetworkError(); bool DoConnect(); - bool DoDisconnect(); + void DoDisconnect(); bool InitSocketOpts(SOCKET socket); bool ConnectWithTimeout(void* address, int address_len); #ifndef HAVE_GETADDRINFO diff --git a/daemon/nntp/ArticleDownloader.cpp b/daemon/nntp/ArticleDownloader.cpp index f0c12220..f2460204 100644 --- a/daemon/nntp/ArticleDownloader.cpp +++ b/daemon/nntp/ArticleDownloader.cpp @@ -283,7 +283,7 @@ void ArticleDownloader::Run() } } - FreeConnection(status == adFinished); + FreeConnection(status == adFinished || (status == adRetry && m_connection && m_connection->GetStatus() == Connection::csConnected)); if (m_articleWriter.GetDuplicate()) { diff --git a/daemon/nntp/NewsServer.cpp b/daemon/nntp/NewsServer.cpp index 9fca8535..550904d8 100644 --- a/daemon/nntp/NewsServer.cpp +++ b/daemon/nntp/NewsServer.cpp @@ -3,6 +3,7 @@ * * Copyright (C) 2004 Sven Henkel * Copyright (C) 2007-2016 Andrey Prygunkov + * Copyright (C) 2026 Denis * * This program is free software; you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by @@ -35,3 +36,36 @@ NewsServer::NewsServer(int id, bool active, const char* name, const char* host, m_name.Format("server%i", id); } } + +void NewsServer::DelayConnect() +{ + constexpr std::chrono::milliseconds CONNECTION_DELAY{15}; + using Clock = std::chrono::steady_clock; + + Clock::time_point deadline{Clock::time_point::max()}; + + { + std::lock_guard lock(m_connectMutex); + + auto now = Clock::now(); + if (m_lastConnectTime == Clock::time_point::min()) + { + m_lastConnectTime = now; + return; + } + + if (now >= m_lastConnectTime + CONNECTION_DELAY) + { + m_lastConnectTime = now; + return; + } + + m_lastConnectTime += CONNECTION_DELAY; + deadline = m_lastConnectTime; + } + + if (deadline != Clock::time_point::max()) + { + std::this_thread::sleep_until(deadline); + } +} diff --git a/daemon/nntp/NewsServer.h b/daemon/nntp/NewsServer.h index b82bdf5e..84318201 100644 --- a/daemon/nntp/NewsServer.h +++ b/daemon/nntp/NewsServer.h @@ -3,7 +3,7 @@ * * Copyright (C) 2004 Sven Henkel * Copyright (C) 2007-2015 Andrey Prygunkov - * Copyright (C) 2025 Denis + * Copyright (C) 2025-2026 Denis * * This program is free software; you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by @@ -28,6 +28,8 @@ #ifndef NEWSSERVER_H #define NEWSSERVER_H +#include +#include #include "NString.h" class NewsServer @@ -62,6 +64,14 @@ class NewsServer void SetBlockTime(time_t blockTime) { m_blockTime = blockTime; } unsigned int GetCertVerificationLevel() const { return m_certVerificationfLevel; } + /** + * @brief Delays the connection attempt to enforce a minimum interval of 15ms between connections. + * + * This rate limiter prevents "502 Too Many Connections" and "481 Exceeded Maximum Connections" + * errors on Usenet servers by spacing out rapid connection attempts. + */ + void DelayConnect(); + private: int m_id; int m_stateId = 0; @@ -83,6 +93,8 @@ class NewsServer bool m_optional = false; time_t m_blockTime = 0; unsigned int m_certVerificationfLevel; + std::chrono::steady_clock::time_point m_lastConnectTime{std::chrono::steady_clock::time_point::min()}; + std::mutex m_connectMutex; }; typedef std::vector> Servers; diff --git a/daemon/nntp/NntpConnection.cpp b/daemon/nntp/NntpConnection.cpp index 05e64637..5a0e9951 100644 --- a/daemon/nntp/NntpConnection.cpp +++ b/daemon/nntp/NntpConnection.cpp @@ -3,6 +3,7 @@ * * Copyright (C) 2004 Sven Henkel * Copyright (C) 2007-2016 Andrey Prygunkov + * Copyright (C) 2024-2026 Denis * * This program is free software; you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by @@ -195,6 +196,11 @@ bool NntpConnection::Connect() return true; } + if (m_newsServer) + { + m_newsServer->DelayConnect(); + } + if (!Connection::Connect()) { return false; @@ -227,14 +233,14 @@ bool NntpConnection::Connect() return true; } -bool NntpConnection::Disconnect() +void NntpConnection::Disconnect() { if (m_status == csConnected) { Request("quit\r\n"); m_activeGroup = nullptr; } - return Connection::Disconnect(); + Connection::Disconnect(); } void NntpConnection::ReportErrorAnswer(const char* msgPrefix, const char* answer) diff --git a/daemon/nntp/NntpConnection.h b/daemon/nntp/NntpConnection.h index 74cc74b1..a457b6ea 100644 --- a/daemon/nntp/NntpConnection.h +++ b/daemon/nntp/NntpConnection.h @@ -3,6 +3,7 @@ * * Copyright (C) 2004 Sven Henkel * Copyright (C) 2007-2016 Andrey Prygunkov + * Copyright (C) 2026 Denis * * This program is free software; you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by @@ -30,8 +31,8 @@ class NntpConnection : public Connection { public: NntpConnection(NewsServer* newsServer); - virtual bool Connect(); - virtual bool Disconnect(); + bool Connect() override; + void Disconnect() override; NewsServer* GetNewsServer() { return m_newsServer; } const char* Request(const char* req); const char* JoinGroup(const char* grp); diff --git a/daemon/nntp/ServerPool.cpp b/daemon/nntp/ServerPool.cpp index ae504022..233bb9df 100644 --- a/daemon/nntp/ServerPool.cpp +++ b/daemon/nntp/ServerPool.cpp @@ -3,7 +3,7 @@ * * Copyright (C) 2004 Sven Henkel * Copyright (C) 2007-2016 Andrey Prygunkov - * Copyright (C) 2024 Denis + * Copyright (C) 2024-2026 Denis * * This program is free software; you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by @@ -23,8 +23,9 @@ #include "nzbget.h" #include "ServerPool.h" #include "Util.h" +#include "QueueCoordinator.h" -static const int CONNECTION_HOLD_SECODNS = 5; +static constexpr int CONNECTION_HOLD_SECONDS = 30; void ServerPool::PooledConnection::SetFreeTimeNow() { @@ -113,7 +114,7 @@ void ServerPool::InitConnections() { debug("Initializing connections in ServerPool"); - Guard guard(m_connectionsMutex); + std::lock_guard guard(m_connectionsMutex); NormalizeLevels(); m_levels.clear(); @@ -162,7 +163,7 @@ void ServerPool::InitConnections() */ NntpConnection* ServerPool::GetConnection(int level, NewsServer* wantServer, RawServerList* ignoreServers) { - Guard guard(m_connectionsMutex); + std::lock_guard guard(m_connectionsMutex); for (; level < (int)m_levels.size() && m_levels[level] > 0; level++) { @@ -256,7 +257,7 @@ void ServerPool::FreeConnection(NntpConnection* connection, bool used) debug("Freeing used connection"); } - Guard guard(m_connectionsMutex); + std::lock_guard guard(m_connectionsMutex); ((PooledConnection*)connection)->SetInUse(false); if (used) @@ -274,7 +275,7 @@ void ServerPool::BlockServer(NewsServer* newsServer) { bool newBlock = false; { - Guard guard(m_connectionsMutex); + std::lock_guard guard(m_connectionsMutex); time_t curTime = Util::CurrentTime(); newBlock = newsServer->GetBlockTime() != curTime; newsServer->SetBlockTime(curTime); @@ -301,7 +302,7 @@ bool ServerPool::IsServerBlocked(NewsServer* newsServer) void ServerPool::CloseUnusedConnections() { - Guard guard(m_connectionsMutex); + std::lock_guard guard(m_connectionsMutex); time_t curtime = Util::CurrentTime(); @@ -350,9 +351,11 @@ void ServerPool::CloseUnusedConnections() } } - // if there are no in-use connections on the level and the hold time out has - // expired - close all connections of the level. - if (!hasInUseConnections && inactiveTime > CONNECTION_HOLD_SECODNS) + // If there are no in-use connections on the level and the hold timeout has expired, + // close all connections of that level. For the primary level (level 0), we keep + // connections alive as long as the queue has more jobs + if (!hasInUseConnections && inactiveTime > CONNECTION_HOLD_SECONDS && + (level > 0 || !g_QueueCoordinator || !g_QueueCoordinator->HasMoreJobs())) { for (PooledConnection* connection : &m_connections) { @@ -381,7 +384,7 @@ void ServerPool::LogDebugInfo() info(" Max-Level: %i", m_maxNormLevel); - Guard guard(m_connectionsMutex); + std::lock_guard guard(m_connectionsMutex); time_t curTime = Util::CurrentTime(); diff --git a/daemon/nntp/ServerPool.h b/daemon/nntp/ServerPool.h index a3c0dbbe..54ed4cde 100644 --- a/daemon/nntp/ServerPool.h +++ b/daemon/nntp/ServerPool.h @@ -3,7 +3,7 @@ * * Copyright (C) 2004 Sven Henkel * Copyright (C) 2007-2016 Andrey Prygunkov - * Copyright (C) 2024 Denis + * Copyright (C) 2024-2026 Denis * * This program is free software; you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by @@ -24,12 +24,10 @@ #define SERVERPOOL_H #include "Log.h" -#include "Container.h" -#include "Thread.h" #include "NewsServer.h" #include "NntpConnection.h" -class ServerPool : public Debuggable +class ServerPool final : public Debuggable { public: typedef std::vector RawServerList; @@ -50,10 +48,10 @@ class ServerPool : public Debuggable bool IsServerBlocked(NewsServer* newsServer); protected: - virtual void LogDebugInfo(); + void LogDebugInfo() override; private: - class PooledConnection : public NntpConnection + class PooledConnection final : public NntpConnection { public: using NntpConnection::NntpConnection; @@ -62,8 +60,8 @@ class ServerPool : public Debuggable time_t GetFreeTime() { return m_freeTime; } void SetFreeTimeNow(); private: - bool m_inUse = false; time_t m_freeTime = 0; + bool m_inUse = false; }; typedef std::vector Levels; @@ -73,8 +71,8 @@ class ServerPool : public Debuggable RawServerList m_sortedServers; Connections m_connections; Levels m_levels; + std::mutex m_connectionsMutex; int m_maxNormLevel = 0; - Mutex m_connectionsMutex; int m_timeout = 60; int m_retryInterval = 0; int m_generation = 0; diff --git a/tests/nntp/ServerPool.cpp b/tests/nntp/ServerPool.cpp index 6ddeda76..c00a75e5 100644 --- a/tests/nntp/ServerPool.cpp +++ b/tests/nntp/ServerPool.cpp @@ -327,4 +327,20 @@ BOOST_AUTO_TEST_CASE(VerifyNormLevelCorrectness) if (con) pool.FreeConnection(con, false); } +BOOST_AUTO_TEST_CASE(NewsServerDelayConnectTest) +{ + NewsServer server(1, true, "test", "localhost", 119, 0, "", "", false, false, nullptr, 1, 0, 0, 0, false, Options::cvStrict); + + auto start = std::chrono::steady_clock::now(); + server.DelayConnect(); + auto duration1 = std::chrono::duration_cast(std::chrono::steady_clock::now() - start).count(); + + start = std::chrono::steady_clock::now(); + server.DelayConnect(); + auto duration2 = std::chrono::duration_cast(std::chrono::steady_clock::now() - start).count(); + + BOOST_CHECK(duration1 < 50); + BOOST_CHECK(duration2 >= 5); +} + BOOST_AUTO_TEST_SUITE_END()