From 1fb8b82230f704e50a72780030ca7b60458cb51b Mon Sep 17 00:00:00 2001 From: Noah Oblath Date: Thu, 21 May 2026 16:38:26 -0700 Subject: [PATCH 01/45] Added build of rmqcpp (along with all dependencies) and removed librabbitmq --- Dockerfile | 33 ++++++++++++++++++++++++++++++--- 1 file changed, 30 insertions(+), 3 deletions(-) diff --git a/Dockerfile b/Dockerfile index 6e27a384..95f250ce 100644 --- a/Dockerfile +++ b/Dockerfile @@ -9,8 +9,10 @@ ARG build_examples=FALSE ARG enable_testing=FALSE ARG narg=2 -# Most dependencies +ENV VCPKG_FORCE_SYSTEM_BINARIES=1 +ENV VCPKG_ROOT=/usr/local/vcpkg +# Most dependencies RUN apt-get update && \ apt-get clean && \ apt-get --fix-missing -y install \ @@ -21,9 +23,17 @@ RUN apt-get update && \ libboost-chrono-dev \ libboost-filesystem-dev \ libboost-system-dev \ - librabbitmq-dev \ libyaml-cpp-dev \ - rapidjson-dev && \ + rapidjson-dev \ + libzstd-dev \ + libssl-dev \ + zlib1g-dev \ + ninja-build \ + pkg-config \ + curl \ + tar \ + unzip \ + zip && \ # pybind11-dev \ # wget && \ rm -rf /var/lib/apt/lists/* @@ -43,6 +53,23 @@ RUN cd /usr/local && \ cd / && \ rm -rf /usr/local/${pybind11_name} +ARG rmqcpp_checkout= +RUN cd /usr/local && \ + git clone https://github.com/Microsoft/vcpkg.git && \ + /usr/local/vcpkg/bootstrap-vcpkg.sh && \ + git clone https://github.com/bloomberg/rmqcpp.git && \ + cd /usr/local/rmqcpp && \ + git checkout ${rmqcpp_checkout} && \ + ${VCPKG_ROOT}/vcpkg install --triplet arm64-linux && \ + mkdir build && \ + cd build && \ + cmake -DCMAKE_BUILD_TYPE=Release \ + -DCMAKE_TOOLCHAIN_FILE=${VCPKG_ROOT}/scripts/buildsystems/vcpkg.cmake \ + -DCMAKE_INSTALL_PREFIX=/usr/local \ + -DBUILD_TESTING=OFF \ + .. && \ + make -j${narg} install + FROM base AS devel RUN apt-get update && \ From 719158426f13567816efad91fd1910b4acd94fc7 Mon Sep 17 00:00:00 2001 From: Noah Oblath Date: Wed, 27 May 2026 16:36:24 -0700 Subject: [PATCH 02/45] Removing SimpleAmqpClient and starting to use rmqcpp instead --- .gitmodules | 3 -- CMakeLists.txt | 57 +++--------------------- DriplineConfig.cmake.in | 2 +- external/CMakeLists.txt | 54 ---------------------- external/SimpleAmqpClient | 1 - external/SimpleAmqpClientConfig.cmake.in | 16 ------- 6 files changed, 6 insertions(+), 127 deletions(-) delete mode 100644 external/CMakeLists.txt delete mode 160000 external/SimpleAmqpClient delete mode 100644 external/SimpleAmqpClientConfig.cmake.in diff --git a/.gitmodules b/.gitmodules index c1f3202a..1ca03f96 100644 --- a/.gitmodules +++ b/.gitmodules @@ -1,6 +1,3 @@ [submodule "scarab"] path = scarab url = https://github.com/project8/scarab -[submodule "external/SimpleAmqpClient"] - path = external/SimpleAmqpClient - url = https://github.com/project8/SimpleAmqpClient diff --git a/CMakeLists.txt b/CMakeLists.txt index d1f23a61..61c3ad8b 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -75,57 +75,14 @@ if( Dripline_ENABLE_TESTING ) endif() -#################### -# SimpleAmqpClient # -#################### - -# SimpleAmqpClient needs to be built independently from, but as a submodule and dependency of, dripline-cpp. -# In other words, the configuration and build environments need to be kept separate. +########## +# rmqcpp # +########## -# First we run the configure stage of SimpleAmqpClient during the configure (cmake) process of dripline-cpp. -# This build takes place in its own build directory, but it installs in the same place as dripline-cpp. -# We do this here instead of below in the ExternalProject so that dripline-cpp can learn about the SimpleAmqpClient targets at the configure stage. - -if( NOT EXISTS ${PROJECT_BINARY_DIR}/external ) - execute_process( - COMMAND mkdir external - WORKING_DIRECTORY ${PROJECT_BINARY_DIR} - ) -endif() - -execute_process( - COMMAND cmake -D CMAKE_INSTALL_PREFIX:STRING=${CMAKE_INSTALL_PREFIX} - -D PACKAGE_CONFIG_PREFIX:STRING=${PACKAGE_CONFIG_PREFIX} - -D TOP_PROJECT_CMAKE_CONFIG_DIR:STRING=${TOP_PROJECT_CMAKE_CONFIG_DIR} - -D LIB_INSTALL_DIR:STRING=${LIB_INSTALL_DIR} - -D BIN_INSTALL_DIR:STRING=${BIN_INSTALL_DIR} - -D CMAKE_BUILD_TYPE=RELEASE - -D CMAKE_CXX_STANDARD=${CMAKE_CXX_STANDARD} - ${PROJECT_SOURCE_DIR}/external - WORKING_DIRECTORY ${PROJECT_BINARY_DIR}/external -) - -# Second we use the ExternalProject module to build and install SimpleAmqpClient. -# This takes place during the build phase of dripline-cpp. -# Note that we disable the configure stage of the ExternalProject because we already took care of the configure stage above. -include(ExternalProject) - -ExternalProject_Add( DriplineExternal - PREFIX ${CMAKE_INSTALL_PREFIX} - SOURCE_DIR ${PROJECT_SOURCE_DIR}/external - BINARY_DIR ${PROJECT_BINARY_DIR}/external - CONFIGURE_COMMAND "" - BUILD_ALWAYS 1 - BUILD_COMMAND $(MAKE) - INSTALL_DIR ${CMAKE_INSTALL_PREFIX} - INSTALL_COMMAND $(MAKE) install -) - -# Finally we use find_package() to learn about the Kassiopeia build targets. -find_package( SimpleAmqpClient REQUIRED CONFIG HINTS ${PROJECT_BINARY_DIR}/external ) +find_package( rmqcpp REQUIRED ) # Added to the public libraries so that downstream code can successfully get the necessary include directories -list( APPEND PUBLIC_EXT_LIBS SimpleAmqpClient::SimpleAmqpClient ) +list( APPEND PUBLIC_EXT_LIBS rmqcpp::rmq ) ################## @@ -143,10 +100,6 @@ else( Dripline_BUILD_PYTHON ) set_option( Scarab_BUILD_PYTHON FALSE ) endif( Dripline_BUILD_PYTHON ) -# SimpleAmqpClient needs system and chrono -list( APPEND Scarab_BOOST_COMPONENTS chrono ) - - ##################### # Prepare for Build # ##################### diff --git a/DriplineConfig.cmake.in b/DriplineConfig.cmake.in index 4defc0f3..14ddc0bd 100644 --- a/DriplineConfig.cmake.in +++ b/DriplineConfig.cmake.in @@ -6,7 +6,7 @@ get_filename_component( Dripline_CMAKE_DIR "${CMAKE_CURRENT_LIST_FILE}" PATH ) include( CMakeFindDependencyMacro ) find_dependency( Scarab REQUIRED PATHS ${Dripline_CMAKE_DIR}/Scarab @Scarab_BINARY_LOCATION@ ) -find_dependency( SimpleAmqpClient REQUIRED PATHS ${Dripline_CMAKE_DIR}/SimpleAmqpClient ${Dripline_CMAKE_DIR}/external ) +find_dependency( rmqcpp REQUIRED ) if( NOT TARGET Dripline::@Dripline_FULL_TARGET_NAME@ ) if( TARGET @Dripline_FULL_TARGET_NAME@ ) diff --git a/external/CMakeLists.txt b/external/CMakeLists.txt deleted file mode 100644 index b3c96a89..00000000 --- a/external/CMakeLists.txt +++ /dev/null @@ -1,54 +0,0 @@ -# This is a wrapper project used to build any external dependencies that require wrapping. -# -# SimpleAmqpClient requires wrapping so that modern CMake practices (use of targets and interfaces) can be applied. - -# Minimum cmake verison 3.1 required by dripline-cpp -cmake_minimum_required (VERSION 3.12) - -cmake_policy( SET CMP0048 NEW ) # version in project() -project( DriplineExternal VERSION 0.0.0 ) - -################## -# SimpleAmqpClient -################## - -option( BUILD_API_DOCS "" OFF ) -add_subdirectory( SimpleAmqpClient ) - -# Because the SimpleAmqpClient CMake build doesn't take care of this stuff, we have to do it ourselves - -# Set interface include directories -set_property( TARGET SimpleAmqpClient - APPEND PROPERTY INTERFACE_INCLUDE_DIRECTORIES - "$" - "$/include>" -) - -# Build-interface targets -export( TARGETS SimpleAmqpClient - NAMESPACE SimpleAmqpClient:: - FILE ${CMAKE_CURRENT_BINARY_DIR}/SimpleAmqpClientTargets.cmake -) - -# Install-interface targets -# The SimpleAmqpClientTargets export was added to SimpleAmqpClient in the project8 fork -# Install SimpleAmqpClientTargets.cmake in the Dripline package-config install directory -install( EXPORT SimpleAmqpClientTargets - FILE - SimpleAmqpClientTargets.cmake - NAMESPACE - SimpleAmqpClient:: - DESTINATION - ${PACKAGE_CONFIG_PREFIX}/SimpleAmqpClient -) - -# Process SimpleAmqpClient.cmake.in -configure_file( ${CMAKE_CURRENT_SOURCE_DIR}/SimpleAmqpClientConfig.cmake.in ${CMAKE_CURRENT_BINARY_DIR}/SimpleAmqpClientConfig.cmake @ONLY ) - -install( - FILES - ${CMAKE_CURRENT_BINARY_DIR}/SimpleAmqpClientConfig.cmake - DESTINATION - ${TOP_PROJECT_CMAKE_CONFIG_DIR}/SimpleAmqpClient -) - diff --git a/external/SimpleAmqpClient b/external/SimpleAmqpClient deleted file mode 160000 index 84ac3883..00000000 --- a/external/SimpleAmqpClient +++ /dev/null @@ -1 +0,0 @@ -Subproject commit 84ac388383dc82b8fad542317fdc106f7412aca3 diff --git a/external/SimpleAmqpClientConfig.cmake.in b/external/SimpleAmqpClientConfig.cmake.in deleted file mode 100644 index b7a0aa90..00000000 --- a/external/SimpleAmqpClientConfig.cmake.in +++ /dev/null @@ -1,16 +0,0 @@ -# SimpleAmqpClientConfig.cmake - -get_filename_component( SimpleAmqpClient_CMAKE_DIR "${CMAKE_CURRENT_LIST_FILE}" PATH ) - -# Find the dependencies -include( CMakeFindDependencyMacro ) -find_dependency( Boost 1.46 REQUIRED COMPONENTS chrono ) - -# Import library targets if it's not already present (usable at build time or as installed) -if( NOT TARGET SimpleAmqpClient::SimpleAmqpClient ) - if( TARGET SimpleAmqpClient ) - add_library( SimpleAmqpClient::SimpleAmqpClient ALIAS SimpleAmqpClient ) - else() - include( "${SimpleAmqpClient_CMAKE_DIR}/SimpleAmqpClientTargets.cmake" ) - endif() -endif() From 64fa2befdb117cd27a2aaf98386d6c864333ed75 Mon Sep 17 00:00:00 2001 From: Noah Oblath Date: Wed, 27 May 2026 16:36:49 -0700 Subject: [PATCH 03/45] Adding the dependencies for rmqcpp and rmqcpp itself to the Dockerfile --- Dockerfile | 45 +++++++++++++++++++++++++++++++++++++-------- 1 file changed, 37 insertions(+), 8 deletions(-) diff --git a/Dockerfile b/Dockerfile index 95f250ce..2b90d598 100644 --- a/Dockerfile +++ b/Dockerfile @@ -20,14 +20,11 @@ RUN apt-get update && \ cmake \ # gdb \ git \ - libboost-chrono-dev \ - libboost-filesystem-dev \ - libboost-system-dev \ libyaml-cpp-dev \ rapidjson-dev \ - libzstd-dev \ libssl-dev \ zlib1g-dev \ + libzstd-dev \ ninja-build \ pkg-config \ curl \ @@ -43,7 +40,7 @@ ARG pybind11_checkout=v3.0.1 ARG pybind11_repo=https://github.com/pybind/pybind11.git ARG pybind11_name=pybind11 RUN cd /usr/local && \ - git clone ${pybind11_repo} && \ + git clone ${pybind11_repo} ${pybind11_name} && \ cd ${pybind11_name} && \ git checkout ${pybind11_checkout} && \ mkdir build && \ @@ -53,6 +50,7 @@ RUN cd /usr/local && \ cd / && \ rm -rf /usr/local/${pybind11_name} +ARG TARGETARCH ARG rmqcpp_checkout= RUN cd /usr/local && \ git clone https://github.com/Microsoft/vcpkg.git && \ @@ -60,15 +58,46 @@ RUN cd /usr/local && \ git clone https://github.com/bloomberg/rmqcpp.git && \ cd /usr/local/rmqcpp && \ git checkout ${rmqcpp_checkout} && \ - ${VCPKG_ROOT}/vcpkg install --triplet arm64-linux && \ + case "${TARGETARCH}" in \ + amd64) TRIPLET="x64-linux-release" ;; \ + arm64) TRIPLET="arm64-linux-release" ;; \ + *) echo "Unsupported architecture: ${TARGETARCH}"; exit 1 ;; \ + esac && \ + ${VCPKG_ROOT}/vcpkg install --triplet ${TRIPLET} && \ mkdir build && \ cd build && \ cmake -DCMAKE_BUILD_TYPE=Release \ -DCMAKE_TOOLCHAIN_FILE=${VCPKG_ROOT}/scripts/buildsystems/vcpkg.cmake \ + -DVCPKG_TARGET_TRIPLET=${TRIPLET} \ -DCMAKE_INSTALL_PREFIX=/usr/local \ -DBUILD_TESTING=OFF \ .. && \ - make -j${narg} install + make -j${narg} install && \ + cd / && \ + ${VCPKG_ROOT}/vcpkg install --triplet ${TRIPLET} boost-filesystem boost-system boost-chrono boost-variant && \ + cp -a /usr/local/rmqcpp/vcpkg_installed/${TRIPLET}/include/. /usr/local/include/ && \ + cp -a /usr/local/rmqcpp/vcpkg_installed/${TRIPLET}/lib/. /usr/local/lib/ && \ + cp -a /usr/local/rmqcpp/vcpkg_installed/${TRIPLET}/share/. /usr/local/share/ && \ + cp -a ${VCPKG_ROOT}/installed/${TRIPLET}/include/. /usr/local/include/ && \ + cp -a ${VCPKG_ROOT}/installed/${TRIPLET}/lib/. /usr/local/lib/ && \ + cp -a ${VCPKG_ROOT}/installed/${TRIPLET}/share/. /usr/local/share/ && \ + rm -rf /usr/local/share/zstd && \ + # Create a shim file to adapt the vcpkg install of "pcre2" to the expected standard "libpcre2-8" + mkdir -p /usr/local/share/libpcre2-8 && \ + printf '%s\n' \ + 'include("/usr/local/share/pcre2/pcre2-config.cmake")' \ + 'if(NOT TARGET libpcre2-8::pcre2-8 AND TARGET pcre2::pcre2-8-static)' \ + ' add_library(libpcre2-8::pcre2-8 ALIAS pcre2::pcre2-8-static)' \ + 'endif()' \ + 'set(libpcre2-8_FOUND TRUE)' \ + > /usr/local/share/libpcre2-8/libpcre2-8Config.cmake && \ + # rmqcpp's rmqcppConfig.cmake is missing the use of find_dependency(zstd), so we add it + RMQCPP_CONFIG=$(find /usr/local -name rmqcppConfig.cmake 2>/dev/null | head -1) && \ + test -n "${RMQCPP_CONFIG}" && \ + if ! grep -q 'find_dependency(zstd' "${RMQCPP_CONFIG}"; then \ + sed -i '/include.*rmqcppTargets/i find_dependency(zstd CONFIG)' "${RMQCPP_CONFIG}"; \ + fi + #rm -rf /usr/local/rmqcpp /usr/local/vcpkg /root/.cache/vcpkg FROM base AS devel @@ -101,7 +130,7 @@ RUN mkdir -p /usr/local/build && \ cmake ../src && \ # unclear why I have to run cmake twice cmake -DCMAKE_BUILD_TYPE=${build_type} \ - -DCMAKE_INSTALL_PREFIX:PATH=/usr/local \ + -DCMAKE_INSTALL_PREFIX:PATH=/usr/local \ -DDripline_BUILD_EXAMPLES:BOOL=${build_examples} \ -DDripline_ENABLE_TESTING:BOOL=${enable_testing} \ -DDripline_BUILD_PYTHON:BOOL=TRUE \ From 27f7a667b6abb5f735b45d6121cc7da7a5ca9956 Mon Sep 17 00:00:00 2001 From: Noah Oblath Date: Wed, 27 May 2026 17:30:29 -0700 Subject: [PATCH 04/45] Added missing boost modules --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 2b90d598..68ac9f33 100644 --- a/Dockerfile +++ b/Dockerfile @@ -74,7 +74,7 @@ RUN cd /usr/local && \ .. && \ make -j${narg} install && \ cd / && \ - ${VCPKG_ROOT}/vcpkg install --triplet ${TRIPLET} boost-filesystem boost-system boost-chrono boost-variant && \ + ${VCPKG_ROOT}/vcpkg install --triplet ${TRIPLET} boost-filesystem boost-system boost-chrono boost-variant boost-uuid && \ cp -a /usr/local/rmqcpp/vcpkg_installed/${TRIPLET}/include/. /usr/local/include/ && \ cp -a /usr/local/rmqcpp/vcpkg_installed/${TRIPLET}/lib/. /usr/local/lib/ && \ cp -a /usr/local/rmqcpp/vcpkg_installed/${TRIPLET}/share/. /usr/local/share/ && \ From da9267a0a27d0a1dd523a4fd6e0b4a7104d7ae7f Mon Sep 17 00:00:00 2001 From: Noah Oblath Date: Wed, 27 May 2026 17:31:09 -0700 Subject: [PATCH 05/45] Switched dl-cpp type aliases and conversion functions from SimpleAmqpClient types to rmqcpp types. --- library/amqp.cc | 151 +++++++++++++++++++++----------------- library/amqp.hh | 32 ++++---- testing/test_amqp.cc | 169 +++++++++++++++++++++++++++++++------------ 3 files changed, 220 insertions(+), 132 deletions(-) diff --git a/library/amqp.cc b/library/amqp.cc index 643f487e..570a466b 100644 --- a/library/amqp.cc +++ b/library/amqp.cc @@ -9,20 +9,25 @@ #include "amqp.hh" +#include "bdlde_base64encoder.h" +#include "bdlt_iso8601util.h" + +using namespace BloombergLP; + namespace dripline { - DRIPLINE_API scarab::param_ptr_t table_to_param( const AmqpClient::Table& a_table ) + DRIPLINE_API scarab::param_ptr_t table_to_param( const rmqt::FieldTable& a_table ) { scarab::param_ptr_t t_param( new scarab::param_node() ); scarab::param_node& t_node = (*t_param).as_node(); for( auto i_entry = a_table.begin(); i_entry != a_table.end(); ++i_entry ) { - t_node.add( i_entry->first, table_to_param( i_entry->second ) ); + t_node.add( std::string(i_entry->first), table_to_param( i_entry->second ) ); } return t_param; } - DRIPLINE_API scarab::param_ptr_t table_to_param( const AmqpClient::Array& a_array ) + DRIPLINE_API scarab::param_ptr_t table_to_param( const rmqt::FieldArray& a_array ) { scarab::param_ptr_t t_param( new scarab::param_array() ); scarab::param_array& t_array = (*t_param).as_array(); @@ -33,94 +38,104 @@ namespace dripline return t_param; } - DRIPLINE_API scarab::param_ptr_t table_to_param( const AmqpClient::TableValue& a_value ) + DRIPLINE_API scarab::param_ptr_t table_to_param( const rmqt::FieldValue& a_value ) { - using AmqpClient::TableValue; - - switch( a_value.GetType() ) + if( a_value.isUnset() ) + return scarab::param_ptr_t( new scarab::param() ); + if( a_value.is< bool >() ) + return scarab::param_ptr_t( new scarab::param_value( a_value.the< bool >() ) ); + if( a_value.is< int8_t >() ) + return scarab::param_ptr_t( new scarab::param_value( (int64_t)a_value.the< int8_t >() ) ); + if( a_value.is< int16_t >() ) + return scarab::param_ptr_t( new scarab::param_value( (int64_t)a_value.the< int16_t >() ) ); + if( a_value.is< int32_t >() ) + return scarab::param_ptr_t( new scarab::param_value( (int64_t)a_value.the< int32_t >() ) ); + if( a_value.is< int64_t >() ) + return scarab::param_ptr_t( new scarab::param_value( a_value.the< int64_t >() ) ); + if( a_value.is< uint8_t >() ) + return scarab::param_ptr_t( new scarab::param_value( (uint64_t)a_value.the< uint8_t >() ) ); + if( a_value.is< uint16_t >() ) + return scarab::param_ptr_t( new scarab::param_value( (uint64_t)a_value.the< uint16_t >() ) ); + if( a_value.is< uint32_t >() ) + return scarab::param_ptr_t( new scarab::param_value( (uint64_t)a_value.the< uint32_t >() ) ); + if( a_value.is< float >() ) + return scarab::param_ptr_t( new scarab::param_value( (double)a_value.the< float >() ) ); + if( a_value.is< double >() ) + return scarab::param_ptr_t( new scarab::param_value( a_value.the< double >() ) ); + if( a_value.is< bsl::string >() ) + return scarab::param_ptr_t( new scarab::param_value( std::string( a_value.the< bsl::string >() ) ) ); + if( a_value.is< bsl::shared_ptr< rmqt::FieldArray > >() ) + { + const auto& t_array_ptr = a_value.the< bsl::shared_ptr< rmqt::FieldArray > >(); + if( ! t_array_ptr ) throw std::domain_error( "Null FieldArray pointer in rmqcpp FieldValue" ); + return table_to_param( *t_array_ptr ); + } + if( a_value.is< bsl::shared_ptr< rmqt::FieldTable > >() ) + { + const auto& t_table_ptr = a_value.the< bsl::shared_ptr< rmqt::FieldTable > >(); + if( ! t_table_ptr ) throw std::domain_error( "Null FieldTable pointer in rmqcpp FieldValue" ); + return table_to_param( *t_table_ptr ); + } + if( a_value.is< bsl::vector< bsl::uint8_t > >() ) + { + const bsl::vector< bsl::uint8_t >& t_bytes = a_value.the< bsl::vector< bsl::uint8_t > >(); + bdlde::Base64Encoder t_encoder( 0 ); // 0 maxLineLength = no line breaks + int t_max_len = bdlde::Base64Encoder::encodedLength( (int)t_bytes.size(), 0 ); + bsl::string t_b64( t_max_len, '\0' ); + int t_num_out = 0, t_num_in = 0; + t_encoder.convert( t_b64.begin(), &t_num_out, &t_num_in, + reinterpret_cast< const char* >( t_bytes.data() ), + reinterpret_cast< const char* >( t_bytes.data() + t_bytes.size() ) ); + int t_end_out = 0; + t_encoder.endConvert( t_b64.begin() + t_num_out, &t_end_out ); + t_b64.resize( t_num_out + t_end_out ); + return scarab::param_ptr_t( new scarab::param_value( std::string( t_b64 ) ) ); + } + if( a_value.is< bdlt::Datetime >() ) { - case TableValue::ValueType::VT_void: - return scarab::param_ptr_t( new scarab::param() ); - break; - case TableValue::ValueType::VT_bool: - return scarab::param_ptr_t( new scarab::param_value( a_value.GetBool() ) ); - break; - case TableValue::ValueType::VT_int8: - return scarab::param_ptr_t( new scarab::param_value( a_value.GetInt8() ) ); - break; - case TableValue::ValueType::VT_int16: - return scarab::param_ptr_t( new scarab::param_value( a_value.GetInt16() ) ); - break; - case TableValue::ValueType::VT_int32: - return scarab::param_ptr_t( new scarab::param_value( a_value.GetInt32() ) ); - break; - case TableValue::ValueType::VT_int64: - return scarab::param_ptr_t( new scarab::param_value( a_value.GetInt64() ) ); - break; - case TableValue::ValueType::VT_float: - return scarab::param_ptr_t( new scarab::param_value( a_value.GetFloat() ) ); - break; - case TableValue::ValueType::VT_double: - return scarab::param_ptr_t( new scarab::param_value( a_value.GetDouble() ) ); - break; - case TableValue::ValueType::VT_string: - return scarab::param_ptr_t( new scarab::param_value( a_value.GetString() ) ); - break; - case TableValue::ValueType::VT_array: - return table_to_param( a_value.GetArray() ); - break; - case TableValue::ValueType::VT_table: - return table_to_param( a_value.GetTable() ); - break; - case TableValue::ValueType::VT_uint8: - return scarab::param_ptr_t( new scarab::param_value( a_value.GetUint8() ) ); - break; - case TableValue::ValueType::VT_uint16: - return scarab::param_ptr_t( new scarab::param_value( a_value.GetUint16() ) ); - break; - case TableValue::ValueType::VT_uint32: - return scarab::param_ptr_t( new scarab::param_value( a_value.GetUint32() ) ); - break; - default: - throw std::domain_error( "Invalid SimpleAMQPClient TableValue type" ); + char t_buf[ 64 ]; + int t_len = bdlt::Iso8601Util::generate( t_buf, sizeof( t_buf ), a_value.the< bdlt::Datetime >() ); + if( t_len < 0 ) throw std::domain_error( "Failed to convert bdlt::Datetime to ISO 8601 string" ); + return scarab::param_ptr_t( new scarab::param_value( std::string( t_buf, t_len ) ) ); } - // should never get here + // uint64_t (deprecated in AMQP field tables) has no scarab equivalent + throw std::domain_error( "Unsupported rmqcpp FieldValue type" ); return scarab::param_ptr_t(); } - DRIPLINE_API AmqpClient::TableValue param_to_table( const scarab::param_node& a_node ) + DRIPLINE_API rmqt::FieldValue param_to_table( const scarab::param_node& a_node ) { - AmqpClient::Table t_table; + auto t_table = bsl::make_shared< rmqt::FieldTable >(); for( auto i_entry = a_node.begin(); i_entry != a_node.end(); ++i_entry ) { - t_table.insert( AmqpClient::TableEntry( i_entry.name(), param_to_table(*i_entry) ) ); + (*t_table)[ bsl::string( i_entry.name() ) ] = param_to_table( *i_entry ); } - return t_table; + return rmqt::FieldValue( t_table ); } - DRIPLINE_API AmqpClient::TableValue param_to_table( const scarab::param_array& a_array ) + DRIPLINE_API rmqt::FieldValue param_to_table( const scarab::param_array& a_array ) { - AmqpClient::Array t_array; + auto t_array = bsl::make_shared< rmqt::FieldArray >(); for( auto i_entry = a_array.begin(); i_entry != a_array.end(); ++i_entry ) { - t_array.push_back( param_to_table( *i_entry ) ); + t_array->push_back( param_to_table( *i_entry ) ); } - return t_array; + return rmqt::FieldValue( t_array ); } - DRIPLINE_API AmqpClient::TableValue param_to_table( const scarab::param_value& a_value ) + DRIPLINE_API rmqt::FieldValue param_to_table( const scarab::param_value& a_value ) { - if( a_value.is_bool() ) return AmqpClient::TableValue( a_value.as_bool() ); - if( a_value.is_int() ) return AmqpClient::TableValue( a_value.as_int() ); - if( a_value.is_uint() ) return AmqpClient::TableValue( (uint32_t)a_value.as_uint() ); - if( a_value.is_double() ) return AmqpClient::TableValue( a_value.as_double() ); - if( a_value.is_string() ) return AmqpClient::TableValue( a_value.as_string() ); + if( a_value.is_bool() ) return rmqt::FieldValue( a_value.as_bool() ); + if( a_value.is_int() ) return rmqt::FieldValue( (int64_t)a_value.as_int() ); + if( a_value.is_uint() ) return rmqt::FieldValue( (uint32_t)a_value.as_uint() ); + if( a_value.is_double() ) return rmqt::FieldValue( a_value.as_double() ); + if( a_value.is_string() ) return rmqt::FieldValue( bsl::string( a_value.as_string() ) ); throw std::domain_error( "Invalid param value type" ); } - DRIPLINE_API AmqpClient::TableValue param_to_table( const scarab::param& a_param ) + DRIPLINE_API rmqt::FieldValue param_to_table( const scarab::param& a_param ) { - if( a_param.is_null() ) return AmqpClient::TableValue(); + if( a_param.is_null() ) return rmqt::FieldValue(); if( a_param.is_node() ) return param_to_table( a_param.as_node() ); if( a_param.is_array() ) return param_to_table( a_param.as_array() ); if( a_param.is_value() ) return param_to_table( a_param.as_value() ); diff --git a/library/amqp.hh b/library/amqp.hh index 8ba2d178..ed4c371e 100644 --- a/library/amqp.hh +++ b/library/amqp.hh @@ -12,33 +12,29 @@ #include "param.hh" -#include "SimpleAmqpClient/SimpleAmqpClient.h" - -#include "SimpleAmqpClient/AmqpException.h" -#include "SimpleAmqpClient/AmqpLibraryException.h" +#include "rmqt_fieldvalue.h" +#include "rmqt_message.h" +#include "rmqp_messageguard.h" +#include "bsl_memory.h" namespace dripline { // convenience typedefs - typedef AmqpClient::Channel::ptr_t amqp_channel_ptr; - typedef AmqpClient::Envelope::ptr_t amqp_envelope_ptr; - typedef AmqpClient::BasicMessage::ptr_t amqp_message_ptr; - - typedef AmqpClient::AmqpException amqp_exception; - typedef AmqpClient::AmqpLibraryException amqp_lib_exception; + typedef BloombergLP::rmqp::TransferrableMessageGuard amqp_envelope_ptr; + typedef bsl::shared_ptr< BloombergLP::rmqt::Message > amqp_message_ptr; typedef std::vector< amqp_message_ptr > amqp_split_message_ptrs; // conversion functions - DRIPLINE_API scarab::param_ptr_t table_to_param( const AmqpClient::Table& a_table ); - DRIPLINE_API scarab::param_ptr_t table_to_param( const AmqpClient::Array& a_array ); - DRIPLINE_API scarab::param_ptr_t table_to_param( const AmqpClient::TableValue& a_value ); - - DRIPLINE_API AmqpClient::TableValue param_to_table( const scarab::param_node& a_node ); - DRIPLINE_API AmqpClient::TableValue param_to_table( const scarab::param_array& a_array ); - DRIPLINE_API AmqpClient::TableValue param_to_table( const scarab::param_value& a_value ); - DRIPLINE_API AmqpClient::TableValue param_to_table( const scarab::param& a_value ); + DRIPLINE_API scarab::param_ptr_t table_to_param( const BloombergLP::rmqt::FieldTable& a_table ); + DRIPLINE_API scarab::param_ptr_t table_to_param( const BloombergLP::rmqt::FieldArray& a_array ); + DRIPLINE_API scarab::param_ptr_t table_to_param( const BloombergLP::rmqt::FieldValue& a_value ); + + DRIPLINE_API BloombergLP::rmqt::FieldValue param_to_table( const scarab::param_node& a_node ); + DRIPLINE_API BloombergLP::rmqt::FieldValue param_to_table( const scarab::param_array& a_array ); + DRIPLINE_API BloombergLP::rmqt::FieldValue param_to_table( const scarab::param_value& a_value ); + DRIPLINE_API BloombergLP::rmqt::FieldValue param_to_table( const scarab::param& a_value ); } /* namespace dripline */ diff --git a/testing/test_amqp.cc b/testing/test_amqp.cc index 68897a92..77aeace8 100644 --- a/testing/test_amqp.cc +++ b/testing/test_amqp.cc @@ -7,15 +7,21 @@ #include "amqp.hh" +#include "bdlt_datetime.h" + +#include "catch2/catch_approx.hpp" #include "catch2/catch_test_macros.hpp" +using namespace BloombergLP; +using Catch::Approx; + TEST_CASE( "amqp_table", "[amqp]" ) { SECTION( "null" ) { scarab::param t_param; - AmqpClient::TableValue t_conv_table_value = dripline::param_to_table( t_param ); - REQUIRE( t_conv_table_value.GetType() == AmqpClient::TableValue::VT_void ); + rmqt::FieldValue t_conv_table_value = dripline::param_to_table( t_param ); + REQUIRE( t_conv_table_value.isUnset() ); scarab::param_ptr_t t_conv2_param = dripline::table_to_param( t_conv_table_value ); REQUIRE( t_conv2_param->is_null() ); @@ -25,9 +31,9 @@ TEST_CASE( "amqp_table", "[amqp]" ) { // bool scarab::param_value t_value( true ); - AmqpClient::TableValue t_conv_table_value = dripline::param_to_table( t_value ); - REQUIRE( t_conv_table_value.GetType() == AmqpClient::TableValue::VT_bool ); - REQUIRE( t_conv_table_value.GetBool() == t_value.as_bool() ); + rmqt::FieldValue t_conv_table_value = dripline::param_to_table( t_value ); + REQUIRE( t_conv_table_value.is< bool >() ); + REQUIRE( t_conv_table_value.the< bool >() == t_value.as_bool() ); scarab::param_ptr_t t_conv2_param = dripline::table_to_param( t_conv_table_value ); REQUIRE( t_conv2_param->is_value() ); @@ -37,8 +43,8 @@ TEST_CASE( "amqp_table", "[amqp]" ) // int t_value.set< int64_t >( -5 ); t_conv_table_value = dripline::param_to_table( t_value ); - REQUIRE( t_conv_table_value.GetType() == AmqpClient::TableValue::VT_int64 ); - REQUIRE( t_conv_table_value.GetInt64() == t_value.as_int() ); + REQUIRE( t_conv_table_value.is< int64_t >() ); + REQUIRE( t_conv_table_value.the< int64_t >() == t_value.as_int() ); t_conv2_param = dripline::table_to_param( t_conv_table_value ); REQUIRE( t_conv2_param->is_value() ); @@ -48,9 +54,9 @@ TEST_CASE( "amqp_table", "[amqp]" ) // uint t_value.set< uint32_t >( 50U ); t_conv_table_value = dripline::param_to_table( t_value ); - REQUIRE( t_conv_table_value.GetType() == AmqpClient::TableValue::VT_uint32 ); - REQUIRE( t_conv_table_value.GetUint32() == t_value.as_uint() ); - + REQUIRE( t_conv_table_value.is< uint32_t >() ); + REQUIRE( t_conv_table_value.the< uint32_t >() == t_value.as_uint() ); + t_conv2_param = dripline::table_to_param( t_conv_table_value ); REQUIRE( t_conv2_param->is_value() ); REQUIRE( t_conv2_param->as_value().is_uint() ); @@ -59,8 +65,8 @@ TEST_CASE( "amqp_table", "[amqp]" ) // double t_value.set< double >( 500.0 ); t_conv_table_value = dripline::param_to_table( t_value ); - REQUIRE( t_conv_table_value.GetType() == AmqpClient::TableValue::VT_double ); - REQUIRE( t_conv_table_value.GetDouble() == t_value.as_double() ); + REQUIRE( t_conv_table_value.is< double >() ); + REQUIRE( t_conv_table_value.the< double >() == t_value.as_double() ); t_conv2_param = dripline::table_to_param( t_conv_table_value ); REQUIRE( t_conv2_param->is_value() ); @@ -68,6 +74,19 @@ TEST_CASE( "amqp_table", "[amqp]" ) REQUIRE( t_conv2_param->as_value().as_double() == t_value.as_double() ); } + SECTION( "string" ) + { + scarab::param_value t_value( std::string("hello world") ); + rmqt::FieldValue t_conv_table_value = dripline::param_to_table( t_value ); + REQUIRE( t_conv_table_value.is< bsl::string >() ); + REQUIRE( std::string( t_conv_table_value.the< bsl::string >() ) == t_value.as_string() ); + + scarab::param_ptr_t t_conv2_param = dripline::table_to_param( t_conv_table_value ); + REQUIRE( t_conv2_param->is_value() ); + REQUIRE( t_conv2_param->as_value().is_string() ); + REQUIRE( t_conv2_param->as_value().as_string() == t_value.as_string() ); + } + SECTION( "node" ) { scarab::param_node t_node; @@ -76,15 +95,15 @@ TEST_CASE( "amqp_table", "[amqp]" ) t_node.add( "value2", 50U ); t_node.add( "value3", 500.0 ); - AmqpClient::TableValue t_conv_table_value = dripline::param_to_table( t_node ); - REQUIRE( t_conv_table_value.GetType() == AmqpClient::TableValue::VT_table ); + rmqt::FieldValue t_conv_table_value = dripline::param_to_table( t_node ); + REQUIRE( t_conv_table_value.is< bsl::shared_ptr< rmqt::FieldTable > >() ); - AmqpClient::Table t_conv_table = t_conv_table_value.GetTable(); + rmqt::FieldTable t_conv_table = *t_conv_table_value.the< bsl::shared_ptr< rmqt::FieldTable > >(); REQUIRE( t_conv_table.size() == t_node.size() ); - REQUIRE( t_conv_table["value0"].GetBool() == t_node["value0"]().as_bool() ); - REQUIRE( t_conv_table["value1"].GetInt64() == t_node["value1"]().as_int() ); - REQUIRE( t_conv_table["value2"].GetUint32() == t_node["value2"]().as_uint() ); - REQUIRE( t_conv_table["value3"].GetDouble() == t_node["value3"]().as_double() ); + REQUIRE( t_conv_table["value0"].the< bool >() == t_node["value0"]().as_bool() ); + REQUIRE( t_conv_table["value1"].the< int64_t >() == t_node["value1"]().as_int() ); + REQUIRE( t_conv_table["value2"].the< uint32_t >() == t_node["value2"]().as_uint() ); + REQUIRE( t_conv_table["value3"].the< double >() == t_node["value3"]().as_double() ); scarab::param_ptr_t t_conv2_param = dripline::table_to_param( t_conv_table ); REQUIRE( t_conv2_param->is_node() ); @@ -105,15 +124,15 @@ TEST_CASE( "amqp_table", "[amqp]" ) t_array.push_back( 50U ); t_array.push_back( 500.0 ); - AmqpClient::TableValue t_conv_table_value = dripline::param_to_table( t_array ); - REQUIRE( t_conv_table_value.GetType() == AmqpClient::TableValue::VT_array ); + rmqt::FieldValue t_conv_table_value = dripline::param_to_table( t_array ); + REQUIRE( t_conv_table_value.is< bsl::shared_ptr< rmqt::FieldArray > >() ); - AmqpClient::Array t_conv_array = t_conv_table_value.GetArray(); + rmqt::FieldArray t_conv_array = *t_conv_table_value.the< bsl::shared_ptr< rmqt::FieldArray > >(); REQUIRE( t_conv_array.size() == t_array.size() ); - REQUIRE( t_conv_array[0].GetBool() == t_array[0]().as_bool() ); - REQUIRE( t_conv_array[1].GetInt64() == t_array[1]().as_int() ); - REQUIRE( t_conv_array[2].GetUint32() == t_array[2]().as_uint() ); - REQUIRE( t_conv_array[3].GetDouble() == t_array[3]().as_double() ); + REQUIRE( t_conv_array[0].the< bool >() == t_array[0]().as_bool() ); + REQUIRE( t_conv_array[1].the< int64_t >() == t_array[1]().as_int() ); + REQUIRE( t_conv_array[2].the< uint32_t >() == t_array[2]().as_uint() ); + REQUIRE( t_conv_array[3].the< double >() == t_array[3]().as_double() ); scarab::param_ptr_t t_conv2_param = dripline::table_to_param( t_conv_array ); REQUIRE( t_conv2_param->is_array() ); @@ -150,32 +169,33 @@ TEST_CASE( "amqp_table", "[amqp]" ) t_node0.add( "array1", t_array1 ); t_node0.add( "node1", t_node1 ); - // conversion to a Table object - AmqpClient::TableValue t_conv_table_value = dripline::param_to_table( t_node0 ); + // conversion to a FieldTable object + rmqt::FieldValue t_conv_table_value = dripline::param_to_table( t_node0 ); - // validate structure of the Table object - REQUIRE( t_conv_table_value.GetType() == AmqpClient::TableValue::VT_table ); + // validate structure of the FieldTable object + REQUIRE( t_conv_table_value.is< bsl::shared_ptr< rmqt::FieldTable > >() ); + const rmqt::FieldTable& t_ft0 = *t_conv_table_value.the< bsl::shared_ptr< rmqt::FieldTable > >(); - REQUIRE( t_conv_table_value.GetTable()["null1"].GetType() == AmqpClient::TableValue::VT_void ); - REQUIRE( t_conv_table_value.GetTable()["value1"].GetType() == AmqpClient::TableValue::VT_uint32 ); - REQUIRE( t_conv_table_value.GetTable()["array1"].GetType() == AmqpClient::TableValue::VT_array ); - REQUIRE( t_conv_table_value.GetTable()["node1"].GetType() == AmqpClient::TableValue::VT_table ); + REQUIRE( t_ft0.at("null1").isUnset() ); + REQUIRE( t_ft0.at("value1").is< uint32_t >() ); + REQUIRE( t_ft0.at("array1").is< bsl::shared_ptr< rmqt::FieldArray > >() ); + REQUIRE( t_ft0.at("node1").is< bsl::shared_ptr< rmqt::FieldTable > >() ); - REQUIRE( t_conv_table_value.GetTable()["array1"].GetArray()[0].GetType() == AmqpClient::TableValue::VT_uint32 ); - REQUIRE( t_conv_table_value.GetTable()["array1"].GetArray()[1].GetType() == AmqpClient::TableValue::VT_array ); - REQUIRE( t_conv_table_value.GetTable()["array1"].GetArray()[2].GetType() == AmqpClient::TableValue::VT_table ); + const rmqt::FieldArray& t_fa1 = *t_ft0.at("array1").the< bsl::shared_ptr< rmqt::FieldArray > >(); + REQUIRE( t_fa1[0].is< uint32_t >() ); + REQUIRE( t_fa1[1].is< bsl::shared_ptr< rmqt::FieldArray > >() ); + REQUIRE( t_fa1[2].is< bsl::shared_ptr< rmqt::FieldTable > >() ); - REQUIRE( t_conv_table_value.GetTable()["array1"].GetArray()[1].GetArray()[0].GetType() == AmqpClient::TableValue::VT_uint32 ); + REQUIRE( (*t_fa1[1].the< bsl::shared_ptr< rmqt::FieldArray > >())[0].is< uint32_t >() ); + REQUIRE( t_fa1[2].the< bsl::shared_ptr< rmqt::FieldTable > >()->at("value3").is< uint32_t >() ); - REQUIRE( t_conv_table_value.GetTable()["array1"].GetArray()[2].GetTable()["value3"].GetType() == AmqpClient::TableValue::VT_uint32 ); - - REQUIRE( t_conv_table_value.GetTable()["node1"].GetTable()["value2"].GetType() == AmqpClient::TableValue::VT_uint32 ); - REQUIRE( t_conv_table_value.GetTable()["node1"].GetTable()["array2"].GetType() == AmqpClient::TableValue::VT_array ); - REQUIRE( t_conv_table_value.GetTable()["node1"].GetTable()["node2"].GetType() == AmqpClient::TableValue::VT_table ); - - REQUIRE( t_conv_table_value.GetTable()["node1"].GetTable()["array2"].GetArray()[0].GetType() == AmqpClient::TableValue::VT_uint32 ); + const rmqt::FieldTable& t_ft1 = *t_ft0.at("node1").the< bsl::shared_ptr< rmqt::FieldTable > >(); + REQUIRE( t_ft1.at("value2").is< uint32_t >() ); + REQUIRE( t_ft1.at("array2").is< bsl::shared_ptr< rmqt::FieldArray > >() ); + REQUIRE( t_ft1.at("node2").is< bsl::shared_ptr< rmqt::FieldTable > >() ); - REQUIRE( t_conv_table_value.GetTable()["node1"].GetTable()["node2"].GetTable()["value3"].GetType() == AmqpClient::TableValue::VT_uint32 ); + REQUIRE( (*t_ft1.at("array2").the< bsl::shared_ptr< rmqt::FieldArray > >())[0].is< uint32_t >() ); + REQUIRE( t_ft1.at("node2").the< bsl::shared_ptr< rmqt::FieldTable > >()->at("value3").is< uint32_t >() ); // return conversion to a param object scarab::param_ptr_t t_conv2_param = dripline::table_to_param( t_conv_table_value ); @@ -221,4 +241,61 @@ TEST_CASE( "amqp_table", "[amqp]" ) REQUIRE( t_conv2_node["node1"]["node2"]["value3"].is_value() ); REQUIRE( t_conv2_node["node1"]["node2"]["value3"]().as_uint() == t_node2["value3"]().as_uint() ); } + + SECTION( "rmqcpp_int_types" ) + { + // int8_t, int16_t, int32_t → scarab integer + REQUIRE( dripline::table_to_param( rmqt::FieldValue(int8_t(-1)) )->as_value().is_int() ); + REQUIRE( dripline::table_to_param( rmqt::FieldValue(int8_t(-1)) )->as_value().as_int() == -1 ); + + REQUIRE( dripline::table_to_param( rmqt::FieldValue(int16_t(-100)) )->as_value().is_int() ); + REQUIRE( dripline::table_to_param( rmqt::FieldValue(int16_t(-100)) )->as_value().as_int() == -100 ); + + REQUIRE( dripline::table_to_param( rmqt::FieldValue(int32_t(-100000)) )->as_value().is_int() ); + REQUIRE( dripline::table_to_param( rmqt::FieldValue(int32_t(-100000)) )->as_value().as_int() == -100000 ); + + // uint8_t, uint16_t → scarab uint + REQUIRE( dripline::table_to_param( rmqt::FieldValue(uint8_t(5)) )->as_value().is_uint() ); + REQUIRE( dripline::table_to_param( rmqt::FieldValue(uint8_t(5)) )->as_value().as_uint() == 5U ); + + REQUIRE( dripline::table_to_param( rmqt::FieldValue(uint16_t(5000)) )->as_value().is_uint() ); + REQUIRE( dripline::table_to_param( rmqt::FieldValue(uint16_t(5000)) )->as_value().as_uint() == 5000U ); + + // float → scarab double + REQUIRE( dripline::table_to_param( rmqt::FieldValue(float(1.5f)) )->as_value().is_double() ); + REQUIRE( dripline::table_to_param( rmqt::FieldValue(float(1.5f)) )->as_value().as_double() == Approx(1.5) ); + } + + SECTION( "binary_to_base64" ) + { + // "Hello" in ASCII bytes encodes to "SGVsbG8=" in base64 + bsl::vector< bsl::uint8_t > t_binary = {0x48, 0x65, 0x6c, 0x6c, 0x6f}; + scarab::param_ptr_t t_result = dripline::table_to_param( rmqt::FieldValue(t_binary) ); + REQUIRE( t_result->is_value() ); + REQUIRE( t_result->as_value().is_string() ); + REQUIRE( t_result->as_value().as_string() == "SGVsbG8=" ); + + // empty binary encodes to empty string + bsl::vector< bsl::uint8_t > t_empty; + scarab::param_ptr_t t_empty_result = dripline::table_to_param( rmqt::FieldValue(t_empty) ); + REQUIRE( t_empty_result->as_value().as_string() == "" ); + } + + SECTION( "datetime_to_iso8601" ) + { + bdlt::Datetime t_dt( 2025, 1, 15, 10, 30, 45 ); + scarab::param_ptr_t t_result = dripline::table_to_param( rmqt::FieldValue(t_dt) ); + REQUIRE( t_result->is_value() ); + REQUIRE( t_result->as_value().is_string() ); + // check date and time portion without depending on sub-second format + REQUIRE( t_result->as_value().as_string().substr( 0, 19 ) == "2025-01-15T10:30:45" ); + } + + SECTION( "unsupported_types" ) + { + // uint64_t is deprecated in AMQP field tables and has no scarab equivalent + rmqt::FieldValue t_uint64_val; + t_uint64_val.assign< uint64_t >( 42ULL ); + REQUIRE_THROWS_AS( dripline::table_to_param( t_uint64_val ), std::domain_error ); + } } From 2f4e0a92f3a708fb100e0d84bd552fdabf64584b Mon Sep 17 00:00:00 2001 From: Noah Oblath Date: Thu, 28 May 2026 11:41:47 -0700 Subject: [PATCH 06/45] Finished most updates to core --- library/core.cc | 551 ++++++++++++++++++------------------------------ library/core.hh | 59 ++---- 2 files changed, 225 insertions(+), 385 deletions(-) diff --git a/library/core.cc b/library/core.cc index 3e04b3f4..f29cd519 100644 --- a/library/core.cc +++ b/library/core.cc @@ -12,6 +12,16 @@ #include "dripline_exceptions.hh" #include "message.hh" +#include "rmqa_consumer.h" +#include "rmqa_rabbitcontext.h" +#include "rmqa_vhost.h" +#include "rmqa_producer.h" +#include "rmqa_topology.h" +#include "rmqp_producer.h" +#include "rmqp_messageguard.h" +#include "rmqt_simpleendpoint.h" +#include "rmqt_plaincredentials.h" + #include "authentication.hh" #include "exponential_backoff.hh" #include "logger.hh" @@ -28,20 +38,13 @@ namespace dripline sent_msg_pkg::~sent_msg_pkg() { - if( f_channel ) + if( f_reply_consumer ) { - try - { - LDEBUG( dlog, "Stopping consuming messages" ); - f_channel->BasicCancel( f_consumer_tag ); - } - catch( amqp_exception& e ) + LDEBUG( dlog, "Canceling reply consumer" ); + auto t_result = f_reply_consumer->cancelAndDrain(); + if( ! t_result ) { - LERROR( dlog, "AMQP exception caught while canceling the channel: (" << e.reply_code() << ") " << e.reply_text() ); - } - catch( amqp_lib_exception& e ) - { - LERROR( dlog, "AMQP library exception caught while canceling the channel: (" << e.ErrorCode() << ") " << e.what() ); + LWARN( dlog, "Error while canceling reply consumer: " << t_result.error() ); } } } @@ -58,7 +61,12 @@ namespace dripline f_heartbeat_routing_key(), f_max_payload_size(), f_make_connection(), - f_max_connection_attempts() + f_max_connection_attempts(), + f_rabbit_context(), + f_vhost(), + f_requests_producer(), + f_alerts_producer(), + f_connection_mutex( std::make_shared< std::mutex >() ) { // Get the default values, and merge in the supplied a_config // a_config's default value is also dripline_config, but the user can supply an arbitrary node. @@ -149,450 +157,297 @@ namespace dripline } } - sent_msg_pkg_ptr core::send( request_ptr_t a_request, amqp_channel_ptr a_channel ) const + sent_msg_pkg_ptr core::send( request_ptr_t a_request ) const { LDEBUG( dlog, "Sending request with routing key <" << a_request->routing_key() << ">" ); if ( ! f_make_connection || core::s_offline ) { throw a_request; - //throw dripline_error() << "cannot send reply when make_connection is false"; } - return do_send( std::static_pointer_cast< message >( a_request ), f_requests_exchange, true, a_channel ); + return do_send( std::static_pointer_cast< message >( a_request ), f_requests_exchange, true ); } - sent_msg_pkg_ptr core::send( reply_ptr_t a_reply, amqp_channel_ptr a_channel ) const + sent_msg_pkg_ptr core::send( reply_ptr_t a_reply ) const { LDEBUG( dlog, "Sending reply with routing key <" << a_reply->routing_key() << ">" ); if ( ! f_make_connection || core::s_offline ) { throw a_reply; - //throw dripline_error() << "cannot send reply when make_connection is false"; } - return do_send( std::static_pointer_cast< message >( a_reply ), f_requests_exchange, false, a_channel ); + return do_send( std::static_pointer_cast< message >( a_reply ), f_requests_exchange, false ); } - sent_msg_pkg_ptr core::send( alert_ptr_t a_alert, amqp_channel_ptr a_channel ) const + sent_msg_pkg_ptr core::send( alert_ptr_t a_alert ) const { LDEBUG( dlog, "Sending alert with routing key <" << a_alert->routing_key() << ">" ); if ( ! f_make_connection || core::s_offline ) { throw a_alert; - //throw dripline_error() << "cannot send reply when make_connection is false"; } - return do_send( std::static_pointer_cast< message >( a_alert ), f_alerts_exchange, false, a_channel ); + return do_send( std::static_pointer_cast< message >( a_alert ), f_alerts_exchange, false ); } - sent_msg_pkg_ptr core::do_send( message_ptr_t a_message, const std::string& a_exchange, bool a_expect_reply, amqp_channel_ptr a_channel ) const + sent_msg_pkg_ptr core::do_send( message_ptr_t a_message, const std::string& a_exchange, bool a_expect_reply ) const { // throws connection_error if it could not connect with the broker - // throws dripline_error if there's a problem with the exchange or creating the AMQP message object(s) - // returns the receive_reply package if the message was completely or partially sent - // the f_successful_send flag will be set accordingly: true if completely sent; false if partially sent - // if there was an error sending the message, that will be returned in f_send_error_message, which will be empty otherwise + // throws dripline_error if there's a problem creating the AMQP message object(s) + // returns the sent_msg_pkg; f_successful_send indicates whether the send succeeded - // lambda to create a string with the basic information about the send attempt auto t_diagnostic_string_maker = [a_message, this]() -> std::string { - return std::string("Broker: ") + f_address +"\nPort: " + std::to_string(f_port) + "\nRouting Key: " + a_message->routing_key(); + return std::string("Broker: ") + f_address + "\nPort: " + std::to_string(f_port) + "\nRouting Key: " + a_message->routing_key(); }; - amqp_channel_ptr t_channel = a_channel ? a_channel : open_channel(); - if( ! t_channel ) - { - throw connection_error() << "Unable to open channel to send message\n" << t_diagnostic_string_maker(); - } + open_connection(); - if( ! setup_exchange( t_channel, a_exchange ) ) + if( ! f_vhost ) { - throw dripline_error() << "Unable to setup the exchange <" << a_exchange << "> to send message\n" << t_diagnostic_string_maker(); + throw connection_error() << "Not connected to broker\n" << t_diagnostic_string_maker(); } - // create empty receive-reply object sent_msg_pkg_ptr t_receive_reply = std::make_shared< sent_msg_pkg >(); - std::unique_lock< std::mutex > t_rr_lock( t_receive_reply->f_mutex ); - - if( a_expect_reply ) - { - t_receive_reply->f_channel = t_channel; - - // create the reply-to queue, and bind the queue to the routing key over the given exchange - std::string t_reply_to = t_channel->DeclareQueue( "" ); - t_channel->BindQueue( t_reply_to, a_exchange, t_reply_to ); - // set the reply-to in the message because now we have the queue to which to reply - a_message->reply_to() = t_reply_to; - - // begin consuming on the reply-to queue - t_receive_reply->f_consumer_tag = t_channel->BasicConsume( t_reply_to ); - LDEBUG( dlog, "Reply-to for request: " << t_reply_to ); - LDEBUG( dlog, "Consumer tag for reply: " << t_receive_reply->f_consumer_tag ); - } - - // convert the dripline::message object to an AMQP message - amqp_split_message_ptrs t_amqp_messages = a_message->create_amqp_messages( f_max_payload_size ); - if( t_amqp_messages.empty() ) - { - throw dripline_error() << "Unable to convert the dripline::message object to AMQP message(s) to be sent\n" << t_diagnostic_string_maker(); - } try { - LDEBUG( dlog, "Sending message to <" << a_message->routing_key() << ">" ); - for( amqp_message_ptr& t_amqp_message : t_amqp_messages ) + if( a_expect_reply ) { - // send the message - // the first boolean argument is whether it's mandatory that the message be delivered to a queue. - // this is only the case for requests, where we expect something to be listening. - t_channel->BasicPublish( a_exchange, a_message->routing_key(), t_amqp_message, a_message->is_request(), false ); + send_withreply( a_message, a_exchange, t_receive_reply ); } - LDEBUG( dlog, "Message sent in " << t_amqp_messages.size() << " chunks" ); + else + { + if( ! send_noreply( a_message, a_exchange ) ) + { + t_receive_reply->f_successful_send = false; + t_receive_reply->f_send_error_message = "Error in send_noreply\n" + t_diagnostic_string_maker(); + return t_receive_reply; + } + } + LDEBUG( dlog, "Message sent to <" << a_message->routing_key() << ">" ); t_receive_reply->f_successful_send = true; t_receive_reply->f_send_error_message.clear(); } - catch( AmqpClient::ConnectionClosedException& e ) - { - LERROR( dlog, "Unable to send message because the connection is closed: " << e.what() ); - throw connection_error() << "Unable to send message because the connection is closed: " << e.what() << '\n' << t_diagnostic_string_maker(); - } - catch( AmqpClient::AmqpLibraryException& e ) + catch( connection_error& ) { - LERROR( dlog, "AMQP error while sending message: " << e.what() ); - t_receive_reply->f_successful_send = false; - t_receive_reply->f_send_error_message = std::string("AMQP error while sending message: ") + std::string(e.what()) + '\n' + t_diagnostic_string_maker(); - } - catch( AmqpClient::MessageReturnedException& e ) - { - LERROR( dlog, "Message was returned: " << e.what() ); - t_receive_reply->f_successful_send = false; - t_receive_reply->f_send_error_message = std::string("Message was returned: ") + std::string(e.what()) + '\n' + t_diagnostic_string_maker(); + throw; } catch( std::exception& e ) { LERROR( dlog, "Error while sending message: " << e.what() ); t_receive_reply->f_successful_send = false; - t_receive_reply->f_send_error_message = std::string("Error while sending message: ") + std::string(e.what()) + '\n' + t_diagnostic_string_maker(); + t_receive_reply->f_send_error_message = std::string("Error while sending message: ") + e.what() + '\n' + t_diagnostic_string_maker(); } return t_receive_reply; } - amqp_channel_ptr core::open_channel() const + void core::open_connection() const { - // Exceptions that can be encountered while opening a channel - // SimpleAmqpClient::Channel::Open(opts) - // std::runtime_error -- options are invalid; auth not specified - // std::logic_error -- unhandled auth type - // std::bad_alloc -- connection is null - // amqp_exception -- unsure of what would cause this - // amqp_lib_exception -- unable to make connection to the broker; maybe other things + std::lock_guard< std::mutex > t_lock( *f_connection_mutex ); - if ( ! f_make_connection || core::s_offline ) + if( f_vhost ) { - return amqp_channel_ptr(); - //throw dripline_error() << "Should not call open_channel when offline"; + return; // already connected + } + if( ! f_make_connection || s_offline ) + { + return; } - amqp_channel_ptr t_ret_ptr = amqp_channel_ptr(); + LINFO( dlog, "Opening AMQP connection to " << f_address << ":" << f_port ); + LDEBUG( dlog, "Using broker authentication: " << f_username << ":" << f_password ); - auto t_open_conn_fcn = [&]()->bool - { - try - { - LINFO( dlog, "Opening AMQP connection and creating channel to " << f_address << ":" << f_port ); - LDEBUG( dlog, "Using broker authentication: " << f_username << ":" << f_password ); - struct AmqpClient::Channel::OpenOpts opts; - opts.host = f_address; - opts.port = f_port; - opts.auth = AmqpClient::Channel::OpenOpts::BasicAuth(f_username, f_password); - t_ret_ptr = AmqpClient::Channel::Open( opts ); - return true; - } - catch( amqp_exception& e ) - { - if( e.is_soft_error() ) - { - LWARN( dlog, "Recoverable AMQP exception caught while opening channel: (" << e.reply_code() << ") " << e.reply_text() ); - return false; - } - // otherwise error is non-recoverable - throw; - } - catch( amqp_lib_exception& e ) - { - LERROR( dlog, "AMQP Library Exception caught while creating channel: (" << e.ErrorCode() << ") " << e.what() ); - if( e.ErrorCode() == -9 ) - { - LERROR( dlog, "This error means the client could not connect to the broker.\n" << - "Check that you have the address and port correct, and that the broker is running.") - } - return false; - } - // std::exceptions are non-recoverable, so don't catch them - }; + using namespace BloombergLP; - scarab::exponential_backoff<> t_open_conn_backoff( t_open_conn_fcn, f_max_connection_attempts ); - auto t_exp_cancel_wrap = wrap_cancelable( t_open_conn_backoff ); - scarab::signal_handler::add_cancelable( t_exp_cancel_wrap ); + f_rabbit_context = bsl::make_shared< rmqa::RabbitContext >(); - int t_expback_return = 0; - try - { - LDEBUG( dlog, "Attempting to open channel; will make up to " << f_max_connection_attempts << " attempts" ); - t_expback_return = t_open_conn_backoff.go(); - // either succeeded or failed after multiple attempts - } - catch( amqp_exception& e ) - { - LERROR( dlog, "Unrecoverable AMQP exception caught while opening channel: (" << e.reply_code() << ") " << e.reply_text() ); - } - catch(const std::exception& e) - { - // unrecoverable error causing a std::exception - LERROR( dlog, "Standard exception caught while creating channel: " << e.what() ); - } + auto t_endpoint = bsl::make_shared< rmqt::SimpleEndpoint >( f_address, "/", (bsl::uint16_t)f_port ); + auto t_credentials = bsl::make_shared< rmqt::PlainCredentials >( f_username, f_password ); + f_vhost = f_rabbit_context->createVHostConnection( "dripline", t_endpoint, t_credentials ); - if( t_expback_return == 0 ) + if( ! f_vhost ) { - LERROR( dlog, "Failed to open a channel; no more attempts will be made" ); + f_rabbit_context.reset(); + throw connection_error() << "Unable to create vhost connection to " << f_address << ":" << f_port; } - - return t_ret_ptr; - } - bool core::setup_exchange( amqp_channel_ptr a_channel, const std::string& a_exchange ) - { - if( s_offline || ! a_channel ) + // Create requests producer { - return false; + rmqa::Topology t_topo; + auto t_ex = t_topo.addExchange( f_requests_exchange, rmqt::ExchangeType::TOPIC ); + auto t_result = f_vhost->createProducer( t_topo, t_ex, 10 ); + if( ! t_result ) + { + f_vhost.reset(); + f_rabbit_context.reset(); + throw connection_error() << "Unable to create requests producer: " << t_result.error(); + } + f_requests_producer = t_result.value(); } - try + // Create alerts producer { - LDEBUG( dlog, "Declaring exchange <" << a_exchange << ">" ); - a_channel->DeclareExchange( a_exchange, AmqpClient::Channel::EXCHANGE_TYPE_TOPIC, false, false, false ); - return true; - } - catch( amqp_exception& e ) - { - LERROR( dlog, "AMQP exception caught while declaring exchange: (" << e.reply_code() << ") " << e.reply_text() ); - return false; - } - catch( amqp_lib_exception& e ) - { - LERROR( dlog, "AMQP library exception caught while declaring exchange: (" << e.ErrorCode() << ") " << e.what() ); - return false; + rmqa::Topology t_topo; + auto t_ex = t_topo.addExchange( f_alerts_exchange, rmqt::ExchangeType::TOPIC ); + auto t_result = f_vhost->createProducer( t_topo, t_ex, 10 ); + if( ! t_result ) + { + f_requests_producer.reset(); + f_vhost.reset(); + f_rabbit_context.reset(); + throw connection_error() << "Unable to create alerts producer: " << t_result.error(); + } + f_alerts_producer = t_result.value(); } + + LINFO( dlog, "AMQP connection established" ); } - bool core::setup_queue( amqp_channel_ptr a_channel, const std::string& a_queue_name ) + void core::send_withreply( message_ptr_t a_message, const std::string& a_exchange, sent_msg_pkg_ptr a_pkg ) const { - if( s_offline || ! a_channel ) - { - return false; - } + using namespace BloombergLP; + + // Generate a unique name for the temporary reply queue + std::string t_reply_to = string_from_uuid( generate_random_uuid() ); + a_message->reply_to() = t_reply_to; + LDEBUG( dlog, "Reply-to for request: " << t_reply_to ); + + // Build topology: declare the exchange (idempotent), add auto-delete reply queue, bind + rmqa::Topology t_reply_topo; + auto t_ex = t_reply_topo.addExchange( a_exchange, rmqt::ExchangeType::TOPIC ); + auto t_queue = t_reply_topo.addQueue( t_reply_to, rmqt::AutoDelete::ON, rmqt::Durable::OFF ); + t_reply_topo.bind( t_ex, t_queue, t_reply_to ); + + // Create promise; the consumer callback will fulfil it when the complete reply is assembled + a_pkg->f_reply_promise = std::make_shared< std::promise< reply_ptr_t > >(); + auto t_promise = a_pkg->f_reply_promise; + + // Inline accumulator struct for multi-chunk replies + struct reply_pack + { + amqp_split_message_ptrs f_messages; + unsigned f_chunks_received; + std::string f_routing_key; + std::mutex f_mutex; + reply_pack() : f_messages(), f_chunks_received( 0 ), f_routing_key(), f_mutex() {} + }; + auto t_pack = std::make_shared< reply_pack >(); - try + auto t_callback = [t_pack, t_promise]( rmqp::MessageGuard& guard ) { - LDEBUG( dlog, "Declaring queue <" << a_queue_name << ">" ); - a_channel->DeclareQueue( a_queue_name, false, false, true, true ); - return true; - } - catch( amqp_exception& e ) - { - LERROR( dlog, "AMQP exception caught while declaring queue: (" << e.reply_code() << ") " << e.reply_text() ); - return false; - } - catch( amqp_lib_exception& e ) - { - LERROR( dlog, "AMQP library exception caught while declaring queue: (" << e.ErrorCode() << ") " << e.what() ); - return false; - } + auto t_amqp_message = bsl::make_shared< rmqt::Message >( guard.message() ); + const std::string t_msg_id( t_amqp_message->messageId() ); + auto t_parsed_id = message::parse_message_id( t_msg_id ); + unsigned t_chunk_idx = std::get< 1 >( t_parsed_id ); + unsigned t_total_chunks = std::get< 2 >( t_parsed_id ); - } + bool t_complete = false; + { + std::lock_guard< std::mutex > t_lock( t_pack->f_mutex ); + if( t_pack->f_messages.empty() ) + { + t_pack->f_messages.resize( t_total_chunks ); + t_pack->f_routing_key = std::string( guard.envelope().routingKey() ); + } + t_pack->f_messages[t_chunk_idx] = t_amqp_message; + ++t_pack->f_chunks_received; + t_complete = ( t_pack->f_chunks_received == t_pack->f_messages.size() ); + } - bool core::bind_key( amqp_channel_ptr a_channel, const std::string& a_exchange, const std::string& a_queue_name, const std::string& a_routing_key ) - { - if( s_offline || ! a_channel ) - { - return false; - } + guard.ack(); - try - { - LDEBUG( dlog, "Binding key <" << a_routing_key << "> to queue <" << a_queue_name << "> over exchange <" << a_exchange << ">" ); - a_channel->BindQueue( a_queue_name, a_exchange, a_routing_key ); + if( t_complete ) + { + try + { + message_ptr_t t_message = message::process_message( t_pack->f_messages, t_pack->f_routing_key ); + reply_ptr_t t_reply = std::dynamic_pointer_cast< msg_reply >( t_message ); + if( t_reply ) + { + t_promise->set_value( t_reply ); + } + else + { + t_promise->set_exception( std::make_exception_ptr( + dripline_error() << "Expected reply but received a different message type" ) ); + } + } + catch( ... ) + { + t_promise->set_exception( std::current_exception() ); + } + } + }; - return true; - } - catch( amqp_exception& e ) + // Create the reply consumer on the temporary queue + auto t_consumer_result = f_vhost->createConsumer( t_reply_topo, t_queue, t_callback, "reply-" + t_reply_to, 1 ); + if( ! t_consumer_result ) { - LERROR( dlog, "AMQP exception caught while declaring binding key <" << a_routing_key << ">: (" << e.reply_code() << ") " << e.reply_text() ); - return false; - } - catch( amqp_lib_exception& e ) - { - LERROR( dlog, "AMQP library exception caught while binding key <" << a_routing_key << ">: (" << e.ErrorCode() << ") " << e.what() ); - return false; + throw connection_error() << "Unable to create reply consumer: " << t_consumer_result.error(); } - } + a_pkg->f_reply_consumer = t_consumer_result.value(); - std::string core::start_consuming( amqp_channel_ptr a_channel, const std::string& a_queue_name ) - { - if( s_offline || ! a_channel ) + // Send message chunks via the requests producer + amqp_split_message_ptrs t_amqp_messages = a_message->create_amqp_messages( f_max_payload_size ); + if( t_amqp_messages.empty() ) { - return std::string(); + throw dripline_error() << "Unable to convert dripline::message to AMQP message(s)"; } - try - { - LDEBUG( dlog, "Starting to consume messages on queue <" << a_queue_name << ">" ); - // second bool is setting no_ack to false - return a_channel->BasicConsume( a_queue_name, "", true, false ); - } - catch( amqp_exception& e ) + LDEBUG( dlog, "Sending request to <" << a_message->routing_key() << "> in " << t_amqp_messages.size() << " chunk(s)" ); + for( amqp_message_ptr& t_amqp_message : t_amqp_messages ) { - LERROR( dlog, "AMQP exception caught while starting consuming messages on <" << a_queue_name << ">: (" << e.reply_code() << ") " << e.reply_text() ); - return std::string(); - } - catch( amqp_lib_exception& e ) - { - LERROR( dlog, "AMQP library exception caught while starting consuming messages on <" << a_queue_name << ">: (" << e.ErrorCode() << ") " << e.what() ); - return std::string(); + auto t_status = f_requests_producer->send( + *t_amqp_message, + a_message->routing_key(), + []( const rmqt::Message&, const bsl::string&, const rmqt::ConfirmResponse& ) {} ); + if( t_status != rmqp::Producer::SENDING ) + { + throw dripline_error() << "Failed to enqueue request chunk for sending; status=" << t_status; + } } } - bool core::stop_consuming( amqp_channel_ptr a_channel, std::string& a_consumer_tag ) + bool core::send_noreply( message_ptr_t a_message, const std::string& a_exchange ) const { - if( s_offline || ! a_channel ) - { - return false; - } + using namespace BloombergLP; - try - { - LDEBUG( dlog, "Stopping consuming messages for consumer <" << a_consumer_tag << ">" ); - a_channel->BasicCancel( a_consumer_tag ); - a_consumer_tag.clear(); - return true; - } - catch( amqp_exception& e ) - { - LERROR( dlog, "AMQP exception caught while stopping consuming messages on <" << a_consumer_tag << ">: (" << e.reply_code() << ") " << e.reply_text() ); - return false; - } - catch( amqp_lib_exception& e ) + bsl::shared_ptr< rmqa::Producer > t_producer; + if( a_exchange == f_alerts_exchange ) { - LERROR( dlog, "AMQP library exception caught while stopping consuming messages on <" << a_consumer_tag << ">: (" << e.ErrorCode() << ") " << e.what() ); - return false; + t_producer = f_alerts_producer; } - catch( AmqpClient::ConsumerTagNotFoundException& e ) + else { - LERROR( dlog, "Fatal AMQP exception encountered while stopping consuming messages on <" << a_consumer_tag << ">: " << e.what() ); - return false; + t_producer = f_requests_producer; } - catch( std::exception& e ) - { - LERROR( dlog, "Standard exception caught while stopping consuming messages on <" << a_consumer_tag << ">: " << e.what() ); - return false; - } - } - bool core::remove_queue( amqp_channel_ptr a_channel, const std::string& a_queue_name ) - { - if( s_offline || ! a_channel ) + if( ! t_producer ) { + LERROR( dlog, "No producer available for exchange <" << a_exchange << ">" ); return false; } - try - { - LDEBUG( dlog, "Deleting queue <" << a_queue_name << ">" ); - a_channel->DeleteQueue( a_queue_name, false ); - return true; - } - catch( AmqpClient::ConnectionClosedException& e ) - { - LERROR( dlog, "Fatal AMQP exception encountered removing queue <" << a_queue_name << ">: " << e.what() ); - return false; - } - catch( amqp_lib_exception& e ) - { - LERROR( dlog, "AMQP library exception caught while removing queue <" << a_queue_name << ">: (" << e.ErrorCode() << ") " << e.what() ); - return false; - } - catch( std::exception& e ) + amqp_split_message_ptrs t_amqp_messages = a_message->create_amqp_messages( f_max_payload_size ); + if( t_amqp_messages.empty() ) { - LERROR( dlog, "Standard exception caught while removing queue <" << a_queue_name << ">: " << e.what() ); + LERROR( dlog, "Unable to convert dripline::message to AMQP message(s)" ); return false; } - } - - void core::listen_for_message( amqp_envelope_ptr& a_envelope, core::post_listen_status& a_status, amqp_channel_ptr a_channel, const std::string& a_consumer_tag, int a_timeout_ms, bool a_do_ack ) - { - if( s_offline || ! a_channel ) - { - a_status = core::post_listen_status::unknown; - return; - } - while( true ) + LDEBUG( dlog, "Sending message to <" << a_message->routing_key() << "> in " << t_amqp_messages.size() << " chunk(s)" ); + bool t_all_sent = true; + for( amqp_message_ptr& t_amqp_message : t_amqp_messages ) { - try + auto t_status = t_producer->send( + *t_amqp_message, + a_message->routing_key(), + []( const rmqt::Message&, const bsl::string&, const rmqt::ConfirmResponse& ) {} ); + if( t_status != rmqp::Producer::SENDING ) { - if( a_timeout_ms > 0 ) - { - a_channel->BasicConsumeMessage( a_consumer_tag, a_envelope, a_timeout_ms ); - } - else - { - a_envelope = a_channel->BasicConsumeMessage( a_consumer_tag ); - } - if( a_envelope ) - { - if( a_do_ack ) a_channel->BasicAck( a_envelope ); - a_status = post_listen_status::message_received; - } - else - { - a_status = post_listen_status::timeout; - } - return; - } - catch( AmqpClient::ConnectionClosedException& e ) - { - LERROR( dlog, "Fatal AMQP exception encountered: " << e.what() ); - a_status = post_listen_status::hard_error; - return; - } - catch( AmqpClient::ConsumerCancelledException& e ) - { - LERROR( dlog, "Fatal AMQP exception encountered: " << e.what() ); - a_status = post_listen_status::hard_error; - return; - } - catch( AmqpClient::AmqpException& e ) - { - if( e.is_soft_error() ) - { - LWARN( dlog, "Non-fatal AMQP exception encountered: " << e.reply_text() ); - a_status = post_listen_status::soft_error; - return; - } - LERROR( dlog, "Fatal AMQP exception encountered: " << e.reply_text() ); - a_status = post_listen_status::hard_error; - return; - } - catch( std::exception& e ) - { - LERROR( dlog, "Standard exception caught: " << e.what() ); - a_status = post_listen_status::hard_error; - return; - } - catch(...) - { - LERROR( dlog, "Unknown exception caught" ); - a_status = post_listen_status::hard_error; - return; + LERROR( dlog, "Failed to enqueue message chunk for sending; status=" << t_status ); + t_all_sent = false; } } + return t_all_sent; } } /* namespace dripline */ diff --git a/library/core.hh b/library/core.hh index d8ca0e85..1b52f084 100644 --- a/library/core.hh +++ b/library/core.hh @@ -11,7 +11,9 @@ #include "dripline_config.hh" #include "message.hh" +#include #include +#include #include #include @@ -21,6 +23,8 @@ namespace scarab class param_node; } +namespace BloombergLP { namespace rmqa { class Consumer; class RabbitContext; class VHost; class Producer; } } + namespace dripline { /*! @@ -38,9 +42,8 @@ namespace dripline */ struct DRIPLINE_API sent_msg_pkg { - std::mutex f_mutex; - amqp_channel_ptr f_channel; - std::string f_consumer_tag; + bsl::shared_ptr< BloombergLP::rmqa::Consumer > f_reply_consumer; ///< rmqcpp consumer on the temporary reply queue (null if no reply expected) + std::shared_ptr< std::promise< reply_ptr_t > > f_reply_promise; ///< fulfilled by the reply consumer callback when the reply is assembled bool f_successful_send; std::string f_send_error_message; ~sent_msg_pkg(); @@ -76,15 +79,6 @@ namespace dripline public: static bool s_offline; - enum class post_listen_status - { - unknown, ///< Initialized or unknown status - message_received, ///< A message was received, and the channel is still valid - timeout, ///< A timeout occurred, and the channel is still valid - soft_error, ///< An error occurred, but the channel should still be valid - hard_error ///< An error occurred, and the channel is no longer valid - }; - public: /* \brief Extracts necessary configuration and authentication information and prepares the DL object to interact with the RabbitMQ broker. Does not initiate connection to the broker. @@ -111,20 +105,17 @@ namespace dripline core& operator=( core&& a_orig ) = default; public: - /// Sends a request message and returns a channel on which to listen for a reply. + /// Sends a request message and waits for a reply via an rmqcpp consumer on a temporary queue. /// Default exchange is "requests" - /// Caller can supply a channel; if one is not supplied, a new channel will be established - virtual sent_msg_pkg_ptr send( request_ptr_t a_request, amqp_channel_ptr a_channel = amqp_channel_ptr() ) const; + virtual sent_msg_pkg_ptr send( request_ptr_t a_request ) const; /// Sends a reply message /// Default exchange is "requests" - /// Caller can supply a channel; if one is not supplied, a new channel will be established - virtual sent_msg_pkg_ptr send( reply_ptr_t a_reply, amqp_channel_ptr a_channel = amqp_channel_ptr() ) const; + virtual sent_msg_pkg_ptr send( reply_ptr_t a_reply ) const; /// Sends an alert message /// Default exchange is "alerts" - /// Caller can supply a channel; if one is not supplied, a new channel will be established - virtual sent_msg_pkg_ptr send( alert_ptr_t a_alert, amqp_channel_ptr a_channel = amqp_channel_ptr() ) const; + virtual sent_msg_pkg_ptr send( alert_ptr_t a_alert ) const; mv_referrable( std::string, address ); mv_accessible( unsigned, port ); @@ -144,29 +135,23 @@ namespace dripline protected: friend class receiver; - sent_msg_pkg_ptr do_send( message_ptr_t a_message, const std::string& a_exchange, bool a_expect_reply, amqp_channel_ptr a_channel = amqp_channel_ptr() ) const; + sent_msg_pkg_ptr do_send( message_ptr_t a_message, const std::string& a_exchange, bool a_expect_reply ) const; - amqp_channel_ptr send_withreply( message_ptr_t a_message, std::string& a_reply_consumer_tag, const std::string& a_exchange ) const; + /// Sets up a temporary reply queue, starts an rmqcpp consumer on it, then sends the message. + /// Stores the consumer and reply promise in a_pkg. + void send_withreply( message_ptr_t a_message, const std::string& a_exchange, sent_msg_pkg_ptr a_pkg ) const; bool send_noreply( message_ptr_t a_message, const std::string& a_exchange ) const; - amqp_channel_ptr open_channel() const; - - static bool setup_exchange( amqp_channel_ptr a_channel, const std::string& a_exchange ); - - static bool setup_queue( amqp_channel_ptr a_channel, const std::string& a_queue_name ); - - static bool bind_key( amqp_channel_ptr a_channel, const std::string& a_exchange, const std::string& a_queue_name, const std::string& a_routing_key ); + /// Lazily establishes the RabbitMQ connection and creates the requests/alerts producers. + /// Thread-safe; subsequent calls are no-ops if already connected. + void open_connection() const; - static std::string start_consuming( amqp_channel_ptr a_channel, const std::string& a_queue_name ); - - static bool stop_consuming( amqp_channel_ptr a_channel, std::string& a_consumer_tag ); - - static bool remove_queue( amqp_channel_ptr a_channel, const std::string& a_queue_name ); - - public: - /// listen for a single AMQP message - static void listen_for_message( amqp_envelope_ptr& a_envelope, post_listen_status& a_status, amqp_channel_ptr a_channel, const std::string& a_consumer_tag, int a_timeout_ms = 0, bool a_do_ack = true ); + mutable bsl::shared_ptr< BloombergLP::rmqa::RabbitContext > f_rabbit_context; + mutable bsl::shared_ptr< BloombergLP::rmqa::VHost > f_vhost; + mutable bsl::shared_ptr< BloombergLP::rmqa::Producer > f_requests_producer; + mutable bsl::shared_ptr< BloombergLP::rmqa::Producer > f_alerts_producer; + mutable std::shared_ptr< std::mutex > f_connection_mutex; }; } /* namespace dripline */ From a5b3bc89ec358877526c919f55d9be64615a9611 Mon Sep 17 00:00:00 2001 From: Noah Oblath Date: Thu, 28 May 2026 12:08:57 -0700 Subject: [PATCH 07/45] Completed most updates to concurrent_receiver --- library/receiver.cc | 241 +++++++++++--------------------------------- library/receiver.hh | 26 ++--- 2 files changed, 75 insertions(+), 192 deletions(-) diff --git a/library/receiver.cc b/library/receiver.cc index f7e79714..3296d896 100644 --- a/library/receiver.cc +++ b/library/receiver.cc @@ -12,9 +12,14 @@ #include "dripline_exceptions.hh" #include "message.hh" +#include "rmqa_vhost.h" +#include "rmqp_messageguard.h" + #include "logger.hh" #include "signal_handler.hh" +#include + LOGGER( dlog, "receiver" ); namespace dripline @@ -46,8 +51,7 @@ namespace dripline receiver::receiver() : scarab::cancelable(), f_incoming_messages(), - f_single_message_wait_ms( 1000 ), - f_reply_listen_timeout_ms( 1000 ) + f_single_message_wait_ms( 1000 ) {} receiver& receiver::operator=( receiver&& a_orig ) @@ -63,10 +67,10 @@ namespace dripline { try { - amqp_message_ptr t_message = a_envelope->Message(); - LDEBUG( dlog, "Received a message chunk <" << t_message->MessageId() ); + amqp_message_ptr t_message = bsl::make_shared< BloombergLP::rmqt::Message >( a_envelope->message() ); + LDEBUG( dlog, "Received a message chunk <" << std::string( t_message->messageId() ) ); - auto t_parsed_message_id = message::parse_message_id( t_message->MessageId() ); + auto t_parsed_message_id = message::parse_message_id( std::string( t_message->messageId() ) ); std::string t_message_id( std::get<0>(t_parsed_message_id) ); if( incoming_messages().count( t_message_id ) == 0 ) { @@ -78,7 +82,7 @@ namespace dripline t_pack.f_messages.resize( std::get<2>(t_parsed_message_id) ); // put in place the first message chunk received t_pack.f_messages[std::get<1>(t_parsed_message_id)] = t_message; - t_pack.f_routing_key = a_envelope->RoutingKey(); + t_pack.f_routing_key = std::string( a_envelope->envelope().routingKey() ); t_pack.f_chunks_received = 1; if( t_pack.f_messages.size() == 1 ) @@ -205,18 +209,12 @@ namespace dripline reply_ptr_t receiver::wait_for_reply( const sent_msg_pkg_ptr a_receive_reply, int a_timeout_ms ) { - core::post_listen_status t_temp = core::post_listen_status::unknown; - return wait_for_reply( a_receive_reply, t_temp, a_timeout_ms ); - } - - reply_ptr_t receiver::wait_for_reply( const sent_msg_pkg_ptr a_receive_reply, core::post_listen_status& a_status, int a_timeout_ms ) - { - if ( ! a_receive_reply->f_channel ) + if( ! a_receive_reply->f_reply_consumer ) { return reply_ptr_t(); } - if( a_timeout_ms != 0 ) + if( a_timeout_ms != 0 ) { LDEBUG( dlog, "Waiting for a reply (timeout: " << a_timeout_ms << " ms)" ); } @@ -225,189 +223,37 @@ namespace dripline LDEBUG( dlog, "Waiting for a reply (no timeout)" ); } - // Assign the chunk timeout time; it should be f_reply_listen_timeout_ms unless a_timeout_ms is shorter than f_reply_listen_timeout_ms - unsigned t_chunk_timeout_ms = f_reply_listen_timeout_ms; - if( a_timeout_ms > 0 && a_timeout_ms < t_chunk_timeout_ms ) - { - t_chunk_timeout_ms = a_timeout_ms; - } - - // for checking the wait_for_reply timeout - auto t_timeout_time = std::chrono::system_clock::now() + std::chrono::milliseconds(a_timeout_ms); + auto t_future = a_receive_reply->f_reply_promise->get_future(); + auto t_deadline = std::chrono::steady_clock::now() + std::chrono::milliseconds( a_timeout_ms ); - // wait for messages until either: - // 1. the channel is no longer valid (return empty reply pointer; a_chan_valid will be false) - // 2. listening times out (return empty reply pointer; a_chan_valid will be true) - // 3. a full dripline message is received (return message) - // 4. error processing a recieved amqp message (return empty reply pointer) - while( ! is_canceled() && (a_timeout_ms == 0 || std::chrono::system_clock::now() < t_timeout_time) ) + while( ! is_canceled() ) { - amqp_envelope_ptr t_envelope; - core::listen_for_message( t_envelope, a_status, a_receive_reply->f_channel, a_receive_reply->f_consumer_tag, t_chunk_timeout_ms, false ); - - // check whether we canceled while listening - if( is_canceled() ) - { - LDEBUG( dlog, "Receiver was canceled before receiving reply" ); - return reply_ptr_t(); - } - - // there was a soft error listening on the channel; no message received - if( a_status == core::post_listen_status::soft_error ) + auto t_status = t_future.wait_for( std::chrono::milliseconds( 100 ) ); + if( t_status == std::future_status::ready ) { - LWARN( dlog, "There was a soft error while listening for a reply; no message received" ); - continue; + return t_future.get(); } - - // there was a hard error listening on the channel; no message received - if( a_status == core::post_listen_status::hard_error ) + if( a_timeout_ms > 0 && std::chrono::steady_clock::now() >= t_deadline ) { - LERROR( dlog, "There was a hard error error while listening for a reply; no message received" ); + LWARN( dlog, "Timed out waiting for reply" ); return reply_ptr_t(); } - - // listening timed out - if( a_status == core::post_listen_status::timeout ) - { - LTRACE( dlog, "Listening for reply message chunks timed out" ); - // continue to wait for message chunks - continue; - } - - // unknown state; should not get here - if( a_status == core::post_listen_status::unknown ) - { - LERROR( dlog, "An unknown status occurred while listening for messages" ); - return reply_ptr_t(); - } - - // because core::listen_for_message uses whether or not the envelope is empty to differentiate between a received message and a timeout, - // we will never have an empty envelope and a status == message_received - - // go ahead with message processing - try - { - amqp_message_ptr t_message = t_envelope->Message(); - LDEBUG( dlog, "Received a message chunk <" << t_message->MessageId() ); - - auto t_parsed_message_id = message::parse_message_id( t_message->MessageId() ); - if( f_incoming_messages.count( std::get<0>(t_parsed_message_id) ) == 0 ) - { - // this path: first chunk for this message - LDEBUG( dlog, "This is the first chunk for this message; creating new message pack" ); - // create the new message_pack object - incoming_message_pack& t_pack = f_incoming_messages[std::get<0>(t_parsed_message_id)]; - // set the f_messages vector to the expected size - t_pack.f_messages.resize( std::get<2>(t_parsed_message_id) ); - // put in place the first message chunk received - t_pack.f_messages[std::get<1>(t_parsed_message_id)] = t_message; - t_pack.f_routing_key = t_envelope->RoutingKey(); - t_pack.f_chunks_received = 1; - - if( t_pack.f_chunks_received == t_pack.f_messages.size() ) - { - return process_received_reply( t_pack, std::get<0>(t_parsed_message_id) ); - } - // else, need more chunks - } - else - { - // this path: have already received chunks from this message - LDEBUG( dlog, "This is not the first chunk for this message; adding to message pack" ); - incoming_message_pack& t_pack = f_incoming_messages[std::get<0>(t_parsed_message_id)]; - if( t_pack.f_processing.load() ) - { - LWARN( dlog, "Message <" << std::get<0>(t_parsed_message_id) << "> is already being processed\n" << - "Just received chunk " << std::get<1>(t_parsed_message_id) << " of " << std::get<2>(t_parsed_message_id) ); - } - else - { - if( t_pack.f_messages[std::get<1>(t_parsed_message_id)] ) - { - LWARN( dlog, "Received duplicate message chunk for message <" << std::get<0>(t_parsed_message_id) << ">; chunk " << std::get<1>(t_parsed_message_id) ); - } - else - { - // add chunk to set of chunks - t_pack.f_messages[std::get<1>(t_parsed_message_id)] = t_message; - ++t_pack.f_chunks_received; - if( t_pack.f_chunks_received == t_pack.f_messages.size() ) - { - return process_received_reply( t_pack, std::get<0>(t_parsed_message_id) ); - } - } - } - } - - } - catch( dripline_error& e ) - { - LERROR( dlog, "There was a problem processing the message: " << e.what() ); - a_status = core::post_listen_status::soft_error; - return reply_ptr_t(); - } - - } // end while( ! is_canceled() && not timed out ) - - // check if listening timed out - if( ! is_canceled() && std::chrono::system_clock::now() > t_timeout_time ) - { - LINFO( dlog, "Listening for reply message timed out" ); - a_status = core::post_listen_status::timeout; - return reply_ptr_t(); - } - - LDEBUG( dlog, "Receiver was canceled" ); - return reply_ptr_t(); - } - - reply_ptr_t receiver::process_received_reply( incoming_message_pack& a_pack, const std::string& a_message_id ) - { - a_pack.f_processing.store( true ); - try - { - message_ptr_t t_message = message::process_message( a_pack.f_messages, a_pack.f_routing_key ); - - f_incoming_messages.erase( a_message_id ); - - if( t_message->is_reply() ) - { - return std::static_pointer_cast< msg_reply >( t_message ); - } - else - { - throw dripline_error() << "Non-reply message received"; - } - } - catch( dripline_error& e ) - { - LERROR( dlog, "Dripline exception caught while handling message: " << e.what() ); - } - catch( amqp_exception& e ) - { - LERROR( dlog, "AMQP exception caught while sending reply: (" << e.reply_code() << ") " << e.reply_text() ); - } - catch( amqp_lib_exception& e ) - { - LERROR( dlog, "AMQP Library Exception caught while sending reply: (" << e.ErrorCode() << ") " << e.what() ); - } - catch( std::exception& e ) - { - LERROR( dlog, "Standard exception caught while sending reply: " << e.what() ); } + LDEBUG( dlog, "Receiver canceled while waiting for reply" ); return reply_ptr_t(); - } concurrent_receiver::concurrent_receiver() : receiver(), - f_message_queue() + f_message_queue(), + f_consumer() {} concurrent_receiver::concurrent_receiver( concurrent_receiver&& a_orig ) : receiver( std::move(a_orig) ), - f_message_queue() + f_message_queue(), + f_consumer( std::move(a_orig.f_consumer) ) {} concurrent_receiver::~concurrent_receiver() @@ -416,7 +262,7 @@ namespace dripline concurrent_receiver& concurrent_receiver::operator=( concurrent_receiver&& a_orig ) { receiver::operator=( std::move(a_orig) ); - // nothing to do with message queue + f_consumer = std::move(a_orig.f_consumer); return *this; } @@ -447,6 +293,41 @@ namespace dripline } } + void concurrent_receiver::start_listening( bsl::shared_ptr< BloombergLP::rmqa::VHost > a_vhost, + const BloombergLP::rmqa::Topology& a_topology, + const BloombergLP::rmqt::QueueHandle& a_queue_handle, + const std::string& a_label ) + { + using namespace BloombergLP; + auto t_result = a_vhost->createConsumer( + a_topology, a_queue_handle, + [this]( rmqp::MessageGuard& guard ) { + amqp_envelope_ptr t_envelope = guard.transferOwnership(); + t_envelope->ack(); + handle_message_chunk( std::move(t_envelope) ); + }, + a_label, + 1 ); + if( ! t_result ) + { + throw connection_error() << "Unable to create consumer: " << t_result.error(); + } + f_consumer = t_result.value(); + } + + void concurrent_receiver::stop_listening() + { + if( f_consumer ) + { + auto t_result = f_consumer->cancelAndDrain(); + if( ! t_result ) + { + LWARN( dlog, "Error canceling consumer: " << t_result.error() ); + } + f_consumer.reset(); + } + } + } /* namespace dripline */ diff --git a/library/receiver.hh b/library/receiver.hh index 5021cdea..7d526070 100644 --- a/library/receiver.hh +++ b/library/receiver.hh @@ -12,6 +12,8 @@ #include "dripline_api.hh" #include "dripline_fwd.hh" +#include "rmqa_topology.h" + #include "cancelable.hh" #include "concurrent_queue.hh" #include "member_variables.hh" @@ -105,8 +107,6 @@ namespace dripline mv_referrable( incoming_message_map, incoming_messages ); /// Wait time for all message chunks from a single dripline message mv_accessible( unsigned, single_message_wait_ms ); - /// Listen timeout for individual message chunks when waiting for replies - mv_accessible( unsigned, reply_listen_timeout_ms ); public: @@ -118,18 +118,9 @@ namespace dripline @return Reply message */ reply_ptr_t wait_for_reply( const sent_msg_pkg_ptr a_receive_reply, int a_timeout_ms = 0 ); - /*! - User interface for waiting for a reply message. - This can be called multiple times to receive multiple replies. - @param[in] a_receive_reply The sent-message package from the request. - @param[in] a_timeout_ms Timeout for waiting for a reply; if it's 0, there will be no timeout. - @param[out] a_chan_valid Returns whether or not the channel is still valid for use after receiving a message. - @return Reply message - */ - reply_ptr_t wait_for_reply( const sent_msg_pkg_ptr a_receive_reply, core::post_listen_status& a_status, int a_timeout_ms = 0 ); protected: - reply_ptr_t process_received_reply( incoming_message_pack& a_pack, const std::string& a_message_id ); + // (no protected helpers currently) }; @@ -172,6 +163,16 @@ namespace dripline /// Handles messages that appear in the concurrent queue by calling `submit_message()`. void execute(); + /// Creates an rmqcpp consumer on the given queue and begins receiving messages. + /// Each received message is passed to handle_message_chunk(). + void start_listening( bsl::shared_ptr< BloombergLP::rmqa::VHost > a_vhost, + const BloombergLP::rmqa::Topology& a_topology, + const BloombergLP::rmqt::QueueHandle& a_queue_handle, + const std::string& a_label = "" ); + + /// Cancels the rmqcpp consumer, drains in-flight messages, and releases it. + void stop_listening(); + protected: /// Handles messages according to the use case. It's to be implemented by the class inheriting from concurrent_receiver /// For a concrete example, see @ref service or @ref endpoint_listener_receiver. @@ -179,6 +180,7 @@ namespace dripline mv_referrable( scarab::concurrent_queue< message_ptr_t >, message_queue ); mv_referrable( std::thread, receiver_thread ); + bsl::shared_ptr< BloombergLP::rmqa::Consumer > f_consumer; }; } /* namespace dripline */ From 50b822cbb0beb15063b358747d412da702f9786a Mon Sep 17 00:00:00 2001 From: Noah Oblath Date: Thu, 28 May 2026 12:38:02 -0700 Subject: [PATCH 08/45] Moved endpoint_listener_receiver to endpoint files. Removed listener.hh/cc --- library/CMakeLists.txt | 2 - library/endpoint.cc | 41 ++++++++++++ library/endpoint.hh | 29 ++++++++ library/listener.cc | 148 ----------------------------------------- library/listener.hh | 132 ------------------------------------ 5 files changed, 70 insertions(+), 282 deletions(-) delete mode 100644 library/listener.cc delete mode 100644 library/listener.hh diff --git a/library/CMakeLists.txt b/library/CMakeLists.txt index 5a85083b..addfb980 100644 --- a/library/CMakeLists.txt +++ b/library/CMakeLists.txt @@ -22,7 +22,6 @@ set( dripline_HEADERS endpoint.hh heartbeater.hh hub.hh - listener.hh message.hh monitor.hh monitor_config.hh @@ -49,7 +48,6 @@ set( dripline_SOURCES endpoint.cc heartbeater.cc hub.cc - listener.cc message.cc monitor.cc monitor_config.cc diff --git a/library/endpoint.cc b/library/endpoint.cc index 85e617b7..795881c6 100644 --- a/library/endpoint.cc +++ b/library/endpoint.cc @@ -431,4 +431,45 @@ namespace dripline return a_request->reply( dl_success(), "Hello, " + a_request->sender_exe() ); } + endpoint_listener_receiver::endpoint_listener_receiver( endpoint_ptr_t a_endpoint_ptr ) : + scarab::cancelable(), + concurrent_receiver(), + f_endpoint( a_endpoint_ptr ) + {} + + endpoint_listener_receiver::endpoint_listener_receiver( endpoint_listener_receiver&& a_orig ) : + scarab::cancelable( std::move(a_orig) ), + concurrent_receiver( std::move(a_orig) ), + f_endpoint( std::move(a_orig.f_endpoint) ) + {} + + endpoint_listener_receiver::~endpoint_listener_receiver() + {} + + endpoint_listener_receiver& endpoint_listener_receiver::operator=( endpoint_listener_receiver&& a_orig ) + { + concurrent_receiver::operator=( std::move(a_orig) ); + f_endpoint = std::move(a_orig.f_endpoint); + return *this; + } + + void endpoint_listener_receiver::submit_message( message_ptr_t a_message ) + { + try + { + f_endpoint->sort_message( a_message ); + return; + } + catch( dripline_error& e ) + { + LERROR( dlog, "<" << f_endpoint->name() << ">: Dripline exception caught while handling message: " << e.what() ); + throw; + } + catch( std::exception& e ) + { + LERROR( dlog, "<" << f_endpoint->name() << ">: Standard exception caught while handling message: " << e.what() ); + throw; + } + } + } /* namespace dripline */ diff --git a/library/endpoint.hh b/library/endpoint.hh index e5ce09d2..7cd18926 100644 --- a/library/endpoint.hh +++ b/library/endpoint.hh @@ -9,6 +9,7 @@ #define DRIPLINE_ENDPOINT_HH_ #include "message.hh" +#include "receiver.hh" #include "return_codes.hh" namespace dripline @@ -252,6 +253,34 @@ namespace dripline return a_request->reply( dl_success(), "No action taken (this is the default method)" ); } + /*! + @class endpoint_listener_receiver + @author N.S. Oblath + + @brief Decorator class for a plain endpoint: adds concurrent_receiver capabilities. + + @details + The endpoint_listener_receiver is used by @ref service to wrap an endpoint that is to listen for messages asynchronously. + */ + class DRIPLINE_API endpoint_listener_receiver : public concurrent_receiver + { + public: + explicit endpoint_listener_receiver( endpoint_ptr_t a_endpoint_ptr ); + endpoint_listener_receiver( const endpoint_listener_receiver& ) = delete; + endpoint_listener_receiver( endpoint_listener_receiver&& a_orig ); + virtual ~endpoint_listener_receiver(); + + endpoint_listener_receiver& operator=( const endpoint_listener_receiver& ) = delete; + endpoint_listener_receiver& operator=( endpoint_listener_receiver&& a_orig ); + + protected: + /// Direct submission of messages to the endpoint + virtual void submit_message( message_ptr_t a_message ) override; + + /// Pointer to the decorated endpoint + mv_referrable( endpoint_ptr_t, endpoint ); + }; + } /* namespace dripline */ #endif /* DRIPLINE_ENDPOINT_HH_ */ diff --git a/library/listener.cc b/library/listener.cc deleted file mode 100644 index 233bc902..00000000 --- a/library/listener.cc +++ /dev/null @@ -1,148 +0,0 @@ -/* - * listener.cc - * - * Created on: Jun 23, 2019 - * Author: N.S. Oblath - */ - -#define DRIPLINE_API_EXPORTS - -#include "listener.hh" - -#include "endpoint.hh" - -#include "core.hh" -#include "dripline_exceptions.hh" - -#include "logger.hh" - -namespace dripline -{ - LOGGER( dlog, "listener" ); - - listener::listener() : - cancelable(), - f_channel(), - f_consumer_tag(), - f_listen_timeout_ms( 1000 ), - f_listener_thread() - {} - - listener& listener::operator=( listener&& a_orig ) - { - cancelable::operator=( std::move(a_orig) ); - f_channel = std::move( a_orig.f_channel ); - f_consumer_tag = std::move( a_orig.f_consumer_tag ); - f_listen_timeout_ms = a_orig.f_listen_timeout_ms; - f_listener_thread = std::move( a_orig.f_listener_thread ); - return *this; - } - - endpoint_listener_receiver::endpoint_listener_receiver( endpoint_ptr_t a_endpoint_ptr ) : - scarab::cancelable(), - listener_receiver(), - f_endpoint( a_endpoint_ptr ) - {} - - endpoint_listener_receiver::endpoint_listener_receiver( endpoint_listener_receiver&& a_orig ) : - scarab::cancelable( std::move(a_orig) ), - listener_receiver( std::move(a_orig) ), - f_endpoint( std::move(a_orig.f_endpoint) ) - {} - - endpoint_listener_receiver::~endpoint_listener_receiver() - {} - - endpoint_listener_receiver& endpoint_listener_receiver::operator=( endpoint_listener_receiver&& a_orig ) - { - listener_receiver::operator=( std::move(a_orig) ); - f_endpoint = std::move(a_orig.f_endpoint); - return *this; - } - - bool endpoint_listener_receiver::listen_on_queue() - { - LINFO( dlog, "Listening for incoming messages on <" << f_endpoint->name() << ">" ); - - while( ! f_canceled.load() ) - { - - amqp_envelope_ptr t_envelope; - core::post_listen_status t_post_listen_status = core::post_listen_status::unknown; - core::listen_for_message( t_envelope, t_post_listen_status, f_channel, f_consumer_tag, f_listen_timeout_ms ); - - if( f_canceled.load() ) - { - LDEBUG( dlog, "Service canceled" ); - return true; - } - - if( t_post_listen_status == core::post_listen_status::timeout ) - { - // we end up here every time the listen times out with no message received - continue; - } - - if( t_post_listen_status == core::post_listen_status::soft_error ) - { - LWARN( dlog, "A soft error ocurred while listening for messages for <" << f_endpoint->name() << ">. The channel is still valid" ); - continue; - } - - if( t_post_listen_status == core::post_listen_status::hard_error ) - { - LERROR( dlog, "A hard error ocurred while listening for messages for <" << f_endpoint->name() << ">. The channel is no longer valid" ); - return false; - } - - if( t_post_listen_status == core::post_listen_status::unknown ) - { - LERROR( dlog, "An unknown status occurred while listening for messages for <" << f_endpoint->name() << ">" ); - return false; - } - - // remaining status is core::post_listen_status::message_received - - handle_message_chunk( t_envelope ); - - if( f_canceled.load() ) - { - LDEBUG( dlog, "Listener <" << f_endpoint->name() << "> canceled" ); - return true; - } - } - return true; - } - - void endpoint_listener_receiver::submit_message( message_ptr_t a_message ) - { - try - { - f_endpoint->sort_message( a_message ); - return; - } - catch( dripline_error& e ) - { - LERROR( dlog, "<" << f_endpoint->name() << ">: Dripline exception caught while handling message: " << e.what() ); - throw; - } - catch( amqp_exception& e ) - { - LERROR( dlog, "<" << f_endpoint->name() << ">: AMQP exception caught while handling message: (" << e.reply_code() << ") " << e.reply_text() ); - throw; - } - catch( amqp_lib_exception& e ) - { - LERROR( dlog, "<" << f_endpoint->name() << ">: AMQP Library Exception caught while handling message: (" << e.ErrorCode() << ") " << e.what() ); - throw; - } - catch( std::exception& e ) - { - LERROR( dlog, "<" << f_endpoint->name() << ">: Standard exception caught while handling message: " << e.what() ); - throw; - } - - return; - } - -} /* namespace dripline */ diff --git a/library/listener.hh b/library/listener.hh deleted file mode 100644 index 82e1305b..00000000 --- a/library/listener.hh +++ /dev/null @@ -1,132 +0,0 @@ -/* - * listener.hh - * - * Created on: Jun 23, 2019 - * Author: N.S. Oblath - */ - -#ifndef DRIPLINE_LISTENER_HH_ -#define DRIPLINE_LISTENER_HH_ - -#include "dripline_fwd.hh" - -#include "receiver.hh" - -#include "cancelable.hh" -#include "member_variables.hh" - -#include - -namespace dripline -{ - /*! - @class listener - @author N.S. Oblath - - @brief A listener is a class capable of listening for AMQP messages on an AMQP channel. This class provides the - basic framework for doing that. - - @details - The listener class is a mix-in class that provides the interface for listening for messages: - the pure virtual function `listen_on_queue()`. - This function should be run in the thread provided by this class. - However, it needs to be implemented by the inheriting class. - - The typical use case involves at least two threads: - 1. A listener gets messages from the AMQP channel (using `listen_on_queue(), e.g. @ref service or @ref endpoint_listener_receiver) and - calls `receiver::handle_message_chunk()` - 2. A receiver has a timing thread waiting for multiple message chunks (if relevant); - when the message is complete, `receiver::process_message()` is called. - - This class also provides the objects and information needed for listening on a queue: - * The AMQP channel - * A consumer tag - * A timeout (in ms) - * The thread object for listening - */ - class DRIPLINE_API listener : public virtual scarab::cancelable - { - public: - listener(); - listener( const listener& ) = delete; - listener( listener&& a_orig ) = default; - virtual ~listener() = default; - - listener& operator=( const listener& ) = delete; - listener& operator=( listener&& a_orig ); - - /// Returns false if the return is due to an error in this function; returns true otherwise (namely because it was canceled) - virtual bool listen_on_queue() = 0; - - mv_referrable( amqp_channel_ptr, channel ); - - mv_referrable( std::string, consumer_tag ); - - mv_accessible( unsigned, listen_timeout_ms ); - - mv_referrable( std::thread, listener_thread ); - }; - - /*! - @class listener_receiver - @author N.S. Oblath - - @brief Convenience class to bring together @ref listener and @ref concurrent_receiver - */ - class DRIPLINE_API listener_receiver : public listener, public concurrent_receiver - { - public: - listener_receiver() : scarab::cancelable(), listener(), concurrent_receiver() {} - listener_receiver( const listener_receiver& ) = delete; - listener_receiver( listener_receiver&& a_orig ) : - scarab::cancelable( std::move(a_orig) ), - listener( std::move(a_orig) ), - concurrent_receiver( std::move(a_orig) ) - {} - - listener_receiver& operator=( const listener_receiver& ) = delete; - listener_receiver& operator=( listener_receiver&& a_orig ) - { - listener::operator=( std::move(a_orig) ); - concurrent_receiver::operator=( std::move(a_orig) ); - return *this; - } - }; - - /*! - @class endpoint_listener_receiver - @author N.S. Oblath - - @brief Decorator class for a plain endpoint: adds listener_receiver capabilities. - - @details - The endpoint_listener_receiver is used by @ref service to wrap an endpoint that is to listen for messages asynchronously. - */ - class DRIPLINE_API endpoint_listener_receiver : public listener_receiver - { - public: - endpoint_listener_receiver( endpoint_ptr_t a_endpoint_ptr ); - endpoint_listener_receiver( const endpoint_listener_receiver& ) = delete; - endpoint_listener_receiver( endpoint_listener_receiver&& a_orig ); - virtual ~endpoint_listener_receiver(); - - endpoint_listener_receiver& operator=( const endpoint_listener_receiver& ) = delete; - endpoint_listener_receiver& operator=( endpoint_listener_receiver&& a_orig ); - - /// Listens for AMQP messages and then passes them to be handled as Dripline message chunks - /// Returns false if the return is due to an error in this function; returns true otherwise (namely because it was canceled) - virtual bool listen_on_queue(); - - protected: - /// Direct submission of messages to the endpoint - virtual void submit_message( message_ptr_t a_message ); - - /// Pointer to the decorated endpoint - mv_referrable( endpoint_ptr_t, endpoint ); - }; - - - -} /* namespace dripline */ - -#endif /* DRIPLINE_LISTENER_HH_ */ From ec9879181570bdbebe95f605655df6a9c72861b2 Mon Sep 17 00:00:00 2001 From: Noah Oblath Date: Thu, 28 May 2026 13:20:00 -0700 Subject: [PATCH 09/45] Modifications to service are complete --- library/service.cc | 277 +++++++-------------------------------------- library/service.hh | 56 +++------ 2 files changed, 54 insertions(+), 279 deletions(-) diff --git a/library/service.cc b/library/service.cc index d450b9b2..499880bd 100644 --- a/library/service.cc +++ b/library/service.cc @@ -33,7 +33,7 @@ namespace dripline core( a_config.has("dripline_mesh") ? a_config["dripline_mesh"].as_node() : dripline_config(), a_auth, a_make_connection ), endpoint( a_config.get_value( "name", "dlcpp_service" ) ), - listener_receiver(), + concurrent_receiver(), heartbeater( this ), scheduler<>(), f_auth( a_auth ), @@ -47,9 +47,7 @@ namespace dripline { LDEBUG( dlog, "Service (cpp) created with config:\n" << a_config ); // get more values from the config - // default of f_listen_timeout_ms is in the listener class - f_listen_timeout_ms = a_config.get_value( "loop_timeout_ms", f_listen_timeout_ms ); - heartbeater::f_check_timeout_ms = f_listen_timeout_ms; + heartbeater::f_check_timeout_ms = a_config.get_value( "loop_timeout_ms", 1000 ); // default of f_single_message_wait_ms is in the receiver class f_single_message_wait_ms = a_config.get_value( "message_wait_ms", f_single_message_wait_ms ); // default of f_heartbeat_interval_s is in the heartbeater class @@ -89,7 +87,7 @@ namespace dripline cancelable::operator=( std::move(a_orig) ); core::operator=( std::move(a_orig) ); endpoint::operator=( std::move(a_orig)); - listener_receiver::operator=( std::move(a_orig) ); + concurrent_receiver::operator=( std::move(a_orig) ); heartbeater::operator=( std::move(a_orig) ); scheduler<>::operator=( std::move(a_orig) ); @@ -121,12 +119,12 @@ namespace dripline bool service::add_async_child( endpoint_ptr_t a_endpoint_ptr ) { - lr_ptr_t t_listener_receiver_ptr = std::dynamic_pointer_cast< listener_receiver >( a_endpoint_ptr ); - if( ! t_listener_receiver_ptr ) + elr_ptr_t t_elr_ptr = std::dynamic_pointer_cast< endpoint_listener_receiver >( a_endpoint_ptr ); + if( ! t_elr_ptr ) { - t_listener_receiver_ptr.reset( new endpoint_listener_receiver( a_endpoint_ptr ) ); + t_elr_ptr = std::make_shared< endpoint_listener_receiver >( a_endpoint_ptr ); } - auto t_inserted = f_async_children.insert( std::make_pair( a_endpoint_ptr->name(), t_listener_receiver_ptr ) ); + auto t_inserted = f_async_children.insert( std::make_pair( a_endpoint_ptr->name(), t_elr_ptr ) ); if( t_inserted.second ) { a_endpoint_ptr->set_service( this ); @@ -208,20 +206,22 @@ namespace dripline LINFO( dlog, "Connecting to <" << f_address << ":" << f_port << ">" ); - if( ! open_channels() ) return false; + try + { + open_connection(); + } + catch( connection_error& e ) + { + LERROR( dlog, "Unable to connect to the broker: " << e.what() ); + return false; + } f_status = status::channel_created; - if( ! setup_exchange( f_channel, f_requests_exchange ) ) return false; - if( ! setup_exchange( f_channel, f_alerts_exchange ) ) return false; - f_status = status::exchange_declared; - - if( ! setup_queues() ) return false; - f_status = status::queue_declared; - - if( ! bind_keys() ) return false; - f_status = status::queue_bound; - - if( ! start_consuming() ) return false; + // TODO (Phase 6): set up service topology, declare queues, bind routing keys + // - declare service queue (f_name) and routing keys (f_name + ".#", f_broadcast_key + ".#") + // - for each sync child, bind child_name + ".#" to the service queue + // - call start_listening( f_vhost, topology, service_queue_handle, f_name ) + // - for each async child, declare its queue and call child->start_listening(...) f_status = status::consuming; return true; @@ -262,35 +262,26 @@ namespace dripline LINFO( dlog, "Starting receiver thread" ); f_receiver_thread = std::thread( &concurrent_receiver::execute, this ); - // lambda to cancel everything on an error from listener::listen_on_queue() - bool t_listen_error = false; - auto t_cancel_on_listen_error = [&t_listen_error, this](listener& a_listener) { - if( ! a_listener.listen_on_queue() ) - { - t_listen_error = true; - this->cancel( RETURN_ERROR ); - } - }; - if( ! f_async_children.empty() ) { LINFO( dlog, "Starting async children" ); } else { LDEBUG( dlog, "No async children to start" ); } - for( async_map_t::iterator t_child_it = f_async_children.begin(); - t_child_it != f_async_children.end(); - ++t_child_it ) + for( auto& t_child_pair : f_async_children ) { - t_child_it->second->receiver_thread() = std::thread( &concurrent_receiver::execute, static_cast< listener_receiver* >(t_child_it->second.get()) ); - t_child_it->second->listener_thread() = std::thread( t_cancel_on_listen_error, std::ref(*t_child_it->second.get()) ); + t_child_pair.second->receiver_thread() = std::thread( &concurrent_receiver::execute, t_child_pair.second.get() ); + // TODO (Phase 6): call t_child_pair.second->start_listening( f_vhost, topology, queue_handle, t_child_pair.first ) } - LINFO( dlog, "Starting listener thread" ); - t_cancel_on_listen_error( *this ); + // Block until canceled + while( ! is_canceled() ) + { + std::this_thread::sleep_for( std::chrono::milliseconds( 100 ) ); + } - for( async_map_t::iterator t_child_it = f_async_children.begin(); - t_child_it != f_async_children.end(); - ++t_child_it ) + for( auto& t_child_pair : f_async_children ) { - t_child_it->second->listener_thread().join(); - t_child_it->second->receiver_thread().join(); + if( t_child_pair.second->receiver_thread().joinable() ) + { + t_child_pair.second->receiver_thread().join(); + } } f_receiver_thread.join(); @@ -303,12 +294,10 @@ namespace dripline { f_scheduler_thread.join(); } - - if( t_listen_error) throw dripline_error() << "Something went wrong while listening for messages"; } catch( std::system_error& e ) { - LERROR( dlog, "Could not start the a thread due to a system error: " << e.what() ); + LERROR( dlog, "Could not start a thread due to a system error: " << e.what() ); return false; } catch( dripline_error& e ) @@ -329,182 +318,20 @@ namespace dripline { LINFO( dlog, "Stopping service on <" << f_name << ">" ); - if( f_status >= status::listening ) // listening or processing + if( f_status >= status::listening ) { this->cancel( dl_success().rc_value() ); f_status = status::consuming; } - if( f_status >= status::queue_bound ) // queue_bound or consuming - { - if( ! stop_consuming() ) return false; - f_status = status::queue_bound; - } + stop_listening(); - if( f_status >= status::queue_declared ) // queue_declared or queue_bound + for( auto& t_child_pair : f_async_children ) { - if( ! remove_queue() ) return false; - f_status = status::exchange_declared; + t_child_pair.second->stop_listening(); } - return true; - } - - bool service::open_channels() - { - LDEBUG( dlog, "Opening channel for service <" << f_name << ">" ); - f_channel = open_channel(); - if( ! f_channel ) return false; - - for( async_map_t::iterator t_child_it = f_async_children.begin(); - t_child_it != f_async_children.end(); - ++t_child_it ) - { - LDEBUG( dlog, "Opening channel for child <" << t_child_it->first << ">" ); - t_child_it->second->channel() = open_channel(); - t_child_it->second->set_listen_timeout_ms( f_listen_timeout_ms ); - } - return true; - } - - bool service::setup_queues() - { - LDEBUG( dlog, "Setting up queue for service <" << f_name << ">" ); - if( ! setup_queue( f_channel, f_name ) ) return false; - - for( async_map_t::iterator t_child_it = f_async_children.begin(); - t_child_it != f_async_children.end(); - ++t_child_it ) - { - LDEBUG( dlog, "Setting up queue for async child <" << t_child_it->first << ">" ); - if( ! setup_queue( t_child_it->second->channel(), t_child_it->first ) ) return false; - } - - return true; - } - - bool service::bind_keys() - { - LDEBUG( dlog, "Binding primary service keys" ); - if( ! bind_key( f_channel, f_requests_exchange, f_name, f_name + ".#" ) ) return false; - if( ! bind_key( f_channel, f_requests_exchange, f_name, f_broadcast_key + ".#" ) ) return false; - - LDEBUG( dlog, "Binding keys for synchronous children" ); - for( sync_map_t::const_iterator t_child_it = f_sync_children.begin(); - t_child_it != f_sync_children.end(); - ++t_child_it ) - { - if( ! bind_key( f_channel, f_requests_exchange, f_name, t_child_it->first + ".#" ) ) return false; - } - - LDEBUG( dlog, "Binding keys for asynchronous children" ); - for( async_map_t::iterator t_child_it = f_async_children.begin(); - t_child_it != f_async_children.end(); - ++t_child_it ) - { - if( ! bind_key( t_child_it->second->channel(), f_requests_exchange, t_child_it->first, t_child_it->first + ".#" ) ) return false; - } - - return true; - } - - bool service::start_consuming() - { - f_consumer_tag = core::start_consuming( f_channel, f_name ); - if( f_consumer_tag.empty() ) return false; - - for( async_map_t::iterator t_child_it = f_async_children.begin(); - t_child_it != f_async_children.end(); - ++t_child_it ) - { - t_child_it->second->consumer_tag() = core::start_consuming( t_child_it->second->channel(), t_child_it->first ); - if( t_child_it->second->consumer_tag().empty() ) return false; - } - return true; - } - - bool service::stop_consuming() - { - // doesn't stop on failure; continues trying to stop consuming - bool t_success = true; - t_success = core::stop_consuming( f_channel, f_consumer_tag ); - for( async_map_t::iterator t_child_it = f_async_children.begin(); - t_child_it != f_async_children.end(); - ++t_child_it ) - { - t_success = core::stop_consuming( t_child_it->second->channel(), t_child_it->second->consumer_tag() ); - } - return t_success; - } - - bool service::remove_queue() - { - // doesn't stop on failure; continues trying to remove queues - bool t_success = true; - t_success = core::remove_queue( f_channel, f_name ); - for( async_map_t::iterator t_child_it = f_async_children.begin(); - t_child_it != f_async_children.end(); - ++t_child_it ) - { - t_success = core::remove_queue( t_child_it->second->channel(), t_child_it->first ); - } - return t_success; - } - - bool service::listen_on_queue() - { - LINFO( dlog, "Listening for incoming messages on <" << f_name << ">" ); - - while( ! is_canceled() ) - { - amqp_envelope_ptr t_envelope; - core::post_listen_status t_post_listen_status = core::post_listen_status::unknown; - core::listen_for_message( t_envelope, t_post_listen_status, f_channel, f_consumer_tag, f_listen_timeout_ms ); - - if( f_canceled.load() ) - { - LDEBUG( dlog, "Service canceled" ); - return true; - } - - if( t_post_listen_status == core::post_listen_status::timeout ) - { - // we end up here every time the listen times out with no message received - continue; - } - - if( t_post_listen_status == core::post_listen_status::soft_error ) - { - LWARN( dlog, "A soft error ocurred while listening for messages for <" << f_name << ">. The channel is still valid" ); - continue; - } - - if( t_post_listen_status == core::post_listen_status::hard_error ) - { - LERROR( dlog, "A hard error ocurred while listening for messages for <" << f_name << ">. The channel is no longer valid" ); - return false; - } - - if( t_post_listen_status == core::post_listen_status::unknown ) - { - LERROR( dlog, "An unknown status occurred while listening for messages for <" << f_name << ">" ); - return false; - } - - // remaining status is core::post_listen_status::message_received - - f_status = status::processing; - - handle_message_chunk( t_envelope ); - - if( f_canceled.load() ) - { - LDEBUG( dlog, "Service <" << f_name << "> canceled" ); - return true; - } - - f_status = status::listening; - } + f_status = status::nothing; return true; } @@ -520,16 +347,6 @@ namespace dripline LERROR( dlog, "<" << f_name << "> Dripline exception caught while handling message: " << e.what() ); throw; } - catch( amqp_exception& e ) - { - LERROR( dlog, "<" << f_name << "> AMQP exception caught while handling message: (" << e.reply_code() << ") " << e.reply_text() ); - throw; - } - catch( amqp_lib_exception& e ) - { - LERROR( dlog, "<" << f_name << "> AMQP Library Exception caught while handling message: (" << e.ErrorCode() << ") " << e.what() ); - throw; - } catch( std::exception& e ) { LERROR( dlog, "<" << f_name << "> Standard exception caught while handling message: " << e.what() ); @@ -539,20 +356,6 @@ namespace dripline return; } - void service::send_reply( reply_ptr_t a_reply ) const - { - LDEBUG( dlog, "Sending reply message to <" << a_reply->routing_key() << ">:\n" << - " Return code: " << a_reply->get_return_code() << '\n' << - " Return message: " << a_reply->return_message() << '\n' << - " Payload:\n" << a_reply->payload() ); - - if( ! send( a_reply ) ) - { - LWARN( dlog, "Something went wrong while sending the reply" ); - } - return; - } - reply_ptr_t service::on_request_message( request_ptr_t a_request ) { std::string t_first_token( a_request->routing_key() ); diff --git a/library/service.hh b/library/service.hh index b0a31f78..706e72e2 100644 --- a/library/service.hh +++ b/library/service.hh @@ -11,9 +11,8 @@ #include "core.hh" #include "endpoint.hh" #include "heartbeater.hh" -#include "scheduler.hh" -#include "listener.hh" #include "receiver.hh" +#include "scheduler.hh" #include "dripline_exceptions.hh" #include "service_config.hh" @@ -79,7 +78,7 @@ namespace dripline class DRIPLINE_API service : public core, public endpoint, - public listener_receiver, + public concurrent_receiver, public heartbeater, public scheduler<> { @@ -147,14 +146,14 @@ namespace dripline bool add_async_child( endpoint_ptr_t a_endpoint_ptr ); public: - /// Sends a request message and returns a channel on which to listen for a reply. - virtual sent_msg_pkg_ptr send( request_ptr_t a_request, amqp_channel_ptr a_channel = amqp_channel_ptr() ) const; + /// Sends a request message + virtual sent_msg_pkg_ptr send( request_ptr_t a_request ) const; /// Sends a reply message - virtual sent_msg_pkg_ptr send( reply_ptr_t a_reply, amqp_channel_ptr a_channel = amqp_channel_ptr() ) const; + virtual sent_msg_pkg_ptr send( reply_ptr_t a_reply ) const; /// Sends an alert message - virtual sent_msg_pkg_ptr send( alert_ptr_t a_alert, amqp_channel_ptr a_channel = amqp_channel_ptr() ) const; + virtual sent_msg_pkg_ptr send( alert_ptr_t a_alert ) const; public: /** @@ -182,34 +181,13 @@ namespace dripline /// If this returns false, the service should quit with an error bool stop(); - protected: - virtual bool open_channels(); - - virtual bool setup_queues(); - - virtual bool bind_keys(); - - virtual bool start_consuming(); - - virtual bool stop_consuming(); - - virtual bool remove_queue(); - - public: - /// Waits for AMQP messages arriving on the channel - /// Returns false if the return is due to an error in this function; returns true otherwise (namely because it was canceled) - virtual bool listen_on_queue(); - - /// Sends a reply message - virtual void send_reply( reply_ptr_t a_reply ) const; - mv_accessible( uuid_t, id ); public: typedef std::map< std::string, endpoint_ptr_t > sync_map_t; mv_referrable( sync_map_t, sync_children ); - typedef std::map< std::string, lr_ptr_t > async_map_t; + typedef std::map< std::string, elr_ptr_t > async_map_t; mv_referrable( async_map_t, async_children ); mv_referrable( std::string, broadcast_key ); @@ -224,28 +202,22 @@ namespace dripline virtual void do_cancellation( int a_code ); }; - inline sent_msg_pkg_ptr service::send( request_ptr_t a_request, amqp_channel_ptr a_channel ) const + inline sent_msg_pkg_ptr service::send( request_ptr_t a_request ) const { a_request->sender_service_name() = f_name; - // we don't use f_channel on this core::send command because a channel can only be used in a single thread, - // and f_channel is primarily meant for listening with the listener thread. - return core::send( a_request, a_channel ); + return core::send( a_request ); } - inline sent_msg_pkg_ptr service::send( reply_ptr_t a_reply, amqp_channel_ptr a_channel ) const + inline sent_msg_pkg_ptr service::send( reply_ptr_t a_reply ) const { - a_reply->sender_service_name() = f_name ; - // we don't use f_channel on this core::send command because a channel can only be used in a single thread, - // and f_channel is primarily meant for listening with the listener thread. - return core::send( a_reply, a_channel ); + a_reply->sender_service_name() = f_name; + return core::send( a_reply ); } - inline sent_msg_pkg_ptr service::send( alert_ptr_t a_alert, amqp_channel_ptr a_channel ) const + inline sent_msg_pkg_ptr service::send( alert_ptr_t a_alert ) const { a_alert->sender_service_name() = f_name; - // we don't use f_channel on this core::send command because a channel can only be used in a single thread, - // and f_channel is primarily meant for listening with the listener thread. - return core::send( a_alert, a_channel ); + return core::send( a_alert ); } } /* namespace dripline */ From 56ddfb31e8864c785c279483be467b638eb4be7e Mon Sep 17 00:00:00 2001 From: Noah Oblath Date: Thu, 28 May 2026 13:36:01 -0700 Subject: [PATCH 10/45] Updated monitor --- library/monitor.cc | 127 +++++++++------------------------------------ library/monitor.hh | 10 +--- 2 files changed, 26 insertions(+), 111 deletions(-) diff --git a/library/monitor.cc b/library/monitor.cc index 5ac79902..9c0fc094 100644 --- a/library/monitor.cc +++ b/library/monitor.cc @@ -15,6 +15,9 @@ #include "logger.hh" #include "signal_handler.hh" +#include +#include + LOGGER( dlog, "monitor" ); namespace dripline @@ -23,7 +26,7 @@ namespace dripline monitor::monitor( const scarab::param_node& a_config, const scarab::authentication& a_auth ) : scarab::cancelable(), core( a_config["dripline_mesh"].as_node(), a_auth ), - listener_receiver(), + concurrent_receiver(), f_status( status::nothing ), f_name( std::string("monitor_") + string_from_uuid(generate_random_uuid()) ), f_json_print( false ), @@ -94,24 +97,22 @@ namespace dripline LINFO( dlog, "Connecting to <" << f_address << ":" << f_port << ">" ); - LDEBUG( dlog, "Opening channel for message monitor <" << f_name << ">" ); - f_channel = open_channel(); - if( ! f_channel ) return false; + try + { + open_connection(); + } + catch( connection_error& e ) + { + LERROR( dlog, "Unable to connect to the broker: " << e.what() ); + return false; + } f_status = status::channel_created; - if( ! setup_exchange( f_channel, f_requests_exchange ) ) return false; - if( ! setup_exchange( f_channel, f_alerts_exchange ) ) return false; - f_status = status::exchange_declared; - - LDEBUG( dlog, "Setting up queue for message monitor <" << f_name << ">" ); - if( ! setup_queue( f_channel, f_name ) ) return false; - f_status = status::queue_declared; - - if( ! bind_keys() ) return false; - f_status = status::queue_bound; - - f_consumer_tag = start_consuming( f_channel, f_name ); - if( f_consumer_tag.empty() ) return false; + // TODO (Phase 6): set up monitor topology, declare queue, bind routing keys + // - declare monitor queue (f_name) + // - bind each request key to the requests exchange + // - bind each alerts key to the alerts exchange + // - call start_listening( f_vhost, topology, queue_handle, f_name ) f_status = status::consuming; return true; @@ -134,16 +135,17 @@ namespace dripline { f_receiver_thread = std::thread( &concurrent_receiver::execute, this ); - if( ! listen_on_queue() ) + // Block until canceled + while( ! is_canceled() ) { - throw dripline_error() << "Something went wrong while listening for messages"; + std::this_thread::sleep_for( std::chrono::milliseconds( 100 ) ); } f_receiver_thread.join(); } catch( std::system_error& e ) { - LERROR( dlog, "Could not start the a thread due to a system error: " << e.what() ); + LERROR( dlog, "Could not start a thread due to a system error: " << e.what() ); return false; } catch( dripline_error& e ) @@ -165,94 +167,15 @@ namespace dripline { LINFO( dlog, "Stopping message monitor <" << f_name << ">" ); - if( f_status >= status::listening ) // listening + if( f_status >= status::listening ) { this->cancel( dl_success().rc_value() ); f_status = status::consuming; } - if( f_status >= status::queue_bound ) // queue_bound or consuming - { - if( ! stop_consuming( f_channel, f_consumer_tag ) ) return false; - f_status = status::queue_bound; - } + stop_listening(); - if( f_status >= status::queue_declared ) // queue_declared or queue_bound - { - if( ! remove_queue( f_channel, f_name ) ) return false; - f_status = status::exchange_declared; - } - - return true; - } - - bool monitor::bind_keys() - { - LDEBUG( dlog, "Binding request keys for message monitor <" << f_name << ">" ); - for( auto t_req_key_it = f_requests_keys.begin(); t_req_key_it != f_requests_keys.end(); ++t_req_key_it ) - { - if( ! bind_key( f_channel, f_requests_exchange, f_name, *t_req_key_it ) ) return false; - } - - LDEBUG( dlog, "Binding alerts keys for message monitor <" << f_name << ">" ); - for( auto t_al_key_it = f_alerts_keys.begin(); t_al_key_it != f_alerts_keys.end(); ++t_al_key_it ) - { - if( ! bind_key( f_channel, f_alerts_exchange, f_name, *t_al_key_it ) ) return false; - } - - return true; - } - - bool monitor::listen_on_queue() - { - LINFO( dlog, "Listening for incoming messages on <" << f_name << ">" ); - - while( ! is_canceled() ) - { - amqp_envelope_ptr t_envelope; - core::post_listen_status t_post_listen_status = core::post_listen_status::unknown; - core::listen_for_message( t_envelope, t_post_listen_status, f_channel, f_consumer_tag, f_listen_timeout_ms ); - - if( f_canceled.load() ) - { - LDEBUG( dlog, "Monitor <" << f_name << "> canceled" ); - return true; - } - - if( t_post_listen_status == core::post_listen_status::timeout ) - { - // we end up here every time the listen times out with no message received - continue; - } - - if( t_post_listen_status == core::post_listen_status::soft_error ) - { - LWARN( dlog, "A soft error ocurred while listening for messages for monitor <" << f_name << ">. The channel is still valid" ); - continue; - } - - if( t_post_listen_status == core::post_listen_status::hard_error ) - { - LERROR( dlog, "A hard error ocurred while listening for messages for monitor <" << f_name << ">. The channel is no longer valid" ); - return false; - } - - if( t_post_listen_status == core::post_listen_status::unknown ) - { - LERROR( dlog, "An unknown status occurred while listening for messages for monitor <" << f_name << ">" ); - return false; - } - - // remaining status is core::post_listen_status::message_received - - handle_message_chunk( t_envelope ); - - if( f_canceled.load() ) - { - LDEBUG( dlog, "Monitor <" << f_name << "> canceled" ); - return true; - } - } + f_status = status::nothing; return true; } diff --git a/library/monitor.hh b/library/monitor.hh index 771f8c7e..df80fece 100644 --- a/library/monitor.hh +++ b/library/monitor.hh @@ -9,7 +9,6 @@ #define DRIPLINE_MONITOR_HH_ #include "core.hh" -#include "listener.hh" #include "receiver.hh" namespace scarab @@ -43,7 +42,7 @@ namespace DRIPLINE_API dripline */ class monitor : public core, - public listener_receiver + public concurrent_receiver { protected: enum class status @@ -93,14 +92,7 @@ namespace DRIPLINE_API dripline /// Stops listening for messages and closes the AMQP connection. bool stop(); - protected: - bool bind_keys(); - public: - /// Waits for a single AMQP message and processes it. - /// Returns false if the return is due to an error in this function; returns true otherwise (namely because it was canceled) - virtual bool listen_on_queue(); - /// Handles a single Dripline message by printing it to stdout. /// Printing is done via a prog-level message in the logger. virtual void submit_message( message_ptr_t a_message ); From 9d67226a8ed24625c61c96c3ec42bfb7c7e7d974 Mon Sep 17 00:00:00 2001 From: Noah Oblath Date: Thu, 28 May 2026 13:52:30 -0700 Subject: [PATCH 11/45] Updated agent and relayer, and cleaned up a few other outdated things. --- library/agent.cc | 24 ++++-------------------- library/dripline_fwd.hh | 6 ------ library/receiver.hh | 13 +++++-------- library/relayer.cc | 8 +------- library/relayer.hh | 9 --------- library/service.cc | 20 +------------------- library/service.hh | 2 +- 7 files changed, 12 insertions(+), 70 deletions(-) diff --git a/library/agent.cc b/library/agent.cc index 88002b66..85be2ce1 100644 --- a/library/agent.cc +++ b/library/agent.cc @@ -208,16 +208,15 @@ namespace dripline return; } - if( ! t_receive_reply->f_consumer_tag.empty() ) // this indicates that the reply queue was created, and we've started consuming on it; we should wait for a reply + if( t_receive_reply->f_reply_consumer ) // this indicates that the reply queue was created, and we've started consuming on it; we should wait for a reply { LINFO( dlog, "Waiting for a reply from the server; use ctrl-c to cancel" ); // timed blocking call to wait for incoming message receiver t_msg_receiver; - core::post_listen_status t_post_listen_status = core::post_listen_status::unknown; auto t_rec_cancel_wrap = wrap_cancelable( t_msg_receiver ); scarab::signal_handler::add_cancelable( t_rec_cancel_wrap ); - dripline::reply_ptr_t t_reply = t_msg_receiver.wait_for_reply( t_receive_reply, t_post_listen_status, f_agent->get_timeout() ); + dripline::reply_ptr_t t_reply = t_msg_receiver.wait_for_reply( t_receive_reply, f_agent->get_timeout() ); if( t_msg_receiver.is_canceled() ) { @@ -266,23 +265,8 @@ namespace dripline } else { - if( t_post_listen_status == core::post_listen_status::timeout ) - { - LWARN( dlog, "Timed out or while waiting for reply" ); - f_agent->set_return( dl_client_error_timeout().rc_value() ); - } - else - { - if( t_post_listen_status == core::post_listen_status::hard_error ) - { - LERROR( dlog, "Error while waiting for reply" ); - } - else - { - LERROR( dlog, "Unknown state while waiting for reply: " << (int)t_post_listen_status ); - } - f_agent->set_return( dl_client_error().rc_value() ); - } + LWARN( dlog, "Timed out or error while waiting for reply" ); + f_agent->set_return( dl_client_error_timeout().rc_value() ); } f_agent->set_reply( t_reply ); } diff --git a/library/dripline_fwd.hh b/library/dripline_fwd.hh index 5f76fd4e..839494d2 100644 --- a/library/dripline_fwd.hh +++ b/library/dripline_fwd.hh @@ -27,12 +27,6 @@ namespace dripline struct sent_msg_pkg; typedef std::shared_ptr< sent_msg_pkg > sent_msg_pkg_ptr; - class listener; - typedef std::shared_ptr< listener > listener_ptr_t; - - class listener_receiver; - typedef std::shared_ptr< listener_receiver > lr_ptr_t; - class endpoint_listener_receiver; typedef std::shared_ptr< endpoint_listener_receiver > elr_ptr_t; diff --git a/library/receiver.hh b/library/receiver.hh index 7d526070..15c8f6c1 100644 --- a/library/receiver.hh +++ b/library/receiver.hh @@ -131,17 +131,14 @@ namespace dripline @brief Receives and processes messages concurrently @details - This class enables Dripline messages to be received and processed concurrently. It is intended to be used in - conjuction with a listener, as in @ref listener_receiver. + This class enables Dripline messages to be received and processed concurrently. - The typical use case involves three threads: - 1. A listener gets messages from the AMQP channel (using `listen_on_queue(), e.g. @ref service or @ref endpoint_listener_receiver) and + The typical use case involves two threads: + 1. A consumer callback (set up via `start_listening()`) receives messages from the AMQP broker and calls `receiver::handle_message_chunk()` - 2. A receiver has a timing thread waiting for multiple message chunks (if relevant); - when the message is complete, `concurrent_receiver::process_message()` is called, which deposits the message in a concurrent queue. - 3. A concurrent_receiver picks up the complete message from the concurrent queue, and processes the message using `submit_message()`. + 2. A concurrent_receiver picks up the complete message from the concurrent queue (via `execute()`), and processes the message using `submit_message()`. - The `execute()` function implements thread 3. + The `execute()` function implements thread 2. A class deriving from concurrent_receiver must implement `submit_message()`. */ diff --git a/library/relayer.cc b/library/relayer.cc index c5fc0509..5ec72a9d 100644 --- a/library/relayer.cc +++ b/library/relayer.cc @@ -129,12 +129,6 @@ namespace dripline } reply_ptr_t relayer::wait_for_reply( const wait_for_send_pkg_ptr a_receive_reply, int a_timeout_ms ) - { - core::post_listen_status t_temp = core::post_listen_status::unknown; - return wait_for_reply( a_receive_reply, t_temp, a_timeout_ms ); - } - - reply_ptr_t relayer::wait_for_reply( const wait_for_send_pkg_ptr a_receive_reply, core::post_listen_status& a_status, int a_timeout_ms ) { std::unique_lock< std::mutex > t_lock( a_receive_reply->f_mutex ); auto t_deadline = std::chrono::system_clock::now() + std::chrono::milliseconds( a_timeout_ms ); @@ -147,7 +141,7 @@ namespace dripline return reply_ptr_t(); } } - return f_msg_receiver.wait_for_reply( a_receive_reply->f_sent_msg_pkg_ptr, a_status, a_timeout_ms ); + return f_msg_receiver.wait_for_reply( a_receive_reply->f_sent_msg_pkg_ptr, a_timeout_ms ); } } diff --git a/library/relayer.hh b/library/relayer.hh index cb57f3fb..27593cf9 100644 --- a/library/relayer.hh +++ b/library/relayer.hh @@ -86,15 +86,6 @@ namespace dripline @return Reply message */ reply_ptr_t wait_for_reply( const wait_for_send_pkg_ptr a_receive_reply, int a_timeout_ms = 0 ); - /*! - User interface for waiting for a reply message. - This can be called multiple times to receive multiple replies. - @param[in] a_receive_reply The sent-message package from the request. - @param[in] a_timeout_ms Timeout for waiting for a reply; if it's 0, there will be no timeout. - @param[out] a_status Returns the output status after receiving a message (or failing to do so). - @return Reply message - */ - reply_ptr_t wait_for_reply( const wait_for_send_pkg_ptr a_receive_reply, core::post_listen_status& a_status, int a_timeout_ms = 0 ); private: void do_cancellation( int a_code ); diff --git a/library/service.cc b/library/service.cc index 499880bd..49323c49 100644 --- a/library/service.cc +++ b/library/service.cc @@ -53,24 +53,6 @@ namespace dripline // default of f_heartbeat_interval_s is in the heartbeater class f_heartbeat_interval_s = a_config.get_value( "heartbeat_interval_s", f_heartbeat_interval_s ); } -/* - service::service( const bool a_make_connection, const scarab::param_node& a_config, const scarab::authentication& a_auth ) : - scarab::cancelable(), - core( a_make_connection, a_config ), - endpoint( "" ), - listener_receiver(), - heartbeater(), - scheduler<>(), - std::enable_shared_from_this< service >(), - f_status( status::nothing ), - f_enable_scheduling( a_config.get_value("enable-scheduling", false ) ), - f_id( generate_random_uuid() ), - f_sync_children(), - f_async_children(), - f_broadcast_key() - { - } -*/ service::~service() { @@ -141,7 +123,7 @@ namespace dripline { unsigned n_failures = 0; bool t_do_repeat = true; // start true so that we get into the repeat loop - // Repeat loop for listening: we may call to listen_on_queue() multiple times + // Repeat loop for restarting on connection failure while( t_do_repeat ) { t_do_repeat = false; // set false because we'll only do the repeat based on the conditions below diff --git a/library/service.hh b/library/service.hh index 706e72e2..0e67c9e3 100644 --- a/library/service.hh +++ b/library/service.hh @@ -56,7 +56,7 @@ namespace dripline A service has a number of key characteristics (most of which come from its parent classes): * `core` -- Has all of the basic AMQP capabilities, sending messages, and making and manipulating connections * `endpoint` -- Handles Dripline messages - * `listener_receiver` -- Asynchronously recieves AMQP messages and turns them into Dripline messages + * `concurrent_receiver` -- Asynchronously receives AMQP messages and turns them into Dripline messages * `heartbeater` -- Sends periodic heartbeat messages * `scheduler` -- Can schedule events From b39f5cf27fe950ff5241941b45996b7811a9f566 Mon Sep 17 00:00:00 2001 From: Noah Oblath Date: Thu, 28 May 2026 20:12:26 -0700 Subject: [PATCH 12/45] Updated message.hh/cc to translate messages to and from rmqcpp --- library/message.cc | 119 +++++++++++++++++++++++++++------------------ library/message.hh | 22 ++++----- 2 files changed, 83 insertions(+), 58 deletions(-) diff --git a/library/message.cc b/library/message.cc index 67d4f48b..f7d44598 100644 --- a/library/message.cc +++ b/library/message.cc @@ -21,6 +21,9 @@ #include "time.hh" #include "version_wrapper.hh" +#include "bsl_vector.h" +#include "rmqt_properties.h" + #include #include @@ -101,16 +104,6 @@ namespace dripline } } -/* - message_ptr_t message::process_envelope( amqp_envelope_ptr a_envelope ) - { - if( ! a_envelope ) - { - throw dripline_error() << "Empty envelope received"; - } - return message::process_message( a_envelope->Message(), a_envelope->RoutingKey() ); - } -*/ std::tuple< std::string, unsigned, unsigned > message::parse_message_id( const string& a_message_id ) { std::string::size_type t_first_separator = a_message_id.find_first_of( s_message_id_separator ); @@ -152,16 +145,18 @@ namespace dripline throw dripline_error() << "All messages provided for processing were invalid"; } - unsigned t_payload_chunk_length = t_first_valid_message->Body().size(); + unsigned t_payload_chunk_length = t_first_valid_message->payloadSize(); encoding t_encoding; - if( t_first_valid_message->ContentEncoding() == "application/json" ) + const std::string t_content_encoding = t_first_valid_message->properties().contentEncoding.isNull() ? "" : + std::string( t_first_valid_message->properties().contentEncoding.value() ); + if( t_content_encoding == "application/json" ) { t_encoding = encoding::json; } else { - throw dripline_error() << "Unable to parse message with content type <" << t_first_valid_message->ContentEncoding() << ">"; + throw dripline_error() << "Unable to parse message with content type <" << t_content_encoding << ">"; } // Build up the body @@ -177,7 +172,7 @@ namespace dripline continue; } - t_payload_str += t_message->Body(); + t_payload_str += std::string( reinterpret_cast< const char* >( t_message->payload() ), t_message->payloadSize() ); } // Attempt to parse @@ -214,28 +209,46 @@ namespace dripline using scarab::at; - using AmqpClient::Table; - using AmqpClient::TableEntry; - using AmqpClient::TableValue; - Table t_properties = t_first_valid_message->HeaderTable(); + const auto& t_hdr_ptr = t_first_valid_message->headers(); + const BloombergLP::rmqt::FieldTable& t_properties = t_hdr_ptr ? *t_hdr_ptr : BloombergLP::rmqt::FieldTable(); + + auto t_hdr_uint = [&t_properties]( const char* a_key, unsigned a_default ) -> unsigned { + auto it = t_properties.find( bsl::string(a_key) ); + if( it == t_properties.end() || !it->second.is< bsl::uint32_t >() ) return a_default; + return (unsigned)it->second.the< bsl::uint32_t >(); + }; + auto t_hdr_str = [&t_properties]( const char* a_key, const char* a_default ) -> std::string { + auto it = t_properties.find( bsl::string(a_key) ); + if( it == t_properties.end() || !it->second.is< bsl::string >() ) return a_default; + return std::string( it->second.the< bsl::string >() ); + }; + auto t_hdr_table = [&t_properties]( const char* a_key ) -> const BloombergLP::rmqt::FieldTable* { + auto it = t_properties.find( bsl::string(a_key) ); + if( it == t_properties.end() || + !it->second.is< bsl::shared_ptr< BloombergLP::rmqt::FieldTable > >() ) return nullptr; + const auto& p = it->second.the< bsl::shared_ptr< BloombergLP::rmqt::FieldTable > >(); + return p ? p.get() : nullptr; + }; // Create the message, of whichever type message_ptr_t t_message; - msg_t t_msg_type = to_msg_t( at( t_properties, std::string("message_type"), TableValue(to_uint(msg_t::unknown)) ).GetInteger() ); + msg_t t_msg_type = to_msg_t( t_hdr_uint("message_type", to_uint(msg_t::unknown)) ); switch( t_msg_type ) { case msg_t::request: { + const std::string t_reply_to = t_first_valid_message->properties().replyTo.isNull() ? "" : + std::string( t_first_valid_message->properties().replyTo.value() ); request_ptr_t t_request = msg_request::create( std::move(t_payload), - to_op_t( at( t_properties, std::string("message_operation"), TableValue(to_uint(op_t::unknown)) ).GetInteger() ), + to_op_t( t_hdr_uint("message_operation", to_uint(op_t::unknown)) ), a_routing_key, - at( t_properties, std::string("specifier"), TableValue("") ).GetString(), - t_first_valid_message->ReplyTo(), + t_hdr_str("specifier", ""), + t_reply_to, t_encoding); bool t_lockout_key_valid = true; - t_request->lockout_key() = uuid_from_string( at( t_properties, std::string("lockout_key"), TableValue("") ).GetString(), t_lockout_key_valid ); + t_request->lockout_key() = uuid_from_string( t_hdr_str("lockout_key", ""), t_lockout_key_valid ); t_request->set_lockout_key_valid( t_lockout_key_valid ); t_message = t_request; @@ -244,11 +257,11 @@ namespace dripline case msg_t::reply: { reply_ptr_t t_reply = msg_reply::create( - at( t_properties, std::string("return_code"), TableValue(999U) ).GetInteger(), - at( t_properties, std::string("return_message"), TableValue("") ).GetString(), + t_hdr_uint("return_code", 999U), + t_hdr_str("return_message", ""), std::move(t_payload), a_routing_key, - at( t_properties, std::string("specifier"), TableValue("") ).GetString(), + t_hdr_str("specifier", ""), t_encoding); t_message = t_reply; @@ -259,7 +272,7 @@ namespace dripline alert_ptr_t t_alert = msg_alert::create( std::move(t_payload), a_routing_key, - at( t_properties, std::string("specifier"), TableValue("") ).GetString(), + t_hdr_str("specifier", ""), t_encoding); t_message = t_alert; @@ -278,17 +291,19 @@ namespace dripline t_message->set_is_valid( false ); } - t_message->correlation_id() = t_first_valid_message->CorrelationId(); - t_message->message_id() = t_first_valid_message->MessageId(); + t_message->correlation_id() = t_first_valid_message->properties().correlationId.isNull() ? "" : + std::string( t_first_valid_message->properties().correlationId.value() ); + t_message->message_id() = std::string( t_first_valid_message->messageId() ); // remove the message chunk information from the message id t_message->message_id() = t_message->message_id().substr( 0, t_message->message_id().find_first_of(s_message_id_separator) ); - t_message->timestamp() = at( t_properties, std::string("timestamp"), TableValue("") ).GetString(); + t_message->timestamp() = t_hdr_str("timestamp", ""); - Table t_sender_info = at( t_properties, std::string("sender_info"), TableValue(Table()) ).GetTable(); - scarab::param_ptr_t t_sender_info_param = table_to_param( t_sender_info ); - t_message->set_sender_info( t_sender_info_param->as_node() ); - - t_message->payload() = *t_payload; + const BloombergLP::rmqt::FieldTable* t_si_tbl = t_hdr_table("sender_info"); + if( t_si_tbl ) + { + scarab::param_ptr_t t_sender_info_param = table_to_param( *t_si_tbl ); + t_message->set_sender_info( t_sender_info_param->as_node() ); + } return t_message; } @@ -315,22 +330,32 @@ namespace dripline unsigned i_chunk = 0; for( string& t_body_part : t_body_parts ) { - amqp_message_ptr t_message = AmqpClient::BasicMessage::Create( t_body_part ); + // Build raw payload bytes + auto t_raw_data = bsl::make_shared< bsl::vector< bsl::uint8_t > >( + t_body_part.begin(), t_body_part.end() ); + + // Build AMQP properties + BloombergLP::rmqt::Properties t_props; + t_props.contentEncoding = bsl::string( interpret_encoding() ); + t_props.correlationId = bsl::string( f_correlation_id ); + t_props.messageId = bsl::string( t_base_message_id + std::to_string(i_chunk) + t_total_chunks_str ); + if( ! f_reply_to.empty() ) + { + t_props.replyTo = bsl::string( f_reply_to ); + } - t_message->ContentEncoding( interpret_encoding() ); - t_message->CorrelationId( f_correlation_id ); - t_message->MessageId( t_base_message_id + std::to_string(i_chunk) + t_total_chunks_str ); - t_message->ReplyTo( f_reply_to ); + // Build header FieldTable + auto t_headers = bsl::make_shared< BloombergLP::rmqt::FieldTable >(); + (*t_headers)[bsl::string("message_type")] = BloombergLP::rmqt::FieldValue( (bsl::uint32_t)to_uint(message_type()) ); + (*t_headers)[bsl::string("specifier")] = BloombergLP::rmqt::FieldValue( bsl::string(f_specifier.to_string()) ); + (*t_headers)[bsl::string("timestamp")] = BloombergLP::rmqt::FieldValue( bsl::string(f_timestamp) ); + (*t_headers)[bsl::string("sender_info")] = param_to_table( get_sender_info() ); - AmqpClient::Table t_properties; - t_properties.insert( AmqpClient::TableEntry( "message_type", to_uint(message_type()) ) ); - t_properties.insert( AmqpClient::TableEntry( "specifier", f_specifier.to_string() ) ); - t_properties.insert( AmqpClient::TableEntry( "timestamp", f_timestamp ) ); - t_properties.insert( AmqpClient::TableEntry( "sender_info", param_to_table( get_sender_info() ) ) ); + this->derived_modify_amqp_message( *t_headers ); - this->derived_modify_amqp_message( t_message, t_properties ); + t_props.headers = t_headers; - t_message->HeaderTable( t_properties ); + amqp_message_ptr t_message = bsl::make_shared< BloombergLP::rmqt::Message >( t_raw_data, t_props ); t_message_parts[i_chunk] = t_message; diff --git a/library/message.hh b/library/message.hh index c6fe091b..b8c46a66 100644 --- a/library/message.hh +++ b/library/message.hh @@ -88,7 +88,7 @@ namespace dripline std::string encode_full_message( unsigned a_max_size, const scarab::param_node& a_options = scarab::param_node() ) const; protected: - virtual void derived_modify_amqp_message( amqp_message_ptr a_amqp_msg, AmqpClient::Table& a_properties ) const = 0; + virtual void derived_modify_amqp_message( BloombergLP::rmqt::FieldTable& a_headers ) const = 0; virtual void derived_modify_message_param( scarab::param_node& a_message_node ) const = 0; @@ -200,7 +200,7 @@ namespace dripline reply_ptr_t reply( const unsigned a_return_code, const std::string& a_ret_msg, scarab::param_ptr_t a_payload = scarab::param_ptr_t( new scarab::param() ) ) const; private: - void derived_modify_amqp_message( amqp_message_ptr a_amqp_msg, AmqpClient::Table& a_properties ) const; + void derived_modify_amqp_message( BloombergLP::rmqt::FieldTable& a_headers ) const; virtual void derived_modify_message_param( scarab::param_node& a_message_node ) const; public: @@ -261,7 +261,7 @@ namespace dripline bool is_alert() const; private: - void derived_modify_amqp_message( amqp_message_ptr a_amqp_msg, AmqpClient::Table& a_properties ) const; + void derived_modify_amqp_message( BloombergLP::rmqt::FieldTable& a_headers ) const; virtual void derived_modify_message_param( scarab::param_node& a_message_node ) const; public: @@ -315,7 +315,7 @@ namespace dripline bool is_alert() const; private: - void derived_modify_amqp_message( amqp_message_ptr a_amqp_msg, AmqpClient::Table& a_properties ) const; + void derived_modify_amqp_message( BloombergLP::rmqt::FieldTable& a_headers ) const; virtual void derived_modify_message_param( scarab::param_node& a_message_node ) const; public: @@ -381,10 +381,10 @@ namespace dripline return false; } - inline void msg_request::derived_modify_amqp_message( amqp_message_ptr /*a_amqp_msg*/, AmqpClient::Table& a_properties ) const + inline void msg_request::derived_modify_amqp_message( BloombergLP::rmqt::FieldTable& a_headers ) const { - a_properties.insert( AmqpClient::TableEntry( "message_operation", AmqpClient::TableValue(to_uint(f_message_operation)) ) ); - a_properties.insert( AmqpClient::TableEntry( "lockout_key", AmqpClient::TableValue(string_from_uuid(lockout_key())) ) ); + a_headers[bsl::string("message_operation")] = BloombergLP::rmqt::FieldValue( (bsl::uint32_t)to_uint(f_message_operation) ); + a_headers[bsl::string("lockout_key")] = BloombergLP::rmqt::FieldValue( bsl::string(string_from_uuid(lockout_key())) ); return; } @@ -435,10 +435,10 @@ namespace dripline return false; } - inline void msg_reply::derived_modify_amqp_message( amqp_message_ptr, AmqpClient::Table& a_properties ) const + inline void msg_reply::derived_modify_amqp_message( BloombergLP::rmqt::FieldTable& a_headers ) const { - a_properties.insert( AmqpClient::TableEntry( "return_code", AmqpClient::TableValue(f_return_code) ) ); - a_properties.insert( AmqpClient::TableEntry( "return_message", AmqpClient::TableValue(f_return_message) ) ); + a_headers[bsl::string("return_code")] = BloombergLP::rmqt::FieldValue( (bsl::uint32_t)f_return_code ); + a_headers[bsl::string("return_message")] = BloombergLP::rmqt::FieldValue( bsl::string(f_return_message) ); return; } @@ -467,7 +467,7 @@ namespace dripline return true; } - inline void msg_alert::derived_modify_amqp_message( amqp_message_ptr, AmqpClient::Table& ) const + inline void msg_alert::derived_modify_amqp_message( BloombergLP::rmqt::FieldTable& ) const { return; } From 8fb87a136104ff3939b701b873a8aa58c1e62d3f Mon Sep 17 00:00:00 2001 From: Noah Oblath Date: Thu, 28 May 2026 20:56:58 -0700 Subject: [PATCH 13/45] Completed the next phase of updates, removing execution in threads and updating testing --- library/monitor.cc | 4 - library/receiver.cc | 191 ++++++++++------------------ library/receiver.hh | 36 ++---- library/service.cc | 14 -- testing/test_concurrent_receiver.cc | 30 +---- testing/test_service.cc | 22 +--- 6 files changed, 88 insertions(+), 209 deletions(-) diff --git a/library/monitor.cc b/library/monitor.cc index 9c0fc094..7503d164 100644 --- a/library/monitor.cc +++ b/library/monitor.cc @@ -133,15 +133,11 @@ namespace dripline try { - f_receiver_thread = std::thread( &concurrent_receiver::execute, this ); - // Block until canceled while( ! is_canceled() ) { std::this_thread::sleep_for( std::chrono::milliseconds( 100 ) ); } - - f_receiver_thread.join(); } catch( std::system_error& e ) { diff --git a/library/receiver.cc b/library/receiver.cc index 3296d896..e91ecf2e 100644 --- a/library/receiver.cc +++ b/library/receiver.cc @@ -16,7 +16,6 @@ #include "rmqp_messageguard.h" #include "logger.hh" -#include "signal_handler.hh" #include @@ -28,23 +27,18 @@ namespace dripline f_messages(), f_chunks_received(), f_routing_key(), - f_thread(), - f_mutex(), - f_conv(), - f_processing( false ) + f_creation_time(), + f_mutex() {} incoming_message_pack::incoming_message_pack( incoming_message_pack&& a_orig ) : f_messages( std::move(a_orig.f_messages) ), f_chunks_received( a_orig.f_chunks_received ), f_routing_key( std::move(a_orig.f_routing_key) ), - f_thread( std::move(a_orig.f_thread) ), - f_mutex(), - f_conv(), - f_processing( a_orig.f_processing.load() ) + f_creation_time( a_orig.f_creation_time ), + f_mutex() { a_orig.f_chunks_received = 0; - a_orig.f_processing.store( false ); } @@ -59,7 +53,6 @@ namespace dripline cancelable::operator=( std::move(a_orig) ); f_incoming_messages = std::move(a_orig.f_incoming_messages); f_single_message_wait_ms = a_orig.f_single_message_wait_ms; - f_reply_listen_timeout_ms = a_orig.f_reply_listen_timeout_ms; return *this; } @@ -68,65 +61,78 @@ namespace dripline try { amqp_message_ptr t_message = bsl::make_shared< BloombergLP::rmqt::Message >( a_envelope->message() ); - LDEBUG( dlog, "Received a message chunk <" << std::string( t_message->messageId() ) ); + LDEBUG( dlog, "Received a message chunk <" << std::string( t_message->messageId() ) << ">" ); + + auto t_parsed = message::parse_message_id( std::string( t_message->messageId() ) ); + const std::string t_message_id( std::get<0>(t_parsed) ); + const unsigned t_chunk_idx = std::get<1>(t_parsed); + const unsigned t_num_chunks = std::get<2>(t_parsed); + + amqp_split_message_ptrs t_complete_messages; + std::string t_complete_routing_key; + bool t_is_complete = false; - auto t_parsed_message_id = message::parse_message_id( std::string( t_message->messageId() ) ); - std::string t_message_id( std::get<0>(t_parsed_message_id) ); - if( incoming_messages().count( t_message_id ) == 0 ) { - // this path: first chunk for this message - LDEBUG( dlog, "This is the first chunk for this message; creating new message pack" ); - // create the new message_pack object - incoming_message_pack& t_pack = incoming_messages()[t_message_id]; - // set the f_messages vector to the expected size - t_pack.f_messages.resize( std::get<2>(t_parsed_message_id) ); - // put in place the first message chunk received - t_pack.f_messages[std::get<1>(t_parsed_message_id)] = t_message; - t_pack.f_routing_key = std::string( a_envelope->envelope().routingKey() ); - t_pack.f_chunks_received = 1; - - if( t_pack.f_messages.size() == 1 ) + std::lock_guard< std::mutex > t_map_lock( f_incoming_messages_mutex ); + + // Lazy stale sweep: evict entries older than f_single_message_wait_ms + auto t_now = std::chrono::steady_clock::now(); + for( auto it = f_incoming_messages.begin(); it != f_incoming_messages.end(); ) { - // if we only expect one chunk, we can bypass creating a separate thread, etc - LDEBUG( dlog, "Single-chunk message being sent directly to processing" ); - process_message_pack( t_pack, t_message_id ); + auto t_age_ms = std::chrono::duration_cast< std::chrono::milliseconds >( t_now - it->second.f_creation_time ).count(); + if( static_cast< unsigned >( t_age_ms ) > f_single_message_wait_ms ) + { + LWARN( dlog, "Evicting stale incomplete message <" << it->first << ">" ); + it = f_incoming_messages.erase( it ); + } + else + { + ++it; + } } - else + + // Find or insert the entry for this message + bool t_is_new = ( f_incoming_messages.find( t_message_id ) == f_incoming_messages.end() ); + incoming_message_pack& t_pack = f_incoming_messages[ t_message_id ]; + if( t_is_new ) { - // start the thread to wait for message chunks - t_pack.f_thread = std::thread([this, &t_pack, &t_parsed_message_id](){ wait_for_message(t_pack, std::get<0>(t_parsed_message_id)); }); - t_pack.f_thread.detach(); + LDEBUG( dlog, "First chunk for message <" << t_message_id << ">; creating new message pack" ); + t_pack.f_messages.resize( t_num_chunks ); + t_pack.f_routing_key = std::string( a_envelope->envelope().routingKey() ); + t_pack.f_creation_time = t_now; + t_pack.f_chunks_received = 0; } - } - else - { - // this path: have already received chunks from this message - LDEBUG( dlog, "This is not the first chunk for this message; adding to message pack" ); - incoming_message_pack& t_pack = incoming_messages()[std::get<0>(t_parsed_message_id)]; - if( t_pack.f_processing.load() ) + + // Store the chunk under per-entry mutex (lock order: map → entry) + std::unique_lock< std::mutex > t_entry_lock( t_pack.f_mutex ); + if( t_pack.f_messages[ t_chunk_idx ] ) { - LWARN( dlog, "Message <" << std::get<0>(t_parsed_message_id) << "> is already being processed\n" << - "Just received chunk " << std::get<1>(t_parsed_message_id) << " of " << std::get<2>(t_parsed_message_id) ); + LWARN( dlog, "Received duplicate chunk " << t_chunk_idx << " for message <" << t_message_id << ">; ignoring" ); } else { - // lock mutex to access f_messages - std::unique_lock< std::mutex > t_lock( t_pack.f_mutex ); - if( t_pack.f_messages[std::get<1>(t_parsed_message_id)] ) - { - LWARN( dlog, "Received duplicate message chunk for message <" << std::get<0>(t_parsed_message_id) << ">; chunk " << std::get<1>(t_parsed_message_id) ); - } - else - { - // add chunk to set of chunks - t_pack.f_messages[std::get<1>(t_parsed_message_id)] = t_message; - ++t_pack.f_chunks_received; - t_lock.unlock(); - // inform the message-processing thread it should check whether it has the complete message - t_pack.f_conv.notify_one(); - } + t_pack.f_messages[ t_chunk_idx ] = t_message; + ++t_pack.f_chunks_received; + LDEBUG( dlog, "Stored chunk " << t_chunk_idx << " for message <" << t_message_id << ">; " + << t_pack.f_chunks_received << " / " << t_pack.f_messages.size() << " received" ); + } + + if( t_pack.f_chunks_received == t_pack.f_messages.size() ) + { + // Message complete: extract data, release entry lock, then erase from map + t_complete_messages = std::move( t_pack.f_messages ); + t_complete_routing_key = std::move( t_pack.f_routing_key ); + t_entry_lock.unlock(); + f_incoming_messages.erase( t_message_id ); + t_is_complete = true; + LDEBUG( dlog, "Message <" << t_message_id << "> complete; processing" ); } - } // new/current message if/else block + } // map lock (and entry lock if not yet released) released here + + if( t_is_complete ) + { + process_message_pack( t_complete_messages, t_complete_routing_key ); + } } catch( dripline_error& e ) { @@ -140,49 +146,11 @@ namespace dripline return; } - void receiver::wait_for_message( incoming_message_pack& a_pack, const std::string& a_message_id ) + void receiver::process_message_pack( amqp_split_message_ptrs& a_messages, const std::string& a_routing_key ) { - std::unique_lock< std::mutex > t_lock( a_pack.f_mutex ); - - LDEBUG( dlog, "Waiting for message; chunks received: " << a_pack.f_chunks_received << " chunks expected: " << a_pack.f_messages.size() ); - - // if the message is already complete, submit it for processing - if( a_pack.f_chunks_received == a_pack.f_messages.size() ) - { - t_lock.release(); // process_message() will unlock the mutex before erasing the message pack - process_message_pack( a_pack, a_message_id ); - return; - } - - auto t_now = std::chrono::system_clock::now(); - while( a_pack.f_conv.wait_until( t_lock, t_now + std::chrono::milliseconds(f_single_message_wait_ms) ) == std::cv_status::no_timeout ) - { - // if the message is complete during the waiting period, submit it for processing - if( a_pack.f_chunks_received == a_pack.f_messages.size() ) - { - t_lock.release(); // process_message() will unlock the mutex before erasing the message pack - process_message_pack( a_pack, a_message_id ); - return; - } - } - - // once the waiting period is over, submit it whether it's complete or not - t_lock.release(); // process_message() will unlock the mutex before erasing the message pack - LWARN( dlog, "Timed out; message may be incomplete" ); - process_message_pack( a_pack, a_message_id ); - - return; - } - - void receiver::process_message_pack( incoming_message_pack& a_pack, const std::string& a_message_id ) - { - a_pack.f_processing.store( true ); try { - message_ptr_t t_message = message::process_message( a_pack.f_messages, a_pack.f_routing_key ); - - a_pack.f_mutex.unlock(); - incoming_messages().erase( a_message_id ); + message_ptr_t t_message = message::process_message( a_messages, a_routing_key ); // if the message is not valid at this point, continue processing it, and we'll deal with it in the endpoint class @@ -246,13 +214,11 @@ namespace dripline concurrent_receiver::concurrent_receiver() : receiver(), - f_message_queue(), f_consumer() {} concurrent_receiver::concurrent_receiver( concurrent_receiver&& a_orig ) : receiver( std::move(a_orig) ), - f_message_queue(), f_consumer( std::move(a_orig.f_consumer) ) {} @@ -268,31 +234,10 @@ namespace dripline void concurrent_receiver::process_message( message_ptr_t a_message ) { - f_message_queue.push( a_message ); + this->submit_message( a_message ); return; } - void concurrent_receiver::execute() - { - try - { - while( ! is_canceled() ) - { - message_ptr_t t_message; - if( f_message_queue.timed_wait_and_pop( t_message ) ) - { - this->submit_message( t_message ); - } - } - } - catch( const std::exception& e ) - { - // shutdown gracefully on an exception - LERROR( dlog, "Exception caught; shutting down.\n" << "\t" << e.what() ); - scarab::signal_handler::cancel_all( RETURN_ERROR ); - } - } - void concurrent_receiver::start_listening( bsl::shared_ptr< BloombergLP::rmqa::VHost > a_vhost, const BloombergLP::rmqa::Topology& a_topology, const BloombergLP::rmqt::QueueHandle& a_queue_handle, diff --git a/library/receiver.hh b/library/receiver.hh index 15c8f6c1..3458bb8f 100644 --- a/library/receiver.hh +++ b/library/receiver.hh @@ -15,13 +15,11 @@ #include "rmqa_topology.h" #include "cancelable.hh" -#include "concurrent_queue.hh" #include "member_variables.hh" -#include -#include +#include #include -#include +#include namespace dripline { @@ -36,10 +34,8 @@ namespace dripline amqp_split_message_ptrs f_messages; unsigned f_chunks_received; std::string f_routing_key; - std::thread f_thread; + std::chrono::steady_clock::time_point f_creation_time; std::mutex f_mutex; - std::condition_variable f_conv; - std::atomic< bool > f_processing; incoming_message_pack(); incoming_message_pack( const incoming_message_pack& ) = delete; incoming_message_pack( incoming_message_pack&& a_orig ); @@ -93,11 +89,8 @@ namespace dripline /// For single-chunk messages, processes the message immediately. void handle_message_chunk( amqp_envelope_ptr a_envelope ); - /// Waits for messages for a set amount of time (`single_message_wait_ms`), and submits the message pack for processing. - /// Intended to be used in a separate thread for each message pack. - void wait_for_message( incoming_message_pack& a_pack, const std::string& a_message_id ); /// Converts a message pack into a Dripline message, and then submits the message for processing. - void process_message_pack( incoming_message_pack& a_pack, const std::string& a_message_id ); + void process_message_pack( amqp_split_message_ptrs& a_messages, const std::string& a_routing_key ); /// Processes a single Dripline message. /// This is the default implementation that always throws a `dripline_error`. @@ -122,6 +115,9 @@ namespace dripline protected: // (no protected helpers currently) + private: + mutable std::mutex f_incoming_messages_mutex; + }; /*! @@ -131,14 +127,11 @@ namespace dripline @brief Receives and processes messages concurrently @details - This class enables Dripline messages to be received and processed concurrently. - - The typical use case involves two threads: - 1. A consumer callback (set up via `start_listening()`) receives messages from the AMQP broker and - calls `receiver::handle_message_chunk()` - 2. A concurrent_receiver picks up the complete message from the concurrent queue (via `execute()`), and processes the message using `submit_message()`. + This class enables Dripline messages to be received and processed. - The `execute()` function implements thread 2. + A consumer callback (set up via `start_listening()`) receives messages from the AMQP broker and + calls `receiver::handle_message_chunk()`, which assembles chunks and calls `submit_message()` + directly (synchronously in the rmqcpp callback thread). A class deriving from concurrent_receiver must implement `submit_message()`. */ @@ -154,12 +147,9 @@ namespace dripline concurrent_receiver& operator=( concurrent_receiver&& a_orig ); public: - /// Deposits the message in the concurrent queue (called by the listener) + /// Dispatches the message directly to `submit_message()`. virtual void process_message( message_ptr_t a_message ); - /// Handles messages that appear in the concurrent queue by calling `submit_message()`. - void execute(); - /// Creates an rmqcpp consumer on the given queue and begins receiving messages. /// Each received message is passed to handle_message_chunk(). void start_listening( bsl::shared_ptr< BloombergLP::rmqa::VHost > a_vhost, @@ -175,8 +165,6 @@ namespace dripline /// For a concrete example, see @ref service or @ref endpoint_listener_receiver. virtual void submit_message( message_ptr_t a_message ) = 0; - mv_referrable( scarab::concurrent_queue< message_ptr_t >, message_queue ); - mv_referrable( std::thread, receiver_thread ); bsl::shared_ptr< BloombergLP::rmqa::Consumer > f_consumer; }; diff --git a/library/service.cc b/library/service.cc index 49323c49..99c4901e 100644 --- a/library/service.cc +++ b/library/service.cc @@ -241,14 +241,10 @@ namespace dripline LINFO( dlog, "Scheduler disabled" ); } - LINFO( dlog, "Starting receiver thread" ); - f_receiver_thread = std::thread( &concurrent_receiver::execute, this ); - if( ! f_async_children.empty() ) { LINFO( dlog, "Starting async children" ); } else { LDEBUG( dlog, "No async children to start" ); } for( auto& t_child_pair : f_async_children ) { - t_child_pair.second->receiver_thread() = std::thread( &concurrent_receiver::execute, t_child_pair.second.get() ); // TODO (Phase 6): call t_child_pair.second->start_listening( f_vhost, topology, queue_handle, t_child_pair.first ) } @@ -258,16 +254,6 @@ namespace dripline std::this_thread::sleep_for( std::chrono::milliseconds( 100 ) ); } - for( auto& t_child_pair : f_async_children ) - { - if( t_child_pair.second->receiver_thread().joinable() ) - { - t_child_pair.second->receiver_thread().join(); - } - } - - f_receiver_thread.join(); - if( f_heartbeat_thread.joinable() ) { f_heartbeat_thread.join(); diff --git a/testing/test_concurrent_receiver.cc b/testing/test_concurrent_receiver.cc index a20fafa3..94d12195 100644 --- a/testing/test_concurrent_receiver.cc +++ b/testing/test_concurrent_receiver.cc @@ -13,10 +13,6 @@ #include "catch2/catch_test_macros.hpp" -#include -#include -#include - namespace dripline { class concurrent_receiver_tester : public concurrent_receiver @@ -24,8 +20,10 @@ namespace dripline public: using concurrent_receiver::concurrent_receiver; + int f_submit_count = 0; + void submit_message( message_ptr_t ) - {} + { ++f_submit_count; } }; } @@ -35,30 +33,12 @@ TEST_CASE( "cr_process_message", "[concurrent_receiver]" ) dripline::request_ptr_t t_request_ptr = dripline::msg_request::create( scarab::param_ptr_t( new scarab::param() ), dripline::op_t::get, "dlcpp_service", "", "" ); - // we process the message before executing the concurrent_receiver. - // this means the message will be queued. - t_concrecv.process_message( t_request_ptr ); - t_concrecv.process_message( t_request_ptr ); - t_concrecv.process_message( t_request_ptr ); + // process_message() now calls submit_message() directly and synchronously; no queue or execute() thread t_concrecv.process_message( t_request_ptr ); t_concrecv.process_message( t_request_ptr ); t_concrecv.process_message( t_request_ptr ); - t_concrecv.process_message( t_request_ptr ); - t_concrecv.process_message( t_request_ptr ); - t_concrecv.process_message( t_request_ptr ); - t_concrecv.process_message( t_request_ptr ); - REQUIRE( t_concrecv.message_queue().size() == 10 ); - - // here we launch the execution asynchronously. - // we'll give it 1 second to execute, which should be enough, though in principle it's not a 100% guarantee that it'll be done in time. - // we then cancel the service and move on to verify that the queue is empty. - auto t_do_execute = [&](){ t_concrecv.concurrent_receiver::execute(); }; - auto t_exe_future = std::async(std::launch::async, t_do_execute); - std::this_thread::sleep_for( std::chrono::seconds(1) ); - t_concrecv.cancel(); - t_exe_future.wait(); - REQUIRE( t_concrecv.message_queue().empty() ); + REQUIRE( t_concrecv.f_submit_count == 3 ); } diff --git a/testing/test_service.cc b/testing/test_service.cc index ad2753d7..63f2245e 100644 --- a/testing/test_service.cc +++ b/testing/test_service.cc @@ -13,31 +13,15 @@ #include "catch2/catch_test_macros.hpp" -#include -#include -#include - TEST_CASE( "process_message", "[service]" ) { dripline::service t_service( scarab::param_node(), scarab::authentication(), false); dripline::request_ptr_t t_request_ptr = dripline::msg_request::create( scarab::param_ptr_t( new scarab::param() ), dripline::op_t::get, "dlcpp_service", "", "" ); - // we process the message before executing the concurrent_receiver. - // this means the message will be queued. - t_service.process_message( t_request_ptr ); - REQUIRE( t_service.message_queue().size() == 1 ); - - // here we launch the execution asynchronously. - // we'll give it 1 second to execute, which should be enough, though in principle it's not a 100% guarantee that it'll be done in time. - // we then cancel the service and move on to verify that the queue is empty. - auto t_do_execute = [&](){ t_service.concurrent_receiver::execute(); }; - auto t_exe_future = std::async(std::launch::async, t_do_execute); - std::this_thread::sleep_for( std::chrono::seconds(1) ); - t_service.cancel(); - t_exe_future.wait(); - - REQUIRE( t_service.message_queue().empty() ); + // process_message() now calls submit_message() directly and synchronously. + // With make_connection=false, no AMQP operations are attempted. + REQUIRE_NOTHROW( t_service.process_message( t_request_ptr ) ); } From 50782165807699df4b77489c02e1108f6d9145e1 Mon Sep 17 00:00:00 2001 From: Noah Oblath Date: Fri, 29 May 2026 11:43:35 -0700 Subject: [PATCH 14/45] Adding move constructor to receiver --- library/receiver.cc | 7 +++++++ library/receiver.hh | 2 +- 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/library/receiver.cc b/library/receiver.cc index e91ecf2e..2d64cd01 100644 --- a/library/receiver.cc +++ b/library/receiver.cc @@ -12,6 +12,7 @@ #include "dripline_exceptions.hh" #include "message.hh" +#include "rmqa_consumer.h" #include "rmqa_vhost.h" #include "rmqp_messageguard.h" @@ -48,6 +49,12 @@ namespace dripline f_single_message_wait_ms( 1000 ) {} + receiver::receiver( receiver&& a_orig ) : + scarab::cancelable( std::move(a_orig) ), + f_incoming_messages( std::move(a_orig.f_incoming_messages) ), + f_single_message_wait_ms( a_orig.f_single_message_wait_ms ) + {} + receiver& receiver::operator=( receiver&& a_orig ) { cancelable::operator=( std::move(a_orig) ); diff --git a/library/receiver.hh b/library/receiver.hh index 3458bb8f..4df8abe3 100644 --- a/library/receiver.hh +++ b/library/receiver.hh @@ -77,7 +77,7 @@ namespace dripline public: receiver(); receiver( const receiver& a_orig ) = delete; - receiver( receiver&& a_orig ) = default; + receiver( receiver&& a_orig ); virtual ~receiver() = default; receiver& operator=( const receiver& a_orig ) = delete; From 66a6a50f5d4ab86ee05d82a1fffd26fc8c1fa298 Mon Sep 17 00:00:00 2001 From: Noah Oblath Date: Fri, 29 May 2026 11:44:01 -0700 Subject: [PATCH 15/45] Setup mesh topology in service and monitor --- library/monitor.cc | 35 ++++++++++++++++++++++++++++++----- library/service.cc | 45 +++++++++++++++++++++++++++++++++------------ 2 files changed, 63 insertions(+), 17 deletions(-) diff --git a/library/monitor.cc b/library/monitor.cc index 7503d164..d0613f61 100644 --- a/library/monitor.cc +++ b/library/monitor.cc @@ -108,11 +108,36 @@ namespace dripline } f_status = status::channel_created; - // TODO (Phase 6): set up monitor topology, declare queue, bind routing keys - // - declare monitor queue (f_name) - // - bind each request key to the requests exchange - // - bind each alerts key to the alerts exchange - // - call start_listening( f_vhost, topology, queue_handle, f_name ) + try + { + using namespace BloombergLP; + + // Monitor queue: ephemeral (auto-delete, non-durable); f_name already contains a UUID + rmqa::Topology t_topo; + auto t_monitor_queue = t_topo.addQueue( bsl::string(f_name), rmqt::AutoDelete::ON, rmqt::Durable::OFF ); + if( ! f_requests_keys.empty() ) + { + auto t_req_ex = t_topo.addExchange( bsl::string(f_requests_exchange), rmqt::ExchangeType::TOPIC ); + for( const auto& t_key : f_requests_keys ) + { + t_topo.bind( t_req_ex, t_monitor_queue, bsl::string(t_key) ); + } + } + if( ! f_alerts_keys.empty() ) + { + auto t_alerts_ex = t_topo.addExchange( bsl::string(f_alerts_exchange), rmqt::ExchangeType::TOPIC ); + for( const auto& t_key : f_alerts_keys ) + { + t_topo.bind( t_alerts_ex, t_monitor_queue, bsl::string(t_key) ); + } + } + start_listening( f_vhost, t_topo, t_monitor_queue, f_name ); + } + catch( connection_error& e ) + { + LERROR( dlog, "Unable to set up monitor topology: " << e.what() ); + return false; + } f_status = status::consuming; return true; diff --git a/library/service.cc b/library/service.cc index 99c4901e..9c91fb88 100644 --- a/library/service.cc +++ b/library/service.cc @@ -199,11 +199,39 @@ namespace dripline } f_status = status::channel_created; - // TODO (Phase 6): set up service topology, declare queues, bind routing keys - // - declare service queue (f_name) and routing keys (f_name + ".#", f_broadcast_key + ".#") - // - for each sync child, bind child_name + ".#" to the service queue - // - call start_listening( f_vhost, topology, service_queue_handle, f_name ) - // - for each async child, declare its queue and call child->start_listening(...) + try + { + using namespace BloombergLP; + + // Build service queue topology: durable, not auto-delete + rmqa::Topology t_topo; + auto t_req_ex = t_topo.addExchange( bsl::string(f_requests_exchange), rmqt::ExchangeType::TOPIC ); + auto t_service_queue = t_topo.addQueue( bsl::string(f_name), rmqt::AutoDelete::OFF, rmqt::Durable::ON ); + t_topo.bind( t_req_ex, t_service_queue, bsl::string(f_name + ".#") ); + t_topo.bind( t_req_ex, t_service_queue, bsl::string(f_broadcast_key + ".#") ); + for( const auto& t_child_pair : f_sync_children ) + { + // Sync children share the service queue + t_topo.bind( t_req_ex, t_service_queue, bsl::string(t_child_pair.first + ".#") ); + } + start_listening( f_vhost, t_topo, t_service_queue, f_name ); + + // Each async child gets its own durable queue + for( auto& t_child_pair : f_async_children ) + { + const std::string& t_child_name = t_child_pair.first; + rmqa::Topology t_child_topo; + auto t_child_ex = t_child_topo.addExchange( bsl::string(f_requests_exchange), rmqt::ExchangeType::TOPIC ); + auto t_child_queue = t_child_topo.addQueue( bsl::string(t_child_name), rmqt::AutoDelete::OFF, rmqt::Durable::ON ); + t_child_topo.bind( t_child_ex, t_child_queue, bsl::string(t_child_name + ".#") ); + t_child_pair.second->start_listening( f_vhost, t_child_topo, t_child_queue, t_child_name ); + } + } + catch( connection_error& e ) + { + LERROR( dlog, "Unable to set up service topology: " << e.what() ); + return false; + } f_status = status::consuming; return true; @@ -241,13 +269,6 @@ namespace dripline LINFO( dlog, "Scheduler disabled" ); } - if( ! f_async_children.empty() ) { LINFO( dlog, "Starting async children" ); } - else { LDEBUG( dlog, "No async children to start" ); } - for( auto& t_child_pair : f_async_children ) - { - // TODO (Phase 6): call t_child_pair.second->start_listening( f_vhost, topology, queue_handle, t_child_pair.first ) - } - // Block until canceled while( ! is_canceled() ) { From d8b052be25a13bb544837127b9fd3759e7dcd69e Mon Sep 17 00:00:00 2001 From: Noah Oblath Date: Fri, 29 May 2026 12:00:38 -0700 Subject: [PATCH 16/45] Added unit tests for multi-chunk message assembly --- testing/test_concurrent_receiver.cc | 130 ++++++++++++++++++++++++++++ testing/test_messages.cc | 31 +++++++ 2 files changed, 161 insertions(+) diff --git a/testing/test_concurrent_receiver.cc b/testing/test_concurrent_receiver.cc index 94d12195..9af5d040 100644 --- a/testing/test_concurrent_receiver.cc +++ b/testing/test_concurrent_receiver.cc @@ -5,14 +5,20 @@ * Author: N.S. Oblath */ +#include "amqp.hh" #include "dripline_exceptions.hh" #include "message.hh" #include "receiver.hh" #include "param_node.hh" +#include "rmqp_messageguard.h" +#include "rmqt_envelope.h" + #include "catch2/catch_test_macros.hpp" +#include + namespace dripline { class concurrent_receiver_tester : public concurrent_receiver @@ -27,6 +33,44 @@ namespace dripline }; } +// Minimal rmqp::MessageGuard implementation for constructing amqp_envelope_ptr in tests. +// TransferrableMessageGuard = bsl::shared_ptr, so any concrete +// subclass wrapped in a bsl::shared_ptr is a valid amqp_envelope_ptr. +struct fake_message_guard : public BloombergLP::rmqp::MessageGuard +{ + BloombergLP::rmqt::Message d_message; + BloombergLP::rmqt::Envelope d_envelope; + + fake_message_guard( const BloombergLP::rmqt::Message& a_msg, + const std::string& a_routing_key ) + : d_message( a_msg ) + , d_envelope( 0, 0, + bsl::string(""), + bsl::string(""), + bsl::string( a_routing_key.c_str() ), + false ) + {} + + const BloombergLP::rmqt::Message& message() const override { return d_message; } + const BloombergLP::rmqt::Envelope& envelope() const override { return d_envelope; } + void ack() override {} + void nack( bool ) override {} + BloombergLP::rmqp::Consumer* consumer() const override { return nullptr; } + BloombergLP::rmqp::TransferrableMessageGuard transferOwnership() override + { + return BloombergLP::rmqp::TransferrableMessageGuard(); + } +}; + +// Wraps an already-serialized chunk (amqp_message_ptr) and routing key into an +// amqp_envelope_ptr suitable for passing to receiver::handle_message_chunk(). +static dripline::amqp_envelope_ptr make_test_envelope( const dripline::amqp_message_ptr& a_chunk, + const std::string& a_routing_key ) +{ + return bsl::make_shared< fake_message_guard >( *a_chunk, a_routing_key ); +} + + TEST_CASE( "cr_process_message", "[concurrent_receiver]" ) { dripline::concurrent_receiver_tester t_concrecv; @@ -42,6 +86,92 @@ TEST_CASE( "cr_process_message", "[concurrent_receiver]" ) } +TEST_CASE( "cr_multi_chunk_assembly", "[concurrent_receiver]" ) +{ + // Payload serializes to ~54 chars of JSON; max_size=20 forces at least 3 chunks. + auto t_payload = scarab::param_ptr_t( new scarab::param_node() ); + t_payload->as_node().add( "data", "abcdefghijklmnopqrstuvwxyz0123456789" ); + + dripline::request_ptr_t t_req_ptr = dripline::msg_request::create( + std::move( t_payload ), + dripline::op_t::get, + "multi.chunk.rk" ); + + const std::string t_routing_key = t_req_ptr->routing_key(); + dripline::amqp_split_message_ptrs t_chunks = t_req_ptr->create_amqp_messages( 20 ); + + REQUIRE( t_chunks.size() > 1 ); + + SECTION( "in-order delivery" ) + { + dripline::concurrent_receiver_tester t_concrecv; + for( auto& t_chunk : t_chunks ) + { + t_concrecv.handle_message_chunk( make_test_envelope( t_chunk, t_routing_key ) ); + } + // All chunks delivered: process_message_pack called exactly once + REQUIRE( t_concrecv.f_submit_count == 1 ); + // Map should be empty after successful assembly + REQUIRE( t_concrecv.incoming_messages().empty() ); + } + + SECTION( "reverse-order delivery" ) + { + dripline::concurrent_receiver_tester t_concrecv; + for( auto it = t_chunks.rbegin(); it != t_chunks.rend(); ++it ) + { + t_concrecv.handle_message_chunk( make_test_envelope( *it, t_routing_key ) ); + } + REQUIRE( t_concrecv.f_submit_count == 1 ); + REQUIRE( t_concrecv.incoming_messages().empty() ); + } +} + +TEST_CASE( "cr_stale_eviction", "[concurrent_receiver]" ) +{ + dripline::concurrent_receiver_tester t_concrecv; + // Short eviction threshold so the test doesn't need a long sleep. + t_concrecv.set_single_message_wait_ms( 50 ); + + // Build a 2-chunk message A; deliver only the first chunk so it stays incomplete. + auto t_payload_a = scarab::param_ptr_t( new scarab::param_node() ); + t_payload_a->as_node().add( "data", "abcdefghijklmnopqrstuvwxyz0123456789" ); + + dripline::request_ptr_t t_req_a = dripline::msg_request::create( + std::move( t_payload_a ), + dripline::op_t::get, + "stale.rk" ); + + dripline::amqp_split_message_ptrs t_chunks_a = t_req_a->create_amqp_messages( 20 ); + REQUIRE( t_chunks_a.size() > 1 ); + + // Deliver only chunk 0 — incomplete entry is inserted into the map. + t_concrecv.handle_message_chunk( make_test_envelope( t_chunks_a[0], "stale.rk" ) ); + REQUIRE( t_concrecv.f_submit_count == 0 ); + REQUIRE( t_concrecv.incoming_messages().size() == 1 ); + + // Wait longer than the eviction threshold so the entry becomes stale. + std::this_thread::sleep_for( std::chrono::milliseconds( 100 ) ); + + // Deliver a fresh 1-chunk message B. The lazy sweep in handle_message_chunk + // runs at entry and evicts message A's stale entry before processing B. + auto t_payload_b = scarab::param_ptr_t( new scarab::param_node() ); + dripline::request_ptr_t t_req_b = dripline::msg_request::create( + std::move( t_payload_b ), + dripline::op_t::get, + "fresh.rk" ); + + dripline::amqp_split_message_ptrs t_chunks_b = t_req_b->create_amqp_messages(); + REQUIRE( t_chunks_b.size() == 1 ); + + t_concrecv.handle_message_chunk( make_test_envelope( t_chunks_b[0], "fresh.rk" ) ); + + // Message A evicted (never processed); message B processed successfully. + REQUIRE( t_concrecv.f_submit_count == 1 ); + REQUIRE( t_concrecv.incoming_messages().empty() ); +} + + diff --git a/testing/test_messages.cc b/testing/test_messages.cc index 647ca624..1c42b5e8 100644 --- a/testing/test_messages.cc +++ b/testing/test_messages.cc @@ -161,5 +161,36 @@ TEST_CASE( "message-conversion", "[message]" ) REQUIRE( *t_alert_ptr == *t_conv_alert_ptr ); } + SECTION( "multi_chunk" ) + { + // Payload serializes to ~50 chars of JSON; max_size=20 forces 3 chunks. + auto t_payload = scarab::param_ptr_t( new scarab::param_node() ); + t_payload->as_node().add( "data", "abcdefghijklmnopqrstuvwxyz0123456789" ); + + dripline::request_ptr_t t_req_ptr = dripline::msg_request::create( + std::move( t_payload ), + dripline::op_t::set, + "multi.chunk.rk" ); + + REQUIRE( t_req_ptr ); + + dripline::amqp_split_message_ptrs t_amqp_msg_ptrs = t_req_ptr->create_amqp_messages( 20 ); + + REQUIRE( t_amqp_msg_ptrs.size() > 1 ); + + dripline::message_ptr_t t_conv_msg_ptr = dripline::message::process_message( t_amqp_msg_ptrs, "multi.chunk.rk" ); + + REQUIRE( t_conv_msg_ptr ); + REQUIRE( t_conv_msg_ptr->get_is_valid() ); + REQUIRE( t_conv_msg_ptr->is_request() ); + REQUIRE( t_conv_msg_ptr->routing_key() == "multi.chunk.rk" ); + REQUIRE( t_conv_msg_ptr->correlation_id() == t_req_ptr->correlation_id() ); + REQUIRE( t_conv_msg_ptr->message_type() == dripline::msg_t::request ); + + dripline::request_ptr_t t_conv_req_ptr = std::static_pointer_cast< dripline::msg_request >( t_conv_msg_ptr ); + REQUIRE( t_conv_req_ptr->get_message_operation() == dripline::op_t::set ); + REQUIRE( *t_req_ptr == *t_conv_req_ptr ); + } + } From 8afa5e6c79ab97f3ed57dc72af36169da37ce6c1 Mon Sep 17 00:00:00 2001 From: Noah Oblath Date: Fri, 29 May 2026 12:15:02 -0700 Subject: [PATCH 17/45] Added new unit tests as suggested by claude sonnet 4.6 --- testing/CMakeLists.txt | 5 + testing/test_configs.cc | 51 ++++++ testing/test_dripline_constants.cc | 117 +++++++++++++ testing/test_endpoint_handlers.cc | 273 +++++++++++++++++++++++++++++ testing/test_hub.cc | 194 ++++++++++++++++++++ testing/test_reply_cache.cc | 76 ++++++++ 6 files changed, 716 insertions(+) create mode 100644 testing/test_configs.cc create mode 100644 testing/test_dripline_constants.cc create mode 100644 testing/test_endpoint_handlers.cc create mode 100644 testing/test_hub.cc create mode 100644 testing/test_reply_cache.cc diff --git a/testing/CMakeLists.txt b/testing/CMakeLists.txt index bffe5f76..b99ac11e 100644 --- a/testing/CMakeLists.txt +++ b/testing/CMakeLists.txt @@ -27,11 +27,16 @@ set( testing_SOURCES test_agent.cc test_amqp.cc test_concurrent_receiver.cc + test_configs.cc test_core.cc + test_dripline_constants.cc test_dripline_error.cc test_endpoint.cc + test_endpoint_handlers.cc + test_hub.cc test_lockout.cc test_messages.cc + test_reply_cache.cc test_return_codes.cc test_scheduler.cc test_service.cc diff --git a/testing/test_configs.cc b/testing/test_configs.cc new file mode 100644 index 00000000..134b7eae --- /dev/null +++ b/testing/test_configs.cc @@ -0,0 +1,51 @@ +/* + * test_configs.cc + * + * Created on: May 29, 2026 + * Author: N.S. Oblath + */ + +#include "dripline_config.hh" +#include "monitor_config.hh" +#include "service_config.hh" + +#include "catch2/catch_test_macros.hpp" + +TEST_CASE( "dripline_config_defaults", "[config]" ) +{ + // Pass false to skip reading the optional ~/.dripline_mesh.yaml file + dripline::dripline_config t_config( false ); + + REQUIRE( t_config["broker_port"]().as_uint() == 5672 ); + REQUIRE( t_config["broker"]().as_string() == "localhost" ); + REQUIRE( t_config["requests_exchange"]().as_string() == "requests" ); + REQUIRE( t_config["alerts_exchange"]().as_string() == "alerts" ); + REQUIRE( t_config["heartbeat_routing_key"]().as_string() == "heartbeat" ); + REQUIRE( t_config["max_connection_attempts"]().as_uint() == 10 ); + REQUIRE( t_config.has( "max_payload_size" ) ); +} + +TEST_CASE( "service_config_defaults", "[config]" ) +{ + dripline::service_config t_config( "my_service" ); + + // service_config wraps its sub-configs under "dripline_mesh" and adds service-level keys + REQUIRE( t_config.has( "dripline_mesh" ) ); + REQUIRE( t_config["name"]().as_string() == "my_service" ); + REQUIRE( t_config["loop_timeout_ms"]().as_uint() == 1000 ); + REQUIRE( t_config["message_wait_ms"]().as_uint() == 1000 ); + REQUIRE( t_config["heartbeat_interval_s"]().as_uint() == 60 ); +} + +TEST_CASE( "service_config_default_name", "[config]" ) +{ + // Default name argument is "a_service" + dripline::service_config t_config; + REQUIRE( t_config["name"]().as_string() == "a_service" ); +} + +TEST_CASE( "monitor_config_defaults", "[config]" ) +{ + dripline::monitor_config t_config; + REQUIRE( t_config.has( "dripline_mesh" ) ); +} diff --git a/testing/test_dripline_constants.cc b/testing/test_dripline_constants.cc new file mode 100644 index 00000000..a36f1627 --- /dev/null +++ b/testing/test_dripline_constants.cc @@ -0,0 +1,117 @@ +/* + * test_dripline_constants.cc + * + * Created on: May 29, 2026 + * Author: N.S. Oblath + */ + +#include "dripline_constants.hh" +#include "dripline_exceptions.hh" + +#include "catch2/catch_test_macros.hpp" + +#include + +TEST_CASE( "op_t_conversions", "[constants]" ) +{ + SECTION( "to_uint" ) + { + REQUIRE( dripline::to_uint( dripline::op_t::set ) == 0 ); + REQUIRE( dripline::to_uint( dripline::op_t::get ) == 1 ); + REQUIRE( dripline::to_uint( dripline::op_t::cmd ) == 9 ); + } + + SECTION( "to_op_t from uint" ) + { + REQUIRE( dripline::to_op_t( 0u ) == dripline::op_t::set ); + REQUIRE( dripline::to_op_t( 1u ) == dripline::op_t::get ); + REQUIRE( dripline::to_op_t( 9u ) == dripline::op_t::cmd ); + } + + SECTION( "to_string" ) + { + REQUIRE( dripline::to_string( dripline::op_t::set ) == "set" ); + REQUIRE( dripline::to_string( dripline::op_t::get ) == "get" ); + REQUIRE( dripline::to_string( dripline::op_t::cmd ) == "cmd" ); + REQUIRE( dripline::to_string( dripline::op_t::unknown ) == "unknown" ); + } + + SECTION( "to_op_t from string - valid" ) + { + REQUIRE( dripline::to_op_t( std::string("set") ) == dripline::op_t::set ); + REQUIRE( dripline::to_op_t( std::string("get") ) == dripline::op_t::get ); + REQUIRE( dripline::to_op_t( std::string("cmd") ) == dripline::op_t::cmd ); + REQUIRE( dripline::to_op_t( std::string("unknown") ) == dripline::op_t::unknown ); + } + + SECTION( "to_op_t from string - invalid throws" ) + { + REQUIRE_THROWS_AS( dripline::to_op_t( std::string("invalid") ), dripline::dripline_error ); + REQUIRE_THROWS_AS( dripline::to_op_t( std::string("") ), dripline::dripline_error ); + } + + SECTION( "round-trip uint->op_t->string" ) + { + REQUIRE( dripline::to_string( dripline::to_op_t( 1u ) ) == "get" ); + REQUIRE( dripline::to_uint( dripline::to_op_t( dripline::to_uint( dripline::op_t::cmd ) ) ) == 9u ); + } + + SECTION( "streaming" ) + { + std::ostringstream t_os; + t_os << dripline::op_t::get; + REQUIRE( t_os.str() == "1" ); + } +} + +TEST_CASE( "msg_t_conversions", "[constants]" ) +{ + SECTION( "to_uint" ) + { + REQUIRE( dripline::to_uint( dripline::msg_t::reply ) == 2 ); + REQUIRE( dripline::to_uint( dripline::msg_t::request ) == 3 ); + REQUIRE( dripline::to_uint( dripline::msg_t::alert ) == 4 ); + } + + SECTION( "to_msg_t from uint" ) + { + REQUIRE( dripline::to_msg_t( 2u ) == dripline::msg_t::reply ); + REQUIRE( dripline::to_msg_t( 3u ) == dripline::msg_t::request ); + REQUIRE( dripline::to_msg_t( 4u ) == dripline::msg_t::alert ); + } + + SECTION( "to_string" ) + { + REQUIRE( dripline::to_string( dripline::msg_t::reply ) == "reply" ); + REQUIRE( dripline::to_string( dripline::msg_t::request ) == "request" ); + REQUIRE( dripline::to_string( dripline::msg_t::alert ) == "alert" ); + REQUIRE( dripline::to_string( dripline::msg_t::unknown ) == "unknown" ); + } + + SECTION( "to_msg_t from string - valid" ) + { + REQUIRE( dripline::to_msg_t( std::string("reply") ) == dripline::msg_t::reply ); + REQUIRE( dripline::to_msg_t( std::string("request") ) == dripline::msg_t::request ); + REQUIRE( dripline::to_msg_t( std::string("alert") ) == dripline::msg_t::alert ); + REQUIRE( dripline::to_msg_t( std::string("unknown") ) == dripline::msg_t::unknown ); + } + + SECTION( "to_msg_t from string - invalid throws" ) + { + REQUIRE_THROWS_AS( dripline::to_msg_t( std::string("invalid") ), dripline::dripline_error ); + REQUIRE_THROWS_AS( dripline::to_msg_t( std::string("") ), dripline::dripline_error ); + } + + SECTION( "round-trip uint->msg_t->string" ) + { + REQUIRE( dripline::to_string( dripline::to_msg_t( 3u ) ) == "request" ); + REQUIRE( dripline::to_uint( dripline::to_msg_t( dripline::to_uint( dripline::msg_t::alert ) ) ) == 4u ); + } + + SECTION( "streaming" ) + { + std::ostringstream t_os; + t_os << dripline::msg_t::reply; + REQUIRE( t_os.str() == "2" ); + } +} diff --git a/testing/test_endpoint_handlers.cc b/testing/test_endpoint_handlers.cc new file mode 100644 index 00000000..5ca4354f --- /dev/null +++ b/testing/test_endpoint_handlers.cc @@ -0,0 +1,273 @@ +/* + * test_endpoint_handlers.cc + * + * Created on: May 29, 2026 + * Author: N.S. Oblath + */ + +#include "dripline_exceptions.hh" +#include "endpoint.hh" +#include "return_codes.hh" +#include "throw_reply.hh" + +#include "param_node.hh" + +#include "catch2/catch_test_macros.hpp" + +// Helper: make a plain request for a given op and specifier +static dripline::request_ptr_t make_request( dripline::op_t a_op, const std::string& a_specifier ) +{ + return dripline::msg_request::create( + scarab::param_ptr_t( new scarab::param() ), + a_op, + "test.routing.key", + a_specifier, + "" ); +} + +// ------------------------------------------------------------------------- +// Built-in wire-protocol handlers +// ------------------------------------------------------------------------- + +TEST_CASE( "endpoint_ping", "[endpoint]" ) +{ + dripline::endpoint t_endpoint( "test_endpoint" ); + + auto t_request = make_request( dripline::op_t::cmd, "ping" ); + auto t_reply = t_endpoint.submit_request_message( t_request ); + + REQUIRE( t_reply != nullptr ); + REQUIRE( t_reply->get_return_code() == dripline::dl_success::s_value ); +} + +TEST_CASE( "endpoint_lock_unlock", "[endpoint]" ) +{ + dripline::endpoint t_endpoint( "test_endpoint" ); + + SECTION( "lock succeeds on unlocked endpoint" ) + { + REQUIRE_FALSE( t_endpoint.is_locked() ); + + auto t_request = make_request( dripline::op_t::cmd, "lock" ); + auto t_reply = t_endpoint.submit_request_message( t_request ); + + REQUIRE( t_reply->get_return_code() == dripline::dl_success::s_value ); + REQUIRE( t_endpoint.is_locked() ); + // reply payload must contain the new lockout key + REQUIRE( t_reply->payload().is_node() ); + REQUIRE( t_reply->payload().as_node().has( "key" ) ); + } + + SECTION( "lock fails when already locked" ) + { + // pre-lock using the direct API + t_endpoint.enable_lockout( scarab::param_node() ); + REQUIRE( t_endpoint.is_locked() ); + + auto t_request = make_request( dripline::op_t::cmd, "lock" ); + auto t_reply = t_endpoint.submit_request_message( t_request ); + + // handle_lock_request returns dl_resource_error when already locked + REQUIRE( t_reply->get_return_code() == dripline::dl_resource_error::s_value ); + REQUIRE( t_endpoint.is_locked() ); + } + + SECTION( "force-unlock succeeds regardless of key" ) + { + t_endpoint.enable_lockout( scarab::param_node() ); + REQUIRE( t_endpoint.is_locked() ); + + // payload with force=true; lockout_key in request stays nil (wrong key) + scarab::param_ptr_t t_payload( new scarab::param_node() ); + t_payload->as_node().add( "force", true ); + auto t_request = dripline::msg_request::create( + std::move( t_payload ), + dripline::op_t::cmd, + "test.routing.key", + "unlock", + "" ); + + auto t_reply = t_endpoint.submit_request_message( t_request ); + + REQUIRE( t_reply->get_return_code() == dripline::dl_success::s_value ); + REQUIRE_FALSE( t_endpoint.is_locked() ); + } + + SECTION( "unlock with correct key succeeds" ) + { + dripline::uuid_t t_key = t_endpoint.enable_lockout( scarab::param_node() ); + REQUIRE( t_endpoint.is_locked() ); + + auto t_request = make_request( dripline::op_t::cmd, "unlock" ); + t_request->lockout_key() = t_key; + + auto t_reply = t_endpoint.submit_request_message( t_request ); + + REQUIRE( t_reply->get_return_code() == dripline::dl_success::s_value ); + REQUIRE_FALSE( t_endpoint.is_locked() ); + } + + SECTION( "unlock when already unlocked returns warning" ) + { + REQUIRE_FALSE( t_endpoint.is_locked() ); + + auto t_request = make_request( dripline::op_t::cmd, "unlock" ); + auto t_reply = t_endpoint.submit_request_message( t_request ); + + REQUIRE( t_reply->get_return_code() == dripline::dl_warning_no_action_taken::s_value ); + } +} + +TEST_CASE( "endpoint_is_locked", "[endpoint]" ) +{ + dripline::endpoint t_endpoint( "test_endpoint" ); + + SECTION( "is-locked returns false when unlocked" ) + { + auto t_request = make_request( dripline::op_t::get, "is-locked" ); + auto t_reply = t_endpoint.submit_request_message( t_request ); + + REQUIRE( t_reply->get_return_code() == dripline::dl_success::s_value ); + REQUIRE( t_reply->payload().is_node() ); + REQUIRE( t_reply->payload().as_node().has( "is_locked" ) ); + REQUIRE( t_reply->payload().as_node()["is_locked"]().as_bool() == false ); + } + + SECTION( "is-locked returns true when locked" ) + { + t_endpoint.enable_lockout( scarab::param_node() ); + + auto t_request = make_request( dripline::op_t::get, "is-locked" ); + auto t_reply = t_endpoint.submit_request_message( t_request ); + + REQUIRE( t_reply->get_return_code() == dripline::dl_success::s_value ); + REQUIRE( t_reply->payload().as_node()["is_locked"]().as_bool() == true ); + } +} + +TEST_CASE( "endpoint_set_condition", "[endpoint]" ) +{ + dripline::endpoint t_endpoint( "test_endpoint" ); + + // default __do_handle_set_condition_request returns dl_success with "No action taken" + auto t_request = make_request( dripline::op_t::cmd, "set_condition" ); + auto t_reply = t_endpoint.submit_request_message( t_request ); + + REQUIRE( t_reply->get_return_code() == dripline::dl_success::s_value ); +} + +// ------------------------------------------------------------------------- +// Lockout authentication enforcement +// ------------------------------------------------------------------------- + +TEST_CASE( "endpoint_lockout_access_denied", "[endpoint]" ) +{ + dripline::endpoint t_endpoint( "test_endpoint" ); + + // Lock the endpoint; the stored key is non-nil. Requests with nil key fail. + t_endpoint.enable_lockout( scarab::param_node() ); + REQUIRE( t_endpoint.is_locked() ); + + SECTION( "set request with wrong key is denied" ) + { + // default lockout_key on a fresh request is a nil UUID → fails authenticate() + auto t_request = make_request( dripline::op_t::set, "anything" ); + auto t_reply = t_endpoint.submit_request_message( t_request ); + + REQUIRE( t_reply->get_return_code() == dripline::dl_service_error_access_denied::s_value ); + } + + SECTION( "cmd request with wrong key is denied (non-exempt specifier)" ) + { + auto t_request = make_request( dripline::op_t::cmd, "lock" ); + // lockout_key in request is nil → authenticate() returns false + auto t_reply = t_endpoint.submit_request_message( t_request ); + + REQUIRE( t_reply->get_return_code() == dripline::dl_service_error_access_denied::s_value ); + } + + SECTION( "ping is exempt from lockout" ) + { + auto t_request = make_request( dripline::op_t::cmd, "ping" ); + auto t_reply = t_endpoint.submit_request_message( t_request ); + + REQUIRE( t_reply->get_return_code() == dripline::dl_success::s_value ); + } + + SECTION( "unlock is exempt from lockout" ) + { + // force-unlock without the correct key should still be allowed through + scarab::param_ptr_t t_payload( new scarab::param_node() ); + t_payload->as_node().add( "force", true ); + auto t_request = dripline::msg_request::create( + std::move( t_payload ), + dripline::op_t::cmd, + "test.routing.key", + "unlock", + "" ); + auto t_reply = t_endpoint.submit_request_message( t_request ); + + REQUIRE( t_reply->get_return_code() == dripline::dl_success::s_value ); + } +} + +// ------------------------------------------------------------------------- +// throw_reply integration: a thrown throw_reply must become the reply +// ------------------------------------------------------------------------- + +namespace +{ + struct throw_reply_endpoint : public dripline::endpoint + { + throw_reply_endpoint() : dripline::endpoint( "throw_reply_ep" ) {} + + dripline::reply_ptr_t do_get_request( const dripline::request_ptr_t a_request ) override + { + throw dripline::throw_reply( dripline::dl_amqp_error() ) << "test throw_reply message"; + } + }; +} // anonymous namespace + +TEST_CASE( "endpoint_throw_reply_caught", "[endpoint]" ) +{ + throw_reply_endpoint t_endpoint; + + auto t_request = make_request( dripline::op_t::get, "anything" ); + + // submit_request_message must NOT propagate the throw_reply; it must convert it to a reply + REQUIRE_NOTHROW( t_endpoint.submit_request_message( t_request ) ); + + auto t_reply = t_endpoint.submit_request_message( t_request ); + REQUIRE( t_reply != nullptr ); + REQUIRE( t_reply->get_return_code() == dripline::dl_amqp_error::s_value ); + REQUIRE( t_reply->return_message() == "test throw_reply message" ); +} + +// ------------------------------------------------------------------------- +// endpoint_listener_receiver dispatch +// ------------------------------------------------------------------------- + +TEST_CASE( "endpoint_listener_receiver_dispatch", "[endpoint]" ) +{ + auto t_ep_ptr = std::make_shared< dripline::endpoint >( "test_elr_endpoint" ); + dripline::endpoint_listener_receiver t_elr( t_ep_ptr ); + + SECTION( "request message is handled without throwing" ) + { + dripline::message_ptr_t t_msg = dripline::msg_request::create( + scarab::param_ptr_t( new scarab::param() ), + dripline::op_t::get, + "test.routing.key", + "", + "" ); + REQUIRE_NOTHROW( t_elr.submit_message( t_msg ) ); + } + + SECTION( "alert message propagates dripline_error (default handler)" ) + { + dripline::message_ptr_t t_msg = dripline::msg_alert::create( + scarab::param_ptr_t( new scarab::param() ), + "" ); + REQUIRE_THROWS_AS( t_elr.submit_message( t_msg ), dripline::dripline_error ); + } +} diff --git a/testing/test_hub.cc b/testing/test_hub.cc new file mode 100644 index 00000000..691427d8 --- /dev/null +++ b/testing/test_hub.cc @@ -0,0 +1,194 @@ +/* + * test_hub.cc + * + * Created on: May 29, 2026 + * Author: N.S. Oblath + */ + +#include "hub.hh" +#include "return_codes.hh" + +#include "authentication.hh" +#include "param_node.hh" + +#include "catch2/catch_test_macros.hpp" + +TEST_CASE( "hub_get_handler", "[hub]" ) +{ + dripline::hub t_hub( scarab::param_node(), scarab::authentication(), false ); + + SECTION( "registered handler is dispatched" ) + { + bool t_called = false; + t_hub.register_get_handler( "my_value", [&t_called]( dripline::request_ptr_t a_req ) { + t_called = true; + return a_req->reply( dripline::dl_success(), "ok" ); + } ); + + auto t_request = dripline::msg_request::create( + scarab::param_ptr_t( new scarab::param() ), + dripline::op_t::get, + "test.endpoint", + "my_value", + "" ); + + auto t_reply = t_hub.submit_request_message( t_request ); + REQUIRE( t_called ); + REQUIRE( t_reply->get_return_code() == dripline::dl_success::s_value ); + } + + SECTION( "unrecognized specifier returns bad-payload error" ) + { + auto t_request = dripline::msg_request::create( + scarab::param_ptr_t( new scarab::param() ), + dripline::op_t::get, + "test.endpoint", + "not_registered", + "" ); + + auto t_reply = t_hub.submit_request_message( t_request ); + REQUIRE( t_reply->get_return_code() == dripline::dl_service_error_bad_payload::s_value ); + } + + SECTION( "removed handler is no longer dispatched" ) + { + bool t_called = false; + t_hub.register_get_handler( "to_remove", [&t_called]( dripline::request_ptr_t a_req ) { + t_called = true; + return a_req->reply( dripline::dl_success(), "ok" ); + } ); + + t_hub.remove_get_handler( "to_remove" ); + + auto t_request = dripline::msg_request::create( + scarab::param_ptr_t( new scarab::param() ), + dripline::op_t::get, + "test.endpoint", + "to_remove", + "" ); + + auto t_reply = t_hub.submit_request_message( t_request ); + REQUIRE_FALSE( t_called ); + REQUIRE( t_reply->get_return_code() == dripline::dl_service_error_bad_payload::s_value ); + } +} + +TEST_CASE( "hub_set_handler", "[hub]" ) +{ + dripline::hub t_hub( scarab::param_node(), scarab::authentication(), false ); + + SECTION( "registered handler is dispatched" ) + { + bool t_called = false; + t_hub.register_set_handler( "my_param", [&t_called]( dripline::request_ptr_t a_req ) { + t_called = true; + return a_req->reply( dripline::dl_success(), "set ok" ); + } ); + + auto t_request = dripline::msg_request::create( + scarab::param_ptr_t( new scarab::param() ), + dripline::op_t::set, + "test.endpoint", + "my_param", + "" ); + + auto t_reply = t_hub.submit_request_message( t_request ); + REQUIRE( t_called ); + REQUIRE( t_reply->get_return_code() == dripline::dl_success::s_value ); + } + + SECTION( "unrecognized specifier returns bad-payload error" ) + { + auto t_request = dripline::msg_request::create( + scarab::param_ptr_t( new scarab::param() ), + dripline::op_t::set, + "test.endpoint", + "not_registered", + "" ); + + auto t_reply = t_hub.submit_request_message( t_request ); + REQUIRE( t_reply->get_return_code() == dripline::dl_service_error_bad_payload::s_value ); + } + + SECTION( "removed handler is no longer dispatched" ) + { + bool t_called = false; + t_hub.register_set_handler( "to_remove", [&t_called]( dripline::request_ptr_t a_req ) { + t_called = true; + return a_req->reply( dripline::dl_success(), "ok" ); + } ); + + t_hub.remove_set_handler( "to_remove" ); + + auto t_request = dripline::msg_request::create( + scarab::param_ptr_t( new scarab::param() ), + dripline::op_t::set, + "test.endpoint", + "to_remove", + "" ); + + auto t_reply = t_hub.submit_request_message( t_request ); + REQUIRE_FALSE( t_called ); + REQUIRE( t_reply->get_return_code() == dripline::dl_service_error_bad_payload::s_value ); + } +} + +TEST_CASE( "hub_cmd_handler", "[hub]" ) +{ + dripline::hub t_hub( scarab::param_node(), scarab::authentication(), false ); + + SECTION( "registered handler is dispatched" ) + { + bool t_called = false; + t_hub.register_cmd_handler( "do_thing", [&t_called]( dripline::request_ptr_t a_req ) { + t_called = true; + return a_req->reply( dripline::dl_success(), "cmd ok" ); + } ); + + auto t_request = dripline::msg_request::create( + scarab::param_ptr_t( new scarab::param() ), + dripline::op_t::cmd, + "test.endpoint", + "do_thing", + "" ); + + auto t_reply = t_hub.submit_request_message( t_request ); + REQUIRE( t_called ); + REQUIRE( t_reply->get_return_code() == dripline::dl_success::s_value ); + } + + SECTION( "unrecognized specifier returns bad-payload error" ) + { + auto t_request = dripline::msg_request::create( + scarab::param_ptr_t( new scarab::param() ), + dripline::op_t::cmd, + "test.endpoint", + "not_registered", + "" ); + + auto t_reply = t_hub.submit_request_message( t_request ); + REQUIRE( t_reply->get_return_code() == dripline::dl_service_error_bad_payload::s_value ); + } + + SECTION( "removed handler is no longer dispatched" ) + { + bool t_called = false; + t_hub.register_cmd_handler( "to_remove", [&t_called]( dripline::request_ptr_t a_req ) { + t_called = true; + return a_req->reply( dripline::dl_success(), "ok" ); + } ); + + t_hub.remove_cmd_handler( "to_remove" ); + + auto t_request = dripline::msg_request::create( + scarab::param_ptr_t( new scarab::param() ), + dripline::op_t::cmd, + "test.endpoint", + "to_remove", + "" ); + + auto t_reply = t_hub.submit_request_message( t_request ); + REQUIRE_FALSE( t_called ); + REQUIRE( t_reply->get_return_code() == dripline::dl_service_error_bad_payload::s_value ); + } +} diff --git a/testing/test_reply_cache.cc b/testing/test_reply_cache.cc new file mode 100644 index 00000000..af2aef34 --- /dev/null +++ b/testing/test_reply_cache.cc @@ -0,0 +1,76 @@ +/* + * test_reply_cache.cc + * + * Created on: May 29, 2026 + * Author: N.S. Oblath + */ + +#include "reply_cache.hh" +#include "return_codes.hh" + +#include "param_node.hh" +#include "param_value.hh" + +#include "catch2/catch_test_macros.hpp" + +TEST_CASE( "reply_cache_set_cache", "[reply_cache]" ) +{ + dripline::reply_cache* t_cache = dripline::reply_cache::get_instance(); + REQUIRE( t_cache != nullptr ); + + SECTION( "set_cache stores return code, message, and payload" ) + { + scarab::param_ptr_t t_payload( new scarab::param_value( 42 ) ); + + t_cache->set_cache( dripline::dl_amqp_error(), "test message", std::move( t_payload ) ); + + REQUIRE( t_cache->ret_code().rc_value() == dripline::dl_amqp_error::s_value ); + REQUIRE( t_cache->return_message() == "test message" ); + REQUIRE( t_cache->payload().as_value().as_uint() == 42u ); + } + + SECTION( "set_cache can be overwritten" ) + { + t_cache->set_cache( dripline::dl_success(), "first", scarab::param_ptr_t( new scarab::param_value( 1 ) ) ); + t_cache->set_cache( dripline::dl_unhandled_exception(), "second", scarab::param_ptr_t( new scarab::param_value( 2 ) ) ); + + REQUIRE( t_cache->ret_code().rc_value() == dripline::dl_unhandled_exception::s_value ); + REQUIRE( t_cache->return_message() == "second" ); + REQUIRE( t_cache->payload().as_value().as_uint() == 2u ); + } +} + +TEST_CASE( "reply_cache_free_function", "[reply_cache]" ) +{ + // set_reply_cache() is the free-function interface to the singleton + dripline::set_reply_cache( + dripline::dl_warning_no_action_taken(), + "via free function", + scarab::param_ptr_t( new scarab::param_node() ) ); + + dripline::reply_cache* t_cache = dripline::reply_cache::get_instance(); + REQUIRE( t_cache->ret_code().rc_value() == dripline::dl_warning_no_action_taken::s_value ); + REQUIRE( t_cache->return_message() == "via free function" ); + REQUIRE( t_cache->payload().is_node() ); +} + +TEST_CASE( "reply_cache_assign_throw_reply", "[reply_cache]" ) +{ + dripline::reply_cache* t_cache = dripline::reply_cache::get_instance(); + + dripline::throw_reply t_src( dripline::dl_resource_error() ); + t_src.set_payload( scarab::param_ptr_t( new scarab::param_value( 99 ) ) ); + t_src << "assigned from throw_reply"; + + *t_cache = t_src; // operator=(const throw_reply&) + + REQUIRE( t_cache->ret_code().rc_value() == dripline::dl_resource_error::s_value ); + REQUIRE( t_cache->return_message() == "assigned from throw_reply" ); + REQUIRE( t_cache->payload().as_value().as_uint() == 99u ); +} + +TEST_CASE( "reply_cache_singleton_identity", "[reply_cache]" ) +{ + // Two calls to get_instance() must return the same pointer + REQUIRE( dripline::reply_cache::get_instance() == dripline::reply_cache::get_instance() ); +} From 1397939b985d7dcfd363982580a37da5199ca614 Mon Sep 17 00:00:00 2001 From: Noah Oblath Date: Fri, 29 May 2026 12:24:44 -0700 Subject: [PATCH 18/45] Added new testing files to the CMakeLists.txt file. --- testing/CMakeLists.txt | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/testing/CMakeLists.txt b/testing/CMakeLists.txt index b99ac11e..7b2a4151 100644 --- a/testing/CMakeLists.txt +++ b/testing/CMakeLists.txt @@ -37,6 +37,11 @@ set( testing_SOURCES test_lockout.cc test_messages.cc test_reply_cache.cc + test_endpoint_handlers.cc + test_hub.cc + test_lockout.cc + test_messages.cc + test_reply_cache.cc test_return_codes.cc test_scheduler.cc test_service.cc From 9e895792019f0bd968a1844da3eb0d46475d7458 Mon Sep 17 00:00:00 2001 From: Noah Oblath Date: Fri, 29 May 2026 13:52:10 -0700 Subject: [PATCH 19/45] Debugging based on new unit tests --- library/endpoint.hh | 1 - library/receiver.hh | 2 +- testing/test_endpoint_handlers.cc | 13 ++++++++----- testing/test_hub.cc | 18 +++++++++--------- testing/test_reply_cache.cc | 2 +- 5 files changed, 19 insertions(+), 17 deletions(-) diff --git a/library/endpoint.hh b/library/endpoint.hh index 7cd18926..395d1e64 100644 --- a/library/endpoint.hh +++ b/library/endpoint.hh @@ -273,7 +273,6 @@ namespace dripline endpoint_listener_receiver& operator=( const endpoint_listener_receiver& ) = delete; endpoint_listener_receiver& operator=( endpoint_listener_receiver&& a_orig ); - protected: /// Direct submission of messages to the endpoint virtual void submit_message( message_ptr_t a_message ) override; diff --git a/library/receiver.hh b/library/receiver.hh index 4df8abe3..3a964997 100644 --- a/library/receiver.hh +++ b/library/receiver.hh @@ -160,11 +160,11 @@ namespace dripline /// Cancels the rmqcpp consumer, drains in-flight messages, and releases it. void stop_listening(); - protected: /// Handles messages according to the use case. It's to be implemented by the class inheriting from concurrent_receiver /// For a concrete example, see @ref service or @ref endpoint_listener_receiver. virtual void submit_message( message_ptr_t a_message ) = 0; + protected: bsl::shared_ptr< BloombergLP::rmqa::Consumer > f_consumer; }; diff --git a/testing/test_endpoint_handlers.cc b/testing/test_endpoint_handlers.cc index 5ca4354f..f989253f 100644 --- a/testing/test_endpoint_handlers.cc +++ b/testing/test_endpoint_handlers.cc @@ -18,7 +18,7 @@ static dripline::request_ptr_t make_request( dripline::op_t a_op, const std::string& a_specifier ) { return dripline::msg_request::create( - scarab::param_ptr_t( new scarab::param() ), + scarab::param_ptr_t( new scarab::param_node() ), a_op, "test.routing.key", a_specifier, @@ -60,14 +60,17 @@ TEST_CASE( "endpoint_lock_unlock", "[endpoint]" ) SECTION( "lock fails when already locked" ) { - // pre-lock using the direct API - t_endpoint.enable_lockout( scarab::param_node() ); + // pre-lock using the direct API; capture the key so the re-lock request can authenticate + dripline::uuid_t t_key = t_endpoint.enable_lockout( scarab::param_node() ); REQUIRE( t_endpoint.is_locked() ); + // The re-lock request must carry the correct key, otherwise __do_cmd_request rejects + // it with dl_service_error_access_denied before even reaching handle_lock_request. auto t_request = make_request( dripline::op_t::cmd, "lock" ); + t_request->lockout_key() = t_key; auto t_reply = t_endpoint.submit_request_message( t_request ); - // handle_lock_request returns dl_resource_error when already locked + // handle_lock_request sees is_locked()==true, calls enable_lockout which returns nil → resource error REQUIRE( t_reply->get_return_code() == dripline::dl_resource_error::s_value ); REQUIRE( t_endpoint.is_locked() ); } @@ -95,7 +98,7 @@ TEST_CASE( "endpoint_lock_unlock", "[endpoint]" ) SECTION( "unlock with correct key succeeds" ) { - dripline::uuid_t t_key = t_endpoint.enable_lockout( scarab::param_node() ); + dripline::uuid_t t_key = t_endpoint.enable_lockout( scarab::param_node{} ); REQUIRE( t_endpoint.is_locked() ); auto t_request = make_request( dripline::op_t::cmd, "unlock" ); diff --git a/testing/test_hub.cc b/testing/test_hub.cc index 691427d8..56787334 100644 --- a/testing/test_hub.cc +++ b/testing/test_hub.cc @@ -28,7 +28,7 @@ TEST_CASE( "hub_get_handler", "[hub]" ) auto t_request = dripline::msg_request::create( scarab::param_ptr_t( new scarab::param() ), dripline::op_t::get, - "test.endpoint", + "dlcpp_service", "my_value", "" ); @@ -42,7 +42,7 @@ TEST_CASE( "hub_get_handler", "[hub]" ) auto t_request = dripline::msg_request::create( scarab::param_ptr_t( new scarab::param() ), dripline::op_t::get, - "test.endpoint", + "dlcpp_service", "not_registered", "" ); @@ -63,7 +63,7 @@ TEST_CASE( "hub_get_handler", "[hub]" ) auto t_request = dripline::msg_request::create( scarab::param_ptr_t( new scarab::param() ), dripline::op_t::get, - "test.endpoint", + "dlcpp_service", "to_remove", "" ); @@ -88,7 +88,7 @@ TEST_CASE( "hub_set_handler", "[hub]" ) auto t_request = dripline::msg_request::create( scarab::param_ptr_t( new scarab::param() ), dripline::op_t::set, - "test.endpoint", + "dlcpp_service", "my_param", "" ); @@ -102,7 +102,7 @@ TEST_CASE( "hub_set_handler", "[hub]" ) auto t_request = dripline::msg_request::create( scarab::param_ptr_t( new scarab::param() ), dripline::op_t::set, - "test.endpoint", + "dlcpp_service", "not_registered", "" ); @@ -123,7 +123,7 @@ TEST_CASE( "hub_set_handler", "[hub]" ) auto t_request = dripline::msg_request::create( scarab::param_ptr_t( new scarab::param() ), dripline::op_t::set, - "test.endpoint", + "dlcpp_service", "to_remove", "" ); @@ -148,7 +148,7 @@ TEST_CASE( "hub_cmd_handler", "[hub]" ) auto t_request = dripline::msg_request::create( scarab::param_ptr_t( new scarab::param() ), dripline::op_t::cmd, - "test.endpoint", + "dlcpp_service", "do_thing", "" ); @@ -162,7 +162,7 @@ TEST_CASE( "hub_cmd_handler", "[hub]" ) auto t_request = dripline::msg_request::create( scarab::param_ptr_t( new scarab::param() ), dripline::op_t::cmd, - "test.endpoint", + "dlcpp_service", "not_registered", "" ); @@ -183,7 +183,7 @@ TEST_CASE( "hub_cmd_handler", "[hub]" ) auto t_request = dripline::msg_request::create( scarab::param_ptr_t( new scarab::param() ), dripline::op_t::cmd, - "test.endpoint", + "dlcpp_service", "to_remove", "" ); diff --git a/testing/test_reply_cache.cc b/testing/test_reply_cache.cc index af2aef34..e03a4da0 100644 --- a/testing/test_reply_cache.cc +++ b/testing/test_reply_cache.cc @@ -58,7 +58,7 @@ TEST_CASE( "reply_cache_assign_throw_reply", "[reply_cache]" ) { dripline::reply_cache* t_cache = dripline::reply_cache::get_instance(); - dripline::throw_reply t_src( dripline::dl_resource_error() ); + dripline::throw_reply t_src( dripline::dl_resource_error{} ); t_src.set_payload( scarab::param_ptr_t( new scarab::param_value( 99 ) ) ); t_src << "assigned from throw_reply"; From b8630010ef18a8d2dc36b89feef3b918ec1dea33 Mon Sep 17 00:00:00 2001 From: Noah Oblath Date: Fri, 29 May 2026 13:52:46 -0700 Subject: [PATCH 20/45] Fixing old comment --- library/agent.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/library/agent.cc b/library/agent.cc index 85be2ce1..d0c4324b 100644 --- a/library/agent.cc +++ b/library/agent.cc @@ -81,7 +81,7 @@ namespace dripline t_config.remove( "auth_file" ); t_config.remove( "auth_groups" ); - f_agent->set_timeout( t_config.get_value( "timeout", 10U ) * 1000 ); // convert seconds (dripline agent user interface) to milliseconds (expected by SimpleAmqpClient) + f_agent->set_timeout( t_config.get_value( "timeout", 10U ) * 1000 ); // convert seconds (dripline agent user interface) to milliseconds t_config.erase( "timeout" ); f_agent->set_json_print( t_config.get_value( "json_print", f_agent->get_json_print() ) ); t_config.erase( "json_print" ); From bfbbcdf18a8432b3946a0d7d27ad001a166e7981 Mon Sep 17 00:00:00 2001 From: Noah Oblath Date: Fri, 29 May 2026 17:20:01 -0700 Subject: [PATCH 21/45] Fixing leftover reference to the external directory --- Dockerfile | 1 - 1 file changed, 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 68ac9f33..20aecbd7 100644 --- a/Dockerfile +++ b/Dockerfile @@ -114,7 +114,6 @@ FROM base # note that the build dir is *not* in source, this is so that the source can me mounted onto the container without covering the build target COPY .git /usr/local/src/.git -COPY external /usr/local/src/external COPY documentation /usr/local/src/documentation COPY scarab /usr/local/src/scarab COPY library /usr/local/src/library From f2c11bfc221685528d7f803bdd579f13d0919324 Mon Sep 17 00:00:00 2001 From: Noah Oblath Date: Fri, 29 May 2026 17:33:27 -0700 Subject: [PATCH 22/45] Copied vcpkg-based dependency install to test_builds job; made a couple small changes to the Dockerfile version too; pinned rmqcpp version to current main branch commit --- .github/workflows/publish.yaml | 58 ++++++++++++++++++++++++++++++++-- Dockerfile | 13 +++++--- 2 files changed, 64 insertions(+), 7 deletions(-) diff --git a/.github/workflows/publish.yaml b/.github/workflows/publish.yaml index ae9db989..f2e0def3 100644 --- a/.github/workflows/publish.yaml +++ b/.github/workflows/publish.yaml @@ -21,6 +21,7 @@ env: REGISTRY_OLD: docker.io BASE_IMAGE_REPO: python BASE_IMAGE_TAG: '3.12.1-slim-bookworm' + RMQCPP_CHECKOUT: cc6885319ccb97b8a6d13e09e83a52c43aab16c7 # narg/NARG (number-of-CPU argument to pass to the builds) are based on OS-specific CPU counts in the GitHub-hosted GHA runners. # checked 1/4/2024: https://docs.github.com/en/actions/using-github-hosted-runners/about-github-hosted-runners/about-github-hosted-runners#supported-runners-and-hardware-resources @@ -83,6 +84,7 @@ jobs: build_examples=TRUE enable_testing=TRUE narg=${{ env.NARG }} + rmqcpp_checkout=${{ env.RMQCPP_CHECKOUT }} platforms: linux/amd64 tags: ${{ env.DL_CPP_TAG }} @@ -154,7 +156,6 @@ jobs: if: startsWith(matrix.os, 'macos') run: | brew install \ - boost \ rabbitmq-c \ rapidjson \ yaml-cpp @@ -164,10 +165,60 @@ jobs: run: | sudo apt-get update DEBIAN_FRONTEND=noninteractive sudo apt-get install -y \ - libboost-all-dev \ librabbitmq-dev \ rapidjson-dev \ libyaml-cpp-dev + + - name: Install dependencies -- common + run: | + cd /usr/local + git clone https://github.com/Microsoft/vcpkg.git + /usr/local/vcpkg/bootstrap-vcpkg.sh + git clone https://github.com/bloomberg/rmqcpp.git + cd /usr/local/rmqcpp + git checkout ${{ env.RMQCPP_CHECKOUT }} + case "${TARGETARCH}" in \ + amd64) TRIPLET="x64-linux-release" ;; \ + arm64) TRIPLET="arm64-linux-release" ;; \ + *) echo "Unsupported architecture: ${TARGETARCH}"; exit 1 ;; \ + esac + ${VCPKG_ROOT}/vcpkg install --triplet ${TRIPLET} + mkdir -p /usr/local/rmqcpp/build + cd /usr/local/rmqcpp/build + cmake -DCMAKE_BUILD_TYPE=Release \ + -DCMAKE_TOOLCHAIN_FILE=${VCPKG_ROOT}/scripts/buildsystems/vcpkg.cmake \ + -DVCPKG_TARGET_TRIPLET=${TRIPLET} \ + -DCMAKE_INSTALL_PREFIX=/usr/local \ + -DBUILD_TESTING=OFF \ + .. + make -j${narg} install + cd / + ${VCPKG_ROOT}/vcpkg install --triplet ${TRIPLET} \ + boost-filesystem \ + boost-system \ + boost-chrono \ + boost-variant \ + boost-uuid + cp -a /usr/local/rmqcpp/vcpkg_installed/${TRIPLET}/include/. /usr/local/include/ + cp -a /usr/local/rmqcpp/vcpkg_installed/${TRIPLET}/lib/. /usr/local/lib/ + cp -a /usr/local/rmqcpp/vcpkg_installed/${TRIPLET}/share/. /usr/local/share/ + cp -a ${VCPKG_ROOT}/installed/${TRIPLET}/include/. /usr/local/include/ + cp -a ${VCPKG_ROOT}/installed/${TRIPLET}/lib/. /usr/local/lib/ + cp -a ${VCPKG_ROOT}/installed/${TRIPLET}/share/. /usr/local/share/ + rm -rf /usr/local/share/zstd + mkdir -p /usr/local/share/libpcre2-8 + printf '%s\n' \ + 'include("/usr/local/share/pcre2/pcre2-config.cmake")' \ + 'if(NOT TARGET libpcre2-8::pcre2-8 AND TARGET pcre2::pcre2-8-static)' \ + ' add_library(libpcre2-8::pcre2-8 ALIAS pcre2::pcre2-8-static)' \ + 'endif()' \ + 'set(libpcre2-8_FOUND TRUE)' \ + > /usr/local/share/libpcre2-8/libpcre2-8Config.cmake + RMQCPP_CONFIG=$(find /usr/local -name rmqcppConfig.cmake 2>/dev/null | head -1) + test -n "${RMQCPP_CONFIG}" + if ! grep -q 'find_dependency(zstd' "${RMQCPP_CONFIG}"; then \ + sed -i '/include.*rmqcppTargets/i find_dependency(zstd CONFIG)' "${RMQCPP_CONFIG}"; \ + fi - name: Configure run: | @@ -303,7 +354,8 @@ jobs: build_type=${{ matrix.build-type }} build_examples=${{ matrix.build-examples }} enable_testing=${{ matrix.enable-testing }} - narg=${{env.NARG }} + narg=${{ env.NARG }} + rmqcpp_checkout=${{ env.RMQCPP_CHECKOUT }} tags: ${{ steps.docker_meta.outputs.tags }} platforms: linux/amd64,linux/arm64,linux/arm/v7 diff --git a/Dockerfile b/Dockerfile index 20aecbd7..edcf2f52 100644 --- a/Dockerfile +++ b/Dockerfile @@ -51,7 +51,7 @@ RUN cd /usr/local && \ rm -rf /usr/local/${pybind11_name} ARG TARGETARCH -ARG rmqcpp_checkout= +ARG rmqcpp_checkout=cc6885319ccb97b8a6d13e09e83a52c43aab16c7 RUN cd /usr/local && \ git clone https://github.com/Microsoft/vcpkg.git && \ /usr/local/vcpkg/bootstrap-vcpkg.sh && \ @@ -64,8 +64,8 @@ RUN cd /usr/local && \ *) echo "Unsupported architecture: ${TARGETARCH}"; exit 1 ;; \ esac && \ ${VCPKG_ROOT}/vcpkg install --triplet ${TRIPLET} && \ - mkdir build && \ - cd build && \ + mkdir -p /usr/local/rmqcpp/build && \ + cd /usr/local/rmqcpp/build && \ cmake -DCMAKE_BUILD_TYPE=Release \ -DCMAKE_TOOLCHAIN_FILE=${VCPKG_ROOT}/scripts/buildsystems/vcpkg.cmake \ -DVCPKG_TARGET_TRIPLET=${TRIPLET} \ @@ -74,7 +74,12 @@ RUN cd /usr/local && \ .. && \ make -j${narg} install && \ cd / && \ - ${VCPKG_ROOT}/vcpkg install --triplet ${TRIPLET} boost-filesystem boost-system boost-chrono boost-variant boost-uuid && \ + ${VCPKG_ROOT}/vcpkg install --triplet ${TRIPLET} \ + boost-filesystem \ + boost-system \ + boost-chrono \ + boost-variant \ + boost-uuid && \ cp -a /usr/local/rmqcpp/vcpkg_installed/${TRIPLET}/include/. /usr/local/include/ && \ cp -a /usr/local/rmqcpp/vcpkg_installed/${TRIPLET}/lib/. /usr/local/lib/ && \ cp -a /usr/local/rmqcpp/vcpkg_installed/${TRIPLET}/share/. /usr/local/share/ && \ From f90ad8c27392f0808e69c9b71ee5c094e2690248 Mon Sep 17 00:00:00 2001 From: Noah Oblath Date: Fri, 29 May 2026 21:37:49 -0700 Subject: [PATCH 23/45] Switch to using build-in vcpkg on the github actions runners; Add tmate for debugging --- .github/workflows/publish.yaml | 76 ++++++++++++++-------------------- 1 file changed, 32 insertions(+), 44 deletions(-) diff --git a/.github/workflows/publish.yaml b/.github/workflows/publish.yaml index f2e0def3..cba99bf9 100644 --- a/.github/workflows/publish.yaml +++ b/.github/workflows/publish.yaml @@ -132,8 +132,10 @@ jobs: include: - os: ubuntu-22.04 narg: 2 + triplet: x64-linux-release - os: macos-latest narg: 3 + triplet: arm64-osx-release steps: @@ -171,54 +173,40 @@ jobs: - name: Install dependencies -- common run: | - cd /usr/local - git clone https://github.com/Microsoft/vcpkg.git - /usr/local/vcpkg/bootstrap-vcpkg.sh + ${VCPKG_INSTALLATION_ROOT}/vcpkg install --triplet ${{ matrix.triplet }} \ + boost-filesystem \ + boost-system \ + boost-chrono \ + boost-variant \ + boost-uuid git clone https://github.com/bloomberg/rmqcpp.git - cd /usr/local/rmqcpp + cd rmqcpp git checkout ${{ env.RMQCPP_CHECKOUT }} - case "${TARGETARCH}" in \ - amd64) TRIPLET="x64-linux-release" ;; \ - arm64) TRIPLET="arm64-linux-release" ;; \ - *) echo "Unsupported architecture: ${TARGETARCH}"; exit 1 ;; \ - esac - ${VCPKG_ROOT}/vcpkg install --triplet ${TRIPLET} - mkdir -p /usr/local/rmqcpp/build - cd /usr/local/rmqcpp/build + ${VCPKG_INSTALLATION_ROOT}/vcpkg install --triplet ${{ matrix.triplet }} + mkdir -p build + cd build cmake -DCMAKE_BUILD_TYPE=Release \ - -DCMAKE_TOOLCHAIN_FILE=${VCPKG_ROOT}/scripts/buildsystems/vcpkg.cmake \ - -DVCPKG_TARGET_TRIPLET=${TRIPLET} \ + -DCMAKE_TOOLCHAIN_FILE=${VCPKG_INSTALLATION_ROOT}/scripts/buildsystems/vcpkg.cmake \ + -DVCPKG_TARGET_TRIPLET=${{ matrix.triplet }} \ -DCMAKE_INSTALL_PREFIX=/usr/local \ -DBUILD_TESTING=OFF \ .. make -j${narg} install - cd / - ${VCPKG_ROOT}/vcpkg install --triplet ${TRIPLET} \ - boost-filesystem \ - boost-system \ - boost-chrono \ - boost-variant \ - boost-uuid - cp -a /usr/local/rmqcpp/vcpkg_installed/${TRIPLET}/include/. /usr/local/include/ - cp -a /usr/local/rmqcpp/vcpkg_installed/${TRIPLET}/lib/. /usr/local/lib/ - cp -a /usr/local/rmqcpp/vcpkg_installed/${TRIPLET}/share/. /usr/local/share/ - cp -a ${VCPKG_ROOT}/installed/${TRIPLET}/include/. /usr/local/include/ - cp -a ${VCPKG_ROOT}/installed/${TRIPLET}/lib/. /usr/local/lib/ - cp -a ${VCPKG_ROOT}/installed/${TRIPLET}/share/. /usr/local/share/ - rm -rf /usr/local/share/zstd - mkdir -p /usr/local/share/libpcre2-8 - printf '%s\n' \ - 'include("/usr/local/share/pcre2/pcre2-config.cmake")' \ - 'if(NOT TARGET libpcre2-8::pcre2-8 AND TARGET pcre2::pcre2-8-static)' \ - ' add_library(libpcre2-8::pcre2-8 ALIAS pcre2::pcre2-8-static)' \ - 'endif()' \ - 'set(libpcre2-8_FOUND TRUE)' \ - > /usr/local/share/libpcre2-8/libpcre2-8Config.cmake - RMQCPP_CONFIG=$(find /usr/local -name rmqcppConfig.cmake 2>/dev/null | head -1) - test -n "${RMQCPP_CONFIG}" - if ! grep -q 'find_dependency(zstd' "${RMQCPP_CONFIG}"; then \ - sed -i '/include.*rmqcppTargets/i find_dependency(zstd CONFIG)' "${RMQCPP_CONFIG}"; \ - fi + + +# mkdir -p /usr/local/share/libpcre2-8 +# printf '%s\n' \ +# 'include("/usr/local/share/pcre2/pcre2-config.cmake")' \ +# 'if(NOT TARGET libpcre2-8::pcre2-8 AND TARGET pcre2::pcre2-8-static)' \ +# ' add_library(libpcre2-8::pcre2-8 ALIAS pcre2::pcre2-8-static)' \ +# 'endif()' \ +# 'set(libpcre2-8_FOUND TRUE)' \ +# > /usr/local/share/libpcre2-8/libpcre2-8Config.cmake +# RMQCPP_CONFIG=$(find /usr/local -name rmqcppConfig.cmake 2>/dev/null | head -1) +# test -n "${RMQCPP_CONFIG}" +# if ! grep -q 'find_dependency(zstd' "${RMQCPP_CONFIG}"; then \ +# sed -i '/include.*rmqcppTargets/i find_dependency(zstd CONFIG)' "${RMQCPP_CONFIG}"; \ +# fi - name: Configure run: | @@ -237,9 +225,9 @@ jobs: ./run_dl_tests # For debugging -# - name: Setup tmate session -# if: ${{ ! success() }} -# uses: mxschmitt/action-tmate@v3 + - name: Setup tmate session + if: ${{ ! success() }} + uses: mxschmitt/action-tmate@v3 build_and_push: From 43faa8d77d570eb09352dc1cef735e514011cd8b Mon Sep 17 00:00:00 2001 From: Noah Oblath Date: Tue, 2 Jun 2026 17:00:03 -0700 Subject: [PATCH 24/45] Fixing durability and updating newman tests --- library/service.cc | 6 +- testing/integration/docker-compose-test.yaml | 2 + .../rabbitmq_for_dl_collection.json | 557 +++++++++--------- 3 files changed, 289 insertions(+), 276 deletions(-) diff --git a/library/service.cc b/library/service.cc index 9c91fb88..3831dc7a 100644 --- a/library/service.cc +++ b/library/service.cc @@ -203,10 +203,10 @@ namespace dripline { using namespace BloombergLP; - // Build service queue topology: durable, not auto-delete + // Build service queue topology: non-durable, auto-delete rmqa::Topology t_topo; auto t_req_ex = t_topo.addExchange( bsl::string(f_requests_exchange), rmqt::ExchangeType::TOPIC ); - auto t_service_queue = t_topo.addQueue( bsl::string(f_name), rmqt::AutoDelete::OFF, rmqt::Durable::ON ); + auto t_service_queue = t_topo.addQueue( bsl::string(f_name), rmqt::AutoDelete::ON, rmqt::Durable::OFF ); t_topo.bind( t_req_ex, t_service_queue, bsl::string(f_name + ".#") ); t_topo.bind( t_req_ex, t_service_queue, bsl::string(f_broadcast_key + ".#") ); for( const auto& t_child_pair : f_sync_children ) @@ -222,7 +222,7 @@ namespace dripline const std::string& t_child_name = t_child_pair.first; rmqa::Topology t_child_topo; auto t_child_ex = t_child_topo.addExchange( bsl::string(f_requests_exchange), rmqt::ExchangeType::TOPIC ); - auto t_child_queue = t_child_topo.addQueue( bsl::string(t_child_name), rmqt::AutoDelete::OFF, rmqt::Durable::ON ); + auto t_child_queue = t_child_topo.addQueue( bsl::string(t_child_name), rmqt::AutoDelete::ON, rmqt::Durable::OFF ); t_child_topo.bind( t_child_ex, t_child_queue, bsl::string(t_child_name + ".#") ); t_child_pair.second->start_listening( f_vhost, t_child_topo, t_child_queue, t_child_name ); } diff --git a/testing/integration/docker-compose-test.yaml b/testing/integration/docker-compose-test.yaml index 5f7ea6d6..c566585a 100644 --- a/testing/integration/docker-compose-test.yaml +++ b/testing/integration/docker-compose-test.yaml @@ -8,7 +8,9 @@ services: - oscillator-endpoints command: > bash -c "sleep 1 && + echo 'Running newman tests' && newman run /root/rabbitmq_for_dl_collection.json && + echo 'Running DL tests' && /root/dl-tests.sh" volumes: - ./dl-tests.sh:/root/dl-tests.sh diff --git a/testing/integration/rabbitmq_for_dl_collection.json b/testing/integration/rabbitmq_for_dl_collection.json index 8368e739..66175374 100644 --- a/testing/integration/rabbitmq_for_dl_collection.json +++ b/testing/integration/rabbitmq_for_dl_collection.json @@ -1,275 +1,286 @@ { - "info": { - "_postman_id": "745e3298-6aa9-4739-a0b3-77543eb49ad9", - "name": "RabbitMQ for Dripline", - "schema": "https://schema.getpostman.com/json/collection/v2.1.0/collection.json", - "_exporter_id": "34537662" - }, - "item": [ - { - "name": "rabbit-broker:15672/api/exchanges/%2f/requests", - "event": [ - { - "listen": "test", - "script": { - "exec": [ - "pm.test(\"Status code is 200\", function () {", - " pm.response.to.have.status(200);", - "});", - "", - "pm.test(\"The response has the expected properties\", function () {", - " const responseJson = pm.response.json();", - " pm.expect(responseJson.durable).to.be.a(\"boolean\");", - " pm.expect(responseJson.durable).to.eql(false);", - " pm.expect(responseJson.name).to.eql(\"requests\");", - " pm.expect(responseJson.type).to.eql(\"topic\");", - "});", - "" - ], - "type": "text/javascript", - "packages": {} - } - } - ], - "request": { - "auth": { - "type": "basic", - "basic": [ - { - "key": "password", - "value": "dripline", - "type": "string" - }, - { - "key": "username", - "value": "dripline", - "type": "string" - } - ] - }, - "method": "GET", - "header": [], - "url": { - "raw": "rabbit-broker:15672/api/exchanges/%2f/requests", - "host": [ - "rabbit-broker" - ], - "port": "15672", - "path": [ - "api", - "exchanges", - "%2f", - "requests" - ] - } - }, - "response": [] - }, - { - "name": "rabbit-broker:15672/api/exchanges/%2f/alerts", - "event": [ - { - "listen": "test", - "script": { - "exec": [ - "pm.test(\"Status code is 200\", function () {", - " pm.response.to.have.status(200);", - "});", - "", - "pm.test(\"The response has the expected properties\", function () {", - " const responseJson = pm.response.json();", - " pm.expect(responseJson.durable).to.be.a(\"boolean\");", - " pm.expect(responseJson.durable).to.eql(false);", - " pm.expect(responseJson.name).to.eql(\"alerts\");", - " pm.expect(responseJson.type).to.eql(\"topic\");", - "});", - "" - ], - "type": "text/javascript", - "packages": {} - } - } - ], - "request": { - "auth": { - "type": "basic", - "basic": [ - { - "key": "password", - "value": "dripline", - "type": "string" - }, - { - "key": "username", - "value": "dripline", - "type": "string" - } - ] - }, - "method": "GET", - "header": [], - "url": { - "raw": "rabbit-broker:15672/api/exchanges/%2f/alerts", - "host": [ - "rabbit-broker" - ], - "port": "15672", - "path": [ - "api", - "exchanges", - "%2f", - "alerts" - ] - } - }, - "response": [] - }, - { - "name": "rabbit-broker:15672/api/queues/%2f/simple", - "event": [ - { - "listen": "test", - "script": { - "exec": [ - "pm.test(\"Status code is 200\", function () {", - " pm.response.to.have.status(200);", - "});", - "", - "pm.test(\"The response has the expected properties\", function () {", - " const responseJson = pm.response.json();", - " pm.expect(responseJson.durable).to.be.a(\"boolean\");", - " pm.expect(responseJson.durable).to.eql(false);", - " pm.expect(responseJson.name).to.eql(\"simple\");", - "});", - "" - ], - "type": "text/javascript", - "packages": {} - } - } - ], - "request": { - "auth": { - "type": "basic", - "basic": [ - { - "key": "password", - "value": "dripline", - "type": "string" - }, - { - "key": "username", - "value": "dripline", - "type": "string" - } - ] - }, - "method": "GET", - "header": [], - "url": { - "raw": "rabbit-broker:15672/api/queues/%2f/simple", - "host": [ - "rabbit-broker" - ], - "port": "15672", - "path": [ - "api", - "queues", - "%2f", - "simple" - ] - } - }, - "response": [] - }, - { - "name": "rabbit-broker:15672/api/bindings/%2f/e/requests/q/simple", - "event": [ - { - "listen": "test", - "script": { - "exec": [ - "pm.test(\"Status code is 200\", function () {", - " pm.response.to.have.status(200);", - "});", - "", - "pm.test(\"The response has the right properties\", function () {", - " const responseJson = pm.response.json();", - " pm.expect(responseJson).to.be.an(\"array\");", - "", - " pm.expect(responseJson[0].source).to.eql(\"requests\");", - " pm.expect(responseJson[0].destination).to.eql(\"simple\");", - " pm.expect(responseJson[0].routing_key).to.eql(\"broadcast.#\");", - " pm.expect(responseJson[1].source).to.eql(\"requests\");", - " pm.expect(responseJson[1].destination).to.eql(\"simple\");", - " pm.expect(responseJson[1].routing_key).to.eql(\"simple.#\");", - "});", - "", - "" - ], - "type": "text/javascript", - "packages": {} - } - } - ], - "request": { - "method": "GET", - "header": [], - "url": { - "raw": "rabbit-broker:15672/api/bindings/%2f/e/requests/q/simple", - "host": [ - "rabbit-broker" - ], - "port": "15672", - "path": [ - "api", - "bindings", - "%2f", - "e", - "requests", - "q", - "simple" - ] - } - }, - "response": [] - } - ], - "auth": { - "type": "basic", - "basic": [ - { - "key": "password", - "value": "dripline", - "type": "string" - }, - { - "key": "username", - "value": "dripline", - "type": "string" - } - ] - }, - "event": [ - { - "listen": "prerequest", - "script": { - "type": "text/javascript", - "packages": {}, - "exec": [ - "" - ] - } - }, - { - "listen": "test", - "script": { - "type": "text/javascript", - "packages": {}, - "exec": [ - "" - ] - } - } - ] + "info": { + "_postman_id": "745e3298-6aa9-4739-a0b3-77543eb49ad9", + "name": "RabbitMQ for Dripline", + "schema": "https://schema.getpostman.com/json/collection/v2.1.0/collection.json", + "_exporter_id": "34537662" + }, + "item": [ + { + "name": "rabbit-broker:15672/api/exchanges/%2f/requests", + "event": [ + { + "listen": "test", + "script": { + "exec": [ + "pm.test(\"Status code is 200\", function () {", + " pm.response.to.have.status(200);", + "});", + "", + "pm.test(\"The response has the expected properties\", function () {", + " const responseJson = pm.response.json();", + " pm.expect(responseJson.durable).to.be.a(\"boolean\");", + " pm.expect(responseJson.durable).to.eql(true);", + " pm.expect(responseJson.name).to.eql(\"requests\");", + " pm.expect(responseJson.type).to.eql(\"topic\");", + "});", + "" + ], + "type": "text/javascript", + "packages": {}, + "requests": {} + } + } + ], + "request": { + "auth": { + "type": "basic", + "basic": [ + { + "key": "password", + "value": "dripline", + "type": "string" + }, + { + "key": "username", + "value": "dripline", + "type": "string" + } + ] + }, + "method": "GET", + "header": [], + "url": { + "raw": "rabbit-broker:15672/api/exchanges/%2f/requests", + "host": [ + "rabbit-broker" + ], + "port": "15672", + "path": [ + "api", + "exchanges", + "%2f", + "requests" + ] + } + }, + "response": [] + }, + { + "name": "rabbit-broker:15672/api/exchanges/%2f/alerts", + "event": [ + { + "listen": "test", + "script": { + "exec": [ + "pm.test(\"Status code is 200\", function () {", + " pm.response.to.have.status(200);", + "});", + "", + "pm.test(\"The response has the expected properties\", function () {", + " const responseJson = pm.response.json();", + " pm.expect(responseJson.durable).to.be.a(\"boolean\");", + " pm.expect(responseJson.durable).to.eql(true);", + " pm.expect(responseJson.name).to.eql(\"alerts\");", + " pm.expect(responseJson.type).to.eql(\"topic\");", + "});", + "" + ], + "type": "text/javascript", + "packages": {}, + "requests": {} + } + } + ], + "request": { + "auth": { + "type": "basic", + "basic": [ + { + "key": "password", + "value": "dripline", + "type": "string" + }, + { + "key": "username", + "value": "dripline", + "type": "string" + } + ] + }, + "method": "GET", + "header": [], + "url": { + "raw": "rabbit-broker:15672/api/exchanges/%2f/alerts", + "host": [ + "rabbit-broker" + ], + "port": "15672", + "path": [ + "api", + "exchanges", + "%2f", + "alerts" + ] + } + }, + "response": [] + }, + { + "name": "rabbit-broker:15672/api/queues/%2f/simple", + "event": [ + { + "listen": "test", + "script": { + "exec": [ + "pm.test(\"Status code is 200\", function () {", + " pm.response.to.have.status(200);", + "});", + "", + "pm.test(\"The response has the expected properties\", function () {", + " const responseJson = pm.response.json();", + " pm.expect(responseJson.durable).to.be.a(\"boolean\");", + " pm.expect(responseJson.durable).to.eql(false);", + " pm.expect(responseJson.name).to.eql(\"simple\");", + "});", + "" + ], + "type": "text/javascript", + "packages": {}, + "requests": {} + } + } + ], + "request": { + "auth": { + "type": "basic", + "basic": [ + { + "key": "password", + "value": "dripline", + "type": "string" + }, + { + "key": "username", + "value": "dripline", + "type": "string" + } + ] + }, + "method": "GET", + "header": [], + "url": { + "raw": "rabbit-broker:15672/api/queues/%2f/simple", + "host": [ + "rabbit-broker" + ], + "port": "15672", + "path": [ + "api", + "queues", + "%2f", + "simple" + ] + } + }, + "response": [] + }, + { + "name": "rabbit-broker:15672/api/bindings/%2f/e/requests/q/simple", + "event": [ + { + "listen": "test", + "script": { + "exec": [ + "pm.test(\"Status code is 200\", function () {", + " pm.response.to.have.status(200);", + "});", + "", + "pm.test(\"The response has the right properties\", function () {", + " const responseJson = pm.response.json();", + " pm.expect(responseJson).to.be.an(\"array\");", + "", + " pm.expect(responseJson[0].source).to.eql(\"requests\");", + " pm.expect(responseJson[0].destination).to.eql(\"simple\");", + " pm.expect(responseJson[0].routing_key).to.eql(\"broadcast.#\");", + " pm.expect(responseJson[1].source).to.eql(\"requests\");", + " pm.expect(responseJson[1].destination).to.eql(\"simple\");", + " pm.expect(responseJson[1].routing_key).to.eql(\"simple.#\");", + "});", + "", + "" + ], + "type": "text/javascript", + "packages": {}, + "requests": {} + } + }, + { + "listen": "prerequest", + "script": { + "packages": {}, + "type": "text/javascript" + } + } + ], + "request": { + "method": "GET", + "header": [], + "url": { + "raw": "rabbit-broker:15672/api/bindings/%2f/e/requests/q/simple", + "host": [ + "rabbit-broker" + ], + "port": "15672", + "path": [ + "api", + "bindings", + "%2f", + "e", + "requests", + "q", + "simple" + ] + } + }, + "response": [] + } + ], + "auth": { + "type": "basic", + "basic": [ + { + "key": "password", + "value": "dripline", + "type": "string" + }, + { + "key": "username", + "value": "dripline", + "type": "string" + } + ] + }, + "event": [ + { + "listen": "prerequest", + "script": { + "type": "text/javascript", + "packages": {}, + "exec": [ + "" + ] + } + }, + { + "listen": "test", + "script": { + "type": "text/javascript", + "packages": {}, + "exec": [ + "" + ] + } + } + ] } \ No newline at end of file From dcfeee5b68cfe9b8a856fc04291699150eb44daa Mon Sep 17 00:00:00 2001 From: Noah Oblath Date: Wed, 3 Jun 2026 14:08:12 -0700 Subject: [PATCH 25/45] Add the multi-chunk integration test --- testing/integration/dl-tests.sh | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/testing/integration/dl-tests.sh b/testing/integration/dl-tests.sh index f6a610e3..036cf9a8 100755 --- a/testing/integration/dl-tests.sh +++ b/testing/integration/dl-tests.sh @@ -12,6 +12,11 @@ dl-agent -vv set simple 500 } +@test "multi-chunk set" { + LARGE_VALUE=$(python3 -c "print('x' * 12000)") + dl-agent -vv --max-payload 500 set simple "${LARGE_VALUE}" +} + @test "user/pass on CL" { dl-agent -vv -u dripline --password dripline get simple } From 48705326a8aedb86f969d19d14bb63d3d78412df Mon Sep 17 00:00:00 2001 From: Noah Oblath Date: Wed, 3 Jun 2026 15:11:08 -0700 Subject: [PATCH 26/45] Renamed dl-tests.sh to run-dl-tests.sh and use --exit-code-from option to avoid false positive problem --- testing/integration/{do-testing.sh => run-dl-tests.sh} | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) rename testing/integration/{do-testing.sh => run-dl-tests.sh} (97%) diff --git a/testing/integration/do-testing.sh b/testing/integration/run-dl-tests.sh similarity index 97% rename from testing/integration/do-testing.sh rename to testing/integration/run-dl-tests.sh index f583947f..555805a3 100755 --- a/testing/integration/do-testing.sh +++ b/testing/integration/run-dl-tests.sh @@ -17,7 +17,7 @@ cleanup () { } trap 'cleanup ; printf "${RED}Tests Failed For Unexpected Reasons${NC}\n"' HUP INT QUIT PIPE TERM -IMG_TAG=$1 docker compose -p integration -f docker-compose.yaml -f docker-compose-test.yaml build && IMG_TAG=$1 docker compose -p integration -f docker-compose.yaml -f docker-compose-test.yaml up -d +IMG_TAG=$1 docker compose -p integration -f docker-compose.yaml -f docker-compose-test.yaml build && IMG_TAG=$1 docker compose -p integration -f docker-compose.yaml -f docker-compose-test.yaml up --exit-code-from test if [ $? -ne 0 ] ; then printf "${RED}Docker Compose Failed${NC}\n" From 0f2548a650d7497dc2bc0e64081e8823fe954d35 Mon Sep 17 00:00:00 2001 From: Noah Oblath Date: Wed, 3 Jun 2026 15:21:50 -0700 Subject: [PATCH 27/45] Added broker-restart test --- testing/integration/broker-restart-test.sh | 10 ++++ .../docker-compose-broker-restart-test.yaml | 16 +++++ .../integration/run-broker-restart-test.sh | 59 +++++++++++++++++++ 3 files changed, 85 insertions(+) create mode 100755 testing/integration/broker-restart-test.sh create mode 100644 testing/integration/docker-compose-broker-restart-test.yaml create mode 100755 testing/integration/run-broker-restart-test.sh diff --git a/testing/integration/broker-restart-test.sh b/testing/integration/broker-restart-test.sh new file mode 100755 index 00000000..4582c2c0 --- /dev/null +++ b/testing/integration/broker-restart-test.sh @@ -0,0 +1,10 @@ +#! /usr/bin/env bats + +@test "ping simple" { + until curl -sf -u dripline:dripline http://rabbit-broker:15672/api/overview > /dev/null 2>&1; do + echo "Broker not yet recovered; waiting..." + sleep 2 + done + sleep 2 # let the service reconnect + dl-agent -vv cmd simple -s ping +} diff --git a/testing/integration/docker-compose-broker-restart-test.yaml b/testing/integration/docker-compose-broker-restart-test.yaml new file mode 100644 index 00000000..2253a8e3 --- /dev/null +++ b/testing/integration/docker-compose-broker-restart-test.yaml @@ -0,0 +1,16 @@ +services: + test: + image: ghcr.io/driplineorg/dripline-cpp:${IMG_TAG:-latest-testing} + depends_on: + - rabbit-broker + - simple-service + command: > + bash -c "sleep 5 && + echo 'Running broker-restart test' && + /root/broker-restart-test.sh" + volumes: + - ./broker-restart-test.sh:/root/broker-restart-test.sh + - ./dripline_mesh.yaml:/root/.dripline_mesh.yaml + environment: + - DRIPLINE_USER=dripline + - DRIPLINE_PASSWORD=dripline diff --git a/testing/integration/run-broker-restart-test.sh b/testing/integration/run-broker-restart-test.sh new file mode 100755 index 00000000..13b8f66b --- /dev/null +++ b/testing/integration/run-broker-restart-test.sh @@ -0,0 +1,59 @@ +#! /bin/bash + +# Runs the test suite using docker compose +# +# Usage: +# do-testing.sh [image tag] + +# source: https://blog.harrison.dev/2016/06/19/integration-testing-with-docker-compose.html + +RED='\033[0;31m' +GREEN='\033[0;32m' +NC='\033[0m' + +cleanup () { + docker compose -p integration kill + docker compose -p integration rm -f +} +trap 'cleanup ; printf "${RED}Tests Failed For Unexpected Reasons${NC}\n"' HUP INT QUIT PIPE TERM + +IMG_TAG=$1 docker compose -p integration -f docker-compose.yaml -f docker-compose-broker-restart-test.yaml build && IMG_TAG=$1 docker compose -p integration -f docker-compose.yaml -f docker-compose-broker-restart-test.yaml up -d rabbit-broker simple-service + +# Wait for broker health +#echo "Waiting for broker to become healthy..." +#until docker exec integration-rabbit-broker-1 \ +# curl -sf -u dripline:dripline http://localhost:15672/api/overview > /dev/null 2>&1; do +# echo " broker not yet healthy, waiting..." +# sleep 2 +#done + +# Pause to allow the service to connect +sleep 5 + +# Restart broker +# Broker will restart and simple-service will continually try to reconnect +# Test for service connection (dl-agent cmd -s ping simple) will wait until broker is back +docker compose -p integration restart rabbit-broker + +# Now run the test container +IMG_TAG=$1 docker compose -p integration -f docker-compose.yaml -f docker-compose-broker-restart-test.yaml up --exit-code-from test test + +if [ $? -ne 0 ] ; then + printf "${RED}Docker Compose Failed${NC}\n" + exit -1 +fi + +TEST_EXIT_CODE=`docker wait integration-test-1` +docker logs integration-test-1 +if [ -z ${TEST_EXIT_CODE+x} ] || [ "$TEST_EXIT_CODE" -ne 0 ] ; then + docker logs integration-test-1 + docker logs integration-simple-service-1 + docker logs integration-rabbit-broker-1 + printf "${RED}Tests Failed${NC} - Exit Code: $TEST_EXIT_CODE\n" +else + printf "${GREEN}Tests Passed${NC}\n" +fi + +cleanup + +exit $TEST_EXIT_CODE From 18b51361c27c2eb83a7f55034d5415ec7338460f Mon Sep 17 00:00:00 2001 From: Noah Oblath Date: Wed, 3 Jun 2026 15:32:29 -0700 Subject: [PATCH 28/45] Updated integration test documentation --- testing/integration/README.md | 56 +++++++++++++++++++++++++++-------- 1 file changed, 43 insertions(+), 13 deletions(-) diff --git a/testing/integration/README.md b/testing/integration/README.md index 9fcb0608..b4ae198d 100644 --- a/testing/integration/README.md +++ b/testing/integration/README.md @@ -9,9 +9,13 @@ If you build the image, we recommend you tag the image with something like `ghcr ## The Tests -The next two sub-sections detail the tests that are run. Skip to [the next section](#run-the-tests) to actually run the tests. +There are two independent test suites, each with its own convenience script and Docker Compose extension file. -### Broker Setup +### DL Tests (`run-dl-tests.sh`) + +This suite exercises broker setup and Dripline service functionality. It uses `docker-compose-test.yaml` as the Docker Compose extension. + +#### Broker Setup The first tests run demonstrate that the RabbitMQ broker has been configured correctly once the services are connected. We check that the expected exchanges and queues are in place by sending HTTP requests to the broker and validating the responses. The `newman` CLI application is used to run a Postman collection. @@ -21,39 +25,65 @@ The command to run the collection is: The collection file is setup to run in the Docker compose environment, where the broker is addressed at `rabbit-broker`. If you want to run it on a host machine on which the Docker setup is running and port 15672 has been opened, you can change instances of `rabbit-broker` in the JSON file to `localhost`, and use the same command. -### Dripline Services +#### Dripline Services We test use of Dripline services by sending requests to those services and validating the results. The tests are specified in `dl-tests.sh`. They're run and checked using the [bats framework](https://bats-core.readthedocs.io/en/stable/index.html). Currently the tests include: -* `dl-agent cmd ping -s simple` -* `dl-agent get simple` -* `dl-agent set simple 500` +* `ping simple` — sends a ping command to the simple service +* `get simple` — sends a get request to the simple service +* `set simple 500` — sends a set request to the simple service +* `multi-chunk set` — sends a large set request that is split into multiple AMQP message chunks +* `user/pass on CL` — verifies authentication with username and password supplied on the command line +* `user on CL/pass in file` — verifies authentication with username on the command line and password read from a file +* `auth file` — verifies authentication using a JSON auth file +* `oscillator hub` — sends a get request to the oscillator hub service +* `oscillator endpoints` — sends a get request to an oscillator endpoint service You can run these tests directly by executing the script: > /path/to/dl-tests.sh -## Run the Tests +#### Running the DL Tests You can run the tests directly with `docker compose`: > IMG_TAG=[image tag] docker compose -f docker-compose.yaml -f docker-compose-test.yaml up -Or you can use the convenience script, `do-testing.sh`: +Or you can use the convenience script, `run-dl-tests.sh`: + + > ./run-dl-tests.sh [image tag] + +Note that the terminal output is suppressed using the `run-dl-tests.sh` script. Output from the test container will be printed. Output from the service(s) in use will be printed on error. + +--- + +### Broker-Restart Test (`run-broker-restart-test.sh`) - > ./do-testing.sh [image tag] +This suite verifies that Dripline services automatically reconnect to the broker after the broker is restarted. It uses `docker-compose-broker-restart-test.yaml` as the Docker Compose extension. -Note that the terminal output is suppressed using the `do-testing.sh` script. Output from the test container will be printed. Output from the service(s) in use will be printed on error. +The test sequence is: +1. Start the broker (`rabbit-broker`) and the simple service (`simple-service`). +2. Wait 5 seconds for the service to connect. +3. Restart the broker with `docker compose restart rabbit-broker`. +4. Start the test container, which waits for the broker to recover and then waits an additional 2 seconds for the service to reconnect. +5. Send a ping to the simple service to confirm it has successfully reconnected. + +The test is specified in `broker-restart-test.sh` and run with the [bats framework](https://bats-core.readthedocs.io/en/stable/index.html). + +#### Running the Broker-Restart Test + + > ./run-broker-restart-test.sh [image tag] + +--- ## Implementation Notes * Synchronizing containers - *Anything using the broker needs to wait until the broker is ready. This is done using a (Docker) healthcheck. The health status of the broker is determined by a curl request to the HTTP server that RabbitMQ comes with. Once it has a healthy status, the DL services start. + * Anything using the broker needs to wait until the broker is ready. This is done using a (Docker) healthcheck. The health status of the broker is determined by a curl request to the HTTP server that RabbitMQ comes with. Once it has a healthy status, the DL services start. * Sending a request to a service requires that the service is running; so far we wait for 1 second after the broker is ready, and assume the service will be running. This could be changed in the future to do something like wait-for-broker.sh except by sending pings to the relevant service. -* To include the test docker-compose extension, add a second -f argument: +* To include a test docker-compose extension, add a second `-f` argument, e.g.: > docker compose -f docker-compose.yaml -f docker-compose-test.yaml up * Specify the docker image tag by setting an environment variable when you run, e.g. > IMG_TAG=v[version]-dev docker compose -f docker-compose.yaml up * There's a default tag (latest-dev), so if this isn't specified, everything should still run. - From b429bfdfc8c48e06aaa80dd50171cdcd61b21ea2 Mon Sep 17 00:00:00 2001 From: Noah Oblath Date: Wed, 3 Jun 2026 16:04:10 -0700 Subject: [PATCH 29/45] Renamed concurrent_receiver to message_dispatcher and updated source documentation. --- AGENTS.md | 4 +- changelog.md | 15 +++ library/CMakeLists.txt | 2 + library/README.md | 23 +++- library/endpoint.cc | 8 +- library/endpoint.hh | 6 +- library/message_dispatcher.cc | 86 ++++++++++++++ library/message_dispatcher.hh | 74 ++++++++++++ library/monitor.cc | 2 +- library/monitor.hh | 4 +- library/receiver.cc | 67 ----------- library/receiver.hh | 72 ++---------- library/service.cc | 4 +- library/service.hh | 23 ++-- testing/CMakeLists.txt | 7 +- testing/test_message_dispatcher.cc | 175 +++++++++++++++++++++++++++++ 16 files changed, 406 insertions(+), 166 deletions(-) create mode 100644 library/message_dispatcher.cc create mode 100644 library/message_dispatcher.hh create mode 100644 testing/test_message_dispatcher.cc diff --git a/AGENTS.md b/AGENTS.md index 4e29b635..32e6891d 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -18,9 +18,9 @@ The goal is to make safe, minimal, style-consistent changes to dripline-cpp. - `core` owns AMQP connectivity and send/listen primitives. - `message` and derived types (`msg_request`, `msg_reply`, `msg_alert`) implement protocol objects and chunking. -- `receiver` and `listener` manage chunk assembly and concurrent processing. +- `receiver` manages chunk assembly; `message_dispatcher` (formerly `concurrent_receiver`) holds the rmqcpp Consumer and dispatches assembled messages. - `endpoint` implements request dispatch and lockout semantics. -- `service` composes endpoint + listener/receiver + heartbeater + scheduler. +- `service` composes endpoint + message_dispatcher + heartbeater + scheduler. - `hub` maps message specifiers to user-registered handlers. - `agent` and `monitor` provide CLI-oriented message send/observe tooling. diff --git a/changelog.md b/changelog.md index 23bab2e1..dcc5ebb2 100644 --- a/changelog.md +++ b/changelog.md @@ -9,6 +9,21 @@ Types of changes: Added, Changed, Deprecated, Removed, Fixed, Security ## [Unreleased] +### Changed + +- Replaced SimpleAmqpClient with rmqcpp (Bloomberg) as the AMQP backend. + - `concurrent_receiver` renamed to `message_dispatcher`; extracted into `library/message_dispatcher.hh` / `library/message_dispatcher.cc`. + - `listener` and `listener_receiver` classes deleted; Consumer lifecycle absorbed into `message_dispatcher`. + - Message delivery is now callback-based (rmqcpp thread pool); dripline-cpp no longer manages listener or receiver threads. + - Stale incomplete multi-chunk messages are lazily evicted after `single_message_wait_ms` ms. + - `core::send()` no longer accepts an `amqp_channel_ptr` parameter (breaking API change). + - `receiver::wait_for_reply()` now uses a `std::future` instead of a polling loop. + +### Removed + +- `library/listener.hh` and `library/listener.cc` (deleted). +- `external/SimpleAmqpClient/` submodule (deleted). + ## [2.10.11] -- 2026-01-23 diff --git a/library/CMakeLists.txt b/library/CMakeLists.txt index addfb980..b4baf9b9 100644 --- a/library/CMakeLists.txt +++ b/library/CMakeLists.txt @@ -23,6 +23,7 @@ set( dripline_HEADERS heartbeater.hh hub.hh message.hh + message_dispatcher.hh monitor.hh monitor_config.hh receiver.hh @@ -49,6 +50,7 @@ set( dripline_SOURCES heartbeater.cc hub.cc message.cc + message_dispatcher.cc monitor.cc monitor_config.cc receiver.cc diff --git a/library/README.md b/library/README.md index b51bb8b4..d5461465 100644 --- a/library/README.md +++ b/library/README.md @@ -1,3 +1,22 @@ -# dripline-cpp/source +# dripline-cpp/library -This directory contains all of the source files for dripline-cpp, both executables and the library. +This directory contains all of the source files for the dripline-cpp library. + +## Key Classes + +| Class | Files | Description | +|---|---|---| +| `core` | `core.hh` / `core.cc` | AMQP connectivity, send primitives, VHost/Topology management | +| `message`, `msg_request`, `msg_reply`, `msg_alert` | `message.hh` / `message.cc` | Protocol objects, serialization, chunking | +| `receiver` | `receiver.hh` / `receiver.cc` | Chunk assembly, incoming-message map, stale-entry eviction, `wait_for_reply()` | +| `message_dispatcher` | `message_dispatcher.hh` / `message_dispatcher.cc` | rmqcpp Consumer lifecycle (`start_listening` / `stop_listening`), dispatches assembled messages to `submit_message()`. Formerly named `concurrent_receiver`. | +| `endpoint` | `endpoint.hh` / `endpoint.cc` | Request dispatch, lockout semantics | +| `endpoint_listener_receiver` | `endpoint.hh` / `endpoint.cc` | Decorator: wraps an `endpoint` with `message_dispatcher` capabilities for async child use | +| `service` | `service.hh` / `service.cc` | Composes `core` + `endpoint` + `message_dispatcher` + `heartbeater` + `scheduler` | +| `monitor` | `monitor.hh` / `monitor.cc` | Composes `core` + `message_dispatcher`; prints received messages | +| `hub` | `hub.hh` / `hub.cc` | Maps specifiers to user-registered handler functions | +| `agent` | `agent.hh` / `agent.cc` | CLI-oriented one-shot message sender | +| `heartbeater` | `heartbeater.hh` / `heartbeater.cc` | Periodic heartbeat sender | +| `scheduler` | `scheduler.hh` | Event scheduler | +| `relayer` | `relayer.hh` / `relayer.cc` | Forwards messages between services | +| `reply_cache` | `reply_cache.hh` / `reply_cache.cc` | Python-binding reply cache (built only with `Scarab_BUILD_PYTHON`) | diff --git a/library/endpoint.cc b/library/endpoint.cc index 795881c6..5e6c705e 100644 --- a/library/endpoint.cc +++ b/library/endpoint.cc @@ -432,14 +432,12 @@ namespace dripline } endpoint_listener_receiver::endpoint_listener_receiver( endpoint_ptr_t a_endpoint_ptr ) : - scarab::cancelable(), - concurrent_receiver(), + message_dispatcher(), f_endpoint( a_endpoint_ptr ) {} endpoint_listener_receiver::endpoint_listener_receiver( endpoint_listener_receiver&& a_orig ) : - scarab::cancelable( std::move(a_orig) ), - concurrent_receiver( std::move(a_orig) ), + message_dispatcher( std::move(a_orig) ), f_endpoint( std::move(a_orig.f_endpoint) ) {} @@ -448,7 +446,7 @@ namespace dripline endpoint_listener_receiver& endpoint_listener_receiver::operator=( endpoint_listener_receiver&& a_orig ) { - concurrent_receiver::operator=( std::move(a_orig) ); + message_dispatcher::operator=( std::move(a_orig) ); f_endpoint = std::move(a_orig.f_endpoint); return *this; } diff --git a/library/endpoint.hh b/library/endpoint.hh index 395d1e64..dffd6bf0 100644 --- a/library/endpoint.hh +++ b/library/endpoint.hh @@ -9,7 +9,7 @@ #define DRIPLINE_ENDPOINT_HH_ #include "message.hh" -#include "receiver.hh" +#include "message_dispatcher.hh" #include "return_codes.hh" namespace dripline @@ -257,12 +257,12 @@ namespace dripline @class endpoint_listener_receiver @author N.S. Oblath - @brief Decorator class for a plain endpoint: adds concurrent_receiver capabilities. + @brief Decorator class for a plain endpoint: adds message_dispatcher capabilities. @details The endpoint_listener_receiver is used by @ref service to wrap an endpoint that is to listen for messages asynchronously. */ - class DRIPLINE_API endpoint_listener_receiver : public concurrent_receiver + class DRIPLINE_API endpoint_listener_receiver : public message_dispatcher { public: explicit endpoint_listener_receiver( endpoint_ptr_t a_endpoint_ptr ); diff --git a/library/message_dispatcher.cc b/library/message_dispatcher.cc new file mode 100644 index 00000000..fed6de66 --- /dev/null +++ b/library/message_dispatcher.cc @@ -0,0 +1,86 @@ +/* + * message_dispatcher.cc + * + * Created on: Jun 3, 2026 + * Author: N.S. Oblath + */ + +#define DRIPLINE_API_EXPORTS + +#include "message_dispatcher.hh" + +#include "dripline_exceptions.hh" + +#include "rmqa_consumer.h" +#include "rmqa_vhost.h" +#include "rmqp_messageguard.h" + +#include "logger.hh" + +LOGGER( dlog, "message_dispatcher" ); + +namespace dripline +{ + + message_dispatcher::message_dispatcher() : + receiver(), + f_consumer() + {} + + message_dispatcher::message_dispatcher( message_dispatcher&& a_orig ) : + receiver( std::move(a_orig) ), + f_consumer( std::move(a_orig.f_consumer) ) + {} + + message_dispatcher::~message_dispatcher() + {} + + message_dispatcher& message_dispatcher::operator=( message_dispatcher&& a_orig ) + { + receiver::operator=( std::move(a_orig) ); + f_consumer = std::move(a_orig.f_consumer); + return *this; + } + + void message_dispatcher::process_message( message_ptr_t a_message ) + { + this->submit_message( a_message ); + return; + } + + void message_dispatcher::start_listening( bsl::shared_ptr< BloombergLP::rmqa::VHost > a_vhost, + const BloombergLP::rmqa::Topology& a_topology, + const BloombergLP::rmqt::QueueHandle& a_queue_handle, + const std::string& a_label ) + { + using namespace BloombergLP; + auto t_result = a_vhost->createConsumer( + a_topology, a_queue_handle, + [this]( rmqp::MessageGuard& guard ) { + amqp_envelope_ptr t_envelope = guard.transferOwnership(); + t_envelope->ack(); + handle_message_chunk( std::move(t_envelope) ); + }, + a_label, + 1 ); + if( ! t_result ) + { + throw connection_error() << "Unable to create consumer: " << t_result.error(); + } + f_consumer = t_result.value(); + } + + void message_dispatcher::stop_listening() + { + if( f_consumer ) + { + auto t_result = f_consumer->cancelAndDrain(); + if( ! t_result ) + { + LWARN( dlog, "Error canceling consumer: " << t_result.error() ); + } + f_consumer.reset(); + } + } + +} /* namespace dripline */ diff --git a/library/message_dispatcher.hh b/library/message_dispatcher.hh new file mode 100644 index 00000000..67ad7608 --- /dev/null +++ b/library/message_dispatcher.hh @@ -0,0 +1,74 @@ +/* + * message_dispatcher.hh + * + * Created on: Jun 3, 2026 + * Author: N.S. Oblath + */ + +#ifndef DRIPLINE_MESSAGE_DISPATCHER_HH_ +#define DRIPLINE_MESSAGE_DISPATCHER_HH_ + +#include "receiver.hh" + +#include "rmqa_topology.h" + +namespace dripline +{ + + /*! + @class message_dispatcher + @author N.S. Oblath + + @brief Receives assembled Dripline messages from an AMQP consumer and dispatches them. + + @details + This class manages the lifecycle of an rmqcpp Consumer (via `start_listening()` / + `stop_listening()`) and overrides `process_message()` to dispatch each assembled + Dripline message directly to `submit_message()` (synchronously in the rmqcpp + callback thread). + + A class deriving from `message_dispatcher` must implement `submit_message()` to + define what happens with each received message. + + @note + The name `concurrent_receiver` was used for this class prior to the rmqcpp migration. + After the migration the class no longer manages any concurrency itself — rmqcpp's + internal thread pool handles delivery — so it was renamed to `message_dispatcher`. + */ + class DRIPLINE_API message_dispatcher : public receiver + { + public: + message_dispatcher(); + message_dispatcher( const message_dispatcher& ) = delete; + message_dispatcher( message_dispatcher&& a_orig ); + virtual ~message_dispatcher(); + + message_dispatcher& operator=( const message_dispatcher& ) = delete; + message_dispatcher& operator=( message_dispatcher&& a_orig ); + + public: + /// Dispatches the message directly to `submit_message()`. + virtual void process_message( message_ptr_t a_message ); + + /// Creates an rmqcpp consumer on the given queue and begins receiving messages. + /// Each received message is passed to handle_message_chunk(). + void start_listening( bsl::shared_ptr< BloombergLP::rmqa::VHost > a_vhost, + const BloombergLP::rmqa::Topology& a_topology, + const BloombergLP::rmqt::QueueHandle& a_queue_handle, + const std::string& a_label = "" ); + + /// Cancels the rmqcpp consumer, drains in-flight messages, and releases it. + void stop_listening(); + + /// Handles messages according to the use case. Must be implemented by the + /// class inheriting from message_dispatcher. + /// For a concrete example, see @ref service or @ref endpoint_listener_receiver. + virtual void submit_message( message_ptr_t a_message ) = 0; + + protected: + bsl::shared_ptr< BloombergLP::rmqa::Consumer > f_consumer; + }; + +} /* namespace dripline */ + +#endif /* DRIPLINE_MESSAGE_DISPATCHER_HH_ */ diff --git a/library/monitor.cc b/library/monitor.cc index d0613f61..14662f33 100644 --- a/library/monitor.cc +++ b/library/monitor.cc @@ -26,7 +26,7 @@ namespace dripline monitor::monitor( const scarab::param_node& a_config, const scarab::authentication& a_auth ) : scarab::cancelable(), core( a_config["dripline_mesh"].as_node(), a_auth ), - concurrent_receiver(), + message_dispatcher(), f_status( status::nothing ), f_name( std::string("monitor_") + string_from_uuid(generate_random_uuid()) ), f_json_print( false ), diff --git a/library/monitor.hh b/library/monitor.hh index df80fece..87d95dbd 100644 --- a/library/monitor.hh +++ b/library/monitor.hh @@ -9,7 +9,7 @@ #define DRIPLINE_MONITOR_HH_ #include "core.hh" -#include "receiver.hh" +#include "message_dispatcher.hh" namespace scarab { @@ -42,7 +42,7 @@ namespace DRIPLINE_API dripline */ class monitor : public core, - public concurrent_receiver + public message_dispatcher { protected: enum class status diff --git a/library/receiver.cc b/library/receiver.cc index 2d64cd01..2ed5a1e6 100644 --- a/library/receiver.cc +++ b/library/receiver.cc @@ -12,10 +12,6 @@ #include "dripline_exceptions.hh" #include "message.hh" -#include "rmqa_consumer.h" -#include "rmqa_vhost.h" -#include "rmqp_messageguard.h" - #include "logger.hh" #include @@ -219,67 +215,4 @@ namespace dripline return reply_ptr_t(); } - concurrent_receiver::concurrent_receiver() : - receiver(), - f_consumer() - {} - - concurrent_receiver::concurrent_receiver( concurrent_receiver&& a_orig ) : - receiver( std::move(a_orig) ), - f_consumer( std::move(a_orig.f_consumer) ) - {} - - concurrent_receiver::~concurrent_receiver() - {} - - concurrent_receiver& concurrent_receiver::operator=( concurrent_receiver&& a_orig ) - { - receiver::operator=( std::move(a_orig) ); - f_consumer = std::move(a_orig.f_consumer); - return *this; - } - - void concurrent_receiver::process_message( message_ptr_t a_message ) - { - this->submit_message( a_message ); - return; - } - - void concurrent_receiver::start_listening( bsl::shared_ptr< BloombergLP::rmqa::VHost > a_vhost, - const BloombergLP::rmqa::Topology& a_topology, - const BloombergLP::rmqt::QueueHandle& a_queue_handle, - const std::string& a_label ) - { - using namespace BloombergLP; - auto t_result = a_vhost->createConsumer( - a_topology, a_queue_handle, - [this]( rmqp::MessageGuard& guard ) { - amqp_envelope_ptr t_envelope = guard.transferOwnership(); - t_envelope->ack(); - handle_message_chunk( std::move(t_envelope) ); - }, - a_label, - 1 ); - if( ! t_result ) - { - throw connection_error() << "Unable to create consumer: " << t_result.error(); - } - f_consumer = t_result.value(); - } - - void concurrent_receiver::stop_listening() - { - if( f_consumer ) - { - auto t_result = f_consumer->cancelAndDrain(); - if( ! t_result ) - { - LWARN( dlog, "Error canceling consumer: " << t_result.error() ); - } - f_consumer.reset(); - } - } - - - } /* namespace dripline */ diff --git a/library/receiver.hh b/library/receiver.hh index 3a964997..8d58bc2f 100644 --- a/library/receiver.hh +++ b/library/receiver.hh @@ -12,8 +12,6 @@ #include "dripline_api.hh" #include "dripline_fwd.hh" -#include "rmqa_topology.h" - #include "cancelable.hh" #include "member_variables.hh" @@ -51,26 +49,26 @@ namespace dripline @brief A receiver is able to collect Dripline message chunks and reassemble them into a complete Dripline message. @details - This is a mix-in class for synchronously receiving and processing Dripline messages. + This is a mix-in class for receiving and processing Dripline messages. - Dripline messages can be broken up into multiple chunks, each of which is transported as an AMQP message. - A receiver is responsible for handling message chunks, storing incomplete Dripline messages, and eventually + Dripline messages can be broken up into multiple chunks, each of which is transported as an AMQP message. + A receiver is responsible for handling message chunks, storing incomplete Dripline messages, and eventually processing complete Dripline messages. The receiver class contains an interface specifically for users waiting to receive reply messages: `wait_for_reply()`. - When the first message chunk for a message is received, one of two things happens: - 1. if the message comprises one chunk, then the message is processed immediately; - 2. if the message comprises multiple chunks, then a separate thread is spun up to wait for the remaining chunks. + When a message chunk arrives via `handle_message_chunk()`, it is stored in the incoming-message map. + Message chunks for a given message can be received in any order. Once all chunks for a message have + arrived, `process_message_pack()` is called inline (no separate thread is spawned). - Incomplete messages are stored in the incoming-message map. Message chunks for a given message can be received - in any order. The receiver will wait `single_message_wait_ms` ms for all of the chunks of a message to arrive - before timing out processing the incomplete message. + Stale incomplete messages (entries older than `single_message_wait_ms` ms) are lazily evicted at the + start of each `handle_message_chunk()` call. The actual assembly of message chunks into complete messages is done in @ref message. - The `receiver` class itself does not know how to process a message. This must be implemented by the class derived from `receiver`. - The default implementation of `process_message()` will throw a `dripline_error`. + The `receiver` class itself does not know how to process a message. This must be implemented by the + class derived from `receiver`. The default implementation of `process_message()` will throw a + `dripline_error`. */ class DRIPLINE_API receiver : public virtual scarab::cancelable { @@ -120,54 +118,6 @@ namespace dripline }; - /*! - @class concurrent_receiver - @author N.S. Oblath - - @brief Receives and processes messages concurrently - - @details - This class enables Dripline messages to be received and processed. - - A consumer callback (set up via `start_listening()`) receives messages from the AMQP broker and - calls `receiver::handle_message_chunk()`, which assembles chunks and calls `submit_message()` - directly (synchronously in the rmqcpp callback thread). - - A class deriving from concurrent_receiver must implement `submit_message()`. - */ - class DRIPLINE_API concurrent_receiver : public receiver - { - public: - concurrent_receiver(); - concurrent_receiver( const concurrent_receiver& ) = delete; - concurrent_receiver( concurrent_receiver&& a_orig ); - virtual ~concurrent_receiver(); - - concurrent_receiver& operator=( const concurrent_receiver& ) = delete; - concurrent_receiver& operator=( concurrent_receiver&& a_orig ); - - public: - /// Dispatches the message directly to `submit_message()`. - virtual void process_message( message_ptr_t a_message ); - - /// Creates an rmqcpp consumer on the given queue and begins receiving messages. - /// Each received message is passed to handle_message_chunk(). - void start_listening( bsl::shared_ptr< BloombergLP::rmqa::VHost > a_vhost, - const BloombergLP::rmqa::Topology& a_topology, - const BloombergLP::rmqt::QueueHandle& a_queue_handle, - const std::string& a_label = "" ); - - /// Cancels the rmqcpp consumer, drains in-flight messages, and releases it. - void stop_listening(); - - /// Handles messages according to the use case. It's to be implemented by the class inheriting from concurrent_receiver - /// For a concrete example, see @ref service or @ref endpoint_listener_receiver. - virtual void submit_message( message_ptr_t a_message ) = 0; - - protected: - bsl::shared_ptr< BloombergLP::rmqa::Consumer > f_consumer; - }; - } /* namespace dripline */ #endif /* DRIPLINE_RECEIVER_HH_ */ diff --git a/library/service.cc b/library/service.cc index 3831dc7a..82531edc 100644 --- a/library/service.cc +++ b/library/service.cc @@ -33,7 +33,7 @@ namespace dripline core( a_config.has("dripline_mesh") ? a_config["dripline_mesh"].as_node() : dripline_config(), a_auth, a_make_connection ), endpoint( a_config.get_value( "name", "dlcpp_service" ) ), - concurrent_receiver(), + message_dispatcher(), heartbeater( this ), scheduler<>(), f_auth( a_auth ), @@ -69,7 +69,7 @@ namespace dripline cancelable::operator=( std::move(a_orig) ); core::operator=( std::move(a_orig) ); endpoint::operator=( std::move(a_orig)); - concurrent_receiver::operator=( std::move(a_orig) ); + message_dispatcher::operator=( std::move(a_orig) ); heartbeater::operator=( std::move(a_orig) ); scheduler<>::operator=( std::move(a_orig) ); diff --git a/library/service.hh b/library/service.hh index 0e67c9e3..20def9c6 100644 --- a/library/service.hh +++ b/library/service.hh @@ -11,7 +11,7 @@ #include "core.hh" #include "endpoint.hh" #include "heartbeater.hh" -#include "receiver.hh" +#include "message_dispatcher.hh" #include "scheduler.hh" #include "dripline_exceptions.hh" @@ -56,21 +56,14 @@ namespace dripline A service has a number of key characteristics (most of which come from its parent classes): * `core` -- Has all of the basic AMQP capabilities, sending messages, and making and manipulating connections * `endpoint` -- Handles Dripline messages - * `concurrent_receiver` -- Asynchronously receives AMQP messages and turns them into Dripline messages + * `message_dispatcher` -- Receives AMQP messages via rmqcpp callbacks and dispatches them as Dripline messages * `heartbeater` -- Sends periodic heartbeat messages * `scheduler` -- Can schedule events - As is apparent from the above descriptions, a service is responsible for a number of threads - when it executes: - * Listening -- grabs AMQP messages off the channel when they arrive - * Message-wait -- any incomplete multi-part Dripline message will setup a thread to wait - * until the message is complete, and then submits it for handling - * Receiver -- grabs completed Dripline messages and handles it - * Async endpoint listening -- same as abovefor each asynchronous endpoint - * Async endpoint message-wait -- same as above for each asynchronous endpoint - * Async endpoint receiver -- same as above for each asynchronous endpoint - * Heatbeater -- sends regular heartbeat messages - * Scheduler -- executes scheduled events + Message delivery is handled by rmqcpp's internal thread pool via callbacks; dripline-cpp + manages only the following threads: + * Heartbeater -- sends regular heartbeat messages + * Scheduler -- executes scheduled events In addition to receiving messages from the broker, a user or client code can give messages directly to the service using `process_message(message)`. @@ -78,7 +71,7 @@ namespace dripline class DRIPLINE_API service : public core, public endpoint, - public concurrent_receiver, + public message_dispatcher, public heartbeater, public scheduler<> { @@ -193,7 +186,7 @@ namespace dripline mv_referrable( std::string, broadcast_key ); protected: - /// Implementation of submit_message (from concurrent_receiver) + /// Implementation of submit_message (from message_dispatcher) virtual void submit_message( message_ptr_t a_message ); virtual reply_ptr_t on_request_message( const request_ptr_t a_request ); diff --git a/testing/CMakeLists.txt b/testing/CMakeLists.txt index 7b2a4151..25097501 100644 --- a/testing/CMakeLists.txt +++ b/testing/CMakeLists.txt @@ -26,7 +26,6 @@ set( testing_SOURCES run_dl_tests.cc test_agent.cc test_amqp.cc - test_concurrent_receiver.cc test_configs.cc test_core.cc test_dripline_constants.cc @@ -35,11 +34,7 @@ set( testing_SOURCES test_endpoint_handlers.cc test_hub.cc test_lockout.cc - test_messages.cc - test_reply_cache.cc - test_endpoint_handlers.cc - test_hub.cc - test_lockout.cc + test_message_dispatcher.cc test_messages.cc test_reply_cache.cc test_return_codes.cc diff --git a/testing/test_message_dispatcher.cc b/testing/test_message_dispatcher.cc new file mode 100644 index 00000000..306c4753 --- /dev/null +++ b/testing/test_message_dispatcher.cc @@ -0,0 +1,175 @@ +/* + * test_message_dispatcher.cc + * + * Created on: Nov 19, 2025 + * Author: N.S. Oblath + * + * Renamed from test_concurrent_receiver.cc (Jun 2026) following the + * concurrent_receiver → message_dispatcher rename (Phase 8 of rmqcpp migration). + */ + +#include "amqp.hh" +#include "dripline_exceptions.hh" +#include "message.hh" +#include "message_dispatcher.hh" + +#include "param_node.hh" + +#include "rmqp_messageguard.h" +#include "rmqt_envelope.h" + +#include "catch2/catch_test_macros.hpp" + +#include + +namespace dripline +{ + class message_dispatcher_tester : public message_dispatcher + { + public: + using message_dispatcher::message_dispatcher; + + int f_submit_count = 0; + + void submit_message( message_ptr_t ) + { ++f_submit_count; } + }; +} + +// Minimal rmqp::MessageGuard implementation for constructing amqp_envelope_ptr in tests. +// TransferrableMessageGuard = bsl::shared_ptr, so any concrete +// subclass wrapped in a bsl::shared_ptr is a valid amqp_envelope_ptr. +struct fake_message_guard : public BloombergLP::rmqp::MessageGuard +{ + BloombergLP::rmqt::Message d_message; + BloombergLP::rmqt::Envelope d_envelope; + + fake_message_guard( const BloombergLP::rmqt::Message& a_msg, + const std::string& a_routing_key ) + : d_message( a_msg ) + , d_envelope( 0, 0, + bsl::string(""), + bsl::string(""), + bsl::string( a_routing_key.c_str() ), + false ) + {} + + const BloombergLP::rmqt::Message& message() const override { return d_message; } + const BloombergLP::rmqt::Envelope& envelope() const override { return d_envelope; } + void ack() override {} + void nack( bool ) override {} + BloombergLP::rmqp::Consumer* consumer() const override { return nullptr; } + BloombergLP::rmqp::TransferrableMessageGuard transferOwnership() override + { + return BloombergLP::rmqp::TransferrableMessageGuard(); + } +}; + +// Wraps an already-serialized chunk (amqp_message_ptr) and routing key into an +// amqp_envelope_ptr suitable for passing to receiver::handle_message_chunk(). +static dripline::amqp_envelope_ptr make_test_envelope( const dripline::amqp_message_ptr& a_chunk, + const std::string& a_routing_key ) +{ + return bsl::make_shared< fake_message_guard >( *a_chunk, a_routing_key ); +} + + +TEST_CASE( "md_process_message", "[message_dispatcher]" ) +{ + dripline::message_dispatcher_tester t_dispatcher; + + dripline::request_ptr_t t_request_ptr = dripline::msg_request::create( scarab::param_ptr_t( new scarab::param() ), dripline::op_t::get, "dlcpp_service", "", "" ); + + // process_message() now calls submit_message() directly and synchronously; no queue or execute() thread + t_dispatcher.process_message( t_request_ptr ); + t_dispatcher.process_message( t_request_ptr ); + t_dispatcher.process_message( t_request_ptr ); + + REQUIRE( t_dispatcher.f_submit_count == 3 ); + +} + +TEST_CASE( "md_multi_chunk_assembly", "[message_dispatcher]" ) +{ + // Payload serializes to ~54 chars of JSON; max_size=20 forces at least 3 chunks. + auto t_payload = scarab::param_ptr_t( new scarab::param_node() ); + t_payload->as_node().add( "data", "abcdefghijklmnopqrstuvwxyz0123456789" ); + + dripline::request_ptr_t t_req_ptr = dripline::msg_request::create( + std::move( t_payload ), + dripline::op_t::get, + "multi.chunk.rk" ); + + const std::string t_routing_key = t_req_ptr->routing_key(); + dripline::amqp_split_message_ptrs t_chunks = t_req_ptr->create_amqp_messages( 20 ); + + REQUIRE( t_chunks.size() > 1 ); + + SECTION( "in-order delivery" ) + { + dripline::message_dispatcher_tester t_dispatcher; + for( auto& t_chunk : t_chunks ) + { + t_dispatcher.handle_message_chunk( make_test_envelope( t_chunk, t_routing_key ) ); + } + // All chunks delivered: process_message_pack called exactly once + REQUIRE( t_dispatcher.f_submit_count == 1 ); + // Map should be empty after successful assembly + REQUIRE( t_dispatcher.incoming_messages().empty() ); + } + + SECTION( "reverse-order delivery" ) + { + dripline::message_dispatcher_tester t_dispatcher; + for( auto it = t_chunks.rbegin(); it != t_chunks.rend(); ++it ) + { + t_dispatcher.handle_message_chunk( make_test_envelope( *it, t_routing_key ) ); + } + REQUIRE( t_dispatcher.f_submit_count == 1 ); + REQUIRE( t_dispatcher.incoming_messages().empty() ); + } +} + +TEST_CASE( "md_stale_eviction", "[message_dispatcher]" ) +{ + dripline::message_dispatcher_tester t_dispatcher; + // Short eviction threshold so the test doesn't need a long sleep. + t_dispatcher.set_single_message_wait_ms( 50 ); + + // Build a 2-chunk message A; deliver only the first chunk so it stays incomplete. + auto t_payload_a = scarab::param_ptr_t( new scarab::param_node() ); + t_payload_a->as_node().add( "data", "abcdefghijklmnopqrstuvwxyz0123456789" ); + + dripline::request_ptr_t t_req_a = dripline::msg_request::create( + std::move( t_payload_a ), + dripline::op_t::get, + "stale.rk" ); + + dripline::amqp_split_message_ptrs t_chunks_a = t_req_a->create_amqp_messages( 20 ); + REQUIRE( t_chunks_a.size() > 1 ); + + // Deliver only chunk 0 — incomplete entry is inserted into the map. + t_dispatcher.handle_message_chunk( make_test_envelope( t_chunks_a[0], "stale.rk" ) ); + REQUIRE( t_dispatcher.f_submit_count == 0 ); + REQUIRE( t_dispatcher.incoming_messages().size() == 1 ); + + // Wait longer than the eviction threshold so the entry becomes stale. + std::this_thread::sleep_for( std::chrono::milliseconds( 100 ) ); + + // Deliver a fresh 1-chunk message B. The lazy sweep in handle_message_chunk + // runs at entry and evicts message A's stale entry before processing B. + auto t_payload_b = scarab::param_ptr_t( new scarab::param_node() ); + dripline::request_ptr_t t_req_b = dripline::msg_request::create( + std::move( t_payload_b ), + dripline::op_t::get, + "fresh.rk" ); + + dripline::amqp_split_message_ptrs t_chunks_b = t_req_b->create_amqp_messages(); + REQUIRE( t_chunks_b.size() == 1 ); + + t_dispatcher.handle_message_chunk( make_test_envelope( t_chunks_b[0], "fresh.rk" ) ); + + // Message A evicted (never processed); message B processed successfully. + REQUIRE( t_dispatcher.f_submit_count == 1 ); + REQUIRE( t_dispatcher.incoming_messages().empty() ); +} From 20248a100768afac8edcec0b88b5d8318cb75205 Mon Sep 17 00:00:00 2001 From: Noah Oblath Date: Wed, 3 Jun 2026 16:13:22 -0700 Subject: [PATCH 30/45] Updated documentation for recent code changes and fixing a few bugs/holes --- documentation/source/building.rst | 6 +++ documentation/source/configuration.rst | 4 +- documentation/source/dl_agent.rst | 4 +- documentation/source/dl_mon.rst | 4 +- documentation/source/getting_started.rst | 6 +-- documentation/source/library.rst | 68 +++++++++++++++--------- documentation/source/offline.rst | 7 ++- 7 files changed, 60 insertions(+), 39 deletions(-) diff --git a/documentation/source/building.rst b/documentation/source/building.rst index 28b03a06..fea826f6 100644 --- a/documentation/source/building.rst +++ b/documentation/source/building.rst @@ -33,6 +33,12 @@ Dripline build options include: ``Dripline_MAX_PAYLOAD_SIZE`` (INT) Maximum payload size in bytes + ``Dripline_OFFLINE`` (BOOL) + if ON, builds the library in offline mode (no broker connection required). + In this mode, sending a message throws the message object rather than transmitting it, + which is useful for unit testing. + See :ref:`Offline & No-Connection ` for more details. + ``Dripline_PYTHON_THROW_REPLY_KEYWORD`` keyword used by the python wrapping that indicates that the thrown object is a reply message to be sent diff --git a/documentation/source/configuration.rst b/documentation/source/configuration.rst index fd1b167f..f94becc6 100644 --- a/documentation/source/configuration.rst +++ b/documentation/source/configuration.rst @@ -34,7 +34,7 @@ Here is the possible set of parameters for the ``dripline_mesh`` block, with a s loop_timeout_ms: (unsigned int) time used in loops for checking for application shutdown in milliseconds message_wait_ms: (unsigned int) timeout for waiting for a message in milliseconds heartbeat_routing_key: (string) routing key for sending and receiving heartbeat messages - hearteat_interval_s: (unsigned int) interval for sending heartbeats in seconds + heartbeat_interval_s: (unsigned int) interval for sending heartbeats in seconds return_codes: - name: (string) return-code name (must be unique) value: (unsigned int) return-code value (must be unique) @@ -57,7 +57,7 @@ The defaults for all of these parameters are given in the class ``dripline_confi loop_timeout_ms: 1000 message_wait_ms: 1000 heartbeat_routing_key: heartbeat - hearteat_interval_s: 60 + heartbeat_interval_s: 60 .. _default-mesh-yaml: diff --git a/documentation/source/dl_agent.rst b/documentation/source/dl_agent.rst index c56214ce..8006e9ec 100644 --- a/documentation/source/dl_agent.rst +++ b/documentation/source/dl_agent.rst @@ -37,8 +37,8 @@ Options --requests-exchange TEXT Set the name of the requests exchange --alerts-exchange TEXT Set the name of the alerts exchange --max-payload UINT Set the maximum payload size (in bytes) - --loop-timeout-msdripline.loop-timeout-ms UINT - --message-wait-msdripline.message-wait-ms UINT + --loop-timeout-ms UINT Set the loop timeout in ms + --message-wait-ms UINT Set the message wait time in ms --heartbeat-routing-key TEXT Set the first token of heartbeat routing keys: [token].[origin] --heartbeat-interval-s UINT Set the interval between heartbeats in s -s,--specifier TEXT Set the specifier diff --git a/documentation/source/dl_mon.rst b/documentation/source/dl_mon.rst index 59ad6739..aaed4f69 100644 --- a/documentation/source/dl_mon.rst +++ b/documentation/source/dl_mon.rst @@ -32,8 +32,8 @@ Options --requests-exchange TEXT Set the name of the requests exchange --alerts-exchange TEXT Set the name of the alerts exchange --max-payload UINT Set the maximum payload size (in bytes) - --loop-timeout-msdripline.loop-timeout-ms UINT - --message-wait-msdripline.message-wait-ms UINT + --loop-timeout-ms UINT Set the loop timeout in ms + --message-wait-ms UINT Set the message wait time in ms --heartbeat-routing-key TEXT Set the first token of heartbeat routing keys: [token].[origin] --heartbeat-interval-s UINT Set the interval between heartbeats in s diff --git a/documentation/source/getting_started.rst b/documentation/source/getting_started.rst index 48e167cc..c62f11c9 100644 --- a/documentation/source/getting_started.rst +++ b/documentation/source/getting_started.rst @@ -15,8 +15,9 @@ Requirements * CMake 3.12 * C++17 -* Boost 1.46 -* rabbitmqc +* Boost 1.46 (including Boost.Asio) +* `rmqcpp `_ (Bloomberg AMQP client library) +* BDE libraries (Bloomberg Development Environment; required by rmqcpp) * libyaml-cpp * rapidjson * quill (logging library) @@ -25,7 +26,6 @@ Requirements Included Packages ================= -* `SimpleAmqpClient `_ (unchanged fork of `upstream SimpleAmqpClient `_) * `scarab `_ diff --git a/documentation/source/library.rst b/documentation/source/library.rst index f9d4708b..4265de7f 100644 --- a/documentation/source/library.rst +++ b/documentation/source/library.rst @@ -21,8 +21,8 @@ There are also a number of classes that implement various features of the above * :ref:`Core`: interface for the RabbitMQ client library; interface includes interacting with the broker and for sending and receiving messages * :ref:`Heartbeater`: implements the heartbeat behavior -* :ref:`Listeners`: listen for AMQP messages (dripline message chunks or whole messages) -* :ref:`Receivers`: collect dripline message chunks and assembles them into complete messages +* :ref:`Message Dispatcher`: manages the rmqcpp Consumer lifecycle and dispatches assembled Dripline messages +* :ref:`Receiver`: collects Dripline message chunks and assembles them into complete messages * :ref:`Scheduler`: executes scheduled events * :ref:`Specifier`: parses specifier strings * :ref:`Version Store`: stores version information for a particular application context @@ -61,7 +61,7 @@ It's range of capabilities are largely defined by the classes it inherits from: * ``core`` * ``endpoint`` -* ``listener_receiver`` +* ``message_dispatcher`` * ``heartbeater`` * ``scheduler`` @@ -74,9 +74,15 @@ The interface for running a service consists of three functions: Or you can use ``run()`` to perform the start-->listen-->stop sequence. A service can have both synchronous and asynchronous child endpoints. With the former, requests are -handled synchronously with the recieving of messages and with processing messages bound for itself. +handled synchronously with the receiving of messages and with processing messages bound for itself. With the latter, requests are passed to the appropriate endpoint, which handles them in its own thread. +Message delivery is handled by rmqcpp's internal thread pool via callbacks. Dripline-cpp manages +only the following threads: + +* **Heartbeat thread** — sends regular heartbeat messages (optional) +* **Scheduler thread** — executes scheduled events (optional) + .. _messages: Messages @@ -132,7 +138,7 @@ Core ---- The ``core`` class provides an interface for the basic AMQP functionality. It wraps the -more general RabbitMQ API in a dripline-specific interface. +rmqcpp RabbitMQ API in a dripline-specific interface. The class includes a number of static utility functions for interacting with the broker. @@ -150,42 +156,52 @@ The heartbeat is an alert sent to a pre-determined routing key, which is given a ``execute()`` function. The interval for sending the heartbeats is ``f_heartbeat_interval_s``, which is in seconds. The default interval is 60 s. -.. _listeners: +.. _message-dispatcher: -Listeners ---------- +Message Dispatcher +------------------ + +The ``message_dispatcher`` class manages the lifecycle of an rmqcpp Consumer +(via ``start_listening()`` / ``stop_listening()``) and dispatches each assembled +Dripline message to ``submit_message()``. -A listener is a class capable of listening on an AMQP channel for AMQP messages, -which represent either a dripline message chunk or an entire dripline message. -The ``listener`` class provides the basic framework for doing that. +Message delivery is callback-based: rmqcpp's internal thread pool invokes the delivery +callback, which passes each AMQP message chunk to ``receiver::handle_message_chunk()``. +Once all chunks of a Dripline message have arrived, the assembled message is dispatched +synchronously to ``submit_message()`` in the rmqcpp callback thread. -The typical use case involves at least two threads: -1. A listener gets messages from the AMQP channel (using ``listen_on_queue()``, - e.g. ``service`` or ``endpoint_listener_receiver``) and - calls ``receiver::handle_message_chunk()`` -2. A receiver has a timing thread waiting for multiple message chunks (if relevant); - when the message is complete, ``receiver::process_message()`` is called. +A class deriving from ``message_dispatcher`` must implement ``submit_message()`` to define +what happens with each received message. The two concrete implementations in dripline-cpp are: -``listener_receiver`` is a convenience class that brings together ``listener`` and ``concurrent_receiver``. +* ``service`` — dispatches messages to itself or to its child endpoints +* ``endpoint_listener_receiver`` — a decorator class that wraps a plain ``endpoint`` and adds + ``message_dispatcher`` capabilities, allowing it to act as an asynchronous child endpoint of a ``service`` -``endpoint_listener_receiver`` is a decorator class for a "plain" endpoint: -it adds ``listener_receiver`` capabilities, allowing it to act as an asynchronous endpoint of a ``service``. +.. note:: + Prior to the rmqcpp migration, this class was named ``concurrent_receiver``. The name was changed + because the class no longer manages any concurrency itself — rmqcpp's thread pool handles delivery. .. _receivers: -Receivers ---------- +Receiver +-------- A receiver is able to collect Dripline message chunks and reassemble them into a complete dripline message. Dripline messages can be broken up into multiple chunks, each of which is transported as an AMQP message. -A receiver is responsible for handling message chunks, storing incomplete dripline messages, and eventually +A receiver is responsible for handling message chunks, storing incomplete dripline messages, and eventually processing complete dripline messages. -The ``receiver`` class contains an interface specifically for users waiting to receive reply messages: `wait_for_reply()`. +When a message chunk arrives via ``handle_message_chunk()``, it is stored in the incoming-message map. +Message chunks for a given message can be received in any order. Once all chunks for a message have +arrived, ``process_message_pack()`` is called inline (no separate thread is spawned). + +Stale incomplete messages (entries older than ``single_message_wait_ms`` ms) are lazily evicted at the +start of each ``handle_message_chunk()`` call. -The ``concurrent_receiver`` class allows client code to concurrently receive and process messages -(i.e. in separate threads). +The ``receiver`` class contains an interface specifically for users waiting to receive reply messages: +``wait_for_reply()``. This uses a ``std::future`` to wait for the reply, which is fulfilled by the +reply consumer's callback when the reply arrives. .. _scheduler: diff --git a/documentation/source/offline.rst b/documentation/source/offline.rst index 311c0cd2..12c6b3e3 100644 --- a/documentation/source/offline.rst +++ b/documentation/source/offline.rst @@ -8,14 +8,13 @@ Offline ------- This mode is a build-time option that is typically used for unit-testing purposes. +It is enabled by setting the ``Dripline_OFFLINE`` CMake flag (see :ref:`Build Options `). In this mode: * Sending a message results in the message object being thrown (in the C++ sense). This feature is useful for testing the various send-message options. -* Receiving messages is currently disabled, as is any other function that uses the SimpleAMQP library API. Such methods will respond with error conditions. Some message-receive functionality could be tested using the Offline mode in the future, e.g. by manually submitting a SimpleAMQP envelope object. - -Offline mode is selected by building the library with the ``Dripline_OFFLINE`` flag enabled in CMake. +* Receiving messages is currently disabled, as is any other function that uses the AMQP library (rmqcpp) API. Such methods will respond with error conditions. Some message-receive functionality could be tested using the Offline mode in the future, e.g. by manually submitting an AMQP envelope object. No-Connection ------------- @@ -24,6 +23,6 @@ Some dripline classes have flags in the constructors that allow them to be used Depending on how it's used, this could be done as either a build-time or a run-time option. -For an object in no-connection mode, methods that use the SimpleAMQP API will respond with error conditions. +For an object in no-connection mode, methods that use the AMQP library (rmqcpp) API will respond with error conditions. This feature is useful, for example, when a user wants to be able to handle message objects that are not received from a broker (e.g. they're created manually). From da8a395c65e21a66ad69c8e37f9c6417c2666715 Mon Sep 17 00:00:00 2001 From: Noah Oblath Date: Wed, 3 Jun 2026 16:19:14 -0700 Subject: [PATCH 31/45] Fixed integration test execution for GitHub Actions --- .github/workflows/publish.yaml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/publish.yaml b/.github/workflows/publish.yaml index cba99bf9..8e2db68e 100644 --- a/.github/workflows/publish.yaml +++ b/.github/workflows/publish.yaml @@ -111,7 +111,8 @@ jobs: - name: Integration Tests run: | cd testing/integration - ./do-testing.sh ${{ env.INT_TAG }} + ./run-dl-tests.sh ${{ env.INT_TAG }} + ./run-broker-restart-test.sh ${{ env.INT_TAG }} # For debugging # - name: Setup tmate session From e2ed1b8cee8e2c1b75754acb6be4127b021ab7dd Mon Sep 17 00:00:00 2001 From: Noah Oblath Date: Wed, 3 Jun 2026 16:27:25 -0700 Subject: [PATCH 32/45] Moved unit testing to a directory parallel to integration testing --- CMakeLists.txt | 2 +- DockerfileBase | 46 - testing/catch/catch.hpp | 17975 ---------------- testing/{ => unit}/CMakeLists.txt | 2 +- testing/{ => unit}/README.md | 0 testing/{ => unit}/run_dl_tests.cc | 0 testing/{ => unit}/test_agent.cc | 0 testing/{ => unit}/test_amqp.cc | 0 .../{ => unit}/test_concurrent_receiver.cc | 0 testing/{ => unit}/test_configs.cc | 0 testing/{ => unit}/test_core.cc | 0 testing/{ => unit}/test_dripline_constants.cc | 0 testing/{ => unit}/test_dripline_error.cc | 0 testing/{ => unit}/test_endpoint.cc | 0 testing/{ => unit}/test_endpoint_handlers.cc | 0 .../{ => unit}/test_examples_oscillator.cc | 0 testing/{ => unit}/test_hub.cc | 0 testing/{ => unit}/test_lockout.cc | 0 testing/{ => unit}/test_message_dispatcher.cc | 0 testing/{ => unit}/test_messages.cc | 0 testing/{ => unit}/test_reply_cache.cc | 0 testing/{ => unit}/test_return_codes.cc | 0 testing/{ => unit}/test_scheduler.cc | 0 testing/{ => unit}/test_service.cc | 0 testing/{ => unit}/test_specifier.cc | 0 testing/{ => unit}/test_throw_reply.cc | 0 testing/{ => unit}/test_uuid.cc | 0 testing/{ => unit}/test_version_store.cc | 0 testing/{ => unit}/use_catch.cc | 0 29 files changed, 2 insertions(+), 18023 deletions(-) delete mode 100644 DockerfileBase delete mode 100644 testing/catch/catch.hpp rename testing/{ => unit}/CMakeLists.txt (97%) rename testing/{ => unit}/README.md (100%) rename testing/{ => unit}/run_dl_tests.cc (100%) rename testing/{ => unit}/test_agent.cc (100%) rename testing/{ => unit}/test_amqp.cc (100%) rename testing/{ => unit}/test_concurrent_receiver.cc (100%) rename testing/{ => unit}/test_configs.cc (100%) rename testing/{ => unit}/test_core.cc (100%) rename testing/{ => unit}/test_dripline_constants.cc (100%) rename testing/{ => unit}/test_dripline_error.cc (100%) rename testing/{ => unit}/test_endpoint.cc (100%) rename testing/{ => unit}/test_endpoint_handlers.cc (100%) rename testing/{ => unit}/test_examples_oscillator.cc (100%) rename testing/{ => unit}/test_hub.cc (100%) rename testing/{ => unit}/test_lockout.cc (100%) rename testing/{ => unit}/test_message_dispatcher.cc (100%) rename testing/{ => unit}/test_messages.cc (100%) rename testing/{ => unit}/test_reply_cache.cc (100%) rename testing/{ => unit}/test_return_codes.cc (100%) rename testing/{ => unit}/test_scheduler.cc (100%) rename testing/{ => unit}/test_service.cc (100%) rename testing/{ => unit}/test_specifier.cc (100%) rename testing/{ => unit}/test_throw_reply.cc (100%) rename testing/{ => unit}/test_uuid.cc (100%) rename testing/{ => unit}/test_version_store.cc (100%) rename testing/{ => unit}/use_catch.cc (100%) diff --git a/CMakeLists.txt b/CMakeLists.txt index 61c3ad8b..21c26907 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -148,7 +148,7 @@ endif() ######### if( Dripline_ENABLE_TESTING ) - add_subdirectory( testing ) + add_subdirectory( testing/unit ) endif() ################## diff --git a/DockerfileBase b/DockerfileBase deleted file mode 100644 index 3d233fde..00000000 --- a/DockerfileBase +++ /dev/null @@ -1,46 +0,0 @@ -#FROM debian:9 -FROM amd64/python:3.7 - -# Most dependencies - -RUN apt-get update && \ - apt-get clean && \ - apt-get --fix-missing -y install \ - build-essential \ - cmake \ - gdb \ - git \ - libboost-chrono-dev \ - libboost-filesystem-dev \ - libboost-system-dev \ - librabbitmq-dev \ - wget && \ - rm -rf /var/lib/apt/lists/* - -RUN mkdir -p /usr/local/deps && \ - git clone https://github.com/jbeder/yaml-cpp && \ - cd yaml-cpp && \ - mkdir build && \ - cd build && \ - cmake -DCMAKE_BUILD_TYPE=Release \ - -DYAML_CPP_BUILD_CONTRIB:BOOL=FALSE \ - -DYAML_CPP_BUILD_TOOLS:BOOL=FALSE \ - -DYAML_CPP_BUILD_TESTS:BOOL=FALSE \ - -DYAML_BUILD_SHARED_LIBS=TRUE \ - .. && \ - make install - -RUN cd /usr/local/deps && \ - git clone https://github.com/Tencent/rapidjson && \ - cd rapidjson && \ - mkdir build && \ - cd build && \ - cmake -DCMAKE_BUILD_TYPE=Release \ - -DRAPIDJSON_BUILD_DOC=FALSE \ - -DRAPIDJSON_BUILD_EXAMPLES=FALSE \ - -DRAPIDJSON_BUILD_TESTS=FALSE \ - -DRAPIDJSON_BUILD_THIRDPARTY_GTEST=FALSE \ - .. && \ - make install - - diff --git a/testing/catch/catch.hpp b/testing/catch/catch.hpp deleted file mode 100644 index 47cd43a7..00000000 --- a/testing/catch/catch.hpp +++ /dev/null @@ -1,17975 +0,0 @@ -/* - * Catch v2.13.10 - * Generated: 2022-10-16 11:01:23.452308 - * ---------------------------------------------------------- - * This file has been merged from multiple headers. Please don't edit it directly - * Copyright (c) 2022 Two Blue Cubes Ltd. All rights reserved. - * - * Distributed under the Boost Software License, Version 1.0. (See accompanying - * file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) - */ -#ifndef TWOBLUECUBES_SINGLE_INCLUDE_CATCH_HPP_INCLUDED -#define TWOBLUECUBES_SINGLE_INCLUDE_CATCH_HPP_INCLUDED -// start catch.hpp - - -#define CATCH_VERSION_MAJOR 2 -#define CATCH_VERSION_MINOR 13 -#define CATCH_VERSION_PATCH 10 - -#ifdef __clang__ -# pragma clang system_header -#elif defined __GNUC__ -# pragma GCC system_header -#endif - -// start catch_suppress_warnings.h - -#ifdef __clang__ -# ifdef __ICC // icpc defines the __clang__ macro -# pragma warning(push) -# pragma warning(disable: 161 1682) -# else // __ICC -# pragma clang diagnostic push -# pragma clang diagnostic ignored "-Wpadded" -# pragma clang diagnostic ignored "-Wswitch-enum" -# pragma clang diagnostic ignored "-Wcovered-switch-default" -# endif -#elif defined __GNUC__ - // Because REQUIREs trigger GCC's -Wparentheses, and because still - // supported version of g++ have only buggy support for _Pragmas, - // Wparentheses have to be suppressed globally. -# pragma GCC diagnostic ignored "-Wparentheses" // See #674 for details - -# pragma GCC diagnostic push -# pragma GCC diagnostic ignored "-Wunused-variable" -# pragma GCC diagnostic ignored "-Wpadded" -#endif -// end catch_suppress_warnings.h -#if defined(CATCH_CONFIG_MAIN) || defined(CATCH_CONFIG_RUNNER) -# define CATCH_IMPL -# define CATCH_CONFIG_ALL_PARTS -#endif - -// In the impl file, we want to have access to all parts of the headers -// Can also be used to sanely support PCHs -#if defined(CATCH_CONFIG_ALL_PARTS) -# define CATCH_CONFIG_EXTERNAL_INTERFACES -# if defined(CATCH_CONFIG_DISABLE_MATCHERS) -# undef CATCH_CONFIG_DISABLE_MATCHERS -# endif -# if !defined(CATCH_CONFIG_ENABLE_CHRONO_STRINGMAKER) -# define CATCH_CONFIG_ENABLE_CHRONO_STRINGMAKER -# endif -#endif - -#if !defined(CATCH_CONFIG_IMPL_ONLY) -// start catch_platform.h - -// See e.g.: -// https://opensource.apple.com/source/CarbonHeaders/CarbonHeaders-18.1/TargetConditionals.h.auto.html -#ifdef __APPLE__ -# include -# if (defined(TARGET_OS_OSX) && TARGET_OS_OSX == 1) || \ - (defined(TARGET_OS_MAC) && TARGET_OS_MAC == 1) -# define CATCH_PLATFORM_MAC -# elif (defined(TARGET_OS_IPHONE) && TARGET_OS_IPHONE == 1) -# define CATCH_PLATFORM_IPHONE -# endif - -#elif defined(linux) || defined(__linux) || defined(__linux__) -# define CATCH_PLATFORM_LINUX - -#elif defined(WIN32) || defined(__WIN32__) || defined(_WIN32) || defined(_MSC_VER) || defined(__MINGW32__) -# define CATCH_PLATFORM_WINDOWS -#endif - -// end catch_platform.h - -#ifdef CATCH_IMPL -# ifndef CLARA_CONFIG_MAIN -# define CLARA_CONFIG_MAIN_NOT_DEFINED -# define CLARA_CONFIG_MAIN -# endif -#endif - -// start catch_user_interfaces.h - -namespace Catch { - unsigned int rngSeed(); -} - -// end catch_user_interfaces.h -// start catch_tag_alias_autoregistrar.h - -// start catch_common.h - -// start catch_compiler_capabilities.h - -// Detect a number of compiler features - by compiler -// The following features are defined: -// -// CATCH_CONFIG_COUNTER : is the __COUNTER__ macro supported? -// CATCH_CONFIG_WINDOWS_SEH : is Windows SEH supported? -// CATCH_CONFIG_POSIX_SIGNALS : are POSIX signals supported? -// CATCH_CONFIG_DISABLE_EXCEPTIONS : Are exceptions enabled? -// **************** -// Note to maintainers: if new toggles are added please document them -// in configuration.md, too -// **************** - -// In general each macro has a _NO_ form -// (e.g. CATCH_CONFIG_NO_POSIX_SIGNALS) which disables the feature. -// Many features, at point of detection, define an _INTERNAL_ macro, so they -// can be combined, en-mass, with the _NO_ forms later. - -#ifdef __cplusplus - -# if (__cplusplus >= 201402L) || (defined(_MSVC_LANG) && _MSVC_LANG >= 201402L) -# define CATCH_CPP14_OR_GREATER -# endif - -# if (__cplusplus >= 201703L) || (defined(_MSVC_LANG) && _MSVC_LANG >= 201703L) -# define CATCH_CPP17_OR_GREATER -# endif - -#endif - -// Only GCC compiler should be used in this block, so other compilers trying to -// mask themselves as GCC should be ignored. -#if defined(__GNUC__) && !defined(__clang__) && !defined(__ICC) && !defined(__CUDACC__) && !defined(__LCC__) -# define CATCH_INTERNAL_START_WARNINGS_SUPPRESSION _Pragma( "GCC diagnostic push" ) -# define CATCH_INTERNAL_STOP_WARNINGS_SUPPRESSION _Pragma( "GCC diagnostic pop" ) - -# define CATCH_INTERNAL_IGNORE_BUT_WARN(...) (void)__builtin_constant_p(__VA_ARGS__) - -#endif - -#if defined(__clang__) - -# define CATCH_INTERNAL_START_WARNINGS_SUPPRESSION _Pragma( "clang diagnostic push" ) -# define CATCH_INTERNAL_STOP_WARNINGS_SUPPRESSION _Pragma( "clang diagnostic pop" ) - -// As of this writing, IBM XL's implementation of __builtin_constant_p has a bug -// which results in calls to destructors being emitted for each temporary, -// without a matching initialization. In practice, this can result in something -// like `std::string::~string` being called on an uninitialized value. -// -// For example, this code will likely segfault under IBM XL: -// ``` -// REQUIRE(std::string("12") + "34" == "1234") -// ``` -// -// Therefore, `CATCH_INTERNAL_IGNORE_BUT_WARN` is not implemented. -# if !defined(__ibmxl__) && !defined(__CUDACC__) -# define CATCH_INTERNAL_IGNORE_BUT_WARN(...) (void)__builtin_constant_p(__VA_ARGS__) /* NOLINT(cppcoreguidelines-pro-type-vararg, hicpp-vararg) */ -# endif - -# define CATCH_INTERNAL_SUPPRESS_GLOBALS_WARNINGS \ - _Pragma( "clang diagnostic ignored \"-Wexit-time-destructors\"" ) \ - _Pragma( "clang diagnostic ignored \"-Wglobal-constructors\"") - -# define CATCH_INTERNAL_SUPPRESS_PARENTHESES_WARNINGS \ - _Pragma( "clang diagnostic ignored \"-Wparentheses\"" ) - -# define CATCH_INTERNAL_SUPPRESS_UNUSED_WARNINGS \ - _Pragma( "clang diagnostic ignored \"-Wunused-variable\"" ) - -# define CATCH_INTERNAL_SUPPRESS_ZERO_VARIADIC_WARNINGS \ - _Pragma( "clang diagnostic ignored \"-Wgnu-zero-variadic-macro-arguments\"" ) - -# define CATCH_INTERNAL_SUPPRESS_UNUSED_TEMPLATE_WARNINGS \ - _Pragma( "clang diagnostic ignored \"-Wunused-template\"" ) - -#endif // __clang__ - -//////////////////////////////////////////////////////////////////////////////// -// Assume that non-Windows platforms support posix signals by default -#if !defined(CATCH_PLATFORM_WINDOWS) - #define CATCH_INTERNAL_CONFIG_POSIX_SIGNALS -#endif - -//////////////////////////////////////////////////////////////////////////////// -// We know some environments not to support full POSIX signals -#if defined(__CYGWIN__) || defined(__QNX__) || defined(__EMSCRIPTEN__) || defined(__DJGPP__) - #define CATCH_INTERNAL_CONFIG_NO_POSIX_SIGNALS -#endif - -#ifdef __OS400__ -# define CATCH_INTERNAL_CONFIG_NO_POSIX_SIGNALS -# define CATCH_CONFIG_COLOUR_NONE -#endif - -//////////////////////////////////////////////////////////////////////////////// -// Android somehow still does not support std::to_string -#if defined(__ANDROID__) -# define CATCH_INTERNAL_CONFIG_NO_CPP11_TO_STRING -# define CATCH_INTERNAL_CONFIG_ANDROID_LOGWRITE -#endif - -//////////////////////////////////////////////////////////////////////////////// -// Not all Windows environments support SEH properly -#if defined(__MINGW32__) -# define CATCH_INTERNAL_CONFIG_NO_WINDOWS_SEH -#endif - -//////////////////////////////////////////////////////////////////////////////// -// PS4 -#if defined(__ORBIS__) -# define CATCH_INTERNAL_CONFIG_NO_NEW_CAPTURE -#endif - -//////////////////////////////////////////////////////////////////////////////// -// Cygwin -#ifdef __CYGWIN__ - -// Required for some versions of Cygwin to declare gettimeofday -// see: http://stackoverflow.com/questions/36901803/gettimeofday-not-declared-in-this-scope-cygwin -# define _BSD_SOURCE -// some versions of cygwin (most) do not support std::to_string. Use the libstd check. -// https://gcc.gnu.org/onlinedocs/gcc-4.8.2/libstdc++/api/a01053_source.html line 2812-2813 -# if !((__cplusplus >= 201103L) && defined(_GLIBCXX_USE_C99) \ - && !defined(_GLIBCXX_HAVE_BROKEN_VSWPRINTF)) - -# define CATCH_INTERNAL_CONFIG_NO_CPP11_TO_STRING - -# endif -#endif // __CYGWIN__ - -//////////////////////////////////////////////////////////////////////////////// -// Visual C++ -#if defined(_MSC_VER) - -// Universal Windows platform does not support SEH -// Or console colours (or console at all...) -# if defined(WINAPI_FAMILY) && (WINAPI_FAMILY == WINAPI_FAMILY_APP) -# define CATCH_CONFIG_COLOUR_NONE -# else -# define CATCH_INTERNAL_CONFIG_WINDOWS_SEH -# endif - -# if !defined(__clang__) // Handle Clang masquerading for msvc - -// MSVC traditional preprocessor needs some workaround for __VA_ARGS__ -// _MSVC_TRADITIONAL == 0 means new conformant preprocessor -// _MSVC_TRADITIONAL == 1 means old traditional non-conformant preprocessor -# if !defined(_MSVC_TRADITIONAL) || (defined(_MSVC_TRADITIONAL) && _MSVC_TRADITIONAL) -# define CATCH_INTERNAL_CONFIG_TRADITIONAL_MSVC_PREPROCESSOR -# endif // MSVC_TRADITIONAL - -// Only do this if we're not using clang on Windows, which uses `diagnostic push` & `diagnostic pop` -# define CATCH_INTERNAL_START_WARNINGS_SUPPRESSION __pragma( warning(push) ) -# define CATCH_INTERNAL_STOP_WARNINGS_SUPPRESSION __pragma( warning(pop) ) -# endif // __clang__ - -#endif // _MSC_VER - -#if defined(_REENTRANT) || defined(_MSC_VER) -// Enable async processing, as -pthread is specified or no additional linking is required -# define CATCH_INTERNAL_CONFIG_USE_ASYNC -#endif // _MSC_VER - -//////////////////////////////////////////////////////////////////////////////// -// Check if we are compiled with -fno-exceptions or equivalent -#if defined(__EXCEPTIONS) || defined(__cpp_exceptions) || defined(_CPPUNWIND) -# define CATCH_INTERNAL_CONFIG_EXCEPTIONS_ENABLED -#endif - -//////////////////////////////////////////////////////////////////////////////// -// DJGPP -#ifdef __DJGPP__ -# define CATCH_INTERNAL_CONFIG_NO_WCHAR -#endif // __DJGPP__ - -//////////////////////////////////////////////////////////////////////////////// -// Embarcadero C++Build -#if defined(__BORLANDC__) - #define CATCH_INTERNAL_CONFIG_POLYFILL_ISNAN -#endif - -//////////////////////////////////////////////////////////////////////////////// - -// Use of __COUNTER__ is suppressed during code analysis in -// CLion/AppCode 2017.2.x and former, because __COUNTER__ is not properly -// handled by it. -// Otherwise all supported compilers support COUNTER macro, -// but user still might want to turn it off -#if ( !defined(__JETBRAINS_IDE__) || __JETBRAINS_IDE__ >= 20170300L ) - #define CATCH_INTERNAL_CONFIG_COUNTER -#endif - -//////////////////////////////////////////////////////////////////////////////// - -// RTX is a special version of Windows that is real time. -// This means that it is detected as Windows, but does not provide -// the same set of capabilities as real Windows does. -#if defined(UNDER_RTSS) || defined(RTX64_BUILD) - #define CATCH_INTERNAL_CONFIG_NO_WINDOWS_SEH - #define CATCH_INTERNAL_CONFIG_NO_ASYNC - #define CATCH_CONFIG_COLOUR_NONE -#endif - -#if !defined(_GLIBCXX_USE_C99_MATH_TR1) -#define CATCH_INTERNAL_CONFIG_GLOBAL_NEXTAFTER -#endif - -// Various stdlib support checks that require __has_include -#if defined(__has_include) - // Check if string_view is available and usable - #if __has_include() && defined(CATCH_CPP17_OR_GREATER) - # define CATCH_INTERNAL_CONFIG_CPP17_STRING_VIEW - #endif - - // Check if optional is available and usable - # if __has_include() && defined(CATCH_CPP17_OR_GREATER) - # define CATCH_INTERNAL_CONFIG_CPP17_OPTIONAL - # endif // __has_include() && defined(CATCH_CPP17_OR_GREATER) - - // Check if byte is available and usable - # if __has_include() && defined(CATCH_CPP17_OR_GREATER) - # include - # if defined(__cpp_lib_byte) && (__cpp_lib_byte > 0) - # define CATCH_INTERNAL_CONFIG_CPP17_BYTE - # endif - # endif // __has_include() && defined(CATCH_CPP17_OR_GREATER) - - // Check if variant is available and usable - # if __has_include() && defined(CATCH_CPP17_OR_GREATER) - # if defined(__clang__) && (__clang_major__ < 8) - // work around clang bug with libstdc++ https://bugs.llvm.org/show_bug.cgi?id=31852 - // fix should be in clang 8, workaround in libstdc++ 8.2 - # include - # if defined(__GLIBCXX__) && defined(_GLIBCXX_RELEASE) && (_GLIBCXX_RELEASE < 9) - # define CATCH_CONFIG_NO_CPP17_VARIANT - # else - # define CATCH_INTERNAL_CONFIG_CPP17_VARIANT - # endif // defined(__GLIBCXX__) && defined(_GLIBCXX_RELEASE) && (_GLIBCXX_RELEASE < 9) - # else - # define CATCH_INTERNAL_CONFIG_CPP17_VARIANT - # endif // defined(__clang__) && (__clang_major__ < 8) - # endif // __has_include() && defined(CATCH_CPP17_OR_GREATER) -#endif // defined(__has_include) - -#if defined(CATCH_INTERNAL_CONFIG_COUNTER) && !defined(CATCH_CONFIG_NO_COUNTER) && !defined(CATCH_CONFIG_COUNTER) -# define CATCH_CONFIG_COUNTER -#endif -#if defined(CATCH_INTERNAL_CONFIG_WINDOWS_SEH) && !defined(CATCH_CONFIG_NO_WINDOWS_SEH) && !defined(CATCH_CONFIG_WINDOWS_SEH) && !defined(CATCH_INTERNAL_CONFIG_NO_WINDOWS_SEH) -# define CATCH_CONFIG_WINDOWS_SEH -#endif -// This is set by default, because we assume that unix compilers are posix-signal-compatible by default. -#if defined(CATCH_INTERNAL_CONFIG_POSIX_SIGNALS) && !defined(CATCH_INTERNAL_CONFIG_NO_POSIX_SIGNALS) && !defined(CATCH_CONFIG_NO_POSIX_SIGNALS) && !defined(CATCH_CONFIG_POSIX_SIGNALS) -# define CATCH_CONFIG_POSIX_SIGNALS -#endif -// This is set by default, because we assume that compilers with no wchar_t support are just rare exceptions. -#if !defined(CATCH_INTERNAL_CONFIG_NO_WCHAR) && !defined(CATCH_CONFIG_NO_WCHAR) && !defined(CATCH_CONFIG_WCHAR) -# define CATCH_CONFIG_WCHAR -#endif - -#if !defined(CATCH_INTERNAL_CONFIG_NO_CPP11_TO_STRING) && !defined(CATCH_CONFIG_NO_CPP11_TO_STRING) && !defined(CATCH_CONFIG_CPP11_TO_STRING) -# define CATCH_CONFIG_CPP11_TO_STRING -#endif - -#if defined(CATCH_INTERNAL_CONFIG_CPP17_OPTIONAL) && !defined(CATCH_CONFIG_NO_CPP17_OPTIONAL) && !defined(CATCH_CONFIG_CPP17_OPTIONAL) -# define CATCH_CONFIG_CPP17_OPTIONAL -#endif - -#if defined(CATCH_INTERNAL_CONFIG_CPP17_STRING_VIEW) && !defined(CATCH_CONFIG_NO_CPP17_STRING_VIEW) && !defined(CATCH_CONFIG_CPP17_STRING_VIEW) -# define CATCH_CONFIG_CPP17_STRING_VIEW -#endif - -#if defined(CATCH_INTERNAL_CONFIG_CPP17_VARIANT) && !defined(CATCH_CONFIG_NO_CPP17_VARIANT) && !defined(CATCH_CONFIG_CPP17_VARIANT) -# define CATCH_CONFIG_CPP17_VARIANT -#endif - -#if defined(CATCH_INTERNAL_CONFIG_CPP17_BYTE) && !defined(CATCH_CONFIG_NO_CPP17_BYTE) && !defined(CATCH_CONFIG_CPP17_BYTE) -# define CATCH_CONFIG_CPP17_BYTE -#endif - -#if defined(CATCH_CONFIG_EXPERIMENTAL_REDIRECT) -# define CATCH_INTERNAL_CONFIG_NEW_CAPTURE -#endif - -#if defined(CATCH_INTERNAL_CONFIG_NEW_CAPTURE) && !defined(CATCH_INTERNAL_CONFIG_NO_NEW_CAPTURE) && !defined(CATCH_CONFIG_NO_NEW_CAPTURE) && !defined(CATCH_CONFIG_NEW_CAPTURE) -# define CATCH_CONFIG_NEW_CAPTURE -#endif - -#if !defined(CATCH_INTERNAL_CONFIG_EXCEPTIONS_ENABLED) && !defined(CATCH_CONFIG_DISABLE_EXCEPTIONS) -# define CATCH_CONFIG_DISABLE_EXCEPTIONS -#endif - -#if defined(CATCH_INTERNAL_CONFIG_POLYFILL_ISNAN) && !defined(CATCH_CONFIG_NO_POLYFILL_ISNAN) && !defined(CATCH_CONFIG_POLYFILL_ISNAN) -# define CATCH_CONFIG_POLYFILL_ISNAN -#endif - -#if defined(CATCH_INTERNAL_CONFIG_USE_ASYNC) && !defined(CATCH_INTERNAL_CONFIG_NO_ASYNC) && !defined(CATCH_CONFIG_NO_USE_ASYNC) && !defined(CATCH_CONFIG_USE_ASYNC) -# define CATCH_CONFIG_USE_ASYNC -#endif - -#if defined(CATCH_INTERNAL_CONFIG_ANDROID_LOGWRITE) && !defined(CATCH_CONFIG_NO_ANDROID_LOGWRITE) && !defined(CATCH_CONFIG_ANDROID_LOGWRITE) -# define CATCH_CONFIG_ANDROID_LOGWRITE -#endif - -#if defined(CATCH_INTERNAL_CONFIG_GLOBAL_NEXTAFTER) && !defined(CATCH_CONFIG_NO_GLOBAL_NEXTAFTER) && !defined(CATCH_CONFIG_GLOBAL_NEXTAFTER) -# define CATCH_CONFIG_GLOBAL_NEXTAFTER -#endif - -// Even if we do not think the compiler has that warning, we still have -// to provide a macro that can be used by the code. -#if !defined(CATCH_INTERNAL_START_WARNINGS_SUPPRESSION) -# define CATCH_INTERNAL_START_WARNINGS_SUPPRESSION -#endif -#if !defined(CATCH_INTERNAL_STOP_WARNINGS_SUPPRESSION) -# define CATCH_INTERNAL_STOP_WARNINGS_SUPPRESSION -#endif -#if !defined(CATCH_INTERNAL_SUPPRESS_PARENTHESES_WARNINGS) -# define CATCH_INTERNAL_SUPPRESS_PARENTHESES_WARNINGS -#endif -#if !defined(CATCH_INTERNAL_SUPPRESS_GLOBALS_WARNINGS) -# define CATCH_INTERNAL_SUPPRESS_GLOBALS_WARNINGS -#endif -#if !defined(CATCH_INTERNAL_SUPPRESS_UNUSED_WARNINGS) -# define CATCH_INTERNAL_SUPPRESS_UNUSED_WARNINGS -#endif -#if !defined(CATCH_INTERNAL_SUPPRESS_ZERO_VARIADIC_WARNINGS) -# define CATCH_INTERNAL_SUPPRESS_ZERO_VARIADIC_WARNINGS -#endif - -// The goal of this macro is to avoid evaluation of the arguments, but -// still have the compiler warn on problems inside... -#if !defined(CATCH_INTERNAL_IGNORE_BUT_WARN) -# define CATCH_INTERNAL_IGNORE_BUT_WARN(...) -#endif - -#if defined(__APPLE__) && defined(__apple_build_version__) && (__clang_major__ < 10) -# undef CATCH_INTERNAL_SUPPRESS_UNUSED_TEMPLATE_WARNINGS -#elif defined(__clang__) && (__clang_major__ < 5) -# undef CATCH_INTERNAL_SUPPRESS_UNUSED_TEMPLATE_WARNINGS -#endif - -#if !defined(CATCH_INTERNAL_SUPPRESS_UNUSED_TEMPLATE_WARNINGS) -# define CATCH_INTERNAL_SUPPRESS_UNUSED_TEMPLATE_WARNINGS -#endif - -#if defined(CATCH_CONFIG_DISABLE_EXCEPTIONS) -#define CATCH_TRY if ((true)) -#define CATCH_CATCH_ALL if ((false)) -#define CATCH_CATCH_ANON(type) if ((false)) -#else -#define CATCH_TRY try -#define CATCH_CATCH_ALL catch (...) -#define CATCH_CATCH_ANON(type) catch (type) -#endif - -#if defined(CATCH_INTERNAL_CONFIG_TRADITIONAL_MSVC_PREPROCESSOR) && !defined(CATCH_CONFIG_NO_TRADITIONAL_MSVC_PREPROCESSOR) && !defined(CATCH_CONFIG_TRADITIONAL_MSVC_PREPROCESSOR) -#define CATCH_CONFIG_TRADITIONAL_MSVC_PREPROCESSOR -#endif - -// end catch_compiler_capabilities.h -#define INTERNAL_CATCH_UNIQUE_NAME_LINE2( name, line ) name##line -#define INTERNAL_CATCH_UNIQUE_NAME_LINE( name, line ) INTERNAL_CATCH_UNIQUE_NAME_LINE2( name, line ) -#ifdef CATCH_CONFIG_COUNTER -# define INTERNAL_CATCH_UNIQUE_NAME( name ) INTERNAL_CATCH_UNIQUE_NAME_LINE( name, __COUNTER__ ) -#else -# define INTERNAL_CATCH_UNIQUE_NAME( name ) INTERNAL_CATCH_UNIQUE_NAME_LINE( name, __LINE__ ) -#endif - -#include -#include -#include - -// We need a dummy global operator<< so we can bring it into Catch namespace later -struct Catch_global_namespace_dummy {}; -std::ostream& operator<<(std::ostream&, Catch_global_namespace_dummy); - -namespace Catch { - - struct CaseSensitive { enum Choice { - Yes, - No - }; }; - - class NonCopyable { - NonCopyable( NonCopyable const& ) = delete; - NonCopyable( NonCopyable && ) = delete; - NonCopyable& operator = ( NonCopyable const& ) = delete; - NonCopyable& operator = ( NonCopyable && ) = delete; - - protected: - NonCopyable(); - virtual ~NonCopyable(); - }; - - struct SourceLineInfo { - - SourceLineInfo() = delete; - SourceLineInfo( char const* _file, std::size_t _line ) noexcept - : file( _file ), - line( _line ) - {} - - SourceLineInfo( SourceLineInfo const& other ) = default; - SourceLineInfo& operator = ( SourceLineInfo const& ) = default; - SourceLineInfo( SourceLineInfo&& ) noexcept = default; - SourceLineInfo& operator = ( SourceLineInfo&& ) noexcept = default; - - bool empty() const noexcept { return file[0] == '\0'; } - bool operator == ( SourceLineInfo const& other ) const noexcept; - bool operator < ( SourceLineInfo const& other ) const noexcept; - - char const* file; - std::size_t line; - }; - - std::ostream& operator << ( std::ostream& os, SourceLineInfo const& info ); - - // Bring in operator<< from global namespace into Catch namespace - // This is necessary because the overload of operator<< above makes - // lookup stop at namespace Catch - using ::operator<<; - - // Use this in variadic streaming macros to allow - // >> +StreamEndStop - // as well as - // >> stuff +StreamEndStop - struct StreamEndStop { - std::string operator+() const; - }; - template - T const& operator + ( T const& value, StreamEndStop ) { - return value; - } -} - -#define CATCH_INTERNAL_LINEINFO \ - ::Catch::SourceLineInfo( __FILE__, static_cast( __LINE__ ) ) - -// end catch_common.h -namespace Catch { - - struct RegistrarForTagAliases { - RegistrarForTagAliases( char const* alias, char const* tag, SourceLineInfo const& lineInfo ); - }; - -} // end namespace Catch - -#define CATCH_REGISTER_TAG_ALIAS( alias, spec ) \ - CATCH_INTERNAL_START_WARNINGS_SUPPRESSION \ - CATCH_INTERNAL_SUPPRESS_GLOBALS_WARNINGS \ - namespace{ Catch::RegistrarForTagAliases INTERNAL_CATCH_UNIQUE_NAME( AutoRegisterTagAlias )( alias, spec, CATCH_INTERNAL_LINEINFO ); } \ - CATCH_INTERNAL_STOP_WARNINGS_SUPPRESSION - -// end catch_tag_alias_autoregistrar.h -// start catch_test_registry.h - -// start catch_interfaces_testcase.h - -#include - -namespace Catch { - - class TestSpec; - - struct ITestInvoker { - virtual void invoke () const = 0; - virtual ~ITestInvoker(); - }; - - class TestCase; - struct IConfig; - - struct ITestCaseRegistry { - virtual ~ITestCaseRegistry(); - virtual std::vector const& getAllTests() const = 0; - virtual std::vector const& getAllTestsSorted( IConfig const& config ) const = 0; - }; - - bool isThrowSafe( TestCase const& testCase, IConfig const& config ); - bool matchTest( TestCase const& testCase, TestSpec const& testSpec, IConfig const& config ); - std::vector filterTests( std::vector const& testCases, TestSpec const& testSpec, IConfig const& config ); - std::vector const& getAllTestCasesSorted( IConfig const& config ); - -} - -// end catch_interfaces_testcase.h -// start catch_stringref.h - -#include -#include -#include -#include - -namespace Catch { - - /// A non-owning string class (similar to the forthcoming std::string_view) - /// Note that, because a StringRef may be a substring of another string, - /// it may not be null terminated. - class StringRef { - public: - using size_type = std::size_t; - using const_iterator = const char*; - - private: - static constexpr char const* const s_empty = ""; - - char const* m_start = s_empty; - size_type m_size = 0; - - public: // construction - constexpr StringRef() noexcept = default; - - StringRef( char const* rawChars ) noexcept; - - constexpr StringRef( char const* rawChars, size_type size ) noexcept - : m_start( rawChars ), - m_size( size ) - {} - - StringRef( std::string const& stdString ) noexcept - : m_start( stdString.c_str() ), - m_size( stdString.size() ) - {} - - explicit operator std::string() const { - return std::string(m_start, m_size); - } - - public: // operators - auto operator == ( StringRef const& other ) const noexcept -> bool; - auto operator != (StringRef const& other) const noexcept -> bool { - return !(*this == other); - } - - auto operator[] ( size_type index ) const noexcept -> char { - assert(index < m_size); - return m_start[index]; - } - - public: // named queries - constexpr auto empty() const noexcept -> bool { - return m_size == 0; - } - constexpr auto size() const noexcept -> size_type { - return m_size; - } - - // Returns the current start pointer. If the StringRef is not - // null-terminated, throws std::domain_exception - auto c_str() const -> char const*; - - public: // substrings and searches - // Returns a substring of [start, start + length). - // If start + length > size(), then the substring is [start, size()). - // If start > size(), then the substring is empty. - auto substr( size_type start, size_type length ) const noexcept -> StringRef; - - // Returns the current start pointer. May not be null-terminated. - auto data() const noexcept -> char const*; - - constexpr auto isNullTerminated() const noexcept -> bool { - return m_start[m_size] == '\0'; - } - - public: // iterators - constexpr const_iterator begin() const { return m_start; } - constexpr const_iterator end() const { return m_start + m_size; } - }; - - auto operator += ( std::string& lhs, StringRef const& sr ) -> std::string&; - auto operator << ( std::ostream& os, StringRef const& sr ) -> std::ostream&; - - constexpr auto operator "" _sr( char const* rawChars, std::size_t size ) noexcept -> StringRef { - return StringRef( rawChars, size ); - } -} // namespace Catch - -constexpr auto operator "" _catch_sr( char const* rawChars, std::size_t size ) noexcept -> Catch::StringRef { - return Catch::StringRef( rawChars, size ); -} - -// end catch_stringref.h -// start catch_preprocessor.hpp - - -#define CATCH_RECURSION_LEVEL0(...) __VA_ARGS__ -#define CATCH_RECURSION_LEVEL1(...) CATCH_RECURSION_LEVEL0(CATCH_RECURSION_LEVEL0(CATCH_RECURSION_LEVEL0(__VA_ARGS__))) -#define CATCH_RECURSION_LEVEL2(...) CATCH_RECURSION_LEVEL1(CATCH_RECURSION_LEVEL1(CATCH_RECURSION_LEVEL1(__VA_ARGS__))) -#define CATCH_RECURSION_LEVEL3(...) CATCH_RECURSION_LEVEL2(CATCH_RECURSION_LEVEL2(CATCH_RECURSION_LEVEL2(__VA_ARGS__))) -#define CATCH_RECURSION_LEVEL4(...) CATCH_RECURSION_LEVEL3(CATCH_RECURSION_LEVEL3(CATCH_RECURSION_LEVEL3(__VA_ARGS__))) -#define CATCH_RECURSION_LEVEL5(...) CATCH_RECURSION_LEVEL4(CATCH_RECURSION_LEVEL4(CATCH_RECURSION_LEVEL4(__VA_ARGS__))) - -#ifdef CATCH_CONFIG_TRADITIONAL_MSVC_PREPROCESSOR -#define INTERNAL_CATCH_EXPAND_VARGS(...) __VA_ARGS__ -// MSVC needs more evaluations -#define CATCH_RECURSION_LEVEL6(...) CATCH_RECURSION_LEVEL5(CATCH_RECURSION_LEVEL5(CATCH_RECURSION_LEVEL5(__VA_ARGS__))) -#define CATCH_RECURSE(...) CATCH_RECURSION_LEVEL6(CATCH_RECURSION_LEVEL6(__VA_ARGS__)) -#else -#define CATCH_RECURSE(...) CATCH_RECURSION_LEVEL5(__VA_ARGS__) -#endif - -#define CATCH_REC_END(...) -#define CATCH_REC_OUT - -#define CATCH_EMPTY() -#define CATCH_DEFER(id) id CATCH_EMPTY() - -#define CATCH_REC_GET_END2() 0, CATCH_REC_END -#define CATCH_REC_GET_END1(...) CATCH_REC_GET_END2 -#define CATCH_REC_GET_END(...) CATCH_REC_GET_END1 -#define CATCH_REC_NEXT0(test, next, ...) next CATCH_REC_OUT -#define CATCH_REC_NEXT1(test, next) CATCH_DEFER ( CATCH_REC_NEXT0 ) ( test, next, 0) -#define CATCH_REC_NEXT(test, next) CATCH_REC_NEXT1(CATCH_REC_GET_END test, next) - -#define CATCH_REC_LIST0(f, x, peek, ...) , f(x) CATCH_DEFER ( CATCH_REC_NEXT(peek, CATCH_REC_LIST1) ) ( f, peek, __VA_ARGS__ ) -#define CATCH_REC_LIST1(f, x, peek, ...) , f(x) CATCH_DEFER ( CATCH_REC_NEXT(peek, CATCH_REC_LIST0) ) ( f, peek, __VA_ARGS__ ) -#define CATCH_REC_LIST2(f, x, peek, ...) f(x) CATCH_DEFER ( CATCH_REC_NEXT(peek, CATCH_REC_LIST1) ) ( f, peek, __VA_ARGS__ ) - -#define CATCH_REC_LIST0_UD(f, userdata, x, peek, ...) , f(userdata, x) CATCH_DEFER ( CATCH_REC_NEXT(peek, CATCH_REC_LIST1_UD) ) ( f, userdata, peek, __VA_ARGS__ ) -#define CATCH_REC_LIST1_UD(f, userdata, x, peek, ...) , f(userdata, x) CATCH_DEFER ( CATCH_REC_NEXT(peek, CATCH_REC_LIST0_UD) ) ( f, userdata, peek, __VA_ARGS__ ) -#define CATCH_REC_LIST2_UD(f, userdata, x, peek, ...) f(userdata, x) CATCH_DEFER ( CATCH_REC_NEXT(peek, CATCH_REC_LIST1_UD) ) ( f, userdata, peek, __VA_ARGS__ ) - -// Applies the function macro `f` to each of the remaining parameters, inserts commas between the results, -// and passes userdata as the first parameter to each invocation, -// e.g. CATCH_REC_LIST_UD(f, x, a, b, c) evaluates to f(x, a), f(x, b), f(x, c) -#define CATCH_REC_LIST_UD(f, userdata, ...) CATCH_RECURSE(CATCH_REC_LIST2_UD(f, userdata, __VA_ARGS__, ()()(), ()()(), ()()(), 0)) - -#define CATCH_REC_LIST(f, ...) CATCH_RECURSE(CATCH_REC_LIST2(f, __VA_ARGS__, ()()(), ()()(), ()()(), 0)) - -#define INTERNAL_CATCH_EXPAND1(param) INTERNAL_CATCH_EXPAND2(param) -#define INTERNAL_CATCH_EXPAND2(...) INTERNAL_CATCH_NO## __VA_ARGS__ -#define INTERNAL_CATCH_DEF(...) INTERNAL_CATCH_DEF __VA_ARGS__ -#define INTERNAL_CATCH_NOINTERNAL_CATCH_DEF -#define INTERNAL_CATCH_STRINGIZE(...) INTERNAL_CATCH_STRINGIZE2(__VA_ARGS__) -#ifndef CATCH_CONFIG_TRADITIONAL_MSVC_PREPROCESSOR -#define INTERNAL_CATCH_STRINGIZE2(...) #__VA_ARGS__ -#define INTERNAL_CATCH_STRINGIZE_WITHOUT_PARENS(param) INTERNAL_CATCH_STRINGIZE(INTERNAL_CATCH_REMOVE_PARENS(param)) -#else -// MSVC is adding extra space and needs another indirection to expand INTERNAL_CATCH_NOINTERNAL_CATCH_DEF -#define INTERNAL_CATCH_STRINGIZE2(...) INTERNAL_CATCH_STRINGIZE3(__VA_ARGS__) -#define INTERNAL_CATCH_STRINGIZE3(...) #__VA_ARGS__ -#define INTERNAL_CATCH_STRINGIZE_WITHOUT_PARENS(param) (INTERNAL_CATCH_STRINGIZE(INTERNAL_CATCH_REMOVE_PARENS(param)) + 1) -#endif - -#define INTERNAL_CATCH_MAKE_NAMESPACE2(...) ns_##__VA_ARGS__ -#define INTERNAL_CATCH_MAKE_NAMESPACE(name) INTERNAL_CATCH_MAKE_NAMESPACE2(name) - -#define INTERNAL_CATCH_REMOVE_PARENS(...) INTERNAL_CATCH_EXPAND1(INTERNAL_CATCH_DEF __VA_ARGS__) - -#ifndef CATCH_CONFIG_TRADITIONAL_MSVC_PREPROCESSOR -#define INTERNAL_CATCH_MAKE_TYPE_LIST2(...) decltype(get_wrapper()) -#define INTERNAL_CATCH_MAKE_TYPE_LIST(...) INTERNAL_CATCH_MAKE_TYPE_LIST2(INTERNAL_CATCH_REMOVE_PARENS(__VA_ARGS__)) -#else -#define INTERNAL_CATCH_MAKE_TYPE_LIST2(...) INTERNAL_CATCH_EXPAND_VARGS(decltype(get_wrapper())) -#define INTERNAL_CATCH_MAKE_TYPE_LIST(...) INTERNAL_CATCH_EXPAND_VARGS(INTERNAL_CATCH_MAKE_TYPE_LIST2(INTERNAL_CATCH_REMOVE_PARENS(__VA_ARGS__))) -#endif - -#define INTERNAL_CATCH_MAKE_TYPE_LISTS_FROM_TYPES(...)\ - CATCH_REC_LIST(INTERNAL_CATCH_MAKE_TYPE_LIST,__VA_ARGS__) - -#define INTERNAL_CATCH_REMOVE_PARENS_1_ARG(_0) INTERNAL_CATCH_REMOVE_PARENS(_0) -#define INTERNAL_CATCH_REMOVE_PARENS_2_ARG(_0, _1) INTERNAL_CATCH_REMOVE_PARENS(_0), INTERNAL_CATCH_REMOVE_PARENS_1_ARG(_1) -#define INTERNAL_CATCH_REMOVE_PARENS_3_ARG(_0, _1, _2) INTERNAL_CATCH_REMOVE_PARENS(_0), INTERNAL_CATCH_REMOVE_PARENS_2_ARG(_1, _2) -#define INTERNAL_CATCH_REMOVE_PARENS_4_ARG(_0, _1, _2, _3) INTERNAL_CATCH_REMOVE_PARENS(_0), INTERNAL_CATCH_REMOVE_PARENS_3_ARG(_1, _2, _3) -#define INTERNAL_CATCH_REMOVE_PARENS_5_ARG(_0, _1, _2, _3, _4) INTERNAL_CATCH_REMOVE_PARENS(_0), INTERNAL_CATCH_REMOVE_PARENS_4_ARG(_1, _2, _3, _4) -#define INTERNAL_CATCH_REMOVE_PARENS_6_ARG(_0, _1, _2, _3, _4, _5) INTERNAL_CATCH_REMOVE_PARENS(_0), INTERNAL_CATCH_REMOVE_PARENS_5_ARG(_1, _2, _3, _4, _5) -#define INTERNAL_CATCH_REMOVE_PARENS_7_ARG(_0, _1, _2, _3, _4, _5, _6) INTERNAL_CATCH_REMOVE_PARENS(_0), INTERNAL_CATCH_REMOVE_PARENS_6_ARG(_1, _2, _3, _4, _5, _6) -#define INTERNAL_CATCH_REMOVE_PARENS_8_ARG(_0, _1, _2, _3, _4, _5, _6, _7) INTERNAL_CATCH_REMOVE_PARENS(_0), INTERNAL_CATCH_REMOVE_PARENS_7_ARG(_1, _2, _3, _4, _5, _6, _7) -#define INTERNAL_CATCH_REMOVE_PARENS_9_ARG(_0, _1, _2, _3, _4, _5, _6, _7, _8) INTERNAL_CATCH_REMOVE_PARENS(_0), INTERNAL_CATCH_REMOVE_PARENS_8_ARG(_1, _2, _3, _4, _5, _6, _7, _8) -#define INTERNAL_CATCH_REMOVE_PARENS_10_ARG(_0, _1, _2, _3, _4, _5, _6, _7, _8, _9) INTERNAL_CATCH_REMOVE_PARENS(_0), INTERNAL_CATCH_REMOVE_PARENS_9_ARG(_1, _2, _3, _4, _5, _6, _7, _8, _9) -#define INTERNAL_CATCH_REMOVE_PARENS_11_ARG(_0, _1, _2, _3, _4, _5, _6, _7, _8, _9, _10) INTERNAL_CATCH_REMOVE_PARENS(_0), INTERNAL_CATCH_REMOVE_PARENS_10_ARG(_1, _2, _3, _4, _5, _6, _7, _8, _9, _10) - -#define INTERNAL_CATCH_VA_NARGS_IMPL(_0, _1, _2, _3, _4, _5, _6, _7, _8, _9, _10, N, ...) N - -#define INTERNAL_CATCH_TYPE_GEN\ - template struct TypeList {};\ - template\ - constexpr auto get_wrapper() noexcept -> TypeList { return {}; }\ - template class...> struct TemplateTypeList{};\ - template class...Cs>\ - constexpr auto get_wrapper() noexcept -> TemplateTypeList { return {}; }\ - template\ - struct append;\ - template\ - struct rewrap;\ - template class, typename...>\ - struct create;\ - template class, typename>\ - struct convert;\ - \ - template \ - struct append { using type = T; };\ - template< template class L1, typename...E1, template class L2, typename...E2, typename...Rest>\ - struct append, L2, Rest...> { using type = typename append, Rest...>::type; };\ - template< template class L1, typename...E1, typename...Rest>\ - struct append, TypeList, Rest...> { using type = L1; };\ - \ - template< template class Container, template class List, typename...elems>\ - struct rewrap, List> { using type = TypeList>; };\ - template< template class Container, template class List, class...Elems, typename...Elements>\ - struct rewrap, List, Elements...> { using type = typename append>, typename rewrap, Elements...>::type>::type; };\ - \ - template