From 43a39036ac0595b86875d330ff247b85d3b6f98f Mon Sep 17 00:00:00 2001 From: davemarco <83603688+davemarco@users.noreply.github.com> Date: Thu, 11 Jun 2026 13:36:57 -0400 Subject: [PATCH 01/56] feat(clp-s): Support count and count-by-time aggregation output to the results cache. Allows --count and --count-by-time to be used with the results-cache output handler, writing aggregation results directly to MongoDB via _id-keyed atomic $inc upserts so that partial results from multiple search processes merge correctly without a reducer. --- .../core/src/clp_s/CommandLineArguments.cpp | 62 ++++++-- .../core/src/clp_s/CommandLineArguments.hpp | 14 ++ .../core/src/clp_s/OutputHandlerImpl.cpp | 133 ++++++++++++++++-- .../core/src/clp_s/OutputHandlerImpl.hpp | 92 ++++++++++++ .../core/src/clp_s/archive_constants.hpp | 9 ++ components/core/src/clp_s/clp-s.cpp | 35 ++++- components/core/src/clp_s/kv_ir_search.cpp | 17 ++- 7 files changed, 329 insertions(+), 33 deletions(-) diff --git a/components/core/src/clp_s/CommandLineArguments.cpp b/components/core/src/clp_s/CommandLineArguments.cpp index 752f12ffa3..517689c8b1 100644 --- a/components/core/src/clp_s/CommandLineArguments.cpp +++ b/components/core/src/clp_s/CommandLineArguments.cpp @@ -3,6 +3,7 @@ #include #include #include +#include #include #include @@ -845,6 +846,15 @@ CommandLineArguments::parse_arguments(int argc, char const** argv) { po::value( &results_cache_options.dataset)->value_name("DATASET"), "The dataset name to include in each result document" + )( + "count", + "Count the number of results" + )( + "count-by-time", + po::value( + &results_cache_options.count_by_time_bucket_size + )->value_name("SIZE"), + "Count the number of results in each time span of the given size (ms)" ); FileOutputHandlerOptions file_options{}; @@ -924,6 +934,17 @@ CommandLineArguments::parse_arguments(int argc, char const** argv) { << " --host localhost" << " --port 14009" << " --job-id 1" << std::endl; + std::cerr << std::endl; + + std::cerr << " # Search archives in archives-dir for logs matching a KQL query" + R"( "level: INFO" and output a count aggregation to the results cache)" + << std::endl; + std::cerr << " " << m_program_name << R"( s archives-dir "level: INFO")" + << " " << cResultsCacheOutputHandlerName + << " --uri mongodb://127.0.0.1:27017/test" + " --collection test" + " --count" + << std::endl; po::options_description visible_options; visible_options.add(general_options); @@ -1114,32 +1135,40 @@ void CommandLineArguments::parse_reducer_output_handler_options( throw std::invalid_argument("job-id cannot be negative."); } - bool has_aggregation{}; + auto const aggregation_type{ + parse_aggregation_options(parsed_options, reducer_options.count_by_time_bucket_size) + }; + if (false == aggregation_type.has_value()) { + throw std::invalid_argument( + "The reducer output handler currently only supports count and" + " count-by-time aggregations." + ); + } + reducer_options.aggregation_type = aggregation_type.value(); +} + +auto CommandLineArguments::parse_aggregation_options( + po::variables_map const& parsed_options, + int64_t count_by_time_bucket_size +) -> std::optional { + std::optional aggregation_type; if (parsed_options.count("count")) { - reducer_options.aggregation_type = AggregationType::Count; - has_aggregation = true; + aggregation_type = AggregationType::Count; } if (parsed_options.count("count-by-time")) { - if (has_aggregation) { + if (aggregation_type.has_value()) { throw std::invalid_argument( "The --count-by-time and --count options are mutually exclusive." ); } - if (reducer_options.count_by_time_bucket_size <= 0) { + if (count_by_time_bucket_size <= 0) { throw std::invalid_argument("Value for count-by-time must be greater than zero."); } - reducer_options.aggregation_type = AggregationType::CountByTime; - has_aggregation = true; - } - - if (false == has_aggregation) { - throw std::invalid_argument( - "The reducer output handler currently only supports count and" - " count-by-time aggregations." - ); + aggregation_type = AggregationType::CountByTime; } + return aggregation_type; } void CommandLineArguments::parse_results_cache_output_handler_options( @@ -1171,6 +1200,11 @@ void CommandLineArguments::parse_results_cache_output_handler_options( if (0 == results_cache_options.max_num_results) { throw std::invalid_argument("max-num-results cannot be 0."); } + + results_cache_options.aggregation_type = parse_aggregation_options( + parsed_options, + results_cache_options.count_by_time_bucket_size + ); } void CommandLineArguments::parse_file_output_handler_options( diff --git a/components/core/src/clp_s/CommandLineArguments.hpp b/components/core/src/clp_s/CommandLineArguments.hpp index ee4f7c6a94..f6b8f088da 100644 --- a/components/core/src/clp_s/CommandLineArguments.hpp +++ b/components/core/src/clp_s/CommandLineArguments.hpp @@ -42,6 +42,8 @@ class CommandLineArguments { std::string dataset; uint64_t batch_size{1000}; uint64_t max_num_results{1000}; + std::optional aggregation_type; + int64_t count_by_time_bucket_size{}; // Milliseconds }; struct FileOutputHandlerOptions { @@ -156,6 +158,18 @@ class CommandLineArguments { NetworkOutputHandlerOptions& network_options ); + /** + * Parses and validates the aggregation options (count and count-by-time) supported by some + * output handlers. + * @param parsed_options + * @param count_by_time_bucket_size + * @return The requested aggregation type, or std::nullopt if no aggregation was requested. + */ + [[nodiscard]] static auto parse_aggregation_options( + boost::program_options::variables_map const& parsed_options, + int64_t count_by_time_bucket_size + ) -> std::optional; + /** * Validates output options related to the Reducer output handler. * @param options_description diff --git a/components/core/src/clp_s/OutputHandlerImpl.cpp b/components/core/src/clp_s/OutputHandlerImpl.cpp index 1d9aa9118f..cb1a4114cd 100644 --- a/components/core/src/clp_s/OutputHandlerImpl.cpp +++ b/components/core/src/clp_s/OutputHandlerImpl.cpp @@ -8,6 +8,8 @@ #include #include #include +#include +#include #include #include #include @@ -24,6 +26,39 @@ using std::string; using std::string_view; namespace clp_s { +// The two count-writing paths (direct-to-results-cache and via the reducer) must use the same +// field name for the count value. NOTE: only the field name is shared. Count-by-time documents +// ({timestamp, count}) have the same shape on both paths, but for plain count the reducer writes +// a nested record-group document ({group_tags, records: [{count}]}) while +// CountToResultsCacheOutputHandler writes a flat {count} document, so consumers of the two paths +// differ. +static_assert( + string_view{constants::results_cache::search::cCount} + == string_view{reducer::CountOperator::cRecordElementKey} +); + +namespace { +/** + * Connects to the results cache and returns the requested collection. + * @tparam OperationFailedT The handler-specific exception type to throw on failure. + * @param uri + * @param collection + * @param client Returns the connected client, which must outlive the returned collection. + * @return The collection. + */ +template +auto connect_to_results_cache(string const& uri, string const& collection, mongocxx::client& client) + -> mongocxx::collection { + try { + auto mongo_uri = mongocxx::uri(uri); + client = mongocxx::client(mongo_uri); + return client[mongo_uri.database()][collection]; + } catch (mongocxx::exception const& e) { + throw OperationFailedT(ErrorCode::ErrorCodeBadParamDbUri, __FILENAME__, __LINE__); + } +} +} // namespace + void FileOutputHandler::write( string_view message, epochtime_t timestamp, @@ -78,14 +113,8 @@ ResultsCacheOutputHandler::ResultsCacheOutputHandler( m_batch_size{batch_size}, m_max_num_results{max_num_results}, m_dataset{std::move(dataset)} { - try { - auto mongo_uri = mongocxx::uri(uri); - m_client = mongocxx::client(mongo_uri); - m_collection = m_client[mongo_uri.database()][collection]; - m_results.reserve(m_batch_size); - } catch (mongocxx::exception const& e) { - throw OperationFailed(ErrorCode::ErrorCodeBadParamDbUri, __FILENAME__, __LINE__); - } + m_collection = connect_to_results_cache(uri, collection, m_client); + m_results.reserve(m_batch_size); } ErrorCode ResultsCacheOutputHandler::finish() { @@ -214,4 +243,92 @@ ErrorCode CountByTimeOutputHandler::finish() { } return ErrorCode::ErrorCodeSuccess; } + +CountToResultsCacheOutputHandler::CountToResultsCacheOutputHandler( + string const& uri, + string const& collection +) + : ::clp_s::search::OutputHandler(false, false) { + m_collection = connect_to_results_cache(uri, collection, m_client); +} + +ErrorCode CountToResultsCacheOutputHandler::finish() { + if (0 == m_count) { + return ErrorCode::ErrorCodeSuccess; + } + + // Filtering on _id makes the upsert safe under concurrent writers: _id's built-in unique + // index guarantees that racing upserts resolve to a single document. + try { + auto const filter = bsoncxx::builder::basic::make_document( + bsoncxx::builder::basic::kvp( + constants::results_cache::cDocId, + constants::results_cache::search::cCount + ) + ); + auto const inc_count = bsoncxx::builder::basic::make_document( + bsoncxx::builder::basic::kvp(constants::results_cache::search::cCount, m_count) + ); + auto const update = bsoncxx::builder::basic::make_document( + bsoncxx::builder::basic::kvp("$inc", inc_count) + ); + m_collection + .update_one(filter.view(), update.view(), mongocxx::options::update{}.upsert(true)); + } catch (mongocxx::exception const& e) { + return ErrorCode::ErrorCodeFailureDbBulkWrite; + } + return ErrorCode::ErrorCodeSuccess; +} + +CountByTimeToResultsCacheOutputHandler::CountByTimeToResultsCacheOutputHandler( + string const& uri, + string const& collection, + int64_t count_by_time_bucket_size +) + : ::clp_s::search::OutputHandler(true, false), + m_count_by_time_bucket_size(count_by_time_bucket_size) { + m_collection = connect_to_results_cache(uri, collection, m_client); +} + +ErrorCode CountByTimeToResultsCacheOutputHandler::finish() { + if (m_bucket_counts.empty()) { + return ErrorCode::ErrorCodeSuccess; + } + + // Each bucket document is keyed by its bucket timestamp via _id, whose built-in unique index + // makes concurrent upserts from multiple writers resolve to a single document per bucket. The + // timestamp is duplicated into a regular field on first insert so that readers don't need to + // inspect _id. + try { + auto bulk_write = m_collection.create_bulk_write(); + for (auto const& [bucket_timestamp, count] : m_bucket_counts) { + auto filter = bsoncxx::builder::basic::make_document( + bsoncxx::builder::basic::kvp(constants::results_cache::cDocId, bucket_timestamp) + ); + auto const inc_count = bsoncxx::builder::basic::make_document( + bsoncxx::builder::basic::kvp(constants::results_cache::search::cCount, count) + ); + auto const set_timestamp_on_insert = bsoncxx::builder::basic::make_document( + bsoncxx::builder::basic::kvp( + constants::results_cache::search::cTimestamp, + bucket_timestamp + ) + ); + auto update = bsoncxx::builder::basic::make_document( + bsoncxx::builder::basic::kvp("$inc", inc_count), + bsoncxx::builder::basic::kvp("$setOnInsert", set_timestamp_on_insert) + ); + + // The documents are moved into the operation so that they outlive this loop + // iteration. + mongocxx::model::update_one update_op{std::move(filter), std::move(update)}; + update_op.upsert(true); + bulk_write.append(update_op); + } + bulk_write.execute(); + } catch (mongocxx::exception const& e) { + return ErrorCode::ErrorCodeFailureDbBulkWrite; + } + return ErrorCode::ErrorCodeSuccess; +} } // namespace clp_s diff --git a/components/core/src/clp_s/OutputHandlerImpl.hpp b/components/core/src/clp_s/OutputHandlerImpl.hpp index 7cef8674b0..ccd126a35f 100644 --- a/components/core/src/clp_s/OutputHandlerImpl.hpp +++ b/components/core/src/clp_s/OutputHandlerImpl.hpp @@ -6,6 +6,7 @@ #include #include +#include #include #include #include @@ -279,6 +280,97 @@ class CountByTimeOutputHandler : public ::clp_s::search::OutputHandler { int64_t m_count_by_time_bucket_size; }; +/** + * Output handler that performs a count aggregation and writes the result to the results cache + * (MongoDB). The count is written with an atomic upsert that increments the stored count, so + * results from multiple search processes writing to the same collection merge correctly. + */ +class CountToResultsCacheOutputHandler : public ::clp_s::search::OutputHandler { +public: + // Types + class OperationFailed : public TraceableException { + public: + // Constructors + OperationFailed(ErrorCode error_code, char const* const filename, int line_number) + : TraceableException(error_code, filename, line_number) {} + }; + + // Constructors + CountToResultsCacheOutputHandler(std::string const& uri, std::string const& collection); + + // Methods inherited from OutputHandler + void write( + std::string_view message, + epochtime_t timestamp, + std::string_view archive_id, + int64_t log_event_idx + ) override {} + + void write(std::string_view message) override { m_count += 1; } + + /** + * Upserts the count to the results cache. + * @return ErrorCodeSuccess on success + * @return ErrorCodeFailureDbBulkWrite on failure to write to the results cache + */ + ErrorCode finish() override; + +private: + mongocxx::client m_client; + mongocxx::collection m_collection; + int64_t m_count{}; +}; + +/** + * Output handler that performs a count aggregation bucketed by time and writes the results to the + * results cache (MongoDB). Each bucket's count is written with an atomic upsert that increments + * the stored count, so results from multiple search processes writing to the same collection merge + * correctly. + */ +class CountByTimeToResultsCacheOutputHandler : public ::clp_s::search::OutputHandler { +public: + // Types + class OperationFailed : public TraceableException { + public: + // Constructors + OperationFailed(ErrorCode error_code, char const* const filename, int line_number) + : TraceableException(error_code, filename, line_number) {} + }; + + // Constructors + CountByTimeToResultsCacheOutputHandler( + std::string const& uri, + std::string const& collection, + int64_t count_by_time_bucket_size + ); + + // Methods inherited from OutputHandler + void write( + std::string_view message, + epochtime_t timestamp, + std::string_view archive_id, + int64_t log_event_idx + ) override { + int64_t bucket = (timestamp / m_count_by_time_bucket_size) * m_count_by_time_bucket_size; + m_bucket_counts[bucket] += 1; + } + + void write(std::string_view message) override {} + + /** + * Upserts the bucket counts to the results cache. + * @return ErrorCodeSuccess on success + * @return ErrorCodeFailureDbBulkWrite on failure to write to the results cache + */ + ErrorCode finish() override; + +private: + mongocxx::client m_client; + mongocxx::collection m_collection; + std::map m_bucket_counts; + int64_t m_count_by_time_bucket_size; +}; + /** * Output handler that records all results in a provided vector. */ diff --git a/components/core/src/clp_s/archive_constants.hpp b/components/core/src/clp_s/archive_constants.hpp index 796c5a5bc3..2f2b949a2d 100644 --- a/components/core/src/clp_s/archive_constants.hpp +++ b/components/core/src/clp_s/archive_constants.hpp @@ -44,6 +44,11 @@ constexpr std::string_view cFileSplitNumber{"_file_split_number"}; constexpr std::string_view cArchiveCreatorId{"_archive_creator_id"}; } // namespace range_index +namespace results_cache { +// MongoDB's primary-key field, which is backed by a built-in unique index. +constexpr char cDocId[]{"_id"}; +} // namespace results_cache + namespace results_cache::decompression { constexpr char cPath[]{"path"}; constexpr char cStreamId[]{"stream_id"}; @@ -59,6 +64,10 @@ constexpr char cTimestamp[]{"timestamp"}; constexpr char cMessage[]{"message"}; constexpr char cArchiveId[]{"archive_id"}; constexpr std::string_view cDataset{"dataset"}; +// Must match reducer::CountOperator::cRecordElementKey (enforced by a static_assert in +// OutputHandlerImpl.cpp, which also documents how the two count-writing paths' document shapes +// differ). +constexpr char cCount[]{"count"}; } // namespace results_cache::search } // namespace clp_s::constants #endif // CLP_S_ARCHIVE_CONSTANTS_HPP diff --git a/components/core/src/clp_s/clp-s.cpp b/components/core/src/clp_s/clp-s.cpp index 31308b6ad0..ea123aece1 100644 --- a/components/core/src/clp_s/clp-s.cpp +++ b/components/core/src/clp_s/clp-s.cpp @@ -264,13 +264,34 @@ bool search_archive( }, [&](CommandLineArguments::ResultsCacheOutputHandlerOptions const& options) -> void { - output_handler = std::make_unique( - options.uri, - options.collection, - options.batch_size, - options.max_num_results, - options.dataset - ); + if (false == options.aggregation_type.has_value()) { + output_handler = std::make_unique( + options.uri, + options.collection, + options.batch_size, + options.max_num_results, + options.dataset + ); + } else if (CommandLineArguments::AggregationType::Count + == options.aggregation_type.value()) + { + output_handler + = std::make_unique( + options.uri, + options.collection + ); + } else if (CommandLineArguments::AggregationType::CountByTime + == options.aggregation_type.value()) + { + output_handler = std::make_unique< + clp_s::CountByTimeToResultsCacheOutputHandler + >(options.uri, + options.collection, + options.count_by_time_bucket_size); + } else { + SPDLOG_ERROR("Unhandled aggregation type."); + output_handler = nullptr; + } }, [&](CommandLineArguments::StdoutOutputHandlerOptions const& options) -> void { diff --git a/components/core/src/clp_s/kv_ir_search.cpp b/components/core/src/clp_s/kv_ir_search.cpp index be344120f2..5f642729cc 100644 --- a/components/core/src/clp_s/kv_ir_search.cpp +++ b/components/core/src/clp_s/kv_ir_search.cpp @@ -1,3 +1,4 @@ + #include "kv_ir_search.hpp" #include @@ -246,10 +247,18 @@ auto search_kv_ir_stream( return KvIrSearchError{KvIrSearchErrorEnum::ProjectionSupportNotImplemented}; } - if (std::holds_alternative( - command_line_arguments.get_output_handler_options() - )) - { + auto const& output_handler_options{command_line_arguments.get_output_handler_options()}; + auto const* results_cache_options + = std::get_if( + &output_handler_options + ); + bool const aggregation_was_requested + = std::holds_alternative( + output_handler_options + ) + || (nullptr != results_cache_options + && results_cache_options->aggregation_type.has_value()); + if (aggregation_was_requested) { SPDLOG_ERROR("kv-ir search: Count support is not implemented."); return KvIrSearchError{KvIrSearchErrorEnum::CountSupportNotImplemented}; } From 33ad6b789410df510b6e0e169f20d571ff97cca6 Mon Sep 17 00:00:00 2001 From: davemarco <83603688+davemarco@users.noreply.github.com> Date: Thu, 11 Jun 2026 14:35:41 -0400 Subject: [PATCH 02/56] refactor(clp-s): Clean up count aggregation code to better match project conventions. --- .../core/src/clp_s/CommandLineArguments.cpp | 48 +++++++++---------- .../core/src/clp_s/CommandLineArguments.hpp | 7 +-- .../core/src/clp_s/OutputHandlerImpl.cpp | 2 - components/core/src/clp_s/kv_ir_search.cpp | 1 - 4 files changed, 28 insertions(+), 30 deletions(-) diff --git a/components/core/src/clp_s/CommandLineArguments.cpp b/components/core/src/clp_s/CommandLineArguments.cpp index 517689c8b1..65db13e735 100644 --- a/components/core/src/clp_s/CommandLineArguments.cpp +++ b/components/core/src/clp_s/CommandLineArguments.cpp @@ -1105,6 +1105,30 @@ void CommandLineArguments::parse_network_dest_output_handler_options( } } +auto CommandLineArguments::parse_aggregation_options( + po::variables_map const& parsed_options, + int64_t count_by_time_bucket_size +) -> std::optional { + std::optional aggregation_type; + if (parsed_options.count("count")) { + aggregation_type = AggregationType::Count; + } + if (parsed_options.count("count-by-time")) { + if (aggregation_type.has_value()) { + throw std::invalid_argument( + "The --count-by-time and --count options are mutually exclusive." + ); + } + + if (count_by_time_bucket_size <= 0) { + throw std::invalid_argument("Value for count-by-time must be greater than zero."); + } + + aggregation_type = AggregationType::CountByTime; + } + return aggregation_type; +} + void CommandLineArguments::parse_reducer_output_handler_options( po::options_description const& options_description, std::vector const& options, @@ -1147,30 +1171,6 @@ void CommandLineArguments::parse_reducer_output_handler_options( reducer_options.aggregation_type = aggregation_type.value(); } -auto CommandLineArguments::parse_aggregation_options( - po::variables_map const& parsed_options, - int64_t count_by_time_bucket_size -) -> std::optional { - std::optional aggregation_type; - if (parsed_options.count("count")) { - aggregation_type = AggregationType::Count; - } - if (parsed_options.count("count-by-time")) { - if (aggregation_type.has_value()) { - throw std::invalid_argument( - "The --count-by-time and --count options are mutually exclusive." - ); - } - - if (count_by_time_bucket_size <= 0) { - throw std::invalid_argument("Value for count-by-time must be greater than zero."); - } - - aggregation_type = AggregationType::CountByTime; - } - return aggregation_type; -} - void CommandLineArguments::parse_results_cache_output_handler_options( po::options_description const& options_description, std::vector const& options, diff --git a/components/core/src/clp_s/CommandLineArguments.hpp b/components/core/src/clp_s/CommandLineArguments.hpp index f6b8f088da..d17302b193 100644 --- a/components/core/src/clp_s/CommandLineArguments.hpp +++ b/components/core/src/clp_s/CommandLineArguments.hpp @@ -159,10 +159,11 @@ class CommandLineArguments { ); /** - * Parses and validates the aggregation options (count and count-by-time) supported by some - * output handlers. + * Validates the aggregation options (count and count-by-time) for output handlers that + * support aggregations. * @param parsed_options - * @param count_by_time_bucket_size + * @param count_by_time_bucket_size The parsed value of the count-by-time option; only + * validated when that option was specified. * @return The requested aggregation type, or std::nullopt if no aggregation was requested. */ [[nodiscard]] static auto parse_aggregation_options( diff --git a/components/core/src/clp_s/OutputHandlerImpl.cpp b/components/core/src/clp_s/OutputHandlerImpl.cpp index cb1a4114cd..b5f84347a5 100644 --- a/components/core/src/clp_s/OutputHandlerImpl.cpp +++ b/components/core/src/clp_s/OutputHandlerImpl.cpp @@ -319,8 +319,6 @@ ErrorCode CountByTimeToResultsCacheOutputHandler::finish() { bsoncxx::builder::basic::kvp("$setOnInsert", set_timestamp_on_insert) ); - // The documents are moved into the operation so that they outlive this loop - // iteration. mongocxx::model::update_one update_op{std::move(filter), std::move(update)}; update_op.upsert(true); bulk_write.append(update_op); diff --git a/components/core/src/clp_s/kv_ir_search.cpp b/components/core/src/clp_s/kv_ir_search.cpp index 5f642729cc..cde0d9b55d 100644 --- a/components/core/src/clp_s/kv_ir_search.cpp +++ b/components/core/src/clp_s/kv_ir_search.cpp @@ -1,4 +1,3 @@ - #include "kv_ir_search.hpp" #include From 4b6a39a0e7ea3ab8c6ecfa5ee93c8b6e3127add4 Mon Sep 17 00:00:00 2001 From: davemarco <83603688+davemarco@users.noreply.github.com> Date: Thu, 11 Jun 2026 14:43:46 -0400 Subject: [PATCH 03/56] docs(clp-s): Clarify that the existing count aggregation example in the search usage text outputs to the reducer. --- components/core/src/clp_s/CommandLineArguments.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/components/core/src/clp_s/CommandLineArguments.cpp b/components/core/src/clp_s/CommandLineArguments.cpp index 65db13e735..f3ff521fb0 100644 --- a/components/core/src/clp_s/CommandLineArguments.cpp +++ b/components/core/src/clp_s/CommandLineArguments.cpp @@ -927,7 +927,7 @@ CommandLineArguments::parse_arguments(int argc, char const** argv) { std::cerr << std::endl; std::cerr << " # Search archives in archives-dir for logs matching a KQL query" - R"( "level: INFO" and output perform a count aggregation)" + R"( "level: INFO" and output a count aggregation to the reducer)" << std::endl; std::cerr << " " << m_program_name << R"( s archives-dir "level: INFO")" << " " << cReducerOutputHandlerName << " --count" From 8ed417afb333c7b61f5765059d16d84406271939 Mon Sep 17 00:00:00 2001 From: davemarco <83603688+davemarco@users.noreply.github.com> Date: Fri, 12 Jun 2026 17:00:25 -0400 Subject: [PATCH 04/56] refactor(clp-s): Rename count output handlers to indicate their destination and simplify comments. --- .../core/src/clp_s/OutputHandlerImpl.cpp | 31 +++++-------------- .../core/src/clp_s/OutputHandlerImpl.hpp | 24 ++++++-------- .../core/src/clp_s/archive_constants.hpp | 3 -- components/core/src/clp_s/clp-s.cpp | 14 ++++----- components/core/src/clp_s/kv_ir_search.cpp | 15 ++++----- 5 files changed, 33 insertions(+), 54 deletions(-) diff --git a/components/core/src/clp_s/OutputHandlerImpl.cpp b/components/core/src/clp_s/OutputHandlerImpl.cpp index b5f84347a5..83cd3ab7ac 100644 --- a/components/core/src/clp_s/OutputHandlerImpl.cpp +++ b/components/core/src/clp_s/OutputHandlerImpl.cpp @@ -26,24 +26,13 @@ using std::string; using std::string_view; namespace clp_s { -// The two count-writing paths (direct-to-results-cache and via the reducer) must use the same -// field name for the count value. NOTE: only the field name is shared. Count-by-time documents -// ({timestamp, count}) have the same shape on both paths, but for plain count the reducer writes -// a nested record-group document ({group_tags, records: [{count}]}) while -// CountToResultsCacheOutputHandler writes a flat {count} document, so consumers of the two paths -// differ. -static_assert( - string_view{constants::results_cache::search::cCount} - == string_view{reducer::CountOperator::cRecordElementKey} -); - namespace { /** * Connects to the results cache and returns the requested collection. - * @tparam OperationFailedT The handler-specific exception type to throw on failure. + * @tparam OperationFailedT The type of exception to throw on failure. * @param uri * @param collection - * @param client Returns the connected client, which must outlive the returned collection. + * @param client Returns the connected client. * @return The collection. */ template @@ -209,18 +198,18 @@ void ResultsCacheOutputHandler::write( } } -CountOutputHandler::CountOutputHandler(int reducer_socket_fd) +CountToReducerOutputHandler::CountToReducerOutputHandler(int reducer_socket_fd) : ::clp_s::search::OutputHandler(false, false), m_reducer_socket_fd(reducer_socket_fd), m_pipeline(reducer::PipelineInputMode::InterStage) { m_pipeline.add_pipeline_stage(std::make_shared()); } -void CountOutputHandler::write(string_view message) { +void CountToReducerOutputHandler::write(string_view message) { m_pipeline.push_record(reducer::EmptyRecord{}); } -ErrorCode CountOutputHandler::finish() { +ErrorCode CountToReducerOutputHandler::finish() { if (false == reducer::send_pipeline_results(m_reducer_socket_fd, std::move(m_pipeline.finish()))) { @@ -229,7 +218,7 @@ ErrorCode CountOutputHandler::finish() { return ErrorCode::ErrorCodeSuccess; } -ErrorCode CountByTimeOutputHandler::finish() { +ErrorCode CountByTimeToReducerOutputHandler::finish() { if (false == reducer::send_pipeline_results( m_reducer_socket_fd, @@ -257,8 +246,6 @@ ErrorCode CountToResultsCacheOutputHandler::finish() { return ErrorCode::ErrorCodeSuccess; } - // Filtering on _id makes the upsert safe under concurrent writers: _id's built-in unique - // index guarantees that racing upserts resolve to a single document. try { auto const filter = bsoncxx::builder::basic::make_document( bsoncxx::builder::basic::kvp( @@ -295,10 +282,8 @@ ErrorCode CountByTimeToResultsCacheOutputHandler::finish() { return ErrorCode::ErrorCodeSuccess; } - // Each bucket document is keyed by its bucket timestamp via _id, whose built-in unique index - // makes concurrent upserts from multiple writers resolve to a single document per bucket. The - // timestamp is duplicated into a regular field on first insert so that readers don't need to - // inspect _id. + // The bucket timestamp is also stored in a `timestamp` field so that the documents match the + // `{timestamp, count}` shape the reducer writes. try { auto bulk_write = m_collection.create_bulk_write(); for (auto const& [bucket_timestamp, count] : m_bucket_counts) { diff --git a/components/core/src/clp_s/OutputHandlerImpl.hpp b/components/core/src/clp_s/OutputHandlerImpl.hpp index ccd126a35f..9a741e82f2 100644 --- a/components/core/src/clp_s/OutputHandlerImpl.hpp +++ b/components/core/src/clp_s/OutputHandlerImpl.hpp @@ -215,10 +215,10 @@ class ResultsCacheOutputHandler : public ::clp_s::search::OutputHandler { /** * Output handler that performs a count aggregation and sends the results to a reducer. */ -class CountOutputHandler : public ::clp_s::search::OutputHandler { +class CountToReducerOutputHandler : public ::clp_s::search::OutputHandler { public: // Constructors - CountOutputHandler(int reducer_socket_fd); + CountToReducerOutputHandler(int reducer_socket_fd); // Methods inherited from OutputHandler void write( @@ -246,10 +246,10 @@ class CountOutputHandler : public ::clp_s::search::OutputHandler { * Output handler that performs a count aggregation bucketed by time and sends the results to a * reducer. */ -class CountByTimeOutputHandler : public ::clp_s::search::OutputHandler { +class CountByTimeToReducerOutputHandler : public ::clp_s::search::OutputHandler { public: // Constructors - CountByTimeOutputHandler(int reducer_socket_fd, int64_t count_by_time_bucket_size) + CountByTimeToReducerOutputHandler(int reducer_socket_fd, int64_t count_by_time_bucket_size) : search::OutputHandler{true, false}, m_reducer_socket_fd{reducer_socket_fd}, m_count_by_time_bucket_size{count_by_time_bucket_size} {} @@ -281,9 +281,7 @@ class CountByTimeOutputHandler : public ::clp_s::search::OutputHandler { }; /** - * Output handler that performs a count aggregation and writes the result to the results cache - * (MongoDB). The count is written with an atomic upsert that increments the stored count, so - * results from multiple search processes writing to the same collection merge correctly. + * Output handler that performs a count aggregation and writes the results to the results cache. */ class CountToResultsCacheOutputHandler : public ::clp_s::search::OutputHandler { public: @@ -309,9 +307,9 @@ class CountToResultsCacheOutputHandler : public ::clp_s::search::OutputHandler { void write(std::string_view message) override { m_count += 1; } /** - * Upserts the count to the results cache. + * Flushes the count. * @return ErrorCodeSuccess on success - * @return ErrorCodeFailureDbBulkWrite on failure to write to the results cache + * @return ErrorCodeFailureDbBulkWrite on database error */ ErrorCode finish() override; @@ -323,9 +321,7 @@ class CountToResultsCacheOutputHandler : public ::clp_s::search::OutputHandler { /** * Output handler that performs a count aggregation bucketed by time and writes the results to the - * results cache (MongoDB). Each bucket's count is written with an atomic upsert that increments - * the stored count, so results from multiple search processes writing to the same collection merge - * correctly. + * results cache. */ class CountByTimeToResultsCacheOutputHandler : public ::clp_s::search::OutputHandler { public: @@ -358,9 +354,9 @@ class CountByTimeToResultsCacheOutputHandler : public ::clp_s::search::OutputHan void write(std::string_view message) override {} /** - * Upserts the bucket counts to the results cache. + * Flushes the counts. * @return ErrorCodeSuccess on success - * @return ErrorCodeFailureDbBulkWrite on failure to write to the results cache + * @return ErrorCodeFailureDbBulkWrite on database error */ ErrorCode finish() override; diff --git a/components/core/src/clp_s/archive_constants.hpp b/components/core/src/clp_s/archive_constants.hpp index 2f2b949a2d..212d593d5e 100644 --- a/components/core/src/clp_s/archive_constants.hpp +++ b/components/core/src/clp_s/archive_constants.hpp @@ -64,9 +64,6 @@ constexpr char cTimestamp[]{"timestamp"}; constexpr char cMessage[]{"message"}; constexpr char cArchiveId[]{"archive_id"}; constexpr std::string_view cDataset{"dataset"}; -// Must match reducer::CountOperator::cRecordElementKey (enforced by a static_assert in -// OutputHandlerImpl.cpp, which also documents how the two count-writing paths' document shapes -// differ). constexpr char cCount[]{"count"}; } // namespace results_cache::search } // namespace clp_s::constants diff --git a/components/core/src/clp_s/clp-s.cpp b/components/core/src/clp_s/clp-s.cpp index ea123aece1..3875dfa1cb 100644 --- a/components/core/src/clp_s/clp-s.cpp +++ b/components/core/src/clp_s/clp-s.cpp @@ -247,16 +247,16 @@ bool search_archive( -> void { if (CommandLineArguments::AggregationType::Count == options.aggregation_type) { - output_handler = std::make_unique( - reducer_socket_fd - ); + output_handler + = std::make_unique( + reducer_socket_fd + ); } else if (CommandLineArguments::AggregationType::CountByTime == options.aggregation_type) { - output_handler = std::make_unique( - reducer_socket_fd, - options.count_by_time_bucket_size - ); + output_handler = std::make_unique< + clp_s::CountByTimeToReducerOutputHandler + >(reducer_socket_fd, options.count_by_time_bucket_size); } else { SPDLOG_ERROR("Unhandled aggregation type."); output_handler = nullptr; diff --git a/components/core/src/clp_s/kv_ir_search.cpp b/components/core/src/clp_s/kv_ir_search.cpp index cde0d9b55d..bb610eb0c7 100644 --- a/components/core/src/clp_s/kv_ir_search.cpp +++ b/components/core/src/clp_s/kv_ir_search.cpp @@ -247,17 +247,18 @@ auto search_kv_ir_stream( } auto const& output_handler_options{command_line_arguments.get_output_handler_options()}; + bool const reducer_aggregation_requested + = std::holds_alternative( + output_handler_options + ); auto const* results_cache_options = std::get_if( &output_handler_options ); - bool const aggregation_was_requested - = std::holds_alternative( - output_handler_options - ) - || (nullptr != results_cache_options - && results_cache_options->aggregation_type.has_value()); - if (aggregation_was_requested) { + bool const results_cache_aggregation_requested + = nullptr != results_cache_options + && results_cache_options->aggregation_type.has_value(); + if (reducer_aggregation_requested || results_cache_aggregation_requested) { SPDLOG_ERROR("kv-ir search: Count support is not implemented."); return KvIrSearchError{KvIrSearchErrorEnum::CountSupportNotImplemented}; } From 7455500fce56380a42f0fc6b1f0295a202aac1a2 Mon Sep 17 00:00:00 2001 From: marco Date: Thu, 18 Jun 2026 15:28:19 -0400 Subject: [PATCH 05/56] feat(clp-s): Support count and count-by-time aggregation output to stdout. Adds `--count` and `--count-by-time` to the stdout output handler, emitting per-archive results as newline-delimited JSON. The options may be given without naming the `stdout` handler (e.g. `clp-s s --count`), and the kv-ir search path now reports these aggregations as unsupported, consistent with the reducer and results-cache handlers. --- .../core/src/clp_s/CommandLineArguments.cpp | 76 +++++++++++++++---- .../core/src/clp_s/CommandLineArguments.hpp | 18 ++++- .../core/src/clp_s/OutputHandlerImpl.cpp | 64 ++++++++++++++++ .../core/src/clp_s/OutputHandlerImpl.hpp | 40 ++++++++++ components/core/src/clp_s/clp-s.cpp | 12 ++- components/core/src/clp_s/kv_ir_search.cpp | 8 +- 6 files changed, 200 insertions(+), 18 deletions(-) diff --git a/components/core/src/clp_s/CommandLineArguments.cpp b/components/core/src/clp_s/CommandLineArguments.cpp index f3ff521fb0..f89d30f027 100644 --- a/components/core/src/clp_s/CommandLineArguments.cpp +++ b/components/core/src/clp_s/CommandLineArguments.cpp @@ -39,7 +39,8 @@ constexpr std::string_view cStdoutCacheOutputHandlerName{"stdout"}; */ auto collect_subcommands( std::map const& subcommands, - std::vector const& options + std::vector const& options, + std::optional const& default_subcommand = std::nullopt ) -> std::map>; /** @@ -161,7 +162,8 @@ void validate_archive_paths( auto collect_subcommands( std::map const& subcommands, - std::vector const& options + std::vector const& options, + std::optional const& default_subcommand ) -> std::map> { std::optional current_subcommand; std::optional> current_subcommand_arguments; @@ -200,7 +202,14 @@ auto collect_subcommands( if (false == current_subcommand.has_value() || false == current_subcommand_arguments.has_value()) { - throw std::invalid_argument(fmt::format("unrecognized option \"{}\"", option)); + // A leading option token (e.g. `--count`) with no named subcommand selects the default + // subcommand, letting its options be specified without naming the subcommand. + if (false == default_subcommand.has_value() || option.empty() || '-' != option.front()) { + throw std::invalid_argument(fmt::format("unrecognized option \"{}\"", option)); + } + current_subcommand = default_subcommand; + current_subcommand_arguments.emplace(); + current_subcommand_options_description = subcommands.at(default_subcommand.value()); } current_subcommand_arguments.value().emplace_back(option); @@ -865,6 +874,21 @@ CommandLineArguments::parse_arguments(int argc, char const** argv) { "File output path" ); + StdoutOutputHandlerOptions stdout_options{}; + po::options_description stdout_output_handler_options("Stdout Output Handler Options"); + // clang-format off + stdout_output_handler_options.add_options()( + "count", + "Count the number of results" + )( + "count-by-time", + po::value( + &stdout_options.count_by_time_bucket_size + )->value_name("SIZE"), + "Count the number of results in each time span of the given size (ms)" + ); + // clang-format on + std::vector unrecognized_options = po::collect_unrecognized(parsed.options, po::include_positional); unrecognized_options.erase(unrecognized_options.begin()); @@ -945,6 +969,13 @@ CommandLineArguments::parse_arguments(int argc, char const** argv) { " --collection test" " --count" << std::endl; + std::cerr << std::endl; + + std::cerr << " # Search archives in archives-dir for logs matching a KQL query" + R"( "level: INFO" and output a count aggregation to stdout)" + << std::endl; + std::cerr << " " << m_program_name << R"( s archives-dir "level: INFO")" + << " --count" << std::endl; po::options_description visible_options; visible_options.add(general_options); @@ -953,6 +984,7 @@ CommandLineArguments::parse_arguments(int argc, char const** argv) { visible_options.add(network_output_handler_options); visible_options.add(results_cache_output_handler_options); visible_options.add(reducer_output_handler_options); + visible_options.add(stdout_output_handler_options); std::cerr << visible_options << '\n'; return ParsingResult::InfoCommand; } @@ -984,11 +1016,13 @@ CommandLineArguments::parse_arguments(int argc, char const** argv) { {std::string{cReducerOutputHandlerName}, &reducer_output_handler_options}, {std::string{cResultsCacheOutputHandlerName}, &results_cache_output_handler_options}, - {std::string{cStdoutCacheOutputHandlerName}, nullptr} - }; - auto const output_options_map{ - collect_subcommands(subcommands, unrecognized_output_options) + {std::string{cStdoutCacheOutputHandlerName}, &stdout_output_handler_options} }; + auto const output_options_map{collect_subcommands( + subcommands, + unrecognized_output_options, + std::string{cStdoutCacheOutputHandlerName} + )}; validate_archive_paths(archive_path, archive_id, m_input_paths); @@ -1047,14 +1081,14 @@ CommandLineArguments::parse_arguments(int argc, char const** argv) { std::move(results_cache_options) ); } else if (cStdoutCacheOutputHandlerName == output_handler_name) { - m_output_handler_options.emplace(); - if (false == output_handler_options.empty()) { - std::string error_msg{fmt::format( - "stdout output handler does not support \"{}\"", - output_handler_options.front() - )}; - throw std::invalid_argument(error_msg); - } + parse_stdout_output_handler_options( + stdout_output_handler_options, + output_handler_options, + stdout_options + ); + m_output_handler_options.emplace( + std::move(stdout_options) + ); } else if (cFileOutputHandlerName == output_handler_name) { parse_file_output_handler_options( file_output_handler_options, @@ -1222,6 +1256,18 @@ void CommandLineArguments::parse_file_output_handler_options( } } +void CommandLineArguments::parse_stdout_output_handler_options( + po::options_description const& options_description, + std::vector const& options, + StdoutOutputHandlerOptions& stdout_options +) { + po::variables_map parsed_options; + parse_subcommand_options(options_description, options, parsed_options); + + stdout_options.aggregation_type + = parse_aggregation_options(parsed_options, stdout_options.count_by_time_bucket_size); +} + void CommandLineArguments::print_basic_usage() const { std::cerr << "Usage: " << m_program_name << " [OPTIONS] COMMAND [COMMAND ARGUMENTS]" << std::endl; diff --git a/components/core/src/clp_s/CommandLineArguments.hpp b/components/core/src/clp_s/CommandLineArguments.hpp index d17302b193..f3edc59de0 100644 --- a/components/core/src/clp_s/CommandLineArguments.hpp +++ b/components/core/src/clp_s/CommandLineArguments.hpp @@ -63,7 +63,10 @@ class CommandLineArguments { int64_t count_by_time_bucket_size{}; // Milliseconds }; - struct StdoutOutputHandlerOptions {}; + struct StdoutOutputHandlerOptions { + std::optional aggregation_type; + int64_t count_by_time_bucket_size{}; // Milliseconds + }; using OutputHandlerOptionsVariant = std:: variant const& options, + StdoutOutputHandlerOptions& stdout_options + ); + void print_basic_usage() const; void print_compression_usage() const; diff --git a/components/core/src/clp_s/OutputHandlerImpl.cpp b/components/core/src/clp_s/OutputHandlerImpl.cpp index 83cd3ab7ac..75aebe84bf 100644 --- a/components/core/src/clp_s/OutputHandlerImpl.cpp +++ b/components/core/src/clp_s/OutputHandlerImpl.cpp @@ -12,12 +12,15 @@ #include #include #include +#include #include #include "../clp/networking/socket_utils.hpp" #include "../reducer/CountOperator.hpp" #include "../reducer/network_utils.hpp" #include "../reducer/Record.hpp" +#include "../reducer/RecordGroup.hpp" +#include "../reducer/RecordGroupIterator.hpp" #include "archive_constants.hpp" #include "search/OutputHandler.hpp" #include "TraceableException.hpp" @@ -233,6 +236,67 @@ ErrorCode CountByTimeToReducerOutputHandler::finish() { return ErrorCode::ErrorCodeSuccess; } +AggregationToStdoutOutputHandler::AggregationToStdoutOutputHandler( + string archive_id, + bool count_by_time, + int64_t count_by_time_bucket_size +) + : ::clp_s::search::OutputHandler(count_by_time, false), + m_archive_id{std::move(archive_id)}, + m_count_by_time_bucket_size{count_by_time_bucket_size}, + m_pipeline{reducer::PipelineInputMode::InterStage} { + m_pipeline.add_pipeline_stage(std::make_shared()); +} + +void AggregationToStdoutOutputHandler::write(string_view message) { + m_pipeline.push_record(reducer::EmptyRecord{}); +} + +void AggregationToStdoutOutputHandler::write( + string_view message, + epochtime_t timestamp, + string_view archive_id, + int64_t log_event_idx +) { + int64_t const bucket = (timestamp / m_count_by_time_bucket_size) * m_count_by_time_bucket_size; + m_bucket_counts[bucket] += 1; +} + +ErrorCode AggregationToStdoutOutputHandler::finish() { + // count-by-time results are serialized from the bucket counts; count results come from the + // CountOperator pipeline. `should_output_metadata()` is true only for count-by-time. + std::unique_ptr results; + if (should_output_metadata()) { + results = std::make_unique( + m_bucket_counts, + reducer::CountOperator::cRecordElementKey + ); + } else { + results = m_pipeline.finish(); + } + for (; false == results->done(); results->next()) { + auto& group{results->get()}; + nlohmann::json result; + result[constants::results_cache::search::cArchiveId] = m_archive_id; + + auto const& tags{group.get_tags()}; + if (false == tags.empty()) { + // For count-by-time the group tag is the (stringified) time bucket. + result[constants::results_cache::search::cTimestamp] = std::stoll(tags.front()); + } + + // A count group contains exactly one record holding the count. + auto& record_it{group.record_iter()}; + if (false == record_it.done()) { + result[constants::results_cache::search::cCount] + = record_it.get().get_int64_value(reducer::CountOperator::cRecordElementKey); + } + + std::cout << result.dump() << '\n'; + } + return ErrorCode::ErrorCodeSuccess; +} + CountToResultsCacheOutputHandler::CountToResultsCacheOutputHandler( string const& uri, string const& collection diff --git a/components/core/src/clp_s/OutputHandlerImpl.hpp b/components/core/src/clp_s/OutputHandlerImpl.hpp index 9a741e82f2..80bfa7f810 100644 --- a/components/core/src/clp_s/OutputHandlerImpl.hpp +++ b/components/core/src/clp_s/OutputHandlerImpl.hpp @@ -367,6 +367,46 @@ class CountByTimeToResultsCacheOutputHandler : public ::clp_s::search::OutputHan int64_t m_count_by_time_bucket_size; }; +/** + * Output handler that performs a count or count-by-time aggregation and writes the results to + * standard output as newline-delimited JSON, one object per group. + * + * Mirroring the reducer count handlers: count accumulates through a `reducer::CountOperator` + * pipeline (yielding a single total), while count-by-time accumulates a count per time bucket. Both + * are serialized through a `reducer::RecordGroupIterator` in `finish()`. + */ +class AggregationToStdoutOutputHandler : public ::clp_s::search::OutputHandler { +public: + // Constructors + AggregationToStdoutOutputHandler( + std::string archive_id, + bool count_by_time, + int64_t count_by_time_bucket_size + ); + + // Methods inherited from OutputHandler + void write( + std::string_view message, + epochtime_t timestamp, + std::string_view archive_id, + int64_t log_event_idx + ) override; + + void write(std::string_view message) override; + + /** + * Serializes the aggregation results to stdout as newline-delimited JSON. + * @return ErrorCodeSuccess on success + */ + ErrorCode finish() override; + +private: + std::string m_archive_id; + int64_t m_count_by_time_bucket_size; + reducer::Pipeline m_pipeline; + std::map m_bucket_counts; +}; + /** * Output handler that records all results in a provided vector. */ diff --git a/components/core/src/clp_s/clp-s.cpp b/components/core/src/clp_s/clp-s.cpp index 3875dfa1cb..a4ecf80c6b 100644 --- a/components/core/src/clp_s/clp-s.cpp +++ b/components/core/src/clp_s/clp-s.cpp @@ -295,7 +295,17 @@ bool search_archive( }, [&](CommandLineArguments::StdoutOutputHandlerOptions const& options) -> void { - output_handler = std::make_unique(); + if (false == options.aggregation_type.has_value()) { + output_handler = std::make_unique(); + } else { + output_handler + = std::make_unique( + std::string{archive_reader->get_archive_id()}, + CommandLineArguments::AggregationType::CountByTime + == options.aggregation_type.value(), + options.count_by_time_bucket_size + ); + } } }, command_line_arguments.get_output_handler_options() diff --git a/components/core/src/clp_s/kv_ir_search.cpp b/components/core/src/clp_s/kv_ir_search.cpp index bb610eb0c7..8a232b5561 100644 --- a/components/core/src/clp_s/kv_ir_search.cpp +++ b/components/core/src/clp_s/kv_ir_search.cpp @@ -258,7 +258,13 @@ auto search_kv_ir_stream( bool const results_cache_aggregation_requested = nullptr != results_cache_options && results_cache_options->aggregation_type.has_value(); - if (reducer_aggregation_requested || results_cache_aggregation_requested) { + auto const* stdout_options + = std::get_if(&output_handler_options); + bool const stdout_aggregation_requested + = nullptr != stdout_options && stdout_options->aggregation_type.has_value(); + if (reducer_aggregation_requested || results_cache_aggregation_requested + || stdout_aggregation_requested) + { SPDLOG_ERROR("kv-ir search: Count support is not implemented."); return KvIrSearchError{KvIrSearchErrorEnum::CountSupportNotImplemented}; } From 61f9848932d939926dd627d23093677572c4069d Mon Sep 17 00:00:00 2001 From: davemarco <83603688+davemarco@users.noreply.github.com> Date: Mon, 22 Jun 2026 11:25:07 -0400 Subject: [PATCH 06/56] Address review comments. --- .../core/src/clp_s/OutputHandlerImpl.cpp | 100 +++++++++--------- .../core/src/clp_s/OutputHandlerImpl.hpp | 75 +++++++------ .../core/src/clp_s/archive_constants.hpp | 5 - components/core/src/clp_s/clp-s.cpp | 12 ++- components/core/src/clp_s/kv_ir_search.cpp | 16 +-- 5 files changed, 109 insertions(+), 99 deletions(-) diff --git a/components/core/src/clp_s/OutputHandlerImpl.cpp b/components/core/src/clp_s/OutputHandlerImpl.cpp index 83cd3ab7ac..e840b4aa1f 100644 --- a/components/core/src/clp_s/OutputHandlerImpl.cpp +++ b/components/core/src/clp_s/OutputHandlerImpl.cpp @@ -3,13 +3,12 @@ #include #include #include +#include #include #include #include #include -#include -#include #include #include #include @@ -36,7 +35,7 @@ namespace { * @return The collection. */ template -auto connect_to_results_cache(string const& uri, string const& collection, mongocxx::client& client) +auto connect_to_results_cache(string_view uri, string_view collection, mongocxx::client& client) -> mongocxx::collection { try { auto mongo_uri = mongocxx::uri(uri); @@ -198,18 +197,18 @@ void ResultsCacheOutputHandler::write( } } -CountToReducerOutputHandler::CountToReducerOutputHandler(int reducer_socket_fd) +CountReducerOutputHandler::CountReducerOutputHandler(int reducer_socket_fd) : ::clp_s::search::OutputHandler(false, false), m_reducer_socket_fd(reducer_socket_fd), m_pipeline(reducer::PipelineInputMode::InterStage) { m_pipeline.add_pipeline_stage(std::make_shared()); } -void CountToReducerOutputHandler::write(string_view message) { +auto CountReducerOutputHandler::write(string_view message) -> void { m_pipeline.push_record(reducer::EmptyRecord{}); } -ErrorCode CountToReducerOutputHandler::finish() { +auto CountReducerOutputHandler::finish() -> ErrorCode { if (false == reducer::send_pipeline_results(m_reducer_socket_fd, std::move(m_pipeline.finish()))) { @@ -218,7 +217,7 @@ ErrorCode CountToReducerOutputHandler::finish() { return ErrorCode::ErrorCodeSuccess; } -ErrorCode CountByTimeToReducerOutputHandler::finish() { +auto CountByTimeReducerOutputHandler::finish() -> ErrorCode { if (false == reducer::send_pipeline_results( m_reducer_socket_fd, @@ -233,82 +232,79 @@ ErrorCode CountByTimeToReducerOutputHandler::finish() { return ErrorCode::ErrorCodeSuccess; } -CountToResultsCacheOutputHandler::CountToResultsCacheOutputHandler( - string const& uri, - string const& collection +CountResultsCacheOutputHandler::CountResultsCacheOutputHandler( + string_view uri, + string_view collection, + string archive_id ) - : ::clp_s::search::OutputHandler(false, false) { + : ::clp_s::search::OutputHandler{false, false}, + m_archive_id{std::move(archive_id)} { m_collection = connect_to_results_cache(uri, collection, m_client); } -ErrorCode CountToResultsCacheOutputHandler::finish() { +auto CountResultsCacheOutputHandler::finish() -> ErrorCode { if (0 == m_count) { return ErrorCode::ErrorCodeSuccess; } try { - auto const filter = bsoncxx::builder::basic::make_document( - bsoncxx::builder::basic::kvp( - constants::results_cache::cDocId, - constants::results_cache::search::cCount + m_collection.insert_one( + bsoncxx::builder::basic::make_document( + bsoncxx::builder::basic::kvp( + constants::results_cache::search::cArchiveId, + m_archive_id + ), + bsoncxx::builder::basic::kvp( + constants::results_cache::search::cCount, + m_count + ) ) ); - auto const inc_count = bsoncxx::builder::basic::make_document( - bsoncxx::builder::basic::kvp(constants::results_cache::search::cCount, m_count) - ); - auto const update = bsoncxx::builder::basic::make_document( - bsoncxx::builder::basic::kvp("$inc", inc_count) - ); - m_collection - .update_one(filter.view(), update.view(), mongocxx::options::update{}.upsert(true)); } catch (mongocxx::exception const& e) { return ErrorCode::ErrorCodeFailureDbBulkWrite; } return ErrorCode::ErrorCodeSuccess; } -CountByTimeToResultsCacheOutputHandler::CountByTimeToResultsCacheOutputHandler( - string const& uri, - string const& collection, +CountByTimeResultsCacheOutputHandler::CountByTimeResultsCacheOutputHandler( + string_view uri, + string_view collection, + string archive_id, int64_t count_by_time_bucket_size ) - : ::clp_s::search::OutputHandler(true, false), - m_count_by_time_bucket_size(count_by_time_bucket_size) { + : ::clp_s::search::OutputHandler{true, false}, + m_archive_id{std::move(archive_id)}, + m_count_by_time_bucket_size{count_by_time_bucket_size} { m_collection = connect_to_results_cache(uri, collection, m_client); } -ErrorCode CountByTimeToResultsCacheOutputHandler::finish() { +auto CountByTimeResultsCacheOutputHandler::finish() -> ErrorCode { if (m_bucket_counts.empty()) { return ErrorCode::ErrorCodeSuccess; } - // The bucket timestamp is also stored in a `timestamp` field so that the documents match the - // `{timestamp, count}` shape the reducer writes. try { - auto bulk_write = m_collection.create_bulk_write(); + std::vector documents; + documents.reserve(m_bucket_counts.size()); for (auto const& [bucket_timestamp, count] : m_bucket_counts) { - auto filter = bsoncxx::builder::basic::make_document( - bsoncxx::builder::basic::kvp(constants::results_cache::cDocId, bucket_timestamp) - ); - auto const inc_count = bsoncxx::builder::basic::make_document( - bsoncxx::builder::basic::kvp(constants::results_cache::search::cCount, count) - ); - auto const set_timestamp_on_insert = bsoncxx::builder::basic::make_document( - bsoncxx::builder::basic::kvp( - constants::results_cache::search::cTimestamp, - bucket_timestamp + documents.emplace_back( + bsoncxx::builder::basic::make_document( + bsoncxx::builder::basic::kvp( + constants::results_cache::search::cArchiveId, + m_archive_id + ), + bsoncxx::builder::basic::kvp( + constants::results_cache::search::cTimestamp, + bucket_timestamp + ), + bsoncxx::builder::basic::kvp( + constants::results_cache::search::cCount, + count + ) ) ); - auto update = bsoncxx::builder::basic::make_document( - bsoncxx::builder::basic::kvp("$inc", inc_count), - bsoncxx::builder::basic::kvp("$setOnInsert", set_timestamp_on_insert) - ); - - mongocxx::model::update_one update_op{std::move(filter), std::move(update)}; - update_op.upsert(true); - bulk_write.append(update_op); } - bulk_write.execute(); + m_collection.insert_many(documents); } catch (mongocxx::exception const& e) { return ErrorCode::ErrorCodeFailureDbBulkWrite; } diff --git a/components/core/src/clp_s/OutputHandlerImpl.hpp b/components/core/src/clp_s/OutputHandlerImpl.hpp index 9a741e82f2..7b6d4ad3bc 100644 --- a/components/core/src/clp_s/OutputHandlerImpl.hpp +++ b/components/core/src/clp_s/OutputHandlerImpl.hpp @@ -215,29 +215,31 @@ class ResultsCacheOutputHandler : public ::clp_s::search::OutputHandler { /** * Output handler that performs a count aggregation and sends the results to a reducer. */ -class CountToReducerOutputHandler : public ::clp_s::search::OutputHandler { +class CountReducerOutputHandler : public ::clp_s::search::OutputHandler { public: // Constructors - CountToReducerOutputHandler(int reducer_socket_fd); + CountReducerOutputHandler(int reducer_socket_fd); - // Methods inherited from OutputHandler - void write( + // Methods implementing OutputHandler + auto write( std::string_view message, epochtime_t timestamp, std::string_view archive_id, int64_t log_event_idx - ) override {} + ) -> void override {} - void write(std::string_view message) override; + auto write(std::string_view message) -> void override; + // Methods overriding OutputHandler /** * Flushes the count. * @return ErrorCodeSuccess on success * @return ErrorCodeFailureNetwork on network error */ - ErrorCode finish() override; + auto finish() -> ErrorCode override; private: + // Data members int m_reducer_socket_fd; reducer::Pipeline m_pipeline; }; @@ -246,35 +248,37 @@ class CountToReducerOutputHandler : public ::clp_s::search::OutputHandler { * Output handler that performs a count aggregation bucketed by time and sends the results to a * reducer. */ -class CountByTimeToReducerOutputHandler : public ::clp_s::search::OutputHandler { +class CountByTimeReducerOutputHandler : public ::clp_s::search::OutputHandler { public: // Constructors - CountByTimeToReducerOutputHandler(int reducer_socket_fd, int64_t count_by_time_bucket_size) + CountByTimeReducerOutputHandler(int reducer_socket_fd, int64_t count_by_time_bucket_size) : search::OutputHandler{true, false}, m_reducer_socket_fd{reducer_socket_fd}, m_count_by_time_bucket_size{count_by_time_bucket_size} {} - // Methods inherited from OutputHandler - void write( + // Methods implementing OutputHandler + auto write( std::string_view message, epochtime_t timestamp, std::string_view archive_id, int64_t log_event_idx - ) override { + ) -> void override { int64_t bucket = (timestamp / m_count_by_time_bucket_size) * m_count_by_time_bucket_size; m_bucket_counts[bucket] += 1; } - void write(std::string_view message) override {} + auto write(std::string_view message) -> void override {} + // Methods overriding OutputHandler /** * Flushes the counts. * @return ErrorCodeSuccess on success * @return ErrorCodeFailureNetwork on network error */ - ErrorCode finish() override; + auto finish() -> ErrorCode override; private: + // Data members int m_reducer_socket_fd; std::map m_bucket_counts; int64_t m_count_by_time_bucket_size; @@ -283,7 +287,7 @@ class CountByTimeToReducerOutputHandler : public ::clp_s::search::OutputHandler /** * Output handler that performs a count aggregation and writes the results to the results cache. */ -class CountToResultsCacheOutputHandler : public ::clp_s::search::OutputHandler { +class CountResultsCacheOutputHandler : public ::clp_s::search::OutputHandler { public: // Types class OperationFailed : public TraceableException { @@ -294,28 +298,35 @@ class CountToResultsCacheOutputHandler : public ::clp_s::search::OutputHandler { }; // Constructors - CountToResultsCacheOutputHandler(std::string const& uri, std::string const& collection); + CountResultsCacheOutputHandler( + std::string_view uri, + std::string_view collection, + std::string archive_id + ); - // Methods inherited from OutputHandler - void write( + // Methods implementing OutputHandler + auto write( std::string_view message, epochtime_t timestamp, std::string_view archive_id, int64_t log_event_idx - ) override {} + ) -> void override {} - void write(std::string_view message) override { m_count += 1; } + auto write(std::string_view message) -> void override { m_count += 1; } + // Methods overriding OutputHandler /** * Flushes the count. * @return ErrorCodeSuccess on success * @return ErrorCodeFailureDbBulkWrite on database error */ - ErrorCode finish() override; + auto finish() -> ErrorCode override; private: + // Data members mongocxx::client m_client; mongocxx::collection m_collection; + std::string m_archive_id; int64_t m_count{}; }; @@ -323,7 +334,7 @@ class CountToResultsCacheOutputHandler : public ::clp_s::search::OutputHandler { * Output handler that performs a count aggregation bucketed by time and writes the results to the * results cache. */ -class CountByTimeToResultsCacheOutputHandler : public ::clp_s::search::OutputHandler { +class CountByTimeResultsCacheOutputHandler : public ::clp_s::search::OutputHandler { public: // Types class OperationFailed : public TraceableException { @@ -334,35 +345,39 @@ class CountByTimeToResultsCacheOutputHandler : public ::clp_s::search::OutputHan }; // Constructors - CountByTimeToResultsCacheOutputHandler( - std::string const& uri, - std::string const& collection, + CountByTimeResultsCacheOutputHandler( + std::string_view uri, + std::string_view collection, + std::string archive_id, int64_t count_by_time_bucket_size ); - // Methods inherited from OutputHandler - void write( + // Methods implementing OutputHandler + auto write( std::string_view message, epochtime_t timestamp, std::string_view archive_id, int64_t log_event_idx - ) override { + ) -> void override { int64_t bucket = (timestamp / m_count_by_time_bucket_size) * m_count_by_time_bucket_size; m_bucket_counts[bucket] += 1; } - void write(std::string_view message) override {} + auto write(std::string_view message) -> void override {} + // Methods overriding OutputHandler /** * Flushes the counts. * @return ErrorCodeSuccess on success * @return ErrorCodeFailureDbBulkWrite on database error */ - ErrorCode finish() override; + auto finish() -> ErrorCode override; private: + // Data members mongocxx::client m_client; mongocxx::collection m_collection; + std::string m_archive_id; std::map m_bucket_counts; int64_t m_count_by_time_bucket_size; }; diff --git a/components/core/src/clp_s/archive_constants.hpp b/components/core/src/clp_s/archive_constants.hpp index 212d593d5e..5ccc9d4112 100644 --- a/components/core/src/clp_s/archive_constants.hpp +++ b/components/core/src/clp_s/archive_constants.hpp @@ -44,11 +44,6 @@ constexpr std::string_view cFileSplitNumber{"_file_split_number"}; constexpr std::string_view cArchiveCreatorId{"_archive_creator_id"}; } // namespace range_index -namespace results_cache { -// MongoDB's primary-key field, which is backed by a built-in unique index. -constexpr char cDocId[]{"_id"}; -} // namespace results_cache - namespace results_cache::decompression { constexpr char cPath[]{"path"}; constexpr char cStreamId[]{"stream_id"}; diff --git a/components/core/src/clp_s/clp-s.cpp b/components/core/src/clp_s/clp-s.cpp index 3875dfa1cb..e902ce179e 100644 --- a/components/core/src/clp_s/clp-s.cpp +++ b/components/core/src/clp_s/clp-s.cpp @@ -248,14 +248,14 @@ bool search_archive( if (CommandLineArguments::AggregationType::Count == options.aggregation_type) { output_handler - = std::make_unique( + = std::make_unique( reducer_socket_fd ); } else if (CommandLineArguments::AggregationType::CountByTime == options.aggregation_type) { output_handler = std::make_unique< - clp_s::CountByTimeToReducerOutputHandler + clp_s::CountByTimeReducerOutputHandler >(reducer_socket_fd, options.count_by_time_bucket_size); } else { SPDLOG_ERROR("Unhandled aggregation type."); @@ -276,17 +276,19 @@ bool search_archive( == options.aggregation_type.value()) { output_handler - = std::make_unique( + = std::make_unique( options.uri, - options.collection + options.collection, + std::string{archive_reader->get_archive_id()} ); } else if (CommandLineArguments::AggregationType::CountByTime == options.aggregation_type.value()) { output_handler = std::make_unique< - clp_s::CountByTimeToResultsCacheOutputHandler + clp_s::CountByTimeResultsCacheOutputHandler >(options.uri, options.collection, + std::string{archive_reader->get_archive_id()}, options.count_by_time_bucket_size); } else { SPDLOG_ERROR("Unhandled aggregation type."); diff --git a/components/core/src/clp_s/kv_ir_search.cpp b/components/core/src/clp_s/kv_ir_search.cpp index bb610eb0c7..7ec61bf236 100644 --- a/components/core/src/clp_s/kv_ir_search.cpp +++ b/components/core/src/clp_s/kv_ir_search.cpp @@ -247,19 +247,21 @@ auto search_kv_ir_stream( } auto const& output_handler_options{command_line_arguments.get_output_handler_options()}; - bool const reducer_aggregation_requested - = std::holds_alternative( + auto const reducer_aggregation_requested{ + std::holds_alternative( output_handler_options - ); + ) + }; auto const* results_cache_options = std::get_if( &output_handler_options ); - bool const results_cache_aggregation_requested - = nullptr != results_cache_options - && results_cache_options->aggregation_type.has_value(); + auto const results_cache_aggregation_requested{ + nullptr != results_cache_options + && results_cache_options->aggregation_type.has_value() + }; if (reducer_aggregation_requested || results_cache_aggregation_requested) { - SPDLOG_ERROR("kv-ir search: Count support is not implemented."); + SPDLOG_ERROR("kv-ir search: Aggregation support is not implemented."); return KvIrSearchError{KvIrSearchErrorEnum::CountSupportNotImplemented}; } From a0f3622ecac5f44aa6a4c1911ee91bb610951e40 Mon Sep 17 00:00:00 2001 From: davemarco <83603688+davemarco@users.noreply.github.com> Date: Mon, 22 Jun 2026 11:52:43 -0400 Subject: [PATCH 07/56] Use `string const&` for results-cache count handler uri/collection params. --- components/core/src/clp_s/OutputHandlerImpl.cpp | 12 ++++++------ components/core/src/clp_s/OutputHandlerImpl.hpp | 8 ++++---- 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/components/core/src/clp_s/OutputHandlerImpl.cpp b/components/core/src/clp_s/OutputHandlerImpl.cpp index e840b4aa1f..dca9b0e7fe 100644 --- a/components/core/src/clp_s/OutputHandlerImpl.cpp +++ b/components/core/src/clp_s/OutputHandlerImpl.cpp @@ -35,10 +35,10 @@ namespace { * @return The collection. */ template -auto connect_to_results_cache(string_view uri, string_view collection, mongocxx::client& client) +auto connect_to_results_cache(string const& uri, string const& collection, mongocxx::client& client) -> mongocxx::collection { try { - auto mongo_uri = mongocxx::uri(uri); + auto mongo_uri = mongocxx::uri{uri}; client = mongocxx::client(mongo_uri); return client[mongo_uri.database()][collection]; } catch (mongocxx::exception const& e) { @@ -233,8 +233,8 @@ auto CountByTimeReducerOutputHandler::finish() -> ErrorCode { } CountResultsCacheOutputHandler::CountResultsCacheOutputHandler( - string_view uri, - string_view collection, + string const& uri, + string const& collection, string archive_id ) : ::clp_s::search::OutputHandler{false, false}, @@ -267,8 +267,8 @@ auto CountResultsCacheOutputHandler::finish() -> ErrorCode { } CountByTimeResultsCacheOutputHandler::CountByTimeResultsCacheOutputHandler( - string_view uri, - string_view collection, + string const& uri, + string const& collection, string archive_id, int64_t count_by_time_bucket_size ) diff --git a/components/core/src/clp_s/OutputHandlerImpl.hpp b/components/core/src/clp_s/OutputHandlerImpl.hpp index 7b6d4ad3bc..c24d01a90d 100644 --- a/components/core/src/clp_s/OutputHandlerImpl.hpp +++ b/components/core/src/clp_s/OutputHandlerImpl.hpp @@ -299,8 +299,8 @@ class CountResultsCacheOutputHandler : public ::clp_s::search::OutputHandler { // Constructors CountResultsCacheOutputHandler( - std::string_view uri, - std::string_view collection, + std::string const& uri, + std::string const& collection, std::string archive_id ); @@ -346,8 +346,8 @@ class CountByTimeResultsCacheOutputHandler : public ::clp_s::search::OutputHandl // Constructors CountByTimeResultsCacheOutputHandler( - std::string_view uri, - std::string_view collection, + std::string const& uri, + std::string const& collection, std::string archive_id, int64_t count_by_time_bucket_size ); From 1c282e6e4bf15497dca56c1e702f9479dd2da6bd Mon Sep 17 00:00:00 2001 From: davemarco <83603688+davemarco@users.noreply.github.com> Date: Mon, 22 Jun 2026 13:38:45 -0400 Subject: [PATCH 08/56] Add forward declaration for results-cache helper and apply clang-format. --- components/core/src/clp_s/OutputHandlerImpl.cpp | 4 ++++ components/core/src/clp_s/clp-s.cpp | 15 ++++++++------- components/core/src/clp_s/kv_ir_search.cpp | 3 +-- 3 files changed, 13 insertions(+), 9 deletions(-) diff --git a/components/core/src/clp_s/OutputHandlerImpl.cpp b/components/core/src/clp_s/OutputHandlerImpl.cpp index dca9b0e7fe..a10d0de8d6 100644 --- a/components/core/src/clp_s/OutputHandlerImpl.cpp +++ b/components/core/src/clp_s/OutputHandlerImpl.cpp @@ -35,6 +35,10 @@ namespace { * @return The collection. */ template +auto connect_to_results_cache(string const& uri, string const& collection, mongocxx::client& client) + -> mongocxx::collection; + +template auto connect_to_results_cache(string const& uri, string const& collection, mongocxx::client& client) -> mongocxx::collection { try { diff --git a/components/core/src/clp_s/clp-s.cpp b/components/core/src/clp_s/clp-s.cpp index e902ce179e..5b491ce84d 100644 --- a/components/core/src/clp_s/clp-s.cpp +++ b/components/core/src/clp_s/clp-s.cpp @@ -247,16 +247,17 @@ bool search_archive( -> void { if (CommandLineArguments::AggregationType::Count == options.aggregation_type) { - output_handler - = std::make_unique( - reducer_socket_fd - ); + output_handler = std::make_unique( + reducer_socket_fd + ); } else if (CommandLineArguments::AggregationType::CountByTime == options.aggregation_type) { - output_handler = std::make_unique< - clp_s::CountByTimeReducerOutputHandler - >(reducer_socket_fd, options.count_by_time_bucket_size); + output_handler + = std::make_unique( + reducer_socket_fd, + options.count_by_time_bucket_size + ); } else { SPDLOG_ERROR("Unhandled aggregation type."); output_handler = nullptr; diff --git a/components/core/src/clp_s/kv_ir_search.cpp b/components/core/src/clp_s/kv_ir_search.cpp index 7ec61bf236..468bd418ae 100644 --- a/components/core/src/clp_s/kv_ir_search.cpp +++ b/components/core/src/clp_s/kv_ir_search.cpp @@ -257,8 +257,7 @@ auto search_kv_ir_stream( &output_handler_options ); auto const results_cache_aggregation_requested{ - nullptr != results_cache_options - && results_cache_options->aggregation_type.has_value() + nullptr != results_cache_options && results_cache_options->aggregation_type.has_value() }; if (reducer_aggregation_requested || results_cache_aggregation_requested) { SPDLOG_ERROR("kv-ir search: Aggregation support is not implemented."); From 228e31ac0bfbbdbe7a5dac69ea688e0bb896218b Mon Sep 17 00:00:00 2001 From: davemarco <83603688+davemarco@users.noreply.github.com> Date: Mon, 22 Jun 2026 13:49:02 -0400 Subject: [PATCH 09/56] Add ms unit to count_by_time_bucket_size field name. --- .../core/src/clp_s/CommandLineArguments.cpp | 12 ++++++------ .../core/src/clp_s/CommandLineArguments.hpp | 8 ++++---- components/core/src/clp_s/OutputHandlerImpl.cpp | 4 ++-- components/core/src/clp_s/OutputHandlerImpl.hpp | 16 +++++++++------- components/core/src/clp_s/clp-s.cpp | 4 ++-- 5 files changed, 23 insertions(+), 21 deletions(-) diff --git a/components/core/src/clp_s/CommandLineArguments.cpp b/components/core/src/clp_s/CommandLineArguments.cpp index f3ff521fb0..189f8139be 100644 --- a/components/core/src/clp_s/CommandLineArguments.cpp +++ b/components/core/src/clp_s/CommandLineArguments.cpp @@ -809,7 +809,7 @@ CommandLineArguments::parse_arguments(int argc, char const** argv) { )( "count-by-time", po::value( - &reducer_options.count_by_time_bucket_size + &reducer_options.count_by_time_bucket_size_ms )->value_name("SIZE"), "Count the number of results in each time span of the given size (ms)" ); @@ -852,7 +852,7 @@ CommandLineArguments::parse_arguments(int argc, char const** argv) { )( "count-by-time", po::value( - &results_cache_options.count_by_time_bucket_size + &results_cache_options.count_by_time_bucket_size_ms )->value_name("SIZE"), "Count the number of results in each time span of the given size (ms)" ); @@ -1107,7 +1107,7 @@ void CommandLineArguments::parse_network_dest_output_handler_options( auto CommandLineArguments::parse_aggregation_options( po::variables_map const& parsed_options, - int64_t count_by_time_bucket_size + int64_t count_by_time_bucket_size_ms ) -> std::optional { std::optional aggregation_type; if (parsed_options.count("count")) { @@ -1120,7 +1120,7 @@ auto CommandLineArguments::parse_aggregation_options( ); } - if (count_by_time_bucket_size <= 0) { + if (count_by_time_bucket_size_ms <= 0) { throw std::invalid_argument("Value for count-by-time must be greater than zero."); } @@ -1160,7 +1160,7 @@ void CommandLineArguments::parse_reducer_output_handler_options( } auto const aggregation_type{ - parse_aggregation_options(parsed_options, reducer_options.count_by_time_bucket_size) + parse_aggregation_options(parsed_options, reducer_options.count_by_time_bucket_size_ms) }; if (false == aggregation_type.has_value()) { throw std::invalid_argument( @@ -1203,7 +1203,7 @@ void CommandLineArguments::parse_results_cache_output_handler_options( results_cache_options.aggregation_type = parse_aggregation_options( parsed_options, - results_cache_options.count_by_time_bucket_size + results_cache_options.count_by_time_bucket_size_ms ); } diff --git a/components/core/src/clp_s/CommandLineArguments.hpp b/components/core/src/clp_s/CommandLineArguments.hpp index d17302b193..ce44b4cbf4 100644 --- a/components/core/src/clp_s/CommandLineArguments.hpp +++ b/components/core/src/clp_s/CommandLineArguments.hpp @@ -43,7 +43,7 @@ class CommandLineArguments { uint64_t batch_size{1000}; uint64_t max_num_results{1000}; std::optional aggregation_type; - int64_t count_by_time_bucket_size{}; // Milliseconds + int64_t count_by_time_bucket_size_ms{}; }; struct FileOutputHandlerOptions { @@ -60,7 +60,7 @@ class CommandLineArguments { int port{-1}; reducer::job_id_t job_id{-1}; AggregationType aggregation_type{AggregationType::Count}; - int64_t count_by_time_bucket_size{}; // Milliseconds + int64_t count_by_time_bucket_size_ms{}; }; struct StdoutOutputHandlerOptions {}; @@ -162,13 +162,13 @@ class CommandLineArguments { * Validates the aggregation options (count and count-by-time) for output handlers that * support aggregations. * @param parsed_options - * @param count_by_time_bucket_size The parsed value of the count-by-time option; only + * @param count_by_time_bucket_size_ms The parsed value of the count-by-time option; only * validated when that option was specified. * @return The requested aggregation type, or std::nullopt if no aggregation was requested. */ [[nodiscard]] static auto parse_aggregation_options( boost::program_options::variables_map const& parsed_options, - int64_t count_by_time_bucket_size + int64_t count_by_time_bucket_size_ms ) -> std::optional; /** diff --git a/components/core/src/clp_s/OutputHandlerImpl.cpp b/components/core/src/clp_s/OutputHandlerImpl.cpp index a10d0de8d6..bbaee32958 100644 --- a/components/core/src/clp_s/OutputHandlerImpl.cpp +++ b/components/core/src/clp_s/OutputHandlerImpl.cpp @@ -274,11 +274,11 @@ CountByTimeResultsCacheOutputHandler::CountByTimeResultsCacheOutputHandler( string const& uri, string const& collection, string archive_id, - int64_t count_by_time_bucket_size + int64_t count_by_time_bucket_size_ms ) : ::clp_s::search::OutputHandler{true, false}, m_archive_id{std::move(archive_id)}, - m_count_by_time_bucket_size{count_by_time_bucket_size} { + m_count_by_time_bucket_size_ms{count_by_time_bucket_size_ms} { m_collection = connect_to_results_cache(uri, collection, m_client); } diff --git a/components/core/src/clp_s/OutputHandlerImpl.hpp b/components/core/src/clp_s/OutputHandlerImpl.hpp index c24d01a90d..cf0df29c52 100644 --- a/components/core/src/clp_s/OutputHandlerImpl.hpp +++ b/components/core/src/clp_s/OutputHandlerImpl.hpp @@ -251,10 +251,10 @@ class CountReducerOutputHandler : public ::clp_s::search::OutputHandler { class CountByTimeReducerOutputHandler : public ::clp_s::search::OutputHandler { public: // Constructors - CountByTimeReducerOutputHandler(int reducer_socket_fd, int64_t count_by_time_bucket_size) + CountByTimeReducerOutputHandler(int reducer_socket_fd, int64_t count_by_time_bucket_size_ms) : search::OutputHandler{true, false}, m_reducer_socket_fd{reducer_socket_fd}, - m_count_by_time_bucket_size{count_by_time_bucket_size} {} + m_count_by_time_bucket_size_ms{count_by_time_bucket_size_ms} {} // Methods implementing OutputHandler auto write( @@ -263,7 +263,8 @@ class CountByTimeReducerOutputHandler : public ::clp_s::search::OutputHandler { std::string_view archive_id, int64_t log_event_idx ) -> void override { - int64_t bucket = (timestamp / m_count_by_time_bucket_size) * m_count_by_time_bucket_size; + int64_t bucket + = (timestamp / m_count_by_time_bucket_size_ms) * m_count_by_time_bucket_size_ms; m_bucket_counts[bucket] += 1; } @@ -281,7 +282,7 @@ class CountByTimeReducerOutputHandler : public ::clp_s::search::OutputHandler { // Data members int m_reducer_socket_fd; std::map m_bucket_counts; - int64_t m_count_by_time_bucket_size; + int64_t m_count_by_time_bucket_size_ms; }; /** @@ -349,7 +350,7 @@ class CountByTimeResultsCacheOutputHandler : public ::clp_s::search::OutputHandl std::string const& uri, std::string const& collection, std::string archive_id, - int64_t count_by_time_bucket_size + int64_t count_by_time_bucket_size_ms ); // Methods implementing OutputHandler @@ -359,7 +360,8 @@ class CountByTimeResultsCacheOutputHandler : public ::clp_s::search::OutputHandl std::string_view archive_id, int64_t log_event_idx ) -> void override { - int64_t bucket = (timestamp / m_count_by_time_bucket_size) * m_count_by_time_bucket_size; + int64_t bucket + = (timestamp / m_count_by_time_bucket_size_ms) * m_count_by_time_bucket_size_ms; m_bucket_counts[bucket] += 1; } @@ -379,7 +381,7 @@ class CountByTimeResultsCacheOutputHandler : public ::clp_s::search::OutputHandl mongocxx::collection m_collection; std::string m_archive_id; std::map m_bucket_counts; - int64_t m_count_by_time_bucket_size; + int64_t m_count_by_time_bucket_size_ms; }; /** diff --git a/components/core/src/clp_s/clp-s.cpp b/components/core/src/clp_s/clp-s.cpp index 5b491ce84d..e358852482 100644 --- a/components/core/src/clp_s/clp-s.cpp +++ b/components/core/src/clp_s/clp-s.cpp @@ -256,7 +256,7 @@ bool search_archive( output_handler = std::make_unique( reducer_socket_fd, - options.count_by_time_bucket_size + options.count_by_time_bucket_size_ms ); } else { SPDLOG_ERROR("Unhandled aggregation type."); @@ -290,7 +290,7 @@ bool search_archive( >(options.uri, options.collection, std::string{archive_reader->get_archive_id()}, - options.count_by_time_bucket_size); + options.count_by_time_bucket_size_ms); } else { SPDLOG_ERROR("Unhandled aggregation type."); output_handler = nullptr; From d0f097becb6dffd921080a48ec62890b15e2139d Mon Sep 17 00:00:00 2001 From: davemarco <83603688+davemarco@users.noreply.github.com> Date: Mon, 22 Jun 2026 11:25:07 -0400 Subject: [PATCH 10/56] Address review comments. --- .../core/src/clp_s/OutputHandlerImpl.cpp | 100 +++++++++--------- .../core/src/clp_s/OutputHandlerImpl.hpp | 75 +++++++------ .../core/src/clp_s/archive_constants.hpp | 5 - components/core/src/clp_s/clp-s.cpp | 12 ++- components/core/src/clp_s/kv_ir_search.cpp | 21 ++-- 5 files changed, 112 insertions(+), 101 deletions(-) diff --git a/components/core/src/clp_s/OutputHandlerImpl.cpp b/components/core/src/clp_s/OutputHandlerImpl.cpp index 75aebe84bf..85c5b8bf4a 100644 --- a/components/core/src/clp_s/OutputHandlerImpl.cpp +++ b/components/core/src/clp_s/OutputHandlerImpl.cpp @@ -3,13 +3,12 @@ #include #include #include +#include #include #include #include #include -#include -#include #include #include #include @@ -39,7 +38,7 @@ namespace { * @return The collection. */ template -auto connect_to_results_cache(string const& uri, string const& collection, mongocxx::client& client) +auto connect_to_results_cache(string_view uri, string_view collection, mongocxx::client& client) -> mongocxx::collection { try { auto mongo_uri = mongocxx::uri(uri); @@ -201,18 +200,18 @@ void ResultsCacheOutputHandler::write( } } -CountToReducerOutputHandler::CountToReducerOutputHandler(int reducer_socket_fd) +CountReducerOutputHandler::CountReducerOutputHandler(int reducer_socket_fd) : ::clp_s::search::OutputHandler(false, false), m_reducer_socket_fd(reducer_socket_fd), m_pipeline(reducer::PipelineInputMode::InterStage) { m_pipeline.add_pipeline_stage(std::make_shared()); } -void CountToReducerOutputHandler::write(string_view message) { +auto CountReducerOutputHandler::write(string_view message) -> void { m_pipeline.push_record(reducer::EmptyRecord{}); } -ErrorCode CountToReducerOutputHandler::finish() { +auto CountReducerOutputHandler::finish() -> ErrorCode { if (false == reducer::send_pipeline_results(m_reducer_socket_fd, std::move(m_pipeline.finish()))) { @@ -221,7 +220,7 @@ ErrorCode CountToReducerOutputHandler::finish() { return ErrorCode::ErrorCodeSuccess; } -ErrorCode CountByTimeToReducerOutputHandler::finish() { +auto CountByTimeReducerOutputHandler::finish() -> ErrorCode { if (false == reducer::send_pipeline_results( m_reducer_socket_fd, @@ -297,82 +296,79 @@ ErrorCode AggregationToStdoutOutputHandler::finish() { return ErrorCode::ErrorCodeSuccess; } -CountToResultsCacheOutputHandler::CountToResultsCacheOutputHandler( - string const& uri, - string const& collection +CountResultsCacheOutputHandler::CountResultsCacheOutputHandler( + string_view uri, + string_view collection, + string archive_id ) - : ::clp_s::search::OutputHandler(false, false) { + : ::clp_s::search::OutputHandler{false, false}, + m_archive_id{std::move(archive_id)} { m_collection = connect_to_results_cache(uri, collection, m_client); } -ErrorCode CountToResultsCacheOutputHandler::finish() { +auto CountResultsCacheOutputHandler::finish() -> ErrorCode { if (0 == m_count) { return ErrorCode::ErrorCodeSuccess; } try { - auto const filter = bsoncxx::builder::basic::make_document( - bsoncxx::builder::basic::kvp( - constants::results_cache::cDocId, - constants::results_cache::search::cCount + m_collection.insert_one( + bsoncxx::builder::basic::make_document( + bsoncxx::builder::basic::kvp( + constants::results_cache::search::cArchiveId, + m_archive_id + ), + bsoncxx::builder::basic::kvp( + constants::results_cache::search::cCount, + m_count + ) ) ); - auto const inc_count = bsoncxx::builder::basic::make_document( - bsoncxx::builder::basic::kvp(constants::results_cache::search::cCount, m_count) - ); - auto const update = bsoncxx::builder::basic::make_document( - bsoncxx::builder::basic::kvp("$inc", inc_count) - ); - m_collection - .update_one(filter.view(), update.view(), mongocxx::options::update{}.upsert(true)); } catch (mongocxx::exception const& e) { return ErrorCode::ErrorCodeFailureDbBulkWrite; } return ErrorCode::ErrorCodeSuccess; } -CountByTimeToResultsCacheOutputHandler::CountByTimeToResultsCacheOutputHandler( - string const& uri, - string const& collection, +CountByTimeResultsCacheOutputHandler::CountByTimeResultsCacheOutputHandler( + string_view uri, + string_view collection, + string archive_id, int64_t count_by_time_bucket_size ) - : ::clp_s::search::OutputHandler(true, false), - m_count_by_time_bucket_size(count_by_time_bucket_size) { + : ::clp_s::search::OutputHandler{true, false}, + m_archive_id{std::move(archive_id)}, + m_count_by_time_bucket_size{count_by_time_bucket_size} { m_collection = connect_to_results_cache(uri, collection, m_client); } -ErrorCode CountByTimeToResultsCacheOutputHandler::finish() { +auto CountByTimeResultsCacheOutputHandler::finish() -> ErrorCode { if (m_bucket_counts.empty()) { return ErrorCode::ErrorCodeSuccess; } - // The bucket timestamp is also stored in a `timestamp` field so that the documents match the - // `{timestamp, count}` shape the reducer writes. try { - auto bulk_write = m_collection.create_bulk_write(); + std::vector documents; + documents.reserve(m_bucket_counts.size()); for (auto const& [bucket_timestamp, count] : m_bucket_counts) { - auto filter = bsoncxx::builder::basic::make_document( - bsoncxx::builder::basic::kvp(constants::results_cache::cDocId, bucket_timestamp) - ); - auto const inc_count = bsoncxx::builder::basic::make_document( - bsoncxx::builder::basic::kvp(constants::results_cache::search::cCount, count) - ); - auto const set_timestamp_on_insert = bsoncxx::builder::basic::make_document( - bsoncxx::builder::basic::kvp( - constants::results_cache::search::cTimestamp, - bucket_timestamp + documents.emplace_back( + bsoncxx::builder::basic::make_document( + bsoncxx::builder::basic::kvp( + constants::results_cache::search::cArchiveId, + m_archive_id + ), + bsoncxx::builder::basic::kvp( + constants::results_cache::search::cTimestamp, + bucket_timestamp + ), + bsoncxx::builder::basic::kvp( + constants::results_cache::search::cCount, + count + ) ) ); - auto update = bsoncxx::builder::basic::make_document( - bsoncxx::builder::basic::kvp("$inc", inc_count), - bsoncxx::builder::basic::kvp("$setOnInsert", set_timestamp_on_insert) - ); - - mongocxx::model::update_one update_op{std::move(filter), std::move(update)}; - update_op.upsert(true); - bulk_write.append(update_op); } - bulk_write.execute(); + m_collection.insert_many(documents); } catch (mongocxx::exception const& e) { return ErrorCode::ErrorCodeFailureDbBulkWrite; } diff --git a/components/core/src/clp_s/OutputHandlerImpl.hpp b/components/core/src/clp_s/OutputHandlerImpl.hpp index 80bfa7f810..7631dec223 100644 --- a/components/core/src/clp_s/OutputHandlerImpl.hpp +++ b/components/core/src/clp_s/OutputHandlerImpl.hpp @@ -215,29 +215,31 @@ class ResultsCacheOutputHandler : public ::clp_s::search::OutputHandler { /** * Output handler that performs a count aggregation and sends the results to a reducer. */ -class CountToReducerOutputHandler : public ::clp_s::search::OutputHandler { +class CountReducerOutputHandler : public ::clp_s::search::OutputHandler { public: // Constructors - CountToReducerOutputHandler(int reducer_socket_fd); + CountReducerOutputHandler(int reducer_socket_fd); - // Methods inherited from OutputHandler - void write( + // Methods implementing OutputHandler + auto write( std::string_view message, epochtime_t timestamp, std::string_view archive_id, int64_t log_event_idx - ) override {} + ) -> void override {} - void write(std::string_view message) override; + auto write(std::string_view message) -> void override; + // Methods overriding OutputHandler /** * Flushes the count. * @return ErrorCodeSuccess on success * @return ErrorCodeFailureNetwork on network error */ - ErrorCode finish() override; + auto finish() -> ErrorCode override; private: + // Data members int m_reducer_socket_fd; reducer::Pipeline m_pipeline; }; @@ -246,35 +248,37 @@ class CountToReducerOutputHandler : public ::clp_s::search::OutputHandler { * Output handler that performs a count aggregation bucketed by time and sends the results to a * reducer. */ -class CountByTimeToReducerOutputHandler : public ::clp_s::search::OutputHandler { +class CountByTimeReducerOutputHandler : public ::clp_s::search::OutputHandler { public: // Constructors - CountByTimeToReducerOutputHandler(int reducer_socket_fd, int64_t count_by_time_bucket_size) + CountByTimeReducerOutputHandler(int reducer_socket_fd, int64_t count_by_time_bucket_size) : search::OutputHandler{true, false}, m_reducer_socket_fd{reducer_socket_fd}, m_count_by_time_bucket_size{count_by_time_bucket_size} {} - // Methods inherited from OutputHandler - void write( + // Methods implementing OutputHandler + auto write( std::string_view message, epochtime_t timestamp, std::string_view archive_id, int64_t log_event_idx - ) override { + ) -> void override { int64_t bucket = (timestamp / m_count_by_time_bucket_size) * m_count_by_time_bucket_size; m_bucket_counts[bucket] += 1; } - void write(std::string_view message) override {} + auto write(std::string_view message) -> void override {} + // Methods overriding OutputHandler /** * Flushes the counts. * @return ErrorCodeSuccess on success * @return ErrorCodeFailureNetwork on network error */ - ErrorCode finish() override; + auto finish() -> ErrorCode override; private: + // Data members int m_reducer_socket_fd; std::map m_bucket_counts; int64_t m_count_by_time_bucket_size; @@ -283,7 +287,7 @@ class CountByTimeToReducerOutputHandler : public ::clp_s::search::OutputHandler /** * Output handler that performs a count aggregation and writes the results to the results cache. */ -class CountToResultsCacheOutputHandler : public ::clp_s::search::OutputHandler { +class CountResultsCacheOutputHandler : public ::clp_s::search::OutputHandler { public: // Types class OperationFailed : public TraceableException { @@ -294,28 +298,35 @@ class CountToResultsCacheOutputHandler : public ::clp_s::search::OutputHandler { }; // Constructors - CountToResultsCacheOutputHandler(std::string const& uri, std::string const& collection); + CountResultsCacheOutputHandler( + std::string_view uri, + std::string_view collection, + std::string archive_id + ); - // Methods inherited from OutputHandler - void write( + // Methods implementing OutputHandler + auto write( std::string_view message, epochtime_t timestamp, std::string_view archive_id, int64_t log_event_idx - ) override {} + ) -> void override {} - void write(std::string_view message) override { m_count += 1; } + auto write(std::string_view message) -> void override { m_count += 1; } + // Methods overriding OutputHandler /** * Flushes the count. * @return ErrorCodeSuccess on success * @return ErrorCodeFailureDbBulkWrite on database error */ - ErrorCode finish() override; + auto finish() -> ErrorCode override; private: + // Data members mongocxx::client m_client; mongocxx::collection m_collection; + std::string m_archive_id; int64_t m_count{}; }; @@ -323,7 +334,7 @@ class CountToResultsCacheOutputHandler : public ::clp_s::search::OutputHandler { * Output handler that performs a count aggregation bucketed by time and writes the results to the * results cache. */ -class CountByTimeToResultsCacheOutputHandler : public ::clp_s::search::OutputHandler { +class CountByTimeResultsCacheOutputHandler : public ::clp_s::search::OutputHandler { public: // Types class OperationFailed : public TraceableException { @@ -334,35 +345,39 @@ class CountByTimeToResultsCacheOutputHandler : public ::clp_s::search::OutputHan }; // Constructors - CountByTimeToResultsCacheOutputHandler( - std::string const& uri, - std::string const& collection, + CountByTimeResultsCacheOutputHandler( + std::string_view uri, + std::string_view collection, + std::string archive_id, int64_t count_by_time_bucket_size ); - // Methods inherited from OutputHandler - void write( + // Methods implementing OutputHandler + auto write( std::string_view message, epochtime_t timestamp, std::string_view archive_id, int64_t log_event_idx - ) override { + ) -> void override { int64_t bucket = (timestamp / m_count_by_time_bucket_size) * m_count_by_time_bucket_size; m_bucket_counts[bucket] += 1; } - void write(std::string_view message) override {} + auto write(std::string_view message) -> void override {} + // Methods overriding OutputHandler /** * Flushes the counts. * @return ErrorCodeSuccess on success * @return ErrorCodeFailureDbBulkWrite on database error */ - ErrorCode finish() override; + auto finish() -> ErrorCode override; private: + // Data members mongocxx::client m_client; mongocxx::collection m_collection; + std::string m_archive_id; std::map m_bucket_counts; int64_t m_count_by_time_bucket_size; }; diff --git a/components/core/src/clp_s/archive_constants.hpp b/components/core/src/clp_s/archive_constants.hpp index 212d593d5e..5ccc9d4112 100644 --- a/components/core/src/clp_s/archive_constants.hpp +++ b/components/core/src/clp_s/archive_constants.hpp @@ -44,11 +44,6 @@ constexpr std::string_view cFileSplitNumber{"_file_split_number"}; constexpr std::string_view cArchiveCreatorId{"_archive_creator_id"}; } // namespace range_index -namespace results_cache { -// MongoDB's primary-key field, which is backed by a built-in unique index. -constexpr char cDocId[]{"_id"}; -} // namespace results_cache - namespace results_cache::decompression { constexpr char cPath[]{"path"}; constexpr char cStreamId[]{"stream_id"}; diff --git a/components/core/src/clp_s/clp-s.cpp b/components/core/src/clp_s/clp-s.cpp index a4ecf80c6b..2966db298b 100644 --- a/components/core/src/clp_s/clp-s.cpp +++ b/components/core/src/clp_s/clp-s.cpp @@ -248,14 +248,14 @@ bool search_archive( if (CommandLineArguments::AggregationType::Count == options.aggregation_type) { output_handler - = std::make_unique( + = std::make_unique( reducer_socket_fd ); } else if (CommandLineArguments::AggregationType::CountByTime == options.aggregation_type) { output_handler = std::make_unique< - clp_s::CountByTimeToReducerOutputHandler + clp_s::CountByTimeReducerOutputHandler >(reducer_socket_fd, options.count_by_time_bucket_size); } else { SPDLOG_ERROR("Unhandled aggregation type."); @@ -276,17 +276,19 @@ bool search_archive( == options.aggregation_type.value()) { output_handler - = std::make_unique( + = std::make_unique( options.uri, - options.collection + options.collection, + std::string{archive_reader->get_archive_id()} ); } else if (CommandLineArguments::AggregationType::CountByTime == options.aggregation_type.value()) { output_handler = std::make_unique< - clp_s::CountByTimeToResultsCacheOutputHandler + clp_s::CountByTimeResultsCacheOutputHandler >(options.uri, options.collection, + std::string{archive_reader->get_archive_id()}, options.count_by_time_bucket_size); } else { SPDLOG_ERROR("Unhandled aggregation type."); diff --git a/components/core/src/clp_s/kv_ir_search.cpp b/components/core/src/clp_s/kv_ir_search.cpp index 8a232b5561..92354e66de 100644 --- a/components/core/src/clp_s/kv_ir_search.cpp +++ b/components/core/src/clp_s/kv_ir_search.cpp @@ -247,25 +247,28 @@ auto search_kv_ir_stream( } auto const& output_handler_options{command_line_arguments.get_output_handler_options()}; - bool const reducer_aggregation_requested - = std::holds_alternative( + auto const reducer_aggregation_requested{ + std::holds_alternative( output_handler_options - ); + ) + }; auto const* results_cache_options = std::get_if( &output_handler_options ); - bool const results_cache_aggregation_requested - = nullptr != results_cache_options - && results_cache_options->aggregation_type.has_value(); + auto const results_cache_aggregation_requested{ + nullptr != results_cache_options + && results_cache_options->aggregation_type.has_value() + }; auto const* stdout_options = std::get_if(&output_handler_options); - bool const stdout_aggregation_requested - = nullptr != stdout_options && stdout_options->aggregation_type.has_value(); + auto const stdout_aggregation_requested{ + nullptr != stdout_options && stdout_options->aggregation_type.has_value() + }; if (reducer_aggregation_requested || results_cache_aggregation_requested || stdout_aggregation_requested) { - SPDLOG_ERROR("kv-ir search: Count support is not implemented."); + SPDLOG_ERROR("kv-ir search: Aggregation support is not implemented."); return KvIrSearchError{KvIrSearchErrorEnum::CountSupportNotImplemented}; } From f95b1f662c29d85f11e8fa3b98fae0987419b41e Mon Sep 17 00:00:00 2001 From: davemarco <83603688+davemarco@users.noreply.github.com> Date: Mon, 22 Jun 2026 11:52:43 -0400 Subject: [PATCH 11/56] Use `string const&` for results-cache count handler uri/collection params. --- components/core/src/clp_s/OutputHandlerImpl.cpp | 12 ++++++------ components/core/src/clp_s/OutputHandlerImpl.hpp | 8 ++++---- 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/components/core/src/clp_s/OutputHandlerImpl.cpp b/components/core/src/clp_s/OutputHandlerImpl.cpp index 85c5b8bf4a..1ba5367af3 100644 --- a/components/core/src/clp_s/OutputHandlerImpl.cpp +++ b/components/core/src/clp_s/OutputHandlerImpl.cpp @@ -38,10 +38,10 @@ namespace { * @return The collection. */ template -auto connect_to_results_cache(string_view uri, string_view collection, mongocxx::client& client) +auto connect_to_results_cache(string const& uri, string const& collection, mongocxx::client& client) -> mongocxx::collection { try { - auto mongo_uri = mongocxx::uri(uri); + auto mongo_uri = mongocxx::uri{uri}; client = mongocxx::client(mongo_uri); return client[mongo_uri.database()][collection]; } catch (mongocxx::exception const& e) { @@ -297,8 +297,8 @@ ErrorCode AggregationToStdoutOutputHandler::finish() { } CountResultsCacheOutputHandler::CountResultsCacheOutputHandler( - string_view uri, - string_view collection, + string const& uri, + string const& collection, string archive_id ) : ::clp_s::search::OutputHandler{false, false}, @@ -331,8 +331,8 @@ auto CountResultsCacheOutputHandler::finish() -> ErrorCode { } CountByTimeResultsCacheOutputHandler::CountByTimeResultsCacheOutputHandler( - string_view uri, - string_view collection, + string const& uri, + string const& collection, string archive_id, int64_t count_by_time_bucket_size ) diff --git a/components/core/src/clp_s/OutputHandlerImpl.hpp b/components/core/src/clp_s/OutputHandlerImpl.hpp index 7631dec223..37f1bc761a 100644 --- a/components/core/src/clp_s/OutputHandlerImpl.hpp +++ b/components/core/src/clp_s/OutputHandlerImpl.hpp @@ -299,8 +299,8 @@ class CountResultsCacheOutputHandler : public ::clp_s::search::OutputHandler { // Constructors CountResultsCacheOutputHandler( - std::string_view uri, - std::string_view collection, + std::string const& uri, + std::string const& collection, std::string archive_id ); @@ -346,8 +346,8 @@ class CountByTimeResultsCacheOutputHandler : public ::clp_s::search::OutputHandl // Constructors CountByTimeResultsCacheOutputHandler( - std::string_view uri, - std::string_view collection, + std::string const& uri, + std::string const& collection, std::string archive_id, int64_t count_by_time_bucket_size ); From 20bddad9943946671d91956422af91bd01e6b12c Mon Sep 17 00:00:00 2001 From: davemarco <83603688+davemarco@users.noreply.github.com> Date: Mon, 22 Jun 2026 13:38:45 -0400 Subject: [PATCH 12/56] Add forward declaration for results-cache helper and apply clang-format. --- components/core/src/clp_s/OutputHandlerImpl.cpp | 4 ++++ components/core/src/clp_s/clp-s.cpp | 15 ++++++++------- components/core/src/clp_s/kv_ir_search.cpp | 3 +-- 3 files changed, 13 insertions(+), 9 deletions(-) diff --git a/components/core/src/clp_s/OutputHandlerImpl.cpp b/components/core/src/clp_s/OutputHandlerImpl.cpp index 1ba5367af3..1484456efe 100644 --- a/components/core/src/clp_s/OutputHandlerImpl.cpp +++ b/components/core/src/clp_s/OutputHandlerImpl.cpp @@ -38,6 +38,10 @@ namespace { * @return The collection. */ template +auto connect_to_results_cache(string const& uri, string const& collection, mongocxx::client& client) + -> mongocxx::collection; + +template auto connect_to_results_cache(string const& uri, string const& collection, mongocxx::client& client) -> mongocxx::collection { try { diff --git a/components/core/src/clp_s/clp-s.cpp b/components/core/src/clp_s/clp-s.cpp index 2966db298b..ee03f243a5 100644 --- a/components/core/src/clp_s/clp-s.cpp +++ b/components/core/src/clp_s/clp-s.cpp @@ -247,16 +247,17 @@ bool search_archive( -> void { if (CommandLineArguments::AggregationType::Count == options.aggregation_type) { - output_handler - = std::make_unique( - reducer_socket_fd - ); + output_handler = std::make_unique( + reducer_socket_fd + ); } else if (CommandLineArguments::AggregationType::CountByTime == options.aggregation_type) { - output_handler = std::make_unique< - clp_s::CountByTimeReducerOutputHandler - >(reducer_socket_fd, options.count_by_time_bucket_size); + output_handler + = std::make_unique( + reducer_socket_fd, + options.count_by_time_bucket_size + ); } else { SPDLOG_ERROR("Unhandled aggregation type."); output_handler = nullptr; diff --git a/components/core/src/clp_s/kv_ir_search.cpp b/components/core/src/clp_s/kv_ir_search.cpp index 92354e66de..96ce99ad18 100644 --- a/components/core/src/clp_s/kv_ir_search.cpp +++ b/components/core/src/clp_s/kv_ir_search.cpp @@ -257,8 +257,7 @@ auto search_kv_ir_stream( &output_handler_options ); auto const results_cache_aggregation_requested{ - nullptr != results_cache_options - && results_cache_options->aggregation_type.has_value() + nullptr != results_cache_options && results_cache_options->aggregation_type.has_value() }; auto const* stdout_options = std::get_if(&output_handler_options); From f2aa7e354b97b1bc3ac4b72c54a77643a1b39377 Mon Sep 17 00:00:00 2001 From: davemarco <83603688+davemarco@users.noreply.github.com> Date: Mon, 22 Jun 2026 13:49:02 -0400 Subject: [PATCH 13/56] Add ms unit to count_by_time_bucket_size field name. --- .../core/src/clp_s/CommandLineArguments.cpp | 12 ++++++------ .../core/src/clp_s/CommandLineArguments.hpp | 8 ++++---- components/core/src/clp_s/OutputHandlerImpl.cpp | 4 ++-- components/core/src/clp_s/OutputHandlerImpl.hpp | 16 +++++++++------- components/core/src/clp_s/clp-s.cpp | 4 ++-- 5 files changed, 23 insertions(+), 21 deletions(-) diff --git a/components/core/src/clp_s/CommandLineArguments.cpp b/components/core/src/clp_s/CommandLineArguments.cpp index f89d30f027..b50f368b68 100644 --- a/components/core/src/clp_s/CommandLineArguments.cpp +++ b/components/core/src/clp_s/CommandLineArguments.cpp @@ -818,7 +818,7 @@ CommandLineArguments::parse_arguments(int argc, char const** argv) { )( "count-by-time", po::value( - &reducer_options.count_by_time_bucket_size + &reducer_options.count_by_time_bucket_size_ms )->value_name("SIZE"), "Count the number of results in each time span of the given size (ms)" ); @@ -861,7 +861,7 @@ CommandLineArguments::parse_arguments(int argc, char const** argv) { )( "count-by-time", po::value( - &results_cache_options.count_by_time_bucket_size + &results_cache_options.count_by_time_bucket_size_ms )->value_name("SIZE"), "Count the number of results in each time span of the given size (ms)" ); @@ -1141,7 +1141,7 @@ void CommandLineArguments::parse_network_dest_output_handler_options( auto CommandLineArguments::parse_aggregation_options( po::variables_map const& parsed_options, - int64_t count_by_time_bucket_size + int64_t count_by_time_bucket_size_ms ) -> std::optional { std::optional aggregation_type; if (parsed_options.count("count")) { @@ -1154,7 +1154,7 @@ auto CommandLineArguments::parse_aggregation_options( ); } - if (count_by_time_bucket_size <= 0) { + if (count_by_time_bucket_size_ms <= 0) { throw std::invalid_argument("Value for count-by-time must be greater than zero."); } @@ -1194,7 +1194,7 @@ void CommandLineArguments::parse_reducer_output_handler_options( } auto const aggregation_type{ - parse_aggregation_options(parsed_options, reducer_options.count_by_time_bucket_size) + parse_aggregation_options(parsed_options, reducer_options.count_by_time_bucket_size_ms) }; if (false == aggregation_type.has_value()) { throw std::invalid_argument( @@ -1237,7 +1237,7 @@ void CommandLineArguments::parse_results_cache_output_handler_options( results_cache_options.aggregation_type = parse_aggregation_options( parsed_options, - results_cache_options.count_by_time_bucket_size + results_cache_options.count_by_time_bucket_size_ms ); } diff --git a/components/core/src/clp_s/CommandLineArguments.hpp b/components/core/src/clp_s/CommandLineArguments.hpp index f3edc59de0..14bf242ae3 100644 --- a/components/core/src/clp_s/CommandLineArguments.hpp +++ b/components/core/src/clp_s/CommandLineArguments.hpp @@ -43,7 +43,7 @@ class CommandLineArguments { uint64_t batch_size{1000}; uint64_t max_num_results{1000}; std::optional aggregation_type; - int64_t count_by_time_bucket_size{}; // Milliseconds + int64_t count_by_time_bucket_size_ms{}; }; struct FileOutputHandlerOptions { @@ -60,7 +60,7 @@ class CommandLineArguments { int port{-1}; reducer::job_id_t job_id{-1}; AggregationType aggregation_type{AggregationType::Count}; - int64_t count_by_time_bucket_size{}; // Milliseconds + int64_t count_by_time_bucket_size_ms{}; }; struct StdoutOutputHandlerOptions { @@ -165,13 +165,13 @@ class CommandLineArguments { * Validates the aggregation options (count and count-by-time) for output handlers that * support aggregations. * @param parsed_options - * @param count_by_time_bucket_size The parsed value of the count-by-time option; only + * @param count_by_time_bucket_size_ms The parsed value of the count-by-time option; only * validated when that option was specified. * @return The requested aggregation type, or std::nullopt if no aggregation was requested. */ [[nodiscard]] static auto parse_aggregation_options( boost::program_options::variables_map const& parsed_options, - int64_t count_by_time_bucket_size + int64_t count_by_time_bucket_size_ms ) -> std::optional; /** diff --git a/components/core/src/clp_s/OutputHandlerImpl.cpp b/components/core/src/clp_s/OutputHandlerImpl.cpp index 1484456efe..f0bf178799 100644 --- a/components/core/src/clp_s/OutputHandlerImpl.cpp +++ b/components/core/src/clp_s/OutputHandlerImpl.cpp @@ -338,11 +338,11 @@ CountByTimeResultsCacheOutputHandler::CountByTimeResultsCacheOutputHandler( string const& uri, string const& collection, string archive_id, - int64_t count_by_time_bucket_size + int64_t count_by_time_bucket_size_ms ) : ::clp_s::search::OutputHandler{true, false}, m_archive_id{std::move(archive_id)}, - m_count_by_time_bucket_size{count_by_time_bucket_size} { + m_count_by_time_bucket_size_ms{count_by_time_bucket_size_ms} { m_collection = connect_to_results_cache(uri, collection, m_client); } diff --git a/components/core/src/clp_s/OutputHandlerImpl.hpp b/components/core/src/clp_s/OutputHandlerImpl.hpp index 37f1bc761a..d2c4ad27d8 100644 --- a/components/core/src/clp_s/OutputHandlerImpl.hpp +++ b/components/core/src/clp_s/OutputHandlerImpl.hpp @@ -251,10 +251,10 @@ class CountReducerOutputHandler : public ::clp_s::search::OutputHandler { class CountByTimeReducerOutputHandler : public ::clp_s::search::OutputHandler { public: // Constructors - CountByTimeReducerOutputHandler(int reducer_socket_fd, int64_t count_by_time_bucket_size) + CountByTimeReducerOutputHandler(int reducer_socket_fd, int64_t count_by_time_bucket_size_ms) : search::OutputHandler{true, false}, m_reducer_socket_fd{reducer_socket_fd}, - m_count_by_time_bucket_size{count_by_time_bucket_size} {} + m_count_by_time_bucket_size_ms{count_by_time_bucket_size_ms} {} // Methods implementing OutputHandler auto write( @@ -263,7 +263,8 @@ class CountByTimeReducerOutputHandler : public ::clp_s::search::OutputHandler { std::string_view archive_id, int64_t log_event_idx ) -> void override { - int64_t bucket = (timestamp / m_count_by_time_bucket_size) * m_count_by_time_bucket_size; + int64_t bucket + = (timestamp / m_count_by_time_bucket_size_ms) * m_count_by_time_bucket_size_ms; m_bucket_counts[bucket] += 1; } @@ -281,7 +282,7 @@ class CountByTimeReducerOutputHandler : public ::clp_s::search::OutputHandler { // Data members int m_reducer_socket_fd; std::map m_bucket_counts; - int64_t m_count_by_time_bucket_size; + int64_t m_count_by_time_bucket_size_ms; }; /** @@ -349,7 +350,7 @@ class CountByTimeResultsCacheOutputHandler : public ::clp_s::search::OutputHandl std::string const& uri, std::string const& collection, std::string archive_id, - int64_t count_by_time_bucket_size + int64_t count_by_time_bucket_size_ms ); // Methods implementing OutputHandler @@ -359,7 +360,8 @@ class CountByTimeResultsCacheOutputHandler : public ::clp_s::search::OutputHandl std::string_view archive_id, int64_t log_event_idx ) -> void override { - int64_t bucket = (timestamp / m_count_by_time_bucket_size) * m_count_by_time_bucket_size; + int64_t bucket + = (timestamp / m_count_by_time_bucket_size_ms) * m_count_by_time_bucket_size_ms; m_bucket_counts[bucket] += 1; } @@ -379,7 +381,7 @@ class CountByTimeResultsCacheOutputHandler : public ::clp_s::search::OutputHandl mongocxx::collection m_collection; std::string m_archive_id; std::map m_bucket_counts; - int64_t m_count_by_time_bucket_size; + int64_t m_count_by_time_bucket_size_ms; }; /** diff --git a/components/core/src/clp_s/clp-s.cpp b/components/core/src/clp_s/clp-s.cpp index ee03f243a5..af2a4d9221 100644 --- a/components/core/src/clp_s/clp-s.cpp +++ b/components/core/src/clp_s/clp-s.cpp @@ -256,7 +256,7 @@ bool search_archive( output_handler = std::make_unique( reducer_socket_fd, - options.count_by_time_bucket_size + options.count_by_time_bucket_size_ms ); } else { SPDLOG_ERROR("Unhandled aggregation type."); @@ -290,7 +290,7 @@ bool search_archive( >(options.uri, options.collection, std::string{archive_reader->get_archive_id()}, - options.count_by_time_bucket_size); + options.count_by_time_bucket_size_ms); } else { SPDLOG_ERROR("Unhandled aggregation type."); output_handler = nullptr; From f93586a307600d389a4ca85c79e0b24ba0b843b4 Mon Sep 17 00:00:00 2001 From: davemarco <83603688+davemarco@users.noreply.github.com> Date: Mon, 22 Jun 2026 15:25:15 -0400 Subject: [PATCH 14/56] feat(clp-s): Support min and max aggregation output to stdout and the results cache. Adds --min/--max aggregations, extracting the field's numeric value from each matched record's marshalled JSON. Unifies the results-cache count handlers into a single AggregationToResultsCacheOutputHandler handling count, count-by-time, min, and max. --- .../core/src/clp_s/CommandLineArguments.cpp | 106 ++++++- .../core/src/clp_s/CommandLineArguments.hpp | 17 +- .../core/src/clp_s/OutputHandlerImpl.cpp | 288 +++++++++++++----- .../core/src/clp_s/OutputHandlerImpl.hpp | 116 ++++--- .../core/src/clp_s/archive_constants.hpp | 3 + components/core/src/clp_s/clp-s.cpp | 28 +- components/core/src/clp_s/kv_ir_search.cpp | 5 +- 7 files changed, 402 insertions(+), 161 deletions(-) diff --git a/components/core/src/clp_s/CommandLineArguments.cpp b/components/core/src/clp_s/CommandLineArguments.cpp index b50f368b68..b72ec27723 100644 --- a/components/core/src/clp_s/CommandLineArguments.cpp +++ b/components/core/src/clp_s/CommandLineArguments.cpp @@ -13,6 +13,7 @@ #include "../clp/type_utils.hpp" #include "../reducer/types.hpp" #include "FileReader.hpp" +#include "search/ast/SearchUtils.hpp" namespace po = boost::program_options; @@ -204,7 +205,8 @@ auto collect_subcommands( { // A leading option token (e.g. `--count`) with no named subcommand selects the default // subcommand, letting its options be specified without naming the subcommand. - if (false == default_subcommand.has_value() || option.empty() || '-' != option.front()) { + if (false == default_subcommand.has_value() || option.empty() || '-' != option.front()) + { throw std::invalid_argument(fmt::format("unrecognized option \"{}\"", option)); } current_subcommand = default_subcommand; @@ -864,6 +866,16 @@ CommandLineArguments::parse_arguments(int argc, char const** argv) { &results_cache_options.count_by_time_bucket_size_ms )->value_name("SIZE"), "Count the number of results in each time span of the given size (ms)" + )( + "min", + po::value( + &results_cache_options.aggregation_field)->value_name("FIELD"), + "Output the minimum value of the given field" + )( + "max", + po::value( + &results_cache_options.aggregation_field)->value_name("FIELD"), + "Output the maximum value of the given field" ); FileOutputHandlerOptions file_options{}; @@ -883,9 +895,19 @@ CommandLineArguments::parse_arguments(int argc, char const** argv) { )( "count-by-time", po::value( - &stdout_options.count_by_time_bucket_size + &stdout_options.count_by_time_bucket_size_ms )->value_name("SIZE"), "Count the number of results in each time span of the given size (ms)" + )( + "min", + po::value( + &stdout_options.aggregation_field)->value_name("FIELD"), + "Output the minimum value of the given field" + )( + "max", + po::value( + &stdout_options.aggregation_field)->value_name("FIELD"), + "Output the maximum value of the given field" ); // clang-format on @@ -976,6 +998,25 @@ CommandLineArguments::parse_arguments(int argc, char const** argv) { << std::endl; std::cerr << " " << m_program_name << R"( s archives-dir "level: INFO")" << " --count" << std::endl; + std::cerr << std::endl; + + std::cerr << " # Search archives in archives-dir for logs matching a KQL query" + R"( "level: INFO" and output the maximum value of `latency` to stdout)" + << std::endl; + std::cerr << " " << m_program_name << R"( s archives-dir "level: INFO")" + << " --max latency" << std::endl; + std::cerr << std::endl; + + std::cerr << " # Search archives in archives-dir for logs matching a KQL query" + R"( "level: INFO" and output the minimum value of `latency` to the)" + " results cache" + << std::endl; + std::cerr << " " << m_program_name << R"( s archives-dir "level: INFO")" + << " " << cResultsCacheOutputHandlerName + << " --uri mongodb://127.0.0.1:27017/test" + " --collection test" + " --min latency" + << std::endl; po::options_description visible_options; visible_options.add(general_options); @@ -1141,24 +1182,53 @@ void CommandLineArguments::parse_network_dest_output_handler_options( auto CommandLineArguments::parse_aggregation_options( po::variables_map const& parsed_options, - int64_t count_by_time_bucket_size_ms + int64_t count_by_time_bucket_size_ms, + std::string_view aggregation_field ) -> std::optional { std::optional aggregation_type; - if (parsed_options.count("count")) { - aggregation_type = AggregationType::Count; - } - if (parsed_options.count("count-by-time")) { + auto const set_aggregation_type = [&](AggregationType type) { if (aggregation_type.has_value()) { throw std::invalid_argument( - "The --count-by-time and --count options are mutually exclusive." + "The --count, --count-by-time, --min, and --max options are mutually exclusive." ); } + aggregation_type = type; + }; + if (parsed_options.count("count")) { + set_aggregation_type(AggregationType::Count); + } + if (parsed_options.count("count-by-time")) { + set_aggregation_type(AggregationType::CountByTime); if (count_by_time_bucket_size_ms <= 0) { throw std::invalid_argument("Value for count-by-time must be greater than zero."); } + } + if (parsed_options.count("min")) { + set_aggregation_type(AggregationType::Min); + } + if (parsed_options.count("max")) { + set_aggregation_type(AggregationType::Max); + } - aggregation_type = AggregationType::CountByTime; + if (AggregationType::Min == aggregation_type || AggregationType::Max == aggregation_type) { + if (aggregation_field.empty()) { + throw std::invalid_argument("The field for --min/--max cannot be an empty string."); + } + if (search::ast::has_unescaped_wildcards(aggregation_field)) { + throw std::invalid_argument("The field for --min/--max cannot contain wildcards."); + } + std::vector tokens; + std::string descriptor_namespace; + if (false + == search::ast::tokenize_column_descriptor( + std::string{aggregation_field}, + tokens, + descriptor_namespace + )) + { + throw std::invalid_argument("The field for --min/--max is not a valid field path."); + } } return aggregation_type; } @@ -1193,9 +1263,11 @@ void CommandLineArguments::parse_reducer_output_handler_options( throw std::invalid_argument("job-id cannot be negative."); } - auto const aggregation_type{ - parse_aggregation_options(parsed_options, reducer_options.count_by_time_bucket_size_ms) - }; + auto const aggregation_type{parse_aggregation_options( + parsed_options, + reducer_options.count_by_time_bucket_size_ms, + std::string_view{} + )}; if (false == aggregation_type.has_value()) { throw std::invalid_argument( "The reducer output handler currently only supports count and" @@ -1237,7 +1309,8 @@ void CommandLineArguments::parse_results_cache_output_handler_options( results_cache_options.aggregation_type = parse_aggregation_options( parsed_options, - results_cache_options.count_by_time_bucket_size_ms + results_cache_options.count_by_time_bucket_size_ms, + results_cache_options.aggregation_field ); } @@ -1264,8 +1337,11 @@ void CommandLineArguments::parse_stdout_output_handler_options( po::variables_map parsed_options; parse_subcommand_options(options_description, options, parsed_options); - stdout_options.aggregation_type - = parse_aggregation_options(parsed_options, stdout_options.count_by_time_bucket_size); + stdout_options.aggregation_type = parse_aggregation_options( + parsed_options, + stdout_options.count_by_time_bucket_size_ms, + stdout_options.aggregation_field + ); } void CommandLineArguments::print_basic_usage() const { diff --git a/components/core/src/clp_s/CommandLineArguments.hpp b/components/core/src/clp_s/CommandLineArguments.hpp index 14bf242ae3..772bd89dcd 100644 --- a/components/core/src/clp_s/CommandLineArguments.hpp +++ b/components/core/src/clp_s/CommandLineArguments.hpp @@ -34,6 +34,8 @@ class CommandLineArguments { enum class AggregationType : uint8_t { Count, CountByTime, + Min, + Max, }; struct ResultsCacheOutputHandlerOptions { @@ -44,6 +46,8 @@ class CommandLineArguments { uint64_t max_num_results{1000}; std::optional aggregation_type; int64_t count_by_time_bucket_size_ms{}; + // Field whose value is aggregated by `--min`/`--max`. + std::string aggregation_field; }; struct FileOutputHandlerOptions { @@ -65,7 +69,9 @@ class CommandLineArguments { struct StdoutOutputHandlerOptions { std::optional aggregation_type; - int64_t count_by_time_bucket_size{}; // Milliseconds + int64_t count_by_time_bucket_size_ms{}; + // Field whose value is aggregated by `--min`/`--max`. + std::string aggregation_field; }; using OutputHandlerOptionsVariant = std:: @@ -162,16 +168,19 @@ class CommandLineArguments { ); /** - * Validates the aggregation options (count and count-by-time) for output handlers that - * support aggregations. + * Validates the aggregation options (count, count-by-time, min, and max) for output handlers + * that support aggregations. The options are mutually exclusive. * @param parsed_options * @param count_by_time_bucket_size_ms The parsed value of the count-by-time option; only * validated when that option was specified. + * @param aggregation_field The parsed value of the min/max option; only validated when one of + * those options was specified. * @return The requested aggregation type, or std::nullopt if no aggregation was requested. */ [[nodiscard]] static auto parse_aggregation_options( boost::program_options::variables_map const& parsed_options, - int64_t count_by_time_bucket_size_ms + int64_t count_by_time_bucket_size_ms, + std::string_view aggregation_field ) -> std::optional; /** diff --git a/components/core/src/clp_s/OutputHandlerImpl.cpp b/components/core/src/clp_s/OutputHandlerImpl.cpp index f0bf178799..2d915faa1b 100644 --- a/components/core/src/clp_s/OutputHandlerImpl.cpp +++ b/components/core/src/clp_s/OutputHandlerImpl.cpp @@ -3,6 +3,7 @@ #include #include #include +#include #include #include @@ -21,6 +22,7 @@ #include "../reducer/RecordGroup.hpp" #include "../reducer/RecordGroupIterator.hpp" #include "archive_constants.hpp" +#include "search/ast/SearchUtils.hpp" #include "search/OutputHandler.hpp" #include "TraceableException.hpp" @@ -28,6 +30,8 @@ using std::string; using std::string_view; namespace clp_s { +using AggregationType = CommandLineArguments::AggregationType; + namespace { /** * Connects to the results cache and returns the requested collection. @@ -52,8 +56,93 @@ auto connect_to_results_cache(string const& uri, string const& collection, mongo throw OperationFailedT(ErrorCode::ErrorCodeBadParamDbUri, __FILENAME__, __LINE__); } } + +/** + * @param type + * @return Whether the aggregation needs per-record metadata (i.e. the timestamp). + */ +[[nodiscard]] auto aggregation_needs_metadata(AggregationType type) -> bool { + return AggregationType::CountByTime == type; +} + +/** + * @param type + * @return Whether the aggregation needs the record marshalled into its JSON message. + */ +[[nodiscard]] auto aggregation_needs_marshalling(AggregationType type) -> bool { + return AggregationType::Min == type || AggregationType::Max == type; +} } // namespace +FieldMinMaxAggregator::FieldMinMaxAggregator(bool find_max, string_view field) + : m_find_max{find_max} { + string descriptor_namespace; + if (false + == search::ast::tokenize_column_descriptor( + string{field}, + m_field_path, + descriptor_namespace + )) + { + // A malformed field tokenizes to nothing, so `update` never matches and min/max yields no + // result. The field is validated during command-line parsing, so this is just defensive. + m_field_path.clear(); + return; + } + if (false == descriptor_namespace.empty()) { + // Namespaced fields (e.g. the auto-generated `@` namespace) are marshalled under a + // top-level object keyed by the namespace string, so navigate into it first. + m_field_path.insert(m_field_path.begin(), descriptor_namespace); + } +} + +auto FieldMinMaxAggregator::beats_extreme(Value candidate) const -> bool { + auto const& current{m_extreme.value()}; + // Compare integers exactly; only fall back to a (lossy) double comparison when the types + // differ. + if (std::holds_alternative(candidate) && std::holds_alternative(current)) { + auto const lhs{std::get(candidate)}; + auto const rhs{std::get(current)}; + return m_find_max ? lhs > rhs : lhs < rhs; + } + auto const as_double{[](Value value) { + return std::visit([](auto held) { return static_cast(held); }, value); + }}; + return m_find_max ? as_double(candidate) > as_double(current) + : as_double(candidate) < as_double(current); +} + +auto FieldMinMaxAggregator::update(string_view message) -> void { + nlohmann::json doc; + try { + doc = nlohmann::json::parse(message); + } catch (nlohmann::json::exception const&) { + return; + } + + nlohmann::json const* node{&doc}; + for (auto const& key : m_field_path) { + if (false == node->is_object()) { + return; + } + auto const it{node->find(key)}; + if (node->end() == it) { + return; + } + node = &it.value(); + } + + if (false == node->is_number()) { + return; + } + Value const candidate{ + node->is_number_integer() ? Value{node->get()} : Value{node->get()} + }; + if (false == m_extreme.has_value() || beats_extreme(candidate)) { + m_extreme = candidate; + } +} + void FileOutputHandler::write( string_view message, epochtime_t timestamp, @@ -241,18 +330,32 @@ auto CountByTimeReducerOutputHandler::finish() -> ErrorCode { AggregationToStdoutOutputHandler::AggregationToStdoutOutputHandler( string archive_id, - bool count_by_time, - int64_t count_by_time_bucket_size + CommandLineArguments::AggregationType aggregation_type, + int64_t count_by_time_bucket_size_ms, + string_view aggregation_field ) - : ::clp_s::search::OutputHandler(count_by_time, false), + : ::clp_s::search:: + OutputHandler{aggregation_needs_metadata(aggregation_type), aggregation_needs_marshalling(aggregation_type)}, m_archive_id{std::move(archive_id)}, - m_count_by_time_bucket_size{count_by_time_bucket_size}, - m_pipeline{reducer::PipelineInputMode::InterStage} { - m_pipeline.add_pipeline_stage(std::make_shared()); + m_aggregation_type{aggregation_type}, + m_count_by_time_bucket_size_ms{count_by_time_bucket_size_ms}, + m_aggregation_field{aggregation_field} { + if (AggregationType::Count == m_aggregation_type) { + m_pipeline.emplace(reducer::PipelineInputMode::InterStage); + m_pipeline->add_pipeline_stage(std::make_shared()); + } else if (AggregationType::Min == m_aggregation_type + || AggregationType::Max == m_aggregation_type) + { + m_min_max.emplace(AggregationType::Max == m_aggregation_type, aggregation_field); + } } void AggregationToStdoutOutputHandler::write(string_view message) { - m_pipeline.push_record(reducer::EmptyRecord{}); + if (m_pipeline.has_value()) { + m_pipeline->push_record(reducer::EmptyRecord{}); + } else if (m_min_max.has_value()) { + m_min_max->update(message); + } } void AggregationToStdoutOutputHandler::write( @@ -261,21 +364,36 @@ void AggregationToStdoutOutputHandler::write( string_view archive_id, int64_t log_event_idx ) { - int64_t const bucket = (timestamp / m_count_by_time_bucket_size) * m_count_by_time_bucket_size; + int64_t const bucket + = (timestamp / m_count_by_time_bucket_size_ms) * m_count_by_time_bucket_size_ms; m_bucket_counts[bucket] += 1; } ErrorCode AggregationToStdoutOutputHandler::finish() { - // count-by-time results are serialized from the bucket counts; count results come from the - // CountOperator pipeline. `should_output_metadata()` is true only for count-by-time. + if (AggregationType::Min == m_aggregation_type || AggregationType::Max == m_aggregation_type) { + if (m_min_max.has_value() && m_min_max->has_value()) { + nlohmann::json result; + result[constants::results_cache::search::cArchiveId] = m_archive_id; + result[constants::results_cache::search::cField] = m_aggregation_field; + auto const* const key = AggregationType::Max == m_aggregation_type + ? constants::results_cache::search::cMax + : constants::results_cache::search::cMin; + std::visit([&](auto value) { result[key] = value; }, m_min_max->get_value()); + std::cout << result.dump() << '\n'; + } + return ErrorCode::ErrorCodeSuccess; + } + + // count results come from the CountOperator pipeline; count-by-time results are serialized from + // the bucket counts. std::unique_ptr results; - if (should_output_metadata()) { + if (AggregationType::CountByTime == m_aggregation_type) { results = std::make_unique( m_bucket_counts, reducer::CountOperator::cRecordElementKey ); } else { - results = m_pipeline.finish(); + results = m_pipeline->finish(); } for (; false == results->done(); results->next()) { auto& group{results->get()}; @@ -300,79 +418,107 @@ ErrorCode AggregationToStdoutOutputHandler::finish() { return ErrorCode::ErrorCodeSuccess; } -CountResultsCacheOutputHandler::CountResultsCacheOutputHandler( +AggregationToResultsCacheOutputHandler::AggregationToResultsCacheOutputHandler( string const& uri, string const& collection, - string archive_id + string archive_id, + CommandLineArguments::AggregationType aggregation_type, + int64_t count_by_time_bucket_size_ms, + string_view aggregation_field ) - : ::clp_s::search::OutputHandler{false, false}, - m_archive_id{std::move(archive_id)} { + : ::clp_s::search:: + OutputHandler{aggregation_needs_metadata(aggregation_type), aggregation_needs_marshalling(aggregation_type)}, + m_archive_id{std::move(archive_id)}, + m_aggregation_type{aggregation_type}, + m_count_by_time_bucket_size_ms{count_by_time_bucket_size_ms}, + m_aggregation_field{aggregation_field} { + if (AggregationType::Min == m_aggregation_type || AggregationType::Max == m_aggregation_type) { + m_min_max.emplace(AggregationType::Max == m_aggregation_type, aggregation_field); + } m_collection = connect_to_results_cache(uri, collection, m_client); } -auto CountResultsCacheOutputHandler::finish() -> ErrorCode { - if (0 == m_count) { - return ErrorCode::ErrorCodeSuccess; - } - +auto AggregationToResultsCacheOutputHandler::finish() -> ErrorCode { try { - m_collection.insert_one( - bsoncxx::builder::basic::make_document( + switch (m_aggregation_type) { + case AggregationType::Count: { + if (0 == m_count) { + return ErrorCode::ErrorCodeSuccess; + } + m_collection.insert_one( + bsoncxx::builder::basic::make_document( + bsoncxx::builder::basic::kvp( + constants::results_cache::search::cArchiveId, + m_archive_id + ), + bsoncxx::builder::basic::kvp( + constants::results_cache::search::cCount, + m_count + ) + ) + ); + break; + } + case AggregationType::CountByTime: { + if (m_bucket_counts.empty()) { + return ErrorCode::ErrorCodeSuccess; + } + std::vector documents; + documents.reserve(m_bucket_counts.size()); + for (auto const& [bucket_timestamp, count] : m_bucket_counts) { + documents.emplace_back( + bsoncxx::builder::basic::make_document( + bsoncxx::builder::basic::kvp( + constants::results_cache::search::cArchiveId, + m_archive_id + ), + bsoncxx::builder::basic::kvp( + constants::results_cache::search::cTimestamp, + bucket_timestamp + ), + bsoncxx::builder::basic::kvp( + constants::results_cache::search::cCount, + count + ) + ) + ); + } + m_collection.insert_many(documents); + break; + } + case AggregationType::Min: + case AggregationType::Max: { + if (false == m_min_max.has_value() || false == m_min_max->has_value()) { + return ErrorCode::ErrorCodeSuccess; + } + bsoncxx::builder::basic::document document; + document.append( bsoncxx::builder::basic::kvp( constants::results_cache::search::cArchiveId, m_archive_id ), bsoncxx::builder::basic::kvp( - constants::results_cache::search::cCount, - m_count + constants::results_cache::search::cField, + m_aggregation_field ) - ) - ); - } catch (mongocxx::exception const& e) { - return ErrorCode::ErrorCodeFailureDbBulkWrite; - } - return ErrorCode::ErrorCodeSuccess; -} - -CountByTimeResultsCacheOutputHandler::CountByTimeResultsCacheOutputHandler( - string const& uri, - string const& collection, - string archive_id, - int64_t count_by_time_bucket_size_ms -) - : ::clp_s::search::OutputHandler{true, false}, - m_archive_id{std::move(archive_id)}, - m_count_by_time_bucket_size_ms{count_by_time_bucket_size_ms} { - m_collection = connect_to_results_cache(uri, collection, m_client); -} - -auto CountByTimeResultsCacheOutputHandler::finish() -> ErrorCode { - if (m_bucket_counts.empty()) { - return ErrorCode::ErrorCodeSuccess; - } - - try { - std::vector documents; - documents.reserve(m_bucket_counts.size()); - for (auto const& [bucket_timestamp, count] : m_bucket_counts) { - documents.emplace_back( - bsoncxx::builder::basic::make_document( - bsoncxx::builder::basic::kvp( - constants::results_cache::search::cArchiveId, - m_archive_id - ), - bsoncxx::builder::basic::kvp( - constants::results_cache::search::cTimestamp, - bucket_timestamp - ), - bsoncxx::builder::basic::kvp( - constants::results_cache::search::cCount, - count - ) - ) - ); + ); + auto const append_extreme = [&](auto const& key) { + std::visit( + [&](auto value) { + document.append(bsoncxx::builder::basic::kvp(key, value)); + }, + m_min_max->get_value() + ); + }; + if (AggregationType::Max == m_aggregation_type) { + append_extreme(constants::results_cache::search::cMax); + } else { + append_extreme(constants::results_cache::search::cMin); + } + m_collection.insert_one(document.view()); + break; + } } - m_collection.insert_many(documents); } catch (mongocxx::exception const& e) { return ErrorCode::ErrorCodeFailureDbBulkWrite; } diff --git a/components/core/src/clp_s/OutputHandlerImpl.hpp b/components/core/src/clp_s/OutputHandlerImpl.hpp index d2c4ad27d8..6a925c18d7 100644 --- a/components/core/src/clp_s/OutputHandlerImpl.hpp +++ b/components/core/src/clp_s/OutputHandlerImpl.hpp @@ -7,9 +7,11 @@ #include #include +#include #include #include #include +#include #include #include @@ -17,6 +19,7 @@ #include "../reducer/Pipeline.hpp" #include "../reducer/RecordGroupIterator.hpp" +#include "CommandLineArguments.hpp" #include "Defs.hpp" #include "FileWriter.hpp" #include "search/OutputHandler.hpp" @@ -286,56 +289,53 @@ class CountByTimeReducerOutputHandler : public ::clp_s::search::OutputHandler { }; /** - * Output handler that performs a count aggregation and writes the results to the results cache. + * Accumulates the minimum or maximum value of a single numeric field across the records it is fed. + * The field is identified by a KQL column descriptor (e.g. `a.b.c`, or `@x` for the auto-generated + * namespace) resolved against each record's marshalled JSON. Records that lack the field or whose + * value is non-numeric are ignored. The extreme preserves whether it was an integer or a + * floating-point value so integer fields aren't degraded to `double`. */ -class CountResultsCacheOutputHandler : public ::clp_s::search::OutputHandler { +class FieldMinMaxAggregator { public: // Types - class OperationFailed : public TraceableException { - public: - // Constructors - OperationFailed(ErrorCode error_code, char const* const filename, int line_number) - : TraceableException(error_code, filename, line_number) {} - }; + using Value = std::variant; // Constructors - CountResultsCacheOutputHandler( - std::string const& uri, - std::string const& collection, - std::string archive_id - ); + FieldMinMaxAggregator(bool find_max, std::string_view field); - // Methods implementing OutputHandler - auto write( - std::string_view message, - epochtime_t timestamp, - std::string_view archive_id, - int64_t log_event_idx - ) -> void override {} + // Methods + /** + * Updates the running extreme with the target field's value extracted from `message`. + * @param message The marshalled JSON of a matched record. + */ + auto update(std::string_view message) -> void; - auto write(std::string_view message) -> void override { m_count += 1; } + [[nodiscard]] auto has_value() const -> bool { return m_extreme.has_value(); } - // Methods overriding OutputHandler /** - * Flushes the count. - * @return ErrorCodeSuccess on success - * @return ErrorCodeFailureDbBulkWrite on database error + * @return The accumulated extreme, preserving its integer or floating-point type. */ - auto finish() -> ErrorCode override; + [[nodiscard]] auto get_value() const -> Value { return m_extreme.value(); } private: + // Methods + /** + * @param candidate + * @return Whether `candidate` is more extreme (per the min/max mode) than the current extreme. + */ + [[nodiscard]] auto beats_extreme(Value candidate) const -> bool; + // Data members - mongocxx::client m_client; - mongocxx::collection m_collection; - std::string m_archive_id; - int64_t m_count{}; + bool m_find_max; + std::vector m_field_path; + std::optional m_extreme; }; /** - * Output handler that performs a count aggregation bucketed by time and writes the results to the - * results cache. + * Output handler that performs an aggregation (count, count-by-time, min, or max) and writes the + * results to the results cache (MongoDB). Each archive writes its own documents. */ -class CountByTimeResultsCacheOutputHandler : public ::clp_s::search::OutputHandler { +class AggregationToResultsCacheOutputHandler : public ::clp_s::search::OutputHandler { public: // Types class OperationFailed : public TraceableException { @@ -346,11 +346,13 @@ class CountByTimeResultsCacheOutputHandler : public ::clp_s::search::OutputHandl }; // Constructors - CountByTimeResultsCacheOutputHandler( + AggregationToResultsCacheOutputHandler( std::string const& uri, std::string const& collection, std::string archive_id, - int64_t count_by_time_bucket_size_ms + CommandLineArguments::AggregationType aggregation_type, + int64_t count_by_time_bucket_size_ms, + std::string_view aggregation_field ); // Methods implementing OutputHandler @@ -360,16 +362,22 @@ class CountByTimeResultsCacheOutputHandler : public ::clp_s::search::OutputHandl std::string_view archive_id, int64_t log_event_idx ) -> void override { - int64_t bucket + int64_t const bucket = (timestamp / m_count_by_time_bucket_size_ms) * m_count_by_time_bucket_size_ms; m_bucket_counts[bucket] += 1; } - auto write(std::string_view message) -> void override {} + auto write(std::string_view message) -> void override { + if (CommandLineArguments::AggregationType::Count == m_aggregation_type) { + m_count += 1; + } else if (m_min_max.has_value()) { + m_min_max->update(message); + } + } // Methods overriding OutputHandler /** - * Flushes the counts. + * Writes the aggregation results to the results cache. * @return ErrorCodeSuccess on success * @return ErrorCodeFailureDbBulkWrite on database error */ @@ -380,28 +388,33 @@ class CountByTimeResultsCacheOutputHandler : public ::clp_s::search::OutputHandl mongocxx::client m_client; mongocxx::collection m_collection; std::string m_archive_id; - std::map m_bucket_counts; + CommandLineArguments::AggregationType m_aggregation_type; int64_t m_count_by_time_bucket_size_ms; + int64_t m_count{}; + std::map m_bucket_counts; + std::string m_aggregation_field; + std::optional m_min_max; }; /** - * Output handler that performs a count or count-by-time aggregation and writes the results to - * standard output as newline-delimited JSON, one object per group. + * Output handler that performs an aggregation (count, count-by-time, min, or max) and writes the + * results to standard output as newline-delimited JSON, one object per group. * - * Mirroring the reducer count handlers: count accumulates through a `reducer::CountOperator` - * pipeline (yielding a single total), while count-by-time accumulates a count per time bucket. Both - * are serialized through a `reducer::RecordGroupIterator` in `finish()`. + * Count accumulates through a `reducer::CountOperator` pipeline (yielding a single total), + * count-by-time accumulates a count per time bucket, and min/max accumulate the extreme value of a + * target field. */ class AggregationToStdoutOutputHandler : public ::clp_s::search::OutputHandler { public: // Constructors AggregationToStdoutOutputHandler( std::string archive_id, - bool count_by_time, - int64_t count_by_time_bucket_size + CommandLineArguments::AggregationType aggregation_type, + int64_t count_by_time_bucket_size_ms, + std::string_view aggregation_field ); - // Methods inherited from OutputHandler + // Methods implementing OutputHandler void write( std::string_view message, epochtime_t timestamp, @@ -411,6 +424,7 @@ class AggregationToStdoutOutputHandler : public ::clp_s::search::OutputHandler { void write(std::string_view message) override; + // Methods overriding OutputHandler /** * Serializes the aggregation results to stdout as newline-delimited JSON. * @return ErrorCodeSuccess on success @@ -418,10 +432,14 @@ class AggregationToStdoutOutputHandler : public ::clp_s::search::OutputHandler { ErrorCode finish() override; private: + // Data members std::string m_archive_id; - int64_t m_count_by_time_bucket_size; - reducer::Pipeline m_pipeline; + CommandLineArguments::AggregationType m_aggregation_type; + int64_t m_count_by_time_bucket_size_ms; + std::optional m_pipeline; std::map m_bucket_counts; + std::string m_aggregation_field; + std::optional m_min_max; }; /** diff --git a/components/core/src/clp_s/archive_constants.hpp b/components/core/src/clp_s/archive_constants.hpp index 5ccc9d4112..0ab8cf824e 100644 --- a/components/core/src/clp_s/archive_constants.hpp +++ b/components/core/src/clp_s/archive_constants.hpp @@ -60,6 +60,9 @@ constexpr char cMessage[]{"message"}; constexpr char cArchiveId[]{"archive_id"}; constexpr std::string_view cDataset{"dataset"}; constexpr char cCount[]{"count"}; +constexpr char cMin[]{"min"}; +constexpr char cMax[]{"max"}; +constexpr char cField[]{"field"}; } // namespace results_cache::search } // namespace clp_s::constants #endif // CLP_S_ARCHIVE_CONSTANTS_HPP diff --git a/components/core/src/clp_s/clp-s.cpp b/components/core/src/clp_s/clp-s.cpp index af2a4d9221..738c57c663 100644 --- a/components/core/src/clp_s/clp-s.cpp +++ b/components/core/src/clp_s/clp-s.cpp @@ -273,27 +273,15 @@ bool search_archive( options.max_num_results, options.dataset ); - } else if (CommandLineArguments::AggregationType::Count - == options.aggregation_type.value()) - { - output_handler - = std::make_unique( - options.uri, - options.collection, - std::string{archive_reader->get_archive_id()} - ); - } else if (CommandLineArguments::AggregationType::CountByTime - == options.aggregation_type.value()) - { + } else { output_handler = std::make_unique< - clp_s::CountByTimeResultsCacheOutputHandler + clp_s::AggregationToResultsCacheOutputHandler >(options.uri, options.collection, std::string{archive_reader->get_archive_id()}, - options.count_by_time_bucket_size_ms); - } else { - SPDLOG_ERROR("Unhandled aggregation type."); - output_handler = nullptr; + options.aggregation_type.value(), + options.count_by_time_bucket_size_ms, + options.aggregation_field); } }, [&](CommandLineArguments::StdoutOutputHandlerOptions const& options) @@ -304,9 +292,9 @@ bool search_archive( output_handler = std::make_unique( std::string{archive_reader->get_archive_id()}, - CommandLineArguments::AggregationType::CountByTime - == options.aggregation_type.value(), - options.count_by_time_bucket_size + options.aggregation_type.value(), + options.count_by_time_bucket_size_ms, + options.aggregation_field ); } } diff --git a/components/core/src/clp_s/kv_ir_search.cpp b/components/core/src/clp_s/kv_ir_search.cpp index 96ce99ad18..90c02079b0 100644 --- a/components/core/src/clp_s/kv_ir_search.cpp +++ b/components/core/src/clp_s/kv_ir_search.cpp @@ -259,8 +259,9 @@ auto search_kv_ir_stream( auto const results_cache_aggregation_requested{ nullptr != results_cache_options && results_cache_options->aggregation_type.has_value() }; - auto const* stdout_options - = std::get_if(&output_handler_options); + auto const* stdout_options = std::get_if( + &output_handler_options + ); auto const stdout_aggregation_requested{ nullptr != stdout_options && stdout_options->aggregation_type.has_value() }; From e4b5c7bdd6b39a52dd580759aa12861c7382367e Mon Sep 17 00:00:00 2001 From: davemarco <83603688+davemarco@users.noreply.github.com> Date: Thu, 25 Jun 2026 11:06:19 -0400 Subject: [PATCH 15/56] refactor(clp-s): Address review comments on results-cache count handlers. - Take string_view for the results-cache handler uri/collection/dataset/archive_id params and materialize std::string only at the mongocxx call sites. - Drop the redundant ::clp_s:: qualifier on the count handlers' base class. - Rename the count-by-time write timestamp param to timestamp_ms to match the bucket size unit. --- .../core/src/clp_s/OutputHandlerImpl.cpp | 48 +++++++++++-------- .../core/src/clp_s/OutputHandlerImpl.hpp | 34 ++++++------- components/core/src/clp_s/clp-s.cpp | 4 +- 3 files changed, 46 insertions(+), 40 deletions(-) diff --git a/components/core/src/clp_s/OutputHandlerImpl.cpp b/components/core/src/clp_s/OutputHandlerImpl.cpp index bbaee32958..c7b0f17c5d 100644 --- a/components/core/src/clp_s/OutputHandlerImpl.cpp +++ b/components/core/src/clp_s/OutputHandlerImpl.cpp @@ -35,16 +35,22 @@ namespace { * @return The collection. */ template -auto connect_to_results_cache(string const& uri, string const& collection, mongocxx::client& client) - -> mongocxx::collection; +auto connect_to_results_cache( + string_view uri, + string_view collection, + mongocxx::client& client +) -> mongocxx::collection; template -auto connect_to_results_cache(string const& uri, string const& collection, mongocxx::client& client) - -> mongocxx::collection { +auto connect_to_results_cache( + string_view uri, + string_view collection, + mongocxx::client& client +) -> mongocxx::collection { try { - auto mongo_uri = mongocxx::uri{uri}; + auto mongo_uri = mongocxx::uri{string{uri}}; client = mongocxx::client(mongo_uri); - return client[mongo_uri.database()][collection]; + return client[mongo_uri.database()][string{collection}]; } catch (mongocxx::exception const& e) { throw OperationFailedT(ErrorCode::ErrorCodeBadParamDbUri, __FILENAME__, __LINE__); } @@ -94,17 +100,17 @@ void NetworkOutputHandler::write( } ResultsCacheOutputHandler::ResultsCacheOutputHandler( - string const& uri, - string const& collection, + string_view uri, + string_view collection, uint64_t batch_size, uint64_t max_num_results, - string dataset, + string_view dataset, bool should_output_timestamp ) : ::clp_s::search::OutputHandler{should_output_timestamp, true}, m_batch_size{batch_size}, m_max_num_results{max_num_results}, - m_dataset{std::move(dataset)} { + m_dataset{dataset} { m_collection = connect_to_results_cache(uri, collection, m_client); m_results.reserve(m_batch_size); } @@ -202,7 +208,7 @@ void ResultsCacheOutputHandler::write( } CountReducerOutputHandler::CountReducerOutputHandler(int reducer_socket_fd) - : ::clp_s::search::OutputHandler(false, false), + : search::OutputHandler(false, false), m_reducer_socket_fd(reducer_socket_fd), m_pipeline(reducer::PipelineInputMode::InterStage) { m_pipeline.add_pipeline_stage(std::make_shared()); @@ -237,12 +243,12 @@ auto CountByTimeReducerOutputHandler::finish() -> ErrorCode { } CountResultsCacheOutputHandler::CountResultsCacheOutputHandler( - string const& uri, - string const& collection, - string archive_id + string_view uri, + string_view collection, + string_view archive_id ) - : ::clp_s::search::OutputHandler{false, false}, - m_archive_id{std::move(archive_id)} { + : search::OutputHandler{false, false}, + m_archive_id{archive_id} { m_collection = connect_to_results_cache(uri, collection, m_client); } @@ -271,13 +277,13 @@ auto CountResultsCacheOutputHandler::finish() -> ErrorCode { } CountByTimeResultsCacheOutputHandler::CountByTimeResultsCacheOutputHandler( - string const& uri, - string const& collection, - string archive_id, + string_view uri, + string_view collection, + string_view archive_id, int64_t count_by_time_bucket_size_ms ) - : ::clp_s::search::OutputHandler{true, false}, - m_archive_id{std::move(archive_id)}, + : search::OutputHandler{true, false}, + m_archive_id{archive_id}, m_count_by_time_bucket_size_ms{count_by_time_bucket_size_ms} { m_collection = connect_to_results_cache(uri, collection, m_client); } diff --git a/components/core/src/clp_s/OutputHandlerImpl.hpp b/components/core/src/clp_s/OutputHandlerImpl.hpp index cf0df29c52..6d4df7fc7f 100644 --- a/components/core/src/clp_s/OutputHandlerImpl.hpp +++ b/components/core/src/clp_s/OutputHandlerImpl.hpp @@ -165,11 +165,11 @@ class ResultsCacheOutputHandler : public ::clp_s::search::OutputHandler { // Constructor ResultsCacheOutputHandler( - std::string const& uri, - std::string const& collection, + std::string_view uri, + std::string_view collection, uint64_t batch_size, uint64_t max_num_results, - std::string dataset, + std::string_view dataset, bool should_output_metadata = true ); @@ -215,7 +215,7 @@ class ResultsCacheOutputHandler : public ::clp_s::search::OutputHandler { /** * Output handler that performs a count aggregation and sends the results to a reducer. */ -class CountReducerOutputHandler : public ::clp_s::search::OutputHandler { +class CountReducerOutputHandler : public search::OutputHandler { public: // Constructors CountReducerOutputHandler(int reducer_socket_fd); @@ -248,7 +248,7 @@ class CountReducerOutputHandler : public ::clp_s::search::OutputHandler { * Output handler that performs a count aggregation bucketed by time and sends the results to a * reducer. */ -class CountByTimeReducerOutputHandler : public ::clp_s::search::OutputHandler { +class CountByTimeReducerOutputHandler : public search::OutputHandler { public: // Constructors CountByTimeReducerOutputHandler(int reducer_socket_fd, int64_t count_by_time_bucket_size_ms) @@ -259,12 +259,12 @@ class CountByTimeReducerOutputHandler : public ::clp_s::search::OutputHandler { // Methods implementing OutputHandler auto write( std::string_view message, - epochtime_t timestamp, + epochtime_t timestamp_ms, std::string_view archive_id, int64_t log_event_idx ) -> void override { int64_t bucket - = (timestamp / m_count_by_time_bucket_size_ms) * m_count_by_time_bucket_size_ms; + = (timestamp_ms / m_count_by_time_bucket_size_ms) * m_count_by_time_bucket_size_ms; m_bucket_counts[bucket] += 1; } @@ -288,7 +288,7 @@ class CountByTimeReducerOutputHandler : public ::clp_s::search::OutputHandler { /** * Output handler that performs a count aggregation and writes the results to the results cache. */ -class CountResultsCacheOutputHandler : public ::clp_s::search::OutputHandler { +class CountResultsCacheOutputHandler : public search::OutputHandler { public: // Types class OperationFailed : public TraceableException { @@ -300,9 +300,9 @@ class CountResultsCacheOutputHandler : public ::clp_s::search::OutputHandler { // Constructors CountResultsCacheOutputHandler( - std::string const& uri, - std::string const& collection, - std::string archive_id + std::string_view uri, + std::string_view collection, + std::string_view archive_id ); // Methods implementing OutputHandler @@ -335,7 +335,7 @@ class CountResultsCacheOutputHandler : public ::clp_s::search::OutputHandler { * Output handler that performs a count aggregation bucketed by time and writes the results to the * results cache. */ -class CountByTimeResultsCacheOutputHandler : public ::clp_s::search::OutputHandler { +class CountByTimeResultsCacheOutputHandler : public search::OutputHandler { public: // Types class OperationFailed : public TraceableException { @@ -347,21 +347,21 @@ class CountByTimeResultsCacheOutputHandler : public ::clp_s::search::OutputHandl // Constructors CountByTimeResultsCacheOutputHandler( - std::string const& uri, - std::string const& collection, - std::string archive_id, + std::string_view uri, + std::string_view collection, + std::string_view archive_id, int64_t count_by_time_bucket_size_ms ); // Methods implementing OutputHandler auto write( std::string_view message, - epochtime_t timestamp, + epochtime_t timestamp_ms, std::string_view archive_id, int64_t log_event_idx ) -> void override { int64_t bucket - = (timestamp / m_count_by_time_bucket_size_ms) * m_count_by_time_bucket_size_ms; + = (timestamp_ms / m_count_by_time_bucket_size_ms) * m_count_by_time_bucket_size_ms; m_bucket_counts[bucket] += 1; } diff --git a/components/core/src/clp_s/clp-s.cpp b/components/core/src/clp_s/clp-s.cpp index e8bebec2a9..ae94022050 100644 --- a/components/core/src/clp_s/clp-s.cpp +++ b/components/core/src/clp_s/clp-s.cpp @@ -344,7 +344,7 @@ bool search_archive( = std::make_unique( options.uri, options.collection, - std::string{archive_reader->get_archive_id()} + archive_reader->get_archive_id() ); } else if (CommandLineArguments::AggregationType::CountByTime == options.aggregation_type.value()) @@ -353,7 +353,7 @@ bool search_archive( clp_s::CountByTimeResultsCacheOutputHandler >(options.uri, options.collection, - std::string{archive_reader->get_archive_id()}, + archive_reader->get_archive_id(), options.count_by_time_bucket_size_ms); } else { SPDLOG_ERROR("Unhandled aggregation type."); From f1a7ab7e4ccb9801f4123d9a8da73539a47e5005 Mon Sep 17 00:00:00 2001 From: davemarco <83603688+davemarco@users.noreply.github.com> Date: Thu, 25 Jun 2026 11:22:14 -0400 Subject: [PATCH 16/56] refactor(clp-s): Apply clang-format to connect_to_results_cache signature. --- components/core/src/clp_s/OutputHandlerImpl.cpp | 14 ++++---------- 1 file changed, 4 insertions(+), 10 deletions(-) diff --git a/components/core/src/clp_s/OutputHandlerImpl.cpp b/components/core/src/clp_s/OutputHandlerImpl.cpp index c7b0f17c5d..388433c5ee 100644 --- a/components/core/src/clp_s/OutputHandlerImpl.cpp +++ b/components/core/src/clp_s/OutputHandlerImpl.cpp @@ -35,18 +35,12 @@ namespace { * @return The collection. */ template -auto connect_to_results_cache( - string_view uri, - string_view collection, - mongocxx::client& client -) -> mongocxx::collection; +auto connect_to_results_cache(string_view uri, string_view collection, mongocxx::client& client) + -> mongocxx::collection; template -auto connect_to_results_cache( - string_view uri, - string_view collection, - mongocxx::client& client -) -> mongocxx::collection { +auto connect_to_results_cache(string_view uri, string_view collection, mongocxx::client& client) + -> mongocxx::collection { try { auto mongo_uri = mongocxx::uri{string{uri}}; client = mongocxx::client(mongo_uri); From 704ccf596de06485099f8be67257918843969810 Mon Sep 17 00:00:00 2001 From: davemarco <83603688+davemarco@users.noreply.github.com> Date: Thu, 25 Jun 2026 12:45:05 -0400 Subject: [PATCH 17/56] Separate aggregation operator args from output handler args. --- .../core/src/clp_s/CommandLineArguments.cpp | 100 ++++++++---------- .../core/src/clp_s/CommandLineArguments.hpp | 21 ++-- components/core/src/clp_s/clp-s.cpp | 37 ++++--- components/core/src/clp_s/kv_ir_search.cpp | 23 +--- 4 files changed, 80 insertions(+), 101 deletions(-) diff --git a/components/core/src/clp_s/CommandLineArguments.cpp b/components/core/src/clp_s/CommandLineArguments.cpp index d652af4af4..74407a6dd5 100644 --- a/components/core/src/clp_s/CommandLineArguments.cpp +++ b/components/core/src/clp_s/CommandLineArguments.cpp @@ -782,6 +782,19 @@ CommandLineArguments::parse_arguments(int argc, char const** argv) { // clang-format on search_options.add(match_options); + po::options_description aggregation_options("Aggregation Controls"); + // clang-format off + aggregation_options.add_options()( + "count", + "Count the number of results" + )( + "count-by-time", + po::value(&m_count_by_time_bucket_size_ms)->value_name("SIZE"), + "Count the number of results in each time span of the given size (ms)" + ); + // clang-format on + search_options.add(aggregation_options); + NetworkOutputHandlerOptions network_options{}; po::options_description network_output_handler_options( "Network Output Handler Options" @@ -818,15 +831,6 @@ CommandLineArguments::parse_arguments(int argc, char const** argv) { po::value( &reducer_options.job_id)->value_name("ID"), "Job ID for the requested aggregation operation" - )( - "count", - "Count the number of results" - )( - "count-by-time", - po::value( - &reducer_options.count_by_time_bucket_size_ms - )->value_name("SIZE"), - "Count the number of results in each time span of the given size (ms)" ); // clang-format on @@ -861,15 +865,6 @@ CommandLineArguments::parse_arguments(int argc, char const** argv) { po::value( &results_cache_options.dataset)->value_name("DATASET"), "The dataset name to include in each result document" - )( - "count", - "Count the number of results" - )( - "count-by-time", - po::value( - &results_cache_options.count_by_time_bucket_size_ms - )->value_name("SIZE"), - "Count the number of results in each time span of the given size (ms)" ); FileOutputHandlerOptions file_options{}; @@ -882,18 +877,6 @@ CommandLineArguments::parse_arguments(int argc, char const** argv) { StdoutOutputHandlerOptions stdout_options{}; po::options_description stdout_output_handler_options("Stdout Output Handler Options"); - // clang-format off - stdout_output_handler_options.add_options()( - "count", - "Count the number of results" - )( - "count-by-time", - po::value( - &stdout_options.count_by_time_bucket_size_ms - )->value_name("SIZE"), - "Count the number of results in each time span of the given size (ms)" - ); - // clang-format on std::vector unrecognized_options = po::collect_unrecognized(parsed.options, po::include_positional); @@ -960,8 +943,8 @@ CommandLineArguments::parse_arguments(int argc, char const** argv) { R"( "level: INFO" and output a count aggregation to the reducer)" << std::endl; std::cerr << " " << m_program_name << R"( s archives-dir "level: INFO")" - << " " << cReducerOutputHandlerName << " --count" - << " --host localhost" + << " --count" + << " " << cReducerOutputHandlerName << " --host localhost" << " --port 14009" << " --job-id 1" << std::endl; std::cerr << std::endl; @@ -970,10 +953,10 @@ CommandLineArguments::parse_arguments(int argc, char const** argv) { R"( "level: INFO" and output a count aggregation to the results cache)" << std::endl; std::cerr << " " << m_program_name << R"( s archives-dir "level: INFO")" + << " --count" << " " << cResultsCacheOutputHandlerName << " --uri mongodb://127.0.0.1:27017/test" " --collection test" - " --count" << std::endl; std::cerr << std::endl; @@ -986,6 +969,7 @@ CommandLineArguments::parse_arguments(int argc, char const** argv) { po::options_description visible_options; visible_options.add(general_options); visible_options.add(match_options); + visible_options.add(aggregation_options); visible_options.add(file_output_handler_options); visible_options.add(network_output_handler_options); visible_options.add(results_cache_output_handler_options); @@ -1058,7 +1042,35 @@ CommandLineArguments::parse_arguments(int argc, char const** argv) { throw std::invalid_argument("clp-s only supports one output handler at a time"); } + m_aggregation_type = parse_aggregation_options( + parsed_command_line_options, + m_count_by_time_bucket_size_ms + ); + for (auto const& [output_handler_name, output_handler_options] : output_options_map) { + // Validate the requested aggregation (if any) against the selected output handler: + // the reducer requires one, the results cache and stdout treat it as optional, and + // the file and network handlers do not support it. + if (cReducerOutputHandlerName == output_handler_name) { + if (false == m_aggregation_type.has_value()) { + throw std::invalid_argument( + "The reducer output handler currently only supports count and" + " count-by-time aggregations." + ); + } + } else if ((cFileOutputHandlerName == output_handler_name + || cNetworkOutputHandlerName == output_handler_name) + && m_aggregation_type.has_value()) + { + throw std::invalid_argument( + fmt::format( + "The {} output handler does not support count or count-by-time" + " aggregations.", + output_handler_name + ) + ); + } + if (cNetworkOutputHandlerName == output_handler_name) { parse_network_dest_output_handler_options( network_output_handler_options, @@ -1198,17 +1210,6 @@ void CommandLineArguments::parse_reducer_output_handler_options( if (reducer_options.job_id < 0) { throw std::invalid_argument("job-id cannot be negative."); } - - auto const aggregation_type{ - parse_aggregation_options(parsed_options, reducer_options.count_by_time_bucket_size_ms) - }; - if (false == aggregation_type.has_value()) { - throw std::invalid_argument( - "The reducer output handler currently only supports count and" - " count-by-time aggregations." - ); - } - reducer_options.aggregation_type = aggregation_type.value(); } void CommandLineArguments::parse_results_cache_output_handler_options( @@ -1240,11 +1241,6 @@ void CommandLineArguments::parse_results_cache_output_handler_options( if (0 == results_cache_options.max_num_results) { throw std::invalid_argument("max-num-results cannot be 0."); } - - results_cache_options.aggregation_type = parse_aggregation_options( - parsed_options, - results_cache_options.count_by_time_bucket_size_ms - ); } void CommandLineArguments::parse_file_output_handler_options( @@ -1254,6 +1250,7 @@ void CommandLineArguments::parse_file_output_handler_options( ) { po::variables_map parsed_options; parse_subcommand_options(options_description, options, parsed_options); + if (parsed_options.count("path") == 0) { throw std::invalid_argument("path must be specified."); } @@ -1269,11 +1266,6 @@ void CommandLineArguments::parse_stdout_output_handler_options( ) { po::variables_map parsed_options; parse_subcommand_options(options_description, options, parsed_options); - - stdout_options.aggregation_type = parse_aggregation_options( - parsed_options, - stdout_options.count_by_time_bucket_size_ms - ); } void CommandLineArguments::print_basic_usage() const { diff --git a/components/core/src/clp_s/CommandLineArguments.hpp b/components/core/src/clp_s/CommandLineArguments.hpp index ad0d5c31e2..d80524ebbb 100644 --- a/components/core/src/clp_s/CommandLineArguments.hpp +++ b/components/core/src/clp_s/CommandLineArguments.hpp @@ -42,8 +42,6 @@ class CommandLineArguments { std::string dataset; uint64_t batch_size{1000}; uint64_t max_num_results{1000}; - std::optional aggregation_type; - int64_t count_by_time_bucket_size_ms{}; }; struct FileOutputHandlerOptions { @@ -59,14 +57,9 @@ class CommandLineArguments { std::string host; int port{-1}; reducer::job_id_t job_id{-1}; - AggregationType aggregation_type{AggregationType::Count}; - int64_t count_by_time_bucket_size_ms{}; }; - struct StdoutOutputHandlerOptions { - std::optional aggregation_type; - int64_t count_by_time_bucket_size_ms{}; - }; + struct StdoutOutputHandlerOptions {}; using OutputHandlerOptionsVariant = std:: variant std::optional const& { + return m_aggregation_type; + } + + [[nodiscard]] auto get_count_by_time_bucket_size_ms() const -> int64_t { + return m_count_by_time_bucket_size_ms; + } + [[nodiscard]] auto get_retain_float_format() const -> bool { return false == m_no_retain_float_format; } @@ -273,6 +274,10 @@ class CommandLineArguments { bool m_ignore_case{false}; bool m_enable_telemetry{false}; std::vector m_projection_columns; + + // Aggregation operator options (shared across all output handlers) + std::optional m_aggregation_type; + int64_t m_count_by_time_bucket_size_ms{}; }; } // namespace clp_s diff --git a/components/core/src/clp_s/clp-s.cpp b/components/core/src/clp_s/clp-s.cpp index 051375f37d..7664d0aa4c 100644 --- a/components/core/src/clp_s/clp-s.cpp +++ b/components/core/src/clp_s/clp-s.cpp @@ -307,20 +307,20 @@ bool search_archive( options.port ); }, - [&](CommandLineArguments::ReducerOutputHandlerOptions const& options) - -> void { - if (CommandLineArguments::AggregationType::Count - == options.aggregation_type) { + [&](CommandLineArguments::ReducerOutputHandlerOptions const&) -> void { + auto const& aggregation_type + = command_line_arguments.get_aggregation_type(); + if (CommandLineArguments::AggregationType::Count == aggregation_type) { output_handler = std::make_unique( reducer_socket_fd ); } else if (CommandLineArguments::AggregationType::CountByTime - == options.aggregation_type) - { + == aggregation_type) { output_handler = std::make_unique( reducer_socket_fd, - options.count_by_time_bucket_size_ms + command_line_arguments + .get_count_by_time_bucket_size_ms() ); } else { SPDLOG_ERROR("Unhandled aggregation type."); @@ -329,7 +329,9 @@ bool search_archive( }, [&](CommandLineArguments::ResultsCacheOutputHandlerOptions const& options) -> void { - if (false == options.aggregation_type.has_value()) { + auto const& aggregation_type + = command_line_arguments.get_aggregation_type(); + if (false == aggregation_type.has_value()) { output_handler = std::make_unique( options.uri, options.collection, @@ -338,8 +340,7 @@ bool search_archive( options.dataset ); } else if (CommandLineArguments::AggregationType::Count - == options.aggregation_type.value()) - { + == aggregation_type.value()) { output_handler = std::make_unique( options.uri, @@ -347,30 +348,32 @@ bool search_archive( archive_reader->get_archive_id() ); } else if (CommandLineArguments::AggregationType::CountByTime - == options.aggregation_type.value()) + == aggregation_type.value()) { output_handler = std::make_unique< clp_s::CountByTimeResultsCacheOutputHandler >(options.uri, options.collection, archive_reader->get_archive_id(), - options.count_by_time_bucket_size_ms); + command_line_arguments.get_count_by_time_bucket_size_ms()); } else { SPDLOG_ERROR("Unhandled aggregation type."); output_handler = nullptr; } }, - [&](CommandLineArguments::StdoutOutputHandlerOptions const& options) - -> void { - if (false == options.aggregation_type.has_value()) { + [&](CommandLineArguments::StdoutOutputHandlerOptions const&) -> void { + auto const& aggregation_type + = command_line_arguments.get_aggregation_type(); + if (false == aggregation_type.has_value()) { output_handler = std::make_unique(); } else { output_handler = std::make_unique( archive_reader->get_archive_id(), CommandLineArguments::AggregationType::CountByTime - == options.aggregation_type.value(), - options.count_by_time_bucket_size_ms + == aggregation_type.value(), + command_line_arguments + .get_count_by_time_bucket_size_ms() ); } } diff --git a/components/core/src/clp_s/kv_ir_search.cpp b/components/core/src/clp_s/kv_ir_search.cpp index 90c02079b0..2e16ea896b 100644 --- a/components/core/src/clp_s/kv_ir_search.cpp +++ b/components/core/src/clp_s/kv_ir_search.cpp @@ -246,28 +246,7 @@ auto search_kv_ir_stream( return KvIrSearchError{KvIrSearchErrorEnum::ProjectionSupportNotImplemented}; } - auto const& output_handler_options{command_line_arguments.get_output_handler_options()}; - auto const reducer_aggregation_requested{ - std::holds_alternative( - output_handler_options - ) - }; - auto const* results_cache_options - = std::get_if( - &output_handler_options - ); - auto const results_cache_aggregation_requested{ - nullptr != results_cache_options && results_cache_options->aggregation_type.has_value() - }; - auto const* stdout_options = std::get_if( - &output_handler_options - ); - auto const stdout_aggregation_requested{ - nullptr != stdout_options && stdout_options->aggregation_type.has_value() - }; - if (reducer_aggregation_requested || results_cache_aggregation_requested - || stdout_aggregation_requested) - { + if (command_line_arguments.get_aggregation_type().has_value()) { SPDLOG_ERROR("kv-ir search: Aggregation support is not implemented."); return KvIrSearchError{KvIrSearchErrorEnum::CountSupportNotImplemented}; } From 79fda753530f5805c01ce588e6570f20b39a9311 Mon Sep 17 00:00:00 2001 From: davemarco <83603688+davemarco@users.noreply.github.com> Date: Thu, 25 Jun 2026 12:57:30 -0400 Subject: [PATCH 18/56] Drop now-unused default subcommand from collect_subcommands. --- .../core/src/clp_s/CommandLineArguments.cpp | 24 +++++-------------- 1 file changed, 6 insertions(+), 18 deletions(-) diff --git a/components/core/src/clp_s/CommandLineArguments.cpp b/components/core/src/clp_s/CommandLineArguments.cpp index 74407a6dd5..1a79f543b9 100644 --- a/components/core/src/clp_s/CommandLineArguments.cpp +++ b/components/core/src/clp_s/CommandLineArguments.cpp @@ -39,8 +39,7 @@ constexpr std::string_view cStdoutCacheOutputHandlerName{"stdout"}; */ auto collect_subcommands( std::map const& subcommands, - std::vector const& options, - std::optional const& default_subcommand = std::nullopt + std::vector const& options ) -> std::map>; /** @@ -162,8 +161,7 @@ void validate_archive_paths( auto collect_subcommands( std::map const& subcommands, - std::vector const& options, - std::optional const& default_subcommand + std::vector const& options ) -> std::map> { std::optional current_subcommand; std::optional> current_subcommand_arguments; @@ -202,15 +200,7 @@ auto collect_subcommands( if (false == current_subcommand.has_value() || false == current_subcommand_arguments.has_value()) { - // A leading option token (e.g. `--count`) with no named subcommand selects the default - // subcommand, letting its options be specified without naming the subcommand. - if (false == default_subcommand.has_value() || option.empty() || '-' != option.front()) - { - throw std::invalid_argument(fmt::format("unrecognized option \"{}\"", option)); - } - current_subcommand = default_subcommand; - current_subcommand_arguments.emplace(); - current_subcommand_options_description = subcommands.at(default_subcommand.value()); + throw std::invalid_argument(fmt::format("unrecognized option \"{}\"", option)); } current_subcommand_arguments.value().emplace_back(option); @@ -1008,11 +998,9 @@ CommandLineArguments::parse_arguments(int argc, char const** argv) { &results_cache_output_handler_options}, {std::string{cStdoutCacheOutputHandlerName}, &stdout_output_handler_options} }; - auto const output_options_map{collect_subcommands( - subcommands, - unrecognized_output_options, - std::string{cStdoutCacheOutputHandlerName} - )}; + auto const output_options_map{ + collect_subcommands(subcommands, unrecognized_output_options) + }; validate_archive_paths(archive_path, archive_id, m_input_paths); From 77ce3d8f8570724bbcb134c6a035c4aef3f70b81 Mon Sep 17 00:00:00 2001 From: davemarco <83603688+davemarco@users.noreply.github.com> Date: Thu, 25 Jun 2026 13:06:58 -0400 Subject: [PATCH 19/56] Clean up leftover stdout output-handler code. --- .../core/src/clp_s/CommandLineArguments.cpp | 24 ++----------------- .../core/src/clp_s/CommandLineArguments.hpp | 13 ---------- .../core/src/clp_s/OutputHandlerImpl.cpp | 8 +++---- .../core/src/clp_s/OutputHandlerImpl.hpp | 12 ++++++---- 4 files changed, 13 insertions(+), 44 deletions(-) diff --git a/components/core/src/clp_s/CommandLineArguments.cpp b/components/core/src/clp_s/CommandLineArguments.cpp index 1a79f543b9..d985472dc6 100644 --- a/components/core/src/clp_s/CommandLineArguments.cpp +++ b/components/core/src/clp_s/CommandLineArguments.cpp @@ -865,9 +865,6 @@ CommandLineArguments::parse_arguments(int argc, char const** argv) { "File output path" ); - StdoutOutputHandlerOptions stdout_options{}; - po::options_description stdout_output_handler_options("Stdout Output Handler Options"); - std::vector unrecognized_options = po::collect_unrecognized(parsed.options, po::include_positional); unrecognized_options.erase(unrecognized_options.begin()); @@ -964,7 +961,6 @@ CommandLineArguments::parse_arguments(int argc, char const** argv) { visible_options.add(network_output_handler_options); visible_options.add(results_cache_output_handler_options); visible_options.add(reducer_output_handler_options); - visible_options.add(stdout_output_handler_options); std::cerr << visible_options << '\n'; return ParsingResult::InfoCommand; } @@ -996,7 +992,7 @@ CommandLineArguments::parse_arguments(int argc, char const** argv) { {std::string{cReducerOutputHandlerName}, &reducer_output_handler_options}, {std::string{cResultsCacheOutputHandlerName}, &results_cache_output_handler_options}, - {std::string{cStdoutCacheOutputHandlerName}, &stdout_output_handler_options} + {std::string{cStdoutCacheOutputHandlerName}, nullptr} }; auto const output_options_map{ collect_subcommands(subcommands, unrecognized_output_options) @@ -1087,14 +1083,7 @@ CommandLineArguments::parse_arguments(int argc, char const** argv) { std::move(results_cache_options) ); } else if (cStdoutCacheOutputHandlerName == output_handler_name) { - parse_stdout_output_handler_options( - stdout_output_handler_options, - output_handler_options, - stdout_options - ); - m_output_handler_options.emplace( - std::move(stdout_options) - ); + m_output_handler_options.emplace(); } else if (cFileOutputHandlerName == output_handler_name) { parse_file_output_handler_options( file_output_handler_options, @@ -1247,15 +1236,6 @@ void CommandLineArguments::parse_file_output_handler_options( } } -void CommandLineArguments::parse_stdout_output_handler_options( - po::options_description const& options_description, - std::vector const& options, - StdoutOutputHandlerOptions& stdout_options -) { - po::variables_map parsed_options; - parse_subcommand_options(options_description, options, parsed_options); -} - void CommandLineArguments::print_basic_usage() const { std::cerr << "Usage: " << m_program_name << " [OPTIONS] COMMAND [COMMAND ARGUMENTS]" << std::endl; diff --git a/components/core/src/clp_s/CommandLineArguments.hpp b/components/core/src/clp_s/CommandLineArguments.hpp index d80524ebbb..c9e8ffd99f 100644 --- a/components/core/src/clp_s/CommandLineArguments.hpp +++ b/components/core/src/clp_s/CommandLineArguments.hpp @@ -217,19 +217,6 @@ class CommandLineArguments { FileOutputHandlerOptions& file_options ); - /** - * Validates output options related to the stdout output handler. - * @param options_description - * @param options Vector of options previously parsed by boost::program_options and which may - * contain options that have the unrecognized flag set. - * @param stdout_options The parsed representation of the stdout output handler options. - */ - void parse_stdout_output_handler_options( - boost::program_options::options_description const& options_description, - std::vector const& options, - StdoutOutputHandlerOptions& stdout_options - ); - void print_basic_usage() const; void print_compression_usage() const; diff --git a/components/core/src/clp_s/OutputHandlerImpl.cpp b/components/core/src/clp_s/OutputHandlerImpl.cpp index a2820fc393..fea2626821 100644 --- a/components/core/src/clp_s/OutputHandlerImpl.cpp +++ b/components/core/src/clp_s/OutputHandlerImpl.cpp @@ -251,22 +251,22 @@ AggregationToStdoutOutputHandler::AggregationToStdoutOutputHandler( m_pipeline.add_pipeline_stage(std::make_shared()); } -void AggregationToStdoutOutputHandler::write(string_view message) { +auto AggregationToStdoutOutputHandler::write(string_view message) -> void { m_pipeline.push_record(reducer::EmptyRecord{}); } -void AggregationToStdoutOutputHandler::write( +auto AggregationToStdoutOutputHandler::write( string_view message, epochtime_t timestamp_ms, string_view archive_id, int64_t log_event_idx -) { +) -> void { int64_t const bucket = (timestamp_ms / m_count_by_time_bucket_size_ms) * m_count_by_time_bucket_size_ms; m_bucket_counts[bucket] += 1; } -ErrorCode AggregationToStdoutOutputHandler::finish() { +auto AggregationToStdoutOutputHandler::finish() -> ErrorCode { // count-by-time results are serialized from the bucket counts; count results come from the // CountOperator pipeline. `should_output_metadata()` is true only for count-by-time. std::unique_ptr results; diff --git a/components/core/src/clp_s/OutputHandlerImpl.hpp b/components/core/src/clp_s/OutputHandlerImpl.hpp index 9b0a8b2607..2fe535b588 100644 --- a/components/core/src/clp_s/OutputHandlerImpl.hpp +++ b/components/core/src/clp_s/OutputHandlerImpl.hpp @@ -401,23 +401,25 @@ class AggregationToStdoutOutputHandler : public search::OutputHandler { int64_t count_by_time_bucket_size_ms ); - // Methods inherited from OutputHandler - void write( + // Methods implementing OutputHandler + auto write( std::string_view message, epochtime_t timestamp_ms, std::string_view archive_id, int64_t log_event_idx - ) override; + ) -> void override; - void write(std::string_view message) override; + auto write(std::string_view message) -> void override; + // Methods overriding OutputHandler /** * Serializes the aggregation results to stdout as newline-delimited JSON. * @return ErrorCodeSuccess on success */ - ErrorCode finish() override; + auto finish() -> ErrorCode override; private: + // Data members std::string m_archive_id; int64_t m_count_by_time_bucket_size_ms; reducer::Pipeline m_pipeline; From d4ee4b3b9fb1840b82c9595170d8ac349cd23156 Mon Sep 17 00:00:00 2001 From: davemarco <83603688+davemarco@users.noreply.github.com> Date: Thu, 25 Jun 2026 13:16:08 -0400 Subject: [PATCH 20/56] Validate aggregation support in each output handler's parse function. --- .../core/src/clp_s/CommandLineArguments.cpp | 46 +++++++++---------- .../core/src/clp_s/CommandLineArguments.hpp | 7 +++ 2 files changed, 30 insertions(+), 23 deletions(-) diff --git a/components/core/src/clp_s/CommandLineArguments.cpp b/components/core/src/clp_s/CommandLineArguments.cpp index d985472dc6..088dd1c441 100644 --- a/components/core/src/clp_s/CommandLineArguments.cpp +++ b/components/core/src/clp_s/CommandLineArguments.cpp @@ -1032,29 +1032,6 @@ CommandLineArguments::parse_arguments(int argc, char const** argv) { ); for (auto const& [output_handler_name, output_handler_options] : output_options_map) { - // Validate the requested aggregation (if any) against the selected output handler: - // the reducer requires one, the results cache and stdout treat it as optional, and - // the file and network handlers do not support it. - if (cReducerOutputHandlerName == output_handler_name) { - if (false == m_aggregation_type.has_value()) { - throw std::invalid_argument( - "The reducer output handler currently only supports count and" - " count-by-time aggregations." - ); - } - } else if ((cFileOutputHandlerName == output_handler_name - || cNetworkOutputHandlerName == output_handler_name) - && m_aggregation_type.has_value()) - { - throw std::invalid_argument( - fmt::format( - "The {} output handler does not support count or count-by-time" - " aggregations.", - output_handler_name - ) - ); - } - if (cNetworkOutputHandlerName == output_handler_name) { parse_network_dest_output_handler_options( network_output_handler_options, @@ -1119,6 +1096,8 @@ void CommandLineArguments::parse_network_dest_output_handler_options( po::variables_map parsed_options; parse_subcommand_options(options_description, options, parsed_options); + reject_aggregation_for_handler(cNetworkOutputHandlerName); + if (parsed_options.count("host") == 0) { throw std::invalid_argument("host must be specified."); } @@ -1158,6 +1137,18 @@ auto CommandLineArguments::parse_aggregation_options( return aggregation_type; } +void CommandLineArguments::reject_aggregation_for_handler(std::string_view handler_name) const { + if (m_aggregation_type.has_value()) { + throw std::invalid_argument( + fmt::format( + "The {} output handler does not support count or count-by-time" + " aggregations.", + handler_name + ) + ); + } +} + void CommandLineArguments::parse_reducer_output_handler_options( po::options_description const& options_description, std::vector const& options, @@ -1187,6 +1178,13 @@ void CommandLineArguments::parse_reducer_output_handler_options( if (reducer_options.job_id < 0) { throw std::invalid_argument("job-id cannot be negative."); } + + if (false == m_aggregation_type.has_value()) { + throw std::invalid_argument( + "The reducer output handler currently only supports count and count-by-time" + " aggregations." + ); + } } void CommandLineArguments::parse_results_cache_output_handler_options( @@ -1228,6 +1226,8 @@ void CommandLineArguments::parse_file_output_handler_options( po::variables_map parsed_options; parse_subcommand_options(options_description, options, parsed_options); + reject_aggregation_for_handler(cFileOutputHandlerName); + if (parsed_options.count("path") == 0) { throw std::invalid_argument("path must be specified."); } diff --git a/components/core/src/clp_s/CommandLineArguments.hpp b/components/core/src/clp_s/CommandLineArguments.hpp index c9e8ffd99f..fe03a8040a 100644 --- a/components/core/src/clp_s/CommandLineArguments.hpp +++ b/components/core/src/clp_s/CommandLineArguments.hpp @@ -177,6 +177,13 @@ class CommandLineArguments { int64_t count_by_time_bucket_size_ms ) -> std::optional; + /** + * Throws if an aggregation was requested, for output handlers that don't support aggregations. + * @param handler_name The name of the output handler, used in the error message. + * @throws std::invalid_argument if an aggregation operator was specified. + */ + void reject_aggregation_for_handler(std::string_view handler_name) const; + /** * Validates output options related to the Reducer output handler. * @param options_description From 47b13da4c1af38596b647a775883b8ab6e2d86c5 Mon Sep 17 00:00:00 2001 From: davemarco <83603688+davemarco@users.noreply.github.com> Date: Thu, 25 Jun 2026 13:25:21 -0400 Subject: [PATCH 21/56] Drop redundant comment on aggregation members. --- components/core/src/clp_s/CommandLineArguments.hpp | 1 - 1 file changed, 1 deletion(-) diff --git a/components/core/src/clp_s/CommandLineArguments.hpp b/components/core/src/clp_s/CommandLineArguments.hpp index fe03a8040a..8ff787133f 100644 --- a/components/core/src/clp_s/CommandLineArguments.hpp +++ b/components/core/src/clp_s/CommandLineArguments.hpp @@ -269,7 +269,6 @@ class CommandLineArguments { bool m_enable_telemetry{false}; std::vector m_projection_columns; - // Aggregation operator options (shared across all output handlers) std::optional m_aggregation_type; int64_t m_count_by_time_bucket_size_ms{}; }; From 135a6d928c1320944730c21b19eb10612ad9f634 Mon Sep 17 00:00:00 2001 From: davemarco <83603688+davemarco@users.noreply.github.com> Date: Thu, 25 Jun 2026 13:25:21 -0400 Subject: [PATCH 22/56] Select stdout aggregation by type instead of a bool flag. --- components/core/src/clp_s/OutputHandlerImpl.cpp | 14 +++++++++----- components/core/src/clp_s/OutputHandlerImpl.hpp | 14 ++++++++------ components/core/src/clp_s/clp-s.cpp | 3 +-- 3 files changed, 18 insertions(+), 13 deletions(-) diff --git a/components/core/src/clp_s/OutputHandlerImpl.cpp b/components/core/src/clp_s/OutputHandlerImpl.cpp index fea2626821..7442c0090e 100644 --- a/components/core/src/clp_s/OutputHandlerImpl.cpp +++ b/components/core/src/clp_s/OutputHandlerImpl.cpp @@ -241,11 +241,15 @@ auto CountByTimeReducerOutputHandler::finish() -> ErrorCode { AggregationToStdoutOutputHandler::AggregationToStdoutOutputHandler( string_view archive_id, - bool count_by_time, + CommandLineArguments::AggregationType aggregation_type, int64_t count_by_time_bucket_size_ms ) - : search::OutputHandler(count_by_time, false), + : search::OutputHandler( + CommandLineArguments::AggregationType::CountByTime == aggregation_type, + false + ), m_archive_id{archive_id}, + m_aggregation_type{aggregation_type}, m_count_by_time_bucket_size_ms{count_by_time_bucket_size_ms}, m_pipeline{reducer::PipelineInputMode::InterStage} { m_pipeline.add_pipeline_stage(std::make_shared()); @@ -267,10 +271,10 @@ auto AggregationToStdoutOutputHandler::write( } auto AggregationToStdoutOutputHandler::finish() -> ErrorCode { - // count-by-time results are serialized from the bucket counts; count results come from the - // CountOperator pipeline. `should_output_metadata()` is true only for count-by-time. + // Count-by-time results are serialized from the per-bucket counts; count results come from the + // CountOperator pipeline. std::unique_ptr results; - if (should_output_metadata()) { + if (CommandLineArguments::AggregationType::CountByTime == m_aggregation_type) { results = std::make_unique( m_bucket_counts, reducer::CountOperator::cRecordElementKey diff --git a/components/core/src/clp_s/OutputHandlerImpl.hpp b/components/core/src/clp_s/OutputHandlerImpl.hpp index 2fe535b588..d028cd0cd5 100644 --- a/components/core/src/clp_s/OutputHandlerImpl.hpp +++ b/components/core/src/clp_s/OutputHandlerImpl.hpp @@ -17,6 +17,7 @@ #include "../reducer/Pipeline.hpp" #include "../reducer/RecordGroupIterator.hpp" +#include "CommandLineArguments.hpp" #include "Defs.hpp" #include "FileWriter.hpp" #include "search/OutputHandler.hpp" @@ -385,19 +386,19 @@ class CountByTimeResultsCacheOutputHandler : public search::OutputHandler { }; /** - * Output handler that performs a count or count-by-time aggregation and writes the results to - * standard output as newline-delimited JSON, one object per group. + * Output handler that writes a search aggregation's results to standard output as newline-delimited + * JSON, one object per group. * - * Mirroring the reducer count handlers: count accumulates through a `reducer::CountOperator` - * pipeline (yielding a single total), while count-by-time accumulates a count per time bucket. Both - * are serialized through a `reducer::RecordGroupIterator` in `finish()`. + * The aggregation to perform is selected by `AggregationType`. Count accumulates through a + * `reducer::CountOperator` pipeline (yielding a single total); count-by-time accumulates a count + * per time bucket. Results are serialized through a `reducer::RecordGroupIterator` in `finish()`. */ class AggregationToStdoutOutputHandler : public search::OutputHandler { public: // Constructors AggregationToStdoutOutputHandler( std::string_view archive_id, - bool count_by_time, + CommandLineArguments::AggregationType aggregation_type, int64_t count_by_time_bucket_size_ms ); @@ -421,6 +422,7 @@ class AggregationToStdoutOutputHandler : public search::OutputHandler { private: // Data members std::string m_archive_id; + CommandLineArguments::AggregationType m_aggregation_type; int64_t m_count_by_time_bucket_size_ms; reducer::Pipeline m_pipeline; std::map m_bucket_counts; diff --git a/components/core/src/clp_s/clp-s.cpp b/components/core/src/clp_s/clp-s.cpp index 7664d0aa4c..b85d796dbd 100644 --- a/components/core/src/clp_s/clp-s.cpp +++ b/components/core/src/clp_s/clp-s.cpp @@ -370,8 +370,7 @@ bool search_archive( output_handler = std::make_unique( archive_reader->get_archive_id(), - CommandLineArguments::AggregationType::CountByTime - == aggregation_type.value(), + aggregation_type.value(), command_line_arguments .get_count_by_time_bucket_size_ms() ); From abefa20dba6ca4e9e8200674e5102f29b0071c15 Mon Sep 17 00:00:00 2001 From: davemarco <83603688+davemarco@users.noreply.github.com> Date: Thu, 25 Jun 2026 14:22:58 -0400 Subject: [PATCH 23/56] Split stdout aggregation into per-aggregation handlers. --- .../core/src/clp_s/OutputHandlerImpl.cpp | 69 ++++--------------- .../core/src/clp_s/OutputHandlerImpl.hpp | 64 +++++++++++++---- components/core/src/clp_s/clp-s.cpp | 16 ++++- 3 files changed, 75 insertions(+), 74 deletions(-) diff --git a/components/core/src/clp_s/OutputHandlerImpl.cpp b/components/core/src/clp_s/OutputHandlerImpl.cpp index 7442c0090e..07df55e89a 100644 --- a/components/core/src/clp_s/OutputHandlerImpl.cpp +++ b/components/core/src/clp_s/OutputHandlerImpl.cpp @@ -239,67 +239,24 @@ auto CountByTimeReducerOutputHandler::finish() -> ErrorCode { return ErrorCode::ErrorCodeSuccess; } -AggregationToStdoutOutputHandler::AggregationToStdoutOutputHandler( - string_view archive_id, - CommandLineArguments::AggregationType aggregation_type, - int64_t count_by_time_bucket_size_ms -) - : search::OutputHandler( - CommandLineArguments::AggregationType::CountByTime == aggregation_type, - false - ), - m_archive_id{archive_id}, - m_aggregation_type{aggregation_type}, - m_count_by_time_bucket_size_ms{count_by_time_bucket_size_ms}, - m_pipeline{reducer::PipelineInputMode::InterStage} { - m_pipeline.add_pipeline_stage(std::make_shared()); -} - -auto AggregationToStdoutOutputHandler::write(string_view message) -> void { - m_pipeline.push_record(reducer::EmptyRecord{}); -} +auto CountToStdoutOutputHandler::finish() -> ErrorCode { + if (0 == m_count) { + return ErrorCode::ErrorCodeSuccess; + } -auto AggregationToStdoutOutputHandler::write( - string_view message, - epochtime_t timestamp_ms, - string_view archive_id, - int64_t log_event_idx -) -> void { - int64_t const bucket - = (timestamp_ms / m_count_by_time_bucket_size_ms) * m_count_by_time_bucket_size_ms; - m_bucket_counts[bucket] += 1; + nlohmann::json result; + result[constants::results_cache::search::cArchiveId] = m_archive_id; + result[constants::results_cache::search::cCount] = m_count; + std::cout << result.dump() << '\n'; + return ErrorCode::ErrorCodeSuccess; } -auto AggregationToStdoutOutputHandler::finish() -> ErrorCode { - // Count-by-time results are serialized from the per-bucket counts; count results come from the - // CountOperator pipeline. - std::unique_ptr results; - if (CommandLineArguments::AggregationType::CountByTime == m_aggregation_type) { - results = std::make_unique( - m_bucket_counts, - reducer::CountOperator::cRecordElementKey - ); - } else { - results = m_pipeline.finish(); - } - for (; false == results->done(); results->next()) { - auto& group{results->get()}; +auto CountByTimeToStdoutOutputHandler::finish() -> ErrorCode { + for (auto const& [bucket_timestamp, count] : m_bucket_counts) { nlohmann::json result; result[constants::results_cache::search::cArchiveId] = m_archive_id; - - auto const& tags{group.get_tags()}; - if (false == tags.empty()) { - // For count-by-time the group tag is the (stringified) time bucket. - result[constants::results_cache::search::cTimestamp] = std::stoll(tags.front()); - } - - // A count group contains exactly one record holding the count. - auto& record_it{group.record_iter()}; - if (false == record_it.done()) { - result[constants::results_cache::search::cCount] - = record_it.get().get_int64_value(reducer::CountOperator::cRecordElementKey); - } - + result[constants::results_cache::search::cTimestamp] = bucket_timestamp; + result[constants::results_cache::search::cCount] = count; std::cout << result.dump() << '\n'; } return ErrorCode::ErrorCodeSuccess; diff --git a/components/core/src/clp_s/OutputHandlerImpl.hpp b/components/core/src/clp_s/OutputHandlerImpl.hpp index d028cd0cd5..2d2c06502d 100644 --- a/components/core/src/clp_s/OutputHandlerImpl.hpp +++ b/components/core/src/clp_s/OutputHandlerImpl.hpp @@ -386,21 +386,53 @@ class CountByTimeResultsCacheOutputHandler : public search::OutputHandler { }; /** - * Output handler that writes a search aggregation's results to standard output as newline-delimited - * JSON, one object per group. - * - * The aggregation to perform is selected by `AggregationType`. Count accumulates through a - * `reducer::CountOperator` pipeline (yielding a single total); count-by-time accumulates a count - * per time bucket. Results are serialized through a `reducer::RecordGroupIterator` in `finish()`. + * Output handler that performs a count aggregation and writes the result to standard output as a + * JSON object. */ -class AggregationToStdoutOutputHandler : public search::OutputHandler { +class CountToStdoutOutputHandler : public search::OutputHandler { public: // Constructors - AggregationToStdoutOutputHandler( + explicit CountToStdoutOutputHandler(std::string_view archive_id) + : search::OutputHandler{false, false}, + m_archive_id{archive_id} {} + + // Methods implementing OutputHandler + auto write( + std::string_view message, + epochtime_t timestamp, + std::string_view archive_id, + int64_t log_event_idx + ) -> void override {} + + auto write(std::string_view message) -> void override { m_count += 1; } + + // Methods overriding OutputHandler + /** + * Writes the count to standard output as a JSON object. + * @return ErrorCodeSuccess on success + */ + auto finish() -> ErrorCode override; + +private: + // Data members + std::string m_archive_id; + int64_t m_count{}; +}; + +/** + * Output handler that performs a count aggregation bucketed by time and writes the per-bucket + * results to standard output as newline-delimited JSON, one object per time bucket. + */ +class CountByTimeToStdoutOutputHandler : public search::OutputHandler { +public: + // Constructors + CountByTimeToStdoutOutputHandler( std::string_view archive_id, - CommandLineArguments::AggregationType aggregation_type, int64_t count_by_time_bucket_size_ms - ); + ) + : search::OutputHandler{true, false}, + m_archive_id{archive_id}, + m_count_by_time_bucket_size_ms{count_by_time_bucket_size_ms} {} // Methods implementing OutputHandler auto write( @@ -408,13 +440,17 @@ class AggregationToStdoutOutputHandler : public search::OutputHandler { epochtime_t timestamp_ms, std::string_view archive_id, int64_t log_event_idx - ) -> void override; + ) -> void override { + int64_t const bucket + = (timestamp_ms / m_count_by_time_bucket_size_ms) * m_count_by_time_bucket_size_ms; + m_bucket_counts[bucket] += 1; + } - auto write(std::string_view message) -> void override; + auto write(std::string_view message) -> void override {} // Methods overriding OutputHandler /** - * Serializes the aggregation results to stdout as newline-delimited JSON. + * Writes the per-bucket counts to standard output as newline-delimited JSON. * @return ErrorCodeSuccess on success */ auto finish() -> ErrorCode override; @@ -422,9 +458,7 @@ class AggregationToStdoutOutputHandler : public search::OutputHandler { private: // Data members std::string m_archive_id; - CommandLineArguments::AggregationType m_aggregation_type; int64_t m_count_by_time_bucket_size_ms; - reducer::Pipeline m_pipeline; std::map m_bucket_counts; }; diff --git a/components/core/src/clp_s/clp-s.cpp b/components/core/src/clp_s/clp-s.cpp index b85d796dbd..92d76620f4 100644 --- a/components/core/src/clp_s/clp-s.cpp +++ b/components/core/src/clp_s/clp-s.cpp @@ -366,14 +366,24 @@ bool search_archive( = command_line_arguments.get_aggregation_type(); if (false == aggregation_type.has_value()) { output_handler = std::make_unique(); - } else { + } else if (CommandLineArguments::AggregationType::Count + == aggregation_type.value()) { + output_handler + = std::make_unique( + archive_reader->get_archive_id() + ); + } else if (CommandLineArguments::AggregationType::CountByTime + == aggregation_type.value()) + { output_handler - = std::make_unique( + = std::make_unique( archive_reader->get_archive_id(), - aggregation_type.value(), command_line_arguments .get_count_by_time_bucket_size_ms() ); + } else { + SPDLOG_ERROR("Unhandled aggregation type."); + output_handler = nullptr; } } }, From ca0858347cc55e95cd18e34a800491c8d45deddb Mon Sep 17 00:00:00 2001 From: davemarco <83603688+davemarco@users.noreply.github.com> Date: Thu, 25 Jun 2026 14:37:14 -0400 Subject: [PATCH 24/56] Rename stdout count handlers to match siblings and drop unused include. --- components/core/src/clp_s/OutputHandlerImpl.cpp | 5 ++--- components/core/src/clp_s/OutputHandlerImpl.hpp | 8 ++++---- components/core/src/clp_s/clp-s.cpp | 9 ++++----- 3 files changed, 10 insertions(+), 12 deletions(-) diff --git a/components/core/src/clp_s/OutputHandlerImpl.cpp b/components/core/src/clp_s/OutputHandlerImpl.cpp index 07df55e89a..2f9555d019 100644 --- a/components/core/src/clp_s/OutputHandlerImpl.cpp +++ b/components/core/src/clp_s/OutputHandlerImpl.cpp @@ -18,7 +18,6 @@ #include "../reducer/CountOperator.hpp" #include "../reducer/network_utils.hpp" #include "../reducer/Record.hpp" -#include "../reducer/RecordGroup.hpp" #include "../reducer/RecordGroupIterator.hpp" #include "archive_constants.hpp" #include "search/OutputHandler.hpp" @@ -239,7 +238,7 @@ auto CountByTimeReducerOutputHandler::finish() -> ErrorCode { return ErrorCode::ErrorCodeSuccess; } -auto CountToStdoutOutputHandler::finish() -> ErrorCode { +auto CountStdoutOutputHandler::finish() -> ErrorCode { if (0 == m_count) { return ErrorCode::ErrorCodeSuccess; } @@ -251,7 +250,7 @@ auto CountToStdoutOutputHandler::finish() -> ErrorCode { return ErrorCode::ErrorCodeSuccess; } -auto CountByTimeToStdoutOutputHandler::finish() -> ErrorCode { +auto CountByTimeStdoutOutputHandler::finish() -> ErrorCode { for (auto const& [bucket_timestamp, count] : m_bucket_counts) { nlohmann::json result; result[constants::results_cache::search::cArchiveId] = m_archive_id; diff --git a/components/core/src/clp_s/OutputHandlerImpl.hpp b/components/core/src/clp_s/OutputHandlerImpl.hpp index 2d2c06502d..f6269a6d1f 100644 --- a/components/core/src/clp_s/OutputHandlerImpl.hpp +++ b/components/core/src/clp_s/OutputHandlerImpl.hpp @@ -389,10 +389,10 @@ class CountByTimeResultsCacheOutputHandler : public search::OutputHandler { * Output handler that performs a count aggregation and writes the result to standard output as a * JSON object. */ -class CountToStdoutOutputHandler : public search::OutputHandler { +class CountStdoutOutputHandler : public search::OutputHandler { public: // Constructors - explicit CountToStdoutOutputHandler(std::string_view archive_id) + explicit CountStdoutOutputHandler(std::string_view archive_id) : search::OutputHandler{false, false}, m_archive_id{archive_id} {} @@ -423,10 +423,10 @@ class CountToStdoutOutputHandler : public search::OutputHandler { * Output handler that performs a count aggregation bucketed by time and writes the per-bucket * results to standard output as newline-delimited JSON, one object per time bucket. */ -class CountByTimeToStdoutOutputHandler : public search::OutputHandler { +class CountByTimeStdoutOutputHandler : public search::OutputHandler { public: // Constructors - CountByTimeToStdoutOutputHandler( + CountByTimeStdoutOutputHandler( std::string_view archive_id, int64_t count_by_time_bucket_size_ms ) diff --git a/components/core/src/clp_s/clp-s.cpp b/components/core/src/clp_s/clp-s.cpp index 92d76620f4..851d332396 100644 --- a/components/core/src/clp_s/clp-s.cpp +++ b/components/core/src/clp_s/clp-s.cpp @@ -368,15 +368,14 @@ bool search_archive( output_handler = std::make_unique(); } else if (CommandLineArguments::AggregationType::Count == aggregation_type.value()) { - output_handler - = std::make_unique( - archive_reader->get_archive_id() - ); + output_handler = std::make_unique( + archive_reader->get_archive_id() + ); } else if (CommandLineArguments::AggregationType::CountByTime == aggregation_type.value()) { output_handler - = std::make_unique( + = std::make_unique( archive_reader->get_archive_id(), command_line_arguments .get_count_by_time_bucket_size_ms() From 6ef3aa9a96ecc15daf559dfd08987968418ff983 Mon Sep 17 00:00:00 2001 From: davemarco <83603688+davemarco@users.noreply.github.com> Date: Thu, 25 Jun 2026 14:59:48 -0400 Subject: [PATCH 25/56] Reject unrecognized stdout output handler options. --- components/core/src/clp_s/CommandLineArguments.cpp | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/components/core/src/clp_s/CommandLineArguments.cpp b/components/core/src/clp_s/CommandLineArguments.cpp index 088dd1c441..968ecdd4fa 100644 --- a/components/core/src/clp_s/CommandLineArguments.cpp +++ b/components/core/src/clp_s/CommandLineArguments.cpp @@ -1060,6 +1060,13 @@ CommandLineArguments::parse_arguments(int argc, char const** argv) { std::move(results_cache_options) ); } else if (cStdoutCacheOutputHandlerName == output_handler_name) { + if (false == output_handler_options.empty()) { + std::string error_msg{fmt::format( + "stdout output handler does not support \"{}\"", + output_handler_options.front() + )}; + throw std::invalid_argument(error_msg); + } m_output_handler_options.emplace(); } else if (cFileOutputHandlerName == output_handler_name) { parse_file_output_handler_options( From 8381d826f45230f3659b7b84283fbd33ffdff5e2 Mon Sep 17 00:00:00 2001 From: davemarco <83603688+davemarco@users.noreply.github.com> Date: Thu, 25 Jun 2026 15:21:47 -0400 Subject: [PATCH 26/56] Generalize aggregation output handler comments and drop unused include. --- components/core/src/clp_s/CommandLineArguments.cpp | 6 +----- components/core/src/clp_s/CommandLineArguments.hpp | 4 ++-- components/core/src/clp_s/OutputHandlerImpl.cpp | 1 - components/core/src/clp_s/OutputHandlerImpl.hpp | 11 +++++------ 4 files changed, 8 insertions(+), 14 deletions(-) diff --git a/components/core/src/clp_s/CommandLineArguments.cpp b/components/core/src/clp_s/CommandLineArguments.cpp index 968ecdd4fa..8c94c2282b 100644 --- a/components/core/src/clp_s/CommandLineArguments.cpp +++ b/components/core/src/clp_s/CommandLineArguments.cpp @@ -1147,11 +1147,7 @@ auto CommandLineArguments::parse_aggregation_options( void CommandLineArguments::reject_aggregation_for_handler(std::string_view handler_name) const { if (m_aggregation_type.has_value()) { throw std::invalid_argument( - fmt::format( - "The {} output handler does not support count or count-by-time" - " aggregations.", - handler_name - ) + fmt::format("The {} output handler does not support aggregations.", handler_name) ); } } diff --git a/components/core/src/clp_s/CommandLineArguments.hpp b/components/core/src/clp_s/CommandLineArguments.hpp index 8ff787133f..b96ef4dadf 100644 --- a/components/core/src/clp_s/CommandLineArguments.hpp +++ b/components/core/src/clp_s/CommandLineArguments.hpp @@ -178,9 +178,9 @@ class CommandLineArguments { ) -> std::optional; /** - * Throws if an aggregation was requested, for output handlers that don't support aggregations. + * Throws if an aggregation was requested. * @param handler_name The name of the output handler, used in the error message. - * @throws std::invalid_argument if an aggregation operator was specified. + * @throws std::invalid_argument if an aggregation was requested. */ void reject_aggregation_for_handler(std::string_view handler_name) const; diff --git a/components/core/src/clp_s/OutputHandlerImpl.cpp b/components/core/src/clp_s/OutputHandlerImpl.cpp index 2f9555d019..3d8aa76780 100644 --- a/components/core/src/clp_s/OutputHandlerImpl.cpp +++ b/components/core/src/clp_s/OutputHandlerImpl.cpp @@ -18,7 +18,6 @@ #include "../reducer/CountOperator.hpp" #include "../reducer/network_utils.hpp" #include "../reducer/Record.hpp" -#include "../reducer/RecordGroupIterator.hpp" #include "archive_constants.hpp" #include "search/OutputHandler.hpp" #include "TraceableException.hpp" diff --git a/components/core/src/clp_s/OutputHandlerImpl.hpp b/components/core/src/clp_s/OutputHandlerImpl.hpp index f6269a6d1f..6dd09d0c2b 100644 --- a/components/core/src/clp_s/OutputHandlerImpl.hpp +++ b/components/core/src/clp_s/OutputHandlerImpl.hpp @@ -386,8 +386,7 @@ class CountByTimeResultsCacheOutputHandler : public search::OutputHandler { }; /** - * Output handler that performs a count aggregation and writes the result to standard output as a - * JSON object. + * Output handler that performs a count aggregation and writes the results to standard output. */ class CountStdoutOutputHandler : public search::OutputHandler { public: @@ -408,7 +407,7 @@ class CountStdoutOutputHandler : public search::OutputHandler { // Methods overriding OutputHandler /** - * Writes the count to standard output as a JSON object. + * Flushes the count. * @return ErrorCodeSuccess on success */ auto finish() -> ErrorCode override; @@ -420,8 +419,8 @@ class CountStdoutOutputHandler : public search::OutputHandler { }; /** - * Output handler that performs a count aggregation bucketed by time and writes the per-bucket - * results to standard output as newline-delimited JSON, one object per time bucket. + * Output handler that performs a count aggregation bucketed by time and writes the results to + * standard output. */ class CountByTimeStdoutOutputHandler : public search::OutputHandler { public: @@ -450,7 +449,7 @@ class CountByTimeStdoutOutputHandler : public search::OutputHandler { // Methods overriding OutputHandler /** - * Writes the per-bucket counts to standard output as newline-delimited JSON. + * Flushes the counts. * @return ErrorCodeSuccess on success */ auto finish() -> ErrorCode override; From 4c5e4ca67039da7b40e9760c880a06fabdb6998c Mon Sep 17 00:00:00 2001 From: davemarco <83603688+davemarco@users.noreply.github.com> Date: Thu, 25 Jun 2026 16:55:30 -0400 Subject: [PATCH 27/56] Reimplement aggregations as a variant of aggregators with decoupled output sinks; add min/max. --- components/core/src/clp_s/Aggregation.cpp | 140 ++++++++++++++ components/core/src/clp_s/Aggregation.hpp | 123 ++++++++++++ components/core/src/clp_s/CMakeLists.txt | 2 + .../core/src/clp_s/CommandLineArguments.cpp | 68 +++++-- .../core/src/clp_s/CommandLineArguments.hpp | 31 ++- .../core/src/clp_s/OutputHandlerImpl.cpp | 108 +++-------- .../core/src/clp_s/OutputHandlerImpl.hpp | 180 +++++++----------- components/core/src/clp_s/clp-s.cpp | 74 +++---- components/core/src/clp_s/kv_ir_search.cpp | 2 +- 9 files changed, 455 insertions(+), 273 deletions(-) create mode 100644 components/core/src/clp_s/Aggregation.cpp create mode 100644 components/core/src/clp_s/Aggregation.hpp diff --git a/components/core/src/clp_s/Aggregation.cpp b/components/core/src/clp_s/Aggregation.cpp new file mode 100644 index 0000000000..b70d2b5710 --- /dev/null +++ b/components/core/src/clp_s/Aggregation.cpp @@ -0,0 +1,140 @@ +#include "Aggregation.hpp" + +#include +#include +#include +#include +#include +#include + +#include + +#include "archive_constants.hpp" +#include "search/ast/SearchUtils.hpp" + +using std::string; +using std::string_view; + +namespace clp_s { +auto CountAggregation::get_results() const -> std::vector { + if (0 == m_count) { + return {}; + } + AggregationResult result; + result.emplace_back(constants::results_cache::search::cCount, m_count); + return {std::move(result)}; +} + +auto CountByTimeAggregation::get_results() const -> std::vector { + std::vector results; + results.reserve(m_bucket_counts.size()); + for (auto const& [bucket_timestamp, count] : m_bucket_counts) { + AggregationResult result; + result.emplace_back(constants::results_cache::search::cTimestamp, bucket_timestamp); + result.emplace_back(constants::results_cache::search::cCount, count); + results.push_back(std::move(result)); + } + return results; +} + +MinMaxAggregation::MinMaxAggregation(bool find_max, string_view field) + : m_find_max{find_max}, + m_field{field} { + string descriptor_namespace; + if (false + == search::ast::tokenize_column_descriptor( + string{field}, + m_field_path, + descriptor_namespace + )) + { + // A malformed field tokenizes to nothing, so `add_record` never matches and min/max yields + // no result. The field is validated during command-line parsing, so this is just defensive. + m_field_path.clear(); + return; + } + if (false == descriptor_namespace.empty()) { + // Namespaced fields (e.g. the auto-generated `@` namespace) are marshalled under a + // top-level object keyed by the namespace string, so navigate into it first. + m_field_path.insert(m_field_path.begin(), descriptor_namespace); + } +} + +auto MinMaxAggregation::beats_extreme(Extreme candidate) const -> bool { + auto const& current{m_extreme.value()}; + // Compare integers exactly; only fall back to a (lossy) double comparison when the types + // differ. + if (std::holds_alternative(candidate) && std::holds_alternative(current)) { + auto const lhs{std::get(candidate)}; + auto const rhs{std::get(current)}; + return m_find_max ? lhs > rhs : lhs < rhs; + } + auto const as_double{[](Extreme value) { + return std::visit([](auto held) { return static_cast(held); }, value); + }}; + return m_find_max ? as_double(candidate) > as_double(current) + : as_double(candidate) < as_double(current); +} + +auto MinMaxAggregation::add_record(string_view message, epochtime_t) -> void { + nlohmann::json doc; + try { + doc = nlohmann::json::parse(message); + } catch (nlohmann::json::exception const&) { + return; + } + + nlohmann::json const* node{&doc}; + for (auto const& key : m_field_path) { + if (false == node->is_object()) { + return; + } + auto const it{node->find(key)}; + if (node->end() == it) { + return; + } + node = &it.value(); + } + + if (false == node->is_number()) { + return; + } + Extreme const candidate{ + node->is_number_integer() ? Extreme{node->get()} : Extreme{node->get()} + }; + if (false == m_extreme.has_value() || beats_extreme(candidate)) { + m_extreme = candidate; + } +} + +auto MinMaxAggregation::get_results() const -> std::vector { + if (false == m_extreme.has_value()) { + return {}; + } + AggregationResult result; + result.emplace_back(constants::results_cache::search::cField, m_field); + auto const* const key{ + m_find_max ? constants::results_cache::search::cMax + : constants::results_cache::search::cMin + }; + AggregationValue const value{ + std::visit([](auto held) -> AggregationValue { return held; }, m_extreme.value()) + }; + result.emplace_back(key, value); + return {std::move(result)}; +} + +auto aggregation_needs_metadata(Aggregation const& aggregation) -> bool { + return std::visit( + [](auto const& agg) { return std::decay_t::cNeedsMetadata; }, + aggregation + ); +} + +auto aggregation_needs_marshalled_record(Aggregation const& aggregation) -> bool { + return std::visit( + [](auto const& agg) { return std::decay_t::cNeedsMarshalledRecord; }, + aggregation + ); +} +} // namespace clp_s diff --git a/components/core/src/clp_s/Aggregation.hpp b/components/core/src/clp_s/Aggregation.hpp new file mode 100644 index 0000000000..32313a7f4c --- /dev/null +++ b/components/core/src/clp_s/Aggregation.hpp @@ -0,0 +1,123 @@ +#ifndef CLP_S_AGGREGATION_HPP +#define CLP_S_AGGREGATION_HPP + +#include +#include +#include +#include +#include +#include +#include +#include + +#include "Defs.hpp" + +namespace clp_s { +/** + * A single typed value in an aggregation result document. + */ +using AggregationValue = std::variant; + +/** + * One aggregation result document: an ordered list of typed key-value pairs. A sink serializes + * these to its destination (JSON for stdout, BSON for the results cache); the archive id is added + * by the sink, not the aggregator. + */ +using AggregationResult = std::vector>; + +/** + * Counts the number of matched records. + */ +class CountAggregation { +public: + static constexpr bool cNeedsMetadata = false; + static constexpr bool cNeedsMarshalledRecord = false; + + auto + add_record([[maybe_unused]] std::string_view message, [[maybe_unused]] epochtime_t timestamp_ms) + -> void { + m_count += 1; + } + + [[nodiscard]] auto get_results() const -> std::vector; + +private: + int64_t m_count{}; +}; + +/** + * Counts the number of matched records in each time bucket of a fixed size. + */ +class CountByTimeAggregation { +public: + static constexpr bool cNeedsMetadata = true; + static constexpr bool cNeedsMarshalledRecord = false; + + explicit CountByTimeAggregation(int64_t bucket_size_ms) : m_bucket_size_ms{bucket_size_ms} {} + + [[nodiscard]] auto get_bucket_size_ms() const -> int64_t { return m_bucket_size_ms; } + + auto add_record([[maybe_unused]] std::string_view message, epochtime_t timestamp_ms) -> void { + int64_t const bucket = (timestamp_ms / m_bucket_size_ms) * m_bucket_size_ms; + m_bucket_counts[bucket] += 1; + } + + [[nodiscard]] auto get_results() const -> std::vector; + +private: + int64_t m_bucket_size_ms; + std::map m_bucket_counts; +}; + +/** + * Tracks the minimum or maximum value of a target field across matched records. Integer fields are + * compared exactly and only degraded to a (lossy) double comparison when comparing against a + * floating-point value, so integer fields aren't degraded to `double`. + */ +class MinMaxAggregation { +public: + static constexpr bool cNeedsMetadata = false; + static constexpr bool cNeedsMarshalledRecord = true; + + MinMaxAggregation(bool find_max, std::string_view field); + + auto add_record(std::string_view message, [[maybe_unused]] epochtime_t timestamp_ms) -> void; + + [[nodiscard]] auto get_results() const -> std::vector; + +private: + using Extreme = std::variant; + + /** + * @param candidate + * @return Whether `candidate` is more extreme (per the min/max mode) than the current extreme. + */ + [[nodiscard]] auto beats_extreme(Extreme candidate) const -> bool; + + bool m_find_max; + std::string m_field; + std::vector m_field_path; + std::optional m_extreme; +}; + +/** + * A search aggregation to perform. Replaces a `(type, params)` representation: each alternative + * carries exactly the fields its aggregation needs, and `std::visit` dispatches `add_record` / + * `get_results` to the active one. + */ +using Aggregation = std::variant; + +/** + * @param aggregation + * @return Whether the aggregation needs per-record metadata (i.e. the timestamp). + */ +[[nodiscard]] auto aggregation_needs_metadata(Aggregation const& aggregation) -> bool; + +/** + * @param aggregation + * @return Whether the aggregation needs each matched record marshalled into its JSON message. + */ +[[nodiscard]] auto aggregation_needs_marshalled_record(Aggregation const& aggregation) -> bool; +} // namespace clp_s + +#endif // CLP_S_AGGREGATION_HPP diff --git a/components/core/src/clp_s/CMakeLists.txt b/components/core/src/clp_s/CMakeLists.txt index b4bdfad644..ed8d19a963 100644 --- a/components/core/src/clp_s/CMakeLists.txt +++ b/components/core/src/clp_s/CMakeLists.txt @@ -461,6 +461,8 @@ endif() set( CLP_S_EXE_SOURCES + Aggregation.cpp + Aggregation.hpp CommandLineArguments.cpp CommandLineArguments.hpp ErrorCode.hpp diff --git a/components/core/src/clp_s/CommandLineArguments.cpp b/components/core/src/clp_s/CommandLineArguments.cpp index 8c94c2282b..17d4f49290 100644 --- a/components/core/src/clp_s/CommandLineArguments.cpp +++ b/components/core/src/clp_s/CommandLineArguments.cpp @@ -5,6 +5,7 @@ #include #include #include +#include #include #include @@ -13,6 +14,7 @@ #include "../clp/type_utils.hpp" #include "../reducer/types.hpp" #include "FileReader.hpp" +#include "search/ast/SearchUtils.hpp" namespace po = boost::program_options; @@ -772,6 +774,8 @@ CommandLineArguments::parse_arguments(int argc, char const** argv) { // clang-format on search_options.add(match_options); + int64_t count_by_time_bucket_size_ms{}; + std::string aggregation_field; po::options_description aggregation_options("Aggregation Controls"); // clang-format off aggregation_options.add_options()( @@ -779,8 +783,16 @@ CommandLineArguments::parse_arguments(int argc, char const** argv) { "Count the number of results" )( "count-by-time", - po::value(&m_count_by_time_bucket_size_ms)->value_name("SIZE"), + po::value(&count_by_time_bucket_size_ms)->value_name("SIZE"), "Count the number of results in each time span of the given size (ms)" + )( + "min", + po::value(&aggregation_field)->value_name("FIELD"), + "Find the minimum value of the given field" + )( + "max", + po::value(&aggregation_field)->value_name("FIELD"), + "Find the maximum value of the given field" ); // clang-format on search_options.add(aggregation_options); @@ -1026,9 +1038,10 @@ CommandLineArguments::parse_arguments(int argc, char const** argv) { throw std::invalid_argument("clp-s only supports one output handler at a time"); } - m_aggregation_type = parse_aggregation_options( + m_aggregation = parse_aggregation_options( parsed_command_line_options, - m_count_by_time_bucket_size_ms + count_by_time_bucket_size_ms, + aggregation_field ); for (auto const& [output_handler_name, output_handler_options] : output_options_map) { @@ -1122,30 +1135,49 @@ void CommandLineArguments::parse_network_dest_output_handler_options( auto CommandLineArguments::parse_aggregation_options( po::variables_map const& parsed_options, - int64_t count_by_time_bucket_size_ms -) -> std::optional { - std::optional aggregation_type; - if (parsed_options.count("count")) { - aggregation_type = AggregationType::Count; - } - if (parsed_options.count("count-by-time")) { - if (aggregation_type.has_value()) { + int64_t count_by_time_bucket_size_ms, + std::string_view aggregation_field +) -> std::optional { + std::optional aggregation; + auto const set_aggregation = [&](Aggregation value) { + if (aggregation.has_value()) { throw std::invalid_argument( - "The --count-by-time and --count options are mutually exclusive." + "The --count, --count-by-time, --min, and --max options are mutually exclusive." ); } + aggregation = std::move(value); + }; + auto const validate_aggregation_field = [&]() { + if (aggregation_field.empty()) { + throw std::invalid_argument("The --min and --max options require a field."); + } + if (search::ast::has_unescaped_wildcards(aggregation_field)) { + throw std::invalid_argument("The --min and --max field must not contain wildcards."); + } + }; + if (parsed_options.count("count")) { + set_aggregation(CountAggregation{}); + } + if (parsed_options.count("count-by-time")) { if (count_by_time_bucket_size_ms <= 0) { throw std::invalid_argument("Value for count-by-time must be greater than zero."); } - - aggregation_type = AggregationType::CountByTime; + set_aggregation(CountByTimeAggregation{count_by_time_bucket_size_ms}); + } + if (parsed_options.count("min")) { + validate_aggregation_field(); + set_aggregation(MinMaxAggregation{false, aggregation_field}); } - return aggregation_type; + if (parsed_options.count("max")) { + validate_aggregation_field(); + set_aggregation(MinMaxAggregation{true, aggregation_field}); + } + return aggregation; } void CommandLineArguments::reject_aggregation_for_handler(std::string_view handler_name) const { - if (m_aggregation_type.has_value()) { + if (m_aggregation.has_value()) { throw std::invalid_argument( fmt::format("The {} output handler does not support aggregations.", handler_name) ); @@ -1182,7 +1214,9 @@ void CommandLineArguments::parse_reducer_output_handler_options( throw std::invalid_argument("job-id cannot be negative."); } - if (false == m_aggregation_type.has_value()) { + if (false == m_aggregation.has_value() + || std::holds_alternative(m_aggregation.value())) + { throw std::invalid_argument( "The reducer output handler currently only supports count and count-by-time" " aggregations." diff --git a/components/core/src/clp_s/CommandLineArguments.hpp b/components/core/src/clp_s/CommandLineArguments.hpp index b96ef4dadf..69d2e759ea 100644 --- a/components/core/src/clp_s/CommandLineArguments.hpp +++ b/components/core/src/clp_s/CommandLineArguments.hpp @@ -12,6 +12,7 @@ #include #include "../reducer/types.hpp" +#include "Aggregation.hpp" #include "Defs.hpp" #include "InputConfig.hpp" @@ -31,11 +32,6 @@ class CommandLineArguments { Search = 's' }; - enum class AggregationType : uint8_t { - Count, - CountByTime, - }; - struct ResultsCacheOutputHandlerOptions { std::string uri; std::string collection; @@ -123,12 +119,8 @@ class CommandLineArguments { return m_output_handler_options; } - [[nodiscard]] auto get_aggregation_type() const -> std::optional const& { - return m_aggregation_type; - } - - [[nodiscard]] auto get_count_by_time_bucket_size_ms() const -> int64_t { - return m_count_by_time_bucket_size_ms; + [[nodiscard]] auto get_aggregation() const -> std::optional const& { + return m_aggregation; } [[nodiscard]] auto get_retain_float_format() const -> bool { @@ -165,17 +157,21 @@ class CommandLineArguments { ); /** - * Validates the aggregation options (count and count-by-time) for output handlers that - * support aggregations. + * Builds the requested aggregation (count, count-by-time, min, or max) from the parsed options. * @param parsed_options * @param count_by_time_bucket_size_ms The parsed value of the count-by-time option; only * validated when that option was specified. - * @return The requested aggregation type, or std::nullopt if no aggregation was requested. + * @param aggregation_field The parsed value of the min/max option; only validated when one of + * those options was specified. + * @return The requested aggregation, or std::nullopt if no aggregation was requested. + * @throws std::invalid_argument if more than one aggregation is specified, the count-by-time + * bucket size is non-positive, or a min/max field is missing or contains wildcards. */ [[nodiscard]] static auto parse_aggregation_options( boost::program_options::variables_map const& parsed_options, - int64_t count_by_time_bucket_size_ms - ) -> std::optional; + int64_t count_by_time_bucket_size_ms, + std::string_view aggregation_field + ) -> std::optional; /** * Throws if an aggregation was requested. @@ -269,8 +265,7 @@ class CommandLineArguments { bool m_enable_telemetry{false}; std::vector m_projection_columns; - std::optional m_aggregation_type; - int64_t m_count_by_time_bucket_size_ms{}; + std::optional m_aggregation; }; } // namespace clp_s diff --git a/components/core/src/clp_s/OutputHandlerImpl.cpp b/components/core/src/clp_s/OutputHandlerImpl.cpp index 3d8aa76780..03951c2d80 100644 --- a/components/core/src/clp_s/OutputHandlerImpl.cpp +++ b/components/core/src/clp_s/OutputHandlerImpl.cpp @@ -3,8 +3,11 @@ #include #include #include +#include #include +#include +#include #include #include #include @@ -237,102 +240,43 @@ auto CountByTimeReducerOutputHandler::finish() -> ErrorCode { return ErrorCode::ErrorCodeSuccess; } -auto CountStdoutOutputHandler::finish() -> ErrorCode { - if (0 == m_count) { - return ErrorCode::ErrorCodeSuccess; - } - - nlohmann::json result; - result[constants::results_cache::search::cArchiveId] = m_archive_id; - result[constants::results_cache::search::cCount] = m_count; - std::cout << result.dump() << '\n'; - return ErrorCode::ErrorCodeSuccess; -} - -auto CountByTimeStdoutOutputHandler::finish() -> ErrorCode { - for (auto const& [bucket_timestamp, count] : m_bucket_counts) { - nlohmann::json result; - result[constants::results_cache::search::cArchiveId] = m_archive_id; - result[constants::results_cache::search::cTimestamp] = bucket_timestamp; - result[constants::results_cache::search::cCount] = count; - std::cout << result.dump() << '\n'; +auto StdoutSink::write(AggregationResult const& result) -> void { + nlohmann::json document; + document[constants::results_cache::search::cArchiveId] = m_archive_id; + for (auto const& [key, value] : result) { + std::visit([&](auto const& field_value) { document[key] = field_value; }, value); } - return ErrorCode::ErrorCodeSuccess; + std::cout << document.dump() << '\n'; } -CountResultsCacheOutputHandler::CountResultsCacheOutputHandler( - string_view uri, - string_view collection, - string_view archive_id -) - : search::OutputHandler{false, false}, - m_archive_id{archive_id} { +ResultsCacheSink::ResultsCacheSink(string_view uri, string_view collection, string_view archive_id) + : m_archive_id{archive_id} { m_collection = connect_to_results_cache(uri, collection, m_client); } -auto CountResultsCacheOutputHandler::finish() -> ErrorCode { - if (0 == m_count) { - return ErrorCode::ErrorCodeSuccess; - } - - try { - m_collection.insert_one( - bsoncxx::builder::basic::make_document( - bsoncxx::builder::basic::kvp( - constants::results_cache::search::cArchiveId, - m_archive_id - ), - bsoncxx::builder::basic::kvp( - constants::results_cache::search::cCount, - m_count - ) - ) +auto ResultsCacheSink::write(AggregationResult const& result) -> void { + bsoncxx::builder::basic::document document; + document.append( + bsoncxx::builder::basic::kvp(constants::results_cache::search::cArchiveId, m_archive_id) + ); + for (auto const& [key, value] : result) { + std::visit( + [&](auto const& field_value) { + document.append(bsoncxx::builder::basic::kvp(key, field_value)); + }, + value ); - } catch (mongocxx::exception const& e) { - return ErrorCode::ErrorCodeFailureDbBulkWrite; } - return ErrorCode::ErrorCodeSuccess; -} - -CountByTimeResultsCacheOutputHandler::CountByTimeResultsCacheOutputHandler( - string_view uri, - string_view collection, - string_view archive_id, - int64_t count_by_time_bucket_size_ms -) - : search::OutputHandler{true, false}, - m_archive_id{archive_id}, - m_count_by_time_bucket_size_ms{count_by_time_bucket_size_ms} { - m_collection = connect_to_results_cache(uri, collection, m_client); + m_results.push_back(document.extract()); } -auto CountByTimeResultsCacheOutputHandler::finish() -> ErrorCode { - if (m_bucket_counts.empty()) { +auto ResultsCacheSink::finish() -> ErrorCode { + if (m_results.empty()) { return ErrorCode::ErrorCodeSuccess; } try { - std::vector documents; - documents.reserve(m_bucket_counts.size()); - for (auto const& [bucket_timestamp, count] : m_bucket_counts) { - documents.emplace_back( - bsoncxx::builder::basic::make_document( - bsoncxx::builder::basic::kvp( - constants::results_cache::search::cArchiveId, - m_archive_id - ), - bsoncxx::builder::basic::kvp( - constants::results_cache::search::cTimestamp, - bucket_timestamp - ), - bsoncxx::builder::basic::kvp( - constants::results_cache::search::cCount, - count - ) - ) - ); - } - m_collection.insert_many(documents); + m_collection.insert_many(m_results); } catch (mongocxx::exception const& e) { return ErrorCode::ErrorCodeFailureDbBulkWrite; } diff --git a/components/core/src/clp_s/OutputHandlerImpl.hpp b/components/core/src/clp_s/OutputHandlerImpl.hpp index 6dd09d0c2b..19fbca87fd 100644 --- a/components/core/src/clp_s/OutputHandlerImpl.hpp +++ b/components/core/src/clp_s/OutputHandlerImpl.hpp @@ -7,9 +7,11 @@ #include #include +#include #include #include #include +#include #include #include @@ -17,6 +19,7 @@ #include "../reducer/Pipeline.hpp" #include "../reducer/RecordGroupIterator.hpp" +#include "Aggregation.hpp" #include "CommandLineArguments.hpp" #include "Defs.hpp" #include "FileWriter.hpp" @@ -287,56 +290,58 @@ class CountByTimeReducerOutputHandler : public search::OutputHandler { }; /** - * Output handler that performs a count aggregation and writes the results to the results cache. + * Consumes a search aggregation's result documents and writes them to a destination. */ -class CountResultsCacheOutputHandler : public search::OutputHandler { +class AggregationSink { public: - // Types - class OperationFailed : public TraceableException { - public: - // Constructors - OperationFailed(ErrorCode error_code, char const* const filename, int line_number) - : TraceableException(error_code, filename, line_number) {} - }; - // Constructors - CountResultsCacheOutputHandler( - std::string_view uri, - std::string_view collection, - std::string_view archive_id - ); + AggregationSink() = default; - // Methods implementing OutputHandler - auto write( - std::string_view message, - epochtime_t timestamp, - std::string_view archive_id, - int64_t log_event_idx - ) -> void override {} + // Destructor + virtual ~AggregationSink() = default; - auto write(std::string_view message) -> void override { m_count += 1; } + // Delete copy and move + AggregationSink(AggregationSink const&) = delete; + auto operator=(AggregationSink const&) -> AggregationSink& = delete; + AggregationSink(AggregationSink&&) = delete; + auto operator=(AggregationSink&&) -> AggregationSink& = delete; - // Methods overriding OutputHandler + // Methods /** - * Flushes the count. + * Writes one result document. The archive id is added by the sink. + * @param result + */ + virtual auto write(AggregationResult const& result) -> void = 0; + + /** + * Flushes any buffered results. * @return ErrorCodeSuccess on success - * @return ErrorCodeFailureDbBulkWrite on database error */ - auto finish() -> ErrorCode override; + [[nodiscard]] virtual auto finish() -> ErrorCode = 0; +}; + +/** + * Sink that writes aggregation results to standard output as newline-delimited JSON. + */ +class StdoutSink : public AggregationSink { +public: + // Constructors + explicit StdoutSink(std::string_view archive_id) : m_archive_id{archive_id} {} + + // Methods implementing AggregationSink + auto write(AggregationResult const& result) -> void override; + + [[nodiscard]] auto finish() -> ErrorCode override { return ErrorCode::ErrorCodeSuccess; } private: // Data members - mongocxx::client m_client; - mongocxx::collection m_collection; std::string m_archive_id; - int64_t m_count{}; }; /** - * Output handler that performs a count aggregation bucketed by time and writes the results to the - * results cache. + * Sink that writes aggregation results to a MongoDB results-cache collection. */ -class CountByTimeResultsCacheOutputHandler : public search::OutputHandler { +class ResultsCacheSink : public AggregationSink { public: // Types class OperationFailed : public TraceableException { @@ -347,91 +352,42 @@ class CountByTimeResultsCacheOutputHandler : public search::OutputHandler { }; // Constructors - CountByTimeResultsCacheOutputHandler( + ResultsCacheSink( std::string_view uri, std::string_view collection, - std::string_view archive_id, - int64_t count_by_time_bucket_size_ms + std::string_view archive_id ); - // Methods implementing OutputHandler - auto write( - std::string_view message, - epochtime_t timestamp_ms, - std::string_view archive_id, - int64_t log_event_idx - ) -> void override { - int64_t bucket - = (timestamp_ms / m_count_by_time_bucket_size_ms) * m_count_by_time_bucket_size_ms; - m_bucket_counts[bucket] += 1; - } - - auto write(std::string_view message) -> void override {} + // Methods implementing AggregationSink + auto write(AggregationResult const& result) -> void override; - // Methods overriding OutputHandler /** - * Flushes the counts. + * Flushes the buffered result documents to the results cache. * @return ErrorCodeSuccess on success * @return ErrorCodeFailureDbBulkWrite on database error */ - auto finish() -> ErrorCode override; + [[nodiscard]] auto finish() -> ErrorCode override; private: // Data members mongocxx::client m_client; mongocxx::collection m_collection; std::string m_archive_id; - std::map m_bucket_counts; - int64_t m_count_by_time_bucket_size_ms; -}; - -/** - * Output handler that performs a count aggregation and writes the results to standard output. - */ -class CountStdoutOutputHandler : public search::OutputHandler { -public: - // Constructors - explicit CountStdoutOutputHandler(std::string_view archive_id) - : search::OutputHandler{false, false}, - m_archive_id{archive_id} {} - - // Methods implementing OutputHandler - auto write( - std::string_view message, - epochtime_t timestamp, - std::string_view archive_id, - int64_t log_event_idx - ) -> void override {} - - auto write(std::string_view message) -> void override { m_count += 1; } - - // Methods overriding OutputHandler - /** - * Flushes the count. - * @return ErrorCodeSuccess on success - */ - auto finish() -> ErrorCode override; - -private: - // Data members - std::string m_archive_id; - int64_t m_count{}; + std::vector m_results; }; /** - * Output handler that performs a count aggregation bucketed by time and writes the results to - * standard output. + * Output handler that runs a search `Aggregation` and writes its results to an `AggregationSink`. + * The aggregation (the *what*) and the sink (the *where*) compose: adding an aggregation is one + * `Aggregation` alternative, adding a destination is one `AggregationSink`. */ -class CountByTimeStdoutOutputHandler : public search::OutputHandler { +class AggregationOutputHandler : public search::OutputHandler { public: // Constructors - CountByTimeStdoutOutputHandler( - std::string_view archive_id, - int64_t count_by_time_bucket_size_ms - ) - : search::OutputHandler{true, false}, - m_archive_id{archive_id}, - m_count_by_time_bucket_size_ms{count_by_time_bucket_size_ms} {} + AggregationOutputHandler(Aggregation aggregation, std::unique_ptr sink) + : search::OutputHandler{aggregation_needs_metadata(aggregation), aggregation_needs_marshalled_record(aggregation)}, + m_aggregation{std::move(aggregation)}, + m_sink{std::move(sink)} {} // Methods implementing OutputHandler auto write( @@ -440,25 +396,37 @@ class CountByTimeStdoutOutputHandler : public search::OutputHandler { std::string_view archive_id, int64_t log_event_idx ) -> void override { - int64_t const bucket - = (timestamp_ms / m_count_by_time_bucket_size_ms) * m_count_by_time_bucket_size_ms; - m_bucket_counts[bucket] += 1; + std::visit( + [&](auto& aggregation) { aggregation.add_record(message, timestamp_ms); }, + m_aggregation + ); } - auto write(std::string_view message) -> void override {} + auto write(std::string_view message) -> void override { + std::visit([&](auto& aggregation) { aggregation.add_record(message, 0); }, m_aggregation); + } // Methods overriding OutputHandler /** - * Flushes the counts. + * Drains the aggregation's results into the sink. * @return ErrorCodeSuccess on success + * @return The sink's error code on failure */ - auto finish() -> ErrorCode override; + auto finish() -> ErrorCode override { + auto const results{std::visit( + [](auto& aggregation) { return aggregation.get_results(); }, + m_aggregation + )}; + for (auto const& result : results) { + m_sink->write(result); + } + return m_sink->finish(); + } private: // Data members - std::string m_archive_id; - int64_t m_count_by_time_bucket_size_ms; - std::map m_bucket_counts; + Aggregation m_aggregation; + std::unique_ptr m_sink; }; /** diff --git a/components/core/src/clp_s/clp-s.cpp b/components/core/src/clp_s/clp-s.cpp index 851d332396..e17a19b76e 100644 --- a/components/core/src/clp_s/clp-s.cpp +++ b/components/core/src/clp_s/clp-s.cpp @@ -308,30 +308,28 @@ bool search_archive( ); }, [&](CommandLineArguments::ReducerOutputHandlerOptions const&) -> void { - auto const& aggregation_type - = command_line_arguments.get_aggregation_type(); - if (CommandLineArguments::AggregationType::Count == aggregation_type) { + // The reducer only supports count and count-by-time; min/max are + // rejected during command-line parsing. + auto const& aggregation{ + command_line_arguments.get_aggregation().value() + }; + if (std::holds_alternative(aggregation)) { output_handler = std::make_unique( reducer_socket_fd ); - } else if (CommandLineArguments::AggregationType::CountByTime - == aggregation_type) { + } else { output_handler = std::make_unique( reducer_socket_fd, - command_line_arguments - .get_count_by_time_bucket_size_ms() + std::get(aggregation) + .get_bucket_size_ms() ); - } else { - SPDLOG_ERROR("Unhandled aggregation type."); - output_handler = nullptr; } }, [&](CommandLineArguments::ResultsCacheOutputHandlerOptions const& options) -> void { - auto const& aggregation_type - = command_line_arguments.get_aggregation_type(); - if (false == aggregation_type.has_value()) { + auto const& aggregation{command_line_arguments.get_aggregation()}; + if (false == aggregation.has_value()) { output_handler = std::make_unique( options.uri, options.collection, @@ -339,50 +337,28 @@ bool search_archive( options.max_num_results, options.dataset ); - } else if (CommandLineArguments::AggregationType::Count - == aggregation_type.value()) { - output_handler - = std::make_unique( + } else { + output_handler = std::make_unique( + aggregation.value(), + std::make_unique( options.uri, options.collection, archive_reader->get_archive_id() - ); - } else if (CommandLineArguments::AggregationType::CountByTime - == aggregation_type.value()) - { - output_handler = std::make_unique< - clp_s::CountByTimeResultsCacheOutputHandler - >(options.uri, - options.collection, - archive_reader->get_archive_id(), - command_line_arguments.get_count_by_time_bucket_size_ms()); - } else { - SPDLOG_ERROR("Unhandled aggregation type."); - output_handler = nullptr; + ) + ); } }, [&](CommandLineArguments::StdoutOutputHandlerOptions const&) -> void { - auto const& aggregation_type - = command_line_arguments.get_aggregation_type(); - if (false == aggregation_type.has_value()) { + auto const& aggregation{command_line_arguments.get_aggregation()}; + if (false == aggregation.has_value()) { output_handler = std::make_unique(); - } else if (CommandLineArguments::AggregationType::Count - == aggregation_type.value()) { - output_handler = std::make_unique( - archive_reader->get_archive_id() - ); - } else if (CommandLineArguments::AggregationType::CountByTime - == aggregation_type.value()) - { - output_handler - = std::make_unique( - archive_reader->get_archive_id(), - command_line_arguments - .get_count_by_time_bucket_size_ms() - ); } else { - SPDLOG_ERROR("Unhandled aggregation type."); - output_handler = nullptr; + output_handler = std::make_unique( + aggregation.value(), + std::make_unique( + archive_reader->get_archive_id() + ) + ); } } }, diff --git a/components/core/src/clp_s/kv_ir_search.cpp b/components/core/src/clp_s/kv_ir_search.cpp index 2e16ea896b..65b969d3f9 100644 --- a/components/core/src/clp_s/kv_ir_search.cpp +++ b/components/core/src/clp_s/kv_ir_search.cpp @@ -246,7 +246,7 @@ auto search_kv_ir_stream( return KvIrSearchError{KvIrSearchErrorEnum::ProjectionSupportNotImplemented}; } - if (command_line_arguments.get_aggregation_type().has_value()) { + if (command_line_arguments.get_aggregation().has_value()) { SPDLOG_ERROR("kv-ir search: Aggregation support is not implemented."); return KvIrSearchError{KvIrSearchErrorEnum::CountSupportNotImplemented}; } From 06a0f8e40cc71c213680da27f1a0add646279fae Mon Sep 17 00:00:00 2001 From: davemarco <83603688+davemarco@users.noreply.github.com> Date: Fri, 26 Jun 2026 18:05:36 -0400 Subject: [PATCH 28/56] Extract aggregation sinks and results-cache connect helper into their own files. Move AggregationSink/StdoutSink/ResultsCacheSink out of OutputHandlerImpl into AggregationSink.{hpp,cpp} (they are a distinct abstraction from search::OutputHandler), and lift the shared connect_to_results_cache helper into ResultsCacheUtils.hpp so both ResultsCacheOutputHandler and ResultsCacheSink share one definition instead of a file-private one. AggregationOutputHandler stays in OutputHandlerImpl since it is a search::OutputHandler. --- components/core/src/clp_s/AggregationSink.cpp | 63 +++++++++++ components/core/src/clp_s/AggregationSink.hpp | 105 ++++++++++++++++++ components/core/src/clp_s/CMakeLists.txt | 3 + .../core/src/clp_s/OutputHandlerImpl.cpp | 72 +----------- .../core/src/clp_s/OutputHandlerImpl.hpp | 89 +-------------- .../core/src/clp_s/ResultsCacheUtils.hpp | 39 +++++++ 6 files changed, 213 insertions(+), 158 deletions(-) create mode 100644 components/core/src/clp_s/AggregationSink.cpp create mode 100644 components/core/src/clp_s/AggregationSink.hpp create mode 100644 components/core/src/clp_s/ResultsCacheUtils.hpp diff --git a/components/core/src/clp_s/AggregationSink.cpp b/components/core/src/clp_s/AggregationSink.cpp new file mode 100644 index 0000000000..d50ab9ee8d --- /dev/null +++ b/components/core/src/clp_s/AggregationSink.cpp @@ -0,0 +1,63 @@ +#include "AggregationSink.hpp" + +#include +#include +#include + +#include +#include +#include +#include +#include +#include + +#include "archive_constants.hpp" +#include "ErrorCode.hpp" +#include "ResultsCacheUtils.hpp" + +using std::string_view; + +namespace clp_s { +auto StdoutSink::write(AggregationResult const& result) -> void { + nlohmann::json document; + document[constants::results_cache::search::cArchiveId] = m_archive_id; + for (auto const& [key, value] : result) { + std::visit([&](auto const& field_value) { document[key] = field_value; }, value); + } + std::cout << document.dump() << '\n'; +} + +ResultsCacheSink::ResultsCacheSink(string_view uri, string_view collection, string_view archive_id) + : m_archive_id{archive_id} { + m_collection = connect_to_results_cache(uri, collection, m_client); +} + +auto ResultsCacheSink::write(AggregationResult const& result) -> void { + bsoncxx::builder::basic::document document; + document.append( + bsoncxx::builder::basic::kvp(constants::results_cache::search::cArchiveId, m_archive_id) + ); + for (auto const& [key, value] : result) { + std::visit( + [&](auto const& field_value) { + document.append(bsoncxx::builder::basic::kvp(key, field_value)); + }, + value + ); + } + m_results.push_back(document.extract()); +} + +auto ResultsCacheSink::finish() -> ErrorCode { + if (m_results.empty()) { + return ErrorCode::ErrorCodeSuccess; + } + + try { + m_collection.insert_many(m_results); + } catch (mongocxx::exception const& e) { + return ErrorCode::ErrorCodeFailureDbBulkWrite; + } + return ErrorCode::ErrorCodeSuccess; +} +} // namespace clp_s diff --git a/components/core/src/clp_s/AggregationSink.hpp b/components/core/src/clp_s/AggregationSink.hpp new file mode 100644 index 0000000000..2b05ece16e --- /dev/null +++ b/components/core/src/clp_s/AggregationSink.hpp @@ -0,0 +1,105 @@ +#ifndef CLP_S_AGGREGATIONSINK_HPP +#define CLP_S_AGGREGATIONSINK_HPP + +#include +#include +#include + +#include +#include +#include + +#include "Aggregation.hpp" +#include "ErrorCode.hpp" +#include "TraceableException.hpp" + +namespace clp_s { +/** + * Consumes a search aggregation's result documents and writes them to a destination. + */ +class AggregationSink { +public: + // Constructors + AggregationSink() = default; + + // Destructor + virtual ~AggregationSink() = default; + + // Delete copy and move + AggregationSink(AggregationSink const&) = delete; + auto operator=(AggregationSink const&) -> AggregationSink& = delete; + AggregationSink(AggregationSink&&) = delete; + auto operator=(AggregationSink&&) -> AggregationSink& = delete; + + // Methods + /** + * Writes one result document. The archive id is added by the sink. + * @param result + */ + virtual auto write(AggregationResult const& result) -> void = 0; + + /** + * Flushes any buffered results. + * @return ErrorCodeSuccess on success + */ + [[nodiscard]] virtual auto finish() -> ErrorCode = 0; +}; + +/** + * Sink that writes aggregation results to standard output as newline-delimited JSON. + */ +class StdoutSink : public AggregationSink { +public: + // Constructors + explicit StdoutSink(std::string_view archive_id) : m_archive_id{archive_id} {} + + // Methods implementing AggregationSink + auto write(AggregationResult const& result) -> void override; + + [[nodiscard]] auto finish() -> ErrorCode override { return ErrorCode::ErrorCodeSuccess; } + +private: + // Data members + std::string m_archive_id; +}; + +/** + * Sink that writes aggregation results to a MongoDB results-cache collection. + */ +class ResultsCacheSink : public AggregationSink { +public: + // Types + class OperationFailed : public TraceableException { + public: + // Constructors + OperationFailed(ErrorCode error_code, char const* const filename, int line_number) + : TraceableException(error_code, filename, line_number) {} + }; + + // Constructors + ResultsCacheSink( + std::string_view uri, + std::string_view collection, + std::string_view archive_id + ); + + // Methods implementing AggregationSink + auto write(AggregationResult const& result) -> void override; + + /** + * Flushes the buffered result documents to the results cache. + * @return ErrorCodeSuccess on success + * @return ErrorCodeFailureDbBulkWrite on database error + */ + [[nodiscard]] auto finish() -> ErrorCode override; + +private: + // Data members + mongocxx::client m_client; + mongocxx::collection m_collection; + std::string m_archive_id; + std::vector m_results; +}; +} // namespace clp_s + +#endif // CLP_S_AGGREGATIONSINK_HPP diff --git a/components/core/src/clp_s/CMakeLists.txt b/components/core/src/clp_s/CMakeLists.txt index ed8d19a963..cdd07ab1c8 100644 --- a/components/core/src/clp_s/CMakeLists.txt +++ b/components/core/src/clp_s/CMakeLists.txt @@ -463,6 +463,8 @@ set( CLP_S_EXE_SOURCES Aggregation.cpp Aggregation.hpp + AggregationSink.cpp + AggregationSink.hpp CommandLineArguments.cpp CommandLineArguments.hpp ErrorCode.hpp @@ -470,6 +472,7 @@ set( kv_ir_search.hpp OutputHandlerImpl.cpp OutputHandlerImpl.hpp + ResultsCacheUtils.hpp TraceableException.hpp ) diff --git a/components/core/src/clp_s/OutputHandlerImpl.cpp b/components/core/src/clp_s/OutputHandlerImpl.cpp index 03951c2d80..37a26e4a6d 100644 --- a/components/core/src/clp_s/OutputHandlerImpl.cpp +++ b/components/core/src/clp_s/OutputHandlerImpl.cpp @@ -3,7 +3,6 @@ #include #include #include -#include #include #include @@ -12,9 +11,7 @@ #include #include #include -#include #include -#include #include #include "../clp/networking/socket_utils.hpp" @@ -22,6 +19,7 @@ #include "../reducer/network_utils.hpp" #include "../reducer/Record.hpp" #include "archive_constants.hpp" +#include "ResultsCacheUtils.hpp" #include "search/OutputHandler.hpp" #include "TraceableException.hpp" @@ -29,32 +27,6 @@ using std::string; using std::string_view; namespace clp_s { -namespace { -/** - * Connects to the results cache and returns the requested collection. - * @tparam OperationFailedT The type of exception to throw on failure. - * @param uri - * @param collection - * @param client Returns the connected client. - * @return The collection. - */ -template -auto connect_to_results_cache(string_view uri, string_view collection, mongocxx::client& client) - -> mongocxx::collection; - -template -auto connect_to_results_cache(string_view uri, string_view collection, mongocxx::client& client) - -> mongocxx::collection { - try { - auto mongo_uri = mongocxx::uri{string{uri}}; - client = mongocxx::client(mongo_uri); - return client[mongo_uri.database()][string{collection}]; - } catch (mongocxx::exception const& e) { - throw OperationFailedT(ErrorCode::ErrorCodeBadParamDbUri, __FILENAME__, __LINE__); - } -} -} // namespace - void FileOutputHandler::write( string_view message, epochtime_t timestamp, @@ -240,46 +212,4 @@ auto CountByTimeReducerOutputHandler::finish() -> ErrorCode { return ErrorCode::ErrorCodeSuccess; } -auto StdoutSink::write(AggregationResult const& result) -> void { - nlohmann::json document; - document[constants::results_cache::search::cArchiveId] = m_archive_id; - for (auto const& [key, value] : result) { - std::visit([&](auto const& field_value) { document[key] = field_value; }, value); - } - std::cout << document.dump() << '\n'; -} - -ResultsCacheSink::ResultsCacheSink(string_view uri, string_view collection, string_view archive_id) - : m_archive_id{archive_id} { - m_collection = connect_to_results_cache(uri, collection, m_client); -} - -auto ResultsCacheSink::write(AggregationResult const& result) -> void { - bsoncxx::builder::basic::document document; - document.append( - bsoncxx::builder::basic::kvp(constants::results_cache::search::cArchiveId, m_archive_id) - ); - for (auto const& [key, value] : result) { - std::visit( - [&](auto const& field_value) { - document.append(bsoncxx::builder::basic::kvp(key, field_value)); - }, - value - ); - } - m_results.push_back(document.extract()); -} - -auto ResultsCacheSink::finish() -> ErrorCode { - if (m_results.empty()) { - return ErrorCode::ErrorCodeSuccess; - } - - try { - m_collection.insert_many(m_results); - } catch (mongocxx::exception const& e) { - return ErrorCode::ErrorCodeFailureDbBulkWrite; - } - return ErrorCode::ErrorCodeSuccess; -} } // namespace clp_s diff --git a/components/core/src/clp_s/OutputHandlerImpl.hpp b/components/core/src/clp_s/OutputHandlerImpl.hpp index 19fbca87fd..415431ac7c 100644 --- a/components/core/src/clp_s/OutputHandlerImpl.hpp +++ b/components/core/src/clp_s/OutputHandlerImpl.hpp @@ -20,6 +20,7 @@ #include "../reducer/Pipeline.hpp" #include "../reducer/RecordGroupIterator.hpp" #include "Aggregation.hpp" +#include "AggregationSink.hpp" #include "CommandLineArguments.hpp" #include "Defs.hpp" #include "FileWriter.hpp" @@ -289,93 +290,6 @@ class CountByTimeReducerOutputHandler : public search::OutputHandler { int64_t m_count_by_time_bucket_size_ms; }; -/** - * Consumes a search aggregation's result documents and writes them to a destination. - */ -class AggregationSink { -public: - // Constructors - AggregationSink() = default; - - // Destructor - virtual ~AggregationSink() = default; - - // Delete copy and move - AggregationSink(AggregationSink const&) = delete; - auto operator=(AggregationSink const&) -> AggregationSink& = delete; - AggregationSink(AggregationSink&&) = delete; - auto operator=(AggregationSink&&) -> AggregationSink& = delete; - - // Methods - /** - * Writes one result document. The archive id is added by the sink. - * @param result - */ - virtual auto write(AggregationResult const& result) -> void = 0; - - /** - * Flushes any buffered results. - * @return ErrorCodeSuccess on success - */ - [[nodiscard]] virtual auto finish() -> ErrorCode = 0; -}; - -/** - * Sink that writes aggregation results to standard output as newline-delimited JSON. - */ -class StdoutSink : public AggregationSink { -public: - // Constructors - explicit StdoutSink(std::string_view archive_id) : m_archive_id{archive_id} {} - - // Methods implementing AggregationSink - auto write(AggregationResult const& result) -> void override; - - [[nodiscard]] auto finish() -> ErrorCode override { return ErrorCode::ErrorCodeSuccess; } - -private: - // Data members - std::string m_archive_id; -}; - -/** - * Sink that writes aggregation results to a MongoDB results-cache collection. - */ -class ResultsCacheSink : public AggregationSink { -public: - // Types - class OperationFailed : public TraceableException { - public: - // Constructors - OperationFailed(ErrorCode error_code, char const* const filename, int line_number) - : TraceableException(error_code, filename, line_number) {} - }; - - // Constructors - ResultsCacheSink( - std::string_view uri, - std::string_view collection, - std::string_view archive_id - ); - - // Methods implementing AggregationSink - auto write(AggregationResult const& result) -> void override; - - /** - * Flushes the buffered result documents to the results cache. - * @return ErrorCodeSuccess on success - * @return ErrorCodeFailureDbBulkWrite on database error - */ - [[nodiscard]] auto finish() -> ErrorCode override; - -private: - // Data members - mongocxx::client m_client; - mongocxx::collection m_collection; - std::string m_archive_id; - std::vector m_results; -}; - /** * Output handler that runs a search `Aggregation` and writes its results to an `AggregationSink`. * The aggregation (the *what*) and the sink (the *where*) compose: adding an aggregation is one @@ -479,3 +393,4 @@ class VectorOutputHandler : public ::clp_s::search::OutputHandler { } // namespace clp_s #endif // CLP_S_OUTPUTHANDLERIMPL_HPP + \ No newline at end of file diff --git a/components/core/src/clp_s/ResultsCacheUtils.hpp b/components/core/src/clp_s/ResultsCacheUtils.hpp new file mode 100644 index 0000000000..01a64c2db6 --- /dev/null +++ b/components/core/src/clp_s/ResultsCacheUtils.hpp @@ -0,0 +1,39 @@ +#ifndef CLP_S_RESULTSCACHEUTILS_HPP +#define CLP_S_RESULTSCACHEUTILS_HPP + +#include +#include + +#include +#include +#include +#include + +#include "ErrorCode.hpp" + +namespace clp_s { +/** + * Connects to the results cache and returns the requested collection. + * @tparam OperationFailedT The type of exception to throw on failure. + * @param uri + * @param collection + * @param client Returns the connected client. + * @return The collection. + */ +template +auto connect_to_results_cache( + std::string_view uri, + std::string_view collection, + mongocxx::client& client +) -> mongocxx::collection { + try { + auto mongo_uri = mongocxx::uri{std::string{uri}}; + client = mongocxx::client(mongo_uri); + return client[mongo_uri.database()][std::string{collection}]; + } catch (mongocxx::exception const& e) { + throw OperationFailedT(ErrorCode::ErrorCodeBadParamDbUri, __FILENAME__, __LINE__); + } +} +} // namespace clp_s + +#endif // CLP_S_RESULTSCACHEUTILS_HPP From dc20723d36474c55f62cd08a8a10b18f38a38036 Mon Sep 17 00:00:00 2001 From: davemarco <83603688+davemarco@users.noreply.github.com> Date: Sun, 28 Jun 2026 13:56:43 -0400 Subject: [PATCH 29/56] Throw on malformed min/max field and trim aggregation comments. Reject an un-tokenizable --min/--max field with std::invalid_argument (surfaced as a clean arg error) instead of silently clearing the path and yielding no result. Also tighten doc comments in Aggregation.hpp. --- components/core/src/clp_s/Aggregation.cpp | 6 ++---- components/core/src/clp_s/Aggregation.hpp | 12 +++--------- 2 files changed, 5 insertions(+), 13 deletions(-) diff --git a/components/core/src/clp_s/Aggregation.cpp b/components/core/src/clp_s/Aggregation.cpp index b70d2b5710..d91fa84939 100644 --- a/components/core/src/clp_s/Aggregation.cpp +++ b/components/core/src/clp_s/Aggregation.cpp @@ -1,5 +1,6 @@ #include "Aggregation.hpp" +#include #include #include #include @@ -48,10 +49,7 @@ MinMaxAggregation::MinMaxAggregation(bool find_max, string_view field) descriptor_namespace )) { - // A malformed field tokenizes to nothing, so `add_record` never matches and min/max yields - // no result. The field is validated during command-line parsing, so this is just defensive. - m_field_path.clear(); - return; + throw std::invalid_argument("Invalid --min/--max field: " + string{field}); } if (false == descriptor_namespace.empty()) { // Namespaced fields (e.g. the auto-generated `@` namespace) are marshalled under a diff --git a/components/core/src/clp_s/Aggregation.hpp b/components/core/src/clp_s/Aggregation.hpp index 32313a7f4c..14d685666c 100644 --- a/components/core/src/clp_s/Aggregation.hpp +++ b/components/core/src/clp_s/Aggregation.hpp @@ -19,9 +19,7 @@ namespace clp_s { using AggregationValue = std::variant; /** - * One aggregation result document: an ordered list of typed key-value pairs. A sink serializes - * these to its destination (JSON for stdout, BSON for the results cache); the archive id is added - * by the sink, not the aggregator. + * One aggregation result document: an ordered list of typed key-value pairs. */ using AggregationResult = std::vector>; @@ -70,9 +68,7 @@ class CountByTimeAggregation { }; /** - * Tracks the minimum or maximum value of a target field across matched records. Integer fields are - * compared exactly and only degraded to a (lossy) double comparison when comparing against a - * floating-point value, so integer fields aren't degraded to `double`. + * Tracks the minimum or maximum value of a target field across matched records. */ class MinMaxAggregation { public: @@ -101,9 +97,7 @@ class MinMaxAggregation { }; /** - * A search aggregation to perform. Replaces a `(type, params)` representation: each alternative - * carries exactly the fields its aggregation needs, and `std::visit` dispatches `add_record` / - * `get_results` to the active one. + * A search aggregation to perform. */ using Aggregation = std::variant; From 85c9ebfdfd79234c307f5d660026b058c9a0d258 Mon Sep 17 00:00:00 2001 From: davemarco <83603688+davemarco@users.noreply.github.com> Date: Sun, 28 Jun 2026 14:06:38 -0400 Subject: [PATCH 30/56] Compare min/max int64 and double operands exactly to avoid precision loss above 2^53. --- components/core/src/clp_s/Aggregation.cpp | 92 ++++++++++++++++++++--- 1 file changed, 80 insertions(+), 12 deletions(-) diff --git a/components/core/src/clp_s/Aggregation.cpp b/components/core/src/clp_s/Aggregation.cpp index d91fa84939..717e448e17 100644 --- a/components/core/src/clp_s/Aggregation.cpp +++ b/components/core/src/clp_s/Aggregation.cpp @@ -1,5 +1,7 @@ #include "Aggregation.hpp" +#include +#include #include #include #include @@ -17,6 +19,77 @@ using std::string; using std::string_view; namespace clp_s { +namespace { +// `2^63`: exactly representable as a double and one past `INT64_MAX`. Any double `>=` this exceeds +// every `int64_t`. +constexpr double cInt64UpperBound{9223372036854775808.0}; +// `-2^63 == INT64_MIN`, exactly representable as a double. +constexpr double cInt64Min{-9223372036854775808.0}; + +[[nodiscard]] auto is_less(int64_t lhs, int64_t rhs) -> bool { + return lhs < rhs; +} + +[[nodiscard]] auto is_less(double lhs, double rhs) -> bool { + return lhs < rhs; +} + +/** + * Determines whether an integer is strictly less than a double without precision loss. Casting + * `lhs` to a double would be lossy above `2^53`, so we instead range-check `rhs` against the + * `int64_t` bounds and then compare integer parts exactly, letting the fractional part of `rhs` + * break ties. + * @param lhs + * @param rhs + * @return Whether `lhs < rhs`. + */ +[[nodiscard]] auto is_less(int64_t lhs, double rhs) -> bool { + if (std::isnan(rhs)) { + return false; + } + if (rhs >= cInt64UpperBound) { + return true; + } + if (rhs < cInt64Min) { + return false; + } + // `rhs` is now in `[INT64_MIN, INT64_MAX]`, so `trunc(rhs)` casts to `int64_t` exactly. + auto const truncated{std::trunc(rhs)}; + auto const rhs_int{static_cast(truncated)}; + if (lhs != rhs_int) { + return lhs < rhs_int; + } + // Integer parts are equal; `lhs < rhs` iff `rhs` has a positive fractional part. + return rhs > truncated; +} + +/** + * Determines whether a double is strictly less than an integer without precision loss. See the + * `int64_t`/`double` overload for the rationale. + * @param lhs + * @param rhs + * @return Whether `lhs < rhs`. + */ +[[nodiscard]] auto is_less(double lhs, int64_t rhs) -> bool { + if (std::isnan(lhs)) { + return false; + } + if (lhs >= cInt64UpperBound) { + return false; + } + if (lhs < cInt64Min) { + return true; + } + auto const truncated{std::trunc(lhs)}; + auto const lhs_int{static_cast(truncated)}; + if (lhs_int != rhs) { + return lhs_int < rhs; + } + // Integer parts are equal; `lhs < rhs` iff `lhs` has a negative fractional part. + return lhs < truncated; +} +} // namespace + auto CountAggregation::get_results() const -> std::vector { if (0 == m_count) { return {}; @@ -60,18 +133,13 @@ MinMaxAggregation::MinMaxAggregation(bool find_max, string_view field) auto MinMaxAggregation::beats_extreme(Extreme candidate) const -> bool { auto const& current{m_extreme.value()}; - // Compare integers exactly; only fall back to a (lossy) double comparison when the types - // differ. - if (std::holds_alternative(candidate) && std::holds_alternative(current)) { - auto const lhs{std::get(candidate)}; - auto const rhs{std::get(current)}; - return m_find_max ? lhs > rhs : lhs < rhs; - } - auto const as_double{[](Extreme value) { - return std::visit([](auto held) { return static_cast(held); }, value); - }}; - return m_find_max ? as_double(candidate) > as_double(current) - : as_double(candidate) < as_double(current); + // `is_less` compares every `int64_t`/`double` combination exactly (no lossy casts). A candidate + // beats the current maximum when the current value is smaller, and the current minimum when the + // candidate is smaller. + if (m_find_max) { + return std::visit([](auto cand, auto cur) { return is_less(cur, cand); }, candidate, current); + } + return std::visit([](auto cand, auto cur) { return is_less(cand, cur); }, candidate, current); } auto MinMaxAggregation::add_record(string_view message, epochtime_t) -> void { From cc4be185428937e49d146d0098f6813a288143bf Mon Sep 17 00:00:00 2001 From: davemarco <83603688+davemarco@users.noreply.github.com> Date: Sun, 28 Jun 2026 14:09:41 -0400 Subject: [PATCH 31/56] Extract exact int64/double comparison into NumericCompare.hpp. --- components/core/src/clp_s/Aggregation.cpp | 73 +------------------ components/core/src/clp_s/NumericCompare.hpp | 77 ++++++++++++++++++++ 2 files changed, 78 insertions(+), 72 deletions(-) create mode 100644 components/core/src/clp_s/NumericCompare.hpp diff --git a/components/core/src/clp_s/Aggregation.cpp b/components/core/src/clp_s/Aggregation.cpp index 717e448e17..958ef889b8 100644 --- a/components/core/src/clp_s/Aggregation.cpp +++ b/components/core/src/clp_s/Aggregation.cpp @@ -1,6 +1,5 @@ #include "Aggregation.hpp" -#include #include #include #include @@ -13,83 +12,13 @@ #include #include "archive_constants.hpp" +#include "NumericCompare.hpp" #include "search/ast/SearchUtils.hpp" using std::string; using std::string_view; namespace clp_s { -namespace { -// `2^63`: exactly representable as a double and one past `INT64_MAX`. Any double `>=` this exceeds -// every `int64_t`. -constexpr double cInt64UpperBound{9223372036854775808.0}; -// `-2^63 == INT64_MIN`, exactly representable as a double. -constexpr double cInt64Min{-9223372036854775808.0}; - -[[nodiscard]] auto is_less(int64_t lhs, int64_t rhs) -> bool { - return lhs < rhs; -} - -[[nodiscard]] auto is_less(double lhs, double rhs) -> bool { - return lhs < rhs; -} - -/** - * Determines whether an integer is strictly less than a double without precision loss. Casting - * `lhs` to a double would be lossy above `2^53`, so we instead range-check `rhs` against the - * `int64_t` bounds and then compare integer parts exactly, letting the fractional part of `rhs` - * break ties. - * @param lhs - * @param rhs - * @return Whether `lhs < rhs`. - */ -[[nodiscard]] auto is_less(int64_t lhs, double rhs) -> bool { - if (std::isnan(rhs)) { - return false; - } - if (rhs >= cInt64UpperBound) { - return true; - } - if (rhs < cInt64Min) { - return false; - } - // `rhs` is now in `[INT64_MIN, INT64_MAX]`, so `trunc(rhs)` casts to `int64_t` exactly. - auto const truncated{std::trunc(rhs)}; - auto const rhs_int{static_cast(truncated)}; - if (lhs != rhs_int) { - return lhs < rhs_int; - } - // Integer parts are equal; `lhs < rhs` iff `rhs` has a positive fractional part. - return rhs > truncated; -} - -/** - * Determines whether a double is strictly less than an integer without precision loss. See the - * `int64_t`/`double` overload for the rationale. - * @param lhs - * @param rhs - * @return Whether `lhs < rhs`. - */ -[[nodiscard]] auto is_less(double lhs, int64_t rhs) -> bool { - if (std::isnan(lhs)) { - return false; - } - if (lhs >= cInt64UpperBound) { - return false; - } - if (lhs < cInt64Min) { - return true; - } - auto const truncated{std::trunc(lhs)}; - auto const lhs_int{static_cast(truncated)}; - if (lhs_int != rhs) { - return lhs_int < rhs; - } - // Integer parts are equal; `lhs < rhs` iff `lhs` has a negative fractional part. - return lhs < truncated; -} -} // namespace - auto CountAggregation::get_results() const -> std::vector { if (0 == m_count) { return {}; diff --git a/components/core/src/clp_s/NumericCompare.hpp b/components/core/src/clp_s/NumericCompare.hpp new file mode 100644 index 0000000000..688ad41f41 --- /dev/null +++ b/components/core/src/clp_s/NumericCompare.hpp @@ -0,0 +1,77 @@ +#ifndef CLP_S_NUMERICCOMPARE_HPP +#define CLP_S_NUMERICCOMPARE_HPP + +#include +#include + +namespace clp_s { +// `2^63`: exactly representable as a double and one past `INT64_MAX`. Any double `>=` this exceeds +// every `int64_t`. +constexpr double cInt64UpperBound{9223372036854775808.0}; +// `-2^63 == INT64_MIN`, exactly representable as a double. +constexpr double cInt64Min{-9223372036854775808.0}; + +[[nodiscard]] inline auto is_less(int64_t lhs, int64_t rhs) -> bool { + return lhs < rhs; +} + +[[nodiscard]] inline auto is_less(double lhs, double rhs) -> bool { + return lhs < rhs; +} + +/** + * Determines whether an integer is strictly less than a double without precision loss. Casting + * `lhs` to a double would be lossy above `2^53`, so we instead range-check `rhs` against the + * `int64_t` bounds and then compare integer parts exactly, letting the fractional part of `rhs` + * break ties. + * @param lhs + * @param rhs + * @return Whether `lhs < rhs`. + */ +[[nodiscard]] inline auto is_less(int64_t lhs, double rhs) -> bool { + if (std::isnan(rhs)) { + return false; + } + if (rhs >= cInt64UpperBound) { + return true; + } + if (rhs < cInt64Min) { + return false; + } + // `rhs` is now in `[INT64_MIN, INT64_MAX]`, so `trunc(rhs)` casts to `int64_t` exactly. + auto const truncated{std::trunc(rhs)}; + auto const rhs_int{static_cast(truncated)}; + if (lhs != rhs_int) { + return lhs < rhs_int; + } + // Integer parts are equal; `lhs < rhs` iff `rhs` has a positive fractional part. + return rhs > truncated; +} + +/** + * Determines whether a double is strictly less than an integer without precision loss. See the + * `int64_t`/`double` overload for the rationale. + * @param lhs + * @param rhs + * @return Whether `lhs < rhs`. + */ +[[nodiscard]] inline auto is_less(double lhs, int64_t rhs) -> bool { + if (std::isnan(lhs)) { + return false; + } + if (lhs >= cInt64UpperBound) { + return false; + } + if (lhs < cInt64Min) { + return true; + } + auto const truncated{std::trunc(lhs)}; + auto const lhs_int{static_cast(truncated)}; + if (lhs_int != rhs) { + return lhs_int < rhs; + } + // Integer parts are equal; `lhs < rhs` iff `lhs` has a negative fractional part. + return lhs < truncated; +} +} // namespace clp_s +#endif // CLP_S_NUMERICCOMPARE_HPP From b3db8724d485e1ff784e14bad1b6709cad6781fd Mon Sep 17 00:00:00 2001 From: davemarco <83603688+davemarco@users.noreply.github.com> Date: Sun, 28 Jun 2026 14:16:53 -0400 Subject: [PATCH 32/56] Cite SQLite's sqlite3IntFloatCompare as the basis for the exact comparison. --- components/core/src/clp_s/NumericCompare.hpp | 3 +++ 1 file changed, 3 insertions(+) diff --git a/components/core/src/clp_s/NumericCompare.hpp b/components/core/src/clp_s/NumericCompare.hpp index 688ad41f41..c15e86897f 100644 --- a/components/core/src/clp_s/NumericCompare.hpp +++ b/components/core/src/clp_s/NumericCompare.hpp @@ -24,6 +24,9 @@ constexpr double cInt64Min{-9223372036854775808.0}; * `lhs` to a double would be lossy above `2^53`, so we instead range-check `rhs` against the * `int64_t` bounds and then compare integer parts exactly, letting the fractional part of `rhs` * break ties. + * + * Based on SQLite's `sqlite3IntFloatCompare`: + * https://github.com/sqlite/sqlite/blob/master/src/vdbeaux.c * @param lhs * @param rhs * @return Whether `lhs < rhs`. From c34494d9a70b13bd749292a242b9728f88ebf5f0 Mon Sep 17 00:00:00 2001 From: davemarco <83603688+davemarco@users.noreply.github.com> Date: Sun, 28 Jun 2026 14:30:43 -0400 Subject: [PATCH 33/56] Trim aggregation comments. --- components/core/src/clp_s/Aggregation.cpp | 7 ++----- components/core/src/clp_s/Aggregation.hpp | 2 +- 2 files changed, 3 insertions(+), 6 deletions(-) diff --git a/components/core/src/clp_s/Aggregation.cpp b/components/core/src/clp_s/Aggregation.cpp index 958ef889b8..437e791d6c 100644 --- a/components/core/src/clp_s/Aggregation.cpp +++ b/components/core/src/clp_s/Aggregation.cpp @@ -54,17 +54,14 @@ MinMaxAggregation::MinMaxAggregation(bool find_max, string_view field) throw std::invalid_argument("Invalid --min/--max field: " + string{field}); } if (false == descriptor_namespace.empty()) { - // Namespaced fields (e.g. the auto-generated `@` namespace) are marshalled under a - // top-level object keyed by the namespace string, so navigate into it first. + // The tokenizer strips the namespace prefix out of the path, but namespaced fields live + // under a top-level object keyed by that namespace. Prepend it back so the path matches. m_field_path.insert(m_field_path.begin(), descriptor_namespace); } } auto MinMaxAggregation::beats_extreme(Extreme candidate) const -> bool { auto const& current{m_extreme.value()}; - // `is_less` compares every `int64_t`/`double` combination exactly (no lossy casts). A candidate - // beats the current maximum when the current value is smaller, and the current minimum when the - // candidate is smaller. if (m_find_max) { return std::visit([](auto cand, auto cur) { return is_less(cur, cand); }, candidate, current); } diff --git a/components/core/src/clp_s/Aggregation.hpp b/components/core/src/clp_s/Aggregation.hpp index 14d685666c..037e2d5d47 100644 --- a/components/core/src/clp_s/Aggregation.hpp +++ b/components/core/src/clp_s/Aggregation.hpp @@ -103,7 +103,7 @@ using Aggregation = std::variant bool; From 8002b1bafceae5c8bc0c1e72dd5d3f5ebcee5697 Mon Sep 17 00:00:00 2001 From: davemarco <83603688+davemarco@users.noreply.github.com> Date: Sun, 28 Jun 2026 14:35:56 -0400 Subject: [PATCH 34/56] Clarify copy/move comment and trim sink doc comments. --- components/core/src/clp_s/AggregationSink.hpp | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/components/core/src/clp_s/AggregationSink.hpp b/components/core/src/clp_s/AggregationSink.hpp index 2b05ece16e..e0839c4f30 100644 --- a/components/core/src/clp_s/AggregationSink.hpp +++ b/components/core/src/clp_s/AggregationSink.hpp @@ -25,7 +25,7 @@ class AggregationSink { // Destructor virtual ~AggregationSink() = default; - // Delete copy and move + // Explicitly disable copy and move constructor/assignment AggregationSink(AggregationSink const&) = delete; auto operator=(AggregationSink const&) -> AggregationSink& = delete; AggregationSink(AggregationSink&&) = delete; @@ -33,7 +33,7 @@ class AggregationSink { // Methods /** - * Writes one result document. The archive id is added by the sink. + * Writes one result document. * @param result */ virtual auto write(AggregationResult const& result) -> void = 0; @@ -87,7 +87,7 @@ class ResultsCacheSink : public AggregationSink { auto write(AggregationResult const& result) -> void override; /** - * Flushes the buffered result documents to the results cache. + * Flushes the buffered result documents. * @return ErrorCodeSuccess on success * @return ErrorCodeFailureDbBulkWrite on database error */ From f538e6f3533f0e5b667199f043758c9cee70e70e Mon Sep 17 00:00:00 2001 From: davemarco <83603688+davemarco@users.noreply.github.com> Date: Sun, 28 Jun 2026 14:52:31 -0400 Subject: [PATCH 35/56] Clarify NumericCompare comments. --- components/core/src/clp_s/NumericCompare.hpp | 35 ++++++++++++-------- 1 file changed, 22 insertions(+), 13 deletions(-) diff --git a/components/core/src/clp_s/NumericCompare.hpp b/components/core/src/clp_s/NumericCompare.hpp index c15e86897f..676488bcb5 100644 --- a/components/core/src/clp_s/NumericCompare.hpp +++ b/components/core/src/clp_s/NumericCompare.hpp @@ -4,11 +4,16 @@ #include #include +// Exact less-than comparisons between `int64_t` and `double` values. +// +// Comparing the two types by casting the integer to a double is lossy: doubles can only represent +// integers exactly up to `2^53`, so larger `int64_t` values round and the comparison can give the +// wrong answer. The overloads below instead compare exactly across both types. namespace clp_s { -// `2^63`: exactly representable as a double and one past `INT64_MAX`. Any double `>=` this exceeds -// every `int64_t`. +// `2^63`: the smallest double too large to fit in an `int64_t` (exactly representable, one past +// `INT64_MAX`). Any double `>=` this is greater than every `int64_t`. constexpr double cInt64UpperBound{9223372036854775808.0}; -// `-2^63 == INT64_MIN`, exactly representable as a double. +// `-2^63 == INT64_MIN` (exactly representable). Any double `<` this is less than every `int64_t`. constexpr double cInt64Min{-9223372036854775808.0}; [[nodiscard]] inline auto is_less(int64_t lhs, int64_t rhs) -> bool { @@ -20,60 +25,64 @@ constexpr double cInt64Min{-9223372036854775808.0}; } /** - * Determines whether an integer is strictly less than a double without precision loss. Casting - * `lhs` to a double would be lossy above `2^53`, so we instead range-check `rhs` against the - * `int64_t` bounds and then compare integer parts exactly, letting the fractional part of `rhs` - * break ties. + * Compares an integer against a double exactly. After ruling out doubles that fall outside the + * `int64_t` range, `rhs` is split into its integer and fractional parts: the integer parts are + * compared directly, and the fractional part of `rhs` breaks ties. * - * Based on SQLite's `sqlite3IntFloatCompare`: + * Adapted from SQLite's `sqlite3IntFloatCompare`: * https://github.com/sqlite/sqlite/blob/master/src/vdbeaux.c * @param lhs * @param rhs * @return Whether `lhs < rhs`. */ [[nodiscard]] inline auto is_less(int64_t lhs, double rhs) -> bool { + // NaN is unordered, so it is never less than anything. if (std::isnan(rhs)) { return false; } + // `rhs` lies outside the `int64_t` range, so the result is determined by its sign. if (rhs >= cInt64UpperBound) { return true; } if (rhs < cInt64Min) { return false; } - // `rhs` is now in `[INT64_MIN, INT64_MAX]`, so `trunc(rhs)` casts to `int64_t` exactly. + // `rhs` is within the `int64_t` range, so truncating it toward zero and casting is exact. auto const truncated{std::trunc(rhs)}; auto const rhs_int{static_cast(truncated)}; if (lhs != rhs_int) { return lhs < rhs_int; } - // Integer parts are equal; `lhs < rhs` iff `rhs` has a positive fractional part. + // Same integer part: `lhs < rhs` exactly when `rhs` has a positive fractional part. return rhs > truncated; } /** - * Determines whether a double is strictly less than an integer without precision loss. See the - * `int64_t`/`double` overload for the rationale. + * Compares a double against an integer exactly. The mirror image of the `int64_t`/`double` overload; + * see it for the rationale. * @param lhs * @param rhs * @return Whether `lhs < rhs`. */ [[nodiscard]] inline auto is_less(double lhs, int64_t rhs) -> bool { + // NaN is unordered, so it is never less than anything. if (std::isnan(lhs)) { return false; } + // `lhs` lies outside the `int64_t` range, so the result is determined by its sign. if (lhs >= cInt64UpperBound) { return false; } if (lhs < cInt64Min) { return true; } + // `lhs` is within the `int64_t` range, so truncating it toward zero and casting is exact. auto const truncated{std::trunc(lhs)}; auto const lhs_int{static_cast(truncated)}; if (lhs_int != rhs) { return lhs_int < rhs; } - // Integer parts are equal; `lhs < rhs` iff `lhs` has a negative fractional part. + // Same integer part: `lhs < rhs` exactly when `lhs` has a negative fractional part. return lhs < truncated; } } // namespace clp_s From 60baba904407082963c723a402d23d75e25ec1f9 Mon Sep 17 00:00:00 2001 From: davemarco <83603688+davemarco@users.noreply.github.com> Date: Sun, 28 Jun 2026 15:05:09 -0400 Subject: [PATCH 36/56] Rename NumericCompare to IntFloatCompare and clarify its comments. --- components/core/src/clp_s/Aggregation.cpp | 2 +- ...NumericCompare.hpp => IntFloatCompare.hpp} | 38 ++++++++++++------- 2 files changed, 25 insertions(+), 15 deletions(-) rename components/core/src/clp_s/{NumericCompare.hpp => IntFloatCompare.hpp} (58%) diff --git a/components/core/src/clp_s/Aggregation.cpp b/components/core/src/clp_s/Aggregation.cpp index 437e791d6c..db1afdca3d 100644 --- a/components/core/src/clp_s/Aggregation.cpp +++ b/components/core/src/clp_s/Aggregation.cpp @@ -12,7 +12,7 @@ #include #include "archive_constants.hpp" -#include "NumericCompare.hpp" +#include "IntFloatCompare.hpp" #include "search/ast/SearchUtils.hpp" using std::string; diff --git a/components/core/src/clp_s/NumericCompare.hpp b/components/core/src/clp_s/IntFloatCompare.hpp similarity index 58% rename from components/core/src/clp_s/NumericCompare.hpp rename to components/core/src/clp_s/IntFloatCompare.hpp index 676488bcb5..132c5ebbe4 100644 --- a/components/core/src/clp_s/NumericCompare.hpp +++ b/components/core/src/clp_s/IntFloatCompare.hpp @@ -1,19 +1,30 @@ -#ifndef CLP_S_NUMERICCOMPARE_HPP -#define CLP_S_NUMERICCOMPARE_HPP +#ifndef CLP_S_INTFLOATCOMPARE_HPP +#define CLP_S_INTFLOATCOMPARE_HPP #include #include // Exact less-than comparisons between `int64_t` and `double` values. // -// Comparing the two types by casting the integer to a double is lossy: doubles can only represent -// integers exactly up to `2^53`, so larger `int64_t` values round and the comparison can give the -// wrong answer. The overloads below instead compare exactly across both types. +// A double can represent every integer exactly only up to `2^53`. Beyond that the representable +// integers have gaps, so casting a larger `int64_t` to a double snaps it to the nearest value a +// double can hold (e.g. `2^53 + 1` becomes `2^53`). Comparing an `int64_t` and a double through +// that cast can therefore return the wrong result. The overloads below compare the two types +// exactly, without casting. +// +// Adapted from SQLite's `sqlite3IntFloatCompare`: +// https://github.com/sqlite/sqlite/blob/master/src/vdbeaux.c namespace clp_s { -// `2^63`: the smallest double too large to fit in an `int64_t` (exactly representable, one past -// `INT64_MAX`). Any double `>=` this is greater than every `int64_t`. +// Doubles span a far wider range than `int64_t`, so a double can be larger or smaller than any +// `int64_t`. These constants bracket the `int64_t` range, letting the comparisons below detect such +// out-of-range doubles before converting them to `int64_t` (which would overflow). + +// `2^63`, one greater than `INT64_MAX`: any double `>=` this is larger than every `int64_t`. We use +// `2^63` rather than `INT64_MAX` because `2^63` is exactly representable as a double and `INT64_MAX` +// is not. constexpr double cInt64UpperBound{9223372036854775808.0}; -// `-2^63 == INT64_MIN` (exactly representable). Any double `<` this is less than every `int64_t`. +// `-2^63`, equal to `INT64_MIN`: any double `<` this is smaller than every `int64_t`. Both `-2^63` +// and `INT64_MIN` are the same value and are exactly representable as a double. constexpr double cInt64Min{-9223372036854775808.0}; [[nodiscard]] inline auto is_less(int64_t lhs, int64_t rhs) -> bool { @@ -28,9 +39,6 @@ constexpr double cInt64Min{-9223372036854775808.0}; * Compares an integer against a double exactly. After ruling out doubles that fall outside the * `int64_t` range, `rhs` is split into its integer and fractional parts: the integer parts are * compared directly, and the fractional part of `rhs` breaks ties. - * - * Adapted from SQLite's `sqlite3IntFloatCompare`: - * https://github.com/sqlite/sqlite/blob/master/src/vdbeaux.c * @param lhs * @param rhs * @return Whether `lhs < rhs`. @@ -40,10 +48,11 @@ constexpr double cInt64Min{-9223372036854775808.0}; if (std::isnan(rhs)) { return false; } - // `rhs` lies outside the `int64_t` range, so the result is determined by its sign. + // `rhs` is larger than every `int64_t`, so `lhs` must be less than it. if (rhs >= cInt64UpperBound) { return true; } + // `rhs` is smaller than every `int64_t`, so `lhs` cannot be less than it. if (rhs < cInt64Min) { return false; } @@ -69,10 +78,11 @@ constexpr double cInt64Min{-9223372036854775808.0}; if (std::isnan(lhs)) { return false; } - // `lhs` lies outside the `int64_t` range, so the result is determined by its sign. + // `lhs` is larger than every `int64_t`, so it cannot be less than `rhs`. if (lhs >= cInt64UpperBound) { return false; } + // `lhs` is smaller than every `int64_t`, so it must be less than `rhs`. if (lhs < cInt64Min) { return true; } @@ -86,4 +96,4 @@ constexpr double cInt64Min{-9223372036854775808.0}; return lhs < truncated; } } // namespace clp_s -#endif // CLP_S_NUMERICCOMPARE_HPP +#endif // CLP_S_INTFLOATCOMPARE_HPP From afe05185bdc15b25db8201bb94851b9dc1b533dc Mon Sep 17 00:00:00 2001 From: davemarco <83603688+davemarco@users.noreply.github.com> Date: Sun, 28 Jun 2026 15:12:24 -0400 Subject: [PATCH 37/56] Trim redundant IntFloatCompare comments. --- components/core/src/clp_s/IntFloatCompare.hpp | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/components/core/src/clp_s/IntFloatCompare.hpp b/components/core/src/clp_s/IntFloatCompare.hpp index 132c5ebbe4..e6c2158264 100644 --- a/components/core/src/clp_s/IntFloatCompare.hpp +++ b/components/core/src/clp_s/IntFloatCompare.hpp @@ -15,9 +15,9 @@ // Adapted from SQLite's `sqlite3IntFloatCompare`: // https://github.com/sqlite/sqlite/blob/master/src/vdbeaux.c namespace clp_s { -// Doubles span a far wider range than `int64_t`, so a double can be larger or smaller than any -// `int64_t`. These constants bracket the `int64_t` range, letting the comparisons below detect such -// out-of-range doubles before converting them to `int64_t` (which would overflow). +// Doubles span a far wider range than `int64_t`. These constants bracket the `int64_t` range, +// letting the comparisons below detect out-of-range doubles before converting them to `int64_t` +// (which would overflow). // `2^63`, one greater than `INT64_MAX`: any double `>=` this is larger than every `int64_t`. We use // `2^63` rather than `INT64_MAX` because `2^63` is exactly representable as a double and `INT64_MAX` @@ -67,8 +67,7 @@ constexpr double cInt64Min{-9223372036854775808.0}; } /** - * Compares a double against an integer exactly. The mirror image of the `int64_t`/`double` overload; - * see it for the rationale. + * Compares a double against an integer exactly. * @param lhs * @param rhs * @return Whether `lhs < rhs`. From 1a3b902f4ac768e061fc6216a729de6fc14e420e Mon Sep 17 00:00:00 2001 From: davemarco <83603688+davemarco@users.noreply.github.com> Date: Sun, 28 Jun 2026 15:18:07 -0400 Subject: [PATCH 38/56] Simplify aggregation handler comment and fix trailing whitespace/newline. --- components/core/src/clp_s/IntFloatCompare.hpp | 1 + components/core/src/clp_s/OutputHandlerImpl.hpp | 3 --- 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/components/core/src/clp_s/IntFloatCompare.hpp b/components/core/src/clp_s/IntFloatCompare.hpp index e6c2158264..df49ebff87 100644 --- a/components/core/src/clp_s/IntFloatCompare.hpp +++ b/components/core/src/clp_s/IntFloatCompare.hpp @@ -95,4 +95,5 @@ constexpr double cInt64Min{-9223372036854775808.0}; return lhs < truncated; } } // namespace clp_s + #endif // CLP_S_INTFLOATCOMPARE_HPP diff --git a/components/core/src/clp_s/OutputHandlerImpl.hpp b/components/core/src/clp_s/OutputHandlerImpl.hpp index 415431ac7c..7ed4fb8f70 100644 --- a/components/core/src/clp_s/OutputHandlerImpl.hpp +++ b/components/core/src/clp_s/OutputHandlerImpl.hpp @@ -292,8 +292,6 @@ class CountByTimeReducerOutputHandler : public search::OutputHandler { /** * Output handler that runs a search `Aggregation` and writes its results to an `AggregationSink`. - * The aggregation (the *what*) and the sink (the *where*) compose: adding an aggregation is one - * `Aggregation` alternative, adding a destination is one `AggregationSink`. */ class AggregationOutputHandler : public search::OutputHandler { public: @@ -393,4 +391,3 @@ class VectorOutputHandler : public ::clp_s::search::OutputHandler { } // namespace clp_s #endif // CLP_S_OUTPUTHANDLERIMPL_HPP - \ No newline at end of file From e95f81bd8292b9bf6d6e6e044162984ccb1b0322 Mon Sep 17 00:00:00 2001 From: davemarco <83603688+davemarco@users.noreply.github.com> Date: Sun, 28 Jun 2026 15:53:51 -0400 Subject: [PATCH 39/56] Use angle-bracket includes for new clp_s headers per the new guideline. Also wrap an IntFloatCompare comment and drop a stray blank line. --- components/core/src/clp_s/Aggregation.cpp | 12 ++++++++---- components/core/src/clp_s/Aggregation.hpp | 2 +- components/core/src/clp_s/AggregationSink.cpp | 6 +++--- components/core/src/clp_s/AggregationSink.hpp | 6 +++--- components/core/src/clp_s/IntFloatCompare.hpp | 4 ++-- components/core/src/clp_s/OutputHandlerImpl.cpp | 1 - components/core/src/clp_s/ResultsCacheUtils.hpp | 2 +- 7 files changed, 18 insertions(+), 15 deletions(-) diff --git a/components/core/src/clp_s/Aggregation.cpp b/components/core/src/clp_s/Aggregation.cpp index db1afdca3d..33ad37aab0 100644 --- a/components/core/src/clp_s/Aggregation.cpp +++ b/components/core/src/clp_s/Aggregation.cpp @@ -11,9 +11,9 @@ #include -#include "archive_constants.hpp" -#include "IntFloatCompare.hpp" -#include "search/ast/SearchUtils.hpp" +#include +#include +#include using std::string; using std::string_view; @@ -63,7 +63,11 @@ MinMaxAggregation::MinMaxAggregation(bool find_max, string_view field) auto MinMaxAggregation::beats_extreme(Extreme candidate) const -> bool { auto const& current{m_extreme.value()}; if (m_find_max) { - return std::visit([](auto cand, auto cur) { return is_less(cur, cand); }, candidate, current); + return std::visit( + [](auto cand, auto cur) { return is_less(cur, cand); }, + candidate, + current + ); } return std::visit([](auto cand, auto cur) { return is_less(cand, cur); }, candidate, current); } diff --git a/components/core/src/clp_s/Aggregation.hpp b/components/core/src/clp_s/Aggregation.hpp index 037e2d5d47..3137222a75 100644 --- a/components/core/src/clp_s/Aggregation.hpp +++ b/components/core/src/clp_s/Aggregation.hpp @@ -10,7 +10,7 @@ #include #include -#include "Defs.hpp" +#include namespace clp_s { /** diff --git a/components/core/src/clp_s/AggregationSink.cpp b/components/core/src/clp_s/AggregationSink.cpp index d50ab9ee8d..6ce94ac6e3 100644 --- a/components/core/src/clp_s/AggregationSink.cpp +++ b/components/core/src/clp_s/AggregationSink.cpp @@ -11,9 +11,9 @@ #include #include -#include "archive_constants.hpp" -#include "ErrorCode.hpp" -#include "ResultsCacheUtils.hpp" +#include +#include +#include using std::string_view; diff --git a/components/core/src/clp_s/AggregationSink.hpp b/components/core/src/clp_s/AggregationSink.hpp index e0839c4f30..afccf8207c 100644 --- a/components/core/src/clp_s/AggregationSink.hpp +++ b/components/core/src/clp_s/AggregationSink.hpp @@ -9,9 +9,9 @@ #include #include -#include "Aggregation.hpp" -#include "ErrorCode.hpp" -#include "TraceableException.hpp" +#include +#include +#include namespace clp_s { /** diff --git a/components/core/src/clp_s/IntFloatCompare.hpp b/components/core/src/clp_s/IntFloatCompare.hpp index df49ebff87..0da766343e 100644 --- a/components/core/src/clp_s/IntFloatCompare.hpp +++ b/components/core/src/clp_s/IntFloatCompare.hpp @@ -20,8 +20,8 @@ namespace clp_s { // (which would overflow). // `2^63`, one greater than `INT64_MAX`: any double `>=` this is larger than every `int64_t`. We use -// `2^63` rather than `INT64_MAX` because `2^63` is exactly representable as a double and `INT64_MAX` -// is not. +// `2^63` rather than `INT64_MAX` because `2^63` is exactly representable as a double and +// `INT64_MAX` is not. constexpr double cInt64UpperBound{9223372036854775808.0}; // `-2^63`, equal to `INT64_MIN`: any double `<` this is smaller than every `int64_t`. Both `-2^63` // and `INT64_MIN` are the same value and are exactly representable as a double. diff --git a/components/core/src/clp_s/OutputHandlerImpl.cpp b/components/core/src/clp_s/OutputHandlerImpl.cpp index 37a26e4a6d..ff81e1e926 100644 --- a/components/core/src/clp_s/OutputHandlerImpl.cpp +++ b/components/core/src/clp_s/OutputHandlerImpl.cpp @@ -211,5 +211,4 @@ auto CountByTimeReducerOutputHandler::finish() -> ErrorCode { } return ErrorCode::ErrorCodeSuccess; } - } // namespace clp_s diff --git a/components/core/src/clp_s/ResultsCacheUtils.hpp b/components/core/src/clp_s/ResultsCacheUtils.hpp index 01a64c2db6..08248c35b5 100644 --- a/components/core/src/clp_s/ResultsCacheUtils.hpp +++ b/components/core/src/clp_s/ResultsCacheUtils.hpp @@ -9,7 +9,7 @@ #include #include -#include "ErrorCode.hpp" +#include namespace clp_s { /** From 069a495d4e5a497404bed5d7714c16a2b9f675fe Mon Sep 17 00:00:00 2001 From: davemarco <83603688+davemarco@users.noreply.github.com> Date: Sun, 28 Jun 2026 16:08:47 -0400 Subject: [PATCH 40/56] Address review: validate bucket size, clear flushed buffer, explicit reducer dispatch, rename aggregation error. - CountByTimeAggregation rejects non-positive bucket sizes. - ResultsCacheSink clears its buffer after a successful flush to avoid duplicate inserts. - Reducer output dispatch handles count-by-time explicitly and throws on unsupported aggregations. - Rename KvIrSearchErrorEnum::CountSupportNotImplemented to AggregationSupportNotImplemented. --- components/core/src/clp_s/Aggregation.hpp | 7 ++++++- components/core/src/clp_s/AggregationSink.cpp | 1 + components/core/src/clp_s/clp-s.cpp | 13 +++++++++++-- components/core/src/clp_s/kv_ir_search.cpp | 6 +++--- components/core/src/clp_s/kv_ir_search.hpp | 4 ++-- 5 files changed, 23 insertions(+), 8 deletions(-) diff --git a/components/core/src/clp_s/Aggregation.hpp b/components/core/src/clp_s/Aggregation.hpp index 3137222a75..f3f6d0fb4a 100644 --- a/components/core/src/clp_s/Aggregation.hpp +++ b/components/core/src/clp_s/Aggregation.hpp @@ -4,6 +4,7 @@ #include #include #include +#include #include #include #include @@ -51,7 +52,11 @@ class CountByTimeAggregation { static constexpr bool cNeedsMetadata = true; static constexpr bool cNeedsMarshalledRecord = false; - explicit CountByTimeAggregation(int64_t bucket_size_ms) : m_bucket_size_ms{bucket_size_ms} {} + explicit CountByTimeAggregation(int64_t bucket_size_ms) : m_bucket_size_ms{bucket_size_ms} { + if (bucket_size_ms <= 0) { + throw std::invalid_argument("CountByTimeAggregation bucket size must be positive."); + } + } [[nodiscard]] auto get_bucket_size_ms() const -> int64_t { return m_bucket_size_ms; } diff --git a/components/core/src/clp_s/AggregationSink.cpp b/components/core/src/clp_s/AggregationSink.cpp index 6ce94ac6e3..3f596edd95 100644 --- a/components/core/src/clp_s/AggregationSink.cpp +++ b/components/core/src/clp_s/AggregationSink.cpp @@ -55,6 +55,7 @@ auto ResultsCacheSink::finish() -> ErrorCode { try { m_collection.insert_many(m_results); + m_results.clear(); } catch (mongocxx::exception const& e) { return ErrorCode::ErrorCodeFailureDbBulkWrite; } diff --git a/components/core/src/clp_s/clp-s.cpp b/components/core/src/clp_s/clp-s.cpp index e17a19b76e..faca0ce43b 100644 --- a/components/core/src/clp_s/clp-s.cpp +++ b/components/core/src/clp_s/clp-s.cpp @@ -4,6 +4,7 @@ #include #include #include +#include #include #include #include @@ -317,13 +318,20 @@ bool search_archive( output_handler = std::make_unique( reducer_socket_fd ); - } else { + } else if (std::holds_alternative( + aggregation + )) { output_handler = std::make_unique( reducer_socket_fd, std::get(aggregation) .get_bucket_size_ms() ); + } else { + throw std::invalid_argument( + "The reducer output handler only supports the count and " + "count-by-time aggregations." + ); } }, [&](CommandLineArguments::ResultsCacheOutputHandlerOptions const& options) @@ -514,7 +522,8 @@ int main(int argc, char const* argv[]) { if (KvIrSearchError{KvIrSearchErrorEnum::ProjectionSupportNotImplemented} == error || KvIrSearchError{KvIrSearchErrorEnum::UnsupportedOutputHandlerType} == error - || KvIrSearchError{KvIrSearchErrorEnum::CountSupportNotImplemented} == error) + || KvIrSearchError{KvIrSearchErrorEnum::AggregationSupportNotImplemented} + == error) { // These errors are treated as non-fatal because they result from unsupported // features. However, this approach may cause archives with this extension to be diff --git a/components/core/src/clp_s/kv_ir_search.cpp b/components/core/src/clp_s/kv_ir_search.cpp index 65b969d3f9..5a1574aa56 100644 --- a/components/core/src/clp_s/kv_ir_search.cpp +++ b/components/core/src/clp_s/kv_ir_search.cpp @@ -248,7 +248,7 @@ auto search_kv_ir_stream( if (command_line_arguments.get_aggregation().has_value()) { SPDLOG_ERROR("kv-ir search: Aggregation support is not implemented."); - return KvIrSearchError{KvIrSearchErrorEnum::CountSupportNotImplemented}; + return KvIrSearchError{KvIrSearchErrorEnum::AggregationSupportNotImplemented}; } auto const raw_reader{ @@ -313,8 +313,8 @@ auto KvIrSearchErrorCategory::message(KvIrSearchErrorEnum error_enum) const -> s switch (error_enum) { case KvIrSearchErrorEnum::ClpLegacyError: return "clp legacy error."; - case KvIrSearchErrorEnum::CountSupportNotImplemented: - return "Count support is not implemented."; + case KvIrSearchErrorEnum::AggregationSupportNotImplemented: + return "Aggregation support is not implemented."; case KvIrSearchErrorEnum::DeserializerCreationFailure: return "Failed to create `clp::ffi::ir_stream::Deserializer`."; case KvIrSearchErrorEnum::ProjectionSupportNotImplemented: diff --git a/components/core/src/clp_s/kv_ir_search.hpp b/components/core/src/clp_s/kv_ir_search.hpp index 4d5d9e98a7..adf9c15d85 100644 --- a/components/core/src/clp_s/kv_ir_search.hpp +++ b/components/core/src/clp_s/kv_ir_search.hpp @@ -14,7 +14,7 @@ namespace clp_s { enum class KvIrSearchErrorEnum : uint8_t { ClpLegacyError = 1, - CountSupportNotImplemented, + AggregationSupportNotImplemented, DeserializerCreationFailure, ProjectionSupportNotImplemented, StreamReaderCreationFailure, @@ -31,7 +31,7 @@ using KvIrSearchError = ystdlib::error_handling::ErrorCode; * @param reducer_socket_fd * @return A void result on success, or an error code indicating the failure: * - KvIrSearchErrorEnum::ClpLegacyError if a `clp::TraceableException` is caught. - * - KvIrSearchErrorEnum::CountSupportNotImplemented if count-related features are enabled. + * - KvIrSearchErrorEnum::AggregationSupportNotImplemented if an aggregation is requested. * - KvIrSearchErrorEnum::ProjectionSupportNotImplemented if projection is non-empty. * - KvIrSearchErrorEnum::StreamReaderCreationFailure if the stream reader cannot be successfully * created. From d990339dc7198c040fb14c791561822605d18d56 Mon Sep 17 00:00:00 2001 From: davemarco <83603688+davemarco@users.noreply.github.com> Date: Sun, 28 Jun 2026 22:47:52 -0400 Subject: [PATCH 41/56] Move results-cache connect helper into a .cpp so the header lints under clang-tidy. Make connect_to_results_cache a non-template function defined in ResultsCacheUtils.cpp instead of a header-only template. The template existed only to throw each caller's nested OperationFailed, but the call sites are caught as std::exception and TraceableException::what() ignores the exception type, so collapsing them into a single ResultsCacheConnectionError is behavior-preserving. This also gives the header a sibling .cpp, so clang-tidy resolves the mongocxx include path (header-only templates fall back to a non-mongo compile command and fail with "mongocxx/client.hpp file not found"). Removes the now-dead nested OperationFailed classes and the reducer parse-time comment. --- components/core/src/clp_s/AggregationSink.cpp | 2 +- components/core/src/clp_s/AggregationSink.hpp | 9 ------ components/core/src/clp_s/CMakeLists.txt | 1 + components/core/src/clp_s/IntFloatCompare.hpp | 13 +++----- .../core/src/clp_s/OutputHandlerImpl.cpp | 2 +- .../core/src/clp_s/OutputHandlerImpl.hpp | 7 ----- .../core/src/clp_s/ResultsCacheUtils.cpp | 31 +++++++++++++++++++ .../core/src/clp_s/ResultsCacheUtils.hpp | 25 ++++++--------- components/core/src/clp_s/clp-s.cpp | 2 -- 9 files changed, 48 insertions(+), 44 deletions(-) create mode 100644 components/core/src/clp_s/ResultsCacheUtils.cpp diff --git a/components/core/src/clp_s/AggregationSink.cpp b/components/core/src/clp_s/AggregationSink.cpp index 3f596edd95..cb06467b55 100644 --- a/components/core/src/clp_s/AggregationSink.cpp +++ b/components/core/src/clp_s/AggregationSink.cpp @@ -29,7 +29,7 @@ auto StdoutSink::write(AggregationResult const& result) -> void { ResultsCacheSink::ResultsCacheSink(string_view uri, string_view collection, string_view archive_id) : m_archive_id{archive_id} { - m_collection = connect_to_results_cache(uri, collection, m_client); + m_collection = connect_to_results_cache(uri, collection, m_client); } auto ResultsCacheSink::write(AggregationResult const& result) -> void { diff --git a/components/core/src/clp_s/AggregationSink.hpp b/components/core/src/clp_s/AggregationSink.hpp index afccf8207c..5479cbac39 100644 --- a/components/core/src/clp_s/AggregationSink.hpp +++ b/components/core/src/clp_s/AggregationSink.hpp @@ -11,7 +11,6 @@ #include #include -#include namespace clp_s { /** @@ -68,14 +67,6 @@ class StdoutSink : public AggregationSink { */ class ResultsCacheSink : public AggregationSink { public: - // Types - class OperationFailed : public TraceableException { - public: - // Constructors - OperationFailed(ErrorCode error_code, char const* const filename, int line_number) - : TraceableException(error_code, filename, line_number) {} - }; - // Constructors ResultsCacheSink( std::string_view uri, diff --git a/components/core/src/clp_s/CMakeLists.txt b/components/core/src/clp_s/CMakeLists.txt index cdd07ab1c8..1437d66b27 100644 --- a/components/core/src/clp_s/CMakeLists.txt +++ b/components/core/src/clp_s/CMakeLists.txt @@ -472,6 +472,7 @@ set( kv_ir_search.hpp OutputHandlerImpl.cpp OutputHandlerImpl.hpp + ResultsCacheUtils.cpp ResultsCacheUtils.hpp TraceableException.hpp ) diff --git a/components/core/src/clp_s/IntFloatCompare.hpp b/components/core/src/clp_s/IntFloatCompare.hpp index 0da766343e..a6082b455e 100644 --- a/components/core/src/clp_s/IntFloatCompare.hpp +++ b/components/core/src/clp_s/IntFloatCompare.hpp @@ -23,8 +23,8 @@ namespace clp_s { // `2^63` rather than `INT64_MAX` because `2^63` is exactly representable as a double and // `INT64_MAX` is not. constexpr double cInt64UpperBound{9223372036854775808.0}; -// `-2^63`, equal to `INT64_MIN`: any double `<` this is smaller than every `int64_t`. Both `-2^63` -// and `INT64_MIN` are the same value and are exactly representable as a double. +// `-2^63`, which equals `INT64_MIN` and is exactly representable as a double: any double `<` this +// is smaller than every `int64_t`. constexpr double cInt64Min{-9223372036854775808.0}; [[nodiscard]] inline auto is_less(int64_t lhs, int64_t rhs) -> bool { @@ -56,13 +56,13 @@ constexpr double cInt64Min{-9223372036854775808.0}; if (rhs < cInt64Min) { return false; } - // `rhs` is within the `int64_t` range, so truncating it toward zero and casting is exact. + // `rhs` is in `int64_t` range, so its truncated value fits an `int64_t` with no rounding. auto const truncated{std::trunc(rhs)}; auto const rhs_int{static_cast(truncated)}; if (lhs != rhs_int) { return lhs < rhs_int; } - // Same integer part: `lhs < rhs` exactly when `rhs` has a positive fractional part. + // Equal integer parts: `lhs < rhs` only if `rhs` carries a fractional remainder. return rhs > truncated; } @@ -73,25 +73,20 @@ constexpr double cInt64Min{-9223372036854775808.0}; * @return Whether `lhs < rhs`. */ [[nodiscard]] inline auto is_less(double lhs, int64_t rhs) -> bool { - // NaN is unordered, so it is never less than anything. if (std::isnan(lhs)) { return false; } - // `lhs` is larger than every `int64_t`, so it cannot be less than `rhs`. if (lhs >= cInt64UpperBound) { return false; } - // `lhs` is smaller than every `int64_t`, so it must be less than `rhs`. if (lhs < cInt64Min) { return true; } - // `lhs` is within the `int64_t` range, so truncating it toward zero and casting is exact. auto const truncated{std::trunc(lhs)}; auto const lhs_int{static_cast(truncated)}; if (lhs_int != rhs) { return lhs_int < rhs; } - // Same integer part: `lhs < rhs` exactly when `lhs` has a negative fractional part. return lhs < truncated; } } // namespace clp_s diff --git a/components/core/src/clp_s/OutputHandlerImpl.cpp b/components/core/src/clp_s/OutputHandlerImpl.cpp index ff81e1e926..4a7ae20813 100644 --- a/components/core/src/clp_s/OutputHandlerImpl.cpp +++ b/components/core/src/clp_s/OutputHandlerImpl.cpp @@ -81,7 +81,7 @@ ResultsCacheOutputHandler::ResultsCacheOutputHandler( m_batch_size{batch_size}, m_max_num_results{max_num_results}, m_dataset{dataset} { - m_collection = connect_to_results_cache(uri, collection, m_client); + m_collection = connect_to_results_cache(uri, collection, m_client); m_results.reserve(m_batch_size); } diff --git a/components/core/src/clp_s/OutputHandlerImpl.hpp b/components/core/src/clp_s/OutputHandlerImpl.hpp index 7ed4fb8f70..2bb37fc4a1 100644 --- a/components/core/src/clp_s/OutputHandlerImpl.hpp +++ b/components/core/src/clp_s/OutputHandlerImpl.hpp @@ -161,13 +161,6 @@ class ResultsCacheOutputHandler : public ::clp_s::search::OutputHandler { } }; - class OperationFailed : public TraceableException { - public: - // Constructors - OperationFailed(ErrorCode error_code, char const* const filename, int line_number) - : TraceableException(error_code, filename, line_number) {} - }; - // Constructor ResultsCacheOutputHandler( std::string_view uri, diff --git a/components/core/src/clp_s/ResultsCacheUtils.cpp b/components/core/src/clp_s/ResultsCacheUtils.cpp new file mode 100644 index 0000000000..81384fe4d9 --- /dev/null +++ b/components/core/src/clp_s/ResultsCacheUtils.cpp @@ -0,0 +1,31 @@ +#include "ResultsCacheUtils.hpp" + +#include +#include + +#include +#include +#include +#include + +#include + +namespace clp_s { +auto connect_to_results_cache( + std::string_view uri, + std::string_view collection, + mongocxx::client& client +) -> mongocxx::collection { + try { + auto mongo_uri = mongocxx::uri{std::string{uri}}; + client = mongocxx::client(mongo_uri); + return client[mongo_uri.database()][std::string{collection}]; + } catch (mongocxx::exception const&) { + throw ResultsCacheConnectionError( + ErrorCode::ErrorCodeBadParamDbUri, + __FILENAME__, + __LINE__ + ); + } +} +} // namespace clp_s diff --git a/components/core/src/clp_s/ResultsCacheUtils.hpp b/components/core/src/clp_s/ResultsCacheUtils.hpp index 08248c35b5..a614b76047 100644 --- a/components/core/src/clp_s/ResultsCacheUtils.hpp +++ b/components/core/src/clp_s/ResultsCacheUtils.hpp @@ -1,39 +1,34 @@ #ifndef CLP_S_RESULTSCACHEUTILS_HPP #define CLP_S_RESULTSCACHEUTILS_HPP -#include #include #include #include -#include -#include #include +#include namespace clp_s { +class ResultsCacheConnectionError : public TraceableException { +public: + ResultsCacheConnectionError(ErrorCode error_code, char const* const filename, int line_number) + : TraceableException{error_code, filename, line_number} {} +}; + /** * Connects to the results cache and returns the requested collection. - * @tparam OperationFailedT The type of exception to throw on failure. * @param uri * @param collection * @param client Returns the connected client. * @return The collection. + * @throw ResultsCacheConnectionError if connecting to the results cache fails. */ -template -auto connect_to_results_cache( +[[nodiscard]] auto connect_to_results_cache( std::string_view uri, std::string_view collection, mongocxx::client& client -) -> mongocxx::collection { - try { - auto mongo_uri = mongocxx::uri{std::string{uri}}; - client = mongocxx::client(mongo_uri); - return client[mongo_uri.database()][std::string{collection}]; - } catch (mongocxx::exception const& e) { - throw OperationFailedT(ErrorCode::ErrorCodeBadParamDbUri, __FILENAME__, __LINE__); - } -} +) -> mongocxx::collection; } // namespace clp_s #endif // CLP_S_RESULTSCACHEUTILS_HPP diff --git a/components/core/src/clp_s/clp-s.cpp b/components/core/src/clp_s/clp-s.cpp index faca0ce43b..f416740375 100644 --- a/components/core/src/clp_s/clp-s.cpp +++ b/components/core/src/clp_s/clp-s.cpp @@ -309,8 +309,6 @@ bool search_archive( ); }, [&](CommandLineArguments::ReducerOutputHandlerOptions const&) -> void { - // The reducer only supports count and count-by-time; min/max are - // rejected during command-line parsing. auto const& aggregation{ command_line_arguments.get_aggregation().value() }; From e56b93f95ef3586135561b743dcf2816d4ad379c Mon Sep 17 00:00:00 2001 From: davemarco <83603688+davemarco@users.noreply.github.com> Date: Thu, 2 Jul 2026 16:19:20 -0400 Subject: [PATCH 42/56] Constrain aggregators with a concept and template the output handler; address review. --- components/core/src/clp_s/Aggregation.cpp | 15 ------ components/core/src/clp_s/Aggregation.hpp | 38 ++++++++----- components/core/src/clp_s/AggregationSink.hpp | 10 ++-- .../core/src/clp_s/OutputHandlerImpl.cpp | 3 +- .../core/src/clp_s/OutputHandlerImpl.hpp | 54 ++++++++++++------- components/core/src/clp_s/clp-s.cpp | 4 +- 6 files changed, 72 insertions(+), 52 deletions(-) diff --git a/components/core/src/clp_s/Aggregation.cpp b/components/core/src/clp_s/Aggregation.cpp index 33ad37aab0..e8af540315 100644 --- a/components/core/src/clp_s/Aggregation.cpp +++ b/components/core/src/clp_s/Aggregation.cpp @@ -4,7 +4,6 @@ #include #include #include -#include #include #include #include @@ -119,18 +118,4 @@ auto MinMaxAggregation::get_results() const -> std::vector { result.emplace_back(key, value); return {std::move(result)}; } - -auto aggregation_needs_metadata(Aggregation const& aggregation) -> bool { - return std::visit( - [](auto const& agg) { return std::decay_t::cNeedsMetadata; }, - aggregation - ); -} - -auto aggregation_needs_marshalled_record(Aggregation const& aggregation) -> bool { - return std::visit( - [](auto const& agg) { return std::decay_t::cNeedsMarshalledRecord; }, - aggregation - ); -} } // namespace clp_s diff --git a/components/core/src/clp_s/Aggregation.hpp b/components/core/src/clp_s/Aggregation.hpp index f3f6d0fb4a..06e6ef29be 100644 --- a/components/core/src/clp_s/Aggregation.hpp +++ b/components/core/src/clp_s/Aggregation.hpp @@ -1,6 +1,7 @@ #ifndef CLP_S_AGGREGATION_HPP #define CLP_S_AGGREGATION_HPP +#include #include #include #include @@ -24,6 +25,31 @@ using AggregationValue = std::variant; */ using AggregationResult = std::vector>; +/** + * Requirement for the search aggregator interface. + * @tparam AggregatorType The type of the aggregator. + */ +template +concept AggregatorReq + = requires(AggregatorType aggregator, std::string_view message, epochtime_t timestamp_ms) { + /** + * Folds one matched record into the running aggregate. + */ + { aggregator.add_record(message, timestamp_ms) } -> std::same_as; + + /** + * @return The aggregate's result documents. + */ + { aggregator.get_results() } -> std::same_as>; + + /** + * Whether the caller must supply per-record metadata and the marshalled record, + * respectively. + */ + { AggregatorType::cNeedsMetadata } -> std::convertible_to; + { AggregatorType::cNeedsMarshalledRecord } -> std::convertible_to; + }; + /** * Counts the number of matched records. */ @@ -105,18 +131,6 @@ class MinMaxAggregation { * A search aggregation to perform. */ using Aggregation = std::variant; - -/** - * @param aggregation - * @return Whether the aggregation needs per-record metadata. - */ -[[nodiscard]] auto aggregation_needs_metadata(Aggregation const& aggregation) -> bool; - -/** - * @param aggregation - * @return Whether the aggregation needs each matched record marshalled into its JSON message. - */ -[[nodiscard]] auto aggregation_needs_marshalled_record(Aggregation const& aggregation) -> bool; } // namespace clp_s #endif // CLP_S_AGGREGATION_HPP diff --git a/components/core/src/clp_s/AggregationSink.hpp b/components/core/src/clp_s/AggregationSink.hpp index 5479cbac39..f6797c3b3c 100644 --- a/components/core/src/clp_s/AggregationSink.hpp +++ b/components/core/src/clp_s/AggregationSink.hpp @@ -21,15 +21,17 @@ class AggregationSink { // Constructors AggregationSink() = default; - // Destructor - virtual ~AggregationSink() = default; - - // Explicitly disable copy and move constructor/assignment + // Delete copy constructor and assignment operator AggregationSink(AggregationSink const&) = delete; auto operator=(AggregationSink const&) -> AggregationSink& = delete; + + // Delete move constructor and assignment operator AggregationSink(AggregationSink&&) = delete; auto operator=(AggregationSink&&) -> AggregationSink& = delete; + // Destructor + virtual ~AggregationSink() = default; + // Methods /** * Writes one result document. diff --git a/components/core/src/clp_s/OutputHandlerImpl.cpp b/components/core/src/clp_s/OutputHandlerImpl.cpp index 4a7ae20813..e2ebbe48f2 100644 --- a/components/core/src/clp_s/OutputHandlerImpl.cpp +++ b/components/core/src/clp_s/OutputHandlerImpl.cpp @@ -14,12 +14,13 @@ #include #include +#include + #include "../clp/networking/socket_utils.hpp" #include "../reducer/CountOperator.hpp" #include "../reducer/network_utils.hpp" #include "../reducer/Record.hpp" #include "archive_constants.hpp" -#include "ResultsCacheUtils.hpp" #include "search/OutputHandler.hpp" #include "TraceableException.hpp" diff --git a/components/core/src/clp_s/OutputHandlerImpl.hpp b/components/core/src/clp_s/OutputHandlerImpl.hpp index 2bb37fc4a1..d3cd9284d9 100644 --- a/components/core/src/clp_s/OutputHandlerImpl.hpp +++ b/components/core/src/clp_s/OutputHandlerImpl.hpp @@ -11,17 +11,20 @@ #include #include #include +#include +#include #include #include #include #include +#include +#include +#include + #include "../reducer/Pipeline.hpp" #include "../reducer/RecordGroupIterator.hpp" -#include "Aggregation.hpp" -#include "AggregationSink.hpp" -#include "CommandLineArguments.hpp" #include "Defs.hpp" #include "FileWriter.hpp" #include "search/OutputHandler.hpp" @@ -285,12 +288,15 @@ class CountByTimeReducerOutputHandler : public search::OutputHandler { /** * Output handler that runs a search `Aggregation` and writes its results to an `AggregationSink`. + * @tparam AggT The concrete aggregator, selected at construction by + * `make_aggregation_output_handler`. */ +template class AggregationOutputHandler : public search::OutputHandler { public: // Constructors - AggregationOutputHandler(Aggregation aggregation, std::unique_ptr sink) - : search::OutputHandler{aggregation_needs_metadata(aggregation), aggregation_needs_marshalled_record(aggregation)}, + AggregationOutputHandler(AggT aggregation, std::unique_ptr sink) + : search::OutputHandler{AggT::cNeedsMetadata, AggT::cNeedsMarshalledRecord}, m_aggregation{std::move(aggregation)}, m_sink{std::move(sink)} {} @@ -301,15 +307,10 @@ class AggregationOutputHandler : public search::OutputHandler { std::string_view archive_id, int64_t log_event_idx ) -> void override { - std::visit( - [&](auto& aggregation) { aggregation.add_record(message, timestamp_ms); }, - m_aggregation - ); + m_aggregation.add_record(message, timestamp_ms); } - auto write(std::string_view message) -> void override { - std::visit([&](auto& aggregation) { aggregation.add_record(message, 0); }, m_aggregation); - } + auto write(std::string_view message) -> void override { m_aggregation.add_record(message, 0); } // Methods overriding OutputHandler /** @@ -318,11 +319,7 @@ class AggregationOutputHandler : public search::OutputHandler { * @return The sink's error code on failure */ auto finish() -> ErrorCode override { - auto const results{std::visit( - [](auto& aggregation) { return aggregation.get_results(); }, - m_aggregation - )}; - for (auto const& result : results) { + for (auto const& result : m_aggregation.get_results()) { m_sink->write(result); } return m_sink->finish(); @@ -330,10 +327,31 @@ class AggregationOutputHandler : public search::OutputHandler { private: // Data members - Aggregation m_aggregation; + AggT m_aggregation; std::unique_ptr m_sink; }; +/** + * Builds the `AggregationOutputHandler` specialization for `aggregation`'s active alternative. + * @param aggregation The aggregation to run. + * @param sink The destination for the aggregation's results. + * @return The constructed output handler. + */ +[[nodiscard]] inline auto +make_aggregation_output_handler(Aggregation aggregation, std::unique_ptr sink) + -> std::unique_ptr { + return std::visit( + [&](auto&& agg) -> std::unique_ptr { + using AggT = std::decay_t; + return std::make_unique>( + std::move(agg), + std::move(sink) + ); + }, + std::move(aggregation) + ); +} + /** * Output handler that records all results in a provided vector. */ diff --git a/components/core/src/clp_s/clp-s.cpp b/components/core/src/clp_s/clp-s.cpp index 16916be63c..3ecde01487 100644 --- a/components/core/src/clp_s/clp-s.cpp +++ b/components/core/src/clp_s/clp-s.cpp @@ -343,7 +343,7 @@ bool search_archive( options.dataset ); } else { - output_handler = std::make_unique( + output_handler = clp_s::make_aggregation_output_handler( aggregation.value(), std::make_unique( options.uri, @@ -358,7 +358,7 @@ bool search_archive( if (false == aggregation.has_value()) { output_handler = std::make_unique(); } else { - output_handler = std::make_unique( + output_handler = clp_s::make_aggregation_output_handler( aggregation.value(), std::make_unique( archive_reader->get_archive_id() From 13ac1ac917aa01c7c82ae7142e3d5730038ae157 Mon Sep 17 00:00:00 2001 From: davemarco <83603688+davemarco@users.noreply.github.com> Date: Fri, 3 Jul 2026 11:04:53 -0400 Subject: [PATCH 43/56] Clean up aggregation doc comments --- components/core/src/clp_s/Aggregation.hpp | 18 ++++++++++++------ components/core/src/clp_s/AggregationSink.hpp | 6 +++--- .../core/src/clp_s/OutputHandlerImpl.hpp | 7 +++---- .../core/src/clp_s/ResultsCacheUtils.hpp | 1 + 4 files changed, 19 insertions(+), 13 deletions(-) diff --git a/components/core/src/clp_s/Aggregation.hpp b/components/core/src/clp_s/Aggregation.hpp index 06e6ef29be..04738b89eb 100644 --- a/components/core/src/clp_s/Aggregation.hpp +++ b/components/core/src/clp_s/Aggregation.hpp @@ -26,27 +26,33 @@ using AggregationValue = std::variant; using AggregationResult = std::vector>; /** - * Requirement for the search aggregator interface. + * Requirements a type must satisfy to be used as an aggregator. * @tparam AggregatorType The type of the aggregator. */ template concept AggregatorReq = requires(AggregatorType aggregator, std::string_view message, epochtime_t timestamp_ms) { /** - * Folds one matched record into the running aggregate. + * Adds a record to the aggregate. + * @param message The message in the log event. + * @param timestamp_ms The timestamp of the log event. */ { aggregator.add_record(message, timestamp_ms) } -> std::same_as; /** - * @return The aggregate's result documents. + * Gets the aggregate's results. + * @return The result documents produced by the aggregation. */ { aggregator.get_results() } -> std::same_as>; /** - * Whether the caller must supply per-record metadata and the marshalled record, - * respectively. + * Whether the caller must supply per-record metadata. */ { AggregatorType::cNeedsMetadata } -> std::convertible_to; + + /** + * Whether the caller must supply the marshalled record. + */ { AggregatorType::cNeedsMarshalledRecord } -> std::convertible_to; }; @@ -128,7 +134,7 @@ class MinMaxAggregation { }; /** - * A search aggregation to perform. + * One of the supported aggregations that a search can apply to its matched records. */ using Aggregation = std::variant; } // namespace clp_s diff --git a/components/core/src/clp_s/AggregationSink.hpp b/components/core/src/clp_s/AggregationSink.hpp index f6797c3b3c..5c1e071b45 100644 --- a/components/core/src/clp_s/AggregationSink.hpp +++ b/components/core/src/clp_s/AggregationSink.hpp @@ -14,7 +14,7 @@ namespace clp_s { /** - * Consumes a search aggregation's result documents and writes them to a destination. + * Consumes an aggregation's result documents and writes them to a destination. */ class AggregationSink { public: @@ -35,13 +35,13 @@ class AggregationSink { // Methods /** * Writes one result document. - * @param result + * @param result The result document to write. */ virtual auto write(AggregationResult const& result) -> void = 0; /** * Flushes any buffered results. - * @return ErrorCodeSuccess on success + * @return ErrorCodeSuccess on success or relevant error code on error */ [[nodiscard]] virtual auto finish() -> ErrorCode = 0; }; diff --git a/components/core/src/clp_s/OutputHandlerImpl.hpp b/components/core/src/clp_s/OutputHandlerImpl.hpp index d3cd9284d9..41fade4c92 100644 --- a/components/core/src/clp_s/OutputHandlerImpl.hpp +++ b/components/core/src/clp_s/OutputHandlerImpl.hpp @@ -287,9 +287,8 @@ class CountByTimeReducerOutputHandler : public search::OutputHandler { }; /** - * Output handler that runs a search `Aggregation` and writes its results to an `AggregationSink`. - * @tparam AggT The concrete aggregator, selected at construction by - * `make_aggregation_output_handler`. + * Output handler that runs an `Aggregation` and writes its results to an `AggregationSink`. + * @tparam AggT The type of aggregator to run. */ template class AggregationOutputHandler : public search::OutputHandler { @@ -332,7 +331,7 @@ class AggregationOutputHandler : public search::OutputHandler { }; /** - * Builds the `AggregationOutputHandler` specialization for `aggregation`'s active alternative. + * Creates an output handler that runs the given aggregation. * @param aggregation The aggregation to run. * @param sink The destination for the aggregation's results. * @return The constructed output handler. diff --git a/components/core/src/clp_s/ResultsCacheUtils.hpp b/components/core/src/clp_s/ResultsCacheUtils.hpp index a614b76047..b0639e53dc 100644 --- a/components/core/src/clp_s/ResultsCacheUtils.hpp +++ b/components/core/src/clp_s/ResultsCacheUtils.hpp @@ -12,6 +12,7 @@ namespace clp_s { class ResultsCacheConnectionError : public TraceableException { public: + // Constructors ResultsCacheConnectionError(ErrorCode error_code, char const* const filename, int line_number) : TraceableException{error_code, filename, line_number} {} }; From 31c7b0c75875cce0b00dfe5eabc7b2f023af0395 Mon Sep 17 00:00:00 2001 From: davemarco <83603688+davemarco@users.noreply.github.com> Date: Mon, 6 Jul 2026 10:19:14 -0400 Subject: [PATCH 44/56] feat(clp-s): Support unique-value aggregation output to stdout and the results cache. Add a --unique FIELD aggregation that collects the distinct scalar values of a field across matched records, emitting one {field, value} result document per value through the existing aggregation sinks. Factor the shared field-path tokenization and JSON navigation out of MinMaxAggregation into reusable helpers, and rewrite the reducer output-handler restriction as a whitelist. --- components/core/src/clp_s/Aggregation.cpp | 126 +++++++++++++----- components/core/src/clp_s/Aggregation.hpp | 24 +++- .../core/src/clp_s/CommandLineArguments.cpp | 20 ++- .../core/src/clp_s/CommandLineArguments.hpp | 4 +- .../core/src/clp_s/archive_constants.hpp | 1 + 5 files changed, 134 insertions(+), 41 deletions(-) diff --git a/components/core/src/clp_s/Aggregation.cpp b/components/core/src/clp_s/Aggregation.cpp index e8af540315..3eb1e69914 100644 --- a/components/core/src/clp_s/Aggregation.cpp +++ b/components/core/src/clp_s/Aggregation.cpp @@ -18,6 +18,62 @@ using std::string; using std::string_view; namespace clp_s { +namespace { +/** + * Tokenizes an aggregation's target field into its path components. + * @param field + * @param field_path Returns the tokenized path components. + * @throws std::invalid_argument if `field` cannot be tokenized. + */ +auto tokenize_aggregation_field(string_view field, std::vector& field_path) -> void { + string descriptor_namespace; + if (false + == search::ast::tokenize_column_descriptor(string{field}, field_path, descriptor_namespace)) + { + throw std::invalid_argument("Invalid aggregation field: " + string{field}); + } + if (false == descriptor_namespace.empty()) { + // The tokenizer strips the namespace prefix out of the path, but namespaced fields live + // under a top-level object keyed by that namespace. Prepend it back so the path matches. + field_path.insert(field_path.begin(), descriptor_namespace); + } +} + +/** + * Parses `message` as JSON and navigates to the node at `field_path`. + * @param message + * @param field_path + * @param doc Returns the parsed document, which owns the returned node. + * @return A pointer, valid for the lifetime of `doc`, to the node at `field_path`. + * @return nullptr if `message` is not valid JSON, or if any path component is missing or traverses + * a non-object. + */ +auto find_field_node( + string_view message, + std::vector const& field_path, + nlohmann::json& doc +) -> nlohmann::json const* { + try { + doc = nlohmann::json::parse(message); + } catch (nlohmann::json::exception const&) { + return nullptr; + } + + nlohmann::json const* node{&doc}; + for (auto const& key : field_path) { + if (false == node->is_object()) { + return nullptr; + } + auto const it{node->find(key)}; + if (node->end() == it) { + return nullptr; + } + node = &it.value(); + } + return node; +} +} // namespace + auto CountAggregation::get_results() const -> std::vector { if (0 == m_count) { return {}; @@ -42,21 +98,7 @@ auto CountByTimeAggregation::get_results() const -> std::vector bool { @@ -73,25 +115,8 @@ auto MinMaxAggregation::beats_extreme(Extreme candidate) const -> bool { auto MinMaxAggregation::add_record(string_view message, epochtime_t) -> void { nlohmann::json doc; - try { - doc = nlohmann::json::parse(message); - } catch (nlohmann::json::exception const&) { - return; - } - - nlohmann::json const* node{&doc}; - for (auto const& key : m_field_path) { - if (false == node->is_object()) { - return; - } - auto const it{node->find(key)}; - if (node->end() == it) { - return; - } - node = &it.value(); - } - - if (false == node->is_number()) { + auto const* const node{find_field_node(message, m_field_path, doc)}; + if (nullptr == node || false == node->is_number()) { return; } Extreme const candidate{ @@ -118,4 +143,37 @@ auto MinMaxAggregation::get_results() const -> std::vector { result.emplace_back(key, value); return {std::move(result)}; } + +UniqueAggregation::UniqueAggregation(string_view field) : m_field{field} { + tokenize_aggregation_field(field, m_field_path); +} + +auto UniqueAggregation::add_record(string_view message, epochtime_t) -> void { + nlohmann::json doc; + auto const* const node{find_field_node(message, m_field_path, doc)}; + if (nullptr == node) { + return; + } + + if (node->is_number_integer()) { + m_values.emplace(node->get()); + } else if (node->is_number_float()) { + m_values.emplace(node->get()); + } else if (node->is_string()) { + m_values.emplace(node->get()); + } + // Values that aren't scalars (e.g. objects, arrays, booleans, or null) are ignored. +} + +auto UniqueAggregation::get_results() const -> std::vector { + std::vector results; + results.reserve(m_values.size()); + for (auto const& value : m_values) { + AggregationResult result; + result.emplace_back(constants::results_cache::search::cField, m_field); + result.emplace_back(constants::results_cache::search::cValue, value); + results.push_back(std::move(result)); + } + return results; +} } // namespace clp_s diff --git a/components/core/src/clp_s/Aggregation.hpp b/components/core/src/clp_s/Aggregation.hpp index 04738b89eb..6d64af55f3 100644 --- a/components/core/src/clp_s/Aggregation.hpp +++ b/components/core/src/clp_s/Aggregation.hpp @@ -5,6 +5,7 @@ #include #include #include +#include #include #include #include @@ -133,10 +134,31 @@ class MinMaxAggregation { std::optional m_extreme; }; +/** + * Collects the distinct values of a target field across matched records. + */ +class UniqueAggregation { +public: + static constexpr bool cNeedsMetadata = false; + static constexpr bool cNeedsMarshalledRecord = true; + + explicit UniqueAggregation(std::string_view field); + + auto add_record(std::string_view message, [[maybe_unused]] epochtime_t timestamp_ms) -> void; + + [[nodiscard]] auto get_results() const -> std::vector; + +private: + std::string m_field; + std::vector m_field_path; + std::set m_values; +}; + /** * One of the supported aggregations that a search can apply to its matched records. */ -using Aggregation = std::variant; +using Aggregation = std:: + variant; } // namespace clp_s #endif // CLP_S_AGGREGATION_HPP diff --git a/components/core/src/clp_s/CommandLineArguments.cpp b/components/core/src/clp_s/CommandLineArguments.cpp index ca22f95ffe..9544659368 100644 --- a/components/core/src/clp_s/CommandLineArguments.cpp +++ b/components/core/src/clp_s/CommandLineArguments.cpp @@ -793,6 +793,10 @@ CommandLineArguments::parse_arguments(int argc, char const** argv) { "max", po::value(&aggregation_field)->value_name("FIELD"), "Find the maximum value of the given field" + )( + "unique", + po::value(&aggregation_field)->value_name("FIELD"), + "Find the distinct values of the given field" ); // clang-format on search_options.add(aggregation_options); @@ -1142,17 +1146,20 @@ auto CommandLineArguments::parse_aggregation_options( auto const set_aggregation = [&](Aggregation value) { if (aggregation.has_value()) { throw std::invalid_argument( - "The --count, --count-by-time, --min, and --max options are mutually exclusive." + "The --count, --count-by-time, --min, --max, and --unique options are mutually" + " exclusive." ); } aggregation = std::move(value); }; auto const validate_aggregation_field = [&]() { if (aggregation_field.empty()) { - throw std::invalid_argument("The --min and --max options require a field."); + throw std::invalid_argument("The --min, --max, and --unique options require a field."); } if (search::ast::has_unescaped_wildcards(aggregation_field)) { - throw std::invalid_argument("The --min and --max field must not contain wildcards."); + throw std::invalid_argument( + "The --min, --max, and --unique field must not contain wildcards." + ); } }; @@ -1173,6 +1180,10 @@ auto CommandLineArguments::parse_aggregation_options( validate_aggregation_field(); set_aggregation(MinMaxAggregation{true, aggregation_field}); } + if (parsed_options.count("unique")) { + validate_aggregation_field(); + set_aggregation(UniqueAggregation{aggregation_field}); + } return aggregation; } @@ -1216,7 +1227,8 @@ void CommandLineArguments::parse_reducer_output_handler_options( } if (false == m_aggregation.has_value() - || std::holds_alternative(m_aggregation.value())) + || (false == std::holds_alternative(m_aggregation.value()) + && false == std::holds_alternative(m_aggregation.value()))) { throw std::invalid_argument( "The reducer output handler currently only supports count and count-by-time" diff --git a/components/core/src/clp_s/CommandLineArguments.hpp b/components/core/src/clp_s/CommandLineArguments.hpp index 3ee00246cc..1470d3c32c 100644 --- a/components/core/src/clp_s/CommandLineArguments.hpp +++ b/components/core/src/clp_s/CommandLineArguments.hpp @@ -160,10 +160,10 @@ class CommandLineArguments { * Builds the requested aggregation from the parsed options. * @param parsed_options * @param count_by_time_bucket_size_ms Bucket size for count-by-time. Only used by that option. - * @param aggregation_field Field for min/max. Only used by those options. + * @param aggregation_field Field for min/max/unique. Only used by those options. * @return The requested aggregation, or std::nullopt if none was requested. * @throws std::invalid_argument if multiple aggregations are specified, the bucket size is - * non-positive, or a min/max field is empty or contains wildcards. + * non-positive, or a min/max/unique field is empty or contains wildcards. */ [[nodiscard]] static auto parse_aggregation_options( boost::program_options::variables_map const& parsed_options, diff --git a/components/core/src/clp_s/archive_constants.hpp b/components/core/src/clp_s/archive_constants.hpp index 0ab8cf824e..0dbf3d4c68 100644 --- a/components/core/src/clp_s/archive_constants.hpp +++ b/components/core/src/clp_s/archive_constants.hpp @@ -63,6 +63,7 @@ constexpr char cCount[]{"count"}; constexpr char cMin[]{"min"}; constexpr char cMax[]{"max"}; constexpr char cField[]{"field"}; +constexpr char cValue[]{"value"}; } // namespace results_cache::search } // namespace clp_s::constants #endif // CLP_S_ARCHIVE_CONSTANTS_HPP From 7d2f35eb2744bed6a62c94a6fb266fbdf9fce9d8 Mon Sep 17 00:00:00 2001 From: davemarco <83603688+davemarco@users.noreply.github.com> Date: Mon, 6 Jul 2026 10:32:07 -0400 Subject: [PATCH 45/56] Collect boolean values in --unique and batch results-cache flushing. Add bool to AggregationValue so --unique treats booleans as distinct scalar values, and flush ResultsCacheSink in batches so high-cardinality aggregations don't buffer every document, surfacing mid-stream database errors. --- components/core/src/clp_s/Aggregation.cpp | 4 +- components/core/src/clp_s/Aggregation.hpp | 2 +- components/core/src/clp_s/AggregationSink.cpp | 48 ++++++++++++++----- components/core/src/clp_s/AggregationSink.hpp | 20 +++++++- components/core/src/clp_s/clp-s.cpp | 1 + 5 files changed, 59 insertions(+), 16 deletions(-) diff --git a/components/core/src/clp_s/Aggregation.cpp b/components/core/src/clp_s/Aggregation.cpp index 3eb1e69914..8b99a4ecde 100644 --- a/components/core/src/clp_s/Aggregation.cpp +++ b/components/core/src/clp_s/Aggregation.cpp @@ -161,8 +161,10 @@ auto UniqueAggregation::add_record(string_view message, epochtime_t) -> void { m_values.emplace(node->get()); } else if (node->is_string()) { m_values.emplace(node->get()); + } else if (node->is_boolean()) { + m_values.emplace(node->get()); } - // Values that aren't scalars (e.g. objects, arrays, booleans, or null) are ignored. + // Values that aren't scalars (e.g. objects, arrays, or null) are ignored. } auto UniqueAggregation::get_results() const -> std::vector { diff --git a/components/core/src/clp_s/Aggregation.hpp b/components/core/src/clp_s/Aggregation.hpp index 6d64af55f3..a4636846d8 100644 --- a/components/core/src/clp_s/Aggregation.hpp +++ b/components/core/src/clp_s/Aggregation.hpp @@ -19,7 +19,7 @@ namespace clp_s { /** * A single typed value in an aggregation result document. */ -using AggregationValue = std::variant; +using AggregationValue = std::variant; /** * One aggregation result document: an ordered list of typed key-value pairs. diff --git a/components/core/src/clp_s/AggregationSink.cpp b/components/core/src/clp_s/AggregationSink.cpp index cb06467b55..3dc94852c7 100644 --- a/components/core/src/clp_s/AggregationSink.cpp +++ b/components/core/src/clp_s/AggregationSink.cpp @@ -1,5 +1,6 @@ #include "AggregationSink.hpp" +#include #include #include #include @@ -27,12 +28,38 @@ auto StdoutSink::write(AggregationResult const& result) -> void { std::cout << document.dump() << '\n'; } -ResultsCacheSink::ResultsCacheSink(string_view uri, string_view collection, string_view archive_id) - : m_archive_id{archive_id} { +ResultsCacheSink::ResultsCacheSink( + string_view uri, + string_view collection, + uint64_t batch_size, + string_view archive_id +) + : m_batch_size{batch_size}, + m_archive_id{archive_id} { m_collection = connect_to_results_cache(uri, collection, m_client); } +auto ResultsCacheSink::flush_buffer() -> ErrorCode { + if (m_results.empty()) { + return ErrorCode::ErrorCodeSuccess; + } + + try { + m_collection.insert_many(m_results); + } catch (mongocxx::exception const& e) { + return ErrorCode::ErrorCodeFailureDbBulkWrite; + } + m_results.clear(); + return ErrorCode::ErrorCodeSuccess; +} + auto ResultsCacheSink::write(AggregationResult const& result) -> void { + // Once a flush has failed, stop buffering so memory stays bounded; the error surfaces in + // `finish()`. + if (ErrorCode::ErrorCodeSuccess != m_flush_error) { + return; + } + bsoncxx::builder::basic::document document; document.append( bsoncxx::builder::basic::kvp(constants::results_cache::search::cArchiveId, m_archive_id) @@ -46,19 +73,16 @@ auto ResultsCacheSink::write(AggregationResult const& result) -> void { ); } m_results.push_back(document.extract()); -} -auto ResultsCacheSink::finish() -> ErrorCode { - if (m_results.empty()) { - return ErrorCode::ErrorCodeSuccess; + if (m_results.size() >= m_batch_size) { + m_flush_error = flush_buffer(); } +} - try { - m_collection.insert_many(m_results); - m_results.clear(); - } catch (mongocxx::exception const& e) { - return ErrorCode::ErrorCodeFailureDbBulkWrite; +auto ResultsCacheSink::finish() -> ErrorCode { + if (ErrorCode::ErrorCodeSuccess != m_flush_error) { + return m_flush_error; } - return ErrorCode::ErrorCodeSuccess; + return flush_buffer(); } } // namespace clp_s diff --git a/components/core/src/clp_s/AggregationSink.hpp b/components/core/src/clp_s/AggregationSink.hpp index 5c1e071b45..6c6464b1f8 100644 --- a/components/core/src/clp_s/AggregationSink.hpp +++ b/components/core/src/clp_s/AggregationSink.hpp @@ -1,6 +1,7 @@ #ifndef CLP_S_AGGREGATIONSINK_HPP #define CLP_S_AGGREGATIONSINK_HPP +#include #include #include #include @@ -73,25 +74,40 @@ class ResultsCacheSink : public AggregationSink { ResultsCacheSink( std::string_view uri, std::string_view collection, + uint64_t batch_size, std::string_view archive_id ); // Methods implementing AggregationSink + /** + * Buffers a result document, flushing the buffer to the database once it reaches the batch size. + * @param result The result document to write. + */ auto write(AggregationResult const& result) -> void override; /** - * Flushes the buffered result documents. + * Flushes any remaining buffered result documents. * @return ErrorCodeSuccess on success - * @return ErrorCodeFailureDbBulkWrite on database error + * @return ErrorCodeFailureDbBulkWrite if any flush (including an earlier batched one) failed */ [[nodiscard]] auto finish() -> ErrorCode override; private: + // Methods + /** + * Inserts the buffered result documents into the collection and clears the buffer. + * @return ErrorCodeSuccess on success, leaving the buffer empty + * @return ErrorCodeFailureDbBulkWrite on database error, leaving the buffer intact + */ + [[nodiscard]] auto flush_buffer() -> ErrorCode; + // Data members mongocxx::client m_client; mongocxx::collection m_collection; + uint64_t m_batch_size; std::string m_archive_id; std::vector m_results; + ErrorCode m_flush_error{ErrorCode::ErrorCodeSuccess}; }; } // namespace clp_s diff --git a/components/core/src/clp_s/clp-s.cpp b/components/core/src/clp_s/clp-s.cpp index 3ecde01487..1d4bb4945e 100644 --- a/components/core/src/clp_s/clp-s.cpp +++ b/components/core/src/clp_s/clp-s.cpp @@ -348,6 +348,7 @@ bool search_archive( std::make_unique( options.uri, options.collection, + options.batch_size, archive_reader->get_archive_id() ) ); From e838098fed3b9df1a61f9a26fc98f1dc4f57d4ee Mon Sep 17 00:00:00 2001 From: davemarco <83603688+davemarco@users.noreply.github.com> Date: Mon, 6 Jul 2026 12:47:10 -0400 Subject: [PATCH 46/56] refactor(clp-s): Address review feedback on min/max aggregation. - Rename the per-record aggregator classes and their variant from `*Aggregation`/`Aggregation` to `*Aggregator`/`Aggregator`, keeping "aggregation" for the operation's data/output (value, result, sink, output handler). - Rename ambiguous `_ms` identifiers to `_millisecs`. - Use `` includes, brace-init constexpr members, and tidy doc-comments (possessive wording, hoist implementation notes). Co-Authored-By: Claude Opus 4.8 (1M context) --- components/core/src/clp_s/Aggregation.cpp | 16 +-- components/core/src/clp_s/Aggregation.hpp | 108 ++++++++++-------- .../core/src/clp_s/CommandLineArguments.cpp | 41 +++---- .../core/src/clp_s/CommandLineArguments.hpp | 16 +-- components/core/src/clp_s/IntFloatCompare.hpp | 17 ++- .../core/src/clp_s/OutputHandlerImpl.hpp | 35 +++--- components/core/src/clp_s/clp-s.cpp | 26 ++--- components/core/src/clp_s/kv_ir_search.cpp | 2 +- 8 files changed, 137 insertions(+), 124 deletions(-) diff --git a/components/core/src/clp_s/Aggregation.cpp b/components/core/src/clp_s/Aggregation.cpp index e8af540315..160e2219c3 100644 --- a/components/core/src/clp_s/Aggregation.cpp +++ b/components/core/src/clp_s/Aggregation.cpp @@ -18,7 +18,7 @@ using std::string; using std::string_view; namespace clp_s { -auto CountAggregation::get_results() const -> std::vector { +auto CountAggregator::get_results() const -> std::vector { if (0 == m_count) { return {}; } @@ -27,7 +27,7 @@ auto CountAggregation::get_results() const -> std::vector { return {std::move(result)}; } -auto CountByTimeAggregation::get_results() const -> std::vector { +auto CountByTimeAggregator::get_results() const -> std::vector { std::vector results; results.reserve(m_bucket_counts.size()); for (auto const& [bucket_timestamp, count] : m_bucket_counts) { @@ -39,7 +39,9 @@ auto CountByTimeAggregation::get_results() const -> std::vector bool { +auto MinMaxAggregator::beats_extreme(Extreme candidate) const -> bool { auto const& current{m_extreme.value()}; if (m_find_max) { return std::visit( @@ -71,7 +71,7 @@ auto MinMaxAggregation::beats_extreme(Extreme candidate) const -> bool { return std::visit([](auto cand, auto cur) { return is_less(cand, cur); }, candidate, current); } -auto MinMaxAggregation::add_record(string_view message, epochtime_t) -> void { +auto MinMaxAggregator::add_record(string_view message, epochtime_t) -> void { nlohmann::json doc; try { doc = nlohmann::json::parse(message); @@ -102,7 +102,7 @@ auto MinMaxAggregation::add_record(string_view message, epochtime_t) -> void { } } -auto MinMaxAggregation::get_results() const -> std::vector { +auto MinMaxAggregator::get_results() const -> std::vector { if (false == m_extreme.has_value()) { return {}; } diff --git a/components/core/src/clp_s/Aggregation.hpp b/components/core/src/clp_s/Aggregation.hpp index 04738b89eb..0e94b34904 100644 --- a/components/core/src/clp_s/Aggregation.hpp +++ b/components/core/src/clp_s/Aggregation.hpp @@ -16,12 +16,12 @@ namespace clp_s { /** - * A single typed value in an aggregation result document. + * A single typed value in an aggregation's result document. */ using AggregationValue = std::variant; /** - * One aggregation result document: an ordered list of typed key-value pairs. + * One aggregation's result document: an ordered list of typed key-value pairs. */ using AggregationResult = std::vector>; @@ -30,43 +30,47 @@ using AggregationResult = std::vector>; * @tparam AggregatorType The type of the aggregator. */ template -concept AggregatorReq - = requires(AggregatorType aggregator, std::string_view message, epochtime_t timestamp_ms) { - /** - * Adds a record to the aggregate. - * @param message The message in the log event. - * @param timestamp_ms The timestamp of the log event. - */ - { aggregator.add_record(message, timestamp_ms) } -> std::same_as; - - /** - * Gets the aggregate's results. - * @return The result documents produced by the aggregation. - */ - { aggregator.get_results() } -> std::same_as>; - - /** - * Whether the caller must supply per-record metadata. - */ - { AggregatorType::cNeedsMetadata } -> std::convertible_to; - - /** - * Whether the caller must supply the marshalled record. - */ - { AggregatorType::cNeedsMarshalledRecord } -> std::convertible_to; - }; +concept AggregatorReq = requires( + AggregatorType aggregator, + std::string_view message, + epochtime_t timestamp_millisecs +) { + /** + * Adds a record to the aggregate. + * @param message The message in the log event. + * @param timestamp_millisecs The timestamp of the log event. + */ + { aggregator.add_record(message, timestamp_millisecs) } -> std::same_as; + + /** + * Gets the aggregate's results. + * @return The result documents produced by the aggregation. + */ + { aggregator.get_results() } -> std::same_as>; + + /** + * Whether the caller must supply per-record metadata. + */ + { AggregatorType::cNeedsMetadata } -> std::convertible_to; + + /** + * Whether the caller must supply the marshalled record. + */ + { AggregatorType::cNeedsMarshalledRecord } -> std::convertible_to; +}; /** * Counts the number of matched records. */ -class CountAggregation { +class CountAggregator { public: - static constexpr bool cNeedsMetadata = false; - static constexpr bool cNeedsMarshalledRecord = false; + static constexpr bool cNeedsMetadata{false}; + static constexpr bool cNeedsMarshalledRecord{false}; - auto - add_record([[maybe_unused]] std::string_view message, [[maybe_unused]] epochtime_t timestamp_ms) - -> void { + auto add_record( + [[maybe_unused]] std::string_view message, + [[maybe_unused]] epochtime_t timestamp_millisecs + ) -> void { m_count += 1; } @@ -79,42 +83,48 @@ class CountAggregation { /** * Counts the number of matched records in each time bucket of a fixed size. */ -class CountByTimeAggregation { +class CountByTimeAggregator { public: - static constexpr bool cNeedsMetadata = true; - static constexpr bool cNeedsMarshalledRecord = false; + static constexpr bool cNeedsMetadata{true}; + static constexpr bool cNeedsMarshalledRecord{false}; - explicit CountByTimeAggregation(int64_t bucket_size_ms) : m_bucket_size_ms{bucket_size_ms} { - if (bucket_size_ms <= 0) { - throw std::invalid_argument("CountByTimeAggregation bucket size must be positive."); + explicit CountByTimeAggregator(int64_t bucket_size_millisecs) + : m_bucket_size_millisecs{bucket_size_millisecs} { + if (bucket_size_millisecs <= 0) { + throw std::invalid_argument("CountByTimeAggregator bucket size must be positive."); } } - [[nodiscard]] auto get_bucket_size_ms() const -> int64_t { return m_bucket_size_ms; } + [[nodiscard]] auto get_bucket_size_millisecs() const -> int64_t { + return m_bucket_size_millisecs; + } - auto add_record([[maybe_unused]] std::string_view message, epochtime_t timestamp_ms) -> void { - int64_t const bucket = (timestamp_ms / m_bucket_size_ms) * m_bucket_size_ms; + auto add_record([[maybe_unused]] std::string_view message, epochtime_t timestamp_millisecs) + -> void { + int64_t const bucket + = (timestamp_millisecs / m_bucket_size_millisecs) * m_bucket_size_millisecs; m_bucket_counts[bucket] += 1; } [[nodiscard]] auto get_results() const -> std::vector; private: - int64_t m_bucket_size_ms; + int64_t m_bucket_size_millisecs; std::map m_bucket_counts; }; /** * Tracks the minimum or maximum value of a target field across matched records. */ -class MinMaxAggregation { +class MinMaxAggregator { public: - static constexpr bool cNeedsMetadata = false; - static constexpr bool cNeedsMarshalledRecord = true; + static constexpr bool cNeedsMetadata{false}; + static constexpr bool cNeedsMarshalledRecord{true}; - MinMaxAggregation(bool find_max, std::string_view field); + MinMaxAggregator(bool find_max, std::string_view field); - auto add_record(std::string_view message, [[maybe_unused]] epochtime_t timestamp_ms) -> void; + auto add_record(std::string_view message, [[maybe_unused]] epochtime_t timestamp_millisecs) + -> void; [[nodiscard]] auto get_results() const -> std::vector; @@ -136,7 +146,7 @@ class MinMaxAggregation { /** * One of the supported aggregations that a search can apply to its matched records. */ -using Aggregation = std::variant; +using Aggregator = std::variant; } // namespace clp_s #endif // CLP_S_AGGREGATION_HPP diff --git a/components/core/src/clp_s/CommandLineArguments.cpp b/components/core/src/clp_s/CommandLineArguments.cpp index ca22f95ffe..97dd4e997e 100644 --- a/components/core/src/clp_s/CommandLineArguments.cpp +++ b/components/core/src/clp_s/CommandLineArguments.cpp @@ -11,10 +11,11 @@ #include #include +#include + #include "../clp/type_utils.hpp" #include "../reducer/types.hpp" #include "FileReader.hpp" -#include "search/ast/SearchUtils.hpp" namespace po = boost::program_options; @@ -774,7 +775,7 @@ CommandLineArguments::parse_arguments(int argc, char const** argv) { // clang-format on search_options.add(match_options); - int64_t count_by_time_bucket_size_ms{}; + int64_t count_by_time_bucket_size_millisecs{}; std::string aggregation_field; po::options_description aggregation_options("Aggregation Controls"); // clang-format off @@ -783,7 +784,7 @@ CommandLineArguments::parse_arguments(int argc, char const** argv) { "Count the number of results" )( "count-by-time", - po::value(&count_by_time_bucket_size_ms)->value_name("SIZE"), + po::value(&count_by_time_bucket_size_millisecs)->value_name("SIZE"), "Count the number of results in each time span of the given size (ms)" )( "min", @@ -1038,9 +1039,9 @@ CommandLineArguments::parse_arguments(int argc, char const** argv) { throw std::invalid_argument("clp-s only supports one output handler at a time"); } - m_aggregation = parse_aggregation_options( + m_aggregator = parse_aggregation_options( parsed_command_line_options, - count_by_time_bucket_size_ms, + count_by_time_bucket_size_millisecs, aggregation_field ); @@ -1135,17 +1136,17 @@ void CommandLineArguments::parse_network_dest_output_handler_options( auto CommandLineArguments::parse_aggregation_options( po::variables_map const& parsed_options, - int64_t count_by_time_bucket_size_ms, + int64_t count_by_time_bucket_size_millisecs, std::string_view aggregation_field -) -> std::optional { - std::optional aggregation; - auto const set_aggregation = [&](Aggregation value) { - if (aggregation.has_value()) { +) -> std::optional { + std::optional aggregator; + auto const set_aggregator = [&](Aggregator value) { + if (aggregator.has_value()) { throw std::invalid_argument( "The --count, --count-by-time, --min, and --max options are mutually exclusive." ); } - aggregation = std::move(value); + aggregator = std::move(value); }; auto const validate_aggregation_field = [&]() { if (aggregation_field.empty()) { @@ -1157,28 +1158,28 @@ auto CommandLineArguments::parse_aggregation_options( }; if (parsed_options.count("count")) { - set_aggregation(CountAggregation{}); + set_aggregator(CountAggregator{}); } if (parsed_options.count("count-by-time")) { - if (count_by_time_bucket_size_ms <= 0) { + if (count_by_time_bucket_size_millisecs <= 0) { throw std::invalid_argument("Value for count-by-time must be greater than zero."); } - set_aggregation(CountByTimeAggregation{count_by_time_bucket_size_ms}); + set_aggregator(CountByTimeAggregator{count_by_time_bucket_size_millisecs}); } if (parsed_options.count("min")) { validate_aggregation_field(); - set_aggregation(MinMaxAggregation{false, aggregation_field}); + set_aggregator(MinMaxAggregator{false, aggregation_field}); } if (parsed_options.count("max")) { validate_aggregation_field(); - set_aggregation(MinMaxAggregation{true, aggregation_field}); + set_aggregator(MinMaxAggregator{true, aggregation_field}); } - return aggregation; + return aggregator; } auto CommandLineArguments::reject_aggregation_for_handler(std::string_view handler_name) const -> void { - if (m_aggregation.has_value()) { + if (m_aggregator.has_value()) { throw std::invalid_argument( fmt::format("The {} output handler does not support aggregations.", handler_name) ); @@ -1215,8 +1216,8 @@ void CommandLineArguments::parse_reducer_output_handler_options( throw std::invalid_argument("job-id cannot be negative."); } - if (false == m_aggregation.has_value() - || std::holds_alternative(m_aggregation.value())) + if (false == m_aggregator.has_value() + || std::holds_alternative(m_aggregator.value())) { throw std::invalid_argument( "The reducer output handler currently only supports count and count-by-time" diff --git a/components/core/src/clp_s/CommandLineArguments.hpp b/components/core/src/clp_s/CommandLineArguments.hpp index 3ee00246cc..4f28fda308 100644 --- a/components/core/src/clp_s/CommandLineArguments.hpp +++ b/components/core/src/clp_s/CommandLineArguments.hpp @@ -11,8 +11,9 @@ #include #include +#include + #include "../reducer/types.hpp" -#include "Aggregation.hpp" #include "Defs.hpp" #include "InputConfig.hpp" @@ -119,8 +120,8 @@ class CommandLineArguments { return m_output_handler_options; } - [[nodiscard]] auto get_aggregation() const -> std::optional const& { - return m_aggregation; + [[nodiscard]] auto get_aggregator() const -> std::optional const& { + return m_aggregator; } [[nodiscard]] auto get_retain_float_format() const -> bool { @@ -159,7 +160,8 @@ class CommandLineArguments { /** * Builds the requested aggregation from the parsed options. * @param parsed_options - * @param count_by_time_bucket_size_ms Bucket size for count-by-time. Only used by that option. + * @param count_by_time_bucket_size_millisecs Bucket size for count-by-time. Only used by that + * option. * @param aggregation_field Field for min/max. Only used by those options. * @return The requested aggregation, or std::nullopt if none was requested. * @throws std::invalid_argument if multiple aggregations are specified, the bucket size is @@ -167,9 +169,9 @@ class CommandLineArguments { */ [[nodiscard]] static auto parse_aggregation_options( boost::program_options::variables_map const& parsed_options, - int64_t count_by_time_bucket_size_ms, + int64_t count_by_time_bucket_size_millisecs, std::string_view aggregation_field - ) -> std::optional; + ) -> std::optional; /** * Throws if an aggregation was requested. @@ -263,7 +265,7 @@ class CommandLineArguments { bool m_enable_telemetry{false}; std::vector m_projection_columns; - std::optional m_aggregation; + std::optional m_aggregator; }; } // namespace clp_s diff --git a/components/core/src/clp_s/IntFloatCompare.hpp b/components/core/src/clp_s/IntFloatCompare.hpp index a6082b455e..484d3a1a4b 100644 --- a/components/core/src/clp_s/IntFloatCompare.hpp +++ b/components/core/src/clp_s/IntFloatCompare.hpp @@ -36,38 +36,37 @@ constexpr double cInt64Min{-9223372036854775808.0}; } /** - * Compares an integer against a double exactly. After ruling out doubles that fall outside the - * `int64_t` range, `rhs` is split into its integer and fractional parts: the integer parts are - * compared directly, and the fractional part of `rhs` breaks ties. + * Compares an integer and a double exactly, without a cast that would lose precision. If `rhs` is + * greater than every `int64_t`, `lhs < rhs` is true; if `rhs` is less than every `int64_t`, it is + * false. Otherwise `rhs` is truncated to a whole number and compared to `lhs`; on a tie, `lhs` is + * smaller only when `rhs` has a positive fractional part. * @param lhs * @param rhs * @return Whether `lhs < rhs`. */ [[nodiscard]] inline auto is_less(int64_t lhs, double rhs) -> bool { - // NaN is unordered, so it is never less than anything. if (std::isnan(rhs)) { return false; } - // `rhs` is larger than every `int64_t`, so `lhs` must be less than it. if (rhs >= cInt64UpperBound) { return true; } - // `rhs` is smaller than every `int64_t`, so `lhs` cannot be less than it. if (rhs < cInt64Min) { return false; } - // `rhs` is in `int64_t` range, so its truncated value fits an `int64_t` with no rounding. auto const truncated{std::trunc(rhs)}; auto const rhs_int{static_cast(truncated)}; if (lhs != rhs_int) { return lhs < rhs_int; } - // Equal integer parts: `lhs < rhs` only if `rhs` carries a fractional remainder. return rhs > truncated; } /** - * Compares a double against an integer exactly. + * Compares a double and an integer exactly, without a cast that would lose precision. If `lhs` is + * greater than every `int64_t`, `lhs < rhs` is false; if `lhs` is less than every `int64_t`, it is + * true. Otherwise `lhs` is truncated to a whole number and compared to `rhs`; on a tie, `lhs` is + * smaller only when it has a negative fractional part. * @param lhs * @param rhs * @return Whether `lhs < rhs`. diff --git a/components/core/src/clp_s/OutputHandlerImpl.hpp b/components/core/src/clp_s/OutputHandlerImpl.hpp index 41fade4c92..1290e5fff2 100644 --- a/components/core/src/clp_s/OutputHandlerImpl.hpp +++ b/components/core/src/clp_s/OutputHandlerImpl.hpp @@ -252,20 +252,23 @@ class CountReducerOutputHandler : public search::OutputHandler { class CountByTimeReducerOutputHandler : public search::OutputHandler { public: // Constructors - CountByTimeReducerOutputHandler(int reducer_socket_fd, int64_t count_by_time_bucket_size_ms) + CountByTimeReducerOutputHandler( + int reducer_socket_fd, + int64_t count_by_time_bucket_size_millisecs + ) : search::OutputHandler{true, false}, m_reducer_socket_fd{reducer_socket_fd}, - m_count_by_time_bucket_size_ms{count_by_time_bucket_size_ms} {} + m_count_by_time_bucket_size_millisecs{count_by_time_bucket_size_millisecs} {} // Methods implementing OutputHandler auto write( std::string_view message, - epochtime_t timestamp_ms, + epochtime_t timestamp_millisecs, std::string_view archive_id, int64_t log_event_idx ) -> void override { - int64_t bucket - = (timestamp_ms / m_count_by_time_bucket_size_ms) * m_count_by_time_bucket_size_ms; + int64_t bucket = (timestamp_millisecs / m_count_by_time_bucket_size_millisecs) + * m_count_by_time_bucket_size_millisecs; m_bucket_counts[bucket] += 1; } @@ -283,7 +286,7 @@ class CountByTimeReducerOutputHandler : public search::OutputHandler { // Data members int m_reducer_socket_fd; std::map m_bucket_counts; - int64_t m_count_by_time_bucket_size_ms; + int64_t m_count_by_time_bucket_size_millisecs; }; /** @@ -294,22 +297,22 @@ template class AggregationOutputHandler : public search::OutputHandler { public: // Constructors - AggregationOutputHandler(AggT aggregation, std::unique_ptr sink) + AggregationOutputHandler(AggT aggregator, std::unique_ptr sink) : search::OutputHandler{AggT::cNeedsMetadata, AggT::cNeedsMarshalledRecord}, - m_aggregation{std::move(aggregation)}, + m_aggregator{std::move(aggregator)}, m_sink{std::move(sink)} {} // Methods implementing OutputHandler auto write( std::string_view message, - epochtime_t timestamp_ms, + epochtime_t timestamp_millisecs, std::string_view archive_id, int64_t log_event_idx ) -> void override { - m_aggregation.add_record(message, timestamp_ms); + m_aggregator.add_record(message, timestamp_millisecs); } - auto write(std::string_view message) -> void override { m_aggregation.add_record(message, 0); } + auto write(std::string_view message) -> void override { m_aggregator.add_record(message, 0); } // Methods overriding OutputHandler /** @@ -318,7 +321,7 @@ class AggregationOutputHandler : public search::OutputHandler { * @return The sink's error code on failure */ auto finish() -> ErrorCode override { - for (auto const& result : m_aggregation.get_results()) { + for (auto const& result : m_aggregator.get_results()) { m_sink->write(result); } return m_sink->finish(); @@ -326,18 +329,18 @@ class AggregationOutputHandler : public search::OutputHandler { private: // Data members - AggT m_aggregation; + AggT m_aggregator; std::unique_ptr m_sink; }; /** * Creates an output handler that runs the given aggregation. - * @param aggregation The aggregation to run. + * @param aggregator The aggregation to run. * @param sink The destination for the aggregation's results. * @return The constructed output handler. */ [[nodiscard]] inline auto -make_aggregation_output_handler(Aggregation aggregation, std::unique_ptr sink) +make_aggregation_output_handler(Aggregator aggregator, std::unique_ptr sink) -> std::unique_ptr { return std::visit( [&](auto&& agg) -> std::unique_ptr { @@ -347,7 +350,7 @@ make_aggregation_output_handler(Aggregation aggregation, std::unique_ptr void { - auto const& aggregation{ - command_line_arguments.get_aggregation().value() - }; - if (std::holds_alternative(aggregation)) { + auto const& aggregator{command_line_arguments.get_aggregator().value()}; + if (std::holds_alternative(aggregator)) { output_handler = std::make_unique( reducer_socket_fd ); - } else if (std::holds_alternative( - aggregation + } else if (std::holds_alternative( + aggregator )) { output_handler = std::make_unique( reducer_socket_fd, - std::get(aggregation) - .get_bucket_size_ms() + std::get(aggregator) + .get_bucket_size_millisecs() ); } else { throw std::invalid_argument( @@ -333,8 +331,8 @@ bool search_archive( }, [&](CommandLineArguments::ResultsCacheOutputHandlerOptions const& options) -> void { - auto const& aggregation{command_line_arguments.get_aggregation()}; - if (false == aggregation.has_value()) { + auto const& aggregator{command_line_arguments.get_aggregator()}; + if (false == aggregator.has_value()) { output_handler = std::make_unique( options.uri, options.collection, @@ -344,7 +342,7 @@ bool search_archive( ); } else { output_handler = clp_s::make_aggregation_output_handler( - aggregation.value(), + aggregator.value(), std::make_unique( options.uri, options.collection, @@ -354,12 +352,12 @@ bool search_archive( } }, [&](CommandLineArguments::StdoutOutputHandlerOptions const&) -> void { - auto const& aggregation{command_line_arguments.get_aggregation()}; - if (false == aggregation.has_value()) { + auto const& aggregator{command_line_arguments.get_aggregator()}; + if (false == aggregator.has_value()) { output_handler = std::make_unique(); } else { output_handler = clp_s::make_aggregation_output_handler( - aggregation.value(), + aggregator.value(), std::make_unique( archive_reader->get_archive_id() ) diff --git a/components/core/src/clp_s/kv_ir_search.cpp b/components/core/src/clp_s/kv_ir_search.cpp index 5a1574aa56..f4979cd263 100644 --- a/components/core/src/clp_s/kv_ir_search.cpp +++ b/components/core/src/clp_s/kv_ir_search.cpp @@ -246,7 +246,7 @@ auto search_kv_ir_stream( return KvIrSearchError{KvIrSearchErrorEnum::ProjectionSupportNotImplemented}; } - if (command_line_arguments.get_aggregation().has_value()) { + if (command_line_arguments.get_aggregator().has_value()) { SPDLOG_ERROR("kv-ir search: Aggregation support is not implemented."); return KvIrSearchError{KvIrSearchErrorEnum::AggregationSupportNotImplemented}; } From fdbc260b8c4cab486a4d3a65b93c78048260d988 Mon Sep 17 00:00:00 2001 From: davemarco <83603688+davemarco@users.noreply.github.com> Date: Tue, 7 Jul 2026 13:51:23 -0400 Subject: [PATCH 47/56] Document beats_extreme's candidate parameter. --- components/core/src/clp_s/Aggregation.hpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/components/core/src/clp_s/Aggregation.hpp b/components/core/src/clp_s/Aggregation.hpp index 0e94b34904..02f7d90494 100644 --- a/components/core/src/clp_s/Aggregation.hpp +++ b/components/core/src/clp_s/Aggregation.hpp @@ -132,7 +132,7 @@ class MinMaxAggregator { using Extreme = std::variant; /** - * @param candidate + * @param candidate The value to compare against the current extreme. * @return Whether `candidate` is more extreme (per the min/max mode) than the current extreme. */ [[nodiscard]] auto beats_extreme(Extreme candidate) const -> bool; From 11061c5d44e6f558369f201ccda96c74d57d4158 Mon Sep 17 00:00:00 2001 From: davemarco <83603688+davemarco@users.noreply.github.com> Date: Tue, 7 Jul 2026 14:19:25 -0400 Subject: [PATCH 48/56] Correct Aggregator variant's doc comment to say "aggregators". --- components/core/src/clp_s/Aggregation.hpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/components/core/src/clp_s/Aggregation.hpp b/components/core/src/clp_s/Aggregation.hpp index 02f7d90494..5fc0cd2d6b 100644 --- a/components/core/src/clp_s/Aggregation.hpp +++ b/components/core/src/clp_s/Aggregation.hpp @@ -144,7 +144,7 @@ class MinMaxAggregator { }; /** - * One of the supported aggregations that a search can apply to its matched records. + * One of the supported aggregators that a search can apply to its matched records. */ using Aggregator = std::variant; } // namespace clp_s From 940ddcb0b971406e94ae0f0832805b0ba45083bf Mon Sep 17 00:00:00 2001 From: davemarco <83603688+davemarco@users.noreply.github.com> Date: Tue, 7 Jul 2026 14:56:59 -0400 Subject: [PATCH 49/56] Reject min/max aggregation on namespaced fields and add a max usage example. --- components/core/src/clp_s/Aggregation.cpp | 7 ++++--- components/core/src/clp_s/CommandLineArguments.cpp | 8 ++++++++ 2 files changed, 12 insertions(+), 3 deletions(-) diff --git a/components/core/src/clp_s/Aggregation.cpp b/components/core/src/clp_s/Aggregation.cpp index 160e2219c3..eeae73d841 100644 --- a/components/core/src/clp_s/Aggregation.cpp +++ b/components/core/src/clp_s/Aggregation.cpp @@ -39,8 +39,6 @@ auto CountByTimeAggregator::get_results() const -> std::vector Date: Thu, 9 Jul 2026 17:14:11 -0400 Subject: [PATCH 50/56] Rename Aggregation/IntFloatCompare files to snake_case and add member category comments. --- components/core/src/clp_s/AggregationSink.hpp | 2 +- components/core/src/clp_s/CMakeLists.txt | 4 ++-- .../core/src/clp_s/CommandLineArguments.hpp | 2 +- .../core/src/clp_s/OutputHandlerImpl.hpp | 2 +- .../{Aggregation.cpp => aggregators.cpp} | 4 ++-- .../{Aggregation.hpp => aggregators.hpp} | 19 ++++++++++++++++--- ...FloatCompare.hpp => int_float_compare.hpp} | 7 ++++--- 7 files changed, 27 insertions(+), 13 deletions(-) rename components/core/src/clp_s/{Aggregation.cpp => aggregators.cpp} (98%) rename components/core/src/clp_s/{Aggregation.hpp => aggregators.hpp} (92%) rename components/core/src/clp_s/{IntFloatCompare.hpp => int_float_compare.hpp} (96%) diff --git a/components/core/src/clp_s/AggregationSink.hpp b/components/core/src/clp_s/AggregationSink.hpp index 5c1e071b45..794c084587 100644 --- a/components/core/src/clp_s/AggregationSink.hpp +++ b/components/core/src/clp_s/AggregationSink.hpp @@ -9,7 +9,7 @@ #include #include -#include +#include #include namespace clp_s { diff --git a/components/core/src/clp_s/CMakeLists.txt b/components/core/src/clp_s/CMakeLists.txt index dfad31d9b0..3c58147773 100644 --- a/components/core/src/clp_s/CMakeLists.txt +++ b/components/core/src/clp_s/CMakeLists.txt @@ -480,10 +480,10 @@ endif() set( CLP_S_EXE_SOURCES - Aggregation.cpp - Aggregation.hpp AggregationSink.cpp AggregationSink.hpp + aggregators.cpp + aggregators.hpp CommandLineArguments.cpp CommandLineArguments.hpp ErrorCode.hpp diff --git a/components/core/src/clp_s/CommandLineArguments.hpp b/components/core/src/clp_s/CommandLineArguments.hpp index 4f28fda308..30d12b998f 100644 --- a/components/core/src/clp_s/CommandLineArguments.hpp +++ b/components/core/src/clp_s/CommandLineArguments.hpp @@ -11,7 +11,7 @@ #include #include -#include +#include #include "../reducer/types.hpp" #include "Defs.hpp" diff --git a/components/core/src/clp_s/OutputHandlerImpl.hpp b/components/core/src/clp_s/OutputHandlerImpl.hpp index 1290e5fff2..cf5ef5440c 100644 --- a/components/core/src/clp_s/OutputHandlerImpl.hpp +++ b/components/core/src/clp_s/OutputHandlerImpl.hpp @@ -19,8 +19,8 @@ #include #include -#include #include +#include #include #include "../reducer/Pipeline.hpp" diff --git a/components/core/src/clp_s/Aggregation.cpp b/components/core/src/clp_s/aggregators.cpp similarity index 98% rename from components/core/src/clp_s/Aggregation.cpp rename to components/core/src/clp_s/aggregators.cpp index eeae73d841..9976ddb9e7 100644 --- a/components/core/src/clp_s/Aggregation.cpp +++ b/components/core/src/clp_s/aggregators.cpp @@ -1,4 +1,4 @@ -#include "Aggregation.hpp" +#include "aggregators.hpp" #include #include @@ -11,7 +11,7 @@ #include #include -#include +#include #include using std::string; diff --git a/components/core/src/clp_s/Aggregation.hpp b/components/core/src/clp_s/aggregators.hpp similarity index 92% rename from components/core/src/clp_s/Aggregation.hpp rename to components/core/src/clp_s/aggregators.hpp index 5fc0cd2d6b..f98858f102 100644 --- a/components/core/src/clp_s/Aggregation.hpp +++ b/components/core/src/clp_s/aggregators.hpp @@ -1,5 +1,5 @@ -#ifndef CLP_S_AGGREGATION_HPP -#define CLP_S_AGGREGATION_HPP +#ifndef CLP_S_AGGREGATORS_HPP +#define CLP_S_AGGREGATORS_HPP #include #include @@ -64,9 +64,11 @@ concept AggregatorReq = requires( */ class CountAggregator { public: + // Static constants static constexpr bool cNeedsMetadata{false}; static constexpr bool cNeedsMarshalledRecord{false}; + // Methods auto add_record( [[maybe_unused]] std::string_view message, [[maybe_unused]] epochtime_t timestamp_millisecs @@ -77,6 +79,7 @@ class CountAggregator { [[nodiscard]] auto get_results() const -> std::vector; private: + // Data members int64_t m_count{}; }; @@ -85,9 +88,11 @@ class CountAggregator { */ class CountByTimeAggregator { public: + // Static constants static constexpr bool cNeedsMetadata{true}; static constexpr bool cNeedsMarshalledRecord{false}; + // Constructors explicit CountByTimeAggregator(int64_t bucket_size_millisecs) : m_bucket_size_millisecs{bucket_size_millisecs} { if (bucket_size_millisecs <= 0) { @@ -95,6 +100,7 @@ class CountByTimeAggregator { } } + // Methods [[nodiscard]] auto get_bucket_size_millisecs() const -> int64_t { return m_bucket_size_millisecs; } @@ -109,6 +115,7 @@ class CountByTimeAggregator { [[nodiscard]] auto get_results() const -> std::vector; private: + // Data members int64_t m_bucket_size_millisecs; std::map m_bucket_counts; }; @@ -118,25 +125,31 @@ class CountByTimeAggregator { */ class MinMaxAggregator { public: + // Static constants static constexpr bool cNeedsMetadata{false}; static constexpr bool cNeedsMarshalledRecord{true}; + // Constructors MinMaxAggregator(bool find_max, std::string_view field); + // Methods auto add_record(std::string_view message, [[maybe_unused]] epochtime_t timestamp_millisecs) -> void; [[nodiscard]] auto get_results() const -> std::vector; private: + // Types using Extreme = std::variant; + // Methods /** * @param candidate The value to compare against the current extreme. * @return Whether `candidate` is more extreme (per the min/max mode) than the current extreme. */ [[nodiscard]] auto beats_extreme(Extreme candidate) const -> bool; + // Data members bool m_find_max; std::string m_field; std::vector m_field_path; @@ -149,4 +162,4 @@ class MinMaxAggregator { using Aggregator = std::variant; } // namespace clp_s -#endif // CLP_S_AGGREGATION_HPP +#endif // CLP_S_AGGREGATORS_HPP diff --git a/components/core/src/clp_s/IntFloatCompare.hpp b/components/core/src/clp_s/int_float_compare.hpp similarity index 96% rename from components/core/src/clp_s/IntFloatCompare.hpp rename to components/core/src/clp_s/int_float_compare.hpp index 484d3a1a4b..5c7acf6847 100644 --- a/components/core/src/clp_s/IntFloatCompare.hpp +++ b/components/core/src/clp_s/int_float_compare.hpp @@ -1,5 +1,5 @@ -#ifndef CLP_S_INTFLOATCOMPARE_HPP -#define CLP_S_INTFLOATCOMPARE_HPP +#ifndef CLP_S_INT_FLOAT_COMPARE_HPP +#define CLP_S_INT_FLOAT_COMPARE_HPP #include #include @@ -23,6 +23,7 @@ namespace clp_s { // `2^63` rather than `INT64_MAX` because `2^63` is exactly representable as a double and // `INT64_MAX` is not. constexpr double cInt64UpperBound{9223372036854775808.0}; + // `-2^63`, which equals `INT64_MIN` and is exactly representable as a double: any double `<` this // is smaller than every `int64_t`. constexpr double cInt64Min{-9223372036854775808.0}; @@ -90,4 +91,4 @@ constexpr double cInt64Min{-9223372036854775808.0}; } } // namespace clp_s -#endif // CLP_S_INTFLOATCOMPARE_HPP +#endif // CLP_S_INT_FLOAT_COMPARE_HPP From 45a219e986ee1f6d60be1c4e2360e75277476242 Mon Sep 17 00:00:00 2001 From: davemarco <83603688+davemarco@users.noreply.github.com> Date: Thu, 9 Jul 2026 17:21:57 -0400 Subject: [PATCH 51/56] Remove a redundant comment and an unused exception binding. --- components/core/src/clp_s/Aggregation.cpp | 1 - components/core/src/clp_s/AggregationSink.cpp | 2 +- 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/components/core/src/clp_s/Aggregation.cpp b/components/core/src/clp_s/Aggregation.cpp index 8b99a4ecde..dfd4112867 100644 --- a/components/core/src/clp_s/Aggregation.cpp +++ b/components/core/src/clp_s/Aggregation.cpp @@ -164,7 +164,6 @@ auto UniqueAggregation::add_record(string_view message, epochtime_t) -> void { } else if (node->is_boolean()) { m_values.emplace(node->get()); } - // Values that aren't scalars (e.g. objects, arrays, or null) are ignored. } auto UniqueAggregation::get_results() const -> std::vector { diff --git a/components/core/src/clp_s/AggregationSink.cpp b/components/core/src/clp_s/AggregationSink.cpp index 3dc94852c7..1e6a0dea85 100644 --- a/components/core/src/clp_s/AggregationSink.cpp +++ b/components/core/src/clp_s/AggregationSink.cpp @@ -46,7 +46,7 @@ auto ResultsCacheSink::flush_buffer() -> ErrorCode { try { m_collection.insert_many(m_results); - } catch (mongocxx::exception const& e) { + } catch (mongocxx::exception const&) { return ErrorCode::ErrorCodeFailureDbBulkWrite; } m_results.clear(); From 3e351b1a855a237cc45b98abfcd1c7ce6ad7543c Mon Sep 17 00:00:00 2001 From: davemarco <83603688+davemarco@users.noreply.github.com> Date: Sun, 12 Jul 2026 22:50:21 -0400 Subject: [PATCH 52/56] Add --count-by group-by count aggregation to clp-s --- .../core/src/clp_s/CommandLineArguments.cpp | 18 ++++-- components/core/src/clp_s/aggregators.cpp | 64 ++++++++++++++++--- components/core/src/clp_s/aggregators.hpp | 33 +++++++++- 3 files changed, 100 insertions(+), 15 deletions(-) diff --git a/components/core/src/clp_s/CommandLineArguments.cpp b/components/core/src/clp_s/CommandLineArguments.cpp index 89e8aa737a..0a2a451c38 100644 --- a/components/core/src/clp_s/CommandLineArguments.cpp +++ b/components/core/src/clp_s/CommandLineArguments.cpp @@ -798,6 +798,10 @@ CommandLineArguments::parse_arguments(int argc, char const** argv) { "unique", po::value(&aggregation_field)->value_name("FIELD"), "Find the distinct values of the given field" + )( + "count-by", + po::value(&aggregation_field)->value_name("FIELD"), + "Count the number of results grouped by the value of the given field" ); // clang-format on search_options.add(aggregation_options); @@ -1155,19 +1159,21 @@ auto CommandLineArguments::parse_aggregation_options( auto const set_aggregator = [&](Aggregator value) { if (aggregator.has_value()) { throw std::invalid_argument( - "The --count, --count-by-time, --min, --max, and --unique options are mutually" - " exclusive." + "The --count, --count-by-time, --min, --max, --unique, and --count-by options" + " are mutually exclusive." ); } aggregator = std::move(value); }; auto const validate_aggregation_field = [&]() { if (aggregation_field.empty()) { - throw std::invalid_argument("The --min, --max, and --unique options require a field."); + throw std::invalid_argument( + "The --min, --max, --unique, and --count-by options require a field." + ); } if (search::ast::has_unescaped_wildcards(aggregation_field)) { throw std::invalid_argument( - "The --min, --max, and --unique field must not contain wildcards." + "The --min, --max, --unique, and --count-by field must not contain wildcards." ); } }; @@ -1193,6 +1199,10 @@ auto CommandLineArguments::parse_aggregation_options( validate_aggregation_field(); set_aggregator(UniqueAggregator{aggregation_field}); } + if (parsed_options.count("count-by")) { + validate_aggregation_field(); + set_aggregator(GroupByCountAggregator{aggregation_field}); + } return aggregator; } diff --git a/components/core/src/clp_s/aggregators.cpp b/components/core/src/clp_s/aggregators.cpp index 5191b1dc9a..88c63a5005 100644 --- a/components/core/src/clp_s/aggregators.cpp +++ b/components/core/src/clp_s/aggregators.cpp @@ -1,6 +1,7 @@ #include "aggregators.hpp" #include +#include #include #include #include @@ -72,6 +73,28 @@ find_field_node(string_view message, std::vector const& field_path, nloh } return node; } + +/** + * Extracts a scalar node's value as an `AggregationValue`. + * @param node + * @return The node's value if it is an integer, float, string, or boolean. + * @return std::nullopt otherwise. + */ +auto node_to_aggregation_value(nlohmann::json const& node) -> std::optional { + if (node.is_number_integer()) { + return node.get(); + } + if (node.is_number_float()) { + return node.get(); + } + if (node.is_string()) { + return node.get(); + } + if (node.is_boolean()) { + return node.get(); + } + return std::nullopt; +} } // namespace auto CountAggregator::get_results() const -> std::vector { @@ -95,6 +118,35 @@ auto CountByTimeAggregator::get_results() const -> std::vector void { + nlohmann::json doc; + auto const* const node{find_field_node(message, m_field_path, doc)}; + if (nullptr == node) { + return; + } + auto value{node_to_aggregation_value(*node)}; + if (value.has_value()) { + m_counts[std::move(value.value())] += 1; + } +} + +auto GroupByCountAggregator::get_results() const -> std::vector { + std::vector results; + results.reserve(m_counts.size()); + for (auto const& [value, count] : m_counts) { + AggregationResult result; + result.emplace_back(constants::results_cache::search::cField, m_field); + result.emplace_back(constants::results_cache::search::cValue, value); + result.emplace_back(constants::results_cache::search::cCount, count); + results.push_back(std::move(result)); + } + return results; +} + MinMaxAggregator::MinMaxAggregator(bool find_max, string_view field) : m_find_max{find_max}, m_field{field} { @@ -154,15 +206,9 @@ auto UniqueAggregator::add_record(string_view message, epochtime_t) -> void { if (nullptr == node) { return; } - - if (node->is_number_integer()) { - m_values.emplace(node->get()); - } else if (node->is_number_float()) { - m_values.emplace(node->get()); - } else if (node->is_string()) { - m_values.emplace(node->get()); - } else if (node->is_boolean()) { - m_values.emplace(node->get()); + auto value{node_to_aggregation_value(*node)}; + if (value.has_value()) { + m_values.emplace(std::move(value.value())); } } diff --git a/components/core/src/clp_s/aggregators.hpp b/components/core/src/clp_s/aggregators.hpp index e63992066b..ff7485a286 100644 --- a/components/core/src/clp_s/aggregators.hpp +++ b/components/core/src/clp_s/aggregators.hpp @@ -121,6 +121,31 @@ class CountByTimeAggregator { std::map m_bucket_counts; }; +/** + * Counts the number of matched records grouped by the value of a target field. + */ +class GroupByCountAggregator { +public: + // Static constants + static constexpr bool cNeedsMetadata{false}; + static constexpr bool cNeedsMarshalledRecord{true}; + + // Constructors + explicit GroupByCountAggregator(std::string_view field); + + // Methods + auto add_record(std::string_view message, [[maybe_unused]] epochtime_t timestamp_millisecs) + -> void; + + [[nodiscard]] auto get_results() const -> std::vector; + +private: + // Data members + std::string m_field; + std::vector m_field_path; + std::map m_counts; +}; + /** * Tracks the minimum or maximum value of a target field across matched records. */ @@ -185,8 +210,12 @@ class UniqueAggregator { /** * One of the supported aggregators that a search can apply to its matched records. */ -using Aggregator - = std::variant; +using Aggregator = std::variant< + CountAggregator, + CountByTimeAggregator, + GroupByCountAggregator, + MinMaxAggregator, + UniqueAggregator>; } // namespace clp_s #endif // CLP_S_AGGREGATORS_HPP From acad47e97bdf7d12d28da3423380343be9167ea9 Mon Sep 17 00:00:00 2001 From: davemarco <83603688+davemarco@users.noreply.github.com> Date: Tue, 14 Jul 2026 21:40:06 -0400 Subject: [PATCH 53/56] Factor out node_to_aggregation_value helper and clarify flush-failure handling docs --- components/core/src/clp_s/AggregationSink.cpp | 2 -- components/core/src/clp_s/AggregationSink.hpp | 3 +- components/core/src/clp_s/aggregators.cpp | 35 ++++++++++++++----- 3 files changed, 28 insertions(+), 12 deletions(-) diff --git a/components/core/src/clp_s/AggregationSink.cpp b/components/core/src/clp_s/AggregationSink.cpp index 1e6a0dea85..435a80c48a 100644 --- a/components/core/src/clp_s/AggregationSink.cpp +++ b/components/core/src/clp_s/AggregationSink.cpp @@ -54,8 +54,6 @@ auto ResultsCacheSink::flush_buffer() -> ErrorCode { } auto ResultsCacheSink::write(AggregationResult const& result) -> void { - // Once a flush has failed, stop buffering so memory stays bounded; the error surfaces in - // `finish()`. if (ErrorCode::ErrorCodeSuccess != m_flush_error) { return; } diff --git a/components/core/src/clp_s/AggregationSink.hpp b/components/core/src/clp_s/AggregationSink.hpp index 7d3a64b907..d76b7fbd1b 100644 --- a/components/core/src/clp_s/AggregationSink.hpp +++ b/components/core/src/clp_s/AggregationSink.hpp @@ -81,7 +81,8 @@ class ResultsCacheSink : public AggregationSink { // Methods implementing AggregationSink /** * Buffers a result document, flushing the buffer to the database once it reaches the batch - * size. + * size. If an earlier flush failed, the document is dropped rather than buffered so that memory + * stays bounded; the error is reported by `finish()`. * @param result The result document to write. */ auto write(AggregationResult const& result) -> void override; diff --git a/components/core/src/clp_s/aggregators.cpp b/components/core/src/clp_s/aggregators.cpp index 5191b1dc9a..c17ff4c3a8 100644 --- a/components/core/src/clp_s/aggregators.cpp +++ b/components/core/src/clp_s/aggregators.cpp @@ -1,6 +1,7 @@ #include "aggregators.hpp" #include +#include #include #include #include @@ -72,6 +73,28 @@ find_field_node(string_view message, std::vector const& field_path, nloh } return node; } + +/** + * Extracts a scalar node's value as an `AggregationValue`. + * @param node + * @return The node's value if it is an integer, float, string, or boolean. + * @return std::nullopt otherwise. + */ +auto node_to_aggregation_value(nlohmann::json const& node) -> std::optional { + if (node.is_number_integer()) { + return node.get(); + } + if (node.is_number_float()) { + return node.get(); + } + if (node.is_string()) { + return node.get(); + } + if (node.is_boolean()) { + return node.get(); + } + return std::nullopt; +} } // namespace auto CountAggregator::get_results() const -> std::vector { @@ -154,15 +177,9 @@ auto UniqueAggregator::add_record(string_view message, epochtime_t) -> void { if (nullptr == node) { return; } - - if (node->is_number_integer()) { - m_values.emplace(node->get()); - } else if (node->is_number_float()) { - m_values.emplace(node->get()); - } else if (node->is_string()) { - m_values.emplace(node->get()); - } else if (node->is_boolean()) { - m_values.emplace(node->get()); + auto value{node_to_aggregation_value(*node)}; + if (value.has_value()) { + m_values.emplace(std::move(value.value())); } } From 8da43ed0704068d21b8d8a04fc6822e285e3ce5e Mon Sep 17 00:00:00 2001 From: davemarco <83603688+davemarco@users.noreply.github.com> Date: Tue, 14 Jul 2026 22:07:14 -0400 Subject: [PATCH 54/56] Extract shared JSON value helper and rename find_field_value --- components/core/src/clp_s/aggregators.cpp | 73 +++++++++++------------ 1 file changed, 36 insertions(+), 37 deletions(-) diff --git a/components/core/src/clp_s/aggregators.cpp b/components/core/src/clp_s/aggregators.cpp index c17ff4c3a8..e1674e65b4 100644 --- a/components/core/src/clp_s/aggregators.cpp +++ b/components/core/src/clp_s/aggregators.cpp @@ -21,11 +21,10 @@ using std::string_view; namespace clp_s { namespace { /** - * Tokenizes an aggregation's target field into its path components. + * Tokenizes an aggregation's target field into its key path. * @param field - * @param field_path Returns the tokenized path components. - * @throws std::invalid_argument if `field` cannot be tokenized, or if it is in a non-default - * namespace. + * @param field_path Returns the path's keys. + * @throws std::invalid_argument if `field` is malformed, or if it names a non-default namespace. */ auto tokenize_aggregation_field(string_view field, std::vector& field_path) -> void { string descriptor_namespace; @@ -43,16 +42,15 @@ auto tokenize_aggregation_field(string_view field, std::vector& field_pa } /** - * Parses `message` as JSON and navigates to the node at `field_path`. - * @param message + * Parses `message` as JSON and locates the value at `field_path`. + * @param message A matched record, marshalled to a JSON string. * @param field_path - * @param doc Returns the parsed document, which owns the returned node. - * @return A pointer, valid for the lifetime of `doc`, to the node at `field_path`. - * @return nullptr if `message` is not valid JSON, or if any path component is missing or traverses - * a non-object. + * @param doc Returns the parsed document. + * @return The value at `field_path`. + * @return nullptr if `message` isn't valid JSON, or if `field_path` doesn't resolve to a value. */ auto -find_field_node(string_view message, std::vector const& field_path, nlohmann::json& doc) +find_field_value(string_view message, std::vector const& field_path, nlohmann::json& doc) -> nlohmann::json const* { try { doc = nlohmann::json::parse(message); @@ -60,38 +58,38 @@ find_field_node(string_view message, std::vector const& field_path, nloh return nullptr; } - nlohmann::json const* node{&doc}; + nlohmann::json const* current{&doc}; for (auto const& key : field_path) { - if (false == node->is_object()) { + if (false == current->is_object()) { return nullptr; } - auto const it{node->find(key)}; - if (node->end() == it) { + auto const it{current->find(key)}; + if (current->end() == it) { return nullptr; } - node = &it.value(); + current = &it.value(); } - return node; + return current; } /** - * Extracts a scalar node's value as an `AggregationValue`. - * @param node - * @return The node's value if it is an integer, float, string, or boolean. + * Converts a scalar JSON value to an `AggregationValue`. + * @param value + * @return `value` as an `AggregationValue` if it's an integer, float, string, or boolean. * @return std::nullopt otherwise. */ -auto node_to_aggregation_value(nlohmann::json const& node) -> std::optional { - if (node.is_number_integer()) { - return node.get(); +auto to_aggregation_value(nlohmann::json const& value) -> std::optional { + if (value.is_number_integer()) { + return value.get(); } - if (node.is_number_float()) { - return node.get(); + if (value.is_number_float()) { + return value.get(); } - if (node.is_string()) { - return node.get(); + if (value.is_string()) { + return value.get(); } - if (node.is_boolean()) { - return node.get(); + if (value.is_boolean()) { + return value.get(); } return std::nullopt; } @@ -138,12 +136,13 @@ auto MinMaxAggregator::beats_extreme(Extreme candidate) const -> bool { auto MinMaxAggregator::add_record(string_view message, epochtime_t) -> void { nlohmann::json doc; - auto const* const node{find_field_node(message, m_field_path, doc)}; - if (nullptr == node || false == node->is_number()) { + auto const* const value{find_field_value(message, m_field_path, doc)}; + if (nullptr == value || false == value->is_number()) { return; } Extreme const candidate{ - node->is_number_integer() ? Extreme{node->get()} : Extreme{node->get()} + value->is_number_integer() ? Extreme{value->get()} + : Extreme{value->get()} }; if (false == m_extreme.has_value() || beats_extreme(candidate)) { m_extreme = candidate; @@ -173,13 +172,13 @@ UniqueAggregator::UniqueAggregator(string_view field) : m_field{field} { auto UniqueAggregator::add_record(string_view message, epochtime_t) -> void { nlohmann::json doc; - auto const* const node{find_field_node(message, m_field_path, doc)}; - if (nullptr == node) { + auto const* const value{find_field_value(message, m_field_path, doc)}; + if (nullptr == value) { return; } - auto value{node_to_aggregation_value(*node)}; - if (value.has_value()) { - m_values.emplace(std::move(value.value())); + auto aggregation_value{to_aggregation_value(*value)}; + if (aggregation_value.has_value()) { + m_values.emplace(std::move(aggregation_value.value())); } } From 2d2f5c66e847f229ed028477cee743d1d96b0752 Mon Sep 17 00:00:00 2001 From: davemarco <83603688+davemarco@users.noreply.github.com> Date: Tue, 14 Jul 2026 22:13:18 -0400 Subject: [PATCH 55/56] Add --unique usage example and tidy aggregation doc comments --- components/core/src/clp_s/AggregationSink.hpp | 7 +++---- components/core/src/clp_s/CommandLineArguments.cpp | 8 ++++++++ components/core/src/clp_s/aggregators.hpp | 2 +- 3 files changed, 12 insertions(+), 5 deletions(-) diff --git a/components/core/src/clp_s/AggregationSink.hpp b/components/core/src/clp_s/AggregationSink.hpp index d76b7fbd1b..b9867f1201 100644 --- a/components/core/src/clp_s/AggregationSink.hpp +++ b/components/core/src/clp_s/AggregationSink.hpp @@ -81,8 +81,7 @@ class ResultsCacheSink : public AggregationSink { // Methods implementing AggregationSink /** * Buffers a result document, flushing the buffer to the database once it reaches the batch - * size. If an earlier flush failed, the document is dropped rather than buffered so that memory - * stays bounded; the error is reported by `finish()`. + * size. Documents are dropped after an earlier flush failure; the error surfaces in `finish()`. * @param result The result document to write. */ auto write(AggregationResult const& result) -> void override; @@ -90,7 +89,7 @@ class ResultsCacheSink : public AggregationSink { /** * Flushes any remaining buffered result documents. * @return ErrorCodeSuccess on success - * @return ErrorCodeFailureDbBulkWrite if any flush (including an earlier batched one) failed + * @return ErrorCodeFailureDbBulkWrite if this flush or an earlier batched flush failed */ [[nodiscard]] auto finish() -> ErrorCode override; @@ -98,7 +97,7 @@ class ResultsCacheSink : public AggregationSink { // Methods /** * Inserts the buffered result documents into the collection and clears the buffer. - * @return ErrorCodeSuccess on success, leaving the buffer empty + * @return ErrorCodeSuccess on success * @return ErrorCodeFailureDbBulkWrite on database error, leaving the buffer intact */ [[nodiscard]] auto flush_buffer() -> ErrorCode; diff --git a/components/core/src/clp_s/CommandLineArguments.cpp b/components/core/src/clp_s/CommandLineArguments.cpp index 89e8aa737a..a477fe0e06 100644 --- a/components/core/src/clp_s/CommandLineArguments.cpp +++ b/components/core/src/clp_s/CommandLineArguments.cpp @@ -977,6 +977,14 @@ CommandLineArguments::parse_arguments(int argc, char const** argv) { << std::endl; std::cerr << " " << m_program_name << R"( s archives-dir "level: INFO")" << " --max latency" << std::endl; + std::cerr << std::endl; + + std::cerr << " # Search archives in archives-dir for logs matching a KQL query" + R"( "level: INFO" and output the distinct values of field "latency")" + " to stdout" + << std::endl; + std::cerr << " " << m_program_name << R"( s archives-dir "level: INFO")" + << " --unique latency" << std::endl; po::options_description visible_options; visible_options.add(general_options); diff --git a/components/core/src/clp_s/aggregators.hpp b/components/core/src/clp_s/aggregators.hpp index e63992066b..61b9e78c5d 100644 --- a/components/core/src/clp_s/aggregators.hpp +++ b/components/core/src/clp_s/aggregators.hpp @@ -28,7 +28,7 @@ using AggregationResult = std::vector>; /** * Requirements a type must satisfy to be used as an aggregator. - * @tparam AggregatorType The type of the aggregator. + * @tparam AggregatorType */ template concept AggregatorReq = requires( From 04c6a1567f606a13f60a650fab7b8043a3597c2a Mon Sep 17 00:00:00 2001 From: davemarco <83603688+davemarco@users.noreply.github.com> Date: Tue, 14 Jul 2026 22:23:49 -0400 Subject: [PATCH 56/56] docs(clp-s): trim redundant flush_buffer buffer-state comments --- components/core/src/clp_s/AggregationSink.hpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/components/core/src/clp_s/AggregationSink.hpp b/components/core/src/clp_s/AggregationSink.hpp index b9867f1201..004b5ef4ba 100644 --- a/components/core/src/clp_s/AggregationSink.hpp +++ b/components/core/src/clp_s/AggregationSink.hpp @@ -96,9 +96,9 @@ class ResultsCacheSink : public AggregationSink { private: // Methods /** - * Inserts the buffered result documents into the collection and clears the buffer. + * Inserts the buffered result documents into the collection. * @return ErrorCodeSuccess on success - * @return ErrorCodeFailureDbBulkWrite on database error, leaving the buffer intact + * @return ErrorCodeFailureDbBulkWrite on database error */ [[nodiscard]] auto flush_buffer() -> ErrorCode;