From 89adc4ebd4a94215d847fb516fe56e38b424e658 Mon Sep 17 00:00:00 2001 From: Xuhui Chen <90581933+ShangDanLuXian@users.noreply.github.com> Date: Wed, 15 Jul 2026 17:25:42 +0000 Subject: [PATCH] feat(clp-s::filter): Add interfaces and definitions for the top-level index framework. --- components/core/src/clp_s/CMakeLists.txt | 1 + .../core/src/clp_s/filter/CMakeLists.txt | 5 + .../core/src/clp_s/filter/IndexBuilder.hpp | 60 +++++++ .../filter/IndexBuilderSpecification.hpp | 98 +++++++++++ .../core/src/clp_s/filter/IndexDefs.hpp | 161 ++++++++++++++++++ .../core/src/clp_s/filter/IndexRunner.hpp | 62 +++++++ .../filter/PackedFilterSpecification.hpp | 35 ++++ .../filter/tests/test-clp_s-index_defs.cpp | 58 +++++++ 8 files changed, 480 insertions(+) create mode 100644 components/core/src/clp_s/filter/IndexBuilder.hpp create mode 100644 components/core/src/clp_s/filter/IndexBuilderSpecification.hpp create mode 100644 components/core/src/clp_s/filter/IndexDefs.hpp create mode 100644 components/core/src/clp_s/filter/IndexRunner.hpp create mode 100644 components/core/src/clp_s/filter/PackedFilterSpecification.hpp create mode 100644 components/core/src/clp_s/filter/tests/test-clp_s-index_defs.cpp diff --git a/components/core/src/clp_s/CMakeLists.txt b/components/core/src/clp_s/CMakeLists.txt index 5dc71c956..0f497c734 100644 --- a/components/core/src/clp_s/CMakeLists.txt +++ b/components/core/src/clp_s/CMakeLists.txt @@ -552,6 +552,7 @@ if(CLP_BUILD_TESTING) INTERFACE filter/tests/test-clp_s-bitmap_view.cpp filter/tests/test-clp_s-bloom_filter.cpp + filter/tests/test-clp_s-index_defs.cpp filter/tests/test-clp_s-xxhash.cpp tests/clp_s_test_utils.cpp tests/clp_s_test_utils.hpp diff --git a/components/core/src/clp_s/filter/CMakeLists.txt b/components/core/src/clp_s/filter/CMakeLists.txt index 52e020d20..e06c04703 100644 --- a/components/core/src/clp_s/filter/CMakeLists.txt +++ b/components/core/src/clp_s/filter/CMakeLists.txt @@ -13,6 +13,11 @@ set( FilterReader.hpp HashAlgorithm.cpp HashAlgorithm.hpp + IndexBuilder.hpp + IndexBuilderSpecification.hpp + IndexDefs.hpp + IndexRunner.hpp + PackedFilterSpecification.hpp XxHash.cpp XxHash.hpp ) diff --git a/components/core/src/clp_s/filter/IndexBuilder.hpp b/components/core/src/clp_s/filter/IndexBuilder.hpp new file mode 100644 index 000000000..6b2c154d6 --- /dev/null +++ b/components/core/src/clp_s/filter/IndexBuilder.hpp @@ -0,0 +1,60 @@ +#ifndef CLP_S_FILTER_INDEX_BUILDER_HPP +#define CLP_S_FILTER_INDEX_BUILDER_HPP + +#include +#include +#include + +#include + +namespace clp_s { +class ArchiveReader; +} // namespace clp_s + +namespace clp_s::filter { +/** + * Interface for building an index over the archives of a Packed Filter. The framework feeds an + * implementation one archive at a time so that only a single archive needs to be loaded into memory + * at once; the implementation produces that archive's serialized blob, which the framework collects + * to assemble the Packed Filter. + */ +class IndexBuilder { +public: + // Constructors + IndexBuilder() = default; + + // Delete copy constructor and assignment operator + IndexBuilder(IndexBuilder const&) = delete; + auto operator=(IndexBuilder const&) -> IndexBuilder& = delete; + + // Default move constructor and assignment operator + IndexBuilder(IndexBuilder&&) noexcept = default; + auto operator=(IndexBuilder&&) noexcept -> IndexBuilder& = default; + + // Destructor + virtual ~IndexBuilder() = default; + + // Methods + /** + * Builds the index's data for an archive and stores the resulting per-archive blob. + * + * @param local_archive_id The Packed-Filter-local ID of the archive, in the range + * [0, num_archives). + * @param archive_reader A reader for the archive identified by `local_archive_id`. + * @return A void result on success, or an error code indicating the failure: + * - Error codes are defined by the derived class. + */ + [[nodiscard]] virtual auto + add_archive(uint16_t local_archive_id, clp_s::ArchiveReader const& archive_reader) + -> ystdlib::error_handling::Result + = 0; + + /** + * @return The serialized blob for each added archive, indexed by local archive ID. The returned + * views remain valid until the next call to `add_archive` or the builder's destruction. + */ + [[nodiscard]] virtual auto get_archive_blobs() const -> std::vector> = 0; +}; +} // namespace clp_s::filter + +#endif // CLP_S_FILTER_INDEX_BUILDER_HPP diff --git a/components/core/src/clp_s/filter/IndexBuilderSpecification.hpp b/components/core/src/clp_s/filter/IndexBuilderSpecification.hpp new file mode 100644 index 000000000..f7db53f7c --- /dev/null +++ b/components/core/src/clp_s/filter/IndexBuilderSpecification.hpp @@ -0,0 +1,98 @@ +#ifndef CLP_S_FILTER_INDEX_BUILDER_SPECIFICATION_HPP +#define CLP_S_FILTER_INDEX_BUILDER_SPECIFICATION_HPP + +#include +#include + +#include +#include + +#include +#include +#include + +namespace clp_s::filter { +/** + * Describes a single `IndexBuilder` implementation registered for an index: the archive sections it + * reads, the range of archive versions it supports, its index version, and a factory for creating + * instances of it. + * + * The range of supported archive versions is half-open: + * [first_supported_archive_version, last_supported_archive_version). A null last version indicates + * an open range that includes the newest archive version. + */ +class IndexBuilderSpecification { +public: + // Types + /** + * A factory for creating an `IndexBuilder`. + * + * @param config Implementation-defined configuration. + * @param packed_filter_spec A description of the Packed Filter being built. + * @return A result containing the created builder on success, or an error code indicating the + * failure: + * - Error codes are defined by the implementation. + */ + using Factory = auto (*)( + nlohmann::json const& config, + PackedFilterSpecification const& packed_filter_spec + ) -> ystdlib::error_handling::Result>; + + // Constructors + IndexBuilderSpecification( + archive_section_bitmap_t archive_section_bitmap, + archive_version_t first_supported_archive_version, + std::optional last_supported_archive_version, + index_version_t index_version, + Factory factory + ) + : m_archive_section_bitmap{archive_section_bitmap}, + m_first_supported_archive_version{first_supported_archive_version}, + m_last_supported_archive_version{last_supported_archive_version}, + m_index_version{index_version}, + m_factory{factory} {} + + // Methods + /** + * @param archive_version + * @return Whether this builder supports `archive_version`. + */ + [[nodiscard]] auto supports_archive_version(archive_version_t archive_version) const -> bool { + if (archive_version < m_first_supported_archive_version) { + return false; + } + return false == m_last_supported_archive_version.has_value() + || archive_version < m_last_supported_archive_version.value(); + } + + /** + * Creates an `IndexBuilder` using the registered factory. + * + * @param config Implementation-defined configuration. + * @param packed_filter_spec A description of the Packed Filter being built. + * @return Forwards the factory's return values. + */ + [[nodiscard]] auto create_builder( + nlohmann::json const& config, + PackedFilterSpecification const& packed_filter_spec + ) const -> ystdlib::error_handling::Result> { + return m_factory(config, packed_filter_spec); + } + + [[nodiscard]] auto get_archive_section_bitmap() const -> archive_section_bitmap_t { + return m_archive_section_bitmap; + } + + [[nodiscard]] auto get_index_version() const -> index_version_t { return m_index_version; } + +private: + // Variables + archive_section_bitmap_t m_archive_section_bitmap; + archive_version_t m_first_supported_archive_version; + std::optional m_last_supported_archive_version; + index_version_t m_index_version; + Factory m_factory; +}; +} // namespace clp_s::filter + +#endif // CLP_S_FILTER_INDEX_BUILDER_SPECIFICATION_HPP diff --git a/components/core/src/clp_s/filter/IndexDefs.hpp b/components/core/src/clp_s/filter/IndexDefs.hpp new file mode 100644 index 000000000..ea25a846b --- /dev/null +++ b/components/core/src/clp_s/filter/IndexDefs.hpp @@ -0,0 +1,161 @@ +#ifndef CLP_S_FILTER_INDEX_DEFS_HPP +#define CLP_S_FILTER_INDEX_DEFS_HPP + +#include +#include + +namespace clp_s::filter { +/** + * Identifier for an index type. Encoded as part of the metadata in a Packed Filter, so its width + * is part of the on-disk format. The ID space is partitioned into the reserved ranges described by + * `IndexIdRange` and the `c*IndexIdBegin` constants below. + */ +using index_id_t = uint16_t; + +/** + * A 32-bit semantic version of an index implementation, encoded the same way as a clp-s archive + * version: an 8-bit major version, an 8-bit minor version, and a 16-bit patch version. + * + * An index increments its major version when it makes an incompatible change to its serialized + * format, or when the archive format changes such that the index must operate in a significantly + * different way on newer archives. Minor version changes correspond to other backwards-compatible + * changes within a major version, and patch versions represent bug fixes. + */ +using index_version_t = uint32_t; + +/** + * A clp-s archive version, encoded as an 8-bit major version, an 8-bit minor version, and a 16-bit + * patch version. + */ +using archive_version_t = uint32_t; + +/** + * A bitmap of `ArchiveSection` flags describing which sections of an archive an index needs to + * read. Since this is a purely in-memory construct, its width may change freely. + */ +using archive_section_bitmap_t = uint8_t; + +/** + * Sections of an archive that an index may need to read while building. Callers build an + * `archive_section_bitmap_t` by ORing these flags together. + * + * NOTE: An actual index implementation may need finer-grained flags than these; the set is + * expected to grow as indices are added. + */ +enum class ArchiveSection : archive_section_bitmap_t { + Metadata = 1, + SchemaMetadata = 1 << 1, + Dictionaries = 1 << 2, + EncodedRecordTables = 1 << 3, +}; + +/** + * Reserved ranges of the Index ID space. + */ +enum class IndexIdRange : uint8_t { + OfficialOpenSource = 1, + OfficialClosedSource, + Custom, +}; + +// First Index ID reserved for official open-source indices. +constexpr index_id_t cOfficialOpenSourceIndexIdBegin{0x0000}; + +// First Index ID reserved for official closed-source indices. +constexpr index_id_t cOfficialClosedSourceIndexIdBegin{0x1000}; + +// First Index ID free for any custom index. +constexpr index_id_t cCustomIndexIdBegin{0x2000}; + +/** + * @param index_id + * @return The reserved range that `index_id` falls into. + */ +[[nodiscard]] constexpr auto classify_index_id(index_id_t index_id) -> IndexIdRange { + if (cCustomIndexIdBegin <= index_id) { + return IndexIdRange::Custom; + } + if (cOfficialClosedSourceIndexIdBegin <= index_id) { + return IndexIdRange::OfficialClosedSource; + } + return IndexIdRange::OfficialOpenSource; +} + +/** + * Encodes a semantic version into an `index_version_t`. + * + * @param major_version + * @param minor_version + * @param patch_version + * @return The encoded index version. + */ +[[nodiscard]] constexpr auto +make_index_version(uint8_t major_version, uint8_t minor_version, uint16_t patch_version) + -> index_version_t { + constexpr uint32_t cMajorVersionOffset{24U}; + constexpr uint32_t cMinorVersionOffset{16U}; + return (static_cast(major_version) << cMajorVersionOffset) + | (static_cast(minor_version) << cMinorVersionOffset) + | static_cast(patch_version); +} + +/** + * Decomposes an `index_version_t` into its major, minor, and patch components. + * + * @param index_version + * @return A tuple of the major version, the minor version, and the patch version. + */ +[[nodiscard]] constexpr auto decompose_index_version(index_version_t index_version) + -> std::tuple { + constexpr uint32_t cMajorVersionOffset{24U}; + constexpr uint32_t cMinorVersionOffset{16U}; + auto const major_version{static_cast(index_version >> cMajorVersionOffset)}; + auto const minor_version{static_cast(index_version >> cMinorVersionOffset)}; + auto const patch_version{static_cast(index_version)}; + return std::make_tuple(major_version, minor_version, patch_version); +} + +/** + * @param lhs + * @param rhs + * @return A bitmap with the flags of both `lhs` and `rhs` set. + */ +[[nodiscard]] constexpr auto operator|(ArchiveSection lhs, ArchiveSection rhs) + -> archive_section_bitmap_t { + return static_cast( + static_cast(lhs) | static_cast(rhs) + ); +} + +/** + * @param lhs + * @param rhs + * @return A copy of `lhs` with `rhs`'s flag additionally set. + */ +[[nodiscard]] constexpr auto operator|(archive_section_bitmap_t lhs, ArchiveSection rhs) + -> archive_section_bitmap_t { + return static_cast(lhs | static_cast(rhs)); +} + +/** + * @param section + * @return A bitmap with only `section`'s flag set. + */ +[[nodiscard]] constexpr auto to_archive_section_bitmap(ArchiveSection section) + -> archive_section_bitmap_t { + return static_cast(section); +} + +/** + * @param bitmap + * @param section + * @return Whether `section`'s flag is set in `bitmap`. + */ +[[nodiscard]] constexpr auto contains(archive_section_bitmap_t bitmap, ArchiveSection section) + -> bool { + auto const flag{static_cast(section)}; + return 0 != (bitmap & flag); +} +} // namespace clp_s::filter + +#endif // CLP_S_FILTER_INDEX_DEFS_HPP diff --git a/components/core/src/clp_s/filter/IndexRunner.hpp b/components/core/src/clp_s/filter/IndexRunner.hpp new file mode 100644 index 000000000..dbf35c6bd --- /dev/null +++ b/components/core/src/clp_s/filter/IndexRunner.hpp @@ -0,0 +1,62 @@ +#ifndef CLP_S_FILTER_INDEX_RUNNER_HPP +#define CLP_S_FILTER_INDEX_RUNNER_HPP + +#include +#include + +#include + +#include + +namespace clp_s::search::ast { +class Expression; +} // namespace clp_s::search::ast + +namespace clp_s::filter { +/** + * View over the set of candidate archives passed to `IndexRunner::filter`, where bit i corresponds + * to the local archive ID i. The component type fixes the word size used to represent the candidate + * set as it is passed across the FFI boundary. + */ +using CandidateArchiveBitmapView = BitmapView; + +/** + * Interface for using a deserialized index to narrow the set of archives that could match a query + * at filtering time. + */ +class IndexRunner { +public: + // Constructors + IndexRunner() = default; + + // Delete copy constructor and assignment operator + IndexRunner(IndexRunner const&) = delete; + auto operator=(IndexRunner const&) -> IndexRunner& = delete; + + // Default move constructor and assignment operator + IndexRunner(IndexRunner&&) noexcept = default; + auto operator=(IndexRunner&&) noexcept -> IndexRunner& = default; + + // Destructor + virtual ~IndexRunner() = default; + + // Methods + /** + * Narrows the set of candidate archives for a query by clearing the bits of archives that + * cannot contain any results matching the query. + * + * @param query The root of the query AST to filter against. + * @param candidate_archive_bitmap The set of candidate archives, where bit i corresponds to the + * local archive ID i. Returns the narrowed set of candidate archives. + * @return A void result on success, or an error code indicating the failure: + * - Error codes are defined by the derived class. + */ + [[nodiscard]] virtual auto filter( + std::shared_ptr const& query, + CandidateArchiveBitmapView& candidate_archive_bitmap + ) -> ystdlib::error_handling::Result + = 0; +}; +} // namespace clp_s::filter + +#endif // CLP_S_FILTER_INDEX_RUNNER_HPP diff --git a/components/core/src/clp_s/filter/PackedFilterSpecification.hpp b/components/core/src/clp_s/filter/PackedFilterSpecification.hpp new file mode 100644 index 000000000..8e3ba935a --- /dev/null +++ b/components/core/src/clp_s/filter/PackedFilterSpecification.hpp @@ -0,0 +1,35 @@ +#ifndef CLP_S_FILTER_PACKED_FILTER_SPECIFICATION_HPP +#define CLP_S_FILTER_PACKED_FILTER_SPECIFICATION_HPP + +#include + +#include + +namespace clp_s::filter { +/** + * Framework-provided description of a Packed Filter being built. Every archive in a Packed Filter + * is guaranteed to share the same archive version, which simplifies managing support for different + * archive versions across index implementations. + */ +class PackedFilterSpecification { +public: + // Constructors + PackedFilterSpecification(size_t num_archives, archive_version_t archive_version) + : m_num_archives{num_archives}, + m_archive_version{archive_version} {} + + // Methods + [[nodiscard]] auto get_num_archives() const -> size_t { return m_num_archives; } + + [[nodiscard]] auto get_archive_version() const -> archive_version_t { + return m_archive_version; + } + +private: + // Variables + size_t m_num_archives; + archive_version_t m_archive_version; +}; +} // namespace clp_s::filter + +#endif // CLP_S_FILTER_PACKED_FILTER_SPECIFICATION_HPP diff --git a/components/core/src/clp_s/filter/tests/test-clp_s-index_defs.cpp b/components/core/src/clp_s/filter/tests/test-clp_s-index_defs.cpp new file mode 100644 index 000000000..437de414b --- /dev/null +++ b/components/core/src/clp_s/filter/tests/test-clp_s-index_defs.cpp @@ -0,0 +1,58 @@ +#include +#include + +#include + +#include + +using clp_s::filter::ArchiveSection; + +TEST_CASE("index-defs-index-version-round-trip", "[clp_s][filter]") { + constexpr uint8_t cMajorVersion{1}; + constexpr uint8_t cMinorVersion{2}; + constexpr uint16_t cPatchVersion{3}; + auto const index_version{ + clp_s::filter::make_index_version(cMajorVersion, cMinorVersion, cPatchVersion) + }; + auto const [major_version, minor_version, patch_version]{ + clp_s::filter::decompose_index_version(index_version) + }; + REQUIRE(cMajorVersion == major_version); + REQUIRE(cMinorVersion == minor_version); + REQUIRE(cPatchVersion == patch_version); +} + +TEST_CASE("index-defs-classify-index-id", "[clp_s][filter]") { + REQUIRE(clp_s::filter::IndexIdRange::OfficialOpenSource + == clp_s::filter::classify_index_id(clp_s::filter::cOfficialOpenSourceIndexIdBegin)); + REQUIRE(clp_s::filter::IndexIdRange::OfficialOpenSource + == clp_s::filter::classify_index_id( + clp_s::filter::cOfficialClosedSourceIndexIdBegin - 1 + )); + REQUIRE(clp_s::filter::IndexIdRange::OfficialClosedSource + == clp_s::filter::classify_index_id(clp_s::filter::cOfficialClosedSourceIndexIdBegin)); + REQUIRE(clp_s::filter::IndexIdRange::OfficialClosedSource + == clp_s::filter::classify_index_id(clp_s::filter::cCustomIndexIdBegin - 1)); + REQUIRE(clp_s::filter::IndexIdRange::Custom + == clp_s::filter::classify_index_id(clp_s::filter::cCustomIndexIdBegin)); +} + +TEST_CASE("index-defs-archive-section-bitmap", "[clp_s][filter]") { + auto const bitmap{ArchiveSection::Metadata | ArchiveSection::Dictionaries}; + REQUIRE(clp_s::filter::contains(bitmap, ArchiveSection::Metadata)); + REQUIRE(clp_s::filter::contains(bitmap, ArchiveSection::Dictionaries)); + REQUIRE_FALSE(clp_s::filter::contains(bitmap, ArchiveSection::SchemaMetadata)); + REQUIRE_FALSE(clp_s::filter::contains(bitmap, ArchiveSection::EncodedRecordTables)); + + auto const extended_bitmap{bitmap | ArchiveSection::EncodedRecordTables}; + REQUIRE(clp_s::filter::contains(extended_bitmap, ArchiveSection::EncodedRecordTables)); + + REQUIRE(clp_s::filter::to_archive_section_bitmap(ArchiveSection::Metadata) + == clp_s::filter::to_archive_section_bitmap(ArchiveSection::Metadata)); + REQUIRE_FALSE( + clp_s::filter::contains( + clp_s::filter::to_archive_section_bitmap(ArchiveSection::Metadata), + ArchiveSection::Dictionaries + ) + ); +}