feat(clp-s::filter): Add interfaces and definitions for the top-level…

components/core/src/clp_s/CMakeLists.txt

@@ -552,6 +552,7 @@ if(CLP_BUILD_TESTING)
                 INTERFACE
                 filter/tests/test-clp_s-bitmap_view.cpp
                 filter/tests/test-clp_s-bloom_filter.cpp
+                filter/tests/test-clp_s-index_defs.cpp
                 filter/tests/test-clp_s-xxhash.cpp
                 tests/clp_s_test_utils.cpp
                 tests/clp_s_test_utils.hpp

components/core/src/clp_s/filter/CMakeLists.txt

@@ -13,6 +13,11 @@ set(
         FilterReader.hpp
         HashAlgorithm.cpp
         HashAlgorithm.hpp
+        IndexBuilder.hpp
+        IndexBuilderSpecification.hpp
+        IndexDefs.hpp
+        IndexRunner.hpp
+        PackedFilterSpecification.hpp
         XxHash.cpp
         XxHash.hpp
 )

components/core/src/clp_s/filter/IndexBuilder.hpp

@@ -0,0 +1,60 @@
+#ifndef CLP_S_FILTER_INDEX_BUILDER_HPP
+#define CLP_S_FILTER_INDEX_BUILDER_HPP
+
+#include <cstdint>
+#include <span>
+#include <vector>
+
+#include <ystdlib/error_handling/Result.hpp>
+
+namespace clp_s {
+class ArchiveReader;
+}  // namespace clp_s
+
+namespace clp_s::filter {
+/**
+ * Interface for building an index over the archives of a Packed Filter. The framework feeds an
+ * implementation one archive at a time so that only a single archive needs to be loaded into memory
+ * at once; the implementation produces that archive's serialized blob, which the framework collects
+ * to assemble the Packed Filter.
+ */
+class IndexBuilder {
+public:
+    // Constructors
+    IndexBuilder() = default;
+
+    // Delete copy constructor and assignment operator
+    IndexBuilder(IndexBuilder const&) = delete;
+    auto operator=(IndexBuilder const&) -> IndexBuilder& = delete;
+
+    // Default move constructor and assignment operator
+    IndexBuilder(IndexBuilder&&) noexcept = default;
+    auto operator=(IndexBuilder&&) noexcept -> IndexBuilder& = default;
+
+    // Destructor
+    virtual ~IndexBuilder() = default;
+
+    // Methods
+    /**
+     * Builds the index's data for an archive and stores the resulting per-archive blob.
+     *
+     * @param local_archive_id The Packed-Filter-local ID of the archive, in the range
+     * [0, num_archives).
+     * @param archive_reader A reader for the archive identified by `local_archive_id`.
+     * @return A void result on success, or an error code indicating the failure:
+     * - Error codes are defined by the derived class.
+     */
+    [[nodiscard]] virtual auto
+    add_archive(uint16_t local_archive_id, clp_s::ArchiveReader const& archive_reader)
+            -> ystdlib::error_handling::Result<void>
+            = 0;
+
+    /**
+     * @return The serialized blob for each added archive, indexed by local archive ID. The returned
+     * views remain valid until the next call to `add_archive` or the builder's destruction.
+     */
+    [[nodiscard]] virtual auto get_archive_blobs() const -> std::vector<std::span<char const>> = 0;
+};
+}  // namespace clp_s::filter
+
+#endif  // CLP_S_FILTER_INDEX_BUILDER_HPP

components/core/src/clp_s/filter/IndexBuilderSpecification.hpp

@@ -0,0 +1,98 @@
+#ifndef CLP_S_FILTER_INDEX_BUILDER_SPECIFICATION_HPP
+#define CLP_S_FILTER_INDEX_BUILDER_SPECIFICATION_HPP
+
+#include <memory>
+#include <optional>
+
+#include <nlohmann/json_fwd.hpp>
+#include <ystdlib/error_handling/Result.hpp>
+
+#include <clp_s/filter/IndexBuilder.hpp>
+#include <clp_s/filter/IndexDefs.hpp>
+#include <clp_s/filter/PackedFilterSpecification.hpp>
+
+namespace clp_s::filter {
+/**
+ * Describes a single `IndexBuilder` implementation registered for an index: the archive sections it
+ * reads, the range of archive versions it supports, its index version, and a factory for creating
+ * instances of it.
+ *
+ * The range of supported archive versions is half-open:
+ * [first_supported_archive_version, last_supported_archive_version). A null last version indicates
+ * an open range that includes the newest archive version.
+ */
+class IndexBuilderSpecification {
+public:
+    // Types
+    /**
+     * A factory for creating an `IndexBuilder`.
+     *
+     * @param config Implementation-defined configuration.
+     * @param packed_filter_spec A description of the Packed Filter being built.
+     * @return A result containing the created builder on success, or an error code indicating the
+     * failure:
+     * - Error codes are defined by the implementation.
+     */
+    using Factory = auto (*)(
+            nlohmann::json const& config,
+            PackedFilterSpecification const& packed_filter_spec
+    ) -> ystdlib::error_handling::Result<std::unique_ptr<IndexBuilder>>;
+
+    // Constructors
+    IndexBuilderSpecification(
+            archive_section_bitmap_t archive_section_bitmap,
+            archive_version_t first_supported_archive_version,
+            std::optional<archive_version_t> last_supported_archive_version,
+            index_version_t index_version,
+            Factory factory
+    )
+            : m_archive_section_bitmap{archive_section_bitmap},
+              m_first_supported_archive_version{first_supported_archive_version},
+              m_last_supported_archive_version{last_supported_archive_version},
+              m_index_version{index_version},
+              m_factory{factory} {}
+
+    // Methods
+    /**
+     * @param archive_version
+     * @return Whether this builder supports `archive_version`.
+     */
+    [[nodiscard]] auto supports_archive_version(archive_version_t archive_version) const -> bool {
+        if (archive_version < m_first_supported_archive_version) {
+            return false;
+        }
+        return false == m_last_supported_archive_version.has_value()
+               || archive_version < m_last_supported_archive_version.value();
+    }
+
+    /**
+     * Creates an `IndexBuilder` using the registered factory.
+     *
+     * @param config Implementation-defined configuration.
+     * @param packed_filter_spec A description of the Packed Filter being built.
+     * @return Forwards the factory's return values.
+     */
+    [[nodiscard]] auto create_builder(
+            nlohmann::json const& config,
+            PackedFilterSpecification const& packed_filter_spec
+    ) const -> ystdlib::error_handling::Result<std::unique_ptr<IndexBuilder>> {
+        return m_factory(config, packed_filter_spec);
+    }
+
+    [[nodiscard]] auto get_archive_section_bitmap() const -> archive_section_bitmap_t {
+        return m_archive_section_bitmap;
+    }
+
+    [[nodiscard]] auto get_index_version() const -> index_version_t { return m_index_version; }
+
+private:
+    // Variables
+    archive_section_bitmap_t m_archive_section_bitmap;
+    archive_version_t m_first_supported_archive_version;
+    std::optional<archive_version_t> m_last_supported_archive_version;
+    index_version_t m_index_version;
+    Factory m_factory;
+};
+}  // namespace clp_s::filter
+
+#endif  // CLP_S_FILTER_INDEX_BUILDER_SPECIFICATION_HPP

components/core/src/clp_s/filter/IndexDefs.hpp

@@ -0,0 +1,161 @@
+#ifndef CLP_S_FILTER_INDEX_DEFS_HPP
+#define CLP_S_FILTER_INDEX_DEFS_HPP
+
+#include <cstdint>
+#include <tuple>
+
+namespace clp_s::filter {
+/**
+ * Identifier for an index type. Encoded as part of the metadata in a Packed Filter, so its width
+ * is part of the on-disk format. The ID space is partitioned into the reserved ranges described by
+ * `IndexIdRange` and the `c*IndexIdBegin` constants below.
+ */
+using index_id_t = uint16_t;
+
+/**
+ * A 32-bit semantic version of an index implementation, encoded the same way as a clp-s archive
+ * version: an 8-bit major version, an 8-bit minor version, and a 16-bit patch version.
+ *
+ * An index increments its major version when it makes an incompatible change to its serialized
+ * format, or when the archive format changes such that the index must operate in a significantly
+ * different way on newer archives. Minor version changes correspond to other backwards-compatible
+ * changes within a major version, and patch versions represent bug fixes.
+ */
+using index_version_t = uint32_t;
+
+/**
+ * A clp-s archive version, encoded as an 8-bit major version, an 8-bit minor version, and a 16-bit
+ * patch version.
+ */
+using archive_version_t = uint32_t;
+
+/**
+ * A bitmap of `ArchiveSection` flags describing which sections of an archive an index needs to
+ * read. Since this is a purely in-memory construct, its width may change freely.
+ */
+using archive_section_bitmap_t = uint8_t;
+
+/**
+ * Sections of an archive that an index may need to read while building. Callers build an
+ * `archive_section_bitmap_t` by ORing these flags together.
+ *
+ * NOTE: An actual index implementation may need finer-grained flags than these; the set is
+ * expected to grow as indices are added.
+ */
+enum class ArchiveSection : archive_section_bitmap_t {
+    Metadata = 1,
+    SchemaMetadata = 1 << 1,
+    Dictionaries = 1 << 2,
+    EncodedRecordTables = 1 << 3,
+};
+
+/**
+ * Reserved ranges of the Index ID space.
+ */
+enum class IndexIdRange : uint8_t {
+    OfficialOpenSource = 1,
+    OfficialClosedSource,
+    Custom,
+};
+
+// First Index ID reserved for official open-source indices.
+constexpr index_id_t cOfficialOpenSourceIndexIdBegin{0x0000};
+
+// First Index ID reserved for official closed-source indices.
+constexpr index_id_t cOfficialClosedSourceIndexIdBegin{0x1000};
+
+// First Index ID free for any custom index.
+constexpr index_id_t cCustomIndexIdBegin{0x2000};
+
+/**
+ * @param index_id
+ * @return The reserved range that `index_id` falls into.
+ */
+[[nodiscard]] constexpr auto classify_index_id(index_id_t index_id) -> IndexIdRange {
+    if (cCustomIndexIdBegin <= index_id) {
+        return IndexIdRange::Custom;
+    }
+    if (cOfficialClosedSourceIndexIdBegin <= index_id) {
+        return IndexIdRange::OfficialClosedSource;
+    }
+    return IndexIdRange::OfficialOpenSource;
+}
+
+/**
+ * Encodes a semantic version into an `index_version_t`.
+ *
+ * @param major_version
+ * @param minor_version
+ * @param patch_version
+ * @return The encoded index version.
+ */
+[[nodiscard]] constexpr auto
+make_index_version(uint8_t major_version, uint8_t minor_version, uint16_t patch_version)
+        -> index_version_t {
+    constexpr uint32_t cMajorVersionOffset{24U};
+    constexpr uint32_t cMinorVersionOffset{16U};
+    return (static_cast<index_version_t>(major_version) << cMajorVersionOffset)
+           | (static_cast<index_version_t>(minor_version) << cMinorVersionOffset)
+           | static_cast<index_version_t>(patch_version);
+}
+
+/**
+ * Decomposes an `index_version_t` into its major, minor, and patch components.
+ *
+ * @param index_version
+ * @return A tuple of the major version, the minor version, and the patch version.
+ */
+[[nodiscard]] constexpr auto decompose_index_version(index_version_t index_version)
+        -> std::tuple<uint8_t, uint8_t, uint16_t> {
+    constexpr uint32_t cMajorVersionOffset{24U};
+    constexpr uint32_t cMinorVersionOffset{16U};
+    auto const major_version{static_cast<uint8_t>(index_version >> cMajorVersionOffset)};
+    auto const minor_version{static_cast<uint8_t>(index_version >> cMinorVersionOffset)};
+    auto const patch_version{static_cast<uint16_t>(index_version)};
+    return std::make_tuple(major_version, minor_version, patch_version);
+}
+
+/**
+ * @param lhs
+ * @param rhs
+ * @return A bitmap with the flags of both `lhs` and `rhs` set.
+ */
+[[nodiscard]] constexpr auto operator|(ArchiveSection lhs, ArchiveSection rhs)
+        -> archive_section_bitmap_t {
+    return static_cast<archive_section_bitmap_t>(
+            static_cast<archive_section_bitmap_t>(lhs) | static_cast<archive_section_bitmap_t>(rhs)
+    );
+}
+
+/**
+ * @param lhs
+ * @param rhs
+ * @return A copy of `lhs` with `rhs`'s flag additionally set.
+ */
+[[nodiscard]] constexpr auto operator|(archive_section_bitmap_t lhs, ArchiveSection rhs)
+        -> archive_section_bitmap_t {
+    return static_cast<archive_section_bitmap_t>(lhs | static_cast<archive_section_bitmap_t>(rhs));
+}
+
+/**
+ * @param section
+ * @return A bitmap with only `section`'s flag set.
+ */
+[[nodiscard]] constexpr auto to_archive_section_bitmap(ArchiveSection section)
+        -> archive_section_bitmap_t {
+    return static_cast<archive_section_bitmap_t>(section);
+}
+
+/**
+ * @param bitmap
+ * @param section
+ * @return Whether `section`'s flag is set in `bitmap`.
+ */
+[[nodiscard]] constexpr auto contains(archive_section_bitmap_t bitmap, ArchiveSection section)
+        -> bool {
+    auto const flag{static_cast<archive_section_bitmap_t>(section)};
+    return 0 != (bitmap & flag);
+}
+}  // namespace clp_s::filter
+
+#endif  // CLP_S_FILTER_INDEX_DEFS_HPP