dictionary API

include/openzl/zl_dict.h

@@ -0,0 +1,120 @@
+// Copyright (c) Meta Platforms, Inc. and affiliates.
+
+#ifndef OPENZL_ZL_DICT_H
+#define OPENZL_ZL_DICT_H
+
+#include <stddef.h> // size_t
+#include <stdint.h> // uint8_t
+#include "openzl/zl_common_types.h"
+#include "openzl/zl_errors.h"       // ZL_RESULT_OF
+#include "openzl/zl_localParams.h"  // ZL_LocalParams
+#include "openzl/zl_opaque_types.h" // ZL_Dict
+#include "openzl/zl_portability.h"  // ZL_NOEXCEPT_FUNC_PTR
+
+#if defined(__cplusplus)
+extern "C" {
+#endif
+
+/*
+ * ================================== ZL_Dict ==================================
+ *
+ * The ZL_Dict is a dictionary object for use with both the encoder and decoder.
+ * Dictionaries are only supported for codecs. Dictionaries are also
+ * instance-specific. Two copies of the same codec within the same graph can
+ * have different associated dictionaries. The ZL_Dict APIs are exposed for
+ * *creating* dictionaries and are likely not what you are looking for. See the
+ * ZL_DictLoader for how to *use* dictionaries.
+ */
+
+typedef struct {
+    uint8_t bytes[32];
+} ZL_DictID;
+
+/**
+ * Helper function to pack a serialized dict buffer into the ZL_Dict wire
+ * format.
+ * @param dst the buffer to write the serialized dict into. The size must be
+ * larger than the @p src size by at least 60 bytes.
+ * @param codecID the codec associated with this dict.
+ * @return on success,the resulting serialized size. Otherwise, an error.
+ */
+ZL_Report ZL_Dict_pack(
+        void* dst,
+        size_t dstCapacity,
+        ZL_IDType codecID,
+        const void* src,
+        size_t srcSize);
+
+typedef ZL_Dict* ZL_DictPtr;
+ZL_RESULT_DECLARE_TYPE(ZL_DictPtr);
+
+/*
+ * =============================== ZL_DictBundle ===============================
+ *
+ * The ZL_DictBundle contains information about all the ZL_Dict objects required
+ * by a particular compressor. Note that any individual frame generated by a
+ * compressor may not require every single dictionary in the bundle to
+ * (de)compress. However, if it is *possible* for a frame to require a dict
+ * object to decompress, it must be recorded in the bundle.
+ */
+
+typedef struct {
+    uint8_t bytes[32];
+} ZL_DictBundleID;
+
+typedef struct {
+    ZL_DictBundleID bundleID;
+    size_t numDicts;
+    ZL_DictID* dicts;
+} ZL_DictBundle;
+
+typedef ZL_DictBundle* ZL_DictBundlePtr;
+ZL_RESULT_DECLARE_TYPE(ZL_DictBundlePtr);
+
+/**
+ * Helper function to pack a list of dict IDs into the ZL_DictBundle wire
+ * format.
+ * @param dst the buffer to write the serialized bundle into. The size must be
+ * larger than the byte size of @p dictIDs by at least 48 bytes.
+ */
+ZL_Report
+ZL_DictBundle_serialize(void* dst, size_t dstCapacity, ZL_DictBundle* bundle);
+
+/*
+ * =============================== ZL_DictLoader ===============================
+ *
+ * The ZL_DictLoader is responsible for fetching and (de)materializing dicts and
+ * bundles. It is used by both the ZL_Compressor/ZL_CCtx and ZL_DCtx to provide
+ * dictionary functionality.
+ *
+ * The fetching APIs are provided unimplemented to enable you to customize blob
+ * sources and caching behavior. (A simple reference implementation is also
+ * provided in ZL_StaticSetDictLoader. This is sufficient for use with a static
+ * set of known dictionaries and bundles.)
+ *
+ * Before usage, individual codec de/materializers must be registered with the
+ * ZL_DictLoader.
+ */
+typedef struct {
+    void* opaque; // optional opaque pointer. The ZL_DictLoader will take
+                  // ownership of this pointer.
+    void (*freeOpaque)(
+            void* opaque); // Function to free the opaque pointer, if provided.
+    /**
+     * Functions to fetch bundles and dicts. Implementors are expected to use
+     * the TODO helper functions to parse raw blobs. Implementors are encouraged
+     * to cache bundles and dicts using a custom state stored in @p opaque .
+     */
+    ZL_RESULT_OF(ZL_DictBundlePtr) (
+            *fetchDictBundle)(void* opaque, const ZL_DictBundleID* id);
+    ZL_RESULT_OF(ZL_DictPtr) (*fetchDict)(void* opaque, const ZL_DictID* id);
+} ZL_DictLoaderDesc;
+
+ZL_DictLoader* ZL_DictLoader_create(const ZL_DictLoaderDesc* desc);
+void ZL_DictLoader_free(ZL_DictLoader* loader);
+
+#if defined(__cplusplus)
+} // extern "C"
+#endif
+
+#endif // OPENZL_ZL_DICT_H

include/openzl/zl_opaque_types.h

@@ -62,6 +62,8 @@ typedef struct ZL_Selector_s ZL_Selector;
 typedef struct ZL_Graph_s ZL_Graph;
 typedef struct ZL_Edge_s ZL_Edge;
 typedef struct ZL_Segmenter_s ZL_Segmenter;
+typedef struct ZL_Dict_s ZL_Dict;
+typedef struct ZL_DictLoader_s ZL_DictLoader;
 
 // Generic List construction macro (C99)
 #define ZL_LIST_SIZE(_type, ...) \

src/openzl/dict/BUCK

@@ -0,0 +1,15 @@
+load("@fbcode_macros//build_defs:cpp_library.bzl", "cpp_library")
+
+oncall("fill in oncall here @nocommit")
+
+cpp_library(
+    name = "dict",
+    srcs = ["dict.h"],
+    labels = ["autodeps2_generated"],
+)
+
+cpp_library(
+    name = "dictloader",
+    srcs = ["dictloader.h"],
+    labels = ["autodeps2_generated"],
+)

src/openzl/dict/dict.h

@@ -0,0 +1,12 @@
+// Copyright (c) Meta Platforms, Inc. and affiliates.
+
+#ifndef OPENZL_DICT_DICT_H
+#define OPENZL_DICT_DICT_H
+
+struct ZL_Dict_s {
+    ZL_DictID dictID;
+    ZL_IDType materializingCodec;
+    void* dictObj;
+}
+
+#endif // OPENZL_DICT_DICT_H

src/openzl/dict/dictloader.h

@@ -0,0 +1,10 @@
+// Copyright (c) Meta Platforms, Inc. and affiliates.
+
+#ifndef OPENZL_DICT_DICTLOADER_H
+#define OPENZL_DICT_DICTLOADER_H
+
+struct ZL_DictLoader_s {
+    void* todo;
+}
+
+#endif // OPENZL_DICT_DICTLOADER_H

tests/BUCK

@@ -17,6 +17,7 @@ zs_unittest(
     deps = [
         "..:common",
         "..:compress",
+        # "..:dict",
         "../cpp:openzl_cpp",
         "../tools:json",
         "datagen:datagen",

tests/unittest/dict/Dict.h

@@ -0,0 +1,178 @@
+// Copyright (c) Meta Platforms, Inc. and affiliates.
+
+#ifndef ZSTRONG_TESTS_DICT_H
+#define ZSTRONG_TESTS_DICT_H
+
+#include <stdint.h>
+#include <map>
+
+#include "openzl/common/allocation.h"
+#include "openzl/zl_errors.h"
+#include "openzl/zl_opaque_types.h"
+
+typedef struct {
+    uint8_t bytes[32];
+} ZL_SHA256;
+ZL_RESULT_DECLARE_TYPE(ZL_SHA256);
+
+typedef ZL_SHA256 ZL_DictID;
+// ZL_Dict
+typedef struct {
+    ZL_DictID dictID; // this is a SHA of the serialized ZL_Dict
+    ZL_IDType materializingCodec;
+    void (*dematerializeFn)(void* dict);
+    void* dictObj;
+} ZL_Dict;
+
+// the dict loader needs to support search using both a
+// 32-bit ID and a SHA.
+typedef struct {
+    uint32_t id;
+    ZL_SHA256 sha; // this is a SHA of the serialized ZL_DictBundle
+} ZL_DictBundleID;
+
+// ZL_DictBundle
+typedef struct {
+    ZL_DictBundleID bundleID;
+    bool compressionOnly;
+    size_t numDicts;
+    ZL_SHA256* dicts;
+} ZL_DictBundle;
+
+// ======================================================
+// ==================== ZL_DictBundle ===================
+// ======================================================
+
+size_t ZL_DictBundle_numDicts(const ZL_DictBundle* bundle);
+const ZL_DictID* ZL_DictBundle_dictIDs(const ZL_DictBundle* bundle);
+
+// ======================================================
+// =================== ZL_DictLoader ====================
+// ======================================================
+
+// For clarity of illustration, this sample uses C++ class semantics. In prod,
+// the C API will be similar but semantically more complex.
+class ZL_DictLoader {
+   public:
+    // Use ZL_DictBundle_* helper functions
+    virtual ZL_RESULT_OF(ZL_DictBundle*)
+            fetchDictBundle(const ZL_DictBundleID* id) = 0;
+
+    // A Compressor or DCtx is required to materialize properly
+    virtual ZL_RESULT_OF(ZL_Dict*) fetchDict(
+            const ZL_DictID* id,
+            const ZL_Compressor* materializingCompressor) = 0;
+    virtual ZL_RESULT_OF(ZL_Dict*) fetchDict(
+            const ZL_DictID* id,
+            const ZL_DCtx* materializingDCtx) = 0;
+};
+
+class ZL_StaticSetDictLoader : public ZL_DictLoader {
+   public:
+    ZL_StaticSetDictLoader(
+            std::string bundleBlob,
+            std::vector<std::string> dictBlobs)
+            : alloc_(ALLOC_Arena_HeapArena_create()),
+              bundle_(bundleBlob),
+              dictBlobs_(dictBlobs)
+    {
+    }
+
+    ZL_Dict* fetchDict(
+            const ZL_DictID* id,
+            const ZL_Compressor* materializingCompressor) override
+    {
+        size_t idx = dictHashes_.find(id) - dictHashes_.begin();
+        auto blob  = dictBlobs_[idx];
+        // preflight checks here
+        auto codecID;
+        auto payload;
+        auto res = ZL_Compressor_materializeDict(
+                materializingCompressor,
+                codecID,
+                payload.data(),
+                payload.size(),
+                this->alloc_);
+        ZL_ERR_IF_ERR(res);
+        return ZL_WRAP_VALUE(ZL_RES_value(res).ptr);
+    }
+    // other inherited functions not shown
+
+   private:
+    ZL_DictBundle bundle_;
+    std::vector<std::string> dictBlobs_;
+    std::vector<ZL_DictID> dictHashes_;
+
+    std::map<std::pair<ZL_DictID, void* /* materializing fn ptr*/>, void*>
+            dicts_;
+};
+
+class MCDictLoader : public ZL_DictLoader {
+   public:
+    MCDictLoader(ZL_DictAlloc* alloc);
+    bool hasBundle(ZL_DictBundleID bundleID);
+
+    // Implementors of ZL_DictLoader are encouraged to cache dicts, but it's not
+    // required. For Managed Compression, this will be a requirement.
+    ZL_Report prefetchDictsForBundle(
+            const ZL_DictBundleID* id,
+            ZL_Compressor* materializingCompressor);
+    ZL_Report prefetchDictsForBundle(
+            const ZL_DictBundleID* id,
+            ZL_DCtx* materializingDCtx);
+}
+
+struct ZL_MaterializationResult {
+    void* ptr;
+    void* materializeFnPtr;
+    void* dematerializeFnPtr;
+};
+
+// ======================================================
+// ================== Compression APIs ==================
+// ======================================================
+
+ZL_RESULT_OF(ZL_MaterializationResult)
+ZL_Compressor_materializeDict(
+        ZL_Compressor* compressor,
+        ZL_IDtype codecID,
+        void* blob,
+        size_t blobSize,
+        ZL_DictAlloc* allocator);
+
+// Attach in the same way we do ZL_CCtx_refCompressor()
+ZL_Report ZL_CCtx_refDictLoader(ZL_CCtx* cctx, ZL_DictLoader* loader);
+
+// If there is a dict bundle already set (e.g. via training), return its ID.
+ZL_DictBundleID ZL_Compressor_getDictBundleID(ZL_Compressor* compressor);
+
+void ZL_Compressor_useDictBundle(
+        ZL_Compressor* compressor,
+        ZL_DictBundleID bundleSha);
+
+// ======================================================
+// ================= Decompression APIs =================
+// ======================================================
+
+ZL_RESULT_OF(ZL_MaterializationResult)
+ZL_DCtx_materializeDict(
+        ZL_DCtx* dctx,
+        ZL_IDtype codecID,
+        void* blob,
+        size_t blobSize,
+        ZL_DictAlloc* allocator);
+
+// Attach in the same way we do ZL_CCtx_refCompressor()
+ZL_Report ZL_DCtx_refDictLoader(ZL_DCtx* dctx, ZL_DictLoader* loader);
+
+// ======================================================
+// ===================== Codec APIs =====================
+// ======================================================
+
+// If there is an associated dict, we can fetch it at encode/decode time. It
+// will be the job of the CCtx/DCtx to properly request materialization and pass
+// these blobs down
+void* ZL_Encoder_getDict(ZL_Encoder* eictx);
+void* ZL_Decoder_getDict(ZL_Decoder* dictx);
+
+#endif // ZSTRONG_TESTS_DICT_H