From 2014b02505dd9b84149d74a54891582524a8e63c Mon Sep 17 00:00:00 2001 From: Victor Zhang Date: Mon, 9 Mar 2026 12:03:49 -0700 Subject: [PATCH 1/3] dict API v2 Summary: This API is a pared down version that pushes a lot of the heavy lifting into the ZL_DictLoader implementation. Rather than fully specifying the dict store construct, we create a minimal API that enables the integration with Managed Compression and is sufficient for the purposes of the cctx/dctx. Note the use of C++ in the definitions of the ZL_DictLoader itself. I anticipate our biggest use cases will be sophisticated systems like Managed Compression, for whom an ergonomic C++ interface is a P0. The C API will use the same tricks as we have always used to enable C++ class semantics. Differential Revision: D95292077 --- tests/BUCK | 1 + tests/unittest/dict/Dict.h | 178 ++++++++++++++++++++++++++++++++ tests/unittest/dict/DictE2E.cpp | 87 ++++++++++++++++ 3 files changed, 266 insertions(+) create mode 100644 tests/unittest/dict/Dict.h create mode 100644 tests/unittest/dict/DictE2E.cpp diff --git a/tests/BUCK b/tests/BUCK index 274b58098..2d4278199 100644 --- a/tests/BUCK +++ b/tests/BUCK @@ -17,6 +17,7 @@ zs_unittest( deps = [ "..:common", "..:compress", + # "..:dict", "../cpp:openzl_cpp", "../tools:json", "datagen:datagen", diff --git a/tests/unittest/dict/Dict.h b/tests/unittest/dict/Dict.h new file mode 100644 index 000000000..d28ca03d8 --- /dev/null +++ b/tests/unittest/dict/Dict.h @@ -0,0 +1,178 @@ +// Copyright (c) Meta Platforms, Inc. and affiliates. + +#ifndef ZSTRONG_TESTS_DICT_H +#define ZSTRONG_TESTS_DICT_H + +#include +#include + +#include "openzl/common/allocation.h" +#include "openzl/zl_errors.h" +#include "openzl/zl_opaque_types.h" + +typedef struct { + uint8_t bytes[32]; +} ZL_SHA256; +ZL_RESULT_DECLARE_TYPE(ZL_SHA256); + +typedef ZL_SHA256 ZL_DictID; +// ZL_Dict +typedef struct { + ZL_DictID dictID; // this is a SHA of the serialized ZL_Dict + ZL_IDType materializingCodec; + void (*dematerializeFn)(void* dict); + void* dictObj; +} ZL_Dict; + +// the dict loader needs to support search using both a +// 32-bit ID and a SHA. +typedef struct { + uint32_t id; + ZL_SHA256 sha; // this is a SHA of the serialized ZL_DictBundle +} ZL_DictBundleID; + +// ZL_DictBundle +typedef struct { + ZL_DictBundleID bundleID; + bool compressionOnly; + size_t numDicts; + ZL_SHA256* dicts; +} ZL_DictBundle; + +// ====================================================== +// ==================== ZL_DictBundle =================== +// ====================================================== + +size_t ZL_DictBundle_numDicts(const ZL_DictBundle* bundle); +const ZL_DictID* ZL_DictBundle_dictIDs(const ZL_DictBundle* bundle); + +// ====================================================== +// =================== ZL_DictLoader ==================== +// ====================================================== + +// For clarity of illustration, this sample uses C++ class semantics. In prod, +// the C API will be similar but semantically more complex. +class ZL_DictLoader { + public: + // Use ZL_DictBundle_* helper functions + virtual ZL_RESULT_OF(ZL_DictBundle*) + fetchDictBundle(const ZL_DictBundleID* id) = 0; + + // A Compressor or DCtx is required to materialize properly + virtual ZL_RESULT_OF(ZL_Dict*) fetchDict( + const ZL_DictID* id, + const ZL_Compressor* materializingCompressor) = 0; + virtual ZL_RESULT_OF(ZL_Dict*) fetchDict( + const ZL_DictID* id, + const ZL_DCtx* materializingDCtx) = 0; +}; + +class ZL_StaticSetDictLoader : public ZL_DictLoader { + public: + ZL_StaticSetDictLoader( + std::string bundleBlob, + std::vector dictBlobs) + : alloc_(ALLOC_Arena_HeapArena_create()), + bundle_(bundleBlob), + dictBlobs_(dictBlobs) + { + } + + ZL_Dict* fetchDict( + const ZL_DictID* id, + const ZL_Compressor* materializingCompressor) override + { + size_t idx = dictHashes_.find(id) - dictHashes_.begin(); + auto blob = dictBlobs_[idx]; + // preflight checks here + auto codecID; + auto payload; + auto res = ZL_Compressor_materializeDict( + materializingCompressor, + codecID, + payload.data(), + payload.size(), + this->alloc_); + ZL_ERR_IF_ERR(res); + return ZL_WRAP_VALUE(ZL_RES_value(res).ptr); + } + // other inherited functions not shown + + private: + ZL_DictBundle bundle_; + std::vector dictBlobs_; + std::vector dictHashes_; + + std::map, void*> + dicts_; +}; + +class MCDictLoader : public ZL_DictLoader { + public: + MCDictLoader(ZL_DictAlloc* alloc); + bool hasBundle(ZL_DictBundleID bundleID); + + // Implementors of ZL_DictLoader are encouraged to cache dicts, but it's not + // required. For Managed Compression, this will be a requirement. + ZL_Report prefetchDictsForBundle( + const ZL_DictBundleID* id, + ZL_Compressor* materializingCompressor); + ZL_Report prefetchDictsForBundle( + const ZL_DictBundleID* id, + ZL_DCtx* materializingDCtx); +} + +struct ZL_MaterializationResult { + void* ptr; + void* materializeFnPtr; + void* dematerializeFnPtr; +}; + +// ====================================================== +// ================== Compression APIs ================== +// ====================================================== + +ZL_RESULT_OF(ZL_MaterializationResult) +ZL_Compressor_materializeDict( + ZL_Compressor* compressor, + ZL_IDtype codecID, + void* blob, + size_t blobSize, + ZL_DictAlloc* allocator); + +// Attach in the same way we do ZL_CCtx_refCompressor() +ZL_Report ZL_CCtx_refDictLoader(ZL_CCtx* cctx, ZL_DictLoader* loader); + +// If there is a dict bundle already set (e.g. via training), return its ID. +ZL_DictBundleID ZL_Compressor_getDictBundleID(ZL_Compressor* compressor); + +void ZL_Compressor_useDictBundle( + ZL_Compressor* compressor, + ZL_DictBundleID bundleSha); + +// ====================================================== +// ================= Decompression APIs ================= +// ====================================================== + +ZL_RESULT_OF(ZL_MaterializationResult) +ZL_DCtx_materializeDict( + ZL_DCtx* dctx, + ZL_IDtype codecID, + void* blob, + size_t blobSize, + ZL_DictAlloc* allocator); + +// Attach in the same way we do ZL_CCtx_refCompressor() +ZL_Report ZL_DCtx_refDictLoader(ZL_DCtx* dctx, ZL_DictLoader* loader); + +// ====================================================== +// ===================== Codec APIs ===================== +// ====================================================== + +// If there is an associated dict, we can fetch it at encode/decode time. It +// will be the job of the CCtx/DCtx to properly request materialization and pass +// these blobs down +void* ZL_Encoder_getDict(ZL_Encoder* eictx); +void* ZL_Decoder_getDict(ZL_Decoder* dictx); + +#endif // ZSTRONG_TESTS_DICT_H diff --git a/tests/unittest/dict/DictE2E.cpp b/tests/unittest/dict/DictE2E.cpp new file mode 100644 index 000000000..081a45e09 --- /dev/null +++ b/tests/unittest/dict/DictE2E.cpp @@ -0,0 +1,87 @@ +// Copyright (c) Meta Platforms, Inc. and affiliates. + +#include + +#include "openzl/common/allocation.h" +#include "openzl/zl_compress.h" +#include "openzl/zl_compressor.h" + +#include "Dict.h" + +using namespace ::testing; + +TEST(DictE2E, bundleOSS) +{ + ZL_Compressor* compressor; // pre-existing + std::string infoStr = "bundle info"; + std::vector dictStrs = { "dict1", "dict2" }; + + // Default dict behavior. Creates a new arena for managed allocations + ZL_StaticSetDictLoader dictLoader(infoStr, dictStrs); + // (Optional) check dicts + ZL_DictBundleID bundleID = ZL_Compressor_getDictBundleID(compressor); + ZL_DictBundle* bundle = ZL_RES_value(dictLoader.fetchDictBundle(bundleID)); + for (size_t i = 0; i < ZL_DictBundle_numDicts(bundle); ++i) { + ZL_DictID dictID = ZL_DictBundle_dictIDs(bundle)[i]; + // Example: pre-fetch dict to ensure availability + ZL_REQUIRE_SUCCESS(dictLoader.fetchDict(&dictID, compressor)); + } + ZL_CCtx* cctx; + ZL_CCtx_refDictLoader(cctx, dictLoader); +} + +TEST(DictE2E, bundleMC) +{ + std::vector compressorPool; + std::map + bundlesInCfg; // simulate configerator + std::map dictsInCfg; // simulate configerator + + ZL_DictLoader globalStore; // custom dict loader + for (auto* comp : compressorPool) { + ZL_DictBundleID bundleID = ZL_Compressor_getDictBundleID(comp); + if (!ZL_SHA256_isValid(&bundleID)) { + // no dicts for this compressor + continue; + } + if (globalStore.hasBundle(bundleID)) { // MC-specific function + // no need to double-materialize + continue; + } + // If we are unable to materialize all dicts, that's an error + ZL_REQUIRE_SUCCESS(globalStore.prefetchDictsForBundle( + bundleID, comp)); // MC-specific function + } +} + +TEST(DictE2E, bundleDecomp) +{ + ZL_DCtx* dctx; + std::map bundlesInCfg; // simulate configerator + std::map dictsInCfg; // simulate configerator + MCDictLoader globalStore; // one per process + + std::vector compressedBlobs = { "blob1", "blob2" }; + + ZL_DCtx_refDictLoader(dctx, globalStore); + for (auto& blob : compressedBlobs) { + // This is an illustration of a hypothetical call flow. In practice, the + // user would simply call: + // ZL_DCtx_refDictLoader(dctx, globalStore); + // ZL_DCtx_decompress(dctx, blob.data(), blob.size()); + ZL_DictBundleID bundleID = + DCtx_getBundleIDFromFrame(dctx, blob.data(), blob.size()); + auto res = globalStore.fetchDictBundle(bundleID); + ZL_ERR_IF_ERR(res); + ZL_DictBundle* bundle = ZL_RES_value(res); + ZL_DictID dictIDs[] = ZL_DictBundle_dictIDs(bundle); + std::vector localIDsFromFrame; + for (auto idx : localIDsFromFrame) { + ZL_DictID dictID = dictIDs[idx]; + auto dictRes = globalStore.fetchDict(dictID); + ZL_ERR_IF_ERR(dictRes); + ZL_Dict* dict = ZL_RES_value(dictRes); + // do something with the dict + } + } +} From dd8b44fb244c709540cdbbd55d0a92a9d7458185 Mon Sep 17 00:00:00 2001 From: Victor Zhang Date: Mon, 9 Mar 2026 12:03:49 -0700 Subject: [PATCH 2/3] materializer registry API Differential Revision: D95403498 --- tests/unittest/dict/MaterializationE2E.cpp | 53 +++++++++++++++++++ tests/unittest/dict/zl_materializerregistry.h | 27 ++++++++++ 2 files changed, 80 insertions(+) create mode 100644 tests/unittest/dict/MaterializationE2E.cpp create mode 100644 tests/unittest/dict/zl_materializerregistry.h diff --git a/tests/unittest/dict/MaterializationE2E.cpp b/tests/unittest/dict/MaterializationE2E.cpp new file mode 100644 index 000000000..a20991e66 --- /dev/null +++ b/tests/unittest/dict/MaterializationE2E.cpp @@ -0,0 +1,53 @@ +// (c) Meta Platforms, Inc. and affiliates. Confidential and proprietary. + +#include +#include "zl_materializerregistry.h" + +#include "openzl/zl_ctransform.h" +#include "openzl/zl_dtransform.h" + +using namespace ::testing; + +TEST(MaterializationE2E, HowToRegisterANode) +{ + ZL_IDType trid = ZL_IDType{ 67 }; + ZL_MIGraphDesc graphDesc{ + .CTid = trid, + .inputTypes = nullptr, + // other stuff here + }; + ZL_MIEncoderDesc encoderDesc{ + .gd = graphDesc, + .transform_f = nullptr, + .name = "enc", + }; + ZL_MIDecoderDesc decoderDesc{ + .gd = graphDesc, + .transform_f = nullptr, + .name = "dec", + }; + ZL_MaterializerDesc matDesc = { + .codecID = trid, // new field + .materializeFn = nullptr, + .dematerializeFn = nullptr, + }; + + ZL_Compressor* compressor; + ZL_Compressor_registerMIEncoder(compressor, &encoderDesc); + + ZL_DCtx* dctx; + ZL_DCtx_registerMIDecoder(dctx, &decoderDesc); + + ZL_MaterializerRegistry* registry; + ZL_MaterializerRegistry_registerNode(registry, &matDesc); +} + +TEST(MaterializationE2E, HowToUse) +{ + ZL_MaterializerRegistry* registry; // pre-existing + + ZL_DictLoader loader(registry); + ZL_DictID id; + loader->fetchDict(id); // no need to pass a compressor or dctx--the registry + // has all the info to materialize and dematerialize +} diff --git a/tests/unittest/dict/zl_materializerregistry.h b/tests/unittest/dict/zl_materializerregistry.h new file mode 100644 index 000000000..8f1438e45 --- /dev/null +++ b/tests/unittest/dict/zl_materializerregistry.h @@ -0,0 +1,27 @@ +// Copyright (c) Meta Platforms, Inc. and affiliates. + +#ifndef ZSTRONG_TESTS_DICT_MATERIALIZER_H +#define ZSTRONG_TESTS_DICT_MATERIALIZER_H + +#include + +#include "openzl/zl_materializer.h" +#include "openzl/zl_opaque_types.h" + +struct ZL_MaterializerRegistry_s { + std::map registry; +}; + +typedef struct ZL_MaterializerRegistry_s ZL_MaterializerRegistry; + +ZL_Report ZL_MaterializerRegistry_registerNode( + ZL_MaterializerRegistry* registry, + ZL_MaterializerDesc* desc); +void* ZL_MaterializerRegistry_getMaterializer( + ZL_MaterializerRegistry* registry, + ZL_IDType codecId); +void* ZL_MaterializerRegistry_getDematerializer( + ZL_MaterializerRegistry* registry, + ZL_IDType codecId); + +#endif // ZSTRONG_TESTS_DICT_MATERIALIZER_H From a998a51d326c7d688978557a57e53f60f97815c0 Mon Sep 17 00:00:00 2001 From: Victor Zhang Date: Mon, 9 Mar 2026 12:03:49 -0700 Subject: [PATCH 3/3] dictionary API Differential Revision: D95832106 --- include/openzl/zl_dict.h | 120 +++++++++++++++++++++++++++++++ include/openzl/zl_opaque_types.h | 2 + src/openzl/dict/BUCK | 15 ++++ src/openzl/dict/dict.h | 12 ++++ src/openzl/dict/dictloader.h | 10 +++ 5 files changed, 159 insertions(+) create mode 100644 include/openzl/zl_dict.h create mode 100644 src/openzl/dict/BUCK create mode 100644 src/openzl/dict/dict.h create mode 100644 src/openzl/dict/dictloader.h diff --git a/include/openzl/zl_dict.h b/include/openzl/zl_dict.h new file mode 100644 index 000000000..f501fbbe5 --- /dev/null +++ b/include/openzl/zl_dict.h @@ -0,0 +1,120 @@ +// Copyright (c) Meta Platforms, Inc. and affiliates. + +#ifndef OPENZL_ZL_DICT_H +#define OPENZL_ZL_DICT_H + +#include // size_t +#include // uint8_t +#include "openzl/zl_common_types.h" +#include "openzl/zl_errors.h" // ZL_RESULT_OF +#include "openzl/zl_localParams.h" // ZL_LocalParams +#include "openzl/zl_opaque_types.h" // ZL_Dict +#include "openzl/zl_portability.h" // ZL_NOEXCEPT_FUNC_PTR + +#if defined(__cplusplus) +extern "C" { +#endif + +/* + * ================================== ZL_Dict ================================== + * + * The ZL_Dict is a dictionary object for use with both the encoder and decoder. + * Dictionaries are only supported for codecs. Dictionaries are also + * instance-specific. Two copies of the same codec within the same graph can + * have different associated dictionaries. The ZL_Dict APIs are exposed for + * *creating* dictionaries and are likely not what you are looking for. See the + * ZL_DictLoader for how to *use* dictionaries. + */ + +typedef struct { + uint8_t bytes[32]; +} ZL_DictID; + +/** + * Helper function to pack a serialized dict buffer into the ZL_Dict wire + * format. + * @param dst the buffer to write the serialized dict into. The size must be + * larger than the @p src size by at least 60 bytes. + * @param codecID the codec associated with this dict. + * @return on success,the resulting serialized size. Otherwise, an error. + */ +ZL_Report ZL_Dict_pack( + void* dst, + size_t dstCapacity, + ZL_IDType codecID, + const void* src, + size_t srcSize); + +typedef ZL_Dict* ZL_DictPtr; +ZL_RESULT_DECLARE_TYPE(ZL_DictPtr); + +/* + * =============================== ZL_DictBundle =============================== + * + * The ZL_DictBundle contains information about all the ZL_Dict objects required + * by a particular compressor. Note that any individual frame generated by a + * compressor may not require every single dictionary in the bundle to + * (de)compress. However, if it is *possible* for a frame to require a dict + * object to decompress, it must be recorded in the bundle. + */ + +typedef struct { + uint8_t bytes[32]; +} ZL_DictBundleID; + +typedef struct { + ZL_DictBundleID bundleID; + size_t numDicts; + ZL_DictID* dicts; +} ZL_DictBundle; + +typedef ZL_DictBundle* ZL_DictBundlePtr; +ZL_RESULT_DECLARE_TYPE(ZL_DictBundlePtr); + +/** + * Helper function to pack a list of dict IDs into the ZL_DictBundle wire + * format. + * @param dst the buffer to write the serialized bundle into. The size must be + * larger than the byte size of @p dictIDs by at least 48 bytes. + */ +ZL_Report +ZL_DictBundle_serialize(void* dst, size_t dstCapacity, ZL_DictBundle* bundle); + +/* + * =============================== ZL_DictLoader =============================== + * + * The ZL_DictLoader is responsible for fetching and (de)materializing dicts and + * bundles. It is used by both the ZL_Compressor/ZL_CCtx and ZL_DCtx to provide + * dictionary functionality. + * + * The fetching APIs are provided unimplemented to enable you to customize blob + * sources and caching behavior. (A simple reference implementation is also + * provided in ZL_StaticSetDictLoader. This is sufficient for use with a static + * set of known dictionaries and bundles.) + * + * Before usage, individual codec de/materializers must be registered with the + * ZL_DictLoader. + */ +typedef struct { + void* opaque; // optional opaque pointer. The ZL_DictLoader will take + // ownership of this pointer. + void (*freeOpaque)( + void* opaque); // Function to free the opaque pointer, if provided. + /** + * Functions to fetch bundles and dicts. Implementors are expected to use + * the TODO helper functions to parse raw blobs. Implementors are encouraged + * to cache bundles and dicts using a custom state stored in @p opaque . + */ + ZL_RESULT_OF(ZL_DictBundlePtr) ( + *fetchDictBundle)(void* opaque, const ZL_DictBundleID* id); + ZL_RESULT_OF(ZL_DictPtr) (*fetchDict)(void* opaque, const ZL_DictID* id); +} ZL_DictLoaderDesc; + +ZL_DictLoader* ZL_DictLoader_create(const ZL_DictLoaderDesc* desc); +void ZL_DictLoader_free(ZL_DictLoader* loader); + +#if defined(__cplusplus) +} // extern "C" +#endif + +#endif // OPENZL_ZL_DICT_H diff --git a/include/openzl/zl_opaque_types.h b/include/openzl/zl_opaque_types.h index a75b06997..d2bac3761 100644 --- a/include/openzl/zl_opaque_types.h +++ b/include/openzl/zl_opaque_types.h @@ -62,6 +62,8 @@ typedef struct ZL_Selector_s ZL_Selector; typedef struct ZL_Graph_s ZL_Graph; typedef struct ZL_Edge_s ZL_Edge; typedef struct ZL_Segmenter_s ZL_Segmenter; +typedef struct ZL_Dict_s ZL_Dict; +typedef struct ZL_DictLoader_s ZL_DictLoader; // Generic List construction macro (C99) #define ZL_LIST_SIZE(_type, ...) \ diff --git a/src/openzl/dict/BUCK b/src/openzl/dict/BUCK new file mode 100644 index 000000000..09415e8ce --- /dev/null +++ b/src/openzl/dict/BUCK @@ -0,0 +1,15 @@ +load("@fbcode_macros//build_defs:cpp_library.bzl", "cpp_library") + +oncall("fill in oncall here @nocommit") + +cpp_library( + name = "dict", + srcs = ["dict.h"], + labels = ["autodeps2_generated"], +) + +cpp_library( + name = "dictloader", + srcs = ["dictloader.h"], + labels = ["autodeps2_generated"], +) diff --git a/src/openzl/dict/dict.h b/src/openzl/dict/dict.h new file mode 100644 index 000000000..fd1ddd454 --- /dev/null +++ b/src/openzl/dict/dict.h @@ -0,0 +1,12 @@ +// Copyright (c) Meta Platforms, Inc. and affiliates. + +#ifndef OPENZL_DICT_DICT_H +#define OPENZL_DICT_DICT_H + +struct ZL_Dict_s { + ZL_DictID dictID; + ZL_IDType materializingCodec; + void* dictObj; +} + +#endif // OPENZL_DICT_DICT_H diff --git a/src/openzl/dict/dictloader.h b/src/openzl/dict/dictloader.h new file mode 100644 index 000000000..34496c871 --- /dev/null +++ b/src/openzl/dict/dictloader.h @@ -0,0 +1,10 @@ +// Copyright (c) Meta Platforms, Inc. and affiliates. + +#ifndef OPENZL_DICT_DICTLOADER_H +#define OPENZL_DICT_DICTLOADER_H + +struct ZL_DictLoader_s { + void* todo; +} + +#endif // OPENZL_DICT_DICTLOADER_H