From 51100d946d445304a86eb4852a59d51adcaf0aa0 Mon Sep 17 00:00:00 2001 From: Victor Zhang Date: Tue, 7 Apr 2026 11:18:32 -0700 Subject: [PATCH 1/2] Add ~16MB auto-chunking segmenter to XLDB int graphs Summary: Add a segmenter to all XLDB numeric graphs (U8/U16/U32/U64/F32) that auto-chunks input at ~16MB boundaries, aligned to element width. This is implemented inside makeXldbGenericIntCGraphImpl so it applies to all int/float graph variants automatically. Fallback (ZSTD) and Store graphs are not segmented. There is an existing numeric segmenter standard graph, but we can't use it because it doesn't allow us to specify the chunk size or inner graph. The next diff in this stack will add support in the numeric segmenter. When a new OpenZL release is issued, we will migrate to the numeric segmenter. Differential Revision: D99729801 --- tests/datagen/distributions/VecLengthDistribution.h | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/tests/datagen/distributions/VecLengthDistribution.h b/tests/datagen/distributions/VecLengthDistribution.h index 784b0c127..39f170474 100644 --- a/tests/datagen/distributions/VecLengthDistribution.h +++ b/tests/datagen/distributions/VecLengthDistribution.h @@ -16,7 +16,7 @@ class VecLengthDistribution : public Distribution { explicit VecLengthDistribution( std::shared_ptr generator, size_t min, - size_t max = kMaxVecLength) + size_t max = (size_t)1u << 17) : Distribution(generator), min_(min), max_(max) { if (min > max) { @@ -57,8 +57,6 @@ class VecLengthDistribution : public Distribution { } private: - static constexpr size_t kMaxVecLength = (size_t)1u << 17; - const size_t min_; const size_t max_; }; From f84df91d73dcd392c35a4af5d194ee9fd458ab73 Mon Sep 17 00:00:00 2001 From: Victor Zhang Date: Tue, 7 Apr 2026 11:22:28 -0700 Subject: [PATCH 2/2] Make numeric segmenter chunk size and head graph configurable (#603) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Summary: Pull Request resolved: https://github.com/facebook/openzl/pull/603 Make the numeric segmenter accept local params for the chunk byte size and head graph, replacing the previously hardcoded values. - Chunk size: read via ZL_Segmenter_getLocalIntParam() using ZL_SEGM_NUMERIC_CHUNK_BYTE_SIZE_MAX_PID. Falls back to 16MB default if not provided. - Head graph: read via ZL_Segmenter_getCustomGraphs(). Falls back to ZL_GRAPH_NUMERIC_COMPRESS if no custom graph is provided. Both changes are backward-compatible — existing callers that don't provide these params get identical behavior to before. Differential Revision: D99729917 --- .../compress/segmenters/segmenter_numeric.c | 24 +++++++++++++------ .../compress/segmenters/segmenter_numeric.h | 2 ++ 2 files changed, 19 insertions(+), 7 deletions(-) diff --git a/src/openzl/compress/segmenters/segmenter_numeric.c b/src/openzl/compress/segmenters/segmenter_numeric.c index e1755f5d5..86651b574 100644 --- a/src/openzl/compress/segmenters/segmenter_numeric.c +++ b/src/openzl/compress/segmenters/segmenter_numeric.c @@ -3,6 +3,7 @@ #include "openzl/compress/segmenters/segmenter_numeric.h" #include "openzl/common/assertion.h" #include "openzl/compress/private_nodes.h" +#include "openzl/zl_selector.h" // ZL_LP_INVALID_PARAMID ...which is probably not where this constant should be ZL_Report SEGM_numeric(ZL_Segmenter* sctx) { @@ -14,14 +15,23 @@ ZL_Report SEGM_numeric(ZL_Segmenter* sctx) size_t const width = ZL_Input_eltWidth(input); ZL_ASSERT(width == 1 || width == 2 || width == 4 || width == 8); - // Note: Currently, static chunk size. - // Tomorrow: global parameter, then local parameter. - size_t const chunkByteSizeMax = 16 << 20; - size_t const chunkEltSizeMax = chunkByteSizeMax / width; + ZL_IntParam const chunkParam = ZL_Segmenter_getLocalIntParam( + sctx, ZL_SEGM_NUMERIC_CHUNK_BYTE_SIZE_MAX_PID); + size_t const chunkByteSizeMax = + (chunkParam.paramId != ZL_LP_INVALID_PARAMID) + ? (size_t)chunkParam.paramValue + : (16 << 20) /* default to 16MB */; + ZL_ERR_IF_LT( + chunkByteSizeMax, + width, + nodeParameter_invalid, + "chunk size must produce at least one element"); + size_t const chunkEltSizeMax = chunkByteSizeMax / width; - // Note: Currently, static head graph. - // Tomorrow: selectable - ZL_GraphID const headGraph = ZL_GRAPH_NUMERIC_COMPRESS; + ZL_GraphIDList const customGraphs = ZL_Segmenter_getCustomGraphs(sctx); + ZL_GraphID const headGraph = (customGraphs.nbGraphIDs >= 1) + ? customGraphs.graphids[0] + : ZL_GRAPH_NUMERIC_COMPRESS; size_t numElts = ZL_Input_numElts(input); while (numElts > chunkEltSizeMax) { diff --git a/src/openzl/compress/segmenters/segmenter_numeric.h b/src/openzl/compress/segmenters/segmenter_numeric.h index 90303d1b3..7efa58804 100644 --- a/src/openzl/compress/segmenters/segmenter_numeric.h +++ b/src/openzl/compress/segmenters/segmenter_numeric.h @@ -10,6 +10,8 @@ ZL_BEGIN_C_DECLS ZL_Report SEGM_numeric(ZL_Segmenter* sctx); +#define ZL_SEGM_NUMERIC_CHUNK_BYTE_SIZE_MAX_PID 1 + #define SEGM_NUMERIC_DESC \ { \ .name = "!zl.segmenter_numeric", \