Skip to content

Make numeric segmenter chunk size and head graph configurable (#603) #603

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
Victor-C-Zhang wants to merge 2 commits into
base: dev
Choose a base branch
from
Open

Make numeric segmenter chunk size and head graph configurable (#603) #603

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
51100d9
Add ~16MB auto-chunking segmenter to XLDB int graphs
Apr 7, 2026
f84df91
Make numeric segmenter chunk size and head graph configurable (#603)
Victor-C-Zhang Apr 7, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
24 changes: 17 additions & 7 deletions src/openzl/compress/segmenters/segmenter_numeric.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@
#include "openzl/compress/segmenters/segmenter_numeric.h"
#include "openzl/common/assertion.h"
#include "openzl/compress/private_nodes.h"
#include "openzl/zl_selector.h" // ZL_LP_INVALID_PARAMID ...which is probably not where this constant should be

ZL_Report SEGM_numeric(ZL_Segmenter* sctx)
{
Expand All @@ -14,14 +15,23 @@ ZL_Report SEGM_numeric(ZL_Segmenter* sctx)
size_t const width = ZL_Input_eltWidth(input);
ZL_ASSERT(width == 1 || width == 2 || width == 4 || width == 8);

// Note: Currently, static chunk size.
// Tomorrow: global parameter, then local parameter.
size_t const chunkByteSizeMax = 16 << 20;
size_t const chunkEltSizeMax = chunkByteSizeMax / width;
ZL_IntParam const chunkParam = ZL_Segmenter_getLocalIntParam(
sctx, ZL_SEGM_NUMERIC_CHUNK_BYTE_SIZE_MAX_PID);
size_t const chunkByteSizeMax =
(chunkParam.paramId != ZL_LP_INVALID_PARAMID)
? (size_t)chunkParam.paramValue
: (16 << 20) /* default to 16MB */;
ZL_ERR_IF_LT(
chunkByteSizeMax,
width,
nodeParameter_invalid,
"chunk size must produce at least one element");
size_t const chunkEltSizeMax = chunkByteSizeMax / width;

// Note: Currently, static head graph.
// Tomorrow: selectable
ZL_GraphID const headGraph = ZL_GRAPH_NUMERIC_COMPRESS;
ZL_GraphIDList const customGraphs = ZL_Segmenter_getCustomGraphs(sctx);
ZL_GraphID const headGraph = (customGraphs.nbGraphIDs >= 1)
? customGraphs.graphids[0]
: ZL_GRAPH_NUMERIC_COMPRESS;

size_t numElts = ZL_Input_numElts(input);
while (numElts > chunkEltSizeMax) {
Expand Down
2 changes: 2 additions & 0 deletions src/openzl/compress/segmenters/segmenter_numeric.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,8 @@ ZL_BEGIN_C_DECLS

ZL_Report SEGM_numeric(ZL_Segmenter* sctx);

#define ZL_SEGM_NUMERIC_CHUNK_BYTE_SIZE_MAX_PID 1

#define SEGM_NUMERIC_DESC \
{ \
.name = "!zl.segmenter_numeric", \
Expand Down
4 changes: 1 addition & 3 deletions tests/datagen/distributions/VecLengthDistribution.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,7 @@ class VecLengthDistribution : public Distribution<size_t> {
explicit VecLengthDistribution(
std::shared_ptr<RandWrapper> generator,
size_t min,
size_t max = kMaxVecLength)
size_t max = (size_t)1u << 17)
: Distribution<size_t>(generator), min_(min), max_(max)
{
if (min > max) {
Expand Down Expand Up @@ -57,8 +57,6 @@ class VecLengthDistribution : public Distribution<size_t> {
}

private:
static constexpr size_t kMaxVecLength = (size_t)1u << 17;

const size_t min_;
const size_t max_;
};
Expand Down
Loading