Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
34 commits
Select commit Hold shift + click to select a range
6a89390
igzip: fix raw deflate over-consumption via read_in_length
asonje Mar 20, 2026
93f144e
rename trailer_overconsumption_fixed -> read_in_correction_applied; r…
asonje Mar 20, 2026
92eaf59
add IGZIP inflate/deflate counters to statistics; enable ENABLE_STATI…
asonje Mar 20, 2026
b48d118
add iaa_fallback_igzip: IAA can fall back to IGZIP before software zlib
asonje Mar 20, 2026
0a857e6
cmake: set ENABLE_STATISTICS=OFF by default
asonje Mar 20, 2026
1971197
- Reset statistics to off by default and formatting
asonje Mar 20, 2026
f7ee0ec
IGZIP: lift Z_FINISH-only restriction; enable full streaming compress
asonje Apr 3, 2026
ccd3d6c
fix iaa_fallback_igzip compress: set path_selected=IGZIP after fallback
asonje Apr 14, 2026
37fdc50
iaa: replace marker-based IsIAADecompressible with 512-byte threshold
asonje Apr 14, 2026
fd6ebbb
README: document USE_IGZIP cmake option and ISA-L dependency
asonje Apr 30, 2026
0366ddd
zlib_accel: guard pre_avail_in declaration with USE_IGZIP
asonje Apr 30, 2026
07338b4
Clang format
asonje Apr 30, 2026
8c1bf9a
remove cmake.txt; fix null deref in deflateSetDictionary
asonje May 4, 2026
dd02236
igzip: remove dead SupportedOptions/IGZIPShouldFallback stubs
asonje May 4, 2026
8ba37be
zlib_accel: fix read_in_correction_applied semantics and reset asymmetry
asonje May 4, 2026
cd7cb91
iaa: deprecate iaa_prepend_empty_block config option
asonje May 4, 2026
db0a00f
tests: add IAA->IGZIP fallback coverage
asonje May 4, 2026
a75858e
tests: restore IGZIP path assertions for SYNC_FLUSH regression tests
asonje May 4, 2026
6ef5f24
igzip: document read_in_correction_applied reset and ZSTATE_NEW_HDR c…
asonje May 4, 2026
8f63d4b
address round-2 Copilot review: README doc, igzip.h comment, IGZIP st…
asonje May 5, 2026
612c560
fix: restore gzip_flag after deflateReset on IGZIP stream (Cassandra …
asonje May 6, 2026
577fbfa
Merge branch 'intel:igzip' into igzip
asonje Jun 2, 2026
8120b29
Rename iaa_fallback_igzip to igzip_fallback, extend to QAT, default=1
asonje Jun 2, 2026
ecd1420
fix: build error when USE_IGZIP + USE_IAA/USE_QAT all enabled
asonje Jun 2, 2026
c042d3c
tests: assert return codes and guard QAT->IGZIP tests with USE_IGZIP
asonje Jun 2, 2026
eb89dd3
fixup: use const char* ternaries to identify accelerator in fallback …
asonje Jun 3, 2026
65da917
Merge branch 'intel:igzip' into igzip
asonje Jul 1, 2026
34e2b0a
igzip: remove workarounds for ISA-L bugs 1–3, 5 (fixed in public v2.3…
asonje Jul 1, 2026
827ae09
cmake: require ISA-L >= 2.32.1 when USE_IGZIP=ON
asonje Jul 1, 2026
381c6bf
format: apply clang-format
asonje Jul 1, 2026
5e95437
address Copilot review comments on PR #60
asonje Jul 6, 2026
8e2b8c4
cmake,igzip: address round-3 Copilot review comments on PR #60
asonje Jul 7, 2026
baf06f5
igzip: address round-4 vkarpenk review comments on PR #60
asonje Jul 9, 2026
348932e
Formatted code
asonje Jul 9, 2026
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
53 changes: 52 additions & 1 deletion common.cmake
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,58 @@ if(USE_IAA)
endif()

if(USE_IGZIP)
add_compile_definitions(USE_IGZIP)
add_compile_definitions(USE_IGZIP)

# Enforce minimum ISA-L version. v2.32.1 contains critical igzip bugfixes:
# - next_in/avail_in not reset after block finish (inflate boundary bug)
# - zlib DICTID stored/read in wrong byte order
# - gzip_flag mutated by isal_deflate when writing wrapper header
# Earlier versions require defensive workarounds that have been removed.
if(ISAL_PATH)
set(_isal_api_header "${ISAL_PATH}/include/isal_api.h")
else()
find_path(_isal_api_include isal_api.h PATH_SUFFIXES include)
if(_isal_api_include)
set(_isal_api_header "${_isal_api_include}/isal_api.h")
endif()
unset(_isal_api_include CACHE)
endif()
if(NOT _isal_api_header OR NOT EXISTS "${_isal_api_header}")
message(FATAL_ERROR "ISA-L header (isal_api.h) not found. Set -DISAL_PATH=<path> or install ISA-L system-wide.")
endif()
file(STRINGS "${_isal_api_header}" _isal_major_line REGEX "^#define[ \t]+ISAL_MAJOR_VERSION[ \t]+")
file(STRINGS "${_isal_api_header}" _isal_minor_line REGEX "^#define[ \t]+ISAL_MINOR_VERSION[ \t]+")
file(STRINGS "${_isal_api_header}" _isal_patch_line REGEX "^#define[ \t]+ISAL_PATCH_VERSION[ \t]+")
if(NOT _isal_major_line OR NOT _isal_minor_line OR NOT _isal_patch_line)
message(FATAL_ERROR "Could not find ISAL_MAJOR/MINOR/PATCH_VERSION macros in ${_isal_api_header}. ISA-L header may be malformed or too old.")
endif()
list(GET _isal_major_line 0 _isal_major_line)
list(GET _isal_minor_line 0 _isal_minor_line)
list(GET _isal_patch_line 0 _isal_patch_line)
string(REGEX REPLACE "^#define[ \t]+ISAL_MAJOR_VERSION[ \t]+([0-9]+).*" "\\1" _isal_major "${_isal_major_line}")
string(REGEX REPLACE "^#define[ \t]+ISAL_MINOR_VERSION[ \t]+([0-9]+).*" "\\1" _isal_minor "${_isal_minor_line}")
string(REGEX REPLACE "^#define[ \t]+ISAL_PATCH_VERSION[ \t]+([0-9]+).*" "\\1" _isal_patch "${_isal_patch_line}")
foreach(_v _isal_major _isal_minor _isal_patch)
if(NOT "${${_v}}" MATCHES "^[0-9]+$")
message(FATAL_ERROR "Failed to parse ISA-L version from ${_isal_api_header}: '${_v}' = '${${_v}}'")
endif()
endforeach()
set(_isal_version_found "${_isal_major}.${_isal_minor}.${_isal_patch}")
message(STATUS "Found ISA-L version: ${_isal_version_found} (${_isal_api_header})")
if(_isal_version_found VERSION_LESS "2.32.1")
message(FATAL_ERROR
"ISA-L >= 2.32.1 required, found ${_isal_version_found}.\n"
"Update ISA-L or pass -DISAL_PATH=<path> to cmake.\n"
"Release: https://github.com/intel/isa-l/releases/tag/v2.32.1")
endif()
unset(_isal_api_header)
unset(_isal_major_line)
unset(_isal_minor_line)
unset(_isal_patch_line)
unset(_isal_major)
unset(_isal_minor)
unset(_isal_patch)
unset(_isal_version_found)
endif()

if(USE_QAT)
Expand Down
142 changes: 36 additions & 106 deletions igzip.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -120,14 +120,12 @@ struct isal_zstream *InitCompressIGZIP(int level, int windowBits) {
return isal_strm;
}

int CompressIGZIP(struct isal_zstream *isal_strm, int flush, uint8_t *input,
uint32_t *input_length, uint8_t *output,
uint32_t *output_length, unsigned long *total_in,
unsigned long *total_out) {
int CompressIGZIP(struct isal_zstream *isal_strm, int flush,
const uint8_t *input, uint32_t *input_length, uint8_t *output,
uint32_t *output_length, const unsigned long *total_in,
const unsigned long *total_out) {
int ret;

(void)total_in;
(void)total_out;
if (!isal_strm) {
Log(LogLevel::LOG_ERROR, "CompressIGZIP() Line ", __LINE__,
" deflate isal_strm is NULL\n");
Expand All @@ -138,7 +136,7 @@ int CompressIGZIP(struct isal_zstream *isal_strm, int flush, uint8_t *input,
isal_strm->next_out = output;
const uint32_t original_avail_out = *output_length;
isal_strm->avail_out = original_avail_out;
isal_strm->next_in = input;
isal_strm->next_in = const_cast<uint8_t *>(input);
const uint32_t original_avail_in = *input_length;
isal_strm->avail_in = original_avail_in;
isal_strm->total_out = *total_out;
Expand Down Expand Up @@ -303,21 +301,16 @@ struct inflate_state *InitUncompressIGZIP(int windowBits) {

isal_strm_inflate->avail_in = 0;
isal_strm_inflate->next_in = NULL;
// strm->total_out = 0;
// strm->total_in = 0;

// s->read_in_correction_applied = 0;

ConfigureInflateWindow(isal_strm_inflate, windowBits);

return isal_strm_inflate;
}

IGZIPNoInputAction IGZIPHandleActiveStreamNoInput(
z_streamp strm, struct inflate_state *isal_strm_inflate, int window_bits,
int *read_in_correction_applied, int *ret) {
z_streamp strm, struct inflate_state *isal_strm_inflate, int *ret) {
if (strm == nullptr || isal_strm_inflate == nullptr || ret == nullptr ||
read_in_correction_applied == nullptr || strm->avail_in != 0) {
strm->avail_in != 0) {
return IGZIP_NO_INPUT_NOT_HANDLED;
}

Expand All @@ -326,22 +319,17 @@ IGZIPNoInputAction IGZIPHandleActiveStreamNoInput(
bool end_of_stream = true;

*ret = UncompressIGZIP(isal_strm_inflate, strm->next_in, &input_len,
strm->next_out, &output_len, window_bits,
read_in_correction_applied, &strm->total_in,
strm->next_out, &output_len, &strm->total_in,
&strm->total_out, &end_of_stream);

if (*ret == Z_DATA_ERROR) {
Log(LogLevel::LOG_INFO, "IGZIPHandleActiveStreamNoInput() Line ", __LINE__,
" requested zlib fallback for raw INPUT_DONE ambiguity\n");
return IGZIP_NO_INPUT_FALLBACK_ZLIB;
}

if (*ret == 0) {
strm->next_out += output_len;
strm->avail_out -= output_len;
Comment on lines 325 to 327
strm->total_out += output_len;
if (output_len > 0) {
*ret = end_of_stream ? Z_STREAM_END : Z_OK;
if (end_of_stream) {
*ret = Z_STREAM_END;
} else if (output_len > 0) {
*ret = Z_OK;
} else {
*ret = Z_BUF_ERROR;
}
Expand All @@ -354,12 +342,11 @@ IGZIPNoInputAction IGZIPHandleActiveStreamNoInput(

IGZIPInflatePathAction IGZIPRunInflateAndSelectPathAction(
z_streamp strm, struct inflate_state **isal_strm_inflate, int window_bits,
int *read_in_correction_applied, uint32_t *input_length,
uint32_t *output_length, int *ret, bool *end_of_stream,
uint32_t pre_avail_in) {
uint32_t *input_length, uint32_t *output_length, int *ret,
bool *end_of_stream) {
if (strm == nullptr || isal_strm_inflate == nullptr ||
input_length == nullptr || output_length == nullptr || ret == nullptr ||
end_of_stream == nullptr || read_in_correction_applied == nullptr) {
end_of_stream == nullptr) {
if (ret != nullptr) {
*ret = Z_DATA_ERROR;
}
Expand All @@ -377,25 +364,12 @@ IGZIPInflatePathAction IGZIPRunInflateAndSelectPathAction(
}

*ret = UncompressIGZIP(*isal_strm_inflate, strm->next_in, input_length,
strm->next_out, output_length, window_bits,
read_in_correction_applied, &strm->total_in,
strm->next_out, output_length, &strm->total_in,
&strm->total_out, end_of_stream);

const uint32_t remaining_after_igzip =
(pre_avail_in >= *input_length) ? (pre_avail_in - *input_length) : 0;

if (*ret == 0 && window_bits < 0 && *end_of_stream &&
remaining_after_igzip > 0 && *read_in_correction_applied == 0 &&
strm->total_in == 0 && strm->total_out == 0) {
Log(LogLevel::LOG_ERROR,
"IGZIPRunInflateAndSelectPathAction() raw boundary guard FIRED strm=",
static_cast<void *>(strm), " bytes_in=", *input_length,
" bytes_out=", *output_length, " pre_avail_in=", pre_avail_in,
" remaining_in=", remaining_after_igzip, "\n");
*ret = 1;
*end_of_stream = false;
return IGZIP_INFLATE_PATH_FALLBACK_RAW_BOUNDARY;
}
// Raw boundary guard removed: ISA-L PR#215 (cd72fd7) fixes avail_in
// over-consumption at BLOCK_FINISH in isal_inflate; the guard is no
// longer needed for that case.

if (*ret == Z_NEED_DICT) {
return IGZIP_INFLATE_PATH_FALLBACK_NEED_DICT;
Expand All @@ -410,11 +384,11 @@ IGZIPInflatePathAction IGZIPRunInflateAndSelectPathAction(
return IGZIP_INFLATE_PATH_NONE;
}

int UncompressIGZIP(struct inflate_state *isal_strm_inflate, uint8_t *input,
uint32_t *input_length, uint8_t *output,
uint32_t *output_length, int window_bits,
int *read_in_correction_applied, unsigned long *total_in,
unsigned long *total_out, bool *end_of_stream) {
int UncompressIGZIP(struct inflate_state *isal_strm_inflate,
const uint8_t *input, uint32_t *input_length,
uint8_t *output, uint32_t *output_length,
const unsigned long *total_in,
const unsigned long *total_out, bool *end_of_stream) {
(void)total_in;

if (!isal_strm_inflate) {
Expand All @@ -428,7 +402,7 @@ int UncompressIGZIP(struct inflate_state *isal_strm_inflate, uint8_t *input,
isal_strm_inflate->avail_out = original_avail_out;
const uint32_t original_avail_in = *input_length;
isal_strm_inflate->avail_in = original_avail_in;
isal_strm_inflate->next_in = input;
isal_strm_inflate->next_in = const_cast<uint8_t *>(input);
isal_strm_inflate->total_out = *total_out;

const int decomp = isal_inflate(isal_strm_inflate);
Expand All @@ -444,48 +418,14 @@ int UncompressIGZIP(struct inflate_state *isal_strm_inflate, uint8_t *input,
consumed_before_adjust = 0;
}

uint32_t rewind_adjust_bytes = 0;

// WORKAROUND: ISA-L raw-deflate over-consumption fix.
// ISAL pre-loads input in 8-byte word chunks into a 64-bit shift register
// (read_in). After BLOCK_FINISH, read_in_length >> 3 is the exact byte
// count over-consumed, covering all avail_in scenarios: [0], [1,7], [8],
// and >8 (multi-frame), where prior heuristics were blind or inaccurate.
if (window_bits < 0 &&
(decomp == ISAL_DECOMP_OK || decomp == ISAL_END_INPUT) &&
isal_strm_inflate->block_state == ISAL_BLOCK_FINISH) {
const uint32_t read_in_correction =
(isal_strm_inflate->read_in_length > 0)
? static_cast<uint32_t>(isal_strm_inflate->read_in_length >> 3)
: 0u;
Log(LogLevel::LOG_INFO, "UncompressIGZIP() Line ", __LINE__,
" raw_finish avail_in ", isal_strm_inflate->avail_in,
" read_in_length_bits ", isal_strm_inflate->read_in_length,
" read_in_correction_bytes ", read_in_correction, "\n");
if (read_in_correction > 0) {
rewind_adjust_bytes = (read_in_correction <= consumed_before_adjust)
? read_in_correction
: consumed_before_adjust;
*read_in_correction_applied = 1;
}
}

// WORKAROUND: BLOCK_INPUT_DONE — output-buffer-limited with ambiguous
// trailer bytes. read_in_length does not apply here (not yet at
// BLOCK_FINISH); request caller fallback to zlib. BLOCK_FINISH is fully
// handled above.
if (window_bits < 0 && decomp == ISAL_DECOMP_OK &&
*read_in_correction_applied == 0 &&
isal_strm_inflate->block_state == ISAL_BLOCK_INPUT_DONE &&
isal_strm_inflate->avail_in < 8 && isal_strm_inflate->avail_in > 0) {
Log(LogLevel::LOG_INFO, "UncompressIGZIP() Line ", __LINE__,
" raw INPUT_DONE ambiguity detected: over_consumed ",
8u - isal_strm_inflate->avail_in, ", requesting zlib fallback\n");
return Z_DATA_ERROR;
}
// Bug 2 guard removed: ISA-L PR#215 (cd72fd7) makes BLOCK_FINISH correctly
// restore avail_in after over-consumption into read_in. Continuing past
// BLOCK_INPUT_DONE (even with avail_in 1-7) reaches BLOCK_FINISH with the
// correct stream boundary. No fallback needed.
// (ISA-L >= 2.32.1 enforced at configure time via common.cmake.)

*output_length = original_avail_out - isal_strm_inflate->avail_out;
*input_length = consumed_before_adjust - rewind_adjust_bytes;
*input_length = consumed_before_adjust;
input = isal_strm_inflate->next_in;
output = isal_strm_inflate->next_out;

Expand Down Expand Up @@ -570,33 +510,23 @@ int inflateSetDictionary(z_streamp strm, unsigned char *dict_data,
return isal_inflate_set_dict(s->isal_strm_inflate, dict_data, dict_len);
}

void ResetCompressIGZIP(struct isal_zstream *isal_strm, int windowBits) {
// isal_deflate_reset preserves gzip_flag, hist_bits, level, and level_buf.
// gzip_flag must be restored: after the first chunk ISA-L changes it from
// IGZIP_ZLIB (3) to IGZIP_ZLIB_NO_HDR (4) to suppress the header on
// continuation calls. Without this reset, the next stream reused via
// deflateReset would produce headerless output, causing decompressors
// (e.g. Java Inflater with nowrap=false) to reject every subsequent chunk.
void ResetCompressIGZIP(struct isal_zstream *isal_strm) {
// ISA-L PR#215 (30e90b4) stops gzip_flag from being mutated during
// compression, so isal_deflate_reset preserves it correctly. No manual
// ConfigureDeflateWindow call is needed here.
isal_deflate_reset(isal_strm);
isal_strm->end_of_stream = 0;
isal_strm->flush = NO_FLUSH;
ConfigureDeflateWindow(isal_strm, windowBits);
}

int ResetUncompressIGZIP(struct inflate_state *isal_strm_inflate,
int *read_in_correction_applied) {
int ResetUncompressIGZIP(struct inflate_state *isal_strm_inflate) {
if (!isal_strm_inflate) {
Log(LogLevel::LOG_ERROR, "ResetUncompressIGZIP() Line ", __LINE__,
" isal_strm_inflate is NULL\n");
return Z_STREAM_ERROR;
}

// Reset ISA-L inflate state. isal_inflate_reset clears the internal
// read_in / read_in_length buffer, so any over-consumption correction
// applied during the previous stream session no longer applies.
// Clear the flag so the new session fires the correction fresh if needed.
isal_inflate_reset(isal_strm_inflate);
*read_in_correction_applied = 0;

return Z_OK;
}
Expand Down
36 changes: 14 additions & 22 deletions igzip.h
Original file line number Diff line number Diff line change
Expand Up @@ -12,9 +12,6 @@ typedef struct internal_state2 {
int level;
int w_bits;
struct inflate_state *isal_strm_inflate;
int read_in_correction_applied; /* Set once per stream session when the
read_in_length over-consumption correction
fires; cleared only on inflateReset */
} inflate_state2;

typedef struct internal_state {
Expand All @@ -25,46 +22,41 @@ typedef struct internal_state {
} deflate_state;

struct isal_zstream *InitCompressIGZIP(int level, int windowBits);
int CompressIGZIP(struct isal_zstream *isal_strm, int flush, uint8_t *input,
uint32_t *input_length, uint8_t *output,
uint32_t *output_length, unsigned long *total_in,
unsigned long *total_out);
int CompressIGZIP(struct isal_zstream *isal_strm, int flush,
const uint8_t *input, uint32_t *input_length, uint8_t *output,
uint32_t *output_length, const unsigned long *total_in,
const unsigned long *total_out);
bool IsIGZIPDeflateFinished(const struct isal_zstream *stream);
enum IGZIPNoInputAction {
IGZIP_NO_INPUT_NOT_HANDLED,
IGZIP_NO_INPUT_RETURN,
IGZIP_NO_INPUT_FALLBACK_ZLIB,
};

enum IGZIPInflatePathAction {
IGZIP_INFLATE_PATH_NONE,
IGZIP_INFLATE_PATH_SET_IGZIP,
IGZIP_INFLATE_PATH_FALLBACK_NEED_DICT,
IGZIP_INFLATE_PATH_FALLBACK_DATA_ERROR,
IGZIP_INFLATE_PATH_FALLBACK_RAW_BOUNDARY,
};

IGZIPNoInputAction IGZIPHandleActiveStreamNoInput(
z_streamp strm, struct inflate_state *isal_strm_inflate, int window_bits,
int *read_in_correction_applied, int *ret);
z_streamp strm, struct inflate_state *isal_strm_inflate, int *ret);

IGZIPInflatePathAction IGZIPRunInflateAndSelectPathAction(
z_streamp strm, struct inflate_state **isal_strm_inflate, int window_bits,
int *read_in_correction_applied, uint32_t *input_length,
uint32_t *output_length, int *ret, bool *end_of_stream,
uint32_t pre_avail_in);
uint32_t *input_length, uint32_t *output_length, int *ret,
bool *end_of_stream);

int EndCompressIGZIP(struct isal_zstream *isal_strm);
void ResetCompressIGZIP(struct isal_zstream *isal_strm, int windowBits);
void ResetCompressIGZIP(struct isal_zstream *isal_strm);

struct inflate_state *InitUncompressIGZIP(int windowBits);
int UncompressIGZIP(struct inflate_state *isal_strm_inflate, uint8_t *input,
uint32_t *input_length, uint8_t *output,
uint32_t *output_length, int window_bits,
int *read_in_correction_applied, unsigned long *total_in,
unsigned long *total_out, bool *end_of_stream);
int UncompressIGZIP(struct inflate_state *isal_strm_inflate,
const uint8_t *input, uint32_t *input_length,
uint8_t *output, uint32_t *output_length,
const unsigned long *total_in,
const unsigned long *total_out, bool *end_of_stream);
int EndUncompressIGZIP(struct inflate_state *isal_strm_inflate);
int ResetUncompressIGZIP(struct inflate_state *isal_strm_inflate,
int *read_in_correction_applied);
int ResetUncompressIGZIP(struct inflate_state *isal_strm_inflate);
// #define Z_DEFAULT_COMPRESSION 6
#endif
Loading