diff --git a/.gitignore b/.gitignore index fc1dadd..aedc64f 100644 --- a/.gitignore +++ b/.gitignore @@ -1,2 +1,5 @@ /build/ -/tests/build \ No newline at end of file +/tests/build +cmake.txt +/resources/ + diff --git a/CMakeLists.txt b/CMakeLists.txt index 1b36c5e..445c160 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -15,7 +15,7 @@ if(ENABLE_STATISTICS) add_compile_definitions(ENABLE_STATISTICS) endif() -add_library(${PROJECT_NAME} SHARED config/config_reader.cpp config/config.cpp zlib_accel.cpp iaa.cpp qat.cpp utils.cpp statistics.cpp) +add_library(${PROJECT_NAME} SHARED config/config_reader.cpp config/config.cpp zlib_accel.cpp iaa.cpp qat.cpp utils.cpp statistics.cpp igzip.cpp) add_custom_target(format find .. -iname '*.h' -o -iname '*.cpp' | xargs clang-format -style=Google -i diff --git a/README.md b/README.md index 4ed2712..496a659 100644 --- a/README.md +++ b/README.md @@ -59,8 +59,10 @@ make CMake supports the following options: - USE_QAT (ON/OFF): include QAT acceleration - USE_IAA (ON/OFF): include IAA acceleration +- USE_IGZIP (ON/OFF): include IGZIP acceleration (requires ISA-L) - QPL_PATH: path to QPL for IAA acceleration (if not in a standard directory) - QATZIP_PATH: path to QATzip for QAT acceleration (if not in a standard directory) +- ISAL_PATH: path to ISA-L for IGZIP acceleration (if not in a standard directory) - DEBUG_LOG (ON/OFF): enable logging - ENABLE_STATISTICS (ON/OFF): enable statistics - COVERAGE (ON/OFF): enable test coverage (more details in a later section) @@ -84,6 +86,9 @@ Requirements for IAA - [accel-config](https://github.com/intel/idxd-config) - [Query Processing Library](https://github.com/intel/qpl) +Requirements for IGZIP +- [ISA-L (Intel Intelligent Storage Acceleration Library)](https://github.com/intel/isa-l) + A setup with both QAT and IAA enabled has been tested on an AWS m7i.metal-24xl instance (Ubuntu 22.04, kernel 6.8.0). Refer to the links above for instructions on how to install the dependencies. @@ -178,20 +183,22 @@ use_zlib_uncompress - Enable zlib for decompression - Setting to 1 is recommended, to allow fall back to zlib in case accelerators cannot be used or experience an error. +igzip_fallback +- Values: 0,1. Default: 1 +- If 1, and an IAA or QAT compression/decompression operation fails, the request is retried using IGZIP (if enabled) before falling back to software zlib. Useful on machines where hardware accelerators are intermittently unavailable. + iaa_compress_percentage - Values: 0-100. Default: 50 - If both IAA and QAT are enabled, percentage of compression calls to offload to IAA. iaa_prepend_empty_block - Values: 0,1. Default: 0 -- Prepend an empty stored block to the compressed data to "mark" that the data was compressed by IAA. -- IAA has a 4kB history window limit and it is not able to decompress blocks that use a longer history window (up to 32kB per deflate standard). -- During decompression, this marker indicates that the data was compressed by IAA and is therefore guarateed decompressible by IAA. +- **Deprecated.** This option is retained for backward compatibility and will be removed in a future release. Setting it to 1 has no effect on decompression. +- Background: the original design prepended a 5-byte empty stored-block marker to IAA-compressed output so the decompressor could identify IAA-produced data (which uses a 4kB history window). This approach was abandoned because QPL hardware always consumes all `available_in` bytes regardless of where the stream boundary falls, making marker-based detection unreliable when the caller does not supply the exact compressed size. IAA decompression eligibility is now determined by a 512-byte minimum input length threshold: callers such as Java's `ZipInputStream` feed chunks of ≤512 bytes when the compressed size is unknown, while Lucene stored-field reads always supply the exact size (>512 bytes). iaa_uncompress_percentage - Values: 0-100. Default: 50 - If both IAA and QAT are enabled, percentage of decompression calls to offload to IAA. -- If iaa_prepend_empty_block = 1, this percentage is only applied to data with the empty block marker. qat_periodical_polling = 0 - Values: 0,1. Default: 0 diff --git a/common.cmake b/common.cmake index 7c60c0f..e3c94a0 100644 --- a/common.cmake +++ b/common.cmake @@ -2,6 +2,7 @@ # SPDX-License-Identifier: Apache-2.0 option(USE_IAA "Use IAA (requires QPL)" OFF) +option(USE_IGZIP "Use IGZIP (requires isa-l)" OFF) option(USE_QAT "Use QAT (requires QATzip)" OFF) option(DEBUG_LOG "for logging" ON) option(COVERAGE "for coverage" OFF) @@ -14,6 +15,61 @@ if(USE_IAA) add_compile_definitions(USE_IAA) endif() +if(USE_IGZIP) + add_compile_definitions(USE_IGZIP) + + # Enforce minimum ISA-L version. v2.32.1 contains critical igzip bugfixes: + # - next_in/avail_in not reset after block finish (inflate boundary bug) + # - zlib DICTID stored/read in wrong byte order + # - gzip_flag mutated by isal_deflate when writing wrapper header + # Earlier versions require defensive workarounds that have been removed. + if(ISAL_PATH) + set(_isal_api_header "${ISAL_PATH}/include/isal_api.h") + else() + find_path(_isal_api_include isal_api.h PATH_SUFFIXES include) + if(_isal_api_include) + set(_isal_api_header "${_isal_api_include}/isal_api.h") + endif() + unset(_isal_api_include CACHE) + endif() + if(NOT _isal_api_header OR NOT EXISTS "${_isal_api_header}") + message(FATAL_ERROR "ISA-L header (isal_api.h) not found. Set -DISAL_PATH= or install ISA-L system-wide.") + endif() + file(STRINGS "${_isal_api_header}" _isal_major_line REGEX "^#define[ \t]+ISAL_MAJOR_VERSION[ \t]+") + file(STRINGS "${_isal_api_header}" _isal_minor_line REGEX "^#define[ \t]+ISAL_MINOR_VERSION[ \t]+") + file(STRINGS "${_isal_api_header}" _isal_patch_line REGEX "^#define[ \t]+ISAL_PATCH_VERSION[ \t]+") + if(NOT _isal_major_line OR NOT _isal_minor_line OR NOT _isal_patch_line) + message(FATAL_ERROR "Could not find ISAL_MAJOR/MINOR/PATCH_VERSION macros in ${_isal_api_header}. ISA-L header may be malformed or too old.") + endif() + list(GET _isal_major_line 0 _isal_major_line) + list(GET _isal_minor_line 0 _isal_minor_line) + list(GET _isal_patch_line 0 _isal_patch_line) + string(REGEX REPLACE "^#define[ \t]+ISAL_MAJOR_VERSION[ \t]+([0-9]+).*" "\\1" _isal_major "${_isal_major_line}") + string(REGEX REPLACE "^#define[ \t]+ISAL_MINOR_VERSION[ \t]+([0-9]+).*" "\\1" _isal_minor "${_isal_minor_line}") + string(REGEX REPLACE "^#define[ \t]+ISAL_PATCH_VERSION[ \t]+([0-9]+).*" "\\1" _isal_patch "${_isal_patch_line}") + foreach(_v _isal_major _isal_minor _isal_patch) + if(NOT "${${_v}}" MATCHES "^[0-9]+$") + message(FATAL_ERROR "Failed to parse ISA-L version from ${_isal_api_header}: '${_v}' = '${${_v}}'") + endif() + endforeach() + set(_isal_version_found "${_isal_major}.${_isal_minor}.${_isal_patch}") + message(STATUS "Found ISA-L version: ${_isal_version_found} (${_isal_api_header})") + if(_isal_version_found VERSION_LESS "2.32.1") + message(FATAL_ERROR + "ISA-L >= 2.32.1 required, found ${_isal_version_found}.\n" + "Update ISA-L or pass -DISAL_PATH= to cmake.\n" + "Release: https://github.com/intel/isa-l/releases/tag/v2.32.1") + endif() + unset(_isal_api_header) + unset(_isal_major_line) + unset(_isal_minor_line) + unset(_isal_patch_line) + unset(_isal_major) + unset(_isal_minor) + unset(_isal_patch) + unset(_isal_version_found) +endif() + if(USE_QAT) add_compile_definitions(USE_QAT) endif() @@ -84,16 +140,31 @@ if(USE_IAA) else() message(STATUS "Using QPL_PATH: ${QPL_PATH}") include_directories(${QPL_PATH}/include/qpl ${QPL_PATH}/include) - link_directories(PUBLIC ${QPL_PATH}/lib64 ${QPL_PATH}/lib) + link_directories(${QPL_PATH}/lib64 ${QPL_PATH}/lib) link_libraries(qpl) endif() endif() +if(USE_IGZIP) + if(NOT DEFINED ISAL_PATH) + find_package(isal REQUIRED) + if(isal_FOUND) + message(STATUS "Found ISA-L: ${isal_DIR}") + link_libraries(isal) + endif() + else() + message(STATUS "Using ISAL_PATH: ${ISAL_PATH}") + include_directories(${ISAL_PATH}/include) + link_directories(${ISAL_PATH}/.libs) + link_libraries(isal) + endif() +endif() + if(USE_QAT) if(DEFINED QATZIP_PATH) message(STATUS "Using QATZIP_PATH: ${QATZIP_PATH}") include_directories(${QATZIP_PATH}/include) - link_directories(PUBLIC ${QATZIP_PATH}/src/.libs) + link_directories(${QATZIP_PATH}/src/.libs) endif() link_libraries(qatzip) endif() diff --git a/config/config.cpp b/config/config.cpp index aefe066..c2a85c8 100644 --- a/config/config.cpp +++ b/config/config.cpp @@ -21,6 +21,8 @@ uint32_t configs[CONFIG_MAX] = { 0, /*use_iaa_uncompress*/ 1, /*use_zlib_compress*/ 1, /*use_zlib_uncompress*/ + 0, /*use_igzip_compress*/ + 0, /*use_igzip_uncompress*/ 50, /*iaa_compress_percentage*/ 50, /*iaa_uncompress_percentage*/ 0, /*iaa_prepend_empty_block*/ @@ -28,6 +30,7 @@ uint32_t configs[CONFIG_MAX] = { 1, /*qat_compression_level*/ 0, /*qat_compression_allow_chunking*/ 0, /*ignore_zlib_dictionary*/ + 1, /*igzip_fallback*/ 1, /*log_level*/ 1000 /*log_stats_samples*/ }; @@ -45,6 +48,8 @@ bool LoadConfigFile(std::string& file_content, const char* file_path) { "use_iaa_uncompress", "use_zlib_compress", "use_zlib_uncompress", + "use_igzip_compress", + "use_igzip_uncompress", "iaa_compress_percentage", "iaa_uncompress_percentage", "iaa_prepend_empty_block", @@ -52,6 +57,7 @@ bool LoadConfigFile(std::string& file_content, const char* file_path) { "qat_compression_level", "qat_compression_allow_chunking", "ignore_zlib_dictionary", + "igzip_fallback", "log_level", "log_stats_samples" }; @@ -78,6 +84,8 @@ bool LoadConfigFile(std::string& file_content, const char* file_path) { trySetConfig(USE_IAA_UNCOMPRESS, 1, 0); trySetConfig(USE_ZLIB_COMPRESS, 1, 0); trySetConfig(USE_ZLIB_UNCOMPRESS, 1, 0); + trySetConfig(USE_IGZIP_COMPRESS, 1, 0); + trySetConfig(USE_IGZIP_UNCOMPRESS, 1, 0); trySetConfig(IAA_COMPRESS_PERCENTAGE, 100, 0); trySetConfig(IAA_UNCOMPRESS_PERCENTAGE, 100, 0); trySetConfig(IAA_PREPEND_EMPTY_BLOCK, 1, 0); @@ -85,6 +93,7 @@ bool LoadConfigFile(std::string& file_content, const char* file_path) { trySetConfig(QAT_COMPRESSION_LEVEL, 9, 1); trySetConfig(QAT_COMPRESSION_ALLOW_CHUNKING, 1, 0); trySetConfig(IGNORE_ZLIB_DICTIONARY, 1, 0); + trySetConfig(IGZIP_FALLBACK, 1, 0); trySetConfig(LOG_LEVEL, 3, 0); trySetConfig(LOG_STATS_SAMPLES, UINT32_MAX, 0); diff --git a/config/config.h b/config/config.h index acc8055..838d1cd 100644 --- a/config/config.h +++ b/config/config.h @@ -16,13 +16,16 @@ enum ConfigOption { USE_IAA_UNCOMPRESS, USE_ZLIB_COMPRESS, USE_ZLIB_UNCOMPRESS, + USE_IGZIP_COMPRESS, + USE_IGZIP_UNCOMPRESS, IAA_COMPRESS_PERCENTAGE, IAA_UNCOMPRESS_PERCENTAGE, - IAA_PREPEND_EMPTY_BLOCK, + IAA_PREPEND_EMPTY_BLOCK, // DEPRECATED — see README for details QAT_PERIODICAL_POLLING, QAT_COMPRESSION_LEVEL, QAT_COMPRESSION_ALLOW_CHUNKING, IGNORE_ZLIB_DICTIONARY, + IGZIP_FALLBACK, LOG_LEVEL, LOG_STATS_SAMPLES, CONFIG_MAX diff --git a/config/default_config b/config/default_config index 827b7b2..b70a92b 100644 --- a/config/default_config +++ b/config/default_config @@ -4,6 +4,8 @@ use_iaa_compress = 0 use_iaa_uncompress = 0 use_zlib_compress = 1 use_zlib_uncompress = 1 +use_igzip_compress = 0 +use_igzip_uncompress = 0 iaa_compress_percentage = 50 iaa_uncompress_percentage = 50 iaa_prepend_empty_block = 0 diff --git a/iaa.cpp b/iaa.cpp index 2bbcd27..5a22f87 100644 --- a/iaa.cpp +++ b/iaa.cpp @@ -99,9 +99,19 @@ int CompressIAA(uint8_t* input, uint32_t* input_length, uint8_t* output, output_shift += GZIP_EXT_XHDR_SIZE; } - // If prepending an empty block, leave space for it to be added - // For zlib format, we don't need an empty block as a marker, as the zlib - // header includes info about the window size + // DEPRECATED: iaa_prepend_empty_block is no longer used by the decompressor. + // The original design used a 5-byte empty stored-block marker written at the + // start of IAA-compressed output so that the decompressor could detect and + // trust IAA-compressed data (which uses a 4kB history window). This approach + // was abandoned because QPL hardware always consumes all available_in bytes + // regardless of where the BFINAL=1 token falls (overconsumption bug), so a + // caller-supplied exact boundary is required instead. IsIAADecompressible now + // uses a 512-byte minimum input length threshold to gate IAA decompression: + // Java ZipInputStream feeds <=512-byte chunks (csize unknown), triggering + // overconsumption; Lucene stored-field reads always provide the exact + // compressed size (>512 bytes), where consuming all input is correct. + // The config option is retained for backward compatibility and will be + // removed in a future release. bool prepend_empty_block = false; CompressedFormat format = GetCompressedFormat(window_bits); if (format != CompressedFormat::ZLIB && @@ -255,44 +265,32 @@ bool SupportedOptionsIAA(int window_bits, uint32_t input_length, return false; } -bool PrependedEmptyBlockPresent(uint8_t* input, uint32_t input_length, - CompressedFormat format) { - uint32_t header_length = GetHeaderLength(format); - if (header_length + PREPENDED_BLOCK_LENGTH > input_length) { - return false; - } - - if (input[header_length] == 0 && input[header_length + 1] == 0 && - input[header_length + 2] == 0 && input[header_length + 3] == 0xFF && - input[header_length + 4] == 0xFF) { - Log(LogLevel::LOG_INFO, "PrependedEmptyBlockPresent() Line ", __LINE__, - " Empty block detected\n"); - return true; - } - - return false; -} - bool IsIAADecompressible(uint8_t* input, uint32_t input_length, int window_bits) { CompressedFormat format = GetCompressedFormat(window_bits); if (format == CompressedFormat::ZLIB) { int window = GetWindowSizeFromZlibHeader(input, input_length); - Log(LogLevel::LOG_INFO, "IsIAADecompressible() Line ", __LINE__, " window ", - window, "\n"); return window <= 12; - } else { - // if no empty block markers selected, we cannot tell for sure it's - // IAA-decompression, but we assume it is. - if (configs[IAA_PREPEND_EMPTY_BLOCK] == 0) { - return true; - } else if (configs[IAA_PREPEND_EMPTY_BLOCK] == 1 && - PrependedEmptyBlockPresent(input, input_length, format)) { - return true; - } else { - return false; - } } + // For raw deflate and gzip formats, QPL always reports total_in == + // available_in regardless of where BFINAL=1 falls in the stream. This is + // safe only when the caller provides avail_in == actual_compressed_size + // (e.g. Lucene stored-field reads, where the exact compressed size is known + // from the .fdt file format). + // + // Callers that do not know the compressed size a priori — notably Java's + // ZipInputStream, which uses a fixed 512-byte internal buffer and feeds + // chunks of that size to inflate() — will have avail_in > actual_csize. + // QPL consuming all 512 bytes then reporting total_in=512 when actual_csize + // was 2 triggers ZipException at the Java level. + // + // Guard: only attempt IAA when input_length > 512. ZipInputStream always + // feeds chunks of at most 512 bytes, so any call above that threshold is + // guaranteed not to be a ZipInputStream-chunked read. Lucene stored-field + // entries are typically much larger than 512 bytes; the few that are smaller + // fall back to IGZIP which is also correct. + static constexpr uint32_t kZipInputStreamBufferSize = 512; + return input_length > kZipInputStreamBufferSize; } #endif // USE_IAA diff --git a/igzip.cpp b/igzip.cpp new file mode 100644 index 0000000..01f2fe3 --- /dev/null +++ b/igzip.cpp @@ -0,0 +1,479 @@ +// Copyright (C) 2025 Intel Corporation +// SPDX-License-Identifier: Apache-2.0 + +#ifdef USE_IGZIP + +#include "igzip.h" + +#include +#include + +#include "logging.h" + +static uint16_t ClampHistBits(int bits) { + if (bits < 0) { + return 0; + } + if (bits > ISAL_DEF_MAX_HIST_BITS) { + return ISAL_DEF_MAX_HIST_BITS; + } + return (uint16_t)bits; +} + +static void ConfigureDeflateWindow(struct isal_zstream *isal_strm, + int windowBits) { + if (windowBits < 0) { + isal_strm->gzip_flag = IGZIP_DEFLATE; + isal_strm->hist_bits = ClampHistBits(-windowBits); + return; + } + + if (windowBits >= 24 && windowBits <= 31) { + isal_strm->gzip_flag = IGZIP_GZIP; + isal_strm->hist_bits = ClampHistBits(windowBits - 16); + return; + } + + isal_strm->gzip_flag = IGZIP_ZLIB; + isal_strm->hist_bits = ClampHistBits(windowBits); +} + +static void ConfigureInflateWindow(struct inflate_state *isal_strm_inflate, + int windowBits) { + if (windowBits < 0) { + isal_strm_inflate->crc_flag = IGZIP_DEFLATE; + isal_strm_inflate->hist_bits = ClampHistBits(-windowBits); + return; + } + + if ((windowBits >= 24 && windowBits <= 31) || + (windowBits >= 40 && windowBits <= 47)) { + isal_strm_inflate->crc_flag = IGZIP_GZIP; + isal_strm_inflate->hist_bits = + ClampHistBits(windowBits > 31 ? windowBits - 32 : windowBits - 16); + return; + } + + isal_strm_inflate->crc_flag = IGZIP_ZLIB; + isal_strm_inflate->hist_bits = ClampHistBits(windowBits); +} + +bool IsIGZIPDeflateFinished(const struct isal_zstream *stream) { + if (stream == nullptr) { + return false; + } + const enum isal_zstate_state state = stream->internal_state.state; + // ZSTATE_TMP_END is a temporary state and may require reentry to + // flush remaining output; only ZSTATE_END is terminal. + return state == ZSTATE_END; +} + +struct isal_zstream *InitCompressIGZIP(int level, int windowBits) { + Log(LogLevel::LOG_INFO, + "InitCompressIGZIP() initializing deflate with level ", level, + ", windowBits ", windowBits, "\n"); + + struct isal_zstream *isal_strm = + (struct isal_zstream *)malloc(sizeof(struct isal_zstream)); + if (!isal_strm) { + Log(LogLevel::LOG_ERROR, + "InitCompressIGZIP() memory allocation for isal_zstream failed\n"); + return nullptr; + } + + /* Setup ISA-L compression context */ + isal_deflate_init(isal_strm); + + isal_strm->end_of_stream = 0; + isal_strm->flush = NO_FLUSH; + + // Map Zlib levels to ISA-L levels + if (level >= 1 && level <= 2) { + isal_strm->level = 1; + isal_strm->level_buf = (uint8_t *)malloc(ISAL_DEF_LVL1_DEFAULT); + isal_strm->level_buf_size = ISAL_DEF_LVL1_DEFAULT; + } else if ((level >= 3 && level <= 6) || level == -1) { + isal_strm->level = 2; + isal_strm->level_buf = (uint8_t *)malloc(ISAL_DEF_LVL2_DEFAULT); + isal_strm->level_buf_size = ISAL_DEF_LVL2_DEFAULT; + } else if (level >= 7 && level <= 9) { + isal_strm->level = 3; + isal_strm->level_buf = (uint8_t *)malloc(ISAL_DEF_LVL3_DEFAULT); + isal_strm->level_buf_size = ISAL_DEF_LVL3_DEFAULT; + } else { + Log(LogLevel::LOG_ERROR, "InitCompressIGZIP() invalid compression level\n"); + free(isal_strm); + return nullptr; + } + + if (!isal_strm->level_buf) { + free(isal_strm); + Log(LogLevel::LOG_ERROR, + "InitCompressIGZIP() memory allocation for level_buf failed\n"); + return nullptr; + } + + ConfigureDeflateWindow(isal_strm, windowBits); + + return isal_strm; +} + +int CompressIGZIP(struct isal_zstream *isal_strm, int flush, + const uint8_t *input, uint32_t *input_length, uint8_t *output, + uint32_t *output_length, const unsigned long *total_in, + const unsigned long *total_out) { + int ret; + + if (!isal_strm) { + Log(LogLevel::LOG_ERROR, "CompressIGZIP() deflate isal_strm is NULL\n"); + return -1; + } + + // set stream->avail_in, next_in, avail_out, next_out (from zstream)​ + isal_strm->next_out = output; + const uint32_t original_avail_out = *output_length; + isal_strm->avail_out = original_avail_out; + isal_strm->next_in = const_cast(input); + const uint32_t original_avail_in = *input_length; + isal_strm->avail_in = original_avail_in; + isal_strm->total_out = *total_out; + isal_strm->total_in = *total_in; + + // stream->flush mapping + switch (flush) { + case Z_NO_FLUSH: + isal_strm->flush = NO_FLUSH; + break; + case Z_SYNC_FLUSH: + case Z_PARTIAL_FLUSH: + case Z_BLOCK: + isal_strm->flush = SYNC_FLUSH; + break; + case Z_FULL_FLUSH: + isal_strm->flush = FULL_FLUSH; + break; + case Z_FINISH: + isal_strm->flush = FULL_FLUSH; + isal_strm->end_of_stream = 1; + break; + default: + Log(LogLevel::LOG_ERROR, "CompressIGZIP() invalid flush value\n"); + return -1; + } + + Log(LogLevel::LOG_INFO, "CompressIGZIP() gzip_flag ", isal_strm->gzip_flag, + ", hist_bits ", isal_strm->hist_bits, ", flush ", isal_strm->flush, + ", level ", isal_strm->level, ", avail_in ", isal_strm->avail_in, + ", avail_out ", (uint32_t)isal_strm->avail_out, ", total_out ", + (uint32_t)isal_strm->total_out, ", total_in ", + (uint32_t)isal_strm->total_in, "\n"); + + // ISA-L always emits sync bytes on SYNC_FLUSH regardless of pending data. + // When the stream is already byte-aligned (ZSTATE_NEW_HDR) and there is no + // new input, no real progress can be made — return 0 progress so the caller + // reports Z_BUF_ERROR, matching zlib's semantics for empty flush calls. + // ZSTATE_NEW_HDR is the idle/byte-aligned state in ISA-L's internal deflate + // state machine; validated against ISA-L v2.32.0 (commit c196241). + if (isal_strm->avail_in == 0 && isal_strm->flush == SYNC_FLUSH && + isal_strm->end_of_stream == 0 && + isal_strm->internal_state.state == ZSTATE_NEW_HDR) { + *output_length = 0; + *input_length = 0; + return 0; + } + + int comp = isal_deflate(isal_strm); + + *output_length = original_avail_out - isal_strm->avail_out; + *input_length = original_avail_in - isal_strm->avail_in; + + Log(LogLevel::LOG_INFO, "CompressIGZIP() after isal_deflate: avail_in ", + isal_strm->avail_in, ", avail_out ", (uint32_t)isal_strm->avail_out, + ", bytes_consumed ", *input_length, ", bytes_produced ", *output_length, + "\n"); + + ret = (comp == COMP_OK) ? Z_OK : Z_STREAM_ERROR; + + if (ret == Z_OK) { + Log(LogLevel::LOG_INFO, + "CompressIGZIP() deflate finished successfully Z_OK\n"); + } else { + Log(LogLevel::LOG_ERROR, + "CompressIGZIP() deflate finished with error code ", ret, "\n"); + switch (comp) { + case INVALID_FLUSH: + Log(LogLevel::LOG_ERROR, "CompressIGZIP() invalid flush\n"); + break; + case INVALID_PARAM: + Log(LogLevel::LOG_ERROR, "CompressIGZIP() invalid parameter\n"); + break; + case STATELESS_OVERFLOW: + Log(LogLevel::LOG_ERROR, "CompressIGZIP() stateless overflow\n"); + break; + case ISAL_INVALID_OPERATION: + Log(LogLevel::LOG_ERROR, "CompressIGZIP() invalid operation\n"); + break; + case ISAL_INVALID_STATE: + Log(LogLevel::LOG_ERROR, "CompressIGZIP() invalid state\n"); + break; + case ISAL_INVALID_LEVEL: + Log(LogLevel::LOG_ERROR, "CompressIGZIP() invalid level\n"); + break; + case ISAL_INVALID_LEVEL_BUF: + Log(LogLevel::LOG_ERROR, "CompressIGZIP() invalid level buffer\n"); + break; + } + } + + return ret; +} + +int EndCompressIGZIP(struct isal_zstream *isal_strm) { + if (!isal_strm) { + Log(LogLevel::LOG_ERROR, "EndCompressIGZIP() isal_stream is NULL\n"); + return -1; + } + + // Free allocated memory for level_buf and isal_strm + if (isal_strm->level_buf) { + free(isal_strm->level_buf); + } + free(isal_strm); + + Log(LogLevel::LOG_INFO, "EndCompressIGZIP() deflate end\n"); + return Z_OK; +} + +struct inflate_state *InitUncompressIGZIP(int windowBits) { + struct inflate_state *isal_strm_inflate = + (struct inflate_state *)malloc(sizeof(struct inflate_state)); + if (!isal_strm_inflate) { + Log(LogLevel::LOG_ERROR, + "InitUncompressIGZIP() memory allocation for inflate_state failed\n"); + return nullptr; + } + + Log(LogLevel::LOG_INFO, + "InitUncompressIGZIP() initializing inflate with windowBits ", windowBits, + "\n"); + + /* Setup ISA-L decompression context */ + isal_inflate_init(isal_strm_inflate); + + isal_strm_inflate->avail_in = 0; + isal_strm_inflate->next_in = NULL; + + ConfigureInflateWindow(isal_strm_inflate, windowBits); + + return isal_strm_inflate; +} + +void IGZIPHandleActiveStreamNoInput(z_streamp strm, + struct inflate_state *isal_strm_inflate, + int *ret) { + // Caller guarantees strm->avail_in == 0 before calling this function. + if (strm == nullptr || isal_strm_inflate == nullptr || ret == nullptr) { + return; + } + + uint32_t input_len = 0; + uint32_t output_len = strm->avail_out; + bool end_of_stream = true; + + *ret = UncompressIGZIP(isal_strm_inflate, strm->next_in, &input_len, + strm->next_out, &output_len, &strm->total_in, + &strm->total_out, &end_of_stream); + + if (*ret == 0) { + strm->next_out += output_len; + strm->avail_out -= output_len; + // This is the only site that updates total_out for the avail_in==0 path. + // The caller returns immediately after this call, so the main + // inflate() update block is never reached — no double-counting. + strm->total_out += output_len; + if (end_of_stream) { + *ret = Z_STREAM_END; + } else if (output_len > 0) { + *ret = Z_OK; + } else { + *ret = Z_BUF_ERROR; + } + return; + } + + *ret = Z_BUF_ERROR; +} + +IGZIPInflatePathAction IGZIPRunInflateAndSelectPathAction( + z_streamp strm, struct inflate_state **isal_strm_inflate, int window_bits, + uint32_t *input_length, uint32_t *output_length, int *ret, + bool *end_of_stream) { + if (strm == nullptr || isal_strm_inflate == nullptr || + input_length == nullptr || output_length == nullptr || ret == nullptr || + end_of_stream == nullptr) { + if (ret != nullptr) { + *ret = Z_DATA_ERROR; + } + return IGZIP_INFLATE_PATH_NONE; + } + + if (*isal_strm_inflate == nullptr) { + *isal_strm_inflate = InitUncompressIGZIP(window_bits); + if (*isal_strm_inflate == nullptr) { + Log(LogLevel::LOG_ERROR, + "IGZIPRunInflateAndSelectPathAction() failed to initialize igzip " + "inflate stream\n"); + *ret = Z_DATA_ERROR; + return IGZIP_INFLATE_PATH_NONE; + } + } + + *ret = UncompressIGZIP(*isal_strm_inflate, strm->next_in, input_length, + strm->next_out, output_length, &strm->total_in, + &strm->total_out, end_of_stream); + + // Raw boundary guard removed: ISA-L PR#215 (cd72fd7) fixes avail_in + // over-consumption at BLOCK_FINISH in isal_inflate; the guard is no + // longer needed for that case. + + if (*ret == Z_NEED_DICT) { + return IGZIP_INFLATE_PATH_FALLBACK_NEED_DICT; + } + if (*ret == Z_DATA_ERROR) { + return IGZIP_INFLATE_PATH_FALLBACK_DATA_ERROR; + } + if (*ret == 0) { + return IGZIP_INFLATE_PATH_SET_IGZIP; + } + + return IGZIP_INFLATE_PATH_NONE; +} + +int UncompressIGZIP(struct inflate_state *isal_strm_inflate, + const uint8_t *input, uint32_t *input_length, + uint8_t *output, uint32_t *output_length, + const unsigned long *total_in, + const unsigned long *total_out, bool *end_of_stream) { + (void)total_in; + + if (!isal_strm_inflate) { + Log(LogLevel::LOG_ERROR, "UncompressIGZIP() isal_strm_inflate is NULL\n"); + return Z_STREAM_ERROR; + } + // set stream->avail_in, next_in, avail_out, next_out (from zstream)​ + isal_strm_inflate->next_out = output; + const uint32_t original_avail_out = *output_length; + isal_strm_inflate->avail_out = original_avail_out; + const uint32_t original_avail_in = *input_length; + isal_strm_inflate->avail_in = original_avail_in; + isal_strm_inflate->next_in = const_cast(input); + isal_strm_inflate->total_out = *total_out; + + const int decomp = isal_inflate(isal_strm_inflate); + + uint32_t consumed_before_adjust = 0; + if (isal_strm_inflate->avail_in <= original_avail_in) { + consumed_before_adjust = original_avail_in - isal_strm_inflate->avail_in; + } else { + Log(LogLevel::LOG_ERROR, "UncompressIGZIP() invalid avail_in ", + isal_strm_inflate->avail_in, " greater than original_avail_in ", + original_avail_in, ", clamping consumed bytes to 0\n"); + consumed_before_adjust = 0; + } + + // Bug 2 guard removed: ISA-L PR#215 (cd72fd7) makes BLOCK_FINISH correctly + // restore avail_in after over-consumption into read_in. Continuing past + // BLOCK_INPUT_DONE (even with avail_in 1-7) reaches BLOCK_FINISH with the + // correct stream boundary. No fallback needed. + // (ISA-L >= 2.32.1 enforced at configure time via common.cmake.) + + *output_length = original_avail_out - isal_strm_inflate->avail_out; + *input_length = consumed_before_adjust; + + if (end_of_stream != nullptr) { + *end_of_stream = (isal_strm_inflate->block_state == ISAL_BLOCK_FINISH); + } + + int ret = + Z_DATA_ERROR; // default: any unhandled ISA-L error maps to Z_DATA_ERROR + if (decomp == ISAL_DECOMP_OK || decomp == ISAL_END_INPUT) { + ret = 0; + } else if (decomp == ISAL_NEED_DICT) { + ret = Z_NEED_DICT; + } + + if (ret == Z_OK) { + Log(LogLevel::LOG_INFO, + "UncompressIGZIP() inflate finished successfully Z_OK\n"); + } else { + Log(LogLevel::LOG_ERROR, + "UncompressIGZIP() inflate finished with error code ", ret, "\n"); + switch (decomp) { + case ISAL_INVALID_BLOCK: + Log(LogLevel::LOG_ERROR, + "UncompressIGZIP() ISA-L error - Invalid block\n"); + break; + case ISAL_INVALID_SYMBOL: + Log(LogLevel::LOG_ERROR, + "UncompressIGZIP() ISA-L error - Invalid symbol\n"); + break; + case ISAL_INVALID_LOOKBACK: + Log(LogLevel::LOG_ERROR, + "UncompressIGZIP() ISA-L error - Invalid lookback\n"); + break; + case ISAL_END_INPUT: + Log(LogLevel::LOG_ERROR, + "UncompressIGZIP() ISA-L error - End of input reached " + "unexpectedly\n"); + break; + case ISAL_UNSUPPORTED_METHOD: + Log(LogLevel::LOG_ERROR, + "UncompressIGZIP() ISA-L error - Unsupported method\n"); + break; + case ISAL_NEED_DICT: + Log(LogLevel::LOG_ERROR, + "UncompressIGZIP() ISA-L error - Need dictionary\n"); + break; + default: + Log(LogLevel::LOG_ERROR, "UncompressIGZIP() ISA-L error code ", decomp, + "\n"); + break; + } + } + + return ret; +} + +int EndUncompressIGZIP(struct inflate_state *isal_strm_inflate) { + if (!isal_strm_inflate) { + Log(LogLevel::LOG_ERROR, "EndUncompressIGZIP() z_streamp is NULL\n"); + return Z_STREAM_ERROR; + } + + free(isal_strm_inflate); + + Log(LogLevel::LOG_INFO, "EndUncompressIGZIP() inflate end\n"); + return Z_OK; +} + +void ResetCompressIGZIP(struct isal_zstream *isal_strm) { + // ISA-L PR#215 (30e90b4) stops gzip_flag from being mutated during + // compression, so isal_deflate_reset preserves it correctly. No manual + // ConfigureDeflateWindow call is needed here. + isal_deflate_reset(isal_strm); + isal_strm->end_of_stream = 0; + isal_strm->flush = NO_FLUSH; +} + +int ResetUncompressIGZIP(struct inflate_state *isal_strm_inflate) { + if (!isal_strm_inflate) { + Log(LogLevel::LOG_ERROR, + "ResetUncompressIGZIP() isal_strm_inflate is NULL\n"); + return Z_STREAM_ERROR; + } + + isal_inflate_reset(isal_strm_inflate); + + return Z_OK; +} +#endif diff --git a/igzip.h b/igzip.h new file mode 100644 index 0000000..40b51eb --- /dev/null +++ b/igzip.h @@ -0,0 +1,44 @@ +// Copyright (C) 2025 Intel Corporation +// SPDX-License-Identifier: Apache-2.0 + +#pragma once + +#ifdef USE_IGZIP +#include +#include + +struct isal_zstream *InitCompressIGZIP(int level, int windowBits); +int CompressIGZIP(struct isal_zstream *isal_strm, int flush, + const uint8_t *input, uint32_t *input_length, uint8_t *output, + uint32_t *output_length, const unsigned long *total_in, + const unsigned long *total_out); +bool IsIGZIPDeflateFinished(const struct isal_zstream *stream); + +enum IGZIPInflatePathAction { + IGZIP_INFLATE_PATH_NONE, + IGZIP_INFLATE_PATH_SET_IGZIP, + IGZIP_INFLATE_PATH_FALLBACK_NEED_DICT, + IGZIP_INFLATE_PATH_FALLBACK_DATA_ERROR, +}; + +void IGZIPHandleActiveStreamNoInput(z_streamp strm, + struct inflate_state *isal_strm_inflate, + int *ret); + +IGZIPInflatePathAction IGZIPRunInflateAndSelectPathAction( + z_streamp strm, struct inflate_state **isal_strm_inflate, int window_bits, + uint32_t *input_length, uint32_t *output_length, int *ret, + bool *end_of_stream); + +int EndCompressIGZIP(struct isal_zstream *isal_strm); +void ResetCompressIGZIP(struct isal_zstream *isal_strm); + +struct inflate_state *InitUncompressIGZIP(int windowBits); +int UncompressIGZIP(struct inflate_state *isal_strm_inflate, + const uint8_t *input, uint32_t *input_length, + uint8_t *output, uint32_t *output_length, + const unsigned long *total_in, + const unsigned long *total_out, bool *end_of_stream); +int EndUncompressIGZIP(struct inflate_state *isal_strm_inflate); +int ResetUncompressIGZIP(struct inflate_state *isal_strm_inflate); +#endif diff --git a/sharded_map.h b/sharded_map.h index ab8dda8..debb87e 100644 --- a/sharded_map.h +++ b/sharded_map.h @@ -4,6 +4,7 @@ #pragma once #include +#include #include #include #include diff --git a/statistics.cpp b/statistics.cpp index 9dabeb6..0610678 100644 --- a/statistics.cpp +++ b/statistics.cpp @@ -18,9 +18,10 @@ using namespace config; const std::array stat_names{ {"deflate_count", "deflate_error_count", "deflate_qat_count", "deflate_qat_error_count", "deflate_iaa_count", "deflate_iaa_error_count", - "deflate_zlib_count", "inflate_count", "inflate_error_count", - "inflate_qat_count", "inflate_qat_error_count", "inflate_iaa_count", - "inflate_iaa_error_count", "inflate_zlib_count"}}; + "deflate_igzip_count", "deflate_igzip_error_count", "deflate_zlib_count", + "inflate_count", "inflate_error_count", "inflate_qat_count", + "inflate_qat_error_count", "inflate_iaa_count", "inflate_iaa_error_count", + "inflate_igzip_count", "inflate_igzip_error_count", "inflate_zlib_count"}}; thread_local std::array stats{}; diff --git a/statistics.h b/statistics.h index 8cee404..b3654ce 100644 --- a/statistics.h +++ b/statistics.h @@ -17,6 +17,8 @@ enum class Statistic : size_t { DEFLATE_QAT_ERROR_COUNT, DEFLATE_IAA_COUNT, DEFLATE_IAA_ERROR_COUNT, + DEFLATE_IGZIP_COUNT, + DEFLATE_IGZIP_ERROR_COUNT, DEFLATE_ZLIB_COUNT, INFLATE_COUNT, INFLATE_ERROR_COUNT, @@ -24,6 +26,8 @@ enum class Statistic : size_t { INFLATE_QAT_ERROR_COUNT, INFLATE_IAA_COUNT, INFLATE_IAA_ERROR_COUNT, + INFLATE_IGZIP_COUNT, + INFLATE_IGZIP_ERROR_COUNT, INFLATE_ZLIB_COUNT, STATS_COUNT }; diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index b0c4564..0ac0465 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -24,7 +24,7 @@ add_executable(zlib_accel_test ) add_custom_target(run - COMMAND ./zlib_accel_test + COMMAND ./zlib_accel_test # --gtest_filter=ConfigLoaderTest.LoadValidConfig DEPENDS zlib_accel_test ) diff --git a/tests/zlib_accel_test.cpp b/tests/zlib_accel_test.cpp index 61a7189..4e8b234 100644 --- a/tests/zlib_accel_test.cpp +++ b/tests/zlib_accel_test.cpp @@ -6,6 +6,7 @@ #include #include +#include #include #include #include @@ -29,6 +30,10 @@ using namespace config; +#ifdef USE_IGZIP +#include "../igzip.h" +#endif + enum BlockCompressibilityType { compressible_block, incompressible_block, @@ -183,6 +188,41 @@ int ZlibUncompressUtility2(const char* input, size_t input_length, return st; } +int ZlibCompressWithLevel(const char* input, size_t input_length, + std::string* output, int level, int window_bits, + int flush, size_t* output_upper_bound, + ExecutionPath* execution_path) { + z_stream stream; + memset(&stream, 0, sizeof(z_stream)); + + int st = deflateInit2(&stream, level, Z_DEFLATED, window_bits, 8, + Z_DEFAULT_STRATEGY); + if (st != Z_OK) { + deflateEnd(&stream); + return st; + } + + stream.next_in = (Bytef*)input; + stream.avail_in = static_cast(input_length); + + *output_upper_bound = + deflateBound(&stream, static_cast(input_length)); + output->resize(*output_upper_bound); + stream.avail_out = static_cast(*output_upper_bound); + stream.next_out = reinterpret_cast(&(*output)[0]); + + st = deflate(&stream, flush); + *execution_path = GetDeflateExecutionPath(&stream); + if (st != Z_STREAM_END) { + deflateEnd(&stream); + return st; + } + output->resize(stream.total_out); + + deflateEnd(&stream); + return st; +} + int ZlibCompressGzipFile(const char* input, size_t input_length) { const char* filename = "file.gz"; remove(filename); @@ -240,16 +280,25 @@ void SetCompressPath(ExecutionPath path, bool zlib_fallback, switch (path) { case ZLIB: SetConfig(USE_IAA_COMPRESS, 0); + SetConfig(USE_IGZIP_COMPRESS, 0); SetConfig(USE_QAT_COMPRESS, 0); SetConfig(USE_ZLIB_COMPRESS, 1); break; case QAT: SetConfig(USE_IAA_COMPRESS, 0); + SetConfig(USE_IGZIP_COMPRESS, 0); SetConfig(USE_QAT_COMPRESS, 1); SetConfig(USE_ZLIB_COMPRESS, zlib_fallback ? 1 : 0); break; case IAA: SetConfig(USE_IAA_COMPRESS, 1); + SetConfig(USE_IGZIP_COMPRESS, 0); + SetConfig(USE_QAT_COMPRESS, 0); + SetConfig(USE_ZLIB_COMPRESS, zlib_fallback ? 1 : 0); + break; + case IGZIP: + SetConfig(USE_IGZIP_COMPRESS, 1); + SetConfig(USE_IAA_COMPRESS, 0); SetConfig(USE_QAT_COMPRESS, 0); SetConfig(USE_ZLIB_COMPRESS, zlib_fallback ? 1 : 0); break; @@ -265,16 +314,25 @@ void SetUncompressPath(ExecutionPath path, bool zlib_fallback, switch (path) { case ZLIB: SetConfig(USE_IAA_UNCOMPRESS, 0); + SetConfig(USE_IGZIP_UNCOMPRESS, 0); SetConfig(USE_QAT_UNCOMPRESS, 0); SetConfig(USE_ZLIB_UNCOMPRESS, 1); break; case QAT: SetConfig(USE_IAA_UNCOMPRESS, 0); + SetConfig(USE_IGZIP_UNCOMPRESS, 0); SetConfig(USE_QAT_UNCOMPRESS, 1); SetConfig(USE_ZLIB_UNCOMPRESS, zlib_fallback ? 1 : 0); break; case IAA: SetConfig(USE_IAA_UNCOMPRESS, 1); + SetConfig(USE_IGZIP_UNCOMPRESS, 0); + SetConfig(USE_QAT_UNCOMPRESS, 0); + SetConfig(USE_ZLIB_UNCOMPRESS, zlib_fallback ? 1 : 0); + break; + case IGZIP: + SetConfig(USE_IAA_UNCOMPRESS, 0); + SetConfig(USE_IGZIP_UNCOMPRESS, 1); SetConfig(USE_QAT_UNCOMPRESS, 0); SetConfig(USE_ZLIB_UNCOMPRESS, zlib_fallback ? 1 : 0); break; @@ -331,6 +389,8 @@ struct TestParam { return "QAT"; case IAA: return "IAA"; + case IGZIP: + return "IGZIP"; } return ""; } @@ -590,6 +650,14 @@ TEST_P(ZlibTest, CompressDecompress) { test_param.iaa_prepend_empty_block, test_param.qat_compression_allow_chunking); + // For IGZIP->IAA compatibility checks, force max zlib level so IGZIP uses + // ISA-L level 3 (stricter match selection), which makes long-history + // limitations consistently observable. + const int compression_level = (test_param.execution_path_compress == IGZIP && + test_param.execution_path_uncompress == IAA) + ? 9 + : -1; + size_t input_length = test_param.block_size; BlockCompressibilityType block_type = test_param.block_type; char* input = GenerateBlock(input_length, block_type); @@ -598,9 +666,10 @@ TEST_P(ZlibTest, CompressDecompress) { std::string compressed; size_t output_upper_bound; ExecutionPath execution_path = UNDEFINED; - int ret = ZlibCompress( - input, input_length, &compressed, test_param.window_bits_compress, - test_param.flush_compress, &output_upper_bound, &execution_path); + int ret = ZlibCompressWithLevel( + input, input_length, &compressed, compression_level, + test_param.window_bits_compress, test_param.flush_compress, + &output_upper_bound, &execution_path); VerifyStatIncremented(Statistic::DEFLATE_COUNT); bool compress_fallback_expected = @@ -621,6 +690,8 @@ TEST_P(ZlibTest, CompressDecompress) { VerifyStatIncremented(Statistic::DEFLATE_QAT_COUNT); } else if (test_param.execution_path_compress == IAA) { VerifyStatIncremented(Statistic::DEFLATE_IAA_COUNT); + } else if (test_param.execution_path_compress == IGZIP) { + VerifyStatIncremented(Statistic::DEFLATE_IGZIP_COUNT); } else if (test_param.execution_path_compress == ZLIB) { VerifyStatIncremented(Statistic::DEFLATE_ZLIB_COUNT); } @@ -674,6 +745,9 @@ TEST_P(ZlibTest, CompressDecompress) { VerifyStatIncremented(Statistic::INFLATE_QAT_COUNT); } else if (test_param.execution_path_uncompress == IAA) { VerifyStatIncremented(Statistic::INFLATE_IAA_COUNT); + } else if (test_param.execution_path_uncompress == IGZIP) { + VerifyStatIncrementedUpTo(Statistic::INFLATE_IGZIP_COUNT, + test_param.input_chunks_uncompress); } else if (test_param.execution_path_uncompress == ZLIB) { VerifyStatIncrementedUpTo(Statistic::INFLATE_ZLIB_COUNT, test_param.input_chunks_uncompress); @@ -729,6 +803,10 @@ INSTANTIATE_TEST_SUITE_P( #ifdef USE_IAA , IAA +#endif +#ifdef USE_IGZIP + , + IGZIP #endif ), testing::Values(false, true), @@ -740,6 +818,10 @@ INSTANTIATE_TEST_SUITE_P( #ifdef USE_IAA , IAA +#endif +#ifdef USE_IGZIP + , + IGZIP #endif ), testing::Values(false, true), testing::Values(-15, 15, 31), @@ -827,6 +909,10 @@ INSTANTIATE_TEST_SUITE_P( #ifdef USE_IAA , IAA +#endif +#ifdef USE_IGZIP + , + IGZIP #endif ), testing::Values(false, true), @@ -838,6 +924,10 @@ INSTANTIATE_TEST_SUITE_P( #ifdef USE_IAA , IAA +#endif +#ifdef USE_IGZIP + , + IGZIP #endif ), testing::Values(false, true), testing::Values(15), @@ -923,6 +1013,10 @@ INSTANTIATE_TEST_SUITE_P( #ifdef USE_IAA , IAA +#endif +#ifdef USE_IGZIP + , + IGZIP #endif ), testing::Values(false, true), @@ -934,6 +1028,10 @@ INSTANTIATE_TEST_SUITE_P( #ifdef USE_IAA , IAA +#endif +#ifdef USE_IGZIP + , + IGZIP #endif ), testing::Values(false, true), testing::Values(15), @@ -960,6 +1058,14 @@ TEST_P(ZlibPartialAndMultiStreamTest, CompressDecompressPartialStream) { test_param.iaa_prepend_empty_block, test_param.qat_compression_allow_chunking); + // For IGZIP->IAA compatibility checks, force max zlib level so IGZIP uses + // ISA-L level 3 (stricter match selection), which makes long-history + // limitations consistently observable. + const int compression_level = (test_param.execution_path_compress == IGZIP && + test_param.execution_path_uncompress == IAA) + ? 9 + : -1; + size_t input_length = test_param.block_size; BlockCompressibilityType block_type = test_param.block_type; char* input = GenerateBlock(input_length, block_type); @@ -968,9 +1074,10 @@ TEST_P(ZlibPartialAndMultiStreamTest, CompressDecompressPartialStream) { std::string compressed; size_t output_upper_bound; ExecutionPath execution_path = UNDEFINED; - int ret = ZlibCompress( - input, input_length, &compressed, test_param.window_bits_compress, - test_param.flush_compress, &output_upper_bound, &execution_path); + int ret = ZlibCompressWithLevel( + input, input_length, &compressed, compression_level, + test_param.window_bits_compress, test_param.flush_compress, + &output_upper_bound, &execution_path); bool error_expected = ZlibCompressExpectError(test_param, input_length, output_upper_bound); @@ -998,8 +1105,9 @@ TEST_P(ZlibPartialAndMultiStreamTest, CompressDecompressPartialStream) { window_bits_uncompress, test_param.flush_uncompress, test_param.input_chunks_uncompress, &execution_path); - // Only zlib decompression won't return an error + // zlib and igzip decompression may return partial progress instead of error if (test_param.execution_path_uncompress == ZLIB || + test_param.execution_path_uncompress == IGZIP || test_param.zlib_fallback_uncompress) { ASSERT_EQ(ret, Z_OK); ASSERT_TRUE(uncompressed_length < input_length); @@ -1026,6 +1134,14 @@ TEST_P(ZlibPartialAndMultiStreamTest, CompressDecompressMultiStream) { test_param.iaa_prepend_empty_block, test_param.qat_compression_allow_chunking); + // For IGZIP->IAA compatibility checks, force max zlib level so IGZIP uses + // ISA-L level 3 (stricter match selection), which makes long-history + // limitations consistently observable. + const int compression_level = (test_param.execution_path_compress == IGZIP && + test_param.execution_path_uncompress == IAA) + ? 9 + : -1; + size_t input_length = test_param.block_size; BlockCompressibilityType block_type = test_param.block_type; char* input = GenerateBlock(input_length, block_type); @@ -1036,9 +1152,10 @@ TEST_P(ZlibPartialAndMultiStreamTest, CompressDecompressMultiStream) { size_t input_length1 = input_length / 2; size_t output_upper_bound1; ExecutionPath execution_path = UNDEFINED; - int ret = ZlibCompress( - input, input_length1, &compressed1, test_param.window_bits_compress, - test_param.flush_compress, &output_upper_bound1, &execution_path); + int ret = ZlibCompressWithLevel( + input, input_length1, &compressed1, compression_level, + test_param.window_bits_compress, test_param.flush_compress, + &output_upper_bound1, &execution_path); bool error_expected = ZlibCompressExpectError(test_param, input_length1, output_upper_bound1); @@ -1054,9 +1171,10 @@ TEST_P(ZlibPartialAndMultiStreamTest, CompressDecompressMultiStream) { size_t input_length2 = input_length - input_length / 2; size_t output_upper_bound2; execution_path = UNDEFINED; - ret = ZlibCompress(input + input_length1, input_length2, &compressed2, - test_param.window_bits_compress, test_param.flush_compress, - &output_upper_bound2, &execution_path); + ret = ZlibCompressWithLevel( + input + input_length1, input_length2, &compressed2, compression_level, + test_param.window_bits_compress, test_param.flush_compress, + &output_upper_bound2, &execution_path); error_expected = ZlibCompressExpectError(test_param, input_length2, output_upper_bound2); @@ -1101,8 +1219,9 @@ TEST_P(ZlibPartialAndMultiStreamTest, CompressDecompressMultiStream) { ASSERT_EQ(uncompressed_length, input_length1); ASSERT_TRUE(memcmp(uncompressed, input, uncompressed_length) == 0); - // IAA does not handle concatenated streams - if (test_param.execution_path_uncompress != IAA) { + // IAA/IGZIP may consume bytes beyond first-stream boundary + if (test_param.execution_path_uncompress != IAA && + test_param.execution_path_uncompress != IGZIP) { ASSERT_EQ(input_consumed, compressed1.length()); } } @@ -1111,6 +1230,1638 @@ TEST_P(ZlibPartialAndMultiStreamTest, CompressDecompressMultiStream) { DestroyBlock(input); } +#ifdef USE_IGZIP +TEST(IGZIPInflateRegressionTest, EmptyInputContinuationKeepsIGZIPPath) { + SetCompressPath(ZLIB, false, false, false); + SetUncompressPath(IGZIP, false, false); + + const size_t input_length = 1 << 20; + char* input = GenerateBlock(input_length, compressible_block); + ASSERT_NE(input, nullptr); + + std::string compressed; + size_t output_upper_bound; + ExecutionPath execution_path = UNDEFINED; + int ret = ZlibCompress(input, input_length, &compressed, -15, Z_FINISH, + &output_upper_bound, &execution_path); + ASSERT_EQ(ret, Z_STREAM_END); + ASSERT_EQ(execution_path, ZLIB); + + z_stream stream; + memset(&stream, 0, sizeof(z_stream)); + ASSERT_EQ(inflateInit2(&stream, -15), Z_OK); + + std::vector output_chunk(8192); + stream.next_in = reinterpret_cast(compressed.data()); + stream.avail_in = static_cast(compressed.size()); + + int iter = 0; + int last_ret = Z_OK; + while (iter++ < 2048) { + stream.next_out = reinterpret_cast(output_chunk.data()); + stream.avail_out = static_cast(output_chunk.size()); + last_ret = inflate(&stream, Z_SYNC_FLUSH); + ASSERT_NE(last_ret, Z_DATA_ERROR); + ASSERT_EQ(GetInflateExecutionPath(&stream), IGZIP); + + if (last_ret == Z_STREAM_END) { + break; + } + + if (stream.avail_in == 0 && last_ret == Z_OK) { + break; + } + } + + ASSERT_NE(last_ret, Z_STREAM_END); + ASSERT_EQ(stream.avail_in, 0u); + + stream.next_out = reinterpret_cast(output_chunk.data()); + stream.avail_out = static_cast(output_chunk.size()); + stream.next_in = nullptr; + stream.avail_in = 0; + + int continuation_ret = inflate(&stream, Z_SYNC_FLUSH); + EXPECT_TRUE(continuation_ret == Z_BUF_ERROR || continuation_ret == Z_OK || + continuation_ret == Z_STREAM_END); + EXPECT_EQ(GetInflateExecutionPath(&stream), IGZIP); + + inflateEnd(&stream); + DestroyBlock(input); +} + +TEST(IGZIPInflateRegressionTest, RawContinuationMustNotIncreaseAvailIn) { + SetCompressPath(ZLIB, false, false, false); + SetUncompressPath(IGZIP, false, false); + + const size_t input_length = 16384; + char* input = GenerateBlock(input_length, compressible_block); + ASSERT_NE(input, nullptr); + + std::string compressed; + size_t output_upper_bound; + ExecutionPath execution_path = UNDEFINED; + int ret = ZlibCompress(input, input_length, &compressed, -15, Z_FINISH, + &output_upper_bound, &execution_path); + ASSERT_EQ(ret, Z_STREAM_END); + ASSERT_EQ(execution_path, ZLIB); + + z_stream stream; + memset(&stream, 0, sizeof(z_stream)); + ASSERT_EQ(inflateInit2(&stream, -15), Z_OK); + + std::vector output_chunk(8192); + stream.next_in = reinterpret_cast(compressed.data()); + stream.avail_in = static_cast(compressed.size()); + stream.next_out = reinterpret_cast(output_chunk.data()); + stream.avail_out = static_cast(output_chunk.size()); + + ret = inflate(&stream, Z_SYNC_FLUSH); + ASSERT_EQ(GetInflateExecutionPath(&stream), IGZIP); + ASSERT_TRUE(ret == Z_OK || ret == Z_STREAM_END); + + while (ret == Z_OK && stream.avail_in > 0) { + stream.next_out = reinterpret_cast(output_chunk.data()); + stream.avail_out = static_cast(output_chunk.size()); + ret = inflate(&stream, Z_SYNC_FLUSH); + ASSERT_EQ(GetInflateExecutionPath(&stream), IGZIP); + ASSERT_TRUE(ret == Z_OK || ret == Z_STREAM_END); + } + + ASSERT_EQ(stream.avail_in, 0u); + + stream.next_out = reinterpret_cast(output_chunk.data()); + stream.avail_out = static_cast(output_chunk.size()); + stream.next_in = nullptr; + stream.avail_in = 0; + ret = inflate(&stream, Z_SYNC_FLUSH); + ASSERT_TRUE(ret == Z_BUF_ERROR || ret == Z_OK || ret == Z_STREAM_END); + ASSERT_EQ(GetInflateExecutionPath(&stream), IGZIP); + + uint8_t one_byte = 0; + stream.next_in = &one_byte; + stream.avail_in = 1; + stream.next_out = reinterpret_cast(output_chunk.data()); + stream.avail_out = static_cast(output_chunk.size()); + + ret = inflate(&stream, Z_SYNC_FLUSH); + EXPECT_NE(ret, Z_DATA_ERROR); + EXPECT_LE(stream.avail_in, 1u); + + inflateEnd(&stream); + DestroyBlock(input); +} + +TEST(IGZIPInflateRegressionTest, RawTrailingByteMustNotIncreaseAvailIn) { + SetCompressPath(ZLIB, false, false, false); + SetUncompressPath(IGZIP, false, false); + + const size_t input_length = 4096; + char* input = GenerateBlock(input_length, compressible_block); + ASSERT_NE(input, nullptr); + + std::string compressed; + size_t output_upper_bound; + ExecutionPath execution_path = UNDEFINED; + int ret = ZlibCompress(input, input_length, &compressed, -15, Z_FINISH, + &output_upper_bound, &execution_path); + ASSERT_EQ(ret, Z_STREAM_END); + ASSERT_EQ(execution_path, ZLIB); + + std::string compressed_with_trailing = compressed; + compressed_with_trailing.push_back('\0'); + + z_stream stream; + memset(&stream, 0, sizeof(z_stream)); + ASSERT_EQ(inflateInit2(&stream, -15), Z_OK); + + std::vector uncompressed(input_length * 2); + stream.next_in = reinterpret_cast(compressed_with_trailing.data()); + stream.avail_in = static_cast(compressed_with_trailing.size()); + stream.next_out = reinterpret_cast(uncompressed.data()); + stream.avail_out = static_cast(uncompressed.size()); + + ret = inflate(&stream, Z_SYNC_FLUSH); + EXPECT_NE(ret, Z_DATA_ERROR); + EXPECT_EQ(GetInflateExecutionPath(&stream), IGZIP); + + // For valid raw-deflate stream with one extra byte, inflate may leave that + // byte unconsumed, but avail_in must never increase. + EXPECT_LE(stream.avail_in, 1u); + + inflateEnd(&stream); + DestroyBlock(input); +} + +TEST(IGZIPInflateRegressionTest, + RawOneByteContinuationMustNotIncreaseAvailInAcrossSizes) { + SetCompressPath(ZLIB, false, false, false); + SetUncompressPath(IGZIP, false, false); + + std::vector input_sizes = {1, 2, 3, 7, 8, 15, 16, + 31, 32, 63, 64, 127, 128, 255, + 256, 511, 512, 1023, 1024, 2047, 2048, + 4095, 4096, 8191, 8192, 16384, 32768}; + + for (size_t input_length : input_sizes) { + std::vector input(input_length); + for (size_t i = 0; i < input_length; ++i) { + input[i] = static_cast((i * 131u + 17u) & 0xFFu); + } + + std::string compressed; + size_t output_upper_bound; + ExecutionPath execution_path = UNDEFINED; + int ret = ZlibCompress(input.data(), input_length, &compressed, -15, + Z_FINISH, &output_upper_bound, &execution_path); + ASSERT_EQ(ret, Z_STREAM_END); + ASSERT_EQ(execution_path, ZLIB); + + z_stream stream; + memset(&stream, 0, sizeof(z_stream)); + ASSERT_EQ(inflateInit2(&stream, -15), Z_OK); + + std::vector output_chunk(1024); + stream.next_in = reinterpret_cast(compressed.data()); + stream.avail_in = static_cast(compressed.size()); + + int last_ret = Z_OK; + for (int iter = 0; iter < 4096; ++iter) { + stream.next_out = reinterpret_cast(output_chunk.data()); + stream.avail_out = static_cast(output_chunk.size()); + last_ret = inflate(&stream, Z_SYNC_FLUSH); + ASSERT_EQ(GetInflateExecutionPath(&stream), IGZIP); + ASSERT_NE(last_ret, Z_DATA_ERROR); + + if (stream.avail_in == 0 || last_ret == Z_STREAM_END) { + break; + } + } + + ASSERT_EQ(stream.avail_in, 0u); + + stream.next_out = reinterpret_cast(output_chunk.data()); + stream.avail_out = static_cast(output_chunk.size()); + stream.next_in = nullptr; + stream.avail_in = 0; + int empty_ret = inflate(&stream, Z_SYNC_FLUSH); + EXPECT_TRUE(empty_ret == Z_BUF_ERROR || empty_ret == Z_OK || + empty_ret == Z_STREAM_END); + EXPECT_EQ(GetInflateExecutionPath(&stream), IGZIP); + + uint8_t trailing_byte = 0; + stream.next_in = &trailing_byte; + stream.avail_in = 1; + stream.next_out = reinterpret_cast(output_chunk.data()); + stream.avail_out = static_cast(output_chunk.size()); + + int one_byte_ret = inflate(&stream, Z_SYNC_FLUSH); + EXPECT_NE(one_byte_ret, Z_DATA_ERROR); + EXPECT_LE(stream.avail_in, 1u) << "input_length=" << input_length; + + inflateEnd(&stream); + } +} + +TEST(IGZIPInflateRegressionTest, + TinyRawEntryMustNotOverconsumePastStreamBoundary) { + SetCompressPath(ZLIB, false, false, false); + SetUncompressPath(IGZIP, false, false); + + // Empty raw-deflate entries are often tiny (commonly 2 bytes). The + // decompressor must stop exactly at stream end and leave trailing bytes for + // the caller. + const std::string empty_payload; + std::string compressed; + size_t output_upper_bound; + ExecutionPath execution_path = UNDEFINED; + int ret = + ZlibCompress(empty_payload.data(), empty_payload.size(), &compressed, -15, + Z_FINISH, &output_upper_bound, &execution_path); + ASSERT_EQ(ret, Z_STREAM_END); + ASSERT_EQ(execution_path, ZLIB); + ASSERT_GT(compressed.size(), 0u); + + const size_t trailing_len = 510; + std::string input = compressed; + input.append(trailing_len, static_cast(0xA5)); + + z_stream stream; + memset(&stream, 0, sizeof(z_stream)); + ASSERT_EQ(inflateInit2(&stream, -15), Z_OK); + + std::vector output(512); + stream.next_in = reinterpret_cast(input.data()); + stream.avail_in = static_cast(input.size()); + stream.next_out = reinterpret_cast(output.data()); + stream.avail_out = static_cast(output.size()); + + ret = inflate(&stream, Z_SYNC_FLUSH); + const ExecutionPath observed_path = GetInflateExecutionPath(&stream); + ASSERT_TRUE(observed_path == IGZIP || observed_path == ZLIB); + ASSERT_NE(ret, Z_DATA_ERROR); + + // No output is expected for empty payload. Most importantly, all trailing + // bytes must remain unconsumed for the caller. + EXPECT_EQ(output.size() - stream.avail_out, 0u); + EXPECT_EQ(stream.avail_in, trailing_len) + << "compressed_size=" << compressed.size(); + + inflateEnd(&stream); +} + +TEST(IGZIPInflateRegressionTest, RawStreamEndMustPreserveEightTrailingBytes) { + SetCompressPath(ZLIB, false, false, false); + SetUncompressPath(IGZIP, false, false); + + const size_t input_length = 32768; + char* input = GenerateBlock(input_length, compressible_block); + ASSERT_NE(input, nullptr); + + std::string compressed; + size_t output_upper_bound; + ExecutionPath execution_path = UNDEFINED; + int ret = ZlibCompress(input, input_length, &compressed, -15, Z_FINISH, + &output_upper_bound, &execution_path); + ASSERT_EQ(ret, Z_STREAM_END); + ASSERT_EQ(execution_path, ZLIB); + + std::string with_trailing = compressed; + with_trailing.append("ABCDEFGH", 8); + + z_stream stream; + memset(&stream, 0, sizeof(z_stream)); + ASSERT_EQ(inflateInit2(&stream, -15), Z_OK); + + std::vector output(input_length * 2); + stream.next_in = reinterpret_cast(with_trailing.data()); + stream.avail_in = static_cast(with_trailing.size()); + + int last_ret = Z_OK; + for (int iter = 0; iter < 64; ++iter) { + stream.next_out = reinterpret_cast(output.data()); + stream.avail_out = static_cast(output.size()); + last_ret = inflate(&stream, Z_SYNC_FLUSH); + const ExecutionPath path = GetInflateExecutionPath(&stream); + ASSERT_TRUE(path == IGZIP || path == ZLIB); + ASSERT_NE(last_ret, Z_DATA_ERROR); + if (last_ret == Z_STREAM_END) { + break; + } + if (stream.avail_out > 0 && stream.avail_in == 0) { + break; + } + } + + EXPECT_EQ(last_ret, Z_STREAM_END); + EXPECT_EQ(stream.avail_in, 8u); + + inflateEnd(&stream); + DestroyBlock(input); +} + +TEST(IGZIPInflateRegressionTest, RawSplitInputDefersCorrectionUntilStreamEnd) { + SetCompressPath(ZLIB, false, false, false); + SetUncompressPath(IGZIP, false, false); + + const size_t input_length = 65536; + char* input = GenerateBlock(input_length, compressible_block); + ASSERT_NE(input, nullptr); + + std::string compressed; + size_t output_upper_bound; + ExecutionPath execution_path = UNDEFINED; + int ret = ZlibCompress(input, input_length, &compressed, -15, Z_FINISH, + &output_upper_bound, &execution_path); + ASSERT_EQ(ret, Z_STREAM_END); + ASSERT_EQ(execution_path, ZLIB); + ASSERT_GT(compressed.size(), 2u); + + std::string with_trailing = compressed; + with_trailing.append("ABCDEFGH", 8); + + z_stream stream; + memset(&stream, 0, sizeof(z_stream)); + ASSERT_EQ(inflateInit2(&stream, -15), Z_OK); + + std::vector output(input_length * 2); + const size_t split_offset = compressed.size() - 1; + + stream.next_in = reinterpret_cast(with_trailing.data()); + stream.avail_in = static_cast(split_offset); + stream.next_out = reinterpret_cast(output.data()); + stream.avail_out = static_cast(output.size()); + + ret = inflate(&stream, Z_SYNC_FLUSH); + { + const ExecutionPath path = GetInflateExecutionPath(&stream); + ASSERT_TRUE(path == IGZIP || path == ZLIB); + } + ASSERT_NE(ret, Z_DATA_ERROR); + + stream.next_in = + reinterpret_cast(with_trailing.data() + split_offset); + stream.avail_in = + static_cast(with_trailing.size() - split_offset); + + int last_ret = ret; + for (int iter = 0; iter < 64; ++iter) { + stream.next_out = reinterpret_cast(output.data()); + stream.avail_out = static_cast(output.size()); + last_ret = inflate(&stream, Z_SYNC_FLUSH); + const ExecutionPath path = GetInflateExecutionPath(&stream); + ASSERT_TRUE(path == IGZIP || path == ZLIB); + ASSERT_NE(last_ret, Z_DATA_ERROR); + if (last_ret == Z_STREAM_END) { + break; + } + if (stream.avail_out > 0 && stream.avail_in == 0) { + break; + } + } + + EXPECT_EQ(last_ret, Z_STREAM_END); + EXPECT_EQ(stream.avail_in, 8u); + + inflateEnd(&stream); + DestroyBlock(input); +} + +TEST(IGZIPDeflateRegressionTest, + RepeatedFinishWithEmptyInputMustReturnStreamEnd) { + SetCompressPath(IGZIP, true, false, false); + SetUncompressPath(ZLIB, false, false); + if (GetConfig(USE_ZLIB_COMPRESS) == 0) { + GTEST_SKIP() << "USE_ZLIB_COMPRESS=0 disables fallback-first contract"; + } + + const char* input = "igzip-finish-regression-payload"; + const size_t input_length = strlen(input); + + z_stream stream; + memset(&stream, 0, sizeof(z_stream)); + ASSERT_EQ(deflateInit2(&stream, Z_DEFAULT_COMPRESSION, Z_DEFLATED, 15, 8, + Z_DEFAULT_STRATEGY), + Z_OK); + + std::vector output(4096); + stream.next_in = reinterpret_cast(const_cast(input)); + stream.avail_in = static_cast(input_length); + + int ret = Z_OK; + for (int iter = 0; iter < 32; ++iter) { + stream.next_out = reinterpret_cast(output.data()); + stream.avail_out = static_cast(output.size()); + ret = deflate(&stream, Z_FINISH); + const ExecutionPath path = GetDeflateExecutionPath(&stream); + ASSERT_TRUE(path == IGZIP || path == ZLIB); + ASSERT_NE(ret, Z_DATA_ERROR); + if (ret == Z_STREAM_END) { + break; + } + } + + ASSERT_EQ(ret, Z_STREAM_END); + + for (int iter = 0; iter < 64; ++iter) { + stream.next_in = nullptr; + stream.avail_in = 0; + stream.next_out = reinterpret_cast(output.data()); + stream.avail_out = static_cast(output.size()); + + int finish_ret = deflate(&stream, Z_FINISH); + EXPECT_EQ(finish_ret, Z_STREAM_END) << "iter=" << iter; + const ExecutionPath path = GetDeflateExecutionPath(&stream); + EXPECT_TRUE(path == IGZIP || path == ZLIB) << "iter=" << iter; + } + + deflateEnd(&stream); +} + +TEST(IGZIPDeflateRegressionTest, ResetMustNotStallSyncFlushOnSameStream) { + SetCompressPath(IGZIP, true, false, false); + SetUncompressPath(ZLIB, false, false); + if (GetConfig(USE_ZLIB_COMPRESS) == 0) { + GTEST_SKIP() << "USE_ZLIB_COMPRESS=0 disables fallback-first contract"; + } + + z_stream stream; + memset(&stream, 0, sizeof(z_stream)); + ASSERT_EQ(deflateInit2(&stream, Z_DEFAULT_COMPRESSION, Z_DEFLATED, 15, 8, + Z_DEFAULT_STRATEGY), + Z_OK); + + std::vector output(4096); + std::vector input(512, 'A'); + + for (int cycle = 0; cycle < 20; ++cycle) { + stream.next_in = reinterpret_cast(input.data()); + stream.avail_in = static_cast(input.size()); + stream.next_out = reinterpret_cast(output.data()); + stream.avail_out = static_cast(output.size()); + + int ret = deflate(&stream, Z_NO_FLUSH); + ASSERT_TRUE(ret == Z_OK || ret == Z_BUF_ERROR) << "cycle=" << cycle; + ASSERT_EQ(GetDeflateExecutionPath(&stream), IGZIP) << "cycle=" << cycle; + + stream.next_in = nullptr; + stream.avail_in = 0; + stream.next_out = reinterpret_cast(output.data()); + stream.avail_out = static_cast(output.size()); + + ret = deflate(&stream, Z_SYNC_FLUSH); + ASSERT_EQ(ret, Z_OK) << "cycle=" << cycle; + ASSERT_LT(stream.avail_out, output.size()) << "cycle=" << cycle; + ASSERT_EQ(GetDeflateExecutionPath(&stream), IGZIP) << "cycle=" << cycle; + + stream.next_in = nullptr; + stream.avail_in = 0; + stream.next_out = reinterpret_cast(output.data()); + stream.avail_out = static_cast(output.size()); + + ret = deflate(&stream, Z_FINISH); + ASSERT_EQ(ret, Z_STREAM_END) << "cycle=" << cycle; + + ASSERT_EQ(deflateReset(&stream), Z_OK) << "cycle=" << cycle; + } + + deflateEnd(&stream); +} + +TEST(IGZIPDeflateRegressionTest, + RepeatedEmptySyncFlushMustEventuallyReportNoProgress) { + SetCompressPath(IGZIP, true, false, false); + SetUncompressPath(ZLIB, false, false); + if (GetConfig(USE_ZLIB_COMPRESS) == 0) { + GTEST_SKIP() << "USE_ZLIB_COMPRESS=0 disables fallback-first contract"; + } + + z_stream stream; + memset(&stream, 0, sizeof(z_stream)); + ASSERT_EQ(deflateInit2(&stream, Z_DEFAULT_COMPRESSION, Z_DEFLATED, 15, 8, + Z_DEFAULT_STRATEGY), + Z_OK); + + std::vector output(4096); + std::vector input(1024, 'B'); + + stream.next_in = reinterpret_cast(input.data()); + stream.avail_in = static_cast(input.size()); + stream.next_out = reinterpret_cast(output.data()); + stream.avail_out = static_cast(output.size()); + int ret = deflate(&stream, Z_NO_FLUSH); + ASSERT_TRUE(ret == Z_OK || ret == Z_BUF_ERROR); + ASSERT_EQ(GetDeflateExecutionPath(&stream), IGZIP); + + bool observed_buf_error = false; + for (int iter = 0; iter < 128; ++iter) { + stream.next_in = nullptr; + stream.avail_in = 0; + stream.next_out = reinterpret_cast(output.data()); + stream.avail_out = static_cast(output.size()); + + ret = deflate(&stream, Z_SYNC_FLUSH); + ASSERT_EQ(GetDeflateExecutionPath(&stream), IGZIP) << "iter=" << iter; + ASSERT_NE(ret, Z_DATA_ERROR) << "iter=" << iter; + + if (ret == Z_BUF_ERROR) { + observed_buf_error = true; + break; + } + } + + EXPECT_TRUE(observed_buf_error) + << "Repeated empty Z_SYNC_FLUSH calls must eventually stop producing " + << "new bytes and return Z_BUF_ERROR"; + + stream.next_in = nullptr; + stream.avail_in = 0; + stream.next_out = reinterpret_cast(output.data()); + stream.avail_out = static_cast(output.size()); + ret = deflate(&stream, Z_FINISH); + EXPECT_EQ(ret, Z_STREAM_END); + + deflateEnd(&stream); +} + +// Regression test for: deflateReset on a reused IGZIP stream must restore the +// zlib header (gzip_flag = IGZIP_ZLIB) so that each independent chunk is +// self-contained and decompressible by a fresh zlib inflater. Without the +// fix, isal_deflate_reset preserved gzip_flag = IGZIP_ZLIB_NO_HDR (4) and the +// second and subsequent chunks were emitted without a zlib header, causing +// Java's Inflater (nowrap=false) to report "incorrect header check". +TEST(IGZIPDeflateRegressionTest, + ResetMustRestoreZlibHeaderForSubsequentChunks) { + SetCompressPath(IGZIP, false, false, false); + SetUncompressPath(ZLIB, false, false); + + z_stream cstream; + memset(&cstream, 0, sizeof(z_stream)); + ASSERT_EQ(deflateInit2(&cstream, Z_DEFAULT_COMPRESSION, Z_DEFLATED, + /*windowBits=*/15, /*memLevel=*/8, Z_DEFAULT_STRATEGY), + Z_OK); + + const int kChunks = 5; + const int kChunkSize = 16384; + const int kCompBound = deflateBound(&cstream, kChunkSize); + + for (int chunk = 0; chunk < kChunks; ++chunk) { + // Each chunk is independent data compressed with a fresh IGZIP stream + // (via deflateReset). + std::vector input(kChunkSize, + static_cast('A' + chunk % 26)); + std::vector compressed(kCompBound); + + cstream.next_in = input.data(); + cstream.avail_in = static_cast(input.size()); + cstream.next_out = compressed.data(); + cstream.avail_out = static_cast(compressed.size()); + + ASSERT_EQ(deflate(&cstream, Z_FINISH), Z_STREAM_END) << "chunk=" << chunk; + ASSERT_EQ(GetDeflateExecutionPath(&cstream), IGZIP) << "chunk=" << chunk; + + const size_t compressed_size = compressed.size() - cstream.avail_out; + + // Decompress with a fresh zlib inflater — requires a valid zlib header. + std::vector decompressed(kChunkSize); + z_stream dstream; + memset(&dstream, 0, sizeof(z_stream)); + ASSERT_EQ(inflateInit(&dstream), Z_OK) << "chunk=" << chunk; + dstream.next_in = compressed.data(); + dstream.avail_in = static_cast(compressed_size); + dstream.next_out = decompressed.data(); + dstream.avail_out = static_cast(decompressed.size()); + ASSERT_EQ(inflate(&dstream, Z_FINISH), Z_STREAM_END) + << "chunk=" << chunk + << ": decompression failed (missing zlib header after deflateReset?)"; + ASSERT_EQ(decompressed, input) << "chunk=" << chunk; + inflateEnd(&dstream); + + ASSERT_EQ(deflateReset(&cstream), Z_OK) << "chunk=" << chunk; + } + + deflateEnd(&cstream); +} + +// Regression: after inflate() returns Z_STREAM_END and avail_in is left +// pointing at trailing bytes (Bug 1 fix), a subsequent inflate() call with +// those trailing bytes must return Z_STREAM_END, not Z_BUF_ERROR. The IGZIP +// stream is still active (not freed), and isal_inflate on a finished stream +// returns ISAL_END_INPUT with 0 consumed — the avail_in>0 path in inflate() +// previously fell through to Z_BUF_ERROR when input_len==output_len==0. +TEST(IGZIPInflateRegressionTest, + InflateAfterStreamEndWithTrailingBytesReturnsStreamEnd) { + SetCompressPath(IGZIP, false, false, false); + SetUncompressPath(IGZIP, false, false); + + // Compress with IGZIP (raw deflate). + z_stream cstream; + memset(&cstream, 0, sizeof(z_stream)); + const size_t kPayloadSize = 4096; + ASSERT_EQ(deflateInit2(&cstream, Z_DEFAULT_COMPRESSION, Z_DEFLATED, -15, 8, + Z_DEFAULT_STRATEGY), + Z_OK); + std::vector payload(kPayloadSize, 'Q'); + std::vector compressed(deflateBound(&cstream, kPayloadSize)); + cstream.next_in = payload.data(); + cstream.avail_in = static_cast(kPayloadSize); + cstream.next_out = compressed.data(); + cstream.avail_out = static_cast(compressed.size()); + ASSERT_EQ(deflate(&cstream, Z_FINISH), Z_STREAM_END); + ASSERT_EQ(GetDeflateExecutionPath(&cstream), IGZIP); + const size_t compressed_size = compressed.size() - cstream.avail_out; + deflateEnd(&cstream); + + // Build input buffer: compressed stream + 8 trailing bytes. + std::vector input_buf(compressed.begin(), + compressed.begin() + compressed_size); + input_buf.insert(input_buf.end(), {'A', 'B', 'C', 'D', 'E', 'F', 'G', 'H'}); + + z_stream dstream; + memset(&dstream, 0, sizeof(z_stream)); + ASSERT_EQ(inflateInit2(&dstream, -15), Z_OK); + std::vector output(kPayloadSize * 2); + + dstream.next_in = input_buf.data(); + dstream.avail_in = static_cast(input_buf.size()); + dstream.next_out = output.data(); + dstream.avail_out = static_cast(output.size()); + + // First call: IGZIP decompresses the stream and leaves 8 trailing bytes. + ASSERT_EQ(inflate(&dstream, Z_SYNC_FLUSH), Z_STREAM_END); + ASSERT_EQ(GetInflateExecutionPath(&dstream), IGZIP); + ASSERT_EQ(dstream.avail_in, 8u) << "Bug 1 fix should rewind trailing bytes"; + + // Second call: avail_in==8 > 0, IGZIP stream still active. isal_inflate on + // a finished stream produces no I/O — must return Z_STREAM_END, not + // Z_BUF_ERROR. + dstream.next_out = output.data(); + dstream.avail_out = static_cast(output.size()); + EXPECT_EQ(inflate(&dstream, Z_SYNC_FLUSH), Z_STREAM_END); + + inflateEnd(&dstream); +} + +// Regression: after inflate() returns Z_STREAM_END with avail_in==0, a +// subsequent inflate() call with avail_in==0 (IGZIPHandleActiveStreamNoInput +// path) must return Z_STREAM_END, not Z_BUF_ERROR. +TEST(IGZIPInflateRegressionTest, + InflateWithZeroInputAfterStreamEndReturnsStreamEnd) { + SetCompressPath(IGZIP, false, false, false); + SetUncompressPath(IGZIP, false, false); + + z_stream cstream; + memset(&cstream, 0, sizeof(z_stream)); + const size_t kPayloadSize = 4096; + ASSERT_EQ(deflateInit2(&cstream, Z_DEFAULT_COMPRESSION, Z_DEFLATED, -15, 8, + Z_DEFAULT_STRATEGY), + Z_OK); + std::vector payload(kPayloadSize, 'Q'); + std::vector compressed(deflateBound(&cstream, kPayloadSize)); + cstream.next_in = payload.data(); + cstream.avail_in = static_cast(kPayloadSize); + cstream.next_out = compressed.data(); + cstream.avail_out = static_cast(compressed.size()); + ASSERT_EQ(deflate(&cstream, Z_FINISH), Z_STREAM_END); + ASSERT_EQ(GetDeflateExecutionPath(&cstream), IGZIP); + const size_t compressed_size = compressed.size() - cstream.avail_out; + deflateEnd(&cstream); + + z_stream dstream; + memset(&dstream, 0, sizeof(z_stream)); + ASSERT_EQ(inflateInit2(&dstream, -15), Z_OK); + std::vector output(kPayloadSize * 2); + + dstream.next_in = compressed.data(); + dstream.avail_in = static_cast(compressed_size); + dstream.next_out = output.data(); + dstream.avail_out = static_cast(output.size()); + + // First call: consumes all compressed bytes exactly, avail_in drops to 0. + ASSERT_EQ(inflate(&dstream, Z_SYNC_FLUSH), Z_STREAM_END); + ASSERT_EQ(GetInflateExecutionPath(&dstream), IGZIP); + ASSERT_EQ(dstream.avail_in, 0u); + + // Second call: avail_in==0, IGZIP stream still active + // (IGZIPHandleActiveStreamNoInput path). isal_inflate on a finished stream + // with no input produces no output — must return Z_STREAM_END, not + // Z_BUF_ERROR. + dstream.next_out = output.data(); + dstream.avail_out = static_cast(output.size()); + EXPECT_EQ(inflate(&dstream, Z_SYNC_FLUSH), Z_STREAM_END); + + inflateEnd(&dstream); +} + +TEST(IGZIPDeflateRegressionTest, DictionaryStreamMustStayOnZlibAcrossReset) { + SetCompressPath(IGZIP, false, false, false); + SetUncompressPath(ZLIB, false, false); + SetConfig(IGNORE_ZLIB_DICTIONARY, 0); + + z_stream stream; + memset(&stream, 0, sizeof(z_stream)); + ASSERT_EQ(deflateInit2(&stream, Z_DEFAULT_COMPRESSION, Z_DEFLATED, 15, 8, + Z_DEFAULT_STRATEGY), + Z_OK); + + const char dict[] = "0123456789abcdef"; + ASSERT_EQ(deflateSetDictionary(&stream, reinterpret_cast(dict), + static_cast(sizeof(dict) - 1)), + Z_OK); + EXPECT_EQ(GetDeflateExecutionPath(&stream), ZLIB); + + std::vector out1(4096); + const char* msg1 = "dictionary-stream-first-message"; + stream.next_in = reinterpret_cast(const_cast(msg1)); + stream.avail_in = static_cast(strlen(msg1)); + int ret = Z_OK; + for (int iter = 0; iter < 16; ++iter) { + stream.next_out = reinterpret_cast(out1.data()); + stream.avail_out = static_cast(out1.size()); + ret = deflate(&stream, Z_FINISH); + ASSERT_NE(ret, Z_DATA_ERROR); + if (ret == Z_STREAM_END) { + break; + } + } + ASSERT_EQ(ret, Z_STREAM_END); + EXPECT_EQ(GetDeflateExecutionPath(&stream), ZLIB); + const size_t out1_size = out1.size() - stream.avail_out; + + ASSERT_EQ(deflateReset(&stream), Z_OK); + + // Re-set dictionary after reset (standard zlib API usage). + ASSERT_EQ(deflateSetDictionary(&stream, reinterpret_cast(dict), + static_cast(sizeof(dict) - 1)), + Z_OK); + EXPECT_EQ(GetDeflateExecutionPath(&stream), ZLIB); + + std::vector out2(4096); + const char* msg2 = "dictionary-stream-second-message"; + stream.next_in = reinterpret_cast(const_cast(msg2)); + stream.avail_in = static_cast(strlen(msg2)); + ret = Z_OK; + for (int iter = 0; iter < 16; ++iter) { + stream.next_out = reinterpret_cast(out2.data()); + stream.avail_out = static_cast(out2.size()); + ret = deflate(&stream, Z_FINISH); + ASSERT_NE(ret, Z_DATA_ERROR); + if (ret == Z_STREAM_END) { + break; + } + } + ASSERT_EQ(ret, Z_STREAM_END); + EXPECT_EQ(GetDeflateExecutionPath(&stream), ZLIB); + const size_t out2_size = out2.size() - stream.avail_out; + + z_stream verify; + memset(&verify, 0, sizeof(verify)); + ASSERT_EQ(inflateInit2(&verify, 15), Z_OK); + + std::vector verify_out(1024); + verify.next_in = reinterpret_cast(out2.data()); + verify.avail_in = static_cast(out2_size); + verify.next_out = reinterpret_cast(verify_out.data()); + verify.avail_out = static_cast(verify_out.size()); + + ret = inflate(&verify, Z_NO_FLUSH); + EXPECT_EQ(ret, Z_NEED_DICT); + + ASSERT_EQ(inflateSetDictionary(&verify, reinterpret_cast(dict), + static_cast(sizeof(dict) - 1)), + Z_OK); + ret = inflate(&verify, Z_FINISH); + EXPECT_EQ(ret, Z_STREAM_END); + + inflateEnd(&verify); + + EXPECT_GT(out1_size, 0u); + EXPECT_GT(out2_size, 0u); + + deflateEnd(&stream); +} + +TEST(IGZIPDeflateRegressionTest, + ParallelDictionaryResetStreamsRemainRoundTripSafe) { + SetCompressPath(IGZIP, false, false, false); + SetUncompressPath(ZLIB, false, false); + SetConfig(IGNORE_ZLIB_DICTIONARY, 0); + + const std::string dictionary = + "abcdefghijklmnopqrstuvwxyz0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZ"; + const int thread_count = 4; + const int iterations_per_thread = 120; + + std::atomic failed{false}; + std::string failure_reason; + std::mutex failure_mutex; + + auto set_failure = [&](const std::string& message) { + bool expected = false; + if (failed.compare_exchange_strong(expected, true)) { + std::lock_guard lock(failure_mutex); + failure_reason = message; + } + }; + + auto worker = [&](int worker_id) { + z_stream cstream; + memset(&cstream, 0, sizeof(cstream)); + int ret = deflateInit2(&cstream, Z_DEFAULT_COMPRESSION, Z_DEFLATED, 15, 8, + Z_DEFAULT_STRATEGY); + if (ret != Z_OK) { + set_failure("deflateInit2 failed"); + return; + } + + for (int i = 0; i < iterations_per_thread && !failed.load(); ++i) { + if (deflateReset(&cstream) != Z_OK) { + set_failure("deflateReset failed"); + break; + } + + ret = deflateSetDictionary( + &cstream, reinterpret_cast(dictionary.data()), + static_cast(dictionary.size())); + if (ret != Z_OK) { + set_failure("deflateSetDictionary failed"); + break; + } + + if (GetDeflateExecutionPath(&cstream) != ZLIB) { + set_failure("dictionary stream did not stay on zlib"); + break; + } + + std::string input; + input.reserve(64 * 1024); + for (int j = 0; j < 1024; ++j) { + input += "tid="; + input += std::to_string(worker_id); + input += " iter="; + input += std::to_string(i); + input += " row="; + input += std::to_string(j); + input += " payload="; + input += dictionary; + input += "\n"; + } + + std::vector compressed(compressBound(input.size())); + cstream.next_in = + reinterpret_cast(const_cast(input.data())); + cstream.avail_in = static_cast(input.size()); + cstream.next_out = compressed.data(); + cstream.avail_out = static_cast(compressed.size()); + + int def_ret = Z_OK; + for (int guard = 0; guard < 64; ++guard) { + def_ret = deflate(&cstream, Z_FINISH); + if (def_ret == Z_STREAM_END) { + break; + } + if (def_ret != Z_OK && def_ret != Z_BUF_ERROR) { + break; + } + } + if (def_ret != Z_STREAM_END) { + set_failure("deflate did not reach Z_STREAM_END"); + break; + } + + const size_t compressed_size = compressed.size() - cstream.avail_out; + if (compressed_size == 0) { + set_failure("compressed payload was empty"); + break; + } + + z_stream dstream; + memset(&dstream, 0, sizeof(dstream)); + ret = inflateInit2(&dstream, 15); + if (ret != Z_OK) { + set_failure("inflateInit2 failed"); + break; + } + + std::vector output(input.size() + 1024); + dstream.next_in = compressed.data(); + dstream.avail_in = static_cast(compressed_size); + dstream.next_out = output.data(); + dstream.avail_out = static_cast(output.size()); + + ret = inflate(&dstream, Z_NO_FLUSH); + if (ret == Z_NEED_DICT) { + ret = inflateSetDictionary( + &dstream, reinterpret_cast(dictionary.data()), + static_cast(dictionary.size())); + if (ret == Z_OK) { + ret = inflate(&dstream, Z_FINISH); + } + } + + const size_t produced = output.size() - dstream.avail_out; + const bool output_matches = + (produced == input.size()) && + (memcmp(output.data(), input.data(), input.size()) == 0); + + inflateEnd(&dstream); + + if (ret != Z_STREAM_END) { + set_failure("inflate did not reach Z_STREAM_END"); + break; + } + if (!output_matches) { + set_failure("round-trip data mismatch"); + break; + } + } + + deflateEnd(&cstream); + }; + + std::vector workers; + workers.reserve(thread_count); + for (int worker_id = 0; worker_id < thread_count; ++worker_id) { + workers.emplace_back(worker, worker_id); + } + for (auto& worker_thread : workers) { + worker_thread.join(); + } + + ASSERT_FALSE(failed.load()) << failure_reason; +} + +TEST(IGZIPDeflateRegressionTest, + IGZIPDictionaryOutputMustBeZlibDictionaryCompatible) { + SetCompressPath(IGZIP, false, false, false); + SetUncompressPath(ZLIB, false, false); + SetConfig(IGNORE_ZLIB_DICTIONARY, 0); + + const std::string dictionary = + "abcdefghijklmnopqrstuvwxyz0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZ"; + std::string input; + for (int i = 0; i < 512; ++i) { + input += "doc:"; + input += dictionary; + input += "|"; + } + + z_stream cstream; + memset(&cstream, 0, sizeof(cstream)); + ASSERT_EQ(deflateInit2(&cstream, Z_DEFAULT_COMPRESSION, Z_DEFLATED, 15, 8, + Z_DEFAULT_STRATEGY), + Z_OK); + ASSERT_EQ(deflateSetDictionary( + &cstream, reinterpret_cast(dictionary.data()), + static_cast(dictionary.size())), + Z_OK); + ASSERT_EQ(GetDeflateExecutionPath(&cstream), ZLIB); + + std::vector compressed(compressBound(input.size())); + cstream.next_in = reinterpret_cast(const_cast(input.data())); + cstream.avail_in = static_cast(input.size()); + cstream.next_out = compressed.data(); + cstream.avail_out = static_cast(compressed.size()); + + int ret = Z_OK; + for (int iter = 0; iter < 32; ++iter) { + ret = deflate(&cstream, Z_FINISH); + if (ret == Z_STREAM_END) { + break; + } + } + ASSERT_EQ(ret, Z_STREAM_END); + const size_t compressed_size = compressed.size() - cstream.avail_out; + deflateEnd(&cstream); + + z_stream dstream; + memset(&dstream, 0, sizeof(dstream)); + ASSERT_EQ(inflateInit2(&dstream, 15), Z_OK); + + std::vector uncompressed(input.size() + 1024); + dstream.next_in = compressed.data(); + dstream.avail_in = static_cast(compressed_size); + dstream.next_out = uncompressed.data(); + dstream.avail_out = static_cast(uncompressed.size()); + + ret = inflate(&dstream, Z_NO_FLUSH); + if (ret == Z_NEED_DICT) { + ASSERT_EQ(inflateSetDictionary( + &dstream, reinterpret_cast(dictionary.data()), + static_cast(dictionary.size())), + Z_OK); + ret = inflate(&dstream, Z_FINISH); + } + + EXPECT_EQ(ret, Z_STREAM_END); + const size_t produced = uncompressed.size() - dstream.avail_out; + ASSERT_EQ(produced, input.size()); + EXPECT_EQ(memcmp(uncompressed.data(), input.data(), input.size()), 0); + + inflateEnd(&dstream); +} + +TEST(IGZIPDeflateRegressionTest, + FinishWithTinyOutputBufferMustNotTruncateStream) { + SetCompressPath(IGZIP, true, false, false); + SetUncompressPath(ZLIB, false, false); + if (GetConfig(USE_ZLIB_COMPRESS) == 0) { + GTEST_SKIP() << "USE_ZLIB_COMPRESS=0 disables fallback-first contract"; + } + + std::string input; + input.reserve(256 * 1024); + for (int i = 0; i < 4096; ++i) { + input += "{\"k\":"; + input += std::to_string(i); + input += ",\"msg\":\"abcdefghijklmnopqrstuvwxyz\"}"; + } + + z_stream cstream; + memset(&cstream, 0, sizeof(cstream)); + ASSERT_EQ(deflateInit2(&cstream, Z_DEFAULT_COMPRESSION, Z_DEFLATED, 15, 8, + Z_DEFAULT_STRATEGY), + Z_OK); + + cstream.next_in = reinterpret_cast(const_cast(input.data())); + cstream.avail_in = static_cast(input.size()); + + std::vector compressed; + compressed.reserve(input.size()); + + int ret = Z_OK; + for (int iter = 0; iter < 20000; ++iter) { + unsigned char out_chunk[64]; + cstream.next_out = out_chunk; + cstream.avail_out = sizeof(out_chunk); + + ret = deflate(&cstream, Z_FINISH); + const ExecutionPath path = GetDeflateExecutionPath(&cstream); + ASSERT_TRUE(path == IGZIP || path == ZLIB); + ASSERT_NE(ret, Z_DATA_ERROR) << "iter=" << iter; + + const size_t produced = sizeof(out_chunk) - cstream.avail_out; + compressed.insert(compressed.end(), out_chunk, out_chunk + produced); + + if (ret == Z_STREAM_END) { + break; + } + } + + ASSERT_EQ(ret, Z_STREAM_END); + deflateEnd(&cstream); + + z_stream dstream; + memset(&dstream, 0, sizeof(dstream)); + ASSERT_EQ(inflateInit2(&dstream, 15), Z_OK); + + std::vector output(input.size() + 1024); + dstream.next_in = compressed.data(); + dstream.avail_in = static_cast(compressed.size()); + dstream.next_out = output.data(); + dstream.avail_out = static_cast(output.size()); + + for (int iter = 0; iter < 1024; ++iter) { + ret = inflate(&dstream, Z_NO_FLUSH); + ASSERT_NE(ret, Z_DATA_ERROR); + if (ret == Z_STREAM_END) { + break; + } + if (ret == Z_BUF_ERROR && dstream.avail_in == 0) { + break; + } + } + + ASSERT_EQ(ret, Z_STREAM_END); + const size_t out_size = output.size() - dstream.avail_out; + ASSERT_EQ(out_size, input.size()); + EXPECT_EQ(memcmp(output.data(), input.data(), input.size()), 0); + + inflateEnd(&dstream); +} + +TEST(IGZIPDeflateRegressionTest, SyncFlushWithInputMustStayOnIGZIPPath) { + // Originally this test asserted the path was ZLIB (IGZIPShouldFallbackDeflate + // redirected SYNC_FLUSH to zlib). That function is removed; IGZIP now handles + // SYNC_FLUSH natively and the stream must stay on IGZIP throughout. + SetCompressPath(IGZIP, false, false, false); + SetUncompressPath(ZLIB, false, false); + + std::string input; + input.reserve(64 * 1024); + for (int i = 0; i < 1024; ++i) { + input += "record-"; + input += std::to_string(i); + input += "-abcdefghijklmnopqrstuvwxyz"; + } + + z_stream cstream; + memset(&cstream, 0, sizeof(cstream)); + ASSERT_EQ(deflateInit2(&cstream, Z_DEFAULT_COMPRESSION, Z_DEFLATED, 15, 8, + Z_DEFAULT_STRATEGY), + Z_OK); + + std::vector compressed; + compressed.reserve(input.size()); + + cstream.next_in = reinterpret_cast(const_cast(input.data())); + cstream.avail_in = static_cast(input.size()); + + unsigned char sync_chunk[256]; + cstream.next_out = sync_chunk; + cstream.avail_out = sizeof(sync_chunk); + + int ret = deflate(&cstream, Z_SYNC_FLUSH); + ASSERT_NE(ret, Z_DATA_ERROR); + ASSERT_EQ(GetDeflateExecutionPath(&cstream), IGZIP); + const size_t sync_produced = sizeof(sync_chunk) - cstream.avail_out; + compressed.insert(compressed.end(), sync_chunk, sync_chunk + sync_produced); + + for (int iter = 0; iter < 8192; ++iter) { + unsigned char out_chunk[256]; + cstream.next_out = out_chunk; + cstream.avail_out = sizeof(out_chunk); + cstream.next_in = nullptr; + cstream.avail_in = 0; + + ret = deflate(&cstream, Z_FINISH); + ASSERT_NE(ret, Z_DATA_ERROR); + ASSERT_EQ(GetDeflateExecutionPath(&cstream), IGZIP); + + const size_t produced = sizeof(out_chunk) - cstream.avail_out; + compressed.insert(compressed.end(), out_chunk, out_chunk + produced); + + if (ret == Z_STREAM_END) { + break; + } + } + + ASSERT_EQ(ret, Z_STREAM_END); + deflateEnd(&cstream); + + z_stream dstream; + memset(&dstream, 0, sizeof(dstream)); + ASSERT_EQ(inflateInit2(&dstream, 15), Z_OK); + + std::vector output(input.size() + 1024); + dstream.next_in = compressed.data(); + dstream.avail_in = static_cast(compressed.size()); + dstream.next_out = output.data(); + dstream.avail_out = static_cast(output.size()); + + for (int iter = 0; iter < 1024; ++iter) { + ret = inflate(&dstream, Z_NO_FLUSH); + ASSERT_NE(ret, Z_DATA_ERROR); + if (ret == Z_STREAM_END) { + break; + } + } + + ASSERT_EQ(ret, Z_STREAM_END); + const size_t out_size = output.size() - dstream.avail_out; + ASSERT_EQ(out_size, input.size()); + EXPECT_EQ(memcmp(output.data(), input.data(), input.size()), 0); + + inflateEnd(&dstream); +} + +#ifdef USE_IGZIP +TEST(IGZIPDeflateRegressionTest, Compress2MustDetectTruncatedOutput) { + // Regression: compress2() returned Z_OK on a truncated stream when destLen + // was too small to hold the ISA-L trailer. ISA-L consumes all input + // (COMP_OK) but leaves the stream in a non-terminal state; the old + // input_len != sourceLen check missed this case. + SetCompressPath(IGZIP, /*zlib_fallback=*/false, false, false); + + const std::vector source(64, 'A'); + + // Determine the true compressed size with an adequately sized buffer. + uLongf correct_destLen = compressBound(source.size()); + std::vector correct_dest(correct_destLen); + ASSERT_EQ(compress2(correct_dest.data(), &correct_destLen, source.data(), + source.size(), Z_DEFAULT_COMPRESSION), + Z_OK); + + // Call again with a buffer one byte too small for the complete output. + uLongf small_destLen = correct_destLen - 1; + std::vector small_dest(small_destLen); + int ret = compress2(small_dest.data(), &small_destLen, source.data(), + source.size(), Z_DEFAULT_COMPRESSION); + // Must NOT silently succeed on truncated output. + EXPECT_NE(ret, Z_OK); +} +#endif // USE_IGZIP + +TEST(IGZIPInflateRegressionTest, + NeedDictFromIGZIPMustFallbackToZlibOnFirstInflateCall) { + SetCompressPath(ZLIB, false, false, false); + SetUncompressPath(IGZIP, false, false); + SetConfig(IGNORE_ZLIB_DICTIONARY, 0); + + const std::string dictionary = + "abcdefghijklmnopqrstuvwxyz0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZ"; + std::string input; + for (int i = 0; i < 512; ++i) { + input += "doc:"; + input += dictionary; + input += "|"; + } + + z_stream cstream; + memset(&cstream, 0, sizeof(cstream)); + ASSERT_EQ(deflateInit2(&cstream, Z_DEFAULT_COMPRESSION, Z_DEFLATED, 15, 8, + Z_DEFAULT_STRATEGY), + Z_OK); + ASSERT_EQ(deflateSetDictionary( + &cstream, reinterpret_cast(dictionary.data()), + static_cast(dictionary.size())), + Z_OK); + ASSERT_EQ(GetDeflateExecutionPath(&cstream), ZLIB); + + std::vector compressed(compressBound(input.size())); + cstream.next_in = reinterpret_cast(const_cast(input.data())); + cstream.avail_in = static_cast(input.size()); + cstream.next_out = compressed.data(); + cstream.avail_out = static_cast(compressed.size()); + + int ret = Z_OK; + for (int iter = 0; iter < 64; ++iter) { + ret = deflate(&cstream, Z_FINISH); + if (ret == Z_STREAM_END) { + break; + } + } + ASSERT_EQ(ret, Z_STREAM_END); + const size_t compressed_size = compressed.size() - cstream.avail_out; + deflateEnd(&cstream); + + z_stream dstream; + memset(&dstream, 0, sizeof(dstream)); + ASSERT_EQ(inflateInit2(&dstream, 15), Z_OK); + + std::vector uncompressed(input.size() + 1024); + dstream.next_in = compressed.data(); + dstream.avail_in = static_cast(compressed_size); + dstream.next_out = uncompressed.data(); + dstream.avail_out = static_cast(uncompressed.size()); + + ret = inflate(&dstream, Z_NO_FLUSH); + ASSERT_EQ(GetInflateExecutionPath(&dstream), ZLIB); + ASSERT_EQ(ret, Z_NEED_DICT); + + ASSERT_EQ(inflateSetDictionary( + &dstream, reinterpret_cast(dictionary.data()), + static_cast(dictionary.size())), + Z_OK); + ret = inflate(&dstream, Z_FINISH); + ASSERT_EQ(ret, Z_STREAM_END); + + const size_t out_size = uncompressed.size() - dstream.avail_out; + ASSERT_EQ(out_size, input.size()); + EXPECT_EQ(memcmp(uncompressed.data(), input.data(), input.size()), 0); + + inflateEnd(&dstream); +} + +#ifdef USE_IAA +// IAA->IGZIP fallback tests. +// On machines without IAA hardware, CompressIAA/UncompressIAA return non-zero, +// which naturally triggers the fallback path. These tests verify: +// - When igzip_fallback=1: the stream lands on IGZIP after IAA fails. +// - When igzip_fallback=0: the stream falls through to zlib, not IGZIP. + +TEST(IAAFallbackIGZIPTest, DeflateUsesIGZIPWhenIAAFailsAndFallbackEnabled) { + SetCompressPath(IAA, /*zlib_fallback=*/true, + /*iaa_prepend_empty_block=*/false, + /*qat_compression_allow_chunking=*/false); + SetConfig(USE_IGZIP_COMPRESS, 1); + SetConfig(IGZIP_FALLBACK, 1); + + const size_t input_length = 64 * 1024; + char* input = GenerateBlock(input_length, compressible_block); + ASSERT_NE(input, nullptr); + + z_stream stream; + memset(&stream, 0, sizeof(z_stream)); + ASSERT_EQ(deflateInit2(&stream, Z_DEFAULT_COMPRESSION, Z_DEFLATED, -15, 8, + Z_DEFAULT_STRATEGY), + Z_OK); + + std::vector output(deflateBound(&stream, input_length)); + stream.next_in = reinterpret_cast(input); + stream.avail_in = static_cast(input_length); + stream.next_out = output.data(); + stream.avail_out = static_cast(output.size()); + + int ret = deflate(&stream, Z_FINISH); + ASSERT_EQ(ret, Z_STREAM_END); + + // If IAA hardware is absent, the fallback must have routed to IGZIP. + // If IAA hardware is present and succeeds, IAA path is also acceptable. + const ExecutionPath path = GetDeflateExecutionPath(&stream); + EXPECT_TRUE(path == IGZIP || path == IAA) + << "Expected IGZIP (fallback) or IAA (hardware success), got " + << static_cast(path); + + deflateEnd(&stream); + SetConfig(IGZIP_FALLBACK, 0); + DestroyBlock(input); +} + +TEST(IAAFallbackIGZIPTest, DeflateDoesNotUseIGZIPWhenFallbackDisabled) { + SetCompressPath(IAA, /*zlib_fallback=*/true, + /*iaa_prepend_empty_block=*/false, + /*qat_compression_allow_chunking=*/false); + SetConfig(USE_IGZIP_COMPRESS, 1); + SetConfig(IGZIP_FALLBACK, 0); + + const size_t input_length = 64 * 1024; + char* input = GenerateBlock(input_length, compressible_block); + ASSERT_NE(input, nullptr); + + z_stream stream; + memset(&stream, 0, sizeof(z_stream)); + ASSERT_EQ(deflateInit2(&stream, Z_DEFAULT_COMPRESSION, Z_DEFLATED, -15, 8, + Z_DEFAULT_STRATEGY), + Z_OK); + + std::vector output(deflateBound(&stream, input_length)); + stream.next_in = reinterpret_cast(input); + stream.avail_in = static_cast(input_length); + stream.next_out = output.data(); + stream.avail_out = static_cast(output.size()); + + const int ret = deflate(&stream, Z_FINISH); + ASSERT_EQ(ret, Z_STREAM_END); + + // With fallback disabled, IGZIP must not be selected for an IAA-configured + // stream; it should fall through to zlib. + const ExecutionPath path = GetDeflateExecutionPath(&stream); + EXPECT_NE(path, IGZIP) << "IGZIP must not be used when igzip_fallback=0"; + + deflateEnd(&stream); + SetConfig(IGZIP_FALLBACK, 0); + DestroyBlock(input); +} + +TEST(IAAFallbackIGZIPTest, InflateUsesIGZIPWhenIAAFailsAndFallbackEnabled) { + // Compress with IGZIP so the output is IGZIP-compatible (4kB window). + SetCompressPath(IGZIP, false, false, false); + SetUncompressPath(IAA, /*zlib_fallback=*/true, + /*iaa_prepend_empty_block=*/false); + SetConfig(USE_IGZIP_UNCOMPRESS, 1); + SetConfig(IGZIP_FALLBACK, 1); + + const size_t input_length = 64 * 1024; + char* input = GenerateBlock(input_length, compressible_block); + ASSERT_NE(input, nullptr); + + std::string compressed; + size_t output_upper_bound; + ExecutionPath compress_path = UNDEFINED; + int ret = ZlibCompress(input, input_length, &compressed, -15, Z_FINISH, + &output_upper_bound, &compress_path); + ASSERT_EQ(ret, Z_STREAM_END); + + z_stream dstream; + memset(&dstream, 0, sizeof(z_stream)); + ASSERT_EQ(inflateInit2(&dstream, -15), Z_OK); + + std::vector uncompressed(input_length); + dstream.next_in = reinterpret_cast(compressed.data()); + dstream.avail_in = static_cast(compressed.size()); + dstream.next_out = reinterpret_cast(uncompressed.data()); + dstream.avail_out = static_cast(uncompressed.size()); + + ret = inflate(&dstream, Z_FINISH); + ASSERT_EQ(ret, Z_STREAM_END); + + // If IAA hardware is absent, fallback must route to IGZIP. + // If IAA hardware is present and succeeds, IAA is also acceptable. + const ExecutionPath path = GetInflateExecutionPath(&dstream); + EXPECT_TRUE(path == IGZIP || path == IAA) + << "Expected IGZIP (fallback) or IAA (hardware success), got " + << static_cast(path); + + EXPECT_EQ(memcmp(uncompressed.data(), input, input_length), 0); + + inflateEnd(&dstream); + SetConfig(IGZIP_FALLBACK, 0); + DestroyBlock(input); +} + +TEST(IAAFallbackIGZIPTest, InflateDoesNotUseIGZIPWhenFallbackDisabled) { + SetCompressPath(IGZIP, false, false, false); + SetUncompressPath(IAA, /*zlib_fallback=*/true, + /*iaa_prepend_empty_block=*/false); + SetConfig(USE_IGZIP_UNCOMPRESS, 1); + SetConfig(IGZIP_FALLBACK, 0); + + const size_t input_length = 64 * 1024; + char* input = GenerateBlock(input_length, compressible_block); + ASSERT_NE(input, nullptr); + + std::string compressed; + size_t output_upper_bound; + ExecutionPath compress_path = UNDEFINED; + int ret = ZlibCompress(input, input_length, &compressed, -15, Z_FINISH, + &output_upper_bound, &compress_path); + ASSERT_EQ(ret, Z_STREAM_END); + + z_stream dstream; + memset(&dstream, 0, sizeof(z_stream)); + ASSERT_EQ(inflateInit2(&dstream, -15), Z_OK); + + std::vector uncompressed(input_length); + dstream.next_in = reinterpret_cast(compressed.data()); + dstream.avail_in = static_cast(compressed.size()); + dstream.next_out = reinterpret_cast(uncompressed.data()); + dstream.avail_out = static_cast(uncompressed.size()); + + inflate(&dstream, Z_FINISH); + + // With fallback disabled, IGZIP must not be selected for an IAA-configured + // inflate stream; it should fall through to zlib. + const ExecutionPath path = GetInflateExecutionPath(&dstream); + EXPECT_NE(path, IGZIP) << "IGZIP must not be used when igzip_fallback=0"; + + inflateEnd(&dstream); + SetConfig(IGZIP_FALLBACK, 0); + DestroyBlock(input); +} +#endif // USE_IAA + +#if defined(USE_QAT) && defined(USE_IGZIP) +// QAT->IGZIP fallback tests. +// On machines without QAT hardware, CompressQAT/UncompressQAT return non-zero, +// which naturally triggers the fallback path. These tests verify: +// - When igzip_fallback=1: the stream lands on IGZIP after QAT fails. +// - When igzip_fallback=0: the stream falls through to zlib, not IGZIP. + +TEST(QATFallbackIGZIPTest, DeflateUsesIGZIPWhenQATFailsAndFallbackEnabled) { + SetCompressPath(QAT, /*zlib_fallback=*/true, + /*iaa_prepend_empty_block=*/false, + /*qat_compression_allow_chunking=*/false); + SetConfig(USE_IGZIP_COMPRESS, 1); + SetConfig(IGZIP_FALLBACK, 1); + + const size_t input_length = 64 * 1024; + char* input = GenerateBlock(input_length, compressible_block); + ASSERT_NE(input, nullptr); + + z_stream stream; + memset(&stream, 0, sizeof(z_stream)); + ASSERT_EQ(deflateInit2(&stream, Z_DEFAULT_COMPRESSION, Z_DEFLATED, -15, 8, + Z_DEFAULT_STRATEGY), + Z_OK); + + std::vector output(deflateBound(&stream, input_length)); + stream.next_in = reinterpret_cast(input); + stream.avail_in = static_cast(input_length); + stream.next_out = output.data(); + stream.avail_out = static_cast(output.size()); + + int ret = deflate(&stream, Z_FINISH); + ASSERT_EQ(ret, Z_STREAM_END); + + // If QAT hardware is absent, fallback must route to IGZIP. + // If QAT hardware is present and succeeds, QAT path is also acceptable. + const ExecutionPath path = GetDeflateExecutionPath(&stream); + EXPECT_TRUE(path == IGZIP || path == QAT) + << "Expected IGZIP (fallback) or QAT (hardware success), got " + << static_cast(path); + + deflateEnd(&stream); + SetConfig(IGZIP_FALLBACK, 0); + DestroyBlock(input); +} + +TEST(QATFallbackIGZIPTest, DeflateDoesNotUseIGZIPWhenFallbackDisabled) { + SetCompressPath(QAT, /*zlib_fallback=*/true, + /*iaa_prepend_empty_block=*/false, + /*qat_compression_allow_chunking=*/false); + SetConfig(USE_IGZIP_COMPRESS, 1); + SetConfig(IGZIP_FALLBACK, 0); + + const size_t input_length = 64 * 1024; + char* input = GenerateBlock(input_length, compressible_block); + ASSERT_NE(input, nullptr); + + z_stream stream; + memset(&stream, 0, sizeof(z_stream)); + ASSERT_EQ(deflateInit2(&stream, Z_DEFAULT_COMPRESSION, Z_DEFLATED, -15, 8, + Z_DEFAULT_STRATEGY), + Z_OK); + + std::vector output(deflateBound(&stream, input_length)); + stream.next_in = reinterpret_cast(input); + stream.avail_in = static_cast(input_length); + stream.next_out = output.data(); + stream.avail_out = static_cast(output.size()); + + int ret = deflate(&stream, Z_FINISH); + ASSERT_EQ(ret, Z_STREAM_END); + + // With fallback disabled, IGZIP must not be selected for a QAT-configured + // stream; it should fall through to zlib. + const ExecutionPath path = GetDeflateExecutionPath(&stream); + EXPECT_NE(path, IGZIP) << "IGZIP must not be used when igzip_fallback=0"; + + deflateEnd(&stream); + SetConfig(IGZIP_FALLBACK, 0); + DestroyBlock(input); +} + +TEST(QATFallbackIGZIPTest, InflateUsesIGZIPWhenQATFailsAndFallbackEnabled) { + // Compress with IGZIP so the output is IGZIP-compatible (4kB window). + SetCompressPath(IGZIP, false, false, false); + SetUncompressPath(QAT, /*zlib_fallback=*/true, + /*iaa_prepend_empty_block=*/false); + SetConfig(USE_IGZIP_UNCOMPRESS, 1); + SetConfig(IGZIP_FALLBACK, 1); + + const size_t input_length = 64 * 1024; + char* input = GenerateBlock(input_length, compressible_block); + ASSERT_NE(input, nullptr); + + std::string compressed; + size_t output_upper_bound; + ExecutionPath compress_path = UNDEFINED; + int ret = ZlibCompress(input, input_length, &compressed, -15, Z_FINISH, + &output_upper_bound, &compress_path); + ASSERT_EQ(ret, Z_STREAM_END); + + z_stream dstream; + memset(&dstream, 0, sizeof(z_stream)); + ASSERT_EQ(inflateInit2(&dstream, -15), Z_OK); + + std::vector uncompressed(input_length); + dstream.next_in = reinterpret_cast(compressed.data()); + dstream.avail_in = static_cast(compressed.size()); + dstream.next_out = reinterpret_cast(uncompressed.data()); + dstream.avail_out = static_cast(uncompressed.size()); + + ret = inflate(&dstream, Z_FINISH); + ASSERT_EQ(ret, Z_STREAM_END); + + // If QAT hardware is absent, fallback must route to IGZIP. + // If QAT hardware is present and succeeds, QAT is also acceptable. + const ExecutionPath path = GetInflateExecutionPath(&dstream); + EXPECT_TRUE(path == IGZIP || path == QAT) + << "Expected IGZIP (fallback) or QAT (hardware success), got " + << static_cast(path); + + EXPECT_EQ(memcmp(uncompressed.data(), input, input_length), 0); + + inflateEnd(&dstream); + SetConfig(IGZIP_FALLBACK, 0); + DestroyBlock(input); +} + +TEST(QATFallbackIGZIPTest, InflateDoesNotUseIGZIPWhenFallbackDisabled) { + SetCompressPath(IGZIP, false, false, false); + SetUncompressPath(QAT, /*zlib_fallback=*/true, + /*iaa_prepend_empty_block=*/false); + SetConfig(USE_IGZIP_UNCOMPRESS, 1); + SetConfig(IGZIP_FALLBACK, 0); + + const size_t input_length = 64 * 1024; + char* input = GenerateBlock(input_length, compressible_block); + ASSERT_NE(input, nullptr); + + std::string compressed; + size_t output_upper_bound; + ExecutionPath compress_path = UNDEFINED; + int ret = ZlibCompress(input, input_length, &compressed, -15, Z_FINISH, + &output_upper_bound, &compress_path); + ASSERT_EQ(ret, Z_STREAM_END); + + z_stream dstream; + memset(&dstream, 0, sizeof(z_stream)); + ASSERT_EQ(inflateInit2(&dstream, -15), Z_OK); + + std::vector uncompressed(input_length); + dstream.next_in = reinterpret_cast(compressed.data()); + dstream.avail_in = static_cast(compressed.size()); + dstream.next_out = reinterpret_cast(uncompressed.data()); + dstream.avail_out = static_cast(uncompressed.size()); + + ret = inflate(&dstream, Z_FINISH); + ASSERT_EQ(ret, Z_STREAM_END); + + // With fallback disabled, IGZIP must not be selected for a QAT-configured + // inflate stream; it should fall through to zlib. + const ExecutionPath path = GetInflateExecutionPath(&dstream); + EXPECT_NE(path, IGZIP) << "IGZIP must not be used when igzip_fallback=0"; + + inflateEnd(&dstream); + SetConfig(IGZIP_FALLBACK, 0); + DestroyBlock(input); +} +#endif // USE_QAT && USE_IGZIP + +#endif + INSTANTIATE_TEST_SUITE_P( CompressDecompress, ZlibPartialAndMultiStreamTest, testing::Combine( @@ -1122,6 +2873,10 @@ INSTANTIATE_TEST_SUITE_P( #ifdef USE_IAA , IAA +#endif +#ifdef USE_IGZIP + , + IGZIP #endif ), testing::Values(false, true), @@ -1133,6 +2888,10 @@ INSTANTIATE_TEST_SUITE_P( #ifdef USE_IAA , IAA +#endif +#ifdef USE_IGZIP + , + IGZIP #endif ), testing::Values(false, true), testing::Values(-15, 15, 31), @@ -1203,6 +2962,10 @@ INSTANTIATE_TEST_SUITE_P( #ifdef USE_IAA , IAA +#endif +#ifdef USE_IGZIP + , + IGZIP #endif ), testing::Values(false, true), @@ -1214,6 +2977,10 @@ INSTANTIATE_TEST_SUITE_P( #ifdef USE_IAA , IAA +#endif +#ifdef USE_IGZIP + , + IGZIP #endif ), testing::Values(false, true), testing::Values(31), @@ -1226,6 +2993,120 @@ INSTANTIATE_TEST_SUITE_P( class ConfigLoaderTest : public ::testing::Test {}; +#if defined(USE_IGZIP) || defined(USE_QAT) || defined(USE_IAA) +class DictionaryMidstreamFallbackRegressionTest : public ::testing::Test {}; + +static void RunMidstreamSetDictionaryRegression(ExecutionPath accel_path) { + SetCompressPath(accel_path, true, false, false); + SetUncompressPath(ZLIB, false, false); + SetConfig(IGNORE_ZLIB_DICTIONARY, 0); + + std::string first_chunk; + std::string second_chunk; + for (int i = 0; i < 256; ++i) { + first_chunk += "first:"; + first_chunk += std::to_string(i); + first_chunk += ":abcdefghijklmnopqrstuvwxyz|"; + + second_chunk += "second:"; + second_chunk += std::to_string(i); + second_chunk += ":0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZ|"; + } + const std::string expected = first_chunk + second_chunk; + + z_stream cstream; + memset(&cstream, 0, sizeof(cstream)); + ASSERT_EQ(deflateInit2(&cstream, Z_DEFAULT_COMPRESSION, Z_DEFLATED, 15, 8, + Z_DEFAULT_STRATEGY), + Z_OK); + + std::vector compressed(compressBound(expected.size()) + 512); + cstream.next_out = compressed.data(); + cstream.avail_out = static_cast(compressed.size()); + + cstream.next_in = + reinterpret_cast(const_cast(first_chunk.data())); + cstream.avail_in = static_cast(first_chunk.size()); + int ret = deflate(&cstream, Z_NO_FLUSH); + ASSERT_TRUE(ret == Z_OK || ret == Z_BUF_ERROR); + const ExecutionPath observed_before_dict = GetDeflateExecutionPath(&cstream); + EXPECT_TRUE(observed_before_dict == accel_path || + observed_before_dict == ZLIB); + + const unsigned char dict[] = "midstream-dictionary"; + const int dict_ret = + deflateSetDictionary(&cstream, dict, static_cast(sizeof(dict) - 1)); + + // zlib semantics require dictionary to be set before any deflate output. + // Accepting this call midstream can desynchronize accelerator and zlib state. + EXPECT_EQ(dict_ret, Z_STREAM_ERROR); + // Rejected dictionary update must not mutate active stream path. + EXPECT_EQ(GetDeflateExecutionPath(&cstream), observed_before_dict); + + cstream.next_in = + reinterpret_cast(const_cast(second_chunk.data())); + cstream.avail_in = static_cast(second_chunk.size()); + for (int guard = 0; guard < 128; ++guard) { + ret = deflate(&cstream, Z_FINISH); + ASSERT_NE(ret, Z_DATA_ERROR); + if (ret == Z_STREAM_END) { + break; + } + } + ASSERT_EQ(ret, Z_STREAM_END); + + const size_t compressed_size = compressed.size() - cstream.avail_out; + ASSERT_GT(compressed_size, 0u); + deflateEnd(&cstream); + + z_stream dstream; + memset(&dstream, 0, sizeof(dstream)); + ASSERT_EQ(inflateInit2(&dstream, 15), Z_OK); + + std::vector uncompressed(expected.size() + 1024); + dstream.next_in = compressed.data(); + dstream.avail_in = static_cast(compressed_size); + dstream.next_out = uncompressed.data(); + dstream.avail_out = static_cast(uncompressed.size()); + + for (int guard = 0; guard < 128; ++guard) { + ret = inflate(&dstream, Z_NO_FLUSH); + ASSERT_NE(ret, Z_DATA_ERROR); + if (ret == Z_STREAM_END) { + break; + } + } + + ASSERT_EQ(ret, Z_STREAM_END); + const size_t produced = uncompressed.size() - dstream.avail_out; + ASSERT_EQ(produced, expected.size()); + EXPECT_EQ(memcmp(uncompressed.data(), expected.data(), expected.size()), 0); + + inflateEnd(&dstream); +} + +#ifdef USE_IGZIP +TEST_F(DictionaryMidstreamFallbackRegressionTest, + IGZIPRejectsMidstreamSetDictionary) { + RunMidstreamSetDictionaryRegression(IGZIP); +} +#endif + +#ifdef USE_QAT +TEST_F(DictionaryMidstreamFallbackRegressionTest, + QATRejectsMidstreamSetDictionary) { + RunMidstreamSetDictionaryRegression(QAT); +} +#endif + +#ifdef USE_IAA +TEST_F(DictionaryMidstreamFallbackRegressionTest, + IAARejectsMidstreamSetDictionary) { + RunMidstreamSetDictionaryRegression(IAA); +} +#endif +#endif + void CreateAndWriteTempConfigFile(const char* file_path) { std::ofstream temp_file(file_path); temp_file << "use_qat_compress=5000\n"; @@ -1272,6 +3153,8 @@ TEST_F(ConfigLoaderTest, LoadValidConfig) { EXPECT_EQ(GetConfig(USE_QAT_UNCOMPRESS), 1); EXPECT_EQ(GetConfig(USE_IAA_COMPRESS), 0); EXPECT_EQ(GetConfig(USE_IAA_UNCOMPRESS), 0); + EXPECT_EQ(GetConfig(USE_IGZIP_COMPRESS), 0); + EXPECT_EQ(GetConfig(USE_IGZIP_UNCOMPRESS), 0); EXPECT_EQ(GetConfig(USE_ZLIB_COMPRESS), 1); EXPECT_EQ(GetConfig(USE_ZLIB_UNCOMPRESS), 1); EXPECT_EQ(GetConfig(LOG_LEVEL), 1); diff --git a/zlib_accel.cpp b/zlib_accel.cpp index 26188e7..bcffdb1 100644 --- a/zlib_accel.cpp +++ b/zlib_accel.cpp @@ -5,6 +5,7 @@ #include #include +#include #include #include @@ -20,6 +21,9 @@ #ifdef USE_IAA #include "iaa.h" #endif +#ifdef USE_IGZIP +#include "igzip.h" +#endif #ifdef USE_QAT #include "qat.h" #endif @@ -27,6 +31,10 @@ using namespace config; +// Window bits: 15 = zlib (deflate) format, 31 = gzip format +static constexpr int kWindowBitsZlib = 15; +static constexpr int kWindowBitsGzip = 31; + // Disable cfi-icall as it makes calls to orig* functions fail #if defined(__clang__) #pragma clang attribute push(__attribute__((no_sanitize("cfi-icall"))), \ @@ -177,6 +185,8 @@ static void cleanup_zlib_accel(void) { // Avoid recursive call (e.g., if QATzip falls back to zlib internally) static thread_local bool in_call = false; +constexpr uint8_t ZLIB_FDICT_MASK = 0x20; + struct DeflateSettings { DeflateSettings(int _level, int _method, int _window_bits, int _mem_level, int _strategy) @@ -192,12 +202,14 @@ struct DeflateSettings { int mem_level; int strategy; ExecutionPath path = UNDEFINED; + struct isal_zstream* isal_strm = nullptr; }; struct InflateSettings { InflateSettings(int _window_bits) : window_bits(_window_bits) {} int window_bits; ExecutionPath path = UNDEFINED; + struct inflate_state* isal_strm = nullptr; }; class DeflateStreamSettings { @@ -234,12 +246,26 @@ class InflateStreamSettings { }; InflateStreamSettings inflate_stream_settings; +static void SetDeflatePath(DeflateSettings* settings, ExecutionPath new_path) { + if (settings == nullptr || settings->path == new_path) { + return; + } + settings->path = new_path; +} + +static void SetInflatePath(InflateSettings* settings, ExecutionPath new_path) { + if (settings == nullptr || settings->path == new_path) { + return; + } + settings->path = new_path; +} + int ZEXPORT deflateInit_(z_streamp strm, int level, const char* version, int stream_size) { Log(LogLevel::LOG_INFO, "deflateInit_ Line ", __LINE__, ", strm ", static_cast(strm), ", level ", level, "\n"); - deflate_stream_settings.Set(strm, level, Z_DEFLATED, 15, 8, + deflate_stream_settings.Set(strm, level, Z_DEFLATED, kWindowBitsZlib, 8, Z_DEFAULT_STRATEGY); return orig_deflateInit_(strm, level, version, stream_size); } @@ -263,8 +289,19 @@ int ZEXPORT deflateSetDictionary(z_streamp strm, const Bytef* dictionary, Log(LogLevel::LOG_INFO, "deflateSetDictionary Line ", __LINE__, ", strm ", static_cast(strm), ", dictLength ", dictLength, "\n"); DeflateSettings* deflate_settings = deflate_stream_settings.Get(strm); - deflate_settings->path = ZLIB; - return orig_deflateSetDictionary(strm, dictionary, dictLength); + // Reject mid-stream: if an accelerator is active, the underlying zlib + // stream has not been advanced, so orig_deflateSetDictionary would + // incorrectly accept the call. Per zlib spec, dictionary must be set before + // compression begins. + if (deflate_settings != nullptr && deflate_settings->path != UNDEFINED && + deflate_settings->path != ZLIB) { + return Z_STREAM_ERROR; + } + const int ret = orig_deflateSetDictionary(strm, dictionary, dictLength); + if (ret == Z_OK) { + SetDeflatePath(deflate_settings, ZLIB); + } + return ret; } Log(LogLevel::LOG_INFO, "deflateSetDictionary Line ", __LINE__, " ignored because ignore_zlib_dictionary is set to ", @@ -280,30 +317,43 @@ int ZEXPORT deflate(z_streamp strm, int flush) { Log(LogLevel::LOG_INFO, "deflate Line ", __LINE__, ", strm ", static_cast(strm), ", avail_in ", strm->avail_in, ", avail_out ", strm->avail_out, ", flush ", flush, ", in_call ", in_call, ", path ", - static_cast(deflate_settings->path), "\n"); + static_cast(deflate_settings->path), ", path_name ", + static_cast(deflate_settings->path), ", window_bits ", + deflate_settings->window_bits, ", total_in ", strm->total_in, + ", total_out ", strm->total_out, ", adler ", strm->adler, "\n"); int ret = 1; bool iaa_available = false; bool qat_available = false; - if (!in_call && flush == Z_FINISH && deflate_settings->path != ZLIB) { + bool igzip_available = false; + if (!in_call && deflate_settings->path != ZLIB) { uint32_t input_len = strm->avail_in; uint32_t output_len = strm->avail_out; + bool igzip_stream_active = false; #ifdef USE_IAA - iaa_available = configs[USE_IAA_COMPRESS] && + iaa_available = (flush == Z_FINISH) && configs[USE_IAA_COMPRESS] && SupportedOptionsIAA(deflate_settings->window_bits, input_len, output_len); #endif #ifdef USE_QAT qat_available = - configs[USE_QAT_COMPRESS] && output_len >= QAT_DEST_BUFFER_MIN_SIZE && + (flush == Z_FINISH) && configs[USE_QAT_COMPRESS] && + output_len >= QAT_DEST_BUFFER_MIN_SIZE && SupportedOptionsQAT(deflate_settings->window_bits, input_len); #endif +#ifdef USE_IGZIP + igzip_stream_active = (deflate_settings->path == IGZIP && + deflate_settings->isal_strm != nullptr); + igzip_available = configs[USE_IGZIP_COMPRESS]; +#endif // If both accelerators are enabled, send configured ratio of requests to // one or the other ExecutionPath path_selected = ZLIB; - if (iaa_available && qat_available) { + if (igzip_stream_active) { + path_selected = IGZIP; + } else if (iaa_available && qat_available) { if (static_cast(std::rand() % 100) < configs[IAA_COMPRESS_PERCENTAGE]) { path_selected = IAA; @@ -314,6 +364,8 @@ int ZEXPORT deflate(z_streamp strm, int flush) { path_selected = IAA; } else if (qat_available) { path_selected = QAT; + } else if (igzip_available) { + path_selected = IGZIP; } if (path_selected == IAA) { @@ -325,7 +377,7 @@ int ZEXPORT deflate(z_streamp strm, int flush) { ret = CompressIAA(strm->next_in, &input_len, strm->next_out, &output_len, qpl_path_hardware, deflate_settings->window_bits, max_compressed_size); - deflate_settings->path = IAA; + SetDeflatePath(deflate_settings, IAA); in_call = false; INCREMENT_STAT(DEFLATE_IAA_COUNT); INCREMENT_STAT_COND(ret != 0, DEFLATE_IAA_ERROR_COUNT); @@ -335,13 +387,58 @@ int ZEXPORT deflate(z_streamp strm, int flush) { in_call = true; ret = CompressQAT(strm->next_in, &input_len, strm->next_out, &output_len, deflate_settings->window_bits); - deflate_settings->path = QAT; + SetDeflatePath(deflate_settings, QAT); in_call = false; INCREMENT_STAT(DEFLATE_QAT_COUNT); INCREMENT_STAT_COND(ret != 0, DEFLATE_QAT_ERROR_COUNT); #endif // USE_QAT + } else if (path_selected == IGZIP) { +#ifdef USE_IGZIP + if (deflate_settings->isal_strm == nullptr) { + deflate_settings->isal_strm = InitCompressIGZIP( + deflate_settings->level, deflate_settings->window_bits); + } + if (deflate_settings->isal_strm != nullptr) { + in_call = true; + ret = CompressIGZIP(deflate_settings->isal_strm, flush, strm->next_in, + &input_len, strm->next_out, &output_len, + &strm->total_in, &strm->total_out); + SetDeflatePath(deflate_settings, IGZIP); + in_call = false; + + INCREMENT_STAT(DEFLATE_IGZIP_COUNT); + INCREMENT_STAT_COND(ret != 0, DEFLATE_IGZIP_ERROR_COUNT); + } +#endif } +#ifdef USE_IGZIP + // Accelerator->IGZIP fallback: if IAA or QAT failed and IGZIP is + // available, retry with IGZIP before falling through to software zlib. + if ((path_selected == IAA || path_selected == QAT) && ret != 0 && + configs[IGZIP_FALLBACK] && igzip_available) { + // Accelerator may have modified input_len/output_len on failure. + // Restore them before retrying with IGZIP. + input_len = strm->avail_in; + output_len = strm->avail_out; + if (deflate_settings->isal_strm == nullptr) { + deflate_settings->isal_strm = InitCompressIGZIP( + deflate_settings->level, deflate_settings->window_bits); + } + if (deflate_settings->isal_strm != nullptr) { + in_call = true; + ret = CompressIGZIP(deflate_settings->isal_strm, flush, strm->next_in, + &input_len, strm->next_out, &output_len, + &strm->total_in, &strm->total_out); + SetDeflatePath(deflate_settings, IGZIP); + in_call = false; + path_selected = IGZIP; // use IGZIP return-code semantics below + INCREMENT_STAT(DEFLATE_IGZIP_COUNT); + INCREMENT_STAT_COND(ret != 0, DEFLATE_IGZIP_ERROR_COUNT); + } + } +#endif // USE_IGZIP accelerator fallback + if (ret == 0) { strm->next_in += input_len; strm->avail_in -= input_len; @@ -349,25 +446,44 @@ int ZEXPORT deflate(z_streamp strm, int flush) { strm->next_out += output_len; strm->avail_out -= output_len; strm->total_out += output_len; - if (strm->avail_in == 0) { - ret = Z_STREAM_END; + if (path_selected == IGZIP) { + const bool no_progress = (input_len == 0 && output_len == 0); + bool finish_done = false; +#ifdef USE_IGZIP + finish_done = (flush == Z_FINISH) && + IsIGZIPDeflateFinished(deflate_settings->isal_strm); +#endif + + if (finish_done) { + ret = Z_STREAM_END; + } else if (!no_progress) { + ret = Z_OK; + } else { + ret = Z_BUF_ERROR; + } } else { - ret = Z_BUF_ERROR; + if (strm->avail_in == 0) { + ret = Z_STREAM_END; + } else { + ret = Z_BUF_ERROR; + } } Log(LogLevel::LOG_INFO, "deflate Line ", __LINE__, ", strm ", static_cast(strm), ", accelerator return code ", ret, - ", avail_in ", strm->avail_in, ", avail_out ", strm->avail_out, - ", path ", static_cast(deflate_settings->path), "\n"); + ", bytes_in ", input_len, ", bytes_out ", output_len, ", avail_in ", + strm->avail_in, ", avail_out ", strm->avail_out, ", path ", + static_cast(deflate_settings->path), ", path_name ", + static_cast(deflate_settings->path), "\n"); return ret; } } - if (in_call || configs[USE_ZLIB_COMPRESS]) { + if (in_call || configs[USE_ZLIB_COMPRESS] || deflate_settings->path == ZLIB) { ret = orig_deflate(strm, flush); INCREMENT_STAT(DEFLATE_ZLIB_COUNT); if (!in_call) { - deflate_settings->path = ZLIB; + SetDeflatePath(deflate_settings, ZLIB); } } else { ret = Z_DATA_ERROR; @@ -376,6 +492,7 @@ int ZEXPORT deflate(z_streamp strm, int flush) { Log(LogLevel::LOG_INFO, "deflate Line ", __LINE__, ", strm ", static_cast(strm), ", zlib return code ", ret, ", avail_in ", strm->avail_in, ", avail_out ", strm->avail_out, ", path ", + static_cast(deflate_settings->path), ", path_name ", static_cast(deflate_settings->path), "\n"); INCREMENT_STAT_COND(ret < 0, DEFLATE_ERROR_COUNT); @@ -385,6 +502,13 @@ int ZEXPORT deflate(z_streamp strm, int flush) { int ZEXPORT deflateEnd(z_streamp strm) { Log(LogLevel::LOG_INFO, "deflateEnd Line ", __LINE__, ", strm ", static_cast(strm), "\n"); + DeflateSettings* deflate_settings = deflate_stream_settings.Get(strm); + if (deflate_settings != nullptr && deflate_settings->isal_strm != nullptr) { +#ifdef USE_IGZIP + EndCompressIGZIP(deflate_settings->isal_strm); +#endif + deflate_settings->isal_strm = nullptr; + } deflate_stream_settings.Unset(strm); return orig_deflateEnd(strm); } @@ -393,18 +517,24 @@ int ZEXPORT deflateReset(z_streamp strm) { Log(LogLevel::LOG_INFO, "deflateReset Line ", __LINE__, ", strm ", static_cast(strm), "\n"); DeflateSettings* deflate_settings = deflate_stream_settings.Get(strm); + int ret = orig_deflateReset(strm); if (deflate_settings != nullptr) { - deflate_settings->path = UNDEFINED; + SetDeflatePath(deflate_settings, UNDEFINED); + +#ifdef USE_IGZIP + if (deflate_settings->isal_strm != nullptr) { + ResetCompressIGZIP(deflate_settings->isal_strm); + } +#endif } - return orig_deflateReset(strm); + return ret; } int ZEXPORT inflateInit_(z_streamp strm, const char* version, int stream_size) { - inflate_stream_settings.Set(strm, 15); + inflate_stream_settings.Set(strm, kWindowBitsZlib); Log(LogLevel::LOG_INFO, "inflateInit_ Line ", __LINE__, ", strm ", static_cast(strm), "\n"); - return orig_inflateInit_(strm, version, stream_size); } @@ -413,7 +543,6 @@ int ZEXPORT inflateInit2_(z_streamp strm, int window_bits, const char* version, inflate_stream_settings.Set(strm, window_bits); Log(LogLevel::LOG_INFO, "inflateInit2_ Line ", __LINE__, ", strm ", static_cast(strm), ", window_bits ", window_bits, "\n"); - return orig_inflateInit2_(strm, window_bits, version, stream_size); } @@ -421,10 +550,13 @@ int ZEXPORT inflateSetDictionary(z_streamp strm, const Bytef* dictionary, uInt dictLength) { if (!configs[IGNORE_ZLIB_DICTIONARY]) { Log(LogLevel::LOG_INFO, "inflateSetDictionary Line ", __LINE__, ", strm ", - static_cast(strm), "dictLength ", dictLength, "\n"); + static_cast(strm), ", dictLength ", dictLength, "\n"); InflateSettings* inflate_settings = inflate_stream_settings.Get(strm); - inflate_settings->path = ZLIB; - return orig_inflateSetDictionary(strm, dictionary, dictLength); + const int ret = orig_inflateSetDictionary(strm, dictionary, dictLength); + if (ret == Z_OK) { + SetInflatePath(inflate_settings, ZLIB); + } + return ret; } Log(LogLevel::LOG_INFO, "inflateSetDictionary Line ", __LINE__, " ignored because ignore_zlib_dictionary is set to ", @@ -434,13 +566,18 @@ int ZEXPORT inflateSetDictionary(z_streamp strm, const Bytef* dictionary, int ZEXPORT inflate(z_streamp strm, int flush) { InflateSettings* inflate_settings = inflate_stream_settings.Get(strm); + INCREMENT_STAT(INFLATE_COUNT); PrintStats(); Log(LogLevel::LOG_INFO, "inflate Line ", __LINE__, ", strm ", static_cast(strm), ", avail_in ", strm->avail_in, ", avail_out ", strm->avail_out, ", flush ", flush, ", in_call ", in_call, ", path ", - static_cast(inflate_settings->path), "\n"); + static_cast(inflate_settings->path), ", path_name ", + static_cast(inflate_settings->path), ", window_bits ", + inflate_settings->window_bits, ", total_in ", strm->total_in, + ", total_out ", strm->total_out, ", adler ", strm->adler, "\n"); + PrintDeflateBlockHeader(LogLevel::LOG_INFO, strm->next_in, strm->avail_in, inflate_settings->window_bits); @@ -448,6 +585,36 @@ int ZEXPORT inflate(z_streamp strm, int flush) { bool end_of_stream = true; bool iaa_available = false; bool qat_available = false; + bool igzip_available = false; + + const bool igzip_stream_active = (inflate_settings->path == IGZIP && + inflate_settings->isal_strm != nullptr); + +#ifdef USE_IGZIP + const bool igzip_supported_options = + !in_call && configs[USE_IGZIP_UNCOMPRESS]; + + // Keep stateful IGZIP stream handling on the same engine. + // For avail_in==0, let IGZIP process any buffered bits in its internal + // state before reporting Z_BUF_ERROR. + if (!in_call && igzip_stream_active && strm->avail_in == 0) { + in_call = true; + IGZIPHandleActiveStreamNoInput(strm, inflate_settings->isal_strm, &ret); + in_call = false; + return ret; + } +#endif + + // Early detection: if this is a zlib-format stream with the FDICT bit set + // in the header, pin to ZLIB immediately so dictionary streams never reach + // any accelerator (QAT/IAA/IGZIP don't support preset dictionaries). + if (!in_call && inflate_settings->path == UNDEFINED && + inflate_settings->window_bits >= 8 && + inflate_settings->window_bits <= kWindowBitsZlib && strm->avail_in >= 2 && + (strm->next_in[1] & ZLIB_FDICT_MASK)) { + SetInflatePath(inflate_settings, ZLIB); + } + if (!in_call && strm->avail_in > 0 && inflate_settings->path != ZLIB) { uint32_t input_len = strm->avail_in; uint32_t output_len = strm->avail_out; @@ -465,11 +632,16 @@ int ZEXPORT inflate(z_streamp strm, int flush) { configs[USE_QAT_UNCOMPRESS] && SupportedOptionsQAT(inflate_settings->window_bits, input_len); #endif +#ifdef USE_IGZIP + igzip_available = igzip_supported_options; +#endif // If both accelerators are enabled, send configured ratio of requests to // one or the other ExecutionPath path_selected = ZLIB; - if (iaa_available && qat_available) { + if (igzip_stream_active) { + path_selected = IGZIP; + } else if (iaa_available && qat_available) { if (static_cast(std::rand()) % 100 < configs[IAA_UNCOMPRESS_PERCENTAGE]) { path_selected = IAA; @@ -480,15 +652,16 @@ int ZEXPORT inflate(z_streamp strm, int flush) { path_selected = IAA; } else if (qat_available) { path_selected = QAT; + } else if (igzip_available) { + path_selected = IGZIP; } - if (path_selected == IAA) { #ifdef USE_IAA in_call = true; ret = UncompressIAA(strm->next_in, &input_len, strm->next_out, &output_len, qpl_path_hardware, inflate_settings->window_bits, &end_of_stream); - inflate_settings->path = IAA; + SetInflatePath(inflate_settings, IAA); // IAA inflate is stateless in this wrapper. If stream end was not // reached, use zlib for stateful continuation. if (!end_of_stream) { @@ -504,7 +677,7 @@ int ZEXPORT inflate(z_streamp strm, int flush) { ret = UncompressQAT(strm->next_in, &input_len, strm->next_out, &output_len, inflate_settings->window_bits, &end_of_stream); - inflate_settings->path = QAT; + SetInflatePath(inflate_settings, QAT); // QATzip does not support stateful decompression // Fall back to zlib if end-of-stream not reached in one call if (!end_of_stream) { @@ -514,8 +687,74 @@ int ZEXPORT inflate(z_streamp strm, int flush) { INCREMENT_STAT(INFLATE_QAT_COUNT); INCREMENT_STAT_COND(ret != 0, INFLATE_QAT_ERROR_COUNT); #endif // USE_QAT + } else if (path_selected == IGZIP) { +#ifdef USE_IGZIP + in_call = true; + const IGZIPInflatePathAction path_action = + IGZIPRunInflateAndSelectPathAction( + strm, &inflate_settings->isal_strm, inflate_settings->window_bits, + &input_len, &output_len, &ret, &end_of_stream); + in_call = false; + + if (inflate_settings->isal_strm == nullptr) { + return Z_DATA_ERROR; + } + + if (path_action == IGZIP_INFLATE_PATH_FALLBACK_NEED_DICT) { + Log(LogLevel::LOG_ERROR, " strm=", static_cast(strm), + " source=igzip", " total_in=", strm->total_in, + " total_out=", strm->total_out, " adler=", strm->adler, "\n"); + SetInflatePath(inflate_settings, ZLIB); + } else if (path_action == IGZIP_INFLATE_PATH_FALLBACK_DATA_ERROR) { + SetInflatePath(inflate_settings, ZLIB); + } else if (path_action == IGZIP_INFLATE_PATH_SET_IGZIP && + inflate_settings->path != ZLIB) { + SetInflatePath(inflate_settings, IGZIP); + } + INCREMENT_STAT(INFLATE_IGZIP_COUNT); + INCREMENT_STAT_COND(ret != 0, INFLATE_IGZIP_ERROR_COUNT); +#endif } +#ifdef USE_IGZIP + // Accelerator->IGZIP fallback: if IAA or QAT failed and IGZIP is + // available, retry with IGZIP before falling through to software zlib. + if ((path_selected == IAA || path_selected == QAT) && ret != 0 && + configs[IGZIP_FALLBACK] && igzip_available) { + // Accelerator may have modified input_len/output_len on failure. + // Restore them before retrying with IGZIP. + input_len = strm->avail_in; + output_len = strm->avail_out; + end_of_stream = true; + in_call = true; + const IGZIPInflatePathAction path_action = + IGZIPRunInflateAndSelectPathAction( + strm, &inflate_settings->isal_strm, inflate_settings->window_bits, + &input_len, &output_len, &ret, &end_of_stream); + in_call = false; + + if (inflate_settings->isal_strm == nullptr) { + return Z_DATA_ERROR; + } + + const char* failed_accelerator = (path_selected == QAT) ? "QAT" : "IAA"; + if (path_action == IGZIP_INFLATE_PATH_FALLBACK_NEED_DICT) { + Log(LogLevel::LOG_ERROR, " strm=", static_cast(strm), + " source=igzip (", failed_accelerator, " fallback)", + " total_in=", strm->total_in, " total_out=", strm->total_out, + " adler=", strm->adler, "\n"); + SetInflatePath(inflate_settings, ZLIB); + } else if (path_action == IGZIP_INFLATE_PATH_FALLBACK_DATA_ERROR) { + SetInflatePath(inflate_settings, ZLIB); + } else if (path_action == IGZIP_INFLATE_PATH_SET_IGZIP && + inflate_settings->path != ZLIB) { + SetInflatePath(inflate_settings, IGZIP); + } + INCREMENT_STAT(INFLATE_IGZIP_COUNT); + INCREMENT_STAT_COND(ret != 0, INFLATE_IGZIP_ERROR_COUNT); + } +#endif // USE_IGZIP accelerator fallback + if (ret == 0) { strm->next_in += input_len; strm->avail_in -= input_len; @@ -523,30 +762,36 @@ int ZEXPORT inflate(z_streamp strm, int flush) { strm->next_out += output_len; strm->avail_out -= output_len; strm->total_out += output_len; - if (input_len > 0 || output_len > 0) { - if (end_of_stream) { - ret = Z_STREAM_END; - } else { - ret = Z_OK; - } + if (end_of_stream) { + ret = Z_STREAM_END; + } else if (input_len > 0 || output_len > 0) { + ret = Z_OK; } else { ret = Z_BUF_ERROR; } Log(LogLevel::LOG_INFO, "inflate Line ", __LINE__, ", strm ", static_cast(strm), ", accelerator return code ", ret, - ", avail_in ", strm->avail_in, ", avail_out ", strm->avail_out, - ", end_of_stream ", end_of_stream, ", path ", - static_cast(inflate_settings->path), "\n"); + ", bytes_in ", input_len, ", bytes_out ", output_len, ", avail_in ", + strm->avail_in, ", avail_out ", strm->avail_out, ", end_of_stream ", + end_of_stream, ", path ", static_cast(inflate_settings->path), + ", path_name ", static_cast(inflate_settings->path), + ", window_bits ", inflate_settings->window_bits, "\n"); return ret; } } - if (in_call || configs[USE_ZLIB_UNCOMPRESS]) { + if (in_call || configs[USE_ZLIB_UNCOMPRESS] || + inflate_settings->path == ZLIB) { ret = orig_inflate(strm, flush); + if (ret == Z_NEED_DICT) { + Log(LogLevel::LOG_ERROR, " strm=", static_cast(strm), + " source=zlib", " total_in=", strm->total_in, + " total_out=", strm->total_out, " adler=", strm->adler, "\n"); + } INCREMENT_STAT(INFLATE_ZLIB_COUNT); if (!in_call) { - inflate_settings->path = ZLIB; + SetInflatePath(inflate_settings, ZLIB); } } else { ret = Z_DATA_ERROR; @@ -555,7 +800,9 @@ int ZEXPORT inflate(z_streamp strm, int flush) { Log(LogLevel::LOG_INFO, "inflate Line ", __LINE__, ", strm ", static_cast(strm), ", zlib return code ", ret, ", avail_in ", strm->avail_in, ", avail_out ", strm->avail_out, ", path ", - static_cast(inflate_settings->path), "\n"); + static_cast(inflate_settings->path), ", path_name ", + static_cast(inflate_settings->path), ", window_bits ", + inflate_settings->window_bits, "\n"); INCREMENT_STAT_COND(ret < 0, INFLATE_ERROR_COUNT); return ret; @@ -564,6 +811,13 @@ int ZEXPORT inflate(z_streamp strm, int flush) { int ZEXPORT inflateEnd(z_streamp strm) { Log(LogLevel::LOG_INFO, "inflateEnd Line ", __LINE__, ", strm ", static_cast(strm), "\n"); + InflateSettings* inflate_settings = inflate_stream_settings.Get(strm); + if (inflate_settings != nullptr && inflate_settings->isal_strm != nullptr) { +#ifdef USE_IGZIP + EndUncompressIGZIP(inflate_settings->isal_strm); +#endif + inflate_settings->isal_strm = nullptr; + } inflate_stream_settings.Unset(strm); return orig_inflateEnd(strm); } @@ -572,13 +826,24 @@ int ZEXPORT inflateReset(z_streamp strm) { Log(LogLevel::LOG_INFO, "inflateReset Line ", __LINE__, ", strm ", static_cast(strm), "\n"); InflateSettings* inflate_settings = inflate_stream_settings.Get(strm); + int ret = orig_inflateReset(strm); if (inflate_settings != nullptr) { - inflate_settings->path = UNDEFINED; + SetInflatePath(inflate_settings, UNDEFINED); + if (inflate_settings->isal_strm != nullptr) { +#ifdef USE_IGZIP + ResetUncompressIGZIP(inflate_settings->isal_strm); +#endif + } } - return orig_inflateReset(strm); + return ret; } +// Note: compress2 / uncompress2 are one-shot stateless paths. They do NOT +// honor IGZIP_FALLBACK: if the preferred accelerator fails, control falls +// directly to software zlib, bypassing IGZIP even when IGZIP_FALLBACK=1. +// This diverges from the streaming deflate()/inflate() paths intentionally — +// stateless paths have no retry loop and the added complexity is not warranted. int ZEXPORT compress2(Bytef* dest, uLongf* destLen, const Bytef* source, uLong sourceLen, int level) { Log(LogLevel::LOG_INFO, "compress2 Line ", __LINE__, ", sourceLen ", @@ -591,13 +856,17 @@ int ZEXPORT compress2(Bytef* dest, uLongf* destLen, const Bytef* source, bool iaa_available = false; bool qat_available = false; + bool igzip_available = false; #ifdef USE_IAA iaa_available = configs[USE_IAA_COMPRESS] && - SupportedOptionsIAA(15, input_len, output_len); + SupportedOptionsIAA(kWindowBitsZlib, input_len, output_len); #endif #ifdef USE_QAT - qat_available = - configs[USE_QAT_COMPRESS] && SupportedOptionsQAT(15, input_len); + qat_available = configs[USE_QAT_COMPRESS] && + SupportedOptionsQAT(kWindowBitsZlib, input_len); +#endif +#ifdef USE_IGZIP + igzip_available = configs[USE_IGZIP_COMPRESS]; #endif ExecutionPath path_selected = ZLIB; @@ -605,22 +874,46 @@ int ZEXPORT compress2(Bytef* dest, uLongf* destLen, const Bytef* source, path_selected = IAA; } else if (qat_available) { path_selected = QAT; + } else if (igzip_available) { + path_selected = IGZIP; } if (path_selected == IAA) { #ifdef USE_IAA in_call = true; ret = CompressIAA(const_cast(source), &input_len, dest, - &output_len, qpl_path_hardware, 15); + &output_len, qpl_path_hardware, kWindowBitsZlib); in_call = false; #endif // USE_IAA } else if (path_selected == QAT) { #ifdef USE_QAT in_call = true; ret = CompressQAT(const_cast(source), &input_len, dest, - &output_len, 15); + &output_len, kWindowBitsZlib); in_call = false; #endif // USE_QAT + } else if (path_selected == IGZIP) { +#ifdef USE_IGZIP + in_call = true; + struct isal_zstream* isal_strm = InitCompressIGZIP(level, kWindowBitsZlib); + if (isal_strm == nullptr) { + ret = 1; + } else { + unsigned long total_in = 0; + unsigned long total_out = 0; + ret = CompressIGZIP(isal_strm, Z_FINISH, const_cast(source), + &input_len, dest, &output_len, &total_in, &total_out); + if (ret == 0 && !IsIGZIPDeflateFinished(isal_strm)) { + // compress2 is one-shot: if the stream did not reach terminal state + // (ZSTATE_END), the output is incomplete (e.g. trailer truncated by a + // too-small destLen, or not all input consumed). Mirrors the + // !end_of_stream check in the uncompress2 IGZIP path. + ret = 1; + } + EndCompressIGZIP(isal_strm); + } + in_call = false; +#endif } if (ret == 0) { @@ -663,15 +956,19 @@ int ZEXPORT uncompress2(Bytef* dest, uLongf* destLen, const Bytef* source, bool iaa_available = false; bool qat_available = false; + bool igzip_available = false; #ifdef USE_IAA - iaa_available = - configs[USE_IAA_UNCOMPRESS] && - SupportedOptionsIAA(15, input_len, output_len) && - IsIAADecompressible(const_cast(source), input_len, 15); + iaa_available = configs[USE_IAA_UNCOMPRESS] && + SupportedOptionsIAA(kWindowBitsZlib, input_len, output_len) && + IsIAADecompressible(const_cast(source), input_len, + kWindowBitsZlib); #endif #ifdef USE_QAT - qat_available = - configs[USE_QAT_UNCOMPRESS] && SupportedOptionsQAT(15, input_len); + qat_available = configs[USE_QAT_UNCOMPRESS] && + SupportedOptionsQAT(kWindowBitsZlib, input_len); +#endif +#ifdef USE_IGZIP + igzip_available = configs[USE_IGZIP_UNCOMPRESS]; #endif ExecutionPath path_selected = ZLIB; @@ -679,13 +976,16 @@ int ZEXPORT uncompress2(Bytef* dest, uLongf* destLen, const Bytef* source, path_selected = IAA; } else if (qat_available) { path_selected = QAT; + } else if (igzip_available) { + path_selected = IGZIP; } if (path_selected == IAA) { #ifdef USE_IAA in_call = true; ret = UncompressIAA(const_cast(source), &input_len, dest, - &output_len, qpl_path_hardware, 15, &end_of_stream); + &output_len, qpl_path_hardware, kWindowBitsZlib, + &end_of_stream); if (!end_of_stream) { ret = 1; } @@ -695,9 +995,34 @@ int ZEXPORT uncompress2(Bytef* dest, uLongf* destLen, const Bytef* source, #ifdef USE_QAT in_call = true; ret = UncompressQAT(const_cast(source), &input_len, dest, - &output_len, 15, &end_of_stream); + &output_len, kWindowBitsZlib, &end_of_stream); in_call = false; #endif // USE_QAT + } else if (path_selected == IGZIP) { +#ifdef USE_IGZIP + in_call = true; + struct inflate_state* isal_strm = InitUncompressIGZIP(kWindowBitsZlib); + if (isal_strm == nullptr) { + ret = 1; + } else { + unsigned long total_in = 0; + unsigned long total_out = 0; + ret = UncompressIGZIP(isal_strm, const_cast(source), &input_len, + dest, &output_len, &total_in, &total_out, + &end_of_stream); + EndUncompressIGZIP(isal_strm); + if (ret == 0 && !end_of_stream) { + // uncompress2 is one-shot: if ISA-L did not reach end-of-stream, + // the output is incomplete. end_of_stream is used instead of + // input_len residual because IGZIP may consume all bytes without + // seeing the zlib trailer, so checking consumed bytes alone is + // insufficient. Both checks test the same class of failure (partial + // decompression) by the most reliable indicator for each path. + ret = 1; + } + } + in_call = false; +#endif } if (ret == 0) { @@ -767,10 +1092,10 @@ struct GzipFile { io_buf_content = 0; memset(&deflate_stream, 0, sizeof(z_stream)); - orig_deflateInit2_(&deflate_stream, -1, Z_DEFLATED, 31, 8, + orig_deflateInit2_(&deflate_stream, -1, Z_DEFLATED, kWindowBitsGzip, 8, Z_DEFAULT_STRATEGY, ZLIB_VERSION, (int)sizeof(z_stream)); memset(&inflate_stream, 0, sizeof(z_stream)); - orig_inflateInit2_(&inflate_stream, 31, ZLIB_VERSION, + orig_inflateInit2_(&inflate_stream, kWindowBitsGzip, ZLIB_VERSION, (int)sizeof(z_stream)); } @@ -938,14 +1263,19 @@ static int GzwriteAcceleratorCompress(GzipFile* gz, uint8_t* input, int ret = 1; bool iaa_available = false; bool qat_available = false; + bool igzip_available = false; #ifdef USE_IAA - iaa_available = configs[USE_IAA_COMPRESS] && - SupportedOptionsIAA(31, *input_length, *output_length); + iaa_available = + configs[USE_IAA_COMPRESS] && + SupportedOptionsIAA(kWindowBitsGzip, *input_length, *output_length); #endif #ifdef USE_QAT - qat_available = - configs[USE_QAT_COMPRESS] && SupportedOptionsQAT(31, *input_length); + qat_available = configs[USE_QAT_COMPRESS] && + SupportedOptionsQAT(kWindowBitsGzip, *input_length); +#endif +#ifdef USE_IGZIP + igzip_available = configs[USE_IGZIP_COMPRESS]; #endif ExecutionPath path_selected = ZLIB; @@ -953,23 +1283,43 @@ static int GzwriteAcceleratorCompress(GzipFile* gz, uint8_t* input, path_selected = QAT; } else if (iaa_available) { path_selected = IAA; + } else if (igzip_available) { + path_selected = IGZIP; } if (path_selected == IAA) { #ifdef USE_IAA in_call = true; ret = CompressIAA(input, input_length, output, output_length, - qpl_path_hardware, 31, 0, true); + qpl_path_hardware, kWindowBitsGzip, 0, true); gz->path = IAA; in_call = false; #endif // USE_IAA } else if (path_selected == QAT) { #ifdef USE_QAT in_call = true; - ret = CompressQAT(input, input_length, output, output_length, 31, true); + ret = CompressQAT(input, input_length, output, output_length, + kWindowBitsGzip, true); gz->path = QAT; in_call = false; #endif // USE_QAT + } else if (path_selected == IGZIP) { +#ifdef USE_IGZIP + in_call = true; + struct isal_zstream* isal_strm = + InitCompressIGZIP(Z_DEFAULT_COMPRESSION, kWindowBitsGzip); + if (isal_strm == nullptr) { + ret = 1; + } else { + unsigned long total_in = 0; + unsigned long total_out = 0; + ret = CompressIGZIP(isal_strm, Z_FINISH, input, input_length, output, + output_length, &total_in, &total_out); + EndCompressIGZIP(isal_strm); + gz->path = IGZIP; + } + in_call = false; +#endif } return ret; } @@ -988,15 +1338,20 @@ static int GzreadAcceleratorUncompress(GzipFile* gz, uint8_t* input, int ret = 1; bool iaa_available = false; bool qat_available = false; + bool igzip_available = false; #ifdef USE_IAA - iaa_available = configs[USE_IAA_UNCOMPRESS] && - SupportedOptionsIAA(31, *input_length, *output_length) && - IsIAADecompressible(input, *input_length, 31); + iaa_available = + configs[USE_IAA_UNCOMPRESS] && + SupportedOptionsIAA(kWindowBitsGzip, *input_length, *output_length) && + IsIAADecompressible(input, *input_length, kWindowBitsGzip); #endif #ifdef USE_QAT - qat_available = - configs[USE_QAT_UNCOMPRESS] && SupportedOptionsQAT(31, *input_length); + qat_available = configs[USE_QAT_UNCOMPRESS] && + SupportedOptionsQAT(kWindowBitsGzip, *input_length); +#endif +#ifdef USE_IGZIP + igzip_available = configs[USE_IGZIP_UNCOMPRESS]; #endif ExecutionPath path_selected = ZLIB; @@ -1004,24 +1359,44 @@ static int GzreadAcceleratorUncompress(GzipFile* gz, uint8_t* input, path_selected = QAT; } else if (iaa_available) { path_selected = IAA; + } else if (igzip_available) { + path_selected = IGZIP; } if (path_selected == IAA) { #ifdef USE_IAA in_call = true; - ret = UncompressIAA(input, input_length, output, output_length, - qpl_path_hardware, 31, end_of_stream, true); + ret = + UncompressIAA(input, input_length, output, output_length, + qpl_path_hardware, kWindowBitsGzip, end_of_stream, true); gz->path = IAA; in_call = false; #endif // USE_IAA } else if (path_selected == QAT) { #ifdef USE_QAT in_call = true; - ret = UncompressQAT(input, input_length, output, output_length, 31, - end_of_stream, true); + ret = UncompressQAT(input, input_length, output, output_length, + kWindowBitsGzip, end_of_stream, true); gz->path = QAT; in_call = false; #endif // USE_QAT + } else if (path_selected == IGZIP) { +#ifdef USE_IGZIP + in_call = true; + struct inflate_state* isal_strm = InitUncompressIGZIP(kWindowBitsGzip); + if (isal_strm == nullptr) { + ret = 1; + } else { + unsigned long total_in = 0; + unsigned long total_out = 0; + ret = + UncompressIGZIP(isal_strm, input, input_length, output, output_length, + &total_in, &total_out, end_of_stream); + EndUncompressIGZIP(isal_strm); + gz->path = IGZIP; + } + in_call = false; +#endif } return ret; } @@ -1105,8 +1480,9 @@ int ZEXPORT gzwrite(gzFile file, voidpc buf, unsigned len) { static_cast(file), ", buf ", buf, ", len ", len, "\n"); unsigned int written_bytes = 0; - bool accelerator_selected = - configs[USE_IAA_COMPRESS] || configs[USE_QAT_COMPRESS]; + bool accelerator_selected = configs[USE_IAA_COMPRESS] || + configs[USE_QAT_COMPRESS] || + configs[USE_IGZIP_COMPRESS]; if (gz->path != ZLIB && accelerator_selected) { gz->AllocateBuffers(); gz->data_buf_size = 256 << 10; @@ -1164,8 +1540,9 @@ int ZEXPORT gzread(gzFile file, voidp buf, unsigned len) { int ret = 1; uint32_t read_bytes = 0; - bool accelerator_selected = - configs[USE_IAA_UNCOMPRESS] || configs[USE_QAT_UNCOMPRESS]; + bool accelerator_selected = configs[USE_IAA_UNCOMPRESS] || + configs[USE_QAT_UNCOMPRESS] || + configs[USE_IGZIP_UNCOMPRESS]; if (gz->path != ZLIB && accelerator_selected) { gz->AllocateBuffers(); gz->data_buf_size = 512 << 10; @@ -1348,7 +1725,6 @@ int ZEXPORT gzclose(gzFile file) { } else { ret = orig_gzclose(file); } - Log(LogLevel::LOG_INFO, "gzclose Line ", __LINE__, ", file ", static_cast(file), ", return code ", ret, ", buffered processed ", gz->data_buf_pos, "\n"); diff --git a/zlib_accel.h b/zlib_accel.h index c763ea9..2dd2c78 100644 --- a/zlib_accel.h +++ b/zlib_accel.h @@ -7,7 +7,7 @@ #include // Visible for testing -enum ExecutionPath { UNDEFINED, ZLIB, QAT, IAA }; +enum ExecutionPath { UNDEFINED, ZLIB, QAT, IAA, IGZIP }; ExecutionPath GetDeflateExecutionPath(z_streamp strm); ExecutionPath GetInflateExecutionPath(z_streamp strm);