Initial draft PR to establish igzip integration branch for ongoing IGZIP fixes and improvements

.gitignore

@@ -1,2 +1,5 @@
 /build/
-/tests/build
+/tests/build
+cmake.txt
+/resources/
+

CMakeLists.txt

@@ -15,7 +15,7 @@ if(ENABLE_STATISTICS)
   add_compile_definitions(ENABLE_STATISTICS)
 endif()
 
-add_library(${PROJECT_NAME} SHARED config/config_reader.cpp config/config.cpp zlib_accel.cpp iaa.cpp qat.cpp utils.cpp statistics.cpp)
+add_library(${PROJECT_NAME} SHARED config/config_reader.cpp config/config.cpp zlib_accel.cpp iaa.cpp qat.cpp utils.cpp statistics.cpp igzip.cpp)
 
 add_custom_target(format
     find .. -iname '*.h' -o -iname '*.cpp' | xargs clang-format -style=Google -i

README.md

@@ -59,8 +59,10 @@ make
 CMake supports the following options:
 - USE_QAT (ON/OFF): include QAT acceleration
 - USE_IAA (ON/OFF): include IAA acceleration
+- USE_IGZIP (ON/OFF): include IGZIP acceleration (requires ISA-L)
 - QPL_PATH: path to QPL for IAA acceleration (if not in a standard directory)
 - QATZIP_PATH: path to QATzip for QAT acceleration (if not in a standard directory)
+- ISAL_PATH: path to ISA-L for IGZIP acceleration (if not in a standard directory)
 - DEBUG_LOG (ON/OFF): enable logging
 - ENABLE_STATISTICS (ON/OFF): enable statistics
 - COVERAGE (ON/OFF): enable test coverage (more details in a later section)
@@ -84,6 +86,9 @@ Requirements for IAA
 - [accel-config](https://github.com/intel/idxd-config)
 - [Query Processing Library](https://github.com/intel/qpl)
 
+Requirements for IGZIP
+- [ISA-L (Intel Intelligent Storage Acceleration Library)](https://github.com/intel/isa-l)
+
 A setup with both QAT and IAA enabled has been tested on an AWS m7i.metal-24xl instance (Ubuntu 22.04, kernel 6.8.0).
 Refer to the links above for instructions on how to install the dependencies.
 
@@ -178,20 +183,22 @@ use_zlib_uncompress
 - Enable zlib for decompression
 - Setting to 1 is recommended, to allow fall back to zlib in case accelerators cannot be used or experience an error.
 
+igzip_fallback
+- Values: 0,1. Default: 1
+- If 1, and an IAA or QAT compression/decompression operation fails, the request is retried using IGZIP (if enabled) before falling back to software zlib. Useful on machines where hardware accelerators are intermittently unavailable.
+
 iaa_compress_percentage
 - Values: 0-100. Default: 50
 - If both IAA and QAT are enabled, percentage of compression calls to offload to IAA.
 
 iaa_prepend_empty_block
 - Values: 0,1. Default: 0
-- Prepend an empty stored block to the compressed data to "mark" that the data was compressed by IAA.
-- IAA has a 4kB history window limit and it is not able to decompress blocks that use a longer history window (up to 32kB per deflate standard).
-- During decompression, this marker indicates that the data was compressed by IAA and is therefore guarateed decompressible by IAA.
+- **Deprecated.** This option is retained for backward compatibility and will be removed in a future release. Setting it to 1 has no effect on decompression.
+- Background: the original design prepended a 5-byte empty stored-block marker to IAA-compressed output so the decompressor could identify IAA-produced data (which uses a 4kB history window). This approach was abandoned because QPL hardware always consumes all `available_in` bytes regardless of where the stream boundary falls, making marker-based detection unreliable when the caller does not supply the exact compressed size. IAA decompression eligibility is now determined by a 512-byte minimum input length threshold: callers such as Java's `ZipInputStream` feed chunks of ≤512 bytes when the compressed size is unknown, while Lucene stored-field reads always supply the exact size (>512 bytes).
 
 iaa_uncompress_percentage
 - Values: 0-100. Default: 50
 - If both IAA and QAT are enabled, percentage of decompression calls to offload to IAA.
-- If iaa_prepend_empty_block = 1, this percentage is only applied to data with the empty block marker.
 
 qat_periodical_polling = 0
 - Values: 0,1. Default: 0

common.cmake

@@ -2,6 +2,7 @@
 # SPDX-License-Identifier: Apache-2.0
 
 option(USE_IAA "Use IAA (requires QPL)" OFF)
+option(USE_IGZIP "Use IGZIP (requires isa-l)" OFF)
 option(USE_QAT "Use QAT (requires QATzip)" OFF)
 option(DEBUG_LOG "for logging" ON)
 option(COVERAGE "for coverage" OFF)
@@ -14,6 +15,61 @@ if(USE_IAA)
   add_compile_definitions(USE_IAA)
 endif()
 
+if(USE_IGZIP)
+  add_compile_definitions(USE_IGZIP)
+
+  # Enforce minimum ISA-L version. v2.32.1 contains critical igzip bugfixes:
+  #   - next_in/avail_in not reset after block finish (inflate boundary bug)
+  #   - zlib DICTID stored/read in wrong byte order
+  #   - gzip_flag mutated by isal_deflate when writing wrapper header
+  # Earlier versions require defensive workarounds that have been removed.
+  if(ISAL_PATH)
+    set(_isal_api_header "${ISAL_PATH}/include/isal_api.h")
+  else()
+    find_path(_isal_api_include isal_api.h PATH_SUFFIXES include)
+    if(_isal_api_include)
+      set(_isal_api_header "${_isal_api_include}/isal_api.h")
+    endif()
+    unset(_isal_api_include CACHE)
+  endif()
+  if(NOT _isal_api_header OR NOT EXISTS "${_isal_api_header}")
+    message(FATAL_ERROR "ISA-L header (isal_api.h) not found. Set -DISAL_PATH=<path> or install ISA-L system-wide.")
+  endif()
+  file(STRINGS "${_isal_api_header}" _isal_major_line REGEX "^#define[ \t]+ISAL_MAJOR_VERSION[ \t]+")
+  file(STRINGS "${_isal_api_header}" _isal_minor_line REGEX "^#define[ \t]+ISAL_MINOR_VERSION[ \t]+")
+  file(STRINGS "${_isal_api_header}" _isal_patch_line REGEX "^#define[ \t]+ISAL_PATCH_VERSION[ \t]+")
+  if(NOT _isal_major_line OR NOT _isal_minor_line OR NOT _isal_patch_line)
+    message(FATAL_ERROR "Could not find ISAL_MAJOR/MINOR/PATCH_VERSION macros in ${_isal_api_header}. ISA-L header may be malformed or too old.")
+  endif()
+  list(GET _isal_major_line 0 _isal_major_line)
+  list(GET _isal_minor_line 0 _isal_minor_line)
+  list(GET _isal_patch_line 0 _isal_patch_line)
+  string(REGEX REPLACE "^#define[ \t]+ISAL_MAJOR_VERSION[ \t]+([0-9]+).*" "\\1" _isal_major "${_isal_major_line}")
+  string(REGEX REPLACE "^#define[ \t]+ISAL_MINOR_VERSION[ \t]+([0-9]+).*" "\\1" _isal_minor "${_isal_minor_line}")
+  string(REGEX REPLACE "^#define[ \t]+ISAL_PATCH_VERSION[ \t]+([0-9]+).*" "\\1" _isal_patch "${_isal_patch_line}")
+  foreach(_v _isal_major _isal_minor _isal_patch)
+    if(NOT "${${_v}}" MATCHES "^[0-9]+$")
+      message(FATAL_ERROR "Failed to parse ISA-L version from ${_isal_api_header}: '${_v}' = '${${_v}}'")
+    endif()
+  endforeach()
+  set(_isal_version_found "${_isal_major}.${_isal_minor}.${_isal_patch}")
+  message(STATUS "Found ISA-L version: ${_isal_version_found} (${_isal_api_header})")
+  if(_isal_version_found VERSION_LESS "2.32.1")
+    message(FATAL_ERROR
+      "ISA-L >= 2.32.1 required, found ${_isal_version_found}.\n"
+      "Update ISA-L or pass -DISAL_PATH=<path> to cmake.\n"
+      "Release: https://github.com/intel/isa-l/releases/tag/v2.32.1")
+  endif()
+  unset(_isal_api_header)
+  unset(_isal_major_line)
+  unset(_isal_minor_line)
+  unset(_isal_patch_line)
+  unset(_isal_major)
+  unset(_isal_minor)
+  unset(_isal_patch)
+  unset(_isal_version_found)
+endif()
+
 if(USE_QAT)
   add_compile_definitions(USE_QAT)
 endif()
@@ -84,16 +140,31 @@ if(USE_IAA)
   else()
     message(STATUS "Using QPL_PATH: ${QPL_PATH}")
     include_directories(${QPL_PATH}/include/qpl ${QPL_PATH}/include)
-    link_directories(PUBLIC ${QPL_PATH}/lib64 ${QPL_PATH}/lib)
+    link_directories(${QPL_PATH}/lib64 ${QPL_PATH}/lib)
     link_libraries(qpl)
   endif()
 endif()
 
+if(USE_IGZIP)
+	if(NOT DEFINED ISAL_PATH)
+	  find_package(isal REQUIRED)
+	  if(isal_FOUND)
+	    message(STATUS "Found ISA-L: ${isal_DIR}")
+	    link_libraries(isal)
+	  endif()
+	  else()
+    message(STATUS "Using ISAL_PATH: ${ISAL_PATH}")
+    include_directories(${ISAL_PATH}/include)
+    link_directories(${ISAL_PATH}/.libs)
+    link_libraries(isal)
+  endif()
+endif()
+
 if(USE_QAT)
   if(DEFINED QATZIP_PATH)
     message(STATUS "Using QATZIP_PATH: ${QATZIP_PATH}")
     include_directories(${QATZIP_PATH}/include)
-    link_directories(PUBLIC ${QATZIP_PATH}/src/.libs)
+    link_directories(${QATZIP_PATH}/src/.libs)
   endif()
   link_libraries(qatzip)
 endif()

config/config.cpp

@@ -21,13 +21,16 @@ uint32_t configs[CONFIG_MAX] = {
     0,   /*use_iaa_uncompress*/
     1,   /*use_zlib_compress*/
     1,   /*use_zlib_uncompress*/
+    0,   /*use_igzip_compress*/
+    0,   /*use_igzip_uncompress*/
     50,  /*iaa_compress_percentage*/
     50,  /*iaa_uncompress_percentage*/
     0,   /*iaa_prepend_empty_block*/
     0,   /*qat_periodical_polling*/
     1,   /*qat_compression_level*/
     0,   /*qat_compression_allow_chunking*/
     0,   /*ignore_zlib_dictionary*/
+    1,   /*igzip_fallback*/
     1,   /*log_level*/
     1000 /*log_stats_samples*/
 };
@@ -45,13 +48,16 @@ bool LoadConfigFile(std::string& file_content, const char* file_path) {
     "use_iaa_uncompress",
     "use_zlib_compress",
     "use_zlib_uncompress",
+    "use_igzip_compress",
+    "use_igzip_uncompress",
     "iaa_compress_percentage",
     "iaa_uncompress_percentage",
     "iaa_prepend_empty_block",
     "qat_periodical_polling",
     "qat_compression_level",
 	  "qat_compression_allow_chunking",
     "ignore_zlib_dictionary",
+    "igzip_fallback",
     "log_level",
     "log_stats_samples"
   };
@@ -78,13 +84,16 @@ bool LoadConfigFile(std::string& file_content, const char* file_path) {
   trySetConfig(USE_IAA_UNCOMPRESS, 1, 0);
   trySetConfig(USE_ZLIB_COMPRESS, 1, 0);
   trySetConfig(USE_ZLIB_UNCOMPRESS, 1, 0);
+  trySetConfig(USE_IGZIP_COMPRESS, 1, 0);
+  trySetConfig(USE_IGZIP_UNCOMPRESS, 1, 0);
   trySetConfig(IAA_COMPRESS_PERCENTAGE, 100, 0);
   trySetConfig(IAA_UNCOMPRESS_PERCENTAGE, 100, 0);
   trySetConfig(IAA_PREPEND_EMPTY_BLOCK, 1, 0);
   trySetConfig(QAT_PERIODICAL_POLLING, 1, 0);
   trySetConfig(QAT_COMPRESSION_LEVEL, 9, 1);
   trySetConfig(QAT_COMPRESSION_ALLOW_CHUNKING, 1, 0);
   trySetConfig(IGNORE_ZLIB_DICTIONARY, 1, 0);
+  trySetConfig(IGZIP_FALLBACK, 1, 0);
   trySetConfig(LOG_LEVEL, 3, 0);
   trySetConfig(LOG_STATS_SAMPLES, UINT32_MAX, 0);
 

config/config.h

@@ -16,13 +16,16 @@ enum ConfigOption {
   USE_IAA_UNCOMPRESS,
   USE_ZLIB_COMPRESS,
   USE_ZLIB_UNCOMPRESS,
+  USE_IGZIP_COMPRESS,
+  USE_IGZIP_UNCOMPRESS,
   IAA_COMPRESS_PERCENTAGE,
   IAA_UNCOMPRESS_PERCENTAGE,
-  IAA_PREPEND_EMPTY_BLOCK,
+  IAA_PREPEND_EMPTY_BLOCK,  // DEPRECATED — see README for details
   QAT_PERIODICAL_POLLING,
   QAT_COMPRESSION_LEVEL,
   QAT_COMPRESSION_ALLOW_CHUNKING,
   IGNORE_ZLIB_DICTIONARY,
+  IGZIP_FALLBACK,
   LOG_LEVEL,
   LOG_STATS_SAMPLES,
   CONFIG_MAX

config/default_config

@@ -4,6 +4,8 @@ use_iaa_compress = 0
 use_iaa_uncompress = 0
 use_zlib_compress = 1
 use_zlib_uncompress = 1
+use_igzip_compress = 0
+use_igzip_uncompress = 0
 iaa_compress_percentage = 50
 iaa_uncompress_percentage = 50
 iaa_prepend_empty_block = 0

iaa.cpp

@@ -99,9 +99,19 @@ int CompressIAA(uint8_t* input, uint32_t* input_length, uint8_t* output,
     output_shift += GZIP_EXT_XHDR_SIZE;
   }
 
-  // If prepending an empty block, leave space for it to be added
-  // For zlib format, we don't need an empty block as a marker, as the zlib
-  // header includes info about the window size
+  // DEPRECATED: iaa_prepend_empty_block is no longer used by the decompressor.
+  // The original design used a 5-byte empty stored-block marker written at the
+  // start of IAA-compressed output so that the decompressor could detect and
+  // trust IAA-compressed data (which uses a 4kB history window). This approach
+  // was abandoned because QPL hardware always consumes all available_in bytes
+  // regardless of where the BFINAL=1 token falls (overconsumption bug), so a
+  // caller-supplied exact boundary is required instead. IsIAADecompressible now
+  // uses a 512-byte minimum input length threshold to gate IAA decompression:
+  // Java ZipInputStream feeds <=512-byte chunks (csize unknown), triggering
+  // overconsumption; Lucene stored-field reads always provide the exact
+  // compressed size (>512 bytes), where consuming all input is correct.
+  // The config option is retained for backward compatibility and will be
+  // removed in a future release.
   bool prepend_empty_block = false;
   CompressedFormat format = GetCompressedFormat(window_bits);
   if (format != CompressedFormat::ZLIB &&
@@ -255,44 +265,32 @@ bool SupportedOptionsIAA(int window_bits, uint32_t input_length,
   return false;
 }
 
-bool PrependedEmptyBlockPresent(uint8_t* input, uint32_t input_length,
-                                CompressedFormat format) {
-  uint32_t header_length = GetHeaderLength(format);
-  if (header_length + PREPENDED_BLOCK_LENGTH > input_length) {
-    return false;
-  }
-
-  if (input[header_length] == 0 && input[header_length + 1] == 0 &&
-      input[header_length + 2] == 0 && input[header_length + 3] == 0xFF &&
-      input[header_length + 4] == 0xFF) {
-    Log(LogLevel::LOG_INFO, "PrependedEmptyBlockPresent() Line ", __LINE__,
-        " Empty block detected\n");
-    return true;
-  }
-
-  return false;
-}
-
 bool IsIAADecompressible(uint8_t* input, uint32_t input_length,
                          int window_bits) {
   CompressedFormat format = GetCompressedFormat(window_bits);
   if (format == CompressedFormat::ZLIB) {
     int window = GetWindowSizeFromZlibHeader(input, input_length);
-    Log(LogLevel::LOG_INFO, "IsIAADecompressible() Line ", __LINE__, " window ",
-        window, "\n");
     return window <= 12;
-  } else {
-    // if no empty block markers selected, we cannot tell for sure it's
-    // IAA-decompression, but we assume it is.
-    if (configs[IAA_PREPEND_EMPTY_BLOCK] == 0) {
-      return true;
-    } else if (configs[IAA_PREPEND_EMPTY_BLOCK] == 1 &&
-               PrependedEmptyBlockPresent(input, input_length, format)) {
-      return true;
-    } else {
-      return false;
-    }
   }
+  // For raw deflate and gzip formats, QPL always reports total_in ==
+  // available_in regardless of where BFINAL=1 falls in the stream. This is
+  // safe only when the caller provides avail_in == actual_compressed_size
+  // (e.g. Lucene stored-field reads, where the exact compressed size is known
+  // from the .fdt file format).
+  //
+  // Callers that do not know the compressed size a priori — notably Java's
+  // ZipInputStream, which uses a fixed 512-byte internal buffer and feeds
+  // chunks of that size to inflate() — will have avail_in > actual_csize.
+  // QPL consuming all 512 bytes then reporting total_in=512 when actual_csize
+  // was 2 triggers ZipException at the Java level.
+  //
+  // Guard: only attempt IAA when input_length > 512. ZipInputStream always
+  // feeds chunks of at most 512 bytes, so any call above that threshold is
+  // guaranteed not to be a ZipInputStream-chunked read. Lucene stored-field
+  // entries are typically much larger than 512 bytes; the few that are smaller
+  // fall back to IGZIP which is also correct.
+  static constexpr uint32_t kZipInputStreamBufferSize = 512;
+  return input_length > kZipInputStreamBufferSize;
 }
 
 #endif  // USE_IAA