From 3641494188ee4bce1e4bfa04668c518bb33deb69 Mon Sep 17 00:00:00 2001 From: T0k1To Date: Wed, 8 Jul 2026 18:17:32 -0300 Subject: [PATCH 1/3] img: shard the image read cache into per-bank locks for concurrent readers Restructure the TSK_IMG_INFO read cache from a single global cache_lock into 8 independent banks of 4 entries (total capacity unchanged: 32 x 64KB = 2MB), each bank with its own lock: - Cache blocks are 64KB-aligned and the address space maps to banks by 256KB regions, so every block belongs to exactly one bank and lookups scan only that bank's 4 entries (the old code scanned all 32 on every tsk_img_read call). - cache_lock keeps its role as the backend lock: image format backends (raw, AFF, EWF, logical, pool) hold shared state (seek positions, handles) and remain serialized by it. Strict lock order bank->backend and no thread ever holds two banks, so the design is deadlock-free by construction. - Sequential-access detection with a bounded read-ahead of the next 64KB block, tracked per bank; disable at runtime with TSK_IMG_READAHEAD=0. - Lock setup centralized in tsk_img_lock_init()/tsk_img_lock_deinit(), used by img_open and the APFS/LVM pool wrappers. - Requests crossing a 64KB block boundary bypass the cache (direct serialized read, as large reads already did). Benchmarks on this branch (ext4 corpus, 15k files, warm cache, median of 3 interleaved A/B runs, 4-core/8-thread laptop): tsk_img_read 4KB 1 thread: 4180 -> 4507 MB/s (+8%) 2 threads: 3122 -> 3939 MB/s (+26%) 4 threads: 3060 -> 3651 MB/s (+19%) 8 threads: 2575 -> 3436 MB/s (+33%) Validation: tsk_loaddb database byte-identical to the unpatched build (154,616 dump statements); fls -r listing identical (15,379 lines); icat output bit-identical on sampled inodes; fs_thread_test with 8 threads produces identical per-thread logs; ThreadSanitizer clean on fs_thread_test and an 8-thread read microbenchmark. This directly benefits IPED's parallel in-process image reading (ReadContentInputStream via JNI from multiple worker threads). --- tsk/img/img_io.c | 244 ++++++++++++++++++++++++++-------- tsk/img/img_open.cpp | 10 +- tsk/img/tsk_img.h | 38 +++++- tsk/pool/apfs_pool_compat.cpp | 4 +- tsk/pool/lvm_pool_compat.cpp | 4 +- 5 files changed, 232 insertions(+), 68 deletions(-) diff --git a/tsk/img/img_io.c b/tsk/img/img_io.c index ef835bd20d..12f94868b2 100755 --- a/tsk/img/img_io.c +++ b/tsk/img/img_io.c @@ -12,6 +12,51 @@ #include "tsk_img_i.h" +/* Read-ahead kill switch: TSK_IMG_READAHEAD=0 disables the sequential + * prefetch. Checked once; the racy first initialization is benign since + * every thread computes the same value. */ +static int +img_readahead_enabled(void) +{ + static int enabled = -1; + if (enabled < 0) { + const char *env = getenv("TSK_IMG_READAHEAD"); + enabled = (env == NULL || strcmp(env, "0") != 0) ? 1 : 0; + } + return enabled; +} + +/** + * \ingroup imglib + * Initialize the backend lock and the per-bank cache locks of a + * TSK_IMG_INFO. Every code path that creates a TSK_IMG_INFO (including + * the pool wrappers) must call this instead of initializing + * cache_lock directly. + */ +void +tsk_img_lock_init(TSK_IMG_INFO * a_img_info) +{ + int i; + tsk_init_lock(&(a_img_info->cache_lock)); + for (i = 0; i < TSK_IMG_INFO_CACHE_BANKS; i++) { + tsk_init_lock(&(a_img_info->cache_bank_locks[i])); + } +} + +/** + * \ingroup imglib + * Destroy the locks initialized by tsk_img_lock_init(). + */ +void +tsk_img_lock_deinit(TSK_IMG_INFO * a_img_info) +{ + int i; + tsk_deinit_lock(&(a_img_info->cache_lock)); + for (i = 0; i < TSK_IMG_INFO_CACHE_BANKS; i++) { + tsk_deinit_lock(&(a_img_info->cache_bank_locks[i])); + } +} + // This function assumes that we hold the cache_lock even though we're not modifying // the cache. This is because the lower-level read callbacks make the same assumption. static ssize_t tsk_img_read_no_cache(TSK_IMG_INFO * a_img_info, TSK_OFF_T a_off, @@ -53,15 +98,33 @@ static ssize_t tsk_img_read_no_cache(TSK_IMG_INFO * a_img_info, TSK_OFF_T a_off, * @param a_buf Buffer to read into * @param a_len Number of bytes to read into buffer * @returns -1 on error or number of bytes read + * + * Cache design: blocks are TSK_IMG_INFO_CACHE_LEN bytes, aligned to + * TSK_IMG_INFO_CACHE_LEN. The image address space is divided into + * regions of TSK_IMG_INFO_CACHE_BANK_ENTRIES consecutive blocks; each + * region maps to one cache bank, so a block always lives in exactly one + * bank and a lookup only scans that bank's entries under that bank's + * lock. Backend reads are serialized by cache_lock (the image-format + * backends keep shared state such as seek positions). Lock order is + * strictly bank lock -> cache_lock, and no thread ever holds two bank + * locks (the read-ahead only prefetches blocks of the same bank), so + * the locking is deadlock-free. */ ssize_t tsk_img_read(TSK_IMG_INFO * a_img_info, TSK_OFF_T a_off, char *a_buf, size_t a_len) { #define CACHE_AGE 1000 + const TSK_OFF_T region_len = + (TSK_OFF_T) TSK_IMG_INFO_CACHE_LEN * TSK_IMG_INFO_CACHE_BANK_ENTRIES; ssize_t read_count = 0; int cache_index = 0; int cache_next = 0; // index to lowest age cache (to use next) + int bank = 0; + int bank_first = 0; + int bank_last = 0; // exclusive + int hit = 0; + TSK_OFF_T block_off = 0; size_t len2 = 0; if (a_img_info == NULL) { @@ -99,29 +162,27 @@ tsk_img_read(TSK_IMG_INFO * a_img_info, TSK_OFF_T a_off, return -1; } - /* cache_lock is used for both the cache in IMG_INFO and - * the shared variables in the img type specific INFO structs. - * grab it now so that it is held before any reads. - */ - tsk_take_lock(&(a_img_info->cache_lock)); - - // if they ask for more than the cache length, skip the cache - if ((a_len + (a_off % 512)) > TSK_IMG_INFO_CACHE_LEN) { - read_count = tsk_img_read_no_cache(a_img_info, a_off, a_buf, a_len); - tsk_release_lock(&(a_img_info->cache_lock)); - return read_count; - } - // TODO: why not just return 0 here (and be POSIX compliant)? // and why not check earlier for this condition? if (a_off >= a_img_info->size) { - tsk_release_lock(&(a_img_info->cache_lock)); tsk_error_reset(); tsk_error_set_errno(TSK_ERR_IMG_READ_OFF); tsk_error_set_errstr("tsk_img_read - %" PRIdOFF, a_off); return -1; } + /* Requests that cross a cache-block boundary (or exceed the block + * size) bypass the cache. cache_lock serializes the backend call + * and protects the shared variables in the img type specific INFO + * structs. */ + if ((size_t) (a_off % TSK_IMG_INFO_CACHE_LEN) + a_len > + TSK_IMG_INFO_CACHE_LEN) { + tsk_take_lock(&(a_img_info->cache_lock)); + read_count = tsk_img_read_no_cache(a_img_info, a_off, a_buf, a_len); + tsk_release_lock(&(a_img_info->cache_lock)); + return read_count; + } + /* See if the requested length is going to be too long. * we'll use this length when checking the cache. */ len2 = a_len; @@ -132,30 +193,49 @@ tsk_img_read(TSK_IMG_INFO * a_img_info, TSK_OFF_T a_off, len2 = (size_t) (a_img_info->size - a_off); } - // check if it is in the cache - for (cache_index = 0; - cache_index < TSK_IMG_INFO_CACHE_NUM; cache_index++) { + block_off = (a_off / TSK_IMG_INFO_CACHE_LEN) * TSK_IMG_INFO_CACHE_LEN; + bank = (int) ((a_off / region_len) % TSK_IMG_INFO_CACHE_BANKS); + bank_first = bank * TSK_IMG_INFO_CACHE_BANK_ENTRIES; + bank_last = bank_first + TSK_IMG_INFO_CACHE_BANK_ENTRIES; + cache_next = bank_first; + + tsk_take_lock(&(a_img_info->cache_bank_locks[bank])); + + // Detect sequential access pattern for read-ahead (per bank). + // Consider sequential if the new offset equals the expected continuation + // (previous offset + previous length) or is within 8KB of it. + { + TSK_OFF_T expected = a_img_info->last_read_offset[bank] + + (TSK_OFF_T) a_img_info->last_read_len[bank]; + if ((a_off >= a_img_info->last_read_offset[bank]) && + ((a_off == expected) || ((a_off > expected) && (a_off < expected + 8192)))) { + // saturate so the counter cannot overflow on very long streaks + if (a_img_info->sequential_streak[bank] < 1000) + a_img_info->sequential_streak[bank]++; + } else { + a_img_info->sequential_streak[bank] = 0; + } + a_img_info->last_read_offset[bank] = a_off; + } + + // check if the block is in this bank of the cache + for (cache_index = bank_first; cache_index < bank_last; cache_index++) { // Look into the in-use cache entries if (a_img_info->cache_len[cache_index] > 0) { - // the read_count check makes sure we don't go back in after data was read - if ((read_count == 0) - && (a_img_info->cache_off[cache_index] <= a_off) + if ((hit == 0) + && (a_img_info->cache_off[cache_index] == block_off) && (a_img_info->cache_off[cache_index] + - a_img_info->cache_len[cache_index] >= a_off + len2)) { - - /* - if (tsk_verbose) - fprintf(stderr, - "tsk_img_read: Read found in cache %d\n", cache_index ); - */ + (TSK_OFF_T) a_img_info->cache_len[cache_index] >= + a_off + (TSK_OFF_T) len2)) { // We found it... memcpy(a_buf, - &a_img_info->cache[cache_index][a_off - - a_img_info->cache_off[cache_index]], len2); + &a_img_info->cache[cache_index][a_off - block_off], + len2); read_count = (ssize_t) len2; + hit = 1; // reset its "age" since it was useful a_img_info->cache_age[cache_index] = CACHE_AGE; @@ -181,31 +261,19 @@ tsk_img_read(TSK_IMG_INFO * a_img_info, TSK_OFF_T a_off, } // if we didn't find it, then load it into the cache_next entry - if (read_count == 0) { - size_t read_size = 0; - - // round the offset down to a sector boundary - a_img_info->cache_off[cache_next] = (a_off / 512) * 512; - - /* - if (tsk_verbose) - fprintf(stderr, - "tsk_img_read: Loading data into cache %d (%" PRIdOFF - ")\n", cache_next, a_img_info->cache_off[cache_next]); - */ - - // Read a full cache block or the remaining data. - read_size = TSK_IMG_INFO_CACHE_LEN; - - if ((a_img_info->cache_off[cache_next] + (TSK_OFF_T)read_size) > - a_img_info->size) { - read_size = - (size_t) (a_img_info->size - - a_img_info->cache_off[cache_next]); + if (hit == 0) { + size_t read_size = TSK_IMG_INFO_CACHE_LEN; + + if ((block_off + (TSK_OFF_T) read_size) > a_img_info->size) { + read_size = (size_t) (a_img_info->size - block_off); } - read_count = a_img_info->read(a_img_info, - a_img_info->cache_off[cache_next], + /* Backend reads are serialized: the image-format callbacks keep + * shared state (seek positions, library handles) that assumes a + * single caller at a time. Lock order: bank lock -> cache_lock. */ + tsk_take_lock(&(a_img_info->cache_lock)); + + read_count = a_img_info->read(a_img_info, block_off, a_img_info->cache[cache_next], read_size); // if no error, then set the variables and copy the data @@ -217,9 +285,10 @@ tsk_img_read(TSK_IMG_INFO * a_img_info, TSK_OFF_T a_off, TSK_OFF_T rel_off = 0; a_img_info->cache_age[cache_next] = CACHE_AGE; a_img_info->cache_len[cache_next] = read_count; + a_img_info->cache_off[cache_next] = block_off; // Determine the offset relative to the start of the cached data. - rel_off = a_off - a_img_info->cache_off[cache_next]; + rel_off = a_off - block_off; // Make sure we were able to read sufficient data into the cache. if (rel_off > (TSK_OFF_T) read_count) { @@ -234,6 +303,67 @@ tsk_img_read(TSK_IMG_INFO * a_img_info, TSK_OFF_T a_off, memcpy(a_buf, &(a_img_info->cache[cache_next][rel_off]), len2); } read_count = (ssize_t) len2; + + // Read-ahead: if sequential access pattern detected, preload the + // next block into a low-priority slot of the SAME bank (blocks + // of other banks are never touched, keeping the locking + // deadlock-free). Can be disabled with TSK_IMG_READAHEAD=0. + if (img_readahead_enabled() + && a_img_info->sequential_streak[bank] >= 2) { + TSK_OFF_T ra_off = block_off + TSK_IMG_INFO_CACHE_LEN; + + if ((ra_off < a_img_info->size) + && ((ra_off / region_len) == (block_off / region_len))) { + int ra_slot = -1; + int ra_slot_unused = 0; + int already_cached = 0; + int i; + + // Find best slot for prefetch (prefer unused, else + // lowest age) and skip if the next block is already + // in this bank. + for (i = bank_first; i < bank_last; i++) { + if ((a_img_info->cache_len[i] > 0) + && (a_img_info->cache_off[i] == ra_off)) { + already_cached = 1; + break; + } + if (i == cache_next) + continue; + if (a_img_info->cache_len[i] == 0) { + if (!ra_slot_unused) { + ra_slot = i; + ra_slot_unused = 1; + } + } + else if (!ra_slot_unused + && (ra_slot == -1 + || a_img_info->cache_age[i] < + a_img_info->cache_age[ra_slot])) { + ra_slot = i; + } + } + + if (!already_cached && ra_slot != -1) { + size_t ra_rdsize = TSK_IMG_INFO_CACHE_LEN; + ssize_t ra_count; + + if (ra_off + (TSK_OFF_T)ra_rdsize > a_img_info->size) { + ra_rdsize = (size_t)(a_img_info->size - ra_off); + } + + ra_count = a_img_info->read(a_img_info, ra_off, + a_img_info->cache[ra_slot], ra_rdsize); + + if (ra_count > 0) { + a_img_info->cache_off[ra_slot] = ra_off; + a_img_info->cache_len[ra_slot] = ra_count; + // Assign lower age so it can be evicted more easily + a_img_info->cache_age[ra_slot] = CACHE_AGE / 4; + } + } + } + } } else { a_img_info->cache_len[cache_next] = 0; @@ -241,10 +371,16 @@ tsk_img_read(TSK_IMG_INFO * a_img_info, TSK_OFF_T a_off, a_img_info->cache_off[cache_next] = 0; // Something went wrong so let's try skipping the cache + // (cache_lock is already held, as tsk_img_read_no_cache expects) read_count = tsk_img_read_no_cache(a_img_info, a_off, a_buf, a_len); } + + tsk_release_lock(&(a_img_info->cache_lock)); } - tsk_release_lock(&(a_img_info->cache_lock)); + a_img_info->last_read_len[bank] = + (read_count > 0) ? (size_t) read_count : 0; + + tsk_release_lock(&(a_img_info->cache_bank_locks[bank])); return read_count; } diff --git a/tsk/img/img_open.cpp b/tsk/img/img_open.cpp index 2b72d2da51..d433d7ec25 100644 --- a/tsk/img/img_open.cpp +++ b/tsk/img/img_open.cpp @@ -278,8 +278,8 @@ tsk_img_open(int num_img, return NULL; } - /* we have a good img_info, set up the cache lock */ - tsk_init_lock(&(img_info->cache_lock)); + /* we have a good img_info, set up the cache locks */ + tsk_img_lock_init(img_info); return img_info; } @@ -381,7 +381,7 @@ tsk_img_open_utf8(int num_img, free(images16); if (retval) { - tsk_init_lock(&(retval->cache_lock)); + tsk_img_lock_init(retval); } return retval; } @@ -474,7 +474,7 @@ tsk_img_open_external( img_info->close = close; img_info->imgstat = imgstat; - tsk_init_lock(&(img_info->cache_lock)); + tsk_img_lock_init(img_info); return img_info; } @@ -574,6 +574,6 @@ tsk_img_close(TSK_IMG_INFO * a_img_info) if (a_img_info == NULL) { return; } - tsk_deinit_lock(&(a_img_info->cache_lock)); + tsk_img_lock_deinit(a_img_info); a_img_info->close(a_img_info); } diff --git a/tsk/img/tsk_img.h b/tsk/img/tsk_img.h index ebd37562ed..9ad21a07c1 100644 --- a/tsk/img/tsk_img.h +++ b/tsk/img/tsk_img.h @@ -84,6 +84,16 @@ extern "C" { #define TSK_IMG_INFO_CACHE_NUM 32 #define TSK_IMG_INFO_CACHE_LEN 65536 +/* The read cache is split into independent banks, each with its own + * lock, so that concurrent readers only contend when they access the + * same region of the image. Cache blocks are aligned to + * TSK_IMG_INFO_CACHE_LEN and the image address space is mapped onto + * banks in runs of TSK_IMG_INFO_CACHE_BANK_ENTRIES consecutive blocks, + * so every block belongs to exactly one bank. */ +#define TSK_IMG_INFO_CACHE_BANKS 8 +#define TSK_IMG_INFO_CACHE_BANK_ENTRIES \ + (TSK_IMG_INFO_CACHE_NUM / TSK_IMG_INFO_CACHE_BANKS) + typedef struct TSK_IMG_INFO TSK_IMG_INFO; #define TSK_IMG_INFO_TAG 0x39204231 @@ -102,17 +112,35 @@ extern "C" { // the following are protected by cache_lock in IMG_INFO TSK_TCHAR **images; ///< Image names - tsk_lock_t cache_lock; ///< Lock for cache and associated values - char cache[TSK_IMG_INFO_CACHE_NUM][TSK_IMG_INFO_CACHE_LEN]; ///< read cache (r/w shared - lock) - TSK_OFF_T cache_off[TSK_IMG_INFO_CACHE_NUM]; ///< starting byte offset of corresponding cache entry (r/w shared - lock) - int cache_age[TSK_IMG_INFO_CACHE_NUM]; ///< "Age" of corresponding cache entry, higher means more recently used (r/w shared - lock) - size_t cache_len[TSK_IMG_INFO_CACHE_NUM]; ///< Length of cache entry used (0 if never used) (r/w shared - lock) + /* cache_lock serializes calls into the image-format backends + * (which keep shared state such as seek positions and handle + * caches) and any non-cache shared variables in the + * format-specific INFO structs. The cache arrays themselves are + * protected by the per-bank locks: entry i belongs to bank + * i / TSK_IMG_INFO_CACHE_BANK_ENTRIES. Lock order is always + * bank lock -> cache_lock; never the reverse. */ + tsk_lock_t cache_lock; ///< Lock for backend I/O and format-specific shared state + tsk_lock_t cache_bank_locks[TSK_IMG_INFO_CACHE_BANKS]; ///< One lock per cache bank + char cache[TSK_IMG_INFO_CACHE_NUM][TSK_IMG_INFO_CACHE_LEN]; ///< read cache (r/w shared - bank lock) + TSK_OFF_T cache_off[TSK_IMG_INFO_CACHE_NUM]; ///< starting byte offset of corresponding cache entry (r/w shared - bank lock) + int cache_age[TSK_IMG_INFO_CACHE_NUM]; ///< "Age" of corresponding cache entry, higher means more recently used (r/w shared - bank lock) + size_t cache_len[TSK_IMG_INFO_CACHE_NUM]; ///< Length of cache entry used (0 if never used) (r/w shared - bank lock) + + TSK_OFF_T last_read_offset[TSK_IMG_INFO_CACHE_BANKS]; ///< Last offset read per bank (sequential access detection; bank lock) + size_t last_read_len[TSK_IMG_INFO_CACHE_BANKS]; ///< Length of last read per bank (bank lock) + int sequential_streak[TSK_IMG_INFO_CACHE_BANKS]; ///< Consecutive sequential reads per bank (bank lock) ssize_t(*read) (TSK_IMG_INFO * img, TSK_OFF_T off, char *buf, size_t len); ///< \internal External progs should call tsk_img_read() void (*close) (TSK_IMG_INFO *); ///< \internal Progs should call tsk_img_close() void (*imgstat) (TSK_IMG_INFO *, FILE *); ///< Pointer to file type specific function }; + /* Initialize / destroy the cache_lock and the per-bank cache locks + * of a TSK_IMG_INFO. Must be used by every code path that creates + * or tears down a TSK_IMG_INFO (including pool wrappers). */ + extern void tsk_img_lock_init(TSK_IMG_INFO *); + extern void tsk_img_lock_deinit(TSK_IMG_INFO *); + // open and close functions extern TSK_IMG_INFO *tsk_img_open_sing(const TSK_TCHAR * a_image, TSK_IMG_TYPE_ENUM type, unsigned int a_ssize); diff --git a/tsk/pool/apfs_pool_compat.cpp b/tsk/pool/apfs_pool_compat.cpp index 53461abe5c..3cbe046683 100755 --- a/tsk/pool/apfs_pool_compat.cpp +++ b/tsk/pool/apfs_pool_compat.cpp @@ -283,7 +283,7 @@ apfs_img_close(TSK_IMG_INFO * img_info) } // Close the pool image - tsk_deinit_lock(&(img_info->cache_lock)); + tsk_img_lock_deinit(img_info); tsk_img_free(img_info); } @@ -341,7 +341,7 @@ TSK_IMG_INFO * APFSPoolCompat::getImageInfo(const TSK_POOL_INFO *pool_info, TSK_ img_info->spare_size = origInfo->spare_size; img_info->images = origInfo->images; - tsk_init_lock(&(img_info->cache_lock)); + tsk_img_lock_init(img_info); return img_info; diff --git a/tsk/pool/lvm_pool_compat.cpp b/tsk/pool/lvm_pool_compat.cpp index d7c28ee909..39dd80318e 100644 --- a/tsk/pool/lvm_pool_compat.cpp +++ b/tsk/pool/lvm_pool_compat.cpp @@ -77,7 +77,7 @@ lvm_logical_volume_img_close(TSK_IMG_INFO * img_info) IMG_POOL_INFO *pool_img_info = (IMG_POOL_INFO *)img_info; libvslvm_logical_volume_free((libvslvm_logical_volume_t **) &( pool_img_info->impl ), NULL); - tsk_deinit_lock(&(img_info->cache_lock)); + tsk_img_lock_deinit(img_info); tsk_img_free(img_info); } } @@ -168,7 +168,7 @@ TSK_IMG_INFO * LVMPoolCompat::getImageInfo(const TSK_POOL_INFO *pool_info, TSK_D img_info->spare_size = origInfo->spare_size; img_info->images = origInfo->images; - tsk_init_lock(&(img_info->cache_lock)); + tsk_img_lock_init(img_info); return img_info; From a63b4eb3a99fe981f3d6a8b36ce029a4e97a8597 Mon Sep 17 00:00:00 2001 From: T0k1To Date: Wed, 8 Jul 2026 18:17:32 -0300 Subject: [PATCH 2/3] util: parallelize calculateEntropy block counting calculateEntropy() reads up to 99 x 64KB blocks and counts byte frequencies sequentially. Count the blocks with a small thread pool (<= 4 threads, block-stride partitioning), leveraging the sharded image cache for concurrent reads: - Per-block histograms are merged in block order, stopping at the first failed read, replicating the sequential semantics exactly: the result is bit-identical to the sequential pass. - Sequential path kept for Windows, small block counts, thread creation failure, or TSK_ENTROPY_THREADS=0. - The sequential path buffer stays heap-allocated as upstream changed it in 4.15. Measured ~3.4x faster on ext4 (2.28s -> 0.67s over 200 iterations), ~2.2-2.6x on FAT32/ISO9660. ThreadSanitizer clean. Entropy output verified identical to the unpatched build. --- tsk/util/detect_encryption.c | 173 ++++++++++++++++++++++++++++++----- 1 file changed, 149 insertions(+), 24 deletions(-) diff --git a/tsk/util/detect_encryption.c b/tsk/util/detect_encryption.c index e1b837edd5..aea9b0de5f 100644 --- a/tsk/util/detect_encryption.c +++ b/tsk/util/detect_encryption.c @@ -10,6 +10,12 @@ #include "detect_encryption.h" +#include +#include +#ifndef TSK_WIN32 +#include +#endif + // Scans the buffer and returns 1 if the given signature is found, 0 otherwise. // Looks for the signature starting at each byte from startingOffset to endingOffset. int @@ -102,6 +108,49 @@ detectSymantecPGP(const char * buf, size_t len) { return detectSignature(signature, strlen(signature), 0, 32, buf, len); } +#define ENTROPY_BLOCK_LEN 65536 +#define ENTROPY_MAX_BLOCKS 100 + +#ifndef TSK_WIN32 +#include + +// State shared by the entropy worker threads. Blocks are assigned by +// stride (worker t handles blocks where block % numThreads == t); each +// worker fills an independent per-block histogram, and the main thread +// merges them in block order so the result is identical to the +// sequential pass, including the stop at the first failed read. +typedef struct { + TSK_IMG_INFO *img_info; + TSK_DADDR_T offset; + uint64_t firstBlock; + uint64_t endBlock; // exclusive + unsigned stride; + unsigned lane; + int (*blockCounts)[256]; + unsigned char *blockOk; +} entropy_worker_args; + +static void * +entropyWorker(void *ptr) { + entropy_worker_args *args = (entropy_worker_args *)ptr; + char buf[ENTROPY_BLOCK_LEN]; + + for (uint64_t i = args->firstBlock + args->lane; i < args->endBlock; + i += args->stride) { + if (tsk_img_read(args->img_info, args->offset + i * ENTROPY_BLOCK_LEN, + buf, ENTROPY_BLOCK_LEN) != (ssize_t)ENTROPY_BLOCK_LEN) { + continue; // leave blockOk[i] unset; merge stops there + } + for (size_t j = 0; j < ENTROPY_BLOCK_LEN; j++) { + unsigned char b = buf[j] & 0xff; + args->blockCounts[i][b]++; + } + args->blockOk[i] = 1; + } + return NULL; +} +#endif + // Returns the entropy of the beginning of the image. double calculateEntropy(TSK_IMG_INFO * img_info, TSK_DADDR_T offset) { @@ -113,40 +162,116 @@ calculateEntropy(TSK_IMG_INFO * img_info, TSK_DADDR_T offset) { } // Read in blocks of 65536 bytes, skipping the first one that is more likely to contain header data. - size_t bufLen = 65536; - char* buf = (char*)tsk_malloc(bufLen); - if (buf == NULL) { - return 0.0; - } + size_t bufLen = ENTROPY_BLOCK_LEN; size_t bytesRead = 0; - for (uint64_t i = 1; i < 100; i++) { - if ((i + 1) * bufLen > (uint64_t)img_info->size - offset) { - break; - } - if (tsk_img_read(img_info, offset + i * bufLen, buf, bufLen) != (ssize_t) bufLen) { - break; - } + // Number of full blocks available after the skipped header block + uint64_t endBlock = 1; + while (endBlock < ENTROPY_MAX_BLOCKS + && (endBlock + 1) * bufLen <= (uint64_t)img_info->size - offset) { + endBlock++; + } - for (size_t j = 0; j < bufLen; j++) { - unsigned char b = buf[j] & 0xff; - byteCounts[b]++; +#ifndef TSK_WIN32 + // Read and count the blocks with a small pool of threads (the image + // cache supports concurrent readers). Per-block histograms merged in + // block order keep the result bit-identical to the sequential pass. + // Disable with TSK_ENTROPY_THREADS=0. + { + const char *env = getenv("TSK_ENTROPY_THREADS"); + long nproc = sysconf(_SC_NPROCESSORS_ONLN); + unsigned numThreads = (nproc > 4) ? 4 : (nproc > 1 ? (unsigned)nproc : 1); + + if ((env == NULL || strcmp(env, "0") != 0) + && numThreads > 1 && endBlock - 1 >= 2 * numThreads) { + + int (*blockCounts)[256] = (int (*)[256])tsk_malloc( + ENTROPY_MAX_BLOCKS * sizeof(*blockCounts)); + unsigned char *blockOk = (unsigned char *)tsk_malloc(ENTROPY_MAX_BLOCKS); + pthread_t threads[4]; + entropy_worker_args args[4]; + unsigned started = 0; + + if (blockCounts != NULL && blockOk != NULL) { + for (unsigned t = 0; t < numThreads; t++) { + args[t].img_info = img_info; + args[t].offset = offset; + args[t].firstBlock = 1; + args[t].endBlock = endBlock; + args[t].stride = numThreads; + args[t].lane = t; + args[t].blockCounts = blockCounts; + args[t].blockOk = blockOk; + if (pthread_create(&threads[t], NULL, entropyWorker, + &args[t]) != 0) { + break; + } + started++; + } + for (unsigned t = 0; t < started; t++) { + pthread_join(threads[t], NULL); + } + + if (started == numThreads) { + // Merge in block order, stopping at the first failed + // read exactly like the sequential loop. + for (uint64_t i = 1; i < endBlock; i++) { + if (!blockOk[i]) { + break; + } + for (int b = 0; b < 256; b++) { + byteCounts[b] += blockCounts[i][b]; + } + bytesRead += bufLen; + } + free(blockCounts); + free(blockOk); + goto compute; + } + // thread creation failed: fall through to sequential + memset(byteCounts, 0, sizeof(byteCounts)); + bytesRead = 0; + } + free(blockCounts); + free(blockOk); } - bytesRead += bufLen; } +#endif - free(buf); + { + char* buf = (char*)tsk_malloc(bufLen); + if (buf == NULL) { + return 0.0; + } + for (uint64_t i = 1; i < endBlock; i++) { + if (tsk_img_read(img_info, offset + i * bufLen, buf, bufLen) != (ssize_t) bufLen) { + break; + } + + for (size_t j = 0; j < bufLen; j++) { + unsigned char b = buf[j] & 0xff; + byteCounts[b]++; + } + bytesRead += bufLen; + } + free(buf); + } +#ifndef TSK_WIN32 + compute: +#endif // Calculate entropy - double entropy = 0.0; - double log2 = log(2); - for (int i = 0; i < 256; i++) { - if (byteCounts[i] > 0) { - double p = (double)(byteCounts[i]) / bytesRead; - entropy -= p * log(p) / log2; + { + double entropy = 0.0; + double log2 = log(2); + for (int i = 0; i < 256; i++) { + if (byteCounts[i] > 0) { + double p = (double)(byteCounts[i]) / bytesRead; + entropy -= p * log(p) / log2; + } } + return entropy; } - return entropy; } /** From 466fa4cc32ee372e61240d81f24fd704f5c0ceb8 Mon Sep 17 00:00:00 2001 From: T0k1To Date: Wed, 8 Jul 2026 18:17:32 -0300 Subject: [PATCH 3/3] base: replace MD5_memcpy/MD5_memset loops with memcpy/memset The RSA reference MD5 implementation (1991) copies and fills memory with byte-by-byte for loops. Use libc memcpy()/memset() so the compiler can inline and vectorize. NIST vectors pass, including irregular chunking that exercises partial buffering. --- tsk/base/md5c.c | 34 ++++------------------------------ 1 file changed, 4 insertions(+), 30 deletions(-) diff --git a/tsk/base/md5c.c b/tsk/base/md5c.c index 7503943eb8..6cc7380013 100644 --- a/tsk/base/md5c.c +++ b/tsk/base/md5c.c @@ -55,8 +55,6 @@ documentation and/or software. static void MD5Transform(UINT4[4], unsigned char[64]); static void Encode(unsigned char *, UINT4 *, unsigned int); static void Decode(UINT4 *, unsigned char *, unsigned int); -static void MD5_memcpy(POINTER, POINTER, unsigned int); -static void MD5_memset(POINTER, int, unsigned int); static unsigned char PADDING[64] = { 0x80, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, @@ -143,8 +141,7 @@ TSK_MD5_Update(TSK_MD5_CTX * context, unsigned char *input, /* Transform as many times as possible. */ if (inputLen >= partLen) { - MD5_memcpy - ((POINTER) & context->buffer[index], (POINTER) input, partLen); + memcpy(&context->buffer[index], input, partLen); MD5Transform(context->state, context->buffer); for (i = partLen; i + 63 < inputLen; i += 64) @@ -156,8 +153,7 @@ TSK_MD5_Update(TSK_MD5_CTX * context, unsigned char *input, i = 0; /* Buffer remaining input */ - MD5_memcpy - ((POINTER) & context->buffer[index], (POINTER) & input[i], + memcpy(&context->buffer[index], &input[i], inputLen - i); } @@ -190,7 +186,7 @@ TSK_MD5_Final(unsigned char digest[16], TSK_MD5_CTX * context) /* Zeroize sensitive information. */ - MD5_memset((POINTER) context, 0, sizeof(*context)); + memset(context, 0, sizeof(*context)); } /* MD5 basic transformation. Transforms state based on block. @@ -281,7 +277,7 @@ MD5Transform(UINT4 state[4], unsigned char block[64]) /* Zeroize sensitive information. */ - MD5_memset((POINTER) x, 0, sizeof(x)); + memset(x, 0, sizeof(x)); } /* Encodes input (UINT4) into output (unsigned char). Assumes len is @@ -314,25 +310,3 @@ Decode(UINT4 *output, unsigned char *input, unsigned int len) 24); } -/* Note: Replace "for loop" with standard memcpy if possible. - */ - -static void -MD5_memcpy(POINTER output, POINTER input, unsigned int len) -{ - unsigned int i; - - for (i = 0; i < len; i++) - output[i] = input[i]; -} - -/* Note: Replace "for loop" with standard memset if possible. - */ -static void -MD5_memset(POINTER output, int value, unsigned int len) -{ - unsigned int i; - - for (i = 0; i < len; i++) - ((char *) output)[i] = (char) value; -}