From 8a25ff4dd50fd9432c6d77a1532e1dcc7e664625 Mon Sep 17 00:00:00 2001 From: Deamhan Date: Thu, 14 Mar 2019 18:30:02 +0300 Subject: [PATCH 01/54] CMakeLists.txt has been added (works fine with VS2017) --- makefiles/CMakeLists.txt | 178 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 178 insertions(+) create mode 100644 makefiles/CMakeLists.txt diff --git a/makefiles/CMakeLists.txt b/makefiles/CMakeLists.txt new file mode 100644 index 0000000..22f35fc --- /dev/null +++ b/makefiles/CMakeLists.txt @@ -0,0 +1,178 @@ +cmake_minimum_required(VERSION 3.6) + +project(bdelta) +set(CMAKE_CXX_STANDARD 17) + +option(BDELTA_PGO "build using PGO instead of PGI (MSVC only)" "OFF") + +IF(MSVC) + set(CMAKE_CONFIGURATION_TYPES "Debug;Release;PGO" CACHE STRING "" FORCE) +ELSE() + set(CMAKE_CONFIGURATION_TYPES "Debug;Release" CACHE STRING "" FORCE) +ENDIF() + +set (COMMON_SOURCES + + compatibility.h + checksum.h + file.h +) + +set (BDELTA_SOURCES + + ${COMMON_SOURCES} + bdelta.cpp + libbdelta.cpp +) + +set (BPATCH_SOURCES + + ${COMMON_SOURCES} + bpatch.cpp +) + +IF(MSVC) + IF(CMAKE_CXX_COMPILER_VERSION VERSION_LESS 19.15) + message(FATAL_ERROR "Your Visual Studio is too old. Fully updated Visual Studio 2017 or newer is required") + ENDIF() + add_compile_options(/Zi # pdb + /W4 # warning level 4 + /EHsc # exceptions: sync + /J # use unsigned char + /Gd # use cdecl + # treat warnings as errors + /we4715 # not all control paths return a value + /we4828 # disallow invalid characters + # prinf-like functions: format mismatch + /we4473 # : not enough arguments passed for format string + /we4474 # : too many arguments passed for format string + /we4475 # : length modifier cannot be used with type field character in format specifier + /we4476 # : unknown type field character in format specifier + /we4477 # : format string requires an argument of type , but variadic argument has type + /we4478 # : positional and non-positional placeholders cannot be mixed in the same format string + /we4775 # nonstandard extension used in format string of function + /we4776 # % is not allowed in the format string of function + /we4777 # : format string requires an argument of type , but variadic argument has type + /we4778 # : unterminated format string + # macro arg mismatch + /we4002 # too many actual parameters for macro 'identifier' + /we4003 # not enough actual parameters for macro 'identifier' + /Zc:threadSafeInit- # https://connect.microsoft.com/VisualStudio/feedback/details/1789709/visual-c-2015-runtime-broken-on-windows-server-2003-c-11-magic-statics + /MP # multiprocessor compilation + /utf-8 # utf-8 source & exec + /GF # eliminate duplicate strings + /std:c++17) + + SET(CMAKE_CXX_FLAGS ${CMAKE_C_FLAGS}) + + add_definitions(-D_CRT_SECURE_NO_WARNINGS) + + SET(CMAKE_EXE_LINKER_FLAGS "/LARGEADDRESSAWARE /SUBSYSTEM:CONSOLE") + SET(CMAKE_EXE_LINKER_FLAGS_RELEASE "${CMAKE_EXE_LINKER_FLAGS_RELEASE} /LTCG /OPT:REF,ICF /DEBUG") + SET(CMAKE_CXX_FLAGS_RELEASE "${CMAKE_CXX_FLAGS_RELEASE} /MD -DNDEBUG /Ox /Ob2 /Oi /Ot /Oy /GS- /Gy /GR- /GL /Gw /MT") + SET(CMAKE_C_FLAGS_RELEASE ${CMAKE_CXX_FLAGS_RELEASE}) + SET(CMAKE_CXX_FLAGS_DEBUG "${CMAKE_CXX_FLAGS_DEBUG} /MDd /GS /MTd -D_DEBUG") + SET(CMAKE_C_FLAGS_DEBUG ${CMAKE_CXX_FLAGS_DEBUG}) + + set(CMAKE_CXX_FLAGS_PGO ${CMAKE_CXX_FLAGS_RELEASE}) + set(CMAKE_C_FLAGS_PGO ${CMAKE_C_FLAGS_RELEASE}) + + IF (BDELTA_PGO) + set(CMAKE_SHARED_LINKER_FLAGS_PGO "${CMAKE_SHARED_LINKER_FLAGS_RELEASE} /LTCG:PGOptimize") + set(CMAKE_EXE_LINKER_FLAGS_PGO "${CMAKE_EXE_LINKER_FLAGS_PGO} /LTCG:PGOptimize") + ELSE() + set(CMAKE_SHARED_LINKER_FLAGS_PGO "${CMAKE_SHARED_LINKER_FLAGS_RELEASE} /LTCG:PGInstrument") + set(CMAKE_EXE_LINKER_FLAGS_PGO "${CMAKE_EXE_LINKER_FLAGS_PGO} /LTCG:PGInstrument") + ENDIF() + +ELSE() + IF(CMAKE_COMPILER_IS_GNUCC) + IF(CMAKE_CXX_COMPILER_VERSION VERSION_LESS 8.2) + message(FATAL_ERROR "Your GCC is too old. GCC 8.2 or newer is required") + ENDIF() + ENDIF() + add_compile_options(-pipe + -W + -Wall + + -Wextra + -Wwrite-strings + -Wframe-larger-than=16384 + -Wstack-usage=16384 + -fdiagnostics-show-option + -Wmissing-declarations + -Wredundant-decls + -Wcast-qual + -Wsuggest-attribute=noreturn + -Wsuggest-attribute=format + -Wunused-but-set-variable + -Wunused-but-set-parameter + + -Wframe-larger-than=4096 + + -Wno-multichar + -Wno-strict-aliasing + -Wno-missing-field-initializers + + -Werror=return-type + -Werror=pointer-arith + -Werror=format + -Werror=format-extra-args + -Werror=unused-value + -Werror=sizeof-pointer-memaccess + -Werror=implicit-function-declaration + + -Werror=missing-declarations + -Werror=missing-prototypes + -Werror=reorder + -Werror=declaration-after-statement + -Werror=missing-format-attribute + -funsigned-char) + + SET(CMAKE_CXX_FLAGS "${CMAKE_C_FLAGS} -Werror=delete-non-virtual-dtor -U__STRICT_ANSI__ -fno-operator-names -std=c++17") + SET(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -Wimplicit-int -Wmissing-prototypes -Werror=implicit-int") + + SET(CMAKE_CXX_FLAGS_RELEASE "-O2 -s -fno-rtti -DNDEBUG") + SET(CMAKE_C_FLAGS_RELEASE ${CMAKE_CXX_FLAGS_RELEASE}) + SET(CMAKE_CXX_FLAGS_DEBUG "-O0 -g -D_DEBUG -D__DEBUG__ -DDEBUG_LEVEL=3 -D_GLIBCXX_DEBUG -D_GLIBCXX_DEBUG_PEDANTIC") + SET(CMAKE_C_FLAGS_DEBUG ${CMAKE_CXX_FLAGS_DEBUG}) + + SET(CMAKE_EXE_LINKER_FLAGS_DEBUG "${CMAKE_EXE_LINKER_FLAGS} -g") + SET(CMAKE_SHARED_LINKER_FLAGS_DEBUG "${CMAKE_SHARED_LINKER_FLAGS} -g") + + SET(CMAKE_EXE_LINKER_FLAGS_RELEASE "${CMAKE_EXE_LINKER_FLAGS} -s") + SET(CMAKE_SHARED_LINKER_FLAGS_RELEASE "${CMAKE_SHARED_LINKER_FLAGS} -s") + + add_compile_options (-m32) + SET(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -m32 -static-libgcc -static-libstdc++") + SET(CMAKE_SHARED_LINKER_FLAGS "${CMAKE_SHARED_LINKER_FLAGS} -m32 -static-libgcc -static-libstdc++") +ENDIF() + +string(TIMESTAMP BUILD_YEAR "%Y") +add_definitions(-DBUILD_YEAR="${BUILD_YEAR}") + +add_executable ( + bdelta + + ${BDELTA_SOURCES} +) + +add_executable ( + bpatch + + ${BPATCH_SOURCES} +) + +IF (NOT MSVC) + SET ( + LIBS + + stdc++fs + pthread + dl + ) + target_link_libraries(bdelta ${LIBS}) + target_link_libraries(bpatch ${LIBS}) +ENDIF() + +set_property(DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR} PROPERTY VS_STARTUP_PROJECT bdelta) From f0b76666e9530d23f90080f79e401ffd935bce19 Mon Sep 17 00:00:00 2001 From: Deamhan Date: Thu, 14 Mar 2019 19:23:01 +0300 Subject: [PATCH 02/54] CMakeLists.txt: src path has been fixed & USE_CXX17 has been set --- makefiles/CMakeLists.txt | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/makefiles/CMakeLists.txt b/makefiles/CMakeLists.txt index 22f35fc..56cc38b 100644 --- a/makefiles/CMakeLists.txt +++ b/makefiles/CMakeLists.txt @@ -13,22 +13,22 @@ ENDIF() set (COMMON_SOURCES - compatibility.h - checksum.h - file.h + ../src/compatibility.h + ../src/checksum.h + ../src/file.h ) set (BDELTA_SOURCES ${COMMON_SOURCES} - bdelta.cpp - libbdelta.cpp + ../src/bdelta.cpp + ../src/libbdelta.cpp ) set (BPATCH_SOURCES ${COMMON_SOURCES} - bpatch.cpp + ../src/bpatch.cpp ) IF(MSVC) @@ -150,14 +150,15 @@ ENDIF() string(TIMESTAMP BUILD_YEAR "%Y") add_definitions(-DBUILD_YEAR="${BUILD_YEAR}") +add_definitions(-DUSE_CXX17) -add_executable ( +add_executable( bdelta ${BDELTA_SOURCES} ) -add_executable ( +add_executable( bpatch ${BPATCH_SOURCES} From 94381c2b50a7e13faa956b9fc95204e225992490 Mon Sep 17 00:00:00 2001 From: Deamhan Date: Thu, 14 Mar 2019 19:24:11 +0300 Subject: [PATCH 03/54] file.h: helper functions performance has been improved --- src/file.h | 157 ++++++++++++++++++++++++++++++++++++++--------------- 1 file changed, 113 insertions(+), 44 deletions(-) diff --git a/src/file.h b/src/file.h index 0e03fb0..8d1f0de 100644 --- a/src/file.h +++ b/src/file.h @@ -2,85 +2,154 @@ * License, v. 2.0. If a copy of the MPL was not distributed with this * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ +#include +#ifdef USE_CXX17 +#include +#endif // USE_CXX17 #include +#include #define MAX_IO_BLOCK_SIZE (1024 * 1024) -void fread_fixed(FILE *f, void * _buf, unsigned num_bytes) { - char * buf = (char *)_buf; - - while (num_bytes != 0) - { - unsigned block_size = num_bytes; - if (block_size > MAX_IO_BLOCK_SIZE) block_size = MAX_IO_BLOCK_SIZE; - - size_t r = fread(buf, 1, block_size, f); - if (r != block_size) - { - static char read_error_message[128]; - sprintf (read_error_message, "read error: fread_fixed(block_size=%u) != %u", block_size, (unsigned)r); - throw read_error_message; - } - buf += block_size; - num_bytes -= block_size; - } +static char error_message_buffer[512]; + +void fread_fixed(FILE *f, void * _buf, unsigned num_bytes) +{ + char * buf = (char *)_buf; + + while (num_bytes != 0) + { + unsigned block_size = num_bytes; + if (block_size > MAX_IO_BLOCK_SIZE) + block_size = MAX_IO_BLOCK_SIZE; + + size_t r = fread(buf, 1, block_size, f); + if (r != block_size) + { + snprintf(error_message_buffer, sizeof(error_message_buffer), "read error: fread_fixed(block_size=%u) != %u", block_size, (unsigned)r); + throw error_message_buffer; + } + buf += block_size; + num_bytes -= block_size; + } } -void fwrite_fixed(FILE *f, const void * _buf, unsigned num_bytes) { - const char * buf = (const char *)_buf; - - while (num_bytes != 0) - { - unsigned block_size = num_bytes; - if (block_size > MAX_IO_BLOCK_SIZE) block_size = MAX_IO_BLOCK_SIZE; - - size_t r = fwrite(buf, 1, block_size, f); - if (r != block_size) - { - static char write_error_message[128]; - sprintf (write_error_message, "write error: fwrite_fixed(num_bytes=%u) != %u", block_size, (unsigned)r); - throw write_error_message; - } - buf += block_size; - num_bytes -= block_size; - } +void fwrite_fixed(FILE *f, const void * _buf, unsigned num_bytes) +{ + const char * buf = (const char *)_buf; + + while (num_bytes != 0) + { + unsigned block_size = num_bytes; + if (block_size > MAX_IO_BLOCK_SIZE) block_size = MAX_IO_BLOCK_SIZE; + + size_t r = fwrite(buf, 1, block_size, f); + if (r != block_size) + { + snprintf(error_message_buffer, sizeof(error_message_buffer), "write error: fwrite_fixed(num_bytes=%u) != %u", block_size, (unsigned)r); + throw error_message_buffer; + } + buf += block_size; + num_bytes -= block_size; + } } -unsigned read_word(FILE *f) { +#ifdef BIG_ENDIAN + +unsigned read_word(FILE *f) +{ unsigned char b, b2; fread_fixed(f, &b, 1); fread_fixed(f, &b2, 1); return (b2 << 8) + b; } -unsigned read_dword(FILE *f) { +unsigned read_dword(FILE *f) +{ unsigned low = read_word(f); return (read_word(f) << 16) + low; } -void write_word(FILE *f, unsigned number) { +void write_word(FILE *f, unsigned number) +{ unsigned char b = number & 255, b2 = number >> 8; fwrite_fixed(f, &b, 1); fwrite_fixed(f, &b2, 1); } -void write_dword(FILE *f, unsigned number) { +void write_dword(FILE *f, unsigned number) +{ write_word(f, number & 65535); write_word(f, number >> 16); } -bool fileExists(char *fname) { +#else + +template ::value>> +inline T read_type(FILE *f) +{ + T result = 0; + fread_fixed(f, &result, sizeof(result)); + return result; +} + +#define read_byte(f) read_type((f)) +#define read_word(f) read_type((f)) +#define read_dword(f) read_type((f)) + + +inline void write_word(FILE *f, uint16_t number) +{ + fwrite_fixed(f, &number, sizeof(number)); +} + +template ::value>> +inline void write_type(FILE *f, T number) +{ + fwrite_fixed(f, &number, sizeof(number)); +} + +#define write_byte(f, v) write_type((f), (v)) +#define write_word(f, v) write_type((f), (v)) +#define write_dword(f, v) write_type((f), (v)) + +#endif // BIG_ENDIAN + +#ifdef USE_CXX17 + +template +inline bool fileExists(const T& fname) +{ + std::error_code ec; + return std::filesystem::exists(fname, ec); +} + +template +inline uint64_t getLenOfFile(const const T& fname) +{ + std::error_code ec; + return std::filesystem::file_size(fname, ec); +} + +#else + +bool fileExists(const char * fname) +{ FILE *f = fopen(fname, "rb"); - bool exists = (f != NULL); - if (exists) fclose(f); + bool exists = (f != nullptr); + if (exists) + fclose(f); return exists; } -unsigned getLenOfFile(char *fname) { +unsigned getLenOfFile(const char * fname) +{ FILE *f = fopen(fname, "rb"); fseek(f, 0, SEEK_END); unsigned len = ftell(f); fclose(f); return len; } + +#endif // USE_CXX17 From 0273fad2bcc8e8011bcf6b1c7de99c5f74122295 Mon Sep 17 00:00:00 2001 From: Deamhan Date: Thu, 14 Mar 2019 19:26:37 +0300 Subject: [PATCH 04/54] compatibility.h: custom typedefs have been removed, inttypes.h has been added --- src/compatibility.h | 14 +++++--------- 1 file changed, 5 insertions(+), 9 deletions(-) diff --git a/src/compatibility.h b/src/compatibility.h index 979cebb..5dcdfd0 100755 --- a/src/compatibility.h +++ b/src/compatibility.h @@ -3,15 +3,11 @@ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ // Fix for MSVC++, which doesn't support Variable Length Arrays. -#ifdef _MSC_VER - #include - #define STACK_ALLOC(name, type, num) type *name = (type *)alloca(sizeof(type) * num) +#include +#include - typedef unsigned __int8 uint8_t; - typedef unsigned __int16 uint16_t; - typedef unsigned __int32 uint32_t; - typedef unsigned __int64 uint64_t; +#ifdef _MSC_VER + #define STACK_ALLOC(name, type, num) type *name = (type *)alloca(sizeof(type) * num) #else - #include - #define STACK_ALLOC(name, type, num) type name[num] + #define STACK_ALLOC(name, type, num) type name[num] #endif From 62cb8e4b2043470098653a41a09720f326afecbc Mon Sep 17 00:00:00 2001 From: Deamhan Date: Thu, 14 Mar 2019 19:29:01 +0300 Subject: [PATCH 05/54] file.h: \t -> 4 spaces --- src/file.h | 42 +++++++++++++++++++++--------------------- 1 file changed, 21 insertions(+), 21 deletions(-) diff --git a/src/file.h b/src/file.h index 8d1f0de..ad21884 100644 --- a/src/file.h +++ b/src/file.h @@ -58,30 +58,30 @@ void fwrite_fixed(FILE *f, const void * _buf, unsigned num_bytes) unsigned read_word(FILE *f) { - unsigned char b, b2; - fread_fixed(f, &b, 1); - fread_fixed(f, &b2, 1); - return (b2 << 8) + b; + unsigned char b, b2; + fread_fixed(f, &b, 1); + fread_fixed(f, &b2, 1); + return (b2 << 8) + b; } unsigned read_dword(FILE *f) { - unsigned low = read_word(f); - return (read_word(f) << 16) + low; + unsigned low = read_word(f); + return (read_word(f) << 16) + low; } void write_word(FILE *f, unsigned number) { - unsigned char b = number & 255, - b2 = number >> 8; - fwrite_fixed(f, &b, 1); - fwrite_fixed(f, &b2, 1); + unsigned char b = number & 255, + b2 = number >> 8; + fwrite_fixed(f, &b, 1); + fwrite_fixed(f, &b2, 1); } void write_dword(FILE *f, unsigned number) { - write_word(f, number & 65535); - write_word(f, number >> 16); + write_word(f, number & 65535); + write_word(f, number >> 16); } #else @@ -136,20 +136,20 @@ inline uint64_t getLenOfFile(const const T& fname) bool fileExists(const char * fname) { - FILE *f = fopen(fname, "rb"); - bool exists = (f != nullptr); - if (exists) + FILE *f = fopen(fname, "rb"); + bool exists = (f != nullptr); + if (exists) fclose(f); - return exists; + return exists; } unsigned getLenOfFile(const char * fname) { - FILE *f = fopen(fname, "rb"); - fseek(f, 0, SEEK_END); - unsigned len = ftell(f); - fclose(f); - return len; + FILE *f = fopen(fname, "rb"); + fseek(f, 0, SEEK_END); + unsigned len = ftell(f); + fclose(f); + return len; } #endif // USE_CXX17 From 7a85464a4b29d6e4035fddd7b75ebe2334500f22 Mon Sep 17 00:00:00 2001 From: Deamhan Date: Thu, 14 Mar 2019 19:53:52 +0300 Subject: [PATCH 06/54] libbdelta.cpp: exception safe resource deallocation & cleanup --- src/libbdelta.cpp | 841 +++++++++++++++++++++++++--------------------- 1 file changed, 463 insertions(+), 378 deletions(-) diff --git a/src/libbdelta.cpp b/src/libbdelta.cpp index 636013a..653981d 100644 --- a/src/libbdelta.cpp +++ b/src/libbdelta.cpp @@ -2,6 +2,12 @@ * License, v. 2.0. If a copy of the MPL was not distributed with this * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ +#include +#include +#include +#include +#include + #include "compatibility.h" #if !defined(TOKEN_SIZE) || TOKEN_SIZE == 1 @@ -12,449 +18,528 @@ typedef uint16_t Token; typedef uint32_t Token; #endif -#include #include "bdelta.h" #include "checksum.h" -#include -#include -#include + + const bool verbose = false; struct checksum_entry { - Hash::Value cksum; //Rolling checksums - unsigned loc; - checksum_entry() {} - checksum_entry(Hash::Value cksum, unsigned loc) - {this->cksum = cksum; this->loc = loc;} + Hash::Value cksum; //Rolling checksums + unsigned loc; + checksum_entry() {} + checksum_entry(Hash::Value _cksum, unsigned _loc) : cksum(_cksum), loc(_loc) {} }; struct Range { - unsigned p, num; - Range() {} - Range(unsigned p, unsigned num) {this->p = p; this->num = num;} + unsigned p, num; + Range() {} + Range(unsigned _p, unsigned _num) : p(_p), num(_num) {} }; -struct Match { - unsigned p1, p2, num; - Match(unsigned p1, unsigned p2, unsigned num) - {this->p1 = p1; this->p2 = p2; this->num = num;} +struct Match +{ + unsigned p1, p2, num; + Match(unsigned _p1, unsigned _p2, unsigned _num) : p1(_p1), p2(_p2), num(_num) {} }; -struct _BDelta_Instance { - bdelta_readCallback cb; - void *handle1, *handle2; - unsigned data1_size, data2_size; - std::list matches; - std::list::iterator accessplace; - int access_int; - int errorcode; - - const Token *read1(void *buf, unsigned place, unsigned num) - {return (const Token*)cb(handle1, buf, place, num);} - const Token *read2(void *buf, unsigned place, unsigned num) - {return (const Token*)cb(handle2, buf, place, num);} +struct _BDelta_Instance +{ + bdelta_readCallback cb; + void *handle1, *handle2; + unsigned data1_size, data2_size; + std::list matches; + std::list::iterator accessplace; + int access_int; + int errorcode; + + const Token *read1(void *buf, unsigned place, unsigned num) + { + return (const Token*)cb(handle1, buf, place, num); + } + const Token *read2(void *buf, unsigned place, unsigned num) + { + return (const Token*)cb(handle2, buf, place, num); + } }; -struct Checksums_Instance { - unsigned blocksize; - unsigned htablesize; - checksum_entry **htable; // Points to first match in checksums - checksum_entry *checksums; // Sorted list of all checksums - unsigned numchecksums; - - Checksums_Instance(int blocksize) {this->blocksize = blocksize;} - void add(checksum_entry ck) { - checksums[numchecksums] = ck; - ++numchecksums; - } - unsigned tableIndex(Hash::Value hashValue) { - return Hash::modulo(hashValue, htablesize); - } +struct Checksums_Instance +{ + unsigned blocksize; + unsigned htablesize; + checksum_entry **htable; // Points to first match in checksums + checksum_entry *checksums; // Sorted list of all checksums + unsigned numchecksums; + + Checksums_Instance(int blocksize) { this->blocksize = blocksize; } + void add(checksum_entry ck) + { + checksums[numchecksums] = ck; + ++numchecksums; + } + unsigned tableIndex(Hash::Value hashValue) + { + return Hash::modulo(hashValue, htablesize); + } }; -unsigned match_buf_forward(const void *buf1, const void *buf2, unsigned num) { - unsigned i = 0; - while (i < num && ((const Token*)buf1)[i]==((const Token*)buf2)[i]) ++i; - return i; +unsigned match_buf_forward(const void *buf1, const void *buf2, unsigned num) +{ + unsigned i = 0; + while (i < num && ((const Token*)buf1)[i] == ((const Token*)buf2)[i]) + ++i; + return i; } -unsigned match_buf_backward(const void *buf1, const void *buf2, unsigned num) { - int i = num; - do --i; - while (i >= 0 && ((const Token*)buf1)[i] == ((const Token*)buf2)[i]); - return num - i - 1; +unsigned match_buf_backward(const void *buf1, const void *buf2, unsigned num) +{ + int i = num; + do + { + --i; + } + while (i >= 0 && ((const Token*)buf1)[i] == ((const Token*)buf2)[i]); + return (num - i - 1); } -unsigned match_forward(BDelta_Instance *b, unsigned p1, unsigned p2) { - unsigned num = 0, match, numtoread; - do { - numtoread = std::min(b->data1_size - p1, b->data2_size - p2); - if (numtoread > 4096) numtoread = 4096; - Token buf1[4096], buf2[4096]; - const Token *read1 = b->read1(buf1, p1, numtoread), - *read2 = b->read2(buf2, p2, numtoread); - p1 += numtoread; p2 += numtoread; - match = match_buf_forward(read1, read2, numtoread); - num += match; - } while (match && match == numtoread); - return num; +unsigned match_forward(BDelta_Instance *b, unsigned p1, unsigned p2) +{ + unsigned num = 0, match, numtoread; + do + { + numtoread = std::min(b->data1_size - p1, b->data2_size - p2); + if (numtoread > 4096) numtoread = 4096; + Token buf1[4096], buf2[4096]; + const Token *read1 = b->read1(buf1, p1, numtoread), + *read2 = b->read2(buf2, p2, numtoread); + p1 += numtoread; p2 += numtoread; + match = match_buf_forward(read1, read2, numtoread); + num += match; + } while (match && match == numtoread); + return num; } -unsigned match_backward(BDelta_Instance *b, unsigned p1, unsigned p2, unsigned blocksize) { - unsigned num = 0, match, numtoread; - do { - numtoread = std::min(p1, p2); - if (numtoread > blocksize) numtoread = blocksize; - if (numtoread > 4096) numtoread = 4096; - p1 -= numtoread; p2 -= numtoread; - Token buf1[4096], buf2[4096]; - const Token *read1 = b->read1(buf1, p1, numtoread), - *read2 = b->read2(buf2, p2, numtoread); - match = match_buf_backward(read1, read2, numtoread); - num += match; - } while (match && match == numtoread); - return num; +unsigned match_backward(BDelta_Instance *b, unsigned p1, unsigned p2, unsigned blocksize) +{ + unsigned num = 0, match, numtoread; + do + { + numtoread = std::min(p1, p2); + if (numtoread > blocksize) + numtoread = blocksize; + if (numtoread > 4096) + numtoread = 4096; + p1 -= numtoread; + p2 -= numtoread; + Token buf1[4096], buf2[4096]; + const Token *read1 = b->read1(buf1, p1, numtoread), + *read2 = b->read2(buf2, p2, numtoread); + match = match_buf_backward(read1, read2, numtoread); + num += match; + } while (match && match == numtoread); + return num; } // Iterator helper function template static inline T bdelta_next(T i) {return ++i;} -struct UnusedRange { - unsigned p, num; - std::list::iterator ml, mr; - UnusedRange() {} - UnusedRange(unsigned p, unsigned num, std::list::iterator ml, std::list::iterator mr) { - this->p = p; this->num = num; this->ml = ml; this->mr = mr; - } +struct UnusedRange +{ + unsigned p, num; + std::list::iterator ml, mr; + UnusedRange() {} + UnusedRange(unsigned _p, unsigned _num, std::list::iterator _ml, std::list::iterator _mr) + : p(_p), num(_num), ml(_ml), mr(_mr) + {} }; // Sort first by location, second by match length (larger matches first) -bool comparep(UnusedRange r1, UnusedRange r2) { - if (r1.p != r2.p) - return r1.p < r2.p; - return r1.num > r2.num; +bool comparep(UnusedRange r1, UnusedRange r2) +{ + if (r1.p != r2.p) + return r1.p < r2.p; + return r1.num > r2.num; } -bool comparemrp2(UnusedRange r1, UnusedRange r2) { - if (r1.mr->p2 != r2.mr->p2) - return r1.mr->p2 < r2.mr->p2; - return r1.mr->num > r2.mr->num; +bool comparemrp2(UnusedRange r1, UnusedRange r2) +{ + if (r1.mr->p2 != r2.mr->p2) + return r1.mr->p2 < r2.mr->p2; + return r1.mr->num > r2.mr->num; } -bool compareMatchP2(Match r1, Match r2) { - if (r1.p2 != r2.p2) - return r1.p2 < r2.p2; - return r1.num > r2.num; +bool compareMatchP2(Match r1, Match r2) +{ + if (r1.p2 != r2.p2) + return r1.p2 < r2.p2; + return r1.num > r2.num; } -void addMatch(BDelta_Instance *b, unsigned p1, unsigned p2, unsigned num, std::list::iterator place) { - Match newMatch = Match(p1, p2, num); - while (place != b->matches.begin() && ! compareMatchP2(*place, newMatch)) - --place; - while (place != b->matches.end() && compareMatchP2(*place, newMatch)) - ++place; - b->matches.insert(place, newMatch); +void addMatch(BDelta_Instance *b, unsigned p1, unsigned p2, unsigned num, std::list::iterator place) +{ + Match newMatch = Match(p1, p2, num); + while (place != b->matches.begin() && ! compareMatchP2(*place, newMatch)) + --place; + while (place != b->matches.end() && compareMatchP2(*place, newMatch)) + ++place; + b->matches.insert(place, newMatch); } template -T absoluteDifference(T a, T b) { - return std::max(a, b) - std::min(a, b); +T absoluteDifference(T a, T b) +{ + return std::max(a, b) - std::min(a, b); } -void findMatches(BDelta_Instance *b, Checksums_Instance *h, unsigned minMatchSize, unsigned start, unsigned end, unsigned place, std::list::iterator iterPlace) { - const unsigned blocksize = h->blocksize; - STACK_ALLOC(buf1, Token, blocksize); - STACK_ALLOC(buf2, Token, blocksize); - - unsigned best1, best2, bestnum = 0; - unsigned processMatchesPos; - const Token *inbuf = b->read2(buf1, start, blocksize), - *outbuf; - Hash hash = Hash(inbuf, blocksize); - unsigned buf_loc = blocksize; - for (unsigned j = start + blocksize; ; ++j) { - unsigned thisTableIndex = h->tableIndex(hash.getValue()); - checksum_entry *c = h->htable[thisTableIndex]; - if (c) { - do { - if (c->cksum == hash.getValue()) { - unsigned p1 = c->loc, p2 = j - blocksize; - unsigned fnum = match_forward(b, p1, p2); - if (fnum >= blocksize) { - unsigned bnum = match_backward(b, p1, p2, blocksize); - unsigned num = fnum + bnum; - if (num >= minMatchSize) { - p1 -= bnum; p2 -= bnum; - bool foundBetter; - if (bestnum) { - double oldValue = double(bestnum) / (absoluteDifference(place, best1) + blocksize * 2), - newValue = double(num) / (absoluteDifference(place, p1) + blocksize * 2); - foundBetter = newValue > oldValue; - } else { - foundBetter = true; - processMatchesPos = std::min(j + blocksize - 1, end); - } - if (foundBetter) { - best1 = p1; - best2 = p2; - bestnum = num; - } - - } - } - } - ++c; - } while (h->tableIndex(c->cksum) == thisTableIndex); - } - - if (bestnum && j >= processMatchesPos) { - addMatch(b, best1, best2, bestnum, iterPlace); - place = best1 + bestnum; - unsigned matchEnd = best2 + bestnum; - if (matchEnd > j) { - if (matchEnd >= end) - j = end; - else { - // Fast forward over matched area. - j = matchEnd - blocksize; - inbuf = b->read2(buf1, j, blocksize); - hash = Hash(inbuf, blocksize); - buf_loc = blocksize; - j += blocksize; - } - } - bestnum = 0; - } - - if (buf_loc == blocksize) { - buf_loc = 0; - std::swap(inbuf, outbuf); - inbuf = b->read2(outbuf == buf1 ? buf2 : buf1, j, std::min(end - j, blocksize)); - } - - if (j >= end) - break; - - hash.advance(outbuf[buf_loc], inbuf[buf_loc]); - ++buf_loc; - } +void findMatches(BDelta_Instance *b, Checksums_Instance *h, unsigned minMatchSize, unsigned start, unsigned end, unsigned place, std::list::iterator iterPlace) +{ + const unsigned blocksize = h->blocksize; + STACK_ALLOC(buf1, Token, blocksize); + STACK_ALLOC(buf2, Token, blocksize); + + unsigned best1 = 0, best2 = 0, bestnum = 0; + unsigned processMatchesPos = 0; + const Token *inbuf = b->read2(buf1, start, blocksize), + *outbuf = nullptr; + Hash hash = Hash(inbuf, blocksize); + unsigned buf_loc = blocksize; + for (unsigned j = start + blocksize; ; ++j) + { + unsigned thisTableIndex = h->tableIndex(hash.getValue()); + checksum_entry *c = h->htable[thisTableIndex]; + if (c) + { + do + { + if (c->cksum == hash.getValue()) + { + unsigned p1 = c->loc, p2 = j - blocksize; + unsigned fnum = match_forward(b, p1, p2); + if (fnum >= blocksize) + { + unsigned bnum = match_backward(b, p1, p2, blocksize); + unsigned num = fnum + bnum; + if (num >= minMatchSize) + { + p1 -= bnum; p2 -= bnum; + bool foundBetter; + if (bestnum) + { + double oldValue = double(bestnum) / (absoluteDifference(place, best1) + blocksize * 2), + newValue = double(num) / (absoluteDifference(place, p1) + blocksize * 2); + foundBetter = newValue > oldValue; + } + else + { + foundBetter = true; + processMatchesPos = std::min(j + blocksize - 1, end); + } + if (foundBetter) + { + best1 = p1; + best2 = p2; + bestnum = num; + } + + } + } + } + ++c; + } while (h->tableIndex(c->cksum) == thisTableIndex); + } + + if (bestnum && j >= processMatchesPos) + { + addMatch(b, best1, best2, bestnum, iterPlace); + place = best1 + bestnum; + unsigned matchEnd = best2 + bestnum; + if (matchEnd > j) + { + if (matchEnd >= end) + j = end; + else + { + // Fast forward over matched area. + j = matchEnd - blocksize; + inbuf = b->read2(buf1, j, blocksize); + hash = Hash(inbuf, blocksize); + buf_loc = blocksize; + j += blocksize; + } + } + bestnum = 0; + } + + if (buf_loc == blocksize) + { + buf_loc = 0; + std::swap(inbuf, outbuf); + inbuf = b->read2(outbuf == buf1 ? buf2 : buf1, j, std::min(end - j, blocksize)); + } + + if (j >= end) + break; + + hash.advance(outbuf[buf_loc], inbuf[buf_loc]); + ++buf_loc; + } } -struct Checksums_Compare { - Checksums_Instance &ci; - Checksums_Compare(Checksums_Instance &h) : ci(h) {} - bool operator() (checksum_entry c1, checksum_entry c2) { - unsigned ti1 = ci.tableIndex(c1.cksum), ti2 = ci.tableIndex(c2.cksum); - if (ti1 == ti2) - if (c1.cksum == c2.cksum) - return c1.loc < c2.loc; - else - return c1.cksum < c2.cksum; - else - return ti1 < ti2; - } +struct Checksums_Compare +{ + Checksums_Instance &ci; + Checksums_Compare(Checksums_Instance &h) : ci(h) {} + bool operator() (checksum_entry c1, checksum_entry c2) + { + unsigned ti1 = ci.tableIndex(c1.cksum), ti2 = ci.tableIndex(c2.cksum); + if (ti1 == ti2) + { + if (c1.cksum == c2.cksum) + return c1.loc < c2.loc; + else + return c1.cksum < c2.cksum; + } + else + return ti1 < ti2; + } }; BDelta_Instance *bdelta_init_alg(unsigned data1_size, unsigned data2_size, - bdelta_readCallback cb, void *handle1, void *handle2, - unsigned tokenSize) { - if (tokenSize != sizeof(Token)) { - printf("Error: BDelta library compiled for token size of %lu.\n", (unsigned long)sizeof (Token)); - return 0; - } - BDelta_Instance *b = new BDelta_Instance; - if (!b) return 0; - b->data1_size = data1_size; - b->data2_size = data2_size; - b->cb = cb; - b->handle1 = handle1; - b->handle2 = handle2; - b->access_int = -1; - return b; + bdelta_readCallback cb, void *handle1, void *handle2, + unsigned tokenSize) +{ + if (tokenSize != sizeof(Token)) + { + printf("Error: BDelta library compiled for token size of %lu.\n", (unsigned long)sizeof (Token)); + return 0; + } + BDelta_Instance *b = new BDelta_Instance; + if (!b) + return nullptr; + b->data1_size = data1_size; + b->data2_size = data2_size; + b->cb = cb; + b->handle1 = handle1; + b->handle2 = handle2; + b->access_int = -1; + return b; } -void bdelta_done_alg(BDelta_Instance *b) { - b->matches.clear(); - delete b; +void bdelta_done_alg(BDelta_Instance *b) +{ + b->matches.clear(); + delete b; } // Adapted from http://graphics.stanford.edu/~seander/bithacks.html#RoundUpPowerOf2 -unsigned roundUpPowerOf2(unsigned v) { - --v; - for (int i = 1; i <= 16; i *= 2) - v |= v >> i; - return v + 1; +unsigned roundUpPowerOf2(unsigned v) +{ + --v; + for (int i = 1; i <= 16; i *= 2) + v |= (v >> i); + return (v + 1); } -void bdelta_pass_2(BDelta_Instance *b, unsigned blocksize, unsigned minMatchSize, UnusedRange *unused, unsigned numunused, UnusedRange *unused2, unsigned numunused2) { - Checksums_Instance h(blocksize); - b->access_int = -1; - - unsigned numblocks = 0; - for (unsigned i = 0; i < numunused; ++i) { - numblocks += unused[i].num / blocksize; - } - - // numblocks = size / blocksize; - h.htablesize = std::max((unsigned)2, roundUpPowerOf2(numblocks)); - h.htable = new checksum_entry*[h.htablesize]; - if (!h.htable) {b->errorcode = BDELTA_MEM_ERROR; return;} - h.checksums = new checksum_entry[numblocks + 2]; - if (!h.checksums) {b->errorcode = BDELTA_MEM_ERROR; return;} - - h.numchecksums = 0; - // unsigned numchecksums = 0; - STACK_ALLOC(buf, Token, blocksize); - for (unsigned i = 0; i < numunused; ++i) { - unsigned first = unused[i].p, last = unused[i].p + unused[i].num; - for (unsigned loc = first; loc + blocksize <= last; loc += blocksize) { - const Token *read = b->read1(buf, loc, blocksize); - Hash::Value blocksum = Hash(read, blocksize).getValue(); - // Adjacent checksums are never repeated. - //if (! h.numchecksums || blocksum != h.checksums[h.numchecksums - 1].cksum) - h.add(checksum_entry(blocksum, loc)); - } - } - - if (h.numchecksums) { - std::sort(h.checksums, h.checksums + h.numchecksums, Checksums_Compare(h)); - const unsigned maxIdenticalChecksums = 2; - unsigned writeLoc = 0, readLoc, testAhead; - for (readLoc = 0; readLoc < h.numchecksums; readLoc = testAhead) { - for (testAhead = readLoc; testAhead < h.numchecksums && h.checksums[readLoc].cksum == h.checksums[testAhead].cksum; ++testAhead) - ; - if (testAhead - readLoc <= maxIdenticalChecksums) - for (unsigned i = readLoc; i < testAhead; ++i) - h.checksums[writeLoc++] = h.checksums[i]; - } - h.numchecksums = writeLoc; - } - h.checksums[h.numchecksums].cksum = std::numeric_limits::max(); // If there's only one checksum, we might hit this and not know it, - h.checksums[h.numchecksums].loc = 0; // So we'll just read from the beginning of the file to prevent crashes. - h.checksums[h.numchecksums + 1].cksum = 0; - - for (unsigned i = 0; i < h.htablesize; ++i) h.htable[i] = 0; - for (int i = h.numchecksums - 1; i >= 0; --i) - h.htable[h.tableIndex(h.checksums[i].cksum)] = &h.checksums[i]; - - for (unsigned i = 0; i < numunused2; ++i) - if (unused2[i].num >= blocksize) - findMatches(b, &h, minMatchSize, unused2[i].p, unused2[i].p + unused2[i].num, unused[i].p, unused2[i].mr); - - delete [] h.htable; - delete [] h.checksums; +void bdelta_pass_2(BDelta_Instance *b, unsigned blocksize, unsigned minMatchSize, UnusedRange *unused, unsigned numunused, UnusedRange *unused2, unsigned numunused2) +{ + Checksums_Instance h(blocksize); + b->access_int = -1; + + unsigned numblocks = 0; + for (unsigned i = 0; i < numunused; ++i) + numblocks += unused[i].num / blocksize; + + h.htablesize = std::max((unsigned)2, roundUpPowerOf2(numblocks)); + h.htable = new checksum_entry*[h.htablesize]; + if (!h.htable) + { + b->errorcode = BDELTA_MEM_ERROR; + return; + } + std::unique_ptr htable_holder(h.htable); + + h.checksums = new checksum_entry[numblocks + 2]; + if (!h.checksums) + { + b->errorcode = BDELTA_MEM_ERROR; + return; + } + std::unique_ptr checksums_holder(h.checksums); + + h.numchecksums = 0; + STACK_ALLOC(buf, Token, blocksize); + for (unsigned i = 0; i < numunused; ++i) + { + unsigned first = unused[i].p, last = unused[i].p + unused[i].num; + for (unsigned loc = first; loc + blocksize <= last; loc += blocksize) + { + const Token *read = b->read1(buf, loc, blocksize); + Hash::Value blocksum = Hash(read, blocksize).getValue(); + // Adjacent checksums are never repeated. + h.add(checksum_entry(blocksum, loc)); + } + } + + if (h.numchecksums) + { + std::sort(h.checksums, h.checksums + h.numchecksums, Checksums_Compare(h)); + const unsigned maxIdenticalChecksums = 2; + unsigned writeLoc = 0, readLoc, testAhead; + for (readLoc = 0; readLoc < h.numchecksums; readLoc = testAhead) + { + for (testAhead = readLoc; testAhead < h.numchecksums && h.checksums[readLoc].cksum == h.checksums[testAhead].cksum; ++testAhead) + ; + if (testAhead - readLoc <= maxIdenticalChecksums) + for (unsigned i = readLoc; i < testAhead; ++i) + h.checksums[writeLoc++] = h.checksums[i]; + } + h.numchecksums = writeLoc; + } + h.checksums[h.numchecksums].cksum = std::numeric_limits::max(); // If there's only one checksum, we might hit this and not know it, + h.checksums[h.numchecksums].loc = 0; // So we'll just read from the beginning of the file to prevent crashes. + h.checksums[h.numchecksums + 1].cksum = 0; + + memset(h.htable, 0, h.htablesize * sizeof(h.htable[0])); + for (int i = h.numchecksums - 1; i >= 0; --i) + h.htable[h.tableIndex(h.checksums[i].cksum)] = &h.checksums[i]; + + for (unsigned i = 0; i < numunused2; ++i) + { + if (unused2[i].num >= blocksize) + findMatches(b, &h, minMatchSize, unused2[i].p, unused2[i].p + unused2[i].num, unused[i].p, unused2[i].mr); + } } -void bdelta_swap_inputs(BDelta_Instance *b) { - for (std::list::iterator l = b->matches.begin(); l != b->matches.end(); ++l) - std::swap(l->p1, l->p2); - std::swap(b->data1_size, b->data2_size); - std::swap(b->handle1, b->handle2); - b->matches.sort(compareMatchP2); +void bdelta_swap_inputs(BDelta_Instance *b) +{ + for (std::list::iterator l = b->matches.begin(); l != b->matches.end(); ++l) + std::swap(l->p1, l->p2); + std::swap(b->data1_size, b->data2_size); + std::swap(b->handle1, b->handle2); + b->matches.sort(compareMatchP2); } -void bdelta_clean_matches(BDelta_Instance *b, unsigned flags) { - std::list::iterator nextL = b->matches.begin(); - if (nextL == b->matches.end()) return; - while (true) { - std::list::iterator l = nextL; - if (++nextL == b->matches.end()) - break; - - int overlap = l->p2 + l->num - nextL->p2; - if (overlap >= 0) { - if (overlap >= nextL->num) { - b->matches.erase(nextL); - nextL = l; - continue; - } - if (flags & BDELTA_REMOVE_OVERLAP) - l->num -= overlap; - } - } +void bdelta_clean_matches(BDelta_Instance *b, unsigned flags) +{ + std::list::iterator nextL = b->matches.begin(); + if (nextL == b->matches.end()) + return; + while (true) + { + std::list::iterator l = nextL; + if (++nextL == b->matches.end()) + break; + + int overlap = l->p2 + l->num - nextL->p2; + if (overlap >= 0) + { + if ((unsigned)overlap >= nextL->num) + { + b->matches.erase(nextL); + nextL = l; + continue; + } + if (flags & BDELTA_REMOVE_OVERLAP) + l->num -= overlap; + } + } } -void bdelta_showMatches(BDelta_Instance *b) { - for (std::list::iterator l = b->matches.begin(); l != b->matches.end(); ++l) - printf("(%d, %d, %d), ", l->p1, l->p2, l->num); - printf ("\n\n"); +void bdelta_showMatches(BDelta_Instance *b) +{ + for (std::list::iterator l = b->matches.begin(); l != b->matches.end(); ++l) + printf("(%d, %d, %d), ", l->p1, l->p2, l->num); + printf ("\n\n"); } -void get_unused_blocks(UnusedRange *unused, unsigned *numunusedptr) { - unsigned nextStartPos = 0; - for (unsigned i = 1; i < *numunusedptr; ++i) { - unsigned startPos = nextStartPos; - nextStartPos = std::max(startPos, unused[i].p + unused[i].num); - unused[i] = UnusedRange(startPos, unused[i].p < startPos ? 0 : unused[i].p - startPos, unused[i-1].mr, unused[i].mr); - } +void get_unused_blocks(UnusedRange *unused, unsigned *numunusedptr) +{ + unsigned nextStartPos = 0; + for (unsigned i = 1; i < *numunusedptr; ++i) + { + unsigned startPos = nextStartPos; + nextStartPos = std::max(startPos, unused[i].p + unused[i].num); + unused[i] = UnusedRange(startPos, unused[i].p < startPos ? 0 : unused[i].p - startPos, unused[i-1].mr, unused[i].mr); + } } -bool isZeroMatch(Match &m) {return m.num == 0;} - -void bdelta_pass(BDelta_Instance *b, unsigned blocksize, unsigned minMatchSize, unsigned maxHoleSize, unsigned flags) { - // Place an empty Match at beginning so we can assume there's a Match to the left of every hole. - b->matches.push_front(Match(0, 0, 0)); - // Trick for including the free range at the end. - b->matches.push_back(Match(b->data1_size, b->data2_size, 0)); - - UnusedRange *unused = new UnusedRange[b->matches.size() + 1], - *unused2 = new UnusedRange[b->matches.size() + 1]; - unsigned numunused = 0, numunused2 = 0; - for (std::list::iterator l = b->matches.begin(); l != b->matches.end(); ++l) { - unused[numunused++] = UnusedRange(l->p1, l->num, l, l); - unused2[numunused2++] = UnusedRange(l->p2, l->num, l, l); - } - - std::sort(unused + 1, unused + numunused, comparep); // Leave empty match at beginning - //std::sort(unused2, unused2 + numunused2, comparep); - - get_unused_blocks(unused, &numunused); - get_unused_blocks(unused2, &numunused2); - //std::sort(unused2, unused2 + numunused2, comparemrp2); - - if (flags & BDELTA_GLOBAL) - bdelta_pass_2(b, blocksize, minMatchSize, unused, numunused, unused2, numunused2); - else { - std::sort(unused + 1, unused + numunused, comparemrp2); - for (unsigned i = 1; i < numunused; ++i) { - UnusedRange u1 = unused[i], u2 = unused2[i]; - if (u1.num >= blocksize && u2.num >= blocksize) - if (! maxHoleSize || (u1.num <= maxHoleSize && u2.num <= maxHoleSize)) - if (! (flags & BDELTA_SIDES_ORDERED) || (bdelta_next(u1.ml) == u1.mr && bdelta_next(u2.ml) == u2.mr)) - bdelta_pass_2(b, blocksize, minMatchSize, &u1, 1, &u2, 1); - } - } - - if (verbose) printf("pass (blocksize: %u, matches: %lu)\n", blocksize, (unsigned long)b->matches.size()); - - // Get rid of the dummy values we placed at the ends. - b->matches.erase(std::find_if(b->matches.begin(), b->matches.end(), isZeroMatch)); - b->matches.pop_back(); - - delete [] unused; - delete [] unused2; +inline bool isZeroMatch(Match &m) { return m.num == 0; } + +void bdelta_pass(BDelta_Instance *b, unsigned blocksize, unsigned minMatchSize, unsigned maxHoleSize, unsigned flags) +{ + // Place an empty Match at beginning so we can assume there's a Match to the left of every hole. + b->matches.push_front(Match(0, 0, 0)); + // Trick for including the free range at the end. + b->matches.push_back(Match(b->data1_size, b->data2_size, 0)); + + UnusedRange *unused = new UnusedRange[b->matches.size() + 1], + *unused2 = new UnusedRange[b->matches.size() + 1]; + + std::unique_ptr unused_holder(unused), unused2_holder(unused2); + + unsigned numunused = 0, numunused2 = 0; + for (std::list::iterator l = b->matches.begin(); l != b->matches.end(); ++l) + { + unused[numunused++] = UnusedRange(l->p1, l->num, l, l); + unused2[numunused2++] = UnusedRange(l->p2, l->num, l, l); + } + + std::sort(unused + 1, unused + numunused, comparep); // Leave empty match at beginning + + get_unused_blocks(unused, &numunused); + get_unused_blocks(unused2, &numunused2); + + if (flags & BDELTA_GLOBAL) + bdelta_pass_2(b, blocksize, minMatchSize, unused, numunused, unused2, numunused2); + else + { + std::sort(unused + 1, unused + numunused, comparemrp2); + for (unsigned i = 1; i < numunused; ++i) + { + UnusedRange u1 = unused[i], u2 = unused2[i]; + if (u1.num >= blocksize && u2.num >= blocksize) + if (! maxHoleSize || (u1.num <= maxHoleSize && u2.num <= maxHoleSize)) + if (! (flags & BDELTA_SIDES_ORDERED) || (bdelta_next(u1.ml) == u1.mr && bdelta_next(u2.ml) == u2.mr)) + bdelta_pass_2(b, blocksize, minMatchSize, &u1, 1, &u2, 1); + } + } + + if (verbose) + printf("pass (blocksize: %u, matches: %lu)\n", blocksize, (unsigned long)b->matches.size()); + + // Get rid of the dummy values we placed at the ends. + b->matches.erase(std::find_if(b->matches.begin(), b->matches.end(), isZeroMatch)); + b->matches.pop_back(); } -unsigned bdelta_numMatches(BDelta_Instance *b) { - return b->matches.size(); +unsigned bdelta_numMatches(BDelta_Instance *b) +{ + return b->matches.size(); } -void bdelta_getMatch(BDelta_Instance *b, unsigned matchNum, - unsigned *p1, unsigned *p2, unsigned *num) { - int &access_int = b->access_int; - std::list::iterator &accessplace = b->accessplace; - if (access_int == -1) {access_int = 0; accessplace = b->matches.begin();} - while ((unsigned)access_int < matchNum) { - ++accessplace; - ++access_int; - } - while ((unsigned)access_int > matchNum) { - --accessplace; - --access_int; - } - *p1 = accessplace->p1; - *p2 = accessplace->p2; - *num = accessplace->num; +void bdelta_getMatch(BDelta_Instance *b, unsigned matchNum, unsigned *p1, unsigned *p2, unsigned *num) +{ + int &access_int = b->access_int; + std::list::iterator &accessplace = b->accessplace; + if (access_int == -1) + { + access_int = 0; + accessplace = b->matches.begin(); + } + while ((unsigned)access_int < matchNum) + { + ++accessplace; + ++access_int; + } + while ((unsigned)access_int > matchNum) + { + --accessplace; + --access_int; + } + *p1 = accessplace->p1; + *p2 = accessplace->p2; + *num = accessplace->num; } -int bdelta_getError(BDelta_Instance *instance) { - return instance->errorcode; +inline int bdelta_getError(BDelta_Instance *instance) +{ + return instance->errorcode; } From 0accb4ffb97e35acaddddd48c16701c37a0c965d Mon Sep 17 00:00:00 2001 From: Deamhan Date: Thu, 14 Mar 2019 19:56:52 +0300 Subject: [PATCH 07/54] checksum.h: cleanup --- src/checksum.h | 81 ++++++++++++++++++++++++++------------------------ 1 file changed, 42 insertions(+), 39 deletions(-) diff --git a/src/checksum.h b/src/checksum.h index 40b9118..f12715c 100755 --- a/src/checksum.h +++ b/src/checksum.h @@ -2,47 +2,50 @@ * License, v. 2.0. If a copy of the MPL was not distributed with this * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ -struct Hash { -public: - typedef uint64_t Value; - Hash() {} - Hash(const Token *buf, unsigned blocksize) { - value = 0; - for (unsigned num = 0; num < blocksize; ++num) - advance_add(buf[num]); - oldCoefficient = powHash(multiplyAmount, blocksize); - } - void advance(Token out, Token in) { - advance_remove(out); - advance_add(in); - } - static unsigned modulo(Value hash, unsigned d) { - // Assumes d is power of 2. - return hash & (d - 1); - } - Value getValue() {return value >> extraProcBits;} +struct Hash +{ + typedef uint64_t Value; + Hash() {} + Hash(const Token * buf, unsigned blocksize) + { + value = 0; + for (unsigned num = 0; num < blocksize; ++num) + advance_add(buf[num]); + oldCoefficient = powHash(multiplyAmount, blocksize); + } + void advance(Token out, Token in) + { + advance_remove(out); + advance_add(in); + } + static unsigned modulo(Value hash, unsigned d) + { + // Assumes d is power of 2. + return (hash & (d - 1)); + } + Value getValue() { return value >> extraProcBits; } private: - typedef uint64_t ProcValue; - static const unsigned extraProcBits = (sizeof(ProcValue) - sizeof(Value)) * 8; + typedef uint64_t ProcValue; + static const unsigned extraProcBits = (sizeof(ProcValue) - sizeof(Value)) * 8; - static const ProcValue multiplyAmount = (1ll << extraProcBits) | 181; - ProcValue oldCoefficient, value; + static const ProcValue multiplyAmount = ((1ll << extraProcBits) | 181); + ProcValue oldCoefficient, value; - void advance_add(Token in) { - value += in; - value *= multiplyAmount; - } - void advance_remove(Token out) { - value -= out * oldCoefficient; - } - static ProcValue powHash(ProcValue x, unsigned y) { - ProcValue res = 1; - while (y) { - if (y & 1) res *= x; - x *= x; - y >>= 1; - } - return res; - } + void advance_add(Token in) + { + value += in; + value *= multiplyAmount; + } + void advance_remove(Token out) { value -= out * oldCoefficient; } + static ProcValue powHash(ProcValue x, unsigned y) + { + ProcValue res = 1; + while (y) { + if (y & 1) res *= x; + x *= x; + y >>= 1; + } + return res; + } }; From fecc3310914967f6f23899c6af9574ff93137383 Mon Sep 17 00:00:00 2001 From: Deamhan Date: Thu, 14 Mar 2019 19:58:29 +0300 Subject: [PATCH 08/54] bdelta.h: cleanup --- src/bdelta.h | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/src/bdelta.h b/src/bdelta.h index a9aee38..1934623 100644 --- a/src/bdelta.h +++ b/src/bdelta.h @@ -14,8 +14,8 @@ typedef struct _BDelta_Instance BDelta_Instance; typedef const void *(*bdelta_readCallback)(void *handle, void *buf, unsigned place, unsigned num); BDelta_Instance *bdelta_init_alg(unsigned data1_size, unsigned data2_size, - bdelta_readCallback cb, void *handle1, void *handle2, - unsigned tokenSize); + bdelta_readCallback cb, void *handle1, void *handle2, + unsigned tokenSize); void bdelta_done_alg(BDelta_Instance *b); void bdelta_pass(BDelta_Instance *b, unsigned blockSize, unsigned minMatchSize, unsigned maxHoleSize, unsigned flags); @@ -26,7 +26,7 @@ void bdelta_clean_matches(BDelta_Instance *b, unsigned flags); unsigned bdelta_numMatches(BDelta_Instance *b); void bdelta_getMatch(BDelta_Instance *b, unsigned matchNum, - unsigned *p1, unsigned *p2, unsigned *num); + unsigned *p1, unsigned *p2, unsigned *num); int bdelta_getError(BDelta_Instance *b); void bdelta_showMatches(BDelta_Instance *b); @@ -39,9 +39,9 @@ void bdelta_showMatches(BDelta_Instance *b); #define BDELTA_REMOVE_OVERLAP 1 enum BDELTA_RESULT { - BDELTA_OK = 0, - BDELTA_MEM_ERROR = -1, - BDELTA_READ_ERROR = -2 + BDELTA_OK = 0, + BDELTA_MEM_ERROR = -1, + BDELTA_READ_ERROR = -2 }; #ifdef __cplusplus From 4634f547356ed4bccda81e43f81aec25df008979 Mon Sep 17 00:00:00 2001 From: Deamhan Date: Thu, 14 Mar 2019 20:01:48 +0300 Subject: [PATCH 09/54] bdelta.cpp: exception safe resource deallocation has been added & cleanup --- src/bdelta.cpp | 343 ++++++++++++++++++++++++------------------------- 1 file changed, 170 insertions(+), 173 deletions(-) diff --git a/src/bdelta.cpp b/src/bdelta.cpp index 6f68846..02193bd 100644 --- a/src/bdelta.cpp +++ b/src/bdelta.cpp @@ -2,6 +2,7 @@ * License, v. 2.0. If a copy of the MPL was not distributed with this * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ +#include #include #include #include @@ -10,183 +11,179 @@ #include "file.h" #include "compatibility.h" -const void *f_read(void *f, void *buf, unsigned place, unsigned num) { - fseek((FILE *)f, place, SEEK_SET); - fread_fixed((FILE *)f, buf, num); - return buf; +const void *f_read(void *f, void *buf, unsigned place, unsigned num) +{ + fseek((FILE *)f, place, SEEK_SET); + fread_fixed((FILE *)f, buf, num); + return buf; } -const void *m_read(void *f, void * buf, unsigned place, unsigned num) { - if (0) { - /* - * BDelta uses only returned pointer - * and does not modify it's contents. - * - * But bugs happen. - */ - memcpy (buf, (char*)f + place, num); - return buf; - } - return (const char*)f + place; +inline const void * m_read(void *f, void * /*buf*/, unsigned place, unsigned /*num*/) +{ + return (const char*)f + place; } -void my_pass(BDelta_Instance *b, unsigned blocksize, unsigned minMatchSize, unsigned flags) { - bdelta_pass(b, blocksize, minMatchSize, 0, flags); - bdelta_clean_matches(b, BDELTA_REMOVE_OVERLAP); +void my_pass(BDelta_Instance *b, unsigned blocksize, unsigned minMatchSize, unsigned flags) +{ + bdelta_pass(b, blocksize, minMatchSize, 0, flags); + bdelta_clean_matches(b, BDELTA_REMOVE_OVERLAP); } -int main(int argc, char **argv) { - try { - bool all_ram_mode = false; - char * m1 = NULL; - char * m2 = NULL; - - if (argc > 1 && strcmp(argv[1], "--all-in-ram") == 0) - { - all_ram_mode = true; - --argc; - ++argv; - } - if (argc != 4) { - printf("usage: bdelta [--all-in-ram] \n"); - printf("needs two files to compare + output file:\n"); - exit(1); - } - if (!fileExists(argv[1]) || !fileExists(argv[2])) { - printf("one of the input files does not exist\n"); - exit(1); - } - unsigned size = getLenOfFile(argv[1]); - unsigned size2 = getLenOfFile(argv[2]); - FILE *f1 = fopen(argv[1], "rb"), - *f2 = fopen(argv[2], "rb"); - - BDelta_Instance *b; - - if (all_ram_mode) - { - m1 = new char[size]; - m2 = new char[size2]; - fread_fixed(f1, m1, size); - fread_fixed(f2, m2, size2); - - b = bdelta_init_alg(size, size2, m_read, m1, m2, 1); - } - else - { - b = bdelta_init_alg(size, size2, f_read, f1, f2, 1); - } - int nummatches; - - // List of primes for reference. Taken from Wikipedia. - // 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 - // 1-20 2 3 5 7 11 13 17 19 23 29 31 37 41 43 47 53 59 61 67 71 - // 21-40 73 79 83 89 97 101 103 107 109 113 127 131 137 139 149 151 157 163 167 173 - // 41-60 179 181 191 193 197 199 211 223 227 229 233 239 241 251 257 263 269 271 277 281 - // 61-80 283 293 307 311 313 317 331 337 347 349 353 359 367 373 379 383 389 397 401 409 - // 81-100 419 421 431 433 439 443 449 457 461 463 467 479 487 491 499 503 509 521 523 541 - // 101-120 547 557 563 569 571 577 587 593 599 601 607 613 617 619 631 641 643 647 653 659 - // 121-140 661 673 677 683 691 701 709 719 727 733 739 743 751 757 761 769 773 787 797 809 - // 141-160 811 821 823 827 829 839 853 857 859 863 877 881 883 887 907 911 919 929 937 941 - // 161-180 947 953 967 971 977 983 991 997 - - my_pass(b, 997, 1994, 0); - my_pass(b, 503, 1006, 0); - my_pass(b, 127, 254, 0); - my_pass(b, 31, 62, 0); - my_pass(b, 7, 14, 0); - my_pass(b, 5, 10, 0); - my_pass(b, 3, 6, 0); - my_pass(b, 13, 26, BDELTA_GLOBAL); - my_pass(b, 7, 14, 0); - my_pass(b, 5, 10, 0); - - nummatches = bdelta_numMatches(b); - - unsigned * copyloc1 = new unsigned[nummatches + 1]; - unsigned * copyloc2 = new unsigned[nummatches + 1]; - unsigned * copynum = new unsigned[nummatches + 1]; - - FILE *fout = fopen(argv[3], "wb"); - if (!fout) { - printf("couldn't open output file\n"); - exit(1); - } - - const char *magic = "BDT"; - fwrite_fixed(fout, magic, 3); - unsigned short version = 1; - write_word(fout, version); - unsigned char intsize = 4; - fwrite_fixed(fout, &intsize, 1); - write_dword(fout, size); - write_dword(fout, size2); - write_dword(fout, nummatches); - - unsigned lastp1 = 0, - lastp2 = 0; - for (int i = 0; i < nummatches; ++i) { - unsigned p1, p2, num; - bdelta_getMatch(b, i, &p1, &p2, &num); - // printf("%*x, %*x, %*x, %*x\n", 10, p1, 10, p2, 10, num, 10, p2-lastp2); - copyloc1[i] = p1 - lastp1; - write_dword(fout, copyloc1[i]); - copyloc2[i] = p2 - lastp2; - write_dword(fout, copyloc2[i]); - copynum[i] = num; - write_dword(fout, copynum[i]); - lastp1 = p1 + num; - lastp2 = p2 + num; - } - if (size2 != lastp2) { - copyloc1[nummatches] = 0; copynum[nummatches] = 0; - copyloc2[nummatches] = size2 - lastp2; - ++nummatches; - } - -// write_unsigned_list(adds, nummatches+1, fout); -// write_unsigned_list(copynum, nummatches, fout); -// write_signed_list(copyloc, nummatches, fout); - -// fwrite(copyloc1, 4, nummatches, fout); -// fwrite(copyloc2, 4, nummatches, fout); -// fwrite(copynum, 4, nummatches, fout); - unsigned fp = 0; - for (int i = 0; i < nummatches; ++i) { - unsigned num = copyloc2[i]; - while (num > 0) { - unsigned towrite = (num > 4096) ? 4096 : num; - unsigned char buf[4096]; - const void * b; - if (all_ram_mode) - b = m_read(m2, buf, fp, towrite); - else - b = f_read(f2, buf, fp, towrite); - fwrite_fixed(fout, b, towrite); - num -= towrite; - fp += towrite; - } - // fp+=copyloc2[i]; - if (i != nummatches) fp += copynum[i]; - } - - fclose(fout); - - bdelta_done_alg(b); - - delete [] m1; - delete [] m2; - - fclose(f1); - fclose(f2); - - delete [] copynum; - delete [] copyloc2; - delete [] copyloc1; - - } catch (const char * desc){ - fprintf (stderr, "FATAL: %s\n", desc); - return -1; - } - return 0; +int main(int argc, char **argv) +{ + try + { + bool all_ram_mode = false; + std::unique_ptr m1; + std::unique_ptr m2; + + if (argc > 1 && strcmp(argv[1], "--all-in-ram") == 0) + { + all_ram_mode = true; + --argc; + ++argv; + } + if (argc != 4) + { + printf("usage: bdelta [--all-in-ram] \n"); + printf("needs two files to compare + output file:\n"); + exit(1); + } + if (!fileExists(argv[1]) || !fileExists(argv[2])) + { + printf("one of the input files does not exist\n"); + exit(1); + } + unsigned size = (unsigned)getLenOfFile(argv[1]); + unsigned size2 = (unsigned)getLenOfFile(argv[2]); + FILE * f1 = fopen(argv[1], "rb"); + if (f1 == nullptr) + { + printf("unable to open file %s\n", argv[1]); + exit(1); + } + std::unique_ptr f1_holder(f1, fclose); + + FILE * f2 = fopen(argv[2], "rb"); + if (f2 == nullptr) + { + printf("unable to open file %s\n", argv[2]); + exit(1); + } + std::unique_ptr f2_holder(f2, fclose); + + BDelta_Instance * b = nullptr; + + if (all_ram_mode) + { + m1.reset(new char[size]); + m2.reset(new char[size2]); + fread_fixed(f1, m1.get(), size); + fread_fixed(f2, m2.get(), size2); + + b = bdelta_init_alg(size, size2, m_read, m1.get(), m2.get(), 1); + } + else + b = bdelta_init_alg(size, size2, f_read, f1, f2, 1); + + std::unique_ptr b_holder(b, bdelta_done_alg); + + int nummatches = 0; + + // List of primes for reference. Taken from Wikipedia. + // 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 + // 1-20 2 3 5 7 11 13 17 19 23 29 31 37 41 43 47 53 59 61 67 71 + // 21-40 73 79 83 89 97 101 103 107 109 113 127 131 137 139 149 151 157 163 167 173 + // 41-60 179 181 191 193 197 199 211 223 227 229 233 239 241 251 257 263 269 271 277 281 + // 61-80 283 293 307 311 313 317 331 337 347 349 353 359 367 373 379 383 389 397 401 409 + // 81-100 419 421 431 433 439 443 449 457 461 463 467 479 487 491 499 503 509 521 523 541 + // 101-120 547 557 563 569 571 577 587 593 599 601 607 613 617 619 631 641 643 647 653 659 + // 121-140 661 673 677 683 691 701 709 719 727 733 739 743 751 757 761 769 773 787 797 809 + // 141-160 811 821 823 827 829 839 853 857 859 863 877 881 883 887 907 911 919 929 937 941 + // 161-180 947 953 967 971 977 983 991 997 + + my_pass(b, 997, 1994, 0); + my_pass(b, 503, 1006, 0); + my_pass(b, 127, 254, 0); + my_pass(b, 31, 62, 0); + my_pass(b, 7, 14, 0); + my_pass(b, 5, 10, 0); + my_pass(b, 3, 6, 0); + my_pass(b, 13, 26, BDELTA_GLOBAL); + my_pass(b, 7, 14, 0); + my_pass(b, 5, 10, 0); + + nummatches = bdelta_numMatches(b); + + unsigned * copyloc1 = new unsigned[nummatches + 1]; + unsigned * copyloc2 = new unsigned[nummatches + 1]; + unsigned * copynum = new unsigned[nummatches + 1]; + std::unique_ptr copyloc1_holder(copyloc1), copyloc2_holder(copyloc2), copynum_holder(copynum); + + FILE *fout = fopen(argv[3], "wb"); + if (fout == nullptr) + { + printf("couldn't open output file\n"); + exit(1); + } + std::unique_ptr fout_holder(fout, fclose); + + const char * magic = "BDT"; + fwrite_fixed(fout, magic, 3); + unsigned short version = 1; + write_word(fout, version); + unsigned char intsize = 4; + fwrite_fixed(fout, &intsize, 1); + write_dword(fout, size); + write_dword(fout, size2); + write_dword(fout, nummatches); + + unsigned lastp1 = 0, lastp2 = 0; + for (int i = 0; i < nummatches; ++i) + { + unsigned p1 = 0, p2 = 0, num = 0; + bdelta_getMatch(b, i, &p1, &p2, &num); + copyloc1[i] = p1 - lastp1; + write_dword(fout, copyloc1[i]); + copyloc2[i] = p2 - lastp2; + write_dword(fout, copyloc2[i]); + copynum[i] = num; + write_dword(fout, copynum[i]); + lastp1 = p1 + num; + lastp2 = p2 + num; + } + if (size2 != lastp2) + { + copyloc1[nummatches] = 0; + copynum[nummatches] = 0; + copyloc2[nummatches] = size2 - lastp2; + ++nummatches; + } + + unsigned fp = 0; + for (int i = 0; i < nummatches; ++i) + { + unsigned num = copyloc2[i]; + while (num > 0) + { + unsigned towrite = (num > 4096) ? 4096 : num; + unsigned char buf[4096]; + const void * block = all_ram_mode ? m_read(m2.get(), buf, fp, towrite) : f_read(f2, buf, fp, towrite); + fwrite_fixed(fout, block, towrite); + num -= towrite; + fp += towrite; + } + + if (i != nummatches) + fp += copynum[i]; + } + + } + catch (const char * desc) + { + fprintf (stderr, "FATAL: %s\n", desc); + return -1; + } + + return 0; } From f427b104d41e29858c0bde43926b637fbf43ddb2 Mon Sep 17 00:00:00 2001 From: Deamhan Date: Thu, 14 Mar 2019 20:05:47 +0300 Subject: [PATCH 10/54] bpatch.cpp: exception safe resource deallocation & cleanup --- src/bpatch.cpp | 185 +++++++++++++++++++++++++++---------------------- 1 file changed, 104 insertions(+), 81 deletions(-) diff --git a/src/bpatch.cpp b/src/bpatch.cpp index 8817ce6..f5c35f3 100644 --- a/src/bpatch.cpp +++ b/src/bpatch.cpp @@ -7,98 +7,121 @@ #include "file.h" #include "compatibility.h" -bool copy_bytes_to_file(FILE *infile, FILE *outfile, unsigned numleft) { - size_t numread; - do { - char buf[1024]; - numread = fread(buf, 1, numleft > 1024 ? 1024 : numleft, infile); - if (fwrite(buf, 1, numread, outfile) != numread) { - printf("Could not write temporary data. Possibly out of space\n"); - return false; - } - numleft -= numread; - } while (numleft && !(numread < 1024 && numleft)); - return (numleft == 0); +bool copy_bytes_to_file(FILE *infile, FILE *outfile, unsigned numleft) +{ + size_t numread; + do { + char buf[1024]; + numread = fread(buf, 1, numleft > 1024 ? 1024 : numleft, infile); + if (fwrite(buf, 1, numread, outfile) != numread) { + printf("Could not write temporary data. Possibly out of space\n"); + return false; + } + numleft -= numread; + } while (numleft && !(numread < 1024 && numleft)); + return (numleft == 0); } -int main(int argc, char **argv) { - try { - if (argc != 4) { - printf("usage: bpatch \n"); - printf("needs a reference file, file to output, and patchfile:\n"); - return 1; - } +int main(int argc, char **argv) +{ + try + { + if (argc != 4) + { + printf("usage: bpatch \n"); + printf("needs a reference file, file to output, and patchfile:\n"); + return 1; + } - if (!fileExists(argv[1]) || !fileExists(argv[3])) { - printf("one of the input files does not exist\n"); - return 1; - } + if (!fileExists(argv[1]) || !fileExists(argv[3])) + { + printf("one of the input files does not exist\n"); + return 1; + } - FILE *patchfile = fopen(argv[3], "rb"); - char magic[3]; - fread_fixed(patchfile, magic, 3); - if (strncmp(magic, "BDT", 3)) { - printf("Given file is not a recognized patchfile\n"); - return 1; - } - unsigned short version = read_word(patchfile); - if (version != 1) { - printf("unsupported patch version\n"); - return 1; - } - char intsize; - fread_fixed(patchfile, &intsize, 1); - if (intsize != 4) { - printf("unsupported file pointer size\n"); - return 1; - } - unsigned size1 = read_dword(patchfile), - size2 = read_dword(patchfile); + FILE *patchfile = fopen(argv[3], "rb"); + char magic[3]; + fread_fixed(patchfile, magic, 3); + if (strncmp(magic, "BDT", 3)) + { + printf("Given file is not a recognized patchfile\n"); + return 1; + } + unsigned short version = read_word(patchfile); + if (version != 1) + { + printf("unsupported patch version\n"); + return 1; + } + char intsize; + fread_fixed(patchfile, &intsize, 1); + if (intsize != 4) + { + printf("unsupported file pointer size\n"); + return 1; + } + /*unsigned size1 =*/ read_dword(patchfile); + unsigned size2 = read_dword(patchfile); - unsigned nummatches = read_dword(patchfile); + unsigned nummatches = read_dword(patchfile); - unsigned * copyloc1 = new unsigned[nummatches + 1]; - unsigned * copyloc2 = new unsigned[nummatches + 1]; - unsigned * copynum = new unsigned[nummatches + 1]; + unsigned * copyloc1 = new unsigned[nummatches + 1]; + unsigned * copyloc2 = new unsigned[nummatches + 1]; + unsigned * copynum = new unsigned[nummatches + 1]; + std::unique_ptr copyloc1_holder(copyloc1), copyloc2_holder(copyloc2), copynum_holder(copynum); - for (unsigned i = 0; i < nummatches; ++i) { - copyloc1[i] = read_dword(patchfile); - copyloc2[i] = read_dword(patchfile); - copynum[i] = read_dword(patchfile); - size2 -= copyloc2[i] + copynum[i]; - } - if (size2) { - copyloc1[nummatches] = 0; copynum[nummatches] = 0; - copyloc2[nummatches] = size2; - ++nummatches; - } + for (unsigned i = 0; i < nummatches; ++i) + { + copyloc1[i] = read_dword(patchfile); + copyloc2[i] = read_dword(patchfile); + copynum[i] = read_dword(patchfile); + size2 -= copyloc2[i] + copynum[i]; + } + if (size2) { + copyloc1[nummatches] = 0; copynum[nummatches] = 0; + copyloc2[nummatches] = size2; + ++nummatches; + } - FILE *ref = fopen(argv[1], "rb"); - FILE *outfile = fopen(argv[2], "wb"); + FILE * ref = fopen(argv[1], "rb"); + if (ref == nullptr) + { + printf("Error: unable to open file %s\n", argv[1]); + return -1; + } + std::unique_ptr ref_holder(ref, fclose); - for (unsigned i = 0; i < nummatches; ++i) { - if (!copy_bytes_to_file(patchfile, outfile, copyloc2[i])) { - printf("Error. patchfile is truncated\n"); - return -1; - } + FILE * outfile = fopen(argv[2], "wb"); + if (outfile == nullptr) + { + printf("Error: unable to open file %s\n", argv[2]); + return -1; + } + std::unique_ptr outfile_holder(outfile, fclose); - int copyloc = copyloc1[i]; - fseek(ref, copyloc, SEEK_CUR); + for (unsigned i = 0; i < nummatches; ++i) + { + if (!copy_bytes_to_file(patchfile, outfile, copyloc2[i])) + { + printf("Error: patchfile is truncated\n"); + return -1; + } - if (!copy_bytes_to_file(ref, outfile, copynum[i])) { - printf("Error while copying from reference file\n"); - return -1; - } - } + int copyloc = copyloc1[i]; + fseek(ref, copyloc, SEEK_CUR); - delete [] copynum; - delete [] copyloc2; - delete [] copyloc1; + if (!copy_bytes_to_file(ref, outfile, copynum[i])) + { + printf("Error while copying from reference file\n"); + return -1; + } + } + } + catch (const char * desc) + { + fprintf (stderr, "FATAL: %s\n", desc); + return -1; + } - } catch (const char * desc){ - fprintf (stderr, "FATAL: %s\n", desc); - return -1; - } - - return 0; + return 0; } From 55586dd1c993083693688922dc6b1005f3918573 Mon Sep 17 00:00:00 2001 From: Deamhan Date: Thu, 14 Mar 2019 20:14:54 +0300 Subject: [PATCH 11/54] multiple header include protection has been added --- src/checksum.h | 4 ++++ src/compatibility.h | 5 +++++ src/file.h | 5 +++++ 3 files changed, 14 insertions(+) diff --git a/src/checksum.h b/src/checksum.h index f12715c..7fcc5cb 100755 --- a/src/checksum.h +++ b/src/checksum.h @@ -2,6 +2,9 @@ * License, v. 2.0. If a copy of the MPL was not distributed with this * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ +#ifndef __CHECKSUM_H__ +#define __CHECKSUM_H__ + struct Hash { typedef uint64_t Value; @@ -49,3 +52,4 @@ struct Hash } }; +#endif // __CHECKSUM_H__ diff --git a/src/compatibility.h b/src/compatibility.h index 5dcdfd0..968d9f3 100755 --- a/src/compatibility.h +++ b/src/compatibility.h @@ -2,6 +2,9 @@ * License, v. 2.0. If a copy of the MPL was not distributed with this * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ +#ifndef __COMPATIBILITY_H__ +#define __COMPATIBILITY_H__ + // Fix for MSVC++, which doesn't support Variable Length Arrays. #include #include @@ -11,3 +14,5 @@ #else #define STACK_ALLOC(name, type, num) type name[num] #endif + +#endif // __COMPATIBILITY_H__ diff --git a/src/file.h b/src/file.h index ad21884..a2e0cf7 100644 --- a/src/file.h +++ b/src/file.h @@ -2,6 +2,9 @@ * License, v. 2.0. If a copy of the MPL was not distributed with this * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ +#ifndef __FILE_H__ +#define __FILE_H__ + #include #ifdef USE_CXX17 #include @@ -153,3 +156,5 @@ unsigned getLenOfFile(const char * fname) } #endif // USE_CXX17 + +#endif // __FILE_H__ From 856c7826f0ee494c3ec5ad701f9a3946c5c64c8c Mon Sep 17 00:00:00 2001 From: Deamhan Date: Thu, 14 Mar 2019 20:18:17 +0300 Subject: [PATCH 12/54] file.h: double 'const' has been eliminated --- src/file.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/file.h b/src/file.h index a2e0cf7..7e6276c 100644 --- a/src/file.h +++ b/src/file.h @@ -129,7 +129,7 @@ inline bool fileExists(const T& fname) } template -inline uint64_t getLenOfFile(const const T& fname) +inline uint64_t getLenOfFile(const T& fname) { std::error_code ec; return std::filesystem::file_size(fname, ec); From 25f6104a9d4746ae0ffc17ae8856fce2dd29cea4 Mon Sep 17 00:00:00 2001 From: Deamhan Date: Fri, 15 Mar 2019 10:11:45 +0300 Subject: [PATCH 13/54] GCC build issues have been fixed --- src/bdelta.cpp | 6 +++--- src/bpatch.cpp | 2 +- src/file.h | 32 +++++++++++++------------------- src/libbdelta.cpp | 25 +++++++++++++------------ 4 files changed, 30 insertions(+), 35 deletions(-) diff --git a/src/bdelta.cpp b/src/bdelta.cpp index 02193bd..73c21bf 100644 --- a/src/bdelta.cpp +++ b/src/bdelta.cpp @@ -11,19 +11,19 @@ #include "file.h" #include "compatibility.h" -const void *f_read(void *f, void *buf, unsigned place, unsigned num) +static const void *f_read(void *f, void *buf, unsigned place, unsigned num) { fseek((FILE *)f, place, SEEK_SET); fread_fixed((FILE *)f, buf, num); return buf; } -inline const void * m_read(void *f, void * /*buf*/, unsigned place, unsigned /*num*/) +static inline const void * m_read(void *f, void * /*buf*/, unsigned place, unsigned /*num*/) { return (const char*)f + place; } -void my_pass(BDelta_Instance *b, unsigned blocksize, unsigned minMatchSize, unsigned flags) +static void my_pass(BDelta_Instance *b, unsigned blocksize, unsigned minMatchSize, unsigned flags) { bdelta_pass(b, blocksize, minMatchSize, 0, flags); bdelta_clean_matches(b, BDELTA_REMOVE_OVERLAP); diff --git a/src/bpatch.cpp b/src/bpatch.cpp index f5c35f3..a94a4e0 100644 --- a/src/bpatch.cpp +++ b/src/bpatch.cpp @@ -7,7 +7,7 @@ #include "file.h" #include "compatibility.h" -bool copy_bytes_to_file(FILE *infile, FILE *outfile, unsigned numleft) +static bool copy_bytes_to_file(FILE *infile, FILE *outfile, unsigned numleft) { size_t numread; do { diff --git a/src/file.h b/src/file.h index 7e6276c..cd4787b 100644 --- a/src/file.h +++ b/src/file.h @@ -16,7 +16,7 @@ static char error_message_buffer[512]; -void fread_fixed(FILE *f, void * _buf, unsigned num_bytes) +static void fread_fixed(FILE *f, void * _buf, unsigned num_bytes) { char * buf = (char *)_buf; @@ -37,7 +37,7 @@ void fread_fixed(FILE *f, void * _buf, unsigned num_bytes) } } -void fwrite_fixed(FILE *f, const void * _buf, unsigned num_bytes) +static void fwrite_fixed(FILE *f, const void * _buf, unsigned num_bytes) { const char * buf = (const char *)_buf; @@ -57,9 +57,9 @@ void fwrite_fixed(FILE *f, const void * _buf, unsigned num_bytes) } } -#ifdef BIG_ENDIAN +#ifdef BIG_ENDIAN_HOST -unsigned read_word(FILE *f) +static inline unsigned read_word(FILE *f) { unsigned char b, b2; fread_fixed(f, &b, 1); @@ -67,13 +67,13 @@ unsigned read_word(FILE *f) return (b2 << 8) + b; } -unsigned read_dword(FILE *f) +static inline unsigned read_dword(FILE *f) { unsigned low = read_word(f); return (read_word(f) << 16) + low; } -void write_word(FILE *f, unsigned number) +static inline void write_word(FILE *f, unsigned number) { unsigned char b = number & 255, b2 = number >> 8; @@ -81,7 +81,7 @@ void write_word(FILE *f, unsigned number) fwrite_fixed(f, &b2, 1); } -void write_dword(FILE *f, unsigned number) +static inline void write_dword(FILE *f, unsigned number) { write_word(f, number & 65535); write_word(f, number >> 16); @@ -90,7 +90,7 @@ void write_dword(FILE *f, unsigned number) #else template ::value>> -inline T read_type(FILE *f) +static inline T read_type(FILE *f) { T result = 0; fread_fixed(f, &result, sizeof(result)); @@ -101,14 +101,8 @@ inline T read_type(FILE *f) #define read_word(f) read_type((f)) #define read_dword(f) read_type((f)) - -inline void write_word(FILE *f, uint16_t number) -{ - fwrite_fixed(f, &number, sizeof(number)); -} - template ::value>> -inline void write_type(FILE *f, T number) +static inline void write_type(FILE *f, T number) { fwrite_fixed(f, &number, sizeof(number)); } @@ -122,14 +116,14 @@ inline void write_type(FILE *f, T number) #ifdef USE_CXX17 template -inline bool fileExists(const T& fname) +static inline bool fileExists(const T& fname) { std::error_code ec; return std::filesystem::exists(fname, ec); } template -inline uint64_t getLenOfFile(const T& fname) +static inline uint64_t getLenOfFile(const T& fname) { std::error_code ec; return std::filesystem::file_size(fname, ec); @@ -137,7 +131,7 @@ inline uint64_t getLenOfFile(const T& fname) #else -bool fileExists(const char * fname) +static bool fileExists(const char * fname) { FILE *f = fopen(fname, "rb"); bool exists = (f != nullptr); @@ -146,7 +140,7 @@ bool fileExists(const char * fname) return exists; } -unsigned getLenOfFile(const char * fname) +static unsigned getLenOfFile(const char * fname) { FILE *f = fopen(fname, "rb"); fseek(f, 0, SEEK_END); diff --git a/src/libbdelta.cpp b/src/libbdelta.cpp index 653981d..4dd2067 100644 --- a/src/libbdelta.cpp +++ b/src/libbdelta.cpp @@ -3,6 +3,7 @@ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ #include +#include #include #include #include @@ -83,14 +84,14 @@ struct Checksums_Instance }; -unsigned match_buf_forward(const void *buf1, const void *buf2, unsigned num) +static unsigned match_buf_forward(const void *buf1, const void *buf2, unsigned num) { unsigned i = 0; while (i < num && ((const Token*)buf1)[i] == ((const Token*)buf2)[i]) ++i; return i; } -unsigned match_buf_backward(const void *buf1, const void *buf2, unsigned num) +static unsigned match_buf_backward(const void *buf1, const void *buf2, unsigned num) { int i = num; do @@ -100,7 +101,7 @@ unsigned match_buf_backward(const void *buf1, const void *buf2, unsigned num) while (i >= 0 && ((const Token*)buf1)[i] == ((const Token*)buf2)[i]); return (num - i - 1); } -unsigned match_forward(BDelta_Instance *b, unsigned p1, unsigned p2) +static unsigned match_forward(BDelta_Instance *b, unsigned p1, unsigned p2) { unsigned num = 0, match, numtoread; do @@ -117,7 +118,7 @@ unsigned match_forward(BDelta_Instance *b, unsigned p1, unsigned p2) return num; } -unsigned match_backward(BDelta_Instance *b, unsigned p1, unsigned p2, unsigned blocksize) +static unsigned match_backward(BDelta_Instance *b, unsigned p1, unsigned p2, unsigned blocksize) { unsigned num = 0, match, numtoread; do @@ -153,26 +154,26 @@ struct UnusedRange }; // Sort first by location, second by match length (larger matches first) -bool comparep(UnusedRange r1, UnusedRange r2) +static bool comparep(UnusedRange r1, UnusedRange r2) { if (r1.p != r2.p) return r1.p < r2.p; return r1.num > r2.num; } -bool comparemrp2(UnusedRange r1, UnusedRange r2) +static bool comparemrp2(UnusedRange r1, UnusedRange r2) { if (r1.mr->p2 != r2.mr->p2) return r1.mr->p2 < r2.mr->p2; return r1.mr->num > r2.mr->num; } -bool compareMatchP2(Match r1, Match r2) +static bool compareMatchP2(Match r1, Match r2) { if (r1.p2 != r2.p2) return r1.p2 < r2.p2; return r1.num > r2.num; } -void addMatch(BDelta_Instance *b, unsigned p1, unsigned p2, unsigned num, std::list::iterator place) +static void addMatch(BDelta_Instance *b, unsigned p1, unsigned p2, unsigned num, std::list::iterator place) { Match newMatch = Match(p1, p2, num); while (place != b->matches.begin() && ! compareMatchP2(*place, newMatch)) @@ -188,7 +189,7 @@ T absoluteDifference(T a, T b) return std::max(a, b) - std::min(a, b); } -void findMatches(BDelta_Instance *b, Checksums_Instance *h, unsigned minMatchSize, unsigned start, unsigned end, unsigned place, std::list::iterator iterPlace) +static void findMatches(BDelta_Instance *b, Checksums_Instance *h, unsigned minMatchSize, unsigned start, unsigned end, unsigned place, std::list::iterator iterPlace) { const unsigned blocksize = h->blocksize; STACK_ALLOC(buf1, Token, blocksize); @@ -330,7 +331,7 @@ void bdelta_done_alg(BDelta_Instance *b) // Adapted from http://graphics.stanford.edu/~seander/bithacks.html#RoundUpPowerOf2 -unsigned roundUpPowerOf2(unsigned v) +static unsigned roundUpPowerOf2(unsigned v) { --v; for (int i = 1; i <= 16; i *= 2) @@ -338,7 +339,7 @@ unsigned roundUpPowerOf2(unsigned v) return (v + 1); } -void bdelta_pass_2(BDelta_Instance *b, unsigned blocksize, unsigned minMatchSize, UnusedRange *unused, unsigned numunused, UnusedRange *unused2, unsigned numunused2) +static void bdelta_pass_2(BDelta_Instance *b, unsigned blocksize, unsigned minMatchSize, UnusedRange *unused, unsigned numunused, UnusedRange *unused2, unsigned numunused2) { Checksums_Instance h(blocksize); b->access_int = -1; @@ -450,7 +451,7 @@ void bdelta_showMatches(BDelta_Instance *b) printf ("\n\n"); } -void get_unused_blocks(UnusedRange *unused, unsigned *numunusedptr) +static void get_unused_blocks(UnusedRange *unused, unsigned *numunusedptr) { unsigned nextStartPos = 0; for (unsigned i = 1; i < *numunusedptr; ++i) From d98c35ae18262d28db757680b288d3b7ff87037f Mon Sep 17 00:00:00 2001 From: Deamhan Date: Fri, 15 Mar 2019 12:03:26 +0300 Subject: [PATCH 14/54] big stack objects have been moved to heap, thread safety has been improved, cleanup --- makefiles/CMakeLists.txt | 7 ++++--- src/bdelta.cpp | 14 +++++++------ src/bpatch.cpp | 31 +++++++++++++++++++--------- src/file.h | 44 +++------------------------------------- src/libbdelta.cpp | 29 +++++++++++++------------- 5 files changed, 50 insertions(+), 75 deletions(-) diff --git a/makefiles/CMakeLists.txt b/makefiles/CMakeLists.txt index 56cc38b..fe41a09 100644 --- a/makefiles/CMakeLists.txt +++ b/makefiles/CMakeLists.txt @@ -13,15 +13,16 @@ ENDIF() set (COMMON_SOURCES - ../src/compatibility.h - ../src/checksum.h - ../src/file.h + ../src/file.h + ../src/file.cpp ) set (BDELTA_SOURCES ${COMMON_SOURCES} ../src/bdelta.cpp + ../src/bdelta.h + ../src/checksum.h ../src/libbdelta.cpp ) diff --git a/src/bdelta.cpp b/src/bdelta.cpp index 73c21bf..ac211de 100644 --- a/src/bdelta.cpp +++ b/src/bdelta.cpp @@ -2,14 +2,15 @@ * License, v. 2.0. If a copy of the MPL was not distributed with this * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ +#include #include #include #include #include +#include #include "bdelta.h" #include "file.h" -#include "compatibility.h" static const void *f_read(void *f, void *buf, unsigned place, unsigned num) { @@ -161,14 +162,15 @@ int main(int argc, char **argv) } unsigned fp = 0; + const size_t WRITE_BUFFER_SIZE = 4096; + std::unique_ptr write_buffer(new uint8_t[WRITE_BUFFER_SIZE]); for (int i = 0; i < nummatches; ++i) { unsigned num = copyloc2[i]; while (num > 0) { - unsigned towrite = (num > 4096) ? 4096 : num; - unsigned char buf[4096]; - const void * block = all_ram_mode ? m_read(m2.get(), buf, fp, towrite) : f_read(f2, buf, fp, towrite); + unsigned towrite = std::min(num, WRITE_BUFFER_SIZE); + const void * block = all_ram_mode ? m_read(m2.get(), write_buffer.get(), fp, towrite) : f_read(f2, write_buffer.get(), fp, towrite); fwrite_fixed(fout, block, towrite); num -= towrite; fp += towrite; @@ -179,9 +181,9 @@ int main(int argc, char **argv) } } - catch (const char * desc) + catch (const std::exception& ex) { - fprintf (stderr, "FATAL: %s\n", desc); + fprintf (stderr, "FATAL: %s\n", ex.what()); return -1; } diff --git a/src/bpatch.cpp b/src/bpatch.cpp index a94a4e0..218e6c6 100644 --- a/src/bpatch.cpp +++ b/src/bpatch.cpp @@ -2,24 +2,35 @@ * License, v. 2.0. If a copy of the MPL was not distributed with this * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ +#include +#include #include #include + #include "file.h" -#include "compatibility.h" static bool copy_bytes_to_file(FILE *infile, FILE *outfile, unsigned numleft) { - size_t numread; - do { - char buf[1024]; - numread = fread(buf, 1, numleft > 1024 ? 1024 : numleft, infile); - if (fwrite(buf, 1, numread, outfile) != numread) { + const size_t BUFFER_SIZE = 65536; + std::unique_ptr buf(new uint8_t[BUFFER_SIZE]); + while (numleft != 0) + { + const size_t len = std::min(BUFFER_SIZE, numleft); + size_t numread = fread(buf.get(), 1, len, infile); + if (len != numread) + { + printf("Could not read data\n"); + return false; + } + if (fwrite(buf.get(), 1, numread, outfile) != numread) + { printf("Could not write temporary data. Possibly out of space\n"); return false; } numleft -= numread; - } while (numleft && !(numread < 1024 && numleft)); - return (numleft == 0); + } + + return true; } int main(int argc, char **argv) @@ -117,9 +128,9 @@ int main(int argc, char **argv) } } } - catch (const char * desc) + catch (const std::exception& ex) { - fprintf (stderr, "FATAL: %s\n", desc); + fprintf (stderr, "FATAL: %s\n", ex.what()); return -1; } diff --git a/src/file.h b/src/file.h index cd4787b..1dd0ee9 100644 --- a/src/file.h +++ b/src/file.h @@ -10,52 +10,14 @@ #include #endif // USE_CXX17 #include +#include #include #define MAX_IO_BLOCK_SIZE (1024 * 1024) -static char error_message_buffer[512]; +void fread_fixed(FILE *f, void * _buf, unsigned num_bytes); +void fwrite_fixed(FILE *f, const void * _buf, unsigned num_bytes); -static void fread_fixed(FILE *f, void * _buf, unsigned num_bytes) -{ - char * buf = (char *)_buf; - - while (num_bytes != 0) - { - unsigned block_size = num_bytes; - if (block_size > MAX_IO_BLOCK_SIZE) - block_size = MAX_IO_BLOCK_SIZE; - - size_t r = fread(buf, 1, block_size, f); - if (r != block_size) - { - snprintf(error_message_buffer, sizeof(error_message_buffer), "read error: fread_fixed(block_size=%u) != %u", block_size, (unsigned)r); - throw error_message_buffer; - } - buf += block_size; - num_bytes -= block_size; - } -} - -static void fwrite_fixed(FILE *f, const void * _buf, unsigned num_bytes) -{ - const char * buf = (const char *)_buf; - - while (num_bytes != 0) - { - unsigned block_size = num_bytes; - if (block_size > MAX_IO_BLOCK_SIZE) block_size = MAX_IO_BLOCK_SIZE; - - size_t r = fwrite(buf, 1, block_size, f); - if (r != block_size) - { - snprintf(error_message_buffer, sizeof(error_message_buffer), "write error: fwrite_fixed(num_bytes=%u) != %u", block_size, (unsigned)r); - throw error_message_buffer; - } - buf += block_size; - num_bytes -= block_size; - } -} #ifdef BIG_ENDIAN_HOST diff --git a/src/libbdelta.cpp b/src/libbdelta.cpp index 4dd2067..9ad2926 100644 --- a/src/libbdelta.cpp +++ b/src/libbdelta.cpp @@ -4,13 +4,12 @@ #include #include +#include #include #include #include #include -#include "compatibility.h" - #if !defined(TOKEN_SIZE) || TOKEN_SIZE == 1 typedef uint8_t Token; #elif TOKEN_SIZE == 2 @@ -104,11 +103,12 @@ static unsigned match_buf_backward(const void *buf1, const void *buf2, unsigned static unsigned match_forward(BDelta_Instance *b, unsigned p1, unsigned p2) { unsigned num = 0, match, numtoread; + const size_t BUFFER_SIZE = 4096; + std::unique_ptr buffer(new Token[BUFFER_SIZE * 2]); + Token * buf1 = buffer.get(), * buf2 = buf1 + BUFFER_SIZE; do { - numtoread = std::min(b->data1_size - p1, b->data2_size - p2); - if (numtoread > 4096) numtoread = 4096; - Token buf1[4096], buf2[4096]; + numtoread = std::min(std::min(b->data1_size - p1, b->data2_size - p2), BUFFER_SIZE); const Token *read1 = b->read1(buf1, p1, numtoread), *read2 = b->read2(buf2, p2, numtoread); p1 += numtoread; p2 += numtoread; @@ -121,16 +121,14 @@ static unsigned match_forward(BDelta_Instance *b, unsigned p1, unsigned p2) static unsigned match_backward(BDelta_Instance *b, unsigned p1, unsigned p2, unsigned blocksize) { unsigned num = 0, match, numtoread; + const size_t BUFFER_SIZE = 4096; + std::unique_ptr buffer(new Token[BUFFER_SIZE * 2]); + Token * buf1 = buffer.get(), * buf2 = buf1 + BUFFER_SIZE; do { - numtoread = std::min(p1, p2); - if (numtoread > blocksize) - numtoread = blocksize; - if (numtoread > 4096) - numtoread = 4096; + numtoread = std::min(std::min(std::min(p1, p2), blocksize), BUFFER_SIZE); p1 -= numtoread; p2 -= numtoread; - Token buf1[4096], buf2[4096]; const Token *read1 = b->read1(buf1, p1, numtoread), *read2 = b->read2(buf2, p2, numtoread); match = match_buf_backward(read1, read2, numtoread); @@ -192,8 +190,9 @@ T absoluteDifference(T a, T b) static void findMatches(BDelta_Instance *b, Checksums_Instance *h, unsigned minMatchSize, unsigned start, unsigned end, unsigned place, std::list::iterator iterPlace) { const unsigned blocksize = h->blocksize; - STACK_ALLOC(buf1, Token, blocksize); - STACK_ALLOC(buf2, Token, blocksize); + + std::unique_ptr buffer(new Token[blocksize * 2]); + Token * buf1 = buffer.get(), * buf2 = buf1 + blocksize; unsigned best1 = 0, best2 = 0, bestnum = 0; unsigned processMatchesPos = 0; @@ -366,13 +365,13 @@ static void bdelta_pass_2(BDelta_Instance *b, unsigned blocksize, unsigned minMa std::unique_ptr checksums_holder(h.checksums); h.numchecksums = 0; - STACK_ALLOC(buf, Token, blocksize); + std::unique_ptr buf(new Token[blocksize]); for (unsigned i = 0; i < numunused; ++i) { unsigned first = unused[i].p, last = unused[i].p + unused[i].num; for (unsigned loc = first; loc + blocksize <= last; loc += blocksize) { - const Token *read = b->read1(buf, loc, blocksize); + const Token *read = b->read1(buf.get(), loc, blocksize); Hash::Value blocksum = Hash(read, blocksize).getValue(); // Adjacent checksums are never repeated. h.add(checksum_entry(blocksum, loc)); From 3e7e48979bb55b947e325d02f7ecf12b17e83cfd Mon Sep 17 00:00:00 2001 From: Deamhan Date: Fri, 15 Mar 2019 12:15:02 +0300 Subject: [PATCH 15/54] libbdelta.cpp: performance improvement & cleanup --- src/libbdelta.cpp | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/src/libbdelta.cpp b/src/libbdelta.cpp index 9ad2926..e856d2f 100644 --- a/src/libbdelta.cpp +++ b/src/libbdelta.cpp @@ -9,6 +9,7 @@ #include #include #include +#include #if !defined(TOKEN_SIZE) || TOKEN_SIZE == 1 typedef uint8_t Token; @@ -66,14 +67,16 @@ struct Checksums_Instance { unsigned blocksize; unsigned htablesize; - checksum_entry **htable; // Points to first match in checksums + checksum_entry **htable; // Points to first match in checksums checksum_entry *checksums; // Sorted list of all checksums unsigned numchecksums; - Checksums_Instance(int blocksize) { this->blocksize = blocksize; } - void add(checksum_entry ck) + explicit Checksums_Instance(int _blocksize) : blocksize(_blocksize) {} + + template + void add(T&& ck) { - checksums[numchecksums] = ck; + checksums[numchecksums] = std::forward(ck); ++numchecksums; } unsigned tableIndex(Hash::Value hashValue) @@ -285,7 +288,7 @@ static void findMatches(BDelta_Instance *b, Checksums_Instance *h, unsigned minM struct Checksums_Compare { Checksums_Instance &ci; - Checksums_Compare(Checksums_Instance &h) : ci(h) {} + explicit Checksums_Compare(Checksums_Instance &h) : ci(h) {} bool operator() (checksum_entry c1, checksum_entry c2) { unsigned ti1 = ci.tableIndex(c1.cksum), ti2 = ci.tableIndex(c2.cksum); From 310a364b5d294277ff95417627dd37dd26b1b958 Mon Sep 17 00:00:00 2001 From: Deamhan Date: Fri, 15 Mar 2019 19:36:57 +0300 Subject: [PATCH 16/54] libbdelta.cpp: some performance improvements & cleanup --- src/libbdelta.cpp | 59 +++++++++++++++++++++++++++++------------------ 1 file changed, 36 insertions(+), 23 deletions(-) diff --git a/src/libbdelta.cpp b/src/libbdelta.cpp index e856d2f..8e8e7a7 100644 --- a/src/libbdelta.cpp +++ b/src/libbdelta.cpp @@ -24,14 +24,16 @@ typedef uint32_t Token; const bool verbose = false; -struct checksum_entry { +struct checksum_entry +{ Hash::Value cksum; //Rolling checksums unsigned loc; checksum_entry() {} checksum_entry(Hash::Value _cksum, unsigned _loc) : cksum(_cksum), loc(_loc) {} }; -struct Range { +struct Range +{ unsigned p, num; Range() {} Range(unsigned _p, unsigned _num) : p(_p), num(_num) {} @@ -155,31 +157,40 @@ struct UnusedRange }; // Sort first by location, second by match length (larger matches first) -static bool comparep(UnusedRange r1, UnusedRange r2) +class comparep { - if (r1.p != r2.p) - return r1.p < r2.p; - return r1.num > r2.num; -} -static bool comparemrp2(UnusedRange r1, UnusedRange r2) +public: + bool operator()(UnusedRange r1, UnusedRange r2) + { + return ((r1.p != r2.p) ? (r1.p < r2.p) : (r1.num > r2.num)); + } +}; + +class comparemrp2 { - if (r1.mr->p2 != r2.mr->p2) - return r1.mr->p2 < r2.mr->p2; - return r1.mr->num > r2.mr->num; -} -static bool compareMatchP2(Match r1, Match r2) +public: + bool operator()(UnusedRange r1, UnusedRange r2) + { + return ((r1.mr->p2 != r2.mr->p2)? (r1.mr->p2 < r2.mr->p2) : (r1.mr->num > r2.mr->num)); + } +}; + +class compareMatchP2 { - if (r1.p2 != r2.p2) - return r1.p2 < r2.p2; - return r1.num > r2.num; -} +public: + bool operator()(Match r1, Match r2) + { + return ((r1.p2 != r2.p2) ? (r1.p2 < r2.p2) : (r1.num > r2.num)); + } +}; static void addMatch(BDelta_Instance *b, unsigned p1, unsigned p2, unsigned num, std::list::iterator place) { Match newMatch = Match(p1, p2, num); - while (place != b->matches.begin() && ! compareMatchP2(*place, newMatch)) + compareMatchP2 comp; + while (place != b->matches.begin() && !comp(*place, newMatch)) --place; - while (place != b->matches.end() && compareMatchP2(*place, newMatch)) + while (place != b->matches.end() && comp(*place, newMatch)) ++place; b->matches.insert(place, newMatch); } @@ -190,6 +201,8 @@ T absoluteDifference(T a, T b) return std::max(a, b) - std::min(a, b); } + + static void findMatches(BDelta_Instance *b, Checksums_Instance *h, unsigned minMatchSize, unsigned start, unsigned end, unsigned place, std::list::iterator iterPlace) { const unsigned blocksize = h->blocksize; @@ -352,7 +365,7 @@ static void bdelta_pass_2(BDelta_Instance *b, unsigned blocksize, unsigned minMa h.htablesize = std::max((unsigned)2, roundUpPowerOf2(numblocks)); h.htable = new checksum_entry*[h.htablesize]; - if (!h.htable) + if (h.htable == nullptr) { b->errorcode = BDELTA_MEM_ERROR; return; @@ -417,7 +430,7 @@ void bdelta_swap_inputs(BDelta_Instance *b) std::swap(l->p1, l->p2); std::swap(b->data1_size, b->data2_size); std::swap(b->handle1, b->handle2); - b->matches.sort(compareMatchP2); + b->matches.sort(compareMatchP2()); } void bdelta_clean_matches(BDelta_Instance *b, unsigned flags) @@ -485,7 +498,7 @@ void bdelta_pass(BDelta_Instance *b, unsigned blocksize, unsigned minMatchSize, unused2[numunused2++] = UnusedRange(l->p2, l->num, l, l); } - std::sort(unused + 1, unused + numunused, comparep); // Leave empty match at beginning + std::sort(unused + 1, unused + numunused, comparep()); // Leave empty match at beginning get_unused_blocks(unused, &numunused); get_unused_blocks(unused2, &numunused2); @@ -494,7 +507,7 @@ void bdelta_pass(BDelta_Instance *b, unsigned blocksize, unsigned minMatchSize, bdelta_pass_2(b, blocksize, minMatchSize, unused, numunused, unused2, numunused2); else { - std::sort(unused + 1, unused + numunused, comparemrp2); + std::sort(unused + 1, unused + numunused, comparemrp2()); for (unsigned i = 1; i < numunused; ++i) { UnusedRange u1 = unused[i], u2 = unused2[i]; From a79ee72405c1f108a3508600b922aa38bbfbc58e Mon Sep 17 00:00:00 2001 From: Deamhan Date: Fri, 15 Mar 2019 22:40:13 +0300 Subject: [PATCH 17/54] libbdelta.cpp: memory management performance has been improved --- src/libbdelta.cpp | 89 +++++++++++++++++++++++++++++------------------ 1 file changed, 56 insertions(+), 33 deletions(-) diff --git a/src/libbdelta.cpp b/src/libbdelta.cpp index 8e8e7a7..c62d143 100644 --- a/src/libbdelta.cpp +++ b/src/libbdelta.cpp @@ -9,6 +9,7 @@ #include #include #include +#include #include #if !defined(TOKEN_SIZE) || TOKEN_SIZE == 1 @@ -95,6 +96,7 @@ static unsigned match_buf_forward(const void *buf1, const void *buf2, unsigned n ++i; return i; } + static unsigned match_buf_backward(const void *buf1, const void *buf2, unsigned num) { int i = num; @@ -105,15 +107,16 @@ static unsigned match_buf_backward(const void *buf1, const void *buf2, unsigned while (i >= 0 && ((const Token*)buf1)[i] == ((const Token*)buf2)[i]); return (num - i - 1); } + +static const size_t TOKEN_BUFFER_SIZE = 4096; +static thread_local std::unique_ptr match_forward_buffer; static unsigned match_forward(BDelta_Instance *b, unsigned p1, unsigned p2) { unsigned num = 0, match, numtoread; - const size_t BUFFER_SIZE = 4096; - std::unique_ptr buffer(new Token[BUFFER_SIZE * 2]); - Token * buf1 = buffer.get(), * buf2 = buf1 + BUFFER_SIZE; + Token * buf1 = match_forward_buffer.get(), * buf2 = buf1 + TOKEN_BUFFER_SIZE; do { - numtoread = std::min(std::min(b->data1_size - p1, b->data2_size - p2), BUFFER_SIZE); + numtoread = std::min(std::min(b->data1_size - p1, b->data2_size - p2), TOKEN_BUFFER_SIZE); const Token *read1 = b->read1(buf1, p1, numtoread), *read2 = b->read2(buf2, p2, numtoread); p1 += numtoread; p2 += numtoread; @@ -123,15 +126,14 @@ static unsigned match_forward(BDelta_Instance *b, unsigned p1, unsigned p2) return num; } +static thread_local std::unique_ptr match_backward_buffer; static unsigned match_backward(BDelta_Instance *b, unsigned p1, unsigned p2, unsigned blocksize) { unsigned num = 0, match, numtoread; - const size_t BUFFER_SIZE = 4096; - std::unique_ptr buffer(new Token[BUFFER_SIZE * 2]); - Token * buf1 = buffer.get(), * buf2 = buf1 + BUFFER_SIZE; + Token * buf1 = match_backward_buffer.get(), * buf2 = buf1 + TOKEN_BUFFER_SIZE; do { - numtoread = std::min(std::min(std::min(p1, p2), blocksize), BUFFER_SIZE); + numtoread = std::min(std::min(std::min(p1, p2), blocksize), TOKEN_BUFFER_SIZE); p1 -= numtoread; p2 -= numtoread; const Token *read1 = b->read1(buf1, p1, numtoread), @@ -202,13 +204,14 @@ T absoluteDifference(T a, T b) } - +static constexpr size_t BUFFER_DEFAULT_SIZE = 16 * 1024; +static thread_local std::vector findMatchesBuffer; static void findMatches(BDelta_Instance *b, Checksums_Instance *h, unsigned minMatchSize, unsigned start, unsigned end, unsigned place, std::list::iterator iterPlace) { const unsigned blocksize = h->blocksize; - std::unique_ptr buffer(new Token[blocksize * 2]); - Token * buf1 = buffer.get(), * buf2 = buf1 + blocksize; + findMatchesBuffer.resize(blocksize * 2); + Token * buf1 = findMatchesBuffer.data(), * buf2 = buf1 + blocksize; unsigned best1 = 0, best2 = 0, bestnum = 0; unsigned processMatchesPos = 0; @@ -317,6 +320,11 @@ struct Checksums_Compare } }; +static thread_local std::vector bdelta_pass_2_htable; +static thread_local std::vector bdelta_pass_2_hchecksums; +static thread_local std::vector bdelta_pass_2_buf; +static thread_local std::vector bdelta_pass_unused; + BDelta_Instance *bdelta_init_alg(unsigned data1_size, unsigned data2_size, bdelta_readCallback cb, void *handle1, void *handle2, unsigned tokenSize) @@ -335,6 +343,16 @@ BDelta_Instance *bdelta_init_alg(unsigned data1_size, unsigned data2_size, b->handle1 = handle1; b->handle2 = handle2; b->access_int = -1; + + match_forward_buffer.reset(new Token[TOKEN_BUFFER_SIZE * 2]); + match_backward_buffer.reset(new Token[TOKEN_BUFFER_SIZE * 2]); + + findMatchesBuffer.resize(BUFFER_DEFAULT_SIZE); + bdelta_pass_2_htable.resize(BUFFER_DEFAULT_SIZE); + bdelta_pass_2_hchecksums.resize(BUFFER_DEFAULT_SIZE); + bdelta_pass_2_buf.resize(BUFFER_DEFAULT_SIZE); + bdelta_pass_unused.resize(BUFFER_DEFAULT_SIZE); + return b; } @@ -342,6 +360,21 @@ void bdelta_done_alg(BDelta_Instance *b) { b->matches.clear(); delete b; + + match_forward_buffer.reset(); + match_backward_buffer.reset(); + + findMatchesBuffer.clear(); + bdelta_pass_2_htable.clear(); + bdelta_pass_2_hchecksums.clear(); + bdelta_pass_2_buf.clear(); + bdelta_pass_unused.clear(); + + findMatchesBuffer.shrink_to_fit(); + bdelta_pass_2_htable.shrink_to_fit(); + bdelta_pass_2_hchecksums.shrink_to_fit(); + bdelta_pass_2_buf.shrink_to_fit(); + bdelta_pass_unused.shrink_to_fit(); } @@ -364,30 +397,20 @@ static void bdelta_pass_2(BDelta_Instance *b, unsigned blocksize, unsigned minMa numblocks += unused[i].num / blocksize; h.htablesize = std::max((unsigned)2, roundUpPowerOf2(numblocks)); - h.htable = new checksum_entry*[h.htablesize]; - if (h.htable == nullptr) - { - b->errorcode = BDELTA_MEM_ERROR; - return; - } - std::unique_ptr htable_holder(h.htable); + bdelta_pass_2_htable.resize(h.htablesize); + h.htable = bdelta_pass_2_htable.data(); - h.checksums = new checksum_entry[numblocks + 2]; - if (!h.checksums) - { - b->errorcode = BDELTA_MEM_ERROR; - return; - } - std::unique_ptr checksums_holder(h.checksums); + bdelta_pass_2_hchecksums.resize(numblocks + 2); + h.checksums = bdelta_pass_2_hchecksums.data(); h.numchecksums = 0; - std::unique_ptr buf(new Token[blocksize]); + bdelta_pass_2_buf.resize(blocksize); for (unsigned i = 0; i < numunused; ++i) { unsigned first = unused[i].p, last = unused[i].p + unused[i].num; for (unsigned loc = first; loc + blocksize <= last; loc += blocksize) { - const Token *read = b->read1(buf.get(), loc, blocksize); + const Token *read = b->read1(bdelta_pass_2_buf.data(), loc, blocksize); Hash::Value blocksum = Hash(read, blocksize).getValue(); // Adjacent checksums are never repeated. h.add(checksum_entry(blocksum, loc)); @@ -482,14 +505,14 @@ inline bool isZeroMatch(Match &m) { return m.num == 0; } void bdelta_pass(BDelta_Instance *b, unsigned blocksize, unsigned minMatchSize, unsigned maxHoleSize, unsigned flags) { // Place an empty Match at beginning so we can assume there's a Match to the left of every hole. - b->matches.push_front(Match(0, 0, 0)); + b->matches.emplace_front(0, 0, 0); // Trick for including the free range at the end. - b->matches.push_back(Match(b->data1_size, b->data2_size, 0)); - - UnusedRange *unused = new UnusedRange[b->matches.size() + 1], - *unused2 = new UnusedRange[b->matches.size() + 1]; + b->matches.emplace_back(b->data1_size, b->data2_size, 0); - std::unique_ptr unused_holder(unused), unused2_holder(unused2); + const size_t BUFFER_SIZE = b->matches.size() + 1; + bdelta_pass_unused.resize(BUFFER_SIZE * 2); + UnusedRange *unused = bdelta_pass_unused.data(), + *unused2 = unused + BUFFER_SIZE; unsigned numunused = 0, numunused2 = 0; for (std::list::iterator l = b->matches.begin(); l != b->matches.end(); ++l) From 014a92818879f5fd35fa4f401bce256531beca61 Mon Sep 17 00:00:00 2001 From: Deamhan Date: Fri, 15 Mar 2019 23:17:46 +0300 Subject: [PATCH 18/54] file.h: cleanup --- src/file.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/file.h b/src/file.h index 1dd0ee9..c723af3 100644 --- a/src/file.h +++ b/src/file.h @@ -13,7 +13,7 @@ #include #include -#define MAX_IO_BLOCK_SIZE (1024 * 1024) +static constexpr size_t MAX_IO_BLOCK_SIZE = (1024 * 1024); void fread_fixed(FILE *f, void * _buf, unsigned num_bytes); void fwrite_fixed(FILE *f, const void * _buf, unsigned num_bytes); From 74bc6b0c31e211d6bd724cdc71ece364d393346f Mon Sep 17 00:00:00 2001 From: Deamhan Date: Fri, 15 Mar 2019 23:18:52 +0300 Subject: [PATCH 19/54] bdelta.cpp: write buffer has been increased --- src/bdelta.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/bdelta.cpp b/src/bdelta.cpp index ac211de..2a6d3d0 100644 --- a/src/bdelta.cpp +++ b/src/bdelta.cpp @@ -162,7 +162,7 @@ int main(int argc, char **argv) } unsigned fp = 0; - const size_t WRITE_BUFFER_SIZE = 4096; + constexpr size_t WRITE_BUFFER_SIZE = 1024 * 1024; std::unique_ptr write_buffer(new uint8_t[WRITE_BUFFER_SIZE]); for (int i = 0; i < nummatches; ++i) { From 6ae1dd38dcd4a5a525fe473200979f333b5e3eb7 Mon Sep 17 00:00:00 2001 From: Deamhan Date: Fri, 15 Mar 2019 23:31:14 +0300 Subject: [PATCH 20/54] file.cpp has been added --- src/file.cpp | 51 +++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 51 insertions(+) create mode 100644 src/file.cpp diff --git a/src/file.cpp b/src/file.cpp new file mode 100644 index 0000000..f96db46 --- /dev/null +++ b/src/file.cpp @@ -0,0 +1,51 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#include "file.h" + +#include + +void fread_fixed(FILE *f, void * _buf, unsigned num_bytes) +{ + char * buf = (char *)_buf; + + while (num_bytes != 0) + { + unsigned block_size = num_bytes; + if (block_size > MAX_IO_BLOCK_SIZE) + block_size = MAX_IO_BLOCK_SIZE; + + size_t r = fread(buf, 1, block_size, f); + if (r != block_size) + { + const size_t BUFFER_SIZE = 512; + std::unique_ptr error_message_buffer(new char[BUFFER_SIZE]); + snprintf(error_message_buffer.get(), BUFFER_SIZE, "read error: fread_fixed(block_size=%u) != %u", block_size, (unsigned)r); + throw std::system_error(EIO, std::generic_category(), std::string(error_message_buffer.get())); + } + buf += block_size; + num_bytes -= block_size; + } +} + +void fwrite_fixed(FILE *f, const void * _buf, unsigned num_bytes) +{ + const char * buf = (const char *)_buf; + + while (num_bytes != 0) + { + unsigned block_size = std::min(num_bytes, MAX_IO_BLOCK_SIZE); + + size_t r = fwrite(buf, 1, block_size, f); + if (r != block_size) + { + const size_t BUFFER_SIZE = 512; + std::unique_ptr error_message_buffer(new char[BUFFER_SIZE]); + snprintf(error_message_buffer.get(), BUFFER_SIZE, "write error: fwrite_fixed(num_bytes=%u) != %u", block_size, (unsigned)r); + throw std::system_error(EIO, std::generic_category(), std::string(error_message_buffer.get())); + } + buf += block_size; + num_bytes -= block_size; + } +} From 15b963f4f0b3905c393da2becd9873fb4f5aeb15 Mon Sep 17 00:00:00 2001 From: Deamhan Date: Fri, 15 Mar 2019 23:40:56 +0300 Subject: [PATCH 21/54] compatibility.h has been removed --- src/compatibility.h | 18 ------------------ 1 file changed, 18 deletions(-) delete mode 100755 src/compatibility.h diff --git a/src/compatibility.h b/src/compatibility.h deleted file mode 100755 index 968d9f3..0000000 --- a/src/compatibility.h +++ /dev/null @@ -1,18 +0,0 @@ -/* This Source Code Form is subject to the terms of the Mozilla Public - * License, v. 2.0. If a copy of the MPL was not distributed with this - * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ - -#ifndef __COMPATIBILITY_H__ -#define __COMPATIBILITY_H__ - -// Fix for MSVC++, which doesn't support Variable Length Arrays. -#include -#include - -#ifdef _MSC_VER - #define STACK_ALLOC(name, type, num) type *name = (type *)alloca(sizeof(type) * num) -#else - #define STACK_ALLOC(name, type, num) type name[num] -#endif - -#endif // __COMPATIBILITY_H__ From 4d51c6ad711cb6a710aa4aa5b35ddbd7ac6ec50c Mon Sep 17 00:00:00 2001 From: Deamhan Date: Fri, 15 Mar 2019 23:58:42 +0300 Subject: [PATCH 22/54] I/O performance upgrade --- src/bdelta.cpp | 3 +++ src/bpatch.cpp | 3 +++ 2 files changed, 6 insertions(+) diff --git a/src/bdelta.cpp b/src/bdelta.cpp index 2a6d3d0..0479303 100644 --- a/src/bdelta.cpp +++ b/src/bdelta.cpp @@ -121,12 +121,15 @@ int main(int argc, char **argv) unsigned * copynum = new unsigned[nummatches + 1]; std::unique_ptr copyloc1_holder(copyloc1), copyloc2_holder(copyloc2), copynum_holder(copynum); + const size_t fout_buffer_size = 65536; + std::unique_ptr fout_buffer(new char[fout_buffer_size]); FILE *fout = fopen(argv[3], "wb"); if (fout == nullptr) { printf("couldn't open output file\n"); exit(1); } + setvbuf(fout, fout_buffer.get(), _IOFBF, fout_buffer_size); std::unique_ptr fout_holder(fout, fclose); const char * magic = "BDT"; diff --git a/src/bpatch.cpp b/src/bpatch.cpp index 218e6c6..5a401b1 100644 --- a/src/bpatch.cpp +++ b/src/bpatch.cpp @@ -102,12 +102,15 @@ int main(int argc, char **argv) } std::unique_ptr ref_holder(ref, fclose); + const size_t fout_buffer_size = 65536; + std::unique_ptr outfile_buffer(new char[fout_buffer_size]); FILE * outfile = fopen(argv[2], "wb"); if (outfile == nullptr) { printf("Error: unable to open file %s\n", argv[2]); return -1; } + setvbuf(outfile, outfile_buffer.get(), _IOFBF, fout_buffer_size); std::unique_ptr outfile_holder(outfile, fclose); for (unsigned i = 0; i < nummatches; ++i) From 05d58dc4a9483be9a825a7bb8fb4331692354bb3 Mon Sep 17 00:00:00 2001 From: Deamhan Date: Sat, 16 Mar 2019 01:04:47 +0300 Subject: [PATCH 23/54] windows: I/O performance has been improved --- src/file.cpp | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/src/file.cpp b/src/file.cpp index f96db46..548ea81 100644 --- a/src/file.cpp +++ b/src/file.cpp @@ -6,6 +6,12 @@ #include +#ifdef _MSC_VER +#define fast_fwrite _fwrite_nolock +#else +#define fast_fwrite fwrite +#endif // _MSC_VER + void fread_fixed(FILE *f, void * _buf, unsigned num_bytes) { char * buf = (char *)_buf; @@ -37,7 +43,7 @@ void fwrite_fixed(FILE *f, const void * _buf, unsigned num_bytes) { unsigned block_size = std::min(num_bytes, MAX_IO_BLOCK_SIZE); - size_t r = fwrite(buf, 1, block_size, f); + size_t r = fast_fwrite(buf, 1, block_size, f); if (r != block_size) { const size_t BUFFER_SIZE = 512; From d5e7e41beb53d4dfc2e26234d3564a032309e3bd Mon Sep 17 00:00:00 2001 From: Deamhan Date: Sat, 16 Mar 2019 01:13:55 +0300 Subject: [PATCH 24/54] file.cpp: file I/O is unlocked now (it reduces locking overhead) --- src/file.cpp | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/src/file.cpp b/src/file.cpp index 548ea81..a7b7a77 100644 --- a/src/file.cpp +++ b/src/file.cpp @@ -7,9 +7,8 @@ #include #ifdef _MSC_VER -#define fast_fwrite _fwrite_nolock -#else -#define fast_fwrite fwrite +#define fwrite_unlocked _fwrite_nolock +#define fread_unlocked _fread_nolock #endif // _MSC_VER void fread_fixed(FILE *f, void * _buf, unsigned num_bytes) @@ -22,7 +21,7 @@ void fread_fixed(FILE *f, void * _buf, unsigned num_bytes) if (block_size > MAX_IO_BLOCK_SIZE) block_size = MAX_IO_BLOCK_SIZE; - size_t r = fread(buf, 1, block_size, f); + size_t r = fread_unlocked(buf, 1, block_size, f); if (r != block_size) { const size_t BUFFER_SIZE = 512; @@ -43,7 +42,7 @@ void fwrite_fixed(FILE *f, const void * _buf, unsigned num_bytes) { unsigned block_size = std::min(num_bytes, MAX_IO_BLOCK_SIZE); - size_t r = fast_fwrite(buf, 1, block_size, f); + size_t r = fwrite_unlocked(buf, 1, block_size, f); if (r != block_size) { const size_t BUFFER_SIZE = 512; From 97a4fca8e59772a9a80b74e831377b9ddd874d72 Mon Sep 17 00:00:00 2001 From: Deamhan Date: Sat, 16 Mar 2019 17:31:21 +0300 Subject: [PATCH 25/54] --all-in-ram performance has been improved --- src/bdelta.cpp | 14 ++++---------- src/libbdelta.cpp | 21 +++++++++++---------- 2 files changed, 15 insertions(+), 20 deletions(-) diff --git a/src/bdelta.cpp b/src/bdelta.cpp index 0479303..51b6a98 100644 --- a/src/bdelta.cpp +++ b/src/bdelta.cpp @@ -19,11 +19,6 @@ static const void *f_read(void *f, void *buf, unsigned place, unsigned num) return buf; } -static inline const void * m_read(void *f, void * /*buf*/, unsigned place, unsigned /*num*/) -{ - return (const char*)f + place; -} - static void my_pass(BDelta_Instance *b, unsigned blocksize, unsigned minMatchSize, unsigned flags) { bdelta_pass(b, blocksize, minMatchSize, 0, flags); @@ -34,13 +29,12 @@ int main(int argc, char **argv) { try { - bool all_ram_mode = false; std::unique_ptr m1; std::unique_ptr m2; - if (argc > 1 && strcmp(argv[1], "--all-in-ram") == 0) + const bool all_ram_mode = (argc > 1 && strcmp(argv[1], "--all-in-ram") == 0); + if (all_ram_mode) { - all_ram_mode = true; --argc; ++argv; } @@ -82,7 +76,7 @@ int main(int argc, char **argv) fread_fixed(f1, m1.get(), size); fread_fixed(f2, m2.get(), size2); - b = bdelta_init_alg(size, size2, m_read, m1.get(), m2.get(), 1); + b = bdelta_init_alg(size, size2, nullptr, m1.get(), m2.get(), 1); } else b = bdelta_init_alg(size, size2, f_read, f1, f2, 1); @@ -173,7 +167,7 @@ int main(int argc, char **argv) while (num > 0) { unsigned towrite = std::min(num, WRITE_BUFFER_SIZE); - const void * block = all_ram_mode ? m_read(m2.get(), write_buffer.get(), fp, towrite) : f_read(f2, write_buffer.get(), fp, towrite); + const void * block = all_ram_mode ? (const void *)((char*)m2.get() + fp) : f_read(f2, write_buffer.get(), fp, towrite); fwrite_fixed(fout, block, towrite); num -= towrite; fp += towrite; diff --git a/src/libbdelta.cpp b/src/libbdelta.cpp index c62d143..db61d9b 100644 --- a/src/libbdelta.cpp +++ b/src/libbdelta.cpp @@ -48,7 +48,7 @@ struct Match struct _BDelta_Instance { - bdelta_readCallback cb; + const bdelta_readCallback cb; void *handle1, *handle2; unsigned data1_size, data2_size; std::list matches; @@ -56,13 +56,20 @@ struct _BDelta_Instance int access_int; int errorcode; + _BDelta_Instance(bdelta_readCallback _cb, void * _handle1, void * _handle2, + unsigned _data1_size, unsigned _data2_size) + : cb(_cb), handle1(_handle1), handle2(_handle2) + , data1_size(_data1_size), data2_size(_data2_size) + , access_int(-1), errorcode(0) + {} + const Token *read1(void *buf, unsigned place, unsigned num) { - return (const Token*)cb(handle1, buf, place, num); + return (cb == nullptr)? ((const Token*)((char *)handle1 + place)) : ((const Token*)cb(handle1, buf, place, num)); } const Token *read2(void *buf, unsigned place, unsigned num) { - return (const Token*)cb(handle2, buf, place, num); + return (cb == nullptr) ? ((const Token*)((char *)handle2 + place)) : ((const Token*)cb(handle2, buf, place, num)); } }; @@ -334,15 +341,9 @@ BDelta_Instance *bdelta_init_alg(unsigned data1_size, unsigned data2_size, printf("Error: BDelta library compiled for token size of %lu.\n", (unsigned long)sizeof (Token)); return 0; } - BDelta_Instance *b = new BDelta_Instance; + BDelta_Instance *b = new BDelta_Instance(cb, handle1, handle2, data1_size, data2_size); if (!b) return nullptr; - b->data1_size = data1_size; - b->data2_size = data2_size; - b->cb = cb; - b->handle1 = handle1; - b->handle2 = handle2; - b->access_int = -1; match_forward_buffer.reset(new Token[TOKEN_BUFFER_SIZE * 2]); match_backward_buffer.reset(new Token[TOKEN_BUFFER_SIZE * 2]); From 88a95bf406f83e91a1172703541b096ee905220b Mon Sep 17 00:00:00 2001 From: Deamhan Date: Sat, 16 Mar 2019 17:32:27 +0300 Subject: [PATCH 26/54] outdated & unused files have been removed --- README | 12 -- makefiles/vs90/bdelta_python.vcproj | 235 ---------------------------- setup.py | 22 --- src/Makefile | 45 ------ src/bdelta.pyx | 61 -------- test/library.cpp | 31 ---- test/py_bindings.py | 18 --- test/suite.py | 40 ----- version.py | 120 -------------- 9 files changed, 584 deletions(-) delete mode 100644 makefiles/vs90/bdelta_python.vcproj delete mode 100644 setup.py delete mode 100644 src/Makefile delete mode 100644 src/bdelta.pyx delete mode 100644 test/library.cpp delete mode 100644 test/py_bindings.py delete mode 100644 test/suite.py delete mode 100644 version.py diff --git a/README b/README index 3b6ede3..036850b 100644 --- a/README +++ b/README @@ -34,18 +34,6 @@ is designed to give a great deal of control over each pass. This functionality is not well-documented, so contact me if you need help. I hope to add more comments to the code in future releases. - -Other Bindings -============== - -A simple interface for accessing the API with Python is included. This -interface is kept up-to-date since I do a lot of work in Python these days. To -istall it, just type "python build.py install" in the root folder. You'll need -Cython to build/install, but Cython is not required after installation. - -See test/py_bindings.py for a usage example. - - Delta File Format ================= diff --git a/makefiles/vs90/bdelta_python.vcproj b/makefiles/vs90/bdelta_python.vcproj deleted file mode 100644 index 6fee8e2..0000000 --- a/makefiles/vs90/bdelta_python.vcproj +++ /dev/null @@ -1,235 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - diff --git a/setup.py b/setup.py deleted file mode 100644 index 3a792a1..0000000 --- a/setup.py +++ /dev/null @@ -1,22 +0,0 @@ -from distutils.core import setup, Extension -from Cython.Distutils import build_ext -from version import get_git_version -import sys - -bdelta_module = Extension( - "bdelta", - ["src/bdelta.pyx", "src/libbdelta.cpp"], - define_macros = [('TOKEN_SIZE', '2')], - extra_compile_args = ['/EHsc'] if sys.platform == 'win32' else None -) - -setup( - name = 'BDelta', - version = get_git_version(), - description = 'Python Bindings for BDelta', - author = 'John Whitney', - author_email = 'jjw@deltup.org', - url = 'http://bdelta.org', - cmdclass = {'build_ext': build_ext}, - ext_modules = [bdelta_module] -) diff --git a/src/Makefile b/src/Makefile deleted file mode 100644 index 65fb88d..0000000 --- a/src/Makefile +++ /dev/null @@ -1,45 +0,0 @@ -DESTDIR= -PREFIX ?= /usr -BINDIR ?= $(PREFIX)/bin -LIBDIR ?= ${PREFIX}/lib -CXXFLAGS += -O2 - -ifeq ($(shell uname -s),Darwin) - SHAREDLIB := libbdelta.dylib -else - SHAREDLIB := libbdelta.so -endif - -ALL_TARGETS = bpatch $(SHAREDLIB) bdelta - -all: $(ALL_TARGETS) - -libbdelta.a: libbdelta.cpp compatibility.h checksum.h file.h - $(CXX) -c $(CXXFLAGS) $< -o $<.o - $(AR) rs $@ $<.o - -libbdelta.so: libbdelta.cpp compatibility.h checksum.h file.h - $(CXX) -shared -fPIC $(CXXFLAGS) $(LDFLAGS) $< -o $@ - -libbdelta.dylib: libbdelta.cpp compatibility.h checksum.h file.h - $(CXX) -dynamiclib $(CXXFLAGS) $< -o $@ - -bdelta: bdelta.cpp bdelta.h compatibility.h file.h $(SHAREDLIB) - $(CXX) $< -o $@ $(CXXFLAGS) $(LDFLAGS) -L. -lbdelta - -bpatch: bpatch.cpp compatibility.h bdelta.h file.h - -% : %.cpp - $(CXX) $< -o $@ $(CXXFLAGS) $(LDFLAGS) - -install: libbdelta.so bdelta bpatch - mkdir -p $(DESTDIR)$(BINDIR) - mkdir -p $(DESTDIR)$(LIBDIR) - install -m 755 libbdelta.so $(DESTDIR)$(LIBDIR) - install -m 755 bdelta $(DESTDIR)$(BINDIR) - install -m 755 bpatch $(DESTDIR)$(BINDIR) - -clean: - -rm $(ALL_TARGETS) - -.PHONY: clean diff --git a/src/bdelta.pyx b/src/bdelta.pyx deleted file mode 100644 index 8affe78..0000000 --- a/src/bdelta.pyx +++ /dev/null @@ -1,61 +0,0 @@ -# This Source Code Form is subject to the terms of the Mozilla Public -# License, v. 2.0. If a copy of the MPL was not distributed with this -# file, You can obtain one at http://mozilla.org/MPL/2.0/. - -cdef extern from "bdelta.h": - ctypedef struct BDelta_Instance: - pass - - ctypedef void *const_void_ptr "const void*" - ctypedef void *(*bdelta_readCallback)(void *handle, void *buf, unsigned place, unsigned num) - BDelta_Instance *bdelta_init_alg(unsigned data1_size, unsigned data2_size, - bdelta_readCallback cb, void *handle1, void *handle2, - unsigned tokenSize) - void bdelta_done_alg(BDelta_Instance *b) - - void bdelta_pass(BDelta_Instance *b, unsigned blockSize, unsigned minMatchSize, unsigned maxHoleSize, unsigned flags) - - void bdelta_swap_inputs(BDelta_Instance *b) - void bdelta_clean_matches(BDelta_Instance *b, unsigned flags) - - unsigned bdelta_numMatches(BDelta_Instance *b) - - void bdelta_getMatch(BDelta_Instance *b, unsigned matchNum, - unsigned *p1, unsigned *p2, unsigned *num) - - int bdelta_getError(BDelta_Instance *b) - void bdelta_showMatches(BDelta_Instance *b) - - cdef enum PassFlags: - BDELTA_GLOBAL, - BDELTA_SIDES_ORDERED - cdef enum CleanFlags: - BDELTA_REMOVE_OVERLAP - -cdef const_void_ptr readCallback(void *handle, void *buf, unsigned place, unsigned num): - cdef char *str = handle - return str + ((place + 1) * 2); - -cdef class BDelta: - cdef BDelta_Instance *_b - cdef bytes str1, str2 - - def __cinit__(self, str1, str2): - self.str1 = str1.encode('UTF-16') - self.str2 = str2.encode('UTF-16') - self._b = bdelta_init_alg(len(str1), len(str2), readCallback, self.str1, self.str2, 2) - - def __dealloc__(self): - self.str1 = None - self.str2 = None - bdelta_done_alg(self._b) - - def b_pass(self, blockSize, minMatchSize, maxHoleSize, globalScope = False, sidesOrdered = False): - bdelta_pass(self._b, blockSize, minMatchSize, maxHoleSize, - (BDELTA_GLOBAL if globalScope else 0) | (BDELTA_SIDES_ORDERED if sidesOrdered else 0)) - - def matches(self): - cdef unsigned p1, p2, num - for i in xrange(bdelta_numMatches(self._b)): - bdelta_getMatch(self._b, i, &p1, &p2, &num) - yield (int(p1), int(p2), int(num)) \ No newline at end of file diff --git a/test/library.cpp b/test/library.cpp deleted file mode 100644 index 9008fc6..0000000 --- a/test/library.cpp +++ /dev/null @@ -1,31 +0,0 @@ -/* Any copyright is dedicated to the Public Domain. - * http://creativecommons.org/publicdomain/zero/1.0/ */ - -// "Hello BDelta!" - -#include "bdelta.h" -#include -#include - -const int smallestMatch = 8; -char *a = "abcdefghijklmnopqrstuvwxyz", *b = "abcdefghijklmnopqrstuvwxyz"; -void *a_read(unsigned place, unsigned num) { - return (char*)(a + place); -} - -void *b_read(unsigned place, unsigned num) { - return (char*)(b + place); -} -main() { - void *bi = bdelta_init_alg(strlen(a), strlen(b), a_read, b_read); - int nummatches; - for (int i = 64; i >= smallestMatch; i/=2) - nummatches = bdelta_pass(bi, i); - for (int i = 0; i < nummatches; ++i) { - unsigned p1, p2, num; - bdelta_getMatch(bi, i, &p1, &p2, &num); - printf("Got match\n"); - } - -} - diff --git a/test/py_bindings.py b/test/py_bindings.py deleted file mode 100644 index f600cd7..0000000 --- a/test/py_bindings.py +++ /dev/null @@ -1,18 +0,0 @@ -# Any copyright is dedicated to the Public Domain. -# http://creativecommons.org/publicdomain/zero/1.0/ - -from bdelta import BDelta - -a = u"The quick brown fox jumped over the lazy dog" -b = u"The quick drowned fox jumped over the lazy dog" - -b = BDelta(a, b) - -b.b_pass(13, 27, 0) # Find all matches that are at least 27 chars long -print list(b.matches()) # [(15, 17, 29)]) - -b.b_pass(3, 5, 0) # Find all matches that are at least 5 chars long -print list(b.matches()) # [(0, 0, 10), (15, 17, 29)] - -b.b_pass(2, 3, 0) # Find all matches that are at least 3 chars long -print list(b.matches()) # [(0, 0, 10), (11, 11, 4), (15, 17, 29)] diff --git a/test/suite.py b/test/suite.py deleted file mode 100644 index 5eb821b..0000000 --- a/test/suite.py +++ /dev/null @@ -1,40 +0,0 @@ -# Any copyright is dedicated to the Public Domain. -# http://creativecommons.org/publicdomain/zero/1.0/ - -# First argument should be a test suite folder -# Such folders contain subfolders with files 1 (old -# version), and 2 (new version) of data that you -# want to test with the delta algorithm - -import sys -import os -import filecmp -import time - -basePath = sys.argv[1] -initialTime = time.time() - -def isValidFile(x): - return not x.startswith('.') and os.path.isdir(os.path.join(basePath, x)) - -testFolders = filter(isValidFile, os.listdir(basePath)) -print "Running tests:", testFolders -print "BDelta 0.10 file sizes:" -for test in testFolders: - def path(fName): - return os.path.join(basePath, test, fName) - t1 = time.time() - os.system("../src/bdelta %s %s %s" % (path("1"), path("2"), path("patch.bdelta"))) - t2 = time.time() - os.system("../src/bpatch %s %s %s" % (path("1"), path("2.new"), path("patch.bdelta"))) - print ("%20s:" % test), - if filecmp.cmp(path("2"), path("2.new"), shallow=False): - print "uncompressed=%s\t" % os.stat(path("patch.bdelta")).st_size, - os.system("bzip2 %s" % path("patch.bdelta")) - print "bz2=%s\t" % os.stat(path("patch.bdelta.bz2")).st_size, - print "time=%f" % (t2-t1) - else: - print "ERROR" - os.remove(path("patch.bdelta.bz2")) - os.remove(path("2.new")) -print "Total time:", (time.time()-initialTime) diff --git a/version.py b/version.py deleted file mode 100644 index d8ebfee..0000000 --- a/version.py +++ /dev/null @@ -1,120 +0,0 @@ -# -*- coding: utf-8 -*- -# Author: Douglas Creager -# This file is placed into the public domain. - -# Calculates the current version number. If possible, this is the -# output of “git describe”, modified to conform to the versioning -# scheme that setuptools uses. If “git describe” returns an error -# (most likely because we're in an unpacked copy of a release tarball, -# rather than in a git working copy), then we fall back on reading the -# contents of the RELEASE-VERSION file. -# -# To use this script, simply import it your setup.py file, and use the -# results of get_git_version() as your package version: -# -# from version import * -# -# setup( -# version=get_git_version(), -# . -# . -# . -# ) -# -# This will automatically update the RELEASE-VERSION file, if -# necessary. Note that the RELEASE-VERSION file should *not* be -# checked into git; please add it to your top-level .gitignore file. -# -# You'll probably want to distribute the RELEASE-VERSION file in your -# sdist tarballs; to do this, just create a MANIFEST.in file that -# contains the following line: -# -# include RELEASE-VERSION - -__all__ = ("get_git_version") - -from subprocess import Popen, PIPE - - -def call_git_describe(abbrev=4): - try: - p = Popen(['git', 'describe', '--abbrev=%d' % abbrev], - stdout=PIPE, stderr=PIPE) - p.stderr.close() - line = p.stdout.readlines()[0] - return line.strip() - - except: - return None - - -def read_release_version(): - try: - f = open("RELEASE-VERSION", "r") - - try: - version = f.readlines()[0] - return version.strip() - - finally: - f.close() - - except: - return None - - -def write_release_version(version): - f = open("RELEASE-VERSION", "w") - f.write("%s\n" % version) - f.close() - - -def get_git_version(abbrev=4): - # Read in the version that's currently in RELEASE-VERSION. - - release_version = read_release_version() - - # First try to get the current version using “git describe”. - - version = call_git_describe(abbrev) - - #adapt to PEP 386 compatible versioning scheme - version = pep386adapt(version) - - # If that doesn't work, fall back on the value that's in - # RELEASE-VERSION. - - if version is None: - version = release_version - - # If we still don't have anything, that's an error. - - if version is None: - raise ValueError("Cannot find the version number!") - - # If the current version is different from what's in the - # RELEASE-VERSION file, update the file to be current. - - if version != release_version: - write_release_version(version) - - # Finally, return the current version. - - return version - - -def pep386adapt(version): - if version is not None: - if '-' in version: - # adapt git-describe version to be in line with PEP 386 - parts = version.split('-') - parts[-2] = 'post'+parts[-2] - version = '.'.join(parts[:-1]) - elif version[0] == 'v': - version = version[1:] - - return version - - -if __name__ == "__main__": - print get_git_version() \ No newline at end of file From b6dcb4380e5ec738f8bcf351dd42a9bd78bc68ad Mon Sep 17 00:00:00 2001 From: Deamhan Date: Sat, 16 Mar 2019 17:45:59 +0300 Subject: [PATCH 27/54] CMakeLists.txt: GCC32 option has been added --- makefiles/CMakeLists.txt | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/makefiles/CMakeLists.txt b/makefiles/CMakeLists.txt index fe41a09..c350ed4 100644 --- a/makefiles/CMakeLists.txt +++ b/makefiles/CMakeLists.txt @@ -4,6 +4,7 @@ project(bdelta) set(CMAKE_CXX_STANDARD 17) option(BDELTA_PGO "build using PGO instead of PGI (MSVC only)" "OFF") +option(GCC32 "build 32 bit executable instead of 64 bit (GCC only)" "OFF") IF(MSVC) set(CMAKE_CONFIGURATION_TYPES "Debug;Release;PGO" CACHE STRING "" FORCE) @@ -143,10 +144,15 @@ ELSE() SET(CMAKE_EXE_LINKER_FLAGS_RELEASE "${CMAKE_EXE_LINKER_FLAGS} -s") SET(CMAKE_SHARED_LINKER_FLAGS_RELEASE "${CMAKE_SHARED_LINKER_FLAGS} -s") + + SET(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -static-libgcc -static-libstdc++") + SET(CMAKE_SHARED_LINKER_FLAGS "${CMAKE_SHARED_LINKER_FLAGS} -static-libgcc -static-libstdc++") +IF (GCC32) add_compile_options (-m32) - SET(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -m32 -static-libgcc -static-libstdc++") - SET(CMAKE_SHARED_LINKER_FLAGS "${CMAKE_SHARED_LINKER_FLAGS} -m32 -static-libgcc -static-libstdc++") + SET(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -m32") + SET(CMAKE_SHARED_LINKER_FLAGS "${CMAKE_SHARED_LINKER_FLAGS} -m32") +ENDIF() ENDIF() string(TIMESTAMP BUILD_YEAR "%Y") From 3e7f0af1d3197f4da970724e174c2af523645dc8 Mon Sep 17 00:00:00 2001 From: Deamhan Date: Sat, 16 Mar 2019 17:47:02 +0300 Subject: [PATCH 28/54] .gitignore --- .gitignore | 10 ++-------- 1 file changed, 2 insertions(+), 8 deletions(-) diff --git a/.gitignore b/.gitignore index 64a2f62..25c1a25 100644 --- a/.gitignore +++ b/.gitignore @@ -1,9 +1,3 @@ -*.pyc -*.dylib - -build -RELEASE-VERSION -src/bdelta -src/bpatch -src/bdelta.c +*vs*/ +*cppcheck* From 3978f861c7aaa9ba01625519a1d400396339b343 Mon Sep 17 00:00:00 2001 From: Deamhan Date: Sun, 17 Mar 2019 07:25:36 +0300 Subject: [PATCH 29/54] explicit buffer allocations have been removed --- src/bdelta.cpp | 5 ++--- src/bpatch.cpp | 5 ++--- 2 files changed, 4 insertions(+), 6 deletions(-) diff --git a/src/bdelta.cpp b/src/bdelta.cpp index 51b6a98..e7db031 100644 --- a/src/bdelta.cpp +++ b/src/bdelta.cpp @@ -115,15 +115,14 @@ int main(int argc, char **argv) unsigned * copynum = new unsigned[nummatches + 1]; std::unique_ptr copyloc1_holder(copyloc1), copyloc2_holder(copyloc2), copynum_holder(copynum); - const size_t fout_buffer_size = 65536; - std::unique_ptr fout_buffer(new char[fout_buffer_size]); FILE *fout = fopen(argv[3], "wb"); if (fout == nullptr) { printf("couldn't open output file\n"); exit(1); } - setvbuf(fout, fout_buffer.get(), _IOFBF, fout_buffer_size); + constexpr size_t fout_buffer_size = 256 * 1024; + setvbuf(fout, nullptr, _IOFBF, fout_buffer_size); std::unique_ptr fout_holder(fout, fclose); const char * magic = "BDT"; diff --git a/src/bpatch.cpp b/src/bpatch.cpp index 5a401b1..52d32f8 100644 --- a/src/bpatch.cpp +++ b/src/bpatch.cpp @@ -102,15 +102,14 @@ int main(int argc, char **argv) } std::unique_ptr ref_holder(ref, fclose); - const size_t fout_buffer_size = 65536; - std::unique_ptr outfile_buffer(new char[fout_buffer_size]); FILE * outfile = fopen(argv[2], "wb"); if (outfile == nullptr) { printf("Error: unable to open file %s\n", argv[2]); return -1; } - setvbuf(outfile, outfile_buffer.get(), _IOFBF, fout_buffer_size); + constexpr size_t outfile_buffer_size = 256 * 1024; + setvbuf(outfile, nullptr, _IOFBF, fout_buffer_size); std::unique_ptr outfile_holder(outfile, fclose); for (unsigned i = 0; i < nummatches; ++i) From 59a97c454b9a8f74a2d4b49769bfdc99e9eb5da2 Mon Sep 17 00:00:00 2001 From: Deamhan Date: Sun, 17 Mar 2019 07:26:44 +0300 Subject: [PATCH 30/54] file.cpp: some cleanup --- src/file.cpp | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/src/file.cpp b/src/file.cpp index a7b7a77..0a4afe0 100644 --- a/src/file.cpp +++ b/src/file.cpp @@ -17,9 +17,7 @@ void fread_fixed(FILE *f, void * _buf, unsigned num_bytes) while (num_bytes != 0) { - unsigned block_size = num_bytes; - if (block_size > MAX_IO_BLOCK_SIZE) - block_size = MAX_IO_BLOCK_SIZE; + unsigned block_size = std::min(num_bytes, MAX_IO_BLOCK_SIZE); size_t r = fread_unlocked(buf, 1, block_size, f); if (r != block_size) From ffd96494fb3d421f8a3bba127d0c3104d53978f5 Mon Sep 17 00:00:00 2001 From: Deamhan Date: Sun, 17 Mar 2019 08:56:07 +0300 Subject: [PATCH 31/54] bpatch.cpp: mistake has been fixed --- src/bpatch.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/bpatch.cpp b/src/bpatch.cpp index 52d32f8..4f0c49c 100644 --- a/src/bpatch.cpp +++ b/src/bpatch.cpp @@ -109,7 +109,7 @@ int main(int argc, char **argv) return -1; } constexpr size_t outfile_buffer_size = 256 * 1024; - setvbuf(outfile, nullptr, _IOFBF, fout_buffer_size); + setvbuf(outfile, nullptr, _IOFBF, outfile_buffer_size); std::unique_ptr outfile_holder(outfile, fclose); for (unsigned i = 0; i < nummatches; ++i) From f78caa752f3d9f953abc842c6fb428fa65b8d676 Mon Sep 17 00:00:00 2001 From: Deamhan Date: Mon, 18 Mar 2019 18:21:16 +0300 Subject: [PATCH 32/54] cleanup --- src/bdelta.cpp | 47 ++++++++++++++++++++++---------------------- src/bpatch.cpp | 53 +++++++++++++++++++++++++++----------------------- 2 files changed, 52 insertions(+), 48 deletions(-) diff --git a/src/bdelta.cpp b/src/bdelta.cpp index e7db031..bbcffda 100644 --- a/src/bdelta.cpp +++ b/src/bdelta.cpp @@ -51,21 +51,21 @@ int main(int argc, char **argv) } unsigned size = (unsigned)getLenOfFile(argv[1]); unsigned size2 = (unsigned)getLenOfFile(argv[2]); - FILE * f1 = fopen(argv[1], "rb"); - if (f1 == nullptr) + std::unique_ptr f1(fopen(argv[1], "rb"), fclose); + if (!f1) { printf("unable to open file %s\n", argv[1]); exit(1); } - std::unique_ptr f1_holder(f1, fclose); + - FILE * f2 = fopen(argv[2], "rb"); - if (f2 == nullptr) + std::unique_ptr f2(fopen(argv[2], "rb"), fclose); + if (!f2) { printf("unable to open file %s\n", argv[2]); exit(1); } - std::unique_ptr f2_holder(f2, fclose); + BDelta_Instance * b = nullptr; @@ -73,13 +73,13 @@ int main(int argc, char **argv) { m1.reset(new char[size]); m2.reset(new char[size2]); - fread_fixed(f1, m1.get(), size); - fread_fixed(f2, m2.get(), size2); + fread_fixed(f1.get(), m1.get(), size); + fread_fixed(f2.get(), m2.get(), size2); b = bdelta_init_alg(size, size2, nullptr, m1.get(), m2.get(), 1); } else - b = bdelta_init_alg(size, size2, f_read, f1, f2, 1); + b = bdelta_init_alg(size, size2, f_read, f1.get(), f2.get(), 1); std::unique_ptr b_holder(b, bdelta_done_alg); @@ -115,25 +115,24 @@ int main(int argc, char **argv) unsigned * copynum = new unsigned[nummatches + 1]; std::unique_ptr copyloc1_holder(copyloc1), copyloc2_holder(copyloc2), copynum_holder(copynum); - FILE *fout = fopen(argv[3], "wb"); - if (fout == nullptr) + std::unique_ptr fout(fopen(argv[3], "wb"), fclose); + if (!fout) { printf("couldn't open output file\n"); exit(1); } constexpr size_t fout_buffer_size = 256 * 1024; - setvbuf(fout, nullptr, _IOFBF, fout_buffer_size); - std::unique_ptr fout_holder(fout, fclose); + setvbuf(fout.get(), nullptr, _IOFBF, fout_buffer_size); const char * magic = "BDT"; - fwrite_fixed(fout, magic, 3); + fwrite_fixed(fout.get(), magic, 3); unsigned short version = 1; - write_word(fout, version); + write_word(fout.get(), version); unsigned char intsize = 4; - fwrite_fixed(fout, &intsize, 1); - write_dword(fout, size); - write_dword(fout, size2); - write_dword(fout, nummatches); + fwrite_fixed(fout.get(), &intsize, 1); + write_dword(fout.get(), size); + write_dword(fout.get(), size2); + write_dword(fout.get(), nummatches); unsigned lastp1 = 0, lastp2 = 0; for (int i = 0; i < nummatches; ++i) @@ -141,11 +140,11 @@ int main(int argc, char **argv) unsigned p1 = 0, p2 = 0, num = 0; bdelta_getMatch(b, i, &p1, &p2, &num); copyloc1[i] = p1 - lastp1; - write_dword(fout, copyloc1[i]); + write_dword(fout.get(), copyloc1[i]); copyloc2[i] = p2 - lastp2; - write_dword(fout, copyloc2[i]); + write_dword(fout.get(), copyloc2[i]); copynum[i] = num; - write_dword(fout, copynum[i]); + write_dword(fout.get(), copynum[i]); lastp1 = p1 + num; lastp2 = p2 + num; } @@ -166,8 +165,8 @@ int main(int argc, char **argv) while (num > 0) { unsigned towrite = std::min(num, WRITE_BUFFER_SIZE); - const void * block = all_ram_mode ? (const void *)((char*)m2.get() + fp) : f_read(f2, write_buffer.get(), fp, towrite); - fwrite_fixed(fout, block, towrite); + const void * block = all_ram_mode ? (const void *)((char*)m2.get() + fp) : f_read(f2.get(), write_buffer.get(), fp, towrite); + fwrite_fixed(fout.get(), block, towrite); num -= towrite; fp += towrite; } diff --git a/src/bpatch.cpp b/src/bpatch.cpp index 4f0c49c..2cf67ad 100644 --- a/src/bpatch.cpp +++ b/src/bpatch.cpp @@ -9,6 +9,11 @@ #include "file.h" +#ifdef _MSC_VER +#define fread_unlocked _fread_nolock +#define fwrite_unlocked _fwrite_nolock +#endif + static bool copy_bytes_to_file(FILE *infile, FILE *outfile, unsigned numleft) { const size_t BUFFER_SIZE = 65536; @@ -16,13 +21,13 @@ static bool copy_bytes_to_file(FILE *infile, FILE *outfile, unsigned numleft) while (numleft != 0) { const size_t len = std::min(BUFFER_SIZE, numleft); - size_t numread = fread(buf.get(), 1, len, infile); + size_t numread = fread_unlocked(buf.get(), 1, len, infile); if (len != numread) { printf("Could not read data\n"); return false; } - if (fwrite(buf.get(), 1, numread, outfile) != numread) + if (fwrite_unlocked(buf.get(), 1, numread, outfile) != numread) { printf("Could not write temporary data. Possibly out of space\n"); return false; @@ -50,31 +55,31 @@ int main(int argc, char **argv) return 1; } - FILE *patchfile = fopen(argv[3], "rb"); + std::unique_ptr patchfile(fopen(argv[3], "rb"), fclose); char magic[3]; - fread_fixed(patchfile, magic, 3); + fread_fixed(patchfile.get(), magic, 3); if (strncmp(magic, "BDT", 3)) { printf("Given file is not a recognized patchfile\n"); return 1; } - unsigned short version = read_word(patchfile); + unsigned short version = read_word(patchfile.get()); if (version != 1) { printf("unsupported patch version\n"); return 1; } char intsize; - fread_fixed(patchfile, &intsize, 1); + fread_fixed(patchfile.get(), &intsize, 1); if (intsize != 4) { printf("unsupported file pointer size\n"); return 1; } - /*unsigned size1 =*/ read_dword(patchfile); - unsigned size2 = read_dword(patchfile); + /*unsigned size1 =*/ read_dword(patchfile.get()); + unsigned size2 = read_dword(patchfile.get()); - unsigned nummatches = read_dword(patchfile); + unsigned nummatches = read_dword(patchfile.get()); unsigned * copyloc1 = new unsigned[nummatches + 1]; unsigned * copyloc2 = new unsigned[nummatches + 1]; @@ -83,47 +88,47 @@ int main(int argc, char **argv) for (unsigned i = 0; i < nummatches; ++i) { - copyloc1[i] = read_dword(patchfile); - copyloc2[i] = read_dword(patchfile); - copynum[i] = read_dword(patchfile); + copyloc1[i] = read_dword(patchfile.get()); + copyloc2[i] = read_dword(patchfile.get()); + copynum[i] = read_dword(patchfile.get()); size2 -= copyloc2[i] + copynum[i]; } - if (size2) { + if (size2) + { copyloc1[nummatches] = 0; copynum[nummatches] = 0; copyloc2[nummatches] = size2; ++nummatches; } - FILE * ref = fopen(argv[1], "rb"); - if (ref == nullptr) + std::unique_ptr ref(fopen(argv[1], "rb"), fclose); + if (!ref) { printf("Error: unable to open file %s\n", argv[1]); return -1; } - std::unique_ptr ref_holder(ref, fclose); + - FILE * outfile = fopen(argv[2], "wb"); - if (outfile == nullptr) + std::unique_ptr outfile(fopen(argv[2], "wb"), fclose); + if (!outfile) { printf("Error: unable to open file %s\n", argv[2]); return -1; } constexpr size_t outfile_buffer_size = 256 * 1024; - setvbuf(outfile, nullptr, _IOFBF, outfile_buffer_size); - std::unique_ptr outfile_holder(outfile, fclose); - + setvbuf(outfile.get(), nullptr, _IOFBF, outfile_buffer_size); + for (unsigned i = 0; i < nummatches; ++i) { - if (!copy_bytes_to_file(patchfile, outfile, copyloc2[i])) + if (!copy_bytes_to_file(patchfile.get(), outfile.get(), copyloc2[i])) { printf("Error: patchfile is truncated\n"); return -1; } int copyloc = copyloc1[i]; - fseek(ref, copyloc, SEEK_CUR); + fseek(ref.get(), copyloc, SEEK_CUR); - if (!copy_bytes_to_file(ref, outfile, copynum[i])) + if (!copy_bytes_to_file(ref.get(), outfile.get(), copynum[i])) { printf("Error while copying from reference file\n"); return -1; From 199fd458e0e95c3a1b10e90e0cc5e781c8f644cf Mon Sep 17 00:00:00 2001 From: Deamhan Date: Mon, 18 Mar 2019 18:24:15 +0300 Subject: [PATCH 33/54] Checksums_Instance initialization has been moved to constructor --- src/libbdelta.cpp | 29 ++++++++++++++--------------- 1 file changed, 14 insertions(+), 15 deletions(-) diff --git a/src/libbdelta.cpp b/src/libbdelta.cpp index db61d9b..2b7683c 100644 --- a/src/libbdelta.cpp +++ b/src/libbdelta.cpp @@ -75,19 +75,23 @@ struct _BDelta_Instance struct Checksums_Instance { - unsigned blocksize; - unsigned htablesize; - checksum_entry **htable; // Points to first match in checksums - checksum_entry *checksums; // Sorted list of all checksums + const unsigned blocksize; + const unsigned htablesize; + checksum_entry ** const htable; // Points to first match in checksums + checksum_entry * const checksums; // Sorted list of all checksums unsigned numchecksums; - explicit Checksums_Instance(int _blocksize) : blocksize(_blocksize) {} + Checksums_Instance(unsigned _blocksize, unsigned _htablesize, checksum_entry ** _htable, checksum_entry * _checksums) + : blocksize(_blocksize), htablesize(_htablesize), htable(_htable) + , checksums(_checksums), numchecksums(0) + { + memset(htable, 0, htablesize * sizeof(htable[0])); + } template void add(T&& ck) { - checksums[numchecksums] = std::forward(ck); - ++numchecksums; + checksums[++numchecksums] = std::forward(ck); } unsigned tableIndex(Hash::Value hashValue) { @@ -390,21 +394,17 @@ static unsigned roundUpPowerOf2(unsigned v) static void bdelta_pass_2(BDelta_Instance *b, unsigned blocksize, unsigned minMatchSize, UnusedRange *unused, unsigned numunused, UnusedRange *unused2, unsigned numunused2) { - Checksums_Instance h(blocksize); b->access_int = -1; unsigned numblocks = 0; for (unsigned i = 0; i < numunused; ++i) numblocks += unused[i].num / blocksize; - h.htablesize = std::max((unsigned)2, roundUpPowerOf2(numblocks)); - bdelta_pass_2_htable.resize(h.htablesize); - h.htable = bdelta_pass_2_htable.data(); - + bdelta_pass_2_htable.resize(std::max((unsigned)2, roundUpPowerOf2(numblocks))); bdelta_pass_2_hchecksums.resize(numblocks + 2); - h.checksums = bdelta_pass_2_hchecksums.data(); - h.numchecksums = 0; + Checksums_Instance h(blocksize, bdelta_pass_2_htable.size(), bdelta_pass_2_htable.data(), bdelta_pass_2_hchecksums.data()); + bdelta_pass_2_buf.resize(blocksize); for (unsigned i = 0; i < numunused; ++i) { @@ -437,7 +437,6 @@ static void bdelta_pass_2(BDelta_Instance *b, unsigned blocksize, unsigned minMa h.checksums[h.numchecksums].loc = 0; // So we'll just read from the beginning of the file to prevent crashes. h.checksums[h.numchecksums + 1].cksum = 0; - memset(h.htable, 0, h.htablesize * sizeof(h.htable[0])); for (int i = h.numchecksums - 1; i >= 0; --i) h.htable[h.tableIndex(h.checksums[i].cksum)] = &h.checksums[i]; From c29009223e023a314a7e05d24034560e8710a68d Mon Sep 17 00:00:00 2001 From: Deamhan Date: Mon, 18 Mar 2019 20:59:00 +0300 Subject: [PATCH 34/54] checksum.h: some cleanup --- src/checksum.h | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/src/checksum.h b/src/checksum.h index 7fcc5cb..af62565 100755 --- a/src/checksum.h +++ b/src/checksum.h @@ -36,15 +36,16 @@ struct Hash void advance_add(Token in) { - value += in; - value *= multiplyAmount; + value = (value + in) * multiplyAmount; } void advance_remove(Token out) { value -= out * oldCoefficient; } static ProcValue powHash(ProcValue x, unsigned y) { ProcValue res = 1; - while (y) { - if (y & 1) res *= x; + while (y) + { + if (y & 1) + res *= x; x *= x; y >>= 1; } From 60cd44e85e22ef795620e1757571cc21cad1110c Mon Sep 17 00:00:00 2001 From: Deamhan Date: Mon, 18 Mar 2019 20:59:48 +0300 Subject: [PATCH 35/54] libbdelta.cpp: small performance optimization --- src/libbdelta.cpp | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/libbdelta.cpp b/src/libbdelta.cpp index 2b7683c..619a339 100644 --- a/src/libbdelta.cpp +++ b/src/libbdelta.cpp @@ -397,8 +397,9 @@ static void bdelta_pass_2(BDelta_Instance *b, unsigned blocksize, unsigned minMa b->access_int = -1; unsigned numblocks = 0; - for (unsigned i = 0; i < numunused; ++i) - numblocks += unused[i].num / blocksize; + for (unsigned i = 0; i < numunused; ++i) + numblocks += unused[i].num; + numblocks /= blocksize; bdelta_pass_2_htable.resize(std::max((unsigned)2, roundUpPowerOf2(numblocks))); bdelta_pass_2_hchecksums.resize(numblocks + 2); From a2f65100bda222d4abcce5980dafbee0449a4298 Mon Sep 17 00:00:00 2001 From: Deamhan Date: Mon, 18 Mar 2019 22:01:29 +0300 Subject: [PATCH 36/54] libbdelta.cpp: cleanup --- src/libbdelta.cpp | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/libbdelta.cpp b/src/libbdelta.cpp index 619a339..e2be9c5 100644 --- a/src/libbdelta.cpp +++ b/src/libbdelta.cpp @@ -532,18 +532,18 @@ void bdelta_pass(BDelta_Instance *b, unsigned blocksize, unsigned minMatchSize, else { std::sort(unused + 1, unused + numunused, comparemrp2()); - for (unsigned i = 1; i < numunused; ++i) + for (unsigned i = 1; i < numunused; ++i) { UnusedRange u1 = unused[i], u2 = unused2[i]; if (u1.num >= blocksize && u2.num >= blocksize) - if (! maxHoleSize || (u1.num <= maxHoleSize && u2.num <= maxHoleSize)) - if (! (flags & BDELTA_SIDES_ORDERED) || (bdelta_next(u1.ml) == u1.mr && bdelta_next(u2.ml) == u2.mr)) + if (!maxHoleSize || (u1.num <= maxHoleSize && u2.num <= maxHoleSize)) + if (!(flags & BDELTA_SIDES_ORDERED) || (bdelta_next(u1.ml) == u1.mr && bdelta_next(u2.ml) == u2.mr)) bdelta_pass_2(b, blocksize, minMatchSize, &u1, 1, &u2, 1); } } if (verbose) - printf("pass (blocksize: %u, matches: %lu)\n", blocksize, (unsigned long)b->matches.size()); + printf("pass (blocksize: %u, matches: %u)\n", blocksize, (unsigned)b->matches.size()); // Get rid of the dummy values we placed at the ends. b->matches.erase(std::find_if(b->matches.begin(), b->matches.end(), isZeroMatch)); From 7ef3b5c72aa7ff04185d7631c792b7ebf06d1a6c Mon Sep 17 00:00:00 2001 From: Deamhan Date: Tue, 19 Mar 2019 00:05:54 +0300 Subject: [PATCH 37/54] libbdelta.cpp: increment issue has been fixed --- src/libbdelta.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libbdelta.cpp b/src/libbdelta.cpp index e2be9c5..2d547b4 100644 --- a/src/libbdelta.cpp +++ b/src/libbdelta.cpp @@ -91,7 +91,7 @@ struct Checksums_Instance template void add(T&& ck) { - checksums[++numchecksums] = std::forward(ck); + checksums[numchecksums++] = std::forward(ck); } unsigned tableIndex(Hash::Value hashValue) { From 70b0ffdd039d86e1bfeda988ee53db0e8b667033 Mon Sep 17 00:00:00 2001 From: Deamhan Date: Wed, 20 Mar 2019 18:29:17 +0300 Subject: [PATCH 38/54] libbdelta.cpp: cleanup --- src/libbdelta.cpp | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/libbdelta.cpp b/src/libbdelta.cpp index 2d547b4..3765170 100644 --- a/src/libbdelta.cpp +++ b/src/libbdelta.cpp @@ -100,7 +100,7 @@ struct Checksums_Instance }; -static unsigned match_buf_forward(const void *buf1, const void *buf2, unsigned num) +static inline unsigned match_buf_forward(const void *buf1, const void *buf2, unsigned num) { unsigned i = 0; while (i < num && ((const Token*)buf1)[i] == ((const Token*)buf2)[i]) @@ -108,7 +108,7 @@ static unsigned match_buf_forward(const void *buf1, const void *buf2, unsigned n return i; } -static unsigned match_buf_backward(const void *buf1, const void *buf2, unsigned num) +static inline unsigned match_buf_backward(const void *buf1, const void *buf2, unsigned num) { int i = num; do @@ -401,7 +401,7 @@ static void bdelta_pass_2(BDelta_Instance *b, unsigned blocksize, unsigned minMa numblocks += unused[i].num; numblocks /= blocksize; - bdelta_pass_2_htable.resize(std::max((unsigned)2, roundUpPowerOf2(numblocks))); + bdelta_pass_2_htable.resize(std::max(2u, roundUpPowerOf2(numblocks))); bdelta_pass_2_hchecksums.resize(numblocks + 2); Checksums_Instance h(blocksize, bdelta_pass_2_htable.size(), bdelta_pass_2_htable.data(), bdelta_pass_2_hchecksums.data()); @@ -579,7 +579,7 @@ void bdelta_getMatch(BDelta_Instance *b, unsigned matchNum, unsigned *p1, unsign *num = accessplace->num; } -inline int bdelta_getError(BDelta_Instance *instance) +int bdelta_getError(BDelta_Instance *instance) { return instance->errorcode; } From 7a6e10cc01b81058b81cdf8735ded98b40c821c2 Mon Sep 17 00:00:00 2001 From: Deamhan Date: Wed, 20 Mar 2019 18:30:44 +0300 Subject: [PATCH 39/54] file.cpp: missing header has been added --- src/file.cpp | 1 + 1 file changed, 1 insertion(+) diff --git a/src/file.cpp b/src/file.cpp index 0a4afe0..46b5b7c 100644 --- a/src/file.cpp +++ b/src/file.cpp @@ -5,6 +5,7 @@ #include "file.h" #include +#include #ifdef _MSC_VER #define fwrite_unlocked _fwrite_nolock From c774369dfce06bbfd71416a9df26cf2fe07138bc Mon Sep 17 00:00:00 2001 From: Deamhan Date: Wed, 20 Mar 2019 18:31:44 +0300 Subject: [PATCH 40/54] bdelta.def has been added (for libbdelta.dll) --- src/bdelta.def | 11 +++++++++++ 1 file changed, 11 insertions(+) create mode 100644 src/bdelta.def diff --git a/src/bdelta.def b/src/bdelta.def new file mode 100644 index 0000000..55123ff --- /dev/null +++ b/src/bdelta.def @@ -0,0 +1,11 @@ +LIBRARY bdelta +EXPORTS + bdelta_init_alg + bdelta_done_alg + bdelta_pass + bdelta_swap_inputs + bdelta_clean_matches + bdelta_numMatches + bdelta_getMatch + bdelta_getError + bdelta_showMatches From 89c5bb95c4ffb2d28886ec8c98ad0f543241fc05 Mon Sep 17 00:00:00 2001 From: Deamhan Date: Wed, 20 Mar 2019 18:32:14 +0300 Subject: [PATCH 41/54] CMakeLists.txt: some new options have been added --- makefiles/CMakeLists.txt | 87 +++++++++++++++++++++++++++++++++------- 1 file changed, 72 insertions(+), 15 deletions(-) diff --git a/makefiles/CMakeLists.txt b/makefiles/CMakeLists.txt index c350ed4..5b80214 100644 --- a/makefiles/CMakeLists.txt +++ b/makefiles/CMakeLists.txt @@ -1,10 +1,15 @@ cmake_minimum_required(VERSION 3.6) project(bdelta) -set(CMAKE_CXX_STANDARD 17) -option(BDELTA_PGO "build using PGO instead of PGI (MSVC only)" "OFF") -option(GCC32 "build 32 bit executable instead of 64 bit (GCC only)" "OFF") +option(BDELTA_PGO "build using PGO instead of PGI (MSVC only)" "OFF") +option(GCC32 "build 32 bit executable instead of 64 bit (GCC only)" "OFF") +option(CXX17 "enable C++17 features" "ON") +option(BIG_ENDIAN_HOST "build for big endian host" "OFF") + +IF (CXX17) + set(CMAKE_CXX_STANDARD 17) +ENDIF() IF(MSVC) set(CMAKE_CONFIGURATION_TYPES "Debug;Release;PGO" CACHE STRING "" FORCE) @@ -33,8 +38,27 @@ set (BPATCH_SOURCES ../src/bpatch.cpp ) +set (BDELTA_STATIC_LIB_SOURCES + + ${COMMON_SOURCES} + ../src/bdelta.h + ../src/checksum.h + ../src/libbdelta.cpp +) + +set (BDELTA_SHARED_LIB_SOURCES + + ${BDELTA_STATIC_LIB_SOURCES} +) + IF(MSVC) - IF(CMAKE_CXX_COMPILER_VERSION VERSION_LESS 19.15) + set (BDELTA_SHARED_LIB_SOURCES + + ${BDELTA_SHARED_LIB_SOURCES} + ../src/bdelta.def + ) + + IF(CXX17 AND CMAKE_CXX_COMPILER_VERSION VERSION_LESS 19.15) message(FATAL_ERROR "Your Visual Studio is too old. Fully updated Visual Studio 2017 or newer is required") ENDIF() add_compile_options(/Zi # pdb @@ -63,7 +87,11 @@ IF(MSVC) /MP # multiprocessor compilation /utf-8 # utf-8 source & exec /GF # eliminate duplicate strings - /std:c++17) + ) + + IF (CXX17) + add_compile_options(/std:c++17) + ENDIF() SET(CMAKE_CXX_FLAGS ${CMAKE_C_FLAGS}) @@ -88,7 +116,7 @@ IF(MSVC) ENDIF() ELSE() - IF(CMAKE_COMPILER_IS_GNUCC) + IF(CXX17 AND CMAKE_COMPILER_IS_GNUCC) IF(CMAKE_CXX_COMPILER_VERSION VERSION_LESS 8.2) message(FATAL_ERROR "Your GCC is too old. GCC 8.2 or newer is required") ENDIF() @@ -155,9 +183,9 @@ IF (GCC32) ENDIF() ENDIF() -string(TIMESTAMP BUILD_YEAR "%Y") -add_definitions(-DBUILD_YEAR="${BUILD_YEAR}") -add_definitions(-DUSE_CXX17) +IF (CXX17) + add_definitions(-DUSE_CXX17) +ENDIF() add_executable( bdelta @@ -171,14 +199,43 @@ add_executable( ${BPATCH_SOURCES} ) +add_library ( + libbdelta-static + STATIC + + ${BDELTA_STATIC_LIB_SOURCES} +) + +add_library ( + libbdelta-shared + SHARED + + ${BDELTA_SHARED_LIB_SOURCES} +) + +SET_TARGET_PROPERTIES( + libbdelta-static + PROPERTIES PREFIX "" + OUTPUT_NAME libbdelta + PDB_NAME libbdelta-static +) + +SET_TARGET_PROPERTIES( + libbdelta-shared + PROPERTIES PREFIX "" + OUTPUT_NAME libbdelta + PDB_NAME libbdelta-shared +) + IF (NOT MSVC) - SET ( - LIBS + IF (CXX17) + SET ( + LIBS + + stdc++fs + ) + ENDIF() - stdc++fs - pthread - dl - ) target_link_libraries(bdelta ${LIBS}) target_link_libraries(bpatch ${LIBS}) ENDIF() From 7a6337f4f140575eaf85f4878db60ab4ade80414 Mon Sep 17 00:00:00 2001 From: Deamhan Date: Wed, 20 Mar 2019 18:36:27 +0300 Subject: [PATCH 42/54] pre CXX17 build case under VS has been fixed --- src/bdelta.cpp | 1 + src/bpatch.cpp | 1 + 2 files changed, 2 insertions(+) diff --git a/src/bdelta.cpp b/src/bdelta.cpp index bbcffda..2bb1b66 100644 --- a/src/bdelta.cpp +++ b/src/bdelta.cpp @@ -2,6 +2,7 @@ * License, v. 2.0. If a copy of the MPL was not distributed with this * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ +#include #include #include #include diff --git a/src/bpatch.cpp b/src/bpatch.cpp index 2cf67ad..e8dd1f7 100644 --- a/src/bpatch.cpp +++ b/src/bpatch.cpp @@ -2,6 +2,7 @@ * License, v. 2.0. If a copy of the MPL was not distributed with this * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ +#include #include #include #include From a2e2cf00bf26c686d844e31c2f3b2f51e663920a Mon Sep 17 00:00:00 2001 From: Deamhan Date: Wed, 20 Mar 2019 19:11:04 +0300 Subject: [PATCH 43/54] CMakeLists.txt: -std=c++17 has been moved under CXX17 --- makefiles/CMakeLists.txt | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/makefiles/CMakeLists.txt b/makefiles/CMakeLists.txt index 5b80214..91e36a4 100644 --- a/makefiles/CMakeLists.txt +++ b/makefiles/CMakeLists.txt @@ -159,7 +159,10 @@ ELSE() -Werror=missing-format-attribute -funsigned-char) - SET(CMAKE_CXX_FLAGS "${CMAKE_C_FLAGS} -Werror=delete-non-virtual-dtor -U__STRICT_ANSI__ -fno-operator-names -std=c++17") + SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Werror=delete-non-virtual-dtor -U__STRICT_ANSI__ -fno-operator-names") + IF (CXX17) + SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=c++17") + ENDIF() SET(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -Wimplicit-int -Wmissing-prototypes -Werror=implicit-int") SET(CMAKE_CXX_FLAGS_RELEASE "-O2 -s -fno-rtti -DNDEBUG") From 2ddd76675d076183911bcab1db25ce71d67922a0 Mon Sep 17 00:00:00 2001 From: Deamhan Date: Thu, 21 Mar 2019 22:14:55 +0300 Subject: [PATCH 44/54] file.h: some cleanup --- src/file.h | 20 ++++++++------------ 1 file changed, 8 insertions(+), 12 deletions(-) diff --git a/src/file.h b/src/file.h index c723af3..0f87bcc 100644 --- a/src/file.h +++ b/src/file.h @@ -9,6 +9,7 @@ #ifdef USE_CXX17 #include #endif // USE_CXX17 +#include #include #include #include @@ -93,22 +94,17 @@ static inline uint64_t getLenOfFile(const T& fname) #else -static bool fileExists(const char * fname) +static inline bool fileExists(const char * fname) { - FILE *f = fopen(fname, "rb"); - bool exists = (f != nullptr); - if (exists) - fclose(f); - return exists; + std::unique_ptr f(fopen(fname, "rb"), fclose); + return (bool)f; } -static unsigned getLenOfFile(const char * fname) +static inline unsigned getLenOfFile(const char * fname) { - FILE *f = fopen(fname, "rb"); - fseek(f, 0, SEEK_END); - unsigned len = ftell(f); - fclose(f); - return len; + std::unique_ptr f(fopen(fname, "rb"), fclose); + fseek(f.get(), 0, SEEK_END); + return ftell(f.get()); } #endif // USE_CXX17 From 31f0a9c556194bd65d91bfd3af27dd77d44eb187 Mon Sep 17 00:00:00 2001 From: Deamhan Date: Fri, 22 Mar 2019 17:45:33 +0300 Subject: [PATCH 45/54] 'noexcept' has been added where it is possible & some cleanup --- src/bdelta.h | 13 ++++----- src/checksum.h | 20 +++++++------- src/libbdelta.cpp | 67 +++++++++++++++++++++++------------------------ 3 files changed, 50 insertions(+), 50 deletions(-) diff --git a/src/bdelta.h b/src/bdelta.h index 1934623..57bc57d 100644 --- a/src/bdelta.h +++ b/src/bdelta.h @@ -4,6 +4,8 @@ #ifdef __cplusplus extern "C" { +#else +#define noexcept #endif // __cplusplus typedef struct _BDelta_Instance BDelta_Instance; @@ -20,16 +22,15 @@ void bdelta_done_alg(BDelta_Instance *b); void bdelta_pass(BDelta_Instance *b, unsigned blockSize, unsigned minMatchSize, unsigned maxHoleSize, unsigned flags); -void bdelta_swap_inputs(BDelta_Instance *b); +void bdelta_swap_inputs(BDelta_Instance *b) noexcept; void bdelta_clean_matches(BDelta_Instance *b, unsigned flags); -unsigned bdelta_numMatches(BDelta_Instance *b); +unsigned bdelta_numMatches(BDelta_Instance *b) noexcept; -void bdelta_getMatch(BDelta_Instance *b, unsigned matchNum, - unsigned *p1, unsigned *p2, unsigned *num); +void bdelta_getMatch(BDelta_Instance *b, unsigned matchNum, unsigned *p1, unsigned *p2, unsigned *num) noexcept; -int bdelta_getError(BDelta_Instance *b); -void bdelta_showMatches(BDelta_Instance *b); +int bdelta_getError(BDelta_Instance *b) noexcept; +void bdelta_showMatches(BDelta_Instance *b) noexcept; // Flags for bdelta_pass() #define BDELTA_GLOBAL 1 diff --git a/src/checksum.h b/src/checksum.h index af62565..3e0eadb 100755 --- a/src/checksum.h +++ b/src/checksum.h @@ -8,38 +8,38 @@ struct Hash { typedef uint64_t Value; - Hash() {} - Hash(const Token * buf, unsigned blocksize) + Hash() = default; + Hash(const Token * buf, unsigned blocksize) noexcept { value = 0; for (unsigned num = 0; num < blocksize; ++num) advance_add(buf[num]); oldCoefficient = powHash(multiplyAmount, blocksize); } - void advance(Token out, Token in) + void advance(Token out, Token in) noexcept { advance_remove(out); advance_add(in); } - static unsigned modulo(Value hash, unsigned d) + static unsigned modulo(Value hash, unsigned d) noexcept { // Assumes d is power of 2. return (hash & (d - 1)); } - Value getValue() { return value >> extraProcBits; } + Value getValue() noexcept { return value >> extraProcBits; } private: typedef uint64_t ProcValue; - static const unsigned extraProcBits = (sizeof(ProcValue) - sizeof(Value)) * 8; + static constexpr unsigned extraProcBits = (sizeof(ProcValue) - sizeof(Value)) * 8; - static const ProcValue multiplyAmount = ((1ll << extraProcBits) | 181); + static constexpr ProcValue multiplyAmount = ((1ll << extraProcBits) | 181); ProcValue oldCoefficient, value; - void advance_add(Token in) + void advance_add(Token in) noexcept { value = (value + in) * multiplyAmount; } - void advance_remove(Token out) { value -= out * oldCoefficient; } - static ProcValue powHash(ProcValue x, unsigned y) + void advance_remove(Token out) noexcept { value -= out * oldCoefficient; } + static ProcValue powHash(ProcValue x, unsigned y) noexcept { ProcValue res = 1; while (y) diff --git a/src/libbdelta.cpp b/src/libbdelta.cpp index 3765170..4400bd2 100644 --- a/src/libbdelta.cpp +++ b/src/libbdelta.cpp @@ -23,27 +23,26 @@ typedef uint32_t Token; #include "bdelta.h" #include "checksum.h" - const bool verbose = false; struct checksum_entry { Hash::Value cksum; //Rolling checksums unsigned loc; - checksum_entry() {} - checksum_entry(Hash::Value _cksum, unsigned _loc) : cksum(_cksum), loc(_loc) {} + checksum_entry() = default; + checksum_entry(Hash::Value _cksum, unsigned _loc) noexcept : cksum(_cksum), loc(_loc) {} }; struct Range { unsigned p, num; - Range() {} - Range(unsigned _p, unsigned _num) : p(_p), num(_num) {} + Range() = default; + Range(unsigned _p, unsigned _num) noexcept : p(_p), num(_num) {} }; struct Match { unsigned p1, p2, num; - Match(unsigned _p1, unsigned _p2, unsigned _num) : p1(_p1), p2(_p2), num(_num) {} + Match(unsigned _p1, unsigned _p2, unsigned _num) noexcept : p1(_p1), p2(_p2), num(_num) {} }; struct _BDelta_Instance @@ -57,7 +56,7 @@ struct _BDelta_Instance int errorcode; _BDelta_Instance(bdelta_readCallback _cb, void * _handle1, void * _handle2, - unsigned _data1_size, unsigned _data2_size) + unsigned _data1_size, unsigned _data2_size) noexcept : cb(_cb), handle1(_handle1), handle2(_handle2) , data1_size(_data1_size), data2_size(_data2_size) , access_int(-1), errorcode(0) @@ -81,7 +80,7 @@ struct Checksums_Instance checksum_entry * const checksums; // Sorted list of all checksums unsigned numchecksums; - Checksums_Instance(unsigned _blocksize, unsigned _htablesize, checksum_entry ** _htable, checksum_entry * _checksums) + Checksums_Instance(unsigned _blocksize, unsigned _htablesize, checksum_entry ** _htable, checksum_entry * _checksums) noexcept : blocksize(_blocksize), htablesize(_htablesize), htable(_htable) , checksums(_checksums), numchecksums(0) { @@ -89,18 +88,18 @@ struct Checksums_Instance } template - void add(T&& ck) + void add(T&& ck) noexcept { checksums[numchecksums++] = std::forward(ck); } - unsigned tableIndex(Hash::Value hashValue) + unsigned tableIndex(Hash::Value hashValue) noexcept { return Hash::modulo(hashValue, htablesize); } }; -static inline unsigned match_buf_forward(const void *buf1, const void *buf2, unsigned num) +static inline unsigned match_buf_forward(const void *buf1, const void *buf2, unsigned num) noexcept { unsigned i = 0; while (i < num && ((const Token*)buf1)[i] == ((const Token*)buf2)[i]) @@ -108,7 +107,7 @@ static inline unsigned match_buf_forward(const void *buf1, const void *buf2, uns return i; } -static inline unsigned match_buf_backward(const void *buf1, const void *buf2, unsigned num) +static inline unsigned match_buf_backward(const void *buf1, const void *buf2, unsigned num) noexcept { int i = num; do @@ -157,14 +156,14 @@ static unsigned match_backward(BDelta_Instance *b, unsigned p1, unsigned p2, uns // Iterator helper function template -static inline T bdelta_next(T i) {return ++i;} +static inline T bdelta_next(T i) noexcept { return ++i; } struct UnusedRange { unsigned p, num; std::list::iterator ml, mr; - UnusedRange() {} - UnusedRange(unsigned _p, unsigned _num, std::list::iterator _ml, std::list::iterator _mr) + UnusedRange() = default; + UnusedRange(unsigned _p, unsigned _num, std::list::iterator _ml, std::list::iterator _mr) noexcept : p(_p), num(_num), ml(_ml), mr(_mr) {} }; @@ -173,7 +172,7 @@ struct UnusedRange class comparep { public: - bool operator()(UnusedRange r1, UnusedRange r2) + bool operator()(UnusedRange r1, UnusedRange r2) noexcept { return ((r1.p != r2.p) ? (r1.p < r2.p) : (r1.num > r2.num)); } @@ -182,7 +181,7 @@ class comparep class comparemrp2 { public: - bool operator()(UnusedRange r1, UnusedRange r2) + bool operator()(UnusedRange r1, UnusedRange r2) noexcept { return ((r1.mr->p2 != r2.mr->p2)? (r1.mr->p2 < r2.mr->p2) : (r1.mr->num > r2.mr->num)); } @@ -191,7 +190,7 @@ class comparemrp2 class compareMatchP2 { public: - bool operator()(Match r1, Match r2) + bool operator()(Match r1, Match r2) noexcept { return ((r1.p2 != r2.p2) ? (r1.p2 < r2.p2) : (r1.num > r2.num)); } @@ -209,7 +208,7 @@ static void addMatch(BDelta_Instance *b, unsigned p1, unsigned p2, unsigned num, } template -T absoluteDifference(T a, T b) +T absoluteDifference(T a, T b) noexcept { return std::max(a, b) - std::min(a, b); } @@ -315,8 +314,8 @@ static void findMatches(BDelta_Instance *b, Checksums_Instance *h, unsigned minM struct Checksums_Compare { Checksums_Instance &ci; - explicit Checksums_Compare(Checksums_Instance &h) : ci(h) {} - bool operator() (checksum_entry c1, checksum_entry c2) + explicit Checksums_Compare(Checksums_Instance &h) noexcept : ci(h) {} + bool operator() (checksum_entry c1, checksum_entry c2) noexcept { unsigned ti1 = ci.tableIndex(c1.cksum), ti2 = ci.tableIndex(c2.cksum); if (ti1 == ti2) @@ -338,7 +337,7 @@ static thread_local std::vector bdelta_pass_unused; BDelta_Instance *bdelta_init_alg(unsigned data1_size, unsigned data2_size, bdelta_readCallback cb, void *handle1, void *handle2, - unsigned tokenSize) + unsigned tokenSize) { if (tokenSize != sizeof(Token)) { @@ -361,7 +360,7 @@ BDelta_Instance *bdelta_init_alg(unsigned data1_size, unsigned data2_size, return b; } -void bdelta_done_alg(BDelta_Instance *b) +void bdelta_done_alg(BDelta_Instance *b) { b->matches.clear(); delete b; @@ -384,7 +383,7 @@ void bdelta_done_alg(BDelta_Instance *b) // Adapted from http://graphics.stanford.edu/~seander/bithacks.html#RoundUpPowerOf2 -static unsigned roundUpPowerOf2(unsigned v) +static unsigned roundUpPowerOf2(unsigned v) noexcept { --v; for (int i = 1; i <= 16; i *= 2) @@ -448,7 +447,7 @@ static void bdelta_pass_2(BDelta_Instance *b, unsigned blocksize, unsigned minMa } } -void bdelta_swap_inputs(BDelta_Instance *b) +void bdelta_swap_inputs(BDelta_Instance *b) noexcept { for (std::list::iterator l = b->matches.begin(); l != b->matches.end(); ++l) std::swap(l->p1, l->p2); @@ -457,7 +456,7 @@ void bdelta_swap_inputs(BDelta_Instance *b) b->matches.sort(compareMatchP2()); } -void bdelta_clean_matches(BDelta_Instance *b, unsigned flags) +void bdelta_clean_matches(BDelta_Instance *b, unsigned flags) { std::list::iterator nextL = b->matches.begin(); if (nextL == b->matches.end()) @@ -483,14 +482,14 @@ void bdelta_clean_matches(BDelta_Instance *b, unsigned flags) } } -void bdelta_showMatches(BDelta_Instance *b) +void bdelta_showMatches(BDelta_Instance *b) noexcept { for (std::list::iterator l = b->matches.begin(); l != b->matches.end(); ++l) printf("(%d, %d, %d), ", l->p1, l->p2, l->num); - printf ("\n\n"); + printf("\n\n"); } -static void get_unused_blocks(UnusedRange *unused, unsigned *numunusedptr) +static void get_unused_blocks(UnusedRange *unused, unsigned *numunusedptr) noexcept { unsigned nextStartPos = 0; for (unsigned i = 1; i < *numunusedptr; ++i) @@ -501,9 +500,9 @@ static void get_unused_blocks(UnusedRange *unused, unsigned *numunusedptr) } } -inline bool isZeroMatch(Match &m) { return m.num == 0; } +inline bool isZeroMatch(Match &m) noexcept { return m.num == 0; } -void bdelta_pass(BDelta_Instance *b, unsigned blocksize, unsigned minMatchSize, unsigned maxHoleSize, unsigned flags) +void bdelta_pass(BDelta_Instance *b, unsigned blocksize, unsigned minMatchSize, unsigned maxHoleSize, unsigned flags) { // Place an empty Match at beginning so we can assume there's a Match to the left of every hole. b->matches.emplace_front(0, 0, 0); @@ -550,12 +549,12 @@ void bdelta_pass(BDelta_Instance *b, unsigned blocksize, unsigned minMatchSize, b->matches.pop_back(); } -unsigned bdelta_numMatches(BDelta_Instance *b) +unsigned bdelta_numMatches(BDelta_Instance *b) noexcept { return b->matches.size(); } -void bdelta_getMatch(BDelta_Instance *b, unsigned matchNum, unsigned *p1, unsigned *p2, unsigned *num) +void bdelta_getMatch(BDelta_Instance *b, unsigned matchNum, unsigned *p1, unsigned *p2, unsigned *num) noexcept { int &access_int = b->access_int; std::list::iterator &accessplace = b->accessplace; @@ -579,7 +578,7 @@ void bdelta_getMatch(BDelta_Instance *b, unsigned matchNum, unsigned *p1, unsign *num = accessplace->num; } -int bdelta_getError(BDelta_Instance *instance) +int bdelta_getError(BDelta_Instance *instance) noexcept { return instance->errorcode; } From 79da65604147c7743faabe65d11ca91d4846dffe Mon Sep 17 00:00:00 2001 From: Deamhan Date: Tue, 9 Apr 2019 23:34:51 +0300 Subject: [PATCH 46/54] exception-free bdelta implementation is ready to use --- makefiles/CMakeLists.txt | 15 +- src/bdelta.cpp | 314 +++++++++++++----------- src/bdelta.h | 33 +-- src/bpatch.cpp | 169 +++++++------ src/file.cpp | 24 +- src/file.h | 72 +++--- src/libbdelta.cpp | 248 +++++++++++-------- src/noexcept.h | 516 +++++++++++++++++++++++++++++++++++++++ 8 files changed, 994 insertions(+), 397 deletions(-) create mode 100644 src/noexcept.h diff --git a/makefiles/CMakeLists.txt b/makefiles/CMakeLists.txt index 91e36a4..5f86541 100644 --- a/makefiles/CMakeLists.txt +++ b/makefiles/CMakeLists.txt @@ -30,6 +30,7 @@ set (BDELTA_SOURCES ../src/bdelta.h ../src/checksum.h ../src/libbdelta.cpp + ../src/noexcept.h ) set (BPATCH_SOURCES @@ -44,6 +45,7 @@ set (BDELTA_STATIC_LIB_SOURCES ../src/bdelta.h ../src/checksum.h ../src/libbdelta.cpp + ../src/noexcept.h ) set (BDELTA_SHARED_LIB_SOURCES @@ -87,6 +89,8 @@ IF(MSVC) /MP # multiprocessor compilation /utf-8 # utf-8 source & exec /GF # eliminate duplicate strings + + /EHs-c- # disable c++ exceptions ) IF (CXX17) @@ -96,6 +100,7 @@ IF(MSVC) SET(CMAKE_CXX_FLAGS ${CMAKE_C_FLAGS}) add_definitions(-D_CRT_SECURE_NO_WARNINGS) + add_definitions(-D_HAS_EXCEPTIONS=0) SET(CMAKE_EXE_LINKER_FLAGS "/LARGEADDRESSAWARE /SUBSYSTEM:CONSOLE") SET(CMAKE_EXE_LINKER_FLAGS_RELEASE "${CMAKE_EXE_LINKER_FLAGS_RELEASE} /LTCG /OPT:REF,ICF /DEBUG") @@ -157,12 +162,12 @@ ELSE() -Werror=reorder -Werror=declaration-after-statement -Werror=missing-format-attribute - -funsigned-char) + -funsigned-char + + -fno-exceptions + ) - SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Werror=delete-non-virtual-dtor -U__STRICT_ANSI__ -fno-operator-names") - IF (CXX17) - SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=c++17") - ENDIF() + SET(CMAKE_CXX_FLAGS "${CMAKE_C_FLAGS} -Werror=delete-non-virtual-dtor -U__STRICT_ANSI__ -fno-operator-names -std=c++17") SET(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -Wimplicit-int -Wmissing-prototypes -Werror=implicit-int") SET(CMAKE_CXX_FLAGS_RELEASE "-O2 -s -fno-rtti -DNDEBUG") diff --git a/src/bdelta.cpp b/src/bdelta.cpp index 2bb1b66..2d5eaaa 100644 --- a/src/bdelta.cpp +++ b/src/bdelta.cpp @@ -8,179 +8,199 @@ #include #include #include -#include #include "bdelta.h" #include "file.h" -static const void *f_read(void *f, void *buf, unsigned place, unsigned num) +static const void * f_read(void *f, void *buf, unsigned place, unsigned num, BDelta_Instance * /*b*/, BDELTA_RESULT& result) { fseek((FILE *)f, place, SEEK_SET); - fread_fixed((FILE *)f, buf, num); + result = fread_fixed((FILE *)f, buf, num) ? BDELTA_OK : BDELTA_READ_ERROR; return buf; } +static const char * fatal_alloc = "FATAL: unable to allocate memory\n"; +static const char * fatal_read = "FATAL: unable to read file\n"; +static const char * fatal_write = "FATAL: unable to write file\n"; + static void my_pass(BDelta_Instance *b, unsigned blocksize, unsigned minMatchSize, unsigned flags) { bdelta_pass(b, blocksize, minMatchSize, 0, flags); + switch (*bdelta_getError(b)) + { + case BDELTA_MEM_ERROR: + printf(fatal_alloc); + exit(1); + case BDELTA_READ_ERROR: + printf(fatal_read); + exit(2); + case BDELTA_WRITE_ERROR: + printf(fatal_write); + exit(3); + } + bdelta_clean_matches(b, BDELTA_REMOVE_OVERLAP); } +#define check_write_result(result) if(!(result)) { printf(fatal_read); return 3; } +#define check_read_result(result) if(!(result)) { printf(fatal_write); return 2; } +#define check_alloc(ptr) if (!(ptr)) { printf(fatal_alloc); exit(1); } + int main(int argc, char **argv) { - try + std::unique_ptr m1(nullptr, free), m2(nullptr, free); + + const bool all_ram_mode = (argc > 1 && strcmp(argv[1], "--all-in-ram") == 0); + if (all_ram_mode) + { + --argc; + ++argv; + } + if (argc != 4) + { + printf("usage: bdelta [--all-in-ram] \n"); + printf("needs two files to compare + output file:\n"); + exit(1); + } + if (!fileExists(argv[1]) || !fileExists(argv[2])) + { + printf("one of the input files does not exist\n"); + exit(1); + } + uint32_t size = (uint32_t)getLenOfFile(argv[1]); + uint32_t size2 = (uint32_t)getLenOfFile(argv[2]); + std::unique_ptr f1(fopen(argv[1], "rb"), fclose); + if (!f1) { - std::unique_ptr m1; - std::unique_ptr m2; + printf("unable to open file %s\n", argv[1]); + exit(1); + } - const bool all_ram_mode = (argc > 1 && strcmp(argv[1], "--all-in-ram") == 0); - if (all_ram_mode) - { - --argc; - ++argv; - } - if (argc != 4) - { - printf("usage: bdelta [--all-in-ram] \n"); - printf("needs two files to compare + output file:\n"); - exit(1); - } - if (!fileExists(argv[1]) || !fileExists(argv[2])) - { - printf("one of the input files does not exist\n"); - exit(1); - } - unsigned size = (unsigned)getLenOfFile(argv[1]); - unsigned size2 = (unsigned)getLenOfFile(argv[2]); - std::unique_ptr f1(fopen(argv[1], "rb"), fclose); - if (!f1) - { - printf("unable to open file %s\n", argv[1]); - exit(1); - } - - std::unique_ptr f2(fopen(argv[2], "rb"), fclose); - if (!f2) - { - printf("unable to open file %s\n", argv[2]); - exit(1); - } - - - BDelta_Instance * b = nullptr; + std::unique_ptr f2(fopen(argv[2], "rb"), fclose); + if (!f2) + { + printf("unable to open file %s\n", argv[2]); + exit(1); + } - if (all_ram_mode) - { - m1.reset(new char[size]); - m2.reset(new char[size2]); - fread_fixed(f1.get(), m1.get(), size); - fread_fixed(f2.get(), m2.get(), size2); - b = bdelta_init_alg(size, size2, nullptr, m1.get(), m2.get(), 1); - } - else - b = bdelta_init_alg(size, size2, f_read, f1.get(), f2.get(), 1); - - std::unique_ptr b_holder(b, bdelta_done_alg); - - int nummatches = 0; - - // List of primes for reference. Taken from Wikipedia. - // 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 - // 1-20 2 3 5 7 11 13 17 19 23 29 31 37 41 43 47 53 59 61 67 71 - // 21-40 73 79 83 89 97 101 103 107 109 113 127 131 137 139 149 151 157 163 167 173 - // 41-60 179 181 191 193 197 199 211 223 227 229 233 239 241 251 257 263 269 271 277 281 - // 61-80 283 293 307 311 313 317 331 337 347 349 353 359 367 373 379 383 389 397 401 409 - // 81-100 419 421 431 433 439 443 449 457 461 463 467 479 487 491 499 503 509 521 523 541 - // 101-120 547 557 563 569 571 577 587 593 599 601 607 613 617 619 631 641 643 647 653 659 - // 121-140 661 673 677 683 691 701 709 719 727 733 739 743 751 757 761 769 773 787 797 809 - // 141-160 811 821 823 827 829 839 853 857 859 863 877 881 883 887 907 911 919 929 937 941 - // 161-180 947 953 967 971 977 983 991 997 - - my_pass(b, 997, 1994, 0); - my_pass(b, 503, 1006, 0); - my_pass(b, 127, 254, 0); - my_pass(b, 31, 62, 0); - my_pass(b, 7, 14, 0); - my_pass(b, 5, 10, 0); - my_pass(b, 3, 6, 0); - my_pass(b, 13, 26, BDELTA_GLOBAL); - my_pass(b, 7, 14, 0); - my_pass(b, 5, 10, 0); - - nummatches = bdelta_numMatches(b); - - unsigned * copyloc1 = new unsigned[nummatches + 1]; - unsigned * copyloc2 = new unsigned[nummatches + 1]; - unsigned * copynum = new unsigned[nummatches + 1]; - std::unique_ptr copyloc1_holder(copyloc1), copyloc2_holder(copyloc2), copynum_holder(copynum); - - std::unique_ptr fout(fopen(argv[3], "wb"), fclose); - if (!fout) - { - printf("couldn't open output file\n"); - exit(1); - } - constexpr size_t fout_buffer_size = 256 * 1024; - setvbuf(fout.get(), nullptr, _IOFBF, fout_buffer_size); - - const char * magic = "BDT"; - fwrite_fixed(fout.get(), magic, 3); - unsigned short version = 1; - write_word(fout.get(), version); - unsigned char intsize = 4; - fwrite_fixed(fout.get(), &intsize, 1); - write_dword(fout.get(), size); - write_dword(fout.get(), size2); - write_dword(fout.get(), nummatches); - - unsigned lastp1 = 0, lastp2 = 0; - for (int i = 0; i < nummatches; ++i) - { - unsigned p1 = 0, p2 = 0, num = 0; - bdelta_getMatch(b, i, &p1, &p2, &num); - copyloc1[i] = p1 - lastp1; - write_dword(fout.get(), copyloc1[i]); - copyloc2[i] = p2 - lastp2; - write_dword(fout.get(), copyloc2[i]); - copynum[i] = num; - write_dword(fout.get(), copynum[i]); - lastp1 = p1 + num; - lastp2 = p2 + num; - } - if (size2 != lastp2) - { - copyloc1[nummatches] = 0; - copynum[nummatches] = 0; - copyloc2[nummatches] = size2 - lastp2; - ++nummatches; - } + BDelta_Instance * b = nullptr; + + if (all_ram_mode) + { + m1.reset((uint8_t*)malloc(size)); + m2.reset((uint8_t*)malloc(size2)); + check_alloc(m1 && m2); + fread_fixed(f1.get(), m1.get(), size); + fread_fixed(f2.get(), m2.get(), size2); - unsigned fp = 0; - constexpr size_t WRITE_BUFFER_SIZE = 1024 * 1024; - std::unique_ptr write_buffer(new uint8_t[WRITE_BUFFER_SIZE]); - for (int i = 0; i < nummatches; ++i) + b = bdelta_init_alg(size, size2, nullptr, m1.get(), m2.get(), 1); + } + else + b = bdelta_init_alg(size, size2, f_read, f1.get(), f2.get(), 1); + + check_alloc(b); + + std::unique_ptr b_holder(b, bdelta_done_alg); + + int nummatches = 0; + + // List of primes for reference. Taken from Wikipedia. + // 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 + // 1-20 2 3 5 7 11 13 17 19 23 29 31 37 41 43 47 53 59 61 67 71 + // 21-40 73 79 83 89 97 101 103 107 109 113 127 131 137 139 149 151 157 163 167 173 + // 41-60 179 181 191 193 197 199 211 223 227 229 233 239 241 251 257 263 269 271 277 281 + // 61-80 283 293 307 311 313 317 331 337 347 349 353 359 367 373 379 383 389 397 401 409 + // 81-100 419 421 431 433 439 443 449 457 461 463 467 479 487 491 499 503 509 521 523 541 + // 101-120 547 557 563 569 571 577 587 593 599 601 607 613 617 619 631 641 643 647 653 659 + // 121-140 661 673 677 683 691 701 709 719 727 733 739 743 751 757 761 769 773 787 797 809 + // 141-160 811 821 823 827 829 839 853 857 859 863 877 881 883 887 907 911 919 929 937 941 + // 161-180 947 953 967 971 977 983 991 997 + + my_pass(b, 997, 1994, 0); + my_pass(b, 503, 1006, 0); + my_pass(b, 127, 254, 0); + my_pass(b, 31, 62, 0); + my_pass(b, 7, 14, 0); + my_pass(b, 5, 10, 0); + my_pass(b, 3, 6, 0); + my_pass(b, 13, 26, BDELTA_GLOBAL); + my_pass(b, 7, 14, 0); + my_pass(b, 5, 10, 0); + + nummatches = bdelta_numMatches(b); + + const size_t mem_len = (nummatches + 1) * sizeof(uint32_t); + std::unique_ptr copyloc1((uint32_t*)malloc(mem_len), free), + copyloc2((uint32_t*)malloc(mem_len), free), + copynum((uint32_t*)malloc(mem_len), free); + + check_alloc(copyloc1 && copyloc2 && copynum); + + std::unique_ptr fout(fopen(argv[3], "wb"), fclose); + if (!fout) + { + printf("couldn't open output file\n"); + exit(1); + } + constexpr size_t fout_buffer_size = 256 * 1024; + setvbuf(fout.get(), nullptr, _IOFBF, fout_buffer_size); + + const char * magic = "BDT"; + check_write_result(fwrite_fixed(fout.get(), magic, 3)); + uint16_t version = 1; + check_write_result(write_value(fout.get(), &version)); + uint8_t intsize = 4; + check_write_result(write_value(fout.get(), &intsize)); + bool result = write_value(fout.get(), &size) + && write_value(fout.get(), &size2) + && write_value(fout.get(), &nummatches); + check_write_result(result); + + unsigned lastp1 = 0, lastp2 = 0; + for (int i = 0; i < nummatches; ++i) + { + unsigned p1 = 0, p2 = 0, num = 0; + bdelta_getMatch(b, i, &p1, &p2, &num); + copyloc1[i] = p1 - lastp1; + result = write_value(fout.get(), ©loc1[i]); + copyloc2[i] = p2 - lastp2; + result = result && write_value(fout.get(), ©loc2[i]); + copynum[i] = num; + result = result && write_value(fout.get(), ©num[i]); + check_write_result(result); + lastp1 = p1 + num; + lastp2 = p2 + num; + } + if (size2 != lastp2) + { + copyloc1[nummatches] = 0; + copynum[nummatches] = 0; + copyloc2[nummatches] = size2 - lastp2; + ++nummatches; + } + + unsigned fp = 0; + constexpr size_t WRITE_BUFFER_SIZE = 1024 * 1024; + std::unique_ptr write_buffer((uint8_t*)malloc(WRITE_BUFFER_SIZE), free); + check_alloc(write_buffer); + BDELTA_RESULT * res = bdelta_getError(b); + for (int i = 0; i < nummatches; ++i) + { + unsigned num = copyloc2[i]; + while (num > 0) { - unsigned num = copyloc2[i]; - while (num > 0) - { - unsigned towrite = std::min(num, WRITE_BUFFER_SIZE); - const void * block = all_ram_mode ? (const void *)((char*)m2.get() + fp) : f_read(f2.get(), write_buffer.get(), fp, towrite); - fwrite_fixed(fout.get(), block, towrite); - num -= towrite; - fp += towrite; - } - - if (i != nummatches) - fp += copynum[i]; + unsigned towrite = std::min(num, WRITE_BUFFER_SIZE); + const void * block = all_ram_mode ? (const void *)((char*)m2.get() + fp) : f_read(f2.get(), write_buffer.get(), fp, towrite, b, *res); + check_read_result(*res == BDELTA_OK); + check_write_result(fwrite_fixed(fout.get(), block, towrite)); + num -= towrite; + fp += towrite; } - } - catch (const std::exception& ex) - { - fprintf (stderr, "FATAL: %s\n", ex.what()); - return -1; + if (i != nummatches) + fp += copynum[i]; } return 0; diff --git a/src/bdelta.h b/src/bdelta.h index 57bc57d..6f7430b 100644 --- a/src/bdelta.h +++ b/src/bdelta.h @@ -1,6 +1,8 @@ /* This Source Code Form is subject to the terms of the Mozilla Public * License, v. 2.0. If a copy of the MPL was not distributed with this * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ +#ifndef __BDELTA_H__ +#define __BDELTA_H__ #ifdef __cplusplus extern "C" { @@ -8,28 +10,35 @@ extern "C" { #define noexcept #endif // __cplusplus +enum BDELTA_RESULT { + BDELTA_OK = 0, + BDELTA_MEM_ERROR = -1, + BDELTA_READ_ERROR = -2, + BDELTA_WRITE_ERROR = -3 +}; + typedef struct _BDelta_Instance BDelta_Instance; // Callback function must return a pointer to the data requested. // A "fill and forget" buffer is provided, but can be ignored, so // long as the data persists throughout the life of bdelta_pass(). -typedef const void *(*bdelta_readCallback)(void *handle, void *buf, unsigned place, unsigned num); +typedef const void *(*bdelta_readCallback)(void *handle, void *buf, unsigned place, unsigned num, BDelta_Instance * b, BDELTA_RESULT& result); -BDelta_Instance *bdelta_init_alg(unsigned data1_size, unsigned data2_size, - bdelta_readCallback cb, void *handle1, void *handle2, - unsigned tokenSize); -void bdelta_done_alg(BDelta_Instance *b); +BDelta_Instance * bdelta_init_alg(unsigned data1_size, unsigned data2_size, + bdelta_readCallback cb, void *handle1, void *handle2, + unsigned tokenSize) noexcept; +void bdelta_done_alg(BDelta_Instance *b) noexcept; -void bdelta_pass(BDelta_Instance *b, unsigned blockSize, unsigned minMatchSize, unsigned maxHoleSize, unsigned flags); +void bdelta_pass(BDelta_Instance *b, unsigned blockSize, unsigned minMatchSize, unsigned maxHoleSize, unsigned flags) noexcept; void bdelta_swap_inputs(BDelta_Instance *b) noexcept; -void bdelta_clean_matches(BDelta_Instance *b, unsigned flags); +void bdelta_clean_matches(BDelta_Instance *b, unsigned flags) noexcept; unsigned bdelta_numMatches(BDelta_Instance *b) noexcept; void bdelta_getMatch(BDelta_Instance *b, unsigned matchNum, unsigned *p1, unsigned *p2, unsigned *num) noexcept; -int bdelta_getError(BDelta_Instance *b) noexcept; +BDELTA_RESULT * bdelta_getError(BDelta_Instance *b) noexcept; void bdelta_showMatches(BDelta_Instance *b) noexcept; // Flags for bdelta_pass() @@ -39,12 +48,8 @@ void bdelta_showMatches(BDelta_Instance *b) noexcept; // Flags for bdelta_clean_matches() #define BDELTA_REMOVE_OVERLAP 1 -enum BDELTA_RESULT { - BDELTA_OK = 0, - BDELTA_MEM_ERROR = -1, - BDELTA_READ_ERROR = -2 -}; - #ifdef __cplusplus } #endif // __cplusplus + +#endif // __BDELTA_H__ diff --git a/src/bpatch.cpp b/src/bpatch.cpp index e8dd1f7..fa8260b 100644 --- a/src/bpatch.cpp +++ b/src/bpatch.cpp @@ -6,6 +6,7 @@ #include #include #include +#include #include #include "file.h" @@ -18,7 +19,7 @@ static bool copy_bytes_to_file(FILE *infile, FILE *outfile, unsigned numleft) { const size_t BUFFER_SIZE = 65536; - std::unique_ptr buf(new uint8_t[BUFFER_SIZE]); + std::unique_ptr buf((uint8_t*)malloc(BUFFER_SIZE), free); while (numleft != 0) { const size_t len = std::min(BUFFER_SIZE, numleft); @@ -39,107 +40,103 @@ static bool copy_bytes_to_file(FILE *infile, FILE *outfile, unsigned numleft) return true; } +#define check_read_result(result) if(!(result)) { printf("FATAL: I/O error on reading\n"); return 2; } + int main(int argc, char **argv) { - try + if (argc != 4) { - if (argc != 4) - { - printf("usage: bpatch \n"); - printf("needs a reference file, file to output, and patchfile:\n"); - return 1; - } + printf("usage: bpatch \n"); + printf("needs a reference file, file to output, and patchfile:\n"); + return 1; + } - if (!fileExists(argv[1]) || !fileExists(argv[3])) - { - printf("one of the input files does not exist\n"); - return 1; - } + if (!fileExists(argv[1]) || !fileExists(argv[3])) + { + printf("one of the input files does not exist\n"); + return 1; + } - std::unique_ptr patchfile(fopen(argv[3], "rb"), fclose); - char magic[3]; - fread_fixed(patchfile.get(), magic, 3); - if (strncmp(magic, "BDT", 3)) - { - printf("Given file is not a recognized patchfile\n"); - return 1; - } - unsigned short version = read_word(patchfile.get()); - if (version != 1) - { - printf("unsupported patch version\n"); - return 1; - } - char intsize; - fread_fixed(patchfile.get(), &intsize, 1); - if (intsize != 4) - { - printf("unsupported file pointer size\n"); - return 1; - } - /*unsigned size1 =*/ read_dword(patchfile.get()); - unsigned size2 = read_dword(patchfile.get()); + std::unique_ptr patchfile(fopen(argv[3], "rb"), fclose); + char magic[3]; + check_read_result(fread_fixed(patchfile.get(), magic, 3)); + if (strncmp(magic, "BDT", 3)) + { + printf("Given file is not a recognized patchfile\n"); + return 1; + } + uint16_t version = 0; + uint8_t intsize = 0; + check_read_result(read_value(patchfile.get(), &version)); + if (version != 1) + { + printf("unsupported patch version\n"); + return 1; + } + check_read_result(read_value(patchfile.get(), &intsize)); + if (intsize != 4) + { + printf("unsupported file pointer size\n"); + return 1; + } + uint32_t size1, size2, nummatches; + bool result = read_value(patchfile.get(), &size1) + && read_value(patchfile.get(), &size2) + && read_value(patchfile.get(), &nummatches); + check_read_result(result); + + std::unique_ptr copyloc1((uint32_t*)malloc(nummatches + 1), free), + copyloc2((uint32_t*)malloc(nummatches + 1), free), + copynum((uint32_t*)malloc(nummatches + 1), free); - unsigned nummatches = read_dword(patchfile.get()); + for (unsigned i = 0; i < nummatches; ++i) + { + result = read_value(patchfile.get(), ©loc1[i]) + && read_value(patchfile.get(), ©loc2[i]) + && read_value(patchfile.get(), ©num[i]); + check_read_result(result); + size2 -= copyloc2[i] + copynum[i]; + } + if (size2) + { + copyloc1[nummatches] = 0; + copynum[nummatches] = 0; + copyloc2[nummatches] = size2; + ++nummatches; + } - unsigned * copyloc1 = new unsigned[nummatches + 1]; - unsigned * copyloc2 = new unsigned[nummatches + 1]; - unsigned * copynum = new unsigned[nummatches + 1]; - std::unique_ptr copyloc1_holder(copyloc1), copyloc2_holder(copyloc2), copynum_holder(copynum); + std::unique_ptr ref(fopen(argv[1], "rb"), fclose); + if (!ref) + { + printf("Error: unable to open file %s\n", argv[1]); + return -1; + } - for (unsigned i = 0; i < nummatches; ++i) - { - copyloc1[i] = read_dword(patchfile.get()); - copyloc2[i] = read_dword(patchfile.get()); - copynum[i] = read_dword(patchfile.get()); - size2 -= copyloc2[i] + copynum[i]; - } - if (size2) - { - copyloc1[nummatches] = 0; copynum[nummatches] = 0; - copyloc2[nummatches] = size2; - ++nummatches; - } - std::unique_ptr ref(fopen(argv[1], "rb"), fclose); - if (!ref) + std::unique_ptr outfile(fopen(argv[2], "wb"), fclose); + if (!outfile) + { + printf("Error: unable to open file %s\n", argv[2]); + return -1; + } + constexpr size_t outfile_buffer_size = 256 * 1024; + setvbuf(outfile.get(), nullptr, _IOFBF, outfile_buffer_size); + + for (unsigned i = 0; i < nummatches; ++i) + { + if (!copy_bytes_to_file(patchfile.get(), outfile.get(), copyloc2[i])) { - printf("Error: unable to open file %s\n", argv[1]); + printf("Error: patchfile is truncated\n"); return -1; } - - std::unique_ptr outfile(fopen(argv[2], "wb"), fclose); - if (!outfile) + fseek(ref.get(), copyloc1[i], SEEK_CUR); + + if (!copy_bytes_to_file(ref.get(), outfile.get(), copynum[i])) { - printf("Error: unable to open file %s\n", argv[2]); + printf("Error while copying from reference file\n"); return -1; } - constexpr size_t outfile_buffer_size = 256 * 1024; - setvbuf(outfile.get(), nullptr, _IOFBF, outfile_buffer_size); - - for (unsigned i = 0; i < nummatches; ++i) - { - if (!copy_bytes_to_file(patchfile.get(), outfile.get(), copyloc2[i])) - { - printf("Error: patchfile is truncated\n"); - return -1; - } - - int copyloc = copyloc1[i]; - fseek(ref.get(), copyloc, SEEK_CUR); - - if (!copy_bytes_to_file(ref.get(), outfile.get(), copynum[i])) - { - printf("Error while copying from reference file\n"); - return -1; - } - } - } - catch (const std::exception& ex) - { - fprintf (stderr, "FATAL: %s\n", ex.what()); - return -1; } return 0; diff --git a/src/file.cpp b/src/file.cpp index 46b5b7c..80b02f1 100644 --- a/src/file.cpp +++ b/src/file.cpp @@ -12,7 +12,7 @@ #define fread_unlocked _fread_nolock #endif // _MSC_VER -void fread_fixed(FILE *f, void * _buf, unsigned num_bytes) +bool fread_fixed(FILE *f, void * _buf, unsigned num_bytes) noexcept { char * buf = (char *)_buf; @@ -22,18 +22,16 @@ void fread_fixed(FILE *f, void * _buf, unsigned num_bytes) size_t r = fread_unlocked(buf, 1, block_size, f); if (r != block_size) - { - const size_t BUFFER_SIZE = 512; - std::unique_ptr error_message_buffer(new char[BUFFER_SIZE]); - snprintf(error_message_buffer.get(), BUFFER_SIZE, "read error: fread_fixed(block_size=%u) != %u", block_size, (unsigned)r); - throw std::system_error(EIO, std::generic_category(), std::string(error_message_buffer.get())); - } + return false; + buf += block_size; num_bytes -= block_size; } + + return true; } -void fwrite_fixed(FILE *f, const void * _buf, unsigned num_bytes) +bool fwrite_fixed(FILE *f, const void * _buf, unsigned num_bytes) noexcept { const char * buf = (const char *)_buf; @@ -43,13 +41,11 @@ void fwrite_fixed(FILE *f, const void * _buf, unsigned num_bytes) size_t r = fwrite_unlocked(buf, 1, block_size, f); if (r != block_size) - { - const size_t BUFFER_SIZE = 512; - std::unique_ptr error_message_buffer(new char[BUFFER_SIZE]); - snprintf(error_message_buffer.get(), BUFFER_SIZE, "write error: fwrite_fixed(num_bytes=%u) != %u", block_size, (unsigned)r); - throw std::system_error(EIO, std::generic_category(), std::string(error_message_buffer.get())); - } + return false; + buf += block_size; num_bytes -= block_size; } + + return true; } diff --git a/src/file.h b/src/file.h index 0f87bcc..90a3ec4 100644 --- a/src/file.h +++ b/src/file.h @@ -16,64 +16,70 @@ static constexpr size_t MAX_IO_BLOCK_SIZE = (1024 * 1024); -void fread_fixed(FILE *f, void * _buf, unsigned num_bytes); -void fwrite_fixed(FILE *f, const void * _buf, unsigned num_bytes); - +bool fread_fixed(FILE *f, void * _buf, unsigned num_bytes) noexcept; +bool fwrite_fixed(FILE *f, const void * _buf, unsigned num_bytes) noexcept; #ifdef BIG_ENDIAN_HOST +template +struct HalfType; + +template <> +struct HalfType +{ + typedef uint8_t type; +}; + +template <> +struct HalfType +{ + typedef uint16_t type; +}; -static inline unsigned read_word(FILE *f) +template +static inline bool read_value(FILE* f, T* value) { - unsigned char b, b2; - fread_fixed(f, &b, 1); - fread_fixed(f, &b2, 1); - return (b2 << 8) + b; + typename HalfType::type low, high; + if (!(fread_fixed(f, &low, sizeof(low)) && fread_fixed(f, &high, sizeof(high)))) + return false; + constexpr int shift = sizeof(low) * 8; + *value = ((high << shift) + low); + return true; } -static inline unsigned read_dword(FILE *f) +template <> static inline bool read_value(FILE* f, uint8_t* value); { - unsigned low = read_word(f); - return (read_word(f) << 16) + low; + return fread_fixed(f, value, sizeof(*value)); } -static inline void write_word(FILE *f, unsigned number) +template +static inline bool write_value(FILE* f, T* value) { - unsigned char b = number & 255, - b2 = number >> 8; - fwrite_fixed(f, &b, 1); - fwrite_fixed(f, &b2, 1); + constexpr int shift = sizeof(HalfType::type) * 8; + constexpr int mask = ((1 << shift) - 1); + + typename HalfType::type low = (*value & mask), high = (*value >> 8); + return (fwrite_fixed(f, &low, sizeof(low)) && fwrite_fixed(f, &high, sizeof(high))); } -static inline void write_dword(FILE *f, unsigned number) +template <> static inline bool write_value(FILE * f, uint8_t* value); { - write_word(f, number & 65535); - write_word(f, number >> 16); + return write_fixed(f, value, sizeof(*value)); } #else template ::value>> -static inline T read_type(FILE *f) +static inline bool read_value(FILE* f, T* value) { - T result = 0; - fread_fixed(f, &result, sizeof(result)); - return result; + return fread_fixed(f, value, sizeof(*value)); } -#define read_byte(f) read_type((f)) -#define read_word(f) read_type((f)) -#define read_dword(f) read_type((f)) - template ::value>> -static inline void write_type(FILE *f, T number) +static inline bool write_value(FILE* f, T* value) { - fwrite_fixed(f, &number, sizeof(number)); + return fwrite_fixed(f, value, sizeof(*value)); } -#define write_byte(f, v) write_type((f), (v)) -#define write_word(f, v) write_type((f), (v)) -#define write_dword(f, v) write_type((f), (v)) - #endif // BIG_ENDIAN #ifdef USE_CXX17 diff --git a/src/libbdelta.cpp b/src/libbdelta.cpp index 4400bd2..186fd22 100644 --- a/src/libbdelta.cpp +++ b/src/libbdelta.cpp @@ -6,10 +6,9 @@ #include #include #include -#include #include #include -#include +#include #include #if !defined(TOKEN_SIZE) || TOKEN_SIZE == 1 @@ -22,6 +21,7 @@ typedef uint32_t Token; #include "bdelta.h" #include "checksum.h" +#include "noexcept.h" const bool verbose = false; struct checksum_entry @@ -45,30 +45,67 @@ struct Match Match(unsigned _p1, unsigned _p2, unsigned _num) noexcept : p1(_p1), p2(_p2), num(_num) {} }; +typedef NoThrowList MatchList; +typedef typename MatchList::iterator MatchListIterator; + +static const size_t TOKEN_BUFFER_SIZE = 4096; +static constexpr size_t BUFFER_DEFAULT_SIZE = 16 * 1024; + +struct UnusedRange +{ + unsigned p, num; + MatchListIterator ml, mr; + UnusedRange() = default; + UnusedRange(unsigned _p, unsigned _num, const MatchListIterator& _ml, const MatchListIterator& _mr) noexcept + : p(_p), num(_num), ml(_ml), mr(_mr) + {} + void set(unsigned _p, unsigned _num, const MatchListIterator& _ml, const MatchListIterator& _mr) + { + p = _p; + num = _num; + ml = _ml; + mr = _mr; + } +}; + struct _BDelta_Instance { const bdelta_readCallback cb; void *handle1, *handle2; unsigned data1_size, data2_size; - std::list matches; - std::list::iterator accessplace; + MatchList matches; + MatchListIterator accessplace; int access_int; - int errorcode; + BDELTA_RESULT errorcode; + + NoThrowMemoryStorage match_buffer; + NoThrowMemoryStorage find_matches_buffer; + NoThrowMemoryStorage bdelta_pass_2_htable; + NoThrowMemoryStorage bdelta_pass_2_hchecksums; + NoThrowMemoryStorage bdelta_pass_2_buf; + NoThrowMemoryStorage bdelta_pass_unused; _BDelta_Instance(bdelta_readCallback _cb, void * _handle1, void * _handle2, unsigned _data1_size, unsigned _data2_size) noexcept : cb(_cb), handle1(_handle1), handle2(_handle2) , data1_size(_data1_size), data2_size(_data2_size) - , access_int(-1), errorcode(0) - {} + , access_int(-1), errorcode(BDELTA_OK), match_buffer(2 * TOKEN_BUFFER_SIZE) + , find_matches_buffer(2 * BUFFER_DEFAULT_SIZE), bdelta_pass_2_htable(2 * BUFFER_DEFAULT_SIZE) + , bdelta_pass_2_hchecksums(2 * BUFFER_DEFAULT_SIZE), bdelta_pass_2_buf(2 * BUFFER_DEFAULT_SIZE) + , bdelta_pass_unused(2 * BUFFER_DEFAULT_SIZE) + { + if (!(match_buffer && find_matches_buffer && bdelta_pass_2_htable + && bdelta_pass_2_hchecksums && bdelta_pass_2_buf && bdelta_pass_unused)) + errorcode = BDELTA_MEM_ERROR; + } - const Token *read1(void *buf, unsigned place, unsigned num) + const Token *read1(void *buf, unsigned place, unsigned num) noexcept { - return (cb == nullptr)? ((const Token*)((char *)handle1 + place)) : ((const Token*)cb(handle1, buf, place, num)); + return (cb == nullptr)? ((const Token*)((char *)handle1 + place)) : ((const Token*)cb(handle1, buf, place, num, this, this->errorcode)); } - const Token *read2(void *buf, unsigned place, unsigned num) + const Token *read2(void *buf, unsigned place, unsigned num) noexcept { - return (cb == nullptr) ? ((const Token*)((char *)handle2 + place)) : ((const Token*)cb(handle2, buf, place, num)); + return (cb == nullptr)? ((const Token*)((char *)handle2 + place)) : ((const Token*)cb(handle2, buf, place, num, this, this->errorcode)); } }; @@ -118,29 +155,31 @@ static inline unsigned match_buf_backward(const void *buf1, const void *buf2, un return (num - i - 1); } -static const size_t TOKEN_BUFFER_SIZE = 4096; -static thread_local std::unique_ptr match_forward_buffer; -static unsigned match_forward(BDelta_Instance *b, unsigned p1, unsigned p2) +static unsigned match_forward(BDelta_Instance *b, unsigned p1, unsigned p2) noexcept { unsigned num = 0, match, numtoread; - Token * buf1 = match_forward_buffer.get(), * buf2 = buf1 + TOKEN_BUFFER_SIZE; + Token * buf1 = b->match_buffer.get(), * buf2 = buf1 + TOKEN_BUFFER_SIZE; do { numtoread = std::min(std::min(b->data1_size - p1, b->data2_size - p2), TOKEN_BUFFER_SIZE); const Token *read1 = b->read1(buf1, p1, numtoread), *read2 = b->read2(buf2, p2, numtoread); - p1 += numtoread; p2 += numtoread; + + if (b->errorcode != BDELTA_OK) + return 0; + + p1 += numtoread; + p2 += numtoread; match = match_buf_forward(read1, read2, numtoread); num += match; } while (match && match == numtoread); return num; } -static thread_local std::unique_ptr match_backward_buffer; -static unsigned match_backward(BDelta_Instance *b, unsigned p1, unsigned p2, unsigned blocksize) +static unsigned match_backward(BDelta_Instance *b, unsigned p1, unsigned p2, unsigned blocksize) noexcept { - unsigned num = 0, match, numtoread; - Token * buf1 = match_backward_buffer.get(), * buf2 = buf1 + TOKEN_BUFFER_SIZE; + unsigned num = 0u, match, numtoread; + Token * buf1 = b->match_buffer.get(), * buf2 = buf1 + TOKEN_BUFFER_SIZE; do { numtoread = std::min(std::min(std::min(p1, p2), blocksize), TOKEN_BUFFER_SIZE); @@ -148,6 +187,10 @@ static unsigned match_backward(BDelta_Instance *b, unsigned p1, unsigned p2, uns p2 -= numtoread; const Token *read1 = b->read1(buf1, p1, numtoread), *read2 = b->read2(buf2, p2, numtoread); + + if (b->errorcode != BDELTA_OK) + return 0; + match = match_buf_backward(read1, read2, numtoread); num += match; } while (match && match == numtoread); @@ -158,16 +201,6 @@ static unsigned match_backward(BDelta_Instance *b, unsigned p1, unsigned p2, uns template static inline T bdelta_next(T i) noexcept { return ++i; } -struct UnusedRange -{ - unsigned p, num; - std::list::iterator ml, mr; - UnusedRange() = default; - UnusedRange(unsigned _p, unsigned _num, std::list::iterator _ml, std::list::iterator _mr) noexcept - : p(_p), num(_num), ml(_ml), mr(_mr) - {} -}; - // Sort first by location, second by match length (larger matches first) class comparep { @@ -196,7 +229,7 @@ class compareMatchP2 } }; -static void addMatch(BDelta_Instance *b, unsigned p1, unsigned p2, unsigned num, std::list::iterator place) +static void addMatch(BDelta_Instance *b, unsigned p1, unsigned p2, unsigned num, MatchListIterator place) noexcept { Match newMatch = Match(p1, p2, num); compareMatchP2 comp; @@ -204,7 +237,8 @@ static void addMatch(BDelta_Instance *b, unsigned p1, unsigned p2, unsigned num, --place; while (place != b->matches.end() && comp(*place, newMatch)) ++place; - b->matches.insert(place, newMatch); + if (b->matches.insert(place, newMatch) == nullptr) + b->errorcode = BDELTA_MEM_ERROR; } template @@ -214,14 +248,17 @@ T absoluteDifference(T a, T b) noexcept } -static constexpr size_t BUFFER_DEFAULT_SIZE = 16 * 1024; -static thread_local std::vector findMatchesBuffer; -static void findMatches(BDelta_Instance *b, Checksums_Instance *h, unsigned minMatchSize, unsigned start, unsigned end, unsigned place, std::list::iterator iterPlace) +static void findMatches(BDelta_Instance *b, Checksums_Instance *h, unsigned minMatchSize, unsigned start, unsigned end, unsigned place, MatchListIterator iterPlace) noexcept { const unsigned blocksize = h->blocksize; - findMatchesBuffer.resize(blocksize * 2); - Token * buf1 = findMatchesBuffer.data(), * buf2 = buf1 + blocksize; + if (!b->find_matches_buffer.resize(blocksize * 2)) + { + b->errorcode = BDELTA_MEM_ERROR; + return; + } + + Token * buf1 = b->find_matches_buffer.get(), * buf2 = buf1 + blocksize; unsigned best1 = 0, best2 = 0, bestnum = 0; unsigned processMatchesPos = 0; @@ -241,9 +278,15 @@ static void findMatches(BDelta_Instance *b, Checksums_Instance *h, unsigned minM { unsigned p1 = c->loc, p2 = j - blocksize; unsigned fnum = match_forward(b, p1, p2); + if (b->errorcode != BDELTA_OK) + return; + if (fnum >= blocksize) { unsigned bnum = match_backward(b, p1, p2, blocksize); + if (b->errorcode != BDELTA_OK) + return; + unsigned num = fnum + bnum; if (num >= minMatchSize) { @@ -277,6 +320,9 @@ static void findMatches(BDelta_Instance *b, Checksums_Instance *h, unsigned minM if (bestnum && j >= processMatchesPos) { addMatch(b, best1, best2, bestnum, iterPlace); + if (b->errorcode != BDELTA_OK) + return; + place = best1 + bestnum; unsigned matchEnd = best2 + bestnum; if (matchEnd > j) @@ -330,55 +376,25 @@ struct Checksums_Compare } }; -static thread_local std::vector bdelta_pass_2_htable; -static thread_local std::vector bdelta_pass_2_hchecksums; -static thread_local std::vector bdelta_pass_2_buf; -static thread_local std::vector bdelta_pass_unused; - BDelta_Instance *bdelta_init_alg(unsigned data1_size, unsigned data2_size, bdelta_readCallback cb, void *handle1, void *handle2, - unsigned tokenSize) + unsigned tokenSize) noexcept { if (tokenSize != sizeof(Token)) { printf("Error: BDelta library compiled for token size of %lu.\n", (unsigned long)sizeof (Token)); - return 0; + return nullptr; } - BDelta_Instance *b = new BDelta_Instance(cb, handle1, handle2, data1_size, data2_size); - if (!b) + void * p = malloc(sizeof(BDelta_Instance)); + if (p == nullptr) return nullptr; - match_forward_buffer.reset(new Token[TOKEN_BUFFER_SIZE * 2]); - match_backward_buffer.reset(new Token[TOKEN_BUFFER_SIZE * 2]); - - findMatchesBuffer.resize(BUFFER_DEFAULT_SIZE); - bdelta_pass_2_htable.resize(BUFFER_DEFAULT_SIZE); - bdelta_pass_2_hchecksums.resize(BUFFER_DEFAULT_SIZE); - bdelta_pass_2_buf.resize(BUFFER_DEFAULT_SIZE); - bdelta_pass_unused.resize(BUFFER_DEFAULT_SIZE); - - return b; + return (new(p) BDelta_Instance(cb, handle1, handle2, data1_size, data2_size)); } -void bdelta_done_alg(BDelta_Instance *b) +void bdelta_done_alg(BDelta_Instance *b) noexcept { - b->matches.clear(); - delete b; - - match_forward_buffer.reset(); - match_backward_buffer.reset(); - - findMatchesBuffer.clear(); - bdelta_pass_2_htable.clear(); - bdelta_pass_2_hchecksums.clear(); - bdelta_pass_2_buf.clear(); - bdelta_pass_unused.clear(); - - findMatchesBuffer.shrink_to_fit(); - bdelta_pass_2_htable.shrink_to_fit(); - bdelta_pass_2_hchecksums.shrink_to_fit(); - bdelta_pass_2_buf.shrink_to_fit(); - bdelta_pass_unused.shrink_to_fit(); + free(b); } @@ -391,7 +407,7 @@ static unsigned roundUpPowerOf2(unsigned v) noexcept return (v + 1); } -static void bdelta_pass_2(BDelta_Instance *b, unsigned blocksize, unsigned minMatchSize, UnusedRange *unused, unsigned numunused, UnusedRange *unused2, unsigned numunused2) +static void bdelta_pass_2(BDelta_Instance *b, unsigned blocksize, unsigned minMatchSize, UnusedRange *unused, unsigned numunused, UnusedRange *unused2, unsigned numunused2) noexcept { b->access_int = -1; @@ -400,18 +416,29 @@ static void bdelta_pass_2(BDelta_Instance *b, unsigned blocksize, unsigned minMa numblocks += unused[i].num; numblocks /= blocksize; - bdelta_pass_2_htable.resize(std::max(2u, roundUpPowerOf2(numblocks))); - bdelta_pass_2_hchecksums.resize(numblocks + 2); + b->bdelta_pass_2_htable.resize(std::max(2u, roundUpPowerOf2(numblocks))); + b->bdelta_pass_2_hchecksums.resize(numblocks + 2); + if (!(b->bdelta_pass_2_htable.resize(std::max(2u, roundUpPowerOf2(numblocks))) && b->bdelta_pass_2_hchecksums.resize(numblocks + 2))) + { + b->errorcode = BDELTA_MEM_ERROR; + return; + } - Checksums_Instance h(blocksize, bdelta_pass_2_htable.size(), bdelta_pass_2_htable.data(), bdelta_pass_2_hchecksums.data()); + Checksums_Instance h(blocksize, b->bdelta_pass_2_htable.size(), b->bdelta_pass_2_htable.get(), b->bdelta_pass_2_hchecksums.get()); + + b->bdelta_pass_2_buf.resize(blocksize); + if (!b->bdelta_pass_2_buf.resize(blocksize)) + { + b->errorcode = BDELTA_MEM_ERROR; + return; + } - bdelta_pass_2_buf.resize(blocksize); for (unsigned i = 0; i < numunused; ++i) { unsigned first = unused[i].p, last = unused[i].p + unused[i].num; for (unsigned loc = first; loc + blocksize <= last; loc += blocksize) { - const Token *read = b->read1(bdelta_pass_2_buf.data(), loc, blocksize); + const Token *read = b->read1(b->bdelta_pass_2_buf.get(), loc, blocksize); Hash::Value blocksum = Hash(read, blocksize).getValue(); // Adjacent checksums are never repeated. h.add(checksum_entry(blocksum, loc)); @@ -443,27 +470,31 @@ static void bdelta_pass_2(BDelta_Instance *b, unsigned blocksize, unsigned minMa for (unsigned i = 0; i < numunused2; ++i) { if (unused2[i].num >= blocksize) + { findMatches(b, &h, minMatchSize, unused2[i].p, unused2[i].p + unused2[i].num, unused[i].p, unused2[i].mr); + if (b->errorcode != BDELTA_OK) + return; + } } } void bdelta_swap_inputs(BDelta_Instance *b) noexcept { - for (std::list::iterator l = b->matches.begin(); l != b->matches.end(); ++l) - std::swap(l->p1, l->p2); + for (auto& m : b->matches) + std::swap(m.p1, m.p2); std::swap(b->data1_size, b->data2_size); std::swap(b->handle1, b->handle2); b->matches.sort(compareMatchP2()); } -void bdelta_clean_matches(BDelta_Instance *b, unsigned flags) +void bdelta_clean_matches(BDelta_Instance *b, unsigned flags) noexcept { - std::list::iterator nextL = b->matches.begin(); + auto nextL = b->matches.begin(); if (nextL == b->matches.end()) return; while (true) { - std::list::iterator l = nextL; + auto l = nextL; if (++nextL == b->matches.end()) break; @@ -484,8 +515,8 @@ void bdelta_clean_matches(BDelta_Instance *b, unsigned flags) void bdelta_showMatches(BDelta_Instance *b) noexcept { - for (std::list::iterator l = b->matches.begin(); l != b->matches.end(); ++l) - printf("(%d, %d, %d), ", l->p1, l->p2, l->num); + for (const auto& m : b->matches) + printf("(%d, %d, %d), ", m.p1, m.p2, m.num); printf("\n\n"); } @@ -502,23 +533,36 @@ static void get_unused_blocks(UnusedRange *unused, unsigned *numunusedptr) noexc inline bool isZeroMatch(Match &m) noexcept { return m.num == 0; } -void bdelta_pass(BDelta_Instance *b, unsigned blocksize, unsigned minMatchSize, unsigned maxHoleSize, unsigned flags) +void bdelta_pass(BDelta_Instance *b, unsigned blocksize, unsigned minMatchSize, unsigned maxHoleSize, unsigned flags) noexcept { // Place an empty Match at beginning so we can assume there's a Match to the left of every hole. - b->matches.emplace_front(0, 0, 0); + if (b->matches.emplace_front(0, 0, 0) == nullptr) + { + b->errorcode = BDELTA_MEM_ERROR; + return; + } // Trick for including the free range at the end. - b->matches.emplace_back(b->data1_size, b->data2_size, 0); + if (b->matches.emplace_back(b->data1_size, b->data2_size, 0) == nullptr) + { + b->errorcode = BDELTA_MEM_ERROR; + return; + } const size_t BUFFER_SIZE = b->matches.size() + 1; - bdelta_pass_unused.resize(BUFFER_SIZE * 2); - UnusedRange *unused = bdelta_pass_unused.data(), + if (!b->bdelta_pass_unused.resize(BUFFER_SIZE * 2)) + { + b->errorcode = BDELTA_MEM_ERROR; + return; + } + + UnusedRange *unused = b->bdelta_pass_unused.get(), *unused2 = unused + BUFFER_SIZE; unsigned numunused = 0, numunused2 = 0; - for (std::list::iterator l = b->matches.begin(); l != b->matches.end(); ++l) + for (auto l = b->matches.begin(); l != b->matches.end(); ++l) { - unused[numunused++] = UnusedRange(l->p1, l->num, l, l); - unused2[numunused2++] = UnusedRange(l->p2, l->num, l, l); + unused[numunused++].set(l->p1, l->num, l, l); + unused2[numunused2++].set(l->p2, l->num, l, l); } std::sort(unused + 1, unused + numunused, comparep()); // Leave empty match at beginning @@ -527,7 +571,11 @@ void bdelta_pass(BDelta_Instance *b, unsigned blocksize, unsigned minMatchSize, get_unused_blocks(unused2, &numunused2); if (flags & BDELTA_GLOBAL) + { bdelta_pass_2(b, blocksize, minMatchSize, unused, numunused, unused2, numunused2); + if (b->errorcode != BDELTA_OK) + return; + } else { std::sort(unused + 1, unused + numunused, comparemrp2()); @@ -537,7 +585,11 @@ void bdelta_pass(BDelta_Instance *b, unsigned blocksize, unsigned minMatchSize, if (u1.num >= blocksize && u2.num >= blocksize) if (!maxHoleSize || (u1.num <= maxHoleSize && u2.num <= maxHoleSize)) if (!(flags & BDELTA_SIDES_ORDERED) || (bdelta_next(u1.ml) == u1.mr && bdelta_next(u2.ml) == u2.mr)) + { bdelta_pass_2(b, blocksize, minMatchSize, &u1, 1, &u2, 1); + if (b->errorcode != BDELTA_OK) + return; + } } } @@ -557,7 +609,7 @@ unsigned bdelta_numMatches(BDelta_Instance *b) noexcept void bdelta_getMatch(BDelta_Instance *b, unsigned matchNum, unsigned *p1, unsigned *p2, unsigned *num) noexcept { int &access_int = b->access_int; - std::list::iterator &accessplace = b->accessplace; + auto& accessplace = b->accessplace; if (access_int == -1) { access_int = 0; @@ -578,7 +630,7 @@ void bdelta_getMatch(BDelta_Instance *b, unsigned matchNum, unsigned *p1, unsign *num = accessplace->num; } -int bdelta_getError(BDelta_Instance *instance) noexcept +BDELTA_RESULT * bdelta_getError(BDelta_Instance *instance) noexcept { - return instance->errorcode; + return &(instance->errorcode); } diff --git a/src/noexcept.h b/src/noexcept.h new file mode 100644 index 0000000..dcba14f --- /dev/null +++ b/src/noexcept.h @@ -0,0 +1,516 @@ +#ifndef __NOEXCEPT_H__ +#define __NOEXCEPT_H__ + +#include +#include + +template +struct Construct +{ + void construct(T * p, size_t count) const noexcept + { + for (size_t i = 0; i < count; ++i) + new(p + i) T(); + } + + void destruct(T * p, size_t count) const noexcept + { + for (size_t i = 0; i < count; ++i) + (p + i)->~T(); + } + + void move(T * old_p, size_t old_size, T * new_p, size_t new_size) const noexcept + { + size_t i = 0; + for (; i < old_size; ++i) + new (new_p + i) T(std::move(old_p[i])); + + for (; i < new_size; ++i) + new(p + i) T(); + } +}; + +template +struct NoConstruct +{ + void construct(T * /*p*/, size_t /*count*/) const noexcept + {} + + void destruct(T * /*p*/, size_t /*count*/) const noexcept + {} + + void move(T * old_p, size_t old_size, T * new_p, size_t /*new_size*/) const noexcept + { + memcpy(new_p, old_p, old_size * sizeof(T)); + } +}; + +template class Constructor> +struct NoRealloc +{ + void realloc(T * old_p, size_t old_size, T * new_p, size_t new_size, Constructor& constructor) + { + constructor.destruct(old_p, old_size); + constructor.construct(new_p, new_size); + } +}; + +template class Constructor> +struct Realloc +{ + void realloc(T * old_p, size_t old_size, T * new_p, size_t new_size, Constructor& constructor) + { + constructor.move(old_p, old_size, new_p, new_size); + constructor.destruct(old_p, old_size); + } +}; + +template class Constructor = NoConstruct, template class> class Reallocator = NoRealloc> +class NoThrowMemoryStorage +{ +public: + T * get() const noexcept { return m_data; } + + NoThrowMemoryStorage(size_t size) noexcept + : m_data((T*)malloc(size * sizeof(T))) + { + if (m_data != nullptr) + { + m_size = size; + m_capacity = size; + m_constructor.construct(m_data, m_size); + } + else + { + m_size = 0; + m_capacity = 0; + } + } + + ~NoThrowMemoryStorage() + { + if (m_data != nullptr) + { + m_constructor.destruct(m_data, m_capacity); + free(m_data); + } + } + + bool resize(size_t size) noexcept + { + if (m_capacity >= size) + { + m_size = size; + return true; + } + + T * new_data = (T*)malloc(size * sizeof(T)); + if (new_data == nullptr) + { + if (m_data != nullptr) + { + m_constructor.destruct(m_data, m_size); + free(m_data); + m_size = 0; + m_capacity = 0; + } + + return false; + } + + if (m_data != nullptr) + m_reallocator.realloc(m_data, m_size, new_data, size, m_constructor); + else + m_constructor.construct(new_data, size); + + m_data = new_data; + m_size = size; + m_capacity = size; + + return true; + } + + size_t size() const noexcept { return m_size; } + + operator bool() const noexcept { return (m_data != nullptr); } + +protected: + T * m_data; + size_t m_size; + size_t m_capacity; + + Constructor m_constructor; + Reallocator m_reallocator; +}; + +class PtrStack +{ +protected: + NoThrowMemoryStorage m_storage; + size_t m_top; +public: + PtrStack(size_t capacity) noexcept : m_storage(capacity), m_top(0) {} + bool pop(void *& p) noexcept + { + if (m_top == 0) + return false; + + p = m_storage.get()[--m_top]; + return true; + } + + bool push(void * p) noexcept + { + size_t size = m_storage.size(); + if (m_top >= size) + { + if (!m_storage.resize(size * 2)) + return false; + } + + m_storage.get()[m_top++] = p; + return true; + } + + size_t size() const noexcept { return m_top; } + size_t capacity() const noexcept { return m_storage.size(); } + bool good() const noexcept { return (m_storage.get() != nullptr); } +}; + +template +class NoThrowList +{ +public: + struct Node + { + Node * prev; + Node * next; + T data; + }; + + NoThrowList() noexcept : m_head(nullptr), m_tail(nullptr), m_size(0), m_allocator_stack(1024 * 1024), m_blocks_stack(64 * 1024) + {} + + class iterator + { + public: + typedef T value_type; + typedef T* pointer; + typedef T& reference; + typedef std::bidirectional_iterator_tag iterator_category; + + iterator() noexcept : m_node(nullptr) + {} + + explicit iterator(Node * node) noexcept : m_node(node) + {} + + iterator& operator ++() noexcept + { + m_node = m_node->next; + return *this; + } + + iterator& operator --() noexcept + { + m_node = m_node->prev; + return *this; + } + + bool operator == (const iterator& rhs) noexcept + { + return (m_node == rhs.m_node); + } + + bool operator != (const iterator& rhs) noexcept + { + return !(*this == rhs); + } + + Node * get() const noexcept + { + return m_node; + } + + T& operator * () const noexcept + { + return m_node->data; + } + + T* operator ->() const noexcept + { + return &m_node->data; + } + + bool empty() const noexcept + { + return (m_node == nullptr); + } + + protected: + Node * m_node; + }; + + iterator end() const noexcept + { + return iterator(); + } + + iterator begin() const noexcept + { + return iterator(m_head); + } + + bool empty() const noexcept + { + return (m_head == nullptr); + } + + size_t size() + { + return m_size; + } + + template + T* emplace_back(Args... args) noexcept + { + Node * new_node = allocate(); + if (new_node == nullptr) + return nullptr; + + new (&new_node->data) T(std::forward(args)...); + new_node->next = nullptr; + + if (m_tail == nullptr) + m_head = new_node; + else + m_tail->next = new_node; + + new_node->prev = m_tail; + m_tail = new_node; + ++m_size; + + return &new_node->data; + } + + template + T* emplace_front(Args... args) noexcept + { + Node * new_node = allocate(); + if (new_node == nullptr) + return nullptr; + + new (&new_node->data) T(std::forward(args)...); + new_node->prev = nullptr; + + if (m_head == nullptr) + m_tail = new_node; + else + m_head->prev = new_node; + + new_node->next = m_head; + m_head = new_node; + ++m_size; + + return &new_node->data; + } + + iterator erase(const iterator& it) noexcept + { + Node * node = it.get(); + Node * const prev = node->prev, + * const next = node->next; + + if (prev == nullptr) + { + m_head = m_head->next; + if (m_head != nullptr) + m_head->prev = nullptr; + } + else + prev->next = next; + + if (next == nullptr) + { + m_tail = m_tail->prev; + if (m_tail != nullptr) + m_tail->next = nullptr; + } + else + next->prev = prev; + + node->data.T::~T(); + deallocate(node); + --m_size; + + return iterator(next); + } + + void pop_back() + { + if (empty()) + return; + + Node * const prev = m_tail->prev; + if (prev == nullptr) + m_head = nullptr; + + m_tail->data.T::~T(); + deallocate(m_tail); + + m_tail = prev; + if (m_tail != nullptr) + m_tail->next = nullptr; + --m_size; + } + + T * insert(iterator where, const T& data) + { + Node * node = where.get(); + if (node == nullptr) + return emplace_back(data); + else if (node == m_head) + return emplace_front(data); + else + { + Node * new_node = allocate(); + if (new_node == nullptr) + return nullptr; + + Node * const prev = node->prev; + new (&new_node->data) T(data); + new_node->prev = prev; + prev->next = new_node; + new_node->next = node; + node->prev = new_node; + ++m_size; + + return &new_node->data; + } + } + + template + void sort(Comaparer comp) + { + qsort(comp, m_head, nullptr, &m_head); + Node * prev = nullptr; + for (Node * cur = m_head; cur != nullptr; cur = cur->next) + { + cur->prev = prev; + prev = cur; + } + + m_tail = prev; + } + + ~NoThrowList() noexcept + { + Node * p = m_head; + while (p != nullptr) + { + Node * next = p->next; + p->data.T::~T(); + //deallocate(p); + p = next; + } + + void * block = nullptr; + while(m_blocks_stack.pop(block)) + free(block); + + m_size = 0; + m_head = nullptr; + m_tail = nullptr; + } + +protected: + Node * m_head; + Node * m_tail; + size_t m_size; + PtrStack m_allocator_stack; + PtrStack m_blocks_stack; + + Node * allocate() + { + if (!m_allocator_stack.good()) + return nullptr; + + void * p = nullptr; + if (m_allocator_stack.pop(p)) + return (Node*)p; + + if (!m_blocks_stack.good()) + return nullptr; + + size_t count = m_allocator_stack.capacity(); + Node * block = (Node*)malloc(sizeof(Node) * count); + if (!m_blocks_stack.push(block)) + { + free(block); + return nullptr; + } + + for (size_t i = 1; i < count; ++i) + m_allocator_stack.push(block + i); + + return block; + } + void deallocate(Node * p) + { + m_allocator_stack.push(p); + } + + template + void qsort(Comaparer& comp, Node * hd, Node * tl, Node ** rtn) + { + int nlo, nhi; + Node *lo, *hi, *q, *p; + + /* Invariant: Return head sorted with `tl' appended. */ + while (hd != nullptr) + { + nlo = nhi = 0; + lo = hi = nullptr; + q = hd; + p = hd->next; + + /* Partition and count sizes. */ + while (p != nullptr) + { + q = p->next; + if (!comp(hd->data, p->data)) + { + p->next = lo; + lo = p; + ++nlo; + } + else + { + p->next = hi; + hi = p; + ++nhi; + } + p = q; + } + + /* Recurse to establish invariant for sublists of hd, + choosing shortest list first to limit stack. */ + if (nlo < nhi) + { + qsort(comp, lo, hd, rtn); + rtn = &hd->next; + hd = hi; /* Eliminated tail-recursive call. */ + } + else + { + qsort(comp, hi, tl, &hd->next); + tl = hd; + hd = lo; /* Eliminated tail-recursive call. */ + } + } + /* Base case of recurrence. Invariant is easy here. */ + *rtn = tl; + } +}; + +#endif // __NOEXCEPT_H__ From 73a01c73fca64b48efc238bc994f15a44528e5ab Mon Sep 17 00:00:00 2001 From: Deamhan Date: Thu, 11 Apr 2019 08:20:23 +0300 Subject: [PATCH 47/54] cleanup --- makefiles/CMakeLists.txt | 4 +--- src/bdelta.cpp | 4 ++-- src/checksum.h | 1 - src/libbdelta.cpp | 11 +++-------- src/noexcept.h | 6 ++++-- 5 files changed, 10 insertions(+), 16 deletions(-) diff --git a/makefiles/CMakeLists.txt b/makefiles/CMakeLists.txt index 5f86541..7410d8a 100644 --- a/makefiles/CMakeLists.txt +++ b/makefiles/CMakeLists.txt @@ -65,7 +65,6 @@ IF(MSVC) ENDIF() add_compile_options(/Zi # pdb /W4 # warning level 4 - /EHsc # exceptions: sync /J # use unsigned char /Gd # use cdecl # treat warnings as errors @@ -89,14 +88,13 @@ IF(MSVC) /MP # multiprocessor compilation /utf-8 # utf-8 source & exec /GF # eliminate duplicate strings - - /EHs-c- # disable c++ exceptions ) IF (CXX17) add_compile_options(/std:c++17) ENDIF() + string(REPLACE "/EHsc" "/EHs-c-" CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS}") # disable c++ exceptions SET(CMAKE_CXX_FLAGS ${CMAKE_C_FLAGS}) add_definitions(-D_CRT_SECURE_NO_WARNINGS) diff --git a/src/bdelta.cpp b/src/bdelta.cpp index 2d5eaaa..13587f1 100644 --- a/src/bdelta.cpp +++ b/src/bdelta.cpp @@ -37,9 +37,9 @@ static void my_pass(BDelta_Instance *b, unsigned blocksize, unsigned minMatchSiz case BDELTA_WRITE_ERROR: printf(fatal_write); exit(3); + default: + bdelta_clean_matches(b, BDELTA_REMOVE_OVERLAP); } - - bdelta_clean_matches(b, BDELTA_REMOVE_OVERLAP); } #define check_write_result(result) if(!(result)) { printf(fatal_read); return 3; } diff --git a/src/checksum.h b/src/checksum.h index 3e0eadb..65fab4e 100755 --- a/src/checksum.h +++ b/src/checksum.h @@ -8,7 +8,6 @@ struct Hash { typedef uint64_t Value; - Hash() = default; Hash(const Token * buf, unsigned blocksize) noexcept { value = 0; diff --git a/src/libbdelta.cpp b/src/libbdelta.cpp index 186fd22..689afdb 100644 --- a/src/libbdelta.cpp +++ b/src/libbdelta.cpp @@ -28,14 +28,12 @@ struct checksum_entry { Hash::Value cksum; //Rolling checksums unsigned loc; - checksum_entry() = default; checksum_entry(Hash::Value _cksum, unsigned _loc) noexcept : cksum(_cksum), loc(_loc) {} }; struct Range { unsigned p, num; - Range() = default; Range(unsigned _p, unsigned _num) noexcept : p(_p), num(_num) {} }; @@ -55,7 +53,7 @@ struct UnusedRange { unsigned p, num; MatchListIterator ml, mr; - UnusedRange() = default; + UnusedRange() noexcept : p(0), num(0) {} UnusedRange(unsigned _p, unsigned _num, const MatchListIterator& _ml, const MatchListIterator& _mr) noexcept : p(_p), num(_num), ml(_ml), mr(_mr) {} @@ -135,7 +133,6 @@ struct Checksums_Instance } }; - static inline unsigned match_buf_forward(const void *buf1, const void *buf2, unsigned num) noexcept { unsigned i = 0; @@ -247,7 +244,6 @@ T absoluteDifference(T a, T b) noexcept return std::max(a, b) - std::min(a, b); } - static void findMatches(BDelta_Instance *b, Checksums_Instance *h, unsigned minMatchSize, unsigned start, unsigned end, unsigned place, MatchListIterator iterPlace) noexcept { const unsigned blocksize = h->blocksize; @@ -264,7 +260,7 @@ static void findMatches(BDelta_Instance *b, Checksums_Instance *h, unsigned minM unsigned processMatchesPos = 0; const Token *inbuf = b->read2(buf1, start, blocksize), *outbuf = nullptr; - Hash hash = Hash(inbuf, blocksize); + Hash hash(inbuf, blocksize); unsigned buf_loc = blocksize; for (unsigned j = start + blocksize; ; ++j) { @@ -397,7 +393,6 @@ void bdelta_done_alg(BDelta_Instance *b) noexcept free(b); } - // Adapted from http://graphics.stanford.edu/~seander/bithacks.html#RoundUpPowerOf2 static unsigned roundUpPowerOf2(unsigned v) noexcept { @@ -527,7 +522,7 @@ static void get_unused_blocks(UnusedRange *unused, unsigned *numunusedptr) noexc { unsigned startPos = nextStartPos; nextStartPos = std::max(startPos, unused[i].p + unused[i].num); - unused[i] = UnusedRange(startPos, unused[i].p < startPos ? 0 : unused[i].p - startPos, unused[i-1].mr, unused[i].mr); + unused[i].set(startPos, unused[i].p < startPos ? 0 : unused[i].p - startPos, unused[i-1].mr, unused[i].mr); } } diff --git a/src/noexcept.h b/src/noexcept.h index dcba14f..5dcecc7 100644 --- a/src/noexcept.h +++ b/src/noexcept.h @@ -1,6 +1,7 @@ #ifndef __NOEXCEPT_H__ #define __NOEXCEPT_H__ +#include #include #include @@ -26,7 +27,7 @@ struct Construct new (new_p + i) T(std::move(old_p[i])); for (; i < new_size; ++i) - new(p + i) T(); + new(new_p + i) T(); } }; @@ -194,9 +195,10 @@ class NoThrowList class iterator { public: - typedef T value_type; + typedef T value_type; typedef T* pointer; typedef T& reference; + typedef void difference_type; typedef std::bidirectional_iterator_tag iterator_category; iterator() noexcept : m_node(nullptr) From a6823df5cca381c10686d55f5b3190d6f3b5868e Mon Sep 17 00:00:00 2001 From: Deamhan Date: Thu, 11 Apr 2019 20:24:56 +0300 Subject: [PATCH 48/54] some minor improvements --- src/bdelta.def | 2 +- src/bpatch.cpp | 4 ++-- src/libbdelta.cpp | 46 ++++++++++++++++------------------------------ 3 files changed, 19 insertions(+), 33 deletions(-) diff --git a/src/bdelta.def b/src/bdelta.def index 55123ff..a2f86b6 100644 --- a/src/bdelta.def +++ b/src/bdelta.def @@ -1,4 +1,4 @@ -LIBRARY bdelta +LIBRARY libbdelta EXPORTS bdelta_init_alg bdelta_done_alg diff --git a/src/bpatch.cpp b/src/bpatch.cpp index fa8260b..3d19e75 100644 --- a/src/bpatch.cpp +++ b/src/bpatch.cpp @@ -31,7 +31,7 @@ static bool copy_bytes_to_file(FILE *infile, FILE *outfile, unsigned numleft) } if (fwrite_unlocked(buf.get(), 1, numread, outfile) != numread) { - printf("Could not write temporary data. Possibly out of space\n"); + printf("Could not write temporary data. Possibly out of space\n"); return false; } numleft -= numread; @@ -79,7 +79,7 @@ int main(int argc, char **argv) printf("unsupported file pointer size\n"); return 1; } - uint32_t size1, size2, nummatches; + uint32_t size1 = 0, size2 = 0, nummatches = 0; bool result = read_value(patchfile.get(), &size1) && read_value(patchfile.get(), &size2) && read_value(patchfile.get(), &nummatches); diff --git a/src/libbdelta.cpp b/src/libbdelta.cpp index 689afdb..1fa540b 100644 --- a/src/libbdelta.cpp +++ b/src/libbdelta.cpp @@ -198,25 +198,6 @@ static unsigned match_backward(BDelta_Instance *b, unsigned p1, unsigned p2, uns template static inline T bdelta_next(T i) noexcept { return ++i; } -// Sort first by location, second by match length (larger matches first) -class comparep -{ -public: - bool operator()(UnusedRange r1, UnusedRange r2) noexcept - { - return ((r1.p != r2.p) ? (r1.p < r2.p) : (r1.num > r2.num)); - } -}; - -class comparemrp2 -{ -public: - bool operator()(UnusedRange r1, UnusedRange r2) noexcept - { - return ((r1.mr->p2 != r2.mr->p2)? (r1.mr->p2 < r2.mr->p2) : (r1.mr->num > r2.mr->num)); - } -}; - class compareMatchP2 { public: @@ -515,10 +496,10 @@ void bdelta_showMatches(BDelta_Instance *b) noexcept printf("\n\n"); } -static void get_unused_blocks(UnusedRange *unused, unsigned *numunusedptr) noexcept +static void get_unused_blocks(UnusedRange *unused, unsigned numunusedptr) noexcept { unsigned nextStartPos = 0; - for (unsigned i = 1; i < *numunusedptr; ++i) + for (unsigned i = 1; i < numunusedptr; ++i) { unsigned startPos = nextStartPos; nextStartPos = std::max(startPos, unused[i].p + unused[i].num); @@ -526,8 +507,6 @@ static void get_unused_blocks(UnusedRange *unused, unsigned *numunusedptr) noexc } } -inline bool isZeroMatch(Match &m) noexcept { return m.num == 0; } - void bdelta_pass(BDelta_Instance *b, unsigned blocksize, unsigned minMatchSize, unsigned maxHoleSize, unsigned flags) noexcept { // Place an empty Match at beginning so we can assume there's a Match to the left of every hole. @@ -560,12 +539,16 @@ void bdelta_pass(BDelta_Instance *b, unsigned blocksize, unsigned minMatchSize, unused2[numunused2++].set(l->p2, l->num, l, l); } - std::sort(unused + 1, unused + numunused, comparep()); // Leave empty match at beginning + // Leave empty match at beginning + std::sort(unused + 1, unused + numunused, [](UnusedRange r1, UnusedRange r2) noexcept + { + return ((r1.p != r2.p) ? (r1.p < r2.p) : (r1.num > r2.num)); + }); - get_unused_blocks(unused, &numunused); - get_unused_blocks(unused2, &numunused2); + get_unused_blocks(unused, numunused); + get_unused_blocks(unused2, numunused2); - if (flags & BDELTA_GLOBAL) + if ((flags & BDELTA_GLOBAL) != 0) { bdelta_pass_2(b, blocksize, minMatchSize, unused, numunused, unused2, numunused2); if (b->errorcode != BDELTA_OK) @@ -573,13 +556,16 @@ void bdelta_pass(BDelta_Instance *b, unsigned blocksize, unsigned minMatchSize, } else { - std::sort(unused + 1, unused + numunused, comparemrp2()); + std::sort(unused + 1, unused + numunused, [](UnusedRange r1, UnusedRange r2) noexcept + { + return ((r1.mr->p2 != r2.mr->p2) ? (r1.mr->p2 < r2.mr->p2) : (r1.mr->num > r2.mr->num)); + }); for (unsigned i = 1; i < numunused; ++i) { UnusedRange u1 = unused[i], u2 = unused2[i]; if (u1.num >= blocksize && u2.num >= blocksize) if (!maxHoleSize || (u1.num <= maxHoleSize && u2.num <= maxHoleSize)) - if (!(flags & BDELTA_SIDES_ORDERED) || (bdelta_next(u1.ml) == u1.mr && bdelta_next(u2.ml) == u2.mr)) + if ((flags & BDELTA_SIDES_ORDERED) == 0 || (bdelta_next(u1.ml) == u1.mr && bdelta_next(u2.ml) == u2.mr)) { bdelta_pass_2(b, blocksize, minMatchSize, &u1, 1, &u2, 1); if (b->errorcode != BDELTA_OK) @@ -592,7 +578,7 @@ void bdelta_pass(BDelta_Instance *b, unsigned blocksize, unsigned minMatchSize, printf("pass (blocksize: %u, matches: %u)\n", blocksize, (unsigned)b->matches.size()); // Get rid of the dummy values we placed at the ends. - b->matches.erase(std::find_if(b->matches.begin(), b->matches.end(), isZeroMatch)); + b->matches.erase(std::find_if(b->matches.begin(), b->matches.end(), [](const Match &m) noexcept { return m.num == 0; })); b->matches.pop_back(); } From 618f26874d205dc94c4c75191d8327010b002a53 Mon Sep 17 00:00:00 2001 From: Deamhan Date: Fri, 12 Apr 2019 16:33:59 +0300 Subject: [PATCH 49/54] MinGW compatibility issue has been fixed & performance has been improved --- makefiles/CMakeLists.txt | 3 ++- src/file.cpp | 5 ++++- src/libbdelta.cpp | 43 ++++++++++++++++++---------------------- 3 files changed, 25 insertions(+), 26 deletions(-) diff --git a/makefiles/CMakeLists.txt b/makefiles/CMakeLists.txt index 7410d8a..f3b9f49 100644 --- a/makefiles/CMakeLists.txt +++ b/makefiles/CMakeLists.txt @@ -163,12 +163,13 @@ ELSE() -funsigned-char -fno-exceptions + -fno-rtti ) SET(CMAKE_CXX_FLAGS "${CMAKE_C_FLAGS} -Werror=delete-non-virtual-dtor -U__STRICT_ANSI__ -fno-operator-names -std=c++17") SET(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -Wimplicit-int -Wmissing-prototypes -Werror=implicit-int") - SET(CMAKE_CXX_FLAGS_RELEASE "-O2 -s -fno-rtti -DNDEBUG") + SET(CMAKE_CXX_FLAGS_RELEASE "-O2 -s -DNDEBUG -fomit-frame-pointer") SET(CMAKE_C_FLAGS_RELEASE ${CMAKE_CXX_FLAGS_RELEASE}) SET(CMAKE_CXX_FLAGS_DEBUG "-O0 -g -D_DEBUG -D__DEBUG__ -DDEBUG_LEVEL=3 -D_GLIBCXX_DEBUG -D_GLIBCXX_DEBUG_PEDANTIC") SET(CMAKE_C_FLAGS_DEBUG ${CMAKE_CXX_FLAGS_DEBUG}) diff --git a/src/file.cpp b/src/file.cpp index 80b02f1..bd9e84e 100644 --- a/src/file.cpp +++ b/src/file.cpp @@ -10,7 +10,10 @@ #ifdef _MSC_VER #define fwrite_unlocked _fwrite_nolock #define fread_unlocked _fread_nolock -#endif // _MSC_VER +#elif __MINGW32__ +#define fwrite_unlocked fwrite +#define fread_unlocked fread +#endif bool fread_fixed(FILE *f, void * _buf, unsigned num_bytes) noexcept { diff --git a/src/libbdelta.cpp b/src/libbdelta.cpp index 1fa540b..3aa8e87 100644 --- a/src/libbdelta.cpp +++ b/src/libbdelta.cpp @@ -198,22 +198,17 @@ static unsigned match_backward(BDelta_Instance *b, unsigned p1, unsigned p2, uns template static inline T bdelta_next(T i) noexcept { return ++i; } -class compareMatchP2 +const auto compareMatchP2 = [](const Match& r1, const Match& r2) noexcept { -public: - bool operator()(Match r1, Match r2) noexcept - { - return ((r1.p2 != r2.p2) ? (r1.p2 < r2.p2) : (r1.num > r2.num)); - } + return ((r1.p2 != r2.p2) ? (r1.p2 < r2.p2) : (r1.num > r2.num)); }; static void addMatch(BDelta_Instance *b, unsigned p1, unsigned p2, unsigned num, MatchListIterator place) noexcept { Match newMatch = Match(p1, p2, num); - compareMatchP2 comp; - while (place != b->matches.begin() && !comp(*place, newMatch)) + while (place != b->matches.begin() && !compareMatchP2(*place, newMatch)) --place; - while (place != b->matches.end() && comp(*place, newMatch)) + while (place != b->matches.end() && compareMatchP2(*place, newMatch)) ++place; if (b->matches.insert(place, newMatch) == nullptr) b->errorcode = BDELTA_MEM_ERROR; @@ -383,7 +378,7 @@ static unsigned roundUpPowerOf2(unsigned v) noexcept return (v + 1); } -static void bdelta_pass_2(BDelta_Instance *b, unsigned blocksize, unsigned minMatchSize, UnusedRange *unused, unsigned numunused, UnusedRange *unused2, unsigned numunused2) noexcept +static void bdelta_pass_2(BDelta_Instance *b, uint32_t blocksize, uint32_t minMatchSize, UnusedRange *unused, uint32_t numunused, UnusedRange *unused2) noexcept { b->access_int = -1; @@ -443,7 +438,7 @@ static void bdelta_pass_2(BDelta_Instance *b, unsigned blocksize, unsigned minMa for (int i = h.numchecksums - 1; i >= 0; --i) h.htable[h.tableIndex(h.checksums[i].cksum)] = &h.checksums[i]; - for (unsigned i = 0; i < numunused2; ++i) + for (unsigned i = 0; i < numunused; ++i) { if (unused2[i].num >= blocksize) { @@ -460,7 +455,7 @@ void bdelta_swap_inputs(BDelta_Instance *b) noexcept std::swap(m.p1, m.p2); std::swap(b->data1_size, b->data2_size); std::swap(b->handle1, b->handle2); - b->matches.sort(compareMatchP2()); + b->matches.sort(compareMatchP2); } void bdelta_clean_matches(BDelta_Instance *b, unsigned flags) noexcept @@ -496,10 +491,10 @@ void bdelta_showMatches(BDelta_Instance *b) noexcept printf("\n\n"); } -static void get_unused_blocks(UnusedRange *unused, unsigned numunusedptr) noexcept +static void get_unused_blocks(UnusedRange *unused, uint32_t numunused) noexcept { unsigned nextStartPos = 0; - for (unsigned i = 1; i < numunusedptr; ++i) + for (unsigned i = 1; i < numunused; ++i) { unsigned startPos = nextStartPos; nextStartPos = std::max(startPos, unused[i].p + unused[i].num); @@ -507,7 +502,7 @@ static void get_unused_blocks(UnusedRange *unused, unsigned numunusedptr) noexce } } -void bdelta_pass(BDelta_Instance *b, unsigned blocksize, unsigned minMatchSize, unsigned maxHoleSize, unsigned flags) noexcept +void bdelta_pass(BDelta_Instance *b, uint32_t blocksize, uint32_t minMatchSize, uint32_t maxHoleSize, uint32_t flags) noexcept { // Place an empty Match at beginning so we can assume there's a Match to the left of every hole. if (b->matches.emplace_front(0, 0, 0) == nullptr) @@ -532,31 +527,31 @@ void bdelta_pass(BDelta_Instance *b, unsigned blocksize, unsigned minMatchSize, UnusedRange *unused = b->bdelta_pass_unused.get(), *unused2 = unused + BUFFER_SIZE; - unsigned numunused = 0, numunused2 = 0; + uint32_t numunused = 0; for (auto l = b->matches.begin(); l != b->matches.end(); ++l) { - unused[numunused++].set(l->p1, l->num, l, l); - unused2[numunused2++].set(l->p2, l->num, l, l); + unused[numunused].set(l->p1, l->num, l, l); + unused2[numunused++].set(l->p2, l->num, l, l); } // Leave empty match at beginning - std::sort(unused + 1, unused + numunused, [](UnusedRange r1, UnusedRange r2) noexcept + std::sort(unused + 1, unused + numunused, [](const UnusedRange& r1, const UnusedRange& r2) noexcept { return ((r1.p != r2.p) ? (r1.p < r2.p) : (r1.num > r2.num)); }); get_unused_blocks(unused, numunused); - get_unused_blocks(unused2, numunused2); + get_unused_blocks(unused2, numunused); if ((flags & BDELTA_GLOBAL) != 0) { - bdelta_pass_2(b, blocksize, minMatchSize, unused, numunused, unused2, numunused2); + bdelta_pass_2(b, blocksize, minMatchSize, unused, numunused, unused2); if (b->errorcode != BDELTA_OK) return; } - else + else { - std::sort(unused + 1, unused + numunused, [](UnusedRange r1, UnusedRange r2) noexcept + std::sort(unused + 1, unused + numunused, [](const UnusedRange& r1, const UnusedRange& r2) noexcept { return ((r1.mr->p2 != r2.mr->p2) ? (r1.mr->p2 < r2.mr->p2) : (r1.mr->num > r2.mr->num)); }); @@ -567,7 +562,7 @@ void bdelta_pass(BDelta_Instance *b, unsigned blocksize, unsigned minMatchSize, if (!maxHoleSize || (u1.num <= maxHoleSize && u2.num <= maxHoleSize)) if ((flags & BDELTA_SIDES_ORDERED) == 0 || (bdelta_next(u1.ml) == u1.mr && bdelta_next(u2.ml) == u2.mr)) { - bdelta_pass_2(b, blocksize, minMatchSize, &u1, 1, &u2, 1); + bdelta_pass_2(b, blocksize, minMatchSize, &u1, 1, &u2); if (b->errorcode != BDELTA_OK) return; } From 0ba557c5ec02f3b239ff7fb32943cd37ed465bfd Mon Sep 17 00:00:00 2001 From: Deamhan Date: Sat, 13 Jul 2019 13:03:42 +0300 Subject: [PATCH 50/54] noexcept.h: comment has been added --- src/noexcept.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/noexcept.h b/src/noexcept.h index 5dcecc7..a6db97e 100644 --- a/src/noexcept.h +++ b/src/noexcept.h @@ -412,7 +412,7 @@ class NoThrowList { Node * next = p->next; p->data.T::~T(); - //deallocate(p); + // we dont't need deallocate() because m_blocks_stack will free memory p = next; } From 9f175cd852e53d352e6c6a05defd4c65ec72acdc Mon Sep 17 00:00:00 2001 From: Deamhan Date: Sat, 13 Jul 2019 13:35:29 +0300 Subject: [PATCH 51/54] noexcept.h: inplace_realloc optimization has been add for _WIN32 --- src/noexcept.h | 77 +++++++++++++++++++++++++++++++++++++------------- 1 file changed, 57 insertions(+), 20 deletions(-) diff --git a/src/noexcept.h b/src/noexcept.h index a6db97e..f3e29a2 100644 --- a/src/noexcept.h +++ b/src/noexcept.h @@ -5,6 +5,34 @@ #include #include +#ifdef _WIN32 +#define NOMINMAX +#include + +static const HANDLE hDefaultHeap = GetProcessHeap(); + +inline void * bdelta_malloc(size_t size) +{ + return HeapAlloc(hDefaultHeap, 0, size); +} + +inline void bdelta_free(void * p) +{ + HeapFree(hDefaultHeap, 0, p); +} + +inline bool inplace_realloc(void * old_p, size_t new_size) +{ + return (HeapReAlloc(hDefaultHeap, HEAP_REALLOC_IN_PLACE_ONLY, old_p, new_size) != nullptr); +} +#else +#define bdelta_malloc malloc +#define bdelta_free free + +// TODO: implement it for linux +constexpr bool inplace_realloc(void * /*old_p*/, size_t /*new_size*/) { return false; } +#endif _WIN32 + template struct Construct { @@ -73,7 +101,7 @@ class NoThrowMemoryStorage T * get() const noexcept { return m_data; } NoThrowMemoryStorage(size_t size) noexcept - : m_data((T*)malloc(size * sizeof(T))) + : m_data((T*)bdelta_malloc(size * sizeof(T))) { if (m_data != nullptr) { @@ -93,7 +121,7 @@ class NoThrowMemoryStorage if (m_data != nullptr) { m_constructor.destruct(m_data, m_capacity); - free(m_data); + bdelta_free(m_data); } } @@ -101,30 +129,39 @@ class NoThrowMemoryStorage { if (m_capacity >= size) { + if (size > m_size) + m_constructor.construct(m_data + m_size, size - m_size); + else if (size < m_size) + m_constructor.destruct(m_data + size, m_size - size); m_size = size; return true; } - T * new_data = (T*)malloc(size * sizeof(T)); - if (new_data == nullptr) + if (m_data != nullptr && inplace_realloc(m_data, size)) + m_constructor.construct(m_data + m_capacity, size - m_capacity); + else { - if (m_data != nullptr) + T * new_data = (T*)bdelta_malloc(size * sizeof(T)); + if (new_data == nullptr) { - m_constructor.destruct(m_data, m_size); - free(m_data); - m_size = 0; - m_capacity = 0; - } + if (m_data != nullptr) + { + m_constructor.destruct(m_data, m_size); + bdelta_free(m_data); + m_size = 0; + m_capacity = 0; + } - return false; - } + return false; + } - if (m_data != nullptr) - m_reallocator.realloc(m_data, m_size, new_data, size, m_constructor); - else - m_constructor.construct(new_data, size); + if (m_data != nullptr) + m_reallocator.realloc(m_data, m_size, new_data, size, m_constructor); + else + m_constructor.construct(new_data, size); - m_data = new_data; + m_data = new_data; + } m_size = size; m_capacity = size; @@ -418,7 +455,7 @@ class NoThrowList void * block = nullptr; while(m_blocks_stack.pop(block)) - free(block); + bdelta_free(block); m_size = 0; m_head = nullptr; @@ -445,10 +482,10 @@ class NoThrowList return nullptr; size_t count = m_allocator_stack.capacity(); - Node * block = (Node*)malloc(sizeof(Node) * count); + Node * block = (Node*)bdelta_malloc(sizeof(Node) * count); if (!m_blocks_stack.push(block)) { - free(block); + bdelta_free(block); return nullptr; } From 18317fecbee7e70d80f410a53b6e4828254d5b54 Mon Sep 17 00:00:00 2001 From: Deamhan Date: Sun, 18 Jan 2026 13:37:07 +0300 Subject: [PATCH 52/54] added minor build fixes --- src/bdelta.cpp | 12 ++++++------ src/libbdelta.cpp | 4 ++-- src/noexcept.h | 2 +- 3 files changed, 9 insertions(+), 9 deletions(-) diff --git a/src/bdelta.cpp b/src/bdelta.cpp index 13587f1..4801dde 100644 --- a/src/bdelta.cpp +++ b/src/bdelta.cpp @@ -29,22 +29,22 @@ static void my_pass(BDelta_Instance *b, unsigned blocksize, unsigned minMatchSiz switch (*bdelta_getError(b)) { case BDELTA_MEM_ERROR: - printf(fatal_alloc); + printf("%s", fatal_alloc); exit(1); case BDELTA_READ_ERROR: - printf(fatal_read); + printf("%s", fatal_read); exit(2); case BDELTA_WRITE_ERROR: - printf(fatal_write); + printf("%s", fatal_write); exit(3); default: bdelta_clean_matches(b, BDELTA_REMOVE_OVERLAP); } } -#define check_write_result(result) if(!(result)) { printf(fatal_read); return 3; } -#define check_read_result(result) if(!(result)) { printf(fatal_write); return 2; } -#define check_alloc(ptr) if (!(ptr)) { printf(fatal_alloc); exit(1); } +#define check_write_result(result) if(!(result)) { printf("%s", fatal_read); return 3; } +#define check_read_result(result) if(!(result)) { printf("%s", fatal_write); return 2; } +#define check_alloc(ptr) if (!(ptr)) { printf("%s", fatal_alloc); exit(1); } int main(int argc, char **argv) { diff --git a/src/libbdelta.cpp b/src/libbdelta.cpp index 3aa8e87..d8d2e93 100644 --- a/src/libbdelta.cpp +++ b/src/libbdelta.cpp @@ -395,7 +395,7 @@ static void bdelta_pass_2(BDelta_Instance *b, uint32_t blocksize, uint32_t minMa return; } - Checksums_Instance h(blocksize, b->bdelta_pass_2_htable.size(), b->bdelta_pass_2_htable.get(), b->bdelta_pass_2_hchecksums.get()); + Checksums_Instance h(blocksize, (unsigned)b->bdelta_pass_2_htable.size(), b->bdelta_pass_2_htable.get(), b->bdelta_pass_2_hchecksums.get()); b->bdelta_pass_2_buf.resize(blocksize); if (!b->bdelta_pass_2_buf.resize(blocksize)) @@ -579,7 +579,7 @@ void bdelta_pass(BDelta_Instance *b, uint32_t blocksize, uint32_t minMatchSize, unsigned bdelta_numMatches(BDelta_Instance *b) noexcept { - return b->matches.size(); + return (unsigned)b->matches.size(); } void bdelta_getMatch(BDelta_Instance *b, unsigned matchNum, unsigned *p1, unsigned *p2, unsigned *num) noexcept diff --git a/src/noexcept.h b/src/noexcept.h index f3e29a2..610ae71 100644 --- a/src/noexcept.h +++ b/src/noexcept.h @@ -31,7 +31,7 @@ inline bool inplace_realloc(void * old_p, size_t new_size) // TODO: implement it for linux constexpr bool inplace_realloc(void * /*old_p*/, size_t /*new_size*/) { return false; } -#endif _WIN32 +#endif // _WIN32 template struct Construct From fc9d7c78b4f990cd5c716078a05d28267b607cd8 Mon Sep 17 00:00:00 2001 From: Deamhan Date: Sun, 18 Jan 2026 13:58:30 +0300 Subject: [PATCH 53/54] fixed memory allocation issue for bpatch --- src/bpatch.cpp | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/src/bpatch.cpp b/src/bpatch.cpp index 3d19e75..6d46e22 100644 --- a/src/bpatch.cpp +++ b/src/bpatch.cpp @@ -85,9 +85,10 @@ int main(int argc, char **argv) && read_value(patchfile.get(), &nummatches); check_read_result(result); - std::unique_ptr copyloc1((uint32_t*)malloc(nummatches + 1), free), - copyloc2((uint32_t*)malloc(nummatches + 1), free), - copynum((uint32_t*)malloc(nummatches + 1), free); + const size_t alloc_size = sizeof(uint32_t) * (nummatches + 1); + std::unique_ptr copyloc1((uint32_t*)malloc(alloc_size), free), + copyloc2((uint32_t*)malloc(alloc_size), free), + copynum((uint32_t*)malloc(alloc_size), free); for (unsigned i = 0; i < nummatches; ++i) { From cbffacce075e412339a9eb5b17e4df4fc1baf2f5 Mon Sep 17 00:00:00 2001 From: Deamhan Date: Wed, 24 Jun 2026 15:02:47 +0300 Subject: [PATCH 54/54] fixed negative relative offset issue --- src/bpatch.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/bpatch.cpp b/src/bpatch.cpp index 6d46e22..5a32853 100644 --- a/src/bpatch.cpp +++ b/src/bpatch.cpp @@ -131,7 +131,7 @@ int main(int argc, char **argv) return -1; } - fseek(ref.get(), copyloc1[i], SEEK_CUR); + fseek(ref.get(), (int32_t)copyloc1[i], SEEK_CUR); if (!copy_bytes_to_file(ref.get(), outfile.get(), copynum[i])) {