diff --git a/.gitignore b/.gitignore index 64a2f62..25c1a25 100644 --- a/.gitignore +++ b/.gitignore @@ -1,9 +1,3 @@ -*.pyc -*.dylib - -build -RELEASE-VERSION -src/bdelta -src/bpatch -src/bdelta.c +*vs*/ +*cppcheck* diff --git a/README b/README index 3b6ede3..036850b 100644 --- a/README +++ b/README @@ -34,18 +34,6 @@ is designed to give a great deal of control over each pass. This functionality is not well-documented, so contact me if you need help. I hope to add more comments to the code in future releases. - -Other Bindings -============== - -A simple interface for accessing the API with Python is included. This -interface is kept up-to-date since I do a lot of work in Python these days. To -istall it, just type "python build.py install" in the root folder. You'll need -Cython to build/install, but Cython is not required after installation. - -See test/py_bindings.py for a usage example. - - Delta File Format ================= diff --git a/makefiles/CMakeLists.txt b/makefiles/CMakeLists.txt new file mode 100644 index 0000000..f3b9f49 --- /dev/null +++ b/makefiles/CMakeLists.txt @@ -0,0 +1,250 @@ +cmake_minimum_required(VERSION 3.6) + +project(bdelta) + +option(BDELTA_PGO "build using PGO instead of PGI (MSVC only)" "OFF") +option(GCC32 "build 32 bit executable instead of 64 bit (GCC only)" "OFF") +option(CXX17 "enable C++17 features" "ON") +option(BIG_ENDIAN_HOST "build for big endian host" "OFF") + +IF (CXX17) + set(CMAKE_CXX_STANDARD 17) +ENDIF() + +IF(MSVC) + set(CMAKE_CONFIGURATION_TYPES "Debug;Release;PGO" CACHE STRING "" FORCE) +ELSE() + set(CMAKE_CONFIGURATION_TYPES "Debug;Release" CACHE STRING "" FORCE) +ENDIF() + +set (COMMON_SOURCES + + ../src/file.h + ../src/file.cpp +) + +set (BDELTA_SOURCES + + ${COMMON_SOURCES} + ../src/bdelta.cpp + ../src/bdelta.h + ../src/checksum.h + ../src/libbdelta.cpp + ../src/noexcept.h +) + +set (BPATCH_SOURCES + + ${COMMON_SOURCES} + ../src/bpatch.cpp +) + +set (BDELTA_STATIC_LIB_SOURCES + + ${COMMON_SOURCES} + ../src/bdelta.h + ../src/checksum.h + ../src/libbdelta.cpp + ../src/noexcept.h +) + +set (BDELTA_SHARED_LIB_SOURCES + + ${BDELTA_STATIC_LIB_SOURCES} +) + +IF(MSVC) + set (BDELTA_SHARED_LIB_SOURCES + + ${BDELTA_SHARED_LIB_SOURCES} + ../src/bdelta.def + ) + + IF(CXX17 AND CMAKE_CXX_COMPILER_VERSION VERSION_LESS 19.15) + message(FATAL_ERROR "Your Visual Studio is too old. Fully updated Visual Studio 2017 or newer is required") + ENDIF() + add_compile_options(/Zi # pdb + /W4 # warning level 4 + /J # use unsigned char + /Gd # use cdecl + # treat warnings as errors + /we4715 # not all control paths return a value + /we4828 # disallow invalid characters + # prinf-like functions: format mismatch + /we4473 # : not enough arguments passed for format string + /we4474 # : too many arguments passed for format string + /we4475 # : length modifier cannot be used with type field character in format specifier + /we4476 # : unknown type field character in format specifier + /we4477 # : format string requires an argument of type , but variadic argument has type + /we4478 # : positional and non-positional placeholders cannot be mixed in the same format string + /we4775 # nonstandard extension used in format string of function + /we4776 # % is not allowed in the format string of function + /we4777 # : format string requires an argument of type , but variadic argument has type + /we4778 # : unterminated format string + # macro arg mismatch + /we4002 # too many actual parameters for macro 'identifier' + /we4003 # not enough actual parameters for macro 'identifier' + /Zc:threadSafeInit- # https://connect.microsoft.com/VisualStudio/feedback/details/1789709/visual-c-2015-runtime-broken-on-windows-server-2003-c-11-magic-statics + /MP # multiprocessor compilation + /utf-8 # utf-8 source & exec + /GF # eliminate duplicate strings + ) + + IF (CXX17) + add_compile_options(/std:c++17) + ENDIF() + + string(REPLACE "/EHsc" "/EHs-c-" CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS}") # disable c++ exceptions + SET(CMAKE_CXX_FLAGS ${CMAKE_C_FLAGS}) + + add_definitions(-D_CRT_SECURE_NO_WARNINGS) + add_definitions(-D_HAS_EXCEPTIONS=0) + + SET(CMAKE_EXE_LINKER_FLAGS "/LARGEADDRESSAWARE /SUBSYSTEM:CONSOLE") + SET(CMAKE_EXE_LINKER_FLAGS_RELEASE "${CMAKE_EXE_LINKER_FLAGS_RELEASE} /LTCG /OPT:REF,ICF /DEBUG") + SET(CMAKE_CXX_FLAGS_RELEASE "${CMAKE_CXX_FLAGS_RELEASE} /MD -DNDEBUG /Ox /Ob2 /Oi /Ot /Oy /GS- /Gy /GR- /GL /Gw /MT") + SET(CMAKE_C_FLAGS_RELEASE ${CMAKE_CXX_FLAGS_RELEASE}) + SET(CMAKE_CXX_FLAGS_DEBUG "${CMAKE_CXX_FLAGS_DEBUG} /MDd /GS /MTd -D_DEBUG") + SET(CMAKE_C_FLAGS_DEBUG ${CMAKE_CXX_FLAGS_DEBUG}) + + set(CMAKE_CXX_FLAGS_PGO ${CMAKE_CXX_FLAGS_RELEASE}) + set(CMAKE_C_FLAGS_PGO ${CMAKE_C_FLAGS_RELEASE}) + + IF (BDELTA_PGO) + set(CMAKE_SHARED_LINKER_FLAGS_PGO "${CMAKE_SHARED_LINKER_FLAGS_RELEASE} /LTCG:PGOptimize") + set(CMAKE_EXE_LINKER_FLAGS_PGO "${CMAKE_EXE_LINKER_FLAGS_PGO} /LTCG:PGOptimize") + ELSE() + set(CMAKE_SHARED_LINKER_FLAGS_PGO "${CMAKE_SHARED_LINKER_FLAGS_RELEASE} /LTCG:PGInstrument") + set(CMAKE_EXE_LINKER_FLAGS_PGO "${CMAKE_EXE_LINKER_FLAGS_PGO} /LTCG:PGInstrument") + ENDIF() + +ELSE() + IF(CXX17 AND CMAKE_COMPILER_IS_GNUCC) + IF(CMAKE_CXX_COMPILER_VERSION VERSION_LESS 8.2) + message(FATAL_ERROR "Your GCC is too old. GCC 8.2 or newer is required") + ENDIF() + ENDIF() + add_compile_options(-pipe + -W + -Wall + + -Wextra + -Wwrite-strings + -Wframe-larger-than=16384 + -Wstack-usage=16384 + -fdiagnostics-show-option + -Wmissing-declarations + -Wredundant-decls + -Wcast-qual + -Wsuggest-attribute=noreturn + -Wsuggest-attribute=format + -Wunused-but-set-variable + -Wunused-but-set-parameter + + -Wframe-larger-than=4096 + + -Wno-multichar + -Wno-strict-aliasing + -Wno-missing-field-initializers + + -Werror=return-type + -Werror=pointer-arith + -Werror=format + -Werror=format-extra-args + -Werror=unused-value + -Werror=sizeof-pointer-memaccess + -Werror=implicit-function-declaration + + -Werror=missing-declarations + -Werror=missing-prototypes + -Werror=reorder + -Werror=declaration-after-statement + -Werror=missing-format-attribute + -funsigned-char + + -fno-exceptions + -fno-rtti + ) + + SET(CMAKE_CXX_FLAGS "${CMAKE_C_FLAGS} -Werror=delete-non-virtual-dtor -U__STRICT_ANSI__ -fno-operator-names -std=c++17") + SET(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -Wimplicit-int -Wmissing-prototypes -Werror=implicit-int") + + SET(CMAKE_CXX_FLAGS_RELEASE "-O2 -s -DNDEBUG -fomit-frame-pointer") + SET(CMAKE_C_FLAGS_RELEASE ${CMAKE_CXX_FLAGS_RELEASE}) + SET(CMAKE_CXX_FLAGS_DEBUG "-O0 -g -D_DEBUG -D__DEBUG__ -DDEBUG_LEVEL=3 -D_GLIBCXX_DEBUG -D_GLIBCXX_DEBUG_PEDANTIC") + SET(CMAKE_C_FLAGS_DEBUG ${CMAKE_CXX_FLAGS_DEBUG}) + + SET(CMAKE_EXE_LINKER_FLAGS_DEBUG "${CMAKE_EXE_LINKER_FLAGS} -g") + SET(CMAKE_SHARED_LINKER_FLAGS_DEBUG "${CMAKE_SHARED_LINKER_FLAGS} -g") + + SET(CMAKE_EXE_LINKER_FLAGS_RELEASE "${CMAKE_EXE_LINKER_FLAGS} -s") + SET(CMAKE_SHARED_LINKER_FLAGS_RELEASE "${CMAKE_SHARED_LINKER_FLAGS} -s") + + SET(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -static-libgcc -static-libstdc++") + SET(CMAKE_SHARED_LINKER_FLAGS "${CMAKE_SHARED_LINKER_FLAGS} -static-libgcc -static-libstdc++") + +IF (GCC32) + add_compile_options (-m32) + SET(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -m32") + SET(CMAKE_SHARED_LINKER_FLAGS "${CMAKE_SHARED_LINKER_FLAGS} -m32") +ENDIF() +ENDIF() + +IF (CXX17) + add_definitions(-DUSE_CXX17) +ENDIF() + +add_executable( + bdelta + + ${BDELTA_SOURCES} +) + +add_executable( + bpatch + + ${BPATCH_SOURCES} +) + +add_library ( + libbdelta-static + STATIC + + ${BDELTA_STATIC_LIB_SOURCES} +) + +add_library ( + libbdelta-shared + SHARED + + ${BDELTA_SHARED_LIB_SOURCES} +) + +SET_TARGET_PROPERTIES( + libbdelta-static + PROPERTIES PREFIX "" + OUTPUT_NAME libbdelta + PDB_NAME libbdelta-static +) + +SET_TARGET_PROPERTIES( + libbdelta-shared + PROPERTIES PREFIX "" + OUTPUT_NAME libbdelta + PDB_NAME libbdelta-shared +) + +IF (NOT MSVC) + IF (CXX17) + SET ( + LIBS + + stdc++fs + ) + ENDIF() + + target_link_libraries(bdelta ${LIBS}) + target_link_libraries(bpatch ${LIBS}) +ENDIF() + +set_property(DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR} PROPERTY VS_STARTUP_PROJECT bdelta) diff --git a/makefiles/vs90/bdelta_python.vcproj b/makefiles/vs90/bdelta_python.vcproj deleted file mode 100644 index 6fee8e2..0000000 --- a/makefiles/vs90/bdelta_python.vcproj +++ /dev/null @@ -1,235 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - diff --git a/setup.py b/setup.py deleted file mode 100644 index 3a792a1..0000000 --- a/setup.py +++ /dev/null @@ -1,22 +0,0 @@ -from distutils.core import setup, Extension -from Cython.Distutils import build_ext -from version import get_git_version -import sys - -bdelta_module = Extension( - "bdelta", - ["src/bdelta.pyx", "src/libbdelta.cpp"], - define_macros = [('TOKEN_SIZE', '2')], - extra_compile_args = ['/EHsc'] if sys.platform == 'win32' else None -) - -setup( - name = 'BDelta', - version = get_git_version(), - description = 'Python Bindings for BDelta', - author = 'John Whitney', - author_email = 'jjw@deltup.org', - url = 'http://bdelta.org', - cmdclass = {'build_ext': build_ext}, - ext_modules = [bdelta_module] -) diff --git a/src/Makefile b/src/Makefile deleted file mode 100644 index 65fb88d..0000000 --- a/src/Makefile +++ /dev/null @@ -1,45 +0,0 @@ -DESTDIR= -PREFIX ?= /usr -BINDIR ?= $(PREFIX)/bin -LIBDIR ?= ${PREFIX}/lib -CXXFLAGS += -O2 - -ifeq ($(shell uname -s),Darwin) - SHAREDLIB := libbdelta.dylib -else - SHAREDLIB := libbdelta.so -endif - -ALL_TARGETS = bpatch $(SHAREDLIB) bdelta - -all: $(ALL_TARGETS) - -libbdelta.a: libbdelta.cpp compatibility.h checksum.h file.h - $(CXX) -c $(CXXFLAGS) $< -o $<.o - $(AR) rs $@ $<.o - -libbdelta.so: libbdelta.cpp compatibility.h checksum.h file.h - $(CXX) -shared -fPIC $(CXXFLAGS) $(LDFLAGS) $< -o $@ - -libbdelta.dylib: libbdelta.cpp compatibility.h checksum.h file.h - $(CXX) -dynamiclib $(CXXFLAGS) $< -o $@ - -bdelta: bdelta.cpp bdelta.h compatibility.h file.h $(SHAREDLIB) - $(CXX) $< -o $@ $(CXXFLAGS) $(LDFLAGS) -L. -lbdelta - -bpatch: bpatch.cpp compatibility.h bdelta.h file.h - -% : %.cpp - $(CXX) $< -o $@ $(CXXFLAGS) $(LDFLAGS) - -install: libbdelta.so bdelta bpatch - mkdir -p $(DESTDIR)$(BINDIR) - mkdir -p $(DESTDIR)$(LIBDIR) - install -m 755 libbdelta.so $(DESTDIR)$(LIBDIR) - install -m 755 bdelta $(DESTDIR)$(BINDIR) - install -m 755 bpatch $(DESTDIR)$(BINDIR) - -clean: - -rm $(ALL_TARGETS) - -.PHONY: clean diff --git a/src/bdelta.cpp b/src/bdelta.cpp index 6f68846..4801dde 100644 --- a/src/bdelta.cpp +++ b/src/bdelta.cpp @@ -2,191 +2,206 @@ * License, v. 2.0. If a copy of the MPL was not distributed with this * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ +#include +#include +#include #include #include #include #include "bdelta.h" #include "file.h" -#include "compatibility.h" -const void *f_read(void *f, void *buf, unsigned place, unsigned num) { - fseek((FILE *)f, place, SEEK_SET); - fread_fixed((FILE *)f, buf, num); - return buf; +static const void * f_read(void *f, void *buf, unsigned place, unsigned num, BDelta_Instance * /*b*/, BDELTA_RESULT& result) +{ + fseek((FILE *)f, place, SEEK_SET); + result = fread_fixed((FILE *)f, buf, num) ? BDELTA_OK : BDELTA_READ_ERROR; + return buf; } -const void *m_read(void *f, void * buf, unsigned place, unsigned num) { - if (0) { - /* - * BDelta uses only returned pointer - * and does not modify it's contents. - * - * But bugs happen. - */ - memcpy (buf, (char*)f + place, num); - return buf; - } - return (const char*)f + place; +static const char * fatal_alloc = "FATAL: unable to allocate memory\n"; +static const char * fatal_read = "FATAL: unable to read file\n"; +static const char * fatal_write = "FATAL: unable to write file\n"; + +static void my_pass(BDelta_Instance *b, unsigned blocksize, unsigned minMatchSize, unsigned flags) +{ + bdelta_pass(b, blocksize, minMatchSize, 0, flags); + switch (*bdelta_getError(b)) + { + case BDELTA_MEM_ERROR: + printf("%s", fatal_alloc); + exit(1); + case BDELTA_READ_ERROR: + printf("%s", fatal_read); + exit(2); + case BDELTA_WRITE_ERROR: + printf("%s", fatal_write); + exit(3); + default: + bdelta_clean_matches(b, BDELTA_REMOVE_OVERLAP); + } } -void my_pass(BDelta_Instance *b, unsigned blocksize, unsigned minMatchSize, unsigned flags) { - bdelta_pass(b, blocksize, minMatchSize, 0, flags); - bdelta_clean_matches(b, BDELTA_REMOVE_OVERLAP); -} - -int main(int argc, char **argv) { - try { - bool all_ram_mode = false; - char * m1 = NULL; - char * m2 = NULL; - - if (argc > 1 && strcmp(argv[1], "--all-in-ram") == 0) - { - all_ram_mode = true; - --argc; - ++argv; - } - if (argc != 4) { - printf("usage: bdelta [--all-in-ram] \n"); - printf("needs two files to compare + output file:\n"); - exit(1); - } - if (!fileExists(argv[1]) || !fileExists(argv[2])) { - printf("one of the input files does not exist\n"); - exit(1); - } - unsigned size = getLenOfFile(argv[1]); - unsigned size2 = getLenOfFile(argv[2]); - FILE *f1 = fopen(argv[1], "rb"), - *f2 = fopen(argv[2], "rb"); - - BDelta_Instance *b; - - if (all_ram_mode) - { - m1 = new char[size]; - m2 = new char[size2]; - fread_fixed(f1, m1, size); - fread_fixed(f2, m2, size2); - - b = bdelta_init_alg(size, size2, m_read, m1, m2, 1); - } - else - { - b = bdelta_init_alg(size, size2, f_read, f1, f2, 1); - } - int nummatches; - - // List of primes for reference. Taken from Wikipedia. - // 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 - // 1-20 2 3 5 7 11 13 17 19 23 29 31 37 41 43 47 53 59 61 67 71 - // 21-40 73 79 83 89 97 101 103 107 109 113 127 131 137 139 149 151 157 163 167 173 - // 41-60 179 181 191 193 197 199 211 223 227 229 233 239 241 251 257 263 269 271 277 281 - // 61-80 283 293 307 311 313 317 331 337 347 349 353 359 367 373 379 383 389 397 401 409 - // 81-100 419 421 431 433 439 443 449 457 461 463 467 479 487 491 499 503 509 521 523 541 - // 101-120 547 557 563 569 571 577 587 593 599 601 607 613 617 619 631 641 643 647 653 659 - // 121-140 661 673 677 683 691 701 709 719 727 733 739 743 751 757 761 769 773 787 797 809 - // 141-160 811 821 823 827 829 839 853 857 859 863 877 881 883 887 907 911 919 929 937 941 - // 161-180 947 953 967 971 977 983 991 997 - - my_pass(b, 997, 1994, 0); - my_pass(b, 503, 1006, 0); - my_pass(b, 127, 254, 0); - my_pass(b, 31, 62, 0); - my_pass(b, 7, 14, 0); - my_pass(b, 5, 10, 0); - my_pass(b, 3, 6, 0); - my_pass(b, 13, 26, BDELTA_GLOBAL); - my_pass(b, 7, 14, 0); - my_pass(b, 5, 10, 0); - - nummatches = bdelta_numMatches(b); - - unsigned * copyloc1 = new unsigned[nummatches + 1]; - unsigned * copyloc2 = new unsigned[nummatches + 1]; - unsigned * copynum = new unsigned[nummatches + 1]; - - FILE *fout = fopen(argv[3], "wb"); - if (!fout) { - printf("couldn't open output file\n"); - exit(1); - } - - const char *magic = "BDT"; - fwrite_fixed(fout, magic, 3); - unsigned short version = 1; - write_word(fout, version); - unsigned char intsize = 4; - fwrite_fixed(fout, &intsize, 1); - write_dword(fout, size); - write_dword(fout, size2); - write_dword(fout, nummatches); - - unsigned lastp1 = 0, - lastp2 = 0; - for (int i = 0; i < nummatches; ++i) { - unsigned p1, p2, num; - bdelta_getMatch(b, i, &p1, &p2, &num); - // printf("%*x, %*x, %*x, %*x\n", 10, p1, 10, p2, 10, num, 10, p2-lastp2); - copyloc1[i] = p1 - lastp1; - write_dword(fout, copyloc1[i]); - copyloc2[i] = p2 - lastp2; - write_dword(fout, copyloc2[i]); - copynum[i] = num; - write_dword(fout, copynum[i]); - lastp1 = p1 + num; - lastp2 = p2 + num; - } - if (size2 != lastp2) { - copyloc1[nummatches] = 0; copynum[nummatches] = 0; - copyloc2[nummatches] = size2 - lastp2; - ++nummatches; - } - -// write_unsigned_list(adds, nummatches+1, fout); -// write_unsigned_list(copynum, nummatches, fout); -// write_signed_list(copyloc, nummatches, fout); - -// fwrite(copyloc1, 4, nummatches, fout); -// fwrite(copyloc2, 4, nummatches, fout); -// fwrite(copynum, 4, nummatches, fout); - unsigned fp = 0; - for (int i = 0; i < nummatches; ++i) { - unsigned num = copyloc2[i]; - while (num > 0) { - unsigned towrite = (num > 4096) ? 4096 : num; - unsigned char buf[4096]; - const void * b; - if (all_ram_mode) - b = m_read(m2, buf, fp, towrite); - else - b = f_read(f2, buf, fp, towrite); - fwrite_fixed(fout, b, towrite); - num -= towrite; - fp += towrite; - } - // fp+=copyloc2[i]; - if (i != nummatches) fp += copynum[i]; - } - - fclose(fout); - - bdelta_done_alg(b); - - delete [] m1; - delete [] m2; - - fclose(f1); - fclose(f2); - - delete [] copynum; - delete [] copyloc2; - delete [] copyloc1; - - } catch (const char * desc){ - fprintf (stderr, "FATAL: %s\n", desc); - return -1; - } - return 0; +#define check_write_result(result) if(!(result)) { printf("%s", fatal_read); return 3; } +#define check_read_result(result) if(!(result)) { printf("%s", fatal_write); return 2; } +#define check_alloc(ptr) if (!(ptr)) { printf("%s", fatal_alloc); exit(1); } + +int main(int argc, char **argv) +{ + std::unique_ptr m1(nullptr, free), m2(nullptr, free); + + const bool all_ram_mode = (argc > 1 && strcmp(argv[1], "--all-in-ram") == 0); + if (all_ram_mode) + { + --argc; + ++argv; + } + if (argc != 4) + { + printf("usage: bdelta [--all-in-ram] \n"); + printf("needs two files to compare + output file:\n"); + exit(1); + } + if (!fileExists(argv[1]) || !fileExists(argv[2])) + { + printf("one of the input files does not exist\n"); + exit(1); + } + uint32_t size = (uint32_t)getLenOfFile(argv[1]); + uint32_t size2 = (uint32_t)getLenOfFile(argv[2]); + std::unique_ptr f1(fopen(argv[1], "rb"), fclose); + if (!f1) + { + printf("unable to open file %s\n", argv[1]); + exit(1); + } + + + std::unique_ptr f2(fopen(argv[2], "rb"), fclose); + if (!f2) + { + printf("unable to open file %s\n", argv[2]); + exit(1); + } + + + BDelta_Instance * b = nullptr; + + if (all_ram_mode) + { + m1.reset((uint8_t*)malloc(size)); + m2.reset((uint8_t*)malloc(size2)); + check_alloc(m1 && m2); + fread_fixed(f1.get(), m1.get(), size); + fread_fixed(f2.get(), m2.get(), size2); + + b = bdelta_init_alg(size, size2, nullptr, m1.get(), m2.get(), 1); + } + else + b = bdelta_init_alg(size, size2, f_read, f1.get(), f2.get(), 1); + + check_alloc(b); + + std::unique_ptr b_holder(b, bdelta_done_alg); + + int nummatches = 0; + + // List of primes for reference. Taken from Wikipedia. + // 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 + // 1-20 2 3 5 7 11 13 17 19 23 29 31 37 41 43 47 53 59 61 67 71 + // 21-40 73 79 83 89 97 101 103 107 109 113 127 131 137 139 149 151 157 163 167 173 + // 41-60 179 181 191 193 197 199 211 223 227 229 233 239 241 251 257 263 269 271 277 281 + // 61-80 283 293 307 311 313 317 331 337 347 349 353 359 367 373 379 383 389 397 401 409 + // 81-100 419 421 431 433 439 443 449 457 461 463 467 479 487 491 499 503 509 521 523 541 + // 101-120 547 557 563 569 571 577 587 593 599 601 607 613 617 619 631 641 643 647 653 659 + // 121-140 661 673 677 683 691 701 709 719 727 733 739 743 751 757 761 769 773 787 797 809 + // 141-160 811 821 823 827 829 839 853 857 859 863 877 881 883 887 907 911 919 929 937 941 + // 161-180 947 953 967 971 977 983 991 997 + + my_pass(b, 997, 1994, 0); + my_pass(b, 503, 1006, 0); + my_pass(b, 127, 254, 0); + my_pass(b, 31, 62, 0); + my_pass(b, 7, 14, 0); + my_pass(b, 5, 10, 0); + my_pass(b, 3, 6, 0); + my_pass(b, 13, 26, BDELTA_GLOBAL); + my_pass(b, 7, 14, 0); + my_pass(b, 5, 10, 0); + + nummatches = bdelta_numMatches(b); + + const size_t mem_len = (nummatches + 1) * sizeof(uint32_t); + std::unique_ptr copyloc1((uint32_t*)malloc(mem_len), free), + copyloc2((uint32_t*)malloc(mem_len), free), + copynum((uint32_t*)malloc(mem_len), free); + + check_alloc(copyloc1 && copyloc2 && copynum); + + std::unique_ptr fout(fopen(argv[3], "wb"), fclose); + if (!fout) + { + printf("couldn't open output file\n"); + exit(1); + } + constexpr size_t fout_buffer_size = 256 * 1024; + setvbuf(fout.get(), nullptr, _IOFBF, fout_buffer_size); + + const char * magic = "BDT"; + check_write_result(fwrite_fixed(fout.get(), magic, 3)); + uint16_t version = 1; + check_write_result(write_value(fout.get(), &version)); + uint8_t intsize = 4; + check_write_result(write_value(fout.get(), &intsize)); + bool result = write_value(fout.get(), &size) + && write_value(fout.get(), &size2) + && write_value(fout.get(), &nummatches); + check_write_result(result); + + unsigned lastp1 = 0, lastp2 = 0; + for (int i = 0; i < nummatches; ++i) + { + unsigned p1 = 0, p2 = 0, num = 0; + bdelta_getMatch(b, i, &p1, &p2, &num); + copyloc1[i] = p1 - lastp1; + result = write_value(fout.get(), ©loc1[i]); + copyloc2[i] = p2 - lastp2; + result = result && write_value(fout.get(), ©loc2[i]); + copynum[i] = num; + result = result && write_value(fout.get(), ©num[i]); + check_write_result(result); + lastp1 = p1 + num; + lastp2 = p2 + num; + } + if (size2 != lastp2) + { + copyloc1[nummatches] = 0; + copynum[nummatches] = 0; + copyloc2[nummatches] = size2 - lastp2; + ++nummatches; + } + + unsigned fp = 0; + constexpr size_t WRITE_BUFFER_SIZE = 1024 * 1024; + std::unique_ptr write_buffer((uint8_t*)malloc(WRITE_BUFFER_SIZE), free); + check_alloc(write_buffer); + BDELTA_RESULT * res = bdelta_getError(b); + for (int i = 0; i < nummatches; ++i) + { + unsigned num = copyloc2[i]; + while (num > 0) + { + unsigned towrite = std::min(num, WRITE_BUFFER_SIZE); + const void * block = all_ram_mode ? (const void *)((char*)m2.get() + fp) : f_read(f2.get(), write_buffer.get(), fp, towrite, b, *res); + check_read_result(*res == BDELTA_OK); + check_write_result(fwrite_fixed(fout.get(), block, towrite)); + num -= towrite; + fp += towrite; + } + + if (i != nummatches) + fp += copynum[i]; + } + + return 0; } diff --git a/src/bdelta.def b/src/bdelta.def new file mode 100644 index 0000000..a2f86b6 --- /dev/null +++ b/src/bdelta.def @@ -0,0 +1,11 @@ +LIBRARY libbdelta +EXPORTS + bdelta_init_alg + bdelta_done_alg + bdelta_pass + bdelta_swap_inputs + bdelta_clean_matches + bdelta_numMatches + bdelta_getMatch + bdelta_getError + bdelta_showMatches diff --git a/src/bdelta.h b/src/bdelta.h index a9aee38..6f7430b 100644 --- a/src/bdelta.h +++ b/src/bdelta.h @@ -1,35 +1,45 @@ /* This Source Code Form is subject to the terms of the Mozilla Public * License, v. 2.0. If a copy of the MPL was not distributed with this * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ +#ifndef __BDELTA_H__ +#define __BDELTA_H__ #ifdef __cplusplus extern "C" { +#else +#define noexcept #endif // __cplusplus +enum BDELTA_RESULT { + BDELTA_OK = 0, + BDELTA_MEM_ERROR = -1, + BDELTA_READ_ERROR = -2, + BDELTA_WRITE_ERROR = -3 +}; + typedef struct _BDelta_Instance BDelta_Instance; // Callback function must return a pointer to the data requested. // A "fill and forget" buffer is provided, but can be ignored, so // long as the data persists throughout the life of bdelta_pass(). -typedef const void *(*bdelta_readCallback)(void *handle, void *buf, unsigned place, unsigned num); +typedef const void *(*bdelta_readCallback)(void *handle, void *buf, unsigned place, unsigned num, BDelta_Instance * b, BDELTA_RESULT& result); -BDelta_Instance *bdelta_init_alg(unsigned data1_size, unsigned data2_size, - bdelta_readCallback cb, void *handle1, void *handle2, - unsigned tokenSize); -void bdelta_done_alg(BDelta_Instance *b); +BDelta_Instance * bdelta_init_alg(unsigned data1_size, unsigned data2_size, + bdelta_readCallback cb, void *handle1, void *handle2, + unsigned tokenSize) noexcept; +void bdelta_done_alg(BDelta_Instance *b) noexcept; -void bdelta_pass(BDelta_Instance *b, unsigned blockSize, unsigned minMatchSize, unsigned maxHoleSize, unsigned flags); +void bdelta_pass(BDelta_Instance *b, unsigned blockSize, unsigned minMatchSize, unsigned maxHoleSize, unsigned flags) noexcept; -void bdelta_swap_inputs(BDelta_Instance *b); -void bdelta_clean_matches(BDelta_Instance *b, unsigned flags); +void bdelta_swap_inputs(BDelta_Instance *b) noexcept; +void bdelta_clean_matches(BDelta_Instance *b, unsigned flags) noexcept; -unsigned bdelta_numMatches(BDelta_Instance *b); +unsigned bdelta_numMatches(BDelta_Instance *b) noexcept; -void bdelta_getMatch(BDelta_Instance *b, unsigned matchNum, - unsigned *p1, unsigned *p2, unsigned *num); +void bdelta_getMatch(BDelta_Instance *b, unsigned matchNum, unsigned *p1, unsigned *p2, unsigned *num) noexcept; -int bdelta_getError(BDelta_Instance *b); -void bdelta_showMatches(BDelta_Instance *b); +BDELTA_RESULT * bdelta_getError(BDelta_Instance *b) noexcept; +void bdelta_showMatches(BDelta_Instance *b) noexcept; // Flags for bdelta_pass() #define BDELTA_GLOBAL 1 @@ -38,12 +48,8 @@ void bdelta_showMatches(BDelta_Instance *b); // Flags for bdelta_clean_matches() #define BDELTA_REMOVE_OVERLAP 1 -enum BDELTA_RESULT { - BDELTA_OK = 0, - BDELTA_MEM_ERROR = -1, - BDELTA_READ_ERROR = -2 -}; - #ifdef __cplusplus } #endif // __cplusplus + +#endif // __BDELTA_H__ diff --git a/src/bdelta.pyx b/src/bdelta.pyx deleted file mode 100644 index 8affe78..0000000 --- a/src/bdelta.pyx +++ /dev/null @@ -1,61 +0,0 @@ -# This Source Code Form is subject to the terms of the Mozilla Public -# License, v. 2.0. If a copy of the MPL was not distributed with this -# file, You can obtain one at http://mozilla.org/MPL/2.0/. - -cdef extern from "bdelta.h": - ctypedef struct BDelta_Instance: - pass - - ctypedef void *const_void_ptr "const void*" - ctypedef void *(*bdelta_readCallback)(void *handle, void *buf, unsigned place, unsigned num) - BDelta_Instance *bdelta_init_alg(unsigned data1_size, unsigned data2_size, - bdelta_readCallback cb, void *handle1, void *handle2, - unsigned tokenSize) - void bdelta_done_alg(BDelta_Instance *b) - - void bdelta_pass(BDelta_Instance *b, unsigned blockSize, unsigned minMatchSize, unsigned maxHoleSize, unsigned flags) - - void bdelta_swap_inputs(BDelta_Instance *b) - void bdelta_clean_matches(BDelta_Instance *b, unsigned flags) - - unsigned bdelta_numMatches(BDelta_Instance *b) - - void bdelta_getMatch(BDelta_Instance *b, unsigned matchNum, - unsigned *p1, unsigned *p2, unsigned *num) - - int bdelta_getError(BDelta_Instance *b) - void bdelta_showMatches(BDelta_Instance *b) - - cdef enum PassFlags: - BDELTA_GLOBAL, - BDELTA_SIDES_ORDERED - cdef enum CleanFlags: - BDELTA_REMOVE_OVERLAP - -cdef const_void_ptr readCallback(void *handle, void *buf, unsigned place, unsigned num): - cdef char *str = handle - return str + ((place + 1) * 2); - -cdef class BDelta: - cdef BDelta_Instance *_b - cdef bytes str1, str2 - - def __cinit__(self, str1, str2): - self.str1 = str1.encode('UTF-16') - self.str2 = str2.encode('UTF-16') - self._b = bdelta_init_alg(len(str1), len(str2), readCallback, self.str1, self.str2, 2) - - def __dealloc__(self): - self.str1 = None - self.str2 = None - bdelta_done_alg(self._b) - - def b_pass(self, blockSize, minMatchSize, maxHoleSize, globalScope = False, sidesOrdered = False): - bdelta_pass(self._b, blockSize, minMatchSize, maxHoleSize, - (BDELTA_GLOBAL if globalScope else 0) | (BDELTA_SIDES_ORDERED if sidesOrdered else 0)) - - def matches(self): - cdef unsigned p1, p2, num - for i in xrange(bdelta_numMatches(self._b)): - bdelta_getMatch(self._b, i, &p1, &p2, &num) - yield (int(p1), int(p2), int(num)) \ No newline at end of file diff --git a/src/bpatch.cpp b/src/bpatch.cpp index 8817ce6..5a32853 100644 --- a/src/bpatch.cpp +++ b/src/bpatch.cpp @@ -2,103 +2,143 @@ * License, v. 2.0. If a copy of the MPL was not distributed with this * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ +#include +#include +#include #include +#include #include + #include "file.h" -#include "compatibility.h" - -bool copy_bytes_to_file(FILE *infile, FILE *outfile, unsigned numleft) { - size_t numread; - do { - char buf[1024]; - numread = fread(buf, 1, numleft > 1024 ? 1024 : numleft, infile); - if (fwrite(buf, 1, numread, outfile) != numread) { - printf("Could not write temporary data. Possibly out of space\n"); - return false; - } - numleft -= numread; - } while (numleft && !(numread < 1024 && numleft)); - return (numleft == 0); + +#ifdef _MSC_VER +#define fread_unlocked _fread_nolock +#define fwrite_unlocked _fwrite_nolock +#endif + +static bool copy_bytes_to_file(FILE *infile, FILE *outfile, unsigned numleft) +{ + const size_t BUFFER_SIZE = 65536; + std::unique_ptr buf((uint8_t*)malloc(BUFFER_SIZE), free); + while (numleft != 0) + { + const size_t len = std::min(BUFFER_SIZE, numleft); + size_t numread = fread_unlocked(buf.get(), 1, len, infile); + if (len != numread) + { + printf("Could not read data\n"); + return false; + } + if (fwrite_unlocked(buf.get(), 1, numread, outfile) != numread) + { + printf("Could not write temporary data. Possibly out of space\n"); + return false; + } + numleft -= numread; + } + + return true; } -int main(int argc, char **argv) { - try { - if (argc != 4) { - printf("usage: bpatch \n"); - printf("needs a reference file, file to output, and patchfile:\n"); - return 1; - } - - if (!fileExists(argv[1]) || !fileExists(argv[3])) { - printf("one of the input files does not exist\n"); - return 1; - } - - FILE *patchfile = fopen(argv[3], "rb"); - char magic[3]; - fread_fixed(patchfile, magic, 3); - if (strncmp(magic, "BDT", 3)) { - printf("Given file is not a recognized patchfile\n"); - return 1; - } - unsigned short version = read_word(patchfile); - if (version != 1) { - printf("unsupported patch version\n"); - return 1; - } - char intsize; - fread_fixed(patchfile, &intsize, 1); - if (intsize != 4) { - printf("unsupported file pointer size\n"); - return 1; - } - unsigned size1 = read_dword(patchfile), - size2 = read_dword(patchfile); - - unsigned nummatches = read_dword(patchfile); - - unsigned * copyloc1 = new unsigned[nummatches + 1]; - unsigned * copyloc2 = new unsigned[nummatches + 1]; - unsigned * copynum = new unsigned[nummatches + 1]; - - for (unsigned i = 0; i < nummatches; ++i) { - copyloc1[i] = read_dword(patchfile); - copyloc2[i] = read_dword(patchfile); - copynum[i] = read_dword(patchfile); - size2 -= copyloc2[i] + copynum[i]; - } - if (size2) { - copyloc1[nummatches] = 0; copynum[nummatches] = 0; - copyloc2[nummatches] = size2; - ++nummatches; - } - - FILE *ref = fopen(argv[1], "rb"); - FILE *outfile = fopen(argv[2], "wb"); - - for (unsigned i = 0; i < nummatches; ++i) { - if (!copy_bytes_to_file(patchfile, outfile, copyloc2[i])) { - printf("Error. patchfile is truncated\n"); - return -1; - } - - int copyloc = copyloc1[i]; - fseek(ref, copyloc, SEEK_CUR); - - if (!copy_bytes_to_file(ref, outfile, copynum[i])) { - printf("Error while copying from reference file\n"); - return -1; - } - } - - delete [] copynum; - delete [] copyloc2; - delete [] copyloc1; - - } catch (const char * desc){ - fprintf (stderr, "FATAL: %s\n", desc); - return -1; - } - - return 0; +#define check_read_result(result) if(!(result)) { printf("FATAL: I/O error on reading\n"); return 2; } + +int main(int argc, char **argv) +{ + if (argc != 4) + { + printf("usage: bpatch \n"); + printf("needs a reference file, file to output, and patchfile:\n"); + return 1; + } + + if (!fileExists(argv[1]) || !fileExists(argv[3])) + { + printf("one of the input files does not exist\n"); + return 1; + } + + std::unique_ptr patchfile(fopen(argv[3], "rb"), fclose); + char magic[3]; + check_read_result(fread_fixed(patchfile.get(), magic, 3)); + if (strncmp(magic, "BDT", 3)) + { + printf("Given file is not a recognized patchfile\n"); + return 1; + } + uint16_t version = 0; + uint8_t intsize = 0; + check_read_result(read_value(patchfile.get(), &version)); + if (version != 1) + { + printf("unsupported patch version\n"); + return 1; + } + check_read_result(read_value(patchfile.get(), &intsize)); + if (intsize != 4) + { + printf("unsupported file pointer size\n"); + return 1; + } + uint32_t size1 = 0, size2 = 0, nummatches = 0; + bool result = read_value(patchfile.get(), &size1) + && read_value(patchfile.get(), &size2) + && read_value(patchfile.get(), &nummatches); + check_read_result(result); + + const size_t alloc_size = sizeof(uint32_t) * (nummatches + 1); + std::unique_ptr copyloc1((uint32_t*)malloc(alloc_size), free), + copyloc2((uint32_t*)malloc(alloc_size), free), + copynum((uint32_t*)malloc(alloc_size), free); + + for (unsigned i = 0; i < nummatches; ++i) + { + result = read_value(patchfile.get(), ©loc1[i]) + && read_value(patchfile.get(), ©loc2[i]) + && read_value(patchfile.get(), ©num[i]); + check_read_result(result); + size2 -= copyloc2[i] + copynum[i]; + } + if (size2) + { + copyloc1[nummatches] = 0; + copynum[nummatches] = 0; + copyloc2[nummatches] = size2; + ++nummatches; + } + + std::unique_ptr ref(fopen(argv[1], "rb"), fclose); + if (!ref) + { + printf("Error: unable to open file %s\n", argv[1]); + return -1; + } + + + std::unique_ptr outfile(fopen(argv[2], "wb"), fclose); + if (!outfile) + { + printf("Error: unable to open file %s\n", argv[2]); + return -1; + } + constexpr size_t outfile_buffer_size = 256 * 1024; + setvbuf(outfile.get(), nullptr, _IOFBF, outfile_buffer_size); + + for (unsigned i = 0; i < nummatches; ++i) + { + if (!copy_bytes_to_file(patchfile.get(), outfile.get(), copyloc2[i])) + { + printf("Error: patchfile is truncated\n"); + return -1; + } + + fseek(ref.get(), (int32_t)copyloc1[i], SEEK_CUR); + + if (!copy_bytes_to_file(ref.get(), outfile.get(), copynum[i])) + { + printf("Error while copying from reference file\n"); + return -1; + } + } + + return 0; } diff --git a/src/checksum.h b/src/checksum.h index 40b9118..65fab4e 100755 --- a/src/checksum.h +++ b/src/checksum.h @@ -2,47 +2,54 @@ * License, v. 2.0. If a copy of the MPL was not distributed with this * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ -struct Hash { -public: - typedef uint64_t Value; - Hash() {} - Hash(const Token *buf, unsigned blocksize) { - value = 0; - for (unsigned num = 0; num < blocksize; ++num) - advance_add(buf[num]); - oldCoefficient = powHash(multiplyAmount, blocksize); - } - void advance(Token out, Token in) { - advance_remove(out); - advance_add(in); - } - static unsigned modulo(Value hash, unsigned d) { - // Assumes d is power of 2. - return hash & (d - 1); - } - Value getValue() {return value >> extraProcBits;} +#ifndef __CHECKSUM_H__ +#define __CHECKSUM_H__ + +struct Hash +{ + typedef uint64_t Value; + Hash(const Token * buf, unsigned blocksize) noexcept + { + value = 0; + for (unsigned num = 0; num < blocksize; ++num) + advance_add(buf[num]); + oldCoefficient = powHash(multiplyAmount, blocksize); + } + void advance(Token out, Token in) noexcept + { + advance_remove(out); + advance_add(in); + } + static unsigned modulo(Value hash, unsigned d) noexcept + { + // Assumes d is power of 2. + return (hash & (d - 1)); + } + Value getValue() noexcept { return value >> extraProcBits; } private: - typedef uint64_t ProcValue; - static const unsigned extraProcBits = (sizeof(ProcValue) - sizeof(Value)) * 8; + typedef uint64_t ProcValue; + static constexpr unsigned extraProcBits = (sizeof(ProcValue) - sizeof(Value)) * 8; - static const ProcValue multiplyAmount = (1ll << extraProcBits) | 181; - ProcValue oldCoefficient, value; + static constexpr ProcValue multiplyAmount = ((1ll << extraProcBits) | 181); + ProcValue oldCoefficient, value; - void advance_add(Token in) { - value += in; - value *= multiplyAmount; - } - void advance_remove(Token out) { - value -= out * oldCoefficient; - } - static ProcValue powHash(ProcValue x, unsigned y) { - ProcValue res = 1; - while (y) { - if (y & 1) res *= x; - x *= x; - y >>= 1; - } - return res; - } + void advance_add(Token in) noexcept + { + value = (value + in) * multiplyAmount; + } + void advance_remove(Token out) noexcept { value -= out * oldCoefficient; } + static ProcValue powHash(ProcValue x, unsigned y) noexcept + { + ProcValue res = 1; + while (y) + { + if (y & 1) + res *= x; + x *= x; + y >>= 1; + } + return res; + } }; +#endif // __CHECKSUM_H__ diff --git a/src/compatibility.h b/src/compatibility.h deleted file mode 100755 index 979cebb..0000000 --- a/src/compatibility.h +++ /dev/null @@ -1,17 +0,0 @@ -/* This Source Code Form is subject to the terms of the Mozilla Public - * License, v. 2.0. If a copy of the MPL was not distributed with this - * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ - -// Fix for MSVC++, which doesn't support Variable Length Arrays. -#ifdef _MSC_VER - #include - #define STACK_ALLOC(name, type, num) type *name = (type *)alloca(sizeof(type) * num) - - typedef unsigned __int8 uint8_t; - typedef unsigned __int16 uint16_t; - typedef unsigned __int32 uint32_t; - typedef unsigned __int64 uint64_t; -#else - #include - #define STACK_ALLOC(name, type, num) type name[num] -#endif diff --git a/src/file.cpp b/src/file.cpp new file mode 100644 index 0000000..bd9e84e --- /dev/null +++ b/src/file.cpp @@ -0,0 +1,54 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#include "file.h" + +#include +#include + +#ifdef _MSC_VER +#define fwrite_unlocked _fwrite_nolock +#define fread_unlocked _fread_nolock +#elif __MINGW32__ +#define fwrite_unlocked fwrite +#define fread_unlocked fread +#endif + +bool fread_fixed(FILE *f, void * _buf, unsigned num_bytes) noexcept +{ + char * buf = (char *)_buf; + + while (num_bytes != 0) + { + unsigned block_size = std::min(num_bytes, MAX_IO_BLOCK_SIZE); + + size_t r = fread_unlocked(buf, 1, block_size, f); + if (r != block_size) + return false; + + buf += block_size; + num_bytes -= block_size; + } + + return true; +} + +bool fwrite_fixed(FILE *f, const void * _buf, unsigned num_bytes) noexcept +{ + const char * buf = (const char *)_buf; + + while (num_bytes != 0) + { + unsigned block_size = std::min(num_bytes, MAX_IO_BLOCK_SIZE); + + size_t r = fwrite_unlocked(buf, 1, block_size, f); + if (r != block_size) + return false; + + buf += block_size; + num_bytes -= block_size; + } + + return true; +} diff --git a/src/file.h b/src/file.h index 0e03fb0..90a3ec4 100644 --- a/src/file.h +++ b/src/file.h @@ -2,85 +2,117 @@ * License, v. 2.0. If a copy of the MPL was not distributed with this * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ +#ifndef __FILE_H__ +#define __FILE_H__ + +#include +#ifdef USE_CXX17 +#include +#endif // USE_CXX17 +#include #include +#include +#include + +static constexpr size_t MAX_IO_BLOCK_SIZE = (1024 * 1024); + +bool fread_fixed(FILE *f, void * _buf, unsigned num_bytes) noexcept; +bool fwrite_fixed(FILE *f, const void * _buf, unsigned num_bytes) noexcept; + +#ifdef BIG_ENDIAN_HOST +template +struct HalfType; + +template <> +struct HalfType +{ + typedef uint8_t type; +}; + +template <> +struct HalfType +{ + typedef uint16_t type; +}; + +template +static inline bool read_value(FILE* f, T* value) +{ + typename HalfType::type low, high; + if (!(fread_fixed(f, &low, sizeof(low)) && fread_fixed(f, &high, sizeof(high)))) + return false; + constexpr int shift = sizeof(low) * 8; + *value = ((high << shift) + low); + return true; +} -#define MAX_IO_BLOCK_SIZE (1024 * 1024) - -void fread_fixed(FILE *f, void * _buf, unsigned num_bytes) { - char * buf = (char *)_buf; - - while (num_bytes != 0) - { - unsigned block_size = num_bytes; - if (block_size > MAX_IO_BLOCK_SIZE) block_size = MAX_IO_BLOCK_SIZE; - - size_t r = fread(buf, 1, block_size, f); - if (r != block_size) - { - static char read_error_message[128]; - sprintf (read_error_message, "read error: fread_fixed(block_size=%u) != %u", block_size, (unsigned)r); - throw read_error_message; - } - buf += block_size; - num_bytes -= block_size; - } +template <> static inline bool read_value(FILE* f, uint8_t* value); +{ + return fread_fixed(f, value, sizeof(*value)); } -void fwrite_fixed(FILE *f, const void * _buf, unsigned num_bytes) { - const char * buf = (const char *)_buf; - - while (num_bytes != 0) - { - unsigned block_size = num_bytes; - if (block_size > MAX_IO_BLOCK_SIZE) block_size = MAX_IO_BLOCK_SIZE; - - size_t r = fwrite(buf, 1, block_size, f); - if (r != block_size) - { - static char write_error_message[128]; - sprintf (write_error_message, "write error: fwrite_fixed(num_bytes=%u) != %u", block_size, (unsigned)r); - throw write_error_message; - } - buf += block_size; - num_bytes -= block_size; - } +template +static inline bool write_value(FILE* f, T* value) +{ + constexpr int shift = sizeof(HalfType::type) * 8; + constexpr int mask = ((1 << shift) - 1); + + typename HalfType::type low = (*value & mask), high = (*value >> 8); + return (fwrite_fixed(f, &low, sizeof(low)) && fwrite_fixed(f, &high, sizeof(high))); +} + +template <> static inline bool write_value(FILE * f, uint8_t* value); +{ + return write_fixed(f, value, sizeof(*value)); } -unsigned read_word(FILE *f) { - unsigned char b, b2; - fread_fixed(f, &b, 1); - fread_fixed(f, &b2, 1); - return (b2 << 8) + b; +#else + +template ::value>> +static inline bool read_value(FILE* f, T* value) +{ + return fread_fixed(f, value, sizeof(*value)); } -unsigned read_dword(FILE *f) { - unsigned low = read_word(f); - return (read_word(f) << 16) + low; +template ::value>> +static inline bool write_value(FILE* f, T* value) +{ + return fwrite_fixed(f, value, sizeof(*value)); } -void write_word(FILE *f, unsigned number) { - unsigned char b = number & 255, - b2 = number >> 8; - fwrite_fixed(f, &b, 1); - fwrite_fixed(f, &b2, 1); +#endif // BIG_ENDIAN + +#ifdef USE_CXX17 + +template +static inline bool fileExists(const T& fname) +{ + std::error_code ec; + return std::filesystem::exists(fname, ec); } -void write_dword(FILE *f, unsigned number) { - write_word(f, number & 65535); - write_word(f, number >> 16); +template +static inline uint64_t getLenOfFile(const T& fname) +{ + std::error_code ec; + return std::filesystem::file_size(fname, ec); } -bool fileExists(char *fname) { - FILE *f = fopen(fname, "rb"); - bool exists = (f != NULL); - if (exists) fclose(f); - return exists; +#else + +static inline bool fileExists(const char * fname) +{ + std::unique_ptr f(fopen(fname, "rb"), fclose); + return (bool)f; } -unsigned getLenOfFile(char *fname) { - FILE *f = fopen(fname, "rb"); - fseek(f, 0, SEEK_END); - unsigned len = ftell(f); - fclose(f); - return len; +static inline unsigned getLenOfFile(const char * fname) +{ + std::unique_ptr f(fopen(fname, "rb"), fclose); + fseek(f.get(), 0, SEEK_END); + return ftell(f.get()); } + +#endif // USE_CXX17 + +#endif // __FILE_H__ diff --git a/src/libbdelta.cpp b/src/libbdelta.cpp index 636013a..d8d2e93 100644 --- a/src/libbdelta.cpp +++ b/src/libbdelta.cpp @@ -2,7 +2,14 @@ * License, v. 2.0. If a copy of the MPL was not distributed with this * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ -#include "compatibility.h" +#include +#include +#include +#include +#include +#include +#include +#include #if !defined(TOKEN_SIZE) || TOKEN_SIZE == 1 typedef uint8_t Token; @@ -12,449 +19,594 @@ typedef uint16_t Token; typedef uint32_t Token; #endif -#include #include "bdelta.h" #include "checksum.h" -#include -#include -#include +#include "noexcept.h" + const bool verbose = false; -struct checksum_entry { - Hash::Value cksum; //Rolling checksums - unsigned loc; - checksum_entry() {} - checksum_entry(Hash::Value cksum, unsigned loc) - {this->cksum = cksum; this->loc = loc;} +struct checksum_entry +{ + Hash::Value cksum; //Rolling checksums + unsigned loc; + checksum_entry(Hash::Value _cksum, unsigned _loc) noexcept : cksum(_cksum), loc(_loc) {} }; -struct Range { - unsigned p, num; - Range() {} - Range(unsigned p, unsigned num) {this->p = p; this->num = num;} +struct Range +{ + unsigned p, num; + Range(unsigned _p, unsigned _num) noexcept : p(_p), num(_num) {} }; -struct Match { - unsigned p1, p2, num; - Match(unsigned p1, unsigned p2, unsigned num) - {this->p1 = p1; this->p2 = p2; this->num = num;} +struct Match +{ + unsigned p1, p2, num; + Match(unsigned _p1, unsigned _p2, unsigned _num) noexcept : p1(_p1), p2(_p2), num(_num) {} }; -struct _BDelta_Instance { - bdelta_readCallback cb; - void *handle1, *handle2; - unsigned data1_size, data2_size; - std::list matches; - std::list::iterator accessplace; - int access_int; - int errorcode; - - const Token *read1(void *buf, unsigned place, unsigned num) - {return (const Token*)cb(handle1, buf, place, num);} - const Token *read2(void *buf, unsigned place, unsigned num) - {return (const Token*)cb(handle2, buf, place, num);} +typedef NoThrowList MatchList; +typedef typename MatchList::iterator MatchListIterator; + +static const size_t TOKEN_BUFFER_SIZE = 4096; +static constexpr size_t BUFFER_DEFAULT_SIZE = 16 * 1024; + +struct UnusedRange +{ + unsigned p, num; + MatchListIterator ml, mr; + UnusedRange() noexcept : p(0), num(0) {} + UnusedRange(unsigned _p, unsigned _num, const MatchListIterator& _ml, const MatchListIterator& _mr) noexcept + : p(_p), num(_num), ml(_ml), mr(_mr) + {} + void set(unsigned _p, unsigned _num, const MatchListIterator& _ml, const MatchListIterator& _mr) + { + p = _p; + num = _num; + ml = _ml; + mr = _mr; + } }; -struct Checksums_Instance { - unsigned blocksize; - unsigned htablesize; - checksum_entry **htable; // Points to first match in checksums - checksum_entry *checksums; // Sorted list of all checksums - unsigned numchecksums; - - Checksums_Instance(int blocksize) {this->blocksize = blocksize;} - void add(checksum_entry ck) { - checksums[numchecksums] = ck; - ++numchecksums; - } - unsigned tableIndex(Hash::Value hashValue) { - return Hash::modulo(hashValue, htablesize); - } +struct _BDelta_Instance +{ + const bdelta_readCallback cb; + void *handle1, *handle2; + unsigned data1_size, data2_size; + MatchList matches; + MatchListIterator accessplace; + int access_int; + BDELTA_RESULT errorcode; + + NoThrowMemoryStorage match_buffer; + NoThrowMemoryStorage find_matches_buffer; + NoThrowMemoryStorage bdelta_pass_2_htable; + NoThrowMemoryStorage bdelta_pass_2_hchecksums; + NoThrowMemoryStorage bdelta_pass_2_buf; + NoThrowMemoryStorage bdelta_pass_unused; + + _BDelta_Instance(bdelta_readCallback _cb, void * _handle1, void * _handle2, + unsigned _data1_size, unsigned _data2_size) noexcept + : cb(_cb), handle1(_handle1), handle2(_handle2) + , data1_size(_data1_size), data2_size(_data2_size) + , access_int(-1), errorcode(BDELTA_OK), match_buffer(2 * TOKEN_BUFFER_SIZE) + , find_matches_buffer(2 * BUFFER_DEFAULT_SIZE), bdelta_pass_2_htable(2 * BUFFER_DEFAULT_SIZE) + , bdelta_pass_2_hchecksums(2 * BUFFER_DEFAULT_SIZE), bdelta_pass_2_buf(2 * BUFFER_DEFAULT_SIZE) + , bdelta_pass_unused(2 * BUFFER_DEFAULT_SIZE) + { + if (!(match_buffer && find_matches_buffer && bdelta_pass_2_htable + && bdelta_pass_2_hchecksums && bdelta_pass_2_buf && bdelta_pass_unused)) + errorcode = BDELTA_MEM_ERROR; + } + + const Token *read1(void *buf, unsigned place, unsigned num) noexcept + { + return (cb == nullptr)? ((const Token*)((char *)handle1 + place)) : ((const Token*)cb(handle1, buf, place, num, this, this->errorcode)); + } + const Token *read2(void *buf, unsigned place, unsigned num) noexcept + { + return (cb == nullptr)? ((const Token*)((char *)handle2 + place)) : ((const Token*)cb(handle2, buf, place, num, this, this->errorcode)); + } }; +struct Checksums_Instance +{ + const unsigned blocksize; + const unsigned htablesize; + checksum_entry ** const htable; // Points to first match in checksums + checksum_entry * const checksums; // Sorted list of all checksums + unsigned numchecksums; + + Checksums_Instance(unsigned _blocksize, unsigned _htablesize, checksum_entry ** _htable, checksum_entry * _checksums) noexcept + : blocksize(_blocksize), htablesize(_htablesize), htable(_htable) + , checksums(_checksums), numchecksums(0) + { + memset(htable, 0, htablesize * sizeof(htable[0])); + } + + template + void add(T&& ck) noexcept + { + checksums[numchecksums++] = std::forward(ck); + } + unsigned tableIndex(Hash::Value hashValue) noexcept + { + return Hash::modulo(hashValue, htablesize); + } +}; -unsigned match_buf_forward(const void *buf1, const void *buf2, unsigned num) { - unsigned i = 0; - while (i < num && ((const Token*)buf1)[i]==((const Token*)buf2)[i]) ++i; - return i; +static inline unsigned match_buf_forward(const void *buf1, const void *buf2, unsigned num) noexcept +{ + unsigned i = 0; + while (i < num && ((const Token*)buf1)[i] == ((const Token*)buf2)[i]) + ++i; + return i; } -unsigned match_buf_backward(const void *buf1, const void *buf2, unsigned num) { - int i = num; - do --i; - while (i >= 0 && ((const Token*)buf1)[i] == ((const Token*)buf2)[i]); - return num - i - 1; + +static inline unsigned match_buf_backward(const void *buf1, const void *buf2, unsigned num) noexcept +{ + int i = num; + do + { + --i; + } + while (i >= 0 && ((const Token*)buf1)[i] == ((const Token*)buf2)[i]); + return (num - i - 1); } -unsigned match_forward(BDelta_Instance *b, unsigned p1, unsigned p2) { - unsigned num = 0, match, numtoread; - do { - numtoread = std::min(b->data1_size - p1, b->data2_size - p2); - if (numtoread > 4096) numtoread = 4096; - Token buf1[4096], buf2[4096]; - const Token *read1 = b->read1(buf1, p1, numtoread), - *read2 = b->read2(buf2, p2, numtoread); - p1 += numtoread; p2 += numtoread; - match = match_buf_forward(read1, read2, numtoread); - num += match; - } while (match && match == numtoread); - return num; + +static unsigned match_forward(BDelta_Instance *b, unsigned p1, unsigned p2) noexcept +{ + unsigned num = 0, match, numtoread; + Token * buf1 = b->match_buffer.get(), * buf2 = buf1 + TOKEN_BUFFER_SIZE; + do + { + numtoread = std::min(std::min(b->data1_size - p1, b->data2_size - p2), TOKEN_BUFFER_SIZE); + const Token *read1 = b->read1(buf1, p1, numtoread), + *read2 = b->read2(buf2, p2, numtoread); + + if (b->errorcode != BDELTA_OK) + return 0; + + p1 += numtoread; + p2 += numtoread; + match = match_buf_forward(read1, read2, numtoread); + num += match; + } while (match && match == numtoread); + return num; } -unsigned match_backward(BDelta_Instance *b, unsigned p1, unsigned p2, unsigned blocksize) { - unsigned num = 0, match, numtoread; - do { - numtoread = std::min(p1, p2); - if (numtoread > blocksize) numtoread = blocksize; - if (numtoread > 4096) numtoread = 4096; - p1 -= numtoread; p2 -= numtoread; - Token buf1[4096], buf2[4096]; - const Token *read1 = b->read1(buf1, p1, numtoread), - *read2 = b->read2(buf2, p2, numtoread); - match = match_buf_backward(read1, read2, numtoread); - num += match; - } while (match && match == numtoread); - return num; +static unsigned match_backward(BDelta_Instance *b, unsigned p1, unsigned p2, unsigned blocksize) noexcept +{ + unsigned num = 0u, match, numtoread; + Token * buf1 = b->match_buffer.get(), * buf2 = buf1 + TOKEN_BUFFER_SIZE; + do + { + numtoread = std::min(std::min(std::min(p1, p2), blocksize), TOKEN_BUFFER_SIZE); + p1 -= numtoread; + p2 -= numtoread; + const Token *read1 = b->read1(buf1, p1, numtoread), + *read2 = b->read2(buf2, p2, numtoread); + + if (b->errorcode != BDELTA_OK) + return 0; + + match = match_buf_backward(read1, read2, numtoread); + num += match; + } while (match && match == numtoread); + return num; } // Iterator helper function template -static inline T bdelta_next(T i) {return ++i;} - -struct UnusedRange { - unsigned p, num; - std::list::iterator ml, mr; - UnusedRange() {} - UnusedRange(unsigned p, unsigned num, std::list::iterator ml, std::list::iterator mr) { - this->p = p; this->num = num; this->ml = ml; this->mr = mr; - } -}; +static inline T bdelta_next(T i) noexcept { return ++i; } -// Sort first by location, second by match length (larger matches first) -bool comparep(UnusedRange r1, UnusedRange r2) { - if (r1.p != r2.p) - return r1.p < r2.p; - return r1.num > r2.num; -} -bool comparemrp2(UnusedRange r1, UnusedRange r2) { - if (r1.mr->p2 != r2.mr->p2) - return r1.mr->p2 < r2.mr->p2; - return r1.mr->num > r2.mr->num; -} -bool compareMatchP2(Match r1, Match r2) { - if (r1.p2 != r2.p2) - return r1.p2 < r2.p2; - return r1.num > r2.num; -} +const auto compareMatchP2 = [](const Match& r1, const Match& r2) noexcept +{ + return ((r1.p2 != r2.p2) ? (r1.p2 < r2.p2) : (r1.num > r2.num)); +}; -void addMatch(BDelta_Instance *b, unsigned p1, unsigned p2, unsigned num, std::list::iterator place) { - Match newMatch = Match(p1, p2, num); - while (place != b->matches.begin() && ! compareMatchP2(*place, newMatch)) - --place; - while (place != b->matches.end() && compareMatchP2(*place, newMatch)) - ++place; - b->matches.insert(place, newMatch); +static void addMatch(BDelta_Instance *b, unsigned p1, unsigned p2, unsigned num, MatchListIterator place) noexcept +{ + Match newMatch = Match(p1, p2, num); + while (place != b->matches.begin() && !compareMatchP2(*place, newMatch)) + --place; + while (place != b->matches.end() && compareMatchP2(*place, newMatch)) + ++place; + if (b->matches.insert(place, newMatch) == nullptr) + b->errorcode = BDELTA_MEM_ERROR; } template -T absoluteDifference(T a, T b) { - return std::max(a, b) - std::min(a, b); +T absoluteDifference(T a, T b) noexcept +{ + return std::max(a, b) - std::min(a, b); } -void findMatches(BDelta_Instance *b, Checksums_Instance *h, unsigned minMatchSize, unsigned start, unsigned end, unsigned place, std::list::iterator iterPlace) { - const unsigned blocksize = h->blocksize; - STACK_ALLOC(buf1, Token, blocksize); - STACK_ALLOC(buf2, Token, blocksize); - - unsigned best1, best2, bestnum = 0; - unsigned processMatchesPos; - const Token *inbuf = b->read2(buf1, start, blocksize), - *outbuf; - Hash hash = Hash(inbuf, blocksize); - unsigned buf_loc = blocksize; - for (unsigned j = start + blocksize; ; ++j) { - unsigned thisTableIndex = h->tableIndex(hash.getValue()); - checksum_entry *c = h->htable[thisTableIndex]; - if (c) { - do { - if (c->cksum == hash.getValue()) { - unsigned p1 = c->loc, p2 = j - blocksize; - unsigned fnum = match_forward(b, p1, p2); - if (fnum >= blocksize) { - unsigned bnum = match_backward(b, p1, p2, blocksize); - unsigned num = fnum + bnum; - if (num >= minMatchSize) { - p1 -= bnum; p2 -= bnum; - bool foundBetter; - if (bestnum) { - double oldValue = double(bestnum) / (absoluteDifference(place, best1) + blocksize * 2), - newValue = double(num) / (absoluteDifference(place, p1) + blocksize * 2); - foundBetter = newValue > oldValue; - } else { - foundBetter = true; - processMatchesPos = std::min(j + blocksize - 1, end); - } - if (foundBetter) { - best1 = p1; - best2 = p2; - bestnum = num; - } - - } - } - } - ++c; - } while (h->tableIndex(c->cksum) == thisTableIndex); - } - - if (bestnum && j >= processMatchesPos) { - addMatch(b, best1, best2, bestnum, iterPlace); - place = best1 + bestnum; - unsigned matchEnd = best2 + bestnum; - if (matchEnd > j) { - if (matchEnd >= end) - j = end; - else { - // Fast forward over matched area. - j = matchEnd - blocksize; - inbuf = b->read2(buf1, j, blocksize); - hash = Hash(inbuf, blocksize); - buf_loc = blocksize; - j += blocksize; - } - } - bestnum = 0; - } - - if (buf_loc == blocksize) { - buf_loc = 0; - std::swap(inbuf, outbuf); - inbuf = b->read2(outbuf == buf1 ? buf2 : buf1, j, std::min(end - j, blocksize)); - } - - if (j >= end) - break; - - hash.advance(outbuf[buf_loc], inbuf[buf_loc]); - ++buf_loc; - } +static void findMatches(BDelta_Instance *b, Checksums_Instance *h, unsigned minMatchSize, unsigned start, unsigned end, unsigned place, MatchListIterator iterPlace) noexcept +{ + const unsigned blocksize = h->blocksize; + + if (!b->find_matches_buffer.resize(blocksize * 2)) + { + b->errorcode = BDELTA_MEM_ERROR; + return; + } + + Token * buf1 = b->find_matches_buffer.get(), * buf2 = buf1 + blocksize; + + unsigned best1 = 0, best2 = 0, bestnum = 0; + unsigned processMatchesPos = 0; + const Token *inbuf = b->read2(buf1, start, blocksize), + *outbuf = nullptr; + Hash hash(inbuf, blocksize); + unsigned buf_loc = blocksize; + for (unsigned j = start + blocksize; ; ++j) + { + unsigned thisTableIndex = h->tableIndex(hash.getValue()); + checksum_entry *c = h->htable[thisTableIndex]; + if (c) + { + do + { + if (c->cksum == hash.getValue()) + { + unsigned p1 = c->loc, p2 = j - blocksize; + unsigned fnum = match_forward(b, p1, p2); + if (b->errorcode != BDELTA_OK) + return; + + if (fnum >= blocksize) + { + unsigned bnum = match_backward(b, p1, p2, blocksize); + if (b->errorcode != BDELTA_OK) + return; + + unsigned num = fnum + bnum; + if (num >= minMatchSize) + { + p1 -= bnum; p2 -= bnum; + bool foundBetter; + if (bestnum) + { + double oldValue = double(bestnum) / (absoluteDifference(place, best1) + blocksize * 2), + newValue = double(num) / (absoluteDifference(place, p1) + blocksize * 2); + foundBetter = newValue > oldValue; + } + else + { + foundBetter = true; + processMatchesPos = std::min(j + blocksize - 1, end); + } + if (foundBetter) + { + best1 = p1; + best2 = p2; + bestnum = num; + } + + } + } + } + ++c; + } while (h->tableIndex(c->cksum) == thisTableIndex); + } + + if (bestnum && j >= processMatchesPos) + { + addMatch(b, best1, best2, bestnum, iterPlace); + if (b->errorcode != BDELTA_OK) + return; + + place = best1 + bestnum; + unsigned matchEnd = best2 + bestnum; + if (matchEnd > j) + { + if (matchEnd >= end) + j = end; + else + { + // Fast forward over matched area. + j = matchEnd - blocksize; + inbuf = b->read2(buf1, j, blocksize); + hash = Hash(inbuf, blocksize); + buf_loc = blocksize; + j += blocksize; + } + } + bestnum = 0; + } + + if (buf_loc == blocksize) + { + buf_loc = 0; + std::swap(inbuf, outbuf); + inbuf = b->read2(outbuf == buf1 ? buf2 : buf1, j, std::min(end - j, blocksize)); + } + + if (j >= end) + break; + + hash.advance(outbuf[buf_loc], inbuf[buf_loc]); + ++buf_loc; + } } -struct Checksums_Compare { - Checksums_Instance &ci; - Checksums_Compare(Checksums_Instance &h) : ci(h) {} - bool operator() (checksum_entry c1, checksum_entry c2) { - unsigned ti1 = ci.tableIndex(c1.cksum), ti2 = ci.tableIndex(c2.cksum); - if (ti1 == ti2) - if (c1.cksum == c2.cksum) - return c1.loc < c2.loc; - else - return c1.cksum < c2.cksum; - else - return ti1 < ti2; - } +struct Checksums_Compare +{ + Checksums_Instance &ci; + explicit Checksums_Compare(Checksums_Instance &h) noexcept : ci(h) {} + bool operator() (checksum_entry c1, checksum_entry c2) noexcept + { + unsigned ti1 = ci.tableIndex(c1.cksum), ti2 = ci.tableIndex(c2.cksum); + if (ti1 == ti2) + { + if (c1.cksum == c2.cksum) + return c1.loc < c2.loc; + else + return c1.cksum < c2.cksum; + } + else + return ti1 < ti2; + } }; BDelta_Instance *bdelta_init_alg(unsigned data1_size, unsigned data2_size, - bdelta_readCallback cb, void *handle1, void *handle2, - unsigned tokenSize) { - if (tokenSize != sizeof(Token)) { - printf("Error: BDelta library compiled for token size of %lu.\n", (unsigned long)sizeof (Token)); - return 0; - } - BDelta_Instance *b = new BDelta_Instance; - if (!b) return 0; - b->data1_size = data1_size; - b->data2_size = data2_size; - b->cb = cb; - b->handle1 = handle1; - b->handle2 = handle2; - b->access_int = -1; - return b; + bdelta_readCallback cb, void *handle1, void *handle2, + unsigned tokenSize) noexcept +{ + if (tokenSize != sizeof(Token)) + { + printf("Error: BDelta library compiled for token size of %lu.\n", (unsigned long)sizeof (Token)); + return nullptr; + } + void * p = malloc(sizeof(BDelta_Instance)); + if (p == nullptr) + return nullptr; + + return (new(p) BDelta_Instance(cb, handle1, handle2, data1_size, data2_size)); } -void bdelta_done_alg(BDelta_Instance *b) { - b->matches.clear(); - delete b; +void bdelta_done_alg(BDelta_Instance *b) noexcept +{ + free(b); } - // Adapted from http://graphics.stanford.edu/~seander/bithacks.html#RoundUpPowerOf2 -unsigned roundUpPowerOf2(unsigned v) { - --v; - for (int i = 1; i <= 16; i *= 2) - v |= v >> i; - return v + 1; +static unsigned roundUpPowerOf2(unsigned v) noexcept +{ + --v; + for (int i = 1; i <= 16; i *= 2) + v |= (v >> i); + return (v + 1); } -void bdelta_pass_2(BDelta_Instance *b, unsigned blocksize, unsigned minMatchSize, UnusedRange *unused, unsigned numunused, UnusedRange *unused2, unsigned numunused2) { - Checksums_Instance h(blocksize); - b->access_int = -1; - - unsigned numblocks = 0; - for (unsigned i = 0; i < numunused; ++i) { - numblocks += unused[i].num / blocksize; - } - - // numblocks = size / blocksize; - h.htablesize = std::max((unsigned)2, roundUpPowerOf2(numblocks)); - h.htable = new checksum_entry*[h.htablesize]; - if (!h.htable) {b->errorcode = BDELTA_MEM_ERROR; return;} - h.checksums = new checksum_entry[numblocks + 2]; - if (!h.checksums) {b->errorcode = BDELTA_MEM_ERROR; return;} - - h.numchecksums = 0; - // unsigned numchecksums = 0; - STACK_ALLOC(buf, Token, blocksize); - for (unsigned i = 0; i < numunused; ++i) { - unsigned first = unused[i].p, last = unused[i].p + unused[i].num; - for (unsigned loc = first; loc + blocksize <= last; loc += blocksize) { - const Token *read = b->read1(buf, loc, blocksize); - Hash::Value blocksum = Hash(read, blocksize).getValue(); - // Adjacent checksums are never repeated. - //if (! h.numchecksums || blocksum != h.checksums[h.numchecksums - 1].cksum) - h.add(checksum_entry(blocksum, loc)); - } - } - - if (h.numchecksums) { - std::sort(h.checksums, h.checksums + h.numchecksums, Checksums_Compare(h)); - const unsigned maxIdenticalChecksums = 2; - unsigned writeLoc = 0, readLoc, testAhead; - for (readLoc = 0; readLoc < h.numchecksums; readLoc = testAhead) { - for (testAhead = readLoc; testAhead < h.numchecksums && h.checksums[readLoc].cksum == h.checksums[testAhead].cksum; ++testAhead) - ; - if (testAhead - readLoc <= maxIdenticalChecksums) - for (unsigned i = readLoc; i < testAhead; ++i) - h.checksums[writeLoc++] = h.checksums[i]; - } - h.numchecksums = writeLoc; - } - h.checksums[h.numchecksums].cksum = std::numeric_limits::max(); // If there's only one checksum, we might hit this and not know it, - h.checksums[h.numchecksums].loc = 0; // So we'll just read from the beginning of the file to prevent crashes. - h.checksums[h.numchecksums + 1].cksum = 0; - - for (unsigned i = 0; i < h.htablesize; ++i) h.htable[i] = 0; - for (int i = h.numchecksums - 1; i >= 0; --i) - h.htable[h.tableIndex(h.checksums[i].cksum)] = &h.checksums[i]; - - for (unsigned i = 0; i < numunused2; ++i) - if (unused2[i].num >= blocksize) - findMatches(b, &h, minMatchSize, unused2[i].p, unused2[i].p + unused2[i].num, unused[i].p, unused2[i].mr); - - delete [] h.htable; - delete [] h.checksums; +static void bdelta_pass_2(BDelta_Instance *b, uint32_t blocksize, uint32_t minMatchSize, UnusedRange *unused, uint32_t numunused, UnusedRange *unused2) noexcept +{ + b->access_int = -1; + + unsigned numblocks = 0; + for (unsigned i = 0; i < numunused; ++i) + numblocks += unused[i].num; + numblocks /= blocksize; + + b->bdelta_pass_2_htable.resize(std::max(2u, roundUpPowerOf2(numblocks))); + b->bdelta_pass_2_hchecksums.resize(numblocks + 2); + if (!(b->bdelta_pass_2_htable.resize(std::max(2u, roundUpPowerOf2(numblocks))) && b->bdelta_pass_2_hchecksums.resize(numblocks + 2))) + { + b->errorcode = BDELTA_MEM_ERROR; + return; + } + + Checksums_Instance h(blocksize, (unsigned)b->bdelta_pass_2_htable.size(), b->bdelta_pass_2_htable.get(), b->bdelta_pass_2_hchecksums.get()); + + b->bdelta_pass_2_buf.resize(blocksize); + if (!b->bdelta_pass_2_buf.resize(blocksize)) + { + b->errorcode = BDELTA_MEM_ERROR; + return; + } + + for (unsigned i = 0; i < numunused; ++i) + { + unsigned first = unused[i].p, last = unused[i].p + unused[i].num; + for (unsigned loc = first; loc + blocksize <= last; loc += blocksize) + { + const Token *read = b->read1(b->bdelta_pass_2_buf.get(), loc, blocksize); + Hash::Value blocksum = Hash(read, blocksize).getValue(); + // Adjacent checksums are never repeated. + h.add(checksum_entry(blocksum, loc)); + } + } + + if (h.numchecksums) + { + std::sort(h.checksums, h.checksums + h.numchecksums, Checksums_Compare(h)); + const unsigned maxIdenticalChecksums = 2; + unsigned writeLoc = 0, readLoc, testAhead; + for (readLoc = 0; readLoc < h.numchecksums; readLoc = testAhead) + { + for (testAhead = readLoc; testAhead < h.numchecksums && h.checksums[readLoc].cksum == h.checksums[testAhead].cksum; ++testAhead) + ; + if (testAhead - readLoc <= maxIdenticalChecksums) + for (unsigned i = readLoc; i < testAhead; ++i) + h.checksums[writeLoc++] = h.checksums[i]; + } + h.numchecksums = writeLoc; + } + h.checksums[h.numchecksums].cksum = std::numeric_limits::max(); // If there's only one checksum, we might hit this and not know it, + h.checksums[h.numchecksums].loc = 0; // So we'll just read from the beginning of the file to prevent crashes. + h.checksums[h.numchecksums + 1].cksum = 0; + + for (int i = h.numchecksums - 1; i >= 0; --i) + h.htable[h.tableIndex(h.checksums[i].cksum)] = &h.checksums[i]; + + for (unsigned i = 0; i < numunused; ++i) + { + if (unused2[i].num >= blocksize) + { + findMatches(b, &h, minMatchSize, unused2[i].p, unused2[i].p + unused2[i].num, unused[i].p, unused2[i].mr); + if (b->errorcode != BDELTA_OK) + return; + } + } } -void bdelta_swap_inputs(BDelta_Instance *b) { - for (std::list::iterator l = b->matches.begin(); l != b->matches.end(); ++l) - std::swap(l->p1, l->p2); - std::swap(b->data1_size, b->data2_size); - std::swap(b->handle1, b->handle2); - b->matches.sort(compareMatchP2); +void bdelta_swap_inputs(BDelta_Instance *b) noexcept +{ + for (auto& m : b->matches) + std::swap(m.p1, m.p2); + std::swap(b->data1_size, b->data2_size); + std::swap(b->handle1, b->handle2); + b->matches.sort(compareMatchP2); } -void bdelta_clean_matches(BDelta_Instance *b, unsigned flags) { - std::list::iterator nextL = b->matches.begin(); - if (nextL == b->matches.end()) return; - while (true) { - std::list::iterator l = nextL; - if (++nextL == b->matches.end()) - break; - - int overlap = l->p2 + l->num - nextL->p2; - if (overlap >= 0) { - if (overlap >= nextL->num) { - b->matches.erase(nextL); - nextL = l; - continue; - } - if (flags & BDELTA_REMOVE_OVERLAP) - l->num -= overlap; - } - } +void bdelta_clean_matches(BDelta_Instance *b, unsigned flags) noexcept +{ + auto nextL = b->matches.begin(); + if (nextL == b->matches.end()) + return; + while (true) + { + auto l = nextL; + if (++nextL == b->matches.end()) + break; + + int overlap = l->p2 + l->num - nextL->p2; + if (overlap >= 0) + { + if ((unsigned)overlap >= nextL->num) + { + b->matches.erase(nextL); + nextL = l; + continue; + } + if (flags & BDELTA_REMOVE_OVERLAP) + l->num -= overlap; + } + } } -void bdelta_showMatches(BDelta_Instance *b) { - for (std::list::iterator l = b->matches.begin(); l != b->matches.end(); ++l) - printf("(%d, %d, %d), ", l->p1, l->p2, l->num); - printf ("\n\n"); +void bdelta_showMatches(BDelta_Instance *b) noexcept +{ + for (const auto& m : b->matches) + printf("(%d, %d, %d), ", m.p1, m.p2, m.num); + printf("\n\n"); } -void get_unused_blocks(UnusedRange *unused, unsigned *numunusedptr) { - unsigned nextStartPos = 0; - for (unsigned i = 1; i < *numunusedptr; ++i) { - unsigned startPos = nextStartPos; - nextStartPos = std::max(startPos, unused[i].p + unused[i].num); - unused[i] = UnusedRange(startPos, unused[i].p < startPos ? 0 : unused[i].p - startPos, unused[i-1].mr, unused[i].mr); - } +static void get_unused_blocks(UnusedRange *unused, uint32_t numunused) noexcept +{ + unsigned nextStartPos = 0; + for (unsigned i = 1; i < numunused; ++i) + { + unsigned startPos = nextStartPos; + nextStartPos = std::max(startPos, unused[i].p + unused[i].num); + unused[i].set(startPos, unused[i].p < startPos ? 0 : unused[i].p - startPos, unused[i-1].mr, unused[i].mr); + } } -bool isZeroMatch(Match &m) {return m.num == 0;} - -void bdelta_pass(BDelta_Instance *b, unsigned blocksize, unsigned minMatchSize, unsigned maxHoleSize, unsigned flags) { - // Place an empty Match at beginning so we can assume there's a Match to the left of every hole. - b->matches.push_front(Match(0, 0, 0)); - // Trick for including the free range at the end. - b->matches.push_back(Match(b->data1_size, b->data2_size, 0)); - - UnusedRange *unused = new UnusedRange[b->matches.size() + 1], - *unused2 = new UnusedRange[b->matches.size() + 1]; - unsigned numunused = 0, numunused2 = 0; - for (std::list::iterator l = b->matches.begin(); l != b->matches.end(); ++l) { - unused[numunused++] = UnusedRange(l->p1, l->num, l, l); - unused2[numunused2++] = UnusedRange(l->p2, l->num, l, l); - } - - std::sort(unused + 1, unused + numunused, comparep); // Leave empty match at beginning - //std::sort(unused2, unused2 + numunused2, comparep); - - get_unused_blocks(unused, &numunused); - get_unused_blocks(unused2, &numunused2); - //std::sort(unused2, unused2 + numunused2, comparemrp2); - - if (flags & BDELTA_GLOBAL) - bdelta_pass_2(b, blocksize, minMatchSize, unused, numunused, unused2, numunused2); - else { - std::sort(unused + 1, unused + numunused, comparemrp2); - for (unsigned i = 1; i < numunused; ++i) { - UnusedRange u1 = unused[i], u2 = unused2[i]; - if (u1.num >= blocksize && u2.num >= blocksize) - if (! maxHoleSize || (u1.num <= maxHoleSize && u2.num <= maxHoleSize)) - if (! (flags & BDELTA_SIDES_ORDERED) || (bdelta_next(u1.ml) == u1.mr && bdelta_next(u2.ml) == u2.mr)) - bdelta_pass_2(b, blocksize, minMatchSize, &u1, 1, &u2, 1); - } - } - - if (verbose) printf("pass (blocksize: %u, matches: %lu)\n", blocksize, (unsigned long)b->matches.size()); - - // Get rid of the dummy values we placed at the ends. - b->matches.erase(std::find_if(b->matches.begin(), b->matches.end(), isZeroMatch)); - b->matches.pop_back(); - - delete [] unused; - delete [] unused2; +void bdelta_pass(BDelta_Instance *b, uint32_t blocksize, uint32_t minMatchSize, uint32_t maxHoleSize, uint32_t flags) noexcept +{ + // Place an empty Match at beginning so we can assume there's a Match to the left of every hole. + if (b->matches.emplace_front(0, 0, 0) == nullptr) + { + b->errorcode = BDELTA_MEM_ERROR; + return; + } + // Trick for including the free range at the end. + if (b->matches.emplace_back(b->data1_size, b->data2_size, 0) == nullptr) + { + b->errorcode = BDELTA_MEM_ERROR; + return; + } + + const size_t BUFFER_SIZE = b->matches.size() + 1; + if (!b->bdelta_pass_unused.resize(BUFFER_SIZE * 2)) + { + b->errorcode = BDELTA_MEM_ERROR; + return; + } + + UnusedRange *unused = b->bdelta_pass_unused.get(), + *unused2 = unused + BUFFER_SIZE; + + uint32_t numunused = 0; + for (auto l = b->matches.begin(); l != b->matches.end(); ++l) + { + unused[numunused].set(l->p1, l->num, l, l); + unused2[numunused++].set(l->p2, l->num, l, l); + } + + // Leave empty match at beginning + std::sort(unused + 1, unused + numunused, [](const UnusedRange& r1, const UnusedRange& r2) noexcept + { + return ((r1.p != r2.p) ? (r1.p < r2.p) : (r1.num > r2.num)); + }); + + get_unused_blocks(unused, numunused); + get_unused_blocks(unused2, numunused); + + if ((flags & BDELTA_GLOBAL) != 0) + { + bdelta_pass_2(b, blocksize, minMatchSize, unused, numunused, unused2); + if (b->errorcode != BDELTA_OK) + return; + } + else + { + std::sort(unused + 1, unused + numunused, [](const UnusedRange& r1, const UnusedRange& r2) noexcept + { + return ((r1.mr->p2 != r2.mr->p2) ? (r1.mr->p2 < r2.mr->p2) : (r1.mr->num > r2.mr->num)); + }); + for (unsigned i = 1; i < numunused; ++i) + { + UnusedRange u1 = unused[i], u2 = unused2[i]; + if (u1.num >= blocksize && u2.num >= blocksize) + if (!maxHoleSize || (u1.num <= maxHoleSize && u2.num <= maxHoleSize)) + if ((flags & BDELTA_SIDES_ORDERED) == 0 || (bdelta_next(u1.ml) == u1.mr && bdelta_next(u2.ml) == u2.mr)) + { + bdelta_pass_2(b, blocksize, minMatchSize, &u1, 1, &u2); + if (b->errorcode != BDELTA_OK) + return; + } + } + } + + if (verbose) + printf("pass (blocksize: %u, matches: %u)\n", blocksize, (unsigned)b->matches.size()); + + // Get rid of the dummy values we placed at the ends. + b->matches.erase(std::find_if(b->matches.begin(), b->matches.end(), [](const Match &m) noexcept { return m.num == 0; })); + b->matches.pop_back(); } -unsigned bdelta_numMatches(BDelta_Instance *b) { - return b->matches.size(); +unsigned bdelta_numMatches(BDelta_Instance *b) noexcept +{ + return (unsigned)b->matches.size(); } -void bdelta_getMatch(BDelta_Instance *b, unsigned matchNum, - unsigned *p1, unsigned *p2, unsigned *num) { - int &access_int = b->access_int; - std::list::iterator &accessplace = b->accessplace; - if (access_int == -1) {access_int = 0; accessplace = b->matches.begin();} - while ((unsigned)access_int < matchNum) { - ++accessplace; - ++access_int; - } - while ((unsigned)access_int > matchNum) { - --accessplace; - --access_int; - } - *p1 = accessplace->p1; - *p2 = accessplace->p2; - *num = accessplace->num; +void bdelta_getMatch(BDelta_Instance *b, unsigned matchNum, unsigned *p1, unsigned *p2, unsigned *num) noexcept +{ + int &access_int = b->access_int; + auto& accessplace = b->accessplace; + if (access_int == -1) + { + access_int = 0; + accessplace = b->matches.begin(); + } + while ((unsigned)access_int < matchNum) + { + ++accessplace; + ++access_int; + } + while ((unsigned)access_int > matchNum) + { + --accessplace; + --access_int; + } + *p1 = accessplace->p1; + *p2 = accessplace->p2; + *num = accessplace->num; } -int bdelta_getError(BDelta_Instance *instance) { - return instance->errorcode; +BDELTA_RESULT * bdelta_getError(BDelta_Instance *instance) noexcept +{ + return &(instance->errorcode); } diff --git a/src/noexcept.h b/src/noexcept.h new file mode 100644 index 0000000..610ae71 --- /dev/null +++ b/src/noexcept.h @@ -0,0 +1,555 @@ +#ifndef __NOEXCEPT_H__ +#define __NOEXCEPT_H__ + +#include +#include +#include + +#ifdef _WIN32 +#define NOMINMAX +#include + +static const HANDLE hDefaultHeap = GetProcessHeap(); + +inline void * bdelta_malloc(size_t size) +{ + return HeapAlloc(hDefaultHeap, 0, size); +} + +inline void bdelta_free(void * p) +{ + HeapFree(hDefaultHeap, 0, p); +} + +inline bool inplace_realloc(void * old_p, size_t new_size) +{ + return (HeapReAlloc(hDefaultHeap, HEAP_REALLOC_IN_PLACE_ONLY, old_p, new_size) != nullptr); +} +#else +#define bdelta_malloc malloc +#define bdelta_free free + +// TODO: implement it for linux +constexpr bool inplace_realloc(void * /*old_p*/, size_t /*new_size*/) { return false; } +#endif // _WIN32 + +template +struct Construct +{ + void construct(T * p, size_t count) const noexcept + { + for (size_t i = 0; i < count; ++i) + new(p + i) T(); + } + + void destruct(T * p, size_t count) const noexcept + { + for (size_t i = 0; i < count; ++i) + (p + i)->~T(); + } + + void move(T * old_p, size_t old_size, T * new_p, size_t new_size) const noexcept + { + size_t i = 0; + for (; i < old_size; ++i) + new (new_p + i) T(std::move(old_p[i])); + + for (; i < new_size; ++i) + new(new_p + i) T(); + } +}; + +template +struct NoConstruct +{ + void construct(T * /*p*/, size_t /*count*/) const noexcept + {} + + void destruct(T * /*p*/, size_t /*count*/) const noexcept + {} + + void move(T * old_p, size_t old_size, T * new_p, size_t /*new_size*/) const noexcept + { + memcpy(new_p, old_p, old_size * sizeof(T)); + } +}; + +template class Constructor> +struct NoRealloc +{ + void realloc(T * old_p, size_t old_size, T * new_p, size_t new_size, Constructor& constructor) + { + constructor.destruct(old_p, old_size); + constructor.construct(new_p, new_size); + } +}; + +template class Constructor> +struct Realloc +{ + void realloc(T * old_p, size_t old_size, T * new_p, size_t new_size, Constructor& constructor) + { + constructor.move(old_p, old_size, new_p, new_size); + constructor.destruct(old_p, old_size); + } +}; + +template class Constructor = NoConstruct, template class> class Reallocator = NoRealloc> +class NoThrowMemoryStorage +{ +public: + T * get() const noexcept { return m_data; } + + NoThrowMemoryStorage(size_t size) noexcept + : m_data((T*)bdelta_malloc(size * sizeof(T))) + { + if (m_data != nullptr) + { + m_size = size; + m_capacity = size; + m_constructor.construct(m_data, m_size); + } + else + { + m_size = 0; + m_capacity = 0; + } + } + + ~NoThrowMemoryStorage() + { + if (m_data != nullptr) + { + m_constructor.destruct(m_data, m_capacity); + bdelta_free(m_data); + } + } + + bool resize(size_t size) noexcept + { + if (m_capacity >= size) + { + if (size > m_size) + m_constructor.construct(m_data + m_size, size - m_size); + else if (size < m_size) + m_constructor.destruct(m_data + size, m_size - size); + m_size = size; + return true; + } + + if (m_data != nullptr && inplace_realloc(m_data, size)) + m_constructor.construct(m_data + m_capacity, size - m_capacity); + else + { + T * new_data = (T*)bdelta_malloc(size * sizeof(T)); + if (new_data == nullptr) + { + if (m_data != nullptr) + { + m_constructor.destruct(m_data, m_size); + bdelta_free(m_data); + m_size = 0; + m_capacity = 0; + } + + return false; + } + + if (m_data != nullptr) + m_reallocator.realloc(m_data, m_size, new_data, size, m_constructor); + else + m_constructor.construct(new_data, size); + + m_data = new_data; + } + m_size = size; + m_capacity = size; + + return true; + } + + size_t size() const noexcept { return m_size; } + + operator bool() const noexcept { return (m_data != nullptr); } + +protected: + T * m_data; + size_t m_size; + size_t m_capacity; + + Constructor m_constructor; + Reallocator m_reallocator; +}; + +class PtrStack +{ +protected: + NoThrowMemoryStorage m_storage; + size_t m_top; +public: + PtrStack(size_t capacity) noexcept : m_storage(capacity), m_top(0) {} + bool pop(void *& p) noexcept + { + if (m_top == 0) + return false; + + p = m_storage.get()[--m_top]; + return true; + } + + bool push(void * p) noexcept + { + size_t size = m_storage.size(); + if (m_top >= size) + { + if (!m_storage.resize(size * 2)) + return false; + } + + m_storage.get()[m_top++] = p; + return true; + } + + size_t size() const noexcept { return m_top; } + size_t capacity() const noexcept { return m_storage.size(); } + bool good() const noexcept { return (m_storage.get() != nullptr); } +}; + +template +class NoThrowList +{ +public: + struct Node + { + Node * prev; + Node * next; + T data; + }; + + NoThrowList() noexcept : m_head(nullptr), m_tail(nullptr), m_size(0), m_allocator_stack(1024 * 1024), m_blocks_stack(64 * 1024) + {} + + class iterator + { + public: + typedef T value_type; + typedef T* pointer; + typedef T& reference; + typedef void difference_type; + typedef std::bidirectional_iterator_tag iterator_category; + + iterator() noexcept : m_node(nullptr) + {} + + explicit iterator(Node * node) noexcept : m_node(node) + {} + + iterator& operator ++() noexcept + { + m_node = m_node->next; + return *this; + } + + iterator& operator --() noexcept + { + m_node = m_node->prev; + return *this; + } + + bool operator == (const iterator& rhs) noexcept + { + return (m_node == rhs.m_node); + } + + bool operator != (const iterator& rhs) noexcept + { + return !(*this == rhs); + } + + Node * get() const noexcept + { + return m_node; + } + + T& operator * () const noexcept + { + return m_node->data; + } + + T* operator ->() const noexcept + { + return &m_node->data; + } + + bool empty() const noexcept + { + return (m_node == nullptr); + } + + protected: + Node * m_node; + }; + + iterator end() const noexcept + { + return iterator(); + } + + iterator begin() const noexcept + { + return iterator(m_head); + } + + bool empty() const noexcept + { + return (m_head == nullptr); + } + + size_t size() + { + return m_size; + } + + template + T* emplace_back(Args... args) noexcept + { + Node * new_node = allocate(); + if (new_node == nullptr) + return nullptr; + + new (&new_node->data) T(std::forward(args)...); + new_node->next = nullptr; + + if (m_tail == nullptr) + m_head = new_node; + else + m_tail->next = new_node; + + new_node->prev = m_tail; + m_tail = new_node; + ++m_size; + + return &new_node->data; + } + + template + T* emplace_front(Args... args) noexcept + { + Node * new_node = allocate(); + if (new_node == nullptr) + return nullptr; + + new (&new_node->data) T(std::forward(args)...); + new_node->prev = nullptr; + + if (m_head == nullptr) + m_tail = new_node; + else + m_head->prev = new_node; + + new_node->next = m_head; + m_head = new_node; + ++m_size; + + return &new_node->data; + } + + iterator erase(const iterator& it) noexcept + { + Node * node = it.get(); + Node * const prev = node->prev, + * const next = node->next; + + if (prev == nullptr) + { + m_head = m_head->next; + if (m_head != nullptr) + m_head->prev = nullptr; + } + else + prev->next = next; + + if (next == nullptr) + { + m_tail = m_tail->prev; + if (m_tail != nullptr) + m_tail->next = nullptr; + } + else + next->prev = prev; + + node->data.T::~T(); + deallocate(node); + --m_size; + + return iterator(next); + } + + void pop_back() + { + if (empty()) + return; + + Node * const prev = m_tail->prev; + if (prev == nullptr) + m_head = nullptr; + + m_tail->data.T::~T(); + deallocate(m_tail); + + m_tail = prev; + if (m_tail != nullptr) + m_tail->next = nullptr; + --m_size; + } + + T * insert(iterator where, const T& data) + { + Node * node = where.get(); + if (node == nullptr) + return emplace_back(data); + else if (node == m_head) + return emplace_front(data); + else + { + Node * new_node = allocate(); + if (new_node == nullptr) + return nullptr; + + Node * const prev = node->prev; + new (&new_node->data) T(data); + new_node->prev = prev; + prev->next = new_node; + new_node->next = node; + node->prev = new_node; + ++m_size; + + return &new_node->data; + } + } + + template + void sort(Comaparer comp) + { + qsort(comp, m_head, nullptr, &m_head); + Node * prev = nullptr; + for (Node * cur = m_head; cur != nullptr; cur = cur->next) + { + cur->prev = prev; + prev = cur; + } + + m_tail = prev; + } + + ~NoThrowList() noexcept + { + Node * p = m_head; + while (p != nullptr) + { + Node * next = p->next; + p->data.T::~T(); + // we dont't need deallocate() because m_blocks_stack will free memory + p = next; + } + + void * block = nullptr; + while(m_blocks_stack.pop(block)) + bdelta_free(block); + + m_size = 0; + m_head = nullptr; + m_tail = nullptr; + } + +protected: + Node * m_head; + Node * m_tail; + size_t m_size; + PtrStack m_allocator_stack; + PtrStack m_blocks_stack; + + Node * allocate() + { + if (!m_allocator_stack.good()) + return nullptr; + + void * p = nullptr; + if (m_allocator_stack.pop(p)) + return (Node*)p; + + if (!m_blocks_stack.good()) + return nullptr; + + size_t count = m_allocator_stack.capacity(); + Node * block = (Node*)bdelta_malloc(sizeof(Node) * count); + if (!m_blocks_stack.push(block)) + { + bdelta_free(block); + return nullptr; + } + + for (size_t i = 1; i < count; ++i) + m_allocator_stack.push(block + i); + + return block; + } + void deallocate(Node * p) + { + m_allocator_stack.push(p); + } + + template + void qsort(Comaparer& comp, Node * hd, Node * tl, Node ** rtn) + { + int nlo, nhi; + Node *lo, *hi, *q, *p; + + /* Invariant: Return head sorted with `tl' appended. */ + while (hd != nullptr) + { + nlo = nhi = 0; + lo = hi = nullptr; + q = hd; + p = hd->next; + + /* Partition and count sizes. */ + while (p != nullptr) + { + q = p->next; + if (!comp(hd->data, p->data)) + { + p->next = lo; + lo = p; + ++nlo; + } + else + { + p->next = hi; + hi = p; + ++nhi; + } + p = q; + } + + /* Recurse to establish invariant for sublists of hd, + choosing shortest list first to limit stack. */ + if (nlo < nhi) + { + qsort(comp, lo, hd, rtn); + rtn = &hd->next; + hd = hi; /* Eliminated tail-recursive call. */ + } + else + { + qsort(comp, hi, tl, &hd->next); + tl = hd; + hd = lo; /* Eliminated tail-recursive call. */ + } + } + /* Base case of recurrence. Invariant is easy here. */ + *rtn = tl; + } +}; + +#endif // __NOEXCEPT_H__ diff --git a/test/library.cpp b/test/library.cpp deleted file mode 100644 index 9008fc6..0000000 --- a/test/library.cpp +++ /dev/null @@ -1,31 +0,0 @@ -/* Any copyright is dedicated to the Public Domain. - * http://creativecommons.org/publicdomain/zero/1.0/ */ - -// "Hello BDelta!" - -#include "bdelta.h" -#include -#include - -const int smallestMatch = 8; -char *a = "abcdefghijklmnopqrstuvwxyz", *b = "abcdefghijklmnopqrstuvwxyz"; -void *a_read(unsigned place, unsigned num) { - return (char*)(a + place); -} - -void *b_read(unsigned place, unsigned num) { - return (char*)(b + place); -} -main() { - void *bi = bdelta_init_alg(strlen(a), strlen(b), a_read, b_read); - int nummatches; - for (int i = 64; i >= smallestMatch; i/=2) - nummatches = bdelta_pass(bi, i); - for (int i = 0; i < nummatches; ++i) { - unsigned p1, p2, num; - bdelta_getMatch(bi, i, &p1, &p2, &num); - printf("Got match\n"); - } - -} - diff --git a/test/py_bindings.py b/test/py_bindings.py deleted file mode 100644 index f600cd7..0000000 --- a/test/py_bindings.py +++ /dev/null @@ -1,18 +0,0 @@ -# Any copyright is dedicated to the Public Domain. -# http://creativecommons.org/publicdomain/zero/1.0/ - -from bdelta import BDelta - -a = u"The quick brown fox jumped over the lazy dog" -b = u"The quick drowned fox jumped over the lazy dog" - -b = BDelta(a, b) - -b.b_pass(13, 27, 0) # Find all matches that are at least 27 chars long -print list(b.matches()) # [(15, 17, 29)]) - -b.b_pass(3, 5, 0) # Find all matches that are at least 5 chars long -print list(b.matches()) # [(0, 0, 10), (15, 17, 29)] - -b.b_pass(2, 3, 0) # Find all matches that are at least 3 chars long -print list(b.matches()) # [(0, 0, 10), (11, 11, 4), (15, 17, 29)] diff --git a/test/suite.py b/test/suite.py deleted file mode 100644 index 5eb821b..0000000 --- a/test/suite.py +++ /dev/null @@ -1,40 +0,0 @@ -# Any copyright is dedicated to the Public Domain. -# http://creativecommons.org/publicdomain/zero/1.0/ - -# First argument should be a test suite folder -# Such folders contain subfolders with files 1 (old -# version), and 2 (new version) of data that you -# want to test with the delta algorithm - -import sys -import os -import filecmp -import time - -basePath = sys.argv[1] -initialTime = time.time() - -def isValidFile(x): - return not x.startswith('.') and os.path.isdir(os.path.join(basePath, x)) - -testFolders = filter(isValidFile, os.listdir(basePath)) -print "Running tests:", testFolders -print "BDelta 0.10 file sizes:" -for test in testFolders: - def path(fName): - return os.path.join(basePath, test, fName) - t1 = time.time() - os.system("../src/bdelta %s %s %s" % (path("1"), path("2"), path("patch.bdelta"))) - t2 = time.time() - os.system("../src/bpatch %s %s %s" % (path("1"), path("2.new"), path("patch.bdelta"))) - print ("%20s:" % test), - if filecmp.cmp(path("2"), path("2.new"), shallow=False): - print "uncompressed=%s\t" % os.stat(path("patch.bdelta")).st_size, - os.system("bzip2 %s" % path("patch.bdelta")) - print "bz2=%s\t" % os.stat(path("patch.bdelta.bz2")).st_size, - print "time=%f" % (t2-t1) - else: - print "ERROR" - os.remove(path("patch.bdelta.bz2")) - os.remove(path("2.new")) -print "Total time:", (time.time()-initialTime) diff --git a/version.py b/version.py deleted file mode 100644 index d8ebfee..0000000 --- a/version.py +++ /dev/null @@ -1,120 +0,0 @@ -# -*- coding: utf-8 -*- -# Author: Douglas Creager -# This file is placed into the public domain. - -# Calculates the current version number. If possible, this is the -# output of “git describe”, modified to conform to the versioning -# scheme that setuptools uses. If “git describe” returns an error -# (most likely because we're in an unpacked copy of a release tarball, -# rather than in a git working copy), then we fall back on reading the -# contents of the RELEASE-VERSION file. -# -# To use this script, simply import it your setup.py file, and use the -# results of get_git_version() as your package version: -# -# from version import * -# -# setup( -# version=get_git_version(), -# . -# . -# . -# ) -# -# This will automatically update the RELEASE-VERSION file, if -# necessary. Note that the RELEASE-VERSION file should *not* be -# checked into git; please add it to your top-level .gitignore file. -# -# You'll probably want to distribute the RELEASE-VERSION file in your -# sdist tarballs; to do this, just create a MANIFEST.in file that -# contains the following line: -# -# include RELEASE-VERSION - -__all__ = ("get_git_version") - -from subprocess import Popen, PIPE - - -def call_git_describe(abbrev=4): - try: - p = Popen(['git', 'describe', '--abbrev=%d' % abbrev], - stdout=PIPE, stderr=PIPE) - p.stderr.close() - line = p.stdout.readlines()[0] - return line.strip() - - except: - return None - - -def read_release_version(): - try: - f = open("RELEASE-VERSION", "r") - - try: - version = f.readlines()[0] - return version.strip() - - finally: - f.close() - - except: - return None - - -def write_release_version(version): - f = open("RELEASE-VERSION", "w") - f.write("%s\n" % version) - f.close() - - -def get_git_version(abbrev=4): - # Read in the version that's currently in RELEASE-VERSION. - - release_version = read_release_version() - - # First try to get the current version using “git describe”. - - version = call_git_describe(abbrev) - - #adapt to PEP 386 compatible versioning scheme - version = pep386adapt(version) - - # If that doesn't work, fall back on the value that's in - # RELEASE-VERSION. - - if version is None: - version = release_version - - # If we still don't have anything, that's an error. - - if version is None: - raise ValueError("Cannot find the version number!") - - # If the current version is different from what's in the - # RELEASE-VERSION file, update the file to be current. - - if version != release_version: - write_release_version(version) - - # Finally, return the current version. - - return version - - -def pep386adapt(version): - if version is not None: - if '-' in version: - # adapt git-describe version to be in line with PEP 386 - parts = version.split('-') - parts[-2] = 'post'+parts[-2] - version = '.'.join(parts[:-1]) - elif version[0] == 'v': - version = version[1:] - - return version - - -if __name__ == "__main__": - print get_git_version() \ No newline at end of file