From 293a77214b2dcf0ab051bc3235b465289c917348 Mon Sep 17 00:00:00 2001 From: Samuel Li Date: Thu, 28 Mar 2024 07:07:41 -0600 Subject: [PATCH 1/5] define C functions --- include/SPERR_C_API.h | 41 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 41 insertions(+) diff --git a/include/SPERR_C_API.h b/include/SPERR_C_API.h index 4be76f98..75683f2d 100644 --- a/include/SPERR_C_API.h +++ b/include/SPERR_C_API.h @@ -155,6 +155,47 @@ int sperr_trunc_3d( void** dst, /* Output: buffer for the truncated bitstream, allocated by this function */ size_t* dst_len); /* Output: length of `dst` in byte */ +/* + * Experimental API: a single function call for compression and decompression respectively, + * no matter the dimensionality of the input data. + * Note 1: this pair of functions need to be used together; there's no interoperability + * between bitstreams produced by this pair of function and specific 3D/2D functions. + * Note 2: this experimental function might change. For greater API stability, please use + * specific 3D/2D compression and decompression functions. + * + * Compression mode meanings: + * mode == 1 --> fixed bit-per-pixel (BPP) + * mode == 2 --> fixed peak signal-to-noise ratio (PSNR) + * mode == 3 --> fixed point-wise error (PWE) + * + * Return value meanings: + * zero: success + * non-zero: error + */ +int sperr_compress( + const void* src, /* Input: buffer that contains a 2D slice or a 3D volume */ + int is_float, /* Input: input buffer type: 1 == float, 0 == double */ + size_t num_vals, /* Input: number of values in the input buffer */ + int num_dims, /* Input: logical num. of dimensions of the input data. Usually 2 or 3. */ + const size_t* dims, /* Input: length of each logical dimension of the input data. */ + const size_t* chunks, /* Input: length of preferred chunk dims in 3D; ignored in 2D. */ + int mode, /* Input: compression mode */ + double quality, /* Input: compression quality */ + size_t num_threads, /* Input: num. of OpenMP threads in 3D; ignored in 2D. */ + void** dst, /* Output: buffer for the output bitstream, allocated by this function. */ + size_t* dst_len); /* Output: length of `dst` in byte */ + +int sperr_decompress( + const void* src, /* Input: compressed bitstream */ + size_t src_len, /* Input: length of the input bitstream in byte */ + int output_float, /* Input: output data type: 1 == float, 0 == double */ + size_t num_threads, /* Input: num. of OpenMP threads in 3D; ignored in 2D. */ + size_t* out_dims, /* Output: length of each logical dimension of the decompressed data. + It must be memory *already* allocated by the caller with at least 3 + elements to fit the lengh of each dimension in 3D cases. + In 2D cases, the last dimension length will be set to be 1. */ + void** dst); /* Output: buffer for the decompressed data, allocated by this function */ + #ifdef __cplusplus } /* end of extern "C" */ } /* end of namespace C_API */ From d97ed1a24152003e8b7bb9441b3457e9aec982cf Mon Sep 17 00:00:00 2001 From: Samuel Li Date: Thu, 28 Mar 2024 12:50:10 -0600 Subject: [PATCH 2/5] catch all functions pass the compiler --- include/SPERR_C_API.h | 2 +- src/SPERR_C_API.cpp | 67 +++++++++++++++++++++++++++++++++++++++++-- 2 files changed, 66 insertions(+), 3 deletions(-) diff --git a/include/SPERR_C_API.h b/include/SPERR_C_API.h index 75683f2d..27a263bc 100644 --- a/include/SPERR_C_API.h +++ b/include/SPERR_C_API.h @@ -65,7 +65,7 @@ int sperr_comp_2d( * Decompress a 2D SPERR-compressed buffer that is produced by sperr_comp_2d(). * Note that this bitstream shoult NOT contain a header. I.e., a bitstream produced by * sperr_comp_2d() with `out_inc_header = 0`, or with `out_inc_header = 1` and has its - * first 10 bytes stipped. + * first 10 bytes stripped. * * Return value meanings: * 0: success diff --git a/src/SPERR_C_API.cpp b/src/SPERR_C_API.cpp index 867fb681..a13163ae 100644 --- a/src/SPERR_C_API.cpp +++ b/src/SPERR_C_API.cpp @@ -50,7 +50,8 @@ int C_API::sperr_comp_2d(const void* src, return -1; auto stream = sperr::vec8_type(); - if (out_inc_header) { // Assemble a header that's the same as the header in SPERR3D_OMP_C(). + if (out_inc_header) { + // Assemble a header that's the same as the header in SPERR3D_OMP_C(). // The header would contain the following information // -- a version number (1 byte) // -- 8 booleans (1 byte) @@ -58,7 +59,7 @@ int C_API::sperr_comp_2d(const void* src, // // Version number - stream.resize(10); + stream.resize(10); // 10 bytes in total stream[0] = static_cast(SPERR_VERSION_MAJOR); // 8 booleans: @@ -278,3 +279,65 @@ int C_API::sperr_trunc_3d(const void* src, return 0; } } + +int C_API::sperr_compress(const void* src, + int is_float, + size_t num_vals, + int num_dims, + const size_t* dims, + const size_t* chunks, + int mode, + double quality, + size_t num_threads, + void** dst, + size_t* dst_len) +{ + int ret = 0; + + switch (num_dims) { + case 2: + // Sanity check + if (dims[0] * dims[1] != num_vals) + return -1; + ret = sperr_comp_2d(src, is_float, dims[0], dims[1], mode, quality, 1, dst, dst_len); + break; + case 3: + // Sanity check + if (dims[0] * dims[1] * dims[2] != num_vals) + return -1; + ret = sperr_comp_3d(src, is_float, dims[0], dims[1], dims[2], chunks[0], chunks[1], chunks[2], + mode, quality, num_threads, dst, dst_len); + break; + default: + ret = -2; + } + + return ret; +} + +int C_API::sperr_decompress(const void* src, + size_t src_len, + int output_float, + size_t num_threads, + size_t* out_dims, + void** dst) +{ + int is_float = 0; + sperr_parse_header(src, out_dims, out_dims + 1, out_dims + 2, &is_float); + + int ret = 0; + if (out_dims[2] > 1) { + // 3D case + ret = sperr_decomp_3d(src, src_len, output_float, num_threads, out_dims, out_dims + 1, + out_dims + 2, dst); + } + else if (out_dims[2] == 1) { + // 2D case + const auto* ptr = static_cast(src); + ret = sperr_decomp_2d(ptr + 10, src_len - 10, output_float, out_dims[0], out_dims[1], dst); + } + else + ret = -1; + + return ret; +} From cf887f03d1c77b661a53f2ca79241da87f38849b Mon Sep 17 00:00:00 2001 From: Samuel Li Date: Thu, 28 Mar 2024 18:51:09 -0600 Subject: [PATCH 3/5] fix test.sh --- examples/C_API/test.sh | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/examples/C_API/test.sh b/examples/C_API/test.sh index 3b3c9191..a94dda5d 100755 --- a/examples/C_API/test.sh +++ b/examples/C_API/test.sh @@ -29,7 +29,7 @@ CPPDECOMP=cpp.data FILE=./test_data/lena512.float Q=2.5 ./2d.out $FILE 512 512 1 $Q -./bin/sperr2d -c --ftype 32 --dims 512 512 --o_bitstream $CPPSTREAM --o_decomp_f $CPPDECOMP --bpp $Q $FILE +./bin/sperr2d -c --ftype 32 --dims 512 512 --bitstream $CPPSTREAM --decomp_f $CPPDECOMP --bpp $Q $FILE if diff $CSTREAM $CPPSTREAM; then echo "--> C and C++ utilities produce the same bitstream on 2D test data $FILE" @@ -51,7 +51,7 @@ rm -f $CSTREAM $CPPSTREAM $CDECOMP $CPPDECOMP FILE=./test_data/999x999.float Q=90.0 ./2d.out $FILE 999 999 2 $Q -./bin/sperr2d -c --ftype 32 --dims 999 999 --o_bitstream $CPPSTREAM --o_decomp_f $CPPDECOMP --psnr $Q $FILE +./bin/sperr2d -c --ftype 32 --dims 999 999 --bitstream $CPPSTREAM --decomp_f $CPPDECOMP --psnr $Q $FILE if diff $CSTREAM $CPPSTREAM; then echo "--> C and C++ utilities produce the same bitstream on 2D test data $FILE" @@ -74,7 +74,7 @@ rm -f $CSTREAM $CPPSTREAM $CDECOMP $CPPDECOMP FILE=./test_data/vorticity.512_512 Q=1e-8 ./2d.out $FILE 512 512 3 $Q -./bin/sperr2d -c --ftype 32 --dims 512 512 --o_bitstream $CPPSTREAM --o_decomp_f $CPPDECOMP --pwe $Q $FILE +./bin/sperr2d -c --ftype 32 --dims 512 512 --bitstream $CPPSTREAM --decomp_f $CPPDECOMP --pwe $Q $FILE if diff $CSTREAM $CPPSTREAM; then echo "--> C and C++ utilities produce the same bitstream on 2D test data $FILE" @@ -97,7 +97,7 @@ rm -f $CSTREAM $CPPSTREAM $CDECOMP $CPPDECOMP FILE=./test_data/density_128x128x256.d64 Q=2.6 ./3d.out $FILE 128 128 256 1 $Q -d -./bin/sperr3d -c --ftype 64 --dims 128 128 256 --o_bitstream $CPPSTREAM --o_decomp_f64 $CPPDECOMP --bpp $Q $FILE +./bin/sperr3d -c --ftype 64 --dims 128 128 256 --bitstream $CPPSTREAM --decomp_d $CPPDECOMP --bpp $Q $FILE if diff $CSTREAM $CPPSTREAM; then echo "--> C and C++ utilities produce the same bitstream on 3D test data $FILE" @@ -120,7 +120,7 @@ rm -f $CSTREAM $CPPSTREAM $CDECOMP $CPPDECOMP FILE=./test_data/density_128x128x256.d64 Q=102.5 ./3d.out $FILE 128 128 256 2 $Q -d -./bin/sperr3d -c --ftype 64 --dims 128 128 256 --o_bitstream $CPPSTREAM --o_decomp_f64 $CPPDECOMP --psnr $Q $FILE +./bin/sperr3d -c --ftype 64 --dims 128 128 256 --bitstream $CPPSTREAM --decomp_d $CPPDECOMP --psnr $Q $FILE if diff $CSTREAM $CPPSTREAM; then echo "--> C and C++ utilities produce the same bitstream on 3D test data $FILE" @@ -143,7 +143,7 @@ rm -f $CSTREAM $CPPSTREAM $CDECOMP $CPPDECOMP FILE=./test_data/density_128x128x256.d64 Q=4e-5 ./3d.out $FILE 128 128 256 3 $Q -d -./bin/sperr3d -c --ftype 64 --dims 128 128 256 --o_bitstream $CPPSTREAM --o_decomp_f64 $CPPDECOMP --pwe $Q $FILE +./bin/sperr3d -c --ftype 64 --dims 128 128 256 --bitstream $CPPSTREAM --decomp_d $CPPDECOMP --pwe $Q $FILE if diff $CSTREAM $CPPSTREAM; then echo "--> C and C++ utilities produce the same bitstream on 3D test data $FILE" From 9619cdd63066efb88ce8da3a06a48228fa31b44b Mon Sep 17 00:00:00 2001 From: Samuel Li Date: Thu, 28 Mar 2024 19:40:48 -0600 Subject: [PATCH 4/5] 2d catch all passes test --- examples/C_API/2d.c | 29 +++++++++++++++++++++++++++++ 1 file changed, 29 insertions(+) diff --git a/examples/C_API/2d.c b/examples/C_API/2d.c index e77cc5e1..351e7372 100644 --- a/examples/C_API/2d.c +++ b/examples/C_API/2d.c @@ -3,6 +3,7 @@ #include #include #include +#include /* memcmp() */ /* * Given a file name, this function reads in its content and allocates a buffer `dst` to store it. @@ -106,8 +107,36 @@ int main(int argc, char** argv) fwrite(outbuf, sizeof(double), out_dimx * out_dimy, f); fclose(f); + /* Test the "catch all" function calls. */ + size_t dims[3] = {dimx, dimy, 1}; + size_t stream2_len = 0; + void* stream2 = NULL; + rtn = sperr_compress(inbuf, is_float, dimx * dimy, 2, dims, dims, mode, quality, 1, + &stream2, &stream2_len); + if (rtn != 0) { + printf("Catch-all compression failed with error %d\n", rtn); + return 1; + } + if (stream2_len != stream_len || memcmp(bitstream, stream2, stream_len)) { + printf("Catch-all compression not consistent!\n"); + return 1; + } + + void* outbuf2 = NULL; + rtn = sperr_decompress(stream2, stream2_len, is_float, 1, dims, &outbuf2); + if (rtn != 0) { + printf("Catch-all decompression failed with error %d\n", rtn); + return 1; + } + if (memcmp(outbuf, outbuf2, dimx * dimy * (is_float ? sizeof(float) : sizeof(double)))) { + printf("Catch-all decompression not consistent!\n"); + return 1; + } + /* Clean up */ free(outbuf); + free(outbuf2); free(bitstream); + free(stream2); free(inbuf); } From 526163dd21e51683a5658e51717ad8a59e718df7 Mon Sep 17 00:00:00 2001 From: Samuel Li Date: Fri, 29 Mar 2024 12:05:04 -0600 Subject: [PATCH 5/5] 3d catch all pass tests too --- examples/C_API/2d.c | 2 +- examples/C_API/3d.c | 37 ++++++++++++++++++++++++++++++++----- examples/C_API/test.sh | 6 +++--- include/SPERR_C_API.h | 10 ++++------ 4 files changed, 40 insertions(+), 15 deletions(-) diff --git a/examples/C_API/2d.c b/examples/C_API/2d.c index 351e7372..a3a33c96 100644 --- a/examples/C_API/2d.c +++ b/examples/C_API/2d.c @@ -107,7 +107,7 @@ int main(int argc, char** argv) fwrite(outbuf, sizeof(double), out_dimx * out_dimy, f); fclose(f); - /* Test the "catch all" function calls. */ + /* Test the "catch all" functions. */ size_t dims[3] = {dimx, dimy, 1}; size_t stream2_len = 0; void* stream2 = NULL; diff --git a/examples/C_API/3d.c b/examples/C_API/3d.c index c6905459..a8a0255b 100644 --- a/examples/C_API/3d.c +++ b/examples/C_API/3d.c @@ -3,6 +3,7 @@ #include #include #include +#include /* memcmp() */ /* * Given a file name, this function reads in its content and allocates a buffer `dst` to store it. @@ -45,9 +46,7 @@ int main(int argc, char** argv) is_float = 0; /* Hard code the preferred chunk size in this example. */ - const size_t chunk_x = 256; - const size_t chunk_y = 256; - const size_t chunk_z = 256; + const size_t chunks[3] = {128, 128, 128}; /* Read in a file and put its content in `inbuf` */ void* inbuf = NULL; /* Will be free'd later */ @@ -70,8 +69,8 @@ int main(int argc, char** argv) /* Compress `inbuf` and put the compressed bitstream in `bitstream` */ void* bitstream = NULL; /* Will be free'd later */ size_t stream_len = 0; - int rtn = sperr_comp_3d(inbuf, is_float, dimx, dimy, dimz, chunk_x, chunk_y, chunk_z, mode, - quality, nthreads, &bitstream, &stream_len); + int rtn = sperr_comp_3d(inbuf, is_float, dimx, dimy, dimz, chunks[0], chunks[1], chunks[2], + mode, quality, nthreads, &bitstream, &stream_len); if (rtn != 0) { printf("Compression error with code %d\n", rtn); return rtn; @@ -114,8 +113,36 @@ int main(int argc, char** argv) fclose(f); f = NULL; + /* Test the "catch all" functions. */ + size_t dims[3] = {dimx, dimy, dimz}; + size_t stream2_len = 0; + void* stream2 = NULL; + rtn = sperr_compress(inbuf, is_float, dimx * dimy * dimz, 3, dims, chunks, mode, quality, 0, + &stream2, &stream2_len); + if (rtn != 0) { + printf("Catch-all compression failed with error %d\n", rtn); + return 1; + } + if (stream2_len != stream_len || memcmp(bitstream, stream2, stream2_len)) { + printf("Catch-all compression not consistent!\n"); + return 1; + } + + void* outbuf2 = NULL; + rtn = sperr_decompress(stream2, stream2_len, is_float, 0, dims, &outbuf2); + if (rtn != 0) { + printf("Catch-all decompression failed with error %d\n", rtn); + return 1; + } + if (memcmp(outbuf, outbuf2, dimx * dimy * dimz * (is_float ? sizeof(float) : sizeof(double)))) { + printf("Catch-all decompression not consistent!\n"); + return 1; + } + /* Clean up */ free(outbuf); + free(outbuf2); + free(stream2); free(bitstream); free(inbuf); } diff --git a/examples/C_API/test.sh b/examples/C_API/test.sh index a94dda5d..45a21782 100755 --- a/examples/C_API/test.sh +++ b/examples/C_API/test.sh @@ -97,7 +97,7 @@ rm -f $CSTREAM $CPPSTREAM $CDECOMP $CPPDECOMP FILE=./test_data/density_128x128x256.d64 Q=2.6 ./3d.out $FILE 128 128 256 1 $Q -d -./bin/sperr3d -c --ftype 64 --dims 128 128 256 --bitstream $CPPSTREAM --decomp_d $CPPDECOMP --bpp $Q $FILE +./bin/sperr3d -c --ftype 64 --dims 128 128 256 --chunks 128 128 128 --bitstream $CPPSTREAM --decomp_d $CPPDECOMP --bpp $Q $FILE if diff $CSTREAM $CPPSTREAM; then echo "--> C and C++ utilities produce the same bitstream on 3D test data $FILE" @@ -120,7 +120,7 @@ rm -f $CSTREAM $CPPSTREAM $CDECOMP $CPPDECOMP FILE=./test_data/density_128x128x256.d64 Q=102.5 ./3d.out $FILE 128 128 256 2 $Q -d -./bin/sperr3d -c --ftype 64 --dims 128 128 256 --bitstream $CPPSTREAM --decomp_d $CPPDECOMP --psnr $Q $FILE +./bin/sperr3d -c --ftype 64 --dims 128 128 256 --chunks 128 128 128 --bitstream $CPPSTREAM --decomp_d $CPPDECOMP --psnr $Q $FILE if diff $CSTREAM $CPPSTREAM; then echo "--> C and C++ utilities produce the same bitstream on 3D test data $FILE" @@ -143,7 +143,7 @@ rm -f $CSTREAM $CPPSTREAM $CDECOMP $CPPDECOMP FILE=./test_data/density_128x128x256.d64 Q=4e-5 ./3d.out $FILE 128 128 256 3 $Q -d -./bin/sperr3d -c --ftype 64 --dims 128 128 256 --bitstream $CPPSTREAM --decomp_d $CPPDECOMP --pwe $Q $FILE +./bin/sperr3d -c --ftype 64 --dims 128 128 256 --chunks 128 128 128 --bitstream $CPPSTREAM --decomp_d $CPPDECOMP --pwe $Q $FILE if diff $CSTREAM $CPPSTREAM; then echo "--> C and C++ utilities produce the same bitstream on 3D test data $FILE" diff --git a/include/SPERR_C_API.h b/include/SPERR_C_API.h index 27a263bc..47f198d0 100644 --- a/include/SPERR_C_API.h +++ b/include/SPERR_C_API.h @@ -156,12 +156,10 @@ int sperr_trunc_3d( size_t* dst_len); /* Output: length of `dst` in byte */ /* - * Experimental API: a single function call for compression and decompression respectively, - * no matter the dimensionality of the input data. - * Note 1: this pair of functions need to be used together; there's no interoperability - * between bitstreams produced by this pair of function and specific 3D/2D functions. - * Note 2: this experimental function might change. For greater API stability, please use - * specific 3D/2D compression and decompression functions. + * Experimental API: a single pair of functions that compresses and decompresses, + * regardless of the dimensionality of the input data. + * Note: these functions are experimental in nature, and their behaviors/signatures might change. + * For greater API stability, please use specific 3D/2D compression and decompression functions. * * Compression mode meanings: * mode == 1 --> fixed bit-per-pixel (BPP)