diff --git a/.gitmodules b/.gitmodules index ef9d7a0d5172..db26afefbb5e 100644 --- a/.gitmodules +++ b/.gitmodules @@ -85,3 +85,6 @@ [submodule "thirdparty/cpp-httplib"] path = thirdparty/cpp-httplib url = https://github.com/yhirose/cpp-httplib +[submodule "thirdparty/libdeflate"] + path = thirdparty/libdeflate + url = https://github.com/ebiggers/libdeflate.git diff --git a/requirements/windows.txt b/requirements/windows.txt index 9bd83d7286fc..be1f38e158b9 100644 --- a/requirements/windows.txt +++ b/requirements/windows.txt @@ -19,6 +19,7 @@ gdcm gtest hidapi jsoncpp +libdeflate libe57format libharu libzip diff --git a/source/MRMesh/CMakeLists.txt b/source/MRMesh/CMakeLists.txt index 3621ac029714..5015198d36a8 100644 --- a/source/MRMesh/CMakeLists.txt +++ b/source/MRMesh/CMakeLists.txt @@ -65,6 +65,27 @@ ELSE() ENDIF() target_link_libraries(${PROJECT_NAME} PRIVATE libzip::zip) +# libdeflate drives MRZlib's zlibCompressStream (compress path only; +# decompression and the compressZip/libzip pipeline remain on zlib). We avoid +# libdeflate's own CMake config package here: the config it installs bakes +# the build-time absolute include path via $ +# rather than a relocatable relative path, which breaks the Ubuntu/Emscripten +# Docker flow where thirdparty is built at /home/MeshLib and then COPYed to +# /usr/local/lib/meshlib-thirdparty-lib/. Plain find_library / find_path uses +# the search paths already set up by MeshLib's root CMakeLists (for the +# submodule build) and the vcpkg toolchain (on Windows); same binary either +# way. +find_library(LIBDEFLATE_LIBRARY + NAMES deflate libdeflate + REQUIRED +) +find_path(LIBDEFLATE_INCLUDE_DIR + NAMES libdeflate.h + REQUIRED +) +target_link_libraries(${PROJECT_NAME} PRIVATE ${LIBDEFLATE_LIBRARY}) +target_include_directories(${PROJECT_NAME} PRIVATE ${LIBDEFLATE_INCLUDE_DIR}) + # TODO: CMake config target_include_directories(${PROJECT_NAME} PUBLIC diff --git a/source/MRMesh/MRZlib.cpp b/source/MRMesh/MRZlib.cpp index 259298dc21aa..a4d7c30e7ad7 100644 --- a/source/MRMesh/MRZlib.cpp +++ b/source/MRMesh/MRZlib.cpp @@ -2,9 +2,12 @@ #include "MRBuffer.h" #include "MRFinally.h" +#include #include #include +#include +#include namespace { @@ -61,58 +64,69 @@ namespace MR Expected zlibCompressStream( std::istream& in, std::ostream& out, const ZlibCompressParams& params ) { - Buffer inChunk( cChunkSize ), outChunk( cChunkSize ); - z_stream stream { - .zalloc = Z_NULL, - .zfree = Z_NULL, - .opaque = Z_NULL, - }; - int ret; - if ( Z_OK != ( ret = deflateInit2( &stream, params.level, Z_DEFLATED, windowBitsFor( params.rawDeflate ), kDefaultMemLevel, Z_DEFAULT_STRATEGY ) ) ) - return unexpected( zlibToString( ret ) ); - - MR_FINALLY { - deflateEnd( &stream ); - }; - - if ( params.stats ) - *params.stats = {}; - + // libdeflate exposes only a whole-buffer compression API, so drain the + // input stream into memory first. Memory ceiling = input size; callers + // that need streaming compression of very large inputs should chunk at + // a layer above this function (only this overload takes the fast path; + // the other direction still streams in zlibDecompressStream below). + std::vector inBuf; while ( !in.eof() ) { - in.read( inChunk.data(), inChunk.size() ); + const size_t offset = inBuf.size(); + inBuf.resize( offset + cChunkSize ); + in.read( reinterpret_cast( inBuf.data() + offset ), cChunkSize ); if ( in.bad() ) return unexpected( "I/O error" ); - stream.next_in = reinterpret_cast( inChunk.data() ); - stream.avail_in = (unsigned)in.gcount(); - assert( stream.avail_in <= (unsigned)inChunk.size() ); + inBuf.resize( offset + static_cast( in.gcount() ) ); + } - if ( params.stats ) - { - params.stats->crc32 = (uint32_t)crc32( params.stats->crc32, stream.next_in, stream.avail_in ); - params.stats->uncompressedSize += stream.avail_in; - } + // libdeflate accepts levels 1-12. Map the zlib-style conventions we + // inherit from ZlibCompressParams: -1 (default) -> libdeflate 6, which + // matches zlib's default; 0 (zlib's "store-only") -> libdeflate 1 + // because libdeflate has no stored-only mode. The ±4-byte tolerance on + // ZlibCompressStats's size check absorbs the resulting minor differences + // versus the stock-zlib reference blobs. + int level = params.level; + if ( level < 0 ) + level = 6; + else if ( level == 0 ) + level = 1; + else if ( level > 12 ) + level = 12; - const auto flush = in.eof() ? Z_FINISH : Z_NO_FLUSH; - do - { - stream.next_out = reinterpret_cast( outChunk.data() ); - stream.avail_out = (unsigned)outChunk.size(); - ret = deflate( &stream, flush ); - if ( Z_OK != ret && Z_STREAM_END != ret ) - return unexpected( zlibToString( ret ) ); - - assert( stream.avail_out <= (unsigned)outChunk.size() ); - const unsigned written = (unsigned)outChunk.size() - stream.avail_out; - out.write( outChunk.data(), written ); - if ( out.bad() ) - return unexpected( "I/O error" ); - if ( params.stats ) - params.stats->compressedSize += written; - } - while ( stream.avail_out == 0 ); + if ( params.stats ) + { + params.stats->crc32 = libdeflate_crc32( 0, inBuf.data(), inBuf.size() ); + params.stats->uncompressedSize = inBuf.size(); + params.stats->compressedSize = 0; } + libdeflate_compressor* comp = libdeflate_alloc_compressor( level ); + if ( !comp ) + return unexpected( "libdeflate_alloc_compressor failed" ); + MR_FINALLY { + libdeflate_free_compressor( comp ); + }; + + const bool raw = params.rawDeflate; + const size_t bound = raw + ? libdeflate_deflate_compress_bound( comp, inBuf.size() ) + : libdeflate_zlib_compress_bound( comp, inBuf.size() ); + std::vector outBuf( bound ); + + const size_t produced = raw + ? libdeflate_deflate_compress( comp, inBuf.data(), inBuf.size(), outBuf.data(), outBuf.size() ) + : libdeflate_zlib_compress( comp, inBuf.data(), inBuf.size(), outBuf.data(), outBuf.size() ); + if ( produced == 0 ) + return unexpected( "libdeflate compression failed" ); + + out.write( reinterpret_cast( outBuf.data() ), static_cast( produced ) ); + if ( out.bad() ) + return unexpected( "I/O error" ); + + if ( params.stats ) + params.stats->compressedSize = produced; + return {}; } diff --git a/source/MRTest/MRZipCompressTests.cpp b/source/MRTest/MRZipCompressTests.cpp index 9cf9ff58b068..2e8b3eeacf7a 100644 --- a/source/MRTest/MRZipCompressTests.cpp +++ b/source/MRTest/MRZipCompressTests.cpp @@ -28,7 +28,7 @@ TEST( MRMesh, CompressSphereToZip ) UniqueTemporaryFolder srcFolder; ASSERT_TRUE( bool( srcFolder ) ); - constexpr int targetVerts = 1000; // increase it to make the file being compressed larger, 100'000 vertices -> 12M bytes + constexpr int targetVerts = 100000; // increase it to make the file being compressed larger, 100'000 vertices -> 12M bytes SphereParams params; params.radius = 1.0f; params.numMeshVertices = targetVerts; @@ -82,9 +82,9 @@ TEST( MRMesh, CompressManySmallFilesToZip ) ASSERT_TRUE( bool( srcFolder ) ); // increase both below numbers to make the files being compressed larger, 200 * 2 files * 60'000 bytes -> 24M bytes - constexpr int numBinaryFiles = 20; + constexpr int numBinaryFiles = 200; constexpr int numJsonFiles = numBinaryFiles; - constexpr size_t bytesPerFile = 6000; + constexpr size_t bytesPerFile = 60000; // Simple LCG used to produce deterministic pseudo-random bytes. // Keeps the test reproducible across runs and platforms while avoiding diff --git a/thirdparty/CMakeLists.txt b/thirdparty/CMakeLists.txt index e4734cf29af3..1f8c4d125a54 100644 --- a/thirdparty/CMakeLists.txt +++ b/thirdparty/CMakeLists.txt @@ -85,6 +85,17 @@ ENDIF() add_subdirectory(./OpenCTM-git ./OpenCTM) +# Build libdeflate from submodule on every platform except Windows; Windows +# picks it up from vcpkg (see requirements/windows.txt) because the MSBuild +# matrix legs don't run the thirdparty CMake build. MRZlib's compress path +# uses it on all platforms regardless of where the binary comes from. +IF(NOT WIN32) + set(LIBDEFLATE_BUILD_GZIP OFF CACHE BOOL "") + set(LIBDEFLATE_BUILD_TESTS OFF CACHE BOOL "") + set(LIBDEFLATE_BUILD_STATIC_LIB OFF CACHE BOOL "") + add_subdirectory(./libdeflate) +ENDIF() + option(PHMAP_INSTALL "" ON) add_subdirectory(./parallel-hashmap) diff --git a/thirdparty/libdeflate b/thirdparty/libdeflate new file mode 160000 index 000000000000..96836d7d9d10 --- /dev/null +++ b/thirdparty/libdeflate @@ -0,0 +1 @@ +Subproject commit 96836d7d9d10e3e0d53e6edb54eb908514e336c4