From d91067adb15951899d1d3f2728bbfd866af85dd6 Mon Sep 17 00:00:00 2001 From: Fedor Chelnokov Date: Wed, 22 Apr 2026 10:14:47 +0300 Subject: [PATCH 1/4] ZipCompressTests: performance testing --- source/MRTest/MRZipCompressTests.cpp | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/source/MRTest/MRZipCompressTests.cpp b/source/MRTest/MRZipCompressTests.cpp index f8b27379d917..2ec47f238475 100644 --- a/source/MRTest/MRZipCompressTests.cpp +++ b/source/MRTest/MRZipCompressTests.cpp @@ -27,7 +27,7 @@ TEST( MRMesh, CompressSphereToZip ) UniqueTemporaryFolder srcFolder; ASSERT_TRUE( bool( srcFolder ) ); - constexpr int targetVerts = 1000; // increase it to make the file being compressed larger, 100'000 vertices -> 12M bytes + constexpr int targetVerts = 100000; // increase it to make the file being compressed larger, 100'000 vertices -> 12M bytes SphereParams params; params.radius = 1.0f; params.numMeshVertices = targetVerts; @@ -81,9 +81,9 @@ TEST( MRMesh, CompressManySmallFilesToZip ) ASSERT_TRUE( bool( srcFolder ) ); // increase both below numbers to make the files being compressed larger, 200 * 2 files * 60'000 bytes -> 24M bytes - constexpr int numBinaryFiles = 20; + constexpr int numBinaryFiles = 200; constexpr int numJsonFiles = numBinaryFiles; - constexpr size_t bytesPerFile = 6000; + constexpr size_t bytesPerFile = 60000; // Simple LCG used to produce deterministic pseudo-random bytes. // Keeps the test reproducible across runs and platforms while avoiding From 4b90342f324d808797714e1a0abc85a91c5f65ef Mon Sep 17 00:00:00 2001 From: Fedor Chelnokov Date: Tue, 21 Apr 2026 22:04:52 +0300 Subject: [PATCH 2/4] MRZlib: implement zlibCompressStream via libdeflate MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Only the stats-enabled zlibCompressStream(in, out, const ZlibCompressParams&) overload changes: it now reads the input stream into memory and hands compression + CRC-32 to libdeflate (~20-40% faster than zlib-ng, larger delta in Debug builds since libdeflate's base implementation doesn't depend on runtime SIMD dispatch). The thin int-level overload continues to forward to this one. Decompression (zlibDecompressStream) and the compressZip/libzip pipeline both stay on stock zlib -- this PR is intentionally scoped to the single function the user requested. Tradeoff: libdeflate has no streaming input API, so the compressor materialises the whole input in memory. Memory ceiling = input stream size; callers that compress very large inputs (multi-GB) should chunk above this function. Level mapping: ZlibCompressParams's -1 (zlib default) -> libdeflate 6; ZlibCompressParams's 0 (zlib stored-only) -> libdeflate 1 because libdeflate has no stored-only mode. The existing MRMesh.ZlibCompressStats test allows ±4 bytes around the stock-zlib reference sizes, which is enough to absorb the minor byte-level differences in libdeflate's output. Adds thirdparty/libdeflate submodule pinned at v1.24, built as a shared library (LIBDEFLATE_BUILD_GZIP/STATIC/TESTS off) from thirdparty/ CMakeLists.txt on all platforms. MRMesh finds it via find_package (libdeflate CONFIG) hinted at the thirdparty install prefix. --- .gitmodules | 3 ++ source/MRMesh/CMakeLists.txt | 10 ++++ source/MRMesh/MRZlib.cpp | 102 ++++++++++++++++++++--------------- thirdparty/CMakeLists.txt | 7 +++ thirdparty/libdeflate | 1 + 5 files changed, 79 insertions(+), 44 deletions(-) create mode 160000 thirdparty/libdeflate diff --git a/.gitmodules b/.gitmodules index ef9d7a0d5172..db26afefbb5e 100644 --- a/.gitmodules +++ b/.gitmodules @@ -85,3 +85,6 @@ [submodule "thirdparty/cpp-httplib"] path = thirdparty/cpp-httplib url = https://github.com/yhirose/cpp-httplib +[submodule "thirdparty/libdeflate"] + path = thirdparty/libdeflate + url = https://github.com/ebiggers/libdeflate.git diff --git a/source/MRMesh/CMakeLists.txt b/source/MRMesh/CMakeLists.txt index 3621ac029714..3dc141ff3441 100644 --- a/source/MRMesh/CMakeLists.txt +++ b/source/MRMesh/CMakeLists.txt @@ -65,6 +65,16 @@ ELSE() ENDIF() target_link_libraries(${PROJECT_NAME} PRIVATE libzip::zip) +# libdeflate drives MRZlib's zlibCompressStream (compress path only; +# decompression and the compressZip/libzip pipeline remain on zlib). Built +# from thirdparty/libdeflate and installed to the thirdparty prefix; hint +# find_package there explicitly since the prefix is not on CMake's default +# search path on all platforms. +find_package(libdeflate CONFIG REQUIRED + HINTS ${MESHLIB_THIRDPARTY_ROOT_DIR}/lib/cmake/libdeflate +) +target_link_libraries(${PROJECT_NAME} PRIVATE libdeflate::libdeflate_shared) + # TODO: CMake config target_include_directories(${PROJECT_NAME} PUBLIC diff --git a/source/MRMesh/MRZlib.cpp b/source/MRMesh/MRZlib.cpp index 259298dc21aa..a4d7c30e7ad7 100644 --- a/source/MRMesh/MRZlib.cpp +++ b/source/MRMesh/MRZlib.cpp @@ -2,9 +2,12 @@ #include "MRBuffer.h" #include "MRFinally.h" +#include #include #include +#include +#include namespace { @@ -61,58 +64,69 @@ namespace MR Expected zlibCompressStream( std::istream& in, std::ostream& out, const ZlibCompressParams& params ) { - Buffer inChunk( cChunkSize ), outChunk( cChunkSize ); - z_stream stream { - .zalloc = Z_NULL, - .zfree = Z_NULL, - .opaque = Z_NULL, - }; - int ret; - if ( Z_OK != ( ret = deflateInit2( &stream, params.level, Z_DEFLATED, windowBitsFor( params.rawDeflate ), kDefaultMemLevel, Z_DEFAULT_STRATEGY ) ) ) - return unexpected( zlibToString( ret ) ); - - MR_FINALLY { - deflateEnd( &stream ); - }; - - if ( params.stats ) - *params.stats = {}; - + // libdeflate exposes only a whole-buffer compression API, so drain the + // input stream into memory first. Memory ceiling = input size; callers + // that need streaming compression of very large inputs should chunk at + // a layer above this function (only this overload takes the fast path; + // the other direction still streams in zlibDecompressStream below). + std::vector inBuf; while ( !in.eof() ) { - in.read( inChunk.data(), inChunk.size() ); + const size_t offset = inBuf.size(); + inBuf.resize( offset + cChunkSize ); + in.read( reinterpret_cast( inBuf.data() + offset ), cChunkSize ); if ( in.bad() ) return unexpected( "I/O error" ); - stream.next_in = reinterpret_cast( inChunk.data() ); - stream.avail_in = (unsigned)in.gcount(); - assert( stream.avail_in <= (unsigned)inChunk.size() ); + inBuf.resize( offset + static_cast( in.gcount() ) ); + } - if ( params.stats ) - { - params.stats->crc32 = (uint32_t)crc32( params.stats->crc32, stream.next_in, stream.avail_in ); - params.stats->uncompressedSize += stream.avail_in; - } + // libdeflate accepts levels 1-12. Map the zlib-style conventions we + // inherit from ZlibCompressParams: -1 (default) -> libdeflate 6, which + // matches zlib's default; 0 (zlib's "store-only") -> libdeflate 1 + // because libdeflate has no stored-only mode. The ±4-byte tolerance on + // ZlibCompressStats's size check absorbs the resulting minor differences + // versus the stock-zlib reference blobs. + int level = params.level; + if ( level < 0 ) + level = 6; + else if ( level == 0 ) + level = 1; + else if ( level > 12 ) + level = 12; - const auto flush = in.eof() ? Z_FINISH : Z_NO_FLUSH; - do - { - stream.next_out = reinterpret_cast( outChunk.data() ); - stream.avail_out = (unsigned)outChunk.size(); - ret = deflate( &stream, flush ); - if ( Z_OK != ret && Z_STREAM_END != ret ) - return unexpected( zlibToString( ret ) ); - - assert( stream.avail_out <= (unsigned)outChunk.size() ); - const unsigned written = (unsigned)outChunk.size() - stream.avail_out; - out.write( outChunk.data(), written ); - if ( out.bad() ) - return unexpected( "I/O error" ); - if ( params.stats ) - params.stats->compressedSize += written; - } - while ( stream.avail_out == 0 ); + if ( params.stats ) + { + params.stats->crc32 = libdeflate_crc32( 0, inBuf.data(), inBuf.size() ); + params.stats->uncompressedSize = inBuf.size(); + params.stats->compressedSize = 0; } + libdeflate_compressor* comp = libdeflate_alloc_compressor( level ); + if ( !comp ) + return unexpected( "libdeflate_alloc_compressor failed" ); + MR_FINALLY { + libdeflate_free_compressor( comp ); + }; + + const bool raw = params.rawDeflate; + const size_t bound = raw + ? libdeflate_deflate_compress_bound( comp, inBuf.size() ) + : libdeflate_zlib_compress_bound( comp, inBuf.size() ); + std::vector outBuf( bound ); + + const size_t produced = raw + ? libdeflate_deflate_compress( comp, inBuf.data(), inBuf.size(), outBuf.data(), outBuf.size() ) + : libdeflate_zlib_compress( comp, inBuf.data(), inBuf.size(), outBuf.data(), outBuf.size() ); + if ( produced == 0 ) + return unexpected( "libdeflate compression failed" ); + + out.write( reinterpret_cast( outBuf.data() ), static_cast( produced ) ); + if ( out.bad() ) + return unexpected( "I/O error" ); + + if ( params.stats ) + params.stats->compressedSize = produced; + return {}; } diff --git a/thirdparty/CMakeLists.txt b/thirdparty/CMakeLists.txt index e4734cf29af3..14f25a1f6217 100644 --- a/thirdparty/CMakeLists.txt +++ b/thirdparty/CMakeLists.txt @@ -85,6 +85,13 @@ ENDIF() add_subdirectory(./OpenCTM-git ./OpenCTM) +# Always build libdeflate from submodule (MRZlib's compress path delegates +# to it on every platform). +set(LIBDEFLATE_BUILD_GZIP OFF CACHE BOOL "") +set(LIBDEFLATE_BUILD_TESTS OFF CACHE BOOL "") +set(LIBDEFLATE_BUILD_STATIC_LIB OFF CACHE BOOL "") +add_subdirectory(./libdeflate) + option(PHMAP_INSTALL "" ON) add_subdirectory(./parallel-hashmap) diff --git a/thirdparty/libdeflate b/thirdparty/libdeflate new file mode 160000 index 000000000000..96836d7d9d10 --- /dev/null +++ b/thirdparty/libdeflate @@ -0,0 +1 @@ +Subproject commit 96836d7d9d10e3e0d53e6edb54eb908514e336c4 From b4f6ab730cbc446843271e72fc9972d43d1d7288 Mon Sep 17 00:00:00 2001 From: Fedor Chelnokov Date: Tue, 21 Apr 2026 22:27:39 +0300 Subject: [PATCH 3/4] libdeflate: use vcpkg on Windows, submodule on every other platform Windows CI (both MSBuild and CMake legs) doesn't run the thirdparty CMake build step that builds submodules into the install prefix -- dependencies come from vcpkg via thirdparty/install.bat. So gate the add_subdirectory(./libdeflate) on NOT WIN32 and add 'libdeflate' to requirements/windows.txt so vcpkg supplies it there. All other platforms keep building the submodule-pinned v1.24: - Ubuntu apt (22.04 ships 1.10, 24.04 ships 1.19 -- both older than v1.24, so we prefer the pinned submodule build for consistent perf) - Rocky Linux vcpkg (CMake toolchain, can consume submodule fine) - macOS Homebrew (brew 1.25 is current but keeping submodule for version parity across non-Windows legs) - Emscripten (no package manager) MRMesh's find_package(libdeflate CONFIG HINTS ...) call is unchanged and works uniformly: on Windows the vcpkg toolchain's CMAKE_PREFIX_PATH resolves it; elsewhere the HINTS path at the thirdparty install prefix picks up our submodule build. Same libdeflate::libdeflate_shared target. --- requirements/windows.txt | 1 + thirdparty/CMakeLists.txt | 16 ++++++++++------ 2 files changed, 11 insertions(+), 6 deletions(-) diff --git a/requirements/windows.txt b/requirements/windows.txt index 9bd83d7286fc..be1f38e158b9 100644 --- a/requirements/windows.txt +++ b/requirements/windows.txt @@ -19,6 +19,7 @@ gdcm gtest hidapi jsoncpp +libdeflate libe57format libharu libzip diff --git a/thirdparty/CMakeLists.txt b/thirdparty/CMakeLists.txt index 14f25a1f6217..1f8c4d125a54 100644 --- a/thirdparty/CMakeLists.txt +++ b/thirdparty/CMakeLists.txt @@ -85,12 +85,16 @@ ENDIF() add_subdirectory(./OpenCTM-git ./OpenCTM) -# Always build libdeflate from submodule (MRZlib's compress path delegates -# to it on every platform). -set(LIBDEFLATE_BUILD_GZIP OFF CACHE BOOL "") -set(LIBDEFLATE_BUILD_TESTS OFF CACHE BOOL "") -set(LIBDEFLATE_BUILD_STATIC_LIB OFF CACHE BOOL "") -add_subdirectory(./libdeflate) +# Build libdeflate from submodule on every platform except Windows; Windows +# picks it up from vcpkg (see requirements/windows.txt) because the MSBuild +# matrix legs don't run the thirdparty CMake build. MRZlib's compress path +# uses it on all platforms regardless of where the binary comes from. +IF(NOT WIN32) + set(LIBDEFLATE_BUILD_GZIP OFF CACHE BOOL "") + set(LIBDEFLATE_BUILD_TESTS OFF CACHE BOOL "") + set(LIBDEFLATE_BUILD_STATIC_LIB OFF CACHE BOOL "") + add_subdirectory(./libdeflate) +ENDIF() option(PHMAP_INSTALL "" ON) add_subdirectory(./parallel-hashmap) From 594977a01e0a0a50cced5d0be02038093e7ec55e Mon Sep 17 00:00:00 2001 From: Fedor Chelnokov Date: Tue, 21 Apr 2026 22:55:59 +0300 Subject: [PATCH 4/4] MRMesh: find libdeflate via find_library/find_path (workaround non-relocatable config) libdeflate's own libdeflateConfig.cmake is not relocatable: it sets the exported target's INTERFACE_INCLUDE_DIRECTORIES via $ (absolute path). On MeshLib's Ubuntu and Emscripten Docker images, thirdparty is built at /home/MeshLib during the build stage and then COPYed to /usr/local/lib/meshlib- thirdparty-lib/ in the production stage. The baked-in /home/MeshLib/ include path no longer exists in the production image, so CMake fails the imported target's path-existence check with: Imported target "libdeflate::libdeflate_shared" includes non-existent path "/home/MeshLib/include" in its INTERFACE_INCLUDE_DIRECTORIES Other thirdparty libs (httplib, nlohmann_json, ...) are fine because their configs use the relative CMAKE_INSTALL_INCLUDEDIR and stay relocatable. libdeflate up to at least v1.24 does not. Switch to plain find_library + find_path. MeshLib's root CMakeLists already adds ${MESHLIB_THIRDPARTY_ROOT_DIR}/lib and /include to the link/include search paths, and the vcpkg toolchain on Windows does the equivalent. Same binary, no config-package indirection, avoids the bug. --- source/MRMesh/CMakeLists.txt | 25 ++++++++++++++++++------- 1 file changed, 18 insertions(+), 7 deletions(-) diff --git a/source/MRMesh/CMakeLists.txt b/source/MRMesh/CMakeLists.txt index 3dc141ff3441..5015198d36a8 100644 --- a/source/MRMesh/CMakeLists.txt +++ b/source/MRMesh/CMakeLists.txt @@ -66,14 +66,25 @@ ENDIF() target_link_libraries(${PROJECT_NAME} PRIVATE libzip::zip) # libdeflate drives MRZlib's zlibCompressStream (compress path only; -# decompression and the compressZip/libzip pipeline remain on zlib). Built -# from thirdparty/libdeflate and installed to the thirdparty prefix; hint -# find_package there explicitly since the prefix is not on CMake's default -# search path on all platforms. -find_package(libdeflate CONFIG REQUIRED - HINTS ${MESHLIB_THIRDPARTY_ROOT_DIR}/lib/cmake/libdeflate +# decompression and the compressZip/libzip pipeline remain on zlib). We avoid +# libdeflate's own CMake config package here: the config it installs bakes +# the build-time absolute include path via $ +# rather than a relocatable relative path, which breaks the Ubuntu/Emscripten +# Docker flow where thirdparty is built at /home/MeshLib and then COPYed to +# /usr/local/lib/meshlib-thirdparty-lib/. Plain find_library / find_path uses +# the search paths already set up by MeshLib's root CMakeLists (for the +# submodule build) and the vcpkg toolchain (on Windows); same binary either +# way. +find_library(LIBDEFLATE_LIBRARY + NAMES deflate libdeflate + REQUIRED ) -target_link_libraries(${PROJECT_NAME} PRIVATE libdeflate::libdeflate_shared) +find_path(LIBDEFLATE_INCLUDE_DIR + NAMES libdeflate.h + REQUIRED +) +target_link_libraries(${PROJECT_NAME} PRIVATE ${LIBDEFLATE_LIBRARY}) +target_include_directories(${PROJECT_NAME} PRIVATE ${LIBDEFLATE_INCLUDE_DIR}) # TODO: CMake config target_include_directories(${PROJECT_NAME}