MRZlib: implement zlibCompressStream via libdeflate

.gitmodules

@@ -85,3 +85,6 @@
 [submodule "thirdparty/cpp-httplib"]
 	path = thirdparty/cpp-httplib
 	url = https://github.com/yhirose/cpp-httplib
+[submodule "thirdparty/libdeflate"]
+	path = thirdparty/libdeflate
+	url = https://github.com/ebiggers/libdeflate.git

requirements/windows.txt

@@ -19,6 +19,7 @@ gdcm
 gtest
 hidapi
 jsoncpp
+libdeflate
 libe57format
 libharu
 libzip

source/MRMesh/CMakeLists.txt

@@ -65,6 +65,27 @@ ELSE()
 ENDIF()
 target_link_libraries(${PROJECT_NAME} PRIVATE libzip::zip)
 
+# libdeflate drives MRZlib's zlibCompressStream (compress path only;
+# decompression and the compressZip/libzip pipeline remain on zlib). We avoid
+# libdeflate's own CMake config package here: the config it installs bakes
+# the build-time absolute include path via $<INSTALL_INTERFACE:${CMAKE_INSTALL_FULL_INCLUDEDIR}>
+# rather than a relocatable relative path, which breaks the Ubuntu/Emscripten
+# Docker flow where thirdparty is built at /home/MeshLib and then COPYed to
+# /usr/local/lib/meshlib-thirdparty-lib/. Plain find_library / find_path uses
+# the search paths already set up by MeshLib's root CMakeLists (for the
+# submodule build) and the vcpkg toolchain (on Windows); same binary either
+# way.
+find_library(LIBDEFLATE_LIBRARY
+  NAMES deflate libdeflate
+  REQUIRED
+)
+find_path(LIBDEFLATE_INCLUDE_DIR
+  NAMES libdeflate.h
+  REQUIRED
+)
+target_link_libraries(${PROJECT_NAME} PRIVATE ${LIBDEFLATE_LIBRARY})
+target_include_directories(${PROJECT_NAME} PRIVATE ${LIBDEFLATE_INCLUDE_DIR})
+
 # TODO: CMake config
 target_include_directories(${PROJECT_NAME}
   PUBLIC

source/MRMesh/MRZlib.cpp

@@ -2,9 +2,12 @@
 #include "MRBuffer.h"
 #include "MRFinally.h"
 
+#include <libdeflate.h>
 #include <zlib.h>
 
 #include <cassert>
+#include <cstdint>
+#include <vector>
 
 namespace
 {
@@ -61,58 +64,69 @@ namespace MR
 
 Expected<void> zlibCompressStream( std::istream& in, std::ostream& out, const ZlibCompressParams& params )
 {
-    Buffer<char> inChunk( cChunkSize ), outChunk( cChunkSize );
-    z_stream stream {
-        .zalloc = Z_NULL,
-        .zfree = Z_NULL,
-        .opaque = Z_NULL,
-    };
-    int ret;
-    if ( Z_OK != ( ret = deflateInit2( &stream, params.level, Z_DEFLATED, windowBitsFor( params.rawDeflate ), kDefaultMemLevel, Z_DEFAULT_STRATEGY ) ) )
-        return unexpected( zlibToString( ret ) );
-
-    MR_FINALLY {
-        deflateEnd( &stream );
-    };
-
-    if ( params.stats )
-        *params.stats = {};
-
+    // libdeflate exposes only a whole-buffer compression API, so drain the
+    // input stream into memory first. Memory ceiling = input size; callers
+    // that need streaming compression of very large inputs should chunk at
+    // a layer above this function (only this overload takes the fast path;
+    // the other direction still streams in zlibDecompressStream below).
+    std::vector<uint8_t> inBuf;
     while ( !in.eof() )
     {
-        in.read( inChunk.data(), inChunk.size() );
+        const size_t offset = inBuf.size();
+        inBuf.resize( offset + cChunkSize );
+        in.read( reinterpret_cast<char*>( inBuf.data() + offset ), cChunkSize );
         if ( in.bad() )
             return unexpected( "I/O error" );
-        stream.next_in = reinterpret_cast<uint8_t*>( inChunk.data() );
-        stream.avail_in = (unsigned)in.gcount();
-        assert( stream.avail_in <= (unsigned)inChunk.size() );
+        inBuf.resize( offset + static_cast<size_t>( in.gcount() ) );
+    }
 
-        if ( params.stats )
-        {
-            params.stats->crc32 = (uint32_t)crc32( params.stats->crc32, stream.next_in, stream.avail_in );
-            params.stats->uncompressedSize += stream.avail_in;
-        }
+    // libdeflate accepts levels 1-12. Map the zlib-style conventions we
+    // inherit from ZlibCompressParams: -1 (default) -> libdeflate 6, which
+    // matches zlib's default; 0 (zlib's "store-only") -> libdeflate 1
+    // because libdeflate has no stored-only mode. The ±4-byte tolerance on
+    // ZlibCompressStats's size check absorbs the resulting minor differences
+    // versus the stock-zlib reference blobs.
+    int level = params.level;
+    if ( level < 0 )
+        level = 6;
+    else if ( level == 0 )
+        level = 1;
+    else if ( level > 12 )
+        level = 12;
 
-        const auto flush = in.eof() ? Z_FINISH : Z_NO_FLUSH;
-        do
-        {
-            stream.next_out = reinterpret_cast<uint8_t*>( outChunk.data() );
-            stream.avail_out = (unsigned)outChunk.size();
-            ret = deflate( &stream, flush );
-            if ( Z_OK != ret && Z_STREAM_END != ret )
-                return unexpected( zlibToString( ret ) );
-
-            assert( stream.avail_out <= (unsigned)outChunk.size() );
-            const unsigned written = (unsigned)outChunk.size() - stream.avail_out;
-            out.write( outChunk.data(), written );
-            if ( out.bad() )
-                return unexpected( "I/O error" );
-            if ( params.stats )
-                params.stats->compressedSize += written;
-        }
-        while ( stream.avail_out == 0 );
+    if ( params.stats )
+    {
+        params.stats->crc32 = libdeflate_crc32( 0, inBuf.data(), inBuf.size() );
+        params.stats->uncompressedSize = inBuf.size();
+        params.stats->compressedSize = 0;
     }
 
+    libdeflate_compressor* comp = libdeflate_alloc_compressor( level );
+    if ( !comp )
+        return unexpected( "libdeflate_alloc_compressor failed" );
+    MR_FINALLY {
+        libdeflate_free_compressor( comp );
+    };
+
+    const bool raw = params.rawDeflate;
+    const size_t bound = raw
+        ? libdeflate_deflate_compress_bound( comp, inBuf.size() )
+        : libdeflate_zlib_compress_bound( comp, inBuf.size() );
+    std::vector<uint8_t> outBuf( bound );
+
+    const size_t produced = raw
+        ? libdeflate_deflate_compress( comp, inBuf.data(), inBuf.size(), outBuf.data(), outBuf.size() )
+        : libdeflate_zlib_compress( comp, inBuf.data(), inBuf.size(), outBuf.data(), outBuf.size() );
+    if ( produced == 0 )
+        return unexpected( "libdeflate compression failed" );
+
+    out.write( reinterpret_cast<const char*>( outBuf.data() ), static_cast<std::streamsize>( produced ) );
+    if ( out.bad() )
+        return unexpected( "I/O error" );
+
+    if ( params.stats )
+        params.stats->compressedSize = produced;
+
     return {};
 }
 

source/MRTest/MRZipCompressTests.cpp

@@ -28,7 +28,7 @@ TEST( MRMesh, CompressSphereToZip )
     UniqueTemporaryFolder srcFolder;
     ASSERT_TRUE( bool( srcFolder ) );
 
-    constexpr int targetVerts = 1000; // increase it to make the file being compressed larger, 100'000 vertices -> 12M bytes
+    constexpr int targetVerts = 100000; // increase it to make the file being compressed larger, 100'000 vertices -> 12M bytes
     SphereParams params;
     params.radius = 1.0f;
     params.numMeshVertices = targetVerts;
@@ -82,9 +82,9 @@ TEST( MRMesh, CompressManySmallFilesToZip )
     ASSERT_TRUE( bool( srcFolder ) );
 
     // increase both below numbers to make the files being compressed larger, 200 * 2 files * 60'000 bytes -> 24M bytes
-    constexpr int numBinaryFiles = 20;
+    constexpr int numBinaryFiles = 200;
     constexpr int numJsonFiles = numBinaryFiles;
-    constexpr size_t bytesPerFile = 6000;
+    constexpr size_t bytesPerFile = 60000;
 
     // Simple LCG used to produce deterministic pseudo-random bytes.
     // Keeps the test reproducible across runs and platforms while avoiding

thirdparty/CMakeLists.txt

@@ -85,6 +85,17 @@ ENDIF()
 
 add_subdirectory(./OpenCTM-git ./OpenCTM)
 
+# Build libdeflate from submodule on every platform except Windows; Windows
+# picks it up from vcpkg (see requirements/windows.txt) because the MSBuild
+# matrix legs don't run the thirdparty CMake build. MRZlib's compress path
+# uses it on all platforms regardless of where the binary comes from.
+IF(NOT WIN32)
+  set(LIBDEFLATE_BUILD_GZIP OFF CACHE BOOL "")
+  set(LIBDEFLATE_BUILD_TESTS OFF CACHE BOOL "")
+  set(LIBDEFLATE_BUILD_STATIC_LIB OFF CACHE BOOL "")
+  add_subdirectory(./libdeflate)
+ENDIF()
+
 option(PHMAP_INSTALL "" ON)
 add_subdirectory(./parallel-hashmap)