MRZlib: route compress and decompress streams through zlib-ng (native mode)

.gitmodules

@@ -85,3 +85,6 @@
 [submodule "thirdparty/cpp-httplib"]
 	path = thirdparty/cpp-httplib
 	url = https://github.com/yhirose/cpp-httplib
+[submodule "thirdparty/zlib-ng"]
+	path = thirdparty/zlib-ng
+	url = https://github.com/zlib-ng/zlib-ng.git

requirements/macos.txt

@@ -24,3 +24,4 @@ tbb
 tinyxml2
 tl-expected
 zlib
+zlib-ng

requirements/vcpkg-linux.txt

@@ -37,3 +37,4 @@ tiff
 tinygltf
 tinyxml2
 tl-expected
+zlib-ng

requirements/windows.txt

@@ -30,5 +30,6 @@ tiff
 tinygltf
 tinyxml2
 tl-expected
+zlib-ng
 glad
 glfw3

source/MRMesh/CMakeLists.txt

@@ -65,6 +65,80 @@ ELSE()
 ENDIF()
 target_link_libraries(${PROJECT_NAME} PRIVATE libzip::zip)
 
+# zlib-ng (native mode, zng_ prefix) powers MRZlib's zlibCompressStream;
+# decompression and the compressZip/libzip pipeline stay on stock zlib.
+#
+# Windows: use the proper CMake config package that vcpkg's zlib-ng port
+# ships. Multi-config Debug/Release matters here: find_library() picks one
+# .lib and uses it for every configuration, so a Debug build links against
+# the release zlib-ng2.dll and the applocal-copy step doesn't copy it next
+# to the .exe (STATUS_DLL_NOT_FOUND at launch, seen on msvc-2019 Debug
+# CMake with the x64-windows-meshlib-iterator-debug triplet). The
+# find_package(CONFIG) target has IMPORTED_CONFIGURATIONS=DEBUG;RELEASE
+# and drives applocal-copy correctly.
+#
+# Other platforms: find_library + find_path. On Ubuntu apt and Emscripten
+# we build zlib-ng from thirdparty/zlib-ng and its generated config has
+# the same non-relocatable-include-path bug we hit with libdeflate (breaks
+# under the Docker multi-stage COPY-shuffle). Plain library/path lookup
+# sidesteps it, and those platforms are single-config so the multi-config
+# concern above doesn't apply.
+IF(WIN32)
+  # Can't use find_package(zlib-ng CONFIG): the vcpkg zlib-ng 2.1.5 port
+  # that vcpkg 2024.10.21 (pinned for msvc-2019 legs) ships doesn't install
+  # a CMake config file. Can't use a plain find_library() either: it loses
+  # Debug/Release selectivity, linking Debug MRMesh.dll against release
+  # zlib-ng2.dll and breaking applocal-copy (see the diagnostic output on
+  # the iterator-debug triplet, STATUS_DLL_NOT_FOUND at MeshViewer.exe).
+  #
+  # Manually construct an imported target with per-configuration IMPLIB +
+  # LOCATION paths, locating the files directly under vcpkg's triplet
+  # install tree. Gives applocal-copy the metadata it needs to copy the
+  # correct DLL next to the .exe per configuration.
+  find_path(LIBZLIBNG_INCLUDE_DIR NAMES zlib-ng.h REQUIRED)
+  find_library(LIBZLIBNG_LIBRARY_RELEASE
+    NAMES zlib-ng
+    PATHS "${VCPKG_INSTALLED_DIR}/${VCPKG_TARGET_TRIPLET}/lib"
+    NO_DEFAULT_PATH REQUIRED
+  )
+  find_library(LIBZLIBNG_LIBRARY_DEBUG
+    NAMES zlib-ngd zlib-ng
+    PATHS "${VCPKG_INSTALLED_DIR}/${VCPKG_TARGET_TRIPLET}/debug/lib"
+    NO_DEFAULT_PATH REQUIRED
+  )
+  find_program(LIBZLIBNG_DLL_RELEASE
+    NAMES zlib-ng2.dll zlib-ng.dll
+    PATHS "${VCPKG_INSTALLED_DIR}/${VCPKG_TARGET_TRIPLET}/bin"
+    NO_DEFAULT_PATH REQUIRED
+  )
+  find_program(LIBZLIBNG_DLL_DEBUG
+    NAMES zlib-ngd2.dll zlib-ng2.dll zlib-ng.dll
+    PATHS "${VCPKG_INSTALLED_DIR}/${VCPKG_TARGET_TRIPLET}/debug/bin"
+    NO_DEFAULT_PATH REQUIRED
+  )
+  add_library(zlib-ng::zlib SHARED IMPORTED)
+  set_target_properties(zlib-ng::zlib PROPERTIES
+    IMPORTED_CONFIGURATIONS "DEBUG;RELEASE"
+    IMPORTED_IMPLIB_RELEASE  "${LIBZLIBNG_LIBRARY_RELEASE}"
+    IMPORTED_LOCATION_RELEASE "${LIBZLIBNG_DLL_RELEASE}"
+    IMPORTED_IMPLIB_DEBUG    "${LIBZLIBNG_LIBRARY_DEBUG}"
+    IMPORTED_LOCATION_DEBUG  "${LIBZLIBNG_DLL_DEBUG}"
+    INTERFACE_INCLUDE_DIRECTORIES "${LIBZLIBNG_INCLUDE_DIR}"
+  )
+  target_link_libraries(${PROJECT_NAME} PRIVATE zlib-ng::zlib)
+ELSE()
+  find_library(LIBZLIBNG_LIBRARY
+    NAMES z-ng zlib-ng
+    REQUIRED
+  )
+  find_path(LIBZLIBNG_INCLUDE_DIR
+    NAMES zlib-ng.h
+    REQUIRED
+  )
+  target_link_libraries(${PROJECT_NAME} PRIVATE ${LIBZLIBNG_LIBRARY})
+  target_include_directories(${PROJECT_NAME} PRIVATE ${LIBZLIBNG_INCLUDE_DIR})
+ENDIF()
+
 # TODO: CMake config
 target_include_directories(${PROJECT_NAME}
   PUBLIC

source/MRMesh/MRZlib.cpp

@@ -2,50 +2,47 @@
 #include "MRBuffer.h"
 #include "MRFinally.h"
 
-#include <zlib.h>
+// zlib-ng in native mode: the zng_ prefix on every symbol keeps it ABI-
+// distinct from stock zlib (which libzip still links as before), and
+// <zlib-ng.h> re-exports the MAX_WBITS / Z_* constants we need under the
+// same spelling as <zlib.h>, so this TU doesn't include stock zlib's
+// header at all.
+#include <zlib-ng.h>
 
 #include <cassert>
+#include <cstdint>
 
 namespace
 {
 
 constexpr size_t cChunkSize = 256 * 1024; // 256 KiB
 
-// zlib's `windowBits` argument is sign-encoded: positive = zlib wrapper (RFC 1950);
-// negative = raw deflate (RFC 1951, no wrapper). Magnitude is log2(window size);
-// MAX_WBITS = 15 gives a 32 KiB window.
+// windowBits is sign-encoded the same way zlib documents it: positive =
+// zlib wrapper (RFC 1950), negative = raw deflate (RFC 1951, no wrapper).
+// Magnitude is log2(window size); MAX_WBITS = 15 gives a 32 KiB window.
+// zlib-ng re-exports the same MAX_WBITS macro through zconf-ng.h, so we
+// can use it here without pulling in stock zlib's headers.
 constexpr int kZlibWrapperBits = MAX_WBITS;
 constexpr int kRawDeflateBits  = -MAX_WBITS;
 
-// memLevel controls zlib's internal state size. 8 is zlib's default (its internal
-// DEF_MEM_LEVEL in deflate.c is not exported; redeclared here).
+// memLevel controls the compressor's internal state size. 8 is zlib's and
+// zlib-ng's shared default (matches the old MRZlib choice).
 constexpr int kDefaultMemLevel = 8;
-static_assert( kDefaultMemLevel <= MAX_MEM_LEVEL );
 
-std::string zlibToString( int code )
+std::string zngToString( int code )
 {
     switch ( code )
     {
-        case Z_OK:
-            return "ok";
-        case Z_STREAM_END:
-            return "stream end";
-        case Z_NEED_DICT:
-            return "need dict";
-        case Z_ERRNO:
-            return "errno";
-        case Z_STREAM_ERROR:
-            return "stream error";
-        case Z_DATA_ERROR:
-            return "data error";
-        case Z_MEM_ERROR:
-            return "mem error";
-        case Z_BUF_ERROR:
-            return "buf error";
-        case Z_VERSION_ERROR:
-            return "version error";
-        default:
-            return "unknown code";
+        case Z_OK:             return "ok";
+        case Z_STREAM_END:     return "stream end";
+        case Z_NEED_DICT:      return "need dict";
+        case Z_ERRNO:          return "errno";
+        case Z_STREAM_ERROR:   return "stream error";
+        case Z_DATA_ERROR:     return "data error";
+        case Z_MEM_ERROR:      return "mem error";
+        case Z_BUF_ERROR:      return "buf error";
+        case Z_VERSION_ERROR:  return "version error";
+        default:               return "unknown code";
     }
 }
 
@@ -62,48 +59,47 @@ namespace MR
 Expected<void> zlibCompressStream( std::istream& in, std::ostream& out, const ZlibCompressParams& params )
 {
     Buffer<char> inChunk( cChunkSize ), outChunk( cChunkSize );
-    z_stream stream {
-        .zalloc = Z_NULL,
-        .zfree = Z_NULL,
-        .opaque = Z_NULL,
-    };
+    zng_stream stream{};
     int ret;
-    if ( Z_OK != ( ret = deflateInit2( &stream, params.level, Z_DEFLATED, windowBitsFor( params.rawDeflate ), kDefaultMemLevel, Z_DEFAULT_STRATEGY ) ) )
-        return unexpected( zlibToString( ret ) );
+    if ( Z_OK != ( ret = zng_deflateInit2( &stream, params.level, Z_DEFLATED,
+                                           windowBitsFor( params.rawDeflate ),
+                                           kDefaultMemLevel, Z_DEFAULT_STRATEGY ) ) )
+        return unexpected( zngToString( ret ) );
 
     MR_FINALLY {
-        deflateEnd( &stream );
+        zng_deflateEnd( &stream );
     };
 
     if ( params.stats )
         *params.stats = {};
 
     while ( !in.eof() )
     {
-        in.read( inChunk.data(), inChunk.size() );
+        in.read( inChunk.data(), static_cast<std::streamsize>( inChunk.size() ) );
         if ( in.bad() )
             return unexpected( "I/O error" );
         stream.next_in = reinterpret_cast<uint8_t*>( inChunk.data() );
-        stream.avail_in = (unsigned)in.gcount();
-        assert( stream.avail_in <= (unsigned)inChunk.size() );
+        stream.avail_in = static_cast<unsigned>( in.gcount() );
+        assert( stream.avail_in <= static_cast<unsigned>( inChunk.size() ) );
 
         if ( params.stats )
         {
-            params.stats->crc32 = (uint32_t)crc32( params.stats->crc32, stream.next_in, stream.avail_in );
+            params.stats->crc32 = static_cast<uint32_t>(
+                zng_crc32( params.stats->crc32, stream.next_in, stream.avail_in ) );
             params.stats->uncompressedSize += stream.avail_in;
         }
 
-        const auto flush = in.eof() ? Z_FINISH : Z_NO_FLUSH;
+        const int flush = in.eof() ? Z_FINISH : Z_NO_FLUSH;
         do
         {
             stream.next_out = reinterpret_cast<uint8_t*>( outChunk.data() );
-            stream.avail_out = (unsigned)outChunk.size();
-            ret = deflate( &stream, flush );
+            stream.avail_out = static_cast<unsigned>( outChunk.size() );
+            ret = zng_deflate( &stream, flush );
             if ( Z_OK != ret && Z_STREAM_END != ret )
-                return unexpected( zlibToString( ret ) );
+                return unexpected( zngToString( ret ) );
 
-            assert( stream.avail_out <= (unsigned)outChunk.size() );
-            const unsigned written = (unsigned)outChunk.size() - stream.avail_out;
+            assert( stream.avail_out <= static_cast<unsigned>( outChunk.size() ) );
+            const unsigned written = static_cast<unsigned>( outChunk.size() ) - stream.avail_out;
             out.write( outChunk.data(), written );
             if ( out.bad() )
                 return unexpected( "I/O error" );
@@ -124,38 +120,34 @@ Expected<void> zlibCompressStream( std::istream& in, std::ostream& out, int leve
 Expected<void> zlibDecompressStream( std::istream& in, std::ostream& out, const ZlibParams& params )
 {
     Buffer<char> inChunk( cChunkSize ), outChunk( cChunkSize );
-    z_stream stream {
-        .zalloc = Z_NULL,
-        .zfree = Z_NULL,
-        .opaque = Z_NULL,
-    };
+    zng_stream stream{};
     int ret;
-    if ( Z_OK != ( ret = inflateInit2( &stream, windowBitsFor( params.rawDeflate ) ) ) )
-        return unexpected( zlibToString( ret ) );
+    if ( Z_OK != ( ret = zng_inflateInit2( &stream, windowBitsFor( params.rawDeflate ) ) ) )
+        return unexpected( zngToString( ret ) );
 
     MR_FINALLY {
-        inflateEnd( &stream );
+        zng_inflateEnd( &stream );
     };
 
     while ( !in.eof() )
     {
-        in.read( inChunk.data(), inChunk.size() );
+        in.read( inChunk.data(), static_cast<std::streamsize>( inChunk.size() ) );
         if ( in.bad() )
             return unexpected( "I/O error" );
         stream.next_in = reinterpret_cast<uint8_t*>( inChunk.data() );
-        stream.avail_in = (unsigned)in.gcount();
-        assert( stream.avail_in <= (unsigned)inChunk.size() );
+        stream.avail_in = static_cast<unsigned>( in.gcount() );
+        assert( stream.avail_in <= static_cast<unsigned>( inChunk.size() ) );
 
         do
         {
             stream.next_out = reinterpret_cast<uint8_t*>( outChunk.data() );
-            stream.avail_out = (unsigned)outChunk.size();
-            ret = inflate( &stream, Z_NO_FLUSH );
+            stream.avail_out = static_cast<unsigned>( outChunk.size() );
+            ret = zng_inflate( &stream, Z_NO_FLUSH );
             if ( Z_OK != ret && Z_STREAM_END != ret )
-                return unexpected( zlibToString( ret ) );
+                return unexpected( zngToString( ret ) );
 
-            assert( stream.avail_out <= (unsigned)outChunk.size() );
-            out.write( outChunk.data(), (unsigned)outChunk.size() - stream.avail_out );
+            assert( stream.avail_out <= static_cast<unsigned>( outChunk.size() ) );
+            out.write( outChunk.data(), static_cast<unsigned>( outChunk.size() ) - stream.avail_out );
             if ( out.bad() )
                 return unexpected( "I/O error" );
 

thirdparty/CMakeLists.txt

@@ -85,6 +85,40 @@ ENDIF()
 
 add_subdirectory(./OpenCTM-git ./OpenCTM)
 
+# Build zlib-ng from our submodule (native mode -- zng_ prefix, does not shadow
+# stock zlib) on Ubuntu apt and Emscripten, which have no zlib-ng package.
+# Windows + macOS + any MESHLIB_USE_VCPKG config pick up the package-manager
+# copy instead (see requirements/{windows,macos,vcpkg-linux}.txt).
+IF(NOT WIN32 AND NOT APPLE AND NOT MESHLIB_USE_VCPKG)
+  set(ZLIB_COMPAT       OFF CACHE BOOL "")
+  set(ZLIB_ENABLE_TESTS OFF CACHE BOOL "")
+  set(ZLIBNG_ENABLE_TESTS OFF CACHE BOOL "")
+  set(WITH_GTEST        OFF CACHE BOOL "")
+  set(WITH_GZFILEOP     OFF CACHE BOOL "")
+  IF(EMSCRIPTEN)
+    # zlib-ng's CMakeLists treats an undefined BUILD_SHARED_LIBS as "build both
+    # shared and static targets". Emscripten then demotes the SHARED one to
+    # STATIC, after which both rules emit the same libz-ng.a and ninja errors
+    # with "multiple rules generate libz-ng.a [-w dupbuild=err]". Force static
+    # only for this add_subdirectory -- Emscripten produces static output
+    # either way.
+    #
+    # After add_subdirectory(./zlib-ng) we MUST restore BUILD_SHARED_LIBS to
+    # its prior *undefined* state, not "defined but empty". jsoncpp further
+    # down expands ${BUILD_SHARED_LIBS} into set_target_properties() pairs and
+    # silently breaks with "set_target_properties called with incorrect number
+    # of arguments" when the expansion yields empty. jsoncpp's own option()
+    # fallback only fires for undefined, not empty. unset() is therefore the
+    # right tool here (BUILD_SHARED_LIBS was undefined before this block on
+    # every Emscripten MeshLib build).
+    set(BUILD_SHARED_LIBS OFF)
+    add_subdirectory(./zlib-ng)
+    unset(BUILD_SHARED_LIBS)
+  ELSE()
+    add_subdirectory(./zlib-ng)
+  ENDIF()
+ENDIF()
+
 option(PHMAP_INSTALL "" ON)
 add_subdirectory(./parallel-hashmap)