MRZlib: route compress and decompress streams through zlib-ng (native mode)

.gitmodules

@@ -85,3 +85,6 @@
 [submodule "thirdparty/cpp-httplib"]
 	path = thirdparty/cpp-httplib
 	url = https://github.com/yhirose/cpp-httplib
+[submodule "thirdparty/zlib-ng"]
+	path = thirdparty/zlib-ng
+	url = https://github.com/zlib-ng/zlib-ng.git

requirements/macos.txt

@@ -24,3 +24,4 @@ tbb
 tinyxml2
 tl-expected
 zlib
+zlib-ng

requirements/vcpkg-linux.txt

@@ -37,3 +37,4 @@ tiff
 tinygltf
 tinyxml2
 tl-expected
+zlib-ng

requirements/windows.txt

@@ -30,5 +30,6 @@ tiff
 tinygltf
 tinyxml2
 tl-expected
+zlib-ng
 glad
 glfw3

source/MRMesh/CMakeLists.txt

@@ -65,6 +65,15 @@ ELSE()
 ENDIF()
 target_link_libraries(${PROJECT_NAME} PRIVATE libzip::zip)
 
+# zlib-ng (native mode, zng_ prefix) powers MRZlib's compress/decompress streams.
+# Sourced via find_package on vcpkg/Homebrew (where the port installs a CMake
+# config), or via add_subdirectory + ALIAS in thirdparty/CMakeLists.txt on
+# Ubuntu apt and Emscripten.
+IF(NOT TARGET zlib-ng::zlib)
+  find_package(zlib-ng CONFIG REQUIRED)
+ENDIF()
+target_link_libraries(${PROJECT_NAME} PRIVATE zlib-ng::zlib)
+
 # TODO: CMake config
 target_include_directories(${PROJECT_NAME}
   PUBLIC

source/MRMesh/MRZlib.cpp

@@ -2,50 +2,47 @@
 #include "MRBuffer.h"
 #include "MRFinally.h"
 
-#include <zlib.h>
+// zlib-ng in native mode: the zng_ prefix on every symbol keeps it ABI-
+// distinct from stock zlib (which libzip still links as before), and
+// <zlib-ng.h> re-exports the MAX_WBITS / Z_* constants we need under the
+// same spelling as <zlib.h>, so this TU doesn't include stock zlib's
+// header at all.
+#include <zlib-ng.h>
 
 #include <cassert>
+#include <cstdint>
 
 namespace
 {
 
 constexpr size_t cChunkSize = 256 * 1024; // 256 KiB
 
-// zlib's `windowBits` argument is sign-encoded: positive = zlib wrapper (RFC 1950);
-// negative = raw deflate (RFC 1951, no wrapper). Magnitude is log2(window size);
-// MAX_WBITS = 15 gives a 32 KiB window.
+// windowBits is sign-encoded the same way zlib documents it: positive =
+// zlib wrapper (RFC 1950), negative = raw deflate (RFC 1951, no wrapper).
+// Magnitude is log2(window size); MAX_WBITS = 15 gives a 32 KiB window.
+// zlib-ng re-exports the same MAX_WBITS macro through zconf-ng.h, so we
+// can use it here without pulling in stock zlib's headers.
 constexpr int kZlibWrapperBits = MAX_WBITS;
 constexpr int kRawDeflateBits  = -MAX_WBITS;
 
-// memLevel controls zlib's internal state size. 8 is zlib's default (its internal
-// DEF_MEM_LEVEL in deflate.c is not exported; redeclared here).
+// memLevel controls the compressor's internal state size. 8 is zlib's and
+// zlib-ng's shared default (matches the old MRZlib choice).
 constexpr int kDefaultMemLevel = 8;
-static_assert( kDefaultMemLevel <= MAX_MEM_LEVEL );
 
-std::string zlibToString( int code )
+std::string zngToString( int code )
 {
     switch ( code )
     {
-        case Z_OK:
-            return "ok";
-        case Z_STREAM_END:
-            return "stream end";
-        case Z_NEED_DICT:
-            return "need dict";
-        case Z_ERRNO:
-            return "errno";
-        case Z_STREAM_ERROR:
-            return "stream error";
-        case Z_DATA_ERROR:
-            return "data error";
-        case Z_MEM_ERROR:
-            return "mem error";
-        case Z_BUF_ERROR:
-            return "buf error";
-        case Z_VERSION_ERROR:
-            return "version error";
-        default:
-            return "unknown code";
+        case Z_OK:             return "ok";
+        case Z_STREAM_END:     return "stream end";
+        case Z_NEED_DICT:      return "need dict";
+        case Z_ERRNO:          return "errno";
+        case Z_STREAM_ERROR:   return "stream error";
+        case Z_DATA_ERROR:     return "data error";
+        case Z_MEM_ERROR:      return "mem error";
+        case Z_BUF_ERROR:      return "buf error";
+        case Z_VERSION_ERROR:  return "version error";
+        default:               return "unknown code";
     }
 }
 
@@ -62,48 +59,47 @@ namespace MR
 Expected<void> zlibCompressStream( std::istream& in, std::ostream& out, const ZlibCompressParams& params )
 {
     Buffer<char> inChunk( cChunkSize ), outChunk( cChunkSize );
-    z_stream stream {
-        .zalloc = Z_NULL,
-        .zfree = Z_NULL,
-        .opaque = Z_NULL,
-    };
+    zng_stream stream{};
     int ret;
-    if ( Z_OK != ( ret = deflateInit2( &stream, params.level, Z_DEFLATED, windowBitsFor( params.rawDeflate ), kDefaultMemLevel, Z_DEFAULT_STRATEGY ) ) )
-        return unexpected( zlibToString( ret ) );
+    if ( Z_OK != ( ret = zng_deflateInit2( &stream, params.level, Z_DEFLATED,
+                                           windowBitsFor( params.rawDeflate ),
+                                           kDefaultMemLevel, Z_DEFAULT_STRATEGY ) ) )
+        return unexpected( zngToString( ret ) );
 
     MR_FINALLY {
-        deflateEnd( &stream );
+        zng_deflateEnd( &stream );
     };
 
     if ( params.stats )
         *params.stats = {};
 
     while ( !in.eof() )
     {
-        in.read( inChunk.data(), inChunk.size() );
+        in.read( inChunk.data(), static_cast<std::streamsize>( inChunk.size() ) );
         if ( in.bad() )
             return unexpected( "I/O error" );
         stream.next_in = reinterpret_cast<uint8_t*>( inChunk.data() );
-        stream.avail_in = (unsigned)in.gcount();
-        assert( stream.avail_in <= (unsigned)inChunk.size() );
+        stream.avail_in = static_cast<unsigned>( in.gcount() );
+        assert( stream.avail_in <= static_cast<unsigned>( inChunk.size() ) );
 
         if ( params.stats )
         {
-            params.stats->crc32 = (uint32_t)crc32( params.stats->crc32, stream.next_in, stream.avail_in );
+            params.stats->crc32 = static_cast<uint32_t>(
+                zng_crc32( params.stats->crc32, stream.next_in, stream.avail_in ) );
             params.stats->uncompressedSize += stream.avail_in;
         }
 
-        const auto flush = in.eof() ? Z_FINISH : Z_NO_FLUSH;
+        const int flush = in.eof() ? Z_FINISH : Z_NO_FLUSH;
         do
         {
             stream.next_out = reinterpret_cast<uint8_t*>( outChunk.data() );
-            stream.avail_out = (unsigned)outChunk.size();
-            ret = deflate( &stream, flush );
+            stream.avail_out = static_cast<unsigned>( outChunk.size() );
+            ret = zng_deflate( &stream, flush );
             if ( Z_OK != ret && Z_STREAM_END != ret )
-                return unexpected( zlibToString( ret ) );
+                return unexpected( zngToString( ret ) );
 
-            assert( stream.avail_out <= (unsigned)outChunk.size() );
-            const unsigned written = (unsigned)outChunk.size() - stream.avail_out;
+            assert( stream.avail_out <= static_cast<unsigned>( outChunk.size() ) );
+            const unsigned written = static_cast<unsigned>( outChunk.size() ) - stream.avail_out;
             out.write( outChunk.data(), written );
             if ( out.bad() )
                 return unexpected( "I/O error" );
@@ -124,38 +120,34 @@ Expected<void> zlibCompressStream( std::istream& in, std::ostream& out, int leve
 Expected<void> zlibDecompressStream( std::istream& in, std::ostream& out, const ZlibParams& params )
 {
     Buffer<char> inChunk( cChunkSize ), outChunk( cChunkSize );
-    z_stream stream {
-        .zalloc = Z_NULL,
-        .zfree = Z_NULL,
-        .opaque = Z_NULL,
-    };
+    zng_stream stream{};
     int ret;
-    if ( Z_OK != ( ret = inflateInit2( &stream, windowBitsFor( params.rawDeflate ) ) ) )
-        return unexpected( zlibToString( ret ) );
+    if ( Z_OK != ( ret = zng_inflateInit2( &stream, windowBitsFor( params.rawDeflate ) ) ) )
+        return unexpected( zngToString( ret ) );
 
     MR_FINALLY {
-        inflateEnd( &stream );
+        zng_inflateEnd( &stream );
     };
 
     while ( !in.eof() )
     {
-        in.read( inChunk.data(), inChunk.size() );
+        in.read( inChunk.data(), static_cast<std::streamsize>( inChunk.size() ) );
         if ( in.bad() )
             return unexpected( "I/O error" );
         stream.next_in = reinterpret_cast<uint8_t*>( inChunk.data() );
-        stream.avail_in = (unsigned)in.gcount();
-        assert( stream.avail_in <= (unsigned)inChunk.size() );
+        stream.avail_in = static_cast<unsigned>( in.gcount() );
+        assert( stream.avail_in <= static_cast<unsigned>( inChunk.size() ) );
 
         do
         {
             stream.next_out = reinterpret_cast<uint8_t*>( outChunk.data() );
-            stream.avail_out = (unsigned)outChunk.size();
-            ret = inflate( &stream, Z_NO_FLUSH );
+            stream.avail_out = static_cast<unsigned>( outChunk.size() );
+            ret = zng_inflate( &stream, Z_NO_FLUSH );
             if ( Z_OK != ret && Z_STREAM_END != ret )
-                return unexpected( zlibToString( ret ) );
+                return unexpected( zngToString( ret ) );
 
-            assert( stream.avail_out <= (unsigned)outChunk.size() );
-            out.write( outChunk.data(), (unsigned)outChunk.size() - stream.avail_out );
+            assert( stream.avail_out <= static_cast<unsigned>( outChunk.size() ) );
+            out.write( outChunk.data(), static_cast<unsigned>( outChunk.size() ) - stream.avail_out );
             if ( out.bad() )
                 return unexpected( "I/O error" );
 

thirdparty/CMakeLists.txt

@@ -85,6 +85,28 @@ ENDIF()
 
 add_subdirectory(./OpenCTM-git ./OpenCTM)
 
+# zlib-ng from our submodule on Ubuntu apt + Emscripten (no package available);
+# Windows/macOS/vcpkg pick it up via requirements/*.txt and find_package.
+IF(NOT WIN32 AND NOT APPLE AND NOT MESHLIB_USE_VCPKG)
+  set(ZLIB_COMPAT       OFF CACHE BOOL "")
+  set(ZLIB_ENABLE_TESTS OFF CACHE BOOL "")
+  set(ZLIBNG_ENABLE_TESTS OFF CACHE BOOL "")
+  set(WITH_GTEST        OFF CACHE BOOL "")
+  set(WITH_GZFILEOP     OFF CACHE BOOL "")
+  IF(EMSCRIPTEN)
+    # Force static: undefined BUILD_SHARED_LIBS makes zlib-ng emit duplicate
+    # libz-ng.a rules under Emscripten. Restore to undefined (not empty)
+    # afterwards -- jsoncpp downstream breaks on empty BUILD_SHARED_LIBS.
+    set(BUILD_SHARED_LIBS OFF)
+    add_subdirectory(./zlib-ng)
+    unset(BUILD_SHARED_LIBS)
+  ELSE()
+    add_subdirectory(./zlib-ng)
+  ENDIF()
+  # zlib-ng exports zlib-ng::zlib only at install time; alias for in-tree use.
+  add_library(zlib-ng::zlib ALIAS zlib)
+ENDIF()
+
 option(PHMAP_INSTALL "" ON)
 add_subdirectory(./parallel-hashmap)
 

thirdparty/vcpkg/ports/zlib-ng/portfile.cmake

@@ -0,0 +1,89 @@
+vcpkg_from_github(
+    OUT_SOURCE_PATH SOURCE_PATH
+    REPO zlib-ng/zlib-ng
+    REF "${VERSION}"
+    SHA512 e2057c764f1d5aaee738edee7e977182c5b097e3c95489dcd8de813f237d92a05daaa86d68d44b331d9fec5d1802586a8f6cfb658ba849874aaa14e72a8107f5
+    HEAD_REF develop
+)
+
+# MeshLib: strip zlib-ng's GNU symbol version script and the matching .symver
+# pragmas in its C sources.
+#
+# Upstream's CMakeLists.txt defines -DHAVE_SYMVER (which turns on __asm__(
+# ".symver foo, foo@@ZLIB_NG_2.0.0") pragmas in zbuild.h) and passes
+# -Wl,--version-script=zlib-ng.map to the linker whenever the target is
+# non-Apple, non-AIX UNIX. Both together tag every exported symbol in
+# libz-ng.so with ZLIB_NG_2.0.0 / ZLIB_NG_2.1.0 version nodes, which end up
+# in DT_VERNEED of anything linking against libz-ng.
+#
+# auditwheel's manylinux policy database has no entry for (libz-ng.so.2,
+# ZLIB_NG_*), so the MeshLib NuGet wheel-repair step fails with "too-recent
+# versioned symbols" even though no actual symbol is too recent. We don't
+# exercise zlib-ng's ABI-versioning machinery (our consumers rebuild against
+# whatever libz-ng we ship), so we neutralize both knobs by flipping the
+# guarding condition to FALSE. Upstream's zlib-ng.map file is left on disk
+# but never wired into the build.
+vcpkg_replace_string(
+    "${SOURCE_PATH}/CMakeLists.txt"
+    "if(NOT APPLE AND NOT CMAKE_SYSTEM_NAME STREQUAL AIX)"
+    "if(FALSE)  # MeshLib: symbol versioning disabled, see thirdparty/vcpkg/ports/zlib-ng/portfile.cmake"
+)
+
+# Set ZLIB_COMPAT in the triplet file to turn on
+if(NOT DEFINED ZLIB_COMPAT)
+    set(ZLIB_COMPAT OFF)
+endif()
+
+vcpkg_cmake_configure(
+    SOURCE_PATH "${SOURCE_PATH}"
+    OPTIONS
+        "-DZLIB_FULL_VERSION=${ZLIB_FULL_VERSION}"
+        -DZLIB_ENABLE_TESTS=OFF
+        -DWITH_NEW_STRATEGIES=ON
+        -DZLIB_COMPAT=${ZLIB_COMPAT}
+    OPTIONS_RELEASE
+        -DWITH_OPTIM=ON
+)
+vcpkg_cmake_install()
+vcpkg_copy_pdbs()
+
+# Condition in `WIN32`, from https://github.com/zlib-ng/zlib-ng/blob/2.1.5/CMakeLists.txt#L1081-L1100
+# (dynamic) for `zlib` or (static `MSVC) for `zlibstatic` or default `z`
+# i.e. (windows) and not (static mingw) https://learn.microsoft.com/en-us/vcpkg/maintainers/variables#vcpkg_target_is_system
+if(VCPKG_TARGET_IS_WINDOWS AND (NOT (VCPKG_LIBRARY_LINKAGE STREQUAL static AND VCPKG_TARGET_IS_MINGW)))
+    set(_port_suffix)
+    if(ZLIB_COMPAT)
+        set(_port_suffix "")
+    else()
+        set(_port_suffix "-ng")
+    endif()
+
+    set(_port_output_name)
+    if(VCPKG_LIBRARY_LINKAGE STREQUAL "dynamic")
+        set(_port_output_name "zlib${_port_suffix}")
+    else()
+        set(_port_output_name "zlibstatic${_port_suffix}")
+    endif()
+
+    # CMAKE_DEBUG_POSTFIX from https://github.com/zlib-ng/zlib-ng/blob/2.1.5/CMakeLists.txt#L494
+    if(NOT DEFINED VCPKG_BUILD_TYPE OR VCPKG_BUILD_TYPE STREQUAL "release")
+        vcpkg_replace_string("${CURRENT_PACKAGES_DIR}/lib/pkgconfig/zlib${_port_suffix}.pc" " -lz${_port_suffix}" " -l${_port_output_name}")
+    endif()
+    if(NOT DEFINED VCPKG_BUILD_TYPE OR VCPKG_BUILD_TYPE STREQUAL "debug")
+        vcpkg_replace_string("${CURRENT_PACKAGES_DIR}/debug/lib/pkgconfig/zlib${_port_suffix}.pc" " -lz${_port_suffix}" " -l${_port_output_name}d")
+    endif()
+endif()
+
+vcpkg_fixup_pkgconfig()
+
+if(ZLIB_COMPAT)
+    set(_cmake_dir "ZLIB")
+else()
+    set(_cmake_dir "zlib-ng")
+endif()
+vcpkg_cmake_config_fixup(CONFIG_PATH lib/cmake/${_cmake_dir})
+
+file(REMOVE_RECURSE "${CURRENT_PACKAGES_DIR}/debug/share"
+                    "${CURRENT_PACKAGES_DIR}/debug/include"
+)
+vcpkg_install_copyright(FILE_LIST "${SOURCE_PATH}/LICENSE.md")

thirdparty/vcpkg/ports/zlib-ng/vcpkg.json

@@ -0,0 +1,18 @@
+{
+  "name": "zlib-ng",
+  "version": "2.3.3",
+  "port-version": 1,
+  "description": "zlib replacement with optimizations for 'next generation' systems",
+  "homepage": "https://github.com/zlib-ng/zlib-ng",
+  "license": "Zlib",
+  "dependencies": [
+    {
+      "name": "vcpkg-cmake",
+      "host": true
+    },
+    {
+      "name": "vcpkg-cmake-config",
+      "host": true
+    }
+  ]
+}