MRZlib: route compress/decompress through zlib-ng (native) — green snapshot of #5959

.gitmodules

@@ -85,3 +85,6 @@
 [submodule "thirdparty/cpp-httplib"]
 	path = thirdparty/cpp-httplib
 	url = https://github.com/yhirose/cpp-httplib
+[submodule "thirdparty/zlib-ng"]
+	path = thirdparty/zlib-ng
+	url = https://github.com/zlib-ng/zlib-ng.git

requirements/macos.txt

@@ -24,3 +24,4 @@ tbb
 tinyxml2
 tl-expected
 zlib
+zlib-ng

requirements/vcpkg-linux.txt

@@ -37,3 +37,4 @@ tiff
 tinygltf
 tinyxml2
 tl-expected
+zlib-ng

requirements/windows.txt

@@ -30,5 +30,6 @@ tiff
 tinygltf
 tinyxml2
 tl-expected
+zlib-ng
 glad
 glfw3

source/MRMesh/CMakeLists.txt

@@ -65,6 +65,80 @@ ELSE()
 ENDIF()
 target_link_libraries(${PROJECT_NAME} PRIVATE libzip::zip)
 
+# zlib-ng (native mode, zng_ prefix) powers MRZlib's zlibCompressStream;
+# decompression and the compressZip/libzip pipeline stay on stock zlib.
+#
+# Windows: use the proper CMake config package that vcpkg's zlib-ng port
+# ships. Multi-config Debug/Release matters here: find_library() picks one
+# .lib and uses it for every configuration, so a Debug build links against
+# the release zlib-ng2.dll and the applocal-copy step doesn't copy it next
+# to the .exe (STATUS_DLL_NOT_FOUND at launch, seen on msvc-2019 Debug
+# CMake with the x64-windows-meshlib-iterator-debug triplet). The
+# find_package(CONFIG) target has IMPORTED_CONFIGURATIONS=DEBUG;RELEASE
+# and drives applocal-copy correctly.
+#
+# Other platforms: find_library + find_path. On Ubuntu apt and Emscripten
+# we build zlib-ng from thirdparty/zlib-ng and its generated config has
+# the same non-relocatable-include-path bug we hit with libdeflate (breaks
+# under the Docker multi-stage COPY-shuffle). Plain library/path lookup
+# sidesteps it, and those platforms are single-config so the multi-config
+# concern above doesn't apply.
+IF(WIN32)
+  # Can't use find_package(zlib-ng CONFIG): the vcpkg zlib-ng 2.1.5 port
+  # that vcpkg 2024.10.21 (pinned for msvc-2019 legs) ships doesn't install
+  # a CMake config file. Can't use a plain find_library() either: it loses
+  # Debug/Release selectivity, linking Debug MRMesh.dll against release
+  # zlib-ng2.dll and breaking applocal-copy (see the diagnostic output on
+  # the iterator-debug triplet, STATUS_DLL_NOT_FOUND at MeshViewer.exe).
+  #
+  # Manually construct an imported target with per-configuration IMPLIB +
+  # LOCATION paths, locating the files directly under vcpkg's triplet
+  # install tree. Gives applocal-copy the metadata it needs to copy the
+  # correct DLL next to the .exe per configuration.
+  find_path(LIBZLIBNG_INCLUDE_DIR NAMES zlib-ng.h REQUIRED)
+  find_library(LIBZLIBNG_LIBRARY_RELEASE
+    NAMES zlib-ng
+    PATHS "${VCPKG_INSTALLED_DIR}/${VCPKG_TARGET_TRIPLET}/lib"
+    NO_DEFAULT_PATH REQUIRED
+  )
+  find_library(LIBZLIBNG_LIBRARY_DEBUG
+    NAMES zlib-ngd zlib-ng
+    PATHS "${VCPKG_INSTALLED_DIR}/${VCPKG_TARGET_TRIPLET}/debug/lib"
+    NO_DEFAULT_PATH REQUIRED
+  )
+  find_program(LIBZLIBNG_DLL_RELEASE
+    NAMES zlib-ng2.dll zlib-ng.dll
+    PATHS "${VCPKG_INSTALLED_DIR}/${VCPKG_TARGET_TRIPLET}/bin"
+    NO_DEFAULT_PATH REQUIRED
+  )
+  find_program(LIBZLIBNG_DLL_DEBUG
+    NAMES zlib-ngd2.dll zlib-ng2.dll zlib-ng.dll
+    PATHS "${VCPKG_INSTALLED_DIR}/${VCPKG_TARGET_TRIPLET}/debug/bin"
+    NO_DEFAULT_PATH REQUIRED
+  )
+  add_library(zlib-ng::zlib SHARED IMPORTED)
+  set_target_properties(zlib-ng::zlib PROPERTIES
+    IMPORTED_CONFIGURATIONS "DEBUG;RELEASE"
+    IMPORTED_IMPLIB_RELEASE  "${LIBZLIBNG_LIBRARY_RELEASE}"
+    IMPORTED_LOCATION_RELEASE "${LIBZLIBNG_DLL_RELEASE}"
+    IMPORTED_IMPLIB_DEBUG    "${LIBZLIBNG_LIBRARY_DEBUG}"
+    IMPORTED_LOCATION_DEBUG  "${LIBZLIBNG_DLL_DEBUG}"
+    INTERFACE_INCLUDE_DIRECTORIES "${LIBZLIBNG_INCLUDE_DIR}"
+  )
+  target_link_libraries(${PROJECT_NAME} PRIVATE zlib-ng::zlib)
+ELSE()
+  find_library(LIBZLIBNG_LIBRARY
+    NAMES z-ng zlib-ng
+    REQUIRED
+  )
+  find_path(LIBZLIBNG_INCLUDE_DIR
+    NAMES zlib-ng.h
+    REQUIRED
+  )
+  target_link_libraries(${PROJECT_NAME} PRIVATE ${LIBZLIBNG_LIBRARY})
+  target_include_directories(${PROJECT_NAME} PRIVATE ${LIBZLIBNG_INCLUDE_DIR})
+ENDIF()
+
 # TODO: CMake config
 target_include_directories(${PROJECT_NAME}
   PUBLIC

source/MRMesh/MRZlib.cpp

@@ -2,7 +2,7 @@
 #include "MRBuffer.h"
 #include "MRFinally.h"
 
-#include <zlib.h>
+#include <zlib-ng.h>
 
 #include <cassert>
 
@@ -22,7 +22,7 @@ constexpr int kRawDeflateBits  = -MAX_WBITS;
 constexpr int kDefaultMemLevel = 8;
 static_assert( kDefaultMemLevel <= MAX_MEM_LEVEL );
 
-std::string zlibToString( int code )
+std::string zngToString( int code )
 {
     switch ( code )
     {
@@ -62,17 +62,17 @@ namespace MR
 Expected<void> zlibCompressStream( std::istream& in, std::ostream& out, const ZlibCompressParams& params )
 {
     Buffer<char> inChunk( cChunkSize ), outChunk( cChunkSize );
-    z_stream stream {
+    zng_stream stream {
         .zalloc = Z_NULL,
         .zfree = Z_NULL,
         .opaque = Z_NULL,
     };
     int ret;
-    if ( Z_OK != ( ret = deflateInit2( &stream, params.level, Z_DEFLATED, windowBitsFor( params.rawDeflate ), kDefaultMemLevel, Z_DEFAULT_STRATEGY ) ) )
-        return unexpected( zlibToString( ret ) );
+    if ( Z_OK != ( ret = zng_deflateInit2( &stream, params.level, Z_DEFLATED, windowBitsFor( params.rawDeflate ), kDefaultMemLevel, Z_DEFAULT_STRATEGY ) ) )
+        return unexpected( zngToString( ret ) );
 
     MR_FINALLY {
-        deflateEnd( &stream );
+        zng_deflateEnd( &stream );
     };
 
     if ( params.stats )
@@ -89,7 +89,7 @@ Expected<void> zlibCompressStream( std::istream& in, std::ostream& out, const Zl
 
         if ( params.stats )
         {
-            params.stats->crc32 = (uint32_t)crc32( params.stats->crc32, stream.next_in, stream.avail_in );
+            params.stats->crc32 = (uint32_t)zng_crc32( params.stats->crc32, stream.next_in, stream.avail_in );
             params.stats->uncompressedSize += stream.avail_in;
         }
 
@@ -98,9 +98,9 @@ Expected<void> zlibCompressStream( std::istream& in, std::ostream& out, const Zl
         {
             stream.next_out = reinterpret_cast<uint8_t*>( outChunk.data() );
             stream.avail_out = (unsigned)outChunk.size();
-            ret = deflate( &stream, flush );
+            ret = zng_deflate( &stream, flush );
             if ( Z_OK != ret && Z_STREAM_END != ret )
-                return unexpected( zlibToString( ret ) );
+                return unexpected( zngToString( ret ) );
 
             assert( stream.avail_out <= (unsigned)outChunk.size() );
             const unsigned written = (unsigned)outChunk.size() - stream.avail_out;
@@ -124,17 +124,17 @@ Expected<void> zlibCompressStream( std::istream& in, std::ostream& out, int leve
 Expected<void> zlibDecompressStream( std::istream& in, std::ostream& out, const ZlibParams& params )
 {
     Buffer<char> inChunk( cChunkSize ), outChunk( cChunkSize );
-    z_stream stream {
+    zng_stream stream {
         .zalloc = Z_NULL,
         .zfree = Z_NULL,
         .opaque = Z_NULL,
     };
     int ret;
-    if ( Z_OK != ( ret = inflateInit2( &stream, windowBitsFor( params.rawDeflate ) ) ) )
-        return unexpected( zlibToString( ret ) );
+    if ( Z_OK != ( ret = zng_inflateInit2( &stream, windowBitsFor( params.rawDeflate ) ) ) )
+        return unexpected( zngToString( ret ) );
 
     MR_FINALLY {
-        inflateEnd( &stream );
+        zng_inflateEnd( &stream );
     };
 
     while ( !in.eof() )
@@ -150,9 +150,9 @@ Expected<void> zlibDecompressStream( std::istream& in, std::ostream& out, const
         {
             stream.next_out = reinterpret_cast<uint8_t*>( outChunk.data() );
             stream.avail_out = (unsigned)outChunk.size();
-            ret = inflate( &stream, Z_NO_FLUSH );
+            ret = zng_inflate( &stream, Z_NO_FLUSH );
             if ( Z_OK != ret && Z_STREAM_END != ret )
-                return unexpected( zlibToString( ret ) );
+                return unexpected( zngToString( ret ) );
 
             assert( stream.avail_out <= (unsigned)outChunk.size() );
             out.write( outChunk.data(), (unsigned)outChunk.size() - stream.avail_out );

thirdparty/CMakeLists.txt

@@ -85,6 +85,41 @@ ENDIF()
 
 add_subdirectory(./OpenCTM-git ./OpenCTM)
 
+# Build zlib-ng from our submodule (native mode -- zng_ prefix, does not shadow
+# stock zlib) on Ubuntu apt and Emscripten, which have no zlib-ng package.
+# macOS picks up the Homebrew copy instead (see requirements/macos.txt).
+# Windows and MESHLIB_USE_VCPKG configs never enter this file -- they get
+# zlib-ng from vcpkg (see requirements/{windows,vcpkg-*}.txt).
+IF(NOT APPLE)
+  set(ZLIB_COMPAT       OFF CACHE BOOL "")
+  set(ZLIB_ENABLE_TESTS OFF CACHE BOOL "")
+  set(ZLIBNG_ENABLE_TESTS OFF CACHE BOOL "")
+  set(WITH_GTEST        OFF CACHE BOOL "")
+  set(WITH_GZFILEOP     OFF CACHE BOOL "")
+  IF(EMSCRIPTEN)
+    # zlib-ng's CMakeLists treats an undefined BUILD_SHARED_LIBS as "build both
+    # shared and static targets". Emscripten then demotes the SHARED one to
+    # STATIC, after which both rules emit the same libz-ng.a and ninja errors
+    # with "multiple rules generate libz-ng.a [-w dupbuild=err]". Force static
+    # only for this add_subdirectory -- Emscripten produces static output
+    # either way.
+    #
+    # After add_subdirectory(./zlib-ng) we MUST restore BUILD_SHARED_LIBS to
+    # its prior *undefined* state, not "defined but empty". jsoncpp further
+    # down expands ${BUILD_SHARED_LIBS} into set_target_properties() pairs and
+    # silently breaks with "set_target_properties called with incorrect number
+    # of arguments" when the expansion yields empty. jsoncpp's own option()
+    # fallback only fires for undefined, not empty. unset() is therefore the
+    # right tool here (BUILD_SHARED_LIBS was undefined before this block on
+    # every Emscripten MeshLib build).
+    set(BUILD_SHARED_LIBS OFF)
+    add_subdirectory(./zlib-ng)
+    unset(BUILD_SHARED_LIBS)
+  ELSE()
+    add_subdirectory(./zlib-ng)
+  ENDIF()
+ENDIF()
+
 option(PHMAP_INSTALL "" ON)
 add_subdirectory(./parallel-hashmap)
 

thirdparty/vcpkg/ports/zlib-ng/portfile.cmake

@@ -0,0 +1,89 @@
+vcpkg_from_github(
+    OUT_SOURCE_PATH SOURCE_PATH
+    REPO zlib-ng/zlib-ng
+    REF "${VERSION}"
+    SHA512 e2057c764f1d5aaee738edee7e977182c5b097e3c95489dcd8de813f237d92a05daaa86d68d44b331d9fec5d1802586a8f6cfb658ba849874aaa14e72a8107f5
+    HEAD_REF develop
+)
+
+# MeshLib: strip zlib-ng's GNU symbol version script and the matching .symver
+# pragmas in its C sources.
+#
+# Upstream's CMakeLists.txt defines -DHAVE_SYMVER (which turns on __asm__(
+# ".symver foo, foo@@ZLIB_NG_2.0.0") pragmas in zbuild.h) and passes
+# -Wl,--version-script=zlib-ng.map to the linker whenever the target is
+# non-Apple, non-AIX UNIX. Both together tag every exported symbol in
+# libz-ng.so with ZLIB_NG_2.0.0 / ZLIB_NG_2.1.0 version nodes, which end up
+# in DT_VERNEED of anything linking against libz-ng.
+#
+# auditwheel's manylinux policy database has no entry for (libz-ng.so.2,
+# ZLIB_NG_*), so the MeshLib NuGet wheel-repair step fails with "too-recent
+# versioned symbols" even though no actual symbol is too recent. We don't
+# exercise zlib-ng's ABI-versioning machinery (our consumers rebuild against
+# whatever libz-ng we ship), so we neutralize both knobs by flipping the
+# guarding condition to FALSE. Upstream's zlib-ng.map file is left on disk
+# but never wired into the build.
+vcpkg_replace_string(
+    "${SOURCE_PATH}/CMakeLists.txt"
+    "if(NOT APPLE AND NOT CMAKE_SYSTEM_NAME STREQUAL AIX)"
+    "if(FALSE)  # MeshLib: symbol versioning disabled, see thirdparty/vcpkg/ports/zlib-ng/portfile.cmake"
+)
+
+# Set ZLIB_COMPAT in the triplet file to turn on
+if(NOT DEFINED ZLIB_COMPAT)
+    set(ZLIB_COMPAT OFF)
+endif()
+
+vcpkg_cmake_configure(
+    SOURCE_PATH "${SOURCE_PATH}"
+    OPTIONS
+        "-DZLIB_FULL_VERSION=${ZLIB_FULL_VERSION}"
+        -DZLIB_ENABLE_TESTS=OFF
+        -DWITH_NEW_STRATEGIES=ON
+        -DZLIB_COMPAT=${ZLIB_COMPAT}
+    OPTIONS_RELEASE
+        -DWITH_OPTIM=ON
+)
+vcpkg_cmake_install()
+vcpkg_copy_pdbs()
+
+# Condition in `WIN32`, from https://github.com/zlib-ng/zlib-ng/blob/2.1.5/CMakeLists.txt#L1081-L1100
+# (dynamic) for `zlib` or (static `MSVC) for `zlibstatic` or default `z`
+# i.e. (windows) and not (static mingw) https://learn.microsoft.com/en-us/vcpkg/maintainers/variables#vcpkg_target_is_system
+if(VCPKG_TARGET_IS_WINDOWS AND (NOT (VCPKG_LIBRARY_LINKAGE STREQUAL static AND VCPKG_TARGET_IS_MINGW)))
+    set(_port_suffix)
+    if(ZLIB_COMPAT)
+        set(_port_suffix "")
+    else()
+        set(_port_suffix "-ng")
+    endif()
+
+    set(_port_output_name)
+    if(VCPKG_LIBRARY_LINKAGE STREQUAL "dynamic")
+        set(_port_output_name "zlib${_port_suffix}")
+    else()
+        set(_port_output_name "zlibstatic${_port_suffix}")
+    endif()
+
+    # CMAKE_DEBUG_POSTFIX from https://github.com/zlib-ng/zlib-ng/blob/2.1.5/CMakeLists.txt#L494
+    if(NOT DEFINED VCPKG_BUILD_TYPE OR VCPKG_BUILD_TYPE STREQUAL "release")
+        vcpkg_replace_string("${CURRENT_PACKAGES_DIR}/lib/pkgconfig/zlib${_port_suffix}.pc" " -lz${_port_suffix}" " -l${_port_output_name}")
+    endif()
+    if(NOT DEFINED VCPKG_BUILD_TYPE OR VCPKG_BUILD_TYPE STREQUAL "debug")
+        vcpkg_replace_string("${CURRENT_PACKAGES_DIR}/debug/lib/pkgconfig/zlib${_port_suffix}.pc" " -lz${_port_suffix}" " -l${_port_output_name}d")
+    endif()
+endif()
+
+vcpkg_fixup_pkgconfig()
+
+if(ZLIB_COMPAT)
+    set(_cmake_dir "ZLIB")
+else()
+    set(_cmake_dir "zlib-ng")
+endif()
+vcpkg_cmake_config_fixup(CONFIG_PATH lib/cmake/${_cmake_dir})
+
+file(REMOVE_RECURSE "${CURRENT_PACKAGES_DIR}/debug/share"
+                    "${CURRENT_PACKAGES_DIR}/debug/include"
+)
+vcpkg_install_copyright(FILE_LIST "${SOURCE_PATH}/LICENSE.md")

thirdparty/vcpkg/ports/zlib-ng/vcpkg.json

@@ -0,0 +1,18 @@
+{
+  "name": "zlib-ng",
+  "version": "2.3.3",
+  "port-version": 1,
+  "description": "zlib replacement with optimizations for 'next generation' systems",
+  "homepage": "https://github.com/zlib-ng/zlib-ng",
+  "license": "Zlib",
+  "dependencies": [
+    {
+      "name": "vcpkg-cmake",
+      "host": true
+    },
+    {
+      "name": "vcpkg-cmake-config",
+      "host": true
+    }
+  ]
+}