diff --git a/.gitmodules b/.gitmodules index ef9d7a0d5172..0dfdab471dec 100644 --- a/.gitmodules +++ b/.gitmodules @@ -85,3 +85,6 @@ [submodule "thirdparty/cpp-httplib"] path = thirdparty/cpp-httplib url = https://github.com/yhirose/cpp-httplib +[submodule "thirdparty/zlib-ng"] + path = thirdparty/zlib-ng + url = https://github.com/zlib-ng/zlib-ng.git diff --git a/requirements/macos.txt b/requirements/macos.txt index 9324a774689d..f6fefab9825a 100644 --- a/requirements/macos.txt +++ b/requirements/macos.txt @@ -24,3 +24,4 @@ tbb tinyxml2 tl-expected zlib +zlib-ng diff --git a/requirements/vcpkg-linux.txt b/requirements/vcpkg-linux.txt index d590e19fdff1..a770ca2f3987 100644 --- a/requirements/vcpkg-linux.txt +++ b/requirements/vcpkg-linux.txt @@ -37,3 +37,4 @@ tiff tinygltf tinyxml2 tl-expected +zlib-ng diff --git a/requirements/windows.txt b/requirements/windows.txt index 9bd83d7286fc..942effad832d 100644 --- a/requirements/windows.txt +++ b/requirements/windows.txt @@ -30,5 +30,6 @@ tiff tinygltf tinyxml2 tl-expected +zlib-ng glad glfw3 diff --git a/source/MRMesh/CMakeLists.txt b/source/MRMesh/CMakeLists.txt index 3621ac029714..ee809142e17f 100644 --- a/source/MRMesh/CMakeLists.txt +++ b/source/MRMesh/CMakeLists.txt @@ -65,6 +65,80 @@ ELSE() ENDIF() target_link_libraries(${PROJECT_NAME} PRIVATE libzip::zip) +# zlib-ng (native mode, zng_ prefix) powers MRZlib's zlibCompressStream; +# decompression and the compressZip/libzip pipeline stay on stock zlib. +# +# Windows: use the proper CMake config package that vcpkg's zlib-ng port +# ships. Multi-config Debug/Release matters here: find_library() picks one +# .lib and uses it for every configuration, so a Debug build links against +# the release zlib-ng2.dll and the applocal-copy step doesn't copy it next +# to the .exe (STATUS_DLL_NOT_FOUND at launch, seen on msvc-2019 Debug +# CMake with the x64-windows-meshlib-iterator-debug triplet). The +# find_package(CONFIG) target has IMPORTED_CONFIGURATIONS=DEBUG;RELEASE +# and drives applocal-copy correctly. +# +# Other platforms: find_library + find_path. On Ubuntu apt and Emscripten +# we build zlib-ng from thirdparty/zlib-ng and its generated config has +# the same non-relocatable-include-path bug we hit with libdeflate (breaks +# under the Docker multi-stage COPY-shuffle). Plain library/path lookup +# sidesteps it, and those platforms are single-config so the multi-config +# concern above doesn't apply. +IF(WIN32) + # Can't use find_package(zlib-ng CONFIG): the vcpkg zlib-ng 2.1.5 port + # that vcpkg 2024.10.21 (pinned for msvc-2019 legs) ships doesn't install + # a CMake config file. Can't use a plain find_library() either: it loses + # Debug/Release selectivity, linking Debug MRMesh.dll against release + # zlib-ng2.dll and breaking applocal-copy (see the diagnostic output on + # the iterator-debug triplet, STATUS_DLL_NOT_FOUND at MeshViewer.exe). + # + # Manually construct an imported target with per-configuration IMPLIB + + # LOCATION paths, locating the files directly under vcpkg's triplet + # install tree. Gives applocal-copy the metadata it needs to copy the + # correct DLL next to the .exe per configuration. + find_path(LIBZLIBNG_INCLUDE_DIR NAMES zlib-ng.h REQUIRED) + find_library(LIBZLIBNG_LIBRARY_RELEASE + NAMES zlib-ng + PATHS "${VCPKG_INSTALLED_DIR}/${VCPKG_TARGET_TRIPLET}/lib" + NO_DEFAULT_PATH REQUIRED + ) + find_library(LIBZLIBNG_LIBRARY_DEBUG + NAMES zlib-ngd zlib-ng + PATHS "${VCPKG_INSTALLED_DIR}/${VCPKG_TARGET_TRIPLET}/debug/lib" + NO_DEFAULT_PATH REQUIRED + ) + find_program(LIBZLIBNG_DLL_RELEASE + NAMES zlib-ng2.dll zlib-ng.dll + PATHS "${VCPKG_INSTALLED_DIR}/${VCPKG_TARGET_TRIPLET}/bin" + NO_DEFAULT_PATH REQUIRED + ) + find_program(LIBZLIBNG_DLL_DEBUG + NAMES zlib-ngd2.dll zlib-ng2.dll zlib-ng.dll + PATHS "${VCPKG_INSTALLED_DIR}/${VCPKG_TARGET_TRIPLET}/debug/bin" + NO_DEFAULT_PATH REQUIRED + ) + add_library(zlib-ng::zlib SHARED IMPORTED) + set_target_properties(zlib-ng::zlib PROPERTIES + IMPORTED_CONFIGURATIONS "DEBUG;RELEASE" + IMPORTED_IMPLIB_RELEASE "${LIBZLIBNG_LIBRARY_RELEASE}" + IMPORTED_LOCATION_RELEASE "${LIBZLIBNG_DLL_RELEASE}" + IMPORTED_IMPLIB_DEBUG "${LIBZLIBNG_LIBRARY_DEBUG}" + IMPORTED_LOCATION_DEBUG "${LIBZLIBNG_DLL_DEBUG}" + INTERFACE_INCLUDE_DIRECTORIES "${LIBZLIBNG_INCLUDE_DIR}" + ) + target_link_libraries(${PROJECT_NAME} PRIVATE zlib-ng::zlib) +ELSE() + find_library(LIBZLIBNG_LIBRARY + NAMES z-ng zlib-ng + REQUIRED + ) + find_path(LIBZLIBNG_INCLUDE_DIR + NAMES zlib-ng.h + REQUIRED + ) + target_link_libraries(${PROJECT_NAME} PRIVATE ${LIBZLIBNG_LIBRARY}) + target_include_directories(${PROJECT_NAME} PRIVATE ${LIBZLIBNG_INCLUDE_DIR}) +ENDIF() + # TODO: CMake config target_include_directories(${PROJECT_NAME} PUBLIC diff --git a/source/MRMesh/MRZlib.cpp b/source/MRMesh/MRZlib.cpp index 259298dc21aa..e83cf589ff4a 100644 --- a/source/MRMesh/MRZlib.cpp +++ b/source/MRMesh/MRZlib.cpp @@ -2,50 +2,47 @@ #include "MRBuffer.h" #include "MRFinally.h" -#include +// zlib-ng in native mode: the zng_ prefix on every symbol keeps it ABI- +// distinct from stock zlib (which libzip still links as before), and +// re-exports the MAX_WBITS / Z_* constants we need under the +// same spelling as , so this TU doesn't include stock zlib's +// header at all. +#include #include +#include namespace { constexpr size_t cChunkSize = 256 * 1024; // 256 KiB -// zlib's `windowBits` argument is sign-encoded: positive = zlib wrapper (RFC 1950); -// negative = raw deflate (RFC 1951, no wrapper). Magnitude is log2(window size); -// MAX_WBITS = 15 gives a 32 KiB window. +// windowBits is sign-encoded the same way zlib documents it: positive = +// zlib wrapper (RFC 1950), negative = raw deflate (RFC 1951, no wrapper). +// Magnitude is log2(window size); MAX_WBITS = 15 gives a 32 KiB window. +// zlib-ng re-exports the same MAX_WBITS macro through zconf-ng.h, so we +// can use it here without pulling in stock zlib's headers. constexpr int kZlibWrapperBits = MAX_WBITS; constexpr int kRawDeflateBits = -MAX_WBITS; -// memLevel controls zlib's internal state size. 8 is zlib's default (its internal -// DEF_MEM_LEVEL in deflate.c is not exported; redeclared here). +// memLevel controls the compressor's internal state size. 8 is zlib's and +// zlib-ng's shared default (matches the old MRZlib choice). constexpr int kDefaultMemLevel = 8; -static_assert( kDefaultMemLevel <= MAX_MEM_LEVEL ); -std::string zlibToString( int code ) +std::string zngToString( int code ) { switch ( code ) { - case Z_OK: - return "ok"; - case Z_STREAM_END: - return "stream end"; - case Z_NEED_DICT: - return "need dict"; - case Z_ERRNO: - return "errno"; - case Z_STREAM_ERROR: - return "stream error"; - case Z_DATA_ERROR: - return "data error"; - case Z_MEM_ERROR: - return "mem error"; - case Z_BUF_ERROR: - return "buf error"; - case Z_VERSION_ERROR: - return "version error"; - default: - return "unknown code"; + case Z_OK: return "ok"; + case Z_STREAM_END: return "stream end"; + case Z_NEED_DICT: return "need dict"; + case Z_ERRNO: return "errno"; + case Z_STREAM_ERROR: return "stream error"; + case Z_DATA_ERROR: return "data error"; + case Z_MEM_ERROR: return "mem error"; + case Z_BUF_ERROR: return "buf error"; + case Z_VERSION_ERROR: return "version error"; + default: return "unknown code"; } } @@ -62,17 +59,15 @@ namespace MR Expected zlibCompressStream( std::istream& in, std::ostream& out, const ZlibCompressParams& params ) { Buffer inChunk( cChunkSize ), outChunk( cChunkSize ); - z_stream stream { - .zalloc = Z_NULL, - .zfree = Z_NULL, - .opaque = Z_NULL, - }; + zng_stream stream{}; int ret; - if ( Z_OK != ( ret = deflateInit2( &stream, params.level, Z_DEFLATED, windowBitsFor( params.rawDeflate ), kDefaultMemLevel, Z_DEFAULT_STRATEGY ) ) ) - return unexpected( zlibToString( ret ) ); + if ( Z_OK != ( ret = zng_deflateInit2( &stream, params.level, Z_DEFLATED, + windowBitsFor( params.rawDeflate ), + kDefaultMemLevel, Z_DEFAULT_STRATEGY ) ) ) + return unexpected( zngToString( ret ) ); MR_FINALLY { - deflateEnd( &stream ); + zng_deflateEnd( &stream ); }; if ( params.stats ) @@ -80,30 +75,31 @@ Expected zlibCompressStream( std::istream& in, std::ostream& out, const Zl while ( !in.eof() ) { - in.read( inChunk.data(), inChunk.size() ); + in.read( inChunk.data(), static_cast( inChunk.size() ) ); if ( in.bad() ) return unexpected( "I/O error" ); stream.next_in = reinterpret_cast( inChunk.data() ); - stream.avail_in = (unsigned)in.gcount(); - assert( stream.avail_in <= (unsigned)inChunk.size() ); + stream.avail_in = static_cast( in.gcount() ); + assert( stream.avail_in <= static_cast( inChunk.size() ) ); if ( params.stats ) { - params.stats->crc32 = (uint32_t)crc32( params.stats->crc32, stream.next_in, stream.avail_in ); + params.stats->crc32 = static_cast( + zng_crc32( params.stats->crc32, stream.next_in, stream.avail_in ) ); params.stats->uncompressedSize += stream.avail_in; } - const auto flush = in.eof() ? Z_FINISH : Z_NO_FLUSH; + const int flush = in.eof() ? Z_FINISH : Z_NO_FLUSH; do { stream.next_out = reinterpret_cast( outChunk.data() ); - stream.avail_out = (unsigned)outChunk.size(); - ret = deflate( &stream, flush ); + stream.avail_out = static_cast( outChunk.size() ); + ret = zng_deflate( &stream, flush ); if ( Z_OK != ret && Z_STREAM_END != ret ) - return unexpected( zlibToString( ret ) ); + return unexpected( zngToString( ret ) ); - assert( stream.avail_out <= (unsigned)outChunk.size() ); - const unsigned written = (unsigned)outChunk.size() - stream.avail_out; + assert( stream.avail_out <= static_cast( outChunk.size() ) ); + const unsigned written = static_cast( outChunk.size() ) - stream.avail_out; out.write( outChunk.data(), written ); if ( out.bad() ) return unexpected( "I/O error" ); @@ -124,38 +120,34 @@ Expected zlibCompressStream( std::istream& in, std::ostream& out, int leve Expected zlibDecompressStream( std::istream& in, std::ostream& out, const ZlibParams& params ) { Buffer inChunk( cChunkSize ), outChunk( cChunkSize ); - z_stream stream { - .zalloc = Z_NULL, - .zfree = Z_NULL, - .opaque = Z_NULL, - }; + zng_stream stream{}; int ret; - if ( Z_OK != ( ret = inflateInit2( &stream, windowBitsFor( params.rawDeflate ) ) ) ) - return unexpected( zlibToString( ret ) ); + if ( Z_OK != ( ret = zng_inflateInit2( &stream, windowBitsFor( params.rawDeflate ) ) ) ) + return unexpected( zngToString( ret ) ); MR_FINALLY { - inflateEnd( &stream ); + zng_inflateEnd( &stream ); }; while ( !in.eof() ) { - in.read( inChunk.data(), inChunk.size() ); + in.read( inChunk.data(), static_cast( inChunk.size() ) ); if ( in.bad() ) return unexpected( "I/O error" ); stream.next_in = reinterpret_cast( inChunk.data() ); - stream.avail_in = (unsigned)in.gcount(); - assert( stream.avail_in <= (unsigned)inChunk.size() ); + stream.avail_in = static_cast( in.gcount() ); + assert( stream.avail_in <= static_cast( inChunk.size() ) ); do { stream.next_out = reinterpret_cast( outChunk.data() ); - stream.avail_out = (unsigned)outChunk.size(); - ret = inflate( &stream, Z_NO_FLUSH ); + stream.avail_out = static_cast( outChunk.size() ); + ret = zng_inflate( &stream, Z_NO_FLUSH ); if ( Z_OK != ret && Z_STREAM_END != ret ) - return unexpected( zlibToString( ret ) ); + return unexpected( zngToString( ret ) ); - assert( stream.avail_out <= (unsigned)outChunk.size() ); - out.write( outChunk.data(), (unsigned)outChunk.size() - stream.avail_out ); + assert( stream.avail_out <= static_cast( outChunk.size() ) ); + out.write( outChunk.data(), static_cast( outChunk.size() ) - stream.avail_out ); if ( out.bad() ) return unexpected( "I/O error" ); diff --git a/thirdparty/CMakeLists.txt b/thirdparty/CMakeLists.txt index e4734cf29af3..f2afb7fa7419 100644 --- a/thirdparty/CMakeLists.txt +++ b/thirdparty/CMakeLists.txt @@ -85,6 +85,40 @@ ENDIF() add_subdirectory(./OpenCTM-git ./OpenCTM) +# Build zlib-ng from our submodule (native mode -- zng_ prefix, does not shadow +# stock zlib) on Ubuntu apt and Emscripten, which have no zlib-ng package. +# Windows + macOS + any MESHLIB_USE_VCPKG config pick up the package-manager +# copy instead (see requirements/{windows,macos,vcpkg-linux}.txt). +IF(NOT WIN32 AND NOT APPLE AND NOT MESHLIB_USE_VCPKG) + set(ZLIB_COMPAT OFF CACHE BOOL "") + set(ZLIB_ENABLE_TESTS OFF CACHE BOOL "") + set(ZLIBNG_ENABLE_TESTS OFF CACHE BOOL "") + set(WITH_GTEST OFF CACHE BOOL "") + set(WITH_GZFILEOP OFF CACHE BOOL "") + IF(EMSCRIPTEN) + # zlib-ng's CMakeLists treats an undefined BUILD_SHARED_LIBS as "build both + # shared and static targets". Emscripten then demotes the SHARED one to + # STATIC, after which both rules emit the same libz-ng.a and ninja errors + # with "multiple rules generate libz-ng.a [-w dupbuild=err]". Force static + # only for this add_subdirectory -- Emscripten produces static output + # either way. + # + # After add_subdirectory(./zlib-ng) we MUST restore BUILD_SHARED_LIBS to + # its prior *undefined* state, not "defined but empty". jsoncpp further + # down expands ${BUILD_SHARED_LIBS} into set_target_properties() pairs and + # silently breaks with "set_target_properties called with incorrect number + # of arguments" when the expansion yields empty. jsoncpp's own option() + # fallback only fires for undefined, not empty. unset() is therefore the + # right tool here (BUILD_SHARED_LIBS was undefined before this block on + # every Emscripten MeshLib build). + set(BUILD_SHARED_LIBS OFF) + add_subdirectory(./zlib-ng) + unset(BUILD_SHARED_LIBS) + ELSE() + add_subdirectory(./zlib-ng) + ENDIF() +ENDIF() + option(PHMAP_INSTALL "" ON) add_subdirectory(./parallel-hashmap) diff --git a/thirdparty/vcpkg/ports/zlib-ng/portfile.cmake b/thirdparty/vcpkg/ports/zlib-ng/portfile.cmake new file mode 100644 index 000000000000..a842eabbd402 --- /dev/null +++ b/thirdparty/vcpkg/ports/zlib-ng/portfile.cmake @@ -0,0 +1,89 @@ +vcpkg_from_github( + OUT_SOURCE_PATH SOURCE_PATH + REPO zlib-ng/zlib-ng + REF "${VERSION}" + SHA512 e2057c764f1d5aaee738edee7e977182c5b097e3c95489dcd8de813f237d92a05daaa86d68d44b331d9fec5d1802586a8f6cfb658ba849874aaa14e72a8107f5 + HEAD_REF develop +) + +# MeshLib: strip zlib-ng's GNU symbol version script and the matching .symver +# pragmas in its C sources. +# +# Upstream's CMakeLists.txt defines -DHAVE_SYMVER (which turns on __asm__( +# ".symver foo, foo@@ZLIB_NG_2.0.0") pragmas in zbuild.h) and passes +# -Wl,--version-script=zlib-ng.map to the linker whenever the target is +# non-Apple, non-AIX UNIX. Both together tag every exported symbol in +# libz-ng.so with ZLIB_NG_2.0.0 / ZLIB_NG_2.1.0 version nodes, which end up +# in DT_VERNEED of anything linking against libz-ng. +# +# auditwheel's manylinux policy database has no entry for (libz-ng.so.2, +# ZLIB_NG_*), so the MeshLib NuGet wheel-repair step fails with "too-recent +# versioned symbols" even though no actual symbol is too recent. We don't +# exercise zlib-ng's ABI-versioning machinery (our consumers rebuild against +# whatever libz-ng we ship), so we neutralize both knobs by flipping the +# guarding condition to FALSE. Upstream's zlib-ng.map file is left on disk +# but never wired into the build. +vcpkg_replace_string( + "${SOURCE_PATH}/CMakeLists.txt" + "if(NOT APPLE AND NOT CMAKE_SYSTEM_NAME STREQUAL AIX)" + "if(FALSE) # MeshLib: symbol versioning disabled, see thirdparty/vcpkg/ports/zlib-ng/portfile.cmake" +) + +# Set ZLIB_COMPAT in the triplet file to turn on +if(NOT DEFINED ZLIB_COMPAT) + set(ZLIB_COMPAT OFF) +endif() + +vcpkg_cmake_configure( + SOURCE_PATH "${SOURCE_PATH}" + OPTIONS + "-DZLIB_FULL_VERSION=${ZLIB_FULL_VERSION}" + -DZLIB_ENABLE_TESTS=OFF + -DWITH_NEW_STRATEGIES=ON + -DZLIB_COMPAT=${ZLIB_COMPAT} + OPTIONS_RELEASE + -DWITH_OPTIM=ON +) +vcpkg_cmake_install() +vcpkg_copy_pdbs() + +# Condition in `WIN32`, from https://github.com/zlib-ng/zlib-ng/blob/2.1.5/CMakeLists.txt#L1081-L1100 +# (dynamic) for `zlib` or (static `MSVC) for `zlibstatic` or default `z` +# i.e. (windows) and not (static mingw) https://learn.microsoft.com/en-us/vcpkg/maintainers/variables#vcpkg_target_is_system +if(VCPKG_TARGET_IS_WINDOWS AND (NOT (VCPKG_LIBRARY_LINKAGE STREQUAL static AND VCPKG_TARGET_IS_MINGW))) + set(_port_suffix) + if(ZLIB_COMPAT) + set(_port_suffix "") + else() + set(_port_suffix "-ng") + endif() + + set(_port_output_name) + if(VCPKG_LIBRARY_LINKAGE STREQUAL "dynamic") + set(_port_output_name "zlib${_port_suffix}") + else() + set(_port_output_name "zlibstatic${_port_suffix}") + endif() + + # CMAKE_DEBUG_POSTFIX from https://github.com/zlib-ng/zlib-ng/blob/2.1.5/CMakeLists.txt#L494 + if(NOT DEFINED VCPKG_BUILD_TYPE OR VCPKG_BUILD_TYPE STREQUAL "release") + vcpkg_replace_string("${CURRENT_PACKAGES_DIR}/lib/pkgconfig/zlib${_port_suffix}.pc" " -lz${_port_suffix}" " -l${_port_output_name}") + endif() + if(NOT DEFINED VCPKG_BUILD_TYPE OR VCPKG_BUILD_TYPE STREQUAL "debug") + vcpkg_replace_string("${CURRENT_PACKAGES_DIR}/debug/lib/pkgconfig/zlib${_port_suffix}.pc" " -lz${_port_suffix}" " -l${_port_output_name}d") + endif() +endif() + +vcpkg_fixup_pkgconfig() + +if(ZLIB_COMPAT) + set(_cmake_dir "ZLIB") +else() + set(_cmake_dir "zlib-ng") +endif() +vcpkg_cmake_config_fixup(CONFIG_PATH lib/cmake/${_cmake_dir}) + +file(REMOVE_RECURSE "${CURRENT_PACKAGES_DIR}/debug/share" + "${CURRENT_PACKAGES_DIR}/debug/include" +) +vcpkg_install_copyright(FILE_LIST "${SOURCE_PATH}/LICENSE.md") diff --git a/thirdparty/vcpkg/ports/zlib-ng/vcpkg.json b/thirdparty/vcpkg/ports/zlib-ng/vcpkg.json new file mode 100644 index 000000000000..380f3c9ab75e --- /dev/null +++ b/thirdparty/vcpkg/ports/zlib-ng/vcpkg.json @@ -0,0 +1,18 @@ +{ + "name": "zlib-ng", + "version": "2.3.3", + "port-version": 1, + "description": "zlib replacement with optimizations for 'next generation' systems", + "homepage": "https://github.com/zlib-ng/zlib-ng", + "license": "Zlib", + "dependencies": [ + { + "name": "vcpkg-cmake", + "host": true + }, + { + "name": "vcpkg-cmake-config", + "host": true + } + ] +} diff --git a/thirdparty/zlib-ng b/thirdparty/zlib-ng new file mode 160000 index 000000000000..860e4cff7917 --- /dev/null +++ b/thirdparty/zlib-ng @@ -0,0 +1 @@ +Subproject commit 860e4cff7917d93f54f5d7f0bc1d0e8b1a3cb988