diff --git a/OneBranchPipelines/build-release-package-pipeline.yml b/OneBranchPipelines/build-release-package-pipeline.yml index 5f999289f..2c39429ef 100644 --- a/OneBranchPipelines/build-release-package-pipeline.yml +++ b/OneBranchPipelines/build-release-package-pipeline.yml @@ -69,6 +69,23 @@ parameters: displayName: 'Run SDL Security Tasks' type: boolean default: true + + # Which package to build. Both are produced from this same pipeline but are + # fully independent stage groups, so building only one halves the run time and + # lets you run two builds (one per package) concurrently. + # mssql-python -> Windows/macOS/Linux wheels + Consolidate + # mssql-python-odbc -> ODBC data wheels + ConsolidateOdbc + # both -> everything (previous default behaviour) + # Scheduled (daily) builds ignore this and always build 'both' (see + # effectiveBuildPackage below) so the nightly keeps producing every artifact. + - name: buildPackage + displayName: 'Package to Build' + type: string + values: + - 'mssql-python' + - 'mssql-python-odbc' + - 'both' + default: 'mssql-python' # ========================= # PLATFORM CONFIGURATIONS @@ -138,6 +155,33 @@ parameters: - { tag: 'musllinux', arch: 'x86_64', platform: 'linux/amd64' } - { tag: 'musllinux', arch: 'aarch64', platform: 'linux/arm64' } + # ========================= + # ODBC PACKAGE CONFIGURATIONS (mssql-python-odbc) + # ========================= + # The standalone mssql-python-odbc package ships ONLY pre-built ODBC driver + # binaries (data) — there is NO compiled extension — so ONE py3-none- + # wheel per platform serves every supported Python version. Hence there is no + # per-Python matrix here: 7 wheels total (2 Windows + 1 macOS + 4 Linux). + # These stages build alongside the mssql-python wheels and are consolidated + # SEPARATELY (see the ConsolidateOdbc stage) so the release pipeline can publish + # mssql-python and mssql-python-odbc independently. + - name: odbcWindowsConfigs + type: object + default: + - { arch: 'x64' } + - { arch: 'arm64' } + - name: odbcMacosConfigs + type: object + default: + - { name: 'universal2' } + - name: odbcLinuxConfigs + type: object + default: + - { tag: 'manylinux_2_28', arch: 'x86_64', platform: 'linux/amd64' } + - { tag: 'manylinux_2_28', arch: 'aarch64', platform: 'linux/arm64' } + - { tag: 'musllinux', arch: 'x86_64', platform: 'linux/amd64' } + - { tag: 'musllinux', arch: 'aarch64', platform: 'linux/arm64' } + # ========================= # PIPELINE VARIABLES # ========================= @@ -149,6 +193,14 @@ variables: value: 'Official' ${{ else }}: value: '${{ parameters.oneBranchType }}' + + # Determine which package(s) to build: scheduled (nightly) builds always build + # 'both' so every artifact keeps refreshing; manual/PR builds honour the parameter. + - name: effectiveBuildPackage + ${{ if eq(variables['Build.Reason'], 'Schedule') }}: + value: 'both' + ${{ else }}: + value: '${{ parameters.buildPackage }}' # Variable template imports # Each file provides specific variable groups: @@ -326,7 +378,13 @@ extends: # Version automatically detected from wheel metadata (setup.py) sbom: enabled: ${{ parameters.runSdlTasks }} - packageName: 'mssql-python' + # Name the SBOM after the package actually being built so an odbc-only build + # does not mislabel the driver wheels as 'mssql-python'. Combined ('both') and + # mssql-python builds keep the primary package name. + ${{ if eq(variables.effectiveBuildPackage, 'mssql-python-odbc') }}: + packageName: 'mssql-python-odbc' + ${{ else }}: + packageName: 'mssql-python' # TSA - Threat and Security Assessment # Uploads scan results to Microsoft's TSA tool for tracking @@ -338,8 +396,10 @@ extends: # ========================= # PIPELINE STAGES # ========================= - # Total stages: 9 Windows + 5 macOS + 4 Linux + 1 Consolidate = 19 stages - # Stages run in parallel (no dependencies between platform builds) + # mssql-python: 9 Windows + 5 macOS + 4 Linux + 1 Consolidate = 19 stages + # mssql-python-odbc: 2 Windows + 1 macOS + 4 Linux + 1 ConsolidateOdbc = 8 stages + # Total: 27 stages. All build stages run in parallel; each Consolidate* stage + # waits only on its own package's build stages. stages: # ========================= # WINDOWS BUILD STAGES @@ -355,16 +415,18 @@ extends: # 5. Builds wheel # 6. Publishes artifacts (wheels + PYD + PDB) # 7. ESRP malware scanning - - ${{ each config in parameters.windowsConfigs }}: - - template: /OneBranchPipelines/stages/build-windows-single-stage.yml@self - parameters: - stageName: Win_py${{ config.pyVer }}_${{ config.arch }} - jobName: BuildWheel - # Convert pyVer '310' → pythonVersion '3.10' - pythonVersion: ${{ format('{0}.{1}', substring(config.pyVer, 0, 1), substring(config.pyVer, 1, 2)) }} - shortPyVer: ${{ config.pyVer }} - architecture: ${{ config.arch }} - oneBranchType: '${{ variables.effectiveOneBranchType }}' + # Gate: build mssql-python stages unless an odbc-only build was requested + - ${{ if ne(variables.effectiveBuildPackage, 'mssql-python-odbc') }}: + - ${{ each config in parameters.windowsConfigs }}: + - template: /OneBranchPipelines/stages/build-windows-single-stage.yml@self + parameters: + stageName: Win_py${{ config.pyVer }}_${{ config.arch }} + jobName: BuildWheel + # Convert pyVer '310' → pythonVersion '3.10' + pythonVersion: ${{ format('{0}.{1}', substring(config.pyVer, 0, 1), substring(config.pyVer, 1, 2)) }} + shortPyVer: ${{ config.pyVer }} + architecture: ${{ config.arch }} + oneBranchType: '${{ variables.effectiveOneBranchType }}' # ========================= # MACOS BUILD STAGES @@ -381,15 +443,16 @@ extends: # 6. Builds wheel # 7. Publishes artifacts (wheels + .so) # 8. ESRP malware scanning - - ${{ each config in parameters.macosConfigs }}: - - template: /OneBranchPipelines/stages/build-macos-single-stage.yml@self - parameters: - stageName: MacOS_py${{ config.pyVer }} - jobName: BuildWheel - # Convert pyVer '310' → pythonVersion '3.10' - pythonVersion: ${{ format('{0}.{1}', substring(config.pyVer, 0, 1), substring(config.pyVer, 1, 2)) }} - shortPyVer: ${{ config.pyVer }} - oneBranchType: '${{ variables.effectiveOneBranchType }}' + - ${{ if ne(variables.effectiveBuildPackage, 'mssql-python-odbc') }}: + - ${{ each config in parameters.macosConfigs }}: + - template: /OneBranchPipelines/stages/build-macos-single-stage.yml@self + parameters: + stageName: MacOS_py${{ config.pyVer }} + jobName: BuildWheel + # Convert pyVer '310' → pythonVersion '3.10' + pythonVersion: ${{ format('{0}.{1}', substring(config.pyVer, 0, 1), substring(config.pyVer, 1, 2)) }} + shortPyVer: ${{ config.pyVer }} + oneBranchType: '${{ variables.effectiveOneBranchType }}' # ========================= # LINUX BUILD STAGES @@ -411,15 +474,16 @@ extends: # d. Runs pytest against SQL Server # 4. Publishes artifacts (all 5 wheels) # 5. Component Governance + AntiMalware scanning - - ${{ each config in parameters.linuxConfigs }}: - - template: /OneBranchPipelines/stages/build-linux-single-stage.yml@self - parameters: - stageName: Linux_${{ config.tag }}_${{ config.arch }} - jobName: BuildWheels - linuxTag: ${{ config.tag }} - arch: ${{ config.arch }} - dockerPlatform: ${{ config.platform }} - oneBranchType: '${{ variables.effectiveOneBranchType }}' + - ${{ if ne(variables.effectiveBuildPackage, 'mssql-python-odbc') }}: + - ${{ each config in parameters.linuxConfigs }}: + - template: /OneBranchPipelines/stages/build-linux-single-stage.yml@self + parameters: + stageName: Linux_${{ config.tag }}_${{ config.arch }} + jobName: BuildWheels + linuxTag: ${{ config.tag }} + arch: ${{ config.arch }} + dockerPlatform: ${{ config.platform }} + oneBranchType: '${{ variables.effectiveOneBranchType }}' # ========================= # CONSOLIDATE STAGE @@ -433,36 +497,149 @@ extends: # - dist/bindings/macOS/*.so (macOS universal2 binaries) # - dist/bindings/Linux/*.so (Linux native extensions) # This stage also runs final BinSkim scan on all binaries - - stage: Consolidate - displayName: 'Consolidate All Artifacts' - dependsOn: - # Windows dependencies (9 stages) - - Win_py310_x64 - - Win_py311_x64 - - Win_py312_x64 - - Win_py313_x64 - - Win_py314_x64 - - Win_py311_arm64 - - Win_py312_arm64 - - Win_py313_arm64 - - Win_py314_arm64 - # macOS dependencies (5 stages) - - MacOS_py310 - - MacOS_py311 - - MacOS_py312 - - MacOS_py313 - - MacOS_py314 - # Linux dependencies (4 stages) - - Linux_manylinux_2_28_x86_64 - - Linux_manylinux_2_28_aarch64 - - Linux_musllinux_x86_64 - - Linux_musllinux_aarch64 - jobs: - - template: /OneBranchPipelines/jobs/consolidate-artifacts-job.yml@self - parameters: - # CRITICAL: Use effectiveOneBranchType to ensure scheduled builds run as 'Official' - # Using parameters.oneBranchType would break scheduled builds (they'd run as 'NonOfficial') - oneBranchType: '${{ variables.effectiveOneBranchType }}' + - ${{ if ne(variables.effectiveBuildPackage, 'mssql-python-odbc') }}: + - stage: Consolidate + displayName: 'Consolidate All Artifacts' + dependsOn: + # Windows dependencies (9 stages) + - Win_py310_x64 + - Win_py311_x64 + - Win_py312_x64 + - Win_py313_x64 + - Win_py314_x64 + - Win_py311_arm64 + - Win_py312_arm64 + - Win_py313_arm64 + - Win_py314_arm64 + # macOS dependencies (5 stages) + - MacOS_py310 + - MacOS_py311 + - MacOS_py312 + - MacOS_py313 + - MacOS_py314 + # Linux dependencies (4 stages) + - Linux_manylinux_2_28_x86_64 + - Linux_manylinux_2_28_aarch64 + - Linux_musllinux_x86_64 + - Linux_musllinux_aarch64 + jobs: + - template: /OneBranchPipelines/jobs/consolidate-artifacts-job.yml@self + parameters: + # CRITICAL: Use effectiveOneBranchType to ensure scheduled builds run as 'Official' + # Using parameters.oneBranchType would break scheduled builds (they'd run as 'NonOfficial') + oneBranchType: '${{ variables.effectiveOneBranchType }}' # Note: Symbol publishing handled directly in Windows build stages # PDB files uploaded to Microsoft Symbol Server for debugging + + # ========================================================================= + # ODBC PACKAGE BUILD STAGES (mssql-python-odbc) + # ========================================================================= + # 7 data-only wheels (no native compile, no pytest) built from setup_odbc.py. + # They run in parallel with the mssql-python stages and are consolidated by a + # SEPARATE ConsolidateOdbc stage so the release pipeline can ship each package + # independently. SDL "just works": these wheels contain the SAME ODBC driver + # binaries already scanned in mssql_python/libs by the main build — no new + # first-party native code is introduced. + # + # ODBC Windows stages (2): ODBC_Win_x64, ODBC_Win_arm64 + # Gate: build mssql-python-odbc stages unless an mssql-python-only build was requested + - ${{ if ne(variables.effectiveBuildPackage, 'mssql-python') }}: + - ${{ each config in parameters.odbcWindowsConfigs }}: + - template: /OneBranchPipelines/stages/build-odbc-windows-stage.yml@self + parameters: + stageName: ODBC_Win_${{ config.arch }} + jobName: BuildWheel + architecture: ${{ config.arch }} + oneBranchType: '${{ variables.effectiveOneBranchType }}' + + # ODBC macOS stage (1): ODBC_MacOS_universal2 + - ${{ if ne(variables.effectiveBuildPackage, 'mssql-python') }}: + - ${{ each config in parameters.odbcMacosConfigs }}: + - template: /OneBranchPipelines/stages/build-odbc-macos-stage.yml@self + parameters: + stageName: ODBC_MacOS_${{ config.name }} + jobName: BuildWheel + oneBranchType: '${{ variables.effectiveOneBranchType }}' + + # ODBC Linux stages (4): manylinux_2_28 x86_64/aarch64, musllinux x86_64/aarch64 + - ${{ if ne(variables.effectiveBuildPackage, 'mssql-python') }}: + - ${{ each config in parameters.odbcLinuxConfigs }}: + - template: /OneBranchPipelines/stages/build-odbc-linux-stage.yml@self + parameters: + stageName: ODBC_Linux_${{ config.tag }}_${{ config.arch }} + jobName: BuildWheel + linuxTag: ${{ config.tag }} + arch: ${{ config.arch }} + dockerPlatform: ${{ config.platform }} + oneBranchType: '${{ variables.effectiveOneBranchType }}' + + # ========================================================================= + # CONSOLIDATE ODBC STAGE + # ========================================================================= + # Collects the 7 mssql-python-odbc wheels into a single dist/ folder and + # publishes them as `drop_ConsolidateOdbc_ConsolidateArtifacts` (distinct from + # the mssql-python `drop_Consolidate_ConsolidateArtifacts`). The release + # pipeline selects one artifact based on its `releasePackage` parameter. + - ${{ if ne(variables.effectiveBuildPackage, 'mssql-python') }}: + - stage: ConsolidateOdbc + displayName: 'Consolidate All ODBC Artifacts' + dependsOn: + - ODBC_Win_x64 + - ODBC_Win_arm64 + - ODBC_MacOS_universal2 + - ODBC_Linux_manylinux_2_28_x86_64 + - ODBC_Linux_manylinux_2_28_aarch64 + - ODBC_Linux_musllinux_x86_64 + - ODBC_Linux_musllinux_aarch64 + jobs: + - template: /OneBranchPipelines/jobs/consolidate-odbc-artifacts-job.yml@self + parameters: + oneBranchType: '${{ variables.effectiveOneBranchType }}' + expectedWheelCount: 7 + + # ========================================================================= + # WHEEL INSTALLATION TEST STAGE(S) + # ========================================================================= + # Post-consolidation validation of the package split from an end user's + # perspective: install the ACTUAL published mssql-python + mssql-python-odbc + # wheels together, remove the bundled ODBC driver so the external package is + # the ONLY driver source, then run the full pytest suite. Runs only when BOTH + # packages are built in the same run (buildPackage = 'both'), because it needs + # both consolidated artifacts (drop_Consolidate_* and drop_ConsolidateOdbc_*). + - ${{ if eq(variables.effectiveBuildPackage, 'both') }}: + - template: /OneBranchPipelines/stages/wheel-installation-test-stage.yml@self + parameters: + stageName: TestBothWheels_manylinux_2_28_x86_64 + jobName: InstallAndTest + linuxTag: manylinux_2_28 + arch: x86_64 + dockerPlatform: linux/amd64 + oneBranchType: '${{ variables.effectiveOneBranchType }}' + - template: /OneBranchPipelines/stages/wheel-installation-test-stage.yml@self + parameters: + stageName: TestBothWheels_musllinux_x86_64 + jobName: InstallAndTest + linuxTag: musllinux + arch: x86_64 + dockerPlatform: linux/amd64 + oneBranchType: '${{ variables.effectiveOneBranchType }}' + # Windows + macOS counterparts run natively (no Docker) on the same pools the + # build stages use. They prove the external-package resolution end-to-end on all + # three OSes - Windows additionally covers mssql-auth.dll resolution, macOS covers + # the universal2 dylib @loader_path/rpath resolution. + - template: /OneBranchPipelines/stages/wheel-installation-test-windows-stage.yml@self + parameters: + stageName: TestBothWheels_Windows_x64 + jobName: InstallAndTest + pythonVersion: '3.12' + shortPyVer: '312' + architecture: x64 + oneBranchType: '${{ variables.effectiveOneBranchType }}' + - template: /OneBranchPipelines/stages/wheel-installation-test-macos-stage.yml@self + parameters: + stageName: TestBothWheels_macOS_universal2 + jobName: InstallAndTest + pythonVersion: '3.12' + shortPyVer: '312' + oneBranchType: '${{ variables.effectiveOneBranchType }}' diff --git a/OneBranchPipelines/dummy-release-pipeline.yml b/OneBranchPipelines/dummy-release-pipeline.yml index f9e548fb0..ac0c38666 100644 --- a/OneBranchPipelines/dummy-release-pipeline.yml +++ b/OneBranchPipelines/dummy-release-pipeline.yml @@ -12,6 +12,16 @@ pr: none # Parameters for DUMMY release pipeline parameters: + # Which package to test-release. Both are produced by the same build pipeline + # (definition 2199); this switches the consolidated artifact and messaging. + - name: releasePackage + displayName: '[TEST] Package to Release' + type: string + values: + - 'mssql-python' + - 'mssql-python-odbc' + default: 'mssql-python' + - name: publishSymbols displayName: '[TEST] Publish Symbols to Symbol Servers' type: boolean @@ -32,6 +42,16 @@ variables: - group: 'ESRP Federated Creds (AME)' # Contains ESRP signing credentials - group: 'Symbols Publishing' # Contains SymbolServer, SymbolTokenUri variables + # Select which consolidated artifact to download based on the target package. + # Both are produced by the same build pipeline (definition 2199): + # mssql-python -> drop_Consolidate_ConsolidateArtifacts + # mssql-python-odbc -> drop_ConsolidateOdbc_ConsolidateArtifacts + - name: consolidatedArtifactName + ${{ if eq(parameters.releasePackage, 'mssql-python-odbc') }}: + value: 'drop_ConsolidateOdbc_ConsolidateArtifacts' + ${{ else }}: + value: 'drop_Consolidate_ConsolidateArtifacts' + # OneBranch resources resources: repositories: @@ -114,7 +134,7 @@ extends: definition: 2199 # Build-Release-Package-Pipeline definition ID buildVersionToDownload: 'specific' buildId: $(resources.pipeline.buildPipeline.runID) # Use the build run selected in UI - artifactName: 'drop_Consolidate_ConsolidateArtifacts' # Consolidated artifact with dist/ and symbols/ + artifactName: '${{ variables.consolidatedArtifactName }}' # mssql-python or mssql-python-odbc consolidated wheels targetPath: '$(Build.SourcesDirectory)/artifacts' # Step 2: List downloaded artifacts for verification @@ -175,13 +195,66 @@ extends: Write-Host "Symbols: $(if ($symbols) { $symbols.Count } else { 0 }) files" Write-Host "=====================================" - # Step 3: Validate mssql-py-core is a stable version (no dev/alpha/beta/rc) - - task: PowerShell@2 - displayName: '[TEST] Validate mssql-py-core is a stable version' - inputs: - targetType: 'filePath' - filePath: '$(Build.SourcesDirectory)\OneBranchPipelines\scripts\validate-release-versions.ps1' - arguments: '-VersionFile "$(Build.SourcesDirectory)\artifacts\dist\mssql-py-core.version"' + # Step 3: Validate mssql-py-core is a stable version (mssql-python only; + # mssql-python-odbc ships no mssql-py-core.version file) + - ${{ if eq(parameters.releasePackage, 'mssql-python') }}: + - task: PowerShell@2 + displayName: '[TEST] Validate mssql-py-core is a stable version' + inputs: + targetType: 'filePath' + filePath: '$(Build.SourcesDirectory)\OneBranchPipelines\scripts\validate-release-versions.ps1' + arguments: '-VersionFile "$(Build.SourcesDirectory)\artifacts\dist\mssql-py-core.version"' + + # Guard: if the released mssql-python wheel declares an + # mssql-python-odbc dependency, it MUST be the exact pinned version. + # A pre-pin build declares no such dependency and is allowed through + # (inert until setup.py adds install_requires), so this does not break + # an unrelated hotfix released before the pin lands. It DOES catch a + # wheel pinned to the wrong odbc version. Update $expectedVersion when + # the odbc pin bumps. + - task: PowerShell@2 + displayName: '[TEST] Validate mssql-python-odbc pin in wheel metadata' + inputs: + targetType: 'inline' + script: | + $ErrorActionPreference = 'Stop' + $expectedVersion = '18.6.2' + $wheels = Get-ChildItem -Path "$(Build.SourcesDirectory)/artifacts/dist" -Filter "mssql_python-*.whl" + if ($wheels.Count -eq 0) { + Write-Error "No mssql_python-*.whl found in artifacts/dist to validate the odbc pin." + exit 1 + } + Add-Type -AssemblyName System.IO.Compression.FileSystem + $wheel = $wheels[0].FullName + Write-Host "Inspecting wheel metadata: $wheel" + $zip = [System.IO.Compression.ZipFile]::OpenRead($wheel) + $metaEntry = $zip.Entries | Where-Object { $_.FullName -match '\.dist-info/METADATA$' } | Select-Object -First 1 + if (-not $metaEntry) { + $zip.Dispose() + Write-Error "METADATA not found inside $wheel" + exit 1 + } + $reader = New-Object System.IO.StreamReader($metaEntry.Open()) + $metadata = $reader.ReadToEnd() + $reader.Dispose() + $zip.Dispose() + $odbcLines = ($metadata -split "`n") | Where-Object { $_ -match 'Requires-Dist:\s*mssql[-_]python[-_]odbc' } + if (-not $odbcLines) { + Write-Warning "Wheel declares no mssql-python-odbc dependency (pre-pin build). Skipping odbc pin enforcement - expected until setup.py adds install_requires=mssql-python-odbc==$expectedVersion." + } else { + Write-Host "odbc dependency in metadata:" + $odbcLines | ForEach-Object { Write-Host " $($_.Trim())" } + # Anchor the version with a negative lookahead so 18.6.20 / + # 18.6.2.1 do NOT satisfy an 18.6.2 pin, while still allowing a + # trailing space, ';' environment marker, or end of line. + $pinRegex = 'mssql[-_]python[-_]odbc\s*==\s*' + [regex]::Escape($expectedVersion) + '(?![\d.])' + $pinned = $odbcLines | Where-Object { $_ -match $pinRegex } + if (-not $pinned) { + Write-Error "mssql-python wheel declares an mssql-python-odbc dependency but not the exact pin ==$expectedVersion. Release from the POST-pin build with the correct odbc version." + exit 1 + } + Write-Host "OK: wheel depends on mssql-python-odbc==$expectedVersion" + } # Step 4: Verify wheel integrity - task: PowerShell@2 @@ -217,8 +290,8 @@ extends: Write-Host "`nAll wheels verified successfully!" - # Step 5: Publish Symbols (if enabled and symbols exist) - - ${{ if eq(parameters.publishSymbols, true) }}: + # Step 5: Publish Symbols (if enabled and symbols exist; mssql-python only) + - ${{ if and(eq(parameters.publishSymbols, true), eq(parameters.releasePackage, 'mssql-python')) }}: - template: /OneBranchPipelines/steps/symbol-publishing-step.yml@self parameters: SymbolsFolder: '$(Build.SourcesDirectory)/symbols' @@ -233,13 +306,13 @@ extends: Write-Host "====================================" Write-Host "⚠️ TEST PIPELINE - DRY RUN MODE ⚠️" Write-Host "====================================" - Write-Host "Package: mssql-python (TEST)" + Write-Host "Package: ${{ parameters.releasePackage }} (TEST)" Write-Host "" Write-Host "Actions performed:" Write-Host "✓ Downloaded wheels from build pipeline" Write-Host "✓ Verified wheel integrity" Write-Host "✓ Downloaded symbols from build pipeline" - if ("${{ parameters.publishSymbols }}" -eq "True") { + if ("${{ parameters.publishSymbols }}" -eq "True" -and "${{ parameters.releasePackage }}" -eq "mssql-python") { Write-Host "✓ Published symbols to SqlClientDrivers org" } Write-Host "✗ ESRP dummy release NOT performed (parameter disabled)" @@ -280,7 +353,7 @@ extends: definition: 2199 buildVersionToDownload: 'specific' buildId: $(resources.pipeline.buildPipeline.runID) - artifactName: 'drop_Consolidate_ConsolidateArtifacts' + artifactName: '${{ variables.consolidatedArtifactName }}' targetPath: '$(Build.SourcesDirectory)/artifacts' # ⚠️ IMPORTANT: Uses Maven ContentType for testing - NOT PyPI! @@ -313,11 +386,11 @@ extends: Write-Host "====================================" Write-Host "⚠️ TEST PIPELINE - DUMMY RELEASE COMPLETED ⚠️" Write-Host "====================================" - Write-Host "Package: mssql-python (TEST)" + Write-Host "Package: ${{ parameters.releasePackage }} (TEST)" Write-Host "ContentType: Maven (NOT PyPI - Safe for Testing)" Write-Host "Owners: $(owner)" Write-Host "Approvers: $(approver)" - Write-Host "Symbols Published: ${{ parameters.publishSymbols }}" + Write-Host "Symbols Published: $(if ('${{ parameters.publishSymbols }}' -eq 'True' -and '${{ parameters.releasePackage }}' -eq 'mssql-python') { 'True' } else { 'False' })" Write-Host "=====================================" Write-Host "" Write-Host "⚠️ IMPORTANT: This was a DUMMY release using Maven ContentType" @@ -326,7 +399,7 @@ extends: Write-Host "What was tested:" Write-Host "✓ Artifact download from build pipeline" Write-Host "✓ Wheel integrity verification" - if ("${{ parameters.publishSymbols }}" -eq "True") { + if ("${{ parameters.publishSymbols }}" -eq "True" -and "${{ parameters.releasePackage }}" -eq "mssql-python") { Write-Host "✓ Symbol publishing to SqlClientDrivers org" } Write-Host "✓ ESRP release workflow (Maven ContentType)" diff --git a/OneBranchPipelines/jobs/consolidate-artifacts-job.yml b/OneBranchPipelines/jobs/consolidate-artifacts-job.yml index 478803aa3..40cda12b0 100644 --- a/OneBranchPipelines/jobs/consolidate-artifacts-job.yml +++ b/OneBranchPipelines/jobs/consolidate-artifacts-job.yml @@ -1,7 +1,7 @@ # Consolidate Artifacts Job Template # Downloads artifacts from all platform build stages and consolidates into single dist/ folder -# Works with individual stage artifacts (15 stages total: 7 Windows + 4 macOS + 4 Linux) -# Each Linux stage builds 4 Python versions, resulting in 27 total wheels +# Works with individual stage artifacts (18 stages total: 9 Windows + 5 macOS + 4 Linux) +# Each Linux stage builds 5 Python versions (3.10-3.14), resulting in 34 total wheels parameters: - name: oneBranchType type: string @@ -29,13 +29,20 @@ jobs: - checkout: self fetchDepth: 1 - # Download ALL artifacts from current build - # Matrix jobs publish as: Windows_, macOS_, Linux_ - # This downloads all of them automatically (27 total artifacts) + # Download only the mssql-python platform-stage artifacts from the current build. + # The build definition now ALSO produces mssql-python-odbc artifacts (drop_ODBC_*) + # in the SAME run, so we scope this download with itemPattern to the mssql-python + # stages (drop_Win_*, drop_MacOS_*, drop_Linux_*). Without this, the odbc wheels + # would be swept into this package's dist/ and shipped to the mssql-python PyPI + # project. The odbc wheels are consolidated separately (ConsolidateOdbc stage). - task: DownloadPipelineArtifact@2 displayName: 'Download All Platform Artifacts' inputs: buildType: 'current' + itemPattern: | + drop_Win_*/** + drop_MacOS_*/** + drop_Linux_*/** targetPath: '$(Pipeline.Workspace)/all-artifacts' # Consolidate all wheels into single dist/ directory @@ -68,12 +75,17 @@ jobs: echo "" WHEEL_COUNT=$(ls -1 $(ob_outputDirectory)/dist/*.whl 2>/dev/null | wc -l) echo "Total wheel count: $WHEEL_COUNT" - echo "Expected: 27 wheels (7 Windows + 4 macOS + 16 Linux)" + echo "Expected: 34 wheels (9 Windows + 5 macOS + 20 Linux)" - if [ "$WHEEL_COUNT" -ne 27 ]; then - echo "WARNING: Expected 27 wheels but found $WHEEL_COUNT" + # Hard-fail on a wheel-count mismatch (symmetric with the odbc consolidate job). + # A mismatch means a build stage silently produced a partial set (e.g. a missing + # Python version) or the build matrix changed without updating this count. + # Update the expected 34 whenever the Windows/macOS/Linux build matrix changes. + if [ "$WHEEL_COUNT" -ne 34 ]; then + echo "ERROR: Expected 34 wheels but found $WHEEL_COUNT" >&2 + exit 1 else - echo "SUCCESS: All 27 wheels consolidated!" + echo "SUCCESS: All 34 wheels consolidated!" fi displayName: 'Consolidate wheels from all platforms' diff --git a/OneBranchPipelines/jobs/consolidate-odbc-artifacts-job.yml b/OneBranchPipelines/jobs/consolidate-odbc-artifacts-job.yml new file mode 100644 index 000000000..f0d8a0715 --- /dev/null +++ b/OneBranchPipelines/jobs/consolidate-odbc-artifacts-job.yml @@ -0,0 +1,75 @@ +# Consolidate ODBC Artifacts Job Template +# Downloads the per-platform `mssql-python-odbc` wheels from all build stages and +# consolidates them into a single dist/ folder for the release pipeline to publish. +# Expected: 7 wheels (2 Windows + 1 macOS universal2 + 4 Linux). +parameters: + - name: oneBranchType + type: string + default: 'Official' + - name: expectedWheelCount + type: number + default: 7 + +jobs: + - job: ConsolidateArtifacts + displayName: 'Consolidate All ODBC Platform Artifacts' + condition: succeeded() + + pool: + type: linux + isCustom: true + name: Azure Pipelines + vmImage: 'ubuntu-latest' + + variables: + # Consolidation only moves files; no binaries to scan. + - name: ob_sdl_binskim_enabled + value: false + - name: ob_outputDirectory + value: '$(Build.ArtifactStagingDirectory)' + + steps: + - checkout: self + fetchDepth: 1 + + # Download only the mssql-python-odbc platform-stage artifacts (drop_ODBC_*) from + # the current build. The same run also builds the mssql-python wheels (drop_Win_*, + # drop_MacOS_*, drop_Linux_*); itemPattern scopes this download to the odbc stages + # so ONLY the 7 odbc wheels are consolidated here (not the mssql-python wheels). + - task: DownloadPipelineArtifact@2 + displayName: 'Download All ODBC Platform Artifacts' + inputs: + buildType: 'current' + itemPattern: 'drop_ODBC_*/**' + targetPath: '$(Pipeline.Workspace)/all-artifacts' + + - bash: | + set -e + mkdir -p $(ob_outputDirectory)/dist + + echo "Finding all .whl files..." + find $(Pipeline.Workspace)/all-artifacts -name "*.whl" -exec ls -lh {} \; + + echo "Copying all wheels to consolidated dist/..." + find $(Pipeline.Workspace)/all-artifacts -name "*.whl" -exec cp -v {} $(ob_outputDirectory)/dist/ \; + + echo "Consolidated wheels:" + ls -lh $(ob_outputDirectory)/dist/ + WHEEL_COUNT=$(ls -1 $(ob_outputDirectory)/dist/*.whl 2>/dev/null | wc -l) + echo "Total wheel count: $WHEEL_COUNT (expected ${{ parameters.expectedWheelCount }})" + if [ "$WHEEL_COUNT" -ne "${{ parameters.expectedWheelCount }}" ]; then + echo "ERROR: expected ${{ parameters.expectedWheelCount }} wheels but found $WHEEL_COUNT" >&2 + exit 1 + fi + echo "SUCCESS: all ${{ parameters.expectedWheelCount }} ODBC wheels consolidated." + displayName: 'Consolidate ODBC wheels' + + - task: PublishPipelineArtifact@1 + displayName: 'Publish Consolidated ODBC Artifacts' + inputs: + targetPath: '$(ob_outputDirectory)' + # Distinct name so it does not collide with the mssql-python consolidate + # artifact (drop_Consolidate_ConsolidateArtifacts) in the same build run. + # Matches the OneBranch auto-name for a stage named `ConsolidateOdbc`. + artifact: 'drop_ConsolidateOdbc_ConsolidateArtifacts' + publishLocation: 'pipeline' diff --git a/OneBranchPipelines/official-release-pipeline.yml b/OneBranchPipelines/official-release-pipeline.yml index a5ec227f6..43cc32d68 100644 --- a/OneBranchPipelines/official-release-pipeline.yml +++ b/OneBranchPipelines/official-release-pipeline.yml @@ -1,5 +1,7 @@ -# OneBranch Official Release Pipeline for mssql-python -# Downloads wheel and symbol artifacts from build pipeline, publishes symbols, and releases wheels to PyPI via ESRP +# OneBranch Official Release Pipeline (mssql-python and mssql-python-odbc) +# Selects the package via the `releasePackage` parameter, downloads that package's +# consolidated wheels (and symbols for mssql-python) from the build pipeline +# (definition 2199), publishes symbols, and releases wheels to PyPI via ESRP. # This pipeline is ALWAYS Official - no NonOfficial option name: $(Year:YY)$(DayOfYear)$(Rev:.r)-Release @@ -10,6 +12,16 @@ pr: none # Parameters for release pipeline parameters: + # Which package to release. Both are produced by the same build pipeline + # (definition 2199); this switches the consolidated artifact and messaging. + - name: releasePackage + displayName: 'Package to Release' + type: string + values: + - 'mssql-python' + - 'mssql-python-odbc' + default: 'mssql-python' + - name: publishSymbols displayName: 'Publish Symbols to Symbol Servers' type: boolean @@ -30,6 +42,16 @@ variables: - group: 'ESRP Federated Creds (AME)' # Contains ESRP signing credentials - group: 'Symbols Publishing' # Contains SymbolServer, SymbolTokenUri variables + # Select which consolidated artifact to download/publish based on the target + # package. Both are produced by the same build pipeline (definition 2199): + # mssql-python -> drop_Consolidate_ConsolidateArtifacts + # mssql-python-odbc -> drop_ConsolidateOdbc_ConsolidateArtifacts + - name: consolidatedArtifactName + ${{ if eq(parameters.releasePackage, 'mssql-python-odbc') }}: + value: 'drop_ConsolidateOdbc_ConsolidateArtifacts' + ${{ else }}: + value: 'drop_Consolidate_ConsolidateArtifacts' + # OneBranch resources resources: repositories: @@ -117,7 +139,7 @@ extends: definition: 2199 # Build-Release-Package-Pipeline definition ID buildVersionToDownload: 'specific' buildId: $(resources.pipeline.buildPipeline.runID) # Use the build run selected in UI - artifactName: 'drop_Consolidate_ConsolidateArtifacts' # Consolidated artifact with dist/ and symbols/ + artifactName: '${{ variables.consolidatedArtifactName }}' # mssql-python or mssql-python-odbc consolidated wheels targetPath: '$(Build.SourcesDirectory)/artifacts' # Step 2: List downloaded artifacts for verification @@ -173,13 +195,66 @@ extends: Write-Host "Symbols: $(if ($symbols) { $symbols.Count } else { 0 }) files" Write-Host "=====================================" - # Step 3: Validate mssql-py-core is a stable version (no dev/alpha/beta/rc) - - task: PowerShell@2 - displayName: 'Validate mssql-py-core is a stable version' - inputs: - targetType: 'filePath' - filePath: '$(Build.SourcesDirectory)\OneBranchPipelines\scripts\validate-release-versions.ps1' - arguments: '-VersionFile "$(Build.SourcesDirectory)\artifacts\dist\mssql-py-core.version"' + # Step 3: Validate mssql-py-core is a stable version (mssql-python only; + # mssql-python-odbc ships no mssql-py-core.version file) + - ${{ if eq(parameters.releasePackage, 'mssql-python') }}: + - task: PowerShell@2 + displayName: 'Validate mssql-py-core is a stable version' + inputs: + targetType: 'filePath' + filePath: '$(Build.SourcesDirectory)\OneBranchPipelines\scripts\validate-release-versions.ps1' + arguments: '-VersionFile "$(Build.SourcesDirectory)\artifacts\dist\mssql-py-core.version"' + + # Guard: if the released mssql-python wheel declares an + # mssql-python-odbc dependency, it MUST be the exact pinned version. + # A pre-pin build declares no such dependency and is allowed through + # (inert until setup.py adds install_requires), so this does not break + # an unrelated hotfix released before the pin lands. It DOES catch a + # wheel pinned to the wrong odbc version. Update $expectedVersion when + # the odbc pin bumps. + - task: PowerShell@2 + displayName: 'Validate mssql-python-odbc pin in wheel metadata' + inputs: + targetType: 'inline' + script: | + $ErrorActionPreference = 'Stop' + $expectedVersion = '18.6.2' + $wheels = Get-ChildItem -Path "$(Build.SourcesDirectory)/artifacts/dist" -Filter "mssql_python-*.whl" + if ($wheels.Count -eq 0) { + Write-Error "No mssql_python-*.whl found in artifacts/dist to validate the odbc pin." + exit 1 + } + Add-Type -AssemblyName System.IO.Compression.FileSystem + $wheel = $wheels[0].FullName + Write-Host "Inspecting wheel metadata: $wheel" + $zip = [System.IO.Compression.ZipFile]::OpenRead($wheel) + $metaEntry = $zip.Entries | Where-Object { $_.FullName -match '\.dist-info/METADATA$' } | Select-Object -First 1 + if (-not $metaEntry) { + $zip.Dispose() + Write-Error "METADATA not found inside $wheel" + exit 1 + } + $reader = New-Object System.IO.StreamReader($metaEntry.Open()) + $metadata = $reader.ReadToEnd() + $reader.Dispose() + $zip.Dispose() + $odbcLines = ($metadata -split "`n") | Where-Object { $_ -match 'Requires-Dist:\s*mssql[-_]python[-_]odbc' } + if (-not $odbcLines) { + Write-Warning "Wheel declares no mssql-python-odbc dependency (pre-pin build). Skipping odbc pin enforcement - expected until setup.py adds install_requires=mssql-python-odbc==$expectedVersion." + } else { + Write-Host "odbc dependency in metadata:" + $odbcLines | ForEach-Object { Write-Host " $($_.Trim())" } + # Anchor the version with a negative lookahead so 18.6.20 / + # 18.6.2.1 do NOT satisfy an 18.6.2 pin, while still allowing a + # trailing space, ';' environment marker, or end of line. + $pinRegex = 'mssql[-_]python[-_]odbc\s*==\s*' + [regex]::Escape($expectedVersion) + '(?![\d.])' + $pinned = $odbcLines | Where-Object { $_ -match $pinRegex } + if (-not $pinned) { + Write-Error "mssql-python wheel declares an mssql-python-odbc dependency but not the exact pin ==$expectedVersion. Release from the POST-pin build with the correct odbc version." + exit 1 + } + Write-Host "OK: wheel depends on mssql-python-odbc==$expectedVersion" + } # Step 4: Verify wheel integrity - task: PowerShell@2 @@ -215,8 +290,8 @@ extends: Write-Host "`nAll wheels verified successfully!" - # Step 5: Publish Symbols (if enabled and symbols exist) - - ${{ if eq(parameters.publishSymbols, true) }}: + # Step 5: Publish Symbols (mssql-python only; mssql-python-odbc has no PDBs) + - ${{ if and(eq(parameters.publishSymbols, true), eq(parameters.releasePackage, 'mssql-python')) }}: - template: /OneBranchPipelines/steps/symbol-publishing-step.yml@self parameters: SymbolsFolder: '$(Build.SourcesDirectory)/symbols' @@ -231,13 +306,13 @@ extends: Write-Host "====================================" Write-Host "DRY RUN MODE - No Release Performed" Write-Host "====================================" - Write-Host "Package: mssql-python" + Write-Host "Package: ${{ parameters.releasePackage }}" Write-Host "" Write-Host "Actions performed:" Write-Host "- Downloaded wheels from build pipeline" Write-Host "- Verified wheel integrity" Write-Host "- Downloaded symbols from build pipeline" - if ("${{ parameters.publishSymbols }}" -eq "True") { + if ("${{ parameters.publishSymbols }}" -eq "True" -and "${{ parameters.releasePackage }}" -eq "mssql-python") { Write-Host "- Published symbols to SqlClientDrivers org" } Write-Host "" @@ -273,7 +348,7 @@ extends: definition: 2199 buildVersionToDownload: 'specific' buildId: $(resources.pipeline.buildPipeline.runID) - artifactName: 'drop_Consolidate_ConsolidateArtifacts' + artifactName: '${{ variables.consolidatedArtifactName }}' targetPath: '$(Build.SourcesDirectory)/artifacts' - task: EsrpRelease@9 @@ -303,16 +378,17 @@ extends: Write-Host "====================================" Write-Host "ESRP Release Completed" Write-Host "====================================" - Write-Host "Package: mssql-python" + Write-Host "Package: ${{ parameters.releasePackage }}" Write-Host "Target: PyPI" Write-Host "Owners: $(owner)" Write-Host "Approvers: $(approver)" - Write-Host "Symbols Published: ${{ parameters.publishSymbols }}" + $symbolsPublished = ("${{ parameters.publishSymbols }}" -eq "True") -and ("${{ parameters.releasePackage }}" -eq "mssql-python") + Write-Host "Symbols Published: $symbolsPublished" Write-Host "=====================================" Write-Host "" Write-Host "Next steps:" Write-Host "1. Verify release in ESRP portal" Write-Host "2. Wait for approval workflow completion" - Write-Host "3. Verify package on PyPI: https://pypi.org/project/mssql-python/" + Write-Host "3. Verify package on PyPI: https://pypi.org/project/${{ parameters.releasePackage }}/" Write-Host "4. Verify symbols in SqlClientDrivers org (if published)" Write-Host "=====================================" diff --git a/OneBranchPipelines/stages/build-odbc-linux-stage.yml b/OneBranchPipelines/stages/build-odbc-linux-stage.yml new file mode 100644 index 000000000..51da9144b --- /dev/null +++ b/OneBranchPipelines/stages/build-odbc-linux-stage.yml @@ -0,0 +1,197 @@ +# ODBC Linux Single Configuration Stage Template +# Builds the platform-specific `mssql-python-odbc` wheel for ONE libc/arch combo +# (manylinux_2_28 or musllinux_1_2 x x86_64 or aarch64). +# +# Data-only package: no native compilation, no per-Python matrix, no pytest. +# A PyPA build container is still used so `platform.libc_ver()` reports the right +# libc and the wheel is tagged manylinux_2_28_* or musllinux_1_2_* correctly. +# aarch64 wheels are produced under QEMU emulation. Driver binaries come from the +# committed mssql_python/libs/ tree that is bind-mounted into the container. +parameters: + # Stage identifier (e.g., 'ODBC_Linux_manylinux_2_28_x86_64'). + - name: stageName + type: string + - name: jobName + type: string + default: 'BuildWheel' + # Distribution type: 'manylinux_2_28' (glibc 2.28+) or 'musllinux' (musl libc). + - name: linuxTag + type: string + # CPU architecture: 'x86_64' (AMD64) or 'aarch64' (ARM64). + - name: arch + type: string + # Docker platform for QEMU emulation: 'linux/amd64' or 'linux/arm64'. + - name: dockerPlatform + type: string + - name: oneBranchType + type: string + default: 'Official' + +stages: + - stage: ${{ parameters.stageName }} + displayName: 'ODBC Linux ${{ parameters.linuxTag }} ${{ parameters.arch }}' + jobs: + - job: ${{ parameters.jobName }} + displayName: 'Build ODBC Wheel - ${{ parameters.linuxTag }} ${{ parameters.arch }}' + # Reuse the same custom 1ES Linux pool as the mssql-python build. + pool: + type: linux + isCustom: true + name: Python-1ES-pool + demands: + - imageOverride -equals PYTHON-1ES-UB2404 + timeoutInMinutes: 60 + + variables: + # BinSkim needs ICU libs not present in manylinux/musllinux containers. + - name: ob_sdl_binskim_enabled + value: false + - name: ob_outputDirectory + value: '$(Build.ArtifactStagingDirectory)' + - name: LinuxContainerImage + value: 'onebranch.azurecr.io/linux/ubuntu-2204:latest' + - name: LINUX_TAG + value: ${{ parameters.linuxTag }} + - name: ARCH + value: ${{ parameters.arch }} + - name: DOCKER_PLATFORM + value: ${{ parameters.dockerPlatform }} + + steps: + - checkout: self + fetchDepth: 1 + + - bash: | + set -e + if ! docker info > /dev/null 2>&1; then + echo "Starting Docker daemon..." + sudo dockerd > docker.log 2>&1 & + sleep 10 + fi + docker --version + displayName: 'Setup and start Docker daemon' + + - script: | + sudo apt-get install -y qemu-user-static + displayName: 'Enable QEMU (for aarch64)' + + - script: | + rm -rf $(ob_outputDirectory)/wheels + mkdir -p $(ob_outputDirectory)/wheels + displayName: 'Prepare artifact directories' + + - task: AzureCLI@2 + displayName: 'Login to ACR (tdslibrs)' + inputs: + azureSubscription: 'Magnitude Test-mssql-rs-mssql-python' + scriptType: 'bash' + scriptLocation: 'inlineScript' + inlineScript: | + az acr login --name tdslibrs + + - script: | + set -euxo pipefail + # Same PyPA images as the mssql-python Linux build (mirrored into ACR). + if [[ "$(LINUX_TAG)" == "manylinux_2_28" ]]; then + IMAGE="tdslibrs.azurecr.io/import/python-build/manylinux_2_28_$(ARCH):latest" + elif [[ "$(LINUX_TAG)" == "musllinux" ]]; then + IMAGE="tdslibrs.azurecr.io/import/python-build/musllinux_1_2_$(ARCH):latest" + else + echo "ERROR: Unsupported LINUX_TAG '$(LINUX_TAG)'. Expected: manylinux_2_28 or musllinux" >&2 + exit 1 + fi + docker run -d --name build-$(LINUX_TAG)-$(ARCH) \ + --platform $(DOCKER_PLATFORM) \ + -v $(Build.SourcesDirectory):/workspace \ + -w /workspace \ + $IMAGE tail -f /dev/null + displayName: 'Start $(LINUX_TAG) $(ARCH) container' + + # Build the single py3-none wheel inside the container. Any CPython works; no + # compilation runs. setup_odbc.py's get_platform_info() auto-detects musl via + # platform.libc_ver(): on musllinux images it ALWAYS emits musllinux_1_2_ + # and ignores MANYLINUX_TAG. MANYLINUX_TAG is only consulted on glibc images, so + # hardcoding "manylinux_2_28" below is correct for both image families. + - script: | + set -euxo pipefail + if [[ "$(LINUX_TAG)" == "manylinux_2_28" ]]; then SHELL_EXE=bash; else SHELL_EXE=sh; fi + docker exec \ + -e targetArch="$(ARCH)" \ + -e MANYLINUX_TAG="manylinux_2_28" \ + build-$(LINUX_TAG)-$(ARCH) $SHELL_EXE -lc ' + set -eux + # Pick any available CPython from the PyPA image. + PY=$(ls -d /opt/python/cp312-cp312/bin/python 2>/dev/null || ls -d /opt/python/cp3*/bin/python | head -1) + echo "Using: $($PY --version)" + $PY -m pip install -q -U pip setuptools wheel twine + cd /workspace + $PY setup_odbc.py bdist_wheel + $PY -m twine check /workspace/dist/*.whl + echo "Produced wheels:"; ls -lh /workspace/dist/*.whl + ' + displayName: 'Build ODBC wheel in container' + + - script: | + set -euxo pipefail + cp -v $(Build.SourcesDirectory)/dist/*.whl $(ob_outputDirectory)/wheels/ + displayName: 'Stage wheel artifact' + + # Guard against the .gitignore '*.so' trap or a silent sync_libs() skip + # shipping a data wheel with NO driver binary (twine check would still pass). + # The driver filename contains 'msodbcsql' on every platform. + - script: | + set -eu + wheel_dir="$(ob_outputDirectory)/wheels" + matches=$(ls "$wheel_dir"/*.whl 2>/dev/null || true) + if [ -z "$matches" ]; then + echo "ERROR: no wheels found to verify in $wheel_dir" >&2 + exit 1 + fi + for whl in $matches; do + echo "Verifying ODBC driver binary present in: $whl" + entries=$(python3 -m zipfile -l "$whl") + if ! echo "$entries" | grep -qi 'msodbcsql'; then + echo "ERROR: ODBC driver binary (msodbcsql*) missing from $whl" >&2 + exit 1 + fi + # The msodbcsql driver loads the ODBC installer lib at runtime; assert + # the companion is packaged too, not just the driver itself. + if ! echo "$entries" | grep -qi 'libodbcinst'; then + echo "ERROR: ODBC installer lib (libodbcinst*) missing from $whl" >&2 + exit 1 + fi + echo "OK: $whl contains the ODBC driver binary and libodbcinst" + done + displayName: 'Assert wheel contains ODBC driver binary' + + - script: | + docker rm -f build-$(LINUX_TAG)-$(ARCH) || true + displayName: 'Stop and remove build container' + condition: always() + + - task: PublishPipelineArtifact@1 + displayName: 'Publish ODBC Linux Artifact' + inputs: + targetPath: '$(ob_outputDirectory)' + artifact: 'drop_${{ parameters.stageName }}_${{ parameters.jobName }}' + publishLocation: 'pipeline' + + - template: /OneBranchPipelines/steps/malware-scanning-step.yml@self + parameters: + scanPath: '$(ob_outputDirectory)' + artifactType: 'pkg' + + - ${{ if eq(parameters.oneBranchType, 'Official') }}: + - task: EsrpMalwareScanning@5 + displayName: 'ESRP MalwareScanning - ODBC Wheel (Official)' + inputs: + ConnectedServiceName: '$(SigningEsrpConnectedServiceName)' + AppRegistrationClientId: '$(SigningAppRegistrationClientId)' + AppRegistrationTenantId: '$(SigningAppRegistrationTenantId)' + EsrpClientId: '$(SigningEsrpClientId)' + UseMSIAuthentication: true + FolderPath: '$(ob_outputDirectory)/wheels' + Pattern: '*.whl' + SessionTimeout: 60 + CleanupTempStorage: 1 + VerboseLogin: 1 diff --git a/OneBranchPipelines/stages/build-odbc-macos-stage.yml b/OneBranchPipelines/stages/build-odbc-macos-stage.yml new file mode 100644 index 000000000..7378bc868 --- /dev/null +++ b/OneBranchPipelines/stages/build-odbc-macos-stage.yml @@ -0,0 +1,134 @@ +# ODBC macOS Single Configuration Stage Template +# Builds the universal2 `mssql-python-odbc` wheel (arm64 + x86_64 driver binaries in +# one wheel). +# +# Data-only package: no native compilation, no per-Python matrix, no pytest. +# `setup_odbc.py` returns the macosx_15_0_universal2 platform tag and copies both +# macOS arch subtrees from mssql_python/libs, so a single build serves both slices. +parameters: + # Stage identifier (e.g., 'ODBC_MacOS_universal2'). + - name: stageName + type: string + - name: jobName + type: string + default: 'BuildWheel' + - name: oneBranchType + type: string + default: 'Official' + - name: pythonVersion + type: string + default: '3.12' + +stages: + - stage: ${{ parameters.stageName }} + displayName: 'ODBC macOS Universal2' + jobs: + - job: ${{ parameters.jobName }} + displayName: 'Build ODBC Wheel - macOS Universal2' + # macOS pools declare as 'linux' type (Azure Pipelines quirk). + pool: + type: linux + isCustom: true + name: Azure Pipelines + vmImage: 'macOS-14' + timeoutInMinutes: 60 + + variables: + # BinSkim targets PE binaries; macOS uses Mach-O, so disable it here. + - name: ob_sdl_binskim_enabled + value: false + - name: ob_outputDirectory + value: '$(Build.ArtifactStagingDirectory)' + # OneBranch-required variable (unused on macOS stages). + - name: LinuxContainerImage + value: 'onebranch.azurecr.io/linux/ubuntu-2204:latest' + + steps: + - checkout: self + fetchDepth: 1 + + - task: UsePythonVersion@0 + inputs: + versionSpec: '${{ parameters.pythonVersion }}' + addToPath: true + displayName: 'Use Python ${{ parameters.pythonVersion }}' + + - script: | + set -e + python -m pip install --upgrade pip + python -m pip install setuptools wheel build twine + displayName: 'Install build tooling' + + # get_platform_info() forces the macosx_15_0_universal2 tag; sync_libs() copies + # both arm64 + x86_64 driver subtrees, so no post-build retag is required. + - script: | + set -e + python setup_odbc.py bdist_wheel + echo "Produced wheels:"; ls -lh dist/*.whl + displayName: 'Build ODBC wheel (universal2)' + + - script: | + set -e + python -m twine check dist/*.whl + displayName: 'twine check wheel' + + - task: CopyFiles@2 + inputs: + SourceFolder: '$(Build.SourcesDirectory)/dist' + Contents: '*.whl' + TargetFolder: '$(ob_outputDirectory)/wheels' + displayName: 'Stage wheel artifact' + + # Guard against a data wheel that packaged no driver binary (twine check + # would still pass). The driver filename contains 'msodbcsql' on every platform. + - script: | + set -eu + wheel_dir="$(ob_outputDirectory)/wheels" + matches=$(ls "$wheel_dir"/*.whl 2>/dev/null || true) + if [ -z "$matches" ]; then + echo "ERROR: no wheels found to verify in $wheel_dir" >&2 + exit 1 + fi + for whl in $matches; do + echo "Verifying ODBC driver binary present in: $whl" + entries=$(python -m zipfile -l "$whl") + if ! echo "$entries" | grep -qi 'msodbcsql'; then + echo "ERROR: ODBC driver binary (msodbcsql*) missing from $whl" >&2 + exit 1 + fi + # The msodbcsql driver loads the ODBC installer lib at runtime; assert + # the companion is packaged too, not just the driver itself. + if ! echo "$entries" | grep -qi 'libodbcinst'; then + echo "ERROR: ODBC installer lib (libodbcinst*) missing from $whl" >&2 + exit 1 + fi + echo "OK: $whl contains the ODBC driver binary and libodbcinst" + done + displayName: 'Assert wheel contains ODBC driver binary' + + - task: PublishPipelineArtifact@1 + displayName: 'Publish ODBC macOS Artifact' + inputs: + targetPath: '$(ob_outputDirectory)' + artifact: 'drop_${{ parameters.stageName }}_${{ parameters.jobName }}' + publishLocation: 'pipeline' + + - template: /OneBranchPipelines/steps/malware-scanning-step.yml@self + parameters: + scanPath: '$(ob_outputDirectory)' + artifactType: 'pkg' + + - ${{ if eq(parameters.oneBranchType, 'Official') }}: + - task: EsrpMalwareScanning@5 + displayName: 'ESRP MalwareScanning - ODBC Wheel (Official)' + inputs: + ConnectedServiceName: '$(SigningEsrpConnectedServiceName)' + AppRegistrationClientId: '$(SigningAppRegistrationClientId)' + AppRegistrationTenantId: '$(SigningAppRegistrationTenantId)' + EsrpClientId: '$(SigningEsrpClientId)' + UseMSIAuthentication: true + FolderPath: '$(ob_outputDirectory)/wheels' + Pattern: '*.whl' + SessionTimeout: 60 + CleanupTempStorage: 1 + VerboseLogin: 1 diff --git a/OneBranchPipelines/stages/build-odbc-windows-stage.yml b/OneBranchPipelines/stages/build-odbc-windows-stage.yml new file mode 100644 index 000000000..3fc9903b8 --- /dev/null +++ b/OneBranchPipelines/stages/build-odbc-windows-stage.yml @@ -0,0 +1,153 @@ +# ODBC Windows Single Configuration Stage Template +# Builds the platform-specific `mssql-python-odbc` wheel for ONE Windows architecture. +# +# Unlike the mssql-python build, this package ships ONLY pre-built ODBC driver +# binaries (data) — there is NO compiled Python extension. A single +# `py3-none-` wheel therefore serves every supported Python version +# (3.10+), so this stage does NOT use a per-Python-version matrix and does NOT +# run pytest against SQL Server. The driver binaries are already committed under +# `mssql_python/libs/`, so `checkout: self` is all that is needed; `setup_odbc.py` +# packages the current platform's subtree. +parameters: + # Stage identifier (e.g., 'ODBC_Win_x64'). + - name: stageName + type: string + # Job identifier within the stage. + - name: jobName + type: string + default: 'BuildWheel' + # Target architecture: 'x64' (AMD64) or 'arm64' (ARM64). + - name: architecture + type: string + # OneBranch build type: 'Official' (production) or 'NonOfficial' (dev/test). + - name: oneBranchType + type: string + default: 'Official' + # Any supported interpreter can produce the Python-agnostic wheel. + - name: pythonVersion + type: string + default: '3.12' + +stages: + - stage: ${{ parameters.stageName }} + displayName: 'ODBC Windows ${{ parameters.architecture }}' + jobs: + - job: ${{ parameters.jobName }} + displayName: 'Build ODBC Wheel - Windows ${{ parameters.architecture }}' + # Reuse the same custom 1ES Windows pool as the mssql-python build. + pool: + type: windows + isCustom: true + name: Python-1ES-pool + demands: + - imageOverride -equals PYTHON-1ES-MMS2022 + timeoutInMinutes: 60 + + variables: + # OneBranch output directory for artifacts. + ob_outputDirectory: '$(Build.ArtifactStagingDirectory)' + # OneBranch-required variable (unused on Windows stages). + LinuxContainerImage: 'onebranch.azurecr.io/linux/ubuntu-2204:latest' + + steps: + # Driver binaries live in mssql_python/libs (committed); shallow checkout is enough. + - checkout: self + fetchDepth: 1 + + # The ARM64 wheel is pure data, so it can be produced on an x64 host — there is + # no native compilation and hence no ARM64 python.lib cross-compile machinery. + - task: UsePythonVersion@0 + inputs: + versionSpec: '${{ parameters.pythonVersion }}' + architecture: 'x64' + addToPath: true + displayName: 'Use Python ${{ parameters.pythonVersion }}' + + - powershell: | + $ErrorActionPreference = "Stop" + python -m pip install --upgrade pip + python -m pip install setuptools wheel build twine + displayName: 'Install build tooling' + + # Build the platform-specific, Python-agnostic wheel. ARCHITECTURE controls the + # platform tag (win_amd64 / win_arm64). No native compilation happens here — + # setup_odbc.py only packages the driver binaries from mssql_python/libs. + - powershell: | + $ErrorActionPreference = "Stop" + $env:ARCHITECTURE = "${{ parameters.architecture }}" + python setup_odbc.py bdist_wheel + Write-Host "Produced wheels:" + Get-ChildItem dist\*.whl | ForEach-Object { Write-Host " - $($_.Name)" } + displayName: 'Build ODBC wheel (ARCHITECTURE=${{ parameters.architecture }})' + + - powershell: | + $ErrorActionPreference = "Stop" + python -m twine check dist\*.whl + displayName: 'twine check wheel' + + - task: CopyFiles@2 + inputs: + SourceFolder: '$(Build.SourcesDirectory)\dist' + Contents: '*.whl' + TargetFolder: '$(ob_outputDirectory)\wheels' + displayName: 'Stage wheel artifact' + + # Guard against a data wheel that packaged an incomplete driver set (twine + # check would still pass). Assert BOTH the driver DLL (msodbcsql*) and the + # Windows auth component (mssql-auth.dll): the native loader treats the + # external package as COMPLETE on Windows only when mssql-auth.dll is + # co-located with the driver - without it the loader silently falls back to + # bundled libs (or fails), which the Linux-only install test cannot catch. + - powershell: | + $ErrorActionPreference = "Stop" + $wheels = Get-ChildItem -Path "$(ob_outputDirectory)\wheels" -Filter "*.whl" + if ($wheels.Count -eq 0) { + Write-Error "No wheels found to verify in $(ob_outputDirectory)\wheels" + exit 1 + } + Add-Type -AssemblyName System.IO.Compression.FileSystem + foreach ($whl in $wheels) { + $zip = [System.IO.Compression.ZipFile]::OpenRead($whl.FullName) + $names = $zip.Entries | ForEach-Object { $_.FullName } + $zip.Dispose() + if (-not ($names | Where-Object { $_ -match 'msodbcsql' })) { + Write-Error "ODBC driver binary (msodbcsql*) missing from $($whl.Name)" + exit 1 + } + if (-not ($names | Where-Object { $_ -match 'mssql-auth\.dll' })) { + Write-Error "Windows auth component (mssql-auth.dll) missing from $($whl.Name) - loader would not treat the external package as complete" + exit 1 + } + Write-Host "OK: $($whl.Name) contains the ODBC driver binary and mssql-auth.dll" + } + displayName: 'Assert wheel contains ODBC driver binary' + + # OneBranch requires artifact naming: drop__. + - task: PublishPipelineArtifact@1 + displayName: 'Publish ODBC Windows Artifact' + inputs: + targetPath: '$(ob_outputDirectory)' + artifact: 'drop_${{ parameters.stageName }}_${{ parameters.jobName }}' + publishLocation: 'pipeline' + + # Component Governance + OneBranch AntiMalware notification. + - template: /OneBranchPipelines/steps/malware-scanning-step.yml@self + parameters: + scanPath: '$(ob_outputDirectory)' + artifactType: 'pkg' + + # Scan the redistributed driver binaries + wheel for malware (Official only). + - ${{ if eq(parameters.oneBranchType, 'Official') }}: + - task: EsrpMalwareScanning@5 + displayName: 'ESRP MalwareScanning - ODBC Wheel (Official)' + inputs: + ConnectedServiceName: '$(SigningEsrpConnectedServiceName)' + AppRegistrationClientId: '$(SigningAppRegistrationClientId)' + AppRegistrationTenantId: '$(SigningAppRegistrationTenantId)' + EsrpClientId: '$(SigningEsrpClientId)' + UseMSIAuthentication: true + FolderPath: '$(ob_outputDirectory)/wheels' + Pattern: '*.whl' + SessionTimeout: 60 + CleanupTempStorage: 1 + VerboseLogin: 1 diff --git a/OneBranchPipelines/stages/wheel-installation-test-macos-stage.yml b/OneBranchPipelines/stages/wheel-installation-test-macos-stage.yml new file mode 100644 index 000000000..0c324df23 --- /dev/null +++ b/OneBranchPipelines/stages/wheel-installation-test-macos-stage.yml @@ -0,0 +1,211 @@ +# ========================================================================================= +# End-User "Both Wheels Together" Installation Test - macOS +# ========================================================================================= +# macOS counterpart of wheel-installation-test-stage.yml (which is Docker/Linux-only). +# Installs the ACTUAL published mssql-python (universal2) + mssql-python-odbc (universal2) +# wheels into a fresh venv, runs the full pytest suite (bundled libs intact, external +# package already preferred by GetOdbcLibsBaseDir), then DELETES the bundled +# mssql_python/libs and connects once more in a fresh process. With the bundled fallback +# gone, a successful live query can ONLY be using the driver from the separately installed +# mssql_python_odbc package. This also exercises the universal2 dylib @loader_path/rpath +# resolution from the external package. +# +# Runs natively on a macOS-15 + Colima/Docker SQL Server agent. macOS 15 is required +# because the universal2 wheels are retagged to a macosx_15_0 minimum deployment target. +# A single representative Python version is sufficient: the external package RESOLUTION +# path (GetOdbcLibsBaseDir in ddbc_bindings.cpp) is Python-version independent; per-version +# native builds are already covered by the macOS build stages. +# ========================================================================================= +parameters: + # Stage identifier (e.g., 'TestBothWheels_macOS_universal2') + - name: stageName + type: string + # Job identifier within the stage + - name: jobName + type: string + default: 'InstallAndTest' + # Python version in X.Y format used to select the mssql-python wheel + - name: pythonVersion + type: string + default: '3.12' + # Python version as 3-digit string (e.g., '312') matching the cp wheel tag + - name: shortPyVer + type: string + default: '312' + # OneBranch build type: 'Official' or 'NonOfficial' + - name: oneBranchType + type: string + default: 'Official' + +stages: + - stage: ${{ parameters.stageName }} + displayName: 'Test Both Wheels macOS universal2' + # Only meaningful when both consolidated artifacts exist in this run. + dependsOn: + - Consolidate + - ConsolidateOdbc + jobs: + - job: ${{ parameters.jobName }} + displayName: 'Install both wheels + pytest - macOS universal2' + + # macOS-15 (Sequoia). type:linux is an Azure Pipelines quirk - macOS pools + # declare as 'linux' type. The build stage retags the universal2 wheels to + # macosx_15_0 (min deployment target), so pip only accepts them on macOS 15+. + # The build stage itself runs on macOS-14 because it tests the freshly built + # .so directly (never pip-installs the retagged wheel); this install test does + # pip-install it, so it MUST run on macOS 15+ or pip rejects the wheel with + # "not a supported wheel on this platform". + pool: + type: linux + isCustom: true + name: Azure Pipelines + vmImage: 'macOS-15' + timeoutInMinutes: 120 + + variables: + # BinSkim is Windows-PE focused; macOS uses Mach-O. + - name: ob_sdl_binskim_enabled + value: false + - name: ob_outputDirectory + value: '$(Build.ArtifactStagingDirectory)' + # OneBranch-required variable (unused on macOS agents). + - name: LinuxContainerImage + value: 'onebranch.azurecr.io/linux/ubuntu-2204:latest' + + steps: + - checkout: self + fetchDepth: 1 + + - task: UsePythonVersion@0 + inputs: + versionSpec: '${{ parameters.pythonVersion }}' + addToPath: true + displayName: 'Use Python ${{ parameters.pythonVersion }}' + + # Download the two CONSOLIDATED wheel artifacts produced earlier in this run. + - task: DownloadPipelineArtifact@2 + displayName: 'Download consolidated mssql-python wheels' + inputs: + buildType: 'current' + artifactName: 'drop_Consolidate_ConsolidateArtifacts' + targetPath: '$(Pipeline.Workspace)/mssql-python-wheels' + + - task: DownloadPipelineArtifact@2 + displayName: 'Download consolidated mssql-python-odbc wheels' + inputs: + buildType: 'current' + artifactName: 'drop_ConsolidateOdbc_ConsolidateArtifacts' + targetPath: '$(Pipeline.Workspace)/mssql-python-odbc-wheels' + + # Install Docker CLI + Colima (macOS Docker runtime) and start a SQL Server + # 2022 container on the host loopback (localhost:1433), same as the build stage. + - script: | + brew update + brew install docker colima + colima start --vm-type vz --cpu 4 --memory 8 || { + echo "vz VM failed, trying qemu..." + colima start --vm-type qemu --cpu 4 --memory 8 + } + sleep 30 + docker context use colima >/dev/null || true + docker version + displayName: 'Install and start Docker (Colima)' + timeoutInMinutes: 15 + + - script: | + set -euxo pipefail + docker pull mcr.microsoft.com/mssql/server:2022-latest + docker run --name sqlserver \ + -e ACCEPT_EULA=Y \ + -e MSSQL_SA_PASSWORD="${DB_PASSWORD}" \ + -p 1433:1433 -d \ + mcr.microsoft.com/mssql/server:2022-latest + echo "Waiting for SQL Server to be ready..." + for i in {1..30}; do + # sqlcmd runs INSIDE the mssql container; localhost is the container's own + # loopback (a readiness probe), not debug code reaching a dev box. + if docker exec sqlserver /opt/mssql-tools18/bin/sqlcmd \ + -S localhost -U SA -P "$DB_PASSWORD" -C -Q "SELECT 1" >/dev/null 2>&1; then # DevSkim: ignore DS162092 + echo "SQL Server is ready"; break + fi + sleep 2 + done + displayName: 'Start SQL Server (Docker)' + env: + DB_PASSWORD: $(DB_PASSWORD) + + - script: | + set -euo pipefail + + PYTAG="cp${{ parameters.shortPyVer }}" + + # Select the universal2 wheels matching THIS Python version (mssql-python) + # and the data-only odbc package (py3-none-macosx-universal2). + MP_WHEEL=$(ls "$(Pipeline.Workspace)"/mssql-python-wheels/dist/mssql_python-*${PYTAG}-*macosx*universal2*.whl 2>/dev/null | head -1) + if [ -z "$MP_WHEEL" ]; then echo "ERROR: no mssql-python wheel for ${PYTAG}/macosx universal2"; exit 1; fi + ODBC_WHEEL=$(ls "$(Pipeline.Workspace)"/mssql-python-odbc-wheels/dist/mssql_python_odbc-*macosx*universal2*.whl 2>/dev/null | head -1) + if [ -z "$ODBC_WHEEL" ]; then echo "ERROR: no mssql-python-odbc wheel for macosx universal2"; exit 1; fi + + # Fresh isolated venv = clean end-user machine. + TEST_DIR="$(Agent.TempDirectory)/enduser_${PYTAG}" + rm -rf "$TEST_DIR" + python3 -m venv "$TEST_DIR" + VENV_PY="$TEST_DIR/bin/python" + + # PYTHONSAFEPATH so the repo checkout can never shadow the installed wheel. + export PYTHONSAFEPATH=1 + + "$VENV_PY" -m pip install -q -U pip + + # Install the odbc wheel FIRST so mssql-python's pinned mssql-python-odbc==X + # requirement is satisfied from this LOCAL wheel (pip never reaches PyPI). + echo "Installing mssql-python-odbc: $ODBC_WHEEL" + "$VENV_PY" -m pip install -q "$ODBC_WHEEL" + echo "Installing mssql-python: $MP_WHEEL" + "$VENV_PY" -m pip install -q "$MP_WHEEL" + + # Show where the external ODBC package lives (the split target). + "$VENV_PY" -c "import mssql_python_odbc, os; print('mssql_python_odbc libs:', mssql_python_odbc.get_libs_dir()); assert os.path.isdir(mssql_python_odbc.get_libs_dir())" + + # POSITIVE assertion: prove the native loader WILL resolve the driver from + # the external mssql_python_odbc package (mirrors GetOdbcLibsBaseDir). + "$VENV_PY" -c "import mssql_python.ddbc_bindings as d, mssql_python_odbc, os; base=os.path.dirname(mssql_python_odbc.__file__); drv=d.GetDriverPathCpp(base); print('external driver path:', drv); assert os.path.exists(drv), 'external ODBC driver not found - loader would NOT select external package: '+drv; print('POSITIVE: loader will resolve the driver from the external mssql_python_odbc package')" + + # PHASE 1: run the FULL existing suite with bundled libs INTACT. Copy the + # suite OUT of the source tree so collection imports the INSTALLED package. + cp -r "$(Build.SourcesDirectory)/tests" "$TEST_DIR/tests" + cp "$(Build.SourcesDirectory)/pytest.ini" "$TEST_DIR/pytest.ini" + "$VENV_PY" -m pip install -q pytest + if [ -f "$(Build.SourcesDirectory)/requirements.txt" ]; then + "$VENV_PY" -m pip install -q -r "$(Build.SourcesDirectory)/requirements.txt" + fi + cd "$TEST_DIR" + # 127.0.0.1 targets the CI SQL Server 2022 container on the agent loopback (a + # test fixture), not debug code reaching a real host. + export DB_CONNECTION_STRING="Server=tcp:127.0.0.1,1433;Database=master;Uid=SA;Pwd=$DB_PASSWORD;TrustServerCertificate=yes" # DevSkim: ignore DS162092 + "$VENV_PY" -m pytest "$TEST_DIR/tests" -v --maxfail=1 -m "not stress" + echo "Full suite passed for ${PYTAG} with both wheels installed (external driver preferred)" + + # PHASE 2 - DECISIVE negative proof: remove the ODBC driver that + # mssql_python still bundles, then connect again in a FRESH process. + MP_LIBS=$("$VENV_PY" -c "import mssql_python, os; print(os.path.join(os.path.dirname(mssql_python.__file__), 'libs'))") + case "$MP_LIBS" in + "$(Build.SourcesDirectory)"/*) + echo "ERROR: resolved mssql_python to the source tree ($MP_LIBS), not the installed wheel"; exit 1 ;; + esac + if [ -d "$MP_LIBS" ]; then + echo "Removing bundled mssql_python/libs to force use of external ODBC package: $MP_LIBS" + rm -rf "$MP_LIBS" + else + echo "Note: mssql_python has no bundled libs/ (already split at build time)" + fi + + "$VENV_PY" -c "import mssql_python; print('mssql_python', mssql_python.__version__, 'imported OK (bundled libs removed)')" + # 127.0.0.1 targets the CI SQL Server 2022 container on the agent loopback (a + # test fixture), not debug code reaching a real host. + export DB_CONNECTION_STRING="Server=tcp:127.0.0.1,1433;Database=master;Uid=SA;Pwd=$DB_PASSWORD;TrustServerCertificate=yes" # DevSkim: ignore DS162092 + "$VENV_PY" -c "import mssql_python, os; c=mssql_python.connect(os.environ['DB_CONNECTION_STRING']); cur=c.cursor(); cur.execute('SELECT 1'); print('connection OK via EXTERNAL driver:', cur.fetchone()); c.close()" + echo "DECISIVE: connected with bundled libs removed - driver resolved from external mssql_python_odbc package" + displayName: 'Install both wheels + full pytest + external-only proof' + env: + DB_PASSWORD: $(DB_PASSWORD) diff --git a/OneBranchPipelines/stages/wheel-installation-test-stage.yml b/OneBranchPipelines/stages/wheel-installation-test-stage.yml new file mode 100644 index 000000000..1fe0c1a62 --- /dev/null +++ b/OneBranchPipelines/stages/wheel-installation-test-stage.yml @@ -0,0 +1,342 @@ +# ========================================================================================= +# End-User "Both Wheels Together" Installation Test Stage +# ========================================================================================= +# Validates the mssql-python / mssql-python-odbc PACKAGE SPLIT from an end-user's +# perspective, using the ACTUAL published wheel artifacts (not raw build binaries): +# +# pip install mssql_python---.whl +# pip install mssql_python_odbc--py3-none-.whl +# +# The proof is both POSITIVE and NEGATIVE: +# * POSITIVE: before touching anything we assert that the native loader WILL pick +# the external package - GetDriverPathCpp(dir(mssql_python_odbc)) points at an +# existing driver file, which is exactly the condition GetOdbcLibsBaseDir uses +# to treat the external package as authoritative. +# * NEGATIVE: we then DELETE the ODBC driver binaries that mssql-python still +# bundles under `mssql_python/libs/`. With the bundled fallback removed, a +# successful connection + passing test suite proves the driver was resolved +# from the separately-installed `mssql_python_odbc` package. +# (see GetOdbcLibsBaseDir in ddbc_bindings.cpp, which prefers the external package +# and falls back to bundled libs). Without the deletion the bundled fallback would +# mask a broken split and the test would pass even if the ODBC package were ignored. +# +# Scope: Linux manylinux_2_28 + musllinux (x86_64), all supported Python versions. +# This stage tests the INSTALL/RESOLUTION path, not per-arch compilation (already +# covered by the build stages). It runs only when BOTH packages are built in the +# same run (buildPackage = 'both'), because it needs both consolidated artifacts. +# ========================================================================================= +parameters: + # Stage identifier (e.g., 'TestBothWheels_manylinux_2_28_x86_64') + - name: stageName + type: string + # Job identifier within the stage + - name: jobName + type: string + default: 'InstallAndTest' + # Linux distribution type: 'manylinux_2_28' (glibc 2.28+) or 'musllinux' (musl libc) + - name: linuxTag + type: string + # CPU architecture (test stage targets x86_64 only) + - name: arch + type: string + default: 'x86_64' + # Docker platform for the build/test container + - name: dockerPlatform + type: string + default: 'linux/amd64' + # OneBranch build type: 'Official' or 'NonOfficial' + - name: oneBranchType + type: string + default: 'Official' + +stages: + - stage: ${{ parameters.stageName }} + displayName: 'Test Both Wheels ${{ parameters.linuxTag }} ${{ parameters.arch }}' + # Only meaningful when both consolidated artifacts exist in this run. + dependsOn: + - Consolidate + - ConsolidateOdbc + jobs: + - job: ${{ parameters.jobName }} + displayName: 'Install both wheels + pytest - ${{ parameters.linuxTag }} ${{ parameters.arch }}' + + pool: + type: linux + isCustom: true + name: Python-1ES-pool + demands: + - imageOverride -equals PYTHON-1ES-UB2404 + timeoutInMinutes: 90 + + variables: + - name: ob_sdl_binskim_enabled + value: false + - name: ob_outputDirectory + value: '$(Build.ArtifactStagingDirectory)' + - name: LinuxContainerImage + value: 'onebranch.azurecr.io/linux/ubuntu-2204:latest' + - name: LINUX_TAG + value: ${{ parameters.linuxTag }} + - name: ARCH + value: ${{ parameters.arch }} + - name: DOCKER_PLATFORM + value: ${{ parameters.dockerPlatform }} + + steps: + - checkout: self + fetchDepth: 1 + + # Download the two CONSOLIDATED wheel artifacts produced earlier in this run: + # drop_Consolidate_ConsolidateArtifacts -> mssql-python wheels (all platforms/pyvers) + # drop_ConsolidateOdbc_ConsolidateArtifacts -> mssql-python-odbc wheels (7 platforms) + - task: DownloadPipelineArtifact@2 + displayName: 'Download consolidated mssql-python wheels' + inputs: + buildType: 'current' + artifactName: 'drop_Consolidate_ConsolidateArtifacts' + targetPath: '$(Pipeline.Workspace)/mssql-python-wheels' + + - task: DownloadPipelineArtifact@2 + displayName: 'Download consolidated mssql-python-odbc wheels' + inputs: + buildType: 'current' + artifactName: 'drop_ConsolidateOdbc_ConsolidateArtifacts' + targetPath: '$(Pipeline.Workspace)/mssql-python-odbc-wheels' + + - bash: | + set -euxo pipefail + if ! docker info > /dev/null 2>&1; then + echo "Docker daemon not running, starting it..." + sudo dockerd > docker.log 2>&1 & + sleep 10 + fi + docker --version + displayName: 'Ensure Docker daemon is running' + + - task: AzureCLI@2 + displayName: 'Login to ACR (tdslibrs)' + inputs: + azureSubscription: 'Magnitude Test-mssql-rs-mssql-python' + scriptType: 'bash' + scriptLocation: 'inlineScript' + inlineScript: | + az acr login --name tdslibrs + + - script: | + set -euxo pipefail + if [[ "$(LINUX_TAG)" == "manylinux_2_28" ]]; then + IMAGE="tdslibrs.azurecr.io/import/python-build/manylinux_2_28_$(ARCH):latest" + elif [[ "$(LINUX_TAG)" == "musllinux" ]]; then + IMAGE="tdslibrs.azurecr.io/import/python-build/musllinux_1_2_$(ARCH):latest" + else + echo "ERROR: Unsupported LINUX_TAG '$(LINUX_TAG)'" >&2 + exit 1 + fi + + # Mount the repo and downloaded wheels read-only. Read-only /workspace is + # deliberate: this stage must exercise the INSTALLED wheels, never the + # source checkout, so nothing here should ever write into the repo tree. + docker run -d --name test-$(LINUX_TAG)-$(ARCH) \ + --platform $(DOCKER_PLATFORM) \ + -v $(Build.SourcesDirectory):/workspace:ro \ + -v $(Pipeline.Workspace)/mssql-python-wheels:/wheels/mssql-python:ro \ + -v $(Pipeline.Workspace)/mssql-python-odbc-wheels:/wheels/mssql-python-odbc:ro \ + -w /workspace \ + $IMAGE \ + tail -f /dev/null + displayName: 'Start $(LINUX_TAG) $(ARCH) test container' + + # Install the ODBC driver's runtime SYSTEM dependencies INTO the test + # container. msodbcsql18 links MIT Kerberos (libkrb5.so.3 / + # libgssapi_krb5.so.2) as a system dependency - these are never bundled in + # the wheel (the driver's RUNPATH is only $ORIGIN, which resolves the + # co-located libodbcinst.so.2 but NOT krb5). The manylinux_2_28 base image + # ships krb5-libs by default, but the minimal musllinux/Alpine image does + # NOT, so without this the driver's dlopen fails with + # "Error loading shared library libkrb5.so.3: No such file or directory" and, + # because that throw happens inside std::call_once on musl, the process + # aborts (core dump) at import instead of raising a clean ImportError. This + # mirrors the krb5-libs install already performed by + # build-linux-single-stage.yml and reflects the real runtime requirement for + # Alpine end users. + - script: | + set -euxo pipefail + if [[ "$(LINUX_TAG)" == "musllinux" ]]; then + docker exec test-$(LINUX_TAG)-$(ARCH) sh -lc 'apk add --no-cache krb5-libs keyutils-libs' + else + docker exec test-$(LINUX_TAG)-$(ARCH) bash -lc 'dnf -y install krb5-libs keyutils-libs || yum -y install krb5-libs keyutils-libs || true' + fi + displayName: 'Install ODBC driver runtime deps (krb5) in $(LINUX_TAG) container' + + # Start a SQL Server 2022 container on the host so pytest has a live server. + - script: | + set -euxo pipefail + docker run -d --name sqlserver-$(LINUX_TAG)-$(ARCH) \ + --platform linux/amd64 \ + -e ACCEPT_EULA=Y \ + -e MSSQL_SA_PASSWORD="$(DB_PASSWORD)" \ + -p 1433:1433 \ + mcr.microsoft.com/mssql/server:2022-latest + + echo "Waiting for SQL Server to be ready..." + for i in {1..30}; do + if docker exec sqlserver-$(LINUX_TAG)-$(ARCH) /opt/mssql-tools18/bin/sqlcmd \ + -S localhost -U SA -P "$(DB_PASSWORD)" -C -Q "SELECT 1" >/dev/null 2>&1; then # DevSkim: ignore DS162092 + echo "SQL Server is ready" + break + fi + sleep 2 + done + + SQL_IP=$(docker inspect -f '{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}' sqlserver-$(LINUX_TAG)-$(ARCH)) + echo "SQL Server IP: $SQL_IP" + echo "##vso[task.setvariable variable=SQL_IP]$SQL_IP" + displayName: 'Start SQL Server container for testing' + env: + DB_PASSWORD: $(DB_PASSWORD) + + # For each Python version: install BOTH wheels, remove the bundled ODBC + # driver from the installed mssql_python package, then run pytest. A pass + # proves the driver was loaded from the separate mssql_python_odbc package. + - script: | + set -euxo pipefail + if [[ "$(LINUX_TAG)" == "manylinux_2_28" ]]; then SHELL_EXE=bash; else SHELL_EXE=sh; fi + + # Count how many Python versions actually ran the full test so the stage + # cannot go green having silently skipped every version (e.g. if none of + # the /opt/python/ interpreters exist in the image). + TESTED=0 + # Inner script uses exit code 3 to signal "interpreter absent - skipped". + SKIP_RC=3 + + for PYBIN in cp310 cp311 cp312 cp313 cp314; do + echo "" + echo "=====================================================" + echo "End-user both-wheels test: $PYBIN on $(LINUX_TAG)/$(ARCH)" + echo "=====================================================" + + set +e + docker exec -e PYBIN=$PYBIN -e LINUX_TAG="$(LINUX_TAG)" -e SQL_IP=$(SQL_IP) -e DB_PASSWORD="$(DB_PASSWORD)" \ + test-$(LINUX_TAG)-$(ARCH) $SHELL_EXE -lc ' + set -euo pipefail + + PY=/opt/python/${PYBIN}-${PYBIN}/bin/python + test -x "$PY" || { echo "Python $PY missing - skipping ${PYBIN}"; exit 3; } + echo "Using: $($PY --version)" + + # Fresh isolated venv = clean end-user machine + TEST_DIR="/enduser_${PYBIN}" + rm -rf "$TEST_DIR" + $PY -m venv "$TEST_DIR" + VENV_PY="$TEST_DIR/bin/python" + + # Run all Python invocations from the venv dir (never /workspace) and + # with PYTHONSAFEPATH so the repo checkout at /workspace/mssql_python + # can NEVER shadow the freshly installed wheel. Without this, cwd + # (/workspace) lands on sys.path and "import mssql_python" would resolve + # to the source tree instead of the package under test. + export PYTHONSAFEPATH=1 + cd "$TEST_DIR" + + $VENV_PY -m pip install -q -U pip + + # Platform wheel tag: manylinux_2_28 vs musllinux_1_2, x86_64 + if [ "$LINUX_TAG" = "manylinux_2_28" ]; then PLAT="manylinux"; else PLAT="musllinux"; fi + + # Select the mssql-python wheel matching THIS Python version + libc + MP_WHEEL=$(ls /wheels/mssql-python/dist/mssql_python-*${PYBIN}-*${PLAT}*x86_64*.whl 2>/dev/null | head -1) + if [ -z "$MP_WHEEL" ]; then + echo "ERROR: no mssql-python wheel for ${PYBIN}/${PLAT}"; exit 1 + fi + + # ODBC wheel is data-only (py3-none-), one per platform + ODBC_WHEEL=$(ls /wheels/mssql-python-odbc/dist/mssql_python_odbc-*${PLAT}*x86_64*.whl 2>/dev/null | head -1) + if [ -z "$ODBC_WHEEL" ]; then + echo "ERROR: no mssql-python-odbc wheel for ${PLAT}"; exit 1 + fi + + # Install the odbc wheel FIRST so that once mssql-python pins + # mssql-python-odbc==X (setup.py install_requires), the requirement + # is already satisfied from this LOCAL wheel and pip never reaches + # PyPI for a version that may not be published yet. + echo "Installing mssql-python-odbc: $ODBC_WHEEL" + $VENV_PY -m pip install -q "$ODBC_WHEEL" + echo "Installing mssql-python: $MP_WHEEL" + $VENV_PY -m pip install -q "$MP_WHEEL" + + # Show where the external ODBC package lives (the split target) + $VENV_PY -c "import mssql_python_odbc, os; print(\"mssql_python_odbc libs:\", mssql_python_odbc.get_libs_dir()); assert os.path.isdir(mssql_python_odbc.get_libs_dir())" + + # POSITIVE assertion: prove the native loader WILL resolve the driver + # from the external mssql_python_odbc package. This mirrors exactly what + # GetOdbcLibsBaseDir() does in ddbc_bindings.cpp: it takes the parent dir + # of mssql_python_odbc.__file__ as the base and treats the external + # package as authoritative iff GetDriverPathCpp(base) exists. If this + # driver file is present, the loader is guaranteed to pick the external + # package (non-Windows only checks the driver; Windows also needs + # mssql-auth.dll, which this platform test does not cover). + $VENV_PY -c "import mssql_python.ddbc_bindings as d, mssql_python_odbc, os; base=os.path.dirname(mssql_python_odbc.__file__); drv=d.GetDriverPathCpp(base); print(\"external driver path:\", drv); assert os.path.exists(drv), \"external ODBC driver not found - loader would NOT select external package: \"+drv; print(\"POSITIVE: loader will resolve the driver from the external mssql_python_odbc package\")" + + # DECISIVE (negative) STEP: remove the ODBC driver bundled inside + # mssql_python. With the fallback gone, any successful connection must + # be using the driver from the separate mssql_python_odbc package. + MP_LIBS=$($VENV_PY -c "import mssql_python, os; print(os.path.join(os.path.dirname(mssql_python.__file__), \"libs\"))") + case "$MP_LIBS" in + /workspace/*) + echo "ERROR: resolved mssql_python to the source tree ($MP_LIBS), not the installed wheel"; exit 1 ;; + esac + if [ -d "$MP_LIBS" ]; then + echo "Removing bundled mssql_python/libs to force use of external ODBC package: $MP_LIBS" + rm -rf "$MP_LIBS" + else + echo "Note: mssql_python has no bundled libs/ (already split at build time)" + fi + + # Smoke test: import + live connection using ONLY the external driver + $VENV_PY -c "import mssql_python; print(\"mssql_python\", mssql_python.__version__, \"imported OK\")" + DB_CONNECTION_STRING="Server=$SQL_IP;Database=master;Uid=SA;Pwd=$DB_PASSWORD;TrustServerCertificate=yes" \ + $VENV_PY -c "import mssql_python, os; c=mssql_python.connect(os.environ[\"DB_CONNECTION_STRING\"]); cur=c.cursor(); cur.execute(\"SELECT 1\"); print(\"connection OK:\", cur.fetchone()); c.close()" + + # Copy the suite OUT of /workspace so pytest collection cannot import + # the source-tree package either, then run against the installed wheels. + cp -r /workspace/tests "$TEST_DIR/tests" + cp /workspace/pytest.ini "$TEST_DIR/pytest.ini" 2>/dev/null || true + $VENV_PY -m pip install -q pytest + if [ -f /workspace/requirements.txt ]; then + $VENV_PY -m pip install -q -r /workspace/requirements.txt + fi + DB_CONNECTION_STRING="Server=$SQL_IP;Database=master;Uid=SA;Pwd=$DB_PASSWORD;TrustServerCertificate=yes" \ + $VENV_PY -m pytest "$TEST_DIR/tests" -v --maxfail=1 + echo "All tests passed for ${PYBIN} using the EXTERNAL mssql_python_odbc driver" + ' + rc=$? + set -e + + if [ "$rc" -eq "$SKIP_RC" ]; then + echo "Skipped $PYBIN (interpreter not present in image)" + elif [ "$rc" -ne 0 ]; then + echo "ERROR: both-wheels test failed for $PYBIN (exit $rc)" + exit "$rc" + else + TESTED=$((TESTED + 1)) + echo "Both-wheels test complete for $PYBIN" + fi + done + + if [ "$TESTED" -eq 0 ]; then + echo "ERROR: no Python versions were tested - every interpreter was missing. Failing the stage." + exit 1 + fi + + echo "=====================================================" + echo "All available Python versions ($TESTED) passed the both-wheels end-user test" + echo "=====================================================" + displayName: 'Install both wheels + pytest (Python 3.10-3.14)' + env: + DB_PASSWORD: $(DB_PASSWORD) + + - script: | + docker stop test-$(LINUX_TAG)-$(ARCH) sqlserver-$(LINUX_TAG)-$(ARCH) || true + docker rm test-$(LINUX_TAG)-$(ARCH) sqlserver-$(LINUX_TAG)-$(ARCH) || true + displayName: 'Cleanup containers' + condition: always() diff --git a/OneBranchPipelines/stages/wheel-installation-test-windows-stage.yml b/OneBranchPipelines/stages/wheel-installation-test-windows-stage.yml new file mode 100644 index 000000000..8a45e39b7 --- /dev/null +++ b/OneBranchPipelines/stages/wheel-installation-test-windows-stage.yml @@ -0,0 +1,196 @@ +# ========================================================================================= +# End-User "Both Wheels Together" Installation Test - WINDOWS +# ========================================================================================= +# Windows counterpart of wheel-installation-test-stage.yml (which is Docker/Linux-only). +# Installs the ACTUAL published mssql-python + mssql-python-odbc Windows wheels into a +# fresh venv, runs the full pytest suite (bundled libs intact, external package already +# preferred by GetOdbcLibsBaseDir), then DELETES the bundled mssql_python/libs and +# connects once more in a fresh process. With the bundled fallback gone, a successful +# live query can ONLY be using the driver from the separately installed +# mssql_python_odbc package. +# +# Windows adds coverage the Linux stage explicitly cannot: on Windows the loader treats +# the external package as authoritative only when BOTH the ODBC driver AND mssql-auth.dll +# are present, so this proves the auth DLL is resolved from the external package too. +# +# Runs natively on the same 1ES Windows Server + SQL Server LocalDB pool as the build +# stage (no Docker). A single representative Python version is sufficient: the external +# package RESOLUTION path (GetOdbcLibsBaseDir in ddbc_bindings.cpp) is Python-version +# independent; per-version native builds are already covered by the Windows build stages. +# ========================================================================================= +parameters: + # Stage identifier (e.g., 'TestBothWheels_Windows_x64') + - name: stageName + type: string + # Job identifier within the stage + - name: jobName + type: string + default: 'InstallAndTest' + # Python version in X.Y format used to select the mssql-python wheel + - name: pythonVersion + type: string + default: '3.12' + # Python version as 3-digit string (e.g., '312') matching the cp wheel tag + - name: shortPyVer + type: string + default: '312' + # Target architecture (test stage targets x64 only; ARM64 binaries can't run on x64 host) + - name: architecture + type: string + default: 'x64' + # OneBranch build type: 'Official' or 'NonOfficial' + - name: oneBranchType + type: string + default: 'Official' + +stages: + - stage: ${{ parameters.stageName }} + displayName: 'Test Both Wheels Windows ${{ parameters.architecture }}' + # Only meaningful when both consolidated artifacts exist in this run. + dependsOn: + - Consolidate + - ConsolidateOdbc + jobs: + - job: ${{ parameters.jobName }} + displayName: 'Install both wheels + pytest - Windows ${{ parameters.architecture }}' + + # Same custom 1ES pool the Windows build stage uses: Windows Server 2022 with + # SQL Server LocalDB pre-installed, so pytest has a live server without Docker. + pool: + type: windows + isCustom: true + name: Python-1ES-pool + demands: + - imageOverride -equals PYTHON-1ES-MMS2022 + timeoutInMinutes: 120 + + variables: + ob_outputDirectory: '$(Build.ArtifactStagingDirectory)' + # OneBranch-required variable (unused on Windows agents). + LinuxContainerImage: 'onebranch.azurecr.io/linux/ubuntu-2204:latest' + + steps: + - checkout: self + fetchDepth: 1 + + - task: UsePythonVersion@0 + inputs: + versionSpec: '${{ parameters.pythonVersion }}' + architecture: 'x64' + addToPath: true + displayName: 'Use Python ${{ parameters.pythonVersion }}' + + # Download the two CONSOLIDATED wheel artifacts produced earlier in this run. + - task: DownloadPipelineArtifact@2 + displayName: 'Download consolidated mssql-python wheels' + inputs: + buildType: 'current' + artifactName: 'drop_Consolidate_ConsolidateArtifacts' + targetPath: '$(Pipeline.Workspace)/mssql-python-wheels' + + - task: DownloadPipelineArtifact@2 + displayName: 'Download consolidated mssql-python-odbc wheels' + inputs: + buildType: 'current' + artifactName: 'drop_ConsolidateOdbc_ConsolidateArtifacts' + targetPath: '$(Pipeline.Workspace)/mssql-python-odbc-wheels' + + # Start SQL Server LocalDB (pre-installed on the 1ES Windows image). + - powershell: | + sqllocaldb create MSSQLLocalDB + sqllocaldb start MSSQLLocalDB + displayName: 'Start LocalDB instance' + + - powershell: | + sqlcmd -S "(localdb)\MSSQLLocalDB" -Q "CREATE DATABASE TestDB" + sqlcmd -S "(localdb)\MSSQLLocalDB" -Q "CREATE LOGIN testuser WITH PASSWORD = '$(DB_PASSWORD)'" + sqlcmd -S "(localdb)\MSSQLLocalDB" -d TestDB -Q "CREATE USER testuser FOR LOGIN testuser" + sqlcmd -S "(localdb)\MSSQLLocalDB" -d TestDB -Q "ALTER ROLE db_owner ADD MEMBER testuser" + displayName: 'Setup database and user' + env: + DB_PASSWORD: $(DB_PASSWORD) + + - powershell: | + $ErrorActionPreference = 'Stop' + + # Run every venv-python invocation through this helper so a non-zero exit + # (native commands do NOT trip $ErrorActionPreference) fails the step loudly. + # No param() block: PowerShell then routes ALL tokens (including -m, -c, -q) + # into the automatic $args instead of trying to bind them as parameters. + $script:VenvPy = $null + function Invoke-Py { + & $script:VenvPy @args + if ($LASTEXITCODE -ne 0) { throw "python exited ${LASTEXITCODE}: $($args -join ' ')" } + } + + $PYTAG = 'cp${{ parameters.shortPyVer }}' + $PLATTAG = 'win_amd64' + + # Select the mssql-python wheel matching THIS Python version + platform. + $mpWheel = Get-ChildItem "$(Pipeline.Workspace)\mssql-python-wheels\dist\mssql_python-*$PYTAG-*$PLATTAG*.whl" -ErrorAction SilentlyContinue | Select-Object -First 1 + if (-not $mpWheel) { throw "ERROR: no mssql-python wheel for $PYTAG/$PLATTAG" } + # The odbc wheel is data-only (py3-none-), one per platform. + $odbcWheel = Get-ChildItem "$(Pipeline.Workspace)\mssql-python-odbc-wheels\dist\mssql_python_odbc-*$PLATTAG*.whl" -ErrorAction SilentlyContinue | Select-Object -First 1 + if (-not $odbcWheel) { throw "ERROR: no mssql-python-odbc wheel for $PLATTAG" } + + # Fresh isolated venv = clean end-user machine. + $testDir = Join-Path $env:TEMP "enduser_$PYTAG" + if (Test-Path $testDir) { Remove-Item -Recurse -Force $testDir } + python -m venv $testDir + if ($LASTEXITCODE -ne 0) { throw "venv creation failed" } + $script:VenvPy = Join-Path $testDir 'Scripts\python.exe' + + # PYTHONSAFEPATH so the repo checkout can never shadow the installed wheel. + $env:PYTHONSAFEPATH = '1' + + Invoke-Py -m pip install -q -U pip + + # Install the odbc wheel FIRST so mssql-python's pinned mssql-python-odbc==X + # requirement is satisfied from this LOCAL wheel (pip never reaches PyPI). + Write-Host "Installing mssql-python-odbc: $($odbcWheel.Name)" + Invoke-Py -m pip install -q "$($odbcWheel.FullName)" + Write-Host "Installing mssql-python: $($mpWheel.Name)" + Invoke-Py -m pip install -q "$($mpWheel.FullName)" + + # Show where the external ODBC package lives (the split target). + Invoke-Py -c "import mssql_python_odbc, os; d=mssql_python_odbc.get_libs_dir(); print('mssql_python_odbc libs:', d); assert os.path.isdir(d)" + + # POSITIVE assertion: prove the native loader WILL resolve the driver from + # the external mssql_python_odbc package (mirrors GetOdbcLibsBaseDir). + Invoke-Py -c "import mssql_python.ddbc_bindings as d, mssql_python_odbc, os; base=os.path.dirname(mssql_python_odbc.__file__); drv=d.GetDriverPathCpp(base); print('external driver path:', drv); assert os.path.exists(drv), 'external ODBC driver not found - loader would NOT select external package: '+drv; print('POSITIVE: loader will resolve the driver from the external mssql_python_odbc package')" + + # PHASE 1: run the FULL existing suite with bundled libs INTACT. Copy the + # suite OUT of the source tree so collection imports the INSTALLED package. + Copy-Item -Recurse "$(Build.SourcesDirectory)\tests" (Join-Path $testDir 'tests') + Copy-Item "$(Build.SourcesDirectory)\pytest.ini" (Join-Path $testDir 'pytest.ini') + Invoke-Py -m pip install -q pytest + if (Test-Path "$(Build.SourcesDirectory)\requirements.txt") { + Invoke-Py -m pip install -q -r "$(Build.SourcesDirectory)\requirements.txt" + } + $env:DB_CONNECTION_STRING = "Server=(localdb)\MSSQLLocalDB;Database=TestDB;Uid=testuser;Pwd=$($env:DB_PASSWORD);TrustServerCertificate=yes" + Push-Location $testDir + try { + Invoke-Py -m pytest (Join-Path $testDir 'tests') -v --maxfail=1 -m "not stress" + } finally { + Pop-Location + } + Write-Host "Full suite passed for $PYTAG with both wheels installed (external driver preferred)" + + # PHASE 2 - DECISIVE negative proof: remove the ODBC driver that + # mssql_python still bundles, then connect again in a FRESH process. + $mpLibs = & $script:VenvPy -c "import mssql_python, os; print(os.path.join(os.path.dirname(mssql_python.__file__), 'libs'))" + if ($LASTEXITCODE -ne 0) { throw "failed to resolve mssql_python libs dir" } + if ($mpLibs -like "$(Build.SourcesDirectory)*") { throw "ERROR: resolved mssql_python to the source tree ($mpLibs), not the installed wheel" } + if (Test-Path $mpLibs) { + Write-Host "Removing bundled mssql_python/libs to force use of external ODBC package: $mpLibs" + Remove-Item -Recurse -Force $mpLibs + } else { + Write-Host "Note: mssql_python has no bundled libs/ (already split at build time)" + } + + Invoke-Py -c "import mssql_python; print('mssql_python', mssql_python.__version__, 'imported OK (bundled libs removed)')" + Invoke-Py -c "import mssql_python, os; c=mssql_python.connect(os.environ['DB_CONNECTION_STRING']); cur=c.cursor(); cur.execute('SELECT 1'); print('connection OK via EXTERNAL driver:', cur.fetchone()); c.close()" + Write-Host "DECISIVE: connected with bundled libs removed - driver + mssql-auth.dll resolved from external mssql_python_odbc package" + displayName: 'Install both wheels + full pytest + external-only proof' + env: + DB_PASSWORD: $(DB_PASSWORD) diff --git a/tests/test_000_dependencies.py b/tests/test_000_dependencies.py index f38ba06e2..02784181a 100644 --- a/tests/test_000_dependencies.py +++ b/tests/test_000_dependencies.py @@ -22,6 +22,7 @@ def __init__(self): self.platform_name = platform.system().lower() self.raw_architecture = platform.machine().lower() self.module_dir = self._get_module_directory() + self.libs_base_dir = self._get_libs_base_dir() self.normalized_arch = self._normalize_architecture() def _get_module_directory(self): @@ -35,6 +36,33 @@ def _get_module_directory(self): # Fallback to relative path from tests directory return Path(__file__).parent.parent / "mssql_python" + def _get_libs_base_dir(self): + """Directory that contains the ``libs/`` driver payload the loader will use. + + Mirrors the native resolver (``GetOdbcLibsBaseDir`` in ddbc_bindings.cpp): + the external ``mssql_python_odbc`` package is authoritative whenever it is + installed and actually ships the driver; otherwise the driver is resolved + from ``mssql_python``'s own bundled ``libs/``. Keeping the expected paths in + lockstep with the loader means these dependency assertions validate the SAME + driver the runtime loads -- including the package-split scenario where the + bundled ``mssql_python/libs`` is absent and the driver comes only from the + separately installed ``mssql_python_odbc`` package. When that package is not + installed the base is unchanged, so single-wheel CI keeps enforcing that + ``mssql_python`` bundles its own libs. + """ + try: + import mssql_python_odbc + import mssql_python.ddbc_bindings as ddbc + + external_base = Path(mssql_python_odbc.__file__).parent + driver = ddbc.GetDriverPathCpp(str(external_base)) + if driver and Path(driver).exists(): + return external_base + except Exception: + # No external package (or it can't resolve a driver) -> bundled libs. + pass + return self.module_dir + def _normalize_architecture(self): """Normalize architecture names for the given platform.""" arch_lower = self.raw_architecture.lower() @@ -152,7 +180,7 @@ def get_expected_dependencies(self): def _get_windows_dependencies(self): """Get Windows dependencies based on architecture.""" - base_path = self.module_dir / "libs" / "windows" / self.normalized_arch + base_path = self.libs_base_dir / "libs" / "windows" / self.normalized_arch dependencies = [ base_path / self._driver_filename(), @@ -169,7 +197,7 @@ def _get_macos_dependencies(self): # macOS uses universal2 binaries, but we need to check both arch directories for arch in ["arm64", "x86_64"]: - base_path = self.module_dir / "libs" / "macos" / arch / "lib" + base_path = self.libs_base_dir / "libs" / "macos" / arch / "lib" dependencies.extend( [ base_path / self._driver_filename(), @@ -190,7 +218,7 @@ def _get_linux_dependencies(self): elif runtime_arch in ["aarch64"]: runtime_arch = "arm64" - base_path = self.module_dir / "libs" / "linux" / distro_name / runtime_arch / "lib" + base_path = self.libs_base_dir / "libs" / "linux" / distro_name / runtime_arch / "lib" dependencies = [ base_path / self._driver_filename(), @@ -232,7 +260,7 @@ def get_expected_driver_path(self): if platform_name == "windows": driver_path = ( - Path(self.module_dir) + Path(self.libs_base_dir) / "libs" / "windows" / normalized_arch @@ -241,7 +269,7 @@ def get_expected_driver_path(self): elif platform_name == "darwin": driver_path = ( - Path(self.module_dir) + Path(self.libs_base_dir) / "libs" / "macos" / normalized_arch @@ -252,7 +280,7 @@ def get_expected_driver_path(self): elif platform_name == "linux": distro_name = self._detect_linux_distro() driver_path = ( - Path(self.module_dir) + Path(self.libs_base_dir) / "libs" / "linux" / distro_name @@ -356,7 +384,7 @@ class TestArchitectureSpecificDependencies: def test_windows_vcredist_dependency(self): """Test that Windows builds include vcredist dependencies.""" vcredist_path = ( - dependency_tester.module_dir + dependency_tester.libs_base_dir / "libs" / "windows" / dependency_tester.normalized_arch @@ -372,7 +400,7 @@ def test_windows_vcredist_dependency(self): def test_windows_auth_dependency(self): """Test that Windows builds include authentication library.""" auth_path = ( - dependency_tester.module_dir + dependency_tester.libs_base_dir / "libs" / "windows" / dependency_tester.normalized_arch @@ -385,7 +413,7 @@ def test_windows_auth_dependency(self): def test_macos_universal_dependencies(self): """Test that macOS builds include dependencies for both architectures.""" for arch in ["arm64", "x86_64"]: - base_path = dependency_tester.module_dir / "libs" / "macos" / arch / "lib" + base_path = dependency_tester.libs_base_dir / "libs" / "macos" / arch / "lib" msodbcsql_path = base_path / dependency_tester._driver_filename() libodbcinst_path = base_path / "libodbcinst.2.dylib" @@ -456,7 +484,7 @@ def direct_bundled_deps(dylib_path, bundled_names): problems = [] checked_arches = [] for arch in ["arm64", "x86_64"]: - lib_dir = dependency_tester.module_dir / "libs" / "macos" / arch / "lib" + lib_dir = dependency_tester.libs_base_dir / "libs" / "macos" / arch / "lib" driver = lib_dir / "libmsodbcsql.18.dylib" if not driver.exists(): continue @@ -508,7 +536,7 @@ def test_linux_distribution_dependencies(self): distro_name = dependency_tester._detect_linux_distro() # Test that the distribution directory exists - distro_path = dependency_tester.module_dir / "libs" / "linux" / distro_name + distro_path = dependency_tester.libs_base_dir / "libs" / "linux" / distro_name assert distro_path.exists(), f"Linux distribution directory not found: {distro_path}" @@ -583,9 +611,9 @@ def test_get_driver_path_from_ddbc_bindings(): try: import mssql_python.ddbc_bindings as ddbc - module_dir = dependency_tester.module_dir + libs_base = dependency_tester.libs_base_dir - driver_path = ddbc.GetDriverPathCpp(str(module_dir)) + driver_path = ddbc.GetDriverPathCpp(str(libs_base)) # The driver path should be same as one returned by the Python function expected_path = dependency_tester.get_expected_driver_path()