diff --git a/.github/workflows/conda-audit.yml b/.github/workflows/conda-audit.yml new file mode 100644 index 000000000..66b9a18ed --- /dev/null +++ b/.github/workflows/conda-audit.yml @@ -0,0 +1,85 @@ +--- +# N3: PR-triggered conda build + masking-immune RUNPATH audit. +# +# The OneBranch conda-build pipeline is `trigger: none` / `pr: none`, so its blocking +# audit never runs on a PR. This lightweight GitHub Actions job builds ONE real +# linux-64 conda package from the SHIPPED PyPI wheels and runs +# eng/scripts/audit_bundled_binaries.py on it, so a regression in the $ORIGIN climb, +# the declared conda deps (krb5/libtool/openssl), or the expected DT_NEEDED set fails +# the PR automatically -- the full-agent runtime masking cannot hide it. +# +# The PyPI mssql-python-odbc binaries are not yet pre-baked with the climb, so the +# recipe's assertion-only default would (correctly) refuse to mutate them; this PR +# gate sets CONDA_ALLOW_UNSIGNED_PATCH=1 to build a DEV-patched climb and audit THAT. +# The signed release path stays assertion-only (see conda/mssql-python/build.sh). +name: conda-audit + +on: + pull_request: + paths: + - 'conda/**' + - 'eng/scripts/audit_bundled_binaries.py' + - 'OneBranchPipelines/scripts/build-conda-packages.sh' + - 'tests/test_027_conda_release_metadata.py' + - 'tests/test_029_bundled_binary_audit.py' + - '.github/workflows/conda-audit.yml' + +permissions: + contents: read + +jobs: + linux-conda-audit: + runs-on: ubuntu-latest + timeout-minutes: 30 + steps: + - uses: actions/checkout@v4 + + - uses: actions/setup-python@v5 + with: + python-version: '3.11' + + - name: Unit-test the audit + release validator + run: | + python -m pip install --quiet pytest zstandard + # --noconftest: tests/conftest.py imports mssql_python (the native ddbc_bindings + # extension), which is NOT built in this repackage-only gate. test_027/test_029 + # are pure conda validator/audit tests using only built-in fixtures, so skip + # conftest to avoid that unrelated import. + python -m pytest --noconftest \ + tests/test_029_bundled_binary_audit.py \ + tests/test_027_conda_release_metadata.py -q + + - name: Fetch the shipped linux wheels from PyPI + run: | + set -euo pipefail + mkdir -p wheels + # --no-deps: pull ONLY the two wheels we repackage (not azure-identity etc). + python -m pip download --no-deps mssql-python mssql-python-odbc -d wheels + echo "Downloaded:"; ls -1 wheels + + - name: Build + audit a linux-64 conda package + env: + # PyPI odbc binaries are not pre-baked; build a DEV-patched climb here and + # audit it. NEVER used on the signed release path. + CONDA_ALLOW_UNSIGNED_PATCH: '1' + run: | + set -euo pipefail + mssql_whl="$(ls wheels/mssql_python-*.whl | grep -v mssql_python_odbc | head -1)" + odbc_whl="$(ls wheels/mssql_python_odbc-*.whl | head -1)" + MSSQL_VER="$(basename "$mssql_whl" | sed -nE 's/^mssql_python-([^-]+)-.*/\1/p')" + ODBC_VER="$(basename "$odbc_whl" | sed -nE 's/^mssql_python_odbc-([^-]+)-.*/\1/p')" + echo "mssql-python=$MSSQL_VER mssql-python-odbc=$ODBC_VER" + bash OneBranchPipelines/scripts/build-conda-packages.sh \ + "$PWD/wheels" \ + "$PWD/conda" \ + "$RUNNER_TEMP/conda-bld" \ + "$MSSQL_VER" \ + "$ODBC_VER" \ + "3.11" \ + "linux-64" + + - name: Explicit standalone audit of the built package + run: | + set -euo pipefail + python -m pip install --quiet zstandard + python eng/scripts/audit_bundled_binaries.py --root "$RUNNER_TEMP/conda-bld/bld" diff --git a/OneBranchPipelines/conda-build-pipeline.yml b/OneBranchPipelines/conda-build-pipeline.yml new file mode 100644 index 000000000..c445fb1a9 --- /dev/null +++ b/OneBranchPipelines/conda-build-pipeline.yml @@ -0,0 +1,322 @@ +# ========================================================================================= +# OneBranch STANDALONE Conda Build Pipeline (mssql-python) +# ========================================================================================= +# Builds the SELF-CONTAINED mssql-python conda package (which vendors the ODBC Driver 18 +# payload -- there is NO separate companion package) for every conda subdir, WITHOUT +# rebuilding the wheels. It consumes the wheels already produced by the wheel build +# pipeline (definition 2199): the mssql-python wheels (drop_Consolidate_ConsolidateArtifacts) +# and the mssql-python-odbc wheels (drop_ConsolidateOdbc_ConsolidateArtifacts), then +# repackages + validates them into conda packages on the matching platform agent. +# +# WHY PER-OS (not one job for all OSs): unlike the odbc wheels (a pure data re-tag), +# conda-build provisions a REAL per-subdir host env and `pip install`s the matching wheel +# (see conda/mssql-python/build.sh|bld.bat). A win-64 / osx-* / linux-* host env cannot be +# created on a foreign OS, so each conda package must be built on its matching platform -- +# exactly like the wheels and the conda-forge feedstocks. The subdirs that CAN be +# cross-targeted on one agent are done via CONDA_SUBDIR (osx-64 under Rosetta 2 on the +# Intel mac agent; linux-aarch64 under QEMU on the x86_64 linux agent). +# +# This pipeline ONLY builds + validates + consolidates the conda packages as an artifact +# (drop_ConsolidateConda_ConsolidateArtifacts). Publishing is the companion +# conda-release-pipeline.yml. Validated locally that the recipe builds + imports; the +# per-OS legs + SDL settings need an actual ADO run to confirm. +# ========================================================================================= +name: $(Year:YY)$(DayOfYear)$(Rev:.r)-CondaBuild + +trigger: none +pr: none + +parameters: + - name: oneBranchType + displayName: 'OneBranch Template Type' + type: string + values: + - 'Official' + - 'NonOfficial' + default: 'NonOfficial' + # Python versions to build conda packages for (Windows loops these; the POSIX legs + # auto-detect the same set from the downloaded wheels). + - name: pythonVersions + displayName: 'Python versions (comma-separated)' + type: string + default: '3.10,3.11,3.12,3.13,3.14' + # H1: the Encrypt=yes probe connection string. OpenSSL is dlopen'd lazily (not in + # DT_NEEDED), so ONLY a live TLS handshake proves libssl/libcrypto resolve from + # $PREFIX/lib -- no static audit can. Supply a reachable server (from a secret var) + # to activate conda/tls_connect_probe.py on the Linux leg; empty = the probe SKIPs. + - name: condaTlsProbeConn + displayName: 'TLS probe: a full SQL Server connection string; NOT a yes/no toggle; empty = skip' + type: string + default: '' + # H1: enable the minimal-base ldd reachability gate (CONDA_ASSERT_PREFIX_REACHABLE). + # It fails CLOSED if the driver binds a system (or absent) krb5/gssapi/libltdl, so it + # is only valid on a leg with NO system copies of those libs -- set true ONLY when + # the Linux agent/container is a curated minimal base, else it will (correctly) fail + # on a full agent's system libs. + - name: enableMinimalReachabilityGate + displayName: 'Enable minimal-base ldd reachability gate (needs a minimal Linux base)' + type: boolean + default: false + +variables: + - name: effectiveOneBranchType + ${{ if eq(variables['Build.Reason'], 'Schedule') }}: + value: 'Official' + ${{ else }}: + value: '${{ parameters.oneBranchType }}' + - template: /OneBranchPipelines/variables/common-variables.yml@self + - template: /OneBranchPipelines/variables/onebranch-variables.yml@self + +resources: + repositories: + - repository: templates + type: git + name: 'OneBranch.Pipelines/GovernedTemplates' + ref: 'refs/heads/main' + # The wheel build pipeline whose consolidated wheel artifacts this pipeline repackages. + pipelines: + - pipeline: buildPipeline + source: 'Build-Release-Package-Pipeline' + trigger: none + +extends: + template: 'v2/OneBranch.${{ variables.effectiveOneBranchType }}.CrossPlat.yml@templates' + parameters: + featureFlags: + WindowsHostVersion: + Version: '2022' + # Minimal SDL: this pipeline compiles NOTHING (it repackages prebuilt, already-scanned + # wheels), so the heavy binary analyzers add no coverage. Keep the credential / inclusive + # -language / baseline guardrails and publish logs. + globalSdl: + baseline: + baselineFile: $(Build.SourcesDirectory)/.gdn/.gdnbaselines + suppressionSet: default + suppression: + suppressionFile: $(Build.SourcesDirectory)/.gdn/.gdnsuppress + suppressionSet: default + credscan: + enabled: true + policheck: + enabled: true + break: true + exclusionFile: '$(REPO_ROOT)/.config/PolicheckExclusions.xml' + publishLogs: + enabled: true + + stages: + # ========================= + # CONDA win-64 (native, per-Python) + # ========================= + - stage: CondaWin64 + displayName: 'Conda win-64' + jobs: + - job: BuildConda + displayName: 'Build + validate win-64 conda' + timeoutInMinutes: 120 + pool: + type: windows + isCustom: true + name: Python-1ES-pool + demands: + - imageOverride -equals PYTHON-1ES-MMS2022 + variables: + ob_outputDirectory: '$(Build.ArtifactStagingDirectory)' + steps: + - checkout: self + fetchDepth: 1 + - task: DownloadPipelineArtifact@2 + displayName: 'Download mssql-python wheels' + inputs: + buildType: 'specific' + project: '$(System.TeamProject)' + definition: 2199 + buildVersionToDownload: 'specific' + buildId: $(resources.pipeline.buildPipeline.runID) + artifactName: 'drop_Consolidate_ConsolidateArtifacts' + targetPath: '$(Build.SourcesDirectory)' + - task: DownloadPipelineArtifact@2 + displayName: 'Download mssql-python-odbc wheels' + inputs: + buildType: 'specific' + project: '$(System.TeamProject)' + definition: 2199 + buildVersionToDownload: 'specific' + buildId: $(resources.pipeline.buildPipeline.runID) + artifactName: 'drop_ConsolidateOdbc_ConsolidateArtifacts' + targetPath: '$(Pipeline.Workspace)/odbc_wheels' + # win-64 conda build (all Python versions in one pass via the ps1 loop). + - template: /OneBranchPipelines/steps/conda-build-validate-step.yml@self + parameters: + pythonVersion: '${{ parameters.pythonVersions }}' + condaSubdir: 'win-64' + targetArch: 'x64' + odbcWheelDir: '$(Pipeline.Workspace)/odbc_wheels' + odbcWheelFilter: 'mssql_python_odbc-*win_amd64.whl' + - task: PublishPipelineArtifact@1 + displayName: 'Publish win-64 conda artifact' + inputs: + targetPath: '$(ob_outputDirectory)' + artifact: 'drop_CondaWin64_BuildConda' + publishLocation: 'pipeline' + + # ========================= + # CONDA osx-arm64 + osx-64 (Intel mac agent: arm64 cross, x86_64 native) + # ========================= + - stage: CondaMacOS + displayName: 'Conda macOS (osx-arm64 + osx-64)' + jobs: + - job: BuildConda + displayName: 'Build + validate macOS conda' + timeoutInMinutes: 120 + # macOS pools declare as type:linux (Azure Pipelines quirk). + pool: + type: linux + isCustom: true + name: Azure Pipelines + vmImage: 'macos-latest' + variables: + ob_outputDirectory: '$(Build.ArtifactStagingDirectory)' + steps: + - checkout: self + fetchDepth: 1 + - task: DownloadPipelineArtifact@2 + displayName: 'Download mssql-python wheels' + inputs: + buildType: 'specific' + project: '$(System.TeamProject)' + definition: 2199 + buildVersionToDownload: 'specific' + buildId: $(resources.pipeline.buildPipeline.runID) + artifactName: 'drop_Consolidate_ConsolidateArtifacts' + targetPath: '$(Build.SourcesDirectory)' + - task: DownloadPipelineArtifact@2 + displayName: 'Download mssql-python-odbc wheels' + inputs: + buildType: 'specific' + project: '$(System.TeamProject)' + definition: 2199 + buildVersionToDownload: 'specific' + buildId: $(resources.pipeline.buildPipeline.runID) + artifactName: 'drop_ConsolidateOdbc_ConsolidateArtifacts' + targetPath: '$(Pipeline.Workspace)/odbc_wheels' + # osx-arm64: CROSS-built on the Intel agent (BEST-EFFORT -- the runtime import + # auto-skips; the static arm64-slice audit stands in). + - template: /OneBranchPipelines/steps/conda-build-validate-step-posix.yml@self + parameters: + condaSubdir: 'osx-arm64' + condaTargetSubdir: 'osx-arm64' + continueOnError: true + odbcWheelDir: '$(Pipeline.Workspace)/odbc_wheels' + odbcWheelFilter: 'mssql_python_odbc-*macosx*universal2.whl' + pythonVersions: '${{ parameters.pythonVersions }}' + # osx-64: NATIVE on the Intel agent (BLOCKING -- real import + driver-load proof). + - template: /OneBranchPipelines/steps/conda-build-validate-step-posix.yml@self + parameters: + condaSubdir: 'osx-64' + condaTargetSubdir: 'osx-64' + odbcWheelDir: '$(Pipeline.Workspace)/odbc_wheels' + odbcWheelFilter: 'mssql_python_odbc-*macosx*universal2.whl' + pythonVersions: '${{ parameters.pythonVersions }}' + - task: PublishPipelineArtifact@1 + displayName: 'Publish macOS conda artifact' + inputs: + targetPath: '$(ob_outputDirectory)' + artifact: 'drop_CondaMacOS_BuildConda' + publishLocation: 'pipeline' + + # ========================= + # CONDA linux-64 + linux-aarch64 (x86_64 agent: native + QEMU cross) + # ========================= + - stage: CondaLinux + displayName: 'Conda Linux (linux-64 + linux-aarch64)' + jobs: + - job: BuildConda + displayName: 'Build + validate Linux conda' + timeoutInMinutes: 120 + pool: + type: linux + isCustom: true + name: Azure Pipelines + vmImage: 'ubuntu-latest' + variables: + ob_outputDirectory: '$(Build.ArtifactStagingDirectory)' + # H1: activate the runtime gates in build-conda-packages.sh. The TLS probe + # runs when a connection string is supplied; the ldd reachability gate runs + # (fail-closed) only when explicitly enabled on a minimal base. + CONDA_TLS_PROBE_CONN: ${{ parameters.condaTlsProbeConn }} + ${{ if parameters.enableMinimalReachabilityGate }}: + CONDA_ASSERT_PREFIX_REACHABLE: '1' + steps: + - checkout: self + fetchDepth: 1 + - task: DownloadPipelineArtifact@2 + displayName: 'Download mssql-python wheels' + inputs: + buildType: 'specific' + project: '$(System.TeamProject)' + definition: 2199 + buildVersionToDownload: 'specific' + buildId: $(resources.pipeline.buildPipeline.runID) + artifactName: 'drop_Consolidate_ConsolidateArtifacts' + targetPath: '$(Build.SourcesDirectory)' + - task: DownloadPipelineArtifact@2 + displayName: 'Download mssql-python-odbc wheels' + inputs: + buildType: 'specific' + project: '$(System.TeamProject)' + definition: 2199 + buildVersionToDownload: 'specific' + buildId: $(resources.pipeline.buildPipeline.runID) + artifactName: 'drop_ConsolidateOdbc_ConsolidateArtifacts' + targetPath: '$(Pipeline.Workspace)/odbc_wheels' + # linux-64: NATIVE glibc x86_64 host. + - template: /OneBranchPipelines/steps/conda-build-validate-step-posix.yml@self + parameters: + condaSubdir: 'linux-64' + odbcWheelDir: '$(Pipeline.Workspace)/odbc_wheels' + odbcWheelFilter: 'mssql_python_odbc-*manylinux_2_28_x86_64.whl' + pythonVersions: '${{ parameters.pythonVersions }}' + # linux-aarch64: CROSS-target via QEMU binfmt. MANDATORY (blocking): a + # leg that cannot run its own aarch64 import is a real failure, not a + # silent pass -- otherwise a broken aarch64 conda package would ship + # unvalidated. Install the aarch64 glibc loader/libs (libc6-arm64-cross) + # so the emulated aarch64 Python can find /lib/ld-linux-aarch64.so.1, and + # register QEMU binfmt for host execution; both must succeed. + - bash: | + set -euo pipefail + sudo apt-get update + sudo apt-get install -y qemu-user-static binfmt-support libc6-arm64-cross + docker run --rm --privileged multiarch/qemu-user-static --reset -p yes + displayName: 'Install libc6-arm64-cross + register QEMU binfmt (aarch64)' + - template: /OneBranchPipelines/steps/conda-build-validate-step-posix.yml@self + parameters: + condaSubdir: 'linux-aarch64' + condaTargetSubdir: 'linux-aarch64' + odbcWheelDir: '$(Pipeline.Workspace)/odbc_wheels' + odbcWheelFilter: 'mssql_python_odbc-*manylinux_2_28_aarch64.whl' + pythonVersions: '${{ parameters.pythonVersions }}' + - task: PublishPipelineArtifact@1 + displayName: 'Publish Linux conda artifact' + inputs: + targetPath: '$(ob_outputDirectory)' + artifact: 'drop_CondaLinux_BuildConda' + publishLocation: 'pipeline' + + # ========================= + # CONSOLIDATE all conda packages into one artifact + # ========================= + - stage: ConsolidateConda + displayName: 'Consolidate All Conda Packages' + dependsOn: + - CondaWin64 + - CondaMacOS + - CondaLinux + jobs: + - template: /OneBranchPipelines/jobs/consolidate-conda-artifacts-job.yml@self + parameters: + # This standalone pipeline's conda legs publish drop_Conda* artifacts. + downloadItemPattern: | + drop_CondaWin64_*/** + drop_CondaMacOS_*/** + drop_CondaLinux_*/** diff --git a/OneBranchPipelines/conda-release-pipeline.yml b/OneBranchPipelines/conda-release-pipeline.yml new file mode 100644 index 000000000..07f7c46ed --- /dev/null +++ b/OneBranchPipelines/conda-release-pipeline.yml @@ -0,0 +1,146 @@ +# ========================================================================================= +# OneBranch STANDALONE Conda Release Pipeline (mssql-python) +# ========================================================================================= +# Decoupled from the wheel/PyPI release. Downloads the consolidated conda packages produced +# by the STANDALONE conda-build pipeline (conda-build-pipeline.yml, artifact +# drop_ConsolidateConda_ConsolidateArtifacts) -- NOT the wheel pipeline (def 2199), which no +# longer produces any conda artifact -- enforces the release-readiness gate (conda-release- +# step: required subdirs + full Python matrix + one version), and -- when publishToConda=true -- publishes the SELF-CONTAINED +# mssql-python conda package to Anaconda.org via anaconda-client (conda-publish-step; ESRP +# has no Conda ContentType). There is a single self-contained package (it vendors the ODBC +# Driver 18 payload); no companion. +# +# Always Official (a publish is a deliberate, gated action). Manual trigger only. +# The OneBranch YAML needs an actual ADO run to fully validate. +# ========================================================================================= +name: $(Year:YY)$(DayOfYear)$(Rev:.r)-CondaRelease + +trigger: none +pr: none + +parameters: + - name: publishToConda + displayName: 'Publish Conda Packages to Anaconda.org (PRODUCTION)' + type: boolean + default: false # Safety: default to a validate-only dry run. + - name: condaChannel + displayName: 'Anaconda.org channel/org to publish to' + type: string + default: 'microsoft' + - name: condaLabel + displayName: 'Anaconda.org channel label' + type: string + default: 'main' + # Infra setup: ADO definition id of the STANDALONE conda-build pipeline + # (conda-build-pipeline.yml), whose ConsolidateConda stage produces + # drop_ConsolidateConda_ConsolidateArtifacts. This is NOT the wheel pipeline + # (def 2199). Set it once the conda-build pipeline is registered in ADO; 0 is a + # placeholder that must be overridden before a real run. + - name: condaBuildDefinitionId + displayName: 'Conda-build pipeline ADO definition id' + type: number + default: 0 + # The exact mssql-python version being released. When set, the readiness gate + # asserts EVERY conda package matches it (not just internal one-version + # consistency). Leave empty only for a consistency-only dry run. + - name: mssqlPythonVersion + displayName: 'Expected mssql-python release version (e.g. 1.13.0)' + type: string + default: '' + +variables: + - template: /OneBranchPipelines/variables/common-variables.yml@self + - template: /OneBranchPipelines/variables/onebranch-variables.yml@self + # ANACONDA_API_TOKEN lives in this org-scoped group; included ONLY when publishing so a + # validate-only run never requires the group to exist. + - ${{ if eq(parameters.publishToConda, true) }}: + - group: 'Anaconda Publishing' + +resources: + repositories: + - repository: templates + type: git + name: 'OneBranch.Pipelines/GovernedTemplates' + ref: 'refs/heads/main' + # The build pipeline whose consolidated conda artifact this pipeline validates + publishes. + pipelines: + - pipeline: buildPipeline + # Infra setup: the STANDALONE conda-build pipeline (backed by + # conda-build-pipeline.yml), whose ConsolidateConda stage publishes + # drop_ConsolidateConda_ConsolidateArtifacts. Update to its exact ADO + # definition name once registered. NOT the wheel pipeline + # (Build-Release-Package-Pipeline / def 2199), which no longer builds conda. + source: 'Conda-Build-Pipeline' + trigger: none + +extends: + template: 'v2/OneBranch.Official.CrossPlat.yml@templates' + parameters: + featureFlags: + WindowsHostVersion: + Version: '2022' + globalSdl: + baseline: + baselineFile: $(Build.SourcesDirectory)/.gdn/.gdnbaselines + suppressionSet: default + suppression: + suppressionFile: $(Build.SourcesDirectory)/.gdn/.gdnsuppress + suppressionSet: default + binskim: + enabled: true + break: true + credscan: + enabled: true + policheck: + enabled: true + break: true + exclusionFile: '$(REPO_ROOT)/.config/PolicheckExclusions.xml' + publishLogs: + enabled: true + tsa: + enabled: true + configFile: '$(REPO_ROOT)/.config/tsaoptions.json' + + stages: + - stage: CondaRelease + displayName: 'Validate & Publish Conda Release' + jobs: + # Gate: prove the consolidated conda set is complete (required subdirs + + # full Python matrix + one version) before anything is published. + - job: ValidateConda + displayName: 'Validate consolidated conda packages' + pool: + type: windows + isCustom: true + name: Python-1ES-pool + demands: + - imageOverride -equals PYTHON-1ES-MMS2022 + variables: + ob_outputDirectory: '$(Build.ArtifactStagingDirectory)' + steps: + - template: /OneBranchPipelines/steps/conda-release-step.yml@self + parameters: + buildDefinitionId: ${{ parameters.condaBuildDefinitionId }} + mssqlPythonVersion: '${{ parameters.mssqlPythonVersion }}' + + # PRODUCTION publish (releaseJob) -- runs ONLY when publishToConda=true and + # ONLY after the ValidateConda gate succeeds, so an incomplete set is never + # uploaded. anaconda-client reads ANACONDA_API_TOKEN from the env (never the + # command line), so the token never appears in the logs. + - ${{ if eq(parameters.publishToConda, true) }}: + - job: PublishConda + displayName: 'Publish conda packages to Anaconda.org' + dependsOn: ValidateConda + templateContext: + type: releaseJob + isProduction: true + pool: + type: windows + variables: + ob_outputDirectory: '$(Build.ArtifactStagingDirectory)' + steps: + - template: /OneBranchPipelines/steps/conda-publish-step.yml@self + parameters: + buildDefinitionId: ${{ parameters.condaBuildDefinitionId }} + condaChannel: '${{ parameters.condaChannel }}' + condaLabel: '${{ parameters.condaLabel }}' diff --git a/OneBranchPipelines/jobs/consolidate-conda-artifacts-job.yml b/OneBranchPipelines/jobs/consolidate-conda-artifacts-job.yml new file mode 100644 index 000000000..5dddb12dc --- /dev/null +++ b/OneBranchPipelines/jobs/consolidate-conda-artifacts-job.yml @@ -0,0 +1,126 @@ +# Consolidate Conda Artifacts Job Template +# Collects the per-platform self-contained mssql-python conda packages (which vendor +# the ODBC payload) that each build leg staged under conda// and gathers +# them into a single conda/ tree for the release pipeline to publish. +# +# BEST-EFFORT (build pipeline): conda is a downstream repackage of the ESRP-signed +# wheels and must NEVER block the primary wheel deliverable, so a missing/short set +# only WARNS here. The HARD count gate lives in the release pipeline (which refuses +# to publish an incomplete conda set), symmetric with how the wheel/odbc drops are +# best-effort collected in the build and gated at release time. +# +# Expected packages (validated conda subdirs). The self-contained mssql-python +# package (which vendors the ODBC payload) is emitted per-Python by each build leg; +# there is NO separate companion package: +# win-64 : 5 py x mssql-python = 5 +# osx-64 : 5 py x mssql-python (Intel Mac, cross-built via Rosetta) = 5 +# osx-arm64 : 5 py x mssql-python (Apple Silicon, native) = 5 +# linux-64 : 5 py x mssql-python (glibc x86_64 host, native) = 5 +# linux-aarch64 : 5 py x mssql-python (x86_64 host + QEMU, best-effort) = 5 +# ------------------------------------------------------------------------------ +# TOTAL (PyPI parity minus win-arm64 + musllinux) = 25 +# win-arm64 (no import-validation host on x64) and musllinux (no conda musl subdir) +# are intentionally NOT conda-built. This job is BEST-EFFORT and never hard-fails on +# a short set; the release pipeline's conda-release-step enforces the hard gate +# (required subdirs present + complete Python matrix) before anything is published. +parameters: + - name: oneBranchType + type: string + default: 'Official' + # Artifact item pattern the consolidate job downloads. Defaults to the integrated + # wheel-pipeline leg artifacts; the standalone conda-build pipeline overrides it with + # its drop_Conda* leg artifacts. + - name: downloadItemPattern + type: string + default: | + drop_Win_*/** + drop_MacOS_*/** + drop_Linux_*/** + drop_ODBC_BuildAll_*/** + +jobs: + - job: ConsolidateArtifacts + displayName: 'Consolidate All Conda Packages' + condition: succeeded() + + pool: + type: linux + isCustom: true + name: Azure Pipelines + vmImage: 'ubuntu-latest' + + variables: + # Consolidation only moves files; no binaries to scan. + - name: ob_sdl_binskim_enabled + value: false + - name: ob_outputDirectory + value: '$(Build.ArtifactStagingDirectory)' + + steps: + - checkout: self + fetchDepth: 1 + + # The conda packages are staged INSIDE the mssql-python build-leg artifacts + # (drop_Win_*, drop_MacOS_*, drop_Linux_*) under conda//. Scope the + # download to those stages so every leg's self-contained mssql-python conda is + # gathered in one place. (drop_ODBC_BuildAll_* is included only for its wheels, + # which ride along and are ignored below -- we pick only *.conda / *.tar.bz2.) + - task: DownloadPipelineArtifact@2 + displayName: 'Download All Platform Artifacts' + inputs: + buildType: 'current' + itemPattern: ${{ parameters.downloadItemPattern }} + targetPath: '$(Pipeline.Workspace)/all-artifacts' + + - bash: | + set -e + echo "Collecting conda packages (preserving / layout)..." + mkdir -p $(ob_outputDirectory)/conda + + # Copy every mssql-python* conda package into conda//. Each build + # leg wrote the self-contained mssql-python package under a conda// + # folder, so the parent dir name IS the target subdir. + found=0 + while IFS= read -r p; do + subdir=$(basename "$(dirname "$p")") + mkdir -p "$(ob_outputDirectory)/conda/$subdir" + cp -v "$p" "$(ob_outputDirectory)/conda/$subdir/" + found=1 + done < <(find $(Pipeline.Workspace)/all-artifacts -type f \( -name 'mssql-python*.conda' -o -name 'mssql-python*.tar.bz2' \)) + + echo "" + echo "Consolidated conda tree:" + find $(ob_outputDirectory)/conda -type f | sort + + PKG_COUNT=$(find $(ob_outputDirectory)/conda -type f \( -name '*.conda' -o -name '*.tar.bz2' \) | wc -l) + echo "" + echo "Per-subdir conda package counts:" + for d in $(ob_outputDirectory)/conda/*/; do + [ -d "$d" ] || continue + sub=$(basename "$d") + n=$(find "$d" -type f \( -name '*.conda' -o -name '*.tar.bz2' \) | wc -l) + printf ' %-14s %s\n' "$sub" "$n" + done + echo "Total conda package count: $PKG_COUNT (full PyPI-parity set = 25)" + + # BEST-EFFORT: warn only, never exit non-zero — a conda hiccup on any leg + # must not fail this build or block the wheel release. The release pipeline's + # conda-release-step enforces the hard gate (required subdirs + full Python + # matrix) before anything is published. + if [ "$found" != "1" ]; then + echo "##vso[task.logissue type=warning]No conda packages found in the build-leg artifacts." + else + echo "Collected $PKG_COUNT conda package(s) (best-effort; release-time gate enforces completeness)." + fi + displayName: 'Consolidate conda packages' + + - task: PublishPipelineArtifact@1 + displayName: 'Publish Consolidated Conda Artifacts' + inputs: + targetPath: '$(ob_outputDirectory)' + # Distinct name so it does not collide with the wheel consolidate artifact + # (drop_Consolidate_ConsolidateArtifacts) or the odbc one + # (drop_ConsolidateOdbc_ConsolidateArtifacts) in the same run. Matches the + # OneBranch auto-name for a stage named `ConsolidateConda`. + artifact: 'drop_ConsolidateConda_ConsolidateArtifacts' + publishLocation: 'pipeline' diff --git a/OneBranchPipelines/scripts/.gitattributes b/OneBranchPipelines/scripts/.gitattributes new file mode 100644 index 000000000..dfdb8b771 --- /dev/null +++ b/OneBranchPipelines/scripts/.gitattributes @@ -0,0 +1 @@ +*.sh text eol=lf diff --git a/OneBranchPipelines/scripts/build-conda-packages.ps1 b/OneBranchPipelines/scripts/build-conda-packages.ps1 new file mode 100644 index 000000000..0e7f79377 --- /dev/null +++ b/OneBranchPipelines/scripts/build-conda-packages.ps1 @@ -0,0 +1,276 @@ +<# +.SYNOPSIS + Build and validate the self-contained mssql-python conda package (which vendors the + ODBC Driver 18 payload) from prebuilt (ESRP-signed) wheels, fully offline. + +.DESCRIPTION + Repackages the wheels produced by build definition 2199 into conda packages using + conda-build, then proves the recipes are correct by solving a fresh environment + from the freshly built local channel and importing both packages. + + Runs on the OneBranch Windows 1ES pool (or locally). Builds the win_amd64 slice + for every Python version detected among the mssql_python wheels. Other platforms + (linux-*, osx-*, win_arm64) must be built on matching agents in a follow-up, the + same way the wheel build matrix fans out. + +.PARAMETER WheelsDir + Directory containing ALL downloaded wheels (both packages, all platforms/pythons). + +.PARAMETER RecipeRoot + Path to the repo's conda/ directory (contains mssql-python/ and mssql-python-odbc/). + +.PARAMETER OutputDir + Space-free working/output directory (conda croot, Miniforge install, built pkgs). + +.PARAMETER MssqlPythonVersion + Version to stamp on the mssql-python conda package (e.g. 1.13.0). + +.PARAMETER OdbcVersion + Version to stamp on the mssql-python-odbc conda package (e.g. 18.6.2.1). + +.PARAMETER PythonVersions + Optional comma-separated list (e.g. "3.11,3.12"). Empty = auto-detect from wheels. + +.PARAMETER CondaSubdir + Optional target subdir (e.g. win-arm64) to CROSS-target via CONDA_SUBDIR instead of + the host's native subdir. Empty = build the host's native subdir (win-64). Cross- + targeting only yields a VALIDATED package when the host can run the target Python for + the import check, so it is left unset for the native win-64 leg. + +.PARAMETER Package + Which package(s) to build: + 'all' - companion (ONCE) + binding (per-Python) [default] + 'odbc' - ONLY the Python-agnostic companion, built ONCE (ODBC_BuildAll stage); + validated by importing it under each target Python. + 'binding' - ONLY the per-Python binding; the companion is seeded from + -DriverCondaDir into the local channel so the version-locked + `mssql-python-odbc ==` dependency resolves for the solve/import. + +.PARAMETER DriverCondaDir + Folder holding a prebuilt companion .conda (mssql-python-odbc) under a / + layout, to seed into the local channel (binding mode) instead of rebuilding the + companion per-Python. Empty in 'all'/'odbc' mode. +#> +[CmdletBinding()] +param( + [Parameter(Mandatory = $true)][string]$WheelsDir, + [Parameter(Mandatory = $true)][string]$RecipeRoot, + [Parameter(Mandatory = $true)][string]$OutputDir, + [Parameter(Mandatory = $true)][string]$MssqlPythonVersion, + [Parameter(Mandatory = $true)][string]$OdbcVersion, + [string]$PythonVersions = "", + [string]$CondaSubdir = "", + [ValidateSet('all', 'odbc', 'binding')] + [string]$Package = 'all', + [string]$DriverCondaDir = "" +) + +$ErrorActionPreference = 'Stop' + +function Assert-LastExit([string]$Message) { + if ($LASTEXITCODE -ne 0) { + Write-Error "FAILED (exit $LASTEXITCODE): $Message" + exit 1 + } +} + +Write-Host "==================== conda build inputs ====================" +Write-Host "WheelsDir : $WheelsDir" +Write-Host "RecipeRoot : $RecipeRoot" +Write-Host "OutputDir : $OutputDir" +Write-Host "MssqlPythonVersion : $MssqlPythonVersion" +Write-Host "OdbcVersion : $OdbcVersion" +Write-Host "PythonVersions : $(if ($PythonVersions) { $PythonVersions } else { '(auto-detect)' })" +Write-Host "CondaSubdir : $(if ($CondaSubdir) { $CondaSubdir } else { '(native)' })" +Write-Host "============================================================" + +New-Item -ItemType Directory -Force -Path $OutputDir | Out-Null +$bld = Join-Path $OutputDir 'bld' +New-Item -ItemType Directory -Force -Path $bld | Out-Null + +# --------------------------------------------------------------------------- +# 1. Locate conda, or install Miniforge3 (conda-forge defaults, no license issues) +# --------------------------------------------------------------------------- +$conda = (Get-Command conda -ErrorAction SilentlyContinue).Source +if (-not $conda) { + Write-Host "=== conda not found on PATH; installing Miniforge3 ===" + $installer = Join-Path $OutputDir 'Miniforge3-Windows-x86_64.exe' + $forgeDir = Join-Path $OutputDir 'miniforge' + # Pin Miniforge to a specific release for reproducible, supply-chain-safe builds + # (never 'latest', which floats). Override with MINIFORGE_VERSION. When + # MINIFORGE_SHA256 is set (this installer's checksum for the pinned version) the + # download is verified BEFORE it is executed -- set it in the pipeline to make + # integrity checking mandatory. + $mfver = if ($env:MINIFORGE_VERSION) { $env:MINIFORGE_VERSION } else { '26.3.2-3' } + $url = "https://github.com/conda-forge/miniforge/releases/download/$mfver/Miniforge3-Windows-x86_64.exe" + Write-Host "Downloading pinned Miniforge ${mfver}: $url" + Invoke-WebRequest -Uri $url -OutFile $installer + if ($env:MINIFORGE_SHA256) { + $actual = (Get-FileHash -Algorithm SHA256 -Path $installer).Hash + if ($actual -ne $env:MINIFORGE_SHA256) { + Write-Error "Miniforge installer SHA256 mismatch: expected '$($env:MINIFORGE_SHA256)', got '$actual'." + exit 1 + } + Write-Host "Miniforge installer SHA256 verified." + } + else { + Write-Host "WARNING: MINIFORGE_SHA256 not set -- skipping installer integrity check. Set it in the pipeline to the pinned checksum for $mfver." + } + # NSIS silent install; /D (target dir) MUST be last and unquoted. + Start-Process -FilePath $installer -ArgumentList '/S', '/InstallationType=JustMe', '/AddToPath=0', "/D=$forgeDir" -Wait + $conda = Join-Path $forgeDir 'Scripts\conda.exe' +} +if (-not (Test-Path $conda)) { + Write-Error "conda not available at '$conda' after install attempt." + exit 1 +} +Write-Host "Using conda: $conda" +& $conda --version +Assert-LastExit "conda --version" + +# --------------------------------------------------------------------------- +# 2. Install conda-build (pinned to the stable pre-26 series) +# --------------------------------------------------------------------------- +# Pin conda-build<26: the bleeding-edge 26.7.0 crashes with an internal +# "An unexpected error has occurred" during the LOCAL packaging phase (right +# after "Fixing permissions"); 26.7.1 is not yet released. The mature 25.x +# series builds these recipes cleanly and supports every key we use. +# NOTE: anaconda-client is intentionally NOT installed here — this script only +# builds + validates (it never runs `anaconda upload`). Publishing installs its +# own anaconda-client in conda-publish-step.yml. Keeping it out of the build env +# also drops the anaconda-auth conda plugin, which the crash report fingered. +Write-Host "=== installing conda-build (<26) ===" +& $conda install -y -n base "conda-build<26" +Assert-LastExit "conda install conda-build<26" + +# --------------------------------------------------------------------------- +# 3. Determine which Python versions to build (win_amd64 mssql_python wheels) +# --------------------------------------------------------------------------- +if ([string]::IsNullOrWhiteSpace($PythonVersions)) { + $pyvers = Get-ChildItem -Path $WheelsDir -Filter 'mssql_python-*win_amd64.whl' | + ForEach-Object { if ($_.Name -match 'cp3(\d+)') { "3.$($Matches[1])" } } | + Sort-Object -Unique +} +else { + $pyvers = $PythonVersions.Split(',') | ForEach-Object { $_.Trim() } | Where-Object { $_ } +} +if (-not $pyvers) { + Write-Error "No win_amd64 mssql_python wheels found in '$WheelsDir' to determine Python versions." + exit 1 +} +Write-Host "Building conda packages for Python versions: $($pyvers -join ', ')" + +# --------------------------------------------------------------------------- +# 4. Export the environment consumed by the recipes (jinja + build scripts) +# --------------------------------------------------------------------------- +$env:WHEELS_DIR = $WheelsDir +$env:MSSQL_PYTHON_VERSION = $MssqlPythonVersion +$env:MSSQL_ODBC_VERSION = $OdbcVersion + +# CROSS-target a non-native subdir when requested: conda-build and the verify env's +# `conda create` both honor CONDA_SUBDIR, so the packages are stamped for $CondaSubdir +# and the import check runs the target Python (via Rosetta 2 / QEMU on an emulating +# host). Empty = build the host's native subdir. +if ($CondaSubdir) { + $env:CONDA_SUBDIR = $CondaSubdir + Write-Host "Cross-targeting conda subdir: CONDA_SUBDIR=$($env:CONDA_SUBDIR)" +} + +# --------------------------------------------------------------------------- +# 5. Build the self-contained mssql-python package (per Python). The recipe vendors +# the ODBC Driver 18 payload by extracting the mssql-python-odbc wheel into its +# own site-packages, so there is NO separate companion package to build. +# --------------------------------------------------------------------------- +$bindRecipe = Join-Path $RecipeRoot 'mssql-python' + +if ($Package -eq 'odbc') { + Write-Host "NOTE: -Package odbc is a no-op in the self-contained model (the ODBC payload" + Write-Host "is vendored INTO mssql-python; there is no separate companion). Nothing to build." +} +else { + foreach ($py in $pyvers) { + Write-Host "=== [py $py] build mssql-python (self-contained: vendors the ODBC payload) ===" + & $conda build $bindRecipe --python $py --no-test --no-anaconda-upload --output-folder $bld + Assert-LastExit "conda build mssql-python (py $py)" + } +} + +# --------------------------------------------------------------------------- +# 6. Make the local output folder a VALID conda channel. +# conda-build --output-folder already wrote $bld\\repodata.json for the +# platform we built, but a conda channel is only valid if it ALSO carries +# noarch\repodata.json (even empty) -- otherwise `conda create -c file://$bld` +# fails with "UnavailableInvalidChannel ... must contain noarch/repodata.json". +# Create it directly rather than via `conda index`, whose subcommand is absent +# from miniforge (it moved to the standalone conda-index package). +# --------------------------------------------------------------------------- +$noarchDir = Join-Path $bld 'noarch' +New-Item -ItemType Directory -Force -Path $noarchDir | Out-Null +$noarchRepo = Join-Path $noarchDir 'repodata.json' +if (-not (Test-Path $noarchRepo)) { + '{"info":{"subdir":"noarch"},"packages":{},"packages.conda":{}}' | Set-Content -NoNewline -Encoding ascii $noarchRepo +} + +# --------------------------------------------------------------------------- +# 6b. Masking-immune RUNPATH audit of the freshly built packages (#563). +# --------------------------------------------------------------------------- +# BLOCKING static gate that reads the ELF RUNPATH bytes of the vendored Linux ODBC +# binaries and requires the relative $ORIGIN climb. win-64 packages carry no ELF +# payload so this is a clean no-op here, but it is wired on EVERY leg so a Linux +# package can never reach publish without the #563 self-containment being proven. +$auditScript = Join-Path (Split-Path $RecipeRoot -Parent) 'eng/scripts/audit_bundled_binaries.py' +if (-not (Test-Path $auditScript)) { + Write-Error "RUNPATH audit script not found at $auditScript" + exit 1 +} +Write-Host "=== RUNPATH self-containment audit (eng/scripts/audit_bundled_binaries.py) ===" +& $conda run -n base python -m pip install --quiet --disable-pip-version-check zstandard +& $conda run -n base python $auditScript --root $bld +Assert-LastExit "RUNPATH self-containment audit" + +# --------------------------------------------------------------------------- +# 7. Validate: solve a fresh env from the local channel and import the package. +# Proves azure-identity + the folded-in openssl/krb5 deps resolve AND that the +# repackaged native binding imports with its vendored ODBC payload (driver loads +# at import). +# --------------------------------------------------------------------------- +$localChannel = "file:///" + ($bld -replace '\\', '/') +if ($Package -eq 'odbc') { + Write-Host "NOTE: -Package odbc is a no-op in the self-contained model; nothing to validate." +} +else { + # Run the verify imports from a NEUTRAL dir: `python -c` prepends the cwd to + # sys.path, and the pipeline runs from the repo checkout whose in-tree + # mssql_python\ (source, no compiled .pyd) would shadow the conda-installed + # package -> "No ddbc_bindings module found". $OutputDir is outside the repo. + Set-Location $OutputDir + foreach ($py in $pyvers) { + $envName = "verify_" + ($py -replace '\.', '') + Write-Host "=== [py $py] create verify env from local channel ===" + # -c microsoft (ahead of conda-forge) so azure-core/azure-identity/msal resolve from the + # lean `microsoft` channel, NOT conda-forge whose azure-core recipe over-declares flask/six + # -> celery/boto3/botocore (~9 MB); see conda-forge/azure-core-feedstock#71. + # --strict-channel-priority keeps the freshly built local package authoritative. + & $conda create -y -n $envName -c $localChannel -c microsoft -c conda-forge --strict-channel-priority --override-channels "python=$py" mssql-python + Assert-LastExit "conda create verify env (py $py)" + + Write-Host "=== [py $py] import mssql_python + prove the vendored ODBC payload is present ===" + & $conda run -n $envName python -c "import mssql_python; print('BINDING_OK', mssql_python.__version__)" + Assert-LastExit "import mssql_python (py $py)" + & $conda run -n $envName python -c "import mssql_python_odbc; print('ODBC_PAYLOAD_OK', mssql_python_odbc.__version__)" + Assert-LastExit "import mssql_python_odbc (py $py)" + + Write-Host "=== [py $py] DB-less driver-load proof (real ODBC driver must load, not just the shim) ===" + & $conda run -n $envName python (Join-Path $RecipeRoot 'driver_load_probe.py') + Assert-LastExit "driver-load proof (py $py)" + + Write-Host "=== [py $py] confirm resolved dependencies ===" + & $conda list -n $envName | Select-String -Pattern 'azure-identity|mssql-python|openssl|krb5' + } +} + +Write-Host "==================== built conda artifacts ====================" +Get-ChildItem -Path $bld -Recurse -Include *.conda, *.tar.bz2 | +Where-Object { $_.Name -like 'mssql-python*' } | +ForEach-Object { Write-Host " $($_.FullName)" } +Write-Host "CONDA_BUILD_OK" diff --git a/OneBranchPipelines/scripts/build-conda-packages.sh b/OneBranchPipelines/scripts/build-conda-packages.sh new file mode 100644 index 000000000..8efaf194c --- /dev/null +++ b/OneBranchPipelines/scripts/build-conda-packages.sh @@ -0,0 +1,409 @@ +#!/usr/bin/env bash +# Build + validate the self-contained mssql-python conda package from prebuilt +# (ESRP-signed) wheels, fully offline via a local --find-links dir. The package +# VENDORS the ODBC Driver 18 payload -- the recipe extracts BOTH the code wheel and +# the mssql-python-odbc wheel into one site-packages -- so there is NO separate +# conda package (the v1.11.0 model). +# ============================================================================ +# Bash port of build-conda-packages.ps1 for the macOS and Linux build legs. +# conda-build provisions a per-subdir HOST env and installs the matching wheel +# (see conda/*/build.sh). It runs NATIVELY for linux-64 and osx-64, under QEMU +# binfmt for linux-aarch64, and as a CROSS-build for osx-arm64 on the Intel macOS +# agent (there is no reverse Rosetta, so the arm64 Python is never executed -- +# conda/*/build.sh extract the universal2 wheel without Python and the section-7 +# runtime import is skipped; the pipeline's static arm64-slice audit stands in). +# +# Args: +# $1 WheelsDir find-links dir holding the mssql-python + mssql-python-odbc wheels +# $2 RecipeRoot repo conda/ dir (mssql-python/ + mssql-python-odbc/) +# $3 OutputDir space-free work/output dir (Miniforge + croot + built pkgs) +# $4 MssqlPythonVersion version to stamp on mssql-python +# $5 OdbcVersion version to stamp on mssql-python-odbc +# $6 PythonVersions optional comma-separated (e.g. "3.11,3.12"); empty = auto-detect +# $7 CondaSubdir optional target subdir (e.g. osx-64, osx-arm64, +# linux-aarch64) to CROSS-target via CONDA_SUBDIR; empty = +# build the host's native subdir. The section-7 runtime +# import validation requires the host to be able to RUN the +# target's Python -- true natively, under Rosetta 2 (osx-64 +# on Apple Silicon) and under QEMU binfmt (linux-aarch64 on +# x86_64). For osx-arm64 on the Intel agent it is NOT, so +# that leg auto-skips the import (static arch audit stands in). +set -euo pipefail + +WheelsDir="${1:?WheelsDir required}" +RecipeRoot="${2:?RecipeRoot required}" +OutputDir="${3:?OutputDir required}" +MssqlPythonVersion="${4:?MssqlPythonVersion required}" +OdbcVersion="${5:?OdbcVersion required}" +PythonVersions="${6:-}" +CondaSubdir="${7:-}" + +echo "==================== conda build inputs ====================" +echo "WheelsDir : $WheelsDir" +echo "RecipeRoot : $RecipeRoot" +echo "OutputDir : $OutputDir" +echo "MssqlPythonVersion : $MssqlPythonVersion" +echo "OdbcVersion : $OdbcVersion" +echo "PythonVersions : ${PythonVersions:-(auto-detect)}" +echo "CondaSubdir : ${CondaSubdir:-(native)}" +echo "============================================================" + +mkdir -p "$OutputDir" +bld="$OutputDir/bld" +mkdir -p "$bld" + +# --------------------------------------------------------------------------- +# 1. Locate conda, or install Miniforge3 (conda-forge defaults) for THIS platform +# --------------------------------------------------------------------------- +conda="$(command -v conda || true)" +# Reuse an existing Miniforge install if a previous run already created one. On +# macOS the universal2 build invokes this script once per subdir (osx-64 AND +# osx-arm64) on the SAME agent, sharing $OutputDir; each run is a fresh shell so +# `command -v conda` is empty even though miniforge/ already exists. Without this +# guard the second run re-runs the installer into the existing dir and fails with +# "File or directory already exists: .../conda-bld/miniforge". +if [ -z "$conda" ] && [ -x "$OutputDir/miniforge/bin/conda" ]; then + echo "=== reusing existing Miniforge3 at $OutputDir/miniforge ===" + conda="$OutputDir/miniforge/bin/conda" +fi +if [ -z "$conda" ]; then + echo "=== conda not found on PATH; installing Miniforge3 ===" + os="$(uname -s)"; arch="$(uname -m)" + case "$os-$arch" in + Darwin-arm64) mf="Miniforge3-MacOSX-arm64.sh" ;; + Darwin-x86_64) mf="Miniforge3-MacOSX-x86_64.sh" ;; + Linux-x86_64) mf="Miniforge3-Linux-x86_64.sh" ;; + Linux-aarch64) mf="Miniforge3-Linux-aarch64.sh" ;; + *) echo "ERROR: unsupported platform '$os-$arch' for Miniforge" >&2; exit 1 ;; + esac + forgeDir="$OutputDir/miniforge" + installer="$OutputDir/$mf" + # Pin Miniforge to a specific release for reproducible, supply-chain-safe builds + # (never `latest`, which floats to whatever upstream publishes next). Override with + # MINIFORGE_VERSION. When MINIFORGE_SHA256 is set (to this arch's installer checksum + # for the pinned version) the download is verified BEFORE it is executed -- set it + # in the pipeline to make integrity checking mandatory. + mfver="${MINIFORGE_VERSION:-26.3.2-3}" + url="https://github.com/conda-forge/miniforge/releases/download/${mfver}/$mf" + echo "Downloading pinned Miniforge $mfver: $url" + curl -fL "$url" -o "$installer" + if [ -n "${MINIFORGE_SHA256:-}" ]; then + if command -v sha256sum >/dev/null 2>&1; then + actual="$(sha256sum "$installer" | awk '{print $1}')" + else + actual="$(shasum -a 256 "$installer" | awk '{print $1}')" + fi + if [ "$actual" != "$MINIFORGE_SHA256" ]; then + echo "ERROR: Miniforge installer SHA256 mismatch: expected '$MINIFORGE_SHA256', got '$actual'." >&2 + exit 1 + fi + echo "Miniforge installer SHA256 verified." + else + echo "WARNING: MINIFORGE_SHA256 not set -- skipping installer integrity check. Set it in the pipeline to the pinned checksum of $mf @ $mfver." + fi + # -u = update/reuse an existing target dir instead of erroring, in case a prior + # run left a partial miniforge/ behind that failed the reuse check above. + bash "$installer" -b -u -p "$forgeDir" + conda="$forgeDir/bin/conda" +fi +if ! "$conda" --version >/dev/null 2>&1; then + echo "ERROR: conda not available at '$conda' after install attempt." >&2 + exit 1 +fi +echo "Using conda: $conda" +"$conda" --version + +# --------------------------------------------------------------------------- +# 2. Install conda-build (pinned to the stable pre-26 series) +# --------------------------------------------------------------------------- +# Pin conda-build<26: the bleeding-edge 26.7.0 crashes with an internal +# "An unexpected error has occurred" during the LOCAL packaging phase (right +# after "Fixing permissions"); 26.7.1 is not yet released. The mature 25.x +# series builds these recipes cleanly and supports every key we use. +# NOTE: anaconda-client is intentionally NOT installed here — this script only +# builds + validates (it never runs `anaconda upload`). Publishing installs its +# own anaconda-client in conda-publish-step.yml. Keeping it out of the build env +# also drops the anaconda-auth conda plugin, which the crash report fingered. +# +# Use a DEDICATED env instead of `install -n base`: a pre-installed conda whose base +# is pinned to a too-new python (the GitHub-hosted runner's Miniconda pins python +# 3.14, which no conda-build<26 supports) makes a base install UNSOLVABLE. A fresh env +# lets conda pick a python conda-build<26 supports, independent of the base pin. +# conda-forge only (--override-channels) avoids the defaults-channel ToS; zstandard +# rides along so the RUNPATH audit reads .conda metadata from this same env. +condaBuildEnv="conda_builder" +echo "=== creating dedicated conda-build env ($condaBuildEnv: conda-build<26) ===" +"$conda" create -y -n "$condaBuildEnv" -c conda-forge --override-channels "conda-build<26" zstandard + +# --------------------------------------------------------------------------- +# 3. Determine which Python versions to build (auto-detect from mssql_python wheels) +# --------------------------------------------------------------------------- +if [ -z "$PythonVersions" ]; then + pyvers="$(ls "$WheelsDir"/mssql_python-*.whl 2>/dev/null \ + | grep -v 'mssql_python_odbc' \ + | sed -nE 's/.*-cp3([0-9]+)-.*/3.\1/p' | sort -u)" +else + pyvers="$(echo "$PythonVersions" | tr ',' '\n' | sed 's/[[:space:]]//g' | grep -v '^$')" +fi +if [ -z "$pyvers" ]; then + echo "ERROR: no mssql_python wheels in '$WheelsDir' to determine Python versions." >&2 + exit 1 +fi +echo "Building conda packages for Python versions: $(echo "$pyvers" | tr '\n' ' ')" + +# --------------------------------------------------------------------------- +# 4. Export the environment consumed by the recipes (jinja + build scripts) +# --------------------------------------------------------------------------- +export WHEELS_DIR="$WheelsDir" +export MSSQL_PYTHON_VERSION="$MssqlPythonVersion" +export MSSQL_ODBC_VERSION="$OdbcVersion" +# Forward the DEV-ONLY unsigned-patch escape hatch into conda-build (meta.yaml +# script_env allow-lists it, else conda-build's sanitized env drops it). Empty on the +# signed build -> build.sh stays assertion-only; the PR audit gate sets it to 1 to +# patch the un-baked PyPI binaries. +export CONDA_ALLOW_UNSIGNED_PATCH="${CONDA_ALLOW_UNSIGNED_PATCH:-}" + +# Cross-subdir builds: force conda-build AND the verify `conda create` to target the +# requested subdir instead of the host's native one. Both honor CONDA_SUBDIR, so the +# packages are stamped for $CondaSubdir. The section-7 import validation solves that +# subdir and runs the target Python where the host can execute it (natively, under +# Rosetta 2 for osx-64, or under QEMU binfmt for linux-aarch64); on the osx-arm64 +# cross-build (Intel agent, no reverse Rosetta) section 7 auto-detects that the target +# Python can't run and skips the import. Left unset for a native build. +if [ -n "$CondaSubdir" ]; then + export CONDA_SUBDIR="$CondaSubdir" + echo "Cross-targeting conda subdir: CONDA_SUBDIR=$CONDA_SUBDIR" + # Emulated aarch64 cross-build: the verify env's target-arch Python (section 7) + # runs under qemu-user. Point qemu at the aarch64 glibc loader/libs (installed via + # libc6-arm64-cross on the leg) so it can find /lib/ld-linux-aarch64.so.1. Only the + # emulated aarch64 leg has this dir; elsewhere the var is a harmless no-op. + case "$CONDA_SUBDIR" in + *aarch64) + if [ -d /usr/aarch64-linux-gnu ]; then + export QEMU_LD_PREFIX="${QEMU_LD_PREFIX:-/usr/aarch64-linux-gnu}" + echo "Set QEMU_LD_PREFIX=$QEMU_LD_PREFIX for emulated aarch64 verify" + fi + ;; + esac +fi + +# --------------------------------------------------------------------------- +# 5. Build companion FIRST, then the binding, for each Python version +# --------------------------------------------------------------------------- +bindRecipe="$RecipeRoot/mssql-python" +for py in $pyvers; do + echo "=== [py $py] build mssql-python (self-contained: vendors the ODBC payload) ===" + "$conda" run -n "$condaBuildEnv" conda-build "$bindRecipe" --python "$py" --no-test --no-anaconda-upload --output-folder "$bld" +done + +# --------------------------------------------------------------------------- +# 6. Make the local output folder a VALID conda channel. +# conda-build --output-folder already wrote $bld//repodata.json for the +# platform we built, but a conda channel is only valid if it ALSO carries +# noarch/repodata.json (even empty) -- otherwise `conda create -c file://$bld` +# fails with "UnavailableInvalidChannel ... must contain noarch/repodata.json". +# Create it directly rather than via `conda index`, whose subcommand is absent +# from miniforge (it moved to the standalone conda-index package). +# --------------------------------------------------------------------------- +mkdir -p "$bld/noarch" +if [ ! -f "$bld/noarch/repodata.json" ]; then + printf '%s' '{"info":{"subdir":"noarch"},"packages":{},"packages.conda":{}}' > "$bld/noarch/repodata.json" +fi + +# --------------------------------------------------------------------------- +# 6b. Masking-immune RUNPATH audit of the freshly built packages (#563). +# --------------------------------------------------------------------------- +# BLOCKING static gate: read the ELF RUNPATH BYTES of the vendored Linux ODBC +# binaries in every built .conda and require the relative $ORIGIN climb (+ no +# vendored krb5/openssl/libltdl). Immune to the system-lib masking that hides an +# unreachable conda copy from a runtime ldd/import on a full agent; win/osx +# packages carry no ELF payload and are skipped. `set -e` makes a violation abort. +auditScript="$(cd "$(dirname "$RecipeRoot")" && pwd)/eng/scripts/audit_bundled_binaries.py" +if [ ! -f "$auditScript" ]; then + echo "ERROR: RUNPATH audit script not found at $auditScript" >&2 + exit 1 +fi +echo "=== RUNPATH self-containment audit (eng/scripts/audit_bundled_binaries.py) ===" +"$conda" run -n "$condaBuildEnv" python "$auditScript" --root "$bld" + +# --------------------------------------------------------------------------- +# 7. Validate: solve a fresh env from the local channel and import the package. +# Proves azure-identity + the folded-in openssl/krb5 deps resolve AND that the +# repackaged native binding imports with its vendored ODBC payload (driver loads +# at import). +# --------------------------------------------------------------------------- +# Run from a NEUTRAL dir: `python -c` prepends the cwd to sys.path, and the pipeline +# runs from the repo checkout whose in-tree mssql_python/ (source, no compiled +# extension) would shadow the conda-installed package -> "No ddbc_bindings module +# found". $OutputDir is outside the repo checkout. +cd "$OutputDir" +# conda's channel URL parser treats any path COMPONENT equal to a known conda subdir +# (osx-arm64, osx-64, linux-64, linux-aarch64, win-64, noarch) as the platform subdir +# and STRIPS it from the channel root. The pipeline isolates each leg under a +# subdir-NAMED dir (OutputDir=.../), so $bld's path contains that token and +# `conda create -c "$bld"` would resolve to the WRONG, token-stripped path (.../bld, +# which has no repodata) -> "UnavailableInvalidChannel ... must contain noarch/...". +# Copy the freshly built channel (platform subdir + the section-6 noarch stub) into a +# token-FREE, per-leg-unique dir so conda parses the channel path verbatim. +legName="${OutputDir##*/}" # e.g. osx-arm64 / linux-64 +localChannel="$(dirname "$OutputDir")/verifychan_${legName//[^A-Za-z0-9]/_}" +rm -rf "$localChannel"; mkdir -p "$localChannel" +cp -a "$bld"/. "$localChannel"/ +echo "verify channel (token-free alias of $bld): $localChannel" + +# Emulated CROSS leg: the target-arch conda subdir differs from the host arch, so the +# verify Python runs under QEMU binfmt (e.g. linux-aarch64 on an x86_64 agent). The +# Python binding imports fine under qemu-user, but qemu-user CANNOT reliably initialize +# the native unixODBC environment -- SQLAllocEnv does ltdl/pthread/locale init it +# mis-emulates, surfacing as "Failed to allocate environment handle". The SAME driver +# loads under full-arch emulation AND on the native same-arch leg, and the masking- +# immune static RUNPATH audit already proved self-containment. So on an emulated cross +# leg the RUNTIME driver probes (driver-load, ldd reachability, TLS) are BEST-EFFORT; +# build + audit + import stay blocking. +emulated_cross=0 +host_machine="$(uname -m)" +case "${CONDA_SUBDIR:-}" in + *aarch64 | *arm64) + if [ "$host_machine" != "aarch64" ] && [ "$host_machine" != "arm64" ]; then + emulated_cross=1 + echo "NOTE: emulated CROSS leg (CONDA_SUBDIR=$CONDA_SUBDIR on $host_machine host); runtime driver probes are best-effort under QEMU binfmt, build/audit/import remain blocking." + fi + ;; +esac + +for py in $pyvers; do + # Include the target subdir so the two macOS legs (osx-64 + osx-arm64) that run on + # the SAME agent never collide on the env name, and recreate cleanly so a re-run + # or a leftover env can't fail `conda create`. + sub="${CONDA_SUBDIR:-native}"; sub="${sub//-/_}" + envName="verify_${sub}_${py//./}" + "$conda" env remove -y -n "$envName" >/dev/null 2>&1 || true + echo "=== [py $py] create verify env from local channel ===" + # -c microsoft (ahead of conda-forge) so azure-core/azure-identity/msal resolve from the + # lean `microsoft` channel, NOT conda-forge whose azure-core recipe over-declares flask/six + # -> celery/boto3/botocore (~9 MB); see conda-forge/azure-core-feedstock#71. + # --strict-channel-priority keeps the freshly built local package authoritative. + "$conda" create -y -n "$envName" -c "$localChannel" -c microsoft -c conda-forge --strict-channel-priority --override-channels "python=$py" mssql-python + # Whether the freshly built package's Python can EXECUTE on this host. + target_runnable=1 + "$conda" run -n "$envName" python -c "import sys" >/dev/null 2>&1 || target_runnable=0 + if [ "$target_runnable" = "0" ]; then + # The ONLY leg allowed to skip the runtime proof is the osx-arm64 cross-build on + # an Intel agent (no reverse Rosetta): the arm64 Python genuinely cannot run here, + # and the pipeline's static arm64-slice audit (lipo/otool/file on the arm64 Mach-O + # payload) stands in -- the same assurance as the shipping PyPI universal2 arm64 + # slice. Every OTHER target (linux-64/osx-64 native, linux-aarch64 under QEMU + # binfmt) MUST run its own import; a leg that cannot is a real failure, never a + # silent pass -- otherwise a broken linux-aarch64 package ships unvalidated. + if [ "${CONDA_SUBDIR:-}" = "osx-arm64" ] && [ "$(uname -s)" = "Darwin" ]; then + echo "=== [py $py] osx-arm64 cross on Intel: target Python not executable; skipping runtime import (static arm64-slice audit covers this leg). ===" + continue + fi + echo "ERROR: [py $py] target Python for CONDA_SUBDIR=${CONDA_SUBDIR:-native} is not executable on $(uname -s)/$(uname -m), and this is NOT the osx-arm64 cross-build. Refusing to silently skip validation (linux-aarch64 requires QEMU binfmt to be registered on this leg)." >&2 + exit 1 + fi + echo "=== [py $py] import mssql_python + prove the vendored ODBC payload is present ===" + "$conda" run -n "$envName" python -c "import mssql_python; print('BINDING_OK', mssql_python.__version__)" + "$conda" run -n "$envName" python -c "import mssql_python_odbc; print('ODBC_PAYLOAD_OK', mssql_python_odbc.__version__)" + echo "=== [py $py] DB-less driver-load proof (real ODBC driver must load, not just the shim) ===" + if [ "$emulated_cross" = "1" ]; then + "$conda" run -n "$envName" python "$RecipeRoot/driver_load_probe.py" \ + || echo "SKIP (emulated cross under QEMU binfmt): qemu-user cannot initialize the native ODBC environment (SQLAllocEnv). The masking-immune static RUNPATH audit + the native same-arch leg + full-arch emulation validate the driver; this runtime probe is best-effort on the emulated leg." + else + "$conda" run -n "$envName" python "$RecipeRoot/driver_load_probe.py" + fi + # Minimal-base reachability gate (#563): on a Linux leg with NO system + # krb5/libltdl (set CONDA_ASSERT_PREFIX_REACHABLE=1), prove the vendored driver + # binds the env's OWN $CONDA_PREFIX/lib copies via the $ORIGIN climb -- not a + # system fallthrough that would MASK an unreachable conda lib on a full agent. + # The $ORIGIN climb makes ldd resolve krb5/gssapi/libltdl from $CONDA_PREFIX/lib + # without LD_LIBRARY_PATH; a system or not-found binding fails the leg. + if [ "${CONDA_ASSERT_PREFIX_REACHABLE:-}" = "1" ] && [ "$(uname -s)" = "Linux" ] && [ "$emulated_cross" = "1" ]; then + echo "=== [py $py] reachability gate SKIPPED on the emulated cross leg (qemu-user cannot reliably run the aarch64 driver's ldd/env init); the masking-immune static RUNPATH audit is the authoritative \$ORIGIN-climb guard. ===" + elif [ "${CONDA_ASSERT_PREFIX_REACHABLE:-}" = "1" ] && [ "$(uname -s)" = "Linux" ]; then + echo "=== [py $py] minimal-base ldd reachability gate (driver MUST bind CONDA_PREFIX/lib) ===" + env_prefix="$("$conda" run -n "$envName" python -c 'import os,sys; print(os.environ.get("CONDA_PREFIX") or sys.prefix)')" + # Inspect the SAME driver variant the loader actually binds on THIS host. mssql_python + # (GetDriverPathCpp in ddbc_bindings.cpp) selects libs/linux// by probing + # /etc/*-release; a blind glob instead grabs the alphabetically-first 'alpine' (musl) + # variant, which needs libc.musl (absent on glibc) and whose libodbcinst does NOT link + # libltdl -> a false "libltdl absent" failure. Mirror that selection exactly. + drv="$("$conda" run -n "$envName" python -c 'import mssql_python,glob,os,platform; b=os.path.dirname(mssql_python.__file__); d=("alpine" if os.path.exists("/etc/alpine-release") else "rhel" if (os.path.exists("/etc/redhat-release") or os.path.exists("/etc/centos-release")) else "suse" if (os.path.exists("/etc/SuSE-release") or os.path.exists("/etc/SUSE-brand")) else "debian_ubuntu"); a=("arm64" if platform.machine() in ("aarch64","arm64") else "x86_64"); m=glob.glob(os.path.join(b,"..","mssql_python_odbc","libs","linux",d,a,"lib","libmsodbcsql*")); print(m[0] if m else "")')" + if [ -z "$drv" ]; then + echo "ERROR: [py $py] no libmsodbcsql driver found in the verify env; cannot prove reachability." >&2 + exit 1 + fi + inst="$(dirname "$drv")/libodbcinst.so.2" + # H2: the inspection itself MUST succeed (no `|| true`) -- a failed ldd cannot + # be read as "reachable". Collect the combined transitive ldd of both binaries. + ldd_all="" + for lib in "$drv" "$inst"; do + echo "--- ldd $(basename "$lib") ---" + if ! out="$("$conda" run -n "$envName" ldd "$lib" 2>&1)"; then + echo "$out" + echo "ERROR: [py $py] ldd failed on $(basename "$lib"); cannot verify reachability." >&2 + exit 1 + fi + echo "$out" + ldd_all="$ldd_all +$out" + done + # H2: each required soname MUST be present AND resolve from $env_prefix -- a + # missing (not found) or system binding FAILS closed (never passes on no-match). + reach_fail=0 + for want in libkrb5.so libgssapi_krb5.so libltdl.so; do + hits="$(printf '%s\n' "$ldd_all" | grep -F "$want" || true)" + if [ -z "$hits" ]; then + echo "MISS: required '$want' absent from ldd output (driver stopped resolving it?)." >&2 + reach_fail=1 + continue + fi + n_prefix=0 + n_bad=0 + while IFS= read -r line; do + [ -n "$line" ] || continue + resolved="$(printf '%s' "$line" | sed -nE 's/.*=> +([^ ]+).*/\1/p')" + case "$resolved" in + "$env_prefix"/*) n_prefix=$((n_prefix + 1)); echo "OK $line" ;; + "") n_bad=$((n_bad + 1)); echo "NOTFOUND $line" >&2 ;; + *) n_bad=$((n_bad + 1)); echo "SYSTEM $line" >&2 ;; + esac + done <&2 + reach_fail=1 + fi + done + [ "$reach_fail" = "0" ] || { echo "ERROR: [py $py] reachability gate FAILED -- a required krb5/gssapi/libltdl bound to system or was absent instead of $env_prefix/lib." >&2; exit 1; } + echo "REACHABILITY_OK (krb5 + gssapi_krb5 + libltdl all bound from $env_prefix/lib)" + fi + # Live Encrypt=yes TLS gate -- forces the driver to dlopen its OpenSSL backend + # (libssl/libcrypto), which the DB-less Encrypt=no probe above NEVER exercises. + # Runs (BLOCKING) only when CONDA_TLS_PROBE_CONN points at a reachable server; + # otherwise it SKIPS loudly (it never silently passes). CAVEAT: this is + # conclusive ONLY on a minimal base with NO system OpenSSL -- a system libssl + # lets the driver's dlopen fall through and MASK an unreachable conda + # /lib copy (exactly what full CI agents hide). The masking-IMMUNE guard + # is eng/scripts/audit_bundled_binaries.py, which reads the RUNPATH bytes and + # requires an $ORIGIN/.. climb regardless of any system libs; this gate is the + # complementary end-to-end backstop for a minimal-base leg. + if [ -n "${CONDA_TLS_PROBE_CONN:-}" ]; then + echo "=== [py $py] live Encrypt=yes TLS gate (OpenSSL backend must be reachable) ===" + if [ "$emulated_cross" = "1" ]; then + "$conda" run -n "$envName" python "$RecipeRoot/tls_connect_probe.py" \ + || echo "SKIP (emulated cross under QEMU binfmt): qemu-user cannot run the aarch64 driver's TLS/OpenSSL init; best-effort on the emulated leg (static RUNPATH audit covers OpenSSL layout)." + else + "$conda" run -n "$envName" python "$RecipeRoot/tls_connect_probe.py" + fi + else + echo "=== [py $py] Encrypt=yes TLS gate SKIPPED (set CONDA_TLS_PROBE_CONN on a minimal-base leg to enable) ===" + fi + echo "=== [py $py] confirm resolved dependencies ===" + "$conda" list -n "$envName" | grep -E 'azure-identity|mssql-python|openssl|krb5' || true +done + +echo "==================== built conda artifacts ====================" +find "$bld" -type f \( -name 'mssql-python*.conda' -o -name 'mssql-python*.tar.bz2' \) -print +echo "CONDA_BUILD_OK" diff --git a/OneBranchPipelines/steps/conda-build-validate-step-posix.yml b/OneBranchPipelines/steps/conda-build-validate-step-posix.yml new file mode 100644 index 000000000..e67b621e8 --- /dev/null +++ b/OneBranchPipelines/steps/conda-build-validate-step-posix.yml @@ -0,0 +1,146 @@ +# Conda Build + Validate Step Template (POSIX / bash) +# ============================================================================ +# Bash twin of conda-build-validate-step.yml for the macOS (osx-arm64) and Linux +# (linux-64) build legs. Repackages THIS leg's mssql-python wheel(s) + the +# external mssql-python-odbc wheel into conda packages and validates solve+import +# on the SAME native agent. conda-build provisions a real per-subdir host env, so +# this only runs on the matching native platform (no cross-build, no musl target, +# no aarch64 here — the aarch64 host is x86_64 + QEMU). +# +# This step ONLY builds + validates + stages conda packages as an artifact. It +# does NOT publish anything (publishing happens in the release pipeline), and it is +# BLOCKING by default (continueOnError=false): if conda cannot build/validate the +# packages the leg FAILS, so a broken conda package can never hide behind a green +# build. The one intentionally best-effort exception is the emulated linux-aarch64 +# leg (QEMU flakiness), which overrides continueOnError to true at its call site. +parameters: + # conda subdir this leg targets: 'osx-arm64' or 'linux-64' (display + staging). + - name: condaSubdir + type: string + # Directory holding the freshly built mssql-python wheel(s) for this platform. + - name: mssqlWheelDir + type: string + default: '$(Build.SourcesDirectory)/dist' + # Glob selecting this platform's mssql-python wheel(s) (odbc excluded in-script). + - name: mssqlWheelGlob + type: string + default: 'mssql_python-*.whl' + # Directory holding the downloaded external mssql-python-odbc wheel(s). + # macOS downloads to $(Pipeline.Workspace)/odbc_wheels; Linux flattens them into + # $(Build.SourcesDirectory)/odbc_wheels — pass the right one per leg. + - name: odbcWheelDir + type: string + default: '$(Pipeline.Workspace)/odbc_wheels' + # find -name filter selecting THIS platform's odbc wheel from the consolidated + # odbc drop (which contains ALL 7 platforms). MUST match the leg's OS/arch, + # else conda-build's pip install fails with DistributionNotFound. + - name: odbcWheelFilter + type: string + # Repo conda/ recipe root (contains mssql-python/ and mssql-python-odbc/). + - name: recipeRoot + type: string + default: '$(Build.SourcesDirectory)/conda' + # Space-free working dir for the conda croot + Miniforge + built packages. The + # step APPENDS the target subdir to this (see OUT below) so two legs sharing one + # agent (macOS builds osx-arm64 AND osx-64 on the same Intel agent) never share a + # bld tree -- otherwise the blocking osx-64 leg's staging `find` would also sweep + # up the best-effort osx-arm64 packages (cross-subdir bleed-through). + - name: outputDir + type: string + default: '$(Agent.TempDirectory)/conda-bld' + # Optional comma-separated Python versions; empty = auto-detect from the wheels. + - name: pythonVersions + type: string + default: '' + # Optional target subdir to CROSS-build via CONDA_SUBDIR (e.g. 'osx-64' on an + # Apple-Silicon agent, 'linux-aarch64' on an x86_64 host). Empty = build the + # host's native subdir (osx-arm64 / linux-64). Cross-targeting relies on the host + # being able to RUN the target's Python for the import validation (Rosetta 2 / + # QEMU binfmt); the caller is responsible for that being available on the leg. + - name: condaTargetSubdir + type: string + default: '' + # The shared bash build+validate script. + - name: scriptPath + type: string + default: '$(Build.SourcesDirectory)/OneBranchPipelines/scripts/build-conda-packages.sh' + # BLOCKING by default: if conda cannot build/validate the packages, FAIL the leg + # instead of letting a green build hide a broken conda package. Callers running an + # intentionally best-effort emulated leg (e.g. linux-aarch64 under QEMU) may override + # this to true. + - name: continueOnError + type: boolean + default: false + +steps: + - bash: | + set -euo pipefail + + MSSQL_WHEEL_DIR="${{ parameters.mssqlWheelDir }}" + ODBC_WHEEL_DIR="${{ parameters.odbcWheelDir }}" + # Append the target subdir so each leg on a SHARED agent (macOS osx-arm64 + + # osx-64) gets its OWN bld tree -- no cross-subdir bleed-through at staging. + OUT="${{ parameters.outputDir }}/${{ parameters.condaSubdir }}" + LINKS="$OUT/wheels" + rm -rf "$LINKS"; mkdir -p "$LINKS" + + # Gather this platform's mssql-python wheel(s) into ONE find-links dir, + # excluding the odbc package (its filename also starts with mssql_python). + shopt -s nullglob + mssql_found=0 + for w in "$MSSQL_WHEEL_DIR"/${{ parameters.mssqlWheelGlob }}; do + case "$(basename "$w")" in mssql_python_odbc-*) continue ;; esac + cp -f "$w" "$LINKS/"; mssql_found=1 + done + [ "$mssql_found" = "1" ] || { echo "ERROR: no mssql-python wheel in $MSSQL_WHEEL_DIR" >&2; exit 1; } + + # Gather THIS platform's odbc wheel (must match the leg's OS/arch). + odbc_whl="$(find "$ODBC_WHEEL_DIR" -name '${{ parameters.odbcWheelFilter }}' 2>/dev/null | head -1)" + [ -n "$odbc_whl" ] || { echo "ERROR: no wheel matching '${{ parameters.odbcWheelFilter }}' in $ODBC_WHEEL_DIR" >&2; exit 1; } + cp -f "$odbc_whl" "$LINKS/" + echo "find-links wheels:"; ls -1 "$LINKS" + + # Derive versions from the wheel filenames (single source of truth: the + # ESRP-signed wheels), so the conda package version can NEVER drift. + mssql_whl="$(ls "$LINKS"/mssql_python-*.whl | grep -v mssql_python_odbc | head -1)" + MSSQL_VER="$(basename "$mssql_whl" | sed -nE 's/^mssql_python-([^-]+)-.*/\1/p')" + ODBC_VER="$(basename "$odbc_whl" | sed -nE 's/^mssql_python_odbc-([^-]+)-.*/\1/p')" + [ -n "$MSSQL_VER" ] && [ -n "$ODBC_VER" ] || { echo "ERROR: could not derive versions from wheel filenames" >&2; exit 1; } + echo "Derived versions -> mssql-python=$MSSQL_VER mssql-python-odbc=$ODBC_VER" + + # Build + validate the conda packages for this leg's Python version(s). + chmod +x "${{ parameters.scriptPath }}" + bash "${{ parameters.scriptPath }}" \ + "$LINKS" \ + "${{ parameters.recipeRoot }}" \ + "$OUT" \ + "$MSSQL_VER" \ + "$ODBC_VER" \ + "${{ parameters.pythonVersions }}" \ + "${{ parameters.condaTargetSubdir }}" + + # Stage this leg's conda packages onto the artifact for the consolidate stage, + # preserving the conda subdir layout. Stage ONLY packages whose conda-build + # output subdir matches THIS leg's target (metadata-matched, not a blind tree + # sweep) so a shared agent (macOS osx-arm64 + osx-64) can never bleed one leg's + # packages into the other's artifact. conda-build writes each package into a + # bld// folder that equals its info/index.json subdir, so the folder + # name IS the authoritative subdir. + TARGET_SUBDIR="${{ parameters.condaTargetSubdir }}" + [ -n "$TARGET_SUBDIR" ] || TARGET_SUBDIR="${{ parameters.condaSubdir }}" + CONDA_OUT="$(ob_outputDirectory)/conda" + mkdir -p "$CONDA_OUT/$TARGET_SUBDIR" + staged=0 + while IFS= read -r p; do + subdir="$(basename "$(dirname "$p")")" + if [ "$subdir" != "$TARGET_SUBDIR" ]; then + echo " skip (subdir '$subdir' != target '$TARGET_SUBDIR'): $(basename "$p")" + continue + fi + cp -f "$p" "$CONDA_OUT/$TARGET_SUBDIR/" + echo " staged $TARGET_SUBDIR/$(basename "$p")" + staged=1 + done < <(find "$OUT/bld" -type f \( -name 'mssql-python*.conda' -o -name 'mssql-python*.tar.bz2' \)) + [ "$staged" = "1" ] || { echo "ERROR: no conda packages matching target subdir '$TARGET_SUBDIR' were produced in $OUT/bld" >&2; exit 1; } + displayName: 'Conda build + validate (${{ parameters.condaSubdir }})' + continueOnError: ${{ parameters.continueOnError }} diff --git a/OneBranchPipelines/steps/conda-build-validate-step.yml b/OneBranchPipelines/steps/conda-build-validate-step.yml new file mode 100644 index 000000000..59fddde8c --- /dev/null +++ b/OneBranchPipelines/steps/conda-build-validate-step.yml @@ -0,0 +1,141 @@ +# Conda Build + Validate Step Template +# ============================================================================ +# Repackages the prebuilt, ESRP-signed wheels produced by THIS build leg into +# conda packages, then validates them on the SAME native agent (which already +# has the matching wheel, the external mssql-python-odbc wheel, and a live +# SQL Server for pytest). Include this AFTER the wheel is built on a build leg. +# +# WHY THIS RUNS PER-PLATFORM (not on a single host like ODBC_BuildAll): +# `ODBC_BuildAll` cross-produces every wheel on one host because setup_odbc.py +# only RE-TAGS a data zip. conda-build is different: it provisions a real host +# environment for the target subdir and `pip install`s the matching wheel +# (see conda/*/bld.bat|build.sh). A linux-64 / osx-* host env cannot be created +# on a Windows agent, so — exactly like the wheels and like the conda-forge +# pyodbc-feedstock — each conda package must be built on its matching platform. +# +# SCOPE / LIMITATIONS (first cut, intentionally conservative): +# * x64 / native only. Skipped on cross-arch legs (e.g. Windows ARM64) because +# the import validation cannot execute on the x64 host AND conda cannot +# provision a win-arm64 host env there — same reason pytest is skipped there. +# * musllinux has NO conda target (conda linux-* is glibc), so this step is +# never included on the musllinux legs. +# +# This step ONLY builds + validates + stages conda packages as an artifact. It +# does NOT publish anything — publishing (anaconda upload / ESRP) happens in the +# release pipeline, exactly like the wheels. +parameters: + # Python version this leg builds, X.Y (e.g. '3.13'). One conda build per leg. + - name: pythonVersion + type: string + # conda subdir to stamp on the packages (win-64, osx-64, osx-arm64, + # linux-64, linux-aarch64). Must match the platform of THIS agent. + - name: condaSubdir + type: string + # Target architecture of the wheel build; the step is skipped unless 'x64' + # (or a native arch) so cross-compiled legs don't attempt a conda build. + - name: targetArch + type: string + default: 'x64' + # Directory holding the freshly built mssql-python wheel (setup.py bdist_wheel). + - name: mssqlWheelDir + type: string + default: '$(Build.SourcesDirectory)/dist' + # Directory holding the downloaded external mssql-python-odbc wheel(s) + # (populated by the leg's `installOdbcWheel` download step). + - name: odbcWheelDir + type: string + default: '$(Pipeline.Workspace)/odbc_wheels' + # Filename filter selecting THIS platform's mssql-python-odbc wheel from the + # consolidated odbc drop (which contains ALL 7 platforms). MUST match the leg's + # OS/arch, otherwise conda-build's `pip install` on this host fails with + # DistributionNotFound (a macOS/linux wheel is not installable on win-64, etc.). + - name: odbcWheelFilter + type: string + default: 'mssql_python_odbc-*win_amd64.whl' + # Repo conda/ recipe root (contains mssql-python/ and mssql-python-odbc/). + - name: recipeRoot + type: string + default: '$(Build.SourcesDirectory)/conda' + # Space-free working dir for the conda croot + Miniforge + built packages. + - name: outputDir + type: string + default: '$(Agent.TempDirectory)/conda-bld' + # The shared build+validate script (installs Miniforge/conda-build, builds the + # self-contained mssql-python package, indexes a local channel, solves + imports). + - name: scriptPath + type: string + default: '$(Build.SourcesDirectory)/OneBranchPipelines/scripts/build-conda-packages.ps1' + # BLOCKING by default: if conda cannot build/validate the packages, FAIL the leg + # instead of letting a green build hide a broken conda package. Callers running an + # intentionally best-effort emulated leg (e.g. linux-aarch64 under QEMU) may override + # this to true. + - name: continueOnError + type: boolean + default: false + +steps: + - powershell: | + $ErrorActionPreference = 'Stop' + + $mssqlWheelDir = "${{ parameters.mssqlWheelDir }}" + $odbcWheelDir = "${{ parameters.odbcWheelDir }}" + + # Gather both packages' wheels into ONE find-links dir the recipes install from. + $links = Join-Path "${{ parameters.outputDir }}" 'wheels' + New-Item -ItemType Directory -Force -Path $links | Out-Null + + $mssqlWheels = @(Get-ChildItem -Path $mssqlWheelDir -Filter 'mssql_python-*.whl' -ErrorAction SilentlyContinue | + Where-Object { $_.Name -notlike 'mssql_python_odbc-*' }) + if ($mssqlWheels.Count -eq 0) { Write-Error "No mssql_python-*.whl found in $mssqlWheelDir"; exit 1 } + + $odbcWheel = Get-ChildItem -Path $odbcWheelDir -Recurse -Filter '${{ parameters.odbcWheelFilter }}' -ErrorAction SilentlyContinue | + Select-Object -First 1 + if (-not $odbcWheel) { Write-Error "No wheel matching '${{ parameters.odbcWheelFilter }}' found in $odbcWheelDir"; exit 1 } + + # Copy EVERY mssql-python wheel (a standalone conda build passes all Python versions + # at once; a per-Python wheel leg simply has one) plus this platform's odbc wheel. + $mssqlWheels | ForEach-Object { Copy-Item $_.FullName -Destination $links -Force } + Copy-Item $odbcWheel.FullName -Destination $links -Force + Write-Host "find-links wheels:" + Get-ChildItem $links | ForEach-Object { Write-Host " - $($_.Name)" } + + # Derive the versions from the wheel filenames (single source of truth: the + # ESRP-signed wheels themselves) so the conda package version can NEVER drift + # from the wheel. Filenames: mssql_python--cp3X-...whl and + # mssql_python_odbc--py3-none-...whl. + if ($mssqlWheels[0].Name -notmatch '^mssql_python-([^-]+)-') { Write-Error "Cannot parse version from $($mssqlWheels[0].Name)"; exit 1 } + $mssqlVer = $Matches[1] + if ($odbcWheel.Name -notmatch '^mssql_python_odbc-([^-]+)-') { Write-Error "Cannot parse version from $($odbcWheel.Name)"; exit 1 } + $odbcVer = $Matches[1] + Write-Host "Derived versions -> mssql-python=$mssqlVer mssql-python-odbc=$odbcVer" + + # Build + validate the self-contained mssql-python conda package for THIS leg's + # single Python version (it vendors the ODBC payload from the odbc wheel above). + & "${{ parameters.scriptPath }}" ` + -WheelsDir $links ` + -RecipeRoot "${{ parameters.recipeRoot }}" ` + -OutputDir "${{ parameters.outputDir }}" ` + -MssqlPythonVersion $mssqlVer ` + -OdbcVersion $odbcVer ` + -PythonVersions "${{ parameters.pythonVersion }}" + if ($LASTEXITCODE -ne 0) { Write-Error "conda build+validate failed (exit $LASTEXITCODE)"; exit 1 } + + # Stage the built conda packages onto the leg's artifact so the consolidate + # stage can collect them (mirrors how the wheels ride the same artifact). + $condaOut = Join-Path "$(ob_outputDirectory)" 'conda' + New-Item -ItemType Directory -Force -Path $condaOut | Out-Null + $built = Get-ChildItem -Path (Join-Path "${{ parameters.outputDir }}" 'bld') -Recurse -Include *.conda, *.tar.bz2 | + Where-Object { $_.Name -like 'mssql-python*' } + if (-not $built) { Write-Error "No conda packages were produced under $($links)"; exit 1 } + foreach ($p in $built) { + # Preserve the conda subdir folder layout (e.g. win-64/) so the channel + # is valid when consolidated and indexed downstream. + $subdir = Split-Path -Leaf (Split-Path -Parent $p.FullName) + $dest = Join-Path $condaOut $subdir + New-Item -ItemType Directory -Force -Path $dest | Out-Null + Copy-Item $p.FullName -Destination $dest -Force + Write-Host " staged $subdir/$($p.Name)" + } + displayName: 'Conda build + validate (${{ parameters.condaSubdir }} py${{ parameters.pythonVersion }})' + condition: ne('${{ parameters.targetArch }}', 'arm64') + continueOnError: ${{ parameters.continueOnError }} diff --git a/OneBranchPipelines/steps/conda-publish-step.yml b/OneBranchPipelines/steps/conda-publish-step.yml new file mode 100644 index 000000000..8229ae324 --- /dev/null +++ b/OneBranchPipelines/steps/conda-publish-step.yml @@ -0,0 +1,280 @@ +# Conda Publish Step Template +# ============================================================================ +# Publishes the consolidated conda packages produced by the STANDALONE conda-build +# pipeline (conda-build-pipeline.yml, artifact +# drop_ConsolidateConda_ConsolidateArtifacts) to an Anaconda.org channel using +# anaconda-client (`anaconda upload`). +# +# Reference pattern: azure-sdk-for-python's conda publish +# (eng/pipelines/templates/stages/archetype-conda-release.yml), which runs +# `anaconda upload --user Microsoft --skip-existing` inside a 1ES releaseJob and +# authenticates via the ANACONDA_API_TOKEN env var. ESRP has NO Conda ContentType, +# so anaconda-client is the sanctioned publish path. +# +# Two adaptations vs. azure-sdk: +# 1. Our packages are NON-noarch (native binding + driver), so they live under +# per-platform subdirs (win-64 / osx-arm64 / linux-64), NOT a single noarch +# folder. We recurse every subdir. +# 2. The self-contained mssql-python package vendors the ODBC payload, so there is +# no separate companion to order; we still refuse to publish an incomplete or +# mislabeled set. +# +# The caller MUST: +# - run this ONLY after the conda-release-step.yml readiness gate has passed, and +# - supply ANACONDA_API_TOKEN (variable group 'Anaconda Publishing') to the job. +# This template never puts the token on the command line (anaconda-client reads it +# from the env), so it never appears in the logs. +parameters: + # Infra setup: ADO definition id of the STANDALONE conda-build pipeline + # (conda-build-pipeline.yml) that produced the consolidated conda artifact -- NOT + # the wheel pipeline (def 2199). The release pipeline passes the real id; 0 is a + # placeholder that must be overridden. + - name: buildDefinitionId + type: number + default: 0 + # Consolidated conda artifact name (see consolidate-conda-artifacts-job.yml). + - name: condaArtifactName + type: string + default: 'drop_ConsolidateConda_ConsolidateArtifacts' + # Target Anaconda.org channel/org (e.g. 'microsoft'). Empty is rejected so a + # test pipeline can never accidentally push to the production channel. + - name: condaChannel + type: string + default: '' + # Channel label to publish under (production packages go to 'main'). + - name: condaLabel + type: string + default: 'main' + # Comma-separated subdirs a complete release MUST contain (PyPI parity minus + # win-arm64 and musllinux). Guards against publishing an incomplete set even if + # this step is run standalone. Keep in sync with conda-release-step.yml. + - name: requiredSubdirs + type: string + default: 'win-64,osx-64,osx-arm64,linux-64,linux-aarch64' + # Comma-separated superset of subdirs allowed to appear; anything else fails. + - name: allowedSubdirs + type: string + default: 'win-64,win-arm64,osx-64,osx-arm64,linux-64,linux-aarch64' + # Python used to install/run anaconda-client in the release job. + - name: pythonVersion + type: string + default: '3.12' + # Optional display-name prefix (e.g. '[TEST] ' for the dummy pipeline). + - name: labelPrefix + type: string + default: '' + +steps: + # Infra guard: refuse to run against the placeholder definition id (see release step). + - task: PowerShell@2 + displayName: '${{ parameters.labelPrefix }}Guard: conda-build definition id is configured' + inputs: + targetType: 'inline' + script: | + $id = ${{ parameters.buildDefinitionId }} + if ($id -eq 0) { + Write-Error "condaBuildDefinitionId is 0 (placeholder). Set both the release pipeline's resource 'source:' name and condaBuildDefinitionId to the registered standalone conda-build pipeline." + exit 1 + } + Write-Host "conda-build definition id = $id" + + - task: DownloadPipelineArtifact@2 + displayName: '${{ parameters.labelPrefix }}Download consolidated conda packages (publish)' + inputs: + buildType: 'specific' + project: '$(System.TeamProject)' + definition: ${{ parameters.buildDefinitionId }} + buildVersionToDownload: 'specific' + buildId: $(resources.pipeline.buildPipeline.runID) + artifactName: '${{ parameters.condaArtifactName }}' + targetPath: '$(Build.SourcesDirectory)/conda-artifacts' + + - task: UsePythonVersion@0 + displayName: '${{ parameters.labelPrefix }}Use Python ${{ parameters.pythonVersion }}' + inputs: + versionSpec: '${{ parameters.pythonVersion }}' + addToPath: true + + - task: PowerShell@2 + displayName: '${{ parameters.labelPrefix }}Install anaconda-client' + inputs: + targetType: 'inline' + script: | + $ErrorActionPreference = 'Stop' + python -m pip install --upgrade pip + # Pin anaconda-client: this PRODUCTION job holds ANACONDA_API_TOKEN, so the + # publish tool must be a fixed, reviewed version -- never float 'latest' from + # public PyPI into a credentialed release. Infra hardening: move this to an + # internal feed and hash-pin (--require-hashes). + python -m pip install "anaconda-client==1.12.3" + # anaconda-client installs the `anaconda` console script onto PATH. + anaconda --version + + - task: PowerShell@2 + displayName: '${{ parameters.labelPrefix }}Publish conda packages to anaconda.org/${{ parameters.condaChannel }}' + env: + # anaconda-client reads ANACONDA_API_TOKEN automatically, so the token is + # never passed on the command line and never appears in the logs. Supplied by + # the caller from the 'Anaconda Publishing' variable group. + ANACONDA_API_TOKEN: $(ANACONDA_API_TOKEN) + inputs: + targetType: 'inline' + script: | + $ErrorActionPreference = 'Stop' + + # Refuse to publish without an explicit target channel (protects the dummy + # pipeline, whose default channel is empty, from ever hitting production). + if ([string]::IsNullOrWhiteSpace("${{ parameters.condaChannel }}")) { + Write-Error "condaChannel is empty. Supply the target Anaconda.org channel/org (production = 'microsoft')." + exit 1 + } + + if ([string]::IsNullOrWhiteSpace($env:ANACONDA_API_TOKEN)) { + Write-Error "ANACONDA_API_TOKEN is not set. Add the 'Anaconda Publishing' variable group to the job." + exit 1 + } + + $root = "$(Build.SourcesDirectory)/conda-artifacts/conda" + if (-not (Test-Path $root)) { + Write-Error "Consolidated conda tree not found at $root. Did ConsolidateConda run in the selected build?" + exit 1 + } + + $required = '${{ parameters.requiredSubdirs }}'.Split(',') | ForEach-Object { $_.Trim() } | Where-Object { $_ } + $allowed = '${{ parameters.allowedSubdirs }}'.Split(',') | ForEach-Object { $_.Trim() } | Where-Object { $_ } + + $pkgs = @(Get-ChildItem -Path $root -Recurse -Include *.conda, *.tar.bz2) + if ($pkgs.Count -eq 0) { Write-Error "No conda packages found under $root. Refusing to publish an empty set."; exit 1 } + + $bySubdir = $pkgs | Group-Object { $_.Directory.Name } + function Get-Binding($grp) { @($grp | Where-Object { $_.Name -like 'mssql-python-*' -and $_.Name -notlike 'mssql-python-odbc-*' }) } + + Write-Host "Discovered $($pkgs.Count) conda package(s) across $($bySubdir.Count) subdir(s):" + foreach ($g in ($bySubdir | Sort-Object Name)) { + Write-Host (" {0,-14} mssql-python={1}" -f $g.Name, (Get-Binding $g.Group).Count) + } + + $failed = $false + + # No mis-stamped subdir. + foreach ($g in $bySubdir) { + if ($allowed -notcontains $g.Name) { + Write-Host "##vso[task.logissue type=error]Unexpected conda subdir '$($g.Name)' (not in allowedSubdirs). Refusing to publish." + $failed = $true + } + } + + # Every required subdir present with at least one mssql-python package. + $foundSubdirs = @($bySubdir | ForEach-Object { $_.Name }) + foreach ($req in $required) { + if ($foundSubdirs -notcontains $req) { + Write-Host "##vso[task.logissue type=error]Required conda subdir '$req' is MISSING. Refusing to publish an incomplete set." + $failed = $true + continue + } + if ((Get-Binding ($bySubdir | Where-Object { $_.Name -eq $req }).Group).Count -eq 0) { + Write-Host "##vso[task.logissue type=error]Subdir '$req' has no mssql-python package." + $failed = $true + } + } + + # The self-contained mssql-python vendors the ODBC payload, so a stray + # mssql-python-odbc package (the old separate companion) must never ship. + $stray = @($pkgs | Where-Object { $_.Name -like 'mssql-python-odbc-*' }) + if ($stray.Count -gt 0) { + Write-Host "##vso[task.logissue type=error]Found $($stray.Count) stray mssql-python-odbc package(s); the self-contained model ships only mssql-python. Refusing to publish." + $failed = $true + } + + $binding = Get-Binding $pkgs + if ($binding.Count -eq 0) { + Write-Host "##vso[task.logissue type=error]No mssql-python packages found. Refusing to publish an empty set." + $failed = $true + } + + if ($failed) { Write-Error "Conda publish pre-check FAILED. Refusing to publish an incomplete/mislabeled set."; exit 1 } + + # Staged-then-promoted publish: upload the FULL set to a BUILD-UNIQUE staging + # label first; only after every package lands AND its bytes are verified do we + # promote to the public label. A mid-upload failure leaves a partial set on the + # staging label ONLY -- the public label is never partially updated. The staging + # label includes the build id so concurrent/retried runs never collide. + $stagingLabel = "${{ parameters.condaLabel }}_staging_$(Build.BuildId)" + + function Invoke-Anaconda($argsList) { + for ($attempt = 1; $attempt -le 3; $attempt++) { + & anaconda @argsList + if ($LASTEXITCODE -eq 0) { return $true } + Write-Host "attempt $attempt failed; retrying in 5s ..." + Start-Sleep -Seconds 5 + } + return $false + } + + function Get-Spec($file) { + # /// from --.. + # The package name may contain '-', so peel the trailing version + build. + $stem = $file.Name -replace '\.(conda|tar\.bz2)$', '' + $parts = $stem.Split('-') + if ($parts.Count -lt 3) { Write-Error "Cannot parse conda spec from '$($file.Name)'."; exit 1 } + $version = $parts[$parts.Count - 2] + $name = ($parts[0..($parts.Count - 3)]) -join '-' + return "${{ parameters.condaChannel }}/$name/$version/$($file.Name)" + } + + Write-Host "==== Stage: upload $($binding.Count) package(s) to label '$stagingLabel' ====" + foreach ($p in ($binding | Sort-Object FullName)) { + Write-Host "Uploading $($p.Directory.Name)/$($p.Name) -> $stagingLabel ..." + if (-not (Invoke-Anaconda @('upload', '--user', "${{ parameters.condaChannel }}", '--label', $stagingLabel, '--skip-existing', "$($p.FullName)"))) { + Write-Error "Failed to stage $($p.Name) after 3 attempts. The public label was NOT touched." + exit 1 + } + } + + Write-Host "==== Verify: SHA-256 + staging membership for every package before promoting ====" + foreach ($p in ($binding | Sort-Object FullName)) { + $spec = Get-Spec $p + $localSha = (Get-FileHash -Algorithm SHA256 -Path $p.FullName).Hash.ToLower() + $show = (& anaconda show $spec 2>&1 | Out-String) + if ($LASTEXITCODE -ne 0) { + Write-Error "Staged package '$spec' is not resolvable on anaconda.org; refusing to promote a partial set." + exit 1 + } + # anaconda show reports the uploaded file's checksum. When a sha256 is present, + # require an EXACT match to the local artifact (true byte integrity, unlike + # --skip-existing's name-only check). + $m = [regex]::Match($show, '(?i)sha256[^0-9a-f]*([0-9a-f]{64})') + if ($m.Success) { + $remoteSha = $m.Groups[1].Value.ToLower() + if ($remoteSha -ne $localSha) { + Write-Error "SHA-256 mismatch for '$spec': local $localSha != remote $remoteSha. Refusing to promote a corrupted/wrong artifact." + exit 1 + } + Write-Host "SHA-256 OK $($p.Name) ($localSha)" + } + else { + Write-Host "NOTE: anaconda show exposed no sha256 for '$spec'; verified membership + local sha256 $localSha." + } + } + + Write-Host "==== Promote (idempotent): move the verified set from '$stagingLabel' to '${{ parameters.condaLabel }}' ====" + foreach ($p in ($binding | Sort-Object FullName)) { + $spec = Get-Spec $p + if (Invoke-Anaconda @('move', '--from-label', $stagingLabel, '--to-label', "${{ parameters.condaLabel }}", $spec)) { + Write-Host "Promoted $($p.Name)." + continue + } + # Idempotent rerun: a move can fail because THIS package was already promoted + # by a prior (interrupted) run. Treat 'already on the target label' as done; + # anything else is a real failure that must be re-run. + $show = (& anaconda show $spec 2>&1 | Out-String) + if ($show -match [regex]::Escape("${{ parameters.condaLabel }}")) { + Write-Host "Already on '${{ parameters.condaLabel }}' (idempotent rerun): $($p.Name)." + continue + } + Write-Error "Failed to promote '$spec' to '${{ parameters.condaLabel }}' and it is not already there. Re-run to finish promotion." + exit 1 + } + + Write-Host "" + Write-Host "Published $($binding.Count) mssql-python conda package(s) to anaconda.org/${{ parameters.condaChannel }} (label ${{ parameters.condaLabel }}) via staging label '$stagingLabel'." diff --git a/OneBranchPipelines/steps/conda-release-step.yml b/OneBranchPipelines/steps/conda-release-step.yml new file mode 100644 index 000000000..f6a59305c --- /dev/null +++ b/OneBranchPipelines/steps/conda-release-step.yml @@ -0,0 +1,198 @@ +# Conda Release Readiness Step Template +# ============================================================================ +# Downloads the consolidated conda packages produced by the STANDALONE conda-build +# pipeline (conda-build-pipeline.yml, artifact +# drop_ConsolidateConda_ConsolidateArtifacts) and enforces the RELEASE-TIME hard +# gate that the build pipeline intentionally does +# NOT enforce. +# +# Why the gate lives HERE and not in the build: +# - BUILD pipeline: conda is collected BEST-EFFORT (warn-only) so a conda hiccup +# on any leg can never fail the build or block the primary wheel release. +# - RELEASE pipeline: conda completeness is GATED -- an incomplete conda set +# must never be shipped. +# +# What it enforces (all from each package's AUTHORITATIVE info/index.json, never +# folder names or bare counts -- so a mis-stamped subdir or a dropped Python +# variant cannot slip through): +# - real subdir: every package's info/index.json `subdir` is in `allowedSubdirs` +# AND equals the folder it was staged into (catches an osx-64 package copied +# into osx-arm64/, which a folder-name check cannot). +# - required subdirs: every subdir in `requiredSubdirs` (the PyPI-parity set) +# is present. +# - Python matrix: on every required subdir the mssql-python binding covers +# EVERY expected Python (`pythonVersions`) -- catches e.g. 3 of 5 win-64 +# bindings shipping against the single companion. +# - versions: all packages share one version (and match the expected version +# when `mssqlPythonVersion` is supplied). +# +# The check is implemented in conda/validate_conda_release.py (unit-tested by +# tests/test_027_conda_release_metadata.py), which reads the zstd-compressed +# info/index.json embedded in every .conda. +# +# win-arm64 is intentionally NOT in the parity set: an x64 agent cannot import- +# validate an arm64 conda package (same reason its wheel skips pytest), so it needs +# a native win-arm64 agent before it can be gated. musllinux has no conda subdir at +# all (conda Linux is glibc-only), so it is correctly absent. +# +# Publishing to anaconda.org is a SEPARATE, still-to-be-finalized step (ESRP Conda +# ContentType vs anaconda-client upload); this template only proves the artifact is +# complete and correctly paired so publishing can proceed safely. +parameters: + # Infra setup: ADO definition id of the STANDALONE conda-build pipeline + # (conda-build-pipeline.yml) that produced the conda artifact -- NOT the wheel + # pipeline (def 2199), which no longer builds conda. The release pipeline passes + # the real id; 0 is a placeholder that must be overridden. + - name: buildDefinitionId + type: number + default: 0 + # Consolidated conda artifact name (see consolidate-conda-artifacts-job.yml). + - name: condaArtifactName + type: string + default: 'drop_ConsolidateConda_ConsolidateArtifacts' + # Comma-separated subdirs that a complete release MUST contain (PyPI parity minus + # win-arm64 and musllinux, which have no validated conda build). If an emulated + # leg (osx-64 / linux-aarch64) ever proves too flaky to gate on, drop it here — + # no code change needed. + - name: requiredSubdirs + type: string + default: 'win-64,osx-64,osx-arm64,linux-64,linux-aarch64' + # Comma-separated superset of subdirs that are ALLOWED to appear. Any discovered + # subdir outside this set fails the gate (guards against a mis-stamped subdir). + # win-arm64 is allowed-but-not-required so a future native-agent build can land + # without tripping the gate. + - name: allowedSubdirs + type: string + default: 'win-64,win-arm64,osx-64,osx-arm64,linux-64,linux-aarch64' + # Comma-separated Python versions the binding matrix MUST cover on every required + # subdir. The gate reads each binding's pyXY build tag from info/index.json. + - name: pythonVersions + type: string + default: '3.10,3.11,3.12,3.13,3.14' + # Optional EXACT expected versions. When set, the gate asserts every package's + # info/index.json version matches; when empty it still enforces one-version-per- + # package consistency plus the subdir / matrix / pairing checks. + - name: mssqlPythonVersion + type: string + default: '' + - name: odbcVersion + type: string + default: '' + # Optional display-name prefix (e.g. '[TEST] ' for the dummy pipeline). + - name: labelPrefix + type: string + default: '' + +steps: + # Infra guard: the standalone conda-build definition id MUST be set. A 0 placeholder + # means the resource `source:` name and this numeric id disagree, so the download + # would silently target the wrong pipeline -- fail fast with a clear message. + - task: PowerShell@2 + displayName: '${{ parameters.labelPrefix }}Guard: conda-build definition id is configured' + inputs: + targetType: 'inline' + script: | + $id = ${{ parameters.buildDefinitionId }} + if ($id -eq 0) { + Write-Error "condaBuildDefinitionId is 0 (placeholder). Register the standalone conda-build pipeline in ADO, then set BOTH the release pipeline's resource 'source:' name AND condaBuildDefinitionId to it." + exit 1 + } + Write-Host "conda-build definition id = $id" + + - task: DownloadPipelineArtifact@2 + displayName: '${{ parameters.labelPrefix }}Download consolidated conda packages' + inputs: + buildType: 'specific' + project: '$(System.TeamProject)' + definition: ${{ parameters.buildDefinitionId }} + buildVersionToDownload: 'specific' + buildId: $(resources.pipeline.buildPipeline.runID) + artifactName: '${{ parameters.condaArtifactName }}' + targetPath: '$(Build.SourcesDirectory)/conda-artifacts' + + # N3 provenance: bind the downloaded artifact to the release commit. The conda-build + # pipeline stamps its source commit on the run; if it differs from the commit being + # released, a stale/older/different build was selected -- refuse it. + - task: PowerShell@2 + displayName: '${{ parameters.labelPrefix }}Provenance: conda artifact built from the release commit' + inputs: + targetType: 'inline' + script: | + $built = "$(resources.pipeline.buildPipeline.sourceCommit)" + $release = "$(Build.SourceVersion)" + Write-Host "conda-build source commit: $built" + Write-Host "release source commit: $release" + if ([string]::IsNullOrWhiteSpace($built)) { + Write-Error "Could not determine the conda-build source commit (resources.pipeline.buildPipeline.sourceCommit is empty)." + exit 1 + } + if ($built -ne $release) { + Write-Error "Provenance mismatch: the selected conda artifact was built from $built but this release is $release. Refusing to publish an artifact from a different/older build." + exit 1 + } + Write-Host "Provenance OK: artifact built from the release commit." + + # N3 re-audit: re-run the masking-immune RUNPATH audit on the EXACT downloaded + # artifacts (not just metadata) so a stale pre-audit .conda can never be published. + - task: PowerShell@2 + displayName: '${{ parameters.labelPrefix }}Re-audit the exact release artifacts (RUNPATH self-containment)' + inputs: + targetType: 'inline' + script: | + $ErrorActionPreference = 'Stop' + $root = "$(Build.SourcesDirectory)/conda-artifacts/conda" + $audit = "$(Build.SourcesDirectory)/eng/scripts/audit_bundled_binaries.py" + if (-not (Test-Path $root)) { Write-Error "Consolidated conda tree not found at $root."; exit 1 } + if (-not (Test-Path $audit)) { Write-Error "Audit script not found at $audit."; exit 1 } + python -m pip install --quiet --disable-pip-version-check zstandard + python "$audit" --root "$root" + if ($LASTEXITCODE -ne 0) { + Write-Error "Release-boundary RUNPATH audit FAILED on the downloaded conda artifacts. Refusing to publish." + exit 1 + } + Write-Host "Release-boundary RUNPATH audit passed on the exact downloaded artifacts." + + - task: PowerShell@2 + displayName: '${{ parameters.labelPrefix }}Validate conda release readiness (metadata: subdirs + Python matrix + versions)' + inputs: + targetType: 'inline' + script: | + $ErrorActionPreference = 'Stop' + $root = "$(Build.SourcesDirectory)/conda-artifacts/conda" + if (-not (Test-Path $root)) { + Write-Error "Consolidated conda tree not found at $root. Was ConsolidateConda produced by the selected build run?" + exit 1 + } + + $gate = "$(Build.SourcesDirectory)/conda/validate_conda_release.py" + if (-not (Test-Path $gate)) { + Write-Error "Metadata gate script not found at $gate." + exit 1 + } + + # The gate reads each package's AUTHORITATIVE info/index.json (a zstd tar + # inside every .conda) rather than trusting folder names or counts. zstd is + # stdlib on py3.14+; install the `zstandard` backend so the reader always works. + python -m pip install --quiet --disable-pip-version-check zstandard + if ($LASTEXITCODE -ne 0) { + Write-Error "Failed to install the 'zstandard' backend needed to read .conda metadata." + exit 1 + } + + # Pass EXACT version expectations only when supplied; the gate still enforces + # one-version-per-package consistency when they are empty. + $extra = @() + if ('${{ parameters.mssqlPythonVersion }}'.Trim()) { $extra += @('--mssql-python-version', ('${{ parameters.mssqlPythonVersion }}'.Trim())) } + if ('${{ parameters.odbcVersion }}'.Trim()) { $extra += @('--mssql-python-odbc-version', ('${{ parameters.odbcVersion }}'.Trim())) } + + python "$gate" ` + --root "$root" ` + --required-subdirs '${{ parameters.requiredSubdirs }}' ` + --allowed-subdirs '${{ parameters.allowedSubdirs }}' ` + --pythons '${{ parameters.pythonVersions }}' ` + @extra + if ($LASTEXITCODE -ne 0) { + Write-Error "Conda release readiness FAILED. Refusing to proceed with an incomplete/mis-labeled/mis-paired conda set." + exit 1 + } + Write-Host "Conda set is release-ready (the publish step is gated separately)." diff --git a/README.md b/README.md index 9d7aca493..9e3a69938 100644 --- a/README.md +++ b/README.md @@ -57,6 +57,11 @@ tdnf distro-sync && tdnf install -y libtool-ltdl krb5-libs glibc-iconv pip install mssql-python ``` +**Conda:** mssql-python is also published as a self-contained conda package — the ODBC Driver 18 payload and its native dependencies (`krb5`, `openssl`, `libltdl`) are resolved by conda, so none of the system `apt`/`dnf`/`apk`/`zypper` steps above are required. Both channels are needed: `mssql-python` comes from the `microsoft` channel and its dependencies resolve from `conda-forge`. +```bash +conda install -c microsoft -c conda-forge mssql-python +``` + ## Key Features ### Supported Platforms diff --git a/conda/.gitattributes b/conda/.gitattributes new file mode 100644 index 000000000..68446d138 --- /dev/null +++ b/conda/.gitattributes @@ -0,0 +1,3 @@ +# Keep shell build scripts LF so conda-build works on Linux/macOS agents, +# regardless of the checkout host's core.autocrlf setting. +*.sh text eol=lf diff --git a/conda/driver_load_probe.py b/conda/driver_load_probe.py new file mode 100644 index 000000000..dd6cadb19 --- /dev/null +++ b/conda/driver_load_probe.py @@ -0,0 +1,139 @@ +"""DB-less ODBC driver-load proof for the conda test-before-live gate. + +Importing ``mssql_python`` and issuing the first ``connect()`` triggers the +one-time native ODBC driver load (``std::call_once`` in the C++ binding). To +prove the driver payload is present AND architecture-correct WITHOUT a live SQL +Server, we attempt a connection to an unreachable local port and classify the +failure. + +FAIL-CLOSED classification (this is the whole point of the probe): + +* We treat the outcome as PASS **only** when there is positive proof the native + driver loaded -- either a clean connect, or a *connection-stage* diagnostic + that only the loaded ``msodbcsql`` driver can emit (its ``[Microsoft][ODBC + Driver 18 for SQL Server]`` branding, a SQL Server network provider error, a + TLS handshake error, or a login / auth outcome). See ``_DRIVER_LOADED_MARKERS``. +* Every other exception is treated as a load failure -> non-zero exit. This + includes the C++ ``LoadDriverOrThrowException`` family + ("Failed to load the driver...", "Failed to load library: ", + "Failed to load required function pointers...", "ODBC driver not found...", + the ``mssql-auth.dll`` errors) and the macOS ``dlopen`` / ``dlerror`` detail -- + none of which contain a loaded-driver marker, so a broken / missing / + mis-architecture driver can never report PASS. + +This gates on the actual DRIVER, not just the tiny ``mssql_python_odbc`` Python +shim, and needs no ``DB_CONNECTION_STRING`` secret. A real live ``SELECT 1`` still +runs separately whenever a server is wired. + +Exit code 0 = driver loaded; non-zero = driver did not load (blocks publish). +""" + +import sys + +# Positive signals: the native ODBC driver LOADED and reached the network / TLS +# / auth stage (or connected). These are the ONLY outcomes that count as PASS. +# All markers are matched case-insensitively. +_DRIVER_LOADED_MARKERS = ( + # The loaded msodbcsql driver brands every diagnostic it emits; a driver + # that failed to load / link / resolve its symbols never gets far enough to + # print this, so it is the strongest single proof of a successful load. + "odbc driver 18 for sql server", + "microsoft][odbc", + # SQL Server network / transport providers -- reached only after load. + "tcp provider", + "named pipes provider", + "shared memory provider", + "sql server network interfaces", + # Connection / login outcomes that prove the handshake was attempted. + "login timeout expired", + "a network-related or instance-specific error", + "server was not found", + "server is not found", + "actively refused", # Windows WSAECONNREFUSED (target port closed) + "connection refused", # posix ECONNREFUSED (target port closed) + "communication link failure", + "unable to establish", + "login failed for user", # authentication stage reached + "cannot open database", # server reached, database validation + # TLS handshake reached -> both the driver and its crypto backend loaded. + "ssl provider", + "ssl security error", + "certificate", +) + +# Negative signals: the native driver did NOT load / link / resolve. Listed only +# to produce a clearer FAIL message -- classification is allowlist-based, so an +# unrecognized exception still fails closed even if it matches nothing here. +_DRIVER_LOAD_FAILURE_MARKERS = ( + "failed to load the driver", + "failed to load library", + "failed to load required function pointers", + "odbc driver not found", + "mssql-auth.dll", + "mssql-python-odbc", + "cannot open shared object", # linux dlopen failure + "image not found", # macOS dlopen failure + "no such file or directory", # driver binary absent + "can't open lib", # unixODBC could not open the driver + "unsupported architecture", + "unsupported platform", +) + + +def driver_loaded(exc): + """FAIL-CLOSED classifier for the connect outcome. + + Returns ``True`` only when there is positive proof the native ODBC driver + loaded: a clean connect (``exc is None``) or a connection-stage diagnostic + that the loaded driver alone can emit. Every other exception -- including the + C++ "Failed to load the driver..." family and anything unrecognized -- + returns ``False`` so the probe exits non-zero. + """ + if exc is None: + return True + msg = str(exc).lower() + return any(marker in msg for marker in _DRIVER_LOADED_MARKERS) + + +def describe(exc): + """Short, human-readable reason string for the probe's stdout / exit line.""" + if exc is None: + return "clean connect" + msg = str(exc) + low = msg.lower() + for marker in _DRIVER_LOAD_FAILURE_MARKERS: + if marker in low: + return "driver load failure -> " + msg[:300] + return msg[:300] + + +def main(): + # Deferred so this module can be imported (and ``driver_loaded`` unit-tested) + # WITHOUT triggering the native ``mssql_python`` import, which needs the + # compiled extension + driver payload. + import mssql_python + + # Unreachable endpoint (nothing listens on TCP port 1) -> the driver loads, + # attempts the socket, and fails fast at the network stage. The loopback:1 is a + # dummy DB-less probe target, never a live endpoint. + conn_str = "Server=127.0.0.1,1;Database=x;Uid=x;Pwd=x;Encrypt=no;TrustServerCertificate=yes;" # DevSkim: ignore DS162092 + outcome = None + try: + conn = mssql_python.connect(conn_str) + # Reaching a real server on 127.0.0.1:1 is not expected, but a successful + # connect still proves the driver loaded. Close it and pass. + try: + conn.close() + except Exception: # noqa: BLE001 - best-effort cleanup only + pass + except Exception as exc: # noqa: BLE001 - deliberately classified below + outcome = exc + + if driver_loaded(outcome): + print("DRIVER_LOADED (" + describe(outcome) + ")") + return + sys.exit("DRIVER DID NOT LOAD / wrong arch / missing companion: " + describe(outcome)) + + +if __name__ == "__main__": + main() diff --git a/conda/mssql-python/bld.bat b/conda/mssql-python/bld.bat new file mode 100644 index 000000000..d73fbb16f --- /dev/null +++ b/conda/mssql-python/bld.bat @@ -0,0 +1,23 @@ +@echo on +REM Repackage the prebuilt, ESRP-signed wheel into a conda package (offline), and +REM vendor the ODBC Driver 18 payload INTO it (v1.11.0 model: libs ship inside). +REM PKG_NAME / PKG_VERSION are exported by conda-build; WHEELS_DIR + MSSQL_ODBC_VERSION +REM by the pipeline. +setlocal enabledelayedexpansion +"%PYTHON%" -m pip install --no-deps --no-index --find-links "%WHEELS_DIR%" %PKG_NAME%==%PKG_VERSION% -vv +if errorlevel 1 exit 1 + +REM Extract the python-agnostic py3-none-win odbc wheel into the SAME site-packages +REM so mssql_python_odbc\libs\ sits beside mssql_python\ (the loader finds the driver +REM there). WHEELS_DIR is staged per-target, so a single matching odbc wheel is present. +set "SP=%PREFIX%\Lib\site-packages" +if not exist "%SP%" mkdir "%SP%" +set "ODBC_WHL=" +for %%W in ("%WHEELS_DIR%\mssql_python_odbc-%MSSQL_ODBC_VERSION%-py3-none-win_*.whl") do set "ODBC_WHL=%%~fW" +if not defined ODBC_WHL ( + echo ERROR: no mssql_python_odbc==%MSSQL_ODBC_VERSION% py3-none-win wheel in "%WHEELS_DIR%" + exit 1 +) +echo Extracting "!ODBC_WHL!" into "%SP%" +tar -xf "!ODBC_WHL!" -C "%SP%" +if errorlevel 1 exit 1 diff --git a/conda/mssql-python/build.sh b/conda/mssql-python/build.sh new file mode 100644 index 000000000..94aea5617 --- /dev/null +++ b/conda/mssql-python/build.sh @@ -0,0 +1,114 @@ +#!/bin/bash +# Repackage the prebuilt, ESRP-signed wheel into a conda package (offline). +# PKG_NAME / PKG_VERSION are exported by conda-build; WHEELS_DIR by the pipeline/harness. +set -euo pipefail +# Cross-arch (emulated) build: when repackaging the aarch64 wheel on an x86_64 host, +# $PYTHON is the target-arch interpreter and runs under qemu-user. Point qemu at the +# aarch64 glibc loader/libs (installed via libc6-arm64-cross) so it can find +# /lib/ld-linux-aarch64.so.1 instead of aborting with "Could not open". The dir only +# exists on the emulated aarch64 leg; setting the var elsewhere is a harmless no-op. +[ -d /usr/aarch64-linux-gnu ] && export QEMU_LD_PREFIX="${QEMU_LD_PREFIX:-/usr/aarch64-linux-gnu}" + +# This package is SELF-CONTAINED (v1.11.0 model): the ODBC Driver 18 payload ships +# INSIDE it, so there is NO separate mssql-python-odbc conda package. We land BOTH +# the code wheel AND the python-agnostic py3-none- odbc wheel in the SAME +# site-packages, so mssql_python_odbc/libs/ sits beside mssql_python/ and the C++ +# loader resolves the driver there. WHEELS_DIR is staged per-target by the pipeline, +# so exactly one matching odbc wheel is present. +odbc_ver="${MSSQL_ODBC_VERSION:?MSSQL_ODBC_VERSION not set}" + +# The normal path installs with the host-env Python -- native builds, and the +# QEMU-emulated linux-aarch64 leg where the aarch64 Python runs under binfmt. pip +# resolves the correct site-packages for BOTH wheels, so no unzip is needed there. +# The osx-arm64 conda package is CROSS-built on an Intel macOS agent (no reverse +# Rosetta): the arm64 host Python CANNOT execute and pip would abort, so extract +# both wheels (zips) WITHOUT Python -- the same approach the Windows bld.bat uses +# with `tar`. macOS ships `unzip`. The arm64 slice comes from the universal2 wheel; +# conda-build still stamps osx-arm64. +if "$PYTHON" -c "import sys" >/dev/null 2>&1; then + "$PYTHON" -m pip install --no-deps --no-index --find-links "$WHEELS_DIR" "$PKG_NAME==$PKG_VERSION" -vv + "$PYTHON" -m pip install --no-deps --no-index --find-links "$WHEELS_DIR" "mssql-python-odbc==$odbc_ver" -vv +else + echo "Host Python '$PYTHON' is not executable on this agent (non-emulated cross-build);" + echo "extracting both wheels into \$SP_DIR without running Python." + mkdir -p "$SP_DIR" + pkg_underscore="${PKG_NAME//-/_}" + code_whl="" + for w in "$WHEELS_DIR/${pkg_underscore}-${PKG_VERSION}-"*.whl; do + [ -e "$w" ] && { code_whl="$w"; break; } + done + [ -n "$code_whl" ] || { echo "ERROR: no ${PKG_NAME}==${PKG_VERSION} wheel in '$WHEELS_DIR'" >&2; exit 1; } + odbc_whl="" + for w in "$WHEELS_DIR"/mssql_python_odbc-"$odbc_ver"-py3-none-*.whl; do + [ -e "$w" ] && { odbc_whl="$w"; break; } + done + [ -n "$odbc_whl" ] || { echo "ERROR: no mssql_python_odbc==$odbc_ver py3-none wheel in '$WHEELS_DIR'" >&2; exit 1; } + echo "Extracting '$code_whl' -> '$SP_DIR'" + unzip -oq "$code_whl" -d "$SP_DIR" + echo "Extracting '$odbc_whl' -> '$SP_DIR'" + unzip -oq "$odbc_whl" -d "$SP_DIR" +fi + +# --------------------------------------------------------------------------- +# Linux driver reachability (#563) -- the core fix. +# --------------------------------------------------------------------------- +# Declaring krb5/openssl/libltdl as conda deps drops one consistent copy of each +# into $PREFIX/lib, but that is INERT on its own: the vendored ODBC binaries ship +# with a bare DT_RUNPATH=$ORIGIN (no climb), so on a minimal conda base the loader +# never looks in $PREFIX/lib -- it falls through to SYSTEM krb5 (the #563 mixing +# crash) and cannot find libltdl.so.7 at all. Reachability, not declaration, is the +# lever: stamp a PURELY RELATIVE $ORIGIN climb (the ELF twin of the macOS +# @loader_path flow) onto libmsodbcsql* and libodbcinst.so.2 so they resolve THIS +# env's own $PREFIX/lib, location-independently. +# +# SIGNATURE SAFETY: +# The Linux ODBC .so are NOT ESRP code-signed -- the mssql-python-odbc pipeline only +# MALWARE-SCANS them (there is no CodeSign task). Only Windows .dll (Authenticode) +# and macOS .dylib (codesign) are code-signed, and this recipe never patches those +# (the climb is Linux-only). So stamping the relative $ORIGIN climb here breaks no +# signature. If the binaries already carry the EXACT climb (e.g. a future odbc-side +# pre-bake before signing), the patch is a byte-for-byte no-op. +# +# The canonical RUNPATH is exactly "$ORIGIN:$ORIGIN/" -- the patch below emits +# that literal form, and the static audit (eng/scripts/audit_bundled_binaries.py) +# requires the same exact climb entry. +# +# Linux-only by construction: the glob matches nothing in a macOS payload +# (libs/macos/...), so this whole block is a natural no-op on the osx legs. +prefix_lib="$PREFIX/lib" +shopt -s nullglob +have_linux_payload=0 +[ -d "$SP_DIR/mssql_python_odbc/libs/linux" ] && have_linux_payload=1 +drivers_seen=0 +for libdir in "$SP_DIR"/mssql_python_odbc/libs/linux/*/*/lib; do + # Compute the EXACT expected climb from THIS driver dir up to $PREFIX/lib (derived + # from the real install layout, never a hard-coded ../ count). + climb="$("$PYTHON" -c 'import os,sys; print(os.path.relpath(sys.argv[1], sys.argv[2]))' "$prefix_lib" "$libdir")" + want="\$ORIGIN:\$ORIGIN/$climb" + for so in "$libdir"/libmsodbcsql-*.so.* "$libdir"/libodbcinst.so.2; do + [ -e "$so" ] || continue + drivers_seen=$((drivers_seen + 1)) + got="$(patchelf --print-rpath "$so" 2>/dev/null || true)" + if [ "$got" = "$want" ]; then + echo "RPATH-OK (already baked) $(basename "$so") -> $got" + continue + fi + # Stamp the exact $ORIGIN climb (safe -- these Linux .so are not code-signed). + patchelf --set-rpath "$want" "$so" + got="$(patchelf --print-rpath "$so")" + # H2: compare EXACTLY to the intended value, not just "no absolute entry". + if [ "$got" != "$want" ]; then + echo "ERROR: patch did not yield the exact expected RUNPATH ('$got' != '$want')." >&2 + exit 1 + fi + echo "RPATH-PATCHED $(basename "$so") -> $got" + done +done +shopt -u nullglob +# H2: a Linux payload with NO driver found is a bypass hole -- a bare `conda build` +# skipping the orchestrator audit would then ship un-asserted drivers. Fail loudly. +if [ "$have_linux_payload" = "1" ] && [ "$drivers_seen" = "0" ]; then + echo "ERROR: Linux ODBC payload present but no libmsodbcsql*/libodbcinst.so.2 found to assert the #563 climb." >&2 + exit 1 +fi +[ "$drivers_seen" -gt 0 ] && echo "LINUX_RPATH_CLIMB_OK" || true diff --git a/conda/mssql-python/meta.yaml b/conda/mssql-python/meta.yaml new file mode 100644 index 000000000..6c3b5b10b --- /dev/null +++ b/conda/mssql-python/meta.yaml @@ -0,0 +1,124 @@ +{% set version = environ.get('MSSQL_PYTHON_VERSION', '1.13.0') %} +{% set odbc_version = environ.get('MSSQL_ODBC_VERSION', '18.6.2.1') %} + +package: + name: mssql-python + version: "{{ version }}" + +build: + number: 0 + # Like the companion, this recipe REPACKAGES a prebuilt wheel (the compiled + # ddbc_bindings extension + bundled runtime); it compiles nothing. conda-build's + # overlinking/overdepending checks target from-source builds and mis-fire on + # vendored binaries (e.g. ddbc_bindings linking the driver that lives in the + # separate companion package), so downgrade both from errors to warnings -- CI + # enables them as errors by default. + error_overlinking: false + error_overdepending: false + # Like the companion, this recipe vendors PREBUILT, signed binaries (the compiled + # ddbc_bindings extension + bundled VC++ runtime); conda-build must neither rewrite + # nor scan them: + # - binary_relocation: rewriting RPATH / install-name in a signed binary corrupts + # the signature. + # - detect_binary_files_with_prefix: the build-prefix scan over the signed native + # binaries is meaningless for a pure repackage and is the packaging step that + # fails right after "Fixing permissions" on these recipes. + binary_relocation: false + detect_binary_files_with_prefix: false + # macOS builds this recipe for BOTH osx-64 (native on the Intel agent) and + # osx-arm64 (CROSS-built there). conda-build's .pyc byte-compilation runs the + # TARGET Python, which for osx-arm64 cannot execute on Intel -- so skip pyc on + # macOS (Python regenerates it at import). Linux/Windows legs are unaffected. + skip_compile_pyc: + - "**/*.py" # [osx] + # WHEELS_DIR is exported by the pipeline (or a local harness) and passed into the + # isolated conda-build environment so bld.bat / build.sh can install the prebuilt, + # ESRP-signed wheel from --find-links, fully offline. This mirrors CI: no PyPI. + script_env: + - WHEELS_DIR + # MSSQL_ODBC_VERSION lets build.sh / bld.bat locate the matching mssql-python-odbc + # wheel to vendor INTO this package (the driver payload now ships inside). + - MSSQL_ODBC_VERSION + # conda-build sanitizes the recipe's env and forwards ONLY allow-listed vars, so + # build.sh's DEV-ONLY escape hatch must be listed here to be reachable. The PR + # audit gate sets CONDA_ALLOW_UNSIGNED_PATCH=1 to patch the un-baked PyPI binaries; + # it MUST NEVER be set on the signed release build (conda-build-pipeline.yml + # hard-guards that), so the release path stays strictly assertion-only. + - CONDA_ALLOW_UNSIGNED_PATCH + +requirements: + # Linux reachability fix (#563): patchelf stamps a relative $ORIGIN RPATH climb + # onto the vendored ODBC binaries in build.sh so the DECLARED conda + # krb5/openssl/libltdl in $PREFIX/lib are actually REACHABLE. Declaration alone is + # inert -- the driver ships with a bare DT_RUNPATH=$ORIGIN (no climb) and would + # fall through to SYSTEM krb5/libltdl (the exact #563 mixing bug). Build-time only, + # Linux only (macOS uses @loader_path, Windows has no RPATH). + build: + - patchelf # [linux] + host: + - python + - pip + run: + - python + # No version floor: on the `microsoft` channel azure-identity / azure-core / msal + # ship as CalVer (e.g. 2026.06.01), so a semver floor like `>=1.12.0` is a + # misleading no-op there (every published build already satisfies it). + - azure-identity + # --- ODBC Driver 18 payload deps (folded in from the former companion) -------- + # The proprietary driver libs now ship INSIDE this package (the v1.11.0 model: + # libs bundled in the wheel), so there is NO separate `mssql-python-odbc` conda + # package and its declared, security-serviced deps live here instead. + # + # OpenSSL: the driver dlopen's libssl/libcrypto for TLS (Encrypt=yes). Because it + # is dlopen'd (not an ELF NEEDED) conda-build's overlinking can't see it, so it + # must be declared. Linux-only, pinned >=3,<4 (Driver 18 supports the OpenSSL + # 1.1/3.0 ABI only; conda-forge has begun shipping openssl 4). macOS is excluded + # (the signed dylib dlopen's OpenSSL from a hardcoded Homebrew path -- users + # `brew install openssl`); Windows uses SChannel. + - openssl >=3,<4 # [linux] + # Kerberos: libmsodbcsql NEEDs libkrb5.so.3 + libgssapi_krb5.so.2 on Linux. macOS + # uses Kerberos.framework and Windows uses SSPI, so krb5 is Linux-only. + - krb5 # [linux] + # libltdl: libodbcinst.so.2 (the unixODBC driver manager the ODBC driver loads + # through) NEEDs libltdl.so.7, which is NOT bundled on Linux. Declare libtool + # (which provides libltdl.so.7); the $ORIGIN RPATH climb in build.sh makes the + # driver resolve THIS env's copy instead of failing "libltdl.so.7 not found". + # Linux-only (macOS bundles libltdl.7.dylib; Windows has no ltdl). + - libtool # [linux] + # Windows VC++ runtime: msodbcsql18.dll imports VCRUNTIME140.dll, but the vendored + # vcredist ships only msvcp140.dll. Declare the security-serviced conda runtime. + - vc14_runtime # [win] + # conda drops the wheel's platform tag (manylinux_2_28 / macosx_15_0), so + # re-assert that floor as a virtual-package run constraint. The bundled wheels + # already required these, so this is never stricter than what shipped. + - __glibc >=2.28 # [linux] + - __osx >=15.0 # [osx] + +test: + imports: + - mssql_python + +about: + home: https://github.com/microsoft/mssql-python + # This package ships BOTH the MIT-licensed mssql-python code AND the proprietary + # Microsoft ODBC Driver 18 payload (+ the bundled VC++ runtime on Windows), so the + # license is the MIT code license AND the Microsoft proprietary EULA. + license: MIT AND LicenseRef-Microsoft-Proprietary + license_file: + - ../../LICENSE + # The proprietary EULAs live canonically with the ODBC payload; reference them + # there instead of committing duplicate copies in the recipe. conda-build resolves + # license_file relative to the recipe dir at package time and embeds them in + # info/licenses/. + - ../../mssql_python_odbc/licenses/MICROSOFT_ODBC_DRIVER_FOR_SQL_SERVER_LICENSE.txt + - ../../mssql_python_odbc/licenses/MICROSOFT_VISUAL_STUDIO_LICENSE.txt + summary: Microsoft driver for Python to interact with SQL Server and Azure SQL. + description: | + mssql-python is a DB API 2.0 (PEP 249) compliant driver for SQL Server, + Azure SQL, and Azure Synapse. This conda package is self-contained: the + proprietary Microsoft ODBC Driver 18 payload ships inside it (the same model as + the v1.11.0 wheel), so no separate driver package is required. + +extra: + recipe-maintainers: + - jahnvithakkar diff --git a/conda/tls_connect_probe.py b/conda/tls_connect_probe.py new file mode 100644 index 000000000..720db0a2b --- /dev/null +++ b/conda/tls_connect_probe.py @@ -0,0 +1,234 @@ +"""Live ``Encrypt=yes`` TLS gate: prove the driver's OpenSSL backend is REACHABLE. + +Why this exists (and why ``driver_load_probe.py`` is not enough): the Linux +``libmsodbcsql`` links ``libkrb5``/``libgssapi_krb5`` at load time but resolves +its OpenSSL backend (``libssl``/``libcrypto``) by **dlopen at TLS time** -- there +is no ``libssl``/``libcrypto`` ``DT_NEEDED`` or soname string in the binary, so +the crypto libraries are only touched when an actual encrypted handshake runs. +An ``Encrypt=no`` connect (what ``driver_load_probe.py`` does) NEVER exercises +that path, so it cannot reveal an unreachable OpenSSL -- e.g. a conda env where +the declared ``openssl`` lives in ``/lib`` that the vendored driver's +RUNPATH does not reach. Only a real ``Encrypt=yes`` handshake forces the dlopen. + +FAIL-CLOSED contract: + +* ``Encrypt`` is forced to ``yes`` (mandatory encryption), so the pre-login TLS + handshake MUST complete before any LOGIN7 packet is sent. Therefore ANY outcome + that reaches the authentication / database stage -- a clean connect, a + ``Login failed for user`` (18456), or a ``Cannot open database`` -- is POSITIVE + proof that OpenSSL loaded, negotiated, and established the encrypted channel. + These are the only PASS outcomes (see ``_TLS_COMPLETED_MARKERS``). +* Every other outcome fails closed (non-zero exit). In particular an OpenSSL that + could not be loaded surfaces BEFORE login as an ``SSL Provider`` / + ``libssl``/``libcrypto`` / ``cannot open shared object`` error -- classified + here as ``OPENSSL BACKEND UNREACHABLE`` (see ``_OPENSSL_UNREACHABLE_MARKERS``), + which is exactly the conda RUNPATH bug this gate is meant to catch. + +IMPORTANT -- masking caveat: this gate is only CONCLUSIVE on a minimal base with +NO system OpenSSL on the default loader path. On a full agent (or any host with a +system ``libssl``) the driver's dlopen can fall through to the system copy and the +handshake succeeds even when the conda ``/lib`` copy is unreachable -- +masking the very bug, just like the hosted CI agents do today. Run it in a +minimal container (no system OpenSSL) against a reachable server to make it +meaningful. The masking-IMMUNE static guard is +``eng/scripts/audit_bundled_binaries.py`` (it reads the RUNPATH bytes and requires +an ``$ORIGIN/..`` climb regardless of what system libs exist); this live gate is +the complementary end-to-end backstop. + +Config: set ``CONDA_TLS_PROBE_CONN`` to a reachable SQL Server connection string +(creds may be wrong -- reaching ``Login failed`` still proves TLS). If it is not +set the gate SKIPS loudly (exit 0) -- it never silently passes. + +Exit code 0 = TLS handshake completed (OpenSSL reachable) OR skipped; non-zero = +OpenSSL backend unreachable / handshake did not complete (blocks publish). +""" + +import os +import sys + +# Outcomes that can ONLY occur AFTER a mandatory (Encrypt=yes) TLS handshake has +# completed -- i.e. positive proof the dlopen'd OpenSSL backend loaded and +# negotiated the encrypted channel. Matched case-insensitively. +_TLS_COMPLETED_MARKERS = ( + "login failed for user", # LOGIN7 rejected -> handshake already done + "18456", # SQL Server login-failed error number + "cannot open database", # authenticated, database validation stage + "changed database context", # connected successfully + "password did not match", +) + +# Markers that mean the crypto backend could NOT be loaded / the handshake never +# ran. Listed for a crisp FAIL message -- classification is allowlist-based, so an +# unrecognized outcome fails closed even if it matches nothing here. +_OPENSSL_UNREACHABLE_MARKERS = ( + "libssl", + "libcrypto", + "cannot open shared object", # linux dlopen failure of the crypto backend + "image not found", # macOS dlopen failure + "openssl", + "ssl provider", # an SSL Provider error before login = crypto/handshake fail + "ssl routines", + "encryption not supported", + "unable to load", + "cannot load", +) + + +def tls_completed(exc): + """FAIL-CLOSED classifier: True only when the TLS handshake provably completed. + + ``exc is None`` (clean connect) or a post-handshake authentication/database + diagnostic returns True; every other outcome -- including an OpenSSL-load + failure or anything unrecognized -- returns False so the gate exits non-zero. + """ + if exc is None: + return True + msg = str(exc).lower() + return any(marker in msg for marker in _TLS_COMPLETED_MARKERS) + + +def describe(exc): + """Short, human-readable reason string for the gate's stdout / exit line.""" + if exc is None: + return "clean connect (TLS handshake completed)" + msg = str(exc) + low = msg.lower() + for marker in _OPENSSL_UNREACHABLE_MARKERS: + if marker in low: + return "OpenSSL backend unreachable -> " + msg[:300] + return msg[:300] + + +def _split_top_level(conn): + """Split an ODBC connection string on TOP-LEVEL ``;`` only. + + An ODBC value wrapped in ``{...}`` may itself contain ``;`` (MS-ODBCSTR), so a + naive ``split(';')`` would shred braced values. Track brace depth and break only + at depth 0. + """ + segments = [] + buf = "" + depth = 0 + for ch in conn.strip(): + if ch == "{": + depth += 1 + buf += ch + elif ch == "}": + depth = max(0, depth - 1) + buf += ch + elif ch == ";" and depth == 0: + segments.append(buf) + buf = "" + else: + buf += ch + segments.append(buf) + return segments + + +def force_tls(conn): + """Force ``Encrypt=yes`` and ``TrustServerCertificate=yes`` on the string. + + Encrypt=yes makes the pre-login TLS handshake mandatory (the whole point of + the gate). TrustServerCertificate=yes lets it reach the auth stage against a + local dev server's self-signed cert -- this is a local connectivity gate, NOT + a security assertion, and must never be copied into a production connection. + + Rebuild-from-tokens (NOT regex substitution): brace-aware split on top-level + ``;``, DROP any existing Encrypt / TrustServerCertificate segment (case- + insensitive -- including a valueless ``Encrypt`` or a duplicate), then append the + canonical pair exactly once. A regex substitution can corrupt the string -- e.g. + ``Encrypt=;yes`` becomes ``Encrypt=yes;yes``, leaving a bare ``yes`` the parser + rejects with "keyword 'yes' has no value", and a duplicate ``Encrypt`` slips + through as a "Duplicate keyword" error; rebuilding from tokens cannot. + """ + kept = [] + for seg in _split_top_level(conn): + token = seg.strip() + if not token: + continue + key = token.split("=", 1)[0].strip().lower() + if key in ("encrypt", "trustservercertificate"): + continue # drop any existing (incl. valueless / duplicate); re-added below + kept.append(token) + kept.append("Encrypt=yes") + kept.append("TrustServerCertificate=yes") + return ";".join(kept) + + +def _redact(conn): + """Render the connection string's STRUCTURE with every value masked. + + Safe to log: shows the keys (and their order) so a malformed string is + diagnosable, but never a secret value. A segment with no ``=`` -- the exact shape + that trips the parser -- is surfaced verbatim so the failure explains itself. + """ + shown = [] + for seg in _split_top_level(conn): + token = seg.strip() + if not token: + continue + if "=" in token: + shown.append(token.split("=", 1)[0].strip() + "=***") + else: + shown.append("<>") + return ";".join(shown) + + +def _is_probe_connection_string(raw): + """True if ``raw`` looks like an ODBC connection string (has a key=value pair). + + Guards the common misconfiguration of treating CONDA_TLS_PROBE_CONN as a yes/no + toggle: a bare ``yes``/``true``/``1`` has no ``=``, so it cannot be a connection + string and must not be handed to the parser (which would fail on a bare keyword). + """ + return "=" in raw + + +def main(): + raw = os.environ.get("CONDA_TLS_PROBE_CONN", "").strip() + if not raw: + print( + "TLS_PROBE_SKIPPED: set CONDA_TLS_PROBE_CONN to a reachable SQL Server " + "connection string (on a minimal base with no system OpenSSL) to run " + "this Encrypt=yes gate." + ) + return + + if not _is_probe_connection_string(raw): + # A bare word like "yes"/"true"/"1" is the "I thought it was a yes/no toggle" + # misconfiguration. It is NOT a connection string, so feeding it to the driver + # only fails the leg on an unrelated parse error ("keyword 'yes' has no value"). + # Skip LOUDLY instead -- the static RUNPATH audit still guards OpenSSL layout. + print( + "TLS_PROBE_SKIPPED: CONDA_TLS_PROBE_CONN is set but is not a connection string " + "(no 'key=value' pair). It is NOT a yes/no toggle -- set it to a reachable SQL " + "Server connection string (with Server, user and password keywords) to run the " + "Encrypt=yes gate, or leave it empty to skip." + ) + return + + conn_str = force_tls(raw) + print("TLS_PROBE using (values redacted): " + _redact(conn_str)) + + # Deferred so this module can be imported (and the classifier unit-tested) + # WITHOUT the compiled extension / driver payload. + import mssql_python + + outcome = None + try: + conn = mssql_python.connect(conn_str) + try: + conn.close() + except Exception: # noqa: BLE001 - best-effort cleanup only + pass + except Exception as exc: # noqa: BLE001 - deliberately classified below + outcome = exc + + if tls_completed(outcome): + print("TLS_OK (OpenSSL backend reachable; " + describe(outcome) + ")") + return + sys.exit("TLS/OPENSSL BACKEND UNREACHABLE: " + describe(outcome)) + + +if __name__ == "__main__": + main() diff --git a/conda/validate_conda_release.py b/conda/validate_conda_release.py new file mode 100644 index 000000000..710c619fb --- /dev/null +++ b/conda/validate_conda_release.py @@ -0,0 +1,284 @@ +"""Metadata-based conda release-readiness gate. + +The release pipeline must never ship an incomplete conda set. This module reads +the AUTHORITATIVE ``info/index.json`` embedded in every ``.conda`` / ``.tar.bz2`` +(never folder names or bare counts) and validates the self-contained +``mssql-python`` package -- which vendors the ODBC Driver 18 payload, so there is +NO separate companion package: + +* every package's real ``subdir`` is in the allowed set AND matches its folder + (catches a mislabeled / mis-stamped leg); +* the only package name is ``mssql-python`` and its version matches the expected + release version (or, if none supplied, is internally consistent -- one version); +* the full (required-subdir x Python) matrix is complete -- every required + platform ships a package for every expected Python. + +Exit code 0 = release-ready; non-zero = a violation was found (blocks publish). +""" + +from __future__ import annotations + +import argparse +import io +import json +import re +import sys +import tarfile +import zipfile +from collections import defaultdict + +_BINDING_NAME = "mssql-python" + +_PY_TAG_RE = re.compile(r"py(\d)(\d{1,2})") +_PY_DEP_RE = re.compile(r"python\s+(\d+)\.(\d+)") + + +def _zstd_decompress(raw: bytes) -> bytes: + """Decompress a zstandard blob, preferring the 3.14+ stdlib backend.""" + try: # Python 3.14+ + from compression import zstd # type: ignore + + return zstd.decompress(raw) + except Exception: # pragma: no cover - exercised via the third-party path + pass + import zstandard # third-party fallback + + return zstandard.ZstdDecompressor().decompress(raw) + + +def read_index_json(path: str) -> dict: + """Return the parsed ``info/index.json`` from a ``.conda`` / ``.tar.bz2``.""" + if path.endswith(".conda"): + with zipfile.ZipFile(path) as zf: + info_name = next( + (n for n in zf.namelist() if n.startswith("info-") and n.endswith(".tar.zst")), + None, + ) + if info_name is None: + raise ValueError(f"{path}: no info-*.tar.zst member (malformed .conda package)") + info_blob = zf.read(info_name) + with tarfile.open(fileobj=io.BytesIO(_zstd_decompress(info_blob))) as tf: + member = tf.extractfile("info/index.json") + if member is None: # pragma: no cover - malformed package + raise ValueError(f"{path}: info/index.json missing") + return json.load(member) + if path.endswith(".tar.bz2"): + with tarfile.open(path, "r:bz2") as tf: + member = tf.extractfile("info/index.json") + if member is None: # pragma: no cover - malformed package + raise ValueError(f"{path}: info/index.json missing") + return json.load(member) + raise ValueError(f"{path}: unrecognized conda package extension") + + +def python_tag_from_index(index: dict) -> str: + """Extract the ``X.Y`` Python version a package is built for, or ``''``. + + Uses the build string's ``pyXY`` token first (authoritative for conda-build + Python packages), then falls back to a ``python X.Y`` run dependency. A + Python-agnostic package (build string ``0``) has neither and returns ``''``. + """ + match = _PY_TAG_RE.search(str(index.get("build", ""))) + if match: + return f"{match.group(1)}.{match.group(2)}" + for dep in index.get("depends", []) or []: + match = _PY_DEP_RE.match(str(dep)) + if match: + return f"{match.group(1)}.{match.group(2)}" + return "" + + +def validate( + packages: list[dict], + required_subdirs: list[str], + allowed_subdirs: list[str], + expected_pythons: list[str], + expected_versions: dict | None = None, +) -> list[str]: + """Return a list of human-readable violation strings (empty == release-ready). + + ``packages`` is a list of dicts with keys: ``folder`` (staged subdir folder), + ``subdir`` (real info/index.json subdir), ``name``, ``version``, ``build``, + ``python`` (``X.Y`` or ``''``). + """ + errors: list[str] = [] + expected_versions = expected_versions or {} + + # 1. Authoritative subdir must be allowed AND match the folder it was staged in. + for p in packages: + ident = f"{p['name']}-{p['version']}-{p['build']}" + if p["subdir"] not in allowed_subdirs: + errors.append( + f"{ident}: real subdir '{p['subdir']}' is not in allowed set {allowed_subdirs}." + ) + if p["subdir"] != p["folder"]: + errors.append( + f"MISLABELED: {ident} is staged in folder '{p['folder']}' but its " + f"info/index.json subdir is '{p['subdir']}'." + ) + + # 2. Only the self-contained mssql-python package may appear; versions match + # expected (or are internally consistent -- one version per package). + seen_versions: dict = defaultdict(set) + for p in packages: + if p["name"] != _BINDING_NAME: + errors.append( + f"unexpected package name '{p['name']}' ({p['version']}); the " + f"self-contained conda package ships only '{_BINDING_NAME}'." + ) + continue + seen_versions[p["name"]].add(p["version"]) + for name, versions in seen_versions.items(): + if len(versions) > 1: + errors.append( + f"{name}: multiple versions present {sorted(versions)} " + f"(a release must ship exactly one version per package)." + ) + exp = expected_versions.get(name) + if exp is not None: + for v in versions: + if v != exp: + errors.append(f"{name}: version '{v}' != expected '{exp}'.") + + # 2b. Reject duplicate (name, version, subdir, python) keys. Two packages with + # an identical key are never legitimate -- it means one leg's package bled + # into another subdir's staging folder (the shared-output-dir hazard) or was + # staged twice. The per-subdir matrix check below collapses variants into a + # set, so a duplicate would silently MASK a genuinely missing variant; fail + # loudly on the duplicate instead. + key_folders: dict = defaultdict(list) + for p in packages: + key_folders[(p["name"], p["version"], p["subdir"], p["python"])].append(p["folder"]) + for (name, version, subdir, python), folders in sorted(key_folders.items()): + if len(folders) > 1: + errors.append( + f"DUPLICATE: {name}-{version} (subdir '{subdir}', python " + f"'{python or '-'}') appears {len(folders)}x (staged in {sorted(folders)})." + ) + + # Group by the REAL (metadata) subdir, never the folder name. + by_subdir: dict = defaultdict(list) + for p in packages: + by_subdir[p["subdir"]].append(p) + + # 3. Required subdirs must be PRESENT; every present ALLOWED subdir must ship a + # COMPLETE per-Python matrix. Validating present-but-not-required subdirs too + # (not just the required set) stops a partially built allowed subdir -- e.g. a + # half-finished win-arm64 -- from slipping through to publish just because it + # is not in the required set. + for sub in required_subdirs: + if not by_subdir.get(sub): + errors.append(f"required subdir '{sub}' is MISSING.") + + for sub in sorted(by_subdir): + if sub not in allowed_subdirs: + # Not an allowed subdir: already flagged per-package in step 1. Skip the + # matrix work so the error set stays focused on the root cause. + continue + grp = by_subdir[sub] + bindings = [p for p in grp if p["name"] == _BINDING_NAME] + if not bindings: + errors.append(f"subdir '{sub}': no {_BINDING_NAME} package.") + continue + + for p in bindings: + if not p["python"]: + errors.append( + f"{p['name']}-{p['version']}-{p['build']} in '{sub}' has no " + f"detectable Python tag (build string should carry pyXY)." + ) + got_pythons = sorted({p["python"] for p in bindings if p["python"]}) + missing = [py for py in expected_pythons if py not in got_pythons] + if missing: + errors.append( + f"subdir '{sub}': matrix INCOMPLETE -- missing Python {missing} " + f"(present: {got_pythons or 'none'})." + ) + + return errors + + +def collect_packages(root: str) -> list[dict]: + """Read every ``.conda`` / ``.tar.bz2`` under ``root`` into package dicts.""" + import glob + import os + + paths = sorted( + glob.glob(os.path.join(root, "**", "*.conda"), recursive=True) + + glob.glob(os.path.join(root, "**", "*.tar.bz2"), recursive=True) + ) + packages = [] + for path in paths: + index = read_index_json(path) + packages.append( + { + "folder": os.path.basename(os.path.dirname(path)), + "subdir": str(index.get("subdir", "")), + "name": str(index.get("name", "")), + "version": str(index.get("version", "")), + "build": str(index.get("build", "")), + "python": python_tag_from_index(index), + "path": path, + } + ) + return packages + + +def _split(value: str) -> list[str]: + return [x.strip() for x in value.split(",") if x.strip()] + + +def main(argv: list | None = None) -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--root", required=True, help="Root of the consolidated conda tree.") + parser.add_argument( + "--required-subdirs", default="win-64,osx-64,osx-arm64,linux-64,linux-aarch64" + ) + parser.add_argument( + "--allowed-subdirs", + default="win-64,win-arm64,osx-64,osx-arm64,linux-64,linux-aarch64", + ) + parser.add_argument("--pythons", default="3.10,3.11,3.12,3.13,3.14") + parser.add_argument("--mssql-python-version", default=None) + parser.add_argument("--mssql-python-odbc-version", default=None) + args = parser.parse_args(argv) + + packages = collect_packages(args.root) + if not packages: + print(f"ERROR: no conda packages found under {args.root}.", file=sys.stderr) + return 1 + + expected_versions = {} + if args.mssql_python_version: + expected_versions[_BINDING_NAME] = args.mssql_python_version + # --mssql-python-odbc-version is accepted for back-compat but ignored: the + # self-contained mssql-python package vendors the ODBC payload, so there is no + # separate companion package to version. + + print(f"Discovered {len(packages)} conda package(s):") + for p in sorted(packages, key=lambda x: (x["subdir"], x["name"], x["python"])): + print( + f" {p['subdir']:<14} {p['name']:<18} {p['version']:<12} " + f"py={p['python'] or '-':<5} build={p['build']}" + ) + + errors = validate( + packages, + required_subdirs=_split(args.required_subdirs), + allowed_subdirs=_split(args.allowed_subdirs), + expected_pythons=_split(args.pythons), + expected_versions=expected_versions, + ) + + if errors: + print("\nConda release readiness FAILED:", file=sys.stderr) + for e in errors: + print(f" - {e}", file=sys.stderr) + return 1 + + print("\nOK: metadata-validated conda set is release-ready (subdirs, Python matrix, pairing).") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/eng/scripts/audit_bundled_binaries.py b/eng/scripts/audit_bundled_binaries.py new file mode 100644 index 000000000..ba4bf0a0c --- /dev/null +++ b/eng/scripts/audit_bundled_binaries.py @@ -0,0 +1,471 @@ +#!/usr/bin/env python3 +"""Masking-immune audit of the vendored Linux ODBC binaries in built conda packages. + +The #563 reachability fix lives in the ELF RUNPATH of the vendored driver, not in a +comment or a runtime probe. A runtime ``ldd``/import check can PASS on any host that +happens to carry a system ``krb5``/``libltdl`` -- the driver silently binds the +system copy and a wrong/missing conda climb goes unnoticed (exactly what the full CI +agents hide). This audit is immune to that masking: it reads the ELF bytes straight +out of each built ``.conda`` payload -- via the ``PT_DYNAMIC`` program header the +*loader itself* uses -- and asserts, statically and exactly: + + * ``libmsodbcsql*`` and ``libodbcinst.so.2`` carry the EXACT relative ``$ORIGIN`` + climb that lands on the package-root ``lib`` (== ``$PREFIX/lib``), computed from + each binary's own location -- not a substring, not "any ``..``". A too-short, + overshooting, or ``$ORIGINATOR`` climb FAILS; + * that climb entry appears in the EFFECTIVE RUNPATH: the loader honours + ``DT_RUNPATH`` and IGNORES ``DT_RPATH`` when ``DT_RUNPATH`` is present, so a good + ``DT_RPATH`` decoy behind a bad ``DT_RUNPATH`` FAILS; + * no absolute RPATH entry exists (stay relocatable); + * the run deps that SERVICE the driver -- ``krb5``, ``libtool`` (libltdl provider), + ``openssl`` -- are DECLARED in ``info/index.json`` ``depends`` (deleting a dep + from ``meta.yaml`` must fail here, not just be masked at runtime), and the driver + still ``DT_NEEDED``s ``libkrb5``/``libgssapi_krb5``/``libodbcinst`` (and + ``libodbcinst`` still needs ``libltdl``) so a driver that stopped needing krb5 is + caught too; + * no ``krb5``/``openssl``/``libltdl`` is VENDORED inside the payload (they are + serviced by conda, never bundled). + +Non-Linux packages (``win-*`` / ``osx-*``) have no such ELF payload and are skipped. +An unreadable/malformed package FAILS (it is never silently treated as non-Linux). + +Exit code 0 = every Linux package is exactly self-contained; non-zero = a violation +was found (blocks the build/release). +""" + +from __future__ import annotations + +import argparse +import glob +import io +import json +import os +import posixpath +import struct +import sys +import tarfile +import zipfile + +# --- ELF constants --------------------------------------------------------- +_DT_NEEDED = 1 +_DT_STRTAB = 5 +_DT_RPATH = 15 +_DT_RUNPATH = 29 +_PT_LOAD = 1 +_PT_DYNAMIC = 2 + +# Driver binaries whose RUNPATH must carry the exact $ORIGIN climb. +_DRIVER_PREFIXES = ("libmsodbcsql-", "libmsodbcsql.") +_ODBCINST = "libodbcinst.so.2" + +# Libraries that must be serviced by DECLARED conda deps, never vendored into the +# Linux payload (bundling these is the anti-pattern the recipe avoids). +_MUST_NOT_VENDOR = ("libkrb5", "libgssapi", "libssl", "libcrypto", "libltdl") + +# conda run-deps that SERVICE the driver's krb5/openssl/libltdl. Missing any means +# the $PREFIX/lib copy the RUNPATH climb points at would not exist -- declaration is +# as load-bearing as the climb itself. +_REQUIRED_DEPS = ("krb5", "libtool", "openssl") + +# Expected DT_NEEDED soname substrings, so a driver that silently STOPPED needing +# krb5 (making the declared dep moot) is caught too. +_DRIVER_NEEDED = ("libkrb5", "libgssapi_krb5", "libodbcinst") +_ODBCINST_NEEDED = ("libltdl",) + + +def _zstd_decompress(raw: bytes) -> bytes: + """Decompress a zstandard blob, preferring the 3.14+ stdlib backend.""" + try: # Python 3.14+ + from compression import zstd # type: ignore + + return zstd.decompress(raw) + except Exception: + pass + import zstandard # third-party fallback + + return zstandard.ZstdDecompressor().decompress(raw) + + +def _is_elf(data: bytes) -> bool: + return len(data) >= 64 and data[:4] == b"\x7fELF" + + +def elf_dynamic(data: bytes) -> dict: + """Return ``{'runpath': str|None, 'rpath': str|None, 'needed': [str]}``. + + Parses the ``PT_DYNAMIC`` program header -- the segment the LOADER actually uses + -- and maps ``DT_STRTAB``'s virtual address to a file offset through the + ``PT_LOAD`` segments, so this matches the loader's own view rather than a section + table that a stripped/rewritten binary might not carry. Handles ELF32/ELF64 and + both endiannesses; the shipped drivers are ELF64-LE. + """ + out: dict = {"runpath": None, "rpath": None, "needed": []} + if not _is_elf(data): + return out + is64 = data[4] == 2 + en = "<" if data[5] == 1 else ">" + + if is64: + e_phoff = struct.unpack_from(en + "Q", data, 0x20)[0] + e_phentsize = struct.unpack_from(en + "H", data, 0x36)[0] + e_phnum = struct.unpack_from(en + "H", data, 0x38)[0] + else: + e_phoff = struct.unpack_from(en + "I", data, 0x1C)[0] + e_phentsize = struct.unpack_from(en + "H", data, 0x2A)[0] + e_phnum = struct.unpack_from(en + "H", data, 0x2C)[0] + if not e_phoff or not e_phnum: + return out + + loads = [] # (p_vaddr, p_offset, p_filesz) + dyn = None # (p_offset, p_filesz) + for i in range(e_phnum): + off = e_phoff + i * e_phentsize + if off + e_phentsize > len(data): + return out + p_type = struct.unpack_from(en + "I", data, off)[0] + if is64: + p_offset = struct.unpack_from(en + "Q", data, off + 8)[0] + p_vaddr = struct.unpack_from(en + "Q", data, off + 16)[0] + p_filesz = struct.unpack_from(en + "Q", data, off + 32)[0] + else: + p_offset = struct.unpack_from(en + "I", data, off + 4)[0] + p_vaddr = struct.unpack_from(en + "I", data, off + 8)[0] + p_filesz = struct.unpack_from(en + "I", data, off + 16)[0] + if p_type == _PT_LOAD: + loads.append((p_vaddr, p_offset, p_filesz)) + elif p_type == _PT_DYNAMIC: + dyn = (p_offset, p_filesz) + if dyn is None: + return out + dyn_off, dyn_size = dyn + + def vaddr_to_off(vaddr: int): + for v, o, sz in loads: + if v <= vaddr < v + sz: + return vaddr - v + o + return None + + strtab_vaddr = None + runpath_rel = None + rpath_rel = None + needed_rel: list[int] = [] + entsize = 16 if is64 else 8 + for off in range(dyn_off, dyn_off + dyn_size, entsize): + if off + entsize > len(data): + break + if is64: + d_tag = struct.unpack_from(en + "q", data, off)[0] + d_val = struct.unpack_from(en + "Q", data, off + 8)[0] + else: + d_tag = struct.unpack_from(en + "i", data, off)[0] + d_val = struct.unpack_from(en + "I", data, off + 4)[0] + if d_tag == 0: # DT_NULL terminates the array + break + if d_tag == _DT_STRTAB: + strtab_vaddr = d_val + elif d_tag == _DT_RUNPATH: + runpath_rel = d_val + elif d_tag == _DT_RPATH: + rpath_rel = d_val + elif d_tag == _DT_NEEDED: + needed_rel.append(d_val) + if strtab_vaddr is None: + return out + strtab_off = vaddr_to_off(strtab_vaddr) + if strtab_off is None: + return out + + def read_str(rel: int) -> str: + pos = strtab_off + rel + end = data.find(b"\x00", pos) + return data[pos : (end if end >= 0 else len(data))].decode("utf-8", "replace") + + if runpath_rel is not None: + out["runpath"] = read_str(runpath_rel) + if rpath_rel is not None: + out["rpath"] = read_str(rpath_rel) + out["needed"] = [read_str(n) for n in needed_rel] + return out + + +def effective_runpath(dyn: dict): + """The loader ignores ``DT_RPATH`` when ``DT_RUNPATH`` is present.""" + return dyn["runpath"] if dyn["runpath"] is not None else dyn["rpath"] + + +def _entries(runpath) -> list[str]: + return [e for e in (runpath or "").split(":") if e] + + +def expected_climb_entry(member_name: str) -> str: + """Exact ``$ORIGIN/`` from the member's own dir to package-root ``lib``. + + conda stores python files at ``lib/pythonX.Y/site-packages/...`` and + ``$PREFIX/lib`` == package-root ``lib``, so the climb is the POSIX relpath from + the driver's directory to the top-level ``lib`` (never a hard-coded ``../`` count). + """ + member_dir = posixpath.dirname(member_name) + climb = posixpath.relpath("lib", member_dir) + return "$ORIGIN/" + climb + + +def _iter_payload_members(path: str): + """Yield ``(member_name, data_bytes)`` for the files in a conda package payload.""" + if path.endswith(".conda"): + with zipfile.ZipFile(path) as zf: + pkg_name = next( + (n for n in zf.namelist() if n.startswith("pkg-") and n.endswith(".tar.zst")), + None, + ) + if pkg_name is None: + return + blob = _zstd_decompress(zf.read(pkg_name)) + with tarfile.open(fileobj=io.BytesIO(blob)) as tf: + for m in tf.getmembers(): + if not m.isfile(): + continue + f = tf.extractfile(m) + if f is not None: + yield m.name, f.read() + elif path.endswith(".tar.bz2"): + with tarfile.open(path, "r:bz2") as tf: + for m in tf.getmembers(): + if not m.isfile(): + continue + f = tf.extractfile(m) + if f is not None: + yield m.name, f.read() + + +def read_index(path: str) -> dict: + """Return the package's ``info/index.json`` as a dict. + + RAISES on a malformed/unreadable package -- callers must NOT swallow this into a + silent "non-Linux, skip" (a truncated Linux package would then slip through). + """ + if path.endswith(".conda"): + with zipfile.ZipFile(path) as zf: + info_name = next( + (n for n in zf.namelist() if n.startswith("info-") and n.endswith(".tar.zst")), + None, + ) + if info_name is None: + raise ValueError("no info-*.tar.zst member (malformed .conda)") + blob = _zstd_decompress(zf.read(info_name)) + with tarfile.open(fileobj=io.BytesIO(blob)) as tf: + member = tf.extractfile("info/index.json") + if member is None: + raise ValueError("info/index.json missing") + return json.load(member) + if path.endswith(".tar.bz2"): + with tarfile.open(path, "r:bz2") as tf: + member = tf.extractfile("info/index.json") + if member is None: + raise ValueError("info/index.json missing") + return json.load(member) + raise ValueError("unrecognized conda package extension") + + +def _dep_names(depends) -> set: + """The package names (first token) of an ``info/index.json`` ``depends`` list.""" + names = set() + for d in depends or []: + token = str(d).strip().split() + if token: + names.add(token[0]) + return names + + +def audit_package(path: str) -> list[str]: + """Return a list of violation strings for one package (empty == clean).""" + base_name = os.path.basename(path) + try: + index = read_index(path) + except Exception as exc: # H2: malformed/unreadable must FAIL, never skip. + return [f"{base_name}: unreadable/malformed package metadata ({exc})."] + + subdir = str(index.get("subdir", "")) + if not subdir.startswith("linux"): + print(f" SKIP (no Linux ELF payload): {base_name} [subdir={subdir or '?'}]") + return [] + + errors: list[str] = [] + + # N2a: the run deps that SERVICE the driver's krb5/openssl/libltdl must be declared. + dep_names = _dep_names(index.get("depends")) + for req in _REQUIRED_DEPS: + if req not in dep_names: + errors.append( + f"{base_name}: info/index.json depends is missing '{req}' -- the " + f"$PREFIX/lib copy the RUNPATH climb targets would not exist. " + f"depends={sorted(dep_names)}" + ) + # openssl must be RANGE-pinned for Driver 18 (which supports only the OpenSSL + # 1.1/3.0 ABI; conda-forge has begun shipping openssl 4), not merely present. + if "openssl" in dep_names: + spec = next( + (str(d) for d in (index.get("depends") or []) if str(d).split()[:1] == ["openssl"]), + "openssl", + ) + constraint = spec[len("openssl") :].strip() + if ">=3" not in constraint or "<4" not in constraint: + errors.append( + f"{base_name}: openssl dep '{spec}' is not range-pinned '>=3,<4' " + f"(Driver 18 supports only the OpenSSL 1.1/3.0 ABI)." + ) + + lib_dirs: set = set() + dirs_with_driver: set = set() + dirs_with_inst: set = set() + vendored: list[str] = [] + + for name, data in _iter_payload_members(path): + base = posixpath.basename(name) + norm = "/" + name + member_dir = posixpath.dirname(name) + # Track every driver lib dir (mssql_python_odbc/libs/linux///lib). + if "/libs/linux/" in norm and member_dir.endswith("/lib"): + lib_dirs.add(member_dir) + + # Flag any crypto/krb5/ltdl library vendored into the Linux payload. + if "/libs/linux/" in norm and any( + base.startswith(p) and ".so" in base for p in _MUST_NOT_VENDOR + ): + vendored.append(name) + continue + + is_driver = any(base.startswith(p) for p in _DRIVER_PREFIXES) + is_inst = base == _ODBCINST + if not (is_driver or is_inst): + continue + if not _is_elf(data): + errors.append(f"{name}: expected an ELF binary but the header is not ELF.") + continue + + dyn = elf_dynamic(data) + entries = _entries(effective_runpath(dyn)) + needed = dyn["needed"] + # musl/alpine variants (NEEDED libc.musl*) link differently -- their libodbcinst + # statically resolves libltdl, so the glibc DT_NEEDED requirements below do not + # apply. There is no musl conda subdir (conda Linux is glibc-only); these variants + # ride along in the payload but are never the conda load target. The climb / + # presence / no-vendored checks still apply to them. + is_musl = any("libc.musl" in n for n in needed) + want = expected_climb_entry(name) + + # Bare $ORIGIN must ALSO be present: it is how the driver resolves its + # co-located sibling libodbcinst.so.2. Losing it breaks driver-manager loading + # even when the $PREFIX/lib climb entry is intact. + if "$ORIGIN" not in entries: + errors.append( + f"{name}: effective RUNPATH {entries or '[none]'} lacks bare '$ORIGIN' " + f"(co-located sibling resolution for libodbcinst.so.2). NEEDED={needed}" + ) + # N1: the EXACT climb entry must be present in the EFFECTIVE RUNPATH. + if want not in entries: + errors.append( + f"{name}: effective RUNPATH {entries or '[none]'} does not contain the " + f"exact climb entry '{want}' to $PREFIX/lib (the loader uses DT_RUNPATH " + f"when present, else DT_RPATH). NEEDED={needed}" + ) + # Stay relocatable: reject ANY absolute entry. + abs_entries = [e for e in entries if e.startswith("/")] + if abs_entries: + errors.append( + f"{name}: RUNPATH has ABSOLUTE entries {abs_entries}; must stay " + f"relocatable (relative $ORIGIN only)." + ) + + # N2b: the expected DT_NEEDED set must still be present (glibc variants only; + # musl links these statically / differently, and is not a conda target). + if is_driver: + dirs_with_driver.add(member_dir) + if not is_musl: + for want_need in _DRIVER_NEEDED: + if not any(want_need in n for n in needed): + errors.append( + f"{name}: driver no longer NEEDs '{want_need}*' (NEEDED={needed}); " + f"the declared conda dep would go unused and reachability is unproven." + ) + if is_inst: + dirs_with_inst.add(member_dir) + if not is_musl: + for want_need in _ODBCINST_NEEDED: + if not any(want_need in n for n in needed): + errors.append( + f"{name}: libodbcinst.so.2 no longer NEEDs '{want_need}*' " + f"(NEEDED={needed})." + ) + print(f" {subdir}/{base}: effective RUNPATH={entries} NEEDED={needed}") + + if vendored: + errors.append( + f"{base_name}: vendors libraries that must be DECLARED conda deps, not " + f"bundled: {sorted(vendored)} (krb5/openssl/libltdl are serviced by conda, " + f"never shipped inside the payload)." + ) + # Per-subdir presence: EVERY discovered driver lib dir must ship BOTH a driver and + # libodbcinst.so.2. A package-global count would let a driver missing from ONE + # distro subdir (alpine/debian_ubuntu/rhel/suse) slip past. + if not lib_dirs: + errors.append( + f"{base_name}: no mssql_python_odbc/libs/linux/*/*/lib directory found in a " + f"Linux package." + ) + for d in sorted(lib_dirs): + if d not in dirs_with_driver: + errors.append(f"{base_name}: '{d}' has no libmsodbcsql* driver.") + if d not in dirs_with_inst: + errors.append(f"{base_name}: '{d}' has no libodbcinst.so.2.") + return errors + + +def main(argv: list | None = None) -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--root", help="Directory to scan recursively for *.conda / *.tar.bz2.") + parser.add_argument("packages", nargs="*", help="Explicit package paths to audit.") + args = parser.parse_args(argv) + + paths = list(args.packages) + if args.root: + paths += glob.glob(os.path.join(args.root, "**", "*.conda"), recursive=True) + paths += glob.glob(os.path.join(args.root, "**", "*.tar.bz2"), recursive=True) + paths = sorted(set(paths)) + + if not paths: + print( + "ERROR: no conda packages to audit (pass --root DIR or package paths).", file=sys.stderr + ) + return 1 + + print(f"Auditing RUNPATH self-containment of {len(paths)} conda package(s):") + all_errors: list[str] = [] + linux_checked = 0 + for p in paths: + try: + if str(read_index(p).get("subdir", "")).startswith("linux"): + linux_checked += 1 + except Exception: + # A malformed package is a violation, reported by audit_package below. + pass + all_errors.extend(audit_package(p)) + + if all_errors: + print("\nRUNPATH audit FAILED:", file=sys.stderr) + for e in all_errors: + print(f" - {e}", file=sys.stderr) + return 1 + + if linux_checked == 0: + print("\nOK: no Linux packages present; nothing to audit (win/osx have no ELF payload).") + else: + print( + f"\nOK: all {linux_checked} Linux package(s) carry the EXACT $ORIGIN climb, keep " + f"their krb5/gssapi/libltdl NEEDEDs, declare krb5/libtool/openssl, and vendor no " + f"crypto (conda services them)." + ) + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/tests/test_026_driver_load_probe.py b/tests/test_026_driver_load_probe.py new file mode 100644 index 000000000..71feceb5d --- /dev/null +++ b/tests/test_026_driver_load_probe.py @@ -0,0 +1,144 @@ +"""Fail-closed classification tests for ``conda/driver_load_probe.py``. + +The conda test-before-publish gate runs ``conda/driver_load_probe.py`` to prove +the repackaged native ODBC driver actually loads (not just the tiny +``mssql_python_odbc`` shim). The probe MUST fail closed: a broken / missing / +mis-architecture driver -- whose failure surfaces as the C++ +``LoadDriverOrThrowException`` family ("Failed to load the driver...", "Failed +to load library: ", "Failed to load required function pointers...") -- has +to make the probe exit non-zero, while a genuine connection-stage failure +(driver loaded, TCP/TLS/auth attempted) has to pass. + +These are pure, no-DB unit tests: the probe's native ``import mssql_python`` is +deferred into ``main()``, so the classifier can be loaded and exercised with a +stubbed connector without the compiled extension or a live SQL Server. +""" + +import importlib.util +import sys +import types +from pathlib import Path + +import pytest + +_PROBE_PATH = Path(__file__).resolve().parent.parent / "conda" / "driver_load_probe.py" + +# The conda/ sources are not shipped inside the built wheel, so the installed-wheel +# test leg copies only tests/ into an isolated dir. Skip the whole module (rather than +# erroring at collection/run) when the conda source it exercises is absent. +if not _PROBE_PATH.is_file(): + pytest.skip( + f"conda source not present ({_PROBE_PATH}); skipping conda driver-load probe tests", + allow_module_level=True, + ) + + +def _load_probe(): + """Import ``conda/driver_load_probe.py`` as a standalone module.""" + spec = importlib.util.spec_from_file_location("driver_load_probe_under_test", _PROBE_PATH) + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +# Messages the loaded msodbcsql driver emits once it has reached the network / +# TLS / auth stage. Every one of these MUST classify as "driver loaded" (PASS). +_LOADED_MESSAGES = [ + "Driver Error: Connection operation failed; DDBC Error: [Microsoft][ODBC Driver 18 for " + "SQL Server]TCP Provider: No connection could be made because the target machine actively " + "refused it.", + "[Microsoft][ODBC Driver 18 for SQL Server]Login timeout expired", + "[Microsoft][ODBC Driver 18 for SQL Server]TCP Provider: Error code 0x2726", + "[Microsoft][ODBC Driver 18 for SQL Server]A network-related or instance-specific error " + "has occurred", + "[Microsoft][ODBC Driver 18 for SQL Server]SSL Provider: certificate verify failed", + "[Microsoft][ODBC Driver 18 for SQL Server]Login failed for user 'x'.", + "connection refused", +] + +# Messages that mean the native driver did NOT load / link / resolve. Every one +# of these MUST classify as "not loaded" (FAIL / non-zero exit). +_LOAD_FAILURE_MESSAGES = [ + "Failed to load the driver. Please read the documentation " + "(https://github.com/microsoft/mssql-python#installation) to install the required " + "dependencies.", + "Failed to load library: C:\\x\\msodbcsql18.dll", + "Failed to load required function pointers from driver.", + "ODBC driver not found at: /x/libmsodbcsql-18.5.so.2.1", + "Failed to load mssql-auth.dll. Please ensure it is present in the expected directory.", + "mssql-auth.dll not found. If you are using Entra ID, please ensure it is present.", + "The mssql-python-odbc package (which ships the ODBC driver binaries) is not installed.", + "dlopen(...): image not found", + "libcrypto.so.3: cannot open shared object file: No such file or directory", + "Unsupported architecture", + # Fail-closed default: an unexpected / unrelated error is NOT proof of load. + "some totally unexpected internal error", +] + + +@pytest.mark.parametrize("msg", _LOADED_MESSAGES) +def test_driver_loaded_true_for_connection_stage_errors(msg): + probe = _load_probe() + assert probe.driver_loaded(RuntimeError(msg)) is True + + +@pytest.mark.parametrize("msg", _LOAD_FAILURE_MESSAGES) +def test_driver_loaded_false_for_load_failures(msg): + probe = _load_probe() + assert probe.driver_loaded(RuntimeError(msg)) is False + + +def test_driver_loaded_true_for_clean_connect(): + probe = _load_probe() + assert probe.driver_loaded(None) is True + + +def _run_main_with_stub(monkeypatch, connect): + """Run ``probe.main()`` with a stubbed ``mssql_python`` module.""" + probe = _load_probe() + stub = types.ModuleType("mssql_python") + stub.connect = connect + monkeypatch.setitem(sys.modules, "mssql_python", stub) + return probe + + +def test_main_exits_nonzero_on_simulated_load_failure(monkeypatch): + def connect(_conn_str): + raise RuntimeError( + "Failed to load the driver. Please read the documentation to install the " + "required dependencies." + ) + + probe = _run_main_with_stub(monkeypatch, connect) + with pytest.raises(SystemExit) as excinfo: + probe.main() + # sys.exit() -> non-zero (truthy) exit code carrying the reason. + assert excinfo.value.code + assert "DRIVER DID NOT LOAD" in str(excinfo.value.code) + + +def test_main_passes_on_simulated_network_failure(monkeypatch): + def connect(_conn_str): + raise RuntimeError( + "[Microsoft][ODBC Driver 18 for SQL Server]TCP Provider: No connection could be " + "made because the target machine actively refused it." + ) + + probe = _run_main_with_stub(monkeypatch, connect) + # A genuine connection-stage failure must NOT raise SystemExit (exit 0). + probe.main() + + +def test_main_passes_on_clean_connect(monkeypatch): + closed = {"value": False} + + class _Conn: + def close(self): + closed["value"] = True + + def connect(_conn_str): + return _Conn() + + probe = _run_main_with_stub(monkeypatch, connect) + probe.main() + assert closed["value"] is True diff --git a/tests/test_027_conda_release_metadata.py b/tests/test_027_conda_release_metadata.py new file mode 100644 index 000000000..406a871f9 --- /dev/null +++ b/tests/test_027_conda_release_metadata.py @@ -0,0 +1,221 @@ +"""Unit tests for the metadata-based conda release gate. + +``conda/validate_conda_release.py`` reads each package's authoritative +``info/index.json`` and enforces: real-subdir == folder, allowed subdirs, the +full (subdir x Python) matrix, and exact versions for the self-contained +``mssql-python`` package (which vendors the ODBC payload -- no companion). These +tests exercise the pure ``validate()`` logic with synthetic package records (no +real ``.conda`` needed) plus one optional round-trip through the metadata reader. +""" + +import importlib.util +import io +import json +import tarfile +from pathlib import Path + +import pytest + +_MODULE_PATH = Path(__file__).resolve().parent.parent / "conda" / "validate_conda_release.py" + +# The conda/ sources are not shipped inside the built wheel, so the installed-wheel +# test leg copies only tests/ into an isolated dir. Skip the whole module (rather than +# erroring at collection) when the conda source it exercises is absent. +if not _MODULE_PATH.is_file(): + pytest.skip( + f"conda source not present ({_MODULE_PATH}); skipping conda release metadata tests", + allow_module_level=True, + ) + + +def _load_module(): + spec = importlib.util.spec_from_file_location("validate_conda_release_under_test", _MODULE_PATH) + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +vcr = _load_module() + +_REQUIRED = ["win-64", "osx-64", "osx-arm64", "linux-64", "linux-aarch64"] +_ALLOWED = ["win-64", "win-arm64", "osx-64", "osx-arm64", "linux-64", "linux-aarch64"] +_PYTHONS = ["3.10", "3.11", "3.12", "3.13", "3.14"] +_MP_VER = "1.13.0" + + +def _binding(subdir, py, folder=None, version=_MP_VER): + return { + "folder": folder or subdir, + "subdir": subdir, + "name": "mssql-python", + "version": version, + "build": f"py{py.replace('.', '')}_0", + "python": py, + } + + +def _healthy_set(): + """A complete release: the self-contained mssql-python package for every + (required subdir x Python).""" + pkgs = [] + for sub in _REQUIRED: + for py in _PYTHONS: + pkgs.append(_binding(sub, py)) + return pkgs + + +def _run(pkgs, expected_versions=None): + return vcr.validate( + pkgs, + required_subdirs=_REQUIRED, + allowed_subdirs=_ALLOWED, + expected_pythons=_PYTHONS, + expected_versions=( + expected_versions if expected_versions is not None else {"mssql-python": _MP_VER} + ), + ) + + +def test_healthy_set_passes(): + assert _run(_healthy_set()) == [] + + +def test_mislabeled_subdir_fails(): + pkgs = _healthy_set() + # An osx-64 package physically staged into the osx-arm64 folder. + pkgs.append(_binding("osx-64", "3.12", folder="osx-arm64")) + errors = _run(pkgs) + assert any("MISLABELED" in e for e in errors) + + +def test_missing_python_variant_on_win64_fails(): + # This is the exact 8e7f217f regression: drop a win-64 binding; presence-pairing + # against the single companion used to pass, metadata matrix must now fail. + pkgs = [p for p in _healthy_set() if not (p["subdir"] == "win-64" and p["python"] == "3.12")] + errors = _run(pkgs) + assert any("win-64" in e and "INCOMPLETE" in e and "3.12" in e for e in errors) + + +def test_stray_companion_package_fails(): + # The self-contained model ships ONLY mssql-python; a stray companion package + # (the old separate mssql-python-odbc) must now be rejected as unexpected. + pkgs = _healthy_set() + pkgs.append( + { + "folder": "linux-64", + "subdir": "linux-64", + "name": "mssql-python-odbc", + "version": "18.6.2.1", + "build": "0", + "python": "", + } + ) + errors = _run(pkgs) + assert any("unexpected package name" in e and "mssql-python-odbc" in e for e in errors) + + +def test_unexpected_subdir_fails(): + pkgs = _healthy_set() + pkgs.append(_binding("linux-ppc64le", "3.12")) + errors = _run(pkgs) + assert any("linux-ppc64le" in e and "allowed" in e for e in errors) + + +def test_version_mismatch_fails(): + pkgs = _healthy_set() + pkgs.append(_binding("linux-64", "3.14", version="9.9.9")) # stray wrong-version binding + # remove the correct 3.14 to avoid duplicate-python noise masking the version check + pkgs = [ + p + for p in pkgs + if not (p["subdir"] == "linux-64" and p["python"] == "3.14" and p["version"] == _MP_VER) + ] + errors = _run(pkgs) + assert any("version" in e.lower() for e in errors) + + +def test_multiple_versions_same_package_fails(): + pkgs = _healthy_set() + pkgs.append(_binding("linux-64", "3.10", version="1.12.0", folder="linux-64")) + errors = _run(pkgs, expected_versions={}) # no expected -> consistency check must still fail + assert any("multiple versions" in e for e in errors) + + +def test_missing_required_subdir_fails(): + pkgs = [p for p in _healthy_set() if p["subdir"] != "linux-aarch64"] + errors = _run(pkgs) + assert any("linux-aarch64" in e and "MISSING" in e for e in errors) + + +def test_duplicate_package_fails(): + # The identical package staged twice (same name/version/subdir/python) -- e.g. a + # leg's package collected twice from a shared output dir. A set-based matrix + # check would silently absorb it; the gate must reject the duplicate outright so + # it can never mask a genuinely missing variant. + pkgs = _healthy_set() + pkgs.append(_binding("linux-64", "3.12")) # exact duplicate of an existing entry + errors = _run(pkgs) + assert any("DUPLICATE" in e and "linux-64" in e and "3.12" in e for e in errors) + + +def test_present_allowed_subdir_partial_matrix_fails(): + # win-arm64 is ALLOWED but not REQUIRED. If it shows up only partially built it + # must still fail the gate, else a half-finished allowed subdir slips to publish + # simply because it is not in the required set. + pkgs = _healthy_set() + pkgs.append(_binding("win-arm64", "3.10")) # only one of five Pythons + errors = _run(pkgs) + assert any("win-arm64" in e and "INCOMPLETE" in e for e in errors) + + +def test_python_tag_from_index(): + assert vcr.python_tag_from_index({"build": "py311_0"}) == "3.11" + assert vcr.python_tag_from_index({"build": "py310h1a2b3c_0"}) == "3.10" + assert ( + vcr.python_tag_from_index({"build": "0", "depends": ["python 3.12.* *_cpython"]}) == "3.12" + ) + assert vcr.python_tag_from_index({"build": "0"}) == "" + + +def _zstd_available(): + try: + from compression import zstd # noqa: F401 # py3.14+ + + return True + except Exception: + try: + import zstandard # noqa: F401 + + return True + except Exception: + return False + + +@pytest.mark.skipif(not _zstd_available(), reason="no zstandard backend available") +def test_read_index_json_roundtrip(tmp_path): + import zipfile + + index = {"name": "mssql-python", "version": _MP_VER, "build": "py312_0", "subdir": "win-64"} + # Build info/index.json -> tar -> zstd -> .conda zip, then read it back. + tar_buf = io.BytesIO() + with tarfile.open(fileobj=tar_buf, mode="w") as tf: + data = json.dumps(index).encode() + ti = tarfile.TarInfo("info/index.json") + ti.size = len(data) + tf.addfile(ti, io.BytesIO(data)) + try: + from compression import zstd # py3.14+ + + compressed = zstd.compress(tar_buf.getvalue()) + except Exception: + import zstandard + + compressed = zstandard.ZstdCompressor().compress(tar_buf.getvalue()) + + conda_path = tmp_path / "mssql-python-1.13.0-py312_0.conda" + with zipfile.ZipFile(conda_path, "w") as zf: + zf.writestr("info-mssql-python-1.13.0-py312_0.tar.zst", compressed) + + got = vcr.read_index_json(str(conda_path)) + assert got["subdir"] == "win-64" + assert vcr.python_tag_from_index(got) == "3.12" diff --git a/tests/test_028_tls_connect_probe.py b/tests/test_028_tls_connect_probe.py new file mode 100644 index 000000000..c5f5ccd7d --- /dev/null +++ b/tests/test_028_tls_connect_probe.py @@ -0,0 +1,232 @@ +"""Fail-closed classification tests for ``conda/tls_connect_probe.py``. + +The live ``Encrypt=yes`` conda gate runs ``conda/tls_connect_probe.py`` to prove +the driver's dlopen'd OpenSSL backend (``libssl``/``libcrypto``) is REACHABLE -- +something the DB-less ``Encrypt=no`` ``driver_load_probe.py`` cannot show, because +the crypto libraries are only touched by a real TLS handshake. The classifier MUST +fail closed: only an outcome that provably means the mandatory pre-login TLS +handshake completed (a clean connect, a ``Login failed`` / 18456, or a +``Cannot open database``) may PASS; an OpenSSL-load failure or anything +unrecognized MUST fail. + +These are pure, no-DB unit tests: the probe's native ``import mssql_python`` is +deferred into ``main()``, so the classifier + ``force_tls`` can be exercised +without the compiled extension or a live SQL Server. +""" + +import importlib.util +from pathlib import Path + +import pytest + +_PROBE_PATH = Path(__file__).resolve().parent.parent / "conda" / "tls_connect_probe.py" + +# The conda/ sources are not shipped inside the built wheel, so the installed-wheel +# test leg copies only tests/ into an isolated dir. Skip the whole module (rather than +# erroring at collection/run) when the conda source it exercises is absent. +if not _PROBE_PATH.is_file(): + pytest.skip( + f"conda source not present ({_PROBE_PATH}); skipping conda TLS-probe tests", + allow_module_level=True, + ) + + +def _load_probe(): + """Import ``conda/tls_connect_probe.py`` as a standalone module.""" + spec = importlib.util.spec_from_file_location("tls_connect_probe_under_test", _PROBE_PATH) + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +# Outcomes that can only occur AFTER a mandatory Encrypt=yes handshake completes; +# every one MUST classify as "TLS completed" (PASS -> OpenSSL was reachable). +_TLS_COMPLETED_MESSAGES = [ + "[Microsoft][ODBC Driver 18 for SQL Server]Login failed for user 'x'.", + "Login failed for user 'sa'. (18456)", + '[Microsoft][ODBC Driver 18 for SQL Server]Cannot open database "X" requested by the ' + "login. The login failed.", + "Changed database context to 'master'.", +] + +# Outcomes that mean the crypto backend never loaded / the handshake never +# completed; every one MUST classify as "not completed" (FAIL / non-zero exit). +_TLS_FAILURE_MESSAGES = [ + "[Microsoft][ODBC Driver 18 for SQL Server]SSL Provider: The certificate chain was issued " + "by an authority that is not trusted.", + "libssl.so.3: cannot open shared object file: No such file or directory", + "libcrypto.so.3: cannot open shared object file: No such file or directory", + "[Microsoft][ODBC Driver 18 for SQL Server]TCP Provider: Error code 0x2726", + "[Microsoft][ODBC Driver 18 for SQL Server]Login timeout expired", + "dlopen(libssl.dylib): image not found", + # Fail-closed default: an unexpected / unrelated error is NOT proof of a + # completed handshake. + "some totally unexpected internal error", +] + + +@pytest.mark.parametrize("msg", _TLS_COMPLETED_MESSAGES) +def test_tls_completed_true_for_post_handshake_outcomes(msg): + probe = _load_probe() + assert probe.tls_completed(RuntimeError(msg)) is True + + +@pytest.mark.parametrize("msg", _TLS_FAILURE_MESSAGES) +def test_tls_completed_false_for_pre_handshake_failures(msg): + probe = _load_probe() + assert probe.tls_completed(RuntimeError(msg)) is False + + +def test_tls_completed_true_for_clean_connect(): + probe = _load_probe() + assert probe.tls_completed(None) is True + + +def test_force_tls_appends_when_absent(): + probe = _load_probe() + out = probe.force_tls("Server=dbserver;Database=x") + assert "Encrypt=yes" in out + assert "TrustServerCertificate=yes" in out + + +def test_force_tls_overrides_encrypt_no(): + probe = _load_probe() + out = probe.force_tls("Server=dbserver;Encrypt=no;Database=x") + low = out.lower() + assert "encrypt=yes" in low + assert "encrypt=no" not in low + + +def test_force_tls_is_idempotent(): + probe = _load_probe() + once = probe.force_tls("Server=dbserver;Database=x") + twice = probe.force_tls(once) + assert once == twice + # Exactly one Encrypt= and one TrustServerCertificate= key. + assert twice.lower().count("encrypt=") == 1 + assert twice.lower().count("trustservercertificate=") == 1 + + +# --- hardening: force_tls must never hand the parser a malformed string ---------- +# The mssql_python parser splits on top-level ';' and rejects any segment without an +# '=' ("keyword '' has no value") or a duplicated keyword. A regex substitution +# could produce exactly those; rebuilding from tokens must not. + + +def _every_segment_has_value(probe, conn): + """Mirror the parser's rule: each non-empty top-level ';'-segment needs '='.""" + return all("=" in seg.strip() for seg in probe._split_top_level(conn) if seg.strip()) + + +def _key_count(probe, conn, wanted): + total = 0 + for seg in probe._split_top_level(conn): + token = seg.strip() + if token and token.split("=", 1)[0].strip().lower() == wanted: + total += 1 + return total + + +def test_force_tls_dedups_duplicate_encrypt(): + """A duplicate Encrypt (old regex fixed only the first -> parser 'Duplicate keyword').""" + probe = _load_probe() + out = probe.force_tls("Server=dbserver;Encrypt=yes;Database=x;Encrypt=no") + assert _key_count(probe, out, "encrypt") == 1 + assert "encrypt=no" not in out.lower() + assert _every_segment_has_value(probe, out) + + +def test_force_tls_handles_valueless_encrypt(): + """A bare 'Encrypt' (no '=value') must not leave a value-less keyword behind.""" + probe = _load_probe() + out = probe.force_tls("Server=dbserver;Encrypt;Database=x") + assert _key_count(probe, out, "encrypt") == 1 + assert _every_segment_has_value(probe, out) + + +def test_force_tls_preserves_braced_value_with_semicolon(): + """An ODBC braced value may contain ';'; it must survive intact (MS-ODBCSTR).""" + probe = _load_probe() + out = probe.force_tls("Server=dbserver;Pwd={a;b};Encrypt=no") + assert "Pwd={a;b}" in out + assert _key_count(probe, out, "encrypt") == 1 + assert "encrypt=no" not in out.lower() + assert _every_segment_has_value(probe, out) + + +@pytest.mark.parametrize( + "raw", + [ + "Server=dbserver;Database=master", + "Server=dbserver", + "server=dbserver;encrypt=no;trustservercertificate=no", + "Server = dbserver ; Encrypt = no ; TrustServerCertificate = no", + "Server=dbserver;Encrypt=Strict", + "Encrypt=yes;Server=dbserver", + "Server=tcp:dbserver,1433;Database=master", + ], +) +def test_force_tls_output_is_parseable(raw): + """Every realistic input yields a string whose every segment has a value and + carries exactly one Encrypt=yes / TrustServerCertificate=yes.""" + probe = _load_probe() + out = probe.force_tls(raw) + assert _every_segment_has_value(probe, out) + assert _key_count(probe, out, "encrypt") == 1 + assert _key_count(probe, out, "trustservercertificate") == 1 + assert "encrypt=yes" in out.lower() + assert "trustservercertificate=yes" in out.lower() + + +def test_split_top_level_respects_braces(): + probe = _load_probe() + assert probe._split_top_level("Server=x;Pwd={a;b};Encrypt=no") == [ + "Server=x", + "Pwd={a;b}", + "Encrypt=no", + ] + + +def test_redact_masks_values_and_flags_bare_segments(): + """The debug line must never leak a value and must surface a no-value segment.""" + probe = _load_probe() + red = probe._redact("Server=dbserver;Pwd=REDACTME;Encrypt=yes") + assert "REDACTME" not in red + assert "Pwd=***" in red + assert "Server=***" in red + # a segment with no '=' (the shape that trips the parser) is surfaced verbatim. + assert "<=3,<4"] + + +def _make_pkg( + tmp_path, + runpath=_GOOD_RUNPATH, + rpath=None, + subdir="linux-64", + vendored=None, + depends=None, + driver_needed=None, + inst_needed=None, +): + """Write a minimal .tar.bz2 conda package with two ELF driver binaries.""" + p = tmp_path / "mssql-python-1.13.0-py312_0.tar.bz2" + with tarfile.open(p, "w:bz2") as tf: + + def add(name, data): + ti = tarfile.TarInfo(name) + ti.size = len(data) + tf.addfile(ti, io.BytesIO(data)) + + add( + "info/index.json", + json.dumps( + { + "name": "mssql-python", + "version": "1.13.0", + "build": "py312_0", + "subdir": subdir, + "depends": _GOOD_DEPENDS if depends is None else depends, + } + ).encode(), + ) + add( + f"{_LIBDIR}/libmsodbcsql-18.6.so.2.1", + _make_elf64( + runpath=runpath, + rpath=rpath, + needed=_DRIVER_NEEDED if driver_needed is None else driver_needed, + ), + ) + add( + f"{_LIBDIR}/libodbcinst.so.2", + _make_elf64( + runpath=runpath, + rpath=rpath, + needed=_INST_NEEDED if inst_needed is None else inst_needed, + ), + ) + if vendored: + add(f"{_LIBDIR}/{vendored}", b"\x7fELF fake-vendored") + return str(p) + + +# --- low-level parser ------------------------------------------------------- + + +def test_elf_dynamic_pt_parse(): + data = _make_elf64(runpath=_GOOD_RUNPATH, needed=["libkrb5.so.3", "libodbcinst.so.2"]) + dyn = audit.elf_dynamic(data) + assert dyn["runpath"] == _GOOD_RUNPATH + assert dyn["rpath"] is None + assert "libkrb5.so.3" in dyn["needed"] and "libodbcinst.so.2" in dyn["needed"] + + +def test_effective_runpath_prefers_runpath_over_rpath(): + # DT_RUNPATH present -> loader ignores DT_RPATH. + dyn = audit.elf_dynamic(_make_elf64(runpath="$ORIGIN", rpath=_GOOD_RUNPATH)) + assert audit.effective_runpath(dyn) == "$ORIGIN" + # Only DT_RPATH present -> that is the effective one. + dyn2 = audit.elf_dynamic(_make_elf64(rpath=_GOOD_RUNPATH)) + assert audit.effective_runpath(dyn2) == _GOOD_RUNPATH + + +def test_expected_climb_entry_is_exact(): + member = f"{_LIBDIR}/libmsodbcsql-18.6.so.2.1" + assert audit.expected_climb_entry(member) == _CLIMB_ENTRY + + +# --- audit_package: the happy path ----------------------------------------- + + +def test_audit_passes_with_exact_climb(tmp_path): + assert audit.audit_package(_make_pkg(tmp_path)) == [] + + +# --- N1: wrong climb variants must all FAIL -------------------------------- + + +def test_audit_fails_wrong_depth_too_short(tmp_path): + errors = audit.audit_package(_make_pkg(tmp_path, runpath="$ORIGIN:$ORIGIN/..")) + assert any("exact climb entry" in e for e in errors) + + +def test_audit_fails_overshoot(tmp_path): + over = "$ORIGIN:$ORIGIN/../../../../../../../../.." # one level too many + errors = audit.audit_package(_make_pkg(tmp_path, runpath=over)) + assert any("exact climb entry" in e for e in errors) + + +def test_audit_fails_decoy_rpath_behind_bad_runpath(tmp_path): + # Good climb hidden in DT_RPATH, but DT_RUNPATH (which the loader uses) is bare. + errors = audit.audit_package(_make_pkg(tmp_path, runpath="$ORIGIN", rpath=_GOOD_RUNPATH)) + assert any("exact climb entry" in e for e in errors) + + +def test_audit_fails_malformed_originator(tmp_path): + bad = "$ORIGINATOR/../../../../../../../.." # startswith('$ORIGIN') but wrong token + errors = audit.audit_package(_make_pkg(tmp_path, runpath=bad)) + assert any("exact climb entry" in e for e in errors) + + +def test_audit_fails_on_absolute_rpath(tmp_path): + errors = audit.audit_package(_make_pkg(tmp_path, runpath="$ORIGIN:/opt/lib")) + assert any("ABSOLUTE" in e for e in errors) + + +def test_audit_fails_missing_bare_origin(tmp_path): + # Climb entry present but bare $ORIGIN dropped -> the driver can no longer resolve + # its co-located sibling libodbcinst.so.2 even though $PREFIX/lib is reachable. + errors = audit.audit_package(_make_pkg(tmp_path, runpath=_CLIMB_ENTRY)) + assert any("bare '$ORIGIN'" in e for e in errors) + + +# --- N2: declared deps + DT_NEEDED ----------------------------------------- + + +def test_audit_fails_missing_declared_krb5(tmp_path): + depends = ["python", "azure-identity", "libtool", "openssl >=3,<4"] # no krb5 + errors = audit.audit_package(_make_pkg(tmp_path, depends=depends)) + assert any("missing 'krb5'" in e for e in errors) + + +def test_audit_fails_missing_declared_libtool(tmp_path): + depends = ["python", "azure-identity", "krb5", "openssl >=3,<4"] # no libtool + errors = audit.audit_package(_make_pkg(tmp_path, depends=depends)) + assert any("missing 'libtool'" in e for e in errors) + + +def test_audit_fails_missing_declared_openssl(tmp_path): + depends = ["python", "azure-identity", "krb5", "libtool"] # no openssl + errors = audit.audit_package(_make_pkg(tmp_path, depends=depends)) + assert any("missing 'openssl'" in e for e in errors) + + +def test_audit_fails_openssl_not_range_pinned(tmp_path): + # openssl present but not pinned to the Driver-18 ABI range (>=3,<4). + depends = ["python", "azure-identity", "krb5", "libtool", "openssl"] + errors = audit.audit_package(_make_pkg(tmp_path, depends=depends)) + assert any("range-pinned" in e for e in errors) + + +def test_audit_fails_driver_missing_from_one_subdir(tmp_path): + # debian_ubuntu is complete, but rhel ships only libodbcinst (driver dropped). A + # package-global count would pass since debian_ubuntu supplies a driver; per-subdir + # presence must catch the rhel gap. + rhel_lib = "lib/python3.12/site-packages/mssql_python_odbc/libs/linux/rhel/x86_64/lib" + p = tmp_path / "mssql-python-1.13.0-py312_0.tar.bz2" + with tarfile.open(p, "w:bz2") as tf: + + def add(name, data): + ti = tarfile.TarInfo(name) + ti.size = len(data) + tf.addfile(ti, io.BytesIO(data)) + + add( + "info/index.json", + json.dumps( + { + "name": "mssql-python", + "version": "1.13.0", + "build": "py312_0", + "subdir": "linux-64", + "depends": _GOOD_DEPENDS, + } + ).encode(), + ) + add( + f"{_LIBDIR}/libmsodbcsql-18.6.so.2.1", _make_elf64(_GOOD_RUNPATH, needed=_DRIVER_NEEDED) + ) + add(f"{_LIBDIR}/libodbcinst.so.2", _make_elf64(_GOOD_RUNPATH, needed=_INST_NEEDED)) + # rhel: libodbcinst only -- the driver is missing from this subdir. + add(f"{rhel_lib}/libodbcinst.so.2", _make_elf64(_GOOD_RUNPATH, needed=_INST_NEEDED)) + errors = audit.audit_package(str(p)) + assert any("rhel" in e and "libmsodbcsql" in e for e in errors) + + +def test_audit_fails_driver_lost_needed(tmp_path): + # Driver stopped NEEDing libgssapi_krb5 -> declared krb5 dep is now moot. + errors = audit.audit_package( + _make_pkg(tmp_path, driver_needed=["libkrb5.so.3", "libodbcinst.so.2"]) + ) + assert any("libgssapi_krb5" in e and "no longer NEED" in e for e in errors) + + +def test_audit_fails_odbcinst_lost_libltdl(tmp_path): + errors = audit.audit_package(_make_pkg(tmp_path, inst_needed=["libc.so.6"])) + assert any("libltdl" in e and "no longer NEED" in e for e in errors) + + +def test_audit_allows_musl_variant_without_libltdl(tmp_path): + # The alpine/musl libodbcinst NEEDs libc.musl* and statically links ltdl, so the + # glibc libltdl DT_NEEDED requirement must NOT fail it. Package has a complete glibc + # debian_ubuntu variant plus an alpine/musl variant. + alpine_lib = "lib/python3.12/site-packages/mssql_python_odbc/libs/linux/alpine/x86_64/lib" + p = tmp_path / "mssql-python-1.13.0-py312_0.tar.bz2" + with tarfile.open(p, "w:bz2") as tf: + + def add(name, data): + ti = tarfile.TarInfo(name) + ti.size = len(data) + tf.addfile(ti, io.BytesIO(data)) + + add( + "info/index.json", + json.dumps( + { + "name": "mssql-python", + "version": "1.13.0", + "build": "py312_0", + "subdir": "linux-64", + "depends": _GOOD_DEPENDS, + } + ).encode(), + ) + # glibc debian_ubuntu (complete: NEEDs libltdl/krb5). + add( + f"{_LIBDIR}/libmsodbcsql-18.6.so.2.1", _make_elf64(_GOOD_RUNPATH, needed=_DRIVER_NEEDED) + ) + add(f"{_LIBDIR}/libodbcinst.so.2", _make_elf64(_GOOD_RUNPATH, needed=_INST_NEEDED)) + # alpine/musl: driver + inst link libc.musl and do NOT NEED libltdl. + add( + f"{alpine_lib}/libmsodbcsql-18.6.so.2.1", + _make_elf64( + _GOOD_RUNPATH, + needed=[ + "libodbcinst.so.2", + "libkrb5.so.3", + "libgssapi_krb5.so.2", + "libc.musl-x86_64.so.1", + ], + ), + ) + add( + f"{alpine_lib}/libodbcinst.so.2", + _make_elf64(_GOOD_RUNPATH, needed=["libc.musl-x86_64.so.1"]), + ) + assert audit.audit_package(str(p)) == [] + + +# --- vendoring + malformed + non-linux ------------------------------------- + + +def test_audit_fails_on_vendored_crypto(tmp_path): + errors = audit.audit_package(_make_pkg(tmp_path, vendored="libkrb5.so.3")) + assert any("vendors" in e for e in errors) + + +def test_audit_fails_malformed_package(tmp_path): + # A .tar.bz2 with no info/index.json must FAIL (never silently skipped as non-Linux). + p = tmp_path / "broken-1.0-0.tar.bz2" + with tarfile.open(p, "w:bz2") as tf: + data = b"not an index" + ti = tarfile.TarInfo("some/other/file") + ti.size = len(data) + tf.addfile(ti, io.BytesIO(data)) + errors = audit.audit_package(str(p)) + assert any("unreadable/malformed" in e for e in errors) + + +def test_audit_skips_non_linux(tmp_path): + assert audit.audit_package(_make_pkg(tmp_path, subdir="win-64")) == []