diff --git a/eng/pipelines/templates/stages/1es-redirect.yml b/eng/pipelines/templates/stages/1es-redirect.yml index 00911bc510b8..367ca4cec87f 100644 --- a/eng/pipelines/templates/stages/1es-redirect.yml +++ b/eng/pipelines/templates/stages/1es-redirect.yml @@ -19,6 +19,10 @@ parameters: - name: oneESTemplateTag type: string default: release +- name: EnableCompiledCodeql + type: boolean + default: false + extends: ${{ if and(parameters.Use1ESOfficial, eq(parameters.oneESTemplateTag, 'canary')) }}: @@ -55,9 +59,16 @@ extends: enabled: false justificationForDisabling: "ESLint injected task has failures because it uses an old version of mkdirp. We should not fail for tools not controlled by the repo. See: https://dev.azure.com/azur 19 e-sdk/internal/_build/results?buildId=3556850" codeql: - compiled: - enabled: false - justificationForDisabling: "To reduce redundant CG runs across all our pipeline jobs we are disabling and only running in our main build job." + ${{ if eq(parameters.EnableCompiledCodeql, true) }}: + # "cpp" covers both C and C++ code. Language is specified because + # checkout happens after the injected "CodeQL Initialize" step + language: cpp + compiled: + enabled: true + ${{ else }}: + compiled: + enabled: false + justificationForDisabling: "To reduce redundant CG runs across all our pipeline jobs we are disabling and only running in our main build job." componentgovernance: enabled: false justificationForDisabling: "To reduce redundant CG runs across all our pipeline jobs we are disabling and only running in our main build job." diff --git a/eng/pipelines/templates/stages/archetype-sdk-client.yml b/eng/pipelines/templates/stages/archetype-sdk-client.yml index 53efd1ab54b0..109f9e6bd012 100644 --- a/eng/pipelines/templates/stages/archetype-sdk-client.yml +++ b/eng/pipelines/templates/stages/archetype-sdk-client.yml @@ -81,6 +81,9 @@ parameters: - name: oneESTemplateTag type: string default: release + - name: EnableCompiledCodeql + type: boolean + default: false - name: EnvVars type: object default: {} @@ -89,6 +92,7 @@ extends: template: /eng/pipelines/templates/stages/1es-redirect.yml parameters: oneESTemplateTag: ${{ parameters.oneESTemplateTag }} + EnableCompiledCodeql: ${{ parameters.EnableCompiledCodeql }} stages: - ${{ if and(eq(parameters.SkipPrValidation, true), eq(variables['Build.Reason'], 'Manual')) }}: - stage: NoOp diff --git a/eng/tools/azure-sdk-tools/ci_tools/build.py b/eng/tools/azure-sdk-tools/ci_tools/build.py index 26ac6fcfd15a..9aea4d49258c 100644 --- a/eng/tools/azure-sdk-tools/ci_tools/build.py +++ b/eng/tools/azure-sdk-tools/ci_tools/build.py @@ -221,8 +221,16 @@ def create_package( setup_directory_or_file: str, dest_folder: str, enable_wheel: bool = True, enable_sdist: bool = True ): """ - Uses the invoking python executable to build a wheel and sdist file given a setup.py or setup.py directory. Outputs - into a distribution directory and defaults to the value of get_artifact_directory(). + Builds a wheel and/or sdist file given a setup.py, pyproject.toml, or directory containing either. + + For packages with compiled extensions (ext_modules): + - setup.py: uses cibuildwheel to build platform-specific wheels + - pyproject.toml: uses cibuildwheel to build platform-specific wheels (respects [tool.cibuildwheel] config) + + For pure Python packages: + - Uses python -m build + + Outputs into a distribution directory and defaults to get_artifact_directory(). """ dist = get_artifact_directory(dest_folder) @@ -231,36 +239,54 @@ def create_package( should_log_build_output = logger.getEffectiveLevel() <= logging.DEBUG if setup_parsed.is_pyproject: - # when building with pyproject, we will use `python -m build` to build the package - # -n argument will not use an isolated environment, which means the current environment must have all the dependencies of the package installed, to successfully - # pull in the dynamic `__version__` attribute. This is because setuptools is actually walking the __init__.py to get that attribute, which will fail - # if the imports within the setup.py don't work. Perhaps an isolated environment is better, pulling all the "dependencies" into the [build-system].requires list - - # given the additional requirements of the package, we should install them in the current environment before attempting to build the package - # we assume the presence of `wheel`, `build`, `setuptools>=61.0.0` - pip_output = get_pip_list_output(sys.executable) - necessary_install_requirements = [ - req for req in setup_parsed.requires if parse_require(req).name not in pip_output.keys() - ] - run_logged( - [sys.executable, "-m", "pip", "install", *necessary_install_requirements], - cwd=setup_parsed.folder, - check=False, - should_stream_to_console=should_log_build_output, - ) - run_logged( - [ - sys.executable, - "-m", - "build", - f"-n{'s' if enable_sdist else ''}{'w' if enable_wheel else ''}", - "-o", - dist, - ], - cwd=setup_parsed.folder, - check=True, - should_stream_to_console=should_log_build_output, - ) + # when building with pyproject, check if package has compiled extensions + if enable_wheel and setup_parsed.ext_modules: + # Use cibuildwheel for compiled extensions (respects [tool.cibuildwheel] config) + run_logged( + [sys.executable, "-vv", "-m", "cibuildwheel", "--output-dir", dist], + cwd=setup_parsed.folder, + check=True, + should_stream_to_console=should_log_build_output, + ) + if enable_sdist: + # Build sdist separately with python -m build + run_logged( + [sys.executable, "-m", "build", "-s", "-o", dist], + cwd=setup_parsed.folder, + check=True, + should_stream_to_console=should_log_build_output, + ) + else: + # Use python -m build for pure Python packages + # -n argument will not use an isolated environment, which means the current environment must have all the dependencies of the package installed, to successfully + # pull in the dynamic `__version__` attribute. This is because setuptools is actually walking the __init__.py to get that attribute, which will fail + # if the imports within the setup.py don't work. Perhaps an isolated environment is better, pulling all the "dependencies" into the [build-system].requires list + + # given the additional requirements of the package, we should install them in the current environment before attempting to build the package + # we assume the presence of `wheel`, `build`, `setuptools>=61.0.0` + pip_output = get_pip_list_output(sys.executable) + necessary_install_requirements = [ + req for req in setup_parsed.requires if parse_require(req).name not in pip_output.keys() + ] + run_logged( + [sys.executable, "-m", "pip", "install", *necessary_install_requirements], + cwd=setup_parsed.folder, + check=False, + should_stream_to_console=should_log_build_output, + ) + run_logged( + [ + sys.executable, + "-m", + "build", + f"-n{'s' if enable_sdist else ''}{'w' if enable_wheel else ''}", + "-o", + dist, + ], + cwd=setup_parsed.folder, + check=True, + should_stream_to_console=should_log_build_output, + ) else: if enable_wheel: if setup_parsed.ext_modules: diff --git a/eng/tools/azure-sdk-tools/ci_tools/parsing/parse_functions.py b/eng/tools/azure-sdk-tools/ci_tools/parsing/parse_functions.py index a8e112434323..2830ee15ae70 100644 --- a/eng/tools/azure-sdk-tools/ci_tools/parsing/parse_functions.py +++ b/eng/tools/azure-sdk-tools/ci_tools/parsing/parse_functions.py @@ -703,6 +703,25 @@ def parse_pyproject( ext_modules = get_value_from_dict(toml_dict, "tool.setuptools.ext-modules", []) ext_modules = [Extension(**moduleArgDict) for moduleArgDict in ext_modules] + # Declaring ext-modules in [tool.setuptools.ext-modules] is still experimental in setuptools, and the + # abi3 / py_limited_api wheel tag cannot be expressed declaratively, so compiled-extension packages keep + # their Extension(...) definition in setup.py. When such a package also has a [project] table, pyproject + # wins over setup.py during parsing and the extension would otherwise go undetected, causing the build to + # be mis-routed to `python -m build` (pure-Python) instead of cibuildwheel. Fall back to setup.py here so + # the extension is still discovered. Guarded so a setup.py that cannot be parsed cannot regress packages + # that parse fine today. + if not ext_modules: + sibling_setup_py = os.path.join(package_directory, "setup.py") + if os.path.exists(sibling_setup_py): + try: + setup_py_result = parse_setup_py(sibling_setup_py) + ext_package = ext_package or setup_py_result[11] # ext_package + ext_modules = setup_py_result[12] # ext_modules + except Exception as e: # pragma: no cover - defensive, preserves prior behavior + logging.warning( + f"Found setup.py alongside {pyproject_filename} but could not parse it for ext_modules: {e}" + ) + # fmt: off return ( name, # str diff --git a/sdk/storage/ci.yml b/sdk/storage/ci.yml index 390eab91bb75..d35537c2ab07 100644 --- a/sdk/storage/ci.yml +++ b/sdk/storage/ci.yml @@ -39,6 +39,9 @@ extends: ServiceDirectory: storage TestProxy: true TestTimeoutInMinutes: 120 + # Enable Compiled CodeQL becuase azure-storage-extensions has C code that + # must be scanned + EnableCompiledCodeql: true Artifacts: - name: azure-storage-blob safeName: azurestorageblob