Add CodeQL native C build scanning to storage pipeline

eng/pipelines/templates/stages/1es-redirect.yml

@@ -19,6 +19,10 @@ parameters:
 - name: oneESTemplateTag
   type: string
   default: release
+- name: EnableCompiledCodeql
+  type: boolean
+  default: false
+
 
 extends:
   ${{ if and(parameters.Use1ESOfficial, eq(parameters.oneESTemplateTag, 'canary')) }}:
@@ -55,9 +59,16 @@ extends:
         enabled: false
         justificationForDisabling: "ESLint injected task has failures because it uses an old version of mkdirp. We should not fail for tools not controlled by the repo. See: https://dev.azure.com/azur     19 e-sdk/internal/_build/results?buildId=3556850"
       codeql:
-        compiled:
-          enabled: false
-          justificationForDisabling: "To reduce redundant CG runs across all our pipeline jobs we are disabling and only running in our main build job."
+        ${{ if eq(parameters.EnableCompiledCodeql, true) }}:
+          # "cpp" covers both C and C++ code. Language is specified because 
+          # checkout happens after the injected "CodeQL Initialize" step
+          language: cpp
+          compiled:
+            enabled: true
+        ${{ else }}:
+          compiled: 
+            enabled: false
+            justificationForDisabling: "To reduce redundant CG runs across all our pipeline jobs we are disabling and only running in our main build job."
       componentgovernance:
         enabled: false
         justificationForDisabling: "To reduce redundant CG runs across all our pipeline jobs we are disabling and only running in our main build job."

eng/pipelines/templates/stages/archetype-sdk-client.yml

@@ -81,6 +81,9 @@ parameters:
   - name: oneESTemplateTag
     type: string
     default: release
+  - name: EnableCompiledCodeql
+    type: boolean
+    default: false
   - name: EnvVars
     type: object
     default: {}
@@ -89,6 +92,7 @@ extends:
   template: /eng/pipelines/templates/stages/1es-redirect.yml
   parameters:
     oneESTemplateTag: ${{ parameters.oneESTemplateTag }}
+    EnableCompiledCodeql: ${{ parameters.EnableCompiledCodeql }}
     stages:
       - ${{ if and(eq(parameters.SkipPrValidation, true), eq(variables['Build.Reason'], 'Manual')) }}:
         - stage: NoOp

eng/tools/azure-sdk-tools/ci_tools/build.py

@@ -221,8 +221,16 @@ def create_package(
     setup_directory_or_file: str, dest_folder: str, enable_wheel: bool = True, enable_sdist: bool = True
 ):
     """
-    Uses the invoking python executable to build a wheel and sdist file given a setup.py or setup.py directory. Outputs
-    into a distribution directory and defaults to the value of get_artifact_directory().
+    Builds a wheel and/or sdist file given a setup.py, pyproject.toml, or directory containing either.
+
+    For packages with compiled extensions (ext_modules):
+    - setup.py: uses cibuildwheel to build platform-specific wheels
+    - pyproject.toml: uses cibuildwheel to build platform-specific wheels (respects [tool.cibuildwheel] config)
+
+    For pure Python packages:
+    - Uses python -m build
+
+    Outputs into a distribution directory and defaults to get_artifact_directory().
     """
 
     dist = get_artifact_directory(dest_folder)
@@ -231,36 +239,54 @@ def create_package(
     should_log_build_output = logger.getEffectiveLevel() <= logging.DEBUG
 
     if setup_parsed.is_pyproject:
-        # when building with pyproject, we will use `python -m build` to build the package
-        # -n argument will not use an isolated environment, which means the current environment must have all the dependencies of the package installed, to successfully
-        # pull in the dynamic `__version__` attribute. This is because setuptools is actually walking the __init__.py to get that attribute, which will fail
-        # if the imports within the setup.py don't work. Perhaps an isolated environment is better, pulling all the "dependencies" into the [build-system].requires list
-
-        # given the additional requirements of the package, we should install them in the current environment before attempting to build the package
-        # we assume the presence of `wheel`, `build`, `setuptools>=61.0.0`
-        pip_output = get_pip_list_output(sys.executable)
-        necessary_install_requirements = [
-            req for req in setup_parsed.requires if parse_require(req).name not in pip_output.keys()
-        ]
-        run_logged(
-            [sys.executable, "-m", "pip", "install", *necessary_install_requirements],
-            cwd=setup_parsed.folder,
-            check=False,
-            should_stream_to_console=should_log_build_output,
-        )
-        run_logged(
-            [
-                sys.executable,
-                "-m",
-                "build",
-                f"-n{'s' if enable_sdist else ''}{'w' if enable_wheel else ''}",
-                "-o",
-                dist,
-            ],
-            cwd=setup_parsed.folder,
-            check=True,
-            should_stream_to_console=should_log_build_output,
-        )
+        # when building with pyproject, check if package has compiled extensions
+        if enable_wheel and setup_parsed.ext_modules:
+            # Use cibuildwheel for compiled extensions (respects [tool.cibuildwheel] config)
+            run_logged(
+                [sys.executable, "-vv", "-m", "cibuildwheel", "--output-dir", dist],
+                cwd=setup_parsed.folder,
+                check=True,
+                should_stream_to_console=should_log_build_output,
+            )
+            if enable_sdist:
+                # Build sdist separately with python -m build
+                run_logged(
+                    [sys.executable, "-m", "build", "-s", "-o", dist],
+                    cwd=setup_parsed.folder,
+                    check=True,
+                    should_stream_to_console=should_log_build_output,
+                )
+        else:
+            # Use python -m build for pure Python packages
+            # -n argument will not use an isolated environment, which means the current environment must have all the dependencies of the package installed, to successfully
+            # pull in the dynamic `__version__` attribute. This is because setuptools is actually walking the __init__.py to get that attribute, which will fail
+            # if the imports within the setup.py don't work. Perhaps an isolated environment is better, pulling all the "dependencies" into the [build-system].requires list
+
+            # given the additional requirements of the package, we should install them in the current environment before attempting to build the package
+            # we assume the presence of `wheel`, `build`, `setuptools>=61.0.0`
+            pip_output = get_pip_list_output(sys.executable)
+            necessary_install_requirements = [
+                req for req in setup_parsed.requires if parse_require(req).name not in pip_output.keys()
+            ]
+            run_logged(
+                [sys.executable, "-m", "pip", "install", *necessary_install_requirements],
+                cwd=setup_parsed.folder,
+                check=False,
+                should_stream_to_console=should_log_build_output,
+            )
+            run_logged(
+                [
+                    sys.executable,
+                    "-m",
+                    "build",
+                    f"-n{'s' if enable_sdist else ''}{'w' if enable_wheel else ''}",
+                    "-o",
+                    dist,
+                ],
+                cwd=setup_parsed.folder,
+                check=True,
+                should_stream_to_console=should_log_build_output,
+            )
     else:
         if enable_wheel:
             if setup_parsed.ext_modules:

eng/tools/azure-sdk-tools/ci_tools/parsing/parse_functions.py

@@ -703,6 +703,25 @@ def parse_pyproject(
     ext_modules = get_value_from_dict(toml_dict, "tool.setuptools.ext-modules", [])
     ext_modules = [Extension(**moduleArgDict) for moduleArgDict in ext_modules]
 
+    # Declaring ext-modules in [tool.setuptools.ext-modules] is still experimental in setuptools, and the
+    # abi3 / py_limited_api wheel tag cannot be expressed declaratively, so compiled-extension packages keep
+    # their Extension(...) definition in setup.py. When such a package also has a [project] table, pyproject
+    # wins over setup.py during parsing and the extension would otherwise go undetected, causing the build to
+    # be mis-routed to `python -m build` (pure-Python) instead of cibuildwheel. Fall back to setup.py here so
+    # the extension is still discovered. Guarded so a setup.py that cannot be parsed cannot regress packages
+    # that parse fine today.
+    if not ext_modules:
+        sibling_setup_py = os.path.join(package_directory, "setup.py")
+        if os.path.exists(sibling_setup_py):
+            try:
+                setup_py_result = parse_setup_py(sibling_setup_py)
+                ext_package = ext_package or setup_py_result[11]  # ext_package
+                ext_modules = setup_py_result[12]  # ext_modules
+            except Exception as e:  # pragma: no cover - defensive, preserves prior behavior
+                logging.warning(
+                    f"Found setup.py alongside {pyproject_filename} but could not parse it for ext_modules: {e}"
+                )
+
     # fmt: off
     return (
         name,                   # str

sdk/storage/ci.yml

@@ -39,6 +39,9 @@ extends:
     ServiceDirectory: storage
     TestProxy: true
     TestTimeoutInMinutes: 120
+    # Enable Compiled CodeQL becuase azure-storage-extensions has C code that 
+    # must be scanned
+    EnableCompiledCodeql: true
     Artifacts:
     - name: azure-storage-blob
       safeName: azurestorageblob