From 0b674a85bc50f7460faf54f30f7b6ebb8a3772d3 Mon Sep 17 00:00:00 2001 From: subramaniak Date: Wed, 1 Jul 2026 07:00:31 +0000 Subject: [PATCH 1/6] feat: add DR-008 resolved-dependency resolve + override mechanism --- scripts/known_good/models/module.py | 3 + scripts/known_good/resolved_dependencies.py | 421 ++++++++++++++++++ .../tests/test_resolved_dependencies.py | 289 ++++++++++++ .../update_module_from_known_good.py | 142 +++--- 4 files changed, 795 insertions(+), 60 deletions(-) create mode 100644 scripts/known_good/resolved_dependencies.py create mode 100644 scripts/known_good/tests/test_resolved_dependencies.py diff --git a/scripts/known_good/models/module.py b/scripts/known_good/models/module.py index 72cae75c678..de01225a368 100644 --- a/scripts/known_good/models/module.py +++ b/scripts/known_good/models/module.py @@ -36,6 +36,7 @@ class Metadata: exclude_test_targets: list[str] = field(default_factory=lambda: []) langs: list[str] = field(default_factory=lambda: ["cpp", "rust"]) rust_coverage_config: str | None = "ferrocene-coverage" # Optional field for Rust coverage configuration + bazel_config: list[str] = field(default_factory=lambda: []) @classmethod def from_dict(cls, data: Dict[str, Any]) -> Metadata: @@ -53,6 +54,7 @@ def from_dict(cls, data: Dict[str, Any]) -> Metadata: exclude_test_targets=data.get("exclude_test_targets", []), langs=data.get("langs", ["cpp", "rust"]), rust_coverage_config=data.get("rust_coverage_config", "ferrocene-coverage"), + bazel_config=data.get("bazel_config", []), ) def to_dict(self) -> Dict[str, Any]: @@ -67,6 +69,7 @@ def to_dict(self) -> Dict[str, Any]: "exclude_test_targets": self.exclude_test_targets, "langs": self.langs, "rust_coverage_config": self.rust_coverage_config, + "bazel_config": self.bazel_config, } diff --git a/scripts/known_good/resolved_dependencies.py b/scripts/known_good/resolved_dependencies.py new file mode 100644 index 00000000000..6baebf85cdf --- /dev/null +++ b/scripts/known_good/resolved_dependencies.py @@ -0,0 +1,421 @@ +#!/usr/bin/env python3 +# ******************************************************************************* +# Copyright (c) 2026 Contributors to the Eclipse Foundation +# +# See the NOTICE file(s) distributed with this work for additional +# information regarding copyright ownership. +# +# This program and the accompanying materials are made available under the +# terms of the Apache License Version 2.0 which is available at +# https://www.apache.org/licenses/LICENSE-2.0 +# +# SPDX-License-Identifier: Apache-2.0 +# ******************************************************************************* +"""Resolved dependency versions from the reference_integration root. + +DR-008 Option 4 requires that the dependency versions ``reference_integration`` +resolves are pushed *into* each module so the module's own unit tests + coverage +run against the resolved set (not against the versions the module declares in its +released ``MODULE.bazel``). + +This module provides :class:`ResolvedDependencies`, which: + +* holds the resolved version/commit per dependency (sourced from ref_int's root — + either ``known_good.json`` for local runs, or the Stage-1 ``stage1-resolved-deps`` + artifact for CI runs so the resolution flows Stage 1 -> Stage 2), and +* exposes an interface to **scan** an individual module's ``MODULE.bazel`` and + **overwrite** the declared dependency versions to match the resolved set, by + appending the matching ``git_override`` / ``single_version_override`` directives. + +The injection is append-only and operates on the CI checkout of the module — it is +never committed back to the module's released sources (DR-008 "temporary mechanism"). +""" + +from __future__ import annotations + +import argparse +import json +import logging +import re +import sys +from pathlib import Path +from typing import Dict, List, Optional + +# Import ``models`` + ``generate_override_directive`` whether this file is loaded as +# ``known_good.resolved_dependencies`` (scripts/ on path, e.g. from quality_runners.py) +# or as ``resolved_dependencies`` (scripts/known_good/ on path). Preferring the +# package-qualified form keeps a single ``Module`` class identity in the package context. +_HERE = Path(__file__).resolve().parent +try: + from known_good.models.known_good import load_known_good + from known_good.models.module import Module + from known_good.update_module_from_known_good import generate_override_directive +except ImportError: + if str(_HERE) not in sys.path: + sys.path.insert(0, str(_HERE)) + from models.known_good import load_known_good # noqa: E402 + from models.module import Module # noqa: E402 + from update_module_from_known_good import generate_override_directive # noqa: E402 + +# Marker delimiting the block we append, so injection is idempotent / detectable. +INJECTION_BEGIN = "# --- BEGIN ref_int resolved-deps injection (DR-008 Option 4) ---" +INJECTION_END = "# --- END ref_int resolved-deps injection (DR-008 Option 4) ---" + +# The single file that carries the resolved set from Stage 1 (resolve) to Stage 2 +# (per-module validation). It is the only handoff needed: first-party commits + +# third-party resolved versions, merged. The lock travels alongside only as evidence. +MANIFEST_NAME = "resolved_versions.json" + +# Built-in / non-registry modules that must not be given a single_version_override. +_SKIP_MODULES = {"bazel_tools"} + +# Capture the module name from any ``bazel_dep(name = "...")`` call (name is the first arg). +_BAZEL_DEP_RE = re.compile(r'bazel_dep\(\s*name\s*=\s*"([^"]+)"') +# Capture an existing override target so we don't inject a duplicate for the same module. +_OVERRIDE_RE = re.compile( + r'(?:git_override|single_version_override|local_path_override|archive_override)\(\s*module_name\s*=\s*"([^"]+)"' +) +# Parsers for reconstructing the resolved set from generated score_modules_*.MODULE.bazel. +_GIT_OVERRIDE_BLOCK_RE = re.compile(r"git_override\((?P.*?)\)", re.S) +_SINGLE_VERSION_BLOCK_RE = re.compile(r"single_version_override\((?P.*?)\)", re.S) +_FIELD_RE = lambda field: re.compile(rf'{field}\s*=\s*"([^"]+)"') # noqa: E731 + + +class ResolvedDependencies: + """Resolved dependency versions from the reference_integration root. + + Holds a ``name -> Module`` map of the dependencies ref_int pins, and provides an + interface to scan + overwrite a module's ``MODULE.bazel`` to those versions. + """ + + def __init__(self, resolved: Dict[str, Module]): + self._resolved = resolved + + # -- construction: "resolved deps versions from ref_int root" -------------------- + + @classmethod + def from_known_good(cls, known_good_path: Path) -> "ResolvedDependencies": + """Build from ``known_good.json`` (local / dev source of the resolved pins).""" + kg = load_known_good(Path(known_good_path).resolve()) + resolved: Dict[str, Module] = {} + for group in kg.modules.values(): + for module in group.values(): + resolved[module.name] = module + return cls(resolved) + + @classmethod + def from_resolved_artifact(cls, artifact_dir: Path) -> "ResolvedDependencies": + """Build from the Stage-1 ``stage1-resolved-deps`` artifact. + + The handoff is the single ``resolved_versions.json`` manifest (see + :meth:`from_mod_graph` / :meth:`to_file`). For backward compatibility, if the + manifest is absent the older format is parsed: the generated + ``score_modules_*.MODULE.bazel`` override files, gated on the presence of + ``MODULE.bazel.lock`` as evidence of full resolution. + """ + artifact_dir = Path(artifact_dir) + + manifest = artifact_dir / MANIFEST_NAME + if manifest.is_file(): + return cls.from_file(manifest) + + # Legacy fallback: reconstruct from the generated override files. + lock = artifact_dir / "MODULE.bazel.lock" + if not lock.is_file(): + raise FileNotFoundError( + f"Neither {MANIFEST_NAME} nor MODULE.bazel.lock found in resolved-deps artifact " + f"{artifact_dir}; Stage 2 must consume the Stage-1 resolved dependency set." + ) + + module_files = sorted(artifact_dir.glob("score_modules_*.MODULE.bazel")) + if not module_files: + raise FileNotFoundError(f"No score_modules_*.MODULE.bazel files in resolved-deps artifact {artifact_dir}.") + + resolved: Dict[str, Module] = {} + for mf in module_files: + for module in cls._parse_override_file(mf.read_text()): + resolved[module.name] = module + return cls(resolved) + + @classmethod + def from_mod_graph(cls, mod_graph_json: Path, override_files: List[Path]) -> "ResolvedDependencies": + """Build the *complete* resolved set by merging two sources. + + * The override directives ref_int actually declares — parsed from its root + ``MODULE.bazel`` and the ``bazel_common/*.MODULE.bazel`` files it ``include()``s. + This carries every module ref_int pins by a non-registry source as its real + directive: ``git_override(commit, remote)`` for ``score_*`` plus third-party like + ``trlc`` / ``flatbuffers`` / ``rules_oci``, and ``single_version_override`` where + ref_int pins a registry version. The graph cannot supply these — it reports + overridden modules as version ``0.0.0``. + * ``bazel mod graph --output=json`` — the post-MVS resolved version of every other + (registry) module (protobuf, abseil, rules_rust, ...), emitted as + ``single_version_override`` so each module under test is forced to the exact + version ref_int resolved (MVS is graph-global, so a module's own subgraph could + otherwise select a different version). + + ``archive_override`` / ``local_path_override`` targets (e.g. ``rules_boost``) cannot + be represented and are logged as not carried. + """ + resolved: Dict[str, Module] = {} + unrepresentable: List[str] = [] + for f in override_files: + # Drop comment-only lines first: hand-written MODULE.bazel files contain + # commented-out overrides (e.g. "# git_override(... rules_rpm ...)") that must + # not be captured. Inline trailing comments (after a value) are left intact. + text = "\n".join(ln for ln in Path(f).read_text().splitlines() if not ln.lstrip().startswith("#")) + for module in cls._parse_override_file(text): # git_override + single_version_override + resolved[module.name] = module + for m in re.finditer(r'(archive_override|local_path_override)\(\s*module_name\s*=\s*"([^"]+)"', text): + unrepresentable.append(f"{m.group(2)} ({m.group(1)})") + + graph = json.loads(Path(mod_graph_json).read_text()) + versions: Dict[str, str] = {} + _collect_resolved_versions(graph, versions) + skipped: List[str] = [] + for name, version in versions.items(): + if name in resolved or name in _SKIP_MODULES: + continue # already carried by an override directive, or non-overridable + if not version or version == "0.0.0": + # Non-registry version: ref_int pins it via an override we did not capture + # (e.g. archive_override). single_version_override cannot reproduce it. + skipped.append(name) + continue + resolved[name] = Module(name=name, hash="", repo="", version=version) + + if unrepresentable: + logging.warning( + "Overrides not carried into manifest (need manual handling): %s", ", ".join(unrepresentable) + ) + if skipped: + logging.warning( + "Graph modules at version 0.0.0 with no carried override, skipped: %s", ", ".join(sorted(skipped)) + ) + return cls(resolved) + + def to_file(self, path: Path) -> None: + """Serialize the resolved set to the JSON manifest (Stage 1 -> Stage 2 handoff). + + Only the fields needed to regenerate the override directive are stored + (``version`` for single_version_override; ``repo`` + ``hash`` for git_override). + Metadata is intentionally omitted — the manifest carries dependency pins, not the + module-under-test's test configuration (that comes from known_good.json). + """ + modules = {} + for name in sorted(self._resolved): + m = self._resolved[name] + entry: Dict[str, object] = {"version": m.version} if m.version else {"repo": m.repo, "hash": m.hash} + if m.bazel_patches: + entry["bazel_patches"] = m.bazel_patches + modules[name] = entry + Path(path).write_text(json.dumps({"modules": modules}, indent=2) + "\n") + + @classmethod + def from_file(cls, path: Path) -> "ResolvedDependencies": + """Load a resolved set previously written by :meth:`to_file`.""" + data = json.loads(Path(path).read_text()) + resolved = {name: Module.from_dict(name, md) for name, md in data.get("modules", {}).items()} + return cls(resolved) + + @staticmethod + def _parse_override_file(text: str) -> List[Module]: + """Reconstruct Module objects from generated git/single_version override blocks.""" + modules: List[Module] = [] + + for match in _GIT_OVERRIDE_BLOCK_RE.finditer(text): + body = match.group("body") + name = _field(body, "module_name") + commit = _field(body, "commit") + remote = _field(body, "remote") + if name and commit and remote: + modules.append(Module(name=name, hash=commit, repo=remote)) + + for match in _SINGLE_VERSION_BLOCK_RE.finditer(text): + body = match.group("body") + name = _field(body, "module_name") + version = _field(body, "version") + if name and version: + modules.append(Module(name=name, hash="", repo="", version=version)) + + return modules + + # -- interface: scan + overwrite a module's MODULE.bazel ------------------------- + + @property + def names(self) -> set[str]: + return set(self._resolved) + + def get(self, name: str) -> Optional[Module]: + return self._resolved.get(name) + + def scan(self, module_bazel: Path) -> List[str]: + """Return the names of dependencies a module declares via ``bazel_dep``.""" + text = Path(module_bazel).read_text() + # Ignore anything inside a previous injection block so re-scans are stable. + text = self._strip_injection(text) + return _BAZEL_DEP_RE.findall(text) + + def overwrite(self, module_bazel: Path, *, module_under_test: Optional[str] = None, write: bool = True) -> str: + """Overwrite a module's declared dependency versions with the resolved set. + + Appends a ``git_override`` / ``single_version_override`` directive for every + dependency the module declares that we have a resolved version for, so the + module (and all its transitive deps) build against ref_int's resolved versions. + + * Skips the module under test itself (the root is never overridden). + * Skips dependencies that already carry an override in the file. + * Re-running is idempotent: a prior injection block is replaced. + """ + module_bazel = Path(module_bazel) + original = self._strip_injection(module_bazel.read_text()) + + declared = set(_BAZEL_DEP_RE.findall(original)) + already_overridden = set(_OVERRIDE_RE.findall(original)) + + from dataclasses import replace as _replace + + directives: List[str] = [] + # Inject overrides only for deps the module actually declares (intersected with the + # resolved set). Bazel fails with "root module specifies overrides on nonexistent + # module(s)" if an override targets a module that is not in this module's dependency + # graph, so the full resolved set cannot be injected wholesale — a declared bazel_dep + # is by definition in the graph, which makes its override safe. + for name in sorted(declared): + if name == module_under_test: + continue # the module under test is the root; never override it + if name in already_overridden: + continue # respect an override the module already declares + module = self._resolved.get(name) + if module is None: + continue # dep ref_int does not pin; resolves normally + # Strip bazel_patches: they reference //patches/... labels in ref_int's + # workspace which do not exist inside another module's checkout. + module = _replace(module, bazel_patches=None) + directive = generate_override_directive(module) + if directive is None: + continue + directives.append(directive) + + if not directives: + patched = original + else: + body = "\n".join(directives) + patched = f"{original.rstrip()}\n\n{INJECTION_BEGIN}\n{body}\n{INJECTION_END}\n" + + if write: + module_bazel.write_text(patched) + return patched + + @staticmethod + def _strip_injection(text: str) -> str: + """Remove a previously appended injection block, if present.""" + pattern = re.compile( + re.escape(INJECTION_BEGIN) + r".*?" + re.escape(INJECTION_END) + r"\n?", + re.S, + ) + return pattern.sub("", text).rstrip() + "\n" if pattern.search(text) else text + + +def _field(body: str, field: str) -> str: + match = _FIELD_RE(field).search(body) + return match.group(1) if match else "" + + +def _collect_resolved_versions(node: dict, acc: Dict[str, str]) -> None: + """Walk a ``bazel mod graph --output=json`` tree, recording name -> resolved version. + + Each node carries the post-MVS ``name`` and ``version``; a module can appear many + times in the graph but always at the single resolved version, so deduping by name is + safe. The ```` node has an empty version and is skipped implicitly. + """ + for dep in node.get("dependencies", []): + name, version = dep.get("name"), dep.get("version") + if name and version: + acc[name] = version + _collect_resolved_versions(dep, acc) + + +def _parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser( + description="Resolve (Stage 1) or inject (Stage 2) ref_int's resolved dependency set (DR-008 Option 4)." + ) + parser.add_argument( + "module_bazel", + type=Path, + nargs="?", + default=None, + help="Inject mode: path to the module's MODULE.bazel to overwrite. Omit when using --export.", + ) + parser.add_argument( + "--known-good-path", + type=Path, + default=_HERE.parents[1] / "known_good.json", + help="Resolved set source: known_good.json (default; first-party commit pins).", + ) + parser.add_argument( + "--resolved-deps", + type=Path, + default=None, + help="Inject mode: Stage-1 stage1-resolved-deps artifact dir (overrides --known-good-path).", + ) + parser.add_argument( + "--mod-graph", + type=Path, + default=None, + help="Export mode: 'bazel mod graph --output=json' output, merged with known_good.json.", + ) + parser.add_argument( + "--export", + type=Path, + default=None, + help=f"Export mode: write the merged resolved set to this {MANIFEST_NAME} manifest and exit.", + ) + parser.add_argument( + "--module-under-test", + default=None, + help="Name of the module under test (never overridden as it is the root).", + ) + parser.add_argument("--dry-run", action="store_true", help="Print patched content instead of writing.") + return parser.parse_args() + + +def main() -> None: + args = _parse_args() + + # Export mode (Stage 1): build the manifest by merging the override directives ref_int + # declares (root MODULE.bazel + bazel_common/*.MODULE.bazel) with the resolved registry + # versions from 'bazel mod graph'. + if args.export is not None: + if args.mod_graph is None: + raise SystemExit("--export requires --mod-graph (output of 'bazel mod graph --output=json')") + repo_root = _HERE.parents[1] + override_files = [repo_root / "MODULE.bazel", *sorted((repo_root / "bazel_common").glob("*.MODULE.bazel"))] + override_files = [f for f in override_files if f.is_file()] + resolved = ResolvedDependencies.from_mod_graph(args.mod_graph, override_files) + Path(args.export).parent.mkdir(parents=True, exist_ok=True) + resolved.to_file(args.export) + print(f"Wrote resolved dependency manifest ({len(resolved.names)} modules) to {args.export}") + return + + # Inject mode (Stage 2): overwrite a module's MODULE.bazel with the resolved set. + if args.module_bazel is None: + raise SystemExit("module_bazel is required unless --export is given") + + if args.resolved_deps: + resolved = ResolvedDependencies.from_resolved_artifact(args.resolved_deps) + else: + resolved = ResolvedDependencies.from_known_good(args.known_good_path) + + patched = resolved.overwrite( + args.module_bazel, + module_under_test=args.module_under_test, + write=not args.dry_run, + ) + if args.dry_run: + print(patched) + else: + print(f"Injected resolved-deps overrides into {args.module_bazel}") + + +if __name__ == "__main__": + main() diff --git a/scripts/known_good/tests/test_resolved_dependencies.py b/scripts/known_good/tests/test_resolved_dependencies.py new file mode 100644 index 00000000000..37608b76004 --- /dev/null +++ b/scripts/known_good/tests/test_resolved_dependencies.py @@ -0,0 +1,289 @@ +# ******************************************************************************* +# Copyright (c) 2026 Contributors to the Eclipse Foundation +# +# See the NOTICE file(s) distributed with this work for additional +# information regarding copyright ownership. +# +# This program and the accompanying materials are made available under the +# terms of the Apache License Version 2.0 which is available at +# https://www.apache.org/licenses/LICENSE-2.0 +# +# SPDX-License-Identifier: Apache-2.0 +# ******************************************************************************* +"""Unit tests for ResolvedDependencies (DR-008 Option 4 dependency injection). + +Self-contained: builds the resolved set from a temporary known_good.json and +overwrites a temporary module MODULE.bazel — no cloned repos or Bazel required. +""" + +import json +import sys +from pathlib import Path + +import pytest + +# Make scripts/known_good importable when run via plain pytest. +_KG_DIR = Path(__file__).resolve().parents[1] +if str(_KG_DIR) not in sys.path: + sys.path.insert(0, str(_KG_DIR)) + +from resolved_dependencies import ( # noqa: E402 + INJECTION_BEGIN, + INJECTION_END, + ResolvedDependencies, +) + +KNOWN_GOOD = { + "modules": { + "target_sw": { + "score_baselibs": { + "repo": "https://github.com/eclipse-score/baselibs.git", + "hash": "cab36dd7de92aaaaaaaaaaaaaaaaaaaaaaaaaaaa", + "bazel_patches": ["patches/baselibs/001-fix.patch"], + }, + "score_logging": { + "repo": "https://github.com/eclipse-score/logging.git", + "hash": "0e9187f79a99bbbbbbbbbbbbbbbbbbbbbbbbbbbb", + }, + "score_persistency": { + "repo": "https://github.com/eclipse-score/persistency.git", + "hash": "4d1fa1ae3c55cccccccccccccccccccccccccccc", + }, + }, + "tooling": { + "score_tooling": { + "repo": "https://github.com/eclipse-score/tooling.git", + "version": "1.2.0", + }, + }, + }, + "timestamp": "2026-01-01T00:00:00+00:00Z", +} + +MODULE_BAZEL = """\ +module(name = "score_persistency", version = "0.0.0") + +bazel_dep(name = "rules_cc", version = "0.2.17") +bazel_dep(name = "score_baselibs", version = "0.2.7") +bazel_dep(name = "score_logging", version = "0.2.0") +bazel_dep(name = "score_tooling", version = "1.0.0") +bazel_dep(name = "score_unpinned", version = "9.9.9") +""" + + +@pytest.fixture +def known_good_file(tmp_path: Path) -> Path: + p = tmp_path / "known_good.json" + p.write_text(json.dumps(KNOWN_GOOD)) + return p + + +@pytest.fixture +def module_bazel(tmp_path: Path) -> Path: + p = tmp_path / "MODULE.bazel" + p.write_text(MODULE_BAZEL) + return p + + +@pytest.fixture +def resolved(known_good_file: Path) -> ResolvedDependencies: + return ResolvedDependencies.from_known_good(known_good_file) + + +class TestFromKnownGood: + def test_names_span_all_groups(self, resolved: ResolvedDependencies): + assert {"score_baselibs", "score_logging", "score_persistency", "score_tooling"} <= resolved.names + + def test_get_returns_resolved_commit(self, resolved: ResolvedDependencies): + assert resolved.get("score_baselibs").hash.startswith("cab36dd7de92") + + def test_version_module_kept(self, resolved: ResolvedDependencies): + assert resolved.get("score_tooling").version == "1.2.0" + + +class TestScan: + def test_returns_declared_deps(self, resolved: ResolvedDependencies, module_bazel: Path): + declared = resolved.scan(module_bazel) + assert "score_baselibs" in declared + assert "score_unpinned" in declared + assert "rules_cc" in declared + + +class TestOverwrite: + def test_pins_declared_resolved_siblings(self, resolved: ResolvedDependencies, module_bazel: Path): + patched = resolved.overwrite(module_bazel, module_under_test="score_persistency", write=False) + block = patched.split(INJECTION_BEGIN)[1].split(INJECTION_END)[0] + assert 'git_override(\n module_name = "score_baselibs"' in block + assert 'commit = "cab36dd7de92aaaaaaaaaaaaaaaaaaaaaaaaaaaa"' in block + # version module -> single_version_override + assert 'single_version_override(\n module_name = "score_tooling"' in block + assert 'version = "1.2.0"' in block + + def test_strips_patches(self, resolved: ResolvedDependencies, module_bazel: Path): + # bazel_patches reference //patches/... labels that exist only in ref_int's + # workspace, so they are stripped from the injected overrides. + patched = resolved.overwrite(module_bazel, module_under_test="score_persistency", write=False) + assert "patches/baselibs/001-fix.patch" not in patched + assert "patch_strip" not in patched + + def test_skips_resolved_dep_not_declared(self, resolved: ResolvedDependencies, tmp_path: Path): + # Only declared deps are injected. Overriding a module that is NOT in the module's + # dependency graph makes Bazel fail ("overrides on nonexistent module(s)"), so a + # resolved dep the module does not declare must NOT be injected. + mod = tmp_path / "MODULE.bazel" + mod.write_text( + 'module(name = "score_persistency", version = "0.0.0")\nbazel_dep(name = "score_baselibs", version = "0.1")\n' + ) + block = resolved.overwrite(mod, module_under_test="score_persistency", write=False).split(INJECTION_BEGIN)[1] + assert 'module_name = "score_baselibs"' in block # declared -> injected + assert 'module_name = "score_logging"' not in block # not declared -> not injected + + def test_skips_root_module(self, resolved: ResolvedDependencies, module_bazel: Path): + patched = resolved.overwrite(module_bazel, module_under_test="score_persistency", write=False) + block = patched.split(INJECTION_BEGIN)[1].split(INJECTION_END)[0] + assert 'module_name = "score_persistency"' not in block + + def test_skips_unpinned_third_party(self, resolved: ResolvedDependencies, module_bazel: Path): + patched = resolved.overwrite(module_bazel, module_under_test="score_persistency", write=False) + block = patched.split(INJECTION_BEGIN)[1].split(INJECTION_END)[0] + assert "score_unpinned" not in block + assert "rules_cc" not in block + + def test_idempotent(self, resolved: ResolvedDependencies, module_bazel: Path): + first = resolved.overwrite(module_bazel, module_under_test="score_persistency", write=True) + second = resolved.overwrite(module_bazel, module_under_test="score_persistency", write=True) + assert first == second + assert second.count(INJECTION_BEGIN) == 1 + + def test_skips_dep_with_existing_override(self, resolved: ResolvedDependencies, tmp_path: Path): + mod = tmp_path / "MODULE.bazel" + mod.write_text( + MODULE_BAZEL + '\ngit_override(\n module_name = "score_logging",\n commit = "deadbeef",\n' + ' remote = "https://example.com/x.git",\n)\n' + ) + patched = resolved.overwrite(mod, module_under_test="score_persistency", write=False) + block = patched.split(INJECTION_BEGIN)[1].split(INJECTION_END)[0] + assert 'module_name = "score_logging"' not in block # respected pre-existing override + + +class TestMetadataBazelConfig: + def test_bazel_config_roundtrip(self): + from models.module import Metadata + + m = Metadata.from_dict({"bazel_config": ["bl-x86_64-linux"]}) + assert m.bazel_config == ["bl-x86_64-linux"] + assert m.to_dict()["bazel_config"] == ["bl-x86_64-linux"] + + def test_bazel_config_default_empty(self): + from models.module import Metadata + + m = Metadata.from_dict({}) + assert m.bazel_config == [] + + def test_bazel_config_multi(self): + from models.module import Metadata + + m = Metadata.from_dict({"bazel_config": ["per-x86_64-linux", "ferrocene-coverage"]}) + assert m.bazel_config == ["per-x86_64-linux", "ferrocene-coverage"] + + +class TestFromModGraph: + @staticmethod + def _graph() -> dict: + # Mirrors 'bazel mod graph --output=json': overridden modules report version 0.0.0. + return { + "key": "", + "name": "ref_int", + "version": "", + "dependencies": [ + {"name": "trlc", "version": "0.0.0"}, # git_override (carried from file) + {"name": "rules_boost", "version": "0.0.0"}, # archive_override (not representable) + {"name": "score_baselibs", "version": "0.0.0"}, # git_override (carried from file) + { + "name": "protobuf", + "version": "29.1", + "dependencies": [ + {"name": "abseil-cpp", "version": "20250512.1"}, + ], + }, + ], + } + + def test_merges_overrides_and_registry_versions(self, tmp_path: Path): + graph = tmp_path / "graph.json" + graph.write_text(json.dumps(self._graph())) + root = tmp_path / "MODULE.bazel" + root.write_text( + 'git_override(\n module_name = "trlc",\n commit = "abc1234",\n' + ' remote = "https://github.com/x/trlc.git",\n)\n' + 'archive_override(\n module_name = "rules_boost",\n urls = ["https://e/x.tar"],\n)\n' + ) + scoremods = tmp_path / "score_modules_target_sw.MODULE.bazel" + scoremods.write_text( + 'git_override(\n module_name = "score_baselibs",\n commit = "def5678",\n' + ' remote = "https://github.com/eclipse-score/baselibs.git",\n)\n' + ) + + rd = ResolvedDependencies.from_mod_graph(graph, [root, scoremods]) + # Overridden modules carried as their real git_override (graph's 0.0.0 ignored). + assert rd.get("trlc").hash == "abc1234" + assert rd.get("score_baselibs").hash == "def5678" + # Registry modules carried from the resolved graph version. + assert rd.get("protobuf").version == "29.1" + assert rd.get("abseil-cpp").version == "20250512.1" + # archive_override target at 0.0.0 is not representable -> not carried. + assert rd.get("rules_boost") is None + + def test_ignores_commented_out_overrides(self, tmp_path: Path): + graph = tmp_path / "graph.json" + graph.write_text(json.dumps({"key": "", "name": "r", "version": "", "dependencies": []})) + root = tmp_path / "MODULE.bazel" + root.write_text( + '# git_override(\n# module_name = "rules_rpm",\n' + '# commit = "a78e559cf81754c199c926229dc6b4443e1ff149",\n' + '# remote = "https://github.com/eclipse-score/inc_os_autosd.git",\n# )\n' + ) + rd = ResolvedDependencies.from_mod_graph(graph, [root]) + assert rd.get("rules_rpm") is None # commented-out override must not be carried + + +class TestManifestRoundtrip: + def test_to_file_is_lean_and_roundtrips(self, tmp_path: Path, resolved: ResolvedDependencies): + manifest = tmp_path / "resolved_versions.json" + resolved.to_file(manifest) + data = json.loads(manifest.read_text())["modules"] + assert "metadata" not in data["score_baselibs"] # lean: no test-config noise + assert data["score_tooling"] == {"version": "1.2.0"} + loaded = ResolvedDependencies.from_file(manifest) + assert loaded.get("score_baselibs").hash == resolved.get("score_baselibs").hash + assert loaded.get("score_tooling").version == "1.2.0" + + +class TestFromResolvedArtifact: + def test_prefers_manifest(self, tmp_path: Path, resolved: ResolvedDependencies): + art = tmp_path / "art" + art.mkdir() + resolved.to_file(art / "resolved_versions.json") + # With the manifest present, no lock / score_modules files are required. + parsed = ResolvedDependencies.from_resolved_artifact(art) + assert parsed.get("score_baselibs").hash == resolved.get("score_baselibs").hash + assert parsed.get("score_tooling").version == "1.2.0" + + def test_requires_manifest_or_lockfile(self, tmp_path: Path): + (tmp_path / "score_modules_target_sw.MODULE.bazel").write_text("bazel_dep(name='x')\n") + with pytest.raises(FileNotFoundError): + ResolvedDependencies.from_resolved_artifact(tmp_path) + + def test_roundtrip_known_good_to_artifact(self, tmp_path: Path, resolved: ResolvedDependencies): + # Build an artifact dir mirroring stage1-resolved-deps, then parse it back. + from update_module_from_known_good import generate_git_override_blocks + + art = tmp_path / "art" + art.mkdir() + (art / "MODULE.bazel.lock").write_text("{}") + blocks = generate_git_override_blocks(list(resolved._resolved.values()), {}) + (art / "score_modules_target_sw.MODULE.bazel").write_text("\n".join(blocks)) + + parsed = ResolvedDependencies.from_resolved_artifact(art) + assert parsed.get("score_baselibs").hash == resolved.get("score_baselibs").hash + assert parsed.get("score_tooling").version == "1.2.0" diff --git a/scripts/known_good/update_module_from_known_good.py b/scripts/known_good/update_module_from_known_good.py index 2d61ea2a805..9174070b97a 100755 --- a/scripts/known_good/update_module_from_known_good.py +++ b/scripts/known_good/update_module_from_known_good.py @@ -35,76 +35,98 @@ from pathlib import Path from typing import Dict, List, Optional -from models import Module -from models.known_good import load_known_good +# Import models whether run standalone (scripts/known_good on path) or imported as part +# of the ``known_good`` package (scripts/ on path, where bare ``models`` is shadowed by +# the separate scripts/models package). +try: + from known_good.models.known_good import load_known_good + from known_good.models.module import Module +except ImportError: + from models import Module + from models.known_good import load_known_good # Configure logging logging.basicConfig(level=logging.WARNING, format="%(levelname)s: %(message)s") +def generate_override_directive(module: Module, repo_commit_dict: Optional[Dict[str, str]] = None) -> Optional[str]: + """Generate the override directive (single_version_override / git_override) for a module. + + Returns just the override call (without the preceding ``bazel_dep(...)`` line), so the + same logic can be reused both to (a) build ref_int's score_modules_*.MODULE.bazel files + (composed with a bazel_dep line by ``generate_git_override_blocks``) and to (b) inject + overrides into a module's own MODULE.bazel where the bazel_dep is already declared + (see ``ResolvedDependencies.overwrite`` in resolved_dependencies.py). + + Returns ``None`` (and logs a warning) when the module has neither a usable version nor a + valid repo+commit, mirroring the skip behaviour of the original generator. + """ + repo_commit_dict = repo_commit_dict or {} + commit = module.hash + + # Allow overriding specific repos via command line + if module.repo in repo_commit_dict: + commit = repo_commit_dict[module.repo] + + # Generate patches lines if bazel_patches exist + patches_lines = "" + if module.bazel_patches: + patches_lines = " patches = [\n" + for patch in module.bazel_patches: + patches_lines += f' "{patch}",\n' + patches_lines += " ],\n" + patch_strip_line = " patch_strip = 1,\n" if patches_lines else "" + + if module.version: + # If version is provided, use single_version_override + return ( + "single_version_override(\n" + f' module_name = "{module.name}",\n' + f"{patch_strip_line}" + f"{patches_lines}" + f' version = "{module.version}",\n' + ")\n" + ) + + if not module.repo or not commit: + logging.warning( + "Skipping module %s with missing repo or commit: repo=%s, commit=%s", + module.name, + module.repo, + commit, + ) + return None + + # Validate commit hash format (7-40 hex characters) + if not re.match(r"^[a-fA-F0-9]{7,40}$", commit): + logging.warning( + "Skipping module %s with invalid commit hash: %s", + module.name, + commit, + ) + return None + + # If no version, use git_override. Only include patch_strip if there are patches to apply. + return ( + "git_override(\n" + f' module_name = "{module.name}",\n' + f' commit = "{commit}",\n' + f"{patch_strip_line}" + f"{patches_lines}" + f' remote = "{module.repo}",\n' + ")\n" + ) + + def generate_git_override_blocks(modules: List[Module], repo_commit_dict: Dict[str, str]) -> List[str]: """Generate bazel_dep and git_override blocks for each module.""" blocks = [] for module in modules: - commit = module.hash - - # Allow overriding specific repos via command line - if module.repo in repo_commit_dict: - commit = repo_commit_dict[module.repo] - - # Generate patches lines if bazel_patches exist - patches_lines = "" - if module.bazel_patches: - patches_lines = " patches = [\n" - for patch in module.bazel_patches: - patches_lines += f' "{patch}",\n' - patches_lines += " ],\n" - patch_strip_line = " patch_strip = 1,\n" if patches_lines else "" - - if module.version: - # If version is provided, use bazel_dep with single_version_override - block = ( - f'bazel_dep(name = "{module.name}")\n' - "single_version_override(\n" - f' module_name = "{module.name}",\n' - f"{patch_strip_line}" - f"{patches_lines}" - f' version = "{module.version}",\n' - ")\n" - ) - else: - if not module.repo or not commit: - logging.warning( - "Skipping module %s with missing repo or commit: repo=%s, commit=%s", - module.name, - module.repo, - commit, - ) - continue - - # Validate commit hash format (7-40 hex characters) - if not re.match(r"^[a-fA-F0-9]{7,40}$", commit): - logging.warning( - "Skipping module %s with invalid commit hash: %s", - module.name, - commit, - ) - continue - - # If no version, use bazel_dep with git_override - # Only include patch_strip if there are patches to apply - block = ( - f'bazel_dep(name = "{module.name}")\n' - "git_override(\n" - f' module_name = "{module.name}",\n' - f' commit = "{commit}",\n' - f"{patch_strip_line}" - f"{patches_lines}" - f' remote = "{module.repo}",\n' - ")\n" - ) - blocks.append(block) + directive = generate_override_directive(module, repo_commit_dict) + if directive is None: + continue + blocks.append(f'bazel_dep(name = "{module.name}")\n' + directive) return blocks From 7a7203bc6a39615ece28f5c31ea1536d53b90bd3 Mon Sep 17 00:00:00 2001 From: subramaniak Date: Thu, 2 Jul 2026 11:42:58 +0530 Subject: [PATCH 2/6] fix: remove bazel_config, always overwrite overrides, clean injection markers, simplify imports --- scripts/known_good/models/module.py | 5 +- scripts/known_good/resolved_dependencies.py | 56 ++++++------------- .../tests/test_resolved_dependencies.py | 40 ++++--------- 3 files changed, 28 insertions(+), 73 deletions(-) diff --git a/scripts/known_good/models/module.py b/scripts/known_good/models/module.py index de01225a368..73adfe5f9a4 100644 --- a/scripts/known_good/models/module.py +++ b/scripts/known_good/models/module.py @@ -35,8 +35,7 @@ class Metadata: extra_test_config: list[str] = field(default_factory=lambda: []) exclude_test_targets: list[str] = field(default_factory=lambda: []) langs: list[str] = field(default_factory=lambda: ["cpp", "rust"]) - rust_coverage_config: str | None = "ferrocene-coverage" # Optional field for Rust coverage configuration - bazel_config: list[str] = field(default_factory=lambda: []) + rust_coverage_config: str | None = "ferrocene-coverage" @classmethod def from_dict(cls, data: Dict[str, Any]) -> Metadata: @@ -54,7 +53,6 @@ def from_dict(cls, data: Dict[str, Any]) -> Metadata: exclude_test_targets=data.get("exclude_test_targets", []), langs=data.get("langs", ["cpp", "rust"]), rust_coverage_config=data.get("rust_coverage_config", "ferrocene-coverage"), - bazel_config=data.get("bazel_config", []), ) def to_dict(self) -> Dict[str, Any]: @@ -69,7 +67,6 @@ def to_dict(self) -> Dict[str, Any]: "exclude_test_targets": self.exclude_test_targets, "langs": self.langs, "rust_coverage_config": self.rust_coverage_config, - "bazel_config": self.bazel_config, } diff --git a/scripts/known_good/resolved_dependencies.py b/scripts/known_good/resolved_dependencies.py index 6baebf85cdf..09a004c5b47 100644 --- a/scripts/known_good/resolved_dependencies.py +++ b/scripts/known_good/resolved_dependencies.py @@ -13,22 +13,15 @@ # ******************************************************************************* """Resolved dependency versions from the reference_integration root. -DR-008 Option 4 requires that the dependency versions ``reference_integration`` -resolves are pushed *into* each module so the module's own unit tests + coverage -run against the resolved set (not against the versions the module declares in its -released ``MODULE.bazel``). - -This module provides :class:`ResolvedDependencies`, which: - -* holds the resolved version/commit per dependency (sourced from ref_int's root — - either ``known_good.json`` for local runs, or the Stage-1 ``stage1-resolved-deps`` - artifact for CI runs so the resolution flows Stage 1 -> Stage 2), and -* exposes an interface to **scan** an individual module's ``MODULE.bazel`` and - **overwrite** the declared dependency versions to match the resolved set, by - appending the matching ``git_override`` / ``single_version_override`` directives. - -The injection is append-only and operates on the CI checkout of the module — it is -never committed back to the module's released sources (DR-008 "temporary mechanism"). +Provides :class:`ResolvedDependencies`, which holds the resolved version/commit per +dependency (sourced from ref_int's root — either ``known_good.json`` for local runs, +or the Stage-1 ``stage1-resolved-deps`` artifact for CI runs), and exposes an interface +to **scan** an individual module's ``MODULE.bazel`` and **overwrite** the declared +dependency versions to match the resolved set by appending the matching +``git_override`` / ``single_version_override`` directives. + +The injection operates on the CI checkout of the module — it is never committed back +to the module's released sources. """ from __future__ import annotations @@ -37,29 +30,18 @@ import json import logging import re -import sys from pathlib import Path from typing import Dict, List, Optional -# Import ``models`` + ``generate_override_directive`` whether this file is loaded as -# ``known_good.resolved_dependencies`` (scripts/ on path, e.g. from quality_runners.py) -# or as ``resolved_dependencies`` (scripts/known_good/ on path). Preferring the -# package-qualified form keeps a single ``Module`` class identity in the package context. _HERE = Path(__file__).resolve().parent -try: - from known_good.models.known_good import load_known_good - from known_good.models.module import Module - from known_good.update_module_from_known_good import generate_override_directive -except ImportError: - if str(_HERE) not in sys.path: - sys.path.insert(0, str(_HERE)) - from models.known_good import load_known_good # noqa: E402 - from models.module import Module # noqa: E402 - from update_module_from_known_good import generate_override_directive # noqa: E402 + +from known_good.models.known_good import load_known_good +from known_good.models.module import Module +from known_good.update_module_from_known_good import generate_override_directive # Marker delimiting the block we append, so injection is idempotent / detectable. -INJECTION_BEGIN = "# --- BEGIN ref_int resolved-deps injection (DR-008 Option 4) ---" -INJECTION_END = "# --- END ref_int resolved-deps injection (DR-008 Option 4) ---" +INJECTION_BEGIN = "# --- BEGIN ref_int resolved-deps injection ---" +INJECTION_END = "# --- END ref_int resolved-deps injection ---" # The single file that carries the resolved set from Stage 1 (resolve) to Stage 2 # (per-module validation). It is the only handoff needed: first-party commits + @@ -71,10 +53,6 @@ # Capture the module name from any ``bazel_dep(name = "...")`` call (name is the first arg). _BAZEL_DEP_RE = re.compile(r'bazel_dep\(\s*name\s*=\s*"([^"]+)"') -# Capture an existing override target so we don't inject a duplicate for the same module. -_OVERRIDE_RE = re.compile( - r'(?:git_override|single_version_override|local_path_override|archive_override)\(\s*module_name\s*=\s*"([^"]+)"' -) # Parsers for reconstructing the resolved set from generated score_modules_*.MODULE.bazel. _GIT_OVERRIDE_BLOCK_RE = re.compile(r"git_override\((?P.*?)\)", re.S) _SINGLE_VERSION_BLOCK_RE = re.compile(r"single_version_override\((?P.*?)\)", re.S) @@ -270,7 +248,6 @@ def overwrite(self, module_bazel: Path, *, module_under_test: Optional[str] = No original = self._strip_injection(module_bazel.read_text()) declared = set(_BAZEL_DEP_RE.findall(original)) - already_overridden = set(_OVERRIDE_RE.findall(original)) from dataclasses import replace as _replace @@ -280,11 +257,10 @@ def overwrite(self, module_bazel: Path, *, module_under_test: Optional[str] = No # module(s)" if an override targets a module that is not in this module's dependency # graph, so the full resolved set cannot be injected wholesale — a declared bazel_dep # is by definition in the graph, which makes its override safe. + # ref_int always decides the version — any existing module-level override is replaced. for name in sorted(declared): if name == module_under_test: continue # the module under test is the root; never override it - if name in already_overridden: - continue # respect an override the module already declares module = self._resolved.get(name) if module is None: continue # dep ref_int does not pin; resolves normally diff --git a/scripts/known_good/tests/test_resolved_dependencies.py b/scripts/known_good/tests/test_resolved_dependencies.py index 37608b76004..a51aded6ead 100644 --- a/scripts/known_good/tests/test_resolved_dependencies.py +++ b/scripts/known_good/tests/test_resolved_dependencies.py @@ -22,12 +22,12 @@ import pytest -# Make scripts/known_good importable when run via plain pytest. -_KG_DIR = Path(__file__).resolve().parents[1] -if str(_KG_DIR) not in sys.path: - sys.path.insert(0, str(_KG_DIR)) +# Make scripts/ importable so known_good.* package resolves when run via plain pytest. +_SCRIPTS_DIR = Path(__file__).resolve().parents[2] +if str(_SCRIPTS_DIR) not in sys.path: + sys.path.insert(0, str(_SCRIPTS_DIR)) -from resolved_dependencies import ( # noqa: E402 +from known_good.resolved_dependencies import ( # noqa: E402 INJECTION_BEGIN, INJECTION_END, ResolvedDependencies, @@ -155,7 +155,8 @@ def test_idempotent(self, resolved: ResolvedDependencies, module_bazel: Path): assert first == second assert second.count(INJECTION_BEGIN) == 1 - def test_skips_dep_with_existing_override(self, resolved: ResolvedDependencies, tmp_path: Path): + def test_overwrites_dep_with_existing_override(self, resolved: ResolvedDependencies, tmp_path: Path): + # ref_int always decides the version — a pre-existing override in the module is replaced. mod = tmp_path / "MODULE.bazel" mod.write_text( MODULE_BAZEL + '\ngit_override(\n module_name = "score_logging",\n commit = "deadbeef",\n' @@ -163,28 +164,9 @@ def test_skips_dep_with_existing_override(self, resolved: ResolvedDependencies, ) patched = resolved.overwrite(mod, module_under_test="score_persistency", write=False) block = patched.split(INJECTION_BEGIN)[1].split(INJECTION_END)[0] - assert 'module_name = "score_logging"' not in block # respected pre-existing override - - -class TestMetadataBazelConfig: - def test_bazel_config_roundtrip(self): - from models.module import Metadata - - m = Metadata.from_dict({"bazel_config": ["bl-x86_64-linux"]}) - assert m.bazel_config == ["bl-x86_64-linux"] - assert m.to_dict()["bazel_config"] == ["bl-x86_64-linux"] - - def test_bazel_config_default_empty(self): - from models.module import Metadata - - m = Metadata.from_dict({}) - assert m.bazel_config == [] - - def test_bazel_config_multi(self): - from models.module import Metadata - - m = Metadata.from_dict({"bazel_config": ["per-x86_64-linux", "ferrocene-coverage"]}) - assert m.bazel_config == ["per-x86_64-linux", "ferrocene-coverage"] + # ref_int's resolved commit must appear in the injection block, overwriting "deadbeef" + assert 'module_name = "score_logging"' in block + assert "deadbeef" not in block class TestFromModGraph: @@ -276,7 +258,7 @@ def test_requires_manifest_or_lockfile(self, tmp_path: Path): def test_roundtrip_known_good_to_artifact(self, tmp_path: Path, resolved: ResolvedDependencies): # Build an artifact dir mirroring stage1-resolved-deps, then parse it back. - from update_module_from_known_good import generate_git_override_blocks + from known_good.update_module_from_known_good import generate_git_override_blocks art = tmp_path / "art" art.mkdir() From 472f92410cb2aabca916a3ad5dbe7586e053110e Mon Sep 17 00:00:00 2001 From: subramaniak Date: Thu, 2 Jul 2026 11:53:24 +0530 Subject: [PATCH 3/6] fix: revert update_module_from_known_good.py, move generate_override_directive inline, add BUILD for bazel run --- scripts/known_good/BUILD | 37 +++++ scripts/known_good/resolved_dependencies.py | 57 ++++++- .../tests/test_resolved_dependencies.py | 11 +- .../update_module_from_known_good.py | 142 ++++++++---------- scripts/tooling/BUILD | 8 + 5 files changed, 168 insertions(+), 87 deletions(-) create mode 100644 scripts/known_good/BUILD diff --git a/scripts/known_good/BUILD b/scripts/known_good/BUILD new file mode 100644 index 00000000000..647a539c997 --- /dev/null +++ b/scripts/known_good/BUILD @@ -0,0 +1,37 @@ +# ******************************************************************************* +# Copyright (c) 2026 Contributors to the Eclipse Foundation +# +# See the NOTICE file(s) distributed with this work for additional +# information regarding copyright ownership. +# +# This program and the accompanying materials are made available under the +# terms of the Apache License Version 2.0 which is available at +# https://www.apache.org/licenses/LICENSE-2.0 +# +# SPDX-License-Identifier: Apache-2.0 +# ******************************************************************************* +load("@rules_python//python:defs.bzl", "py_binary", "py_library") + +# Library target: the known_good package (models + generators). +# Used as a dep by //scripts/tooling and by the resolve_deps binary below. +py_library( + name = "known_good", + srcs = glob( + ["**/*.py"], + exclude = ["tests/**"], + ), + visibility = ["//visibility:public"], +) + +# Runnable binary for the resolve + inject workflow. +# Stage 1 (export): bazel run //scripts/known_good:resolve_deps -- \ +# --mod-graph graph.json --export artifacts/resolved_versions.json +# Stage 2 (inject): bazel run //scripts/known_good:resolve_deps -- \ +# _module/MODULE.bazel --resolved-deps _resolved_deps/ +py_binary( + name = "resolve_deps", + srcs = ["resolved_dependencies.py"], + main = "resolved_dependencies.py", + visibility = ["//visibility:public"], + deps = [":known_good"], +) diff --git a/scripts/known_good/resolved_dependencies.py b/scripts/known_good/resolved_dependencies.py index 09a004c5b47..d77f9dd3f42 100644 --- a/scripts/known_good/resolved_dependencies.py +++ b/scripts/known_good/resolved_dependencies.py @@ -37,12 +37,67 @@ from known_good.models.known_good import load_known_good from known_good.models.module import Module -from known_good.update_module_from_known_good import generate_override_directive # Marker delimiting the block we append, so injection is idempotent / detectable. INJECTION_BEGIN = "# --- BEGIN ref_int resolved-deps injection ---" INJECTION_END = "# --- END ref_int resolved-deps injection ---" + +def generate_override_directive(module: Module, repo_commit_dict: Optional[Dict[str, str]] = None) -> Optional[str]: + """Return the override directive (single_version_override / git_override) for a module. + + Returns just the override call without a preceding ``bazel_dep(...)`` line, so the + same logic can be reused both to build ref_int's score_modules_*.MODULE.bazel files + and to inject overrides into a module's own MODULE.bazel where bazel_dep is already + declared (see :meth:`ResolvedDependencies.overwrite`). + + Returns ``None`` when the module has neither a usable version nor a valid repo+commit. + """ + repo_commit_dict = repo_commit_dict or {} + commit = module.hash + + if module.repo in repo_commit_dict: + commit = repo_commit_dict[module.repo] + + patches_lines = "" + if module.bazel_patches: + patches_lines = " patches = [\n" + for patch in module.bazel_patches: + patches_lines += f' "{patch}",\n' + patches_lines += " ],\n" + patch_strip_line = " patch_strip = 1,\n" if patches_lines else "" + + if module.version: + return ( + "single_version_override(\n" + f' module_name = "{module.name}",\n' + f"{patch_strip_line}" + f"{patches_lines}" + f' version = "{module.version}",\n' + ")\n" + ) + + if not module.repo or not commit: + logging.warning( + "Skipping module %s with missing repo or commit: repo=%s, commit=%s", + module.name, module.repo, commit, + ) + return None + + if not re.match(r"^[a-fA-F0-9]{7,40}$", commit): + logging.warning("Skipping module %s with invalid commit hash: %s", module.name, commit) + return None + + return ( + "git_override(\n" + f' module_name = "{module.name}",\n' + f' commit = "{commit}",\n' + f"{patch_strip_line}" + f"{patches_lines}" + f' remote = "{module.repo}",\n' + ")\n" + ) + # The single file that carries the resolved set from Stage 1 (resolve) to Stage 2 # (per-module validation). It is the only handoff needed: first-party commits + # third-party resolved versions, merged. The lock travels alongside only as evidence. diff --git a/scripts/known_good/tests/test_resolved_dependencies.py b/scripts/known_good/tests/test_resolved_dependencies.py index a51aded6ead..511bb9fc2b2 100644 --- a/scripts/known_good/tests/test_resolved_dependencies.py +++ b/scripts/known_good/tests/test_resolved_dependencies.py @@ -31,6 +31,7 @@ INJECTION_BEGIN, INJECTION_END, ResolvedDependencies, + generate_override_directive, ) KNOWN_GOOD = { @@ -257,13 +258,15 @@ def test_requires_manifest_or_lockfile(self, tmp_path: Path): ResolvedDependencies.from_resolved_artifact(tmp_path) def test_roundtrip_known_good_to_artifact(self, tmp_path: Path, resolved: ResolvedDependencies): - # Build an artifact dir mirroring stage1-resolved-deps, then parse it back. - from known_good.update_module_from_known_good import generate_git_override_blocks - + # Build an artifact dir mirroring stage1-resolved-deps (legacy format), then parse it back. art = tmp_path / "art" art.mkdir() (art / "MODULE.bazel.lock").write_text("{}") - blocks = generate_git_override_blocks(list(resolved._resolved.values()), {}) + blocks = [] + for m in resolved._resolved.values(): + directive = generate_override_directive(m) + if directive: + blocks.append(f'bazel_dep(name = "{m.name}")\n' + directive) (art / "score_modules_target_sw.MODULE.bazel").write_text("\n".join(blocks)) parsed = ResolvedDependencies.from_resolved_artifact(art) diff --git a/scripts/known_good/update_module_from_known_good.py b/scripts/known_good/update_module_from_known_good.py index 9174070b97a..2d61ea2a805 100755 --- a/scripts/known_good/update_module_from_known_good.py +++ b/scripts/known_good/update_module_from_known_good.py @@ -35,98 +35,76 @@ from pathlib import Path from typing import Dict, List, Optional -# Import models whether run standalone (scripts/known_good on path) or imported as part -# of the ``known_good`` package (scripts/ on path, where bare ``models`` is shadowed by -# the separate scripts/models package). -try: - from known_good.models.known_good import load_known_good - from known_good.models.module import Module -except ImportError: - from models import Module - from models.known_good import load_known_good +from models import Module +from models.known_good import load_known_good # Configure logging logging.basicConfig(level=logging.WARNING, format="%(levelname)s: %(message)s") -def generate_override_directive(module: Module, repo_commit_dict: Optional[Dict[str, str]] = None) -> Optional[str]: - """Generate the override directive (single_version_override / git_override) for a module. - - Returns just the override call (without the preceding ``bazel_dep(...)`` line), so the - same logic can be reused both to (a) build ref_int's score_modules_*.MODULE.bazel files - (composed with a bazel_dep line by ``generate_git_override_blocks``) and to (b) inject - overrides into a module's own MODULE.bazel where the bazel_dep is already declared - (see ``ResolvedDependencies.overwrite`` in resolved_dependencies.py). - - Returns ``None`` (and logs a warning) when the module has neither a usable version nor a - valid repo+commit, mirroring the skip behaviour of the original generator. - """ - repo_commit_dict = repo_commit_dict or {} - commit = module.hash - - # Allow overriding specific repos via command line - if module.repo in repo_commit_dict: - commit = repo_commit_dict[module.repo] - - # Generate patches lines if bazel_patches exist - patches_lines = "" - if module.bazel_patches: - patches_lines = " patches = [\n" - for patch in module.bazel_patches: - patches_lines += f' "{patch}",\n' - patches_lines += " ],\n" - patch_strip_line = " patch_strip = 1,\n" if patches_lines else "" - - if module.version: - # If version is provided, use single_version_override - return ( - "single_version_override(\n" - f' module_name = "{module.name}",\n' - f"{patch_strip_line}" - f"{patches_lines}" - f' version = "{module.version}",\n' - ")\n" - ) - - if not module.repo or not commit: - logging.warning( - "Skipping module %s with missing repo or commit: repo=%s, commit=%s", - module.name, - module.repo, - commit, - ) - return None - - # Validate commit hash format (7-40 hex characters) - if not re.match(r"^[a-fA-F0-9]{7,40}$", commit): - logging.warning( - "Skipping module %s with invalid commit hash: %s", - module.name, - commit, - ) - return None - - # If no version, use git_override. Only include patch_strip if there are patches to apply. - return ( - "git_override(\n" - f' module_name = "{module.name}",\n' - f' commit = "{commit}",\n' - f"{patch_strip_line}" - f"{patches_lines}" - f' remote = "{module.repo}",\n' - ")\n" - ) - - def generate_git_override_blocks(modules: List[Module], repo_commit_dict: Dict[str, str]) -> List[str]: """Generate bazel_dep and git_override blocks for each module.""" blocks = [] for module in modules: - directive = generate_override_directive(module, repo_commit_dict) - if directive is None: - continue - blocks.append(f'bazel_dep(name = "{module.name}")\n' + directive) + commit = module.hash + + # Allow overriding specific repos via command line + if module.repo in repo_commit_dict: + commit = repo_commit_dict[module.repo] + + # Generate patches lines if bazel_patches exist + patches_lines = "" + if module.bazel_patches: + patches_lines = " patches = [\n" + for patch in module.bazel_patches: + patches_lines += f' "{patch}",\n' + patches_lines += " ],\n" + patch_strip_line = " patch_strip = 1,\n" if patches_lines else "" + + if module.version: + # If version is provided, use bazel_dep with single_version_override + block = ( + f'bazel_dep(name = "{module.name}")\n' + "single_version_override(\n" + f' module_name = "{module.name}",\n' + f"{patch_strip_line}" + f"{patches_lines}" + f' version = "{module.version}",\n' + ")\n" + ) + else: + if not module.repo or not commit: + logging.warning( + "Skipping module %s with missing repo or commit: repo=%s, commit=%s", + module.name, + module.repo, + commit, + ) + continue + + # Validate commit hash format (7-40 hex characters) + if not re.match(r"^[a-fA-F0-9]{7,40}$", commit): + logging.warning( + "Skipping module %s with invalid commit hash: %s", + module.name, + commit, + ) + continue + + # If no version, use bazel_dep with git_override + # Only include patch_strip if there are patches to apply + block = ( + f'bazel_dep(name = "{module.name}")\n' + "git_override(\n" + f' module_name = "{module.name}",\n' + f' commit = "{commit}",\n' + f"{patch_strip_line}" + f"{patches_lines}" + f' remote = "{module.repo}",\n' + ")\n" + ) + blocks.append(block) return blocks diff --git a/scripts/tooling/BUILD b/scripts/tooling/BUILD index c2088414897..854bae80e41 100644 --- a/scripts/tooling/BUILD +++ b/scripts/tooling/BUILD @@ -81,6 +81,14 @@ py_binary( visibility = ["//visibility:public"], ) +# Alias: expose the resolve_deps script under //scripts/tooling so it can be +# invoked as `bazel run //scripts/tooling:resolve_deps` alongside other tooling scripts. +alias( + name = "resolve_deps", + actual = "//scripts/known_good:resolve_deps", + visibility = ["//visibility:public"], +) + # Tests target score_py_pytest( name = "tooling_tests", From fd7ecf5e469d1344fff35a99a2150b624a513cb6 Mon Sep 17 00:00:00 2001 From: subramaniak Date: Thu, 2 Jul 2026 12:19:11 +0530 Subject: [PATCH 4/6] fix: ruff format fix for scripts/tooling/BUILD --- scripts/known_good/models/module.py | 16 ++-- scripts/known_good/resolved_dependencies.py | 54 ++++++++------ .../tests/test_resolved_dependencies.py | 5 +- .../update_module_from_known_good.py | 16 ++-- scripts/tooling/BUILD | 74 ++++++++++--------- 5 files changed, 87 insertions(+), 78 deletions(-) diff --git a/scripts/known_good/models/module.py b/scripts/known_good/models/module.py index 73adfe5f9a4..7028fa988d1 100644 --- a/scripts/known_good/models/module.py +++ b/scripts/known_good/models/module.py @@ -32,13 +32,13 @@ class Metadata: """ code_root_path: str = "//score/..." - extra_test_config: list[str] = field(default_factory=lambda: []) - exclude_test_targets: list[str] = field(default_factory=lambda: []) + extra_test_config: list[str] = field(default_factory=list) + exclude_test_targets: list[str] = field(default_factory=list) langs: list[str] = field(default_factory=lambda: ["cpp", "rust"]) rust_coverage_config: str | None = "ferrocene-coverage" @classmethod - def from_dict(cls, data: Dict[str, Any]) -> Metadata: + def from_dict(cls, data: dict[str, Any]) -> Metadata: """Create a Metadata instance from a dictionary. Args: @@ -55,7 +55,7 @@ def from_dict(cls, data: Dict[str, Any]) -> Metadata: rust_coverage_config=data.get("rust_coverage_config", "ferrocene-coverage"), ) - def to_dict(self) -> Dict[str, Any]: + def to_dict(self) -> dict[str, Any]: """Convert Metadata instance to dictionary representation. Returns: @@ -82,7 +82,7 @@ class Module: pin_version: bool = False @classmethod - def from_dict(cls, name: str, module_data: Dict[str, Any]) -> Module: + def from_dict(cls, name: str, module_data: dict[str, Any]) -> Module: """Create a Module instance from a dictionary representation. Args: @@ -149,7 +149,7 @@ def from_dict(cls, name: str, module_data: Dict[str, Any]) -> Module: ) @classmethod - def parse_modules(cls, modules_dict: Dict[str, Any]) -> List[Module]: + def parse_modules(cls, modules_dict: dict[str, Any]) -> list[Module]: """Parse modules dictionary into Module dataclass instances. Args: @@ -190,13 +190,13 @@ def owner_repo(self) -> str: return f"{parts[0]}/{parts[1]}" - def to_dict(self) -> Dict[str, Any]: + def to_dict(self) -> dict[str, Any]: """Convert Module instance to dictionary representation for JSON output. Returns: Dictionary with module configuration """ - result: Dict[str, Any] = {"repo": self.repo} + result: dict[str, Any] = {"repo": self.repo} if self.version: result["version"] = self.version else: diff --git a/scripts/known_good/resolved_dependencies.py b/scripts/known_good/resolved_dependencies.py index d77f9dd3f42..543f845c1fc 100644 --- a/scripts/known_good/resolved_dependencies.py +++ b/scripts/known_good/resolved_dependencies.py @@ -31,19 +31,18 @@ import logging import re from pathlib import Path -from typing import Dict, List, Optional - -_HERE = Path(__file__).resolve().parent from known_good.models.known_good import load_known_good from known_good.models.module import Module +_HERE = Path(__file__).resolve().parent + # Marker delimiting the block we append, so injection is idempotent / detectable. INJECTION_BEGIN = "# --- BEGIN ref_int resolved-deps injection ---" INJECTION_END = "# --- END ref_int resolved-deps injection ---" -def generate_override_directive(module: Module, repo_commit_dict: Optional[Dict[str, str]] = None) -> Optional[str]: +def generate_override_directive(module: Module, repo_commit_dict: dict[str, str] | None = None) -> str | None: """Return the override directive (single_version_override / git_override) for a module. Returns just the override call without a preceding ``bazel_dep(...)`` line, so the @@ -80,7 +79,9 @@ def generate_override_directive(module: Module, repo_commit_dict: Optional[Dict[ if not module.repo or not commit: logging.warning( "Skipping module %s with missing repo or commit: repo=%s, commit=%s", - module.name, module.repo, commit, + module.name, + module.repo, + commit, ) return None @@ -98,6 +99,7 @@ def generate_override_directive(module: Module, repo_commit_dict: Optional[Dict[ ")\n" ) + # The single file that carries the resolved set from Stage 1 (resolve) to Stage 2 # (per-module validation). It is the only handoff needed: first-party commits + # third-party resolved versions, merged. The lock travels alongside only as evidence. @@ -121,23 +123,23 @@ class ResolvedDependencies: interface to scan + overwrite a module's ``MODULE.bazel`` to those versions. """ - def __init__(self, resolved: Dict[str, Module]): + def __init__(self, resolved: dict[str, Module]): self._resolved = resolved # -- construction: "resolved deps versions from ref_int root" -------------------- @classmethod - def from_known_good(cls, known_good_path: Path) -> "ResolvedDependencies": + def from_known_good(cls, known_good_path: Path) -> ResolvedDependencies: """Build from ``known_good.json`` (local / dev source of the resolved pins).""" kg = load_known_good(Path(known_good_path).resolve()) - resolved: Dict[str, Module] = {} + resolved: dict[str, Module] = {} for group in kg.modules.values(): for module in group.values(): resolved[module.name] = module return cls(resolved) @classmethod - def from_resolved_artifact(cls, artifact_dir: Path) -> "ResolvedDependencies": + def from_resolved_artifact(cls, artifact_dir: Path) -> ResolvedDependencies: """Build from the Stage-1 ``stage1-resolved-deps`` artifact. The handoff is the single ``resolved_versions.json`` manifest (see @@ -164,14 +166,14 @@ def from_resolved_artifact(cls, artifact_dir: Path) -> "ResolvedDependencies": if not module_files: raise FileNotFoundError(f"No score_modules_*.MODULE.bazel files in resolved-deps artifact {artifact_dir}.") - resolved: Dict[str, Module] = {} + resolved: dict[str, Module] = {} for mf in module_files: for module in cls._parse_override_file(mf.read_text()): resolved[module.name] = module return cls(resolved) @classmethod - def from_mod_graph(cls, mod_graph_json: Path, override_files: List[Path]) -> "ResolvedDependencies": + def from_mod_graph(cls, mod_graph_json: Path, override_files: list[Path]) -> ResolvedDependencies: """Build the *complete* resolved set by merging two sources. * The override directives ref_int actually declares — parsed from its root @@ -190,8 +192,8 @@ def from_mod_graph(cls, mod_graph_json: Path, override_files: List[Path]) -> "Re ``archive_override`` / ``local_path_override`` targets (e.g. ``rules_boost``) cannot be represented and are logged as not carried. """ - resolved: Dict[str, Module] = {} - unrepresentable: List[str] = [] + resolved: dict[str, Module] = {} + unrepresentable: list[str] = [] for f in override_files: # Drop comment-only lines first: hand-written MODULE.bazel files contain # commented-out overrides (e.g. "# git_override(... rules_rpm ...)") that must @@ -203,9 +205,9 @@ def from_mod_graph(cls, mod_graph_json: Path, override_files: List[Path]) -> "Re unrepresentable.append(f"{m.group(2)} ({m.group(1)})") graph = json.loads(Path(mod_graph_json).read_text()) - versions: Dict[str, str] = {} + versions: dict[str, str] = {} _collect_resolved_versions(graph, versions) - skipped: List[str] = [] + skipped: list[str] = [] for name, version in versions.items(): if name in resolved or name in _SKIP_MODULES: continue # already carried by an override directive, or non-overridable @@ -237,23 +239,23 @@ def to_file(self, path: Path) -> None: modules = {} for name in sorted(self._resolved): m = self._resolved[name] - entry: Dict[str, object] = {"version": m.version} if m.version else {"repo": m.repo, "hash": m.hash} + entry: dict[str, object] = {"version": m.version} if m.version else {"repo": m.repo, "hash": m.hash} if m.bazel_patches: entry["bazel_patches"] = m.bazel_patches modules[name] = entry Path(path).write_text(json.dumps({"modules": modules}, indent=2) + "\n") @classmethod - def from_file(cls, path: Path) -> "ResolvedDependencies": + def from_file(cls, path: Path) -> ResolvedDependencies: """Load a resolved set previously written by :meth:`to_file`.""" data = json.loads(Path(path).read_text()) resolved = {name: Module.from_dict(name, md) for name, md in data.get("modules", {}).items()} return cls(resolved) @staticmethod - def _parse_override_file(text: str) -> List[Module]: + def _parse_override_file(text: str) -> list[Module]: """Reconstruct Module objects from generated git/single_version override blocks.""" - modules: List[Module] = [] + modules: list[Module] = [] for match in _GIT_OVERRIDE_BLOCK_RE.finditer(text): body = match.group("body") @@ -278,17 +280,21 @@ def _parse_override_file(text: str) -> List[Module]: def names(self) -> set[str]: return set(self._resolved) - def get(self, name: str) -> Optional[Module]: + @property + def modules(self) -> dict[str, Module]: + return dict(self._resolved) + + def get(self, name: str) -> Module | None: return self._resolved.get(name) - def scan(self, module_bazel: Path) -> List[str]: + def scan(self, module_bazel: Path) -> list[str]: """Return the names of dependencies a module declares via ``bazel_dep``.""" text = Path(module_bazel).read_text() # Ignore anything inside a previous injection block so re-scans are stable. text = self._strip_injection(text) return _BAZEL_DEP_RE.findall(text) - def overwrite(self, module_bazel: Path, *, module_under_test: Optional[str] = None, write: bool = True) -> str: + def overwrite(self, module_bazel: Path, *, module_under_test: str | None = None, write: bool = True) -> str: """Overwrite a module's declared dependency versions with the resolved set. Appends a ``git_override`` / ``single_version_override`` directive for every @@ -306,7 +312,7 @@ def overwrite(self, module_bazel: Path, *, module_under_test: Optional[str] = No from dataclasses import replace as _replace - directives: List[str] = [] + directives: list[str] = [] # Inject overrides only for deps the module actually declares (intersected with the # resolved set). Bazel fails with "root module specifies overrides on nonexistent # module(s)" if an override targets a module that is not in this module's dependency @@ -352,7 +358,7 @@ def _field(body: str, field: str) -> str: return match.group(1) if match else "" -def _collect_resolved_versions(node: dict, acc: Dict[str, str]) -> None: +def _collect_resolved_versions(node: dict, acc: dict[str, str]) -> None: """Walk a ``bazel mod graph --output=json`` tree, recording name -> resolved version. Each node carries the post-MVS ``name`` and ``version``; a module can appear many diff --git a/scripts/known_good/tests/test_resolved_dependencies.py b/scripts/known_good/tests/test_resolved_dependencies.py index 511bb9fc2b2..436669abb39 100644 --- a/scripts/known_good/tests/test_resolved_dependencies.py +++ b/scripts/known_good/tests/test_resolved_dependencies.py @@ -133,7 +133,8 @@ def test_skips_resolved_dep_not_declared(self, resolved: ResolvedDependencies, t # resolved dep the module does not declare must NOT be injected. mod = tmp_path / "MODULE.bazel" mod.write_text( - 'module(name = "score_persistency", version = "0.0.0")\nbazel_dep(name = "score_baselibs", version = "0.1")\n' + 'module(name = "score_persistency", version = "0.0.0")\n' + 'bazel_dep(name = "score_baselibs", version = "0.1")\n' ) block = resolved.overwrite(mod, module_under_test="score_persistency", write=False).split(INJECTION_BEGIN)[1] assert 'module_name = "score_baselibs"' in block # declared -> injected @@ -263,7 +264,7 @@ def test_roundtrip_known_good_to_artifact(self, tmp_path: Path, resolved: Resolv art.mkdir() (art / "MODULE.bazel.lock").write_text("{}") blocks = [] - for m in resolved._resolved.values(): + for m in resolved.modules.values(): directive = generate_override_directive(m) if directive: blocks.append(f'bazel_dep(name = "{m.name}")\n' + directive) diff --git a/scripts/known_good/update_module_from_known_good.py b/scripts/known_good/update_module_from_known_good.py index 2d61ea2a805..b38b4045870 100755 --- a/scripts/known_good/update_module_from_known_good.py +++ b/scripts/known_good/update_module_from_known_good.py @@ -42,7 +42,7 @@ logging.basicConfig(level=logging.WARNING, format="%(levelname)s: %(message)s") -def generate_git_override_blocks(modules: List[Module], repo_commit_dict: Dict[str, str]) -> List[str]: +def generate_git_override_blocks(modules: list[Module], repo_commit_dict: dict[str, str]) -> list[str]: """Generate bazel_dep and git_override blocks for each module.""" blocks = [] @@ -109,7 +109,7 @@ def generate_git_override_blocks(modules: List[Module], repo_commit_dict: Dict[s return blocks -def generate_local_override_blocks(modules: List[Module]) -> List[str]: +def generate_local_override_blocks(modules: list[Module]) -> list[str]: """Generate bazel_dep and local_path_override blocks for each module.""" blocks = [] @@ -127,7 +127,7 @@ def generate_local_override_blocks(modules: List[Module]) -> List[str]: return blocks -def generate_coverage_blocks(modules: List[Module]) -> List[str]: +def generate_coverage_blocks(modules: list[Module]) -> list[str]: """Generate rust_coverage_report blocks for each module with rust impl.""" blocks = ["""load("@score_tooling//:defs.bzl", "rust_coverage_report")"""] @@ -161,9 +161,9 @@ def generate_coverage_blocks(modules: List[Module]) -> List[str]: def generate_file_content( args: argparse.Namespace, - modules: List[Module], - repo_commit_dict: Dict[str, str], - timestamp: Optional[str] = None, + modules: list[Module], + repo_commit_dict: dict[str, str], + timestamp: str | None = None, file_type: str = "module", ) -> str: """Generate the complete content for score_modules.MODULE.bazel.""" @@ -292,9 +292,9 @@ def main() -> None: try: known_good = load_known_good(Path(known_path)) except FileNotFoundError as e: - raise SystemExit(f"ERROR: {e}") + raise SystemExit(f"ERROR: {e}") from e except ValueError as e: - raise SystemExit(f"ERROR: {e}") + raise SystemExit(f"ERROR: {e}") from e if not known_good.modules: raise SystemExit("No modules found in known_good.json") diff --git a/scripts/tooling/BUILD b/scripts/tooling/BUILD index 854bae80e41..6e848c58f51 100644 --- a/scripts/tooling/BUILD +++ b/scripts/tooling/BUILD @@ -22,84 +22,86 @@ load("@score_tooling//python_basics:defs.bzl", "score_py_pytest") # `bazel run //scripts/tooling:requirements.update -- --upgrade` compile_pip_requirements( - name = "requirements", - srcs = [ + name="requirements", + srcs=[ "requirements.in", "@score_tooling//python_basics:requirements.txt", ], - extra_args = [ + extra_args=[ "--no-annotate", ], - requirements_txt = "requirements.txt", - tags = [ + requirements_txt="requirements.txt", + tags=[ "manual", ], ) # Library target py_library( - name = "lib", - srcs = glob(["lib/**/*.py"]), - visibility = ["//visibility:public"], + name="lib", + srcs=glob(["lib/**/*.py"]), + visibility=["//visibility:public"], ) # CLI library target (shared between binary and tests) py_library( - name = "cli", - srcs = glob(["cli/**/*.py"]), - data = [ + name="cli", + srcs=glob(["cli/**/*.py"]), + data=[ ":cli/misc/assets/report_template.html", ], - deps = [":lib"] + all_requirements, + deps=[":lib"] + all_requirements, ) # CLI binary target py_binary( - name = "tooling", - srcs = ["cli/main.py"], - main = "cli/main.py", - visibility = ["//visibility:public"], - deps = [":cli"], + name="tooling", + srcs=["cli/main.py"], + main="cli/main.py", + visibility=["//visibility:public"], + deps=[":cli"], ) # Workflow scripts as executables py_binary( - name = "checkout_repos", - srcs = ["cli/workflow/checkout_repos.py"], - main = "cli/workflow/checkout_repos.py", - visibility = ["//visibility:public"], - deps = [ + name="checkout_repos", + srcs=["cli/workflow/checkout_repos.py"], + main="cli/workflow/checkout_repos.py", + visibility=["//visibility:public"], + deps=[ ":cli", ":lib", - ] + all_requirements, + ] + + all_requirements, ) py_binary( - name = "recategorize_guidelines", - srcs = ["cli/workflow/recategorize_guidelines.py"], - main = "cli/workflow/recategorize_guidelines.py", - visibility = ["//visibility:public"], + name="recategorize_guidelines", + srcs=["cli/workflow/recategorize_guidelines.py"], + main="cli/workflow/recategorize_guidelines.py", + visibility=["//visibility:public"], ) # Alias: expose the resolve_deps script under //scripts/tooling so it can be # invoked as `bazel run //scripts/tooling:resolve_deps` alongside other tooling scripts. alias( - name = "resolve_deps", - actual = "//scripts/known_good:resolve_deps", - visibility = ["//visibility:public"], + name="resolve_deps", + actual="//scripts/known_good:resolve_deps", + visibility=["//visibility:public"], ) # Tests target score_py_pytest( - name = "tooling_tests", - srcs = glob(["tests/**/*.py"]), - data = [ + name="tooling_tests", + srcs=glob(["tests/**/*.py"]), + data=[ ":cli/misc/assets/report_template.html", "//:known_good.json", ], - pytest_config = "//:pyproject.toml", - deps = [ + pytest_config="//:pyproject.toml", + deps=[ ":cli", ":lib", - ] + all_requirements, + ] + + all_requirements, ) From eeca6716cd0c084dda3260aff7ec307f0636d064 Mon Sep 17 00:00:00 2001 From: subramaniak Date: Thu, 2 Jul 2026 07:10:38 +0000 Subject: [PATCH 5/6] fix: revert out-of-scope changes to module.py, update_module_from_known_good.py, and tooling BUILD format --- scripts/known_good/models/module.py | 18 ++--- .../update_module_from_known_good.py | 16 ++-- scripts/tooling/BUILD | 77 +++++++++---------- 3 files changed, 54 insertions(+), 57 deletions(-) diff --git a/scripts/known_good/models/module.py b/scripts/known_good/models/module.py index 7028fa988d1..72cae75c678 100644 --- a/scripts/known_good/models/module.py +++ b/scripts/known_good/models/module.py @@ -32,13 +32,13 @@ class Metadata: """ code_root_path: str = "//score/..." - extra_test_config: list[str] = field(default_factory=list) - exclude_test_targets: list[str] = field(default_factory=list) + extra_test_config: list[str] = field(default_factory=lambda: []) + exclude_test_targets: list[str] = field(default_factory=lambda: []) langs: list[str] = field(default_factory=lambda: ["cpp", "rust"]) - rust_coverage_config: str | None = "ferrocene-coverage" + rust_coverage_config: str | None = "ferrocene-coverage" # Optional field for Rust coverage configuration @classmethod - def from_dict(cls, data: dict[str, Any]) -> Metadata: + def from_dict(cls, data: Dict[str, Any]) -> Metadata: """Create a Metadata instance from a dictionary. Args: @@ -55,7 +55,7 @@ def from_dict(cls, data: dict[str, Any]) -> Metadata: rust_coverage_config=data.get("rust_coverage_config", "ferrocene-coverage"), ) - def to_dict(self) -> dict[str, Any]: + def to_dict(self) -> Dict[str, Any]: """Convert Metadata instance to dictionary representation. Returns: @@ -82,7 +82,7 @@ class Module: pin_version: bool = False @classmethod - def from_dict(cls, name: str, module_data: dict[str, Any]) -> Module: + def from_dict(cls, name: str, module_data: Dict[str, Any]) -> Module: """Create a Module instance from a dictionary representation. Args: @@ -149,7 +149,7 @@ def from_dict(cls, name: str, module_data: dict[str, Any]) -> Module: ) @classmethod - def parse_modules(cls, modules_dict: dict[str, Any]) -> list[Module]: + def parse_modules(cls, modules_dict: Dict[str, Any]) -> List[Module]: """Parse modules dictionary into Module dataclass instances. Args: @@ -190,13 +190,13 @@ def owner_repo(self) -> str: return f"{parts[0]}/{parts[1]}" - def to_dict(self) -> dict[str, Any]: + def to_dict(self) -> Dict[str, Any]: """Convert Module instance to dictionary representation for JSON output. Returns: Dictionary with module configuration """ - result: dict[str, Any] = {"repo": self.repo} + result: Dict[str, Any] = {"repo": self.repo} if self.version: result["version"] = self.version else: diff --git a/scripts/known_good/update_module_from_known_good.py b/scripts/known_good/update_module_from_known_good.py index b38b4045870..2d61ea2a805 100755 --- a/scripts/known_good/update_module_from_known_good.py +++ b/scripts/known_good/update_module_from_known_good.py @@ -42,7 +42,7 @@ logging.basicConfig(level=logging.WARNING, format="%(levelname)s: %(message)s") -def generate_git_override_blocks(modules: list[Module], repo_commit_dict: dict[str, str]) -> list[str]: +def generate_git_override_blocks(modules: List[Module], repo_commit_dict: Dict[str, str]) -> List[str]: """Generate bazel_dep and git_override blocks for each module.""" blocks = [] @@ -109,7 +109,7 @@ def generate_git_override_blocks(modules: list[Module], repo_commit_dict: dict[s return blocks -def generate_local_override_blocks(modules: list[Module]) -> list[str]: +def generate_local_override_blocks(modules: List[Module]) -> List[str]: """Generate bazel_dep and local_path_override blocks for each module.""" blocks = [] @@ -127,7 +127,7 @@ def generate_local_override_blocks(modules: list[Module]) -> list[str]: return blocks -def generate_coverage_blocks(modules: list[Module]) -> list[str]: +def generate_coverage_blocks(modules: List[Module]) -> List[str]: """Generate rust_coverage_report blocks for each module with rust impl.""" blocks = ["""load("@score_tooling//:defs.bzl", "rust_coverage_report")"""] @@ -161,9 +161,9 @@ def generate_coverage_blocks(modules: list[Module]) -> list[str]: def generate_file_content( args: argparse.Namespace, - modules: list[Module], - repo_commit_dict: dict[str, str], - timestamp: str | None = None, + modules: List[Module], + repo_commit_dict: Dict[str, str], + timestamp: Optional[str] = None, file_type: str = "module", ) -> str: """Generate the complete content for score_modules.MODULE.bazel.""" @@ -292,9 +292,9 @@ def main() -> None: try: known_good = load_known_good(Path(known_path)) except FileNotFoundError as e: - raise SystemExit(f"ERROR: {e}") from e + raise SystemExit(f"ERROR: {e}") except ValueError as e: - raise SystemExit(f"ERROR: {e}") from e + raise SystemExit(f"ERROR: {e}") if not known_good.modules: raise SystemExit("No modules found in known_good.json") diff --git a/scripts/tooling/BUILD b/scripts/tooling/BUILD index 6e848c58f51..d989b765db4 100644 --- a/scripts/tooling/BUILD +++ b/scripts/tooling/BUILD @@ -22,86 +22,83 @@ load("@score_tooling//python_basics:defs.bzl", "score_py_pytest") # `bazel run //scripts/tooling:requirements.update -- --upgrade` compile_pip_requirements( - name="requirements", - srcs=[ + name = "requirements", + srcs = [ "requirements.in", "@score_tooling//python_basics:requirements.txt", ], - extra_args=[ + extra_args = [ "--no-annotate", ], - requirements_txt="requirements.txt", - tags=[ + requirements_txt = "requirements.txt", + tags = [ "manual", ], ) # Library target py_library( - name="lib", - srcs=glob(["lib/**/*.py"]), - visibility=["//visibility:public"], + name = "lib", + srcs = glob(["lib/**/*.py"]), + visibility = ["//visibility:public"], ) # CLI library target (shared between binary and tests) py_library( - name="cli", - srcs=glob(["cli/**/*.py"]), - data=[ + name = "cli", + srcs = glob(["cli/**/*.py"]), + data = [ ":cli/misc/assets/report_template.html", ], - deps=[":lib"] + all_requirements, + deps = [":lib"] + all_requirements, ) # CLI binary target py_binary( - name="tooling", - srcs=["cli/main.py"], - main="cli/main.py", - visibility=["//visibility:public"], - deps=[":cli"], + name = "tooling", + srcs = ["cli/main.py"], + main = "cli/main.py", + visibility = ["//visibility:public"], + deps = [":cli"], ) # Workflow scripts as executables py_binary( - name="checkout_repos", - srcs=["cli/workflow/checkout_repos.py"], - main="cli/workflow/checkout_repos.py", - visibility=["//visibility:public"], - deps=[ + name = "checkout_repos", + srcs = ["cli/workflow/checkout_repos.py"], + main = "cli/workflow/checkout_repos.py", + visibility = ["//visibility:public"], + deps = [ ":cli", ":lib", - ] - + all_requirements, + ] + all_requirements, ) py_binary( - name="recategorize_guidelines", - srcs=["cli/workflow/recategorize_guidelines.py"], - main="cli/workflow/recategorize_guidelines.py", - visibility=["//visibility:public"], + name = "recategorize_guidelines", + srcs = ["cli/workflow/recategorize_guidelines.py"], + main = "cli/workflow/recategorize_guidelines.py", + visibility = ["//visibility:public"], ) -# Alias: expose the resolve_deps script under //scripts/tooling so it can be -# invoked as `bazel run //scripts/tooling:resolve_deps` alongside other tooling scripts. +# Alias: expose resolve_deps under //scripts/tooling for `bazel run //scripts/tooling:resolve_deps`. alias( - name="resolve_deps", - actual="//scripts/known_good:resolve_deps", - visibility=["//visibility:public"], + name = "resolve_deps", + actual = "//scripts/known_good:resolve_deps", + visibility = ["//visibility:public"], ) # Tests target score_py_pytest( - name="tooling_tests", - srcs=glob(["tests/**/*.py"]), - data=[ + name = "tooling_tests", + srcs = glob(["tests/**/*.py"]), + data = [ ":cli/misc/assets/report_template.html", "//:known_good.json", ], - pytest_config="//:pyproject.toml", - deps=[ + pytest_config = "//:pyproject.toml", + deps = [ ":cli", ":lib", - ] - + all_requirements, + ] + all_requirements, ) From fbc58a38f6d4023ec2bdb5315d6a8aea683ebb1e Mon Sep 17 00:00:00 2001 From: subramaniak Date: Fri, 3 Jul 2026 07:40:49 +0000 Subject: [PATCH 6/6] feat: warn on unresolved declared deps, add bazel test target for known_good tests --- scripts/known_good/BUILD | 11 +++++++++++ scripts/known_good/resolved_dependencies.py | 14 ++++++++++++-- .../known_good/tests/test_resolved_dependencies.py | 12 ++++++++++++ 3 files changed, 35 insertions(+), 2 deletions(-) diff --git a/scripts/known_good/BUILD b/scripts/known_good/BUILD index 647a539c997..12a21236bac 100644 --- a/scripts/known_good/BUILD +++ b/scripts/known_good/BUILD @@ -11,6 +11,7 @@ # SPDX-License-Identifier: Apache-2.0 # ******************************************************************************* load("@rules_python//python:defs.bzl", "py_binary", "py_library") +load("@score_tooling//python_basics:defs.bzl", "score_py_pytest") # Library target: the known_good package (models + generators). # Used as a dep by //scripts/tooling and by the resolve_deps binary below. @@ -23,6 +24,16 @@ py_library( visibility = ["//visibility:public"], ) +# Tests for the known_good package (currently: ResolvedDependencies). +# Not part of //scripts/tooling:tooling_tests, whose glob is scoped to scripts/tooling/tests/. +score_py_pytest( + name = "known_good_tests", + srcs = glob(["tests/**/*.py"]), + data = ["//:known_good.json"], + pytest_config = "//:pyproject.toml", + deps = [":known_good"], +) + # Runnable binary for the resolve + inject workflow. # Stage 1 (export): bazel run //scripts/known_good:resolve_deps -- \ # --mod-graph graph.json --export artifacts/resolved_versions.json diff --git a/scripts/known_good/resolved_dependencies.py b/scripts/known_good/resolved_dependencies.py index 543f845c1fc..05dc7801768 100644 --- a/scripts/known_good/resolved_dependencies.py +++ b/scripts/known_good/resolved_dependencies.py @@ -302,7 +302,11 @@ def overwrite(self, module_bazel: Path, *, module_under_test: str | None = None, module (and all its transitive deps) build against ref_int's resolved versions. * Skips the module under test itself (the root is never overridden). - * Skips dependencies that already carry an override in the file. + * Always overwrites: any existing override the module already declares is replaced. + * A declared dependency with no entry in the resolved set is expected not to occur + when the resolved set comes from ref_int's full ``bazel mod graph`` (it is a + superset of every module's own graph) — if it does happen, a warning is logged + and that dependency is left to resolve on its own rather than failing the run. * Re-running is idempotent: a prior injection block is replaced. """ module_bazel = Path(module_bazel) @@ -324,7 +328,13 @@ def overwrite(self, module_bazel: Path, *, module_under_test: str | None = None, continue # the module under test is the root; never override it module = self._resolved.get(name) if module is None: - continue # dep ref_int does not pin; resolves normally + logging.warning( + "%s declares %s, which has no entry in the resolved set; " + "leaving it to resolve on its own instead of failing the run.", + module_bazel, + name, + ) + continue # Strip bazel_patches: they reference //patches/... labels in ref_int's # workspace which do not exist inside another module's checkout. module = _replace(module, bazel_patches=None) diff --git a/scripts/known_good/tests/test_resolved_dependencies.py b/scripts/known_good/tests/test_resolved_dependencies.py index 436669abb39..344de47d462 100644 --- a/scripts/known_good/tests/test_resolved_dependencies.py +++ b/scripts/known_good/tests/test_resolved_dependencies.py @@ -17,6 +17,7 @@ """ import json +import logging import sys from pathlib import Path @@ -157,6 +158,17 @@ def test_idempotent(self, resolved: ResolvedDependencies, module_bazel: Path): assert first == second assert second.count(INJECTION_BEGIN) == 1 + def test_warns_on_declared_dep_not_in_resolved_set( + self, resolved: ResolvedDependencies, module_bazel: Path, caplog: pytest.LogCaptureFixture + ): + # "score_unpinned" is declared in MODULE_BAZEL but has no known_good.json entry. + # This is expected to be effectively impossible once the resolved set is sourced + # from the full 'bazel mod graph' (a superset of any module's own graph), so it + # must be surfaced as a warning rather than silently ignored. + with caplog.at_level(logging.WARNING): + resolved.overwrite(module_bazel, module_under_test="score_persistency", write=False) + assert "score_unpinned" in caplog.text + def test_overwrites_dep_with_existing_override(self, resolved: ResolvedDependencies, tmp_path: Path): # ref_int always decides the version — a pre-existing override in the module is replaced. mod = tmp_path / "MODULE.bazel"