From 34302073c2f114ce3520575b38fd5fb12780daae Mon Sep 17 00:00:00 2001 From: p3dr0rv Date: Thu, 30 Jul 2026 23:42:49 -0700 Subject: [PATCH 01/82] Add readiness checks and modify settings.gradle to comment out unused projects - Introduced `checks.py` in the `release-agent/tools` directory, implementing various readiness checks including Azure DevOps build definition access, HTTP reachability, and pipeline variable management. - Updated `settings.gradle` to comment out several project inclusions, including `AcaPlugin`, `LinuxBroker`, `java-linux-test-app`, `LinuxBrokerPackage`, and `NativeAuthSample`, to streamline the build configuration. --- .gitignore | 11 +- build.gradle | 2 +- release-agent/EXTERNAL-REFERENCES.md | 53 + release-agent/README.md | 230 +++ release-agent/config/phases.yaml | 130 ++ release-agent/config/preflight.yaml | 93 + release-agent/config/readiness.yaml | 129 ++ release-agent/config/requirements.yaml | 114 ++ release-agent/config/schedule.yaml | 20 + release-agent/orchestrator/__init__.py | 0 release-agent/orchestrator/cli.py | 53 + release-agent/orchestrator/cli_common.py | 147 ++ .../orchestrator/commands/__init__.py | 21 + .../orchestrator/commands/automation.py | 54 + .../orchestrator/commands/infra_cmd.py | 53 + .../orchestrator/commands/lockdown.py | 88 + release-agent/orchestrator/commands/logs.py | 64 + release-agent/orchestrator/commands/notice.py | 253 +++ release-agent/orchestrator/commands/notify.py | 126 ++ .../orchestrator/commands/pipeline.py | 105 ++ .../orchestrator/commands/readiness.py | 121 ++ .../orchestrator/commands/release.py | 295 ++++ release-agent/orchestrator/discovery.py | 72 + release-agent/orchestrator/engine.py | 695 ++++++++ release-agent/orchestrator/eventlog.py | 108 ++ release-agent/orchestrator/infra.py | 177 ++ release-agent/orchestrator/phase_config.py | 38 + release-agent/orchestrator/readiness.py | 201 +++ release-agent/orchestrator/registry.py | 77 + release-agent/orchestrator/render.py | 370 ++++ release-agent/orchestrator/schedule.py | 105 ++ release-agent/orchestrator/state.py | 112 ++ release-agent/phases/__init__.py | 0 release-agent/phases/agents/__init__.py | 38 + release-agent/phases/agents/preflight.py | 311 ++++ release-agent/phases/readiness_verifiers.py | 87 + release-agent/phases/stub_runner.py | 47 + release-agent/setup/bootstrap.ps1 | 86 + release-agent/skill/SKILL.md | 282 +++ .../templates/early-code-complete-notice.md | 50 + release-agent/tests/test_engine.py | 1563 +++++++++++++++++ release-agent/tools/__init__.py | 0 release-agent/tools/checks.py | 285 +++ settings.gradle | 20 +- 44 files changed, 6874 insertions(+), 12 deletions(-) create mode 100644 release-agent/EXTERNAL-REFERENCES.md create mode 100644 release-agent/README.md create mode 100644 release-agent/config/phases.yaml create mode 100644 release-agent/config/preflight.yaml create mode 100644 release-agent/config/readiness.yaml create mode 100644 release-agent/config/requirements.yaml create mode 100644 release-agent/config/schedule.yaml create mode 100644 release-agent/orchestrator/__init__.py create mode 100644 release-agent/orchestrator/cli.py create mode 100644 release-agent/orchestrator/cli_common.py create mode 100644 release-agent/orchestrator/commands/__init__.py create mode 100644 release-agent/orchestrator/commands/automation.py create mode 100644 release-agent/orchestrator/commands/infra_cmd.py create mode 100644 release-agent/orchestrator/commands/lockdown.py create mode 100644 release-agent/orchestrator/commands/logs.py create mode 100644 release-agent/orchestrator/commands/notice.py create mode 100644 release-agent/orchestrator/commands/notify.py create mode 100644 release-agent/orchestrator/commands/pipeline.py create mode 100644 release-agent/orchestrator/commands/readiness.py create mode 100644 release-agent/orchestrator/commands/release.py create mode 100644 release-agent/orchestrator/discovery.py create mode 100644 release-agent/orchestrator/engine.py create mode 100644 release-agent/orchestrator/eventlog.py create mode 100644 release-agent/orchestrator/infra.py create mode 100644 release-agent/orchestrator/phase_config.py create mode 100644 release-agent/orchestrator/readiness.py create mode 100644 release-agent/orchestrator/registry.py create mode 100644 release-agent/orchestrator/render.py create mode 100644 release-agent/orchestrator/schedule.py create mode 100644 release-agent/orchestrator/state.py create mode 100644 release-agent/phases/__init__.py create mode 100644 release-agent/phases/agents/__init__.py create mode 100644 release-agent/phases/agents/preflight.py create mode 100644 release-agent/phases/readiness_verifiers.py create mode 100644 release-agent/phases/stub_runner.py create mode 100644 release-agent/setup/bootstrap.ps1 create mode 100644 release-agent/skill/SKILL.md create mode 100644 release-agent/templates/early-code-complete-notice.md create mode 100644 release-agent/tests/test_engine.py create mode 100644 release-agent/tools/__init__.py create mode 100644 release-agent/tools/checks.py diff --git a/.gitignore b/.gitignore index 4618e843..2802f33f 100644 --- a/.gitignore +++ b/.gitignore @@ -59,4 +59,13 @@ ehthumbs_vista.db out/ -plugins/buildsystem/bin \ No newline at end of file +plugins/buildsystem/bin + +ICM-investigation/ +# Release Orchestrator generated run-state (per-release, ephemeral) +.release-runs/ + +# Python bytecode +__pycache__/ +*.pyc + diff --git a/build.gradle b/build.gradle index a3e5b735..90a3f6b3 100644 --- a/build.gradle +++ b/build.gradle @@ -67,7 +67,7 @@ buildscript { dependencies { classpath "com.android.tools.build:gradle:${rootProject.ext.gradleVersion}" classpath "org.javassist:javassist:${rootProject.ext.javaAssistVersion}" - classpath "com.microsoft.intune.mam:android-build-plugin:${rootProject.ext.intuneAppSdkVersion}" + // classpath "com.microsoft.intune.mam:android-build-plugin:${rootProject.ext.intuneAppSdkVersion}" classpath "org.jetbrains.kotlin:kotlin-gradle-plugin:${rootProject.ext.kotlinVersion}" // classpath "net.serenity-bdd:serenity-gradle-plugin:1.9.6" // classpath 'com.google.gms:google-services:3.2.1' diff --git a/release-agent/EXTERNAL-REFERENCES.md b/release-agent/EXTERNAL-REFERENCES.md new file mode 100644 index 00000000..f4b5b59a --- /dev/null +++ b/release-agent/EXTERNAL-REFERENCES.md @@ -0,0 +1,53 @@ +# External References + +Everything the Release Orchestrator depends on that lives **outside this codebase**. +If any of these change (URL moved, DL renamed, pipeline re-IDed, template edited, +access revoked), the orchestrator can silently break — so they're catalogued here. +Review this list when something stops working or when onboarding a new release owner. + +Legend for **Access**: `anon` = no auth · `az` = Azure CLI signed-in user · +`AAD-SSO` = browser Microsoft sign-in · `MCP` = via an MCP server · `Google` = Google account (not automatable in Scout). + +## Systems of record (read/write) + +| Ref | What | Used by | Access | Notes | +|---|---|---|---|---| +| ADO pipeline **3038** | "Code Complete Calendar Checker" — CCD source of record | CCD seed, `set-ccd`, `skip-release`, Phase-0 `cron` (verify scheduled) | az | org identitydivision / project Engineering. Real writes gated by --confirm. `cron` step verifies a recent `schedule`-reason run. | +| ADO build def **2828** | Auth Client Android build (org identitydivision / project Engineering) | readiness `build_access` | az | access check only | +| ADO build def **397224** | Android Build Release (org msazure / project One) | readiness `build_access` | az | access check only | +| ADO wiki **IdentityWiki.wiki** page **59148** | "Monthly Releases Payloads History" (parent) | Phase-0 `wiki` agent | az (`az devops wiki`) | child page ` Release`; dup-safe numbering | +| **ICM team 78848** | "Auth Client Android Shield" on-call roster | readiness `oncall_now` | MCP (ICM) | primary = index 0 of currentOnCallContacts | +| **ADX cluster** idsharedeus2.eastus2 / db d496be22d62a46b0a3cf67ea2e736fd8 | release telemetry | readiness `adx_access` | MCP (Kusto) | `print 1` access probe | + +## External web pages (scraped / linked) + +| Ref | URL | Used by | Access | Notes | +|---|---|---|---|---| +| CCOA No-Fly Zones | https://prod.change-manager.msidentity.com/ccoa-periods | Phase-0 `lockdown` | AAD-SSO | scraped via browser; only Production-env periods block | +| **Component Governance alerts** (governed repo **104410** = AD-MFA-phonefactor-phoneApp-android, branch `working`) | https://msazure.governance.visualstudio.com/{One projId}/_apis/ComponentGovernance/GovernedRepositories/104410/Branches/working/Alerts | Phase-0 `cg` | az (`az rest`) | read-only report; active alerts by severity. projId=b32aa71e-…, resource=499b84ac-… | +| **`release` variable group 40** | https://identitydivision.visualstudio.com/Engineering/_library?...variableGroupId=40&path=release | Phase-0 `flight_reminder` (link only) | az/web | feature owners update local flights here — release engineer does NOT | +| Flight pre-mortem example doc | https://microsoft-my.sharepoint-df.com/:w:/p/rapong/cQpEZp0cXp1sQYo4A4M3PQWCEgUCDj364FJa-rq-msg59WlBsw | Phase-0 `flight_reminder` (link only) | AAD-SSO | example shared with feature owners | +| Localization instructions | https://eng.ms/docs/.../combined-release-checklist/localization | Phase-0 `flight_reminder` (link only) | AAD-SSO | confirmed valid 2026-07-29 | +| **Teams chat: "Android Core Team"** | thread `19:976a859f167f44e59c4ceca8b1d23581@thread.v2` | Phase-0 `flight_reminder` target | MCP (WorkIQ) | LIVE target; dry-run posts to the owner's own chat | +| **EcsFlight.kt** (Auth App ECS flights) | https://msazure.visualstudio.com/One/_git/AD-MFA-phonefactor-phoneApp-android?path=/.../ecs/entities/EcsFlight.kt&version=GBworking | Phase-0 `flight_reminder` bullet 4 (link only) | az/web | reviewers check its history since last code complete | +| Early code-complete notice template | https://eng.ms/docs/.../combined-release-checklist/early-code-complete-notice-email-template | Phase-0 `notice` | AAD-SSO | copied locally to `templates/early-code-complete-notice.md` — **re-sync if upstream edits** | +| Hotfix cherry-pick guide | https://eng.ms/docs/.../release/cherry-pick-to-hotfix-guidelines | link inside notice email body | AAD-SSO | referenced, not fetched | +| common-for-android changelog | https://raw.githubusercontent.com/AzureAD/microsoft-authentication-library-common-for-android/dev/changelog.txt | Phase-0 `breaking` | anon | breaking = `[MAJOR]` in `vNext` | +| Play Console vitals | (Google Play Console) | Phase-0 `vitals` (#8) | Google | **NOT automatable in Scout** (Google auth wall) | + +## Outbound email recipients (LIVE runs only) + +> In **dry-run**, every outbound email goes to the **release owner** instead of these. +> Real recipients are used only on a `--live` release. + +| Step | To | Notes | +|---|---|---| +| Phase-0 `notice` (early code-complete) | androididentity@microsoft.com ("Azure Identity Android SDK"), jialh@microsoft.com | provided by release owner 2026-07-29 | + +## Tooling / infra (provisioned by bootstrap) + +| Ref | What | Notes | +|---|---|---| +| Agency CLI | provides the **ICM** and **Kusto** MCP servers | `agency mcp icm` / `agency mcp kusto`; auto-registered into `~/.scout/m-mcp-servers.json` by `cli infra` | +| Azure CLI (`az`) + `azure-devops` extension | pipeline + wiki reads/writes | signed-in user is the release owner | +| Scout | host for the skill + automations | `~/.scout`; bootstrap checks presence | diff --git a/release-agent/README.md b/release-agent/README.md new file mode 100644 index 00000000..be2932ad --- /dev/null +++ b/release-agent/README.md @@ -0,0 +1,230 @@ +# Release Orchestrator (`/release-agent`) + +The **conductor backbone** for the Android monthly release (ADO items **X4** + **X5**). +It drives the whole release as a state-aware smart checklist: it knows the phases, +runs each step's agent, and **holds at gates** for the release engineer to decide. + +> **Build status.** **Phase 0 (pre-flight) has real, tested agents** (early-notice, +> flight/string reminders, BREAKING-OneAuth detection, CG alerts, cron verify, wiki +> payload); the later phases are still **stubs** (mock actions) and get filled in one +> at a time (see the Release-Stabilization roadmap). Building on this shared backbone — +> not 50 one-off scripts — is what makes the "agent carries the knowledge" model real. + +## Architecture (thin skill over a deterministic engine) + +``` + you ──/release-agent──▶ SKILL (conversation layer) ──shell──▶ ENGINE (Python, deterministic) + presents gate briefs, state machine + dispatch + run-state + relays your approve/deny the BRAIN — fully unit-tested +``` + +- **Engine = the brain.** Decides what's next, runs stubbed agents, holds at gates, persists run-state. No LLM — unit-tested and dry-run replayable. +- **Skill = the mouth & ears.** Presents the gate, collects your decision, relays it. Never decides the flow. + +## Layout + +``` +release-agent/ COMMITTED (distributed with android-complete) +├─ config/ +│ ├─ phases.yaml the state machine (phases → steps → gates + CCD anchors), as data +│ ├─ preflight.yaml Phase-0 config (config/.yaml convention; loaded by phase id) +│ ├─ readiness.yaml the entry-gate checklist, as data +│ ├─ schedule.yaml where CCD comes from (pipeline 3038 coords), as data +│ └─ requirements.yaml external dependencies (CLIs, extensions, MCP servers) — single source of truth +├─ orchestrator/ three layers: logic → data → presentation +│ ├─ engine.py the conductor: state machine + dispatch + gates + time-anchoring + status_report +│ ├─ readiness.py ReadinessGate: entry-gate logic (verify/sign/decline) → structured data +│ ├─ schedule.py CCD math (2nd Wednesday, override, phase anchors) — pure, no IO +│ ├─ phase_config.py per-phase config loader (config/.yaml, by phase id) +│ ├─ infra.py infra preflight: check CLIs + register/verify MCP servers into Scout config +│ ├─ render.py presentation only: structured data → text/markdown (swap for other UIs) +│ ├─ state.py Release State Record / run-state (X5) +│ ├─ discovery.py find releases (none / one / many) +│ ├─ registry.py automation registry (track provisioned automations for teardown) +│ ├─ eventlog.py per-release interaction + event log +│ ├─ cli.py thin entry point: builds the parser from commands/, dispatches +│ ├─ cli_common.py shared CLI plumbing (load state, emit, event log, advance block) +│ └─ commands/ one module per command domain (self-registering subparsers) +│ ├─ release.py lifecycle + overrides: init/list/status/next/approve/deny/done/skip/reopen/halt/resume/activate +│ ├─ readiness.py entry gate: checklist/verify/sign/decline +│ ├─ pipeline.py real pipeline writes (gated): set-ccd / skip-release +│ ├─ notify.py daily phase digest (tick advances + reports; notify = read-only) + set-owner +│ ├─ lockdown.py CCOA overlap check: check-lockdown +│ ├─ notice.py Phase-0 scout steps: prepare-notice / prepare-flight-reminder / record-step +│ ├─ logs.py event log: log / journal +│ ├─ automation.py automation registry command +│ └─ infra_cmd.py infra preflight command +├─ phases/ +│ ├─ stub_runner.py mock phase agents (replaced one at a time) +│ ├─ readiness_verifiers.py auto verifiers for the entry gate (pass/fail) +│ └─ agents/ real phase agents — one module per phase, merged into one registry +│ ├─ __init__.py aggregator: merges every phase's REGISTRY (dup-id guarded) +│ └─ preflight.py Phase-0 agents (breaking · wiki · cg · cron) +├─ tools/checks.py real IO (az / http), isolated +├─ skill/SKILL.md the /release-agent Scout skill +├─ setup/bootstrap.ps1 one-time setup (infra preflight, installs skill) +└─ tests/test_engine.py unit + full dry-run-replay tests + +.release-runs// GENERATED, gitignored (per-release working state) +├─ release-state.json the per-release metadata + run-state (owner, CCD, steps, gates, …) +└─ events.jsonl the per-release event/interaction log +.release-runs/_automations.json GENERATED, gitignored — registry of provisioned Scout automations +``` + +**Two homes for data (by lifetime):** +- **Release metadata + run-state** → `.release-runs//release-state.json` (per-release; the `ReleaseState` record). Holds `owner_email`/`owner_name` (the release owner, resolved from the signed-in `az` user at `init`; reminders email this person), `ccd`/`ccd_source`/`ccd_conflict`, `dry_run`, step completion, gate decisions, `last_notified`, etc. Add release-scoped fields here. +- **Tool config** → `release-agent/config/*.yaml` (not release-specific; committed): `phases.yaml`, `readiness.yaml`, `schedule.yaml`, `requirements.yaml`. + +## Architecture — three layers (so it adapts to other interfaces) + +1. **Logic** (`engine.py`, `readiness.py`, `schedule.py`, `state.py`) — pure, deterministic, returns **structured data**. No formatting, no IO. +2. **Presentation** (`render.py`) — pure functions: structured data → text/markdown. A different interface (web UI, TUI) swaps this layer and reuses everything else. +3. **Interface** (`cli.py` + `cli_common.py` + `commands/` + `skill/SKILL.md`) — the CLI is a thin assembler: `cli.py` builds the parser from the self-registering modules in `commands/` (one per domain), and shared plumbing lives in `cli_common.py`. Adding a command is a localized change to one module. + +IO lives in `tools/` and `phases/` (pluggable). Config is data in `config/`. + + +## Run-state: two kinds (the X5 idea) + +- **Derived** — recomputed from systems of record (ADO/Git/Play Console/ADX). Never stored ⇒ never stale. *(reconcilers are stubbed for now.)* +- **Persisted** — decisions/intent, step completion, pending human actions. Stored in `release-state.json`. + +The conductor is **stateless**: on each invocation it loads the record, (later) reconciles against live systems, decides, acts, writes back. That's what lets a release resume across days/sessions. + +## Quick start + +```powershell +# one-time +pwsh ./setup/bootstrap.ps1 +``` + +`bootstrap.ps1` runs an **infrastructure preflight** first (`python -m orchestrator.cli infra`), +driven by **`config/requirements.yaml`** (the single source of truth for external +dependencies). It checks each CLI/host dependency and prints an `install:` hint for +anything missing, then **registers any required MCP servers into Scout's config** +(backing the file up first) and tells you to **restart Scout** so they load. Keep +`requirements.yaml` up to date whenever a new dependency (CLI, package, or MCP +server) is introduced. + +You can run the preflight any time on its own: + +```powershell +python -m orchestrator.cli infra # check + auto-register MCP servers (restart Scout after) +python -m orchestrator.cli infra --no-register # report only +``` + +```powershell +# drive a release (dry-run by default) +cd release-agent +python -m orchestrator.cli init --release 2026-07 +python -m orchestrator.cli next --release 2026-07 # runs until the first gate +python -m orchestrator.cli approve --release 2026-07 --comment "flags reviewed" +python -m orchestrator.cli status --release 2026-07 +``` + +Or in Scout: **`/release-agent`**. + +## Time anchoring — phases open relative to the Code Complete Date (CCD) + +Phases don't fire on demand; they're anchored to the **CCD**. **The CCD is +canonically the 2nd Wednesday of the month.** The orchestrator still reads ADO +pipeline **3038 "Code Complete Calendar Checker"**, but it does **not** silently +adopt the pipeline's `overrideCodeCompleteDate`: if that override is a *different* +in-month date, the tool flags a **conflict** (`ccd_conflict`) and asks the user +which date is real — the default or the pipeline's. `init` computes the default +and reports any conflict. + +- **Phase 0 opens at `CCD-7`** (declared as `anchor: "CCD-7"` on the phase in + `phases.yaml`). Before then the release is **`scheduled`** — the engine runs + nothing and status shows *"opens `` (in N days)"*. Other phases are + dependency-driven for now; add an `anchor:` to any phase to time-gate it too. +- **Simulated clock:** every read/advance command takes `--as-of YYYY-MM-DD` so a + dry-run can jump to CCD-7 and prove a phase opens on schedule. Real runs use today. +- **Resolving a conflict / changing the CCD.** `set-ccd` and `skip-release` + **write back** to pipeline 3038 (override / `skipRelease`) — real production + changes, so they're gated: preview first, then re-run with `--confirm` (a + `--reason` is always required and audited). Pick the default → `set-ccd --default` + clears the pipeline override so they match; pick the pipeline date → `set-ccd + --date `. `status` re-reads the pipeline and re-flags any new conflict. + +```powershell +python -m orchestrator.cli set-ccd --release 2026-07 --date 2026-07-15 --reason "more bake time" # preview +python -m orchestrator.cli set-ccd --release 2026-07 --date 2026-07-15 --reason "more bake time" --confirm +python -m orchestrator.cli status --release 2026-07 --as-of 2026-07-01 # jump the clock in dry-run +python -m orchestrator.cli done --release 2026-07 --note "China upload complete" # clear a reminder hold +``` + +## Push reminders — daily phase digest (reaching you when Scout is closed) + +Everything the engine surfaces is **pull** — you see it when you open Scout. The +**push** layer is a **daily phase status digest** emailed to the release owner: + +- **Setup is interactive → no push.** Readiness + establishing the CCD happen in + Scout, so they're never emailed (unsigned / blocked / halted = silent). +- **First push = a phase opening** (Phase 0 at CCD‑7). Nothing before it. +- **Daily while a phase is open with outstanding work** — once/day, progress + + what still needs you, until the phase's actions are done; then the next phase's + digest takes over when it opens (each phase notifies on open). + +```powershell +python -m orchestrator.cli tick --json # advance to today + {message,subject,owner_email,...} +python -m orchestrator.cli tick --as-of 2026-08-06 # simulate a date (debug) +python -m orchestrator.cli notify --json # read-only: report WITHOUT advancing (manual check) +``` + +A **Scout automation** runs **`tick --json` hourly** and, when `message` is non‑empty, +emails it to `owner_email` (subject from the JSON). `tick` both **advances** the release +to the current date and reports; running hourly means a tick missed while the machine +was off is picked up by the next one, and a once-per-calendar-day guard +(`last_notified_date`) keeps it to one advance-effect and one email per day. `notify` is +the **read-only** variant (report without advancing); `--as-of`/`--force` are debug overrides. + +**Automation registry.** Every automation the orchestrator provisions is recorded in +`.release-runs/_automations.json` (via `cli automation register`) so it can be torn +down cleanly. Automations are **per-release** by default (`--release `), created +at start and removed at that release's close (`automation list --release ` → +delete each → `automation deregister`). Push reminders are per-release too. A +`--shared` scope exists for the rare automation meant to outlive every release. + +## Two kinds of human step + +- **Gate** (`gate: true`) — a *decision*: the conductor holds and you `approve`/`deny`. +- **Reminder** (`owner: human`, no gate) — a *to-do*: the conductor holds + ("ACTION NEEDED"), you go do it, then `done` it. Not a decision — just done / not-yet. + +## Event log (for analysis & improvement) + +Every action is recorded to an append-only JSONL event log so we can improve the +process across engineers and months. The highest-value signal is the **decision +driver** — the reason attached to each gate approve/deny/decline. + +- Per-release trace: `.release-runs//events.jsonl` (one log per release; there is no machine-wide aggregate). + +```powershell +python -m orchestrator.cli log --release 2026-07 # this release's trace +python -m orchestrator.cli log --release 2026-07 --analyze # rolled-up summary +``` + +Events captured include: `release_started`, `readiness_verified/signed/declined`, +`step_ran`, `gate_hold` + `gate_approved`/`gate_denied` (with `driver`), +`reminder_hold`/`reminder_done`, `scheduled_hold`, `ccd_changed`, +`release_skip_set`/`release_skip_cleared`, `step_skipped`/`step_reopened`, +`release_halted`/`release_resumed`, `release_complete`, plus interaction events +(what Scout showed / what the user chose). Logging never breaks the flow (best-effort). + +> The log lives under the gitignored `.release-runs/`, so it's per-machine. Shipping +> logs to a shared store (Kusto/ADO/wiki) for cross-engineer analysis is a future step. + +## Tests + +```powershell +cd release-agent +python tests/test_engine.py # unit + full dry-run replay + readiness + eventlog +``` + +## Design constraints honored (from §7.1 of the stabilization plan) +1. Dry-run/replay is the primary test method (never test on live monthly releases). +2. Run-state schema defined once, upfront (X5), shared by all agents. +3. Sequence by risk/value — agents are independent plug-ins on the backbone. +4. Manual overrides are first-class (approve/deny gates; activate conditional phases). +5. Conductor is stateless; minimize persisted state, derive the rest. diff --git a/release-agent/config/phases.yaml b/release-agent/config/phases.yaml new file mode 100644 index 00000000..7a551a30 --- /dev/null +++ b/release-agent/config/phases.yaml @@ -0,0 +1,130 @@ +# Release Orchestrator — state machine definition (data, not code). +# The conductor reads this to know phases -> steps -> gates -> transitions. +# Phase 0 (preflight) has REAL agents (see phases/agents/preflight.py); later +# phases are still `agent: stub` and get a real agent id as each is implemented. +# +# Field reference: +# phases[].id/name : phase identifier + label +# phases[].checklist_phase : maps to the human checklist Phase number +# phases[].execution : sequential (default) | parallel (independent steps) +# steps[].id/name : step identifier + label +# steps[].agent : which phase-agent handles it ('stub' until built) +# steps[].gate : if true, conductor HOLDS at this step for human approval +# steps[].owner : agent | human (who acts; gates are always human-decided) +# steps[].source : scout -> the skill runs it via MCP + record-step +# steps[].attest : if true, a human confirms it (owner: human) +# steps[].depends_on : step ids in this phase that must finish first +# steps[].maps_to : the Release-Stabilization action-item ID(s) this step uses + +version: 1 + +phases: + - id: preflight + name: "Pre-flight & Code Complete" + checklist_phase: 0 + anchor: "CCD-7" # opens 7 days before the Code Complete Date (start window) + execution: parallel # steps are independent; a hold on one doesn't block the others + steps: + - { id: notice, name: "Send early release notice", agent: stub, owner: agent, source: scout, maps_to: [S0] } + - { id: flight_reminder, name: "Send feature-owner reminders (flights · strings · flag-freeze)", agent: stub, owner: agent, source: scout, maps_to: [S3, S4, S5] } + - { id: confirm_reminders, name: "Confirm feature owners completed flight / string / flag-freeze work", agent: stub, owner: human, attest: true, depends_on: [flight_reminder], maps_to: [S3, S4, S5, S6] } + - { id: lockdown, name: "Detect lockdown/holiday overlap", agent: stub, owner: agent, source: scout, maps_to: [S1] } + - { id: breaking, name: "Detect BREAKING-OneAuth + draft comms", agent: breaking_detect, owner: agent, maps_to: [S2] } + - { id: cg, name: "Report critical CG alerts", agent: cg_alerts, owner: agent, maps_to: [S8] } + - { id: vitals, name: "Confirm Play Console vitals & policy status reviewed", agent: stub, owner: human, attest: true, maps_to: [S9] } + - { id: cron, name: "Verify Calendar Checker scheduled", agent: cron_check, owner: agent, maps_to: [S10] } + - { id: wiki, name: "Create release payload wiki subpage", agent: wiki_payload, owner: agent, maps_to: [S0] } + + - id: ccd + name: "Code Complete Day" + checklist_phase: 1 + steps: + - { id: final_reminder, name: "CCD final code-complete reminder", agent: stub, owner: agent, maps_to: [P1-1] } + - { id: localization, name: "Trigger localization pipeline (PR)", agent: stub, owner: agent, maps_to: [P1-2] } + - { id: precheck_prs, name: "Pre-check open required PRs", agent: stub, owner: agent, maps_to: [P1-3a] } + - { id: branch_cut, name: "Cut the release branch", agent: stub, owner: human, gate: true, maps_to: [P1-3b] } + - { id: verify_trigger, name: "Verify orchestrator fired overnight", agent: stub, owner: agent, maps_to: [P1-4] } + + - id: build_verify + name: "Build & Lib Verification" + checklist_phase: 2 + steps: + - { id: stages_ok, name: "Verify pipeline completed expected stages", agent: stub, owner: agent, maps_to: [B1] } + - { id: retain, name: "Retain the build", agent: stub, owner: agent, maps_to: [B1r] } + - { id: health, name: "Assess build-completeness + product health", agent: stub, owner: agent, maps_to: [B2] } + - { id: ui_auto, name: "Assess UI automation results", agent: stub, owner: agent, maps_to: [B3] } + - { id: payload, name: "Write built versions into payload wiki", agent: stub, owner: agent, maps_to: [B6] } + - { id: mrwp_rc, name: "Point MRWP at cut RC Auth App branch", agent: stub, owner: agent, maps_to: [B4] } + - { id: go_test, name: "Proceed to bug bash", agent: stub, owner: human, gate: true } + + - id: bug_bash + name: "Test / Bug Bash" + checklist_phase: 3 + steps: + - { id: clone_plans, name: "Clone/rename test plans", agent: stub, owner: agent, maps_to: [T3] } + - { id: coordinate, name: "Bug Bash coordinator (invite/monitor/aggregate)", agent: stub, owner: agent, maps_to: [T2] } + - { id: ui_failures, name: "Surface Phase 2 UI failure list", agent: stub, owner: human, maps_to: [T4] } + - { id: signoffs, name: "Chase DID/Dublin sign-offs + telemetry", agent: stub, owner: agent, maps_to: [T5] } + - { id: bash_done, name: "Bug bash complete + signed off", agent: stub, owner: human, gate: true } + + - id: finalize + name: "Finalize & Publish" + checklist_phase: 4 + steps: + - { id: gate_watch, name: "Watch orchestrator gates (1-click approve)", agent: stub, owner: human, gate: true, maps_to: [F1] } + - { id: integ_prs, name: "Auto-create integration PRs", agent: stub, owner: agent, maps_to: [F2] } + - { id: verify_pub, name: "Verify Maven Central + GitHub publication", agent: stub, owner: agent, maps_to: [F3] } + - { id: final_comms, name: "Send final broker email + Teams", agent: stub, owner: agent, maps_to: [F4] } + - { id: nonrc_pin, name: "Final non-RC pin PR + kick + retain", agent: stub, owner: agent, maps_to: [F5] } + - { id: tag, name: "Tag the release commit", agent: stub, owner: agent, maps_to: [F6] } + - { id: backmerge, name: "Back-merge to working", agent: stub, owner: agent, maps_to: [F7] } + + - id: rollout_start + name: "Rollout Start" + checklist_phase: 5 + steps: + - { id: notice, name: "Send initial Auth App release notice", agent: stub, owner: agent, maps_to: [R1] } + - { id: artifact, name: "Verify final non-RC artifact is used", agent: stub, owner: agent, maps_to: [R2] } + - { id: signoff_start, name: "Start Release Sign Off (1-click)", agent: stub, owner: human, gate: true, maps_to: [R3] } + + - id: monitor + name: "Monitoring & Ring Advancement" + checklist_phase: 6 + steps: + - { id: health_report, name: "Crash-% health report (halt/hotfix/proceed)", agent: stub, owner: agent, maps_to: [M1] } + - { id: adoption, name: "Adoption % vs ring threshold", agent: stub, owner: agent, maps_to: [M2] } + - { id: guards, name: "Check advancement guard rules", agent: stub, owner: agent, maps_to: [M4] } + - { id: advance, name: "Ring advance / halt decision", agent: stub, owner: human, gate: true, maps_to: [M3] } + - { id: progress_email, name: "Send progression email on advance", agent: stub, owner: agent, maps_to: [M5] } + - { id: dashboard, name: "Update ADX dashboard params", agent: stub, owner: agent, maps_to: [M6] } + - { id: crash_annot, name: "Flag new vs existing crashes", agent: stub, owner: human, maps_to: [M7] } + - { id: hotfix_eval, name: "Surface hotfix-trigger criteria", agent: stub, owner: human, maps_to: [M8] } + + - id: partner + name: "Partner Stores" + checklist_phase: 7 + steps: + - { id: china, name: "China publish flow", agent: stub, owner: human, maps_to: [PT1] } + - { id: samsung, name: "Samsung portal upload", agent: stub, owner: human, maps_to: [PT1s] } + - { id: ngms, name: "Verify nGMS upload", agent: stub, owner: agent, maps_to: [PT2] } + - { id: teams_dev, name: "Teams Devices announcement", agent: stub, owner: agent, maps_to: [PT3] } + - { id: approvals, name: "Track partner approvals + final notice", agent: stub, owner: agent, maps_to: [PT4] } + - { id: review_times, name: "Maintain review-times table", agent: stub, owner: agent, maps_to: [PT5] } + + - id: hotfix + name: "Hotfix (conditional)" + checklist_phase: 8 + conditional: true + steps: + - { id: cherry, name: "Cherry-pick PRs", agent: stub, owner: agent, maps_to: [H1] } + - { id: rebuild, name: "Rebuild + retain hotfix build", agent: stub, owner: agent, maps_to: [H2] } + - { id: smoke, name: "P0 smoke subset", agent: stub, owner: human, gate: true, maps_to: [H3] } + - { id: expedite, name: "Expedited rollout", agent: stub, owner: agent, maps_to: [H4] } + - { id: alpha_orgs, name: "Track alpha-mitigation orgs to remove", agent: stub, owner: agent, maps_to: [H5] } + + - id: close + name: "Release Close" + checklist_phase: 9 + steps: + - { id: closeout, name: "Close-out checklist (approvals/comms/bugs)", agent: stub, owner: agent, maps_to: [C1] } + - { id: done, name: "Release complete", agent: stub, owner: human, gate: true } diff --git a/release-agent/config/preflight.yaml b/release-agent/config/preflight.yaml new file mode 100644 index 00000000..3f1ba65f --- /dev/null +++ b/release-agent/config/preflight.yaml @@ -0,0 +1,93 @@ +# Config for the REAL Phase-0 pre-flight agents (deterministic; az CLI / HTTP). +# +# Unlike the readiness scout-assisted checks (ICM/Kusto, which have no CLI and are +# run by the skill via MCP), these steps talk to systems that DO have a CLI/HTTP +# surface, so they are plain Python agents — testable and LLM-free. In a dry-run +# release they SIMULATE (no network, no writes); only a real release acts. + +version: 1 + +# Step `cg` (maps_to S8): "Report critical CG alerts". +# Component Governance alerts for the governed repo, read (read-only) from the CG +# governance host via `az rest`. Reports ACTIVE alerts grouped by severity and +# surfaces High/Critical. Report-only — never blocks the release. +cg: + resource: "499b84ac-1321-427f-aa17-267ca6975798" # Azure DevOps resource id (for `az rest`) + governance_host: "https://msazure.governance.visualstudio.com" + project_id: "b32aa71e-8ed2-41b2-9d77-5bc261222004" # msazure/One + governed_repo_id: 104410 # AD-MFA-phonefactor-phoneApp-android + branch: "working" + high_severities: ["critical", "high"] # surfaced/flagged as high-priority + +# Step `cron` (maps_to S10): "Verify Calendar Checker scheduled". +# Pipeline 3038's cron lives in YAML (not exposed in definition triggers), but its +# build history proves the schedule is FIRING: a recent `schedule`-reason run means +# the Calendar Checker is live. Blocks if there's no recent scheduled run (stale). +cron: + pipeline_id: 3038 + org: "https://identitydivision.visualstudio.com" + project: "Engineering" + name: "Code Complete Calendar Checker" + max_staleness_days: 2 # a daily cron should never be older than this + + +# Step `breaking` (maps_to S2): "Detect BREAKING-OneAuth + draft comms". +# common-for-android records breaking changes as [MAJOR] entries in changelog.txt. +# The unreleased "vNext" section holds the changes shipping in THIS release, so we +# scan only that section. Any [MAJOR] there is a breaking change OneAuth must hear +# about — the agent lists them and drafts the comms; a human sends it. +breaking: + changelog_url: "https://raw.githubusercontent.com/AzureAD/microsoft-authentication-library-common-for-android/dev/changelog.txt" + section: "vNext" # scan the unreleased section only + breaking_tag: "[MAJOR]" # breaking-change marker convention in this changelog + +# Step `wiki` (maps_to S0): "Create release payload wiki subpage". +# Creates the per-release payload page under the standing history parent page. +# Phase 2 (step B6) later writes the built versions into this same page. +wiki: + org: "https://identitydivision.visualstudio.com" + project: "IdentityWiki" + wiki: "IdentityWiki.wiki" + parent_path: "/IdentityWiki/Services/Microsoft Authenticator/Release/Android/Monthly Releases Payloads History" + # Child page name = " Release" (e.g. "August 2026 Release"). + # If that page already exists, the agent NOTIFIES and creates the next free + # numbered page instead ("August 2026 2 Release", "August 2026 3 Release", ...). + +# Step `lockdown` (maps_to S1): "Detect lockdown/holiday overlap". +# The CCOA "No-Fly Zones" source is an AAD-gated web app, so it can't be read by +# deterministic Python — the SKILL scrapes it via the authenticated browser and +# passes the periods to `check-lockdown`, which decides overlap DETERMINISTICALLY. +# Only periods whose Environment is Production BLOCK the release; Banner-only +# advisories are ignored. Overlap is checked against the release window +# CCD-7 .. CCD+14. On overlap the step holds for the owner (who runs set-ccd). +lockdown: + url: "https://prod.change-manager.msidentity.com/ccoa-periods" + blocking_environment: "Production" # only Production-env CCOA periods block + window_start_anchor: "CCD-7" + window_end_anchor: "CCD+14" + +# Step `notice` (maps_to S0): "Send early release notice". +# Fills the local template deterministically and sends it. RECIPIENTS: in a +# dry-run every outbound email goes to the release owner (safe rehearsal); a live +# release uses the real `recipients` list below. See EXTERNAL-REFERENCES.md. +notice: + template: "templates/early-code-complete-notice.md" + variant: "initial" # initial (CCD-7) | update (CCD-day) + recipients: # LIVE only; dry-run redirects to owner + - "androididentity@microsoft.com" # "Azure Identity Android SDK" + - "jialh@microsoft.com" + +# Step `flight_reminder` (maps_to S3/S4/S5): "Feature-owner flight & string reminders". +# Three checklist reminders combined into ONE Teams message to the Android Core Team +# group chat. Scout-assisted (Teams send needs WorkIQ). DRY-RUN → the message goes to +# the release owner's own Teams chat; LIVE → the group chat below. See EXTERNAL-REFERENCES.md. +flight_reminder: + live_chat_id: "19:976a859f167f44e59c4ceca8b1d23581@thread.v2" # "Android Core Team" + live_chat_name: "Android Core Team" + links: + variable_group: "https://identitydivision.visualstudio.com/Engineering/_library?itemType=VariableGroups&view=VariableGroupView&variableGroupId=40&path=release" + premortem_example: "https://microsoft-my.sharepoint-df.com/:w:/p/rapong/cQpEZp0cXp1sQYo4A4M3PQWCEgUCDj364FJa-rq-msg59WlBsw" + localization: "https://eng.ms/docs/microsoft-security/identity/entra-developer-application-platform/auth-client/authn-sdk-msal-android/android-auth-libraries/releases/combined-release-checklist/localization" + ecs_flight_history: "https://msazure.visualstudio.com/One/_git/AD-MFA-phonefactor-phoneApp-android?path=/PhoneFactor/ExperimentationLibrary/src/main/java/com/microsoft/authenticator/experimentation/ecs/entities/EcsFlight.kt&version=GBworking" + + diff --git a/release-agent/config/readiness.yaml b/release-agent/config/readiness.yaml new file mode 100644 index 00000000..bd9edd3c --- /dev/null +++ b/release-agent/config/readiness.yaml @@ -0,0 +1,129 @@ +# Release readiness checklist — the ENTRY GATE. +# The engine blocks (status = readiness_gate) until EVERY item is satisfied. +# All items are equally required — there is no per-item priority. +# +# The ONLY distinction is WHO resolves an item (the `verify` field): +# auto -> Scout resolves it (verifies programmatically). Result is pass or fail; +# if it can't be fully proven, it is NOT auto. Two execution sources: +# * (default) a Python verifier in phases/readiness_verifiers.py (uses az/http) +# * source: scout -> the SKILL runs the check via its MCP tools (e.g. ICM) +# and records the result with `record-check`; the Python engine skips it. +# attest -> The engineer resolves it (confirms). Used for anything Scout cannot +# fully prove (physical devices, portal sign-in). +# +# If any item is unsatisfied the gate stays closed. If the engineer cannot satisfy +# an attest item, they resolve it or hand the release to someone who can. +# +# Edit this file to change the entry checklist — it is data, not code. + +title: "Release readiness — entry gate" +instructions: > + Clear every item before Phase 0 can start. Scout verifies the auto items; you + attest the rest. Every item is required — the release cannot start until all are + satisfied. +blocked_message: > + This item is required and is not satisfied, so the release cannot start. Resolve + it — or, if you cannot, hand the release to another engineer who can (notify your + manager / the release team). The new owner runs /release-agent and completes the + checklist themselves. + +items: + # ---- AUTO: fully verified by Scout (pass/fail, no half-measures) ---- + # `label` = short name for the table; `detail` = PLAIN cell text (no inline links — + # markdown links don't render inside Scout table cells). Links go in `links` + # (list of {name,url}) and are rendered as a clickable reference list BELOW the table. + # For build_access the detail + links are generated from the live check results. + - id: build_access + label: "Build definitions accessible" + text: "Release build definitions are accessible" + verify: auto + verifier: build_defs + checks: + - type: ado_build_def + name: "Auth Client Android (Engineering #2828)" + org: "https://identitydivision.visualstudio.com/" + project: "Engineering" + id: 2828 + url: "https://identitydivision.visualstudio.com/Engineering/_build?definitionId=2828" + - type: ado_build_def + name: "Android Build Release (One #397224)" + org: "https://msazure.visualstudio.com/" + project: "One" + id: 397224 + url: "https://msazure.visualstudio.com/One/_build?definitionId=397224" + + # Everything the tool needs to run UNATTENDED (machine on, Scout not focused): + # the MCP servers registered, and the permissions set so scheduled work (the + # daily digest, Teams reminders, browser checks) runs without a prompt. + - id: mcp_servers + label: "MCP servers registered" + text: "The MCP servers the release needs (ICM on-call, Kusto/ADX telemetry) are registered in Scout" + detail: "Scout checks the ICM + Kusto/ADX MCP servers are registered in your Scout config (needed for the on-call and telemetry checks)." + verify: auto + verifier: mcp_servers + + - id: silent_perms + label: "Silent-run permissions" + text: "Permissions allow fully unattended runs — shell, WorkIQ (email + Teams), and the browser are auto-approved so scheduled work never stalls on a prompt" + detail: "Scout checks shell, WorkIQ (email + Teams), and the browser are all auto-approved so the daily automation runs silently when Scout isn't focused." + verify: auto + source: scout # only the skill can read Scout's own settings (m_get_settings) + verifier: silent_perms + # The servers that must be auto-approved for a fully-silent run (kept as data). + required_servers: [shell, workiq, playwright] + # OPT-OUT (soft): enabling silent runs needs the user to turn on the Scout master + # toggle "Allow AI to request permission changes" (allowModelPermissionsChange) — + # which ONLY the user can flip in the UI. The skill offers to enable silent runs; + # if the user declines, it records `degraded` (proceed WITHOUT silent runs) and the + # gate still clears. Downside recorded: the daily digest / Teams reminders / browser + # checks will PROMPT when Scout isn't focused and can stall until the user opens Scout. + opt_out: true + + # ---- ATTEST: the engineer confirms (Scout cannot fully prove these) ---- + # `detail` is the table-cell text and MAY contain inline markdown links + # (they render fine inside Scout table cells). + - id: adx_access + label: "ADX release dashboard" + text: "You can query the ADX release-telemetry cluster (Scout verifies via Kusto)" + detail: "Scout verifies you can query the [ADX release dashboard](https://dataexplorer.azure.com/dashboards/ab7abd7e-c36f-4ff0-88a7-aaa3ec014bd7)'s cluster." + verify: auto + source: scout # the SKILL queries Kusto (the Python engine can't reach the MCP) + verifier: kusto_access + cluster_uri: "https://idsharedeus2.eastus2.kusto.windows.net" + database: "d496be22d62a46b0a3cf67ea2e736fd8" + + - id: play_console_access + label: "Play Console access" + text: "You can open the Play Console app dashboard (sign in and confirm it loads)" + detail: "Open the [Play Console dashboard](https://play.google.com/console/u/0/developers/6720847872553662727/app/4972501392087484764/app-dashboard) and confirm it loads." + verify: attest + + - id: oncall_now + label: "Not on-call now" + text: "You are NOT currently the Android on-call (Scout verifies via ICM)" + detail: "Scout verifies you are not the current Android [on-call](https://aka.ms/androidoncall) for Auth Client Android Shield." + verify: auto + source: scout # the SKILL checks ICM (the Python engine can't reach the MCP) + verifier: oncall + team_id: 78848 + team_name: "Auth Client Android Shield" + + - id: oncall_window + label: "Free during release window" + text: "You are NOT scheduled on-call during the release window" + detail: "Confirm you are NOT scheduled Android [on-call](https://aka.ms/androidoncall) during the release window." + verify: attest + window_start_anchor: "CCD-7" # window opens 7 days before Code Complete + window_end_anchor: "CCD+14" # …through 14 days after + + - id: saw_ame + label: "SAW + AME" + text: "SAW machine and AME account are accessible — log into SAW, confirm the desktop loads, and sign in with your AME credentials" + detail: "Log into [SAW](https://aka.ms/saw), confirm the desktop loads, and sign in with your AME credentials." + verify: attest + + - id: yubikey + label: "YubiKey in hand" + text: "YubiKey in hand (no YubiKey? pick one up at Studio A 1909, Bldg 34/35 cafe, or order a DSR USB-C thumbdrive)" + detail: "YubiKey in hand — no YubiKey? Studio A 1909, Bldg 34/35 cafe, or [order a DSR USB-C](https://cloudmfa-support.azurewebsites.net/SecurityKeyServices/SecurityKey)." + verify: attest diff --git a/release-agent/config/requirements.yaml b/release-agent/config/requirements.yaml new file mode 100644 index 00000000..28e4942c --- /dev/null +++ b/release-agent/config/requirements.yaml @@ -0,0 +1,114 @@ +# Release Orchestrator — external dependency manifest (single source of truth). +# +# Lists everything the tool needs on a machine, so a DIFFERENT release engineer +# can verify their setup is complete. `bootstrap.ps1` reads this file and checks +# each entry (or, for MCP servers, prints them for the engineer to confirm in Scout). +# +# KEEP THIS UPDATED: whenever a feature or phase agent introduces a new dependency +# (a CLI, a Python package, or an MCP server), add it here in the same change. +# +# Entry fields: +# id unique short id +# name human name +# type cli | python | host | mcp +# required_by which part(s) of the tool need it (traceability) +# check shell command that succeeds (exit 0) when present [cli/python/host] +# install how to install it (shown when the check fails) [cli/python/host] +# note guidance (used for `mcp` entries — not shell-checkable) + +requirements: + - id: scout + name: "Microsoft Scout (host app)" + type: host + required_by: [everything — the skill runs inside Scout] + # Detected by the presence of the Scout profile dir (~/.scout). This is a soft + # check surfaced by `infra`; if absent, the engineer must install Scout FIRST + # (MCP servers can't be registered without it). + check: "cmd /c \"if exist \"%USERPROFILE%\\.scout\" (exit 0) else (exit 1)\"" + install: "Install Microsoft Scout first, then re-run bootstrap. (Get it from your team's Scout distribution / the internal Scout install page.)" + + - id: python + name: "Python 3.9+" + type: host + required_by: [engine] + check: "python --version" + install: "Install Python 3.9 or newer from https://www.python.org/downloads/" + + - id: pyyaml + name: "PyYAML" + type: python + required_by: [engine (reads config/*.yaml)] + check: "python -c \"import yaml\"" + install: "python -m pip install pyyaml" + + - id: azure-cli + name: "Azure CLI (az)" + type: cli + required_by: [readiness.build_access] + check: "az version" + install: "https://learn.microsoft.com/cli/azure/install-azure-cli" + + - id: azure-devops-ext + name: "Azure CLI 'azure-devops' extension" + type: cli + required_by: [readiness.build_access] + check: "az extension show --name azure-devops" + install: "az extension add --name azure-devops" + + - id: az-login + name: "Signed in to Azure DevOps (az login)" + type: cli + required_by: [readiness.build_access] + check: "az account show" + install: "az login (must have access to the release build definitions)" + + - id: agency-cli + name: "Agency CLI (provides the ICM MCP server)" + type: cli + required_by: [readiness.oncall_now, mcp.icm] + check: "cmd /c \"%APPDATA%\\agency\\CurrentVersion\\agency.exe --version\"" + install: "Install the Agency platform: https://aka.ms/agency (provides agency.exe under %APPDATA%\\agency)" + +# MCP servers the skill needs inside Scout. Unlike CLIs these live in Scout's own +# config (~/.scout/m-mcp-servers.json) and load at startup — so `bootstrap.ps1` +# REGISTERS any missing ones (create-if-absent, backing up the file first) and then +# tells the engineer to RESTART Scout for them to load. Each entry: +# id short id +# name human name +# scout_key the key to use under "servers" in m-mcp-servers.json +# provider a shell check that the launcher exists (so we don't register a broken server) +# command absolute launcher path (supports %APPDATA% etc.; expanded at register time) +# args launcher args +# required_by which part(s) of the tool need it +# note guidance +mcp_servers: + - id: icm + name: "ICM MCP server (on-call / incidents)" + scout_key: icm + provider: "cmd /c \"%APPDATA%\\agency\\CurrentVersion\\agency.exe --version\"" + command: "%APPDATA%\\agency\\CurrentVersion\\agency.exe" + args: ["mcp", "icm"] + required_by: [readiness.oncall_now, on-call lookups] + note: "Provided by the Agency CLI. After registering, RESTART Scout so it loads; then Scout auto-discovers the ICM tools." + + - id: kusto + name: "Kusto / ADX MCP server (query telemetry)" + scout_key: kusto + provider: "cmd /c \"%APPDATA%\\agency\\CurrentVersion\\agency.exe --version\"" + command: "%APPDATA%\\agency\\CurrentVersion\\agency.exe" + args: ["mcp", "kusto"] + # Multi-cluster: infra appends '--known-services ' built from the + # `kusto_clusters` list below, so ALL our clusters are queryable through one + # MCP. Add a cluster there (data only) to make it available — no code change. + known_services_from: kusto_clusters + required_by: [monitoring/ADX dashboards, Play vitals, crash metrics, readiness.adx_access] + note: "Provided by the Agency CLI (uvx microsoft-fabric-rti-mcp). After registering, RESTART Scout. Add clusters under kusto_clusters." + +# Kusto/ADX clusters the tool queries (data — infra wires these into the kusto MCP +# via --known-services). Add an entry per cluster we use. +kusto_clusters: + - service_uri: "https://idsharedeus2.eastus2.kusto.windows.net" + default_database: "d496be22d62a46b0a3cf67ea2e736fd8" + description: "ID shared EastUS2 (idsharedeus2) — Android release telemetry" + + diff --git a/release-agent/config/schedule.yaml b/release-agent/config/schedule.yaml new file mode 100644 index 00000000..5708fdf6 --- /dev/null +++ b/release-agent/config/schedule.yaml @@ -0,0 +1,20 @@ +# Where the Code Complete Date (CCD) comes from — the system of record. +# +# The orchestrator SEEDS CCD from this pipeline at `init`, and when you change +# CCD (`set-ccd`) or skip the release (`skip-release`) it WRITES BACK to these +# same variables — gated (explicit --confirm) and audited (reason + event log). +# +# CCD rule (mirrors the pipeline's own YAML): +# * override_variable, if set AND its month == the release month -> use it +# * otherwise -> 2nd Wednesday of the month +# +# Per-phase anchors (e.g. Phase 0 opens CCD-7) live next to each phase in +# phases.yaml as `anchor: "CCD-7"`. This file only says WHERE CCD comes from. + +ccd_source: + pipeline_id: 3038 + org: "https://identitydivision.visualstudio.com" + project: "Engineering" + name: "Code Complete Calendar Checker" + override_variable: "overrideCodeCompleteDate" # YYYY-MM-DD, month-scoped + skip_variable: "skipRelease" # any non-empty value suppresses the release diff --git a/release-agent/orchestrator/__init__.py b/release-agent/orchestrator/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/release-agent/orchestrator/cli.py b/release-agent/orchestrator/cli.py new file mode 100644 index 00000000..49f7e464 --- /dev/null +++ b/release-agent/orchestrator/cli.py @@ -0,0 +1,53 @@ +"""Release Orchestrator — CLI entry point (thin assembler). + +The interface the /release-agent skill calls. This file only wires the parser and +dispatches; the command handlers live in `orchestrator/commands/` (one module per +domain) and shared plumbing in `orchestrator/cli_common.py`. + + python -m orchestrator.cli [options] + +State lives in //release-state.json (gitignored). +Config is release-agent/config/*.yaml. +""" +from __future__ import annotations +import argparse +import os +import sys + +# Force UTF-8 stdout so status glyphs don't crash under Windows cp1252 when piped. +try: + sys.stdout.reconfigure(encoding="utf-8") + sys.stderr.reconfigure(encoding="utf-8") +except Exception: + pass + +HERE = os.path.dirname(os.path.abspath(__file__)) +ROOT = os.path.dirname(HERE) # release-agent/ +sys.path.insert(0, ROOT) + +from orchestrator import cli_common as C +from orchestrator.commands import REGISTRARS + + +def build_parser(): + p = argparse.ArgumentParser(prog="release-agent", + description="Release Orchestrator backbone (X4+X5).") + p.add_argument("--config", default=C.DEFAULT_CONFIG) + p.add_argument("--runs-root", default=C.DEFAULT_RUNS_ROOT) + # NOTE: --as-of is defined per-command (status/next/approve/deny/done/resume/notify), + # where it must appear AFTER the subcommand. It is intentionally NOT a global flag: + # argparse lets a subparser's own --as-of silently clobber a global one, which is a + # footgun. Commands that don't take a simulated clock simply omit it. + sub = p.add_subparsers(dest="cmd", required=True) + for register in REGISTRARS: + register(sub) + return p + + +def main(argv=None): + args = build_parser().parse_args(argv) + return args.func(args) + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/release-agent/orchestrator/cli_common.py b/release-agent/orchestrator/cli_common.py new file mode 100644 index 00000000..ade5a53b --- /dev/null +++ b/release-agent/orchestrator/cli_common.py @@ -0,0 +1,147 @@ +"""Shared CLI plumbing for the Release Orchestrator command modules. + +The CLI is split into a thin assembler (`cli.py`) plus one module per command +domain under `commands/`. This module holds the pieces those command modules +share: path resolution, state/orchestrator loading, the event log, user-facing +emit (print + auto-log), and the small render helpers used when advancing. + +Everything here takes explicit parameters (runs_root / release / config) rather +than the argparse namespace, so the helpers are decoupled from the parser and +easy to reuse and test. +""" +from __future__ import annotations + +import os + +from orchestrator.state import ReleaseState +from orchestrator.engine import Orchestrator +from orchestrator import discovery, render, schedule +from orchestrator.eventlog import EventLog +from tools import checks +import yaml as _yaml + +HERE = os.path.dirname(os.path.abspath(__file__)) +ROOT = os.path.dirname(HERE) # release-agent/ +DEFAULT_CONFIG = os.path.join(ROOT, "config", "phases.yaml") +SCHEDULE_CONFIG = os.path.join(ROOT, "config", "schedule.yaml") +REQUIREMENTS_CONFIG = os.path.join(ROOT, "config", "requirements.yaml") +# runs live OUTSIDE release-agent/, in android-complete/.release-runs (gitignored) +DEFAULT_RUNS_ROOT = os.path.join(os.path.dirname(ROOT), ".release-runs") + + +# ---- paths / state ---- +def state_path(runs_root: str, release: str) -> str: + return os.path.join(runs_root, release, "release-state.json") + + +def load_state(runs_root: str, release: str) -> ReleaseState: + return ReleaseState.load(state_path(runs_root, release)) + + +def save_state(st: ReleaseState, runs_root: str, release: str) -> None: + st.save(state_path(runs_root, release)) + + +def parse_as_of(args): + """The simulated clock from --as-of (None ⇒ engine uses today).""" + s = getattr(args, "as_of", None) + return schedule.parse_date(s) if s else None + + +def load_orch(runs_root: str, release: str, config: str, as_of=None): + """Load state + build an Orchestrator wired to the --as-of clock.""" + st = load_state(runs_root, release) + return st, Orchestrator(config, st, as_of=as_of) + + +# ---- config ---- +def ccd_source() -> dict: + """Where CCD comes from (pipeline coords) — from config/schedule.yaml.""" + try: + with open(SCHEDULE_CONFIG, "r", encoding="utf-8") as fh: + return (_yaml.safe_load(fh) or {}).get("ccd_source", {}) or {} + except OSError: + return {} + + +# ---- event log / emit ---- +def elog(runs_root: str, release: str) -> EventLog: + return EventLog(runs_root, release) + + +def emit(runs_root: str, release: str, text: str, kind: str = "message", options=None): + """Print a user-facing block AND auto-log it as scout output, so the log + always captures 'what was shown' without relying on the skill/LLM to journal.""" + print(text) + try: + elog(runs_root, release).scout_said(text, kind=kind, options=options) + except Exception: + pass + + +# ---- advancing the loop (shared by next / approve / done) ---- +TAGS = {"ran": "[ok]", "gate": "[gate]", "reminder": "[action]", "scheduled": "[scheduled]", + "complete": "[done]", "idle": "[--]", "readiness": "[entry-gate]", + "blocked": "[BLOCKED]", "halted": "[HALTED]"} + + +def log_actions(el: EventLog, actions): + """Record engine actions as events (step_ran / gate_hold / complete / blocked).""" + events = { + "ran": "step_ran", "gate": "gate_hold", "reminder": "reminder_hold", + "scheduled": "scheduled_hold", "readiness": "readiness_hold", + "blocked": "blocked_hold", "halted": "halted_hold", "complete": "release_complete", + } + for a in actions: + name = events.get(a.kind) + if not name: + continue + if a.kind in ("ran", "gate", "reminder", "scheduled"): + el.log(name, phase=a.phase, step=a.step, name=a.name) + else: + el.log(name) + + +def advance_block(actions, orch, lead=None) -> str: + """The canonical 'what happened + new status' block for advance commands.""" + out = list(lead or []) + for a in actions: + out.append(f" {TAGS.get(a.kind, '-')} {a.message}") + out.append("\n" + render.status_view(orch.status_report())) + return "\n".join(out) + + +# ---- CCD / pipeline helpers ---- +def refresh_conflict(st: ReleaseState) -> bool: + """Best-effort: re-read the pipeline override and refresh st.ccd_conflict + (a pipeline date that differs from our stored CCD). Returns True if the state + changed (caller should save). Silent on any read failure — never blocks.""" + if not st.ccd: + return False + src = ccd_source() + if not src.get("pipeline_id"): + return False + ok, val, _ = checks.read_pipeline_variable( + src["org"], src["project"], src["pipeline_id"], src["override_variable"]) + if not ok: + return False + conflict = schedule.pipeline_conflict(st.release_id, val, st.ccd) + new = conflict.isoformat() if conflict else None + if new != st.ccd_conflict: + st.ccd_conflict = new + return True + return False + + +def write_ccd_var(src: dict, value: str): + """Write the CCD override variable on the pipeline. Returns CheckResult.""" + return checks.set_pipeline_variable( + src["org"], src["project"], src["pipeline_id"], src["override_variable"], value) + + +def resolve_release_id(runs_root: str, release): + """Return an explicit release id or discover the active one (or None).""" + if release: + return release + rel = discovery.resolve(runs_root, None).get("release") + return rel["release_id"] if rel else None diff --git a/release-agent/orchestrator/commands/__init__.py b/release-agent/orchestrator/commands/__init__.py new file mode 100644 index 00000000..c6417e74 --- /dev/null +++ b/release-agent/orchestrator/commands/__init__.py @@ -0,0 +1,21 @@ +"""Command modules for the Release Orchestrator CLI. + +Each module in this package owns one domain of commands. A module exposes a +`register(subparsers)` function that adds its subparser(s) and wires each to its +handler via `set_defaults(func=...)`. `cli.py` imports REGISTRARS and calls each +one, so adding a command is a localized change (new/edited module only). +""" +from . import release, readiness, pipeline, notify, infra_cmd, automation, logs, lockdown, notice + +# Order controls how subcommands appear in --help. +REGISTRARS = [ + release.register, + readiness.register, + pipeline.register, + notify.register, + lockdown.register, + notice.register, + logs.register, + automation.register, + infra_cmd.register, +] diff --git a/release-agent/orchestrator/commands/automation.py b/release-agent/orchestrator/commands/automation.py new file mode 100644 index 00000000..63786730 --- /dev/null +++ b/release-agent/orchestrator/commands/automation.py @@ -0,0 +1,54 @@ +"""Automation registry command: register / list / deregister provisioned Scout +automations so they can be torn down cleanly at release close.""" +from __future__ import annotations +import json as _json + +from orchestrator.registry import AutomationRegistry + + +def cmd_automation(args): + """Track Scout automations the orchestrator provisions, so they can be torn + down at release close. This only records ids — the skill does the actual + Scout create/delete via m_create_automation / m_delete_automation.""" + reg = AutomationRegistry(args.runs_root) + if args.action == "register": + if not (args.id and args.name): + print("register needs --id and --name.") + return 1 + e = reg.register(args.id, args.name, release=args.release, + shared=args.shared, purpose=args.purpose or "") + where = "shared" if e["scope"] == "shared" else f"release {e['release']}" + print(f"Registered automation {e['id']} ({where}): {e['name']}") + return 0 + if args.action == "deregister": + if not args.id: + print("deregister needs --id.") + return 1 + print("Deregistered." if reg.deregister(args.id) else "No such automation id in registry.") + return 0 + # list + items = reg.list(release=args.release, scope=(args.scope or None)) + if args.json: + print(_json.dumps(items, indent=2)) + return 0 + if not items: + print("No automations registered." if args.release is None + else f"No automations registered for release {args.release}.") + return 0 + for e in items: + where = "shared" if e.get("scope") == "shared" else (e.get("release") or "?") + print(f" {e['id']} [{where}] {e['name']} — {e.get('purpose','')}") + return 0 + + +def register(sub): + au = sub.add_parser("automation", help="Track provisioned automations (register/list/deregister) for teardown") + au.add_argument("action", choices=["register", "list", "deregister"]) + au.add_argument("--id", default=None, help="Scout automation id") + au.add_argument("--name", default="", help="Automation name (for register)") + au.add_argument("--release", default=None, help="Release scope (omit + --shared for machine-wide)") + au.add_argument("--shared", action="store_true", help="Mark as shared/persistent (not torn down per release)") + au.add_argument("--scope", default=None, choices=["shared", "release"], help="Filter list by scope") + au.add_argument("--purpose", default="", help="Short description") + au.add_argument("--json", action="store_true") + au.set_defaults(func=cmd_automation) diff --git a/release-agent/orchestrator/commands/infra_cmd.py b/release-agent/orchestrator/commands/infra_cmd.py new file mode 100644 index 00000000..38df422e --- /dev/null +++ b/release-agent/orchestrator/commands/infra_cmd.py @@ -0,0 +1,53 @@ +"""Infrastructure preflight command: check CLIs + register/verify MCP servers in +Scout. Named infra_cmd to avoid clashing with the orchestrator.infra module.""" +from __future__ import annotations +import json as _json + +from orchestrator import infra +from orchestrator import cli_common as C + + +def cmd_infra(args): + """Infrastructure preflight: check CLI/host deps and register + verify the + MCP servers the skill needs in Scout. Run before the tool-level requirements. + Registers missing MCP servers into Scout's config (backup first) unless + --no-register; --json for machine output.""" + report = infra.run(C.REQUIREMENTS_CONFIG, register=not getattr(args, "no_register", False)) + if getattr(args, "json", False): + print(_json.dumps(report, indent=2)) + return 0 if report["ok"] else 1 + print("Infrastructure preflight") + if not report.get("scout_present", True): + print(" ⛔ Microsoft Scout not detected (~/.scout missing).") + scout_req = next((r for r in report["requirements"] if r["id"] == "scout"), None) + if scout_req and scout_req.get("install"): + print(f" install: {scout_req['install']}") + print(" Install Scout FIRST, then re-run — MCP servers can't be registered without it.") + print(" CLIs / host:") + for r in report["requirements"]: + mark = "OK" if r["ok"] else "MISSING" + print(f" [{mark}] {r['name']}") + if not r["ok"] and r["install"]: + print(f" install: {r['install']}") + print(" MCP servers (Scout config):") + if not report["mcp_servers"]: + print(" (none required)") + for m in report["mcp_servers"]: + label = {"present": "OK", "registered": "REGISTERED", "would_register": "MISSING", + "provider_missing": "PROVIDER MISSING", "launcher_missing": "LAUNCHER MISSING", + "scout_missing": "SCOUT NOT INSTALLED"}.get(m["status"], m["status"].upper()) + print(f" [{label}] {m['name']} — {m['detail']}") + if report["restart_needed"]: + print("\n ⚠ RESTART Scout to load newly-registered MCP server(s).") + if not report["ok"]: + print("\n Some infrastructure is missing — resolve the items above, then re-run.") + else: + print("\n Infrastructure OK.") + return 0 if report["ok"] else 1 + + +def register(sub): + inf = sub.add_parser("infra", help="Infrastructure preflight: check CLIs + register/verify MCP servers in Scout") + inf.add_argument("--no-register", action="store_true", help="Only report; don't register missing MCP servers") + inf.add_argument("--json", action="store_true") + inf.set_defaults(func=cmd_infra) diff --git a/release-agent/orchestrator/commands/lockdown.py b/release-agent/orchestrator/commands/lockdown.py new file mode 100644 index 00000000..1668c86d --- /dev/null +++ b/release-agent/orchestrator/commands/lockdown.py @@ -0,0 +1,88 @@ +"""Lockdown / CCOA overlap check (Phase-0 step `lockdown`, S1). + +The CCOA "No-Fly Zones" source is an AAD-gated web app, so the SKILL scrapes it +via the authenticated browser and passes the periods here as JSON. This command +decides overlap DETERMINISTICALLY (not the LLM) and records the step result: + * no Production-env CCOA overlaps the release window -> step passes. + * one or more overlap -> step holds for the owner (who shifts CCD via set-ccd). +""" +from __future__ import annotations +import json as _json + +from orchestrator import cli_common as C +from orchestrator import schedule + + +def _load_lockdown_cfg() -> dict: + from orchestrator.phase_config import load_phase_config + return load_phase_config("preflight", "lockdown") + + +def overlapping_periods(win_start, win_end, periods, blocking_env="Production"): + """Pure overlap rule. `periods` is a list of dicts with keys name, environment, + start (date), end (date). Returns those whose Environment contains + `blocking_env` (case-insensitive) AND whose [start,end] intersects the window + [win_start,win_end]. Banner-only advisories (no Production) are excluded.""" + hits = [] + for p in periods: + if blocking_env.lower() not in (p.get("environment") or "").lower(): + continue + s, e = p.get("start"), p.get("end") + if s is None or e is None: + continue + if s <= win_end and e >= win_start: # ranges intersect + hits.append(p) + return hits + + +def cmd_check_lockdown(args): + st = C.load_state(args.runs_root, args.release) + if not st.ccd: + print("No CCD set for this release — cannot compute the release window.") + return 1 + cfg = _load_lockdown_cfg() + blocking_env = cfg.get("blocking_environment", "Production") + ccd = schedule.parse_date(st.ccd) + win_start = schedule.anchor_date(ccd, cfg.get("window_start_anchor", "CCD-7")) + win_end = schedule.anchor_date(ccd, cfg.get("window_end_anchor", "CCD+14")) + + try: + raw = _json.loads(args.periods_json or "[]") + except ValueError: + print("Could not parse --periods-json (expected a JSON array).") + return 1 + periods = [] + for p in raw if isinstance(raw, list) else []: + s, e = schedule.parse_date(p.get("start")), schedule.parse_date(p.get("end")) + periods.append({"name": p.get("name", "?"), + "environment": p.get("environment", ""), "start": s, "end": e}) + + hits = overlapping_periods(win_start, win_end, periods, blocking_env) + _, orch = C.load_orch(args.runs_root, args.release, args.config, C.parse_as_of(args)) + window = f"{win_start.isoformat()}..{win_end.isoformat()}" + if not hits: + detail = (f"No {blocking_env} CCOA lockdown overlaps the release window " + f"({window}). Checked {len(periods)} period(s).") + orch.record_scout_step("preflight", "lockdown", "pass", detail) + C.save_state(orch.state, args.runs_root, args.release) + C.emit(args.runs_root, args.release, f"[ok] Lockdown check: {detail}", kind="lockdown") + return 0 + + listed = "; ".join( + f"{h['name']} ({h['start'].isoformat()}..{h['end'].isoformat()})" for h in hits) + detail = (f"{len(hits)} {blocking_env} CCOA lockdown(s) overlap the release window " + f"({window}): {listed}. Shift CCD past the lockdown (set-ccd) if needed.") + orch.record_scout_step("preflight", "lockdown", "attention", detail) + C.save_state(orch.state, args.runs_root, args.release) + C.emit(args.runs_root, args.release, f"[attention] Lockdown overlap — {detail}", kind="lockdown") + return 0 + + +def register(sub): + cl = sub.add_parser("check-lockdown", + help="Decide CCOA lockdown overlap from scraped periods and record the step") + cl.add_argument("--release", required=True) + cl.add_argument("--periods-json", required=True, + help='JSON array of {name, environment, start, end} (dates YYYY-MM-DD, UTC)') + cl.add_argument("--as-of", default=None, help="Simulated clock (YYYY-MM-DD); default today") + cl.set_defaults(func=cmd_check_lockdown) diff --git a/release-agent/orchestrator/commands/logs.py b/release-agent/orchestrator/commands/logs.py new file mode 100644 index 00000000..6c470832 --- /dev/null +++ b/release-agent/orchestrator/commands/logs.py @@ -0,0 +1,64 @@ +"""Event-log commands: log (show/analyze) and journal (record interaction).""" +from __future__ import annotations +import json as _json + +from orchestrator.eventlog import EventLog, summarize +from orchestrator import cli_common as C + + +def cmd_log(args): + """Show or analyze this release's event log (per-release only).""" + el = EventLog(args.runs_root, args.release) + events = el.read(args.limit) + if args.analyze: + print(_json.dumps(summarize(events), indent=2)) + return 0 + if args.json: + print(_json.dumps(events, indent=2)) + return 0 + if not events: + print("No events logged yet.") + return 0 + for e in events: + src = e.get("source", "engine") + loc = f" {e['phase']}/{e.get('step','')}" if e.get("phase") else "" + extra = "" + if e.get("driver"): + extra += f" driver=\"{e['driver']}\"" + if e.get("text"): + t = e["text"].replace("\n", " ") + extra += f" \"{t[:80]}{'…' if len(t) > 80 else ''}\"" + if e.get("choice"): + extra += f" choice={e['choice']}" + print(f" {e['ts']} {src:<6} {e.get('actor','?'):<10} {e['event']}{loc}{extra}") + return 0 + + +def cmd_journal(args): + """Record an INTERACTION event (what Scout showed / what the user chose). + The skill calls this so the per-release log captures the real conversation + for debugging. Best-effort; never affects the flow.""" + el = C.elog(args.runs_root, args.release) + if args.source == "scout": + el.scout_said(args.text or "", kind=args.kind or "message", options=args.option or None) + else: + el.user_said(args.text or "", kind=args.kind or "input", choice=args.choice or None) + return 0 + + +def register(sub): + lg = sub.add_parser("log", help="Show or analyze this release's event log") + lg.add_argument("--release", required=True) + lg.add_argument("--analyze", action="store_true", help="Print a rolled-up summary") + lg.add_argument("--limit", type=int, default=None) + lg.add_argument("--json", action="store_true") + lg.set_defaults(func=cmd_log) + + jn = sub.add_parser("journal", help="Record an interaction event (scout output / user input)") + jn.add_argument("--release", required=True) + jn.add_argument("--source", required=True, choices=["scout", "user"]) + jn.add_argument("--text", default="", help="What was shown / said") + jn.add_argument("--kind", default="", help="e.g. prompt, checklist, message, choice, input") + jn.add_argument("--choice", default="", help="For user: the option id/label chosen") + jn.add_argument("--option", action="append", help="For scout: an option presented (repeatable)") + jn.set_defaults(func=cmd_journal) diff --git a/release-agent/orchestrator/commands/notice.py b/release-agent/orchestrator/commands/notice.py new file mode 100644 index 00000000..05779faa --- /dev/null +++ b/release-agent/orchestrator/commands/notice.py @@ -0,0 +1,253 @@ +"""Early code-complete notice (Phase-0 step `notice`, S0) + generic step recorder. + +Sending email needs the WorkIQ MCP (a skill-layer capability the deterministic +engine can't reach), so `notice` is a scout-assisted step: + + 1. `prepare-notice` (here, deterministic) fills the local template with the + release's CCD/owner and resolves recipients — DRY-RUN redirects every mail to + the release owner; a LIVE release uses the real recipients from preflight.yaml. + It prints {subject, body, recipients, dry_run, ...} for the skill to send. + 2. the skill sends it via workiq_send_email, then calls `record-step` to mark + the step done (or attention on failure). + +`record-step` is a generic recorder for ANY scout-assisted phase step. +""" +from __future__ import annotations +import json as _json +import os + +from orchestrator import cli_common as C +from orchestrator import schedule + +_ROOT = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) + +# Fixed external link used inside the notice body (see EXTERNAL-REFERENCES.md). +HOTFIX_GUIDE_URL = ("https://eng.ms/docs/microsoft-security/identity/" + "entra-developer-application-platform/auth-client/" + "microsoft-authenticator/microsoft-authenticator/release/" + "cherry-pick-to-hotfix-guidelines") + + +def _load_notice_cfg() -> dict: + from orchestrator.phase_config import load_phase_config + return load_phase_config("preflight", "notice") + + +def _ordinal(n: int) -> str: + if 11 <= (n % 100) <= 13: + return f"{n}th" + return f"{n}{ {1: 'st', 2: 'nd', 3: 'rd'}.get(n % 10, 'th') }" + + +def _parse_template(text: str, variant: str): + """Pull the (subject, body) for a variant from the delimited template file. + Sections are marked '===INITIAL:SUBJECT===' / '===INITIAL:BODY===' etc.""" + key = variant.upper() + marks = {"subject": f"==={key}:SUBJECT===", "body": f"==={key}:BODY==="} + out = {} + for field, mark in marks.items(): + if mark not in text: + return None + after = text.split(mark, 1)[1] + # body runs until the next '===...===' marker or EOF + end = after.find("\n===") + out[field] = (after[:end] if end != -1 else after).strip("\n") + return out["subject"].strip(), out["body"] + + +def _fill(s: str, ctx: dict) -> str: + for k, v in ctx.items(): + s = s.replace("{" + k + "}", str(v)) + return s + + +def _esc(s: str) -> str: + return (str(s or "").replace("&", "&").replace("<", "<") + .replace(">", ">")) + + +def _notice_html(variant: str, ctx: dict) -> str: + """Email-safe HTML notice: clean anchor for the hotfix guide + a real table + (inline styles, Outlook-friendly). Same content as the markdown body.""" + date_line = ("**Today**" if variant == "update" + else f"{_esc(ctx['ccd_long'])}.") + if variant == "update": + date_line = "Today" + owner = _esc(ctx["owner"]) + return f"""\ +
+

Hi everyone,

+

This is a reminder that the Microsoft Android Authenticator app and Broker + libraries code complete date for the {_esc(ctx['month'])} release is {date_line}

+

Any check-ins made after code complete will require following + the hotfix cherry-pick guide + and EM approval.

+ + + + + + + + + + + +
MonthCode Complete DateAndroid Release Owner
{_esc(ctx['month'])}{_esc(ctx['ccd_date'])} + Primary (Release Owner — covers Broker + Auth App): @{_esc(ctx['owner_at'])}
+

Thank you,

+

{owner}

+
""" + + +def cmd_prepare_notice(args): + st = C.load_state(args.runs_root, args.release) + if not st.ccd: + print(_json.dumps({"error": "no CCD set for this release"})) + return 1 + cfg = _load_notice_cfg() + variant = getattr(args, "variant", None) or cfg.get("variant", "initial") + tpl_path = os.path.join(_ROOT, cfg.get("template", "templates/early-code-complete-notice.md")) + try: + with open(tpl_path, "r", encoding="utf-8") as fh: + parsed = _parse_template(fh.read(), variant) + except OSError: + print(_json.dumps({"error": f"template not found: {tpl_path}"})) + return 1 + if not parsed: + print(_json.dumps({"error": f"variant '{variant}' not in template"})) + return 1 + subject_tpl, body_tpl = parsed + + ccd = schedule.parse_date(st.ccd) + owner_email = st.owner_email or "" + ctx = { + "month": ccd.strftime("%B"), + "ccd_long": f"{ccd.strftime('%A, %B')} {_ordinal(ccd.day)}, {ccd.year}", + "ccd_date": ccd.strftime("%m/%d/%Y"), + "owner": st.owner_name or owner_email or "the release owner", + "owner_at": (owner_email.split("@")[0] if owner_email else "release-owner"), + } + subject = _fill(subject_tpl, ctx) + body = _fill(body_tpl, ctx) + html = _notice_html(variant, ctx) + + # RECIPIENTS: dry-run → owner only (safe); live → configured real list. + if st.dry_run: + recipients = [owner_email] if owner_email else [] + note = "dry-run: redirected to release owner" + else: + recipients = list(cfg.get("recipients", [])) + note = "live recipients" + subject_out = (f"[DRY-RUN → owner] {subject}" if st.dry_run else subject) + + print(_json.dumps({ + "step": "notice", "release": args.release, "dry_run": st.dry_run, + "subject": subject_out, "body": body, "html": html, "recipients": recipients, + "recipients_note": note, + })) + return 0 + + +def _reminder_ctx(st): + """Shared date/owner context for the Teams reminders.""" + ccd = schedule.parse_date(st.ccd) + owner_email = st.owner_email or "" + ccd7 = schedule.anchor_date(ccd, "CCD-7") + return { + "month": ccd.strftime("%B"), + "ccd_long": f"{ccd.strftime('%A, %B')} {_ordinal(ccd.day)}, {ccd.year}", + "ccd_date": ccd.strftime("%m/%d/%Y"), + "ccd7_date": ccd7.strftime("%m/%d/%Y"), + "owner": st.owner_name or owner_email or "the release owner", + }, owner_email + + +def _reminder_cfg(section: str) -> dict: + from orchestrator.phase_config import load_phase_config + return load_phase_config("preflight", section) + + +def _reminder_payload(step, args, st, cfg, html, owner_email): + """Resolve the Teams target: DRY-RUN → owner's own chat; LIVE → group chat.""" + if st.dry_run: + html = ("

[DRY-RUN → owner] this would go to the " + f"{_esc(cfg.get('live_chat_name', 'Android Core Team'))} group chat.

" + html) + return {"step": step, "release": args.release, "dry_run": True, + "content": html, "content_type": "html", + "send_to": "owner", "owner_email": owner_email, "chat_id": None, + "target_note": "dry-run: send to the release owner's own Teams chat"} + return {"step": step, "release": args.release, "dry_run": False, + "content": html, "content_type": "html", + "send_to": "group", "owner_email": owner_email, + "chat_id": cfg.get("live_chat_id"), + "target_note": cfg.get("live_chat_name", "Android Core Team")} + + +def _flight_reminder_html(ctx: dict, links: dict) -> str: + """Teams-friendly HTML for the combined feature-owner reminders.""" + vg = links.get("variable_group", "#") + pm = links.get("premortem_example", "#") + loc = links.get("localization", "#") + ecs = links.get("ecs_flight_history", "#") + return f"""\ +

Flight & String Reminders — {_esc(ctx['month'])} Release

+

Hi Android Core Team, four reminders as we approach code complete ({_esc(ctx['ccd_long'])}):

+
    +
  1. [Broker] Update local flights. Feature owners — please update your local flights in the release variable group. The release engineer will not update these directly; verify your values are current before we proceed. (Variable group)
    2. +
  2. [Broker & Auth App] Flight pre-mortem docs. It is each Feature Owner's sole responsibility to ensure every flight shipping with a default value of true has a flight pre-mortem doc — explaining the change's purpose and how we plan to monitor it after the release hits PROD. (Example pre-mortem doc)
    3. +
  3. [Auth App] Merge user-facing strings by today (CCD−7). It is each Feature Owner's sole responsibility to ensure every Authenticator PR with a new or modified user-facing string is merged by today ({_esc(ctx['ccd7_date'])}). Strings landing inside the 1-week window are not guaranteed to be localized — call out any late strings and escalate to the team lead if not resolved by end of day. (Localization Instructions)
    4. +
  4. [Auth App] Feature-flag freeze & default-OFF review. Review all features added since the last release and ensure they are default OFF; any feature rolling out default ON requires explicit Team Lead / Engineering Manager approval documented in the release wiki, and the release should be blocked if such approval is missing. No feature work is allowed after Code Complete (CCD) — all feature-flag changes must land before CCD, and any flag changes after CCD require the same cherry-pick approval process as code changes. Review EcsFlight.kt history since the last Code Complete to verify compliance. (EcsFlight.kt history)
    5. +
+

Thanks,
{_esc(ctx['owner'])}

""" + + +def cmd_prepare_flight_reminder(args): + """Build the combined 3-in-1 flight & string reminder Teams message and resolve + its target. DRY-RUN → the release owner's own Teams chat; LIVE → the Android + Core Team group chat. Prints JSON for the skill to send via workiq_send_chat_message.""" + st = C.load_state(args.runs_root, args.release) + if not st.ccd: + print(_json.dumps({"error": "no CCD set for this release"})) + return 1 + cfg = _reminder_cfg("flight_reminder") + ctx, owner_email = _reminder_ctx(st) + html = _flight_reminder_html(ctx, cfg.get("links", {}) or {}) + print(_json.dumps(_reminder_payload("flight_reminder", args, st, cfg, html, owner_email))) + return 0 + + +def cmd_record_step(args): + """Generic recorder for a scout-assisted phase step (skill calls this after + doing the out-of-engine work, e.g. sending the notice email).""" + _, orch = C.load_orch(args.runs_root, args.release, args.config, C.parse_as_of(args)) + act = orch.record_scout_step(args.phase, args.step, args.status, args.detail or "") + C.save_state(orch.state, args.runs_root, args.release) + C.emit(args.runs_root, args.release, + f"[{'ok' if args.status == 'pass' else 'attention'}] {args.step}: {act.message}", + kind="step") + return 0 + + +def register(sub): + pn = sub.add_parser("prepare-notice", + help="Fill the early code-complete notice template and resolve recipients (JSON)") + pn.add_argument("--release", required=True) + pn.add_argument("--variant", default=None, help="initial (default) | update") + pn.set_defaults(func=cmd_prepare_notice) + + pf = sub.add_parser("prepare-flight-reminder", + help="Build the combined flight & string reminder Teams message + target (JSON)") + pf.add_argument("--release", required=True) + pf.set_defaults(func=cmd_prepare_flight_reminder) + + rs = sub.add_parser("record-step", + help="Record a scout-assisted phase step result (pass|attention)") + rs.add_argument("--release", required=True) + rs.add_argument("--phase", default="preflight") + rs.add_argument("--step", required=True) + rs.add_argument("--status", required=True, choices=["pass", "attention"]) + rs.add_argument("--detail", default="") + rs.add_argument("--as-of", default=None) + rs.set_defaults(func=cmd_record_step) diff --git a/release-agent/orchestrator/commands/notify.py b/release-agent/orchestrator/commands/notify.py new file mode 100644 index 00000000..f93b9f39 --- /dev/null +++ b/release-agent/orchestrator/commands/notify.py @@ -0,0 +1,126 @@ +"""Notification + owner commands: notify (daily phase digest) and set-owner.""" +from __future__ import annotations +import json as _json +import os + +from orchestrator.state import ReleaseState +from orchestrator.engine import Orchestrator +from orchestrator import render, schedule +from orchestrator import cli_common as C +from tools import checks + + +def cmd_set_owner(args): + """Set/change the release owner (who reminders are emailed to).""" + st = C.load_state(args.runs_root, args.release) + st.owner_email = (getattr(args, "owner_email", None) or checks.current_az_user()) + if getattr(args, "owner_name", None): + st.owner_name = args.owner_name + C.save_state(st, args.runs_root, args.release) + C.elog(args.runs_root, args.release).log("owner_set", owner=st.owner_email) + if not st.owner_email: + print("Couldn't resolve an owner (no --owner-email and az user unavailable).") + return 1 + who = f"{st.owner_name + ' ' if st.owner_name else ''}{st.owner_email}" + print(f"Release {args.release} owner set to {who}.") + return 0 + + +def cmd_notify(args): + """Emit the daily phase digest IF the active phase is open with outstanding + work, else nothing. Read-only (does NOT advance the flow — use `tick` for that). + De-duped to one per calendar day; --force bypasses; --json prints the mailer + payload {message,subject,owner_email,owner_name,release}.""" + rid = C.resolve_release_id(args.runs_root, args.release) + want_json = getattr(args, "json", False) + if not rid: + if want_json: + print(_json.dumps({"message": "", "html": "", "subject": "", "owner_email": None, + "owner_name": None, "release": None})) + return 0 + payload = _notify_payload(args, rid, advance=False) + if want_json: + print(_json.dumps(payload)) + elif payload["message"]: + print(payload["message"]) + return 0 + + +def _notify_payload(args, rid, advance): + """Shared by `notify` and `tick`. Optionally ADVANCE the flow first + (run_until_gate), then read the state machine and build the once-per-day + digest payload. Returns {message, subject, owner_email, owner_name, release}. + `message` is "" unless a digest is due AND not already sent today (or --force).""" + sp = C.state_path(args.runs_root, rid) + if not os.path.exists(sp): + return {"message": "", "html": "", "subject": "", "owner_email": None, + "owner_name": None, "release": rid} + as_of = C.parse_as_of(args) + if advance: + # Auto-advance: run every agent step that can run, holding at the first + # gate / action-needed. Idempotent — a no-op once holding or not due. + st, orch = C.load_orch(args.runs_root, rid, args.config, as_of) + actions = orch.run_until_gate() + C.save_state(st, args.runs_root, rid) + C.log_actions(C.elog(args.runs_root, rid), actions) + else: + st = ReleaseState.load(sp) + orch = Orchestrator(args.config, st, as_of=as_of) + report = orch.status_report() + msg = render.notification(report) + html = render.notification_html(report) + subject = render.notification_subject(report) + today = (as_of or schedule.today()).isoformat() + fresh = bool(msg) and (getattr(args, "force", False) or st.last_notified_date != today) + if fresh: + st.last_notified_date = today + st.save(sp) + try: + C.elog(args.runs_root, rid).log("notified", text=msg, owner=st.owner_email) + except Exception: + pass + return {"message": msg if fresh else "", "html": html if fresh else "", + "subject": subject, "owner_email": st.owner_email, + "owner_name": st.owner_name, "release": rid} + + +def cmd_tick(args): + """One automation heartbeat: discover the active release, ADVANCE it (run the + agent steps that can run, holding at gates/actions), then emit the daily digest + payload for the mailer. Safe to run often — advancing is idempotent and the + digest is de-duped to once per calendar day. This is what the hourly Scout + automation runs so an open phase makes progress even if the 9am tick was missed + (machine off) — the next tick after the machine is on picks it up.""" + rid = C.resolve_release_id(args.runs_root, args.release) + if not rid: + print(_json.dumps({"message": "", "html": "", "subject": "", "owner_email": None, + "owner_name": None, "release": None})) + return 0 + payload = _notify_payload(args, rid, advance=True) + if getattr(args, "json", False): + print(_json.dumps(payload)) + elif payload["message"]: + print(payload["message"]) + return 0 + + +def register(sub): + so = sub.add_parser("set-owner", help="Set/change the release owner (who reminders are emailed to)") + so.add_argument("--release", required=True) + so.add_argument("--owner-email", default=None, help="Owner email (default: signed-in az user)") + so.add_argument("--owner-name", default=None, help="Owner display name (optional)") + so.set_defaults(func=cmd_set_owner) + + nt = sub.add_parser("notify", help="Emit a push line if something needs the user now (else nothing)") + nt.add_argument("--release", default=None, help="Target release; if omitted, discover the active one") + nt.add_argument("--as-of", default=None, help="Simulated clock (YYYY-MM-DD) — debug override; default today") + nt.add_argument("--force", action="store_true", help="Bypass de-dup (always emit if actionable)") + nt.add_argument("--json", action="store_true", help="Emit {message,subject,owner_email,owner_name,release} for the mailer") + nt.set_defaults(func=cmd_notify) + + tk = sub.add_parser("tick", help="Automation heartbeat: ADVANCE the active release, then emit the digest payload") + tk.add_argument("--release", default=None, help="Target release; if omitted, discover the active one") + tk.add_argument("--as-of", default=None, help="Simulated clock (YYYY-MM-DD) — debug override; default today") + tk.add_argument("--force", action="store_true", help="Bypass the once-per-day digest de-dup") + tk.add_argument("--json", action="store_true", help="Emit {message,subject,owner_email,owner_name,release} for the mailer") + tk.set_defaults(func=cmd_tick) diff --git a/release-agent/orchestrator/commands/pipeline.py b/release-agent/orchestrator/commands/pipeline.py new file mode 100644 index 00000000..004e801d --- /dev/null +++ b/release-agent/orchestrator/commands/pipeline.py @@ -0,0 +1,105 @@ +"""Pipeline-write commands (real production changes to ADO pipeline 3038): +set-ccd and skip-release. Both are gated (preview → --confirm) and audited.""" +from __future__ import annotations + +from orchestrator import schedule +from orchestrator import cli_common as C +from tools import checks + + +def cmd_set_ccd(args): + """Change the Code Complete Date. Writes the pipeline override (real change) — + requires --confirm and a --reason. Without --confirm, previews the write.""" + st = C.load_state(args.runs_root, args.release) + src = C.ccd_source() + if not src.get("pipeline_id"): + print("No CCD source configured (config/schedule.yaml).") + return 1 + if not (args.reason and args.reason.strip()): + print("A --reason is required (audited).") + return 1 + + if args.default: + new_ccd, source, value = schedule.default_ccd(args.release), "default", "" + what = f"clear the override → default {new_ccd.isoformat()} (2nd Wednesday)" + else: + d = schedule.parse_date(args.date) + if not d: + print(f"Bad --date '{args.date}' (expected YYYY-MM-DD).") + return 1 + ry, rm = schedule.parse_release_month(args.release) + if (d.year, d.month) != (ry, rm): + print(f"CCD {d.isoformat()} is not in release month {args.release}. The pipeline " + f"override is month-scoped, so a different month wouldn't apply. " + f"Use the release id for that month instead.") + return 1 + new_ccd, source, value = d, "manual", d.isoformat() + what = f"set CCD override → {value}" + + if not args.confirm: + print(f"[preview] Would {what} on pipeline {src['pipeline_id']} " + f"({src['override_variable']}).\n Re-run with --confirm to write it. Reason: {args.reason.strip()}") + return 0 + + res = C.write_ccd_var(src, value) + if not res.ok: + print(f"Failed to write pipeline variable: {res.detail}") + return 1 + st.ccd = new_ccd.isoformat() + st.ccd_source = source + st.ccd_conflict = None # the date is now settled — clear any conflict + C.save_state(st, args.runs_root, args.release) + C.elog(args.runs_root, args.release).log( + "ccd_changed", value=value or "(default)", source=source, driver=args.reason.strip()) + opens = schedule.anchor_date(new_ccd, "CCD-7").isoformat() + C.emit(args.runs_root, args.release, + f"✅ CCD set to **{st.ccd}** ({source}); pipeline updated. " + f"Phase 0 opens {opens} (CCD-7).\n {res.detail}", kind="ccd") + return 0 + + +def cmd_skip_release(args): + """Suppress the release by setting the pipeline 'skipRelease' switch (real change).""" + st = C.load_state(args.runs_root, args.release) + src = C.ccd_source() + if not (args.reason and args.reason.strip()): + print("A --reason is required (audited).") + return 1 + clearing = bool(getattr(args, "clear", False)) + value = "" if clearing else "skipped" + verb = "clear" if clearing else "set" + if not args.confirm: + print(f"[preview] Would {verb} '{src.get('skip_variable')}' on pipeline " + f"{src.get('pipeline_id')}.\n Re-run with --confirm. Reason: {args.reason.strip()}") + return 0 + res = checks.set_pipeline_variable( + src["org"], src["project"], src["pipeline_id"], src["skip_variable"], value) + if not res.ok: + print(f"Failed to write pipeline variable: {res.detail}") + return 1 + st.skip_release = not clearing + C.save_state(st, args.runs_root, args.release) + C.elog(args.runs_root, args.release).log( + "release_skip_cleared" if clearing else "release_skip_set", driver=args.reason.strip()) + msg = ("✅ Release un-skipped — pipeline will trigger normally." + if clearing else + "🛑 Release marked SKIP in the pipeline — the monthly trigger is suppressed until cleared.") + C.emit(args.runs_root, args.release, msg + f"\n {res.detail}", kind="skip_release") + return 0 + + +def register(sub): + sc = sub.add_parser("set-ccd", help="Change the Code Complete Date (writes pipeline override; --confirm)") + sc.add_argument("--release", required=True) + sc.add_argument("--date", default="", help="New CCD (YYYY-MM-DD), must be in the release month") + sc.add_argument("--default", action="store_true", help="Clear the override → 2nd-Wednesday default") + sc.add_argument("--reason", default="", help="Why (audited — required)") + sc.add_argument("--confirm", action="store_true", help="Actually write to the pipeline (else preview)") + sc.set_defaults(func=cmd_set_ccd) + + sr = sub.add_parser("skip-release", help="Suppress/cancel the release via the pipeline switch (--confirm)") + sr.add_argument("--release", required=True) + sr.add_argument("--clear", action="store_true", help="Clear the skip (re-enable the release)") + sr.add_argument("--reason", default="", help="Why (audited — required)") + sr.add_argument("--confirm", action="store_true", help="Actually write to the pipeline (else preview)") + sr.set_defaults(func=cmd_skip_release) diff --git a/release-agent/orchestrator/commands/readiness.py b/release-agent/orchestrator/commands/readiness.py new file mode 100644 index 00000000..e4a75137 --- /dev/null +++ b/release-agent/orchestrator/commands/readiness.py @@ -0,0 +1,121 @@ +"""Readiness entry-gate commands: checklist, verify, sign, decline.""" +from __future__ import annotations +import json as _json + +from orchestrator import render +from orchestrator import cli_common as C + + +def cmd_checklist(args): + st, orch = C.load_orch(args.runs_root, args.release, args.config) + if getattr(args, "verify", False): + orch.gate.verify() + C.save_state(st, args.runs_root, args.release) + chk = orch.gate.checklist() + if getattr(args, "json", False): + print(_json.dumps(chk, indent=2)) + return 0 + # canonical, consistent display block (the template) — same for every engineer. + # auto-logged as scout output so the log always records what was shown. + C.emit(args.runs_root, args.release, render.readiness_table(chk, args.release), kind="readiness_checklist") + return 0 + + +def cmd_verify(args): + st, orch = C.load_orch(args.runs_root, args.release, args.config) + chk = orch.gate.verify() + C.save_state(st, args.runs_root, args.release) + C.elog(args.runs_root, args.release).log( + "readiness_verified", + results=[{"id": it["id"], "status": it["status"]} for it in chk["auto_items"]]) + for it in chk["auto_items"]: + mark = "OK" if it["status"] == "pass" else "FAIL" + print(f" [{mark}] {it['id']}: {it['status']} — {it.get('message','')}") + return 0 + + +def cmd_sign(args): + st, orch = C.load_orch(args.runs_root, args.release, args.config) + ids = None if args.all else (args.item or []) + chk = orch.gate.sign(ids) + C.save_state(st, args.runs_root, args.release) + C.elog(args.runs_root, args.release).log( + "readiness_signed" if chk["signed"] else "readiness_partial", + items=("all" if ids is None else ids), signed=chk["signed"]) + if chk["signed"]: + print(f"Readiness signed at {chk['signed_at']}. Entry gate cleared — you can now start Phase 0.") + else: + pending = [i["id"] for i in chk["items"] if not i["satisfied"]] + print(f"Recorded. Still pending: {', '.join(pending)}") + return 0 + + +def cmd_decline(args): + st, orch = C.load_orch(args.runs_root, args.release, args.config) + chk = orch.gate.decline(args.item or []) + C.save_state(st, args.runs_root, args.release) + C.elog(args.runs_root, args.release).log( + "readiness_declined", items=args.item or [], blocked=chk["blocked"], driver=args.reason or None) + if chk["blocked"]: + labels = [next((i["label"] for i in chk["items"] if i["id"] == b), b) + for b in chk["blocked_items"]] + msg = ("⛔ BLOCKED — cannot start: " + ", ".join(labels) + ".\n" + " " + chk.get("blocked_message", "").strip()) + else: + msg = f"Recorded as unable: {', '.join(args.item or [])}" + C.emit(args.runs_root, args.release, msg, kind="decline_result") + return 0 + + +def cmd_record_check(args): + """Record the result of a scout-assisted auto readiness check (source: scout), + e.g. the ICM on-call lookup. The skill runs the check via its MCP tools and + calls this to store the pass/fail result in the engine.""" + st, orch = C.load_orch(args.runs_root, args.release, args.config) + res = orch.gate.record_check(args.item, args.status, args.detail or "") + if "error" in res: + print(res["error"]) + return 1 + C.save_state(st, args.runs_root, args.release) + C.elog(args.runs_root, args.release).log( + "readiness_check_recorded", item=args.item, status=args.status, driver=args.detail or None) + item = next((i for i in res["items"] if i["id"] == args.item), None) + mark = {"pass": "OK", "degraded": "WARN"}.get(args.status, "FAIL") + tail = " — entry gate cleared." if res.get("signed") else "" + C.emit(args.runs_root, args.release, + f"[{mark}] {(item or {}).get('label', args.item)}: {args.status}" + f"{(' — ' + args.detail) if args.detail else ''}{tail}", kind="record_check") + return 0 + + +def register(sub): + c = sub.add_parser("checklist", help="Show the readiness entry-gate checklist") + c.add_argument("--release", required=True) + c.add_argument("--verify", action="store_true", help="Run auto verifiers before showing") + c.add_argument("--json", action="store_true") + c.set_defaults(func=cmd_checklist) + + v = sub.add_parser("verify", help="Run the auto readiness verifiers") + v.add_argument("--release", required=True) + v.set_defaults(func=cmd_verify) + + rc = sub.add_parser("record-check", + help="Record a scout-assisted auto check result (e.g. ICM on-call)") + rc.add_argument("--release", required=True) + rc.add_argument("--item", required=True, help="Readiness item id (must be a source:scout auto item)") + rc.add_argument("--status", required=True, choices=["pass", "fail", "degraded"], + help="pass | fail (or 'degraded' for opt-out items the user proceeds without)") + rc.add_argument("--detail", default="", help="Short evidence/summary (e.g. 'not in roster')") + rc.set_defaults(func=cmd_record_check) + + sg = sub.add_parser("sign", help="Attest human readiness items (also runs auto verify)") + sg.add_argument("--release", required=True) + sg.add_argument("--all", action="store_true", help="Attest every human item") + sg.add_argument("--item", action="append", help="Attest a specific item id (repeatable)") + sg.set_defaults(func=cmd_sign) + + dc = sub.add_parser("decline", help="Declare you CANNOT satisfy an item (may block ownership)") + dc.add_argument("--release", required=True) + dc.add_argument("--item", action="append", required=True, help="Item id you cannot satisfy (repeatable)") + dc.add_argument("--reason", default="", help="Why (recorded in the event log)") + dc.set_defaults(func=cmd_decline) diff --git a/release-agent/orchestrator/commands/release.py b/release-agent/orchestrator/commands/release.py new file mode 100644 index 00000000..817645a4 --- /dev/null +++ b/release-agent/orchestrator/commands/release.py @@ -0,0 +1,295 @@ +"""Release lifecycle + manual overrides: init, list, status, next, approve, deny, +done, activate, skip, reopen, halt, resume.""" +from __future__ import annotations +import json as _json +import os + +from orchestrator.state import ReleaseState +from orchestrator.engine import Orchestrator +from orchestrator import discovery, render, schedule +from orchestrator import cli_common as C +from tools import checks + + +def cmd_init(args): + sp = C.state_path(args.runs_root, args.release) + if os.path.exists(sp) and not args.force: + print(f"Release {args.release} already exists at {sp} (use --force to recreate).") + return 1 + st = ReleaseState(release_id=args.release, dry_run=not args.live, status="not_started") + + # Release owner (the engineer running this release) — release metadata. + # Priority: explicit --owner-email (skill can pass the richer profile) then + # the signed-in az user. Never hardcoded. + st.owner_email = (getattr(args, "owner_email", None) or checks.current_az_user()) + st.owner_name = getattr(args, "owner_name", None) or None + owner_note = "" if st.owner_email else " (couldn't resolve owner — set with set-owner)" + + # CCD is canonically the 2nd Wednesday. We still READ the pipeline override, + # but we do NOT silently adopt it — if it differs, we flag a conflict for the + # user to resolve (2nd-Wed default vs the pipeline date). + src = C.ccd_source() + override, note = None, "" + if src.get("pipeline_id"): + ok, val, detail = checks.read_pipeline_variable( + src["org"], src["project"], src["pipeline_id"], src["override_variable"]) + if ok: + override = val + else: + note = f" (couldn't read pipeline override: {detail})" + try: + default = schedule.default_ccd(args.release) + st.ccd = default.isoformat() + st.ccd_source = "default" + conflict = schedule.pipeline_conflict(args.release, override, st.ccd) + st.ccd_conflict = conflict.isoformat() if conflict else None + except (ValueError, IndexError): + default, conflict = None, None + note = " (release id isn't YYYY-MM — CCD not set; use set-ccd)" + st.save(sp) + + mode = "LIVE" if args.live else "dry-run" + C.elog(args.runs_root, args.release).log( + "release_started", mode=mode, forced=bool(args.force), + ccd=st.ccd, ccd_source=st.ccd_source, ccd_conflict=st.ccd_conflict, owner=st.owner_email) + owner_line = f" Owner: {st.owner_name + ' ' if st.owner_name else ''}{st.owner_email or '(unresolved)'}{owner_note}" + if st.ccd: + opens = schedule.anchor_date(default, "CCD-7").isoformat() + lines = [f"Initialized release {args.release} ({mode}).", f" state: {sp}", owner_line, + f" Code Complete Date: {st.ccd} (2nd Wednesday){note}", + f" Phase 0 (Pre-flight) opens {opens} (CCD-7). Until then nothing fires."] + if conflict: + lines.append(f" ⚠ Pipeline override is {conflict.isoformat()}, which differs from the " + f"2nd-Wednesday default. Confirm which is the real CCD before proceeding.") + print("\n".join(lines)) + else: + print(f"Initialized release {args.release} ({mode}).\n state: {sp}\n{owner_line}\n{note}") + return 0 + + +def cmd_list(args): + """List discovered releases (none/one/many). --json for the skill.""" + res = discovery.resolve(args.runs_root, getattr(args, "release", None)) + if args.json: + print(_json.dumps(res, indent=2)) + return 0 + all_ = res["all"] + if res["resolution"] == "none": + if getattr(args, "release", None): + print(f"No release '{args.release}' found. Start one with: init --release {args.release}") + else: + print("No active release on this machine. Start one with: init --release ") + return 0 + print(f"Found {len(all_)} release(s):") + for r in all_: + mark = "->" if r is res["release"] else " " + mode = "dry-run" if r["dry_run"] else "LIVE" + print(f" {mark} {r['release_id']} [{r['status']}, {mode}] updated {r['updated_at']}") + if res["resolution"] == "ambiguous": + print(f"\nMultiple releases found — assuming most recent: {res['release']['release_id']} " + f"(confirm before acting).") + return 0 + + +def cmd_status(args): + st, orch = C.load_orch(args.runs_root, args.release, args.config, C.parse_as_of(args)) + if not getattr(args, "no_pipeline_check", False) and C.refresh_conflict(st): + C.save_state(st, args.runs_root, args.release) + if getattr(args, "json", False): + print(_json.dumps(orch.status_report(), indent=2)) + return 0 + C.emit(args.runs_root, args.release, render.status_view(orch.status_report()), kind="status") + return 0 + + +def cmd_next(args): + st, orch = C.load_orch(args.runs_root, args.release, args.config, C.parse_as_of(args)) + actions = orch.run_until_gate() + C.save_state(st, args.runs_root, args.release) # persist BEFORE any display + C.log_actions(C.elog(args.runs_root, args.release), actions) + C.emit(args.runs_root, args.release, C.advance_block(actions, orch), kind="advance") + return 0 + + +def cmd_approve(args): + st, orch = C.load_orch(args.runs_root, args.release, args.config, C.parse_as_of(args)) + gate_phase, gate_step = st.current_phase, st.current_step + act = orch.approve_gate(args.comment or "") + el = C.elog(args.runs_root, args.release) + if act.kind != "idle": + el.log("gate_approved", phase=gate_phase, step=gate_step, driver=args.comment or None) + actions = orch.run_until_gate() + C.save_state(st, args.runs_root, args.release) # persist BEFORE any display + C.log_actions(el, actions) + C.emit(args.runs_root, args.release, + C.advance_block(actions, orch, lead=[f" {act.message}"]), kind="advance") + return 0 + + +def cmd_deny(args): + st, orch = C.load_orch(args.runs_root, args.release, args.config, C.parse_as_of(args)) + gate_phase, gate_step = st.current_phase, st.current_step + act = orch.deny_gate(args.comment or "") + if act.kind != "idle": + C.elog(args.runs_root, args.release).log( + "gate_denied", phase=gate_phase, step=gate_step, driver=args.comment or None) + C.save_state(st, args.runs_root, args.release) + C.emit(args.runs_root, args.release, + f" {act.message}\n\n" + render.status_view(orch.status_report()), kind="deny") + return 0 + + +def cmd_done(args): + """Mark a reminder (human, non-gate) step done, then advance to the next hold.""" + st, orch = C.load_orch(args.runs_root, args.release, args.config, C.parse_as_of(args)) + act = orch.complete_step(getattr(args, "phase", None), getattr(args, "step", None), args.note or "") + if act.kind == "idle": + print(act.message) + return 1 + el = C.elog(args.runs_root, args.release) + el.log("reminder_done", phase=act.phase, step=act.step, driver=args.note or None) + actions = orch.run_until_gate() + C.save_state(st, args.runs_root, args.release) + C.log_actions(el, actions) + C.emit(args.runs_root, args.release, + C.advance_block(actions, orch, lead=[f" {act.message}"]), kind="advance") + return 0 + + +def cmd_skip(args): + st, orch = C.load_orch(args.runs_root, args.release, args.config) + act = orch.skip_step(args.phase, args.step, args.reason or "") + if act.kind == "idle": # rejected (no reason / bad step) — nothing changed + print(act.message) + return 1 + C.save_state(st, args.runs_root, args.release) + C.elog(args.runs_root, args.release).log("step_skipped", phase=args.phase, step=args.step, driver=args.reason) + C.emit(args.runs_root, args.release, act.message, kind="override") + return 0 + + +def cmd_reopen(args): + st, orch = C.load_orch(args.runs_root, args.release, args.config) + act = orch.reopen_step(args.phase, args.step, args.reason or "") + if act.kind == "idle": + print(act.message) + return 1 + C.save_state(st, args.runs_root, args.release) + C.elog(args.runs_root, args.release).log("step_reopened", phase=args.phase, step=args.step, driver=args.reason or None) + C.emit(args.runs_root, args.release, act.message, kind="override") + return 0 + + +def cmd_halt(args): + st, orch = C.load_orch(args.runs_root, args.release, args.config) + act = orch.halt(args.reason or "") + if act.kind == "idle": + print(act.message) + return 1 + C.save_state(st, args.runs_root, args.release) + C.elog(args.runs_root, args.release).log("release_halted", driver=args.reason) + C.emit(args.runs_root, args.release, act.message, kind="override") + return 0 + + +def cmd_resume(args): + st, orch = C.load_orch(args.runs_root, args.release, args.config, C.parse_as_of(args)) + act = orch.resume(args.reason or "") + if not getattr(args, "no_pipeline_check", False): + C.refresh_conflict(st) + C.save_state(st, args.runs_root, args.release) + C.elog(args.runs_root, args.release).log("release_resumed", driver=args.reason or None) + tail = "" + if st.ccd_conflict: + tail = (f"\n ⚠ Pipeline override {st.ccd_conflict} differs from CCD {st.ccd} — " + f"confirm which is correct (see status).") + C.emit(args.runs_root, args.release, (act.message + tail), kind="override") + return 0 + + +def cmd_activate(args): + st, orch = C.load_orch(args.runs_root, args.release, args.config) + orch.activate_conditional(args.phase) + C.save_state(st, args.runs_root, args.release) + print(f"Activated conditional phase: {args.phase}") + return 0 + + +def register(sub): + i = sub.add_parser("init", help="Start a new release run (dry-run by default)") + i.add_argument("--release", required=True) + i.add_argument("--live", action="store_true") + i.add_argument("--force", action="store_true") + i.add_argument("--owner-email", default=None, help="Release owner email (default: signed-in az user)") + i.add_argument("--owner-name", default=None, help="Release owner display name (optional)") + i.set_defaults(func=cmd_init) + + l = sub.add_parser("list", help="Discover releases (none/one/many)") + l.add_argument("--release", required=False, default=None) + l.add_argument("--json", action="store_true") + l.set_defaults(func=cmd_list) + + s = sub.add_parser("status", help="Show the run-state brief") + s.add_argument("--release", required=True) + s.add_argument("--as-of", default=None, help="Simulated clock (YYYY-MM-DD); default today") + s.add_argument("--json", action="store_true") + s.add_argument("--no-pipeline-check", action="store_true", + help="Skip re-reading the pipeline to detect CCD drift (faster/offline).") + s.set_defaults(func=cmd_status) + + n = sub.add_parser("next", help="Advance until the next gate / completion") + n.add_argument("--release", required=True) + n.add_argument("--as-of", default=None, help="Simulated clock (YYYY-MM-DD); default today") + n.set_defaults(func=cmd_next) + + a = sub.add_parser("approve", help="Approve the current holding gate, continue") + a.add_argument("--release", required=True) + a.add_argument("--as-of", default=None, help="Simulated clock (YYYY-MM-DD); default today") + a.add_argument("--comment", default="") + a.set_defaults(func=cmd_approve) + + d = sub.add_parser("deny", help="Deny the current holding gate") + d.add_argument("--release", required=True) + d.add_argument("--as-of", default=None, help="Simulated clock (YYYY-MM-DD); default today") + d.add_argument("--comment", default="") + d.set_defaults(func=cmd_deny) + + dn = sub.add_parser("done", help="Mark a reminder (human, non-gate) step done, then advance") + dn.add_argument("--release", required=True) + dn.add_argument("--as-of", default=None, help="Simulated clock (YYYY-MM-DD); default today") + dn.add_argument("--phase", default=None, help="Defaults to the current holding step") + dn.add_argument("--step", default=None) + dn.add_argument("--note", default="", help="Optional note (audited)") + dn.set_defaults(func=cmd_done) + + # ---- manual overrides ---- + sk = sub.add_parser("skip", help="Skip a step without running it (reason REQUIRED)") + sk.add_argument("--release", required=True) + sk.add_argument("--phase", required=True) + sk.add_argument("--step", required=True) + sk.add_argument("--reason", required=True, help="Why (audit — required)") + sk.set_defaults(func=cmd_skip) + + ro = sub.add_parser("reopen", help="Reopen a done/skipped step so it runs again") + ro.add_argument("--release", required=True) + ro.add_argument("--phase", required=True) + ro.add_argument("--step", required=True) + ro.add_argument("--reason", default="", help="Why (optional)") + ro.set_defaults(func=cmd_reopen) + + ht = sub.add_parser("halt", help="Emergency hold — nothing advances until resume (reason REQUIRED)") + ht.add_argument("--release", required=True) + ht.add_argument("--reason", required=True, help="Why (audit — required)") + ht.set_defaults(func=cmd_halt) + + rs = sub.add_parser("resume", help="Clear an emergency halt") + rs.add_argument("--release", required=True) + rs.add_argument("--as-of", default=None, help="Simulated clock (YYYY-MM-DD); default today") + rs.add_argument("--no-pipeline-check", action="store_true", help="Skip CCD-drift check") + rs.add_argument("--reason", default="", help="Why (optional)") + rs.set_defaults(func=cmd_resume) + + ac = sub.add_parser("activate", help="Turn on a conditional phase (e.g. hotfix)") + ac.add_argument("--release", required=True) + ac.add_argument("--phase", required=True) + ac.set_defaults(func=cmd_activate) diff --git a/release-agent/orchestrator/discovery.py b/release-agent/orchestrator/discovery.py new file mode 100644 index 00000000..8d4d19e3 --- /dev/null +++ b/release-agent/orchestrator/discovery.py @@ -0,0 +1,72 @@ +"""Release discovery — the none / one / many logic. + +Scans the runs root for release folders and reports what's there so the +/release-agent skill can: + * 0 releases -> tell the user none is active, offer to start one + * 1 release -> use it + * many -> present the assumed one (most recently updated) + ask to confirm + +Deterministic; the skill only presents what this returns. +""" +from __future__ import annotations +import os +import json +from typing import Optional + + +def _summarize(state_file: str) -> Optional[dict]: + try: + with open(state_file, "r", encoding="utf-8") as fh: + data = json.load(fh) + except Exception: + return None + return { + "release_id": data.get("release_id"), + "status": data.get("status"), + "dry_run": data.get("dry_run", True), + "current_phase": data.get("current_phase"), + "current_step": data.get("current_step"), + "updated_at": data.get("updated_at"), + "state_file": state_file, + } + + +def list_releases(runs_root: str) -> list: + """Return summaries of all releases found, newest-updated first.""" + out = [] + if not os.path.isdir(runs_root): + return out + for name in os.listdir(runs_root): + sf = os.path.join(runs_root, name, "release-state.json") + if os.path.isfile(sf): + s = _summarize(sf) + if s: + out.append(s) + out.sort(key=lambda s: s.get("updated_at") or "", reverse=True) + return out + + +def resolve(runs_root: str, requested: Optional[str] = None) -> dict: + """Decide which release to act on. + + Returns a dict: + { "resolution": "none" | "one" | "explicit" | "ambiguous", + "release": , # the chosen/assumed release + "all": [] } # everything found + + - none : no releases exist -> caller should offer to start one + - one : exactly one exists -> use it + - explicit : caller named one and it exists -> use it + - ambiguous : several exist and none named -> 'release' is the assumed + (most recently updated); caller should confirm. + """ + all_ = list_releases(runs_root) + if requested: + match = next((r for r in all_ if r["release_id"] == requested), None) + return {"resolution": "explicit" if match else "none", + "release": match, "all": all_} + if not all_: + return {"resolution": "none", "release": None, "all": all_} + if len(all_) == 1: + return {"resolution": "one", "release": all_[0], "all": all_} + return {"resolution": "ambiguous", "release": all_[0], "all": all_} diff --git a/release-agent/orchestrator/engine.py b/release-agent/orchestrator/engine.py new file mode 100644 index 00000000..37207370 --- /dev/null +++ b/release-agent/orchestrator/engine.py @@ -0,0 +1,695 @@ +"""Release Orchestrator — the conductor (deterministic engine, X4). + +Responsibilities (per §7.1): + 1. Load the release state machine from config/phases.yaml. + 2. Own the dispatch loop: find next step -> run its (stub) agent -> + record result -> advance, or HOLD at a gate for human approval. + 3. Persist run-state via ReleaseState (X5). + +The engine is the BRAIN: it decides what's next. The skill is only the mouth/ears. +No LLM logic here — this is fully unit-testable and dry-run-replayable. +""" +from __future__ import annotations +import os +from dataclasses import dataclass +from datetime import date +from typing import Optional + +import yaml + +from .state import ReleaseState, StepState, GateDecision, _now +from .readiness import ReadinessGate +from . import schedule +from phases import agents + + +@dataclass +class NextAction: + """What the conductor decided on this invocation — the engine's output.""" + kind: str # 'ran' | 'gate' | 'reminder' | 'scheduled' | 'complete' | 'idle' | 'readiness' | 'blocked' | 'halted' + phase: Optional[str] = None + step: Optional[str] = None + name: Optional[str] = None + message: str = "" + + +class Orchestrator: + """The conductor: owns the state machine, dispatch loop, gates, and structured + status. The readiness entry gate is delegated to ReadinessGate (self.gate); + presentation lives in render.py. This class holds no formatting logic.""" + + def __init__(self, config_path: str, state: ReleaseState, readiness_path: str = None, + as_of: date = None): + with open(config_path, "r", encoding="utf-8") as fh: + self.config = yaml.safe_load(fh) + readiness_cfg = None + if readiness_path is None: + readiness_path = os.path.join(os.path.dirname(config_path), "readiness.yaml") + if os.path.exists(readiness_path): + with open(readiness_path, "r", encoding="utf-8") as fh: + readiness_cfg = yaml.safe_load(fh) + self.state = state + self.gate = ReadinessGate(readiness_cfg, state) + # The simulated clock. Defaults to today; `--as-of` overrides it so a + # dry-run can jump to CCD-7 and prove a phase opens on schedule. + self.as_of = as_of or schedule.today() + + # ---- time anchoring (CCD-relative phase windows) ---- + def _ccd(self) -> Optional[date]: + return schedule.parse_date(self.state.ccd) + + def _phase_anchor_date(self, phase: dict) -> Optional[date]: + """The date a phase opens, or None if it has no anchor / CCD is unknown.""" + spec = phase.get("anchor") + ccd = self._ccd() + if not spec or ccd is None: + return None + return schedule.anchor_date(ccd, spec) + + def _phase_due(self, phase: dict) -> bool: + """A phase is due once the clock reaches its anchor. No anchor ⇒ always due.""" + ad = self._phase_anchor_date(phase) + return ad is None or self.as_of >= ad + + @staticmethod + def _is_reminder(step: dict) -> bool: + """A human, non-gate step is a reminder: the engine can't do it, so it + holds and tells the person to do it, then waits for them to mark it done.""" + return step.get("owner") == "human" and not step.get("gate") + + # ---- state-machine traversal ---- + def _iter_steps(self): + """Yield (phase_dict, step_dict) in definition order, skipping conditional + phases unless explicitly activated on the state.""" + for phase in self.config["phases"]: + if phase.get("conditional") and phase["id"] not in self._activated_conditionals(): + continue + for step in phase["steps"]: + yield phase, step + + def _activated_conditionals(self) -> set: + # A conditional phase (e.g. hotfix) is activated by an explicit note flag. + return {n.split("activate:")[1].strip() + for n in self.state.notes if isinstance(n, str) and n.startswith("activate:")} + + def activate_conditional(self, phase_id: str) -> None: + self.state.notes.append(f"activate:{phase_id}") + + def _first_incomplete(self): + for phase, step in self._iter_steps(): + if not self.state.is_done(phase["id"], step["id"]): + return phase, step + return None, None + + # ---- dispatch ---- + def _current_phase(self): + """The first included phase that still has incomplete steps (definition + order). Conditional phases are skipped unless activated.""" + for phase in self.config["phases"]: + if not self._phase_included(phase): + continue + if all(self.state.is_done(phase["id"], s["id"]) for s in phase["steps"]): + continue + return phase + return None + + @staticmethod + def _step_kind(step: dict) -> str: + """Classify a step: gate | scout | attest | reminder | auto.""" + if step.get("gate"): + return "gate" + if step.get("source") == "scout": + return "scout" + if step.get("attest"): + return "attest" + if step.get("owner") == "human": + return "reminder" + return "auto" + + def _deps_met(self, pid: str, step: dict) -> bool: + """True when every step this one depends_on is done (deps are within-phase).""" + for dep in step.get("depends_on", []) or []: + if not self.state.is_done(pid, dep): + return False + return True + + def step_once(self, attempted=None) -> NextAction: + """Advance exactly one step (or hold). For a sequential phase this is the + classic first-incomplete-step logic. For a parallel phase it runs one ready + step whose dependencies are met, letting independent steps progress even + when a sibling is holding. `attempted` (a set, managed by run_until_gate) + prevents re-running an auto step twice within one drain.""" + if self.state.status == "complete": + return NextAction(kind="complete", message="Release already complete.") + + # HALTED: emergency hold set by a human. Nothing advances until resume(). + if self.state.halted: + self.state.status = "halted" + return NextAction( + kind="halted", + message="Release is HALTED" + + (f": {self.state.halt_reason}" if self.state.halt_reason else "") + + ". Run resume to continue.", + ) + + # BLOCKED: an entry-gate item was declared unsatisfiable. + if self.state.blocked: + self.state.status = "blocked" + labels = self.gate.blocked_labels() + msg = (self.gate.config or {}).get("blocked_message", "").strip() + return NextAction( + kind="blocked", + message="Entry gate blocked — cannot start: " + ", ".join(labels) + ". " + msg, + ) + + # ENTRY GATE: nothing runs until the readiness checklist is signed. + if not self.state.readiness_signed: + self.state.status = "readiness_gate" + return NextAction( + kind="readiness", + message="HOLDING at the readiness entry gate. Sign the checklist before Phase 0 can start.", + ) + + phase = self._current_phase() + if phase is None: + self.state.status = "complete" + self.state.current_phase = None + self.state.current_step = None + return NextAction(kind="complete", message="All steps done — release complete.") + + self.state.current_phase = phase["id"] + + # TIME GATE: if this phase hasn't reached its anchor date yet, hold as scheduled. + if not self._phase_due(phase): + opens = self._phase_anchor_date(phase) + self.state.status = "scheduled" + days = (opens - self.as_of).days + first = next((s for s in phase["steps"] + if not self.state.is_done(phase["id"], s["id"])), None) + return NextAction( + kind="scheduled", phase=phase["id"], + step=first["id"] if first else None, + name=first["name"] if first else None, + message=f"{phase['name']} opens {opens.isoformat()} " + f"({schedule.humanize_delta(days)}). Nothing to do yet.", + ) + + if phase.get("execution") == "parallel": + return self._step_parallel(phase, attempted) + return self._step_sequential(phase) + + # ---- sequential dispatch (classic: one step at a time, stop at first hold) ---- + def _step_sequential(self, phase: dict) -> NextAction: + step = next(s for s in phase["steps"] + if not self.state.is_done(phase["id"], s["id"])) + self.state.current_step = step["id"] + + if step.get("gate") and not self._gate_approved(phase["id"], step["id"]): + self.state.status = "holding_gate" + return NextAction( + kind="gate", phase=phase["id"], step=step["id"], name=step["name"], + message=f"HOLDING at gate: {phase['name']} → {step['name']}. Awaiting human decision.", + ) + + if self._is_reminder(step): + self.state.status = "awaiting_action" + key = f"{phase['id']}.{step['id']}" + if key not in self.state.pending_human: + self.state.pending_human.append(key) + if step.get("attest"): + msg = (f"CONFIRM — attest that this is done to proceed: {step['name']}. " + f"Mark it done once you've verified it.") + else: + msg = f"ACTION NEEDED — you need to: {step['name']}. Mark it done when complete." + return NextAction(kind="reminder", phase=phase["id"], step=step["id"], + name=step["name"], message=msg) + + if step.get("source") == "scout": + self.state.status = "awaiting_action" + key = f"{phase['id']}.{step['id']}" + if key not in self.state.pending_human: + self.state.pending_human.append(key) + return NextAction( + kind="reminder", phase=phase["id"], step=step["id"], name=step["name"], + message=f"Scout-assisted check pending — {step['name']}. " + f"Scout runs this automatically when you open it.", + ) + + return self._run_auto_step(phase, step, block_holds=True) + + # ---- parallel dispatch (dependency-aware; independent steps don't block) ---- + def _step_parallel(self, phase: dict, attempted) -> NextAction: + pid = phase["id"] + + def ready(s): + return (not self.state.is_done(pid, s["id"])) and self._deps_met(pid, s) + + # 1) Run ONE ready, not-yet-attempted runnable step (auto agent, or an + # already-approved gate). Independent steps progress even if a sibling holds. + for s in phase["steps"]: + if not ready(s): + continue + kind = self._step_kind(s) + runnable = kind == "auto" or (kind == "gate" and self._gate_approved(pid, s["id"])) + if not runnable: + continue + key = f"{pid}.{s['id']}" + if attempted is not None and key in attempted: + continue + if attempted is not None: + attempted.add(key) + return self._run_auto_step(phase, s, block_holds=False) + + # 2) No more auto progress — surface the holds (all at once). + holds = [] + for s in phase["steps"]: + if not ready(s): + continue + kind = self._step_kind(s) + blocked_auto = kind == "auto" and self.state.get_step(pid, s["id"]).status == "blocked" + unapproved_gate = kind == "gate" and not self._gate_approved(pid, s["id"]) + if kind in ("scout", "attest", "reminder") or blocked_auto or unapproved_gate: + holds.append(s) + + gates = [s for s in holds if self._step_kind(s) == "gate"] + if gates: + g = gates[0] + self.state.status = "holding_gate" + self.state.current_step = g["id"] + return NextAction(kind="gate", phase=pid, step=g["id"], name=g["name"], + message=f"HOLDING at gate: {phase['name']} → {g['name']}. Awaiting human decision.") + + non_gate = [s for s in holds if self._step_kind(s) != "gate"] + for s in non_gate: + key = f"{pid}.{s['id']}" + if key not in self.state.pending_human: + self.state.pending_human.append(key) + if non_gate: + self.state.status = "awaiting_action" + self.state.current_step = non_gate[0]["id"] + names = "; ".join(s["name"] for s in non_gate) + return NextAction(kind="reminder", phase=pid, step=non_gate[0]["id"], + name=non_gate[0]["name"], + message=f"{len(non_gate)} item(s) need you: {names}") + + # Not complete, but nothing is ready — remaining steps wait on unmet deps. + self.state.status = "awaiting_action" + return NextAction(kind="reminder", phase=pid, + message="Waiting on prerequisite steps to complete.") + + def _run_auto_step(self, phase: dict, step: dict, block_holds: bool) -> NextAction: + """Run an agent step. On success → done. On failure: in sequential mode + (block_holds=True) HOLD as action-needed and break; in parallel mode + (block_holds=False) mark it blocked + register it, but return 'ran' so the + drain continues with independent steps.""" + pid = phase["id"] + agent_id = step.get("agent", "stub") + runner = agents.get_runner(agent_id) + result = runner(pid, step, self.state.dry_run, self.state) + key = f"{pid}.{step['id']}" + if not result.ok: + self.state.set_step(pid, step["id"], + StepState(status="blocked", note=result.action, by=result.by)) + if key not in self.state.pending_human: + self.state.pending_human.append(key) + if block_holds: + self.state.status = "awaiting_action" + return NextAction(kind="reminder", phase=pid, step=step["id"], + name=step["name"], + message=f"ACTION NEEDED — {step['name']}: {result.action}") + return NextAction(kind="ran", phase=pid, step=step["id"], name=step["name"], + message=f"BLOCKED — {step['name']}: {result.action}") + self.state.set_step(pid, step["id"], + StepState(status="done", completed_at=_now(), + note=result.action, by=result.by)) + if result.by == "human": + self.state.pending_human = [p for p in self.state.pending_human if p != key] + self.state.status = "running" + return NextAction(kind="ran", phase=pid, step=step["id"], + name=step["name"], message=result.action) + + def run_until_gate(self, max_steps: int = 500) -> list: + """Drive the loop until a gate hold, completion, or step cap. + Returns the list of NextAction taken. `attempted` prevents re-running an + auto step twice within this drain (so a re-blocking step can't loop).""" + actions = [] + attempted = set() + for _ in range(max_steps): + act = self.step_once(attempted) + actions.append(act) + if act.kind in ("gate", "reminder", "scheduled", "complete", "readiness", "blocked", "halted"): + break + return actions + + # ---- manual overrides (human-driven transitions, §7.1 constraint #5) ---- + def _find_step(self, phase_id: str, step_id: str): + phase = next((p for p in self.config["phases"] if p["id"] == phase_id), None) + if phase and any(s["id"] == step_id for s in phase["steps"]): + return phase + return None + + def skip_step(self, phase_id: str, step_id: str, reason: str) -> NextAction: + """Mark a step skipped (counts as done for progression) without running it. + A reason is REQUIRED (audit). For 'doesn't apply' or 'done manually outside the tool'.""" + if not (reason and reason.strip()): + return NextAction(kind="idle", message="A reason is required to skip a step.") + if not self._find_step(phase_id, step_id): + return NextAction(kind="idle", message=f"No such step: {phase_id}/{step_id}") + self.state.set_step(phase_id, step_id, + StepState(status="skipped", completed_at=_now(), + note=f"Skipped: {reason.strip()}", by="human")) + if self.state.status == "holding_gate" and self.state.current_step == step_id: + self.state.status = "running" + return NextAction(kind="ran", phase=phase_id, step=step_id, + message=f"Skipped {phase_id}/{step_id} — {reason.strip()}") + + def complete_step(self, phase_id: str = None, step_id: str = None, note: str = "") -> NextAction: + """Mark a reminder (human, non-gate) step done. Defaults to the step the + conductor is currently holding on. This is how a person clears an + 'ACTION NEEDED' hold once they've actually done the task.""" + phase_id = phase_id or self.state.current_phase + step_id = step_id or self.state.current_step + if not (phase_id and step_id) or not self._find_step(phase_id, step_id): + return NextAction(kind="idle", message=f"No such step: {phase_id}/{step_id}") + self.state.set_step(phase_id, step_id, + StepState(status="done", completed_at=_now(), + note=(note.strip() or "Marked done"), by="human")) + key = f"{phase_id}.{step_id}" + self.state.pending_human = [p for p in self.state.pending_human + if p != key and not p.startswith(key + " ")] + if self.state.status == "awaiting_action": + self.state.status = "running" + tail = f" — {note.strip()}" if note and note.strip() else "" + return NextAction(kind="ran", phase=phase_id, step=step_id, + message=f"Done: {phase_id}/{step_id}{tail}") + + def record_scout_step(self, phase_id: str, step_id: str, status: str, + detail: str = "") -> NextAction: + """Record the outcome of a scout-assisted step (one the skill ran via MCP/ + browser, e.g. the CCOA lockdown check). + * status == "pass" -> mark the step done and let the flow continue. + * status == "attention" -> keep it held (needs the owner) with the detail + (e.g. a Production CCOA lockdown overlaps — the owner must shift CCD).""" + if not self._find_step(phase_id, step_id): + return NextAction(kind="idle", message=f"No such step: {phase_id}/{step_id}") + if status == "pass": + return self.complete_step(phase_id, step_id, detail) + # attention: leave the step outstanding, flagged for the owner. + self.state.set_step(phase_id, step_id, + StepState(status="blocked", note=detail, by="scout")) + self.state.status = "awaiting_action" + key = f"{phase_id}.{step_id}" + if key not in self.state.pending_human: + self.state.pending_human.append(key) + return NextAction(kind="reminder", phase=phase_id, step=step_id, + message=f"Needs your attention — {detail}") + + def reopen_step(self, phase_id: str, step_id: str, reason: str = "") -> NextAction: + """Undo a done/skipped step so the conductor runs it again. Reason optional.""" + if not self._find_step(phase_id, step_id): + return NextAction(kind="idle", message=f"No such step: {phase_id}/{step_id}") + self.state.steps.pop(self.state.key(phase_id, step_id), None) # remove -> pending + # drop any prior gate approval for this step so a gate re-holds + self.state.gate_decisions = [g for g in self.state.gate_decisions + if g.get("step") != f"{phase_id}.{step_id}"] + if self.state.status == "complete": + self.state.status = "running" + note = f" — {reason.strip()}" if reason and reason.strip() else "" + return NextAction(kind="ran", phase=phase_id, step=step_id, + message=f"Reopened {phase_id}/{step_id}{note}") + + def halt(self, reason: str) -> NextAction: + """Emergency hold — nothing advances until resume(). Reason REQUIRED (audit).""" + if not (reason and reason.strip()): + return NextAction(kind="idle", message="A reason is required to halt the release.") + self.state.halted = True + self.state.halt_reason = reason.strip() + self.state.status = "halted" + return NextAction(kind="halted", message=f"Release HALTED — {reason.strip()}") + + def resume(self, reason: str = "") -> NextAction: + """Clear an emergency halt. Reason optional.""" + if not self.state.halted: + return NextAction(kind="idle", message="Release is not halted.") + self.state.halted = False + self.state.halt_reason = None + self.state.status = "running" + note = f" — {reason.strip()}" if reason and reason.strip() else "" + return NextAction(kind="idle", message=f"Release resumed{note}.") + + # ---- gates ---- + def _gate_approved(self, phase: str, step: str) -> bool: + for gd in self.state.gate_decisions: + if gd.get("step") == f"{phase}.{step}" and gd.get("decision") == "approved": + return True + return False + + def approve_gate(self, comment: str = "") -> NextAction: + """Record approval for the current holding gate and continue.""" + phase = self.state.current_phase + step = self.state.current_step + if self.state.status != "holding_gate" or not phase: + return NextAction(kind="idle", message="No gate is currently holding.") + self.state.gate_decisions.append( + asdict_gate(GateDecision(step=f"{phase}.{step}", decision="approved", + at=_now(), comment=comment))) + # mark the gate step done and advance + self.state.set_step(phase, step, + StepState(status="done", completed_at=_now(), + note=f"Gate approved. {comment}".strip(), by="human")) + self.state.status = "running" + return NextAction(kind="ran", phase=phase, step=step, + message=f"Gate approved: {phase} → {step}. {comment}".strip()) + + def deny_gate(self, comment: str = "") -> NextAction: + phase = self.state.current_phase + step = self.state.current_step + if self.state.status != "holding_gate" or not phase: + return NextAction(kind="idle", message="No gate is currently holding.") + self.state.gate_decisions.append( + asdict_gate(GateDecision(step=f"{phase}.{step}", decision="denied", + at=_now(), comment=comment))) + self.state.pending_human.append(f"{phase}.{step} (denied: {comment})") + self.state.status = "blocked" + return NextAction(kind="gate", phase=phase, step=step, + message=f"Gate DENIED: {phase} → {step}. Release blocked. {comment}".strip()) + + # ---- reporting ---- + def _phase_included(self, phase: dict) -> bool: + return (not phase.get("conditional")) or phase["id"] in self._activated_conditionals() + + def _active_phase_report(self) -> Optional[dict]: + """The first incomplete included phase, with its outstanding steps and + whether its time-window is open (due). This is what the daily phase + notification reports on — independent of state.current_phase (which is + only set once the release has been advanced).""" + for phase in self.config["phases"]: + if not self._phase_included(phase): + continue + steps = phase["steps"] + done = sum(1 for s in steps if self.state.is_done(phase["id"], s["id"])) + if done == len(steps): + continue # phase complete — look at the next one + outstanding = [ + {"id": s["id"], "name": s["name"], "gate": bool(s.get("gate")), + "reminder": self._is_reminder(s), "owner": s.get("owner", "agent")} + for s in steps if not self.state.is_done(phase["id"], s["id"]) + ] + completed = [s["name"] for s in steps + if self.state.is_done(phase["id"], s["id"])] + cur = self.state.current_step + steps_view = [] + for s in steps: + sid = s["id"] + s_done = self.state.is_done(phase["id"], sid) + s_blocked = self.state.get_step(phase["id"], sid).status == "blocked" + is_gate = bool(s.get("gate")) + is_rem = self._is_reminder(s) + is_scout = s.get("source") == "scout" + is_attest = bool(s.get("attest")) + if s_done: + status = "done" + elif s_blocked: + status = "blocked" + elif is_gate: + status = "approval" + elif is_attest: + status = "confirm" + elif is_rem or is_scout: + status = "action" + else: + status = "auto" + needs = bool((is_gate or is_rem or is_scout or is_attest or s_blocked) and not s_done) + steps_view.append({ + "id": sid, "name": s["name"], "status": status, + "needs_owner": needs, + "now": bool(sid == cur and not s_done and (is_gate or is_rem or is_scout or is_attest or s_blocked)), + }) + opens = self._phase_anchor_date(phase) + return { + "id": phase["id"], "name": phase["name"], + "num": phase.get("checklist_phase"), + "done": done, "total": len(steps), + "due": self._phase_due(phase), "started": done > 0, + "opens": opens.isoformat() if opens else None, + "opens_in_days": (opens - self.as_of).days if opens else None, + "outstanding": outstanding, + "completed": completed, + "steps": steps_view, + } + return None + + def _phase_map(self): + """Build the phase overview + running totals. Returns + (phases, total, done, current_phase_name, current_phase_obj, current_step_name).""" + phases = [] + total = done = 0 + current_phase_name = current_step_name = None + current_phase_obj = None + for idx, phase in enumerate(self.config["phases"]): + if not self._phase_included(phase): + continue + p_total = len(phase["steps"]) + p_done = sum(1 for s in phase["steps"] if self.state.is_done(phase["id"], s["id"])) + total += p_total + done += p_done + is_current = self.state.current_phase == phase["id"] + due = self._phase_due(phase) + opens = self._phase_anchor_date(phase) + if p_total and p_done == p_total: + state = "done" + elif not due and p_done == 0: + state = "scheduled" + elif is_current or p_done > 0: + state = "current" + else: + state = "pending" + if is_current: + current_phase_name = phase["name"] + current_phase_obj = phase + phases.append({ + "id": phase["id"], "name": phase["name"], + "num": phase.get("checklist_phase", idx), + "done": p_done, "total": p_total, "state": state, + "current": is_current, + "anchor": phase.get("anchor"), + "opens": opens.isoformat() if opens else None, + "opens_in_days": (opens - self.as_of).days if opens else None, + }) + for s in phase["steps"]: + if s["id"] == self.state.current_step and phase["id"] == self.state.current_phase: + current_step_name = s["name"] + return phases, total, done, current_phase_name, current_phase_obj, current_step_name + + def _current_steps(self, current_phase_obj) -> list: + """The current phase's steps, each tagged with a display state.""" + if not current_phase_obj: + return [] + phase_due = self._phase_due(current_phase_obj) + out = [] + for s in current_phase_obj["steps"]: + rec = self.state.steps.get(self.state.key(current_phase_obj["id"], s["id"]), {}) or {} + if rec.get("status") == "skipped": + s_state = "skipped" + elif self.state.is_done(current_phase_obj["id"], s["id"]): + s_state = "done" + elif s["id"] == self.state.current_step and self.state.status == "holding_gate": + s_state = "gate" + elif s["id"] == self.state.current_step and self.state.status == "awaiting_action": + s_state = "reminder" + elif not phase_due: + s_state = "scheduled" + else: + s_state = "pending" + out.append({ + "id": s["id"], "name": s["name"], + "gate": bool(s.get("gate")), + "reminder": self._is_reminder(s), + "owner": s.get("owner", "agent"), + "state": s_state, + }) + return out + + def _hold_view(self, phase_name, step_name) -> dict: + """Detail of the current hold (gate or action-needed) — same shape for both.""" + return { + "phase": self.state.current_phase, + "phase_name": phase_name, + "step": self.state.current_step, + "step_name": step_name, + } + + def _scheduled_view(self) -> Optional[dict]: + """The phase we're waiting on the clock for — derived from the first + incomplete phase's due-ness, so `status` shows it even before `next`.""" + first_incomplete = next( + (p for p in self.config["phases"] + if self._phase_included(p) + and not all(self.state.is_done(p["id"], s["id"]) for s in p["steps"])), + None) + if (first_incomplete is None or self._phase_due(first_incomplete) + or self.state.status in ("complete", "halted", "blocked")): + return None + opens = self._phase_anchor_date(first_incomplete) + return { + "phase": first_incomplete["id"], + "phase_name": first_incomplete["name"], + "opens": opens.isoformat() if opens else None, + "opens_in_days": (opens - self.as_of).days if opens else None, + } + + def status_report(self) -> dict: + """Structured status — presentation layer (render.py) turns this into a view. + Deterministic; no formatting baked in. Assembled from focused builders: + phase map, current-phase steps, current hold, scheduled window, active phase.""" + (phases, total, done, current_phase_name, + current_phase_obj, current_step_name) = self._phase_map() + current_steps = self._current_steps(current_phase_obj) + + gate = action = None + if self.state.status == "holding_gate" and self.state.current_phase: + gate = self._hold_view(current_phase_name, current_step_name) + elif self.state.status == "awaiting_action" and self.state.current_phase: + action = self._hold_view(current_phase_name, current_step_name) + scheduled = self._scheduled_view() + + chk = self.gate.checklist() + return { + "release_id": self.state.release_id, + "status": self.state.status, + "dry_run": self.state.dry_run, + "owner_email": self.state.owner_email, + "owner_name": self.state.owner_name, + "ccd": self.state.ccd, + "ccd_source": self.state.ccd_source, + "ccd_conflict": self.state.ccd_conflict, + "as_of": self.as_of.isoformat(), + "skip_release": self.state.skip_release, + "readiness_signed": self.state.readiness_signed, + "readiness_pending": [i["id"] for i in chk["items"] if not i["satisfied"]], + "blocked": self.state.blocked, + "blocked_items": list(self.state.blocked_items), + "blocked_message": chk.get("blocked_message", ""), + "halted": self.state.halted, + "halt_reason": self.state.halt_reason, + "done": done, "total": total, + "percent": round(100 * done / total) if total else 0, + "phases": phases, + "current_phase": self.state.current_phase, + "current_phase_name": current_phase_name, + "current_step": self.state.current_step, + "current_step_name": current_step_name, + "current_steps": current_steps, + "gate": gate, + "action": action, + "scheduled": scheduled, + "active_phase": self._active_phase_report(), + "pending_human": list(self.state.pending_human), + "gate_decisions": len(self.state.gate_decisions), + "updated_at": self.state.updated_at, + } + + +def asdict_gate(gd: GateDecision) -> dict: + return {"step": gd.step, "decision": gd.decision, "at": gd.at, + "by": gd.by, "comment": gd.comment} diff --git a/release-agent/orchestrator/eventlog.py b/release-agent/orchestrator/eventlog.py new file mode 100644 index 00000000..7df01951 --- /dev/null +++ b/release-agent/orchestrator/eventlog.py @@ -0,0 +1,108 @@ +"""Event log — the per-release record we analyze to debug and improve. + +Scope: PER-RELEASE ONLY. Each release keeps its own complete log at + //events.jsonl +There is no machine-wide aggregate — a release is a self-contained unit and its +log travels with it. + +What it must capture to be useful for debugging: not just the engine commands, +but the actual INTERACTION — + * what Scout presented to the engineer (prompts, the rendered checklist, options) + * what the engineer chose / typed (their input) + * the engine's own events (steps, gate holds, decisions with drivers) + +Each line is one JSON object: + { ts, release_id, actor, source, event, ... } + source: "engine" (deterministic engine action) | "scout" (agent output) | "user" (engineer input) + +Logging is best-effort and MUST never break or alter the interaction. +""" +from __future__ import annotations +import json +import os +import getpass +from datetime import datetime, timezone + + +def _now() -> str: + return datetime.now(timezone.utc).isoformat() + + +def _actor() -> str: + try: + return getpass.getuser() + except Exception: + return os.getenv("USERNAME") or os.getenv("USER") or "unknown" + + +class EventLog: + def __init__(self, runs_root: str, release_id: str): + self.runs_root = runs_root + self.release_id = release_id + self.path = os.path.join(runs_root, release_id, "events.jsonl") + self.actor = _actor() + + def log(self, event: str, source: str = "engine", **fields) -> dict: + rec = {"ts": _now(), "release_id": self.release_id, "actor": self.actor, + "source": source, "event": event} + rec.update({k: v for k, v in fields.items() if v is not None}) + try: + os.makedirs(os.path.dirname(self.path), exist_ok=True) + with open(self.path, "a", encoding="utf-8") as fh: + fh.write(json.dumps(rec, ensure_ascii=False) + "\n") + except Exception: + pass # logging must never break the flow + return rec + + # convenience wrappers for the interaction layer (called by the skill via CLI) + def scout_said(self, text: str, kind: str = "message", options=None) -> dict: + return self.log("scout_output", source="scout", kind=kind, text=text, options=options) + + def user_said(self, text: str, kind: str = "input", choice=None) -> dict: + return self.log("user_input", source="user", kind=kind, text=text, choice=choice) + + def read(self, limit: int = None) -> list: + return _read_jsonl(self.path, limit) + + +def _read_jsonl(path: str, limit: int = None) -> list: + if not os.path.isfile(path): + return [] + out = [] + with open(path, "r", encoding="utf-8") as fh: + for ln in fh: + ln = ln.strip() + if not ln: + continue + try: + out.append(json.loads(ln)) + except Exception: + continue + return out[-limit:] if limit else out + + +def summarize(events: list) -> dict: + """Roll up a single release's events for quick debugging.""" + by_event, by_source = {}, {} + gate_decisions, declines, interactions = [], [], 0 + for e in events: + by_event[e.get("event")] = by_event.get(e.get("event"), 0) + 1 + by_source[e.get("source")] = by_source.get(e.get("source"), 0) + 1 + if e.get("source") in ("scout", "user"): + interactions += 1 + if e.get("event") in ("gate_approved", "gate_denied"): + gate_decisions.append({"phase": e.get("phase"), "step": e.get("step"), + "decision": "approved" if e["event"] == "gate_approved" else "denied", + "driver": e.get("driver"), "actor": e.get("actor"), "ts": e.get("ts")}) + if e.get("event") == "readiness_declined": + declines.append({"items": e.get("items"), "owner_blocked": e.get("owner_blocked"), + "driver": e.get("driver"), "ts": e.get("ts")}) + return { + "release": events[0]["release_id"] if events else None, + "total_events": len(events), + "by_source": by_source, + "by_event": by_event, + "interactions_logged": interactions, + "gate_decisions": gate_decisions, + "declines": declines, + } diff --git a/release-agent/orchestrator/infra.py b/release-agent/orchestrator/infra.py new file mode 100644 index 00000000..3da3ba1a --- /dev/null +++ b/release-agent/orchestrator/infra.py @@ -0,0 +1,177 @@ +"""Infrastructure preflight — verify (and auto-provision) everything the skill +needs on a machine BEFORE the tool-level requirements: CLIs, the launchers that +back MCP servers, and the MCP servers themselves registered into Scout's config. + +Design: + * CLI/host/python deps ("requirements") → shell-checked (same as before). + * MCP servers ("mcp_servers") → live in Scout's own config file + (~/.scout/m-mcp-servers.json), which loads at startup. We can't "install" them, + but we CAN register a missing one into that file (backing it up first) so it + loads on the next Scout restart. Each entry names its `provider` (a shell check + that the launcher exists) so we never register a server whose launcher is absent. + +This module is pure-Python and import-light so bootstrap.ps1 can call it via the +CLI (`python -m orchestrator.cli infra`). It only touches Scout's MCP config when +asked to register, and always backs it up. +""" +from __future__ import annotations + +import json +import os +import shutil +import subprocess +from datetime import datetime + +import yaml + + +def scout_mcp_config_path() -> str: + return os.path.join(os.path.expanduser("~"), ".scout", "m-mcp-servers.json") + + +def expand(s: str) -> str: + """Expand %VARS% / $VARS and ~ in a path string.""" + return os.path.expanduser(os.path.expandvars(s or "")) + + +def load_requirements(path: str) -> dict: + with open(path, "r", encoding="utf-8") as fh: + return yaml.safe_load(fh) or {} + + +def _shell_ok(cmd: str, timeout: int = 30) -> bool: + """True if the shell command exits 0. Used for CLI + provider checks.""" + if not cmd: + return False + try: + r = subprocess.run(cmd, shell=True, capture_output=True, text=True, timeout=timeout) + return r.returncode == 0 + except (subprocess.TimeoutExpired, OSError): + return False + + +def check_requirements(req: dict) -> list: + """Return [{id,name,ok,install}] for each shell-checkable requirement.""" + out = [] + for r in req.get("requirements", []): + out.append({ + "id": r.get("id", ""), "name": r.get("name", r.get("id", "?")), + "ok": _shell_ok(r.get("check", "")), "install": r.get("install", ""), + }) + return out + + +def _load_scout_config(path: str) -> dict: + if not os.path.exists(path): + return {"servers": {}} + with open(path, "r", encoding="utf-8") as fh: + data = json.load(fh) + data.setdefault("servers", {}) + return data + + +def _backup(path: str) -> str: + stamp = datetime.now().strftime("%Y%m%d-%H%M%S") + dst = f"{path}.bak-{stamp}" + shutil.copy2(path, dst) + return dst + + +def ensure_mcp_servers(req: dict, register: bool) -> list: + """Check each required MCP server against Scout's config. When `register` is + True, add any that are missing (whose provider launcher exists), backing up + the config once before the first write. + + Returns [{id,name,scout_key,status,detail}] where status is one of: + present | registered | would_register | provider_missing | launcher_missing + """ + servers = req.get("mcp_servers", []) + if not servers: + return [] + cfg_path = scout_mcp_config_path() + cfg = _load_scout_config(cfg_path) + existing = cfg.get("servers", {}) + results, dirty, backed_up = [], False, None + + for m in servers: + key = m.get("scout_key") or m.get("id") + name = m.get("name", key) + rec = {"id": m.get("id"), "name": name, "scout_key": key} + if key in existing: + results.append({**rec, "status": "present", "detail": "already in Scout config"}) + continue + # Not registered. Is its launcher present? + provider_ok = _shell_ok(m.get("provider", "")) + cmd = expand(m.get("command", "")) + launcher_ok = bool(cmd) and os.path.exists(cmd) + if not provider_ok and not launcher_ok: + results.append({**rec, "status": "provider_missing", + "detail": m.get("note", "provider/launcher not found")}) + continue + if not launcher_ok: + results.append({**rec, "status": "launcher_missing", + "detail": f"launcher not found at {cmd or '(unset)'}"}) + continue + if not register: + results.append({**rec, "status": "would_register", + "detail": "run bootstrap (or infra --register) to add it"}) + continue + # Build args, expanding any dynamic directive (e.g. Kusto known-services + # from a data list) so multi-cluster config stays pure data. + args = list(m.get("args", [])) + ks_from = m.get("known_services_from") + if ks_from: + clusters = req.get(ks_from, []) or [] + known = [{"service_uri": c.get("service_uri"), + "default_database": c.get("default_database"), + "description": c.get("description", "")} + for c in clusters if c.get("service_uri")] + if known: + args += ["--known-services", json.dumps(known)] + # Register it into the config (backup once). + if not backed_up and os.path.exists(cfg_path): + backed_up = _backup(cfg_path) + existing[key] = { + "builtin": False, + "config": {"name": name.split(" MCP")[0].strip() or key, + "type": "command", "command": cmd, "args": args}, + "tools": [], + } + dirty = True + results.append({**rec, "status": "registered", + "detail": "added to Scout config — RESTART Scout to load"}) + + if dirty: + os.makedirs(os.path.dirname(cfg_path), exist_ok=True) + tmp = cfg_path + ".tmp" + with open(tmp, "w", encoding="utf-8") as fh: + json.dump(cfg, fh, indent=2, ensure_ascii=False) + os.replace(tmp, cfg_path) + return results + + +def run(req_path: str, register: bool = True) -> dict: + """Full infra preflight. Returns a structured report. + + If Scout itself isn't present (~/.scout missing), MCP registration is skipped + (there's no config to write) and the report flags scout_missing — install Scout + first, then re-run. + """ + req = load_requirements(req_path) + reqs = check_requirements(req) + scout_present = os.path.isdir(os.path.join(os.path.expanduser("~"), ".scout")) + if scout_present: + mcps = ensure_mcp_servers(req, register) + else: + # can't register into a non-existent Scout config — report as pending + mcps = [{"id": m.get("id"), "name": m.get("name", m.get("id")), + "scout_key": m.get("scout_key") or m.get("id"), + "status": "scout_missing", "detail": "install Scout first"} + for m in req.get("mcp_servers", [])] + restart_needed = any(m["status"] == "registered" for m in mcps) + ok = (scout_present and all(r["ok"] for r in reqs) + and all(m["status"] in ("present", "registered") for m in mcps)) + return {"requirements": reqs, "mcp_servers": mcps, + "scout_present": scout_present, + "ok": ok, "restart_needed": restart_needed, + "scout_mcp_config": scout_mcp_config_path()} diff --git a/release-agent/orchestrator/phase_config.py b/release-agent/orchestrator/phase_config.py new file mode 100644 index 00000000..6bc73031 --- /dev/null +++ b/release-agent/orchestrator/phase_config.py @@ -0,0 +1,38 @@ +"""Per-phase config loader — one convention: config/.yaml. + +Each phase's config lives beside the phase and mirrors its code +(phases/agents/.py ↔ config/.yaml), so adding a phase is a +predictable recipe with no hard-coded filenames scattered across modules. + +Cross-cutting config (requirements.yaml, schedule.yaml, readiness.yaml) is +release-wide, NOT per-phase, and is loaded elsewhere — it does not go through here. + +Import-light (os + yaml only) so both the phase agents (phases/) and the CLI +command modules (orchestrator/commands/) can use it without a cycle. +""" +from __future__ import annotations + +import os + +import yaml + +_ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) + + +def phase_config_path(phase_id: str) -> str: + """Absolute path to a phase's config file: config/.yaml.""" + return os.path.join(_ROOT, "config", f"{phase_id}.yaml") + + +def load_phase_config(phase_id: str, section: str | None = None) -> dict: + """Load config/.yaml. With `section`, return just that top-level key + (e.g. load_phase_config('preflight', 'cg')); without it, the whole document. + Missing file or key → empty dict (agents fall back to their own defaults).""" + try: + with open(phase_config_path(phase_id), "r", encoding="utf-8") as fh: + data = yaml.safe_load(fh) or {} + except OSError: + data = {} + if section is not None: + return data.get(section, {}) or {} + return data diff --git a/release-agent/orchestrator/readiness.py b/release-agent/orchestrator/readiness.py new file mode 100644 index 00000000..2d35ccde --- /dev/null +++ b/release-agent/orchestrator/readiness.py @@ -0,0 +1,201 @@ +"""Release readiness entry gate (logic only — no presentation). + +A self-contained subsystem: given the readiness config (data) + the release +run-state, it computes the checklist, runs auto verifiers, records attestations +and declines, and decides whether the gate is signed/blocked. + +Returns structured data only. Rendering lives in render.py so a different +interface (web UI, other frontend) can present the same data its own way. + +Model: every item is equally required. The only distinction is WHO resolves it: + auto -> Scout verifies it programmatically (pass/fail) + attest -> the engineer confirms it +If any item is unsatisfied the gate is not signed. If the engineer declares they +cannot satisfy an item (decline), the gate is blocked until resolved / handed off. +""" +from __future__ import annotations + +from . import schedule +from .state import ReleaseState, _now + + +class ReadinessGate: + def __init__(self, config: dict, state: ReleaseState): + # config is the parsed readiness.yaml (or None if not configured) + self.config = config + self.state = state + + def _window(self, it: dict): + """Compute a CCD-relative window {start,end} for an item that declares + window_start_anchor / window_end_anchor, using the release CCD. Returns + None when the item has no window or CCD isn't set yet.""" + sa, ea = it.get("window_start_anchor"), it.get("window_end_anchor") + ccd = schedule.parse_date(self.state.ccd) + if not (sa and ea and ccd): + return None + try: + return {"start": schedule.anchor_date(ccd, sa).isoformat(), + "end": schedule.anchor_date(ccd, ea).isoformat()} + except ValueError: + return None + + # ---- queries ---- + def checklist(self) -> dict: + """The entry checklist as structured data (no formatting).""" + if not self.config: + return {"items": [], "signed": True, "title": "", "instructions": ""} + items = [] + for it in self.config.get("items", []): + rec = self.state.readiness_items.get(it["id"], {}) or {} + links = [] + for ln in it.get("links", []): + if isinstance(ln, dict): + links.append({"name": ln.get("name", ln.get("url")), "url": ln.get("url")}) + else: + links.append({"name": ln, "url": ln}) + items.append({ + "id": it["id"], "text": it["text"], + "label": it.get("label", it["id"]), + "detail": it.get("detail"), + "links": links, + "verify": it.get("verify", "attest"), # auto | attest (who resolves it) + "source": it.get("source"), # None (python) | "scout" (skill runs it via MCP) + "verifier": it.get("verifier"), + "team_id": it.get("team_id"), # for scout-assisted checks (e.g. on-call team) + "team_name": it.get("team_name"), + "cluster_uri": it.get("cluster_uri"), # for scout-assisted Kusto checks + "database": it.get("database"), + "required_servers": it.get("required_servers"), # for scout-assisted silent-perms check + "opt_out": it.get("opt_out", False), # soft item: may be waived ("degraded") and still satisfy + "window": self._window(it), # {start,end} for windowed attest items + "status": rec.get("status", "pending"), # pending | pass | fail | attested | unable + "message": rec.get("message"), + "checks": rec.get("checks", []), # per-check results for auto items + "satisfied": self._item_satisfied(it), + }) + return { + "title": self.config.get("title", "Release readiness"), + "instructions": self.config.get("instructions", ""), + "blocked_message": self.config.get("blocked_message", ""), + "items": items, + "auto_items": [i for i in items if i["verify"] == "auto"], + "attest_items": [i for i in items if i["verify"] == "attest"], + "signed": self.state.readiness_signed, + "signed_at": self.state.readiness_signed_at, + "blocked": self.state.blocked, + "blocked_items": list(self.state.blocked_items), + "all_satisfied": all(i["satisfied"] for i in items) if items else True, + } + + @property + def signed(self) -> bool: + return self.state.readiness_signed + + @property + def blocked(self) -> bool: + return self.state.blocked + + # ---- mutations ---- + def verify(self) -> dict: + """Run the AUTO verifiers. Each returns pass or fail — no half-measures. + Items with source: scout are skipped here — the skill runs those via its + MCP tools and records the result with record_check().""" + from phases.readiness_verifiers import get_verifier + if not self.config: + return self.checklist() + for it in self.config.get("items", []): + if it.get("verify") != "auto": + continue + if it.get("source") == "scout": + continue # skill-run (MCP) — not executable here + vf = get_verifier(it.get("verifier")) + if vf is None: + self.state.readiness_items[it["id"]] = {"status": "fail", + "message": f"no verifier '{it.get('verifier')}' registered", "at": _now()} + continue + res = vf(it, self.state.dry_run) + rec = {"status": res.status, "message": res.message, "at": _now()} + if getattr(res, "details", None): + rec["checks"] = res.details + self.state.readiness_items[it["id"]] = rec + self._refresh_signed() + return self.checklist() + + def record_check(self, item_id: str, status: str, message: str = "") -> dict: + """Record the result of a SCOUT-ASSISTED auto check (source: scout), which + the skill performs via its MCP tools (e.g. the ICM on-call lookup). Only + valid for auto items marked source: scout — a Python-verified auto item + (e.g. build_access) cannot be hand-recorded, and attest items use sign(). + + Status is normally 'pass' | 'fail'. Items marked `opt_out: true` (soft items + the user may proceed WITHOUT — e.g. silent_perms) also accept 'degraded', + which SATISFIES the gate while recording that they chose to proceed without + the capability (the downside is captured in `message`).""" + by_id = {it["id"]: it for it in (self.config or {}).get("items", [])} + it = by_id.get(item_id) + if it is None: + return {"error": f"no such readiness item '{item_id}'"} + if not (it.get("verify") == "auto" and it.get("source") == "scout"): + return {"error": f"'{item_id}' is not a scout-assisted auto item (cannot record a result for it)"} + valid = ("pass", "fail") + (("degraded",) if it.get("opt_out") else ()) + if status not in valid: + return {"error": f"status must be one of: {', '.join(valid)}"} + self.state.readiness_items[item_id] = { + "status": status, "message": message, "at": _now(), "source": "scout"} + self._refresh_signed() + return self.checklist() + + def sign(self, item_ids=None) -> dict: + """Attest human (attest) items and run auto verifiers. item_ids=None + attests every attest item. Auto items are only set by verification — + they cannot be hand-waved through. Signs when all items are satisfied.""" + if not self.config: + self.state.readiness_signed = True + self.state.readiness_signed_at = _now() + return self.checklist() + self.verify() # auto items (pass/fail) + all_items = self.config.get("items", []) + attest_ids = [it["id"] for it in all_items if it.get("verify", "attest") == "attest"] + targets = attest_ids if item_ids is None else [i for i in item_ids if i in attest_ids] + for iid in targets: + self.state.readiness_items[iid] = {"status": "attested", "at": _now()} + self._refresh_signed() + return self.checklist() + + def decline(self, item_ids: list) -> dict: + """The engineer declares they CANNOT satisfy one or more items. Every item + is required, so any declined item blocks the gate until resolved / handed off.""" + if not self.config: + return self.checklist() + by_id = {it["id"]: it for it in self.config.get("items", [])} + for iid in item_ids or []: + if iid not in by_id: + continue + self.state.readiness_items[iid] = {"status": "unable", "at": _now()} + self.state.blocked = True + if iid not in self.state.blocked_items: + self.state.blocked_items.append(iid) + self.state.readiness_signed = False + return self.checklist() + + def blocked_labels(self) -> list: + """Human labels for the currently-blocked item ids.""" + by_id = {it["id"]: it.get("label", it["id"]) for it in (self.config or {}).get("items", [])} + return [by_id.get(b, b) for b in self.state.blocked_items] + + # ---- internals ---- + def _item_satisfied(self, item: dict) -> bool: + status = (self.state.readiness_items.get(item["id"], {}) or {}).get("status") + if item.get("verify", "attest") == "auto": + if item.get("opt_out"): + # soft item: fully verified (pass) OR user chose to proceed (degraded) + return status in ("pass", "degraded") + return status == "pass" # auto is fully verified: pass or nothing + return status == "attested" # attest is human-confirmed + + def _refresh_signed(self) -> None: + items = self.config.get("items", []) if self.config else [] + if items and all(self._item_satisfied(it) for it in items): + if not self.state.readiness_signed: + self.state.readiness_signed = True + self.state.readiness_signed_at = _now() diff --git a/release-agent/orchestrator/registry.py b/release-agent/orchestrator/registry.py new file mode 100644 index 00000000..231e6ac5 --- /dev/null +++ b/release-agent/orchestrator/registry.py @@ -0,0 +1,77 @@ +"""Automation registry — tracks the Scout automations the orchestrator provisions +for a release, so they can be cleanly torn down at release close. + +The engine/CLI never call Scout's automation API (creating/deleting automations is +the skill's job via m_create_automation / m_delete_automation). This module only +RECORDS which automations exist, tagged by release + scope, so the skill knows +exactly what to remove at the end — nothing gets orphaned. + +Two scopes: + * shared — machine-wide, reused across releases (e.g. "Release push reminders"). + NOT torn down per release. + * release — provisioned for one release; removed when that release closes. + +Stored machine-wide at /_automations.json (gitignored runtime state). +""" +from __future__ import annotations + +import json +import os +from datetime import datetime, timezone + + +def _now() -> str: + return datetime.now(timezone.utc).isoformat() + + +class AutomationRegistry: + def __init__(self, runs_root: str): + self.runs_root = runs_root + self.path = os.path.join(runs_root, "_automations.json") + + def _load(self) -> list: + try: + with open(self.path, "r", encoding="utf-8") as fh: + data = json.load(fh) + return data if isinstance(data, list) else [] + except (OSError, ValueError): + return [] + + def _save(self, entries: list) -> None: + os.makedirs(self.runs_root, exist_ok=True) + tmp = self.path + ".tmp" + with open(tmp, "w", encoding="utf-8") as fh: + json.dump(entries, fh, indent=2) + os.replace(tmp, self.path) + + def register(self, auto_id: str, name: str, release: str = None, + shared: bool = False, purpose: str = "") -> dict: + """Record an automation (upsert by id). Shared automations store release=None.""" + entry = { + "id": auto_id, + "name": name, + "scope": "shared" if shared else "release", + "release": None if shared else release, + "purpose": purpose, + "registered_at": _now(), + } + entries = [e for e in self._load() if e.get("id") != auto_id] # upsert + entries.append(entry) + self._save(entries) + return entry + + def deregister(self, auto_id: str) -> bool: + entries = self._load() + kept = [e for e in entries if e.get("id") != auto_id] + self._save(kept) + return len(kept) != len(entries) + + def list(self, release: str = None, scope: str = None) -> list: + """List entries. `release` filters to that release's automations (scope + 'release' whose release matches). `scope` filters by 'shared'/'release'.""" + entries = self._load() + if release is not None: + entries = [e for e in entries if e.get("release") == release] + if scope is not None: + entries = [e for e in entries if e.get("scope") == scope] + return entries diff --git a/release-agent/orchestrator/render.py b/release-agent/orchestrator/render.py new file mode 100644 index 00000000..a8119eda --- /dev/null +++ b/release-agent/orchestrator/render.py @@ -0,0 +1,370 @@ +"""Presentation layer — turns structured data into text/markdown. + +Kept separate from the logic (engine.py / readiness.py) so a different interface +(web UI, TUI, another frontend) can consume the same structured data and present +it its own way. These are pure functions: data in, string out. No state, no IO. +""" +from __future__ import annotations + + +# ---- readiness entry gate (the frozen table) ---- +_ICON = {"pass": "✅", "attested": "✅", "fail": "❌", "unable": "⛔", + "degraded": "⚠️", "pending": "⬜"} +_STATUS_WORD = {"pass": "PASS", "attested": "Confirmed", "fail": "FAIL", + "unable": "Unable", "degraded": "Proceeding (not silent)", + "pending": "Outstanding"} + + +def _cell(s: str) -> str: + """Single-line, pipe-safe text for a markdown table cell.""" + return (s or "").replace("\n", " ").replace("|", "\\|").strip() + + +def readiness_table(chk: dict, release_id: str) -> str: + """Canonical markdown table for the entry gate (frozen layout). + `chk` is ReadinessGate.checklist().""" + if not chk.get("items"): + return "No readiness checklist configured." + lines = [ + f"### Readiness Entry Gate — {release_id}", + "", + _cell(chk["instructions"]), + "", + "**Type legend:** `[auto]` Scout verifies · `[attest]` you confirm", + "", + "| | Type | Item | Status |", + "|---|---|---|---|", + ] + for it in chk["items"]: + box = _ICON.get(it["status"], "⬜") + typ = "`[auto]`" if it["verify"] == "auto" else "`[attest]`" + label = it.get("label") or it["id"] + if it.get("checks"): + parts = [f"[{c['name']}]({c['url']}) {'✓' if c.get('ok') else '✗'}" + if c.get("url") else f"{c['name']} {'✓' if c.get('ok') else '✗'}" + for c in it["checks"]] + detail = " · ".join(parts) + else: + detail = it.get("detail") or it.get("text") or "" + win = it.get("window") + if win and win.get("start") and win.get("end"): + detail = f"{detail} (window: {win['start']} → {win['end']})".strip() + item_cell = f"**{_cell(label)}** — {_cell(detail)}" + lines.append(f"| {box} | {typ} | {item_cell} | {_STATUS_WORD.get(it['status'], it['status'])} |") + lines.append("") + + if chk["blocked"]: + labels = [next((i["label"] for i in chk["items"] if i["id"] == b), b) + for b in chk["blocked_items"]] + lines.append("⛔ **Blocked** — cannot start: " + ", ".join(labels) + ".") + lines.append(_cell(chk["blocked_message"])) + elif chk["signed"]: + lines.append("✅ **All items satisfied — entry gate cleared.** Ready to start Phase 0.") + else: + pending = [i["label"] for i in chk["items"] if not i["satisfied"]] + lines.append("**Outstanding:** " + ", ".join(pending)) + return "\n".join(lines) + + +# ---- release status ---- +# Plain-language labels for internal engine states (never show raw state names). +_STATE_LABEL = { + "not_started": "Not started", + "running": "In progress", + "scheduled": "Scheduled — waiting for the window to open", + "awaiting_action": "Action needed from you", + "holding_gate": "Waiting for your approval", + "readiness_gate": "Entry gate — checklist pending", + "blocked": "Blocked", + "halted": "Halted", + "complete": "Complete", +} +_PHASE_ICON = {"done": "✅", "current": "⏸", "pending": "⬜", "scheduled": "🗓"} +_STEP_ICON = {"done": "✅", "gate": "⏸", "reminder": "📌", "scheduled": "🗓", + "pending": "⬜", "skipped": "⏭️"} +_STEP_STATE_WORD = {"done": "Done", "gate": "Awaiting your approval", + "reminder": "Do this — then mark done", "scheduled": "Not open yet", + "pending": "Pending", "skipped": "Skipped"} + + +def status_view(r: dict) -> str: + """Human-readable status: next-action headline → phase map → current-phase steps. + `r` is Orchestrator.status_report().""" + mode = "DRY-RUN" if r["dry_run"] else "LIVE" + bars = 20 + filled = round(bars * r["done"] / r["total"]) if r["total"] else 0 + bar = "█" * filled + "░" * (bars - filled) + label = _STATE_LABEL.get(r["status"], r["status"]) + + lines = [ + f"## Release {r['release_id']} · {mode} · {r['done']}/{r['total']} ({r['percent']}%)", + f"`{bar}`", + ] + # Code Complete Date anchor line (when known). + if r.get("ccd"): + src = {"override": "override", "manual": "confirmed", + "default": "2nd Wednesday"}.get(r.get("ccd_source"), r.get("ccd_source") or "") + srctag = f" ({src})" if src else "" + skip = " · ⚠ release SKIP set in pipeline" if r.get("skip_release") else "" + lines.append(f"**Code Complete:** {r['ccd']}{srctag} · **today:** {r.get('as_of','')}{skip}") + if r.get("ccd_conflict"): + lines.append(f"⚠ **Confirm the date** — the pipeline override is **{r['ccd_conflict']}**, " + f"which differs from the 2nd-Wednesday default (**{r['ccd']}**). " + f"Which is the real Code Complete Date — the default, or the pipeline's?") + if r.get("owner_email"): + who = (f"{r['owner_name']} " if r.get("owner_name") else "") + f"{r['owner_email']}" + lines.append(f"**Owner:** {who}") + lines.append("") + + # 1) Next-action headline — the single most important thing. + if r.get("halted"): + rsn = f" — {r['halt_reason']}" if r.get("halt_reason") else "" + lines.append(f"⛔ **HALTED**{rsn}. Nothing advances until you resume.") + elif r["blocked"]: + lines.append(f"⛔ **Blocked** — cannot start: {', '.join(r['blocked_items'])}. " + "Resolve it, or hand the release to someone who can.") + elif not r["readiness_signed"]: + lines.append("▣ **Entry gate** — the readiness checklist isn't signed yet.") + elif r.get("scheduled"): + sc = r["scheduled"] + when = _delta_phrase(sc.get("opens_in_days")) + lines.append(f"🗓 **Scheduled** — **{sc['phase_name']}** opens **{sc.get('opens','')}** " + f"({when}). Nothing to do yet.") + elif r.get("action"): + a = r["action"] + lines.append(f"📌 **Action needed** — you need to: **{a['step_name']}** " + f"(Phase {_phase_num(r, a['phase'])} · {a['phase_name']}). Mark it done when complete.") + elif r["gate"]: + g = r["gate"] + lines.append(f"⏸ **Next: your decision** — approve or deny **{g['step_name']}** " + f"(Phase {_phase_num(r, g['phase'])} · {g['phase_name']}).") + elif r["status"] == "complete": + lines.append("✔ **Release complete.** All phases done.") + elif r["status"] == "not_started": + lines.append("▶ **Not started yet** — run the next step to begin.") + else: + # in progress, between gates + nxt = _next_pending_step(r) + if nxt: + lines.append(f"▶ **In progress** — next up: **{nxt}** " + f"({r.get('current_phase_name') or ''}).") + else: + lines.append(f"▶ **{label}.**") + + # 2) Phase map (overview). + if r.get("phases"): + lines += ["", "### Phases", "| | # | Phase | Done |", "|---|---|---|---|"] + for p in r["phases"]: + icon = _PHASE_ICON.get(p["state"], "⬜") + note = "" + if p["current"]: + note = " ← you are here" + elif p["state"] == "scheduled" and p.get("opens"): + note = f" · opens {p['opens']} ({_delta_phrase(p.get('opens_in_days'))})" + lines.append(f"| {icon} | {p['num']} | {p['name']}{note} | {p['done']}/{p['total']} |") + lines.append("✅ done · ⏸ in progress · 🗓 scheduled · ⬜ not started") + + # 3) Current-phase detail (drill-down). + if r.get("current_steps"): + lines += ["", f"### ▶ Current phase — {r.get('current_phase_name','')}", + "| | Step | State |", "|---|---|---|"] + for s in r["current_steps"]: + icon = _STEP_ICON.get(s["state"], "⬜") + word = _STEP_STATE_WORD.get(s["state"], s["state"]) + tag = " 🚦" if s["gate"] else (" 📌" if s.get("reminder") else "") + lines.append(f"| {icon} | {s['name']}{tag} | {word} |") + + return "\n".join(lines) + + +def _delta_phrase(days) -> str: + if days is None: + return "" + if days == 0: + return "today" + if days == 1: + return "tomorrow" + if days == -1: + return "yesterday" + return f"in {days} days" if days > 0 else f"{-days} days ago" + + +def _phase_num(r: dict, phase_id: str): + for p in r.get("phases", []): + if p["id"] == phase_id: + return p["num"] + return "?" + + +def _next_pending_step(r: dict): + for s in r.get("current_steps", []): + if s["state"] in ("pending", "gate"): + return s["name"] + return None + + +def notification_subject(r: dict) -> str: + """Email subject for the daily phase digest (empty if nothing to send).""" + ap = r.get("active_phase") + if not ap: + return f"Release {r.get('release_id','?')} — update" + return f"Release {r.get('release_id','?')} — Phase {ap.get('num')} status" + + +def notification(r: dict) -> str: + """The DAILY PHASE DIGEST emailed to the release owner, or "" to stay silent. + `r` is Orchestrator.status_report(). The push automation sends whatever this + returns; the once-per-day cadence is enforced by the CLI (last_notified_date). + + Model (established with the user): + * Setup (readiness + CCD) is interactive in Scout — NO push. So an unsigned + release, a blocked entry gate, or a halted release stay silent here. + * The FIRST push is a phase opening (Phase 0 at CCD-7). Nothing before it + (no pre-open heads-up). + * While a phase is open with outstanding steps, report its status daily. + * Each phase gets its own digest when it opens (general pattern). + """ + rid = r.get("release_id", "?") + if (r.get("halted") or r.get("blocked") or r.get("status") == "complete" + or not r.get("readiness_signed")): + return "" # setup / paused — no push + + ap = r.get("active_phase") + if not ap or not ap.get("due"): + return "" # nothing open yet (scheduled) — no push + + head = f"Release {rid} — Phase {ap['num']}: {ap['name']}" + lines = [head] + if not ap["started"]: + opened = f" (opened {ap['opens']})" if ap.get("opens") else "" + lines.append(f"Phase {ap['num']} has opened{opened} — {ap['total']} steps to work through, none done yet.") + else: + lines.append(f"Progress: {ap['done']} of {ap['total']} steps done.") + + # What the orchestrator has already handled automatically (this phase). + completed = ap.get("completed") or [] + if completed: + lines.append(f"Completed ({len(completed)}):") + for name in completed[:8]: + lines.append(f" ✓ {name}") + + # What needs the owner right now (a live hold), then the human touchpoints ahead. + if r.get("gate"): + lines.append(f"Waiting on your decision: {r['gate']['step_name']} (approve or deny).") + elif r.get("action"): + lines.append(f"Action needed now: {r['action']['step_name']} (do it, then mark done).") + + human = [o for o in ap.get("outstanding", []) if o["gate"] or o["reminder"]] + if human: + lines.append(f"Still needs you ({len(human)}):") + for o in human[:6]: + what = "your approval" if o["gate"] else "your action" + lines.append(f" • {o['name']} — {what}") + + lines.append("Open Scout to continue the release.") + return "\n".join(lines) + + +# ---- HTML digest (nice email UX) ------------------------------------------- +# Email-safe: inline styles + table layout (Outlook-friendly), no external CSS. + +def _esc(s: str) -> str: + return (str(s or "").replace("&", "&").replace("<", "<") + .replace(">", ">").replace('"', """)) + + +# Per-status pill styling (label, text color, background). +_PILL = { + "done": ("✓ Done", "#1a7f37", "#e6f4ea"), + "now": ("⚠ Needs you now", "#b42318", "#fef3f2"), + "blocked": ("⛔ Blocked — fix & rerun", "#b42318", "#fef3f2"), + "approval": ("Your approval", "#b54708", "#fffaeb"), + "confirm": ("Your confirmation", "#b54708", "#fffaeb"), + "action": ("Your action", "#b54708", "#fffaeb"), + "auto": ("Automatic — pending", "#475467", "#f2f4f7"), +} + + +def _pill(status: str) -> str: + label, fg, bg = _PILL.get(status, _PILL["auto"]) + return (f'{label}') + + +def notification_html(r: dict) -> str: + """HTML version of the daily phase digest. Returns "" under the exact same + silence rules as notification() (reuses it as the guard). Presents EVERY step + in the active phase with a status pill, and flags what needs the owner now.""" + if not notification(r): # same silence rules / dedup gate + return "" + rid = _esc(r.get("release_id", "?")) + ap = r.get("active_phase") or {} + phase_title = _esc(f"Phase {ap.get('num')}: {ap.get('name','')}") + done, total = ap.get("done", 0), ap.get("total", 0) + pct = round(100 * done / total) if total else 0 + steps = ap.get("steps", []) + dry = r.get("dry_run") + + # The single item that needs the owner right now (the live hold), if any. + hold = r.get("gate") or r.get("action") + hold_name = _esc(hold["step_name"]) if hold else "" + hold_kind = "approve or deny" if r.get("gate") else "do it, then mark it done" + + # Rows: every step, with the active hold promoted to the "now" pill. + rows = [] + for s in steps: + st = "now" if s.get("now") else s.get("status", "auto") + name = _esc(s.get("name", "")) + star = (' ⚑' + if s.get("needs_owner") else "") + rows.append( + f'{name}{star}' + f'{_pill(st)}' + ) + rows_html = "\n".join(rows) + + attention = "" + if hold: + attention = ( + f'' + f'
' + f'⚑ Needs your attention: {hold_name} ' + f'— {hold_kind}.
' + ) + + dry_badge = ( + 'DRY-RUN' + if dry else "") + + return f"""\ +
+ + + + {attention} + + +
+
Release {rid}{dry_badge}
+
{phase_title}
+
+
Progress: {done} of {total} steps done ({pct}%)
+
+
+
+
+ + + + {rows_html} +
TaskStatus
+
+
Items marked ⚑ need you. Open Scout to continue the release.
+
+
""" + diff --git a/release-agent/orchestrator/schedule.py b/release-agent/orchestrator/schedule.py new file mode 100644 index 00000000..5c5deac6 --- /dev/null +++ b/release-agent/orchestrator/schedule.py @@ -0,0 +1,105 @@ +"""Schedule math for CCD-anchored phases — pure functions, no IO. + +The Code Complete Date (CCD) is the anchor the whole release hangs off of. +This module mirrors the logic of ADO pipeline 3038 "Code Complete Calendar +Checker" so the orchestrator resolves the *same* date the pipeline would: + + * Default : the 2nd Wednesday of the release month. + * Override: a full YYYY-MM-DD, but only if it belongs to the release month + (a stale cross-month override is ignored, exactly like the pipeline). + +Phases anchor to CCD via a spec like "CCD-7" (7 days before) or "CCD+1". +Kept IO-free so it stays deterministic and unit-testable; reading/writing the +pipeline variable lives in tools/checks.py, and CCD is stored on ReleaseState. +""" +from __future__ import annotations + +import calendar +import re +from datetime import date, timedelta +from typing import Optional + + +def parse_release_month(release_id: str) -> Tuple[int, int]: + """'2026-07' -> (2026, 7).""" + parts = release_id.split("-") + return int(parts[0]), int(parts[1]) + + +def second_wednesday(year: int, month: int) -> date: + """The 2nd Wednesday of the month (pipeline 3038's default rule).""" + weeks = calendar.monthcalendar(year, month) + wednesdays = [w[calendar.WEDNESDAY] for w in weeks if w[calendar.WEDNESDAY] != 0] + return date(year, month, wednesdays[1]) + + +def parse_date(s: Optional[str]) -> Optional[date]: + """Parse 'YYYY-MM-DD' -> date, or None if empty/invalid.""" + if not s or not str(s).strip(): + return None + try: + y, m, d = (int(x) for x in str(s).strip().split("-")) + return date(y, m, d) + except (ValueError, TypeError): + return None + + +def default_ccd(release_id: str) -> date: + """The canonical CCD for a release month: the 2nd Wednesday. This is the + source of truth — a differing pipeline override is treated as a *question* + to confirm, not a value to adopt silently.""" + year, month = parse_release_month(release_id) + return second_wednesday(year, month) + + +def pipeline_conflict(release_id: str, override: Optional[str], stored_ccd: Optional[str] = None): + """Return the pipeline override date IF it is a valid in-month date that + DIFFERS from our reference CCD — i.e. a divergence the user must resolve. + Otherwise None (override empty, cross-month, or already in agreement). + + The reference is `stored_ccd` when provided (what this release is anchored + to), else the 2nd-Wednesday default (used at init before anything is stored). + """ + od = parse_date(override) + if not od: + return None + year, month = parse_release_month(release_id) + if (od.year, od.month) != (year, month): + return None # month-scoped, like the pipeline + reference = parse_date(stored_ccd) or default_ccd(release_id) + return od if od != reference else None + + +_ANCHOR_RE = re.compile(r"^CCD\s*([+-]\s*\d+)?$", re.IGNORECASE) + + +def anchor_offset(spec: str) -> int: + """'CCD-7' -> -7, 'CCD+1' -> 1, 'CCD' -> 0.""" + m = _ANCHOR_RE.match((spec or "").strip()) + if not m: + raise ValueError(f"bad anchor spec: {spec!r} (expected e.g. 'CCD-7')") + grp = m.group(1) + return int(grp.replace(" ", "")) if grp else 0 + + +def anchor_date(ccd: date, spec: str) -> date: + """The calendar date a phase with `spec` opens, given CCD.""" + return ccd + timedelta(days=anchor_offset(spec)) + + +def today() -> date: + """'Now' at date granularity. `--as-of` overrides this for testable dry-runs.""" + return date.today() + + +def humanize_delta(days: int) -> str: + """'in 3 days' / 'today' / '2 days ago' — for countdowns.""" + if days == 0: + return "today" + if days == 1: + return "tomorrow" + if days == -1: + return "yesterday" + if days > 0: + return f"in {days} days" + return f"{-days} days ago" diff --git a/release-agent/orchestrator/state.py b/release-agent/orchestrator/state.py new file mode 100644 index 00000000..4d08006a --- /dev/null +++ b/release-agent/orchestrator/state.py @@ -0,0 +1,112 @@ +"""Release Orchestrator — run-state model (X5). + +Two kinds of state, per the architecture: + * DERIVED : recomputed from systems of record (ADO/Git/Play Console/ADX). Never stored here. + * PERSISTED: decisions/intent, step completion, pending human actions, notes. + Stored in release-state.json (the Release State Record). + +This module owns ONLY the persisted state. Reconcile-on-resume (deriving live +state) is a separate concern handled by tools/reconcile.py (stubbed for now). +""" +from __future__ import annotations +import json +import os +from dataclasses import dataclass, field, asdict +from datetime import datetime, timezone +from typing import Optional + + +SCHEMA_VERSION = 1 + + +def _now() -> str: + return datetime.now(timezone.utc).isoformat() + + +@dataclass +class StepState: + """Persisted state for a single step.""" + status: str = "pending" # pending | done | skipped | blocked + completed_at: Optional[str] = None + note: Optional[str] = None + by: Optional[str] = None # 'agent' (stub) or 'human' + + +@dataclass +class GateDecision: + """A recorded human decision at a gate (audit trail).""" + step: str + decision: str # approved | denied | held + at: str + by: str = "human" + comment: Optional[str] = None + + +@dataclass +class ReleaseState: + """The Release State Record — one per monthly release.""" + schema_version: int = SCHEMA_VERSION + release_id: str = "" # e.g. 2026-07 + created_at: str = field(default_factory=_now) + updated_at: str = field(default_factory=_now) + dry_run: bool = True + # Release owner — the engineer running this release (release metadata). The + # push reminders email this address; resolved from the signed-in user at init. + owner_email: Optional[str] = None + owner_name: Optional[str] = None + # Code Complete Date — the anchor the phases hang off of (orchestrator's truth, + # seeded from / written back to pipeline 3038). ccd is 'YYYY-MM-DD'. + ccd: Optional[str] = None + ccd_source: Optional[str] = None # 'default' (2nd Wed) | 'override' | 'manual' + ccd_conflict: Optional[str] = None # a pipeline override date that DIFFERS from ccd (unresolved) + skip_release: bool = False # mirrors the pipeline 'skipRelease' switch (display) + # readiness entry gate (must be signed before Phase 0) + readiness_signed: bool = False + readiness_signed_at: Optional[str] = None + readiness_items: dict = field(default_factory=dict) # item_id -> {status,...} + blocked: bool = False # an item was declared unsatisfiable + blocked_items: list = field(default_factory=list) + # manual overrides (human-driven transitions, §7.1) + halted: bool = False # emergency hold + halt_reason: Optional[str] = None + # cursor + current_phase: Optional[str] = None + current_step: Optional[str] = None + status: str = "not_started" # not_started | running | scheduled | awaiting_action | holding_gate | halted | blocked | complete + # persisted detail + steps: dict = field(default_factory=dict) # "phase.step" -> StepState (as dict) + gate_decisions: list = field(default_factory=list) + pending_human: list = field(default_factory=list) # outstanding human actions + last_notified: Optional[str] = None # last push message (legacy; kept for load compat) + last_notified_date: Optional[str] = None # YYYY-MM-DD of the last daily digest sent + notes: list = field(default_factory=list) + + # ---- persistence ---- + @classmethod + def load(cls, path: str) -> "ReleaseState": + with open(path, "r", encoding="utf-8") as fh: + data = json.load(fh) + return cls(**data) + + def save(self, path: str) -> None: + self.updated_at = _now() + os.makedirs(os.path.dirname(path), exist_ok=True) + tmp = path + ".tmp" + with open(tmp, "w", encoding="utf-8") as fh: + json.dump(asdict(self), fh, indent=2) + os.replace(tmp, path) + + # ---- step helpers ---- + @staticmethod + def key(phase: str, step: str) -> str: + return f"{phase}.{step}" + + def get_step(self, phase: str, step: str) -> StepState: + raw = self.steps.get(self.key(phase, step)) + return StepState(**raw) if raw else StepState() + + def set_step(self, phase: str, step: str, state: StepState) -> None: + self.steps[self.key(phase, step)] = asdict(state) + + def is_done(self, phase: str, step: str) -> bool: + return self.get_step(phase, step).status in ("done", "skipped") diff --git a/release-agent/phases/__init__.py b/release-agent/phases/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/release-agent/phases/agents/__init__.py b/release-agent/phases/agents/__init__.py new file mode 100644 index 00000000..ce2f9be8 --- /dev/null +++ b/release-agent/phases/agents/__init__.py @@ -0,0 +1,38 @@ +"""Phase-agent registry — aggregates every phase's agents into one lookup. + +Each phase's real agents live in `phases/agents/.py`, which exposes a +module-level `REGISTRY = {agent_id: run(phase_id, step, dry_run, state) -> StepResult}`. +This package merges them all into a single `REGISTRY` so the engine does ONE +lookup, and guards against two phases claiming the same agent id. + +Adding a phase's agents is a one-line change: create `phases/agents/.py` +with a `REGISTRY`, then add its name to `_PHASE_MODULES` below. The engine never +changes — it already looks up the merged registry. +""" +from __future__ import annotations + +from importlib import import_module + +from phases.stub_runner import get_runner as _stub_get_runner + +# Phase agent modules, in phase order. Add a new phase's module name here. +_PHASE_MODULES = [ + "preflight", +] + +REGISTRY: dict = {} +for _name in _PHASE_MODULES: + _mod = import_module(f"{__name__}.{_name}") + for _agent_id, _runner in getattr(_mod, "REGISTRY", {}).items(): + if _agent_id in REGISTRY: + raise RuntimeError( + f"duplicate phase-agent id '{_agent_id}': phases.agents.{_name} " + f"collides with an agent already registered by an earlier phase") + REGISTRY[_agent_id] = _runner + + +def get_runner(agent_id: str): + """The ONE dispatch seam the engine uses: return the real phase agent for + `agent_id` if one is registered, otherwise the stub runner (which handles + unbuilt steps). Always returns a callable with the run(...) contract.""" + return REGISTRY.get(agent_id) or _stub_get_runner(agent_id) diff --git a/release-agent/phases/agents/preflight.py b/release-agent/phases/agents/preflight.py new file mode 100644 index 00000000..d7d41d55 --- /dev/null +++ b/release-agent/phases/agents/preflight.py @@ -0,0 +1,311 @@ +"""Real Phase-0 pre-flight agents (deterministic; az CLI / HTTP). + +These replace the `stub` for specific Phase-0 steps with genuine actions: + + * breaking_detect (step `breaking`, S2) — reads the common-for-android + changelog, finds breaking ([MAJOR]) changes in the unreleased section, + and drafts the OneAuth comms. Read-only. + * wiki_payload (step `wiki`, S0) — creates the per-release payload + wiki subpage under the standing history page. A real ADO write. + +Contract (shared with the stub): run(phase_id, step, dry_run, state) -> StepResult + +Dry-run safety: in a dry-run release every agent SIMULATES — no network, no +writes — so tests and dry-run rehearsals never touch production. Only a real +(dry_run=False) release performs the action. +""" +from __future__ import annotations +from urllib import request as _request + +from phases.stub_runner import StepResult + + +# ---- config ---------------------------------------------------------------- +def _load_cfg() -> dict: + """Phase-0 config (config/preflight.yaml), via the shared per-phase loader.""" + from orchestrator.phase_config import load_phase_config + return load_phase_config("preflight") + + +def _fetch_text(url: str, timeout: int = 20) -> str: + req = _request.Request(url, headers={"User-Agent": "release-agent-preflight/1.0"}) + with _request.urlopen(req, timeout=timeout) as resp: + return resp.read().decode("utf-8", "replace") + + +# ---- breaking-change detection (pure, unit-tested) ------------------------- +def parse_breaking(changelog_text: str, section: str = "vNext", + tag: str = "[MAJOR]") -> list: + """Return the list of `tag` (breaking) entry lines inside `section`. + + The changelog is a flat text file: a section header line (e.g. "vNext"), + an underline, then `- [SEVERITY] ... (#PR)` bullets, until the next + "Version X.Y.Z" header. We scan only the requested section. + """ + entries, in_section = [], False + for raw in changelog_text.splitlines(): + s = raw.strip() + if not in_section: + if s == section: + in_section = True + continue + if s.startswith("Version "): + break + if tag in raw: + entries.append(s) + return entries + + +def _draft_breaking_comms(entries: list, state=None) -> str: + release = getattr(state, "release_id", None) or "this release" + bullets = "\n".join(f"- {e}" for e in entries) + return ( + f"Subject: [Action] Breaking OneAuth changes in {release}\n\n" + f"Hi OneAuth team,\n\n" + f"The upcoming Android common release ({release}) contains the following " + f"breaking change(s). Please review for downstream impact before code " + f"complete:\n\n{bullets}\n\n" + f"Thanks,\nRelease Orchestrator" + ) + + +def run_breaking(phase_id: str, step: dict, dry_run: bool, state=None) -> StepResult: + cfg = _load_cfg().get("breaking", {}) + url = cfg.get("changelog_url") + section = cfg.get("section", "vNext") + tag = cfg.get("breaking_tag", "[MAJOR]") + if dry_run: + return StepResult( + True, + f"[dry-run] Would scan the '{section}' changelog section for {tag} " + f"(breaking) entries and draft OneAuth comms.", + "agent", + ) + if not url: + return StepResult(False, "breaking: no changelog_url configured", "agent") + try: + text = _fetch_text(url) + except Exception as e: # noqa: BLE001 - network/parse errors -> hold for human + return StepResult(False, f"breaking: could not fetch changelog ({e})", "agent") + entries = parse_breaking(text, section, tag) + if not entries: + return StepResult( + True, f"No breaking ({tag}) changes in '{section}' — no OneAuth comms needed.", + "agent", + ) + listing = "\n".join(f" - {e}" for e in entries) + draft = _draft_breaking_comms(entries, state) + return StepResult( + True, + f"Detected {len(entries)} breaking ({tag}) change(s) in '{section}':\n{listing}\n\n" + f"--- DRAFT COMMS (send to OneAuth) ---\n{draft}", + "agent", + ) + + +# ---- payload wiki subpage -------------------------------------------------- +def _payload_template(state=None) -> str: + release = getattr(state, "release_id", None) or "unknown" + ccd = getattr(state, "ccd", None) or "TBD" + owner = getattr(state, "owner_name", None) or getattr(state, "owner_email", None) or "TBD" + return ( + f"# {release} — Release Payload\n\n" + f"| Field | Value |\n|---|---|\n" + f"| Release | {release} |\n| Code Complete Date | {ccd} |\n| Release owner | {owner} |\n\n" + f"## Built versions\n\n" + f"_Filled during Build & Lib Verification (Phase 2)._\n\n" + f"| Artifact | Version |\n|---|---|\n| | |\n" + ) + + +def _page_name(state, n: int = 1) -> str: + """Payload page name: ' Release', e.g. 'August 2026 Release'. + A numbered variant ('August 2026 2 Release') is used when a page already + exists for the month (n >= 2).""" + import calendar + release = getattr(state, "release_id", None) or "unknown" + try: + year, month = release.split("-")[:2] + base = f"{calendar.month_name[int(month)]} {int(year)}" + except Exception: # noqa: BLE001 - fall back to the raw id + base = release + return f"{base} {n} Release" if n and n >= 2 else f"{base} Release" + + +def run_wiki(phase_id: str, step: dict, dry_run: bool, state=None) -> StepResult: + cfg = _load_cfg().get("wiki", {}) + org = cfg.get("org") + project = cfg.get("project") + wiki = cfg.get("wiki") + parent = (cfg.get("parent_path") or "").rstrip("/") + base_name = _page_name(state) + base_path = f"{parent}/{base_name}" + if dry_run: + return StepResult( + True, + f"[dry-run] Would create payload wiki subpage '{base_name}' under " + f"'{parent}' (duplicate-safe: a second numbered page if it already exists).", + "agent", + ) + if not (org and project and wiki and parent): + return StepResult(False, "wiki: incomplete configuration", "agent") + from tools.checks import create_wiki_page, wiki_page_exists + + # Duplicate handling: if the month's page already exists, DON'T overwrite — + # notify and create the next free " N Release" page instead. + if wiki_page_exists(org, project, wiki, base_path): + n = 2 + while n <= 50: + cand_name = _page_name(state, n) + cand_path = f"{parent}/{cand_name}" + if not wiki_page_exists(org, project, wiki, cand_path): + res = create_wiki_page(org, project, wiki, cand_path, _payload_template(state)) + if not res.ok: + return StepResult(False, f"wiki: could not create '{cand_path}' — {res.detail}", "agent") + return StepResult( + True, + f"⚠ A payload page already exists for this month ('{base_name}'). " + f"Left it untouched and created a SECOND page: '{cand_name}'. ({res.detail})", + "agent", + ) + n += 1 + return StepResult(False, f"wiki: too many existing pages for '{base_name}'", "agent") + + res = create_wiki_page(org, project, wiki, base_path, _payload_template(state)) + if not res.ok: + return StepResult(False, f"wiki: could not create '{base_path}' — {res.detail}", "agent") + return StepResult(True, f"Payload wiki subpage ready: '{base_name}' ({res.detail})", "agent") + + +# ---- Component Governance alerts (report-only) ----------------------------- +def _cg_summary(alerts: list, high_sev: list): + """Return (active, high) lists from raw alerts. `high` = active alerts whose + severity is in high_sev (critical/high).""" + active = [a for a in alerts if (a.get("alertState") or "").lower() == "active"] + high = [a for a in active if (a.get("severity") or "").lower() in + [s.lower() for s in high_sev]] + return active, high + + +def _cg_report(active: list, high: list) -> str: + from collections import Counter + by_sev = Counter((a.get("severity") or "unknown").lower() for a in active) + counts = ", ".join(f"{n} {s}" for s, n in sorted(by_sev.items(), + key=lambda kv: (-kv[1], kv[0]))) or "none" + lines = [f"Component Governance: {len(active)} active alert(s) — {counts}."] + if high: + lines.append(f"High/Critical ({len(high)}) — review before release:") + for a in high[:15]: + comp = (a.get("component") or {}).get("displayName") or "" + ver = (a.get("component") or {}).get("displayVersion") or "" + rec = (a.get("actionItems") or "").strip().split("\n")[0] + title = a.get("title") or a.get("summary") or "?" + sev = (a.get("severity") or "").capitalize() + comp_str = f" — {comp} {ver}".rstrip() if comp else "" + lines.append(f" • [{sev}] {title}{comp_str}" + (f" — {rec}" if rec else "")) + else: + lines.append("No High/Critical active alerts.") + return "\n".join(lines) + + +def run_cg_alerts(phase_id: str, step: dict, dry_run: bool, state=None) -> StepResult: + """Report active Component Governance alerts. Passes when there are no active + High/Critical alerts; BLOCKS (holds) when there are — the owner must fix the + issues and RERUN this step (re-checks), or override by skipping it.""" + cfg = _load_cfg().get("cg", {}) + if dry_run: + return StepResult( + True, + "[dry-run] Would query Component Governance for active alerts and " + "report counts + High/Critical items (blocking on High/Critical).", + "agent", + ) + required = ("resource", "governance_host", "project_id", "governed_repo_id", "branch") + if not all(cfg.get(k) for k in required): + return StepResult(False, "cg: incomplete configuration", "agent") + from tools.checks import fetch_cg_alerts + ok, alerts, detail = fetch_cg_alerts( + cfg["resource"], cfg["governance_host"], cfg["project_id"], + cfg["governed_repo_id"], cfg["branch"]) + if not ok: + return StepResult(False, f"cg: could not read alerts — {detail}", "agent") + active, high = _cg_summary(alerts, cfg.get("high_severities", ["critical", "high"])) + report = _cg_report(active, high) + if high: + # Block: the report is shown, the step holds. The owner fixes the alerts + # and reruns this step (re-checks), or skips to override. + return StepResult( + False, + report + "\n→ Fix the High/Critical alerts (or wait for remediation), then " + "RERUN this step to re-check — or skip to override with a reason.", + "agent", + ) + return StepResult(True, report, "agent") + + +# ---- Calendar Checker schedule verification -------------------------------- +def _iso_age_days(iso: str): + """Whole days between an ISO-8601 timestamp and now (UTC), or None if unparseable.""" + from datetime import datetime, timezone + if not iso: + return None + try: + s = iso.replace("Z", "+00:00") + dt = datetime.fromisoformat(s) + if dt.tzinfo is None: + dt = dt.replace(tzinfo=timezone.utc) + return (datetime.now(timezone.utc) - dt).days + except ValueError: + return None + + +def run_cron_check(phase_id: str, step: dict, dry_run: bool, state=None) -> StepResult: + """Verify the Calendar Checker pipeline is scheduled AND firing, by confirming a + recent `schedule`-reason run. Passes if a scheduled run is within the staleness + window; BLOCKS (fix + rerun, or skip) if there's none or it's stale.""" + cfg = _load_cfg().get("cron", {}) + name = cfg.get("name", "Calendar Checker") + if dry_run: + return StepResult( + True, + f"[dry-run] Would verify '{name}' (pipeline {cfg.get('pipeline_id')}) has a " + f"recent scheduled run.", + "agent", + ) + if not all(cfg.get(k) for k in ("pipeline_id", "org", "project")): + return StepResult(False, "cron: incomplete configuration", "agent") + from tools.checks import latest_scheduled_build + ok, run, detail = latest_scheduled_build(cfg["org"], cfg["project"], cfg["pipeline_id"]) + if not ok: + return StepResult(False, f"cron: could not read build history — {detail}", "agent") + if not run: + return StepResult( + False, + f"{name}: no scheduled run found in recent history — the cron may be " + f"disabled. Investigate, then rerun this step (or skip to override).", + "agent", + ) + age = _iso_age_days(run.get("queueTime")) + max_stale = cfg.get("max_staleness_days", 2) + when = (run.get("queueTime") or "")[:16] + if age is not None and age > max_stale: + return StepResult( + False, + f"{name}: last scheduled run was {when} ({age}d ago) — stale (> {max_stale}d). " + f"The schedule may be broken. Fix + rerun this step, or skip to override.", + "agent", + ) + return StepResult( + True, + f"{name} is scheduled and firing — last scheduled run {when} ({run.get('result')}).", + "agent", + ) + + +# ---- registry -------------------------------------------------------------- +REGISTRY = { + "breaking_detect": run_breaking, + "wiki_payload": run_wiki, + "cg_alerts": run_cg_alerts, + "cron_check": run_cron_check, +} diff --git a/release-agent/phases/readiness_verifiers.py b/release-agent/phases/readiness_verifiers.py new file mode 100644 index 00000000..d5d7a0eb --- /dev/null +++ b/release-agent/phases/readiness_verifiers.py @@ -0,0 +1,87 @@ +"""Readiness AUTO verifiers. + +An auto verifier must FULLY verify its item — it returns pass or fail, never a +half-measure. If something cannot be fully proven programmatically, it must NOT +be an auto item (make it an attest item in readiness.yaml instead). + +Contract: verify(item, dry_run) -> VerifyResult(status, message) + status: "pass" | "fail" +""" +from __future__ import annotations +from dataclasses import dataclass +import os + + +@dataclass +class VerifyResult: + status: str # "pass" | "fail" + message: str + details: list = None # optional per-check breakdown: [{name, url, ok, detail}] + + @property + def ok(self) -> bool: + return self.status == "pass" + + +def verify_build_defs(item: dict, dry_run: bool) -> VerifyResult: + """Confirm the engineer can access every configured ADO build definition, + using `az pipelines build definition show`. Fully verified access — pass/fail. + Returns per-check details (name, url, ok) so the display can link each one.""" + from tools.checks import check_ado_build_def + + checks = [c for c in item.get("checks", []) if c.get("type") == "ado_build_def"] + if not checks: + return VerifyResult("fail", "no build definitions configured to check", []) + details, any_fail = [], False + for c in checks: + r = check_ado_build_def(c["org"], c["project"], c["id"]) + details.append({"name": c.get("name", str(c["id"])), "url": c.get("url"), + "ok": r.ok, "detail": r.detail}) + if not r.ok: + any_fail = True + msg = "; ".join(f"{'OK' if d['ok'] else 'FAIL'} {d['name']}" for d in details) + return VerifyResult("fail" if any_fail else "pass", msg, details) + + +def verify_mcp_servers(item: dict, dry_run: bool) -> VerifyResult: + """Confirm every MCP server the skill needs (ICM, Kusto/ADX) is registered in + Scout's config. Reuses the infra preflight in READ-ONLY mode (register=False) + against config/requirements.yaml — the single source of truth for MCP deps. + Fully verified: pass only if all are `present`, else fail listing the missing + ones (fix = run bootstrap / `infra --register`, then RESTART Scout).""" + from orchestrator import infra + + root = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) + req_path = os.path.join(root, "config", "requirements.yaml") + try: + req = infra.load_requirements(req_path) + except OSError as e: + return VerifyResult("fail", f"cannot read requirements.yaml: {e}", []) + results = infra.ensure_mcp_servers(req, register=False) + if not results: + return VerifyResult("fail", "no MCP servers configured to check", []) + details, missing = [], [] + for r in results: + ok = r.get("status") == "present" + details.append({"name": r.get("name", r.get("scout_key")), "url": None, + "ok": ok, "detail": r.get("detail", "")}) + if not ok: + missing.append(r.get("scout_key") or r.get("id")) + if missing: + return VerifyResult( + "fail", + "not registered: " + ", ".join(missing) + + " — run bootstrap (or `python -m orchestrator.cli infra`), then RESTART Scout", + details) + return VerifyResult( + "pass", "registered: " + ", ".join(r.get("scout_key", "?") for r in results), details) + + +REGISTRY = { + "build_defs": verify_build_defs, + "mcp_servers": verify_mcp_servers, +} + + +def get_verifier(verifier_id: str): + return REGISTRY.get(verifier_id) diff --git a/release-agent/phases/stub_runner.py b/release-agent/phases/stub_runner.py new file mode 100644 index 00000000..cb61c9f3 --- /dev/null +++ b/release-agent/phases/stub_runner.py @@ -0,0 +1,47 @@ +"""Stub phase runner — the fallback for steps that don't yet have a real agent. + +Phase 0 has real agents (phases/agents/preflight.py); every other step still +maps to `agent: stub`. The stub does NOT perform the real action — it returns a +mock result telling the conductor what a human would do, so the end-to-end flow +can be driven and tested before each real agent exists. + +When a real phase agent is built, it implements the same contract: + run(phase_id, step, dry_run, state) -> StepResult +and replaces the stub for that step's `agent` id (registered in phases/agents/). +""" +from __future__ import annotations +from dataclasses import dataclass + + +@dataclass +class StepResult: + ok: bool + action: str # human-readable description of what happened / should happen + by: str # 'agent' (stub did it) or 'human' (needs a person) + + +def run_stub(phase_id: str, step: dict, dry_run: bool, state=None) -> StepResult: + """Mock action for a step. Agent-owned steps are 'auto-completed' (mock); + human-owned steps return a reminder that a person must act.""" + owner = step.get("owner", "agent") + name = step.get("name", step["id"]) + if owner == "human": + return StepResult( + ok=True, + action=f"[STUB] Reminder — a human must: {name}", + by="human", + ) + prefix = "[STUB/dry-run]" if dry_run else "[STUB]" + return StepResult( + ok=True, + action=f"{prefix} Would run agent for: {name} (mock success)", + by="agent", + ) + + +# Registry: maps an agent id -> runner. For now everything is the stub. +REGISTRY = {"stub": run_stub} + + +def get_runner(agent_id: str): + return REGISTRY.get(agent_id, run_stub) diff --git a/release-agent/setup/bootstrap.ps1 b/release-agent/setup/bootstrap.ps1 new file mode 100644 index 00000000..44a53189 --- /dev/null +++ b/release-agent/setup/bootstrap.ps1 @@ -0,0 +1,86 @@ +<# +.SYNOPSIS + One-time setup for the Release Orchestrator (/release-agent) on this machine. + Small by design: it only prepares Scout + the environment so the real work + can run inside Scout. It does NOT run a release. + +.DESCRIPTION + Steps: + 1. Infrastructure preflight — check CLIs/host deps AND register + verify the + MCP servers the skill needs inside Scout (from config/requirements.yaml). + 2. Install the /release-agent skill into the Scout skills folder. + 3. Print next steps. + +.EXAMPLE + pwsh ./setup/bootstrap.ps1 +#> +[CmdletBinding()] +param( + [string]$ScoutSkillsDir = "$env:USERPROFILE\.scout\m-skills", + [switch]$SkipSkillInstall +) + +$ErrorActionPreference = "Stop" +$AgentRoot = Split-Path -Parent $PSScriptRoot # release-agent/ +$RepoRoot = Split-Path -Parent $AgentRoot # android-complete/ +$ReqFile = Join-Path $AgentRoot "config\requirements.yaml" + +Write-Host "Release Orchestrator bootstrap`n" -ForegroundColor Cyan + +# ---- 1. Infrastructure preflight (CLIs + MCP servers), data-driven ---- +# Delegates to the engine (python -m orchestrator.cli infra), which reads +# config/requirements.yaml, checks every CLI/host dependency, and REGISTERS any +# missing MCP servers into Scout's config (backing it up first). One home for the +# logic; bootstrap just needs python+pyyaml to call it. +Write-Host "1. Infrastructure preflight (from config/requirements.yaml)" +if (-not (Test-Path $ReqFile)) { Write-Host " requirements.yaml not found at $ReqFile" -ForegroundColor Red; exit 1 } + +$haveInfra = $false +try { python -c "import yaml" 2>$null; if ($LASTEXITCODE -eq 0) { $haveInfra = $true } } catch { $haveInfra = $false } + +$ok = $true +$restartNeeded = $false +if (-not $haveInfra) { + Write-Host " [ ] Python + PyYAML ... MISSING (needed to run the preflight)" -ForegroundColor Yellow + Write-Host " install: Python 3.9+ then python -m pip install pyyaml" -ForegroundColor DarkYellow + $ok = $false +} else { + Push-Location $AgentRoot + try { + $out = python -m orchestrator.cli infra 2>&1 + $out | ForEach-Object { Write-Host " $_" } + if ($LASTEXITCODE -ne 0) { $ok = $false } + if ($out -match "RESTART Scout") { $restartNeeded = $true } + } finally { Pop-Location } + + # engine config presence (cheap local sanity check) + Write-Host -NoNewline " [ ] engine config (phases.yaml) ... " + if (Test-Path (Join-Path $AgentRoot 'config\phases.yaml')) { Write-Host "OK" -ForegroundColor Green } + else { Write-Host "MISSING" -ForegroundColor Yellow; $ok = $false } +} + +if (-not $ok) { + Write-Host "`nSome infrastructure is missing — resolve the items above, then re-run." -ForegroundColor Yellow +} +if ($restartNeeded) { + Write-Host "`n>>> RESTART Scout now so the newly-registered MCP server(s) load. <<<" -ForegroundColor Cyan +} + +Write-Host "`n2. Skill install" +if ($SkipSkillInstall) { + Write-Host " Skipped (--SkipSkillInstall)." +} else { + $src = Join-Path $AgentRoot "skill\SKILL.md" + $destDir = Join-Path $ScoutSkillsDir "release-agent" + New-Item -ItemType Directory -Force -Path $destDir | Out-Null + Copy-Item $src (Join-Path $destDir "SKILL.md") -Force + Write-Host " Installed /release-agent skill -> $destDir" -ForegroundColor Green +} + +Write-Host "`n3. Next steps" -ForegroundColor Cyan +Write-Host " * Open Scout and run: /release-agent" +Write-Host " * Or drive the engine directly from $AgentRoot :" +Write-Host " python -m orchestrator.cli init --release 2026-07" +Write-Host " python -m orchestrator.cli next --release 2026-07" +Write-Host " python -m orchestrator.cli status --release 2026-07" +Write-Host "`n Everything runs in DRY-RUN by default. Nothing touches production.`n" diff --git a/release-agent/skill/SKILL.md b/release-agent/skill/SKILL.md new file mode 100644 index 00000000..39506dfa --- /dev/null +++ b/release-agent/skill/SKILL.md @@ -0,0 +1,282 @@ +--- +name: release-agent +description: Drive an Android release end-to-end using the Release Orchestrator backbone. Use when the user invokes /release-agent, says "start a release", "continue the release", "advance the release", "approve the gate", "release status", or asks about release run-state. The engine is deterministic and does the real work; this skill is the conversation layer that discovers releases, presents gate briefs and status, and relays the human decision. +--- + +# /release-agent — Release Orchestrator conductor + +You are the conversation layer over the **Release Orchestrator engine** (deterministic Python). +The engine decides what happens next; you discover releases, present status/gates nicely, and relay decisions. +**Never decide the release flow yourself, and never invent a release** — always call the engine. + +## Where things live +- Engine + config: `C:\repos\android-complete\release-agent\` (run commands from here). +- Run-state: `C:\repos\android-complete\.release-runs\\release-state.json` (gitignored; one per month, e.g. `2026-07`). +- The `setup/bootstrap.ps1` script ONLY prepares the machine. Its first step is an **infrastructure preflight** (`python -m orchestrator.cli infra`): it checks the CLIs/host deps in `config/requirements.yaml` AND registers the **MCP servers** the skill needs into Scout's config — the **ICM** server (on-call lookups) and the **Kusto/ADX** server (telemetry queries), both provided by the Agency CLI — backing the config up and telling the engineer to restart Scout. It also checks **Scout itself is installed** (`~/.scout`); if not, it stops and says to install Scout first. It does **not** start a release — that happens here, in Scout. +- **Kusto is multi-cluster:** clusters live as data under `kusto_clusters` in `config/requirements.yaml`; infra wires them all into the one Kusto MCP via `--known-services`. To make a new cluster queryable, add an entry there and re-run `python -m orchestrator.cli infra` (then restart Scout). +- If an infra check ever fails (a needed MCP server isn't registered, or Scout wasn't restarted after registering), run `python -m orchestrator.cli infra` and tell the user to restart Scout; the manifest is `config/requirements.yaml`. + +## ALWAYS discover first (the none / one / many rule) +On ANY request about a release (status, continue, approve, advance), **do not assume a release id**. +First run discovery and branch on the result: + +``` +python -m orchestrator.cli list --json +``` + +The JSON has `resolution`: +- **`none`** → there is NO active release on this machine. Tell the user briefly, then **use the `m_ask_user` prompt tool** (not a free-text question) to offer starting one — see "Starting a release" below. Only run `init` after they choose. +- **`one`** → use that release (`release.release_id`). Proceed. +- **`ambiguous`** (several exist) → present the list from `all`, then **use `m_ask_user`** to let the user pick which release to act on (one option per release id, most-recent first). Do not act until they choose. +- **`explicit`** (you passed `--release` and it matched) → use it. + +Never run `status`/`next`/`approve` against a release id you haven't confirmed exists via `list`. + +## Starting a release (don't make the user type a date format) + +The release id is just `YYYY-MM`. **You compute it — never ask the user to type the format.** +Work out the current month from today's date (e.g. today 2026-07 → `2026-07`). + +When no release is active (or the user says "start a release"), call the **`m_ask_user`** prompt tool with clickable options, e.g.: +- **"Current month (``)"** ← recommended +- **"A different month"** + +If they pick the current month, run `init --release ` immediately. +If they pick "a different month", then (and only then) ask which month in a follow-up `m_ask_user` free-text prompt (hint: "e.g. next month, or 2026-08") and convert whatever they say into `YYYY-MM` yourself. Accept natural answers ("this month", "next month", "August") — do the date math for them; don't demand a rigid format. +Default to **dry-run**; only pass `--live` if the user explicitly asks for a live run. + +`init` records the **release owner** (the engineer running it) in the release metadata — resolved from the signed-in `az` user, and reminders are emailed to that address. You can pass a richer profile with `--owner-email`/`--owner-name` (e.g. from `workiq_get_my_profile`), or change it later with `set-owner`. Never hardcode a recipient. + +### Ensure push reminders exist (per release — provisioned at start, torn down at close) + +Right after `init`, make sure the **push-reminder automation** exists for THIS release so reminders reach the user even with Scout closed. It is a **per-release** automation: created when the release starts and removed when it closes (see teardown below). +1. List automations (`m_list_automations`). If one named **"Release push reminders"** already exists AND the registry has it scoped to the current release (`automation list --release --json`), **leave it** — don't duplicate. +2. If it's missing, create it with `m_create_automation`: + - **name:** `Release push reminders` + - **schedule:** `every hour` + - **teamsNotify:** `never` + - **prompt:** from `C:\repos\android-complete\release-agent` run `python -m orchestrator.cli tick --json` (this ADVANCES the active release — running the agent steps that can run, holding at gates/actions — then returns `{message, html, subject, owner_email, owner_name, release}`); if `message` is non-empty and `owner_email` is set, email it via `workiq_send_email` (`to: [owner_email]`, `subject:` the `subject` value, `body:` the `html` value with `isHtml: true` — fall back to the plain `message` with `isHtml: false` only if `html` is empty); if `message` is empty, do nothing. (Recipient comes from `owner_email` — never hardcode. Do **not** use `m_send_teams_message` (bot relay 404s) or the Teams self-chat (delivers silently).) +3. **Register it to this release** so it's tracked and torn down at close: + `python -m orchestrator.cli automation register --id --name "Release push reminders" --release --purpose "hourly advance + phase digest email to owner"` + +Do it silently as part of the start flow (the user already opted into push); don't re-ask each release. (The automation runs `tick` in discovery mode, so it targets the active release automatically.) **Why hourly, not once at 9am:** `tick` is idempotent (advancing is a no-op once holding at a gate, and the digest de-dupes to one email per calendar day), so running it every hour means a run missed while the machine was off — e.g. the 9am tick — is simply picked up by the next tick after the machine is on. A single daily trigger would be skipped for that day. + +### Any automation you provision MUST be registered (for teardown) + +Whenever you create a Scout automation for the orchestrator, immediately record it with `automation register` so nothing gets orphaned: +- **Per-release** (the normal case, e.g. push reminders, a phase watcher) → `--release `. **Removed when that release closes.** +- **Shared / persistent** (rare — only something genuinely meant to outlive every release) → `--shared` (no `--release`). Not torn down at close. Default to per-release unless there's a clear reason. + +At **release close** (status complete, the Release Close phase, or the user asks to "clean up automations"), tear down that release's automations: +1. `python -m orchestrator.cli automation list --release --json` — the automations provisioned for this release. +2. For each entry, delete the real Scout automation with `m_delete_automation` (id from the entry), then `python -m orchestrator.cli automation deregister --id `. +3. Shared automations (if any) are **not** in the release-scoped list, so they survive — leave them. +Confirm with the user before deleting, and report what was removed. + +## Code Complete Date (CCD) & phase scheduling + +Phases are **anchored to the Code Complete Date**, not started on demand. **The CCD is the 2nd Wednesday of the release month — that's the canonical default.** `init` computes it and prints when Phase 0 opens. + +`init` also *reads* the pipeline (ADO 3038 `overrideCodeCompleteDate`) but **does not silently adopt it.** If the pipeline holds a **different in-month date**, that's a **conflict to resolve, not an answer**: the status view shows a *"⚠ Confirm the date"* line and `status --json` sets `ccd_conflict`. When you see a conflict, **ask the user which is the real CCD** via `m_ask_user`, e.g.: +- **"Use the 2nd-Wednesday default (``)"** — then offer to sync the pipeline: run `set-ccd --release --default --reason ""` (preview) → show it → `--confirm` to clear the pipeline override so they match. +- **"Use the pipeline date (``)"** — run `set-ccd --release --date --reason "confirmed CCD is " --confirm` (stores it locally; the pipeline already has it). + +Either resolution clears the conflict. Never pick for the user. + +- **Phase 0 (Pre-flight) opens at CCD‑7** (7 days before CCD). You can `init` any time, but until CCD‑7 the release sits in **`scheduled`** — the engine runs nothing. The status view says *"🗓 Scheduled — Pre‑flight opens `` (in N days). Nothing to do yet."* Relay that plainly; don't try to force it forward. +- When the clock reaches CCD‑7, `next` opens Phase 0 and runs its steps up to the first gate — the normal flow resumes. +- **Testing the clock:** every read/advance command accepts `--as-of YYYY-MM-DD` to simulate a date (dry-run only). Real runs use today. + +**Changing the CCD (real production change).** If the user wants to move the date ("give us more time", "cut early"), use `set-ccd`. This **writes the pipeline override** — so it's gated: run it **without `--confirm` first to show the preview**, present that to the user, get an explicit yes (a `--reason` is always required, for audit), then re-run **with `--confirm`**. The override is month-scoped — the date must be in the release month. Use `--default` to revert to the 2nd-Wednesday default. + +**Skipping/cancelling the release.** Same gated pattern: `skip-release` sets the pipeline `skipRelease` switch (preview → confirm, reason required); `skip-release --clear` re-enables it. This suppresses the monthly trigger — treat it as a real, deliberate action and confirm before `--confirm`. + +**Ongoing conflict detection.** `status`/`resume` re-read the pipeline; if someone sets a differing override later, the same `ccd_conflict` surfaces — ask again. (Use `--no-pipeline-check` only if offline.) + +## Push reminders — the daily phase digest (reaching the user when Scout is closed) + +Everything above is **pull** (seen only when the user opens Scout). The **push** layer is a **daily phase status digest** emailed to the release owner, with a deliberate model: + +- **Setup is interactive — no push.** The readiness checklist and establishing the CCD happen live in Scout, so they are **never** emailed. An unsigned release, a blocked entry gate, and a halted release all stay silent. +- **The first push is a phase opening.** Phase 0 opens at **CCD‑7** — that's the first email. Nothing is sent before a phase opens (no pre‑open heads‑up). +- **Daily while a phase has outstanding work.** Once a phase is open, the owner gets a **once‑per‑day** digest (progress + what still needs them) until the phase's actions are done; then the next phase's digest takes over when it opens (each phase notifies on open). + +`tick` is the deterministic automation half: `python -m orchestrator.cli tick --json` first **advances** the active release (runs the agent steps that can run, holding at gates/actions — idempotent), then returns `{message, html, subject, owner_email, owner_name, release}` — `message` is the plain-text digest, `html` is the rich HTML version (full task table with status pills, attention-flagged), both empty when nothing is due today or it was already sent today. (`notify --json` is the read-only variant — same payload but does NOT advance; use it for a manual "what would I be told" check.) `--as-of ` is a debug clock; `--force` bypasses the once‑per‑day guard. + +- A **Scout automation** named **"Release push reminders"** runs `tick --json` (discovery mode) **hourly** and, when `message` is non‑empty, emails it via `workiq_send_email` to `owner_email` (subject from the JSON) — the release owner from release metadata, **never a hardcoded address**; when `message` is empty it stays silent. Running hourly (not once/day) means a tick missed while the machine was off is picked up by the next one, and idempotency + once‑per‑day de‑dup keep it to one advance-effect and one email per day. It is **per‑release**: auto‑provisioned (create‑if‑missing, registered to the release) at start and torn down at close. (Email is the channel because it reliably notifies; the `m_send_teams_message` bot relay 404s without a conversation reference, and the Teams self‑chat delivers silently.) + +If the user asks "how will I be reminded" / "set up notifications," explain this; if the automation doesn't exist, create it (see "Ensure push reminders exist"). Keep the email subject/body exactly as `tick` returns — don't embellish. + +## The readiness ENTRY GATE (right after starting) + +Immediately after `init`, the very first thing is the **readiness checklist** — the entry gate. The engine's `next` refuses to run any step (reports `readiness_gate`) until it's cleared. **Every item is equally required** — there is no priority or "hard vs soft" distinction. The only difference between items is **who resolves them**: + +- **`auto`** — **Scout resolves it** (verifies programmatically, pass/fail). Two execution sources, but the user sees both as `[auto]`: + - *Python-verified* (default): `build_access` (both ADO build definitions, via `az`) and `mcp_servers` (the ICM + Kusto/ADX MCP servers are registered in Scout). The engine's `verify`/`sign` runs these. + - *Scout-assisted* (`source: scout`): the **engine can't reach the MCP/Scout-settings, so YOU run the check** and record the result (see step 3a). Fail-closed **except `silent_perms`** (see below). Today: `oncall_now` (ICM current on-call), `adx_access` (Kusto `print 1` against the ADX cluster), `silent_perms` (Scout permissions allow fully-unattended runs). +- **`attest`** — **the engineer resolves it** (confirms): `play_console_access`, `oncall_window`, `saw_ame`, `yubikey`. + +**Two of the auto items exist so scheduled work runs UNATTENDED** (machine on, Scout not focused): `mcp_servers` (the MCP deps are registered — **hard**, since without them the on-call/telemetry checks can't run) and `silent_perms` (permissions won't stall the daily digest / Teams reminders / browser checks on a prompt — **soft/opt-out**: the user can choose to proceed without silent runs, recorded as `degraded`, with the downside noted). Enabling silent runs needs the user to flip ONE Scout master toggle first (*"Allow AI to request permission changes"*) — only they can (it's read-only from the model); after that I auto-request the rest with a single Allow click. + +**On-call is TWO items (hybrid), because Scout can only see the *current* rotation, not the future one:** +- `oncall_now` (**auto/ICM**) — are you on-call *right now*? Scout verifies this from ICM. +- `oncall_window` (**attest**) — are you free across the whole release window **CCD‑7 → CCD+14**? Scout can't read the future rotation, so you attest it (the checklist shows the concrete dates). + +If any item is unsatisfied the gate stays closed. If the engineer can't satisfy an attest item, they resolve it or hand the release to someone who can — the same for every item. Never describe any item as "not a hard block" or "optional." + +Flow after starting: +1. `python -m orchestrator.cli checklist --release --verify` — this runs the auto checks AND prints the **canonical checklist table (markdown)**. **Reproduce its stdout into your reply as live markdown (NOT wrapped in a ``` code fence)** so Scout renders it as a real table — it is already a finished markdown table with the type labels, per-item status, and clickable links. **Do NOT rebuild, re-format, re-order, re-label, or re-type any of it from memory, and do NOT fence it.** If you reconstruct it you WILL introduce errors (stale icons, mangled/merged URLs); if you fence it, it shows as raw text. Always reproduce the literal command output as rendered markdown. You may add a sentence of your own before or after, but the table block itself must match the output. +2. There are exactly **two types by resolver**: `[auto]` (Scout verifies) and `[attest]` (the user confirms). All items must be satisfied to clear the gate. (Do not add lock icons or a "hard requirement" legend.) +3a. **Run the scout-assisted `[auto]` checks yourself, then record each result** — don't ask the user for these; they're verified, not attested. + - **`oncall_now` (ICM):** call the ICM MCP `get_on_call_schedule_by_team_id` with `teamIds: [78848]` ("Auth Client Android Shield"). Resolve the current user's alias (`get_my_icm_context` or the owner email's local part), then decide by their role in `shiftCurrentOnCalls[].currentOnCallContacts[]`: + - **Not in the roster at all** → `record-check --item oncall_now --status pass --detail "not on the current roster"`. + - **Present but NOT the primary** (i.e. they are a **backup/secondary** — any position other than the first-listed contact) → **pass**: `record-check --item oncall_now --status pass --detail "backup OCE, not primary (primary: )"`. A backup is free to run the release. + - **The PRIMARY / current OCE** (the **first-listed** contact in `currentOnCallContacts`) → `record-check --item oncall_now --status fail --detail "currently the primary on-call for Auth Client Android Shield"`. + - Only the **primary** blocks the gate. If you cannot confidently tell primary from backup (ambiguous ordering, or the user says otherwise), **ask the user** "Are you the primary/current OCE, or backup?" and record accordingly — **never block a backup.** + - **`adx_access` (Kusto):** run a trivial query — `kusto_query` with the item's `cluster_uri` + `database` (from `checklist --json`) and query `print 1`. Success = the engineer has data access to the ADX release dashboard's cluster. + - Query succeeds → `record-check --release --item adx_access --status pass --detail "print 1 succeeded"`. + - Query fails (auth/access error) → `record-check --release --item adx_access --status fail --detail ""`. + - **`silent_perms` (Scout settings — OPT-OUT/soft):** the daily push digest, the Teams reminders and the browser (CCOA/lockdown) checks all run from a background automation while Scout isn't focused — they must not stall on a permission prompt. Call **`m_get_settings`** and read `permissions.servers`. It's satisfied when ALL of the item's `required_servers` (from `checklist --json`: `shell`, `workiq`, `playwright`) have `autoApprove: true` (one server flag each keeps it simple: `workiq.autoApprove` covers both `workiq_send_email` and Teams; `playwright.autoApprove` covers the browser). **This item never hard-blocks — the user may choose to proceed without silent runs.** Flow: + - **Already all auto-approved** → `record-check --release --item silent_perms --status pass --detail "shell/workiq/playwright auto-approved"`. Done. + - **One or more NOT auto-approved** → **offer the choice** with `m_ask_user`: **"Enable silent runs (recommended)"** vs **"Proceed without — I'll get prompts"**. Explain the downside of proceeding: *the daily digest, Teams reminders, and CCOA/lockdown browser checks will pop a permission prompt when Scout isn't focused and can stall until you open Scout and approve them.* + - They pick **Enable** → the only manual step is the Scout master toggle: if `permissions.allowModelPermissionsChange` is `false`, tell them to turn on **Settings → Permissions → "Allow AI to request permission changes"** (I cannot flip it — it's read-only from the model, by design). Once it's `true`, call **`m_request_permission_escalation`** with `servers: { workiq: {autoApprove:true}, playwright: {autoApprove:true} }` (add `shell` if off too); they click **Allow** once, then re-read `m_get_settings` and `record-check … --status pass --detail "enabled silent runs"`. + - They pick **Proceed without** (or won't enable the master toggle) → `record-check --release --item silent_perms --status degraded --detail "proceeding without silent runs — unattended digest/Teams/browser checks will prompt & may stall until Scout is opened"`. **`degraded` satisfies the gate** (the checklist shows it as ⚠️ *Proceeding (not silent)*), so the release can start; the downside is on record. + - A `fail` on `oncall_now`/`adx_access`, or a real problem, keeps the gate closed — treat it like any unsatisfiable required item (resolve or hand off). Do NOT attest these — they're `auto` items you verified. (`silent_perms` is the one soft/opt-out auto item: it uses `degraded`, never `fail`, when the user chooses to proceed.) +3b. Use `m_ask_user` to collect the **attestations**: `play_console_access`, the on-call **window** (`oncall_window` — show the CCD‑7 → CCD+14 dates from the checklist), `saw_ame`, `yubikey`. Offer: **"All confirmed"**, **"I'm scheduled on-call during the window"**, **"I can't open Play Console"**, **"I don't have a SAW machine"**, **"I don't have a YubiKey"**. +4. If they confirm everything → `sign --release --all`, then `next` to begin Phase 0. +5. **If they can't satisfy any attest item** (on-call during the window, no SAW, no YubiKey, no portal access) → `decline --release --item ` (repeat `--item` for each). The gate is now blocked. Tell them plainly: the release can't start until that item is resolved; if they can't resolve it, hand the release to another engineer who can (notify their manager / the release team). Treat every item this way — don't single any out as harder or softer. +6. If an **auto** item shows FAIL (no build-definition access, or you're on-call), the gate stays closed — a real problem to resolve, not something to attest around. + +Never attest an `auto` item on the user's behalf — auto items are only satisfied by real verification (Python check or your recorded ICM result). +Never hand-edit or regenerate the checklist/status blocks — always show the CLI's literal output. + +## Parallel phases — process ALL the holds, not one at a time + +Some phases run **in parallel** (Phase 0 is `execution: parallel`): a single `next` runs **every independent automated step at once** (breaking, CG, cron, wiki — all complete in one call) and then surfaces **all the human/scout holds together** (e.g. *"4 item(s) need you: …"*). So don't treat it as one-step-at-a-time. After `next`, read `status --json` and look at **`pending_human`** (and `active_phase.steps` with their `status`/`needs_owner`) — that's the full set of what's outstanding. Work through **all** of them in this pass: +- **`source: scout`** steps (notice, flight_reminder, lockdown) → run each via MCP/browser + `record-step` (see the sections below). These are independent — do them all. +- **`attest`** steps (confirm_reminders, vitals) → ask the owner to confirm, then `done --step `. +- **`blocked`** steps (cg/cron on a real problem) → show the note; fix + rerun, or skip. +Dependencies still hold: `confirm_reminders` only appears **after** `flight_reminder` is sent (it won't be in `pending_human` until then). Call `next` again after clearing holds to let newly-ready steps surface and, once all are done, advance to the next phase. + +## Scout-assisted phase steps (CCOA lockdown check) + +Some Phase steps read AAD-gated sources the deterministic engine can't reach, so **you run them via the browser and record the result** — same idea as the readiness scout checks, but mid-phase. When advancing, if **`lockdown`** is among the pending holds (`source: scout`, in `pending_human`), handle it like this — silently, without bothering the user unless there's an overlap: + +1. **Scrape the CCOA source.** Navigate (Playwright) to `https://prod.change-manager.msidentity.com/ccoa-periods`. If an AAD account picker appears, click the user's own account (Windows-SSO — no password). Wait for the "CCOA Periods" page. +2. **Extract the periods.** From **"Upcoming CCOA periods"** and the **current-year** "Past NoFly Zones" table, read each row's **Name, Environment, Start Date (UTC), End Date (UTC)**. Build a JSON array: `[{"name","environment","start":"YYYY-MM-DD","end":"YYYY-MM-DD"}, ...]` (use the UTC dates). +3. **Let the engine decide (deterministic).** Run `python -m orchestrator.cli check-lockdown --release --periods-json ''`. It computes the release window (CCD‑7 … CCD+14), keeps only **Production**-environment periods, checks overlap, and records the step: **pass** (no overlap → step done, flow continues) or **attention** (overlap → step holds). +4. **Relay the outcome.** On **pass**, just continue (`next`) — no need to bother the user. On **attention**, surface it: name the overlapping lockdown(s) and window, and tell them to **shift CCD** past the lockdown (`set-ccd`) if they want to proceed; there are **no partners to notify** for this step. + +If you can't reach the browser/SSO in this context, leave the step held — it stays flagged as needing attention and you (or the user, next time Scout is open) can run it then. Don't mark it done without actually running the check. + +## Scout-assisted phase steps (early code-complete notice) + +The Phase-0 `notice` step sends the early code-complete email. Sending needs WorkIQ (a skill capability), so it's scout-assisted like `lockdown`. When `status --json` shows the current step is **`notice`** (holding, `awaiting_action`): + +1. **Prepare it (deterministic).** Run `python -m orchestrator.cli prepare-notice --release `. It fills the local template (`templates/early-code-complete-notice.md`) with the release's CCD/owner and returns JSON `{subject, body, html, recipients, dry_run, recipients_note}`. +2. **Send it.** Email via `workiq_send_email` using the returned `subject` and `recipients` exactly, with **`body:` the `html` value and `isHtml: true`** (the HTML has a clean hotfix-guide link + a proper rendered table — fall back to the plain `body` with `isHtml: false` only if `html` is empty). **Recipients are already resolved for you**: in a **dry-run** they're the **release owner only** (safe rehearsal — the subject is prefixed `[DRY-RUN → owner]`); on a **live** release they're the real distribution list (androididentity@microsoft.com, jialh@microsoft.com — see EXTERNAL-REFERENCES.md). Never override the recipients. +3. **Record it.** After a successful send: `python -m orchestrator.cli record-step --release --step notice --status pass --detail "sent to "`. If the send fails, `--status attention --detail ""` to keep it flagged. + +## Scout-assisted phase steps (flight & string reminders — Teams) + +The Phase-0 `flight_reminder` step posts a **combined 4-in-1 reminder** (update local flights · flight pre-mortem docs · merge user-facing strings by CCD-7 · Auth App feature-flag freeze / default-OFF review) as a **Teams message** to the Android Core Team. Sending Teams needs WorkIQ, so it's scout-assisted. When the current step is **`flight_reminder`** (holding): + +1. **Prepare it.** Run `python -m orchestrator.cli prepare-flight-reminder --release `. It returns JSON `{content, content_type:"html", dry_run, send_to, owner_email, chat_id, target_note}`. +2. **Resolve the chat + send.** + - **Dry-run** (`send_to: "owner"`): get the owner's 1:1 chat with `workiq_create_chat_by_email` (email = `owner_email`), then `workiq_send_chat_message` with that `chatId`, `content` = the returned HTML, `contentType: "html"`. (Safe rehearsal — the message is prefixed `[DRY-RUN → owner]`.) + - **Live** (`send_to: "group"`): `workiq_send_chat_message` with `chatId` = the returned `chat_id` (the Android Core Team thread), `content`, `contentType: "html"`. + Never override the target — `prepare-flight-reminder` already picked owner-vs-group from dry_run. +3. **Record it.** After a successful send: `python -m orchestrator.cli record-step --release --step flight_reminder --status pass --detail "posted to "`; on failure, `--status attention --detail ""`. + +**Sending the reminder is fire-and-forget** — it does NOT prove the feature owners actually did the work. So the very next step is **`confirm_reminders`**, a human **attestation** the engine holds on (`awaiting_action`). When the current step is `confirm_reminders`, ask the release owner (via `m_ask_user`) to confirm the reminded work is actually done — feature owners updated local flights, wrote flight pre-mortem docs, merged user-facing strings by CCD-7, and all features are default-OFF (or default-ON ones are approved in the wiki). Only when they confirm, run `python -m orchestrator.cli done --release --step confirm_reminders --note ""`. If they can't confirm, leave it holding (the release correctly blocks here until the pre-requisite work is verified) — don't mark it done. + +Phase 0's **`vitals`** step ("Confirm Play Console vitals & policy status reviewed") is another **attestation** hold. Play Console has no API for **Policy issues/warnings** (the Reporting API covers only technical vitals, and the Console UI is behind a Google login Scout can't automate), so this is a manual check: when the current step is `vitals`, ask the owner to open Play Console, review **Android vitals** (crash/ANR rate) and **Policy status** (issues/warnings), and confirm they're acceptable. On confirmation, `python -m orchestrator.cli done --release --step vitals --note ""`. If there's an unresolved policy issue or vitals regression, leave it holding. + +## Commands (run from the release-agent folder) + +| Intent | Command | +| --- | --- | +| Discover releases | `python -m orchestrator.cli list --json` | +| Start a new release (dry-run) | `python -m orchestrator.cli init --release ` | +| Start for real | `python -m orchestrator.cli init --release --live` | +| Show readiness entry checklist | `python -m orchestrator.cli checklist --release --json` | +| Run auto readiness verifiers | `python -m orchestrator.cli verify --release ` | +| Attest human items (+auto verify) | `python -m orchestrator.cli sign --release --all` | +| Record a scout-assisted check (ICM on-call now) | `python -m orchestrator.cli record-check --release --item oncall_now --status pass\|fail --detail "..."` | +| Decide CCOA lockdown overlap (from scraped periods) | `python -m orchestrator.cli check-lockdown --release --periods-json '[{"name","environment","start","end"}]'` | +| Prepare the early code-complete notice email (JSON) | `python -m orchestrator.cli prepare-notice --release [--variant initial\|update]` | +| Prepare the flight & string reminders Teams message (JSON) | `python -m orchestrator.cli prepare-flight-reminder --release ` | +| Record a scout-assisted phase step (after sending/doing it) | `python -m orchestrator.cli record-step --release --step --status pass\|attention --detail "..."` | +| Declare you CANNOT satisfy an item | `python -m orchestrator.cli decline --release --item ` | +| Status (structured) | `python -m orchestrator.cli status --release --json` | +| Advance to next gate | `python -m orchestrator.cli next --release ` | +| Approve the holding gate | `python -m orchestrator.cli approve --release --comment ""` | +| Deny the holding gate | `python -m orchestrator.cli deny --release --comment ""` | +| **Done** — mark a reminder (human to-do) complete | `python -m orchestrator.cli done --release [--phase

--step ] --note ""` | +| **Set/change CCD** (writes pipeline; preview→confirm) | `python -m orchestrator.cli set-ccd --release --date --reason "" [--confirm]` | +| **Revert CCD to default** (2nd Wednesday) | `python -m orchestrator.cli set-ccd --release --default --reason "" [--confirm]` | +| **Skip/cancel the release** (writes pipeline) | `python -m orchestrator.cli skip-release --release --reason "" [--confirm]` (add `--clear` to un-skip) | +| **Skip** a step (reason REQUIRED) | `python -m orchestrator.cli skip --release --phase

--step --reason ""` | +| **Reopen** a done/skipped step | `python -m orchestrator.cli reopen --release --phase

--step [--reason "..."]` | +| **Halt** (emergency, reason REQUIRED) | `python -m orchestrator.cli halt --release --reason ""` | +| **Resume** after a halt | `python -m orchestrator.cli resume --release [--reason "..."]` | +| Show / analyze this release's log | `python -m orchestrator.cli log --release ` (add `--analyze`, `--json`) | +| Journal interaction (silent) | `python -m orchestrator.cli journal --release --source scout|user --text "..."` | +| Activate conditional hotfix phase | `python -m orchestrator.cli activate --release --phase hotfix` | +| **Notify** — push line if something needs me (else nothing) | `python -m orchestrator.cli notify [--release ] [--as-of ] [--force]` | +| **Track automations** (register/list/deregister for teardown) | `python -m orchestrator.cli automation register --id --name "" [--shared\|--release ] [--purpose "..."]` · `automation list [--release ] [--json]` · `automation deregister --id ` | + +**Manual overrides** (the release engineer can steer when reality diverges from the plan): +- **skip** — a step doesn't apply this release, or was done manually outside the tool. **A reason is required** (audited). Confirm the reason with the user, then run `skip`. +- **reopen** — a step (incl. an approved gate) needs to run again; reopening a gate makes it re-hold for a fresh decision. +- **halt** — emergency freeze (e.g. production incident). **Reason required.** While halted, `next` refuses to advance and status shows a HALTED banner. Use for "stop everything now." +- **resume** — clear a halt and continue. +Map natural language to these ("skip the CG report, doesn't apply" → `skip … --reason`; "halt, we have an incident" → `halt --reason`; "resume" → `resume`). Never skip or halt without capturing the user's reason. + +The human-readable commands (`checklist`, `status`, `next`, `approve`, `deny`, `decline` without `--json`) emit a **canonical block AND auto-log it** as scout output. **Prefer these and show their output to the user** — that keeps the display consistent AND guarantees the log captures what was shown. Use `--json` only when you need raw fields for your own logic, not for display. + +## Presenting STATUS (make it clean — users ask for this most) + +> **Render it as markdown, never as a code block.** Both the `checklist` and `status` outputs are already markdown (headings + tables). Paste them into your reply as **normal message content so Scout renders the table/stepper** — do **NOT** wrap them in a ``` code fence / triple backticks. Fencing them makes them show as raw text (YAML-looking) instead of a rendered table. Reproduce the content faithfully, but as live markdown. + +**First decide what to show.** If the readiness entry gate isn't cleared yet (not signed, or blocked), the most useful "status" is the **checklist itself** — run `checklist --release --verify` and show that table (don't show the terse status line, and don't ask permission to pull the checklist). Only when the gate is cleared / the release is mid-flight do you show the `status` block. + +For a mid-release status: run `python -m orchestrator.cli status --release ` (no `--json`) and **show its output as rendered markdown** (not fenced) — it's a finished view with a next-action headline, a **phase map** (✅ done · ⏸ in progress · 🗓 scheduled · ⬜ not started) and the **current phase's steps** in a table. It auto-logs what was shown. You may add a sentence before/after, but reproduce the block faithfully; don't re-render from `--json` (that skips the auto-log) and don't invent your own layout. Never surface raw engine state names like `holding_gate`, `awaiting_action`, or `scheduled` — the view already translates them to plain language ("Waiting for your approval", "Action needed from you", "Scheduled"). + +If you need structured fields for branching logic, `status --json` gives: +`release_id, status, dry_run, ccd, ccd_source, as_of, done, total, percent, current_phase_name, current_step_name, gate, action, scheduled, pending_human, readiness_signed, blocked`. + +## Behaviour +1. **"status" / "where are we":** discover the release, then check its state: + - **If the readiness entry gate is not yet cleared** (status `readiness_gate` / not signed, or `blocked`): the useful answer *is* the checklist — so run `checklist --release --verify` and **show that table directly**. Don't show the terse "entry gate not signed" line and don't ask "want me to pull the checklist?" — just pull it. Then prompt for the attestations (see the ENTRY GATE flow). + - **Otherwise** (gate cleared / mid-release): show the `status` output. + - **If no release exists:** say so briefly and use `m_ask_user` to offer starting one. +2. **"start a release":** use `m_ask_user` to offer current-month vs another month (you compute the `YYYY-MM`), run `init`, **ensure the push-reminder automation exists** (create-if-missing — see "Ensure push reminders exist"), then **present the readiness entry checklist and get it signed** (see "The readiness ENTRY GATE"), then `next`, then present the resulting status. +3. **Engine HOLDS at a gate:** stop and present the gate. Use `m_ask_user` to offer **Approve** / **Deny** (never decide for them). +4. **On their decision:** run `approve`/`deny` with their comment, then present the new status (it auto-continues to the next gate). +5. **Engine HOLDS for a reminder (ACTION NEEDED / status `awaiting_action`):** this is a human *to‑do*, not a decision — the engine can't do it and is waiting for the person to do it (e.g. "China publish flow", "Surface Phase 2 UI failure list"). Present it plainly as "you need to do X", and when the user says it's done, run `done --release --note ""` (defaults to the current held step) — the flow then advances. Don't offer Approve/Deny for a reminder; it's just done / not-yet. +6. **A step is BLOCKED (an agent found a real problem):** some agent steps block the flow when they detect something the owner must resolve. Today: **`cg`** (blocks on active **High/Critical** Component Governance alerts — the note lists CVE, component, fix) and **`cron`** (blocks if the Calendar Checker pipeline has no recent scheduled run — i.e. the cron may be broken). Show the owner the note plainly. Their two exits for any blocked step: **(a) fix** the underlying problem, then **rerun** — call `next` again, which re-runs the check; if it's now clean it passes and the flow continues; **(b) override** — `skip --release --phase --step --reason ""` (e.g. accepted risk, tracked separately). Don't mark it done any other way — either the re-check passes or they consciously skip. +7. **Engine is `scheduled`:** Phase 0 hasn't opened yet (before CCD‑7). Tell the user the opens date + countdown from the status view; there's nothing to advance. If they want to start earlier, that's a CCD change (`set-ccd`), not a `next`. +8. **"continue"/"resume":** discover → if the entry gate isn't cleared, show the checklist (as in #1); otherwise brief them with status → then `next`. + +> **Prompt, don't interrogate.** Whenever you need a discrete choice from the user (start? which release? approve or deny?), prefer the `m_ask_user` clickable prompt over a free-text question. Reserve free-text prompts for genuinely open values (like an unusual month). + +## Guardrails +- The engine is the source of truth for sequencing and gate state. When unsure, run `status --json`. +- Gates are **human-decided**. Present and relay; never authorize. +- **Dry-run by default.** Only use `--live` when the user explicitly asks. +- `[STUB...]` output means that step is mocked today — say so plainly; don't imply real work happened. + +## Event logging (silent — never changes the interaction) +Each release keeps its **own** append-only log at `.release-runs//events.jsonl` (per-release only — there is no machine-wide aggregate). It exists purely for debugging and improvement. **Logging is invisible to the user and must never change how you communicate** — do not announce it, and do not add extra questions just to populate it. + +**What's automatic vs. your job:** +- **Scout output is logged automatically** by the CLI whenever you run a human-readable command (`checklist`, `status`, `next`, `approve`, `deny`, `decline` without `--json`). You do NOT need to journal what was shown — just run those commands and show their output. +- **User input is your responsibility** — the engine can't see what the user typed/clicked in Scout. So **every time the user makes a choice or gives input, immediately journal it** (this is required, not optional): + `journal --release --source user --kind choice --text "" --choice "

  • ") == 4 + assert "variableGroupId=40" in out["content"] # variable-group link + + # live + st2 = ReleaseState(release_id=rid, dry_run=False, ccd="2026-08-12", + ccd_source="default", owner_email="pedroro@microsoft.com") + C.save_state(st2, d, rid) + buf2 = io.StringIO() + with contextlib.redirect_stdout(buf2): + ncmd.cmd_prepare_flight_reminder(A) + out2 = _json.loads(buf2.getvalue()) + assert out2["dry_run"] is False and out2["send_to"] == "group" + assert out2["chat_id"] == "19:976a859f167f44e59c4ceca8b1d23581@thread.v2" + assert "[DRY-RUN" not in out2["content"] + + +def test_no_localization_strings_step(): + """The old #5 localization strings step was removed.""" + st, orch = _orch() + preflight = next(p for p in orch.config["phases"] if p["id"] == "preflight") + assert "strings" not in [s["id"] for s in preflight["steps"]] + + +def test_confirm_reminders_is_attestation_hold(): + """In the parallel Phase 0, confirm_reminders is a human attestation that becomes + ready only after flight_reminder is sent; it surfaces as a hold with a confirm + pill and clears via `done`.""" + st, orch = _orch(signed=False) + _pass_scout_checks(orch) + orch.gate.sign() + _clear_notice(orch) + orch.record_scout_step("preflight", "flight_reminder", "pass", "sent") + orch.run_until_gate() + assert st.status == "awaiting_action" + # confirm_reminders (dep on flight_reminder, now done) is among the pending holds + assert "preflight.confirm_reminders" in st.pending_human + ap = orch.status_report()["active_phase"] + step = next(s for s in ap["steps"] if s["id"] == "confirm_reminders") + assert step["status"] == "confirm" and step["needs_owner"] + # owner attests → advances + orch.complete_step("preflight", "confirm_reminders", "verified with feature owners") + assert st.is_done("preflight", "confirm_reminders") + + +def test_confirm_reminders_gated_by_flight_send(): + """confirm_reminders must NOT be offered until flight_reminder is sent (dependency).""" + st, orch = _orch(signed=False) + _pass_scout_checks(orch) + orch.gate.sign() + _clear_notice(orch) + # flight_reminder NOT yet recorded → confirm_reminders is not ready + orch.run_until_gate() + assert "preflight.confirm_reminders" not in st.pending_human + assert "preflight.flight_reminder" in st.pending_human # the send is what's pending + + +def test_parallel_autos_run_despite_pending_holds(): + """Independent auto steps (breaking/cg/cron/wiki) complete even while scout/attest + steps are still holding — a hold no longer blocks its siblings.""" + st, orch = _orch(signed=False) + _pass_scout_checks(orch) + orch.gate.sign() + # nothing cleared: notice/flight/lockdown (scout) + vitals (attest) all hold + orch.run_until_gate() + # yet the independent auto agents ran to completion + for sid in ("breaking", "cg", "cron", "wiki"): + assert st.is_done("preflight", sid), sid + # and the holds are all surfaced together + for sid in ("notice", "flight_reminder", "lockdown", "vitals"): + assert f"preflight.{sid}" in st.pending_human, sid + + +def test_cg_report_summarizes_and_flags_high(): + """The CG report groups active alerts by severity and lists High/Critical.""" + from phases.agents.preflight import _cg_summary, _cg_report + alerts = [ + {"alertState": "active", "severity": "high", "title": "CVE-1", + "component": {"displayName": "io.netty:x", "displayVersion": "4.2.15"}, + "actionItems": "Upgrade to 4.2.16"}, + {"alertState": "active", "severity": "medium", "title": "CVE-2"}, + {"alertState": "autoDismissed", "severity": "high", "title": "CVE-OLD"}, + {"alertState": "fixed", "severity": "critical", "title": "CVE-FIXED"}, + ] + active, high = _cg_summary(alerts, ["critical", "high"]) + assert len(active) == 2 and len(high) == 1 + rep = _cg_report(active, high) + assert "2 active alert(s)" in rep and "1 high" in rep + assert "CVE-1" in rep and "io.netty:x" in rep and "Upgrade to 4.2.16" in rep + assert "CVE-OLD" not in rep and "CVE-FIXED" not in rep # non-active excluded + + +def test_cg_agent_dry_run_simulates(): + from phases.agents import preflight as pa + r = pa.run_cg_alerts("preflight", {"id": "cg"}, True, None) + assert r.ok and "dry-run" in r.action.lower() + + +def test_cg_agent_blocks_on_high(): + """High/Critical active alerts BLOCK the step (ok=False) with a fix-and-rerun message.""" + from phases.agents import preflight as pa + from tools import checks + orig = checks.fetch_cg_alerts + checks.fetch_cg_alerts = lambda *a, **k: (True, [ + {"alertState": "active", "severity": "high", "title": "CVE-9", + "component": {"displayName": "pkg", "displayVersion": "1.0"}}, + ], "ok") + try: + r = pa.run_cg_alerts("preflight", {"id": "cg"}, False, None) + assert not r.ok # High → blocks + assert "CVE-9" in r.action and "RERUN" in r.action + finally: + checks.fetch_cg_alerts = orig + + +def test_cg_agent_passes_when_no_high(): + """Only Medium/Low active alerts → the step passes (report captured).""" + from phases.agents import preflight as pa + from tools import checks + orig = checks.fetch_cg_alerts + checks.fetch_cg_alerts = lambda *a, **k: (True, [ + {"alertState": "active", "severity": "medium", "title": "CVE-M"}, + ], "ok") + try: + r = pa.run_cg_alerts("preflight", {"id": "cg"}, False, None) + assert r.ok and "1 active" in r.action + finally: + checks.fetch_cg_alerts = orig + + +def test_cg_blocked_step_reruns_and_clears_when_fixed(): + """A CG block holds the step; fixing (alerts now clean) + rerunning `next` + re-checks and lets the flow continue.""" + from phases import agents as pa + from phases.stub_runner import StepResult + flag = {"high": True} + + def fake_cg(phase, step, dry_run, st): + if flag["high"]: + return StepResult(False, "CG: 1 critical active\n→ Fix and RERUN or skip.", "agent") + return StepResult(True, "CG: 0 active alerts.", "agent") + orig = pa.REGISTRY["cg_alerts"] + pa.REGISTRY["cg_alerts"] = fake_cg + try: + st, orch = _orch() + _clear_phase0_scout(orch) # clear the earlier scout/human holds + orch.run_until_gate() + assert st.status == "awaiting_action" and st.current_step == "cg" + assert st.get_step("preflight", "cg").status == "blocked" + # the digest shows it blocked / needs owner + step = next(s for s in orch.status_report()["active_phase"]["steps"] if s["id"] == "cg") + assert step["status"] == "blocked" and step["needs_owner"] + # FIX: alerts now clean → RERUN (next) re-checks and passes + flag["high"] = False + orch.run_until_gate() + assert st.is_done("preflight", "cg") + finally: + pa.REGISTRY["cg_alerts"] = orig + + +def test_cg_blocked_step_skip_override(): + """The owner can override a CG block by skipping the step (with a reason).""" + from phases import agents as pa + from phases.stub_runner import StepResult + orig = pa.REGISTRY["cg_alerts"] + pa.REGISTRY["cg_alerts"] = lambda *a, **k: StepResult(False, "CG: 1 high active", "agent") + try: + st, orch = _orch() + _clear_phase0_scout(orch) + orch.run_until_gate() + assert st.current_step == "cg" and st.get_step("preflight", "cg").status == "blocked" + orch.skip_step("preflight", "cg", "accepted risk; tracked separately") + assert st.is_done("preflight", "cg") # skipped counts as done + assert st.get_step("preflight", "cg").status == "skipped" + finally: + pa.REGISTRY["cg_alerts"] = orig + + +def test_cg_agent_fetch_error_holds(): + from phases.agents import preflight as pa + from tools import checks + orig = checks.fetch_cg_alerts + checks.fetch_cg_alerts = lambda *a, **k: (False, [], "403 forbidden") + try: + r = pa.run_cg_alerts("preflight", {"id": "cg"}, False, None) + assert not r.ok and "could not read alerts" in r.action + finally: + checks.fetch_cg_alerts = orig + + +def test_cron_check_dry_run_simulates(): + from phases.agents import preflight as pa + r = pa.run_cron_check("preflight", {"id": "cron"}, True, None) + assert r.ok and "dry-run" in r.action.lower() + + +def test_cron_check_passes_on_recent_scheduled_run(): + from phases.agents import preflight as pa + from tools import checks + from datetime import datetime, timezone + orig = checks.latest_scheduled_build + now_iso = datetime.now(timezone.utc).isoformat() + checks.latest_scheduled_build = lambda *a, **k: (True, { + "queueTime": now_iso, "result": "succeeded", "status": "completed"}, "ok") + try: + r = pa.run_cron_check("preflight", {"id": "cron"}, False, None) + assert r.ok and "scheduled and firing" in r.action + finally: + checks.latest_scheduled_build = orig + + +def test_cron_check_blocks_when_stale(): + from phases.agents import preflight as pa + from tools import checks + orig = checks.latest_scheduled_build + checks.latest_scheduled_build = lambda *a, **k: (True, { + "queueTime": "2026-01-01T06:00:00Z", "result": "succeeded", "status": "completed"}, "ok") + try: + r = pa.run_cron_check("preflight", {"id": "cron"}, False, None) + assert not r.ok and "stale" in r.action + finally: + checks.latest_scheduled_build = orig + + +def test_cron_check_blocks_when_no_scheduled_run(): + from phases.agents import preflight as pa + from tools import checks + orig = checks.latest_scheduled_build + checks.latest_scheduled_build = lambda *a, **k: (True, None, "no scheduled runs in recent history") + try: + r = pa.run_cron_check("preflight", {"id": "cron"}, False, None) + assert not r.ok and "no scheduled run" in r.action + finally: + checks.latest_scheduled_build = orig + + +def test_vitals_is_attestation_hold(): + """Play Console vitals/policy is an attest step (no API for policy): it HOLDS + for the owner to confirm they reviewed it, and clears via `done`.""" + st, orch = _orch() # clears the earlier holds (incl. vitals via _clear_phase0_scout) + # re-open vitals to observe its natural hold + orch.reopen_step("preflight", "vitals") + orch.run_until_gate() + assert st.status == "awaiting_action" + assert st.current_step == "vitals" + step = next(s for s in orch.status_report()["active_phase"]["steps"] if s["id"] == "vitals") + assert step["status"] == "confirm" and step["needs_owner"] + orch.complete_step("preflight", "vitals", "reviewed vitals + policy status in Play Console") + assert st.is_done("preflight", "vitals") + + +if __name__ == "__main__": + fns = [v for k, v in sorted(globals().items()) if k.startswith("test_")] + for fn in fns: + fn() + print(f" PASS {fn.__name__}") + print(f"\n{len(fns)}/{len(fns)} tests passed") diff --git a/release-agent/tools/__init__.py b/release-agent/tools/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/release-agent/tools/checks.py b/release-agent/tools/checks.py new file mode 100644 index 00000000..677f09e7 --- /dev/null +++ b/release-agent/tools/checks.py @@ -0,0 +1,285 @@ +"""Real checks for readiness verifiers (no fakery). + +- ado_build_def: uses `az pipelines build definition show` to prove the signed-in + user can actually access a build definition. Genuine access check. +- http_reachable: HEAD/GET a URL and report the status. NOTE: for auth-gated web + apps (Play Console, ADX) an HTTP 200 only proves the URL is reachable — it does + NOT prove the user has sign-in access, since a login page also returns 200. +""" +from __future__ import annotations +import subprocess +import shutil +from dataclasses import dataclass +from urllib import request as _request +from urllib.error import URLError, HTTPError + + +@dataclass +class CheckResult: + ok: bool + verified_access: bool # True only when we truly proved access (not mere reachability) + detail: str + + +def check_ado_build_def(org: str, project: str, def_id: int, timeout: int = 30) -> CheckResult: + az = shutil.which("az") + if az is None: + return CheckResult(False, False, "az CLI not found") + try: + out = subprocess.run( + [az, "pipelines", "build", "definition", "show", + "--id", str(def_id), "--org", org, "--project", project, + "--query", "name", "-o", "tsv"], + capture_output=True, text=True, timeout=timeout, + ) + except subprocess.TimeoutExpired: + return CheckResult(False, False, f"timeout querying build def {def_id}") + except OSError as e: + return CheckResult(False, False, f"failed to run az: {e}") + if out.returncode == 0 and out.stdout.strip(): + return CheckResult(True, True, f"accessible: '{out.stdout.strip()}'") + err = (out.stderr or "").strip().splitlines() + msg = err[-1] if err else f"cannot access build def {def_id}" + return CheckResult(False, True, msg[:160]) + + +def current_az_user(timeout: int = 20): + """Return the signed-in user's email/UPN from `az account show`, or None. + Used to resolve the release owner without hardcoding an address.""" + az = shutil.which("az") + if az is None: + return None + try: + out = subprocess.run( + [az, "account", "show", "--query", "user.name", "-o", "tsv"], + capture_output=True, text=True, timeout=timeout, + ) + except (subprocess.TimeoutExpired, OSError): + return None + if out.returncode == 0: + val = (out.stdout or "").strip() + return val or None + return None + + +def check_http(url: str, timeout: int = 15) -> CheckResult: + """Reachability check. verified_access is False by design — a 200 from an + auth-gated web app does not prove the user has access.""" + req = _request.Request(url, method="HEAD", headers={"User-Agent": "release-agent-readiness/1.0"}) + try: + with _request.urlopen(req, timeout=timeout) as resp: + code = resp.status + return CheckResult(200 <= code < 400, False, f"reachable (HTTP {code})") + except HTTPError as e: + # Some servers reject HEAD; treat <500 as reachable + return CheckResult(e.code < 500, False, f"reachable (HTTP {e.code})") + except (URLError, TimeoutError) as e: + return CheckResult(False, False, f"unreachable: {e}") + except Exception as e: # noqa + return CheckResult(False, False, f"error: {e}") + + +# ---- CCD pipeline variables (the source of record for Code Complete Date) ---- +# +# These read/write the definition-level variables on pipeline 3038 via the az CLI. +# `overrideCodeCompleteDate` and `skipRelease` are UI/definition variables, which +# is exactly what `az pipelines variable` operates on. + +def read_pipeline_variable(org: str, project: str, def_id: int, name: str, + timeout: int = 30): + """Return (ok, value, detail). value is the variable's string value (may be '') + or None if the variable isn't defined. ok is False only on a real access/CLI error.""" + az = shutil.which("az") + if az is None: + return (False, None, "az CLI not found") + try: + out = subprocess.run( + [az, "pipelines", "variable", "list", "--pipeline-id", str(def_id), + "--org", org, "--project", project, "-o", "json"], + capture_output=True, text=True, timeout=timeout, + ) + except subprocess.TimeoutExpired: + return (False, None, f"timeout listing variables on pipeline {def_id}") + except OSError as e: + return (False, None, f"failed to run az: {e}") + if out.returncode != 0: + err = (out.stderr or "").strip().splitlines() + return (False, None, (err[-1] if err else "az returned non-zero")[:160]) + import json as _json + try: + data = _json.loads(out.stdout or "{}") + except ValueError: + return (False, None, "could not parse az output") + if name in data: + return (True, (data[name] or {}).get("value", "") or "", "ok") + return (True, None, f"variable '{name}' not defined") + + +def set_pipeline_variable(org: str, project: str, def_id: int, name: str, + value: str, timeout: int = 30) -> CheckResult: + """Write a definition variable (update, creating it if absent). A real + production change — callers must gate this behind explicit confirmation.""" + az = shutil.which("az") + if az is None: + return CheckResult(False, False, "az CLI not found") + base = [az, "pipelines", "variable", "{verb}", "--pipeline-id", str(def_id), + "--org", org, "--project", project, "--name", name, "--value", value] + + def _run(verb): + cmd = [c.replace("{verb}", verb) for c in base] + try: + return subprocess.run(cmd, capture_output=True, text=True, timeout=timeout) + except subprocess.TimeoutExpired: + return None + except OSError: + return None + + out = _run("update") + if out is not None and out.returncode == 0: + return CheckResult(True, True, f"{name} = '{value}'") + # update fails if the variable doesn't exist yet — try create. + out2 = _run("create") + if out2 is not None and out2.returncode == 0: + return CheckResult(True, True, f"{name} created = '{value}'") + err = "" + for o in (out2, out): + if o is not None and (o.stderr or "").strip(): + err = (o.stderr or "").strip().splitlines()[-1] + break + return CheckResult(False, True, (err or f"could not set {name}")[:200]) + + +# ---- ADO wiki (payload subpage) -------------------------------------------- +# +# Creates a wiki page via `az devops wiki page create`. A real production write, +# so callers must gate it (the pre-flight agent only calls this on a non-dry-run +# release). Idempotent-ish: an already-existing page is treated as success. + +def wiki_page_exists(org: str, project: str, wiki: str, path: str, + timeout: int = 30): + """True/False whether a wiki page exists, or None if it can't be determined + (e.g. az missing / CLI error) — callers should treat None conservatively.""" + az = shutil.which("az") + if az is None: + return None + try: + out = subprocess.run( + [az, "devops", "wiki", "page", "show", "--path", path, + "--wiki", wiki, "--org", org, "--project", project, "-o", "json"], + capture_output=True, text=True, timeout=timeout, + ) + except (subprocess.TimeoutExpired, OSError): + return None + if out.returncode == 0: + return True + if "could not be found" in (out.stderr or "").lower() or "notfound" in (out.stderr or "").lower(): + return False + return None + + +def create_wiki_page(org: str, project: str, wiki: str, path: str, + content: str, timeout: int = 60) -> CheckResult: + az = shutil.which("az") + if az is None: + return CheckResult(False, False, "az CLI not found") + import tempfile + import os as _os + fd, tmp = tempfile.mkstemp(suffix=".md") + _os.close(fd) + try: + with open(tmp, "w", encoding="utf-8") as fh: + fh.write(content) + cmd = [az, "devops", "wiki", "page", "create", "--path", path, + "--wiki", wiki, "--org", org, "--project", project, + "--file-path", tmp, "--output", "json"] + try: + out = subprocess.run(cmd, capture_output=True, text=True, timeout=timeout) + except subprocess.TimeoutExpired: + return CheckResult(False, True, f"timeout creating wiki page '{path}'") + except OSError as e: + return CheckResult(False, False, f"failed to run az: {e}") + if out.returncode == 0: + return CheckResult(True, True, f"created '{path}'") + stderr = (out.stderr or "").strip() + if "exist" in stderr.lower(): # already there — fine + return CheckResult(True, True, f"page '{path}' already exists") + msg = stderr.splitlines()[-1] if stderr else f"could not create '{path}'" + return CheckResult(False, True, msg[:200]) + finally: + try: + _os.remove(tmp) + except OSError: + pass + + +# ---- Component Governance alerts (read-only) ------------------------------- +# +# CG alerts live on a separate governance host and are read via `az rest` +# (the signed-in user's token). Read-only — we only report. + +def fetch_cg_alerts(resource: str, host: str, project_id: str, repo_id: int, + branch: str, timeout: int = 60): + """Return (ok, alerts, detail). `alerts` is the raw list of alert dicts for + the branch (all states). ok is False on a real CLI/access error.""" + az = shutil.which("az") + if az is None: + return (False, [], "az CLI not found") + url = (f"{host}/{project_id}/_apis/ComponentGovernance/GovernedRepositories/" + f"{repo_id}/Branches/{branch}/Alerts") + try: + out = subprocess.run( + [az, "rest", "--method", "get", "--resource", resource, "--uri", url, "-o", "json"], + capture_output=True, text=True, timeout=timeout, encoding="utf-8", + ) + except subprocess.TimeoutExpired: + return (False, [], "timeout querying Component Governance alerts") + except OSError as e: + return (False, [], f"failed to run az: {e}") + if out.returncode != 0: + err = (out.stderr or "").strip().splitlines() + return (False, [], (err[-1] if err else "az returned non-zero")[:200]) + import json as _json + try: + data = _json.loads(out.stdout or "{}") + except ValueError: + return (False, [], "could not parse az output") + return (True, data.get("value", []) or [], "ok") + + +# ---- scheduled-pipeline verification (Calendar Checker) --------------------- +# +# A YAML pipeline's cron schedule isn't exposed in its definition triggers, but +# whether it's actually FIRING is provable from its build history: a recent +# `schedule`-reason run means the cron is live. Read-only. + +def latest_scheduled_build(org: str, project: str, def_id: int, timeout: int = 60): + """Return (ok, run, detail). `run` is a dict {queueTime, result, status} for + the most recent schedule-reason build, or None if none in recent history.""" + az = shutil.which("az") + if az is None: + return (False, None, "az CLI not found") + try: + out = subprocess.run( + [az, "pipelines", "build", "list", "--definition-ids", str(def_id), + "--org", org, "--project", project, "--top", "25", "-o", "json"], + capture_output=True, text=True, timeout=timeout, encoding="utf-8", + ) + except subprocess.TimeoutExpired: + return (False, None, f"timeout listing builds for definition {def_id}") + except OSError as e: + return (False, None, f"failed to run az: {e}") + if out.returncode != 0: + err = (out.stderr or "").strip().splitlines() + return (False, None, (err[-1] if err else "az returned non-zero")[:200]) + import json as _json + try: + builds = _json.loads(out.stdout or "[]") + except ValueError: + return (False, None, "could not parse az output") + sched = [b for b in builds if b.get("reason") == "schedule"] + if not sched: + return (True, None, "no scheduled runs in recent history") + latest = max(sched, key=lambda b: b.get("queueTime") or "") + return (True, {"queueTime": latest.get("queueTime"), "result": latest.get("result"), + "status": latest.get("status")}, "ok") + diff --git a/settings.gradle b/settings.gradle index 97a748c6..63cdf70b 100644 --- a/settings.gradle +++ b/settings.gradle @@ -186,23 +186,23 @@ project(':labapi').projectDir = new File('common/labapi') include(":keyvault") project(':keyvault').projectDir = new File('common/keyvault') -include(":AcaPlugin") -project(':AcaPlugin').projectDir = new File('plugins/buildsystem') +//include(":AcaPlugin") +//project(':AcaPlugin').projectDir = new File('plugins/buildsystem') include(":broker4j") project(':broker4j').projectDir = new File('broker/broker4j') -include(":LinuxBroker") -project(':LinuxBroker').projectDir = new File('broker/LinuxBroker') +//include(":LinuxBroker") +//project(':LinuxBroker').projectDir = new File('broker/LinuxBroker') include(':LabApiUtilities') project(':LabApiUtilities').projectDir = new File('common/LabApiUtilities') -include(':java-linux-test-app') -project(':java-linux-test-app').projectDir = new File('broker/java-linux-test-app') +//include(':java-linux-test-app') +//project(':java-linux-test-app').projectDir = new File('broker/java-linux-test-app') -include(":LinuxBrokerPackage") -project(':LinuxBrokerPackage').projectDir = new File('/broker/LinuxBrokerPackage') +//include(":LinuxBrokerPackage") +//project(':LinuxBrokerPackage').projectDir = new File('/broker/LinuxBrokerPackage') include(":AzureSample") project(':AzureSample').projectDir = new File('azuresample/app') @@ -219,8 +219,8 @@ project(':mockltw').projectDir = new File('broker/mockbrokers/mockltw') include(":mockbrokerapplib") project(':mockbrokerapplib').projectDir = new File('broker/mockbrokers/mockbrokerapplib') -include(":NativeAuthSample") -project(':NativeAuthSample').projectDir = new File('nativeauthsample/app') +//include(":NativeAuthSample") +//project(':NativeAuthSample').projectDir = new File('nativeauthsample/app') // Authenticator App Projects — only included when the opt-in flag is set. if (includeAuthenticatorApp) { From 19688fdb02ce0275a4562d7ab2029bb0e9c79dc6 Mon Sep 17 00:00:00 2001 From: p3dr0rv Date: Wed, 5 Aug 2026 16:24:42 -0700 Subject: [PATCH 02/82] Implement inter-process state locking for CLI commands to prevent race conditions and ensure safe parallel execution; enhance error messaging for Azure DevOps authentication issues. --- release-agent/orchestrator/cli.py | 7 ++- release-agent/orchestrator/cli_common.py | 76 ++++++++++++++++++++++++ release-agent/orchestrator/engine.py | 5 +- release-agent/orchestrator/render.py | 13 ++++ release-agent/phases/agents/preflight.py | 19 +++++- release-agent/skill/SKILL.md | 8 ++- release-agent/tests/test_engine.py | 70 +++++++++++++++++++++- release-agent/tools/checks.py | 17 +++++- 8 files changed, 205 insertions(+), 10 deletions(-) diff --git a/release-agent/orchestrator/cli.py b/release-agent/orchestrator/cli.py index 49f7e464..d39e49a6 100644 --- a/release-agent/orchestrator/cli.py +++ b/release-agent/orchestrator/cli.py @@ -46,7 +46,12 @@ def build_parser(): def main(argv=None): args = build_parser().parse_args(argv) - return args.func(args) + # Serialize state read-modify-write per release so parallel CLI invocations + # (e.g. the skill firing record-step calls at once) can't clobber each other. + runs_root = getattr(args, "runs_root", None) + release = C.effective_release(runs_root, getattr(args, "release", None)) + with C.state_lock(runs_root, release): + return args.func(args) if __name__ == "__main__": diff --git a/release-agent/orchestrator/cli_common.py b/release-agent/orchestrator/cli_common.py index ade5a53b..101c829e 100644 --- a/release-agent/orchestrator/cli_common.py +++ b/release-agent/orchestrator/cli_common.py @@ -12,6 +12,8 @@ from __future__ import annotations import os +import time +from contextlib import contextmanager from orchestrator.state import ReleaseState from orchestrator.engine import Orchestrator @@ -28,6 +30,80 @@ # runs live OUTSIDE release-agent/, in android-complete/.release-runs (gitignored) DEFAULT_RUNS_ROOT = os.path.join(os.path.dirname(ROOT), ".release-runs") +# ---- inter-process state lock ---- +_LOCK_TIMEOUT = 30.0 # max seconds to wait for another CLI process to release +_LOCK_STALE = 120.0 # a lock older than this is treated as abandoned (crashed proc) + + +@contextmanager +def state_lock(runs_root: str, release): + """Serialize a release's state read-modify-write ACROSS CLI processes. + + Every mutating command loads state, mutates, then saves. Two running at once + (e.g. the skill firing `record-step` calls in parallel, or an hourly `tick` + overlapping an interactive command) would clobber each other — a last-writer- + wins lost update. This exclusive per-release lock makes each CLI invocation + atomic: a second process blocks until the first has saved and released. + Read-only commands hold it only for their brief duration. + + No release (e.g. `list`, `infra`) → no lock: nothing release-scoped to guard. + A lock older than _LOCK_STALE is stolen (its owner crashed). + """ + if not release: + yield + return + lock_dir = os.path.join(runs_root, release) + os.makedirs(lock_dir, exist_ok=True) + lock_path = os.path.join(lock_dir, ".state.lock") + deadline = time.monotonic() + _LOCK_TIMEOUT + fd = None + while True: + try: + fd = os.open(lock_path, os.O_CREAT | os.O_EXCL | os.O_WRONLY) + os.write(fd, str(os.getpid()).encode()) + break + except FileExistsError: + try: + if time.time() - os.path.getmtime(lock_path) > _LOCK_STALE: + os.remove(lock_path) # abandoned by a crashed process + continue + except OSError: + pass + if time.monotonic() > deadline: + raise TimeoutError( + f"could not acquire state lock for release {release} within " + f"{_LOCK_TIMEOUT:.0f}s — another CLI process is holding it") + time.sleep(0.05) + try: + yield + finally: + try: + os.close(fd) + except OSError: + pass + try: + os.remove(lock_path) + except OSError: + pass + + +def effective_release(runs_root, release): + """The release id to lock on. The explicit `--release` when given; otherwise, + for discovery-mode mutating commands (e.g. the hourly `tick`), the single + active release so it's still serialized against interactive commands. Returns + None when ambiguous / none exist (nothing to serialize on).""" + if release: + return release + if not runs_root: + return None + try: + res = discovery.resolve(runs_root, None) + if res.get("resolution") == "one" and res.get("release"): + return res["release"].get("release_id") + except Exception: + pass + return None + # ---- paths / state ---- def state_path(runs_root: str, release: str) -> str: diff --git a/release-agent/orchestrator/engine.py b/release-agent/orchestrator/engine.py index 37207370..0cf05d76 100644 --- a/release-agent/orchestrator/engine.py +++ b/release-agent/orchestrator/engine.py @@ -501,8 +501,9 @@ def _active_phase_report(self) -> Optional[dict]: steps_view = [] for s in steps: sid = s["id"] + stp = self.state.get_step(phase["id"], sid) s_done = self.state.is_done(phase["id"], sid) - s_blocked = self.state.get_step(phase["id"], sid).status == "blocked" + s_blocked = stp.status == "blocked" is_gate = bool(s.get("gate")) is_rem = self._is_reminder(s) is_scout = s.get("source") == "scout" @@ -523,6 +524,7 @@ def _active_phase_report(self) -> Optional[dict]: steps_view.append({ "id": sid, "name": s["name"], "status": status, "needs_owner": needs, + "note": stp.note, # agent result / block reason / detail "now": bool(sid == cur and not s_done and (is_gate or is_rem or is_scout or is_attest or s_blocked)), }) opens = self._phase_anchor_date(phase) @@ -607,6 +609,7 @@ def _current_steps(self, current_phase_obj) -> list: "reminder": self._is_reminder(s), "owner": s.get("owner", "agent"), "state": s_state, + "note": rec.get("note"), # agent result / block reason / detail }) return out diff --git a/release-agent/orchestrator/render.py b/release-agent/orchestrator/render.py index a8119eda..5c393b4d 100644 --- a/release-agent/orchestrator/render.py +++ b/release-agent/orchestrator/render.py @@ -174,6 +174,19 @@ def status_view(r: dict) -> str: tag = " 🚦" if s["gate"] else (" 📌" if s.get("reminder") else "") lines.append(f"| {icon} | {s['name']}{tag} | {word} |") + # 3b) Results & activity — surface each step's stored outcome (agent report, + # created link, block reason) so a done/blocked step isn't just "Done" with + # no evidence. Full multi-line notes (e.g. the CG report) are preserved. + results = [s for s in r["current_steps"] + if s.get("note") and s["state"] in ("done", "blocked")] + if results: + lines += ["", "### Results & activity"] + for s in results: + mark = "⛔" if s["state"] == "blocked" else "✅" + note_lines = [ln.rstrip() for ln in str(s["note"]).strip().split("\n")] + body = " \n".join(ln for ln in note_lines if ln) # markdown hard breaks + lines += ["", f"{mark} **{s['name']}** ", body] + return "\n".join(lines) diff --git a/release-agent/phases/agents/preflight.py b/release-agent/phases/agents/preflight.py index d7d41d55..2b3c1fc0 100644 --- a/release-agent/phases/agents/preflight.py +++ b/release-agent/phases/agents/preflight.py @@ -132,6 +132,12 @@ def _page_name(state, n: int = 1) -> str: return f"{base} {n} Release" if n and n >= 2 else f"{base} Release" +def _wiki_url(org: str, project: str, wiki: str, path: str) -> str: + """Browser URL for an ADO wiki page (so the result links to the created page).""" + from urllib.parse import quote + return f"{(org or '').rstrip('/')}/{project}/_wiki/wikis/{wiki}?pagePath={quote(path or '')}" + + def run_wiki(phase_id: str, step: dict, dry_run: bool, state=None) -> StepResult: cfg = _load_cfg().get("wiki", {}) org = cfg.get("org") @@ -144,7 +150,8 @@ def run_wiki(phase_id: str, step: dict, dry_run: bool, state=None) -> StepResult return StepResult( True, f"[dry-run] Would create payload wiki subpage '{base_name}' under " - f"'{parent}' (duplicate-safe: a second numbered page if it already exists).", + f"'{parent}' (duplicate-safe: a second numbered page if it already exists).\n" + f"Would live at: {_wiki_url(org, project, wiki, base_path)}", "agent", ) if not (org and project and wiki and parent): @@ -165,7 +172,8 @@ def run_wiki(phase_id: str, step: dict, dry_run: bool, state=None) -> StepResult return StepResult( True, f"⚠ A payload page already exists for this month ('{base_name}'). " - f"Left it untouched and created a SECOND page: '{cand_name}'. ({res.detail})", + f"Left it untouched and created a SECOND page: '{cand_name}'.\n" + f"Link: {_wiki_url(org, project, wiki, cand_path)}", "agent", ) n += 1 @@ -174,7 +182,12 @@ def run_wiki(phase_id: str, step: dict, dry_run: bool, state=None) -> StepResult res = create_wiki_page(org, project, wiki, base_path, _payload_template(state)) if not res.ok: return StepResult(False, f"wiki: could not create '{base_path}' — {res.detail}", "agent") - return StepResult(True, f"Payload wiki subpage ready: '{base_name}' ({res.detail})", "agent") + return StepResult( + True, + f"Payload wiki subpage ready: '{base_name}'.\n" + f"Link: {_wiki_url(org, project, wiki, base_path)}", + "agent", + ) # ---- Component Governance alerts (report-only) ----------------------------- diff --git a/release-agent/skill/SKILL.md b/release-agent/skill/SKILL.md index 39506dfa..def9ebb4 100644 --- a/release-agent/skill/SKILL.md +++ b/release-agent/skill/SKILL.md @@ -127,7 +127,7 @@ If any item is unsatisfied the gate stays closed. If the engineer can't satisfy Flow after starting: 1. `python -m orchestrator.cli checklist --release --verify` — this runs the auto checks AND prints the **canonical checklist table (markdown)**. **Reproduce its stdout into your reply as live markdown (NOT wrapped in a ``` code fence)** so Scout renders it as a real table — it is already a finished markdown table with the type labels, per-item status, and clickable links. **Do NOT rebuild, re-format, re-order, re-label, or re-type any of it from memory, and do NOT fence it.** If you reconstruct it you WILL introduce errors (stale icons, mangled/merged URLs); if you fence it, it shows as raw text. Always reproduce the literal command output as rendered markdown. You may add a sentence of your own before or after, but the table block itself must match the output. 2. There are exactly **two types by resolver**: `[auto]` (Scout verifies) and `[attest]` (the user confirms). All items must be satisfied to clear the gate. (Do not add lock icons or a "hard requirement" legend.) -3a. **Run the scout-assisted `[auto]` checks yourself, then record each result** — don't ask the user for these; they're verified, not attested. +3a. **Run the scout-assisted `[auto]` checks yourself, then record each result** — don't ask the user for these; they're verified, not attested. **Do this quietly: run the checks and `record-check` them without narrating each one** (a wall of per-check prose is what buries the table). The only time 3a surfaces anything to the user is the `silent_perms` opt-out choice below. - **`oncall_now` (ICM):** call the ICM MCP `get_on_call_schedule_by_team_id` with `teamIds: [78848]` ("Auth Client Android Shield"). Resolve the current user's alias (`get_my_icm_context` or the owner email's local part), then decide by their role in `shiftCurrentOnCalls[].currentOnCallContacts[]`: - **Not in the roster at all** → `record-check --item oncall_now --status pass --detail "not on the current roster"`. - **Present but NOT the primary** (i.e. they are a **backup/secondary** — any position other than the first-listed contact) → **pass**: `record-check --item oncall_now --status pass --detail "backup OCE, not primary (primary: )"`. A backup is free to run the release. @@ -142,7 +142,7 @@ Flow after starting: - They pick **Enable** → the only manual step is the Scout master toggle: if `permissions.allowModelPermissionsChange` is `false`, tell them to turn on **Settings → Permissions → "Allow AI to request permission changes"** (I cannot flip it — it's read-only from the model, by design). Once it's `true`, call **`m_request_permission_escalation`** with `servers: { workiq: {autoApprove:true}, playwright: {autoApprove:true} }` (add `shell` if off too); they click **Allow** once, then re-read `m_get_settings` and `record-check … --status pass --detail "enabled silent runs"`. - They pick **Proceed without** (or won't enable the master toggle) → `record-check --release --item silent_perms --status degraded --detail "proceeding without silent runs — unattended digest/Teams/browser checks will prompt & may stall until Scout is opened"`. **`degraded` satisfies the gate** (the checklist shows it as ⚠️ *Proceeding (not silent)*), so the release can start; the downside is on record. - A `fail` on `oncall_now`/`adx_access`, or a real problem, keeps the gate closed — treat it like any unsatisfiable required item (resolve or hand off). Do NOT attest these — they're `auto` items you verified. (`silent_perms` is the one soft/opt-out auto item: it uses `degraded`, never `fail`, when the user chooses to proceed.) -3b. Use `m_ask_user` to collect the **attestations**: `play_console_access`, the on-call **window** (`oncall_window` — show the CCD‑7 → CCD+14 dates from the checklist), `saw_ame`, `yubikey`. Offer: **"All confirmed"**, **"I'm scheduled on-call during the window"**, **"I can't open Play Console"**, **"I don't have a SAW machine"**, **"I don't have a YubiKey"**. +3b. **Re-anchor on the TABLE, then attest.** After recording the auto results, **run `python -m orchestrator.cli checklist --release ` again and reproduce its updated markdown table** — the auto items now show ✅ and only the attests are Outstanding. This table is the single source of truth and **must be shown immediately before you ask for attestations**; the auto-check work in 3a pushes the first table far up, so re-render it here every time. Then use `m_ask_user` to collect the four attestations (`play_console_access`; the on-call **window** `oncall_window` — the CCD‑7 → CCD+14 dates are in the table; `saw_ame`; `yubikey`). Offer: **"All confirmed"**, **"I'm scheduled on-call during the window"**, **"I can't open Play Console"**, **"I don't have a SAW machine"**, **"I don't have a YubiKey"**. **Never replace the table with a plain prose list of the items** — the `m_ask_user` prompt accompanies the re-rendered table, it does not substitute for it. 4. If they confirm everything → `sign --release --all`, then `next` to begin Phase 0. 5. **If they can't satisfy any attest item** (on-call during the window, no SAW, no YubiKey, no portal access) → `decline --release --item ` (repeat `--item` for each). The gate is now blocked. Tell them plainly: the release can't start until that item is resolved; if they can't resolve it, hand the release to another engineer who can (notify their manager / the release team). Treat every item this way — don't single any out as harder or softer. 6. If an **auto** item shows FAIL (no build-definition access, or you're on-call), the gate stays closed — a real problem to resolve, not something to attest around. @@ -154,10 +154,14 @@ Never hand-edit or regenerate the checklist/status blocks — always show the CL Some phases run **in parallel** (Phase 0 is `execution: parallel`): a single `next` runs **every independent automated step at once** (breaking, CG, cron, wiki — all complete in one call) and then surfaces **all the human/scout holds together** (e.g. *"4 item(s) need you: …"*). So don't treat it as one-step-at-a-time. After `next`, read `status --json` and look at **`pending_human`** (and `active_phase.steps` with their `status`/`needs_owner`) — that's the full set of what's outstanding. Work through **all** of them in this pass: - **`source: scout`** steps (notice, flight_reminder, lockdown) → run each via MCP/browser + `record-step` (see the sections below). These are independent — do them all. + +> **State writes are safe to parallelize.** The CLI serializes every state read-modify-write per release with an exclusive lock, so firing several `record-step` / `record-check` / `done` calls at once (or an hourly `tick` overlapping your command) can't clobber each other — a second invocation simply waits for the first to save. You don't need to run them one-at-a-time to avoid races. - **`attest`** steps (confirm_reminders, vitals) → ask the owner to confirm, then `done --step `. - **`blocked`** steps (cg/cron on a real problem) → show the note; fix + rerun, or skip. Dependencies still hold: `confirm_reminders` only appears **after** `flight_reminder` is sent (it won't be in `pending_human` until then). Call `next` again after clearing holds to let newly-ready steps surface and, once all are done, advance to the next phase. +> **ALWAYS show the `status` table each iteration — never a bare prose list.** After every `next`, and after every `done`/`record-step` that clears a hold, run `python -m orchestrator.cli status --release ` (no `--json`, add `--as-of` if simulating) and **reproduce its full output as live markdown**: the phase map, the current-phase **steps table** (what's done ✅ / pending / needs you 📌), and the **Results & activity** section (each agent's result — the CG report, the created **wiki link**, block reasons). This is the source of truth the user asked for; the human-readable `status` also auto-logs what was shown. Use `status --json` only for your own branching — but still show the rendered table to the user. Then, alongside the table, tell the user exactly what each outstanding step needs (run the scout steps yourself; for the `attest` steps spell out what to confirm). Do **not** replace the table with your own summarized list of remaining steps. + ## Scout-assisted phase steps (CCOA lockdown check) Some Phase steps read AAD-gated sources the deterministic engine can't reach, so **you run them via the browser and record the result** — same idea as the readiness scout checks, but mid-phase. When advancing, if **`lockdown`** is among the pending holds (`source: scout`, in `pending_human`), handle it like this — silently, without bothering the user unless there's an overlap: diff --git a/release-agent/tests/test_engine.py b/release-agent/tests/test_engine.py index 0b42df0f..f95db780 100644 --- a/release-agent/tests/test_engine.py +++ b/release-agent/tests/test_engine.py @@ -853,8 +853,76 @@ def test_registry_register_list_deregister(): assert len(reg.list()) == 1 -# ---- Phase-0 real pre-flight agents (breaking detect, wiki payload) ---- +# ---- inter-process state lock (parallel CLI mutation safety) ---- + +def test_state_lock_is_exclusive_then_releases(): + """While a release's state lock is held, a second acquisition blocks (times + out); once released it can be acquired again.""" + import threading + with tempfile.TemporaryDirectory() as rr: + R = "2099-02" + os.makedirs(os.path.join(rr, R)) + acquired, timed_out = [], [] + with C.state_lock(rr, R): + orig = C._LOCK_TIMEOUT + C._LOCK_TIMEOUT = 0.3 + def try_acquire(): + try: + with C.state_lock(rr, R): + acquired.append(True) + except TimeoutError: + timed_out.append(True) + t = threading.Thread(target=try_acquire) + t.start(); t.join() + C._LOCK_TIMEOUT = orig + assert timed_out and not acquired # blocked while held + with C.state_lock(rr, R): # released -> acquirable + acquired.append("after") + assert "after" in acquired + + +def test_concurrent_record_check_both_persist(): + """Two record-check CLI invocations fired at the same instant must BOTH + persist — the per-release lock prevents the last-writer-wins clobber that + dropped `notice` in the live test (parallel state-write race).""" + import threading + from orchestrator import cli as _cli + with tempfile.TemporaryDirectory() as rr: + R = "2099-03" + _cli.main(["--runs-root", rr, "init", "--release", R, + "--owner-email", "t@example.com", "--owner-name", "T"]) + barrier = threading.Barrier(2) + + def rec(item): + barrier.wait() # maximize overlap + _cli.main(["--runs-root", rr, "record-check", "--release", R, + "--item", item, "--status", "pass", "--detail", item]) + threads = [threading.Thread(target=rec, args=(i,)) + for i in ("oncall_now", "adx_access")] + for t in threads: + t.start() + for t in threads: + t.join() + st = C.load_state(rr, R) + assert st.readiness_items.get("oncall_now", {}).get("status") == "pass" + assert st.readiness_items.get("adx_access", {}).get("status") == "pass" + + +def test_status_surfaces_agent_result_notes_and_wiki_link(): + """Agent results are stored as the step note and surfaced in status: the + Results & activity section shows each done agent's output, incl. the wiki link.""" + from orchestrator import render + st, orch = _ccd_orch("2026-07-02") # Phase 0 open + orch.run_until_gate() # runs breaking/cg/cron/wiki (set notes) + r = orch.status_report() + steps = {s["id"]: s for s in r["current_steps"]} + assert steps["cg"].get("note") and steps["wiki"].get("note") + view = render.status_view(r) + assert "Results & activity" in view + assert "Would live at" in view # the wiki link surfaces (dry-run) + +# ---- Phase-0 real pre-flight agents (breaking detect, wiki payload) ---- _SAMPLE_CHANGELOG = """vNext ---------- - [MINOR] add a thing (#1) diff --git a/release-agent/tools/checks.py b/release-agent/tools/checks.py index 677f09e7..e490d504 100644 --- a/release-agent/tools/checks.py +++ b/release-agent/tools/checks.py @@ -38,9 +38,22 @@ def check_ado_build_def(org: str, project: str, def_id: int, timeout: int = 30) return CheckResult(False, False, f"failed to run az: {e}") if out.returncode == 0 and out.stdout.strip(): return CheckResult(True, True, f"accessible: '{out.stdout.strip()}'") - err = (out.stderr or "").strip().splitlines() + err_text = (out.stderr or "").strip() + low = err_text.lower() + # Distinguish a real ACCESS problem from an az-not-authenticated-to-ADO state + # (org rejects the CLI's AAD token — often Conditional Access — and returns the + # sign-in page / 401). Give an actionable message instead of a raw 401. + if ("requires user authentication" in low or "unauthorized" in low + or "sign in" in low or "tf400813" in low or "no credentials" in low): + return CheckResult( + False, True, + f"az is not authenticated to Azure DevOps (build def {def_id}); the org " + f"rejected the CLI token. Run `az login`; if it still fails (Conditional " + f"Access), sign in with a PAT: `az devops login --organization {org}` " + f"(Build: Read scope).") + err = err_text.splitlines() msg = err[-1] if err else f"cannot access build def {def_id}" - return CheckResult(False, True, msg[:160]) + return CheckResult(False, True, msg[:200]) def current_az_user(timeout: int = 20): From d5ea1d4a140a3deb11342da81352d3286333add6 Mon Sep 17 00:00:00 2001 From: p3dr0rv Date: Wed, 5 Aug 2026 16:26:36 -0700 Subject: [PATCH 03/82] Ignore local pipeline and wiki repositories --- .gitignore | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.gitignore b/.gitignore index 2802f33f..733b31d1 100644 --- a/.gitignore +++ b/.gitignore @@ -16,6 +16,8 @@ tsl msalcpp design-docs 1ES-Pipelines +AuthClientAndroidPipelines +IdentityWiki.wiki nativeauthsample # Gradle files From dc5d8ab27f3c3ca072cba12aa50ae695b075bd63 Mon Sep 17 00:00:00 2001 From: p3dr0rv Date: Thu, 13 Aug 2026 20:47:57 +0100 Subject: [PATCH 04/82] release-agent: harden readiness gate + modularize the skill Engine (gate integrity): - sign: remove blanket --all; CLI now requires explicit --item ids + refuses a bare sign, and records each attestation individually with an evidence --note. Closes the hole where sign --all attested every human item in one blind call (a release reached Phase 0 without real confirmation). - readiness.sign() carries a per-item evidence note. Readiness config/UX: - silent_perms.required_servers now includes kusto + icm (the MCP servers the adx_access / oncall_now checks call) so they don't prompt on first run. Skill (A: dedupe, B: modular split): - Split the 48KB monolith SKILL.md into a lean 7KB core (golden rules + behaviour dispatch + reference routing table) plus skill/reference/*.md read on demand (readiness-gate, starting-and-scheduling, presenting-status, commands, phases/preflight, phases/_TEMPLATE). Core is now well under Scout's inline limit, so it no longer spills / gets skimmed. - Fix regression where the readiness table was suppressed: render the checklist table FIRST, then handle silent-runs (was "before showing the checklist"). - Anti-assumption rule: never attest/approve on an echoed m_ask_user result. - Scale convention: add a phase = phases.yaml + phases/agents/.py + reference/phases/.md + one routing-table row. bootstrap.ps1: auto-install Python via winget + silent pyyaml; verify Scout present + skill copy; restart Scout by default (guarded when run inside a Scout session); UTF-8 + detached-launch fixes; clearer folder/run instructions. Tests: 101/101 (added regressions for bare-sign refusal, no --all, evidence note). Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- release-agent/README.md | 8 +- release-agent/config/readiness.yaml | 8 +- .../orchestrator/commands/readiness.py | 36 +- release-agent/orchestrator/readiness.py | 13 +- release-agent/setup/bootstrap.ps1 | 235 +++++++++++-- release-agent/skill/SKILL.md | 324 +++--------------- release-agent/skill/reference/commands.md | 52 +++ .../skill/reference/phases/_TEMPLATE.md | 28 ++ .../skill/reference/phases/preflight.md | 54 +++ .../skill/reference/presenting-status.md | 14 + .../skill/reference/readiness-gate.md | 60 ++++ .../reference/starting-and-scheduling.md | 70 ++++ release-agent/tests/test_engine.py | 41 ++- 13 files changed, 625 insertions(+), 318 deletions(-) create mode 100644 release-agent/skill/reference/commands.md create mode 100644 release-agent/skill/reference/phases/_TEMPLATE.md create mode 100644 release-agent/skill/reference/phases/preflight.md create mode 100644 release-agent/skill/reference/presenting-status.md create mode 100644 release-agent/skill/reference/readiness-gate.md create mode 100644 release-agent/skill/reference/starting-and-scheduling.md diff --git a/release-agent/README.md b/release-agent/README.md index be2932ad..7dc8fdfb 100644 --- a/release-agent/README.md +++ b/release-agent/README.md @@ -93,9 +93,13 @@ The conductor is **stateless**: on each invocation it loads the record, (later) ## Quick start +Run the one-time setup from the **`release-agent` folder of your `android-complete` clone**, +using PowerShell 7 (`pwsh`): + ```powershell -# one-time -pwsh ./setup/bootstrap.ps1 +# one-time — from the release-agent folder +cd C:\repos\android-complete\release-agent # adjust to your clone location +pwsh .\setup\bootstrap.ps1 ``` `bootstrap.ps1` runs an **infrastructure preflight** first (`python -m orchestrator.cli infra`), diff --git a/release-agent/config/readiness.yaml b/release-agent/config/readiness.yaml index bd9edd3c..add4d6a1 100644 --- a/release-agent/config/readiness.yaml +++ b/release-agent/config/readiness.yaml @@ -64,13 +64,15 @@ items: - id: silent_perms label: "Silent-run permissions" - text: "Permissions allow fully unattended runs — shell, WorkIQ (email + Teams), and the browser are auto-approved so scheduled work never stalls on a prompt" - detail: "Scout checks shell, WorkIQ (email + Teams), and the browser are all auto-approved so the daily automation runs silently when Scout isn't focused." + text: "Permissions allow fully unattended runs — shell, WorkIQ (email + Teams), the browser, and the ICM + Kusto/ADX MCP servers are auto-approved so scheduled work and the on-call/telemetry checks never stall on a prompt" + detail: "Scout checks shell, WorkIQ (email + Teams), the browser, and the ICM + Kusto/ADX MCP servers are all auto-approved so the daily automation and the readiness on-call/telemetry checks run silently when Scout isn't focused." verify: auto source: scout # only the skill can read Scout's own settings (m_get_settings) verifier: silent_perms # The servers that must be auto-approved for a fully-silent run (kept as data). - required_servers: [shell, workiq, playwright] + # Includes the MCP servers the readiness auto-checks call: `kusto` (adx_access) and + # `icm` (oncall_now) — otherwise those checks prompt on first run. + required_servers: [shell, workiq, playwright, kusto, icm] # OPT-OUT (soft): enabling silent runs needs the user to turn on the Scout master # toggle "Allow AI to request permission changes" (allowModelPermissionsChange) — # which ONLY the user can flip in the UI. The skill offers to enable silent runs; diff --git a/release-agent/orchestrator/commands/readiness.py b/release-agent/orchestrator/commands/readiness.py index e4a75137..5b6786c6 100644 --- a/release-agent/orchestrator/commands/readiness.py +++ b/release-agent/orchestrator/commands/readiness.py @@ -36,17 +36,35 @@ def cmd_verify(args): def cmd_sign(args): st, orch = C.load_orch(args.runs_root, args.release, args.config) - ids = None if args.all else (args.item or []) - chk = orch.gate.sign(ids) + ids = args.item or [] + if not ids: + # No blanket sign: the engineer must name the item(s) they confirmed. + # This closes the integrity hole where `sign --all` attested every human + # item in one blind call with no evidence and no per-item confirmation. + print("Refusing to sign: name the item(s) you confirmed with --item " + "(repeatable). Attest only what the engineer explicitly confirmed.") + return 2 + # Validate the ids are real attest items before recording anything. + chk_before = orch.gate.checklist() + attest_ids = {i["id"] for i in chk_before["attest_items"]} + unknown = [i for i in ids if i not in attest_ids] + if unknown: + print(f"Not attestable (unknown or not an attest item): {', '.join(unknown)}") + return 2 + chk = orch.gate.sign(ids, note=args.note or None) C.save_state(st, args.runs_root, args.release) - C.elog(args.runs_root, args.release).log( - "readiness_signed" if chk["signed"] else "readiness_partial", - items=("all" if ids is None else ids), signed=chk["signed"]) + el = C.elog(args.runs_root, args.release) + # Log EACH attestation individually with its evidence note, so the trail shows + # exactly what was confirmed (not one opaque items:"all"). + for iid in ids: + el.log("readiness_attested", item=iid, driver=args.note or None) + el.log("readiness_signed" if chk["signed"] else "readiness_partial", + items=ids, signed=chk["signed"]) if chk["signed"]: print(f"Readiness signed at {chk['signed_at']}. Entry gate cleared — you can now start Phase 0.") else: pending = [i["id"] for i in chk["items"] if not i["satisfied"]] - print(f"Recorded. Still pending: {', '.join(pending)}") + print(f"Recorded {', '.join(ids)}. Still pending: {', '.join(pending)}") return 0 @@ -110,8 +128,10 @@ def register(sub): sg = sub.add_parser("sign", help="Attest human readiness items (also runs auto verify)") sg.add_argument("--release", required=True) - sg.add_argument("--all", action="store_true", help="Attest every human item") - sg.add_argument("--item", action="append", help="Attest a specific item id (repeatable)") + sg.add_argument("--item", action="append", + help="Attest a specific item id the engineer confirmed (repeatable, required)") + sg.add_argument("--note", default="", + help="Evidence: what the engineer confirmed (recorded per item)") sg.set_defaults(func=cmd_sign) dc = sub.add_parser("decline", help="Declare you CANNOT satisfy an item (may block ownership)") diff --git a/release-agent/orchestrator/readiness.py b/release-agent/orchestrator/readiness.py index 2d35ccde..87838789 100644 --- a/release-agent/orchestrator/readiness.py +++ b/release-agent/orchestrator/readiness.py @@ -145,10 +145,12 @@ def record_check(self, item_id: str, status: str, message: str = "") -> dict: self._refresh_signed() return self.checklist() - def sign(self, item_ids=None) -> dict: + def sign(self, item_ids=None, note=None) -> dict: """Attest human (attest) items and run auto verifiers. item_ids=None - attests every attest item. Auto items are only set by verification — - they cannot be hand-waved through. Signs when all items are satisfied.""" + attests every attest item (library convenience — the CLI never does this; + it requires explicit item ids). Auto items are only set by verification — + they cannot be hand-waved through. `note` records the human's confirmation + as evidence on each attested item. Signs when all items are satisfied.""" if not self.config: self.state.readiness_signed = True self.state.readiness_signed_at = _now() @@ -158,7 +160,10 @@ def sign(self, item_ids=None) -> dict: attest_ids = [it["id"] for it in all_items if it.get("verify", "attest") == "attest"] targets = attest_ids if item_ids is None else [i for i in item_ids if i in attest_ids] for iid in targets: - self.state.readiness_items[iid] = {"status": "attested", "at": _now()} + rec = {"status": "attested", "at": _now()} + if note: + rec["note"] = note + self.state.readiness_items[iid] = rec self._refresh_signed() return self.checklist() diff --git a/release-agent/setup/bootstrap.ps1 b/release-agent/setup/bootstrap.ps1 index 44a53189..f56433a2 100644 --- a/release-agent/setup/bootstrap.ps1 +++ b/release-agent/setup/bootstrap.ps1 @@ -8,24 +8,107 @@ Steps: 1. Infrastructure preflight — check CLIs/host deps AND register + verify the MCP servers the skill needs inside Scout (from config/requirements.yaml). - 2. Install the /release-agent skill into the Scout skills folder. - 3. Print next steps. + If Python 3.9+ is missing it is installed automatically via winget + (per-user, no admin); PyYAML is then installed via pip. Pass + -NoAutoInstallPython to opt out of the Python auto-install. + 2. Install the /release-agent skill into the Scout skills folder — only if + Microsoft Scout is detected (~/.scout present); the copy is then verified. + 3. Print next steps. By default the script closes and relaunches Scout so the + new skill / MCP servers load (skipped automatically when this setup is run + from inside a Scout session, to avoid killing itself). Pass -NoRestartScout + to opt out and restart Scout yourself. + + HOW TO RUN + Run this from the `release-agent` folder of your android-complete clone, + using PowerShell 7 (pwsh): + + cd \android-complete\release-agent + pwsh .\setup\bootstrap.ps1 + + Example (default clone location): + + cd C:\repos\android-complete\release-agent + pwsh .\setup\bootstrap.ps1 + + You can launch it from any working directory as long as you give the full + path to the script (the script locates its own folder), e.g.: + + pwsh C:\repos\android-complete\release-agent\setup\bootstrap.ps1 .EXAMPLE - pwsh ./setup/bootstrap.ps1 + cd C:\repos\android-complete\release-agent + pwsh .\setup\bootstrap.ps1 #> [CmdletBinding()] param( [string]$ScoutSkillsDir = "$env:USERPROFILE\.scout\m-skills", - [switch]$SkipSkillInstall + [switch]$SkipSkillInstall, + [switch]$NoAutoInstallPython, # by default, if Python is missing we install it via winget + [switch]$NoRestartScout # by default we restart Scout so new skill/MCP servers load ) $ErrorActionPreference = "Stop" + +# The Python engine prints UTF-8 (em-dashes, etc.). Make sure this console reads/writes +# UTF-8 so echoed output doesn't turn into mojibake like "ΓÇö". +try { + [Console]::OutputEncoding = [System.Text.Encoding]::UTF8 + $OutputEncoding = [System.Text.Encoding]::UTF8 + $env:PYTHONIOENCODING = "utf-8" + $env:PYTHONUTF8 = "1" +} catch {} + $AgentRoot = Split-Path -Parent $PSScriptRoot # release-agent/ $RepoRoot = Split-Path -Parent $AgentRoot # android-complete/ $ReqFile = Join-Path $AgentRoot "config\requirements.yaml" Write-Host "Release Orchestrator bootstrap`n" -ForegroundColor Cyan +Write-Host " Running from: $AgentRoot" -ForegroundColor DarkGray +Write-Host " (run this from the 'release-agent' folder of your android-complete clone)`n" -ForegroundColor DarkGray + +# Sanity check: make sure we're actually in the release-agent folder. +if (-not (Test-Path (Join-Path $AgentRoot 'setup\bootstrap.ps1'))) { + Write-Host " This does not look like the release-agent folder." -ForegroundColor Red + Write-Host " cd into \android-complete\release-agent and run: pwsh .\setup\bootstrap.ps1`n" -ForegroundColor Red + exit 1 +} + +# True when THIS script was launched from a shell inside a Scout session — Scout sets +# these env vars for its child processes. Restarting Scout from there would kill us. +$InsideScout = [bool]($env:COPILOT_AGENT_SESSION_ID -or $env:COPILOT_CLI) + +# Stop all Scout processes and relaunch the app. Returns $true if it relaunched. +function Restart-Scout { + $procs = @(Get-Process -Name 'scout' -ErrorAction SilentlyContinue) + $exe = ($procs | Where-Object { $_.Path } | Select-Object -First 1).Path + if (-not $exe) { $exe = Join-Path $env:LOCALAPPDATA 'Programs\Microsoft Scout\scout.exe' } + if (-not (Test-Path $exe)) { + Write-Host " Could not locate scout.exe to relaunch." -ForegroundColor Yellow + return $false + } + if ($procs.Count) { + Write-Host " Closing Scout ($($procs.Count) process(es)) ..." -ForegroundColor DarkYellow + $procs | ForEach-Object { try { Stop-Process -Id $_.Id -Force -ErrorAction Stop } catch {} } + # Wait for the processes to actually exit before relaunching (single-instance lock). + for ($i = 0; $i -lt 20 -and (Get-Process -Name 'scout' -ErrorAction SilentlyContinue); $i++) { + Start-Sleep -Milliseconds 250 + } + } + Write-Host " Relaunching Scout ..." -ForegroundColor DarkYellow + # Launch fully DETACHED so Scout does NOT inherit this console — otherwise its + # Electron startup logs spill into the terminal after setup returns. Win32_Process + # Create starts it in a brand-new process with no console attachment. + $launched = $false + try { + $r = Invoke-CimMethod -ClassName Win32_Process -MethodName Create -Arguments @{ CommandLine = "`"$exe`"" } -ErrorAction Stop + if ($r.ReturnValue -eq 0) { $launched = $true } + } catch {} + if (-not $launched) { + # Fallback: explorer.exe launches the app as its child, also detached from our console. + try { Start-Process explorer.exe -ArgumentList "`"$exe`""; $launched = $true } catch {} + } + return $launched +} # ---- 1. Infrastructure preflight (CLIs + MCP servers), data-driven ---- # Delegates to the engine (python -m orchestrator.cli infra), which reads @@ -35,52 +118,156 @@ Write-Host "Release Orchestrator bootstrap`n" -ForegroundColor Cyan Write-Host "1. Infrastructure preflight (from config/requirements.yaml)" if (-not (Test-Path $ReqFile)) { Write-Host " requirements.yaml not found at $ReqFile" -ForegroundColor Red; exit 1 } +# Resolve a working Python (3.9+). Prefer `python`/`python3`, fall back to `py -3`, +# then probe well-known winget/python.org install dirs (PATH may be stale in this session). +function Resolve-Python { + foreach ($cand in @('python','python3')) { + try { & $cand -c "import sys; sys.exit(0 if sys.version_info[:2] >= (3,9) else 1)" 2>$null; if ($LASTEXITCODE -eq 0) { return $cand } } catch {} + } + try { & py -3 -c "import sys; sys.exit(0 if sys.version_info[:2] >= (3,9) else 1)" 2>$null; if ($LASTEXITCODE -eq 0) { return 'py -3' } } catch {} + # Direct probe of standard per-user install locations (newest first) + $globs = @( + "$env:LOCALAPPDATA\Programs\Python\Python3*\python.exe", + "$env:ProgramFiles\Python3*\python.exe" + ) + foreach ($g in $globs) { + $hit = Get-ChildItem $g -ErrorAction SilentlyContinue | Sort-Object FullName -Descending | Select-Object -First 1 + if ($hit) { + try { & $hit.FullName -c "import sys; sys.exit(0 if sys.version_info[:2] >= (3,9) else 1)" 2>$null; if ($LASTEXITCODE -eq 0) { return "`"$($hit.FullName)`"" } } catch {} + } + } + return $null +} + +$PyExe = Resolve-Python + +# If Python is missing, install it for the user via winget (per-user, no admin), then re-resolve. +if (-not $PyExe -and -not $NoAutoInstallPython) { + $winget = (Get-Command winget -ErrorAction SilentlyContinue) + if ($winget) { + Write-Host " [ ] Python 3.9+ not found - installing Python 3.12 via winget (no admin needed) ..." -ForegroundColor DarkYellow + winget install --id Python.Python.3.12 -e --source winget --scope user ` + --accept-package-agreements --accept-source-agreements --disable-interactivity 2>&1 | + ForEach-Object { Write-Host " $_" -ForegroundColor DarkGray } + # Refresh PATH from the registry so the just-installed python is visible this session. + $env:Path = [Environment]::GetEnvironmentVariable('Path','Machine') + ';' + + [Environment]::GetEnvironmentVariable('Path','User') + $PyExe = Resolve-Python + if ($PyExe) { Write-Host " [x] Python installed" -ForegroundColor Green } + else { Write-Host " [ ] Python installed but not detected yet - close and reopen your terminal, then re-run setup." -ForegroundColor Yellow } + } else { + Write-Host " [ ] Python 3.9+ missing and winget is unavailable for auto-install." -ForegroundColor Yellow + } +} + $haveInfra = $false -try { python -c "import yaml" 2>$null; if ($LASTEXITCODE -eq 0) { $haveInfra = $true } } catch { $haveInfra = $false } +if ($PyExe) { + # Python is present. Ensure PyYAML — install it silently via pip, no user action needed. + & cmd /c "$PyExe -c ""import yaml"" 2>nul" + if ($LASTEXITCODE -eq 0) { + $haveInfra = $true + } else { + Write-Host " [ ] PyYAML missing - installing via pip ..." -ForegroundColor DarkYellow + & cmd /c "$PyExe -m pip install --quiet --disable-pip-version-check pyyaml" + & cmd /c "$PyExe -c ""import yaml"" 2>nul" + if ($LASTEXITCODE -eq 0) { $haveInfra = $true; Write-Host " [x] PyYAML installed" -ForegroundColor Green } + } +} $ok = $true $restartNeeded = $false if (-not $haveInfra) { - Write-Host " [ ] Python + PyYAML ... MISSING (needed to run the preflight)" -ForegroundColor Yellow - Write-Host " install: Python 3.9+ then python -m pip install pyyaml" -ForegroundColor DarkYellow + if (-not $PyExe) { + Write-Host " [ ] Python 3.9+ ... still MISSING (needed to run the preflight)" -ForegroundColor Yellow + if ($NoAutoInstallPython) { + Write-Host " Auto-install was disabled (-NoAutoInstallPython)." -ForegroundColor DarkYellow + } + Write-Host " Install Python 3.9+ ('winget install Python.Python.3.12' or https://aka.ms/python)," -ForegroundColor DarkYellow + Write-Host " reopen your terminal, then re-run setup." -ForegroundColor DarkYellow + } else { + Write-Host " [ ] PyYAML ... could not be installed automatically" -ForegroundColor Yellow + } $ok = $false } else { Push-Location $AgentRoot try { - $out = python -m orchestrator.cli infra 2>&1 + $out = & cmd /c "$PyExe -m orchestrator.cli infra 2>&1" $out | ForEach-Object { Write-Host " $_" } if ($LASTEXITCODE -ne 0) { $ok = $false } if ($out -match "RESTART Scout") { $restartNeeded = $true } } finally { Pop-Location } # engine config presence (cheap local sanity check) - Write-Host -NoNewline " [ ] engine config (phases.yaml) ... " - if (Test-Path (Join-Path $AgentRoot 'config\phases.yaml')) { Write-Host "OK" -ForegroundColor Green } - else { Write-Host "MISSING" -ForegroundColor Yellow; $ok = $false } + if (Test-Path (Join-Path $AgentRoot 'config\phases.yaml')) { + Write-Host " [OK] engine config (phases.yaml)" -ForegroundColor Green + } else { + Write-Host " [MISSING] engine config (phases.yaml)" -ForegroundColor Yellow; $ok = $false + } } if (-not $ok) { Write-Host "`nSome infrastructure is missing — resolve the items above, then re-run." -ForegroundColor Yellow } -if ($restartNeeded) { - Write-Host "`n>>> RESTART Scout now so the newly-registered MCP server(s) load. <<<" -ForegroundColor Cyan -} Write-Host "`n2. Skill install" if ($SkipSkillInstall) { Write-Host " Skipped (--SkipSkillInstall)." } else { - $src = Join-Path $AgentRoot "skill\SKILL.md" - $destDir = Join-Path $ScoutSkillsDir "release-agent" - New-Item -ItemType Directory -Force -Path $destDir | Out-Null - Copy-Item $src (Join-Path $destDir "SKILL.md") -Force - Write-Host " Installed /release-agent skill -> $destDir" -ForegroundColor Green + $src = Join-Path $AgentRoot "skill\SKILL.md" + $ScoutRoot = Split-Path -Parent $ScoutSkillsDir # ~/.scout + $destDir = Join-Path $ScoutSkillsDir "release-agent" + $destFile = Join-Path $destDir "SKILL.md" + + if (-not (Test-Path $src)) { + Write-Host " [ ] Source skill not found at $src" -ForegroundColor Red + $ok = $false + } + # Is Scout actually installed? Its per-user data folder (~/.scout) is created on + # first launch. If it's absent, Scout isn't installed/run yet — don't fabricate it. + elseif (-not (Test-Path $ScoutRoot)) { + Write-Host " [ ] Microsoft Scout not detected ($ScoutRoot is missing)." -ForegroundColor Yellow + Write-Host " Install Microsoft Scout and launch it once, then re-run this setup" -ForegroundColor DarkYellow + Write-Host " to install the /release-agent skill." -ForegroundColor DarkYellow + $ok = $false + } + else { + New-Item -ItemType Directory -Force -Path $destDir | Out-Null + Copy-Item $src $destFile -Force + # Verify the skill actually landed (exists + non-empty + size matches source). + $srcLen = (Get-Item $src).Length + if ((Test-Path $destFile) -and ((Get-Item $destFile).Length -eq $srcLen) -and ($srcLen -gt 0)) { + Write-Host " [x] Installed /release-agent skill -> $destFile ($srcLen bytes)" -ForegroundColor Green + $skillInstalled = $true + } else { + Write-Host " [ ] Skill copy could not be verified at $destFile" -ForegroundColor Red + $ok = $false + } + } + + # Report every release-agent skill Scout can currently see. + if (Test-Path $ScoutSkillsDir) { + $installed = Get-ChildItem $ScoutSkillsDir -Directory -ErrorAction SilentlyContinue | + Where-Object { Test-Path (Join-Path $_.FullName 'SKILL.md') } | + Select-Object -ExpandProperty Name + if ($installed) { Write-Host " Skills currently installed in Scout: $($installed -join ', ')" -ForegroundColor DarkGray } + } +} + +$needsReload = ($restartNeeded -or $skillInstalled) +if ($needsReload) { + Write-Host "`nScout must reload to pick up the newly-installed skill / MCP server(s)." + if ($NoRestartScout) { + Write-Host " Auto-restart disabled (-NoRestartScout). Restart Scout yourself so the changes load." -ForegroundColor DarkYellow + } elseif ($InsideScout) { + Write-Host " Skipping auto-restart: this setup is running inside a Scout session," -ForegroundColor Yellow + Write-Host " so restarting would kill it. Close and reopen Scout manually." -ForegroundColor Yellow + } else { + $done = Restart-Scout + if ($done) { Write-Host " Scout restarted — the skill/MCP servers will load on launch." -ForegroundColor Green } + else { Write-Host " Please restart Scout manually so the changes load." -ForegroundColor Yellow } + } } Write-Host "`n3. Next steps" -ForegroundColor Cyan Write-Host " * Open Scout and run: /release-agent" -Write-Host " * Or drive the engine directly from $AgentRoot :" -Write-Host " python -m orchestrator.cli init --release 2026-07" -Write-Host " python -m orchestrator.cli next --release 2026-07" -Write-Host " python -m orchestrator.cli status --release 2026-07" -Write-Host "`n Everything runs in DRY-RUN by default. Nothing touches production.`n" +Write-Host " * The agent drives the whole release from inside Scout - just follow its prompts.`n" diff --git a/release-agent/skill/SKILL.md b/release-agent/skill/SKILL.md index def9ebb4..3aa13e23 100644 --- a/release-agent/skill/SKILL.md +++ b/release-agent/skill/SKILL.md @@ -5,282 +5,54 @@ description: Drive an Android release end-to-end using the Release Orchestrator # /release-agent — Release Orchestrator conductor -You are the conversation layer over the **Release Orchestrator engine** (deterministic Python). -The engine decides what happens next; you discover releases, present status/gates nicely, and relay decisions. -**Never decide the release flow yourself, and never invent a release** — always call the engine. +> **Recommended model:** run on a high-reasoning model (e.g. **claude-opus-4.8**). Release work involves gate decisions, Component Governance / incident judgment, and multi-step state reconciliation. Scout skills can't self-select a model, so switch the session model before invoking if you're on a lighter one. (The unattended "Release push reminders" automation is already pinned to a strong model.) -## Where things live -- Engine + config: `C:\repos\android-complete\release-agent\` (run commands from here). -- Run-state: `C:\repos\android-complete\.release-runs\\release-state.json` (gitignored; one per month, e.g. `2026-07`). -- The `setup/bootstrap.ps1` script ONLY prepares the machine. Its first step is an **infrastructure preflight** (`python -m orchestrator.cli infra`): it checks the CLIs/host deps in `config/requirements.yaml` AND registers the **MCP servers** the skill needs into Scout's config — the **ICM** server (on-call lookups) and the **Kusto/ADX** server (telemetry queries), both provided by the Agency CLI — backing the config up and telling the engineer to restart Scout. It also checks **Scout itself is installed** (`~/.scout`); if not, it stops and says to install Scout first. It does **not** start a release — that happens here, in Scout. -- **Kusto is multi-cluster:** clusters live as data under `kusto_clusters` in `config/requirements.yaml`; infra wires them all into the one Kusto MCP via `--known-services`. To make a new cluster queryable, add an entry there and re-run `python -m orchestrator.cli infra` (then restart Scout). -- If an infra check ever fails (a needed MCP server isn't registered, or Scout wasn't restarted after registering), run `python -m orchestrator.cli infra` and tell the user to restart Scout; the manifest is `config/requirements.yaml`. - -## ALWAYS discover first (the none / one / many rule) -On ANY request about a release (status, continue, approve, advance), **do not assume a release id**. -First run discovery and branch on the result: - -``` -python -m orchestrator.cli list --json -``` - -The JSON has `resolution`: -- **`none`** → there is NO active release on this machine. Tell the user briefly, then **use the `m_ask_user` prompt tool** (not a free-text question) to offer starting one — see "Starting a release" below. Only run `init` after they choose. -- **`one`** → use that release (`release.release_id`). Proceed. -- **`ambiguous`** (several exist) → present the list from `all`, then **use `m_ask_user`** to let the user pick which release to act on (one option per release id, most-recent first). Do not act until they choose. -- **`explicit`** (you passed `--release` and it matched) → use it. - -Never run `status`/`next`/`approve` against a release id you haven't confirmed exists via `list`. - -## Starting a release (don't make the user type a date format) - -The release id is just `YYYY-MM`. **You compute it — never ask the user to type the format.** -Work out the current month from today's date (e.g. today 2026-07 → `2026-07`). - -When no release is active (or the user says "start a release"), call the **`m_ask_user`** prompt tool with clickable options, e.g.: -- **"Current month (``)"** ← recommended -- **"A different month"** - -If they pick the current month, run `init --release ` immediately. -If they pick "a different month", then (and only then) ask which month in a follow-up `m_ask_user` free-text prompt (hint: "e.g. next month, or 2026-08") and convert whatever they say into `YYYY-MM` yourself. Accept natural answers ("this month", "next month", "August") — do the date math for them; don't demand a rigid format. -Default to **dry-run**; only pass `--live` if the user explicitly asks for a live run. - -`init` records the **release owner** (the engineer running it) in the release metadata — resolved from the signed-in `az` user, and reminders are emailed to that address. You can pass a richer profile with `--owner-email`/`--owner-name` (e.g. from `workiq_get_my_profile`), or change it later with `set-owner`. Never hardcode a recipient. - -### Ensure push reminders exist (per release — provisioned at start, torn down at close) - -Right after `init`, make sure the **push-reminder automation** exists for THIS release so reminders reach the user even with Scout closed. It is a **per-release** automation: created when the release starts and removed when it closes (see teardown below). -1. List automations (`m_list_automations`). If one named **"Release push reminders"** already exists AND the registry has it scoped to the current release (`automation list --release --json`), **leave it** — don't duplicate. -2. If it's missing, create it with `m_create_automation`: - - **name:** `Release push reminders` - - **schedule:** `every hour` - - **teamsNotify:** `never` - - **prompt:** from `C:\repos\android-complete\release-agent` run `python -m orchestrator.cli tick --json` (this ADVANCES the active release — running the agent steps that can run, holding at gates/actions — then returns `{message, html, subject, owner_email, owner_name, release}`); if `message` is non-empty and `owner_email` is set, email it via `workiq_send_email` (`to: [owner_email]`, `subject:` the `subject` value, `body:` the `html` value with `isHtml: true` — fall back to the plain `message` with `isHtml: false` only if `html` is empty); if `message` is empty, do nothing. (Recipient comes from `owner_email` — never hardcode. Do **not** use `m_send_teams_message` (bot relay 404s) or the Teams self-chat (delivers silently).) -3. **Register it to this release** so it's tracked and torn down at close: - `python -m orchestrator.cli automation register --id --name "Release push reminders" --release --purpose "hourly advance + phase digest email to owner"` - -Do it silently as part of the start flow (the user already opted into push); don't re-ask each release. (The automation runs `tick` in discovery mode, so it targets the active release automatically.) **Why hourly, not once at 9am:** `tick` is idempotent (advancing is a no-op once holding at a gate, and the digest de-dupes to one email per calendar day), so running it every hour means a run missed while the machine was off — e.g. the 9am tick — is simply picked up by the next tick after the machine is on. A single daily trigger would be skipped for that day. - -### Any automation you provision MUST be registered (for teardown) - -Whenever you create a Scout automation for the orchestrator, immediately record it with `automation register` so nothing gets orphaned: -- **Per-release** (the normal case, e.g. push reminders, a phase watcher) → `--release `. **Removed when that release closes.** -- **Shared / persistent** (rare — only something genuinely meant to outlive every release) → `--shared` (no `--release`). Not torn down at close. Default to per-release unless there's a clear reason. - -At **release close** (status complete, the Release Close phase, or the user asks to "clean up automations"), tear down that release's automations: -1. `python -m orchestrator.cli automation list --release --json` — the automations provisioned for this release. -2. For each entry, delete the real Scout automation with `m_delete_automation` (id from the entry), then `python -m orchestrator.cli automation deregister --id `. -3. Shared automations (if any) are **not** in the release-scoped list, so they survive — leave them. -Confirm with the user before deleting, and report what was removed. - -## Code Complete Date (CCD) & phase scheduling - -Phases are **anchored to the Code Complete Date**, not started on demand. **The CCD is the 2nd Wednesday of the release month — that's the canonical default.** `init` computes it and prints when Phase 0 opens. - -`init` also *reads* the pipeline (ADO 3038 `overrideCodeCompleteDate`) but **does not silently adopt it.** If the pipeline holds a **different in-month date**, that's a **conflict to resolve, not an answer**: the status view shows a *"⚠ Confirm the date"* line and `status --json` sets `ccd_conflict`. When you see a conflict, **ask the user which is the real CCD** via `m_ask_user`, e.g.: -- **"Use the 2nd-Wednesday default (``)"** — then offer to sync the pipeline: run `set-ccd --release --default --reason ""` (preview) → show it → `--confirm` to clear the pipeline override so they match. -- **"Use the pipeline date (``)"** — run `set-ccd --release --date --reason "confirmed CCD is " --confirm` (stores it locally; the pipeline already has it). - -Either resolution clears the conflict. Never pick for the user. - -- **Phase 0 (Pre-flight) opens at CCD‑7** (7 days before CCD). You can `init` any time, but until CCD‑7 the release sits in **`scheduled`** — the engine runs nothing. The status view says *"🗓 Scheduled — Pre‑flight opens `` (in N days). Nothing to do yet."* Relay that plainly; don't try to force it forward. -- When the clock reaches CCD‑7, `next` opens Phase 0 and runs its steps up to the first gate — the normal flow resumes. -- **Testing the clock:** every read/advance command accepts `--as-of YYYY-MM-DD` to simulate a date (dry-run only). Real runs use today. - -**Changing the CCD (real production change).** If the user wants to move the date ("give us more time", "cut early"), use `set-ccd`. This **writes the pipeline override** — so it's gated: run it **without `--confirm` first to show the preview**, present that to the user, get an explicit yes (a `--reason` is always required, for audit), then re-run **with `--confirm`**. The override is month-scoped — the date must be in the release month. Use `--default` to revert to the 2nd-Wednesday default. - -**Skipping/cancelling the release.** Same gated pattern: `skip-release` sets the pipeline `skipRelease` switch (preview → confirm, reason required); `skip-release --clear` re-enables it. This suppresses the monthly trigger — treat it as a real, deliberate action and confirm before `--confirm`. - -**Ongoing conflict detection.** `status`/`resume` re-read the pipeline; if someone sets a differing override later, the same `ccd_conflict` surfaces — ask again. (Use `--no-pipeline-check` only if offline.) - -## Push reminders — the daily phase digest (reaching the user when Scout is closed) - -Everything above is **pull** (seen only when the user opens Scout). The **push** layer is a **daily phase status digest** emailed to the release owner, with a deliberate model: - -- **Setup is interactive — no push.** The readiness checklist and establishing the CCD happen live in Scout, so they are **never** emailed. An unsigned release, a blocked entry gate, and a halted release all stay silent. -- **The first push is a phase opening.** Phase 0 opens at **CCD‑7** — that's the first email. Nothing is sent before a phase opens (no pre‑open heads‑up). -- **Daily while a phase has outstanding work.** Once a phase is open, the owner gets a **once‑per‑day** digest (progress + what still needs them) until the phase's actions are done; then the next phase's digest takes over when it opens (each phase notifies on open). - -`tick` is the deterministic automation half: `python -m orchestrator.cli tick --json` first **advances** the active release (runs the agent steps that can run, holding at gates/actions — idempotent), then returns `{message, html, subject, owner_email, owner_name, release}` — `message` is the plain-text digest, `html` is the rich HTML version (full task table with status pills, attention-flagged), both empty when nothing is due today or it was already sent today. (`notify --json` is the read-only variant — same payload but does NOT advance; use it for a manual "what would I be told" check.) `--as-of ` is a debug clock; `--force` bypasses the once‑per‑day guard. - -- A **Scout automation** named **"Release push reminders"** runs `tick --json` (discovery mode) **hourly** and, when `message` is non‑empty, emails it via `workiq_send_email` to `owner_email` (subject from the JSON) — the release owner from release metadata, **never a hardcoded address**; when `message` is empty it stays silent. Running hourly (not once/day) means a tick missed while the machine was off is picked up by the next one, and idempotency + once‑per‑day de‑dup keep it to one advance-effect and one email per day. It is **per‑release**: auto‑provisioned (create‑if‑missing, registered to the release) at start and torn down at close. (Email is the channel because it reliably notifies; the `m_send_teams_message` bot relay 404s without a conversation reference, and the Teams self‑chat delivers silently.) - -If the user asks "how will I be reminded" / "set up notifications," explain this; if the automation doesn't exist, create it (see "Ensure push reminders exist"). Keep the email subject/body exactly as `tick` returns — don't embellish. - -## The readiness ENTRY GATE (right after starting) - -Immediately after `init`, the very first thing is the **readiness checklist** — the entry gate. The engine's `next` refuses to run any step (reports `readiness_gate`) until it's cleared. **Every item is equally required** — there is no priority or "hard vs soft" distinction. The only difference between items is **who resolves them**: - -- **`auto`** — **Scout resolves it** (verifies programmatically, pass/fail). Two execution sources, but the user sees both as `[auto]`: - - *Python-verified* (default): `build_access` (both ADO build definitions, via `az`) and `mcp_servers` (the ICM + Kusto/ADX MCP servers are registered in Scout). The engine's `verify`/`sign` runs these. - - *Scout-assisted* (`source: scout`): the **engine can't reach the MCP/Scout-settings, so YOU run the check** and record the result (see step 3a). Fail-closed **except `silent_perms`** (see below). Today: `oncall_now` (ICM current on-call), `adx_access` (Kusto `print 1` against the ADX cluster), `silent_perms` (Scout permissions allow fully-unattended runs). -- **`attest`** — **the engineer resolves it** (confirms): `play_console_access`, `oncall_window`, `saw_ame`, `yubikey`. - -**Two of the auto items exist so scheduled work runs UNATTENDED** (machine on, Scout not focused): `mcp_servers` (the MCP deps are registered — **hard**, since without them the on-call/telemetry checks can't run) and `silent_perms` (permissions won't stall the daily digest / Teams reminders / browser checks on a prompt — **soft/opt-out**: the user can choose to proceed without silent runs, recorded as `degraded`, with the downside noted). Enabling silent runs needs the user to flip ONE Scout master toggle first (*"Allow AI to request permission changes"*) — only they can (it's read-only from the model); after that I auto-request the rest with a single Allow click. - -**On-call is TWO items (hybrid), because Scout can only see the *current* rotation, not the future one:** -- `oncall_now` (**auto/ICM**) — are you on-call *right now*? Scout verifies this from ICM. -- `oncall_window` (**attest**) — are you free across the whole release window **CCD‑7 → CCD+14**? Scout can't read the future rotation, so you attest it (the checklist shows the concrete dates). - -If any item is unsatisfied the gate stays closed. If the engineer can't satisfy an attest item, they resolve it or hand the release to someone who can — the same for every item. Never describe any item as "not a hard block" or "optional." +You are the conversation layer over the **Release Orchestrator engine** (deterministic Python). The engine decides what happens next; you discover releases, present status/gates, and relay decisions. **Never decide the release flow yourself, and never invent a release — always call the engine.** -Flow after starting: -1. `python -m orchestrator.cli checklist --release --verify` — this runs the auto checks AND prints the **canonical checklist table (markdown)**. **Reproduce its stdout into your reply as live markdown (NOT wrapped in a ``` code fence)** so Scout renders it as a real table — it is already a finished markdown table with the type labels, per-item status, and clickable links. **Do NOT rebuild, re-format, re-order, re-label, or re-type any of it from memory, and do NOT fence it.** If you reconstruct it you WILL introduce errors (stale icons, mangled/merged URLs); if you fence it, it shows as raw text. Always reproduce the literal command output as rendered markdown. You may add a sentence of your own before or after, but the table block itself must match the output. -2. There are exactly **two types by resolver**: `[auto]` (Scout verifies) and `[attest]` (the user confirms). All items must be satisfied to clear the gate. (Do not add lock icons or a "hard requirement" legend.) -3a. **Run the scout-assisted `[auto]` checks yourself, then record each result** — don't ask the user for these; they're verified, not attested. **Do this quietly: run the checks and `record-check` them without narrating each one** (a wall of per-check prose is what buries the table). The only time 3a surfaces anything to the user is the `silent_perms` opt-out choice below. - - **`oncall_now` (ICM):** call the ICM MCP `get_on_call_schedule_by_team_id` with `teamIds: [78848]` ("Auth Client Android Shield"). Resolve the current user's alias (`get_my_icm_context` or the owner email's local part), then decide by their role in `shiftCurrentOnCalls[].currentOnCallContacts[]`: - - **Not in the roster at all** → `record-check --item oncall_now --status pass --detail "not on the current roster"`. - - **Present but NOT the primary** (i.e. they are a **backup/secondary** — any position other than the first-listed contact) → **pass**: `record-check --item oncall_now --status pass --detail "backup OCE, not primary (primary: )"`. A backup is free to run the release. - - **The PRIMARY / current OCE** (the **first-listed** contact in `currentOnCallContacts`) → `record-check --item oncall_now --status fail --detail "currently the primary on-call for Auth Client Android Shield"`. - - Only the **primary** blocks the gate. If you cannot confidently tell primary from backup (ambiguous ordering, or the user says otherwise), **ask the user** "Are you the primary/current OCE, or backup?" and record accordingly — **never block a backup.** - - **`adx_access` (Kusto):** run a trivial query — `kusto_query` with the item's `cluster_uri` + `database` (from `checklist --json`) and query `print 1`. Success = the engineer has data access to the ADX release dashboard's cluster. - - Query succeeds → `record-check --release --item adx_access --status pass --detail "print 1 succeeded"`. - - Query fails (auth/access error) → `record-check --release --item adx_access --status fail --detail ""`. - - **`silent_perms` (Scout settings — OPT-OUT/soft):** the daily push digest, the Teams reminders and the browser (CCOA/lockdown) checks all run from a background automation while Scout isn't focused — they must not stall on a permission prompt. Call **`m_get_settings`** and read `permissions.servers`. It's satisfied when ALL of the item's `required_servers` (from `checklist --json`: `shell`, `workiq`, `playwright`) have `autoApprove: true` (one server flag each keeps it simple: `workiq.autoApprove` covers both `workiq_send_email` and Teams; `playwright.autoApprove` covers the browser). **This item never hard-blocks — the user may choose to proceed without silent runs.** Flow: - - **Already all auto-approved** → `record-check --release --item silent_perms --status pass --detail "shell/workiq/playwright auto-approved"`. Done. - - **One or more NOT auto-approved** → **offer the choice** with `m_ask_user`: **"Enable silent runs (recommended)"** vs **"Proceed without — I'll get prompts"**. Explain the downside of proceeding: *the daily digest, Teams reminders, and CCOA/lockdown browser checks will pop a permission prompt when Scout isn't focused and can stall until you open Scout and approve them.* - - They pick **Enable** → the only manual step is the Scout master toggle: if `permissions.allowModelPermissionsChange` is `false`, tell them to turn on **Settings → Permissions → "Allow AI to request permission changes"** (I cannot flip it — it's read-only from the model, by design). Once it's `true`, call **`m_request_permission_escalation`** with `servers: { workiq: {autoApprove:true}, playwright: {autoApprove:true} }` (add `shell` if off too); they click **Allow** once, then re-read `m_get_settings` and `record-check … --status pass --detail "enabled silent runs"`. - - They pick **Proceed without** (or won't enable the master toggle) → `record-check --release --item silent_perms --status degraded --detail "proceeding without silent runs — unattended digest/Teams/browser checks will prompt & may stall until Scout is opened"`. **`degraded` satisfies the gate** (the checklist shows it as ⚠️ *Proceeding (not silent)*), so the release can start; the downside is on record. - - A `fail` on `oncall_now`/`adx_access`, or a real problem, keeps the gate closed — treat it like any unsatisfiable required item (resolve or hand off). Do NOT attest these — they're `auto` items you verified. (`silent_perms` is the one soft/opt-out auto item: it uses `degraded`, never `fail`, when the user chooses to proceed.) -3b. **Re-anchor on the TABLE, then attest.** After recording the auto results, **run `python -m orchestrator.cli checklist --release ` again and reproduce its updated markdown table** — the auto items now show ✅ and only the attests are Outstanding. This table is the single source of truth and **must be shown immediately before you ask for attestations**; the auto-check work in 3a pushes the first table far up, so re-render it here every time. Then use `m_ask_user` to collect the four attestations (`play_console_access`; the on-call **window** `oncall_window` — the CCD‑7 → CCD+14 dates are in the table; `saw_ame`; `yubikey`). Offer: **"All confirmed"**, **"I'm scheduled on-call during the window"**, **"I can't open Play Console"**, **"I don't have a SAW machine"**, **"I don't have a YubiKey"**. **Never replace the table with a plain prose list of the items** — the `m_ask_user` prompt accompanies the re-rendered table, it does not substitute for it. -4. If they confirm everything → `sign --release --all`, then `next` to begin Phase 0. -5. **If they can't satisfy any attest item** (on-call during the window, no SAW, no YubiKey, no portal access) → `decline --release --item ` (repeat `--item` for each). The gate is now blocked. Tell them plainly: the release can't start until that item is resolved; if they can't resolve it, hand the release to another engineer who can (notify their manager / the release team). Treat every item this way — don't single any out as harder or softer. -6. If an **auto** item shows FAIL (no build-definition access, or you're on-call), the gate stays closed — a real problem to resolve, not something to attest around. - -Never attest an `auto` item on the user's behalf — auto items are only satisfied by real verification (Python check or your recorded ICM result). -Never hand-edit or regenerate the checklist/status blocks — always show the CLI's literal output. - -## Parallel phases — process ALL the holds, not one at a time - -Some phases run **in parallel** (Phase 0 is `execution: parallel`): a single `next` runs **every independent automated step at once** (breaking, CG, cron, wiki — all complete in one call) and then surfaces **all the human/scout holds together** (e.g. *"4 item(s) need you: …"*). So don't treat it as one-step-at-a-time. After `next`, read `status --json` and look at **`pending_human`** (and `active_phase.steps` with their `status`/`needs_owner`) — that's the full set of what's outstanding. Work through **all** of them in this pass: -- **`source: scout`** steps (notice, flight_reminder, lockdown) → run each via MCP/browser + `record-step` (see the sections below). These are independent — do them all. - -> **State writes are safe to parallelize.** The CLI serializes every state read-modify-write per release with an exclusive lock, so firing several `record-step` / `record-check` / `done` calls at once (or an hourly `tick` overlapping your command) can't clobber each other — a second invocation simply waits for the first to save. You don't need to run them one-at-a-time to avoid races. -- **`attest`** steps (confirm_reminders, vitals) → ask the owner to confirm, then `done --step `. -- **`blocked`** steps (cg/cron on a real problem) → show the note; fix + rerun, or skip. -Dependencies still hold: `confirm_reminders` only appears **after** `flight_reminder` is sent (it won't be in `pending_human` until then). Call `next` again after clearing holds to let newly-ready steps surface and, once all are done, advance to the next phase. - -> **ALWAYS show the `status` table each iteration — never a bare prose list.** After every `next`, and after every `done`/`record-step` that clears a hold, run `python -m orchestrator.cli status --release ` (no `--json`, add `--as-of` if simulating) and **reproduce its full output as live markdown**: the phase map, the current-phase **steps table** (what's done ✅ / pending / needs you 📌), and the **Results & activity** section (each agent's result — the CG report, the created **wiki link**, block reasons). This is the source of truth the user asked for; the human-readable `status` also auto-logs what was shown. Use `status --json` only for your own branching — but still show the rendered table to the user. Then, alongside the table, tell the user exactly what each outstanding step needs (run the scout steps yourself; for the `attest` steps spell out what to confirm). Do **not** replace the table with your own summarized list of remaining steps. - -## Scout-assisted phase steps (CCOA lockdown check) - -Some Phase steps read AAD-gated sources the deterministic engine can't reach, so **you run them via the browser and record the result** — same idea as the readiness scout checks, but mid-phase. When advancing, if **`lockdown`** is among the pending holds (`source: scout`, in `pending_human`), handle it like this — silently, without bothering the user unless there's an overlap: - -1. **Scrape the CCOA source.** Navigate (Playwright) to `https://prod.change-manager.msidentity.com/ccoa-periods`. If an AAD account picker appears, click the user's own account (Windows-SSO — no password). Wait for the "CCOA Periods" page. -2. **Extract the periods.** From **"Upcoming CCOA periods"** and the **current-year** "Past NoFly Zones" table, read each row's **Name, Environment, Start Date (UTC), End Date (UTC)**. Build a JSON array: `[{"name","environment","start":"YYYY-MM-DD","end":"YYYY-MM-DD"}, ...]` (use the UTC dates). -3. **Let the engine decide (deterministic).** Run `python -m orchestrator.cli check-lockdown --release --periods-json ''`. It computes the release window (CCD‑7 … CCD+14), keeps only **Production**-environment periods, checks overlap, and records the step: **pass** (no overlap → step done, flow continues) or **attention** (overlap → step holds). -4. **Relay the outcome.** On **pass**, just continue (`next`) — no need to bother the user. On **attention**, surface it: name the overlapping lockdown(s) and window, and tell them to **shift CCD** past the lockdown (`set-ccd`) if they want to proceed; there are **no partners to notify** for this step. - -If you can't reach the browser/SSO in this context, leave the step held — it stays flagged as needing attention and you (or the user, next time Scout is open) can run it then. Don't mark it done without actually running the check. - -## Scout-assisted phase steps (early code-complete notice) - -The Phase-0 `notice` step sends the early code-complete email. Sending needs WorkIQ (a skill capability), so it's scout-assisted like `lockdown`. When `status --json` shows the current step is **`notice`** (holding, `awaiting_action`): - -1. **Prepare it (deterministic).** Run `python -m orchestrator.cli prepare-notice --release `. It fills the local template (`templates/early-code-complete-notice.md`) with the release's CCD/owner and returns JSON `{subject, body, html, recipients, dry_run, recipients_note}`. -2. **Send it.** Email via `workiq_send_email` using the returned `subject` and `recipients` exactly, with **`body:` the `html` value and `isHtml: true`** (the HTML has a clean hotfix-guide link + a proper rendered table — fall back to the plain `body` with `isHtml: false` only if `html` is empty). **Recipients are already resolved for you**: in a **dry-run** they're the **release owner only** (safe rehearsal — the subject is prefixed `[DRY-RUN → owner]`); on a **live** release they're the real distribution list (androididentity@microsoft.com, jialh@microsoft.com — see EXTERNAL-REFERENCES.md). Never override the recipients. -3. **Record it.** After a successful send: `python -m orchestrator.cli record-step --release --step notice --status pass --detail "sent to "`. If the send fails, `--status attention --detail ""` to keep it flagged. - -## Scout-assisted phase steps (flight & string reminders — Teams) - -The Phase-0 `flight_reminder` step posts a **combined 4-in-1 reminder** (update local flights · flight pre-mortem docs · merge user-facing strings by CCD-7 · Auth App feature-flag freeze / default-OFF review) as a **Teams message** to the Android Core Team. Sending Teams needs WorkIQ, so it's scout-assisted. When the current step is **`flight_reminder`** (holding): - -1. **Prepare it.** Run `python -m orchestrator.cli prepare-flight-reminder --release `. It returns JSON `{content, content_type:"html", dry_run, send_to, owner_email, chat_id, target_note}`. -2. **Resolve the chat + send.** - - **Dry-run** (`send_to: "owner"`): get the owner's 1:1 chat with `workiq_create_chat_by_email` (email = `owner_email`), then `workiq_send_chat_message` with that `chatId`, `content` = the returned HTML, `contentType: "html"`. (Safe rehearsal — the message is prefixed `[DRY-RUN → owner]`.) - - **Live** (`send_to: "group"`): `workiq_send_chat_message` with `chatId` = the returned `chat_id` (the Android Core Team thread), `content`, `contentType: "html"`. - Never override the target — `prepare-flight-reminder` already picked owner-vs-group from dry_run. -3. **Record it.** After a successful send: `python -m orchestrator.cli record-step --release --step flight_reminder --status pass --detail "posted to "`; on failure, `--status attention --detail ""`. - -**Sending the reminder is fire-and-forget** — it does NOT prove the feature owners actually did the work. So the very next step is **`confirm_reminders`**, a human **attestation** the engine holds on (`awaiting_action`). When the current step is `confirm_reminders`, ask the release owner (via `m_ask_user`) to confirm the reminded work is actually done — feature owners updated local flights, wrote flight pre-mortem docs, merged user-facing strings by CCD-7, and all features are default-OFF (or default-ON ones are approved in the wiki). Only when they confirm, run `python -m orchestrator.cli done --release --step confirm_reminders --note ""`. If they can't confirm, leave it holding (the release correctly blocks here until the pre-requisite work is verified) — don't mark it done. - -Phase 0's **`vitals`** step ("Confirm Play Console vitals & policy status reviewed") is another **attestation** hold. Play Console has no API for **Policy issues/warnings** (the Reporting API covers only technical vitals, and the Console UI is behind a Google login Scout can't automate), so this is a manual check: when the current step is `vitals`, ask the owner to open Play Console, review **Android vitals** (crash/ANR rate) and **Policy status** (issues/warnings), and confirm they're acceptable. On confirmation, `python -m orchestrator.cli done --release --step vitals --note ""`. If there's an unresolved policy issue or vitals regression, leave it holding. - -## Commands (run from the release-agent folder) - -| Intent | Command | +## Where things live +- Engine + config: `C:\repos\android-complete\release-agent\` — **run all `python -m orchestrator.cli …` commands from here.** +- Run-state: `C:\repos\android-complete\.release-runs\\release-state.json` (gitignored; one per month, e.g. `2026-08`). +- **Reference docs (this skill's detail):** `C:\repos\android-complete\release-agent\skill\reference\` — read the relevant one on demand (routing table below). The core stays lean; the details live there. +- `setup/bootstrap.ps1` only prepares the machine (infra preflight + installs this skill). If an infra check fails (an MCP server isn't registered, or Scout wasn't restarted), run `python -m orchestrator.cli infra` and tell the user to restart Scout; manifest is `config/requirements.yaml`. + +## GOLDEN RULES (always apply — the deduped essentials) +1. **Discover first, always.** On ANY release request, run `python -m orchestrator.cli list --json` and branch on `resolution`: `none` → offer to start (via `m_ask_user`); `one` → use `release.release_id`; `ambiguous` → list `all`, let the user pick; `explicit` → use it. Never run `status`/`next`/`approve` against an unconfirmed id. +2. **Render CLI output as LIVE MARKDOWN — never fenced.** `checklist`, `status`, `next`, etc. print finished markdown tables. Reproduce their stdout **verbatim as normal message content** so Scout renders the table — do NOT wrap in a ``` code fence, and do NOT rebuild/re-order/re-type from memory (you'll introduce stale icons / broken URLs). A sentence before/after is fine; the block must match. Use `--json` only for your own branching. +3. **The engine is the source of truth.** It owns sequencing and gate state. When unsure, `status --json`. Never hand-edit checklist/status output. +4. **Prompt, don't interrogate.** For any discrete choice (start? which release? approve/deny?) use the `m_ask_user` clickable prompt, not free-text. Reserve free-text for genuinely open values (an unusual month). +5. **Never assume a human decision.** An `m_ask_user` result that merely echoes the offered options is NOT confirmation. Never attest, approve, sign, or mark done until the user explicitly said so. Attesting/approving on an assumption is a release-integrity violation. +6. **Gates are human-decided.** Present and relay Approve/Deny; never authorize yourself. +7. **Dry-run by default.** Only pass `--live` when the user explicitly asks. `[STUB…]` output = mocked step; say so, don't imply real work. +8. **Never hardcode a recipient.** Reminders/notices go to the release `owner_email` from metadata (or engine-resolved DLs). +9. **Log silently.** Human-readable commands auto-log. YOU must journal user choices: `journal --release --source user --kind choice --text "" --choice "
    • " if len(names) < s["failed"] else "") + tag = _chip(_CAT_LABEL.get(s.get("category", "ui"), "UI automation"), "#eef4ff", "#0b5cad") suite_html += ( - f"
    " - f"
    {s['failed']}" - f" / {s['total']}  {T.esc(s['name'])}
    " - f"
      " - f"{items}{more}
    ") - return (f"
    " - f"
    MRWP {prov} — run {r.get('run_id')} " - f"({r.get('ran')}/{r.get('total')} stages)
    " - f"
    Tests: {tline}
    {red}{suite_html}" - f"
    ") + f"
    " + f"" + f"" + f"
    {T.esc(s['name'])}  {tag}" + f"{s['failed']}" + f"/{s['total']} " + f"· {sr}%
    " + f"
      {items}{more}
    ") + + rate_color = "#b42318" if ui_rate >= 5 else ("#b54708" if ui_rate > 0 else "#067647") + return ( + f"" + f"
    " + f"" + f"" + f"
    MRWP {prov}" + f" · run {r.get('run_id')} " + f"· {r.get('ran')}/{r.get('total')} stages{_chip('completed', '#ecfdf3', '#067647')}
    " + # headline = UI-automation failure rate (the RC-critical bucket) + f"
    " + f"{ui_rate}%" + f" UI-automation failure rate  ·  " + f"{ui_pass} passed / " + f"{ui_fail} failed of {ui_total} UI tests
    " + f"{_split_bar(100 - ui_rate)}" + # per-category breakdown + f"{cat_table}
    " + f"{red}{suite_html}" + f"" + f"
    ") + + # Overall headline — UI-automation failures ONLY (the RC-critical bucket), across both providers. + def _ui_sum(field): + return sum((((model.get("mrwp") or {}).get(p) or {}).get("tests", {}) + .get("categories", {}).get("ui", {}).get(field, 0) or 0) + for p in ("ECS", "Local")) + tot_f, tot_t = _ui_sum("failed"), _ui_sum("total") + overall_rate = _fail_rate(tot_f, tot_t) probs = model.get("problems") or [] issues = (("
    Blocking issues (a stage that never " + "border-radius:8px;color:#b42318;'>Blocking issues (a stage that never " "ran = pipeline aborted):
      " + "".join(f"
    • {T.esc(p)}
      • " for p in probs) + "
    ") if probs else "") return f"""\ -
    -

    Hi {T.esc(ctx.get('owner','there'))},

    -

    The Release Candidate for {T.esc(rid)} has been built and RC testing has completed. - Review the results below and approve “RC verified — proceed to bug bash” when ready.

    -

    Pipeline health

    -
      -
    • Code Complete Checker: fired the release (run {ch.get('run_id')}).
      • -
    • Release Orchestrator: healthy — pre-gate stages green, {park}.
      - Versions: {T.esc(vstr)} · - run {o.get('run_id')}
      • -
    -

    RC testing — both provider runs ran to completion

    - {mrwp_block('ECS')} - {mrwp_block('Local')} +
    + + +
    +
    RC Verification Report
    +
    Release {T.esc(rid)} · Phase 2 — Build & RC testing
    +
    + + + + + + + +
    +
    UI-automation failure rate
    +
    {overall_rate}%
    +
    {tot_f} failed / {tot_t} UI tests
    +     		+
    Checker
    +
    ✓ Fired
    +
    run {ch.get('run_id')}
    +     		+
    Orchestrator
    +
    ✓ Healthy
    +
    {park}
    +
    + +

    Versions: {T.esc(vstr)} · + orchestrator run {o.get('run_id')}

    + +

    UI-automation results

    + {mrwp_card('ECS')} + {mrwp_card('Local')} {issues} -

    Next: if the failures are acceptable to carry into bug bash, approve the gate - (the release advances to Phase 3 — Test / Bug Bash). Otherwise investigate the red suites first.

    -

    — Release Orchestrator (Scout)

    + + + +
    + Next: review the failing suites above. If acceptable to carry into bug bash, + approve “RC verified — proceed to bug bash” + (advances to Phase 3). Otherwise investigate the red suites first. +
    +

    — Release Orchestrator (Scout)

    """ diff --git a/release-agent/tests/test_engine.py b/release-agent/tests/test_engine.py index 0cfa4996..8f108857 100644 --- a/release-agent/tests/test_engine.py +++ b/release-agent/tests/test_engine.py @@ -2005,10 +2005,18 @@ def test_build_verify_rc_report_emails_owner(): "versions": {"Common": "24.6.0", "Msal": "8.4.2", "Broker": "16.5.0"}}, "mrwp": {"ECS": {"run_id": 1678863, "complete": True, "ran": 23, "total": 23, "failed_stages": ["UI Automation"], - "tests": {"total": 5871, "passed": 5767, "failed": 104, - "runs": [{"name": "PROD MSAL - RC Broker", "total": 44, "failed": 18}]}}, + "tests": {"total": 5871, "passed": 5767, "failed": 104, "runs": [], + "categories": { + "unit": {"total": 5248, "passed": 5248, "failed": 0}, + "instrumented": {"total": 442, "passed": 440, "failed": 2}, + "ui": {"total": 165, "passed": 63, "failed": 102}}}, + "failed_suites": [{"name": "PROD MSAL - RC Broker (API 32)", + "failed": 18, "total": 44, "category": "ui", + "tests": ["test_1_Foo", "test_2_Bar"]}]}, "Local": {"run_id": 1678864, "complete": True, "ran": 23, "total": 23, - "failed_stages": [], "tests": {"total": 5856, "passed": 5756, "failed": 100, "runs": []}}}, + "failed_stages": [], "tests": {"total": 5856, "passed": 5756, "failed": 100, + "runs": [], "categories": {}}, + "failed_suites": []}}, "problems": []} try: st = ReleaseState(release_id="2026-08", ccd="2026-08-26", @@ -2016,7 +2024,12 @@ def test_build_verify_rc_report_emails_owner(): out = as_dict(_steps.get_step("build_verify", "rc_report").build(st)) assert out["kind"] == "needs_skill" and out["tool"] == "workiq_send_email" assert out["payload"]["to"] == ["dev@microsoft.com"] and out["payload"]["isHtml"] - assert "104 failed" in out["payload"]["body"] and "1678863" in out["payload"]["body"] + body = out["payload"]["body"] + assert "1678863" in body # run id present + assert "UI-automation failure rate" in body # per-category headline metric + assert "61.8%" in body # 102/165 UI failures — the real UI rate + assert "Unit" in body and "Instrumented" in body and "UI automation" in body + assert "test_1_Foo" in body # failing test names still listed assert out["record_as"] == "rc_report" and out["outbound"] is True # no owner → blocked st2 = ReleaseState(release_id="2026-08", ccd="2026-08-26") @@ -2026,6 +2039,18 @@ def test_build_verify_rc_report_emails_owner(): P.release_report = orig +def test_classify_test_run_categories(): + """The test-run classifier buckets into exactly three: unit / instrumented / ui; + anything that isn't unit/instrumented is UI ('the rest are UI', incl. Lab Api Tests).""" + from tools import pipelines as P + assert P.classify_test_run("common4j_UnitTests") == "unit" + assert P.classify_test_run("common_InstrumentedTests") == "instrumented" + assert P.classify_test_run("PROD MSAL - RC Broker (API 32)") == "ui" + assert P.classify_test_run("RC MSAL - PROD Broker (API 28) # 123_build.1") == "ui" + assert P.classify_test_run("Lab Api Tests") == "ui" # NOT 'other' + assert P.classify_test_run("") == "ui" + + def test_get_failed_tests_aggregates_repeated_suites(): """get_failed_tests merges the SAME suite that appears as several runs (the cause of the confusing duplicates) into one entry, summing failures and collecting test names.""" diff --git a/release-agent/tools/pipelines.py b/release-agent/tools/pipelines.py index 2c2381e9..645f9510 100644 --- a/release-agent/tools/pipelines.py +++ b/release-agent/tools/pipelines.py @@ -295,10 +295,33 @@ def stage_completion(stages): "failed": failed, "yellow": yellow, "complete": not never and total > 0} +import re as _re_mod +_UI_API_RE = _re_mod.compile(r"\(API\s*\d+\)", _re_mod.IGNORECASE) +TEST_CATEGORIES = ("unit", "instrumented", "ui") +_CATEGORY_LABEL = {"unit": "Unit", "instrumented": "Instrumented", "ui": "UI automation"} + + +def classify_test_run(name): + """Bucket a test-run/suite name into one of THREE categories: + * '*_UnitTests' → unit + * '*_InstrumentedTests' → instrumented + * everything else → ui (the device UI-automation suites, which carry an + '(API NN)' tag, plus any other run such as + 'Lab Api Tests' — 'the rest are UI'). + """ + low = (name or "").lower() + if "unittest" in low: + return "unit" + if "instrumentedtest" in low: + return "instrumented" + return "ui" + + def get_test_summary(org, project, build_id, timeout=60): """Return (ok, summary, detail) for a build's Test-tab results. summary = - {total, passed, failed, runs:[{name,total,passed,failed}]} aggregated across all - test runs associated with the build (unit / instrumented / UI-automation). + {total, passed, failed, runs:[{name,total,passed,failed,category}], + categories:{unit|instrumented|ui|other: {total,passed,failed}}} aggregated across + all test runs, classified into unit / instrumented / UI-automation / other. Uses the Test Runs REST API directly (az devops invoke mis-routes this one).""" base = org.rstrip("/") @@ -309,16 +332,22 @@ def get_test_summary(org, project, build_id, timeout=60): return (False, None, detail) runs = (data or {}).get("value", []) or [] out_runs, tot, passed = [], 0, 0 + cats = {c: {"total": 0, "passed": 0, "failed": 0} for c in TEST_CATEGORIES} for r in runs: t = r.get("totalTests") or 0 p = r.get("passedTests") or 0 na = r.get("notApplicableTests") or 0 f = max(t - p - na, 0) + cat = classify_test_run(r.get("name")) tot += t passed += p - out_runs.append({"name": r.get("name"), "total": t, "passed": p, "failed": f}) + cats[cat]["total"] += t + cats[cat]["passed"] += p + cats[cat]["failed"] += f + out_runs.append({"name": r.get("name"), "total": t, "passed": p, + "failed": f, "category": cat}) return (True, {"total": tot, "passed": passed, "failed": max(tot - passed, 0), - "runs": out_runs}, "") + "runs": out_runs, "categories": cats}, "") def _suite_base_name(name): @@ -349,7 +378,8 @@ def fcount(r): suites, calls = {}, 0 for r in failing: name = _suite_base_name(r.get("name")) - s = suites.setdefault(name, {"name": name, "failed": 0, "total": 0, "tests": []}) + s = suites.setdefault(name, {"name": name, "failed": 0, "total": 0, + "category": classify_test_run(name), "tests": []}) s["failed"] += fcount(r) s["total"] += r.get("totalTests") or 0 if calls < max_result_calls: From 2a8c5fc3d4b9759914b93fcd4f1c6ec4899f15a3 Mon Sep 17 00:00:00 2001 From: p3dr0rv Date: Thu, 20 Aug 2026 17:53:13 +0100 Subject: [PATCH 63/82] release-agent: document that _ccd_cron emits host-local wall-clock (scheduler is local, not UTC) Empirically verified 2026-08-20: a cron '37 9' fired at 09:37 PDT / 16:37 UTC. Docstring-only note so a UTC conversion is never added, which would shift every CCD-day comm by the host's UTC offset. No logic change. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- release-agent/orchestrator/automations.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/release-agent/orchestrator/automations.py b/release-agent/orchestrator/automations.py index af04cf7a..fc15de9a 100644 --- a/release-agent/orchestrator/automations.py +++ b/release-agent/orchestrator/automations.py @@ -120,7 +120,13 @@ def _ccd_cron(ccd_date, hhmm: str): a CCD more than a week out (these are provisioned at release start) is the wrong date — it fired the CCD-day comms a week early. Cron `M H D Mo *` targets the CCD's day-of-month + month exactly, so a one-shot fires ON the CCD. Returns the NL Scout - accepts (e.g. 'cron: 0 9 26 8 *') or None if inputs are missing/invalid.""" + accepts (e.g. 'cron: 0 9 26 8 *') or None if inputs are missing/invalid. + + TIMEZONE: emit the LOCAL wall-clock time directly — do NOT convert to UTC. + Scout's scheduler interprets cron in host-local time (empirically verified + 2026-08-20: a cron '37 9' fired at 09:37 PDT / 16:37 UTC, not 09:37 UTC). So a + `hhmm` of '09:00' correctly fires at 09:00 local on the CCD. Adding a UTC + conversion here would shift every CCD-day comm by the host's UTC offset.""" if not ccd_date or not hhmm: return None try: From 545ad67cb45c8a0a2b258912dadfa45819c5c1cd Mon Sep 17 00:00:00 2001 From: p3dr0rv Date: Thu, 20 Aug 2026 12:11:50 -0700 Subject: [PATCH 64/82] Implement RC report follow-up command and UI gate logic; enhance documentation and tests --- .../orchestrator/commands/rc_report.py | 48 ++++++++ release-agent/orchestrator/engine.py | 1 + release-agent/skill/SKILL.md | 1 + release-agent/skill/reference/commands.md | 3 +- .../skill/reference/phases/build_verify.md | 47 ++++++++ .../reference/starting-and-scheduling.md | 2 +- release-agent/steps/build_verify/_common.py | 77 +++++++++++- release-agent/steps/build_verify/rc_report.py | 39 ++++-- release-agent/tests/test_engine.py | 113 ++++++++++++++++++ 9 files changed, 317 insertions(+), 14 deletions(-) create mode 100644 release-agent/skill/reference/phases/build_verify.md diff --git a/release-agent/orchestrator/commands/rc_report.py b/release-agent/orchestrator/commands/rc_report.py index 5c370c2a..08473d64 100644 --- a/release-agent/orchestrator/commands/rc_report.py +++ b/release-agent/orchestrator/commands/rc_report.py @@ -52,6 +52,46 @@ def _u(build_id): return K.build_url(build_id) if build_id else "" +def cmd_record_rc_report(args): + """Record the rc_report step's outcome AFTER the skill has emailed the RC report. + + Re-reads the live model, applies the UI-automation quality gate (K.rc_ui_gate), + records `pass` (>=90% UI pass → step done, flow advances to go_test) or `attention` + (<90% → step BLOCKS for owner investigation), and stashes the evaluated pipeline-run + links on the step so its Details point at every artifact behind the verdict. + + This is the follow-up the rc_report NeedsSkill names (`payload.followup_command`), so + the skill runs it instead of a blind `record-step --status pass`.""" + _, orch = C.load_orch(args.runs_root, args.release, args.config, C.parse_as_of(args)) + try: + model = K.rc_report_model(orch.state) + except Exception as e: # pragma: no cover - defensive + print(_json.dumps({"error": f"could not build the RC model ({e})."})) + return 1 + + gate = K.rc_ui_gate(model) + links = K.rc_run_links(model) + status = "pass" if gate["verdict"] == "pass" else "attention" + orch.record_scout_step("build_verify", "rc_report", status, gate["detail"]) + + # record_scout_step doesn't carry links — attach the evaluated-run refs (and stamp + # the recorder as scout) on the resulting step, preserving its status/note. + step = orch.state.get_step("build_verify", "rc_report") + step.links = links + step.by = "scout" + orch.state.set_step("build_verify", "rc_report", step) + C.save_state(orch.state, args.runs_root, args.release) + + C.emit(args.runs_root, args.release, + f"[{'ok' if status == 'pass' else 'attention'}] rc_report: " + f"{gate['detail'].splitlines()[0]}", kind="step") + print(_json.dumps({"verdict": gate["verdict"], "status": status, + "pass_pct": gate["pass_pct"], "ui_total": gate["ui_total"], + "ui_failed": gate["ui_failed"], "threshold": gate["threshold"], + "detail": gate["detail"], "links": links})) + return 0 if status == "pass" else 2 + + def _format(m) -> str: L = [f"## RC Pipeline Status — Release {m['release']}", ""] @@ -138,3 +178,11 @@ def register(sub): rp.add_argument("--release", required=True) rp.add_argument("--json", action="store_true", help="Emit the raw report model") rp.set_defaults(func=cmd_rc_report) + + rr = sub.add_parser( + "record-rc-report", + help="Record the rc_report step after emailing: apply the 90%% UI gate " + "(pass|attention/block) + stash the evaluated run links") + rr.add_argument("--release", required=True) + rr.add_argument("--as-of", default=None, help="Simulated clock (YYYY-MM-DD); default today") + rr.set_defaults(func=cmd_record_rc_report) diff --git a/release-agent/orchestrator/engine.py b/release-agent/orchestrator/engine.py index 2b6ea2bd..567e0748 100644 --- a/release-agent/orchestrator/engine.py +++ b/release-agent/orchestrator/engine.py @@ -648,6 +648,7 @@ def _active_phase_report(self) -> Optional[dict]: "needs_owner": needs, "time_ready": self._step_time_ready(phase, s), # False = waits for its fire_at_local "note": stp.note, # agent result / block reason / detail + "links": list(getattr(stp, "links", None) or []), # durable refs to items evaluated "now": bool(sid == cur and not s_done and (is_gate or is_rem or is_attest or s_blocked)), }) opens = self._phase_anchor_date(phase) diff --git a/release-agent/skill/SKILL.md b/release-agent/skill/SKILL.md index a32bcef8..56bb23f9 100644 --- a/release-agent/skill/SKILL.md +++ b/release-agent/skill/SKILL.md @@ -55,6 +55,7 @@ Discover → (if no gate cleared, run the entry gate) → `next` to advance → | Running the readiness entry gate (right after `init`) | `reference/readiness-gate.md` | | Starting a release / handling CCD / setting up push reminders & automations | `reference/starting-and-scheduling.md` | | Advancing **Phase 0 (Pre-flight)** — notice, flight reminders, lockdown, confirm, vitals | `reference/phases/preflight.md` | +| Advancing **Phase 2 (Build & RC Verification)** — verification chain, RC report email + 90% UI gate, go_test | `reference/phases/build_verify.md` | | Rendering `status`/`checklist` output | `reference/presenting-status.md` | | Looking up a command / manual override / event-logging detail | `reference/commands.md` | | Building a NEW phase's guidance | `reference/phases/_TEMPLATE.md` | diff --git a/release-agent/skill/reference/commands.md b/release-agent/skill/reference/commands.md index 11ce610c..96110fb1 100644 --- a/release-agent/skill/reference/commands.md +++ b/release-agent/skill/reference/commands.md @@ -15,6 +15,7 @@ _Loaded on demand. Run all from `C:\repos\android-complete\release-agent`._ | Resolve a migrated step → outcome JSON (done\|blocked\|needs_human\|needs_skill) | `python -m orchestrator.cli step-action --release --step [--phase

    ] [--param k=v …]` | | Answer a STEP question (knowledge) | `python -m orchestrator.cli step-info --step [--phase

    ]` | | **Phase 2 — RC pipeline + test report** (read-only) | `python -m orchestrator.cli rc-report --release [--json]` → the checker→orchestrator→ECS/Local-MRWP chain + per-run test breakdown | +| **Phase 2 — record RC verdict** (after emailing the report) | `python -m orchestrator.cli record-rc-report --release ` → applies the **90% UI-automation gate** across both MRWP runs, records `pass` (step done → go_test) or `attention` (step **blocks** for investigation), and stashes the checker/orchestrator/ECS/Local run links on the step. This is the follow-up the `rc_report` `needs_skill` names — run it **instead of** `record-step` | | **Simulate a mid-release point** (testing) | `python -m orchestrator.cli sim list` · `python -m orchestrator.cli sim run --scenario [--freeze] [--json]` → **seeds the real release** to a scenario's target (`config/scenarios/.yaml`): fast-forwards the real engine, signs the entry gate + completes earlier phases from mocks, then stops `open`/`gate`/`done` at the target. `data: live` runs the target phase against real `az`; `data: mock` is offline. Any existing state at that id is backed up first, so afterwards you use the **normal** commands (`status`, `rc-report`, `next`, `approve`). `--runs-root ` targets a throwaway sandbox instead; `--freeze` snapshots state to `tests/fixtures/.json` | | Answer an ENTRY-GATE item question (knowledge) | `python -m orchestrator.cli gate-info --item ` (build_access, mcp_servers, ccd_confirmed, silent_perms, teams_notify, adx_access, oncall_now, play_console_access, oncall_window, saw_ame, yubikey) | | Prepare early code-complete notice (JSON) — _legacy; prefer `step-action --step notice`_ | `python -m orchestrator.cli prepare-notice --release [--variant initial\|update]` | @@ -57,7 +58,7 @@ Map natural language to these ("skip the CG report, doesn't apply" → `skip … - **`done`** — already complete; nothing to run. - **`blocked`** — surface `reason` to the owner; don't proceed. - **`needs_human`** — show `prompt` (attestation or reminder to-do). -- **`needs_skill`** — run `tool` with `payload` (an MCP/browser call the engine can't make, already fully resolved), then confirm with `record-step --step --status pass\|attention`. Runs are real — the payload targets the real DL/chat unless the engineer's `mocks.local.yaml` has a `send_to` redirect (then `payload.to`/`chatId` points at them and the subject carries `[TEST → me]`). +- **`needs_skill`** — run `tool` with `payload` (an MCP/browser call the engine can't make, already fully resolved), then confirm with `record-step --step --status pass\|attention`. **UNLESS `payload.followup_command` is set** — then run that engine command **instead** of `record-step` (it records the verdict itself): `rc_report` sets `followup_command: record-rc-report` (send the email, then run `record-rc-report` — it applies the 90% UI gate and records pass/block + links). Runs are real — the payload targets the real DL/chat unless the engineer's `mocks.local.yaml` has a `send_to` redirect (then `payload.to`/`chatId` points at them and the subject carries `[TEST → me]`). If a step isn't migrated yet, `step-action` returns `{"error": …}` with exit 1. **Use `step-action` for scout steps** (`needs_skill` → run the tool, then `record-step`) **and attest steps** (`needs_human` → show the `prompt` via `m_ask_user`, then clear with `done --step `). Migrated: scout — `preflight.notice`, `preflight.flight_reminder`, `preflight.lockdown` (gather-then-decide: its `needs_skill` carries a `_gather` browser-scrape directive + a `check-lockdown` follow-up); attest — `preflight.confirm_reminders`, `preflight.vitals`. **Agent steps** (`preflight.breaking`, `cg`, `cron`, `wiki`) are migrated too but the **engine runs them in-process during `next`** — `step-action` refuses them (exit 1); relay their results from the `status` table. diff --git a/release-agent/skill/reference/phases/build_verify.md b/release-agent/skill/reference/phases/build_verify.md new file mode 100644 index 00000000..e6163f38 --- /dev/null +++ b/release-agent/skill/reference/phases/build_verify.md @@ -0,0 +1,47 @@ +# Reference — Phase `build_verify` (Phase 2 · Build & Lib Verification) + +Opens **CCD+1** — the engineer wakes to a resume. The engine runs the four verification +**agent** steps in-process during `next`; you relay their results and drive the one +**scout** step (`rc_report`) + the human **gate** (`go_test`). + +## Execution model +Sequential. A single `next` runs the agent chain (checker → orchestrator → ECS/Local +MRWP); each **blocks** on a real problem (a stage that never ran, an unhealthy +orchestrator, an auth failure). A blocked step → show the note, then **fix + `next`** to +re-check, or **`skip … --reason`** to override. When the chain is green the scout +`rc_report` step becomes ready, then the `go_test` gate. + +## Automated steps (no skill action — relay from the `status` table) +`checker_fired`, `orchestrator_health`, `mrwp_ecs`, `mrwp_local` — read-only `az` agent +steps run inside `next`. Each records the ADO run it evaluated as a Details 🔗 link. +`step-action` refuses them (exit 1); never dispatch them yourself. + +## `rc_report` — email the RC report + apply the 90% UI gate (`scout`) +- **Trigger:** `status --json` shows current step `rc_report` (state `scout`), after the + four agent steps are done. +- **Resolve:** `step-action --release --phase build_verify --step rc_report` → + `needs_skill` (`workiq_send_email`) with a fully-composed HTML dashboard, plus + `payload.followup_command: record-rc-report`. +- **Act:** send the email verbatim (`payload.to/subject/body`, `isHtml:true`) — honoring + a `send_to` redirect if the engineer set one. **Always send** — the owner gets the + dashboard (failing suites + run links) whether the gate passes or not. +- **Record (the two-hop):** because `followup_command` is set, run + `record-rc-report --release ` **instead of** `record-step`. It re-reads the model, + applies the **90% UI-automation gate** (combined pass rate across ECS + Local): + - **≥ 90% → `pass`** — the step is done; advance to `go_test`. + - **< 90% → `attention`** — the step **BLOCKS** (`awaiting_action`). This is a large UI + failure: the owner must **investigate the root cause** (usually a **fix + an MRWP + re-run**). Exits: fix + re-run, then `next` re-runs `rc_report`; or `skip … --reason` + to override. It records the failing-suite summary + stashes the checker/orchestrator/ + ECS/Local run links on the step. + - (No UI tests found → passes with a ⚠ note.) +- The command prints `{verdict, pass_pct, ui_total, detail, links}` for your branching; + relay the `status` table (the `rc_report` Details shows the verdict + 🔗 links). + +## `go_test` — RC verified, proceed to bug bash (`gate`, human) +Present the settled `status`; `m_ask_user` Approve/Deny; `approve` / `deny --comment`. +Never authorize yourself. If `rc_report` blocked, `go_test` isn't reached until it clears. + +## External references +Engineering pipelines: Checker def 3038, Orchestrator def 2828, MRWP def 2519 +(org `identitydivision.visualstudio.com`, project `Engineering`). diff --git a/release-agent/skill/reference/starting-and-scheduling.md b/release-agent/skill/reference/starting-and-scheduling.md index 453ae6c1..1389eb56 100644 --- a/release-agent/skill/reference/starting-and-scheduling.md +++ b/release-agent/skill/reference/starting-and-scheduling.md @@ -24,7 +24,7 @@ Right after `init`, make sure the **push-reminder automation** exists for THIS r - **teamsNotify:** `never` - **prompt:** From `C:\repos\android-complete\release-agent`, advance the active release **AUTONOMOUSLY** (no user is watching) and send the daily digest: 1. Run `python -m orchestrator.cli status --json`. If there is **no** release, or it is **unsigned / halted / complete**, STOP silently. - 2. **Run Scout's own steps until none remain.** Loop: run `next --json`; read `scout_pending`; if empty, go to step 3; else for EACH id: run `step-action --release --phase --step ` (Phase 0 = `preflight`), perform the returned `needs_skill` `tool`+`payload` via the matching MCP tool (`workiq_send_email` / `workiq_send_chat_message` / `azure_devops-pipelines_run_pipeline`, honoring `test_redirect`), then finalize with `record-step … --status pass` — **UNLESS** the action names a follow-up engine command (e.g. `check-lockdown`, `record-localization-run`/`check-localization`), which you run instead as the action describes. **Headless safety copy:** whenever a `needs_skill` action has **`outbound: true`**, after performing it also `m_send_teams_message` a **ONE-LINE** courtesy copy — `🤖 [release ] Autonomous:

    .` — so the owner sees what went out (never paste the full email/message body). A scout step that records `attention` is left blocked (it surfaces in the digest). Do it silently. + 2. **Run Scout's own steps until none remain.** Loop: run `next --json`; read `scout_pending`; if empty, go to step 3; else for EACH id: run `step-action --release --phase --step ` (Phase 0 = `preflight`), perform the returned `needs_skill` `tool`+`payload` via the matching MCP tool (`workiq_send_email` / `workiq_send_chat_message` / `azure_devops-pipelines_run_pipeline`, honoring `test_redirect`), then finalize with `record-step … --status pass` — **UNLESS** the action names a follow-up engine command (via `payload.followup_command`, or the known two-hop steps: `check-lockdown`, `record-localization-run`/`check-localization`, and **`record-rc-report`** for `rc_report` — send the RC email, then run `record-rc-report`, which applies the 90% UI gate and records pass/block), which you run instead as the action describes. **Headless safety copy:** whenever a `needs_skill` action has **`outbound: true`**, after performing it also `m_send_teams_message` a **ONE-LINE** courtesy copy — `🤖 [release ] Autonomous: .` — so the owner sees what went out (never paste the full email/message body). A scout step that records `attention` is left blocked (it surfaces in the digest). Do it silently. 3. Run `python -m orchestrator.cli tick --json` → `{message, html, subject, owner_email, owner_name, release, channels, teams}`. If `message` is empty, STOP (nothing due / already sent today). Otherwise deliver on every enabled channel: - **Email** (when `channels.email`): `workiq_send_email` (`to:[owner_email]`, `subject:` the value, `body:` the `html` with `isHtml:true` — fall back to plain `message`/`isHtml:false` only if `html` empty). Recipient from `owner_email` — never hardcode. - **Teams** (when `channels.teams` and `teams` is non-null): dispatch on `teams.via` — diff --git a/release-agent/steps/build_verify/_common.py b/release-agent/steps/build_verify/_common.py index 0c1beb04..7d199c98 100644 --- a/release-agent/steps/build_verify/_common.py +++ b/release-agent/steps/build_verify/_common.py @@ -59,6 +59,12 @@ def stash_runs(state, **ids): state.pipeline_runs = pr +# The Phase-2 quality bar: at least this % of UI-automation tests must pass for RC to +# clear to bug bash. Below it, the rc_report step blocks for owner investigation +# (a large UI failure usually means a real regression → fix + re-run MRWP). +RC_UI_PASS_THRESHOLD = 90.0 + + # ---------------------------------------------------------------- RC report email def rc_report_model(state, timeout=120): """The full Phase-2 RC report model (checker → orchestrator → ECS/Local MRWP + @@ -68,10 +74,77 @@ def rc_report_model(state, timeout=120): checker_def=CHECKER_DEF, orch_def=ORCHESTRATOR_DEF, timeout=timeout) +def rc_run_links(model) -> list: + """Durable links to EVERY pipeline run the RC verification evaluated — the Code + Complete Checker, the Release Orchestrator, and both MRWP (ECS + Local) runs — so the + recorded step points at each artifact behind the verdict (surfaced in the step's + Details). Only runs with a resolved id are included.""" + out = [] + ch = (model.get("checker") or {}).get("run_id") + if ch: + out.append({"name": "Code Complete Checker run", "url": build_url(ch)}) + orid = (model.get("orchestrator") or {}).get("run_id") + if orid: + out.append({"name": "Release Orchestrator run", "url": build_url(orid)}) + for prov in ("ECS", "Local"): + rid = ((model.get("mrwp") or {}).get(prov) or {}).get("run_id") + if rid: + out.append({"name": f"MRWP {prov} run", "url": build_url(rid)}) + return out + + +def rc_ui_gate(model) -> dict: + """The Phase-2 RC quality gate. Aggregates UI-automation results across BOTH MRWP + providers (ECS + Local) and decides whether RC quality clears the bar. Returns + {ui_total, ui_passed, ui_failed, pass_pct, threshold, verdict, detail} + `verdict` is 'pass' when pass_pct >= RC_UI_PASS_THRESHOLD (or no UI tests were found), + else 'attention' (below the bar → the rc_report step must block for investigation). + `detail` is the human note recorded on the step / shown to the owner.""" + ui_total = ui_pass = ui_fail = 0 + for prov in ("ECS", "Local"): + ui = (((model.get("mrwp") or {}).get(prov) or {}).get("tests") or {}) \ + .get("categories", {}).get("ui") or {} + ui_total += ui.get("total") or 0 + ui_pass += ui.get("passed") or 0 + ui_fail += ui.get("failed") or 0 + thr = RC_UI_PASS_THRESHOLD + if not ui_total: + return {"ui_total": 0, "ui_passed": 0, "ui_failed": 0, "pass_pct": None, + "threshold": thr, "verdict": "pass", + "detail": ("\u26a0 No UI-automation tests were found in either MRWP run — " + "nothing to gate on. Proceeding, but verify RC test coverage.")} + pass_pct = round(ui_pass * 100.0 / ui_total, 1) + ok = pass_pct >= thr + head = (f"UI-automation pass rate {pass_pct}% ({ui_pass}/{ui_total} passed, " + f"{ui_fail} failed) across ECS + Local") + if ok: + detail = f"{head} \u2014 at or above the {thr:.0f}% gate. RC quality OK to proceed." + else: + detail = (f"{head} \u2014 BELOW the {thr:.0f}% gate. This is a large UI failure; " + f"investigate the root cause (it likely needs a fix + an MRWP re-run). " + f"The failing suites are in the RC report email. Once fixed and re-run, " + f"re-run this step (`next`); to override, `skip`.") + suites = [] + for prov in ("ECS", "Local"): + for s in (((model.get("mrwp") or {}).get(prov) or {}).get("failed_suites") or []): + if s.get("category", "ui") == "ui" and s.get("failed"): + suites.append((prov, s)) + suites.sort(key=lambda ps: -ps[1]["failed"]) + if suites: + detail += "\nTop UI failures:\n" + "\n".join( + f" \u2022 [{prov}] {s['name']}: {s['failed']}/{s['total']} failed" + for prov, s in suites[:6]) + return {"ui_total": ui_total, "ui_passed": ui_pass, "ui_failed": ui_fail, + "pass_pct": pass_pct, "threshold": thr, + "verdict": "pass" if ok else "attention", "detail": detail} + + def rc_email_subject(model) -> str: rid = model.get("release", "?") - return (f"Release {rid} — RC verification report (Phase 2) · " - f"action: approve to proceed to bug bash") + action = ("approve to proceed to bug bash" + if rc_ui_gate(model)["verdict"] == "pass" + else "investigate UI failures before proceeding") + return f"Release {rid} — RC verification report (Phase 2) · action: {action}" def _fail_rate(failed, total) -> float: diff --git a/release-agent/steps/build_verify/rc_report.py b/release-agent/steps/build_verify/rc_report.py index a8b337e0..6a94851c 100644 --- a/release-agent/steps/build_verify/rc_report.py +++ b/release-agent/steps/build_verify/rc_report.py @@ -1,15 +1,23 @@ -"""Step: `rc_report` — email the RC verification report to the release owner -(Phase 2, build_verify), right before the go_test approval gate. +"""Step: `rc_report` — email the RC verification report to the release owner AND apply +the Phase-2 UI-automation quality gate (Phase 2, build_verify), right before the +go_test approval gate. When the four verification steps have resolved the chain, this step composes the Phase-2 RC report (checker → orchestrator → ECS/Local MRWP + per-run test failures) from LIVE pipeline data and emails it to the release owner, so the engineer wakes to -the report on CCD+1 and can review before approving `go_test`. +the report on CCD+1. The report is ALWAYS sent (the owner gets the dashboard of +failures + links either way). The step's OUTCOME is then decided by the UI gate: if the +UI-automation pass rate across both MRWP runs is >= RC_UI_PASS_THRESHOLD (90%) it +records `pass` and the flow advances to go_test; below the bar it records `attention` +(the step BLOCKS) so the owner investigates the large failure (usually a fix + MRWP +re-run) before proceeding. Sending email needs the WorkIQ MCP the engine can't reach, so this is a `scout` step: `build()` composes the email deterministically and returns a -NeedsSkill(workiq_send_email) for the skill to send. Redirect for tests with the -`send_to` payload knob (keeps the send real, points it at you). +NeedsSkill(workiq_send_email); the payload names the `record-rc-report` follow-up +command, which re-reads the model, applies the gate, records pass|attention, and +stashes the evaluated run links on the step. Redirect for tests with the `send_to` +payload knob (keeps the send real, points it at you). """ from __future__ import annotations @@ -29,7 +37,9 @@ def build(state): """Compose the RC verification email → NeedsSkill(workiq_send_email). Blocks if the - owner email is unknown (nowhere to send) — set it with `set-owner`.""" + owner email is unknown (nowhere to send) — set it with `set-owner`. The email is + always sent; the UI gate verdict (recorded by the `record-rc-report` follow-up) then + decides whether the step passes or blocks.""" to = state.owner_email if not to: return Blocked( @@ -40,8 +50,14 @@ def build(state): except Exception as e: # pragma: no cover - defensive return Blocked(f"rc_report: could not build the RC report ({e}).") - probs = model.get("problems") or [] - tail = f"; {len(probs)} blocking issue(s)" if probs else "" + gate = K.rc_ui_gate(model) + if gate["verdict"] == "pass": + summary = (f"Email the RC verification report to the release owner ({to}) — " + f"UI gate PASS") + else: + summary = (f"Email the RC verification report to the release owner ({to}) — " + f"UI gate FAIL ({gate['pass_pct']}% < {int(K.RC_UI_PASS_THRESHOLD)}%); " + f"will block for investigation") return NeedsSkill( tool="workiq_send_email", payload={ @@ -50,9 +66,12 @@ def build(state): "body": html, "isHtml": True, "_plain_body": plain, + # After sending, DON'T blind-record pass: run this engine command instead — + # it applies the 90% UI gate (pass|attention) and stashes the run links. + "followup_command": "record-rc-report", }, record_as=ID, - summary=f"Email the RC verification report to the release owner ({to}){tail}", - note=f"RC report emailed to {to}", + summary=summary, + note=gate["detail"], outbound=True, ) diff --git a/release-agent/tests/test_engine.py b/release-agent/tests/test_engine.py index 8f108857..6624a055 100644 --- a/release-agent/tests/test_engine.py +++ b/release-agent/tests/test_engine.py @@ -2039,6 +2039,119 @@ def test_build_verify_rc_report_emails_owner(): P.release_report = orig +def test_rc_ui_gate_and_run_links(): + """The Phase-2 UI gate aggregates UI-automation results across BOTH MRWP providers: + >=90% combined pass → 'pass'; below → 'attention' (with a failing-suite summary); + no UI tests → pass with a warning. rc_run_links surfaces every evaluated run.""" + from steps.build_verify import _common as K + + def _model(ecs_ui, local_ui, ecs_suites=None): + return { + "release": "2026-08", + "checker": {"fired": True, "run_id": 111}, + "orchestrator": {"found": True, "run_id": 222}, + "mrwp": { + "ECS": {"run_id": 333, "failed_suites": ecs_suites or [], + "tests": {"categories": {"ui": ecs_ui}}}, + "Local": {"run_id": 444, "failed_suites": [], + "tests": {"categories": {"ui": local_ui}}}}} + + # 180/200 = 90.0% → exactly at the bar → pass + g = K.rc_ui_gate(_model({"total": 100, "passed": 100, "failed": 0}, + {"total": 100, "passed": 80, "failed": 20})) + assert g["verdict"] == "pass" and g["pass_pct"] == 90.0 and g["ui_total"] == 200 + + # 160/200 = 80% → below the bar → attention, with the failing suite listed + fail_model = _model({"total": 100, "passed": 60, "failed": 40}, + {"total": 100, "passed": 100, "failed": 0}, + ecs_suites=[{"name": "PROD MSAL - RC Broker (API 32)", + "failed": 40, "total": 100, "category": "ui"}]) + g2 = K.rc_ui_gate(fail_model) + assert g2["verdict"] == "attention" and g2["pass_pct"] == 80.0 + assert "BELOW" in g2["detail"] and "PROD MSAL - RC Broker (API 32)" in g2["detail"] + + # no UI tests anywhere → pass with a warning (absence of data is not a failure) + g3 = K.rc_ui_gate({"mrwp": {"ECS": {"tests": {"categories": {}}}, + "Local": {"tests": {"categories": {}}}}}) + assert g3["verdict"] == "pass" and g3["ui_total"] == 0 and "No UI-automation" in g3["detail"] + + # every evaluated run becomes a durable link + links = K.rc_run_links(fail_model) + names = [l["name"] for l in links] + assert names == ["Code Complete Checker run", "Release Orchestrator run", + "MRWP ECS run", "MRWP Local run"] + assert all("buildId=" in l["url"] for l in links) + + +def test_record_rc_report_applies_ui_gate_and_stashes_links(): + """`record-rc-report` (the follow-up the skill runs after emailing) applies the 90% + UI gate: >=90% → step done; <90% → step BLOCKS (awaiting_action). Either way it + stashes the evaluated run links on the step. release_report is monkeypatched offline.""" + import tempfile as _tf + from tools import pipelines as P + from orchestrator.commands import rc_report as RR + from orchestrator.state import StepState + + def _model(ecs_ui, local_ui): + return {"release": "2026-08", + "checker": {"fired": True, "run_id": 111}, + "orchestrator": {"found": True, "run_id": 222}, + "mrwp": {"ECS": {"run_id": 333, "failed_suites": [], + "tests": {"categories": {"ui": ecs_ui}}}, + "Local": {"run_id": 444, "failed_suites": [], + "tests": {"categories": {"ui": local_ui}}}}, + "problems": []} + + orig = P.release_report + with _tf.TemporaryDirectory() as d: + rid = "2026-08" + _stub_build_defs("pass") + st = ReleaseState(release_id=rid, ccd="2026-08-26", owner_email="dev@microsoft.com") + orch = Orchestrator(CONFIG, st) + _pass_scout_checks(orch); orch.gate.sign() + C.save_state(st, d, rid) + + class A: + runs_root = d; release = rid; config = CONFIG; as_of = None + + try: + # PASS: 190/200 = 95% ≥ 90 → step done, links stashed + P.release_report = lambda *a, **k: _model( + {"total": 100, "passed": 95, "failed": 5}, + {"total": 100, "passed": 95, "failed": 5}) + assert RR.cmd_record_rc_report(A) == 0 + s1 = C.load_state(d, rid) + assert s1.is_done("build_verify", "rc_report") + step1 = s1.get_step("build_verify", "rc_report") + assert [l["name"] for l in step1.links] == [ + "Code Complete Checker run", "Release Orchestrator run", + "MRWP ECS run", "MRWP Local run"] + + # reset the step, then FAIL: 120/200 = 60% < 90 → blocked, links still stashed + s1.set_step("build_verify", "rc_report", StepState()) + C.save_state(s1, d, rid) + P.release_report = lambda *a, **k: _model( + {"total": 100, "passed": 60, "failed": 40}, + {"total": 100, "passed": 60, "failed": 40}) + assert RR.cmd_record_rc_report(A) == 2 + s2 = C.load_state(d, rid) + step2 = s2.get_step("build_verify", "rc_report") + assert step2.status == "blocked" and not s2.is_done("build_verify", "rc_report") + assert s2.status == "awaiting_action" + assert "build_verify.rc_report" in s2.pending_human + assert "BELOW" in step2.note and len(step2.links) == 4 + finally: + P.release_report = orig + + +def test_active_phase_report_steps_carry_links(): + """The digest's active-phase step model exposes each step's durable `links` so the + links to items a step evaluated aren't dropped from the phase report.""" + st, orch = _mock_orch({}) + ap = orch.status_report()["active_phase"] + assert ap and all("links" in s for s in ap["steps"]) + + def test_classify_test_run_categories(): """The test-run classifier buckets into exactly three: unit / instrumented / ui; anything that isn't unit/instrumented is UI ('the rest are UI', incl. Lab Api Tests).""" From 4da6e02ff4e7502702cb4aec60a30ee6e4577f24 Mon Sep 17 00:00:00 2001 From: p3dr0rv Date: Thu, 20 Aug 2026 12:12:18 -0700 Subject: [PATCH 65/82] Refactor Phase 2 RC verification process: remove go_test gate, implement three-tier UI automation quality gate - Updated build_verify_live.yaml to auto-advance rc_report and position at bug-bash entry. - Modified mocks.local.example.yaml to reflect changes in RC report handling. - Enhanced rc_report.py to apply a three-tier UI gate, determining pass/warn/attention outcomes. - Adjusted sim.py to change auto-approve gates from go_test to bash_done. - Revised SKILL.md and commands.md to update user guidance on RC verification and command usage. - Updated build_verify.md to clarify the removal of the go_test gate and the new RC report process. - Enhanced _common.py to summarize UI failing suites for better reporting. - Updated tests to reflect the removal of the go_test gate and ensure correct behavior in the new flow. --- release-agent/config/knowledge.yaml | 22 ++- release-agent/config/phases.yaml | 6 +- .../config/scenarios/at_rc_gate.yaml | 12 +- .../config/scenarios/build_verify_live.yaml | 7 +- release-agent/mocks.local.example.yaml | 2 +- .../orchestrator/commands/rc_report.py | 11 +- release-agent/orchestrator/sim.py | 2 +- release-agent/skill/SKILL.md | 4 +- release-agent/skill/reference/commands.md | 2 +- .../skill/reference/phases/build_verify.md | 39 +++-- release-agent/steps/build_verify/_common.py | 82 +++++---- release-agent/steps/build_verify/rc_report.py | 25 ++- release-agent/tests/test_engine.py | 160 ++++++++++-------- 13 files changed, 216 insertions(+), 158 deletions(-) diff --git a/release-agent/config/knowledge.yaml b/release-agent/config/knowledge.yaml index d83fdf5c..7aceb7db 100644 --- a/release-agent/config/knowledge.yaml +++ b/release-agent/config/knowledge.yaml @@ -309,26 +309,30 @@ build_verify.mrwp_local: a: "The orchestrator triggers MRWP twice with different flight providers — ECS (server-driven flights) and Local (local flight overrides). Both must run to completion; they're verified separately." build_verify.rc_report: - summary: "Email the RC verification report (pipeline health + test failures) to the release owner." + summary: "Email the RC verification report + apply the three-tier 90% UI gate (the Phase-2 go/no-go)." what: > Once the four verification steps have resolved the chain, this composes the Phase-2 RC report — checker fired, orchestrator healthy/parked, both MRWP runs' stage completion, and each run's - failing test suites — from LIVE pipeline data and emails it to the release owner. It runs right - before the go_test gate so the engineer wakes (on CCD+1) to the report and can review before - approving. Scout-assisted (source: scout): the engine composes it; the skill sends via WorkIQ. + failing test suites — from LIVE pipeline data and emails it to the release owner (on CCD+1). It + is the terminal Phase-2 step and the go/no-go: after sending, `record-rc-report` applies a + three-tier gate on the combined UI-automation pass rate across both MRWP runs — 100% is a clean + pass, >=90% passes with a warning (investigate the failing tests in parallel, bug bash not + blocked), and <90% BLOCKS for owner investigation. There is no separate approval gate. + Scout-assisted (source: scout): the engine composes it; the skill sends via WorkIQ. who: > Scout composes the email; the skill sends it via workiq_send_email to the release owner (state.owner_email). Redirect for testing with the step's `send_to` payload knob. where: - - "Release Orchestrator run (versions + parked gate): via the run link in the email" - - "MRWP ECS / Local runs (Test tab): via the run links in the email" + - "Release Orchestrator run (versions + parked gate): via the run link in the email + the step's Details" + - "MRWP ECS / Local runs (Test tab): via the run links in the email + the step's Details" how: > If it blocks with 'no release owner email', set it with `set-owner --email ` - and rerun. The email frames failing tests as bug-bash triage (not blockers) unless a stage never - ran. After it sends, the go_test gate holds for the owner's approval. + and rerun. On a clean/warn pass (>=90% UI) the release auto-advances into bug bash. On a <90% + block, the owner investigates and rules on it (patch a real bug + re-trigger RC, or proceed as an + automation flake): fix + re-run then `next`, or `skip --reason` to override. faqs: - q: "Who gets the email?" - a: "The release owner on record (owner_email). It's a personal review copy, not the broad DL — the owner reviews the RC failures, then approves the go_test gate." + a: "The release owner on record (owner_email). It's a personal review copy, not the broad DL — the owner reviews the RC failures; the 90% UI gate then decides go/no-go automatically." - q: "Does the sim send this email?" a: "No. In a sim the step is marked done without sending; only the real skill flow composes + sends it via WorkIQ." diff --git a/release-agent/config/phases.yaml b/release-agent/config/phases.yaml index 982936c5..e07fcda3 100644 --- a/release-agent/config/phases.yaml +++ b/release-agent/config/phases.yaml @@ -53,8 +53,10 @@ phases: - { id: orchestrator_health, name: "Verify Release Orchestrator health (parked at Remove RC Tags)", owner: agent, maps_to: [B1] } - { id: mrwp_ecs, name: "Verify MRWP (ECS) ran to completion + tests", owner: agent, maps_to: [B2] } - { id: mrwp_local, name: "Verify MRWP (Local) ran to completion + tests", owner: agent, maps_to: [B3] } - - { id: rc_report, name: "Email RC verification report to release owner", owner: agent, source: scout, maps_to: [B4] } - - { id: go_test, name: "RC verified — proceed to bug bash", owner: human, gate: true } + # rc_report is the Phase-2 go/no-go: it emails the RC report AND applies the 90% UI + # gate — clean/warn (>=90%) proceed to bug bash; <90% BLOCKS for owner investigation. + # No separate human approval step: the gate IS the decision. + - { id: rc_report, name: "Email RC verification report + apply the 90% UI gate", owner: agent, source: scout, maps_to: [B4] } - id: bug_bash name: "Test / Bug Bash" diff --git a/release-agent/config/scenarios/at_rc_gate.yaml b/release-agent/config/scenarios/at_rc_gate.yaml index 084f24ac..34585316 100644 --- a/release-agent/config/scenarios/at_rc_gate.yaml +++ b/release-agent/config/scenarios/at_rc_gate.yaml @@ -1,7 +1,9 @@ -# Fast-forward all the way to Phase 2's go_test gate — the four verification steps -# resolve from deterministic OFFLINE mocks (no az), leaving the release holding at the -# human "RC verified — proceed to bug bash" decision. Use this to exercise the gate -# hold, the status render, and the RC digest one-liner without touching the network. +# Fast-forward THROUGH Phase 2 (Build & RC Verification): the four verification steps +# resolve from deterministic OFFLINE mocks (no az) and stash the pipeline ids, then the +# rc_report step auto-advances (go_test was removed — rc_report's 90% UI gate is the +# decision). Lands positioned at the entry of Phase 3 (bug bash). Use this to exercise +# the RC verification chain, the pipeline-id stash, and the status render without touching +# the network. # # release-agent sim run --scenario at_rc_gate # @@ -10,7 +12,7 @@ release_id: 2026-08 ccd: 2026-08-26 ccd_source: confirmed as_of: CCD+2 -target: { phase: build_verify, at: gate } +target: { phase: build_verify, at: done } data: mock mocks: build_verify.checker_fired: diff --git a/release-agent/config/scenarios/build_verify_live.yaml b/release-agent/config/scenarios/build_verify_live.yaml index 0c214fcb..6d8b83f8 100644 --- a/release-agent/config/scenarios/build_verify_live.yaml +++ b/release-agent/config/scenarios/build_verify_live.yaml @@ -1,7 +1,8 @@ # Test Phase 2 END-TO-END against the REAL pipelines: fast-forwards Phases 0-1, # then runs the four verification steps LIVE (real `az` reads of the 2026-08 -# checker → orchestrator → ECS/Local MRWP runs) and lands holding at the go_test -# gate. Seeds the real release, so afterwards use the normal skill: status / approve. +# checker → orchestrator → ECS/Local MRWP runs), auto-advances rc_report, and lands +# positioned at the bug-bash entry (go_test was removed — rc_report's 90% UI gate is the +# decision). Seeds the real release, so afterwards use the normal skill: status / next. # # release-agent sim run --scenario build_verify_live # @@ -10,5 +11,5 @@ release_id: 2026-08 ccd: 2026-08-26 ccd_source: confirmed as_of: CCD+2 -target: { phase: build_verify, at: gate } +target: { phase: build_verify, at: done } data: live # the 4 build_verify steps hit real az; earlier phases are mocked diff --git a/release-agent/mocks.local.example.yaml b/release-agent/mocks.local.example.yaml index ce6e0710..7929d6b9 100644 --- a/release-agent/mocks.local.example.yaml +++ b/release-agent/mocks.local.example.yaml @@ -81,7 +81,7 @@ preflight.cg: # build_verify.ui_auto: { outcome: done } # agent # build_verify.payload: { outcome: done } # agent # build_verify.mrwp_rc: { outcome: done } # agent -# build_verify.go_test: 🚦 gate — NOT mockable +# build_verify.rc_report: { outcome: done } # scout (emails RC report + applies the 90% UI gate) # ---- Phase 3 · bug_bash ---- # bug_bash.clone_plans: { outcome: done } # agent diff --git a/release-agent/orchestrator/commands/rc_report.py b/release-agent/orchestrator/commands/rc_report.py index 08473d64..d641021d 100644 --- a/release-agent/orchestrator/commands/rc_report.py +++ b/release-agent/orchestrator/commands/rc_report.py @@ -55,10 +55,11 @@ def _u(build_id): def cmd_record_rc_report(args): """Record the rc_report step's outcome AFTER the skill has emailed the RC report. - Re-reads the live model, applies the UI-automation quality gate (K.rc_ui_gate), - records `pass` (>=90% UI pass → step done, flow advances to go_test) or `attention` - (<90% → step BLOCKS for owner investigation), and stashes the evaluated pipeline-run - links on the step so its Details point at every artifact behind the verdict. + Re-reads the live model, applies the three-tier UI-automation gate (K.rc_ui_gate), + records `pass` (>=90% UI pass — clean/warn → step done, release auto-advances into bug + bash) or `attention` (<90% → step BLOCKS for owner investigation), and stashes the + evaluated pipeline-run links on the step so its Details point at every artifact behind + the verdict. This is the follow-up the rc_report NeedsSkill names (`payload.followup_command`), so the skill runs it instead of a blind `record-step --status pass`.""" @@ -71,7 +72,7 @@ def cmd_record_rc_report(args): gate = K.rc_ui_gate(model) links = K.rc_run_links(model) - status = "pass" if gate["verdict"] == "pass" else "attention" + status = "attention" if gate["blocking"] else "pass" orch.record_scout_step("build_verify", "rc_report", status, gate["detail"]) # record_scout_step doesn't carry links — attach the evaluated-run refs (and stamp diff --git a/release-agent/orchestrator/sim.py b/release-agent/orchestrator/sim.py index b188d1c3..79304ed0 100644 --- a/release-agent/orchestrator/sim.py +++ b/release-agent/orchestrator/sim.py @@ -24,7 +24,7 @@ as_of: CCD+1 # CCD-relative or absolute YYYY-MM-DD; default: ccd target: {phase: build_verify, at: open} # at: open | gate | done data: live # live | mock (default: mock) - approve_gates: [go_test] # gates to auto-approve while fast-forwarding (see note) + approve_gates: [bash_done] # gates to auto-approve while fast-forwarding (see note) mocks: # merged last (win); fine-grained inputs or outcome mocks build_verify.checker_fired: { triggering: {...} } seed: # direct ReleaseState field overrides (e.g. pipeline_runs) diff --git a/release-agent/skill/SKILL.md b/release-agent/skill/SKILL.md index 56bb23f9..b0f207a4 100644 --- a/release-agent/skill/SKILL.md +++ b/release-agent/skill/SKILL.md @@ -43,7 +43,7 @@ Discover → (if no gate cleared, run the entry gate) → `next` to advance → - **User asks about the RC pipelines / RC tests / "phase 2 status"** ("how are the release pipelines?", "did the RC tests pass?", "show pipeline status", "is the orchestrator done?"): run **`rc-report --release `** and paste its output verbatim — the checker → orchestrator → ECS/Local-MRWP chain with each run's stage completion + Test-tab breakdown (unit/instrumented/UI-automation). It's **read-only** (never gates); use `--json` for your own branching. Red/yellow stages and failed tests are expected here (triaged in bug bash) — only a stage that never ran is a real problem, and it shows under **Issues**. - **User wants to test/validate a phase mid-release without running one from scratch** ("test phase 2", "simulate phase 2", "let me test the RC phase", "drop me in at the RC gate", "test the bug-bash phase"): this is the **sim** — it SEEDS the real release to a mid-release point so you then drive it with the normal skill. Run it for them via the shell; don't hand them python. Pick the scenario by intent (`sim list` shows all): - - "test phase 2" / "test phase 2 against the real pipelines" / "does phase 2 work" → **`sim run --scenario build_verify_live`** (fast-forwards Phases 0-1, runs the 4 verification steps against the **real** 2026-08 `az` runs, lands holding at the go_test gate). + - "test phase 2" / "test phase 2 against the real pipelines" / "does phase 2 work" → **`sim run --scenario build_verify_live`** (fast-forwards Phases 0-1, runs the 4 verification steps against the **real** 2026-08 `az` runs, auto-advances rc_report, lands positioned at the bug-bash entry). - "test phase 2 offline / quickly / without the network" → **`sim run --scenario at_rc_gate`** (same flow, fully mocked). - "drop me at phase 2 so I can step through it myself" → **`sim run --scenario mid_build_verify_open`** (positions at entry, runs nothing — then use `next` to run each step live). @@ -55,7 +55,7 @@ Discover → (if no gate cleared, run the entry gate) → `next` to advance → | Running the readiness entry gate (right after `init`) | `reference/readiness-gate.md` | | Starting a release / handling CCD / setting up push reminders & automations | `reference/starting-and-scheduling.md` | | Advancing **Phase 0 (Pre-flight)** — notice, flight reminders, lockdown, confirm, vitals | `reference/phases/preflight.md` | -| Advancing **Phase 2 (Build & RC Verification)** — verification chain, RC report email + 90% UI gate, go_test | `reference/phases/build_verify.md` | +| Advancing **Phase 2 (Build & RC Verification)** — verification chain, RC report email + three-tier 90% UI gate (no separate gate) | `reference/phases/build_verify.md` | | Rendering `status`/`checklist` output | `reference/presenting-status.md` | | Looking up a command / manual override / event-logging detail | `reference/commands.md` | | Building a NEW phase's guidance | `reference/phases/_TEMPLATE.md` | diff --git a/release-agent/skill/reference/commands.md b/release-agent/skill/reference/commands.md index 96110fb1..f7e87faf 100644 --- a/release-agent/skill/reference/commands.md +++ b/release-agent/skill/reference/commands.md @@ -15,7 +15,7 @@ _Loaded on demand. Run all from `C:\repos\android-complete\release-agent`._ | Resolve a migrated step → outcome JSON (done\|blocked\|needs_human\|needs_skill) | `python -m orchestrator.cli step-action --release --step [--phase

    ] [--param k=v …]` | | Answer a STEP question (knowledge) | `python -m orchestrator.cli step-info --step [--phase

    ]` | | **Phase 2 — RC pipeline + test report** (read-only) | `python -m orchestrator.cli rc-report --release [--json]` → the checker→orchestrator→ECS/Local-MRWP chain + per-run test breakdown | -| **Phase 2 — record RC verdict** (after emailing the report) | `python -m orchestrator.cli record-rc-report --release ` → applies the **90% UI-automation gate** across both MRWP runs, records `pass` (step done → go_test) or `attention` (step **blocks** for investigation), and stashes the checker/orchestrator/ECS/Local run links on the step. This is the follow-up the `rc_report` `needs_skill` names — run it **instead of** `record-step` | +| **Phase 2 — record RC verdict** (after emailing the report) | `python -m orchestrator.cli record-rc-report --release ` → applies the **three-tier 90% UI-automation gate** across both MRWP runs, records `pass` (100% clean / ≥90% warn → step done, release auto-advances into bug bash) or `attention` (<90% → step **blocks** for investigation), and stashes the checker/orchestrator/ECS/Local run links on the step. This is the follow-up the `rc_report` `needs_skill` names — run it **instead of** `record-step` | | **Simulate a mid-release point** (testing) | `python -m orchestrator.cli sim list` · `python -m orchestrator.cli sim run --scenario [--freeze] [--json]` → **seeds the real release** to a scenario's target (`config/scenarios/.yaml`): fast-forwards the real engine, signs the entry gate + completes earlier phases from mocks, then stops `open`/`gate`/`done` at the target. `data: live` runs the target phase against real `az`; `data: mock` is offline. Any existing state at that id is backed up first, so afterwards you use the **normal** commands (`status`, `rc-report`, `next`, `approve`). `--runs-root ` targets a throwaway sandbox instead; `--freeze` snapshots state to `tests/fixtures/.json` | | Answer an ENTRY-GATE item question (knowledge) | `python -m orchestrator.cli gate-info --item ` (build_access, mcp_servers, ccd_confirmed, silent_perms, teams_notify, adx_access, oncall_now, play_console_access, oncall_window, saw_ame, yubikey) | | Prepare early code-complete notice (JSON) — _legacy; prefer `step-action --step notice`_ | `python -m orchestrator.cli prepare-notice --release [--variant initial\|update]` | diff --git a/release-agent/skill/reference/phases/build_verify.md b/release-agent/skill/reference/phases/build_verify.md index e6163f38..79e4b22e 100644 --- a/release-agent/skill/reference/phases/build_verify.md +++ b/release-agent/skill/reference/phases/build_verify.md @@ -2,21 +2,23 @@ Opens **CCD+1** — the engineer wakes to a resume. The engine runs the four verification **agent** steps in-process during `next`; you relay their results and drive the one -**scout** step (`rc_report`) + the human **gate** (`go_test`). +**scout** step (`rc_report`), which is the terminal Phase-2 step **and** the go/no-go — +there is no separate human gate. ## Execution model Sequential. A single `next` runs the agent chain (checker → orchestrator → ECS/Local MRWP); each **blocks** on a real problem (a stage that never ran, an unhealthy orchestrator, an auth failure). A blocked step → show the note, then **fix + `next`** to re-check, or **`skip … --reason`** to override. When the chain is green the scout -`rc_report` step becomes ready, then the `go_test` gate. +`rc_report` step runs — it is the last Phase-2 step and the go/no-go. ## Automated steps (no skill action — relay from the `status` table) `checker_fired`, `orchestrator_health`, `mrwp_ecs`, `mrwp_local` — read-only `az` agent steps run inside `next`. Each records the ADO run it evaluated as a Details 🔗 link. `step-action` refuses them (exit 1); never dispatch them yourself. -## `rc_report` — email the RC report + apply the 90% UI gate (`scout`) +## `rc_report` — email the RC report + apply the 90% UI gate (`scout`, terminal) +This is the Phase-2 go/no-go — there is **no separate approval gate**. - **Trigger:** `status --json` shows current step `rc_report` (state `scout`), after the four agent steps are done. - **Resolve:** `step-action --release --phase build_verify --step rc_report` → @@ -24,23 +26,24 @@ steps run inside `next`. Each records the ADO run it evaluated as a Details 🔗 `payload.followup_command: record-rc-report`. - **Act:** send the email verbatim (`payload.to/subject/body`, `isHtml:true`) — honoring a `send_to` redirect if the engineer set one. **Always send** — the owner gets the - dashboard (failing suites + run links) whether the gate passes or not. + dashboard (failing suites + run links) whatever the verdict is. - **Record (the two-hop):** because `followup_command` is set, run `record-rc-report --release ` **instead of** `record-step`. It re-reads the model, - applies the **90% UI-automation gate** (combined pass rate across ECS + Local): - - **≥ 90% → `pass`** — the step is done; advance to `go_test`. - - **< 90% → `attention`** — the step **BLOCKS** (`awaiting_action`). This is a large UI - failure: the owner must **investigate the root cause** (usually a **fix + an MRWP - re-run**). Exits: fix + re-run, then `next` re-runs `rc_report`; or `skip … --reason` - to override. It records the failing-suite summary + stashes the checker/orchestrator/ - ECS/Local run links on the step. - - (No UI tests found → passes with a ⚠ note.) -- The command prints `{verdict, pass_pct, ui_total, detail, links}` for your branching; - relay the `status` table (the `rc_report` Details shows the verdict + 🔗 links). - -## `go_test` — RC verified, proceed to bug bash (`gate`, human) -Present the settled `status`; `m_ask_user` Approve/Deny; `approve` / `deny --comment`. -Never authorize yourself. If `rc_report` blocked, `go_test` isn't reached until it clears. + applies the **three-tier 90% UI-automation gate** (combined pass rate across ECS + Local): + - **100% → `clean`** — step done; the release auto-advances into Phase 3 (bug bash). + - **≥ 90% & < 100% → `warn`** — step done; auto-advances into bug bash, but the owner + should investigate the failing UI tests **in parallel** (a later step confirms the + retest — bug bash is **not** blocked). + - **< 90% → `attention`** — the step **BLOCKS** (`awaiting_action`). Large UI failure: the + owner investigates and decides — patch a real bug + re-trigger RC, or (if it's an + automation flake to re-run later) proceed to bug bash. Exits: fix + re-run, then + `next` re-runs `rc_report`; or `skip … --reason` to override. + - (No UI tests found → `clean` with a ⚠ note.) + It records the failing-suite summary + stashes the checker/orchestrator/ECS/Local run + links on the step. +- The command prints `{verdict, blocking, pass_pct, ui_total, detail, links}` for your + branching; relay the `status` table (the `rc_report` Details shows the verdict + 🔗 links). + On `clean`/`warn` the engine auto-advances into Phase 3 — brief the owner and continue. ## External references Engineering pipelines: Checker def 3038, Orchestrator def 2828, MRWP def 2519 diff --git a/release-agent/steps/build_verify/_common.py b/release-agent/steps/build_verify/_common.py index 7d199c98..629adf50 100644 --- a/release-agent/steps/build_verify/_common.py +++ b/release-agent/steps/build_verify/_common.py @@ -93,13 +93,35 @@ def rc_run_links(model) -> list: return out +def _ui_failing_suites_summary(model, limit=6) -> str: + """A compact 'Top UI failures' list across both providers, or '' when none.""" + suites = [] + for prov in ("ECS", "Local"): + for s in (((model.get("mrwp") or {}).get(prov) or {}).get("failed_suites") or []): + if s.get("category", "ui") == "ui" and s.get("failed"): + suites.append((prov, s)) + suites.sort(key=lambda ps: -ps[1]["failed"]) + if not suites: + return "" + return "\nTop UI failures:\n" + "\n".join( + f" \u2022 [{prov}] {s['name']}: {s['failed']}/{s['total']} failed" + for prov, s in suites[:limit]) + + def rc_ui_gate(model) -> dict: - """The Phase-2 RC quality gate. Aggregates UI-automation results across BOTH MRWP - providers (ECS + Local) and decides whether RC quality clears the bar. Returns - {ui_total, ui_passed, ui_failed, pass_pct, threshold, verdict, detail} - `verdict` is 'pass' when pass_pct >= RC_UI_PASS_THRESHOLD (or no UI tests were found), - else 'attention' (below the bar → the rc_report step must block for investigation). - `detail` is the human note recorded on the step / shown to the owner.""" + """The Phase-2 RC quality gate — a THREE-tier decision on the combined UI-automation + pass rate across both MRWP providers (ECS + Local). Returns + {ui_total, ui_passed, ui_failed, pass_pct, threshold, verdict, blocking, detail} + where `verdict` is: + * 'clean' — 100% UI pass (or no UI tests found): proceed, no action. + * 'warn' — >= RC_UI_PASS_THRESHOLD (90%) but < 100%: proceed to bug bash, but + the owner should investigate the failing UI tests IN PARALLEL (a + later step confirms the retest — bug bash is NOT blocked). + * 'attention' — < 90%: BLOCK. A large failure the owner must investigate and rule + on (patch a real bug + re-trigger RC, or proceed as an automation + flake to re-run later). + `blocking` is True only for 'attention'. `detail` is the note recorded on the step / + shown to the owner.""" ui_total = ui_pass = ui_fail = 0 for prov in ("ECS", "Local"): ui = (((model.get("mrwp") or {}).get(prov) or {}).get("tests") or {}) \ @@ -108,42 +130,38 @@ def rc_ui_gate(model) -> dict: ui_pass += ui.get("passed") or 0 ui_fail += ui.get("failed") or 0 thr = RC_UI_PASS_THRESHOLD + base = {"ui_total": ui_total, "ui_passed": ui_pass, "ui_failed": ui_fail, "threshold": thr} if not ui_total: - return {"ui_total": 0, "ui_passed": 0, "ui_failed": 0, "pass_pct": None, - "threshold": thr, "verdict": "pass", + return {**base, "pass_pct": None, "verdict": "clean", "blocking": False, "detail": ("\u26a0 No UI-automation tests were found in either MRWP run — " "nothing to gate on. Proceeding, but verify RC test coverage.")} pass_pct = round(ui_pass * 100.0 / ui_total, 1) - ok = pass_pct >= thr head = (f"UI-automation pass rate {pass_pct}% ({ui_pass}/{ui_total} passed, " f"{ui_fail} failed) across ECS + Local") - if ok: - detail = f"{head} \u2014 at or above the {thr:.0f}% gate. RC quality OK to proceed." - else: - detail = (f"{head} \u2014 BELOW the {thr:.0f}% gate. This is a large UI failure; " - f"investigate the root cause (it likely needs a fix + an MRWP re-run). " - f"The failing suites are in the RC report email. Once fixed and re-run, " - f"re-run this step (`next`); to override, `skip`.") - suites = [] - for prov in ("ECS", "Local"): - for s in (((model.get("mrwp") or {}).get(prov) or {}).get("failed_suites") or []): - if s.get("category", "ui") == "ui" and s.get("failed"): - suites.append((prov, s)) - suites.sort(key=lambda ps: -ps[1]["failed"]) - if suites: - detail += "\nTop UI failures:\n" + "\n".join( - f" \u2022 [{prov}] {s['name']}: {s['failed']}/{s['total']} failed" - for prov, s in suites[:6]) - return {"ui_total": ui_total, "ui_passed": ui_pass, "ui_failed": ui_fail, - "pass_pct": pass_pct, "threshold": thr, - "verdict": "pass" if ok else "attention", "detail": detail} + if pass_pct >= 100.0: + return {**base, "pass_pct": pass_pct, "verdict": "clean", "blocking": False, + "detail": (f"UI-automation pass rate 100% ({ui_pass}/{ui_total}) — all UI " + f"tests passed. Proceeding to bug bash.")} + if pass_pct >= thr: + return {**base, "pass_pct": pass_pct, "verdict": "warn", "blocking": False, + "detail": (f"{head} \u2014 at or above the {thr:.0f}% gate but not clean. " + f"Proceeding to bug bash; release owner: investigate the {ui_fail} " + f"failing UI test(s) in parallel (a later step confirms the retest, " + f"so bug bash is not blocked)." + _ui_failing_suites_summary(model))} + return {**base, "pass_pct": pass_pct, "verdict": "attention", "blocking": True, + "detail": (f"{head} \u2014 BELOW the {thr:.0f}% gate. Large UI failure: investigate " + f"the root cause and decide \u2014 patch a real bug + re-trigger RC, or " + f"(if it's an automation flake to re-run later) proceed to bug bash. This " + f"step stays BLOCKED until you `next` after a re-run, or `skip --reason` " + f"to override." + _ui_failing_suites_summary(model))} def rc_email_subject(model) -> str: rid = model.get("release", "?") - action = ("approve to proceed to bug bash" - if rc_ui_gate(model)["verdict"] == "pass" - else "investigate UI failures before proceeding") + v = rc_ui_gate(model)["verdict"] + action = {"clean": "approve to proceed to bug bash", + "warn": "proceeding to bug bash — investigate failing UI tests in parallel", + "attention": "investigate UI failures before proceeding"}[v] return f"Release {rid} — RC verification report (Phase 2) · action: {action}" diff --git a/release-agent/steps/build_verify/rc_report.py b/release-agent/steps/build_verify/rc_report.py index 6a94851c..87252ee9 100644 --- a/release-agent/steps/build_verify/rc_report.py +++ b/release-agent/steps/build_verify/rc_report.py @@ -1,16 +1,18 @@ """Step: `rc_report` — email the RC verification report to the release owner AND apply -the Phase-2 UI-automation quality gate (Phase 2, build_verify), right before the -go_test approval gate. +the Phase-2 UI-automation quality gate (Phase 2, build_verify). This is the terminal +Phase-2 step and the go/no-go — there is no separate human approval gate. When the four verification steps have resolved the chain, this step composes the Phase-2 RC report (checker → orchestrator → ECS/Local MRWP + per-run test failures) from LIVE pipeline data and emails it to the release owner, so the engineer wakes to the report on CCD+1. The report is ALWAYS sent (the owner gets the dashboard of -failures + links either way). The step's OUTCOME is then decided by the UI gate: if the -UI-automation pass rate across both MRWP runs is >= RC_UI_PASS_THRESHOLD (90%) it -records `pass` and the flow advances to go_test; below the bar it records `attention` -(the step BLOCKS) so the owner investigates the large failure (usually a fix + MRWP -re-run) before proceeding. +failures + links either way). The step's OUTCOME is then decided by the three-tier UI +gate on the combined UI-automation pass rate across both MRWP runs: 100% is a clean +pass; >= RC_UI_PASS_THRESHOLD (90%) but < 100% passes with a warning (investigate the +failing tests in parallel — bug bash is NOT blocked); below 90% records `attention` +(the step BLOCKS) so the owner investigates the large failure and rules on it (patch + +re-trigger RC, or proceed as an automation flake). On a clean/warn pass the release +auto-advances into Phase 3 (bug bash). Sending email needs the WorkIQ MCP the engine can't reach, so this is a `scout` step: `build()` composes the email deterministically and returns a @@ -51,9 +53,14 @@ def build(state): return Blocked(f"rc_report: could not build the RC report ({e}).") gate = K.rc_ui_gate(model) - if gate["verdict"] == "pass": + v = gate["verdict"] + if v == "clean": summary = (f"Email the RC verification report to the release owner ({to}) — " - f"UI gate PASS") + f"UI gate CLEAN (100%)") + elif v == "warn": + summary = (f"Email the RC verification report to the release owner ({to}) — " + f"UI gate PASS with warning ({gate['pass_pct']}%); proceed + investigate " + f"failing UI tests in parallel") else: summary = (f"Email the RC verification report to the release owner ({to}) — " f"UI gate FAIL ({gate['pass_pct']}% < {int(K.RC_UI_PASS_THRESHOLD)}%); " diff --git a/release-agent/tests/test_engine.py b/release-agent/tests/test_engine.py index 6624a055..c70edf4b 100644 --- a/release-agent/tests/test_engine.py +++ b/release-agent/tests/test_engine.py @@ -163,6 +163,15 @@ def _orch(signed=True): return st, orch +def _advance_to_first_gate(orch): + """Now that go_test is gone, the first real GATE is Phase-3 `bug_bash.bash_done`, + reached after the Phase-3 `ui_failures` human reminder. Drive to that reminder, clear + it, then drive to the bash_done gate.""" + orch.run_until_gate() # holds at ui_failures (reminder) + orch.complete_step("bug_bash", "ui_failures", "test: UI failures reviewed") + orch.run_until_gate() # holds at bash_done (gate) + + # ---- readiness entry gate ---- def test_entry_gate_blocks_before_signing(): @@ -185,10 +194,10 @@ def test_signing_clears_entry_gate(): _clear_phase0_scout(orch) _clear_ccd_scout(orch) orch.run_until_gate() - # Phase 0 and Phase 1 have no gate (branch cut is automatic); the first real - # gate is go_test at the end of Build & Lib Verification. - assert st.current_step == "go_test" - assert st.status == "holding_gate" + # Phases 0, 1 and 2 have no human gate (rc_report's 90% UI gate is auto); the first + # hold is the Phase-3 'ui_failures' human reminder. + assert st.current_step == "ui_failures" + assert st.status == "awaiting_action" def test_partial_sign_does_not_clear(): @@ -619,40 +628,40 @@ def test_render_is_consistent_and_has_links(): # ---- phase flow (readiness pre-signed) ---- -def test_holds_at_first_gate(): +def test_holds_at_first_hold(): st, orch = _orch() actions = orch.run_until_gate() - assert actions[-1].kind == "gate" - assert actions[-1].step == "go_test" # Phases 0 & 1 are gateless; first gate is go_test (Phase 2) - # auto steps that RUN before the first gate: Phase-0 breaking/cg/cron/wiki (4) + - # Phase-2 build_verify checker_fired/orchestrator_health/mrwp_ecs/mrwp_local (4) + - # rc_report (scout email, mocked done here) (1). The Phase-1 scout steps are - # pre-recorded by _orch's _clear_ccd_scout (not "ran"). - assert sum(1 for a in actions if a.kind == "ran") == 9 + assert actions[-1].kind == "reminder" + assert actions[-1].step == "ui_failures" # Phases 0-2 gateless (rc_report auto); first hold is Phase-3 ui_failures + # auto steps that RUN before the first hold: Phase-0 breaking/cg/cron/wiki (4) + + # Phase-2 checker_fired/orchestrator_health/mrwp_ecs/mrwp_local (4) + rc_report (scout + # email, mocked done here) (1) + Phase-3 clone_plans/coordinate stubs (2). The Phase-1 + # scout steps are pre-recorded by _orch's _clear_ccd_scout (not "ran"). + assert sum(1 for a in actions if a.kind == "ran") == 11 def test_gate_blocks_until_approved(): st, orch = _orch() - orch.run_until_gate() + _advance_to_first_gate(orch) assert st.status == "holding_gate" orch.run_until_gate() assert st.status == "holding_gate" - assert not st.is_done("build_verify", "go_test") + assert not st.is_done("bug_bash", "bash_done") def test_approve_advances(): st, orch = _orch() - orch.run_until_gate() + _advance_to_first_gate(orch) orch.approve_gate("ok") - assert st.is_done("build_verify", "go_test") + assert st.is_done("bug_bash", "bash_done") orch.run_until_gate() - assert st.current_step == "ui_failures" # next stop after go_test: a bug-bash human to-do - assert st.status == "awaiting_action" + assert st.current_step == "gate_watch" # next stop after bash_done: the Phase-4 finalize gate + assert st.status == "holding_gate" def test_deny_blocks(): st, orch = _orch() - orch.run_until_gate() + _advance_to_first_gate(orch) orch.deny_gate("flag not approved") assert st.status == "blocked" assert any("denied" in p for p in st.pending_human) @@ -684,16 +693,16 @@ def test_persistence_roundtrip(): orch.gate.sign() _clear_phase0_scout(orch) _clear_ccd_scout(orch) - orch.run_until_gate() + _advance_to_first_gate(orch) st.save(path) # reload — simulates resuming next day st2 = ReleaseState.load(path) assert st2.status == "holding_gate" - assert st2.current_step == "go_test" + assert st2.current_step == "bash_done" assert st2.readiness_signed # readiness survives the roundtrip orch2 = Orchestrator(CONFIG, st2) orch2.approve_gate("resumed") - assert st2.is_done("build_verify", "go_test") + assert st2.is_done("bug_bash", "bash_done") def test_conditional_hotfix_excluded_by_default(): @@ -713,31 +722,31 @@ def test_conditional_hotfix_excluded_by_default(): def test_skip_requires_reason(): st, orch = _orch() - orch.run_until_gate() # holds at go_test - act = orch.skip_step("build_verify", "go_test", "") # no reason + _advance_to_first_gate(orch) # holds at bash_done + act = orch.skip_step("bug_bash", "bash_done", "") # no reason assert act.kind == "idle" - assert not st.is_done("build_verify", "go_test") # unchanged + assert not st.is_done("bug_bash", "bash_done") # unchanged def test_skip_advances_past_gate(): st, orch = _orch() - orch.run_until_gate() - orch.skip_step("build_verify", "go_test", "n/a this release") - assert st.is_done("build_verify", "go_test") # skipped counts as done - rec = st.steps[st.key("build_verify", "go_test")] + _advance_to_first_gate(orch) + orch.skip_step("bug_bash", "bash_done", "n/a this release") + assert st.is_done("bug_bash", "bash_done") # skipped counts as done + rec = st.steps[st.key("bug_bash", "bash_done")] assert rec["status"] == "skipped" orch.run_until_gate() - assert st.current_step == "ui_failures" # advanced past the gate to the next hold + assert st.current_step == "gate_watch" # advanced past the gate to the Phase-4 gate def test_reopen_step(): st, orch = _orch() - orch.run_until_gate(); orch.approve_gate("ok") - assert st.is_done("build_verify", "go_test") - orch.reopen_step("build_verify", "go_test") - assert not st.is_done("build_verify", "go_test") # back to pending + _advance_to_first_gate(orch); orch.approve_gate("ok") + assert st.is_done("bug_bash", "bash_done") + orch.reopen_step("bug_bash", "bash_done") + assert not st.is_done("bug_bash", "bash_done") # back to pending orch.run_until_gate() - assert st.current_step == "go_test" # gate re-holds + assert st.current_step == "bash_done" # gate re-holds def test_halt_blocks_then_resume(): @@ -978,7 +987,7 @@ def test_no_anchor_when_ccd_unknown_runs_immediately(): """Backward-compatible: with no CCD stored, the anchor is inert and Phase 0 runs.""" st, orch = _orch() # ccd is None orch.run_until_gate() - assert st.status == "holding_gate" # reached flag_freeze, not 'scheduled' + assert st.status == "awaiting_action" # reached the first hold (Phase-3 ui_failures), not 'scheduled' # ---- reminder steps (human, non-gate → hold until done) ---- @@ -1079,11 +1088,11 @@ def test_digest_silent_while_scout_pending(): def test_notify_digest_reports_gate_and_progress(): from orchestrator import render st, orch = _orch() # signed, no CCD → phase due immediately - orch.run_until_gate() # Phases 0 & 1 gateless; holds at go_test (Phase 2) + orch.run_until_gate() # Phases 0-2 gateless; holds at the Phase-3 ui_failures action msg = render.notification(orch.status_report()) assert "Progress:" in msg - assert "Waiting on your decision" in msg and "proceed to bug bash" in msg.lower() - assert "your approval" in msg # lists the human touchpoint + assert "Action needed now" in msg # ui_failures is the live hold + assert "your approval" in msg # the bash_done gate is listed among the human touchpoints def test_notify_json_carries_owner_and_subject(): @@ -1976,19 +1985,20 @@ def test_build_verify_checker_blocks_when_not_triggered(): def test_build_verify_phase_shape(): - """Phase 2 has the 4 verification agent steps + the rc_report scout email + the - go_test human gate (in order), CCD+1 anchored, and the old action stubs are gone.""" + """Phase 2 has the 4 verification agent steps + the rc_report scout step (which emails + the RC report AND applies the 90% UI gate). rc_report is the terminal step — there is + NO separate human gate (the gate IS the decision). CCD+1 anchored.""" import yaml as _yaml cfg = _yaml.safe_load(open(CONFIG, encoding="utf-8")) bv = next(p for p in cfg["phases"] if p["id"] == "build_verify") ids = [s["id"] for s in bv["steps"]] assert ids == ["checker_fired", "orchestrator_health", "mrwp_ecs", "mrwp_local", - "rc_report", "go_test"] + "rc_report"] assert bv.get("anchor") == "CCD+1" rc = next(s for s in bv["steps"] if s["id"] == "rc_report") assert rc.get("source") == "scout" and rc.get("owner") == "agent" - gate = bv["steps"][-1] - assert gate["id"] == "go_test" and gate.get("gate") and gate.get("owner") == "human" + assert bv["steps"][-1]["id"] == "rc_report" # terminal Phase-2 step + assert not any(s.get("gate") for s in bv["steps"]) # no human gate in Phase 2 def test_build_verify_rc_report_emails_owner(): @@ -2040,9 +2050,10 @@ def test_build_verify_rc_report_emails_owner(): def test_rc_ui_gate_and_run_links(): - """The Phase-2 UI gate aggregates UI-automation results across BOTH MRWP providers: - >=90% combined pass → 'pass'; below → 'attention' (with a failing-suite summary); - no UI tests → pass with a warning. rc_run_links surfaces every evaluated run.""" + """The Phase-2 UI gate is three-tier on the combined UI pass rate across BOTH MRWP + providers: 100% → 'clean'; >=90% & <100% → 'warn' (non-blocking, investigate in + parallel); <90% → 'attention' (blocking, with a failing-suite summary); no UI tests → + 'clean' with a warning. rc_run_links surfaces every evaluated run.""" from steps.build_verify import _common as K def _model(ecs_ui, local_ui, ecs_suites=None): @@ -2056,24 +2067,32 @@ def _model(ecs_ui, local_ui, ecs_suites=None): "Local": {"run_id": 444, "failed_suites": [], "tests": {"categories": {"ui": local_ui}}}}} - # 180/200 = 90.0% → exactly at the bar → pass + # 200/200 = 100% → clean (non-blocking) + g0 = K.rc_ui_gate(_model({"total": 100, "passed": 100, "failed": 0}, + {"total": 100, "passed": 100, "failed": 0})) + assert g0["verdict"] == "clean" and g0["blocking"] is False and g0["pass_pct"] == 100.0 + + # 180/200 = 90.0% → exactly at the bar, not clean → warn (non-blocking) g = K.rc_ui_gate(_model({"total": 100, "passed": 100, "failed": 0}, {"total": 100, "passed": 80, "failed": 20})) - assert g["verdict"] == "pass" and g["pass_pct"] == 90.0 and g["ui_total"] == 200 + assert g["verdict"] == "warn" and g["blocking"] is False + assert g["pass_pct"] == 90.0 and g["ui_total"] == 200 + assert "in parallel" in g["detail"] - # 160/200 = 80% → below the bar → attention, with the failing suite listed + # 160/200 = 80% → below the bar → attention (blocking), with the failing suite listed fail_model = _model({"total": 100, "passed": 60, "failed": 40}, {"total": 100, "passed": 100, "failed": 0}, ecs_suites=[{"name": "PROD MSAL - RC Broker (API 32)", "failed": 40, "total": 100, "category": "ui"}]) g2 = K.rc_ui_gate(fail_model) - assert g2["verdict"] == "attention" and g2["pass_pct"] == 80.0 + assert g2["verdict"] == "attention" and g2["blocking"] is True and g2["pass_pct"] == 80.0 assert "BELOW" in g2["detail"] and "PROD MSAL - RC Broker (API 32)" in g2["detail"] - # no UI tests anywhere → pass with a warning (absence of data is not a failure) + # no UI tests anywhere → clean with a warning (absence of data is not a failure) g3 = K.rc_ui_gate({"mrwp": {"ECS": {"tests": {"categories": {}}}, "Local": {"tests": {"categories": {}}}}}) - assert g3["verdict"] == "pass" and g3["ui_total"] == 0 and "No UI-automation" in g3["detail"] + assert g3["verdict"] == "clean" and g3["blocking"] is False + assert g3["ui_total"] == 0 and "No UI-automation" in g3["detail"] # every evaluated run becomes a durable link links = K.rc_run_links(fail_model) @@ -2341,23 +2360,24 @@ def test_digest_shows_rc_line_when_build_verify_active(): def test_sim_fast_forwards_to_rc_gate_offline(): - """The at_rc_gate scenario (fine input mocks, no az) fast-forwards Phases 0-1, - runs the 4 build_verify steps for real on injected inputs, stashes the pipeline - ids, and halts at the go_test gate — all offline.""" + """The at_rc_gate scenario (fine input mocks, no az) fast-forwards Phases 0-1, runs the + 4 build_verify steps for real on injected inputs, stashes the pipeline ids, auto-advances + rc_report, and lands past Phase 2 at the bug-bash entry — all offline.""" import tempfile from orchestrator import sim as SIM with tempfile.TemporaryDirectory() as tmp: res = SIM.run_scenario("at_rc_gate", runs_root=tmp) - assert res.reached and res.stop_kind == "gate" + assert res.reached and res.stop_kind == "done" st = res.state # earlier phases complete assert all(st.is_done("preflight", s) for s in ("notice", "confirm_reminders", "vitals", "wiki")) assert all(st.is_done("ccd", s) for s in ("final_reminder", "localization")) - # the 4 verification steps ran (real build() on mocks) and are done - for s in ("checker_fired", "orchestrator_health", "mrwp_ecs", "mrwp_local"): + # the 4 verification steps ran (real build() on mocks) and rc_report auto-advanced + for s in ("checker_fired", "orchestrator_health", "mrwp_ecs", "mrwp_local", "rc_report"): assert st.is_done("build_verify", s), s - assert not st.is_done("build_verify", "go_test") # gate still holding + from orchestrator.engine import Orchestrator as _O + assert _O(CONFIG, st).current_phase_id() == "bug_bash" # positioned past Phase 2 # pipeline ids were stashed by the steps during the sim assert st.pipeline_runs.get("orchestrator") == "1678611" assert st.pipeline_runs.get("mrwp_ecs") == "1678863" @@ -2379,12 +2399,13 @@ def test_sim_open_positions_at_target_entry(): assert st.is_done("preflight", "wiki") and st.is_done("ccd", "localization") # nothing in the target phase has run assert not any(st.is_done("build_verify", s) for s in - ("checker_fired", "orchestrator_health", "mrwp_ecs", "mrwp_local", "go_test")) + ("checker_fired", "orchestrator_health", "mrwp_ecs", "mrwp_local", "rc_report")) assert st.current_phase == "build_verify" def test_sim_done_mode_completes_phase_and_advances(): - """`at: done` auto-approves the target's gate and lands at the next phase.""" + """`at: done` runs the whole target phase and lands at the next phase (Phase 2 has no + gate now — rc_report auto-advances).""" import tempfile from orchestrator import sim as SIM scenario = {"name": "t_done", "release_id": "2026-08", "ccd": "2026-08-26", @@ -2394,8 +2415,7 @@ def test_sim_done_mode_completes_phase_and_advances(): res = SIM.run_scenario(scenario, runs_root=tmp) st = res.state assert all(st.is_done("build_verify", s) for s in - ("checker_fired", "orchestrator_health", "mrwp_ecs", "mrwp_local", "go_test")) - assert "build_verify.go_test" in res.gates_approved + ("checker_fired", "orchestrator_health", "mrwp_ecs", "mrwp_local", "rc_report")) from orchestrator.engine import Orchestrator orch = Orchestrator(CONFIG, st) assert orch.current_phase_id() == "bug_bash" @@ -2855,8 +2875,8 @@ def test_ccd_steps_blocked_without_ccd(): def test_ccd_phase_shape_and_scout_kinds(): """Phase 1 is three scout comms/trigger steps and NO gate — the branch cut is - automatic (at 11 PM), so there's no manual cut step; the next gate is go_test in - Phase 2.""" + automatic (at 11 PM), so there's no manual cut step; the next hold is the Phase-3 + ui_failures reminder (Phase 2's rc_report gate is automatic).""" import yaml cfg = yaml.safe_load(open(CONFIG, encoding="utf-8")) ccd = next(p for p in cfg["phases"] if p["id"] == "ccd") @@ -3285,11 +3305,11 @@ def test_local_mock_completes_scout_step(): def test_local_mock_never_mocks_a_gate(): """Gate steps are not mockable — a gate still holds for a real decision even if someone lists it in the mock file.""" - st, orch = _mock_orch({"build_verify.go_test": {"outcome": "done"}}, as_of="2026-07-09") - _clear_phase0_scout(orch) # clear Phase-0 holds so we reach the Phase-2 gate + st, orch = _mock_orch({"bug_bash.bash_done": {"outcome": "done"}}, as_of="2026-07-09") + _clear_phase0_scout(orch) # clear Phase-0 holds _clear_ccd_scout(orch) # clear Phase-1 scout comms (Phase 1 is gateless) - orch.run_until_gate() - assert not st.is_done("build_verify", "go_test") + _advance_to_first_gate(orch) # Phases 0-2 gateless; clear ui_failures → hold at bash_done + assert not st.is_done("bug_bash", "bash_done") assert st.status == "holding_gate" From d124d90185d9dfb6cec2c75468d26caa07d7be15 Mon Sep 17 00:00:00 2001 From: p3dr0rv Date: Thu, 20 Aug 2026 20:24:04 +0100 Subject: [PATCH 66/82] release-agent: remove step-id coupling from the generic planner + render MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Design-audit follow-up to the go_test removal — eliminate the same 'step id hardcoded across modules' smell in two more places: 1) automations planner no longer special-cases ccd.localization. A step module may now declare automation_prompt(release, spec) (single source of truth, like fire_at_local); localization's bespoke trigger/poller prompts move into steps/ccd/localization.py and _prompt_for() delegates generically. 2) render/digest no longer hardcode the build_verify phase id for the RC pipeline-run line — driven by a data flag (show_pipeline_runs) in phases.yaml, propagated via engine _active_phase_report. Engine core confirmed free of hardcoded step ids. 181/181 tests pass. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- release-agent/config/phases.yaml | 1 + release-agent/orchestrator/automations.py | 61 +++++++---------------- release-agent/orchestrator/engine.py | 1 + release-agent/orchestrator/render.py | 7 +-- release-agent/steps/ccd/localization.py | 43 ++++++++++++++++ release-agent/tests/test_engine.py | 23 +++++++-- 6 files changed, 86 insertions(+), 50 deletions(-) diff --git a/release-agent/config/phases.yaml b/release-agent/config/phases.yaml index e07fcda3..41e4d664 100644 --- a/release-agent/config/phases.yaml +++ b/release-agent/config/phases.yaml @@ -48,6 +48,7 @@ phases: name: "Build & Lib Verification" checklist_phase: 2 anchor: "CCD+1" # opens the day AFTER Code Complete — the engineer wakes to a resume + show_pipeline_runs: true # surface the RC pipeline-run one-liner (checker/orchestrator/MRWP) in status + digest steps: - { id: checker_fired, name: "Verify Code Complete Checker fired the release", owner: agent, maps_to: [B0] } - { id: orchestrator_health, name: "Verify Release Orchestrator health (parked at Remove RC Tags)", owner: agent, maps_to: [B1] } diff --git a/release-agent/orchestrator/automations.py b/release-agent/orchestrator/automations.py index fc15de9a..bb70d070 100644 --- a/release-agent/orchestrator/automations.py +++ b/release-agent/orchestrator/automations.py @@ -138,50 +138,25 @@ def _ccd_cron(ccd_date, hhmm: str): def _prompt_for(spec: dict, release: str) -> str: """A concrete instruction the automation runs. Scout resolves each step via - step-action, executes the send/trigger, records it, and journals it. The - localization trigger and its poller have bespoke prompts (they don't just - record-step done).""" - step_list = ", ".join(spec["steps"]) - steps = spec.get("steps") or [] + step-action, executes the send/trigger, records it, and journals it. - # Localization poller (interval) — poll the in-flight run. - if spec.get("interval") and steps == ["ccd.localization"]: - return ( - f"Release {release} — localization poller.\n" - f"If localization for {release} is in-flight (it was triggered at noon and " - f"isn't done/blocked yet), poll it once. The ADO MCP can't reach " - f"msazure/One, so read via az (build id is stored on the step):\n" - f"1. status: `az pipelines build show --id " - f"--org https://msazure.visualstudio.com --project One " - f"--query \"{{status:status,result:result}}\" -o json`.\n" - f"2. if completed, find the OneLocBuild@3 log id: `az devops invoke " - f"--org https://msazure.visualstudio.com --area build --resource timeline " - f"--route-parameters project=One buildId= --api-version 7.1 " - f"--query \"records[?name=='OneLocBuild@3'].log.id | [0]\" -o tsv`, then read " - f"it: `az devops invoke --org https://msazure.visualstudio.com --area build " - f"--resource logs --route-parameters project=One buildId= " - f"logId= --api-version 7.1`.\n" - f"3. run `check-localization --release {release} --complete " - f"[--logs \"\"]`.\n" - f"4. act on the printed decision: `timeout` → send the given email; " - f"`complete_pr` → post the given chat message to the Code reviews chat; " - f"`wait`/`complete_none`/`not_started`/`already_final` → nothing to send.\n" - f"Silently journal: `journal --release {release} --source scout --kind " - f"automation --text \"localization-poller: \"`. Stay silent if " - f"there is nothing to do.") - - # Localization trigger (one-shot, noon) — trigger then hand off to the poller. - if not spec.get("interval") and steps == ["ccd.localization"]: - return ( - f"Release {release} — trigger localization.\n" - f"1. run `step-action --release {release} --phase ccd --step localization`;\n" - f"2. run the returned needs_skill action to start pipeline 405133 " - f"(isCreatePrSelected=true); note the queued build id;\n" - f"3. run `record-localization-run --release {release} --build-id ` " - f"— this leaves the step IN-FLIGHT (do NOT record-step done; the poller " - f"finishes it once the run completes or times out);\n" - f"4. silently journal: `journal --release {release} --source scout --kind " - f"automation --text \"ccd-noon triggered ccd.localization\"`.") + A step MAY OWN a bespoke prompt by declaring `automation_prompt(release, spec)` on its + module (the single source of truth, like `fire_at_local`) — used for genuinely bespoke + flows such as the localization trigger + poller. This keeps the planner generic: it + never special-cases a step id. Steps without one get the default send + record-step + prompt below.""" + steps = spec.get("steps") or [] + step_list = ", ".join(steps) + + # Single-step automation whose step owns a bespoke prompt → delegate to the module. + if len(steps) == 1: + phase, _, sid = steps[0].partition(".") + mod = steps_pkg.get_step(phase, sid) + fn = getattr(mod, "automation_prompt", None) + if callable(fn): + prompt = fn(release, spec) + if prompt: + return prompt # Default: send/trigger + record-step done (reminders). return ( diff --git a/release-agent/orchestrator/engine.py b/release-agent/orchestrator/engine.py index 567e0748..868f53d5 100644 --- a/release-agent/orchestrator/engine.py +++ b/release-agent/orchestrator/engine.py @@ -655,6 +655,7 @@ def _active_phase_report(self) -> Optional[dict]: return { "id": phase["id"], "name": phase["name"], "num": phase.get("checklist_phase"), + "show_pipeline_runs": bool(phase.get("show_pipeline_runs")), "done": done, "total": len(steps), "due": self._phase_due(phase), "started": done > 0, "opens": opens.isoformat() if opens else None, diff --git a/release-agent/orchestrator/render.py b/release-agent/orchestrator/render.py index 70b21410..fe383f18 100644 --- a/release-agent/orchestrator/render.py +++ b/release-agent/orchestrator/render.py @@ -447,9 +447,10 @@ def _digest_model(r: dict): hold = ("action", r["action"]["step_name"]) human_all = [o for o in ap.get("outstanding", []) if o["gate"] or o["reminder"]] completed_all = ap.get("completed") or [] - # Phase-2 RC one-liner — only while build_verify is the active phase, best-effort - # (reads state.pipeline_runs; never a live call). Empty until the chain resolves. - rc_line = _pipelines_line(r) if ap.get("id") == "build_verify" else "" + # Phase-2 RC one-liner — only while a phase that opts in (show_pipeline_runs) is + # active, best-effort (reads state.pipeline_runs; never a live call). Empty until the + # chain resolves. + rc_line = _pipelines_line(r) if ap.get("show_pipeline_runs") else "" return { "rid": r.get("release_id", "?"), "ap": ap, diff --git a/release-agent/steps/ccd/localization.py b/release-agent/steps/ccd/localization.py index 1bf3771a..9e61faf1 100644 --- a/release-agent/steps/ccd/localization.py +++ b/release-agent/steps/ccd/localization.py @@ -341,6 +341,49 @@ def build(state): ) +def automation_prompt(release: str, spec: dict) -> str: + """The bespoke automation instruction for THIS step — owned here (single source of + truth, like `fire_at_local`) so the generic automations planner doesn't special-case + the step id. Two shapes: the interval POLLER vs the one-shot noon TRIGGER (the planner + passes the automation `spec`; `interval` set ⇒ poller).""" + if spec.get("interval"): + return ( + f"Release {release} — localization poller.\n" + f"If localization for {release} is in-flight (it was triggered at noon and " + f"isn't done/blocked yet), poll it once. The ADO MCP can't reach " + f"msazure/One, so read via az (build id is stored on the step):\n" + f"1. status: `az pipelines build show --id " + f"--org https://msazure.visualstudio.com --project One " + f"--query \"{{status:status,result:result}}\" -o json`.\n" + f"2. if completed, find the OneLocBuild@3 log id: `az devops invoke " + f"--org https://msazure.visualstudio.com --area build --resource timeline " + f"--route-parameters project=One buildId= --api-version 7.1 " + f"--query \"records[?name=='OneLocBuild@3'].log.id | [0]\" -o tsv`, then read " + f"it: `az devops invoke --org https://msazure.visualstudio.com --area build " + f"--resource logs --route-parameters project=One buildId= " + f"logId= --api-version 7.1`.\n" + f"3. run `check-localization --release {release} --complete " + f"[--logs \"\"]`.\n" + f"4. act on the printed decision: `timeout` → send the given email; " + f"`complete_pr` → post the given chat message to the Code reviews chat; " + f"`wait`/`complete_none`/`not_started`/`already_final` → nothing to send.\n" + f"Silently journal: `journal --release {release} --source scout --kind " + f"automation --text \"localization-poller: \"`. Stay silent if " + f"there is nothing to do.") + + # One-shot (noon) trigger — trigger then hand off to the poller. + return ( + f"Release {release} — trigger localization.\n" + f"1. run `step-action --release {release} --phase ccd --step localization`;\n" + f"2. run the returned needs_skill action to start pipeline 405133 " + f"(isCreatePrSelected=true); note the queued build id;\n" + f"3. run `record-localization-run --release {release} --build-id ` " + f"— this leaves the step IN-FLIGHT (do NOT record-step done; the poller " + f"finishes it once the run completes or times out);\n" + f"4. silently journal: `journal --release {release} --source scout --kind " + f"automation --text \"ccd-noon triggered ccd.localization\"`.") + + KNOWLEDGE = { "summary": "Trigger the loc pipeline at noon, poll it to completion, then post the translations PR for review.", "what": ( diff --git a/release-agent/tests/test_engine.py b/release-agent/tests/test_engine.py index c70edf4b..d942dd1b 100644 --- a/release-agent/tests/test_engine.py +++ b/release-agent/tests/test_engine.py @@ -1453,6 +1453,19 @@ def sync(): assert not u1["ccd-localization-poller"]["changed"] +def test_automation_prompt_delegates_to_step_module(): + """The planner is generic: a step that declares `automation_prompt` owns its bespoke + instruction (localization trigger vs poller), and steps without one get the default + send + record-step prompt — no step id is special-cased in automations.py.""" + from orchestrator import automations as A + by = {a["slug"]: a for a in A.plan(CONFIG, "2026-09", "2026-09-09")["automations"]} + # localization's module owns both bespoke prompts (delegated, not hardcoded here) + assert "trigger localization" in by["ccd-noon"]["prompt"] + assert "localization poller" in by["ccd-localization-poller"]["prompt"] + # a plain multi-step reminder automation uses the generic default prompt + assert "For EACH of these steps in order" in by["ccd-morning"]["prompt"] + + def test_ccd_cron_pins_to_exact_date(): """_ccd_cron builds a cron 'M H D Mo *' targeting the CCD's day+month+time, so a one-shot fires ON the CCD — never the next matching weekday (the early-fire bug).""" @@ -2340,11 +2353,12 @@ def test_build_verify_persists_pipeline_run_ids(): def test_digest_shows_rc_line_when_build_verify_active(): - """When Phase 2 (build_verify) is the active phase and run ids are on state, the daily - digest carries a one-line RC summary; other phases don't show it.""" + """When a phase that opts in (show_pipeline_runs) is active and run ids are on state, + the daily digest carries a one-line RC summary; phases that don't opt in omit it.""" from orchestrator import render r = {"release_id": "2026-08", "readiness_signed": True, "active_phase": {"id": "build_verify", "name": "Build & RC", "num": 2, + "show_pipeline_runs": True, "due": True, "started": True, "done": 2, "total": 5, "outstanding": [], "completed": ["checker_fired", "orchestrator_health"]}, "pipeline_runs": {"checker": "1678599", "orchestrator": "1678611", @@ -2354,8 +2368,9 @@ def test_digest_shows_rc_line_when_build_verify_active(): md = render.notification_markdown(r) assert "RC pipelines:" in text and "orchestrator 1678611" in text assert "MRWP ECS 900001 / Local 900002" in md - # a non-build_verify active phase omits the RC line - r2 = dict(r, active_phase=dict(r["active_phase"], id="prep", name="Prep")) + # a phase that doesn't opt in omits the RC line + r2 = dict(r, active_phase=dict(r["active_phase"], id="prep", name="Prep", + show_pipeline_runs=False)) assert "RC pipelines:" not in render.notification(r2) From 0c612f09ebbee76a12ff975830bf9772d9bee452 Mon Sep 17 00:00:00 2001 From: p3dr0rv Date: Thu, 20 Aug 2026 21:30:55 +0100 Subject: [PATCH 67/82] release-agent: store RC pipeline runs in state (nested schema) + reuse in the report MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit rc_report + the 90% UI gate no longer re-discover pipeline ids live — they read the RECORD the verification steps store in state.pipeline_runs. Schema: { checker{run_id,when}, orchestrator{run_id,versions{},parked}, rcs:[{rc,ecs{...},local{...}}] } — 1 checker + 1 orchestrator, N RC iterations (a re-trigger appends a new ecs/local pair), latest = rcs[-1]. Each provider slot snapshots stage completion + the Test-tab summary + failing suites, so the gate is deterministic/replayable from state alone. Legacy flat pipeline_runs auto-migrates to the nested shape on ReleaseState.load. The live rc-report diagnostic still reads fresh and refreshes the record. render._pipelines_line reads the nested shape (latest RC). 183 tests pass (incl. migration + append-new-rc coverage). Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../orchestrator/commands/rc_report.py | 32 ++- release-agent/orchestrator/render.py | 37 +-- release-agent/orchestrator/state.py | 66 ++++- release-agent/steps/build_verify/_common.py | 136 ++++++++-- .../steps/build_verify/checker_fired.py | 2 +- release-agent/steps/build_verify/mrwp_ecs.py | 3 +- .../steps/build_verify/mrwp_local.py | 3 +- .../steps/build_verify/orchestrator_health.py | 6 +- release-agent/tests/test_engine.py | 247 +++++++++++------- 9 files changed, 381 insertions(+), 151 deletions(-) diff --git a/release-agent/orchestrator/commands/rc_report.py b/release-agent/orchestrator/commands/rc_report.py index d641021d..5e9b1908 100644 --- a/release-agent/orchestrator/commands/rc_report.py +++ b/release-agent/orchestrator/commands/rc_report.py @@ -27,22 +27,28 @@ def cmd_rc_report(args): def _persist(st, model, args): - """Record the resolved run ids on state so status/digest can show them without a - live read. Best-effort — a report must never fail because the state write did.""" + """Record the resolved runs (+ snapshots) on state so status/digest/rc_report read + them without a live call. Best-effort — a report must never fail because the state + write did. (This is the LIVE `rc-report` diagnostic refreshing the record; the verify + steps are the primary writers.)""" if st is None: return - ch = model.get("checker") or {} - o = model.get("orchestrator") or {} - mr = model.get("mrwp") or {} - v = o.get("versions") or {} - vstr = ", ".join(f"{k} {v[k]}" for k in ("Common", "Msal", "Broker") if v.get(k)) or None try: - K.stash_runs(st, - checker=ch.get("run_id"), - orchestrator=o.get("run_id"), - versions=vstr, - mrwp_ecs=(mr.get("ECS") or {}).get("run_id"), - mrwp_local=(mr.get("Local") or {}).get("run_id")) + ch = model.get("checker") or {} + if ch.get("run_id"): + K.stash_checker(st, ch["run_id"], ch.get("when")) + o = model.get("orchestrator") or {} + if o.get("run_id"): + K.stash_orchestrator(st, o["run_id"], + versions={k: v for k, v in (o.get("versions") or {}).items() if v}, + parked=o.get("parked")) + mr = model.get("mrwp") or {} + for slot in ("ECS", "Local"): + m = mr.get(slot) or {} + if m.get("run_id"): + K.stash_mrwp(st, slot, {k: m.get(k) for k in + ("run_id", "complete", "ran", "total", "failed_stages", + "yellow_stages", "never_ran", "tests", "failed_suites")}) C.save_state(st, args.runs_root, args.release) except Exception: pass diff --git a/release-agent/orchestrator/render.py b/release-agent/orchestrator/render.py index fe383f18..7e30208f 100644 --- a/release-agent/orchestrator/render.py +++ b/release-agent/orchestrator/render.py @@ -237,24 +237,31 @@ def attest_prompt_payload(chk: dict, release_id: str) -> dict: def _pipelines_line(r: dict) -> str: - """Compact one-line summary of the Phase-2 release-pipeline run ids recorded on - state (checker → orchestrator → the two MRWP runs). Empty string when none resolved - yet. Shared by the status view and the daily digest so both read from state (no live - az call in the render path).""" - pr = r.get("pipeline_runs") or {} + """Compact one-line summary of the Phase-2 release-pipeline runs recorded on state + (checker → orchestrator → the LATEST RC's two MRWP runs). Empty string when none + resolved yet. Reads the nested pipeline_runs schema (migrating a legacy flat shape); + no live az call in the render path.""" + from orchestrator.state import migrate_pipeline_runs + pr = migrate_pipeline_runs(r.get("pipeline_runs") or {}) parts = [] - if pr.get("checker"): - parts.append(f"checker {pr['checker']}") - if pr.get("orchestrator"): - v = f" ({pr['versions']})" if pr.get("versions") else "" - parts.append(f"orchestrator {pr['orchestrator']}{v}") + ch = pr.get("checker") or {} + if ch.get("run_id"): + parts.append(f"checker {ch['run_id']}") + o = pr.get("orchestrator") or {} + if o.get("run_id"): + v = o.get("versions") or {} + vstr = ", ".join(f"{k} {v[k]}" for k in ("Common", "Msal", "Broker") if v.get(k)) + parts.append(f"orchestrator {o['run_id']}" + (f" ({vstr})" if vstr else "")) + rcs = pr.get("rcs") or [] + rc = rcs[-1] if rcs else {} mr = [] - if pr.get("mrwp_ecs"): - mr.append(f"ECS {pr['mrwp_ecs']}") - if pr.get("mrwp_local"): - mr.append(f"Local {pr['mrwp_local']}") + if (rc.get("ecs") or {}).get("run_id"): + mr.append(f"ECS {rc['ecs']['run_id']}") + if (rc.get("local") or {}).get("run_id"): + mr.append(f"Local {rc['local']['run_id']}") if mr: - parts.append("MRWP " + " / ".join(mr)) + tag = f" (RC{rc['rc']})" if rc.get("rc") and len(rcs) > 1 else "" + parts.append("MRWP" + tag + " " + " / ".join(mr)) return " · ".join(parts) diff --git a/release-agent/orchestrator/state.py b/release-agent/orchestrator/state.py index 8e1301c5..5a9389e1 100644 --- a/release-agent/orchestrator/state.py +++ b/release-agent/orchestrator/state.py @@ -23,6 +23,54 @@ def _now() -> str: return datetime.now(timezone.utc).isoformat() +def _versions_str_to_dict(v) -> dict: + """'Common 24.6.0, Msal 8.4.2, Broker 16.5.0' -> {Common,Msal,Broker}. A dict passes + through; anything unparseable yields {}.""" + if isinstance(v, dict): + return {k: val for k, val in v.items() if val} + if not isinstance(v, str) or not v.strip(): + return {} + out = {} + for part in v.split(","): + toks = part.strip().split() + if len(toks) >= 2: + out[toks[0]] = toks[1] + return out + + +def migrate_pipeline_runs(pr) -> dict: + """Normalize the pipeline_runs container to the nested RC schema (idempotent). + + Accepts the legacy FLAT shape + {checker, orchestrator, versions, mrwp_ecs, mrwp_local, mrwp_id_source, resolved_at} + and lifts it to + {checker:{run_id,...}, orchestrator:{run_id,versions:{},...}, rcs:[{rc:1,ecs,local}]}. + A value already in the nested shape (has 'rcs', or a dict 'checker') is returned as-is. + Empty/None -> {}.""" + if not pr or not isinstance(pr, dict): + return {} + # Already nested? (rcs present, or checker is an object) + if "rcs" in pr or isinstance(pr.get("checker"), dict) or isinstance(pr.get("orchestrator"), dict): + return pr + out = {} + if pr.get("checker"): + out["checker"] = {"run_id": str(pr["checker"]), "resolved_at": pr.get("resolved_at")} + if pr.get("orchestrator"): + out["orchestrator"] = {"run_id": str(pr["orchestrator"]), + "versions": _versions_str_to_dict(pr.get("versions")), + "resolved_at": pr.get("resolved_at")} + ecs, local = pr.get("mrwp_ecs"), pr.get("mrwp_local") + if ecs or local: + rc = {"rc": 1, "resolved_at": pr.get("resolved_at")} + src = pr.get("mrwp_id_source") + if ecs: + rc["ecs"] = {"run_id": str(ecs), "id_source": src} + if local: + rc["local"] = {"run_id": str(local), "id_source": src} + out["rcs"] = [rc] + return out + + @dataclass class StepState: """Persisted state for a single step.""" @@ -88,7 +136,17 @@ class ReleaseState: # refreshed each time Phase 2 resolves the chain. Because a re-triggered 'Trigger RC # Testing' stage spawns NEW MRWP runs, these are re-resolved (newest wins) — not a # fixed cache. Surfaced in status details + the daily digest. - pipeline_runs: dict = field(default_factory=dict) # {checker, orchestrator, mrwp_ecs, mrwp_local, mrwp_id_source, resolved_at} + # Phase-2 release-pipeline runs — the RECORD of what verification resolved, reused by + # the RC report + gate (no re-discovery). Nested schema (see migrate_pipeline_runs): + # { checker: {run_id, when, resolved_at}, + # orchestrator: {run_id, versions:{Common,Msal,Broker}, parked, resolved_at}, + # rcs: [ {rc, ecs:{run_id,id_source,complete,ran,total,failed_stages, + # yellow_stages,never_ran,tests,failed_suites,resolved_at}, + # local:{...same...}, resolved_at} ] } + # There is exactly ONE checker + ONE orchestrator, but MULTIPLE RC iterations (each a + # re-trigger of RC Testing spawns a new ecs/local pair). The LATEST RC is rcs[-1] — the + # report + gate always use it. A per-provider id change appends a new rc entry. + pipeline_runs: dict = field(default_factory=dict) notes: list = field(default_factory=list) # ---- persistence ---- @@ -101,7 +159,11 @@ def load(cls, path: str) -> "ReleaseState": # unattended automation — an unexpected key must never hard-crash it. # Only keys matching a declared field are applied; the rest are dropped. known = {f.name for f in fields(cls)} - return cls(**{k: v for k, v in data.items() if k in known}) + obj = cls(**{k: v for k, v in data.items() if k in known}) + # Migrate the flat pre-nested pipeline_runs shape to the nested RC schema so + # older/hand-edited state files load cleanly and readers see one shape. + obj.pipeline_runs = migrate_pipeline_runs(obj.pipeline_runs) + return obj def save(self, path: str) -> None: self.updated_at = _now() diff --git a/release-agent/steps/build_verify/_common.py b/release-agent/steps/build_verify/_common.py index 629adf50..ce28237c 100644 --- a/release-agent/steps/build_verify/_common.py +++ b/release-agent/steps/build_verify/_common.py @@ -45,17 +45,64 @@ def links_for(build_id, name="ADO run"): return [{"name": name, "url": build_url(build_id)}] -def stash_runs(state, **ids): - """Record resolved pipeline run ids on state.pipeline_runs (drop None values), - stamped with resolved_at. Called each time Phase 2 resolves the chain so the ids - are in state (for status details + the digest). Re-resolved on every pass, so a - re-triggered MRWP run (new id) overwrites the old one.""" +def _now_iso(): from datetime import datetime, timezone - pr = dict(getattr(state, "pipeline_runs", {}) or {}) - for k, v in ids.items(): - if v is not None: - pr[k] = str(v) - pr["resolved_at"] = datetime.now(timezone.utc).isoformat() + return datetime.now(timezone.utc).isoformat() + + +def _pipeline_runs(state) -> dict: + """The nested pipeline_runs container on state (migrating a legacy flat shape).""" + from orchestrator.state import migrate_pipeline_runs + return migrate_pipeline_runs(getattr(state, "pipeline_runs", None) or {}) + + +def stash_checker(state, run_id, when=None): + """Record the (single) Code Complete Checker run that fired the release.""" + pr = _pipeline_runs(state) + pr["checker"] = {"run_id": str(run_id), "when": when, "resolved_at": _now_iso()} + state.pipeline_runs = pr + + +def stash_orchestrator(state, run_id, versions=None, parked=None): + """Record the (single) Release Orchestrator run + its RC versions and parked flag.""" + pr = _pipeline_runs(state) + pr["orchestrator"] = {"run_id": str(run_id), + "versions": versions or {}, + "parked": parked, + "resolved_at": _now_iso()} + state.pipeline_runs = pr + + +def latest_rc(state) -> dict: + """The current RC iteration (the last entry in rcs), or {} when none resolved yet.""" + rcs = _pipeline_runs(state).get("rcs") or [] + return rcs[-1] if rcs else {} + + +def stash_mrwp(state, provider, snapshot): + """Record an MRWP provider run's FULL verification snapshot into the current RC + iteration. `provider` is 'ECS' or 'Local'; `snapshot` carries run_id + stage/test + results (run_id, id_source, complete, ran, total, failed_stages, yellow_stages, + never_ran, tests, failed_suites). + + RC iterations are a list; the LATEST is rcs[-1]. When this provider's slot in the + current RC already holds a DIFFERENT run_id, RC Testing was re-triggered → a NEW rc + entry is appended (rc = last+1) and the snapshot lands there. Same id → idempotent + update. So ecs/local resolving in separate steps (any order) merge into one rc, and a + re-trigger rolls forward to the next rc.""" + key = provider.lower() # 'ecs' | 'local' + pr = _pipeline_runs(state) + rcs = pr.setdefault("rcs", []) + cur = rcs[-1] if rcs else None + existing = (cur or {}).get(key) or {} + if cur is None or (existing.get("run_id") and existing["run_id"] != str(snapshot.get("run_id"))): + cur = {"rc": (rcs[-1]["rc"] + 1) if rcs else 1} + rcs.append(cur) + snap = dict(snapshot) + snap["run_id"] = str(snapshot.get("run_id")) + snap["resolved_at"] = _now_iso() + cur[key] = snap + cur["resolved_at"] = _now_iso() state.pipeline_runs = pr @@ -67,11 +114,46 @@ def stash_runs(state, **ids): # ---------------------------------------------------------------- RC report email def rc_report_model(state, timeout=120): - """The full Phase-2 RC report model (checker → orchestrator → ECS/Local MRWP + - per-run test breakdown) for this release. Pure read; see tools.pipelines.""" - from tools import pipelines as P - return P.release_report(ORG, PROJECT, state.release_id, - checker_def=CHECKER_DEF, orch_def=ORCHESTRATOR_DEF, timeout=timeout) + """The Phase-2 RC report model — assembled from the RECORD in state.pipeline_runs + (the verification steps stored it), NOT a live re-discovery. Uses the LATEST RC + iteration (rcs[-1]). Shape mirrors tools.pipelines.release_report so the gate + email + builders consume it unchanged: + {release, checker{fired,run_id,when}, orchestrator{found,healthy,parked,run_id,versions}, + mrwp{ECS{...}, Local{...}}, problems[], rc} + """ + from orchestrator.state import migrate_pipeline_runs + pr = migrate_pipeline_runs(getattr(state, "pipeline_runs", None) or {}) + ch = pr.get("checker") or {} + o = pr.get("orchestrator") or {} + rcs = pr.get("rcs") or [] + rc = rcs[-1] if rcs else {} + + model = { + "release": state.release_id, + "checker": {"fired": bool(ch.get("run_id")), "run_id": ch.get("run_id"), + "when": ch.get("when")}, + "orchestrator": {"found": bool(o.get("run_id")), "healthy": True, + "run_id": o.get("run_id"), "versions": o.get("versions") or {}, + "parked": o.get("parked")}, + "mrwp": {}, "problems": [], "rc": rc.get("rc"), + } + for slot, prov in (("ecs", "ECS"), ("local", "Local")): + s = rc.get(slot) + if not s: + continue + model["mrwp"][prov] = { + "run_id": s.get("run_id"), "complete": s.get("complete"), + "ran": s.get("ran"), "total": s.get("total"), + "failed_stages": s.get("failed_stages") or [], + "yellow_stages": s.get("yellow_stages") or [], + "never_ran": s.get("never_ran") or [], + "tests": s.get("tests"), "failed_suites": s.get("failed_suites"), + } + if not s.get("complete") and s.get("never_ran"): + model["problems"].append( + f"MRWP {prov}: did NOT run to completion — never-ran: " + f"{', '.join(n for n in s['never_ran'] if n)}.") + return model def rc_run_links(model) -> list: @@ -481,7 +563,8 @@ def verify_mrwp(state, provider): # 3) test summary (best-effort — never blocks; red/yellow tests are triaged later) tests = mock_input("tests", MISSING) - if tests is MISSING: + tests_injected = tests is not MISSING + if not tests_injected: ok, tests, _ = P.get_test_summary(ORG, PROJECT, mid) if not ok: tests = None @@ -495,6 +578,25 @@ def verify_mrwp(state, provider): if comp["yellow"]: extras.append(f"{len(comp['yellow'])} yellow") extra = f" ({', '.join(extras)} — triaged later)" if extras else "" - stash_runs(state, **{f"mrwp_{provider.lower()}": mid}) + + # 4) failing suites (individual test names) — snapshot alongside the summary so the RC + # report + gate read everything from state (no re-discovery). Mockable via `suites`. + # Only fetch LIVE when the summary was read live (tests not injected) — an injected + # summary means an offline/test context, so we don't make the extra network call. + suites = mock_input("suites", MISSING) + if suites is MISSING: + suites = None + if not tests_injected and tests and tests.get("failed"): + okf, fsuites, _ = P.get_failed_tests(ORG, PROJECT, mid) + if okf: + suites = fsuites + + # 5) stash the FULL per-provider snapshot into the current RC iteration. + stash_mrwp(state, provider, { + "run_id": mid, "complete": comp["complete"], "ran": comp["ran"], + "total": comp["total"], "failed_stages": comp["failed"], + "yellow_stages": comp["yellow"], "never_ran": comp["never_ran"], + "tests": tests, "failed_suites": suites, + }) return Done( f"{label} run {mid} ran to completion — {stage_note}{extra}.{tnote}", links=links) diff --git a/release-agent/steps/build_verify/checker_fired.py b/release-agent/steps/build_verify/checker_fired.py index 53bde2f5..081e9b51 100644 --- a/release-agent/steps/build_verify/checker_fired.py +++ b/release-agent/steps/build_verify/checker_fired.py @@ -89,7 +89,7 @@ def _verdict(state, run, result, job): return Blocked( f"Code Complete Checker '{job}' did not succeed (result={result}) in run " f"{bid} ({when}) — the orchestrator was not launched.{K.UNBLOCK_HELP}", links=links) - K.stash_runs(state, checker=bid) + K.stash_checker(state, bid, when) return Done( f"Code Complete Checker fired the release — run {bid} ({when}), '{job}' succeeded.", links=links) diff --git a/release-agent/steps/build_verify/mrwp_ecs.py b/release-agent/steps/build_verify/mrwp_ecs.py index 4fb8d24e..1720466e 100644 --- a/release-agent/steps/build_verify/mrwp_ecs.py +++ b/release-agent/steps/build_verify/mrwp_ecs.py @@ -19,7 +19,8 @@ MOCKABLE = { "mrwp_id": {"kind": "input", "desc": "Inject the ECS MRWP build id (skip orchestrator lookup)."}, "stages": {"kind": "input", "desc": "Inject the ECS run's stage list [{name,state,result}]."}, - "tests": {"kind": "input", "desc": "Inject the ECS test summary {total,passed,failed}."}, + "tests": {"kind": "input", "desc": "Inject the ECS test summary {total,passed,failed,categories}."}, + "suites": {"kind": "input", "desc": "Inject the ECS failing suites [{name,failed,total,category,tests}]."}, } diff --git a/release-agent/steps/build_verify/mrwp_local.py b/release-agent/steps/build_verify/mrwp_local.py index befcc098..972b29be 100644 --- a/release-agent/steps/build_verify/mrwp_local.py +++ b/release-agent/steps/build_verify/mrwp_local.py @@ -17,7 +17,8 @@ MOCKABLE = { "mrwp_id": {"kind": "input", "desc": "Inject the Local MRWP build id (skip orchestrator lookup)."}, "stages": {"kind": "input", "desc": "Inject the Local run's stage list [{name,state,result}]."}, - "tests": {"kind": "input", "desc": "Inject the Local test summary {total,passed,failed}."}, + "tests": {"kind": "input", "desc": "Inject the Local test summary {total,passed,failed,categories}."}, + "suites": {"kind": "input", "desc": "Inject the Local failing suites [{name,failed,total,category,tests}]."}, } diff --git a/release-agent/steps/build_verify/orchestrator_health.py b/release-agent/steps/build_verify/orchestrator_health.py index 84393c9f..82a079c2 100644 --- a/release-agent/steps/build_verify/orchestrator_health.py +++ b/release-agent/steps/build_verify/orchestrator_health.py @@ -82,9 +82,11 @@ def build(state): # The park stage should be PENDING (waiting for the owner). If it already ran, the # gate was approved — surface it (out of the expected Phase-2 state), don't hard-fail. - K.stash_runs(state, orchestrator=bid, versions=vstr) park = by_name.get(cfg["park_stage"]) - if park is not None and park.get("state") == "completed": + park_done = bool(park is not None and park.get("state") == "completed") + K.stash_orchestrator(state, bid, versions={k: v for k, v in versions.items() if v}, + parked=not park_done) + if park_done: return Done( f"Release Orchestrator run {bid} healthy ({vstr}); NOTE '{cfg['park_stage']}' " f"already ran (result={park.get('result')}) — the approval gate was cleared. " diff --git a/release-agent/tests/test_engine.py b/release-agent/tests/test_engine.py index d942dd1b..2ee12a94 100644 --- a/release-agent/tests/test_engine.py +++ b/release-agent/tests/test_engine.py @@ -1951,6 +1951,26 @@ def _bv_build(orch, st, sid): return as_dict(_steps.get_step("build_verify", sid).build(st)) +def _seed_rc_pipeline(st, ecs_ui, local_ui, *, ecs_suites=None, + ecs_id="1678863", local_id="1678864"): + """Seed state.pipeline_runs with a full RC snapshot (checker + orchestrator + one RC + pair) the way the verify steps would, so rc_report / record-rc-report read it from + state (no live re-discovery). `ecs_ui`/`local_ui` are the UI category dicts the gate + consumes ({total,passed,failed}).""" + from steps.build_verify import _common as K + K.stash_checker(st, "1678599", "2026-08-13T06:00") + K.stash_orchestrator(st, "1678611", + versions={"Common": "24.6.0", "Msal": "8.4.2", "Broker": "16.5.0"}, + parked=True) + + def snap(run_id, ui, suites): + return {"run_id": run_id, "complete": True, "ran": 23, "total": 23, + "failed_stages": [], "yellow_stages": [], "never_ran": [], + "tests": {"categories": {"ui": ui}}, "failed_suites": suites or []} + K.stash_mrwp(st, "ECS", snap(ecs_id, ecs_ui, ecs_suites)) + K.stash_mrwp(st, "Local", snap(local_id, local_ui, None)) + + def test_build_verify_steps_pass_with_healthy_mocks(): """With injected healthy inputs, all four build_verify agent steps return done — and mrwp steps surface the test summary + red/yellow counts in the note.""" @@ -2016,50 +2036,49 @@ def test_build_verify_phase_shape(): def test_build_verify_rc_report_emails_owner(): """rc_report composes the RC report email to the release owner as a - NeedsSkill(workiq_send_email); blocks when no owner email is set. release_report is - monkeypatched so it's offline.""" + NeedsSkill(workiq_send_email) from the RECORD in state.pipeline_runs (no live call); + blocks when no owner email is set.""" from orchestrator.outcomes import as_dict - from tools import pipelines as P + from steps.build_verify import _common as K import steps as _steps - orig = P.release_report - P.release_report = lambda *a, **k: { - "release": "2026-08", "checker": {"fired": True, "run_id": 1678599}, - "orchestrator": {"found": True, "healthy": True, "parked": True, "run_id": 1678611, - "versions": {"Common": "24.6.0", "Msal": "8.4.2", "Broker": "16.5.0"}}, - "mrwp": {"ECS": {"run_id": 1678863, "complete": True, "ran": 23, "total": 23, - "failed_stages": ["UI Automation"], - "tests": {"total": 5871, "passed": 5767, "failed": 104, "runs": [], - "categories": { - "unit": {"total": 5248, "passed": 5248, "failed": 0}, - "instrumented": {"total": 442, "passed": 440, "failed": 2}, - "ui": {"total": 165, "passed": 63, "failed": 102}}}, - "failed_suites": [{"name": "PROD MSAL - RC Broker (API 32)", - "failed": 18, "total": 44, "category": "ui", - "tests": ["test_1_Foo", "test_2_Bar"]}]}, - "Local": {"run_id": 1678864, "complete": True, "ran": 23, "total": 23, - "failed_stages": [], "tests": {"total": 5856, "passed": 5756, "failed": 100, - "runs": [], "categories": {}}, - "failed_suites": []}}, - "problems": []} - try: - st = ReleaseState(release_id="2026-08", ccd="2026-08-26", - owner_email="dev@microsoft.com", owner_name="Dev") - out = as_dict(_steps.get_step("build_verify", "rc_report").build(st)) - assert out["kind"] == "needs_skill" and out["tool"] == "workiq_send_email" - assert out["payload"]["to"] == ["dev@microsoft.com"] and out["payload"]["isHtml"] - body = out["payload"]["body"] - assert "1678863" in body # run id present - assert "UI-automation failure rate" in body # per-category headline metric - assert "61.8%" in body # 102/165 UI failures — the real UI rate - assert "Unit" in body and "Instrumented" in body and "UI automation" in body - assert "test_1_Foo" in body # failing test names still listed - assert out["record_as"] == "rc_report" and out["outbound"] is True - # no owner → blocked - st2 = ReleaseState(release_id="2026-08", ccd="2026-08-26") - out2 = as_dict(_steps.get_step("build_verify", "rc_report").build(st2)) - assert out2["kind"] == "blocked" and "owner" in out2["reason"] - finally: - P.release_report = orig + + st = ReleaseState(release_id="2026-08", ccd="2026-08-26", + owner_email="dev@microsoft.com", owner_name="Dev") + # Seed the RC snapshot the verify steps would have stored (full categories + a suite). + K.stash_checker(st, "1678599", "2026-08-13T06:00") + K.stash_orchestrator(st, "1678611", + versions={"Common": "24.6.0", "Msal": "8.4.2", "Broker": "16.5.0"}, + parked=True) + K.stash_mrwp(st, "ECS", { + "run_id": "1678863", "complete": True, "ran": 23, "total": 23, + "failed_stages": ["UI Automation"], "yellow_stages": [], "never_ran": [], + "tests": {"total": 5871, "passed": 5767, "failed": 104, "categories": { + "unit": {"total": 5248, "passed": 5248, "failed": 0}, + "instrumented": {"total": 442, "passed": 440, "failed": 2}, + "ui": {"total": 165, "passed": 63, "failed": 102}}}, + "failed_suites": [{"name": "PROD MSAL - RC Broker (API 32)", "failed": 18, + "total": 44, "category": "ui", "tests": ["test_1_Foo", "test_2_Bar"]}]}) + K.stash_mrwp(st, "Local", { + "run_id": "1678864", "complete": True, "ran": 23, "total": 23, + "failed_stages": [], "yellow_stages": [], "never_ran": [], + "tests": {"total": 5856, "passed": 5756, "failed": 100, "categories": {}}, + "failed_suites": []}) + + out = as_dict(_steps.get_step("build_verify", "rc_report").build(st)) + assert out["kind"] == "needs_skill" and out["tool"] == "workiq_send_email" + assert out["payload"]["to"] == ["dev@microsoft.com"] and out["payload"]["isHtml"] + assert out["payload"]["followup_command"] == "record-rc-report" + body = out["payload"]["body"] + assert "1678863" in body # run id present + assert "UI-automation failure rate" in body # per-category headline metric + assert "61.8%" in body # 102/165 UI failures — the real UI rate + assert "Unit" in body and "Instrumented" in body and "UI automation" in body + assert "test_1_Foo" in body # failing test names still listed + assert out["record_as"] == "rc_report" and out["outbound"] is True + # no owner → blocked + st2 = ReleaseState(release_id="2026-08", ccd="2026-08-26") + out2 = as_dict(_steps.get_step("build_verify", "rc_report").build(st2)) + assert out2["kind"] == "blocked" and "owner" in out2["reason"] def test_rc_ui_gate_and_run_links(): @@ -2116,64 +2135,50 @@ def _model(ecs_ui, local_ui, ecs_suites=None): def test_record_rc_report_applies_ui_gate_and_stashes_links(): - """`record-rc-report` (the follow-up the skill runs after emailing) applies the 90% - UI gate: >=90% → step done; <90% → step BLOCKS (awaiting_action). Either way it - stashes the evaluated run links on the step. release_report is monkeypatched offline.""" + """`record-rc-report` (the follow-up the skill runs after emailing) reads the RC + snapshot from state and applies the 90% UI gate: >=90% → step done; <90% → step + BLOCKS (awaiting_action). Either way it stashes the evaluated run links on the step.""" import tempfile as _tf - from tools import pipelines as P from orchestrator.commands import rc_report as RR from orchestrator.state import StepState - def _model(ecs_ui, local_ui): - return {"release": "2026-08", - "checker": {"fired": True, "run_id": 111}, - "orchestrator": {"found": True, "run_id": 222}, - "mrwp": {"ECS": {"run_id": 333, "failed_suites": [], - "tests": {"categories": {"ui": ecs_ui}}}, - "Local": {"run_id": 444, "failed_suites": [], - "tests": {"categories": {"ui": local_ui}}}}, - "problems": []} - - orig = P.release_report with _tf.TemporaryDirectory() as d: rid = "2026-08" _stub_build_defs("pass") st = ReleaseState(release_id=rid, ccd="2026-08-26", owner_email="dev@microsoft.com") orch = Orchestrator(CONFIG, st) _pass_scout_checks(orch); orch.gate.sign() - C.save_state(st, d, rid) class A: runs_root = d; release = rid; config = CONFIG; as_of = None - try: - # PASS: 190/200 = 95% ≥ 90 → step done, links stashed - P.release_report = lambda *a, **k: _model( - {"total": 100, "passed": 95, "failed": 5}, - {"total": 100, "passed": 95, "failed": 5}) - assert RR.cmd_record_rc_report(A) == 0 - s1 = C.load_state(d, rid) - assert s1.is_done("build_verify", "rc_report") - step1 = s1.get_step("build_verify", "rc_report") - assert [l["name"] for l in step1.links] == [ - "Code Complete Checker run", "Release Orchestrator run", - "MRWP ECS run", "MRWP Local run"] - - # reset the step, then FAIL: 120/200 = 60% < 90 → blocked, links still stashed - s1.set_step("build_verify", "rc_report", StepState()) - C.save_state(s1, d, rid) - P.release_report = lambda *a, **k: _model( - {"total": 100, "passed": 60, "failed": 40}, - {"total": 100, "passed": 60, "failed": 40}) - assert RR.cmd_record_rc_report(A) == 2 - s2 = C.load_state(d, rid) - step2 = s2.get_step("build_verify", "rc_report") - assert step2.status == "blocked" and not s2.is_done("build_verify", "rc_report") - assert s2.status == "awaiting_action" - assert "build_verify.rc_report" in s2.pending_human - assert "BELOW" in step2.note and len(step2.links) == 4 - finally: - P.release_report = orig + # PASS: 190/200 = 95% ≥ 90 → step done, links stashed + _seed_rc_pipeline(st, {"total": 100, "passed": 95, "failed": 5}, + {"total": 100, "passed": 95, "failed": 5}) + C.save_state(st, d, rid) + assert RR.cmd_record_rc_report(A) == 0 + s1 = C.load_state(d, rid) + assert s1.is_done("build_verify", "rc_report") + step1 = s1.get_step("build_verify", "rc_report") + assert [l["name"] for l in step1.links] == [ + "Code Complete Checker run", "Release Orchestrator run", + "MRWP ECS run", "MRWP Local run"] + + # reset the step + re-seed the SAME runs with a failing UI slice (60% < 90) → + # blocked, links still stashed. Same run ids → updates the current rc in place. + s1.set_step("build_verify", "rc_report", StepState()) + _seed_rc_pipeline(s1, {"total": 100, "passed": 60, "failed": 40}, + {"total": 100, "passed": 60, "failed": 40}) + C.save_state(s1, d, rid) + assert RR.cmd_record_rc_report(A) == 2 + s2 = C.load_state(d, rid) + step2 = s2.get_step("build_verify", "rc_report") + assert step2.status == "blocked" and not s2.is_done("build_verify", "rc_report") + assert s2.status == "awaiting_action" + assert "build_verify.rc_report" in s2.pending_human + assert "BELOW" in step2.note and len(step2.links) == 4 + # the same rc was updated in place (not a spurious new RC iteration) + assert len(s2.pipeline_runs["rcs"]) == 1 def test_active_phase_report_steps_carry_links(): @@ -2334,22 +2339,65 @@ def test_mrwp_run_ids_picks_newest_on_retrigger(): def test_build_verify_persists_pipeline_run_ids(): - """The build_verify steps stash the resolved checker/orchestrator/MRWP run ids onto - state.pipeline_runs, and they round-trip through save/load (used by status + digest).""" + """The build_verify steps stash the checker/orchestrator/MRWP runs onto + state.pipeline_runs in the nested RC schema, and it round-trips through save/load.""" import tempfile from orchestrator import cli_common as _C st, orch = _bv_state({}) for sid in ("checker_fired", "orchestrator_health", "mrwp_ecs", "mrwp_local"): _bv_build(orch, st, sid) pr = st.pipeline_runs - assert pr.get("checker") == "1678599" - assert pr.get("orchestrator") == "1678611" - assert pr.get("mrwp_ecs") == "900001" and pr.get("mrwp_local") == "900002" - assert "Broker 1.0.0" in (pr.get("versions") or "") and pr.get("resolved_at") + assert pr["checker"]["run_id"] == "1678599" + assert pr["orchestrator"]["run_id"] == "1678611" + assert pr["orchestrator"]["versions"].get("Broker") == "1.0.0" + rc = pr["rcs"][-1] + assert rc["rc"] == 1 and rc.get("resolved_at") + assert rc["ecs"]["run_id"] == "900001" and rc["local"]["run_id"] == "900002" + assert rc["ecs"]["complete"] and rc["ecs"]["tests"]["failed"] == 4 # snapshot stored with tempfile.TemporaryDirectory() as tmp: _C.save_state(st, tmp, "2026-08") again = _C.load_state(tmp, "2026-08") - assert again.pipeline_runs.get("mrwp_ecs") == "900001" + assert again.pipeline_runs["rcs"][-1]["ecs"]["run_id"] == "900001" + + +def test_migrate_pipeline_runs_flat_to_nested(): + """A legacy FLAT pipeline_runs shape migrates to the nested RC schema on load + (idempotent); an already-nested value passes through unchanged.""" + from orchestrator.state import migrate_pipeline_runs as M + flat = {"checker": "111", "orchestrator": "222", + "versions": "Common 24.6.0, Msal 8.4.2, Broker 16.5.0", + "mrwp_ecs": "333", "mrwp_local": "444", "mrwp_id_source": "tags", + "resolved_at": "2026-08-20T00:00:00Z"} + m = M(flat) + assert m["checker"]["run_id"] == "111" + assert m["orchestrator"]["run_id"] == "222" + assert m["orchestrator"]["versions"] == {"Common": "24.6.0", "Msal": "8.4.2", "Broker": "16.5.0"} + assert m["rcs"] == [{"rc": 1, "resolved_at": "2026-08-20T00:00:00Z", + "ecs": {"run_id": "333", "id_source": "tags"}, + "local": {"run_id": "444", "id_source": "tags"}}] + assert M(m) == m # idempotent + assert M({}) == {} + + +def test_stash_mrwp_appends_new_rc_on_id_change(): + """stash_mrwp merges ecs+local into ONE rc entry, and appends a NEW rc iteration only + when a provider's run id changes (RC Testing re-triggered). Latest = rcs[-1].""" + from steps.build_verify import _common as K + st = ReleaseState(release_id="2026-08") + ecs1 = {"run_id": "900001", "complete": True, "tests": {"categories": {"ui": {"total": 10, "passed": 9, "failed": 1}}}} + K.stash_mrwp(st, "ECS", ecs1) + K.stash_mrwp(st, "Local", {"run_id": "900002", "complete": True}) + assert len(st.pipeline_runs["rcs"]) == 1 # both merged into rc 1 + assert K.latest_rc(st)["rc"] == 1 + # re-resolving the SAME ecs id updates in place — no new rc + K.stash_mrwp(st, "ECS", ecs1) + assert len(st.pipeline_runs["rcs"]) == 1 + # a NEW ecs id → RC re-triggered → append rc 2 + K.stash_mrwp(st, "ECS", {"run_id": "910001", "complete": True}) + K.stash_mrwp(st, "Local", {"run_id": "910002", "complete": True}) + rcs = st.pipeline_runs["rcs"] + assert [r["rc"] for r in rcs] == [1, 2] + assert K.latest_rc(st)["ecs"]["run_id"] == "910001" # latest = last def test_digest_shows_rc_line_when_build_verify_active(): @@ -2361,9 +2409,10 @@ def test_digest_shows_rc_line_when_build_verify_active(): "show_pipeline_runs": True, "due": True, "started": True, "done": 2, "total": 5, "outstanding": [], "completed": ["checker_fired", "orchestrator_health"]}, - "pipeline_runs": {"checker": "1678599", "orchestrator": "1678611", - "versions": "Broker 1.0.0", "mrwp_ecs": "900001", - "mrwp_local": "900002"}} + "pipeline_runs": { + "checker": {"run_id": "1678599"}, + "orchestrator": {"run_id": "1678611", "versions": {"Broker": "1.0.0"}}, + "rcs": [{"rc": 1, "ecs": {"run_id": "900001"}, "local": {"run_id": "900002"}}]}} text = render.notification(r) md = render.notification_markdown(r) assert "RC pipelines:" in text and "orchestrator 1678611" in text @@ -2393,9 +2442,9 @@ def test_sim_fast_forwards_to_rc_gate_offline(): assert st.is_done("build_verify", s), s from orchestrator.engine import Orchestrator as _O assert _O(CONFIG, st).current_phase_id() == "bug_bash" # positioned past Phase 2 - # pipeline ids were stashed by the steps during the sim - assert st.pipeline_runs.get("orchestrator") == "1678611" - assert st.pipeline_runs.get("mrwp_ecs") == "1678863" + # pipeline runs were stashed by the steps during the sim (nested RC schema) + assert st.pipeline_runs["orchestrator"]["run_id"] == "1678611" + assert st.pipeline_runs["rcs"][-1]["ecs"]["run_id"] == "1678863" assert st.readiness_signed From aa0b2ba4fa135efd97486c34c06ef31b58f37a28 Mon Sep 17 00:00:00 2001 From: p3dr0rv Date: Thu, 20 Aug 2026 21:44:16 +0100 Subject: [PATCH 68/82] release-agent: apply the unit-test retry rule in the RC report (recovered = passed + warn) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Unit tests run under a retry rule: a flaky test can appear several times in one run (e.g. Passed/Failed/Passed) and ADO's run aggregate still counts it as a failure. Reconcile per-result by testCaseTitle so a test that passed on any attempt counts as PASSED; a failed-then-passed test is RECOVERED (counted passed, surfaced as a warning); only never-passed tests are real failures. Unit-only — UI/instrumented (and the UI gate) are unchanged. reconcile_retries() + a paged per-run results fetch; get_test_summary re-reads failing UNIT runs per-result; get_failed_tests drops recovered unit titles (and fully-recovered suites); the RC email (plain+HTML) and rc-report diagnostic show a retry warning listing recovered unit tests. Also fixes retry double-counting in totals. Verified live on build 1681651: testNullDrsMetadata + gated_notEnabled_logsNothingAtAll reconcile to passed, unit failed=0. 187/187 tests pass. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../orchestrator/commands/rc_report.py | 11 ++ release-agent/steps/build_verify/_common.py | 36 +++++ release-agent/tests/test_engine.py | 85 ++++++++++++ release-agent/tools/pipelines.py | 126 ++++++++++++++++-- 4 files changed, 247 insertions(+), 11 deletions(-) diff --git a/release-agent/orchestrator/commands/rc_report.py b/release-agent/orchestrator/commands/rc_report.py index 5e9b1908..9339985f 100644 --- a/release-agent/orchestrator/commands/rc_report.py +++ b/release-agent/orchestrator/commands/rc_report.py @@ -177,6 +177,17 @@ def _format(m) -> str: L += ["", "**Issues:**"] for p in probs: L.append(f" - {p}") + # Unit retry warning — failed-then-passed on retry (counted as passed). + recovered = sorted({t for prov in ("ECS", "Local") + for t in ((((m.get("mrwp") or {}).get(prov) or {}).get("tests") or {}) + .get("categories", {}).get("unit", {}).get("recovered") or [])}) + if recovered: + L += ["", f"⚠ **Retry warning** — {len(recovered)} unit test(s) failed then passed " + f"on retry (counted as passed):"] + for t in recovered[:20]: + L.append(f" - {t}") + if len(recovered) > 20: + L.append(f" … and {len(recovered) - 20} more") return "\n".join(L) diff --git a/release-agent/steps/build_verify/_common.py b/release-agent/steps/build_verify/_common.py index ce28237c..56c98a1d 100644 --- a/release-agent/steps/build_verify/_common.py +++ b/release-agent/steps/build_verify/_common.py @@ -238,6 +238,19 @@ def rc_ui_gate(model) -> dict: f"to override." + _ui_failing_suites_summary(model))} +def recovered_unit_tests(model) -> list: + """Unit tests that FAILED then PASSED on retry (the unit retry rule) across both MRWP + providers — counted as passed, but surfaced as a warning in the report. De-duplicated, + sorted.""" + out = set() + for prov in ("ECS", "Local"): + cats = (((model.get("mrwp") or {}).get(prov) or {}).get("tests") or {}) \ + .get("categories", {}) + for t in ((cats.get("unit") or {}).get("recovered") or []): + out.add(t) + return sorted(out) + + def rc_email_subject(model) -> str: rid = model.get("release", "?") v = rc_ui_gate(model)["verdict"] @@ -316,6 +329,14 @@ def _rc_email_plain(model, ctx) -> str: L.append("BLOCKING ISSUES (a stage that never ran = the pipeline aborted):") L += [f" - {p}" for p in probs] L.append("") + recovered = recovered_unit_tests(model) + if recovered: + L.append(f"\u26a0 RETRY WARNING — {len(recovered)} unit test(s) FAILED then PASSED " + f"on retry (counted as passed; verify they aren't genuinely flaky):") + L += [f" - {t}" for t in recovered[:20]] + if len(recovered) > 20: + L.append(f" … and {len(recovered) - 20} more") + L.append("") L.append("NEXT: review the failing tests above. If they're acceptable to carry into " "bug bash, approve the gate (advances to Phase 3 — Test / Bug Bash). " "Otherwise investigate the red suites first.") @@ -448,6 +469,20 @@ def _ui_sum(field): + "".join(f"

  • {T.esc(p)}
    • " for p in probs) + "
    ") if probs else "") + recovered = recovered_unit_tests(model) + retry_warn = "" + if recovered: + shown = recovered[:15] + more = (f"
  • … and " + f"{len(recovered) - len(shown)} more
    • " if len(recovered) > len(shown) else "") + retry_warn = ( + "
    ⚠ Retry warning — " + f"{len(recovered)} unit test(s) failed then passed on retry (counted as " + "passed; verify they aren’t genuinely flaky):" + "
      " + + "".join(f"
    • {T.esc(t)}
      • " for t in shown) + more + "
    ") + return f"""\
    @@ -484,6 +519,7 @@ def _ui_sum(field): {mrwp_card('ECS')} {mrwp_card('Local')} {issues} + {retry_warn}
    diff --git a/release-agent/tests/test_engine.py b/release-agent/tests/test_engine.py index 2ee12a94..bb8fc3fa 100644 --- a/release-agent/tests/test_engine.py +++ b/release-agent/tests/test_engine.py @@ -2189,6 +2189,91 @@ def test_active_phase_report_steps_carry_links(): assert ap and all("links" in s for s in ap["steps"]) +def test_reconcile_retries_pure(): + """reconcile_retries collapses per-attempt results by title: passed if any attempt + passed; recovered if it also failed; failed only if it never passed; NA ignored.""" + from tools import pipelines as P + res = [ + {"testCaseTitle": "testNullDrsMetadata", "outcome": "Passed"}, + {"testCaseTitle": "testNullDrsMetadata", "outcome": "Failed"}, + {"testCaseTitle": "testNullDrsMetadata", "outcome": "Passed"}, # flaky → recovered + {"testCaseTitle": "testAlwaysGreen", "outcome": "Passed"}, + {"testCaseTitle": "testHardFail", "outcome": "Failed"}, + {"testCaseTitle": "testHardFail", "outcome": "Failed"}, # failed every attempt + {"testCaseTitle": "testSkipped", "outcome": "NotExecuted"}, # ignored + ] + r = P.reconcile_retries(res) + assert r["passed"] == 2 and r["failed"] == 1 + assert r["recovered"] == ["testNullDrsMetadata"] + assert r["total"] == 3 and r["na"] == 1 + assert P.reconcile_retries([]) == {"passed": 0, "failed": 0, "recovered": [], "total": 0, "na": 0} + + +def test_get_test_summary_unit_retry_reconciles(): + """A UNIT run whose only failure is a flaky test that passed on retry reconciles to + 0 failed + a `recovered` warning — even though ADO's run aggregate said 1 failed. + (The rule is unit-only; UI/instrumented use the aggregate unchanged.)""" + from tools import pipelines as P + runs = {"value": [{"id": 700, "name": "broker4j_UnitTests", + "totalTests": 5, "passedTests": 4, "notApplicableTests": 0}]} + results = {"value": [ + {"testCaseTitle": "testNullDrsMetadata", "outcome": "Passed"}, + {"testCaseTitle": "testNullDrsMetadata", "outcome": "Failed"}, + {"testCaseTitle": "testNullDrsMetadata", "outcome": "Passed"}, + {"testCaseTitle": "t2", "outcome": "Passed"}, + {"testCaseTitle": "t3", "outcome": "Passed"}, + {"testCaseTitle": "t4", "outcome": "Passed"}, + ]} + orig = P._ado_rest_get + P._ado_rest_get = lambda url, timeout: (True, runs if "buildUri" in url else results, "") + try: + ok, s, _ = P.get_test_summary("O", "P", 700) + assert ok + unit = s["categories"]["unit"] + assert unit["failed"] == 0 and unit["recovered"] == ["testNullDrsMetadata"] + assert unit["passed"] == 4 and unit["total"] == 4 + finally: + P._ado_rest_get = orig + + +def test_get_failed_tests_drops_fully_recovered_unit_suite(): + """A unit suite whose only failure recovered on retry produces NO failing suite.""" + from tools import pipelines as P + runs = {"value": [{"id": 701, "name": "broker4j_UnitTests # 700_build.1", + "totalTests": 5, "passedTests": 4, "notApplicableTests": 0}]} + results = {"value": [ + {"testCaseTitle": "flaky", "outcome": "Failed"}, + {"testCaseTitle": "flaky", "outcome": "Passed"}, + ]} + orig = P._ado_rest_get + P._ado_rest_get = lambda url, timeout: (True, runs if "buildUri" in url else results, "") + try: + ok, suites, _ = P.get_failed_tests("O", "P", 701) + assert ok and suites == [] # the only failure recovered → no failing suite + finally: + P._ado_rest_get = orig + + +def test_rc_report_email_shows_retry_warning(): + """When a unit test recovered on retry, the RC report (plain + HTML) surfaces a retry + warning that lists it (counted as passed but flagged).""" + from steps.build_verify import _common as K + model = {"release": "2026-08", "checker": {"run_id": 1}, + "orchestrator": {"run_id": 2, "versions": {}, "parked": True}, + "mrwp": {"ECS": {"run_id": 3, "ran": 23, "total": 23, + "tests": {"categories": { + "unit": {"total": 100, "passed": 100, "failed": 0, + "recovered": ["testNullDrsMetadata"]}, + "ui": {"total": 10, "passed": 10, "failed": 0}}}}, + "Local": {"run_id": 4, "ran": 23, "total": 23, "tests": {"categories": {}}}}, + "problems": []} + assert K.recovered_unit_tests(model) == ["testNullDrsMetadata"] + plain = K._rc_email_plain(model, {}) + html = K._rc_email_html(model, {}) + assert "RETRY WARNING" in plain and "testNullDrsMetadata" in plain + assert "Retry warning" in html and "testNullDrsMetadata" in html + + def test_classify_test_run_categories(): """The test-run classifier buckets into exactly three: unit / instrumented / ui; anything that isn't unit/instrumented is UI ('the rest are UI', incl. Lab Api Tests).""" diff --git a/release-agent/tools/pipelines.py b/release-agent/tools/pipelines.py index 645f9510..961572eb 100644 --- a/release-agent/tools/pipelines.py +++ b/release-agent/tools/pipelines.py @@ -317,11 +317,73 @@ def classify_test_run(name): return "ui" +# Outcomes that are neither a pass nor a real failure (skipped / not run / inconclusive). +_NA_OUTCOMES = {"NotExecuted", "NotApplicable", "None", "Inconclusive", "Warning", None} + + +def reconcile_retries(results): + """Collapse ADO's per-attempt test results into ONE verdict per test (by title). + + UNIT tests run under a RETRY rule: a flaky test can appear several times in the same + run — e.g. Passed, Failed, Passed. ADO's run aggregate still counts that as a failure, + but the test ultimately PASSED. This groups results by testCaseTitle and rules: + * PASSED — at least one attempt Passed. + * RECOVERED — Passed AND Failed on different attempts (a flaky pass — surfaced as a + warning, but counted as passed). + * FAILED — has a real (non-NA) attempt and NEVER passed. + Not-executed / not-applicable attempts are ignored. Counts are DISTINCT tests. Returns + {passed, failed, recovered:[titles], total, na}.""" + import collections + by = collections.defaultdict(set) + for r in results or []: + title = (r.get("testCaseTitle") or r.get("automatedTestName") or "").strip() + if not title: + continue + by[title].add(r.get("outcome")) + passed = failed = na = 0 + recovered = [] + for title, outs in by.items(): + eff = {o for o in outs if o not in _NA_OUTCOMES} + if not eff: + na += 1 + continue + if "Passed" in eff: + passed += 1 + if "Failed" in eff: + recovered.append(title) + else: + failed += 1 + return {"passed": passed, "failed": failed, "recovered": sorted(recovered), + "total": passed + failed, "na": na} + + +def _run_results(org, project, run_id, timeout=90, page=1000, cap=10000): + """All test results for a run (paged). Returns (ok, [results], detail).""" + base = org.rstrip("/") + out, skip = [], 0 + while len(out) < cap: + url = (f"{base}/{project}/_apis/test/Runs/{run_id}/results" + f"?api-version=7.1&$top={page}&$skip={skip}") + ok, data, detail = _ado_rest_get(url, timeout) + if not ok: + return (False, out, detail) + batch = (data or {}).get("value", []) or [] + out.extend(batch) + if len(batch) < page: + break + skip += page + return (True, out, "") + + def get_test_summary(org, project, build_id, timeout=60): """Return (ok, summary, detail) for a build's Test-tab results. summary = - {total, passed, failed, runs:[{name,total,passed,failed,category}], - categories:{unit|instrumented|ui|other: {total,passed,failed}}} aggregated across - all test runs, classified into unit / instrumented / UI-automation / other. + {total, passed, failed, runs:[{name,total,passed,failed,category,recovered}], + categories:{unit|instrumented|ui: {total,passed,failed,recovered:[titles]}}} + aggregated across all test runs, classified into unit / instrumented / UI-automation. + + UNIT runs apply the retry rule (reconcile_retries): a run WITH failures is re-read at + the per-result level so a flaky test that failed-then-passed counts as passed (and is + reported as `recovered`). UI/instrumented runs use ADO's run aggregate unchanged. Uses the Test Runs REST API directly (az devops invoke mis-routes this one).""" base = org.rstrip("/") @@ -332,21 +394,31 @@ def get_test_summary(org, project, build_id, timeout=60): return (False, None, detail) runs = (data or {}).get("value", []) or [] out_runs, tot, passed = [], 0, 0 - cats = {c: {"total": 0, "passed": 0, "failed": 0} for c in TEST_CATEGORIES} + cats = {c: {"total": 0, "passed": 0, "failed": 0, "recovered": []} for c in TEST_CATEGORIES} for r in runs: t = r.get("totalTests") or 0 p = r.get("passedTests") or 0 na = r.get("notApplicableTests") or 0 f = max(t - p - na, 0) cat = classify_test_run(r.get("name")) + recovered = [] + # UNIT retry rule: re-read a failing unit run per-result and reconcile flaky passes. + if cat == "unit" and f > 0: + ok2, results, _ = _run_results(org, project, r.get("id"), timeout) + if ok2 and results: + rec = reconcile_retries(results) + t, p, f, recovered = rec["total"], rec["passed"], rec["failed"], rec["recovered"] tot += t passed += p cats[cat]["total"] += t cats[cat]["passed"] += p cats[cat]["failed"] += f + if recovered: + cats[cat]["recovered"].extend(recovered) out_runs.append({"name": r.get("name"), "total": t, "passed": p, - "failed": f, "category": cat}) - return (True, {"total": tot, "passed": passed, "failed": max(tot - passed, 0), + "failed": f, "category": cat, "recovered": recovered}) + failed_total = sum(c["failed"] for c in cats.values()) + return (True, {"total": tot, "passed": passed, "failed": failed_total, "runs": out_runs, "categories": cats}, "") @@ -359,9 +431,14 @@ def _suite_base_name(name): def get_failed_tests(org, project, build_id, max_result_calls=20, per_suite_cap=40, timeout=90): """Return (ok, suites, detail) — the individual FAILING tests for a build, aggregated by suite name (the same suite appears as multiple runs; merged). suites is a list of - {name, failed, total, tests:[test titles]}, sorted by failure count desc. Test titles - are fetched for the worst runs first, bounded by max_result_calls; per suite capped at - per_suite_cap names.""" + {name, failed, total, category, tests:[titles], recovered:[titles]}, sorted by failure + count desc. Test titles are fetched for the worst runs first, bounded by + max_result_calls; per suite capped at per_suite_cap names. + + UNIT suites apply the retry rule: a failing unit run is re-read per-result and + reconciled (reconcile_retries), so a flaky test that failed-then-passed is NOT listed + as a failure — it's collected under `recovered` and a suite that fully recovers is + dropped.""" base = org.rstrip("/") url = (f"{base}/{project}/_apis/test/runs" f"?buildUri=vstfs:///Build/Build/{build_id}&api-version=7.1") @@ -378,8 +455,33 @@ def fcount(r): suites, calls = {}, 0 for r in failing: name = _suite_base_name(r.get("name")) + cat = classify_test_run(name) s = suites.setdefault(name, {"name": name, "failed": 0, "total": 0, - "category": classify_test_run(name), "tests": []}) + "category": cat, "tests": [], "recovered": []}) + # UNIT retry rule: reconcile per-result so flaky-recovered tests aren't failures. + if cat == "unit": + ok2, results, _ = _run_results(org, project, r.get("id"), timeout) + if ok2: + rec = reconcile_retries(results) + s["failed"] += rec["failed"] + s["total"] += rec["total"] + for t in rec["recovered"]: + if t not in s["recovered"]: + s["recovered"].append(t) + # only the tests that truly failed (never passed) — reconcile again for names + import collections + by = collections.defaultdict(set) + for res in results: + title = (res.get("testCaseTitle") or res.get("automatedTestName") or "").strip() + if title: + by[title].add(res.get("outcome")) + for title, outs in by.items(): + eff = {o for o in outs if o not in _NA_OUTCOMES} + if eff and "Passed" not in eff and title not in s["tests"] \ + and len(s["tests"]) < per_suite_cap: + s["tests"].append(title) + continue + # NON-unit (UI / instrumented) — ADO aggregate + the failed titles (unchanged). s["failed"] += fcount(r) s["total"] += r.get("totalTests") or 0 if calls < max_result_calls: @@ -392,7 +494,9 @@ def fcount(r): title = (res.get("testCaseTitle") or res.get("automatedTestName") or "").strip() if title and title not in s["tests"] and len(s["tests"]) < per_suite_cap: s["tests"].append(title) - return (True, sorted(suites.values(), key=lambda x: -x["failed"]), "") + # Drop suites whose failures all recovered on retry (unit); keep real failures. + real = [s for s in suites.values() if s["failed"] > 0] + return (True, sorted(real, key=lambda x: -x["failed"]), "") # Coordinates for the release chain (identitydivision/Engineering). The build_verify From 82c2cc30020f3de5bbde2d65fa6c2845dd4dc715 Mon Sep 17 00:00:00 2001 From: p3dr0rv Date: Thu, 20 Aug 2026 22:03:33 +0100 Subject: [PATCH 69/82] =?UTF-8?q?release-agent:=20arch=20review=20batch=20?= =?UTF-8?q?1=20=E2=80=94=20single-source=20ADO=20coords,=20one=20RC-model?= =?UTF-8?q?=20builder,=20test=20net-guard?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit H2 (single-source coordinates): the release toolchain spans MULTIPLE ADO orgs (identitydivision/Engineering for the verification chain, msazure/One for localization+CG, identitydivision/IdentityWiki for the wiki). tools/pipelines.py now names them explicitly (IDENTITYDIVISION/MSAZURE hosts, ENGINEERING_ORG/ENGINEERING_PROJECT) instead of a misleading global ORG/PROJECT; build_verify (Engineering-only) aliases them, cron/wiki import the right target, localization/cg keep their own. H1 (one RC-model builder): assemble_rc_model() is the single canonical builder; release_report (live) and rc_report_model (state) both route through it so they can't drift on shape or problem messages. Guard test asserts agreement; live-verified on 2026-08. L2 (test net-guard): tests/conftest.py autouse fixture makes any un-mocked ADO/az call raise loudly instead of hanging. 188 tests pass. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- release-agent/steps/build_verify/_common.py | 60 ++++---- release-agent/steps/preflight/cron.py | 7 +- release-agent/steps/preflight/wiki.py | 6 +- release-agent/tests/conftest.py | 38 +++++ release-agent/tests/test_engine.py | 26 ++++ release-agent/tools/pipelines.py | 145 +++++++++++++------- 6 files changed, 188 insertions(+), 94 deletions(-) create mode 100644 release-agent/tests/conftest.py diff --git a/release-agent/steps/build_verify/_common.py b/release-agent/steps/build_verify/_common.py index 56c98a1d..7dcec4a8 100644 --- a/release-agent/steps/build_verify/_common.py +++ b/release-agent/steps/build_verify/_common.py @@ -1,26 +1,21 @@ """Shared config + helpers for the Phase 2 (build_verify) release-verification steps. -Underscore-prefixed so steps.discover() skips it (it's not a step). Holds the ADO -coordinates for the three Engineering release pipelines and the recovery / escalation -links surfaced when a step blocks, plus small resolvers the step modules reuse. +Underscore-prefixed so steps.discover() skips it (it's not a step). This whole package is +scoped to ONE ADO target — identitydivision/Engineering (the release-verification chain) — +so the local `ORG`/`PROJECT` names here are unambiguous. Their VALUES are imported from +`tools.pipelines` (the single source; other areas like localization use a DIFFERENT org), +so a coordinate change happens in exactly one place. This module adds the recovery / +escalation links surfaced when a step blocks, plus small resolvers the step modules reuse. """ from __future__ import annotations -ORG = "https://identitydivision.visualstudio.com" -PROJECT = "Engineering" - -CHECKER_DEF = 3038 # Code Complete Calendar Checker (fires the release on the CCD) -ORCHESTRATOR_DEF = 2828 # Release Orchestrator (the spine) -MRWP_DEF = 2519 # Monthly Release Work Pipeline (RC testing; runs ECS + Local) - -# The orchestrator stages that must be green before RC testing is trustworthy, and the -# stage it should be PARKED at (a human approval gate the owner clears in a later phase). -ORCH_REQUIRED_STAGES = [ - "Validate Branch and Versions availability", - "Create Release Branches", - "Trigger RC Testing", -] -ORCH_PARK_STAGE = "Remove RC Tags" +# This package is Engineering-only; alias the explicitly-named source constants to the +# short local names the step CONFIGs use. (Do NOT use these for msazure/One calls.) +from tools.pipelines import ( + ENGINEERING_ORG as ORG, ENGINEERING_PROJECT as PROJECT, + CHECKER_DEF, ORCHESTRATOR_DEF, MRWP_DEF, + ORCH_REQUIRED_STAGES, ORCH_PARK_STAGE, +) # Surfaced in every block reason so the engineer knows how to recover / escalate. RECOVERY_TSG = ("https://eng.ms/docs/microsoft-security/identity/" @@ -116,11 +111,10 @@ def stash_mrwp(state, provider, snapshot): def rc_report_model(state, timeout=120): """The Phase-2 RC report model — assembled from the RECORD in state.pipeline_runs (the verification steps stored it), NOT a live re-discovery. Uses the LATEST RC - iteration (rcs[-1]). Shape mirrors tools.pipelines.release_report so the gate + email - builders consume it unchanged: - {release, checker{fired,run_id,when}, orchestrator{found,healthy,parked,run_id,versions}, - mrwp{ECS{...}, Local{...}}, problems[], rc} + iteration (rcs[-1]) and routes through `tools.pipelines.assemble_rc_model` — the SAME + assembler the live path uses — so the state-based model can't drift from the live one. """ + from tools import pipelines as P from orchestrator.state import migrate_pipeline_runs pr = migrate_pipeline_runs(getattr(state, "pipeline_runs", None) or {}) ch = pr.get("checker") or {} @@ -128,20 +122,16 @@ def rc_report_model(state, timeout=120): rcs = pr.get("rcs") or [] rc = rcs[-1] if rcs else {} - model = { - "release": state.release_id, - "checker": {"fired": bool(ch.get("run_id")), "run_id": ch.get("run_id"), - "when": ch.get("when")}, - "orchestrator": {"found": bool(o.get("run_id")), "healthy": True, - "run_id": o.get("run_id"), "versions": o.get("versions") or {}, - "parked": o.get("parked")}, - "mrwp": {}, "problems": [], "rc": rc.get("rc"), - } + checker = {"fired": bool(ch.get("run_id")), "run_id": ch.get("run_id"), "when": ch.get("when")} + orchestrator = {"found": bool(o.get("run_id")), "healthy": True, + "run_id": o.get("run_id"), "versions": o.get("versions") or {}, + "parked": o.get("parked")} + mrwp = {} for slot, prov in (("ecs", "ECS"), ("local", "Local")): s = rc.get(slot) if not s: continue - model["mrwp"][prov] = { + mrwp[prov] = { "run_id": s.get("run_id"), "complete": s.get("complete"), "ran": s.get("ran"), "total": s.get("total"), "failed_stages": s.get("failed_stages") or [], @@ -149,11 +139,7 @@ def rc_report_model(state, timeout=120): "never_ran": s.get("never_ran") or [], "tests": s.get("tests"), "failed_suites": s.get("failed_suites"), } - if not s.get("complete") and s.get("never_ran"): - model["problems"].append( - f"MRWP {prov}: did NOT run to completion — never-ran: " - f"{', '.join(n for n in s['never_ran'] if n)}.") - return model + return P.assemble_rc_model(state.release_id, checker, orchestrator, mrwp, rc=rc.get("rc")) def rc_run_links(model) -> list: diff --git a/release-agent/steps/preflight/cron.py b/release-agent/steps/preflight/cron.py index edd482c2..a3ace5a8 100644 --- a/release-agent/steps/preflight/cron.py +++ b/release-agent/steps/preflight/cron.py @@ -10,16 +10,17 @@ from orchestrator.outcomes import Done, Blocked from steps.lib.agent import legacy_run from steps.lib.mockctx import mock_input, MISSING +from tools.pipelines import ENGINEERING_ORG, ENGINEERING_PROJECT ID = "cron" KIND = "agent" # Step config (co-located). Pipeline 3038's cron proves it's FIRING via a recent -# schedule-reason run in its build history. +# schedule-reason run in its build history. It's the Engineering Calendar Checker. CONFIG = { "pipeline_id": 3038, - "org": "https://identitydivision.visualstudio.com", - "project": "Engineering", + "org": ENGINEERING_ORG, + "project": ENGINEERING_PROJECT, "name": "Code Complete Calendar Checker", "max_staleness_days": 2, # a daily cron should never be older than this } diff --git a/release-agent/steps/preflight/wiki.py b/release-agent/steps/preflight/wiki.py index cca38f16..4648d0cb 100644 --- a/release-agent/steps/preflight/wiki.py +++ b/release-agent/steps/preflight/wiki.py @@ -10,14 +10,16 @@ from orchestrator.outcomes import Done, Blocked from steps.lib.agent import legacy_run from steps.lib.mockctx import mock_input +from tools.pipelines import IDENTITYDIVISION ID = "wiki" KIND = "agent" # Step config (co-located). Creates the per-release payload page under the standing -# history parent page (Phase 2 later writes the built versions into it). +# history parent page (Phase 2 later writes the built versions into it). NOTE: this is a +# DIFFERENT project (IdentityWiki) from the release chain — only the org host is shared. CONFIG = { - "org": "https://identitydivision.visualstudio.com", + "org": IDENTITYDIVISION, # same collection host as Engineering, different project "project": "IdentityWiki", "wiki": "IdentityWiki.wiki", "parent_path": "/IdentityWiki/Services/Microsoft Authenticator/Release/Android/Monthly Releases Payloads History", diff --git a/release-agent/tests/conftest.py b/release-agent/tests/conftest.py new file mode 100644 index 00000000..b741df60 --- /dev/null +++ b/release-agent/tests/conftest.py @@ -0,0 +1,38 @@ +"""Shared pytest setup for the Release Orchestrator tests. + +Adds the package root to sys.path (tests import `orchestrator.*` / `tools.*` / `steps.*`) +and installs an AUTOUSE network guard: the real ADO/az primitives in `tools.pipelines` +are replaced with a raiser, so any test that reaches a live network call FAILS LOUDLY +with a clear message instead of hanging on `az`. Tests that need controlled responses +monkeypatch these primitives themselves (e.g. `P._ado_rest_get = fake`), which overrides +the guard for the duration of that test. +""" +from __future__ import annotations + +import os +import sys + +import pytest + +ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) # release-agent/ +if ROOT not in sys.path: + sys.path.insert(0, ROOT) + + +@pytest.fixture(autouse=True) +def _no_real_network(monkeypatch): + """Block real ADO/az calls in tests. Any un-mocked network access raises with a hint + naming what to patch — this is what turns an accidental live call into a fast, clear + failure instead of a multi-minute hang.""" + from tools import pipelines as P + + def _blocked(*_a, **_k): + raise RuntimeError( + "test attempted a REAL ADO/az network call — mock it " + "(patch tools.pipelines._ado_rest_get / _ado_rest_get_text / _az_json, " + "or inject the step's input mocks).") + + monkeypatch.setattr(P, "_ado_rest_get", _blocked) + monkeypatch.setattr(P, "_ado_rest_get_text", _blocked) + monkeypatch.setattr(P, "_az_json", _blocked) + yield diff --git a/release-agent/tests/test_engine.py b/release-agent/tests/test_engine.py index bb8fc3fa..41add017 100644 --- a/release-agent/tests/test_engine.py +++ b/release-agent/tests/test_engine.py @@ -2274,6 +2274,32 @@ def test_rc_report_email_shows_retry_warning(): assert "Retry warning" in html and "testNullDrsMetadata" in html +def test_rc_model_shape_agrees_across_live_and_state_paths(): + """H1 guard: the live builder (pipelines.release_report → assemble_rc_model) and the + state builder (steps._common.rc_report_model → assemble_rc_model) produce the SAME + top-level model shape, so the gate/email/diagnostic never drift.""" + from tools import pipelines as P + from steps.build_verify import _common as K + # a canonical assembled model has exactly these top-level keys + m = P.assemble_rc_model("2026-08", {"fired": True, "run_id": 1}, + {"found": True, "healthy": True, "run_id": 2, "versions": {}}, + {"ECS": {"run_id": 3, "complete": True}}, rc=1, id_source="tags") + assert set(m) == {"release", "checker", "orchestrator", "mrwp", "problems", "rc", "mrwp_id_source"} + # the state path yields the same core keys (no id_source — that's live-only) + st = ReleaseState(release_id="2026-08") + _seed_rc_pipeline(st, {"total": 10, "passed": 10, "failed": 0}, + {"total": 10, "passed": 10, "failed": 0}) + sm = K.rc_report_model(st) + assert {"release", "checker", "orchestrator", "mrwp", "problems", "rc"} <= set(sm) + assert sm["rc"] == 1 and sm["problems"] == [] + # a never-ran MRWP snapshot yields the SAME problem string the live path derives + st2 = ReleaseState(release_id="2026-08") + from steps.build_verify import _common as K2 + K2.stash_mrwp(st2, "ECS", {"run_id": "9", "complete": False, "never_ran": ["UI Automation"]}) + pm = K2.rc_report_model(st2) + assert any("did NOT run to completion" in p and "UI Automation" in p for p in pm["problems"]) + + def test_classify_test_run_categories(): """The test-run classifier buckets into exactly three: unit / instrumented / ui; anything that isn't unit/instrumented is UI ('the rest are UI', incl. Lab Api Tests).""" diff --git a/release-agent/tools/pipelines.py b/release-agent/tools/pipelines.py index 961572eb..9df8b9a5 100644 --- a/release-agent/tools/pipelines.py +++ b/release-agent/tools/pipelines.py @@ -499,12 +499,25 @@ def fcount(r): return (True, sorted(real, key=lambda x: -x["failed"]), "") -# Coordinates for the release chain (identitydivision/Engineering). The build_verify -# steps carry their own copies in steps/build_verify/_common.py; this default lets the -# report aggregator run standalone. -CHECKER_DEF = 3038 -ORCHESTRATOR_DEF = 2828 -MRWP_DEF = 2519 +# ── ADO targets — the release toolchain spans MULTIPLE orgs/projects ───────────────── +# There is NO single global org/project. Name each target explicitly so a caller can't +# accidentally point a call at the wrong one: +# • identitydivision — the collection hosting Engineering (release chain) + IdentityWiki +# • msazure — hosts One (localization pipeline 405133, Component Governance) +# These constants cover ONLY the release-VERIFICATION chain (checker / orchestrator / MRWP), +# which lives in identitydivision/Engineering. Other areas own their own coordinates: +# localization → steps/ccd/localization.py (msazure/One); wiki → steps/preflight/wiki.py +# (identitydivision/IdentityWiki); CG → steps/preflight/cg.py (msazure/One). +IDENTITYDIVISION = "https://identitydivision.visualstudio.com" +MSAZURE = "https://msazure.visualstudio.com" + +# The release-verification pipelines: identitydivision / Engineering — SINGLE SOURCE. +# `steps/build_verify/_common.py` imports these (it does not redefine them). +ENGINEERING_ORG = IDENTITYDIVISION +ENGINEERING_PROJECT = "Engineering" +CHECKER_DEF = 3038 # Code Complete Calendar Checker (fires the release on the CCD) +ORCHESTRATOR_DEF = 2828 # Release Orchestrator (the spine) +MRWP_DEF = 2519 # Monthly Release Work Pipeline (RC testing; runs ECS + Local) TRIGGER_JOB = "Trigger Monthly Release" ORCH_REQUIRED_STAGES = [ "Validate Branch and Versions availability", @@ -514,33 +527,79 @@ def fcount(r): ORCH_PARK_STAGE = "Remove RC Tags" +def assemble_rc_model(release, checker, orchestrator, mrwp, *, rc=None, + id_source=None, io_problems=None): + """The ONE canonical Phase-2 RC report model — built from already-resolved pieces, + whether they came from LIVE reads (release_report) or the state snapshot + (steps.build_verify._common.rc_report_model). Both paths call this so they can never + drift on shape or on how `problems` are derived. + + Sections: + checker : {fired, run_id, when} | {fired:False} | {error} + orchestrator : {found, healthy, run_id, versions, parked, failed_stages, park_stage} | {found:False[,error]} + mrwp : {ECS:{...}, Local:{...}} each: run_id, complete, ran, total, + failed_stages, yellow_stages, never_ran, tests, failed_suites [, error] + `problems` is derived here in section order (checker → orchestrator → io_problems → + mrwp) from each section's `error`/structural state, so the messages are identical for + both callers. `io_problems` are non-sectional read failures (e.g. MRWP id resolution). + Returns {release, checker, orchestrator, mrwp, problems, rc[, mrwp_id_source]}. + """ + checker = checker or {} + orchestrator = orchestrator or {} + mrwp = mrwp or {} + problems = [] + + if checker.get("error"): + problems.append(f"Checker: could not read runs ({checker['error']}).") + elif checker.get("fired") is False: + problems.append("Checker: no run has a succeeded 'Trigger Monthly Release' job " + "(release not triggered yet, or before Code Complete Day).") + + o = orchestrator + if o.get("found") is False: + problems.append(f"Orchestrator: could not read runs ({o['error']})." if o.get("error") + else f"Orchestrator: no run found for {release}.") + elif o.get("error"): + problems.append(f"Orchestrator: could not read stages ({o['error']}).") + elif o.get("failed_stages"): + problems.append("Orchestrator: pre-gate stage(s) not green: " + + ", ".join(o["failed_stages"]) + ".") + + problems.extend(io_problems or []) + + for provider in ("ECS", "Local"): + e = mrwp.get(provider) + if not e: + continue + if e.get("error"): + problems.append(f"MRWP {provider}: could not read stages ({e['error']}).") + elif e.get("complete") is False: + nv = ", ".join(n for n in (e.get("never_ran") or []) if n) or "(unknown)" + problems.append(f"MRWP {provider}: did NOT run to completion — never-ran: {nv}.") + + model = {"release": release, "checker": checker, "orchestrator": orchestrator, + "mrwp": mrwp, "problems": problems, "rc": rc} + if id_source is not None: + model["mrwp_id_source"] = id_source + return model + + def release_report(org, project, release_month, checker_def=CHECKER_DEF, orch_def=ORCHESTRATOR_DEF, timeout=90, with_failed_tests=True): - """Assemble the full Phase-2 RC-pipeline + test report for a release month — a pure - read aggregator over the other helpers (does NOT gate; it reports). Returns a model: - - {release, checker:{fired,run_id,when}, - orchestrator:{run_id, found, healthy, parked, park_stage, failed_stages, versions}, - mrwp:{ECS:{...}, Local:{...}}, # each: run_id, complete, ran, total, failed_stages, - # yellow_stages, never_ran, tests:{total,passed,failed,runs}, - # failed_suites:[{name,failed,total,tests:[titles]}] - problems:[...]} # human-readable issues (empty = all good) + """Assemble the full Phase-2 RC-pipeline + test report for a release month by LIVE + reads (does NOT gate; it reports). Resolves the checker / orchestrator / both MRWP + runs, then hands the pieces to `assemble_rc_model` (the shared assembler) so this live + path and the state-snapshot path produce an identical model shape + problems. Any + field that couldn't be read carries an `error` note (surfaced as a problem). `with_failed_tests` (default True) also fetches the individual failing test names per - suite (extra REST calls — set False for a faster stage-only view). Any field that - couldn't be read carries an `error` note; those also land in `problems`. - """ - model = {"release": release_month, "checker": {}, "orchestrator": {}, - "mrwp": {}, "problems": []} - P = problems = model["problems"] - + suite (extra REST calls — set False for a faster stage-only view).""" # --- checker (did the release fire?) --- ok, runs, detail = find_checker_runs(org, project, checker_def, release_month, timeout) - fired = None if not ok: - model["checker"] = {"error": detail} - problems.append(f"Checker: could not read runs ({detail}).") + checker = {"error": detail} else: + fired = None for run in (runs or [])[:25]: ok2, recs, _ = get_timeline(org, project, run.get("id"), timeout) if not ok2: @@ -549,24 +608,15 @@ def release_report(org, project, release_month, checker_def=CHECKER_DEF, if rec is not None and rec.get("result") == "succeeded": fired = run break - if fired: - model["checker"] = {"fired": True, "run_id": fired.get("id"), - "when": (fired.get("queueTime") or "")[:16]} - else: - model["checker"] = {"fired": False} - problems.append("Checker: no run has a succeeded 'Trigger Monthly Release' job " - "(release not triggered yet, or before Code Complete Day).") + checker = ({"fired": True, "run_id": fired.get("id"), + "when": (fired.get("queueTime") or "")[:16]} if fired else {"fired": False}) # --- orchestrator (healthy? parked?) --- ok, orun, detail = find_orchestrator_run(org, project, orch_def, release_month, timeout) if not ok: - model["orchestrator"] = {"found": False, "error": detail} - problems.append(f"Orchestrator: could not read runs ({detail}).") - return model # can't resolve MRWP without the orchestrator + return assemble_rc_model(release_month, checker, {"found": False, "error": detail}, {}) if not orun: - model["orchestrator"] = {"found": False} - problems.append(f"Orchestrator: no run found for {release_month}.") - return model + return assemble_rc_model(release_month, checker, {"found": False}, {}) oid, tags = orun.get("id"), (orun.get("tags") or []) versions = {k: _tag_value(tags, f"Next{k}Version") for k in ("Common", "Msal", "Broker")} @@ -575,7 +625,6 @@ def release_report(org, project, release_month, checker_def=CHECKER_DEF, "park_stage": ORCH_PARK_STAGE, "failed_stages": [], "healthy": None, "parked": None} if not ok: o["error"] = detail - problems.append(f"Orchestrator: could not read stages ({detail}).") else: by = {s.get("name"): s for s in ostages} o["failed_stages"] = [n for n in ORCH_REQUIRED_STAGES @@ -583,37 +632,29 @@ def release_report(org, project, release_month, checker_def=CHECKER_DEF, o["healthy"] = not o["failed_stages"] park = by.get(ORCH_PARK_STAGE) o["parked"] = bool(park and park.get("state") != "completed") - if o["failed_stages"]: - problems.append("Orchestrator: pre-gate stage(s) not green: " - + ", ".join(o["failed_stages"]) + ".") - model["orchestrator"] = o # --- the two MRWP runs (ran to completion? tests?) --- ok, ids, detail, source = mrwp_run_ids(org, project, orun, timeout) if not ok: - problems.append(f"MRWP: could not resolve run ids ({detail}).") - return model - model["mrwp_id_source"] = source # 'tags' or 'logs' + return assemble_rc_model(release_month, checker, o, {}, + io_problems=[f"MRWP: could not resolve run ids ({detail})."]) + mrwp = {} for provider in ("ECS", "Local"): bid = ids.get(provider) entry = {"run_id": bid} ok, stages, detail = get_stages(org, project, bid, timeout) if not ok: entry["error"] = detail - problems.append(f"MRWP {provider}: could not read stages ({detail}).") else: comp = stage_completion(stages) entry.update({"complete": comp["complete"], "ran": comp["ran"], "total": comp["total"], "failed_stages": comp["failed"], "yellow_stages": comp["yellow"], "never_ran": comp["never_ran"]}) - if not comp["complete"]: - nv = ", ".join(n for n in comp["never_ran"] if n) or "(unknown)" - problems.append(f"MRWP {provider}: did NOT run to completion — never-ran: {nv}.") okt, tests, _ = get_test_summary(org, project, bid, timeout) entry["tests"] = tests if okt else None # Individual failing tests, aggregated by suite (deduped across repeated runs). if with_failed_tests and bid and tests and tests.get("failed"): okf, suites, _ = get_failed_tests(org, project, bid, timeout=timeout) entry["failed_suites"] = suites if okf else None - model["mrwp"][provider] = entry - return model + mrwp[provider] = entry + return assemble_rc_model(release_month, checker, o, mrwp, id_source=source) From 01cad029206c24a7d4d1a68dd8553cb3990552ec Mon Sep 17 00:00:00 2001 From: p3dr0rv Date: Thu, 20 Aug 2026 22:15:49 +0100 Subject: [PATCH 70/82] =?UTF-8?q?release-agent:=20arch=20review=20batch=20?= =?UTF-8?q?2=20=E2=80=94=20extract=20status=20views,=20dedupe=20renderers,?= =?UTF-8?q?=20tidy=20imports/excepts?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit M1: extract the ~250-line status view-model (_active_phase_report/_phase_map/_current_steps/_hold_view/_scheduled_view/status_report) into orchestrator/status_views.py as StatusViewMixin; engine.py 783->540 lines. Behaviour identical. M2: sort_failed_suites() + shared recovered_unit_tests() reused across the three RC renderers (plain email / HTML email / CLI report) — one place to change ordering. M3: format_versions() helper collapses six duplicated 'Common/Msal/Broker' join idioms. M4: hoisted the safe lazy imports in build_verify/_common (tools.pipelines, state, datetime, outcomes, mockctx) — no import cycle; left the automations/steps lazies that guard real cycles. L1: narrowed parse-y broad excepts (discovery JSON, engine/schedule YAML+zoneinfo, cli stream reconfigure) to specific exception types; left the intentional best-effort IO ones. L3: documented the orchestrator healthy=True invariant in rc_report_model. 188 tests pass. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- release-agent/orchestrator/cli.py | 2 +- .../orchestrator/commands/rc_report.py | 10 +- release-agent/orchestrator/discovery.py | 2 +- release-agent/orchestrator/engine.py | 260 +---------------- release-agent/orchestrator/render.py | 4 +- release-agent/orchestrator/schedule.py | 2 +- release-agent/orchestrator/status_views.py | 264 ++++++++++++++++++ release-agent/steps/build_verify/_common.py | 45 +-- .../steps/build_verify/orchestrator_health.py | 2 +- release-agent/tools/pipelines.py | 10 + 10 files changed, 314 insertions(+), 287 deletions(-) create mode 100644 release-agent/orchestrator/status_views.py diff --git a/release-agent/orchestrator/cli.py b/release-agent/orchestrator/cli.py index d39e49a6..2e2e8211 100644 --- a/release-agent/orchestrator/cli.py +++ b/release-agent/orchestrator/cli.py @@ -18,7 +18,7 @@ try: sys.stdout.reconfigure(encoding="utf-8") sys.stderr.reconfigure(encoding="utf-8") -except Exception: +except (AttributeError, ValueError): # non-reconfigurable stream / unsupported encoding pass HERE = os.path.dirname(os.path.abspath(__file__)) diff --git a/release-agent/orchestrator/commands/rc_report.py b/release-agent/orchestrator/commands/rc_report.py index 9339985f..54a7163b 100644 --- a/release-agent/orchestrator/commands/rc_report.py +++ b/release-agent/orchestrator/commands/rc_report.py @@ -115,8 +115,7 @@ def _format(m) -> str: err = f" ({o['error']})" if "error" in o else "" L.append(f"⛔ **Release Orchestrator** — no run found{err}.") else: - v = o.get("versions") or {} - vstr = ", ".join(f"{k} {v[k]}" for k in ("Common", "Msal", "Broker") if v.get(k)) or "versions n/a" + vstr = K.format_versions(o.get("versions"), fallback="versions n/a") if o.get("healthy"): park = "parked at 'Remove RC Tags' (awaiting owner approval)" if o.get("parked") \ else f"'{o.get('park_stage')}' already cleared" @@ -158,8 +157,7 @@ def _format(m) -> str: # Failing tests, grouped by suite (UI first), each tagged by category. suites = r.get("failed_suites") if suites: - _ord = {"ui": 0, "instrumented": 1, "unit": 2} - for s in sorted(suites, key=lambda s: (_ord.get(s.get("category", "ui"), 9), -s["failed"])): + for s in K.sort_failed_suites(suites): cat = _lbl.get(s.get("category", "ui"), "UI automation") fr = round(s["failed"] * 100.0 / s["total"], 1) if s["total"] else 0.0 L.append(f" • [{cat}] {s['name']}: {s['failed']}/{s['total']} failed ({fr}%)") @@ -178,9 +176,7 @@ def _format(m) -> str: for p in probs: L.append(f" - {p}") # Unit retry warning — failed-then-passed on retry (counted as passed). - recovered = sorted({t for prov in ("ECS", "Local") - for t in ((((m.get("mrwp") or {}).get(prov) or {}).get("tests") or {}) - .get("categories", {}).get("unit", {}).get("recovered") or [])}) + recovered = K.recovered_unit_tests(m) if recovered: L += ["", f"⚠ **Retry warning** — {len(recovered)} unit test(s) failed then passed " f"on retry (counted as passed):"] diff --git a/release-agent/orchestrator/discovery.py b/release-agent/orchestrator/discovery.py index b21c13a1..6e8cd056 100644 --- a/release-agent/orchestrator/discovery.py +++ b/release-agent/orchestrator/discovery.py @@ -18,7 +18,7 @@ def _summarize(state_file: str) -> Optional[dict]: try: with open(state_file, "r", encoding="utf-8") as fh: data = json.load(fh) - except Exception: + except (OSError, ValueError): # missing/unreadable file or bad JSON → skip it return None return { "release_id": data.get("release_id"), diff --git a/release-agent/orchestrator/engine.py b/release-agent/orchestrator/engine.py index 868f53d5..ae4ac52d 100644 --- a/release-agent/orchestrator/engine.py +++ b/release-agent/orchestrator/engine.py @@ -21,6 +21,7 @@ from .readiness import ReadinessGate from . import schedule from . import mocks as mocks_mod +from .status_views import StatusViewMixin from steps.lib import mockctx import steps from phases import stub_runner @@ -36,7 +37,7 @@ class NextAction: message: str = "" -class Orchestrator: +class Orchestrator(StatusViewMixin): """The conductor: owns the state machine, dispatch loop, gates, and structured status. The readiness entry gate is delegated to ReadinessGate (self.gate); presentation lives in render.py. This class holds no formatting logic.""" @@ -89,7 +90,7 @@ def _config_timezone(config_path: str) -> Optional[str]: if os.path.exists(p): with open(p, "r", encoding="utf-8") as fh: return (yaml.safe_load(fh) or {}).get("timezone") - except Exception: + except (OSError, yaml.YAMLError): # missing/unreadable/invalid schedule.yaml pass return None @@ -591,262 +592,11 @@ def deny_gate(self, comment: str = "") -> NextAction: return NextAction(kind="gate", phase=phase, step=step, message=f"Gate DENIED: {phase} → {step}. Release blocked. {comment}".strip()) - # ---- reporting ---- + # `_phase_included` is a shared helper (used by both the state machine and the status + # views mixin). The status view-model builders live in orchestrator/status_views.py. def _phase_included(self, phase: dict) -> bool: return (not phase.get("conditional")) or phase["id"] in self._activated_conditionals() - def _active_phase_report(self) -> Optional[dict]: - """The first incomplete included phase, with its outstanding steps and - whether its time-window is open (due). This is what the daily phase - notification reports on — independent of state.current_phase (which is - only set once the release has been advanced).""" - for phase in self.config["phases"]: - if not self._phase_included(phase): - continue - steps = phase["steps"] - done = sum(1 for s in steps if self.state.is_done(phase["id"], s["id"])) - if done == len(steps): - continue # phase complete — look at the next one - outstanding = [ - {"id": s["id"], "name": s["name"], "gate": bool(s.get("gate")), - "reminder": self._is_reminder(s), "owner": s.get("owner", "agent")} - for s in steps if not self.state.is_done(phase["id"], s["id"]) - ] - completed = [s["name"] for s in steps - if self.state.is_done(phase["id"], s["id"])] - cur = self.state.current_step - steps_view = [] - for s in steps: - sid = s["id"] - stp = self.state.get_step(phase["id"], sid) - s_done = self.state.is_done(phase["id"], sid) - s_blocked = stp.status == "blocked" - is_gate = bool(s.get("gate")) - is_rem = self._is_reminder(s) - is_scout = s.get("source") == "scout" - is_attest = bool(s.get("attest")) - if s_done: - status = "done" - elif s_blocked: - status = "blocked" - elif is_gate: - status = "approval" - elif is_attest: - status = "confirm" - elif is_rem: - status = "action" # a human to-do — the user must act - elif is_scout: - status = "scout" # Scout runs it automatically (scrape/send via MCP) - else: - status = "auto" - # needs_owner = a genuine USER task. A pending scout step is Scout's - # automatic work (not the user's) until it BLOCKS (s_blocked), so it is - # NOT flagged — only gates, reminders, attests, and blocks are. - needs = bool((is_gate or is_rem or is_attest or s_blocked) and not s_done) - steps_view.append({ - "id": sid, "name": s["name"], "status": status, - "needs_owner": needs, - "time_ready": self._step_time_ready(phase, s), # False = waits for its fire_at_local - "note": stp.note, # agent result / block reason / detail - "links": list(getattr(stp, "links", None) or []), # durable refs to items evaluated - "now": bool(sid == cur and not s_done and (is_gate or is_rem or is_attest or s_blocked)), - }) - opens = self._phase_anchor_date(phase) - return { - "id": phase["id"], "name": phase["name"], - "num": phase.get("checklist_phase"), - "show_pipeline_runs": bool(phase.get("show_pipeline_runs")), - "done": done, "total": len(steps), - "due": self._phase_due(phase), "started": done > 0, - "opens": opens.isoformat() if opens else None, - "opens_in_days": (opens - self.as_of).days if opens else None, - "outstanding": outstanding, - "completed": completed, - "steps": steps_view, - } - return None - - def _phase_map(self): - """Build the phase overview + running totals. Returns - (phases, total, done, current_phase_name, current_phase_obj, current_step_name).""" - phases = [] - total = done = 0 - current_phase_name = current_step_name = None - current_phase_obj = None - for idx, phase in enumerate(self.config["phases"]): - if not self._phase_included(phase): - continue - p_total = len(phase["steps"]) - p_done = sum(1 for s in phase["steps"] if self.state.is_done(phase["id"], s["id"])) - total += p_total - done += p_done - is_current = self.state.current_phase == phase["id"] - due = self._phase_due(phase) - opens = self._phase_anchor_date(phase) - if p_total and p_done == p_total: - state = "done" - elif not due and p_done == 0: - state = "scheduled" - elif is_current or p_done > 0: - state = "current" - else: - state = "pending" - if is_current: - current_phase_name = phase["name"] - current_phase_obj = phase - phases.append({ - "id": phase["id"], "name": phase["name"], - "num": phase.get("checklist_phase", idx), - "done": p_done, "total": p_total, "state": state, - "current": is_current, - "anchor": phase.get("anchor"), - "opens": opens.isoformat() if opens else None, - "opens_in_days": (opens - self.as_of).days if opens else None, - }) - for s in phase["steps"]: - if s["id"] == self.state.current_step and phase["id"] == self.state.current_phase: - current_step_name = s["name"] - return phases, total, done, current_phase_name, current_phase_obj, current_step_name - - def _current_steps(self, current_phase_obj) -> list: - """The current phase's steps, each tagged with a display state.""" - if not current_phase_obj: - return [] - phase_due = self._phase_due(current_phase_obj) - out = [] - for s in current_phase_obj["steps"]: - rec = self.state.steps.get(self.state.key(current_phase_obj["id"], s["id"]), {}) or {} - is_scout = s.get("source") == "scout" and not s.get("attest") - if rec.get("status") == "skipped": - s_state = "skipped" - elif self.state.is_done(current_phase_obj["id"], s["id"]): - s_state = "done" - elif rec.get("status") == "blocked": - s_state = "blocked" # a step hit a real problem — needs the owner - elif s["id"] == self.state.current_step and self.state.status == "holding_gate": - s_state = "gate" - elif is_scout: - s_state = "scout" # Scout's automatic work — never a user "do this" - elif s["id"] == self.state.current_step and self.state.status == "awaiting_action": - s_state = "reminder" - elif not phase_due: - s_state = "scheduled" - else: - s_state = "pending" - out.append({ - "id": s["id"], "name": s["name"], - "gate": bool(s.get("gate")), - "reminder": self._is_reminder(s), - "owner": s.get("owner", "agent"), - "state": s_state, - "note": rec.get("note"), # agent result / block reason / detail - "links": rec.get("links") or [], # durable refs (wiki page, CG alerts) - }) - return out - - def _hold_view(self, phase_name, step_name) -> dict: - """Detail of the current hold (gate or action-needed) — same shape for both.""" - return { - "phase": self.state.current_phase, - "phase_name": phase_name, - "step": self.state.current_step, - "step_name": step_name, - } - - def _scheduled_view(self) -> Optional[dict]: - """The phase we're waiting on the clock for — derived from the first - incomplete phase's due-ness, so `status` shows it even before `next`.""" - first_incomplete = next( - (p for p in self.config["phases"] - if self._phase_included(p) - and not all(self.state.is_done(p["id"], s["id"]) for s in p["steps"])), - None) - if (first_incomplete is None or self._phase_due(first_incomplete) - or self.state.status in ("complete", "halted", "blocked")): - return None - opens = self._phase_anchor_date(first_incomplete) - return { - "phase": first_incomplete["id"], - "phase_name": first_incomplete["name"], - "opens": opens.isoformat() if opens else None, - "opens_in_days": (opens - self.as_of).days if opens else None, - } - - def status_report(self) -> dict: - """Structured status — presentation layer (render.py) turns this into a view. - Deterministic; no formatting baked in. Assembled from focused builders: - phase map, current-phase steps, current hold, scheduled window, active phase.""" - (phases, total, done, current_phase_name, - current_phase_obj, current_step_name) = self._phase_map() - current_steps = self._current_steps(current_phase_obj) - - gate = action = None - if self.state.status == "holding_gate" and self.state.current_phase: - gate = self._hold_view(current_phase_name, current_step_name) - elif self.state.status == "awaiting_action" and self.state.current_phase: - # A scout step is the SKILL's work (run via step-action), not a USER action — - # never surface it as `action` (which the digest reads as "Action needed now"). - phase = next((p for p in self.config["phases"] - if p["id"] == self.state.current_phase), None) - cur = next((s for s in (phase or {}).get("steps", []) - if s["id"] == self.state.current_step), None) if self.state.current_step else None - is_scout_focus = bool(cur and cur.get("source") == "scout" and not cur.get("attest")) - cur_blocked = (self.state.get_step(self.state.current_phase, self.state.current_step).status - == "blocked") if self.state.current_step else False - if not is_scout_focus or cur_blocked: - action = self._hold_view(current_phase_name, current_step_name) - scheduled = self._scheduled_view() - - chk = self.gate.checklist() - active_phase = self._active_phase_report() - # Scout steps ready for the SKILL to execute (perform the MCP send/scrape, then - # record-step). They are NOT user holds — the skill drains these itself; only if - # a scout step records 'attention' does it become a blocked user task. - # GATED ON PHASE DUE: a phase that hasn't reached its anchor (e.g. Code Complete - # Day before the CCD) must expose NO pending scout work — otherwise the autonomous - # automation would drain those steps early, running CCD-day comms ahead of the CCD. - # ALSO GATED ON fire_at_local: a timed step (e.g. the 09:00 CCD comms) is excluded - # until its wall-clock time arrives, so the every-hour worker doesn't fire it early - # — its dedicated cron automation runs it at the pinned time. - scout_pending = ([s["id"] for s in (active_phase or {}).get("steps", []) - if s.get("status") == "scout" and s.get("time_ready", True)] - if (active_phase and active_phase.get("due")) else []) - return { - "release_id": self.state.release_id, - "status": self.state.status, - "owner_email": self.state.owner_email, - "owner_name": self.state.owner_name, - "ccd": self.state.ccd, - "ccd_source": self.state.ccd_source, - "ccd_conflict": self.state.ccd_conflict, - "as_of": self.as_of.isoformat(), - "skip_release": self.state.skip_release, - "readiness_signed": self.state.readiness_signed, - "readiness_pending": [i["id"] for i in chk["items"] if not i["satisfied"]], - "blocked": self.state.blocked, - "blocked_items": list(self.state.blocked_items), - "blocked_message": chk.get("blocked_message", ""), - "halted": self.state.halted, - "halt_reason": self.state.halt_reason, - "done": done, "total": total, - "percent": round(100 * done / total) if total else 0, - "phases": phases, - "current_phase": self.state.current_phase, - "current_phase_name": current_phase_name, - "current_step": self.state.current_step, - "current_step_name": current_step_name, - "current_steps": current_steps, - "gate": gate, - "action": action, - "scheduled": scheduled, - "active_phase": active_phase, - "scout_pending": scout_pending, - "pending_human": list(self.state.pending_human), - "gate_decisions": len(self.state.gate_decisions), - "pipeline_runs": dict(getattr(self.state, "pipeline_runs", {}) or {}), - "updated_at": self.state.updated_at, - } - def asdict_gate(gd: GateDecision) -> dict: return {"step": gd.step, "decision": gd.decision, "at": gd.at, diff --git a/release-agent/orchestrator/render.py b/release-agent/orchestrator/render.py index 7e30208f..ed9bca9c 100644 --- a/release-agent/orchestrator/render.py +++ b/release-agent/orchestrator/render.py @@ -242,6 +242,7 @@ def _pipelines_line(r: dict) -> str: resolved yet. Reads the nested pipeline_runs schema (migrating a legacy flat shape); no live az call in the render path.""" from orchestrator.state import migrate_pipeline_runs + from tools.pipelines import format_versions pr = migrate_pipeline_runs(r.get("pipeline_runs") or {}) parts = [] ch = pr.get("checker") or {} @@ -249,8 +250,7 @@ def _pipelines_line(r: dict) -> str: parts.append(f"checker {ch['run_id']}") o = pr.get("orchestrator") or {} if o.get("run_id"): - v = o.get("versions") or {} - vstr = ", ".join(f"{k} {v[k]}" for k in ("Common", "Msal", "Broker") if v.get(k)) + vstr = format_versions(o.get("versions")) parts.append(f"orchestrator {o['run_id']}" + (f" ({vstr})" if vstr else "")) rcs = pr.get("rcs") or [] rc = rcs[-1] if rcs else {} diff --git a/release-agent/orchestrator/schedule.py b/release-agent/orchestrator/schedule.py index 384cdc38..326e4d57 100644 --- a/release-agent/orchestrator/schedule.py +++ b/release-agent/orchestrator/schedule.py @@ -33,7 +33,7 @@ def get_tz(name: Optional[str] = None): try: from zoneinfo import ZoneInfo return ZoneInfo(name or DEFAULT_TZ) - except Exception: + except (ImportError, KeyError, ValueError): # no tzdata / unknown zone name return None diff --git a/release-agent/orchestrator/status_views.py b/release-agent/orchestrator/status_views.py new file mode 100644 index 00000000..c9efc2e9 --- /dev/null +++ b/release-agent/orchestrator/status_views.py @@ -0,0 +1,264 @@ +"""Status view-model builder — the presentation half of the Orchestrator. + +Extracted from engine.py (which owns the state machine) so the report-model builder is a +separate responsibility. This is a MIXIN on Orchestrator: the methods read engine +internals via self (self.state, self.config, self._phase_due, ...) and return the plain +dict that render.py turns into a view. Behaviour is identical to the in-engine version. +""" +from __future__ import annotations + +from typing import Optional + + +class StatusViewMixin: + def _active_phase_report(self) -> Optional[dict]: + """The first incomplete included phase, with its outstanding steps and + whether its time-window is open (due). This is what the daily phase + notification reports on — independent of state.current_phase (which is + only set once the release has been advanced).""" + for phase in self.config["phases"]: + if not self._phase_included(phase): + continue + steps = phase["steps"] + done = sum(1 for s in steps if self.state.is_done(phase["id"], s["id"])) + if done == len(steps): + continue # phase complete — look at the next one + outstanding = [ + {"id": s["id"], "name": s["name"], "gate": bool(s.get("gate")), + "reminder": self._is_reminder(s), "owner": s.get("owner", "agent")} + for s in steps if not self.state.is_done(phase["id"], s["id"]) + ] + completed = [s["name"] for s in steps + if self.state.is_done(phase["id"], s["id"])] + cur = self.state.current_step + steps_view = [] + for s in steps: + sid = s["id"] + stp = self.state.get_step(phase["id"], sid) + s_done = self.state.is_done(phase["id"], sid) + s_blocked = stp.status == "blocked" + is_gate = bool(s.get("gate")) + is_rem = self._is_reminder(s) + is_scout = s.get("source") == "scout" + is_attest = bool(s.get("attest")) + if s_done: + status = "done" + elif s_blocked: + status = "blocked" + elif is_gate: + status = "approval" + elif is_attest: + status = "confirm" + elif is_rem: + status = "action" # a human to-do — the user must act + elif is_scout: + status = "scout" # Scout runs it automatically (scrape/send via MCP) + else: + status = "auto" + # needs_owner = a genuine USER task. A pending scout step is Scout's + # automatic work (not the user's) until it BLOCKS (s_blocked), so it is + # NOT flagged — only gates, reminders, attests, and blocks are. + needs = bool((is_gate or is_rem or is_attest or s_blocked) and not s_done) + steps_view.append({ + "id": sid, "name": s["name"], "status": status, + "needs_owner": needs, + "time_ready": self._step_time_ready(phase, s), # False = waits for its fire_at_local + "note": stp.note, # agent result / block reason / detail + "links": list(getattr(stp, "links", None) or []), # durable refs to items evaluated + "now": bool(sid == cur and not s_done and (is_gate or is_rem or is_attest or s_blocked)), + }) + opens = self._phase_anchor_date(phase) + return { + "id": phase["id"], "name": phase["name"], + "num": phase.get("checklist_phase"), + "show_pipeline_runs": bool(phase.get("show_pipeline_runs")), + "done": done, "total": len(steps), + "due": self._phase_due(phase), "started": done > 0, + "opens": opens.isoformat() if opens else None, + "opens_in_days": (opens - self.as_of).days if opens else None, + "outstanding": outstanding, + "completed": completed, + "steps": steps_view, + } + return None + + def _phase_map(self): + """Build the phase overview + running totals. Returns + (phases, total, done, current_phase_name, current_phase_obj, current_step_name).""" + phases = [] + total = done = 0 + current_phase_name = current_step_name = None + current_phase_obj = None + for idx, phase in enumerate(self.config["phases"]): + if not self._phase_included(phase): + continue + p_total = len(phase["steps"]) + p_done = sum(1 for s in phase["steps"] if self.state.is_done(phase["id"], s["id"])) + total += p_total + done += p_done + is_current = self.state.current_phase == phase["id"] + due = self._phase_due(phase) + opens = self._phase_anchor_date(phase) + if p_total and p_done == p_total: + state = "done" + elif not due and p_done == 0: + state = "scheduled" + elif is_current or p_done > 0: + state = "current" + else: + state = "pending" + if is_current: + current_phase_name = phase["name"] + current_phase_obj = phase + phases.append({ + "id": phase["id"], "name": phase["name"], + "num": phase.get("checklist_phase", idx), + "done": p_done, "total": p_total, "state": state, + "current": is_current, + "anchor": phase.get("anchor"), + "opens": opens.isoformat() if opens else None, + "opens_in_days": (opens - self.as_of).days if opens else None, + }) + for s in phase["steps"]: + if s["id"] == self.state.current_step and phase["id"] == self.state.current_phase: + current_step_name = s["name"] + return phases, total, done, current_phase_name, current_phase_obj, current_step_name + + def _current_steps(self, current_phase_obj) -> list: + """The current phase's steps, each tagged with a display state.""" + if not current_phase_obj: + return [] + phase_due = self._phase_due(current_phase_obj) + out = [] + for s in current_phase_obj["steps"]: + rec = self.state.steps.get(self.state.key(current_phase_obj["id"], s["id"]), {}) or {} + is_scout = s.get("source") == "scout" and not s.get("attest") + if rec.get("status") == "skipped": + s_state = "skipped" + elif self.state.is_done(current_phase_obj["id"], s["id"]): + s_state = "done" + elif rec.get("status") == "blocked": + s_state = "blocked" # a step hit a real problem — needs the owner + elif s["id"] == self.state.current_step and self.state.status == "holding_gate": + s_state = "gate" + elif is_scout: + s_state = "scout" # Scout's automatic work — never a user "do this" + elif s["id"] == self.state.current_step and self.state.status == "awaiting_action": + s_state = "reminder" + elif not phase_due: + s_state = "scheduled" + else: + s_state = "pending" + out.append({ + "id": s["id"], "name": s["name"], + "gate": bool(s.get("gate")), + "reminder": self._is_reminder(s), + "owner": s.get("owner", "agent"), + "state": s_state, + "note": rec.get("note"), # agent result / block reason / detail + "links": rec.get("links") or [], # durable refs (wiki page, CG alerts) + }) + return out + + def _hold_view(self, phase_name, step_name) -> dict: + """Detail of the current hold (gate or action-needed) — same shape for both.""" + return { + "phase": self.state.current_phase, + "phase_name": phase_name, + "step": self.state.current_step, + "step_name": step_name, + } + + def _scheduled_view(self) -> Optional[dict]: + """The phase we're waiting on the clock for — derived from the first + incomplete phase's due-ness, so `status` shows it even before `next`.""" + first_incomplete = next( + (p for p in self.config["phases"] + if self._phase_included(p) + and not all(self.state.is_done(p["id"], s["id"]) for s in p["steps"])), + None) + if (first_incomplete is None or self._phase_due(first_incomplete) + or self.state.status in ("complete", "halted", "blocked")): + return None + opens = self._phase_anchor_date(first_incomplete) + return { + "phase": first_incomplete["id"], + "phase_name": first_incomplete["name"], + "opens": opens.isoformat() if opens else None, + "opens_in_days": (opens - self.as_of).days if opens else None, + } + + def status_report(self) -> dict: + """Structured status — presentation layer (render.py) turns this into a view. + Deterministic; no formatting baked in. Assembled from focused builders: + phase map, current-phase steps, current hold, scheduled window, active phase.""" + (phases, total, done, current_phase_name, + current_phase_obj, current_step_name) = self._phase_map() + current_steps = self._current_steps(current_phase_obj) + + gate = action = None + if self.state.status == "holding_gate" and self.state.current_phase: + gate = self._hold_view(current_phase_name, current_step_name) + elif self.state.status == "awaiting_action" and self.state.current_phase: + # A scout step is the SKILL's work (run via step-action), not a USER action — + # never surface it as `action` (which the digest reads as "Action needed now"). + phase = next((p for p in self.config["phases"] + if p["id"] == self.state.current_phase), None) + cur = next((s for s in (phase or {}).get("steps", []) + if s["id"] == self.state.current_step), None) if self.state.current_step else None + is_scout_focus = bool(cur and cur.get("source") == "scout" and not cur.get("attest")) + cur_blocked = (self.state.get_step(self.state.current_phase, self.state.current_step).status + == "blocked") if self.state.current_step else False + if not is_scout_focus or cur_blocked: + action = self._hold_view(current_phase_name, current_step_name) + scheduled = self._scheduled_view() + + chk = self.gate.checklist() + active_phase = self._active_phase_report() + # Scout steps ready for the SKILL to execute (perform the MCP send/scrape, then + # record-step). They are NOT user holds — the skill drains these itself; only if + # a scout step records 'attention' does it become a blocked user task. + # GATED ON PHASE DUE: a phase that hasn't reached its anchor (e.g. Code Complete + # Day before the CCD) must expose NO pending scout work — otherwise the autonomous + # automation would drain those steps early, running CCD-day comms ahead of the CCD. + # ALSO GATED ON fire_at_local: a timed step (e.g. the 09:00 CCD comms) is excluded + # until its wall-clock time arrives, so the every-hour worker doesn't fire it early + # — its dedicated cron automation runs it at the pinned time. + scout_pending = ([s["id"] for s in (active_phase or {}).get("steps", []) + if s.get("status") == "scout" and s.get("time_ready", True)] + if (active_phase and active_phase.get("due")) else []) + return { + "release_id": self.state.release_id, + "status": self.state.status, + "owner_email": self.state.owner_email, + "owner_name": self.state.owner_name, + "ccd": self.state.ccd, + "ccd_source": self.state.ccd_source, + "ccd_conflict": self.state.ccd_conflict, + "as_of": self.as_of.isoformat(), + "skip_release": self.state.skip_release, + "readiness_signed": self.state.readiness_signed, + "readiness_pending": [i["id"] for i in chk["items"] if not i["satisfied"]], + "blocked": self.state.blocked, + "blocked_items": list(self.state.blocked_items), + "blocked_message": chk.get("blocked_message", ""), + "halted": self.state.halted, + "halt_reason": self.state.halt_reason, + "done": done, "total": total, + "percent": round(100 * done / total) if total else 0, + "phases": phases, + "current_phase": self.state.current_phase, + "current_phase_name": current_phase_name, + "current_step": self.state.current_step, + "current_step_name": current_step_name, + "current_steps": current_steps, + "gate": gate, + "action": action, + "scheduled": scheduled, + "active_phase": active_phase, + "scout_pending": scout_pending, + "pending_human": list(self.state.pending_human), + "gate_decisions": len(self.state.gate_decisions), + "pipeline_runs": dict(getattr(self.state, "pipeline_runs", {}) or {}), + "updated_at": self.state.updated_at, + } diff --git a/release-agent/steps/build_verify/_common.py b/release-agent/steps/build_verify/_common.py index 7dcec4a8..58bacdef 100644 --- a/release-agent/steps/build_verify/_common.py +++ b/release-agent/steps/build_verify/_common.py @@ -9,13 +9,19 @@ """ from __future__ import annotations +from datetime import datetime, timezone + # This package is Engineering-only; alias the explicitly-named source constants to the # short local names the step CONFIGs use. (Do NOT use these for msazure/One calls.) +from tools import pipelines as P from tools.pipelines import ( ENGINEERING_ORG as ORG, ENGINEERING_PROJECT as PROJECT, CHECKER_DEF, ORCHESTRATOR_DEF, MRWP_DEF, - ORCH_REQUIRED_STAGES, ORCH_PARK_STAGE, + ORCH_REQUIRED_STAGES, ORCH_PARK_STAGE, format_versions, ) +from orchestrator.state import migrate_pipeline_runs +from orchestrator.outcomes import Done, Blocked +from steps.lib.mockctx import mock_input, MISSING # Surfaced in every block reason so the engineer knows how to recover / escalate. RECOVERY_TSG = ("https://eng.ms/docs/microsoft-security/identity/" @@ -41,13 +47,11 @@ def links_for(build_id, name="ADO run"): def _now_iso(): - from datetime import datetime, timezone return datetime.now(timezone.utc).isoformat() def _pipeline_runs(state) -> dict: """The nested pipeline_runs container on state (migrating a legacy flat shape).""" - from orchestrator.state import migrate_pipeline_runs return migrate_pipeline_runs(getattr(state, "pipeline_runs", None) or {}) @@ -114,8 +118,6 @@ def rc_report_model(state, timeout=120): iteration (rcs[-1]) and routes through `tools.pipelines.assemble_rc_model` — the SAME assembler the live path uses — so the state-based model can't drift from the live one. """ - from tools import pipelines as P - from orchestrator.state import migrate_pipeline_runs pr = migrate_pipeline_runs(getattr(state, "pipeline_runs", None) or {}) ch = pr.get("checker") or {} o = pr.get("orchestrator") or {} @@ -123,6 +125,9 @@ def rc_report_model(state, timeout=120): rc = rcs[-1] if rcs else {} checker = {"fired": bool(ch.get("run_id")), "run_id": ch.get("run_id"), "when": ch.get("when")} + # healthy=True is true by construction here: rc_report only runs AFTER orchestrator_health + # passed (a failed pre-gate stage blocks that step, so we never reach this with an + # unhealthy orchestrator). The live path (release_report) derives it from stages. orchestrator = {"found": bool(o.get("run_id")), "healthy": True, "run_id": o.get("run_id"), "versions": o.get("versions") or {}, "parked": o.get("parked")} @@ -257,14 +262,25 @@ def _fail_rate(failed, total) -> float: # Test-run category labels (mirrors tools.pipelines.classify_test_run). _CAT_LABEL = {"unit": "Unit", "instrumented": "Instrumented", "ui": "UI automation"} +# Failing-suite display order: UI first (the RC-critical bucket), then instrumented, unit. +_SUITE_ORDER = {"ui": 0, "instrumented": 1, "unit": 2} + + +def sort_failed_suites(suites): + """Failing suites ordered UI-first then instrumented/unit, each by descending failure + count. One helper so every RC renderer (plain email, HTML email, CLI report) lists + them identically.""" + return sorted(suites or [], + key=lambda s: (_SUITE_ORDER.get(s.get("category", "ui"), 9), + -(s.get("failed") or 0))) + def _rc_email_plain(model, ctx) -> str: """Plain-text form of the RC report email (fallback + logging).""" L = [] rid = model.get("release", "?") o = model.get("orchestrator") or {} - v = o.get("versions") or {} - vstr = ", ".join(f"{k} {v[k]}" for k in ("Common", "Msal", "Broker") if v.get(k)) or "n/a" + vstr = format_versions(o.get("versions"), fallback="n/a") L.append(f"Hi {ctx.get('owner', 'there')},") L.append("") L.append(f"The Release Candidate for {rid} has been built and RC testing has " @@ -297,9 +313,7 @@ def _rc_email_plain(model, ctx) -> str: fs = r.get("failed_stages") or [] if fs: L.append(f" Red stages ({len(fs)}): {', '.join(fs)}") - _ord = {"ui": 0, "instrumented": 1, "unit": 2} - for s in sorted((r.get("failed_suites") or []), - key=lambda s: (_ord.get(s.get("category", "ui"), 9), -s["failed"])): + for s in sort_failed_suites(r.get("failed_suites")): sr = _fail_rate(s["failed"], s["total"]) L.append(f" [{_CAT_LABEL.get(s.get('category', 'ui'), 'UI automation')}] " f"{s['name']} — {s['failed']}/{s['total']} failed ({sr}%):") @@ -337,8 +351,7 @@ def _rc_email_html(model, ctx) -> str: from steps.lib import templating as T rid = model.get("release", "?") o = model.get("orchestrator") or {} - v = o.get("versions") or {} - vstr = ", ".join(f"{k} {v[k]}" for k in ("Common", "Msal", "Broker") if v.get(k)) or "n/a" + vstr = format_versions(o.get("versions"), fallback="n/a") ch = model.get("checker") or {} park = ("parked at ‘Remove RC Tags’" if o.get("parked") else "gate cleared") @@ -392,9 +405,7 @@ def mrwp_card(prov): ("unit", "instrumented", "ui")) # Failing suites — UI first, then instrumented/unit; each tagged by category. - _ord = {"ui": 0, "instrumented": 1, "unit": 2} - suites = sorted((r.get("failed_suites") or []), - key=lambda s: (_ord.get(s.get("category", "ui"), 9), -s["failed"])) + suites = sort_failed_suites(r.get("failed_suites")) suite_html = "" for s in suites: sr = _fail_rate(s["failed"], s["total"]) @@ -540,10 +551,6 @@ def verify_mrwp(state, provider): tests : inject the test summary {total,passed,failed[,runs]} Returns a Done/Blocked outcome. """ - from orchestrator.outcomes import Done, Blocked - from steps.lib.mockctx import mock_input, MISSING - from tools import pipelines as P - label = f"MRWP {provider}" # 1) resolve the MRWP build id for this provider mid = mock_input("mrwp_id", MISSING) diff --git a/release-agent/steps/build_verify/orchestrator_health.py b/release-agent/steps/build_verify/orchestrator_health.py index 82a079c2..0059e8b4 100644 --- a/release-agent/steps/build_verify/orchestrator_health.py +++ b/release-agent/steps/build_verify/orchestrator_health.py @@ -57,7 +57,7 @@ def build(state): links = K.links_for(bid, "Release Orchestrator run") versions = {k: P._tag_value(tags, f"Next{k}Version") for k in ("Common", "Msal", "Broker")} - vstr = ", ".join(f"{k} {v}" for k, v in versions.items() if v) or "versions n/a" + vstr = P.format_versions(versions, fallback="versions n/a") stages = mock_input("stages", MISSING) if stages is MISSING: diff --git a/release-agent/tools/pipelines.py b/release-agent/tools/pipelines.py index 9df8b9a5..adf02aec 100644 --- a/release-agent/tools/pipelines.py +++ b/release-agent/tools/pipelines.py @@ -300,6 +300,16 @@ def stage_completion(stages): TEST_CATEGORIES = ("unit", "instrumented", "ui") _CATEGORY_LABEL = {"unit": "Unit", "instrumented": "Instrumented", "ui": "UI automation"} +_VERSION_KEYS = ("Common", "Msal", "Broker") + + +def format_versions(versions, fallback: str = "") -> str: + """'Common X, Msal Y, Broker Z' from a {Common,Msal,Broker} dict — fixed order, + blanks omitted. Returns `fallback` when nothing is set. One place so every report / + status render formats RC versions identically.""" + v = versions or {} + return ", ".join(f"{k} {v[k]}" for k in _VERSION_KEYS if v.get(k)) or fallback + def classify_test_run(name): """Bucket a test-run/suite name into one of THREE categories: From ea8ce7d62f84ebfab53c1f21e053870f84fc8ba0 Mon Sep 17 00:00:00 2001 From: p3dr0rv Date: Thu, 20 Aug 2026 23:16:52 +0100 Subject: [PATCH 71/82] =?UTF-8?q?release-agent:=20status=20table=20?= =?UTF-8?q?=E2=80=94=20show=20engine-run=20agent=20steps=20as=20automatic,?= =?UTF-8?q?=20not=20'Pending'?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit UX fix: the four Phase-2 verification steps are agent steps Scout runs itself in-process, but they rendered as 'Pending' (⬜) next to the scout rc_report step's 'Scout runs this — automatic' (🤖) — a confusing mixed message implying the human was waiting on the agent steps. New 'auto' display state for engine-run agent steps renders IDENTICALLY to the skill-run 'scout' state (🤖 'Scout runs this — automatic'). 'Pending' is now reserved for a human step queued behind a dependency, so it always means 'you'll act here'. Everything needing the user stays distinct (⏸ gate, 📌 reminder, ⛔ blocked). Render-only; no state-machine change. 189 tests pass (added a display-contract regression test). Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- release-agent/orchestrator/render.py | 10 +++++--- release-agent/orchestrator/status_views.py | 4 +++- release-agent/tests/test_engine.py | 27 ++++++++++++++++++++++ 3 files changed, 37 insertions(+), 4 deletions(-) diff --git a/release-agent/orchestrator/render.py b/release-agent/orchestrator/render.py index ed9bca9c..65687902 100644 --- a/release-agent/orchestrator/render.py +++ b/release-agent/orchestrator/render.py @@ -32,7 +32,7 @@ def step_detail(s: dict, limit: int = 160) -> str: links = s.get("links") or [] if not note and not links: state = s.get("state") - return "—" if state in ("pending", "scheduled", "reminder", "gate") else "" + return "—" if state in ("pending", "scheduled", "reminder", "gate", "auto") else "" text = str(note or "").strip() lead = next((ln.strip() for ln in text.splitlines() if ln.strip()), "") # Prefer STRUCTURED links (first-class, stored on state) over URL-in-prose. @@ -229,10 +229,14 @@ def attest_prompt_payload(chk: dict, release_id: str) -> dict: } _PHASE_ICON = {"done": "✅", "current": "⏸", "pending": "⬜", "scheduled": "🗓"} _STEP_ICON = {"done": "✅", "gate": "⏸", "reminder": "📌", "scheduled": "🗓", - "pending": "⬜", "skipped": "⏭️", "scout": "🤖", "blocked": "⛔"} + "pending": "⬜", "skipped": "⏭️", "scout": "🤖", "auto": "🤖", "blocked": "⛔"} _STEP_STATE_WORD = {"done": "Done", "gate": "Awaiting your approval", "reminder": "Do this — then mark done", "scheduled": "Not open yet", "pending": "Pending", "skipped": "Skipped", + # 'auto' (engine-run agent step) and 'scout' (skill-run MCP step) are BOTH + # Scout's automatic work — shown identically so the human sees one message + # (no action needed), never a confusing "Pending" next to "automatic". + "auto": "Scout runs this — automatic", "scout": "Scout runs this — automatic", "blocked": "Blocked — needs you"} @@ -410,7 +414,7 @@ def _phase_num(r: dict, phase_id: str): def _next_pending_step(r: dict): for s in r.get("current_steps", []): - if s["state"] in ("pending", "gate"): + if s["state"] in ("pending", "gate", "auto"): return s["name"] return None diff --git a/release-agent/orchestrator/status_views.py b/release-agent/orchestrator/status_views.py index c9efc2e9..2cbddb65 100644 --- a/release-agent/orchestrator/status_views.py +++ b/release-agent/orchestrator/status_views.py @@ -147,8 +147,10 @@ def _current_steps(self, current_phase_obj) -> list: s_state = "reminder" elif not phase_due: s_state = "scheduled" + elif s.get("owner") == "human" or s.get("attest") or self._is_reminder(s): + s_state = "pending" # a human step queued behind a dependency else: - s_state = "pending" + s_state = "auto" # an agent step Scout runs itself — no user action out.append({ "id": s["id"], "name": s["name"], "gate": bool(s.get("gate")), diff --git a/release-agent/tests/test_engine.py b/release-agent/tests/test_engine.py index 41add017..38c42d33 100644 --- a/release-agent/tests/test_engine.py +++ b/release-agent/tests/test_engine.py @@ -2189,6 +2189,33 @@ def test_active_phase_report_steps_carry_links(): assert ap and all("links" in s for s in ap["steps"]) +def test_agent_steps_render_as_automatic_not_pending(): + """UX contract: engine-run agent steps ('auto') read IDENTICALLY to skill-run scout + steps — 'Scout runs this — automatic', 🤖 — so a human never sees a confusing + 'Pending' next to 'automatic' for work that needs no action. 'Pending' stays distinct + (a human step still to come).""" + from orchestrator import render + assert render._STEP_STATE_WORD["auto"] == render._STEP_STATE_WORD["scout"] == \ + "Scout runs this — automatic" + assert render._STEP_ICON["auto"] == render._STEP_ICON["scout"] == "🤖" + assert render._STEP_STATE_WORD["pending"] != render._STEP_STATE_WORD["auto"] + + # And the classifier tags the (pending, engine-run) build_verify agent steps as 'auto', + # the scout email as 'scout', never 'pending'. Force Phases 0-1 done so build_verify is + # the current phase with its steps still pending. + from orchestrator.state import StepState + st, orch = _mock_orch({}, as_of="2026-07-09") # build_verify anchor CCD+1 → due + for pid in ("preflight", "ccd"): + ph = next(p for p in orch.config["phases"] if p["id"] == pid) + for s in ph["steps"]: + orch.state.set_step(pid, s["id"], StepState(status="done", by="test")) + orch.state.current_phase = "build_verify" # the engine sets this during `next` + steps = {s["id"]: s for s in orch.status_report()["current_steps"]} + for sid in ("checker_fired", "orchestrator_health", "mrwp_ecs", "mrwp_local"): + assert steps[sid]["state"] == "auto", (sid, steps[sid]["state"]) + assert steps["rc_report"]["state"] == "scout" + + def test_reconcile_retries_pure(): """reconcile_retries collapses per-attempt results by title: passed if any attempt passed; recovered if it also failed; failed only if it never passed; NA ignored.""" From 40a3641daab1fe605a491d2c901f44c58e31dfb5 Mon Sep 17 00:00:00 2001 From: p3dr0rv Date: Thu, 20 Aug 2026 23:34:48 +0100 Subject: [PATCH 72/82] =?UTF-8?q?release-agent:=20phase=20map=20=E2=80=94?= =?UTF-8?q?=20only=20the=20frontier=20shows=20as=20'current'=20(no=20false?= =?UTF-8?q?=202nd=20active=20phase)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Bug: _phase_map marked ANY phase with done>0 as 'current' (⏸). After an upstream reopen left stale progress in a later phase, TWO phases rendered as 'in progress' — e.g. Phase 2 (frontier, 4/5) AND Phase 3 (2/5 leftover). Confusing: looked like Phase 3 ran while Phase 2 was incomplete. Fix: derive the frontier (first-incomplete included phase) once and mark ONLY it 'current'; a later phase with stale partial progress renders 'pending' (its count still shows). Ordering preserved so a not-yet-due frontier stays 'scheduled'. current_phase_obj/current_step_name are now frontier-derived too (equal to the engine cursor during real holds — no behaviour change). Display-only; the engine already processed the frontier first and never re-ran downstream. 190 tests pass (added a regression reproducing the two-active-phase state). Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- release-agent/orchestrator/status_views.py | 18 +++++++++++++----- release-agent/tests/test_engine.py | 22 ++++++++++++++++++++++ 2 files changed, 35 insertions(+), 5 deletions(-) diff --git a/release-agent/orchestrator/status_views.py b/release-agent/orchestrator/status_views.py index 2cbddb65..1aed4fdc 100644 --- a/release-agent/orchestrator/status_views.py +++ b/release-agent/orchestrator/status_views.py @@ -89,6 +89,13 @@ def _phase_map(self): total = done = 0 current_phase_name = current_step_name = None current_phase_obj = None + # The authoritative CURRENT phase is the FRONTIER — the first included phase with + # incomplete steps — NOT merely "any phase with progress". Deriving it here (rather + # than trusting p_done > 0) means exactly ONE phase shows as current, even when an + # upstream reopen left stale progress in a later phase (which would otherwise render + # as a confusing second "in progress" phase). + frontier = self._current_phase() + frontier_id = frontier["id"] if frontier else None for idx, phase in enumerate(self.config["phases"]): if not self._phase_included(phase): continue @@ -96,17 +103,18 @@ def _phase_map(self): p_done = sum(1 for s in phase["steps"] if self.state.is_done(phase["id"], s["id"])) total += p_total done += p_done - is_current = self.state.current_phase == phase["id"] + is_current = phase["id"] == frontier_id due = self._phase_due(phase) opens = self._phase_anchor_date(phase) if p_total and p_done == p_total: state = "done" elif not due and p_done == 0: - state = "scheduled" - elif is_current or p_done > 0: + state = "scheduled" # not open yet — even if it's the frontier + elif is_current: state = "current" else: - state = "pending" + state = "pending" # not the frontier — a later phase, even if it has + # stale partial progress, is not "in progress" now if is_current: current_phase_name = phase["name"] current_phase_obj = phase @@ -120,7 +128,7 @@ def _phase_map(self): "opens_in_days": (opens - self.as_of).days if opens else None, }) for s in phase["steps"]: - if s["id"] == self.state.current_step and phase["id"] == self.state.current_phase: + if s["id"] == self.state.current_step and phase["id"] == frontier_id: current_step_name = s["name"] return phases, total, done, current_phase_name, current_phase_obj, current_step_name diff --git a/release-agent/tests/test_engine.py b/release-agent/tests/test_engine.py index 38c42d33..5b741308 100644 --- a/release-agent/tests/test_engine.py +++ b/release-agent/tests/test_engine.py @@ -974,6 +974,28 @@ def test_phase0_opens_on_ccd_minus_7(): assert st.is_done("preflight", "notice") +def test_only_frontier_phase_shows_current_despite_stale_downstream_progress(): + """Issue-B regression: exactly ONE phase (the frontier = first incomplete) renders as + 'current'. A LATER phase left with stale partial progress (e.g. after reopening an + upstream phase) must NOT also show as 'current' — it's 'pending'.""" + from orchestrator.state import StepState + st, orch = _mock_orch({}, as_of="2026-07-09") + # Phases 0-1 done; build_verify (Phase 2) incomplete = the frontier; bug_bash (Phase 3) + # carries stale progress (2 steps done) as if an upstream reopen rolled Phase 2 back. + for pid in ("preflight", "ccd"): + for s in next(p for p in orch.config["phases"] if p["id"] == pid)["steps"]: + orch.state.set_step(pid, s["id"], StepState(status="done", by="test")) + for sid in ("clone_plans", "coordinate"): + orch.state.set_step("bug_bash", sid, StepState(status="done", by="test")) + phases = {p["id"]: p for p in orch.status_report()["phases"]} + assert phases["build_verify"]["state"] == "current" and phases["build_verify"]["current"] + assert phases["bug_bash"]["state"] == "pending" # stale progress ≠ a 2nd current + assert phases["bug_bash"]["done"] == 2 # the leftover count still shows + assert not phases["bug_bash"]["current"] + # exactly one phase is current + assert sum(1 for p in phases.values() if p["state"] == "current") == 1 + + def test_phase_map_marks_scheduled(): st, orch = _ccd_orch("2026-06-28") orch.run_until_gate() From 44ed6ae7c731ab0a3d577b54b98a2dc63227ce10 Mon Sep 17 00:00:00 2001 From: p3dr0rv Date: Fri, 21 Aug 2026 00:32:27 +0100 Subject: [PATCH 73/82] Phase 2: status-aware RC verify + in-flight state (no false-block on in-progress runs) An MRWP RC run that is still notStarted/inProgress no longer blocks as an aborted pipeline. verify_mrwp now checks the run's overall status first (new pipelines.get_build_status) and returns a new InProgress outcome when it isn't completed - a pending stage on a live run means "not run YET", not aborted. Engine records this as a new non-blocking StepState status "in_flight": not added to pending_human, release stays running (no user action), drain returns a "waiting" hold, and in_flight_since is stamped for the upcoming 6h poller nudge. The step stays not-done so it re-runs on every next/poll until the run completes, then the normal gate applies. Display: in_flight renders as "RC running - Scout is polling" in CLI status + email digest. Adds 2 tests; suite 192 green. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- release-agent/orchestrator/engine.py | 19 +++++++++++- release-agent/orchestrator/outcomes.py | 15 ++++++++++ release-agent/orchestrator/render.py | 9 ++++-- release-agent/orchestrator/state.py | 2 +- release-agent/orchestrator/status_views.py | 5 ++++ release-agent/phases/stub_runner.py | 2 ++ release-agent/steps/build_verify/_common.py | 17 ++++++++++- release-agent/steps/lib/agent.py | 12 +++++--- release-agent/tests/test_engine.py | 33 +++++++++++++++++++++ release-agent/tools/pipelines.py | 19 ++++++++++++ 10 files changed, 124 insertions(+), 9 deletions(-) diff --git a/release-agent/orchestrator/engine.py b/release-agent/orchestrator/engine.py index ae4ac52d..fa86c654 100644 --- a/release-agent/orchestrator/engine.py +++ b/release-agent/orchestrator/engine.py @@ -423,6 +423,23 @@ def _run_auto_step(self, phase: dict, step: dict, block_holds: bool) -> NextActi with mockctx.active(self.mocks.get(f"{pid}.{step['id']}", {})): result = runner(pid, step, self.state) key = f"{pid}.{step['id']}" + # IN-FLIGHT: the step's underlying pipeline run is still executing — NOT a failure. + # Hold the phase as 'waiting on the pipeline' (no user action) and let the poller / + # tick re-run the step until the run completes. Stamp when we first saw it in-flight + # so a poller can send the 6h courtesy nudge. + if getattr(result, "in_flight", False): + prev = self.state.get_step(pid, step["id"]) + data = dict(getattr(prev, "data", {}) or {}) + data.setdefault("in_flight_since", _now()) + data["poll_in_min"] = getattr(result, "poll_in_min", 30) + self.state.set_step(pid, step["id"], + StepState(status="in_flight", note=result.action, by="agent", + links=list(getattr(result, "links", None) or []), + data=data)) + self.state.pending_human = [p for p in self.state.pending_human if p != key] + self.state.status = "running" + return NextAction(kind="waiting", phase=pid, step=step["id"], name=step["name"], + message=f"WAITING — {step['name']}: {result.action}") if not result.ok: self.state.set_step(pid, step["id"], StepState(status="blocked", note=result.action, by=result.by, @@ -455,7 +472,7 @@ def run_until_gate(self, max_steps: int = 500) -> list: for _ in range(max_steps): act = self.step_once(attempted) actions.append(act) - if act.kind in ("gate", "reminder", "scheduled", "complete", "readiness", "blocked", "halted"): + if act.kind in ("gate", "reminder", "scheduled", "complete", "readiness", "blocked", "halted", "waiting"): break return actions diff --git a/release-agent/orchestrator/outcomes.py b/release-agent/orchestrator/outcomes.py index 6106b157..84becb00 100644 --- a/release-agent/orchestrator/outcomes.py +++ b/release-agent/orchestrator/outcomes.py @@ -41,6 +41,21 @@ class Blocked: kind: str = "blocked" +@dataclass +class InProgress: + """An agent step whose underlying work is STILL RUNNING (not a failure, not done). + + Used by the Phase-2 MRWP verification when the RC pipeline run's overall status is + notStarted/inProgress: the step must NOT block as 'aborted' (a never-ran stage during + an in-flight run is just not-run-YET). The engine holds the phase as 'waiting on the + pipeline' — no user action — and a poller re-runs the step every `poll_in_min` minutes + until the run completes, at which point the normal Done/Blocked rules apply.""" + note: str = "" + links: list = field(default_factory=list) + poll_in_min: int = 30 + kind: str = "in_progress" + + @dataclass class NeedsHuman: prompt: str diff --git a/release-agent/orchestrator/render.py b/release-agent/orchestrator/render.py index 65687902..88558c74 100644 --- a/release-agent/orchestrator/render.py +++ b/release-agent/orchestrator/render.py @@ -229,7 +229,8 @@ def attest_prompt_payload(chk: dict, release_id: str) -> dict: } _PHASE_ICON = {"done": "✅", "current": "⏸", "pending": "⬜", "scheduled": "🗓"} _STEP_ICON = {"done": "✅", "gate": "⏸", "reminder": "📌", "scheduled": "🗓", - "pending": "⬜", "skipped": "⏭️", "scout": "🤖", "auto": "🤖", "blocked": "⛔"} + "pending": "⬜", "skipped": "⏭️", "scout": "🤖", "auto": "🤖", "blocked": "⛔", + "in_flight": "⏳"} _STEP_STATE_WORD = {"done": "Done", "gate": "Awaiting your approval", "reminder": "Do this — then mark done", "scheduled": "Not open yet", "pending": "Pending", "skipped": "Skipped", @@ -237,7 +238,10 @@ def attest_prompt_payload(chk: dict, release_id: str) -> dict: # Scout's automatic work — shown identically so the human sees one message # (no action needed), never a confusing "Pending" next to "automatic". "auto": "Scout runs this — automatic", - "scout": "Scout runs this — automatic", "blocked": "Blocked — needs you"} + "scout": "Scout runs this — automatic", "blocked": "Blocked — needs you", + # in_flight = the pipeline run is still executing. No user action — Scout + # is polling every 30 min and re-evaluates when the run completes. + "in_flight": "RC running — Scout is polling"} def _pipelines_line(r: dict) -> str: @@ -559,6 +563,7 @@ def _esc(s: str) -> str: "action": ("Your action", "#b54708", "#fffaeb"), "scout": ("Scout runs this", "#475467", "#f2f4f7"), "auto": ("Automatic — pending", "#475467", "#f2f4f7"), + "in_flight": ("⏳ RC running — polling", "#475467", "#f2f4f7"), } diff --git a/release-agent/orchestrator/state.py b/release-agent/orchestrator/state.py index 5a9389e1..513da2d5 100644 --- a/release-agent/orchestrator/state.py +++ b/release-agent/orchestrator/state.py @@ -74,7 +74,7 @@ def migrate_pipeline_runs(pr) -> dict: @dataclass class StepState: """Persisted state for a single step.""" - status: str = "pending" # pending | done | skipped | blocked + status: str = "pending" # pending | done | skipped | blocked | in_flight completed_at: Optional[str] = None note: Optional[str] = None by: Optional[str] = None # 'agent' (stub) or 'human' diff --git a/release-agent/orchestrator/status_views.py b/release-agent/orchestrator/status_views.py index 1aed4fdc..ce20b6ab 100644 --- a/release-agent/orchestrator/status_views.py +++ b/release-agent/orchestrator/status_views.py @@ -37,6 +37,7 @@ def _active_phase_report(self) -> Optional[dict]: stp = self.state.get_step(phase["id"], sid) s_done = self.state.is_done(phase["id"], sid) s_blocked = stp.status == "blocked" + s_inflight = stp.status == "in_flight" is_gate = bool(s.get("gate")) is_rem = self._is_reminder(s) is_scout = s.get("source") == "scout" @@ -45,6 +46,8 @@ def _active_phase_report(self) -> Optional[dict]: status = "done" elif s_blocked: status = "blocked" + elif s_inflight: + status = "in_flight" # pipeline run still executing — Scout polling elif is_gate: status = "approval" elif is_attest: @@ -147,6 +150,8 @@ def _current_steps(self, current_phase_obj) -> list: s_state = "done" elif rec.get("status") == "blocked": s_state = "blocked" # a step hit a real problem — needs the owner + elif rec.get("status") == "in_flight": + s_state = "in_flight" # pipeline run still executing — Scout is polling elif s["id"] == self.state.current_step and self.state.status == "holding_gate": s_state = "gate" elif is_scout: diff --git a/release-agent/phases/stub_runner.py b/release-agent/phases/stub_runner.py index 0081b528..8dc43d08 100644 --- a/release-agent/phases/stub_runner.py +++ b/release-agent/phases/stub_runner.py @@ -19,6 +19,8 @@ class StepResult: action: str # human-readable description of what happened / should happen by: str # 'agent' (stub did it) or 'human' (needs a person) links: list = None # optional [{name, url}] durable refs (wiki page, CG alerts) + in_flight: bool = False # True → the work is still RUNNING (poll again); not done, not blocked + poll_in_min: int = 30 # re-check cadence when in_flight def run_stub(phase_id: str, step: dict, state=None) -> StepResult: diff --git a/release-agent/steps/build_verify/_common.py b/release-agent/steps/build_verify/_common.py index 58bacdef..ba790f51 100644 --- a/release-agent/steps/build_verify/_common.py +++ b/release-agent/steps/build_verify/_common.py @@ -20,7 +20,7 @@ ORCH_REQUIRED_STAGES, ORCH_PARK_STAGE, format_versions, ) from orchestrator.state import migrate_pipeline_runs -from orchestrator.outcomes import Done, Blocked +from orchestrator.outcomes import Done, Blocked, InProgress from steps.lib.mockctx import mock_input, MISSING # Surfaced in every block reason so the engineer knows how to recover / escalate. @@ -575,6 +575,21 @@ def verify_mrwp(state, provider): links = links_for(mid, f"{label} run") + # 1.5) overall run status — an in-flight run is NOT a failure. If the MRWP run is + # still notStarted/inProgress, its un-run stages just haven't run YET; hold the step + # as in-flight and let the 30-min poller re-evaluate when it completes, instead of + # false-blocking it as an aborted release. `build_status` mock drives this in sim/tests; + # when stages are injected (no live call) we assume the run is complete. + bstatus = mock_input("build_status", MISSING) + if bstatus is MISSING and mock_input("stages", MISSING) is MISSING: + ok_s, bstatus, _bres, _bdetail = P.get_build_status(ORG, PROJECT, mid) + if not ok_s: + bstatus = None # status unknown → fall through to the stage rule (best-effort) + if bstatus not in (MISSING, None) and bstatus != "completed": + return InProgress( + f"{label} run {mid} is still running (status: {bstatus}) — Scout is polling " + f"every 30 min and will re-evaluate the RC when it completes.", links=links) + # 2) stage-completion rule stages = mock_input("stages", MISSING) if stages is MISSING: diff --git a/release-agent/steps/lib/agent.py b/release-agent/steps/lib/agent.py index 07d1b4c7..9223e140 100644 --- a/release-agent/steps/lib/agent.py +++ b/release-agent/steps/lib/agent.py @@ -15,21 +15,25 @@ """ from __future__ import annotations -from orchestrator.outcomes import Done, Blocked +from orchestrator.outcomes import Done, Blocked, InProgress from phases.stub_runner import StepResult def to_step_result(outcome) -> StepResult: - """Map a uniform Outcome to the engine's StepResult (agent steps only ever - return Done/Blocked — never NeedsSkill/NeedsHuman).""" + """Map a uniform Outcome to the engine's StepResult. Agent steps return Done/Blocked, + or InProgress when their underlying pipeline run is still executing (poll again).""" if isinstance(outcome, Done): return StepResult(ok=True, action=outcome.note, by=outcome.by, links=list(outcome.links or [])) if isinstance(outcome, Blocked): return StepResult(ok=False, action=outcome.reason, by="agent", links=list(outcome.links or [])) + if isinstance(outcome, InProgress): + return StepResult(ok=False, action=outcome.note, by="agent", + links=list(outcome.links or []), + in_flight=True, poll_in_min=outcome.poll_in_min) raise TypeError( - f"agent step returned {type(outcome).__name__}; expected Done or Blocked") + f"agent step returned {type(outcome).__name__}; expected Done/Blocked/InProgress") def legacy_run(build): diff --git a/release-agent/tests/test_engine.py b/release-agent/tests/test_engine.py index 5b741308..0a740af9 100644 --- a/release-agent/tests/test_engine.py +++ b/release-agent/tests/test_engine.py @@ -2005,6 +2005,39 @@ def test_build_verify_steps_pass_with_healthy_mocks(): assert "Tests:" in outs["mrwp_ecs"]["note"] and "1 red" in outs["mrwp_ecs"]["note"] +def test_build_verify_mrwp_in_flight_when_run_still_executing(): + """An MRWP run whose OVERALL status is still inProgress is NOT a failure — verify + returns in_progress (Scout polls + re-evaluates on completion) instead of blocking it + as an aborted pipeline. A pending stage on an in-flight run is 'not run YET'.""" + st, orch = _bv_state({"build_verify.mrwp_ecs": { + "mrwp_id": "999", "build_status": "inProgress"}}) + out = _bv_build(orch, st, "mrwp_ecs") + assert out["kind"] == "in_progress" + assert "still running" in out["note"] and "poll" in out["note"].lower() + assert out["poll_in_min"] == 30 + # a completed run still runs the normal rule (control): injected stages, no in-flight + st2, orch2 = _bv_state({}) + assert _bv_build(orch2, st2, "mrwp_ecs")["kind"] == "done" + + +def test_engine_in_flight_step_holds_without_flagging_owner(): + """The engine records an in-flight agent step as status 'in_flight' (not blocked): + it is NOT added to pending_human, the release stays 'running' (no user action), the + drain returns a 'waiting' action, and first-seen time is stamped for the 6h nudge.""" + st, orch = _bv_state({"build_verify.mrwp_ecs": { + "mrwp_id": "999", "build_status": "inProgress"}}) + phase = next(p for p in orch.config["phases"] if p["id"] == "build_verify") + step = next(s for s in phase["steps"] if s["id"] == "mrwp_ecs") + act = orch._run_auto_step(phase, step, block_holds=True) + assert act.kind == "waiting" + rec = st.get_step("build_verify", "mrwp_ecs") + assert rec.status == "in_flight" + assert "build_verify.mrwp_ecs" not in st.pending_human + assert st.status == "running" + assert rec.data.get("in_flight_since") and rec.data.get("poll_in_min") == 30 + assert not st.is_done("build_verify", "mrwp_ecs") # not done → re-runs on next poll + + def test_build_verify_mrwp_blocks_on_never_ran_stage(): """An MRWP run with a skipped/pending stage blocks (aborted pipeline), with the recovery TSG + escalation in the reason.""" diff --git a/release-agent/tools/pipelines.py b/release-agent/tools/pipelines.py index adf02aec..7853c9be 100644 --- a/release-agent/tools/pipelines.py +++ b/release-agent/tools/pipelines.py @@ -260,6 +260,25 @@ def named_record(records, name, types=("Job", "Phase", "Stage")): return None +def get_build_status(org, project, build_id, timeout=60): + """Return (ok, status, result, detail) for a build's OVERALL run. + + status ∈ {notStarted, inProgress, completed, cancelling, postponed, none} + result ∈ {succeeded, partiallySucceeded, failed, canceled, none} (only meaningful + once status == 'completed'). + + This is the Phase-2 completion signal: a run is DONE only when status == 'completed'. + While it's notStarted/inProgress the verify step must treat un-run stages as + 'not run YET' (in-flight), NOT as an aborted release.""" + ok, data, detail = _az_json( + ["pipelines", "build", "show", "--org", org, "--project", project, + "--id", str(build_id), "--query", "{status:status,result:result}"], timeout) + if not ok: + return (False, None, None, detail) + d = data or {} + return (True, d.get("status"), d.get("result"), "") + + def get_stages(org, project, build_id, timeout=60): """Return (ok, stages, detail). `stages` is an ORDER-sorted list of {name, state, result} from the build's timeline (Stage records only).""" From d0624515b4d55a91f66fdc59e4e0c16e64822ba7 Mon Sep 17 00:00:00 2001 From: p3dr0rv Date: Fri, 21 Aug 2026 00:36:15 +0100 Subject: [PATCH 74/82] Phase 2: `rc-retriggered` signal command (reopen RC steps for the newest RC) When the human explicitly signals a NEW RC was triggered - a flaky-suite re-run, or the orchestrator re-running RC testing after a broker cherry-pick - `rc-retriggered --release [--reason]` reopens the two MRWP verifies + rc_report (checker_fired / orchestrator_health are left intact; a re-triggered RC re-runs MRWP against the same orchestrator run). mrwp_run_ids already picks the highest (newest) run id, so the engine re-resolves the new RC and re-applies the gate on the next tick/poll; the status-aware verify holds while it is still in-flight, so an early poll cannot false-fail an in-progress RC. Clears the reopened steps from pending_human, flips status back to running, and journals an `rc_retriggered` event. Adds a test; suite 193 green. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../orchestrator/commands/release.py | 50 +++++++++++++++++++ release-agent/tests/test_engine.py | 30 +++++++++++ 2 files changed, 80 insertions(+) diff --git a/release-agent/orchestrator/commands/release.py b/release-agent/orchestrator/commands/release.py index db422ee0..478fea2c 100644 --- a/release-agent/orchestrator/commands/release.py +++ b/release-agent/orchestrator/commands/release.py @@ -195,6 +195,47 @@ def cmd_reopen(args): return 0 +# The Phase-2 RC-testing steps a re-triggered RC invalidates: the two MRWP verifications +# and the terminal RC report/gate. checker_fired / orchestrator_health are NOT reopened — +# a re-triggered RC re-runs MRWP against the same orchestrator run. +_RC_RETRIGGER_STEPS = ("mrwp_ecs", "mrwp_local", "rc_report") + + +def cmd_rc_retriggered(args): + """The human explicitly signals that a NEW RC has been triggered (a flaky-run + re-trigger, or the orchestrator re-running RC testing after a broker cherry-pick). + + Reopens the Phase-2 RC-testing steps so the engine re-resolves the NEWEST MRWP run + (mrwp_run_ids already picks the highest id) and re-applies the gate. Scout's poller / + next then holds while the new run is in-flight and re-evaluates on completion — so an + early poll can't mark an in-progress RC as a false failure.""" + st, orch = C.load_orch(args.runs_root, args.release, args.config) + reason = (args.reason or "RC re-triggered").strip() + reopened = [] + for sid in _RC_RETRIGGER_STEPS: + act = orch.reopen_step("build_verify", sid, reason) + if act.kind != "idle": + reopened.append(sid) + # a reopened step is no longer an owner action / block + key = f"build_verify.{sid}" + st.pending_human = [p for p in st.pending_human if p != key] + if not reopened: + print("No Phase-2 RC steps found to reopen (is this release in Build & RC " + "Verification?).") + return 1 + if st.status in ("awaiting_action", "holding_gate", "complete", "halted"): + st.status = "running" + C.save_state(st, args.runs_root, args.release) + C.elog(args.runs_root, args.release).log( + "rc_retriggered", driver=reason, steps=",".join(reopened)) + msg = (f"RC re-trigger acknowledged — reopened {', '.join(reopened)}. Scout will " + f"re-resolve the newest RC and re-apply the gate; it holds (no action needed) " + f"while the run is still in-flight and polls every 30 min. Reason: {reason}") + C.emit(args.runs_root, args.release, msg, kind="override") + print(msg) + return 0 + + def cmd_halt(args): st, orch = C.load_orch(args.runs_root, args.release, args.config) act = orch.halt(args.reason or "") @@ -293,6 +334,15 @@ def register(sub): ro.add_argument("--reason", default="", help="Why (optional)") ro.set_defaults(func=cmd_reopen) + rt = sub.add_parser("rc-retriggered", + help="Signal a NEW RC was triggered — reopens the Phase-2 RC steps " + "so Scout re-evaluates the newest RC (holds while in-flight)") + rt.add_argument("--release", required=True) + rt.add_argument("--reason", default="", + help="Why it was re-triggered (e.g. 'flaky broker suite re-run' or " + "'broker cherry-pick #123') — recorded for audit") + rt.set_defaults(func=cmd_rc_retriggered) + ht = sub.add_parser("halt", help="Emergency hold — nothing advances until resume (reason REQUIRED)") ht.add_argument("--release", required=True) ht.add_argument("--reason", required=True, help="Why (audit — required)") diff --git a/release-agent/tests/test_engine.py b/release-agent/tests/test_engine.py index 0a740af9..36f97203 100644 --- a/release-agent/tests/test_engine.py +++ b/release-agent/tests/test_engine.py @@ -749,6 +749,36 @@ def test_reopen_step(): assert st.current_step == "bash_done" # gate re-holds +def test_rc_retriggered_reopens_phase2_rc_steps(): + """`rc-retriggered` reopens the two MRWP verifies + rc_report so Scout re-evaluates the + NEWEST RC, clears them from pending_human, and flips status back to running — while + leaving checker_fired / orchestrator_health (which a re-triggered RC doesn't invalidate) + untouched.""" + import tempfile, argparse + from orchestrator.commands import release as R + from orchestrator.state import StepState + from orchestrator import cli_common as _C + st = ReleaseState(release_id="2026-08", ccd="2026-08-26", ccd_source="confirmed") + for sid in ("checker_fired", "orchestrator_health", "mrwp_ecs", "mrwp_local"): + st.set_step("build_verify", sid, StepState(status="done")) + st.set_step("build_verify", "rc_report", StepState(status="blocked", note="UI 88%")) + st.pending_human = ["build_verify.rc_report"] + st.status = "awaiting_action" + with tempfile.TemporaryDirectory() as d: + _C.save_state(st, d, "2026-08") + ns = argparse.Namespace(runs_root=d, release="2026-08", config=CONFIG, + reason="flaky broker suite re-run") + assert R.cmd_rc_retriggered(ns) == 0 + again = _C.load_state(d, "2026-08") + assert not again.is_done("build_verify", "mrwp_ecs") + assert not again.is_done("build_verify", "mrwp_local") + assert again.get_step("build_verify", "rc_report").status == "pending" + assert again.is_done("build_verify", "checker_fired") # untouched + assert again.is_done("build_verify", "orchestrator_health") # untouched + assert "build_verify.rc_report" not in again.pending_human + assert again.status == "running" + + def test_halt_blocks_then_resume(): st, orch = _orch() orch.halt("prod incident") From 012bad9dd273ec8ae65f726738a604e12dbb891b Mon Sep 17 00:00:00 2001 From: p3dr0rv Date: Fri, 21 Aug 2026 00:41:29 +0100 Subject: [PATCH 75/82] Phase 2: 30-min RC poller + 6h courtesy nudge (poll-rc + on-demand automation) Adds the poller that watches an in-flight re-triggered RC and drives it to a verdict: * poll-rc command: advances the drain (in-flight verify re-checks the run's live status), then emits a deterministic decision - waiting / nudge / resolved / blocked / idle. A run in-flight past NUDGE_AFTER_HOURS (6h) yields ONE courtesy heads-up to the owner (email + Teams text), stamped nudged_at so it never repeats. Not a failure - Scout keeps polling. * build-verify-rc-poller automation (config/automations.yaml): a 30-min interval poller driving build_verify.rc_report, marked on_demand so it is provisioned ONLY when an RC is re-triggered (not at release start) and torn down when poll-rc reports resolved. plan() now surfaces on_demand. * rc_report.automation_prompt: the bespoke poll instruction (act on the decision; deregister on resolved), owned by the step like localization's. Adds 3 tests (waiting->nudge->dedupe, resolved/blocked/idle, plan shape); suite 196 green. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- release-agent/config/automations.yaml | 14 ++ release-agent/orchestrator/automations.py | 4 + .../orchestrator/commands/__init__.py | 3 +- .../orchestrator/commands/rc_poll.py | 129 ++++++++++++++++++ release-agent/steps/build_verify/rc_report.py | 26 ++++ release-agent/tests/test_engine.py | 80 +++++++++++ 6 files changed, 255 insertions(+), 1 deletion(-) create mode 100644 release-agent/orchestrator/commands/rc_poll.py diff --git a/release-agent/config/automations.yaml b/release-agent/config/automations.yaml index cadaa09a..a36903d3 100644 --- a/release-agent/config/automations.yaml +++ b/release-agent/config/automations.yaml @@ -50,3 +50,17 @@ automations: - ccd.localization # poll the triggered run (check-localization) purpose: "Poll the localization run every 10 min; email on 3h timeout, post the PR on completion" + # ON-DEMAND (not provisioned at release start). Created only when Build & RC + # Verification is holding on an in-flight re-triggered RC (the human ran + # `rc-retriggered`); torn down when `poll-rc` reports `resolved`. Interval automation, + # so its step needs no fire_at_local. + - slug: build-verify-rc-poller + name: "Release {release} — RC verification poller" + phase: build_verify + every: "30 minutes" + on_demand: true + steps: + - build_verify.rc_report # poll the in-flight RC (poll-rc): re-gate on completion, nudge at 6h + purpose: "Poll a re-triggered RC every 30 min; hold while in-flight, re-apply the gate on completion, courtesy-nudge the owner at 6h" + + diff --git a/release-agent/orchestrator/automations.py b/release-agent/orchestrator/automations.py index bb70d070..72622438 100644 --- a/release-agent/orchestrator/automations.py +++ b/release-agent/orchestrator/automations.py @@ -210,6 +210,10 @@ def plan(config_path: str, release: str, ccd: str) -> dict: "schedule": sched, # one-shot on the CCD date, or an interval poller "one_shot": one_shot, "interval": interval or None, + # ON-DEMAND automations (e.g. the RC poller) are NOT provisioned at release + # start — the skill creates them only when their trigger condition arises + # (an in-flight re-triggered RC) and tears them down when it clears. + "on_demand": bool(d.get("on_demand")), } spec["prompt"] = _prompt_for(spec, release) # Exactly what to record after creating it, so linkage + schedule are captured diff --git a/release-agent/orchestrator/commands/__init__.py b/release-agent/orchestrator/commands/__init__.py index 18d34009..1ce78330 100644 --- a/release-agent/orchestrator/commands/__init__.py +++ b/release-agent/orchestrator/commands/__init__.py @@ -6,7 +6,7 @@ one, so adding a command is a localized change (new/edited module only). """ from . import (release, readiness, pipeline, notify, infra_cmd, automation, - logs, lockdown, notice, step_action, localization, rc_report, sim) + logs, lockdown, notice, step_action, localization, rc_report, rc_poll, sim) # Order controls how subcommands appear in --help. REGISTRARS = [ @@ -19,6 +19,7 @@ notice.register, localization.register, rc_report.register, + rc_poll.register, sim.register, logs.register, automation.register, diff --git a/release-agent/orchestrator/commands/rc_poll.py b/release-agent/orchestrator/commands/rc_poll.py new file mode 100644 index 00000000..124ef6d1 --- /dev/null +++ b/release-agent/orchestrator/commands/rc_poll.py @@ -0,0 +1,129 @@ +"""`poll-rc` — one poll of an in-flight Phase-2 RC verification (the 30-min RC poller). + +After a re-triggered RC (see `rc-retriggered`), the Build & RC Verification phase holds +on an IN-FLIGHT MRWP run (status-aware verify — see steps/build_verify/_common). This +command is the poller seam the `build-verify-rc-poller` automation calls every 30 min: + + 1. advance the drain (`run_until_gate`) so the in-flight verify step re-checks the run's + live status — still running → stays in-flight; completed → the normal stage rule + + UI gate apply and the phase moves on. + 2. emit a deterministic decision the skill acts on: + waiting — still running; nothing to send. + nudge — running past the 6h courtesy threshold; send the owner a heads-up (once). + resolved — the new RC completed and PASSED the gate; Phase 2 advanced (deregister + the poller). + blocked — the new RC completed but re-blocked the gate (still failing). + idle — nothing in-flight (not in Phase 2, or nothing was re-triggered). + +Decisions are pure functions of state; the 6h nudge stamps `nudged_at` on the step so it +is sent at most once. `--now` overrides the clock for the elapsed/nudge math (tests).""" +from __future__ import annotations +import json as _json +from datetime import datetime, timezone + +from orchestrator import cli_common as C + +# The poll cadence + courtesy-nudge threshold. A re-triggered RC that runs longer than +# NUDGE_AFTER_HOURS gets ONE heads-up to the owner (it is not a failure — Scout keeps +# polling), per the agreed Phase-2 blocked-state handling. +POLL_INTERVAL_MIN = 30 +NUDGE_AFTER_HOURS = 6 + +# The verify steps whose run can be in-flight (checker/orchestrator resolve instantly). +_RC_VERIFY_STEPS = ("mrwp_ecs", "mrwp_local") + + +def _parse_iso(s): + try: + return datetime.fromisoformat(str(s).replace("Z", "+00:00")) + except (ValueError, TypeError): + return None + + +def _elapsed_hours(since_iso, now): + since = _parse_iso(since_iso) + if not since: + return 0.0 + if since.tzinfo is None: + since = since.replace(tzinfo=timezone.utc) + return max(0.0, (now - since).total_seconds() / 3600.0) + + +def _nudge_payload(st, sid, hrs: int) -> dict: + """A SHORT courtesy heads-up (not the full RC report) — the re-triggered RC is taking + a while but Scout is still polling; no action needed yet.""" + label = {"mrwp_ecs": "MRWP (ECS)", "mrwp_local": "MRWP (Local)"}.get(sid, sid) + subject = f"[Release {st.release_id}] Re-triggered RC still running after ~{hrs}h" + body = ( + f"Heads-up: the re-triggered {label} run for release {st.release_id} has been " + f"running for about {hrs} hours. This is NOT a failure — Scout is still polling " + f"every {POLL_INTERVAL_MIN} minutes and will re-apply the RC gate the moment the " + f"run completes, with no action needed from you. If a {hrs}h RC run is unexpected, " + f"open the run in ADO to check for a stuck stage.") + teams = (f"⏳ Release {st.release_id}: the re-triggered {label} RC has been running " + f"~{hrs}h. Scout is still polling every {POLL_INTERVAL_MIN}m and re-applies " + f"the gate on completion — no action needed yet.") + return { + "email": {"to": [st.owner_email] if st.owner_email else [], + "subject": subject, "body": body}, + "teams": {"text": teams}, + } + + +def cmd_poll_rc(args): + now = _parse_iso(args.now) if getattr(args, "now", None) else datetime.now(timezone.utc) + if now is None: + print(_json.dumps({"error": f"bad --now: {args.now!r}"})) + return 1 + + st, orch = C.load_orch(args.runs_root, args.release, args.config, C.parse_as_of(args)) + # Advance: the in-flight verify step re-checks the run's LIVE status. Still running → + # stays in-flight; completed → the stage rule + UI gate run and the phase moves on. + orch.run_until_gate() + st = orch.state + C.save_state(st, args.runs_root, args.release) + + inflight = None + for sid in _RC_VERIFY_STEPS: + s = st.get_step("build_verify", sid) + if s.status == "in_flight": + inflight = (sid, s) + break + + if inflight: + sid, s = inflight + elapsed = _elapsed_hours(s.data.get("in_flight_since"), now) + decision = {"decision": "waiting", "step": sid, + "elapsed_hours": round(elapsed, 2), "poll_in_min": POLL_INTERVAL_MIN} + if elapsed >= NUDGE_AFTER_HOURS and not s.data.get("nudged_at"): + s.data["nudged_at"] = now.isoformat() + st.set_step("build_verify", sid, s) + C.save_state(st, args.runs_root, args.release) + decision["decision"] = "nudge" + decision["nudge"] = _nudge_payload(st, sid, int(elapsed)) + C.emit(args.runs_root, args.release, + f"[rc-poller] {sid} in-flight ~{int(elapsed)}h — 6h courtesy nudge sent " + f"to the owner.", kind="build_verify") + else: + rc = st.get_step("build_verify", "rc_report") + if rc.status == "done": + decision = {"decision": "resolved", "status": "passed", "note": rc.note} + elif rc.status == "blocked": + decision = {"decision": "blocked", "note": rc.note} + else: + decision = {"decision": "idle", + "note": "no in-flight RC in Build & RC Verification"} + + print(_json.dumps(decision)) + return 0 + + +def register(sub): + p = sub.add_parser("poll-rc", + help="One poll of an in-flight Phase-2 RC: advance + emit a " + "waiting/nudge/resolved/blocked/idle decision (30-min poller)") + p.add_argument("--release", required=True) + p.add_argument("--now", default=None, + help="Override 'now' (ISO-8601) for the elapsed / 6h-nudge math") + p.add_argument("--as-of", default=None, help="Simulated clock (YYYY-MM-DD); default today") + p.set_defaults(func=cmd_poll_rc) diff --git a/release-agent/steps/build_verify/rc_report.py b/release-agent/steps/build_verify/rc_report.py index 87252ee9..45bf6e01 100644 --- a/release-agent/steps/build_verify/rc_report.py +++ b/release-agent/steps/build_verify/rc_report.py @@ -82,3 +82,29 @@ def build(state): note=gate["detail"], outbound=True, ) + + +def automation_prompt(release: str, spec: dict) -> str: + """Bespoke instruction for the interval RC poller (owned here, like localization's). + Only the poller shape is used — rc_report has no time-of-day automation.""" + if not spec.get("interval"): + return "" # rc_report is driven by next/tick, not a one-shot automation + return ( + f"Release {release} — RC verification poller (Phase 2, every 30 min).\n" + f"Only act if Build & RC Verification is holding on an IN-FLIGHT re-triggered RC " + f"(the human ran `rc-retriggered`). Poll it once:\n" + f"1. run `poll-rc --release {release}`.\n" + f"2. act on the printed decision:\n" + f" • waiting → still running; send nothing.\n" + f" • nudge → running past 6h; send the courtesy heads-up in decision.nudge " + f"(email decision.nudge.email to the owner AND post decision.nudge.teams.text to " + f"the owner's Scout chat). It is stamped, so it goes out at most once.\n" + f" • resolved → the new RC completed and PASSED the gate; Phase 2 advanced. " + f"Deregister THIS poller (`automation deregister --id `) — " + f"it is no longer needed — and report the pass.\n" + f" • blocked → the new RC completed but re-blocked the UI gate (still failing). " + f"Surface the block to the owner (the 3-exit choice: re-trigger / cherry-pick / " + f"override) and leave the poller in place for the next re-trigger.\n" + f" • idle → nothing in-flight; stay silent.\n" + f"Silently journal: `journal --release {release} --source scout --kind automation " + f"--text \"rc-poller: \"`. Stay silent when there is nothing to send.") diff --git a/release-agent/tests/test_engine.py b/release-agent/tests/test_engine.py index 36f97203..a740a002 100644 --- a/release-agent/tests/test_engine.py +++ b/release-agent/tests/test_engine.py @@ -2561,6 +2561,86 @@ def test_mrwp_run_ids_picks_newest_on_retrigger(): assert ok2 and ids2 == {"ECS": "5", "Local": "6"} +def _run_poll_rc(runs_root, rid, now_iso): + """Run `poll-rc` with the drain stubbed (no live az) and return the decision dict.""" + import io, contextlib, json as _json + from unittest.mock import patch + from orchestrator.commands import rc_poll as RP + + class A: + runs_root = None; release = None; config = CONFIG; as_of = None; now = None + A.runs_root, A.release, A.now = runs_root, rid, now_iso + buf = io.StringIO() + with patch("orchestrator.engine.Orchestrator.run_until_gate", lambda self, **k: []): + with contextlib.redirect_stdout(buf): + rc = RP.cmd_poll_rc(A) + assert rc == 0 + return _json.loads(buf.getvalue().strip().splitlines()[-1]) + + +def test_poll_rc_waits_then_nudges_once_at_6h(): + """The RC poller returns `waiting` while the run is in-flight, sends ONE courtesy + nudge to the owner once it has been in-flight >= 6h, and does not repeat the nudge.""" + import tempfile + from orchestrator.state import StepState + with tempfile.TemporaryDirectory() as d: + rid = "2026-08" + _stub_build_defs("pass") + st = ReleaseState(release_id=rid, ccd="2026-08-26", ccd_source="confirmed", + owner_email="dev@microsoft.com", owner_name="Dev") + st.set_step("build_verify", "mrwp_ecs", + StepState(status="in_flight", note="RC running", + data={"in_flight_since": "2026-08-20T00:00:00+00:00", + "poll_in_min": 30})) + C.save_state(st, d, rid) + # +2h → still waiting + dec = _run_poll_rc(d, rid, "2026-08-20T02:00:00+00:00") + assert dec["decision"] == "waiting" and dec["step"] == "mrwp_ecs" + assert abs(dec["elapsed_hours"] - 2.0) < 0.01 and dec["poll_in_min"] == 30 + # +7h → nudge (once), addressed to the owner + dec = _run_poll_rc(d, rid, "2026-08-20T07:00:00+00:00") + assert dec["decision"] == "nudge" + assert dec["nudge"]["email"]["to"] == ["dev@microsoft.com"] + assert "polling" in dec["nudge"]["teams"]["text"] + # nudged_at stamped → a later poll is waiting again (no repeat nudge) + dec = _run_poll_rc(d, rid, "2026-08-20T09:00:00+00:00") + assert dec["decision"] == "waiting" + + +def test_poll_rc_resolved_blocked_idle(): + """Once no verify step is in-flight, the poller reports the terminal verdict: + resolved (rc_report done) / blocked (rc_report blocked) / idle (nothing running).""" + import tempfile + from orchestrator.state import StepState + with tempfile.TemporaryDirectory() as d: + rid = "2026-08" + _stub_build_defs("pass") + st = ReleaseState(release_id=rid, ccd="2026-08-26", ccd_source="confirmed") + C.save_state(st, d, rid) + assert _run_poll_rc(d, rid, "2026-08-20T09:00:00+00:00")["decision"] == "idle" + + st.set_step("build_verify", "rc_report", StepState(status="done", note="UI CLEAN")) + C.save_state(st, d, rid) + assert _run_poll_rc(d, rid, "2026-08-20T09:00:00+00:00")["decision"] == "resolved" + + st.set_step("build_verify", "rc_report", StepState(status="blocked", note="UI 80%")) + C.save_state(st, d, rid) + r = _run_poll_rc(d, rid, "2026-08-20T09:00:00+00:00") + assert r["decision"] == "blocked" and r["note"] == "UI 80%" + + +def test_rc_poller_automation_is_on_demand_interval(): + """The build-verify-rc-poller is planned as an on-demand 30-min interval automation + driving rc_report, with a bespoke poll prompt (not the default step-action prompt).""" + from orchestrator import automations as A + assert A.validate(CONFIG) == [] + plan = A.plan(CONFIG, "2026-08", "2026-08-26") + rc = next(a for a in plan["automations"] if a["slug"] == "build-verify-rc-poller") + assert rc["on_demand"] and rc["interval"] == "30 minutes" + assert rc["steps"] == ["build_verify.rc_report"] + assert "poll-rc --release 2026-08" in rc["prompt"] and "6h" in rc["prompt"] + + def test_build_verify_persists_pipeline_run_ids(): """The build_verify steps stash the checker/orchestrator/MRWP runs onto state.pipeline_runs in the nested RC schema, and it round-trips through save/load.""" From b519bd062daf2b43882a6e886419d5c80ca98d04 Mon Sep 17 00:00:00 2001 From: p3dr0rv Date: Fri, 21 Aug 2026 00:45:17 +0100 Subject: [PATCH 76/82] Phase 2: three-exit blocked message for the <90% RC UI gate The rc_report `attention` block now spells out THREE exits instead of a vague "fix or override", matching the agreed Phase-2 blocked-state handling: 1. Re-trigger (flaky) - re-run the failed RC test run, then `rc-retriggered --release `; Scout tracks the newest RC (holds in-flight, polls 30m, re-gates on completion). 2. Cherry-pick (real bug) - patch via the broker cherry-pick process (link surfaced), then `rc-retriggered` so Scout tracks the fresh RC to completion. 3. Override (LAST RESORT) - `skip ... --reason`, framed explicitly as a team decision to be discussed first (proceeding to Bug Bash with this many UI failures is not a default). Adds CHERRY_PICK_TSG. Updates the skill docs to present all three (never collapse to two): reference/phases/build_verify.md (3-exit flow + in-flight vs blocked), SKILL.md block handling exception, reference/commands.md (rc-retriggered + poll-rc). Strengthens the gate test; suite 196 green. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- release-agent/skill/SKILL.md | 2 +- release-agent/skill/reference/commands.md | 2 ++ .../skill/reference/phases/build_verify.md | 31 ++++++++++++++++--- release-agent/steps/build_verify/_common.py | 30 +++++++++++++++--- release-agent/tests/test_engine.py | 4 +++ 5 files changed, 58 insertions(+), 11 deletions(-) diff --git a/release-agent/skill/SKILL.md b/release-agent/skill/SKILL.md index b0f207a4..050e0db0 100644 --- a/release-agent/skill/SKILL.md +++ b/release-agent/skill/SKILL.md @@ -36,7 +36,7 @@ Discover → (if no gate cleared, run the entry gate) → `next` to advance → - **Engine HOLDS at a gate:** present it; `m_ask_user` Approve/Deny; run `approve`/`deny --comment` with their reason; present new status. - **Scout steps pending** (`scout_pending` non-empty in `status`): these are **Scout's automated work, NOT a user to-do** — run them yourself, don't wait for the user and don't present them as "you need to". For EACH id in `scout_pending`: `step-action --release --phase

    --step ` → it returns `needs_skill` (an email/Teams/browser action) → perform the returned `tool`+`payload` (respect any `test_redirect`) → `record-step --step --status pass` (or the step's follow-up, e.g. `check-lockdown`). Do this **silently** for each, then re-run `next`. Only once `scout_pending` is empty do you surface the remaining user holds below. (A scout step that records `attention` becomes a `blocked` user task — handle it like any block.) - **Engine HOLDS for a reminder** (`awaiting_action` with `action`/`needs_owner` — an attest or blocked USER task): present as "you need to do X"; when done, `done --release --note ""`. Not a decision — no Approve/Deny. -- **A step is BLOCKED** (agent found a real problem, e.g. `cg` on High/Critical CG alerts, `cron` on a stale Calendar Checker): show the note plainly. Two exits: **(a) fix** → `next` re-runs the check; **(b) override** → `skip --release --phase

    --step --reason ""`. No other way to clear it. +- **A step is BLOCKED** (agent found a real problem, e.g. `cg` on High/Critical CG alerts, `cron` on a stale Calendar Checker): show the note plainly. Two exits: **(a) fix** → `next` re-runs the check; **(b) override** → `skip --release --phase

    --step --reason ""`. No other way to clear it. **Exception — `build_verify.rc_report` (the <90% UI gate)** has a richer **three-exit** flow: **re-trigger** (flaky → re-run RC, then `rc-retriggered --release `), **cherry-pick** (real bug → patch via the broker cherry-pick process, then `rc-retriggered`), or **override** (`skip …`, the last resort — discuss with the team first). After `rc-retriggered`, Scout tracks the newest RC: the verify step is `in_flight` (⏳ no action) while it runs, and the 30-min poller re-applies the gate on completion. See `reference/phases/build_verify.md`. Present all three — don't collapse it to "fix or override". - **Engine is `scheduled`** (before CCD‑7): relay the opens-date + countdown; nothing to advance. Earlier start = a CCD change (`set-ccd`), not `next`. - **"continue"/"resume":** discover → if gate not cleared show the checklist, else brief with status → `next`. - **User asks ABOUT a step** ("what does X do?", "where do I find the Play Console vitals?", "how do I clear this block?", "why is this needed?", "who fixes this?"): run `step-info --phase

    --step ` and answer from it — do NOT guess step details from memory. It returns the step's what/who/where/how/links/FAQs (accurate, curated in `config/knowledge.yaml`). If it returns "no knowledge entry yet", say so rather than inventing an answer. **Then, if a release is active, silently journal the exchange:** `journal --release --kind qa --phase

    --step --question "" --answer ""` — best-effort, never announced, skip entirely when no release run exists. diff --git a/release-agent/skill/reference/commands.md b/release-agent/skill/reference/commands.md index f7e87faf..65f3c475 100644 --- a/release-agent/skill/reference/commands.md +++ b/release-agent/skill/reference/commands.md @@ -40,6 +40,8 @@ _Loaded on demand. Run all from `C:\repos\android-complete\release-agent`._ | Journal a step Q&A (silent) | `python -m orchestrator.cli journal --release --kind qa --phase

    --step --question "..." --answer "..."` | | Localization: record trigger | `python -m orchestrator.cli record-localization-run --release --build-id ` — store the queued build; leaves the step in-flight | | Localization: one poll | `python -m orchestrator.cli check-localization --release --complete [--logs ""]` — wait / timeout(email) / complete(post PR); acts on the printed decision | +| **Phase 2 — signal a re-triggered RC** | `python -m orchestrator.cli rc-retriggered --release [--reason "..."]` — after the owner re-runs RC (flaky) or the orchestrator triggers a fresh RC (broker cherry-pick), reopens `mrwp_ecs`/`mrwp_local`/`rc_report` so Scout re-evaluates the **newest** RC. Holds `in_flight` (no action) while the run executes; the poller re-applies the gate on completion | +| **Phase 2 — one RC poll** | `python -m orchestrator.cli poll-rc --release ` — advances the drain (in-flight verify re-checks live status) and prints a decision: `waiting` / `nudge` (>6h courtesy heads-up to the owner) / `resolved` (passed → deregister the poller) / `blocked` (re-gate failed) / `idle`. Driven by the on-demand `build-verify-rc-poller` automation (every 30 min) | | Activate conditional hotfix phase | `python -m orchestrator.cli activate --release --phase hotfix` | | **Notify** — push line if something needs me | `python -m orchestrator.cli notify [--release ] [--as-of ] [--force]` | | **Plan timed automations** | `automation plan --release [--json]` — derive the per-release CCD automations (name/schedule/steps/prompt) from `config/automations.yaml` + CCD | diff --git a/release-agent/skill/reference/phases/build_verify.md b/release-agent/skill/reference/phases/build_verify.md index 79e4b22e..19c87a97 100644 --- a/release-agent/skill/reference/phases/build_verify.md +++ b/release-agent/skill/reference/phases/build_verify.md @@ -12,6 +12,12 @@ orchestrator, an auth failure). A blocked step → show the note, then **fix + ` re-check, or **`skip … --reason`** to override. When the chain is green the scout `rc_report` step runs — it is the last Phase-2 step and the go/no-go. +**In-flight vs blocked.** An MRWP step whose RC run is still executing is **`in_flight`** +(⏳ "RC running — Scout is polling"), **not** blocked — a stage that hasn't run *yet* on a +live run is not an aborted pipeline. It needs **no owner action**: the engine holds the +phase and the 30-min poller re-checks until the run completes, then the normal stage rule ++ UI gate apply. Only a stage that never ran on a **completed** run blocks. + ## Automated steps (no skill action — relay from the `status` table) `checker_fired`, `orchestrator_health`, `mrwp_ecs`, `mrwp_local` — read-only `az` agent steps run inside `next`. Each records the ADO run it evaluated as a Details 🔗 link. @@ -34,11 +40,26 @@ This is the Phase-2 go/no-go — there is **no separate approval gate**. - **≥ 90% & < 100% → `warn`** — step done; auto-advances into bug bash, but the owner should investigate the failing UI tests **in parallel** (a later step confirms the retest — bug bash is **not** blocked). - - **< 90% → `attention`** — the step **BLOCKS** (`awaiting_action`). Large UI failure: the - owner investigates and decides — patch a real bug + re-trigger RC, or (if it's an - automation flake to re-run later) proceed to bug bash. Exits: fix + re-run, then - `next` re-runs `rc_report`; or `skip … --reason` to override. - - (No UI tests found → `clean` with a ⚠ note.) + - **< 90% → `attention`** — the step **BLOCKS** (`awaiting_action`). This is a large UI + failure. Present the note plainly, then walk the owner through **three exits** (do NOT + reduce it to "fix or override"): + 1. **Re-trigger (flaky)** — if the owner judges the failures are automation flakiness, + they re-run the failed RC test run, then signal **`rc-retriggered --release + --reason "..."`**. That reopens `mrwp_ecs`/`mrwp_local`/`rc_report` so Scout + re-evaluates the **newest** RC. While the new run is still executing the verify step + is **`in_flight`** (⏳ "RC running — Scout is polling") — **no owner action**; the + `build-verify-rc-poller` re-checks every 30 min and re-applies this gate the moment + the run completes. If it runs past 6h the owner gets one courtesy nudge. + 2. **Cherry-pick (real bug)** — if a product bug is driving the failures, the owner + patches it via the **broker cherry-pick process** + (`…/internal-release-checklist/cherry-pick-process-for-broker-libraries`); the + orchestrator then triggers a fresh RC. Same signal: **`rc-retriggered --release + `** so Scout tracks the newest RC to completion. + 3. **Override (LAST RESORT)** — **`skip … --step rc_report --reason ""`**. Frame + this explicitly as the last option: proceeding to Bug Bash with this many UI + failures is a **team decision** and should be **discussed with the team first**, not + taken as a default. The reason is recorded for audit. + (No UI tests found → `clean` with a ⚠ note.) It records the failing-suite summary + stashes the checker/orchestrator/ECS/Local run links on the step. - The command prints `{verdict, blocking, pass_pct, ui_total, detail, links}` for your diff --git a/release-agent/steps/build_verify/_common.py b/release-agent/steps/build_verify/_common.py index ba790f51..4a51c796 100644 --- a/release-agent/steps/build_verify/_common.py +++ b/release-agent/steps/build_verify/_common.py @@ -110,6 +110,13 @@ def stash_mrwp(state, provider, snapshot): # (a large UI failure usually means a real regression → fix + re-run MRWP). RC_UI_PASS_THRESHOLD = 90.0 +# The broker-libraries cherry-pick process — the exit for a REAL product bug behind the +# UI failures (patch → orchestrator triggers a fresh RC). Surfaced in the block detail. +CHERRY_PICK_TSG = ("https://eng.ms/docs/microsoft-security/identity/" + "entra-developer-application-platform/auth-client/" + "authn-sdk-msal-android/android-auth-libraries/releases/" + "internal-release-checklist/cherry-pick-process-for-broker-libraries") + # ---------------------------------------------------------------- RC report email def rc_report_model(state, timeout=120): @@ -222,11 +229,24 @@ def rc_ui_gate(model) -> dict: f"failing UI test(s) in parallel (a later step confirms the retest, " f"so bug bash is not blocked)." + _ui_failing_suites_summary(model))} return {**base, "pass_pct": pass_pct, "verdict": "attention", "blocking": True, - "detail": (f"{head} \u2014 BELOW the {thr:.0f}% gate. Large UI failure: investigate " - f"the root cause and decide \u2014 patch a real bug + re-trigger RC, or " - f"(if it's an automation flake to re-run later) proceed to bug bash. This " - f"step stays BLOCKED until you `next` after a re-run, or `skip --reason` " - f"to override." + _ui_failing_suites_summary(model))} + "detail": ( + f"{head} \u2014 BELOW the {thr:.0f}% gate. The RC report was emailed; the " + f"autonomous tick then halted here (it will NOT auto-advance while blocked). " + f"First decide whether this is automation flakiness or a real product bug, " + f"then take ONE of three exits:\n" + f"1) Re-trigger (flaky) \u2014 if these are flaky suites, re-run the failed RC " + f"test run, then signal `rc-retriggered --release --reason \"...\"`. Scout " + f"tracks the NEW RC: it holds (no action) while the run is in-flight, polls " + f"every 30 min, and re-applies this gate the moment it completes.\n" + f"2) Cherry-pick (real bug) \u2014 if a product bug is driving the failures, patch " + f"it via the broker cherry-pick process ({CHERRY_PICK_TSG}); the orchestrator " + f"then triggers a fresh RC. Signal `rc-retriggered --release ` so Scout " + f"tracks the newest RC to completion.\n" + f"3) Override (LAST RESORT) \u2014 `skip --release --phase build_verify " + f"--step rc_report --reason \"\"`. Only after discussing with the team: " + f"proceeding to Bug Bash with this many UI failures is a team decision, not a " + f"default. The reason is recorded for audit." + + _ui_failing_suites_summary(model))} def recovered_unit_tests(model) -> list: diff --git a/release-agent/tests/test_engine.py b/release-agent/tests/test_engine.py index a740a002..99acb537 100644 --- a/release-agent/tests/test_engine.py +++ b/release-agent/tests/test_engine.py @@ -2204,6 +2204,10 @@ def _model(ecs_ui, local_ui, ecs_suites=None): g2 = K.rc_ui_gate(fail_model) assert g2["verdict"] == "attention" and g2["blocking"] is True and g2["pass_pct"] == 80.0 assert "BELOW" in g2["detail"] and "PROD MSAL - RC Broker (API 32)" in g2["detail"] + # the three exits are spelled out: re-trigger (flaky) / cherry-pick (bug) / override + assert "rc-retriggered" in g2["detail"] + assert "cherry-pick-process-for-broker-libraries" in g2["detail"] + assert "LAST RESORT" in g2["detail"] and "skip" in g2["detail"] # no UI tests anywhere → clean with a warning (absence of data is not a failure) g3 = K.rc_ui_gate({"mrwp": {"ECS": {"tests": {"categories": {}}}, From 32ed96476a698110b503db9eba386d53dcc8bbc9 Mon Sep 17 00:00:00 2001 From: p3dr0rv Date: Fri, 21 Aug 2026 01:20:16 +0100 Subject: [PATCH 77/82] Registry per-release ownership + strip pre-prod legacy/back-compat code MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Relocate release-owned automations into their release folder and remove dead legacy scaffolding (this project is pre-production — no old on-disk state can exist): Registry ownership: * Release-scoped automations now live in //_automations.json, co-located with that release's release-state.json so ownership is explicit and they're removed with the release folder at close. The machine-wide /_automations.json now holds ONLY shared automations. AutomationRegistry takes an optional release; list/ register/deregister read+write the right file(s). Relocated the existing dev registry. Legacy removals (can't exist without production data): * state.migrate_pipeline_runs (flat->nested) + _versions_str_to_dict helper + its test — the flat pipeline_runs shape only briefly existed earlier in dev before the nested redesign; callers now read state.pipeline_runs directly. * ReleaseState.last_notified ("kept for load compat") — read nowhere. * automation sync's steps-fallback for entries "predating slug" — register always sets slug now, so match by slug only. Kept: ReleaseState.load unknown-key drop (genuine robustness for the unattended automation, not legacy). Docs updated. Suite 196 green. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- release-agent/README.md | 21 +++-- .../orchestrator/commands/automation.py | 15 +--- release-agent/orchestrator/registry.py | 88 ++++++++++++++----- release-agent/orchestrator/render.py | 6 +- release-agent/orchestrator/state.py | 67 ++------------ release-agent/steps/build_verify/_common.py | 7 +- release-agent/tests/test_engine.py | 39 ++++---- 7 files changed, 113 insertions(+), 130 deletions(-) diff --git a/release-agent/README.md b/release-agent/README.md index 1101ce30..9a6400e1 100644 --- a/release-agent/README.md +++ b/release-agent/README.md @@ -74,8 +74,9 @@ release-agent/ COMMITTED (distributed with android-complete) .release-runs// GENERATED, gitignored (per-release working state) ├─ release-state.json the per-release metadata + run-state (owner, CCD, steps, gates, …) -└─ events.jsonl the per-release event/interaction log -.release-runs/_automations.json GENERATED, gitignored — registry of provisioned Scout automations +├─ events.jsonl the per-release event/interaction log +└─ _automations.json registry of THIS release's provisioned Scout automations (owned by the release; removed at close) +.release-runs/_automations.json GENERATED, gitignored — registry of SHARED (machine-wide) automations only ``` ## Adding a step (the modular contract) @@ -95,7 +96,7 @@ That's it. Mocking works automatically (`outcome`/knobs); `step-info` shows its guardrail fails loudly if a module and `phases.yaml` drift, so nothing silently breaks. **Two homes for data (by lifetime):** -- **Release metadata + run-state** → `.release-runs//release-state.json` (per-release; the `ReleaseState` record). Holds `owner_email`/`owner_name` (the release owner, resolved from the signed-in `az` user at `init`; reminders email this person), `ccd`/`ccd_source`/`ccd_conflict`, step completion, gate decisions, `last_notified`, etc. Add release-scoped fields here. +- **Release metadata + run-state** → `.release-runs//release-state.json` (per-release; the `ReleaseState` record). Holds `owner_email`/`owner_name` (the release owner, resolved from the signed-in `az` user at `init`; reminders email this person), `ccd`/`ccd_source`/`ccd_conflict`, step completion, gate decisions, `last_notified_date`, etc. Add release-scoped fields here. - **Tool config** → `release-agent/config/*.yaml` (not release-specific; committed): `phases.yaml`, `readiness.yaml`, `schedule.yaml`, `requirements.yaml`. ## Architecture — three layers (so it adapts to other interfaces) @@ -206,12 +207,14 @@ was off is picked up by the next one, and a once-per-calendar-day guard (`last_notified_date`) keeps it to one advance-effect and one email per day. `notify` is the **read-only** variant (report without advancing); `--as-of`/`--force` are debug overrides. -**Automation registry.** Every automation the orchestrator provisions is recorded in -`.release-runs/_automations.json` (via `cli automation register`) so it can be torn -down cleanly. Automations are **per-release** by default (`--release `), created -at start and removed at that release's close (`automation list --release ` → -delete each → `automation deregister`). Push reminders are per-release too. A -`--shared` scope exists for the rare automation meant to outlive every release. +**Automation registry.** Every automation the orchestrator provisions is recorded via +`cli automation register` so it can be torn down cleanly. **Per-release** automations +(`--release `, the default) live in `.release-runs//_automations.json` — +co-located with that release's state so ownership is explicit and they're removed with +the release folder at close (`automation list --release ` → delete each → +`automation deregister`). Push reminders are per-release too. A `--shared` scope (stored +machine-wide at `.release-runs/_automations.json`) exists for the rare automation meant +to outlive every release. ## Two kinds of human step diff --git a/release-agent/orchestrator/commands/automation.py b/release-agent/orchestrator/commands/automation.py index 8ea582a6..58697e5e 100644 --- a/release-agent/orchestrator/commands/automation.py +++ b/release-agent/orchestrator/commands/automation.py @@ -13,7 +13,7 @@ def cmd_automation(args): """Track Scout automations the orchestrator provisions, so they can be torn down at release close. This only records ids + step linkage — the skill does the actual Scout create/delete via m_create_automation / m_delete_automation.""" - reg = AutomationRegistry(args.runs_root) + reg = AutomationRegistry(args.runs_root, getattr(args, "release", None)) if args.action == "plan": return _cmd_plan(args) if args.action == "sync": @@ -108,23 +108,16 @@ def _cmd_sync(args): config_path = getattr(args, "config", None) or C.DEFAULT_CONFIG st = C.load_state(args.runs_root, args.release) ccd = getattr(st, "ccd", None) - reg = AutomationRegistry(args.runs_root) + reg = AutomationRegistry(args.runs_root, getattr(args, "release", None)) registered = reg.list(release=args.release, kind="step-driving") plan = auto_plan.plan(config_path, args.release, ccd) desired_by_slug = {a["slug"]: a for a in plan["automations"]} - desired_by_steps = {tuple(sorted(a["steps"])): a for a in plan["automations"]} updates = [] for e in registered: - # Match by slug (stable, unambiguous). Fall back to steps for older registry - # entries that predate slug — but only when the step set is unique. - spec = desired_by_slug.get(e.get("slug")) - if spec is None and e.get("slug") is None: - key = tuple(sorted(e.get("steps") or [])) - if sum(1 for a in plan["automations"] if tuple(sorted(a["steps"])) == key) == 1: - spec = desired_by_steps.get(key) + spec = desired_by_slug.get(e.get("slug")) # matched by slug (stable, unambiguous) if spec is None: - continue # no unambiguous desired spec — skip + continue # no matching desired spec — skip desired = spec.get("schedule") current = e.get("schedule") updates.append({ diff --git a/release-agent/orchestrator/registry.py b/release-agent/orchestrator/registry.py index 9fc31b1d..b004b84a 100644 --- a/release-agent/orchestrator/registry.py +++ b/release-agent/orchestrator/registry.py @@ -18,10 +18,16 @@ * release-level — operates on the whole release, not a step (e.g. the hourly `tick` push-reminder). Owns NO steps. -Stored machine-wide at /_automations.json (gitignored runtime state). +Storage layout: + * //_automations.json — a release's own automations, co-located + with its release-state.json so ownership is explicit (and they're removed with the + release folder at close). + * /_automations.json — SHARED (machine-wide) automations only; these are + reused across releases and are not tied to any one release folder. """ from __future__ import annotations +import glob import json import os from datetime import datetime, timezone @@ -43,31 +49,51 @@ def _now() -> str: class AutomationRegistry: - def __init__(self, runs_root: str): + def __init__(self, runs_root: str, release: str = None): self.runs_root = runs_root - self.path = os.path.join(runs_root, "_automations.json") + self.release = release + # SHARED (machine-wide) automations only; release automations live in /. + self.shared_path = os.path.join(runs_root, "_automations.json") - def _load(self) -> list: + # ---- paths ---- + def _release_path(self, release: str) -> str: + """A release's own registry file, next to its release-state.json.""" + return os.path.join(self.runs_root, release, "_automations.json") + + def _release_files(self) -> list: + """Every per-release registry file under runs_root.""" + return sorted(glob.glob(os.path.join(self.runs_root, "*", "_automations.json"))) + + # ---- file IO ---- + def _load_file(self, path: str) -> list: try: - with open(self.path, "r", encoding="utf-8") as fh: + with open(path, "r", encoding="utf-8") as fh: data = json.load(fh) return data if isinstance(data, list) else [] except (OSError, ValueError): return [] - def _save(self, entries: list) -> None: - os.makedirs(self.runs_root, exist_ok=True) - tmp = self.path + ".tmp" + def _save_file(self, path: str, entries: list) -> None: + os.makedirs(os.path.dirname(path) or ".", exist_ok=True) + tmp = path + ".tmp" with open(tmp, "w", encoding="utf-8") as fh: json.dump(entries, fh, indent=2) - os.replace(tmp, self.path) + os.replace(tmp, path) + + def _path_for(self, entry: dict) -> str: + """Where an entry is stored: its release folder, else the shared file.""" + if entry.get("scope") == "release" and entry.get("release"): + return self._release_path(entry["release"]) + return self.shared_path + # ---- api ---- def register(self, auto_id: str, name: str, release: str = None, shared: bool = False, purpose: str = "", steps: list = None, kind: str = None, schedule: str = None, slug: str = None) -> dict: - """Record an automation (upsert by id). Shared automations store release=None. - `steps` is the list of '.' ids this automation drives — the - automation<->step linkage used for traceability. + """Record an automation (upsert by id). Shared automations store release=None and + live in the machine-wide file; release automations live in /. `steps` is + the list of '.' ids this automation drives — the automation<->step + linkage used for traceability. `slug` is the stable identity from config/automations.yaml (e.g. 'ccd-noon'). It's the reliable key for `automation sync` — matching by steps alone is @@ -94,30 +120,48 @@ def register(self, auto_id: str, name: str, release: str = None, "slug": slug or None, "kind": kind, "scope": "shared" if shared else "release", - "release": None if shared else release, + "release": None if shared else (release or self.release), "purpose": purpose, "steps": steps, "schedule": schedule or None, "registered_at": _now(), } - entries = [e for e in self._load() if e.get("id") != auto_id] # upsert + # Upsert: drop any prior copy of this id wherever it lived, then write to its + # (possibly new) home file. + self._remove_everywhere(auto_id) + path = self._path_for(entry) + entries = self._load_file(path) entries.append(entry) - self._save(entries) + self._save_file(path, entries) return entry + def _remove_everywhere(self, auto_id: str) -> bool: + """Drop `auto_id` from the shared file and every per-release file. Returns True + if it was found somewhere.""" + removed = False + for path in [self.shared_path, *self._release_files()]: + entries = self._load_file(path) + kept = [e for e in entries if e.get("id") != auto_id] + if len(kept) != len(entries): + self._save_file(path, kept) + removed = True + return removed + def deregister(self, auto_id: str) -> bool: - entries = self._load() - kept = [e for e in entries if e.get("id") != auto_id] - self._save(kept) - return len(kept) != len(entries) + return self._remove_everywhere(auto_id) def list(self, release: str = None, scope: str = None, step: str = None, kind: str = None) -> list: - """List entries. `release` filters to that release's automations (scope - 'release' whose release matches). `scope` filters by 'shared'/'release'. + """List entries. `release` filters to that release's automations (and reads only + that release's file + the shared file). `scope` filters by 'shared'/'release'. `step` filters to automations that DRIVE that '.' id (reverse lookup). `kind` filters by 'step-driving'/'release-level'.""" - entries = self._load() + entries = self._load_file(self.shared_path) + if release is not None: + entries += self._load_file(self._release_path(release)) + else: + for path in self._release_files(): + entries += self._load_file(path) if release is not None: entries = [e for e in entries if e.get("release") == release] if scope is not None: diff --git a/release-agent/orchestrator/render.py b/release-agent/orchestrator/render.py index 88558c74..3733cc4f 100644 --- a/release-agent/orchestrator/render.py +++ b/release-agent/orchestrator/render.py @@ -247,11 +247,9 @@ def attest_prompt_payload(chk: dict, release_id: str) -> dict: def _pipelines_line(r: dict) -> str: """Compact one-line summary of the Phase-2 release-pipeline runs recorded on state (checker → orchestrator → the LATEST RC's two MRWP runs). Empty string when none - resolved yet. Reads the nested pipeline_runs schema (migrating a legacy flat shape); - no live az call in the render path.""" - from orchestrator.state import migrate_pipeline_runs + resolved yet. Reads the nested pipeline_runs schema; no live az call in the render path.""" from tools.pipelines import format_versions - pr = migrate_pipeline_runs(r.get("pipeline_runs") or {}) + pr = r.get("pipeline_runs") or {} parts = [] ch = pr.get("checker") or {} if ch.get("run_id"): diff --git a/release-agent/orchestrator/state.py b/release-agent/orchestrator/state.py index 513da2d5..9b6feb54 100644 --- a/release-agent/orchestrator/state.py +++ b/release-agent/orchestrator/state.py @@ -23,54 +23,6 @@ def _now() -> str: return datetime.now(timezone.utc).isoformat() -def _versions_str_to_dict(v) -> dict: - """'Common 24.6.0, Msal 8.4.2, Broker 16.5.0' -> {Common,Msal,Broker}. A dict passes - through; anything unparseable yields {}.""" - if isinstance(v, dict): - return {k: val for k, val in v.items() if val} - if not isinstance(v, str) or not v.strip(): - return {} - out = {} - for part in v.split(","): - toks = part.strip().split() - if len(toks) >= 2: - out[toks[0]] = toks[1] - return out - - -def migrate_pipeline_runs(pr) -> dict: - """Normalize the pipeline_runs container to the nested RC schema (idempotent). - - Accepts the legacy FLAT shape - {checker, orchestrator, versions, mrwp_ecs, mrwp_local, mrwp_id_source, resolved_at} - and lifts it to - {checker:{run_id,...}, orchestrator:{run_id,versions:{},...}, rcs:[{rc:1,ecs,local}]}. - A value already in the nested shape (has 'rcs', or a dict 'checker') is returned as-is. - Empty/None -> {}.""" - if not pr or not isinstance(pr, dict): - return {} - # Already nested? (rcs present, or checker is an object) - if "rcs" in pr or isinstance(pr.get("checker"), dict) or isinstance(pr.get("orchestrator"), dict): - return pr - out = {} - if pr.get("checker"): - out["checker"] = {"run_id": str(pr["checker"]), "resolved_at": pr.get("resolved_at")} - if pr.get("orchestrator"): - out["orchestrator"] = {"run_id": str(pr["orchestrator"]), - "versions": _versions_str_to_dict(pr.get("versions")), - "resolved_at": pr.get("resolved_at")} - ecs, local = pr.get("mrwp_ecs"), pr.get("mrwp_local") - if ecs or local: - rc = {"rc": 1, "resolved_at": pr.get("resolved_at")} - src = pr.get("mrwp_id_source") - if ecs: - rc["ecs"] = {"run_id": str(ecs), "id_source": src} - if local: - rc["local"] = {"run_id": str(local), "id_source": src} - out["rcs"] = [rc] - return out - - @dataclass class StepState: """Persisted state for a single step.""" @@ -130,14 +82,11 @@ class ReleaseState: steps: dict = field(default_factory=dict) # "phase.step" -> StepState (as dict) gate_decisions: list = field(default_factory=list) pending_human: list = field(default_factory=list) # outstanding human actions - last_notified: Optional[str] = None # last push message (legacy; kept for load compat) last_notified_date: Optional[str] = None # YYYY-MM-DD of the last daily digest sent - # Phase-2 release-pipeline run ids (checker / orchestrator / the two MRWP runs), - # refreshed each time Phase 2 resolves the chain. Because a re-triggered 'Trigger RC - # Testing' stage spawns NEW MRWP runs, these are re-resolved (newest wins) — not a - # fixed cache. Surfaced in status details + the daily digest. # Phase-2 release-pipeline runs — the RECORD of what verification resolved, reused by - # the RC report + gate (no re-discovery). Nested schema (see migrate_pipeline_runs): + # the RC report + gate (no re-discovery). Because a re-triggered 'Trigger RC Testing' + # stage spawns NEW MRWP runs, these are re-resolved (newest wins) — not a fixed cache. + # Nested schema: # { checker: {run_id, when, resolved_at}, # orchestrator: {run_id, versions:{Common,Msal,Broker}, parked, resolved_at}, # rcs: [ {rc, ecs:{run_id,id_source,complete,ran,total,failed_stages, @@ -154,15 +103,11 @@ class ReleaseState: def load(cls, path: str) -> "ReleaseState": with open(path, "r", encoding="utf-8") as fh: data = json.load(fh) - # Tolerate unknown/legacy keys: a persisted state file may predate a - # field rename/removal (or be hand-edited), and this loader runs in an - # unattended automation — an unexpected key must never hard-crash it. - # Only keys matching a declared field are applied; the rest are dropped. + # Drop unknown keys: this loader runs in an unattended automation, so a + # hand-edited or forward-version state file must never hard-crash it. Only keys + # matching a declared field are applied. known = {f.name for f in fields(cls)} obj = cls(**{k: v for k, v in data.items() if k in known}) - # Migrate the flat pre-nested pipeline_runs shape to the nested RC schema so - # older/hand-edited state files load cleanly and readers see one shape. - obj.pipeline_runs = migrate_pipeline_runs(obj.pipeline_runs) return obj def save(self, path: str) -> None: diff --git a/release-agent/steps/build_verify/_common.py b/release-agent/steps/build_verify/_common.py index 4a51c796..cc2c99d4 100644 --- a/release-agent/steps/build_verify/_common.py +++ b/release-agent/steps/build_verify/_common.py @@ -19,7 +19,6 @@ CHECKER_DEF, ORCHESTRATOR_DEF, MRWP_DEF, ORCH_REQUIRED_STAGES, ORCH_PARK_STAGE, format_versions, ) -from orchestrator.state import migrate_pipeline_runs from orchestrator.outcomes import Done, Blocked, InProgress from steps.lib.mockctx import mock_input, MISSING @@ -51,8 +50,8 @@ def _now_iso(): def _pipeline_runs(state) -> dict: - """The nested pipeline_runs container on state (migrating a legacy flat shape).""" - return migrate_pipeline_runs(getattr(state, "pipeline_runs", None) or {}) + """The nested pipeline_runs container on state.""" + return getattr(state, "pipeline_runs", None) or {} def stash_checker(state, run_id, when=None): @@ -125,7 +124,7 @@ def rc_report_model(state, timeout=120): iteration (rcs[-1]) and routes through `tools.pipelines.assemble_rc_model` — the SAME assembler the live path uses — so the state-based model can't drift from the live one. """ - pr = migrate_pipeline_runs(getattr(state, "pipeline_runs", None) or {}) + pr = getattr(state, "pipeline_runs", None) or {} ch = pr.get("checker") or {} o = pr.get("orchestrator") or {} rcs = pr.get("rcs") or [] diff --git a/release-agent/tests/test_engine.py b/release-agent/tests/test_engine.py index 99acb537..24203e24 100644 --- a/release-agent/tests/test_engine.py +++ b/release-agent/tests/test_engine.py @@ -1378,6 +1378,26 @@ def test_registry_register_list_deregister(): assert len(reg.list()) == 1 +def test_registry_relocates_release_automations_into_release_folder(): + """Release-scoped automations live in //_automations.json (owned + by the release); shared ones stay machine-wide.""" + from orchestrator.registry import AutomationRegistry + import os as _os, json as _json + with tempfile.TemporaryDirectory() as tmp: + reg = AutomationRegistry(tmp, release="2026-08") + reg.register("a2", "Phase-3 watcher", release="2026-08", steps=["bug_bash.bash_done"]) + reg.register("sh", "Release push reminders", shared=True, purpose="push") + rel_file = _os.path.join(tmp, "2026-08", "_automations.json") + shared_file = _os.path.join(tmp, "_automations.json") + # the release automation is co-located with the release; shared stays machine-wide + assert [e["id"] for e in _json.load(open(rel_file))] == ["a2"] + assert [e["id"] for e in _json.load(open(shared_file))] == ["sh"] + # release listing reads the release file + shared; deregister finds it in-folder + assert {e["id"] for e in reg.list(release="2026-08")} == {"a2"} + assert reg.deregister("a2") is True + assert reg.list(release="2026-08") == [] + + def test_registry_records_step_linkage_and_reverse_lookup(): """An automation entry records the steps it drives + its kind; list(step=...) is the reverse lookup (which automation owns a step) — the traceability link.""" @@ -2667,25 +2687,6 @@ def test_build_verify_persists_pipeline_run_ids(): assert again.pipeline_runs["rcs"][-1]["ecs"]["run_id"] == "900001" -def test_migrate_pipeline_runs_flat_to_nested(): - """A legacy FLAT pipeline_runs shape migrates to the nested RC schema on load - (idempotent); an already-nested value passes through unchanged.""" - from orchestrator.state import migrate_pipeline_runs as M - flat = {"checker": "111", "orchestrator": "222", - "versions": "Common 24.6.0, Msal 8.4.2, Broker 16.5.0", - "mrwp_ecs": "333", "mrwp_local": "444", "mrwp_id_source": "tags", - "resolved_at": "2026-08-20T00:00:00Z"} - m = M(flat) - assert m["checker"]["run_id"] == "111" - assert m["orchestrator"]["run_id"] == "222" - assert m["orchestrator"]["versions"] == {"Common": "24.6.0", "Msal": "8.4.2", "Broker": "16.5.0"} - assert m["rcs"] == [{"rc": 1, "resolved_at": "2026-08-20T00:00:00Z", - "ecs": {"run_id": "333", "id_source": "tags"}, - "local": {"run_id": "444", "id_source": "tags"}}] - assert M(m) == m # idempotent - assert M({}) == {} - - def test_stash_mrwp_appends_new_rc_on_id_change(): """stash_mrwp merges ecs+local into ONE rc entry, and appends a NEW rc iteration only when a provider's run id changes (RC Testing re-triggered). Latest = rcs[-1].""" From 02a86f4705e064f3231178cd96adcc59f3463d09 Mon Sep 17 00:00:00 2001 From: p3dr0rv Date: Fri, 21 Aug 2026 01:48:21 +0100 Subject: [PATCH 78/82] Phase 3: implement clone_plans as two real agents (Broker + Authenticator) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Replaces the single clone_plans stub with two real ADO-backed agent steps, per the distinct team procedures: * clone_plans_broker — COPIES the Broker master test plan (#2007357, area Engineering\Auth Client\Broker\Android) to a new plan "Android Monthly Release - ", referencing existing test cases (ADO clone default). Follows the eng.ms broker test-plans doc. * clone_plans_auth — CREATES a query-based (dynamic) test suite under "MSAuthenticator Test Passes" (#714514/714515) named "Android/release/MM/YYYY", WIQL mirroring the live suite "Android/release/08/2024" (tag Android, Identity Apps area, not Closed/IgnoreOnPrem). Follows the IDWiki 33580 doc and STOPS before assigning testers (later step). Both are idempotent: the created plan/suite id is stashed on the step and re-confirmed on re-run (auth also reuses a same-named suite) so a re-entered Phase 3 never makes duplicates. New tools/testplans.py wraps the ADO test-plan clone + dynamic-suite-create APIs; adds a pipelines._ado_rest_send POST primitive (the first ADO write path). Query/name shapes and plan ids were verified live read-only against the real plans. config/phases.yaml splits the step; knowledge.yaml documents both for step-info; tests cover clone/idempotency/block offline via mock knobs (net-guarded). Suite 202 green. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- release-agent/config/knowledge.yaml | 50 +++++ release-agent/config/phases.yaml | 3 +- release-agent/mocks.local.example.yaml | 3 +- release-agent/steps/bug_bash/__init__.py | 1 + .../steps/bug_bash/clone_plans_auth.py | 101 ++++++++++ .../steps/bug_bash/clone_plans_broker.py | 80 ++++++++ release-agent/tests/test_engine.py | 77 +++++++- release-agent/tools/pipelines.py | 43 +++++ release-agent/tools/testplans.py | 180 ++++++++++++++++++ 9 files changed, 532 insertions(+), 6 deletions(-) create mode 100644 release-agent/steps/bug_bash/__init__.py create mode 100644 release-agent/steps/bug_bash/clone_plans_auth.py create mode 100644 release-agent/steps/bug_bash/clone_plans_broker.py create mode 100644 release-agent/tools/testplans.py diff --git a/release-agent/config/knowledge.yaml b/release-agent/config/knowledge.yaml index 7aceb7db..c91ac51e 100644 --- a/release-agent/config/knowledge.yaml +++ b/release-agent/config/knowledge.yaml @@ -336,6 +336,56 @@ build_verify.rc_report: - q: "Does the sim send this email?" a: "No. In a sim the step is marked done without sending; only the real skill flow composes + sends it via WorkIQ." +# =============================== Phase 3 · bug_bash =============================== + +bug_bash.clone_plans_broker: + summary: "Copy the Broker master test plan to a new plan for this release." + what: > + Each release COPIES the Broker master template (ADO test plan #2007357, area + 'Engineering\\Auth Client\\Broker\\Android') to a new plan named + 'Android Monthly Release - ' (e.g. 'Android Monthly Release - Aug 2026'), + REFERENCING the existing test cases (the ADO clone default — it shares test cases, + it does not duplicate them). This is the Broker half of the old clone_plans step. + who: "Scout runs it automatically during `next` (an agent step, via the ADO test-plan clone API)." + how: > + Idempotent — the new plan id is stashed on the step; a re-run re-confirms that plan + exists and reports done without cloning again. If the clone fails (auth/API), the + step blocks with the ADO error; fix and `next`, or `skip --reason` to override. + links: + - name: "Broker master test plan (#2007357)" + url: "https://identitydivision.visualstudio.com/Engineering/_testPlans/define?planId=2007357&suiteId=2008656" + - name: "Broker test-plans doc" + url: "https://eng.ms/docs/microsoft-security/identity/entra-developer-application-platform/auth-client/authn-sdk-msal-android/android-auth-libraries/releases/internal-release-checklist/test-plans" + faqs: + - q: "Does it duplicate the test cases?" + a: "No — it references the existing test cases (the 'Reference existing test cases' clone option). Edit test cases only in the master template." + +bug_bash.clone_plans_auth: + summary: "Create the Authenticator bug-bash query-based test suite for this release." + what: > + Creates a NEW query-based (dynamic) test suite under the standing 'MSAuthenticator + Test Passes' plan (#714514, root suite 714515), named 'Android/release/MM/YYYY' + (e.g. 'Android/release/08/2026'). Its WIQL selects the Android bug-bash test cases + (tag 'Android', area 'Engineering\\ISP\\Identity Apps', not Closed, not 'IgnoreOnPrem' + — mirrors the live suite 'Android/release/08/2024'). This is the Authenticator half + of the old clone_plans step. It STOPS after creating the suite — assigning testers is + a later, manual step. + who: "Scout runs it automatically during `next` (an agent step, via the ADO test-suite create API)." + how: > + Idempotent — the created suite id is stashed on the step, and it also reuses an + existing same-named suite rather than making a duplicate. If the create fails + (auth/API), the step blocks with the ADO error; fix and `next`, or `skip --reason`. + links: + - name: "MSAuthenticator Test Passes plan (#714514)" + url: "https://identitydivision.visualstudio.com/Engineering/_testManagement?_a=tests&planId=714514&suiteId=714515" + - name: "How to make a test suite for bug bash (IDWiki 33580)" + url: "https://identitydivision.visualstudio.com/IdentityWiki/_wiki/wikis/IdentityWiki.wiki/33580/How-to-make-test-suite-for-bug-bash-" + faqs: + - q: "Does it assign testers?" + a: "No — it stops after creating the query-based suite. Assigning testers is done later (a separate, manual step)." + - q: "What names the suite?" + a: "The release: 'Android/release/MM/YYYY' — mirroring the live convention (newest was 'Android/release/08/2024')." + # ========================== Entry gate · readiness ========================== # Keyed "readiness.". Queried via `gate-info --item `. Same fields as the # step entries (summary/what/who/where/how/links/faqs). diff --git a/release-agent/config/phases.yaml b/release-agent/config/phases.yaml index 41e4d664..96675dd9 100644 --- a/release-agent/config/phases.yaml +++ b/release-agent/config/phases.yaml @@ -63,7 +63,8 @@ phases: name: "Test / Bug Bash" checklist_phase: 3 steps: - - { id: clone_plans, name: "Clone/rename test plans", owner: agent, maps_to: [T3] } + - { id: clone_plans_broker, name: "Clone Broker test plan (copy master template)", owner: agent, maps_to: [T3] } + - { id: clone_plans_auth, name: "Create Authenticator bug-bash test suite", owner: agent, maps_to: [T3] } - { id: coordinate, name: "Bug Bash coordinator (invite/monitor/aggregate)", owner: agent, maps_to: [T2] } - { id: ui_failures, name: "Surface Phase 2 UI failure list", owner: human, maps_to: [T4] } - { id: signoffs, name: "Chase DID/Dublin sign-offs + telemetry", owner: agent, maps_to: [T5] } diff --git a/release-agent/mocks.local.example.yaml b/release-agent/mocks.local.example.yaml index 7929d6b9..0c195e0e 100644 --- a/release-agent/mocks.local.example.yaml +++ b/release-agent/mocks.local.example.yaml @@ -84,7 +84,8 @@ preflight.cg: # build_verify.rc_report: { outcome: done } # scout (emails RC report + applies the 90% UI gate) # ---- Phase 3 · bug_bash ---- -# bug_bash.clone_plans: { outcome: done } # agent +# bug_bash.clone_plans_broker: { outcome: done } # agent (copy Broker master plan) +# bug_bash.clone_plans_auth: { outcome: done } # agent (create Authenticator query-suite) # bug_bash.coordinate: { outcome: done } # agent # bug_bash.ui_failures: { outcome: done } # reminder (human) # bug_bash.signoffs: { outcome: done } # agent diff --git a/release-agent/steps/bug_bash/__init__.py b/release-agent/steps/bug_bash/__init__.py new file mode 100644 index 00000000..29013d34 --- /dev/null +++ b/release-agent/steps/bug_bash/__init__.py @@ -0,0 +1 @@ +"""Phase 3 (bug_bash) step modules.""" diff --git a/release-agent/steps/bug_bash/clone_plans_auth.py b/release-agent/steps/bug_bash/clone_plans_auth.py new file mode 100644 index 00000000..10582d48 --- /dev/null +++ b/release-agent/steps/bug_bash/clone_plans_auth.py @@ -0,0 +1,101 @@ +"""Step: `clone_plans_auth` — create the Authenticator bug-bash test suite for this +release (Phase 3, bug_bash). + +Per the Authenticator "How to make a test suite for bug bash" doc, each release creates +a NEW query-based (dynamic) test suite under the standing "MSAuthenticator Test Passes" +plan (714514 / rootSuite 714515), named after the release ("Android/release/MM/YYYY"), +whose WIQL selects the Android bug-bash test cases. This is the Authenticator half of the +old `clone_plans` stub. + +We CREATE the suite and STOP — assigning testers is a later, manual step (out of scope +here, per the doc's "Assign Testers" cut line). + +Idempotent: the created suite id is stashed on the step (data.suite_id); a re-run +re-confirms it and reports done without creating a duplicate. As a second guard the step +also looks for an existing same-named child suite before creating. + +Mock knobs (mocks.local.yaml / tests): + suite_id : pretend the suite already exists (this id) — verifies + reports done. + create_id : the id the create should "return" (skip the live create). + existing : id of an already-present same-named suite the name-scan should "find". + fail : a detail string → force a Blocked (simulate an API/auth failure). +""" +from __future__ import annotations + +from orchestrator.outcomes import Done, Blocked +from steps.lib.agent import legacy_run +from steps.lib.mockctx import mock_input, MISSING +from tools import testplans as T + +ID = "clone_plans_auth" +KIND = "agent" + +MOCKABLE = { + "suite_id": {"kind": "input", "desc": "Pretend the query-suite already exists (this id)."}, + "create_id": {"kind": "input", "desc": "Id the create should return (skip the live create)."}, + "existing": {"kind": "input", "desc": "Id an existing same-named suite the name-scan finds."}, + "fail": {"kind": "input", "desc": "Force a Blocked with this detail (simulate an API failure)."}, +} + + +def _links(suite_id): + return [{"name": f"Authenticator bug-bash suite {suite_id}", + "url": T.plan_web_url(T.AUTH_PLAN, suite_id)}] + + +def build(state): + fail = mock_input("fail", MISSING) + if fail is not MISSING: + return Blocked(f"clone_plans_auth: {fail}") + + name = T.auth_suite_name(state.release_id) + step = state.get_step("bug_bash", ID) + + # Already created? (test-injected id, or a stored id from a prior run) → done. + injected = mock_input("suite_id", MISSING) + if injected is not MISSING: + return Done(f"Authenticator bug-bash suite already exists for {state.release_id}: " + f"'{name}' (suite {injected}).", links=_links(injected)) + stored = (step.data or {}).get("suite_id") + if stored: + ok, info, _ = T.get_suite(T.AUTH_PLAN, stored) + if ok and info: + return Done(f"Authenticator bug-bash suite already exists for {state.release_id}: " + f"'{info.get('name') or name}' (suite {stored}).", links=_links(stored)) + # recorded id no longer resolves — fall through and re-create + + # Duplicate guard: is a same-named suite already under the root? (offline-injectable) + existing = mock_input("existing", MISSING) + if existing is MISSING: + ok, existing, detail = T.find_child_suite_by_name(T.AUTH_PLAN, T.AUTH_ROOT_SUITE, name) + if not ok: + hint = " — run `az login`" if str(detail).startswith("AUTH") else "" + return Blocked(f"clone_plans_auth: could not list suites under the " + f"MSAuthenticator plan (#{T.AUTH_PLAN}) ({detail}){hint}.") + if existing: + step.data = dict(step.data or {}); step.data["suite_id"] = existing + state.set_step("bug_bash", ID, step) + return Done(f"Authenticator bug-bash suite already exists for {state.release_id}: " + f"'{name}' (suite {existing}).", links=_links(existing)) + + # Create the query-based suite. + create_id = mock_input("create_id", MISSING) + if create_id is MISSING: + ok, create_id, detail = T.create_auth_query_suite(name, T.auth_bugbash_query()) + if not ok: + hint = " — run `az login`" if str(detail).startswith("AUTH") else "" + return Blocked( + f"clone_plans_auth: could not create the query-based suite '{name}' under " + f"the MSAuthenticator plan (#{T.AUTH_PLAN}) ({detail}){hint}.") + + step.data = dict(step.data or {}) + step.data["suite_id"] = create_id + step.data["suite_name"] = name + state.set_step("bug_bash", ID, step) + return Done( + f"Created the Authenticator bug-bash query-suite '{name}' (suite {create_id}) " + f"under 'MSAuthenticator Test Passes'. Next: assign testers (later step).", + links=_links(create_id)) + + +run = legacy_run(build) diff --git a/release-agent/steps/bug_bash/clone_plans_broker.py b/release-agent/steps/bug_bash/clone_plans_broker.py new file mode 100644 index 00000000..5ca9b0eb --- /dev/null +++ b/release-agent/steps/bug_bash/clone_plans_broker.py @@ -0,0 +1,80 @@ +"""Step: `clone_plans_broker` — copy the Broker master test plan for this release +(Phase 3, bug_bash). + +Per the broker release doc, each release COPIES the master template plan (2007357) +to a new plan "Android Monthly Release - ", referencing the existing test +cases (the ADO clone default — shares test cases, doesn't duplicate them). This is the +Broker half of the old `clone_plans` stub. + +Idempotent: the created plan id is stashed on the step (data.plan_id). On a re-run the +step re-confirms that plan still exists and reports done WITHOUT cloning again — so a +`next` that re-enters Phase 3 never spawns a duplicate plan. + +Mock knobs (mocks.local.yaml / tests): + plan_id : pretend the clone already ran (this plan id) — verifies + reports done. + clone_id : the id the clone POST should "return" (skip the live CloneOperation). + fail : a detail string → force a Blocked (simulate an API/auth failure). +""" +from __future__ import annotations + +from orchestrator.outcomes import Done, Blocked +from steps.lib.agent import legacy_run +from steps.lib.mockctx import mock_input, MISSING +from tools import testplans as T + +ID = "clone_plans_broker" +KIND = "agent" + +MOCKABLE = { + "plan_id": {"kind": "input", "desc": "Pretend the clone already produced this plan id (idempotency test)."}, + "clone_id": {"kind": "input", "desc": "Id the clone should return (skip the live CloneOperation)."}, + "fail": {"kind": "input", "desc": "Force a Blocked with this detail (simulate an API failure)."}, +} + + +def _links(plan_id): + return [{"name": f"Broker test plan {plan_id}", "url": T.plan_web_url(plan_id)}] + + +def build(state): + fail = mock_input("fail", MISSING) + if fail is not MISSING: + return Blocked(f"clone_plans_broker: {fail}") + + dest = T.broker_plan_name(state.release_id) + step = state.get_step("bug_bash", ID) + + # Already cloned? A test injects `plan_id` to assert idempotency; otherwise the + # stored id from a prior run is re-confirmed against ADO. Either way → done, no re-clone. + injected = mock_input("plan_id", MISSING) + if injected is not MISSING: + return Done(f"Broker test plan already cloned for {state.release_id}: " + f"'{dest}' (plan {injected}).", links=_links(injected)) + stored = (step.data or {}).get("plan_id") + if stored: + ok, info, _ = T.get_plan(stored) + if ok and info: + return Done(f"Broker test plan already cloned for {state.release_id}: " + f"'{info.get('name') or dest}' (plan {stored}).", links=_links(stored)) + # recorded id no longer resolves — fall through and re-clone + + # Clone the master (or take the injected clone_id offline). + clone_id = mock_input("clone_id", MISSING) + if clone_id is MISSING: + ok, clone_id, detail = T.clone_broker_plan(dest) + if not ok: + hint = " — run `az login`" if str(detail).startswith("AUTH") else "" + return Blocked( + f"clone_plans_broker: could not copy the master test plan " + f"(#{T.BROKER_MASTER_PLAN}) to '{dest}' ({detail}){hint}.") + + step.data = dict(step.data or {}) + step.data["plan_id"] = clone_id + step.data["plan_name"] = dest + state.set_step("bug_bash", ID, step) + return Done( + f"Copied the Broker master test plan (#{T.BROKER_MASTER_PLAN}) to '{dest}' " + f"(plan {clone_id}), referencing existing test cases.", links=_links(clone_id)) + + +run = legacy_run(build) diff --git a/release-agent/tests/test_engine.py b/release-agent/tests/test_engine.py index 24203e24..19460240 100644 --- a/release-agent/tests/test_engine.py +++ b/release-agent/tests/test_engine.py @@ -72,6 +72,9 @@ def _recent_iso(): {"name": "UI Automation", "state": "completed", "result": "failed"}], "tests": {"total": 100, "passed": 98, "failed": 2}}, "build_verify.rc_report": {"outcome": "done", "note": "RC report emailed (test)"}, # skip live az + send + # Phase-3 bug_bash clone steps — real agents; keep flow tests offline. + "bug_bash.clone_plans_broker": {"outcome": "done", "note": "broker plan cloned (test)"}, + "bug_bash.clone_plans_auth": {"outcome": "done", "note": "auth suite created (test)"}, } @@ -635,9 +638,9 @@ def test_holds_at_first_hold(): assert actions[-1].step == "ui_failures" # Phases 0-2 gateless (rc_report auto); first hold is Phase-3 ui_failures # auto steps that RUN before the first hold: Phase-0 breaking/cg/cron/wiki (4) + # Phase-2 checker_fired/orchestrator_health/mrwp_ecs/mrwp_local (4) + rc_report (scout - # email, mocked done here) (1) + Phase-3 clone_plans/coordinate stubs (2). The Phase-1 - # scout steps are pre-recorded by _orch's _clear_ccd_scout (not "ran"). - assert sum(1 for a in actions if a.kind == "ran") == 11 + # email, mocked done here) (1) + Phase-3 clone_plans_broker/clone_plans_auth/coordinate + # stubs (3). The Phase-1 scout steps are pre-recorded by _orch's _clear_ccd_scout (not "ran"). + assert sum(1 for a in actions if a.kind == "ran") == 12 def test_gate_blocks_until_approved(): @@ -1015,7 +1018,7 @@ def test_only_frontier_phase_shows_current_despite_stale_downstream_progress(): for pid in ("preflight", "ccd"): for s in next(p for p in orch.config["phases"] if p["id"] == pid)["steps"]: orch.state.set_step(pid, s["id"], StepState(status="done", by="test")) - for sid in ("clone_plans", "coordinate"): + for sid in ("clone_plans_broker", "coordinate"): orch.state.set_step("bug_bash", sid, StepState(status="done", by="test")) phases = {p["id"]: p for p in orch.status_report()["phases"]} assert phases["build_verify"]["state"] == "current" and phases["build_verify"]["current"] @@ -2708,6 +2711,72 @@ def test_stash_mrwp_appends_new_rc_on_id_change(): assert K.latest_rc(st)["ecs"]["run_id"] == "910001" # latest = last +# ---- Phase 3: clone_plans_broker / clone_plans_auth ---- + +def _bb_build(sid, mocks, release="2026-08"): + """Build a bug_bash step outcome offline with the given mock inputs active.""" + import steps as _steps + from steps.lib import mockctx + from orchestrator.outcomes import as_dict + st = ReleaseState(release_id=release) + with mockctx.active(mocks): + return st, as_dict(_steps.get_step("bug_bash", sid).build(st)) + + +def test_testplans_names_and_query(): + from tools import testplans as T + assert T.broker_plan_name("2026-08") == "Android Monthly Release - Aug 2026" + assert T.auth_suite_name("2026-08") == "Android/release/08/2026" + q = T.auth_bugbash_query() + assert "contains 'Android'" in q and "IgnoreOnPrem" in q and "Identity Apps" in q + + +def test_clone_plans_broker_clones_then_idempotent(): + """First run copies the master plan and stashes the new plan id; a re-run with that id + stored reports done WITHOUT cloning again.""" + st, out = _bb_build("clone_plans_broker", {"clone_id": "5551212"}) + assert out["kind"] == "done" + assert "Copied the Broker master test plan" in out["note"] and "5551212" in out["note"] + assert st.get_step("bug_bash", "clone_plans_broker").data["plan_id"] == "5551212" + assert out["links"][0]["url"].endswith("planId=5551212") + # idempotent: an injected existing plan id → already-cloned, no re-clone + _, out2 = _bb_build("clone_plans_broker", {"plan_id": "5551212"}) + assert out2["kind"] == "done" and "already cloned" in out2["note"] + + +def test_clone_plans_broker_blocks_on_api_failure(): + _, out = _bb_build("clone_plans_broker", {"fail": "HTTP 403: forbidden"}) + assert out["kind"] == "blocked" and "403" in out["reason"] + + +def test_clone_plans_auth_creates_query_suite_then_idempotent(): + """First run creates the query-based suite (name + stash); a re-run with the suite id + stored reports done without re-creating.""" + st, out = _bb_build("clone_plans_auth", {"existing": None, "create_id": "778899"}) + assert out["kind"] == "done" + assert "Created the Authenticator bug-bash query-suite 'Android/release/08/2026'" in out["note"] + assert "assign testers" in out["note"].lower() # stops before assigning testers + assert st.get_step("bug_bash", "clone_plans_auth").data["suite_id"] == "778899" + # idempotent via injected existing suite id + _, out2 = _bb_build("clone_plans_auth", {"suite_id": "778899"}) + assert out2["kind"] == "done" and "already exists" in out2["note"] + + +def test_clone_plans_auth_reuses_existing_same_named_suite(): + """If a same-named suite already exists under the root, reuse it (no duplicate create).""" + st, out = _bb_build("clone_plans_auth", {"existing": "424242"}) + assert out["kind"] == "done" and "already exists" in out["note"] + assert st.get_step("bug_bash", "clone_plans_auth").data["suite_id"] == "424242" + + +def test_bug_bash_clone_steps_are_real_agents(): + """Both clone steps resolve to real agent modules (KIND=agent) — no longer stubs.""" + import steps as _steps + for sid in ("clone_plans_broker", "clone_plans_auth"): + mod = _steps.get_step("bug_bash", sid) + assert mod is not None and getattr(mod, "KIND", None) == "agent" and hasattr(mod, "run") + + def test_digest_shows_rc_line_when_build_verify_active(): """When a phase that opts in (show_pipeline_runs) is active and run ids are on state, the daily digest carries a one-line RC summary; phases that don't opt in omit it.""" diff --git a/release-agent/tools/pipelines.py b/release-agent/tools/pipelines.py index 7853c9be..75fbd3a7 100644 --- a/release-agent/tools/pipelines.py +++ b/release-agent/tools/pipelines.py @@ -121,6 +121,49 @@ def _ado_rest_get_text(url, timeout): return (False, None, f"REST GET failed: {e}") +def _ado_rest_send(url, method, body, timeout): + """Send a JSON REST request (POST/PATCH/PUT) to ADO with an az-minted bearer token. + Returns (ok, json, detail). The ONE write primitive — used for test-plan creates + where `az devops invoke` has no clean surface.""" + az = shutil.which("az") + if az is None: + return (False, None, "az CLI not found") + try: + tok = subprocess.run( + [az, "account", "get-access-token", "--resource", _ADO_RESOURCE, + "--query", "accessToken", "-o", "tsv"], + capture_output=True, text=True, timeout=timeout, encoding="utf-8") + except (subprocess.TimeoutExpired, OSError) as e: + return (False, None, f"failed to get token: {e}") + if tok.returncode != 0 or not (tok.stdout or "").strip(): + return (False, None, "AUTH: could not get an ADO token (run `az login`)") + token = tok.stdout.strip() + import urllib.request + import urllib.error + data = _json.dumps(body or {}).encode("utf-8") + req = urllib.request.Request( + url, data=data, method=method, + headers={"Authorization": f"Bearer {token}", "Content-Type": "application/json"}) + try: + with urllib.request.urlopen(req, timeout=timeout) as resp: + raw = resp.read().decode("utf-8") + return (True, (_json.loads(raw) if raw.strip() else {}), "") + except urllib.error.HTTPError as e: + detail = f"HTTP {e.code}" + if e.code in (401, 403): + detail = f"AUTH: HTTP {e.code} (run `az login` / check access)" + else: + try: + msg = _json.loads(e.read().decode("utf-8")).get("message") + if msg: + detail = f"HTTP {e.code}: {str(msg)[:200]}" + except Exception: + pass + return (False, None, detail) + except (urllib.error.URLError, ValueError, TimeoutError) as e: + return (False, None, f"REST {method} failed: {e}") + + def _tag_value(tags, key): """Return the value of a `key=value` build tag (e.g. RC-ECS=1678863 → '1678863'), or None. Case-sensitive key match; first match wins.""" diff --git a/release-agent/tools/testplans.py b/release-agent/tools/testplans.py new file mode 100644 index 00000000..62e27bf6 --- /dev/null +++ b/release-agent/tools/testplans.py @@ -0,0 +1,180 @@ +"""ADO Test-Plan operations for Phase 3 (bug_bash) — the two `clone_plans_*` steps. + +Two DIFFERENT release procedures, per the team docs: + + * BROKER — COPY the master test plan (a real ADO "Copy Test Plan" / CloneOperation). + Master: plan 2007357 / rootSuite 2007358 ("Android Monthly Release Master Test + Plan", area 'Engineering\\Auth Client\\Broker\\Android'). Each release clones it to + a new plan "Android Monthly Release - ", REFERENCING the existing test + cases (the ADO clone default — it shares test cases, doesn't duplicate them). + Doc: eng.ms/.../internal-release-checklist/test-plans + + * AUTHENTICATOR — CREATE a new query-based (dynamic) test suite under the standing + "MSAuthenticator Test Passes" plan (714514 / rootSuite 714515), named after the + release "Android/release/MM/YYYY", whose WIQL selects the Android bug-bash test + cases (tag 'Android', not Closed, not 'IgnoreOnPrem' — mirrors the live suite + 3016608 "Android/release/08/2024"). We STOP after creating the suite — assigning + testers is a later, manual step. + Doc: IdentityWiki page 33580 (How to make test suite for bug bash). + +Everything shells out to `az` (bearer token) via tools.pipelines helpers and returns an +(ok, value, detail) triple. Reads are cheap; the two creates are the only writes. +""" +from __future__ import annotations + +from tools import pipelines as P + +ORG = P.ENGINEERING_ORG # https://identitydivision.visualstudio.com +PROJECT = P.ENGINEERING_PROJECT # Engineering +_API = "api-version=7.1" + +# ---- Broker: master template to clone ---- +BROKER_MASTER_PLAN = 2007357 +BROKER_MASTER_ROOT_SUITE = 2007358 +BROKER_AREA_PATH = "Engineering\\Auth Client\\Broker\\Android" +BROKER_ITERATION = "Engineering" + +# ---- Authenticator: standing plan the query-suite hangs under ---- +AUTH_PLAN = 714514 +AUTH_ROOT_SUITE = 714515 +AUTH_AREA_PATH = "Engineering\\ISP\\Identity Apps" + +_MONTHS = ("Jan", "Feb", "Mar", "Apr", "May", "Jun", + "Jul", "Aug", "Sep", "Oct", "Nov", "Dec") + + +def _split_release(release_id: str): + """'2026-08' -> (2026, 8). Raises ValueError on a malformed id.""" + y, m = str(release_id).split("-")[:2] + return int(y), int(m) + + +def broker_plan_name(release_id: str) -> str: + """The Broker clone's destination plan name, e.g. 'Android Monthly Release - Aug 2026'.""" + y, m = _split_release(release_id) + return f"Android Monthly Release - {_MONTHS[m - 1]} {y}" + + +def auth_suite_name(release_id: str) -> str: + """The Authenticator query-suite name, e.g. 'Android/release/08/2026' — mirrors the + live convention (newest suite 'Android/release/08/2024').""" + y, m = _split_release(release_id) + return f"Android/release/{m:02d}/{y}" + + +def auth_bugbash_query() -> str: + """The WIQL for the Authenticator bug-bash query-suite — the Android test cases, + excluding Closed and on-prem-only. Verbatim shape of the live suite 3016608 + ('Android/release/08/2024').""" + return ( + "select [System.Id], [System.WorkItemType], [System.Title], " + "[Microsoft.VSTS.Common.Priority], [System.AssignedTo], [System.AreaPath] " + "from WorkItems where [System.TeamProject] = @project and " + "[System.WorkItemType] in group 'Microsoft.TestCaseCategory' and " + f"[System.AreaPath] under '{AUTH_AREA_PATH}' and " + "[System.Tags] contains 'Android' and [System.State] <> 'Closed' and " + "not [System.Tags] contains 'IgnoreOnPrem'") + + +def _plan_url(plan_id, extra=""): + return f"{ORG}/{PROJECT}/_apis/testplan/plans/{plan_id}?{_API}{extra}" + + +# ---------------------------------------------------------------- reads + +def get_plan(plan_id, timeout=60): + """(ok, {id,name,areaPath,iteration,rootSuiteId}, detail) for a test plan, or block + detail. Used to confirm an already-recorded clone still exists (idempotency).""" + ok, j, d = P._ado_rest_get(_plan_url(plan_id), timeout) + if not ok: + return (False, None, d) + root = (j or {}).get("rootSuite") or {} + return (True, {"id": j.get("id"), "name": j.get("name"), + "areaPath": j.get("areaPath"), "iteration": j.get("iteration"), + "rootSuiteId": root.get("id")}, "") + + +def get_suite(plan_id, suite_id, timeout=60): + """(ok, {id,name,suiteType}, detail) for a suite under a plan.""" + url = f"{ORG}/{PROJECT}/_apis/testplan/Plans/{plan_id}/suites/{suite_id}?{_API}" + ok, j, d = P._ado_rest_get(url, timeout) + if not ok: + return (False, None, d) + return (True, {"id": j.get("id"), "name": j.get("name"), + "suiteType": j.get("suiteType")}, "") + + +def find_child_suite_by_name(plan_id, parent_suite_id, name, timeout=90): + """Find a DIRECT child suite of `parent_suite_id` named `name` (case-insensitive). + Returns (ok, suite_id_or_None, detail). Pages through the plan's suites. A best-effort + duplicate guard for the Authenticator create.""" + want = (name or "").strip().lower() + url = (f"{ORG}/{PROJECT}/_apis/testplan/Plans/{plan_id}/suites?{_API}" + f"&continuationToken=") + token = "" + for _ in range(50): # hard page cap + ok, j, d = P._ado_rest_get(url + token, timeout) + if not ok: + return (False, None, d) + for s in (j or {}).get("value") or []: + if (s.get("name") or "").strip().lower() == want: + parent = s.get("parentSuite") or {} + if str(parent.get("id")) == str(parent_suite_id): + return (True, s.get("id"), "") + token = (j or {}).get("continuationToken") or "" + if not token: + break + return (True, None, "") + + +# ---------------------------------------------------------------- writes + +def clone_broker_plan(dest_name, timeout=120): + """COPY the Broker master plan to a new plan `dest_name`, referencing existing test + cases (ADO clone default). Returns (ok, new_plan_id, detail). + + Mirrors the doc's "Copy Test Plan → Reference existing test cases → Create". + """ + url = f"{ORG}/{PROJECT}/_apis/testplan/Plans/CloneOperation?api-version=7.1-preview.2" + body = { + # copy every suite + hierarchy; do NOT clone requirements. Not setting a + # test-case duplication flag => ADO REFERENCES the existing test cases. + "cloneOptions": {"copyAllSuites": True, "copyAncestorHierarchy": False, + "cloneRequirements": False, "copyComments": False}, + "destinationTestPlan": {"name": dest_name, "project": PROJECT, + "areaPath": BROKER_AREA_PATH, "iteration": BROKER_ITERATION}, + "sourceTestPlan": {"id": BROKER_MASTER_PLAN, "suiteIds": [BROKER_MASTER_ROOT_SUITE]}, + } + ok, j, d = P._ado_rest_send(url, "POST", body, timeout) + if not ok: + return (False, None, d) + dest = (j or {}).get("destinationTestPlan") or {} + pid = dest.get("id") + if not pid: + return (False, None, f"clone returned no destination plan id (state={j.get('state')})") + return (True, pid, "") + + +def create_auth_query_suite(name, query, timeout=90): + """CREATE a query-based (dynamic) test suite `name` under the Authenticator plan's + root suite, selecting the given WIQL. Returns (ok, new_suite_id, detail).""" + url = f"{ORG}/{PROJECT}/_apis/testplan/Plans/{AUTH_PLAN}/suites?api-version=7.1-preview.1" + body = {"suiteType": "dynamicTestSuite", "name": name, + "parentSuite": {"id": AUTH_ROOT_SUITE}, "queryString": query} + ok, j, d = P._ado_rest_send(url, "POST", body, timeout) + if not ok: + return (False, None, d) + sid = (j or {}).get("id") + if not sid: + return (False, None, "suite create returned no id") + return (True, sid, "") + + +# ---------------------------------------------------------------- links + +def plan_web_url(plan_id, suite_id=None): + """A human 'define' URL for a plan (optionally a suite).""" + u = f"{ORG}/{PROJECT}/_testPlans/define?planId={plan_id}" + if suite_id: + u += f"&suiteId={suite_id}" + return u From b7d5b2ad89c056c3b64c9aa9ae4077f5efe124e5 Mon Sep 17 00:00:00 2001 From: p3dr0rv Date: Fri, 21 Aug 2026 02:16:47 +0100 Subject: [PATCH 79/82] Phase 3 clone_plans: live-tested + 2 fixes from real ADO runs; name-override knob MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Live-tested both steps against real ADO (Engineering) with TEST-prefixed names, then deleted the artifacts. The live run exposed two real bugs, now fixed: 1. Broker clone HTTP 400 "CopyAncestorHierarchy cannot be false if multiple suite ids are provided" — the clone body set copyAncestorHierarchy=false while passing the source root suite id. Set it to true; the clone now succeeds (verified: cloned plan + suites, referencing existing test cases). 2. clone_plans_auth created a DUPLICATE suite because its name-scan only saw the first page — ADO returns the suites-list continuation token in the `x-ms-continuationtoken` RESPONSE HEADER, not the body. Added pipelines._ado_rest_get_h (header-aware GET) + _ado_rest_get_all (follows the header token); find_child_suite_by_name now pages all suites and correctly finds an existing same-named suite (verified live — no duplicate). Adds a `name` mock knob to both steps to override the derived plan/suite name (enables safe 'TEST ...' live runs and staging). Extends the test net-guard to _ado_rest_get_h / _ado_rest_send. New offline tests: name override + header-continuation paging. Suite 204. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../steps/bug_bash/clone_plans_auth.py | 5 +- .../steps/bug_bash/clone_plans_broker.py | 5 +- release-agent/tests/conftest.py | 2 + release-agent/tests/test_engine.py | 35 ++++++++++++++ release-agent/tools/pipelines.py | 48 +++++++++++++++++++ release-agent/tools/testplans.py | 33 ++++++------- 6 files changed, 107 insertions(+), 21 deletions(-) diff --git a/release-agent/steps/bug_bash/clone_plans_auth.py b/release-agent/steps/bug_bash/clone_plans_auth.py index 10582d48..c9a28f67 100644 --- a/release-agent/steps/bug_bash/clone_plans_auth.py +++ b/release-agent/steps/bug_bash/clone_plans_auth.py @@ -31,6 +31,7 @@ KIND = "agent" MOCKABLE = { + "name": {"kind": "input", "desc": "Override the suite name (e.g. a 'TEST ...' name for a safe live run)."}, "suite_id": {"kind": "input", "desc": "Pretend the query-suite already exists (this id)."}, "create_id": {"kind": "input", "desc": "Id the create should return (skip the live create)."}, "existing": {"kind": "input", "desc": "Id an existing same-named suite the name-scan finds."}, @@ -48,7 +49,9 @@ def build(state): if fail is not MISSING: return Blocked(f"clone_plans_auth: {fail}") - name = T.auth_suite_name(state.release_id) + name = mock_input("name", MISSING) + if name is MISSING: + name = T.auth_suite_name(state.release_id) step = state.get_step("bug_bash", ID) # Already created? (test-injected id, or a stored id from a prior run) → done. diff --git a/release-agent/steps/bug_bash/clone_plans_broker.py b/release-agent/steps/bug_bash/clone_plans_broker.py index 5ca9b0eb..0ef79de4 100644 --- a/release-agent/steps/bug_bash/clone_plans_broker.py +++ b/release-agent/steps/bug_bash/clone_plans_broker.py @@ -26,6 +26,7 @@ KIND = "agent" MOCKABLE = { + "name": {"kind": "input", "desc": "Override the destination plan name (e.g. a 'TEST ...' name for a safe live run)."}, "plan_id": {"kind": "input", "desc": "Pretend the clone already produced this plan id (idempotency test)."}, "clone_id": {"kind": "input", "desc": "Id the clone should return (skip the live CloneOperation)."}, "fail": {"kind": "input", "desc": "Force a Blocked with this detail (simulate an API failure)."}, @@ -41,7 +42,9 @@ def build(state): if fail is not MISSING: return Blocked(f"clone_plans_broker: {fail}") - dest = T.broker_plan_name(state.release_id) + dest = mock_input("name", MISSING) + if dest is MISSING: + dest = T.broker_plan_name(state.release_id) step = state.get_step("bug_bash", ID) # Already cloned? A test injects `plan_id` to assert idempotency; otherwise the diff --git a/release-agent/tests/conftest.py b/release-agent/tests/conftest.py index b741df60..ba998cf6 100644 --- a/release-agent/tests/conftest.py +++ b/release-agent/tests/conftest.py @@ -33,6 +33,8 @@ def _blocked(*_a, **_k): "or inject the step's input mocks).") monkeypatch.setattr(P, "_ado_rest_get", _blocked) + monkeypatch.setattr(P, "_ado_rest_get_h", _blocked) monkeypatch.setattr(P, "_ado_rest_get_text", _blocked) + monkeypatch.setattr(P, "_ado_rest_send", _blocked) monkeypatch.setattr(P, "_az_json", _blocked) yield diff --git a/release-agent/tests/test_engine.py b/release-agent/tests/test_engine.py index 19460240..5f46b289 100644 --- a/release-agent/tests/test_engine.py +++ b/release-agent/tests/test_engine.py @@ -2777,6 +2777,41 @@ def test_bug_bash_clone_steps_are_real_agents(): assert mod is not None and getattr(mod, "KIND", None) == "agent" and hasattr(mod, "run") +def test_clone_plans_name_override_knob(): + """The `name` mock knob overrides the derived plan/suite name (safe 'TEST ...' runs).""" + st, out = _bb_build("clone_plans_broker", + {"name": "TEST Android Monthly Release - Aug 2026", "clone_id": "1"}) + assert "TEST Android Monthly Release - Aug 2026" in out["note"] + st2, out2 = _bb_build("clone_plans_auth", + {"name": "TEST Android/release/08/2026", "existing": None, "create_id": "2"}) + assert "TEST Android/release/08/2026" in out2["note"] + + +def test_ado_rest_get_all_follows_header_continuation_token(): + """The pager concatenates every page, following the ADO `x-ms-continuationtoken` + RESPONSE HEADER (not a body field) — the bug the live clone_plans_auth test caught, + where only the first page was scanned so a same-named suite was missed → duplicate.""" + from tools import pipelines as P + pages = [ + (True, {"value": [{"id": 1}, {"id": 2}]}, {"x-ms-continuationtoken": "p2"}, ""), + (True, {"value": [{"id": 3}]}, {}, ""), # no token → last page + ] + calls = [] + + def fake_get_h(url, timeout): + calls.append(url) + return pages[len(calls) - 1] + + orig = P._ado_rest_get_h + P._ado_rest_get_h = fake_get_h + try: + ok, items, _ = P._ado_rest_get_all("https://x/_apis/y?api-version=7.1", 30) + finally: + P._ado_rest_get_h = orig + assert ok and [i["id"] for i in items] == [1, 2, 3] + assert "continuationToken=p2" in calls[1] # 2nd request carried the header token + + def test_digest_shows_rc_line_when_build_verify_active(): """When a phase that opts in (show_pipeline_runs) is active and run ids are on state, the daily digest carries a one-line RC summary; phases that don't opt in omit it.""" diff --git a/release-agent/tools/pipelines.py b/release-agent/tools/pipelines.py index 75fbd3a7..f010f608 100644 --- a/release-agent/tools/pipelines.py +++ b/release-agent/tools/pipelines.py @@ -93,6 +93,54 @@ def _ado_rest_get(url, timeout): return (False, None, f"REST GET failed: {e}") +def _ado_rest_get_h(url, timeout): + """Like _ado_rest_get but also returns the response headers: + (ok, json, headers_lower, detail). ADO returns paging tokens in the + `x-ms-continuationtoken` HEADER (not the body), so header access is needed to page.""" + az = shutil.which("az") + if az is None: + return (False, None, {}, "az CLI not found") + try: + tok = subprocess.run( + [az, "account", "get-access-token", "--resource", _ADO_RESOURCE, + "--query", "accessToken", "-o", "tsv"], + capture_output=True, text=True, timeout=timeout, encoding="utf-8") + except (subprocess.TimeoutExpired, OSError) as e: + return (False, None, {}, f"failed to get token: {e}") + if tok.returncode != 0 or not (tok.stdout or "").strip(): + return (False, None, {}, "AUTH: could not get an ADO token (run `az login`)") + token = tok.stdout.strip() + import urllib.request + import urllib.error + req = urllib.request.Request(url, headers={"Authorization": f"Bearer {token}"}) + try: + with urllib.request.urlopen(req, timeout=timeout) as resp: + hdrs = {k.lower(): v for k, v in resp.headers.items()} + return (True, _json.loads(resp.read().decode("utf-8")), hdrs, "") + except urllib.error.HTTPError as e: + detail = f"AUTH: HTTP {e.code}" if e.code in (401, 403) else f"HTTP {e.code}" + return (False, None, {}, detail) + except (urllib.error.URLError, ValueError, TimeoutError) as e: + return (False, None, {}, f"REST GET failed: {e}") + + +def _ado_rest_get_all(url, timeout, cap_pages=60): + """GET every page of a paged ADO collection, following the `x-ms-continuationtoken` + response header. `url` must already carry its api-version (no continuationToken). + Returns (ok, all_items, detail) where all_items is the concatenated `.value` lists.""" + items, token = [], None + for _ in range(cap_pages): + u = url + (f"&continuationToken={token}" if token else "") + ok, j, hdrs, detail = _ado_rest_get_h(u, timeout) + if not ok: + return (False, None, detail) + items += (j or {}).get("value") or [] + token = hdrs.get("x-ms-continuationtoken") + if not token: + break + return (True, items, "") + + def _ado_rest_get_text(url, timeout): """GET an ADO REST url returning PLAIN TEXT (e.g. a build log). (ok, text, detail).""" az = shutil.which("az") diff --git a/release-agent/tools/testplans.py b/release-agent/tools/testplans.py index 62e27bf6..3996c0c4 100644 --- a/release-agent/tools/testplans.py +++ b/release-agent/tools/testplans.py @@ -106,24 +106,18 @@ def get_suite(plan_id, suite_id, timeout=60): def find_child_suite_by_name(plan_id, parent_suite_id, name, timeout=90): """Find a DIRECT child suite of `parent_suite_id` named `name` (case-insensitive). - Returns (ok, suite_id_or_None, detail). Pages through the plan's suites. A best-effort - duplicate guard for the Authenticator create.""" + Returns (ok, suite_id_or_None, detail). Pages through ALL of the plan's suites + (following the ADO continuation-token header). A duplicate guard for the create.""" want = (name or "").strip().lower() - url = (f"{ORG}/{PROJECT}/_apis/testplan/Plans/{plan_id}/suites?{_API}" - f"&continuationToken=") - token = "" - for _ in range(50): # hard page cap - ok, j, d = P._ado_rest_get(url + token, timeout) - if not ok: - return (False, None, d) - for s in (j or {}).get("value") or []: - if (s.get("name") or "").strip().lower() == want: - parent = s.get("parentSuite") or {} - if str(parent.get("id")) == str(parent_suite_id): - return (True, s.get("id"), "") - token = (j or {}).get("continuationToken") or "" - if not token: - break + url = f"{ORG}/{PROJECT}/_apis/testplan/Plans/{plan_id}/suites?{_API}" + ok, suites, detail = P._ado_rest_get_all(url, timeout) + if not ok: + return (False, None, detail) + for s in suites: + if (s.get("name") or "").strip().lower() == want: + parent = s.get("parentSuite") or {} + if str(parent.get("id")) == str(parent_suite_id): + return (True, s.get("id"), "") return (True, None, "") @@ -137,9 +131,10 @@ def clone_broker_plan(dest_name, timeout=120): """ url = f"{ORG}/{PROJECT}/_apis/testplan/Plans/CloneOperation?api-version=7.1-preview.2" body = { - # copy every suite + hierarchy; do NOT clone requirements. Not setting a + # copy every suite + the hierarchy; do NOT clone requirements. Not setting a # test-case duplication flag => ADO REFERENCES the existing test cases. - "cloneOptions": {"copyAllSuites": True, "copyAncestorHierarchy": False, + # (copyAncestorHierarchy must be true when source suiteIds are given, else ADO 400.) + "cloneOptions": {"copyAllSuites": True, "copyAncestorHierarchy": True, "cloneRequirements": False, "copyComments": False}, "destinationTestPlan": {"name": dest_name, "project": PROJECT, "areaPath": BROKER_AREA_PATH, "iteration": BROKER_ITERATION}, From 4ce183502f8d41183445c5f7318b793c578b0d3b Mon Sep 17 00:00:00 2001 From: p3dr0rv Date: Fri, 21 Aug 2026 02:28:53 +0100 Subject: [PATCH 80/82] Fix clone_plans_auth query to match prod (ReleaseBugBash tag, not "not IgnoreOnPrem") MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The Authenticator bug-bash query was over-including: it filtered Android test cases by "not IgnoreOnPrem" (copied from the stale 2024 suite), yielding 66 cases vs prod's 45. Live diff vs the current prod suite 3728419 ("Android release/08/13/2026") showed prod is a strict subset — the 21 extras all LACK the 'ReleaseBugBash' tag (they're ComposeTesting / ComposeSettingTesting / DarkMode cases, not the curated bug-bash set). The IDWiki doc and prod both filter on [System.Tags] contains 'ReleaseBugBash'. Because ADO's tag `contains` is a substring match, that single clause also captures month-specific 'ReleaseBugBash' tags — exactly as the doc describes. Corrected auth_bugbash_query() to filter on 'ReleaseBugBash'; verified live the new query yields 45, matching prod. Updates the test assertion, module docstring, and knowledge FAQ (adds a which-cases entry). Suite 204 green. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- release-agent/config/knowledge.yaml | 12 +++++++----- release-agent/tests/test_engine.py | 2 +- release-agent/tools/testplans.py | 17 +++++++++++------ 3 files changed, 19 insertions(+), 12 deletions(-) diff --git a/release-agent/config/knowledge.yaml b/release-agent/config/knowledge.yaml index c91ac51e..d45eb6d1 100644 --- a/release-agent/config/knowledge.yaml +++ b/release-agent/config/knowledge.yaml @@ -366,10 +366,10 @@ bug_bash.clone_plans_auth: Creates a NEW query-based (dynamic) test suite under the standing 'MSAuthenticator Test Passes' plan (#714514, root suite 714515), named 'Android/release/MM/YYYY' (e.g. 'Android/release/08/2026'). Its WIQL selects the Android bug-bash test cases - (tag 'Android', area 'Engineering\\ISP\\Identity Apps', not Closed, not 'IgnoreOnPrem' - — mirrors the live suite 'Android/release/08/2024'). This is the Authenticator half - of the old clone_plans step. It STOPS after creating the suite — assigning testers is - a later, manual step. + (tag 'Android' + 'ReleaseBugBash', area 'Engineering\\ISP\\Identity Apps', not Closed + — matches the current prod suite 3728419 'Android release/08/13/2026'). This is the + Authenticator half of the old clone_plans step. It STOPS after creating the suite — + assigning testers is a later, manual step. who: "Scout runs it automatically during `next` (an agent step, via the ADO test-suite create API)." how: > Idempotent — the created suite id is stashed on the step, and it also reuses an @@ -384,7 +384,9 @@ bug_bash.clone_plans_auth: - q: "Does it assign testers?" a: "No — it stops after creating the query-based suite. Assigning testers is done later (a separate, manual step)." - q: "What names the suite?" - a: "The release: 'Android/release/MM/YYYY' — mirroring the live convention (newest was 'Android/release/08/2024')." + a: "The release: 'Android/release/MM/YYYY'." + - q: "Which test cases does it include?" + a: "The curated bug-bash set — Android test cases tagged 'ReleaseBugBash' (the tag also matches month-specific 'ReleaseBugBash' tags), not Closed. This matches prod; a broad 'all Android' filter would over-include (e.g. ComposeTesting/DarkMode cases)." # ========================== Entry gate · readiness ========================== # Keyed "readiness.". Queried via `gate-info --item `. Same fields as the diff --git a/release-agent/tests/test_engine.py b/release-agent/tests/test_engine.py index 5f46b289..ff16f933 100644 --- a/release-agent/tests/test_engine.py +++ b/release-agent/tests/test_engine.py @@ -2728,7 +2728,7 @@ def test_testplans_names_and_query(): assert T.broker_plan_name("2026-08") == "Android Monthly Release - Aug 2026" assert T.auth_suite_name("2026-08") == "Android/release/08/2026" q = T.auth_bugbash_query() - assert "contains 'Android'" in q and "IgnoreOnPrem" in q and "Identity Apps" in q + assert "contains 'Android'" in q and "contains 'ReleaseBugBash'" in q and "Identity Apps" in q def test_clone_plans_broker_clones_then_idempotent(): diff --git a/release-agent/tools/testplans.py b/release-agent/tools/testplans.py index 3996c0c4..66660be1 100644 --- a/release-agent/tools/testplans.py +++ b/release-agent/tools/testplans.py @@ -12,8 +12,8 @@ * AUTHENTICATOR — CREATE a new query-based (dynamic) test suite under the standing "MSAuthenticator Test Passes" plan (714514 / rootSuite 714515), named after the release "Android/release/MM/YYYY", whose WIQL selects the Android bug-bash test - cases (tag 'Android', not Closed, not 'IgnoreOnPrem' — mirrors the live suite - 3016608 "Android/release/08/2024"). We STOP after creating the suite — assigning + cases (tag 'Android' + 'ReleaseBugBash', not Closed — matches the current prod suite + 3728419 "Android release/08/13/2026"). We STOP after creating the suite — assigning testers is a later, manual step. Doc: IdentityWiki page 33580 (How to make test suite for bug bash). @@ -63,9 +63,14 @@ def auth_suite_name(release_id: str) -> str: def auth_bugbash_query() -> str: - """The WIQL for the Authenticator bug-bash query-suite — the Android test cases, - excluding Closed and on-prem-only. Verbatim shape of the live suite 3016608 - ('Android/release/08/2024').""" + """The WIQL for the Authenticator bug-bash query-suite — the Android test cases + curated for the release bug bash (tag 'ReleaseBugBash'), excluding Closed. Matches + the current prod suite (e.g. 'Android release/08/13/2026', suite 3728419). + + Per the IDWiki doc, the always-included cases carry tag 'ReleaseBugBash'; month- + specific cases carry 'ReleaseBugBash' (e.g. 'ReleaseBugBashAug'). Because ADO's + `[System.Tags] contains 'X'` is a substring match, the single 'ReleaseBugBash' clause + captures BOTH — no separate month clause needed.""" return ( "select [System.Id], [System.WorkItemType], [System.Title], " "[Microsoft.VSTS.Common.Priority], [System.AssignedTo], [System.AreaPath] " @@ -73,7 +78,7 @@ def auth_bugbash_query() -> str: "[System.WorkItemType] in group 'Microsoft.TestCaseCategory' and " f"[System.AreaPath] under '{AUTH_AREA_PATH}' and " "[System.Tags] contains 'Android' and [System.State] <> 'Closed' and " - "not [System.Tags] contains 'IgnoreOnPrem'") + "[System.Tags] contains 'ReleaseBugBash'") def _plan_url(plan_id, extra=""): From a7cf05bcd0d5dae2f3e28c1f824081522b18999a Mon Sep 17 00:00:00 2001 From: p3dr0rv Date: Fri, 21 Aug 2026 02:34:17 +0100 Subject: [PATCH 81/82] clone_plans_auth: name suite 'Android release/MM/DD/YYYY' from the CCD (match prod) The Authenticator suite name now follows the prod/IDWiki convention exactly - "Android release/MM/DD/YYYY" with the day taken from the release CCD (e.g. CCD 2026-08-13 -> "Android release/08/13/2026", matching prod suite 3728419). Was the day-less, extra-slash "Android/release/08/2026". auth_suite_name(ccd) now derives from the CCD date; the step passes state.ccd and blocks with a clear message if no CCD is set (the name needs the day). Updates the docstrings, knowledge FAQ, and tests; adds a no-CCD block test. Verified live (name + suite create). Suite 205 green. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- release-agent/config/knowledge.yaml | 6 +++--- release-agent/steps/bug_bash/clone_plans_auth.py | 8 ++++++-- release-agent/tests/test_engine.py | 15 +++++++++++---- release-agent/tools/testplans.py | 14 ++++++++------ 4 files changed, 28 insertions(+), 15 deletions(-) diff --git a/release-agent/config/knowledge.yaml b/release-agent/config/knowledge.yaml index d45eb6d1..44bf6a7c 100644 --- a/release-agent/config/knowledge.yaml +++ b/release-agent/config/knowledge.yaml @@ -364,8 +364,8 @@ bug_bash.clone_plans_auth: summary: "Create the Authenticator bug-bash query-based test suite for this release." what: > Creates a NEW query-based (dynamic) test suite under the standing 'MSAuthenticator - Test Passes' plan (#714514, root suite 714515), named 'Android/release/MM/YYYY' - (e.g. 'Android/release/08/2026'). Its WIQL selects the Android bug-bash test cases + Test Passes' plan (#714514, root suite 714515), named 'Android release/MM/DD/YYYY' + from the CCD (e.g. CCD 2026-08-13 -> 'Android release/08/13/2026'). Its WIQL selects the Android bug-bash test cases (tag 'Android' + 'ReleaseBugBash', area 'Engineering\\ISP\\Identity Apps', not Closed — matches the current prod suite 3728419 'Android release/08/13/2026'). This is the Authenticator half of the old clone_plans step. It STOPS after creating the suite — @@ -384,7 +384,7 @@ bug_bash.clone_plans_auth: - q: "Does it assign testers?" a: "No — it stops after creating the query-based suite. Assigning testers is done later (a separate, manual step)." - q: "What names the suite?" - a: "The release: 'Android/release/MM/YYYY'." + a: "From the CCD: 'Android release/MM/DD/YYYY' (e.g. CCD 2026-08-13 → 'Android release/08/13/2026'), matching prod." - q: "Which test cases does it include?" a: "The curated bug-bash set — Android test cases tagged 'ReleaseBugBash' (the tag also matches month-specific 'ReleaseBugBash' tags), not Closed. This matches prod; a broad 'all Android' filter would over-include (e.g. ComposeTesting/DarkMode cases)." diff --git a/release-agent/steps/bug_bash/clone_plans_auth.py b/release-agent/steps/bug_bash/clone_plans_auth.py index c9a28f67..c4c514b9 100644 --- a/release-agent/steps/bug_bash/clone_plans_auth.py +++ b/release-agent/steps/bug_bash/clone_plans_auth.py @@ -3,7 +3,7 @@ Per the Authenticator "How to make a test suite for bug bash" doc, each release creates a NEW query-based (dynamic) test suite under the standing "MSAuthenticator Test Passes" -plan (714514 / rootSuite 714515), named after the release ("Android/release/MM/YYYY"), +plan (714514 / rootSuite 714515), named after the release CCD ("Android release/MM/DD/YYYY"), whose WIQL selects the Android bug-bash test cases. This is the Authenticator half of the old `clone_plans` stub. @@ -51,7 +51,11 @@ def build(state): name = mock_input("name", MISSING) if name is MISSING: - name = T.auth_suite_name(state.release_id) + if not state.ccd: + return Blocked( + "clone_plans_auth: no Code Complete Date on record — can't name the suite " + "'Android release/MM/DD/YYYY'. Set the CCD first (`set-ccd`).") + name = T.auth_suite_name(state.ccd) step = state.get_step("bug_bash", ID) # Already created? (test-injected id, or a stored id from a prior run) → done. diff --git a/release-agent/tests/test_engine.py b/release-agent/tests/test_engine.py index ff16f933..a9652496 100644 --- a/release-agent/tests/test_engine.py +++ b/release-agent/tests/test_engine.py @@ -2713,12 +2713,12 @@ def test_stash_mrwp_appends_new_rc_on_id_change(): # ---- Phase 3: clone_plans_broker / clone_plans_auth ---- -def _bb_build(sid, mocks, release="2026-08"): +def _bb_build(sid, mocks, release="2026-08", ccd="2026-08-13"): """Build a bug_bash step outcome offline with the given mock inputs active.""" import steps as _steps from steps.lib import mockctx from orchestrator.outcomes import as_dict - st = ReleaseState(release_id=release) + st = ReleaseState(release_id=release, ccd=ccd) with mockctx.active(mocks): return st, as_dict(_steps.get_step("bug_bash", sid).build(st)) @@ -2726,7 +2726,8 @@ def _bb_build(sid, mocks, release="2026-08"): def test_testplans_names_and_query(): from tools import testplans as T assert T.broker_plan_name("2026-08") == "Android Monthly Release - Aug 2026" - assert T.auth_suite_name("2026-08") == "Android/release/08/2026" + # suite name comes from the CCD date: 'Android release/MM/DD/YYYY' (matches prod) + assert T.auth_suite_name("2026-08-13") == "Android release/08/13/2026" q = T.auth_bugbash_query() assert "contains 'Android'" in q and "contains 'ReleaseBugBash'" in q and "Identity Apps" in q @@ -2754,7 +2755,7 @@ def test_clone_plans_auth_creates_query_suite_then_idempotent(): stored reports done without re-creating.""" st, out = _bb_build("clone_plans_auth", {"existing": None, "create_id": "778899"}) assert out["kind"] == "done" - assert "Created the Authenticator bug-bash query-suite 'Android/release/08/2026'" in out["note"] + assert "Created the Authenticator bug-bash query-suite 'Android release/08/13/2026'" in out["note"] assert "assign testers" in out["note"].lower() # stops before assigning testers assert st.get_step("bug_bash", "clone_plans_auth").data["suite_id"] == "778899" # idempotent via injected existing suite id @@ -2769,6 +2770,12 @@ def test_clone_plans_auth_reuses_existing_same_named_suite(): assert st.get_step("bug_bash", "clone_plans_auth").data["suite_id"] == "424242" +def test_clone_plans_auth_blocks_without_ccd(): + """The suite name needs the CCD day ('Android release/MM/DD/YYYY') — no CCD → block.""" + st, out = _bb_build("clone_plans_auth", {"existing": None, "create_id": "1"}, ccd=None) + assert out["kind"] == "blocked" and "Code Complete Date" in out["reason"] + + def test_bug_bash_clone_steps_are_real_agents(): """Both clone steps resolve to real agent modules (KIND=agent) — no longer stubs.""" import steps as _steps diff --git a/release-agent/tools/testplans.py b/release-agent/tools/testplans.py index 66660be1..5eb39d5b 100644 --- a/release-agent/tools/testplans.py +++ b/release-agent/tools/testplans.py @@ -11,7 +11,7 @@ * AUTHENTICATOR — CREATE a new query-based (dynamic) test suite under the standing "MSAuthenticator Test Passes" plan (714514 / rootSuite 714515), named after the - release "Android/release/MM/YYYY", whose WIQL selects the Android bug-bash test + release "Android release/MM/DD/YYYY", whose WIQL selects the Android bug-bash test cases (tag 'Android' + 'ReleaseBugBash', not Closed — matches the current prod suite 3728419 "Android release/08/13/2026"). We STOP after creating the suite — assigning testers is a later, manual step. @@ -55,11 +55,13 @@ def broker_plan_name(release_id: str) -> str: return f"Android Monthly Release - {_MONTHS[m - 1]} {y}" -def auth_suite_name(release_id: str) -> str: - """The Authenticator query-suite name, e.g. 'Android/release/08/2026' — mirrors the - live convention (newest suite 'Android/release/08/2024').""" - y, m = _split_release(release_id) - return f"Android/release/{m:02d}/{y}" +def auth_suite_name(ccd: str) -> str: + """The Authenticator query-suite name from the release's Code Complete Date, e.g. + CCD '2026-08-13' -> 'Android release/08/13/2026' — matches the prod convention + (suite 3728419 'Android release/08/13/2026') and the IDWiki doc's + 'Android release/MM/DD/YYYY'. `ccd` is 'YYYY-MM-DD'.""" + y, m, d = str(ccd).split("-")[:3] + return f"Android release/{int(m):02d}/{int(d):02d}/{int(y)}" def auth_bugbash_query() -> str: From 93aa28c922ed6911e777b996a3424a9c04b05b36 Mon Sep 17 00:00:00 2001 From: p3dr0rv Date: Fri, 21 Aug 2026 02:52:06 +0100 Subject: [PATCH 82/82] clone_plans_broker: wait for async CloneOperation to finish (fix "seems empty") MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Investigating a broker clone that "seemed empty" revealed the real gap: ADO's CloneOperation is ASYNCHRONOUS. clone_broker_plan POSTed the clone and returned the destination plan id straight from the POST response, before the background suite/test-case copy finished — so a downstream read (or the user opening the plan) right after the step reported done could see an empty/partial plan. Verified live: a completed clone is a perfect copy of the master (45 suites: 16 static + 29 dynamic, 276 test cases), and cloneStatistics shows clonedTestCasesCount=0 / totalTestCasesCount=271 — i.e. it correctly REFERENCES existing test cases, not duplicates. The clone status lives at cloneOperationResponse.state. clone_broker_plan now polls the CloneOperation until state 'succeeded' (blocks on 'failed') before returning, so the plan is fully populated when the step reports done (measured ~14s for the master). Adds a _clone_op helper for the nested response shape and 2 tests (async-poll-to-succeeded, block-on-failed). Suite 207 green. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- release-agent/tests/test_engine.py | 48 ++++++++++++++++++++++++++++++ release-agent/tools/testplans.py | 48 ++++++++++++++++++++++++++---- 2 files changed, 90 insertions(+), 6 deletions(-) diff --git a/release-agent/tests/test_engine.py b/release-agent/tests/test_engine.py index a9652496..ab007c5d 100644 --- a/release-agent/tests/test_engine.py +++ b/release-agent/tests/test_engine.py @@ -2794,6 +2794,54 @@ def test_clone_plans_name_override_knob(): assert "TEST Android/release/08/2026" in out2["note"] +def test_clone_broker_plan_waits_for_async_completion(): + """clone_broker_plan POSTs the CloneOperation then POLLS the op until 'succeeded' + before returning — so the plan is fully populated when the step reports done (the + async 'seems empty if read too early' gap). state lives under cloneOperationResponse.""" + from tools import pipelines as P + from tools import testplans as T + post = (True, {"destinationTestPlan": {"id": 9001}, + "cloneOperationResponse": {"opId": 77, "state": "queued"}}, "") + # op poll: inProgress, then inProgress, then succeeded + op_pages = [ + (True, {"cloneOperationResponse": {"state": "inProgress"}}, {}, ""), + (True, {"cloneOperationResponse": {"state": "inProgress"}}, {}, ""), + (True, {"cloneOperationResponse": {"state": "succeeded"}}, {}, ""), + ] + calls = {"get": 0} + + def fake_send(url, method, body, timeout): + return post + + def fake_get_h(url, timeout): + i = calls["get"]; calls["get"] += 1 + return op_pages[min(i, len(op_pages) - 1)] + + o_send, o_get = P._ado_rest_send, P._ado_rest_get_h + P._ado_rest_send, P._ado_rest_get_h = fake_send, fake_get_h + try: + ok, pid, d = T.clone_broker_plan("TEST plan", poll_secs=0) + finally: + P._ado_rest_send, P._ado_rest_get_h = o_send, o_get + assert ok and pid == 9001 and d == "" + assert calls["get"] >= 3 # polled until 'succeeded' + + +def test_clone_broker_plan_blocks_on_failed_clone_op(): + """A failed clone operation surfaces as (False, ...) so the step blocks.""" + from tools import pipelines as P + from tools import testplans as T + o_send, o_get = P._ado_rest_send, P._ado_rest_get_h + P._ado_rest_send = lambda *a, **k: (True, {"destinationTestPlan": {"id": 1}, + "cloneOperationResponse": {"opId": 5, "state": "queued"}}, "") + P._ado_rest_get_h = lambda *a, **k: (True, {"cloneOperationResponse": {"state": "failed"}}, {}, "") + try: + ok, pid, d = T.clone_broker_plan("TEST plan", poll_secs=0) + finally: + P._ado_rest_send, P._ado_rest_get_h = o_send, o_get + assert not ok and "failed" in d + + def test_ado_rest_get_all_follows_header_continuation_token(): """The pager concatenates every page, following the ADO `x-ms-continuationtoken` RESPONSE HEADER (not a body field) — the bug the live clone_plans_auth test caught, diff --git a/release-agent/tools/testplans.py b/release-agent/tools/testplans.py index 5eb39d5b..8fc7c1e6 100644 --- a/release-agent/tools/testplans.py +++ b/release-agent/tools/testplans.py @@ -130,16 +130,36 @@ def find_child_suite_by_name(plan_id, parent_suite_id, name, timeout=90): # ---------------------------------------------------------------- writes -def clone_broker_plan(dest_name, timeout=120): +def _clone_op(op_field, *keys): + """Read a field from a CloneOperation response, which nests the live status under + `cloneOperationResponse` (state/opId/completionDate/cloneStatistics) but the ids at + the top level. Tries the nested object first, then the top level.""" + nested = (op_field or {}).get("cloneOperationResponse") or {} + for k in keys: + if nested.get(k) is not None: + return nested.get(k) + if (op_field or {}).get(k) is not None: + return op_field.get(k) + return None + + +def clone_broker_plan(dest_name, timeout=120, poll_secs=3, max_polls=40): """COPY the Broker master plan to a new plan `dest_name`, referencing existing test - cases (ADO clone default). Returns (ok, new_plan_id, detail). - - Mirrors the doc's "Copy Test Plan → Reference existing test cases → Create". + cases (ADO clone default), and WAIT for the async clone to finish. Returns + (ok, new_plan_id, detail). + + ADO's CloneOperation is asynchronous: the POST returns a destination plan id while the + suite/test-case copy runs in the background. We poll the operation until its state is + 'succeeded' before returning, so the plan is fully populated when the step reports done + (a too-early read would otherwise show an empty/partial plan). Mirrors the doc's + "Copy Test Plan → Reference existing test cases → Create". """ + import time url = f"{ORG}/{PROJECT}/_apis/testplan/Plans/CloneOperation?api-version=7.1-preview.2" body = { # copy every suite + the hierarchy; do NOT clone requirements. Not setting a - # test-case duplication flag => ADO REFERENCES the existing test cases. + # test-case duplication flag => ADO REFERENCES the existing test cases + # (cloneStatistics.clonedTestCasesCount stays 0 — shared, not duplicated). # (copyAncestorHierarchy must be true when source suiteIds are given, else ADO 400.) "cloneOptions": {"copyAllSuites": True, "copyAncestorHierarchy": True, "cloneRequirements": False, "copyComments": False}, @@ -153,7 +173,23 @@ def clone_broker_plan(dest_name, timeout=120): dest = (j or {}).get("destinationTestPlan") or {} pid = dest.get("id") if not pid: - return (False, None, f"clone returned no destination plan id (state={j.get('state')})") + return (False, None, f"clone returned no destination plan id (state={_clone_op(j, 'state')})") + op_id = _clone_op(j, "opId") + + # Poll the operation to completion so the plan is populated before we report done. + if op_id: + op_url = (f"{ORG}/{PROJECT}/_apis/testplan/Plans/CloneOperation/{op_id}" + f"?api-version=7.1-preview.2") + for _ in range(max_polls): + ok_o, op, _h, d_o = P._ado_rest_get_h(op_url, 60) + if not ok_o: + break # can't read op → best-effort, return pid + state = str(_clone_op(op, "state") or "").lower() + if state in ("succeeded", "completed"): + break + if state == "failed": + return (False, None, f"clone operation failed (op {op_id})") + time.sleep(poll_secs) return (True, pid, "")