diff --git a/.cursorrules b/.cursorrules index dd7c977..aa461ba 100644 --- a/.cursorrules +++ b/.cursorrules @@ -1,3 +1,6 @@ +# Repository / agent behavior +- Do not add, commit, or suggest pushing interview scripts, LinkedIn drafts, or other personal narrative markdown under `docs/` unless the user explicitly asks. Technical docs (e.g. `docs/Architecture.md`) are fine. + # Code Review Standards - All Python async functions must include detailed logging. - Test scripts must include API timeout handling. diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 0000000..89ac96c --- /dev/null +++ b/.dockerignore @@ -0,0 +1,31 @@ +# Git metadata +.git +.gitignore + +# Python cache / virtual env +__pycache__/ +*.py[cod] +*.pyo +*.pyd +venv/ +env/ +.env + +# Test / coverage artifacts +.coverage +coverage.xml +htmlcov/ +.pytest_cache/ + +# Local outputs and generated data +results/ +logs/ + +# Local editor / OS files +.DS_Store +.vscode/ +.idea/ + +# Large local assets not required for image build +test_images/ +assets/ diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 0524f5a..6d3b45f 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -24,21 +24,15 @@ jobs: - name: Install dependencies run: | python -m pip install --upgrade pip - pip install -r requirements.txt - pip install pytest pytest-cov ruff + pip install -e ".[dev]" - name: Lint + run: ruff check src tests app.py test_connection.py + + - name: Type check (mypy) run: | - ruff check \ - src/ai_quality_agent.py \ - src/eval \ - src/models/inference_adapter.py \ - src/util/failure_memory.py \ - src/util/monitor_performance.py \ - src/agent/orchestrator.py \ - src/engine/image_processor.py \ - src/test_failure_memory_retrieval.py \ - tests + mypy --explicit-package-bases src + MYPYPATH=src mypy --explicit-package-bases app.py test_connection.py - name: Unit tests with coverage run: | diff --git a/.gitignore b/.gitignore index b1af100..74534a8 100644 --- a/.gitignore +++ b/.gitignore @@ -14,6 +14,7 @@ results/**/*.json # Local run artifacts (loopback cache, vector DB, coverage) results/loopback_cache/ results/failure_memory_db/ +logs/**/*.json .coverage coverage.xml htmlcov/ @@ -30,4 +31,11 @@ test_images/ *.gguf # Packaging artifacts -*.egg-info/ \ No newline at end of file +*.egg-info/ + +# Local-only / interview prep (not part of public repo story) +docs/IntegrationGuide.md +docs/AdvocacyCaseStudy.md +docs/InterviewNarratives.md +docs/InterviewLanguagePrep.md +docs/linkedin-self-healing-vision-qa.md \ No newline at end of file diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md new file mode 100644 index 0000000..d4d5df7 --- /dev/null +++ b/CONTRIBUTING.md @@ -0,0 +1,86 @@ +# Contributing + +Thanks for helping improve this project. Small, focused changes are easier to review and merge. + +## Prerequisites + +- Python **3.9+** (CI runs on **3.11**; matching CI locally avoids surprises). +- A clone of the repository. + +## Dependencies (single source of truth) + +**All pinned runtime dependencies are defined in `pyproject.toml` under `[project.dependencies]`.** Do not maintain a second copy of version pins elsewhere. + +- **Developers / CI**: `pip install -e ".[dev]"` (includes pytest, coverage, Ruff, and mypy). +- **Optional**: `pip install -r requirements.txt` — this file only contains `-e .[dev]` as a convenience shim for older habits or docs that still use `-r`. + +When you add or bump a dependency, edit **`pyproject.toml` only**, then reinstall your venv. + +## Local setup + +```bash +cd agentic_testing_framework +python -m venv .venv +source .venv/bin/activate # Windows: .venv\Scripts\activate +python -m pip install -U pip +pip install -e ".[dev]" +``` + +The CLI and tests expect `src` on the module path. Use `PYTHONPATH=src` as shown below (same as CI). + +## Streamlit demo (`app.py`) + +Run from the **repository root** so `agent`, `engine`, etc. resolve: + +```bash +PYTHONPATH=src streamlit run app.py +``` + +Without `PYTHONPATH=src`, **Manual Baseline** still works; **AI Pipeline** needs the orchestrator import path above. + +## Run tests + +```bash +PYTHONPATH=src pytest +``` + +Coverage options are defined in `pyproject.toml` (`--cov=src`, XML report, and **`--cov-fail-under=34`** so total coverage cannot drift far below current levels without CI failing). + +## Lint + +CI runs Ruff on the full Python tree under `src` plus `tests`. Match it before opening a PR: + +```bash +ruff check src tests app.py test_connection.py +``` + +## Type check (mypy) + +Settings live in `pyproject.toml` under `[tool.mypy]`. Run the same checks as CI (two passes avoid duplicate module mapping for `src/` vs repo-root scripts): + +```bash +mypy --explicit-package-bases src +MYPYPATH=src mypy --explicit-package-bases app.py test_connection.py +``` + +## Optional: agent smoke run (CI parity) + +The workflow also runs a short end-to-end report generation: + +```bash +PYTHONPATH=src python src/ai_quality_agent.py --profile dev --performance-analysis --overhead-analysis +``` + +## Pull requests + +1. **Branch**: Open PRs against the repository default branch (usually `main`). +2. **Scope**: One logical change per PR when possible (feature, fix, or docs—not all mixed unless tightly related). +3. **Description**: Summarize *what* changed and *why*; link an issue if one exists. +4. **Green CI**: Ensure tests, Ruff, and **mypy** pass locally. +5. **Docs**: If you change CLI flags, config shape, or inference behavior, update `README.md` and any affected file under `docs/`. + +## Code style + +Follow existing patterns in nearby modules (logging, typing, error messages). Prefer clear names and small functions over clever one-liners. + +Use **`logging.getLogger(__name__)`** instead of `print` for diagnostics. The batch CLI calls **`util.cli_logging.configure_cli_logging()`** in `__main__`, which sets `basicConfig` to include **timestamp**, **level**, and **logger name** when the root logger has no handlers yet. `app.py` does the same at import time when appropriate (e.g. under Streamlit). diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..61059e5 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,24 @@ +# Linux container (default: amd64). llama.cpp here is a **CPU** build with OpenBLAS — +# not Apple Metal (Metal is macOS-only; use a host install if you need GPU on Apple Silicon). +FROM python:3.11-slim + +RUN apt-get update && apt-get install -y \ + build-essential \ + cmake \ + git \ + libopenblas-dev \ + libopencv-dev \ + && rm -rf /var/lib/apt/lists/* + +WORKDIR /app + +# Dependency pins: pyproject.toml only. Runtime install (no [dev] extras). +COPY . . +RUN pip install --no-cache-dir llama-cpp-python \ + && pip install --no-cache-dir . + +ENV PYTHONUNBUFFERED=1 +ENV PYTHONPATH=/app/src +ENV MODEL_PATH=/app/models/your-model-q4_k_m.gguf + +CMD ["python", "src/ai_quality_agent.py", "--profile", "dev"] diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..0f7bcbd --- /dev/null +++ b/LICENSE @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2026 Cheryl + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/README.md b/README.md index 2422477..d2164cb 100644 --- a/README.md +++ b/README.md @@ -4,156 +4,147 @@ ![Pillow](https://img.shields.io/badge/Library-Pillow-orange.svg) [![CI](https://github.com/CHDev2116/agentic_testing_framework/actions/workflows/ci.yml/badge.svg)](https://github.com/CHDev2116/agentic_testing_framework/actions/workflows/ci.yml) -A practical framework to automatically decide whether images are production-ready (`GO` / `REVIEW` / `NO_GO`). +Configuration-driven framework to evaluate image quality and make production release decisions: `GO` / `REVIEW` / `NO_GO`. -Designed for engineers in mobile imaging, model evaluation, and production QA pipelines. - -## 🚀 Quick Start +## Quick Start (CLI Pipeline) ```bash -# 1) Clone git clone https://github.com/CHDev2116/agentic_testing_framework cd agentic_testing_framework - -# 2) Install -pip install -r requirements.txt - -# 3) Run (Dev profile) +python -m pip install -U pip +pip install -e ".[dev]" python3 src/ai_quality_agent.py --profile dev ``` -If no images are found, sample images will be auto-generated. - -Expected output (example): - -```text -=== Summary === -Test Dashboard -- Pass rate: ~60-80% -- Release decision: GO / REVIEW / NO_GO -``` - -For profile comparison, repeatability, backend override, and performance/stress commands, see the `Usage` section below. - ---- - -## ⚡ What is this? +`requirements.txt` is a thin shim (`-e .[dev]`) for `pip install -r requirements.txt`; **all version pins live in `pyproject.toml`** (`[project.dependencies]`). -- AI-powered framework for fast mobile image quality validation on constrained devices. -- Combines physical metrics, multi-backend inference, and arbitration to output `GO` / `REVIEW` / `NO_GO`. -- Includes guardrail-driven closed-loop recovery, benchmarking, repeatability checks, and CI automation. +If no input images are present, sample images are auto-generated. ---- +
+DX: Built for Extensibility -## 🧪 When should you use this? +This repo optimizes for **integrators**: swap runtimes without rewriting the batch pipeline, keep a **fixed downstream contract**, and emit **auditable JSON** (not “score-only” blobs). -- Validating mobile camera quality before release -- Comparing quantized model outputs -- Automating regression checks in CI +### Config-only inference backend selection ---- +- Set `model_settings.inference.backend` in `configs/*.json` to one of: `simulated`, `ollama_vision`, `mock_api`, `llama_cpp`. +- For ad-hoc runs, the CLI can override without editing files: `python3 src/ai_quality_agent.py --profile dev --inference-backend mock_api` (see `--help`). +- **Connecting a live model** (llama.cpp server or Ollama vision): see [`docs/ModelInferenceSetup.md`](docs/ModelInferenceSetup.md). +- Composition root: `build_inference_engine()` in [`src/models/inference_adapter.py`](src/models/inference_adapter.py) selects the concrete engine class from config. -## 🚀 Why this matters +Same codebase path runs locally (simulated / Ollama / llama.cpp HTTP) or against a mock HTTP API—**no forked “deploy-only” branch** unless your infra truly requires it. -Built an AI-powered image quality validation framework that can: +### Orchestrator contract: one method shape, typed-normalized outputs -- Evaluate mobile image quality in milliseconds -- Support multiple inference backends -- Make release decisions (`GO` / `REVIEW` / `NO_GO`) -- Benchmark latency, repeatability, and bias -- Run fully automated via CI/CD +Engines are **not** tied to a shared ABC in this codebase. Each backend class implements the same surface: -Designed for real-world constrained devices and production testing workflows. +`predict_quality(photo_path: str, metrics: dict) -> dict` -## 💼 Real-World Value +Raw backend responses are validated through `InferenceOutput` (`src/models/contracts.py`) and normalized before use, so downstream code sees a **stable schema**: at minimum `decision`, `code`, `msg`, plus optional `confidence`, and `backend` (including `provider->simulated` when fallback fires). -This framework can reduce manual image QA effort, standardize release criteria, -and provide traceable quality decisions for mobile camera and AI imaging pipelines. +**Adding a new backend** today means: implement that method + normalize through `InferenceOutput.from_payload(...)`, then add a branch in `build_inference_engine`. If you want static enforcement later, a `typing.Protocol` (or an ABC) is an incremental hardening step—the factory stays the single registry for CI/review friendliness. -It helps teams ship faster with clearer quality gates, lower review cost, -and more consistent production outcomes. +### Actionable batch artifacts ---- +- Per-inference payloads retain **`code`** (machine-oriented) and **`msg`** (human-oriented) after normalization—failures are classified, not opaque. +- Batch summaries include **`summary.decision_reason`**: a single string that records how **quality-gate** and **aggregated arbitration** were merged (`merge_gate_and_arbitration`), so **why** the merged outcome is `GO` / `REVIEW` / `NO_GO` is reproducible from the JSON without re-running the batch. +- Per-image rows now include **`inference_output`** (typed trace) with step-level planner history (`steps`) and fallback visibility (`fallback_used`). -## 📊 Example Output +Example (trimmed): -```text -=== Summary === -Test Dashboard - - Pass rate: 66.7% - - Avg latency: 4.66 ms - - Release decision: REVIEW +```json +{ + "file": "image4.jpeg", + "decision": { + "decision": "Under-exposed", + "code": "ERR_LIGHT_DARK_002", + "msg": "too dark", + "backend": "llama_cpp" + }, + "inference_output": { + "image_path": "test_images/image4.jpeg", + "final_decision": "NO_GO", + "error_code": "ERR_LIGHT_DARK_002", + "error_message": "too dark", + "total_latency_ms": 9.91, + "steps": [ + { + "attempt": 1, + "signal": "under", + "action": "brighten", + "rationale": "under-exposed signal and safe brightness headroom", + "fallback_used": true, + "metrics_before": {"avg_brightness": 9.8, "sharpness": 14.2}, + "metrics_after": {"avg_brightness": 12.1, "sharpness": 13.9}, + "latency_ms": 4.7 + } + ] + } +} ``` ---- +See also: [`docs/Architecture.md`](docs/Architecture.md) for the provider contract and fallback behavior. -## 🐞 Common Issues +
-### 1. Ollama not responding -- Check if server is running: http://localhost:11434 +## Demo UI (Streamlit) -### 2. No images found -- Framework will auto-generate samples +From the **repository root** (so `src` is importable as top-level packages): -### 3. Slow performance -- Try switching to `simulated` backend - ---- +```bash +PYTHONPATH=src streamlit run app.py +``` -## 📌 Project Overview -The framework is **configuration-driven**, with quality thresholds largely decoupled from execution logic so standards can be adjusted with minimal code changes. +The **AI Pipeline** mode imports `agent.orchestrator`; if imports fail, the UI shows a `PYTHONPATH=src` hint. **Manual Baseline** mode works without the orchestrator. -It is designed for quantized-model QA workflows where repeatability, comparability, and release governance matter as much as raw inference speed. +
+Demo preview & optional assets -## 🤖 AI Honesty Statement +Streamlit UI: generated sample input, **Manual Baseline** vs **AI Pipeline** side-by-side (score, confidence, label, latency), and score delta summary. -Current state: -- **Real**: image metrics are computed from real files (brightness/sharpness). -- **Model inference**: supports `simulated`, `ollama_vision`, `mock_api`, and `llama_cpp` backends (config-driven). +![Streamlit demo: baseline vs AI pipeline comparison](assets/streamlit-comparison.png) -Next improvements: -- Improve robustness and calibration for **Ollama** and **mock API** backends under production-like traffic. -- Expand benchmark datasets for edge cases (low light, blur, high noise) to improve decision reliability. -- Keep the same three-layer architecture so ranking, decision, and benchmarking stay reusable. +Optional: add a short screen recording as `assets/demo.gif` and reference it here for motion (e.g. clicking **Analyze** / **Compare Both Modes**). -## 📍 Core Guarantees (Source of Truth) +
-- **Architecture**: `Engine -> Model -> Eval` with clear module boundaries. -- **Inference backends**: `simulated`, `ollama_vision`, `mock_api`, `llama_cpp` (runtime-configurable). -- **Decision policy**: conservative release gating (`GO` / `REVIEW` / `NO_GO`) with arbitration. -- **Guardrail loopback**: `NO_GO` recovery supports `under` (brighten), `over` (dim), `blurry` (sharpen), bounded by retry/guard thresholds. -- **Retention scope**: auto-clean after 14 days currently applies to `batch_report_*.json` and `error_report_*.json`. -- **CI contract**: lint scope follows `.github/workflows/ci.yml` selected `src` paths plus `tests`. +
+Project identity -For more detail on inference providers, the primary vs demo pipeline, and optional memory profiling (`PIXELQA_MONITOR_MEMORY`), see [`docs/Architecture.md`](docs/Architecture.md). +| Item | Value | +|------|--------| +| **Display name** | Agentic Testing Framework | +| **Python package** (`pyproject.toml`) | `agentic_testing_framework` | +| **Default model profile label** (`configs/*.json` → `model_settings.name`) | `Agentic Testing Framework - Llama 4-bit` | +| **Docker image tag** (example) | `agentic-testing-framework:latest` | +| **Memory profiling** (`src/util/monitor_performance.py`) | Prefer `ATF_MONITOR_MEMORY=1`; legacy alias `PIXELQA_MONITOR_MEMORY` still works | -## 🛠️ Technical Highlights +
-Reference baseline for architecture, backend support, guardrail loopback, retention, and CI scope: see `Core Guarantees (Source of Truth)`. +
+Why this project -### 1. Modular Architecture and Config-Driven Design -* **Largely config-driven**: Uses `configs/*.json` to manage test standards (sharpness/brightness thresholds), so strategies can be adjusted with minimal code changes. -* **Engine layer**: feature extraction from input images (brightness/sharpness metrics). -* **Model layer**: inference abstraction (rule-based today, real-model adapter-ready). -* **Eval layer**: scoring, ranking, benchmark insights, and release decision. +- Automates repetitive image QA with consistent decision policy. +- Supports multiple inference backends (`simulated`, `ollama_vision`, `mock_api`, `llama_cpp`). +- Keeps results traceable with ranking, reports, and guardrail-driven recovery. -### 2. Batch Processing and Performance Monitoring -* **Automated pipeline**: Scans the `test_images/` directory automatically, without manually specifying files. -* **Performance tracking**: Built-in **Latency Tracking** records per-image processing time for inference efficiency analysis. -* **Dashboard summary**: Automatically reports **Pass Rate** and **Average Latency** when testing completes. +
-### 3. Resilience and Error Handling -* **OOM stress simulation**: Includes a random memory-overflow simulator to validate system stability in extreme conditions. -* **Safety-net flow**: Uses `try-except-finally` to ensure the system still produces a context-rich **Crash Report (JSON)** even after failures. +
+Core guarantees (source of truth) -## ✅ Delivery Targets +- **Architecture**: `Engine -> Model -> Eval` with clear boundaries. +- **Decision policy**: conservative release gating (`GO` / `REVIEW` / `NO_GO`). +- **Loopback**: `NO_GO` recovery runs a planner step (`plan_next_action`) to choose brighten/dim/sharpen/stop under retry limits. +- **Planner mode**: `runtime.loopback_planner.mode` supports `simulated` (default) and `llm` (with automatic fallback to simulated on planner errors). +- **Planner health check**: when planner mode is `llm`, startup runs endpoint health check by default (`require_healthy_on_startup=true`) and fails fast if unreachable. Use `--planner-skip-health-check` only for controlled fallback experiments. +- **Retention**: auto-clean for `batch_report_*.json` and `error_report_*.json` after 14 days. +- **CI scope**: Ruff on `src` + `tests` + `app.py` + `test_connection.py`; **mypy** on `src` then on `app.py` / `test_connection.py` with `MYPYPATH=src`; pytest with coverage (including **`--cov-fail-under=34`**). Tests emphasize the **release decision path** (arbitration, inference result normalization, loopback integration) and **golden checks** for batch ranking, release gates, log stability windows, Pillow-based vision metrics, and OpenCV exposure validation—see `tests/`. -- **1. Clone repo and run immediately**: if no input images exist, the runner auto-generates sample images. -- **2. Produce comparable results**: run multiple profiles on the same image set and export a comparison report. -- **3. Provide ranking + decision**: every run outputs per-image ranking and a final release decision (`GO` / `REVIEW` / `NO_GO`). -- **4. Keep a clear three-layer architecture**: `engine` (feature extraction), `model` (inference abstraction), `eval` (scoring + decision). +
-## 🔄 Pipeline Flow (Engine -> Model -> Eval) +
+Pipeline flow ```mermaid flowchart LR @@ -164,280 +155,110 @@ flowchart LR D -- NO_GO: Guardrail Loopback --> B ``` -## 📂 Directory Structure -```text -agentic_testing_framework/ -├── configs/ # Environment-based configs (base/dev/benchmark) -├── src/ -│ ├── engine/ # Feature extraction modules -│ ├── models/ # Inference abstraction layer -│ ├── eval/ # Scoring, ranking, and decision logic -│ └── ai_quality_agent.py # Orchestrator for batch flow and reporting -├── test_images/ # Input images for testing -├── results/ -│ ├── dev/ # Per-run reports for dev profile -│ ├── benchmark/ # Per-run reports for benchmark profile -│ └── comparisons/ # Cross-profile comparison reports -└── README.md - -## 🚀 Usage - -Run from the project root: +
-```bash -# Install dependencies -pip install -r requirements.txt +## Usage -# (Recommended for contributors) install project with test tooling -python3 -m pip install -e ".[dev]" +**Basic runs:** -# Essential: run development profile once (configs/base.json + configs/dev.json) +```bash python3 src/ai_quality_agent.py --profile dev - -# Essential: run benchmark profile python3 src/ai_quality_agent.py --profile benchmark - -# Essential: load a config file directly (applied on top of configs/base.json) python3 src/ai_quality_agent.py --config configs/dev.json ``` -Advanced analysis: +
+Advanced CLI ```bash -# Compare multiple profiles and output a cross-profile ranking python3 src/ai_quality_agent.py --compare-profiles dev benchmark - -# Repeatability test: same image set, run 5 times, report variance python3 src/ai_quality_agent.py --repeatability-test dev --repeatability-runs 5 - -# Temporary backend override (without editing config files) python3 src/ai_quality_agent.py --profile benchmark --inference-backend mock_api - -# Optional performance deep-dive (latency vs image size + simple CPU usage) +python3 src/ai_quality_agent.py --profile dev --loopback-planner llm +python3 src/ai_quality_agent.py --profile dev --loopback-planner llm --planner-timeout-s 10 --planner-model local-planner +python3 src/ai_quality_agent.py --profile dev --loopback-planner llm --planner-skip-health-check python3 src/ai_quality_agent.py --profile dev --performance-analysis - -# One-command stress benchmark (auto-expand input set to >=100 images) python3 src/ai_quality_agent.py --profile dev --stress-test-100 --performance-analysis - -# Lightweight overhead audit for framework self-cost python3 src/ai_quality_agent.py --profile dev --overhead-analysis - -# Vector retrieval smoke test for failure-memory cases +python3 src/ai_quality_agent.py --profile dev --parallel-metrics +python3 src/ai_quality_agent.py --profile dev --async-batch --async-concurrency 4 +python3 src/ai_quality_agent.py --profile dev --async-batch --loopback-planner llm +python3 src/ai_quality_agent.py --profile dev --async-batch --parallel-metrics python3 src/test_failure_memory_retrieval.py ``` -Essential Notes: -- `--profile` supports: `dev`, `benchmark`, `base` -- `--config` accepts either an absolute path or a project-root-relative path -- `REVIEW` / `NO_GO` samples are persisted to a local ChromaDB (`results/failure_memory_db`) with multilingual sentence embeddings -- Guardrail-driven closed loop is enabled for `NO_GO` recovery: `under-exposed` (brighten), `over-exposed` (dim), and `blurry` (sharpen), bounded by `runtime.max_retry` (default `3`) -- Auto-clean currently applies to `batch_report_*.json` and `error_report_*.json` after 14 days - -Advanced Notes: -- `--compare-profiles` runs each profile and creates `results/comparisons/profile_comparison_*.json` -- `--repeatability-test` runs the same profile repeatedly and writes `results/repeatability/repeatability_*.json` -- `--inference-backend` overrides backend at runtime (`simulated`, `ollama_vision`, `mock_api`, `llama_cpp`) -- `--performance-analysis` writes `results/performance/performance_*.json` with latency-size and CPU summaries -- `--stress-test-100` auto-generates synthetic image variants to reach at least 100 images for stable trend analysis -- `--overhead-analysis` writes `results/overhead/overhead_*.json` to quantify framework self-overhead vs model latency -- Loopback guardrails include engine/model agreement checks, oscillation detection, near-over/under exposure cutoffs, and minimum brightness/sharpness gain thresholds -- Performance report includes peak process CPU/memory, tail latency (P95/P99), a correlation matrix, and auto-generated scaling insights - -For canonical design guarantees, treat `Core Guarantees (Source of Truth)` as authoritative when wording differs elsewhere. +
-### 🚨 Automated Error Reporting - -- Per-file failures in batch processing automatically generate `error_report_*.json`. -- Fatal pipeline exceptions are also captured into an error report before re-raising. -- Error reports include timestamp, scope, profile, config source, error type/message, and traceback. -- Error reports are saved under the configured `folders.logs` path and auto-cleaned after 14 days. - -### 🔌 Real Inference Backends - -Inference backend is configured via `model_settings.inference.backend`: -- `simulated` (default): rule-based inference. -- `ollama_vision`: live inference through local Ollama endpoint. -- `mock_api`: external API endpoint for integration testing. -- `llama_cpp`: local OpenAI-compatible endpoint served by `llama-server`. - -Authoritative backend support list is maintained in `Core Guarantees (Source of Truth)`. - -Example backend config: - -```json -"model_settings": { - "inference": { - "backend": "ollama_vision", - "fallback_to_simulated": true, - "ollama": { - "host": "http://localhost:11434", - "model": "llava:7b", - "timeout_s": 45 - }, - "mock_api": { - "url": "http://localhost:8080/infer", - "timeout_s": 10, - "api_key_env": "MOCK_INFER_API_KEY" - } - } -} -``` - -When backend calls fail, the pipeline can fallback to `simulated` inference if `fallback_to_simulated` is enabled. - -### 🦙 llama.cpp Local Server Quickstart - -Start `llama-server` (in a separate terminal): +
+Docker (optional) ```bash -cd /Users/cheryl/public_repos/Quantization/llama.cpp/build/bin/ - -./llama-server \ - -m "/Users/cheryl/public_repos/agentic_testing_framework/src/models/llama-3.1-8b-Q4_K_M.gguf" \ - -ngl -1 \ - --port 8080 \ - --chat-template llama3 +docker build -t agentic-testing-framework:latest . +docker run --rm \ + -v "$(pwd)/test_images:/app/test_images" \ + -v "$(pwd)/results:/app/results" \ + agentic-testing-framework:latest ``` -Check server health: +
-```bash -curl http://127.0.0.1:8080/health -``` +
+Troubleshooting -Optional connectivity smoke test: +- **Ollama not responding**: check `http://localhost:11434` +- **No images found**: samples are auto-generated +- **Slow performance**: try `--inference-backend simulated` -```bash -python3 test_connection.py -``` +
-Run framework with the dev profile (already configured to `llama_cpp`): +
+CI / local tests ```bash -python3 src/ai_quality_agent.py --profile dev -``` - -## 📤 Full Output Example - -Startup mode: PixelQA-Llama-4bit (4-bit) -Starting to process 3 image(s)... - -Processed sample_good.png: [SUCCESS_200] Optimal (4.85ms) -Processed sample_dark.png: [ERR_LIGHT_DARK_002] Under-exposed (4.12ms) - -======================================================= -Test Dashboard - - Total tests: 3 - - Pass rate (Optimal): 66.7% - - Average latency: 4.66 ms - - Release decision: REVIEW -------------------------------------------------------- -Top ranking: - #1 sample_good.png | score=84.2 | Optimal - #2 sample_bright.png | score=31.6 | Over-exposed -======================================================= - -### Typical Performance on M4 Chip - -- Throughput: **~6.42 TPS** (measured via local llama.cpp run) -- Typical end-to-end latency in this framework: **~4-9 ms / image** (profile and backend dependent) -- Use `--performance-analysis` for per-run latency/CPU correlation details - -## 👉 Benchmark Insights - -- Stricter thresholds usually improve screening confidence but lower pass rate. -- Latency alone is not a release signal; combine `pass_rate`, `avg_latency_ms`, and `release_decision`. -- Ranking is a prioritization tool, while final release still follows `GO` / `REVIEW` / `NO_GO`. -- `--compare-profiles` exports these insights to `results/comparisons/profile_comparison_*.json` (`benchmark_insights`). - -## 🔁 Repeatability Example - -Command used: - -```bash -python3 src/ai_quality_agent.py --repeatability-test dev --repeatability-runs 5 +python -m pip install -U pip +pip install -e ".[dev]" +ruff check src tests app.py test_connection.py +mypy --explicit-package-bases src +MYPYPATH=src mypy --explicit-package-bases app.py test_connection.py +PYTHONPATH=src pytest ``` -Observed output (same image set, 5 runs): -- `same_image_set`: `True` -- `pass_rate_variance`: `0.0` -- `avg_latency_variance`: `0.8026` -- `max_per_image_score_variance`: `0.0` -- `decision_distribution`: `{"REVIEW": 5}` +Docker and other installs use **`pyproject.toml` only** for dependency pins (`pip install .` in the Dockerfile). The `requirements.txt` shim is optional for local workflows. -Interpretation: -- The quality outputs are stable across repeated runs on the same image batch. -- Runtime latency varies slightly by environment/load, while ranking and release decision remain consistent in this run. +Workflow reference: `.github/workflows/ci.yml` -Threshold calibration is performed per profile using benchmark feedback to balance pass-rate targets and false-positive risk. -The architecture scales from small local test sets to larger benchmark batches by keeping feature extraction, inference abstraction, and eval logic independently extensible. +
-## 🗺️ Roadmap +
+Deeper documentation & roadmap -- [x] Profile-based config system and report retention -- [x] Multi-backend inference abstraction (`simulated` / `ollama_vision` / `mock_api` / `llama_cpp`) -- [x] Batch quality ranking + release arbitration (`GO` / `REVIEW` / `NO_GO`) -- [x] Repeatability and benchmark comparison workflows -- [x] Automated JSON error reporting with retention cleanup -- [ ] Multi-threading optimization for larger datasets -- [ ] Extended visual analytics (OpenCV-based color/noise diagnostics) +**Docs** -## 🧪 Evaluation & Reliability +- Architecture and provider details: [`docs/Architecture.md`](docs/Architecture.md) +- For benchmark, repeatability, and reliability narratives, use docs + report artifacts under `results/`. -- **Goal**: make release decisions trustworthy, not just repeatable. -- **Validation**: compare against labeled data and track Precision/Recall/FPR/FNR. -- **Bias control**: monitor threshold/model/dataset bias through conflict logging and error distribution. -- **Mitigation**: apply threshold calibration, confidence-aware arbitration, and edge-case dataset expansion. -- **Production policy**: conservative by design; avoid passing low-quality images even at the cost of more false negatives. +**Roadmap (shipped)** -This keeps decisions traceable, measurable, and continuously improvable from local testing to larger benchmark workloads. +- [x] Multi-backend inference abstraction +- [x] Batch ranking + release arbitration +- [x] Repeatability / performance / overhead analysis +- [x] Automated JSON error reporting with retention -## 🎤 Interview TL;DR +**Backlog (intentionally deferred)** -- I built a config-driven image QA framework with a clear `Engine -> Model -> Eval` architecture. -- Core design guarantees are centralized in `Core Guarantees (Source of Truth)` to reduce documentation drift. -- I focused on decision reliability by adding arbitration, bias/error tracking, and automated JSON error reporting with retention cleanup. +- **Multi-threading for very large batches**: not on the near-term roadmap so batch runs stay **single-threaded and easier to reproduce** in CI, benchmarks, and incident debugging. Revisit only after profiling shows preprocessing (not inference I/O) as the clear bottleneck. +- **Extended OpenCV visual diagnostics**: basic histogram-based exposure checks already live in `engine/image_validator.py`; richer diagnostics (e.g. saliency, segmentation-assisted QA) stay **out of scope** until there is a concrete partner or product requirement, to avoid scope creep ahead of a stable inference contract. -## 🧪 CI/CD and Coverage +
-- Workflow: `.github/workflows/ci.yml` -- Stages: - - `lint`: `ruff check` on selected paths in `src/` plus `tests/` (same scope as workflow file) - - `unit tests + coverage`: `PYTHONPATH=src pytest` (produces `coverage.xml`) - - `report generation`: `--performance-analysis --overhead-analysis` - - `artifact upload`: `coverage.xml` and `results/` +## Author -Local run: - -```bash -pip install -r requirements.txt -pip install pytest pytest-cov ruff -PYTHONPATH=src pytest -``` - -## 🎬 Demo Screenshot / GIF - -Add demo media files under: - -- `assets/demo.gif` (recommended) -- `assets/demo.png` - -Then embed with: - -```markdown -![Framework Demo](assets/demo.gif) -``` +Cheryl - AI Optimization & Testing Engineer -## 👨‍💻 My Contributions +## Contributing -**Independently implemented with full-stack ownership of the test lifecycle.** +See [CONTRIBUTING.md](CONTRIBUTING.md) for local setup, tests, lint, and pull request expectations. -- 🧩 **Architecture**: designed the `Engine -> Model -> Eval` system boundaries and decision flow. -- 🐍 **Framework Development**: implemented the Python pipeline, adapters, and guardrail-driven loopback logic. -- 📏 **Evaluation Logic**: built arbitration, release gating, and reliability-oriented quality checks. -- 📊 **Benchmark & Reporting**: delivered repeatability/performance analysis and JSON report outputs. -- ⚙️ **CI/CD**: set up lint, test, coverage, and artifact workflows in GitHub Actions. -- 📝 **Documentation**: authored and maintained technical design, usage guides, and project narratives. +## License -👤 Author -Cheryl - AI Optimization & Testing Engineer \ No newline at end of file +This project is licensed under the [MIT License](LICENSE). diff --git a/app.py b/app.py new file mode 100644 index 0000000..a5844c0 --- /dev/null +++ b/app.py @@ -0,0 +1,359 @@ +from __future__ import annotations + +import logging +import re +import random +import time +from pathlib import Path +from typing import Dict, Optional, Protocol, Tuple, cast + +import numpy as np +import streamlit as st +from PIL import Image, UnidentifiedImageError + +from util.cli_logging import configure_cli_logging + +logger = logging.getLogger(__name__) +configure_cli_logging() + +QualityOrchestrator: Optional[type] = None +try: + from agent.orchestrator import QualityOrchestrator as _QualityOrchestrator + + QualityOrchestrator = _QualityOrchestrator +except ImportError as exc: + logger.warning( + "QualityOrchestrator not importable (%s). From repo root run: " + "PYTHONPATH=src streamlit run app.py", + exc, + ) + + +class _PipelineRunner(Protocol): + def run_pipeline(self, image_metrics: Dict[str, object]) -> Dict[str, object]: ... + + +def _get_orchestrator() -> Optional[_PipelineRunner]: + if QualityOrchestrator is None: + return None + if "orchestrator" not in st.session_state: + st.session_state["orchestrator"] = QualityOrchestrator() + return cast(_PipelineRunner, st.session_state["orchestrator"]) + + +def _build_sample_image() -> Image.Image: + width, height = 640, 360 + x_gradient = np.linspace(40, 220, width, dtype=np.uint8) + y_gradient = np.linspace(0, 25, height, dtype=np.uint8).reshape(height, 1) + red = np.tile(x_gradient, (height, 1)) + green = np.clip(red.astype(np.int16) + y_gradient - 15, 0, 255).astype(np.uint8) + blue = np.clip(240 - red // 2 + y_gradient, 0, 255).astype(np.uint8) + rgb = np.dstack([red, green, blue]) + return Image.fromarray(rgb, mode="RGB") + + +def _load_sample_image() -> Tuple[Image.Image, str]: + for candidate in (Path("sample.jpg"), Path("assets/sample.jpg")): + if candidate.exists(): + return Image.open(candidate).convert("RGB"), str(candidate) + return _build_sample_image(), "generated" + + +def _extract_metrics(image: Image.Image) -> Dict[str, object]: + gray = np.asarray(image.convert("L"), dtype=np.float32) + brightness = float(gray.mean()) + noise_level = float(gray.std()) + gy, gx = np.gradient(gray) + sharpness = float(np.var(gx) + np.var(gy)) + return { + "id": "streamlit_uploaded_image", + "brightness": round(brightness, 2), + "sharpness": round(sharpness, 2), + "noise_level": round(noise_level, 2), + } + + +def _analyze_mock() -> Dict[str, object]: + time.sleep(1.0) + # Baseline mode intentionally simulates unstable manual-style judgments. + score = random.randint(48, 76) + confidence = round(random.uniform(0.45, 0.72), 2) + label = "REVIEW" if score >= 60 else "FAIL" + return { + "score": score, + "confidence": confidence, + "label": label, + "explanation": ( + "Manual-like baseline: subjective and less consistent; " + "this mode is for contrast against AI pipeline stability." + ), + "raw": { + "reviewer_note": random.choice( + [ + "Looks acceptable but uncertain under low light.", + "Borderline sharpness; might require human review.", + "Inconsistent judgement due to subjective threshold.", + ] + ) + }, + } + + +def _normalize_real_result(report: Dict[str, object]) -> Dict[str, object]: + verdict = str(report.get("final_verdict", "REVIEW")) + score_map = {"PASS": 90, "GO": 90, "REVIEW": 70, "FAIL": 40, "NO_GO": 30} + confidence_map = {"PASS": 0.90, "GO": 0.90, "REVIEW": 0.75, "FAIL": 0.60, "NO_GO": 0.55} + return { + "score": score_map.get(verdict, 65), + "confidence": confidence_map.get(verdict, 0.70), + "label": verdict, + "explanation": ( + f"Pipeline stage: {report.get('stage', 'Unknown')}; " + f"engine: {report.get('engine', 'N/A')}" + ), + "raw": report, + } + + +def run_analysis(image: Image.Image, mode: str) -> Dict[str, object]: + if mode == "Manual Baseline (for contrast)": + return _analyze_mock() + + orchestrator = _get_orchestrator() + if orchestrator is None: + return { + "score": 65, + "confidence": 0.7, + "label": "Fallback", + "explanation": "Real pipeline unavailable, fallback to stub output.", + "raw": {"reason": "agent.orchestrator import failed; use PYTHONPATH=src from repo root"}, + } + + try: + metrics = _extract_metrics(image) + report = orchestrator.run_pipeline(metrics) + return _normalize_real_result(report) + except Exception as exc: + logger.exception("AI pipeline run failed") + return { + "score": 60, + "confidence": 0.6, + "label": "Error", + "explanation": "Real pipeline failed; check raw output for details.", + "raw": {"error": str(exc)}, + } + + +def parse_llm_output(text: str) -> Dict[str, object]: + """Demo parser for messy LLM text to normalized fields.""" + parsed: Dict[str, object] = {"score": None, "confidence": None, "label": "unknown"} + lower_text = text.lower() + + score_match = re.search(r"(?:score|points?)\s*[:=]?\s*(\d{1,3})", lower_text) + if score_match: + score = int(score_match.group(1)) + parsed["score"] = max(0, min(score, 100)) + else: + # Fallback: first plausible 0~100 integer in text + generic = re.search(r"\b(\d{1,3})\b", lower_text) + if generic: + score = int(generic.group(1)) + if 0 <= score <= 100: + parsed["score"] = score + + confidence_match = re.search(r"(?:confidence)\s*[:=]?\s*(0(?:\.\d+)?|1(?:\.0+)?)", lower_text) + if confidence_match: + parsed["confidence"] = float(confidence_match.group(1)) + + if any(keyword in lower_text for keyword in ("excellent", "good", "great", "pass")): + parsed["label"] = "positive" + elif any(keyword in lower_text for keyword in ("bad", "poor", "fail")): + parsed["label"] = "negative" + + return parsed + + +st.set_page_config(page_title="Agentic Testing Framework", layout="wide") + +st.title("Agentic Testing Framework") +st.markdown("Replace manual image QA with a repeatable, config-driven pipeline.") +st.caption("From slow & inconsistent to fast & scalable") + +if "result" not in st.session_state: + st.session_state["result"] = None +if "compare_result" not in st.session_state: + st.session_state["compare_result"] = None +if "selected_image" not in st.session_state: + st.session_state["selected_image"] = None +if "source_name" not in st.session_state: + st.session_state["source_name"] = "" + +with st.sidebar: + st.header("Settings") + mode = st.selectbox( + "Analysis mode", + options=["Manual Baseline (for contrast)", "AI Pipeline (real)"], + index=0, + ) + show_raw = st.checkbox("Show raw output", value=True) + show_latency = st.checkbox("Show latency", value=True) + use_sample = st.button("Try sample image") + if mode == "AI Pipeline (real)" and QualityOrchestrator is None: + st.warning( + "`agent.orchestrator` could not be imported. From the **repository root** run: " + "`PYTHONPATH=src streamlit run app.py`" + ) + +uploaded_file = st.file_uploader("Upload image", type=["png", "jpg", "jpeg"]) + +if use_sample: + sample_image, source_name = _load_sample_image() + st.session_state["selected_image"] = sample_image + st.session_state["source_name"] = source_name + st.session_state["result"] = None + st.session_state["compare_result"] = None +elif uploaded_file is not None: + try: + uploaded_image = Image.open(uploaded_file).convert("RGB") + st.session_state["selected_image"] = uploaded_image + st.session_state["source_name"] = uploaded_file.name + st.session_state["result"] = None + st.session_state["compare_result"] = None + except UnidentifiedImageError: + logger.warning("Upload rejected: not a decodable image") + st.error("Cannot decode this file as an image. Please upload PNG/JPG.") + except Exception as exc: + logger.exception("Failed to read uploaded image") + st.error(f"Failed to read upload: {exc}") + +image = st.session_state["selected_image"] + +if image is not None: + col1, col2 = st.columns(2) + with col1: + st.subheader("Input") + st.caption(f"Source: {st.session_state['source_name']}") + st.image(image, width="stretch") + + with col2: + st.subheader("Analysis Result") + if mode == "Manual Baseline (for contrast)": + st.caption("Baseline mode: simulates subjective/manual-style checks.") + else: + st.caption("AI mode: uses orchestrator pipeline for reproducible decisions.") + if st.button("Analyze", type="primary"): + start = time.time() + with st.spinner("Running AI + rules..."): + result = run_analysis(image, mode) + latency = time.time() - start + st.session_state["result"] = {"payload": result, "latency": latency} + st.session_state["compare_result"] = None + + if st.button("Compare Both Modes"): + with st.spinner("Running baseline and AI pipeline..."): + baseline_start = time.time() + baseline_result = run_analysis(image, "Manual Baseline (for contrast)") + baseline_latency = time.time() - baseline_start + + ai_start = time.time() + ai_result = run_analysis(image, "AI Pipeline (real)") + ai_latency = time.time() - ai_start + + st.session_state["compare_result"] = { + "baseline": {"payload": baseline_result, "latency": baseline_latency}, + "ai": {"payload": ai_result, "latency": ai_latency}, + } + st.session_state["result"] = None + + cached_result = st.session_state["result"] + if cached_result: + result = cached_result["payload"] + st.metric("Score", f"{result['score']}/100") + st.metric("Confidence", f"{result['confidence']:.2f}") + st.write(f"**Label:** {result['label']}") + if show_latency: + st.write(f"Latency: {cached_result['latency']:.2f}s") + st.markdown("### Explanation") + st.write(result["explanation"]) + st.info("Robust parsing layer keeps model output structured for downstream QA.") + if show_raw: + with st.expander("Raw output"): + st.json(result["raw"]) + + compare_result = st.session_state["compare_result"] + if compare_result: + st.markdown("### Side-by-side Comparison") + compare_left, compare_right = st.columns(2) + + baseline = compare_result["baseline"] + ai = compare_result["ai"] + + with compare_left: + st.markdown("**Manual Baseline**") + st.metric("Score", f"{baseline['payload']['score']}/100") + st.metric("Confidence", f"{baseline['payload']['confidence']:.2f}") + st.write(f"Label: {baseline['payload']['label']}") + if show_latency: + st.write(f"Latency: {baseline['latency']:.2f}s") + + with compare_right: + st.markdown("**AI Pipeline**") + st.metric("Score", f"{ai['payload']['score']}/100") + st.metric("Confidence", f"{ai['payload']['confidence']:.2f}") + st.write(f"Label: {ai['payload']['label']}") + if show_latency: + st.write(f"Latency: {ai['latency']:.2f}s") + + delta_score = ai["payload"]["score"] - baseline["payload"]["score"] + st.info(f"AI minus Baseline score delta: {delta_score:+.0f} points") +else: + st.info("Upload an image or click 'Try sample image' to start.") + +st.divider() +st.subheader("Impact") +left, right = st.columns(2) +with left: + st.markdown("### Before: Manual / Subjective Checks") + st.write("- Human judgment varies by reviewer") + st.write("- Hard to keep thresholds consistent") + st.write("- Slower and less traceable decisions") +with right: + st.markdown("### After: AI Pipeline Decisions") + st.write("- Config-driven, repeatable decision policy") + st.write("- Structured output for audit and CI") + st.write("- Fast, scalable, and easier to govern") + +st.divider() +st.subheader("How It Works") +st.markdown( + """ +1. Extract image features (blur, noise, exposure) +2. Apply rule-based validation +3. Use AI for semantic reasoning +4. Normalize output into structured format +""" +) + +st.divider() +st.subheader("LLM Output Parsing Demo") +st.caption("Raw LLM output can be messy; parsing normalizes it into stable structured data.") + +raw_outputs = [ + "Score: 85/100, confidence: 0.91, label: good", + "I think this image is around 78 points with decent quality.", + "Result => score=92; label=excellent; confidence=0.95", + "This looks bad. Probably 60.", +] +raw_text = st.selectbox("Select LLM output example", raw_outputs) +demo_left, demo_right = st.columns(2) +with demo_left: + st.markdown("### Raw LLM Output") + st.code(raw_text) +with demo_right: + st.markdown("### Parsed Output") + st.json(parse_llm_output(raw_text)) + +st.info("Without parsing: unstable system. With parsing: reliable pipeline.") + +st.divider() +st.caption("Demo for AI-powered testing / DevRel showcase") \ No newline at end of file diff --git a/assets/streamlit-comparison.png b/assets/streamlit-comparison.png new file mode 100644 index 0000000..778077b Binary files /dev/null and b/assets/streamlit-comparison.png differ diff --git a/configs/base.json b/configs/base.json index db74c5c..880d6b6 100644 --- a/configs/base.json +++ b/configs/base.json @@ -6,7 +6,7 @@ "author": "Cheryl" }, "model_settings": { - "name": "PixelQA-Llama-4bit", + "name": "Agentic Testing Framework - Llama 4-bit", "bit_depth": 4, "quantization_format": "GGUF", "inference": { @@ -45,7 +45,19 @@ "logs": "logs/base" }, "runtime": { - "oom_probability": 0.0 + "oom_probability": 0.0, + "loopback_planner": { + "mode": "simulated", + "require_healthy_on_startup": true, + "llm": { + "host": "http://127.0.0.1:8080", + "endpoint": "/v1/chat/completions", + "model": "local-model", + "timeout_s": 20, + "temperature": 0.0, + "max_tokens": 200 + } + } }, "quality_gate": { "target_pass_rate": 85.0 diff --git a/configs/benchmark.json b/configs/benchmark.json index c66e92d..72be8bd 100644 --- a/configs/benchmark.json +++ b/configs/benchmark.json @@ -6,7 +6,7 @@ "author": "Cheryl" }, "model_settings": { - "name": "PixelQA-Llama-4bit", + "name": "Agentic Testing Framework - Llama 4-bit", "bit_depth": 4, "quantization_format": "GGUF", "inference": { diff --git a/configs/dev.json b/configs/dev.json index 0b99c39..6522adb 100644 --- a/configs/dev.json +++ b/configs/dev.json @@ -6,7 +6,7 @@ "author": "Cheryl" }, "model_settings": { - "name": "PixelQA-Llama-4bit", + "name": "Agentic Testing Framework - Llama 4-bit", "bit_depth": 4, "quantization_format": "GGUF", "inference": { @@ -33,7 +33,11 @@ "logs": "logs/dev" }, "runtime": { - "oom_probability": 0.0 + "oom_probability": 0.0, + "loopback_planner": { + "mode": "simulated", + "require_healthy_on_startup": true + } }, "quality_gate": { "target_pass_rate": 80.0 diff --git a/docs/Architecture.md b/docs/Architecture.md index 3aaeeac..603d509 100644 --- a/docs/Architecture.md +++ b/docs/Architecture.md @@ -1,6 +1,6 @@ -# Architecture: Inference Provider Abstraction (PixelQA-Llama) +# Architecture: Inference Provider Abstraction (Agentic Testing Framework) -This document explains the **Provider abstraction layer** used by PixelQA-Llama: how inference backends are selected, what contract they must satisfy, and how failures are normalized into a stable surface for evaluation and loopback. +This document explains the **Provider abstraction layer** used by this project: how inference backends are selected, what contract they must satisfy, and how failures are normalized into a stable surface for evaluation and loopback. In this codebase, **“Provider” = an inference backend implementation** behind a single orchestrator-facing API. @@ -17,7 +17,7 @@ In this codebase, **“Provider” = an inference backend implementation** behin - Entry: `src/ai_quality_agent.py` (CLI) → `QuantizedVisionAgent` → engine metrics (`vision_math`) → `build_inference_engine` → evaluation / arbitration → reports, plus optional **guardrail-driven loopback** on `NO_GO`. -This is the **main production-oriented path** for PixelQA-style runs. +This is the **main production-oriented path** for batch CLI runs (`ai_quality_agent.py`). **Secondary / demo — staged agent orchestrator** @@ -162,10 +162,10 @@ Decorators (`monitor_performance`, `async_monitor_performance`) **always log wal Enable traced memory in logs when profiling: ```bash -export PIXELQA_MONITOR_MEMORY=1 +export ATF_MONITOR_MEMORY=1 ``` -Accepted truthy values: `1`, `true`, `yes` (case-insensitive). When unset or false, completion logs include elapsed time only. +Accepted truthy values: `1`, `true`, `yes`, `on` (case-insensitive). Legacy alias `PIXELQA_MONITOR_MEMORY` is still honored. When unset or false, completion logs include elapsed time only. ## Adding a new Provider (checklist) diff --git a/docs/ModelInferenceSetup.md b/docs/ModelInferenceSetup.md new file mode 100644 index 0000000..ccdb398 --- /dev/null +++ b/docs/ModelInferenceSetup.md @@ -0,0 +1,139 @@ +# Real model inference setup + +Checklist for moving from **simulated** / fallback inference to a live vision-capable backend. The framework already supports `llama_cpp` and `ollama_vision`; this doc is the operational path. + +## Before you run a batch + +- [ ] At least one image in `test_images/` (or your profile’s `folders.input`). +- [ ] Inference server is running and reachable (see options below). +- [ ] `configs/dev.json` (or your profile) sets `model_settings.inference.backend` to the backend you intend. +- [ ] Timeouts are realistic for your hardware (`timeout_s` in merged config from `configs/base.json`). + +**Success signal in reports:** `decision.backend` is `llama_cpp` or `ollama_vision`, **not** `llama_cpp->simulated` or `ollama_vision->simulated`. The `msg` field should not mention “fallback to simulated inference”. + +--- + +## Option A: llama.cpp (OpenAI-compatible HTTP server) + +`dev` profile defaults to `llama_cpp` and merges host settings from `configs/base.json`: + +| Setting | Default (base) | +|---------|----------------| +| Host | `http://127.0.0.1:8080` | +| Endpoint | `/v1/chat/completions` | +| Model name | `local-model` (must match server) | +| Timeout | `45` seconds | + +### 1. Start the server + +Use your usual llama.cpp / llama-server launch so it exposes **chat completions** on port `8080` (or change config to match). The model name in the server CLI must match `llama_cpp.model` in config. + +### 2. Quick connectivity check + +```bash +PYTHONPATH=src python test_connection.py +``` + +Adjust URL/model inside `test_connection.py` if your server differs. You should see `Connected.` and no connection error. + +### 3. Optional dev overrides + +`configs/dev.json` only needs the backend key today; add a block under `model_settings.inference` if you use a non-default port or model id: + +```json +"inference": { + "backend": "llama_cpp", + "llama_cpp": { + "host": "http://127.0.0.1:8080", + "model": "your-gguf-model-id", + "timeout_s": 60 + } +} +``` + +### 4. Run a small batch (real model) + +```bash +python3 src/ai_quality_agent.py --profile dev \ + --inference-backend llama_cpp \ + --async-batch --async-concurrency 4 \ + --parallel-metrics +``` + +Start with a few images before `--stress-test-100`. + +--- + +## Option B: Ollama (vision model) + +### 1. Install and pull a vision model + +```bash +ollama pull llava:7b +ollama serve # if not already running +``` + +Default in `configs/base.json`: `http://localhost:11434`, model `llava:7b`. + +### 2. Check the API + +```bash +curl -s http://localhost:11434/api/tags +``` + +### 3. Point config at Ollama + +CLI (no file edit): + +```bash +python3 src/ai_quality_agent.py --profile dev --inference-backend ollama_vision +``` + +Or in config: + +```json +"inference": { + "backend": "ollama_vision", + "ollama": { + "host": "http://localhost:11434", + "model": "llava:7b", + "timeout_s": 45 + } +} +``` + +### 4. Run batch + +Same as Option A step 4; use `--inference-backend ollama_vision` if not set in JSON. + +--- + +## Which CLI flags when the model is live + +| Flag | When to use | +|------|-------------| +| `--parallel-metrics` | Many images; CPU metrics (Pillow) are a large share of wall time. | +| `--async-batch` | Waiting on HTTP inference; limits in-flight requests with `--async-concurrency` (default 4). | +| `--repeatability-test dev --repeatability-runs 5` | Check model output stability across runs (meaningless for pure simulated). | + +Simulated-only dev work does **not** need `--async-batch`; real model batches benefit from **both** async I/O and parallel metrics. + +--- + +## Troubleshooting + +| Symptom | Likely cause | +|---------|----------------| +| `404` on `127.0.0.1:8080` | llama server not running or wrong port/path | +| `backend`: `...->simulated` | Request failed; see `msg` for exception text | +| Same pass rate as before, very low latency | Still on simulated path | +| Timeouts / `ERR_MODEL_BACKEND_503` | Increase `timeout_s`; reduce `--async-concurrency` | +| Ollama errors | Model not pulled, wrong host, or non-vision model | + +--- + +## CI vs local + +CI continues to use **simulated** inference for deterministic, fast gates. Real model runs are **local or staging** until you add optional integration jobs with a pinned server image. + +See also: [Architecture.md](Architecture.md), README § “Config-only inference backend selection”. diff --git a/pyproject.toml b/pyproject.toml index b51b384..1360c17 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,20 +1,112 @@ +[build-system] +requires = ["setuptools>=61", "wheel"] +build-backend = "setuptools.build_meta" + [project] name = "agentic_testing_framework" version = "0.1.0" +description = "Configuration-driven image QA with release gating (GO / REVIEW / NO_GO) and multi-backend inference." +readme = "README.md" +requires-python = ">=3.9" +# Single source of dependency pins for this repo (see CONTRIBUTING.md). dependencies = [ - "requests", + "annotated-types==0.7.0", + "anyio==4.13.0", + "beautifulsoup4==4.14.3", + "black==26.3.1", + "certifi==2026.2.25", + "cffi==2.0.0", + "charset-normalizer==3.4.7", + "click==8.3.2", + "chromadb", + "contourpy==1.3.3", + "curl_cffi==0.13.0", + "cycler==0.12.1", + "fonttools==4.62.1", + "frozendict==2.4.7", + "h11==0.16.0", + "httpcore==1.0.9", + "httpx==0.28.1", + "idna==3.11", + "joblib==1.5.3", + "jsonpatch==1.33", + "jsonpointer==3.1.1", + "kiwisolver==1.5.0", + "langchain-core==1.2.25", + "langchain-ollama==1.0.1", + "langsmith==0.7.24", + "matplotlib==3.10.8", + "multitasking==0.0.12", + "mypy_extensions==1.1.0", + "numpy==2.4.4", + "ollama==0.6.1", + "opencv-python-headless==4.13.0.92", + "orjson==3.11.8", + "packaging==26.0", + "pandas==3.0.2", + "pathspec==1.0.4", + "peewee==4.0.4", + "pillow==12.2.0", + "platformdirs==4.9.4", + "protobuf==7.34.1", + "psutil==7.2.2", + "pycparser==3.0", + "pydantic==2.12.5", + "pydantic_core==2.41.5", + "pyparsing==3.3.2", + "python-dateutil==2.9.0.post0", + "pytokens==0.4.1", + "pytz==2026.1.post1", + "PyYAML==6.0.3", + "requests==2.33.1", + "requests-toolbelt==1.0.0", + "scikit-learn==1.8.0", + "scipy==1.17.1", + "sentence-transformers", + "six==1.17.0", + "soupsieve==2.8.3", + "streamlit==1.57.0", + "tenacity==9.1.4", + "threadpoolctl==3.6.0", + "typing-inspection==0.4.2", + "typing_extensions==4.15.0", + "urllib3==2.6.3", + "uuid_utils==0.14.1", + "websockets==16.0", + "xxhash==3.6.0", + "zstandard==0.25.0", ] [project.optional-dependencies] dev = [ "pytest", "pytest-cov", + "ruff", + "mypy", ] -[build-system] -requires = ["setuptools", "wheel"] -build-backend = "setuptools.build_meta" +[tool.setuptools] +py-modules = [ + "ai_quality_agent", + "mock_device", + "verify_capture_success", + "test_failure_memory_retrieval", +] + +[tool.setuptools.package-dir] +"" = "src" + +[tool.setuptools.packages.find] +where = ["src"] [tool.pytest.ini_options] -addopts = "-q --cov=src --cov-report=term-missing --cov-report=xml" -testpaths = ["tests"] \ No newline at end of file +addopts = "-q --cov=src --cov-report=term-missing --cov-report=xml --cov-fail-under=34" +testpaths = ["tests"] + +[tool.mypy] +python_version = "3.11" +explicit_package_bases = true +ignore_missing_imports = true +warn_unused_ignores = true +check_untyped_defs = false +disallow_untyped_defs = false diff --git a/requirements.txt b/requirements.txt index 41c77a3..59b96ba 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,64 +1,4 @@ -annotated-types==0.7.0 -anyio==4.13.0 -beautifulsoup4==4.14.3 -black==26.3.1 -certifi==2026.2.25 -cffi==2.0.0 -charset-normalizer==3.4.7 -click==8.3.2 -chromadb -contourpy==1.3.3 -curl_cffi==0.13.0 -cycler==0.12.1 -fonttools==4.62.1 -frozendict==2.4.7 -h11==0.16.0 -httpcore==1.0.9 -httpx==0.28.1 -idna==3.11 -joblib==1.5.3 -jsonpatch==1.33 -jsonpointer==3.1.1 -kiwisolver==1.5.0 -langchain-core==1.2.25 -langchain-ollama==1.0.1 -langsmith==0.7.24 -matplotlib==3.10.8 -multitasking==0.0.12 -mypy_extensions==1.1.0 -numpy==2.4.4 -ollama==0.6.1 -orjson==3.11.8 -packaging==26.0 -pandas==3.0.2 -pathspec==1.0.4 -peewee==4.0.4 -pillow==12.2.0 -platformdirs==4.9.4 -protobuf==7.34.1 -psutil==7.2.2 -pycparser==3.0 -pydantic==2.12.5 -pydantic_core==2.41.5 -pyparsing==3.3.2 -python-dateutil==2.9.0.post0 -pytokens==0.4.1 -pytz==2026.1.post1 -PyYAML==6.0.3 -requests==2.33.1 -requests-toolbelt==1.0.0 -scikit-learn==1.8.0 -scipy==1.17.1 -sentence-transformers -six==1.17.0 -soupsieve==2.8.3 -tenacity==9.1.4 -threadpoolctl==3.6.0 -typing-inspection==0.4.2 -typing_extensions==4.15.0 -urllib3==2.6.3 -uuid_utils==0.14.1 -websockets==16.0 -xxhash==3.6.0 -yfinance==1.2.0 -zstandard==0.25.0 +# Dependency version pins live only in pyproject.toml ([project.dependencies]). +# This file is a convenience shim for `pip install -r requirements.txt`. +# Equivalent: pip install -e ".[dev]" +-e .[dev] diff --git a/src/agent/loopback_planner.py b/src/agent/loopback_planner.py new file mode 100644 index 0000000..12f958d --- /dev/null +++ b/src/agent/loopback_planner.py @@ -0,0 +1,308 @@ +from __future__ import annotations + +import json +import logging +from importlib import import_module +from typing import Any, Dict, List, Optional, Protocol + +from models.contracts import LoopbackPlan + +logger = logging.getLogger(__name__) + +_REQUESTS_MODULE = None + + +def _get_requests(): + global _REQUESTS_MODULE + if _REQUESTS_MODULE is None: + _REQUESTS_MODULE = import_module("requests") + return _REQUESTS_MODULE + + +class LoopbackPlanner(Protocol): + def plan( + self, + *, + signal: str, + engine_metrics: Dict[str, Any], + thresholds_cfg: Dict[str, Any], + loopback_guard_cfg: Dict[str, Any], + attempt_history: List[Dict[str, Any]], + ) -> LoopbackPlan: + ... + + +class SimulatedLoopbackPlanner: + """Rule-based planner used as deterministic fallback.""" + + def plan( + self, + *, + signal: str, + engine_metrics: Dict[str, Any], + thresholds_cfg: Dict[str, Any], + loopback_guard_cfg: Dict[str, Any], + attempt_history: List[Dict[str, Any]], + ) -> LoopbackPlan: + brightness = float( + engine_metrics.get("avg_brightness", engine_metrics.get("brightness", 0.0)) + ) + sharpness = float(engine_metrics.get("sharpness", 0.0)) + min_brightness = float(thresholds_cfg.get("min_brightness", 40.0)) + max_brightness = float(thresholds_cfg.get("max_brightness", 220.0)) + min_sharpness = float(thresholds_cfg.get("min_sharpness", 20.0)) + overexposure_stop_ratio = float( + loopback_guard_cfg.get("overexposure_stop_ratio", 0.95) + ) + underexposure_stop_ratio = float( + loopback_guard_cfg.get("underexposure_stop_ratio", 1.05) + ) + + if signal == "under": + if brightness >= min_brightness: + return LoopbackPlan( + None, + "engine_disagrees_underexposed", + "model says under but engine brightness is acceptable", + planner_backend="simulated", + ) + if brightness >= (max_brightness * overexposure_stop_ratio): + return LoopbackPlan( + None, + "near_overexposure_guard", + "brighten would likely push image into over-exposure", + planner_backend="simulated", + ) + return LoopbackPlan( + "brighten", + "retry_scheduled", + "under-exposed signal and safe brightness headroom", + planner_backend="simulated", + ) + + if signal == "over": + if brightness <= max_brightness: + return LoopbackPlan( + None, + "engine_disagrees_overexposed", + "model says over but engine brightness is acceptable", + planner_backend="simulated", + ) + if brightness <= (min_brightness * underexposure_stop_ratio): + return LoopbackPlan( + None, + "near_underexposure_guard", + "dimming would likely push image into under-exposure", + planner_backend="simulated", + ) + return LoopbackPlan( + "dim", + "retry_scheduled", + "over-exposed signal and safe dimming headroom", + planner_backend="simulated", + ) + + if signal == "blurry": + if sharpness >= min_sharpness: + return LoopbackPlan( + None, + "engine_disagrees_blurry", + "model says blurry but engine sharpness is acceptable", + planner_backend="simulated", + ) + return LoopbackPlan( + "sharpen", + "retry_scheduled", + "blurry signal and low sharpness metric", + planner_backend="simulated", + ) + + return LoopbackPlan( + None, + f"signal_not_recoverable ({signal})", + "signal is outside supported recovery actions", + planner_backend="simulated", + ) + + +class LLMLoopbackPlanner: + """LLM planner that emits next_action JSON with fallback to simulated.""" + + VALID_ACTIONS = {"brighten", "dim", "sharpen", "stop"} + + def __init__(self, planner_cfg: Dict[str, Any], fallback_planner: LoopbackPlanner): + self.fallback_planner = fallback_planner + self.host = str(planner_cfg.get("host", "http://127.0.0.1:8080")).rstrip("/") + self.endpoint = str(planner_cfg.get("endpoint", "/v1/chat/completions")) + self.model = str(planner_cfg.get("model", "local-model")) + self.timeout_s = float(planner_cfg.get("timeout_s", 20.0)) + self.temperature = float(planner_cfg.get("temperature", 0.0)) + self.max_tokens = int(planner_cfg.get("max_tokens", 200)) + self.health_check_timeout_s = float( + planner_cfg.get("health_check_timeout_s", self.timeout_s) + ) + + def _build_payload( + self, + *, + signal: str, + engine_metrics: Dict[str, Any], + thresholds_cfg: Dict[str, Any], + loopback_guard_cfg: Dict[str, Any], + attempt_history: List[Dict[str, Any]], + ) -> Dict[str, Any]: + prompt = ( + "You are an image QA recovery planner.\n" + "Return STRICT JSON with keys: action, rationale.\n" + "Valid action: brighten, dim, sharpen, stop.\n" + "Choose stop if recovery is not safe or not meaningful.\n" + f"signal={signal}\n" + f"engine_metrics={json.dumps(engine_metrics, ensure_ascii=False)}\n" + f"thresholds={json.dumps(thresholds_cfg, ensure_ascii=False)}\n" + f"loopback_guard={json.dumps(loopback_guard_cfg, ensure_ascii=False)}\n" + f"attempt_history={json.dumps(attempt_history[-3:], ensure_ascii=False)}\n" + ) + return { + "model": self.model, + "messages": [{"role": "user", "content": prompt}], + "temperature": self.temperature, + "max_tokens": self.max_tokens, + "stream": False, + "response_format": {"type": "json_object"}, + } + + @staticmethod + def _extract_json_object(raw_text: str) -> Dict[str, Any]: + try: + return json.loads(raw_text) + except json.JSONDecodeError: + start = raw_text.find("{") + end = raw_text.rfind("}") + if start == -1 or end == -1 or end <= start: + return {} + try: + return json.loads(raw_text[start : end + 1]) + except json.JSONDecodeError: + return {} + + def plan( + self, + *, + signal: str, + engine_metrics: Dict[str, Any], + thresholds_cfg: Dict[str, Any], + loopback_guard_cfg: Dict[str, Any], + attempt_history: List[Dict[str, Any]], + ) -> LoopbackPlan: + payload = self._build_payload( + signal=signal, + engine_metrics=engine_metrics, + thresholds_cfg=thresholds_cfg, + loopback_guard_cfg=loopback_guard_cfg, + attempt_history=attempt_history, + ) + url = f"{self.host}{self.endpoint}" + logger.info("Loopback planner (llm): requesting next action from %s", url) + try: + response = _get_requests().post(url, json=payload, timeout=self.timeout_s) + response.raise_for_status() + body = response.json() + content = str( + body.get("choices", [{}])[0].get("message", {}).get("content", "") + ) + parsed = self._extract_json_object(content) + action = str(parsed.get("action", "stop")).lower() + rationale = str(parsed.get("rationale", "planner returned no rationale")) + if action not in self.VALID_ACTIONS: + logger.warning( + "Loopback planner (llm): invalid action=%s, fallback planner is used", + action, + ) + fallback_plan = self.fallback_planner.plan( + signal=signal, + engine_metrics=engine_metrics, + thresholds_cfg=thresholds_cfg, + loopback_guard_cfg=loopback_guard_cfg, + attempt_history=attempt_history, + ) + return LoopbackPlan( + action=fallback_plan.action, + stop_reason=fallback_plan.stop_reason, + rationale=fallback_plan.rationale, + fallback_used=True, + planner_backend="llm->simulated", + ) + if action == "stop": + return LoopbackPlan( + None, "planner_stop", rationale, fallback_used=False, planner_backend="llm" + ) + return LoopbackPlan( + action, "retry_scheduled", rationale, fallback_used=False, planner_backend="llm" + ) + except Exception as exc: + logger.warning( + "Loopback planner (llm): failed with %s, fallback planner is used", + exc, + ) + fallback_plan = self.fallback_planner.plan( + signal=signal, + engine_metrics=engine_metrics, + thresholds_cfg=thresholds_cfg, + loopback_guard_cfg=loopback_guard_cfg, + attempt_history=attempt_history, + ) + return LoopbackPlan( + action=fallback_plan.action, + stop_reason=fallback_plan.stop_reason, + rationale=fallback_plan.rationale, + fallback_used=True, + planner_backend="llm->simulated", + ) + + def ensure_healthy(self) -> None: + """ + Fail fast when planner endpoint is unreachable. + """ + url = f"{self.host}{self.endpoint}" + payload = { + "model": self.model, + "messages": [{"role": "user", "content": "health_check"}], + "temperature": 0.0, + "max_tokens": 1, + "stream": False, + } + try: + response = _get_requests().post( + url, + json=payload, + timeout=self.health_check_timeout_s, + ) + logger.info( + "Loopback planner health check: reachable endpoint %s (status=%s)", + url, + response.status_code, + ) + except Exception as exc: + raise RuntimeError( + f"LLM planner server is not reachable at {url}. " + "Start the planner backend server or switch --loopback-planner simulated." + ) from exc + + +def create_loopback_planner(config: Dict[str, Any]) -> LoopbackPlanner: + runtime_cfg = config.get("runtime", {}) + planner_cfg = runtime_cfg.get("loopback_planner", {}) + planner_mode = str(planner_cfg.get("mode", "simulated")).lower() + simulated = SimulatedLoopbackPlanner() + if planner_mode == "llm": + llm_cfg = planner_cfg.get("llm", {}) + require_healthy_on_startup = bool( + planner_cfg.get("require_healthy_on_startup", True) + ) + logger.info("Loopback planner: LLM mode enabled") + planner = LLMLoopbackPlanner(planner_cfg=llm_cfg, fallback_planner=simulated) + if require_healthy_on_startup: + planner.ensure_healthy() + return planner + logger.info("Loopback planner: simulated mode enabled") + return simulated diff --git a/src/agent/orchestrator.py b/src/agent/orchestrator.py index e2e0220..973d554 100644 --- a/src/agent/orchestrator.py +++ b/src/agent/orchestrator.py @@ -1,8 +1,13 @@ import json +import logging import time from models.gemma_filter import GemmaFilter from models.llama_analyst import LlamaAnalyst +from util.cli_logging import configure_cli_logging + +logger = logging.getLogger(__name__) + class QualityOrchestrator: def __init__(self): @@ -11,80 +16,82 @@ def __init__(self): def run_pipeline(self, image_metrics): image_id = image_metrics.get("id", "Unknown_IMG") - print(f"\n--- [Pipeline Start] Analyzing {image_id} ---") + logger.info("\n--- [Pipeline Start] Analyzing %s ---", image_id) # --- Stage 1: 快速過濾 --- - print(f"Step 1: Running Gemma-2b for basic check...") + logger.info("Step 1: Running Gemma-2b for basic check...") gemma_raw_response = self.gemma_filter.check_basic_quality(image_metrics) - + gemma_res = self._parse_json(gemma_raw_response) if not gemma_res or not gemma_res.get("pass"): - reason = gemma_res.get('reason') if gemma_res else "Gemma analysis failed" - print(f"❌ Rejected by Gemma: {reason}") + reason = gemma_res.get("reason") if gemma_res else "Gemma analysis failed" + logger.info("Rejected by Gemma: %s", reason) return { "id": image_id, "final_verdict": "FAIL", "stage": "Filter", - "details": gemma_res + "details": gemma_res, } - print(f"✅ Passed Gemma Filter. Reason: {gemma_res.get('reason')}") - + logger.info("Passed Gemma Filter. Reason: %s", gemma_res.get("reason")) + # 在兩個大模型切換間隙，讓 CPU 稍微冷卻 0.5 秒 time.sleep(0.5) # --- Stage 2: 深度分析 --- - print(f"Step 2: Dispatching to Llama-3.1 for deep analysis...") + logger.info("Step 2: Dispatching to Llama-3.1 for deep analysis...") llama_raw_response = self.llama_analyst.analyze_quality(image_metrics) - + llama_res = self._parse_json(llama_raw_response) if not llama_res or llama_res.get("verdict") == "Error": - print(f"⚠️ Llama Analysis stopped by Safety Guard.") + logger.warning("Llama Analysis stopped by Safety Guard.") return {"id": image_id, "error": "Llama analysis timeout or error"} # --- Stage 3: 彙整最終報告 --- - print(f"✅ Final Verdict: {llama_res.get('verdict')}") - + logger.info("Final Verdict: %s", llama_res.get("verdict")) + return { "id": image_id, "final_verdict": llama_res.get("verdict"), "stage": "Full Pipeline", "filter_check": "PASS", "detailed_analysis": llama_res.get("analysis"), - "engine": "Llama-3.1-8b-Q4_K_M" + "engine": "Llama-3.1-8b-Q4_K_M", } def _parse_json(self, text): """ 終極 JSON 解析器：處理大小寫、多餘文字及編碼問題 """ - if not text: return None + if not text: + return None try: # 修正 Python vs JSON 布林值與空值 processed_text = text.replace(": True", ": true").replace(": False", ": false").replace(": None", ": null") - - start_idx = processed_text.find('{') - end_idx = processed_text.rfind('}') - + + start_idx = processed_text.find("{") + end_idx = processed_text.rfind("}") + if start_idx != -1 and end_idx != -1: - json_str = processed_text[start_idx:end_idx + 1] + json_str = processed_text[start_idx : end_idx + 1] return json.loads(json_str) return None - except Exception as e: - print(f"Parsing error logic triggered. Raw snippet: {text[:50]}...") + except Exception: + logger.warning("Parsing error logic triggered. Raw snippet: %s...", text[:50]) return None + if __name__ == "__main__": - # 測試用例 + configure_cli_logging() test_metrics = { "id": "Test_Photo_PASS_CASE", "brightness": 120, "sharpness": 85, - "noise_level": 12 + "noise_level": 12, } - + orchestrator = QualityOrchestrator() report = orchestrator.run_pipeline(test_metrics) - - print("\n--- [Final Report Summary] ---") - print(json.dumps(report, indent=4)) \ No newline at end of file + + logger.info("\n--- [Final Report Summary] ---") + logger.info("%s", json.dumps(report, indent=4)) diff --git a/src/ai_quality_agent.py b/src/ai_quality_agent.py index 810a3e0..f798519 100644 --- a/src/ai_quality_agent.py +++ b/src/ai_quality_agent.py @@ -1,18 +1,30 @@ import argparse +import asyncio import json +import logging import os import random import threading import time import traceback +from contextlib import nullcontext from datetime import datetime from pathlib import Path from statistics import pvariance +from typing import Any, Dict, List, Optional +import httpx from PIL import Image, ImageDraw, ImageEnhance import psutil +from util.cli_logging import configure_cli_logging from util.failure_memory import FailureMemoryStore +from util.metrics_pool import MetricsProcessPool +from util.monitor_performance import async_monitor_performance, gather_with_timing +from agent.loopback_planner import ( + SimulatedLoopbackPlanner, + create_loopback_planner, +) from engine.image_processor import ImageProcessor from engine.vision_math import calculate_metrics from eval.arbitrator import ( @@ -25,8 +37,12 @@ generate_benchmark_insights, get_release_decision, ) +from models.async_inference import predict_quality_async +from models.contracts import AgentInferenceOutput, AgentStep from models.inference_adapter import build_inference_engine +logger = logging.getLogger(__name__) + DEFAULT_CONFIG = { "model_settings": {"name": "Default-Model", "bit_depth": 4}, "thresholds": {"min_sharpness": 20, "min_brightness": 45, "max_brightness": 220}, @@ -85,7 +101,7 @@ def cleanup_old_reports(output_folder_path, max_age_days=REPORT_RETENTION_DAYS): os.remove(file_path) deleted_count += 1 except OSError as e: - print(f"WARNING: Could not remove old report {file_path}: {e}") + logger.warning("Could not remove old report %s: %s", file_path, e) return deleted_count @@ -124,7 +140,7 @@ def ensure_sample_images(image_folder_path): draw.line((idx, 0, idx, 127), fill=255 - intensity) image.save(os.path.join(image_folder_path, file_name)) - print("No input images found. Generated sample dataset in input folder.") + logger.info("No input images found. Generated sample dataset in input folder.") def ensure_stress_test_images(image_folder_path, target_count=100): @@ -164,7 +180,7 @@ def ensure_stress_test_images(image_folder_path, target_count=100): out_name = f"stress_{source_name}_{idx + 1:03d}.jpg" enhancer.save(os.path.join(image_folder_path, out_name), quality=90) - print(f"Stress-test mode: ensured at least {target_count} images in input folder.") + logger.info("Stress-test mode: ensured at least %s images in input folder.", target_count) def load_config(profile="dev", config_path=None): @@ -206,14 +222,67 @@ def load_config(profile="dev", config_path=None): return merged_config, "DEFAULT_CONFIG" +def _apply_runtime_overrides( + config: Dict[str, Any], + config_source: str, + *, + inference_backend_override: Optional[str] = None, + loopback_planner_override: Optional[str] = None, + planner_timeout_s_override: Optional[float] = None, + planner_model_override: Optional[str] = None, + planner_require_healthy_override: Optional[bool] = None, +): + source = config_source + if inference_backend_override: + config.setdefault("model_settings", {}).setdefault("inference", {}) + config["model_settings"]["inference"]["backend"] = inference_backend_override + source = f"{source} + CLI(backend={inference_backend_override})" + if loopback_planner_override: + config.setdefault("runtime", {}).setdefault("loopback_planner", {}) + config["runtime"]["loopback_planner"]["mode"] = loopback_planner_override + source = f"{source} + CLI(loopback_planner={loopback_planner_override})" + if planner_timeout_s_override is not None: + config.setdefault("runtime", {}).setdefault("loopback_planner", {}).setdefault("llm", {}) + config["runtime"]["loopback_planner"]["llm"]["timeout_s"] = float(planner_timeout_s_override) + source = f"{source} + CLI(planner_timeout_s={planner_timeout_s_override})" + if planner_model_override: + config.setdefault("runtime", {}).setdefault("loopback_planner", {}).setdefault("llm", {}) + config["runtime"]["loopback_planner"]["llm"]["model"] = planner_model_override + source = f"{source} + CLI(planner_model={planner_model_override})" + if planner_require_healthy_override is not None: + config.setdefault("runtime", {}).setdefault("loopback_planner", {}) + config["runtime"]["loopback_planner"]["require_healthy_on_startup"] = bool( + planner_require_healthy_override + ) + source = ( + f"{source} + CLI(planner_require_healthy={planner_require_healthy_override})" + ) + return source + + class QuantizedVisionAgent: - def __init__(self, config): + def __init__(self, config, metrics_pool: Optional[MetricsProcessPool] = None): self.config = config self.model_info = config["model_settings"] self.inference_engine = build_inference_engine(config) + self.metrics_pool = metrics_pool self.oom_probability = float(config.get("runtime", {}).get("oom_probability", 0.0)) - print(f"Startup mode: {self.model_info['name']} ({self.model_info['bit_depth']}-bit)") - print(f"Inference backend: {self.inference_engine.backend_name}") + logger.info( + "Startup mode: %s (%s-bit)", + self.model_info["name"], + self.model_info["bit_depth"], + ) + logger.info("Inference backend: %s", self.inference_engine.backend_name) + if metrics_pool is not None: + logger.info( + "Metrics compute: process pool (max_workers=%s)", + metrics_pool.max_workers, + ) + + def _compute_metrics(self, photo_path: str): + if self.metrics_pool is not None: + return self.metrics_pool.calculate(photo_path) + return calculate_metrics(photo_path) def get_all_photos(self): folder_name = self.config["folders"]["input"] @@ -234,7 +303,7 @@ def get_all_photos(self): def analyze_photo_quality(self, photo_path): start_time = time.time() - metrics = calculate_metrics(photo_path) + metrics = self._compute_metrics(photo_path) if metrics is None: return None, {"decision": "Error", "code": "ERR_SYS_IO_404", "msg": "Unable to read file"}, 0 @@ -242,6 +311,41 @@ def analyze_photo_quality(self, photo_path): latency = round((time.time() - start_time) * 1000, 2) return metrics, ai_result, latency + @async_monitor_performance + async def analyze_photo_quality_async( + self, photo_path: str, http_client: httpx.AsyncClient + ): + logger.debug( + "analyze_photo_quality_async: path=%s backend=%s", + photo_path, + self.inference_engine.backend_name, + ) + start_time = time.time() + if self.metrics_pool is not None: + loop = asyncio.get_running_loop() + metrics = await loop.run_in_executor( + self.metrics_pool.executor, + calculate_metrics, + photo_path, + ) + else: + metrics = await asyncio.to_thread(calculate_metrics, photo_path) + if metrics is None: + logger.warning("analyze_photo_quality_async: unable to read %s", photo_path) + return None, {"decision": "Error", "code": "ERR_SYS_IO_404", "msg": "Unable to read file"}, 0 + + ai_result = await predict_quality_async( + self.inference_engine, http_client, photo_path, metrics + ) + latency = round((time.time() - start_time) * 1000, 2) + logger.debug( + "analyze_photo_quality_async: done path=%s decision=%s latency_ms=%.2f", + photo_path, + ai_result.get("decision"), + latency, + ) + return metrics, ai_result, latency + def save_batch_report(report_data, output_folder): current_dir = os.path.dirname(os.path.abspath(__file__)) @@ -256,13 +360,17 @@ def save_batch_report(report_data, output_folder): with open(file_path, "w", encoding="utf-8") as f: json.dump(report_data, f, indent=4, ensure_ascii=False) - print(f"\nBatch test completed. Full report saved to: {file_path}") + logger.info("Batch test completed. Full report saved to: %s", file_path) deleted_count = cleanup_old_reports(full_output_path) if deleted_count > 0: - print(f"Cleaned up {deleted_count} report(s) older than {REPORT_RETENTION_DAYS} days.") + logger.info( + "Cleaned up %s report(s) older than %s days.", + deleted_count, + REPORT_RETENTION_DAYS, + ) current_report_count = count_reports(full_output_path) - print(f"Current report count: {current_report_count}") + logger.info("Current report count: %s", current_report_count) return file_path @@ -272,7 +380,7 @@ def save_comparison_report(comparison_data, output_folder): file_path = comparison_dir / f"profile_comparison_{datetime.now().strftime('%Y%m%d_%H%M%S_%f')}.json" with open(file_path, "w", encoding="utf-8") as f: json.dump(comparison_data, f, indent=4, ensure_ascii=False) - print(f"Comparison report saved to: {file_path}") + logger.info("Comparison report saved to: %s", file_path) return str(file_path) @@ -282,7 +390,7 @@ def save_repeatability_report(repeatability_data, output_folder): file_path = repeatability_dir / f"repeatability_{datetime.now().strftime('%Y%m%d_%H%M%S_%f')}.json" with open(file_path, "w", encoding="utf-8") as f: json.dump(repeatability_data, f, indent=4, ensure_ascii=False) - print(f"Repeatability report saved to: {file_path}") + logger.info("Repeatability report saved to: %s", file_path) return str(file_path) @@ -292,7 +400,7 @@ def save_performance_report(performance_data, output_folder): file_path = performance_dir / f"performance_{datetime.now().strftime('%Y%m%d_%H%M%S_%f')}.json" with open(file_path, "w", encoding="utf-8") as f: json.dump(performance_data, f, indent=4, ensure_ascii=False) - print(f"Performance report saved to: {file_path}") + logger.info("Performance report saved to: %s", file_path) return str(file_path) @@ -302,7 +410,7 @@ def save_overhead_report(overhead_data, output_folder): file_path = overhead_dir / f"overhead_{datetime.now().strftime('%Y%m%d_%H%M%S_%f')}.json" with open(file_path, "w", encoding="utf-8") as f: json.dump(overhead_data, f, indent=4, ensure_ascii=False) - print(f"Overhead report saved to: {file_path}") + logger.info("Overhead report saved to: %s", file_path) return str(file_path) @@ -312,12 +420,13 @@ def save_error_report(error_data, output_folder): file_path = error_dir / f"error_report_{datetime.now().strftime('%Y%m%d_%H%M%S_%f')}.json" with open(file_path, "w", encoding="utf-8") as f: json.dump(error_data, f, indent=4, ensure_ascii=False) - print(f"Error report saved to: {file_path}") + logger.info("Error report saved to: %s", file_path) deleted_count = cleanup_old_error_reports(str(error_dir)) if deleted_count > 0: - print( - f"Cleaned up {deleted_count} error report(s) older than " - f"{REPORT_RETENTION_DAYS} days." + logger.info( + "Cleaned up %s error report(s) older than %s days.", + deleted_count, + REPORT_RETENTION_DAYS, ) return str(file_path) @@ -339,7 +448,7 @@ def cleanup_old_error_reports(error_folder_path, max_age_days=REPORT_RETENTION_D os.remove(file_path) deleted_count += 1 except OSError as e: - print(f"WARNING: Could not remove old error report {file_path}: {e}") + logger.warning("Could not remove old error report %s: %s", file_path, e) return deleted_count @@ -395,34 +504,32 @@ def classify_loopback_signal(ai_result): def decide_loopback_action(signal, engine_metrics, thresholds_cfg, loopback_guard_cfg): - brightness = float(engine_metrics.get("avg_brightness", engine_metrics.get("brightness", 0.0))) - sharpness = float(engine_metrics.get("sharpness", 0.0)) - min_brightness = float(thresholds_cfg.get("min_brightness", 40.0)) - max_brightness = float(thresholds_cfg.get("max_brightness", 220.0)) - min_sharpness = float(thresholds_cfg.get("min_sharpness", 20.0)) - overexposure_stop_ratio = float(loopback_guard_cfg.get("overexposure_stop_ratio", 0.95)) - underexposure_stop_ratio = float(loopback_guard_cfg.get("underexposure_stop_ratio", 1.05)) - - if signal == "under": - if brightness >= min_brightness: - return None, "engine_disagrees_underexposed" - if brightness >= (max_brightness * overexposure_stop_ratio): - return None, "near_overexposure_guard" - return "brighten", "retry_scheduled" - - if signal == "over": - if brightness <= max_brightness: - return None, "engine_disagrees_overexposed" - if brightness <= (min_brightness * underexposure_stop_ratio): - return None, "near_underexposure_guard" - return "dim", "retry_scheduled" - - if signal == "blurry": - if sharpness >= min_sharpness: - return None, "engine_disagrees_blurry" - return "sharpen", "retry_scheduled" - - return None, f"signal_not_recoverable ({signal})" + planner = SimulatedLoopbackPlanner() + plan = planner.plan( + signal=signal, + engine_metrics=engine_metrics, + thresholds_cfg=thresholds_cfg, + loopback_guard_cfg=loopback_guard_cfg, + attempt_history=[], + ) + return plan.action, plan.stop_reason + + +def plan_next_action( + *, + signal: str, + engine_metrics: Dict[str, Any], + thresholds_cfg: Dict[str, Any], + loopback_guard_cfg: Dict[str, Any], +): + planner = SimulatedLoopbackPlanner() + return planner.plan( + signal=signal, + engine_metrics=engine_metrics, + thresholds_cfg=thresholds_cfg, + loopback_guard_cfg=loopback_guard_cfg, + attempt_history=[], + ) def summarize_performance(perf_samples): @@ -480,6 +587,62 @@ def summarize_performance(perf_samples): } +def _build_agent_inference_output( + *, + image_path: str, + attempt_history: List[Dict[str, Any]], + final_ai_result: Dict[str, Any], + total_latency_ms: float, +) -> Dict[str, Any]: + steps: List[AgentStep] = [] + for idx, attempt in enumerate(attempt_history): + signal = str(attempt.get("loopback_signal", "other")).lower() + if signal not in {"under", "over", "blurry", "other"}: + signal = "other" + action = str(attempt.get("action", "stop")).lower() + if action not in {"brighten", "dim", "sharpen", "stop"}: + action = "stop" + metrics_before = { + "avg_brightness": float(attempt.get("avg_brightness", 0.0)), + "sharpness": float(attempt.get("sharpness", 0.0)), + } + metrics_after = None + if idx + 1 < len(attempt_history): + next_attempt = attempt_history[idx + 1] + metrics_after = { + "avg_brightness": float(next_attempt.get("avg_brightness", 0.0)), + "sharpness": float(next_attempt.get("sharpness", 0.0)), + } + steps.append( + AgentStep( + attempt=int(attempt.get("attempt", idx + 1)), + signal=signal, + action=action, + rationale=str(attempt.get("rationale", "")), + fallback_used=bool(attempt.get("planner_fallback_used", False)), + metrics_before=metrics_before, + metrics_after=metrics_after, + latency_ms=float(attempt.get("latency_ms", 0.0)), + ) + ) + + final_release = "NO_GO" + if attempt_history: + release = str(attempt_history[-1].get("release", "NO_GO")).upper() + if release in {"GO", "REVIEW", "NO_GO"}: + final_release = release + + output = AgentInferenceOutput( + image_path=image_path, + final_decision=final_release, + error_code=str(final_ai_result.get("code", "SUCCESS_200")), + error_message=str(final_ai_result.get("msg", final_ai_result.get("decision", "Optimal"))), + steps=steps, + total_latency_ms=float(total_latency_ms), + ) + return output.model_dump() + + def monitor_resources(stop_event, interval=0.1): cpu_usage = [] memory_usage_mb = [] @@ -530,268 +693,489 @@ def _runner(): return result, monitor_result -def benchmark_monitor_overhead(samples=5, sleep_s=0.2): - per_run_ms = [] - process = psutil.Process(os.getpid()) - for _ in range(samples): - cpu_before = time.process_time() - rss_before = process.memory_info().rss / (1024.0 * 1024.0) - start = time.perf_counter() - collect_peak_resources_during(time.sleep, sleep_s) - elapsed_ms = (time.perf_counter() - start) * 1000.0 - cpu_after = time.process_time() - rss_after = process.memory_info().rss / (1024.0 * 1024.0) - per_run_ms.append({ - "wall_ms": round(elapsed_ms, 4), - "extra_wall_ms_vs_sleep": round(max(0.0, elapsed_ms - (sleep_s * 1000.0)), 4), - "cpu_time_ms": round((cpu_after - cpu_before) * 1000.0, 4), - "rss_delta_mb": round(rss_after - rss_before, 4), - }) +async def collect_peak_resources_during_async(awaitable_fn, *args): + stop_event = threading.Event() + monitor_result = {"peak_cpu_usage_pct": 0.0, "peak_memory_mb": 0.0} - avg_extra = sum(item["extra_wall_ms_vs_sleep"] for item in per_run_ms) / len(per_run_ms) - avg_cpu = sum(item["cpu_time_ms"] for item in per_run_ms) / len(per_run_ms) - max_rss_delta = max(item["rss_delta_mb"] for item in per_run_ms) if per_run_ms else 0.0 + def _runner(): + nonlocal monitor_result + try: + monitor_result = monitor_resources(stop_event) + except Exception: + monitor_result = {"peak_cpu_usage_pct": 0.0, "peak_memory_mb": 0.0} + + thread = threading.Thread(target=_runner, daemon=True) + thread.start() + try: + result = await awaitable_fn(*args) + finally: + stop_event.set() + thread.join(timeout=1.0) + return result, monitor_result + + +def _build_photo_process_context(config: Dict[str, Any], agent: QuantizedVisionAgent) -> Dict[str, Any]: + loopback_guard_cfg = config.get("runtime", {}).get("loopback_guard", {}) + loopback_planner = create_loopback_planner(config) return { - "samples": samples, - "sleep_s": sleep_s, - "avg_extra_wall_ms": round(avg_extra, 4), - "avg_cpu_time_ms": round(avg_cpu, 4), - "max_rss_delta_mb": round(max_rss_delta, 4), - "runs": per_run_ms, + "agent": agent, + "image_processor": ImageProcessor(), + "loopback_planner": loopback_planner, + "max_retry": int(config.get("runtime", {}).get("max_retry", 3)), + "thresholds_cfg": config.get("thresholds", {}), + "loopback_guard_cfg": loopback_guard_cfg, + "min_brightness_gain": float(loopback_guard_cfg.get("min_brightness_gain", 4.0)), + "min_sharpness_gain": float(loopback_guard_cfg.get("min_sharpness_gain", 1.0)), + "brighten_factor": float(loopback_guard_cfg.get("brighten_factor", 1.2)), + "dim_factor": float(loopback_guard_cfg.get("dim_factor", 0.85)), + "overexposure_stop_ratio": float(loopback_guard_cfg.get("overexposure_stop_ratio", 0.95)), } -def run_batch_test( - config_profile="dev", - config_path=None, - deterministic=False, - inference_backend_override=None, - performance_analysis=False, - overhead_analysis=False, - stress_test_count=None, -): - config, config_source = load_config(profile=config_profile, config_path=config_path) - error_report_dir = config.get("folders", {}).get("logs", "logs/errors") - if inference_backend_override: - config.setdefault("model_settings", {}).setdefault("inference", {}) - config["model_settings"]["inference"]["backend"] = inference_backend_override - config_source = f"{config_source} + CLI(backend={inference_backend_override})" - print(f"Loaded config source: {config_source}") - agent = QuantizedVisionAgent(config) - photos = agent.get_all_photos() - if stress_test_count: - input_folder = config["folders"]["input"] - current_dir = os.path.dirname(os.path.abspath(__file__)) - base_dir = os.path.dirname(current_dir) - full_input_path = os.path.join(base_dir, input_folder) - ensure_stress_test_images(full_input_path, target_count=int(stress_test_count)) - photos = agent.get_all_photos() - if deterministic: - random.seed(42) - agent.oom_probability = 0.0 +def _process_single_photo(path: str, ctx: Dict[str, Any]) -> Dict[str, Any]: + agent = ctx["agent"] + image_processor = ctx["image_processor"] + file_name = os.path.basename(path) + file_stem = Path(file_name).stem + image_wall_start = time.perf_counter() + + if random.random() < agent.oom_probability: + raise MemoryError("OOM Exception") + + image_meta = get_image_metadata(path) + cpu_start = time.process_time() + attempt_history = [] + current_path = path + final_metrics = None + final_ai_result = None + final_latency = 0.0 + loopback_stop_reason = "not_triggered" + peak_cpu_usage_pct = 0.0 + peak_memory_mb = 0.0 + + for attempt_idx in range(ctx["max_retry"] + 1): + (metrics, ai_result, latency), resource_peaks = collect_peak_resources_during( + agent.analyze_photo_quality, current_path + ) + peak_cpu_usage_pct = max(peak_cpu_usage_pct, resource_peaks.get("peak_cpu_usage_pct", 0.0)) + peak_memory_mb = max(peak_memory_mb, resource_peaks.get("peak_memory_mb", 0.0)) + final_metrics = metrics + final_ai_result = ai_result + final_latency += latency + + if not isinstance(metrics, dict): + attempt_history.append({ + "attempt": attempt_idx + 1, + "image_path": current_path, + "model_decision": ai_result.get("decision"), + "error_code": ai_result.get("code"), + "release": "NO_GO", + "loopback_signal": "other", + "action": "stop", + "rationale": "metrics unavailable; stop loopback", + "latency_ms": latency, + }) + loopback_stop_reason = "metrics_unavailable" + break - if not photos: - print("No testable images were found.") - return None + engine_metrics = { + "avg_brightness": metrics.get("avg_brightness", metrics.get("brightness", 0.0)), + "sharpness": metrics.get("sharpness", 0.0), + } + model_inference = { + "decision": ai_result.get("decision"), + "status": ai_result.get("decision"), + "confidence": ai_result.get("confidence"), + } + release_decision, _ = arbitrate_decision( + engine_metrics, model_inference, ctx["thresholds_cfg"] + ) + loopback_signal = classify_loopback_signal(ai_result) + attempt_history.append({ + "attempt": attempt_idx + 1, + "image_path": current_path, + "model_decision": ai_result.get("decision"), + "error_code": ai_result.get("code"), + "release": release_decision, + "loopback_signal": loopback_signal, + "action": "stop", + "rationale": "release resolved or awaiting planner decision", + "avg_brightness": round(float(engine_metrics.get("avg_brightness", 0.0)), 4), + "sharpness": round(float(engine_metrics.get("sharpness", 0.0)), 4), + "latency_ms": latency, + }) - batch_report = { - "schema_version": "2.0", - "profile": config_profile, - "batch_id": datetime.now().strftime("%Y%m%d_%H%M%S_%f"), - "config_used": config["thresholds"], - "config_source": config_source, - "results": [] - } - perf_samples = [] - failure_memory_store = FailureMemoryStore() - image_processor = ImageProcessor() - max_retry = int(config.get("runtime", {}).get("max_retry", 3)) - thresholds_cfg = config.get("thresholds", {}) - loopback_guard_cfg = config.get("runtime", {}).get("loopback_guard", {}) - min_brightness_gain = float(loopback_guard_cfg.get("min_brightness_gain", 4.0)) - min_sharpness_gain = float(loopback_guard_cfg.get("min_sharpness_gain", 1.0)) - brighten_factor = float(loopback_guard_cfg.get("brighten_factor", 1.2)) - dim_factor = float(loopback_guard_cfg.get("dim_factor", 0.85)) - overexposure_stop_ratio = float(loopback_guard_cfg.get("overexposure_stop_ratio", 0.95)) - process = psutil.Process(os.getpid()) - batch_wall_start = time.perf_counter() - batch_cpu_start = time.process_time() - batch_rss_start_mb = process.memory_info().rss / (1024.0 * 1024.0) - monitor_overhead_baseline = benchmark_monitor_overhead() if overhead_analysis else None - overhead_counters = { - "total_image_wall_ms": 0.0, - "total_model_latency_ms": 0.0, - "total_framework_wall_ms": 0.0, - "total_loopback_retry_count": 0, - "failure_memory_write_ms": 0.0, - "failure_memory_write_count": 0, + current_brightness = float(engine_metrics.get("avg_brightness", 0.0)) + current_sharpness = float(engine_metrics.get("sharpness", 0.0)) + if release_decision != "NO_GO": + loopback_stop_reason = "release_resolved" + break + if attempt_idx >= ctx["max_retry"]: + loopback_stop_reason = "max_retry_reached" + break + + plan = ctx["loopback_planner"].plan( + signal=loopback_signal, + engine_metrics=engine_metrics, + thresholds_cfg=ctx["thresholds_cfg"], + loopback_guard_cfg=ctx.get("loopback_guard_cfg", {}), + attempt_history=attempt_history, + ) + logger.info( + "Loopback planner for %s attempt=%s signal=%s action=%s stop_reason=%s rationale=%s", + file_name, + attempt_idx + 1, + loopback_signal, + plan.action, + plan.stop_reason, + plan.rationale, + ) + attempt_history[-1]["planner_fallback_used"] = bool(plan.fallback_used) + attempt_history[-1]["planner_backend"] = str(plan.planner_backend) + attempt_history[-1]["action"] = str(plan.action or "stop") + attempt_history[-1]["rationale"] = str(plan.rationale) + if not plan.action: + loopback_stop_reason = plan.stop_reason + break + + if len(attempt_history) >= 2: + prev_attempt = attempt_history[-2] + prev_signal = prev_attempt.get("loopback_signal") + prev_brightness = float(prev_attempt.get("avg_brightness", 0.0)) + prev_sharpness = float(prev_attempt.get("sharpness", 0.0)) + brightness_gain = current_brightness - prev_brightness + if prev_signal in {"under", "over"} and prev_signal != loopback_signal: + loopback_stop_reason = "oscillation_detected" + break + if plan.action == "brighten" and brightness_gain < ctx["min_brightness_gain"]: + loopback_stop_reason = f"insufficient_brightness_gain (<{ctx['min_brightness_gain']})" + break + if plan.action == "dim" and (prev_brightness - current_brightness) < ctx["min_brightness_gain"]: + loopback_stop_reason = f"insufficient_dimming_gain (<{ctx['min_brightness_gain']})" + break + if plan.action == "sharpen" and (current_sharpness - prev_sharpness) < ctx["min_sharpness_gain"]: + loopback_stop_reason = f"insufficient_sharpness_gain (<{ctx['min_sharpness_gain']})" + break + + if plan.action == "brighten": + current_path = image_processor.adjust_brightness( + current_path, + level=ctx["brighten_factor"], + file_stem=file_stem, + attempt_idx=attempt_idx + 1, + ) + logger.info( + "Loopback retry %s/%s for %s: detected under-exposed; brightness x%s and re-evaluate.", + attempt_idx + 1, + ctx["max_retry"], + file_name, + ctx["brighten_factor"], + ) + elif plan.action == "dim": + current_path = image_processor.adjust_brightness( + current_path, + level=ctx["dim_factor"], + file_stem=file_stem, + attempt_idx=attempt_idx + 1, + ) + logger.info( + "Loopback retry %s/%s for %s: detected over-exposed; brightness x%s and re-evaluate.", + attempt_idx + 1, + ctx["max_retry"], + file_name, + ctx["dim_factor"], + ) + elif plan.action == "sharpen": + current_path = image_processor.apply_sharpen( + current_path, + file_stem=file_stem, + attempt_idx=attempt_idx + 1, + ) + logger.info( + "Loopback retry %s/%s for %s: detected blurry signal; apply sharpen and re-evaluate.", + attempt_idx + 1, + ctx["max_retry"], + file_name, + ) + loopback_stop_reason = f"retry_scheduled ({plan.action})" + + cpu_delta = max(0.0, time.process_time() - cpu_start) + wall_delta = max(final_latency / 1000.0, 1e-6) + process_cpu_usage_pct = round((cpu_delta / wall_delta) * 100, 4) + logger.info( + "Processed %s: [%s] %s (%sms total)", + file_name, + (final_ai_result or {}).get("code", "?"), + (final_ai_result or {}).get("decision", "?"), + round(final_latency, 2), + ) + + image_wall_ms = (time.perf_counter() - image_wall_start) * 1000.0 + model_latency_ms = float(round(final_latency, 2)) + framework_wall_ms = max(0.0, image_wall_ms - model_latency_ms) + + return { + "row": { + "file": file_name, + "metrics": final_metrics, + "decision": final_ai_result, + "inference_output": _build_agent_inference_output( + image_path=path, + attempt_history=attempt_history, + final_ai_result=final_ai_result or {}, + total_latency_ms=round(final_latency, 2), + ), + "latency_ms": round(final_latency, 2), + "image_meta": image_meta, + "process_cpu_usage_pct": process_cpu_usage_pct, + "loopback": { + "max_retry": ctx["max_retry"], + "min_brightness_gain": ctx["min_brightness_gain"], + "min_sharpness_gain": ctx["min_sharpness_gain"], + "brighten_factor": ctx["brighten_factor"], + "dim_factor": ctx["dim_factor"], + "overexposure_stop_ratio": ctx["overexposure_stop_ratio"], + "retry_count": max(0, len(attempt_history) - 1), + "fallback_used_count": sum( + 1 for item in attempt_history if bool(item.get("planner_fallback_used")) + ), + "fallback_used": any( + bool(item.get("planner_fallback_used")) for item in attempt_history + ), + "stop_reason": loopback_stop_reason, + "attempts": attempt_history, + }, + "status": "SUCCESS", + }, + "perf_sample": { + "file": file_name, + "latency_ms": round(final_latency, 2), + "process_cpu_usage_pct": process_cpu_usage_pct, + "peak_cpu_usage_pct": round(peak_cpu_usage_pct, 4), + "peak_memory_mb": round(peak_memory_mb, 4), + "image_resolution": f"{image_meta.get('width', 0)}x{image_meta.get('height', 0)}", + **image_meta, + }, + "overhead": { + "image_wall_ms": image_wall_ms, + "model_latency_ms": model_latency_ms, + "framework_wall_ms": framework_wall_ms, + "loopback_retry_count": max(0, len(attempt_history) - 1), + }, } - print(f"Starting to process {len(photos)} image(s)...\n") - for path in photos: +@async_monitor_performance +async def _process_single_photo_async( + path: str, + ctx: Dict[str, Any], + http_client: httpx.AsyncClient, + semaphore: asyncio.Semaphore, +) -> Dict[str, Any]: + async with semaphore: + logger.debug("_process_single_photo_async: start path=%s", path) + agent = ctx["agent"] + image_processor = ctx["image_processor"] file_name = os.path.basename(path) file_stem = Path(file_name).stem - try: - image_wall_start = time.perf_counter() - if random.random() < agent.oom_probability: - raise MemoryError("OOM Exception") - - image_meta = get_image_metadata(path) - cpu_start = time.process_time() - attempt_history = [] - current_path = path - final_metrics = None - final_ai_result = None - final_latency = 0.0 - loopback_stop_reason = "not_triggered" - - peak_cpu_usage_pct = 0.0 - peak_memory_mb = 0.0 - - for attempt_idx in range(max_retry + 1): - (metrics, ai_result, latency), resource_peaks = collect_peak_resources_during( - agent.analyze_photo_quality, current_path - ) - peak_cpu_usage_pct = max(peak_cpu_usage_pct, resource_peaks.get("peak_cpu_usage_pct", 0.0)) - peak_memory_mb = max(peak_memory_mb, resource_peaks.get("peak_memory_mb", 0.0)) - final_metrics = metrics - final_ai_result = ai_result - final_latency += latency - - if not isinstance(metrics, dict): - attempt_history.append({ - "attempt": attempt_idx + 1, - "image_path": current_path, - "model_decision": ai_result.get("decision"), - "error_code": ai_result.get("code"), - "release": "NO_GO", - "latency_ms": latency, - }) - loopback_stop_reason = "metrics_unavailable" - break + image_wall_start = time.perf_counter() - engine_metrics = { - "avg_brightness": metrics.get("avg_brightness", metrics.get("brightness", 0.0)), - "sharpness": metrics.get("sharpness", 0.0), - } - model_inference = { - "decision": ai_result.get("decision"), - "status": ai_result.get("decision"), - "confidence": ai_result.get("confidence"), - } - release_decision, _ = arbitrate_decision(engine_metrics, model_inference, config.get("thresholds", {})) - loopback_signal = classify_loopback_signal(ai_result) + if random.random() < agent.oom_probability: + raise MemoryError("OOM Exception") + + image_meta = await asyncio.to_thread(get_image_metadata, path) + cpu_start = time.process_time() + attempt_history = [] + current_path = path + final_metrics = None + final_ai_result = None + final_latency = 0.0 + loopback_stop_reason = "not_triggered" + peak_cpu_usage_pct = 0.0 + peak_memory_mb = 0.0 + + for attempt_idx in range(ctx["max_retry"] + 1): + + async def _analyze(photo_path=current_path): + return await agent.analyze_photo_quality_async(photo_path, http_client) + + (metrics, ai_result, latency), resource_peaks = await collect_peak_resources_during_async( + _analyze + ) + peak_cpu_usage_pct = max(peak_cpu_usage_pct, resource_peaks.get("peak_cpu_usage_pct", 0.0)) + peak_memory_mb = max(peak_memory_mb, resource_peaks.get("peak_memory_mb", 0.0)) + final_metrics = metrics + final_ai_result = ai_result + final_latency += latency + + if not isinstance(metrics, dict): attempt_history.append({ "attempt": attempt_idx + 1, "image_path": current_path, "model_decision": ai_result.get("decision"), "error_code": ai_result.get("code"), - "release": release_decision, - "loopback_signal": loopback_signal, - "avg_brightness": round(float(engine_metrics.get("avg_brightness", 0.0)), 4), - "sharpness": round(float(engine_metrics.get("sharpness", 0.0)), 4), + "release": "NO_GO", + "loopback_signal": "other", + "action": "stop", + "rationale": "metrics unavailable; stop loopback", "latency_ms": latency, }) + loopback_stop_reason = "metrics_unavailable" + break + + engine_metrics = { + "avg_brightness": metrics.get("avg_brightness", metrics.get("brightness", 0.0)), + "sharpness": metrics.get("sharpness", 0.0), + } + model_inference = { + "decision": ai_result.get("decision"), + "status": ai_result.get("decision"), + "confidence": ai_result.get("confidence"), + } + release_decision, _ = arbitrate_decision( + engine_metrics, model_inference, ctx["thresholds_cfg"] + ) + loopback_signal = classify_loopback_signal(ai_result) + attempt_history.append({ + "attempt": attempt_idx + 1, + "image_path": current_path, + "model_decision": ai_result.get("decision"), + "error_code": ai_result.get("code"), + "release": release_decision, + "loopback_signal": loopback_signal, + "action": "stop", + "rationale": "release resolved or awaiting planner decision", + "avg_brightness": round(float(engine_metrics.get("avg_brightness", 0.0)), 4), + "sharpness": round(float(engine_metrics.get("sharpness", 0.0)), 4), + "latency_ms": latency, + }) - current_brightness = float(engine_metrics.get("avg_brightness", 0.0)) - current_sharpness = float(engine_metrics.get("sharpness", 0.0)) - if release_decision != "NO_GO": - loopback_stop_reason = "release_resolved" + current_brightness = float(engine_metrics.get("avg_brightness", 0.0)) + current_sharpness = float(engine_metrics.get("sharpness", 0.0)) + if release_decision != "NO_GO": + loopback_stop_reason = "release_resolved" + break + if attempt_idx >= ctx["max_retry"]: + loopback_stop_reason = "max_retry_reached" + break + + plan = ctx["loopback_planner"].plan( + signal=loopback_signal, + engine_metrics=engine_metrics, + thresholds_cfg=ctx["thresholds_cfg"], + loopback_guard_cfg=ctx.get("loopback_guard_cfg", {}), + attempt_history=attempt_history, + ) + logger.info( + "Loopback planner (async) for %s attempt=%s signal=%s action=%s stop_reason=%s rationale=%s", + file_name, + attempt_idx + 1, + loopback_signal, + plan.action, + plan.stop_reason, + plan.rationale, + ) + attempt_history[-1]["planner_fallback_used"] = bool(plan.fallback_used) + attempt_history[-1]["planner_backend"] = str(plan.planner_backend) + attempt_history[-1]["action"] = str(plan.action or "stop") + attempt_history[-1]["rationale"] = str(plan.rationale) + if not plan.action: + loopback_stop_reason = plan.stop_reason + break + + if len(attempt_history) >= 2: + prev_attempt = attempt_history[-2] + prev_signal = prev_attempt.get("loopback_signal") + prev_brightness = float(prev_attempt.get("avg_brightness", 0.0)) + prev_sharpness = float(prev_attempt.get("sharpness", 0.0)) + brightness_gain = current_brightness - prev_brightness + if prev_signal in {"under", "over"} and prev_signal != loopback_signal: + loopback_stop_reason = "oscillation_detected" + break + if plan.action == "brighten" and brightness_gain < ctx["min_brightness_gain"]: + loopback_stop_reason = f"insufficient_brightness_gain (<{ctx['min_brightness_gain']})" break - if attempt_idx >= max_retry: - loopback_stop_reason = "max_retry_reached" + if plan.action == "dim" and (prev_brightness - current_brightness) < ctx["min_brightness_gain"]: + loopback_stop_reason = f"insufficient_dimming_gain (<{ctx['min_brightness_gain']})" + break + if plan.action == "sharpen" and (current_sharpness - prev_sharpness) < ctx["min_sharpness_gain"]: + loopback_stop_reason = f"insufficient_sharpness_gain (<{ctx['min_sharpness_gain']})" break - next_action, action_stop_reason = decide_loopback_action( - loopback_signal, engine_metrics, thresholds_cfg, loopback_guard_cfg + if plan.action == "brighten": + current_path = await asyncio.to_thread( + image_processor.adjust_brightness, + current_path, + ctx["brighten_factor"], + file_stem, + attempt_idx + 1, ) - if not next_action: - loopback_stop_reason = action_stop_reason - break + elif plan.action == "dim": + current_path = await asyncio.to_thread( + image_processor.adjust_brightness, + current_path, + ctx["dim_factor"], + file_stem, + attempt_idx + 1, + ) + elif plan.action == "sharpen": + current_path = await asyncio.to_thread( + image_processor.apply_sharpen, + current_path, + file_stem, + attempt_idx + 1, + ) + loopback_stop_reason = f"retry_scheduled ({plan.action})" + + cpu_delta = max(0.0, time.process_time() - cpu_start) + wall_delta = max(final_latency / 1000.0, 1e-6) + process_cpu_usage_pct = round((cpu_delta / wall_delta) * 100, 4) + logger.info( + "Processed (async) %s: [%s] %s (%sms total)", + file_name, + (final_ai_result or {}).get("code", "?"), + (final_ai_result or {}).get("decision", "?"), + round(final_latency, 2), + ) - if len(attempt_history) >= 2: - prev_attempt = attempt_history[-2] - prev_signal = prev_attempt.get("loopback_signal") - prev_brightness = float(prev_attempt.get("avg_brightness", 0.0)) - prev_sharpness = float(prev_attempt.get("sharpness", 0.0)) - brightness_gain = current_brightness - prev_brightness - if prev_signal in {"under", "over"} and prev_signal != loopback_signal: - loopback_stop_reason = "oscillation_detected" - break - if next_action == "brighten" and brightness_gain < min_brightness_gain: - loopback_stop_reason = f"insufficient_brightness_gain (<{min_brightness_gain})" - break - if next_action == "dim" and (prev_brightness - current_brightness) < min_brightness_gain: - loopback_stop_reason = f"insufficient_dimming_gain (<{min_brightness_gain})" - break - if next_action == "sharpen" and (current_sharpness - prev_sharpness) < min_sharpness_gain: - loopback_stop_reason = f"insufficient_sharpness_gain (<{min_sharpness_gain})" - break - - if next_action == "brighten": - current_path = image_processor.adjust_brightness( - current_path, - level=brighten_factor, - file_stem=file_stem, - attempt_idx=attempt_idx + 1, - ) - print( - f"Loopback retry {attempt_idx + 1}/{max_retry} for {file_name}: " - f"detected under-exposed; brightness x{brighten_factor} and re-evaluate." - ) - elif next_action == "dim": - current_path = image_processor.adjust_brightness( - current_path, - level=dim_factor, - file_stem=file_stem, - attempt_idx=attempt_idx + 1, - ) - print( - f"Loopback retry {attempt_idx + 1}/{max_retry} for {file_name}: " - f"detected over-exposed; brightness x{dim_factor} and re-evaluate." - ) - elif next_action == "sharpen": - current_path = image_processor.apply_sharpen( - current_path, - file_stem=file_stem, - attempt_idx=attempt_idx + 1, - ) - print( - f"Loopback retry {attempt_idx + 1}/{max_retry} for {file_name}: " - "detected blurry signal; apply sharpen and re-evaluate." - ) - loopback_stop_reason = f"retry_scheduled ({next_action})" - - cpu_delta = max(0.0, time.process_time() - cpu_start) - wall_delta = max(final_latency / 1000.0, 1e-6) - process_cpu_usage_pct = round((cpu_delta / wall_delta) * 100, 4) - print( - f"Processed {file_name}: [{final_ai_result['code']}] {final_ai_result['decision']} " - f"({round(final_latency, 2)}ms total)" - ) + image_wall_ms = (time.perf_counter() - image_wall_start) * 1000.0 + model_latency_ms = float(round(final_latency, 2)) + framework_wall_ms = max(0.0, image_wall_ms - model_latency_ms) - batch_report["results"].append({ + return { + "row": { "file": file_name, "metrics": final_metrics, "decision": final_ai_result, + "inference_output": _build_agent_inference_output( + image_path=path, + attempt_history=attempt_history, + final_ai_result=final_ai_result or {}, + total_latency_ms=round(final_latency, 2), + ), "latency_ms": round(final_latency, 2), "image_meta": image_meta, "process_cpu_usage_pct": process_cpu_usage_pct, "loopback": { - "max_retry": max_retry, - "min_brightness_gain": min_brightness_gain, - "min_sharpness_gain": min_sharpness_gain, - "brighten_factor": brighten_factor, - "dim_factor": dim_factor, - "overexposure_stop_ratio": overexposure_stop_ratio, + "max_retry": ctx["max_retry"], + "min_brightness_gain": ctx["min_brightness_gain"], + "min_sharpness_gain": ctx["min_sharpness_gain"], + "brighten_factor": ctx["brighten_factor"], + "dim_factor": ctx["dim_factor"], + "overexposure_stop_ratio": ctx["overexposure_stop_ratio"], "retry_count": max(0, len(attempt_history) - 1), + "fallback_used_count": sum( + 1 for item in attempt_history if bool(item.get("planner_fallback_used")) + ), + "fallback_used": any( + bool(item.get("planner_fallback_used")) for item in attempt_history + ), "stop_reason": loopback_stop_reason, "attempts": attempt_history, }, - "status": "SUCCESS" - }) - perf_samples.append({ + "status": "SUCCESS", + }, + "perf_sample": { "file": file_name, "latency_ms": round(final_latency, 2), "process_cpu_usage_pct": process_cpu_usage_pct, @@ -799,34 +1183,64 @@ def run_batch_test( "peak_memory_mb": round(peak_memory_mb, 4), "image_resolution": f"{image_meta.get('width', 0)}x{image_meta.get('height', 0)}", **image_meta, - }) - image_wall_ms = (time.perf_counter() - image_wall_start) * 1000.0 - model_latency_ms = float(round(final_latency, 2)) - framework_wall_ms = max(0.0, image_wall_ms - model_latency_ms) - overhead_counters["total_image_wall_ms"] += image_wall_ms - overhead_counters["total_model_latency_ms"] += model_latency_ms - overhead_counters["total_framework_wall_ms"] += framework_wall_ms - overhead_counters["total_loopback_retry_count"] += max(0, len(attempt_history) - 1) + }, + "overhead": { + "image_wall_ms": image_wall_ms, + "model_latency_ms": model_latency_ms, + "framework_wall_ms": framework_wall_ms, + "loopback_retry_count": max(0, len(attempt_history) - 1), + }, + } - except Exception as e: - print(f"Failed to process file {file_name}: {e}") - error_payload = { - "generated_at": datetime.now().isoformat(), - "scope": "single_file", - "profile": config_profile, - "config_source": config_source, - "file": file_name, - "error_type": type(e).__name__, - "error_message": str(e), - "traceback": traceback.format_exc(), - } - save_error_report(error_payload, error_report_dir) - batch_report["results"].append({ - "file": file_name, - "status": "FAILED", - "error": str(e) - }) +def benchmark_monitor_overhead(samples=5, sleep_s=0.2): + per_run_ms = [] + process = psutil.Process(os.getpid()) + for _ in range(samples): + cpu_before = time.process_time() + rss_before = process.memory_info().rss / (1024.0 * 1024.0) + start = time.perf_counter() + collect_peak_resources_during(time.sleep, sleep_s) + elapsed_ms = (time.perf_counter() - start) * 1000.0 + cpu_after = time.process_time() + rss_after = process.memory_info().rss / (1024.0 * 1024.0) + per_run_ms.append({ + "wall_ms": round(elapsed_ms, 4), + "extra_wall_ms_vs_sleep": round(max(0.0, elapsed_ms - (sleep_s * 1000.0)), 4), + "cpu_time_ms": round((cpu_after - cpu_before) * 1000.0, 4), + "rss_delta_mb": round(rss_after - rss_before, 4), + }) + + avg_extra = sum(item["extra_wall_ms_vs_sleep"] for item in per_run_ms) / len(per_run_ms) + avg_cpu = sum(item["cpu_time_ms"] for item in per_run_ms) / len(per_run_ms) + max_rss_delta = max(item["rss_delta_mb"] for item in per_run_ms) if per_run_ms else 0.0 + return { + "samples": samples, + "sleep_s": sleep_s, + "avg_extra_wall_ms": round(avg_extra, 4), + "avg_cpu_time_ms": round(avg_cpu, 4), + "max_rss_delta_mb": round(max_rss_delta, 4), + "runs": per_run_ms, + } + + +def _finalize_batch_report( + *, + batch_report: Dict[str, Any], + config: Dict[str, Any], + config_profile: str, + config_source: str, + perf_samples: List[Dict[str, Any]], + failure_memory_store: FailureMemoryStore, + overhead_counters: Dict[str, Any], + performance_analysis: bool, + overhead_analysis: bool, + batch_wall_start: float, + batch_cpu_start: float, + batch_rss_start_mb: float, + monitor_overhead_baseline: Optional[Dict[str, Any]], + process: psutil.Process, +) -> Dict[str, Any]: total = len(batch_report["results"]) success_count = sum( 1 for row in batch_report["results"] @@ -904,18 +1318,28 @@ def run_batch_test( top_ranking = rankings[:3] - print("\n" + "=" * 55) - print("Test Dashboard") - print(f" - Total tests: {total}") - print(f" - Pass rate (Optimal): {pass_rate:.1f}%") - print(f" - Average latency: {avg_lat:.2f} ms") - print(f" - Release decision (arbitrated): {decision}") - print(f" (gate={gate_decision}, arbitration_batch={arbitration_batch})") - print("-" * 55) - print("Top ranking:") + logger.info("%s", "\n" + "=" * 55) + logger.info("Test Dashboard") + logger.info(" - Total tests: %s", total) + logger.info(" - Pass rate (Optimal): %.1f%%", pass_rate) + logger.info(" - Average latency: %.2f ms", avg_lat) + logger.info(" - Release decision (arbitrated): %s", decision) + logger.info( + " (gate=%s, arbitration_batch=%s)", + gate_decision, + arbitration_batch, + ) + logger.info("%s", "-" * 55) + logger.info("Top ranking:") for item in top_ranking: - print(f" #{item['rank']} {item['file']} | score={item['score']} | {item['decision']}") - print("=" * 55) + logger.info( + " #%s %s | score=%s | %s", + item["rank"], + item["file"], + item["score"], + item["decision"], + ) + logger.info("%s", "=" * 55) batch_report["summary"] = { "total_tests": total, @@ -989,17 +1413,423 @@ def run_batch_test( "summary": batch_report["summary"], "top_ranked": top_ranking, "ranking": rankings, - "image_files": sorted([row["file"] for row in batch_report["results"] if "file" in row]) + "image_files": sorted([row["file"] for row in batch_report["results"] if "file" in row]), } -def run_profile_comparison(profiles, inference_backend_override=None): +def run_batch_test( + config_profile="dev", + config_path=None, + deterministic=False, + inference_backend_override=None, + loopback_planner_override=None, + planner_timeout_s_override=None, + planner_model_override=None, + planner_require_healthy_override=None, + performance_analysis=False, + overhead_analysis=False, + stress_test_count=None, + parallel_metrics=False, + metrics_workers=None, +): + config, config_source = load_config(profile=config_profile, config_path=config_path) + error_report_dir = config.get("folders", {}).get("logs", "logs/errors") + config_source = _apply_runtime_overrides( + config, + config_source, + inference_backend_override=inference_backend_override, + loopback_planner_override=loopback_planner_override, + planner_timeout_s_override=planner_timeout_s_override, + planner_model_override=planner_model_override, + planner_require_healthy_override=planner_require_healthy_override, + ) + logger.info("Loaded config source: %s", config_source) + + pool_cm = ( + MetricsProcessPool(max_workers=metrics_workers) + if parallel_metrics + else nullcontext() + ) + with pool_cm as metrics_pool: + agent = QuantizedVisionAgent( + config, + metrics_pool=metrics_pool if parallel_metrics else None, + ) + return _run_batch_test_body( + agent=agent, + config=config, + config_profile=config_profile, + config_source=config_source, + error_report_dir=error_report_dir, + deterministic=deterministic, + performance_analysis=performance_analysis, + overhead_analysis=overhead_analysis, + stress_test_count=stress_test_count, + parallel_metrics=parallel_metrics, + metrics_workers=metrics_pool.max_workers if parallel_metrics else None, + ) + + +def _run_batch_test_body( + *, + agent: QuantizedVisionAgent, + config: Dict[str, Any], + config_profile: str, + config_source: str, + error_report_dir: str, + deterministic: bool, + performance_analysis: bool, + overhead_analysis: bool, + stress_test_count: Optional[int], + parallel_metrics: bool, + metrics_workers: Optional[int], +): + photos = agent.get_all_photos() + if stress_test_count: + input_folder = config["folders"]["input"] + current_dir = os.path.dirname(os.path.abspath(__file__)) + base_dir = os.path.dirname(current_dir) + full_input_path = os.path.join(base_dir, input_folder) + ensure_stress_test_images(full_input_path, target_count=int(stress_test_count)) + photos = agent.get_all_photos() + if deterministic: + random.seed(42) + agent.oom_probability = 0.0 + + if not photos: + logger.warning("No testable images were found.") + return None + + batch_report = { + "schema_version": "2.0", + "profile": config_profile, + "batch_id": datetime.now().strftime("%Y%m%d_%H%M%S_%f"), + "config_used": config["thresholds"], + "config_source": config_source, + "results": [], + } + if parallel_metrics: + batch_report["parallel_metrics"] = True + batch_report["metrics_workers"] = metrics_workers + + perf_samples = [] + failure_memory_store = FailureMemoryStore() + photo_ctx = _build_photo_process_context(config, agent) + process = psutil.Process(os.getpid()) + batch_wall_start = time.perf_counter() + batch_cpu_start = time.process_time() + batch_rss_start_mb = process.memory_info().rss / (1024.0 * 1024.0) + monitor_overhead_baseline = benchmark_monitor_overhead() if overhead_analysis else None + overhead_counters = { + "total_image_wall_ms": 0.0, + "total_model_latency_ms": 0.0, + "total_framework_wall_ms": 0.0, + "total_loopback_retry_count": 0, + "failure_memory_write_ms": 0.0, + "failure_memory_write_count": 0, + } + + logger.info("Starting to process %s image(s)...", len(photos)) + + for path in photos: + file_name = os.path.basename(path) + try: + outcome = _process_single_photo(path, photo_ctx) + batch_report["results"].append(outcome["row"]) + perf_samples.append(outcome["perf_sample"]) + oh = outcome["overhead"] + overhead_counters["total_image_wall_ms"] += oh["image_wall_ms"] + overhead_counters["total_model_latency_ms"] += oh["model_latency_ms"] + overhead_counters["total_framework_wall_ms"] += oh["framework_wall_ms"] + overhead_counters["total_loopback_retry_count"] += oh["loopback_retry_count"] + + except Exception as e: + logger.exception("Failed to process file %s", file_name) + error_payload = { + "generated_at": datetime.now().isoformat(), + "scope": "single_file", + "profile": config_profile, + "config_source": config_source, + "file": file_name, + "error_type": type(e).__name__, + "error_message": str(e), + "traceback": traceback.format_exc(), + } + save_error_report(error_payload, error_report_dir) + batch_report["results"].append({ + "file": file_name, + "status": "FAILED", + "error": str(e) + }) + + return _finalize_batch_report( + batch_report=batch_report, + config=config, + config_profile=config_profile, + config_source=config_source, + perf_samples=perf_samples, + failure_memory_store=failure_memory_store, + overhead_counters=overhead_counters, + performance_analysis=performance_analysis, + overhead_analysis=overhead_analysis, + batch_wall_start=batch_wall_start, + batch_cpu_start=batch_cpu_start, + batch_rss_start_mb=batch_rss_start_mb, + monitor_overhead_baseline=monitor_overhead_baseline, + process=process, + ) + + +@async_monitor_performance +async def _run_async_batch_processing( + photos: List[str], + photo_ctx: Dict[str, Any], + batch_report: Dict[str, Any], + perf_samples: List[Dict[str, Any]], + overhead_counters: Dict[str, Any], + error_report_dir: str, + config_profile: str, + config_source: str, + concurrency: int, +) -> None: + semaphore = asyncio.Semaphore(max(1, concurrency)) + logger.info( + "Async batch: processing %s image(s) with concurrency=%s", + len(photos), + max(1, concurrency), + ) + + async with httpx.AsyncClient() as http_client: + + async def _handle_photo(path: str) -> Dict[str, Any]: + file_name = os.path.basename(path) + try: + return { + "status": "SUCCESS", + "file_name": file_name, + "outcome": await _process_single_photo_async( + path, photo_ctx, http_client, semaphore + ), + } + except Exception as exc: + logger.exception("Failed to process file %s (async)", file_name) + error_payload = { + "generated_at": datetime.now().isoformat(), + "scope": "single_file", + "profile": config_profile, + "config_source": config_source, + "file": file_name, + "error_type": type(exc).__name__, + "error_message": str(exc), + "traceback": traceback.format_exc(), + } + save_error_report(error_payload, error_report_dir) + return { + "status": "FAILED", + "file_name": file_name, + "error": str(exc), + } + + outcomes = await gather_with_timing( + [_handle_photo(path) for path in photos], + label="async_batch_photos", + ) + + for item in outcomes: + if item["status"] == "SUCCESS": + outcome = item["outcome"] + batch_report["results"].append(outcome["row"]) + perf_samples.append(outcome["perf_sample"]) + oh = outcome["overhead"] + overhead_counters["total_image_wall_ms"] += oh["image_wall_ms"] + overhead_counters["total_model_latency_ms"] += oh["model_latency_ms"] + overhead_counters["total_framework_wall_ms"] += oh["framework_wall_ms"] + overhead_counters["total_loopback_retry_count"] += oh["loopback_retry_count"] + else: + batch_report["results"].append({ + "file": item["file_name"], + "status": "FAILED", + "error": item["error"], + }) + + +def run_batch_test_async( + config_profile="dev", + config_path=None, + deterministic=False, + inference_backend_override=None, + loopback_planner_override=None, + planner_timeout_s_override=None, + planner_model_override=None, + planner_require_healthy_override=None, + performance_analysis=False, + overhead_analysis=False, + stress_test_count=None, + concurrency=4, + parallel_metrics=False, + metrics_workers=None, +): + """ + Parallel batch run: concurrent per-image processing with async HTTP inference. + + Use --parallel-metrics (ProcessPoolExecutor) when metrics CPU is the bottleneck; + async helps most when waiting on llama.cpp / Ollama HTTP responses. + """ + config, config_source = load_config(profile=config_profile, config_path=config_path) + error_report_dir = config.get("folders", {}).get("logs", "logs/errors") + config_source = _apply_runtime_overrides( + config, + config_source, + inference_backend_override=inference_backend_override, + loopback_planner_override=loopback_planner_override, + planner_timeout_s_override=planner_timeout_s_override, + planner_model_override=planner_model_override, + planner_require_healthy_override=planner_require_healthy_override, + ) + logger.info( + "Loaded config source: %s (async batch, concurrency=%s, parallel_metrics=%s)", + config_source, + concurrency, + parallel_metrics, + ) + + pool_cm = ( + MetricsProcessPool(max_workers=metrics_workers) + if parallel_metrics + else nullcontext() + ) + with pool_cm as metrics_pool: + agent = QuantizedVisionAgent( + config, + metrics_pool=metrics_pool if parallel_metrics else None, + ) + return _run_batch_test_async_body( + agent=agent, + config=config, + config_profile=config_profile, + config_source=config_source, + error_report_dir=error_report_dir, + deterministic=deterministic, + performance_analysis=performance_analysis, + overhead_analysis=overhead_analysis, + stress_test_count=stress_test_count, + concurrency=concurrency, + parallel_metrics=parallel_metrics, + metrics_workers=metrics_pool.max_workers if parallel_metrics else None, + ) + + +def _run_batch_test_async_body( + *, + agent: QuantizedVisionAgent, + config: Dict[str, Any], + config_profile: str, + config_source: str, + error_report_dir: str, + deterministic: bool, + performance_analysis: bool, + overhead_analysis: bool, + stress_test_count: Optional[int], + concurrency: int, + parallel_metrics: bool, + metrics_workers: Optional[int], +): + photos = agent.get_all_photos() + if stress_test_count: + input_folder = config["folders"]["input"] + current_dir = os.path.dirname(os.path.abspath(__file__)) + base_dir = os.path.dirname(current_dir) + full_input_path = os.path.join(base_dir, input_folder) + ensure_stress_test_images(full_input_path, target_count=int(stress_test_count)) + photos = agent.get_all_photos() + if deterministic: + random.seed(42) + agent.oom_probability = 0.0 + + if not photos: + logger.warning("No testable images were found.") + return None + + batch_report = { + "schema_version": "2.0", + "profile": config_profile, + "batch_id": datetime.now().strftime("%Y%m%d_%H%M%S_%f"), + "config_used": config["thresholds"], + "config_source": config_source, + "execution_mode": "async", + "async_concurrency": max(1, int(concurrency)), + "results": [], + } + if parallel_metrics: + batch_report["parallel_metrics"] = True + batch_report["metrics_workers"] = metrics_workers + perf_samples: List[Dict[str, Any]] = [] + failure_memory_store = FailureMemoryStore() + photo_ctx = _build_photo_process_context(config, agent) + process = psutil.Process(os.getpid()) + batch_wall_start = time.perf_counter() + batch_cpu_start = time.process_time() + batch_rss_start_mb = process.memory_info().rss / (1024.0 * 1024.0) + monitor_overhead_baseline = benchmark_monitor_overhead() if overhead_analysis else None + overhead_counters = { + "total_image_wall_ms": 0.0, + "total_model_latency_ms": 0.0, + "total_framework_wall_ms": 0.0, + "total_loopback_retry_count": 0, + "failure_memory_write_ms": 0.0, + "failure_memory_write_count": 0, + } + + asyncio.run( + _run_async_batch_processing( + photos, + photo_ctx, + batch_report, + perf_samples, + overhead_counters, + error_report_dir, + config_profile, + config_source, + concurrency=max(1, int(concurrency)), + ) + ) + + return _finalize_batch_report( + batch_report=batch_report, + config=config, + config_profile=config_profile, + config_source=config_source, + perf_samples=perf_samples, + failure_memory_store=failure_memory_store, + overhead_counters=overhead_counters, + performance_analysis=performance_analysis, + overhead_analysis=overhead_analysis, + batch_wall_start=batch_wall_start, + batch_cpu_start=batch_cpu_start, + batch_rss_start_mb=batch_rss_start_mb, + monitor_overhead_baseline=monitor_overhead_baseline, + process=process, + ) + + +def run_profile_comparison( + profiles, + inference_backend_override=None, + loopback_planner_override=None, + planner_timeout_s_override=None, + planner_model_override=None, + planner_require_healthy_override=None, +): profile_outputs = [] for profile in profiles: - print(f"\nRunning profile: {profile}") + logger.info("Running profile: %s", profile) result = run_batch_test( config_profile=profile, inference_backend_override=inference_backend_override, + loopback_planner_override=loopback_planner_override, + planner_timeout_s_override=planner_timeout_s_override, + planner_model_override=planner_model_override, + planner_require_healthy_override=planner_require_healthy_override, overhead_analysis=False, ) if result: @@ -1010,20 +1840,24 @@ def run_profile_comparison(profiles, inference_backend_override=None): key=lambda item: (-item["summary"]["pass_rate"], item["summary"]["avg_latency_ms"]) ) - print("\nProfile ranking (best to worst):") + logger.info("%s", "\nProfile ranking (best to worst):") for idx, item in enumerate(ordered, start=1): summary = item["summary"] - print( - f" #{idx} {item['profile']} | pass={summary['pass_rate']}% | " - f"latency={summary['avg_latency_ms']}ms | decision={summary['release_decision']}" + logger.info( + " #%s %s | pass=%s%% | latency=%sms | decision=%s", + idx, + item["profile"], + summary["pass_rate"], + summary["avg_latency_ms"], + summary["release_decision"], ) benchmark_insights = generate_benchmark_insights(profile_outputs, ordered) - print("\nBenchmark Insights:") + logger.info("%s", "\nBenchmark Insights:") for idx, insight in enumerate(benchmark_insights, start=1): - print(f" [{idx}] Trade-off: {insight['trade_off']}") - print(f" Observation: {insight['observation']}") - print(f" Decision implication: {insight['decision_implication']}") + logger.info(" [%s] Trade-off: %s", idx, insight["trade_off"]) + logger.info(" Observation: %s", insight["observation"]) + logger.info(" Decision implication: %s", insight["decision_implication"]) comparison_report = { "generated_at": datetime.now().isoformat(), @@ -1035,15 +1869,31 @@ def run_profile_comparison(profiles, inference_backend_override=None): return comparison_report -def run_repeatability_test(profile, runs=5, inference_backend_override=None): - print(f"\nRunning repeatability test: profile={profile}, runs={runs}") +def run_repeatability_test( + profile, + runs=5, + inference_backend_override=None, + loopback_planner_override=None, + planner_timeout_s_override=None, + planner_model_override=None, + planner_require_healthy_override=None, +): + logger.info( + "Running repeatability test: profile=%s, runs=%s", + profile, + runs, + ) run_outputs = [] for run_idx in range(1, runs + 1): - print(f"\nRepeatability run {run_idx}/{runs}") + logger.info("Repeatability run %s/%s", run_idx, runs) run_result = run_batch_test( config_profile=profile, deterministic=True, inference_backend_override=inference_backend_override, + loopback_planner_override=loopback_planner_override, + planner_timeout_s_override=planner_timeout_s_override, + planner_model_override=planner_model_override, + planner_require_healthy_override=planner_require_healthy_override, overhead_analysis=False, ) if run_result: @@ -1051,7 +1901,7 @@ def run_repeatability_test(profile, runs=5, inference_backend_override=None): run_outputs.append(run_result) if not run_outputs: - print("Repeatability test failed: no run output produced.") + logger.error("Repeatability test failed: no run output produced.") return None pass_rates = [r["summary"]["pass_rate"] for r in run_outputs] @@ -1092,16 +1942,20 @@ def run_repeatability_test(profile, runs=5, inference_backend_override=None): } save_repeatability_report(repeatability_report, "results") - print("\nRepeatability summary:") - print(f" - Same image set across runs: {image_set_consistent}") - print(f" - Pass-rate variance: {variance_report['pass_rate_variance']}") - print(f" - Avg-latency variance: {variance_report['avg_latency_variance']}") - print(f" - Max per-image score variance: {variance_report['max_image_score_variance']}") - print(f" - Decision distribution: {decision_distribution}") + logger.info("%s", "\nRepeatability summary:") + logger.info(" - Same image set across runs: %s", image_set_consistent) + logger.info(" - Pass-rate variance: %s", variance_report["pass_rate_variance"]) + logger.info(" - Avg-latency variance: %s", variance_report["avg_latency_variance"]) + logger.info( + " - Max per-image score variance: %s", + variance_report["max_image_score_variance"], + ) + logger.info(" - Decision distribution: %s", decision_distribution) return repeatability_report if __name__ == "__main__": + configure_cli_logging() parser = argparse.ArgumentParser(description="Quantized Vision QA batch tester") parser.add_argument( "--profile", @@ -1138,6 +1992,28 @@ def run_repeatability_test(profile, runs=5, inference_backend_override=None): choices=["simulated", "ollama_vision", "mock_api", "llama_cpp"], help="Temporarily override inference backend without editing config" ) + parser.add_argument( + "--loopback-planner", + default=None, + choices=["simulated", "llm"], + help="Temporarily override loopback planner mode without editing config", + ) + parser.add_argument( + "--planner-timeout-s", + type=float, + default=None, + help="Override runtime.loopback_planner.llm.timeout_s from CLI", + ) + parser.add_argument( + "--planner-model", + default=None, + help="Override runtime.loopback_planner.llm.model from CLI", + ) + parser.add_argument( + "--planner-skip-health-check", + action="store_true", + help="Skip startup health check for loopback planner in llm mode", + ) parser.add_argument( "--performance-analysis", action="store_true", @@ -1153,28 +2029,77 @@ def run_repeatability_test(profile, runs=5, inference_backend_override=None): action="store_true", help="Generate overhead report for framework cost (monitoring, loopback, memory writes)" ) + parser.add_argument( + "--async-batch", + action="store_true", + help="Run batch with asyncio parallel per-image processing (httpx for HTTP backends)", + ) + parser.add_argument( + "--async-concurrency", + type=int, + default=4, + help="Max concurrent images when --async-batch is set (default: 4)", + ) + parser.add_argument( + "--parallel-metrics", + action="store_true", + help="Compute sharpness/brightness metrics in a ProcessPoolExecutor (CPU-bound speedup)", + ) + parser.add_argument( + "--metrics-workers", + type=int, + default=None, + help="Process pool size for --parallel-metrics (default: min(cpu_count, 8))", + ) args = parser.parse_args() try: if args.repeatability_test: run_repeatability_test( args.repeatability_test, runs=max(1, args.repeatability_runs), - inference_backend_override=args.inference_backend + inference_backend_override=args.inference_backend, + loopback_planner_override=args.loopback_planner, + planner_timeout_s_override=args.planner_timeout_s, + planner_model_override=args.planner_model, + planner_require_healthy_override=( + False if args.planner_skip_health_check else None + ), ) elif args.compare_profiles: run_profile_comparison( args.compare_profiles, - inference_backend_override=args.inference_backend + inference_backend_override=args.inference_backend, + loopback_planner_override=args.loopback_planner, + planner_timeout_s_override=args.planner_timeout_s, + planner_model_override=args.planner_model, + planner_require_healthy_override=( + False if args.planner_skip_health_check else None + ), ) else: - run_batch_test( + batch_kwargs = dict( config_profile=args.profile, config_path=args.config, inference_backend_override=args.inference_backend, + loopback_planner_override=args.loopback_planner, + planner_timeout_s_override=args.planner_timeout_s, + planner_model_override=args.planner_model, + planner_require_healthy_override=( + False if args.planner_skip_health_check else None + ), performance_analysis=args.performance_analysis, overhead_analysis=args.overhead_analysis, stress_test_count=100 if args.stress_test_100 else None, + parallel_metrics=args.parallel_metrics, + metrics_workers=args.metrics_workers, ) + if args.async_batch: + run_batch_test_async( + **batch_kwargs, + concurrency=max(1, args.async_concurrency), + ) + else: + run_batch_test(**batch_kwargs) except Exception as e: try: fallback_profile = args.profile if hasattr(args, "profile") else "dev" diff --git a/src/engine/__init__.py b/src/engine/__init__.py new file mode 100644 index 0000000..b21930e --- /dev/null +++ b/src/engine/__init__.py @@ -0,0 +1 @@ +# Marks ``engine`` as a package for editable installs (setuptools discovery). diff --git a/src/engine/image_validator.py b/src/engine/image_validator.py index 9d1519a..b6652d1 100644 --- a/src/engine/image_validator.py +++ b/src/engine/image_validator.py @@ -38,7 +38,3 @@ def analyze_exposure(self, image_path): result["verdict"] = "Fail: Overexposed" return result - -# Test usage: -# validator = ImageQualityValidator() -# print(validator.analyze_exposure("test_photo.jpg")) \ No newline at end of file diff --git a/src/engine/vision_math.py b/src/engine/vision_math.py index 66dcce2..363dde2 100644 --- a/src/engine/vision_math.py +++ b/src/engine/vision_math.py @@ -1,13 +1,17 @@ +import logging import os import statistics from PIL import Image +logger = logging.getLogger(__name__) + + def calculate_metrics(photo_path): """ Load a real image and calculate brightness and sharpness metrics. """ if not os.path.exists(photo_path): - print(f"⚠️ Path not found: {photo_path}") + logger.warning("Path not found: %s", photo_path) return None try: @@ -16,8 +20,8 @@ def calculate_metrics(photo_path): img_gray = img.convert("L") # 2. Downscale image for faster computation (128x128). img_small = img_gray.resize((128, 128)) - # 3. Ensure all pixel values are integers. - pixels = [int(p) for p in list(img_small.getdata())] + # Flattened pixel stream (Pillow 10+); avoids deprecated getdata() (removed in Pillow 14). + pixels = [int(p) for p in img_small.get_flattened_data()] if not pixels: return None @@ -26,8 +30,8 @@ def calculate_metrics(photo_path): return { "sharpness": round(statistics.stdev(pixels), 2), "avg_brightness": round(statistics.mean(pixels), 2), - "max_brightness": int(max(pixels)) + "max_brightness": int(max(pixels)), } except Exception as e: - print(f"❌ Image engine computation failed: {e}") - return None \ No newline at end of file + logger.error("Image engine computation failed: %s", e) + return None diff --git a/src/eval/__init__.py b/src/eval/__init__.py new file mode 100644 index 0000000..80fb6e2 --- /dev/null +++ b/src/eval/__init__.py @@ -0,0 +1 @@ +# Marks ``eval`` as a package for editable installs (setuptools discovery). diff --git a/src/eval/log_analyzer.py b/src/eval/log_analyzer.py index cedae37..77b87f4 100644 --- a/src/eval/log_analyzer.py +++ b/src/eval/log_analyzer.py @@ -1,3 +1,10 @@ +import logging + +from util.cli_logging import configure_cli_logging + +logger = logging.getLogger(__name__) + + class LogAnalyzer: def __init__(self, error_tolerance: int): """ @@ -16,30 +23,27 @@ def find_max_stable_sequence(self, logs: str) -> int: for right in range(len(logs)): # If the current status is Error, increase the counter. - if logs[right] == 'E': + if logs[right] == "E": error_count += 1 - + # Shrink the left boundary when error count exceeds tolerance. while error_count > self.k: - if logs[left] == 'E': + if logs[left] == "E": error_count -= 1 left += 1 - + # Update the maximum valid window size. max_length = max(max_length, right - left + 1) - + return max_length + if __name__ == "__main__": - # Simulated result string after ai_quality_agent execution. - # S = Success, E = Error - test_logs = "SSSESSS" - - # Tolerate one error. + configure_cli_logging() + test_logs = "SSSESSS" + analyzer = LogAnalyzer(error_tolerance=1) stable_length = analyzer.find_max_stable_sequence(test_logs) - - print(f"Test log: {test_logs}") - print(f"Longest stable segment with tolerance {analyzer.k}: {stable_length}") - - # Expected result: In SSSESSS, the full sequence includes one E, so length should be 7. \ No newline at end of file + + logger.info("Test log: %s", test_logs) + logger.info("Longest stable segment with tolerance %s: %s", analyzer.k, stable_length) diff --git a/src/mock_device.py b/src/mock_device.py index 9ee3714..1c6fa6b 100644 --- a/src/mock_device.py +++ b/src/mock_device.py @@ -1,5 +1,11 @@ +import logging +import random import time -import random # Used to simulate AI quality scores. + +from util.cli_logging import configure_cli_logging + +logger = logging.getLogger(__name__) + # 1. Simulated AI model class. class MiniVisionModel: @@ -8,43 +14,43 @@ def analyze(self, file_path): # For now, simulate an AI quality score with random values. return round(random.uniform(0.3, 1.0), 2) + # 2. Updated verification function. def verify_capture_success(device_controller, vision_model, previous_latest_file): timeout = 5 start_time = time.time() - - print("🔍 Start monitoring for a new photo...") - + + logger.info("Start monitoring for a new photo...") + try: while time.time() - start_time < timeout: current_file = device_controller.get_latest_photo_path() - + if current_file != previous_latest_file: - print(f"✅ New file detected: {current_file}") - + logger.info("New file detected: %s", current_file) + # --- Add AI quality analysis logic --- score = vision_model.analyze(current_file) - + if score > 0.8: - print(f"✨ Excellent quality (Score: {score})") + logger.info("Excellent quality (Score: %s)", score) return True - elif score < 0.5: - print(f"⚠️ Quality issue detected: image is too blurry (Score: {score})") + if score < 0.5: + logger.warning("Quality issue detected: image is too blurry (Score: %s)", score) return False - else: - print(f"🤔 Quality is borderline; manual review recommended (Score: {score})") - return True - # -------------------------- - + logger.info("Quality is borderline; manual review recommended (Score: %s)", score) + return True + time.sleep(0.5) - - print("⏳ Monitoring timed out: no new photo found.") + + logger.warning("Monitoring timed out: no new photo found.") return False - except Exception as e: - print(f"❌ Exception occurred during test: {e}") + except Exception: + logger.exception("Exception occurred during capture verification") return False + # 3. Simulated mobile environment. class MockDevice: def __init__(self, mode="normal"): @@ -53,23 +59,21 @@ def __init__(self, mode="normal"): def get_latest_photo_path(self): if self.mode == "crash": - raise Exception("OOM: Out of Memory (mobile memory full)") + raise RuntimeError("OOM: Out of Memory (mobile memory full)") if self.has_new_file: return "/sdcard/DCIM/IMG_NEW.jpg" return "/sdcard/DCIM/IMG_OLD.jpg" -# --- Run tests --- -# Initialize AI model. -my_ai_model = MiniVisionModel() +if __name__ == "__main__": + configure_cli_logging() + my_ai_model = MiniVisionModel() -# Scenario 1: capture succeeds and AI checks quality. -success_phone = MockDevice(mode="normal") -success_phone.has_new_file = True -print("\n--- Test: AI quality analysis path ---") -verify_capture_success(success_phone, my_ai_model, "/sdcard/DCIM/IMG_OLD.jpg") + success_phone = MockDevice(mode="normal") + success_phone.has_new_file = True + logger.info("\n--- Test: AI quality analysis path ---") + verify_capture_success(success_phone, my_ai_model, "/sdcard/DCIM/IMG_OLD.jpg") -# Scenario 2: mobile crash path. -crash_phone = MockDevice(mode="crash") -print("\n--- Test: mobile crash path ---") -verify_capture_success(crash_phone, my_ai_model, "/sdcard/DCIM/IMG_OLD.jpg") \ No newline at end of file + crash_phone = MockDevice(mode="crash") + logger.info("\n--- Test: mobile crash path ---") + verify_capture_success(crash_phone, my_ai_model, "/sdcard/DCIM/IMG_OLD.jpg") diff --git a/src/models/__init__.py b/src/models/__init__.py new file mode 100644 index 0000000..89291ee --- /dev/null +++ b/src/models/__init__.py @@ -0,0 +1 @@ +# Marks ``models`` as a package for editable installs (setuptools discovery). diff --git a/src/models/async_inference.py b/src/models/async_inference.py new file mode 100644 index 0000000..4eed09d --- /dev/null +++ b/src/models/async_inference.py @@ -0,0 +1,227 @@ +""" +Async inference helpers using httpx for I/O-bound backends. + +CPU-only simulated inference runs in a thread pool via asyncio.to_thread. +""" + +from __future__ import annotations + +import asyncio +import json +import logging +from typing import Any, Dict + +import httpx + +from models.inference_adapter import ( + LlamaCppInferenceEngine, + MockAPIInferenceEngine, + OllamaVisionInferenceEngine, + SimulatedInferenceEngine, +) +from models.contracts import InferenceOutput + +logger = logging.getLogger(__name__) + + +async def predict_quality_async( + engine: Any, + client: httpx.AsyncClient, + photo_path: str, + metrics: Dict[str, Any], +) -> Dict[str, str]: + """ + Async quality prediction mirroring sync inference_adapter engines. + """ + backend = getattr(engine, "backend_name", type(engine).__name__) + logger.debug( + "predict_quality_async: start backend=%s photo_path=%s", + backend, + photo_path, + ) + if isinstance(engine, SimulatedInferenceEngine): + result = await asyncio.to_thread(engine.predict_quality, photo_path, metrics) + logger.debug("predict_quality_async: done backend=%s (thread pool)", backend) + return result + if isinstance(engine, LlamaCppInferenceEngine): + return await _llama_cpp_predict_async(engine, client, photo_path, metrics) + if isinstance(engine, OllamaVisionInferenceEngine): + return await _ollama_predict_async(engine, client, photo_path, metrics) + if isinstance(engine, MockAPIInferenceEngine): + return await _mock_api_predict_async(engine, client, photo_path, metrics) + result = await asyncio.to_thread(engine.predict_quality, photo_path, metrics) + logger.debug("predict_quality_async: done backend=%s (generic thread pool)", backend) + return result + + +async def _llama_cpp_predict_async( + engine: LlamaCppInferenceEngine, + client: httpx.AsyncClient, + photo_path: str, + metrics: Dict[str, Any], +) -> Dict[str, str]: + payload = { + "model": engine.model, + "messages": engine._build_messages(photo_path, metrics), + "temperature": engine.temperature, + "max_tokens": engine.max_tokens, + "stream": False, + } + if engine.use_response_format: + payload["response_format"] = {"type": "json_object"} + + url = f"{engine.host}{engine.endpoint}" + timeout = httpx.Timeout(engine.timeout_s) + logger.debug( + "predict_quality_async(llama_cpp): POST %s timeout_s=%.1f", + url, + engine.timeout_s, + ) + try: + response = await client.post(url, json=payload, timeout=timeout) + if response.status_code >= 400 and "response_format" in payload: + payload_without_format = dict(payload) + payload_without_format.pop("response_format", None) + response = await client.post(url, json=payload_without_format, timeout=timeout) + response.raise_for_status() + body = response.json() + model_text = str(body.get("choices", [{}])[0].get("message", {}).get("content", "")) + parsed = engine._extract_json_object(model_text) + return InferenceOutput.from_payload( + parsed, + default_msg="llama.cpp returned unparsable response.", + backend=engine.backend_name, + ).to_dict() + except Exception as exc: + logger.warning( + "predict_quality_async(llama_cpp): request failed url=%s error=%s", + url, + exc, + ) + if engine.fallback_to_simulated: + fallback = await asyncio.to_thread( + engine.simulated_fallback.predict_quality, photo_path, metrics + ) + fallback["msg"] = f"llama.cpp fallback to simulated inference: {exc}" + fallback["backend"] = f"{engine.backend_name}->simulated" + return fallback + return InferenceOutput( + decision="Error", + code="ERR_MODEL_BACKEND_503", + msg=f"llama.cpp inference failed: {exc}", + backend=engine.backend_name, + ).to_dict() + + +async def _ollama_predict_async( + engine: OllamaVisionInferenceEngine, + client: httpx.AsyncClient, + photo_path: str, + metrics: Dict[str, Any], +) -> Dict[str, str]: + prompt = ( + f"{engine.prompt_template}\n" + f"Metrics: {json.dumps(metrics, ensure_ascii=False)}\n" + f"Thresholds: {json.dumps(engine.thresholds, ensure_ascii=False)}" + ) + payload = { + "model": engine.model, + "prompt": prompt, + "stream": False, + "images": [await asyncio.to_thread(engine._encode_image, photo_path)], + "format": "json", + } + url = f"{engine.host}/api/generate" + timeout = httpx.Timeout(engine.timeout_s) + logger.debug( + "predict_quality_async(ollama): POST %s timeout_s=%.1f", + url, + engine.timeout_s, + ) + try: + response = await client.post(url, json=payload, timeout=timeout) + response.raise_for_status() + body = response.json() + model_text = str(body.get("response", "")) + parsed = engine._extract_json_object(model_text) + return InferenceOutput.from_payload( + parsed, + default_msg="Ollama returned unparsable response.", + backend=engine.backend_name, + ).to_dict() + except Exception as exc: + logger.warning( + "predict_quality_async(ollama): request failed url=%s error=%s", + url, + exc, + ) + if engine.fallback_to_simulated: + fallback = await asyncio.to_thread( + engine.simulated_fallback.predict_quality, photo_path, metrics + ) + fallback["msg"] = f"Ollama fallback to simulated inference: {exc}" + fallback["backend"] = f"{engine.backend_name}->simulated" + return fallback + return InferenceOutput( + decision="Error", + code="ERR_MODEL_BACKEND_503", + msg=f"Ollama inference failed: {exc}", + backend=engine.backend_name, + ).to_dict() + + +async def _mock_api_predict_async( + engine: MockAPIInferenceEngine, + client: httpx.AsyncClient, + photo_path: str, + metrics: Dict[str, Any], +) -> Dict[str, str]: + import os + + headers = {"Content-Type": "application/json"} + api_key = os.getenv(engine.api_key_env) + if api_key: + headers["Authorization"] = f"Bearer {api_key}" + + payload = { + "photo_path": photo_path, + "metrics": metrics, + "thresholds": engine.thresholds, + } + timeout = httpx.Timeout(engine.timeout_s) + logger.debug( + "predict_quality_async(mock_api): POST %s timeout_s=%.1f", + engine.url, + engine.timeout_s, + ) + try: + response = await client.post( + engine.url, json=payload, headers=headers, timeout=timeout + ) + response.raise_for_status() + body = response.json() + result = body.get("result", body) + return InferenceOutput.from_payload( + result, + default_msg="Mock API returned invalid response.", + backend=engine.backend_name, + ).to_dict() + except Exception as exc: + logger.warning( + "predict_quality_async(mock_api): request failed url=%s error=%s", + engine.url, + exc, + ) + if engine.fallback_to_simulated: + fallback = await asyncio.to_thread( + engine.simulated_fallback.predict_quality, photo_path, metrics + ) + fallback["msg"] = f"Mock API fallback to simulated inference: {exc}" + fallback["backend"] = f"{engine.backend_name}->simulated" + return fallback + return InferenceOutput( + decision="Error", + code="ERR_MODEL_BACKEND_503", + msg=f"Mock API inference failed: {exc}", + backend=engine.backend_name, + ).to_dict() diff --git a/src/models/contracts.py b/src/models/contracts.py new file mode 100644 index 0000000..67eca07 --- /dev/null +++ b/src/models/contracts.py @@ -0,0 +1,109 @@ +from __future__ import annotations + +from dataclasses import dataclass +from typing import Any, Dict, List, Literal, Optional + +from pydantic import BaseModel, Field + + +@dataclass(frozen=True) +class InferenceOutput: + decision: str + code: str + msg: str + confidence: Optional[float] = None + backend: Optional[str] = None + + @classmethod + def from_payload( + cls, + payload: Any, + *, + default_msg: str, + default_decision: str = "Error", + default_code: str = "ERR_MODEL_RESPONSE_422", + backend: Optional[str] = None, + ) -> "InferenceOutput": + if not isinstance(payload, dict): + return cls( + decision=default_decision, + code=default_code, + msg=default_msg, + backend=backend, + ) + + confidence: Optional[float] = None + raw_confidence = payload.get("confidence") + if raw_confidence is not None: + try: + confidence = float(raw_confidence) + except (TypeError, ValueError): + confidence = None + + return cls( + decision=str(payload.get("decision", default_decision)), + code=str(payload.get("code", default_code)), + msg=str(payload.get("msg", default_msg)), + confidence=confidence, + backend=backend, + ) + + def to_dict(self) -> Dict[str, Any]: + data: Dict[str, Any] = { + "decision": self.decision, + "code": self.code, + "msg": self.msg, + } + if self.confidence is not None: + data["confidence"] = self.confidence + if self.backend: + data["backend"] = self.backend + return data + + +@dataclass(frozen=True) +class LoopbackPlan: + action: Optional[str] + stop_reason: str + rationale: str + fallback_used: bool = False + planner_backend: str = "simulated" + + +class AgentStep(BaseModel): + attempt: int = Field(..., description="Current retry round, starts from 1") + signal: Literal["under", "over", "blurry", "other"] = Field( + ..., description="Image signal emitted by evaluator" + ) + action: Literal["brighten", "dim", "sharpen", "stop"] = Field( + ..., description="Planner action taken for this step" + ) + rationale: str = Field(..., description="Why planner selected this action") + fallback_used: bool = Field( + default=False, + description="True when planner falls back from llm to simulated rules", + ) + metrics_before: Dict[str, Any] = Field( + default_factory=dict, description="Metrics before executing this step action" + ) + metrics_after: Optional[Dict[str, Any]] = Field( + default=None, description="Metrics observed after action is executed" + ) + latency_ms: float = Field(..., description="Step latency in milliseconds") + + +class AgentInferenceOutput(BaseModel): + image_path: str + final_decision: Literal["GO", "REVIEW", "NO_GO"] = Field( + ..., description="Final release decision for this image" + ) + error_code: str = Field( + default="SUCCESS_200", description="Machine-oriented decision/error code" + ) + error_message: str = Field( + default="Optimal", description="Human-oriented decision/error message" + ) + steps: List[AgentStep] = Field( + default_factory=list, description="Per-image agent decision trace" + ) + total_latency_ms: float = Field(..., description="Total latency for the image") diff --git a/src/models/inference_adapter.py b/src/models/inference_adapter.py index 3e3b9cb..150f78f 100644 --- a/src/models/inference_adapter.py +++ b/src/models/inference_adapter.py @@ -4,28 +4,12 @@ from importlib import import_module from typing import Any, Dict, List +from models.contracts import InferenceOutput from models.llama_quantizer import LlamaQuantizer -def _normalize_result(result: Dict[str, Any], default_msg: str) -> Dict[str, Any]: - if not isinstance(result, dict): - return { - "decision": "Error", - "code": "ERR_MODEL_RESPONSE_422", - "msg": default_msg, - } - - normalized: Dict[str, Any] = { - "decision": str(result.get("decision", "Error")), - "code": str(result.get("code", "ERR_MODEL_RESPONSE_422")), - "msg": str(result.get("msg", default_msg)), - } - if result.get("confidence") is not None: - try: - normalized["confidence"] = float(result["confidence"]) - except (TypeError, ValueError): - pass - return normalized +def _normalize_result(result: Any, default_msg: str) -> Dict[str, Any]: + return InferenceOutput.from_payload(result, default_msg=default_msg).to_dict() _REQUESTS_MODULE = None @@ -46,9 +30,11 @@ def __init__(self, thresholds: Dict[str, Any]): def predict_quality(self, photo_path: str, metrics: Dict[str, Any]) -> Dict[str, str]: result = self.quantizer.predict_quality(metrics) - normalized = _normalize_result(result, "Simulated inference returned invalid response.") - normalized["backend"] = self.backend_name - return normalized + return InferenceOutput.from_payload( + result, + default_msg="Simulated inference returned invalid response.", + backend=self.backend_name, + ).to_dict() class OllamaVisionInferenceEngine: @@ -113,21 +99,23 @@ def predict_quality(self, photo_path: str, metrics: Dict[str, Any]) -> Dict[str, body = response.json() model_text = str(body.get("response", "")) parsed = self._extract_json_object(model_text) - normalized = _normalize_result(parsed, "Ollama returned unparsable response.") - normalized["backend"] = self.backend_name - return normalized + return InferenceOutput.from_payload( + parsed, + default_msg="Ollama returned unparsable response.", + backend=self.backend_name, + ).to_dict() except Exception as exc: if self.fallback_to_simulated: fallback = self.simulated_fallback.predict_quality(photo_path, metrics) fallback["msg"] = f"Ollama fallback to simulated inference: {exc}" fallback["backend"] = f"{self.backend_name}->simulated" return fallback - return { - "decision": "Error", - "code": "ERR_MODEL_BACKEND_503", - "msg": f"Ollama inference failed: {exc}", - "backend": self.backend_name, - } + return InferenceOutput( + decision="Error", + code="ERR_MODEL_BACKEND_503", + msg=f"Ollama inference failed: {exc}", + backend=self.backend_name, + ).to_dict() class MockAPIInferenceEngine: @@ -161,21 +149,23 @@ def predict_quality(self, photo_path: str, metrics: Dict[str, Any]) -> Dict[str, response.raise_for_status() body = response.json() result = body.get("result", body) - normalized = _normalize_result(result, "Mock API returned invalid response.") - normalized["backend"] = self.backend_name - return normalized + return InferenceOutput.from_payload( + result, + default_msg="Mock API returned invalid response.", + backend=self.backend_name, + ).to_dict() except Exception as exc: if self.fallback_to_simulated: fallback = self.simulated_fallback.predict_quality(photo_path, metrics) fallback["msg"] = f"Mock API fallback to simulated inference: {exc}" fallback["backend"] = f"{self.backend_name}->simulated" return fallback - return { - "decision": "Error", - "code": "ERR_MODEL_BACKEND_503", - "msg": f"Mock API inference failed: {exc}", - "backend": self.backend_name, - } + return InferenceOutput( + decision="Error", + code="ERR_MODEL_BACKEND_503", + msg=f"Mock API inference failed: {exc}", + backend=self.backend_name, + ).to_dict() class LlamaCppInferenceEngine: @@ -279,21 +269,23 @@ def predict_quality(self, photo_path: str, metrics: Dict[str, Any]) -> Dict[str, body.get("choices", [{}])[0].get("message", {}).get("content", "") ) parsed = self._extract_json_object(model_text) - normalized = _normalize_result(parsed, "llama.cpp returned unparsable response.") - normalized["backend"] = self.backend_name - return normalized + return InferenceOutput.from_payload( + parsed, + default_msg="llama.cpp returned unparsable response.", + backend=self.backend_name, + ).to_dict() except Exception as exc: if self.fallback_to_simulated: fallback = self.simulated_fallback.predict_quality(photo_path, metrics) fallback["msg"] = f"llama.cpp fallback to simulated inference: {exc}" fallback["backend"] = f"{self.backend_name}->simulated" return fallback - return { - "decision": "Error", - "code": "ERR_MODEL_BACKEND_503", - "msg": f"llama.cpp inference failed: {exc}", - "backend": self.backend_name, - } + return InferenceOutput( + decision="Error", + code="ERR_MODEL_BACKEND_503", + msg=f"llama.cpp inference failed: {exc}", + backend=self.backend_name, + ).to_dict() def build_inference_engine(config: Dict[str, Any]): diff --git a/src/models/llama_analyst.py b/src/models/llama_analyst.py index 03970f2..b8870bc 100644 --- a/src/models/llama_analyst.py +++ b/src/models/llama_analyst.py @@ -1,7 +1,12 @@ -import requests import json +import logging import time +import requests + +logger = logging.getLogger(__name__) + + class LlamaAnalyst: def __init__(self): # 預設使用 completion 接口以獲得最高穩定性 @@ -10,7 +15,7 @@ def __init__(self): def analyze_quality(self, metrics): start_time = time.time() - + # 任務一：優化 Prompt 結構 (Prefix Prompting) prompt = f"""Analyze these camera metrics and return JSON. Metrics: {metrics} @@ -23,39 +28,37 @@ def analyze_quality(self, metrics): "prompt": prompt, "temperature": 0.0, "max_tokens": 150, - "stop": ["}", "\n\n"] + "stop": ["}", "\n\n"], } - + try: response = requests.post(self.completion_url, json=payload, timeout=30) response.raise_for_status() - + res_data = response.json() - content = res_data.get('content', '').strip() - + content = res_data.get("content", "").strip() + # 手動補回左大括號並確保閉合 full_json = "{" + content if not full_json.endswith("}"): full_json += "}" - + # 任務二：量化指標計算 end_time = time.time() duration = end_time - start_time - + # 估算 Token 數量 (英文約 4 字母一個 token，這在無 usage 回傳時是專業的替代方案) - estimated_tokens = len(content) // 4 + estimated_tokens = len(content) // 4 tps = estimated_tokens / duration if duration > 0 else 0 - - # 專業 Performance Report 輸出 - print(f"\n--- [Llama Performance Report] ---") - print(f"Total Latency : {duration:.2f}s") - print(f"Est. Tokens : {estimated_tokens}") - print(f"Throughput : {tps:.2f} TPS") - print(f"----------------------------------\n") - + + logger.info("\n--- [Llama Performance Report] ---") + logger.info("Total Latency : %.2fs", duration) + logger.info("Est. Tokens : %s", estimated_tokens) + logger.info("Throughput : %.2f TPS", tps) + logger.info("----------------------------------\n") + return full_json - - except Exception as e: - print(f"--- [Llama Inference Failed] ---") - print(f"Error: {str(e)}") - return json.dumps({"verdict": "Error", "analysis": "Pipeline failed."}) \ No newline at end of file + + except Exception: + logger.exception("Llama inference failed") + return json.dumps({"verdict": "Error", "analysis": "Pipeline failed."}) diff --git a/src/test_failure_memory_retrieval.py b/src/test_failure_memory_retrieval.py index 46af543..8515f64 100644 --- a/src/test_failure_memory_retrieval.py +++ b/src/test_failure_memory_retrieval.py @@ -1,5 +1,10 @@ +import logging + +from util.cli_logging import configure_cli_logging from util.failure_memory import FailureMemoryStore +logger = logging.getLogger(__name__) + def main(): store = FailureMemoryStore() @@ -30,19 +35,23 @@ def main(): metadatas = result.get("metadatas", [[]])[0] distances = result.get("distances", [[]])[0] - print(f"Query: {query}") + logger.info("Query: %s", query) if not documents: - print("No similar failure cases found.") + logger.info("No similar failure cases found.") return - print("Top similar failure cases:") + logger.info("Top similar failure cases:") for idx, (doc, meta, dist) in enumerate(zip(documents, metadatas, distances), start=1): - print( - f"{idx}. file={meta.get('file')} | release={meta.get('release_decision')} | " - f"distance={dist:.4f}" + logger.info( + "%s. file=%s | release=%s | distance=%.4f", + idx, + meta.get("file"), + meta.get("release_decision"), + dist, ) - print(f" document={doc}") + logger.info(" document=%s", doc) if __name__ == "__main__": + configure_cli_logging() main() diff --git a/src/util/cli_logging.py b/src/util/cli_logging.py new file mode 100644 index 0000000..488a33b --- /dev/null +++ b/src/util/cli_logging.py @@ -0,0 +1,16 @@ +"""Default logging setup for CLI and script ``__main__`` entrypoints.""" + +from __future__ import annotations + +import logging + +_FORMAT = "%(asctime)s [%(levelname)s] %(name)s: %(message)s" +_DATEFMT = "%Y-%m-%d %H:%M:%S" + + +def configure_cli_logging(level: int = logging.INFO) -> None: + """Attach a stream handler to the root logger if none exist yet.""" + root = logging.getLogger() + if root.handlers: + return + logging.basicConfig(level=level, format=_FORMAT, datefmt=_DATEFMT) diff --git a/src/util/failure_memory.py b/src/util/failure_memory.py index 191e7a2..bca0dfd 100644 --- a/src/util/failure_memory.py +++ b/src/util/failure_memory.py @@ -1,7 +1,10 @@ +import logging import os from importlib import import_module from datetime import datetime, timezone +logger = logging.getLogger(__name__) + os.environ.setdefault("PROTOCOL_BUFFERS_PYTHON_IMPLEMENTATION", "python") @@ -34,7 +37,10 @@ def __init__( ) self.encoder = sentence_transformer_cls(embedding_model) except Exception as e: - print(f"WARNING: Could not load sentence-transformers model ({e}). Using local fallback embeddings.") + logger.warning( + "Could not load sentence-transformers model (%s). Using local fallback embeddings.", + e, + ) def build_document(self, file_name, decision_payload): reason = decision_payload.get("msg") or decision_payload.get("decision") or "Unknown issue" diff --git a/src/util/metrics_pool.py b/src/util/metrics_pool.py new file mode 100644 index 0000000..84e5417 --- /dev/null +++ b/src/util/metrics_pool.py @@ -0,0 +1,61 @@ +""" +Process-pool execution for CPU-bound image metrics (Pillow stdev/mean on pixels). + +Batch runs with many images spend significant time in calculate_metrics; using +multiple processes avoids the GIL and speeds throughput on multi-core hosts. +""" + +from __future__ import annotations + +import logging +import os +from concurrent.futures import ProcessPoolExecutor +from typing import Optional + +from engine.vision_math import calculate_metrics + +logger = logging.getLogger(__name__) + + +def _default_max_workers() -> int: + cpu = os.cpu_count() or 1 + return max(1, min(cpu, 8)) + + +class MetricsProcessPool: + """ + Context manager wrapping ProcessPoolExecutor for calculate_metrics calls. + """ + + def __init__(self, max_workers: Optional[int] = None) -> None: + self.max_workers = max(1, int(max_workers or _default_max_workers())) + self._executor: Optional[ProcessPoolExecutor] = None + + def __enter__(self) -> MetricsProcessPool: + logger.info( + "MetricsProcessPool: starting process pool max_workers=%s", + self.max_workers, + ) + self._executor = ProcessPoolExecutor(max_workers=self.max_workers) + return self + + def __exit__(self, exc_type, exc, tb) -> None: + if self._executor is not None: + logger.debug("MetricsProcessPool: shutting down process pool") + self._executor.shutdown(wait=True) + self._executor = None + + @property + def executor(self) -> ProcessPoolExecutor: + if self._executor is None: + raise RuntimeError("MetricsProcessPool is not active; use as a context manager") + return self._executor + + def calculate(self, photo_path: str): + """ + Run calculate_metrics in a worker process (blocking in caller thread). + """ + if self._executor is None: + raise RuntimeError("MetricsProcessPool is not active") + future = self._executor.submit(calculate_metrics, photo_path) + return future.result() diff --git a/src/util/monitor_performance.py b/src/util/monitor_performance.py index 14934a9..7d242b8 100644 --- a/src/util/monitor_performance.py +++ b/src/util/monitor_performance.py @@ -1,8 +1,9 @@ """ -Performance monitoring helpers for PixelQA agent/util layers. +Performance monitoring helpers for the Agentic Testing Framework (agent/util layers). Provides sync/async decorators that log wall time; optional peak traced allocation -via ``tracemalloc`` when ``PIXELQA_MONITOR_MEMORY`` is enabled (profile/debug). +via ``tracemalloc`` when ``ATF_MONITOR_MEMORY`` is enabled (profile/debug). +Legacy: ``PIXELQA_MONITOR_MEMORY`` is accepted as an alias. Also provides a simple wall-time context manager for inline sections. """ @@ -22,20 +23,21 @@ F = TypeVar("F", bound=Callable[..., Any]) -_ENV_MEMORY_FLAG = "PIXELQA_MONITOR_MEMORY" - def _memory_tracing_enabled() -> bool: """Enable tracemalloc peak/current MB in decorator logs (extra overhead).""" - v = os.environ.get(_ENV_MEMORY_FLAG, "").strip().lower() - return v in ("1", "true", "yes", "on") + for key in ("ATF_MONITOR_MEMORY", "PIXELQA_MONITOR_MEMORY"): + v = os.environ.get(key, "").strip().lower() + if v in ("1", "true", "yes", "on"): + return True + return False def monitor_performance(func: F) -> F: """ Decorator for synchronous callables: records elapsed time; optional peak memory (tracemalloc). - Memory tracing is controlled by env ``PIXELQA_MONITOR_MEMORY`` (default: off) to avoid + Memory tracing is controlled by env ``ATF_MONITOR_MEMORY`` (default: off) to avoid overhead on very hot call paths. Logs entry at DEBUG and completion at INFO so expensive paths stay observable without diff --git a/src/verify_capture_success.py b/src/verify_capture_success.py index 2c169df..386aa7c 100644 --- a/src/verify_capture_success.py +++ b/src/verify_capture_success.py @@ -1,60 +1,78 @@ -import json +import logging import os -import random +import tempfile from datetime import datetime +from pathlib import Path +from typing import Any, Optional + +from PIL import Image + from engine.vision_math import calculate_metrics +from util.cli_logging import configure_cli_logging + +logger = logging.getLogger(__name__) + class QuantizedVisionAgent: - def __init__(self, model_name="PixelQA-Llama-4bit"): + def __init__(self, model_name="Agentic Testing Framework - Llama 4-bit"): self.model_name = model_name - print(f"📦 Loaded quantized model: {self.model_name}") + logger.info("Loaded quantized model: %s", self.model_name) - def verify_capture_success(self, photo_path): - """Step A: Verify that the file exists.""" - return os.path.exists(photo_path) or random.choice([True, False]) # Simulated check + def verify_capture_success(self, photo_path: str) -> bool: + """Return True if the capture file exists (deterministic for this demo script).""" + return os.path.isfile(photo_path) def call_4bit_model_inference(self, metrics): """Step B: Simulate 4-bit model inference based on metrics.""" - # Simulated AI logic: low sharpness implies out-of-focus. if metrics["sharpness"] < 10: return "Fail: Out of Focus (AI Detected)" - elif metrics["avg_brightness"] < 50: + if metrics["avg_brightness"] < 50: return "Fail: Too Dark (AI Detected)" - else: - return "Pass: Quality Meets Standard" + return "Pass: Quality Meets Standard" + -def run_test_pipeline(): +def run_test_pipeline(work_dir: Optional[Path] = None) -> None: + """ + Build a temporary JPEG, run ``calculate_metrics`` on a real path, then simulate inference. + """ agent = QuantizedVisionAgent() - mock_photo = "/sdcard/DCIM/test_shot_002.jpg" - - report = {"timestamp": datetime.now().isoformat(), "test_cases": []} + base = work_dir if work_dir is not None else Path(tempfile.mkdtemp(prefix="atf_verify_")) + base.mkdir(parents=True, exist_ok=True) + mock_photo = base / "mock_shot.jpg" + # Slight variation so sharpness / brightness are non-trivial vs flat fields. + Image.new("RGB", (64, 64), (118, 120, 119)).save(mock_photo, format="JPEG", quality=95) + + report: dict[str, Any] = {"timestamp": datetime.now().isoformat(), "test_cases": []} try: - print(f"🔍 Checking whether file exists: {mock_photo}") - if agent.verify_capture_success(mock_photo): - # Simulate pixel data from a blurry-edge image. - mock_pixels = [120, 122, 121, 119, 120, 121] - metrics = calculate_metrics(mock_pixels) - - # Call 4-bit model for decision. + logger.info("Checking whether file exists: %s", mock_photo) + if agent.verify_capture_success(str(mock_photo)): + metrics = calculate_metrics(str(mock_photo)) + if metrics is None: + logger.error("Metrics unavailable (image load failed).") + return + ai_decision = agent.call_4bit_model_inference(metrics) - - print(f"📊 Numeric metrics: {metrics}") - print(f"🤖 AI decision: {ai_decision}") - - report["test_cases"].append({ - "file": mock_photo, - "ai_decision": ai_decision, - "metrics": metrics - }) + + logger.info("Numeric metrics: %s", metrics) + logger.info("AI decision: %s", ai_decision) + + report["test_cases"].append( + { + "file": str(mock_photo), + "ai_decision": ai_decision, + "metrics": metrics, + } + ) else: - print("❌ Error: File does not exist. Skipping AI analysis.") + logger.warning("File does not exist. Skipping AI analysis.") - except Exception as e: - print(f"💥 System crash: {e}") + except OSError: + logger.exception("System error during verify pipeline") finally: - # Hook your existing save_report(report) here. - print("💾 Test report has been updated.") + logger.info("Test report has been updated.") + if __name__ == "__main__": - run_test_pipeline() \ No newline at end of file + configure_cli_logging() + run_test_pipeline() diff --git a/test_connection.py b/test_connection.py index aee666f..30293f4 100644 --- a/test_connection.py +++ b/test_connection.py @@ -1,31 +1,45 @@ -import requests +import logging import time +import requests + +logger = logging.getLogger(__name__) + + def test_llama_health_check(url="http://localhost:8080/v1", model="llama-3.1-8b"): endpoint = f"{url}/chat/completions" payload = { "model": model, "messages": [{"role": "user", "content": "Ping"}], - "max_tokens": 1, - "temperature": 0.0 + "max_tokens": 1, + "temperature": 0.0, } - + try: - start = time.perf_counter() # 使用更精確的計時器 + start = time.perf_counter() res = requests.post(endpoint, json=payload, timeout=30) - res.raise_for_status() # 直接攔截 4xx/5xx 錯誤 - + res.raise_for_status() + latency = time.perf_counter() - start - data = res.json() - - print(f"✅ [{model}] Connected.") - print(f"⏱️ TTFT (Approx): {latency:.4f}s") - # 這裡可以整合進你的 PixelQA-Llama 效能報告中 - + res.json() + + logger.info("[%s] Connected.", model) + logger.info("TTFT (Approx): %.4fs", latency) + except requests.exceptions.RequestException as e: - print(f"❌ Connection Failed: {e}") + logger.error("Connection Failed: %s", e) except KeyError: - print(f"❌ Malformed Response: {res.text}") + logger.error("Malformed Response: %s", res.text) + if __name__ == "__main__": - test_llama_health_check() \ No newline at end of file + import sys + from pathlib import Path + + _src = Path(__file__).resolve().parent / "src" + if str(_src) not in sys.path: + sys.path.insert(0, str(_src)) + from util.cli_logging import configure_cli_logging # noqa: E402 + + configure_cli_logging() + test_llama_health_check() diff --git a/tests/test_agent_inference_output.py b/tests/test_agent_inference_output.py new file mode 100644 index 0000000..4e7d810 --- /dev/null +++ b/tests/test_agent_inference_output.py @@ -0,0 +1,41 @@ +from ai_quality_agent import _build_agent_inference_output + + +def test_build_agent_inference_output_contains_steps(): + output = _build_agent_inference_output( + image_path="images/a.jpg", + attempt_history=[ + { + "attempt": 1, + "release": "NO_GO", + "loopback_signal": "under", + "action": "brighten", + "rationale": "too dark", + "planner_fallback_used": True, + "avg_brightness": 10.0, + "sharpness": 4.0, + "latency_ms": 12.5, + }, + { + "attempt": 2, + "release": "REVIEW", + "loopback_signal": "other", + "action": "stop", + "rationale": "resolved enough", + "planner_fallback_used": False, + "avg_brightness": 42.0, + "sharpness": 4.5, + "latency_ms": 9.0, + }, + ], + final_ai_result={"code": "ERR_LIGHT_DARK_002", "msg": "Under-exposed"}, + total_latency_ms=21.5, + ) + assert output["image_path"] == "images/a.jpg" + assert output["final_decision"] == "REVIEW" + assert output["error_code"] == "ERR_LIGHT_DARK_002" + assert len(output["steps"]) == 2 + assert output["steps"][0]["fallback_used"] is True + assert output["steps"][0]["metrics_before"]["avg_brightness"] == 10.0 + assert output["steps"][0]["metrics_after"]["avg_brightness"] == 42.0 + diff --git a/tests/test_async_batch.py b/tests/test_async_batch.py new file mode 100644 index 0000000..58ab1ee --- /dev/null +++ b/tests/test_async_batch.py @@ -0,0 +1,60 @@ +from pathlib import Path + +from PIL import Image + +import ai_quality_agent as qa + + +def _make_test_image(path: Path): + image = Image.new("L", (32, 32), color=120) + image.save(path) + + +def test_run_batch_test_async_single_image(monkeypatch, tmp_path): + image_path = tmp_path / "good.png" + _make_test_image(image_path) + + config = { + "thresholds": { + "min_sharpness": 20.0, + "min_brightness": 40.0, + "max_brightness": 220.0, + }, + "runtime": {"oom_probability": 0.0, "max_retry": 0}, + "folders": {"input": str(tmp_path), "output": str(tmp_path / "out"), "logs": str(tmp_path / "logs")}, + "quality_gate": {"target_pass_rate": 80.0}, + "eval_settings": {"conflict_strategy": "conservative", "auto_tag_conflicts": True}, + "model_settings": {"name": "test", "bit_depth": 4, "inference": {"backend": "simulated"}}, + } + + captured_report = {} + + def fake_load_config(profile, config_path): + return config, "TEST_CONFIG" + + def fake_get_all_photos(self): + return [str(image_path)] + + async def fake_analyze_async(self, photo_path, http_client): + return ( + {"sharpness": 50.0, "avg_brightness": 80.0}, + {"decision": "Optimal", "code": "SUCCESS_200", "backend": "simulated"}, + 1.0, + ) + + def fake_save_batch_report(report_data, output_folder): + captured_report["data"] = report_data + return str(tmp_path / "batch_report.json") + + monkeypatch.setattr(qa, "load_config", fake_load_config) + monkeypatch.setattr(qa.QuantizedVisionAgent, "get_all_photos", fake_get_all_photos) + monkeypatch.setattr(qa.QuantizedVisionAgent, "analyze_photo_quality_async", fake_analyze_async) + monkeypatch.setattr(qa, "save_batch_report", fake_save_batch_report) + + result = qa.run_batch_test_async(config_profile="dev", concurrency=2) + + assert result is not None + assert result["summary"]["release_decision"] in {"GO", "REVIEW", "NO_GO"} + assert captured_report["data"]["execution_mode"] == "async" + assert captured_report["data"]["async_concurrency"] == 2 + assert len(captured_report["data"]["results"]) == 1 diff --git a/tests/test_async_inference.py b/tests/test_async_inference.py new file mode 100644 index 0000000..0ba2fe7 --- /dev/null +++ b/tests/test_async_inference.py @@ -0,0 +1,375 @@ +""" +Async HTTP inference tests using httpx.MockTransport (no live servers). + +Includes timeout/connect failure paths per project test conventions. +""" + +from __future__ import annotations + +import asyncio +import json +from pathlib import Path +from typing import Any, Callable, Dict, List, Optional + +import httpx +import pytest +from PIL import Image + +from models.async_inference import predict_quality_async +from models.inference_adapter import ( + LlamaCppInferenceEngine, + MockAPIInferenceEngine, + OllamaVisionInferenceEngine, + SimulatedInferenceEngine, +) + +THRESHOLDS = { + "min_sharpness": 20.0, + "min_brightness": 40.0, + "max_brightness": 220.0, +} +GOOD_METRICS = {"sharpness": 50.0, "avg_brightness": 80.0} + + +def _make_test_image(path: Path) -> None: + Image.new("L", (32, 32), color=120).save(path) + + +def _merge_inference_cfg( + base: Dict[str, Any], + overrides: Dict[str, Any], + nested_keys: tuple[str, ...], +) -> Dict[str, Any]: + merged = dict(base) + for key in nested_keys: + if key in overrides: + merged[key] = {**merged.get(key, {}), **overrides[key]} + merged.update({k: v for k, v in overrides.items() if k not in nested_keys}) + return merged + + +def _llama_inference_cfg(**overrides: Any) -> Dict[str, Any]: + return _merge_inference_cfg( + { + "fallback_to_simulated": True, + "llama_cpp": { + "host": "http://127.0.0.1:8080", + "endpoint": "/v1/chat/completions", + "model": "test-model", + "timeout_s": 5.0, + "use_response_format": True, + }, + }, + overrides, + ("llama_cpp",), + ) + + +def _ollama_inference_cfg(**overrides: Any) -> Dict[str, Any]: + return _merge_inference_cfg( + { + "fallback_to_simulated": True, + "ollama": { + "host": "http://localhost:11434", + "model": "llava:7b", + "timeout_s": 5.0, + }, + }, + overrides, + ("ollama",), + ) + + +def _mock_api_inference_cfg(**overrides: Any) -> Dict[str, Any]: + return _merge_inference_cfg( + { + "fallback_to_simulated": True, + "mock_api": { + "url": "http://localhost:9090/infer", + "timeout_s": 3.0, + "api_key_env": "MOCK_INFER_API_KEY", + }, + }, + overrides, + ("mock_api",), + ) + + +def _run_async(coro): + return asyncio.run(coro) + + +async def _predict( + engine: Any, + handler: Callable[[httpx.Request], httpx.Response], + photo_path: str, + metrics: Dict[str, Any], +) -> Dict[str, str]: + transport = httpx.MockTransport(handler) + async with httpx.AsyncClient(transport=transport) as client: + return await predict_quality_async(engine, client, photo_path, metrics) + + +def test_predict_quality_async_simulated(): + engine = SimulatedInferenceEngine(thresholds=THRESHOLDS) + + def handler(request: httpx.Request) -> httpx.Response: + pytest.fail(f"simulated backend should not call HTTP: {request.url}") + + result = _run_async(_predict(engine, handler, "dummy.jpg", GOOD_METRICS)) + assert result["backend"] == "simulated" + assert result["decision"] in {"Optimal", "Blurry", "Under-exposed", "Over-exposed", "Error"} + + +def test_llama_cpp_async_success(tmp_path: Path): + image_path = tmp_path / "photo.png" + _make_test_image(image_path) + engine = LlamaCppInferenceEngine( + thresholds=THRESHOLDS, + inference_cfg=_llama_inference_cfg(), + ) + model_json = json.dumps( + {"decision": "Optimal", "code": "SUCCESS_200", "msg": "ok", "confidence": 0.91} + ) + + def handler(request: httpx.Request) -> httpx.Response: + assert request.method == "POST" + assert request.url.path == "/v1/chat/completions" + body = json.loads(request.content.decode()) + assert body["model"] == "test-model" + assert body.get("response_format") == {"type": "json_object"} + return httpx.Response( + 200, + json={"choices": [{"message": {"content": model_json}}]}, + ) + + result = _run_async(_predict(engine, handler, str(image_path), GOOD_METRICS)) + assert result["backend"] == "llama_cpp" + assert result["decision"] == "Optimal" + assert result["code"] == "SUCCESS_200" + assert result["confidence"] == pytest.approx(0.91) + + +def test_llama_cpp_async_retries_without_response_format_on_400(tmp_path: Path): + image_path = tmp_path / "photo.png" + _make_test_image(image_path) + engine = LlamaCppInferenceEngine( + thresholds=THRESHOLDS, + inference_cfg=_llama_inference_cfg(), + ) + calls: List[httpx.Request] = [] + model_json = json.dumps({"decision": "Optimal", "code": "SUCCESS_200", "msg": "ok"}) + + def handler(request: httpx.Request) -> httpx.Response: + calls.append(request) + body = json.loads(request.content.decode()) + if len(calls) == 1: + assert body.get("response_format") == {"type": "json_object"} + return httpx.Response(400, json={"error": "response_format not supported"}) + assert "response_format" not in body + return httpx.Response( + 200, + json={"choices": [{"message": {"content": model_json}}]}, + ) + + result = _run_async(_predict(engine, handler, str(image_path), GOOD_METRICS)) + assert len(calls) == 2 + assert result["backend"] == "llama_cpp" + assert result["decision"] == "Optimal" + + +def test_llama_cpp_async_connect_timeout_falls_back(tmp_path: Path): + image_path = tmp_path / "photo.png" + _make_test_image(image_path) + engine = LlamaCppInferenceEngine( + thresholds=THRESHOLDS, + inference_cfg=_llama_inference_cfg(), + ) + + def handler(request: httpx.Request) -> httpx.Response: + raise httpx.ConnectTimeout( + "connection timed out", + request=request, + ) + + result = _run_async(_predict(engine, handler, str(image_path), GOOD_METRICS)) + assert result["backend"] == "llama_cpp->simulated" + assert "fallback to simulated inference" in result["msg"] + + +def test_llama_cpp_async_error_without_fallback(tmp_path: Path): + image_path = tmp_path / "photo.png" + _make_test_image(image_path) + engine = LlamaCppInferenceEngine( + thresholds=THRESHOLDS, + inference_cfg=_llama_inference_cfg(fallback_to_simulated=False), + ) + + def handler(request: httpx.Request) -> httpx.Response: + raise httpx.ReadTimeout( + "read timed out", + request=request, + ) + + result = _run_async(_predict(engine, handler, str(image_path), GOOD_METRICS)) + assert result["backend"] == "llama_cpp" + assert result["decision"] == "Error" + assert result["code"] == "ERR_MODEL_BACKEND_503" + assert "read timed out" in result["msg"] + + +def test_ollama_async_success(tmp_path: Path): + image_path = tmp_path / "photo.png" + _make_test_image(image_path) + engine = OllamaVisionInferenceEngine( + thresholds=THRESHOLDS, + inference_cfg=_ollama_inference_cfg(), + ) + model_json = json.dumps({"decision": "Blurry", "code": "ERR_IMG_BLUR_101", "msg": "soft"}) + + def handler(request: httpx.Request) -> httpx.Response: + assert request.method == "POST" + assert request.url.path == "/api/generate" + body = json.loads(request.content.decode()) + assert body["model"] == "llava:7b" + assert body["format"] == "json" + assert len(body["images"]) == 1 + return httpx.Response(200, json={"response": model_json}) + + result = _run_async(_predict(engine, handler, str(image_path), GOOD_METRICS)) + assert result["backend"] == "ollama_vision" + assert result["decision"] == "Blurry" + assert result["code"] == "ERR_IMG_BLUR_101" + + +def test_ollama_async_http_error_falls_back(tmp_path: Path): + image_path = tmp_path / "photo.png" + _make_test_image(image_path) + engine = OllamaVisionInferenceEngine( + thresholds=THRESHOLDS, + inference_cfg=_ollama_inference_cfg(), + ) + + def handler(request: httpx.Request) -> httpx.Response: + return httpx.Response(503, json={"error": "model unavailable"}) + + result = _run_async(_predict(engine, handler, str(image_path), GOOD_METRICS)) + assert result["backend"] == "ollama_vision->simulated" + assert "Ollama fallback to simulated inference" in result["msg"] + + +def test_mock_api_async_success_with_result_wrapper(): + engine = MockAPIInferenceEngine( + thresholds=THRESHOLDS, + inference_cfg=_mock_api_inference_cfg(), + ) + + def handler(request: httpx.Request) -> httpx.Response: + assert request.url == httpx.URL("http://localhost:9090/infer") + body = json.loads(request.content.decode()) + assert body["photo_path"] == "shots/a.jpg" + assert body["metrics"] == GOOD_METRICS + return httpx.Response( + 200, + json={ + "result": { + "decision": "Optimal", + "code": "SUCCESS_200", + "msg": "mock ok", + } + }, + ) + + result = _run_async(_predict(engine, handler, "shots/a.jpg", GOOD_METRICS)) + assert result["backend"] == "mock_api" + assert result["decision"] == "Optimal" + assert result["msg"] == "mock ok" + + +def test_mock_api_async_success_flat_body(): + engine = MockAPIInferenceEngine( + thresholds=THRESHOLDS, + inference_cfg=_mock_api_inference_cfg(), + ) + + def handler(request: httpx.Request) -> httpx.Response: + return httpx.Response( + 200, + json={"decision": "Under-exposed", "code": "ERR_IMG_DARK_102", "msg": "dark"}, + ) + + result = _run_async(_predict(engine, handler, "x.jpg", GOOD_METRICS)) + assert result["decision"] == "Under-exposed" + assert result["code"] == "ERR_IMG_DARK_102" + + +def test_mock_api_async_sends_bearer_when_api_key_set(monkeypatch: pytest.MonkeyPatch): + monkeypatch.setenv("MOCK_INFER_API_KEY", "secret-token") + engine = MockAPIInferenceEngine( + thresholds=THRESHOLDS, + inference_cfg=_mock_api_inference_cfg(), + ) + seen_auth: List[Optional[str]] = [] + + def handler(request: httpx.Request) -> httpx.Response: + seen_auth.append(request.headers.get("Authorization")) + return httpx.Response( + 200, + json={"decision": "Optimal", "code": "SUCCESS_200", "msg": "ok"}, + ) + + _run_async(_predict(engine, handler, "x.jpg", GOOD_METRICS)) + assert seen_auth == ["Bearer secret-token"] + + +def test_mock_api_async_connect_timeout_without_fallback(): + engine = MockAPIInferenceEngine( + thresholds=THRESHOLDS, + inference_cfg=_mock_api_inference_cfg(fallback_to_simulated=False), + ) + + def handler(request: httpx.Request) -> httpx.Response: + raise httpx.ConnectTimeout( + "connect timed out", + request=request, + ) + + result = _run_async(_predict(engine, handler, "x.jpg", GOOD_METRICS)) + assert result["backend"] == "mock_api" + assert result["decision"] == "Error" + assert result["code"] == "ERR_MODEL_BACKEND_503" + assert "connect timed out" in result["msg"] + + +def test_mock_api_async_uses_configured_timeout_s(): + """Engine timeout_s is forwarded to httpx (API timeout handling).""" + engine = MockAPIInferenceEngine( + thresholds=THRESHOLDS, + inference_cfg=_mock_api_inference_cfg(mock_api={"timeout_s": 7.5}), + ) + seen_timeouts: List[httpx.Timeout] = [] + + def handler(request: httpx.Request) -> httpx.Response: + return httpx.Response( + 200, + json={"decision": "Optimal", "code": "SUCCESS_200", "msg": "ok"}, + ) + + async def _run_with_capture(): + transport = httpx.MockTransport(handler) + async with httpx.AsyncClient(transport=transport) as client: + original_post = client.post + + async def capturing_post(url, **kwargs): + timeout = kwargs.get("timeout") + if isinstance(timeout, httpx.Timeout): + seen_timeouts.append(timeout) + return await original_post(url, **kwargs) + + client.post = capturing_post # type: ignore[method-assign] + return await predict_quality_async(engine, client, "x.jpg", GOOD_METRICS) + + _run_async(_run_with_capture()) + assert len(seen_timeouts) == 1 + assert seen_timeouts[0].connect == 7.5 + assert seen_timeouts[0].read == 7.5 diff --git a/tests/test_benchmark_evaluator.py b/tests/test_benchmark_evaluator.py new file mode 100644 index 0000000..2c26107 --- /dev/null +++ b/tests/test_benchmark_evaluator.py @@ -0,0 +1,88 @@ +"""Golden-style checks for ranking score, sort order, and release gate.""" + +from eval import benchmark_evaluator as be + + +def test_calculate_quality_score_success_vs_error_code_penalty(): + metrics = {"sharpness": 20.0, "avg_brightness": 130.0} + thresholds = { + "min_sharpness": 20.0, + "min_brightness": 40.0, + "max_brightness": 220.0, + } + ok = {"code": "SUCCESS_200", "decision": "GO"} + bad = {"code": "TIMEOUT", "decision": "NO_GO"} + + high = be.calculate_quality_score(metrics, ok, thresholds) + low = be.calculate_quality_score(metrics, bad, thresholds) + + assert high == 100.0 + assert low == round(100.0 * 0.4, 2) + + +def test_calculate_quality_score_non_dict_metrics_returns_zero(): + assert be.calculate_quality_score(None, {"code": "SUCCESS_200"}, {}) == 0.0 + + +def test_build_rankings_sorts_by_score_latency_then_file(): + thresholds = {"min_sharpness": 10, "min_brightness": 40, "max_brightness": 220} + rows = [ + { + "file": "b.png", + "metrics": {"sharpness": 10, "avg_brightness": 130}, + "decision": {"decision": "GO", "code": "SUCCESS_200"}, + "latency_ms": 20, + "status": "OK", + }, + { + "file": "a.png", + "metrics": {"sharpness": 10, "avg_brightness": 130}, + "decision": {"decision": "GO", "code": "SUCCESS_200"}, + "latency_ms": 10, + "status": "OK", + }, + { + "file": "c.png", + "metrics": {"sharpness": 5, "avg_brightness": 130}, + "decision": {"decision": "REVIEW", "code": "SUCCESS_200"}, + "latency_ms": 5, + "status": "OK", + }, + ] + ranked = be.build_rankings(rows, thresholds) + # Same score -> lower latency first; then lexicographic file name as tie-breaker. + assert [r["file"] for r in ranked] == ["a.png", "b.png", "c.png"] + assert [r["rank"] for r in ranked] == [1, 2, 3] + + +def test_get_release_decision_go_review_no_go_boundaries(): + base_cfg = { + "quality_gate": {"target_pass_rate": 90.0}, + "thresholds": {"timeout_ms": 5000}, + } + + go, msg_go = be.get_release_decision(95.0, 2000.0, base_cfg) + assert go == "GO" + assert "90" in msg_go and "2500" in msg_go + + review, _ = be.get_release_decision(80.0, 2000.0, base_cfg) + assert review == "REVIEW" + + no_go, _ = be.get_release_decision(50.0, 2000.0, base_cfg) + assert no_go == "NO_GO" + + +def test_generate_benchmark_insights_returns_three_items(): + ordered = [ + { + "profile": "dev", + "summary": {"release_decision": "GO", "avg_latency_ms": 100, "target_pass_rate": 85}, + } + ] + profile_outputs = [ + {"profile": "dev", "summary": {"avg_latency_ms": 100, "target_pass_rate": 85}}, + {"profile": "strict", "summary": {"avg_latency_ms": 200, "target_pass_rate": 99}}, + ] + insights = be.generate_benchmark_insights(profile_outputs, ordered) + assert len(insights) == 3 + assert all("trade_off" in item for item in insights) diff --git a/tests/test_image_validator.py b/tests/test_image_validator.py new file mode 100644 index 0000000..4bdf09b --- /dev/null +++ b/tests/test_image_validator.py @@ -0,0 +1,55 @@ +"""Exposure histogram rules on synthetic grayscale images (OpenCV read path).""" + +from pathlib import Path + +import numpy as np +from PIL import Image + +from engine.image_validator import ImageQualityValidator + + +def _save_gray(path: Path, value: int) -> None: + Image.new("L", (64, 64), color=value).save(path) + + +def test_analyze_exposure_missing_file(tmp_path): + v = ImageQualityValidator() + out = v.analyze_exposure(str(tmp_path / "missing.png")) + assert out == "Error: Image not found" + + +def test_analyze_exposure_pass_mid_gray(tmp_path): + path = tmp_path / "mid.png" + _save_gray(path, 128) + v = ImageQualityValidator(brightness_threshold=0.7, dark_threshold=0.7) + result = v.analyze_exposure(str(path)) + assert result["verdict"] == "Pass" + assert result["dark_ratio"] < 0.7 + assert result["bright_ratio"] < 0.7 + + +def test_analyze_exposure_fail_too_dark(tmp_path): + path = tmp_path / "dark.png" + _save_gray(path, 0) + v = ImageQualityValidator(dark_threshold=0.5) + result = v.analyze_exposure(str(path)) + assert result["verdict"] == "Fail: Too Dark" + + +def test_analyze_exposure_fail_overexposed(tmp_path): + path = tmp_path / "bright.png" + _save_gray(path, 255) + v = ImageQualityValidator(brightness_threshold=0.5) + result = v.analyze_exposure(str(path)) + assert result["verdict"] == "Fail: Overexposed" + + +def test_histogram_ratios_are_normalized(tmp_path): + """Mass in high bins should dominate bright_ratio (sanity on OpenCV histogram).""" + path = tmp_path / "bright_strip.png" + arr = np.zeros((32, 32), dtype=np.uint8) + arr[:, 16:] = 250 + Image.fromarray(arr, mode="L").save(path) + v = ImageQualityValidator(brightness_threshold=0.2) + result = v.analyze_exposure(str(path)) + assert result["bright_ratio"] >= 0.45 diff --git a/tests/test_inference_adapter.py b/tests/test_inference_adapter.py index b7486f3..3fc0a22 100644 --- a/tests/test_inference_adapter.py +++ b/tests/test_inference_adapter.py @@ -1,4 +1,5 @@ from models.inference_adapter import _normalize_result +from models.contracts import InferenceOutput def test_normalize_result_handles_invalid_payload(): @@ -17,3 +18,13 @@ def test_normalize_result_parses_confidence(): assert normalized["code"] == "SUCCESS_200" assert normalized["msg"] == "ok" assert normalized["confidence"] == 0.88 + + +def test_inference_output_from_payload_preserves_backend(): + output = InferenceOutput.from_payload( + {"decision": "Optimal", "code": "SUCCESS_200", "msg": "ok"}, + default_msg="fallback", + backend="mock_api", + ) + assert output.backend == "mock_api" + assert output.to_dict()["backend"] == "mock_api" diff --git a/tests/test_log_analyzer.py b/tests/test_log_analyzer.py new file mode 100644 index 0000000..89e7073 --- /dev/null +++ b/tests/test_log_analyzer.py @@ -0,0 +1,20 @@ +"""Sliding-window stability with bounded error tolerance (golden strings).""" + +from eval.log_analyzer import LogAnalyzer + + +def test_find_max_stable_sequence_k0_all_success(): + assert LogAnalyzer(0).find_max_stable_sequence("SSSS") == 4 + + +def test_find_max_stable_sequence_k1_full_string_with_one_error(): + # "SSSESSS" — one E inside; k=1 allows entire window. + assert LogAnalyzer(1).find_max_stable_sequence("SSSESSS") == 7 + + +def test_find_max_stable_sequence_k0_breaks_at_each_error(): + assert LogAnalyzer(0).find_max_stable_sequence("SSEESS") == 2 + + +def test_find_max_stable_sequence_empty(): + assert LogAnalyzer(1).find_max_stable_sequence("") == 0 diff --git a/tests/test_loopback_guardrails.py b/tests/test_loopback_guardrails.py index 1754c85..50722bd 100644 --- a/tests/test_loopback_guardrails.py +++ b/tests/test_loopback_guardrails.py @@ -1,4 +1,8 @@ -from ai_quality_agent import classify_loopback_signal, decide_loopback_action +from ai_quality_agent import ( + classify_loopback_signal, + decide_loopback_action, + plan_next_action, +) def test_classify_loopback_signal_blurry(): @@ -39,3 +43,15 @@ def test_decide_loopback_action_for_blurry(): ) assert action == "sharpen" assert reason == "retry_scheduled" + + +def test_plan_next_action_reports_rationale(): + plan = plan_next_action( + signal="under", + engine_metrics={"avg_brightness": 10.0, "sharpness": 30.0}, + thresholds_cfg={"min_brightness": 40.0, "max_brightness": 220.0, "min_sharpness": 20.0}, + loopback_guard_cfg={"overexposure_stop_ratio": 0.95}, + ) + assert plan.action == "brighten" + assert plan.stop_reason == "retry_scheduled" + assert "under-exposed" in plan.rationale diff --git a/tests/test_loopback_planner.py b/tests/test_loopback_planner.py new file mode 100644 index 0000000..a43c785 --- /dev/null +++ b/tests/test_loopback_planner.py @@ -0,0 +1,52 @@ +from agent.loopback_planner import LLMLoopbackPlanner, create_loopback_planner + + +def test_create_loopback_planner_default_simulated(): + planner = create_loopback_planner({"runtime": {}}) + plan = planner.plan( + signal="under", + engine_metrics={"avg_brightness": 10.0, "sharpness": 25.0}, + thresholds_cfg={"min_brightness": 40.0, "max_brightness": 220.0, "min_sharpness": 20.0}, + loopback_guard_cfg={}, + attempt_history=[], + ) + assert plan.action == "brighten" + assert plan.stop_reason == "retry_scheduled" + + +def test_llm_loopback_planner_falls_back_on_network_error(): + planner = LLMLoopbackPlanner( + planner_cfg={"host": "http://127.0.0.1:9", "timeout_s": 0.01}, + fallback_planner=create_loopback_planner({"runtime": {}}), + ) + plan = planner.plan( + signal="blurry", + engine_metrics={"avg_brightness": 120.0, "sharpness": 5.0}, + thresholds_cfg={"min_brightness": 40.0, "max_brightness": 220.0, "min_sharpness": 20.0}, + loopback_guard_cfg={}, + attempt_history=[], + ) + assert plan.action == "sharpen" + assert plan.stop_reason == "retry_scheduled" + assert plan.fallback_used is True + assert plan.planner_backend == "llm->simulated" + + +def test_create_loopback_planner_llm_health_check_fail_fast(): + try: + create_loopback_planner( + { + "runtime": { + "loopback_planner": { + "mode": "llm", + "require_healthy_on_startup": True, + "llm": {"host": "http://127.0.0.1:9", "timeout_s": 0.01}, + } + } + } + ) + except RuntimeError as exc: + assert "not reachable" in str(exc) + else: + raise AssertionError("Expected RuntimeError for unreachable llm planner endpoint") + diff --git a/tests/test_metrics_pool.py b/tests/test_metrics_pool.py new file mode 100644 index 0000000..6f550be --- /dev/null +++ b/tests/test_metrics_pool.py @@ -0,0 +1,16 @@ +from PIL import Image + +from engine.vision_math import calculate_metrics +from util.metrics_pool import MetricsProcessPool + + +def test_metrics_process_pool_matches_inline(tmp_path): + image_path = tmp_path / "sample.png" + Image.new("L", (64, 64), color=100).save(image_path) + + expected = calculate_metrics(str(image_path)) + + with MetricsProcessPool(max_workers=2) as pool: + actual = pool.calculate(str(image_path)) + + assert actual == expected diff --git a/tests/test_runtime_overrides.py b/tests/test_runtime_overrides.py new file mode 100644 index 0000000..d3c7577 --- /dev/null +++ b/tests/test_runtime_overrides.py @@ -0,0 +1,44 @@ +import ai_quality_agent as qa + + +def test_apply_runtime_overrides_updates_backend_and_planner(): + config = { + "model_settings": {"inference": {"backend": "simulated"}}, + "runtime": {"loopback_planner": {"mode": "simulated"}}, + } + source = qa._apply_runtime_overrides( + config, + "BASE", + inference_backend_override="mock_api", + loopback_planner_override="llm", + ) + assert config["model_settings"]["inference"]["backend"] == "mock_api" + assert config["runtime"]["loopback_planner"]["mode"] == "llm" + assert "backend=mock_api" in source + assert "loopback_planner=llm" in source + + +def test_apply_runtime_overrides_updates_planner_llm_fields(): + config = {"runtime": {"loopback_planner": {"mode": "llm", "llm": {}}}} + source = qa._apply_runtime_overrides( + config, + "BASE", + planner_timeout_s_override=7.5, + planner_model_override="llama-planner-q4", + ) + assert config["runtime"]["loopback_planner"]["llm"]["timeout_s"] == 7.5 + assert config["runtime"]["loopback_planner"]["llm"]["model"] == "llama-planner-q4" + assert "planner_timeout_s=7.5" in source + assert "planner_model=llama-planner-q4" in source + + +def test_apply_runtime_overrides_updates_planner_health_policy(): + config = {"runtime": {"loopback_planner": {"mode": "llm"}}} + source = qa._apply_runtime_overrides( + config, + "BASE", + planner_require_healthy_override=False, + ) + assert config["runtime"]["loopback_planner"]["require_healthy_on_startup"] is False + assert "planner_require_healthy=False" in source + diff --git a/tests/test_vision_math.py b/tests/test_vision_math.py new file mode 100644 index 0000000..c120f27 --- /dev/null +++ b/tests/test_vision_math.py @@ -0,0 +1,20 @@ +"""Real-image metric extraction (Pillow path); missing file and bad path edges.""" + +from PIL import Image + +from engine.vision_math import calculate_metrics + + +def test_calculate_metrics_missing_file(tmp_path): + missing = tmp_path / "does_not_exist.png" + assert calculate_metrics(str(missing)) is None + + +def test_calculate_metrics_uniform_image(tmp_path): + path = tmp_path / "gray.png" + Image.new("L", (64, 64), color=100).save(path) + out = calculate_metrics(str(path)) + assert out is not None + assert out["avg_brightness"] == 100.0 + assert out["max_brightness"] == 100 + assert out["sharpness"] == 0.0