From 5ec597a40efe60df7e78165e7c1dd4bc00bf4f2f Mon Sep 17 00:00:00 2001 From: spalen0 Date: Thu, 2 Jul 2026 13:43:59 +0200 Subject: [PATCH] feat: add monitoring.yaml as source of truth for protocol cards - Add monitoring.yaml describing what each protocol monitors, cadence, scripts, and disabled status. - Add utils/monitoring_config.py loader/validator and API endpoint GET /v1/monitoring. - Add tests enforcing sync with automation/jobs.yaml. - Update CLAUDE.md, CONTRIBUTING.md, README.md, and deploy/alerts-api.md. --- CLAUDE.md | 7 + CONTRIBUTING.md | 7 +- README.md | 2 +- api/server.py | 4 + deploy/alerts-api.md | 35 +++ monitoring.yaml | 446 ++++++++++++++++++++++++++++++++ tests/test_monitoring_config.py | 158 +++++++++++ utils/monitoring_config.py | 182 +++++++++++++ 8 files changed, 837 insertions(+), 4 deletions(-) create mode 100644 monitoring.yaml create mode 100644 tests/test_monitoring_config.py create mode 100644 utils/monitoring_config.py diff --git a/CLAUDE.md b/CLAUDE.md index ffa2c075..a2e8d978 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -37,6 +37,13 @@ - **Logging**: Use structured logging instead of print statements - **Testing**: Write unit tests for utility functions and integration tests for protocols +## Monitoring Metadata + +- `monitoring.yaml` at the repo root is the source of truth for the protocol cards on https://curation.yearn.fi/monitoring/. +- When adding, removing, or changing a monitor, update `monitoring.yaml` and the protocol `README.md`. +- The alerts API exposes this metadata at `GET /v1/monitoring` (see `api/server.py`). +- `tests/test_monitoring_config.py` validates that every enabled protocol task in `automation/jobs.yaml` has a `monitoring.yaml` entry and that every task referenced there is scheduled. + ## Best Practices - Create custom exception types for domain-specific errors - Extract repeated patterns into utility functions. Try to keep utility functions small and focused on a single task and store in utils folder. diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 28e916a0..dfc1e76c 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -149,9 +149,10 @@ write_last_value_to_file(cache_filename, "MY_KEY", new_value) 1. Create `protocols/protocol-name/main.py` following the pattern above, with a `main()` function and an `if __name__ == "__main__":` block that wraps it via `run_with_alert(main, PROTOCOL)` (see [Script Entrypoint](#script-entrypoint)). Reference ABIs by their repo-root-relative path, e.g. `load_abi("protocols/protocol-name/abi/Foo.json")` 2. Add a `protocols/protocol-name/README.md` describing what it monitors -3. No `pyproject.toml` change is needed — packages under `protocols/` are discovered automatically (see `[tool.setuptools.packages.find]`) -4. Add the corresponding `TELEGRAM_BOT_TOKEN_*` and `TELEGRAM_CHAT_ID_*` entries to `.env.example` -5. Register the script in `automation/jobs.yaml` under the right profile if it should run on a schedule +3. Add or update the protocol entry in `monitoring.yaml` so the curation website card stays in sync +4. No `pyproject.toml` change is needed — packages under `protocols/` are discovered automatically (see `[tool.setuptools.packages.find]`) +5. Add the corresponding `TELEGRAM_BOT_TOKEN_*` and `TELEGRAM_CHAT_ID_*` entries to `.env.example` +6. Register the script in `automation/jobs.yaml` under the right profile if it should run on a schedule ## Tests diff --git a/README.md b/README.md index 31b7c644..06f6fb42 100644 --- a/README.md +++ b/README.md @@ -80,7 +80,7 @@ uv run protocols/aave/main.py In production the scripts run on a schedule via supercronic on a VPS, defined in [`automation/jobs.yaml`](./automation/jobs.yaml). See [`deploy/`](./deploy/) — [`install.sh`](./deploy/install.sh) provisions a host and [`runbook.md`](./deploy/runbook.md) covers operations. -The optional read-only alerts API exposes persisted alert history from SQLite. See [`deploy/alerts-api.md`](./deploy/alerts-api.md) for endpoint examples and pagination. +The optional read-only alerts API exposes persisted alert history from SQLite and monitoring card metadata. See [`deploy/alerts-api.md`](./deploy/alerts-api.md) for endpoint examples and pagination, or `GET /v1/monitoring` for the protocol card data used by the curation website. ## Code Style diff --git a/api/server.py b/api/server.py index 2b82624b..31e7668b 100644 --- a/api/server.py +++ b/api/server.py @@ -9,6 +9,7 @@ from automation.config import JobsConfig, load_jobs_config from utils.logger import get_logger +from utils.monitoring_config import load_monitoring_config, monitoring_to_json from utils.store import AlertEvent, get_alert, normalize_timestamp, query_alerts logger = get_logger("api.server") @@ -160,6 +161,9 @@ def do_GET(self) -> None: if parsed.path == "/v1/protocols": write_json(self, 200, protocols_to_json(load_jobs_config())) return + if parsed.path == "/v1/monitoring": + write_json(self, 200, monitoring_to_json(load_monitoring_config())) + return if parsed.path == "/v1/alerts": alert_query = parse_alert_query(parsed.query) rows = query_alerts( diff --git a/deploy/alerts-api.md b/deploy/alerts-api.md index e06ea36b..cb7758e1 100644 --- a/deploy/alerts-api.md +++ b/deploy/alerts-api.md @@ -130,6 +130,41 @@ Example response: } ``` +### `GET /v1/monitoring` + +Returns the structured protocol card metadata from `monitoring.yaml`, used by +the curation website to render what each protocol monitors. + +```sh +curl http://127.0.0.1:8923/v1/monitoring +``` + +Example response: + +```json +{ + "version": "1.0", + "data": [ + { + "slug": "3jane", + "display_name": "3Jane", + "description": "USD3/sUSD3 credit market monitoring on Mainnet", + "cadence": "Hourly", + "monitor_count": 11, + "disabled": false, + "tasks": ["protocols/3jane/main.py"], + "monitors": [ + { + "name": "Price Per Share", + "description": "USD3 PPS decrease (any drop vs cached prior, CRITICAL) and sUSD3 PPS decrease (HIGH)" + } + ] + } + ], + "count": 1 +} +``` + ## Delivery Status An alert row means a monitor generated an alert. Telegram delivery is tracked diff --git a/monitoring.yaml b/monitoring.yaml new file mode 100644 index 00000000..f75c50d0 --- /dev/null +++ b/monitoring.yaml @@ -0,0 +1,446 @@ +# Monitoring card metadata for https://curation.yearn.fi/monitoring/ +# +# This file is the source of truth for what each protocol card displays. +# It is rebuilt from the actual repo code and READMEs, not from the website. +# When you add, remove, or change a monitor, update this file and the +# protocol README so the website stays in sync. +# +# Consumed by: +# - the alerts API at GET /v1/monitoring +# - the curation website (directly or via the API) +# +# Keep in sync with: +# - automation/jobs.yaml (scheduled tasks) +# - protocols//README.md (protocol-specific docs) + +version: "1.0" + +protocols: + 3jane: + display_name: "3Jane" + description: "USD3/sUSD3 credit market monitoring on Mainnet" + cadence: "Hourly" + tasks: + - protocols/3jane/main.py + monitors: + - name: "Price Per Share" + description: "USD3 PPS decrease (any drop vs cached prior, CRITICAL) and sUSD3 PPS decrease (HIGH)" + - name: "TVL" + description: "Absolute TVL change >=15% for USD3 or sUSD3" + - name: "Junior Buffer Ratio" + description: "sUSD3 backing <15% of deployed credit" + - name: "USD3 Overcollateralization" + description: "OC <111% (HIGH) or <106% (CRITICAL)" + - name: "Insurance Fund Outflow" + description: "waUSDC outflow >=$50k since prior run" + - name: "Withdraw Liquidity" + description: "USD3 availableWithdrawLimit < $4M" + - name: "Vault Shutdown" + description: "Alert-once when isShutdown() transitions to true" + - name: "Debt Cap" + description: "Any change to ProtocolConfig.getDebtCap()" + - name: "Nominal sUSD3 Backing Floor" + description: "Any floor change; alert-once when floor > sUSD3 backing" + - name: "Protocol Pause" + description: "Alert-once when ProtocolConfig IS_PAUSED flips true" + - name: "Timelock" + description: "CallScheduled events from 24h and 7-day TimelockControllers via Envio" + + aave: + display_name: "Aave V3" + description: "Aave V3 market and governance monitoring" + cadence: "Hourly" + tasks: + - protocols/aave/main.py + - protocols/aave/proposals.py + monitors: + - name: "Market Utilization" + description: "Utilization rate >99% on Mainnet" + - name: "Governance Proposals" + description: "Queued proposals via Aave governance cache API" + - name: "Timelock" + description: "Aave Governance V3 events via Envio timelock monitor" + - name: "Safe Multisigs" + description: "Protocol Guardian and Governance Guardian Safe queues" + + apyusd: + display_name: "APYUSD" + description: "apxUSD rate oracle monitoring on Mainnet" + cadence: "Hourly" + tasks: + - protocols/apyusd/main.py + monitors: + - name: "Rate Oracle" + description: "apxUSD rate change >=10% vs cached prior value" + + cap: + display_name: "CAP" + description: "CAP cUSD liquidity and governance monitoring on Mainnet" + cadence: "Daily" + tasks: + - protocols/cap/liquidity.py + monitors: + - name: "Withdrawable Liquidity" + description: "Total withdrawable liquidity across cUSD assets < $15M" + - name: "Large cUSD Mints" + description: "totalSupply delta >5% vs cached prior" + - name: "Timelock" + description: "CAP TimelockController events via Envio" + - name: "Safe Multisig" + description: "Cap Money Multisig queue" + + compound: + display_name: "Compound V3" + description: "Compound V3 market, collateral, and governance monitoring" + cadence: "Hourly / Daily" + tasks: + - protocols/compound/main.py + - protocols/compound/proposals.py + monitors: + - name: "Market Utilization" + description: "Utilization >99% on Mainnet" + - name: "On-Chain Collateral Risk" + description: "Per-asset allocation ratios vs risk-adjusted thresholds; weighted total risk level" + - name: "Borrow/Supply Ratio" + description: "Borrow/supply ratio >95%" + - name: "Bad Debt" + description: "Negative reserves (getReserves() < 0)" + - name: "Unknown Collateral" + description: "Collateral assets not mapped in SUPPLY_ASSETS_DICT" + - name: "Governance Proposals" + description: "Queued proposals via Tally API" + - name: "Timelock" + description: "Compound Timelock events via Envio" + + ethena: + display_name: "Ethena" + description: "USDe backing and supply monitoring" + cadence: "Daily" + tasks: + - protocols/ethena/ethena.py + monitors: + - name: "USDe Backing Ratio" + description: "Collateral / supply < 1.0 via Ethena transparency API (CRITICAL)" + - name: "Data Freshness" + description: "Collateral, chain, or reserve data older than 12 hours" + + euler: + display_name: "Euler" + description: "Euler market risk and governance monitoring" + cadence: "Hourly" + disabled: true + tasks: + - protocols/euler/markets.py + monitors: + - name: "Debt/Supply Ratio" + description: "Gauntlet dashboard debt/supply ratio >60%" + - name: "Value at Risk" + description: "VaR >1% of borrow amount" + - name: "Liquidation at Risk" + description: "LaR >5% of borrow amount" + - name: "Vault Risk Levels" + description: "Computed risk vs maximum threshold" + - name: "Vault Allocation Ratios" + description: "Risk-adjusted allocation thresholds" + - name: "Safe Multisig" + description: "4/7 multisig transaction queue" + + fluid: + display_name: "Fluid" + description: "Fluid governance and timelock monitoring" + cadence: "Hourly" + tasks: + - protocols/fluid/proposals.py + monitors: + - name: "Governance Proposals" + description: "Queued proposals via Fluid API" + - name: "Timelock" + description: "InstaTimelock events via Envio" + + infinifi: + display_name: "Infinifi" + description: "Infinifi iUSD reserves, backing, and farm monitoring" + cadence: "Hourly" + tasks: + - protocols/infinifi/main.py + monitors: + - name: "Liquid Reserves" + description: "Liquid reserves < $8M" + - name: "Backing Per iUSD" + description: "totalTVL / iUSD supply < 0.999" + - name: "Redemption Pressure" + description: "Pending redemptions / liquid reserves >80%" + - name: "Large iUSD Mints" + description: "totalSupply delta >5% vs cached prior" + - name: "Farm Allocation Shift" + description: "Farm allocation ratio change >30% vs prior run (farms <1% of TVL excluded)" + - name: "Farm Activation" + description: "Farm previously at 0 ratio moves above 3% of TVL" + - name: "Junior TVL Coverage" + description: "Junior TVL <50% of risky farm exposure" + - name: "Timelock" + description: "Longtimelock and Shorttimelock events via Envio" + + lido: + display_name: "Lido" + description: "Lido stETH peg and governance monitoring" + cadence: "Hourly" + tasks: + - protocols/lido/steth/main.py + monitors: + - name: "stETH/ETH Exchange Rate" + description: "Curve stETH/ETH pool rate deviates >0.1% from Lido validator rate" + - name: "Timelock" + description: "Lido Timelock StartVote events via Envio" + - name: "Safe Multisigs" + description: "Emergency Brakes and GateSeal Committee Safe queues" + + lrt-pegs: + display_name: "LRT Pegs" + description: "Liquid restaking token peg and governance monitoring" + cadence: "Hourly" + tasks: + - protocols/lrt-pegs/curve/main.py + - protocols/lrt-pegs/fluid/main.py + - protocols/lrt-pegs/origin_protocol.py + monitors: + - name: "Curve Pool Balance Ratios" + description: "ETH+/WETH, OETH/ETH, weETH-WETH, stETH/ETH pool ratios >90%" + - name: "Fluid Pool Balance Ratio" + description: "weETH/ETH Fluid pool ratio >90% or balance <1 token" + - name: "Origin Protocol" + description: "Wrapped OETH redeem value decrease and vault backing ratio <1 on Mainnet and Base" + - name: "Timelock" + description: "Lombard, EtherFi, KelpDAO, and superOETH timelock events via Envio" + - name: "Safe Multisig" + description: "LBTC boring vault, Origin admin, and tBTC bridge owner Safe queues" + + maker: + display_name: "Maker DAO" + description: "Maker/Sky governance and PSM monitoring" + cadence: "Hourly" + tasks: + - protocols/maker/proposals.py + monitors: + - name: "Executive Proposals" + description: "New scheduled (not cast) executive proposals from Sky governance portal API" + - name: "DSPause / Timelock" + description: "Tenderly alert on plot() in DSPause (16-hour execution delay)" + - name: "PSM USDC Balance" + description: "Tenderly alert when PSM USDC balance < 2B" + + maple: + display_name: "Maple Finance" + description: "Maple syrupUSDC pool monitoring" + cadence: "Hourly" + tasks: + - protocols/maple/main.py + monitors: + - name: "Price Per Share" + description: "Any PPS decrease" + - name: "TVL" + description: "Absolute TVL change >=15%" + - name: "Unrealized Losses" + description: "Any non-zero on-chain unrealized losses; >=0.5% of pool assets via subgraph" + - name: "Withdrawal Queue" + description: "Pending exit value >1% of TVL" + - name: "Pool Liquidity" + description: "Pending withdrawals exceed pool cash" + - name: "Collateralization Ratio" + description: "syrupGlobals collateralRatio <135%" + - name: "Proof of Reserves Divergence" + description: "syrupGlobals collateralValue exceeds PoR attestation by >10%" + - name: "Unknown Collateral" + description: "Collateral assets not in risk mapping" + - name: "Pool Delegate Cover" + description: "Cover balance drops to $0 or decreases vs cached prior" + - name: "Timelock" + description: "Maple GovernorTimelock events via Envio" + - name: "Safe Multisig" + description: "Maple DAO Multisig queue" + + morpho: + display_name: "Morpho" + description: "Morpho market, vault, and governance monitoring" + cadence: "Hourly / Daily" + tasks: + - protocols/morpho/markets.py + - protocols/morpho/markets_v2.py + - protocols/morpho/governance.py + - protocols/morpho/governance_v2.py + monitors: + - name: "Bad Debt Ratio" + description: "Bad debt >0.5% of borrowed TVL in v1 and v2 vault markets" + - name: "Market Allocation Ratios" + description: "Per-market allocation vs risk-adjusted thresholds" + - name: "Vault Risk Levels" + description: "Weighted total risk score vs vault threshold" + - name: "Low Liquidity" + description: "Vault liquidity <1% of assets; YV-collateral unwind liquidity vs collateral at risk" + - name: "V1 Governance" + description: "Pending supply caps, market removals, timelock changes, guardian changes" + - name: "V2 Governance" + description: "Pending timelocked operations, owner/curator changes, sentinel/allocator/adapter changes" + - name: "V2 Unmonitored Vault Wrappers" + description: "V2 vault wrapping a v1 vault not in markets.py:VAULTS_BY_CHAIN" + - name: "Safe Multisig" + description: "Morpho eUSDe predeposit vault owner Safe queue" + + pendle: + display_name: "Pendle" + description: "Pendle PT ratio and governance monitoring" + cadence: "Hourly" + disabled: true + tasks: + - protocols/pendle/main.py + monitors: + - name: "PT to Asset Ratio" + description: "Active vault market PT ratio <0.95" + - name: "Safe Multisig" + description: "2/4 governance multisig queues on Mainnet and Arbitrum" + + rtoken: + display_name: "RToken (ETH+)" + description: "ETH+ RToken collateral and redemption monitoring" + cadence: "Hourly" + tasks: + - protocols/rtoken/monitor_rtoken.py + monitors: + - name: "Collateral Coverage" + description: "basketsNeeded / totalSupply < 105% on Mainnet; <103% on Base (bsdETH disabled)" + - name: "StRSR Exchange Rate" + description: "StRSR exchange rate falls below initial cached value" + - name: "Redemption Available" + description: "redemptionAvailable < 1000 ETH on Mainnet; <600 ETH on Base" + - name: "Timelock" + description: "ETH+ Timelock events via Envio" + + safe: + display_name: "Safe Multisigs" + description: "Cross-protocol Safe multisig transaction monitoring" + cadence: "Every 10 minutes" + tasks: + - protocols/safe/main.py + monitors: + - name: "Queued Transactions" + description: "Pending transactions across all monitored protocol Safe multisigs" + - name: "Unexpected Proposers" + description: "Yearn multisig txs proposed by addresses outside known bot/EOA set" + + silo: + display_name: "Silo" + description: "Silo bad debt and governance monitoring" + cadence: "Hourly" + disabled: true + tasks: [] + monitors: + - name: "Bad Debt" + description: "Positions with riskFactor > 1" + - name: "Timelock" + description: "Contract monitoring (2-day minimum delay)" + - name: "Safe Multisig" + description: "3/6 multisig monitoring on Mainnet, Optimism, and Arbitrum" + + stables: + display_name: "Stablecoins" + description: "Cross-protocol stablecoin peg, oracle, and large-transfer monitoring" + cadence: "Hourly / Every 10 minutes" + tasks: + - protocols/stables/main.py + - protocols/stables/dune_large_transfers.py + - protocols/stables/oracles.py + monitors: + - name: "Stablecoin Prices" + description: "Depeg alerts via DeFiLlama (USDe/sUSDe/iUSD/syrupUSDC/syrupUSDT/USTB thresholds)" + - name: "Large Transfers" + description: "Dune-backed large transfers >=$5M for cUSD and iUSD" + - name: "Oracle Health" + description: "Chainlink staleness, round health, peg deviation, oracle-market divergence; rate-oracle monotonicity/delta" + + strata: + display_name: "Strata" + description: "Strata srUSDe/jrUSDe tranche monitoring" + cadence: "Daily" + tasks: + - protocols/strata/main.py + monitors: + - name: "srUSDe Exchange Rate" + description: "srUSDe convertToAssets decrease vs cached prior" + - name: "Senior Coverage Ratio" + description: "Senior coverage <105%" + - name: "Junior Tranche Drain" + description: "jrUSDe totalAssets drop >=15%" + - name: "Strategy Balance" + description: "sUSDe strategy balance / total deposits drops >=20%" + - name: "TVL" + description: "Total deposits change >=15%" + - name: "sUSDe Dependency" + description: "sUSDe vault rate monotonicity and cooldown duration changes" + - name: "Timelock" + description: "48h and 24h timelock events via Envio" + - name: "Safe Multisig" + description: "Strata Admin Multisig queue" + + timelock: + display_name: "Timelock Alerts" + description: "Cross-protocol timelock event monitoring" + cadence: "Hourly" + tasks: + - protocols/timelock/timelock_alerts.py + monitors: + - name: "Timelock Events" + description: "CallScheduled/ProposalQueued/QueueTransaction/StartVote/ProposalScheduled events across monitored timelocks via Envio" + + ustb: + display_name: "Superstate USTB" + description: "Superstate USTB fund and oracle monitoring" + cadence: "Hourly" + tasks: + - protocols/ustb/main.py + monitors: + - name: "NAV/Share Monotonicity" + description: "Continuous oracle checkpoint NAV decrease" + - name: "Oracle Divergence" + description: "Continuous Oracle vs Chainlink differ by >0.5%" + - name: "Supply Changes" + description: "Total supply change >10% vs prior run" + - name: "Oracle Staleness" + description: "Latest checkpoint effectiveAt >4 days old" + - name: "Stablecoin Price" + description: "USTB price < $10.50 via stables monitor" + + usdai: + display_name: "USDAI" + description: "USDAI collateral, mint ratio, and loan monitoring on Arbitrum" + cadence: "Hourly" + tasks: + - protocols/usdai/main.py + - protocols/usdai/large_mints.py + monitors: + - name: "Backing Invariant" + description: "totalSupply + bridgedSupply - PYUSD balance >= 100 USDai" + - name: "Loan Activity" + description: "Total verified principal change >1% vs prior run" + - name: "Legacy Loan Expiry" + description: "Hardcoded NVIDIA H200s legacy loan past 2028-07-27" + - name: "Large Mints" + description: "totalSupply delta >5% via large_mints.py" + - name: "Safe Multisigs" + description: "USDai and sUSDai Admin Safe queues" + + yearn: + display_name: "Yearn" + description: "Yearn vault flow and timelock monitoring" + cadence: "Hourly / Daily" + tasks: + - protocols/yearn/alert_large_flows.py + - protocols/yearn/check_timelock_delay.py + monitors: + - name: "Large Flows" + description: "Deposit/withdrawal flows >=$1M USD (or 10% of vault totalSupply fallback for unpriced tokens)" + - name: "Timelock Delay" + description: "Yearn TimelockController getMinDelay() < 7 days across Mainnet, Base, Arbitrum, Polygon, Optimism, Katana" + - name: "Timelock Events" + description: "Yearn TimelockController events via Envio" + - name: "Safe Multisigs" + description: "Yearn multisig queues (yChad, Strategist, SAM, Curation) and unexpected-proposer escalation" diff --git a/tests/test_monitoring_config.py b/tests/test_monitoring_config.py new file mode 100644 index 00000000..c28f0041 --- /dev/null +++ b/tests/test_monitoring_config.py @@ -0,0 +1,158 @@ +from __future__ import annotations + +import json +import threading +from http.client import HTTPConnection +from http.server import ThreadingHTTPServer +from pathlib import Path + +import pytest + +from api.server import AlertsHandler +from automation.config import JobsConfig, Profile, Task, load_jobs_config +from utils.monitoring_config import ( + MonitoringConfigError, + load_monitoring_config, + monitoring_to_json, + protocol_to_json, +) + + +def test_load_monitoring_config_parses_root_file(): + config = load_monitoring_config() + assert config.version + assert config.protocols + assert "3jane" in config.protocols + + +def test_protocol_to_json_structure(): + config = load_monitoring_config() + protocol = config.protocols["3jane"] + data = protocol_to_json(protocol) + assert data["slug"] == "3jane" + assert data["display_name"] == "3Jane" + assert data["cadence"] == "Hourly" + assert data["monitor_count"] == len(data["monitors"]) + assert all("name" in m and "description" in m for m in data["monitors"]) + + +def test_monitoring_to_json_is_sorted_and_counted(): + config = load_monitoring_config() + data = monitoring_to_json(config) + assert data["version"] == config.version + assert data["count"] == len(config.protocols) + slugs = [p["slug"] for p in data["data"]] + assert slugs == sorted(slugs) + + +def test_invalid_severity_rejected(): + from utils.monitoring_config import Monitor + + with pytest.raises(MonitoringConfigError, match="invalid severity"): + Monitor(name="x", description="y", severity="WRONG") + + +def test_missing_required_keys_rejected(tmp_path): + path = tmp_path / "monitoring.yaml" + path.write_text("version: '1.0'\nprotocols:\n bad:\n display_name: Bad\n", encoding="utf-8") + with pytest.raises(MonitoringConfigError, match="missing keys"): + load_monitoring_config(path) + + +def _protocol_from_script(script: str) -> str | None: + parts = script.split("/") + if len(parts) < 2 or parts[0] != "protocols" or not parts[1]: + return None + return parts[1] + + +def test_all_jobs_yaml_protocols_have_monitoring_metadata(): + """Every enabled protocol task in jobs.yaml must have a monitoring.yaml entry.""" + jobs = load_jobs_config() + monitoring = load_monitoring_config() + + protocols_with_tasks: set[str] = set() + for profile in jobs.enabled_profiles: + for task in profile.enabled_tasks: + protocol = _protocol_from_script(task.script) + if protocol is not None: + protocols_with_tasks.add(protocol) + + missing = protocols_with_tasks - set(monitoring.protocols) + assert not missing, f"protocols in jobs.yaml without monitoring.yaml entry: {sorted(missing)}" + + +def test_monitoring_tasks_exist_in_jobs_yaml(): + """Every task referenced in an enabled monitoring.yaml entry must be scheduled in jobs.yaml.""" + jobs = load_jobs_config() + monitoring = load_monitoring_config() + + scheduled_tasks: set[str] = set() + for profile in jobs.enabled_profiles: + for task in profile.enabled_tasks: + scheduled_tasks.add(task.script) + + errors: list[str] = [] + for protocol in monitoring.protocols.values(): + if protocol.disabled: + continue + for task in protocol.tasks: + if task not in scheduled_tasks: + errors.append(f"{protocol.slug}: {task}") + + assert not errors, f"monitoring.yaml tasks not found in jobs.yaml: {errors}" + + +def _request(server: ThreadingHTTPServer, method: str, path: str) -> tuple[int, dict]: + host, port = server.server_address + conn = HTTPConnection(host, port) + try: + conn.request(method, path) + response = conn.getresponse() + body = response.read() + return response.status, json.loads(body.decode()) + finally: + conn.close() + + +def _server(): + server = ThreadingHTTPServer(("127.0.0.1", 0), AlertsHandler) + thread = threading.Thread(target=server.serve_forever, daemon=True) + thread.start() + return server + + +def test_disabled_protocols_included(): + config = load_monitoring_config() + disabled = {slug for slug, p in config.protocols.items() if p.disabled} + expected = {"euler", "pendle", "silo"} + assert expected <= disabled, f"expected disabled protocols {expected} to be present, got {disabled}" + + for slug in expected: + data = protocol_to_json(config.protocols[slug]) + assert data["disabled"] is True + + +def test_api_monitoring_route(monkeypatch, tmp_path): + config = JobsConfig( + profiles={ + "hourly": Profile( + name="hourly", + cron="5 * * * *", + tasks=[Task(name="demo", script="protocols/demo/main.py")], + ), + }, + path=Path("jobs.yaml"), + ) + monkeypatch.setattr("api.server.load_jobs_config", lambda: config) + server = _server() + try: + status, body = _request(server, "GET", "/v1/monitoring") + finally: + server.shutdown() + + assert status == 200 + assert body["version"] + assert isinstance(body["data"], list) + assert body["count"] == len(body["data"]) + assert any(p["slug"] == "3jane" for p in body["data"]) diff --git a/utils/monitoring_config.py b/utils/monitoring_config.py new file mode 100644 index 00000000..2dd996bd --- /dev/null +++ b/utils/monitoring_config.py @@ -0,0 +1,182 @@ +"""Load and validate ``monitoring.yaml``. + +``monitoring.yaml`` is the source of truth for the protocol cards shown on the +monitoring website. It describes what each protocol monitors, how often, and +which scripts implement those checks. +""" + +from __future__ import annotations + +import os +from dataclasses import dataclass +from pathlib import Path +from typing import Any + +import yaml + +REPO_ROOT: Path = Path(os.environ.get("REPO_ROOT", Path(__file__).resolve().parent.parent)) +DEFAULT_MONITORING_CONFIG_PATH: Path = REPO_ROOT / "monitoring.yaml" + + +class MonitoringConfigError(ValueError): + """Raised when monitoring.yaml is malformed or violates the schema.""" + + +@dataclass(frozen=True) +class Monitor: + """A single check shown on a protocol card.""" + + name: str + description: str + severity: str | None = None + + def __post_init__(self) -> None: + if self.severity is not None and self.severity not in {"LOW", "MEDIUM", "HIGH", "CRITICAL"}: + raise MonitoringConfigError(f"invalid severity {self.severity!r}") + + +@dataclass(frozen=True) +class ProtocolMonitoring: + """Card metadata for one monitored protocol.""" + + slug: str + display_name: str + description: str + cadence: str + tasks: tuple[str, ...] + monitors: tuple[Monitor, ...] + disabled: bool = False + + @property + def monitor_count(self) -> int: + return len(self.monitors) + + +@dataclass(frozen=True) +class MonitoringConfig: + """Top-level container for monitoring card metadata.""" + + version: str + protocols: dict[str, ProtocolMonitoring] + path: Path + + @property + def sorted_protocols(self) -> list[ProtocolMonitoring]: + return [self.protocols[k] for k in sorted(self.protocols)] + + +def _require_string(value: Any, where: str) -> str: + if not isinstance(value, str) or not value: + raise MonitoringConfigError(f"{where} must be a non-empty string") + return value + + +def _parse_monitor(protocol: str, idx: int, body: Any) -> Monitor: + if not isinstance(body, dict): + raise MonitoringConfigError(f"protocol `{protocol}` monitor #{idx} must be a mapping") + for key in ("name", "description"): + if key not in body: + raise MonitoringConfigError(f"protocol `{protocol}` monitor #{idx} missing `{key}`") + name = _require_string(body["name"], f"protocol `{protocol}` monitor #{idx} name") + description = _require_string(body["description"], f"protocol `{protocol}` monitor #{idx} description") + severity = body.get("severity") + if severity is not None: + severity = _require_string(severity, f"protocol `{protocol}` monitor #{idx} severity") + return Monitor(name=name, description=description, severity=severity) + + +def _parse_protocol(slug: str, body: Any, source: Path) -> ProtocolMonitoring: + if not isinstance(body, dict): + raise MonitoringConfigError(f"{source}: protocol `{slug}` must be a mapping") + + required = ("display_name", "cadence", "tasks", "monitors") + missing = [k for k in required if k not in body] + if missing: + raise MonitoringConfigError(f"{source}: protocol `{slug}` missing keys: {', '.join(missing)}") + + display_name = _require_string(body["display_name"], f"protocol `{slug}` display_name") + cadence = _require_string(body["cadence"], f"protocol `{slug}` cadence") + description = _require_string(body.get("description", ""), f"protocol `{slug}` description") + + tasks_raw = body.get("tasks", []) + if not isinstance(tasks_raw, list): + raise MonitoringConfigError(f"{source}: protocol `{slug}` tasks must be a list") + tasks = tuple(_require_string(t, f"protocol `{slug}` task") for t in tasks_raw) + + monitors_raw = body["monitors"] + if not isinstance(monitors_raw, list) or not monitors_raw: + raise MonitoringConfigError(f"{source}: protocol `{slug}` monitors must be a non-empty list") + monitors = tuple(_parse_monitor(slug, idx, m) for idx, m in enumerate(monitors_raw)) + + disabled = bool(body.get("disabled", False)) + + return ProtocolMonitoring( + slug=slug, + display_name=display_name, + description=description, + cadence=cadence, + tasks=tasks, + monitors=monitors, + disabled=disabled, + ) + + +def load_monitoring_config(path: Path | str | None = None) -> MonitoringConfig: + """Load and validate ``monitoring.yaml`` from ``path``. + + Args: + path: Path to the YAML file. Defaults to ``monitoring.yaml`` in the + repository root. + + Returns: + Parsed and validated monitoring configuration. + """ + config_path = Path(path) if path is not None else DEFAULT_MONITORING_CONFIG_PATH + if not config_path.exists(): + raise MonitoringConfigError(f"monitoring.yaml not found at {config_path}") + + with config_path.open("r") as f: + raw = yaml.safe_load(f) + + if not isinstance(raw, dict): + raise MonitoringConfigError(f"{config_path}: top-level mapping expected") + if "protocols" not in raw: + raise MonitoringConfigError(f"{config_path}: top-level `protocols:` key missing") + + version = _require_string(raw.get("version", ""), "version") + protocols_raw = raw["protocols"] + if not isinstance(protocols_raw, dict) or not protocols_raw: + raise MonitoringConfigError(f"{config_path}: `protocols:` must be a non-empty mapping") + + protocols: dict[str, ProtocolMonitoring] = {} + for slug, body in protocols_raw.items(): + protocols[str(slug)] = _parse_protocol(str(slug), body, config_path) + + return MonitoringConfig(version=version, protocols=protocols, path=config_path) + + +def protocol_to_json(protocol: ProtocolMonitoring) -> dict[str, object]: + """Serialize a protocol's card metadata to a JSON-friendly dict.""" + return { + "slug": protocol.slug, + "display_name": protocol.display_name, + "description": protocol.description, + "cadence": protocol.cadence, + "monitor_count": protocol.monitor_count, + "disabled": protocol.disabled, + "tasks": list(protocol.tasks), + "monitors": [ + { + "name": m.name, + "description": m.description, + **({"severity": m.severity} if m.severity is not None else {}), + } + for m in protocol.monitors + ], + } + + +def monitoring_to_json(config: MonitoringConfig) -> dict[str, object]: + """Serialize the full monitoring config for API consumers.""" + data = [protocol_to_json(p) for p in config.sorted_protocols] + return {"version": config.version, "data": data, "count": len(data)}