From 4ae423d6cffaf70c65559b5255c7ccf32ce84971 Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 17 Jul 2026 09:35:43 +0000 Subject: [PATCH 1/2] fix(monitor): pace registry enumeration to stop throttle truncation #34 made a truncated enumeration fail safely (honest incomplete flag + baseline carry-forward), but a verification run showed it still only listed ~3,346 of ~66k skills: the ~1,300-page listing fires as fast as possible and trips the registry's sustained-listing throttle partway through, at a variable point (429/940/2,940/3,346 across runs). Pace the listing to ~2 req/s (matching the registry's observed limit) so it stays under the throttle, and extend the per-page retry/backoff to outlast a throttle window. Full enumeration then costs ~11 min, within the run's time box. Tests zero the pacing/backoff via the autouse fixture so the suite stays fast. Monitor tests (49) pass; ruff + mypy clean. True coverage is confirmed by the next run's enumerated_count / enumeration_complete. Co-Authored-By: Claude --- src/malwar/monitor/snapshot.py | 24 +++++++++++++++++------- tests/unit/test_monitor.py | 5 +++-- 2 files changed, 20 insertions(+), 9 deletions(-) diff --git a/src/malwar/monitor/snapshot.py b/src/malwar/monitor/snapshot.py index 481f27f..817ee6c 100644 --- a/src/malwar/monitor/snapshot.py +++ b/src/malwar/monitor/snapshot.py @@ -69,13 +69,20 @@ class SkillMeta(NamedTuple): installs: int = 0 -# How many times to retry a single failed listing page before giving up on the -# enumeration. The registry rate-limits, so a long (~66k skill) sweep will hit a -# transient 429/5xx mid-listing; retrying the page keeps one hiccup from -# truncating the whole enumeration. -_PAGE_RETRIES = 4 -# Backoff (seconds) between page retries, indexed by attempt. Capped and finite. -_PAGE_BACKOFF = (2, 4, 8, 16) +# Enumerating the full ~66k-skill registry is ~1,300 listing requests. The +# registry throttles sustained listing, so firing those pages as fast as +# possible trips the limit partway and truncates the enumeration. We therefore +# (a) pace the pages to stay under the throttle, and (b) retry a page that still +# fails, with a backoff long enough to outlast a throttle window. +# +# _PAGE_DELAY paces successive pages (~2 req/s, matching the registry's observed +# limit); the full listing then costs ~11 min, well within the run's time box. +_PAGE_DELAY = 0.5 +# How many times to retry a single failed listing page before giving up. +_PAGE_RETRIES = 6 +# Backoff (seconds) between page retries — extends past a typical throttle +# window so a rate-limited page recovers instead of aborting the sweep. +_PAGE_BACKOFF = (2, 5, 10, 20, 40, 60) async def _enumerate_skills( @@ -143,6 +150,9 @@ async def _enumerate_skills( return metas[:max_skills], True if not cursor: return metas, True # reached the natural end with every page ok + # Pace the next page so sustained listing stays under the throttle. + if _PAGE_DELAY: + await asyncio.sleep(_PAGE_DELAY) # (unreachable: every branch above returns) diff --git a/tests/unit/test_monitor.py b/tests/unit/test_monitor.py index 3d31fe2..783d25d 100644 --- a/tests/unit/test_monitor.py +++ b/tests/unit/test_monitor.py @@ -30,8 +30,9 @@ @pytest.fixture(autouse=True) def _instant_enumeration_backoff(monkeypatch): - """Zero the page-retry backoff so enumeration-failure tests don't sleep.""" - monkeypatch.setattr("malwar.monitor.snapshot._PAGE_BACKOFF", (0, 0, 0, 0)) + """Zero the page pacing/retry backoff so enumeration tests don't sleep.""" + monkeypatch.setattr("malwar.monitor.snapshot._PAGE_BACKOFF", (0, 0, 0, 0, 0, 0)) + monkeypatch.setattr("malwar.monitor.snapshot._PAGE_DELAY", 0) # A skill body that trips MALWAR-FRAUD-001 (agentic affiliate injection). MALICIOUS_BODY = ( From f24a6e7d39855c7db2267e9193acaa7e9b97394c Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 17 Jul 2026 09:46:43 +0000 Subject: [PATCH 2/2] feat(monitor): tunable listing page size to cut enumeration requests Enumerating the ~66k registry at the hardcoded page_size=50 is ~1,330 listing requests, the bulk of the throttle exposure. ClawHub has no bulk/export endpoint (public API is /skills paginated, /search, /skills/{slug}[/file], rate-limited to 120 req/min), so fewer/larger pages is the main lever. Add --page-size (default 100, was 50) to 'crawl monitor' and a page_size workflow dispatch input, so a larger page can be tried without code changes (the API returns its own max if we request more). At 100/page enumeration halves to ~660 requests; if the API honors 250, ~270. Complements the pacing/backoff fix in this branch. yaml + CLI help verified; monitor tests (49) pass, ruff + mypy clean. Co-Authored-By: Claude --- .github/workflows/registry-monitor.yml | 9 +++++++++ src/malwar/cli/commands/crawl.py | 12 ++++++++++++ 2 files changed, 21 insertions(+) diff --git a/.github/workflows/registry-monitor.yml b/.github/workflows/registry-monitor.yml index 1df20ba..8392352 100644 --- a/.github/workflows/registry-monitor.yml +++ b/.github/workflows/registry-monitor.yml @@ -40,6 +40,10 @@ on: description: "Max skills to LLM-verify this run (per-run API cap)" type: string default: "200" + page_size: + description: "Skills per listing page (larger = fewer requests to enumerate)" + type: string + default: "100" # Never let two sweeps race on the snapshot history / the push. concurrency: @@ -97,6 +101,9 @@ jobs: # Manual dispatch may raise the per-run LLM verification cap; scheduled # runs use the steady-state default (200). ESCALATE_BUDGET: ${{ github.event.inputs.escalate_budget || '200' }} + # Listing page size; larger means fewer requests to enumerate the ~66k + # registry (the API returns its own max if this exceeds it). + PAGE_SIZE: ${{ github.event.inputs.page_size || '100' }} run: | FULL="" if [ "$IS_WEEKLY" = "true" ] || [ "$DISPATCH_FULL" = "true" ]; then @@ -106,6 +113,7 @@ jobs: echo "Mode: incremental (only changed skills)." fi echo "Escalation budget: ${ESCALATE_BUDGET}" + echo "Listing page size: ${PAGE_SIZE}" # The whole-registry sweep runs the rule engine + threat intel (fast, # free, 100% recall on the benchmark). Only the AMBIGUOUS band — skills # the rules flagged short of a confident verdict — is escalated to the @@ -127,6 +135,7 @@ jobs: --escalate-backend anthropic \ --escalate-budget "$ESCALATE_BUDGET" \ --max-scans 4000 \ + --page-size "$PAGE_SIZE" \ $FULL - name: Post scan summary to Discord diff --git a/src/malwar/cli/commands/crawl.py b/src/malwar/cli/commands/crawl.py index 9bf23b2..99b3f5d 100644 --- a/src/malwar/cli/commands/crawl.py +++ b/src/malwar/cli/commands/crawl.py @@ -548,6 +548,15 @@ def crawl_monitor( "instead of timing out.", ), ] = None, + page_size: Annotated[ + int, + typer.Option( + "--page-size", + help="Skills requested per registry listing page. Larger pages mean " + "fewer requests to enumerate the whole registry (the API returns its " + "own maximum if this exceeds it).", + ), + ] = 100, ) -> None: """Scan the registry incrementally and diff against the last snapshot. @@ -574,6 +583,7 @@ def crawl_monitor( fail_on_malicious=fail_on_malicious, full=full, max_scans=max_scans, + page_size=page_size, ) ) raise typer.Exit(exit_code) @@ -594,6 +604,7 @@ async def _async_monitor( fail_on_malicious: bool, full: bool = False, max_scans: int | None = None, + page_size: int = 100, ) -> int: from rich.progress import BarColumn, Progress, TextColumn @@ -641,6 +652,7 @@ def _on_progress(done: int, total: int, slug: str) -> None: escalation=backend, escalation_policy=policy, concurrency=concurrency, + page_size=page_size, on_progress=_on_progress, )