diff --git a/.github/workflows/registry-monitor.yml b/.github/workflows/registry-monitor.yml index 1df20ba..8392352 100644 --- a/.github/workflows/registry-monitor.yml +++ b/.github/workflows/registry-monitor.yml @@ -40,6 +40,10 @@ on: description: "Max skills to LLM-verify this run (per-run API cap)" type: string default: "200" + page_size: + description: "Skills per listing page (larger = fewer requests to enumerate)" + type: string + default: "100" # Never let two sweeps race on the snapshot history / the push. concurrency: @@ -97,6 +101,9 @@ jobs: # Manual dispatch may raise the per-run LLM verification cap; scheduled # runs use the steady-state default (200). ESCALATE_BUDGET: ${{ github.event.inputs.escalate_budget || '200' }} + # Listing page size; larger means fewer requests to enumerate the ~66k + # registry (the API returns its own max if this exceeds it). + PAGE_SIZE: ${{ github.event.inputs.page_size || '100' }} run: | FULL="" if [ "$IS_WEEKLY" = "true" ] || [ "$DISPATCH_FULL" = "true" ]; then @@ -106,6 +113,7 @@ jobs: echo "Mode: incremental (only changed skills)." fi echo "Escalation budget: ${ESCALATE_BUDGET}" + echo "Listing page size: ${PAGE_SIZE}" # The whole-registry sweep runs the rule engine + threat intel (fast, # free, 100% recall on the benchmark). Only the AMBIGUOUS band — skills # the rules flagged short of a confident verdict — is escalated to the @@ -127,6 +135,7 @@ jobs: --escalate-backend anthropic \ --escalate-budget "$ESCALATE_BUDGET" \ --max-scans 4000 \ + --page-size "$PAGE_SIZE" \ $FULL - name: Post scan summary to Discord diff --git a/src/malwar/cli/commands/crawl.py b/src/malwar/cli/commands/crawl.py index 9bf23b2..99b3f5d 100644 --- a/src/malwar/cli/commands/crawl.py +++ b/src/malwar/cli/commands/crawl.py @@ -548,6 +548,15 @@ def crawl_monitor( "instead of timing out.", ), ] = None, + page_size: Annotated[ + int, + typer.Option( + "--page-size", + help="Skills requested per registry listing page. Larger pages mean " + "fewer requests to enumerate the whole registry (the API returns its " + "own maximum if this exceeds it).", + ), + ] = 100, ) -> None: """Scan the registry incrementally and diff against the last snapshot. @@ -574,6 +583,7 @@ def crawl_monitor( fail_on_malicious=fail_on_malicious, full=full, max_scans=max_scans, + page_size=page_size, ) ) raise typer.Exit(exit_code) @@ -594,6 +604,7 @@ async def _async_monitor( fail_on_malicious: bool, full: bool = False, max_scans: int | None = None, + page_size: int = 100, ) -> int: from rich.progress import BarColumn, Progress, TextColumn @@ -641,6 +652,7 @@ def _on_progress(done: int, total: int, slug: str) -> None: escalation=backend, escalation_policy=policy, concurrency=concurrency, + page_size=page_size, on_progress=_on_progress, ) diff --git a/src/malwar/monitor/snapshot.py b/src/malwar/monitor/snapshot.py index 481f27f..817ee6c 100644 --- a/src/malwar/monitor/snapshot.py +++ b/src/malwar/monitor/snapshot.py @@ -69,13 +69,20 @@ class SkillMeta(NamedTuple): installs: int = 0 -# How many times to retry a single failed listing page before giving up on the -# enumeration. The registry rate-limits, so a long (~66k skill) sweep will hit a -# transient 429/5xx mid-listing; retrying the page keeps one hiccup from -# truncating the whole enumeration. -_PAGE_RETRIES = 4 -# Backoff (seconds) between page retries, indexed by attempt. Capped and finite. -_PAGE_BACKOFF = (2, 4, 8, 16) +# Enumerating the full ~66k-skill registry is ~1,300 listing requests. The +# registry throttles sustained listing, so firing those pages as fast as +# possible trips the limit partway and truncates the enumeration. We therefore +# (a) pace the pages to stay under the throttle, and (b) retry a page that still +# fails, with a backoff long enough to outlast a throttle window. +# +# _PAGE_DELAY paces successive pages (~2 req/s, matching the registry's observed +# limit); the full listing then costs ~11 min, well within the run's time box. +_PAGE_DELAY = 0.5 +# How many times to retry a single failed listing page before giving up. +_PAGE_RETRIES = 6 +# Backoff (seconds) between page retries — extends past a typical throttle +# window so a rate-limited page recovers instead of aborting the sweep. +_PAGE_BACKOFF = (2, 5, 10, 20, 40, 60) async def _enumerate_skills( @@ -143,6 +150,9 @@ async def _enumerate_skills( return metas[:max_skills], True if not cursor: return metas, True # reached the natural end with every page ok + # Pace the next page so sustained listing stays under the throttle. + if _PAGE_DELAY: + await asyncio.sleep(_PAGE_DELAY) # (unreachable: every branch above returns) diff --git a/tests/unit/test_monitor.py b/tests/unit/test_monitor.py index 3d31fe2..783d25d 100644 --- a/tests/unit/test_monitor.py +++ b/tests/unit/test_monitor.py @@ -30,8 +30,9 @@ @pytest.fixture(autouse=True) def _instant_enumeration_backoff(monkeypatch): - """Zero the page-retry backoff so enumeration-failure tests don't sleep.""" - monkeypatch.setattr("malwar.monitor.snapshot._PAGE_BACKOFF", (0, 0, 0, 0)) + """Zero the page pacing/retry backoff so enumeration tests don't sleep.""" + monkeypatch.setattr("malwar.monitor.snapshot._PAGE_BACKOFF", (0, 0, 0, 0, 0, 0)) + monkeypatch.setattr("malwar.monitor.snapshot._PAGE_DELAY", 0) # A skill body that trips MALWAR-FRAUD-001 (agentic affiliate injection). MALICIOUS_BODY = (