fix(monitor): pace registry enumeration to stop throttle truncation

.github/workflows/registry-monitor.yml

@@ -40,6 +40,10 @@ on:
         description: "Max skills to LLM-verify this run (per-run API cap)"
         type: string
         default: "200"
+      page_size:
+        description: "Skills per listing page (larger = fewer requests to enumerate)"
+        type: string
+        default: "100"
 
 # Never let two sweeps race on the snapshot history / the push.
 concurrency:
@@ -97,6 +101,9 @@ jobs:
           # Manual dispatch may raise the per-run LLM verification cap; scheduled
           # runs use the steady-state default (200).
           ESCALATE_BUDGET: ${{ github.event.inputs.escalate_budget || '200' }}
+          # Listing page size; larger means fewer requests to enumerate the ~66k
+          # registry (the API returns its own max if this exceeds it).
+          PAGE_SIZE: ${{ github.event.inputs.page_size || '100' }}
         run: |
           FULL=""
           if [ "$IS_WEEKLY" = "true" ] || [ "$DISPATCH_FULL" = "true" ]; then
@@ -106,6 +113,7 @@ jobs:
             echo "Mode: incremental (only changed skills)."
           fi
           echo "Escalation budget: ${ESCALATE_BUDGET}"
+          echo "Listing page size: ${PAGE_SIZE}"
           # The whole-registry sweep runs the rule engine + threat intel (fast,
           # free, 100% recall on the benchmark). Only the AMBIGUOUS band — skills
           # the rules flagged short of a confident verdict — is escalated to the
@@ -127,6 +135,7 @@ jobs:
             --escalate-backend anthropic \
             --escalate-budget "$ESCALATE_BUDGET" \
             --max-scans 4000 \
+            --page-size "$PAGE_SIZE" \
             $FULL
 
       - name: Post scan summary to Discord

src/malwar/cli/commands/crawl.py

@@ -548,6 +548,15 @@ def crawl_monitor(
             "instead of timing out.",
         ),
     ] = None,
+    page_size: Annotated[
+        int,
+        typer.Option(
+            "--page-size",
+            help="Skills requested per registry listing page. Larger pages mean "
+            "fewer requests to enumerate the whole registry (the API returns its "
+            "own maximum if this exceeds it).",
+        ),
+    ] = 100,
 ) -> None:
     """Scan the registry incrementally and diff against the last snapshot.
 
@@ -574,6 +583,7 @@ def crawl_monitor(
             fail_on_malicious=fail_on_malicious,
             full=full,
             max_scans=max_scans,
+            page_size=page_size,
         )
     )
     raise typer.Exit(exit_code)
@@ -594,6 +604,7 @@ async def _async_monitor(
     fail_on_malicious: bool,
     full: bool = False,
     max_scans: int | None = None,
+    page_size: int = 100,
 ) -> int:
     from rich.progress import BarColumn, Progress, TextColumn
 
@@ -641,6 +652,7 @@ def _on_progress(done: int, total: int, slug: str) -> None:
             escalation=backend,
             escalation_policy=policy,
             concurrency=concurrency,
+            page_size=page_size,
             on_progress=_on_progress,
         )
 

src/malwar/monitor/snapshot.py

@@ -69,13 +69,20 @@ class SkillMeta(NamedTuple):
     installs: int = 0
 
 
-# How many times to retry a single failed listing page before giving up on the
-# enumeration. The registry rate-limits, so a long (~66k skill) sweep will hit a
-# transient 429/5xx mid-listing; retrying the page keeps one hiccup from
-# truncating the whole enumeration.
-_PAGE_RETRIES = 4
-# Backoff (seconds) between page retries, indexed by attempt. Capped and finite.
-_PAGE_BACKOFF = (2, 4, 8, 16)
+# Enumerating the full ~66k-skill registry is ~1,300 listing requests. The
+# registry throttles sustained listing, so firing those pages as fast as
+# possible trips the limit partway and truncates the enumeration. We therefore
+# (a) pace the pages to stay under the throttle, and (b) retry a page that still
+# fails, with a backoff long enough to outlast a throttle window.
+#
+# _PAGE_DELAY paces successive pages (~2 req/s, matching the registry's observed
+# limit); the full listing then costs ~11 min, well within the run's time box.
+_PAGE_DELAY = 0.5
+# How many times to retry a single failed listing page before giving up.
+_PAGE_RETRIES = 6
+# Backoff (seconds) between page retries — extends past a typical throttle
+# window so a rate-limited page recovers instead of aborting the sweep.
+_PAGE_BACKOFF = (2, 5, 10, 20, 40, 60)
 
 
 async def _enumerate_skills(
@@ -143,6 +150,9 @@ async def _enumerate_skills(
             return metas[:max_skills], True
         if not cursor:
             return metas, True  # reached the natural end with every page ok
+        # Pace the next page so sustained listing stays under the throttle.
+        if _PAGE_DELAY:
+            await asyncio.sleep(_PAGE_DELAY)
 
     # (unreachable: every branch above returns)
 

tests/unit/test_monitor.py

@@ -30,8 +30,9 @@
 
 @pytest.fixture(autouse=True)
 def _instant_enumeration_backoff(monkeypatch):
-    """Zero the page-retry backoff so enumeration-failure tests don't sleep."""
-    monkeypatch.setattr("malwar.monitor.snapshot._PAGE_BACKOFF", (0, 0, 0, 0))
+    """Zero the page pacing/retry backoff so enumeration tests don't sleep."""
+    monkeypatch.setattr("malwar.monitor.snapshot._PAGE_BACKOFF", (0, 0, 0, 0, 0, 0))
+    monkeypatch.setattr("malwar.monitor.snapshot._PAGE_DELAY", 0)
 
 # A skill body that trips MALWAR-FRAUD-001 (agentic affiliate injection).
 MALICIOUS_BODY = (