From e96da402c9f2e79ad37add66af3c36c6865fdc3f Mon Sep 17 00:00:00 2001 From: Tessa Walsh Date: Tue, 28 Jul 2026 12:48:04 -0400 Subject: [PATCH 1/9] Add endpoint for counts of running, paused, and waiting workflows --- backend/btrixcloud/crawlconfigs.py | 104 +++++++++++++++++++++++++++++ backend/btrixcloud/models.py | 27 ++++++++ 2 files changed, 131 insertions(+) diff --git a/backend/btrixcloud/crawlconfigs.py b/backend/btrixcloud/crawlconfigs.py index ed1d1e4d34..84dd44f431 100644 --- a/backend/btrixcloud/crawlconfigs.py +++ b/backend/btrixcloud/crawlconfigs.py @@ -27,6 +27,7 @@ ) from .models import ( + RUNNING_STATES, SUCCESSFUL_STATES, TYPE_ALL_CRAWL_STATES, ConfigRevision, @@ -35,6 +36,7 @@ CrawlConfigDeletedResponse, CrawlConfigIn, CrawlConfigOut, + CrawlConfigRunningCountsResponse, CrawlConfigSearchValues, CrawlConfigUpdateResponse, CrawlerChannel, @@ -1689,6 +1691,90 @@ async def validate_custom_behavior(self, url: str) -> dict[str, bool]: return {"success": True} + async def get_running_counts( + self, org: Organization | None = None + ) -> CrawlConfigRunningCountsResponse: + """Return counts of running workflows, total and status, optionally by org""" + + try: + base_query: dict[str, UUID | dict[str, list[str]]] = {} + if org: + base_query["oid"] = org.id + + total = await self.crawls.count_documents( + {**base_query, "state": {"$in": RUNNING_STATES}} + ) + running = await self.crawls.count_documents( + {**base_query, "state": "running"} + ) + pending_wait = await self.crawls.count_documents( + {**base_query, "state": "pending-wait"} + ) + generate_wacz = await self.crawls.count_documents( + {**base_query, "state": "generate-wacz"} + ) + uploading_wacz = await self.crawls.count_documents( + {**base_query, "state": "uploading-wacz"} + ) + rate_limited = await self.crawls.count_documents( + {**base_query, "state": "rate-limited"} + ) + paused = await self.crawls.count_documents( + {**base_query, "state": "paused"} + ) + paused_storage = await self.crawls.count_documents( + {**base_query, "state": "paused_storage_quota_reached"} + ) + paused_time = await self.crawls.count_documents( + {**base_query, "state": "paused_time_quota_reached"} + ) + paused_read_only = await self.crawls.count_documents( + {**base_query, "state": "paused_org_readonly"} + ) + paused_rate_limit = await self.crawls.count_documents( + {**base_query, "state": "paused_rate_limit_time_reached"} + ) + starting = await self.crawls.count_documents( + {**base_query, "state": "starting"} + ) + waiting_capacity = await self.crawls.count_documents( + {**base_query, "state": "waiting_capacity"} + ) + waiting_org_limit = await self.crawls.count_documents( + {**base_query, "state": "waiting_org_limit"} + ) + waiting_dedupe = await self.crawls.count_documents( + {**base_query, "state": "waiting_dedupe_index"} + ) + + return CrawlConfigRunningCountsResponse( + totalRunningPausedWaiting=total, + # Running states + running=running, + pendingWait=pending_wait, + generateWACZ=generate_wacz, + uploadingWACZ=uploading_wacz, + rateLimited=rate_limited, + # Paused states + paused=paused, + pausedStorageQuotaReached=paused_storage, + pausedTimeQuotaReached=paused_time, + pausedOrgReadOnly=paused_read_only, + pausedRateLimitTimeReached=paused_rate_limit, + # Waiting states + starting=starting, + waitingCapacity=waiting_capacity, + waitingOrgLimit=waiting_org_limit, + waitingDedupeIndex=waiting_dedupe, + ) + except Exception: + logger.exception( + "running_workflow_counts_calculation_failed", + oid=org.id if org else None, + ) + # pylint: disable=raise-missing-from + raise HTTPException(status_code=400, detail="calculation_failure") + # ============================================================================ # pylint: disable=too-many-locals @@ -1945,6 +2031,24 @@ async def get_all_crawler_proxies( return ops.get_crawler_proxies() + @router.get("/running", response_model=CrawlConfigRunningCountsResponse) + async def get_org_crawl_config_running_counts( + org: Organization = Depends(org_viewer_dep), + ): + return await ops.get_running_counts(org) + + @app.get( + "/orgs/all/crawlconfigs/running", + response_model=CrawlConfigRunningCountsResponse, + ) + async def get_all_crawl_config_running_counts( + user: User = Depends(user_dep), + ): + if not user.is_superuser: + raise HTTPException(status_code=403, detail="Not Allowed") + + return await ops.get_running_counts() + @app.get( "/orgs/{oid}/crawlconfigs/{cid}/public/replay.json", response_model=CrawlOutWithResources, diff --git a/backend/btrixcloud/models.py b/backend/btrixcloud/models.py index 9e0080bab8..b13e4ae0d8 100644 --- a/backend/btrixcloud/models.py +++ b/backend/btrixcloud/models.py @@ -709,6 +709,33 @@ class TagsResponse(BaseModel): tags: list[TagCount] +# ============================================================================ +class CrawlConfigRunningCountsResponse(BaseModel): + """Response model for counts of running workflows (total and by status)""" + + totalRunningPausedWaiting: int = 0 + + # Running states + running: int = 0 + pendingWait: int = 0 + generateWACZ: int = 0 + uploadingWACZ: int = 0 + rateLimited: int = 0 + + # Paused states + paused: int = 0 + pausedStorageQuotaReached: int = 0 + pausedTimeQuotaReached: int = 0 + pausedOrgReadOnly: int = 0 + pausedRateLimitTimeReached: int = 0 + + # Waiting states + starting: int = 0 + waitingCapacity: int = 0 + waitingOrgLimit: int = 0 + waitingDedupeIndex: int = 0 + + # ============================================================================ class CrawlConfigSearchValues(BaseModel): """Response model for adding crawlconfigs""" From c22d6a70d08ed1d60cd23c776fdb19c9a771563f Mon Sep 17 00:00:00 2001 From: Tessa Walsh Date: Tue, 28 Jul 2026 16:26:24 -0400 Subject: [PATCH 2/9] Add index to crawls collection for oid and state query --- backend/btrixcloud/crawls.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/backend/btrixcloud/crawls.py b/backend/btrixcloud/crawls.py index 40e5916e94..1f3cc80ec7 100644 --- a/backend/btrixcloud/crawls.py +++ b/backend/btrixcloud/crawls.py @@ -129,6 +129,9 @@ async def init_index(self): ("started", pymongo.ASCENDING), ] ) + await self.crawls.create_index( + [("oid", pymongo.HASHED), ("state", pymongo.DESCENDING)] + ) await self.crawls.create_index([("finished", pymongo.DESCENDING)]) await self.crawls.create_index([("oid", pymongo.HASHED)]) await self.crawls.create_index([("cid", pymongo.HASHED)]) From 85c0180e3ccd5e3145d904ffe829d97843467d3e Mon Sep 17 00:00:00 2001 From: Tessa Walsh Date: Tue, 28 Jul 2026 16:31:23 -0400 Subject: [PATCH 3/9] Add tests --- backend/test/test_run_crawl.py | 35 ++++++++++++++++++++++++++++++++++ 1 file changed, 35 insertions(+) diff --git a/backend/test/test_run_crawl.py b/backend/test/test_run_crawl.py index 5d8cfe81a2..b954bbafb8 100644 --- a/backend/test/test_run_crawl.py +++ b/backend/test/test_run_crawl.py @@ -176,6 +176,41 @@ def test_remove_exclusion(admin_auth_headers, default_org_id): assert r.json()["success"] == True +def test_running_workflow_counts( + admin_auth_headers, crawler_auth_headers, default_org_id +): + # Verify running workflow counts are updated + r = requests.get( + f"{API_PREFIX}/orgs/{default_org_id}/crawlconfigs/running", + headers=admin_auth_headers, + ) + assert r.status_code == 200 + data = r.json() + assert data["totalRunningPausedWaiting"] >= 1 + assert ( + data["running"] >= 1 or data["generateWACZ"] >= 1 or data["uploadingWACZ"] >= 1 + ) + + # Verify again but from non-org-specific endpoint + r = requests.get( + f"{API_PREFIX}/orgs/all/crawlconfigs/running", + headers=admin_auth_headers, + ) + assert r.status_code == 200 + data = r.json() + assert data["totalRunningPausedWaiting"] >= 1 + assert ( + data["running"] >= 1 or data["generateWACZ"] >= 1 or data["uploadingWACZ"] >= 1 + ) + + # Check that non-org-specific endpoint is only available to superadmins + r = requests.get( + f"{API_PREFIX}/orgs/all/crawlconfigs/running", + headers=crawler_auth_headers, + ) + assert r.status_code == 403 + + def test_wait_for_complete(admin_auth_headers, default_org_id): state = None data = None From f8b3220d59fea7f96e8004ed5f4e0750116c50fb Mon Sep 17 00:00:00 2001 From: Tessa Walsh Date: Wed, 29 Jul 2026 10:46:26 -0400 Subject: [PATCH 4/9] Add other category totals, remove extra db call --- backend/btrixcloud/crawlconfigs.py | 27 +++++++++++++++++++++++---- backend/btrixcloud/models.py | 3 +++ backend/test/test_run_crawl.py | 2 ++ 3 files changed, 28 insertions(+), 4 deletions(-) diff --git a/backend/btrixcloud/crawlconfigs.py b/backend/btrixcloud/crawlconfigs.py index 84dd44f431..3d76023e15 100644 --- a/backend/btrixcloud/crawlconfigs.py +++ b/backend/btrixcloud/crawlconfigs.py @@ -27,7 +27,6 @@ ) from .models import ( - RUNNING_STATES, SUCCESSFUL_STATES, TYPE_ALL_CRAWL_STATES, ConfigRevision, @@ -1701,9 +1700,7 @@ async def get_running_counts( if org: base_query["oid"] = org.id - total = await self.crawls.count_documents( - {**base_query, "state": {"$in": RUNNING_STATES}} - ) + # Running states running = await self.crawls.count_documents( {**base_query, "state": "running"} ) @@ -1719,6 +1716,11 @@ async def get_running_counts( rate_limited = await self.crawls.count_documents( {**base_query, "state": "rate-limited"} ) + total_running = ( + running + pending_wait + generate_wacz + uploading_wacz + rate_limited + ) + + # Paused states paused = await self.crawls.count_documents( {**base_query, "state": "paused"} ) @@ -1734,6 +1736,15 @@ async def get_running_counts( paused_rate_limit = await self.crawls.count_documents( {**base_query, "state": "paused_rate_limit_time_reached"} ) + total_paused = ( + paused + + paused_storage + + paused_time + + paused_read_only + + paused_rate_limit + ) + + # Waiting states starting = await self.crawls.count_documents( {**base_query, "state": "starting"} ) @@ -1746,9 +1757,17 @@ async def get_running_counts( waiting_dedupe = await self.crawls.count_documents( {**base_query, "state": "waiting_dedupe_index"} ) + total_waiting = ( + starting + waiting_capacity + waiting_org_limit + waiting_dedupe + ) + + total = total_running + total_paused + total_waiting return CrawlConfigRunningCountsResponse( totalRunningPausedWaiting=total, + totalRunning=total_running, + totalPaused=total_paused, + totalWaiting=total_waiting, # Running states running=running, pendingWait=pending_wait, diff --git a/backend/btrixcloud/models.py b/backend/btrixcloud/models.py index b13e4ae0d8..5b6afe08b1 100644 --- a/backend/btrixcloud/models.py +++ b/backend/btrixcloud/models.py @@ -714,6 +714,9 @@ class CrawlConfigRunningCountsResponse(BaseModel): """Response model for counts of running workflows (total and by status)""" totalRunningPausedWaiting: int = 0 + totalRunning: int = 0 + totalPaused: int = 0 + totalWaiting: int = 0 # Running states running: int = 0 diff --git a/backend/test/test_run_crawl.py b/backend/test/test_run_crawl.py index b954bbafb8..106ab8e517 100644 --- a/backend/test/test_run_crawl.py +++ b/backend/test/test_run_crawl.py @@ -187,6 +187,7 @@ def test_running_workflow_counts( assert r.status_code == 200 data = r.json() assert data["totalRunningPausedWaiting"] >= 1 + assert data["totalRunning"] >= 1 assert ( data["running"] >= 1 or data["generateWACZ"] >= 1 or data["uploadingWACZ"] >= 1 ) @@ -199,6 +200,7 @@ def test_running_workflow_counts( assert r.status_code == 200 data = r.json() assert data["totalRunningPausedWaiting"] >= 1 + assert data["totalRunning"] >= 1 assert ( data["running"] >= 1 or data["generateWACZ"] >= 1 or data["uploadingWACZ"] >= 1 ) From 95c9b8f347d2a02ccda0fdc9bd1ce35b8678401b Mon Sep 17 00:00:00 2001 From: Tessa Walsh Date: Wed, 29 Jul 2026 10:54:27 -0400 Subject: [PATCH 5/9] Simplify typing --- backend/btrixcloud/crawlconfigs.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/backend/btrixcloud/crawlconfigs.py b/backend/btrixcloud/crawlconfigs.py index 3d76023e15..8fb13281af 100644 --- a/backend/btrixcloud/crawlconfigs.py +++ b/backend/btrixcloud/crawlconfigs.py @@ -1696,7 +1696,7 @@ async def get_running_counts( """Return counts of running workflows, total and status, optionally by org""" try: - base_query: dict[str, UUID | dict[str, list[str]]] = {} + base_query: dict[str, UUID | str] = {} if org: base_query["oid"] = org.id From a23ae8fb31bd5111d22d0193f59ac8d392d44d1a Mon Sep 17 00:00:00 2001 From: Tessa Walsh Date: Mon, 3 Aug 2026 13:24:31 -0400 Subject: [PATCH 6/9] Rework counts to use single mongo aggregation --- backend/btrixcloud/crawlconfigs.py | 88 ++++++++++++++---------------- 1 file changed, 42 insertions(+), 46 deletions(-) diff --git a/backend/btrixcloud/crawlconfigs.py b/backend/btrixcloud/crawlconfigs.py index 8fb13281af..310fffb52f 100644 --- a/backend/btrixcloud/crawlconfigs.py +++ b/backend/btrixcloud/crawlconfigs.py @@ -27,6 +27,7 @@ ) from .models import ( + ALL_CRAWL_STATES, SUCCESSFUL_STATES, TYPE_ALL_CRAWL_STATES, ConfigRevision, @@ -1695,47 +1696,51 @@ async def get_running_counts( ) -> CrawlConfigRunningCountsResponse: """Return counts of running workflows, total and status, optionally by org""" + state_count_logger = logger.bind(oid=org.id if org else None) + try: - base_query: dict[str, UUID | str] = {} + match_query: dict[str, UUID | str] = {} if org: - base_query["oid"] = org.id + match_query["oid"] = org.id + + res = await self.crawls.aggregate( + [ + {"$match": match_query}, + {"$group": {"_id": "$state", "count": {"$sum": 1}}}, + {"$project": {"state": "$_id", "count": "$count", "_id": 0}}, + {"$sort": {"count": -1, "state": 1}}, + ] + ).to_list() + + state_counts: dict[str, int] = {} + + for state_dict in res: + state = state_dict["state"] + count = state_dict.get("count", 0) + if state not in ALL_CRAWL_STATES: + state_count_logger.error( + "unexpected_crawl_state_found", state=state, count=count + ) + else: + state_counts[state] = count # Running states - running = await self.crawls.count_documents( - {**base_query, "state": "running"} - ) - pending_wait = await self.crawls.count_documents( - {**base_query, "state": "pending-wait"} - ) - generate_wacz = await self.crawls.count_documents( - {**base_query, "state": "generate-wacz"} - ) - uploading_wacz = await self.crawls.count_documents( - {**base_query, "state": "uploading-wacz"} - ) - rate_limited = await self.crawls.count_documents( - {**base_query, "state": "rate-limited"} - ) + running = state_counts.get("running", 0) + pending_wait = state_counts.get("pending-wait", 0) + generate_wacz = state_counts.get("generate-wacz", 0) + uploading_wacz = state_counts.get("uploading-wacz", 0) + rate_limited = state_counts.get("rate-limited", 0) total_running = ( running + pending_wait + generate_wacz + uploading_wacz + rate_limited ) # Paused states - paused = await self.crawls.count_documents( - {**base_query, "state": "paused"} - ) - paused_storage = await self.crawls.count_documents( - {**base_query, "state": "paused_storage_quota_reached"} - ) - paused_time = await self.crawls.count_documents( - {**base_query, "state": "paused_time_quota_reached"} - ) - paused_read_only = await self.crawls.count_documents( - {**base_query, "state": "paused_org_readonly"} - ) - paused_rate_limit = await self.crawls.count_documents( - {**base_query, "state": "paused_rate_limit_time_reached"} - ) + paused = state_counts.get("paused", 0) + paused_storage = state_counts.get("paused_storage_quota_reached", 0) + paused_time = state_counts.get("paused_time_quota_reached", 0) + paused_read_only = state_counts.get("paused_org_readonly", 0) + paused_rate_limit = state_counts.get("paused_rate_limit_time_reached", 0) + total_paused = ( paused + paused_storage @@ -1745,18 +1750,10 @@ async def get_running_counts( ) # Waiting states - starting = await self.crawls.count_documents( - {**base_query, "state": "starting"} - ) - waiting_capacity = await self.crawls.count_documents( - {**base_query, "state": "waiting_capacity"} - ) - waiting_org_limit = await self.crawls.count_documents( - {**base_query, "state": "waiting_org_limit"} - ) - waiting_dedupe = await self.crawls.count_documents( - {**base_query, "state": "waiting_dedupe_index"} - ) + starting = state_counts.get("starting", 0) + waiting_capacity = state_counts.get("waiting_capacity", 0) + waiting_org_limit = state_counts.get("waiting_org_limit", 0) + waiting_dedupe = state_counts.get("waiting_dedupe", 0) total_waiting = ( starting + waiting_capacity + waiting_org_limit + waiting_dedupe ) @@ -1787,9 +1784,8 @@ async def get_running_counts( waitingDedupeIndex=waiting_dedupe, ) except Exception: - logger.exception( + state_count_logger.exception( "running_workflow_counts_calculation_failed", - oid=org.id if org else None, ) # pylint: disable=raise-missing-from raise HTTPException(status_code=400, detail="calculation_failure") From c6a9a674f53b244a358917cd74c73b90feae7839 Mon Sep 17 00:00:00 2001 From: Tessa Walsh Date: Mon, 3 Aug 2026 16:21:00 -0400 Subject: [PATCH 7/9] Remove unnecessary sort and field renaming from aggregation --- backend/btrixcloud/crawlconfigs.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/backend/btrixcloud/crawlconfigs.py b/backend/btrixcloud/crawlconfigs.py index 310fffb52f..07faa5cb22 100644 --- a/backend/btrixcloud/crawlconfigs.py +++ b/backend/btrixcloud/crawlconfigs.py @@ -1707,15 +1707,14 @@ async def get_running_counts( [ {"$match": match_query}, {"$group": {"_id": "$state", "count": {"$sum": 1}}}, - {"$project": {"state": "$_id", "count": "$count", "_id": 0}}, - {"$sort": {"count": -1, "state": 1}}, + {"$project": {"_id": 1, "count": "$count"}}, ] ).to_list() state_counts: dict[str, int] = {} for state_dict in res: - state = state_dict["state"] + state = state_dict["_id"] count = state_dict.get("count", 0) if state not in ALL_CRAWL_STATES: state_count_logger.error( From ed58a394539f5f1983d6e2429abcd83be89d89fb Mon Sep 17 00:00:00 2001 From: Tessa Walsh Date: Mon, 3 Aug 2026 16:33:35 -0400 Subject: [PATCH 8/9] Remove unnecessary project stage --- backend/btrixcloud/crawlconfigs.py | 1 - 1 file changed, 1 deletion(-) diff --git a/backend/btrixcloud/crawlconfigs.py b/backend/btrixcloud/crawlconfigs.py index 07faa5cb22..5bf31fe6a8 100644 --- a/backend/btrixcloud/crawlconfigs.py +++ b/backend/btrixcloud/crawlconfigs.py @@ -1707,7 +1707,6 @@ async def get_running_counts( [ {"$match": match_query}, {"$group": {"_id": "$state", "count": {"$sum": 1}}}, - {"$project": {"_id": 1, "count": "$count"}}, ] ).to_list() From 8e65c38b3212bc73dd28608e25afc2e6a338799f Mon Sep 17 00:00:00 2001 From: Tessa Walsh Date: Mon, 3 Aug 2026 16:39:54 -0400 Subject: [PATCH 9/9] Use in place of --- backend/btrixcloud/crawlconfigs.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/backend/btrixcloud/crawlconfigs.py b/backend/btrixcloud/crawlconfigs.py index 5bf31fe6a8..ae25c143b6 100644 --- a/backend/btrixcloud/crawlconfigs.py +++ b/backend/btrixcloud/crawlconfigs.py @@ -1706,7 +1706,7 @@ async def get_running_counts( res = await self.crawls.aggregate( [ {"$match": match_query}, - {"$group": {"_id": "$state", "count": {"$sum": 1}}}, + {"$group": {"_id": "$state", "count": {"$count": {}}}}, ] ).to_list()