From 696e912f2baa63e72b315dff1bf3669b80af403b Mon Sep 17 00:00:00 2001 From: sua yoo Date: Fri, 24 Jul 2026 12:34:59 -0700 Subject: [PATCH 1/7] hide qa if running --- frontend/src/pages/org/workflow-detail.ts | 16 +++------------- 1 file changed, 3 insertions(+), 13 deletions(-) diff --git a/frontend/src/pages/org/workflow-detail.ts b/frontend/src/pages/org/workflow-detail.ts index 9cf8c9099e..faefdf96ab 100644 --- a/frontend/src/pages/org/workflow-detail.ts +++ b/frontend/src/pages/org/workflow-detail.ts @@ -1618,18 +1618,6 @@ export class WorkflowDetail extends BtrixElement { const qa = (workflow: Workflow) => { if (!latestCrawl) return html``; - if (workflow.isCrawlRunning) { - return html` - ${noData} - - - - `; - } - if (!isSuccessfullyFinished({ state: workflow.lastCrawlState })) { return notApplicable; } @@ -1681,7 +1669,9 @@ export class WorkflowDetail extends BtrixElement { unitDisplay: "narrow", }), )} - ${this.renderDetailItem(msg("QA Rating"), qa)} + ${this.workflow && !this.workflow.isCrawlRunning + ? this.renderDetailItem(msg("QA Rating"), qa) + : nothing} `; }; From d5eb22ca69a64c1005ae04d70100a4518203acf6 Mon Sep 17 00:00:00 2001 From: sua yoo Date: Fri, 24 Jul 2026 12:42:05 -0700 Subject: [PATCH 2/7] display queue count --- frontend/src/components/ui/desc-list.ts | 11 ++++++++++- frontend/src/pages/org/workflow-detail.ts | 19 ++++++++++++++----- 2 files changed, 24 insertions(+), 6 deletions(-) diff --git a/frontend/src/components/ui/desc-list.ts b/frontend/src/components/ui/desc-list.ts index 1c415a6544..878f4b7e16 100644 --- a/frontend/src/components/ui/desc-list.ts +++ b/frontend/src/components/ui/desc-list.ts @@ -1,6 +1,7 @@ import { css, html, LitElement } from "lit"; import { customElement, property } from "lit/decorators.js"; import { classMap } from "lit/directives/class-map.js"; +import { ifDefined } from "lit/directives/if-defined.js"; /** * Styled
,
and
for displaying data @@ -59,11 +60,19 @@ export class DescListItem extends LitElement { @property({ type: String }) label = ""; + /** + * Value will update intermittently after page load + */ + @property({ type: Boolean }) + live = false; + render() { return html`
${this.label}
-
+
+ +
`; } diff --git a/frontend/src/pages/org/workflow-detail.ts b/frontend/src/pages/org/workflow-detail.ts index faefdf96ab..354b7f4760 100644 --- a/frontend/src/pages/org/workflow-detail.ts +++ b/frontend/src/pages/org/workflow-detail.ts @@ -1132,7 +1132,10 @@ export class WorkflowDetail extends BtrixElement { renderContent: (workflow: Workflow) => TemplateResult | string | number, ) { return html` - + ${when( this.workflow, renderContent, @@ -1606,15 +1609,18 @@ export class WorkflowDetail extends BtrixElement { if (!latestCrawl) return skeleton; if (workflow.isCrawlRunning) { - return [ - this.localize.number(+(latestCrawl.stats?.done || 0)), - this.localize.number(+(latestCrawl.stats?.found || 0)), - ].join(` ${msg("of")} `); + return this.localize.number(+(latestCrawl.stats?.done || 0)); } return this.localize.number(latestCrawl.pageCount || 0); }; + const queuedPages = () => { + if (!latestCrawl) return skeleton; + + return this.localize.number(+(latestCrawl.stats?.found || 0)); + }; + const qa = (workflow: Workflow) => { if (!latestCrawl) return html``; @@ -1664,6 +1670,9 @@ export class WorkflowDetail extends BtrixElement { : execTime(), )} ${this.renderDetailItem(msg("Pages Crawled"), pages)} + ${this.workflow && this.workflow.isCrawlRunning + ? this.renderDetailItem(msg("Pages Queued"), queuedPages) + : nothing} ${this.renderDetailItem(msg("Size"), (workflow) => this.localize.bytes(workflow.lastCrawlSize || 0, { unitDisplay: "narrow", From af79c6ffb3e68f102f1ca692d45b1bbbd3fa0445 Mon Sep 17 00:00:00 2001 From: sua yoo Date: Fri, 24 Jul 2026 13:40:44 -0700 Subject: [PATCH 3/7] update columns --- .../features/archived-items/crawl-status.ts | 9 +- .../crawls/crawl-list/crawl-list-item.ts | 175 +++++++++++------- .../features/crawls/crawl-list/crawl-list.ts | 18 +- frontend/src/layouts/separator.ts | 8 +- frontend/src/pages/org/workflow-detail.ts | 12 +- frontend/src/utils/crawler.ts | 2 +- 6 files changed, 132 insertions(+), 92 deletions(-) diff --git a/frontend/src/features/archived-items/crawl-status.ts b/frontend/src/features/archived-items/crawl-status.ts index cf03060d29..70ad88ffd1 100644 --- a/frontend/src/features/archived-items/crawl-status.ts +++ b/frontend/src/features/archived-items/crawl-status.ts @@ -73,8 +73,10 @@ export class CrawlStatus extends TailwindElement { icon: TemplateResult; label: string; cssColor: string; + cssDarkerColor?: string; } { let color = "var(--sl-color-neutral-400)"; + let darkerColor: string | undefined = undefined; let icon = html` renderName(workflow)); const skipped = isSkipped(this.crawl); + const canceled = this.crawl.state === "canceled"; const hasExec = Boolean(this.crawl.crawlExecSeconds); const notApplicable = html` @@ -71,11 +78,10 @@ export class CrawlListItem extends BtrixElement { return html` { if (e.target === this.dropdownMenu) { return; @@ -97,49 +103,75 @@ export class CrawlListItem extends BtrixElement { )} - ${ - this.href - ? html` - ${label} - ` - : label - } + ${this.href + ? html` + ${label} + ` + : label} - ${ - this.workflowId - ? nothing - : html` - - ${this.safeRender( - (crawl) => html` - - `, - )} - - ` - } + ${this.workflowId + ? nothing + : html` + + ${this.safeRender( + (crawl) => html` + + `, + )} + + `} - ${this.safeRender((crawl) => - crawl.finished - ? html` - - ` - : notApplicable, - )} + ${this.safeRender((crawl) => { + if (crawl.finished) { + return html` + + `; + } + + const done = +(crawl.stats?.done || 0); + const found = +(crawl.stats?.found || 0); + const ratio = done && found ? done / found : 0; + const percentage = ratio * 100; + const { cssColor, cssDarkerColor } = CrawlStatus.getContent({ + state: crawl.state, + }); + + return html` + + `; + })} ${this.safeRender((crawl) => @@ -152,16 +184,33 @@ export class CrawlListItem extends BtrixElement { ${this.safeRender((crawl) => { - if (hasExec) { - const pagesComplete = crawl.finished - ? crawl.pageCount - ? +crawl.pageCount - : 0 - : +(crawl.stats?.done || 0); - - return this.localize.number(pagesComplete, { - notation: "compact", - }); + if (hasExec && !canceled) { + if (crawl.finished) { + return this.localize.number(crawl.pageCount || 0, { + notation: "compact", + }); + } + + const done = +(crawl.stats?.done || 0); + const found = +(crawl.stats?.found || 0); + + return html` + + ${this.localize.number(done, { + notation: "compact", + })} + ${textSeparator(tw`text-neutral-500`)} + ${this.localize.number(found, { + notation: "compact", + })} + + `; } return notApplicable; @@ -169,7 +218,7 @@ export class CrawlListItem extends BtrixElement { ${this.safeRender((crawl) => - hasExec + hasExec && !canceled ? this.localize.bytes( crawl.finished ? crawl.fileSize || 0 @@ -178,18 +227,6 @@ export class CrawlListItem extends BtrixElement { : notApplicable, )} - - ${this.safeRender( - (crawl) => - html``, - )} - ${this.renderActions()} diff --git a/frontend/src/features/crawls/crawl-list/crawl-list.ts b/frontend/src/features/crawls/crawl-list/crawl-list.ts index fd831a0e9a..3e9b2da7e5 100644 --- a/frontend/src/features/crawls/crawl-list/crawl-list.ts +++ b/frontend/src/features/crawls/crawl-list/crawl-list.ts @@ -61,14 +61,13 @@ export class CrawlList extends TailwindElement { render() { const columns = [ "min-content [clickable-start]", - this.workflowId ? undefined : "minmax(22ch, 36ch)", // Name - "minmax(min-content, 22ch)", // Started - "minmax(min-content, 22ch)", // Finished + this.workflowId ? undefined : "minmax(24ch, 40ch)", // Name + "minmax(min-content, 24ch)", // Started + "minmax(min-content, 24ch)", // Finished "1fr", // Execution time "1fr", // Pages "1fr", // Size - "[clickable-end] minmax(max-content, 20ch)", // Run by - "min-content", + "[clickable-end] min-content", // Actions ] .filter((v) => v) .join(" "); @@ -89,19 +88,16 @@ export class CrawlList extends TailwindElement { ${msg("Name")} `} + ${msg("Started")} - ${msg("Date Started")} - - - ${msg("Date Finished")} + ${msg("Finished")} ${msg("Exec Time")} ${msg("Pages")} ${msg("Size")} - ${msg("Run By")} - + ${msg("Row actions")} diff --git a/frontend/src/layouts/separator.ts b/frontend/src/layouts/separator.ts index 1ab2bd17ba..785dca051e 100644 --- a/frontend/src/layouts/separator.ts +++ b/frontend/src/layouts/separator.ts @@ -1,12 +1,12 @@ import { html } from "lit"; +import { tw } from "@/utils/tailwind"; + /** * For separatoring text in the same line, e.g. for breadcrumbs or item details */ -export function textSeparator() { - return html`/ `; } diff --git a/frontend/src/pages/org/workflow-detail.ts b/frontend/src/pages/org/workflow-detail.ts index 354b7f4760..c804aff487 100644 --- a/frontend/src/pages/org/workflow-detail.ts +++ b/frontend/src/pages/org/workflow-detail.ts @@ -1605,7 +1605,7 @@ export class WorkflowDetail extends BtrixElement { }); }; - const pages = (workflow: Workflow) => { + const crawledPages = (workflow: Workflow) => { if (!latestCrawl) return skeleton; if (workflow.isCrawlRunning) { @@ -1615,10 +1615,12 @@ export class WorkflowDetail extends BtrixElement { return this.localize.number(latestCrawl.pageCount || 0); }; - const queuedPages = () => { + const foundPages = () => { if (!latestCrawl) return skeleton; - return this.localize.number(+(latestCrawl.stats?.found || 0)); + const found = +(latestCrawl.stats?.found || 0); + + return this.localize.number(found); }; const qa = (workflow: Workflow) => { @@ -1669,9 +1671,9 @@ export class WorkflowDetail extends BtrixElement { )}` : execTime(), )} - ${this.renderDetailItem(msg("Pages Crawled"), pages)} + ${this.renderDetailItem(msg("Pages Crawled"), crawledPages)} ${this.workflow && this.workflow.isCrawlRunning - ? this.renderDetailItem(msg("Pages Queued"), queuedPages) + ? this.renderDetailItem(msg("Pages Found"), foundPages) : nothing} ${this.renderDetailItem(msg("Size"), (workflow) => this.localize.bytes(workflow.lastCrawlSize || 0, { diff --git a/frontend/src/utils/crawler.ts b/frontend/src/utils/crawler.ts index 0e98ddb597..747aae0ce7 100644 --- a/frontend/src/utils/crawler.ts +++ b/frontend/src/utils/crawler.ts @@ -121,7 +121,7 @@ export function renderName( `; } return html` - +
${item.firstSeed}
From eda066cfb0eb413d5b51488e04a07631c08ad7ad Mon Sep 17 00:00:00 2001 From: sua yoo Date: Fri, 24 Jul 2026 13:46:24 -0700 Subject: [PATCH 4/7] update workflow crawls --- frontend/src/pages/org/workflow-detail.ts | 33 ++++------------------- 1 file changed, 5 insertions(+), 28 deletions(-) diff --git a/frontend/src/pages/org/workflow-detail.ts b/frontend/src/pages/org/workflow-detail.ts index c804aff487..b5b3f43afc 100644 --- a/frontend/src/pages/org/workflow-detail.ts +++ b/frontend/src/pages/org/workflow-detail.ts @@ -2,7 +2,6 @@ import { consume } from "@lit/context"; import { localized, msg, str } from "@lit/localize"; import { Task, TaskStatus } from "@lit/task"; import type { SlDropdown } from "@shoelace-style/shoelace"; -import clsx from "clsx"; import { html, nothing, type PropertyValues, type TemplateResult } from "lit"; import { customElement, property, query, state } from "lit/decorators.js"; import { choose } from "lit/directives/choose.js"; @@ -1146,6 +1145,8 @@ export class WorkflowDetail extends BtrixElement { } private renderCrawls() { + const showReplay = !this.isRunning; + return html`
- ${when( - this.workflow?.isCrawlRunning, - () => - html`
- - ${this.isRunning - ? msg("Workflow crawl is currently in progress.") - : msg("This workflow has an active crawl.")} - - ${this.isRunning ? msg("Watch Crawl") : msg("View Crawl")} - - -
`, - )} -
${when( @@ -1214,14 +1196,9 @@ export class WorkflowDetail extends BtrixElement { crawls.items.map( (crawl: Crawl) => html` From fae56a75ded2a0fb02a2bb51c79d2317c3ab0ba8 Mon Sep 17 00:00:00 2001 From: sua yoo Date: Fri, 24 Jul 2026 14:21:02 -0700 Subject: [PATCH 5/7] update docs --- frontend/docs/docs/user-guide/crawl-workflows.md | 6 ++++++ frontend/docs/docs/user-guide/running-crawl.md | 4 +++- 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/frontend/docs/docs/user-guide/crawl-workflows.md b/frontend/docs/docs/user-guide/crawl-workflows.md index f8f9f27745..1f9e6d6a43 100644 --- a/frontend/docs/docs/user-guide/crawl-workflows.md +++ b/frontend/docs/docs/user-guide/crawl-workflows.md @@ -78,3 +78,9 @@ Workflow runs may be automatically paused, stopped, or skipped due to an enforce | **Storage Quota Reached** | Disk space allocated for the org is full. | | **Time Quota Reached** | All execution time allocated for the org has been spent. | | **Crawling Disabled** | Crawling has been disabled for the entire org. | + +## Crawl Run History + +A list of all crawl runs are shown in the **Crawls** section of the workflow. Selecting the latest crawl run will take you to the **[Watch](running-crawl.md#watch-crawl)** section of the workflow. Selecting a finished crawl will take you to an overview of the archived item where you can review and replay the crawl. + +To delete a crawl run and its associated archived item, select the :bootstrap-three-dots-vertical: action menu button in the row that the crawl is displayed and choose _Delete Crawl_. The crawl run and archived item will be permanently deleted from the organization and will no longer be visible in workflows or collections. diff --git a/frontend/docs/docs/user-guide/running-crawl.md b/frontend/docs/docs/user-guide/running-crawl.md index a1a52fc89c..8c97922e16 100644 --- a/frontend/docs/docs/user-guide/running-crawl.md +++ b/frontend/docs/docs/user-guide/running-crawl.md @@ -2,10 +2,12 @@ Running crawls can be modified from the crawl workflow **Latest Crawl** tab. You may want to modify a running crawl if you find that the workflow is crawling pages that you didn't intend to archive, or if you want a boost of speed. -## Crawl Workflow Status +## Crawl Status When a workflow run is initiated, the workflow status changes to :bootstrap-hourglass-split: Waiting or :btrix-status-dot: Starting, depending on whether the conditions for starting a crawl are in place (such as resource capacity.) The workflow status will change to :btrix-status-dot: Running once the crawler loads the first crawl URL. +A running crawl in the **Crawling** > **Crawl Runs** list and [workflow > **Crawls**](./crawl-workflows.md#crawl-run-history) list will display a progress indicator in the place of a finish date. The progress indicator is based on the ratio of crawled to found pages. The indicator should be used as a rough estimate for how far along the crawl is; the ratio may change over the course of the crawl as the crawler discovers more pages, or if [page exclusions change](#live-exclusion-editing). + ## Watch Crawl You can watch the current state of the browser windows as the crawler visits pages in the **Watch** tab of **Latest Crawl**. A list of queued URLs are displayed below in the **Upcoming Pages** section. From 85362aac4a590881930b854badd46d963cea56fb Mon Sep 17 00:00:00 2001 From: sua yoo Date: Fri, 24 Jul 2026 17:20:36 -0700 Subject: [PATCH 6/7] update superadmin running crawls --- .../features/crawls/crawl-list/crawl-list.ts | 7 +++++-- frontend/src/pages/crawls.ts | 17 +++++++++++++++-- frontend/src/pages/org/crawls.ts | 2 +- 3 files changed, 21 insertions(+), 5 deletions(-) diff --git a/frontend/src/features/crawls/crawl-list/crawl-list.ts b/frontend/src/features/crawls/crawl-list/crawl-list.ts index 3e9b2da7e5..2ccb927d80 100644 --- a/frontend/src/features/crawls/crawl-list/crawl-list.ts +++ b/frontend/src/features/crawls/crawl-list/crawl-list.ts @@ -55,6 +55,9 @@ export class CrawlList extends TailwindElement { @property({ type: String }) workflowId?: string; + @property({ type: Boolean, noAccessor: true }) + runningOnly = false; + @queryAssignedElements({ selector: "btrix-crawl-list-item" }) listItems!: HTMLElement[]; @@ -63,7 +66,7 @@ export class CrawlList extends TailwindElement { "min-content [clickable-start]", this.workflowId ? undefined : "minmax(24ch, 40ch)", // Name "minmax(min-content, 24ch)", // Started - "minmax(min-content, 24ch)", // Finished + "minmax(min-content, 24ch)", // Finished/Progress "1fr", // Execution time "1fr", // Pages "1fr", // Size @@ -90,7 +93,7 @@ export class CrawlList extends TailwindElement { `} ${msg("Started")} - ${msg("Finished")} + ${this.runningOnly ? msg("Progress") : msg("Finished")} ${msg("Exec Time")} + ${this.crawls.items.map(this.renderCrawlItem)} `; diff --git a/frontend/src/pages/org/crawls.ts b/frontend/src/pages/org/crawls.ts index d2a9309010..5a04dd2af6 100644 --- a/frontend/src/pages/org/crawls.ts +++ b/frontend/src/pages/org/crawls.ts @@ -61,7 +61,7 @@ const sortableFields: Record< defaultDirection: "desc", }, pageCount: { - label: msg("Pages"), + label: msg("Pages Crawled"), defaultDirection: "desc", }, fileSize: { From c507b5cc59d43900922d6358a45cdc15845643d7 Mon Sep 17 00:00:00 2001 From: sua yoo Date: Fri, 24 Jul 2026 17:33:22 -0700 Subject: [PATCH 7/7] reduce colors --- .../features/archived-items/crawl-status.ts | 9 ++++--- .../crawls/crawl-list/crawl-list-item.ts | 24 +++++++++++-------- frontend/src/utils/crawler.ts | 5 ++++ 3 files changed, 25 insertions(+), 13 deletions(-) diff --git a/frontend/src/features/archived-items/crawl-status.ts b/frontend/src/features/archived-items/crawl-status.ts index 70ad88ffd1..008af754a8 100644 --- a/frontend/src/features/archived-items/crawl-status.ts +++ b/frontend/src/features/archived-items/crawl-status.ts @@ -219,7 +219,8 @@ export class CrawlStatus extends TailwindElement { break; case "pending-wait": - color = "var(--sl-color-violet-600)"; + color = "var(--sl-color-violet-500)"; + darkerColor = "var(--sl-color-violet-600)"; icon = html`) { + return (RUNNING_STATES as readonly (typeof state)[]).includes(state); +} + export function isActive({ state }: Partial) { return (activeCrawlStates as readonly (typeof state)[]).includes(state); }