diff --git a/frontend/docs/docs/user-guide/crawl-workflows.md b/frontend/docs/docs/user-guide/crawl-workflows.md index f8f9f27745..1f9e6d6a43 100644 --- a/frontend/docs/docs/user-guide/crawl-workflows.md +++ b/frontend/docs/docs/user-guide/crawl-workflows.md @@ -78,3 +78,9 @@ Workflow runs may be automatically paused, stopped, or skipped due to an enforce | **Storage Quota Reached** | Disk space allocated for the org is full. | | **Time Quota Reached** | All execution time allocated for the org has been spent. | | **Crawling Disabled** | Crawling has been disabled for the entire org. | + +## Crawl Run History + +A list of all crawl runs are shown in the **Crawls** section of the workflow. Selecting the latest crawl run will take you to the **[Watch](running-crawl.md#watch-crawl)** section of the workflow. Selecting a finished crawl will take you to an overview of the archived item where you can review and replay the crawl. + +To delete a crawl run and its associated archived item, select the :bootstrap-three-dots-vertical: action menu button in the row that the crawl is displayed and choose _Delete Crawl_. The crawl run and archived item will be permanently deleted from the organization and will no longer be visible in workflows or collections. diff --git a/frontend/docs/docs/user-guide/running-crawl.md b/frontend/docs/docs/user-guide/running-crawl.md index 80570db9b3..16afbaa297 100644 --- a/frontend/docs/docs/user-guide/running-crawl.md +++ b/frontend/docs/docs/user-guide/running-crawl.md @@ -2,10 +2,12 @@ Running crawls can be modified from the crawl workflow **Latest Crawl** tab. You may want to modify a running crawl if you find that the workflow is crawling pages that you didn't intend to archive, or if you want a boost of speed. -## Crawl Workflow Status +## Crawl Status When a workflow run is initiated, the workflow status changes to :bootstrap-hourglass-split: Waiting or :btrix-status-dot: Starting, depending on whether the conditions for starting a crawl are in place (such as resource capacity.) The workflow status will change to :btrix-status-dot: Running once the crawler loads the first crawl URL. +A running crawl in the **Crawling** > **Crawl Runs** list and [workflow > **Crawls**](./crawl-workflows.md#crawl-run-history) list will display a progress indicator in the place of a finish date. The progress indicator is based on the ratio of crawled to found pages. The indicator should be used as a rough estimate for how far along the crawl is; the ratio may change over the course of the crawl as the crawler discovers more pages, or if [page exclusions change](#live-exclusion-editing). + ## Watch Crawl You can watch the current state of the browser windows as the crawler visits pages in the **Watch** tab of **Latest Crawl**. A list of queued URLs are displayed below in the **Upcoming Pages** section. diff --git a/frontend/src/components/ui/desc-list.ts b/frontend/src/components/ui/desc-list.ts index 1c415a6544..878f4b7e16 100644 --- a/frontend/src/components/ui/desc-list.ts +++ b/frontend/src/components/ui/desc-list.ts @@ -1,6 +1,7 @@ import { css, html, LitElement } from "lit"; import { customElement, property } from "lit/decorators.js"; import { classMap } from "lit/directives/class-map.js"; +import { ifDefined } from "lit/directives/if-defined.js"; /** * Styled
,
and
for displaying data @@ -59,11 +60,19 @@ export class DescListItem extends LitElement { @property({ type: String }) label = ""; + /** + * Value will update intermittently after page load + */ + @property({ type: Boolean }) + live = false; + render() { return html`
${this.label}
-
+
+ +
`; } diff --git a/frontend/src/features/archived-items/crawl-status.ts b/frontend/src/features/archived-items/crawl-status.ts index cf03060d29..008af754a8 100644 --- a/frontend/src/features/archived-items/crawl-status.ts +++ b/frontend/src/features/archived-items/crawl-status.ts @@ -73,8 +73,10 @@ export class CrawlStatus extends TailwindElement { icon: TemplateResult; label: string; cssColor: string; + cssDarkerColor?: string; } { let color = "var(--sl-color-neutral-400)"; + let darkerColor: string | undefined = undefined; let icon = html` renderName(workflow)); const skipped = isSkipped(this.crawl); + const canceled = this.crawl.state === "canceled"; const hasExec = Boolean(this.crawl.crawlExecSeconds); const notApplicable = html` @@ -71,11 +78,10 @@ export class CrawlListItem extends BtrixElement { return html` { if (e.target === this.dropdownMenu) { return; @@ -97,49 +103,79 @@ export class CrawlListItem extends BtrixElement { )} - ${ - this.href - ? html` - ${label} - ` - : label - } + ${this.href + ? html` + ${label} + ` + : label} - ${ - this.workflowId - ? nothing - : html` - - ${this.safeRender( - (crawl) => html` - - `, - )} - - ` - } + ${this.workflowId + ? nothing + : html` + + ${this.safeRender( + (crawl) => html` + + `, + )} + + `} - ${this.safeRender((crawl) => - crawl.finished - ? html` - - ` - : notApplicable, - )} + ${this.safeRender((crawl) => { + if (crawl.finished) { + return html` + + `; + } + + const done = +(crawl.stats?.done || 0); + const found = +(crawl.stats?.found || 0); + const ratio = done && found ? done / found : 0; + const percentage = ratio * 100; + + let indicatorColor = "var(--sl-color-neutral-400)"; + let borderColor = "var(--sl-color-neutral-500)"; + + if (isRunning(crawl)) { + const status = CrawlStatus.getContent({ + state: crawl.state, + }); + indicatorColor = status.cssColor; + borderColor = status.cssDarkerColor || status.cssColor; + } + + return html` + + `; + })} ${this.safeRender((crawl) => @@ -152,16 +188,33 @@ export class CrawlListItem extends BtrixElement { ${this.safeRender((crawl) => { - if (hasExec) { - const pagesComplete = crawl.finished - ? crawl.pageCount - ? +crawl.pageCount - : 0 - : +(crawl.stats?.done || 0); - - return this.localize.number(pagesComplete, { - notation: "compact", - }); + if (hasExec && !canceled) { + if (crawl.finished) { + return this.localize.number(crawl.pageCount || 0, { + notation: "compact", + }); + } + + const done = +(crawl.stats?.done || 0); + const found = +(crawl.stats?.found || 0); + + return html` + + ${this.localize.number(done, { + notation: "compact", + })} + ${textSeparator(tw`text-neutral-500`)} + ${this.localize.number(found, { + notation: "compact", + })} + + `; } return notApplicable; @@ -169,7 +222,7 @@ export class CrawlListItem extends BtrixElement { ${this.safeRender((crawl) => - hasExec + hasExec && !canceled ? this.localize.bytes( crawl.finished ? crawl.fileSize || 0 @@ -178,18 +231,6 @@ export class CrawlListItem extends BtrixElement { : notApplicable, )} - - ${this.safeRender( - (crawl) => - html``, - )} - ${this.renderActions()} diff --git a/frontend/src/features/crawls/crawl-list/crawl-list.ts b/frontend/src/features/crawls/crawl-list/crawl-list.ts index fd831a0e9a..2ccb927d80 100644 --- a/frontend/src/features/crawls/crawl-list/crawl-list.ts +++ b/frontend/src/features/crawls/crawl-list/crawl-list.ts @@ -55,20 +55,22 @@ export class CrawlList extends TailwindElement { @property({ type: String }) workflowId?: string; + @property({ type: Boolean, noAccessor: true }) + runningOnly = false; + @queryAssignedElements({ selector: "btrix-crawl-list-item" }) listItems!: HTMLElement[]; render() { const columns = [ "min-content [clickable-start]", - this.workflowId ? undefined : "minmax(22ch, 36ch)", // Name - "minmax(min-content, 22ch)", // Started - "minmax(min-content, 22ch)", // Finished + this.workflowId ? undefined : "minmax(24ch, 40ch)", // Name + "minmax(min-content, 24ch)", // Started + "minmax(min-content, 24ch)", // Finished/Progress "1fr", // Execution time "1fr", // Pages "1fr", // Size - "[clickable-end] minmax(max-content, 20ch)", // Run by - "min-content", + "[clickable-end] min-content", // Actions ] .filter((v) => v) .join(" "); @@ -89,19 +91,16 @@ export class CrawlList extends TailwindElement { ${msg("Name")} `} + ${msg("Started")} - ${msg("Date Started")} - - - ${msg("Date Finished")} + ${this.runningOnly ? msg("Progress") : msg("Finished")} ${msg("Exec Time")} ${msg("Pages")} ${msg("Size")} - ${msg("Run By")} - + ${msg("Row actions")} diff --git a/frontend/src/layouts/separator.ts b/frontend/src/layouts/separator.ts index 1ab2bd17ba..785dca051e 100644 --- a/frontend/src/layouts/separator.ts +++ b/frontend/src/layouts/separator.ts @@ -1,12 +1,12 @@ import { html } from "lit"; +import { tw } from "@/utils/tailwind"; + /** * For separatoring text in the same line, e.g. for breadcrumbs or item details */ -export function textSeparator() { - return html`/ `; } diff --git a/frontend/src/pages/crawls.ts b/frontend/src/pages/crawls.ts index df03be1222..4be25857ef 100644 --- a/frontend/src/pages/crawls.ts +++ b/frontend/src/pages/crawls.ts @@ -15,7 +15,12 @@ import type { Crawl } from "@/types/crawler"; import type { CrawlState } from "@/types/crawlState"; import { activeCrawlStates, isActive } from "@/utils/crawler"; -type SortField = "started" | "firstSeed" | "fileSize"; +type SortField = + | "started" + | "crawlExecSeconds" + | "firstSeed" + | "pageCount" + | "fileSize"; type SortDirection = "asc" | "desc"; const sortableFields: Record< SortField, @@ -25,10 +30,18 @@ const sortableFields: Record< label: msg("Date Started"), defaultDirection: "desc", }, + crawlExecSeconds: { + label: msg("Execution Time"), + defaultDirection: "desc", + }, firstSeed: { label: msg("Crawl Start URL"), defaultDirection: "desc", }, + pageCount: { + label: msg("Pages Crawled"), + defaultDirection: "desc", + }, fileSize: { label: msg("Size"), defaultDirection: "desc", @@ -266,7 +279,7 @@ export class Crawls extends BtrixElement { if (!this.crawls) return; return html` - + ${this.crawls.items.map(this.renderCrawlItem)} `; diff --git a/frontend/src/pages/org/crawls.ts b/frontend/src/pages/org/crawls.ts index d2a9309010..5a04dd2af6 100644 --- a/frontend/src/pages/org/crawls.ts +++ b/frontend/src/pages/org/crawls.ts @@ -61,7 +61,7 @@ const sortableFields: Record< defaultDirection: "desc", }, pageCount: { - label: msg("Pages"), + label: msg("Pages Crawled"), defaultDirection: "desc", }, fileSize: { diff --git a/frontend/src/pages/org/workflow-detail.ts b/frontend/src/pages/org/workflow-detail.ts index 1fde4ac6dd..67c798f97a 100644 --- a/frontend/src/pages/org/workflow-detail.ts +++ b/frontend/src/pages/org/workflow-detail.ts @@ -2,7 +2,6 @@ import { consume } from "@lit/context"; import { localized, msg, str } from "@lit/localize"; import { Task, TaskStatus } from "@lit/task"; import type { SlDropdown } from "@shoelace-style/shoelace"; -import clsx from "clsx"; import { html, nothing, type PropertyValues, type TemplateResult } from "lit"; import { customElement, property, query, state } from "lit/decorators.js"; import { choose } from "lit/directives/choose.js"; @@ -1149,7 +1148,10 @@ export class WorkflowDetail extends BtrixElement { renderContent: (workflow: Workflow) => TemplateResult | string | number, ) { return html` - + ${when( this.workflow, renderContent, @@ -1160,6 +1162,8 @@ export class WorkflowDetail extends BtrixElement { } private renderCrawls() { + const showReplay = !this.isRunning; + return html`
- ${when( - this.workflow?.isCrawlRunning, - () => - html`
- - ${this.isRunning - ? msg("Workflow crawl is currently in progress.") - : msg("This workflow has an active crawl.")} - - ${this.isRunning ? msg("Watch Crawl") : msg("View Crawl")} - - -
`, - )} -
${when( @@ -1228,14 +1213,9 @@ export class WorkflowDetail extends BtrixElement { crawls.items.map( (crawl: Crawl) => html` @@ -1622,34 +1602,27 @@ export class WorkflowDetail extends BtrixElement { }); }; - const pages = (workflow: Workflow) => { + const crawledPages = (workflow: Workflow) => { if (!latestCrawl) return skeleton; if (workflow.isCrawlRunning) { - return [ - this.localize.number(+(latestCrawl.stats?.done || 0)), - this.localize.number(+(latestCrawl.stats?.found || 0)), - ].join(` ${msg("of")} `); + return this.localize.number(+(latestCrawl.stats?.done || 0)); } return this.localize.number(latestCrawl.pageCount || 0); }; + const foundPages = () => { + if (!latestCrawl) return skeleton; + + const found = +(latestCrawl.stats?.found || 0); + + return this.localize.number(found); + }; + const qa = (workflow: Workflow) => { if (!latestCrawl) return html``; - if (workflow.isCrawlRunning) { - return html` - ${noData} - - - - `; - } - if (!isSuccessfullyFinished({ state: workflow.lastCrawlState })) { return notApplicable; } @@ -1695,13 +1668,18 @@ export class WorkflowDetail extends BtrixElement { )}` : execTime(), )} - ${this.renderDetailItem(msg("Pages Crawled"), pages)} + ${this.renderDetailItem(msg("Pages Crawled"), crawledPages)} + ${this.workflow && this.workflow.isCrawlRunning + ? this.renderDetailItem(msg("Pages Found"), foundPages) + : nothing} ${this.renderDetailItem(msg("Size"), (workflow) => this.localize.bytes(workflow.lastCrawlSize || 0, { unitDisplay: "narrow", }), )} - ${this.renderDetailItem(msg("QA Rating"), qa)} + ${this.workflow && !this.workflow.isCrawlRunning + ? this.renderDetailItem(msg("QA Rating"), qa) + : nothing} `; };