feat: Show crawl page progress

frontend/docs/docs/user-guide/crawl-workflows.md

@@ -78,3 +78,9 @@ Workflow runs may be automatically paused, stopped, or skipped due to an enforce
 | **Storage Quota Reached** | Disk space allocated for the org is full. |
 | **Time Quota Reached** | All execution time allocated for the org has been spent. |
 | **Crawling Disabled** | Crawling has been disabled for the entire org. |
+
+## Crawl Run History
+
+A list of all crawl runs are shown in the **Crawls** section of the workflow. Selecting the latest crawl run will take you to the **[Watch](running-crawl.md#watch-crawl)** section of the workflow. Selecting a finished crawl will take you to an overview of the archived item where you can review and replay the crawl.
+
+To delete a crawl run and its associated archived item, select the :bootstrap-three-dots-vertical: action menu button in the row that the crawl is displayed and choose _Delete Crawl_. The crawl run and archived item will be permanently deleted from the organization and will no longer be visible in workflows or collections.

frontend/docs/docs/user-guide/running-crawl.md

@@ -2,10 +2,12 @@
 
 Running crawls can be modified from the crawl workflow **Latest Crawl** tab. You may want to modify a running crawl if you find that the workflow is crawling pages that you didn't intend to archive, or if you want a boost of speed.
 
-## Crawl Workflow Status
+## Crawl Status
 
 When a workflow run is initiated, the workflow status changes to <span class="status-violet-600">:bootstrap-hourglass-split: Waiting</span> or <span class="status-violet-600">:btrix-status-dot: Starting</span>, depending on whether the conditions for starting a crawl are in place (such as resource capacity.) The workflow status will change to <span class="status-green-600">:btrix-status-dot: Running</span> once the crawler loads the first crawl URL.
 
+A running crawl in the **Crawling** > **Crawl Runs** list and [workflow > **Crawls**](./crawl-workflows.md#crawl-run-history) list will display a progress indicator in the place of a finish date. The progress indicator is based on the ratio of crawled to found pages. The indicator should be used as a rough estimate for how far along the crawl is; the ratio may change over the course of the crawl as the crawler discovers more pages, or if [page exclusions change](#live-exclusion-editing).
+
 ## Watch Crawl
 
 You can watch the current state of the browser windows as the crawler visits pages in the **Watch** tab of **Latest Crawl**. A list of queued URLs are displayed below in the **Upcoming Pages** section.

frontend/src/components/ui/desc-list.ts

@@ -1,6 +1,7 @@
 import { css, html, LitElement } from "lit";
 import { customElement, property } from "lit/decorators.js";
 import { classMap } from "lit/directives/class-map.js";
+import { ifDefined } from "lit/directives/if-defined.js";
 
 /**
  * Styled <dl>, <dt> and <dd> for displaying data
@@ -59,11 +60,19 @@ export class DescListItem extends LitElement {
   @property({ type: String })
   label = "";
 
+  /**
+   * Value will update intermittently after page load
+   */
+  @property({ type: Boolean })
+  live = false;
+
   render() {
     return html`<div class="item">
       <div class="content">
         <dt>${this.label}<slot name="label"></slot></dt>
-        <dd><slot></slot></dd>
+        <dd aria-live=${ifDefined(this.live ? "polite" : undefined)}>
+          <slot></slot>
+        </dd>
       </div>
     </div>`;
   }

frontend/src/features/archived-items/crawl-status.ts

@@ -73,8 +73,10 @@ export class CrawlStatus extends TailwindElement {
     icon: TemplateResult;
     label: string;
     cssColor: string;
+    cssDarkerColor?: string;
   } {
     let color = "var(--sl-color-neutral-400)";
+    let darkerColor: string | undefined = undefined;
     let icon = html`<sl-icon
       name="slash-circle"
       class="neutral"
@@ -138,7 +140,8 @@ export class CrawlStatus extends TailwindElement {
         break;
 
       case "running":
-        color = "var(--success)";
+        color = "var(--sl-color-success-500)";
+        darkerColor = "var(--sl-color-success-600)";
         icon = html`<sl-icon
           name="dot"
           library="app"
@@ -150,7 +153,8 @@ export class CrawlStatus extends TailwindElement {
         break;
 
       case "rate-limited":
-        color = "var(--warning)";
+        color = "var(--sl-color-warning-500)";
+        darkerColor = "var(--sl-color-warning-600)";
         icon = html`<sl-icon
           name="dot"
           library="app"
@@ -215,7 +219,8 @@ export class CrawlStatus extends TailwindElement {
         break;
 
       case "pending-wait":
-        color = "var(--sl-color-violet-600)";
+        color = "var(--sl-color-violet-500)";
+        darkerColor = "var(--sl-color-violet-600)";
         icon = html`<sl-icon
           name="dot"
           library="app"
@@ -227,7 +232,8 @@ export class CrawlStatus extends TailwindElement {
         break;
 
       case "generate-wacz":
-        color = "var(--sl-color-violet-600)";
+        color = "var(--sl-color-violet-500)";
+        darkerColor = "var(--sl-color-violet-600)";
         icon = html`<sl-icon
           name="dot"
           library="app"
@@ -239,7 +245,8 @@ export class CrawlStatus extends TailwindElement {
         break;
 
       case "uploading-wacz":
-        color = "var(--sl-color-violet-600)";
+        color = "var(--sl-color-violet-500)";
+        darkerColor = "var(--sl-color-violet-600)";
         icon = html`<sl-icon
           name="dot"
           library="app"
@@ -335,6 +342,7 @@ export class CrawlStatus extends TailwindElement {
       icon,
       label: `${label}${reason ? `: ${reason}` : ""}${substate ? ` (${substate})` : ""}`,
       cssColor: color,
+      cssDarkerColor: darkerColor,
     };
   }
 

frontend/src/features/crawls/crawl-list/crawl-list-item.ts

@@ -1,12 +1,18 @@
 import { localized, msg } from "@lit/localize";
+import clsx from "clsx";
 import { css, html, nothing, type TemplateResult } from "lit";
 import { customElement, property, query } from "lit/decorators.js";
+import { ifDefined } from "lit/directives/if-defined.js";
+import { styleMap } from "lit/directives/style-map.js";
 
 import { BtrixElement } from "@/classes/BtrixElement";
 import type { OverflowDropdown } from "@/components/ui/overflow-dropdown";
+import { CrawlStatus } from "@/features/archived-items/crawl-status";
+import { textSeparator } from "@/layouts/separator";
 import type { Crawl } from "@/types/crawler";
-import { isSkipped, renderName } from "@/utils/crawler";
+import { isRunning, isSkipped, renderName } from "@/utils/crawler";
 import { humanizeExecutionSeconds } from "@/utils/executionTimeFormatter";
+import { tw } from "@/utils/tailwind";
 
 /**
  * @slot menu
@@ -64,18 +70,18 @@ export class CrawlListItem extends BtrixElement {
       : this.safeRender((workflow) => renderName(workflow));
 
     const skipped = isSkipped(this.crawl);
+    const canceled = this.crawl.state === "canceled";
     const hasExec = Boolean(this.crawl.crawlExecSeconds);
     const notApplicable = html`<sl-tooltip content=${msg("Not applicable")}>
       <sl-icon name="slash" class="text-base text-neutral-400"></sl-icon>
     </sl-tooltip>`;
 
     return html`
       <btrix-table-row
-        class=${
-          this.href
-            ? "cursor-pointer select-none transition-colors hover:bg-neutral-50 focus-within:bg-neutral-50 duration-fast"
-            : ""
-        }
+        class=${clsx(
+          this.href &&
+            tw`cursor-pointer select-none transition-colors duration-fast focus-within:bg-neutral-50 hover:bg-neutral-50`,
+        )}
         @click=${async (e: MouseEvent) => {
           if (e.target === this.dropdownMenu) {
             return;
@@ -97,49 +103,79 @@ export class CrawlListItem extends BtrixElement {
           )}
         </btrix-table-cell>
         <btrix-table-cell rowClickTarget=${this.href ? "a" : nothing}>
-          ${
-            this.href
-              ? html`<a href=${this.href} @click=${this.navigate.link}>
-                  ${label}
-                </a>`
-              : label
-          }
+          ${this.href
+            ? html`<a href=${this.href} @click=${this.navigate.link}>
+                ${label}
+              </a>`
+            : label}
         </btrix-table-cell>
-        ${
-          this.workflowId
-            ? nothing
-            : html`
-                <btrix-table-cell>
-                  ${this.safeRender(
-                    (crawl) => html`
-                      <btrix-format-date
-                        date=${crawl.started}
-                        month="2-digit"
-                        day="2-digit"
-                        year="numeric"
-                        hour="2-digit"
-                        minute="2-digit"
-                      ></btrix-format-date>
-                    `,
-                  )}
-                </btrix-table-cell>
-              `
-        }
+        ${this.workflowId
+          ? nothing
+          : html`
+              <btrix-table-cell>
+                ${this.safeRender(
+                  (crawl) => html`
+                    <btrix-format-date
+                      date=${crawl.started}
+                      month="2-digit"
+                      day="2-digit"
+                      year="numeric"
+                      hour="2-digit"
+                      minute="2-digit"
+                    ></btrix-format-date>
+                  `,
+                )}
+              </btrix-table-cell>
+            `}
         <btrix-table-cell>
-          ${this.safeRender((crawl) =>
-            crawl.finished
-              ? html`
-                  <btrix-format-date
-                    date=${crawl.finished}
-                    month="2-digit"
-                    day="2-digit"
-                    year="numeric"
-                    hour="2-digit"
-                    minute="2-digit"
-                  ></btrix-format-date>
-                `
-              : notApplicable,
-          )}
+          ${this.safeRender((crawl) => {
+            if (crawl.finished) {
+              return html`
+                <btrix-format-date
+                  date=${crawl.finished}
+                  month="2-digit"
+                  day="2-digit"
+                  year="numeric"
+                  hour="2-digit"
+                  minute="2-digit"
+                ></btrix-format-date>
+              `;
+            }
+
+            const done = +(crawl.stats?.done || 0);
+            const found = +(crawl.stats?.found || 0);
+            const ratio = done && found ? done / found : 0;
+            const percentage = ratio * 100;
+
+            let indicatorColor = "var(--sl-color-neutral-400)";
+            let borderColor = "var(--sl-color-neutral-500)";
+
+            if (isRunning(crawl)) {
+              const status = CrawlStatus.getContent({
+                state: crawl.state,
+              });
+              indicatorColor = status.cssColor;
+              borderColor = status.cssDarkerColor || status.cssColor;
+            }
+
+            return html`<sl-tooltip
+              content=${this.localize.number(ratio, {
+                style: "percent",
+                maximumFractionDigits: 0,
+              })}
+            >
+              <sl-progress-bar
+                class="w-full"
+                style=${styleMap({
+                  "--indicator-color": indicatorColor,
+                  "--btrix-indicator-border-color": borderColor,
+                })}
+                value=${ifDefined(percentage < 1 ? undefined : percentage)}
+                label=${msg("Page Crawl Progress")}
+                ?indeterminate=${percentage < 1}
+              ></sl-progress-bar>
+            </sl-tooltip>`;
+          })}
         </btrix-table-cell>
         <btrix-table-cell>
           ${this.safeRender((crawl) =>
@@ -152,24 +188,41 @@ export class CrawlListItem extends BtrixElement {
         </btrix-table-cell>
         <btrix-table-cell>
           ${this.safeRender((crawl) => {
-            if (hasExec) {
-              const pagesComplete = crawl.finished
-                ? crawl.pageCount
-                  ? +crawl.pageCount
-                  : 0
-                : +(crawl.stats?.done || 0);
-
-              return this.localize.number(pagesComplete, {
-                notation: "compact",
-              });
+            if (hasExec && !canceled) {
+              if (crawl.finished) {
+                return this.localize.number(crawl.pageCount || 0, {
+                  notation: "compact",
+                });
+              }
+
+              const done = +(crawl.stats?.done || 0);
+              const found = +(crawl.stats?.found || 0);
+
+              return html`<sl-tooltip
+                content="${msg("Pages Crawled")} / ${msg("Pages Found")}"
+              >
+                <span class="inline-flex items-center gap-1">
+                  <span
+                    >${this.localize.number(done, {
+                      notation: "compact",
+                    })}</span
+                  >
+                  ${textSeparator(tw`text-neutral-500`)}
+                  <span class="text-neutral-500"
+                    >${this.localize.number(found, {
+                      notation: "compact",
+                    })}</span
+                  >
+                </span>
+              </sl-tooltip>`;
             }
 
             return notApplicable;
           })}
         </btrix-table-cell>
         <btrix-table-cell>
           ${this.safeRender((crawl) =>
-            hasExec
+            hasExec && !canceled
               ? this.localize.bytes(
                   crawl.finished
                     ? crawl.fileSize || 0
@@ -178,18 +231,6 @@ export class CrawlListItem extends BtrixElement {
               : notApplicable,
           )}
         </btrix-table-cell>
-        <btrix-table-cell>
-          ${this.safeRender(
-            (crawl) =>
-              html`<sl-tooltip content=${crawl.userName}
-                ><btrix-user-chip
-                  class="max-w-full"
-                  userId=${crawl.userid}
-                  userName=${crawl.userName}
-                ></btrix-user-chip
-              ></sl-tooltip>`,
-          )}
-        </btrix-table-cell>
         <btrix-table-cell class="pl-1 pr-1">
           ${this.renderActions()}
         </btrix-table-cell>