Merge pull request #18 from shellui-dev/feat/search-body-indexing

feat: search body-text indexing with snippet extraction

Author: Shewart

src/ShellDocs.Components/Chrome/SearchDialog.razor

@@ -53,6 +53,10 @@
                                 {
                                     <span class="search-result-section">@r.Entry.Section</span>
                                 }
+                                @if (!string.IsNullOrEmpty(r.Snippet))
+                                {
+                                    <span class="search-result-snippet">@r.Snippet</span>
+                                }
                             </span>
                         </button>
                     }
@@ -158,47 +162,78 @@
         if (_highlightedIndex >= _results.Count) _highlightedIndex = 0;
     }
 
-    /* Naive fuzzy score: substring hit boosts by big margin, then per-word
-       token overlap. Good enough for the ~100-entry indexes docs sites have. */
     private static IEnumerable<Scored> Rank(string query, IReadOnlyList<SearchEntry> entries)
     {
         if (string.IsNullOrWhiteSpace(query))
         {
-            return entries.Take(20).Select(e => new Scored(e, 0));
+            return entries.Take(20).Select(e => new Scored(e, 0, null));
         }
         var q = query.Trim().ToLowerInvariant();
         var tokens = q.Split(' ', StringSplitOptions.RemoveEmptyEntries);
         return entries
-            .Select(e => new Scored(e, Score(e, q, tokens)))
+            .Select(e => Evaluate(e, q, tokens))
             .Where(s => s.Score > 0)
             .OrderByDescending(s => s.Score);
     }
 
-    private static int Score(SearchEntry e, string q, string[] tokens)
+    private static Scored Evaluate(SearchEntry e, string q, string[] tokens)
     {
         var title = e.Title.ToLowerInvariant();
         var desc  = (e.Description ?? "").ToLowerInvariant();
         var sec   = (e.Section ?? "").ToLowerInvariant();
+        var body  = (e.Body ?? "").ToLowerInvariant();
         var score = 0;
-        // Exact substring in title: massive boost.
-        if (title.Contains(q)) score += 100;
-        if (title.StartsWith(q)) score += 50;
-        if (sec.Contains(q))   score += 20;
-        if (desc.Contains(q))  score += 10;
-        // Per-token AND — every token must appear somewhere.
+        var bodyHit = false;
+
+        if (title.Contains(q))    score += 100;
+        if (title.StartsWith(q))  score += 50;
+        if (sec.Contains(q))      score += 20;
+        if (desc.Contains(q))     score += 10;
+        if (body.Contains(q))     { score += 6; bodyHit = true; }
+
         foreach (var t in tokens)
         {
-            if (title.Contains(t)) score += 8;
-            else if (sec.Contains(t)) score += 4;
-            else if (desc.Contains(t)) score += 2;
-            else return 0; // hard reject: token missing
+            if (title.Contains(t))      score += 8;
+            else if (sec.Contains(t))   score += 4;
+            else if (desc.Contains(t))  score += 2;
+            else if (body.Contains(t))  { score += 1; bodyHit = true; }
+            else return new Scored(e, 0, null); // hard reject: token missing everywhere
         }
-        // Pages rank slightly higher than headings when otherwise-equal.
+
         if (e.Kind == SearchEntryKind.Page) score += 2;
-        return score;
+
+        // Only surface a snippet when the match came from the body (title/desc
+        // is already shown; snippet is only useful for otherwise-hidden matches).
+        var snippet = bodyHit && !string.IsNullOrEmpty(e.Body)
+            ? BuildSnippet(e.Body!, tokens)
+            : null;
+
+        return new Scored(e, score, snippet);
+    }
+
+    /* Center the snippet on the first token match, ~150 chars, with ellipses
+       when trimmed. Case-preserving. */
+    private static string? BuildSnippet(string body, string[] tokens)
+    {
+        var lower = body.ToLowerInvariant();
+        var hit = -1;
+        foreach (var t in tokens)
+        {
+            var idx = lower.IndexOf(t, StringComparison.Ordinal);
+            if (idx >= 0 && (hit < 0 || idx < hit)) hit = idx;
+        }
+        if (hit < 0) return null;
+
+        const int radius = 75;
+        var start = Math.Max(0, hit - radius);
+        var end = Math.Min(body.Length, hit + radius);
+        var snippet = body[start..end].Trim();
+        if (start > 0) snippet = "… " + snippet;
+        if (end < body.Length) snippet += " …";
+        return snippet;
     }
 
-    private record Scored(SearchEntry Entry, int Score);
+    private record Scored(SearchEntry Entry, int Score, string? Snippet);
 
     public void Dispose()
     {

src/ShellDocs.Components/Chrome/SearchDialog.razor.css

@@ -130,6 +130,15 @@
     text-overflow: ellipsis;
     white-space: nowrap;
 }
+.search-result-snippet {
+    font-size: 0.75rem;
+    color: var(--muted-foreground);
+    line-height: 1.45;
+    display: -webkit-box;
+    -webkit-line-clamp: 2;
+    -webkit-box-orient: vertical;
+    overflow: hidden;
+}
 
 .search-footer {
     display: flex;

src/ShellDocs.Core/MarkdownPlainText.cs

@@ -0,0 +1,41 @@
+using System.Text.RegularExpressions;
+
+namespace ShellDocs.Core;
+
+/* Extracts plain text from markdown for search-body indexing. Strips YAML
+   frontmatter, fenced code blocks, razor component tags, inline HTML, and
+   the surface markdown syntax (headings, emphasis, links, images). Preserves
+   the actual prose so token matching finds body-only hits. */
+public static class MarkdownPlainText
+{
+    private static readonly Regex Frontmatter = new(@"^---\s*\r?\n[\s\S]*?\r?\n---\s*\r?\n", RegexOptions.Compiled);
+    private static readonly Regex FencedBlock = new(@"^```[\s\S]*?^```", RegexOptions.Multiline | RegexOptions.Compiled);
+    private static readonly Regex HtmlTag     = new(@"<[^>]+>", RegexOptions.Compiled);
+    private static readonly Regex Image       = new(@"!\[([^\]]*)\]\([^\)]*\)", RegexOptions.Compiled);
+    private static readonly Regex Link        = new(@"\[([^\]]+)\]\([^\)]*\)", RegexOptions.Compiled);
+    private static readonly Regex InlineCode  = new(@"`([^`]+)`", RegexOptions.Compiled);
+    private static readonly Regex Emphasis    = new(@"(\*\*|__|\*|_)(.+?)\1", RegexOptions.Compiled);
+    private static readonly Regex HeadingHash = new(@"^#{1,6}\s+", RegexOptions.Multiline | RegexOptions.Compiled);
+    private static readonly Regex ListMarker  = new(@"^\s*[-*+]\s+|^\s*\d+\.\s+", RegexOptions.Multiline | RegexOptions.Compiled);
+    private static readonly Regex Blockquote  = new(@"^>\s?", RegexOptions.Multiline | RegexOptions.Compiled);
+    private static readonly Regex WhitespaceRun = new(@"\s+", RegexOptions.Compiled);
+
+    public static string Extract(string markdown, int maxLength = 8000)
+    {
+        if (string.IsNullOrEmpty(markdown)) return "";
+
+        var text = Frontmatter.Replace(markdown, "");
+        text = FencedBlock.Replace(text, " ");
+        text = HtmlTag.Replace(text, " ");
+        text = Image.Replace(text, "$1");
+        text = Link.Replace(text, "$1");
+        text = InlineCode.Replace(text, "$1");
+        text = Emphasis.Replace(text, "$2");
+        text = HeadingHash.Replace(text, "");
+        text = ListMarker.Replace(text, "");
+        text = Blockquote.Replace(text, "");
+        text = WhitespaceRun.Replace(text, " ").Trim();
+
+        return text.Length > maxLength ? text[..maxLength] : text;
+    }
+}

src/ShellDocs.Core/SearchIndex.cs

@@ -4,8 +4,8 @@ namespace ShellDocs.Core;
 
 /* An in-memory search index built from the navigation graph. Each entry
    represents one searchable thing — a page, or a heading within a page.
-   Client-side fuzzy match runs against Title + Description + Section over the
-   wire; body-text indexing lands with the search-index.json build step. */
+   Page entries carry a trimmed plain-text body so client-side match can
+   find hits that aren't in the title/description surface. */
 public sealed class SearchIndex
 {
     public IReadOnlyList<SearchEntry> Entries { get; }
@@ -28,7 +28,8 @@ private static void Walk(NavigationNode node, string? section, List<SearchEntry>
                 Title: node.Title,
                 Description: node.Description,
                 Section: section,
-                Kind: SearchEntryKind.Page));
+                Kind: SearchEntryKind.Page,
+                Body: ExtractBodyFromFile(node.Path)));
 
             // Prefer headings already extracted at render time; otherwise pull
             // them from the source markdown ourselves so the index isn't blank
@@ -75,6 +76,12 @@ private static IReadOnlyList<Heading> ExtractHeadingsFromFile(string? path)
         return list;
     }
 
+    private static string? ExtractBodyFromFile(string? path)
+    {
+        if (string.IsNullOrEmpty(path) || !File.Exists(path)) return null;
+        return MarkdownPlainText.Extract(File.ReadAllText(path));
+    }
+
     private static string Slugify(string text)
     {
         var lowered = text.ToLowerInvariant();
@@ -97,6 +104,7 @@ public record SearchEntry(
     string Title,
     string? Description,
     string? Section,
-    SearchEntryKind Kind);
+    SearchEntryKind Kind,
+    string? Body = null);
 
 public enum SearchEntryKind { Page, Heading }

tests/ShellDocs.Tests/MarkdownPlainTextTests.cs

@@ -0,0 +1,80 @@
+using ShellDocs.Core;
+using Xunit;
+
+namespace ShellDocs.Tests;
+
+public class MarkdownPlainTextTests
+{
+    [Fact]
+    public void Extract_StripsFrontmatter()
+    {
+        var text = MarkdownPlainText.Extract("---\ntitle: Foo\n---\nHello world");
+        Assert.Equal("Hello world", text);
+    }
+
+    [Fact]
+    public void Extract_StripsFencedCodeBlocks()
+    {
+        var md = "Prose before.\n\n```csharp\nvar x = 1;\n```\n\nProse after.";
+        var text = MarkdownPlainText.Extract(md);
+        Assert.Contains("Prose before", text);
+        Assert.Contains("Prose after", text);
+        Assert.DoesNotContain("var x = 1", text);
+    }
+
+    [Fact]
+    public void Extract_StripsRazorComponentTags()
+    {
+        var text = MarkdownPlainText.Extract("Before <Callout Title=\"x\">body</Callout> after.");
+        Assert.DoesNotContain("<Callout", text);
+        Assert.Contains("body", text);
+        Assert.Contains("Before", text);
+        Assert.Contains("after", text);
+    }
+
+    [Fact]
+    public void Extract_UnwrapsLinksAndImages()
+    {
+        var text = MarkdownPlainText.Extract("See [our docs](https://x.com) and ![alt](/img.png).");
+        Assert.Contains("our docs", text);
+        Assert.Contains("alt", text);
+        Assert.DoesNotContain("https://", text);
+        Assert.DoesNotContain("img.png", text);
+    }
+
+    [Fact]
+    public void Extract_UnwrapsEmphasisAndInlineCode()
+    {
+        var text = MarkdownPlainText.Extract("This is **bold**, *italic*, and `code`.");
+        Assert.Contains("bold", text);
+        Assert.Contains("italic", text);
+        Assert.Contains("code", text);
+        Assert.DoesNotContain("**", text);
+        Assert.DoesNotContain("`", text);
+    }
+
+    [Fact]
+    public void Extract_StripsHeadingHashesButKeepsText()
+    {
+        var text = MarkdownPlainText.Extract("## Setup\n\n### Install\n\nRun the CLI.");
+        Assert.Contains("Setup", text);
+        Assert.Contains("Install", text);
+        Assert.Contains("Run the CLI", text);
+        Assert.DoesNotContain("##", text);
+    }
+
+    [Fact]
+    public void Extract_TrimsAtMaxLength()
+    {
+        var long_ = new string('a', 10_000);
+        var text = MarkdownPlainText.Extract(long_, maxLength: 500);
+        Assert.Equal(500, text.Length);
+    }
+
+    [Fact]
+    public void Extract_HandlesEmptyInput()
+    {
+        Assert.Equal("", MarkdownPlainText.Extract(""));
+        Assert.Equal("", MarkdownPlainText.Extract(null!));
+    }
+}

tests/ShellDocs.Tests/SearchIndexTests.cs

@@ -75,4 +75,26 @@ public void FromGraph_PageEntryCarriesDescription()
         var page = index.Entries.First(e => e.Kind == SearchEntryKind.Page);
         Assert.Equal("Get started with ShellDocs", page.Description);
     }
+
+    [Fact]
+    public void FromGraph_PageEntryCarriesExtractedBody()
+    {
+        WritePage("intro.md", "Introduction",
+            bodyHeadings: "## Setup\n\nRun the CLI to scaffold a new project.");
+        var graph = NavigationGraphBuilder.Build(_root);
+        var index = SearchIndex.FromGraph(graph);
+        var page = index.Entries.First(e => e.Kind == SearchEntryKind.Page);
+        Assert.NotNull(page.Body);
+        Assert.Contains("Run the CLI", page.Body);
+    }
+
+    [Fact]
+    public void FromGraph_HeadingEntriesHaveNullBody()
+    {
+        WritePage("intro.md", "Introduction", bodyHeadings: "## Setup\n\nBody prose.");
+        var graph = NavigationGraphBuilder.Build(_root);
+        var index = SearchIndex.FromGraph(graph);
+        var heading = index.Entries.First(e => e.Kind == SearchEntryKind.Heading);
+        Assert.Null(heading.Body);
+    }
 }