From f17678cfb0f55815ac729641c0636bd73b002b4a Mon Sep 17 00:00:00 2001
From: Hiroshi SHIBATA <hsbt@ruby-lang.org>
Date: Tue, 28 Jul 2026 12:14:43 +0900
Subject: [PATCH] Read the Pagefind entry as UTF-8

Pagefind lists the word characters it splits on in `include_characters`,
and they reach past ASCII. On a build host that leaves the locale at
POSIX, Cloudflare Pages among them, `File.read` hands back a US-ASCII
string and the build dies in `JSON.parse` on the first of those bytes.

Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
---
 lib/search_index.rb       |  7 ++++++-
 test/test_search_index.rb | 31 +++++++++++++++++++++++++++++--
 2 files changed, 35 insertions(+), 3 deletions(-)

diff --git a/lib/search_index.rb b/lib/search_index.rb
index 5d8be4276d..966a5fd7e0 100644
--- a/lib/search_index.rb
+++ b/lib/search_index.rb
@@ -86,8 +86,13 @@ def index_key(lang)
     lang.tr("_", "-").downcase
   end
 
+  # The entry is UTF-8 no matter what locale the build runs under: Pagefind
+  # lists the word characters it splits on in `include_characters`, and those
+  # reach past ASCII. Build hosts that leave the locale at POSIX, Cloudflare
+  # Pages among them, give `File.read` a US-ASCII string, and JSON.parse raises
+  # on the first of those bytes as it converts the source to UTF-8.
   def read_entry
-    JSON.parse(File.read(@entry_path))
+    JSON.parse(File.read(@entry_path, encoding: Encoding::UTF_8))
   rescue Errno::ENOENT
     raise Error, "#{@entry_path} is missing, Pagefind wrote no index"
   rescue JSON::ParserError => e
diff --git a/test/test_search_index.rb b/test/test_search_index.rb
index f37e386726..bbfbe238d8 100644
--- a/test/test_search_index.rb
+++ b/test/test_search_index.rb
@@ -17,12 +17,31 @@
     teardown_tempdir
   end
 
+  # `include_characters` is the list of characters Pagefind counts as part of a
+  # word. It reaches past ASCII, which is what makes the entry a UTF-8 file.
   def write_entry(languages)
-    File.write(@entry_path, JSON.generate({ "version" => "1.5.2", "languages" => languages }))
+    entry = {
+      "version" => "1.5.2",
+      "languages" => languages,
+      "include_characters" => ["_", "‿", "⁀", "＿"],
+    }
+    File.write(@entry_path, JSON.generate(entry))
   end
 
   def read_entry
-    JSON.parse(File.read(@entry_path))
+    JSON.parse(File.read(@entry_path, encoding: Encoding::UTF_8))
+  end
+
+  # Stands in for a build host that leaves the locale at POSIX, which is what
+  # Cloudflare Pages does.
+  def with_default_external(encoding)
+    original = Encoding.default_external
+    verbose, $VERBOSE = $VERBOSE, nil
+    Encoding.default_external = encoding
+    yield
+  ensure
+    Encoding.default_external = original
+    $VERBOSE = verbose
   end
 
   def search_index(pagefind: SearchIndex::PAGEFIND)
@@ -148,6 +167,14 @@ def stub_pagefind(status)
       _(error.message).must_match(/missing index "en"/)
     end
 
+    it "reads the entry as UTF-8 whatever locale the build runs under" do
+      write_entry({ "en" => { "hash" => "en_abc", "wasm" => "en", "page_count" => 560 } })
+
+      with_default_external(Encoding::US_ASCII) { search_index.apply_fallbacks }
+
+      _(read_entry["languages"]["bg"]["hash"]).must_equal "en_abc"
+    end
+
     it "raises when the bundle has no languages" do
       write_entry({})