fix(embedding): fix transformers resolve in OpenCode Bun compile host (#210)

.github/workflows/embedding-backend.yml

@@ -14,8 +14,12 @@ on:
       - "bun.lock"
       - "src/services/embedding.ts"
       - "src/services/onnxruntime-resolve.ts"
+      - "src/services/runtime-require.ts"
       - "scripts/verify-embedding-backend.mjs"
       - "scripts/verify-nested-onnxruntime-fixture.mjs"
+      - "scripts/fixtures/compiled-host-entry.mjs"
+      - "tests/runtime-require.test.ts"
+      - "tests/onnxruntime-resolve.test.ts"
       - ".github/workflows/embedding-backend.yml"
   workflow_dispatch:
 
@@ -110,8 +114,11 @@ jobs:
       - name: Build package
         run: bun run build
 
-      - name: Nested fixture + embedding (Bun 1.3.14)
+      - name: Nested fixture + compiled host + embedding (Bun 1.3.14)
         run: bun scripts/verify-nested-onnxruntime-fixture.mjs
 
       - name: Nested fixture + embedding (Node 22)
+        # Node cannot Bun --compile; the compiled-host check is Bun-only above.
         run: node scripts/verify-nested-onnxruntime-fixture.mjs
+        env:
+          SKIP_COMPILE_HOST: "1"

README.md

@@ -230,7 +230,7 @@ Example — remote OpenAI embeddings:
 
 Changing `embeddingModel` (or dimensions) can trigger re-embedding of stored memories on next startup. Prefer picking a model once and sticking with it for a given data directory.
 
-**Intel Mac (`darwin/x64`):** newer `onnxruntime-node` builds may ship without an x64 native binding, so local embedding init can fail. `opencode-mem` pins `onnxruntime-node@1.22.0` and loads transformers through a CJS resolve shim so OpenCode nested installs keep that binding. If init still fails after a plugin upgrade, clear OpenCode's nested plugin cache (`~/.cache/opencode/packages/opencode-mem@*`) and reinstall, or use a remote endpoint via `embeddingApiUrl` + `embeddingApiKey` (example above).
+**Intel Mac (`darwin/x64`):** newer `onnxruntime-node` builds may ship without an x64 native binding, so local embedding init can fail. `opencode-mem` pins `onnxruntime-node@1.22.0` and loads transformers through a CJS resolve shim so OpenCode nested installs keep that binding. Transformers is resolved to an absolute path before that shim is installed so OpenCode's Bun `--compile` host does not fail with `Cannot find module '@huggingface/transformers' from ''`. If init still fails after a plugin upgrade, clear OpenCode's nested plugin cache (`~/.cache/opencode/packages/opencode-mem@*`) and reinstall, or use a remote endpoint via `embeddingApiUrl` + `embeddingApiKey` (example above).
 
 ### Memory Scope
 

scripts/fixtures/compiled-host-entry.mjs

@@ -0,0 +1,80 @@
+#!/usr/bin/env bun
+/**
+ * Minimal OpenCode-shaped Bun host entry for #210.
+ *
+ * OpenCode is shipped via Bun.build({ compile: { autoloadPackageJson: true, ... }})
+ * and dynamically imports external plugins. This entry mimics that load path.
+ *
+ * Env:
+ *   OPENCODE_MEM_PLUGIN_ENTRY — absolute file URL or path to embedding.js
+ */
+const entry = process.env.OPENCODE_MEM_PLUGIN_ENTRY;
+if (!entry) {
+  console.error("OPENCODE_MEM_PLUGIN_ENTRY is required");
+  process.exit(1);
+}
+
+const mod = await import(entry);
+if (typeof mod.loadLocalTransformersBackend !== "function") {
+  console.error("plugin entry does not export loadLocalTransformersBackend");
+  process.exit(1);
+}
+
+const transformers = await mod.loadLocalTransformersBackend();
+if (typeof transformers.pipeline !== "function" || !transformers.env) {
+  console.error("loadLocalTransformersBackend did not expose pipeline/env");
+  process.exit(1);
+}
+
+const { createRequire } = await import("node:module");
+const { dirname, join } = await import("node:path");
+const { existsSync, readFileSync } = await import("node:fs");
+const { fileURLToPath, pathToFileURL } = await import("node:url");
+
+const entryPath = entry.startsWith("file:") ? fileURLToPath(entry) : entry;
+const pluginRequire = createRequire(entryPath);
+const resolveUrl = pathToFileURL(join(dirname(entryPath), "onnxruntime-resolve.js")).href;
+const { getPinnedOnnxruntimePackageRoot, prepareOnnxruntimeForTransformers } =
+  await import(resolveUrl);
+prepareOnnxruntimeForTransformers();
+
+function pkgVersion(entryFile) {
+  let dir = dirname(entryFile);
+  for (let i = 0; i < 6; i++) {
+    const candidate = join(dir, "package.json");
+    if (existsSync(candidate)) {
+      const parsed = JSON.parse(readFileSync(candidate, "utf8"));
+      if (typeof parsed.version === "string" && parsed.version.length > 0) {
+        return parsed.version;
+      }
+    }
+    dir = dirname(dir);
+  }
+  throw new Error(`versioned package.json not found near ${entryFile}`);
+}
+
+const nodeEntry = pluginRequire.resolve("onnxruntime-node");
+const commonEntry = createRequire(nodeEntry).resolve("onnxruntime-common");
+const nodeVersion = pkgVersion(nodeEntry);
+const commonVersion = pkgVersion(commonEntry);
+const pinnedRoot = getPinnedOnnxruntimePackageRoot();
+
+if (nodeVersion !== "1.22.0") {
+  console.error(`expected onnxruntime-node@1.22.0, got ${nodeVersion} at ${nodeEntry}`);
+  process.exit(1);
+}
+if (commonVersion !== "1.22.0") {
+  console.error(`expected onnxruntime-common@1.22.0, got ${commonVersion} at ${commonEntry}`);
+  process.exit(1);
+}
+
+console.log(
+  JSON.stringify({
+    ok: true,
+    nodeEntry,
+    commonEntry,
+    nodeVersion,
+    commonVersion,
+    pinnedRoot,
+  })
+);

scripts/verify-nested-onnxruntime-fixture.mjs

@@ -6,6 +6,14 @@
  * so @huggingface/transformers may keep nested onnxruntime-node@1.24.3.
  * Then verifies the production CJS prepare+load path pins the direct 1.22.0 stack.
  *
+ * Unlike earlier revisions, Transformers is loaded through the production
+ * `loadLocalTransformersBackend()` export (createRuntimeRequire + shim), not via
+ * a separately constructed absolute createRequire() that would hide empty-referrer
+ * failures inside compiled OpenCode/Bun hosts.
+ *
+ * When running under Bun, also compiles a minimal host binary that dynamically
+ * imports the installed plugin — the same pattern OpenCode uses.
+ *
  * npm may hoist dependencies to the consumer root (fixture/node_modules/...) while
  * OpenCode keeps them under the plugin package. Both layouts are accepted as long as
  * transformers can resolve a nested 1.24.x copy and the production shim pins 1.22.0.
@@ -18,6 +26,7 @@
  *   FIXTURE_DIR     — reuse an existing fixture root that already has opencode-mem installed
  *   SKIP_INSTALL    — when FIXTURE_DIR is set, skip pack/install
  *   SKIP_EMBEDDING  — skip the real feature-extraction smoke
+ *   SKIP_COMPILE_HOST — skip Bun --compile host verification
  *   KEEP_FIXTURE    — keep the temp fixture directory
  */
 
@@ -42,12 +51,12 @@ function fail(msg) {
   process.exit(1);
 }
 
-function run(cmd, args, cwd) {
+function run(cmd, args, cwd, env = process.env) {
   const result = spawnSync(cmd, args, {
     cwd,
     encoding: "utf8",
     stdio: ["ignore", "pipe", "pipe"],
-    env: process.env,
+    env,
   });
   if (result.status !== 0) {
     fail(
@@ -102,6 +111,77 @@ function findTransformersPackageJson(searchRoots) {
   return null;
 }
 
+async function verifyCompiledHost(pluginRoot) {
+  if (runtime !== "bun") {
+    log("skipping compiled-host check (requires Bun)");
+    return;
+  }
+  if (process.env.SKIP_COMPILE_HOST === "1") {
+    log("skipping compiled-host check (SKIP_COMPILE_HOST=1)");
+    return;
+  }
+
+  const hostDir = mkdtempSync(join(tmpdir(), "opencode-mem-compiled-host-"));
+  const entrySource = join(repoRoot, "scripts", "fixtures", "compiled-host-entry.mjs");
+  const entryCopy = join(hostDir, "entry.ts");
+  const outfile = join(hostDir, "compiled-host");
+  writeFileSync(entryCopy, readFileSync(entrySource));
+  // Match OpenCode's Bun.build compile options — plain `bun build --compile`
+  // does not reproduce the empty-referrer failure from #210.
+  writeFileSync(
+    join(hostDir, "package.json"),
+    JSON.stringify({ name: "opencode-mem-compiled-host", type: "module" }, null, 2)
+  );
+  writeFileSync(
+    join(hostDir, "build.ts"),
+    `
+await Bun.build({
+  entrypoints: ["./entry.ts"],
+  conditions: ["bun", "node"],
+  format: "esm",
+  compile: {
+    autoloadBunfig: false,
+    autoloadDotenv: false,
+    autoloadTsconfig: true,
+    autoloadPackageJson: true,
+    outfile: ${JSON.stringify(outfile)},
+  },
+});
+`
+  );
+
+  log(`compiling OpenCode-shaped Bun host -> ${outfile}`);
+  run("bun", ["build.ts"], hostDir);
+
+  const embeddingEntry = join(pluginRoot, "dist", "services", "embedding.js");
+  const pluginEntry = pathToFileURL(embeddingEntry).href;
+  log(`running compiled host against ${pluginEntry}`);
+  const stdout = run(outfile, [], hostDir, {
+    ...process.env,
+    OPENCODE_MEM_PLUGIN_ENTRY: pluginEntry,
+  });
+  const line = stdout
+    .split("\n")
+    .map((l) => l.trim())
+    .filter(Boolean)
+    .at(-1);
+  let parsed;
+  try {
+    parsed = JSON.parse(line);
+  } catch {
+    fail(`compiled host did not print JSON status\nstdout:\n${stdout}`);
+  }
+  if (!parsed?.ok) fail(`compiled host reported failure: ${line}`);
+  if (parsed.nodeVersion !== PINNED) {
+    fail(`compiled host onnxruntime-node=${parsed.nodeVersion}, expected ${PINNED}`);
+  }
+  if (parsed.commonVersion !== PINNED) {
+    fail(`compiled host onnxruntime-common=${parsed.commonVersion}, expected ${PINNED}`);
+  }
+  log(`compiled host PASS — pinned node=${parsed.nodeEntry}`);
+  rmSync(hostDir, { recursive: true, force: true });
+}
+
 async function main() {
   log(`runtime=${runtime} platform=${process.platform} arch=${process.arch}`);
 
@@ -121,19 +201,33 @@ async function main() {
     const tarball = run("bash", ["-lc", `ls "${packDir}"/opencode-mem-*.tgz | head -1`]).trim();
     if (!tarball) fail("npm pack produced no tarball");
 
+    // OpenCode installs plugins as a tiny consumer package that depends on the
+    // plugin version, then hoists deps under that cache root (#210 reporter layout).
+    // Intentionally NO root overrides — OpenCode nested installs ignore nested overrides (#184).
     writeFileSync(
       join(fixtureDir, "package.json"),
-      JSON.stringify({ name: "opencode-mem-nested-fixture", private: true }, null, 2)
+      JSON.stringify(
+        {
+          name: "opencode-mem-nested-fixture",
+          private: true,
+          dependencies: {
+            "opencode-mem": `file:${tarball}`,
+          },
+        },
+        null,
+        2
+      )
     );
-    // Intentionally NO root overrides — OpenCode nested installs ignore nested overrides (#184).
-    log(`installing ${tarball} into ${fixtureDir} (ignore-scripts, no overrides)`);
-    run("npm", ["install", "--ignore-scripts", tarball], fixtureDir);
+    log(`installing into ${fixtureDir} (ignore-scripts, OpenCode-shaped hoist, no overrides)`);
+    run("npm", ["install", "--ignore-scripts"], fixtureDir);
   }
 
   const pluginRoot = join(fixtureDir, "node_modules", "opencode-mem");
   if (!existsSync(pluginRoot)) fail(`plugin not installed at ${pluginRoot}`);
 
   const searchRoots = [pluginRoot, fixtureDir];
+  // Diagnostic-only require anchored at the production embedding file. Production
+  // loading must go through loadLocalTransformersBackend() below.
   const pluginRequire = createRequire(join(pluginRoot, "dist", "services", "embedding.js"));
 
   let directNodeEntry;
@@ -177,19 +271,17 @@ async function main() {
     log(`pre-shim transformers resolve -> ${nestedResolved} (@${resolvedPkg.version})`);
   }
 
-  // Load production prepare from the installed plugin dist.
-  const resolveUrl = pathToFileURL(
-    join(pluginRoot, "dist", "services", "onnxruntime-resolve.js")
-  ).href;
-  const { prepareOnnxruntimeForTransformers, getPinnedOnnxruntimePackageRoot } =
-    await import(resolveUrl);
-
-  prepareOnnxruntimeForTransformers();
+  // Load production prepare + transformers through the real embedding module path.
+  // This exercises createRuntimeRequire(import.meta) instead of a hand-built absolute require.
+  const embeddingUrl = pathToFileURL(join(pluginRoot, "dist", "services", "embedding.js")).href;
+  const embeddingMod = await import(embeddingUrl);
+  if (typeof embeddingMod.loadLocalTransformersBackend !== "function") {
+    fail("dist/services/embedding.js does not export loadLocalTransformersBackend");
+  }
 
-  const transformersSpecifier = ["@huggingface", "transformers"].join("/");
-  const transformers = pluginRequire(transformersSpecifier);
+  const transformers = await embeddingMod.loadLocalTransformersBackend();
   if (typeof transformers.pipeline !== "function" || !transformers.env) {
-    fail("CJS transformers load did not expose pipeline/env");
+    fail("production loadLocalTransformersBackend did not expose pipeline/env");
   }
 
   const pinnedNode = pluginRequire.resolve("onnxruntime-node");
@@ -206,12 +298,17 @@ async function main() {
     );
   }
 
+  const resolveUrl = pathToFileURL(
+    join(pluginRoot, "dist", "services", "onnxruntime-resolve.js")
+  ).href;
+  const { getPinnedOnnxruntimePackageRoot } = await import(resolveUrl);
   const pinnedRoot = getPinnedOnnxruntimePackageRoot();
   if (!pinnedRoot.includes("onnxruntime-node")) {
     fail(`unexpected pinned root: ${pinnedRoot}`);
   }
 
   // From nested transformers context, shim must force both packages onto the pin.
+  const transformersSpecifier = ["@huggingface", "transformers"].join("/");
   const transformersEntry = pluginRequire.resolve(transformersSpecifier);
   const fromTransformers = createRequire(transformersEntry);
   const shimmedNode = fromTransformers.resolve("onnxruntime-node");
@@ -256,6 +353,8 @@ async function main() {
     log(`embedding ok: ${dims} dims, L2=${norm.toFixed(4)}`);
   }
 
+  await verifyCompiledHost(pluginRoot);
+
   log("PASS — nested fixture loads production CJS path on onnxruntime 1.22.0 stack");
 
   if (cleanup && process.env.KEEP_FIXTURE !== "1") cleanup();

src/services/embedding.ts

@@ -1,13 +1,13 @@
 import { CONFIG } from "../config.js";
 import { log } from "./logger.js";
-import { createRequire } from "node:module";
 import { join } from "node:path";
 import {
   formatOnnxruntimeInitError,
   prepareOnnxruntimeForTransformers,
 } from "./onnxruntime-resolve.js";
+import { createRuntimeRequire } from "./runtime-require.js";
 
-const requireFromHere = createRequire(import.meta.url);
+const requireFromHere = createRuntimeRequire(import.meta);
 
 const TIMEOUT_MS = 30000;
 const GLOBAL_EMBEDDING_KEY = Symbol.for("opencode-mem.embedding.instance");
@@ -58,8 +58,27 @@ async function ensureTransformersLoaded(): Promise<NonNullable<typeof _transform
   // Pin onnxruntime-node (+ common) to our direct 1.22.0 stack before transformers
   // resolves them (#184 / #210). Load the CJS export so Module._resolveFilename
   // shim applies — the ESM entry's static import bypasses it under OpenCode nested installs.
+  //
+  // Critical ordering for OpenCode's Bun --compile host (#210 follow-up):
+  // resolve the transformers absolute path BEFORE installOnnxruntimeResolveShim().
+  // After the shim is installed, non-onnx Module._resolveFilename fallbacks fail with
+  // `Cannot find module '@huggingface/transformers' from ''` even though the package
+  // exists. Loading via the pre-resolved absolute path keeps the onnx pin shim active
+  // for transformers' nested requires without needing a post-shim package resolve.
+  const transformersSpecifier = getTransformersPackageSpecifier();
+  let transformersEntry: string;
+  try {
+    transformersEntry = requireFromHere.resolve(transformersSpecifier);
+  } catch (error) {
+    throw new Error(
+      `Cannot resolve ${transformersSpecifier} before installing the onnxruntime pin shim. ` +
+        `Original error: ${error instanceof Error ? error.message : String(error)}`,
+      { cause: error }
+    );
+  }
+
   prepareOnnxruntimeForTransformers();
-  const mod = requireFromHere(getTransformersPackageSpecifier()) as HfTransformers;
+  const mod = requireFromHere(transformersEntry) as HfTransformers;
   mod.env.allowLocalModels = true;
   mod.env.allowRemoteModels = true;
   mod.env.cacheDir = join(CONFIG.storagePath, ".cache");
@@ -73,6 +92,14 @@ async function ensureTransformersLoaded(): Promise<NonNullable<typeof _transform
   return _transformers!;
 }
 
+/**
+ * Production transformers load path used by nested OpenCode install fixtures (#210).
+ * Prefer this over a separately anchored createRequire() so empty-referrer hosts are covered.
+ */
+export async function loadLocalTransformersBackend(): Promise<NonNullable<typeof _transformers>> {
+  return ensureTransformersLoaded();
+}
+
 function withTimeout<T>(promise: Promise<T>, ms: number): Promise<T> {
   return Promise.race([
     promise,