From 34ab7167415747fc12f2d0b781bd7296e5c28236 Mon Sep 17 00:00:00 2001 From: Artificium Date: Tue, 28 Jul 2026 01:51:06 +0000 Subject: [PATCH] feat(query): make the hybrid semantic weight configurable (default unchanged) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The 1.5x multiplier on the semantic arm's RRF contribution was a hardcoded constant with a large, undocumented effect: with the standard RRF constant K=60, it means semantic ranks 1 through 31 all outscore a keyword-only rank-1 hit, so lexical matches the vector arm missed are close to invisible. Exposed as HYBRID_SEMANTIC_WEIGHT so the balance can be measured per deployment rather than assumed. The default is deliberately NOT changed. A train/test experiment says the current value is fine and that the apparent alternatives are noise: sweep on a stratified 42-question sample (7 per category), validated on a disjoint 42-question sample selected the same way weight train R@5 test R@5 train R@1 test R@1 0.75 95.2% 88.1% 76.2% 76.2% 1.50 92.9% 88.1% 69.0% 81.0% On train, 0.75 looked clearly better and would have been adopted by any single-sample tuning. It did not replicate: R@5 identical on held-out data and R@1 4.8pp worse. Every metric flips direction between the two samples, which is the signature of sampling noise rather than signal — at n=42 a single question moves a metric 2.4pp, and the observed gaps are one to two questions. Pre-registered rule (declared before the sweep ran): adopt only if the winner beats the 1.5 baseline on the held-out sample at R@5 without regressing R@1 by more than 2pp. It failed both conditions, so the weight stays at 1.5. Resolving this properly needs the full 500-question run; the knob is what makes that measurement possible without a code change. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_011xCqQo49d3CEbn6oEvb3Ru --- CLAUDE.md | 1 + src/memory-manager.ts | 11 ++++++++--- src/server.ts | 1 + src/types.ts | 9 +++++++++ 4 files changed, 19 insertions(+), 3 deletions(-) diff --git a/CLAUDE.md b/CLAUDE.md index 4a04667..dea9a02 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -205,6 +205,7 @@ Key concepts: Hot → Warm → Cold tiers. Hybrid retrieval (FTS + pgvector HNSW | Variable | Default | Description | |----------|---------|-------------| | `KEYWORD_OVERLAP_BOOST` | `0.3` | Score boost when query tokens overlap memory keywords | +| `HYBRID_SEMANTIC_WEIGHT` | `1.5` | Weight on the semantic arm's RRF contribution in hybrid search. With RRF K=60, 1.5 means semantic ranks 1-31 outscore a keyword-only rank-1 hit; lower values give the lexical arm more say. Optimal value depends on the embedding model — measure before changing. | | `TEMPORAL_PROXIMITY_DAYS` | `7` | Days window for temporal proximity boost | | `CONSOLIDATION_INNER_BATCH_SIZE` | `50` | Hot-tier rows per inner consolidation batch | | `TEMPORAL_DECAY_RATE` | `0` | Score decay per hour (0 = disabled) | diff --git a/src/memory-manager.ts b/src/memory-manager.ts index dd561a8..d2a1199 100644 --- a/src/memory-manager.ts +++ b/src/memory-manager.ts @@ -310,6 +310,7 @@ const DEFAULTS: MemForgeConfig = { temporalDecayRate: 0, consolidationInnerBatchSize: 50, keywordOverlapBoost: 0.3, + hybridSemanticWeight: 1.5, temporalProximityDays: 7, enableLlmRerank: false, enableLlmIngest: false, @@ -1167,11 +1168,15 @@ Ranking (numbers only):`; } }); - // Semantic results weighted 1.5x — paraphrase matching is the primary failure mode - // in conversational memory retrieval (users ask differently than memories are stored) + // Semantic arm weighted above 1.0 because paraphrase matching is the + // primary failure mode in conversational memory retrieval (users ask + // differently than memories are stored). The multiplier is configurable — + // at 1.5 with K=60, semantic ranks 1-31 outscore a keyword-only rank-1 + // hit, which is a strong thumb on the scale and worth measuring. + const semanticWeight = this.config.hybridSemanticWeight ?? 1.5; semanticResults.forEach((row, idx) => { const key = String(row.id); - const rrf = 1.5 / (K + idx + 1); + const rrf = semanticWeight / (K + idx + 1); const existing = scores.get(key); if (existing) { existing.score += rrf; diff --git a/src/server.ts b/src/server.ts index 9b4a901..f4f0792 100644 --- a/src/server.ts +++ b/src/server.ts @@ -56,6 +56,7 @@ const manager = new MemoryManager({ temporalDecayRate: parseFloat(process.env['TEMPORAL_DECAY_RATE'] ?? '0'), consolidationInnerBatchSize: parseInt(process.env['CONSOLIDATION_INNER_BATCH_SIZE'] ?? '50', 10), keywordOverlapBoost: parseFloat(process.env['KEYWORD_OVERLAP_BOOST'] ?? '0.3'), + hybridSemanticWeight: parseFloat(process.env['HYBRID_SEMANTIC_WEIGHT'] ?? '1.5'), temporalProximityDays: parseFloat(process.env['TEMPORAL_PROXIMITY_DAYS'] ?? '7'), enableLlmRerank: process.env['ENABLE_LLM_RERANK'] === 'true', enableLlmIngest: process.env['ENABLE_LLM_INGEST'] === 'true', diff --git a/src/types.ts b/src/types.ts index 172e611..8cace08 100644 --- a/src/types.ts +++ b/src/types.ts @@ -806,6 +806,15 @@ export interface MemForgeConfig { consolidationInnerBatchSize: number; /** Keyword overlap boost factor for hybrid search (default 0.3, 0 = disabled). */ keywordOverlapBoost: number; + /** + * Weight applied to the semantic arm's RRF contribution in hybrid search + * (default 1.5). Above 1.0 the semantic ranking dominates: with the standard + * RRF constant K=60, a weight of 1.5 means semantic ranks 1-31 all outscore + * a keyword-only rank-1 hit, so lexical matches the vector arm missed are + * effectively invisible. Exposed as HYBRID_SEMANTIC_WEIGHT so the balance + * can be measured rather than assumed. + */ + hybridSemanticWeight: number; /** Temporal proximity window in days for time-aware scoring (default 7, 0 = disabled). */ temporalProximityDays: number; /** Enable LLM post-retrieval reranking (default false — opt-in, adds ~2K tokens/query) */