From a86e3272395151082f3a394e6ae75897c36d063a Mon Sep 17 00:00:00 2001 From: Harsh16gupta Date: Mon, 13 Jul 2026 23:40:30 +0530 Subject: [PATCH] Refactor clustering strategy options and clean up testEmbed debug utility for V1 --- src/commands/testEmbed.ts | 310 --------------------- src/index.ts | 13 - src/pipeline/pipelineConfig.ts | 4 +- src/pipeline/runPipeline.ts | 11 +- src/webview/components/StrategySection.tsx | 35 ++- src/webview/context/AppStateContext.tsx | 8 +- 6 files changed, 40 insertions(+), 341 deletions(-) delete mode 100644 src/commands/testEmbed.ts diff --git a/src/commands/testEmbed.ts b/src/commands/testEmbed.ts deleted file mode 100644 index 391a560..0000000 --- a/src/commands/testEmbed.ts +++ /dev/null @@ -1,310 +0,0 @@ -import { fetchAllNotes } from '../pipeline/noteReader'; -import { benchmark } from '../pipeline/clustering/benchmark'; -import { averageVectors, blendVectors, computeTitleWeight, cosineSimilarity } from '../pipeline/vectorAggregator'; -import { NoteVector, WorkerMessage } from '../types/embed'; -import { isGenericTitle } from '../utils/titleFilter'; -import { log, logErr } from '../utils/logger'; -import { getEncoding } from 'js-tiktoken'; -import { VectorCache } from '../pipeline/vectorCache'; -import { DEFAULT_CONFIG, isValidEmbeddingVector } from '../pipeline/pipelineConfig'; -import { enrichResultsWithTags } from '../pipeline/clustering/postProcess'; - -// We use cl100k_base to approximate token counts for chunking. -// The embedding model (all-MiniLM-L6-v2) uses a WordPiece tokenizer with a -// 512-token limit. WordPiece has a smaller vocab (~30k vs ~100k) so it produces -// ~1.3-1.5x more tokens than cl100k_base for the same text. A limit of 200 -// cl100k_base tokens expands to ~300 WordPiece tokens in the worst case, -// well within the model's 512-token ceiling. -const enc = getEncoding('cl100k_base'); -const MAX_TOKENS = 200; - -export const runTestEmbed = async (installDir: string) => { - log('Test embed command triggered'); - - const notes = await fetchAllNotes(); - log(`Fetched ${notes.length} notes`); - - if (notes.length === 0) { - log('No notes found. Create some notes and try again.'); - return; - } - - const cache = await VectorCache.create(); - - // Handle deletions: Remove notes from index that are no longer in Joplin - const indexedIds = await cache.getIndexedIds(); - const joplinNoteIds = new Set(notes.map((n) => n.id)); - const idsToDelete = indexedIds.filter((id) => !joplinNoteIds.has(id)); - - if (idsToDelete.length > 0) { - log(`Removing ${idsToDelete.length} obsolete notes from cache`); - await cache.deleteItems(idsToDelete); - } - - await cache.beginUpdate(); - - const worker = new Worker(`${installDir}/worker/embedWorker.js`); - let currentNoteIndex = 0; - let currentChunkIndex = 0; - let currentNoteChunks: string[] = []; - let currentChunkEmbeddings: number[][] = []; - let currentBodyVector: number[] = []; - let isEmbeddingTitle = false; - let totalInferenceTime = 0; - let skippedCount = 0; - let cachedCount = 0; - const batchStartTime = performance.now(); - const noteVectors: NoteVector[] = []; - - worker.onerror = (err: ErrorEvent) => { - logErr('Worker error:', err.message); - cache.cancelUpdate(); - worker.terminate(); - }; - - const prepareNoteChunks = (text: string): string[] => { - const tokens = enc.encode(text); - const chunks: string[] = []; - if (tokens.length === 0) return ['']; - - for (let i = 0; i < tokens.length; i += MAX_TOKENS) { - const chunkTokens = tokens.slice(i, i + MAX_TOKENS); - chunks.push(enc.decode(chunkTokens)); - } - return chunks; - }; - - const finalizeNote = async (vector: number[], titleWeight: number, hash: string) => { - const note = notes[currentNoteIndex]; - noteVectors.push({ noteId: note.id, title: note.title, vector, titleWeight }); - const norm = Math.sqrt(vector.reduce((s, v) => s + v * v, 0)); - log(` → final vector: dim=${vector.length}, titleWeight=${titleWeight.toFixed(3)}, norm=${norm.toFixed(4)}`); - - await cache.upsertItem(note.id, vector, { - title: note.title, - hash, - updatedTime: note.updated_time, - titleWeight, - }); - - currentNoteIndex++; - await processNextNote(); - }; - - let currentNoteHash = ''; - - const processNextNote = async () => { - currentChunkIndex = 0; - currentNoteChunks = []; - currentChunkEmbeddings = []; - currentBodyVector = []; - isEmbeddingTitle = false; - - // Skip notes with empty body and generic title, and bypass cached notes - while (currentNoteIndex < notes.length) { - const note = notes[currentNoteIndex]; - - if (note.body.length === 0 && isGenericTitle(note.title)) { - log( - `[${currentNoteIndex + 1}/${notes.length}] skipped "${note.title.slice(0, 30)}" (empty body, generic title)`, - ); - skippedCount++; - currentNoteIndex++; - continue; - } - - currentNoteHash = cache.computeHash(note.title, note.body); - const cachedItem = await cache.getItem(note.id); - - if (cachedItem && cachedItem.metadata.hash === currentNoteHash) { - if (isValidEmbeddingVector(cachedItem.vector)) { - log(`[${currentNoteIndex + 1}/${notes.length}] cache hit for "${note.title.slice(0, 30)}"`); - noteVectors.push({ - noteId: note.id, - title: note.title, - vector: cachedItem.vector, - titleWeight: cachedItem.metadata.titleWeight ?? 0, - }); - cachedCount++; - currentNoteIndex++; - continue; - } else { - log( - `[${currentNoteIndex + 1}/${notes.length}] cache invalid (contains null/NaN) for "${note.title.slice(0, 30)}"`, - ); - } - } - - break; - } - - if (currentNoteIndex >= notes.length) { - const totalTime = performance.now() - batchStartTime; - log(`-------------------------------------------`); - log(`Batch execution complete!`); - log(`Total notes processed: ${notes.length}`); - log(`Notes embedded: ${noteVectors.length - cachedCount}`); - log(`Notes loaded from cache: ${cachedCount}`); - log(`Notes skipped: ${skippedCount}`); - log(`Sum of inference times: ${Math.round(totalInferenceTime)}ms`); - log(`Real total batch time (including worker message passing): ${Math.round(totalTime)}ms`); - log(`-------------------------------------------`); - - await cache.endUpdate(); - - worker.terminate(); - log('Worker terminated. Embedding complete.'); - - // ── Clustering Benchmark ───────────────────────────── - // Edit this config to compare different algorithms and dimensions. - // Results are printed as a comparison table in the console. - if (noteVectors.length >= 3) { - const vectors = noteVectors.map((nv) => nv.vector); - const results = benchmark(vectors, DEFAULT_CONFIG); - - const notesMap = new Map(notes.map((n) => [n.id, n])); - const allPipelineDocuments = noteVectors.map((nv) => { - const originalNote = notesMap.get(nv.noteId); - return { - title: nv.title, - body: originalNote ? originalNote.body : '', - }; - }); - - enrichResultsWithTags(results, allPipelineDocuments); - - // Log note titles and tags per cluster for all strategies, in order (best to worst) - for (const res of results) { - log(`\nCluster assignments & Analysis (${res.strategyName}):`); - const clusterNotes = new Map(); - for (let i = 0; i < noteVectors.length; i++) { - const c = res.assignments[i]; - if (!clusterNotes.has(c)) clusterNotes.set(c, []); - clusterNotes.get(c)!.push(noteVectors[i].title); - } - for (const [clusterId, titles] of clusterNotes) { - const generatedName = res.clusterNames?.[clusterId]; - const label = - clusterId < 0 - ? 'Noise/Outliers' - : generatedName - ? `${generatedName} (Cluster ${clusterId})` - : `Cluster ${clusterId}`; - const clusterTags = res.tags?.[clusterId] ? ` [Tags: ${res.tags[clusterId].join(', ')}]` : ''; - log(` ${label} (${titles.length} notes)${clusterTags}:`); - for (const title of titles) { - log(` - ${title}`); - } - } - } - } else { - log('Too few notes for clustering (need at least 3).'); - } - - return; - } - - const note = notes[currentNoteIndex]; - - if (note.body.length === 0) { - // Empty body with descriptive title → embed title as the final vector - isEmbeddingTitle = true; - worker.postMessage({ type: 'embed', text: note.title, noteId: note.id }); - } else { - // Has body → chunk and embed sequentially - currentNoteChunks = prepareNoteChunks(note.body); - worker.postMessage({ - type: 'embed', - text: currentNoteChunks[0], - noteId: note.id, - }); - } - }; - - worker.onmessage = async (event: MessageEvent) => { - const data = event.data; - - if (data.type === 'load-result') { - if (data.success) { - log( - `Model loaded in ${(data.loadTime / 1000).toFixed(1)}s, device: ${data.device}, dtype: ${data.dtype}`, - ); - log( - ` Worker WebGPU diagnostics - gpu in navigator: ${data.workerGpuExists}, env.IS_WEBGPU_AVAILABLE: ${data.isWebGpuAvailable}`, - ); - log(`Starting sequential embedding of ${notes.length} notes with chunking...`); - await processNextNote(); - } else { - logErr('Model load failed:', data.error); - cache.cancelUpdate(); - worker.terminate(); - } - return; - } - - if (data.type === 'embed-result') { - const note = notes[currentNoteIndex]; - - if (!data.success) { - logErr(`Failed to embed note "${note.title.slice(0, 30)}":`, data.error); - currentNoteIndex++; - await processNextNote(); - return; - } - - totalInferenceTime += data.inferenceTime; - - if (isEmbeddingTitle) { - const titleEmbedding = data.embedding; - - if (currentBodyVector.length > 0) { - // Body was embedded → blend body + title - const sim = cosineSimilarity(currentBodyVector, titleEmbedding); - const alpha = computeTitleWeight(sim); - const finalVector = blendVectors(currentBodyVector, titleEmbedding, alpha); - log( - ` → title embedded in ${Math.round(data.inferenceTime)}ms, sim=${sim.toFixed(3)}, weight=${alpha.toFixed(3)}`, - ); - await finalizeNote(finalVector, alpha, currentNoteHash); - } else { - // Empty body, descriptive title → title is the entire vector - log( - `[${currentNoteIndex + 1}/${notes.length}] embedded title of "${note.title.slice(0, 30)}" in ${Math.round(data.inferenceTime)}ms (no body)`, - ); - await finalizeNote(titleEmbedding, 1.0, currentNoteHash); - } - } else { - // Body chunk result - currentChunkEmbeddings.push(data.embedding); - log( - `[${currentNoteIndex + 1}/${notes.length}] embedded chunk ${currentChunkIndex + 1}/${currentNoteChunks.length} of "${note.title.slice(0, 30)}" in ${Math.round(data.inferenceTime)}ms`, - ); - - currentChunkIndex++; - if (currentChunkIndex < currentNoteChunks.length) { - // More chunks to process - worker.postMessage({ - type: 'embed', - text: currentNoteChunks[currentChunkIndex], - noteId: note.id, - }); - } else { - // All chunks done → compute body vector - currentBodyVector = averageVectors(currentChunkEmbeddings); - - if (!isGenericTitle(note.title)) { - // Descriptive title → embed it for blending - isEmbeddingTitle = true; - worker.postMessage({ type: 'embed', text: note.title, noteId: note.id }); - } else { - // Generic title → body vector is the final vector - await finalizeNote(currentBodyVector, 0, currentNoteHash); - } - } - } - } - }; - - log('Loading model...'); - worker.postMessage({ type: 'load' }); -}; diff --git a/src/index.ts b/src/index.ts index bd13cc2..007b1de 100644 --- a/src/index.ts +++ b/src/index.ts @@ -1,6 +1,5 @@ import joplin from 'api'; import { MenuItemLocation, ToolbarButtonLocation, SettingItemType as SettingType } from 'api/types'; -import { runTestEmbed } from './commands/testEmbed'; import { runPipeline } from './pipeline/runPipeline'; import { PanelMessage, WebviewMessage } from './types/panel'; import { log } from './utils/logger'; @@ -51,18 +50,6 @@ joplin.plugins.register({ const installDir = await joplin.plugins.installationDir(); - await joplin.commands.register({ - name: 'aiCategorise.testEmbed', - label: 'AI Categorise: Test Embedding', - execute: async () => runTestEmbed(installDir), - }); - - await joplin.views.menuItems.create( - 'aiCategorise.testEmbedMenuItem', - 'aiCategorise.testEmbed', - MenuItemLocation.Tools, - ); - // Panel starts hidden; user opens via toolbar button or View menu const panel = await joplin.views.panels.create('aiCategorise.panel'); await joplin.views.panels.setHtml(panel, '
'); diff --git a/src/pipeline/pipelineConfig.ts b/src/pipeline/pipelineConfig.ts index 78fa4cf..a16d3db 100644 --- a/src/pipeline/pipelineConfig.ts +++ b/src/pipeline/pipelineConfig.ts @@ -17,7 +17,7 @@ export const DEFAULT_CONFIG: CategorizationConfig = { strategies: [ { name: 'kmeans-6', algorithm: 'kmeans', K: 6 }, { name: 'kmedoids-6', algorithm: 'kmedoids', K: 6 }, - { name: 'hdbscan-tuned', algorithm: 'hdbscan', minClusterSize: 4, minSamples: 1 }, - { name: 'hdbscan-conservative', algorithm: 'hdbscan', minClusterSize: 3, minSamples: 2 }, + // { name: 'hdbscan-tuned', algorithm: 'hdbscan', minClusterSize: 4, minSamples: 1 }, + { name: 'hdbscan', algorithm: 'hdbscan', minClusterSize: 3, minSamples: 2 }, ], }; diff --git a/src/pipeline/runPipeline.ts b/src/pipeline/runPipeline.ts index 852342d..3d4860c 100644 --- a/src/pipeline/runPipeline.ts +++ b/src/pipeline/runPipeline.ts @@ -11,7 +11,12 @@ import { isNativeAiReady, fetchNativeEmbeddings } from './nativeEmbeddingPipelin import { DEFAULT_CONFIG, isValidEmbeddingVector } from './pipelineConfig'; import { enrichResultsWithTags } from './clustering/postProcess'; -// See testEmbed.ts for rationale on cl100k_base and the 200-token limit. +// We use cl100k_base to approximate token counts for chunking. +// The embedding model (all-MiniLM-L6-v2) uses a WordPiece tokenizer with a +// 512-token limit. WordPiece has a smaller vocabulary (~30k vs ~100k) so it produces +// ~1.3-1.5x more tokens than cl100k_base for the same text. A limit of 200 +// cl100k_base tokens expands to ~300 WordPiece tokens in the worst case, +// well within the model's 512-token ceiling. const enc = getEncoding('cl100k_base'); const MAX_TOKENS = 200; @@ -25,8 +30,8 @@ export interface PipelineCallbacks { /** * Runs the full embedding + clustering pipeline, reporting progress via callbacks. * - * This is the same logic as testEmbed.ts, but decoupled from console logging - * so the panel (or any other caller) can receive live updates. + * This process is decoupled from console logging so the panel (or any other caller) + * can receive live updates. */ export const runPipeline = async (installDir: string, callbacks: PipelineCallbacks): Promise => { try { diff --git a/src/webview/components/StrategySection.tsx b/src/webview/components/StrategySection.tsx index c0711c5..c7099d6 100644 --- a/src/webview/components/StrategySection.tsx +++ b/src/webview/components/StrategySection.tsx @@ -7,11 +7,21 @@ interface StrategySectionProps { onStrategyChange: (index: number) => void; } -/** Shortens 'hdbscan-5-ms2' → 'hdbscan-5' for pill labels */ -function abbreviateStrategy(name: string): string { - if (!name) return ''; - const parts = name.split('-'); - return parts.length > 2 ? parts[0] + '-' + parts[1] : name; +/** Returns display names for dropdown and pills, marking testing and recommended strategies */ +function getStrategyDisplayName(name: string, isHighest: boolean): string { + let baseName = name; + if (name === 'hdbscan') { + baseName = 'HDBSCAN'; + } else if (name.startsWith('kmeans')) { + baseName = 'K-Means (Testing)'; + } else if (name.startsWith('kmedoids')) { + baseName = 'K-Medoids (Testing)'; + } + + if (isHighest) { + return `${baseName} (Recommended)`; + } + return baseName; } export const StrategySection: React.FC = ({ @@ -38,7 +48,7 @@ export const StrategySection: React.FC = ({ > {strategies.map((s, idx) => ( ))} @@ -51,11 +61,14 @@ export const StrategySection: React.FC = ({
- {strategies.map((s, idx) => ( - - {abbreviateStrategy(s.strategyName)}: {s.silhouetteScore.toFixed(2)} - - ))} + {strategies + .map((s, idx) => ({ s, idx })) + .filter(({ s }) => !s.strategyName.startsWith('kmeans') && !s.strategyName.startsWith('kmedoids')) + .map(({ s, idx }) => ( + + {getStrategyDisplayName(s.strategyName, false)}: {s.silhouetteScore.toFixed(2)} + + ))}
); diff --git a/src/webview/context/AppStateContext.tsx b/src/webview/context/AppStateContext.tsx index 7727eec..b5c1362 100644 --- a/src/webview/context/AppStateContext.tsx +++ b/src/webview/context/AppStateContext.tsx @@ -133,15 +133,19 @@ export const AppStateProvider: React.FC<{ children: React.ReactNode }> = ({ chil }); break; - case 'results': + case 'results': { stopPolling(); setIsRunning(false); setStrategies(msg.strategies || []); setNotes(msg.notes || []); - setSelectedStrategyIndex(0); + const nonTestingIdx = (msg.strategies || []).findIndex( + (s: any) => !s.strategyName.startsWith('kmeans') && !s.strategyName.startsWith('kmedoids'), + ); + setSelectedStrategyIndex(nonTestingIdx !== -1 ? nonTestingIdx : 0); setError(null); setActiveView('dashboard'); break; + } case 'error': stopPolling();