Refactor clustering strategy options and clean up debug utilities

src/index.ts

@@ -1,6 +1,5 @@
 import joplin from 'api';
 import { MenuItemLocation, ToolbarButtonLocation, SettingItemType as SettingType } from 'api/types';
-import { runTestEmbed } from './commands/testEmbed';
 import { runPipeline } from './pipeline/runPipeline';
 import { PanelMessage, WebviewMessage } from './types/panel';
 import { log } from './utils/logger';
@@ -51,18 +50,6 @@ joplin.plugins.register({
 
 		const installDir = await joplin.plugins.installationDir();
 
-		await joplin.commands.register({
-			name: 'aiCategorise.testEmbed',
-			label: 'AI Categorise: Test Embedding',
-			execute: async () => runTestEmbed(installDir),
-		});
-
-		await joplin.views.menuItems.create(
-			'aiCategorise.testEmbedMenuItem',
-			'aiCategorise.testEmbed',
-			MenuItemLocation.Tools,
-		);
-
 		// Panel starts hidden; user opens via toolbar button or View menu
 		const panel = await joplin.views.panels.create('aiCategorise.panel');
 		await joplin.views.panels.setHtml(panel, '<div id="root"></div>');

src/pipeline/pipelineConfig.ts

@@ -17,7 +17,7 @@ export const DEFAULT_CONFIG: CategorizationConfig = {
 	strategies: [
 		{ name: 'kmeans-6', algorithm: 'kmeans', K: 6 },
 		{ name: 'kmedoids-6', algorithm: 'kmedoids', K: 6 },
-		{ name: 'hdbscan-tuned', algorithm: 'hdbscan', minClusterSize: 4, minSamples: 1 },
-		{ name: 'hdbscan-conservative', algorithm: 'hdbscan', minClusterSize: 3, minSamples: 2 },
+		// { name: 'hdbscan-tuned', algorithm: 'hdbscan', minClusterSize: 4, minSamples: 1 },
+		{ name: 'hdbscan', algorithm: 'hdbscan', minClusterSize: 3, minSamples: 2 },
 	],
 };

src/pipeline/runPipeline.ts

@@ -11,7 +11,12 @@ import { isNativeAiReady, fetchNativeEmbeddings } from './nativeEmbeddingPipelin
 import { DEFAULT_CONFIG, isValidEmbeddingVector } from './pipelineConfig';
 import { enrichResultsWithTags } from './clustering/postProcess';
 
-// See testEmbed.ts for rationale on cl100k_base and the 200-token limit.
+// We use cl100k_base to approximate token counts for chunking.
+// The embedding model (all-MiniLM-L6-v2) uses a WordPiece tokenizer with a
+// 512-token limit. WordPiece has a smaller vocabulary (~30k vs ~100k) so it produces
+// ~1.3-1.5x more tokens than cl100k_base for the same text. A limit of 200
+// cl100k_base tokens expands to ~300 WordPiece tokens in the worst case,
+// well within the model's 512-token ceiling.
 const enc = getEncoding('cl100k_base');
 const MAX_TOKENS = 200;
 
@@ -25,8 +30,8 @@ export interface PipelineCallbacks {
 /**
  * Runs the full embedding + clustering pipeline, reporting progress via callbacks.
  *
- * This is the same logic as testEmbed.ts, but decoupled from console logging
- * so the panel (or any other caller) can receive live updates.
+ * This process is decoupled from console logging so the panel (or any other caller)
+ * can receive live updates.
  */
 export const runPipeline = async (installDir: string, callbacks: PipelineCallbacks): Promise<void> => {
 	try {