From 5b61c6e888d24946e8f4d9d8c9561b3366e8f1f3 Mon Sep 17 00:00:00 2001 From: Harsh16gupta Date: Fri, 17 Jul 2026 02:07:57 +0530 Subject: [PATCH] refactor: split large modules into smaller files --- src/commands/applyChanges.ts | 303 ++----- src/commands/applyNotebooks.ts | 176 ++++ src/commands/applyTags.ts | 133 +++ src/pipeline/EmbeddingWorkerOrchestrator.ts | 262 ++++++ src/pipeline/clustering/clusterNaming.ts | 84 ++ src/pipeline/clustering/data/stopWords.ts | 323 ++++++++ src/pipeline/clustering/data/taxonomy.ts | 101 +++ src/pipeline/clustering/postProcess.ts | 852 +------------------- src/pipeline/clustering/tagExtraction.ts | 58 ++ src/pipeline/clustering/tfidf.ts | 260 ++++++ src/pipeline/runPipeline.ts | 280 +------ src/webview/context/AppStateContext.tsx | 326 ++------ src/webview/context/useApplyState.ts | 128 +++ src/webview/context/usePipelineState.ts | 155 ++++ src/webview/context/useSettingsState.ts | 50 ++ 15 files changed, 1942 insertions(+), 1549 deletions(-) create mode 100644 src/commands/applyNotebooks.ts create mode 100644 src/commands/applyTags.ts create mode 100644 src/pipeline/EmbeddingWorkerOrchestrator.ts create mode 100644 src/pipeline/clustering/clusterNaming.ts create mode 100644 src/pipeline/clustering/data/stopWords.ts create mode 100644 src/pipeline/clustering/data/taxonomy.ts create mode 100644 src/pipeline/clustering/tagExtraction.ts create mode 100644 src/pipeline/clustering/tfidf.ts create mode 100644 src/webview/context/useApplyState.ts create mode 100644 src/webview/context/usePipelineState.ts create mode 100644 src/webview/context/useSettingsState.ts diff --git a/src/commands/applyChanges.ts b/src/commands/applyChanges.ts index 89cdaa1..1d9b6d9 100644 --- a/src/commands/applyChanges.ts +++ b/src/commands/applyChanges.ts @@ -1,6 +1,21 @@ import joplin from 'api'; import { ApplyOptions, PanelNote, PanelMessage } from '../types/panel'; import { log } from '../utils/logger'; +import { + fetchExistingTags, + initializeClusterTags, + applyTagsToNote, + removeTagsFromNote, + deleteCreatedTags, +} from './applyTags'; +import { + fetchExistingFolders, + initializeClusterNotebooks, + moveNoteToFolder, + restoreNotebook, + deleteCreatedFolders, + cleanUpFolders, +} from './applyNotebooks'; export interface ChangeLogEntry { timestamp: number; @@ -15,21 +30,6 @@ export interface ChangeLogEntry { createdTagIds?: string[]; } -function matchesKeyword(title: string, body: string, keyword: string): boolean { - const lowerKeyword = keyword.toLowerCase(); - const lowerTitle = title.toLowerCase(); - const lowerBody = body.toLowerCase(); - - try { - const escaped = lowerKeyword.replace(/[-\x2f\\^$*+?.()|[\]{}]/g, '\\$&'); - // Unicode-aware word boundary matching - const regex = new RegExp(`(?:^|[^\\p{L}\\p{N}])` + escaped + `(?:$|[^\\p{L}\\p{N}])`, 'u'); - return regex.test(lowerTitle) || regex.test(lowerBody); - } catch (e) { - return lowerTitle.includes(lowerKeyword) || lowerBody.includes(lowerKeyword); - } -} - export async function applyCategorizationChanges( options: ApplyOptions, notes: PanelNote[], @@ -41,104 +41,33 @@ export async function applyCategorizationChanges( try { setPanelState({ type: 'apply_status', text: 'Fetching existing folders and tags...' }); - const MAX_PAGES = 500; - - // Fetch all existing tags once to map titles to IDs - const allTags: any[] = []; - let tagPage = 1; - while (tagPage <= MAX_PAGES) { - const res = await joplin.data.get(['tags'], { page: tagPage, limit: 100, fields: ['id', 'title'] }); - allTags.push(...res.items); - if (!res.has_more) break; - tagPage++; - } - const existingTagsMap = new Map(allTags.map((t: any) => [t.title.toLowerCase(), t.id])); - - // Fetch all folders once to map titles and parent_ids to IDs - const allFoldersList: any[] = []; - let folderPage = 1; - while (folderPage <= MAX_PAGES) { - const res = await joplin.data.get(['folders'], { - page: folderPage, - limit: 100, - fields: ['id', 'title', 'parent_id'], - }); - allFoldersList.push(...res.items); - if (!res.has_more) break; - folderPage++; - } - const existingFoldersMap = new Map( - allFoldersList.map((f: any) => [`${f.title.toLowerCase()}\x1F${f.parent_id || ''}`, f.id]), - ); - - // Optimized helper to get or create tag - const getOrCreateTagOptimized = async (title: string): Promise<{ id: string; created: boolean }> => { - const lowerTitle = title.toLowerCase(); - const cachedId = existingTagsMap.get(lowerTitle); - if (cachedId) { - return { id: cachedId, created: false }; - } - const created = await joplin.data.post(['tags'], null, { title }); - existingTagsMap.set(lowerTitle, created.id); - return { id: created.id, created: true }; - }; - - // Optimized helper to get or create folder - const getOrCreateFolderOptimized = async ( - title: string, - parentId = '', - ): Promise<{ id: string; created: boolean }> => { - const key = `${title.toLowerCase()}\x1F${parentId}`; - const cachedId = existingFoldersMap.get(key); - if (cachedId) { - return { id: cachedId, created: false }; - } - const created = await joplin.data.post(['folders'], null, { - title, - parent_id: parentId || undefined, - }); - existingFoldersMap.set(key, created.id); - return { id: created.id, created: true }; - }; + // Fetch existing items using modular helpers + const existingTagsMap = await fetchExistingTags(); + const existingFoldersMap = await fetchExistingFolders(); const uniqueClusterIds = Array.from(new Set(assignments.filter((id) => id >= 0))); // 1. Create/retrieve tags if needed - const tagMap: { [clusterId: number]: string } = {}; const createdTagIds: string[] = []; + let tagMap: { [clusterId: number]: string } = {}; if (options.method === 'tags' || options.method === 'both') { - for (const clusterId of uniqueClusterIds) { - const clusterName = clusterNames[clusterId] || `Cluster ${clusterId + 1}`; - const tagName = clusterName; - const { id: tagId, created } = await getOrCreateTagOptimized(tagName); - tagMap[clusterId] = tagId; - if (created) { - createdTagIds.push(tagId); - } - } + tagMap = await initializeClusterTags(uniqueClusterIds, clusterNames, existingTagsMap, createdTagIds); } // 2. Create folders if needed - const folderMap: { [clusterId: number]: string } = {}; const createdFolderIds: string[] = []; + let folderMap: { [clusterId: number]: string } = {}; let uncategorizedFolderId = ''; if (options.method === 'notebooks' || options.method === 'both') { - for (const clusterId of uniqueClusterIds) { - const clusterName = clusterNames[clusterId] || `Cluster ${clusterId + 1}`; - const { id: childFolderId, created } = await getOrCreateFolderOptimized(clusterName); - folderMap[clusterId] = childFolderId; - if (created) { - createdFolderIds.push(childFolderId); - } - } - - if (assignments.includes(-1)) { - const { id: noiseFolderId, created } = await getOrCreateFolderOptimized('Uncategorized'); - uncategorizedFolderId = noiseFolderId; - if (created) { - createdFolderIds.push(noiseFolderId); - } - } + const initFolders = await initializeClusterNotebooks( + uniqueClusterIds, + clusterNames, + assignments, + existingFoldersMap, + createdFolderIds, + ); + folderMap = initFolders.folderMap; + uncategorizedFolderId = initFolders.uncategorizedFolderId; } // 3. Process notes & build change log @@ -183,67 +112,37 @@ export async function applyCategorizationChanges( continue; } - // Handle tags + // Handle tags using modular helper if ((options.method === 'tags' || options.method === 'both') && clusterId >= 0) { - const addedTagIds: string[] = []; - - // Apply the main cluster name tag (as grouping tag) - const mainTagId = tagMap[clusterId]; - if (mainTagId) { - try { - await joplin.data.post(['tags', mainTagId, 'notes'], null, { id: note.noteId }); - addedTagIds.push(mainTagId); - } catch (tagErr) { - log(`Tag ${mainTagId} may already be on note ${note.noteId}: ${tagErr}`); - } - } - - // Get all extracted specific tags for this cluster - const specificTags = clusterTags[clusterId] || []; - for (const tagText of specificTags) { - // Don't duplicate the main cluster tag if it's already applied - const clusterName = clusterNames[clusterId] || `Cluster ${clusterId + 1}`; - if (tagText.toLowerCase() === clusterName.toLowerCase()) { - continue; - } - - if (matchesKeyword(noteTitle, noteBody, tagText)) { - const { id: tagId, created } = await getOrCreateTagOptimized(tagText); - try { - await joplin.data.post(['tags', tagId, 'notes'], null, { id: note.noteId }); - addedTagIds.push(tagId); - if (created) { - createdTagIds.push(tagId); - } - } catch (tagErr) { - log(`Tag ${tagId} may already be on note ${note.noteId}: ${tagErr}`); - } - } - } - + const addedTagIds = await applyTagsToNote( + note, + clusterId, + noteTitle, + noteBody, + clusterNames, + clusterTags, + tagMap, + existingTagsMap, + createdTagIds, + ); if (addedTagIds.length > 0) { changeEntry.addedTagIds = addedTagIds; modified = true; } } - // Handle folders + // Handle folders using modular helper if (options.method === 'notebooks' || options.method === 'both') { - let targetFolderId = ''; - if (clusterId >= 0) { - targetFolderId = folderMap[clusterId]; - } else if (clusterId === -1 && uncategorizedFolderId) { - targetFolderId = uncategorizedFolderId; - } - - if (targetFolderId && targetFolderId !== originalParentId) { - try { - await joplin.data.put(['notes', note.noteId], null, { parent_id: targetFolderId }); - changeEntry.originalParentId = originalParentId; - modified = true; - } catch (moveErr) { - log(`Failed to move note ${note.noteId} to folder ${targetFolderId}: ${moveErr}`); - } + const folderResult = await moveNoteToFolder( + note.noteId, + clusterId, + originalParentId, + folderMap, + uncategorizedFolderId, + ); + if (folderResult.modified) { + changeEntry.originalParentId = folderResult.originalParentId; + modified = true; } } @@ -296,33 +195,17 @@ export async function undoCategorizationChanges(setPanelState: (state: PanelMess // Remove added tag from note (old format) if (entry.addedTagId) { - try { - await joplin.data.delete(['tags', entry.addedTagId, 'notes', entry.noteId]); - } catch (tagErr) { - log(`Undo: tag ${entry.addedTagId} removal failed for note ${entry.noteId}: ${tagErr}`); - } + await removeTagsFromNote(entry.noteId, [entry.addedTagId]); } // Remove added tags from note (new format) if (entry.addedTagIds) { - for (const tagId of entry.addedTagIds) { - try { - await joplin.data.delete(['tags', tagId, 'notes', entry.noteId]); - } catch (tagErr) { - log(`Undo: tag ${tagId} removal failed for note ${entry.noteId}: ${tagErr}`); - } - } + await removeTagsFromNote(entry.noteId, entry.addedTagIds); } // Restore parent notebook if (entry.originalParentId) { - try { - await joplin.data.put(['notes', entry.noteId], null, { parent_id: entry.originalParentId }); - } catch (folderErr) { - log( - `Undo: restoring folder ${entry.originalParentId} failed for note ${entry.noteId}: ${folderErr}`, - ); - } + await restoreNotebook(entry.noteId, entry.originalParentId); } setPanelState({ @@ -335,52 +218,13 @@ export async function undoCategorizationChanges(setPanelState: (state: PanelMess // 2. Delete created folders if they exist (check for notes AND subfolders) if (changeLog.createdFolderIds && changeLog.createdFolderIds.length > 0) { setPanelState({ type: 'undo_status', text: 'Deleting created notebooks...' }); - - // Fetch all folders once to check for subfolders in memory - const undoAllFolders: any[] = []; - let undoPage = 1; - while (undoPage <= 500) { - const res = await joplin.data.get(['folders'], { - page: undoPage, - limit: 100, - fields: ['id', 'parent_id'], - }); - undoAllFolders.push(...res.items); - if (!res.has_more) break; - undoPage++; - } - const undoParentIds = new Set(undoAllFolders.map((f: any) => f.parent_id).filter((pid) => !!pid)); - - for (const folderId of changeLog.createdFolderIds) { - try { - // Skip if folder has subfolders - if (undoParentIds.has(folderId)) { - log(`Undo: skipping folder ${folderId} — has subfolders`); - continue; - } - // Skip if folder still has notes - const notesInFolder = await joplin.data.get(['folders', folderId, 'notes'], { limit: 1 }); - if (notesInFolder.items.length > 0) { - log(`Undo: skipping non-empty folder ${folderId}`); - continue; - } - await joplin.data.delete(['folders', folderId]); - } catch (folderErr) { - log(`Undo: failed to delete created folder ${folderId}: ${folderErr}`); - } - } + await deleteCreatedFolders(changeLog.createdFolderIds); } // 3. Delete created tags if they exist if (changeLog.createdTagIds && changeLog.createdTagIds.length > 0) { setPanelState({ type: 'undo_status', text: 'Deleting created tags...' }); - for (const tagId of changeLog.createdTagIds) { - try { - await joplin.data.delete(['tags', tagId]); - } catch (tagErr) { - log(`Undo: failed to delete created tag ${tagId}: ${tagErr}`); - } - } + await deleteCreatedTags(changeLog.createdTagIds); } // Clear change log @@ -419,36 +263,7 @@ export async function cleanUpEmptyNotebooks(setPanelState: (state: PanelMessage) return; } - let deletedCount = 0; - - // Fetch all folders once to map parent-child relationships in memory - const allFolders: any[] = []; - let page = 1; - while (page <= 500) { - const res = await joplin.data.get(['folders'], { page, limit: 100, fields: ['id', 'parent_id'] }); - allFolders.push(...res.items); - if (!res.has_more) break; - page++; - } - const parentFolderIds = new Set(allFolders.map((f: any) => f.parent_id).filter((pid) => !!pid)); - - for (const folderId of originalParentIds) { - try { - // 1. Check if it has subfolders in memory - if (parentFolderIds.has(folderId)) { - continue; - } - - // 2. Check notes count in this folder - const notesInFolder = await joplin.data.get(['folders', folderId, 'notes'], { limit: 1 }); - if (notesInFolder.items.length === 0) { - await joplin.data.delete(['folders', folderId]); - deletedCount++; - } - } catch (err) { - log(`Cleanup: failed to check or delete folder ${folderId}: ${err}`); - } - } + const deletedCount = await cleanUpFolders(originalParentIds); setPanelState({ type: 'cleanup_complete', diff --git a/src/commands/applyNotebooks.ts b/src/commands/applyNotebooks.ts new file mode 100644 index 0000000..6937c8d --- /dev/null +++ b/src/commands/applyNotebooks.ts @@ -0,0 +1,176 @@ +import joplin from 'api'; +import { log } from '../utils/logger'; + +export async function fetchExistingFolders(): Promise> { + const allFoldersList: any[] = []; + let folderPage = 1; + const MAX_PAGES = 500; + while (folderPage <= MAX_PAGES) { + const res = await joplin.data.get(['folders'], { + page: folderPage, + limit: 100, + fields: ['id', 'title', 'parent_id'], + }); + allFoldersList.push(...res.items); + if (!res.has_more) break; + folderPage++; + } + return new Map( + allFoldersList.map((f: any) => [`${f.title.toLowerCase()}\x1F${f.parent_id || ''}`, f.id]), + ); +} + +export async function getOrCreateFolder( + existingFoldersMap: Map, + title: string, + parentId = '', +): Promise<{ id: string; created: boolean }> { + const key = `${title.toLowerCase()}\x1F${parentId}`; + const cachedId = existingFoldersMap.get(key); + if (cachedId) { + return { id: cachedId, created: false }; + } + const created = await joplin.data.post(['folders'], null, { + title, + parent_id: parentId || undefined, + }); + existingFoldersMap.set(key, created.id); + return { id: created.id, created: true }; +} + +export async function initializeClusterNotebooks( + uniqueClusterIds: number[], + clusterNames: { [clusterId: number]: string }, + assignments: number[], + existingFoldersMap: Map, + createdFolderIds: string[], +): Promise<{ folderMap: { [clusterId: number]: string }; uncategorizedFolderId: string }> { + const folderMap: { [clusterId: number]: string } = {}; + let uncategorizedFolderId = ''; + + for (const clusterId of uniqueClusterIds) { + const clusterName = clusterNames[clusterId] || `Cluster ${clusterId + 1}`; + const { id: childFolderId, created } = await getOrCreateFolder(existingFoldersMap, clusterName); + folderMap[clusterId] = childFolderId; + if (created) { + createdFolderIds.push(childFolderId); + } + } + + if (assignments.includes(-1)) { + const { id: noiseFolderId, created } = await getOrCreateFolder(existingFoldersMap, 'Uncategorized'); + uncategorizedFolderId = noiseFolderId; + if (created) { + createdFolderIds.push(noiseFolderId); + } + } + + return { folderMap, uncategorizedFolderId }; +} + +export async function moveNoteToFolder( + noteId: string, + clusterId: number, + originalParentId: string, + folderMap: { [clusterId: number]: string }, + uncategorizedFolderId: string, +): Promise<{ originalParentId?: string; modified: boolean }> { + let targetFolderId = ''; + if (clusterId >= 0) { + targetFolderId = folderMap[clusterId]; + } else if (clusterId === -1 && uncategorizedFolderId) { + targetFolderId = uncategorizedFolderId; + } + + if (targetFolderId && targetFolderId !== originalParentId) { + try { + await joplin.data.put(['notes', noteId], null, { parent_id: targetFolderId }); + return { originalParentId, modified: true }; + } catch (moveErr) { + log(`Failed to move note ${noteId} to folder ${targetFolderId}: ${moveErr}`); + } + } + + return { modified: false }; +} + +export async function restoreNotebook(noteId: string, originalParentId: string) { + try { + await joplin.data.put(['notes', noteId], null, { parent_id: originalParentId }); + } catch (folderErr) { + log(`Undo: restoring folder ${originalParentId} failed for note ${noteId}: ${folderErr}`); + } +} + +export async function deleteCreatedFolders(createdFolderIds: string[]) { + // Fetch all folders once to check for subfolders in memory + const undoAllFolders: any[] = []; + let undoPage = 1; + const MAX_PAGES = 500; + while (undoPage <= MAX_PAGES) { + const res = await joplin.data.get(['folders'], { + page: undoPage, + limit: 100, + fields: ['id', 'parent_id'], + }); + undoAllFolders.push(...res.items); + if (!res.has_more) break; + undoPage++; + } + const undoParentIds = new Set(undoAllFolders.map((f: any) => f.parent_id).filter((pid) => !!pid)); + + for (const folderId of createdFolderIds) { + try { + // Skip if folder has subfolders + if (undoParentIds.has(folderId)) { + log(`Undo: skipping folder ${folderId} — has subfolders`); + continue; + } + // Skip if folder still has notes + const notesInFolder = await joplin.data.get(['folders', folderId, 'notes'], { limit: 1 }); + if (notesInFolder.items.length > 0) { + log(`Undo: skipping non-empty folder ${folderId}`); + continue; + } + await joplin.data.delete(['folders', folderId]); + } catch (folderErr) { + log(`Undo: failed to delete created folder ${folderId}: ${folderErr}`); + } + } +} + +export async function cleanUpFolders(originalParentIds: Set): Promise { + let deletedCount = 0; + + // Fetch all folders once to map parent-child relationships in memory + const allFolders: any[] = []; + let page = 1; + const MAX_PAGES = 500; + while (page <= MAX_PAGES) { + const res = await joplin.data.get(['folders'], { page, limit: 100, fields: ['id', 'parent_id'] }); + allFolders.push(...res.items); + if (!res.has_more) break; + page++; + } + const parentFolderIds = new Set(allFolders.map((f: any) => f.parent_id).filter((pid) => !!pid)); + + for (const folderId of originalParentIds) { + try { + // 1. Check if it has subfolders in memory + if (parentFolderIds.has(folderId)) { + continue; + } + + // 2. Check notes count in this folder + const notesInFolder = await joplin.data.get(['folders', folderId, 'notes'], { limit: 1 }); + if (notesInFolder.items.length === 0) { + await joplin.data.delete(['folders', folderId]); + deletedCount++; + } + } catch (err) { + log(`Cleanup: failed to check or delete folder ${folderId}: ${err}`); + } + } + + return deletedCount; +} diff --git a/src/commands/applyTags.ts b/src/commands/applyTags.ts new file mode 100644 index 0000000..1ff1b4f --- /dev/null +++ b/src/commands/applyTags.ts @@ -0,0 +1,133 @@ +import joplin from 'api'; +import { PanelNote } from '../types/panel'; +import { log } from '../utils/logger'; + +function matchesKeyword(title: string, body: string, keyword: string): boolean { + const lowerKeyword = keyword.toLowerCase(); + const lowerTitle = title.toLowerCase(); + const lowerBody = body.toLowerCase(); + + try { + const escaped = lowerKeyword.replace(/[-\x2f\\^$*+?.()|[\]{}]/g, '\\$&'); + // Unicode-aware word boundary matching + const regex = new RegExp(`(?:^|[^\\p{L}\\p{N}])` + escaped + `(?:$|[^\\p{L}\\p{N}])`, 'u'); + return regex.test(lowerTitle) || regex.test(lowerBody); + } catch (e) { + return lowerTitle.includes(lowerKeyword) || lowerBody.includes(lowerKeyword); + } +} + +export async function fetchExistingTags(): Promise> { + const allTags: any[] = []; + let tagPage = 1; + const MAX_PAGES = 500; + while (tagPage <= MAX_PAGES) { + const res = await joplin.data.get(['tags'], { page: tagPage, limit: 100, fields: ['id', 'title'] }); + allTags.push(...res.items); + if (!res.has_more) break; + tagPage++; + } + return new Map(allTags.map((t: any) => [t.title.toLowerCase(), t.id])); +} + +export async function getOrCreateTag( + existingTagsMap: Map, + title: string, +): Promise<{ id: string; created: boolean }> { + const lowerTitle = title.toLowerCase(); + const cachedId = existingTagsMap.get(lowerTitle); + if (cachedId) { + return { id: cachedId, created: false }; + } + const created = await joplin.data.post(['tags'], null, { title }); + existingTagsMap.set(lowerTitle, created.id); + return { id: created.id, created: true }; +} + +export async function initializeClusterTags( + uniqueClusterIds: number[], + clusterNames: { [clusterId: number]: string }, + existingTagsMap: Map, + createdTagIds: string[], +): Promise<{ [clusterId: number]: string }> { + const tagMap: { [clusterId: number]: string } = {}; + for (const clusterId of uniqueClusterIds) { + const clusterName = clusterNames[clusterId] || `Cluster ${clusterId + 1}`; + const { id: tagId, created } = await getOrCreateTag(existingTagsMap, clusterName); + tagMap[clusterId] = tagId; + if (created) { + createdTagIds.push(tagId); + } + } + return tagMap; +} + +export async function applyTagsToNote( + note: PanelNote, + clusterId: number, + noteTitle: string, + noteBody: string, + clusterNames: { [clusterId: number]: string }, + clusterTags: { [clusterId: number]: string[] }, + tagMap: { [clusterId: number]: string }, + existingTagsMap: Map, + createdTagIds: string[], +): Promise { + const addedTagIds: string[] = []; + + // Apply the main cluster name tag (as grouping tag) + const mainTagId = tagMap[clusterId]; + if (mainTagId) { + try { + await joplin.data.post(['tags', mainTagId, 'notes'], null, { id: note.noteId }); + addedTagIds.push(mainTagId); + } catch (tagErr) { + log(`Tag ${mainTagId} may already be on note ${note.noteId}: ${tagErr}`); + } + } + + // Get all extracted specific tags for this cluster + const specificTags = clusterTags[clusterId] || []; + for (const tagText of specificTags) { + // Don't duplicate the main cluster tag if it's already applied + const clusterName = clusterNames[clusterId] || `Cluster ${clusterId + 1}`; + if (tagText.toLowerCase() === clusterName.toLowerCase()) { + continue; + } + + if (matchesKeyword(noteTitle, noteBody, tagText)) { + const { id: tagId, created } = await getOrCreateTag(existingTagsMap, tagText); + try { + await joplin.data.post(['tags', tagId, 'notes'], null, { id: note.noteId }); + addedTagIds.push(tagId); + if (created) { + createdTagIds.push(tagId); + } + } catch (tagErr) { + log(`Tag ${tagId} may already be on note ${note.noteId}: ${tagErr}`); + } + } + } + + return addedTagIds; +} + +export async function removeTagsFromNote(noteId: string, addedTagIds: string[]) { + for (const tagId of addedTagIds) { + try { + await joplin.data.delete(['tags', tagId, 'notes', noteId]); + } catch (tagErr) { + log(`Undo: tag ${tagId} removal failed for note ${noteId}: ${tagErr}`); + } + } +} + +export async function deleteCreatedTags(createdTagIds: string[]) { + for (const tagId of createdTagIds) { + try { + await joplin.data.delete(['tags', tagId]); + } catch (tagErr) { + log(`Undo: failed to delete created tag ${tagId}: ${tagErr}`); + } + } +} diff --git a/src/pipeline/EmbeddingWorkerOrchestrator.ts b/src/pipeline/EmbeddingWorkerOrchestrator.ts new file mode 100644 index 0000000..ab2d71c --- /dev/null +++ b/src/pipeline/EmbeddingWorkerOrchestrator.ts @@ -0,0 +1,262 @@ +import { getEncoding } from 'js-tiktoken'; +import { NoteVector, WorkerMessage } from '../types/embed'; +import { VectorCache } from './vectorCache'; +import { isGenericTitle } from '../utils/titleFilter'; +import { log, logErr } from '../utils/logger'; +import { averageVectors, blendVectors, computeTitleWeight, cosineSimilarity } from './vectorAggregator'; +import { isValidEmbeddingVector } from './pipelineConfig'; + +const enc = getEncoding('cl100k_base'); +const MAX_TOKENS = 200; + +export interface OrchestratorCallbacks { + onStatus: (text: string) => void; + onProgress: (current: number, total: number, cached: number, skipped: number) => void; +} + +export interface EmbeddingResult { + noteVectors: NoteVector[]; + cachedCount: number; + skippedCount: number; + totalInferenceTime: number; +} + +export class EmbeddingWorkerOrchestrator { + private currentNoteIndex = 0; + private currentChunkIndex = 0; + private currentNoteChunks: string[] = []; + private currentChunkEmbeddings: number[][] = []; + private currentBodyVector: number[] = []; + private isEmbeddingTitle = false; + private totalInferenceTime = 0; + private skippedCount = 0; + private cachedCount = 0; + private currentNoteHash = ''; + private noteVectors: NoteVector[] = []; + private worker: Worker | null = null; + private resolvePromise: ((value: EmbeddingResult) => void) | null = null; + private rejectPromise: ((reason: any) => void) | null = null; + + constructor( + private installDir: string, + private notes: any[], + private cache: VectorCache, + private callbacks: OrchestratorCallbacks, + ) {} + + public async run(): Promise { + return new Promise((resolve, reject) => { + this.resolvePromise = resolve; + this.rejectPromise = reject; + + try { + this.callbacks.onStatus('Loading model...'); + this.worker = new Worker(`${this.installDir}/worker/embedWorker.js`); + + this.worker.onerror = (err: ErrorEvent) => { + logErr('Worker error:', err.message); + this.cache.cancelUpdate(); + this.terminate(); + reject(new Error('Embedding worker failed: ' + err.message)); + }; + + this.worker.onmessage = async (event: MessageEvent) => { + const data = event.data; + + if (data.type === 'load-result') { + if (data.success) { + log(`Model loaded in ${(data.loadTime / 1000).toFixed(1)}s, device: ${data.device}`); + this.callbacks.onStatus('Embedding notes...'); + await this.processNextNote(); + } else { + logErr('Model load failed:', data.error); + this.cache.cancelUpdate(); + this.terminate(); + reject(new Error('Failed to load embedding model: ' + (data.error || 'unknown error'))); + } + return; + } + + if (data.type === 'embed-result') { + const note = this.notes[this.currentNoteIndex]; + + if (!data.success) { + logErr(`Failed to embed note "${note.title.slice(0, 30)}":`, data.error); + this.currentNoteIndex++; + await this.processNextNote(); + return; + } + + this.totalInferenceTime += data.inferenceTime; + + if (this.isEmbeddingTitle) { + const titleEmbedding = data.embedding; + + if (this.currentBodyVector.length > 0) { + const sim = cosineSimilarity(this.currentBodyVector, titleEmbedding); + const alpha = computeTitleWeight(sim); + const finalVector = blendVectors(this.currentBodyVector, titleEmbedding, alpha); + await this.finalizeNote(finalVector, alpha, this.currentNoteHash); + } else { + await this.finalizeNote(titleEmbedding, 1.0, this.currentNoteHash); + } + } else { + this.currentChunkEmbeddings.push(data.embedding); + log( + `[${this.currentNoteIndex + 1}/${this.notes.length}] embedded chunk ${this.currentChunkIndex + 1}/${this.currentNoteChunks.length} of "${note.title.slice(0, 30)}"`, + ); + + this.currentChunkIndex++; + if (this.currentChunkIndex < this.currentNoteChunks.length) { + this.worker?.postMessage({ + type: 'embed', + text: this.currentNoteChunks[this.currentChunkIndex], + noteId: note.id, + }); + } else { + this.currentBodyVector = averageVectors(this.currentChunkEmbeddings); + + if (!isGenericTitle(note.title)) { + this.isEmbeddingTitle = true; + this.worker?.postMessage({ type: 'embed', text: note.title, noteId: note.id }); + } else { + await this.finalizeNote(this.currentBodyVector, 0, this.currentNoteHash); + } + } + } + } + }; + + this.worker.postMessage({ type: 'load' }); + } catch (err) { + this.terminate(); + reject(err); + } + }); + } + + private terminate() { + if (this.worker) { + this.worker.terminate(); + this.worker = null; + } + } + + private reportProgress() { + // current = notes finalized so far (embedded + cached + skipped) + const processed = this.noteVectors.length + this.skippedCount; + this.callbacks.onProgress(processed, this.notes.length, this.cachedCount, this.skippedCount); + } + + private prepareNoteChunks(text: string): string[] { + const tokens = enc.encode(text); + const chunks: string[] = []; + if (tokens.length === 0) return []; + + for (let i = 0; i < tokens.length; i += MAX_TOKENS) { + const chunkTokens = tokens.slice(i, i + MAX_TOKENS); + chunks.push(enc.decode(chunkTokens)); + } + return chunks; + } + + private async finalizeNote(vector: number[], titleWeight: number, hash: string) { + const note = this.notes[this.currentNoteIndex]; + this.noteVectors.push({ noteId: note.id, title: note.title, vector, titleWeight }); + + await this.cache.upsertItem(note.id, vector, { + title: note.title, + hash, + updatedTime: note.updated_time, + titleWeight, + }); + + this.reportProgress(); + + this.currentNoteIndex++; + await this.processNextNote(); + } + + private async processNextNote() { + this.currentChunkIndex = 0; + this.currentNoteChunks = []; + this.currentChunkEmbeddings = []; + this.currentBodyVector = []; + this.isEmbeddingTitle = false; + + // Skip notes with empty body and generic title, and bypass cached notes + while (this.currentNoteIndex < this.notes.length) { + const note = this.notes[this.currentNoteIndex]; + + if (note.body.length === 0 && isGenericTitle(note.title)) { + log( + `[${this.currentNoteIndex + 1}/${this.notes.length}] skipped "${note.title.slice(0, 30)}" (empty body, generic title)`, + ); + this.skippedCount++; + this.currentNoteIndex++; + this.reportProgress(); + continue; + } + + this.currentNoteHash = this.cache.computeHash(note.title, note.body); + const cachedItem = await this.cache.getItem(note.id); + + if (cachedItem && cachedItem.metadata.hash === this.currentNoteHash) { + if (isValidEmbeddingVector(cachedItem.vector)) { + log( + `[${this.currentNoteIndex + 1}/${this.notes.length}] cache hit for "${note.title.slice(0, 30)}"`, + ); + this.noteVectors.push({ + noteId: note.id, + title: note.title, + vector: cachedItem.vector, + titleWeight: cachedItem.metadata.titleWeight ?? 0, + }); + this.cachedCount++; + this.currentNoteIndex++; + this.reportProgress(); + continue; + } else { + log( + `[${this.currentNoteIndex + 1}/${this.notes.length}] cache invalid (contains null/NaN) for "${note.title.slice(0, 30)}"`, + ); + } + } + + break; + } + + if (this.currentNoteIndex >= this.notes.length) { + this.terminate(); + if (this.resolvePromise) { + this.resolvePromise({ + noteVectors: this.noteVectors, + cachedCount: this.cachedCount, + skippedCount: this.skippedCount, + totalInferenceTime: this.totalInferenceTime, + }); + } + } else { + const note = this.notes[this.currentNoteIndex]; + this.callbacks.onStatus(`Embedding "${note.title.slice(0, 40)}"...`); + + if (note.body.length === 0) { + this.isEmbeddingTitle = true; + this.worker?.postMessage({ type: 'embed', text: note.title, noteId: note.id }); + } else { + this.currentNoteChunks = this.prepareNoteChunks(note.body); + if (this.currentNoteChunks.length === 0) { + // Whitespace-only body — treat as title-only note + this.isEmbeddingTitle = true; + this.worker?.postMessage({ type: 'embed', text: note.title, noteId: note.id }); + } else { + this.worker?.postMessage({ + type: 'embed', + text: this.currentNoteChunks[0], + noteId: note.id, + }); + } + } + } + } +} diff --git a/src/pipeline/clustering/clusterNaming.ts b/src/pipeline/clustering/clusterNaming.ts new file mode 100644 index 0000000..a152a8e --- /dev/null +++ b/src/pipeline/clustering/clusterNaming.ts @@ -0,0 +1,84 @@ +import { TAXONOMY_MAPPING } from './data/taxonomy'; +import { filterDemotedUnigrams } from './tagExtraction'; + +const ACRONYMS = new Set(['sip', 'api', 'ui', 'url', 'html', 'css', 'js', 'db', 'sql', 'onnx']); + +/** + * Capitalizes a phrase to Title Case, preserving common acronyms in uppercase. + */ +export function toTitleCase(phrase: string): string { + return phrase + .split(' ') + .map((word) => { + const lower = word.toLowerCase(); + if (ACRONYMS.has(lower)) { + return word.toUpperCase(); + } + return word.charAt(0).toUpperCase() + word.slice(1); + }) + .join(' '); +} + +/** + * Checks if two phrases share any words (case-insensitive). + */ +export function shareWords(phraseA: string, phraseB: string): boolean { + const wordsA = new Set(phraseA.toLowerCase().split(' ')); + const wordsB = phraseB.toLowerCase().split(' '); + return wordsB.some((w) => wordsA.has(w)); +} + +/** + * Checks the top 3 ngrams of a cluster against a static taxonomy to match common topics. + */ +export function getTaxonomyCategory(scores: { ngram: string; score: number }[]): string | null { + const candidates = scores.slice(0, 3).map((s) => s.ngram.toLowerCase()); + + for (const cand of candidates) { + const words = cand.split(' '); + for (const mapping of TAXONOMY_MAPPING) { + for (const keyword of mapping.keywords) { + if (words.includes(keyword) || cand === keyword) { + return mapping.category; + } + } + } + } + return null; +} + +/** + * Generates a descriptive name for a cluster using the scoring list and clusterId. + */ +export function generateClusterName(scores: { ngram: string; score: number }[], clusterId: number): string { + const filteredScores = filterDemotedUnigrams(scores); + + if (filteredScores.length === 0 || filteredScores[0].score <= 0) { + return clusterId % 2 === 0 ? 'General' : 'Miscellaneous'; + } + + // Try matching against high-level taxonomy first + const taxonomyCategory = getTaxonomyCategory(filteredScores); + if (taxonomyCategory) { + return taxonomyCategory; + } + + const top1 = filteredScores[0]; + let top2: { ngram: string; score: number } | undefined; + + // Find the next highest-scoring phrase that doesn't share any words with the first phrase + for (let i = 1; i < filteredScores.length; i++) { + if (filteredScores[i].score <= 0) break; + if (!shareWords(top1.ngram, filteredScores[i].ngram)) { + top2 = filteredScores[i]; + break; + } + } + + // Join them if the second has at least 60% of the score of the first + if (top2 && top2.score >= top1.score * 0.6) { + return `${toTitleCase(top1.ngram)} & ${toTitleCase(top2.ngram)}`; + } + + return toTitleCase(top1.ngram); +} diff --git a/src/pipeline/clustering/data/stopWords.ts b/src/pipeline/clustering/data/stopWords.ts new file mode 100644 index 0000000..1935113 --- /dev/null +++ b/src/pipeline/clustering/data/stopWords.ts @@ -0,0 +1,323 @@ +export const STOP_WORDS = new Set([ + // English articles, prepositions, conjunctions, pronouns (all length >= 3) + 'about', + 'above', + 'after', + 'again', + 'against', + 'all', + 'and', + 'any', + 'are', + 'arent', + 'because', + 'been', + 'before', + 'being', + 'below', + 'between', + 'both', + 'but', + 'cant', + 'cannot', + 'could', + 'couldnt', + 'did', + 'didnt', + 'does', + 'doesnt', + 'doing', + 'dont', + 'down', + 'during', + 'each', + 'few', + 'for', + 'from', + 'further', + 'had', + 'hadnt', + 'has', + 'hasnt', + 'have', + 'havent', + 'having', + 'hed', + 'hell', + 'hes', + 'her', + 'here', + 'heres', + 'hers', + 'herself', + 'him', + 'himself', + 'his', + 'how', + 'hows', + 'ill', + 'its', + 'itself', + 'lets', + 'more', + 'most', + 'mustnt', + 'myself', + 'nor', + 'not', + 'off', + 'once', + 'only', + 'other', + 'ought', + 'our', + 'ours', + 'ourselves', + 'out', + 'over', + 'own', + 'same', + 'shant', + 'she', + 'shed', + 'shell', + 'shes', + 'should', + 'shouldnt', + 'some', + 'such', + 'than', + 'that', + 'thats', + 'the', + 'their', + 'theirs', + 'them', + 'themselves', + 'then', + 'there', + 'theres', + 'these', + 'they', + 'theyd', + 'theyll', + 'theyre', + 'theyve', + 'this', + 'those', + 'through', + 'too', + 'under', + 'until', + 'very', + 'was', + 'wasnt', + 'wed', + 'well', + 'were', + 'weve', + 'werent', + 'what', + 'whats', + 'when', + 'whens', + 'where', + 'wheres', + 'which', + 'while', + 'who', + 'whos', + 'whom', + 'why', + 'whys', + 'with', + 'wont', + 'would', + 'wouldnt', + 'youd', + 'youll', + 'youre', + 'youve', + 'your', + 'yours', + 'yourself', + 'yourselves', + + // More prepositions, adverbs, and common noise verbs (to clean up phrases) + 'without', + 'within', + 'throughout', + 'around', + 'going', + 'goes', + 'went', + 'getting', + 'got', + 'having', + 'making', + 'taking', + 'actually', + 'really', + 'basically', + 'simply', + 'mainly', + 'mostly', + 'highly', + 'fully', + 'totally', + 'completely', + 'extremely', + 'very', + 'quite', + 'pretty', + 'somewhat', + 'rather', + 'indeed', + 'always', + 'never', + 'sometimes', + 'often', + 'usually', + 'probably', + 'possibly', + 'maybe', + 'crazy', + 'easy', + 'hard', + 'difficult', + 'simple', + 'good', + 'bad', + 'best', + 'worst', + 'better', + 'worse', + 'new', + 'old', + 'first', + 'last', + 'next', + 'prev', + 'previous', + 'current', + 'different', + 'same', + 'other', + 'another', + 'each', + 'every', + 'many', + 'much', + 'few', + 'several', + 'some', + 'any', + 'no', + 'work', + 'thing', + 'things', + 'stuff', + 'name', + 'value', + 'data', + 'user', + 'item', + 'items', + + // Markdown/HTML structure words or general noise words (all length >= 3) + 'http', + 'https', + 'www', + 'com', + 'org', + 'net', + 'html', + 'xml', + 'css', + 'img', + 'href', + 'src', + 'div', + 'span', + 'class', + 'get', + 'post', + 'put', + 'delete', + 'use', + 'using', + 'used', + 'make', + 'made', + 'take', + 'took', + 'see', + 'saw', + 'also', + 'like', + 'one', + 'two', + 'three', + 'four', + 'five', + 'six', + 'seven', + 'eight', + 'nine', + 'ten', + 'first', + 'second', + 'third', + + // Code keywords / programming syntax + 'const', + 'let', + 'var', + 'function', + 'return', + 'class', + 'interface', + 'type', + 'import', + 'export', + 'void', + 'string', + 'number', + 'boolean', + 'any', + 'public', + 'private', + 'protected', + 'async', + 'await', + 'null', + 'undefined', + 'true', + 'false', + 'switch', + 'case', + 'break', + + // Generic Joplin / note-taking fillers (often pollute tags in this context) + 'note', + 'notes', + 'joplin', + 'plugin', + 'folder', + 'folders', + 'notebook', + 'notebooks', + 'tag', + 'tags', + 'todo', + 'todos', + 'task', + 'tasks', + 'file', + 'files', + 'page', + 'pages', + 'data', + 'info', + 'information', +]); diff --git a/src/pipeline/clustering/data/taxonomy.ts b/src/pipeline/clustering/data/taxonomy.ts new file mode 100644 index 0000000..21526ff --- /dev/null +++ b/src/pipeline/clustering/data/taxonomy.ts @@ -0,0 +1,101 @@ +export const TAXONOMY_MAPPING: { keywords: string[]; category: string }[] = [ + { + keywords: ['travel', 'flight', 'trip', 'train', 'vacation', 'backpacking', 'itinerary', 'packing', 'flights'], + category: 'Travel', + }, + { + keywords: [ + 'fund', + 'stock', + 'invest', + 'portfolio', + 'finance', + 'saving', + 'tax', + 'sip', + 'lump', + 'stocks', + 'funds', + 'investment', + 'investments', + ], + category: 'Investment', + }, + { + keywords: ['prep', 'smoothie', 'protein', 'macro', 'macros', 'diet', 'nutrition', 'meal'], + category: 'Meal Prep', + }, + { + keywords: [ + 'recipe', + 'recipes', + 'starter', + 'sourdough', + 'flour', + 'baking', + 'bread', + 'banana', + 'pasta', + 'skillet', + 'cook', + 'cooking', + 'kitchen', + ], + category: 'Recipes', + }, + { + keywords: [ + 'workout', + 'overload', + 'stretch', + 'stretching', + 'routine', + 'pain', + 'fitness', + 'exercise', + 'gym', + 'cardio', + 'back', + 'sitting', + ], + category: 'Workout', + }, + { + keywords: [ + 'code', + 'program', + 'javascript', + 'typescript', + 'node', + 'git', + 'docker', + 'graphql', + 'rest', + 'api', + 'jest', + 'test', + 'error', + 'request', + 'programming', + 'software', + 'developer', + ], + category: 'Programming', + }, + { + keywords: [ + 'psychology', + 'money', + 'meaning', + 'philosophy', + 'ravikant', + 'almanack', + 'book', + 'quotes', + 'thoughts', + 'reading', + 'naval', + ], + category: 'Books & Philosophy', + }, +]; diff --git a/src/pipeline/clustering/postProcess.ts b/src/pipeline/clustering/postProcess.ts index 15f2845..02a8e8b 100644 --- a/src/pipeline/clustering/postProcess.ts +++ b/src/pipeline/clustering/postProcess.ts @@ -1,832 +1,28 @@ import { BenchmarkResult } from '../../types/cluster'; - -export const STOP_WORDS = new Set([ - // English articles, prepositions, conjunctions, pronouns (all length >= 3) - 'about', - 'above', - 'after', - 'again', - 'against', - 'all', - 'and', - 'any', - 'are', - 'arent', - 'because', - 'been', - 'before', - 'being', - 'below', - 'between', - 'both', - 'but', - 'cant', - 'cannot', - 'could', - 'couldnt', - 'did', - 'didnt', - 'does', - 'doesnt', - 'doing', - 'dont', - 'down', - 'during', - 'each', - 'few', - 'for', - 'from', - 'further', - 'had', - 'hadnt', - 'has', - 'hasnt', - 'have', - 'havent', - 'having', - 'hed', - 'hell', - 'hes', - 'her', - 'here', - 'heres', - 'hers', - 'herself', - 'him', - 'himself', - 'his', - 'how', - 'hows', - 'ill', - 'its', - 'itself', - 'lets', - 'more', - 'most', - 'mustnt', - 'myself', - 'nor', - 'not', - 'off', - 'once', - 'only', - 'other', - 'ought', - 'our', - 'ours', - 'ourselves', - 'out', - 'over', - 'own', - 'same', - 'shant', - 'she', - 'shed', - 'shell', - 'shes', - 'should', - 'shouldnt', - 'some', - 'such', - 'than', - 'that', - 'thats', - 'the', - 'their', - 'theirs', - 'them', - 'themselves', - 'then', - 'there', - 'theres', - 'these', - 'they', - 'theyd', - 'theyll', - 'theyre', - 'theyve', - 'this', - 'those', - 'through', - 'too', - 'under', - 'until', - 'very', - 'was', - 'wasnt', - 'wed', - 'well', - 'were', - 'weve', - 'werent', - 'what', - 'whats', - 'when', - 'whens', - 'where', - 'wheres', - 'which', - 'while', - 'who', - 'whos', - 'whom', - 'why', - 'whys', - 'with', - 'wont', - 'would', - 'wouldnt', - 'youd', - 'youll', - 'youre', - 'youve', - 'your', - 'yours', - 'yourself', - 'yourselves', - - // More prepositions, adverbs, and common noise verbs (to clean up phrases) - 'without', - 'within', - 'throughout', - 'around', - 'going', - 'goes', - 'went', - 'getting', - 'got', - 'having', - 'making', - 'taking', - 'actually', - 'really', - 'basically', - 'simply', - 'mainly', - 'mostly', - 'highly', - 'fully', - 'totally', - 'completely', - 'extremely', - 'very', - 'quite', - 'pretty', - 'somewhat', - 'rather', - 'indeed', - 'always', - 'never', - 'sometimes', - 'often', - 'usually', - 'probably', - 'possibly', - 'maybe', - 'crazy', - 'easy', - 'hard', - 'difficult', - 'simple', - 'good', - 'bad', - 'best', - 'worst', - 'better', - 'worse', - 'new', - 'old', - 'first', - 'last', - 'next', - 'prev', - 'previous', - 'current', - 'different', - 'same', - 'other', - 'another', - 'each', - 'every', - 'many', - 'much', - 'few', - 'several', - 'some', - 'any', - 'no', - 'work', - 'thing', - 'things', - 'stuff', - 'name', - 'value', - 'data', - 'user', - 'item', - 'items', - - // Markdown/HTML structure words or general noise words (all length >= 3) - 'http', - 'https', - 'www', - 'com', - 'org', - 'net', - 'html', - 'xml', - 'css', - 'img', - 'href', - 'src', - 'div', - 'span', - 'class', - 'get', - 'post', - 'put', - 'delete', - 'use', - 'using', - 'used', - 'make', - 'made', - 'take', - 'took', - 'see', - 'saw', - 'also', - 'like', - 'one', - 'two', - 'three', - 'four', - 'five', - 'six', - 'seven', - 'eight', - 'nine', - 'ten', - 'first', - 'second', - 'third', - - // Code keywords / programming syntax - 'const', - 'let', - 'var', - 'function', - 'return', - 'class', - 'interface', - 'type', - 'import', - 'export', - 'void', - 'string', - 'number', - 'boolean', - 'any', - 'public', - 'private', - 'protected', - 'async', - 'await', - 'null', - 'undefined', - 'true', - 'false', - 'switch', - 'case', - 'break', - - // Generic Joplin / note-taking fillers (often pollute tags in this context) - 'note', - 'notes', - 'joplin', - 'plugin', - 'folder', - 'folders', - 'notebook', - 'notebooks', - 'tag', - 'tags', - 'todo', - 'todos', - 'task', - 'tasks', - 'file', - 'files', - 'page', - 'pages', - 'data', - 'info', - 'information', -]); - -/** Words that look like plurals but should not be singularized. */ -const SINGULAR_EXCEPTIONS = new Set(['series', 'species', 'means', 'news', 'analysis', 'basis', 'crisis']); - -/** Unigrams with character length at or below this threshold receive a 0.5x scoring penalty. */ -const SHORT_UNIGRAM_THRESHOLD = 4; - -/** - * Strips code blocks, inline code, HTML tags, markdown links/images, and URLs - * from text to avoid polluting tag extraction. - */ -export function cleanText(text: string): string { - if (!text) return ''; - let cleaned = text; - // Strip triple-backtick markdown code blocks - cleaned = cleaned.replace(/```[\s\S]*?```/g, ' '); - // Strip inline code backticks - cleaned = cleaned.replace(/`[^`]*`/g, ' '); - // Strip markdown images ![alt](url) and links [text](url) — keep the text, remove syntax - cleaned = cleaned.replace(/!?\[([^\]]*)\]\([^)]*\)/g, '$1'); - // Strip HTML tags - cleaned = cleaned.replace(/<[^>]*>/g, ' '); - // Strip URLs - cleaned = cleaned.replace(/https?:\/\/\S+/gi, ' '); - return cleaned; -} - -/** - * A lightweight, dependency-free helper to stem basic English plural words to their singular form. - * Handles common cases like -ies -> -y, -es -> - (e.g. boxes -> box), and trailing -s (notes -> note). - */ -export function singularize(word: string): string { - if (word.length <= 3) return word; - if (SINGULAR_EXCEPTIONS.has(word)) return word; - if (word.endsWith('ss')) return word; // e.g. class, process - if (word.endsWith('ies')) return word.slice(0, -3) + 'y'; // e.g. categories -> category - if (word.endsWith('es')) { - const base = word.slice(0, -2); - if ( - base.endsWith('ss') || - base.endsWith('ch') || - base.endsWith('sh') || - base.endsWith('x') || - base.endsWith('z') - ) { - return base; // e.g. classes -> class, boxes -> box - } - return word.slice(0, -1); // e.g. databases -> database, lines -> line - } - if (word.endsWith('s') && !word.endsWith('us') && !word.endsWith('is') && !word.endsWith('as')) { - return word.slice(0, -1); // e.g. notes -> note, tasks -> task - } - return word; -} - -/** - * Lowercases text, cleans it, singularizes it, and tokenizes into alphabetic words - * of length >= 3 that are not in the stop words list. - */ -export function tokenize(text: string): string[] { - const cleaned = cleanText(text).toLowerCase().replace(/[’']/g, ''); - const matches = cleaned.match(/[a-z]+/g) || []; - return matches.map(singularize).filter((w) => w.length >= 3 && !STOP_WORDS.has(w)); -} - -/** - * Generates unigrams, bigrams, and trigrams from a sequence of tokens. - */ -export function getNgrams(tokens: string[]): string[] { - const ngrams: string[] = []; - const N = tokens.length; - for (let i = 0; i < N; i++) { - // Unigram - ngrams.push(tokens[i]); - // Bigram - if (i < N - 1) { - ngrams.push(`${tokens[i]} ${tokens[i + 1]}`); - } - // Trigram - if (i < N - 2) { - ngrams.push(`${tokens[i]} ${tokens[i + 1]} ${tokens[i + 2]}`); - } - } - return ngrams; -} - -export interface DocumentText { - title: string; - body: string; -} - -/** - * Extracts descriptive tags from documents in a cluster using TF-IDF. - */ -export class TfidfExtractor { - private idfs: { [word: string]: number } = {}; - - constructor(allDocuments: DocumentText[]) { - const N = allDocuments.length; - if (N === 0) return; - - const docFreqs: { [word: string]: number } = {}; - - for (const doc of allDocuments) { - // For IDF, we only need unique words/ngrams per document — no title weighting needed - const uniqueWords = this.getUniqueDocumentWords(doc); - for (const word of uniqueWords) { - docFreqs[word] = (docFreqs[word] || 0) + 1; - } - } - - for (const word of Object.keys(docFreqs)) { - const df = docFreqs[word]; - // Max DF rule: If a word/ngram appears in > 60% of all notes, it is too generic, ignore it. - if (df / N > 0.6) { - this.idfs[word] = 0; - } else { - this.idfs[word] = Math.log(N / df) + 1; - } - } - } - - /** - * Splits the text by sentence/line boundaries and generates ngrams within segments. - * This prevents forming cross-boundary ngrams (like joining separate lines or sentences). - */ - private getSegmentNgrams(text: string): string[] { - if (!text) return []; - // Split by sentence punctuation, newlines, markdown headers, and list bullets - const segments = text.split(/[.,?!;:\n\r\-*#()[\]]+/); - const allNgrams: string[] = []; - for (const seg of segments) { - const tokens = tokenize(seg); - const ngrams = getNgrams(tokens); - for (const ng of ngrams) { - // Filter out any ngrams with consecutive duplicate words (e.g. "day day") - if (!hasConsecutiveDuplicates(ng)) { - allNgrams.push(ng); - } - } - } - return allNgrams; - } - - /** - * Returns the unique set of words/ngrams in a document (title + body), used for IDF counting. - * No title weighting — each document contributes at most 1 to each ngram's document frequency. - */ - private getUniqueDocumentWords(doc: DocumentText): Set { - const titleNgrams = this.getSegmentNgrams(doc.title || ''); - const bodyNgrams = this.getSegmentNgrams(doc.body || ''); - return new Set([...titleNgrams, ...bodyNgrams]); - } - - /** - * Returns ngrams for TF scoring with title words weighted 5x higher. - * Uses push loops instead of spread to avoid excess intermediate array allocations. - */ - private getWeightedWords(doc: DocumentText): string[] { - const titleNgrams = this.getSegmentNgrams(doc.title || ''); - const bodyNgrams = this.getSegmentNgrams(doc.body || ''); - const result: string[] = []; - // Title ngrams appear 5 times to boost their term frequency - for (let i = 0; i < 5; i++) { - for (const ng of titleNgrams) { - result.push(ng); - } - } - for (const ng of bodyNgrams) { - result.push(ng); - } - return result; - } - - /** - * Computes sorted TF-IDF scores for ngrams in the cluster documents. - * Incorporates Cluster Frequency (CF) weighting, Length Boosting, and Title Match Boosting. - */ - public extractClusterNgramsWithScores(clusterDocuments: DocumentText[]): { ngram: string; score: number }[] { - if (clusterDocuments.length === 0) return []; - - const tfs: { [ngram: string]: number } = {}; - let totalNgrams = 0; - - for (const doc of clusterDocuments) { - const weighted = this.getWeightedWords(doc); - for (const ng of weighted) { - tfs[ng] = (tfs[ng] || 0) + 1; - totalNgrams++; - } - } - - if (totalNgrams === 0) return []; - - // Count how many documents in the cluster contain each ngram - const docCounts: { [ngram: string]: number } = {}; - for (const doc of clusterDocuments) { - const titleNgrams = this.getSegmentNgrams(doc.title || ''); - const bodyNgrams = this.getSegmentNgrams(doc.body || ''); - const docNgrams = new Set([...titleNgrams, ...bodyNgrams]); - for (const ng of docNgrams) { - docCounts[ng] = (docCounts[ng] || 0) + 1; - } - } - - const scores: { ngram: string; score: number }[] = []; - - for (const ngram of Object.keys(tfs)) { - const idf = this.idfs[ngram] || 0; // default to 0 if word is ignored/generic - if (idf > 0) { - const tf = tfs[ngram] / totalNgrams; - const cf = (docCounts[ngram] || 0) / clusterDocuments.length; - - // Length boost: 1.0x for unigram, 1.5x for bigram, 2.0x for trigram - const wordCount = ngram.split(' ').length; - let lengthBoost = 1.0 + (wordCount - 1) * 0.5; - - // Penalize very short unigrams (length <= 4) to favor longer descriptive phrases - if (wordCount === 1 && ngram.length <= SHORT_UNIGRAM_THRESHOLD) { - lengthBoost *= 0.5; - } - - // Title match boost: 1.5x if it appears in any note title in this cluster - let appearsInTitle = false; - for (const doc of clusterDocuments) { - const titleNgrams = new Set(this.getSegmentNgrams(doc.title || '')); - if (titleNgrams.has(ngram)) { - appearsInTitle = true; - break; - } - } - const titleBoost = appearsInTitle ? 1.5 : 1.0; - - const finalScore = tf * idf * cf * lengthBoost * titleBoost; - scores.push({ ngram, score: finalScore }); - } - } - - scores.sort((a, b) => b.score - a.score); - return scores; - } - - /** - * Computes TF-IDF scores for ngrams in the cluster documents and returns the top K. - */ - public extractClusterTags(clusterDocuments: DocumentText[], topK = 5): string[] { - const scores = this.extractClusterNgramsWithScores(clusterDocuments); - return selectDedupedTags(scores, topK); - } -} - -/** - * Checks if a phrase contains consecutive identical words. - */ -export function hasConsecutiveDuplicates(phrase: string): boolean { - const words = phrase.toLowerCase().split(' '); - for (let i = 0; i < words.length - 1; i++) { - if (words[i] === words[i + 1]) return true; - } - return false; -} - -/** - * Filters out unigrams (single-word candidates) that are part of a stronger - * multi-word candidate (bigram/trigram) with a score >= 50% of the unigram's score. - */ -export function filterDemotedUnigrams(scores: { ngram: string; score: number }[]): { ngram: string; score: number }[] { - return scores.filter((candidate) => { - const wordCount = candidate.ngram.split(' ').length; - if (wordCount === 1) { - const hasStrongerPhrase = scores.some((other) => { - const otherWordCount = other.ngram.split(' ').length; - if (otherWordCount > 1) { - const constituentWords = new Set(other.ngram.toLowerCase().split(' ')); - if (constituentWords.has(candidate.ngram.toLowerCase()) && other.score >= candidate.score * 0.5) { - return true; - } - } - return false; - }); - if (hasStrongerPhrase) { - return false; - } - } - return true; - }); -} - -/** - * Selects up to `topK` tags from pre-computed ngram scores using deduplication rules: - * - Unigrams must be unique (no shared words with already-selected tags) - * - Bigrams/trigrams can share at most 1 word with already-selected tags - */ -export function selectDedupedTags(scores: { ngram: string; score: number }[], topK: number): string[] { - const filteredScores = filterDemotedUnigrams(scores); - const selectedTags: string[] = []; - const usedWords = new Set(); - - for (const candidate of filteredScores) { - if (selectedTags.length >= topK) break; - - const constituentWords = candidate.ngram.split(' '); - const limit = constituentWords.length === 1 ? 0 : 1; - let sharedCount = 0; - for (const w of constituentWords) { - if (usedWords.has(w)) { - sharedCount++; - } - } - - if (sharedCount <= limit) { - selectedTags.push(candidate.ngram); - for (const w of constituentWords) { - usedWords.add(w); - } - } - } - - return selectedTags; -} - -const ACRONYMS = new Set(['sip', 'api', 'ui', 'url', 'html', 'css', 'js', 'db', 'sql', 'onnx']); - -/** - * Capitalizes a phrase to Title Case, preserving common acronyms in uppercase. - */ -export function toTitleCase(phrase: string): string { - return phrase - .split(' ') - .map((word) => { - const lower = word.toLowerCase(); - if (ACRONYMS.has(lower)) { - return word.toUpperCase(); - } - return word.charAt(0).toUpperCase() + word.slice(1); - }) - .join(' '); -} - -/** - * Checks if two phrases share any words (case-insensitive). - */ -export function shareWords(phraseA: string, phraseB: string): boolean { - const wordsA = new Set(phraseA.toLowerCase().split(' ')); - const wordsB = phraseB.toLowerCase().split(' '); - return wordsB.some((w) => wordsA.has(w)); -} - -const TAXONOMY_MAPPING: { keywords: string[]; category: string }[] = [ - { - keywords: ['travel', 'flight', 'trip', 'train', 'vacation', 'backpacking', 'itinerary', 'packing', 'flights'], - category: 'Travel', - }, - { - keywords: [ - 'fund', - 'stock', - 'invest', - 'portfolio', - 'finance', - 'saving', - 'tax', - 'sip', - 'lump', - 'stocks', - 'funds', - 'investment', - 'investments', - ], - category: 'Investment', - }, - { - keywords: ['prep', 'smoothie', 'protein', 'macro', 'macros', 'diet', 'nutrition', 'meal'], - category: 'Meal Prep', - }, - { - keywords: [ - 'recipe', - 'recipes', - 'starter', - 'sourdough', - 'flour', - 'baking', - 'bread', - 'banana', - 'pasta', - 'skillet', - 'cook', - 'cooking', - 'kitchen', - ], - category: 'Recipes', - }, - { - keywords: [ - 'workout', - 'overload', - 'stretch', - 'stretching', - 'routine', - 'pain', - 'fitness', - 'exercise', - 'gym', - 'cardio', - 'back', - 'sitting', - ], - category: 'Workout', - }, - { - keywords: [ - 'code', - 'program', - 'javascript', - 'typescript', - 'node', - 'git', - 'docker', - 'graphql', - 'rest', - 'api', - 'jest', - 'test', - 'error', - 'request', - 'programming', - 'software', - 'developer', - ], - category: 'Programming', - }, - { - keywords: [ - 'psychology', - 'money', - 'meaning', - 'philosophy', - 'ravikant', - 'almanack', - 'book', - 'quotes', - 'thoughts', - 'reading', - 'naval', - ], - category: 'Books & Philosophy', - }, -]; - -/** - * Checks the top 3 ngrams of a cluster against a static taxonomy to match common topics. - */ -export function getTaxonomyCategory(scores: { ngram: string; score: number }[]): string | null { - const candidates = scores.slice(0, 3).map((s) => s.ngram.toLowerCase()); - - for (const cand of candidates) { - const words = cand.split(' '); - for (const mapping of TAXONOMY_MAPPING) { - for (const keyword of mapping.keywords) { - if (words.includes(keyword) || cand === keyword) { - return mapping.category; - } - } - } - } - return null; -} - -/** - * Generates a descriptive name for a cluster using the scoring list and clusterId. - */ -export function generateClusterName(scores: { ngram: string; score: number }[], clusterId: number): string { - const filteredScores = filterDemotedUnigrams(scores); - - if (filteredScores.length === 0 || filteredScores[0].score <= 0) { - return clusterId % 2 === 0 ? 'General' : 'Miscellaneous'; - } - - // Try matching against high-level taxonomy first - const taxonomyCategory = getTaxonomyCategory(filteredScores); - if (taxonomyCategory) { - return taxonomyCategory; - } - - const top1 = filteredScores[0]; - let top2: { ngram: string; score: number } | undefined; - - // Find the next highest-scoring phrase that doesn't share any words with the first phrase - for (let i = 1; i < filteredScores.length; i++) { - if (filteredScores[i].score <= 0) break; - if (!shareWords(top1.ngram, filteredScores[i].ngram)) { - top2 = filteredScores[i]; - break; - } - } - - // Join them if the second has at least 60% of the score of the first - if (top2 && top2.score >= top1.score * 0.6) { - return `${toTitleCase(top1.ngram)} & ${toTitleCase(top2.ngram)}`; - } - - return toTitleCase(top1.ngram); -} +import { DocumentText, TfidfExtractor } from './tfidf'; +import { selectDedupedTags, filterDemotedUnigrams } from './tagExtraction'; +import { generateClusterName, toTitleCase } from './clusterNaming'; + +// Re-export data constants +export { STOP_WORDS } from './data/stopWords'; +export { TAXONOMY_MAPPING } from './data/taxonomy'; + +// Re-export TF-IDF functions & class +export { + DocumentText, + TfidfExtractor, + cleanText, + singularize, + tokenize, + getNgrams, + hasConsecutiveDuplicates, +} from './tfidf'; + +// Re-export tag extraction functions +export { filterDemotedUnigrams, selectDedupedTags } from './tagExtraction'; + +// Re-export cluster naming functions +export { toTitleCase, shareWords, getTaxonomyCategory, generateClusterName } from './clusterNaming'; /** * Enriches benchmark results with extracted TF-IDF tags and cluster names for each cluster. diff --git a/src/pipeline/clustering/tagExtraction.ts b/src/pipeline/clustering/tagExtraction.ts new file mode 100644 index 0000000..46c6871 --- /dev/null +++ b/src/pipeline/clustering/tagExtraction.ts @@ -0,0 +1,58 @@ +/** + * Filters out unigrams (single-word candidates) that are part of a stronger + * multi-word candidate (bigram/trigram) with a score >= 50% of the unigram's score. + */ +export function filterDemotedUnigrams(scores: { ngram: string; score: number }[]): { ngram: string; score: number }[] { + return scores.filter((candidate) => { + const wordCount = candidate.ngram.split(' ').length; + if (wordCount === 1) { + const hasStrongerPhrase = scores.some((other) => { + const otherWordCount = other.ngram.split(' ').length; + if (otherWordCount > 1) { + const constituentWords = new Set(other.ngram.toLowerCase().split(' ')); + if (constituentWords.has(candidate.ngram.toLowerCase()) && other.score >= candidate.score * 0.5) { + return true; + } + } + return false; + }); + if (hasStrongerPhrase) { + return false; + } + } + return true; + }); +} + +/** + * Selects up to `topK` tags from pre-computed ngram scores using deduplication rules: + * - Unigrams must be unique (no shared words with already-selected tags) + * - Bigrams/trigrams can share at most 1 word with already-selected tags + */ +export function selectDedupedTags(scores: { ngram: string; score: number }[], topK: number): string[] { + const filteredScores = filterDemotedUnigrams(scores); + const selectedTags: string[] = []; + const usedWords = new Set(); + + for (const candidate of filteredScores) { + if (selectedTags.length >= topK) break; + + const constituentWords = candidate.ngram.split(' '); + const limit = constituentWords.length === 1 ? 0 : 1; + let sharedCount = 0; + for (const w of constituentWords) { + if (usedWords.has(w)) { + sharedCount++; + } + } + + if (sharedCount <= limit) { + selectedTags.push(candidate.ngram); + for (const w of constituentWords) { + usedWords.add(w); + } + } + } + + return selectedTags; +} diff --git a/src/pipeline/clustering/tfidf.ts b/src/pipeline/clustering/tfidf.ts new file mode 100644 index 0000000..ec67152 --- /dev/null +++ b/src/pipeline/clustering/tfidf.ts @@ -0,0 +1,260 @@ +import { STOP_WORDS } from './data/stopWords'; +import { selectDedupedTags } from './tagExtraction'; + +const SINGULAR_EXCEPTIONS = new Set(['series', 'species', 'means', 'news', 'analysis', 'basis', 'crisis']); +const SHORT_UNIGRAM_THRESHOLD = 4; + +export interface DocumentText { + title: string; + body: string; +} + +/** + * Strips code blocks, inline code, HTML tags, markdown links/images, and URLs + * from text to avoid polluting tag extraction. + */ +export function cleanText(text: string): string { + if (!text) return ''; + let cleaned = text; + // Strip triple-backtick markdown code blocks + cleaned = cleaned.replace(/```[\s\S]*?```/g, ' '); + // Strip inline code backticks + cleaned = cleaned.replace(/`[^`]*`/g, ' '); + // Strip markdown images ![alt](url) and links [text](url) — keep the text, remove syntax + cleaned = cleaned.replace(/!?\[([^\]]*)\]\([^)]*\)/g, '$1'); + // Strip HTML tags + cleaned = cleaned.replace(/<[^>]*>/g, ' '); + // Strip URLs + cleaned = cleaned.replace(/https?:\/\/\S+/gi, ' '); + return cleaned; +} + +/** + * A lightweight, dependency-free helper to stem basic English plural words to their singular form. + * Handles common cases like -ies -> -y, -es -> - (e.g. boxes -> box), and trailing -s (notes -> note). + */ +export function singularize(word: string): string { + if (word.length <= 3) return word; + if (SINGULAR_EXCEPTIONS.has(word)) return word; + if (word.endsWith('ss')) return word; // e.g. class, process + if (word.endsWith('ies')) return word.slice(0, -3) + 'y'; // e.g. categories -> category + if (word.endsWith('es')) { + const base = word.slice(0, -2); + if ( + base.endsWith('ss') || + base.endsWith('ch') || + base.endsWith('sh') || + base.endsWith('x') || + base.endsWith('z') + ) { + return base; // e.g. classes -> class, boxes -> box + } + return word.slice(0, -1); // e.g. databases -> database, lines -> line + } + if (word.endsWith('s') && !word.endsWith('us') && !word.endsWith('is') && !word.endsWith('as')) { + return word.slice(0, -1); // e.g. notes -> note, tasks -> task + } + return word; +} + +/** + * Lowercases text, cleans it, singularizes it, and tokenizes into alphabetic words + * of length >= 3 that are not in the stop words list. + */ +export function tokenize(text: string): string[] { + const cleaned = cleanText(text).toLowerCase().replace(/[’']/g, ''); + const matches = cleaned.match(/[a-z]+/g) || []; + return matches.map(singularize).filter((w) => w.length >= 3 && !STOP_WORDS.has(w)); +} + +/** + * Generates unigrams, bigrams, and trigrams from a sequence of tokens. + */ +export function getNgrams(tokens: string[]): string[] { + const ngrams: string[] = []; + const N = tokens.length; + for (let i = 0; i < N; i++) { + // Unigram + ngrams.push(tokens[i]); + // Bigram + if (i < N - 1) { + ngrams.push(`${tokens[i]} ${tokens[i + 1]}`); + } + // Trigram + if (i < N - 2) { + ngrams.push(`${tokens[i]} ${tokens[i + 1]} ${tokens[i + 2]}`); + } + } + return ngrams; +} + +/** + * Checks if a phrase contains consecutive identical words. + */ +export function hasConsecutiveDuplicates(phrase: string): boolean { + const words = phrase.toLowerCase().split(' '); + for (let i = 0; i < words.length - 1; i++) { + if (words[i] === words[i + 1]) return true; + } + return false; +} + +/** + * Extracts descriptive tags from documents in a cluster using TF-IDF. + */ +export class TfidfExtractor { + private idfs: { [word: string]: number } = {}; + + constructor(allDocuments: DocumentText[]) { + const N = allDocuments.length; + if (N === 0) return; + + const docFreqs: { [word: string]: number } = {}; + + for (const doc of allDocuments) { + // For IDF, we only need unique words/ngrams per document — no title weighting needed + const uniqueWords = this.getUniqueDocumentWords(doc); + for (const word of uniqueWords) { + docFreqs[word] = (docFreqs[word] || 0) + 1; + } + } + + for (const word of Object.keys(docFreqs)) { + const df = docFreqs[word]; + // Max DF rule: If a word/ngram appears in > 60% of all notes, it is too generic, ignore it. + if (df / N > 0.6) { + this.idfs[word] = 0; + } else { + this.idfs[word] = Math.log(N / df) + 1; + } + } + } + + /** + * Splits the text by sentence/line boundaries and generates ngrams within segments. + * This prevents forming cross-boundary ngrams (like joining separate lines or sentences). + */ + private getSegmentNgrams(text: string): string[] { + if (!text) return []; + // Split by sentence punctuation, newlines, markdown headers, and list bullets + const segments = text.split(/[.,?!;:\n\r\-*#()[\]]+/); + const allNgrams: string[] = []; + for (const seg of segments) { + const tokens = tokenize(seg); + const ngrams = getNgrams(tokens); + for (const ng of ngrams) { + // Filter out any ngrams with consecutive duplicate words (e.g. "day day") + if (!hasConsecutiveDuplicates(ng)) { + allNgrams.push(ng); + } + } + } + return allNgrams; + } + + /** + * Returns the unique set of words/ngrams in a document (title + body), used for IDF counting. + * No title weighting — each document contributes at most 1 to each ngram's document frequency. + */ + private getUniqueDocumentWords(doc: DocumentText): Set { + const titleNgrams = this.getSegmentNgrams(doc.title || ''); + const bodyNgrams = this.getSegmentNgrams(doc.body || ''); + return new Set([...titleNgrams, ...bodyNgrams]); + } + + /** + * Returns ngrams for TF scoring with title words weighted 5x higher. + * Uses push loops instead of spread to avoid excess intermediate array allocations. + */ + private getWeightedWords(doc: DocumentText): string[] { + const titleNgrams = this.getSegmentNgrams(doc.title || ''); + const bodyNgrams = this.getSegmentNgrams(doc.body || ''); + const result: string[] = []; + // Title ngrams appear 5 times to boost their term frequency + for (let i = 0; i < 5; i++) { + for (const ng of titleNgrams) { + result.push(ng); + } + } + for (const ng of bodyNgrams) { + result.push(ng); + } + return result; + } + + /** + * Computes sorted TF-IDF scores for ngrams in the cluster documents. + * Incorporates Cluster Frequency (CF) weighting, Length Boosting, and Title Match Boosting. + */ + public extractClusterNgramsWithScores(clusterDocuments: DocumentText[]): { ngram: string; score: number }[] { + if (clusterDocuments.length === 0) return []; + + const tfs: { [ngram: string]: number } = {}; + let totalNgrams = 0; + + for (const doc of clusterDocuments) { + const weighted = this.getWeightedWords(doc); + for (const ng of weighted) { + tfs[ng] = (tfs[ng] || 0) + 1; + totalNgrams++; + } + } + + if (totalNgrams === 0) return []; + + // Count how many documents in the cluster contain each ngram + const docCounts: { [ngram: string]: number } = {}; + for (const doc of clusterDocuments) { + const titleNgrams = this.getSegmentNgrams(doc.title || ''); + const bodyNgrams = this.getSegmentNgrams(doc.body || ''); + const docNgrams = new Set([...titleNgrams, ...bodyNgrams]); + for (const ng of docNgrams) { + docCounts[ng] = (docCounts[ng] || 0) + 1; + } + } + + const scores: { ngram: string; score: number }[] = []; + + for (const ngram of Object.keys(tfs)) { + const idf = this.idfs[ngram] || 0; // default to 0 if word is ignored/generic + if (idf > 0) { + const tf = tfs[ngram] / totalNgrams; + const cf = (docCounts[ngram] || 0) / clusterDocuments.length; + + // Length boost: 1.0x for unigram, 1.5x for bigram, 2.0x for trigram + const wordCount = ngram.split(' ').length; + let lengthBoost = 1.0 + (wordCount - 1) * 0.5; + + // Penalize very short unigrams (length <= 4) to favor longer descriptive phrases + if (wordCount === 1 && ngram.length <= SHORT_UNIGRAM_THRESHOLD) { + lengthBoost *= 0.5; + } + + // Title match boost: 1.5x if it appears in any note title in this cluster + let appearsInTitle = false; + for (const doc of clusterDocuments) { + const titleNgrams = new Set(this.getSegmentNgrams(doc.title || '')); + if (titleNgrams.has(ngram)) { + appearsInTitle = true; + break; + } + } + const titleBoost = appearsInTitle ? 1.5 : 1.0; + + const finalScore = tf * idf * cf * lengthBoost * titleBoost; + scores.push({ ngram, score: finalScore }); + } + } + + scores.sort((a, b) => b.score - a.score); + return scores; + } + + /** + * Computes TF-IDF scores for ngrams in the cluster documents and returns the top K. + */ + public extractClusterTags(clusterDocuments: DocumentText[], topK = 5): string[] { + const scores = this.extractClusterNgramsWithScores(clusterDocuments); + return selectDedupedTags(scores, topK); + } +} diff --git a/src/pipeline/runPipeline.ts b/src/pipeline/runPipeline.ts index 3d4860c..5fac200 100644 --- a/src/pipeline/runPipeline.ts +++ b/src/pipeline/runPipeline.ts @@ -1,24 +1,13 @@ import { fetchAllNotes } from './noteReader'; import { benchmark } from './clustering/benchmark'; -import { averageVectors, blendVectors, computeTitleWeight, cosineSimilarity } from './vectorAggregator'; -import { NoteVector, WorkerMessage } from '../types/embed'; +import { averageVectors } from './vectorAggregator'; import { PanelNote } from '../types/panel'; -import { isGenericTitle } from '../utils/titleFilter'; import { log, logErr } from '../utils/logger'; -import { getEncoding } from 'js-tiktoken'; import { VectorCache } from './vectorCache'; import { isNativeAiReady, fetchNativeEmbeddings } from './nativeEmbeddingPipeline'; import { DEFAULT_CONFIG, isValidEmbeddingVector } from './pipelineConfig'; import { enrichResultsWithTags } from './clustering/postProcess'; - -// We use cl100k_base to approximate token counts for chunking. -// The embedding model (all-MiniLM-L6-v2) uses a WordPiece tokenizer with a -// 512-token limit. WordPiece has a smaller vocabulary (~30k vs ~100k) so it produces -// ~1.3-1.5x more tokens than cl100k_base for the same text. A limit of 200 -// cl100k_base tokens expands to ~300 WordPiece tokens in the worst case, -// well within the model's 512-token ceiling. -const enc = getEncoding('cl100k_base'); -const MAX_TOKENS = 200; +import { EmbeddingWorkerOrchestrator } from './EmbeddingWorkerOrchestrator'; export interface PipelineCallbacks { onStatus: (text: string) => void; @@ -129,243 +118,58 @@ export const runPipeline = async (installDir: string, callbacks: PipelineCallbac await cache.beginUpdate(); - callbacks.onStatus('Loading model...'); - const worker = new Worker(`${installDir}/worker/embedWorker.js`); - - let currentNoteIndex = 0; - let currentChunkIndex = 0; - let currentNoteChunks: string[] = []; - let currentChunkEmbeddings: number[][] = []; - let currentBodyVector: number[] = []; - let isEmbeddingTitle = false; - let totalInferenceTime = 0; - let skippedCount = 0; - let cachedCount = 0; - let currentNoteHash = ''; const batchStartTime = performance.now(); - const noteVectors: NoteVector[] = []; - - const reportProgress = () => { - // current = notes finalized so far (embedded + cached + skipped) - const processed = noteVectors.length + skippedCount; - callbacks.onProgress(processed, notes.length, cachedCount, skippedCount); - }; - - const prepareNoteChunks = (text: string): string[] => { - const tokens = enc.encode(text); - const chunks: string[] = []; - if (tokens.length === 0) return []; - - for (let i = 0; i < tokens.length; i += MAX_TOKENS) { - const chunkTokens = tokens.slice(i, i + MAX_TOKENS); - chunks.push(enc.decode(chunkTokens)); - } - return chunks; - }; - - const finalizeNote = async (vector: number[], titleWeight: number, hash: string) => { - const note = notes[currentNoteIndex]; - noteVectors.push({ noteId: note.id, title: note.title, vector, titleWeight }); - - await cache.upsertItem(note.id, vector, { - title: note.title, - hash, - updatedTime: note.updated_time, - titleWeight, - }); - - reportProgress(); - - currentNoteIndex++; - await processNextNote(); - }; - - const processNextNote = async () => { - currentChunkIndex = 0; - currentNoteChunks = []; - currentChunkEmbeddings = []; - currentBodyVector = []; - isEmbeddingTitle = false; - - // Skip notes with empty body and generic title, and bypass cached notes - while (currentNoteIndex < notes.length) { - const note = notes[currentNoteIndex]; - - if (note.body.length === 0 && isGenericTitle(note.title)) { - log( - `[${currentNoteIndex + 1}/${notes.length}] skipped "${note.title.slice(0, 30)}" (empty body, generic title)`, - ); - skippedCount++; - currentNoteIndex++; - reportProgress(); - continue; - } - - currentNoteHash = cache.computeHash(note.title, note.body); - const cachedItem = await cache.getItem(note.id); - - if (cachedItem && cachedItem.metadata.hash === currentNoteHash) { - if (isValidEmbeddingVector(cachedItem.vector)) { - log(`[${currentNoteIndex + 1}/${notes.length}] cache hit for "${note.title.slice(0, 30)}"`); - noteVectors.push({ - noteId: note.id, - title: note.title, - vector: cachedItem.vector, - titleWeight: cachedItem.metadata.titleWeight ?? 0, - }); - cachedCount++; - currentNoteIndex++; - reportProgress(); - continue; - } else { - log( - `[${currentNoteIndex + 1}/${notes.length}] cache invalid (contains null/NaN) for "${note.title.slice(0, 30)}"`, - ); - } - } - - break; - } - - if (currentNoteIndex >= notes.length) { - const totalTime = performance.now() - batchStartTime; - log( - `Batch complete: ${notes.length} notes, ${noteVectors.length - cachedCount} embedded, ` + - `${cachedCount} cached, ${skippedCount} skipped in ${Math.round(totalTime)}ms ` + - `(inference: ${Math.round(totalInferenceTime)}ms)`, - ); - - await cache.endUpdate(); - worker.terminate(); + const orchestrator = new EmbeddingWorkerOrchestrator(installDir, notes, cache, callbacks); - callbacks.onStatus('Clustering...'); - - if (noteVectors.length < 3) { - callbacks.onError('Too few notes for clustering (need at least 3).'); - return; - } - - const vectors = noteVectors.map((nv) => nv.vector); - const results = benchmark(vectors, DEFAULT_CONFIG); + let result; + try { + result = await orchestrator.run(); + } catch (err) { + cache.cancelUpdate(); + const message = err instanceof Error ? err.message : String(err); + callbacks.onError('Pipeline failed: ' + message); + return; + } - // Post-process to extract tags/keywords for each cluster - const notesMap = new Map(notes.map((n) => [n.id, n])); - const allPipelineDocuments = noteVectors.map((nv) => { - const originalNote = notesMap.get(nv.noteId); - return { - title: nv.title, - body: originalNote ? originalNote.body : '', - }; - }); + const { noteVectors, cachedCount, skippedCount, totalInferenceTime } = result; - enrichResultsWithTags(results, allPipelineDocuments); + const totalTime = performance.now() - batchStartTime; + log( + `Batch complete: ${notes.length} notes, ${noteVectors.length - cachedCount} embedded, ` + + `${cachedCount} cached, ${skippedCount} skipped in ${Math.round(totalTime)}ms ` + + `(inference: ${Math.round(totalInferenceTime)}ms)`, + ); - const panelNotes: PanelNote[] = noteVectors.map((nv) => ({ - noteId: nv.noteId, - title: nv.title, - })); + await cache.endUpdate(); - callbacks.onComplete(results, panelNotes); - return; - } + callbacks.onStatus('Clustering...'); - const note = notes[currentNoteIndex]; - callbacks.onStatus(`Embedding "${note.title.slice(0, 40)}"...`); - - if (note.body.length === 0) { - isEmbeddingTitle = true; - worker.postMessage({ type: 'embed', text: note.title, noteId: note.id }); - } else { - currentNoteChunks = prepareNoteChunks(note.body); - if (currentNoteChunks.length === 0) { - // Whitespace-only body — treat as title-only note - isEmbeddingTitle = true; - worker.postMessage({ type: 'embed', text: note.title, noteId: note.id }); - } else { - worker.postMessage({ - type: 'embed', - text: currentNoteChunks[0], - noteId: note.id, - }); - } - } - }; + if (noteVectors.length < 3) { + callbacks.onError('Too few notes for clustering (need at least 3).'); + return; + } - worker.onerror = (err: ErrorEvent) => { - logErr('Worker error:', err.message); - cache.cancelUpdate(); - worker.terminate(); - callbacks.onError('Embedding worker failed: ' + err.message); - }; - - worker.onmessage = async (event: MessageEvent) => { - const data = event.data; - - if (data.type === 'load-result') { - if (data.success) { - log(`Model loaded in ${(data.loadTime / 1000).toFixed(1)}s, device: ${data.device}`); - callbacks.onStatus('Embedding notes...'); - await processNextNote(); - } else { - logErr('Model load failed:', data.error); - cache.cancelUpdate(); - worker.terminate(); - callbacks.onError('Failed to load embedding model: ' + (data.error || 'unknown error')); - } - return; - } + const vectors = noteVectors.map((nv) => nv.vector); + const results = benchmark(vectors, DEFAULT_CONFIG); - if (data.type === 'embed-result') { - const note = notes[currentNoteIndex]; + // Post-process to extract tags/keywords for each cluster + const notesMap = new Map(notes.map((n) => [n.id, n])); + const allPipelineDocuments = noteVectors.map((nv) => { + const originalNote = notesMap.get(nv.noteId); + return { + title: nv.title, + body: originalNote ? originalNote.body : '', + }; + }); - if (!data.success) { - logErr(`Failed to embed note "${note.title.slice(0, 30)}":`, data.error); - currentNoteIndex++; - await processNextNote(); - return; - } + enrichResultsWithTags(results, allPipelineDocuments); - totalInferenceTime += data.inferenceTime; - - if (isEmbeddingTitle) { - const titleEmbedding = data.embedding; - - if (currentBodyVector.length > 0) { - const sim = cosineSimilarity(currentBodyVector, titleEmbedding); - const alpha = computeTitleWeight(sim); - const finalVector = blendVectors(currentBodyVector, titleEmbedding, alpha); - await finalizeNote(finalVector, alpha, currentNoteHash); - } else { - await finalizeNote(titleEmbedding, 1.0, currentNoteHash); - } - } else { - currentChunkEmbeddings.push(data.embedding); - log( - `[${currentNoteIndex + 1}/${notes.length}] embedded chunk ${currentChunkIndex + 1}/${currentNoteChunks.length} of "${note.title.slice(0, 30)}"`, - ); - - currentChunkIndex++; - if (currentChunkIndex < currentNoteChunks.length) { - worker.postMessage({ - type: 'embed', - text: currentNoteChunks[currentChunkIndex], - noteId: note.id, - }); - } else { - currentBodyVector = averageVectors(currentChunkEmbeddings); - - if (!isGenericTitle(note.title)) { - isEmbeddingTitle = true; - worker.postMessage({ type: 'embed', text: note.title, noteId: note.id }); - } else { - await finalizeNote(currentBodyVector, 0, currentNoteHash); - } - } - } - } - }; + const panelNotes: PanelNote[] = noteVectors.map((nv) => ({ + noteId: nv.noteId, + title: nv.title, + })); - worker.postMessage({ type: 'load' }); + callbacks.onComplete(results, panelNotes); } catch (err) { const message = err instanceof Error ? err.message : String(err); logErr('Pipeline failed:', message); diff --git a/src/webview/context/AppStateContext.tsx b/src/webview/context/AppStateContext.tsx index 39fcc23..b70211c 100644 --- a/src/webview/context/AppStateContext.tsx +++ b/src/webview/context/AppStateContext.tsx @@ -1,5 +1,8 @@ import * as React from 'react'; import { PanelNote, BenchmarkResult, ProgressState, ApplyOptions } from '../../types/panel'; +import { useSettingsState } from './useSettingsState'; +import { useApplyState } from './useApplyState'; +import { usePipelineState } from './usePipelineState'; const POLL_INTERVAL_MS = 500; @@ -55,42 +58,6 @@ interface AppStateContextType { const AppStateContext = React.createContext(undefined); export const AppStateProvider: React.FC<{ children: React.ReactNode }> = ({ children }) => { - const [isRunning, setIsRunning] = React.useState(false); - const [statusText, setStatusText] = React.useState(''); - const [progress, setProgress] = React.useState({ - current: 0, - total: 0, - cached: 0, - skipped: 0, - }); - const [error, setError] = React.useState(null); - const [strategies, setStrategies] = React.useState([]); - const [notes, setNotes] = React.useState([]); - const [selectedStrategyIndex, setSelectedStrategyIndex] = React.useState(0); - const [activeView, setActiveView] = React.useState('idle'); - - const [isApplying, setIsApplying] = React.useState(false); - const [applyProgress, setApplyProgress] = React.useState({ current: 0, total: 0 }); - const [applyError, setApplyError] = React.useState(null); - const [applySuccess, setApplySuccess] = React.useState(false); - - const [isUndoing, setIsUndoing] = React.useState(false); - const [undoProgress, setUndoProgress] = React.useState({ current: 0, total: 0 }); - const [undoError, setUndoError] = React.useState(null); - const [undoSuccess, setUndoSuccess] = React.useState(false); - - const [isCleaningUp, setIsCleaningUp] = React.useState(false); - const [cleanupError, setCleanupError] = React.useState(null); - const [cleanupSuccess, setCleanupSuccess] = React.useState(null); - - const [settings, setSettings] = React.useState({ - metric: 'cosine', - parentNotebook: '', - changeLog: '', - }); - - const hasChangeLog = !!settings.changeLog; - const pollIntervalRef = React.useRef(null); const stopPolling = React.useCallback(() => { @@ -100,20 +67,64 @@ export const AppStateProvider: React.FC<{ children: React.ReactNode }> = ({ chil } }, []); - const fetchSettings = React.useCallback(async () => { - try { - const res = await webviewApi.postMessage({ type: 'getSettings' }); - if (res) { - setSettings({ - metric: (res as any)['categorization.metric'] || 'cosine', - parentNotebook: (res as any)['categorization.parentNotebook'] || '', - changeLog: (res as any)['categorization.changeLog'] || '', - }); - } - } catch (err) { - console.error('Failed to fetch settings:', err); - } - }, []); + // Initialize settings hook + const { settings, hasChangeLog, fetchSettings, updateSetting } = useSettingsState(); + + // Initialize apply state hook + const { + isApplying, + applyProgress, + applyError, + applySuccess, + isUndoing, + undoProgress, + undoError, + undoSuccess, + isCleaningUp, + cleanupError, + cleanupSuccess, + resetApplyState, + applyChanges, + undoChanges, + cleanUpNotebooks, + setIsApplying, + setApplyProgress, + setApplyError, + setApplySuccess, + setIsUndoing, + setUndoProgress, + setUndoError, + setUndoSuccess, + setIsCleaningUp, + setCleanupError, + setCleanupSuccess, + } = useApplyState(() => startPolling()); + + // Initialize pipeline state hook + const { + isRunning, + statusText, + progress, + error, + strategies, + notes, + selectedStrategyIndex, + activeView, + runPipeline, + changeStrategy, + setView, + updateClusterName, + moveNoteToCluster, + addCluster, + setIsRunning, + setStatusText, + setProgress, + setStrategies, + setNotes, + setSelectedStrategyIndex, + setError, + setActiveView, + } = usePipelineState(() => startPolling(), resetApplyState); const handlePollResponse = React.useCallback( (msg: any) => { @@ -227,7 +238,29 @@ export const AppStateProvider: React.FC<{ children: React.ReactNode }> = ({ chil break; } }, - [stopPolling, fetchSettings], + [ + stopPolling, + fetchSettings, + setStatusText, + setProgress, + setStrategies, + setNotes, + setSelectedStrategyIndex, + setError, + setActiveView, + setIsRunning, + setIsApplying, + setApplyProgress, + setApplyError, + setApplySuccess, + setIsUndoing, + setUndoProgress, + setUndoError, + setUndoSuccess, + setIsCleaningUp, + setCleanupError, + setCleanupSuccess, + ], ); const startPolling = React.useCallback(() => { @@ -250,194 +283,9 @@ export const AppStateProvider: React.FC<{ children: React.ReactNode }> = ({ chil }; }, [stopPolling]); - const runPipeline = async () => { - setIsRunning(true); - setStatusText('Starting pipeline...'); - setProgress({ current: 0, total: 0, cached: 0, skipped: 0 }); - setStrategies([]); - setNotes([]); - setError(null); - setActiveView('idle'); - setApplySuccess(false); - setApplyError(null); - setUndoSuccess(false); - setUndoError(null); - setCleanupSuccess(null); - setCleanupError(null); - try { - await webviewApi.postMessage({ type: 'run' }); - } catch (err) { - setError('Failed to start pipeline: ' + String(err)); - setIsRunning(false); - return; - } - startPolling(); - }; - - const changeStrategy = (index: number) => { - setSelectedStrategyIndex(index); - }; - - const setView = (view: ViewType) => { - setActiveView(view); - }; - - const updateClusterName = (clusterId: number, newName: string) => { - setStrategies((prev) => { - const next = [...prev]; - if (next[selectedStrategyIndex]) { - const strat = { ...next[selectedStrategyIndex] }; - const newClusterNames = { ...strat.clusterNames }; - newClusterNames[clusterId] = newName; - strat.clusterNames = newClusterNames; - next[selectedStrategyIndex] = strat; - } - return next; - }); - }; - - const moveNoteToCluster = (noteIndex: number, targetClusterId: number) => { - setStrategies((prev) => { - const next = [...prev]; - if (next[selectedStrategyIndex]) { - const strat = { ...next[selectedStrategyIndex] }; - const newAssignments = [...strat.assignments]; - - if (noteIndex < 0 || noteIndex >= newAssignments.length) return prev; - if (newAssignments[noteIndex] === targetClusterId) return prev; - - newAssignments[noteIndex] = targetClusterId; - strat.assignments = newAssignments; - next[selectedStrategyIndex] = strat; - } - return next; - }); - }; - - const addCluster = (name: string): boolean => { - const trimmedName = name.trim(); - if (!trimmedName) return false; - - const currentStrategy = strategies[selectedStrategyIndex]; - if (!currentStrategy) return false; - - const clusterNames = currentStrategy.clusterNames || {}; - const nameExists = Object.values(clusterNames).some((n) => n.toLowerCase() === trimmedName.toLowerCase()); - - if (nameExists) { - return false; - } - - setStrategies((prev) => { - const next = [...prev]; - const strat = next[selectedStrategyIndex]; - if (strat) { - const newStrat = { ...strat }; - const newClusterNames = { ...strat.clusterNames }; - const newTags = { ...strat.tags }; - - const clusterIds = Object.keys(newClusterNames).map(Number); - const newClusterId = clusterIds.length > 0 ? Math.max(...clusterIds) + 1 : 0; - - newClusterNames[newClusterId] = trimmedName; - newTags[newClusterId] = []; - - newStrat.clusterNames = newClusterNames; - newStrat.tags = newTags; - newStrat.clusterCount = (newStrat.clusterCount || 0) + 1; - - next[selectedStrategyIndex] = newStrat; - } - return next; - }); - - return true; - }; - - const updateSetting = async (key: string, value: any) => { - try { - await webviewApi.postMessage({ - type: 'updateSetting', - key, - value, - }); - const localKey = key.replace('categorization.', ''); - setSettings((prev) => ({ - ...prev, - [localKey]: value, - })); - } catch (err) { - console.error('Failed to update setting:', err); - } - }; - - const applyChanges = async (options: ApplyOptions) => { + const handleApplyChanges = async (options: ApplyOptions) => { const currentStrategy = strategies[selectedStrategyIndex]; - if (!currentStrategy) { - setApplyError('No active strategy selected.'); - return; - } - - setIsApplying(true); - setApplyProgress({ current: 0, total: notes.length }); - setApplyError(null); - setApplySuccess(false); - setUndoSuccess(false); - setUndoError(null); - setCleanupSuccess(null); - setCleanupError(null); - - try { - await webviewApi.postMessage({ - type: 'apply', - options, - notes, - assignments: currentStrategy.assignments, - clusterNames: currentStrategy.clusterNames || {}, - clusterTags: currentStrategy.tags || {}, - }); - startPolling(); - } catch (err) { - setApplyError('Failed to apply changes: ' + String(err)); - setIsApplying(false); - } - }; - - const undoChanges = async () => { - setIsUndoing(true); - setUndoProgress({ current: 0, total: 0 }); - setUndoError(null); - setUndoSuccess(false); - setApplySuccess(false); - setApplyError(null); - setCleanupSuccess(null); - setCleanupError(null); - - try { - await webviewApi.postMessage({ type: 'undo' }); - startPolling(); - } catch (err) { - setUndoError('Failed to start undo operation: ' + String(err)); - setIsUndoing(false); - } - }; - - const cleanUpNotebooks = async () => { - setIsCleaningUp(true); - setCleanupError(null); - setCleanupSuccess(null); - setApplySuccess(false); - setApplyError(null); - setUndoSuccess(false); - setUndoError(null); - - try { - await webviewApi.postMessage({ type: 'cleanUpEmptyNotebooks' }); - startPolling(); - } catch (err) { - setCleanupError('Failed to start cleanup: ' + String(err)); - setIsCleaningUp(false); - } + await applyChanges(options, notes, currentStrategy); }; return ( @@ -464,7 +312,7 @@ export const AppStateProvider: React.FC<{ children: React.ReactNode }> = ({ chil applyProgress, applyError, applySuccess, - applyChanges, + applyChanges: handleApplyChanges, isUndoing, undoProgress, undoError, diff --git a/src/webview/context/useApplyState.ts b/src/webview/context/useApplyState.ts new file mode 100644 index 0000000..e824c50 --- /dev/null +++ b/src/webview/context/useApplyState.ts @@ -0,0 +1,128 @@ +import * as React from 'react'; +import { ApplyOptions, PanelNote, BenchmarkResult } from '../../types/panel'; + +export function useApplyState(startPolling: () => void) { + const [isApplying, setIsApplying] = React.useState(false); + const [applyProgress, setApplyProgress] = React.useState({ current: 0, total: 0 }); + const [applyError, setApplyError] = React.useState(null); + const [applySuccess, setApplySuccess] = React.useState(false); + + const [isUndoing, setIsUndoing] = React.useState(false); + const [undoProgress, setUndoProgress] = React.useState({ current: 0, total: 0 }); + const [undoError, setUndoError] = React.useState(null); + const [undoSuccess, setUndoSuccess] = React.useState(false); + + const [isCleaningUp, setIsCleaningUp] = React.useState(false); + const [cleanupError, setCleanupError] = React.useState(null); + const [cleanupSuccess, setCleanupSuccess] = React.useState(null); + + const resetApplyState = React.useCallback(() => { + setApplySuccess(false); + setApplyError(null); + setUndoSuccess(false); + setUndoError(null); + setCleanupSuccess(null); + setCleanupError(null); + }, []); + + const applyChanges = async ( + options: ApplyOptions, + notes: PanelNote[], + currentStrategy: BenchmarkResult | undefined, + ) => { + if (!currentStrategy) { + setApplyError('No active strategy selected.'); + return; + } + + setIsApplying(true); + setApplyProgress({ current: 0, total: notes.length }); + setApplyError(null); + setApplySuccess(false); + setUndoSuccess(false); + setUndoError(null); + setCleanupSuccess(null); + setCleanupError(null); + + try { + await webviewApi.postMessage({ + type: 'apply', + options, + notes, + assignments: currentStrategy.assignments, + clusterNames: currentStrategy.clusterNames || {}, + clusterTags: currentStrategy.tags || {}, + }); + startPolling(); + } catch (err) { + setApplyError('Failed to apply changes: ' + String(err)); + setIsApplying(false); + } + }; + + const undoChanges = async () => { + setIsUndoing(true); + setUndoProgress({ current: 0, total: 0 }); + setUndoError(null); + setUndoSuccess(false); + setApplySuccess(false); + setApplyError(null); + setCleanupSuccess(null); + setCleanupError(null); + + try { + await webviewApi.postMessage({ type: 'undo' }); + startPolling(); + } catch (err) { + setUndoError('Failed to start undo operation: ' + String(err)); + setIsUndoing(false); + } + }; + + const cleanUpNotebooks = async () => { + setIsCleaningUp(true); + setCleanupError(null); + setCleanupSuccess(null); + setApplySuccess(false); + setApplyError(null); + setUndoSuccess(false); + setUndoError(null); + + try { + await webviewApi.postMessage({ type: 'cleanUpEmptyNotebooks' }); + startPolling(); + } catch (err) { + setCleanupError('Failed to start cleanup: ' + String(err)); + setIsCleaningUp(false); + } + }; + + return { + isApplying, + applyProgress, + applyError, + applySuccess, + isUndoing, + undoProgress, + undoError, + undoSuccess, + isCleaningUp, + cleanupError, + cleanupSuccess, + resetApplyState, + applyChanges, + undoChanges, + cleanUpNotebooks, + setIsApplying, + setApplyProgress, + setApplyError, + setApplySuccess, + setIsUndoing, + setUndoProgress, + setUndoError, + setUndoSuccess, + setIsCleaningUp, + setCleanupError, + setCleanupSuccess, + }; +} diff --git a/src/webview/context/usePipelineState.ts b/src/webview/context/usePipelineState.ts new file mode 100644 index 0000000..86616b8 --- /dev/null +++ b/src/webview/context/usePipelineState.ts @@ -0,0 +1,155 @@ +import * as React from 'react'; +import { PanelNote, BenchmarkResult, ProgressState } from '../../types/panel'; +import { ViewType } from './AppStateContext'; + +export function usePipelineState(startPolling: () => void, resetApplyState: () => void) { + const [isRunning, setIsRunning] = React.useState(false); + const [statusText, setStatusText] = React.useState(''); + const [progress, setProgress] = React.useState({ + current: 0, + total: 0, + cached: 0, + skipped: 0, + }); + const [error, setError] = React.useState(null); + const [strategies, setStrategies] = React.useState([]); + const [notes, setNotes] = React.useState([]); + const [selectedStrategyIndex, setSelectedStrategyIndex] = React.useState(0); + const [activeView, setActiveView] = React.useState('idle'); + + const runPipeline = async () => { + setIsRunning(true); + setStatusText('Starting pipeline...'); + setProgress({ current: 0, total: 0, cached: 0, skipped: 0 }); + setStrategies([]); + setNotes([]); + setError(null); + setActiveView('idle'); + + // Reset apply/undo/cleanup states + resetApplyState(); + + try { + await webviewApi.postMessage({ type: 'run' }); + } catch (err) { + setError('Failed to start pipeline: ' + String(err)); + setIsRunning(false); + return; + } + startPolling(); + }; + + const changeStrategy = React.useCallback((index: number) => { + setSelectedStrategyIndex(index); + }, []); + + const setView = React.useCallback((view: ViewType) => { + setActiveView(view); + }, []); + + const updateClusterName = React.useCallback( + (clusterId: number, newName: string) => { + setStrategies((prev) => { + const next = [...prev]; + if (next[selectedStrategyIndex]) { + const strat = { ...next[selectedStrategyIndex] }; + const newClusterNames = { ...strat.clusterNames }; + newClusterNames[clusterId] = newName; + strat.clusterNames = newClusterNames; + next[selectedStrategyIndex] = strat; + } + return next; + }); + }, + [selectedStrategyIndex], + ); + + const moveNoteToCluster = React.useCallback( + (noteIndex: number, targetClusterId: number) => { + setStrategies((prev) => { + const next = [...prev]; + if (next[selectedStrategyIndex]) { + const strat = { ...next[selectedStrategyIndex] }; + const newAssignments = [...strat.assignments]; + + if (noteIndex < 0 || noteIndex >= newAssignments.length) return prev; + if (newAssignments[noteIndex] === targetClusterId) return prev; + + newAssignments[noteIndex] = targetClusterId; + strat.assignments = newAssignments; + next[selectedStrategyIndex] = strat; + } + return next; + }); + }, + [selectedStrategyIndex], + ); + + const addCluster = React.useCallback( + (name: string): boolean => { + const trimmedName = name.trim(); + if (!trimmedName) return false; + + const currentStrategy = strategies[selectedStrategyIndex]; + if (!currentStrategy) return false; + + const clusterNames = currentStrategy.clusterNames || {}; + const nameExists = Object.values(clusterNames).some((n) => n.toLowerCase() === trimmedName.toLowerCase()); + + if (nameExists) { + return false; + } + + setStrategies((prev) => { + const next = [...prev]; + const strat = next[selectedStrategyIndex]; + if (strat) { + const newStrat = { ...strat }; + const newClusterNames = { ...strat.clusterNames }; + const newTags = { ...strat.tags }; + + const clusterIds = Object.keys(newClusterNames).map(Number); + const newClusterId = clusterIds.length > 0 ? Math.max(...clusterIds) + 1 : 0; + + newClusterNames[newClusterId] = trimmedName; + newTags[newClusterId] = []; + + newStrat.clusterNames = newClusterNames; + newStrat.tags = newTags; + newStrat.clusterCount = (newStrat.clusterCount || 0) + 1; + + next[selectedStrategyIndex] = newStrat; + } + return next; + }); + + return true; + }, + [strategies, selectedStrategyIndex], + ); + + return { + isRunning, + statusText, + progress, + error, + strategies, + notes, + selectedStrategyIndex, + activeView, + runPipeline, + changeStrategy, + setView, + updateClusterName, + moveNoteToCluster, + addCluster, + setIsRunning, + setStatusText, + setProgress, + setStrategies, + setNotes, + setSelectedStrategyIndex, + setError, + setActiveView, + }; +} diff --git a/src/webview/context/useSettingsState.ts b/src/webview/context/useSettingsState.ts new file mode 100644 index 0000000..b213377 --- /dev/null +++ b/src/webview/context/useSettingsState.ts @@ -0,0 +1,50 @@ +import * as React from 'react'; + +export function useSettingsState() { + const [settings, setSettings] = React.useState({ + metric: 'cosine', + parentNotebook: '', + changeLog: '', + }); + + const fetchSettings = React.useCallback(async () => { + try { + const res = await webviewApi.postMessage({ type: 'getSettings' }); + if (res) { + setSettings({ + metric: (res as any)['categorization.metric'] || 'cosine', + parentNotebook: (res as any)['categorization.parentNotebook'] || '', + changeLog: (res as any)['categorization.changeLog'] || '', + }); + } + } catch (err) { + console.error('Failed to fetch settings:', err); + } + }, []); + + const updateSetting = React.useCallback(async (key: string, value: any) => { + try { + await webviewApi.postMessage({ + type: 'updateSetting', + key, + value, + }); + const localKey = key.replace('categorization.', ''); + setSettings((prev) => ({ + ...prev, + [localKey]: value, + })); + } catch (err) { + console.error('Failed to update setting:', err); + } + }, []); + + const hasChangeLog = !!settings.changeLog; + + return { + settings, + hasChangeLog, + fetchSettings, + updateSetting, + }; +}