diff --git a/docs/COMMANDS.md b/docs/COMMANDS.md index d21a3264..3616a68f 100644 --- a/docs/COMMANDS.md +++ b/docs/COMMANDS.md @@ -91,10 +91,12 @@ The SSO credentials remain in CodeMie and are never added to VS Code. The connec For a new provider or an invalid existing key, the command prints this required one-time action: ```text -Press ⇧⌘P (macOS) or Ctrl+Shift+P (Windows/Linux) -Run Chat: Manage Language Models -Right-click CodeMie Profile Model → Update API Key -Enter API key: codemie-proxy +One-time VS Code secret setup required: +1. Open VS Code and Press ⇧⌘P (macOS) or Ctrl+Shift+P (Windows/Linux). +2. Find Chat: Manage Language Models +3. In opened dialog Right-click any CodeMie model → Update API Key +4. Enter API key: codemie-proxy +Reload VS Code, then select a CodeMie model from the model picker ``` VS Code stores that local key in its secret storage and adds the reference to the configuration. The CLI cannot inspect whether an existing secret reference still resolves; if VS Code reports a missing or invalid key, use **Update API Key** again. diff --git a/package.json b/package.json index 6a43458d..db9a4ebc 100644 --- a/package.json +++ b/package.json @@ -37,6 +37,7 @@ "test:unit": "vitest run --project unit", "test:integration": "vitest run --project cli", "test:integration:cli": "vitest run --project cli", + "test:integration:vscode-models": "vitest run --project cli tests/integration/vscode-byok.test.ts tests/integration/vscode-models.live.test.ts", "test:integration:agent": "vitest run --project agent", "test:run": "vitest run --project unit --project cli", "test:all": "vitest run --project unit && vitest run --project cli && vitest run --project agent", diff --git a/src/cli/commands/proxy/__tests__/index.test.ts b/src/cli/commands/proxy/__tests__/index.test.ts index 40bd8e66..88d8f679 100644 --- a/src/cli/commands/proxy/__tests__/index.test.ts +++ b/src/cli/commands/proxy/__tests__/index.test.ts @@ -323,7 +323,7 @@ describe('proxy connect vscode', () => { exitSpy.mockRestore(); }); - it('uses an explicit profile for daemon context and writes its model into VS Code', async () => { + it('uses an explicit profile without requiring its model for VS Code', async () => { const { ConfigLoader } = await import('../../../../utils/config.js'); const { checkStatus, spawnDaemon } = await import('../daemon-manager.js'); const { writeVsCodeLanguageModelsConfig } = await import('../connectors/vscode.js'); @@ -332,7 +332,6 @@ describe('proxy connect vscode', () => { name: 'custom', provider: 'ai-run-sso', baseUrl: 'https://custom.example.com/api', - model: 'custom-profile-model', codeMieProject: 'team-project', codeMieUrl: 'https://custom.example.com', } as Awaited>); @@ -365,12 +364,11 @@ describe('proxy connect vscode', () => { expect(daemonOptions).not.toHaveProperty('model'); expect(writeVsCodeLanguageModelsConfig).toHaveBeenCalledWith( 'http://127.0.0.1:4001', - 'custom-profile-model', false ); }); - it('reuses a matching daemon when only the profile model changes', async () => { + it('reuses a matching daemon independently of the profile model', async () => { const { ConfigLoader } = await import('../../../../utils/config.js'); const { checkStatus, spawnDaemon, stopDaemon } = await import('../daemon-manager.js'); const { checkProxyHealth } = await import('../health-check.js'); @@ -407,7 +405,6 @@ describe('proxy connect vscode', () => { expect(spawnDaemon).not.toHaveBeenCalled(); expect(writeVsCodeLanguageModelsConfig).toHaveBeenCalledWith( 'http://127.0.0.1:4001', - 'new-profile-model', false ); }); diff --git a/src/cli/commands/proxy/connectors/__tests__/vscode.test.ts b/src/cli/commands/proxy/connectors/__tests__/vscode.test.ts index 1c16a047..bda3528a 100644 --- a/src/cli/commands/proxy/connectors/__tests__/vscode.test.ts +++ b/src/cli/commands/proxy/connectors/__tests__/vscode.test.ts @@ -7,8 +7,44 @@ import { afterEach, beforeEach, describe, expect, it } from 'vitest'; import { mkdir, mkdtemp, readFile, rm, writeFile } from 'node:fs/promises'; import { tmpdir } from 'node:os'; import { join } from 'node:path'; +import { VS_CODE_SUPPORTED_MODELS, type VsCodeApiType } from '../vscode-models.js'; import { writeVsCodeLanguageModelsConfigAtPath } from '../vscode.js'; +const EXPECTED_MODEL_IDS = [ + 'claude-sonnet-4-5-20250929', + 'gpt-4.1', + 'gpt-4.1-mini', + 'gpt-5-2025-08-07', + 'gpt-5-mini-2025-08-07', + 'gpt-5-nano-2025-08-07', + 'gpt-5-1-codex-2025-11-13', + 'gpt-5-2-2025-12-11', + 'gpt-5.4-2026-03-05', + 'gpt-5.5-2026-04-24', + 'gpt-5.6-luna-2026-07-09', + 'gpt-5.6-terra-2026-07-09', + 'gemini-3-flash', + 'gemini-3.1-pro', + 'gemini-3.5-flash', + 'claude-4-5-sonnet', + 'claude-sonnet-4-6', + 'claude-sonnet-5', + 'claude-opus-4-5-20251101', + 'claude-opus-4-6-20260205', + 'claude-opus-4-7', + 'claude-opus-4-8', + 'claude-haiku-4-5-20251001', + 'qwen.qwen3-coder-30b-a3b-v1', + 'qwen.qwen3-coder-480b-a35b-v1', + 'moonshotai.kimi-k2.5', +] as const; + +function getApiPath(apiType: VsCodeApiType): string { + if (apiType === 'responses') return '/v1/responses'; + if (apiType === 'messages') return '/v1/messages'; + return '/v1/chat/completions'; +} + describe('writeVsCodeLanguageModelsConfigAtPath', () => { let testDir: string; let configPath: string; @@ -27,26 +63,107 @@ describe('writeVsCodeLanguageModelsConfigAtPath', () => { return JSON.parse(await readFile(configPath, 'utf-8')) as Array>; } - it('writes the selected profile model as the model ID', async () => { + it('writes the exact supported model allowlist under the CodeMie provider', async () => { const result = await writeVsCodeLanguageModelsConfigAtPath( configPath, - 'http://127.0.0.1:4001', - 'gpt-profile-model' + 'http://127.0.0.1:4001' ); const providers = await readProviders(); - const models = providers[0].models as Array>; + const provider = providers[0]; + const models = provider.models as Array>; + expect(result).toEqual({ configPath, requiresSecretConfiguration: true }); - expect(models).toEqual([ - expect.objectContaining({ - id: 'gpt-profile-model', - name: 'CodeMie Profile Model', - url: 'http://127.0.0.1:4001/v1/chat/completions', - }), - ]); + expect(provider).toMatchObject({ + name: 'CodeMie', + vendor: 'customendpoint', + apiType: 'chat-completions', + }); + expect(models.map(model => model.id)).toEqual(EXPECTED_MODEL_IDS); + expect(models.map(model => model.name)).toEqual(EXPECTED_MODEL_IDS); + }); + + it('renders every catalog capability and endpoint without internal metadata', async () => { + await writeVsCodeLanguageModelsConfigAtPath(configPath, 'http://127.0.0.1:4001'); + + const providers = await readProviders(); + const models = providers[0].models as Array>; + + for (const definition of VS_CODE_SUPPORTED_MODELS) { + const model = models.find(candidate => candidate.id === definition.id); + const expected: Record = { + id: definition.id, + name: definition.id, + url: `http://127.0.0.1:4001${getApiPath(definition.apiType)}`, + apiType: definition.apiType, + toolCalling: true, + vision: definition.vision, + streaming: true, + thinking: definition.thinking, + maxInputTokens: definition.maxInputTokens, + maxOutputTokens: definition.maxOutputTokens, + }; + if (definition.adaptiveThinking) expected.adaptiveThinking = true; + if (definition.modelOptions) expected.modelOptions = definition.modelOptions; + if (definition.requestHeaders) expected.requestHeaders = definition.requestHeaders; + if (definition.supportsReasoningEffort) { + expected.supportsReasoningEffort = definition.supportsReasoningEffort; + } + if (definition.reasoningEffortFormat) { + expected.reasoningEffortFormat = definition.reasoningEffortFormat; + } + + expect(model).toEqual(expected); + } + }); + + it('omits top_p for Claude 4.5 models that reject dual sampling parameters', async () => { + await writeVsCodeLanguageModelsConfigAtPath(configPath, 'http://127.0.0.1:4001'); + + const providers = await readProviders(); + const models = providers[0].models as Array>; + const affectedIds = [ + 'claude-sonnet-4-5-20250929', + 'claude-4-5-sonnet', + 'claude-haiku-4-5-20251001', + ]; + + for (const id of affectedIds) { + expect(models.find(model => model.id === id)).toMatchObject({ + modelOptions: { top_p: null }, + }); + } + }); + + it('does not advertise the rejected none effort for GPT-5.1 Codex', async () => { + await writeVsCodeLanguageModelsConfigAtPath(configPath, 'http://127.0.0.1:4001'); + + const providers = await readProviders(); + const models = providers[0].models as Array>; + const codex = models.find(model => model.id === 'gpt-5-1-codex-2025-11-13'); + + expect(codex?.supportsReasoningEffort).toEqual(['low', 'medium', 'high']); + }); + + it('forces bearer authentication for Messages models only', async () => { + await writeVsCodeLanguageModelsConfigAtPath(configPath, 'http://127.0.0.1:4001'); + + const providers = await readProviders(); + const models = providers[0].models as Array>; + + for (const definition of VS_CODE_SUPPORTED_MODELS) { + const model = models.find(candidate => candidate.id === definition.id); + if (definition.apiType === 'messages') { + expect(model?.requestHeaders).toEqual({ + Authorization: 'Bearer ${apiKey}', + }); + } else { + expect(model).not.toHaveProperty('requestHeaders'); + } + } }); - it('updates the managed model and preserves unrelated configuration', async () => { + it('overrides the CodeMie model catalog while preserving the secret and saved settings', async () => { const secretReference = '${input:chat.lm.secret.codemie}'; await writeFile(configPath, JSON.stringify([ { @@ -57,15 +174,15 @@ describe('writeVsCodeLanguageModelsConfigAtPath', () => { { name: 'CodeMie', vendor: 'customendpoint', - apiType: 'chat-completions', + apiType: 'messages', apiKey: secretReference, customProperty: 'preserved', settings: { - 'old-profile-model': { reasoningEffort: 'medium' }, + 'gpt-5.5-2026-04-24': { reasoningEffort: 'high' }, 'custom-setting': { enabled: true }, }, models: [ - { id: 'old-profile-model', name: 'CodeMie Profile Model', stale: true }, + { id: 'stale-catalog-model', name: 'Stale catalog model', stale: true }, { id: 'user-managed-model', name: 'User model', custom: true }, ], }, @@ -73,50 +190,50 @@ describe('writeVsCodeLanguageModelsConfigAtPath', () => { const result = await writeVsCodeLanguageModelsConfigAtPath( configPath, - 'http://127.0.0.1:4010', - 'claude-profile-model' + 'http://127.0.0.1:4010' ); const providers = await readProviders(); - const codemie = providers[1]; + const codeMie = providers[1]; + const models = codeMie.models as Array>; + expect(result.requiresSecretConfiguration).toBe(false); - expect(providers[0]).toMatchObject({ name: 'Other' }); - expect(codemie).toMatchObject({ + expect(providers[0]).toEqual({ + name: 'Other', + vendor: 'customendpoint', + models: [{ id: 'other-model', name: 'Other model' }], + }); + expect(codeMie).toMatchObject({ name: 'CodeMie', + vendor: 'customendpoint', + apiType: 'chat-completions', apiKey: secretReference, customProperty: 'preserved', - settings: { 'custom-setting': { enabled: true } }, + settings: { + 'gpt-5.5-2026-04-24': { reasoningEffort: 'high' }, + 'custom-setting': { enabled: true }, + }, }); - expect(codemie.models).toEqual([ - expect.objectContaining({ id: 'claude-profile-model', name: 'CodeMie Profile Model' }), - { id: 'user-managed-model', name: 'User model', custom: true }, - ]); + expect(models.map(model => model.id)).toEqual(EXPECTED_MODEL_IDS); + expect(models.some(model => model.id === 'user-managed-model')).toBe(false); }); - it('replaces the previous profile model when the selected profile changes', async () => { - await writeVsCodeLanguageModelsConfigAtPath( - configPath, - 'http://127.0.0.1:4001', - 'first-profile-model' - ); + it('updates every managed endpoint without changing saved model settings', async () => { + await writeVsCodeLanguageModelsConfigAtPath(configPath, 'http://127.0.0.1:4001'); const firstProviders = await readProviders(); firstProviders[0].settings = { - 'first-profile-model': { reasoningEffort: 'high' }, - unrelated: { enabled: true }, + 'claude-opus-4-8': { reasoningEffort: 'xhigh' }, }; await writeFile(configPath, JSON.stringify(firstProviders, null, 2), 'utf-8'); - await writeVsCodeLanguageModelsConfigAtPath( - configPath, - 'http://127.0.0.1:4001', - 'second-profile-model' - ); + await writeVsCodeLanguageModelsConfigAtPath(configPath, 'http://127.0.0.1:4010'); const providers = await readProviders(); const models = providers[0].models as Array>; - expect(models).toHaveLength(1); - expect(models[0].id).toBe('second-profile-model'); - expect(providers[0].settings).toEqual({ unrelated: { enabled: true } }); + expect(models.every(model => String(model.url).startsWith('http://127.0.0.1:4010/'))).toBe(true); + expect(providers[0].settings).toEqual({ + 'claude-opus-4-8': { reasoningEffort: 'xhigh' }, + }); }); it.each([ @@ -127,18 +244,9 @@ describe('writeVsCodeLanguageModelsConfigAtPath', () => { await expect(writeVsCodeLanguageModelsConfigAtPath( configPath, - 'http://127.0.0.1:4001', - 'profile-model' + 'http://127.0.0.1:4001' )).rejects.toThrow(); expect(await readFile(configPath, 'utf-8')).toBe(original); }); - - it('rejects an empty profile model before writing configuration', async () => { - await expect(writeVsCodeLanguageModelsConfigAtPath( - configPath, - 'http://127.0.0.1:4001', - ' ' - )).rejects.toThrow('requires a profile model'); - }); }); diff --git a/src/cli/commands/proxy/connectors/vscode-models.ts b/src/cli/commands/proxy/connectors/vscode-models.ts new file mode 100644 index 00000000..9ce2d907 --- /dev/null +++ b/src/cli/commands/proxy/connectors/vscode-models.ts @@ -0,0 +1,299 @@ +export type VsCodeApiType = 'chat-completions' | 'responses' | 'messages'; + +export type VsCodeReasoningEffort = + | 'none' + | 'minimal' + | 'low' + | 'medium' + | 'high' + | 'xhigh' + | 'max'; + +export interface VsCodeModelDefinition { + id: string; + apiType: VsCodeApiType; + vision: boolean; + thinking: boolean; + adaptiveThinking?: true; + modelOptions?: Readonly<{ + temperature?: number | null; + top_p?: number | null; + }>; + requestHeaders?: Readonly>; + supportsReasoningEffort?: readonly VsCodeReasoningEffort[]; + reasoningEffortFormat?: 'chat-completions' | 'responses'; + maxInputTokens: number; + maxOutputTokens: number; +} + +const GPT_5_EFFORTS = ['minimal', 'low', 'medium', 'high'] as const; +const GPT_5_1_EFFORTS = ['low', 'medium', 'high'] as const; +const GPT_5_2_EFFORTS = ['none', 'low', 'medium', 'high'] as const; +const GPT_5_XHIGH_EFFORTS = ['none', 'low', 'medium', 'high', 'xhigh'] as const; +const GPT_5_6_EFFORTS = ['none', 'low', 'medium', 'high', 'xhigh', 'max'] as const; +const GEMINI_FLASH_EFFORTS = ['minimal', 'low', 'medium', 'high'] as const; +const GEMINI_PRO_EFFORTS = ['low', 'medium', 'high'] as const; +const CLAUDE_EFFORTS = ['low', 'medium', 'high'] as const; +const CLAUDE_MAX_EFFORTS = ['low', 'medium', 'high', 'max'] as const; +const CLAUDE_XHIGH_EFFORTS = ['low', 'medium', 'high', 'xhigh', 'max'] as const; +const MESSAGE_AUTH_HEADERS = { + Authorization: 'Bearer ${apiKey}', +} as const; + +export const VS_CODE_SUPPORTED_MODELS: readonly VsCodeModelDefinition[] = [ + { + id: 'claude-sonnet-4-5-20250929', + apiType: 'chat-completions', + vision: true, + thinking: false, + modelOptions: { top_p: null }, + maxInputTokens: 136000, + maxOutputTokens: 64000, + }, + { + id: 'gpt-4.1', + apiType: 'chat-completions', + vision: true, + thinking: false, + maxInputTokens: 1014808, + maxOutputTokens: 32768, + }, + { + id: 'gpt-4.1-mini', + apiType: 'chat-completions', + vision: true, + thinking: false, + maxInputTokens: 1014808, + maxOutputTokens: 32768, + }, + { + id: 'gpt-5-2025-08-07', + apiType: 'chat-completions', + vision: true, + thinking: true, + supportsReasoningEffort: GPT_5_EFFORTS, + reasoningEffortFormat: 'chat-completions', + maxInputTokens: 272000, + maxOutputTokens: 128000, + }, + { + id: 'gpt-5-mini-2025-08-07', + apiType: 'chat-completions', + vision: true, + thinking: true, + supportsReasoningEffort: GPT_5_EFFORTS, + reasoningEffortFormat: 'chat-completions', + maxInputTokens: 272000, + maxOutputTokens: 128000, + }, + { + id: 'gpt-5-nano-2025-08-07', + apiType: 'chat-completions', + vision: true, + thinking: true, + supportsReasoningEffort: GPT_5_EFFORTS, + reasoningEffortFormat: 'chat-completions', + maxInputTokens: 272000, + maxOutputTokens: 128000, + }, + { + id: 'gpt-5-1-codex-2025-11-13', + apiType: 'responses', + vision: true, + thinking: true, + supportsReasoningEffort: GPT_5_1_EFFORTS, + reasoningEffortFormat: 'responses', + maxInputTokens: 272000, + maxOutputTokens: 128000, + }, + { + id: 'gpt-5-2-2025-12-11', + apiType: 'chat-completions', + vision: true, + thinking: true, + supportsReasoningEffort: GPT_5_2_EFFORTS, + reasoningEffortFormat: 'chat-completions', + maxInputTokens: 272000, + maxOutputTokens: 128000, + }, + { + id: 'gpt-5.4-2026-03-05', + apiType: 'chat-completions', + vision: true, + thinking: true, + supportsReasoningEffort: GPT_5_XHIGH_EFFORTS, + reasoningEffortFormat: 'chat-completions', + maxInputTokens: 922000, + maxOutputTokens: 128000, + }, + { + id: 'gpt-5.5-2026-04-24', + apiType: 'responses', + vision: true, + thinking: true, + supportsReasoningEffort: GPT_5_XHIGH_EFFORTS, + reasoningEffortFormat: 'responses', + maxInputTokens: 922000, + maxOutputTokens: 128000, + }, + { + id: 'gpt-5.6-luna-2026-07-09', + apiType: 'responses', + vision: true, + thinking: true, + supportsReasoningEffort: GPT_5_6_EFFORTS, + reasoningEffortFormat: 'responses', + maxInputTokens: 922000, + maxOutputTokens: 128000, + }, + // gpt-5.6-sol-2026-07-09 is intentionally excluded: its CodeMie Responses + // route rejects minimal requests, while Chat rejects tools with reasoning. + // Re-add only after the Responses route passes live VS Code certification. + { + id: 'gpt-5.6-terra-2026-07-09', + apiType: 'responses', + vision: true, + thinking: true, + supportsReasoningEffort: GPT_5_6_EFFORTS, + reasoningEffortFormat: 'responses', + maxInputTokens: 922000, + maxOutputTokens: 128000, + }, + { + id: 'gemini-3-flash', + apiType: 'chat-completions', + vision: true, + thinking: true, + supportsReasoningEffort: GEMINI_FLASH_EFFORTS, + reasoningEffortFormat: 'chat-completions', + maxInputTokens: 983040, + maxOutputTokens: 65536, + }, + { + id: 'gemini-3.1-pro', + apiType: 'chat-completions', + vision: true, + thinking: true, + supportsReasoningEffort: GEMINI_PRO_EFFORTS, + reasoningEffortFormat: 'chat-completions', + maxInputTokens: 983040, + maxOutputTokens: 65536, + }, + { + id: 'gemini-3.5-flash', + apiType: 'chat-completions', + vision: true, + thinking: true, + supportsReasoningEffort: GEMINI_FLASH_EFFORTS, + reasoningEffortFormat: 'chat-completions', + maxInputTokens: 983040, + maxOutputTokens: 65536, + }, + { + id: 'claude-4-5-sonnet', + apiType: 'chat-completions', + vision: true, + thinking: false, + modelOptions: { top_p: null }, + maxInputTokens: 136000, + maxOutputTokens: 64000, + }, + { + id: 'claude-sonnet-4-6', + apiType: 'messages', + vision: true, + thinking: true, + adaptiveThinking: true, + requestHeaders: MESSAGE_AUTH_HEADERS, + supportsReasoningEffort: CLAUDE_MAX_EFFORTS, + maxInputTokens: 936000, + maxOutputTokens: 64000, + }, + { + id: 'claude-sonnet-5', + apiType: 'messages', + vision: true, + thinking: true, + adaptiveThinking: true, + requestHeaders: MESSAGE_AUTH_HEADERS, + supportsReasoningEffort: CLAUDE_XHIGH_EFFORTS, + maxInputTokens: 872000, + maxOutputTokens: 128000, + }, + { + id: 'claude-opus-4-5-20251101', + apiType: 'messages', + vision: true, + thinking: false, + requestHeaders: MESSAGE_AUTH_HEADERS, + supportsReasoningEffort: CLAUDE_EFFORTS, + maxInputTokens: 136000, + maxOutputTokens: 64000, + }, + { + id: 'claude-opus-4-6-20260205', + apiType: 'messages', + vision: true, + thinking: true, + adaptiveThinking: true, + requestHeaders: MESSAGE_AUTH_HEADERS, + supportsReasoningEffort: CLAUDE_MAX_EFFORTS, + maxInputTokens: 872000, + maxOutputTokens: 128000, + }, + { + id: 'claude-opus-4-7', + apiType: 'messages', + vision: true, + thinking: true, + adaptiveThinking: true, + requestHeaders: MESSAGE_AUTH_HEADERS, + supportsReasoningEffort: CLAUDE_XHIGH_EFFORTS, + maxInputTokens: 872000, + maxOutputTokens: 128000, + }, + { + id: 'claude-opus-4-8', + apiType: 'messages', + vision: true, + thinking: true, + adaptiveThinking: true, + requestHeaders: MESSAGE_AUTH_HEADERS, + supportsReasoningEffort: CLAUDE_XHIGH_EFFORTS, + maxInputTokens: 872000, + maxOutputTokens: 128000, + }, + { + id: 'claude-haiku-4-5-20251001', + apiType: 'chat-completions', + vision: true, + thinking: false, + modelOptions: { top_p: null }, + maxInputTokens: 136000, + maxOutputTokens: 64000, + }, + { + id: 'qwen.qwen3-coder-30b-a3b-v1', + apiType: 'chat-completions', + vision: false, + thinking: false, + maxInputTokens: 245760, + maxOutputTokens: 16384, + }, + { + id: 'qwen.qwen3-coder-480b-a35b-v1', + apiType: 'chat-completions', + vision: false, + thinking: false, + maxInputTokens: 114688, + maxOutputTokens: 16384, + }, + { + id: 'moonshotai.kimi-k2.5', + apiType: 'chat-completions', + vision: true, + thinking: false, + maxInputTokens: 245760, + maxOutputTokens: 16384, + }, +]; diff --git a/src/cli/commands/proxy/connectors/vscode.ts b/src/cli/commands/proxy/connectors/vscode.ts index 48e50b4c..95e50711 100644 --- a/src/cli/commands/proxy/connectors/vscode.ts +++ b/src/cli/commands/proxy/connectors/vscode.ts @@ -3,8 +3,12 @@ import { mkdir, readFile, rename, stat, unlink, writeFile } from 'node:fs/promis import { homedir } from 'node:os'; import { dirname, join } from 'node:path'; import { ConfigurationError } from '@/utils/errors.js'; +import { + VS_CODE_SUPPORTED_MODELS, + type VsCodeApiType, + type VsCodeReasoningEffort, +} from './vscode-models.js'; -const MANAGED_MODEL_NAME = 'CodeMie Profile Model'; const SECRET_REFERENCE_PATTERN = /^\$\{input:chat\.lm\.secret\.[^}]+\}$/; interface VsCodeLanguageModelProvider { @@ -21,14 +25,21 @@ interface VsCodeManagedModel { id: string; name: string; url: string; + apiType: VsCodeApiType; toolCalling: true; - vision: true; + vision: boolean; streaming: true; - thinking: true; - supportsReasoningEffort: readonly ['minimal', 'low', 'medium', 'high']; - reasoningEffortFormat: 'chat-completions'; - maxInputTokens: 224000; - maxOutputTokens: 32000; + thinking: boolean; + adaptiveThinking?: true; + modelOptions?: Readonly<{ + temperature?: number | null; + top_p?: number | null; + }>; + requestHeaders?: Readonly>; + supportsReasoningEffort?: readonly VsCodeReasoningEffort[]; + reasoningEffortFormat?: 'chat-completions' | 'responses'; + maxInputTokens: number; + maxOutputTokens: number; } export interface WriteVsCodeConfigResult { @@ -40,18 +51,10 @@ function isRecord(value: unknown): value is Record { return typeof value === 'object' && value !== null && !Array.isArray(value); } -function isManagedModel(model: unknown, currentModelId?: string): boolean { - if (!isRecord(model)) return false; - return model.name === MANAGED_MODEL_NAME || - (currentModelId !== undefined && model.id === currentModelId); -} - function isManagedProvider(provider: unknown): provider is VsCodeLanguageModelProvider { - if (!isRecord(provider)) return false; - - const hasManagedModel = Array.isArray(provider.models) && - provider.models.some(model => isManagedModel(model)); - return hasManagedModel || (provider.vendor === 'customendpoint' && provider.name === 'CodeMie'); + return isRecord(provider) && + provider.vendor === 'customendpoint' && + provider.name === 'CodeMie'; } export function isVsCodeSecretReference(value: unknown): value is string { @@ -94,65 +97,50 @@ export function getVsCodeLanguageModelsPath(insiders = false): string { return join(productDir, 'User', 'chatLanguageModels.json'); } -function buildManagedModel(proxyUrl: string, profileModel: string): VsCodeManagedModel { - return { - id: profileModel, - name: MANAGED_MODEL_NAME, - url: new URL('/v1/chat/completions', proxyUrl).toString(), - toolCalling: true, - vision: true, - streaming: true, - thinking: true, - supportsReasoningEffort: ['minimal', 'low', 'medium', 'high'], - reasoningEffortFormat: 'chat-completions', - maxInputTokens: 224000, - maxOutputTokens: 32000, - }; +function getApiPath(apiType: VsCodeApiType): string { + if (apiType === 'responses') return '/v1/responses'; + if (apiType === 'messages') return '/v1/messages'; + return '/v1/chat/completions'; } -function reconcileModels(existingModels: unknown, managedModel: VsCodeManagedModel): unknown[] { - const models = Array.isArray(existingModels) ? existingModels : []; - const reconciled: unknown[] = []; - let inserted = false; - - for (const model of models) { - if (isManagedModel(model, managedModel.id)) { - if (!inserted) { - reconciled.push(managedModel); - inserted = true; - } - continue; +function buildManagedModels(proxyUrl: string): VsCodeManagedModel[] { + return VS_CODE_SUPPORTED_MODELS.map(definition => { + const model: VsCodeManagedModel = { + id: definition.id, + name: definition.id, + url: new URL(getApiPath(definition.apiType), proxyUrl).toString(), + apiType: definition.apiType, + toolCalling: true, + vision: definition.vision, + streaming: true, + thinking: definition.thinking, + maxInputTokens: definition.maxInputTokens, + maxOutputTokens: definition.maxOutputTokens, + }; + + if (definition.adaptiveThinking) model.adaptiveThinking = true; + if (definition.modelOptions) model.modelOptions = definition.modelOptions; + if (definition.requestHeaders) model.requestHeaders = definition.requestHeaders; + if (definition.supportsReasoningEffort) { + model.supportsReasoningEffort = definition.supportsReasoningEffort; + } + if (definition.reasoningEffortFormat) { + model.reasoningEffortFormat = definition.reasoningEffortFormat; } - reconciled.push(model); - } - if (!inserted) reconciled.push(managedModel); - return reconciled; + return model; + }); } function mergeManagedProviders( providers: VsCodeLanguageModelProvider[], - proxyUrl: string, - profileModel: string + proxyUrl: string ): { provider: VsCodeLanguageModelProvider; requiresSecretConfiguration: boolean } { const existingProvider = Object.assign({}, ...providers); - const existingModels = providers.flatMap(provider => - Array.isArray(provider.models) ? provider.models : [] - ); const existingSettings = Object.assign( {}, ...providers.map(provider => isRecord(provider.settings) ? provider.settings : {}) ); - const previousManagedModelIds = existingModels - .filter(model => isManagedModel(model)) - .map(model => isRecord(model) ? model.id : undefined) - .filter((id): id is string => typeof id === 'string'); - for (const modelId of new Set([ - profileModel, - ...previousManagedModelIds, - ])) { - delete existingSettings[modelId]; - } const existingSecretReference = providers .map(provider => provider.apiKey) .find(isVsCodeSecretReference); @@ -162,12 +150,10 @@ function mergeManagedProviders( name: 'CodeMie', vendor: 'customendpoint', apiType: 'chat-completions', - models: reconcileModels(existingModels, buildManagedModel(proxyUrl, profileModel)), + models: buildManagedModels(proxyUrl), }; - // VS Code owns model configuration state and derives "medium" as the default for this - // non-Claude model. Writing the same setting here races with VS Code's editor and can - // produce duplicate `settings` keys in its unsaved buffer. + // VS Code owns effort selections. Preserve them instead of racing with the editor. if (Object.keys(existingSettings).length > 0) provider.settings = existingSettings; else delete provider.settings; @@ -235,26 +221,18 @@ async function writeAtomically(configPath: string, content: string): Promise { return writeVsCodeLanguageModelsConfigAtPath( getVsCodeLanguageModelsPath(insiders), - proxyUrl, - profileModel + proxyUrl ); } export async function writeVsCodeLanguageModelsConfigAtPath( configPath: string, - proxyUrl: string, - profileModel: string + proxyUrl: string ): Promise { - const normalizedProfileModel = profileModel.trim(); - if (!normalizedProfileModel) { - throw new ConfigurationError('VS Code model configuration requires a profile model.'); - } - const providers = await readProviders(configPath); const managedProviderIndexes = providers .map((provider, index) => isManagedProvider(provider) ? index : -1) @@ -263,7 +241,7 @@ export async function writeVsCodeLanguageModelsConfigAtPath( .map(index => providers[index]) .filter(isManagedProvider); const { provider: managedProvider, requiresSecretConfiguration } = - mergeManagedProviders(managedProviders, proxyUrl, normalizedProfileModel); + mergeManagedProviders(managedProviders, proxyUrl); const firstManagedProviderIndex = managedProviderIndexes[0] ?? providers.length; const managedProviderIndexSet = new Set(managedProviderIndexes); const reconciledProviders = providers.flatMap((provider, index) => { diff --git a/src/cli/commands/proxy/index.ts b/src/cli/commands/proxy/index.ts index de1c97a6..08e7098d 100644 --- a/src/cli/commands/proxy/index.ts +++ b/src/cli/commands/proxy/index.ts @@ -21,6 +21,7 @@ import { import { writeDesktopConfig, getDesktopBaseDir, mapCanonicalToDesktop } from './connectors/desktop.js'; import { fetchManagedMcpServers } from './connectors/managed-mcp-remote.js'; import { writeVsCodeLanguageModelsConfig } from './connectors/vscode.js'; +import { VS_CODE_SUPPORTED_MODELS } from './connectors/vscode-models.js'; import { checkProxyHealth } from './health-check.js'; import { printDesktopInspection } from './inspect-desktop.js'; @@ -509,7 +510,7 @@ export function createProxyCommand(): Command { connect .command('vscode') .description('Configure VS Code BYOK to use the local proxy') - .option('--profile ', 'Profile whose credentials and model to use') + .option('--profile ', 'Profile whose URL, credentials, and project to use') .option('--insiders', 'Configure VS Code Insiders instead of stable VS Code') .option('--verbose', 'Show detailed connection info (URLs, config paths) for debugging') .option('--force', 'Stop any existing proxy and start a fresh one, even if it looks healthy') @@ -528,12 +529,6 @@ export function createProxyCommand(): Command { } const profile = config.name ?? 'default'; - const profileModel = config.model?.trim(); - if (!profileModel) { - throw new ConfigurationError( - `Profile "${profile}" has no model configured.\nRun: codemie setup` - ); - } if (verbose) { console.log( @@ -608,7 +603,6 @@ export function createProxyCommand(): Command { const result = await writeVsCodeLanguageModelsConfig( state!.url, - profileModel, Boolean(opts.insiders) ); logger.info( @@ -618,7 +612,7 @@ export function createProxyCommand(): Command { gatewayUrl: state!.url, profile: state!.profile, project: state!.project, - model: profileModel, + modelCount: VS_CODE_SUPPORTED_MODELS.length, clientType: state!.clientType, requiresSecretConfiguration: result.requiresSecretConfiguration, }) @@ -628,7 +622,7 @@ export function createProxyCommand(): Command { if (verbose) { console.log(` Config: ${result.configPath}`); console.log(` Gateway: ${state!.url}`); - console.log(` Model: ${profileModel}`); + console.log(` Models: ${VS_CODE_SUPPORTED_MODELS.length}`); console.log(` Project: ${config.codeMieProject || '(not configured)'}`); } @@ -636,18 +630,18 @@ export function createProxyCommand(): Command { displaySetupInstructions({ setupInstructions: [ 'One-time VS Code secret setup required:\n', - '1. Press ⇧⌘P (macOS) or Ctrl+Shift+P (Windows/Linux).', - '2. Run: Chat: Manage Language Models', - '3. Right-click CodeMie Profile Model → Update API Key', + '1. Open VS Code and Press ⇧⌘P (macOS) or Ctrl+Shift+P (Windows/Linux).', + '2. Find Chat: Manage Language Models', + '3. In opened dialog Right-click any CodeMie model → Update API Key', `4. Enter API key: ${state!.gatewayKey}\n`, - 'Reload VS Code to apply changes.', + 'Reload VS Code, then select a CodeMie model from the model picker.', ].join('\n'), }); } else { console.log( chalk.dim( ` If VS Code reports a missing or invalid key, open Chat: Manage Language Models, ` + - `then right-click CodeMie Profile Model → Update API Key and enter ${state!.gatewayKey}.` + `then right-click any CodeMie model → Update API Key and enter ${state!.gatewayKey}.` ) ); } diff --git a/src/providers/plugins/sso/proxy/__tests__/transparent-vscode-proxy.integration.test.ts b/src/providers/plugins/sso/proxy/__tests__/transparent-vscode-proxy.integration.test.ts index c6209bae..c685dbe6 100644 --- a/src/providers/plugins/sso/proxy/__tests__/transparent-vscode-proxy.integration.test.ts +++ b/src/providers/plugins/sso/proxy/__tests__/transparent-vscode-proxy.integration.test.ts @@ -112,7 +112,7 @@ describe('transparent VS Code proxy integration', () => { resetPluginRegistry(); }); - it.each(['/v1/responses', '/v1/chat/completions?trace=test'])( + it.each(['/v1/responses', '/v1/messages', '/v1/chat/completions?trace=test'])( 'forwards the caller-selected model and request body unchanged for %s', async (path) => { const captured: CapturedRequest[] = []; const upstream = await startCapturingUpstream(captured); @@ -120,7 +120,7 @@ describe('transparent VS Code proxy integration', () => { const startedProxy = await startProxy(upstream.url); proxies.push(startedProxy.proxy); const body = { - model: 'profile-selected-model', + model: 'caller-selected-model', messages: [{ role: 'user', content: 'Hello' }], tools: [{ type: 'function', name: 'read_file' }], stream: false, @@ -138,53 +138,54 @@ describe('transparent VS Code proxy integration', () => { } ); - it('forwards SSE bytes incrementally and unchanged', async () => { - const chunks = [ - 'data: {"id":"1","choices":[{"delta":{"content":"A"}}]}\n\n', - 'data: {"id":"1","choices":[{"delta":{"content":"B"}}]}\n\n', - 'data: [DONE]\n\n', - ]; - let upstreamClosed = false; - const upstream = await listen(createServer((req, res) => { - void readRequestBody(req).then(() => { - res.statusCode = 200; - res.setHeader('content-type', 'text/event-stream'); - res.write(chunks[0]); - setTimeout(() => { - res.write(chunks[1]); - res.end(chunks[2], () => { upstreamClosed = true; }); - }, 30); - }); - })); - servers.push(upstream.server); - const startedProxy = await startProxy(upstream.url); - proxies.push(startedProxy.proxy); + it.each(['/v1/chat/completions', '/v1/responses', '/v1/messages'])( + 'forwards SSE bytes incrementally and unchanged for %s', async (path) => { + const chunks = [ + 'data: {"id":"1","choices":[{"delta":{"content":"A"}}]}\n\n', + 'data: {"id":"1","choices":[{"delta":{"content":"B"}}]}\n\n', + 'data: [DONE]\n\n', + ]; + let upstreamClosed = false; + const upstream = await listen(createServer((req, res) => { + void readRequestBody(req).then(() => { + res.statusCode = 200; + res.setHeader('content-type', 'text/event-stream'); + res.write(chunks[0]); + setTimeout(() => { + res.write(chunks[1]); + res.end(chunks[2], () => { upstreamClosed = true; }); + }, 30); + }); + })); + servers.push(upstream.server); + const startedProxy = await startProxy(upstream.url); + proxies.push(startedProxy.proxy); - const response = await fetch( - `${startedProxy.url}/v1/chat/completions`, - authenticatedJsonInit({ model: 'profile-selected-model', messages: [], stream: true }) - ); - const reader = response.body!.getReader(); - const first = await reader.read(); - - expect(upstreamClosed).toBe(false); - const received = [Buffer.from(first.value ?? []).toString('utf-8')]; - while (true) { - const next = await reader.read(); - if (next.done) break; - received.push(Buffer.from(next.value).toString('utf-8')); - } + const response = await fetch( + `${startedProxy.url}${path}`, + authenticatedJsonInit({ model: 'caller-selected-model', messages: [], stream: true }) + ); + const reader = response.body!.getReader(); + const first = await reader.read(); - expect(response.headers.get('content-type')).toContain('text/event-stream'); - expect(received.join('')).toBe(chunks.join('')); - }); + expect(upstreamClosed).toBe(false); + const received = [Buffer.from(first.value ?? []).toString('utf-8')]; + while (true) { + const next = await reader.read(); + if (next.done) break; + received.push(Buffer.from(next.value).toString('utf-8')); + } - it('forwards tool-call response fields unchanged', async () => { - const toolCall = { + expect(response.headers.get('content-type')).toContain('text/event-stream'); + expect(received.join('')).toBe(chunks.join('')); + } + ); + + it.each([ + ['/v1/chat/completions', { id: 'completion-1', choices: [{ - index: 0, - delta: { + message: { tool_calls: [{ id: 'call-1', type: 'function', @@ -193,7 +194,27 @@ describe('transparent VS Code proxy integration', () => { }, finish_reason: 'tool_calls', }], - }; + }], + ['/v1/responses', { + id: 'response-1', + output: [{ + id: 'call-1', + type: 'function_call', + name: 'get_test_value', + arguments: '{"name":"value"}', + }], + }], + ['/v1/messages', { + id: 'message-1', + content: [{ + id: 'call-1', + type: 'tool_use', + name: 'get_test_value', + input: { name: 'value' }, + }], + stop_reason: 'tool_use', + }], + ])('forwards tool-call response fields unchanged for %s', async (path, toolCall) => { const upstream = await listen(createServer((req, res) => { void readRequestBody(req).then(() => { res.setHeader('content-type', 'application/json'); @@ -205,8 +226,8 @@ describe('transparent VS Code proxy integration', () => { proxies.push(startedProxy.proxy); const response = await fetch( - `${startedProxy.url}/v1/chat/completions`, - authenticatedJsonInit({ model: 'profile-selected-model', messages: [] }) + `${startedProxy.url}${path}`, + authenticatedJsonInit({ model: 'caller-selected-model', messages: [] }) ); expect(await response.json()).toEqual(toolCall); diff --git a/src/providers/plugins/sso/proxy/plugins/index.ts b/src/providers/plugins/sso/proxy/plugins/index.ts index 0f3fdddc..d8eafb75 100644 --- a/src/providers/plugins/sso/proxy/plugins/index.ts +++ b/src/providers/plugins/sso/proxy/plugins/index.ts @@ -16,6 +16,7 @@ import { RequestSanitizerPlugin } from './request-sanitizer.plugin.js'; import { ClaudeRequestNormalizerPlugin } from './claude-request-normalizer.plugin.js'; import { KimiRequestNormalizerPlugin } from './kimi-request-normalizer.plugin.js'; import { CodexEncryptedContentSanitizerPlugin } from './codex-encrypted-content-sanitizer.plugin.js'; +import { VsCodeRequestNormalizerPlugin } from './vscode-request-normalizer.plugin.js'; import { LoggingPlugin } from './logging.plugin.js'; import { SSOSessionSyncPlugin } from './sso.session-sync.plugin.js'; @@ -36,6 +37,7 @@ export function registerCorePlugins(): void { registry.register(new KimiRequestNormalizerPlugin()); // Priority 14 - caps Kimi output token requests for upstream limits registry.register(new RequestSanitizerPlugin()); // Priority 15 - strips unsupported reasoning params registry.register(new CodexEncryptedContentSanitizerPlugin()); // Priority 16 - strips encrypted reasoning state for Codex + registry.register(new VsCodeRequestNormalizerPlugin()); // Priority 17 - constrains VS Code user identifiers registry.register(new HeaderInjectionPlugin()); registry.register(new LoggingPlugin()); // Always enabled - logs to log files at INFO level registry.register(new SSOSessionSyncPlugin()); // Priority 100 - syncs sessions via multiple processors @@ -56,6 +58,7 @@ export { ClaudeRequestNormalizerPlugin, KimiRequestNormalizerPlugin, CodexEncryptedContentSanitizerPlugin, + VsCodeRequestNormalizerPlugin, LoggingPlugin, }; export { SSOSessionSyncPlugin } from './sso.session-sync.plugin.js'; diff --git a/src/providers/plugins/sso/proxy/plugins/vscode-request-normalizer.plugin.ts b/src/providers/plugins/sso/proxy/plugins/vscode-request-normalizer.plugin.ts new file mode 100644 index 00000000..0cf79d2a --- /dev/null +++ b/src/providers/plugins/sso/proxy/plugins/vscode-request-normalizer.plugin.ts @@ -0,0 +1,85 @@ +/** + * VS Code Request Normalizer + * Priority: 17 (after request sanitizers, before header injection) + * + * Azure-backed OpenAI routes reject `user` identifiers longer than 64 + * characters. The CodeMie Responses path can namespace the supplied value, + * so keep the client portion at or below 32 characters. Longer identifiers + * become a stable truncated SHA-256 digest. When VS Code omits the field, send + * a short explicit fallback to prevent the upstream from deriving an + * overlength default identity. + * + * Scope: VS Code BYOK Responses API traffic only. + */ + +import { createHash } from 'node:crypto'; +import { ProxyPlugin, PluginContext, ProxyInterceptor } from './types.js'; +import { ProxyContext } from '../proxy-types.js'; +import { logger } from '../../../../../utils/logger.js'; + +const VS_CODE_CLIENT_TYPE = 'vscode-byok'; +const MAX_CLIENT_USER_LENGTH = 32; +const FALLBACK_USER = 'vscode-byok'; + +export class VsCodeRequestNormalizerPlugin implements ProxyPlugin { + id = '@codemie/proxy-vscode-request-normalizer'; + name = 'VS Code Request Normalizer'; + version = '1.0.0'; + priority = 17; + + async createInterceptor(context: PluginContext): Promise { + if (context.config.clientType !== VS_CODE_CLIENT_TYPE) { + throw new Error(`Plugin disabled for agent: ${context.config.clientType}`); + } + + return new VsCodeRequestNormalizerInterceptor(); + } +} + +class VsCodeRequestNormalizerInterceptor implements ProxyInterceptor { + name = 'vscode-request-normalizer'; + + async onRequest(context: ProxyContext): Promise { + if (!context.requestBody || !context.headers['content-type']?.includes('application/json')) { + return; + } + + try { + const path = context.url.split('?')[0].replace(/\/+$/, ''); + if (!path.endsWith('/responses')) { + return; + } + + const body = JSON.parse(context.requestBody.toString('utf-8')) as Record; + const existingUser = typeof body.user === 'string' ? body.user : ''; + const normalizedUser = existingUser.length === 0 + ? FALLBACK_USER + : existingUser.length <= MAX_CLIENT_USER_LENGTH + ? existingUser + : createHash('sha256') + .update(existingUser, 'utf-8') + .digest('hex') + .slice(0, MAX_CLIENT_USER_LENGTH); + + if (body.user === normalizedUser) { + return; + } + + body.user = normalizedUser; + + const normalizedBody = JSON.stringify(body); + context.requestBody = Buffer.from(normalizedBody, 'utf-8'); + context.headers['content-length'] = String(context.requestBody.length); + + logger.debug( + `[${this.name}] Normalized Responses user identifier`, + { + originalLength: existingUser.length, + normalizedLength: normalizedUser.length, + } + ); + } catch { + // Not valid JSON or unexpected structure — pass through unchanged. + } + } +} diff --git a/tests/integration/vscode-byok.test.ts b/tests/integration/vscode-byok.test.ts index d88dfb0f..9129f3e8 100644 --- a/tests/integration/vscode-byok.test.ts +++ b/tests/integration/vscode-byok.test.ts @@ -1,5 +1,5 @@ /** - * VS Code BYOK end-to-end integration test + * VS Code BYOK end-to-end integration tests * @group integration */ @@ -9,17 +9,26 @@ import type { AddressInfo } from 'node:net'; import { tmpdir } from 'node:os'; import { join } from 'node:path'; import { afterEach, beforeEach, describe, expect, it } from 'vitest'; -import { writeVsCodeLanguageModelsConfigAtPath } from '../../src/cli/commands/proxy/connectors/vscode.js'; +import { + VS_CODE_SUPPORTED_MODELS, + type VsCodeModelDefinition, + type VsCodeReasoningEffort, +} from '../../src/cli/commands/proxy/connectors/vscode-models.js'; +import { + writeVsCodeLanguageModelsConfigAtPath, +} from '../../src/cli/commands/proxy/connectors/vscode.js'; import { CodeMieProxy } from '../../src/providers/plugins/sso/proxy/sso.proxy.js'; import { GatewayKeyPlugin } from '../../src/providers/plugins/sso/proxy/plugins/gateway-key.plugin.js'; -import { HeaderInjectionPlugin } from '../../src/providers/plugins/sso/proxy/plugins/header-injection.plugin.js'; +import { + HeaderInjectionPlugin, +} from '../../src/providers/plugins/sso/proxy/plugins/header-injection.plugin.js'; import { getPluginRegistry, resetPluginRegistry, } from '../../src/providers/plugins/sso/proxy/plugins/registry.js'; const GATEWAY_KEY = 'test-local-key'; -const PROFILE_MODEL = 'profile-selected-model'; +const PROFILE_MODEL = 'profile-selected-model-that-must-not-be-used'; interface StartedServer { server: Server; @@ -27,14 +36,16 @@ interface StartedServer { } interface CapturedRequest { + url: string; headers: IncomingMessage['headers']; body: Record; } interface LanguageModel { - id?: unknown; - name?: unknown; - url?: unknown; + id: string; + name: string; + url: string; + apiType: string; } interface LanguageModelProvider { @@ -63,7 +74,78 @@ async function readRequestBody(req: IncomingMessage): Promise; } -describe('VS Code BYOK utility flow', () => { +function buildRequestBody( + definition: VsCodeModelDefinition, + effort: VsCodeReasoningEffort | undefined +): Record { + if (definition.apiType === 'responses') { + return { + model: definition.id, + stream: false, + input: [{ role: 'user', content: 'Call get_test_value.' }], + tools: [{ + type: 'function', + name: 'get_test_value', + description: 'Return a synthetic test value.', + parameters: { type: 'object', properties: {} }, + strict: true, + }], + tool_choice: 'required', + ...(effort ? { reasoning: { effort } } : {}), + }; + } + + if (definition.apiType === 'messages') { + return { + model: definition.id, + max_tokens: 1024, + stream: false, + messages: [{ role: 'user', content: 'Call get_test_value.' }], + tools: [{ + name: 'get_test_value', + description: 'Return a synthetic test value.', + input_schema: { type: 'object', properties: {} }, + }], + tool_choice: { type: 'auto' }, + ...(definition.adaptiveThinking ? { thinking: { type: 'adaptive' } } : {}), + ...(effort ? { output_config: { effort } } : {}), + }; + } + + return { + model: definition.id, + stream: false, + messages: [{ role: 'user', content: 'Call get_test_value.' }], + tools: [{ + type: 'function', + function: { + name: 'get_test_value', + description: 'Return a synthetic test value.', + parameters: { type: 'object', properties: {} }, + strict: true, + }, + }], + tool_choice: 'required', + ...(effort ? { reasoning_effort: effort } : {}), + }; +} + +function buildVsCodeAuthHeaders( + definition: VsCodeModelDefinition +): Record { + if (definition.requestHeaders?.Authorization) { + return { + authorization: definition.requestHeaders.Authorization.replace( + '${apiKey}', + GATEWAY_KEY + ), + }; + } + if (definition.apiType === 'messages') return { 'x-api-key': GATEWAY_KEY }; + return { authorization: `Bearer ${GATEWAY_KEY}` }; +} + +describe('VS Code BYOK model matrix', () => { const proxies: CodeMieProxy[] = []; const servers: Server[] = []; let testDir: string; @@ -81,30 +163,17 @@ describe('VS Code BYOK utility flow', () => { resetPluginRegistry(); }); - it('uses the configured profile model through the authenticated local proxy', async () => { + it('forwards every selected model and supported effort without using the profile model', async () => { const captured: CapturedRequest[] = []; - const toolResponse = { - id: 'completion-1', - choices: [{ - message: { - role: 'assistant', - tool_calls: [{ - id: 'call-1', - type: 'function', - function: { - name: 'get_test_value', - arguments: '{"name":"vscode-byok"}', - }, - }], - }, - finish_reason: 'tool_calls', - }], - }; const upstream = await listen(createServer((req, res) => { void readRequestBody(req).then((body) => { - captured.push({ headers: req.headers, body }); + captured.push({ + url: req.url ?? '/', + headers: req.headers, + body, + }); res.setHeader('content-type', 'application/json'); - res.end(JSON.stringify(toolResponse)); + res.end(JSON.stringify({ ok: true })); }); })); servers.push(upstream.server); @@ -125,64 +194,50 @@ describe('VS Code BYOK utility flow', () => { const startedProxy = await proxy.start(); const configPath = join(testDir, 'User', 'chatLanguageModels.json'); - await writeVsCodeLanguageModelsConfigAtPath( - configPath, - startedProxy.url, - PROFILE_MODEL - ); + await writeVsCodeLanguageModelsConfigAtPath(configPath, startedProxy.url); const providers = JSON.parse( await readFile(configPath, 'utf-8') ) as LanguageModelProvider[]; const codeMieProvider = providers.find( - (provider) => provider.name === 'CodeMie' && provider.vendor === 'customendpoint' - ); - const configuredModel = codeMieProvider?.models?.find( - (model) => model.name === 'CodeMie Profile Model' + provider => provider.name === 'CodeMie' && provider.vendor === 'customendpoint' ); - expect(configuredModel).toMatchObject({ - id: PROFILE_MODEL, - url: `${startedProxy.url}/v1/chat/completions`, - }); + expect(codeMieProvider?.models).toHaveLength(VS_CODE_SUPPORTED_MODELS.length); + expect(codeMieProvider?.models?.some(model => model.id === PROFILE_MODEL)).toBe(false); - const requestBody = { - model: configuredModel?.id, - stream: false, - messages: [{ - role: 'user', - content: 'Call get_test_value with name "vscode-byok". Do not answer directly.', - }], - tools: [{ - type: 'function', - function: { - name: 'get_test_value', - description: 'Return a harmless synthetic test value.', - parameters: { - type: 'object', - properties: { name: { type: 'string' } }, - required: ['name'], - additionalProperties: false, - }, - strict: true, - }, - }], - tool_choice: 'required', - }; - const response = await fetch(String(configuredModel?.url), { - method: 'POST', - headers: { - authorization: `Bearer ${GATEWAY_KEY}`, - 'content-type': 'application/json', - }, - body: JSON.stringify(requestBody), - }); + for (const definition of VS_CODE_SUPPORTED_MODELS) { + const configuredModel = codeMieProvider?.models?.find(model => model.id === definition.id); + expect(configuredModel).toMatchObject({ + id: definition.id, + name: definition.id, + apiType: definition.apiType, + }); - expect(response.status).toBe(200); - expect(await response.json()).toEqual(toolResponse); - expect(captured).toHaveLength(1); - expect(captured[0].body).toEqual(requestBody); - expect(captured[0].headers.authorization).toBeUndefined(); - expect(captured[0].headers['x-codemie-client']).toBe('vscode-byok'); - expect(captured[0].headers['x-codemie-project']).toBe('test-project'); + const efforts: ReadonlyArray = + [undefined, ...(definition.supportsReasoningEffort ?? [])]; + for (const effort of efforts) { + const requestBody = buildRequestBody(definition, effort); + const response = await fetch(String(configuredModel?.url), { + method: 'POST', + headers: { + ...buildVsCodeAuthHeaders(definition), + 'content-type': 'application/json', + }, + body: JSON.stringify(requestBody), + }); + + expect(response.status).toBe(200); + expect(await response.json()).toEqual({ ok: true }); + + const forwarded = captured.at(-1); + expect(forwarded?.url).toBe(new URL(String(configuredModel?.url)).pathname); + expect(forwarded?.body).toEqual(requestBody); + expect(forwarded?.headers.authorization).toBeUndefined(); + expect(forwarded?.headers['x-api-key']).toBeUndefined(); + expect(forwarded?.headers['x-codemie-client']).toBe('vscode-byok'); + expect(forwarded?.headers['x-codemie-project']).toBe('test-project'); + expect(forwarded?.headers['x-codemie-cli-model']).toBeUndefined(); + } + } }); }); diff --git a/tests/integration/vscode-models.live.test.ts b/tests/integration/vscode-models.live.test.ts new file mode 100644 index 00000000..afc9669c --- /dev/null +++ b/tests/integration/vscode-models.live.test.ts @@ -0,0 +1,527 @@ +/** + * Opt-in live CodeMie model certification for the VS Code catalog. + * + * Required: + * CODEMIE_VSCODE_LIVE=1 + * CODEMIE_VSCODE_MODELS= + * + * Authentication: + * Either start a VS Code proxy with `codemie proxy connect vscode`, or set: + * CODEMIE_VSCODE_LIVE_URL=http://127.0.0.1:4001 + * CODEMIE_VSCODE_LIVE_API_KEY= + * + * Optional: + * CODEMIE_VSCODE_EFFORTS= + * CODEMIE_VSCODE_PROJECT= + * CODEMIE_VSCODE_REPORT_PATH= + * CODEMIE_VSCODE_REQUEST_TIMEOUT_MS= + * CODEMIE_VSCODE_MAX_ATTEMPTS= + * + * `all` efforts certifies the no-effort/default request and every effort + * advertised for each selected model. + * + * @group integration + */ + +import { mkdir, writeFile } from 'node:fs/promises'; +import { dirname, resolve } from 'node:path'; +import { describe, expect, it } from 'vitest'; +import { isProcessAlive, readState } from '../../src/cli/commands/proxy/daemon-manager.js'; +import { + VS_CODE_SUPPORTED_MODELS, + type VsCodeModelDefinition, + type VsCodeReasoningEffort, +} from '../../src/cli/commands/proxy/connectors/vscode-models.js'; + +const LIVE_ENABLED = process.env.CODEMIE_VSCODE_LIVE === '1'; +const ALL_EFFORTS: readonly VsCodeReasoningEffort[] = [ + 'none', + 'minimal', + 'low', + 'medium', + 'high', + 'xhigh', + 'max', +]; +const DEFAULT_REPORT_PATH = resolve('reports/vscode-model-certification.md'); + +interface LiveConfiguration { + baseUrl: string; + apiKey: string; + project?: string; + source: 'environment' | 'running-vscode-proxy'; +} + +interface CertificationResult { + model: string; + apiType: VsCodeModelDefinition['apiType']; + effort: VsCodeReasoningEffort | 'default'; + passed: boolean; + status: number; + toolCall: boolean; + durationMs: number; + attempts: number; + detail: string; +} + +function buildVsCodeAuthHeaders( + definition: VsCodeModelDefinition, + apiKey: string +): Record { + if (definition.requestHeaders?.Authorization) { + return { + authorization: definition.requestHeaders.Authorization.replace('${apiKey}', apiKey), + }; + } + if (definition.apiType === 'messages') return { 'x-api-key': apiKey }; + return { authorization: `Bearer ${apiKey}` }; +} + +function getApiPath(definition: VsCodeModelDefinition): string { + if (definition.apiType === 'responses') return '/v1/responses'; + if (definition.apiType === 'messages') return '/v1/messages'; + return '/v1/chat/completions'; +} + +function selectModels(selector: string): readonly VsCodeModelDefinition[] { + if (selector === 'all') return VS_CODE_SUPPORTED_MODELS; + + const requestedIds = selector.split(',').map(value => value.trim()).filter(Boolean); + const selected = VS_CODE_SUPPORTED_MODELS.filter(model => requestedIds.includes(model.id)); + const selectedIds = new Set(selected.map(model => model.id)); + const unknownIds = requestedIds.filter(id => !selectedIds.has(id)); + if (unknownIds.length > 0) { + throw new Error(`Unknown VS Code model IDs: ${unknownIds.join(', ')}`); + } + return selected; +} + +function selectEfforts( + definition: VsCodeModelDefinition, + selector: string | undefined +): ReadonlyArray { + if (!selector) return [undefined]; + if (selector === 'all') { + return [undefined, ...(definition.supportsReasoningEffort ?? [])]; + } + + const requested = selector.split(',').map(value => value.trim()).filter(Boolean); + const invalid = requested.filter( + (effort): effort is string => !ALL_EFFORTS.includes(effort as VsCodeReasoningEffort) + ); + if (invalid.length > 0) { + throw new Error(`Unknown reasoning efforts: ${invalid.join(', ')}`); + } + + const efforts = requested as VsCodeReasoningEffort[]; + const supported = definition.supportsReasoningEffort ?? []; + const unsupported = efforts.filter(effort => !supported.includes(effort)); + if (unsupported.length > 0) { + throw new Error( + `${definition.id} does not advertise efforts: ${unsupported.join(', ')}` + ); + } + return efforts; +} + +function buildRequestBody( + definition: VsCodeModelDefinition, + effort: VsCodeReasoningEffort | undefined +): Record { + if (definition.apiType === 'responses') { + return { + model: definition.id, + input: [{ role: 'user', content: 'Call get_test_value with value "ready".' }], + tools: [{ + type: 'function', + name: 'get_test_value', + description: 'Return a supplied synthetic value.', + parameters: { + type: 'object', + properties: { value: { type: 'string' } }, + required: ['value'], + }, + }], + tool_choice: 'required', + stream: true, + ...(effort ? { reasoning: { effort } } : {}), + }; + } + + if (definition.apiType === 'messages') { + return { + model: definition.id, + max_tokens: 1024, + messages: [{ + role: 'user', + content: 'Call get_test_value with value "ready".', + }], + tools: [{ + name: 'get_test_value', + description: 'Return a supplied synthetic value.', + input_schema: { + type: 'object', + properties: { value: { type: 'string' } }, + required: ['value'], + }, + }], + tool_choice: { type: 'auto' }, + stream: true, + ...(definition.adaptiveThinking ? { thinking: { type: 'adaptive' } } : {}), + ...(effort ? { output_config: { effort } } : {}), + }; + } + + return { + model: definition.id, + messages: [{ + role: 'user', + content: 'Call get_test_value with value "ready".', + }], + tools: [{ + type: 'function', + function: { + name: 'get_test_value', + description: 'Return a supplied synthetic value.', + parameters: { + type: 'object', + properties: { value: { type: 'string' } }, + required: ['value'], + }, + }, + }], + tool_choice: 'required', + stream: true, + ...(definition.modelOptions?.temperature === null ? {} : { + ...(definition.modelOptions?.temperature !== undefined + ? { temperature: definition.modelOptions.temperature } + : {}), + }), + ...(definition.modelOptions?.top_p === null ? {} : { + ...(definition.modelOptions?.top_p !== undefined + ? { top_p: definition.modelOptions.top_p } + : {}), + }), + ...(effort ? { reasoning_effort: effort } : {}), + }; +} + +function parseResponsePayloads(rawBody: string): unknown[] { + const trimmed = rawBody.trim(); + if (!trimmed) return []; + + if (trimmed.startsWith('{') || trimmed.startsWith('[')) { + try { + return [JSON.parse(trimmed) as unknown]; + } catch { + return []; + } + } + + const payloads: unknown[] = []; + for (const line of rawBody.split(/\r?\n/)) { + if (!line.startsWith('data:')) continue; + const data = line.slice('data:'.length).trim(); + if (!data || data === '[DONE]') continue; + try { + payloads.push(JSON.parse(data) as unknown); + } catch { + // Ignore non-JSON SSE metadata while retaining all valid event payloads. + } + } + return payloads; +} + +function containsToolCall(value: unknown): boolean { + if (Array.isArray(value)) return value.some(item => containsToolCall(item)); + if (typeof value !== 'object' || value === null) return false; + + const record = value as Record; + if (record.type === 'function_call' || record.type === 'tool_use') return true; + if (Array.isArray(record.tool_calls) && record.tool_calls.length > 0) return true; + return Object.values(record).some(item => containsToolCall(item)); +} + +function findErrorMessage(value: unknown): string | undefined { + if (Array.isArray(value)) { + for (const item of value) { + const message = findErrorMessage(item); + if (message) return message; + } + return undefined; + } + if (typeof value !== 'object' || value === null) return undefined; + + const record = value as Record; + if (typeof record.message === 'string' && ( + record.type === 'error' || + record.code !== undefined || + record.error !== undefined + )) { + return record.message; + } + if (record.error !== undefined) { + const nested = findErrorMessage(record.error); + if (nested) return nested; + } + return undefined; +} + +function compactDetail(value: string): string { + return value + .replace(/Bearer\s+\S+/gi, 'Bearer [redacted]') + .replace(/\s+/g, ' ') + .replace(/\|/g, '\\|') + .trim() + .slice(0, 500); +} + +function getResponseDetail( + responseOk: boolean, + toolCall: boolean, + rawBody: string, + payloads: unknown[] +): string { + if (responseOk && toolCall) return 'Streaming function tool call received.'; + if (responseOk) return 'Request succeeded but no function tool call was detected.'; + + for (const payload of payloads) { + const message = findErrorMessage(payload); + if (message) return compactDetail(message); + } + return compactDetail(rawBody) || 'Request failed without a response body.'; +} + +async function resolveLiveConfiguration(): Promise { + const configuredUrl = process.env.CODEMIE_VSCODE_LIVE_URL?.replace(/\/$/, ''); + const configuredKey = process.env.CODEMIE_VSCODE_LIVE_API_KEY; + if (configuredUrl && configuredKey) { + return { + baseUrl: configuredUrl, + apiKey: configuredKey, + project: process.env.CODEMIE_VSCODE_PROJECT, + source: 'environment', + }; + } + + const state = await readState(); + if (!state || !isProcessAlive(state.pid)) { + throw new Error( + 'Start `codemie proxy connect vscode`, or set CODEMIE_VSCODE_LIVE_URL and ' + + 'CODEMIE_VSCODE_LIVE_API_KEY.' + ); + } + if (state.clientType !== 'vscode-byok') { + throw new Error( + `The running proxy client is "${state.clientType ?? 'unknown'}"; ` + + 'live VS Code certification requires a vscode-byok proxy.' + ); + } + + return { + baseUrl: state.url.replace(/\/$/, ''), + apiKey: state.gatewayKey, + project: process.env.CODEMIE_VSCODE_PROJECT, + source: 'running-vscode-proxy', + }; +} + +async function certifyAttempt( + configuration: LiveConfiguration, + definition: VsCodeModelDefinition, + effort: VsCodeReasoningEffort | undefined, + timeoutMs: number, + attempt: number +): Promise { + const startedAt = Date.now(); + const headers: Record = { + ...buildVsCodeAuthHeaders(definition, configuration.apiKey), + 'content-type': 'application/json', + }; + if (configuration.project) headers['x-codemie-project'] = configuration.project; + + try { + const response = await fetch(`${configuration.baseUrl}${getApiPath(definition)}`, { + method: 'POST', + headers, + body: JSON.stringify(buildRequestBody(definition, effort)), + signal: AbortSignal.timeout(timeoutMs), + }); + const rawBody = await response.text(); + const payloads = parseResponsePayloads(rawBody); + const toolCall = payloads.some(payload => containsToolCall(payload)); + const passed = response.ok && toolCall; + + return { + model: definition.id, + apiType: definition.apiType, + effort: effort ?? 'default', + passed, + status: response.status, + toolCall, + durationMs: Date.now() - startedAt, + attempts: attempt, + detail: getResponseDetail(response.ok, toolCall, rawBody, payloads), + }; + } catch (error) { + return { + model: definition.id, + apiType: definition.apiType, + effort: effort ?? 'default', + passed: false, + status: 0, + toolCall: false, + durationMs: Date.now() - startedAt, + attempts: attempt, + detail: compactDetail(error instanceof Error ? error.message : String(error)), + }; + } +} + +function isRetryable(result: CertificationResult): boolean { + return result.status === 0 || result.status === 429 || result.status >= 500; +} + +async function certifyCombination( + configuration: LiveConfiguration, + definition: VsCodeModelDefinition, + effort: VsCodeReasoningEffort | undefined, + timeoutMs: number, + maxAttempts: number +): Promise { + let lastResult: CertificationResult | undefined; + for (let attempt = 1; attempt <= maxAttempts; attempt++) { + const result = await certifyAttempt( + configuration, + definition, + effort, + timeoutMs, + attempt + ); + if (result.passed || !isRetryable(result)) return result; + lastResult = result; + } + if (!lastResult) throw new Error('Certification did not execute any attempts.'); + return lastResult; +} + +function renderReport( + configuration: LiveConfiguration, + selectedModels: readonly VsCodeModelDefinition[], + results: readonly CertificationResult[] +): string { + const passed = results.filter(result => result.passed).length; + const failed = results.length - passed; + const fullyPassingModels = selectedModels.filter(model => { + const modelResults = results.filter(result => result.model === model.id); + return modelResults.length > 0 && modelResults.every(result => result.passed); + }).length; + + const lines = [ + '# VS Code Model Certification Report', + '', + `- Generated: ${new Date().toISOString()}`, + `- Target: \`${configuration.baseUrl}\``, + `- Authentication source: \`${configuration.source}\``, + '- Flow: streamed function-tool request through the VS Code-scoped CodeMie proxy', + `- Models: ${selectedModels.length}`, + `- Model/effort combinations: ${results.length}`, + `- Passing combinations: ${passed}`, + `- Failing combinations: ${failed}`, + `- Fully passing models: ${fullyPassingModels}/${selectedModels.length}`, + '', + 'No API keys, tokens, request headers, or response payloads are stored in this report.', + '', + '## Results', + '', + '| Model | API | Effort | Result | HTTP | Tool call | Attempts | Duration | Detail |', + '|---|---|---|---:|---:|---:|---:|---:|---|', + ...results.map(result => + `| \`${result.model}\` | ${result.apiType} | \`${result.effort}\` | ` + + `${result.passed ? 'PASS' : 'FAIL'} | ${result.status || 'n/a'} | ` + + `${result.toolCall ? 'yes' : 'no'} | ${result.attempts} | ` + + `${result.durationMs} ms | ${result.detail} |` + ), + '', + '## Advertised Configuration', + '', + '| Model | API | Advertised efforts | VS Code model options |', + '|---|---|---|---|', + ...selectedModels.map(model => { + const efforts = model.supportsReasoningEffort?.join(', ') ?? 'none'; + const modelOptions = model.modelOptions + ? `\`${JSON.stringify(model.modelOptions)}\`` + : 'default'; + return `| \`${model.id}\` | ${model.apiType} | ${efforts} | ${modelOptions} |`; + }), + '', + '## Interpretation', + '', + '- `default` means no explicit reasoning-effort parameter was sent.', + '- Every advertised effort was sent using the model’s configured API body shape.', + '- PASS requires both a successful HTTP response and a streamed function tool call.', + '- A model should remain in the released catalog only when every advertised row passes.', + '', + '## Documentation', + '', + '- [VS Code custom endpoint model configuration](https://code.visualstudio.com/docs/agent-customization/language-models)', + '- [OpenAI model and reasoning guidance](https://developers.openai.com/api/docs/models)', + '- [AWS Claude Messages request parameters](https://docs.aws.amazon.com/bedrock/latest/userguide/model-parameters-anthropic-claude-messages-request-response.html)', + '', + ]; + + return `${lines.join('\n')}\n`; +} + +describe.runIf(LIVE_ENABLED)('VS Code live model certification', () => { + it('streams tools through every requested model and reasoning effort', async () => { + const selector = process.env.CODEMIE_VSCODE_MODELS; + if (!selector) { + throw new Error('CODEMIE_VSCODE_MODELS is required for live certification.'); + } + + const configuration = await resolveLiveConfiguration(); + const selectedModels = selectModels(selector); + expect(selectedModels.length).toBeGreaterThan(0); + + const parsedTimeout = Number(process.env.CODEMIE_VSCODE_REQUEST_TIMEOUT_MS ?? '120000'); + if (!Number.isFinite(parsedTimeout) || parsedTimeout <= 0) { + throw new Error('CODEMIE_VSCODE_REQUEST_TIMEOUT_MS must be a positive number.'); + } + const maxAttempts = Number(process.env.CODEMIE_VSCODE_MAX_ATTEMPTS ?? '2'); + if (!Number.isInteger(maxAttempts) || maxAttempts <= 0) { + throw new Error('CODEMIE_VSCODE_MAX_ATTEMPTS must be a positive integer.'); + } + + const results: CertificationResult[] = []; + for (const definition of selectedModels) { + const efforts = selectEfforts(definition, process.env.CODEMIE_VSCODE_EFFORTS); + for (const effort of efforts) { + results.push(await certifyCombination( + configuration, + definition, + effort, + parsedTimeout, + maxAttempts + )); + } + } + + const reportPath = resolve( + process.env.CODEMIE_VSCODE_REPORT_PATH ?? DEFAULT_REPORT_PATH + ); + await mkdir(dirname(reportPath), { recursive: true }); + await writeFile( + reportPath, + renderReport(configuration, selectedModels, results), + 'utf-8' + ); + + const failures = results.filter(result => !result.passed); + expect( + failures, + `${failures.length} live certification combination(s) failed. ` + + `See ${reportPath}.` + ).toHaveLength(0); + }, 3_600_000); +});