From 9a83bc492f8ccfe079f2c9f36dcdfabbd28dab80 Mon Sep 17 00:00:00 2001 From: Roman Sinyakov Date: Fri, 24 Jul 2026 09:31:05 +0200 Subject: [PATCH 1/4] feat(proxy): add VS Code model catalog --- package.json | 1 + .../commands/proxy/__tests__/index.test.ts | 7 +- .../proxy/connectors/__tests__/vscode.test.ts | 162 ++++++--- .../proxy/connectors/vscode-models.ts | 316 ++++++++++++++++++ src/cli/commands/proxy/connectors/vscode.ts | 122 ++++--- src/cli/commands/proxy/index.ts | 24 +- ...ansparent-vscode-proxy.integration.test.ts | 115 ++++--- tests/integration/vscode-byok.test.ts | 197 ++++++----- tests/integration/vscode-models.live.test.ts | 230 +++++++++++++ 9 files changed, 913 insertions(+), 261 deletions(-) create mode 100644 src/cli/commands/proxy/connectors/vscode-models.ts create mode 100644 tests/integration/vscode-models.live.test.ts diff --git a/package.json b/package.json index 6a43458d..db9a4ebc 100644 --- a/package.json +++ b/package.json @@ -37,6 +37,7 @@ "test:unit": "vitest run --project unit", "test:integration": "vitest run --project cli", "test:integration:cli": "vitest run --project cli", + "test:integration:vscode-models": "vitest run --project cli tests/integration/vscode-byok.test.ts tests/integration/vscode-models.live.test.ts", "test:integration:agent": "vitest run --project agent", "test:run": "vitest run --project unit --project cli", "test:all": "vitest run --project unit && vitest run --project cli && vitest run --project agent", diff --git a/src/cli/commands/proxy/__tests__/index.test.ts b/src/cli/commands/proxy/__tests__/index.test.ts index 40bd8e66..88d8f679 100644 --- a/src/cli/commands/proxy/__tests__/index.test.ts +++ b/src/cli/commands/proxy/__tests__/index.test.ts @@ -323,7 +323,7 @@ describe('proxy connect vscode', () => { exitSpy.mockRestore(); }); - it('uses an explicit profile for daemon context and writes its model into VS Code', async () => { + it('uses an explicit profile without requiring its model for VS Code', async () => { const { ConfigLoader } = await import('../../../../utils/config.js'); const { checkStatus, spawnDaemon } = await import('../daemon-manager.js'); const { writeVsCodeLanguageModelsConfig } = await import('../connectors/vscode.js'); @@ -332,7 +332,6 @@ describe('proxy connect vscode', () => { name: 'custom', provider: 'ai-run-sso', baseUrl: 'https://custom.example.com/api', - model: 'custom-profile-model', codeMieProject: 'team-project', codeMieUrl: 'https://custom.example.com', } as Awaited>); @@ -365,12 +364,11 @@ describe('proxy connect vscode', () => { expect(daemonOptions).not.toHaveProperty('model'); expect(writeVsCodeLanguageModelsConfig).toHaveBeenCalledWith( 'http://127.0.0.1:4001', - 'custom-profile-model', false ); }); - it('reuses a matching daemon when only the profile model changes', async () => { + it('reuses a matching daemon independently of the profile model', async () => { const { ConfigLoader } = await import('../../../../utils/config.js'); const { checkStatus, spawnDaemon, stopDaemon } = await import('../daemon-manager.js'); const { checkProxyHealth } = await import('../health-check.js'); @@ -407,7 +405,6 @@ describe('proxy connect vscode', () => { expect(spawnDaemon).not.toHaveBeenCalled(); expect(writeVsCodeLanguageModelsConfig).toHaveBeenCalledWith( 'http://127.0.0.1:4001', - 'new-profile-model', false ); }); diff --git a/src/cli/commands/proxy/connectors/__tests__/vscode.test.ts b/src/cli/commands/proxy/connectors/__tests__/vscode.test.ts index 1c16a047..1d852577 100644 --- a/src/cli/commands/proxy/connectors/__tests__/vscode.test.ts +++ b/src/cli/commands/proxy/connectors/__tests__/vscode.test.ts @@ -7,8 +7,45 @@ import { afterEach, beforeEach, describe, expect, it } from 'vitest'; import { mkdir, mkdtemp, readFile, rm, writeFile } from 'node:fs/promises'; import { tmpdir } from 'node:os'; import { join } from 'node:path'; +import { VS_CODE_SUPPORTED_MODELS, type VsCodeApiType } from '../vscode-models.js'; import { writeVsCodeLanguageModelsConfigAtPath } from '../vscode.js'; +const EXPECTED_MODEL_IDS = [ + 'claude-sonnet-4-5-20250929', + 'gpt-4.1', + 'gpt-4.1-mini', + 'gpt-5-2025-08-07', + 'gpt-5-mini-2025-08-07', + 'gpt-5-nano-2025-08-07', + 'gpt-5-1-codex-2025-11-13', + 'gpt-5-2-2025-12-11', + 'gpt-5.4-2026-03-05', + 'gpt-5.5-2026-04-24', + 'gpt-5.6-luna-2026-07-09', + 'gpt-5.6-sol-2026-07-09', + 'gpt-5.6-terra-2026-07-09', + 'gemini-3-flash', + 'gemini-3.1-pro', + 'gemini-3.5-flash', + 'claude-4-5-sonnet', + 'claude-sonnet-4-6', + 'claude-sonnet-5', + 'claude-opus-4-5-20251101', + 'claude-opus-4-6-20260205', + 'claude-opus-4-7', + 'claude-opus-4-8', + 'claude-haiku-4-5-20251001', + 'qwen.qwen3-coder-30b-a3b-v1', + 'qwen.qwen3-coder-480b-a35b-v1', + 'moonshotai.kimi-k2.5', +] as const; + +function getApiPath(apiType: VsCodeApiType): string { + if (apiType === 'responses') return '/v1/responses'; + if (apiType === 'messages') return '/v1/messages'; + return '/v1/chat/completions'; +} + describe('writeVsCodeLanguageModelsConfigAtPath', () => { let testDir: string; let configPath: string; @@ -27,26 +64,60 @@ describe('writeVsCodeLanguageModelsConfigAtPath', () => { return JSON.parse(await readFile(configPath, 'utf-8')) as Array>; } - it('writes the selected profile model as the model ID', async () => { + it('writes the exact supported model allowlist under the CodeMie provider', async () => { const result = await writeVsCodeLanguageModelsConfigAtPath( configPath, - 'http://127.0.0.1:4001', - 'gpt-profile-model' + 'http://127.0.0.1:4001' ); const providers = await readProviders(); - const models = providers[0].models as Array>; + const provider = providers[0]; + const models = provider.models as Array>; + expect(result).toEqual({ configPath, requiresSecretConfiguration: true }); - expect(models).toEqual([ - expect.objectContaining({ - id: 'gpt-profile-model', - name: 'CodeMie Profile Model', - url: 'http://127.0.0.1:4001/v1/chat/completions', - }), - ]); + expect(provider).toMatchObject({ + name: 'CodeMie', + vendor: 'customendpoint', + apiType: 'chat-completions', + }); + expect(models.map(model => model.id)).toEqual(EXPECTED_MODEL_IDS); + expect(models.map(model => model.name)).toEqual(EXPECTED_MODEL_IDS); + }); + + it('renders every catalog capability and endpoint without internal metadata', async () => { + await writeVsCodeLanguageModelsConfigAtPath(configPath, 'http://127.0.0.1:4001'); + + const providers = await readProviders(); + const models = providers[0].models as Array>; + + for (const definition of VS_CODE_SUPPORTED_MODELS) { + const model = models.find(candidate => candidate.id === definition.id); + const expected: Record = { + id: definition.id, + name: definition.id, + url: `http://127.0.0.1:4001${getApiPath(definition.apiType)}`, + apiType: definition.apiType, + toolCalling: true, + vision: definition.vision, + streaming: true, + thinking: definition.thinking, + maxInputTokens: definition.maxInputTokens, + maxOutputTokens: definition.maxOutputTokens, + }; + if (definition.adaptiveThinking) expected.adaptiveThinking = true; + if (definition.supportsReasoningEffort) { + expected.supportsReasoningEffort = definition.supportsReasoningEffort; + } + if (definition.reasoningEffortFormat) { + expected.reasoningEffortFormat = definition.reasoningEffortFormat; + } + + expect(model).toEqual(expected); + expect(model).not.toHaveProperty('releaseSlice'); + } }); - it('updates the managed model and preserves unrelated configuration', async () => { + it('replaces legacy CodeMie models while preserving the secret and saved settings', async () => { const secretReference = '${input:chat.lm.secret.codemie}'; await writeFile(configPath, JSON.stringify([ { @@ -57,11 +128,12 @@ describe('writeVsCodeLanguageModelsConfigAtPath', () => { { name: 'CodeMie', vendor: 'customendpoint', - apiType: 'chat-completions', + apiType: 'messages', apiKey: secretReference, customProperty: 'preserved', settings: { 'old-profile-model': { reasoningEffort: 'medium' }, + 'gpt-5.5-2026-04-24': { reasoningEffort: 'high' }, 'custom-setting': { enabled: true }, }, models: [ @@ -73,50 +145,51 @@ describe('writeVsCodeLanguageModelsConfigAtPath', () => { const result = await writeVsCodeLanguageModelsConfigAtPath( configPath, - 'http://127.0.0.1:4010', - 'claude-profile-model' + 'http://127.0.0.1:4010' ); const providers = await readProviders(); - const codemie = providers[1]; + const codeMie = providers[1]; + const models = codeMie.models as Array>; + expect(result.requiresSecretConfiguration).toBe(false); - expect(providers[0]).toMatchObject({ name: 'Other' }); - expect(codemie).toMatchObject({ + expect(providers[0]).toEqual({ + name: 'Other', + vendor: 'customendpoint', + models: [{ id: 'other-model', name: 'Other model' }], + }); + expect(codeMie).toMatchObject({ name: 'CodeMie', + vendor: 'customendpoint', + apiType: 'chat-completions', apiKey: secretReference, customProperty: 'preserved', - settings: { 'custom-setting': { enabled: true } }, + settings: { + 'gpt-5.5-2026-04-24': { reasoningEffort: 'high' }, + 'custom-setting': { enabled: true }, + }, }); - expect(codemie.models).toEqual([ - expect.objectContaining({ id: 'claude-profile-model', name: 'CodeMie Profile Model' }), - { id: 'user-managed-model', name: 'User model', custom: true }, - ]); + expect(codeMie.settings).not.toHaveProperty('old-profile-model'); + expect(models.map(model => model.id)).toEqual(EXPECTED_MODEL_IDS); + expect(models.some(model => model.id === 'user-managed-model')).toBe(false); }); - it('replaces the previous profile model when the selected profile changes', async () => { - await writeVsCodeLanguageModelsConfigAtPath( - configPath, - 'http://127.0.0.1:4001', - 'first-profile-model' - ); + it('updates every managed endpoint without changing saved model settings', async () => { + await writeVsCodeLanguageModelsConfigAtPath(configPath, 'http://127.0.0.1:4001'); const firstProviders = await readProviders(); firstProviders[0].settings = { - 'first-profile-model': { reasoningEffort: 'high' }, - unrelated: { enabled: true }, + 'claude-opus-4-8': { reasoningEffort: 'xhigh' }, }; await writeFile(configPath, JSON.stringify(firstProviders, null, 2), 'utf-8'); - await writeVsCodeLanguageModelsConfigAtPath( - configPath, - 'http://127.0.0.1:4001', - 'second-profile-model' - ); + await writeVsCodeLanguageModelsConfigAtPath(configPath, 'http://127.0.0.1:4010'); const providers = await readProviders(); const models = providers[0].models as Array>; - expect(models).toHaveLength(1); - expect(models[0].id).toBe('second-profile-model'); - expect(providers[0].settings).toEqual({ unrelated: { enabled: true } }); + expect(models.every(model => String(model.url).startsWith('http://127.0.0.1:4010/'))).toBe(true); + expect(providers[0].settings).toEqual({ + 'claude-opus-4-8': { reasoningEffort: 'xhigh' }, + }); }); it.each([ @@ -127,18 +200,9 @@ describe('writeVsCodeLanguageModelsConfigAtPath', () => { await expect(writeVsCodeLanguageModelsConfigAtPath( configPath, - 'http://127.0.0.1:4001', - 'profile-model' + 'http://127.0.0.1:4001' )).rejects.toThrow(); expect(await readFile(configPath, 'utf-8')).toBe(original); }); - - it('rejects an empty profile model before writing configuration', async () => { - await expect(writeVsCodeLanguageModelsConfigAtPath( - configPath, - 'http://127.0.0.1:4001', - ' ' - )).rejects.toThrow('requires a profile model'); - }); }); diff --git a/src/cli/commands/proxy/connectors/vscode-models.ts b/src/cli/commands/proxy/connectors/vscode-models.ts new file mode 100644 index 00000000..7aee0e1f --- /dev/null +++ b/src/cli/commands/proxy/connectors/vscode-models.ts @@ -0,0 +1,316 @@ +export type VsCodeApiType = 'chat-completions' | 'responses' | 'messages'; + +export type VsCodeReasoningEffort = + | 'none' + | 'minimal' + | 'low' + | 'medium' + | 'high' + | 'xhigh' + | 'max'; + +export interface VsCodeModelDefinition { + releaseSlice: 1 | 2 | 3; + id: string; + apiType: VsCodeApiType; + vision: boolean; + thinking: boolean; + adaptiveThinking?: true; + supportsReasoningEffort?: readonly VsCodeReasoningEffort[]; + reasoningEffortFormat?: 'chat-completions' | 'responses'; + maxInputTokens: number; + maxOutputTokens: number; +} + +const GPT_5_EFFORTS = ['minimal', 'low', 'medium', 'high'] as const; +const GPT_5_1_EFFORTS = ['none', 'low', 'medium', 'high'] as const; +const GPT_5_XHIGH_EFFORTS = ['none', 'low', 'medium', 'high', 'xhigh'] as const; +const GPT_5_6_EFFORTS = ['none', 'low', 'medium', 'high', 'xhigh', 'max'] as const; +const GEMINI_FLASH_EFFORTS = ['minimal', 'low', 'medium', 'high'] as const; +const GEMINI_PRO_EFFORTS = ['low', 'medium', 'high'] as const; +const CLAUDE_EFFORTS = ['low', 'medium', 'high'] as const; +const CLAUDE_MAX_EFFORTS = ['low', 'medium', 'high', 'max'] as const; +const CLAUDE_XHIGH_EFFORTS = ['low', 'medium', 'high', 'xhigh', 'max'] as const; + +export const VS_CODE_SUPPORTED_MODELS: readonly VsCodeModelDefinition[] = [ + { + releaseSlice: 3, + id: 'claude-sonnet-4-5-20250929', + apiType: 'chat-completions', + vision: true, + thinking: false, + maxInputTokens: 136000, + maxOutputTokens: 64000, + }, + { + releaseSlice: 1, + id: 'gpt-4.1', + apiType: 'chat-completions', + vision: true, + thinking: false, + maxInputTokens: 1014808, + maxOutputTokens: 32768, + }, + { + releaseSlice: 1, + id: 'gpt-4.1-mini', + apiType: 'chat-completions', + vision: true, + thinking: false, + maxInputTokens: 1014808, + maxOutputTokens: 32768, + }, + { + releaseSlice: 1, + id: 'gpt-5-2025-08-07', + apiType: 'chat-completions', + vision: true, + thinking: true, + supportsReasoningEffort: GPT_5_EFFORTS, + reasoningEffortFormat: 'chat-completions', + maxInputTokens: 272000, + maxOutputTokens: 128000, + }, + { + releaseSlice: 1, + id: 'gpt-5-mini-2025-08-07', + apiType: 'chat-completions', + vision: true, + thinking: true, + supportsReasoningEffort: GPT_5_EFFORTS, + reasoningEffortFormat: 'chat-completions', + maxInputTokens: 272000, + maxOutputTokens: 128000, + }, + { + releaseSlice: 1, + id: 'gpt-5-nano-2025-08-07', + apiType: 'chat-completions', + vision: true, + thinking: true, + supportsReasoningEffort: GPT_5_EFFORTS, + reasoningEffortFormat: 'chat-completions', + maxInputTokens: 272000, + maxOutputTokens: 128000, + }, + { + releaseSlice: 2, + id: 'gpt-5-1-codex-2025-11-13', + apiType: 'responses', + vision: true, + thinking: true, + supportsReasoningEffort: GPT_5_1_EFFORTS, + reasoningEffortFormat: 'responses', + maxInputTokens: 272000, + maxOutputTokens: 128000, + }, + { + releaseSlice: 2, + id: 'gpt-5-2-2025-12-11', + apiType: 'chat-completions', + vision: true, + thinking: true, + supportsReasoningEffort: GPT_5_1_EFFORTS, + reasoningEffortFormat: 'chat-completions', + maxInputTokens: 272000, + maxOutputTokens: 128000, + }, + { + releaseSlice: 2, + id: 'gpt-5.4-2026-03-05', + apiType: 'chat-completions', + vision: true, + thinking: true, + supportsReasoningEffort: GPT_5_XHIGH_EFFORTS, + reasoningEffortFormat: 'chat-completions', + maxInputTokens: 922000, + maxOutputTokens: 128000, + }, + { + releaseSlice: 2, + id: 'gpt-5.5-2026-04-24', + apiType: 'responses', + vision: true, + thinking: true, + supportsReasoningEffort: GPT_5_XHIGH_EFFORTS, + reasoningEffortFormat: 'responses', + maxInputTokens: 922000, + maxOutputTokens: 128000, + }, + { + releaseSlice: 2, + id: 'gpt-5.6-luna-2026-07-09', + apiType: 'responses', + vision: true, + thinking: true, + supportsReasoningEffort: GPT_5_6_EFFORTS, + reasoningEffortFormat: 'responses', + maxInputTokens: 922000, + maxOutputTokens: 128000, + }, + { + releaseSlice: 2, + id: 'gpt-5.6-sol-2026-07-09', + apiType: 'responses', + vision: true, + thinking: true, + supportsReasoningEffort: GPT_5_6_EFFORTS, + reasoningEffortFormat: 'responses', + maxInputTokens: 922000, + maxOutputTokens: 128000, + }, + { + releaseSlice: 2, + id: 'gpt-5.6-terra-2026-07-09', + apiType: 'responses', + vision: true, + thinking: true, + supportsReasoningEffort: GPT_5_6_EFFORTS, + reasoningEffortFormat: 'responses', + maxInputTokens: 922000, + maxOutputTokens: 128000, + }, + { + releaseSlice: 3, + id: 'gemini-3-flash', + apiType: 'chat-completions', + vision: true, + thinking: true, + supportsReasoningEffort: GEMINI_FLASH_EFFORTS, + reasoningEffortFormat: 'chat-completions', + maxInputTokens: 983040, + maxOutputTokens: 65536, + }, + { + releaseSlice: 3, + id: 'gemini-3.1-pro', + apiType: 'chat-completions', + vision: true, + thinking: true, + supportsReasoningEffort: GEMINI_PRO_EFFORTS, + reasoningEffortFormat: 'chat-completions', + maxInputTokens: 983040, + maxOutputTokens: 65536, + }, + { + releaseSlice: 3, + id: 'gemini-3.5-flash', + apiType: 'chat-completions', + vision: true, + thinking: true, + supportsReasoningEffort: GEMINI_FLASH_EFFORTS, + reasoningEffortFormat: 'chat-completions', + maxInputTokens: 983040, + maxOutputTokens: 65536, + }, + { + releaseSlice: 3, + id: 'claude-4-5-sonnet', + apiType: 'chat-completions', + vision: true, + thinking: false, + maxInputTokens: 136000, + maxOutputTokens: 64000, + }, + { + releaseSlice: 3, + id: 'claude-sonnet-4-6', + apiType: 'messages', + vision: true, + thinking: true, + adaptiveThinking: true, + supportsReasoningEffort: CLAUDE_MAX_EFFORTS, + maxInputTokens: 936000, + maxOutputTokens: 64000, + }, + { + releaseSlice: 3, + id: 'claude-sonnet-5', + apiType: 'messages', + vision: true, + thinking: true, + adaptiveThinking: true, + supportsReasoningEffort: CLAUDE_XHIGH_EFFORTS, + maxInputTokens: 872000, + maxOutputTokens: 128000, + }, + { + releaseSlice: 3, + id: 'claude-opus-4-5-20251101', + apiType: 'messages', + vision: true, + thinking: false, + supportsReasoningEffort: CLAUDE_EFFORTS, + maxInputTokens: 136000, + maxOutputTokens: 64000, + }, + { + releaseSlice: 3, + id: 'claude-opus-4-6-20260205', + apiType: 'messages', + vision: true, + thinking: true, + adaptiveThinking: true, + supportsReasoningEffort: CLAUDE_MAX_EFFORTS, + maxInputTokens: 872000, + maxOutputTokens: 128000, + }, + { + releaseSlice: 3, + id: 'claude-opus-4-7', + apiType: 'messages', + vision: true, + thinking: true, + adaptiveThinking: true, + supportsReasoningEffort: CLAUDE_XHIGH_EFFORTS, + maxInputTokens: 872000, + maxOutputTokens: 128000, + }, + { + releaseSlice: 3, + id: 'claude-opus-4-8', + apiType: 'messages', + vision: true, + thinking: true, + adaptiveThinking: true, + supportsReasoningEffort: CLAUDE_XHIGH_EFFORTS, + maxInputTokens: 872000, + maxOutputTokens: 128000, + }, + { + releaseSlice: 3, + id: 'claude-haiku-4-5-20251001', + apiType: 'chat-completions', + vision: true, + thinking: false, + maxInputTokens: 136000, + maxOutputTokens: 64000, + }, + { + releaseSlice: 1, + id: 'qwen.qwen3-coder-30b-a3b-v1', + apiType: 'chat-completions', + vision: false, + thinking: false, + maxInputTokens: 245760, + maxOutputTokens: 16384, + }, + { + releaseSlice: 1, + id: 'qwen.qwen3-coder-480b-a35b-v1', + apiType: 'chat-completions', + vision: false, + thinking: false, + maxInputTokens: 114688, + maxOutputTokens: 16384, + }, + { + releaseSlice: 1, + id: 'moonshotai.kimi-k2.5', + apiType: 'chat-completions', + vision: true, + thinking: false, + maxInputTokens: 245760, + maxOutputTokens: 16384, + }, +]; diff --git a/src/cli/commands/proxy/connectors/vscode.ts b/src/cli/commands/proxy/connectors/vscode.ts index 48e50b4c..c0c82c21 100644 --- a/src/cli/commands/proxy/connectors/vscode.ts +++ b/src/cli/commands/proxy/connectors/vscode.ts @@ -3,8 +3,13 @@ import { mkdir, readFile, rename, stat, unlink, writeFile } from 'node:fs/promis import { homedir } from 'node:os'; import { dirname, join } from 'node:path'; import { ConfigurationError } from '@/utils/errors.js'; +import { + VS_CODE_SUPPORTED_MODELS, + type VsCodeApiType, + type VsCodeReasoningEffort, +} from './vscode-models.js'; -const MANAGED_MODEL_NAME = 'CodeMie Profile Model'; +const LEGACY_MANAGED_MODEL_NAME = 'CodeMie Profile Model'; const SECRET_REFERENCE_PATTERN = /^\$\{input:chat\.lm\.secret\.[^}]+\}$/; interface VsCodeLanguageModelProvider { @@ -21,14 +26,16 @@ interface VsCodeManagedModel { id: string; name: string; url: string; + apiType: VsCodeApiType; toolCalling: true; - vision: true; + vision: boolean; streaming: true; - thinking: true; - supportsReasoningEffort: readonly ['minimal', 'low', 'medium', 'high']; - reasoningEffortFormat: 'chat-completions'; - maxInputTokens: 224000; - maxOutputTokens: 32000; + thinking: boolean; + adaptiveThinking?: true; + supportsReasoningEffort?: readonly VsCodeReasoningEffort[]; + reasoningEffortFormat?: 'chat-completions' | 'responses'; + maxInputTokens: number; + maxOutputTokens: number; } export interface WriteVsCodeConfigResult { @@ -40,18 +47,18 @@ function isRecord(value: unknown): value is Record { return typeof value === 'object' && value !== null && !Array.isArray(value); } -function isManagedModel(model: unknown, currentModelId?: string): boolean { +function isLegacyManagedModel(model: unknown): boolean { if (!isRecord(model)) return false; - return model.name === MANAGED_MODEL_NAME || - (currentModelId !== undefined && model.id === currentModelId); + return model.name === LEGACY_MANAGED_MODEL_NAME; } function isManagedProvider(provider: unknown): provider is VsCodeLanguageModelProvider { if (!isRecord(provider)) return false; - const hasManagedModel = Array.isArray(provider.models) && - provider.models.some(model => isManagedModel(model)); - return hasManagedModel || (provider.vendor === 'customendpoint' && provider.name === 'CodeMie'); + const hasLegacyManagedModel = Array.isArray(provider.models) && + provider.models.some(model => isLegacyManagedModel(model)); + return hasLegacyManagedModel || + (provider.vendor === 'customendpoint' && provider.name === 'CodeMie'); } export function isVsCodeSecretReference(value: unknown): value is string { @@ -94,46 +101,42 @@ export function getVsCodeLanguageModelsPath(insiders = false): string { return join(productDir, 'User', 'chatLanguageModels.json'); } -function buildManagedModel(proxyUrl: string, profileModel: string): VsCodeManagedModel { - return { - id: profileModel, - name: MANAGED_MODEL_NAME, - url: new URL('/v1/chat/completions', proxyUrl).toString(), - toolCalling: true, - vision: true, - streaming: true, - thinking: true, - supportsReasoningEffort: ['minimal', 'low', 'medium', 'high'], - reasoningEffortFormat: 'chat-completions', - maxInputTokens: 224000, - maxOutputTokens: 32000, - }; +function getApiPath(apiType: VsCodeApiType): string { + if (apiType === 'responses') return '/v1/responses'; + if (apiType === 'messages') return '/v1/messages'; + return '/v1/chat/completions'; } -function reconcileModels(existingModels: unknown, managedModel: VsCodeManagedModel): unknown[] { - const models = Array.isArray(existingModels) ? existingModels : []; - const reconciled: unknown[] = []; - let inserted = false; - - for (const model of models) { - if (isManagedModel(model, managedModel.id)) { - if (!inserted) { - reconciled.push(managedModel); - inserted = true; - } - continue; +function buildManagedModels(proxyUrl: string): VsCodeManagedModel[] { + return VS_CODE_SUPPORTED_MODELS.map(definition => { + const model: VsCodeManagedModel = { + id: definition.id, + name: definition.id, + url: new URL(getApiPath(definition.apiType), proxyUrl).toString(), + apiType: definition.apiType, + toolCalling: true, + vision: definition.vision, + streaming: true, + thinking: definition.thinking, + maxInputTokens: definition.maxInputTokens, + maxOutputTokens: definition.maxOutputTokens, + }; + + if (definition.adaptiveThinking) model.adaptiveThinking = true; + if (definition.supportsReasoningEffort) { + model.supportsReasoningEffort = definition.supportsReasoningEffort; + } + if (definition.reasoningEffortFormat) { + model.reasoningEffortFormat = definition.reasoningEffortFormat; } - reconciled.push(model); - } - if (!inserted) reconciled.push(managedModel); - return reconciled; + return model; + }); } function mergeManagedProviders( providers: VsCodeLanguageModelProvider[], - proxyUrl: string, - profileModel: string + proxyUrl: string ): { provider: VsCodeLanguageModelProvider; requiresSecretConfiguration: boolean } { const existingProvider = Object.assign({}, ...providers); const existingModels = providers.flatMap(provider => @@ -143,14 +146,11 @@ function mergeManagedProviders( {}, ...providers.map(provider => isRecord(provider.settings) ? provider.settings : {}) ); - const previousManagedModelIds = existingModels - .filter(model => isManagedModel(model)) + const legacyManagedModelIds = existingModels + .filter(model => isLegacyManagedModel(model)) .map(model => isRecord(model) ? model.id : undefined) .filter((id): id is string => typeof id === 'string'); - for (const modelId of new Set([ - profileModel, - ...previousManagedModelIds, - ])) { + for (const modelId of new Set(legacyManagedModelIds)) { delete existingSettings[modelId]; } const existingSecretReference = providers @@ -162,12 +162,10 @@ function mergeManagedProviders( name: 'CodeMie', vendor: 'customendpoint', apiType: 'chat-completions', - models: reconcileModels(existingModels, buildManagedModel(proxyUrl, profileModel)), + models: buildManagedModels(proxyUrl), }; - // VS Code owns model configuration state and derives "medium" as the default for this - // non-Claude model. Writing the same setting here races with VS Code's editor and can - // produce duplicate `settings` keys in its unsaved buffer. + // VS Code owns effort selections. Preserve them instead of racing with the editor. if (Object.keys(existingSettings).length > 0) provider.settings = existingSettings; else delete provider.settings; @@ -235,26 +233,18 @@ async function writeAtomically(configPath: string, content: string): Promise { return writeVsCodeLanguageModelsConfigAtPath( getVsCodeLanguageModelsPath(insiders), - proxyUrl, - profileModel + proxyUrl ); } export async function writeVsCodeLanguageModelsConfigAtPath( configPath: string, - proxyUrl: string, - profileModel: string + proxyUrl: string ): Promise { - const normalizedProfileModel = profileModel.trim(); - if (!normalizedProfileModel) { - throw new ConfigurationError('VS Code model configuration requires a profile model.'); - } - const providers = await readProviders(configPath); const managedProviderIndexes = providers .map((provider, index) => isManagedProvider(provider) ? index : -1) @@ -263,7 +253,7 @@ export async function writeVsCodeLanguageModelsConfigAtPath( .map(index => providers[index]) .filter(isManagedProvider); const { provider: managedProvider, requiresSecretConfiguration } = - mergeManagedProviders(managedProviders, proxyUrl, normalizedProfileModel); + mergeManagedProviders(managedProviders, proxyUrl); const firstManagedProviderIndex = managedProviderIndexes[0] ?? providers.length; const managedProviderIndexSet = new Set(managedProviderIndexes); const reconciledProviders = providers.flatMap((provider, index) => { diff --git a/src/cli/commands/proxy/index.ts b/src/cli/commands/proxy/index.ts index de1c97a6..08e7098d 100644 --- a/src/cli/commands/proxy/index.ts +++ b/src/cli/commands/proxy/index.ts @@ -21,6 +21,7 @@ import { import { writeDesktopConfig, getDesktopBaseDir, mapCanonicalToDesktop } from './connectors/desktop.js'; import { fetchManagedMcpServers } from './connectors/managed-mcp-remote.js'; import { writeVsCodeLanguageModelsConfig } from './connectors/vscode.js'; +import { VS_CODE_SUPPORTED_MODELS } from './connectors/vscode-models.js'; import { checkProxyHealth } from './health-check.js'; import { printDesktopInspection } from './inspect-desktop.js'; @@ -509,7 +510,7 @@ export function createProxyCommand(): Command { connect .command('vscode') .description('Configure VS Code BYOK to use the local proxy') - .option('--profile ', 'Profile whose credentials and model to use') + .option('--profile ', 'Profile whose URL, credentials, and project to use') .option('--insiders', 'Configure VS Code Insiders instead of stable VS Code') .option('--verbose', 'Show detailed connection info (URLs, config paths) for debugging') .option('--force', 'Stop any existing proxy and start a fresh one, even if it looks healthy') @@ -528,12 +529,6 @@ export function createProxyCommand(): Command { } const profile = config.name ?? 'default'; - const profileModel = config.model?.trim(); - if (!profileModel) { - throw new ConfigurationError( - `Profile "${profile}" has no model configured.\nRun: codemie setup` - ); - } if (verbose) { console.log( @@ -608,7 +603,6 @@ export function createProxyCommand(): Command { const result = await writeVsCodeLanguageModelsConfig( state!.url, - profileModel, Boolean(opts.insiders) ); logger.info( @@ -618,7 +612,7 @@ export function createProxyCommand(): Command { gatewayUrl: state!.url, profile: state!.profile, project: state!.project, - model: profileModel, + modelCount: VS_CODE_SUPPORTED_MODELS.length, clientType: state!.clientType, requiresSecretConfiguration: result.requiresSecretConfiguration, }) @@ -628,7 +622,7 @@ export function createProxyCommand(): Command { if (verbose) { console.log(` Config: ${result.configPath}`); console.log(` Gateway: ${state!.url}`); - console.log(` Model: ${profileModel}`); + console.log(` Models: ${VS_CODE_SUPPORTED_MODELS.length}`); console.log(` Project: ${config.codeMieProject || '(not configured)'}`); } @@ -636,18 +630,18 @@ export function createProxyCommand(): Command { displaySetupInstructions({ setupInstructions: [ 'One-time VS Code secret setup required:\n', - '1. Press ⇧⌘P (macOS) or Ctrl+Shift+P (Windows/Linux).', - '2. Run: Chat: Manage Language Models', - '3. Right-click CodeMie Profile Model → Update API Key', + '1. Open VS Code and Press ⇧⌘P (macOS) or Ctrl+Shift+P (Windows/Linux).', + '2. Find Chat: Manage Language Models', + '3. In opened dialog Right-click any CodeMie model → Update API Key', `4. Enter API key: ${state!.gatewayKey}\n`, - 'Reload VS Code to apply changes.', + 'Reload VS Code, then select a CodeMie model from the model picker.', ].join('\n'), }); } else { console.log( chalk.dim( ` If VS Code reports a missing or invalid key, open Chat: Manage Language Models, ` + - `then right-click CodeMie Profile Model → Update API Key and enter ${state!.gatewayKey}.` + `then right-click any CodeMie model → Update API Key and enter ${state!.gatewayKey}.` ) ); } diff --git a/src/providers/plugins/sso/proxy/__tests__/transparent-vscode-proxy.integration.test.ts b/src/providers/plugins/sso/proxy/__tests__/transparent-vscode-proxy.integration.test.ts index c6209bae..c685dbe6 100644 --- a/src/providers/plugins/sso/proxy/__tests__/transparent-vscode-proxy.integration.test.ts +++ b/src/providers/plugins/sso/proxy/__tests__/transparent-vscode-proxy.integration.test.ts @@ -112,7 +112,7 @@ describe('transparent VS Code proxy integration', () => { resetPluginRegistry(); }); - it.each(['/v1/responses', '/v1/chat/completions?trace=test'])( + it.each(['/v1/responses', '/v1/messages', '/v1/chat/completions?trace=test'])( 'forwards the caller-selected model and request body unchanged for %s', async (path) => { const captured: CapturedRequest[] = []; const upstream = await startCapturingUpstream(captured); @@ -120,7 +120,7 @@ describe('transparent VS Code proxy integration', () => { const startedProxy = await startProxy(upstream.url); proxies.push(startedProxy.proxy); const body = { - model: 'profile-selected-model', + model: 'caller-selected-model', messages: [{ role: 'user', content: 'Hello' }], tools: [{ type: 'function', name: 'read_file' }], stream: false, @@ -138,53 +138,54 @@ describe('transparent VS Code proxy integration', () => { } ); - it('forwards SSE bytes incrementally and unchanged', async () => { - const chunks = [ - 'data: {"id":"1","choices":[{"delta":{"content":"A"}}]}\n\n', - 'data: {"id":"1","choices":[{"delta":{"content":"B"}}]}\n\n', - 'data: [DONE]\n\n', - ]; - let upstreamClosed = false; - const upstream = await listen(createServer((req, res) => { - void readRequestBody(req).then(() => { - res.statusCode = 200; - res.setHeader('content-type', 'text/event-stream'); - res.write(chunks[0]); - setTimeout(() => { - res.write(chunks[1]); - res.end(chunks[2], () => { upstreamClosed = true; }); - }, 30); - }); - })); - servers.push(upstream.server); - const startedProxy = await startProxy(upstream.url); - proxies.push(startedProxy.proxy); + it.each(['/v1/chat/completions', '/v1/responses', '/v1/messages'])( + 'forwards SSE bytes incrementally and unchanged for %s', async (path) => { + const chunks = [ + 'data: {"id":"1","choices":[{"delta":{"content":"A"}}]}\n\n', + 'data: {"id":"1","choices":[{"delta":{"content":"B"}}]}\n\n', + 'data: [DONE]\n\n', + ]; + let upstreamClosed = false; + const upstream = await listen(createServer((req, res) => { + void readRequestBody(req).then(() => { + res.statusCode = 200; + res.setHeader('content-type', 'text/event-stream'); + res.write(chunks[0]); + setTimeout(() => { + res.write(chunks[1]); + res.end(chunks[2], () => { upstreamClosed = true; }); + }, 30); + }); + })); + servers.push(upstream.server); + const startedProxy = await startProxy(upstream.url); + proxies.push(startedProxy.proxy); - const response = await fetch( - `${startedProxy.url}/v1/chat/completions`, - authenticatedJsonInit({ model: 'profile-selected-model', messages: [], stream: true }) - ); - const reader = response.body!.getReader(); - const first = await reader.read(); - - expect(upstreamClosed).toBe(false); - const received = [Buffer.from(first.value ?? []).toString('utf-8')]; - while (true) { - const next = await reader.read(); - if (next.done) break; - received.push(Buffer.from(next.value).toString('utf-8')); - } + const response = await fetch( + `${startedProxy.url}${path}`, + authenticatedJsonInit({ model: 'caller-selected-model', messages: [], stream: true }) + ); + const reader = response.body!.getReader(); + const first = await reader.read(); - expect(response.headers.get('content-type')).toContain('text/event-stream'); - expect(received.join('')).toBe(chunks.join('')); - }); + expect(upstreamClosed).toBe(false); + const received = [Buffer.from(first.value ?? []).toString('utf-8')]; + while (true) { + const next = await reader.read(); + if (next.done) break; + received.push(Buffer.from(next.value).toString('utf-8')); + } - it('forwards tool-call response fields unchanged', async () => { - const toolCall = { + expect(response.headers.get('content-type')).toContain('text/event-stream'); + expect(received.join('')).toBe(chunks.join('')); + } + ); + + it.each([ + ['/v1/chat/completions', { id: 'completion-1', choices: [{ - index: 0, - delta: { + message: { tool_calls: [{ id: 'call-1', type: 'function', @@ -193,7 +194,27 @@ describe('transparent VS Code proxy integration', () => { }, finish_reason: 'tool_calls', }], - }; + }], + ['/v1/responses', { + id: 'response-1', + output: [{ + id: 'call-1', + type: 'function_call', + name: 'get_test_value', + arguments: '{"name":"value"}', + }], + }], + ['/v1/messages', { + id: 'message-1', + content: [{ + id: 'call-1', + type: 'tool_use', + name: 'get_test_value', + input: { name: 'value' }, + }], + stop_reason: 'tool_use', + }], + ])('forwards tool-call response fields unchanged for %s', async (path, toolCall) => { const upstream = await listen(createServer((req, res) => { void readRequestBody(req).then(() => { res.setHeader('content-type', 'application/json'); @@ -205,8 +226,8 @@ describe('transparent VS Code proxy integration', () => { proxies.push(startedProxy.proxy); const response = await fetch( - `${startedProxy.url}/v1/chat/completions`, - authenticatedJsonInit({ model: 'profile-selected-model', messages: [] }) + `${startedProxy.url}${path}`, + authenticatedJsonInit({ model: 'caller-selected-model', messages: [] }) ); expect(await response.json()).toEqual(toolCall); diff --git a/tests/integration/vscode-byok.test.ts b/tests/integration/vscode-byok.test.ts index d88dfb0f..b95a610f 100644 --- a/tests/integration/vscode-byok.test.ts +++ b/tests/integration/vscode-byok.test.ts @@ -1,5 +1,5 @@ /** - * VS Code BYOK end-to-end integration test + * VS Code BYOK end-to-end integration tests * @group integration */ @@ -9,17 +9,26 @@ import type { AddressInfo } from 'node:net'; import { tmpdir } from 'node:os'; import { join } from 'node:path'; import { afterEach, beforeEach, describe, expect, it } from 'vitest'; -import { writeVsCodeLanguageModelsConfigAtPath } from '../../src/cli/commands/proxy/connectors/vscode.js'; +import { + VS_CODE_SUPPORTED_MODELS, + type VsCodeModelDefinition, + type VsCodeReasoningEffort, +} from '../../src/cli/commands/proxy/connectors/vscode-models.js'; +import { + writeVsCodeLanguageModelsConfigAtPath, +} from '../../src/cli/commands/proxy/connectors/vscode.js'; import { CodeMieProxy } from '../../src/providers/plugins/sso/proxy/sso.proxy.js'; import { GatewayKeyPlugin } from '../../src/providers/plugins/sso/proxy/plugins/gateway-key.plugin.js'; -import { HeaderInjectionPlugin } from '../../src/providers/plugins/sso/proxy/plugins/header-injection.plugin.js'; +import { + HeaderInjectionPlugin, +} from '../../src/providers/plugins/sso/proxy/plugins/header-injection.plugin.js'; import { getPluginRegistry, resetPluginRegistry, } from '../../src/providers/plugins/sso/proxy/plugins/registry.js'; const GATEWAY_KEY = 'test-local-key'; -const PROFILE_MODEL = 'profile-selected-model'; +const PROFILE_MODEL = 'profile-selected-model-that-must-not-be-used'; interface StartedServer { server: Server; @@ -27,14 +36,16 @@ interface StartedServer { } interface CapturedRequest { + url: string; headers: IncomingMessage['headers']; body: Record; } interface LanguageModel { - id?: unknown; - name?: unknown; - url?: unknown; + id: string; + name: string; + url: string; + apiType: string; } interface LanguageModelProvider { @@ -63,7 +74,63 @@ async function readRequestBody(req: IncomingMessage): Promise; } -describe('VS Code BYOK utility flow', () => { +function buildRequestBody( + definition: VsCodeModelDefinition, + effort: VsCodeReasoningEffort | undefined +): Record { + if (definition.apiType === 'responses') { + return { + model: definition.id, + stream: false, + input: [{ role: 'user', content: 'Call get_test_value.' }], + tools: [{ + type: 'function', + name: 'get_test_value', + description: 'Return a synthetic test value.', + parameters: { type: 'object', properties: {} }, + strict: true, + }], + tool_choice: 'required', + ...(effort ? { reasoning: { effort } } : {}), + }; + } + + if (definition.apiType === 'messages') { + return { + model: definition.id, + max_tokens: 1024, + stream: false, + messages: [{ role: 'user', content: 'Call get_test_value.' }], + tools: [{ + name: 'get_test_value', + description: 'Return a synthetic test value.', + input_schema: { type: 'object', properties: {} }, + }], + tool_choice: { type: 'any' }, + ...(definition.adaptiveThinking ? { thinking: { type: 'adaptive' } } : {}), + ...(effort ? { output_config: { effort } } : {}), + }; + } + + return { + model: definition.id, + stream: false, + messages: [{ role: 'user', content: 'Call get_test_value.' }], + tools: [{ + type: 'function', + function: { + name: 'get_test_value', + description: 'Return a synthetic test value.', + parameters: { type: 'object', properties: {} }, + strict: true, + }, + }], + tool_choice: 'required', + ...(effort ? { reasoning_effort: effort } : {}), + }; +} + +describe('VS Code BYOK model matrix', () => { const proxies: CodeMieProxy[] = []; const servers: Server[] = []; let testDir: string; @@ -81,30 +148,17 @@ describe('VS Code BYOK utility flow', () => { resetPluginRegistry(); }); - it('uses the configured profile model through the authenticated local proxy', async () => { + it('forwards every selected model and supported effort without using the profile model', async () => { const captured: CapturedRequest[] = []; - const toolResponse = { - id: 'completion-1', - choices: [{ - message: { - role: 'assistant', - tool_calls: [{ - id: 'call-1', - type: 'function', - function: { - name: 'get_test_value', - arguments: '{"name":"vscode-byok"}', - }, - }], - }, - finish_reason: 'tool_calls', - }], - }; const upstream = await listen(createServer((req, res) => { void readRequestBody(req).then((body) => { - captured.push({ headers: req.headers, body }); + captured.push({ + url: req.url ?? '/', + headers: req.headers, + body, + }); res.setHeader('content-type', 'application/json'); - res.end(JSON.stringify(toolResponse)); + res.end(JSON.stringify({ ok: true })); }); })); servers.push(upstream.server); @@ -125,64 +179,49 @@ describe('VS Code BYOK utility flow', () => { const startedProxy = await proxy.start(); const configPath = join(testDir, 'User', 'chatLanguageModels.json'); - await writeVsCodeLanguageModelsConfigAtPath( - configPath, - startedProxy.url, - PROFILE_MODEL - ); + await writeVsCodeLanguageModelsConfigAtPath(configPath, startedProxy.url); const providers = JSON.parse( await readFile(configPath, 'utf-8') ) as LanguageModelProvider[]; const codeMieProvider = providers.find( - (provider) => provider.name === 'CodeMie' && provider.vendor === 'customendpoint' - ); - const configuredModel = codeMieProvider?.models?.find( - (model) => model.name === 'CodeMie Profile Model' + provider => provider.name === 'CodeMie' && provider.vendor === 'customendpoint' ); - expect(configuredModel).toMatchObject({ - id: PROFILE_MODEL, - url: `${startedProxy.url}/v1/chat/completions`, - }); + expect(codeMieProvider?.models).toHaveLength(VS_CODE_SUPPORTED_MODELS.length); + expect(codeMieProvider?.models?.some(model => model.id === PROFILE_MODEL)).toBe(false); - const requestBody = { - model: configuredModel?.id, - stream: false, - messages: [{ - role: 'user', - content: 'Call get_test_value with name "vscode-byok". Do not answer directly.', - }], - tools: [{ - type: 'function', - function: { - name: 'get_test_value', - description: 'Return a harmless synthetic test value.', - parameters: { - type: 'object', - properties: { name: { type: 'string' } }, - required: ['name'], - additionalProperties: false, - }, - strict: true, - }, - }], - tool_choice: 'required', - }; - const response = await fetch(String(configuredModel?.url), { - method: 'POST', - headers: { - authorization: `Bearer ${GATEWAY_KEY}`, - 'content-type': 'application/json', - }, - body: JSON.stringify(requestBody), - }); + for (const definition of VS_CODE_SUPPORTED_MODELS) { + const configuredModel = codeMieProvider?.models?.find(model => model.id === definition.id); + expect(configuredModel).toMatchObject({ + id: definition.id, + name: definition.id, + apiType: definition.apiType, + }); - expect(response.status).toBe(200); - expect(await response.json()).toEqual(toolResponse); - expect(captured).toHaveLength(1); - expect(captured[0].body).toEqual(requestBody); - expect(captured[0].headers.authorization).toBeUndefined(); - expect(captured[0].headers['x-codemie-client']).toBe('vscode-byok'); - expect(captured[0].headers['x-codemie-project']).toBe('test-project'); + const efforts: ReadonlyArray = + definition.supportsReasoningEffort ?? [undefined]; + for (const effort of efforts) { + const requestBody = buildRequestBody(definition, effort); + const response = await fetch(String(configuredModel?.url), { + method: 'POST', + headers: { + authorization: `Bearer ${GATEWAY_KEY}`, + 'content-type': 'application/json', + }, + body: JSON.stringify(requestBody), + }); + + expect(response.status).toBe(200); + expect(await response.json()).toEqual({ ok: true }); + + const forwarded = captured.at(-1); + expect(forwarded?.url).toBe(new URL(String(configuredModel?.url)).pathname); + expect(forwarded?.body).toEqual(requestBody); + expect(forwarded?.headers.authorization).toBeUndefined(); + expect(forwarded?.headers['x-codemie-client']).toBe('vscode-byok'); + expect(forwarded?.headers['x-codemie-project']).toBe('test-project'); + expect(forwarded?.headers['x-codemie-cli-model']).toBeUndefined(); + } + } }); }); diff --git a/tests/integration/vscode-models.live.test.ts b/tests/integration/vscode-models.live.test.ts new file mode 100644 index 00000000..e6c17239 --- /dev/null +++ b/tests/integration/vscode-models.live.test.ts @@ -0,0 +1,230 @@ +/** + * Opt-in live CodeMie model certification for the VS Code catalog. + * + * Required: + * CODEMIE_VSCODE_LIVE=1 + * CODEMIE_VSCODE_LIVE_URL=http://127.0.0.1:4001 + * CODEMIE_VSCODE_LIVE_API_KEY= + * CODEMIE_VSCODE_MODELS= + * + * Optional: + * CODEMIE_VSCODE_EFFORTS= + * CODEMIE_VSCODE_PROJECT= + * + * @group integration + */ + +import { describe, expect, it } from 'vitest'; +import { + VS_CODE_SUPPORTED_MODELS, + type VsCodeModelDefinition, + type VsCodeReasoningEffort, +} from '../../src/cli/commands/proxy/connectors/vscode-models.js'; + +const LIVE_ENABLED = process.env.CODEMIE_VSCODE_LIVE === '1'; +const ALL_EFFORTS: readonly VsCodeReasoningEffort[] = [ + 'none', + 'minimal', + 'low', + 'medium', + 'high', + 'xhigh', + 'max', +]; + +function getApiPath(definition: VsCodeModelDefinition): string { + if (definition.apiType === 'responses') return '/v1/responses'; + if (definition.apiType === 'messages') return '/v1/messages'; + return '/v1/chat/completions'; +} + +function selectModels(selector: string): readonly VsCodeModelDefinition[] { + if (selector === 'all') return VS_CODE_SUPPORTED_MODELS; + + const sliceMatch = /^slice:([123])$/.exec(selector); + if (sliceMatch) { + const slice = Number(sliceMatch[1]); + return VS_CODE_SUPPORTED_MODELS.filter(model => model.releaseSlice === slice); + } + + const requestedIds = selector.split(',').map(value => value.trim()).filter(Boolean); + const selected = VS_CODE_SUPPORTED_MODELS.filter(model => requestedIds.includes(model.id)); + const selectedIds = new Set(selected.map(model => model.id)); + const unknownIds = requestedIds.filter(id => !selectedIds.has(id)); + if (unknownIds.length > 0) { + throw new Error(`Unknown VS Code model IDs: ${unknownIds.join(', ')}`); + } + return selected; +} + +function selectEfforts( + definition: VsCodeModelDefinition, + selector: string | undefined +): ReadonlyArray { + if (!selector) return [undefined]; + if (selector === 'all') return definition.supportsReasoningEffort ?? [undefined]; + + const requested = selector.split(',').map(value => value.trim()).filter(Boolean); + const invalid = requested.filter( + (effort): effort is string => !ALL_EFFORTS.includes(effort as VsCodeReasoningEffort) + ); + if (invalid.length > 0) { + throw new Error(`Unknown reasoning efforts: ${invalid.join(', ')}`); + } + + const efforts = requested as VsCodeReasoningEffort[]; + const supported = definition.supportsReasoningEffort ?? []; + const unsupported = efforts.filter(effort => !supported.includes(effort)); + if (unsupported.length > 0) { + throw new Error( + `${definition.id} does not advertise efforts: ${unsupported.join(', ')}` + ); + } + return efforts; +} + +function buildRequestBody( + definition: VsCodeModelDefinition, + effort: VsCodeReasoningEffort | undefined +): Record { + if (definition.apiType === 'responses') { + return { + model: definition.id, + input: [{ role: 'user', content: 'Call get_test_value with value "ready".' }], + tools: [{ + type: 'function', + name: 'get_test_value', + description: 'Return a supplied synthetic value.', + parameters: { + type: 'object', + properties: { value: { type: 'string' } }, + required: ['value'], + }, + }], + tool_choice: 'required', + ...(effort ? { reasoning: { effort } } : {}), + }; + } + + if (definition.apiType === 'messages') { + return { + model: definition.id, + max_tokens: 1024, + messages: [{ + role: 'user', + content: 'Call get_test_value with value "ready".', + }], + tools: [{ + name: 'get_test_value', + description: 'Return a supplied synthetic value.', + input_schema: { + type: 'object', + properties: { value: { type: 'string' } }, + required: ['value'], + }, + }], + tool_choice: { type: 'any' }, + ...(definition.adaptiveThinking ? { thinking: { type: 'adaptive' } } : {}), + ...(effort ? { output_config: { effort } } : {}), + }; + } + + return { + model: definition.id, + messages: [{ + role: 'user', + content: 'Call get_test_value with value "ready".', + }], + tools: [{ + type: 'function', + function: { + name: 'get_test_value', + description: 'Return a supplied synthetic value.', + parameters: { + type: 'object', + properties: { value: { type: 'string' } }, + required: ['value'], + }, + }, + }], + tool_choice: 'required', + ...(effort ? { reasoning_effort: effort } : {}), + }; +} + +function hasToolCall(apiType: VsCodeModelDefinition['apiType'], body: unknown): boolean { + if (typeof body !== 'object' || body === null) return false; + const response = body as Record; + + if (apiType === 'responses') { + return Array.isArray(response.output) && + response.output.some(item => + typeof item === 'object' && item !== null && + (item as Record).type === 'function_call' + ); + } + + if (apiType === 'messages') { + return Array.isArray(response.content) && + response.content.some(item => + typeof item === 'object' && item !== null && + (item as Record).type === 'tool_use' + ); + } + + if (!Array.isArray(response.choices)) return false; + return response.choices.some(choice => { + if (typeof choice !== 'object' || choice === null) return false; + const message = (choice as Record).message; + return typeof message === 'object' && message !== null && + Array.isArray((message as Record).tool_calls); + }); +} + +describe.runIf(LIVE_ENABLED)('VS Code live model certification', () => { + it('accepts tools and every requested reasoning effort', async () => { + const baseUrl = process.env.CODEMIE_VSCODE_LIVE_URL?.replace(/\/$/, ''); + const apiKey = process.env.CODEMIE_VSCODE_LIVE_API_KEY; + const selector = process.env.CODEMIE_VSCODE_MODELS; + if (!baseUrl || !apiKey || !selector) { + throw new Error( + 'CODEMIE_VSCODE_LIVE_URL, CODEMIE_VSCODE_LIVE_API_KEY, and ' + + 'CODEMIE_VSCODE_MODELS are required for live certification.' + ); + } + + const models = selectModels(selector); + expect(models.length).toBeGreaterThan(0); + + for (const definition of models) { + const efforts = selectEfforts(definition, process.env.CODEMIE_VSCODE_EFFORTS); + for (const effort of efforts) { + const headers: Record = { + authorization: `Bearer ${apiKey}`, + 'content-type': 'application/json', + }; + const project = process.env.CODEMIE_VSCODE_PROJECT; + if (project) headers['x-codemie-project'] = project; + + const response = await fetch(`${baseUrl}${getApiPath(definition)}`, { + method: 'POST', + headers, + body: JSON.stringify(buildRequestBody(definition, effort)), + }); + const rawBody = await response.text(); + + expect( + response.ok, + `${definition.id} (${effort ?? 'default'}) returned ${response.status}: ` + + rawBody.slice(0, 1000) + ).toBe(true); + + const responseBody = JSON.parse(rawBody) as unknown; + expect( + hasToolCall(definition.apiType, responseBody), + `${definition.id} (${effort ?? 'default'}) returned no function tool call` + ).toBe(true); + } + } + }, 1_800_000); +}); From 3ddbb59b9be22d347c043f04a6a65685aeb7e981 Mon Sep 17 00:00:00 2001 From: Roman Sinyakov Date: Fri, 24 Jul 2026 13:41:27 +0200 Subject: [PATCH 2/4] fix(proxy): correct VS Code model compatibility settings --- reports/vscode-model-certification.md | 174 +++++++ .../proxy/connectors/__tests__/vscode.test.ts | 54 ++- .../proxy/connectors/vscode-models.ts | 22 +- src/cli/commands/proxy/connectors/vscode.ts | 32 +- tests/integration/vscode-byok.test.ts | 22 +- tests/integration/vscode-models.live.test.ts | 427 +++++++++++++++--- 6 files changed, 638 insertions(+), 93 deletions(-) create mode 100644 reports/vscode-model-certification.md diff --git a/reports/vscode-model-certification.md b/reports/vscode-model-certification.md new file mode 100644 index 00000000..3853c5ba --- /dev/null +++ b/reports/vscode-model-certification.md @@ -0,0 +1,174 @@ +# VS Code Model Certification Report + +- Generated: 2026-07-24T08:31:16.124Z +- Target: `http://127.0.0.1:4001` +- Authentication source: `running-vscode-proxy` +- Flow: streamed function-tool request through the VS Code-scoped CodeMie proxy +- Models: 27 +- Model/effort combinations: 111 +- Passing combinations: 111 +- Failing combinations: 0 +- Fully passing models: 27/27 + +No API keys, tokens, request headers, or response payloads are stored in this report. + +## Results + +| Model | API | Effort | Result | HTTP | Tool call | Attempts | Duration | Detail | +|---|---|---|---:|---:|---:|---:|---:|---| +| `claude-sonnet-4-5-20250929` | chat-completions | `default` | PASS | 200 | yes | 1 | 2065 ms | Streaming function tool call received. | +| `gpt-4.1` | chat-completions | `default` | PASS | 200 | yes | 1 | 948 ms | Streaming function tool call received. | +| `gpt-4.1-mini` | chat-completions | `default` | PASS | 200 | yes | 1 | 1030 ms | Streaming function tool call received. | +| `gpt-5-2025-08-07` | chat-completions | `default` | PASS | 200 | yes | 1 | 3433 ms | Streaming function tool call received. | +| `gpt-5-2025-08-07` | chat-completions | `minimal` | PASS | 200 | yes | 1 | 1117 ms | Streaming function tool call received. | +| `gpt-5-2025-08-07` | chat-completions | `low` | PASS | 200 | yes | 1 | 2141 ms | Streaming function tool call received. | +| `gpt-5-2025-08-07` | chat-completions | `medium` | PASS | 200 | yes | 1 | 2809 ms | Streaming function tool call received. | +| `gpt-5-2025-08-07` | chat-completions | `high` | PASS | 200 | yes | 1 | 8887 ms | Streaming function tool call received. | +| `gpt-5-mini-2025-08-07` | chat-completions | `default` | PASS | 200 | yes | 1 | 2862 ms | Streaming function tool call received. | +| `gpt-5-mini-2025-08-07` | chat-completions | `minimal` | PASS | 200 | yes | 1 | 831 ms | Streaming function tool call received. | +| `gpt-5-mini-2025-08-07` | chat-completions | `low` | PASS | 200 | yes | 1 | 1922 ms | Streaming function tool call received. | +| `gpt-5-mini-2025-08-07` | chat-completions | `medium` | PASS | 200 | yes | 1 | 2360 ms | Streaming function tool call received. | +| `gpt-5-mini-2025-08-07` | chat-completions | `high` | PASS | 200 | yes | 1 | 3186 ms | Streaming function tool call received. | +| `gpt-5-nano-2025-08-07` | chat-completions | `default` | PASS | 200 | yes | 1 | 4113 ms | Streaming function tool call received. | +| `gpt-5-nano-2025-08-07` | chat-completions | `minimal` | PASS | 200 | yes | 1 | 868 ms | Streaming function tool call received. | +| `gpt-5-nano-2025-08-07` | chat-completions | `low` | PASS | 200 | yes | 1 | 1278 ms | Streaming function tool call received. | +| `gpt-5-nano-2025-08-07` | chat-completions | `medium` | PASS | 200 | yes | 1 | 3463 ms | Streaming function tool call received. | +| `gpt-5-nano-2025-08-07` | chat-completions | `high` | PASS | 200 | yes | 1 | 5720 ms | Streaming function tool call received. | +| `gpt-5-1-codex-2025-11-13` | responses | `default` | PASS | 200 | yes | 1 | 1652 ms | Streaming function tool call received. | +| `gpt-5-1-codex-2025-11-13` | responses | `low` | PASS | 200 | yes | 1 | 2088 ms | Streaming function tool call received. | +| `gpt-5-1-codex-2025-11-13` | responses | `medium` | PASS | 200 | yes | 1 | 2669 ms | Streaming function tool call received. | +| `gpt-5-1-codex-2025-11-13` | responses | `high` | PASS | 200 | yes | 1 | 1656 ms | Streaming function tool call received. | +| `gpt-5-2-2025-12-11` | chat-completions | `default` | PASS | 200 | yes | 1 | 1437 ms | Streaming function tool call received. | +| `gpt-5-2-2025-12-11` | chat-completions | `none` | PASS | 200 | yes | 1 | 1250 ms | Streaming function tool call received. | +| `gpt-5-2-2025-12-11` | chat-completions | `low` | PASS | 200 | yes | 1 | 821 ms | Streaming function tool call received. | +| `gpt-5-2-2025-12-11` | chat-completions | `medium` | PASS | 200 | yes | 1 | 1819 ms | Streaming function tool call received. | +| `gpt-5-2-2025-12-11` | chat-completions | `high` | PASS | 200 | yes | 1 | 968 ms | Streaming function tool call received. | +| `gpt-5.4-2026-03-05` | chat-completions | `default` | PASS | 200 | yes | 1 | 1374 ms | Streaming function tool call received. | +| `gpt-5.4-2026-03-05` | chat-completions | `none` | PASS | 200 | yes | 1 | 1410 ms | Streaming function tool call received. | +| `gpt-5.4-2026-03-05` | chat-completions | `low` | PASS | 200 | yes | 1 | 899 ms | Streaming function tool call received. | +| `gpt-5.4-2026-03-05` | chat-completions | `medium` | PASS | 200 | yes | 1 | 667 ms | Streaming function tool call received. | +| `gpt-5.4-2026-03-05` | chat-completions | `high` | PASS | 200 | yes | 1 | 1465 ms | Streaming function tool call received. | +| `gpt-5.4-2026-03-05` | chat-completions | `xhigh` | PASS | 200 | yes | 1 | 1571 ms | Streaming function tool call received. | +| `gpt-5.5-2026-04-24` | responses | `default` | PASS | 200 | yes | 1 | 3587 ms | Streaming function tool call received. | +| `gpt-5.5-2026-04-24` | responses | `none` | PASS | 200 | yes | 1 | 1782 ms | Streaming function tool call received. | +| `gpt-5.5-2026-04-24` | responses | `low` | PASS | 200 | yes | 1 | 1813 ms | Streaming function tool call received. | +| `gpt-5.5-2026-04-24` | responses | `medium` | PASS | 200 | yes | 1 | 1730 ms | Streaming function tool call received. | +| `gpt-5.5-2026-04-24` | responses | `high` | PASS | 200 | yes | 1 | 1542 ms | Streaming function tool call received. | +| `gpt-5.5-2026-04-24` | responses | `xhigh` | PASS | 200 | yes | 1 | 3337 ms | Streaming function tool call received. | +| `gpt-5.6-luna-2026-07-09` | responses | `default` | PASS | 200 | yes | 1 | 1421 ms | Streaming function tool call received. | +| `gpt-5.6-luna-2026-07-09` | responses | `none` | PASS | 200 | yes | 1 | 2998 ms | Streaming function tool call received. | +| `gpt-5.6-luna-2026-07-09` | responses | `low` | PASS | 200 | yes | 1 | 1437 ms | Streaming function tool call received. | +| `gpt-5.6-luna-2026-07-09` | responses | `medium` | PASS | 200 | yes | 1 | 1311 ms | Streaming function tool call received. | +| `gpt-5.6-luna-2026-07-09` | responses | `high` | PASS | 200 | yes | 1 | 3261 ms | Streaming function tool call received. | +| `gpt-5.6-luna-2026-07-09` | responses | `xhigh` | PASS | 200 | yes | 1 | 1581 ms | Streaming function tool call received. | +| `gpt-5.6-luna-2026-07-09` | responses | `max` | PASS | 200 | yes | 1 | 1379 ms | Streaming function tool call received. | +| `gpt-5.6-sol-2026-07-09` | responses | `default` | PASS | 200 | yes | 1 | 1092 ms | Streaming function tool call received. | +| `gpt-5.6-sol-2026-07-09` | responses | `none` | PASS | 200 | yes | 1 | 2140 ms | Streaming function tool call received. | +| `gpt-5.6-sol-2026-07-09` | responses | `low` | PASS | 200 | yes | 1 | 19806 ms | Streaming function tool call received. | +| `gpt-5.6-sol-2026-07-09` | responses | `medium` | PASS | 200 | yes | 1 | 1136 ms | Streaming function tool call received. | +| `gpt-5.6-sol-2026-07-09` | responses | `high` | PASS | 200 | yes | 1 | 2222 ms | Streaming function tool call received. | +| `gpt-5.6-sol-2026-07-09` | responses | `xhigh` | PASS | 200 | yes | 1 | 1086 ms | Streaming function tool call received. | +| `gpt-5.6-sol-2026-07-09` | responses | `max` | PASS | 200 | yes | 1 | 1269 ms | Streaming function tool call received. | +| `gpt-5.6-terra-2026-07-09` | responses | `default` | PASS | 200 | yes | 1 | 1794 ms | Streaming function tool call received. | +| `gpt-5.6-terra-2026-07-09` | responses | `none` | PASS | 200 | yes | 1 | 2149 ms | Streaming function tool call received. | +| `gpt-5.6-terra-2026-07-09` | responses | `low` | PASS | 200 | yes | 1 | 1468 ms | Streaming function tool call received. | +| `gpt-5.6-terra-2026-07-09` | responses | `medium` | PASS | 200 | yes | 1 | 1873 ms | Streaming function tool call received. | +| `gpt-5.6-terra-2026-07-09` | responses | `high` | PASS | 200 | yes | 1 | 1921 ms | Streaming function tool call received. | +| `gpt-5.6-terra-2026-07-09` | responses | `xhigh` | PASS | 200 | yes | 1 | 1950 ms | Streaming function tool call received. | +| `gpt-5.6-terra-2026-07-09` | responses | `max` | PASS | 200 | yes | 1 | 1266 ms | Streaming function tool call received. | +| `gemini-3-flash` | chat-completions | `default` | PASS | 200 | yes | 1 | 1959 ms | Streaming function tool call received. | +| `gemini-3-flash` | chat-completions | `minimal` | PASS | 200 | yes | 1 | 844 ms | Streaming function tool call received. | +| `gemini-3-flash` | chat-completions | `low` | PASS | 200 | yes | 1 | 847 ms | Streaming function tool call received. | +| `gemini-3-flash` | chat-completions | `medium` | PASS | 200 | yes | 1 | 1269 ms | Streaming function tool call received. | +| `gemini-3-flash` | chat-completions | `high` | PASS | 200 | yes | 1 | 1614 ms | Streaming function tool call received. | +| `gemini-3.1-pro` | chat-completions | `default` | PASS | 200 | yes | 1 | 5728 ms | Streaming function tool call received. | +| `gemini-3.1-pro` | chat-completions | `low` | PASS | 200 | yes | 1 | 2616 ms | Streaming function tool call received. | +| `gemini-3.1-pro` | chat-completions | `medium` | PASS | 200 | yes | 1 | 3136 ms | Streaming function tool call received. | +| `gemini-3.1-pro` | chat-completions | `high` | PASS | 200 | yes | 1 | 3194 ms | Streaming function tool call received. | +| `gemini-3.5-flash` | chat-completions | `default` | PASS | 200 | yes | 1 | 1660 ms | Streaming function tool call received. | +| `gemini-3.5-flash` | chat-completions | `minimal` | PASS | 200 | yes | 1 | 17292 ms | Streaming function tool call received. | +| `gemini-3.5-flash` | chat-completions | `low` | PASS | 200 | yes | 1 | 25835 ms | Streaming function tool call received. | +| `gemini-3.5-flash` | chat-completions | `medium` | PASS | 200 | yes | 1 | 15569 ms | Streaming function tool call received. | +| `gemini-3.5-flash` | chat-completions | `high` | PASS | 200 | yes | 1 | 16069 ms | Streaming function tool call received. | +| `claude-4-5-sonnet` | chat-completions | `default` | PASS | 200 | yes | 1 | 1464 ms | Streaming function tool call received. | +| `claude-sonnet-4-6` | messages | `default` | PASS | 200 | yes | 1 | 3260 ms | Streaming function tool call received. | +| `claude-sonnet-4-6` | messages | `low` | PASS | 200 | yes | 1 | 2286 ms | Streaming function tool call received. | +| `claude-sonnet-4-6` | messages | `medium` | PASS | 200 | yes | 1 | 1935 ms | Streaming function tool call received. | +| `claude-sonnet-4-6` | messages | `high` | PASS | 200 | yes | 1 | 1793 ms | Streaming function tool call received. | +| `claude-sonnet-4-6` | messages | `max` | PASS | 200 | yes | 1 | 1852 ms | Streaming function tool call received. | +| `claude-sonnet-5` | messages | `default` | PASS | 200 | yes | 1 | 1829 ms | Streaming function tool call received. | +| `claude-sonnet-5` | messages | `low` | PASS | 200 | yes | 1 | 1616 ms | Streaming function tool call received. | +| `claude-sonnet-5` | messages | `medium` | PASS | 200 | yes | 1 | 1987 ms | Streaming function tool call received. | +| `claude-sonnet-5` | messages | `high` | PASS | 200 | yes | 1 | 2920 ms | Streaming function tool call received. | +| `claude-sonnet-5` | messages | `xhigh` | PASS | 200 | yes | 1 | 1918 ms | Streaming function tool call received. | +| `claude-sonnet-5` | messages | `max` | PASS | 200 | yes | 1 | 1762 ms | Streaming function tool call received. | +| `claude-opus-4-5-20251101` | messages | `default` | PASS | 200 | yes | 1 | 2114 ms | Streaming function tool call received. | +| `claude-opus-4-5-20251101` | messages | `low` | PASS | 200 | yes | 1 | 1635 ms | Streaming function tool call received. | +| `claude-opus-4-5-20251101` | messages | `medium` | PASS | 200 | yes | 1 | 1844 ms | Streaming function tool call received. | +| `claude-opus-4-5-20251101` | messages | `high` | PASS | 200 | yes | 1 | 1720 ms | Streaming function tool call received. | +| `claude-opus-4-6-20260205` | messages | `default` | PASS | 200 | yes | 1 | 1762 ms | Streaming function tool call received. | +| `claude-opus-4-6-20260205` | messages | `low` | PASS | 200 | yes | 1 | 1919 ms | Streaming function tool call received. | +| `claude-opus-4-6-20260205` | messages | `medium` | PASS | 200 | yes | 1 | 2658 ms | Streaming function tool call received. | +| `claude-opus-4-6-20260205` | messages | `high` | PASS | 200 | yes | 1 | 1746 ms | Streaming function tool call received. | +| `claude-opus-4-6-20260205` | messages | `max` | PASS | 200 | yes | 1 | 1749 ms | Streaming function tool call received. | +| `claude-opus-4-7` | messages | `default` | PASS | 200 | yes | 1 | 1433 ms | Streaming function tool call received. | +| `claude-opus-4-7` | messages | `low` | PASS | 200 | yes | 1 | 1489 ms | Streaming function tool call received. | +| `claude-opus-4-7` | messages | `medium` | PASS | 200 | yes | 1 | 2842 ms | Streaming function tool call received. | +| `claude-opus-4-7` | messages | `high` | PASS | 200 | yes | 1 | 1359 ms | Streaming function tool call received. | +| `claude-opus-4-7` | messages | `xhigh` | PASS | 200 | yes | 1 | 2265 ms | Streaming function tool call received. | +| `claude-opus-4-7` | messages | `max` | PASS | 200 | yes | 1 | 1312 ms | Streaming function tool call received. | +| `claude-opus-4-8` | messages | `default` | PASS | 200 | yes | 1 | 1777 ms | Streaming function tool call received. | +| `claude-opus-4-8` | messages | `low` | PASS | 200 | yes | 1 | 2838 ms | Streaming function tool call received. | +| `claude-opus-4-8` | messages | `medium` | PASS | 200 | yes | 1 | 1640 ms | Streaming function tool call received. | +| `claude-opus-4-8` | messages | `high` | PASS | 200 | yes | 1 | 1518 ms | Streaming function tool call received. | +| `claude-opus-4-8` | messages | `xhigh` | PASS | 200 | yes | 1 | 1535 ms | Streaming function tool call received. | +| `claude-opus-4-8` | messages | `max` | PASS | 200 | yes | 1 | 1778 ms | Streaming function tool call received. | +| `claude-haiku-4-5-20251001` | chat-completions | `default` | PASS | 200 | yes | 1 | 1941 ms | Streaming function tool call received. | +| `qwen.qwen3-coder-30b-a3b-v1` | chat-completions | `default` | PASS | 200 | yes | 1 | 1011 ms | Streaming function tool call received. | +| `qwen.qwen3-coder-480b-a35b-v1` | chat-completions | `default` | PASS | 200 | yes | 1 | 454 ms | Streaming function tool call received. | +| `moonshotai.kimi-k2.5` | chat-completions | `default` | PASS | 200 | yes | 1 | 615 ms | Streaming function tool call received. | + +## Advertised Configuration + +| Model | API | Advertised efforts | VS Code model options | +|---|---|---|---| +| `claude-sonnet-4-5-20250929` | chat-completions | none | `{"top_p":null}` | +| `gpt-4.1` | chat-completions | none | default | +| `gpt-4.1-mini` | chat-completions | none | default | +| `gpt-5-2025-08-07` | chat-completions | minimal, low, medium, high | default | +| `gpt-5-mini-2025-08-07` | chat-completions | minimal, low, medium, high | default | +| `gpt-5-nano-2025-08-07` | chat-completions | minimal, low, medium, high | default | +| `gpt-5-1-codex-2025-11-13` | responses | low, medium, high | default | +| `gpt-5-2-2025-12-11` | chat-completions | none, low, medium, high | default | +| `gpt-5.4-2026-03-05` | chat-completions | none, low, medium, high, xhigh | default | +| `gpt-5.5-2026-04-24` | responses | none, low, medium, high, xhigh | default | +| `gpt-5.6-luna-2026-07-09` | responses | none, low, medium, high, xhigh, max | default | +| `gpt-5.6-sol-2026-07-09` | responses | none, low, medium, high, xhigh, max | default | +| `gpt-5.6-terra-2026-07-09` | responses | none, low, medium, high, xhigh, max | default | +| `gemini-3-flash` | chat-completions | minimal, low, medium, high | default | +| `gemini-3.1-pro` | chat-completions | low, medium, high | default | +| `gemini-3.5-flash` | chat-completions | minimal, low, medium, high | default | +| `claude-4-5-sonnet` | chat-completions | none | `{"top_p":null}` | +| `claude-sonnet-4-6` | messages | low, medium, high, max | default | +| `claude-sonnet-5` | messages | low, medium, high, xhigh, max | default | +| `claude-opus-4-5-20251101` | messages | low, medium, high | default | +| `claude-opus-4-6-20260205` | messages | low, medium, high, max | default | +| `claude-opus-4-7` | messages | low, medium, high, xhigh, max | default | +| `claude-opus-4-8` | messages | low, medium, high, xhigh, max | default | +| `claude-haiku-4-5-20251001` | chat-completions | none | `{"top_p":null}` | +| `qwen.qwen3-coder-30b-a3b-v1` | chat-completions | none | default | +| `qwen.qwen3-coder-480b-a35b-v1` | chat-completions | none | default | +| `moonshotai.kimi-k2.5` | chat-completions | none | default | + +## Interpretation + +- `default` means no explicit reasoning-effort parameter was sent. +- Every advertised effort was sent using the model’s configured API body shape. +- PASS requires both a successful HTTP response and a streamed function tool call. +- A model should remain in the released catalog only when every advertised row passes. + +## Documentation + +- [VS Code custom endpoint model configuration](https://code.visualstudio.com/docs/agent-customization/language-models) +- [OpenAI model and reasoning guidance](https://developers.openai.com/api/docs/models) +- [AWS Claude Messages request parameters](https://docs.aws.amazon.com/bedrock/latest/userguide/model-parameters-anthropic-claude-messages-request-response.html) diff --git a/src/cli/commands/proxy/connectors/__tests__/vscode.test.ts b/src/cli/commands/proxy/connectors/__tests__/vscode.test.ts index 1d852577..838123df 100644 --- a/src/cli/commands/proxy/connectors/__tests__/vscode.test.ts +++ b/src/cli/commands/proxy/connectors/__tests__/vscode.test.ts @@ -105,6 +105,8 @@ describe('writeVsCodeLanguageModelsConfigAtPath', () => { maxOutputTokens: definition.maxOutputTokens, }; if (definition.adaptiveThinking) expected.adaptiveThinking = true; + if (definition.modelOptions) expected.modelOptions = definition.modelOptions; + if (definition.requestHeaders) expected.requestHeaders = definition.requestHeaders; if (definition.supportsReasoningEffort) { expected.supportsReasoningEffort = definition.supportsReasoningEffort; } @@ -117,7 +119,53 @@ describe('writeVsCodeLanguageModelsConfigAtPath', () => { } }); - it('replaces legacy CodeMie models while preserving the secret and saved settings', async () => { + it('omits top_p for Claude 4.5 models that reject dual sampling parameters', async () => { + await writeVsCodeLanguageModelsConfigAtPath(configPath, 'http://127.0.0.1:4001'); + + const providers = await readProviders(); + const models = providers[0].models as Array>; + const affectedIds = [ + 'claude-sonnet-4-5-20250929', + 'claude-4-5-sonnet', + 'claude-haiku-4-5-20251001', + ]; + + for (const id of affectedIds) { + expect(models.find(model => model.id === id)).toMatchObject({ + modelOptions: { top_p: null }, + }); + } + }); + + it('does not advertise the rejected none effort for GPT-5.1 Codex', async () => { + await writeVsCodeLanguageModelsConfigAtPath(configPath, 'http://127.0.0.1:4001'); + + const providers = await readProviders(); + const models = providers[0].models as Array>; + const codex = models.find(model => model.id === 'gpt-5-1-codex-2025-11-13'); + + expect(codex?.supportsReasoningEffort).toEqual(['low', 'medium', 'high']); + }); + + it('forces bearer authentication for Messages models only', async () => { + await writeVsCodeLanguageModelsConfigAtPath(configPath, 'http://127.0.0.1:4001'); + + const providers = await readProviders(); + const models = providers[0].models as Array>; + + for (const definition of VS_CODE_SUPPORTED_MODELS) { + const model = models.find(candidate => candidate.id === definition.id); + if (definition.apiType === 'messages') { + expect(model?.requestHeaders).toEqual({ + Authorization: 'Bearer ${apiKey}', + }); + } else { + expect(model).not.toHaveProperty('requestHeaders'); + } + } + }); + + it('overrides the CodeMie model catalog while preserving the secret and saved settings', async () => { const secretReference = '${input:chat.lm.secret.codemie}'; await writeFile(configPath, JSON.stringify([ { @@ -132,12 +180,11 @@ describe('writeVsCodeLanguageModelsConfigAtPath', () => { apiKey: secretReference, customProperty: 'preserved', settings: { - 'old-profile-model': { reasoningEffort: 'medium' }, 'gpt-5.5-2026-04-24': { reasoningEffort: 'high' }, 'custom-setting': { enabled: true }, }, models: [ - { id: 'old-profile-model', name: 'CodeMie Profile Model', stale: true }, + { id: 'stale-catalog-model', name: 'Stale catalog model', stale: true }, { id: 'user-managed-model', name: 'User model', custom: true }, ], }, @@ -169,7 +216,6 @@ describe('writeVsCodeLanguageModelsConfigAtPath', () => { 'custom-setting': { enabled: true }, }, }); - expect(codeMie.settings).not.toHaveProperty('old-profile-model'); expect(models.map(model => model.id)).toEqual(EXPECTED_MODEL_IDS); expect(models.some(model => model.id === 'user-managed-model')).toBe(false); }); diff --git a/src/cli/commands/proxy/connectors/vscode-models.ts b/src/cli/commands/proxy/connectors/vscode-models.ts index 7aee0e1f..3296695c 100644 --- a/src/cli/commands/proxy/connectors/vscode-models.ts +++ b/src/cli/commands/proxy/connectors/vscode-models.ts @@ -16,6 +16,11 @@ export interface VsCodeModelDefinition { vision: boolean; thinking: boolean; adaptiveThinking?: true; + modelOptions?: Readonly<{ + temperature?: number | null; + top_p?: number | null; + }>; + requestHeaders?: Readonly>; supportsReasoningEffort?: readonly VsCodeReasoningEffort[]; reasoningEffortFormat?: 'chat-completions' | 'responses'; maxInputTokens: number; @@ -23,7 +28,8 @@ export interface VsCodeModelDefinition { } const GPT_5_EFFORTS = ['minimal', 'low', 'medium', 'high'] as const; -const GPT_5_1_EFFORTS = ['none', 'low', 'medium', 'high'] as const; +const GPT_5_1_EFFORTS = ['low', 'medium', 'high'] as const; +const GPT_5_2_EFFORTS = ['none', 'low', 'medium', 'high'] as const; const GPT_5_XHIGH_EFFORTS = ['none', 'low', 'medium', 'high', 'xhigh'] as const; const GPT_5_6_EFFORTS = ['none', 'low', 'medium', 'high', 'xhigh', 'max'] as const; const GEMINI_FLASH_EFFORTS = ['minimal', 'low', 'medium', 'high'] as const; @@ -31,6 +37,9 @@ const GEMINI_PRO_EFFORTS = ['low', 'medium', 'high'] as const; const CLAUDE_EFFORTS = ['low', 'medium', 'high'] as const; const CLAUDE_MAX_EFFORTS = ['low', 'medium', 'high', 'max'] as const; const CLAUDE_XHIGH_EFFORTS = ['low', 'medium', 'high', 'xhigh', 'max'] as const; +const MESSAGE_AUTH_HEADERS = { + Authorization: 'Bearer ${apiKey}', +} as const; export const VS_CODE_SUPPORTED_MODELS: readonly VsCodeModelDefinition[] = [ { @@ -39,6 +48,7 @@ export const VS_CODE_SUPPORTED_MODELS: readonly VsCodeModelDefinition[] = [ apiType: 'chat-completions', vision: true, thinking: false, + modelOptions: { top_p: null }, maxInputTokens: 136000, maxOutputTokens: 64000, }, @@ -110,7 +120,7 @@ export const VS_CODE_SUPPORTED_MODELS: readonly VsCodeModelDefinition[] = [ apiType: 'chat-completions', vision: true, thinking: true, - supportsReasoningEffort: GPT_5_1_EFFORTS, + supportsReasoningEffort: GPT_5_2_EFFORTS, reasoningEffortFormat: 'chat-completions', maxInputTokens: 272000, maxOutputTokens: 128000, @@ -209,6 +219,7 @@ export const VS_CODE_SUPPORTED_MODELS: readonly VsCodeModelDefinition[] = [ apiType: 'chat-completions', vision: true, thinking: false, + modelOptions: { top_p: null }, maxInputTokens: 136000, maxOutputTokens: 64000, }, @@ -219,6 +230,7 @@ export const VS_CODE_SUPPORTED_MODELS: readonly VsCodeModelDefinition[] = [ vision: true, thinking: true, adaptiveThinking: true, + requestHeaders: MESSAGE_AUTH_HEADERS, supportsReasoningEffort: CLAUDE_MAX_EFFORTS, maxInputTokens: 936000, maxOutputTokens: 64000, @@ -230,6 +242,7 @@ export const VS_CODE_SUPPORTED_MODELS: readonly VsCodeModelDefinition[] = [ vision: true, thinking: true, adaptiveThinking: true, + requestHeaders: MESSAGE_AUTH_HEADERS, supportsReasoningEffort: CLAUDE_XHIGH_EFFORTS, maxInputTokens: 872000, maxOutputTokens: 128000, @@ -240,6 +253,7 @@ export const VS_CODE_SUPPORTED_MODELS: readonly VsCodeModelDefinition[] = [ apiType: 'messages', vision: true, thinking: false, + requestHeaders: MESSAGE_AUTH_HEADERS, supportsReasoningEffort: CLAUDE_EFFORTS, maxInputTokens: 136000, maxOutputTokens: 64000, @@ -251,6 +265,7 @@ export const VS_CODE_SUPPORTED_MODELS: readonly VsCodeModelDefinition[] = [ vision: true, thinking: true, adaptiveThinking: true, + requestHeaders: MESSAGE_AUTH_HEADERS, supportsReasoningEffort: CLAUDE_MAX_EFFORTS, maxInputTokens: 872000, maxOutputTokens: 128000, @@ -262,6 +277,7 @@ export const VS_CODE_SUPPORTED_MODELS: readonly VsCodeModelDefinition[] = [ vision: true, thinking: true, adaptiveThinking: true, + requestHeaders: MESSAGE_AUTH_HEADERS, supportsReasoningEffort: CLAUDE_XHIGH_EFFORTS, maxInputTokens: 872000, maxOutputTokens: 128000, @@ -273,6 +289,7 @@ export const VS_CODE_SUPPORTED_MODELS: readonly VsCodeModelDefinition[] = [ vision: true, thinking: true, adaptiveThinking: true, + requestHeaders: MESSAGE_AUTH_HEADERS, supportsReasoningEffort: CLAUDE_XHIGH_EFFORTS, maxInputTokens: 872000, maxOutputTokens: 128000, @@ -283,6 +300,7 @@ export const VS_CODE_SUPPORTED_MODELS: readonly VsCodeModelDefinition[] = [ apiType: 'chat-completions', vision: true, thinking: false, + modelOptions: { top_p: null }, maxInputTokens: 136000, maxOutputTokens: 64000, }, diff --git a/src/cli/commands/proxy/connectors/vscode.ts b/src/cli/commands/proxy/connectors/vscode.ts index c0c82c21..95e50711 100644 --- a/src/cli/commands/proxy/connectors/vscode.ts +++ b/src/cli/commands/proxy/connectors/vscode.ts @@ -9,7 +9,6 @@ import { type VsCodeReasoningEffort, } from './vscode-models.js'; -const LEGACY_MANAGED_MODEL_NAME = 'CodeMie Profile Model'; const SECRET_REFERENCE_PATTERN = /^\$\{input:chat\.lm\.secret\.[^}]+\}$/; interface VsCodeLanguageModelProvider { @@ -32,6 +31,11 @@ interface VsCodeManagedModel { streaming: true; thinking: boolean; adaptiveThinking?: true; + modelOptions?: Readonly<{ + temperature?: number | null; + top_p?: number | null; + }>; + requestHeaders?: Readonly>; supportsReasoningEffort?: readonly VsCodeReasoningEffort[]; reasoningEffortFormat?: 'chat-completions' | 'responses'; maxInputTokens: number; @@ -47,18 +51,10 @@ function isRecord(value: unknown): value is Record { return typeof value === 'object' && value !== null && !Array.isArray(value); } -function isLegacyManagedModel(model: unknown): boolean { - if (!isRecord(model)) return false; - return model.name === LEGACY_MANAGED_MODEL_NAME; -} - function isManagedProvider(provider: unknown): provider is VsCodeLanguageModelProvider { - if (!isRecord(provider)) return false; - - const hasLegacyManagedModel = Array.isArray(provider.models) && - provider.models.some(model => isLegacyManagedModel(model)); - return hasLegacyManagedModel || - (provider.vendor === 'customendpoint' && provider.name === 'CodeMie'); + return isRecord(provider) && + provider.vendor === 'customendpoint' && + provider.name === 'CodeMie'; } export function isVsCodeSecretReference(value: unknown): value is string { @@ -123,6 +119,8 @@ function buildManagedModels(proxyUrl: string): VsCodeManagedModel[] { }; if (definition.adaptiveThinking) model.adaptiveThinking = true; + if (definition.modelOptions) model.modelOptions = definition.modelOptions; + if (definition.requestHeaders) model.requestHeaders = definition.requestHeaders; if (definition.supportsReasoningEffort) { model.supportsReasoningEffort = definition.supportsReasoningEffort; } @@ -139,20 +137,10 @@ function mergeManagedProviders( proxyUrl: string ): { provider: VsCodeLanguageModelProvider; requiresSecretConfiguration: boolean } { const existingProvider = Object.assign({}, ...providers); - const existingModels = providers.flatMap(provider => - Array.isArray(provider.models) ? provider.models : [] - ); const existingSettings = Object.assign( {}, ...providers.map(provider => isRecord(provider.settings) ? provider.settings : {}) ); - const legacyManagedModelIds = existingModels - .filter(model => isLegacyManagedModel(model)) - .map(model => isRecord(model) ? model.id : undefined) - .filter((id): id is string => typeof id === 'string'); - for (const modelId of new Set(legacyManagedModelIds)) { - delete existingSettings[modelId]; - } const existingSecretReference = providers .map(provider => provider.apiKey) .find(isVsCodeSecretReference); diff --git a/tests/integration/vscode-byok.test.ts b/tests/integration/vscode-byok.test.ts index b95a610f..9129f3e8 100644 --- a/tests/integration/vscode-byok.test.ts +++ b/tests/integration/vscode-byok.test.ts @@ -106,7 +106,7 @@ function buildRequestBody( description: 'Return a synthetic test value.', input_schema: { type: 'object', properties: {} }, }], - tool_choice: { type: 'any' }, + tool_choice: { type: 'auto' }, ...(definition.adaptiveThinking ? { thinking: { type: 'adaptive' } } : {}), ...(effort ? { output_config: { effort } } : {}), }; @@ -130,6 +130,21 @@ function buildRequestBody( }; } +function buildVsCodeAuthHeaders( + definition: VsCodeModelDefinition +): Record { + if (definition.requestHeaders?.Authorization) { + return { + authorization: definition.requestHeaders.Authorization.replace( + '${apiKey}', + GATEWAY_KEY + ), + }; + } + if (definition.apiType === 'messages') return { 'x-api-key': GATEWAY_KEY }; + return { authorization: `Bearer ${GATEWAY_KEY}` }; +} + describe('VS Code BYOK model matrix', () => { const proxies: CodeMieProxy[] = []; const servers: Server[] = []; @@ -199,13 +214,13 @@ describe('VS Code BYOK model matrix', () => { }); const efforts: ReadonlyArray = - definition.supportsReasoningEffort ?? [undefined]; + [undefined, ...(definition.supportsReasoningEffort ?? [])]; for (const effort of efforts) { const requestBody = buildRequestBody(definition, effort); const response = await fetch(String(configuredModel?.url), { method: 'POST', headers: { - authorization: `Bearer ${GATEWAY_KEY}`, + ...buildVsCodeAuthHeaders(definition), 'content-type': 'application/json', }, body: JSON.stringify(requestBody), @@ -218,6 +233,7 @@ describe('VS Code BYOK model matrix', () => { expect(forwarded?.url).toBe(new URL(String(configuredModel?.url)).pathname); expect(forwarded?.body).toEqual(requestBody); expect(forwarded?.headers.authorization).toBeUndefined(); + expect(forwarded?.headers['x-api-key']).toBeUndefined(); expect(forwarded?.headers['x-codemie-client']).toBe('vscode-byok'); expect(forwarded?.headers['x-codemie-project']).toBe('test-project'); expect(forwarded?.headers['x-codemie-cli-model']).toBeUndefined(); diff --git a/tests/integration/vscode-models.live.test.ts b/tests/integration/vscode-models.live.test.ts index e6c17239..ade306a8 100644 --- a/tests/integration/vscode-models.live.test.ts +++ b/tests/integration/vscode-models.live.test.ts @@ -3,18 +3,30 @@ * * Required: * CODEMIE_VSCODE_LIVE=1 + * CODEMIE_VSCODE_MODELS= + * + * Authentication: + * Either start a VS Code proxy with `codemie proxy connect vscode`, or set: * CODEMIE_VSCODE_LIVE_URL=http://127.0.0.1:4001 * CODEMIE_VSCODE_LIVE_API_KEY= - * CODEMIE_VSCODE_MODELS= * * Optional: * CODEMIE_VSCODE_EFFORTS= * CODEMIE_VSCODE_PROJECT= + * CODEMIE_VSCODE_REPORT_PATH= + * CODEMIE_VSCODE_REQUEST_TIMEOUT_MS= + * CODEMIE_VSCODE_MAX_ATTEMPTS= + * + * `all` efforts certifies the no-effort/default request and every effort + * advertised for each selected model. * * @group integration */ +import { mkdir, writeFile } from 'node:fs/promises'; +import { dirname, resolve } from 'node:path'; import { describe, expect, it } from 'vitest'; +import { isProcessAlive, readState } from '../../src/cli/commands/proxy/daemon-manager.js'; import { VS_CODE_SUPPORTED_MODELS, type VsCodeModelDefinition, @@ -31,6 +43,39 @@ const ALL_EFFORTS: readonly VsCodeReasoningEffort[] = [ 'xhigh', 'max', ]; +const DEFAULT_REPORT_PATH = resolve('reports/vscode-model-certification.md'); + +interface LiveConfiguration { + baseUrl: string; + apiKey: string; + project?: string; + source: 'environment' | 'running-vscode-proxy'; +} + +interface CertificationResult { + model: string; + apiType: VsCodeModelDefinition['apiType']; + effort: VsCodeReasoningEffort | 'default'; + passed: boolean; + status: number; + toolCall: boolean; + durationMs: number; + attempts: number; + detail: string; +} + +function buildVsCodeAuthHeaders( + definition: VsCodeModelDefinition, + apiKey: string +): Record { + if (definition.requestHeaders?.Authorization) { + return { + authorization: definition.requestHeaders.Authorization.replace('${apiKey}', apiKey), + }; + } + if (definition.apiType === 'messages') return { 'x-api-key': apiKey }; + return { authorization: `Bearer ${apiKey}` }; +} function getApiPath(definition: VsCodeModelDefinition): string { if (definition.apiType === 'responses') return '/v1/responses'; @@ -62,7 +107,9 @@ function selectEfforts( selector: string | undefined ): ReadonlyArray { if (!selector) return [undefined]; - if (selector === 'all') return definition.supportsReasoningEffort ?? [undefined]; + if (selector === 'all') { + return [undefined, ...(definition.supportsReasoningEffort ?? [])]; + } const requested = selector.split(',').map(value => value.trim()).filter(Boolean); const invalid = requested.filter( @@ -102,6 +149,7 @@ function buildRequestBody( }, }], tool_choice: 'required', + stream: true, ...(effort ? { reasoning: { effort } } : {}), }; } @@ -123,7 +171,8 @@ function buildRequestBody( required: ['value'], }, }], - tool_choice: { type: 'any' }, + tool_choice: { type: 'auto' }, + stream: true, ...(definition.adaptiveThinking ? { thinking: { type: 'adaptive' } } : {}), ...(effort ? { output_config: { effort } } : {}), }; @@ -148,83 +197,337 @@ function buildRequestBody( }, }], tool_choice: 'required', + stream: true, + ...(definition.modelOptions?.temperature === null ? {} : { + ...(definition.modelOptions?.temperature !== undefined + ? { temperature: definition.modelOptions.temperature } + : {}), + }), + ...(definition.modelOptions?.top_p === null ? {} : { + ...(definition.modelOptions?.top_p !== undefined + ? { top_p: definition.modelOptions.top_p } + : {}), + }), ...(effort ? { reasoning_effort: effort } : {}), }; } -function hasToolCall(apiType: VsCodeModelDefinition['apiType'], body: unknown): boolean { - if (typeof body !== 'object' || body === null) return false; - const response = body as Record; +function parseResponsePayloads(rawBody: string): unknown[] { + const trimmed = rawBody.trim(); + if (!trimmed) return []; + + if (trimmed.startsWith('{') || trimmed.startsWith('[')) { + try { + return [JSON.parse(trimmed) as unknown]; + } catch { + return []; + } + } + + const payloads: unknown[] = []; + for (const line of rawBody.split(/\r?\n/)) { + if (!line.startsWith('data:')) continue; + const data = line.slice('data:'.length).trim(); + if (!data || data === '[DONE]') continue; + try { + payloads.push(JSON.parse(data) as unknown); + } catch { + // Ignore non-JSON SSE metadata while retaining all valid event payloads. + } + } + return payloads; +} + +function containsToolCall(value: unknown): boolean { + if (Array.isArray(value)) return value.some(item => containsToolCall(item)); + if (typeof value !== 'object' || value === null) return false; + + const record = value as Record; + if (record.type === 'function_call' || record.type === 'tool_use') return true; + if (Array.isArray(record.tool_calls) && record.tool_calls.length > 0) return true; + return Object.values(record).some(item => containsToolCall(item)); +} + +function findErrorMessage(value: unknown): string | undefined { + if (Array.isArray(value)) { + for (const item of value) { + const message = findErrorMessage(item); + if (message) return message; + } + return undefined; + } + if (typeof value !== 'object' || value === null) return undefined; + + const record = value as Record; + if (typeof record.message === 'string' && ( + record.type === 'error' || + record.code !== undefined || + record.error !== undefined + )) { + return record.message; + } + if (record.error !== undefined) { + const nested = findErrorMessage(record.error); + if (nested) return nested; + } + return undefined; +} + +function compactDetail(value: string): string { + return value + .replace(/Bearer\s+\S+/gi, 'Bearer [redacted]') + .replace(/\s+/g, ' ') + .replace(/\|/g, '\\|') + .trim() + .slice(0, 500); +} + +function getResponseDetail( + responseOk: boolean, + toolCall: boolean, + rawBody: string, + payloads: unknown[] +): string { + if (responseOk && toolCall) return 'Streaming function tool call received.'; + if (responseOk) return 'Request succeeded but no function tool call was detected.'; + + for (const payload of payloads) { + const message = findErrorMessage(payload); + if (message) return compactDetail(message); + } + return compactDetail(rawBody) || 'Request failed without a response body.'; +} + +async function resolveLiveConfiguration(): Promise { + const configuredUrl = process.env.CODEMIE_VSCODE_LIVE_URL?.replace(/\/$/, ''); + const configuredKey = process.env.CODEMIE_VSCODE_LIVE_API_KEY; + if (configuredUrl && configuredKey) { + return { + baseUrl: configuredUrl, + apiKey: configuredKey, + project: process.env.CODEMIE_VSCODE_PROJECT, + source: 'environment', + }; + } + + const state = await readState(); + if (!state || !isProcessAlive(state.pid)) { + throw new Error( + 'Start `codemie proxy connect vscode`, or set CODEMIE_VSCODE_LIVE_URL and ' + + 'CODEMIE_VSCODE_LIVE_API_KEY.' + ); + } + if (state.clientType !== 'vscode-byok') { + throw new Error( + `The running proxy client is "${state.clientType ?? 'unknown'}"; ` + + 'live VS Code certification requires a vscode-byok proxy.' + ); + } + + return { + baseUrl: state.url.replace(/\/$/, ''), + apiKey: state.gatewayKey, + project: process.env.CODEMIE_VSCODE_PROJECT, + source: 'running-vscode-proxy', + }; +} + +async function certifyAttempt( + configuration: LiveConfiguration, + definition: VsCodeModelDefinition, + effort: VsCodeReasoningEffort | undefined, + timeoutMs: number, + attempt: number +): Promise { + const startedAt = Date.now(); + const headers: Record = { + ...buildVsCodeAuthHeaders(definition, configuration.apiKey), + 'content-type': 'application/json', + }; + if (configuration.project) headers['x-codemie-project'] = configuration.project; + + try { + const response = await fetch(`${configuration.baseUrl}${getApiPath(definition)}`, { + method: 'POST', + headers, + body: JSON.stringify(buildRequestBody(definition, effort)), + signal: AbortSignal.timeout(timeoutMs), + }); + const rawBody = await response.text(); + const payloads = parseResponsePayloads(rawBody); + const toolCall = payloads.some(payload => containsToolCall(payload)); + const passed = response.ok && toolCall; - if (apiType === 'responses') { - return Array.isArray(response.output) && - response.output.some(item => - typeof item === 'object' && item !== null && - (item as Record).type === 'function_call' - ); + return { + model: definition.id, + apiType: definition.apiType, + effort: effort ?? 'default', + passed, + status: response.status, + toolCall, + durationMs: Date.now() - startedAt, + attempts: attempt, + detail: getResponseDetail(response.ok, toolCall, rawBody, payloads), + }; + } catch (error) { + return { + model: definition.id, + apiType: definition.apiType, + effort: effort ?? 'default', + passed: false, + status: 0, + toolCall: false, + durationMs: Date.now() - startedAt, + attempts: attempt, + detail: compactDetail(error instanceof Error ? error.message : String(error)), + }; } +} - if (apiType === 'messages') { - return Array.isArray(response.content) && - response.content.some(item => - typeof item === 'object' && item !== null && - (item as Record).type === 'tool_use' - ); +function isRetryable(result: CertificationResult): boolean { + return result.status === 0 || result.status === 429 || result.status >= 500; +} + +async function certifyCombination( + configuration: LiveConfiguration, + definition: VsCodeModelDefinition, + effort: VsCodeReasoningEffort | undefined, + timeoutMs: number, + maxAttempts: number +): Promise { + let lastResult: CertificationResult | undefined; + for (let attempt = 1; attempt <= maxAttempts; attempt++) { + const result = await certifyAttempt( + configuration, + definition, + effort, + timeoutMs, + attempt + ); + if (result.passed || !isRetryable(result)) return result; + lastResult = result; } + if (!lastResult) throw new Error('Certification did not execute any attempts.'); + return lastResult; +} - if (!Array.isArray(response.choices)) return false; - return response.choices.some(choice => { - if (typeof choice !== 'object' || choice === null) return false; - const message = (choice as Record).message; - return typeof message === 'object' && message !== null && - Array.isArray((message as Record).tool_calls); - }); +function renderReport( + configuration: LiveConfiguration, + selectedModels: readonly VsCodeModelDefinition[], + results: readonly CertificationResult[] +): string { + const passed = results.filter(result => result.passed).length; + const failed = results.length - passed; + const fullyPassingModels = selectedModels.filter(model => { + const modelResults = results.filter(result => result.model === model.id); + return modelResults.length > 0 && modelResults.every(result => result.passed); + }).length; + + const lines = [ + '# VS Code Model Certification Report', + '', + `- Generated: ${new Date().toISOString()}`, + `- Target: \`${configuration.baseUrl}\``, + `- Authentication source: \`${configuration.source}\``, + '- Flow: streamed function-tool request through the VS Code-scoped CodeMie proxy', + `- Models: ${selectedModels.length}`, + `- Model/effort combinations: ${results.length}`, + `- Passing combinations: ${passed}`, + `- Failing combinations: ${failed}`, + `- Fully passing models: ${fullyPassingModels}/${selectedModels.length}`, + '', + 'No API keys, tokens, request headers, or response payloads are stored in this report.', + '', + '## Results', + '', + '| Model | API | Effort | Result | HTTP | Tool call | Attempts | Duration | Detail |', + '|---|---|---|---:|---:|---:|---:|---:|---|', + ...results.map(result => + `| \`${result.model}\` | ${result.apiType} | \`${result.effort}\` | ` + + `${result.passed ? 'PASS' : 'FAIL'} | ${result.status || 'n/a'} | ` + + `${result.toolCall ? 'yes' : 'no'} | ${result.attempts} | ` + + `${result.durationMs} ms | ${result.detail} |` + ), + '', + '## Advertised Configuration', + '', + '| Model | API | Advertised efforts | VS Code model options |', + '|---|---|---|---|', + ...selectedModels.map(model => { + const efforts = model.supportsReasoningEffort?.join(', ') ?? 'none'; + const modelOptions = model.modelOptions + ? `\`${JSON.stringify(model.modelOptions)}\`` + : 'default'; + return `| \`${model.id}\` | ${model.apiType} | ${efforts} | ${modelOptions} |`; + }), + '', + '## Interpretation', + '', + '- `default` means no explicit reasoning-effort parameter was sent.', + '- Every advertised effort was sent using the model’s configured API body shape.', + '- PASS requires both a successful HTTP response and a streamed function tool call.', + '- A model should remain in the released catalog only when every advertised row passes.', + '', + '## Documentation', + '', + '- [VS Code custom endpoint model configuration](https://code.visualstudio.com/docs/agent-customization/language-models)', + '- [OpenAI model and reasoning guidance](https://developers.openai.com/api/docs/models)', + '- [AWS Claude Messages request parameters](https://docs.aws.amazon.com/bedrock/latest/userguide/model-parameters-anthropic-claude-messages-request-response.html)', + '', + ]; + + return `${lines.join('\n')}\n`; } describe.runIf(LIVE_ENABLED)('VS Code live model certification', () => { - it('accepts tools and every requested reasoning effort', async () => { - const baseUrl = process.env.CODEMIE_VSCODE_LIVE_URL?.replace(/\/$/, ''); - const apiKey = process.env.CODEMIE_VSCODE_LIVE_API_KEY; + it('streams tools through every requested model and reasoning effort', async () => { const selector = process.env.CODEMIE_VSCODE_MODELS; - if (!baseUrl || !apiKey || !selector) { - throw new Error( - 'CODEMIE_VSCODE_LIVE_URL, CODEMIE_VSCODE_LIVE_API_KEY, and ' + - 'CODEMIE_VSCODE_MODELS are required for live certification.' - ); + if (!selector) { + throw new Error('CODEMIE_VSCODE_MODELS is required for live certification.'); } - const models = selectModels(selector); - expect(models.length).toBeGreaterThan(0); + const configuration = await resolveLiveConfiguration(); + const selectedModels = selectModels(selector); + expect(selectedModels.length).toBeGreaterThan(0); + + const parsedTimeout = Number(process.env.CODEMIE_VSCODE_REQUEST_TIMEOUT_MS ?? '120000'); + if (!Number.isFinite(parsedTimeout) || parsedTimeout <= 0) { + throw new Error('CODEMIE_VSCODE_REQUEST_TIMEOUT_MS must be a positive number.'); + } + const maxAttempts = Number(process.env.CODEMIE_VSCODE_MAX_ATTEMPTS ?? '2'); + if (!Number.isInteger(maxAttempts) || maxAttempts <= 0) { + throw new Error('CODEMIE_VSCODE_MAX_ATTEMPTS must be a positive integer.'); + } - for (const definition of models) { + const results: CertificationResult[] = []; + for (const definition of selectedModels) { const efforts = selectEfforts(definition, process.env.CODEMIE_VSCODE_EFFORTS); for (const effort of efforts) { - const headers: Record = { - authorization: `Bearer ${apiKey}`, - 'content-type': 'application/json', - }; - const project = process.env.CODEMIE_VSCODE_PROJECT; - if (project) headers['x-codemie-project'] = project; - - const response = await fetch(`${baseUrl}${getApiPath(definition)}`, { - method: 'POST', - headers, - body: JSON.stringify(buildRequestBody(definition, effort)), - }); - const rawBody = await response.text(); - - expect( - response.ok, - `${definition.id} (${effort ?? 'default'}) returned ${response.status}: ` + - rawBody.slice(0, 1000) - ).toBe(true); - - const responseBody = JSON.parse(rawBody) as unknown; - expect( - hasToolCall(definition.apiType, responseBody), - `${definition.id} (${effort ?? 'default'}) returned no function tool call` - ).toBe(true); + results.push(await certifyCombination( + configuration, + definition, + effort, + parsedTimeout, + maxAttempts + )); } } - }, 1_800_000); + + const reportPath = resolve( + process.env.CODEMIE_VSCODE_REPORT_PATH ?? DEFAULT_REPORT_PATH + ); + await mkdir(dirname(reportPath), { recursive: true }); + await writeFile( + reportPath, + renderReport(configuration, selectedModels, results), + 'utf-8' + ); + + const failures = results.filter(result => !result.passed); + expect( + failures, + `${failures.length} live certification combination(s) failed. ` + + `See ${reportPath}.` + ).toHaveLength(0); + }, 3_600_000); }); From 91b62a983fec5bdce4afde6cdd20bb8244fdf682 Mon Sep 17 00:00:00 2001 From: Roman Sinyakov Date: Fri, 24 Jul 2026 14:34:36 +0200 Subject: [PATCH 3/4] fix(proxy): stabilize VS Code Responses model support --- reports/vscode-model-certification.md | 174 ------------------ .../proxy/connectors/__tests__/vscode.test.ts | 2 - .../proxy/connectors/vscode-models.ts | 41 +---- .../plugins/sso/proxy/plugins/index.ts | 3 + .../vscode-request-normalizer.plugin.ts | 85 +++++++++ tests/integration/vscode-models.live.test.ts | 8 +- 6 files changed, 92 insertions(+), 221 deletions(-) delete mode 100644 reports/vscode-model-certification.md create mode 100644 src/providers/plugins/sso/proxy/plugins/vscode-request-normalizer.plugin.ts diff --git a/reports/vscode-model-certification.md b/reports/vscode-model-certification.md deleted file mode 100644 index 3853c5ba..00000000 --- a/reports/vscode-model-certification.md +++ /dev/null @@ -1,174 +0,0 @@ -# VS Code Model Certification Report - -- Generated: 2026-07-24T08:31:16.124Z -- Target: `http://127.0.0.1:4001` -- Authentication source: `running-vscode-proxy` -- Flow: streamed function-tool request through the VS Code-scoped CodeMie proxy -- Models: 27 -- Model/effort combinations: 111 -- Passing combinations: 111 -- Failing combinations: 0 -- Fully passing models: 27/27 - -No API keys, tokens, request headers, or response payloads are stored in this report. - -## Results - -| Model | API | Effort | Result | HTTP | Tool call | Attempts | Duration | Detail | -|---|---|---|---:|---:|---:|---:|---:|---| -| `claude-sonnet-4-5-20250929` | chat-completions | `default` | PASS | 200 | yes | 1 | 2065 ms | Streaming function tool call received. | -| `gpt-4.1` | chat-completions | `default` | PASS | 200 | yes | 1 | 948 ms | Streaming function tool call received. | -| `gpt-4.1-mini` | chat-completions | `default` | PASS | 200 | yes | 1 | 1030 ms | Streaming function tool call received. | -| `gpt-5-2025-08-07` | chat-completions | `default` | PASS | 200 | yes | 1 | 3433 ms | Streaming function tool call received. | -| `gpt-5-2025-08-07` | chat-completions | `minimal` | PASS | 200 | yes | 1 | 1117 ms | Streaming function tool call received. | -| `gpt-5-2025-08-07` | chat-completions | `low` | PASS | 200 | yes | 1 | 2141 ms | Streaming function tool call received. | -| `gpt-5-2025-08-07` | chat-completions | `medium` | PASS | 200 | yes | 1 | 2809 ms | Streaming function tool call received. | -| `gpt-5-2025-08-07` | chat-completions | `high` | PASS | 200 | yes | 1 | 8887 ms | Streaming function tool call received. | -| `gpt-5-mini-2025-08-07` | chat-completions | `default` | PASS | 200 | yes | 1 | 2862 ms | Streaming function tool call received. | -| `gpt-5-mini-2025-08-07` | chat-completions | `minimal` | PASS | 200 | yes | 1 | 831 ms | Streaming function tool call received. | -| `gpt-5-mini-2025-08-07` | chat-completions | `low` | PASS | 200 | yes | 1 | 1922 ms | Streaming function tool call received. | -| `gpt-5-mini-2025-08-07` | chat-completions | `medium` | PASS | 200 | yes | 1 | 2360 ms | Streaming function tool call received. | -| `gpt-5-mini-2025-08-07` | chat-completions | `high` | PASS | 200 | yes | 1 | 3186 ms | Streaming function tool call received. | -| `gpt-5-nano-2025-08-07` | chat-completions | `default` | PASS | 200 | yes | 1 | 4113 ms | Streaming function tool call received. | -| `gpt-5-nano-2025-08-07` | chat-completions | `minimal` | PASS | 200 | yes | 1 | 868 ms | Streaming function tool call received. | -| `gpt-5-nano-2025-08-07` | chat-completions | `low` | PASS | 200 | yes | 1 | 1278 ms | Streaming function tool call received. | -| `gpt-5-nano-2025-08-07` | chat-completions | `medium` | PASS | 200 | yes | 1 | 3463 ms | Streaming function tool call received. | -| `gpt-5-nano-2025-08-07` | chat-completions | `high` | PASS | 200 | yes | 1 | 5720 ms | Streaming function tool call received. | -| `gpt-5-1-codex-2025-11-13` | responses | `default` | PASS | 200 | yes | 1 | 1652 ms | Streaming function tool call received. | -| `gpt-5-1-codex-2025-11-13` | responses | `low` | PASS | 200 | yes | 1 | 2088 ms | Streaming function tool call received. | -| `gpt-5-1-codex-2025-11-13` | responses | `medium` | PASS | 200 | yes | 1 | 2669 ms | Streaming function tool call received. | -| `gpt-5-1-codex-2025-11-13` | responses | `high` | PASS | 200 | yes | 1 | 1656 ms | Streaming function tool call received. | -| `gpt-5-2-2025-12-11` | chat-completions | `default` | PASS | 200 | yes | 1 | 1437 ms | Streaming function tool call received. | -| `gpt-5-2-2025-12-11` | chat-completions | `none` | PASS | 200 | yes | 1 | 1250 ms | Streaming function tool call received. | -| `gpt-5-2-2025-12-11` | chat-completions | `low` | PASS | 200 | yes | 1 | 821 ms | Streaming function tool call received. | -| `gpt-5-2-2025-12-11` | chat-completions | `medium` | PASS | 200 | yes | 1 | 1819 ms | Streaming function tool call received. | -| `gpt-5-2-2025-12-11` | chat-completions | `high` | PASS | 200 | yes | 1 | 968 ms | Streaming function tool call received. | -| `gpt-5.4-2026-03-05` | chat-completions | `default` | PASS | 200 | yes | 1 | 1374 ms | Streaming function tool call received. | -| `gpt-5.4-2026-03-05` | chat-completions | `none` | PASS | 200 | yes | 1 | 1410 ms | Streaming function tool call received. | -| `gpt-5.4-2026-03-05` | chat-completions | `low` | PASS | 200 | yes | 1 | 899 ms | Streaming function tool call received. | -| `gpt-5.4-2026-03-05` | chat-completions | `medium` | PASS | 200 | yes | 1 | 667 ms | Streaming function tool call received. | -| `gpt-5.4-2026-03-05` | chat-completions | `high` | PASS | 200 | yes | 1 | 1465 ms | Streaming function tool call received. | -| `gpt-5.4-2026-03-05` | chat-completions | `xhigh` | PASS | 200 | yes | 1 | 1571 ms | Streaming function tool call received. | -| `gpt-5.5-2026-04-24` | responses | `default` | PASS | 200 | yes | 1 | 3587 ms | Streaming function tool call received. | -| `gpt-5.5-2026-04-24` | responses | `none` | PASS | 200 | yes | 1 | 1782 ms | Streaming function tool call received. | -| `gpt-5.5-2026-04-24` | responses | `low` | PASS | 200 | yes | 1 | 1813 ms | Streaming function tool call received. | -| `gpt-5.5-2026-04-24` | responses | `medium` | PASS | 200 | yes | 1 | 1730 ms | Streaming function tool call received. | -| `gpt-5.5-2026-04-24` | responses | `high` | PASS | 200 | yes | 1 | 1542 ms | Streaming function tool call received. | -| `gpt-5.5-2026-04-24` | responses | `xhigh` | PASS | 200 | yes | 1 | 3337 ms | Streaming function tool call received. | -| `gpt-5.6-luna-2026-07-09` | responses | `default` | PASS | 200 | yes | 1 | 1421 ms | Streaming function tool call received. | -| `gpt-5.6-luna-2026-07-09` | responses | `none` | PASS | 200 | yes | 1 | 2998 ms | Streaming function tool call received. | -| `gpt-5.6-luna-2026-07-09` | responses | `low` | PASS | 200 | yes | 1 | 1437 ms | Streaming function tool call received. | -| `gpt-5.6-luna-2026-07-09` | responses | `medium` | PASS | 200 | yes | 1 | 1311 ms | Streaming function tool call received. | -| `gpt-5.6-luna-2026-07-09` | responses | `high` | PASS | 200 | yes | 1 | 3261 ms | Streaming function tool call received. | -| `gpt-5.6-luna-2026-07-09` | responses | `xhigh` | PASS | 200 | yes | 1 | 1581 ms | Streaming function tool call received. | -| `gpt-5.6-luna-2026-07-09` | responses | `max` | PASS | 200 | yes | 1 | 1379 ms | Streaming function tool call received. | -| `gpt-5.6-sol-2026-07-09` | responses | `default` | PASS | 200 | yes | 1 | 1092 ms | Streaming function tool call received. | -| `gpt-5.6-sol-2026-07-09` | responses | `none` | PASS | 200 | yes | 1 | 2140 ms | Streaming function tool call received. | -| `gpt-5.6-sol-2026-07-09` | responses | `low` | PASS | 200 | yes | 1 | 19806 ms | Streaming function tool call received. | -| `gpt-5.6-sol-2026-07-09` | responses | `medium` | PASS | 200 | yes | 1 | 1136 ms | Streaming function tool call received. | -| `gpt-5.6-sol-2026-07-09` | responses | `high` | PASS | 200 | yes | 1 | 2222 ms | Streaming function tool call received. | -| `gpt-5.6-sol-2026-07-09` | responses | `xhigh` | PASS | 200 | yes | 1 | 1086 ms | Streaming function tool call received. | -| `gpt-5.6-sol-2026-07-09` | responses | `max` | PASS | 200 | yes | 1 | 1269 ms | Streaming function tool call received. | -| `gpt-5.6-terra-2026-07-09` | responses | `default` | PASS | 200 | yes | 1 | 1794 ms | Streaming function tool call received. | -| `gpt-5.6-terra-2026-07-09` | responses | `none` | PASS | 200 | yes | 1 | 2149 ms | Streaming function tool call received. | -| `gpt-5.6-terra-2026-07-09` | responses | `low` | PASS | 200 | yes | 1 | 1468 ms | Streaming function tool call received. | -| `gpt-5.6-terra-2026-07-09` | responses | `medium` | PASS | 200 | yes | 1 | 1873 ms | Streaming function tool call received. | -| `gpt-5.6-terra-2026-07-09` | responses | `high` | PASS | 200 | yes | 1 | 1921 ms | Streaming function tool call received. | -| `gpt-5.6-terra-2026-07-09` | responses | `xhigh` | PASS | 200 | yes | 1 | 1950 ms | Streaming function tool call received. | -| `gpt-5.6-terra-2026-07-09` | responses | `max` | PASS | 200 | yes | 1 | 1266 ms | Streaming function tool call received. | -| `gemini-3-flash` | chat-completions | `default` | PASS | 200 | yes | 1 | 1959 ms | Streaming function tool call received. | -| `gemini-3-flash` | chat-completions | `minimal` | PASS | 200 | yes | 1 | 844 ms | Streaming function tool call received. | -| `gemini-3-flash` | chat-completions | `low` | PASS | 200 | yes | 1 | 847 ms | Streaming function tool call received. | -| `gemini-3-flash` | chat-completions | `medium` | PASS | 200 | yes | 1 | 1269 ms | Streaming function tool call received. | -| `gemini-3-flash` | chat-completions | `high` | PASS | 200 | yes | 1 | 1614 ms | Streaming function tool call received. | -| `gemini-3.1-pro` | chat-completions | `default` | PASS | 200 | yes | 1 | 5728 ms | Streaming function tool call received. | -| `gemini-3.1-pro` | chat-completions | `low` | PASS | 200 | yes | 1 | 2616 ms | Streaming function tool call received. | -| `gemini-3.1-pro` | chat-completions | `medium` | PASS | 200 | yes | 1 | 3136 ms | Streaming function tool call received. | -| `gemini-3.1-pro` | chat-completions | `high` | PASS | 200 | yes | 1 | 3194 ms | Streaming function tool call received. | -| `gemini-3.5-flash` | chat-completions | `default` | PASS | 200 | yes | 1 | 1660 ms | Streaming function tool call received. | -| `gemini-3.5-flash` | chat-completions | `minimal` | PASS | 200 | yes | 1 | 17292 ms | Streaming function tool call received. | -| `gemini-3.5-flash` | chat-completions | `low` | PASS | 200 | yes | 1 | 25835 ms | Streaming function tool call received. | -| `gemini-3.5-flash` | chat-completions | `medium` | PASS | 200 | yes | 1 | 15569 ms | Streaming function tool call received. | -| `gemini-3.5-flash` | chat-completions | `high` | PASS | 200 | yes | 1 | 16069 ms | Streaming function tool call received. | -| `claude-4-5-sonnet` | chat-completions | `default` | PASS | 200 | yes | 1 | 1464 ms | Streaming function tool call received. | -| `claude-sonnet-4-6` | messages | `default` | PASS | 200 | yes | 1 | 3260 ms | Streaming function tool call received. | -| `claude-sonnet-4-6` | messages | `low` | PASS | 200 | yes | 1 | 2286 ms | Streaming function tool call received. | -| `claude-sonnet-4-6` | messages | `medium` | PASS | 200 | yes | 1 | 1935 ms | Streaming function tool call received. | -| `claude-sonnet-4-6` | messages | `high` | PASS | 200 | yes | 1 | 1793 ms | Streaming function tool call received. | -| `claude-sonnet-4-6` | messages | `max` | PASS | 200 | yes | 1 | 1852 ms | Streaming function tool call received. | -| `claude-sonnet-5` | messages | `default` | PASS | 200 | yes | 1 | 1829 ms | Streaming function tool call received. | -| `claude-sonnet-5` | messages | `low` | PASS | 200 | yes | 1 | 1616 ms | Streaming function tool call received. | -| `claude-sonnet-5` | messages | `medium` | PASS | 200 | yes | 1 | 1987 ms | Streaming function tool call received. | -| `claude-sonnet-5` | messages | `high` | PASS | 200 | yes | 1 | 2920 ms | Streaming function tool call received. | -| `claude-sonnet-5` | messages | `xhigh` | PASS | 200 | yes | 1 | 1918 ms | Streaming function tool call received. | -| `claude-sonnet-5` | messages | `max` | PASS | 200 | yes | 1 | 1762 ms | Streaming function tool call received. | -| `claude-opus-4-5-20251101` | messages | `default` | PASS | 200 | yes | 1 | 2114 ms | Streaming function tool call received. | -| `claude-opus-4-5-20251101` | messages | `low` | PASS | 200 | yes | 1 | 1635 ms | Streaming function tool call received. | -| `claude-opus-4-5-20251101` | messages | `medium` | PASS | 200 | yes | 1 | 1844 ms | Streaming function tool call received. | -| `claude-opus-4-5-20251101` | messages | `high` | PASS | 200 | yes | 1 | 1720 ms | Streaming function tool call received. | -| `claude-opus-4-6-20260205` | messages | `default` | PASS | 200 | yes | 1 | 1762 ms | Streaming function tool call received. | -| `claude-opus-4-6-20260205` | messages | `low` | PASS | 200 | yes | 1 | 1919 ms | Streaming function tool call received. | -| `claude-opus-4-6-20260205` | messages | `medium` | PASS | 200 | yes | 1 | 2658 ms | Streaming function tool call received. | -| `claude-opus-4-6-20260205` | messages | `high` | PASS | 200 | yes | 1 | 1746 ms | Streaming function tool call received. | -| `claude-opus-4-6-20260205` | messages | `max` | PASS | 200 | yes | 1 | 1749 ms | Streaming function tool call received. | -| `claude-opus-4-7` | messages | `default` | PASS | 200 | yes | 1 | 1433 ms | Streaming function tool call received. | -| `claude-opus-4-7` | messages | `low` | PASS | 200 | yes | 1 | 1489 ms | Streaming function tool call received. | -| `claude-opus-4-7` | messages | `medium` | PASS | 200 | yes | 1 | 2842 ms | Streaming function tool call received. | -| `claude-opus-4-7` | messages | `high` | PASS | 200 | yes | 1 | 1359 ms | Streaming function tool call received. | -| `claude-opus-4-7` | messages | `xhigh` | PASS | 200 | yes | 1 | 2265 ms | Streaming function tool call received. | -| `claude-opus-4-7` | messages | `max` | PASS | 200 | yes | 1 | 1312 ms | Streaming function tool call received. | -| `claude-opus-4-8` | messages | `default` | PASS | 200 | yes | 1 | 1777 ms | Streaming function tool call received. | -| `claude-opus-4-8` | messages | `low` | PASS | 200 | yes | 1 | 2838 ms | Streaming function tool call received. | -| `claude-opus-4-8` | messages | `medium` | PASS | 200 | yes | 1 | 1640 ms | Streaming function tool call received. | -| `claude-opus-4-8` | messages | `high` | PASS | 200 | yes | 1 | 1518 ms | Streaming function tool call received. | -| `claude-opus-4-8` | messages | `xhigh` | PASS | 200 | yes | 1 | 1535 ms | Streaming function tool call received. | -| `claude-opus-4-8` | messages | `max` | PASS | 200 | yes | 1 | 1778 ms | Streaming function tool call received. | -| `claude-haiku-4-5-20251001` | chat-completions | `default` | PASS | 200 | yes | 1 | 1941 ms | Streaming function tool call received. | -| `qwen.qwen3-coder-30b-a3b-v1` | chat-completions | `default` | PASS | 200 | yes | 1 | 1011 ms | Streaming function tool call received. | -| `qwen.qwen3-coder-480b-a35b-v1` | chat-completions | `default` | PASS | 200 | yes | 1 | 454 ms | Streaming function tool call received. | -| `moonshotai.kimi-k2.5` | chat-completions | `default` | PASS | 200 | yes | 1 | 615 ms | Streaming function tool call received. | - -## Advertised Configuration - -| Model | API | Advertised efforts | VS Code model options | -|---|---|---|---| -| `claude-sonnet-4-5-20250929` | chat-completions | none | `{"top_p":null}` | -| `gpt-4.1` | chat-completions | none | default | -| `gpt-4.1-mini` | chat-completions | none | default | -| `gpt-5-2025-08-07` | chat-completions | minimal, low, medium, high | default | -| `gpt-5-mini-2025-08-07` | chat-completions | minimal, low, medium, high | default | -| `gpt-5-nano-2025-08-07` | chat-completions | minimal, low, medium, high | default | -| `gpt-5-1-codex-2025-11-13` | responses | low, medium, high | default | -| `gpt-5-2-2025-12-11` | chat-completions | none, low, medium, high | default | -| `gpt-5.4-2026-03-05` | chat-completions | none, low, medium, high, xhigh | default | -| `gpt-5.5-2026-04-24` | responses | none, low, medium, high, xhigh | default | -| `gpt-5.6-luna-2026-07-09` | responses | none, low, medium, high, xhigh, max | default | -| `gpt-5.6-sol-2026-07-09` | responses | none, low, medium, high, xhigh, max | default | -| `gpt-5.6-terra-2026-07-09` | responses | none, low, medium, high, xhigh, max | default | -| `gemini-3-flash` | chat-completions | minimal, low, medium, high | default | -| `gemini-3.1-pro` | chat-completions | low, medium, high | default | -| `gemini-3.5-flash` | chat-completions | minimal, low, medium, high | default | -| `claude-4-5-sonnet` | chat-completions | none | `{"top_p":null}` | -| `claude-sonnet-4-6` | messages | low, medium, high, max | default | -| `claude-sonnet-5` | messages | low, medium, high, xhigh, max | default | -| `claude-opus-4-5-20251101` | messages | low, medium, high | default | -| `claude-opus-4-6-20260205` | messages | low, medium, high, max | default | -| `claude-opus-4-7` | messages | low, medium, high, xhigh, max | default | -| `claude-opus-4-8` | messages | low, medium, high, xhigh, max | default | -| `claude-haiku-4-5-20251001` | chat-completions | none | `{"top_p":null}` | -| `qwen.qwen3-coder-30b-a3b-v1` | chat-completions | none | default | -| `qwen.qwen3-coder-480b-a35b-v1` | chat-completions | none | default | -| `moonshotai.kimi-k2.5` | chat-completions | none | default | - -## Interpretation - -- `default` means no explicit reasoning-effort parameter was sent. -- Every advertised effort was sent using the model’s configured API body shape. -- PASS requires both a successful HTTP response and a streamed function tool call. -- A model should remain in the released catalog only when every advertised row passes. - -## Documentation - -- [VS Code custom endpoint model configuration](https://code.visualstudio.com/docs/agent-customization/language-models) -- [OpenAI model and reasoning guidance](https://developers.openai.com/api/docs/models) -- [AWS Claude Messages request parameters](https://docs.aws.amazon.com/bedrock/latest/userguide/model-parameters-anthropic-claude-messages-request-response.html) diff --git a/src/cli/commands/proxy/connectors/__tests__/vscode.test.ts b/src/cli/commands/proxy/connectors/__tests__/vscode.test.ts index 838123df..bda3528a 100644 --- a/src/cli/commands/proxy/connectors/__tests__/vscode.test.ts +++ b/src/cli/commands/proxy/connectors/__tests__/vscode.test.ts @@ -22,7 +22,6 @@ const EXPECTED_MODEL_IDS = [ 'gpt-5.4-2026-03-05', 'gpt-5.5-2026-04-24', 'gpt-5.6-luna-2026-07-09', - 'gpt-5.6-sol-2026-07-09', 'gpt-5.6-terra-2026-07-09', 'gemini-3-flash', 'gemini-3.1-pro', @@ -115,7 +114,6 @@ describe('writeVsCodeLanguageModelsConfigAtPath', () => { } expect(model).toEqual(expected); - expect(model).not.toHaveProperty('releaseSlice'); } }); diff --git a/src/cli/commands/proxy/connectors/vscode-models.ts b/src/cli/commands/proxy/connectors/vscode-models.ts index 3296695c..9ce2d907 100644 --- a/src/cli/commands/proxy/connectors/vscode-models.ts +++ b/src/cli/commands/proxy/connectors/vscode-models.ts @@ -10,7 +10,6 @@ export type VsCodeReasoningEffort = | 'max'; export interface VsCodeModelDefinition { - releaseSlice: 1 | 2 | 3; id: string; apiType: VsCodeApiType; vision: boolean; @@ -43,7 +42,6 @@ const MESSAGE_AUTH_HEADERS = { export const VS_CODE_SUPPORTED_MODELS: readonly VsCodeModelDefinition[] = [ { - releaseSlice: 3, id: 'claude-sonnet-4-5-20250929', apiType: 'chat-completions', vision: true, @@ -53,7 +51,6 @@ export const VS_CODE_SUPPORTED_MODELS: readonly VsCodeModelDefinition[] = [ maxOutputTokens: 64000, }, { - releaseSlice: 1, id: 'gpt-4.1', apiType: 'chat-completions', vision: true, @@ -62,7 +59,6 @@ export const VS_CODE_SUPPORTED_MODELS: readonly VsCodeModelDefinition[] = [ maxOutputTokens: 32768, }, { - releaseSlice: 1, id: 'gpt-4.1-mini', apiType: 'chat-completions', vision: true, @@ -71,7 +67,6 @@ export const VS_CODE_SUPPORTED_MODELS: readonly VsCodeModelDefinition[] = [ maxOutputTokens: 32768, }, { - releaseSlice: 1, id: 'gpt-5-2025-08-07', apiType: 'chat-completions', vision: true, @@ -82,7 +77,6 @@ export const VS_CODE_SUPPORTED_MODELS: readonly VsCodeModelDefinition[] = [ maxOutputTokens: 128000, }, { - releaseSlice: 1, id: 'gpt-5-mini-2025-08-07', apiType: 'chat-completions', vision: true, @@ -93,7 +87,6 @@ export const VS_CODE_SUPPORTED_MODELS: readonly VsCodeModelDefinition[] = [ maxOutputTokens: 128000, }, { - releaseSlice: 1, id: 'gpt-5-nano-2025-08-07', apiType: 'chat-completions', vision: true, @@ -104,7 +97,6 @@ export const VS_CODE_SUPPORTED_MODELS: readonly VsCodeModelDefinition[] = [ maxOutputTokens: 128000, }, { - releaseSlice: 2, id: 'gpt-5-1-codex-2025-11-13', apiType: 'responses', vision: true, @@ -115,7 +107,6 @@ export const VS_CODE_SUPPORTED_MODELS: readonly VsCodeModelDefinition[] = [ maxOutputTokens: 128000, }, { - releaseSlice: 2, id: 'gpt-5-2-2025-12-11', apiType: 'chat-completions', vision: true, @@ -126,7 +117,6 @@ export const VS_CODE_SUPPORTED_MODELS: readonly VsCodeModelDefinition[] = [ maxOutputTokens: 128000, }, { - releaseSlice: 2, id: 'gpt-5.4-2026-03-05', apiType: 'chat-completions', vision: true, @@ -137,7 +127,6 @@ export const VS_CODE_SUPPORTED_MODELS: readonly VsCodeModelDefinition[] = [ maxOutputTokens: 128000, }, { - releaseSlice: 2, id: 'gpt-5.5-2026-04-24', apiType: 'responses', vision: true, @@ -148,7 +137,6 @@ export const VS_CODE_SUPPORTED_MODELS: readonly VsCodeModelDefinition[] = [ maxOutputTokens: 128000, }, { - releaseSlice: 2, id: 'gpt-5.6-luna-2026-07-09', apiType: 'responses', vision: true, @@ -158,19 +146,10 @@ export const VS_CODE_SUPPORTED_MODELS: readonly VsCodeModelDefinition[] = [ maxInputTokens: 922000, maxOutputTokens: 128000, }, + // gpt-5.6-sol-2026-07-09 is intentionally excluded: its CodeMie Responses + // route rejects minimal requests, while Chat rejects tools with reasoning. + // Re-add only after the Responses route passes live VS Code certification. { - releaseSlice: 2, - id: 'gpt-5.6-sol-2026-07-09', - apiType: 'responses', - vision: true, - thinking: true, - supportsReasoningEffort: GPT_5_6_EFFORTS, - reasoningEffortFormat: 'responses', - maxInputTokens: 922000, - maxOutputTokens: 128000, - }, - { - releaseSlice: 2, id: 'gpt-5.6-terra-2026-07-09', apiType: 'responses', vision: true, @@ -181,7 +160,6 @@ export const VS_CODE_SUPPORTED_MODELS: readonly VsCodeModelDefinition[] = [ maxOutputTokens: 128000, }, { - releaseSlice: 3, id: 'gemini-3-flash', apiType: 'chat-completions', vision: true, @@ -192,7 +170,6 @@ export const VS_CODE_SUPPORTED_MODELS: readonly VsCodeModelDefinition[] = [ maxOutputTokens: 65536, }, { - releaseSlice: 3, id: 'gemini-3.1-pro', apiType: 'chat-completions', vision: true, @@ -203,7 +180,6 @@ export const VS_CODE_SUPPORTED_MODELS: readonly VsCodeModelDefinition[] = [ maxOutputTokens: 65536, }, { - releaseSlice: 3, id: 'gemini-3.5-flash', apiType: 'chat-completions', vision: true, @@ -214,7 +190,6 @@ export const VS_CODE_SUPPORTED_MODELS: readonly VsCodeModelDefinition[] = [ maxOutputTokens: 65536, }, { - releaseSlice: 3, id: 'claude-4-5-sonnet', apiType: 'chat-completions', vision: true, @@ -224,7 +199,6 @@ export const VS_CODE_SUPPORTED_MODELS: readonly VsCodeModelDefinition[] = [ maxOutputTokens: 64000, }, { - releaseSlice: 3, id: 'claude-sonnet-4-6', apiType: 'messages', vision: true, @@ -236,7 +210,6 @@ export const VS_CODE_SUPPORTED_MODELS: readonly VsCodeModelDefinition[] = [ maxOutputTokens: 64000, }, { - releaseSlice: 3, id: 'claude-sonnet-5', apiType: 'messages', vision: true, @@ -248,7 +221,6 @@ export const VS_CODE_SUPPORTED_MODELS: readonly VsCodeModelDefinition[] = [ maxOutputTokens: 128000, }, { - releaseSlice: 3, id: 'claude-opus-4-5-20251101', apiType: 'messages', vision: true, @@ -259,7 +231,6 @@ export const VS_CODE_SUPPORTED_MODELS: readonly VsCodeModelDefinition[] = [ maxOutputTokens: 64000, }, { - releaseSlice: 3, id: 'claude-opus-4-6-20260205', apiType: 'messages', vision: true, @@ -271,7 +242,6 @@ export const VS_CODE_SUPPORTED_MODELS: readonly VsCodeModelDefinition[] = [ maxOutputTokens: 128000, }, { - releaseSlice: 3, id: 'claude-opus-4-7', apiType: 'messages', vision: true, @@ -283,7 +253,6 @@ export const VS_CODE_SUPPORTED_MODELS: readonly VsCodeModelDefinition[] = [ maxOutputTokens: 128000, }, { - releaseSlice: 3, id: 'claude-opus-4-8', apiType: 'messages', vision: true, @@ -295,7 +264,6 @@ export const VS_CODE_SUPPORTED_MODELS: readonly VsCodeModelDefinition[] = [ maxOutputTokens: 128000, }, { - releaseSlice: 3, id: 'claude-haiku-4-5-20251001', apiType: 'chat-completions', vision: true, @@ -305,7 +273,6 @@ export const VS_CODE_SUPPORTED_MODELS: readonly VsCodeModelDefinition[] = [ maxOutputTokens: 64000, }, { - releaseSlice: 1, id: 'qwen.qwen3-coder-30b-a3b-v1', apiType: 'chat-completions', vision: false, @@ -314,7 +281,6 @@ export const VS_CODE_SUPPORTED_MODELS: readonly VsCodeModelDefinition[] = [ maxOutputTokens: 16384, }, { - releaseSlice: 1, id: 'qwen.qwen3-coder-480b-a35b-v1', apiType: 'chat-completions', vision: false, @@ -323,7 +289,6 @@ export const VS_CODE_SUPPORTED_MODELS: readonly VsCodeModelDefinition[] = [ maxOutputTokens: 16384, }, { - releaseSlice: 1, id: 'moonshotai.kimi-k2.5', apiType: 'chat-completions', vision: true, diff --git a/src/providers/plugins/sso/proxy/plugins/index.ts b/src/providers/plugins/sso/proxy/plugins/index.ts index 0f3fdddc..d8eafb75 100644 --- a/src/providers/plugins/sso/proxy/plugins/index.ts +++ b/src/providers/plugins/sso/proxy/plugins/index.ts @@ -16,6 +16,7 @@ import { RequestSanitizerPlugin } from './request-sanitizer.plugin.js'; import { ClaudeRequestNormalizerPlugin } from './claude-request-normalizer.plugin.js'; import { KimiRequestNormalizerPlugin } from './kimi-request-normalizer.plugin.js'; import { CodexEncryptedContentSanitizerPlugin } from './codex-encrypted-content-sanitizer.plugin.js'; +import { VsCodeRequestNormalizerPlugin } from './vscode-request-normalizer.plugin.js'; import { LoggingPlugin } from './logging.plugin.js'; import { SSOSessionSyncPlugin } from './sso.session-sync.plugin.js'; @@ -36,6 +37,7 @@ export function registerCorePlugins(): void { registry.register(new KimiRequestNormalizerPlugin()); // Priority 14 - caps Kimi output token requests for upstream limits registry.register(new RequestSanitizerPlugin()); // Priority 15 - strips unsupported reasoning params registry.register(new CodexEncryptedContentSanitizerPlugin()); // Priority 16 - strips encrypted reasoning state for Codex + registry.register(new VsCodeRequestNormalizerPlugin()); // Priority 17 - constrains VS Code user identifiers registry.register(new HeaderInjectionPlugin()); registry.register(new LoggingPlugin()); // Always enabled - logs to log files at INFO level registry.register(new SSOSessionSyncPlugin()); // Priority 100 - syncs sessions via multiple processors @@ -56,6 +58,7 @@ export { ClaudeRequestNormalizerPlugin, KimiRequestNormalizerPlugin, CodexEncryptedContentSanitizerPlugin, + VsCodeRequestNormalizerPlugin, LoggingPlugin, }; export { SSOSessionSyncPlugin } from './sso.session-sync.plugin.js'; diff --git a/src/providers/plugins/sso/proxy/plugins/vscode-request-normalizer.plugin.ts b/src/providers/plugins/sso/proxy/plugins/vscode-request-normalizer.plugin.ts new file mode 100644 index 00000000..0cf79d2a --- /dev/null +++ b/src/providers/plugins/sso/proxy/plugins/vscode-request-normalizer.plugin.ts @@ -0,0 +1,85 @@ +/** + * VS Code Request Normalizer + * Priority: 17 (after request sanitizers, before header injection) + * + * Azure-backed OpenAI routes reject `user` identifiers longer than 64 + * characters. The CodeMie Responses path can namespace the supplied value, + * so keep the client portion at or below 32 characters. Longer identifiers + * become a stable truncated SHA-256 digest. When VS Code omits the field, send + * a short explicit fallback to prevent the upstream from deriving an + * overlength default identity. + * + * Scope: VS Code BYOK Responses API traffic only. + */ + +import { createHash } from 'node:crypto'; +import { ProxyPlugin, PluginContext, ProxyInterceptor } from './types.js'; +import { ProxyContext } from '../proxy-types.js'; +import { logger } from '../../../../../utils/logger.js'; + +const VS_CODE_CLIENT_TYPE = 'vscode-byok'; +const MAX_CLIENT_USER_LENGTH = 32; +const FALLBACK_USER = 'vscode-byok'; + +export class VsCodeRequestNormalizerPlugin implements ProxyPlugin { + id = '@codemie/proxy-vscode-request-normalizer'; + name = 'VS Code Request Normalizer'; + version = '1.0.0'; + priority = 17; + + async createInterceptor(context: PluginContext): Promise { + if (context.config.clientType !== VS_CODE_CLIENT_TYPE) { + throw new Error(`Plugin disabled for agent: ${context.config.clientType}`); + } + + return new VsCodeRequestNormalizerInterceptor(); + } +} + +class VsCodeRequestNormalizerInterceptor implements ProxyInterceptor { + name = 'vscode-request-normalizer'; + + async onRequest(context: ProxyContext): Promise { + if (!context.requestBody || !context.headers['content-type']?.includes('application/json')) { + return; + } + + try { + const path = context.url.split('?')[0].replace(/\/+$/, ''); + if (!path.endsWith('/responses')) { + return; + } + + const body = JSON.parse(context.requestBody.toString('utf-8')) as Record; + const existingUser = typeof body.user === 'string' ? body.user : ''; + const normalizedUser = existingUser.length === 0 + ? FALLBACK_USER + : existingUser.length <= MAX_CLIENT_USER_LENGTH + ? existingUser + : createHash('sha256') + .update(existingUser, 'utf-8') + .digest('hex') + .slice(0, MAX_CLIENT_USER_LENGTH); + + if (body.user === normalizedUser) { + return; + } + + body.user = normalizedUser; + + const normalizedBody = JSON.stringify(body); + context.requestBody = Buffer.from(normalizedBody, 'utf-8'); + context.headers['content-length'] = String(context.requestBody.length); + + logger.debug( + `[${this.name}] Normalized Responses user identifier`, + { + originalLength: existingUser.length, + normalizedLength: normalizedUser.length, + } + ); + } catch { + // Not valid JSON or unexpected structure — pass through unchanged. + } + } +} diff --git a/tests/integration/vscode-models.live.test.ts b/tests/integration/vscode-models.live.test.ts index ade306a8..afc9669c 100644 --- a/tests/integration/vscode-models.live.test.ts +++ b/tests/integration/vscode-models.live.test.ts @@ -3,7 +3,7 @@ * * Required: * CODEMIE_VSCODE_LIVE=1 - * CODEMIE_VSCODE_MODELS= + * CODEMIE_VSCODE_MODELS= * * Authentication: * Either start a VS Code proxy with `codemie proxy connect vscode`, or set: @@ -86,12 +86,6 @@ function getApiPath(definition: VsCodeModelDefinition): string { function selectModels(selector: string): readonly VsCodeModelDefinition[] { if (selector === 'all') return VS_CODE_SUPPORTED_MODELS; - const sliceMatch = /^slice:([123])$/.exec(selector); - if (sliceMatch) { - const slice = Number(sliceMatch[1]); - return VS_CODE_SUPPORTED_MODELS.filter(model => model.releaseSlice === slice); - } - const requestedIds = selector.split(',').map(value => value.trim()).filter(Boolean); const selected = VS_CODE_SUPPORTED_MODELS.filter(model => requestedIds.includes(model.id)); const selectedIds = new Set(selected.map(model => model.id)); From f0a79f78a62358ff4a9812d9192ac5e2747513ea Mon Sep 17 00:00:00 2001 From: Roman Sinyakov Date: Fri, 24 Jul 2026 14:48:25 +0200 Subject: [PATCH 4/4] docs(proxy): update VS Code setup instructions --- docs/COMMANDS.md | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/docs/COMMANDS.md b/docs/COMMANDS.md index d21a3264..3616a68f 100644 --- a/docs/COMMANDS.md +++ b/docs/COMMANDS.md @@ -91,10 +91,12 @@ The SSO credentials remain in CodeMie and are never added to VS Code. The connec For a new provider or an invalid existing key, the command prints this required one-time action: ```text -Press ⇧⌘P (macOS) or Ctrl+Shift+P (Windows/Linux) -Run Chat: Manage Language Models -Right-click CodeMie Profile Model → Update API Key -Enter API key: codemie-proxy +One-time VS Code secret setup required: +1. Open VS Code and Press ⇧⌘P (macOS) or Ctrl+Shift+P (Windows/Linux). +2. Find Chat: Manage Language Models +3. In opened dialog Right-click any CodeMie model → Update API Key +4. Enter API key: codemie-proxy +Reload VS Code, then select a CodeMie model from the model picker ``` VS Code stores that local key in its secret storage and adds the reference to the configuration. The CLI cannot inspect whether an existing secret reference still resolves; if VS Code reports a missing or invalid key, use **Update API Key** again.