From b356a16e7afdc240abbef6e8dcebb6959e640bec Mon Sep 17 00:00:00 2001 From: warelik Date: Sat, 22 Aug 2026 01:44:50 +0300 Subject: [PATCH] test(degradation): add e2e doctrine harness for translator registry - TestDegradationRequestEffortMapping exercises all 33 registered request pairs (stream and non-stream) for high and no-reasoning payload shapes. - TestDegradationResponseDoctrines marks the four known current-main violations with their open PRs: #190, #191, #193. --- test/e2e_degradation_doctrine_test.go | 388 ++++++++++++++++++++++++++ 1 file changed, 388 insertions(+) create mode 100644 test/e2e_degradation_doctrine_test.go diff --git a/test/e2e_degradation_doctrine_test.go b/test/e2e_degradation_doctrine_test.go new file mode 100644 index 000000000..7a75ced12 --- /dev/null +++ b/test/e2e_degradation_doctrine_test.go @@ -0,0 +1,388 @@ +package test + +import ( + "context" + "fmt" + "strings" + "testing" + + _ "github.com/router-for-me/CLIProxyAPI/v7/internal/thinking/provider/antigravity" + _ "github.com/router-for-me/CLIProxyAPI/v7/internal/thinking/provider/claude" + _ "github.com/router-for-me/CLIProxyAPI/v7/internal/thinking/provider/codex" + _ "github.com/router-for-me/CLIProxyAPI/v7/internal/thinking/provider/gemini" + _ "github.com/router-for-me/CLIProxyAPI/v7/internal/thinking/provider/interactions" + _ "github.com/router-for-me/CLIProxyAPI/v7/internal/thinking/provider/kimi" + _ "github.com/router-for-me/CLIProxyAPI/v7/internal/thinking/provider/openai" + _ "github.com/router-for-me/CLIProxyAPI/v7/internal/thinking/provider/xai" + _ "github.com/router-for-me/CLIProxyAPI/v7/internal/translator" + sdktranslator "github.com/router-for-me/CLIProxyAPI/v7/sdk/translator" + "github.com/tidwall/gjson" +) + +// knownFormats are the schema identifiers observed in the default translator +// registry. Use Has*Transformer to discover which (from,to) pairs are wired. +var knownFormats = []string{ + "openai", + "openai-response", + "claude", + "gemini", + "codex", + "antigravity", + "interactions", + "kiro", +} + +// requestProbe describes a source payload for the high (reasoning enabled) and +// none (reasoning disabled) effort levels used to test (a). +type requestProbe struct { + high []byte + none []byte +} + +var requestProbes = map[string]requestProbe{ + "openai": { + high: []byte(`{"reasoning_effort":"high","messages":[{"role":"user","content":"hi"}]}`), + none: []byte(`{"reasoning_effort":"none","messages":[{"role":"user","content":"hi"}]}`), + }, + "openai-response": { + high: []byte(`{"reasoning":{"effort":"high","summary":"auto"},"input":"hi"}`), + none: []byte(`{"reasoning":{"effort":"none","summary":null},"input":"hi"}`), + }, + "claude": { + high: []byte(`{"thinking":{"type":"enabled","budget_tokens":24576,"display":"summarized"},"messages":[{"role":"user","content":"hi"}]}`), + none: []byte(`{"thinking":{"type":"disabled"},"messages":[{"role":"user","content":"hi"}]}`), + }, + "gemini": { + high: []byte(`{"generationConfig":{"thinkingConfig":{"thinkingLevel":"high","includeThoughts":true}},"contents":[{"role":"user","parts":[{"text":"hi"}]}]}`), + none: []byte(`{"generationConfig":{"thinkingConfig":{"thinkingLevel":"none","includeThoughts":false}},"contents":[{"role":"user","parts":[{"text":"hi"}]}]}`), + }, + "antigravity": { + high: []byte(`{"request":{"generationConfig":{"thinkingConfig":{"thinkingLevel":"high","includeThoughts":true}},"contents":[{"role":"user","parts":[{"text":"hi"}]}]}}`), + none: []byte(`{"request":{"generationConfig":{"thinkingConfig":{"thinkingLevel":"none","includeThoughts":false}},"contents":[{"role":"user","parts":[{"text":"hi"}]}]}}`), + }, + "interactions": { + high: []byte(`{"generation_config":{"thinking_level":"high","thinking_summaries":"auto"},"input":"hi"}`), + none: []byte(`{"generation_config":{"thinking_level":"none","thinking_summaries":"none"},"input":"hi"}`), + }, + "codex": { + high: []byte(`{"reasoning":{"effort":"high","summary":"auto"},"messages":[{"role":"user","content":"hi"}]}`), + none: []byte(`{"reasoning":{"effort":"none","summary":null},"messages":[{"role":"user","content":"hi"}]}`), + }, + "kiro": { + high: []byte(`{"reasoning_effort":"high","messages":[{"role":"user","content":"hi"}]}`), + none: []byte(`{"reasoning_effort":"none","messages":[{"role":"user","content":"hi"}]}`), + }, +} + +// targetEffortCheck verifies that output in the given target format reflects +// the requested effort level (high != none). +type targetEffortCheck struct { + high func(t *testing.T, out []byte) + none func(t *testing.T, out []byte) +} + +var targetEffortChecks = map[string]targetEffortCheck{ + "openai": { + high: func(t *testing.T, out []byte) { + effort := gjson.GetBytes(out, "reasoning_effort").String() + if effort == "" || effort == "none" { + t.Fatalf("openai high: reasoning_effort = %q; out=%s", effort, out) + } + }, + none: func(t *testing.T, out []byte) { + if gjson.GetBytes(out, "reasoning_effort").String() != "none" { + t.Fatalf("openai none: reasoning_effort missing/wrong; out=%s", out) + } + }, + }, + "openai-response": { + high: func(t *testing.T, out []byte) { + effort := gjson.GetBytes(out, "reasoning.effort").String() + if effort == "" || effort == "none" { + t.Fatalf("openai-response high: reasoning.effort = %q; out=%s", effort, out) + } + if gjson.GetBytes(out, "reasoning.summary").String() != "auto" { + t.Fatalf("openai-response high: reasoning.summary != auto; out=%s", out) + } + }, + none: func(t *testing.T, out []byte) { + if gjson.GetBytes(out, "reasoning.effort").String() != "none" { + t.Fatalf("openai-response none: reasoning.effort != none; out=%s", out) + } + if gjson.GetBytes(out, "reasoning.summary").Exists() { + t.Fatalf("openai-response none: reasoning.summary should be absent; out=%s", out) + } + }, + }, + "codex": { + high: func(t *testing.T, out []byte) { + effort := gjson.GetBytes(out, "reasoning.effort").String() + if effort == "" || effort == "none" { + t.Fatalf("codex high: reasoning.effort = %q; out=%s", effort, out) + } + if gjson.GetBytes(out, "reasoning.summary").String() != "auto" { + t.Fatalf("codex high: reasoning.summary != auto; out=%s", out) + } + }, + none: func(t *testing.T, out []byte) { + if gjson.GetBytes(out, "reasoning.effort").String() != "none" { + t.Fatalf("codex none: reasoning.effort != none; out=%s", out) + } + if gjson.GetBytes(out, "reasoning.summary").Exists() { + t.Fatalf("codex none: reasoning.summary should be absent; out=%s", out) + } + }, + }, + "claude": { + high: func(t *testing.T, out []byte) { + thinkingType := gjson.GetBytes(out, "thinking.type").String() + if thinkingType != "enabled" && thinkingType != "adaptive" { + t.Fatalf("claude high: thinking.type = %q; out=%s", thinkingType, out) + } + if gjson.GetBytes(out, "thinking.display").String() != "summarized" { + t.Fatalf("claude high: thinking.display != summarized; out=%s", out) + } + }, + none: func(t *testing.T, out []byte) { + if gjson.GetBytes(out, "thinking.type").String() != "disabled" { + t.Fatalf("claude none: thinking.type != disabled; out=%s", out) + } + if gjson.GetBytes(out, "thinking.display").Exists() { + t.Fatalf("claude none: thinking.display should be absent; out=%s", out) + } + }, + }, + "gemini": { + high: func(t *testing.T, out []byte) { + if !geminiThinkingEnabled(out, "generationConfig.thinkingConfig") { + t.Fatalf("gemini high: no enabled thinking config; out=%s", out) + } + }, + none: func(t *testing.T, out []byte) { + if !geminiThinkingDisabled(out, "generationConfig.thinkingConfig") { + t.Fatalf("gemini none: thinking not disabled; out=%s", out) + } + }, + }, + "antigravity": { + high: func(t *testing.T, out []byte) { + if !geminiThinkingEnabled(out, "request.generationConfig.thinkingConfig") { + t.Fatalf("antigravity high: no enabled thinking config; out=%s", out) + } + }, + none: func(t *testing.T, out []byte) { + if !geminiThinkingDisabled(out, "request.generationConfig.thinkingConfig") { + t.Fatalf("antigravity none: thinking not disabled; out=%s", out) + } + }, + }, + "interactions": { + high: func(t *testing.T, out []byte) { + if gjson.GetBytes(out, "generation_config.thinking_summaries").String() != "auto" { + t.Fatalf("interactions high: thinking_summaries != auto; out=%s", out) + } + if gjson.GetBytes(out, "generation_config.thinking_level").String() != "high" && + gjson.GetBytes(out, "generation_config.thinking_config.thinking_budget").Int() <= 0 { + t.Fatalf("interactions high: no high thinking level/budget; out=%s", out) + } + }, + none: func(t *testing.T, out []byte) { + summaries := gjson.GetBytes(out, "generation_config.thinking_summaries").String() + if summaries != "none" && summaries != "" { + t.Fatalf("interactions none: thinking_summaries = %q; out=%s", summaries, out) + } + if gjson.GetBytes(out, "generation_config.thinking_level").String() != "none" && + gjson.GetBytes(out, "generation_config.thinking_config.thinking_budget").Int() != 0 { + t.Fatalf("interactions none: no none level/budget; out=%s", out) + } + }, + }, + "kiro": { + high: func(t *testing.T, out []byte) { + // Kiro request translators pass through the source format; the + // executor's payload builder later consumes these fields. + effort := gjson.GetBytes(out, "reasoning_effort").String() + if effort != "" && effort != "none" { + return + } + if thinkingType := gjson.GetBytes(out, "thinking.type").String(); thinkingType == "enabled" || thinkingType == "adaptive" { + return + } + t.Fatalf("kiro high: no reasoning intent found; out=%s", out) + }, + none: func(t *testing.T, out []byte) { + if gjson.GetBytes(out, "reasoning_effort").String() == "none" { + return + } + if gjson.GetBytes(out, "thinking.type").String() == "disabled" { + return + } + t.Fatalf("kiro none: reasoning not disabled; out=%s", out) + }, + }, +} + +func geminiThinkingEnabled(out []byte, prefix string) bool { + if gjson.GetBytes(out, prefix+".thinkingLevel").String() == "high" { + if gjson.GetBytes(out, prefix+".includeThoughts").String() == "true" { + return true + } + } + if budget := gjson.GetBytes(out, prefix+".thinkingBudget").Int(); budget > 0 { + if gjson.GetBytes(out, prefix+".includeThoughts").String() == "true" { + return true + } + } + return false +} + +func geminiThinkingDisabled(out []byte, prefix string) bool { + if gjson.GetBytes(out, prefix+".includeThoughts").String() == "true" { + return false + } + if gjson.GetBytes(out, prefix+".thinkingLevel").String() == "none" { + return true + } + if gjson.GetBytes(out, prefix+".thinkingBudget").Int() == 0 { + return true + } + return false +} + +func TestDegradationRequestEffortMapping(t *testing.T) { + for _, from := range knownFormats { + probe, ok := requestProbes[from] + if !ok { + continue + } + for _, to := range knownFormats { + fromF := sdktranslator.FromString(from) + toF := sdktranslator.FromString(to) + if !sdktranslator.HasRequestTransformer(fromF, toF) { + continue + } + check, ok := targetEffortChecks[to] + if !ok { + continue + } + for _, stream := range []bool{false, true} { + name := fmt.Sprintf("%s_to_%s_stream_%v", from, to, stream) + t.Run(name, func(t *testing.T) { + highOut := sdktranslator.TranslateRequest(fromF, toF, "doctrine-model", probe.high, stream) + check.high(t, highOut) + + noneOut := sdktranslator.TranslateRequest(fromF, toF, "doctrine-model", probe.none, stream) + check.none(t, noneOut) + }) + } + } + } +} + +// TestDegradationResponseDoctrines exercises (b)-(d) for response translators. +// Known current-main violations are skipped with the open PR that fixes them. +func TestDegradationResponseDoctrines(t *testing.T) { + cases := []struct { + name string + from string + to string + skipPR string + request []byte + response []byte + check func(t *testing.T, out []byte) + }{ + { + name: "openai_responses_reasoning_fallback", + from: "openai", + to: "openai-response", + skipPR: "#191", + request: []byte(`{"model":"o3-mini","reasoning":{"summary":"auto"},"messages":[{"role":"user","content":"hi"}]}`), + response: []byte(`{"id":"chatcmpl_r","object":"chat.completion","created":1773896263,"model":"o3-mini","choices":[{"index":0,"message":{"role":"assistant","content":"hello","reasoning":"Let me think"}}]}`), + check: func(t *testing.T, out []byte) { + if !gjson.GetBytes(out, "output.#(type==\"reasoning\")").Exists() { + t.Fatalf("reasoning item missing; out=%s", out) + } + }, + }, + { + name: "claude_openai_reasoning_content_canonical", + from: "claude", + to: "openai", + skipPR: "#193", + request: []byte(`{"model":"claude-opus-4-6","thinking":{"type":"adaptive","display":"summarized"},"messages":[{"role":"user","content":"hi"}]}`), + response: []byte(`{"id":"msg_123","type":"message","role":"assistant","model":"claude-opus-4-6","content":[{"type":"thinking","thinking":"First thought. Second thought.","signature":"sig"},{"type":"text","text":"Here is the solution."}],"stop_reason":"end_turn","usage":{"input_tokens":10,"output_tokens":20}}`), + check: func(t *testing.T, out []byte) { + if !gjson.GetBytes(out, "choices.0.message.reasoning_content").Exists() { + t.Fatalf("canonical reasoning_content missing; out=%s", out) + } + if gjson.GetBytes(out, "choices.0.message.reasoning").Exists() { + t.Fatalf("non-canonical reasoning field leaked; out=%s", out) + } + }, + }, + { + name: "gemini_claude_thoughtsignature_preserved", + from: "gemini", + to: "claude", + skipPR: "#190", + request: []byte(`{"model":"gemini-3.5-flash","contents":[{"role":"user","parts":[{"text":"hi"}]}]}`), + response: []byte(`{"responseId":"resp-test","modelVersion":"gemini-test","candidates":[{"content":{"role":"model","parts":[{"thought":true,"text":"thinking text","thoughtSignature":"sig-test"},{"text":"hello world"}]},"finishReason":"STOP"}],"usageMetadata":{"promptTokenCount":21,"candidatesTokenCount":1,"totalTokenCount":131,"thoughtsTokenCount":109}}`), + check: func(t *testing.T, out []byte) { + if got := gjson.GetBytes(out, "content.#(type==\"thinking\").signature").String(); got != "sig-test" { + t.Fatalf("thinking signature = %q, want sig-test; out=%s", got, out) + } + }, + }, + { + name: "gemini_claude_visible_text_with_signature_stays_text", + from: "gemini", + to: "claude", + skipPR: "#190", + request: []byte(`{"model":"gemini-3.5-flash","contents":[{"role":"user","parts":[{"text":"hi"}]}]}`), + response: []byte(`{"responseId":"resp-test","modelVersion":"gemini-test","candidates":[{"content":{"role":"model","parts":[{"text":"hello world","thoughtSignature":"sig-carrier"}]},"finishReason":"STOP"}],"usageMetadata":{"promptTokenCount":21,"candidatesTokenCount":1,"totalTokenCount":131}}`), + check: func(t *testing.T, out []byte) { + if !gjson.GetBytes(out, "content.#(type==\"text\")").Exists() { + t.Fatalf("visible text block missing; out=%s", out) + } + if gjson.GetBytes(out, "content.#(type==\"thinking\")").Exists() { + t.Fatalf("visible text misrouted to thinking; out=%s", out) + } + }, + }, + } + + for _, tc := range cases { + for _, stream := range []bool{false, true} { + name := fmt.Sprintf("%s/stream=%v", tc.name, stream) + t.Run(name, func(t *testing.T) { + if tc.skipPR != "" { + t.Skipf("current main violates this doctrine; fix is %s", tc.skipPR) + } + fromF := sdktranslator.FromString(tc.from) + toF := sdktranslator.FromString(tc.to) + if !sdktranslator.HasResponseTransformer(fromF, toF) { + t.Skipf("no response transformer for %s -> %s", tc.from, tc.to) + } + + var out []byte + if stream { + var param any + chunks := sdktranslator.TranslateStream(context.Background(), fromF, toF, "doctrine-model", tc.request, tc.request, tc.response, ¶m) + if len(chunks) == 0 { + t.Fatal("no response chunks") + } + out = []byte(strings.Join(func() []string { + var s []string + for _, c := range chunks { + s = append(s, string(c)) + } + return s + }(), "\n")) + } else { + out = sdktranslator.TranslateNonStream(context.Background(), fromF, toF, "doctrine-model", tc.request, tc.request, tc.response, nil) + } + tc.check(t, out) + }) + } + } +}