Skip to content

test: conditional doctrine-harness skips #3

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
warelik merged 1 commit into from
Aug 22, 2026
Merged

test: conditional doctrine-harness skips #3

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
14 changes: 4 additions & 10 deletions sdk/cliproxy/auth/e2e_failover_doctrine_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -234,8 +234,6 @@ func newDoctrineManager(t *testing.T, executor *doctrineExecutor, authCount int)
// stays at 0 and no escalation occurs.
// Fix: Plus #198 floors the cooldown at the escalating quota ladder.
func TestSubSecondRetryAfterEscalatesOrRotates(t *testing.T) {
t.Skip("current main violates: sub-second 429 Retry-After bypasses the quota ladder and hammers the same auth (BackoffLevel stays 0). Enable after Plus #198 (quota ladder floor) merges.")

exec := newDoctrineExecutor("claude")
manager, ids, model := newDoctrineManager(t, exec, 1)
manager.SetRetryConfig(2, 1500*time.Millisecond, 5)
Expand All @@ -256,7 +254,7 @@ func TestSubSecondRetryAfterEscalatesOrRotates(t *testing.T) {
t.Fatal("auth disappeared")
}
if auth.Quota.BackoffLevel == 0 {
t.Fatalf("quota BackoffLevel = %d, want > 0 (escalated)", auth.Quota.BackoffLevel)
t.Skipf("current main violates: sub-second 429 Retry-After bypasses the quota ladder and hammers the same auth (BackoffLevel stays 0). Enable after Plus #198 (quota ladder floor) merges.")
}
}

Expand Down Expand Up @@ -354,8 +352,6 @@ func TestEmptyCompletionRotatesStream(t *testing.T) {
// conductor returns success, and the dead auth is not cooled.
// Fix: Plus #195 adds in-stream provider-error detection and rotation.
func TestInStreamProviderErrorDuringBootstrap(t *testing.T) {
t.Skip("current main violates: in-stream provider error envelopes inside a 200 SSE stream are forwarded as content instead of rotating the auth. Enable after Plus #195 (in-stream error failover) merges.")

exec := newDoctrineExecutor("claude")
manager, ids, model := newDoctrineManager(t, exec, 2)

Expand Down Expand Up @@ -389,7 +385,7 @@ func TestInStreamProviderErrorDuringBootstrap(t *testing.T) {
t.Fatal("first auth disappeared")
}
if !auth.Unavailable || auth.NextRetryAfter.IsZero() {
t.Fatalf("first auth should be cooled after in-stream 429, got unavailable=%v next=%v", auth.Unavailable, auth.NextRetryAfter)
t.Skipf("current main violates: in-stream provider error envelopes inside a 200 SSE stream are forwarded as content instead of rotating the auth. Enable after Plus #195 (in-stream error failover) merges.")
}
if exec.StreamCalls(ids[1]) == 0 {
t.Fatal("fallback auth was not tried")
Expand Down Expand Up @@ -477,8 +473,6 @@ func TestAffinityStaysHealthyAfterTransientBlip(t *testing.T) {
// alias is never discovered.
// Fix: Plus #208 resolves API-key model pools for all configured providers.
func TestAliasedAccountDiscoveredWhenSiblingsDie(t *testing.T) {
t.Skip("current main violates: API-key model aliases resolve to a single upstream model, so a sibling behind the alias is not discovered when the first fails. Enable after Plus #208 (multi-provider model pools) merges.")

cfg := &internalconfig.Config{
GeminiKey: []internalconfig.GeminiKey{{
APIKey: "doctrine-key",
Expand Down Expand Up @@ -517,15 +511,15 @@ func TestAliasedAccountDiscoveredWhenSiblingsDie(t *testing.T) {

resp, err := manager.Execute(context.Background(), []string{"gemini"}, cliproxyexecutor.Request{Model: "g25p"}, cliproxyexecutor.Options{})
if err != nil {
t.Fatalf("Execute should fall back to sibling alias model, got error = %v", err)
t.Skipf("current main violates: API-key model aliases resolve to a single upstream model, so a sibling behind the alias is not discovered when the first fails. Enable after Plus #208 (multi-provider model pools) merges.")
}
if !strings.Contains(string(resp.Payload), "ok") {
t.Fatalf("payload = %q, want sibling model content", string(resp.Payload))
}

models := exec.Models(auth.ID)
if len(models) < 2 {
t.Fatalf("executed models = %v, want both alias siblings", models)
t.Skipf("current main violates: API-key model aliases resolve to a single upstream model, so a sibling behind the alias is not discovered when the first fails. Enable after Plus #208 (multi-provider model pools) merges.")
}
if models[0] == models[1] {
t.Fatalf("alias pool rotated to the same model %q", models[0])
Expand Down
43 changes: 27 additions & 16 deletions test/e2e_degradation_doctrine_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -289,7 +289,7 @@ func TestDegradationResponseDoctrines(t *testing.T) {
skipPR string
request []byte
response []byte
check func(t *testing.T, out []byte)
check func(out []byte) string
}{
{
name: "openai_responses_reasoning_fallback",
Expand All @@ -298,10 +298,11 @@ func TestDegradationResponseDoctrines(t *testing.T) {
skipPR: "#191",
request: []byte(`{"model":"o3-mini","reasoning":{"summary":"auto"},"messages":[{"role":"user","content":"hi"}]}`),
response: []byte(`{"id":"chatcmpl_r","object":"chat.completion","created":1773896263,"model":"o3-mini","choices":[{"index":0,"message":{"role":"assistant","content":"hello","reasoning":"Let me think"}}]}`),
check: func(t *testing.T, out []byte) {
check: func(out []byte) string {
if !gjson.GetBytes(out, "output.#(type==\"reasoning\")").Exists() {
t.Fatalf("reasoning item missing; out=%s", out)
return fmt.Sprintf("reasoning item missing; out=%s", out)
}
return ""
},
},
{
Expand All @@ -311,13 +312,14 @@ func TestDegradationResponseDoctrines(t *testing.T) {
skipPR: "#193",
request: []byte(`{"model":"claude-opus-4-6","thinking":{"type":"adaptive","display":"summarized"},"messages":[{"role":"user","content":"hi"}]}`),
response: []byte(`{"id":"msg_123","type":"message","role":"assistant","model":"claude-opus-4-6","content":[{"type":"thinking","thinking":"First thought. Second thought.","signature":"sig"},{"type":"text","text":"Here is the solution."}],"stop_reason":"end_turn","usage":{"input_tokens":10,"output_tokens":20}}`),
check: func(t *testing.T, out []byte) {
check: func(out []byte) string {
if !gjson.GetBytes(out, "choices.0.message.reasoning_content").Exists() {
t.Fatalf("canonical reasoning_content missing; out=%s", out)
return fmt.Sprintf("canonical reasoning_content missing; out=%s", out)
}
if gjson.GetBytes(out, "choices.0.message.reasoning").Exists() {
t.Fatalf("non-canonical reasoning field leaked; out=%s", out)
return fmt.Sprintf("non-canonical reasoning field leaked; out=%s", out)
}
return ""
},
},
{
Expand All @@ -327,10 +329,11 @@ func TestDegradationResponseDoctrines(t *testing.T) {
skipPR: "#190",
request: []byte(`{"model":"gemini-3.5-flash","contents":[{"role":"user","parts":[{"text":"hi"}]}]}`),
response: []byte(`{"responseId":"resp-test","modelVersion":"gemini-test","candidates":[{"content":{"role":"model","parts":[{"thought":true,"text":"thinking text","thoughtSignature":"sig-test"},{"text":"hello world"}]},"finishReason":"STOP"}],"usageMetadata":{"promptTokenCount":21,"candidatesTokenCount":1,"totalTokenCount":131,"thoughtsTokenCount":109}}`),
check: func(t *testing.T, out []byte) {
check: func(out []byte) string {
if got := gjson.GetBytes(out, "content.#(type==\"thinking\").signature").String(); got != "sig-test" {
t.Fatalf("thinking signature = %q, want sig-test; out=%s", got, out)
return fmt.Sprintf("thinking signature = %q, want sig-test; out=%s", got, out)
}
return ""
},
},
{
Expand All @@ -340,13 +343,14 @@ func TestDegradationResponseDoctrines(t *testing.T) {
skipPR: "#190",
request: []byte(`{"model":"gemini-3.5-flash","contents":[{"role":"user","parts":[{"text":"hi"}]}]}`),
response: []byte(`{"responseId":"resp-test","modelVersion":"gemini-test","candidates":[{"content":{"role":"model","parts":[{"text":"hello world","thoughtSignature":"sig-carrier"}]},"finishReason":"STOP"}],"usageMetadata":{"promptTokenCount":21,"candidatesTokenCount":1,"totalTokenCount":131}}`),
check: func(t *testing.T, out []byte) {
check: func(out []byte) string {
if !gjson.GetBytes(out, "content.#(type==\"text\")").Exists() {
t.Fatalf("visible text block missing; out=%s", out)
return fmt.Sprintf("visible text block missing; out=%s", out)
}
if gjson.GetBytes(out, "content.#(type==\"thinking\")").Exists() {
t.Fatalf("visible text misrouted to thinking; out=%s", out)
return fmt.Sprintf("visible text misrouted to thinking; out=%s", out)
}
return ""
},
},
}
Expand All @@ -355,9 +359,6 @@ func TestDegradationResponseDoctrines(t *testing.T) {
for _, stream := range []bool{false, true} {
name := fmt.Sprintf("%s/stream=%v", tc.name, stream)
t.Run(name, func(t *testing.T) {
if tc.skipPR != "" {
t.Skipf("current main violates this doctrine; fix is %s", tc.skipPR)
}
fromF := sdktranslator.FromString(tc.from)
toF := sdktranslator.FromString(tc.to)
if !sdktranslator.HasResponseTransformer(fromF, toF) {
Expand All @@ -369,7 +370,11 @@ func TestDegradationResponseDoctrines(t *testing.T) {
var param any
chunks := sdktranslator.TranslateStream(context.Background(), fromF, toF, "doctrine-model", tc.request, tc.request, tc.response, &param)
if len(chunks) == 0 {
t.Fatal("no response chunks")
if tc.skipPR != "" {
t.Skipf("current main violates this doctrine; fix is %s: no response chunks", tc.skipPR)
} else {
t.Fatal("no response chunks")
}
}
out = []byte(strings.Join(func() []string {
var s []string
Expand All @@ -381,7 +386,13 @@ func TestDegradationResponseDoctrines(t *testing.T) {
} else {
out = sdktranslator.TranslateNonStream(context.Background(), fromF, toF, "doctrine-model", tc.request, tc.request, tc.response, nil)
}
tc.check(t, out)
if msg := tc.check(out); msg != "" {
if tc.skipPR != "" {
t.Skipf("current main violates this doctrine; fix is %s: %s", tc.skipPR, msg)
} else {
t.Fatal(msg)
}
}
})
}
}
Expand Down
Loading