diff --git a/src/agents/failover-error.test.ts b/src/agents/failover-error.test.ts index 772b4707b0c..3bf27c21cff 100644 --- a/src/agents/failover-error.test.ts +++ b/src/agents/failover-error.test.ts @@ -7,6 +7,29 @@ import { resolveFailoverStatus, } from "./failover-error.js"; +// OpenAI 429 example shape: https://help.openai.com/en/articles/5955604-how-can-i-solve-429-too-many-requests-errors +const OPENAI_RATE_LIMIT_MESSAGE = + "Rate limit reached for gpt-4.1-mini in organization org_test on requests per min. Limit: 3.000000 / min. Current: 3.000000 / min."; +// Anthropic overloaded_error example shape: https://docs.anthropic.com/en/api/errors +const ANTHROPIC_OVERLOADED_PAYLOAD = + '{"type":"error","error":{"type":"overloaded_error","message":"Overloaded"},"request_id":"req_test"}'; +// Gemini RESOURCE_EXHAUSTED troubleshooting example: https://ai.google.dev/gemini-api/docs/troubleshooting +const GEMINI_RESOURCE_EXHAUSTED_MESSAGE = + "RESOURCE_EXHAUSTED: Resource has been exhausted (e.g. check quota)."; +// OpenRouter 402 billing example: https://openrouter.ai/docs/api-reference/errors +const OPENROUTER_CREDITS_MESSAGE = "Payment Required: insufficient credits"; +// AWS Bedrock 429 ThrottlingException / 503 ServiceUnavailable: +// https://docs.aws.amazon.com/bedrock/latest/userguide/troubleshooting-api-error-codes.html +const BEDROCK_THROTTLING_EXCEPTION_MESSAGE = + "ThrottlingException: Your request was denied due to exceeding the account quotas for Amazon Bedrock."; +const BEDROCK_SERVICE_UNAVAILABLE_MESSAGE = + "ServiceUnavailable: The service is temporarily unable to handle the request."; +// Groq error codes examples: https://console.groq.com/docs/errors +const GROQ_TOO_MANY_REQUESTS_MESSAGE = + "429 Too Many Requests: Too many requests were sent in a given timeframe."; +const GROQ_SERVICE_UNAVAILABLE_MESSAGE = + "503 Service Unavailable: The server is temporarily unable to handle the request due to overloading or maintenance."; + describe("failover-error", () => { it("infers failover reason from HTTP status", () => { expect(resolveFailoverReasonFromError({ status: 402 })).toBe("billing"); @@ -26,6 +49,57 @@ describe("failover-error", () => { expect(resolveFailoverReasonFromError({ status: 529 })).toBe("rate_limit"); }); + it("classifies documented provider error shapes at the error boundary", () => { + expect( + resolveFailoverReasonFromError({ + status: 429, + message: OPENAI_RATE_LIMIT_MESSAGE, + }), + ).toBe("rate_limit"); + expect( + resolveFailoverReasonFromError({ + status: 529, + message: ANTHROPIC_OVERLOADED_PAYLOAD, + }), + ).toBe("rate_limit"); + expect( + resolveFailoverReasonFromError({ + status: 429, + message: GEMINI_RESOURCE_EXHAUSTED_MESSAGE, + }), + ).toBe("rate_limit"); + expect( + resolveFailoverReasonFromError({ + status: 402, + message: OPENROUTER_CREDITS_MESSAGE, + }), + ).toBe("billing"); + expect( + resolveFailoverReasonFromError({ + status: 429, + message: BEDROCK_THROTTLING_EXCEPTION_MESSAGE, + }), + ).toBe("rate_limit"); + expect( + resolveFailoverReasonFromError({ + status: 503, + message: BEDROCK_SERVICE_UNAVAILABLE_MESSAGE, + }), + ).toBe("timeout"); + expect( + resolveFailoverReasonFromError({ + status: 429, + message: GROQ_TOO_MANY_REQUESTS_MESSAGE, + }), + ).toBe("rate_limit"); + expect( + resolveFailoverReasonFromError({ + status: 503, + message: GROQ_SERVICE_UNAVAILABLE_MESSAGE, + }), + ).toBe("timeout"); + }); + it("infers format errors from error messages", () => { expect( resolveFailoverReasonFromError({ diff --git a/src/agents/model-fallback.test.ts b/src/agents/model-fallback.test.ts index 6f6fdd8b76f..2c58a42c99a 100644 --- a/src/agents/model-fallback.test.ts +++ b/src/agents/model-fallback.test.ts @@ -173,6 +173,17 @@ async function expectSkippedUnavailableProvider(params: { expect(result.attempts[0]?.reason).toBe(params.expectedReason); } +// OpenAI 429 example shape: https://help.openai.com/en/articles/5955604-how-can-i-solve-429-too-many-requests-errors +const OPENAI_RATE_LIMIT_MESSAGE = + "Rate limit reached for gpt-4.1-mini in organization org_test on requests per min. Limit: 3.000000 / min. Current: 3.000000 / min."; +// Anthropic overloaded_error example shape: https://docs.anthropic.com/en/api/errors +const ANTHROPIC_OVERLOADED_PAYLOAD = + '{"type":"error","error":{"type":"overloaded_error","message":"Overloaded"},"request_id":"req_test"}'; +// Internal OpenClaw compatibility marker, not a provider API contract. +const MODEL_COOLDOWN_MESSAGE = "model_cooldown: All credentials for model gpt-5 are cooling down"; +// SDK/transport compatibility marker, not a provider API contract. +const CONNECTION_ERROR_MESSAGE = "Connection error."; + describe("runWithModelFallback", () => { it("keeps openai gpt-5.3 codex on the openai provider before running", async () => { const cfg = makeCfg(); @@ -712,6 +723,38 @@ describe("runWithModelFallback", () => { }); }); + it("falls back on documented OpenAI 429 rate limit responses", async () => { + await expectFallsBackToHaiku({ + provider: "openai", + model: "gpt-4.1-mini", + firstError: Object.assign(new Error(OPENAI_RATE_LIMIT_MESSAGE), { status: 429 }), + }); + }); + + it("falls back on documented overloaded_error payloads", async () => { + await expectFallsBackToHaiku({ + provider: "openai", + model: "gpt-4.1-mini", + firstError: new Error(ANTHROPIC_OVERLOADED_PAYLOAD), + }); + }); + + it("falls back on internal model cooldown markers", async () => { + await expectFallsBackToHaiku({ + provider: "openai", + model: "gpt-4.1-mini", + firstError: new Error(MODEL_COOLDOWN_MESSAGE), + }); + }); + + it("falls back on compatibility connection error messages", async () => { + await expectFallsBackToHaiku({ + provider: "openai", + model: "gpt-4.1-mini", + firstError: new Error(CONNECTION_ERROR_MESSAGE), + }); + }); + it("falls back on timeout abort errors", async () => { const timeoutCause = Object.assign(new Error("request timed out"), { name: "TimeoutError" }); await expectFallsBackToHaiku({ diff --git a/src/agents/pi-embedded-helpers.isbillingerrormessage.test.ts b/src/agents/pi-embedded-helpers.isbillingerrormessage.test.ts index a46857ac851..1ca99e8a993 100644 --- a/src/agents/pi-embedded-helpers.isbillingerrormessage.test.ts +++ b/src/agents/pi-embedded-helpers.isbillingerrormessage.test.ts @@ -17,6 +17,28 @@ import { parseImageSizeError, } from "./pi-embedded-helpers.js"; +// OpenAI 429 example shape: https://help.openai.com/en/articles/5955604-how-can-i-solve-429-too-many-requests-errors +const OPENAI_RATE_LIMIT_MESSAGE = + "Rate limit reached for gpt-4.1-mini in organization org_test on requests per min. Limit: 3.000000 / min. Current: 3.000000 / min."; +// Gemini RESOURCE_EXHAUSTED troubleshooting example: https://ai.google.dev/gemini-api/docs/troubleshooting +const GEMINI_RESOURCE_EXHAUSTED_MESSAGE = + "RESOURCE_EXHAUSTED: Resource has been exhausted (e.g. check quota)."; +// Anthropic overloaded_error example shape: https://docs.anthropic.com/en/api/errors +const ANTHROPIC_OVERLOADED_PAYLOAD = + '{"type":"error","error":{"type":"overloaded_error","message":"Overloaded"},"request_id":"req_test"}'; +// OpenRouter 402 billing example: https://openrouter.ai/docs/api-reference/errors +const OPENROUTER_CREDITS_MESSAGE = "Payment Required: insufficient credits"; +// Together AI error code examples: https://docs.together.ai/docs/error-codes +const TOGETHER_PAYMENT_REQUIRED_MESSAGE = + "402 Payment Required: The account associated with this API key has reached its maximum allowed monthly spending limit."; +const TOGETHER_ENGINE_OVERLOADED_MESSAGE = + "503 Engine Overloaded: The server is experiencing a high volume of requests and is temporarily overloaded."; +// Groq error code examples: https://console.groq.com/docs/errors +const GROQ_TOO_MANY_REQUESTS_MESSAGE = + "429 Too Many Requests: Too many requests were sent in a given timeframe."; +const GROQ_SERVICE_UNAVAILABLE_MESSAGE = + "503 Service Unavailable: The server is temporarily unable to handle the request due to overloading or maintenance."; + describe("isAuthPermanentErrorMessage", () => { it("matches permanent auth failure patterns", () => { const samples = [ @@ -480,7 +502,18 @@ describe("image dimension errors", () => { }); describe("classifyFailoverReason", () => { - it("returns a stable reason", () => { + it("classifies documented provider error messages", () => { + expect(classifyFailoverReason(OPENAI_RATE_LIMIT_MESSAGE)).toBe("rate_limit"); + expect(classifyFailoverReason(GEMINI_RESOURCE_EXHAUSTED_MESSAGE)).toBe("rate_limit"); + expect(classifyFailoverReason(ANTHROPIC_OVERLOADED_PAYLOAD)).toBe("rate_limit"); + expect(classifyFailoverReason(OPENROUTER_CREDITS_MESSAGE)).toBe("billing"); + expect(classifyFailoverReason(TOGETHER_PAYMENT_REQUIRED_MESSAGE)).toBe("billing"); + expect(classifyFailoverReason(TOGETHER_ENGINE_OVERLOADED_MESSAGE)).toBe("timeout"); + expect(classifyFailoverReason(GROQ_TOO_MANY_REQUESTS_MESSAGE)).toBe("rate_limit"); + expect(classifyFailoverReason(GROQ_SERVICE_UNAVAILABLE_MESSAGE)).toBe("timeout"); + }); + + it("classifies internal and compatibility error messages", () => { expect(classifyFailoverReason("invalid api key")).toBe("auth"); expect(classifyFailoverReason("no credentials found")).toBe("auth"); expect(classifyFailoverReason("no api key found")).toBe("auth"); @@ -493,19 +526,11 @@ describe("classifyFailoverReason", () => { "auth", ); expect(classifyFailoverReason("Missing scopes: model.request")).toBe("auth"); - expect(classifyFailoverReason("429 too many requests")).toBe("rate_limit"); - expect(classifyFailoverReason("resource has been exhausted")).toBe("rate_limit"); expect( classifyFailoverReason("model_cooldown: All credentials for model gpt-5 are cooling down"), ).toBe("rate_limit"); expect(classifyFailoverReason("all credentials for model x are cooling down")).toBeNull(); - expect( - classifyFailoverReason( - '{"type":"error","error":{"type":"overloaded_error","message":"Overloaded"}}', - ), - ).toBe("rate_limit"); expect(classifyFailoverReason("invalid request format")).toBe("format"); - expect(classifyFailoverReason("credit balance too low")).toBe("billing"); expect(classifyFailoverReason("deadline exceeded")).toBe("timeout"); expect(classifyFailoverReason("request ended without sending any chunks")).toBe("timeout"); expect(classifyFailoverReason("Connection error.")).toBe("timeout");