test(agents): add provider-backed failover regressions (#36735)

* test(agents): add provider-backed failover fixtures

* test(agents): cover more provider error docs

* test(agents): tighten provider doc fixtures

src/agents/failover-error.test.ts

@@ -7,6 +7,29 @@ import {
   resolveFailoverStatus,
 } from "./failover-error.js";
 
+// OpenAI 429 example shape: https://help.openai.com/en/articles/5955604-how-can-i-solve-429-too-many-requests-errors
+const OPENAI_RATE_LIMIT_MESSAGE =
+  "Rate limit reached for gpt-4.1-mini in organization org_test on requests per min. Limit: 3.000000 / min. Current: 3.000000 / min.";
+// Anthropic overloaded_error example shape: https://docs.anthropic.com/en/api/errors
+const ANTHROPIC_OVERLOADED_PAYLOAD =
+  '{"type":"error","error":{"type":"overloaded_error","message":"Overloaded"},"request_id":"req_test"}';
+// Gemini RESOURCE_EXHAUSTED troubleshooting example: https://ai.google.dev/gemini-api/docs/troubleshooting
+const GEMINI_RESOURCE_EXHAUSTED_MESSAGE =
+  "RESOURCE_EXHAUSTED: Resource has been exhausted (e.g. check quota).";
+// OpenRouter 402 billing example: https://openrouter.ai/docs/api-reference/errors
+const OPENROUTER_CREDITS_MESSAGE = "Payment Required: insufficient credits";
+// AWS Bedrock 429 ThrottlingException / 503 ServiceUnavailable:
+// https://docs.aws.amazon.com/bedrock/latest/userguide/troubleshooting-api-error-codes.html
+const BEDROCK_THROTTLING_EXCEPTION_MESSAGE =
+  "ThrottlingException: Your request was denied due to exceeding the account quotas for Amazon Bedrock.";
+const BEDROCK_SERVICE_UNAVAILABLE_MESSAGE =
+  "ServiceUnavailable: The service is temporarily unable to handle the request.";
+// Groq error codes examples: https://console.groq.com/docs/errors
+const GROQ_TOO_MANY_REQUESTS_MESSAGE =
+  "429 Too Many Requests: Too many requests were sent in a given timeframe.";
+const GROQ_SERVICE_UNAVAILABLE_MESSAGE =
+  "503 Service Unavailable: The server is temporarily unable to handle the request due to overloading or maintenance.";
+
 describe("failover-error", () => {
   it("infers failover reason from HTTP status", () => {
     expect(resolveFailoverReasonFromError({ status: 402 })).toBe("billing");
@@ -26,6 +49,57 @@ describe("failover-error", () => {
     expect(resolveFailoverReasonFromError({ status: 529 })).toBe("rate_limit");
   });
 
+  it("classifies documented provider error shapes at the error boundary", () => {
+    expect(
+      resolveFailoverReasonFromError({
+        status: 429,
+        message: OPENAI_RATE_LIMIT_MESSAGE,
+      }),
+    ).toBe("rate_limit");
+    expect(
+      resolveFailoverReasonFromError({
+        status: 529,
+        message: ANTHROPIC_OVERLOADED_PAYLOAD,
+      }),
+    ).toBe("rate_limit");
+    expect(
+      resolveFailoverReasonFromError({
+        status: 429,
+        message: GEMINI_RESOURCE_EXHAUSTED_MESSAGE,
+      }),
+    ).toBe("rate_limit");
+    expect(
+      resolveFailoverReasonFromError({
+        status: 402,
+        message: OPENROUTER_CREDITS_MESSAGE,
+      }),
+    ).toBe("billing");
+    expect(
+      resolveFailoverReasonFromError({
+        status: 429,
+        message: BEDROCK_THROTTLING_EXCEPTION_MESSAGE,
+      }),
+    ).toBe("rate_limit");
+    expect(
+      resolveFailoverReasonFromError({
+        status: 503,
+        message: BEDROCK_SERVICE_UNAVAILABLE_MESSAGE,
+      }),
+    ).toBe("timeout");
+    expect(
+      resolveFailoverReasonFromError({
+        status: 429,
+        message: GROQ_TOO_MANY_REQUESTS_MESSAGE,
+      }),
+    ).toBe("rate_limit");
+    expect(
+      resolveFailoverReasonFromError({
+        status: 503,
+        message: GROQ_SERVICE_UNAVAILABLE_MESSAGE,
+      }),
+    ).toBe("timeout");
+  });
+
   it("infers format errors from error messages", () => {
     expect(
       resolveFailoverReasonFromError({

src/agents/model-fallback.test.ts

@@ -173,6 +173,17 @@ async function expectSkippedUnavailableProvider(params: {
   expect(result.attempts[0]?.reason).toBe(params.expectedReason);
 }
 
+// OpenAI 429 example shape: https://help.openai.com/en/articles/5955604-how-can-i-solve-429-too-many-requests-errors
+const OPENAI_RATE_LIMIT_MESSAGE =
+  "Rate limit reached for gpt-4.1-mini in organization org_test on requests per min. Limit: 3.000000 / min. Current: 3.000000 / min.";
+// Anthropic overloaded_error example shape: https://docs.anthropic.com/en/api/errors
+const ANTHROPIC_OVERLOADED_PAYLOAD =
+  '{"type":"error","error":{"type":"overloaded_error","message":"Overloaded"},"request_id":"req_test"}';
+// Internal OpenClaw compatibility marker, not a provider API contract.
+const MODEL_COOLDOWN_MESSAGE = "model_cooldown: All credentials for model gpt-5 are cooling down";
+// SDK/transport compatibility marker, not a provider API contract.
+const CONNECTION_ERROR_MESSAGE = "Connection error.";
+
 describe("runWithModelFallback", () => {
   it("keeps openai gpt-5.3 codex on the openai provider before running", async () => {
     const cfg = makeCfg();
@@ -712,6 +723,38 @@ describe("runWithModelFallback", () => {
     });
   });
 
+  it("falls back on documented OpenAI 429 rate limit responses", async () => {
+    await expectFallsBackToHaiku({
+      provider: "openai",
+      model: "gpt-4.1-mini",
+      firstError: Object.assign(new Error(OPENAI_RATE_LIMIT_MESSAGE), { status: 429 }),
+    });
+  });
+
+  it("falls back on documented overloaded_error payloads", async () => {
+    await expectFallsBackToHaiku({
+      provider: "openai",
+      model: "gpt-4.1-mini",
+      firstError: new Error(ANTHROPIC_OVERLOADED_PAYLOAD),
+    });
+  });
+
+  it("falls back on internal model cooldown markers", async () => {
+    await expectFallsBackToHaiku({
+      provider: "openai",
+      model: "gpt-4.1-mini",
+      firstError: new Error(MODEL_COOLDOWN_MESSAGE),
+    });
+  });
+
+  it("falls back on compatibility connection error messages", async () => {
+    await expectFallsBackToHaiku({
+      provider: "openai",
+      model: "gpt-4.1-mini",
+      firstError: new Error(CONNECTION_ERROR_MESSAGE),
+    });
+  });
+
   it("falls back on timeout abort errors", async () => {
     const timeoutCause = Object.assign(new Error("request timed out"), { name: "TimeoutError" });
     await expectFallsBackToHaiku({

src/agents/pi-embedded-helpers.isbillingerrormessage.test.ts

@@ -17,6 +17,28 @@ import {
   parseImageSizeError,
 } from "./pi-embedded-helpers.js";
 
+// OpenAI 429 example shape: https://help.openai.com/en/articles/5955604-how-can-i-solve-429-too-many-requests-errors
+const OPENAI_RATE_LIMIT_MESSAGE =
+  "Rate limit reached for gpt-4.1-mini in organization org_test on requests per min. Limit: 3.000000 / min. Current: 3.000000 / min.";
+// Gemini RESOURCE_EXHAUSTED troubleshooting example: https://ai.google.dev/gemini-api/docs/troubleshooting
+const GEMINI_RESOURCE_EXHAUSTED_MESSAGE =
+  "RESOURCE_EXHAUSTED: Resource has been exhausted (e.g. check quota).";
+// Anthropic overloaded_error example shape: https://docs.anthropic.com/en/api/errors
+const ANTHROPIC_OVERLOADED_PAYLOAD =
+  '{"type":"error","error":{"type":"overloaded_error","message":"Overloaded"},"request_id":"req_test"}';
+// OpenRouter 402 billing example: https://openrouter.ai/docs/api-reference/errors
+const OPENROUTER_CREDITS_MESSAGE = "Payment Required: insufficient credits";
+// Together AI error code examples: https://docs.together.ai/docs/error-codes
+const TOGETHER_PAYMENT_REQUIRED_MESSAGE =
+  "402 Payment Required: The account associated with this API key has reached its maximum allowed monthly spending limit.";
+const TOGETHER_ENGINE_OVERLOADED_MESSAGE =
+  "503 Engine Overloaded: The server is experiencing a high volume of requests and is temporarily overloaded.";
+// Groq error code examples: https://console.groq.com/docs/errors
+const GROQ_TOO_MANY_REQUESTS_MESSAGE =
+  "429 Too Many Requests: Too many requests were sent in a given timeframe.";
+const GROQ_SERVICE_UNAVAILABLE_MESSAGE =
+  "503 Service Unavailable: The server is temporarily unable to handle the request due to overloading or maintenance.";
+
 describe("isAuthPermanentErrorMessage", () => {
   it("matches permanent auth failure patterns", () => {
     const samples = [
@@ -480,7 +502,18 @@ describe("image dimension errors", () => {
 });
 
 describe("classifyFailoverReason", () => {
-  it("returns a stable reason", () => {
+  it("classifies documented provider error messages", () => {
+    expect(classifyFailoverReason(OPENAI_RATE_LIMIT_MESSAGE)).toBe("rate_limit");
+    expect(classifyFailoverReason(GEMINI_RESOURCE_EXHAUSTED_MESSAGE)).toBe("rate_limit");
+    expect(classifyFailoverReason(ANTHROPIC_OVERLOADED_PAYLOAD)).toBe("rate_limit");
+    expect(classifyFailoverReason(OPENROUTER_CREDITS_MESSAGE)).toBe("billing");
+    expect(classifyFailoverReason(TOGETHER_PAYMENT_REQUIRED_MESSAGE)).toBe("billing");
+    expect(classifyFailoverReason(TOGETHER_ENGINE_OVERLOADED_MESSAGE)).toBe("timeout");
+    expect(classifyFailoverReason(GROQ_TOO_MANY_REQUESTS_MESSAGE)).toBe("rate_limit");
+    expect(classifyFailoverReason(GROQ_SERVICE_UNAVAILABLE_MESSAGE)).toBe("timeout");
+  });
+
+  it("classifies internal and compatibility error messages", () => {
     expect(classifyFailoverReason("invalid api key")).toBe("auth");
     expect(classifyFailoverReason("no credentials found")).toBe("auth");
     expect(classifyFailoverReason("no api key found")).toBe("auth");
@@ -493,19 +526,11 @@ describe("classifyFailoverReason", () => {
       "auth",
     );
     expect(classifyFailoverReason("Missing scopes: model.request")).toBe("auth");
-    expect(classifyFailoverReason("429 too many requests")).toBe("rate_limit");
-    expect(classifyFailoverReason("resource has been exhausted")).toBe("rate_limit");
     expect(
       classifyFailoverReason("model_cooldown: All credentials for model gpt-5 are cooling down"),
     ).toBe("rate_limit");
     expect(classifyFailoverReason("all credentials for model x are cooling down")).toBeNull();
-    expect(
-      classifyFailoverReason(
-        '{"type":"error","error":{"type":"overloaded_error","message":"Overloaded"}}',
-      ),
-    ).toBe("rate_limit");
     expect(classifyFailoverReason("invalid request format")).toBe("format");
-    expect(classifyFailoverReason("credit balance too low")).toBe("billing");
     expect(classifyFailoverReason("deadline exceeded")).toBe("timeout");
     expect(classifyFailoverReason("request ended without sending any chunks")).toBe("timeout");
     expect(classifyFailoverReason("Connection error.")).toBe("timeout");