diff --git a/src/agents/failover-error.test.ts b/src/agents/failover-error.test.ts index 4865dbfb560..c3f73e123a4 100644 --- a/src/agents/failover-error.test.ts +++ b/src/agents/failover-error.test.ts @@ -18,6 +18,8 @@ const GEMINI_RESOURCE_EXHAUSTED_MESSAGE = "RESOURCE_EXHAUSTED: Resource has been exhausted (e.g. check quota)."; // OpenRouter 402 billing example: https://openrouter.ai/docs/api-reference/errors const OPENROUTER_CREDITS_MESSAGE = "Payment Required: insufficient credits"; +const TOGETHER_MONTHLY_SPEND_CAP_MESSAGE = + "The account associated with this API key has reached its maximum allowed monthly spending limit."; // Issue-backed Anthropic/OpenAI-compatible insufficient_quota payload under HTTP 400: // https://github.com/openclaw/openclaw/issues/23440 const INSUFFICIENT_QUOTA_PAYLOAD = @@ -201,6 +203,27 @@ describe("failover-error", () => { message: `${"x".repeat(520)} insufficient credits. Monthly spend limit reached.`, }), ).toBe("billing"); + expect( + resolveFailoverReasonFromError({ + status: 402, + message: TOGETHER_MONTHLY_SPEND_CAP_MESSAGE, + }), + ).toBe("billing"); + }); + + it("keeps raw 402 wrappers aligned with status-split temporary spend limits", () => { + const message = "Monthly spend limit reached. Please visit your billing settings."; + expect( + resolveFailoverReasonFromError({ + message: `402 Payment Required: ${message}`, + }), + ).toBe("rate_limit"); + expect( + resolveFailoverReasonFromError({ + status: 402, + message, + }), + ).toBe("rate_limit"); }); it("infers format errors from error messages", () => { diff --git a/src/agents/pi-embedded-helpers.isbillingerrormessage.test.ts b/src/agents/pi-embedded-helpers.isbillingerrormessage.test.ts index e6d5a823c18..097657acd95 100644 --- a/src/agents/pi-embedded-helpers.isbillingerrormessage.test.ts +++ b/src/agents/pi-embedded-helpers.isbillingerrormessage.test.ts @@ -542,6 +542,12 @@ describe("classifyFailoverReasonFromHttpStatus – 402 temporary limits", () => "Insufficient credits. Organization limit reached.", ), ).toBe("billing"); + expect( + classifyFailoverReasonFromHttpStatus( + 402, + "The account associated with this API key has reached its maximum allowed monthly spending limit.", + ), + ).toBe("billing"); }); it("keeps long 402 payloads with explicit billing text as billing", () => { @@ -554,6 +560,17 @@ describe("classifyFailoverReasonFromHttpStatus – 402 temporary limits", () => expect(classifyFailoverReasonFromHttpStatus(402, "")).toBe("billing"); expect(classifyFailoverReasonFromHttpStatus(402, "Payment required")).toBe("billing"); }); + + it("matches raw 402 wrappers and status-split payloads for the same message", () => { + const transientMessage = "Monthly spend limit reached. Please visit your billing settings."; + expect(classifyFailoverReason(`402 Payment Required: ${transientMessage}`)).toBe("rate_limit"); + expect(classifyFailoverReasonFromHttpStatus(402, transientMessage)).toBe("rate_limit"); + + const billingMessage = + "The account associated with this API key has reached its maximum allowed monthly spending limit."; + expect(classifyFailoverReason(`402 Payment Required: ${billingMessage}`)).toBe("billing"); + expect(classifyFailoverReasonFromHttpStatus(402, billingMessage)).toBe("billing"); + }); }); describe("classifyFailoverReason", () => { diff --git a/src/agents/pi-embedded-helpers/errors.ts b/src/agents/pi-embedded-helpers/errors.ts index 2c68ff288d5..8f6dbe652bb 100644 --- a/src/agents/pi-embedded-helpers/errors.ts +++ b/src/agents/pi-embedded-helpers/errors.ts @@ -208,8 +208,9 @@ const HTTP_ERROR_HINTS = [ "permission", ]; +type Http402Reason = "billing" | "rate_limit"; + const BILLING_402_HINTS = [ - "payment required", "insufficient credits", "insufficient quota", "credit balance", @@ -221,9 +222,20 @@ const BILLING_402_HINTS = [ const RETRYABLE_402_RETRY_HINTS = ["try again", "retry", "temporary", "cooldown"] as const; const RETRYABLE_402_LIMIT_HINTS = ["usage limit", "rate limit", "organization usage"] as const; -const RETRYABLE_402_SPEND_HINTS = ["spend limit", "spending limit"] as const; -const RETRYABLE_402_SCOPE_HINTS = ["organization", "workspace"] as const; -const RETRYABLE_402_SCOPE_LIMIT_HINTS = ["limit", "exceeded"] as const; +const BILLING_402_HARD_CAP_RE = + /\b(?:billing hard limit|hard limit reached|maximum allowed(?:\s+(?:daily|weekly|monthly))?(?:\s+spend(?:ing)?)?\s+limit)\b/i; +const RAW_402_MARKER_RE = + /["']?(?:status|code)["']?\s*[:=]\s*402\b|\bhttp\s*402\b|\berror(?:\s+code)?\s*[:=]?\s*402\b|\b(?:got|returned|received)\s+(?:a\s+)?402\b|^\s*402\s+payment required\b/i; +const LEADING_402_WRAPPER_RE = + /^(?:error[:\s-]+)?(?:(?:http\s*)?402(?:\s+payment required)?|payment required)(?:[:\s-]+|$)/i; +const RETRYABLE_402_PERIODIC_USAGE_OR_SPEND_RE = + /\b(?:daily|weekly|monthly)(?:\/(?:daily|weekly|monthly))*\s+(?:usage|spend(?:ing)?)\s+limit(?:s)?(?:\s+(?:exhausted|reached|exceeded))?\b/i; +const RETRYABLE_402_SCOPED_SPEND_LIMIT_RE = + /\b(?:organization|workspace)\s+(?:spend(?:ing)?\s+)?limit(?:s)?(?:\s+(?:reached|exhausted|exceeded))?\b/i; +const RETRYABLE_402_SCOPED_LIMIT_EXCEEDED_RE = + /\b(?:organization|workspace)\b[\s\S]{0,30}\blimit(?:s)?\b[\s\S]{0,30}\bexceeded\b/i; +const RETRYABLE_402_SCOPED_BILLING_PERIOD_RE = + /\b(?:organization|workspace)\b[\s\S]{0,30}\blimit(?:s)?\b[\s\S]{0,30}\bbilling period\b/i; function includesAnyHint(text: string, hints: readonly string[]): boolean { const lower = text.toLowerCase(); @@ -231,35 +243,54 @@ function includesAnyHint(text: string, hints: readonly string[]): boolean { } function hasExplicit402BillingSignal(text: string): boolean { - return includesAnyHint(text, BILLING_402_HINTS); + return includesAnyHint(text, BILLING_402_HINTS) || BILLING_402_HARD_CAP_RE.test(text); } -function hasRetryable402UsageSignal(text: string): boolean { +function hasRetryable402TransientSignal(text: string): boolean { + const lower = text.toLowerCase(); return ( - includesAnyHint(text, RETRYABLE_402_RETRY_HINTS) && - includesAnyHint(text, RETRYABLE_402_LIMIT_HINTS) + (includesAnyHint(text, RETRYABLE_402_RETRY_HINTS) && + includesAnyHint(text, RETRYABLE_402_LIMIT_HINTS)) || + RETRYABLE_402_PERIODIC_USAGE_OR_SPEND_RE.test(text) || + (includesAnyHint(lower, ["daily", "weekly", "monthly"]) && + lower.includes("limit") && + lower.includes("reset")) || + RETRYABLE_402_SCOPED_SPEND_LIMIT_RE.test(text) || + RETRYABLE_402_SCOPED_LIMIT_EXCEEDED_RE.test(text) || + RETRYABLE_402_SCOPED_BILLING_PERIOD_RE.test(text) ); } -function shouldTreat402AsRateLimit(raw: string): boolean { - if (hasExplicit402BillingSignal(raw)) { - return false; +function normalize402Message(raw: string): string { + let normalized = raw.trim(); + while (LEADING_402_WRAPPER_RE.test(normalized)) { + normalized = normalized.replace(LEADING_402_WRAPPER_RE, "").trim(); + } + return normalized; +} + +function classify402Message(message: string): Http402Reason { + const normalized = normalize402Message(message); + if (!normalized) { + return "billing"; } - if (hasRetryable402UsageSignal(raw)) { - return true; + if (hasExplicit402BillingSignal(normalized)) { + return "billing"; } - if (isPeriodicUsageLimitErrorMessage(raw)) { - return true; + if (hasRetryable402TransientSignal(normalized)) { + return "rate_limit"; } - return ( - includesAnyHint(raw, RETRYABLE_402_SPEND_HINTS) || - includesAnyHint(raw, RETRYABLE_402_LIMIT_HINTS) || - (includesAnyHint(raw, RETRYABLE_402_SCOPE_HINTS) && - includesAnyHint(raw, RETRYABLE_402_SCOPE_LIMIT_HINTS)) - ); + return "billing"; +} + +function classifyFailoverReasonFromRaw402Message(raw: string): Http402Reason | null { + if (!RAW_402_MARKER_RE.test(raw)) { + return null; + } + return classify402Message(raw); } function extractLeadingHttpStatus(raw: string): { code: number; rest: string } | null { @@ -315,12 +346,7 @@ export function classifyFailoverReasonFromHttpStatus( } if (status === 402) { - // Some providers surface temporary usage caps as HTTP 402. Keep those - // retryable, but let explicit insufficient-credit signals stay billing. - if (message && shouldTreat402AsRateLimit(message)) { - return "rate_limit"; - } - return "billing"; + return message ? classify402Message(message) : "billing"; } if (status === 429) { return "rate_limit"; @@ -899,6 +925,10 @@ export function classifyFailoverReason(raw: string): FailoverReason | null { if (isModelNotFoundErrorMessage(raw)) { return "model_not_found"; } + const raw402Reason = classifyFailoverReasonFromRaw402Message(raw); + if (raw402Reason) { + return raw402Reason; + } if (isPeriodicUsageLimitErrorMessage(raw)) { return isBillingErrorMessage(raw) ? "billing" : "rate_limit"; }