diff --git a/src/agents/failover-error.ts b/src/agents/failover-error.ts index 47660664c8c..3bdc8650c81 100644 --- a/src/agents/failover-error.ts +++ b/src/agents/failover-error.ts @@ -2,11 +2,10 @@ import { readErrorName } from "../infra/errors.js"; import { classifyFailoverReason, isAuthPermanentErrorMessage, + isTimeoutErrorMessage, type FailoverReason, } from "./pi-embedded-helpers.js"; -const TIMEOUT_HINT_RE = - /timeout|timed out|deadline exceeded|context deadline exceeded|connection error|network error|network request failed|fetch failed|socket hang up|econnrefused|econnreset|econnaborted|enotfound|eai_again|stop reason:\s*(?:abort|error)|reason:\s*(?:abort|error)|unhandled stop reason:\s*(?:abort|error)/i; const ABORT_TIMEOUT_RE = /request was aborted|request aborted/i; export class FailoverError extends Error { @@ -125,7 +124,7 @@ function hasTimeoutHint(err: unknown): boolean { return true; } const message = getErrorMessage(err); - return Boolean(message && TIMEOUT_HINT_RE.test(message)); + return Boolean(message && isTimeoutErrorMessage(message)); } export function isTimeoutError(err: unknown): boolean { diff --git a/src/agents/pi-embedded-helpers/errors.ts b/src/agents/pi-embedded-helpers/errors.ts index cfd1460fc61..30112b74fb6 100644 --- a/src/agents/pi-embedded-helpers/errors.ts +++ b/src/agents/pi-embedded-helpers/errors.ts @@ -3,8 +3,26 @@ import type { OpenClawConfig } from "../../config/config.js"; import { createSubsystemLogger } from "../../logging/subsystem.js"; import { formatSandboxToolPolicyBlockedMessage } from "../sandbox.js"; import { stableStringify } from "../stable-stringify.js"; +import { + isAuthErrorMessage, + isAuthPermanentErrorMessage, + isBillingErrorMessage, + isOverloadedErrorMessage, + isRateLimitErrorMessage, + isTimeoutErrorMessage, + matchesFormatErrorPattern, +} from "./failover-matches.js"; import type { FailoverReason } from "./types.js"; +export { + isAuthErrorMessage, + isAuthPermanentErrorMessage, + isBillingErrorMessage, + isOverloadedErrorMessage, + isRateLimitErrorMessage, + isTimeoutErrorMessage, +} from "./failover-matches.js"; + const log = createSubsystemLogger("errors"); export function formatBillingErrorMessage(provider?: string, model?: string): string { @@ -163,10 +181,6 @@ const ERROR_PREFIX_RE = /^(?:error|api\s*error|openai\s*error|anthropic\s*error|gateway\s*error|request failed|failed|exception)[:\s-]+/i; const CONTEXT_OVERFLOW_ERROR_HEAD_RE = /^(?:context overflow:|request_too_large\b|request size exceeds\b|request exceeds the maximum size\b|context length exceeded\b|maximum context length\b|prompt is too long\b|exceeds model context window\b)/i; -const BILLING_ERROR_HEAD_RE = - /^(?:error[:\s-]+)?billing(?:\s+error)?(?:[:\s-]+|$)|^(?:error[:\s-]+)?(?:credit balance|insufficient credits?|payment required|http\s*402\b)/i; -const BILLING_ERROR_HARD_402_RE = - /["']?(?:status|code)["']?\s*[:=]\s*402\b|\bhttp\s*402\b|\berror(?:\s+code)?\s*[:=]?\s*402\b|^\s*402\s+payment/i; const HTTP_STATUS_PREFIX_RE = /^(?:http\s*)?(\d{3})\s+(.+)$/i; const HTTP_STATUS_CODE_PREFIX_RE = /^(?:http\s*)?(\d{3})(?:\s+([\s\S]+))?$/i; const HTML_ERROR_PREFIX_RE = /^\s*(?: - pattern instanceof RegExp ? pattern.test(value) : value.includes(pattern), - ); -} - -export function isRateLimitErrorMessage(raw: string): boolean { - return matchesErrorPatterns(raw, ERROR_PATTERNS.rateLimit); -} - -export function isTimeoutErrorMessage(raw: string): boolean { - return matchesErrorPatterns(raw, ERROR_PATTERNS.timeout); -} - -/** - * Maximum character length for a string to be considered a billing error message. - * Real API billing errors are short, structured messages (typically under 300 chars). - * Longer text is almost certainly assistant content that happens to mention billing keywords. - */ -const BILLING_ERROR_MAX_LENGTH = 512; - -export function isBillingErrorMessage(raw: string): boolean { - const value = raw.toLowerCase(); - if (!value) { - return false; - } - // Real billing error messages from APIs are short structured payloads. - // Long text (e.g. multi-paragraph assistant responses) that happens to mention - // "billing", "payment", etc. should not be treated as a billing error. - if (raw.length > BILLING_ERROR_MAX_LENGTH) { - // Keep explicit status/code 402 detection for providers that wrap errors in - // larger payloads (for example nested JSON bodies or prefixed metadata). - return BILLING_ERROR_HARD_402_RE.test(value); - } - if (matchesErrorPatterns(value, ERROR_PATTERNS.billing)) { - return true; - } - if (!BILLING_ERROR_HEAD_RE.test(raw)) { - return false; - } - return ( - value.includes("upgrade") || - value.includes("credits") || - value.includes("payment") || - value.includes("plan") - ); -} - export function isMissingToolCallInputError(raw: string): boolean { if (!raw) { return false; @@ -777,18 +652,6 @@ export function isBillingAssistantError(msg: AssistantMessage | undefined): bool return isBillingErrorMessage(msg.errorMessage ?? ""); } -export function isAuthPermanentErrorMessage(raw: string): boolean { - return matchesErrorPatterns(raw, ERROR_PATTERNS.authPermanent); -} - -export function isAuthErrorMessage(raw: string): boolean { - return matchesErrorPatterns(raw, ERROR_PATTERNS.auth); -} - -export function isOverloadedErrorMessage(raw: string): boolean { - return matchesErrorPatterns(raw, ERROR_PATTERNS.overloaded); -} - function isJsonApiInternalServerError(raw: string): boolean { if (!raw) { return false; @@ -852,7 +715,7 @@ export function isImageSizeError(errorMessage?: string): boolean { } export function isCloudCodeAssistFormatError(raw: string): boolean { - return !isImageDimensionErrorMessage(raw) && matchesErrorPatterns(raw, ERROR_PATTERNS.format); + return !isImageDimensionErrorMessage(raw) && matchesFormatErrorPattern(raw); } export function isAuthAssistantError(msg: AssistantMessage | undefined): boolean { diff --git a/src/agents/pi-embedded-helpers/failover-matches.ts b/src/agents/pi-embedded-helpers/failover-matches.ts new file mode 100644 index 00000000000..451852282c6 --- /dev/null +++ b/src/agents/pi-embedded-helpers/failover-matches.ts @@ -0,0 +1,149 @@ +type ErrorPattern = RegExp | string; + +const ERROR_PATTERNS = { + rateLimit: [ + /rate[_ ]limit|too many requests|429/, + "model_cooldown", + "cooling down", + "exceeded your current quota", + "resource has been exhausted", + "quota exceeded", + "resource_exhausted", + "usage limit", + /\btpm\b/i, + "tokens per minute", + ], + overloaded: [ + /overloaded_error|"type"\s*:\s*"overloaded_error"/i, + "overloaded", + "service unavailable", + "high demand", + ], + timeout: [ + "timeout", + "timed out", + "deadline exceeded", + "context deadline exceeded", + "connection error", + "network error", + "network request failed", + "fetch failed", + "socket hang up", + /\beconn(?:refused|reset|aborted)\b/i, + /\benotfound\b/i, + /\beai_again\b/i, + /without sending (?:any )?chunks?/i, + /\bstop reason:\s*(?:abort|error)\b/i, + /\breason:\s*(?:abort|error)\b/i, + /\bunhandled stop reason:\s*(?:abort|error)\b/i, + ], + billing: [ + /["']?(?:status|code)["']?\s*[:=]\s*402\b|\bhttp\s*402\b|\berror(?:\s+code)?\s*[:=]?\s*402\b|\b(?:got|returned|received)\s+(?:a\s+)?402\b|^\s*402\s+payment/i, + "payment required", + "insufficient credits", + "credit balance", + "plans & billing", + "insufficient balance", + ], + authPermanent: [ + /api[_ ]?key[_ ]?(?:revoked|invalid|deactivated|deleted)/i, + "invalid_api_key", + "key has been disabled", + "key has been revoked", + "account has been deactivated", + /could not (?:authenticate|validate).*(?:api[_ ]?key|credentials)/i, + "permission_error", + "not allowed for this organization", + ], + auth: [ + /invalid[_ ]?api[_ ]?key/, + "incorrect api key", + "invalid token", + "authentication", + "re-authenticate", + "oauth token refresh failed", + "unauthorized", + "forbidden", + "access denied", + "insufficient permissions", + "insufficient permission", + /missing scopes?:/i, + "expired", + "token has expired", + /\b401\b/, + /\b403\b/, + "no credentials found", + "no api key found", + ], + format: [ + "string should match pattern", + "tool_use.id", + "tool_use_id", + "messages.1.content.1.tool_use.id", + "invalid request format", + /tool call id was.*must be/i, + ], +} as const; + +const BILLING_ERROR_HEAD_RE = + /^(?:error[:\s-]+)?billing(?:\s+error)?(?:[:\s-]+|$)|^(?:error[:\s-]+)?(?:credit balance|insufficient credits?|payment required|http\s*402\b)/i; +const BILLING_ERROR_HARD_402_RE = + /["']?(?:status|code)["']?\s*[:=]\s*402\b|\bhttp\s*402\b|\berror(?:\s+code)?\s*[:=]?\s*402\b|^\s*402\s+payment/i; +const BILLING_ERROR_MAX_LENGTH = 512; + +function matchesErrorPatterns(raw: string, patterns: readonly ErrorPattern[]): boolean { + if (!raw) { + return false; + } + const value = raw.toLowerCase(); + return patterns.some((pattern) => + pattern instanceof RegExp ? pattern.test(value) : value.includes(pattern), + ); +} + +export function matchesFormatErrorPattern(raw: string): boolean { + return matchesErrorPatterns(raw, ERROR_PATTERNS.format); +} + +export function isRateLimitErrorMessage(raw: string): boolean { + return matchesErrorPatterns(raw, ERROR_PATTERNS.rateLimit); +} + +export function isTimeoutErrorMessage(raw: string): boolean { + return matchesErrorPatterns(raw, ERROR_PATTERNS.timeout); +} + +export function isBillingErrorMessage(raw: string): boolean { + const value = raw.toLowerCase(); + if (!value) { + return false; + } + + if (raw.length > BILLING_ERROR_MAX_LENGTH) { + return BILLING_ERROR_HARD_402_RE.test(value); + } + if (matchesErrorPatterns(value, ERROR_PATTERNS.billing)) { + return true; + } + if (!BILLING_ERROR_HEAD_RE.test(raw)) { + return false; + } + return ( + value.includes("upgrade") || + value.includes("credits") || + value.includes("payment") || + value.includes("plan") + ); +} + +export function isAuthPermanentErrorMessage(raw: string): boolean { + return matchesErrorPatterns(raw, ERROR_PATTERNS.authPermanent); +} + +export function isAuthErrorMessage(raw: string): boolean { + return matchesErrorPatterns(raw, ERROR_PATTERNS.auth); +} + +export function isOverloadedErrorMessage(raw: string): boolean { + return matchesErrorPatterns(raw, ERROR_PATTERNS.overloaded); +}