Files
moltbot/src/cron/service.jobs.test.ts
Forgely3D 4fa11632b4 fix: escalate to model fallback after rate-limit profile rotation cap (#58707)
* fix: escalate to model fallback after rate-limit profile rotation cap

Per-model rate limits (e.g. Anthropic Sonnet-only quotas) are not
relieved by rotating auth profiles — if all profiles share the same
model quota, cycling between them loops forever without falling back
to the next model in the configured fallbacks chain.

Apply the same rotation-cap pattern introduced for overloaded_error
(#58348) to rate_limit errors:

- Add `rateLimitedProfileRotations` to auth.cooldowns config (default: 1)
- After N profile rotations on a rate_limit error, throw FailoverError
  to trigger cross-provider model fallback
- Add `resolveRateLimitProfileRotationLimit` helper following the same
  pattern as `resolveOverloadProfileRotationLimit`

Fixes #58572

* fix: cap prompt-side rate-limit failover (#58707) (thanks @Forgely3D)

* fix: restore latest-main gates for #58707

---------

Co-authored-by: Ember (Forgely3D) <ember@forgely.co>
Co-authored-by: Peter Steinberger <steipete@gmail.com>
2026-04-01 17:54:10 +09:00

571 lines
18 KiB
TypeScript

import { describe, expect, it } from "vitest";
import { applyJobPatch, createJob, recomputeNextRuns } from "./service/jobs.js";
import type { CronServiceState } from "./service/state.js";
import { DEFAULT_TOP_OF_HOUR_STAGGER_MS } from "./stagger.js";
import type { CronJob, CronJobPatch } from "./types.js";
function expectCronStaggerMs(job: CronJob, expected: number): void {
expect(job.schedule.kind).toBe("cron");
if (job.schedule.kind === "cron") {
expect(job.schedule.staggerMs).toBe(expected);
}
}
describe("applyJobPatch", () => {
const createIsolatedAgentTurnJob = (
id: string,
delivery: CronJob["delivery"],
overrides?: Partial<CronJob>,
): CronJob => {
const now = Date.now();
return {
id,
name: id,
enabled: true,
createdAtMs: now,
updatedAtMs: now,
schedule: { kind: "every", everyMs: 60_000 },
sessionTarget: "isolated",
wakeMode: "now",
payload: { kind: "agentTurn", message: "do it" },
delivery,
state: {},
...overrides,
};
};
const switchToMainPatch = (): CronJobPatch => ({
sessionTarget: "main",
payload: { kind: "systemEvent", text: "ping" },
});
const createMainSystemEventJob = (id: string, delivery: CronJob["delivery"]): CronJob => {
return createIsolatedAgentTurnJob(id, delivery, {
sessionTarget: "main",
payload: { kind: "systemEvent", text: "ping" },
});
};
it("clears delivery when switching to main session", () => {
const job = createIsolatedAgentTurnJob("job-1", {
mode: "announce",
channel: "telegram",
to: "123",
});
expect(() => applyJobPatch(job, switchToMainPatch())).not.toThrow();
expect(job.sessionTarget).toBe("main");
expect(job.payload.kind).toBe("systemEvent");
expect(job.delivery).toBeUndefined();
});
it("keeps webhook delivery when switching to main session", () => {
const job = createIsolatedAgentTurnJob("job-webhook", {
mode: "webhook",
to: "https://example.invalid/cron",
});
expect(() => applyJobPatch(job, switchToMainPatch())).not.toThrow();
expect(job.sessionTarget).toBe("main");
expect(job.delivery).toEqual({ mode: "webhook", to: "https://example.invalid/cron" });
});
it("applies explicit delivery patches", () => {
const job = createIsolatedAgentTurnJob("job-2", {
mode: "announce",
channel: "telegram",
to: "123",
});
const patch: CronJobPatch = {
delivery: {
mode: "none",
channel: "signal",
to: "555",
bestEffort: true,
},
};
expect(() => applyJobPatch(job, patch)).not.toThrow();
expect(job.payload.kind).toBe("agentTurn");
if (job.payload.kind === "agentTurn") {
expect(job.payload.message).toBe("do it");
}
expect(job.delivery).toEqual({
mode: "none",
channel: "signal",
to: "555",
bestEffort: true,
});
});
it("applies explicit delivery patches for custom session targets", () => {
const job = createIsolatedAgentTurnJob(
"job-custom-session",
{
mode: "announce",
channel: "telegram",
to: "123",
},
{ sessionTarget: "session:project-alpha" },
);
applyJobPatch(job, {
delivery: { mode: "announce", to: "555" },
});
expect(job.delivery).toEqual({
mode: "announce",
channel: "telegram",
to: "555",
bestEffort: undefined,
});
});
it("merges delivery.accountId from patch and preserves existing", () => {
const job = createIsolatedAgentTurnJob("job-acct", {
mode: "announce",
channel: "telegram",
to: "-100123",
});
applyJobPatch(job, { delivery: { mode: "announce", accountId: " coordinator " } });
expect(job.delivery?.accountId).toBe("coordinator");
expect(job.delivery?.mode).toBe("announce");
expect(job.delivery?.to).toBe("-100123");
// Updating other fields preserves accountId
applyJobPatch(job, { delivery: { mode: "announce", to: "-100999" } });
expect(job.delivery?.accountId).toBe("coordinator");
expect(job.delivery?.to).toBe("-100999");
// Clearing accountId with empty string
applyJobPatch(job, { delivery: { mode: "announce", accountId: "" } });
expect(job.delivery?.accountId).toBeUndefined();
});
it("persists agentTurn payload.lightContext updates when editing existing jobs", () => {
const job = createIsolatedAgentTurnJob("job-light-context", {
mode: "announce",
channel: "telegram",
});
job.payload = {
kind: "agentTurn",
message: "do it",
lightContext: true,
};
applyJobPatch(job, {
payload: {
kind: "agentTurn",
message: "do it",
lightContext: false,
},
});
expect(job.payload.kind).toBe("agentTurn");
if (job.payload.kind === "agentTurn") {
expect(job.payload.lightContext).toBe(false);
}
});
it("applies payload.lightContext when replacing payload kind via patch", () => {
const job = createIsolatedAgentTurnJob("job-light-context-switch", {
mode: "announce",
channel: "telegram",
});
job.payload = { kind: "systemEvent", text: "ping" };
applyJobPatch(job, {
payload: {
kind: "agentTurn",
message: "do it",
lightContext: true,
},
});
const payload = job.payload as CronJob["payload"];
expect(payload.kind).toBe("agentTurn");
if (payload.kind === "agentTurn") {
expect(payload.lightContext).toBe(true);
}
});
it.each([
{ name: "no delivery update", patch: { enabled: true } satisfies CronJobPatch },
{
name: "blank webhook target",
patch: { delivery: { mode: "webhook", to: "" } } satisfies CronJobPatch,
},
{
name: "non-http protocol",
patch: {
delivery: { mode: "webhook", to: "ftp://example.invalid" },
} satisfies CronJobPatch,
},
{
name: "invalid URL",
patch: { delivery: { mode: "webhook", to: "not-a-url" } } satisfies CronJobPatch,
},
] as const)("rejects invalid webhook delivery target URL: $name", ({ patch }) => {
const expectedError = "cron webhook delivery requires delivery.to to be a valid http(s) URL";
const job = createMainSystemEventJob("job-webhook-invalid", { mode: "webhook" });
expect(() => applyJobPatch(job, patch)).toThrow(expectedError);
});
it("trims webhook delivery target URLs", () => {
const job = createMainSystemEventJob("job-webhook-trim", {
mode: "webhook",
to: "https://example.invalid/original",
});
expect(() =>
applyJobPatch(job, { delivery: { mode: "webhook", to: " https://example.invalid/trim " } }),
).not.toThrow();
expect(job.delivery).toEqual({ mode: "webhook", to: "https://example.invalid/trim" });
});
it("rejects failureDestination on main jobs without webhook delivery mode", () => {
const job = createMainSystemEventJob("job-main-failure-dest", {
mode: "announce",
channel: "telegram",
to: "123",
failureDestination: {
mode: "announce",
channel: "telegram",
to: "999",
},
});
expect(() => applyJobPatch(job, { enabled: true })).toThrow(
'cron delivery.failureDestination is only supported for sessionTarget="isolated" unless delivery.mode="webhook"',
);
});
it("validates and trims webhook failureDestination target URLs", () => {
const expectedError =
"cron failure destination webhook requires delivery.failureDestination.to to be a valid http(s) URL";
const job = createIsolatedAgentTurnJob("job-failure-webhook-target", {
mode: "announce",
channel: "telegram",
to: "123",
failureDestination: {
mode: "webhook",
to: "not-a-url",
},
});
expect(() => applyJobPatch(job, { enabled: true })).toThrow(expectedError);
job.delivery = {
mode: "announce",
channel: "telegram",
to: "123",
failureDestination: {
mode: "webhook",
to: " https://example.invalid/failure ",
},
};
expect(() => applyJobPatch(job, { enabled: true })).not.toThrow();
expect(job.delivery?.failureDestination?.to).toBe("https://example.invalid/failure");
});
it("rejects Telegram delivery with invalid target (chatId/topicId format)", () => {
const job = createIsolatedAgentTurnJob("job-telegram-invalid", {
mode: "announce",
channel: "telegram",
to: "-10012345/6789",
});
expect(() => applyJobPatch(job, { enabled: true })).toThrow(
'Invalid Telegram delivery target "-10012345/6789". Use colon (:) as delimiter for topics, not slash. Valid formats: -1001234567890, -1001234567890:123, -1001234567890:topic:123, @username, https://t.me/username',
);
});
it.each([
{ name: "t.me URL", to: "https://t.me/mychannel" },
{ name: "t.me URL (no https)", to: "t.me/mychannel" },
{ name: "valid target (plain chat id)", to: "-1001234567890" },
{ name: "valid target (colon delimiter)", to: "-1001234567890:123" },
{ name: "valid target (topic marker)", to: "-1001234567890:topic:456" },
{ name: "@username", to: "@mybot" },
{ name: "without target", to: undefined },
] as const)("accepts Telegram delivery with $name", ({ to }) => {
const job = createIsolatedAgentTurnJob("job-telegram-valid", {
mode: "announce",
channel: "telegram",
...(to ? { to } : {}),
});
expect(() => applyJobPatch(job, { enabled: true })).not.toThrow();
});
});
function createMockState(now: number, opts?: { defaultAgentId?: string }): CronServiceState {
return {
deps: {
nowMs: () => now,
defaultAgentId: opts?.defaultAgentId,
},
} as unknown as CronServiceState;
}
describe("createJob rejects sessionTarget main for non-default agents", () => {
const now = Date.parse("2026-02-28T12:00:00.000Z");
const mainJobInput = (agentId?: string) => ({
name: "my-main-job",
enabled: true,
schedule: { kind: "every" as const, everyMs: 60_000 },
sessionTarget: "main" as const,
wakeMode: "now" as const,
payload: { kind: "systemEvent" as const, text: "tick" },
...(agentId !== undefined ? { agentId } : {}),
});
it.each([
{ name: "default agent", defaultAgentId: "main", agentId: undefined },
{ name: "explicit default agent", defaultAgentId: "main", agentId: "main" },
{ name: "case-insensitive defaultAgentId match", defaultAgentId: "Main", agentId: "MAIN" },
] as const)("allows creating a main-session job for $name", ({ defaultAgentId, agentId }) => {
const state = createMockState(now, { defaultAgentId });
expect(() => createJob(state, mainJobInput(agentId))).not.toThrow();
});
it.each([
{ name: "non-default agentId", defaultAgentId: "main", agentId: "custom-agent" },
{ name: "missing defaultAgentId", defaultAgentId: undefined, agentId: "custom-agent" },
] as const)("rejects creating a main-session job for $name", ({ defaultAgentId, agentId }) => {
const state = createMockState(now, defaultAgentId ? { defaultAgentId } : undefined);
expect(() => createJob(state, mainJobInput(agentId))).toThrow(
'cron: sessionTarget "main" is only valid for the default agent',
);
});
it("allows isolated session job for non-default agents", () => {
const state = createMockState(now, { defaultAgentId: "main" });
expect(() =>
createJob(state, {
name: "isolated-job",
enabled: true,
schedule: { kind: "every", everyMs: 60_000 },
sessionTarget: "isolated",
wakeMode: "now",
payload: { kind: "agentTurn", message: "do it" },
agentId: "custom-agent",
}),
).not.toThrow();
});
it("rejects failureDestination on main jobs without webhook delivery mode", () => {
const state = createMockState(now, { defaultAgentId: "main" });
expect(() =>
createJob(state, {
...mainJobInput("main"),
delivery: {
mode: "announce",
channel: "telegram",
to: "123",
failureDestination: {
mode: "announce",
channel: "signal",
to: "+15550001111",
},
},
}),
).toThrow('cron channel delivery config is only supported for sessionTarget="isolated"');
});
});
describe("applyJobPatch rejects sessionTarget main for non-default agents", () => {
const now = Date.now();
const createMainJob = (agentId?: string): CronJob => ({
id: "job-main-agent-check",
name: "main-agent-check",
enabled: true,
createdAtMs: now,
updatedAtMs: now,
schedule: { kind: "every", everyMs: 60_000 },
sessionTarget: "main",
wakeMode: "now",
payload: { kind: "systemEvent", text: "tick" },
state: {},
agentId,
});
it.each([
{ name: "rejects patching agentId to non-default", agentId: "custom-agent", shouldThrow: true },
{ name: "allows patching agentId to the default agent", agentId: "main", shouldThrow: false },
] as const)("$name on a main-session job", ({ agentId, shouldThrow }) => {
const job = createMainJob();
const patch = { agentId } as CronJobPatch;
if (shouldThrow) {
expect(() => applyJobPatch(job, patch, { defaultAgentId: "main" })).toThrow(
'cron: sessionTarget "main" is only valid for the default agent',
);
return;
}
expect(() => applyJobPatch(job, patch, { defaultAgentId: "main" })).not.toThrow();
});
});
describe("cron stagger defaults", () => {
it("defaults top-of-hour cron jobs to 5m stagger", () => {
const now = Date.parse("2026-02-08T10:00:00.000Z");
const state = createMockState(now);
const job = createJob(state, {
name: "hourly",
enabled: true,
schedule: { kind: "cron", expr: "0 * * * *", tz: "UTC" },
sessionTarget: "main",
wakeMode: "now",
payload: { kind: "systemEvent", text: "tick" },
});
expectCronStaggerMs(job, DEFAULT_TOP_OF_HOUR_STAGGER_MS);
});
it("keeps exact schedules when staggerMs is explicitly 0", () => {
const now = Date.parse("2026-02-08T10:00:00.000Z");
const state = createMockState(now);
const job = createJob(state, {
name: "exact-hourly",
enabled: true,
schedule: { kind: "cron", expr: "0 * * * *", tz: "UTC", staggerMs: 0 },
sessionTarget: "main",
wakeMode: "now",
payload: { kind: "systemEvent", text: "tick" },
});
expectCronStaggerMs(job, 0);
});
it("preserves existing stagger when editing cron expression without stagger", () => {
const now = Date.now();
const job: CronJob = {
id: "job-keep-stagger",
name: "job-keep-stagger",
enabled: true,
createdAtMs: now,
updatedAtMs: now,
schedule: { kind: "cron", expr: "0 * * * *", tz: "UTC", staggerMs: 120_000 },
sessionTarget: "main",
wakeMode: "now",
payload: { kind: "systemEvent", text: "tick" },
state: {},
};
applyJobPatch(job, {
schedule: { kind: "cron", expr: "0 */2 * * *", tz: "UTC" },
});
expect(job.schedule.kind).toBe("cron");
if (job.schedule.kind === "cron") {
expect(job.schedule.expr).toBe("0 */2 * * *");
expect(job.schedule.staggerMs).toBe(120_000);
}
});
it("applies default stagger when switching from every to top-of-hour cron", () => {
const now = Date.now();
const job: CronJob = {
id: "job-switch-cron",
name: "job-switch-cron",
enabled: true,
createdAtMs: now,
updatedAtMs: now,
schedule: { kind: "every", everyMs: 60_000 },
sessionTarget: "main",
wakeMode: "now",
payload: { kind: "systemEvent", text: "tick" },
state: {},
};
applyJobPatch(job, {
schedule: { kind: "cron", expr: "0 * * * *", tz: "UTC" },
});
expect(job.schedule.kind).toBe("cron");
if (job.schedule.kind === "cron") {
expect(job.schedule.staggerMs).toBe(DEFAULT_TOP_OF_HOUR_STAGGER_MS);
}
});
});
describe("createJob delivery defaults", () => {
const now = Date.parse("2026-02-28T12:00:00.000Z");
it('defaults delivery to { mode: "announce" } for isolated agentTurn jobs without explicit delivery', () => {
const state = createMockState(now);
const job = createJob(state, {
name: "isolated-no-delivery",
enabled: true,
schedule: { kind: "every", everyMs: 60_000 },
sessionTarget: "isolated",
wakeMode: "now",
payload: { kind: "agentTurn", message: "hello" },
});
expect(job.delivery).toEqual({ mode: "announce" });
});
it("preserves explicit delivery for isolated agentTurn jobs", () => {
const state = createMockState(now);
const job = createJob(state, {
name: "isolated-explicit-delivery",
enabled: true,
schedule: { kind: "every", everyMs: 60_000 },
sessionTarget: "isolated",
wakeMode: "now",
payload: { kind: "agentTurn", message: "hello" },
delivery: { mode: "none" },
});
expect(job.delivery).toEqual({ mode: "none" });
});
it("does not set delivery for main systemEvent jobs without explicit delivery", () => {
const state = createMockState(now, { defaultAgentId: "main" });
const job = createJob(state, {
name: "main-no-delivery",
enabled: true,
schedule: { kind: "every", everyMs: 60_000 },
sessionTarget: "main",
wakeMode: "now",
payload: { kind: "systemEvent", text: "ping" },
});
expect(job.delivery).toBeUndefined();
});
});
describe("recomputeNextRuns", () => {
it("backfills missing every anchorMs for legacy loaded jobs", () => {
const now = Date.parse("2026-03-01T12:00:00.000Z");
const createdAtMs = now - 120_000;
const job: CronJob = {
id: "legacy-every",
name: "legacy-every",
enabled: true,
createdAtMs,
updatedAtMs: createdAtMs,
schedule: { kind: "every", everyMs: 60_000 },
sessionTarget: "main",
wakeMode: "now",
payload: { kind: "systemEvent", text: "tick" },
state: {},
};
const state = {
...createMockState(now),
store: { version: 1 as const, jobs: [job] },
} as CronServiceState;
expect(recomputeNextRuns(state)).toBe(true);
expect(job.schedule.kind).toBe("every");
if (job.schedule.kind === "every") {
expect(job.schedule.anchorMs).toBe(createdAtMs);
}
expect(job.state.nextRunAtMs).toBe(now);
});
});