diff --git a/docs/refactor/database-first.md b/docs/refactor/database-first.md index 73ca1e38480..3a50a28dc40 100644 --- a/docs/refactor/database-first.md +++ b/docs/refactor/database-first.md @@ -554,14 +554,15 @@ openclaw doctor --fix ``` `openclaw doctor --fix` invokes the same state migration implementation after -ordinary config preflight. Runtime startup must not import legacy files. +ordinary config preflight and creates a verified backup before import. Runtime +startup must not import legacy files. Migration properties: - One migration pass discovers all legacy file and sidecar database sources and produces a plan before mutating anything. -- A pre-migration backup archive is created unless explicitly disabled by a - dangerous force flag. +- A pre-migration backup archive is created. The standalone migrate command can + skip it only with an explicit dangerous force flag. - Imports are idempotent and keyed by source path, mtime, size, hash, and target table. - Successful source files are removed or archived after the target database has diff --git a/src/commands/doctor-state-migrations.test.ts b/src/commands/doctor-state-migrations.test.ts index 31701c50c94..edf8b9320e5 100644 --- a/src/commands/doctor-state-migrations.test.ts +++ b/src/commands/doctor-state-migrations.test.ts @@ -14,10 +14,8 @@ import { closeOpenClawAgentDatabasesForTest } from "../state/openclaw-agent-db.j import { closeOpenClawStateDatabaseForTest } from "../state/openclaw-state-db.js"; import { autoMigrateLegacyStateDir, - autoMigrateLegacyState, detectLegacyStateMigrations, resetAutoMigrateLegacyStateDirForTest, - resetAutoMigrateLegacyStateForTest, runLegacyStateMigrations, } from "./doctor-state-migrations.js"; @@ -145,7 +143,6 @@ afterEach(async () => { resetPluginStateStoreForTests(); closeOpenClawAgentDatabasesForTest(); closeOpenClawStateDatabaseForTest(); - resetAutoMigrateLegacyStateForTest(); resetAutoMigrateLegacyStateDirForTest(); await Promise.all( tempRoots.map((root) => fs.promises.rm(root, { recursive: true, force: true })), @@ -236,21 +233,6 @@ async function runFreshStateDirMigration(root: string, env = {} as NodeJS.Proces return runStateDirMigration(root, env); } -async function runAutoMigrateLegacyStateWithLog(params: { - root: string; - cfg: OpenClawConfig; - now?: () => number; -}) { - const log = { info: vi.fn(), warn: vi.fn() }; - const result = await autoMigrateLegacyState({ - cfg: params.cfg, - env: { OPENCLAW_STATE_DIR: params.root } as NodeJS.ProcessEnv, - log, - now: params.now, - }); - return { result, log }; -} - function expectTargetAlreadyExistsWarning(result: StateDirMigrationResult, targetDir: string) { expect(result.migrated).toBe(false); expect(result.warnings).toEqual([ @@ -730,46 +712,6 @@ describe("doctor legacy state migrations", () => { expect(fs.existsSync(path.join(backupDir, "foo.txt"))).toBe(true); }); - it("auto-migrates legacy agent dir on startup", async () => { - const { root, cfg } = await makeRootWithEmptyCfg(); - writeLegacyAgentFiles(root, { "auth.json": "{}" }); - - const { result, log } = await runAutoMigrateLegacyStateWithLog({ root, cfg }); - - const targetAgentDir = path.join(root, "agents", "main", "agent"); - expect(fs.existsSync(path.join(targetAgentDir, "auth.json"))).toBe(true); - expect(result.migrated).toBe(true); - expect(log.info).toHaveBeenCalled(); - }); - - it("leaves legacy sessions for doctor on startup", async () => { - const { root, cfg } = await makeRootWithEmptyCfg(); - const legacySessionsDir = writeLegacySessionsFixture({ - root, - sessions: { - "+1555": { sessionId: "a", updatedAt: 10 }, - }, - transcripts: { - "a.jsonl": "a", - }, - }); - - const { result, log } = await runAutoMigrateLegacyStateWithLog({ - root, - cfg, - now: () => 123, - }); - - expect(result.migrated).toBe(false); - expect(log.info).not.toHaveBeenCalled(); - - const targetDir = path.join(root, "agents", "main", "sessions"); - expect(fs.existsSync(path.join(targetDir, "a.jsonl"))).toBe(false); - expect(fs.existsSync(path.join(legacySessionsDir, "a.jsonl"))).toBe(true); - expect(fs.existsSync(path.join(legacySessionsDir, "sessions.json"))).toBe(true); - expect(Object.keys(readSessionsStore({ root, targetDir }))).toHaveLength(0); - }); - it("migrates legacy WhatsApp auth files without touching oauth.json", async () => { const { root, cfg } = await makeRootWithEmptyCfg(); const oauthDir = ensureCredentialsDir(root); @@ -896,21 +838,6 @@ describe("doctor legacy state migrations", () => { expect(store["agent:main:slack:channel:C123"]).toBeUndefined(); }); - it("leaves target sessions with legacy keys for doctor on startup", async () => { - const { root, cfg } = await makeRootWithEmptyCfg(); - const targetDir = path.join(root, "agents", "main", "sessions"); - writeJson5(path.join(targetDir, "sessions.json"), { - main: { sessionId: "legacy", updatedAt: 10 }, - }); - - const { result, log } = await runAutoMigrateLegacyStateWithLog({ root, cfg }); - - expect(result.migrated).toBe(false); - expect(log.info).not.toHaveBeenCalled(); - expect(fs.existsSync(path.join(targetDir, "sessions.json"))).toBe(true); - expect(Object.keys(readSessionsStore({ root, targetDir }))).toHaveLength(0); - }); - it("does nothing when no legacy state dir exists", async () => { const root = await makeTempRoot(); const result = await runStateDirMigration(root); diff --git a/src/commands/doctor-state-migrations.ts b/src/commands/doctor-state-migrations.ts index 50c59a3a0ad..1308b412d90 100644 --- a/src/commands/doctor-state-migrations.ts +++ b/src/commands/doctor-state-migrations.ts @@ -1,12 +1,8 @@ export type { LegacyStateDetection } from "../infra/state-migrations.js"; export { autoMigrateLegacyStateDir, - autoMigrateLegacyAgentDir, - autoMigrateLegacyState, detectLegacyStateMigrations, migrateLegacyAgentDir, resetAutoMigrateLegacyStateDirForTest, - resetAutoMigrateLegacyAgentDirForTest, - resetAutoMigrateLegacyStateForTest, runLegacyStateMigrations, } from "../infra/state-migrations.js"; diff --git a/src/commands/doctor.e2e-harness.ts b/src/commands/doctor.e2e-harness.ts index 40710e7c4f5..3b86177ec2c 100644 --- a/src/commands/doctor.e2e-harness.ts +++ b/src/commands/doctor.e2e-harness.ts @@ -141,6 +141,9 @@ export const autoMigrateLegacyStateDir = vi.fn().mockResolvedValue({ changes: [], warnings: [], }) as unknown as MockFn; +export const createPreMigrationBackup = vi + .fn() + .mockResolvedValue("/tmp/openclaw-backup.tgz") as unknown as MockFn; export const runChannelPluginStartupMaintenance = vi .fn() .mockResolvedValue(undefined) as unknown as MockFn; @@ -462,6 +465,10 @@ vi.mock("./doctor-state-migrations.js", () => ({ runLegacyStateMigrations, })); +vi.mock("../commands/migrate/apply.js", () => ({ + createPreMigrationBackup, +})); + vi.mock("../channels/plugins/lifecycle-startup.js", () => ({ runChannelPluginStartupMaintenance, })); @@ -506,6 +513,7 @@ export async function arrangeLegacyStateMigrationTest(): Promise<{ detectLegacyStateMigrations.mockClear(); runLegacyStateMigrations.mockClear(); + createPreMigrationBackup.mockClear(); detectLegacyStateMigrations.mockResolvedValueOnce( createLegacyStateMigrationDetectionResult({ hasLegacySessions: true, @@ -524,6 +532,7 @@ export async function arrangeLegacyStateMigrationTest(): Promise<{ runtime, detectLegacyStateMigrations, runLegacyStateMigrations, + createPreMigrationBackup, }; } diff --git a/src/commands/doctor.runs-legacy-state-migrations-yes-mode-without.e2e.test.ts b/src/commands/doctor.runs-legacy-state-migrations-yes-mode-without.e2e.test.ts index 65171bee030..732dd160b15 100644 --- a/src/commands/doctor.runs-legacy-state-migrations-yes-mode-without.e2e.test.ts +++ b/src/commands/doctor.runs-legacy-state-migrations-yes-mode-without.e2e.test.ts @@ -39,7 +39,7 @@ describe("doctor command", () => { }); it("runs legacy state migrations in yes mode without prompting", async () => { - const { doctorCommand, runtime, runLegacyStateMigrations } = + const { doctorCommand, runtime, runLegacyStateMigrations, createPreMigrationBackup } = await arrangeLegacyStateMigrationTest(); await (doctorCommand as (runtime: unknown, opts: Record) => Promise)( @@ -47,12 +47,16 @@ describe("doctor command", () => { { yes: true }, ); + expect(createPreMigrationBackup).toHaveBeenCalledTimes(1); expect(runLegacyStateMigrations).toHaveBeenCalledTimes(1); + expect(runLegacyStateMigrations).toHaveBeenCalledWith( + expect.objectContaining({ backupPath: "/tmp/openclaw-backup.tgz" }), + ); expect(confirm).not.toHaveBeenCalled(); }, 30_000); it("runs legacy state migrations in non-interactive mode without prompting", async () => { - const { doctorCommand, runtime, runLegacyStateMigrations } = + const { doctorCommand, runtime, runLegacyStateMigrations, createPreMigrationBackup } = await arrangeLegacyStateMigrationTest(); await (doctorCommand as (runtime: unknown, opts: Record) => Promise)( @@ -60,7 +64,11 @@ describe("doctor command", () => { { nonInteractive: true }, ); + expect(createPreMigrationBackup).toHaveBeenCalledTimes(1); expect(runLegacyStateMigrations).toHaveBeenCalledTimes(1); + expect(runLegacyStateMigrations).toHaveBeenCalledWith( + expect.objectContaining({ backupPath: "/tmp/openclaw-backup.tgz" }), + ); expect(confirm).not.toHaveBeenCalled(); }, 30_000); diff --git a/src/flows/doctor-health-contributions.ts b/src/flows/doctor-health-contributions.ts index 2d5f8e7dd72..4bb0da9f4dc 100644 --- a/src/flows/doctor-health-contributions.ts +++ b/src/flows/doctor-health-contributions.ts @@ -224,6 +224,7 @@ async function runClaudeCliHealth(ctx: DoctorHealthFlowContext): Promise { async function runLegacyStateHealth(ctx: DoctorHealthFlowContext): Promise { const { detectLegacyStateMigrations, runLegacyStateMigrations } = await import("../commands/doctor-state-migrations.js"); + const { createPreMigrationBackup } = await import("../commands/migrate/apply.js"); const { note } = await import("../terminal/note.js"); const legacyState = await detectLegacyStateMigrations({ cfg: ctx.cfg }); if (legacyState.preview.length === 0) { @@ -240,8 +241,13 @@ async function runLegacyStateHealth(ctx: DoctorHealthFlowContext): Promise if (!migrate) { return; } + const backupPath = await createPreMigrationBackup({}); + if (backupPath) { + note(backupPath, "Backup"); + } const migrated = await runLegacyStateMigrations({ detected: legacyState, + backupPath, }); if (migrated.changes.length > 0) { note(migrated.changes.join("\n"), "Doctor changes"); diff --git a/src/infra/state-migrations.ts b/src/infra/state-migrations.ts index 7f3464aefb3..065978d58ec 100644 --- a/src/infra/state-migrations.ts +++ b/src/infra/state-migrations.ts @@ -33,7 +33,6 @@ import { CRESTODIAN_RESCUE_PENDING_OWNER_ID, isRescuePendingOperation, } from "../crestodian/rescue-pending-state.js"; -import { createSubsystemLogger } from "../logging/subsystem.js"; import { buildAgentMainSessionKey, DEFAULT_AGENT_ID, @@ -110,9 +109,7 @@ type MigrationLogger = { warn: (message: string) => void; }; -let autoMigrateChecked = false; let autoMigrateStateDirChecked = false; -let cachedLegacySessionSurfaces: LegacySessionSurface[] | null = null; type LegacySessionSurface = { isLegacyGroupSessionKey?: (key: string) => boolean; @@ -170,6 +167,7 @@ function legacyMigrationSourceKey(source: MigrationSourceReport): string { source.targetTable ?? "", source.sha256 ?? "", String(source.sizeBytes ?? ""), + String(source.mtimeMs ?? ""), String(source.recordCount ?? ""), ].join("\0"), "utf8", @@ -313,8 +311,7 @@ function getLegacySessionSurfaces(): LegacySessionSurface[] { // Legacy migrations run on cold doctor/startup paths. Prefer the narrower // setup plugin surface here so session-key cleanup does not materialize full // bundled channel runtimes. - cachedLegacySessionSurfaces ??= [...listBundledChannelLegacySessionSurfaces()]; - return cachedLegacySessionSurfaces; + return [...listBundledChannelLegacySessionSurfaces()]; } function isSurfaceGroupKey(key: string): boolean { @@ -1989,15 +1986,6 @@ function collectLegacyMigrationSources(detected: LegacyStateDetection): Migratio ); } -export function resetAutoMigrateLegacyStateForTest() { - autoMigrateChecked = false; - cachedLegacySessionSurfaces = null; -} - -export function resetAutoMigrateLegacyAgentDirForTest() { - resetAutoMigrateLegacyStateForTest(); -} - export function resetAutoMigrateLegacyStateDirForTest() { autoMigrateStateDirChecked = false; } @@ -2969,105 +2957,3 @@ export async function runLegacyStateMigrations(params: { throw error; } } - -export async function autoMigrateLegacyAgentDir(params: { - cfg: OpenClawConfig; - env?: NodeJS.ProcessEnv; - homedir?: () => string; - log?: MigrationLogger; - now?: () => number; -}): Promise<{ - migrated: boolean; - skipped: boolean; - changes: string[]; - warnings: string[]; -}> { - return await autoMigrateLegacyState(params); -} - -export async function autoMigrateLegacyState(params: { - cfg: OpenClawConfig; - env?: NodeJS.ProcessEnv; - homedir?: () => string; - log?: MigrationLogger; - now?: () => number; -}): Promise<{ - migrated: boolean; - skipped: boolean; - changes: string[]; - warnings: string[]; -}> { - if (autoMigrateChecked) { - return { migrated: false, skipped: true, changes: [], warnings: [] }; - } - autoMigrateChecked = true; - - const env = params.env ?? process.env; - const stateDirResult = await autoMigrateLegacyStateDir({ - env, - homedir: params.homedir, - log: params.log, - }); - - const logMigrationResults = (changes: string[], warnings: string[]) => { - const logger = params.log ?? createSubsystemLogger("state-migrations"); - if (changes.length > 0) { - logger.info( - `Auto-migrated legacy state:\n${changes.map((entry) => `- ${entry}`).join("\n")}`, - ); - } - if (warnings.length > 0) { - logger.warn( - `Legacy state migration warnings:\n${warnings.map((entry) => `- ${entry}`).join("\n")}`, - ); - } - }; - - if ( - normalizeEnvPathOverride(env.OPENCLAW_AGENT_DIR) || - normalizeEnvPathOverride(env.PI_CODING_AGENT_DIR) - ) { - const changes = [...stateDirResult.changes]; - const warnings = [...stateDirResult.warnings]; - logMigrationResults(changes, warnings); - return { - migrated: stateDirResult.migrated, - skipped: true, - changes, - warnings, - }; - } - - const detected = await detectLegacyStateMigrations({ - cfg: params.cfg, - env, - homedir: params.homedir, - includeSessions: false, - includeChannelPlans: false, - }); - if (!detected.agentDir.hasLegacy) { - const changes = [...stateDirResult.changes]; - const warnings = [...stateDirResult.warnings]; - logMigrationResults(changes, warnings); - return { - migrated: stateDirResult.migrated, - skipped: false, - changes, - warnings, - }; - } - - const now = params.now ?? (() => Date.now()); - const agentDir = await migrateLegacyAgentDir(detected, now); - const changes = [...stateDirResult.changes, ...agentDir.changes]; - const warnings = [...stateDirResult.warnings, ...agentDir.warnings]; - - logMigrationResults(changes, warnings); - - return { - migrated: changes.length > 0, - skipped: false, - changes, - warnings, - }; -}