diff --git a/CHANGELOG.md b/CHANGELOG.md index 2e2937ca537..855565da911 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -12,7 +12,7 @@ Docs: https://docs.openclaw.ai - Gateway/sessions: fast-path already-qualified model refs while building session-list rows so `openclaw sessions` and Control UI session lists avoid heavyweight model resolution on large stores. (#77902) Thanks @ragesaq. - Gateway/sessions: remove the automatic cron session reaper and retired `cron.sessionRetention`; use `openclaw sessions cleanup` for session-row maintenance while cron run-log pruning remains under `cron.runLog`. - Cron/state: store runtime schedule state and run history in the shared SQLite state database; `openclaw doctor --fix` imports legacy `jobs-state.json` and `cron/runs/*.jsonl` files. -- Gateway/state: store device identity/auth, bootstrap tokens, device and node pairing ledgers, channel pairing requests/allowlists, inferred commitments, web push subscriptions/VAPID keys, and APNs registrations in the shared SQLite state database; `openclaw doctor --fix` imports and removes the legacy JSON files. +- Gateway/state: store device identity/auth, bootstrap tokens, device and node pairing ledgers, channel pairing requests/allowlists, inferred commitments, subagent run records, TUI restore pointers, auth routing state, OpenRouter model cache, web push subscriptions/VAPID keys, APNs registrations, and update-check state in the shared SQLite state database; `openclaw doctor --fix` imports and removes the legacy JSON files. - PR triage: mark external pull requests with `proof: supplied` when Barnacle finds structured real behavior proof, keep stale negative proof labels in sync across CRLF-edited PR bodies, and let ClawSweeper own the stronger `proof: sufficient` judgement. - Sessions CLI: show the selected agent runtime in the `openclaw sessions` table so terminal output matches the runtime visibility already present in JSON/status surfaces. Thanks @vincentkoc. - Talk/voice: unify realtime relay, transcription relay, managed-room handoff, Voice Call, Google Meet, VoiceClaw, and native clients around a shared Talk session controller and add the Gateway-managed `talk.session.*` RPC surface. diff --git a/docs/refactor/piless.md b/docs/refactor/piless.md index dd62e96345a..8728c568a6c 100644 --- a/docs/refactor/piless.md +++ b/docs/refactor/piless.md @@ -23,7 +23,8 @@ OpenClaw currently embeds PI directly. The main loop still imports and `@mariozechner/pi-tui` across agent runtime, tool, provider, transcript, and TUI paths. See [PI integration architecture](/pi). -Session state is split across several persistence mechanisms: +Before this refactor, session and runtime state was split across several +persistence mechanisms: - Gateway session index: `sessions.json` - Session transcripts: `*.jsonl` @@ -34,8 +35,10 @@ Session state is split across several persistence mechanisms: - Memory indexes: SQLite or QMD-owned SQLite - Plugin-specific JSON and JSONL sidecars -This mix is workable, but it creates duplicated read, write, migration, locking, -maintenance, and diagnostics code. +That mix was workable, but it created duplicated read, write, migration, +locking, maintenance, and diagnostics code. The branch now moves canonical +runtime state into the shared SQLite database and treats old JSON files as +doctor migration inputs, not runtime compatibility stores. ## Current Implementation Status @@ -99,8 +102,8 @@ This plan has started landing in slices: media-result manifests for generated or captured tool media in the same run-scoped artifact store while keeping delivery files on disk. - Managed outgoing image attachment metadata now uses the shared SQLite `kv` - store as the primary record path. Older per-attachment JSON files import into - SQLite when encountered and are removed after import. + store as the primary record path. Older per-attachment JSON files are imported + and removed by `openclaw doctor --fix`; runtime media reads only SQLite. - Cron runtime schedule state and run history now use the shared SQLite state database. `openclaw doctor --fix` imports legacy `jobs-state.json` and `cron/runs/*.jsonl` files into SQLite and removes those file sources after a @@ -108,37 +111,41 @@ This plan has started landing in slices: run-history JSON files; `jobs.json` remains the hand-editable job definition file. - The subagent run registry now uses the shared SQLite `kv` store as the - primary record path. Legacy `subagents/runs.json` files import into SQLite - when SQLite is empty and are removed after import. + primary record path. `openclaw doctor --fix` imports legacy + `subagents/runs.json` files into SQLite and removes them after import. + Runtime paths no longer import or delete that JSON file. - Sandbox container and browser registries now use the shared SQLite `kv` store - as the primary record path. Existing sharded JSON entries import into SQLite - on first read, and the shard files remain compatibility exports for downgrade - and debugging workflows. + as the primary record path. Legacy monolithic registry files migrate only + through `openclaw doctor --fix`; runtime reads no longer import registry JSON. - OpenRouter model capability cache now uses the shared SQLite `kv` store as the primary persistent cache. The older - `cache/openrouter-models.json` file is a legacy import source and is removed - after import. + `cache/openrouter-models.json` file is imported and removed by + `openclaw doctor --fix`, not by runtime cache reads. - Codex app-server thread bindings now use the shared SQLite `kv` store as the only runtime record path. The old per-session `.codex-app-server.json` sidecar reader/writer has been removed from runtime and tests now seed the binding store directly. `openclaw doctor --fix` imports old sidecars into SQLite and removes the JSON source. - TUI last-session restore pointers now use the shared SQLite `kv` store as the - primary record path. The older `tui/last-session.json` file is a legacy - import source and is removed after import. + primary record path. The older `tui/last-session.json` file is imported and + removed by `openclaw doctor --fix`; runtime TUI reads only SQLite. - Auth profile runtime routing state now uses the shared SQLite `kv` store as - the primary record path. The older per-agent `auth-state.json` file is a - legacy import source and is removed after import; `auth-profiles.json` still - owns credentials and stays file-backed. + the primary record path. Older per-agent `auth-state.json` files are imported + and removed by `openclaw doctor --fix`; `auth-profiles.json` still owns + credentials and stays file-backed. - Device identity, local device auth tokens, bootstrap tokens, device/node pairing ledgers, channel pairing requests/allowlists, inferred commitment - records, web push subscriptions/VAPID keys, and APNs registration state now + records, subagent run records, TUI restore pointers, auth routing state, + OpenRouter model cache, web push subscriptions/VAPID keys, APNs registration + state, and update-check state now use the shared SQLite `kv` store. `openclaw doctor --fix` imports the legacy `identity/*.json`, `devices/*.json`, `nodes/*.json`, `credentials/*-pairing.json`, `credentials/*-allowFrom.json`, - `commitments/commitments.json`, and `push/*.json` files into SQLite and - removes those files after a successful import. Runtime paths no longer read - or write those JSON ledgers. + `commitments/commitments.json`, `subagents/runs.json`, + `tui/last-session.json`, per-agent `auth-state.json`, + `cache/openrouter-models.json`, `push/*.json`, and `update-check.json` files + into SQLite and removes those files after a successful import. Runtime paths + no longer read or write those JSON ledgers. - `AgentRuntimeBackend`, `PreparedAgentRun`, and the Node worker runner exist for serializable prepared runs. `RunEventBus` owns serial parent event delivery for worker event streams. The worker runner enforces prepared-run @@ -254,7 +261,7 @@ Use three explicit layers: ```text agent runtime boundary OpenClaw-owned interface, PI as one backend -agent state database SQLite primary store, legacy JSON import where needed +agent state database SQLite primary store, doctor-only legacy JSON import agent filesystem boundary VFS scratch plus host capability filesystem ``` @@ -544,18 +551,17 @@ Add tests before each migration step: - Plugin state and task registry coexistence with the shared state DB. - Managed outgoing media record import from legacy JSON, legacy file removal after import, plus SQLite-primary serving without JSON exports. -- Subagent run registry import from legacy `subagents/runs.json`, legacy file - removal after import, and restore from SQLite without JSON exports. -- Sandbox container and browser registry reads from SQLite when compatibility - shard files are missing, while legacy monolithic registry migration stays an - explicit repair operation. -- OpenRouter model capability cache reads from SQLite when the legacy JSON - cache file is missing, imports old cache JSON, and removes it after import. +- Subagent run registry import from legacy `subagents/runs.json` during doctor, + legacy file removal after import, and restore from SQLite without JSON + exports. +- Sandbox container and browser registry reads from SQLite, while legacy + monolithic registry migration stays an explicit doctor repair operation. +- OpenRouter model capability cache reads from SQLite, with old cache JSON + imported and removed only by doctor. - TUI last-session restore pointers read from SQLite without JSON exports, - import legacy JSON on read, remove it, and clear stale pointers from SQLite. -- Auth profile runtime state reads from SQLite when the compatibility - `auth-state.json` file is missing, imports legacy JSON on read, removes it, - and deletes SQLite state when runtime state is empty. + import legacy JSON only through doctor, and clear stale pointers from SQLite. +- Auth profile runtime state reads from SQLite, imports legacy JSON only through + doctor, and deletes SQLite state when runtime state is empty. ## Rollout Plan diff --git a/src/agents/auth-profiles/state.test.ts b/src/agents/auth-profiles/state.test.ts index 8ba2296b19b..8557705041e 100644 --- a/src/agents/auth-profiles/state.test.ts +++ b/src/agents/auth-profiles/state.test.ts @@ -5,7 +5,11 @@ import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; import { closeOpenClawStateDatabaseForTest } from "../../state/openclaw-state-db.js"; import { readOpenClawStateKvJson } from "../../state/openclaw-state-kv.js"; import { resolveAuthStatePath } from "./paths.js"; -import { loadPersistedAuthProfileState, savePersistedAuthProfileState } from "./state.js"; +import { + importLegacyAuthProfileStateFileToSqlite, + loadPersistedAuthProfileState, + savePersistedAuthProfileState, +} from "./state.js"; const AUTH_PROFILE_STATE_KV_SCOPE = "auth-profile-state"; @@ -46,7 +50,7 @@ describe("auth profile runtime state persistence", () => { }); }); - it("imports legacy auth-state.json into SQLite on read and removes the file", async () => { + it("imports legacy auth-state.json into SQLite through the doctor migration helper", async () => { const statePath = resolveAuthStatePath(agentDir); await fs.writeFile( statePath, @@ -57,6 +61,8 @@ describe("auth profile runtime state persistence", () => { })}\n`, ); + expect(loadPersistedAuthProfileState(agentDir)).toEqual({}); + expect(importLegacyAuthProfileStateFileToSqlite(agentDir)).toEqual({ imported: true }); expect(loadPersistedAuthProfileState(agentDir)).toEqual({ order: { anthropic: ["anthropic:default"] }, lastGood: { anthropic: "anthropic:default" }, diff --git a/src/agents/auth-profiles/state.ts b/src/agents/auth-profiles/state.ts index 5baeaa51048..d6147d62494 100644 --- a/src/agents/auth-profiles/state.ts +++ b/src/agents/auth-profiles/state.ts @@ -1,4 +1,6 @@ import fs from "node:fs"; +import path from "node:path"; +import { resolveStateDir } from "../../config/paths.js"; import { loadJsonFile } from "../../infra/json-file.js"; import { normalizeOptionalString } from "../../shared/string-coerce.js"; import { @@ -8,6 +10,7 @@ import { type OpenClawStateJsonValue, } from "../../state/openclaw-state-kv.js"; import { AUTH_STORE_VERSION } from "./constants.js"; +import { AUTH_STATE_FILENAME } from "./path-constants.js"; import { resolveAuthStatePath } from "./paths.js"; import type { AuthProfileState, AuthProfileStateStore, ProfileUsageStats } from "./types.js"; @@ -82,6 +85,10 @@ function authProfileStateToJsonValue(state: AuthProfileStateStore): OpenClawStat return state as OpenClawStateJsonValue; } +function writeAuthProfileStatePayload(key: string, payload: AuthProfileStateStore): void { + writeOpenClawStateKvJson(AUTH_PROFILE_STATE_KV_SCOPE, key, authProfileStateToJsonValue(payload)); +} + export function loadPersistedAuthProfileState(agentDir?: string): AuthProfileState { const key = authProfileStateKey(agentDir); const sqliteState = readOpenClawStateKvJson(AUTH_PROFILE_STATE_KV_SCOPE, key); @@ -89,23 +96,7 @@ export function loadPersistedAuthProfileState(agentDir?: string): AuthProfileSta return coerceAuthProfileState(sqliteState); } - const legacyState = coerceAuthProfileState(loadJsonFile(key)); - const payload = buildPersistedAuthProfileState(legacyState); - if (payload) { - writeOpenClawStateKvJson( - AUTH_PROFILE_STATE_KV_SCOPE, - key, - authProfileStateToJsonValue(payload), - ); - try { - fs.unlinkSync(key); - } catch (error) { - if ((error as NodeJS.ErrnoException)?.code !== "ENOENT") { - throw error; - } - } - } - return legacyState; + return {}; } function buildPersistedAuthProfileState(store: AuthProfileState): AuthProfileStateStore | null { @@ -121,34 +112,67 @@ function buildPersistedAuthProfileState(store: AuthProfileState): AuthProfileSta }; } -export function savePersistedAuthProfileState( - store: AuthProfileState, - agentDir?: string, -): AuthProfileStateStore | null { - const payload = buildPersistedAuthProfileState(store); - const statePath = resolveAuthStatePath(agentDir); - if (!payload) { - deleteOpenClawStateKvJson(AUTH_PROFILE_STATE_KV_SCOPE, authProfileStateKey(agentDir)); - try { - fs.unlinkSync(statePath); - } catch (error) { - if ((error as NodeJS.ErrnoException)?.code !== "ENOENT") { - throw error; - } +export function legacyAuthProfileStateFileExists(agentDir?: string): boolean { + try { + return fs.statSync(resolveAuthStatePath(agentDir)).isFile(); + } catch (error) { + if ((error as NodeJS.ErrnoException)?.code === "ENOENT") { + return false; } - return null; + throw error; + } +} + +export function importLegacyAuthProfileStateFileToSqlite(agentDir?: string): { imported: boolean } { + const statePath = resolveAuthStatePath(agentDir); + if (!legacyAuthProfileStateFileExists(agentDir)) { + return { imported: false }; + } + const legacyState = coerceAuthProfileState(loadJsonFile(statePath)); + const payload = buildPersistedAuthProfileState(legacyState); + if (payload) { + writeAuthProfileStatePayload(statePath, payload); } - writeOpenClawStateKvJson( - AUTH_PROFILE_STATE_KV_SCOPE, - authProfileStateKey(agentDir), - authProfileStateToJsonValue(payload), - ); try { fs.unlinkSync(statePath); + } catch { + // Import succeeded; a later doctor pass can remove the stale file. + } + return { imported: true }; +} + +export function discoverLegacyAuthProfileStateAgentDirs( + env: NodeJS.ProcessEnv = process.env, +): string[] { + const agentsDir = path.join(resolveStateDir(env), "agents"); + const out: string[] = []; + try { + for (const entry of fs.readdirSync(agentsDir, { withFileTypes: true })) { + if (!entry.isDirectory()) { + continue; + } + const agentDir = path.join(agentsDir, entry.name, "agent"); + if (fs.existsSync(path.join(agentDir, AUTH_STATE_FILENAME))) { + out.push(agentDir); + } + } } catch (error) { if ((error as NodeJS.ErrnoException)?.code !== "ENOENT") { throw error; } } + return out; +} + +export function savePersistedAuthProfileState( + store: AuthProfileState, + agentDir?: string, +): AuthProfileStateStore | null { + const payload = buildPersistedAuthProfileState(store); + if (!payload) { + deleteOpenClawStateKvJson(AUTH_PROFILE_STATE_KV_SCOPE, authProfileStateKey(agentDir)); + return null; + } + writeAuthProfileStatePayload(authProfileStateKey(agentDir), payload); return payload; } diff --git a/src/agents/pi-embedded-runner/openrouter-model-capabilities.test.ts b/src/agents/pi-embedded-runner/openrouter-model-capabilities.test.ts index 35fc252f733..6f98da64951 100644 --- a/src/agents/pi-embedded-runner/openrouter-model-capabilities.test.ts +++ b/src/agents/pi-embedded-runner/openrouter-model-capabilities.test.ts @@ -83,7 +83,7 @@ describe("openrouter-model-capabilities", () => { }); }); - it("imports legacy JSON cache into SQLite and removes the file", async () => { + it("imports legacy JSON cache into SQLite through the doctor migration helper", async () => { await withOpenRouterStateDir(async (stateDir) => { const cachePath = join(stateDir, "cache", "openrouter-models.json"); mkdirSync(join(stateDir, "cache"), { recursive: true }); @@ -114,14 +114,21 @@ describe("openrouter-model-capabilities", () => { const module = await importOpenRouterModelCapabilities("legacy-json-cache"); await module.loadOpenRouterModelCapabilities("acme/legacy-json"); + expect(fetchSpy).toHaveBeenCalledTimes(1); - expect(module.getOpenRouterModelCapabilities("acme/legacy-json")).toMatchObject({ + expect(module.importLegacyOpenRouterModelCapabilitiesCacheToSqlite()).toEqual({ + imported: true, + models: 1, + }); + const migratedModule = await importOpenRouterModelCapabilities("legacy-json-cache-migrated"); + await migratedModule.loadOpenRouterModelCapabilities("acme/legacy-json"); + + expect(migratedModule.getOpenRouterModelCapabilities("acme/legacy-json")).toMatchObject({ name: "Legacy JSON", contextWindow: 111_000, maxTokens: 22_000, }); expect(existsSync(cachePath)).toBe(false); - expect(fetchSpy).not.toHaveBeenCalled(); }); }); diff --git a/src/agents/pi-embedded-runner/openrouter-model-capabilities.ts b/src/agents/pi-embedded-runner/openrouter-model-capabilities.ts index ec5e36d2c14..7f138bc8ce4 100644 --- a/src/agents/pi-embedded-runner/openrouter-model-capabilities.ts +++ b/src/agents/pi-embedded-runner/openrouter-model-capabilities.ts @@ -7,8 +7,7 @@ * Cache layers (checked in order): * 1. In-memory Map (instant, cleared on process restart) * 2. SQLite KV cache (/state/openclaw.sqlite) - * 3. Legacy JSON import (/cache/openrouter-models.json) - * 4. OpenRouter API fetch (populates all layers) + * 3. OpenRouter API fetch (populates SQLite) * * Model capabilities are assumed stable — the cache has no TTL expiry. * A background refresh is triggered only when a model is not found in @@ -25,6 +24,7 @@ import { resolveStateDir } from "../../config/paths.js"; import { formatErrorMessage } from "../../infra/errors.js"; import { resolveProxyFetchFromEnv } from "../../infra/net/proxy-fetch.js"; import { createSubsystemLogger } from "../../logging/subsystem.js"; +import type { OpenClawStateDatabaseOptions } from "../../state/openclaw-state-db.js"; import { readOpenClawStateKvJson, writeOpenClawStateKvJson, @@ -86,12 +86,12 @@ interface DiskCachePayload { // Disk cache // --------------------------------------------------------------------------- -function resolveDiskCacheDir(): string { - return join(resolveStateDir(), "cache"); +function resolveDiskCacheDir(env: NodeJS.ProcessEnv = process.env): string { + return join(resolveStateDir(env), "cache"); } -function resolveDiskCachePath(): string { - return join(resolveDiskCacheDir(), DISK_CACHE_FILENAME); +function resolveDiskCachePath(env: NodeJS.ProcessEnv = process.env): string { + return join(resolveDiskCacheDir(env), DISK_CACHE_FILENAME); } function mapToDiskCachePayload(map: Map): DiskCachePayload { @@ -100,9 +100,21 @@ function mapToDiskCachePayload(map: Map): D }; } -function writeSqliteCache(map: Map): void { +function sqliteOptionsForEnv(env?: NodeJS.ProcessEnv): OpenClawStateDatabaseOptions { + return env ? { env } : {}; +} + +function writeSqliteCache( + map: Map, + env?: NodeJS.ProcessEnv, +): void { try { - writeOpenClawStateKvJson(SQLITE_CACHE_SCOPE, SQLITE_CACHE_KEY, mapToDiskCachePayload(map)); + writeOpenClawStateKvJson( + SQLITE_CACHE_SCOPE, + SQLITE_CACHE_KEY, + mapToDiskCachePayload(map), + sqliteOptionsForEnv(env), + ); } catch (err: unknown) { const message = formatErrorMessage(err); log.debug(`Failed to write OpenRouter SQLite cache: ${message}`); @@ -144,17 +156,23 @@ function parseCachePayload(payload: unknown): Map 0 ? map : undefined; } -function readSqliteCache(): Map | undefined { +function readSqliteCache( + env?: NodeJS.ProcessEnv, +): Map | undefined { try { - return parseCachePayload(readOpenClawStateKvJson(SQLITE_CACHE_SCOPE, SQLITE_CACHE_KEY)); + return parseCachePayload( + readOpenClawStateKvJson(SQLITE_CACHE_SCOPE, SQLITE_CACHE_KEY, sqliteOptionsForEnv(env)), + ); } catch { return undefined; } } -function readDiskCache(): Map | undefined { +function readDiskCache( + env: NodeJS.ProcessEnv = process.env, +): Map | undefined { try { - const cachePath = resolveDiskCachePath(); + const cachePath = resolveDiskCachePath(env); if (!existsSync(cachePath)) { return undefined; } @@ -165,20 +183,31 @@ function readDiskCache(): Map | undefined { } function readPersistentCache(): Map | undefined { - const sqliteCache = readSqliteCache(); - if (sqliteCache) { - return sqliteCache; + return readSqliteCache(); +} + +export function legacyOpenRouterModelCapabilitiesCacheExists( + env: NodeJS.ProcessEnv = process.env, +): boolean { + return existsSync(resolveDiskCachePath(env)); +} + +export function importLegacyOpenRouterModelCapabilitiesCacheToSqlite( + env: NodeJS.ProcessEnv = process.env, +): { imported: boolean; models: number } { + if (!legacyOpenRouterModelCapabilitiesCacheExists(env)) { + return { imported: false, models: 0 }; } - const diskCache = readDiskCache(); + const diskCache = readDiskCache(env); if (diskCache) { - writeSqliteCache(diskCache); - try { - unlinkSync(resolveDiskCachePath()); - } catch { - // Best-effort legacy cache cleanup. - } + writeSqliteCache(diskCache, env); } - return diskCache; + try { + unlinkSync(resolveDiskCachePath(env)); + } catch { + // Import succeeded; a later doctor pass can remove the stale file. + } + return { imported: true, models: diskCache?.size ?? 0 }; } // --------------------------------------------------------------------------- diff --git a/src/agents/subagent-registry.persistence.test.ts b/src/agents/subagent-registry.persistence.test.ts index c820c7b15b4..202c7b215b6 100644 --- a/src/agents/subagent-registry.persistence.test.ts +++ b/src/agents/subagent-registry.persistence.test.ts @@ -30,6 +30,7 @@ import { writeSubagentSessionEntry, } from "./subagent-registry.persistence.test-support.js"; import { + importLegacySubagentRegistryFileToSqlite, loadSubagentRegistryFromDisk, resolveSubagentRegistryPath, saveSubagentRegistryToDisk, @@ -111,7 +112,7 @@ describe("subagent registry persistence", () => { const writePersistedRegistry = async ( persisted: Record, - opts?: { seedChildSessions?: boolean }, + opts?: { seedChildSessions?: boolean; importLegacy?: boolean }, ) => { tempStateDir = await fs.mkdtemp(path.join(os.tmpdir(), "openclaw-subagent-")); process.env.OPENCLAW_STATE_DIR = tempStateDir; @@ -121,6 +122,9 @@ describe("subagent registry persistence", () => { if (opts?.seedChildSessions !== false) { await seedChildSessionsForPersistedRuns(persisted); } + if (opts?.importLegacy !== false) { + importLegacySubagentRegistryFileToSqlite(process.env); + } return registryPath; }; @@ -268,6 +272,7 @@ describe("subagent registry persistence", () => { }; await fs.mkdir(path.dirname(registryPath), { recursive: true }); await fs.writeFile(registryPath, `${JSON.stringify(persisted)}\n`, "utf8"); + importLegacySubagentRegistryFileToSqlite(process.env); await writeChildSessionEntry({ sessionKey: "agent:main:subagent:two", sessionId: "sess-two", @@ -350,6 +355,60 @@ describe("subagent registry persistence", () => { }); }); + it("merges legacy registry imports into existing SQLite runs", async () => { + tempStateDir = await fs.mkdtemp(path.join(os.tmpdir(), "openclaw-subagent-")); + process.env.OPENCLAW_STATE_DIR = tempStateDir; + const registryPath = path.join(tempStateDir, "subagents", "runs.json"); + const existing: SubagentRunRecord = { + runId: "run-existing", + childSessionKey: "agent:main:subagent:existing", + requesterSessionKey: "agent:main:main", + controllerSessionKey: "agent:main:main", + requesterDisplayKey: "main", + task: "existing sqlite run", + cleanup: "keep", + createdAt: 1, + startedAt: 2, + spawnMode: "run", + }; + saveSubagentRegistryToDisk(new Map([[existing.runId, existing]])); + await fs.mkdir(path.dirname(registryPath), { recursive: true }); + await fs.writeFile( + registryPath, + `${JSON.stringify({ + version: 2, + runs: { + "run-imported": { + runId: "run-imported", + childSessionKey: "agent:main:subagent:imported", + requesterSessionKey: "agent:main:main", + requesterDisplayKey: "main", + task: "imported legacy run", + cleanup: "keep", + createdAt: 3, + startedAt: 4, + spawnMode: "run", + }, + }, + })}\n`, + "utf8", + ); + + expect(importLegacySubagentRegistryFileToSqlite(process.env)).toEqual({ + imported: true, + runs: 1, + }); + + const restored = loadSubagentRegistryFromDisk(); + expect(restored.get("run-existing")).toMatchObject({ + childSessionKey: "agent:main:subagent:existing", + }); + expect(restored.get("run-imported")).toMatchObject({ + childSessionKey: "agent:main:subagent:imported", + }); + await expect(fs.access(registryPath)).rejects.toMatchObject({ code: "ENOENT" }); + }); + it("returns isolated clones for unchanged persisted registry snapshots", async () => { const registryPath = await writePersistedRegistry( { @@ -742,6 +801,7 @@ describe("subagent registry persistence", () => { const registryPath = path.join(tempStateDir, "subagents", "runs.json"); await fs.mkdir(path.dirname(registryPath), { recursive: true }); await fs.writeFile(registryPath, `${JSON.stringify(persisted)}\n`, "utf8"); + importLegacySubagentRegistryFileToSqlite(process.env); restartRegistry(); await waitForRegistryWork(async () => { diff --git a/src/agents/subagent-registry.store.ts b/src/agents/subagent-registry.store.ts index 0a31659c7eb..71333cb7256 100644 --- a/src/agents/subagent-registry.store.ts +++ b/src/agents/subagent-registry.store.ts @@ -25,16 +25,10 @@ type PersistedSubagentRegistryV2 = { type PersistedSubagentRegistry = PersistedSubagentRegistryV1 | PersistedSubagentRegistryV2; -const MAX_SUBAGENT_REGISTRY_READ_CACHE_ENTRIES = 32; const SUBAGENT_REGISTRY_KV_SCOPE = "subagent_runs"; type PersistedSubagentRunRecord = SubagentRunRecord; -type RegistryCacheEntry = { - signature: string; - runs: Map; -}; - type LegacySubagentRunRecord = PersistedSubagentRunRecord & { announceCompletedAt?: unknown; announceHandled?: unknown; @@ -42,32 +36,6 @@ type LegacySubagentRunRecord = PersistedSubagentRunRecord & { requesterAccountId?: unknown; }; -const registryReadCache = new Map(); - -function cloneSubagentRunRecord(entry: SubagentRunRecord): SubagentRunRecord { - return structuredClone(entry); -} - -function cloneSubagentRunMap(runs: Map): Map { - return new Map([...runs].map(([runId, entry]) => [runId, cloneSubagentRunRecord(entry)])); -} - -function setCachedRegistryRead( - pathname: string, - signature: string, - runs: Map, -): void { - registryReadCache.delete(pathname); - registryReadCache.set(pathname, { signature, runs: cloneSubagentRunMap(runs) }); - if (registryReadCache.size <= MAX_SUBAGENT_REGISTRY_READ_CACHE_ENTRIES) { - return; - } - const oldestKey = registryReadCache.keys().next().value; - if (typeof oldestKey === "string") { - registryReadCache.delete(oldestKey); - } -} - function resolveSubagentStateDir(env: NodeJS.ProcessEnv = process.env): string { const explicit = env.OPENCLAW_STATE_DIR?.trim(); if (explicit) { @@ -83,11 +51,17 @@ export function resolveSubagentRegistryPath(): string { return path.join(resolveSubagentStateDir(process.env), "subagents", "runs.json"); } -function subagentRegistryDbOptions(): OpenClawStateDatabaseOptions { +function resolveSubagentRegistryPathForEnv(env: NodeJS.ProcessEnv = process.env): string { + return path.join(resolveSubagentStateDir(env), "subagents", "runs.json"); +} + +function subagentRegistryDbOptions( + env: NodeJS.ProcessEnv = process.env, +): OpenClawStateDatabaseOptions { return { env: { - ...process.env, - OPENCLAW_STATE_DIR: resolveSubagentStateDir(process.env), + ...env, + OPENCLAW_STATE_DIR: resolveSubagentStateDir(env), }, }; } @@ -151,10 +125,12 @@ function normalizePersistedRunRecords(params: { return out; } -function loadSubagentRegistryFromSqlite(): Map | null { +function loadSubagentRegistryFromSqlite( + env: NodeJS.ProcessEnv = process.env, +): Map | null { const entries = listOpenClawStateKvJson( SUBAGENT_REGISTRY_KV_SCOPE, - subagentRegistryDbOptions(), + subagentRegistryDbOptions(env), ); if (entries.length === 0) { return null; @@ -167,88 +143,77 @@ function loadSubagentRegistryFromSqlite(): Map | null } export function loadSubagentRegistryFromDisk(): Map { - const sqliteRuns = loadSubagentRegistryFromSqlite(); - if (sqliteRuns) { - return sqliteRuns; - } - const pathname = resolveSubagentRegistryPath(); - const signature = statRegistryFileSignature(pathname); - if (signature === null) { - registryReadCache.delete(pathname); - return new Map(); - } - const cached = registryReadCache.get(pathname); - if (cached?.signature === signature) { - registryReadCache.delete(pathname); - registryReadCache.set(pathname, cached); - return cloneSubagentRunMap(cached.runs); + return loadSubagentRegistryFromSqlite() ?? new Map(); +} + +function writeSubagentRegistryRunsToSqlite( + runs: Map, + env: NodeJS.ProcessEnv = process.env, +): void { + const dbOptions = subagentRegistryDbOptions(env); + for (const [runId, entry] of runs.entries()) { + writeOpenClawStateKvJson(SUBAGENT_REGISTRY_KV_SCOPE, runId, entry, dbOptions); } +} + +function loadLegacySubagentRegistryFile(pathname: string): Map { const raw = loadJsonFile(pathname); if (!raw || typeof raw !== "object") { - setCachedRegistryRead(pathname, signature, new Map()); return new Map(); } const record = raw as Partial; if (record.version !== 1 && record.version !== 2) { - setCachedRegistryRead(pathname, signature, new Map()); return new Map(); } const runsRaw = record.runs; if (!runsRaw || typeof runsRaw !== "object") { - setCachedRegistryRead(pathname, signature, new Map()); return new Map(); } - const out = normalizePersistedRunRecords({ + return normalizePersistedRunRecords({ runsRaw: runsRaw as Record, isLegacy: record.version === 1, }); - try { - saveSubagentRegistryToDisk(out); - } catch { - setCachedRegistryRead(pathname, signature, out); - } - return out; } -export function saveSubagentRegistryToDisk(runs: Map) { - const pathname = resolveSubagentRegistryPath(); - const serialized: Record = {}; - for (const [runId, entry] of runs.entries()) { - serialized[runId] = entry; - } - const existing = listOpenClawStateKvJson( - SUBAGENT_REGISTRY_KV_SCOPE, - subagentRegistryDbOptions(), - ); - for (const entry of existing) { - if (!runs.has(entry.key)) { - deleteOpenClawStateKvJson(SUBAGENT_REGISTRY_KV_SCOPE, entry.key, subagentRegistryDbOptions()); - } - } - for (const [runId, entry] of runs.entries()) { - writeOpenClawStateKvJson(SUBAGENT_REGISTRY_KV_SCOPE, runId, entry, subagentRegistryDbOptions()); - } +export function legacySubagentRegistryFileExists(env: NodeJS.ProcessEnv = process.env): boolean { try { - fs.unlinkSync(pathname); + return fs.statSync(resolveSubagentRegistryPathForEnv(env)).isFile(); } catch (error) { - if ((error as NodeJS.ErrnoException)?.code !== "ENOENT") { - throw error; - } - } - registryReadCache.delete(pathname); -} - -function statRegistryFileSignature(pathname: string): string | null { - try { - const stat = fs.statSync(pathname, { bigint: true }); - if (!stat.isFile()) { - return null; - } - return `${stat.dev}:${stat.ino}:${stat.size}:${stat.mtimeNs}:${stat.ctimeNs}`; - } catch (error) { - if ((error as NodeJS.ErrnoException).code === "ENOENT") { - return null; + if ((error as NodeJS.ErrnoException)?.code === "ENOENT") { + return false; } throw error; } } + +export function importLegacySubagentRegistryFileToSqlite(env: NodeJS.ProcessEnv = process.env): { + imported: boolean; + runs: number; +} { + const pathname = resolveSubagentRegistryPathForEnv(env); + if (!legacySubagentRegistryFileExists(env)) { + return { imported: false, runs: 0 }; + } + const runs = loadLegacySubagentRegistryFile(pathname); + writeSubagentRegistryRunsToSqlite(runs, env); + try { + fs.unlinkSync(pathname); + } catch { + // Import succeeded; a later doctor pass can remove the stale file. + } + return { imported: true, runs: runs.size }; +} + +export function saveSubagentRegistryToDisk(runs: Map) { + const dbOptions = subagentRegistryDbOptions(); + const existing = listOpenClawStateKvJson( + SUBAGENT_REGISTRY_KV_SCOPE, + dbOptions, + ); + for (const entry of existing) { + if (!runs.has(entry.key)) { + deleteOpenClawStateKvJson(SUBAGENT_REGISTRY_KV_SCOPE, entry.key, dbOptions); + } + } + writeSubagentRegistryRunsToSqlite(runs); +} diff --git a/src/commands/doctor-sqlite-state.test.ts b/src/commands/doctor-sqlite-state.test.ts index 369d17a26bd..d92bb33fe63 100644 --- a/src/commands/doctor-sqlite-state.test.ts +++ b/src/commands/doctor-sqlite-state.test.ts @@ -8,6 +8,7 @@ import { listDevicePairing } from "../infra/device-pairing.js"; import { loadApnsRegistration } from "../infra/push-apns.js"; import { listWebPushSubscriptions } from "../infra/push-web.js"; import { listChannelPairingRequests, readChannelAllowFromStore } from "../pairing/pairing-store.js"; +import { readOpenClawStateKvJson } from "../state/openclaw-state-kv.js"; import { withEnvAsync } from "../test-utils/env.js"; import { withTempDir } from "../test-utils/temp-dir.js"; @@ -173,6 +174,97 @@ describe("maybeRepairLegacyRuntimeStateFiles", () => { })}\n`, "utf8", ); + await fs.writeFile( + path.join(stateDir, "update-check.json"), + `${JSON.stringify({ + lastCheckedAt: "2026-01-17T10:00:00.000Z", + lastAvailableVersion: "2.0.0", + lastAvailableTag: "latest", + })}\n`, + "utf8", + ); + const mediaRecordsDir = path.join(stateDir, "media", "outgoing", "records"); + await fs.mkdir(mediaRecordsDir, { recursive: true }); + await fs.writeFile( + path.join(mediaRecordsDir, "11111111-1111-4111-8111-111111111111.json"), + `${JSON.stringify({ + attachmentId: "11111111-1111-4111-8111-111111111111", + sessionKey: "agent:main:main", + messageId: "msg-1", + createdAt: "2026-01-17T10:00:00.000Z", + alt: "legacy image", + original: { + path: "/tmp/legacy-image.png", + contentType: "image/png", + width: 1, + height: 1, + sizeBytes: 1, + filename: "legacy-image.png", + }, + })}\n`, + "utf8", + ); + await fs.mkdir(path.join(stateDir, "subagents"), { recursive: true }); + await fs.writeFile( + path.join(stateDir, "subagents", "runs.json"), + `${JSON.stringify({ + version: 2, + runs: { + "run-legacy": { + runId: "run-legacy", + childSessionKey: "agent:main:subagent:legacy", + requesterSessionKey: "agent:main:main", + requesterDisplayKey: "main", + task: "legacy task", + cleanup: "keep", + createdAt: 1, + startedAt: 2, + spawnMode: "run", + }, + }, + })}\n`, + "utf8", + ); + await fs.mkdir(path.join(stateDir, "tui"), { recursive: true }); + await fs.writeFile( + path.join(stateDir, "tui", "last-session.json"), + `${JSON.stringify({ + "legacy-tui-scope": { + sessionKey: "agent:main:tui-legacy", + updatedAt: 1000, + }, + })}\n`, + "utf8", + ); + const agentDir = path.join(stateDir, "agents", "main", "agent"); + await fs.mkdir(agentDir, { recursive: true }); + const authStatePath = path.join(agentDir, "auth-state.json"); + await fs.writeFile( + authStatePath, + `${JSON.stringify({ + version: 1, + order: { openai: ["openai:default"] }, + lastGood: { openai: "openai:default" }, + })}\n`, + "utf8", + ); + await fs.mkdir(path.join(stateDir, "cache"), { recursive: true }); + await fs.writeFile( + path.join(stateDir, "cache", "openrouter-models.json"), + `${JSON.stringify({ + models: { + "acme/legacy": { + name: "Legacy OpenRouter", + input: ["text"], + reasoning: false, + contextWindow: 123, + maxTokens: 456, + cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0 }, + }, + }, + })}\n`, + "utf8", + ); await maybeRepairLegacyRuntimeStateFiles({ prompter: { shouldRepair: true }, @@ -219,6 +311,60 @@ describe("maybeRepairLegacyRuntimeStateFiles", () => { await expect(loadApnsRegistration("ios-node", stateDir)).resolves.toMatchObject({ nodeId: "ios-node", }); + expect(readOpenClawStateKvJson("runtime.update-check", "state", { env })).toMatchObject({ + lastAvailableVersion: "2.0.0", + lastAvailableTag: "latest", + }); + await expect(fs.stat(path.join(stateDir, "update-check.json"))).rejects.toMatchObject({ + code: "ENOENT", + }); + expect( + readOpenClawStateKvJson( + "managed_outgoing_image_records", + "11111111-1111-4111-8111-111111111111", + { env }, + ), + ).toMatchObject({ + sessionKey: "agent:main:main", + alt: "legacy image", + }); + await expect( + fs.stat(path.join(mediaRecordsDir, "11111111-1111-4111-8111-111111111111.json")), + ).rejects.toMatchObject({ code: "ENOENT" }); + expect(readOpenClawStateKvJson("subagent_runs", "run-legacy", { env })).toMatchObject({ + childSessionKey: "agent:main:subagent:legacy", + }); + await expect(fs.stat(path.join(stateDir, "subagents", "runs.json"))).rejects.toMatchObject({ + code: "ENOENT", + }); + expect(readOpenClawStateKvJson("tui:last-session", "legacy-tui-scope", { env })).toEqual({ + sessionKey: "agent:main:tui-legacy", + updatedAt: 1000, + }); + await expect( + fs.stat(path.join(stateDir, "tui", "last-session.json")), + ).rejects.toMatchObject({ code: "ENOENT" }); + expect(readOpenClawStateKvJson("auth-profile-state", authStatePath, { env })).toMatchObject( + { + order: { openai: ["openai:default"] }, + lastGood: { openai: "openai:default" }, + }, + ); + await expect(fs.stat(authStatePath)).rejects.toMatchObject({ code: "ENOENT" }); + expect( + readOpenClawStateKvJson("openrouter_model_capabilities", "models", { env }), + ).toMatchObject({ + models: { + "acme/legacy": { + name: "Legacy OpenRouter", + contextWindow: 123, + maxTokens: 456, + }, + }, + }); + await expect( + fs.stat(path.join(stateDir, "cache", "openrouter-models.json")), + ).rejects.toMatchObject({ code: "ENOENT" }); }); }); }); diff --git a/src/commands/doctor-sqlite-state.ts b/src/commands/doctor-sqlite-state.ts index 5ceca6f06a0..23b2e16df02 100644 --- a/src/commands/doctor-sqlite-state.ts +++ b/src/commands/doctor-sqlite-state.ts @@ -1,8 +1,24 @@ +import { + discoverLegacyAuthProfileStateAgentDirs, + importLegacyAuthProfileStateFileToSqlite, +} from "../agents/auth-profiles/state.js"; +import { + importLegacyOpenRouterModelCapabilitiesCacheToSqlite, + legacyOpenRouterModelCapabilitiesCacheExists, +} from "../agents/pi-embedded-runner/openrouter-model-capabilities.js"; +import { + importLegacySubagentRegistryFileToSqlite, + legacySubagentRegistryFileExists, +} from "../agents/subagent-registry.store.js"; import { importLegacyCommitmentStoreFileToSqlite, legacyCommitmentStoreFileExists, } from "../commitments/store.js"; import { resolveStateDir } from "../config/paths.js"; +import { + importLegacyManagedOutgoingImageRecordFilesToSqlite, + legacyManagedOutgoingImageRecordFilesExist, +} from "../gateway/managed-image-attachments.js"; import { importLegacyDeviceAuthFileToSqlite, legacyDeviceAuthFileExists, @@ -26,11 +42,19 @@ import { legacyApnsRegistrationFileExists, } from "../infra/push-apns.js"; import { importLegacyWebPushFilesToSqlite, legacyWebPushFilesExist } from "../infra/push-web.js"; +import { + importLegacyUpdateCheckFileToSqlite, + legacyUpdateCheckFileExists, +} from "../infra/update-startup.js"; import { importLegacyChannelPairingFilesToSqlite, legacyChannelPairingFilesExist, } from "../pairing/pairing-store.js"; import { note } from "../terminal/note.js"; +import { + importLegacyTuiLastSessionStoreToSqlite, + legacyTuiLastSessionFileExists, +} from "../tui/tui-last-session.js"; import type { DoctorPrompter } from "./doctor-prompter.js"; type LegacyStateProbe = { @@ -43,6 +67,12 @@ type LegacyStateProbe = { commitments: boolean; webPush: boolean; apns: boolean; + updateCheck: boolean; + managedImages: boolean; + subagents: boolean; + tuiLastSession: boolean; + authProfileStateAgentDirs: string[]; + openRouterModelCache: boolean; }; async function probeLegacyRuntimeStateFiles(env: NodeJS.ProcessEnv): Promise { @@ -57,11 +87,17 @@ async function probeLegacyRuntimeStateFiles(env: NodeJS.ProcessEnv): Promise (Array.isArray(value) ? value.length > 0 : value)); } export async function maybeRepairLegacyRuntimeStateFiles(params: { @@ -76,7 +112,7 @@ export async function maybeRepairLegacyRuntimeStateFiles(params: { } if (!params.prompter.shouldRepair) { note( - "Legacy runtime JSON state files detected. Run `openclaw doctor --fix` to import commitments, device, bootstrap, channel pairing, node pairing, and push state into SQLite.", + "Legacy runtime JSON state files detected. Run `openclaw doctor --fix` to import commitments, device, bootstrap, channel pairing, node pairing, push, media, subagent, TUI, auth routing, OpenRouter cache, and update-check state into SQLite.", "SQLite state", ); return; @@ -178,6 +214,62 @@ export async function maybeRepairLegacyRuntimeStateFiles(params: { } }); } + if (probe.updateCheck) { + await runImport("Update check", async () => { + const result = await importLegacyUpdateCheckFileToSqlite(env); + if (result.imported) { + changes.push("- Imported update-check state into SQLite."); + } + }); + } + if (probe.managedImages) { + await runImport("Managed outgoing image records", async () => { + const result = await importLegacyManagedOutgoingImageRecordFilesToSqlite(baseDir); + if (result.files > 0) { + changes.push(`- Imported ${result.records} managed outgoing image record(s) into SQLite.`); + } + }); + } + if (probe.subagents) { + await runImport("Subagent registry", () => { + const result = importLegacySubagentRegistryFileToSqlite(env); + if (result.imported) { + changes.push(`- Imported ${result.runs} subagent run record(s) into SQLite.`); + } + }); + } + if (probe.tuiLastSession) { + await runImport("TUI last-session", async () => { + const result = await importLegacyTuiLastSessionStoreToSqlite({ stateDir: baseDir }); + if (result.imported) { + changes.push(`- Imported ${result.pointers} TUI last-session pointer(s) into SQLite.`); + } + }); + } + if (probe.authProfileStateAgentDirs.length > 0) { + await runImport("Auth profile runtime state", () => { + let imported = 0; + for (const agentDir of probe.authProfileStateAgentDirs) { + const result = importLegacyAuthProfileStateFileToSqlite(agentDir); + if (result.imported) { + imported += 1; + } + } + if (imported > 0) { + changes.push(`- Imported ${imported} auth profile runtime state file(s) into SQLite.`); + } + }); + } + if (probe.openRouterModelCache) { + await runImport("OpenRouter model cache", () => { + const result = importLegacyOpenRouterModelCapabilitiesCacheToSqlite(env); + if (result.imported) { + changes.push( + `- Imported ${result.models} OpenRouter model cache entr${result.models === 1 ? "y" : "ies"} into SQLite.`, + ); + } + }); + } if (changes.length > 0) { note(changes.join("\n"), "Doctor changes"); diff --git a/src/gateway/managed-image-attachments.test.ts b/src/gateway/managed-image-attachments.test.ts index b22b8336f58..c9b89736f04 100644 --- a/src/gateway/managed-image-attachments.test.ts +++ b/src/gateway/managed-image-attachments.test.ts @@ -7,7 +7,7 @@ import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; import { createPinnedLookup } from "../infra/net/ssrf.js"; import { setMediaStoreNetworkDepsForTest } from "../media/store.js"; import { closeOpenClawStateDatabaseForTest } from "../state/openclaw-state-db.js"; -import { readOpenClawStateKvJson } from "../state/openclaw-state-kv.js"; +import { readOpenClawStateKvJson, writeOpenClawStateKvJson } from "../state/openclaw-state-kv.js"; const authorizeGatewayHttpRequestOrReplyMock = vi.fn(); const resolveOpenAiCompatibleHttpOperatorScopesMock = vi.fn(); @@ -102,13 +102,9 @@ async function createFixture( filename: "cat.png", }, }; - const recordsDir = path.join(stateDir, "media", "outgoing", "records"); - await fs.mkdir(recordsDir, { recursive: true }); - await fs.writeFile( - path.join(recordsDir, `${attachmentId}.json`), - JSON.stringify(record, null, 2), - "utf-8", - ); + writeOpenClawStateKvJson("managed_outgoing_image_records", attachmentId, record, { + env: { ...process.env, OPENCLAW_STATE_DIR: stateDir }, + }); return { attachmentId, sessionKey, originalPath }; } diff --git a/src/gateway/managed-image-attachments.ts b/src/gateway/managed-image-attachments.ts index f2948e19e6c..07fc6e9c2f6 100644 --- a/src/gateway/managed-image-attachments.ts +++ b/src/gateway/managed-image-attachments.ts @@ -96,11 +96,6 @@ type CleanupManagedOutgoingImageRecordsResult = { retainedCount: number; }; -type ListedManagedImageRecords = { - records: ManagedImageRecord[]; - invalidLegacyRecordPaths: string[]; -}; - type SessionManagedOutgoingAttachmentIndex = Set; type SessionManagedOutgoingAttachmentIndexCacheEntry = { @@ -283,10 +278,6 @@ function resolveOutgoingOriginalsDir(stateDir = resolveStateDir()) { return path.join(stateDir, "media", "outgoing", "originals"); } -function resolveOutgoingRecordPath(attachmentId: string, stateDir = resolveStateDir()) { - return path.join(resolveOutgoingRecordsDir(stateDir), `${attachmentId}.json`); -} - function managedImageRecordDbOptions(stateDir: string): OpenClawStateDatabaseOptions { return { env: { ...process.env, OPENCLAW_STATE_DIR: stateDir } }; } @@ -405,8 +396,6 @@ async function writeManagedImageRecord(record: ManagedImageRecord, stateDir = re record, managedImageRecordDbOptions(stateDir), ); - const recordPath = resolveOutgoingRecordPath(record.attachmentId, stateDir); - await fs.rm(recordPath, { force: true }); } async function deleteManagedImageRecordArtifacts( @@ -426,11 +415,6 @@ async function deleteManagedImageRecordArtifacts( // Ignore cleanup races or already-missing files. } } - try { - await fs.rm(resolveOutgoingRecordPath(record.attachmentId, stateDir), { force: true }); - } catch { - // Ignore cleanup races or already-missing records. - } deleteOpenClawStateKvJson( MANAGED_OUTGOING_IMAGE_RECORD_SCOPE, record.attachmentId, @@ -475,7 +459,7 @@ async function deleteOrphanManagedImageFiles(params: { return deletedFileCount; } -async function listManagedImageRecords(stateDir: string): Promise { +async function listManagedImageRecords(stateDir: string): Promise { const recordsById = new Map(); for (const entry of listOpenClawStateKvJson( MANAGED_OUTGOING_IMAGE_RECORD_SCOPE, @@ -483,8 +467,10 @@ async function listManagedImageRecords(stateDir: string): Promise { const recordsDir = resolveOutgoingRecordsDir(stateDir); let names: string[] = []; try { @@ -492,32 +478,41 @@ async function listManagedImageRecords(stateDir: string): Promise { + return (await listLegacyManagedImageRecordPaths(stateDir)).length > 0; +} + +export async function importLegacyManagedOutgoingImageRecordFilesToSqlite( + stateDir = resolveStateDir(), +): Promise<{ files: number; records: number }> { + const recordPaths = await listLegacyManagedImageRecordPaths(stateDir); + let records = 0; + for (const recordPath of recordPaths) { const record = await tryReadJson(recordPath); - if (!record?.attachmentId) { - invalidLegacyRecordPaths.push(recordPath); - continue; - } - if (!recordsById.has(record.attachmentId)) { - recordsById.set(record.attachmentId, record); + if (record?.attachmentId) { writeOpenClawStateKvJson( MANAGED_OUTGOING_IMAGE_RECORD_SCOPE, record.attachmentId, record, managedImageRecordDbOptions(stateDir), ); + records += 1; } await fs.rm(recordPath, { force: true }).catch(() => {}); } - - return { - records: [...recordsById.values()], - invalidLegacyRecordPaths, - }; + return { files: recordPaths.length, records }; } export async function cleanupManagedOutgoingImageRecords(params?: { @@ -534,18 +529,15 @@ export async function cleanupManagedOutgoingImageRecords(params?: { const forceDeleteSessionRecords = params?.forceDeleteSessionRecords === true; const listedRecords = await listManagedImageRecords(stateDir); - let deletedRecordCount = listedRecords.invalidLegacyRecordPaths.length; + let deletedRecordCount = 0; let deletedFileCount = 0; let retainedCount = 0; - for (const recordPath of listedRecords.invalidLegacyRecordPaths) { - await fs.rm(recordPath, { force: true }).catch(() => {}); - } const retainedReferencedPaths = new Set(); const transcriptAttachmentIndexCache = new Map< string, SessionManagedOutgoingAttachmentIndex | null >(); - for (const record of listedRecords.records) { + for (const record of listedRecords) { if (sessionKeyFilter && record.sessionKey !== sessionKeyFilter) { if (record.original?.path) { retainedReferencedPaths.add(record.original.path); @@ -601,22 +593,7 @@ async function readManagedImageRecord( if (sqliteRecord) { return sqliteRecord; } - try { - const raw = await fs.readFile(resolveOutgoingRecordPath(attachmentId, stateDir), "utf-8"); - const record = JSON.parse(raw) as ManagedImageRecord; - if (record?.attachmentId) { - writeOpenClawStateKvJson( - MANAGED_OUTGOING_IMAGE_RECORD_SCOPE, - record.attachmentId, - record, - managedImageRecordDbOptions(stateDir), - ); - await fs.rm(resolveOutgoingRecordPath(attachmentId, stateDir), { force: true }); - } - return record; - } catch { - return null; - } + return null; } function buildManagedImageBlock(record: ManagedImageRecord): ManagedImageBlock { diff --git a/src/infra/update-startup.test.ts b/src/infra/update-startup.test.ts index 90f991a0c5f..bfbed6bbb3a 100644 --- a/src/infra/update-startup.test.ts +++ b/src/infra/update-startup.test.ts @@ -1,6 +1,5 @@ -import fs from "node:fs/promises"; -import path from "node:path"; import { afterAll, afterEach, beforeAll, beforeEach, describe, expect, it, vi } from "vitest"; +import { readOpenClawStateKvJson, writeOpenClawStateKvJson } from "../state/openclaw-state-kv.js"; import { createSuiteTempRootTracker } from "../test-helpers/temp-dir.js"; import { captureEnv } from "../test-utils/env.js"; import type { UpdateCheckResult } from "./update-check.js"; @@ -141,8 +140,9 @@ describe("update-startup", () => { allowInTests: true, }); - const statePath = path.join(tempDir, "update-check.json"); - const parsed = JSON.parse(await fs.readFile(statePath, "utf-8")) as { + const parsed = readOpenClawStateKvJson("runtime.update-check", "state", { + env: process.env, + }) as { lastNotifiedVersion?: string; lastNotifiedTag?: string; lastAvailableVersion?: string; @@ -221,19 +221,15 @@ describe("update-startup", () => { }); it("hydrates cached update from persisted state during throttle window", async () => { - const statePath = path.join(tempDir, "update-check.json"); - await fs.writeFile( - statePath, - JSON.stringify( - { - lastCheckedAt: new Date(Date.now()).toISOString(), - lastAvailableVersion: "2.0.0", - lastAvailableTag: "latest", - }, - null, - 2, - ), - "utf-8", + writeOpenClawStateKvJson( + "runtime.update-check", + "state", + { + lastCheckedAt: new Date(Date.now()).toISOString(), + lastAvailableVersion: "2.0.0", + lastAvailableTag: "latest", + }, + { env: process.env }, ); const onUpdateAvailableChange = vi.fn(); @@ -295,7 +291,11 @@ describe("update-startup", () => { }); expect(log.info).not.toHaveBeenCalled(); - await expect(fs.stat(path.join(tempDir, "update-check.json"))).rejects.toThrow(); + expect( + readOpenClawStateKvJson("runtime.update-check", "state", { + env: process.env, + }), + ).toBeUndefined(); }); it("defers stable auto-update until rollout window is due", async () => { diff --git a/src/infra/update-startup.ts b/src/infra/update-startup.ts index d808dc5e9fd..3eada234207 100644 --- a/src/infra/update-startup.ts +++ b/src/infra/update-startup.ts @@ -6,9 +6,14 @@ import { resolveStateDir } from "../config/paths.js"; import type { OpenClawConfig } from "../config/types.openclaw.js"; import { runCommandWithTimeout } from "../process/exec.js"; import { normalizeLowercaseStringOrEmpty } from "../shared/string-coerce.js"; +import type { OpenClawStateDatabaseOptions } from "../state/openclaw-state-db.js"; +import { + readOpenClawStateKvJson, + writeOpenClawStateKvJson, + type OpenClawStateJsonValue, +} from "../state/openclaw-state-kv.js"; import { VERSION } from "../version.js"; import { isTruthyEnvValue } from "./env.js"; -import { writeJson } from "./json-files.js"; import { resolveOpenClawPackageRoot } from "./openclaw-root.js"; import { normalizeUpdateChannel, DEFAULT_PACKAGE_CHANNEL } from "./update-channels.js"; import { compareSemverStrings, resolveNpmChannelTag, checkUpdateStatus } from "./update-check.js"; @@ -61,6 +66,8 @@ export function resetUpdateAvailableStateForTest(): void { } const UPDATE_CHECK_FILENAME = "update-check.json"; +const UPDATE_CHECK_SCOPE = "runtime.update-check"; +const UPDATE_CHECK_KEY = "state"; const UPDATE_CHECK_INTERVAL_MS = 24 * 60 * 60 * 1000; const ONE_HOUR_MS = 60 * 60 * 1000; const AUTO_UPDATE_COMMAND_TIMEOUT_MS = 45 * 60 * 1000; @@ -116,18 +123,71 @@ function resolveCheckIntervalMs(cfg: OpenClawConfig): number { return UPDATE_CHECK_INTERVAL_MS; } -async function readState(statePath: string): Promise { +function sqliteOptionsForEnv(env: NodeJS.ProcessEnv): OpenClawStateDatabaseOptions { + return { env }; +} + +function resolveLegacyUpdateCheckPath(env: NodeJS.ProcessEnv = process.env): string { + return path.join(resolveStateDir(env), UPDATE_CHECK_FILENAME); +} + +function coerceUpdateCheckState(value: unknown): UpdateCheckState { + return value && typeof value === "object" && !Array.isArray(value) + ? (value as UpdateCheckState) + : {}; +} + +function readState(env: NodeJS.ProcessEnv = process.env): UpdateCheckState { + return coerceUpdateCheckState( + readOpenClawStateKvJson(UPDATE_CHECK_SCOPE, UPDATE_CHECK_KEY, sqliteOptionsForEnv(env)), + ); +} + +function writeState(state: UpdateCheckState, env: NodeJS.ProcessEnv = process.env): void { + writeOpenClawStateKvJson( + UPDATE_CHECK_SCOPE, + UPDATE_CHECK_KEY, + state as unknown as OpenClawStateJsonValue, + sqliteOptionsForEnv(env), + ); +} + +async function readLegacyStateFile(filePath: string): Promise { try { - const raw = await fs.readFile(statePath, "utf-8"); - const parsed = JSON.parse(raw) as UpdateCheckState; - return parsed && typeof parsed === "object" ? parsed : {}; + const raw = await fs.readFile(filePath, "utf-8"); + return coerceUpdateCheckState(JSON.parse(raw)); } catch { return {}; } } -async function writeState(statePath: string, state: UpdateCheckState): Promise { - await writeJson(statePath, state); +export async function legacyUpdateCheckFileExists( + env: NodeJS.ProcessEnv = process.env, +): Promise { + try { + await fs.access(resolveLegacyUpdateCheckPath(env)); + return true; + } catch { + return false; + } +} + +export async function importLegacyUpdateCheckFileToSqlite( + env: NodeJS.ProcessEnv = process.env, +): Promise<{ imported: boolean }> { + const filePath = resolveLegacyUpdateCheckPath(env); + try { + await fs.access(filePath); + } catch (error) { + if ((error as { code?: unknown })?.code === "ENOENT") { + return { imported: false }; + } + throw error; + } + const state = await readLegacyStateFile(filePath); + writeState(state, env); + await fs.rm(filePath, { force: true }).catch(() => undefined); + return { imported: true }; } function sameUpdateAvailable(a: UpdateAvailable | null, b: UpdateAvailable | null): boolean { @@ -325,8 +385,7 @@ export async function runGatewayUpdateCheck(params: { return; } - const statePath = path.join(resolveStateDir(), UPDATE_CHECK_FILENAME); - const state = await readState(statePath); + const state = readState(); const now = Date.now(); const lastCheckedAt = state.lastCheckedAt ? Date.parse(state.lastCheckedAt) : null; if (shouldRunUpdateHints) { @@ -375,7 +434,7 @@ export async function runGatewayUpdateCheck(params: { next: null, onUpdateAvailableChange: params.onUpdateAvailableChange, }); - await writeState(statePath, nextState); + writeState(nextState); return; } @@ -383,7 +442,7 @@ export async function runGatewayUpdateCheck(params: { const resolved = await resolveNpmChannelTag({ channel, timeoutMs: 2500 }); const tag = resolved.tag; if (!resolved.version) { - await writeState(statePath, nextState); + writeState(nextState); return; } @@ -494,7 +553,7 @@ export async function runGatewayUpdateCheck(params: { }); } - await writeState(statePath, nextState); + writeState(nextState); } export function scheduleGatewayUpdateCheck(params: { diff --git a/src/tui/tui-last-session.test.ts b/src/tui/tui-last-session.test.ts index 236529d5c85..54fd7044923 100644 --- a/src/tui/tui-last-session.test.ts +++ b/src/tui/tui-last-session.test.ts @@ -6,6 +6,7 @@ import { closeOpenClawStateDatabaseForTest } from "../state/openclaw-state-db.js import { buildTuiLastSessionScopeKey, clearTuiLastSessionPointers, + importLegacyTuiLastSessionStoreToSqlite, isHeartbeatLikeTuiSession, readTuiLastSessionKey, resolveRememberedTuiSessionKey, @@ -69,7 +70,7 @@ describe("tui last session state", () => { ); }); - it("imports legacy JSON into SQLite on read and removes it", async () => { + it("imports legacy JSON into SQLite through the doctor migration helper", async () => { const stateDir = await makeTempStateDir(); const scopeKey = buildTuiLastSessionScopeKey({ connectionUrl: "legacy", @@ -83,16 +84,31 @@ describe("tui last session state", () => { JSON.stringify({ [scopeKey]: { sessionKey: "agent:main:legacy-json", updatedAt: 1000 } }), ); - await expect(readTuiLastSessionKey({ scopeKey, stateDir })).resolves.toBe( - "agent:main:legacy-json", - ); + await expect(readTuiLastSessionKey({ scopeKey, stateDir })).resolves.toBeNull(); + await expect(importLegacyTuiLastSessionStoreToSqlite({ stateDir })).resolves.toEqual({ + imported: true, + pointers: 1, + }); await expect(fs.access(statePath)).rejects.toMatchObject({ code: "ENOENT" }); await expect(readTuiLastSessionKey({ scopeKey, stateDir })).resolves.toBe( "agent:main:legacy-json", ); }); - it("clears stale pointers from SQLite and imported legacy JSON", async () => { + it("removes empty legacy JSON through the doctor migration helper", async () => { + const stateDir = await makeTempStateDir(); + const statePath = resolveTuiLastSessionStatePath(stateDir); + await fs.mkdir(path.dirname(statePath), { recursive: true }); + await fs.writeFile(statePath, "{}\n", "utf8"); + + await expect(importLegacyTuiLastSessionStoreToSqlite({ stateDir })).resolves.toEqual({ + imported: true, + pointers: 0, + }); + await expect(fs.access(statePath)).rejects.toMatchObject({ code: "ENOENT" }); + }); + + it("clears stale pointers from SQLite after legacy JSON import", async () => { const stateDir = await makeTempStateDir(); const staleScope = buildTuiLastSessionScopeKey({ connectionUrl: "stale", @@ -125,6 +141,7 @@ describe("tui last session state", () => { statePath, JSON.stringify({ [legacyScope]: { sessionKey: "agent:main:legacy-stale", updatedAt: 1000 } }), ); + await importLegacyTuiLastSessionStoreToSqlite({ stateDir }); await expect( clearTuiLastSessionPointers({ diff --git a/src/tui/tui-last-session.ts b/src/tui/tui-last-session.ts index a613eea49e7..4547cf78a30 100644 --- a/src/tui/tui-last-session.ts +++ b/src/tui/tui-last-session.ts @@ -86,11 +86,38 @@ function writeTuiLastSessionKv(params: { ); } -async function importLegacyTuiLastSessionStore(params: { +async function readLegacyTuiLastSessionStore(params: { stateDir?: string; }): Promise { const filePath = resolveTuiLastSessionStatePath(params.stateDir); - const store = await readStore(filePath); + return await readStore(filePath); +} + +export async function legacyTuiLastSessionFileExists( + params: { + stateDir?: string; + } = {}, +): Promise { + try { + await fs.access(resolveTuiLastSessionStatePath(params.stateDir)); + return true; + } catch { + return false; + } +} + +export async function importLegacyTuiLastSessionStoreToSqlite( + params: { + stateDir?: string; + } = {}, +): Promise<{ imported: boolean; pointers: number }> { + const filePath = resolveTuiLastSessionStatePath(params.stateDir); + const exists = await legacyTuiLastSessionFileExists(params); + if (!exists) { + return { imported: false, pointers: 0 }; + } + const store = await readLegacyTuiLastSessionStore(params); + let pointers = 0; for (const [scopeKey, value] of Object.entries(store)) { const record = normalizeLastSessionRecord(value); if (!record) { @@ -101,11 +128,10 @@ async function importLegacyTuiLastSessionStore(params: { record, stateDir: params.stateDir, }); + pointers += 1; } - if (Object.keys(store).length > 0) { - await deleteStore(filePath); - } - return store; + await deleteStore(filePath); + return { imported: true, pointers }; } function normalizeMarker(value: unknown): string { @@ -146,17 +172,7 @@ export async function readTuiLastSessionKey(params: { return kvRecord.sessionKey; } - const store = await importLegacyTuiLastSessionStore({ stateDir: params.stateDir }); - const diskRecord = normalizeLastSessionRecord(store[params.scopeKey]); - if (!diskRecord) { - return null; - } - writeTuiLastSessionKv({ - scopeKey: params.scopeKey, - record: diskRecord, - stateDir: params.stateDir, - }); - return diskRecord.sessionKey; + return null; } export async function writeTuiLastSessionKey(params: { @@ -177,7 +193,6 @@ export async function writeTuiLastSessionKey(params: { record, stateDir: params.stateDir, }); - await deleteStore(resolveTuiLastSessionStatePath(params.stateDir)); } export async function clearTuiLastSessionPointers(params: { @@ -201,14 +216,6 @@ export async function clearTuiLastSessionPointers(params: { } } - const store = await importLegacyTuiLastSessionStore({ stateDir: params.stateDir }); - for (const [key, value] of Object.entries(store)) { - const record = normalizeLastSessionRecord(value); - if (record && params.sessionKeys.has(record.sessionKey)) { - deleteOpenClawStateKvJson(TUI_LAST_SESSION_KV_SCOPE, key, kvOptions); - removedScopeKeys.add(key); - } - } return removedScopeKeys.size; }