From 208d84df1a05680dce3bf1e63adabbc580d71192 Mon Sep 17 00:00:00 2001 From: Peter Steinberger Date: Wed, 6 May 2026 17:58:25 +0100 Subject: [PATCH] refactor: make sessions json doctor-import only --- docs/channels/channel-routing.md | 12 +- docs/cli/sessions.md | 13 +- docs/refactor/piless.md | 15 +- .../session-management-compaction.md | 8 +- .../register.status-health-sessions.test.ts | 3 - .../register.status-health-sessions.ts | 3 - src/commands/doctor-state-migrations.test.ts | 29 +-- src/commands/doctor.e2e-harness.ts | 1 + src/commands/sessions.test.ts | 23 --- src/commands/sessions.ts | 57 ----- .../sessions/store-backend.sqlite.test.ts | 59 +----- src/config/sessions/store-backend.sqlite.ts | 194 +++++------------- src/infra/state-migrations.ts | 24 ++- 13 files changed, 121 insertions(+), 320 deletions(-) diff --git a/docs/channels/channel-routing.md b/docs/channels/channel-routing.md index d3d48b7c8b0..b6d3e40e060 100644 --- a/docs/channels/channel-routing.md +++ b/docs/channels/channel-routing.md @@ -133,15 +133,15 @@ Canonical session metadata lives in the shared state database: - `~/.openclaw/state/openclaw.sqlite` - JSONL transcripts live alongside the store -Legacy `sessions.json` files are imported on first open and remain the -compatibility/export shape. You can still force or override a JSON file-backed -store via `session.store` and `{agentId}` templating. +Legacy `sessions.json` indexes are imported by `openclaw doctor --fix` and +removed after SQLite has the rows. Custom offline repair stores can still use an +explicit `--store` path, but per-agent runtime metadata should go through the +shared state database. Gateway and ACP session discovery also scans disk-backed agent stores under the default `agents/` root and under templated `session.store` roots. Discovered -stores must stay inside that resolved agent root and use a regular -`sessions.json` file when JSON compatibility mode is selected. Symlinks and -out-of-root paths are ignored. +stores must stay inside that resolved agent root. Symlinks and out-of-root paths +are ignored. ## WebChat behavior diff --git a/docs/cli/sessions.md b/docs/cli/sessions.md index fcba1362d3f..1ac555194bf 100644 --- a/docs/cli/sessions.md +++ b/docs/cli/sessions.md @@ -36,7 +36,6 @@ openclaw sessions --active 120 openclaw sessions --limit 25 openclaw sessions --verbose openclaw sessions --json -openclaw sessions --export-store sessions-debug.json ``` Scope selection: @@ -46,13 +45,10 @@ Scope selection: - `--agent `: one configured agent store - `--all-agents`: aggregate all configured agent stores - `--store `: explicit store path (cannot be combined with `--agent` or `--all-agents`) -- `--export-store `: export the raw resolved store JSON to a file Canonical per-agent session stores use OpenClaw's shared SQLite state database by -default. Custom `--store` paths stay file-backed JSON unless -`OPENCLAW_SESSION_STORE_BACKEND=sqlite` is set. Set -`OPENCLAW_SESSION_STORE_BACKEND=json` to force the legacy `sessions.json` -backend for repair or bisect work. +default. Existing `sessions.json` indexes are imported by `openclaw doctor +--fix`, then removed after SQLite has the rows. - `--limit `: max rows to output (default `100`; `all` restores full output) @@ -97,11 +93,6 @@ JSON examples: } ``` -Use `openclaw sessions --export-store ` to write the raw resolved session -store to JSON for debugging, backup, or SQLite compatibility checks. The command -reads through the active backend, so SQLite-backed stores export the same shape -older `sessions.json` tools expect. - ## Cleanup maintenance Run maintenance now (instead of waiting for the next write cycle): diff --git a/docs/refactor/piless.md b/docs/refactor/piless.md index 143bc564eb8..c569461fd7a 100644 --- a/docs/refactor/piless.md +++ b/docs/refactor/piless.md @@ -46,10 +46,9 @@ This plan has started landing in slices: The shared `kv` table now has a small typed helper for scoped JSON-compatible values so low-risk JSON sidecars can move behind the same SQLite connection without each feature reimplementing read/write/delete glue. -- Canonical per-agent session stores use SQLite by default. Legacy - `sessions.json` imports once, custom `--store` paths remain JSON by default, - and `openclaw sessions --export-store ` exports the compatibility JSON - shape. +- Canonical per-agent session stores use SQLite by default. `openclaw doctor +--fix` imports legacy `sessions.json` indexes into SQLite and removes the JSON + index after import, instead of keeping a parallel compatibility/export store. - Transcript events have a SQLite store primitive with JSONL import/export. Transcript append paths dual-write when the caller already has agent and session scope, including gateway-injected assistant messages. Scoped appends @@ -344,7 +343,8 @@ Migration order: 2. Add shared SQLite connection and migration helpers. 3. Move `sessions.json` behind a `SessionStoreBackend` interface. 4. Make SQLite primary for session entries. -5. Import old `sessions.json` on first open and keep an export/debug command. +5. Import old `sessions.json` from `openclaw doctor --fix`, then remove the JSON + index after SQLite has the rows. 6. Leave `*.jsonl` transcripts on disk while PI owns transcript semantics. 7. After session manager ownership moves behind OpenClaw APIs, store transcript events in SQLite and export JSONL for compatibility. @@ -543,7 +543,8 @@ Phase 0: inventory and contracts Phase 1: SQLite session index - Add shared state DB helper. -- Keep `sessions.json` compatibility. +- Add a doctor migration that imports `sessions.json` into SQLite and removes + the JSON index. - Move session entries to SQLite behind a flag. - Prove current session list, patch, reset, cleanup, and UI flows. @@ -601,7 +602,7 @@ This refactor is successful when: - Core code no longer imports PI packages outside the runtime adapter. - Repeated JSON, transcript, PI adapter, and scratch filesystem logic has one owner each. -- `sessions.json` is compatibility-only, not the primary Gateway store. +- `sessions.json` is a doctor-migrated legacy input, not a compatibility store. - Scratch state and tool artifacts can live in SQLite-backed VFS. - Agents can run in disk, VFS scratch, and VFS-only filesystem modes. - Real workspace writes still use capability-safe host filesystem operations. diff --git a/docs/reference/session-management-compaction.md b/docs/reference/session-management-compaction.md index c6e020fd257..9089fb2f20a 100644 --- a/docs/reference/session-management-compaction.md +++ b/docs/reference/session-management-compaction.md @@ -74,7 +74,9 @@ cached by file path plus `mtimeMs`/`size` and shared across concurrent readers. Per agent, on the Gateway host: -- Store: `~/.openclaw/state/openclaw.sqlite` by default; legacy/custom JSON stores use `~/.openclaw/agents//sessions/sessions.json` +- Store: `~/.openclaw/state/openclaw.sqlite` by default. `openclaw doctor --fix` + imports legacy `~/.openclaw/agents//sessions/sessions.json` indexes + into SQLite and removes the JSON index after import. - Transcripts: `~/.openclaw/agents//sessions/.jsonl` - Telegram topic sessions: `.../-topic-.jsonl` @@ -93,7 +95,7 @@ Session persistence has automatic maintenance controls (`session.maintenance`) f - `maxDiskBytes`: optional sessions-directory budget - `highWaterBytes`: optional target after cleanup (default `80%` of `maxDiskBytes`) -Normal Gateway writes flow through a per-store session writer that serializes in-process mutations. SQLite is the canonical per-agent backend; the JSON writer remains for legacy/custom stores and offline compatibility. Runtime code should prefer `updateSessionStore(...)` or `updateSessionStoreEntry(...)`; direct whole-store saves are compatibility and offline-maintenance tools. When a Gateway is reachable, non-dry-run `openclaw sessions cleanup` and `openclaw agents delete` delegate store mutations to the Gateway so cleanup joins the same writer queue; `--store ` is the explicit offline repair path for direct JSON file maintenance. `maxEntries` cleanup is still batched for production-sized caps, so a store may briefly exceed the configured cap before the next high-water cleanup rewrites it back down. Session store reads do not prune or cap entries during Gateway startup; use writes or `openclaw sessions cleanup --enforce` for cleanup. `openclaw sessions cleanup --enforce` still applies the configured cap immediately and prunes old unreferenced transcript, checkpoint, and trajectory artifacts even when no disk budget is configured. +Normal Gateway writes flow through a per-store session writer that serializes in-process mutations. SQLite is the canonical per-agent backend; `sessions.json` is a legacy doctor-import input, not a parallel export/debug store. Runtime code should prefer `updateSessionStore(...)` or `updateSessionStoreEntry(...)`. When a Gateway is reachable, non-dry-run `openclaw sessions cleanup` and `openclaw agents delete` delegate store mutations to the Gateway so cleanup joins the same writer queue. `maxEntries` cleanup is still batched for production-sized caps, so a store may briefly exceed the configured cap before the next high-water cleanup rewrites it back down. Session store reads do not prune or cap entries during Gateway startup; use writes or `openclaw sessions cleanup --enforce` for cleanup. `openclaw sessions cleanup --enforce` still applies the configured cap immediately and prunes old unreferenced transcript, checkpoint, and trajectory artifacts even when no disk budget is configured. Maintenance keeps durable external conversation pointers such as group sessions and thread-scoped chat sessions, but synthetic runtime entries for cron, hooks, @@ -175,7 +177,7 @@ Implementation detail: the decision happens in `initSessionState()` in `src/auto ## Session store schema -The store's value type is `SessionEntry` in `src/config/sessions/types.ts`. `openclaw sessions --export-store ` exports the legacy JSON shape for support tools. +The store's value type is `SessionEntry` in `src/config/sessions/types.ts`. Key fields (not exhaustive): diff --git a/src/cli/program/register.status-health-sessions.test.ts b/src/cli/program/register.status-health-sessions.test.ts index f6baab133e4..d172074ac3c 100644 --- a/src/cli/program/register.status-health-sessions.test.ts +++ b/src/cli/program/register.status-health-sessions.test.ts @@ -193,8 +193,6 @@ describe("registerStatusHealthSessionsCommands", () => { "120", "--limit", "25", - "--export-store", - "/tmp/exported-sessions.json", ]); expect(setVerbose).toHaveBeenCalledWith(true); @@ -204,7 +202,6 @@ describe("registerStatusHealthSessionsCommands", () => { store: "/tmp/sessions.json", active: "120", limit: "25", - exportStore: "/tmp/exported-sessions.json", }), runtime, ); diff --git a/src/cli/program/register.status-health-sessions.ts b/src/cli/program/register.status-health-sessions.ts index d19b015b75a..c4c21e086f7 100644 --- a/src/cli/program/register.status-health-sessions.ts +++ b/src/cli/program/register.status-health-sessions.ts @@ -133,7 +133,6 @@ export function registerStatusHealthSessionsCommands(program: Command) { .option("--all-agents", "Aggregate sessions across all configured agents", false) .option("--active ", "Only show sessions updated within the past N minutes") .option("--limit ", 'Max sessions to show (default: 100; use "all" for full output)') - .option("--export-store ", "Export the raw resolved session store JSON to a file") .addHelpText( "after", () => @@ -144,7 +143,6 @@ export function registerStatusHealthSessionsCommands(program: Command) { ["openclaw sessions --active 120", "Only last 2 hours."], ["openclaw sessions --limit 25", "Show the newest 25 sessions."], ["openclaw sessions --json", "Machine-readable output."], - ["openclaw sessions --export-store sessions-debug.json", "Export raw store JSON."], ["openclaw sessions --store ./tmp/sessions.json", "Use a specific session store."], ])}\n\n${theme.muted( "Shows token usage per session when the agent reports it; set agents.defaults.contextTokens to cap the window and show %.", @@ -165,7 +163,6 @@ export function registerStatusHealthSessionsCommands(program: Command) { allAgents: Boolean(opts.allAgents), active: opts.active as string | undefined, limit: opts.limit as string | undefined, - exportStore: opts.exportStore as string | undefined, }, defaultRuntime, ); diff --git a/src/commands/doctor-state-migrations.test.ts b/src/commands/doctor-state-migrations.test.ts index f58e0a71dc9..24a2c984c5d 100644 --- a/src/commands/doctor-state-migrations.test.ts +++ b/src/commands/doctor-state-migrations.test.ts @@ -3,6 +3,8 @@ import os from "node:os"; import path from "node:path"; import { afterEach, describe, expect, it, vi } from "vitest"; import type { OpenClawConfig } from "../config/config.js"; +import { loadSqliteSessionStore } from "../config/sessions/store-backend.sqlite.js"; +import { closeOpenClawStateDatabaseForTest } from "../state/openclaw-state-db.js"; import { autoMigrateLegacyStateDir, autoMigrateLegacyState, @@ -219,6 +221,7 @@ async function runTelegramAllowFromMigration(params: { root: string; cfg: OpenCl } afterEach(async () => { + closeOpenClawStateDatabaseForTest(); resetAutoMigrateLegacyStateForTest(); resetAutoMigrateLegacyStateDirForTest(); await Promise.all( @@ -258,11 +261,12 @@ async function detectAndRunMigrations(params: { await runLegacyStateMigrations({ detected, now: params.now }); } -function readSessionsStore(targetDir: string) { - return JSON.parse(fs.readFileSync(path.join(targetDir, "sessions.json"), "utf-8")) as Record< - string, - { sessionId: string } - >; +function readSessionsStore(params: { root: string; targetDir: string }) { + const agentId = path.basename(path.dirname(params.targetDir)); + return loadSqliteSessionStore({ + agentId, + env: { OPENCLAW_STATE_DIR: params.root } as NodeJS.ProcessEnv, + }) as Record; } async function runAndReadSessionsStore(params: { @@ -276,7 +280,7 @@ async function runAndReadSessionsStore(params: { cfg: params.cfg, now: params.now, }); - return readSessionsStore(params.targetDir); + return readSessionsStore({ root: params.root, targetDir: params.targetDir }); } type StateDirMigrationResult = Awaited>; @@ -385,9 +389,8 @@ describe("doctor legacy state migrations", () => { expect(fs.existsSync(path.join(targetDir, "b.jsonl"))).toBe(true); expect(fs.existsSync(path.join(legacySessionsDir, "a.jsonl"))).toBe(false); - const store = JSON.parse( - fs.readFileSync(path.join(targetDir, "sessions.json"), "utf-8"), - ) as Record; + expect(fs.existsSync(path.join(targetDir, "sessions.json"))).toBe(false); + const store = readSessionsStore({ root, targetDir }); expect(store["agent:main:main"]?.sessionId).toBe("b"); expect(store["agent:main:+1555"]?.sessionId).toBe("a"); expect(store["agent:main:+1666"]?.sessionId).toBe("b"); @@ -476,7 +479,8 @@ describe("doctor legacy state migrations", () => { const targetDir = path.join(root, "agents", "main", "sessions"); expect(fs.existsSync(path.join(targetDir, "a.jsonl"))).toBe(true); expect(fs.existsSync(path.join(legacySessionsDir, "a.jsonl"))).toBe(false); - expect(fs.existsSync(path.join(targetDir, "sessions.json"))).toBe(true); + expect(fs.existsSync(path.join(targetDir, "sessions.json"))).toBe(false); + expect(readSessionsStore({ root, targetDir })["agent:main:+1555"]?.sessionId).toBe("a"); }); it("migrates legacy WhatsApp auth files without touching oauth.json", async () => { @@ -699,9 +703,8 @@ describe("doctor legacy state migrations", () => { const { result, log } = await runAutoMigrateLegacyStateWithLog({ root, cfg }); - const store = JSON.parse( - fs.readFileSync(path.join(targetDir, "sessions.json"), "utf-8"), - ) as Record; + expect(fs.existsSync(path.join(targetDir, "sessions.json"))).toBe(false); + const store = readSessionsStore({ root, targetDir }); expect(result.migrated).toBe(true); expect(log.info).toHaveBeenCalled(); expect(store["main"]).toBeUndefined(); diff --git a/src/commands/doctor.e2e-harness.ts b/src/commands/doctor.e2e-harness.ts index 2fb963fbce9..2904113da06 100644 --- a/src/commands/doctor.e2e-harness.ts +++ b/src/commands/doctor.e2e-harness.ts @@ -191,6 +191,7 @@ function createLegacyStateMigrationDetectionResult(params?: { targetAgentId: "main", targetMainKey: "main", targetScope: undefined, + env: process.env, stateDir: "/tmp/state", oauthDir: "/tmp/oauth", sessions: { diff --git a/src/commands/sessions.test.ts b/src/commands/sessions.test.ts index ac9e211213c..52987fd085a 100644 --- a/src/commands/sessions.test.ts +++ b/src/commands/sessions.test.ts @@ -201,29 +201,6 @@ describe("sessionsCommand", () => { expect(main?.totalTokensFresh).toBe(false); }); - it("exports the raw resolved session store for debugging", async () => { - const exportedEntry = { - sessionId: "abc123", - updatedAt: Date.now() - 10 * 60_000, - model: "pi:opus", - }; - const store = writeStore({ - "agent:main:main": exportedEntry, - }); - const exportPath = writeStore({}, "sessions-export-target"); - fs.rmSync(exportPath, { force: true }); - - const { runtime, logs } = makeRuntime(); - await sessionsCommand({ store, exportStore: exportPath }, runtime); - - expect(JSON.parse(fs.readFileSync(exportPath, "utf8"))).toEqual({ - "agent:main:main": exportedEntry, - }); - expect(logs.join("\n")).toContain("Exported 1 session(s)"); - fs.rmSync(store, { force: true }); - fs.rmSync(exportPath, { force: true }); - }); - it("applies --active filtering in JSON output", async () => { const store = writeStore( { diff --git a/src/commands/sessions.ts b/src/commands/sessions.ts index a2447551dfb..6758539f56f 100644 --- a/src/commands/sessions.ts +++ b/src/commands/sessions.ts @@ -6,7 +6,6 @@ import { loadSessionStore, resolveSessionTotalTokens } from "../config/sessions. import type { SessionEntry } from "../config/sessions/types.js"; import type { OpenClawConfig } from "../config/types.openclaw.js"; import { info } from "../globals.js"; -import { writeTextAtomic } from "../infra/json-files.js"; import { parseAgentSessionKey } from "../routing/session-key.js"; import { type RuntimeEnv, writeRuntimeJson } from "../runtime.js"; import { isCronSessionKey } from "../sessions/session-key-utils.js"; @@ -194,44 +193,6 @@ function formatRuntimeCell(runtimeLabel: string, rich: boolean): string { return rich ? theme.info(label) : label; } -async function exportRawSessionStores(params: { - targets: NonNullable>; - outputPath: string; -}): Promise<{ - outputPath: string; - count: number; - stores: Array<{ agentId: string; path: string }>; -}> { - const stores = params.targets.map((target) => { - const sessions = loadSessionStore(target.storePath); - return { - agentId: target.agentId, - path: target.storePath, - sessions, - count: Object.keys(sessions).length, - }; - }); - const single = stores.length === 1 ? stores[0]?.sessions : undefined; - const payload = - single !== undefined - ? single - : { - stores: stores.map((store) => ({ - agentId: store.agentId, - path: store.path, - count: store.count, - sessions: store.sessions, - })), - }; - const outputPath = path.resolve(params.outputPath); - await writeTextAtomic(outputPath, `${JSON.stringify(payload, null, 2)}\n`, { mode: 0o600 }); - return { - outputPath, - count: stores.reduce((sum, store) => sum + store.count, 0), - stores: stores.map((store) => ({ agentId: store.agentId, path: store.path })), - }; -} - function toJsonSessionRow(row: SessionRow): Omit { const { runtimeLabel, ...jsonRow } = row; void runtimeLabel; @@ -246,7 +207,6 @@ export async function sessionsCommand( agent?: string; allAgents?: boolean; limit?: string | number; - exportStore?: string; }, runtime: RuntimeEnv, ) { @@ -271,23 +231,6 @@ export async function sessionsCommand( return; } - if (opts.exportStore) { - const exported = await exportRawSessionStores({ - targets, - outputPath: opts.exportStore, - }); - if (opts.json) { - writeRuntimeJson(runtime, { - exportedPath: exported.outputPath, - count: exported.count, - stores: exported.stores, - }); - } else { - runtime.log(info(`Exported ${exported.count} session(s) to ${exported.outputPath}`)); - } - return; - } - let activeMinutes: number | undefined; if (opts.active !== undefined) { const parsed = Number.parseInt(opts.active, 10); diff --git a/src/config/sessions/store-backend.sqlite.test.ts b/src/config/sessions/store-backend.sqlite.test.ts index 9eafb5c1409..8d6e27680a5 100644 --- a/src/config/sessions/store-backend.sqlite.test.ts +++ b/src/config/sessions/store-backend.sqlite.test.ts @@ -6,7 +6,6 @@ import { writeTextAtomic } from "../../infra/json-files.js"; import { closeOpenClawStateDatabaseForTest } from "../../state/openclaw-state-db.js"; import { resolveStorePath } from "./paths.js"; import { - exportSqliteSessionStore, importJsonSessionStoreToSqlite, loadSqliteSessionStore, saveSqliteSessionStore, @@ -15,7 +14,6 @@ import { loadSessionStore } from "./store-load.js"; import { saveSessionStore, updateSessionStore } from "./store.js"; import type { SessionEntry } from "./types.js"; -const ORIGINAL_SESSION_STORE_BACKEND = process.env.OPENCLAW_SESSION_STORE_BACKEND; const ORIGINAL_STATE_DIR = process.env.OPENCLAW_STATE_DIR; function createTempDir(): string { @@ -24,11 +22,6 @@ function createTempDir(): string { afterEach(() => { closeOpenClawStateDatabaseForTest(); - if (ORIGINAL_SESSION_STORE_BACKEND === undefined) { - delete process.env.OPENCLAW_SESSION_STORE_BACKEND; - } else { - process.env.OPENCLAW_SESSION_STORE_BACKEND = ORIGINAL_SESSION_STORE_BACKEND; - } if (ORIGINAL_STATE_DIR === undefined) { delete process.env.OPENCLAW_STATE_DIR; } else { @@ -68,7 +61,7 @@ describe("SQLite session store backend", () => { }); }); - it("imports legacy sessions.json and exports the SQLite snapshot", async () => { + it("imports legacy sessions.json into SQLite and removes the JSON source", async () => { const stateDir = createTempDir(); const dbPath = path.join(stateDir, "state", "openclaw.sqlite"); const legacyStorePath = path.join(stateDir, "sessions.json"); @@ -96,9 +89,10 @@ describe("SQLite session store backend", () => { sourcePath: legacyStorePath, dbPath, }), - ).toEqual({ imported: 1, sourcePath: legacyStorePath }); + ).toEqual({ imported: 1, sourcePath: legacyStorePath, removedSource: true }); - expect(exportSqliteSessionStore({ agentId: "main", path: dbPath })).toEqual({ + expect(fs.existsSync(legacyStorePath)).toBe(false); + expect(loadSqliteSessionStore({ agentId: "main", path: dbPath })).toEqual({ "discord:user-1": { ...entry, deliveryContext: { @@ -109,10 +103,9 @@ describe("SQLite session store backend", () => { }); }); - it("routes the production session store API through SQLite behind the opt-in flag", async () => { + it("routes the production session store API through SQLite", async () => { const stateDir = createTempDir(); process.env.OPENCLAW_STATE_DIR = stateDir; - process.env.OPENCLAW_SESSION_STORE_BACKEND = "sqlite"; const storePath = resolveStorePath(undefined, { agentId: "ops", env: process.env }); const entry: SessionEntry = { sessionId: "sqlite-primary", @@ -146,7 +139,6 @@ describe("SQLite session store backend", () => { it("uses SQLite by default for canonical per-agent session stores", async () => { const stateDir = createTempDir(); process.env.OPENCLAW_STATE_DIR = stateDir; - delete process.env.OPENCLAW_SESSION_STORE_BACKEND; const storePath = resolveStorePath(undefined, { agentId: "ops", env: process.env }); const entry: SessionEntry = { sessionId: "sqlite-default", @@ -162,29 +154,9 @@ describe("SQLite session store backend", () => { }); }); - it("keeps canonical per-agent session stores file-backed when the JSON backend is forced", async () => { + it("does not import a legacy canonical sessions.json on first SQLite open", async () => { const stateDir = createTempDir(); process.env.OPENCLAW_STATE_DIR = stateDir; - process.env.OPENCLAW_SESSION_STORE_BACKEND = "json"; - const storePath = resolveStorePath(undefined, { agentId: "ops", env: process.env }); - const entry: SessionEntry = { - sessionId: "json-forced", - sessionFile: "json-forced.jsonl", - updatedAt: 100, - }; - - await saveSessionStore(storePath, { "discord:ops": entry }, { skipMaintenance: true }); - - expect(fs.existsSync(storePath)).toBe(true); - expect(loadSessionStore(storePath, { skipCache: true })).toEqual({ - "discord:ops": entry, - }); - }); - - it("imports a legacy canonical sessions.json once on first SQLite open", async () => { - const stateDir = createTempDir(); - process.env.OPENCLAW_STATE_DIR = stateDir; - process.env.OPENCLAW_SESSION_STORE_BACKEND = "sqlite"; const storePath = resolveStorePath(undefined, { agentId: "ops", env: process.env }); const legacyEntry: SessionEntry = { sessionId: "legacy-session", @@ -202,9 +174,7 @@ describe("SQLite session store backend", () => { ), ); - expect(loadSessionStore(storePath, { skipCache: true })).toEqual({ - "discord:ops": legacyEntry, - }); + expect(loadSessionStore(storePath, { skipCache: true })).toEqual({}); await saveSessionStore( storePath, @@ -217,21 +187,6 @@ describe("SQLite session store backend", () => { }, { skipMaintenance: true }, ); - await writeTextAtomic( - storePath, - JSON.stringify( - { - "discord:ops": { - ...legacyEntry, - sessionId: "stale-json-session", - updatedAt: 300, - }, - }, - null, - 2, - ), - ); - expect(loadSessionStore(storePath, { skipCache: true })).toEqual({ "discord:ops": { ...legacyEntry, diff --git a/src/config/sessions/store-backend.sqlite.ts b/src/config/sessions/store-backend.sqlite.ts index e5247ffb2fe..a299a106c16 100644 --- a/src/config/sessions/store-backend.sqlite.ts +++ b/src/config/sessions/store-backend.sqlite.ts @@ -7,7 +7,7 @@ import { runOpenClawStateWriteTransaction, type OpenClawStateDatabaseOptions, } from "../../state/openclaw-state-db.js"; -import { resolveAgentIdFromSessionStorePath, resolveStorePath } from "./paths.js"; +import { resolveAgentIdFromSessionStorePath } from "./paths.js"; import { applySessionStoreMigrations } from "./store-migrations.js"; import { normalizeSessionStore } from "./store-normalize.js"; import type { SessionEntry } from "./types.js"; @@ -21,6 +21,7 @@ export type SqliteSessionStoreOptions = OpenClawStateDatabaseOptions & { export type SessionStoreBackendImportResult = { imported: number; sourcePath: string; + removedSource: boolean; }; type SessionEntryRow = { @@ -28,51 +29,21 @@ type SessionEntryRow = { entry_json: string; }; -export type SqliteSessionStoreBackendMode = "auto" | "json" | "sqlite"; - -export function resolveSqliteSessionStoreBackendMode( - env: NodeJS.ProcessEnv = process.env, -): SqliteSessionStoreBackendMode { - const raw = env.OPENCLAW_SESSION_STORE_BACKEND?.trim().toLowerCase(); - if (raw === "json" || raw === "file" || raw === "files" || raw === "disabled") { - return "json"; - } - if (raw === "sqlite") { - return "sqlite"; - } - return "auto"; -} - -export function isSqliteSessionStoreBackendEnabled(env: NodeJS.ProcessEnv = process.env): boolean { - return resolveSqliteSessionStoreBackendMode(env) !== "json"; -} - -function isCanonicalAgentSessionStorePath(params: { - storePath: string; - agentId: string; - env: NodeJS.ProcessEnv; -}): boolean { - return ( - path.resolve(params.storePath) === - path.resolve(resolveStorePath(undefined, { agentId: params.agentId, env: params.env })) - ); +export function isSqliteSessionStoreBackendEnabled(_env: NodeJS.ProcessEnv = process.env): boolean { + return true; } export function resolveSqliteSessionStoreOptionsForPath( storePath: string, env: NodeJS.ProcessEnv = process.env, ): SqliteSessionStoreOptions | null { - const mode = resolveSqliteSessionStoreBackendMode(env); - if (mode === "json") { + if (!isSqliteSessionStoreBackendEnabled(env)) { return null; } const agentId = resolveAgentIdFromSessionStorePath(storePath); if (!agentId) { return null; } - if (mode === "auto" && !isCanonicalAgentSessionStorePath({ storePath, agentId, env })) { - return null; - } return { agentId, env, sourcePath: storePath }; } @@ -110,63 +81,6 @@ function prepareReplaceStatement(statement: StatementSync, params: Record { let store: Record = {}; try { @@ -228,35 +147,9 @@ function replaceSqliteSessionStore(params: { } } -function importLegacyJsonSessionStoreIfNeeded(options: SqliteSessionStoreOptions): void { - const sourcePath = options.sourcePath?.trim(); - if (!sourcePath || !fs.existsSync(sourcePath)) { - return; - } - runOpenClawStateWriteTransaction((database) => { - if (readImportMarker(database, options)) { - return; - } - const existingRows = countSqliteSessionEntries(database, options); - if (existingRows > 0) { - writeImportMarker({ database, options, imported: 0, sourceExists: true }); - return; - } - const store = parseJsonSessionStoreFromPath(sourcePath); - replaceSqliteSessionStore({ database, options, store }); - writeImportMarker({ - database, - options, - imported: Object.keys(store).length, - sourceExists: true, - }); - }, options); -} - export function loadSqliteSessionStore( options: SqliteSessionStoreOptions, ): Record { - importLegacyJsonSessionStoreIfNeeded(options); const database = openOpenClawStateDatabase(options); const rows = database.db .prepare( @@ -286,38 +179,59 @@ export function saveSqliteSessionStore( normalizeSessionStore(store); runOpenClawStateWriteTransaction((database) => { replaceSqliteSessionStore({ database, options, store }); - writeImportMarker({ - database, - options, - imported: Object.keys(store).length, - sourceExists: Boolean(options.sourcePath && fs.existsSync(options.sourcePath)), - }); }, options); } +function resolveSessionEntryUpdatedAt(entry: SessionEntry): number { + return typeof entry.updatedAt === "number" && Number.isFinite(entry.updatedAt) + ? entry.updatedAt + : 0; +} + +function mergeSessionStoresByUpdatedAt( + existing: Record, + incoming: Record, +): Record { + const merged: Record = { ...existing }; + for (const [key, entry] of Object.entries(incoming)) { + const current = merged[key]; + if (!current || resolveSessionEntryUpdatedAt(entry) >= resolveSessionEntryUpdatedAt(current)) { + merged[key] = entry; + } + } + normalizeSessionStore(merged); + return merged; +} + export function importJsonSessionStoreToSqlite(params: { agentId: string; sourcePath: string; dbPath?: string; + env?: NodeJS.ProcessEnv; now?: () => number; }): SessionStoreBackendImportResult { - const store = parseJsonSessionStoreFromPath(params.sourcePath); - saveSqliteSessionStore( - { - agentId: params.agentId, - sourcePath: params.sourcePath, - ...(params.dbPath ? { path: params.dbPath } : {}), - ...(params.now ? { now: params.now } : {}), - }, - store, - ); - return { imported: Object.keys(store).length, sourcePath: params.sourcePath }; -} - -export function exportSqliteSessionStore( - options: SqliteSessionStoreOptions, -): Record { - return loadSqliteSessionStore(options); + const options = { + agentId: params.agentId, + sourcePath: params.sourcePath, + ...(params.env ? { env: params.env } : {}), + ...(params.dbPath ? { path: params.dbPath } : {}), + ...(params.now ? { now: params.now } : {}), + }; + const importedStore = parseJsonSessionStoreFromPath(params.sourcePath); + const mergedStore = mergeSessionStoresByUpdatedAt(loadSqliteSessionStore(options), importedStore); + saveSqliteSessionStore(options, mergedStore); + let removedSource = false; + try { + fs.rmSync(params.sourcePath, { force: true }); + removedSource = true; + } catch { + removedSource = false; + } + return { + imported: Object.keys(importedStore).length, + sourcePath: params.sourcePath, + removedSource, + }; } export function readJsonSessionStoreRawForImport(pathname: string): string { diff --git a/src/infra/state-migrations.ts b/src/infra/state-migrations.ts index 346a9329428..7f6bb1d8c18 100644 --- a/src/infra/state-migrations.ts +++ b/src/infra/state-migrations.ts @@ -16,6 +16,7 @@ import { import type { SessionEntry } from "../config/sessions.js"; import { saveSessionStore } from "../config/sessions.js"; import { canonicalizeMainSessionAlias } from "../config/sessions/main-session.js"; +import { importJsonSessionStoreToSqlite } from "../config/sessions/store-backend.sqlite.js"; import type { SessionScope } from "../config/sessions/types.js"; import type { OpenClawConfig } from "../config/types.openclaw.js"; import { createSubsystemLogger } from "../logging/subsystem.js"; @@ -46,6 +47,7 @@ export type LegacyStateDetection = { targetAgentId: string; targetMainKey: string; targetScope?: SessionScope; + env: NodeJS.ProcessEnv; stateDir: string; oauthDir: string; sessions: { @@ -706,6 +708,7 @@ export async function detectLegacyStateMigrations(params: { const sessionsLegacyStorePath = path.join(sessionsLegacyDir, "sessions.json"); const sessionsTargetDir = path.join(stateDir, "agents", targetAgentId, "sessions"); const sessionsTargetStorePath = path.join(sessionsTargetDir, "sessions.json"); + const hasTargetJsonSessionStore = fileExists(sessionsTargetStorePath); const legacySessionEntries = safeReadDir(sessionsLegacyDir); const hasLegacySessions = fileExists(sessionsLegacyStorePath) || @@ -740,6 +743,9 @@ export async function detectLegacyStateMigrations(params: { if (legacyKeys.length > 0) { preview.push(`- Sessions: canonicalize legacy keys in ${sessionsTargetStorePath}`); } + if (hasTargetJsonSessionStore) { + preview.push(`- Sessions: import ${sessionsTargetStorePath} into SQLite`); + } if (hasLegacyAgentDir) { preview.push(`- Agent dir: ${legacyAgentDir} → ${targetAgentDir}`); } @@ -751,6 +757,7 @@ export async function detectLegacyStateMigrations(params: { targetAgentId, targetMainKey, targetScope, + env, stateDir, oauthDir, sessions: { @@ -758,7 +765,7 @@ export async function detectLegacyStateMigrations(params: { legacyStorePath: sessionsLegacyStorePath, targetDir: sessionsTargetDir, targetStorePath: sessionsTargetStorePath, - hasLegacy: hasLegacySessions || legacyKeys.length > 0, + hasLegacy: hasLegacySessions || legacyKeys.length > 0 || hasTargetJsonSessionStore, legacyKeys, }, agentDir: { @@ -792,6 +799,7 @@ async function migrateLegacySessions( const targetParsed = fileExists(detected.sessions.targetStorePath) ? readSessionStoreJson5(detected.sessions.targetStorePath) : { store: {}, ok: true }; + const hasTargetSessionStoreFile = fileExists(detected.sessions.targetStorePath); const legacyStore = legacyParsed.store; const targetStore = targetParsed.store; @@ -837,7 +845,9 @@ async function migrateLegacySessions( if ( (legacyParsed.ok || targetParsed.ok) && - (Object.keys(legacyStore).length > 0 || Object.keys(targetStore).length > 0) + (Object.keys(legacyStore).length > 0 || + Object.keys(targetStore).length > 0 || + (hasTargetSessionStoreFile && targetParsed.ok)) ) { const normalized: Record = {}; for (const [key, entry] of Object.entries(merged)) { @@ -851,6 +861,16 @@ async function migrateLegacySessions( skipMaintenance: true, }); changes.push(`Merged sessions store → ${detected.sessions.targetStorePath}`); + if (fileExists(detected.sessions.targetStorePath)) { + const imported = importJsonSessionStoreToSqlite({ + agentId: detected.targetAgentId, + env: detected.env, + sourcePath: detected.sessions.targetStorePath, + }); + changes.push( + `Imported ${imported.imported} session index row(s) into SQLite for agent ${detected.targetAgentId}`, + ); + } if (canonicalizedTarget.legacyKeys.length > 0) { changes.push(`Canonicalized ${canonicalizedTarget.legacyKeys.length} legacy session key(s)`); }