mirror of
https://github.com/moltbot/moltbot.git
synced 2026-05-18 20:19:47 +00:00
refactor: make sessions json doctor-import only
This commit is contained in:
Peter Steinberger
@@ -133,15 +133,15 @@ Canonical session metadata lives in the shared state database:
|
||||
- `~/.openclaw/state/openclaw.sqlite`
|
||||
- JSONL transcripts live alongside the store
|
||||
|
||||
Legacy `sessions.json` files are imported on first open and remain the
|
||||
compatibility/export shape. You can still force or override a JSON file-backed
|
||||
store via `session.store` and `{agentId}` templating.
|
||||
Legacy `sessions.json` indexes are imported by `openclaw doctor --fix` and
|
||||
removed after SQLite has the rows. Custom offline repair stores can still use an
|
||||
explicit `--store` path, but per-agent runtime metadata should go through the
|
||||
shared state database.
|
||||
|
||||
Gateway and ACP session discovery also scans disk-backed agent stores under the
|
||||
default `agents/` root and under templated `session.store` roots. Discovered
|
||||
stores must stay inside that resolved agent root and use a regular
|
||||
`sessions.json` file when JSON compatibility mode is selected. Symlinks and
|
||||
out-of-root paths are ignored.
|
||||
stores must stay inside that resolved agent root. Symlinks and out-of-root paths
|
||||
are ignored.
|
||||
|
||||
## WebChat behavior
|
||||
|
||||
|
||||
@@ -36,7 +36,6 @@ openclaw sessions --active 120
|
||||
openclaw sessions --limit 25
|
||||
openclaw sessions --verbose
|
||||
openclaw sessions --json
|
||||
openclaw sessions --export-store sessions-debug.json
|
||||
```
|
||||
|
||||
Scope selection:
|
||||
@@ -46,13 +45,10 @@ Scope selection:
|
||||
- `--agent <id>`: one configured agent store
|
||||
- `--all-agents`: aggregate all configured agent stores
|
||||
- `--store <path>`: explicit store path (cannot be combined with `--agent` or `--all-agents`)
|
||||
- `--export-store <path>`: export the raw resolved store JSON to a file
|
||||
|
||||
Canonical per-agent session stores use OpenClaw's shared SQLite state database by
|
||||
default. Custom `--store` paths stay file-backed JSON unless
|
||||
`OPENCLAW_SESSION_STORE_BACKEND=sqlite` is set. Set
|
||||
`OPENCLAW_SESSION_STORE_BACKEND=json` to force the legacy `sessions.json`
|
||||
backend for repair or bisect work.
|
||||
default. Existing `sessions.json` indexes are imported by `openclaw doctor
|
||||
--fix`, then removed after SQLite has the rows.
|
||||
|
||||
- `--limit <n|all>`: max rows to output (default `100`; `all` restores full output)
|
||||
|
||||
@@ -97,11 +93,6 @@ JSON examples:
|
||||
}
|
||||
```
|
||||
|
||||
Use `openclaw sessions --export-store <path>` to write the raw resolved session
|
||||
store to JSON for debugging, backup, or SQLite compatibility checks. The command
|
||||
reads through the active backend, so SQLite-backed stores export the same shape
|
||||
older `sessions.json` tools expect.
|
||||
|
||||
## Cleanup maintenance
|
||||
|
||||
Run maintenance now (instead of waiting for the next write cycle):
|
||||
|
||||
@@ -46,10 +46,9 @@ This plan has started landing in slices:
|
||||
The shared `kv` table now has a small typed helper for scoped JSON-compatible
|
||||
values so low-risk JSON sidecars can move behind the same SQLite connection
|
||||
without each feature reimplementing read/write/delete glue.
|
||||
- Canonical per-agent session stores use SQLite by default. Legacy
|
||||
`sessions.json` imports once, custom `--store` paths remain JSON by default,
|
||||
and `openclaw sessions --export-store <path>` exports the compatibility JSON
|
||||
shape.
|
||||
- Canonical per-agent session stores use SQLite by default. `openclaw doctor
|
||||
--fix` imports legacy `sessions.json` indexes into SQLite and removes the JSON
|
||||
index after import, instead of keeping a parallel compatibility/export store.
|
||||
- Transcript events have a SQLite store primitive with JSONL import/export.
|
||||
Transcript append paths dual-write when the caller already has agent and
|
||||
session scope, including gateway-injected assistant messages. Scoped appends
|
||||
@@ -344,7 +343,8 @@ Migration order:
|
||||
2. Add shared SQLite connection and migration helpers.
|
||||
3. Move `sessions.json` behind a `SessionStoreBackend` interface.
|
||||
4. Make SQLite primary for session entries.
|
||||
5. Import old `sessions.json` on first open and keep an export/debug command.
|
||||
5. Import old `sessions.json` from `openclaw doctor --fix`, then remove the JSON
|
||||
index after SQLite has the rows.
|
||||
6. Leave `*.jsonl` transcripts on disk while PI owns transcript semantics.
|
||||
7. After session manager ownership moves behind OpenClaw APIs, store transcript
|
||||
events in SQLite and export JSONL for compatibility.
|
||||
@@ -543,7 +543,8 @@ Phase 0: inventory and contracts
|
||||
Phase 1: SQLite session index
|
||||
|
||||
- Add shared state DB helper.
|
||||
- Keep `sessions.json` compatibility.
|
||||
- Add a doctor migration that imports `sessions.json` into SQLite and removes
|
||||
the JSON index.
|
||||
- Move session entries to SQLite behind a flag.
|
||||
- Prove current session list, patch, reset, cleanup, and UI flows.
|
||||
|
||||
@@ -601,7 +602,7 @@ This refactor is successful when:
|
||||
- Core code no longer imports PI packages outside the runtime adapter.
|
||||
- Repeated JSON, transcript, PI adapter, and scratch filesystem logic has one
|
||||
owner each.
|
||||
- `sessions.json` is compatibility-only, not the primary Gateway store.
|
||||
- `sessions.json` is a doctor-migrated legacy input, not a compatibility store.
|
||||
- Scratch state and tool artifacts can live in SQLite-backed VFS.
|
||||
- Agents can run in disk, VFS scratch, and VFS-only filesystem modes.
|
||||
- Real workspace writes still use capability-safe host filesystem operations.
|
||||
|
||||
@@ -74,7 +74,9 @@ cached by file path plus `mtimeMs`/`size` and shared across concurrent readers.
|
||||
|
||||
Per agent, on the Gateway host:
|
||||
|
||||
- Store: `~/.openclaw/state/openclaw.sqlite` by default; legacy/custom JSON stores use `~/.openclaw/agents/<agentId>/sessions/sessions.json`
|
||||
- Store: `~/.openclaw/state/openclaw.sqlite` by default. `openclaw doctor --fix`
|
||||
imports legacy `~/.openclaw/agents/<agentId>/sessions/sessions.json` indexes
|
||||
into SQLite and removes the JSON index after import.
|
||||
- Transcripts: `~/.openclaw/agents/<agentId>/sessions/<sessionId>.jsonl`
|
||||
- Telegram topic sessions: `.../<sessionId>-topic-<threadId>.jsonl`
|
||||
|
||||
@@ -93,7 +95,7 @@ Session persistence has automatic maintenance controls (`session.maintenance`) f
|
||||
- `maxDiskBytes`: optional sessions-directory budget
|
||||
- `highWaterBytes`: optional target after cleanup (default `80%` of `maxDiskBytes`)
|
||||
|
||||
Normal Gateway writes flow through a per-store session writer that serializes in-process mutations. SQLite is the canonical per-agent backend; the JSON writer remains for legacy/custom stores and offline compatibility. Runtime code should prefer `updateSessionStore(...)` or `updateSessionStoreEntry(...)`; direct whole-store saves are compatibility and offline-maintenance tools. When a Gateway is reachable, non-dry-run `openclaw sessions cleanup` and `openclaw agents delete` delegate store mutations to the Gateway so cleanup joins the same writer queue; `--store <path>` is the explicit offline repair path for direct JSON file maintenance. `maxEntries` cleanup is still batched for production-sized caps, so a store may briefly exceed the configured cap before the next high-water cleanup rewrites it back down. Session store reads do not prune or cap entries during Gateway startup; use writes or `openclaw sessions cleanup --enforce` for cleanup. `openclaw sessions cleanup --enforce` still applies the configured cap immediately and prunes old unreferenced transcript, checkpoint, and trajectory artifacts even when no disk budget is configured.
|
||||
Normal Gateway writes flow through a per-store session writer that serializes in-process mutations. SQLite is the canonical per-agent backend; `sessions.json` is a legacy doctor-import input, not a parallel export/debug store. Runtime code should prefer `updateSessionStore(...)` or `updateSessionStoreEntry(...)`. When a Gateway is reachable, non-dry-run `openclaw sessions cleanup` and `openclaw agents delete` delegate store mutations to the Gateway so cleanup joins the same writer queue. `maxEntries` cleanup is still batched for production-sized caps, so a store may briefly exceed the configured cap before the next high-water cleanup rewrites it back down. Session store reads do not prune or cap entries during Gateway startup; use writes or `openclaw sessions cleanup --enforce` for cleanup. `openclaw sessions cleanup --enforce` still applies the configured cap immediately and prunes old unreferenced transcript, checkpoint, and trajectory artifacts even when no disk budget is configured.
|
||||
|
||||
Maintenance keeps durable external conversation pointers such as group sessions
|
||||
and thread-scoped chat sessions, but synthetic runtime entries for cron, hooks,
|
||||
@@ -175,7 +177,7 @@ Implementation detail: the decision happens in `initSessionState()` in `src/auto
|
||||
|
||||
## Session store schema
|
||||
|
||||
The store's value type is `SessionEntry` in `src/config/sessions/types.ts`. `openclaw sessions --export-store <path>` exports the legacy JSON shape for support tools.
|
||||
The store's value type is `SessionEntry` in `src/config/sessions/types.ts`.
|
||||
|
||||
Key fields (not exhaustive):
|
||||
|
||||
|
||||
@@ -193,8 +193,6 @@ describe("registerStatusHealthSessionsCommands", () => {
|
||||
"120",
|
||||
"--limit",
|
||||
"25",
|
||||
"--export-store",
|
||||
"/tmp/exported-sessions.json",
|
||||
]);
|
||||
|
||||
expect(setVerbose).toHaveBeenCalledWith(true);
|
||||
@@ -204,7 +202,6 @@ describe("registerStatusHealthSessionsCommands", () => {
|
||||
store: "/tmp/sessions.json",
|
||||
active: "120",
|
||||
limit: "25",
|
||||
exportStore: "/tmp/exported-sessions.json",
|
||||
}),
|
||||
runtime,
|
||||
);
|
||||
|
||||
@@ -133,7 +133,6 @@ export function registerStatusHealthSessionsCommands(program: Command) {
|
||||
.option("--all-agents", "Aggregate sessions across all configured agents", false)
|
||||
.option("--active <minutes>", "Only show sessions updated within the past N minutes")
|
||||
.option("--limit <count>", 'Max sessions to show (default: 100; use "all" for full output)')
|
||||
.option("--export-store <path>", "Export the raw resolved session store JSON to a file")
|
||||
.addHelpText(
|
||||
"after",
|
||||
() =>
|
||||
@@ -144,7 +143,6 @@ export function registerStatusHealthSessionsCommands(program: Command) {
|
||||
["openclaw sessions --active 120", "Only last 2 hours."],
|
||||
["openclaw sessions --limit 25", "Show the newest 25 sessions."],
|
||||
["openclaw sessions --json", "Machine-readable output."],
|
||||
["openclaw sessions --export-store sessions-debug.json", "Export raw store JSON."],
|
||||
["openclaw sessions --store ./tmp/sessions.json", "Use a specific session store."],
|
||||
])}\n\n${theme.muted(
|
||||
"Shows token usage per session when the agent reports it; set agents.defaults.contextTokens to cap the window and show %.",
|
||||
@@ -165,7 +163,6 @@ export function registerStatusHealthSessionsCommands(program: Command) {
|
||||
allAgents: Boolean(opts.allAgents),
|
||||
active: opts.active as string | undefined,
|
||||
limit: opts.limit as string | undefined,
|
||||
exportStore: opts.exportStore as string | undefined,
|
||||
},
|
||||
defaultRuntime,
|
||||
);
|
||||
|
||||
@@ -3,6 +3,8 @@ import os from "node:os";
|
||||
import path from "node:path";
|
||||
import { afterEach, describe, expect, it, vi } from "vitest";
|
||||
import type { OpenClawConfig } from "../config/config.js";
|
||||
import { loadSqliteSessionStore } from "../config/sessions/store-backend.sqlite.js";
|
||||
import { closeOpenClawStateDatabaseForTest } from "../state/openclaw-state-db.js";
|
||||
import {
|
||||
autoMigrateLegacyStateDir,
|
||||
autoMigrateLegacyState,
|
||||
@@ -219,6 +221,7 @@ async function runTelegramAllowFromMigration(params: { root: string; cfg: OpenCl
|
||||
}
|
||||
|
||||
afterEach(async () => {
|
||||
closeOpenClawStateDatabaseForTest();
|
||||
resetAutoMigrateLegacyStateForTest();
|
||||
resetAutoMigrateLegacyStateDirForTest();
|
||||
await Promise.all(
|
||||
@@ -258,11 +261,12 @@ async function detectAndRunMigrations(params: {
|
||||
await runLegacyStateMigrations({ detected, now: params.now });
|
||||
}
|
||||
|
||||
function readSessionsStore(targetDir: string) {
|
||||
return JSON.parse(fs.readFileSync(path.join(targetDir, "sessions.json"), "utf-8")) as Record<
|
||||
string,
|
||||
{ sessionId: string }
|
||||
>;
|
||||
function readSessionsStore(params: { root: string; targetDir: string }) {
|
||||
const agentId = path.basename(path.dirname(params.targetDir));
|
||||
return loadSqliteSessionStore({
|
||||
agentId,
|
||||
env: { OPENCLAW_STATE_DIR: params.root } as NodeJS.ProcessEnv,
|
||||
}) as Record<string, { sessionId: string }>;
|
||||
}
|
||||
|
||||
async function runAndReadSessionsStore(params: {
|
||||
@@ -276,7 +280,7 @@ async function runAndReadSessionsStore(params: {
|
||||
cfg: params.cfg,
|
||||
now: params.now,
|
||||
});
|
||||
return readSessionsStore(params.targetDir);
|
||||
return readSessionsStore({ root: params.root, targetDir: params.targetDir });
|
||||
}
|
||||
|
||||
type StateDirMigrationResult = Awaited<ReturnType<typeof autoMigrateLegacyStateDir>>;
|
||||
@@ -385,9 +389,8 @@ describe("doctor legacy state migrations", () => {
|
||||
expect(fs.existsSync(path.join(targetDir, "b.jsonl"))).toBe(true);
|
||||
expect(fs.existsSync(path.join(legacySessionsDir, "a.jsonl"))).toBe(false);
|
||||
|
||||
const store = JSON.parse(
|
||||
fs.readFileSync(path.join(targetDir, "sessions.json"), "utf-8"),
|
||||
) as Record<string, { sessionId: string }>;
|
||||
expect(fs.existsSync(path.join(targetDir, "sessions.json"))).toBe(false);
|
||||
const store = readSessionsStore({ root, targetDir });
|
||||
expect(store["agent:main:main"]?.sessionId).toBe("b");
|
||||
expect(store["agent:main:+1555"]?.sessionId).toBe("a");
|
||||
expect(store["agent:main:+1666"]?.sessionId).toBe("b");
|
||||
@@ -476,7 +479,8 @@ describe("doctor legacy state migrations", () => {
|
||||
const targetDir = path.join(root, "agents", "main", "sessions");
|
||||
expect(fs.existsSync(path.join(targetDir, "a.jsonl"))).toBe(true);
|
||||
expect(fs.existsSync(path.join(legacySessionsDir, "a.jsonl"))).toBe(false);
|
||||
expect(fs.existsSync(path.join(targetDir, "sessions.json"))).toBe(true);
|
||||
expect(fs.existsSync(path.join(targetDir, "sessions.json"))).toBe(false);
|
||||
expect(readSessionsStore({ root, targetDir })["agent:main:+1555"]?.sessionId).toBe("a");
|
||||
});
|
||||
|
||||
it("migrates legacy WhatsApp auth files without touching oauth.json", async () => {
|
||||
@@ -699,9 +703,8 @@ describe("doctor legacy state migrations", () => {
|
||||
|
||||
const { result, log } = await runAutoMigrateLegacyStateWithLog({ root, cfg });
|
||||
|
||||
const store = JSON.parse(
|
||||
fs.readFileSync(path.join(targetDir, "sessions.json"), "utf-8"),
|
||||
) as Record<string, { sessionId: string }>;
|
||||
expect(fs.existsSync(path.join(targetDir, "sessions.json"))).toBe(false);
|
||||
const store = readSessionsStore({ root, targetDir });
|
||||
expect(result.migrated).toBe(true);
|
||||
expect(log.info).toHaveBeenCalled();
|
||||
expect(store["main"]).toBeUndefined();
|
||||
|
||||
@@ -191,6 +191,7 @@ function createLegacyStateMigrationDetectionResult(params?: {
|
||||
targetAgentId: "main",
|
||||
targetMainKey: "main",
|
||||
targetScope: undefined,
|
||||
env: process.env,
|
||||
stateDir: "/tmp/state",
|
||||
oauthDir: "/tmp/oauth",
|
||||
sessions: {
|
||||
|
||||
@@ -201,29 +201,6 @@ describe("sessionsCommand", () => {
|
||||
expect(main?.totalTokensFresh).toBe(false);
|
||||
});
|
||||
|
||||
it("exports the raw resolved session store for debugging", async () => {
|
||||
const exportedEntry = {
|
||||
sessionId: "abc123",
|
||||
updatedAt: Date.now() - 10 * 60_000,
|
||||
model: "pi:opus",
|
||||
};
|
||||
const store = writeStore({
|
||||
"agent:main:main": exportedEntry,
|
||||
});
|
||||
const exportPath = writeStore({}, "sessions-export-target");
|
||||
fs.rmSync(exportPath, { force: true });
|
||||
|
||||
const { runtime, logs } = makeRuntime();
|
||||
await sessionsCommand({ store, exportStore: exportPath }, runtime);
|
||||
|
||||
expect(JSON.parse(fs.readFileSync(exportPath, "utf8"))).toEqual({
|
||||
"agent:main:main": exportedEntry,
|
||||
});
|
||||
expect(logs.join("\n")).toContain("Exported 1 session(s)");
|
||||
fs.rmSync(store, { force: true });
|
||||
fs.rmSync(exportPath, { force: true });
|
||||
});
|
||||
|
||||
it("applies --active filtering in JSON output", async () => {
|
||||
const store = writeStore(
|
||||
{
|
||||
|
||||
@@ -6,7 +6,6 @@ import { loadSessionStore, resolveSessionTotalTokens } from "../config/sessions.
|
||||
import type { SessionEntry } from "../config/sessions/types.js";
|
||||
import type { OpenClawConfig } from "../config/types.openclaw.js";
|
||||
import { info } from "../globals.js";
|
||||
import { writeTextAtomic } from "../infra/json-files.js";
|
||||
import { parseAgentSessionKey } from "../routing/session-key.js";
|
||||
import { type RuntimeEnv, writeRuntimeJson } from "../runtime.js";
|
||||
import { isCronSessionKey } from "../sessions/session-key-utils.js";
|
||||
@@ -194,44 +193,6 @@ function formatRuntimeCell(runtimeLabel: string, rich: boolean): string {
|
||||
return rich ? theme.info(label) : label;
|
||||
}
|
||||
|
||||
async function exportRawSessionStores(params: {
|
||||
targets: NonNullable<ReturnType<typeof resolveSessionStoreTargetsOrExit>>;
|
||||
outputPath: string;
|
||||
}): Promise<{
|
||||
outputPath: string;
|
||||
count: number;
|
||||
stores: Array<{ agentId: string; path: string }>;
|
||||
}> {
|
||||
const stores = params.targets.map((target) => {
|
||||
const sessions = loadSessionStore(target.storePath);
|
||||
return {
|
||||
agentId: target.agentId,
|
||||
path: target.storePath,
|
||||
sessions,
|
||||
count: Object.keys(sessions).length,
|
||||
};
|
||||
});
|
||||
const single = stores.length === 1 ? stores[0]?.sessions : undefined;
|
||||
const payload =
|
||||
single !== undefined
|
||||
? single
|
||||
: {
|
||||
stores: stores.map((store) => ({
|
||||
agentId: store.agentId,
|
||||
path: store.path,
|
||||
count: store.count,
|
||||
sessions: store.sessions,
|
||||
})),
|
||||
};
|
||||
const outputPath = path.resolve(params.outputPath);
|
||||
await writeTextAtomic(outputPath, `${JSON.stringify(payload, null, 2)}\n`, { mode: 0o600 });
|
||||
return {
|
||||
outputPath,
|
||||
count: stores.reduce((sum, store) => sum + store.count, 0),
|
||||
stores: stores.map((store) => ({ agentId: store.agentId, path: store.path })),
|
||||
};
|
||||
}
|
||||
|
||||
function toJsonSessionRow(row: SessionRow): Omit<SessionRow, "runtimeLabel"> {
|
||||
const { runtimeLabel, ...jsonRow } = row;
|
||||
void runtimeLabel;
|
||||
@@ -246,7 +207,6 @@ export async function sessionsCommand(
|
||||
agent?: string;
|
||||
allAgents?: boolean;
|
||||
limit?: string | number;
|
||||
exportStore?: string;
|
||||
},
|
||||
runtime: RuntimeEnv,
|
||||
) {
|
||||
@@ -271,23 +231,6 @@ export async function sessionsCommand(
|
||||
return;
|
||||
}
|
||||
|
||||
if (opts.exportStore) {
|
||||
const exported = await exportRawSessionStores({
|
||||
targets,
|
||||
outputPath: opts.exportStore,
|
||||
});
|
||||
if (opts.json) {
|
||||
writeRuntimeJson(runtime, {
|
||||
exportedPath: exported.outputPath,
|
||||
count: exported.count,
|
||||
stores: exported.stores,
|
||||
});
|
||||
} else {
|
||||
runtime.log(info(`Exported ${exported.count} session(s) to ${exported.outputPath}`));
|
||||
}
|
||||
return;
|
||||
}
|
||||
|
||||
let activeMinutes: number | undefined;
|
||||
if (opts.active !== undefined) {
|
||||
const parsed = Number.parseInt(opts.active, 10);
|
||||
|
||||
@@ -6,7 +6,6 @@ import { writeTextAtomic } from "../../infra/json-files.js";
|
||||
import { closeOpenClawStateDatabaseForTest } from "../../state/openclaw-state-db.js";
|
||||
import { resolveStorePath } from "./paths.js";
|
||||
import {
|
||||
exportSqliteSessionStore,
|
||||
importJsonSessionStoreToSqlite,
|
||||
loadSqliteSessionStore,
|
||||
saveSqliteSessionStore,
|
||||
@@ -15,7 +14,6 @@ import { loadSessionStore } from "./store-load.js";
|
||||
import { saveSessionStore, updateSessionStore } from "./store.js";
|
||||
import type { SessionEntry } from "./types.js";
|
||||
|
||||
const ORIGINAL_SESSION_STORE_BACKEND = process.env.OPENCLAW_SESSION_STORE_BACKEND;
|
||||
const ORIGINAL_STATE_DIR = process.env.OPENCLAW_STATE_DIR;
|
||||
|
||||
function createTempDir(): string {
|
||||
@@ -24,11 +22,6 @@ function createTempDir(): string {
|
||||
|
||||
afterEach(() => {
|
||||
closeOpenClawStateDatabaseForTest();
|
||||
if (ORIGINAL_SESSION_STORE_BACKEND === undefined) {
|
||||
delete process.env.OPENCLAW_SESSION_STORE_BACKEND;
|
||||
} else {
|
||||
process.env.OPENCLAW_SESSION_STORE_BACKEND = ORIGINAL_SESSION_STORE_BACKEND;
|
||||
}
|
||||
if (ORIGINAL_STATE_DIR === undefined) {
|
||||
delete process.env.OPENCLAW_STATE_DIR;
|
||||
} else {
|
||||
@@ -68,7 +61,7 @@ describe("SQLite session store backend", () => {
|
||||
});
|
||||
});
|
||||
|
||||
it("imports legacy sessions.json and exports the SQLite snapshot", async () => {
|
||||
it("imports legacy sessions.json into SQLite and removes the JSON source", async () => {
|
||||
const stateDir = createTempDir();
|
||||
const dbPath = path.join(stateDir, "state", "openclaw.sqlite");
|
||||
const legacyStorePath = path.join(stateDir, "sessions.json");
|
||||
@@ -96,9 +89,10 @@ describe("SQLite session store backend", () => {
|
||||
sourcePath: legacyStorePath,
|
||||
dbPath,
|
||||
}),
|
||||
).toEqual({ imported: 1, sourcePath: legacyStorePath });
|
||||
).toEqual({ imported: 1, sourcePath: legacyStorePath, removedSource: true });
|
||||
|
||||
expect(exportSqliteSessionStore({ agentId: "main", path: dbPath })).toEqual({
|
||||
expect(fs.existsSync(legacyStorePath)).toBe(false);
|
||||
expect(loadSqliteSessionStore({ agentId: "main", path: dbPath })).toEqual({
|
||||
"discord:user-1": {
|
||||
...entry,
|
||||
deliveryContext: {
|
||||
@@ -109,10 +103,9 @@ describe("SQLite session store backend", () => {
|
||||
});
|
||||
});
|
||||
|
||||
it("routes the production session store API through SQLite behind the opt-in flag", async () => {
|
||||
it("routes the production session store API through SQLite", async () => {
|
||||
const stateDir = createTempDir();
|
||||
process.env.OPENCLAW_STATE_DIR = stateDir;
|
||||
process.env.OPENCLAW_SESSION_STORE_BACKEND = "sqlite";
|
||||
const storePath = resolveStorePath(undefined, { agentId: "ops", env: process.env });
|
||||
const entry: SessionEntry = {
|
||||
sessionId: "sqlite-primary",
|
||||
@@ -146,7 +139,6 @@ describe("SQLite session store backend", () => {
|
||||
it("uses SQLite by default for canonical per-agent session stores", async () => {
|
||||
const stateDir = createTempDir();
|
||||
process.env.OPENCLAW_STATE_DIR = stateDir;
|
||||
delete process.env.OPENCLAW_SESSION_STORE_BACKEND;
|
||||
const storePath = resolveStorePath(undefined, { agentId: "ops", env: process.env });
|
||||
const entry: SessionEntry = {
|
||||
sessionId: "sqlite-default",
|
||||
@@ -162,29 +154,9 @@ describe("SQLite session store backend", () => {
|
||||
});
|
||||
});
|
||||
|
||||
it("keeps canonical per-agent session stores file-backed when the JSON backend is forced", async () => {
|
||||
it("does not import a legacy canonical sessions.json on first SQLite open", async () => {
|
||||
const stateDir = createTempDir();
|
||||
process.env.OPENCLAW_STATE_DIR = stateDir;
|
||||
process.env.OPENCLAW_SESSION_STORE_BACKEND = "json";
|
||||
const storePath = resolveStorePath(undefined, { agentId: "ops", env: process.env });
|
||||
const entry: SessionEntry = {
|
||||
sessionId: "json-forced",
|
||||
sessionFile: "json-forced.jsonl",
|
||||
updatedAt: 100,
|
||||
};
|
||||
|
||||
await saveSessionStore(storePath, { "discord:ops": entry }, { skipMaintenance: true });
|
||||
|
||||
expect(fs.existsSync(storePath)).toBe(true);
|
||||
expect(loadSessionStore(storePath, { skipCache: true })).toEqual({
|
||||
"discord:ops": entry,
|
||||
});
|
||||
});
|
||||
|
||||
it("imports a legacy canonical sessions.json once on first SQLite open", async () => {
|
||||
const stateDir = createTempDir();
|
||||
process.env.OPENCLAW_STATE_DIR = stateDir;
|
||||
process.env.OPENCLAW_SESSION_STORE_BACKEND = "sqlite";
|
||||
const storePath = resolveStorePath(undefined, { agentId: "ops", env: process.env });
|
||||
const legacyEntry: SessionEntry = {
|
||||
sessionId: "legacy-session",
|
||||
@@ -202,9 +174,7 @@ describe("SQLite session store backend", () => {
|
||||
),
|
||||
);
|
||||
|
||||
expect(loadSessionStore(storePath, { skipCache: true })).toEqual({
|
||||
"discord:ops": legacyEntry,
|
||||
});
|
||||
expect(loadSessionStore(storePath, { skipCache: true })).toEqual({});
|
||||
|
||||
await saveSessionStore(
|
||||
storePath,
|
||||
@@ -217,21 +187,6 @@ describe("SQLite session store backend", () => {
|
||||
},
|
||||
{ skipMaintenance: true },
|
||||
);
|
||||
await writeTextAtomic(
|
||||
storePath,
|
||||
JSON.stringify(
|
||||
{
|
||||
"discord:ops": {
|
||||
...legacyEntry,
|
||||
sessionId: "stale-json-session",
|
||||
updatedAt: 300,
|
||||
},
|
||||
},
|
||||
null,
|
||||
2,
|
||||
),
|
||||
);
|
||||
|
||||
expect(loadSessionStore(storePath, { skipCache: true })).toEqual({
|
||||
"discord:ops": {
|
||||
...legacyEntry,
|
||||
|
||||
@@ -7,7 +7,7 @@ import {
|
||||
runOpenClawStateWriteTransaction,
|
||||
type OpenClawStateDatabaseOptions,
|
||||
} from "../../state/openclaw-state-db.js";
|
||||
import { resolveAgentIdFromSessionStorePath, resolveStorePath } from "./paths.js";
|
||||
import { resolveAgentIdFromSessionStorePath } from "./paths.js";
|
||||
import { applySessionStoreMigrations } from "./store-migrations.js";
|
||||
import { normalizeSessionStore } from "./store-normalize.js";
|
||||
import type { SessionEntry } from "./types.js";
|
||||
@@ -21,6 +21,7 @@ export type SqliteSessionStoreOptions = OpenClawStateDatabaseOptions & {
|
||||
export type SessionStoreBackendImportResult = {
|
||||
imported: number;
|
||||
sourcePath: string;
|
||||
removedSource: boolean;
|
||||
};
|
||||
|
||||
type SessionEntryRow = {
|
||||
@@ -28,51 +29,21 @@ type SessionEntryRow = {
|
||||
entry_json: string;
|
||||
};
|
||||
|
||||
export type SqliteSessionStoreBackendMode = "auto" | "json" | "sqlite";
|
||||
|
||||
export function resolveSqliteSessionStoreBackendMode(
|
||||
env: NodeJS.ProcessEnv = process.env,
|
||||
): SqliteSessionStoreBackendMode {
|
||||
const raw = env.OPENCLAW_SESSION_STORE_BACKEND?.trim().toLowerCase();
|
||||
if (raw === "json" || raw === "file" || raw === "files" || raw === "disabled") {
|
||||
return "json";
|
||||
}
|
||||
if (raw === "sqlite") {
|
||||
return "sqlite";
|
||||
}
|
||||
return "auto";
|
||||
}
|
||||
|
||||
export function isSqliteSessionStoreBackendEnabled(env: NodeJS.ProcessEnv = process.env): boolean {
|
||||
return resolveSqliteSessionStoreBackendMode(env) !== "json";
|
||||
}
|
||||
|
||||
function isCanonicalAgentSessionStorePath(params: {
|
||||
storePath: string;
|
||||
agentId: string;
|
||||
env: NodeJS.ProcessEnv;
|
||||
}): boolean {
|
||||
return (
|
||||
path.resolve(params.storePath) ===
|
||||
path.resolve(resolveStorePath(undefined, { agentId: params.agentId, env: params.env }))
|
||||
);
|
||||
export function isSqliteSessionStoreBackendEnabled(_env: NodeJS.ProcessEnv = process.env): boolean {
|
||||
return true;
|
||||
}
|
||||
|
||||
export function resolveSqliteSessionStoreOptionsForPath(
|
||||
storePath: string,
|
||||
env: NodeJS.ProcessEnv = process.env,
|
||||
): SqliteSessionStoreOptions | null {
|
||||
const mode = resolveSqliteSessionStoreBackendMode(env);
|
||||
if (mode === "json") {
|
||||
if (!isSqliteSessionStoreBackendEnabled(env)) {
|
||||
return null;
|
||||
}
|
||||
const agentId = resolveAgentIdFromSessionStorePath(storePath);
|
||||
if (!agentId) {
|
||||
return null;
|
||||
}
|
||||
if (mode === "auto" && !isCanonicalAgentSessionStorePath({ storePath, agentId, env })) {
|
||||
return null;
|
||||
}
|
||||
return { agentId, env, sourcePath: storePath };
|
||||
}
|
||||
|
||||
@@ -110,63 +81,6 @@ function prepareReplaceStatement(statement: StatementSync, params: Record<string
|
||||
statement.run(params);
|
||||
}
|
||||
|
||||
function resolveImportMarkerKey(options: SqliteSessionStoreOptions): string | null {
|
||||
const sourcePath = options.sourcePath?.trim();
|
||||
return sourcePath ? `${options.agentId}:${path.resolve(sourcePath)}` : null;
|
||||
}
|
||||
|
||||
function readImportMarker(
|
||||
database: OpenClawStateDatabase,
|
||||
options: SqliteSessionStoreOptions,
|
||||
): string | null {
|
||||
const markerKey = resolveImportMarkerKey(options);
|
||||
if (!markerKey) {
|
||||
return null;
|
||||
}
|
||||
const row = database.db
|
||||
.prepare("SELECT value_json FROM kv WHERE scope = ? AND key = ?")
|
||||
.get("session-store-import", markerKey) as { value_json?: unknown } | undefined;
|
||||
return typeof row?.value_json === "string" ? row.value_json : null;
|
||||
}
|
||||
|
||||
function writeImportMarker(params: {
|
||||
database: OpenClawStateDatabase;
|
||||
options: SqliteSessionStoreOptions;
|
||||
imported: number;
|
||||
sourceExists: boolean;
|
||||
}): void {
|
||||
const markerKey = resolveImportMarkerKey(params.options);
|
||||
if (!markerKey) {
|
||||
return;
|
||||
}
|
||||
params.database.db
|
||||
.prepare(
|
||||
`
|
||||
INSERT OR REPLACE INTO kv (
|
||||
scope,
|
||||
key,
|
||||
value_json,
|
||||
updated_at
|
||||
) VALUES (
|
||||
@scope,
|
||||
@key,
|
||||
@value_json,
|
||||
@updated_at
|
||||
)
|
||||
`,
|
||||
)
|
||||
.run({
|
||||
scope: "session-store-import",
|
||||
key: markerKey,
|
||||
value_json: JSON.stringify({
|
||||
imported: params.imported,
|
||||
sourceExists: params.sourceExists,
|
||||
sourcePath: path.resolve(params.options.sourcePath ?? ""),
|
||||
}),
|
||||
updated_at: resolveNow(params.options),
|
||||
});
|
||||
}
|
||||
|
||||
function countSqliteSessionEntries(
|
||||
database: OpenClawStateDatabase,
|
||||
options: SqliteSessionStoreOptions,
|
||||
@@ -178,6 +92,11 @@ function countSqliteSessionEntries(
|
||||
return typeof count === "bigint" ? Number(count) : count;
|
||||
}
|
||||
|
||||
export function countSqliteSessionStoreEntries(options: SqliteSessionStoreOptions): number {
|
||||
const database = openOpenClawStateDatabase(options);
|
||||
return countSqliteSessionEntries(database, options);
|
||||
}
|
||||
|
||||
function parseJsonSessionStoreFromPath(sourcePath: string): Record<string, SessionEntry> {
|
||||
let store: Record<string, SessionEntry> = {};
|
||||
try {
|
||||
@@ -228,35 +147,9 @@ function replaceSqliteSessionStore(params: {
|
||||
}
|
||||
}
|
||||
|
||||
function importLegacyJsonSessionStoreIfNeeded(options: SqliteSessionStoreOptions): void {
|
||||
const sourcePath = options.sourcePath?.trim();
|
||||
if (!sourcePath || !fs.existsSync(sourcePath)) {
|
||||
return;
|
||||
}
|
||||
runOpenClawStateWriteTransaction((database) => {
|
||||
if (readImportMarker(database, options)) {
|
||||
return;
|
||||
}
|
||||
const existingRows = countSqliteSessionEntries(database, options);
|
||||
if (existingRows > 0) {
|
||||
writeImportMarker({ database, options, imported: 0, sourceExists: true });
|
||||
return;
|
||||
}
|
||||
const store = parseJsonSessionStoreFromPath(sourcePath);
|
||||
replaceSqliteSessionStore({ database, options, store });
|
||||
writeImportMarker({
|
||||
database,
|
||||
options,
|
||||
imported: Object.keys(store).length,
|
||||
sourceExists: true,
|
||||
});
|
||||
}, options);
|
||||
}
|
||||
|
||||
export function loadSqliteSessionStore(
|
||||
options: SqliteSessionStoreOptions,
|
||||
): Record<string, SessionEntry> {
|
||||
importLegacyJsonSessionStoreIfNeeded(options);
|
||||
const database = openOpenClawStateDatabase(options);
|
||||
const rows = database.db
|
||||
.prepare(
|
||||
@@ -286,38 +179,59 @@ export function saveSqliteSessionStore(
|
||||
normalizeSessionStore(store);
|
||||
runOpenClawStateWriteTransaction((database) => {
|
||||
replaceSqliteSessionStore({ database, options, store });
|
||||
writeImportMarker({
|
||||
database,
|
||||
options,
|
||||
imported: Object.keys(store).length,
|
||||
sourceExists: Boolean(options.sourcePath && fs.existsSync(options.sourcePath)),
|
||||
});
|
||||
}, options);
|
||||
}
|
||||
|
||||
function resolveSessionEntryUpdatedAt(entry: SessionEntry): number {
|
||||
return typeof entry.updatedAt === "number" && Number.isFinite(entry.updatedAt)
|
||||
? entry.updatedAt
|
||||
: 0;
|
||||
}
|
||||
|
||||
function mergeSessionStoresByUpdatedAt(
|
||||
existing: Record<string, SessionEntry>,
|
||||
incoming: Record<string, SessionEntry>,
|
||||
): Record<string, SessionEntry> {
|
||||
const merged: Record<string, SessionEntry> = { ...existing };
|
||||
for (const [key, entry] of Object.entries(incoming)) {
|
||||
const current = merged[key];
|
||||
if (!current || resolveSessionEntryUpdatedAt(entry) >= resolveSessionEntryUpdatedAt(current)) {
|
||||
merged[key] = entry;
|
||||
}
|
||||
}
|
||||
normalizeSessionStore(merged);
|
||||
return merged;
|
||||
}
|
||||
|
||||
export function importJsonSessionStoreToSqlite(params: {
|
||||
agentId: string;
|
||||
sourcePath: string;
|
||||
dbPath?: string;
|
||||
env?: NodeJS.ProcessEnv;
|
||||
now?: () => number;
|
||||
}): SessionStoreBackendImportResult {
|
||||
const store = parseJsonSessionStoreFromPath(params.sourcePath);
|
||||
saveSqliteSessionStore(
|
||||
{
|
||||
agentId: params.agentId,
|
||||
sourcePath: params.sourcePath,
|
||||
...(params.dbPath ? { path: params.dbPath } : {}),
|
||||
...(params.now ? { now: params.now } : {}),
|
||||
},
|
||||
store,
|
||||
);
|
||||
return { imported: Object.keys(store).length, sourcePath: params.sourcePath };
|
||||
}
|
||||
|
||||
export function exportSqliteSessionStore(
|
||||
options: SqliteSessionStoreOptions,
|
||||
): Record<string, SessionEntry> {
|
||||
return loadSqliteSessionStore(options);
|
||||
const options = {
|
||||
agentId: params.agentId,
|
||||
sourcePath: params.sourcePath,
|
||||
...(params.env ? { env: params.env } : {}),
|
||||
...(params.dbPath ? { path: params.dbPath } : {}),
|
||||
...(params.now ? { now: params.now } : {}),
|
||||
};
|
||||
const importedStore = parseJsonSessionStoreFromPath(params.sourcePath);
|
||||
const mergedStore = mergeSessionStoresByUpdatedAt(loadSqliteSessionStore(options), importedStore);
|
||||
saveSqliteSessionStore(options, mergedStore);
|
||||
let removedSource = false;
|
||||
try {
|
||||
fs.rmSync(params.sourcePath, { force: true });
|
||||
removedSource = true;
|
||||
} catch {
|
||||
removedSource = false;
|
||||
}
|
||||
return {
|
||||
imported: Object.keys(importedStore).length,
|
||||
sourcePath: params.sourcePath,
|
||||
removedSource,
|
||||
};
|
||||
}
|
||||
|
||||
export function readJsonSessionStoreRawForImport(pathname: string): string {
|
||||
|
||||
@@ -16,6 +16,7 @@ import {
|
||||
import type { SessionEntry } from "../config/sessions.js";
|
||||
import { saveSessionStore } from "../config/sessions.js";
|
||||
import { canonicalizeMainSessionAlias } from "../config/sessions/main-session.js";
|
||||
import { importJsonSessionStoreToSqlite } from "../config/sessions/store-backend.sqlite.js";
|
||||
import type { SessionScope } from "../config/sessions/types.js";
|
||||
import type { OpenClawConfig } from "../config/types.openclaw.js";
|
||||
import { createSubsystemLogger } from "../logging/subsystem.js";
|
||||
@@ -46,6 +47,7 @@ export type LegacyStateDetection = {
|
||||
targetAgentId: string;
|
||||
targetMainKey: string;
|
||||
targetScope?: SessionScope;
|
||||
env: NodeJS.ProcessEnv;
|
||||
stateDir: string;
|
||||
oauthDir: string;
|
||||
sessions: {
|
||||
@@ -706,6 +708,7 @@ export async function detectLegacyStateMigrations(params: {
|
||||
const sessionsLegacyStorePath = path.join(sessionsLegacyDir, "sessions.json");
|
||||
const sessionsTargetDir = path.join(stateDir, "agents", targetAgentId, "sessions");
|
||||
const sessionsTargetStorePath = path.join(sessionsTargetDir, "sessions.json");
|
||||
const hasTargetJsonSessionStore = fileExists(sessionsTargetStorePath);
|
||||
const legacySessionEntries = safeReadDir(sessionsLegacyDir);
|
||||
const hasLegacySessions =
|
||||
fileExists(sessionsLegacyStorePath) ||
|
||||
@@ -740,6 +743,9 @@ export async function detectLegacyStateMigrations(params: {
|
||||
if (legacyKeys.length > 0) {
|
||||
preview.push(`- Sessions: canonicalize legacy keys in ${sessionsTargetStorePath}`);
|
||||
}
|
||||
if (hasTargetJsonSessionStore) {
|
||||
preview.push(`- Sessions: import ${sessionsTargetStorePath} into SQLite`);
|
||||
}
|
||||
if (hasLegacyAgentDir) {
|
||||
preview.push(`- Agent dir: ${legacyAgentDir} → ${targetAgentDir}`);
|
||||
}
|
||||
@@ -751,6 +757,7 @@ export async function detectLegacyStateMigrations(params: {
|
||||
targetAgentId,
|
||||
targetMainKey,
|
||||
targetScope,
|
||||
env,
|
||||
stateDir,
|
||||
oauthDir,
|
||||
sessions: {
|
||||
@@ -758,7 +765,7 @@ export async function detectLegacyStateMigrations(params: {
|
||||
legacyStorePath: sessionsLegacyStorePath,
|
||||
targetDir: sessionsTargetDir,
|
||||
targetStorePath: sessionsTargetStorePath,
|
||||
hasLegacy: hasLegacySessions || legacyKeys.length > 0,
|
||||
hasLegacy: hasLegacySessions || legacyKeys.length > 0 || hasTargetJsonSessionStore,
|
||||
legacyKeys,
|
||||
},
|
||||
agentDir: {
|
||||
@@ -792,6 +799,7 @@ async function migrateLegacySessions(
|
||||
const targetParsed = fileExists(detected.sessions.targetStorePath)
|
||||
? readSessionStoreJson5(detected.sessions.targetStorePath)
|
||||
: { store: {}, ok: true };
|
||||
const hasTargetSessionStoreFile = fileExists(detected.sessions.targetStorePath);
|
||||
const legacyStore = legacyParsed.store;
|
||||
const targetStore = targetParsed.store;
|
||||
|
||||
@@ -837,7 +845,9 @@ async function migrateLegacySessions(
|
||||
|
||||
if (
|
||||
(legacyParsed.ok || targetParsed.ok) &&
|
||||
(Object.keys(legacyStore).length > 0 || Object.keys(targetStore).length > 0)
|
||||
(Object.keys(legacyStore).length > 0 ||
|
||||
Object.keys(targetStore).length > 0 ||
|
||||
(hasTargetSessionStoreFile && targetParsed.ok))
|
||||
) {
|
||||
const normalized: Record<string, SessionEntry> = {};
|
||||
for (const [key, entry] of Object.entries(merged)) {
|
||||
@@ -851,6 +861,16 @@ async function migrateLegacySessions(
|
||||
skipMaintenance: true,
|
||||
});
|
||||
changes.push(`Merged sessions store → ${detected.sessions.targetStorePath}`);
|
||||
if (fileExists(detected.sessions.targetStorePath)) {
|
||||
const imported = importJsonSessionStoreToSqlite({
|
||||
agentId: detected.targetAgentId,
|
||||
env: detected.env,
|
||||
sourcePath: detected.sessions.targetStorePath,
|
||||
});
|
||||
changes.push(
|
||||
`Imported ${imported.imported} session index row(s) into SQLite for agent ${detected.targetAgentId}`,
|
||||
);
|
||||
}
|
||||
if (canonicalizedTarget.legacyKeys.length > 0) {
|
||||
changes.push(`Canonicalized ${canonicalizedTarget.legacyKeys.length} legacy session key(s)`);
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user