refactor: keep session import in doctor

This commit is contained in:
Peter Steinberger
2026-05-06 18:04:35 +01:00
parent 208d84df1a
commit 604a423314
9 changed files with 127 additions and 96 deletions

View File

@@ -136,7 +136,8 @@ Canonical session metadata lives in the shared state database:
Legacy `sessions.json` indexes are imported by `openclaw doctor --fix` and
removed after SQLite has the rows. Custom offline repair stores can still use an
explicit `--store` path, but per-agent runtime metadata should go through the
shared state database.
shared state database. Startup does not import or rewrite legacy session
indexes.
Gateway and ACP session discovery also scans disk-backed agent stores under the
default `agents/` root and under templated `session.store` roots. Discovered

View File

@@ -47,8 +47,10 @@ Scope selection:
- `--store <path>`: explicit store path (cannot be combined with `--agent` or `--all-agents`)
Canonical per-agent session stores use OpenClaw's shared SQLite state database by
default. Existing `sessions.json` indexes are imported by `openclaw doctor
--fix`, then removed after SQLite has the rows.
default. Existing `sessions.json` indexes are imported by the `openclaw doctor`
fix mode, then removed after SQLite has the rows. Gateway startup does not
import or rewrite legacy session indexes; run doctor when you intentionally want
that migration.
- `--limit <n|all>`: max rows to output (default `100`; `all` restores full output)

View File

@@ -46,9 +46,10 @@ This plan has started landing in slices:
The shared `kv` table now has a small typed helper for scoped JSON-compatible
values so low-risk JSON sidecars can move behind the same SQLite connection
without each feature reimplementing read/write/delete glue.
- Canonical per-agent session stores use SQLite by default. `openclaw doctor
--fix` imports legacy `sessions.json` indexes into SQLite and removes the JSON
index after import, instead of keeping a parallel compatibility/export store.
- Canonical per-agent session stores use SQLite by default. The `openclaw doctor`
fix mode imports legacy `sessions.json` indexes into SQLite and removes the
JSON index after import, instead of keeping a startup migration or parallel
compatibility/export store.
- Transcript events have a SQLite store primitive with JSONL import/export.
Transcript append paths dual-write when the caller already has agent and
session scope, including gateway-injected assistant messages. Scoped appends
@@ -343,8 +344,8 @@ Migration order:
2. Add shared SQLite connection and migration helpers.
3. Move `sessions.json` behind a `SessionStoreBackend` interface.
4. Make SQLite primary for session entries.
5. Import old `sessions.json` from `openclaw doctor --fix`, then remove the JSON
index after SQLite has the rows.
5. Import old `sessions.json` only from `openclaw doctor --fix`, then remove the
JSON index after SQLite has the rows.
6. Leave `*.jsonl` transcripts on disk while PI owns transcript semantics.
7. After session manager ownership moves behind OpenClaw APIs, store transcript
events in SQLite and export JSONL for compatibility.

View File

@@ -76,7 +76,8 @@ Per agent, on the Gateway host:
- Store: `~/.openclaw/state/openclaw.sqlite` by default. `openclaw doctor --fix`
imports legacy `~/.openclaw/agents/<agentId>/sessions/sessions.json` indexes
into SQLite and removes the JSON index after import.
into SQLite and removes the JSON index after import; Gateway startup leaves
legacy indexes alone.
- Transcripts: `~/.openclaw/agents/<agentId>/sessions/<sessionId>.jsonl`
- Telegram topic sessions: `.../<sessionId>-topic-<threadId>.jsonl`
@@ -95,7 +96,7 @@ Session persistence has automatic maintenance controls (`session.maintenance`) f
- `maxDiskBytes`: optional sessions-directory budget
- `highWaterBytes`: optional target after cleanup (default `80%` of `maxDiskBytes`)
Normal Gateway writes flow through a per-store session writer that serializes in-process mutations. SQLite is the canonical per-agent backend; `sessions.json` is a legacy doctor-import input, not a parallel export/debug store. Runtime code should prefer `updateSessionStore(...)` or `updateSessionStoreEntry(...)`. When a Gateway is reachable, non-dry-run `openclaw sessions cleanup` and `openclaw agents delete` delegate store mutations to the Gateway so cleanup joins the same writer queue. `maxEntries` cleanup is still batched for production-sized caps, so a store may briefly exceed the configured cap before the next high-water cleanup rewrites it back down. Session store reads do not prune or cap entries during Gateway startup; use writes or `openclaw sessions cleanup --enforce` for cleanup. `openclaw sessions cleanup --enforce` still applies the configured cap immediately and prunes old unreferenced transcript, checkpoint, and trajectory artifacts even when no disk budget is configured.
Normal Gateway writes flow through a per-store session writer that serializes in-process mutations. SQLite is the canonical per-agent backend; `sessions.json` is a legacy doctor-import input, not a parallel export/debug store. Runtime code should prefer `updateSessionStore(...)` or `updateSessionStoreEntry(...)`. When a Gateway is reachable, non-dry-run `openclaw sessions cleanup` and `openclaw agents delete` delegate store mutations to the Gateway so cleanup joins the same writer queue. `maxEntries` cleanup is still batched for production-sized caps, so a store may briefly exceed the configured cap before the next high-water cleanup rewrites it back down. Session store reads do not import, prune, or cap entries during Gateway startup; use `openclaw doctor --fix` for legacy JSON import and writes or `openclaw sessions cleanup --enforce` for cleanup. `openclaw sessions cleanup --enforce` still applies the configured cap immediately and prunes old unreferenced transcript, checkpoint, and trajectory artifacts even when no disk budget is configured.
Maintenance keeps durable external conversation pointers such as group sessions
and thread-scoped chat sessions, but synthetic runtime entries for cron, hooks,

View File

@@ -455,7 +455,7 @@ describe("doctor legacy state migrations", () => {
expect(log.info).toHaveBeenCalled();
});
it("auto-migrates legacy sessions on startup", async () => {
it("leaves legacy sessions for doctor on startup", async () => {
const { root, cfg } = await makeRootWithEmptyCfg();
const legacySessionsDir = writeLegacySessionsFixture({
root,
@@ -473,14 +473,14 @@ describe("doctor legacy state migrations", () => {
now: () => 123,
});
expect(result.migrated).toBe(true);
expect(log.info).toHaveBeenCalled();
expect(result.migrated).toBe(false);
expect(log.info).not.toHaveBeenCalled();
const targetDir = path.join(root, "agents", "main", "sessions");
expect(fs.existsSync(path.join(targetDir, "a.jsonl"))).toBe(true);
expect(fs.existsSync(path.join(legacySessionsDir, "a.jsonl"))).toBe(false);
expect(fs.existsSync(path.join(targetDir, "sessions.json"))).toBe(false);
expect(readSessionsStore({ root, targetDir })["agent:main:+1555"]?.sessionId).toBe("a");
expect(fs.existsSync(path.join(targetDir, "a.jsonl"))).toBe(false);
expect(fs.existsSync(path.join(legacySessionsDir, "a.jsonl"))).toBe(true);
expect(fs.existsSync(path.join(legacySessionsDir, "sessions.json"))).toBe(true);
expect(Object.keys(readSessionsStore({ root, targetDir }))).toHaveLength(0);
});
it("migrates legacy WhatsApp auth files without touching oauth.json", async () => {
@@ -694,7 +694,7 @@ describe("doctor legacy state migrations", () => {
expect(store["agent:main:slack:channel:C123"]).toBeUndefined();
});
it("auto-migrates when only target sessions contain legacy keys", async () => {
it("leaves target sessions with legacy keys for doctor on startup", async () => {
const { root, cfg } = await makeRootWithEmptyCfg();
const targetDir = path.join(root, "agents", "main", "sessions");
writeJson5(path.join(targetDir, "sessions.json"), {
@@ -703,12 +703,10 @@ describe("doctor legacy state migrations", () => {
const { result, log } = await runAutoMigrateLegacyStateWithLog({ root, cfg });
expect(fs.existsSync(path.join(targetDir, "sessions.json"))).toBe(false);
const store = readSessionsStore({ root, targetDir });
expect(result.migrated).toBe(true);
expect(log.info).toHaveBeenCalled();
expect(store["main"]).toBeUndefined();
expect(store["agent:main:main"]?.sessionId).toBe("legacy");
expect(result.migrated).toBe(false);
expect(log.info).not.toHaveBeenCalled();
expect(fs.existsSync(path.join(targetDir, "sessions.json"))).toBe(true);
expect(Object.keys(readSessionsStore({ root, targetDir }))).toHaveLength(0);
});
it("does nothing when no legacy state dir exists", async () => {

View File

@@ -191,6 +191,7 @@ function createLegacyStateMigrationDetectionResult(params?: {
targetAgentId: "main",
targetMainKey: "main",
targetScope: undefined,
cfg: {},
env: process.env,
stateDir: "/tmp/state",
oauthDir: "/tmp/oauth",

View File

@@ -182,6 +182,18 @@ export function saveSqliteSessionStore(
}, options);
}
export function mergeSqliteSessionStore(
options: SqliteSessionStoreOptions,
incoming: Record<string, SessionEntry>,
): { imported: number; stored: number } {
const mergedStore = mergeSessionStoresByUpdatedAt(loadSqliteSessionStore(options), incoming);
saveSqliteSessionStore(options, mergedStore);
return {
imported: Object.keys(incoming).length,
stored: Object.keys(mergedStore).length,
};
}
function resolveSessionEntryUpdatedAt(entry: SessionEntry): number {
return typeof entry.updatedAt === "number" && Number.isFinite(entry.updatedAt)
? entry.updatedAt
@@ -218,8 +230,7 @@ export function importJsonSessionStoreToSqlite(params: {
...(params.now ? { now: params.now } : {}),
};
const importedStore = parseJsonSessionStoreFromPath(params.sourcePath);
const mergedStore = mergeSessionStoresByUpdatedAt(loadSqliteSessionStore(options), importedStore);
saveSqliteSessionStore(options, mergedStore);
const result = mergeSqliteSessionStore(options, importedStore);
let removedSource = false;
try {
fs.rmSync(params.sourcePath, { force: true });
@@ -228,7 +239,7 @@ export function importJsonSessionStoreToSqlite(params: {
removedSource = false;
}
return {
imported: Object.keys(importedStore).length,
imported: result.imported,
sourcePath: params.sourcePath,
removedSource,
};

View File

@@ -3,7 +3,9 @@ import fs from "node:fs/promises";
import path from "node:path";
import { afterEach, describe, expect, it, vi } from "vitest";
import type { OpenClawConfig } from "../config/config.js";
import { loadSqliteSessionStore } from "../config/sessions/store-backend.sqlite.js";
import { resolveChannelAllowFromPath } from "../pairing/pairing-store.js";
import { closeOpenClawStateDatabaseForTest } from "../state/openclaw-state-db.js";
import { createTrackedTempDirs } from "../test-utils/tracked-temp-dirs.js";
import { detectLegacyStateMigrations, runLegacyStateMigrations } from "./state-migrations.js";
@@ -157,6 +159,7 @@ async function createLegacyStateFixture(params?: { includePreKey?: boolean }) {
}
afterEach(async () => {
closeOpenClawStateDatabaseForTest();
await tempDirs.cleanup();
});
@@ -183,6 +186,7 @@ describe("state migrations", () => {
expect(detected.preview).toEqual([
`- Sessions: ${path.join(stateDir, "sessions")}${path.join(stateDir, "agents", "worker-1", "sessions")}`,
`- Sessions: canonicalize legacy keys in ${path.join(stateDir, "agents", "worker-1", "sessions", "sessions.json")}`,
`- Sessions: import ${path.join(stateDir, "agents", "worker-1", "sessions", "sessions.json")} into SQLite`,
`- Agent dir: ${path.join(stateDir, "agent")}${path.join(stateDir, "agents", "worker-1", "agent")}`,
`- MobileAuth auth creds.json: ${path.join(stateDir, "credentials", "creds.json")}${path.join(stateDir, "credentials", "mobileauth", "default", "creds.json")}`,
`- ChatApp pairing allowFrom: ${resolveChannelAllowFromPath("chatapp", env)}${resolveChannelAllowFromPath("chatapp", env, "alpha")}`,
@@ -204,9 +208,9 @@ describe("state migrations", () => {
expect(result.warnings).toStrictEqual([]);
expect(result.changes).toEqual([
`Canonicalized 2 orphaned session key(s) in ${path.join(stateDir, "agents", "worker-1", "sessions", "sessions.json")}`,
`Migrated latest direct-chat session → agent:worker-1:desk`,
`Merged sessions store → ${path.join(stateDir, "agents", "worker-1", "sessions", "sessions.json")}`,
"Canonicalized 2 legacy session key(s)",
"Imported 4 session index row(s) into SQLite for agent worker-1",
"Moved trace.jsonl → agents/worker-1/sessions",
"Moved agent file settings.json → agents/worker-1/agent",
`Moved MobileAuth auth creds.json → ${path.join(stateDir, "credentials", "mobileauth", "default", "creds.json")}`,
@@ -214,12 +218,13 @@ describe("state migrations", () => {
`Copied ChatApp pairing allowFrom → ${resolveChannelAllowFromPath("chatapp", env, "alpha")}`,
]);
const mergedStore = JSON.parse(
await fs.readFile(
path.join(stateDir, "agents", "worker-1", "sessions", "sessions.json"),
"utf8",
),
) as Record<string, { sessionId: string }>;
await expect(
fs.stat(path.join(stateDir, "agents", "worker-1", "sessions", "sessions.json")),
).rejects.toMatchObject({ code: "ENOENT" });
const mergedStore = loadSqliteSessionStore({
agentId: "worker-1",
env,
}) as Record<string, { sessionId: string }>;
expect(mergedStore["agent:worker-1:desk"]?.sessionId).toBe("legacy-direct");
expect(mergedStore["agent:worker-1:mobileauth:group:mobile-room"]?.sessionId).toBe(
"group-session",

View File

@@ -14,9 +14,8 @@ import {
resolveStateDir,
} from "../config/paths.js";
import type { SessionEntry } from "../config/sessions.js";
import { saveSessionStore } from "../config/sessions.js";
import { canonicalizeMainSessionAlias } from "../config/sessions/main-session.js";
import { importJsonSessionStoreToSqlite } from "../config/sessions/store-backend.sqlite.js";
import { mergeSqliteSessionStore } from "../config/sessions/store-backend.sqlite.js";
import type { SessionScope } from "../config/sessions/types.js";
import type { OpenClawConfig } from "../config/types.openclaw.js";
import { createSubsystemLogger } from "../logging/subsystem.js";
@@ -33,6 +32,7 @@ import {
normalizeOptionalLowercaseString,
} from "../shared/string-coerce.js";
import { expandHomePrefix } from "./home-dir.js";
import { writeTextAtomic } from "./json-files.js";
import { isWithinDir } from "./path-safety.js";
import {
ensureDir,
@@ -47,6 +47,7 @@ export type LegacyStateDetection = {
targetAgentId: string;
targetMainKey: string;
targetScope?: SessionScope;
cfg: OpenClawConfig;
env: NodeJS.ProcessEnv;
stateDir: string;
oauthDir: string;
@@ -690,9 +691,13 @@ export async function detectLegacyStateMigrations(params: {
cfg: OpenClawConfig;
env?: NodeJS.ProcessEnv;
homedir?: () => string;
includeSessions?: boolean;
includeChannelPlans?: boolean;
}): Promise<LegacyStateDetection> {
const env = params.env ?? process.env;
const homedir = params.homedir ?? os.homedir;
const includeSessions = params.includeSessions ?? true;
const includeChannelPlans = params.includeChannelPlans ?? true;
const stateDir = resolveStateDir(env, homedir);
const oauthDir = resolveOAuthDir(env, stateDir);
@@ -708,33 +713,36 @@ export async function detectLegacyStateMigrations(params: {
const sessionsLegacyStorePath = path.join(sessionsLegacyDir, "sessions.json");
const sessionsTargetDir = path.join(stateDir, "agents", targetAgentId, "sessions");
const sessionsTargetStorePath = path.join(sessionsTargetDir, "sessions.json");
const hasTargetJsonSessionStore = fileExists(sessionsTargetStorePath);
const legacySessionEntries = safeReadDir(sessionsLegacyDir);
const hasTargetJsonSessionStore = includeSessions && fileExists(sessionsTargetStorePath);
const legacySessionEntries = includeSessions ? safeReadDir(sessionsLegacyDir) : [];
const hasLegacySessions =
fileExists(sessionsLegacyStorePath) ||
(includeSessions && fileExists(sessionsLegacyStorePath)) ||
legacySessionEntries.some((e) => e.isFile() && e.name.endsWith(".jsonl"));
const targetSessionParsed = fileExists(sessionsTargetStorePath)
const targetSessionParsed = hasTargetJsonSessionStore
? readSessionStoreJson5(sessionsTargetStorePath)
: { store: {}, ok: true };
const legacyKeys = targetSessionParsed.ok
? listLegacySessionKeys({
store: targetSessionParsed.store,
agentId: targetAgentId,
mainKey: targetMainKey,
scope: targetScope,
})
: [];
const legacyKeys =
includeSessions && targetSessionParsed.ok
? listLegacySessionKeys({
store: targetSessionParsed.store,
agentId: targetAgentId,
mainKey: targetMainKey,
scope: targetScope,
})
: [];
const legacyAgentDir = path.join(stateDir, "agent");
const targetAgentDir = path.join(stateDir, "agents", targetAgentId, "agent");
const hasLegacyAgentDir = existsDir(legacyAgentDir);
const channelPlans = await collectChannelLegacyStateMigrationPlans({
cfg: params.cfg,
env,
stateDir,
oauthDir,
});
const channelPlans = includeChannelPlans
? await collectChannelLegacyStateMigrationPlans({
cfg: params.cfg,
env,
stateDir,
oauthDir,
})
: [];
const preview: string[] = [];
if (hasLegacySessions) {
@@ -757,6 +765,7 @@ export async function detectLegacyStateMigrations(params: {
targetAgentId,
targetMainKey,
targetScope,
cfg: params.cfg,
env,
stateDir,
oauthDir,
@@ -857,19 +866,23 @@ async function migrateLegacySessions(
}
normalized[key] = normalizedEntry;
}
await saveSessionStore(detected.sessions.targetStorePath, normalized, {
skipMaintenance: true,
});
changes.push(`Merged sessions store → ${detected.sessions.targetStorePath}`);
if (fileExists(detected.sessions.targetStorePath)) {
const imported = importJsonSessionStoreToSqlite({
const imported = mergeSqliteSessionStore(
{
agentId: detected.targetAgentId,
env: detected.env,
sourcePath: detected.sessions.targetStorePath,
});
changes.push(
`Imported ${imported.imported} session index row(s) into SQLite for agent ${detected.targetAgentId}`,
);
},
normalized,
);
changes.push(
`Imported ${imported.imported} session index row(s) into SQLite for agent ${detected.targetAgentId}`,
);
if (targetParsed.ok && fileExists(detected.sessions.targetStorePath)) {
try {
fs.rmSync(detected.sessions.targetStorePath, { force: true });
} catch {
// ignore
}
}
if (canonicalizedTarget.legacyKeys.length > 0) {
changes.push(`Canonicalized ${canonicalizedTarget.legacyKeys.length} legacy session key(s)`);
@@ -985,12 +998,26 @@ export async function runLegacyStateMigrations(params: {
}): Promise<{ changes: string[]; warnings: string[] }> {
const now = params.now ?? (() => Date.now());
const detected = params.detected;
const orphanKeys = await migrateOrphanedSessionKeys({
cfg: detected.cfg,
env: detected.env,
});
const sessions = await migrateLegacySessions(detected, now);
const agentDir = await migrateLegacyAgentDir(detected, now);
const channelPlans = await migrateChannelLegacyStatePlans(detected);
return {
changes: [...sessions.changes, ...agentDir.changes, ...channelPlans.changes],
warnings: [...sessions.warnings, ...agentDir.warnings, ...channelPlans.warnings],
changes: [
...orphanKeys.changes,
...sessions.changes,
...agentDir.changes,
...channelPlans.changes,
],
warnings: [
...orphanKeys.warnings,
...sessions.warnings,
...agentDir.warnings,
...channelPlans.warnings,
],
};
}
@@ -1126,7 +1153,8 @@ export async function migrateOrphanedSessionKeys(params: {
}
}
try {
await saveSessionStore(storePath, normalized, { skipMaintenance: true });
ensureDir(path.dirname(storePath));
await writeTextAtomic(storePath, `${JSON.stringify(normalized, null, 2)}\n`, { mode: 0o600 });
changes.push(`Canonicalized ${totalLegacy} orphaned session key(s) in ${storePath}`);
} catch (err) {
warnings.push(`Failed to write canonicalized store ${storePath}: ${String(err)}`);
@@ -1173,14 +1201,6 @@ export async function autoMigrateLegacyState(params: {
log: params.log,
});
// Canonicalize orphaned session keys regardless of whether legacy migration
// is needed — the orphan-key bug (#29683) affects all installs with
// non-default agent IDs or mainKey configuration.
const orphanKeys = await migrateOrphanedSessionKeys({
cfg: params.cfg,
env,
});
const logMigrationResults = (changes: string[], warnings: string[]) => {
const logger = params.log ?? createSubsystemLogger("state-migrations");
if (changes.length > 0) {
@@ -1196,11 +1216,11 @@ export async function autoMigrateLegacyState(params: {
};
if (env.OPENCLAW_AGENT_DIR?.trim() || env.PI_CODING_AGENT_DIR?.trim()) {
const changes = [...stateDirResult.changes, ...orphanKeys.changes];
const warnings = [...stateDirResult.warnings, ...orphanKeys.warnings];
const changes = [...stateDirResult.changes];
const warnings = [...stateDirResult.warnings];
logMigrationResults(changes, warnings);
return {
migrated: stateDirResult.migrated || orphanKeys.changes.length > 0,
migrated: stateDirResult.migrated,
skipped: true,
changes,
warnings,
@@ -1211,13 +1231,15 @@ export async function autoMigrateLegacyState(params: {
cfg: params.cfg,
env,
homedir: params.homedir,
includeSessions: false,
includeChannelPlans: false,
});
if (!detected.sessions.hasLegacy && !detected.agentDir.hasLegacy) {
const changes = [...stateDirResult.changes, ...orphanKeys.changes];
const warnings = [...stateDirResult.warnings, ...orphanKeys.warnings];
if (!detected.agentDir.hasLegacy) {
const changes = [...stateDirResult.changes];
const warnings = [...stateDirResult.warnings];
logMigrationResults(changes, warnings);
return {
migrated: stateDirResult.migrated || orphanKeys.changes.length > 0,
migrated: stateDirResult.migrated,
skipped: false,
changes,
warnings,
@@ -1225,20 +1247,9 @@ export async function autoMigrateLegacyState(params: {
}
const now = params.now ?? (() => Date.now());
const sessions = await migrateLegacySessions(detected, now);
const agentDir = await migrateLegacyAgentDir(detected, now);
const changes = [
...stateDirResult.changes,
...orphanKeys.changes,
...sessions.changes,
...agentDir.changes,
];
const warnings = [
...stateDirResult.warnings,
...orphanKeys.warnings,
...sessions.warnings,
...agentDir.warnings,
];
const changes = [...stateDirResult.changes, ...agentDir.changes];
const warnings = [...stateDirResult.warnings, ...agentDir.warnings];
logMigrationResults(changes, warnings);