refactor: mark memory wiki legacy importer doctor-only

This commit is contained in:
Peter Steinberger
2026-05-11 01:29:19 +01:00
parent 3d17e8ca00
commit 40cd243eb5
10 changed files with 600 additions and 12 deletions

View File

@@ -2,9 +2,9 @@ import { definePluginEntry } from "./api.js";
import { registerWikiCli } from "./src/cli.js";
import { memoryWikiConfigSchema, resolveMemoryWikiConfig } from "./src/config.js";
import { createWikiCorpusSupplement } from "./src/corpus-supplement.js";
import { createMemoryWikiSourceSyncMigrationProvider } from "./src/doctor-legacy-state.js";
import { registerMemoryWikiGatewayMethods } from "./src/gateway.js";
import { createWikiPromptSectionBuilder } from "./src/prompt-section.js";
import { createMemoryWikiSourceSyncMigrationProvider } from "./src/source-sync-migration.js";
import {
createWikiApplyTool,
createWikiGetTool,

View File

@@ -3,16 +3,16 @@ import os from "node:os";
import path from "node:path";
import { resetPluginBlobStoreForTests } from "openclaw/plugin-sdk/plugin-state-runtime";
import { afterEach, describe, expect, it } from "vitest";
import {
importMemoryWikiLegacyDigestFiles,
legacyMemoryWikiDigestFilesExist,
resolveMemoryWikiLegacyDigestPath,
} from "./digest-state-migration.js";
import {
readMemoryWikiAgentDigestSync,
readMemoryWikiCompiledDigestBundle,
writeMemoryWikiCompiledDigests,
} from "./digest-state.js";
import {
importMemoryWikiLegacyDigestFiles,
legacyMemoryWikiDigestFilesExist,
resolveMemoryWikiLegacyDigestPath,
} from "./doctor-legacy-digest-state.js";
describe("memory wiki compiled digest state", () => {
const previousStateDir = process.env.OPENCLAW_STATE_DIR;

View File

@@ -0,0 +1,70 @@
import fs from "node:fs/promises";
import path from "node:path";
import { type MemoryWikiDigestKind, writeMemoryWikiDigestForMigration } from "./digest-state.js";
export const MEMORY_WIKI_AGENT_DIGEST_LEGACY_PATH = ".openclaw-wiki/cache/agent-digest.json";
export const MEMORY_WIKI_CLAIMS_DIGEST_LEGACY_PATH = ".openclaw-wiki/cache/claims.jsonl";
export function resolveMemoryWikiLegacyDigestPath(
vaultRoot: string,
kind: MemoryWikiDigestKind,
): string {
return path.join(
vaultRoot,
kind === "agent-digest"
? MEMORY_WIKI_AGENT_DIGEST_LEGACY_PATH
: MEMORY_WIKI_CLAIMS_DIGEST_LEGACY_PATH,
);
}
async function importLegacyDigest(params: {
vaultRoot: string;
kind: MemoryWikiDigestKind;
}): Promise<{ imported: boolean; sourcePath: string }> {
const sourcePath = resolveMemoryWikiLegacyDigestPath(params.vaultRoot, params.kind);
const content = await fs.readFile(sourcePath, "utf8");
await writeMemoryWikiDigestForMigration({
vaultRoot: params.vaultRoot,
kind: params.kind,
content,
});
await fs.rm(sourcePath, { force: true });
return { imported: true, sourcePath };
}
export async function legacyMemoryWikiDigestFilesExist(vaultRoot: string): Promise<boolean> {
const results = await Promise.all(
(["agent-digest", "claims-digest"] as const).map((kind) =>
fs
.stat(resolveMemoryWikiLegacyDigestPath(vaultRoot, kind))
.then((stat) => stat.isFile())
.catch(() => false),
),
);
return results.some(Boolean);
}
export async function importMemoryWikiLegacyDigestFiles(params: {
vaultRoot: string;
}): Promise<{ imported: number; warnings: string[]; sourcePaths: string[] }> {
const warnings: string[] = [];
const sourcePaths: string[] = [];
let imported = 0;
for (const kind of ["agent-digest", "claims-digest"] as const) {
try {
const result = await importLegacyDigest({ vaultRoot: params.vaultRoot, kind });
imported += result.imported ? 1 : 0;
sourcePaths.push(result.sourcePath);
} catch (error) {
const sourcePath = resolveMemoryWikiLegacyDigestPath(params.vaultRoot, kind);
if ((error as NodeJS.ErrnoException)?.code === "ENOENT") {
continue;
}
warnings.push(`Failed importing Memory Wiki ${kind}: ${String(error)}`);
sourcePaths.push(sourcePath);
}
}
const cacheDir = path.join(params.vaultRoot, ".openclaw-wiki", "cache");
await fs.rmdir(cacheDir).catch(() => undefined);
return { imported, warnings, sourcePaths };
}

View File

@@ -0,0 +1,48 @@
import fs from "node:fs/promises";
import path from "node:path";
import { type MemoryWikiLogEntry, writeMemoryWikiLogEntryForMigration } from "./log.js";
export function resolveMemoryWikiLegacyLogPath(vaultRoot: string): string {
return path.join(vaultRoot, ".openclaw-wiki", "log.jsonl");
}
function isMemoryWikiLogEntry(value: unknown): value is MemoryWikiLogEntry {
return (
Boolean(value) &&
typeof value === "object" &&
typeof (value as { type?: unknown }).type === "string" &&
typeof (value as { timestamp?: unknown }).timestamp === "string"
);
}
export async function importMemoryWikiLegacyLog(params: {
vaultRoot: string;
}): Promise<{ imported: number; warnings: string[]; sourcePath: string }> {
const sourcePath = resolveMemoryWikiLegacyLogPath(params.vaultRoot);
const warnings: string[] = [];
let imported = 0;
const rawText = await fs.readFile(sourcePath, "utf8");
for (const [index, line] of rawText.split(/\r?\n/u).entries()) {
const trimmed = line.trim();
if (!trimmed) {
continue;
}
try {
const parsed = JSON.parse(trimmed) as unknown;
if (!isMemoryWikiLogEntry(parsed)) {
warnings.push(`Skipped invalid Memory Wiki log entry at ${sourcePath}:${index + 1}`);
continue;
}
await writeMemoryWikiLogEntryForMigration(params.vaultRoot, parsed, `legacy-${index + 1}`);
imported++;
} catch (error) {
warnings.push(
`Failed reading Memory Wiki log entry at ${sourcePath}:${index + 1}: ${String(error)}`,
);
}
}
if (warnings.length === 0) {
await fs.rm(sourcePath, { force: true });
}
return { imported, warnings, sourcePath };
}

View File

@@ -0,0 +1,80 @@
import fs from "node:fs/promises";
import path from "node:path";
import {
type MemoryWikiImportedSourceGroup,
readMemoryWikiSourceSyncState,
writeMemoryWikiSourceSyncState,
} from "./source-sync-state.js";
type MemoryWikiImportedSourceStateEntry = {
group: MemoryWikiImportedSourceGroup;
pagePath: string;
sourcePath: string;
sourceUpdatedAtMs: number;
sourceSize: number;
renderFingerprint: string;
};
export function resolveMemoryWikiLegacySourceSyncStatePath(vaultRoot: string): string {
return path.join(vaultRoot, ".openclaw-wiki", "source-sync.json");
}
function isRecord(value: unknown): value is Record<string, unknown> {
return Boolean(value) && typeof value === "object" && !Array.isArray(value);
}
function parseLegacySourceSyncEntry(raw: unknown): MemoryWikiImportedSourceStateEntry | null {
if (!isRecord(raw)) {
return null;
}
if (raw.group !== "bridge" && raw.group !== "unsafe-local") {
return null;
}
if (
typeof raw.pagePath !== "string" ||
typeof raw.sourcePath !== "string" ||
typeof raw.sourceUpdatedAtMs !== "number" ||
typeof raw.sourceSize !== "number" ||
typeof raw.renderFingerprint !== "string"
) {
return null;
}
return {
group: raw.group,
pagePath: raw.pagePath,
sourcePath: raw.sourcePath,
sourceUpdatedAtMs: raw.sourceUpdatedAtMs,
sourceSize: raw.sourceSize,
renderFingerprint: raw.renderFingerprint,
};
}
export async function importMemoryWikiLegacySourceSyncState(params: {
vaultRoot: string;
}): Promise<{ imported: number; warnings: string[]; sourcePath: string }> {
const sourcePath = resolveMemoryWikiLegacySourceSyncStatePath(params.vaultRoot);
const rawText = await fs.readFile(sourcePath, "utf8");
const raw = JSON.parse(rawText) as unknown;
const warnings: string[] = [];
if (!isRecord(raw) || raw.version !== 1 || !isRecord(raw.entries)) {
return {
imported: 0,
warnings: [`Skipped invalid Memory Wiki source sync file: ${sourcePath}`],
sourcePath,
};
}
const state = await readMemoryWikiSourceSyncState(params.vaultRoot);
let imported = 0;
for (const [syncKey, entry] of Object.entries(raw.entries)) {
const parsed = parseLegacySourceSyncEntry(entry);
if (!parsed) {
warnings.push(`Skipped invalid Memory Wiki source sync entry "${syncKey}".`);
continue;
}
state.entries[syncKey] = parsed;
imported++;
}
await writeMemoryWikiSourceSyncState(params.vaultRoot, state);
await fs.rm(sourcePath, { force: true });
return { imported, warnings, sourcePath };
}

View File

@@ -0,0 +1,76 @@
import fs from "node:fs/promises";
import os from "node:os";
import path from "node:path";
import type { MigrationProviderContext } from "openclaw/plugin-sdk/migration";
import { afterEach, describe, expect, it } from "vitest";
import type { ResolvedMemoryWikiConfig } from "./config.js";
import { createMemoryWikiSourceSyncMigrationProvider } from "./doctor-legacy-state.js";
const tempDirs: string[] = [];
async function createVaultRoot(): Promise<string> {
const root = await fs.mkdtemp(path.join(os.tmpdir(), "memory-wiki-migration-"));
tempDirs.push(root);
return root;
}
function createConfig(vaultRoot: string): ResolvedMemoryWikiConfig {
return {
vaultMode: "isolated",
vault: { path: vaultRoot, renderMode: "native" },
obsidian: { enabled: false, useOfficialCli: false, openAfterWrites: false },
bridge: {
enabled: false,
readMemoryArtifacts: false,
indexDreamReports: false,
indexDailyNotes: false,
indexMemoryRoot: false,
followMemoryEvents: false,
},
unsafeLocal: { allowPrivateMemoryCoreAccess: false, paths: [] },
ingest: { autoCompile: false, maxConcurrentJobs: 1, allowUrlIngest: false },
search: { backend: "shared", corpus: "wiki" },
context: { includeCompiledDigestPrompt: false },
render: { preserveHumanBlocks: true, createBacklinks: true, createDashboards: true },
};
}
afterEach(async () => {
await Promise.all(tempDirs.splice(0).map((dir) => fs.rm(dir, { recursive: true, force: true })));
});
describe("memory wiki source sync migration", () => {
it("removes retired vault metadata files during doctor migration", async () => {
const vaultRoot = await createVaultRoot();
const metadataDir = path.join(vaultRoot, ".openclaw-wiki");
const locksDir = path.join(metadataDir, "locks");
await fs.mkdir(locksDir, { recursive: true });
await fs.writeFile(path.join(metadataDir, "state.json"), '{"version":1}\n', "utf8");
await fs.writeFile(path.join(locksDir, "stale.lock"), "stale", "utf8");
const provider = createMemoryWikiSourceSyncMigrationProvider(createConfig(vaultRoot));
const ctx = {} as MigrationProviderContext;
if (!provider.detect) {
throw new Error("Expected memory wiki migration provider to expose detect");
}
await expect(provider.detect(ctx)).resolves.toMatchObject({
found: true,
confidence: "high",
});
const plan = await provider.plan(ctx);
expect(plan.items.map((item) => item.id)).toContain("memory-wiki-vault-metadata-json");
const result = await provider.apply(ctx, plan);
const item = result.items.find((item) => item.id === "memory-wiki-vault-metadata-json");
expect(item).toMatchObject({
status: "migrated",
details: { removedStateFile: true, removedLocksDir: true },
});
await expect(fs.stat(path.join(metadataDir, "state.json"))).rejects.toMatchObject({
code: "ENOENT",
});
await expect(fs.stat(locksDir)).rejects.toMatchObject({ code: "ENOENT" });
});
});

View File

@@ -0,0 +1,314 @@
import fs from "node:fs/promises";
import path from "node:path";
import type { MigrationProviderPlugin } from "openclaw/plugin-sdk/migration";
import { createMigrationItem, summarizeMigrationItems } from "openclaw/plugin-sdk/migration";
import type { ResolvedMemoryWikiConfig } from "./config.js";
import {
importMemoryWikiLegacyDigestFiles,
legacyMemoryWikiDigestFilesExist,
} from "./doctor-legacy-digest-state.js";
import { importMemoryWikiLegacyLog, resolveMemoryWikiLegacyLogPath } from "./doctor-legacy-log.js";
import {
importMemoryWikiLegacySourceSyncState,
resolveMemoryWikiLegacySourceSyncStatePath,
} from "./doctor-legacy-source-sync-state.js";
import { writeMemoryWikiImportRunRecord } from "./import-runs.js";
const PROVIDER_ID = "memory-wiki-source-sync";
function resolveLegacyVaultStatePath(vaultRoot: string): string {
return path.join(vaultRoot, ".openclaw-wiki", "state.json");
}
function resolveLegacyVaultLocksDir(vaultRoot: string): string {
return path.join(vaultRoot, ".openclaw-wiki", "locks");
}
async function legacySourceExists(vaultRoot: string): Promise<boolean> {
const sourcePath = resolveMemoryWikiLegacySourceSyncStatePath(vaultRoot);
return await fs
.stat(sourcePath)
.then((stat) => stat.isFile())
.catch(() => false);
}
async function legacyLogExists(vaultRoot: string): Promise<boolean> {
return await fs
.stat(resolveMemoryWikiLegacyLogPath(vaultRoot))
.then((stat) => stat.isFile())
.catch(() => false);
}
async function legacyVaultMetadataExists(vaultRoot: string): Promise<boolean> {
const [hasStateFile, hasLocksDir] = await Promise.all([
fs
.stat(resolveLegacyVaultStatePath(vaultRoot))
.then((stat) => stat.isFile())
.catch(() => false),
fs
.stat(resolveLegacyVaultLocksDir(vaultRoot))
.then((stat) => stat.isDirectory())
.catch(() => false),
]);
return hasStateFile || hasLocksDir;
}
async function removeLegacyVaultMetadata(vaultRoot: string): Promise<{
removedStateFile: boolean;
removedLocksDir: boolean;
}> {
const statePath = resolveLegacyVaultStatePath(vaultRoot);
const locksDir = resolveLegacyVaultLocksDir(vaultRoot);
const [hadStateFile, hadLocksDir] = await Promise.all([
fs
.stat(statePath)
.then((stat) => stat.isFile())
.catch(() => false),
fs
.stat(locksDir)
.then((stat) => stat.isDirectory())
.catch(() => false),
]);
if (hadStateFile) {
await fs.rm(statePath, { force: true });
}
if (hadLocksDir) {
await fs.rm(locksDir, { recursive: true, force: true });
}
return { removedStateFile: hadStateFile, removedLocksDir: hadLocksDir };
}
function resolveLegacyImportRunsDir(vaultRoot: string): string {
return path.join(vaultRoot, ".openclaw-wiki", "import-runs");
}
async function listLegacyImportRunJsonFiles(vaultRoot: string): Promise<string[]> {
const importRunsDir = resolveLegacyImportRunsDir(vaultRoot);
const entries = await fs
.readdir(importRunsDir, { withFileTypes: true })
.catch((error: NodeJS.ErrnoException) => {
if (error?.code === "ENOENT") {
return [];
}
throw error;
});
return entries
.filter((entry) => entry.isFile() && entry.name.endsWith(".json"))
.map((entry) => path.join(importRunsDir, entry.name))
.toSorted((left, right) => left.localeCompare(right));
}
function isRecord(value: unknown): value is Record<string, unknown> {
return Boolean(value) && typeof value === "object" && !Array.isArray(value);
}
async function importLegacyImportRunJsonFiles(vaultRoot: string): Promise<{
imported: number;
warnings: string[];
}> {
const warnings: string[] = [];
let imported = 0;
for (const filePath of await listLegacyImportRunJsonFiles(vaultRoot)) {
const raw = JSON.parse(await fs.readFile(filePath, "utf8")) as unknown;
if (!isRecord(raw) || typeof raw.runId !== "string" || !raw.runId.trim()) {
warnings.push(`Skipped invalid Memory Wiki import run file: ${filePath}`);
continue;
}
await writeMemoryWikiImportRunRecord(vaultRoot, {
...raw,
runId: raw.runId.trim(),
});
await fs.rm(filePath, { force: true });
imported++;
}
return { imported, warnings };
}
export function createMemoryWikiSourceSyncMigrationProvider(
config: ResolvedMemoryWikiConfig,
): MigrationProviderPlugin {
const sourcePath = resolveMemoryWikiLegacySourceSyncStatePath(config.vault.path);
const legacyLogPath = resolveMemoryWikiLegacyLogPath(config.vault.path);
const importRunsDir = resolveLegacyImportRunsDir(config.vault.path);
const target = "global SQLite plugin_state_entries(memory-wiki/source-sync)";
const buildPlan: MigrationProviderPlugin["plan"] = async () => {
const hasSourceSync = await legacySourceExists(config.vault.path);
const hasLegacyLog = await legacyLogExists(config.vault.path);
const hasLegacyDigests = await legacyMemoryWikiDigestFilesExist(config.vault.path);
const hasLegacyVaultMetadata = await legacyVaultMetadataExists(config.vault.path);
const importRunFiles = await listLegacyImportRunJsonFiles(config.vault.path);
const items = [
...(hasLegacyVaultMetadata
? [
createMigrationItem({
id: "memory-wiki-vault-metadata-json",
kind: "state",
action: "archive",
source: path.join(config.vault.path, ".openclaw-wiki"),
target: "none; Memory Wiki vault metadata is derived from config and SQLite state",
message: "Remove retired Memory Wiki vault state.json and locks directory.",
}),
]
: []),
...(hasSourceSync
? [
createMigrationItem({
id: "memory-wiki-source-sync-json",
kind: "state",
action: "import",
source: sourcePath,
target,
message: "Import Memory Wiki source sync JSON into SQLite plugin state.",
}),
]
: []),
...(hasLegacyLog
? [
createMigrationItem({
id: "memory-wiki-log-jsonl",
kind: "state",
action: "import",
source: legacyLogPath,
target: "global SQLite plugin_state_entries(memory-wiki/activity-log)",
message: "Import Memory Wiki activity log JSONL into SQLite plugin state.",
}),
]
: []),
...(importRunFiles.length > 0
? [
createMigrationItem({
id: "memory-wiki-import-runs-json",
kind: "state",
action: "import",
source: importRunsDir,
target: "global SQLite plugin_state_entries(memory-wiki/import-runs)",
message: "Import Memory Wiki import-run JSON records into SQLite plugin state.",
details: { recordCount: importRunFiles.length },
}),
]
: []),
...(hasLegacyDigests
? [
createMigrationItem({
id: "memory-wiki-compiled-digest-cache",
kind: "state",
action: "import",
source: path.join(config.vault.path, ".openclaw-wiki", "cache"),
target: "global SQLite plugin_blob_entries(memory-wiki/compiled-digest)",
message: "Import Memory Wiki compiled digest cache into SQLite plugin state.",
}),
]
: []),
];
return {
providerId: PROVIDER_ID,
source: sourcePath,
target,
summary: summarizeMigrationItems(items),
items,
};
};
return {
id: PROVIDER_ID,
label: "Memory Wiki source sync state",
description: "Import the legacy Memory Wiki source sync JSON ledger into SQLite plugin state.",
async detect() {
const found =
(await legacySourceExists(config.vault.path)) ||
(await legacyLogExists(config.vault.path)) ||
(await legacyVaultMetadataExists(config.vault.path)) ||
(await legacyMemoryWikiDigestFilesExist(config.vault.path)) ||
(await listLegacyImportRunJsonFiles(config.vault.path)).length > 0;
return {
found,
source: sourcePath,
label: "Memory Wiki legacy state",
confidence: found ? "high" : "low",
message: found
? `Legacy Memory Wiki state found under ${path.dirname(sourcePath)}.`
: "No legacy Memory Wiki state files found.",
};
},
plan: buildPlan,
async apply(_ctx, plan) {
const selectedPlan = plan ?? (await buildPlan(_ctx));
const items = [...selectedPlan.items];
const warnings = [...(selectedPlan.warnings ?? [])];
for (let itemIndex = 0; itemIndex < items.length; itemIndex += 1) {
const item = items[itemIndex];
if (!item) {
continue;
}
try {
if (item.id === "memory-wiki-vault-metadata-json") {
const result = await removeLegacyVaultMetadata(config.vault.path);
items[itemIndex] = {
...item,
status: "migrated",
details: result,
};
} else if (item.id === "memory-wiki-source-sync-json") {
const result = await importMemoryWikiLegacySourceSyncState({
vaultRoot: config.vault.path,
});
warnings.push(...result.warnings);
items[itemIndex] = {
...item,
status: "migrated",
details: {
imported: result.imported,
},
};
} else if (item.id === "memory-wiki-log-jsonl") {
const result = await importMemoryWikiLegacyLog({
vaultRoot: config.vault.path,
});
warnings.push(...result.warnings);
items[itemIndex] = {
...item,
status: "migrated",
details: {
imported: result.imported,
},
};
} else if (item.id === "memory-wiki-import-runs-json") {
const result = await importLegacyImportRunJsonFiles(config.vault.path);
warnings.push(...result.warnings);
items[itemIndex] = {
...item,
status: "migrated",
details: {
imported: result.imported,
},
};
} else if (item.id === "memory-wiki-compiled-digest-cache") {
const result = await importMemoryWikiLegacyDigestFiles({
vaultRoot: config.vault.path,
});
warnings.push(...result.warnings);
items[itemIndex] = {
...item,
status: "migrated",
details: {
imported: result.imported,
},
};
}
} catch (error) {
items[itemIndex] = {
...item,
status: "error",
reason: error instanceof Error ? error.message : String(error),
};
}
}
return {
...selectedPlan,
summary: summarizeMigrationItems(items),
items,
warnings,
};
},
};
}

View File

@@ -3,7 +3,7 @@ import os from "node:os";
import path from "node:path";
import { resetPluginStateStoreForTests } from "openclaw/plugin-sdk/plugin-state-runtime";
import { afterEach, describe, expect, it } from "vitest";
import { importMemoryWikiLegacyLog, resolveMemoryWikiLegacyLogPath } from "./log-migration.js";
import { importMemoryWikiLegacyLog, resolveMemoryWikiLegacyLogPath } from "./doctor-legacy-log.js";
import { appendMemoryWikiLog, readMemoryWikiLogEntries } from "./log.js";
describe("memory wiki activity log", () => {

View File

@@ -6,7 +6,7 @@ import { afterEach, describe, expect, it } from "vitest";
import {
importMemoryWikiLegacySourceSyncState,
resolveMemoryWikiLegacySourceSyncStatePath,
} from "./source-sync-state-migration.js";
} from "./doctor-legacy-source-sync-state.js";
import {
readMemoryWikiSourceSyncState,
writeMemoryWikiSourceSyncState,

View File

@@ -423,16 +423,16 @@ const allowedExactPaths = new Set([
"extensions/imessage/src/doctor-legacy-state.ts",
"extensions/matrix/src/doctor-legacy-state.ts",
"extensions/matrix/src/doctor-state-imports.ts",
"extensions/memory-wiki/src/digest-state-migration.ts",
"extensions/memory-wiki/src/source-sync-state-migration.ts",
"extensions/memory-wiki/src/source-sync-migration.ts",
"extensions/memory-wiki/src/doctor-legacy-digest-state.ts",
"extensions/memory-wiki/src/doctor-legacy-log.ts",
"extensions/memory-wiki/src/doctor-legacy-source-sync-state.ts",
"extensions/memory-wiki/src/doctor-legacy-state.ts",
"extensions/msteams/src/doctor-legacy-state.ts",
"extensions/nostr/src/doctor-legacy-state.ts",
"extensions/skill-workshop/src/doctor-legacy-state.ts",
"extensions/qqbot/src/doctor-legacy-state.ts",
"extensions/telegram/src/doctor-legacy-state.ts",
"extensions/whatsapp/src/doctor-legacy-state.ts",
"extensions/memory-wiki/src/log-migration.ts",
]);
const allowedPrefixes = ["src/commands/doctor", "src/commands/export-trajectory"];