test: assert crestodian audit through sqlite

This commit is contained in:
Peter Steinberger
2026-05-08 16:15:38 +01:00
parent 5893ceb37c
commit ab0aa4e9b4
12 changed files with 54 additions and 26 deletions

View File

@@ -129,7 +129,7 @@ you pass `--yes` for a direct command:
Applied writes are recorded in:
```text
~/.openclaw/audit/crestodian.jsonl
SQLite core plugin state: core:crestodian/audit
```
Discovery is not audited. Only applied operations and writes are logged.

View File

@@ -46,6 +46,7 @@ function createDeps(overrides?: Partial<QaScenarioRuntimeDeps>): QaScenarioRunti
readConfigSnapshot: fn,
createSession: fn,
readEffectiveTools: fn,
readQaCrestodianAuditEntries: fn,
readSkillStatus: fn,
readRawQaSessionEntries: fn,
setQaActiveMemorySessionDisabled: fn,

View File

@@ -58,6 +58,7 @@ export type QaScenarioRuntimeDeps = {
readConfigSnapshot: QaScenarioRuntimeFunction;
createSession: QaScenarioRuntimeFunction;
readEffectiveTools: QaScenarioRuntimeFunction;
readQaCrestodianAuditEntries: QaScenarioRuntimeFunction;
readSkillStatus: QaScenarioRuntimeFunction;
readRawQaSessionEntries: QaScenarioRuntimeFunction;
setQaActiveMemorySessionDisabled: QaScenarioRuntimeFunction;
@@ -144,6 +145,7 @@ type QaScenarioRuntimeApi<
readConfigSnapshot: TDeps["readConfigSnapshot"];
createSession: TDeps["createSession"];
readEffectiveTools: TDeps["readEffectiveTools"];
readQaCrestodianAuditEntries: TDeps["readQaCrestodianAuditEntries"];
readSkillStatus: TDeps["readSkillStatus"];
readRawQaSessionEntries: TDeps["readRawQaSessionEntries"];
setQaActiveMemorySessionDisabled: TDeps["setQaActiveMemorySessionDisabled"];
@@ -245,6 +247,7 @@ export function createQaScenarioRuntimeApi<
readConfigSnapshot: params.deps.readConfigSnapshot,
createSession: params.deps.createSession,
readEffectiveTools: params.deps.readEffectiveTools,
readQaCrestodianAuditEntries: params.deps.readQaCrestodianAuditEntries,
readSkillStatus: params.deps.readSkillStatus,
readRawQaSessionEntries: params.deps.readRawQaSessionEntries,
setQaActiveMemorySessionDisabled: params.deps.setQaActiveMemorySessionDisabled,

View File

@@ -4,7 +4,10 @@ import {
replaceSqliteSessionTranscriptEvents,
} from "openclaw/plugin-sdk/agent-harness-runtime";
import { upsertSessionEntry } from "openclaw/plugin-sdk/config-runtime";
import { createPluginStateKeyedStore } from "openclaw/plugin-sdk/plugin-state-runtime";
import {
createCorePluginStateKeyedStore,
createPluginStateKeyedStore,
} from "openclaw/plugin-sdk/plugin-state-runtime";
import { liveTurnTimeoutMs } from "./suite-runtime-agent-common.js";
import type {
QaRawSessionEntry,
@@ -18,6 +21,13 @@ type ActiveMemorySessionToggleEntry = {
updatedAt: number;
};
type QaCrestodianAuditEntry = {
timestamp?: string;
operation?: string;
summary?: string;
[key: string]: unknown;
};
function createActiveMemorySessionToggleStore(env: Pick<QaSuiteRuntimeEnv, "gateway">) {
return createPluginStateKeyedStore<ActiveMemorySessionToggleEntry>("active-memory", {
namespace: "session-toggles",
@@ -26,6 +36,15 @@ function createActiveMemorySessionToggleStore(env: Pick<QaSuiteRuntimeEnv, "gate
});
}
function createCrestodianAuditStore(env: Pick<QaSuiteRuntimeEnv, "gateway">) {
return createCorePluginStateKeyedStore<QaCrestodianAuditEntry>({
ownerId: "core:crestodian",
namespace: "audit",
maxEntries: 50_000,
env: env.gateway.runtimeEnv,
});
}
async function createSession(
env: Pick<QaSuiteRuntimeEnv, "gateway" | "primaryModel" | "alternateModel" | "providerMode">,
label: string,
@@ -156,6 +175,11 @@ async function setQaActiveMemorySessionDisabled(
return { sessionKey, disabled: false };
}
async function readQaCrestodianAuditEntries(env: Pick<QaSuiteRuntimeEnv, "gateway">) {
const auditStore = createCrestodianAuditStore(env);
return (await auditStore.entries()).map((entry) => entry.value);
}
async function readEffectiveTools(
env: Pick<QaSuiteRuntimeEnv, "gateway" | "primaryModel" | "alternateModel" | "providerMode">,
sessionKey: string,
@@ -247,6 +271,7 @@ async function readRawQaSessionEntries(env: Pick<QaSuiteRuntimeEnv, "gateway">)
export {
createSession,
readEffectiveTools,
readQaCrestodianAuditEntries,
readRawQaSessionEntries,
readSkillStatus,
setQaActiveMemorySessionDisabled,

View File

@@ -1,6 +1,7 @@
export {
createSession,
readEffectiveTools,
readQaCrestodianAuditEntries,
readRawQaSessionEntries,
readSkillStatus,
setQaActiveMemorySessionDisabled,

View File

@@ -20,6 +20,7 @@ const readConfigSnapshot = vi.hoisted(() => vi.fn());
const waitForConfigRestartSettle = vi.hoisted(() => vi.fn());
const createSession = vi.hoisted(() => vi.fn());
const readEffectiveTools = vi.hoisted(() => vi.fn());
const readQaCrestodianAuditEntries = vi.hoisted(() => vi.fn());
const readSkillStatus = vi.hoisted(() => vi.fn());
const readRawQaSessionEntries = vi.hoisted(() => vi.fn());
const setQaActiveMemorySessionDisabled = vi.hoisted(() => vi.fn());
@@ -87,6 +88,7 @@ vi.mock("./suite-runtime-gateway.js", () => ({
vi.mock("./suite-runtime-agent.js", () => ({
createSession,
readEffectiveTools,
readQaCrestodianAuditEntries,
readSkillStatus,
readRawQaSessionEntries,
setQaActiveMemorySessionDisabled,

View File

@@ -35,6 +35,7 @@ import {
listCronJobs,
readDoctorMemoryStatus,
readEffectiveTools,
readQaCrestodianAuditEntries,
readRawQaSessionEntries,
readSkillStatus,
resolveGeneratedImagePath,
@@ -162,6 +163,7 @@ function createQaSuiteScenarioDeps(params: QaSuiteScenarioDepsParams) {
readConfigSnapshot,
createSession,
readEffectiveTools,
readQaCrestodianAuditEntries,
readSkillStatus,
readRawQaSessionEntries,
setQaActiveMemorySessionDisabled,

View File

@@ -142,17 +142,21 @@ steps:
- assert:
expr: "!JSON.stringify(writtenConfig.channels?.discord ?? {}).includes(setupSpec.discordToken)"
message: Crestodian persisted the raw Discord token.
- set: auditText
- call: readQaCrestodianAuditEntries
saveAs: auditEntries
args:
- ref: env
- set: auditOperations
value:
expr: "await fs.readFile(path.join(stateDir, 'audit', 'crestodian.jsonl'), 'utf8')"
expr: "auditEntries.map((entry) => entry.operation).filter(Boolean)"
- forEach:
items:
ref: setupSpec.auditOperations
item: operation
actions:
- assert:
expr: 'auditText.includes(`"operation":"${operation}"`)'
expr: "auditOperations.includes(operation)"
message:
expr: "`missing audit entry for ${operation}: ${auditText}`"
expr: "`missing audit entry for ${operation}: ${JSON.stringify(auditEntries)}`"
detailsExpr: "`stateDir=${stateDir}\\nconfigPath=${configPath}\\nagent=${JSON.stringify(agent)}\\nDiscord SecretRef=${JSON.stringify(writtenConfig.channels?.discord?.token)}`"
```

View File

@@ -7,6 +7,7 @@ import path from "node:path";
import { runCli, shouldStartCrestodianForBareRoot } from "../../dist/cli/run-main.js";
import { clearConfigCache } from "../../dist/config/config.js";
import type { OpenClawConfig } from "../../dist/config/types.openclaw.js";
import { listCrestodianAuditEntriesForTests } from "../../dist/crestodian/audit.js";
import { runCrestodian } from "../../dist/crestodian/crestodian.js";
import type { RuntimeEnv } from "../../dist/runtime.js";
@@ -160,10 +161,10 @@ async function main() {
"Crestodian persisted the raw Discord token",
);
const auditPath = path.join(stateDir, "audit", "crestodian.jsonl");
const audit = (await fs.readFile(auditPath, "utf8")).trim();
const auditEntries = (await listCrestodianAuditEntriesForTests()).map((entry) => entry.value);
const auditOperations = auditEntries.map((entry) => entry.operation);
for (const operation of spec.auditOperations) {
assert(audit.includes(`"operation":"${operation}"`), `${operation} audit entry missing`);
assert(auditOperations.includes(operation), `${operation} audit entry missing`);
}
console.log("Crestodian first-run Docker E2E passed");

View File

@@ -114,10 +114,10 @@ async function main() {
"planned default model was not written",
);
const auditPath = path.join(stateDir, "audit", "crestodian.jsonl");
const audit = (await fs.readFile(auditPath, "utf8")).trim();
const { listCrestodianAuditEntriesForTests } = await import("../../dist/crestodian/audit.js");
const auditEntries = (await listCrestodianAuditEntriesForTests()).map((entry) => entry.value);
assert(
audit.includes('"operation":"config.setDefaultModel"'),
auditEntries.some((entry) => entry.operation === "config.setDefaultModel"),
"planned model update audit entry missing",
);

View File

@@ -7,6 +7,7 @@ import path from "node:path";
import { handleCrestodianCommand } from "../../dist/auto-reply/reply/commands-crestodian.js";
import { clearConfigCache } from "../../dist/config/config.js";
import type { OpenClawConfig } from "../../dist/config/types.openclaw.js";
import { listCrestodianAuditEntriesForTests } from "../../dist/crestodian/audit.js";
import { runCrestodianRescueMessage } from "../../dist/crestodian/rescue-message.js";
type CommandResult = Awaited<ReturnType<typeof handleCrestodianCommand>>;
@@ -226,10 +227,8 @@ async function main() {
"agent config was not updated",
);
const auditPath = path.join(stateDir, "audit", "crestodian.jsonl");
const auditLines = (await fs.readFile(auditPath, "utf8")).trim().split("\n");
assert(auditLines.length >= 2, "audit log did not record both operations");
const audits = auditLines.map((line) => JSON.parse(line));
const audits = (await listCrestodianAuditEntriesForTests()).map((entry) => entry.value);
assert(audits.length >= 2, "audit log did not record both operations");
assert(
audits.some((audit) => audit.operation === "config.setDefaultModel"),
"model audit operation missing",

View File

@@ -1,6 +1,4 @@
import { randomUUID } from "node:crypto";
import path from "node:path";
import { resolveStateDir } from "../config/paths.js";
import {
createCorePluginStateKeyedStore,
type PluginStateEntry,
@@ -26,13 +24,6 @@ const crestodianAuditStore = createCorePluginStateKeyedStore<CrestodianAuditEntr
maxEntries: CRESTODIAN_AUDIT_MAX_ENTRIES,
});
export function resolveCrestodianAuditPath(
env: NodeJS.ProcessEnv = process.env,
stateDir = resolveStateDir(env),
): string {
return path.join(stateDir, "audit", "crestodian.jsonl");
}
function resolveCrestodianAuditKey(entry: CrestodianAuditEntry): string {
const suffix = randomUUID();
return `${entry.timestamp}:${suffix}`;
@@ -40,7 +31,6 @@ function resolveCrestodianAuditKey(entry: CrestodianAuditEntry): string {
export async function appendCrestodianAuditEntry(
entry: Omit<CrestodianAuditEntry, "timestamp">,
_opts: { env?: NodeJS.ProcessEnv; auditPath?: string } = {},
): Promise<string> {
const record = {
timestamp: new Date().toISOString(),