From ed925c965d3c8efac8be4e604ff5fcf5f72c0440 Mon Sep 17 00:00:00 2001 From: Peter Steinberger Date: Fri, 8 May 2026 15:52:34 +0100 Subject: [PATCH] test: seed memory qa sessions through sqlite --- .../qa-lab/src/scenario-runtime-api.test.ts | 1 + extensions/qa-lab/src/scenario-runtime-api.ts | 3 + .../qa-lab/src/suite-runtime-agent-session.ts | 86 ++++++++++++++++++- extensions/qa-lab/src/suite-runtime-agent.ts | 1 + .../qa-lab/src/suite-runtime-flow.test.ts | 2 + extensions/qa-lab/src/suite-runtime-flow.ts | 2 + qa/scenarios/memory/memory-dreaming-sweep.md | 42 +++++---- qa/scenarios/memory/session-memory-ranking.md | 49 ++++++----- src/plugin-sdk/agent-harness-runtime.ts | 1 + 9 files changed, 144 insertions(+), 43 deletions(-) diff --git a/extensions/qa-lab/src/scenario-runtime-api.test.ts b/extensions/qa-lab/src/scenario-runtime-api.test.ts index a5949a21ad2..3ac4e60aebe 100644 --- a/extensions/qa-lab/src/scenario-runtime-api.test.ts +++ b/extensions/qa-lab/src/scenario-runtime-api.test.ts @@ -48,6 +48,7 @@ function createDeps(overrides?: Partial): QaScenarioRunti readEffectiveTools: fn, readSkillStatus: fn, readRawQaSessionEntries: fn, + seedQaSessionTranscript: fn, runQaCli: fn, extractMediaPathFromText: fn, resolveGeneratedImagePath: fn, diff --git a/extensions/qa-lab/src/scenario-runtime-api.ts b/extensions/qa-lab/src/scenario-runtime-api.ts index c5af21af85f..533e9ce8725 100644 --- a/extensions/qa-lab/src/scenario-runtime-api.ts +++ b/extensions/qa-lab/src/scenario-runtime-api.ts @@ -60,6 +60,7 @@ export type QaScenarioRuntimeDeps = { readEffectiveTools: QaScenarioRuntimeFunction; readSkillStatus: QaScenarioRuntimeFunction; readRawQaSessionEntries: QaScenarioRuntimeFunction; + seedQaSessionTranscript: QaScenarioRuntimeFunction; runQaCli: QaScenarioRuntimeFunction; extractMediaPathFromText: QaScenarioRuntimeFunction; resolveGeneratedImagePath: QaScenarioRuntimeFunction; @@ -144,6 +145,7 @@ type QaScenarioRuntimeApi< readEffectiveTools: TDeps["readEffectiveTools"]; readSkillStatus: TDeps["readSkillStatus"]; readRawQaSessionEntries: TDeps["readRawQaSessionEntries"]; + seedQaSessionTranscript: TDeps["seedQaSessionTranscript"]; runQaCli: TDeps["runQaCli"]; extractMediaPathFromText: TDeps["extractMediaPathFromText"]; resolveGeneratedImagePath: TDeps["resolveGeneratedImagePath"]; @@ -243,6 +245,7 @@ export function createQaScenarioRuntimeApi< readEffectiveTools: params.deps.readEffectiveTools, readSkillStatus: params.deps.readSkillStatus, readRawQaSessionEntries: params.deps.readRawQaSessionEntries, + seedQaSessionTranscript: params.deps.seedQaSessionTranscript, runQaCli: params.deps.runQaCli, extractMediaPathFromText: params.deps.extractMediaPathFromText, resolveGeneratedImagePath: params.deps.resolveGeneratedImagePath, diff --git a/extensions/qa-lab/src/suite-runtime-agent-session.ts b/extensions/qa-lab/src/suite-runtime-agent-session.ts index 78a887a5a72..a5bdf234b9a 100644 --- a/extensions/qa-lab/src/suite-runtime-agent-session.ts +++ b/extensions/qa-lab/src/suite-runtime-agent-session.ts @@ -1,3 +1,9 @@ +import { + createSqliteSessionTranscriptLocator, + CURRENT_SESSION_VERSION, + replaceSqliteSessionTranscriptEvents, +} from "openclaw/plugin-sdk/agent-harness-runtime"; +import { upsertSessionEntry } from "openclaw/plugin-sdk/config-runtime"; import { liveTurnTimeoutMs } from "./suite-runtime-agent-common.js"; import type { QaRawSessionEntry, @@ -27,6 +33,78 @@ async function createSession( return sessionKey; } +async function seedQaSessionTranscript( + env: Pick, + params: { + agentId?: string; + sessionId: string; + sessionKey?: string; + messages: Array<{ role: string; content: unknown; timestamp?: number | string }>; + now?: number; + originLabel?: string; + }, +) { + const agentId = params.agentId?.trim() || "qa"; + const now = params.now ?? Date.now(); + const sessionId = params.sessionId.trim(); + if (!sessionId) { + throw new Error("seedQaSessionTranscript requires sessionId"); + } + const sessionFile = createSqliteSessionTranscriptLocator({ agentId, sessionId }); + const sessionKey = params.sessionKey?.trim() || `agent:${agentId}:seed-${sessionId}`; + let parentId: string | null = null; + const messageEvents = params.messages.map((message, index) => { + const id = `qa-seed-${index + 1}`; + const timestampMs = now - Math.max(1, params.messages.length - index) * 30_000; + const event = { + type: "message" as const, + id, + parentId, + timestamp: new Date(timestampMs).toISOString(), + message: { + ...message, + timestamp: + typeof message.timestamp === "number" || typeof message.timestamp === "string" + ? message.timestamp + : timestampMs, + }, + }; + parentId = id; + return event; + }); + replaceSqliteSessionTranscriptEvents({ + agentId, + sessionId, + transcriptPath: sessionFile, + env: env.gateway.runtimeEnv, + events: [ + { + type: "session", + id: sessionId, + version: CURRENT_SESSION_VERSION, + timestamp: new Date(now - 120_000).toISOString(), + cwd: env.gateway.workspaceDir, + }, + ...messageEvents, + ], + now: () => now, + }); + upsertSessionEntry({ + agentId, + env: env.gateway.runtimeEnv, + sessionKey, + entry: { + sessionId, + updatedAt: now, + sessionFile, + origin: { + label: params.originLabel ?? "QA seeded SQLite transcript", + }, + }, + }); + return { agentId, sessionId, sessionKey, sessionFile }; +} + async function readEffectiveTools( env: Pick, sessionKey: string, @@ -115,4 +193,10 @@ async function readRawQaSessionEntries(env: Pick) ); } -export { createSession, readEffectiveTools, readRawQaSessionEntries, readSkillStatus }; +export { + createSession, + readEffectiveTools, + readRawQaSessionEntries, + readSkillStatus, + seedQaSessionTranscript, +}; diff --git a/extensions/qa-lab/src/suite-runtime-agent.ts b/extensions/qa-lab/src/suite-runtime-agent.ts index e195599b001..6e65cd011b4 100644 --- a/extensions/qa-lab/src/suite-runtime-agent.ts +++ b/extensions/qa-lab/src/suite-runtime-agent.ts @@ -3,6 +3,7 @@ export { readEffectiveTools, readRawQaSessionEntries, readSkillStatus, + seedQaSessionTranscript, } from "./suite-runtime-agent-session.js"; export { forceMemoryIndex, diff --git a/extensions/qa-lab/src/suite-runtime-flow.test.ts b/extensions/qa-lab/src/suite-runtime-flow.test.ts index 6b2d6844f1a..c3bf570bbd5 100644 --- a/extensions/qa-lab/src/suite-runtime-flow.test.ts +++ b/extensions/qa-lab/src/suite-runtime-flow.test.ts @@ -22,6 +22,7 @@ const createSession = vi.hoisted(() => vi.fn()); const readEffectiveTools = vi.hoisted(() => vi.fn()); const readSkillStatus = vi.hoisted(() => vi.fn()); const readRawQaSessionEntries = vi.hoisted(() => vi.fn()); +const seedQaSessionTranscript = vi.hoisted(() => vi.fn()); const runQaCli = vi.hoisted(() => vi.fn()); const extractMediaPathFromText = vi.hoisted(() => vi.fn()); const resolveGeneratedImagePath = vi.hoisted(() => vi.fn()); @@ -87,6 +88,7 @@ vi.mock("./suite-runtime-agent.js", () => ({ readEffectiveTools, readSkillStatus, readRawQaSessionEntries, + seedQaSessionTranscript, runQaCli, extractMediaPathFromText, resolveGeneratedImagePath, diff --git a/extensions/qa-lab/src/suite-runtime-flow.ts b/extensions/qa-lab/src/suite-runtime-flow.ts index 52bcc2ebe4b..67acbd6dfb3 100644 --- a/extensions/qa-lab/src/suite-runtime-flow.ts +++ b/extensions/qa-lab/src/suite-runtime-flow.ts @@ -40,6 +40,7 @@ import { resolveGeneratedImagePath, runAgentPrompt, runQaCli, + seedQaSessionTranscript, startAgentRun, waitForAgentRun, writeWorkspaceSkill, @@ -162,6 +163,7 @@ function createQaSuiteScenarioDeps(params: QaSuiteScenarioDepsParams) { readEffectiveTools, readSkillStatus, readRawQaSessionEntries, + seedQaSessionTranscript, runQaCli, extractMediaPathFromText, resolveGeneratedImagePath, diff --git a/qa/scenarios/memory/memory-dreaming-sweep.md b/qa/scenarios/memory/memory-dreaming-sweep.md index c6bb27addc9..0dad95568ba 100644 --- a/qa/scenarios/memory/memory-dreaming-sweep.md +++ b/qa/scenarios/memory/memory-dreaming-sweep.md @@ -153,25 +153,12 @@ steps: - set: memoryPath value: expr: "path.join(env.gateway.workspaceDir, 'MEMORY.md')" - - set: homeDir - value: - expr: "env.gateway.runtimeEnv.HOME ?? env.gateway.runtimeEnv.OPENCLAW_HOME ?? env.gateway.tempRoot" - - set: sessionsDir - value: - expr: "resolveSessionTranscriptsDirForAgent('qa', env.gateway.runtimeEnv, () => homeDir)" - - set: transcriptPath - value: - expr: "path.join(sessionsDir, `${config.transcriptId}.jsonl`)" - try: actions: - call: fs.mkdir args: - expr: "path.dirname(dailyPath)" - recursive: true - - call: fs.mkdir - args: - - ref: sessionsDir - - recursive: true - call: fs.writeFile args: - ref: dailyPath @@ -180,11 +167,32 @@ steps: - set: now value: expr: "Date.now()" - - call: fs.writeFile + - call: seedQaSessionTranscript + saveAs: seededSession args: - - ref: transcriptPath - - expr: "[JSON.stringify({ type: 'session', id: config.transcriptId, timestamp: new Date(now - 120000).toISOString() }), JSON.stringify({ type: 'message', message: { role: 'user', timestamp: new Date(now - 90000).toISOString(), content: [{ type: 'text', text: config.transcriptUserPrompt }] } }), JSON.stringify({ type: 'message', message: { role: 'assistant', timestamp: new Date(now - 60000).toISOString(), content: [{ type: 'text', text: config.transcriptAssistantReply }] } })].join('\\n') + '\\n'" - - utf8 + - ref: env + - agentId: qa + sessionId: + expr: config.transcriptId + sessionKey: agent:qa:seed-memory-dreaming-sweep + now: + ref: now + originLabel: QA seeded memory dreaming sweep transcript + messages: + - role: user + timestamp: + expr: "now - 90000" + content: + - type: text + text: + expr: config.transcriptUserPrompt + - role: assistant + timestamp: + expr: "now - 60000" + content: + - type: text + text: + expr: config.transcriptAssistantReply - call: fs.rm args: - ref: memoryPath diff --git a/qa/scenarios/memory/session-memory-ranking.md b/qa/scenarios/memory/session-memory-ranking.md index acbbe85870a..4fc8076a6af 100644 --- a/qa/scenarios/memory/session-memory-ranking.md +++ b/qa/scenarios/memory/session-memory-ranking.md @@ -109,36 +109,35 @@ steps: - ref: staleMemoryPath - ref: staleAt - ref: staleAt - - set: transcriptsDir - value: - expr: "resolveSessionTranscriptsDirForAgent('qa', env.gateway.runtimeEnv, () => env.gateway.runtimeEnv.HOME ?? path.join(env.gateway.tempRoot, 'home'))" - - call: fs.mkdir - args: - - ref: transcriptsDir - - recursive: true - - set: transcriptPath - value: - expr: "path.join(transcriptsDir, `${config.transcriptId}.jsonl`)" - set: now value: expr: "Date.now()" - - call: fs.writeFile - args: - - ref: transcriptPath - - expr: "[JSON.stringify({ type: 'session', id: config.transcriptId, timestamp: new Date(now - 120000).toISOString() }), JSON.stringify({ type: 'message', message: { role: 'user', timestamp: new Date(now - 90000).toISOString(), content: [{ type: 'text', text: config.transcriptQuestion }] } }), JSON.stringify({ type: 'message', message: { role: 'assistant', timestamp: new Date(now - 60000).toISOString(), content: [{ type: 'text', text: config.transcriptAnswer }] } })].join('\\n') + '\\n'" - - utf8 - - call: readRawQaSessionStore - saveAs: sessionStore + - call: seedQaSessionTranscript + saveAs: seededSession args: - ref: env - - set: sessionStorePath - value: - expr: "path.join(env.gateway.tempRoot, 'state', 'agents', 'qa', 'sessions', 'sessions.json')" - - call: fs.writeFile - args: - - ref: sessionStorePath - - expr: "JSON.stringify({ ...sessionStore, ['agent:qa:seed-session-memory-ranking']: { sessionId: config.transcriptId, updatedAt: now, sessionFile: transcriptPath, origin: { label: 'QA seeded session memory ranking transcript' } } }, null, 2)" - - utf8 + - agentId: qa + sessionId: + expr: config.transcriptId + sessionKey: agent:qa:seed-session-memory-ranking + now: + ref: now + originLabel: QA seeded session memory ranking transcript + messages: + - role: user + timestamp: + expr: "now - 90000" + content: + - type: text + text: + expr: config.transcriptQuestion + - role: assistant + timestamp: + expr: "now - 60000" + content: + - type: text + text: + expr: config.transcriptAnswer - call: forceMemoryIndex args: - env: diff --git a/src/plugin-sdk/agent-harness-runtime.ts b/src/plugin-sdk/agent-harness-runtime.ts index 9f4988f952f..8c3d74fd549 100644 --- a/src/plugin-sdk/agent-harness-runtime.ts +++ b/src/plugin-sdk/agent-harness-runtime.ts @@ -124,6 +124,7 @@ export { appendSqliteSessionTranscriptEvent, hasSqliteSessionTranscriptEvents, loadSqliteSessionTranscriptEvents, + replaceSqliteSessionTranscriptEvents, resolveSqliteSessionTranscriptScopeForPath, } from "../config/sessions/transcript-store.sqlite.js"; export { createSqliteSessionTranscriptLocator } from "../config/sessions/paths.js";