From 3d9577a76b1a74bee72c4e22f7362eae4d404ca2 Mon Sep 17 00:00:00 2001 From: Peter Steinberger Date: Fri, 8 May 2026 11:03:17 +0100 Subject: [PATCH] refactor: stop docker creating session directories --- scripts/docker/install-sh-e2e/run.sh | 71 ++++++++++++++++---------- scripts/docker/setup.sh | 1 - scripts/e2e/lib/onboard/scenario.sh | 4 +- scripts/e2e/npm-telegram-rtt-docker.sh | 2 +- src/docker-setup.e2e.test.ts | 4 +- src/gateway/server-methods/agents.ts | 5 +- 6 files changed, 51 insertions(+), 36 deletions(-) diff --git a/scripts/docker/install-sh-e2e/run.sh b/scripts/docker/install-sh-e2e/run.sh index 9ec80144a27..53b227c7de4 100755 --- a/scripts/docker/install-sh-e2e/run.sh +++ b/scripts/docker/install-sh-e2e/run.sh @@ -305,7 +305,8 @@ run_agent_turn_logged() { local prompt="$4" local out_json="$5" local started_at - SESSION_JSONL="$(session_jsonl_path "$profile" "$session_id")" + SESSION_DB_PATH="$(session_db_path "$profile")" + SESSION_TRANSCRIPT_ID="$session_id" started_at="$(date +%s)" echo "==> Agent turn start: $label ($profile)" run_agent_turn "$profile" "$session_id" "$prompt" "$out_json" @@ -354,13 +355,25 @@ dump_profile_debug() { echo "missing: ${GATEWAY_LOG:-}" fi - echo "---- session transcript ($profile) ----" - if [[ -n "${SESSION_JSONL:-}" && -f "$SESSION_JSONL" ]]; then - tail -n 80 "$SESSION_JSONL" + echo "---- session transcript rows ($profile) ----" + if [[ -n "${SESSION_DB_PATH:-}" && -f "$SESSION_DB_PATH" && -n "${SESSION_TRANSCRIPT_ID:-}" ]]; then + node - <<'NODE' "$SESSION_DB_PATH" "$SESSION_TRANSCRIPT_ID" || true +const { DatabaseSync } = require("node:sqlite"); +const db = new DatabaseSync(process.argv[2], { readOnly: true }); +const rows = db + .prepare( + "select seq, event_json from transcript_events where session_id = ? order by seq desc limit 80", + ) + .all(process.argv[3]); +for (const row of rows.reverse()) { + console.log(`${row.seq}: ${row.event_json}`); +} +db.close(); +NODE else - echo "missing: ${SESSION_JSONL:-}" - if [[ -n "${SESSION_JSONL:-}" ]]; then - ls -la "$(dirname "$SESSION_JSONL")" 2>/dev/null || true + echo "missing: ${SESSION_DB_PATH:-}" + if [[ -n "${SESSION_DB_PATH:-}" ]]; then + ls -la "$(dirname "$SESSION_DB_PATH")" 2>/dev/null || true fi fi @@ -449,15 +462,20 @@ NODE } assert_session_used_tools() { - local jsonl="$1" - shift - node - <<'NODE' "$jsonl" "$@" -const fs = require("node:fs"); -const jsonl = process.argv[2]; -const required = new Set(process.argv.slice(3)); - -const raw = fs.readFileSync(jsonl, "utf8"); -const lines = raw.split("\n").map((l) => l.trim()).filter(Boolean); + local db_path="$1" + local session_id="$2" + shift 2 + node - <<'NODE' "$db_path" "$session_id" "$@" +const { DatabaseSync } = require("node:sqlite"); +const dbPath = process.argv[2]; +const sessionId = process.argv[3]; +const required = new Set(process.argv.slice(4)); +const db = new DatabaseSync(dbPath, { readOnly: true }); +const rows = db + .prepare("select event_json from transcript_events where session_id = ? order by seq asc") + .all(sessionId); +db.close(); +const lines = rows.map((row) => String(row.event_json ?? "")).filter(Boolean); const seen = new Set(); const toolTypes = new Set([ @@ -510,7 +528,7 @@ for (const line of lines) { const entry = JSON.parse(line); walk(entry, null); } catch { - // ignore unparsable lines + // ignore unparsable rows } } @@ -525,10 +543,9 @@ if (missing.length > 0) { NODE } -session_jsonl_path() { +session_db_path() { local profile="$1" - local session_id="$2" - echo "$HOME/.openclaw-${profile}/agents/main/sessions/${session_id}.jsonl" + echo "$HOME/.openclaw-${profile}/agents/main/agent/openclaw-agent.sqlite" } run_profile() { @@ -632,7 +649,8 @@ run_profile() { IMAGE_PNG="$workspace/proof.png" IMAGE_TXT="$workspace/image.txt" SESSION_ID_PREFIX="e2e-tools-${profile}" - SESSION_JSONL="" + SESSION_DB_PATH="" + SESSION_TRANSCRIPT_ID="" PROOF_VALUE="$(node -e 'console.log(require("node:crypto").randomBytes(16).toString("hex"))')" echo -n "$PROOF_VALUE" >"$PROOF_TXT" @@ -769,11 +787,12 @@ run_profile() { phase_mark_start "Verify tool usage via session transcript ($profile)" # Give the gateway a moment to flush transcripts. sleep 1 - assert_session_used_tools "$(session_jsonl_path "$profile" "$TURN2_SESSION_ID")" write - assert_session_used_tools "$(session_jsonl_path "$profile" "$TURN2B_SESSION_ID")" read - assert_session_used_tools "$(session_jsonl_path "$profile" "$TURN3_SESSION_ID")" exec - assert_session_used_tools "$(session_jsonl_path "$profile" "$TURN3B_SESSION_ID")" write - assert_session_used_tools "$(session_jsonl_path "$profile" "$TURN4_SESSION_ID")" image write + assert_session_used_tools "$(session_db_path "$profile")" "$TURN1_SESSION_ID" read + assert_session_used_tools "$(session_db_path "$profile")" "$TURN2_SESSION_ID" write + assert_session_used_tools "$(session_db_path "$profile")" "$TURN2B_SESSION_ID" read + assert_session_used_tools "$(session_db_path "$profile")" "$TURN3_SESSION_ID" exec + assert_session_used_tools "$(session_db_path "$profile")" "$TURN3B_SESSION_ID" write + assert_session_used_tools "$(session_db_path "$profile")" "$TURN4_SESSION_ID" image write phase_mark_passed "Verify tool usage via session transcript ($profile)" cleanup_profile diff --git a/scripts/docker/setup.sh b/scripts/docker/setup.sh index 881505677ea..85ea12d441e 100755 --- a/scripts/docker/setup.sh +++ b/scripts/docker/setup.sh @@ -274,7 +274,6 @@ mkdir -p "$OPENCLAW_WORKSPACE_DIR" # where the container (even as root) cannot create new host subdirectories. mkdir -p "$OPENCLAW_CONFIG_DIR/identity" mkdir -p "$OPENCLAW_CONFIG_DIR/agents/main/agent" -mkdir -p "$OPENCLAW_CONFIG_DIR/agents/main/sessions" export OPENCLAW_CONFIG_DIR export OPENCLAW_WORKSPACE_DIR diff --git a/scripts/e2e/lib/onboard/scenario.sh b/scripts/e2e/lib/onboard/scenario.sh index d4c9fd4c7fe..b0aa92d1b68 100644 --- a/scripts/e2e/lib/onboard/scenario.sh +++ b/scripts/e2e/lib/onboard/scenario.sh @@ -208,9 +208,9 @@ run_case_local_basic() { # Assert config + workspace scaffolding. workspace_dir="$OPENCLAW_STATE_DIR/workspace" - sessions_dir="$OPENCLAW_STATE_DIR/agents/main/sessions" + agent_db_dir="$OPENCLAW_STATE_DIR/agents/main/agent" - openclaw_e2e_assert_dir "$sessions_dir" + openclaw_e2e_assert_dir "$agent_db_dir" for file in AGENTS.md BOOTSTRAP.md IDENTITY.md SOUL.md TOOLS.md USER.md; do openclaw_e2e_assert_file "$workspace_dir/$file" done diff --git a/scripts/e2e/npm-telegram-rtt-docker.sh b/scripts/e2e/npm-telegram-rtt-docker.sh index 0370cdc3a82..fc25bca6348 100755 --- a/scripts/e2e/npm-telegram-rtt-docker.sh +++ b/scripts/e2e/npm-telegram-rtt-docker.sh @@ -161,7 +161,7 @@ for _ in $(seq 1 60); do sleep 1 done -mkdir -p "$(dirname "$config_path")" "$HOME/.openclaw/workspace" "$HOME/.openclaw/agents/main/sessions" "$HOME/workspace" +mkdir -p "$(dirname "$config_path")" "$HOME/.openclaw/workspace" "$HOME/.openclaw/agents/main/agent" "$HOME/workspace" node /app/scripts/e2e/npm-telegram-rtt-config.mjs \ "$config_path" \ diff --git a/src/docker-setup.e2e.test.ts b/src/docker-setup.e2e.test.ts index ffea67ea011..7bed7cf3dd9 100644 --- a/src/docker-setup.e2e.test.ts +++ b/src/docker-setup.e2e.test.ts @@ -358,7 +358,7 @@ describe("scripts/docker/setup.sh", () => { expect(envFile).toContain("OPENCLAW_TZ=Asia/Shanghai"); }); - it("precreates agent data dirs to avoid EACCES in container", async () => { + it("precreates agent database dirs to avoid EACCES in container", async () => { const activeSandbox = requireSandbox(sandbox); const configDir = join(activeSandbox.rootDir, "config-agent-dirs"); const workspaceDir = join(activeSandbox.rootDir, "workspace-agent-dirs"); @@ -371,8 +371,6 @@ describe("scripts/docker/setup.sh", () => { expect(result.status).toBe(0); const agentDirStat = await stat(join(configDir, "agents", "main", "agent")); expect(agentDirStat.isDirectory()).toBe(true); - const sessionsDirStat = await stat(join(configDir, "agents", "main", "sessions")); - expect(sessionsDirStat.isDirectory()).toBe(true); // Verify that a root-user chown step runs before setup. const log = await readDockerLog(activeSandbox); diff --git a/src/gateway/server-methods/agents.ts b/src/gateway/server-methods/agents.ts index 897e5b204b2..7f168e02197 100644 --- a/src/gateway/server-methods/agents.ts +++ b/src/gateway/server-methods/agents.ts @@ -483,15 +483,14 @@ export const agentsHandlers: GatewayRequestHandlers = { const agentDir = resolveAgentDir(nextConfig, agentId); nextConfig = applyAgentConfig(nextConfig, { agentId, agentDir }); - // Ensure workspace & transcripts exist BEFORE writing config so a failure - // here does not leave a broken config entry behind. + // Ensure workspace exists BEFORE writing config so a failure here does not + // leave a broken config entry behind. const skipBootstrap = Boolean(nextConfig.agents?.defaults?.skipBootstrap); await ensureAgentWorkspace({ dir: workspaceDir, ensureBootstrapFiles: !skipBootstrap, skipOptionalBootstrapFiles: nextConfig.agents?.defaults?.skipOptionalBootstrapFiles, }); - await fs.mkdir(resolveSessionTranscriptsDirForAgent(agentId), { recursive: true }); const persistedIdentity = normalizeIdentityForFile(resolveAgentIdentity(nextConfig, agentId)); if (persistedIdentity) {