From 2a6239084feb8ab183fe366d0b6d61beb7d7780e Mon Sep 17 00:00:00 2001 From: Peter Steinberger Date: Sun, 10 May 2026 01:47:50 +0100 Subject: [PATCH] test(release): route Codex live slash checks through chat --- .../gateway-cli-backend.live-helpers.ts | 4 + .../gateway-codex-harness.live.test.ts | 79 ++++++++++++++++--- 2 files changed, 72 insertions(+), 11 deletions(-) diff --git a/src/gateway/gateway-cli-backend.live-helpers.ts b/src/gateway/gateway-cli-backend.live-helpers.ts index 8d08b3e011b..49e2b87c17f 100644 --- a/src/gateway/gateway-cli-backend.live-helpers.ts +++ b/src/gateway/gateway-cli-backend.live-helpers.ts @@ -20,6 +20,7 @@ import { getFreePortBlockWithPermissionFallback } from "../test-utils/ports.js"; import { GATEWAY_CLIENT_MODES, GATEWAY_CLIENT_NAMES } from "../utils/message-channel.js"; import { startGatewayClientWhenEventLoopReady } from "./client-start-readiness.js"; import { GatewayClient, type GatewayClientOptions } from "./client.js"; +import type { EventFrame } from "./protocol/index.js"; // Aggregate docker live runs can contend on startup enough that the gateway // websocket handshake needs a wider budget than the single-provider reruns. @@ -292,6 +293,7 @@ export async function connectTestGatewayClient(params: { maxAttemptTimeoutMs?: number; clientDisplayName?: string | null; requestTimeoutMs?: number; + onEvent?: (evt: EventFrame) => void; onRetry?: (attempt: number, error: Error) => void; }): Promise { const timeoutMs = params.timeoutMs ?? CLI_GATEWAY_CONNECT_TIMEOUT_MS; @@ -331,6 +333,7 @@ async function connectClientOnce(params: { deviceIdentity?: DeviceIdentity; clientDisplayName?: string | null; requestTimeoutMs?: number; + onEvent?: (evt: EventFrame) => void; }): Promise { return await new Promise((resolve, reject) => { let done = false; @@ -367,6 +370,7 @@ async function connectClientOnce(params: { onHelloOk: () => finish({ client }), onConnectError: (error) => finish({ error }), onClose: failWithClose, + onEvent: params.onEvent, }; if (params.clientDisplayName !== null) { clientOptions.clientDisplayName = params.clientDisplayName ?? "vitest-live"; diff --git a/src/gateway/gateway-codex-harness.live.test.ts b/src/gateway/gateway-codex-harness.live.test.ts index 79436798fe2..9d4490b67b1 100644 --- a/src/gateway/gateway-codex-harness.live.test.ts +++ b/src/gateway/gateway-codex-harness.live.test.ts @@ -31,6 +31,7 @@ import { } from "./live-agent-probes.js"; import { restoreLiveEnv, snapshotLiveEnv, type LiveEnvSnapshot } from "./live-env-test-helpers.js"; import { renderSolidColorPngBase64 } from "./live-image-probe.js"; +import type { EventFrame } from "./protocol/index.js"; const LIVE = isLiveTestEnabled(); const CODEX_HARNESS_LIVE = isTruthyEnvValue(process.env.OPENCLAW_LIVE_CODEX_HARNESS); @@ -279,29 +280,31 @@ async function requestAgentText(params: { async function requestCodexCommandText(params: { client: GatewayClient; command: string; + events: EventFrame[]; expectedText: string | string[]; isExpectedText?: (text: string) => boolean; sessionKey: string; }): Promise { - const { extractPayloadText } = await import("./test-helpers.agent-results.js"); - const payload = await params.client.request( - "agent", + const runId = `idem-${randomUUID()}-codex-command`; + const started = await params.client.request( + "chat.send", { sessionKey: params.sessionKey, - idempotencyKey: `idem-${randomUUID()}-codex-command`, + idempotencyKey: runId, message: params.command, - deliver: false, - thinking: "low", - timeout: CODEX_HARNESS_AGENT_TIMEOUT_SECONDS, }, - { expectFinal: true, timeoutMs: CODEX_HARNESS_REQUEST_TIMEOUT_MS }, + { timeoutMs: CODEX_HARNESS_REQUEST_TIMEOUT_MS }, ); - if (payload?.status !== "ok") { + if (started?.status !== "started") { throw new Error( - `codex command ${params.command} failed: status=${String(payload?.status)} payload=${JSON.stringify(payload)}`, + `codex command ${params.command} did not start correctly: ${JSON.stringify(started)}`, ); } - const text = extractPayloadText(payload.result); + const text = await waitForChatFinalText({ + events: params.events, + runId, + timeoutMs: CODEX_HARNESS_REQUEST_TIMEOUT_MS, + }); const expectedTexts = Array.isArray(params.expectedText) ? params.expectedText : [params.expectedText]; @@ -314,6 +317,54 @@ async function requestCodexCommandText(params: { return text; } +async function waitForChatFinalText(params: { + events: EventFrame[]; + runId: string; + timeoutMs: number; +}): Promise { + const deadline = Date.now() + params.timeoutMs; + while (Date.now() < deadline) { + const text = params.events + .map((event) => extractChatFinalText(event, params.runId)) + .find(Boolean); + if (text) { + return text; + } + await delay(50); + } + throw new Error(`timed out waiting for chat final for ${params.runId}`); +} + +function extractChatFinalText(event: EventFrame, runId: string): string | undefined { + if (event.event !== "chat") { + return undefined; + } + const payload = event.payload; + if (!payload || typeof payload !== "object") { + return undefined; + } + const record = payload as Record; + if (record.runId !== runId || record.state !== "final") { + return undefined; + } + const message = record.message; + if (!message || typeof message !== "object") { + return undefined; + } + const messageRecord = message as Record; + if (typeof messageRecord.text === "string" && messageRecord.text.trim()) { + return messageRecord.text; + } + const content = Array.isArray(messageRecord.content) ? messageRecord.content : []; + return content + .map((entry) => + entry && typeof entry === "object" ? (entry as Record).text : undefined, + ) + .filter((entry): entry is string => typeof entry === "string" && entry.trim().length > 0) + .join("\n") + .trim(); +} + async function verifyCodexImageProbe(params: { client: GatewayClient; sessionKey: string; @@ -752,6 +803,7 @@ describeLive("gateway live (Codex harness)", () => { const deviceIdentity = await ensurePairedTestGatewayClientIdentity({ displayName: "vitest-codex-harness-live", }); + const gatewayEvents: EventFrame[] = []; logCodexLiveStep("config-written", { configPath, modelKey, port }); const server = await startGatewayServer(port, { @@ -766,6 +818,9 @@ describeLive("gateway live (Codex harness)", () => { timeoutMs: GATEWAY_CONNECT_TIMEOUT_MS, requestTimeoutMs: CODEX_HARNESS_REQUEST_TIMEOUT_MS, clientDisplayName: "vitest-codex-harness-live", + onEvent: (event) => { + gatewayEvents.push(event); + }, }); logCodexLiveStep("client-connected"); @@ -811,6 +866,7 @@ describeLive("gateway live (Codex harness)", () => { const statusText = await requestCodexCommandText({ client, + events: gatewayEvents, sessionKey, command: "/codex status", expectedText: [...EXPECTED_CODEX_STATUS_COMMAND_TEXT], @@ -820,6 +876,7 @@ describeLive("gateway live (Codex harness)", () => { const modelsText = await requestCodexCommandText({ client, + events: gatewayEvents, sessionKey, command: "/codex models", expectedText: [...EXPECTED_CODEX_MODELS_COMMAND_TEXT],