mirror of
https://github.com/moltbot/moltbot.git
synced 2026-05-06 15:18:58 +00:00
test(qa): add gateway CPU scenario pack
This commit is contained in:
@@ -124,6 +124,16 @@ the fast Matrix and Telegram lanes before release approval.
|
||||
`aimock` starts a local AIMock-backed provider server for experimental
|
||||
fixture and protocol-mock coverage without replacing the scenario-aware
|
||||
`mock-openai` lane.
|
||||
- `pnpm test:gateway:cpu-scenarios`
|
||||
- Runs the gateway startup bench plus a small mock QA Lab scenario pack
|
||||
(`channel-chat-baseline`, `memory-failure-fallback`,
|
||||
`gateway-restart-inflight-run`) and writes a combined CPU observation
|
||||
summary under `.artifacts/gateway-cpu-scenarios/`.
|
||||
- Flags only sustained hot CPU observations by default (`--cpu-core-warn`
|
||||
plus `--hot-wall-warn-ms`), so short startup bursts are recorded as metrics
|
||||
without looking like the minutes-long gateway peg regression.
|
||||
- Uses built `dist` artifacts; run a build first when the checkout does not
|
||||
already have fresh runtime output.
|
||||
- `pnpm openclaw qa suite --runner multipass`
|
||||
- Runs the same QA suite inside a disposable Multipass Linux VM.
|
||||
- Keeps the same scenario-selection behavior as `qa suite` on the host.
|
||||
|
||||
@@ -22,6 +22,7 @@ import { formatQaGatewayLogsForError, redactQaGatewayDebugText } from "./gateway
|
||||
import { startQaGatewayRpcClient } from "./gateway-rpc-client.js";
|
||||
import { splitQaModelRef, type QaProviderMode } from "./model-selection.js";
|
||||
import { resolveQaNodeExecPath } from "./node-exec.js";
|
||||
import { readProcessTreeCpuMs } from "./process-tree-cpu.js";
|
||||
import {
|
||||
normalizeQaProviderModeEnv,
|
||||
QA_LIVE_PROVIDER_CONFIG_PATH_ENV,
|
||||
@@ -825,6 +826,7 @@ export async function startQaGatewayChild(params: {
|
||||
baseUrl,
|
||||
wsUrl,
|
||||
pid: child.pid ?? null,
|
||||
getProcessCpuMs: () => readProcessTreeCpuMs(activeChild.pid ?? null),
|
||||
token: gatewayToken,
|
||||
workspaceDir,
|
||||
tempRoot,
|
||||
|
||||
16
extensions/qa-lab/src/process-tree-cpu.test.ts
Normal file
16
extensions/qa-lab/src/process-tree-cpu.test.ts
Normal file
@@ -0,0 +1,16 @@
|
||||
import { describe, expect, it } from "vitest";
|
||||
import { parsePsCpuTimeMs } from "./process-tree-cpu.js";
|
||||
|
||||
describe("process tree CPU helpers", () => {
|
||||
it("parses ps CPU time strings", () => {
|
||||
expect(parsePsCpuTimeMs("00:01")).toBe(1_000);
|
||||
expect(parsePsCpuTimeMs("01:02")).toBe(62_000);
|
||||
expect(parsePsCpuTimeMs("01:02:03")).toBe(3_723_000);
|
||||
});
|
||||
|
||||
it("rejects malformed ps CPU time strings", () => {
|
||||
expect(parsePsCpuTimeMs("")).toBeNull();
|
||||
expect(parsePsCpuTimeMs("nope")).toBeNull();
|
||||
expect(parsePsCpuTimeMs("1:2:3:4")).toBeNull();
|
||||
});
|
||||
});
|
||||
72
extensions/qa-lab/src/process-tree-cpu.ts
Normal file
72
extensions/qa-lab/src/process-tree-cpu.ts
Normal file
@@ -0,0 +1,72 @@
|
||||
import { spawnSync } from "node:child_process";
|
||||
|
||||
export function parsePsCpuTimeMs(raw: string): number | null {
|
||||
const parts = raw.trim().split(":").map(Number);
|
||||
if (parts.some((part) => !Number.isFinite(part) || part < 0)) {
|
||||
return null;
|
||||
}
|
||||
if (parts.length === 2) {
|
||||
return Math.round((parts[0] * 60 + parts[1]) * 1000);
|
||||
}
|
||||
if (parts.length === 3) {
|
||||
return Math.round((parts[0] * 60 * 60 + parts[1] * 60 + parts[2]) * 1000);
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
export function readProcessTreeCpuMs(rootPid: number | null | undefined): number | null {
|
||||
if (
|
||||
typeof rootPid !== "number" ||
|
||||
!Number.isInteger(rootPid) ||
|
||||
rootPid <= 0 ||
|
||||
process.platform === "win32"
|
||||
) {
|
||||
return null;
|
||||
}
|
||||
const result = spawnSync("ps", ["-eo", "pid=,ppid=,time="], {
|
||||
encoding: "utf8",
|
||||
stdio: ["ignore", "pipe", "ignore"],
|
||||
});
|
||||
if (result.status !== 0) {
|
||||
return null;
|
||||
}
|
||||
|
||||
const childrenByParent = new Map<number, number[]>();
|
||||
const cpuByPid = new Map<number, number>();
|
||||
for (const line of result.stdout.split("\n")) {
|
||||
const match = line.trim().match(/^(\d+)\s+(\d+)\s+(\S+)$/u);
|
||||
if (!match) {
|
||||
continue;
|
||||
}
|
||||
const [, pidRaw, ppidRaw, cpuRaw] = match;
|
||||
const pid = Number(pidRaw);
|
||||
const ppid = Number(ppidRaw);
|
||||
const cpuMs = parsePsCpuTimeMs(cpuRaw ?? "");
|
||||
if (!Number.isInteger(pid) || !Number.isInteger(ppid) || cpuMs === null) {
|
||||
continue;
|
||||
}
|
||||
cpuByPid.set(pid, cpuMs);
|
||||
const children = childrenByParent.get(ppid) ?? [];
|
||||
children.push(pid);
|
||||
childrenByParent.set(ppid, children);
|
||||
}
|
||||
if (!cpuByPid.has(rootPid)) {
|
||||
return null;
|
||||
}
|
||||
|
||||
let totalCpuMs = 0;
|
||||
const seen = new Set<number>();
|
||||
const stack: number[] = [rootPid];
|
||||
while (stack.length > 0) {
|
||||
const pid = stack.pop();
|
||||
if (pid === undefined || seen.has(pid)) {
|
||||
continue;
|
||||
}
|
||||
seen.add(pid);
|
||||
totalCpuMs += cpuByPid.get(pid) ?? 0;
|
||||
for (const childPid of childrenByParent.get(pid) ?? []) {
|
||||
stack.push(childPid);
|
||||
}
|
||||
}
|
||||
return totalCpuMs;
|
||||
}
|
||||
@@ -7,6 +7,7 @@ export type QaRuntimeGatewayClient = {
|
||||
tempRoot: string;
|
||||
workspaceDir: string;
|
||||
runtimeEnv: NodeJS.ProcessEnv;
|
||||
getProcessCpuMs?: () => number | null;
|
||||
restartAfterStateMutation?: (
|
||||
mutateState: (context: {
|
||||
configPath: string;
|
||||
|
||||
@@ -14,6 +14,11 @@ export type QaSuiteSummaryJson = {
|
||||
passed: number;
|
||||
failed: number;
|
||||
};
|
||||
metrics?: {
|
||||
wallMs: number;
|
||||
gatewayProcessCpuMs?: number | null;
|
||||
gatewayCpuCoreRatio?: number | null;
|
||||
};
|
||||
run: {
|
||||
startedAt: string;
|
||||
finishedAt: string;
|
||||
|
||||
@@ -98,4 +98,20 @@ describe("buildQaSuiteSummaryJson", () => {
|
||||
failed: 1,
|
||||
});
|
||||
});
|
||||
|
||||
it("records optional runtime metrics when provided", () => {
|
||||
const json = buildQaSuiteSummaryJson({
|
||||
...baseParams,
|
||||
metrics: {
|
||||
wallMs: 12_000,
|
||||
gatewayProcessCpuMs: 3_400,
|
||||
gatewayCpuCoreRatio: 0.283,
|
||||
},
|
||||
});
|
||||
expect(json.metrics).toEqual({
|
||||
wallMs: 12_000,
|
||||
gatewayProcessCpuMs: 3_400,
|
||||
gatewayCpuCoreRatio: 0.283,
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
@@ -277,6 +277,7 @@ export type QaSuiteSummaryJsonParams = {
|
||||
scenarios: QaSuiteScenarioResult[];
|
||||
startedAt: Date;
|
||||
finishedAt: Date;
|
||||
metrics?: QaSuiteSummaryJson["metrics"];
|
||||
providerMode: QaProviderMode;
|
||||
primaryModel: string;
|
||||
alternateModel: string;
|
||||
@@ -317,6 +318,7 @@ export function buildQaSuiteSummaryJson(params: QaSuiteSummaryJsonParams): QaSui
|
||||
passed: params.scenarios.filter((scenario) => scenario.status === "pass").length,
|
||||
failed: countQaSuiteFailedScenarios(params.scenarios),
|
||||
},
|
||||
...(params.metrics ? { metrics: params.metrics } : {}),
|
||||
run: {
|
||||
startedAt: params.startedAt.toISOString(),
|
||||
finishedAt: params.finishedAt.toISOString(),
|
||||
@@ -340,6 +342,7 @@ async function writeQaSuiteArtifacts(params: {
|
||||
startedAt: Date;
|
||||
finishedAt: Date;
|
||||
scenarios: QaSuiteScenarioResult[];
|
||||
metrics?: QaSuiteSummaryJson["metrics"];
|
||||
transport: QaTransportAdapter;
|
||||
// Reuse the canonical QaProviderMode union instead of re-declaring it
|
||||
// inline. Loop 6 already unified `QaSuiteSummaryJsonParams.providerMode`
|
||||
@@ -376,6 +379,27 @@ async function writeQaSuiteArtifacts(params: {
|
||||
return { report, reportPath, summaryPath };
|
||||
}
|
||||
|
||||
function buildQaSuiteRuntimeMetrics(params: {
|
||||
startedAt: Date;
|
||||
finishedAt: Date;
|
||||
gatewayProcessCpuStartMs: number | null;
|
||||
gatewayProcessCpuEndMs: number | null;
|
||||
}): QaSuiteSummaryJson["metrics"] {
|
||||
const wallMs = Math.max(1, params.finishedAt.getTime() - params.startedAt.getTime());
|
||||
if (params.gatewayProcessCpuStartMs === null || params.gatewayProcessCpuEndMs === null) {
|
||||
return { wallMs };
|
||||
}
|
||||
const gatewayProcessCpuMs = Math.max(
|
||||
0,
|
||||
params.gatewayProcessCpuEndMs - params.gatewayProcessCpuStartMs,
|
||||
);
|
||||
return {
|
||||
wallMs,
|
||||
gatewayProcessCpuMs,
|
||||
gatewayCpuCoreRatio: Math.round((gatewayProcessCpuMs / wallMs) * 1000) / 1000,
|
||||
};
|
||||
}
|
||||
|
||||
export async function runQaSuite(params?: QaSuiteRunParams): Promise<QaSuiteResult> {
|
||||
const startedAt = new Date();
|
||||
const repoRoot = path.resolve(params?.repoRoot ?? process.cwd());
|
||||
@@ -730,6 +754,7 @@ export async function runQaSuite(params?: QaSuiteRunParams): Promise<QaSuiteResu
|
||||
scenarios: liveScenarioOutcomes,
|
||||
});
|
||||
|
||||
const gatewayProcessCpuStartMs = gateway.getProcessCpuMs?.() ?? null;
|
||||
for (const [index, scenario] of selectedCatalogScenarios.entries()) {
|
||||
const scenarioIdForLog = sanitizeQaSuiteProgressValue(scenario.id);
|
||||
writeQaSuiteProgress(
|
||||
@@ -773,6 +798,12 @@ export async function runQaSuite(params?: QaSuiteRunParams): Promise<QaSuiteResu
|
||||
}
|
||||
|
||||
const finishedAt = new Date();
|
||||
const metrics = buildQaSuiteRuntimeMetrics({
|
||||
startedAt,
|
||||
finishedAt,
|
||||
gatewayProcessCpuStartMs,
|
||||
gatewayProcessCpuEndMs: gateway.getProcessCpuMs?.() ?? null,
|
||||
});
|
||||
const failedCount = scenarios.filter((scenario) => scenario.status === "fail").length;
|
||||
if (scenarios.some((scenario) => scenario.status === "fail")) {
|
||||
preserveGatewayRuntimeDir = path.join(outputDir, "artifacts", "gateway-runtime");
|
||||
@@ -789,6 +820,7 @@ export async function runQaSuite(params?: QaSuiteRunParams): Promise<QaSuiteResu
|
||||
startedAt,
|
||||
finishedAt,
|
||||
scenarios,
|
||||
metrics,
|
||||
transport,
|
||||
providerMode,
|
||||
primaryModel,
|
||||
|
||||
@@ -1513,6 +1513,7 @@
|
||||
"test:fast": "node scripts/run-vitest.mjs run --config test/vitest/vitest.unit.config.ts",
|
||||
"test:force": "node --import tsx scripts/test-force.ts",
|
||||
"test:gateway": "OPENCLAW_GATEWAY_PROJECT_SHARDS=1 node scripts/run-vitest.mjs run --config test/vitest/vitest.gateway.config.ts",
|
||||
"test:gateway:cpu-scenarios": "node scripts/check-gateway-cpu-scenarios.mjs",
|
||||
"test:gateway:watch-regression": "node scripts/check-gateway-watch-regression.mjs",
|
||||
"test:install:e2e": "bash scripts/test-install-sh-e2e-docker.sh",
|
||||
"test:install:e2e:anthropic": "OPENCLAW_E2E_MODELS=anthropic bash scripts/test-install-sh-e2e-docker.sh",
|
||||
|
||||
@@ -29,8 +29,8 @@ execution:
|
||||
kind: flow
|
||||
summary: Toggle reasoning display and GPT-5.5 thinking between off/none and medium, then verify visible reasoning only on the medium turn.
|
||||
config:
|
||||
requiredLiveProvider: openai
|
||||
requiredLiveModel: gpt-5.5
|
||||
requiredProvider: openai
|
||||
requiredModel: gpt-5.5
|
||||
offDirective: /think off
|
||||
maxDirective: /think medium
|
||||
reasoningDirective: /reasoning on
|
||||
@@ -58,7 +58,7 @@ steps:
|
||||
value:
|
||||
expr: splitModelRef(env.primaryModel)
|
||||
- assert:
|
||||
expr: "env.providerMode !== 'live-frontier' || (selected?.provider === config.requiredLiveProvider && selected?.model === config.requiredLiveModel)"
|
||||
expr: "env.providerMode !== 'live-frontier' || (selected?.provider === config.requiredProvider && selected?.model === config.requiredModel)"
|
||||
message:
|
||||
expr: "`expected live GPT-5.5, got ${env.primaryModel}`"
|
||||
- call: state.addInboundMessage
|
||||
|
||||
@@ -40,6 +40,8 @@ execution:
|
||||
summary: Select Anthropic, set adaptive, switch to OpenAI and verify medium fallback, then set xhigh and verify high fallback on a non-xhigh model.
|
||||
config:
|
||||
requiredProviderMode: live-frontier
|
||||
requiredProvider: openai
|
||||
requiredModel: gpt-5.5
|
||||
anthropicModelRef: anthropic/claude-sonnet-4-6
|
||||
openAiXhighModelRef: openai/gpt-5.5
|
||||
noXhighModelRef: anthropic/claude-sonnet-4-6
|
||||
|
||||
@@ -21,6 +21,8 @@ type ProbeResult = {
|
||||
};
|
||||
|
||||
type GatewaySample = {
|
||||
cpuCoreRatio: number | null;
|
||||
cpuMs: number | null;
|
||||
exitCode: number | null;
|
||||
firstOutputMs: number | null;
|
||||
healthz: ProbeResult;
|
||||
@@ -46,6 +48,8 @@ type CaseResult = {
|
||||
samples: GatewaySample[];
|
||||
summary: {
|
||||
firstOutputMs: SummaryStats | null;
|
||||
cpuCoreRatio: SummaryStats | null;
|
||||
cpuMs: SummaryStats | null;
|
||||
healthzMs: SummaryStats | null;
|
||||
maxRssMb: SummaryStats | null;
|
||||
readyLogMs: SummaryStats | null;
|
||||
@@ -269,6 +273,16 @@ function summarizeCase(benchCase: GatewayBenchCase, samples: GatewaySample[]): C
|
||||
.map((sample) => sample.firstOutputMs)
|
||||
.filter((value): value is number => typeof value === "number"),
|
||||
),
|
||||
cpuCoreRatio: summarizeNumbers(
|
||||
samples
|
||||
.map((sample) => sample.cpuCoreRatio)
|
||||
.filter((value): value is number => typeof value === "number"),
|
||||
),
|
||||
cpuMs: summarizeNumbers(
|
||||
samples
|
||||
.map((sample) => sample.cpuMs)
|
||||
.filter((value): value is number => typeof value === "number"),
|
||||
),
|
||||
healthzMs: summarizeNumbers(
|
||||
samples
|
||||
.map((sample) => sample.healthz.ms)
|
||||
@@ -308,6 +322,13 @@ function formatMb(value: number | null): string {
|
||||
return `${value.toFixed(1)}MB`;
|
||||
}
|
||||
|
||||
function formatRatio(value: number | null): string {
|
||||
if (value == null) {
|
||||
return "n/a";
|
||||
}
|
||||
return value.toFixed(3);
|
||||
}
|
||||
|
||||
function formatStats(stats: SummaryStats | null): string {
|
||||
if (!stats) {
|
||||
return "n/a";
|
||||
@@ -322,6 +343,13 @@ function formatMemoryStats(stats: SummaryStats | null): string {
|
||||
return `p50=${formatMb(stats.p50)} avg=${formatMb(stats.avg)} min=${formatMb(stats.min)} max=${formatMb(stats.max)}`;
|
||||
}
|
||||
|
||||
function formatRatioStats(stats: SummaryStats | null): string {
|
||||
if (!stats) {
|
||||
return "n/a";
|
||||
}
|
||||
return `p50=${formatRatio(stats.p50)} avg=${formatRatio(stats.avg)} min=${formatRatio(stats.min)} max=${formatRatio(stats.max)}`;
|
||||
}
|
||||
|
||||
async function getFreePort(): Promise<number> {
|
||||
return new Promise((resolve, reject) => {
|
||||
const server = createServer();
|
||||
@@ -547,6 +575,71 @@ function readProcessRssMb(pid: number | undefined): number | null {
|
||||
return Number.isFinite(rssKb) && rssKb > 0 ? rssKb / 1024 : null;
|
||||
}
|
||||
|
||||
function parsePsCpuTimeMs(raw: string): number | null {
|
||||
const parts = raw.trim().split(":").map(Number);
|
||||
if (parts.some((part) => !Number.isFinite(part) || part < 0)) {
|
||||
return null;
|
||||
}
|
||||
if (parts.length === 2) {
|
||||
return Math.round((parts[0] * 60 + parts[1]) * 1000);
|
||||
}
|
||||
if (parts.length === 3) {
|
||||
return Math.round((parts[0] * 60 * 60 + parts[1] * 60 + parts[2]) * 1000);
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
function readProcessTreeCpuMs(rootPid: number | undefined): number | null {
|
||||
if (!rootPid || process.platform === "win32") {
|
||||
return null;
|
||||
}
|
||||
const result = spawnSync("ps", ["-eo", "pid=,ppid=,time="], {
|
||||
encoding: "utf8",
|
||||
stdio: ["ignore", "pipe", "ignore"],
|
||||
});
|
||||
if (result.status !== 0) {
|
||||
return null;
|
||||
}
|
||||
|
||||
const childrenByParent = new Map<number, number[]>();
|
||||
const cpuByPid = new Map<number, number>();
|
||||
for (const line of result.stdout.split("\n")) {
|
||||
const match = line.trim().match(/^(\d+)\s+(\d+)\s+(\S+)$/u);
|
||||
if (!match) {
|
||||
continue;
|
||||
}
|
||||
const pid = Number(match[1]);
|
||||
const ppid = Number(match[2]);
|
||||
const cpuMs = parsePsCpuTimeMs(match[3]);
|
||||
if (!Number.isInteger(pid) || !Number.isInteger(ppid) || cpuMs === null) {
|
||||
continue;
|
||||
}
|
||||
cpuByPid.set(pid, cpuMs);
|
||||
const children = childrenByParent.get(ppid) ?? [];
|
||||
children.push(pid);
|
||||
childrenByParent.set(ppid, children);
|
||||
}
|
||||
if (!cpuByPid.has(rootPid)) {
|
||||
return null;
|
||||
}
|
||||
|
||||
let totalCpuMs = 0;
|
||||
const seen = new Set<number>();
|
||||
const stack = [rootPid];
|
||||
while (stack.length > 0) {
|
||||
const pid = stack.pop();
|
||||
if (!pid || seen.has(pid)) {
|
||||
continue;
|
||||
}
|
||||
seen.add(pid);
|
||||
totalCpuMs += cpuByPid.get(pid) ?? 0;
|
||||
for (const childPid of childrenByParent.get(pid) ?? []) {
|
||||
stack.push(childPid);
|
||||
}
|
||||
}
|
||||
return totalCpuMs;
|
||||
}
|
||||
|
||||
async function runGatewaySample(options: {
|
||||
benchCase: GatewayBenchCase;
|
||||
entry: string;
|
||||
@@ -583,6 +676,7 @@ async function runGatewaySample(options: {
|
||||
],
|
||||
{ cwd: process.cwd(), detached: process.platform !== "win32", env },
|
||||
);
|
||||
const cpuStartMs = readProcessTreeCpuMs(child.pid);
|
||||
const sampleRss = () => {
|
||||
const rssMb = readProcessRssMb(child.pid);
|
||||
if (rssMb != null) {
|
||||
@@ -636,6 +730,10 @@ async function runGatewaySample(options: {
|
||||
startAt,
|
||||
}),
|
||||
]);
|
||||
const readyAt = performance.now();
|
||||
const cpuEndMs = readProcessTreeCpuMs(child.pid);
|
||||
const cpuMs = cpuStartMs == null || cpuEndMs == null ? null : Math.max(0, cpuEndMs - cpuStartMs);
|
||||
const cpuCoreRatio = cpuMs == null ? null : cpuMs / Math.max(1, readyAt - startAt);
|
||||
const exit = await stopChild(child);
|
||||
clearInterval(rssTimer);
|
||||
sampleRss();
|
||||
@@ -643,6 +741,8 @@ async function runGatewaySample(options: {
|
||||
rmSync(root, { force: true, maxRetries: 3, recursive: true, retryDelay: 100 });
|
||||
|
||||
return {
|
||||
cpuCoreRatio,
|
||||
cpuMs,
|
||||
exitCode: exit.exitCode,
|
||||
firstOutputMs,
|
||||
healthz,
|
||||
@@ -673,11 +773,11 @@ async function runCase(options: {
|
||||
if (index >= options.warmup) {
|
||||
samples.push(sample);
|
||||
console.log(
|
||||
`[gateway-startup-bench] ${options.benchCase.id} run ${samples.length}/${options.runs}: healthz=${formatMs(sample.healthz.ms)} readyz=${formatMs(sample.readyz.ms)} readyLog=${formatMs(sample.readyLogMs)} rss=${formatMb(sample.maxRssMb)}`,
|
||||
`[gateway-startup-bench] ${options.benchCase.id} run ${samples.length}/${options.runs}: healthz=${formatMs(sample.healthz.ms)} readyz=${formatMs(sample.readyz.ms)} readyLog=${formatMs(sample.readyLogMs)} cpu=${formatMs(sample.cpuMs)} cpuCore=${formatRatio(sample.cpuCoreRatio)} rss=${formatMb(sample.maxRssMb)}`,
|
||||
);
|
||||
} else {
|
||||
console.log(
|
||||
`[gateway-startup-bench] ${options.benchCase.id} warmup ${index + 1}/${options.warmup}: healthz=${formatMs(sample.healthz.ms)} readyz=${formatMs(sample.readyz.ms)} rss=${formatMb(sample.maxRssMb)}`,
|
||||
`[gateway-startup-bench] ${options.benchCase.id} warmup ${index + 1}/${options.warmup}: healthz=${formatMs(sample.healthz.ms)} readyz=${formatMs(sample.readyz.ms)} cpu=${formatMs(sample.cpuMs)} cpuCore=${formatRatio(sample.cpuCoreRatio)} rss=${formatMb(sample.maxRssMb)}`,
|
||||
);
|
||||
}
|
||||
}
|
||||
@@ -687,6 +787,8 @@ async function runCase(options: {
|
||||
function printResult(result: CaseResult): void {
|
||||
console.log(`\n${result.name} (${result.id})`);
|
||||
console.log(` first output: ${formatStats(result.summary.firstOutputMs)}`);
|
||||
console.log(` CPU: ${formatStats(result.summary.cpuMs)}`);
|
||||
console.log(` CPU core: ${formatRatioStats(result.summary.cpuCoreRatio)}`);
|
||||
console.log(` /healthz: ${formatStats(result.summary.healthzMs)}`);
|
||||
console.log(` ready log: ${formatStats(result.summary.readyLogMs)}`);
|
||||
console.log(` /readyz: ${formatStats(result.summary.readyzMs)}`);
|
||||
|
||||
280
scripts/check-gateway-cpu-scenarios.mjs
Normal file
280
scripts/check-gateway-cpu-scenarios.mjs
Normal file
@@ -0,0 +1,280 @@
|
||||
#!/usr/bin/env node
|
||||
|
||||
import { spawnSync } from "node:child_process";
|
||||
import fs from "node:fs";
|
||||
import path from "node:path";
|
||||
import process from "node:process";
|
||||
|
||||
const DEFAULT_STARTUP_CASES = ["default", "oneInternalHook", "allInternalHooks"];
|
||||
const DEFAULT_QA_SCENARIOS = [
|
||||
"channel-chat-baseline",
|
||||
"memory-failure-fallback",
|
||||
"gateway-restart-inflight-run",
|
||||
];
|
||||
const DEFAULT_CPU_CORE_WARN = 0.9;
|
||||
const DEFAULT_HOT_WALL_WARN_MS = 30_000;
|
||||
|
||||
function parseArgs(argv) {
|
||||
const options = {
|
||||
outputDir: path.join(
|
||||
process.cwd(),
|
||||
".artifacts",
|
||||
"gateway-cpu-scenarios",
|
||||
new Date().toISOString().replace(/[:.]/g, "-"),
|
||||
),
|
||||
startupCases: [],
|
||||
qaScenarios: [],
|
||||
runs: 1,
|
||||
warmup: 0,
|
||||
skipStartup: false,
|
||||
skipQa: false,
|
||||
cpuCoreWarn: DEFAULT_CPU_CORE_WARN,
|
||||
hotWallWarnMs: DEFAULT_HOT_WALL_WARN_MS,
|
||||
};
|
||||
for (let index = 0; index < argv.length; index += 1) {
|
||||
const arg = argv[index];
|
||||
const readValue = () => {
|
||||
const value = argv[index + 1];
|
||||
if (!value) {
|
||||
throw new Error(`Missing value for ${arg}`);
|
||||
}
|
||||
index += 1;
|
||||
return value;
|
||||
};
|
||||
switch (arg) {
|
||||
case "--output-dir":
|
||||
options.outputDir = path.resolve(readValue());
|
||||
break;
|
||||
case "--startup-case":
|
||||
options.startupCases.push(readValue());
|
||||
break;
|
||||
case "--qa-scenario":
|
||||
options.qaScenarios.push(readValue());
|
||||
break;
|
||||
case "--runs":
|
||||
options.runs = parsePositiveInt(readValue(), "--runs");
|
||||
break;
|
||||
case "--warmup":
|
||||
options.warmup = parseNonNegativeInt(readValue(), "--warmup");
|
||||
break;
|
||||
case "--cpu-core-warn":
|
||||
options.cpuCoreWarn = parsePositiveNumber(readValue(), "--cpu-core-warn");
|
||||
break;
|
||||
case "--hot-wall-warn-ms":
|
||||
options.hotWallWarnMs = parsePositiveInt(readValue(), "--hot-wall-warn-ms");
|
||||
break;
|
||||
case "--skip-startup":
|
||||
options.skipStartup = true;
|
||||
break;
|
||||
case "--skip-qa":
|
||||
options.skipQa = true;
|
||||
break;
|
||||
case "--help":
|
||||
printHelp();
|
||||
process.exit(0);
|
||||
break;
|
||||
default:
|
||||
throw new Error(`Unknown argument: ${arg}`);
|
||||
}
|
||||
}
|
||||
if (options.startupCases.length === 0) {
|
||||
options.startupCases = [...DEFAULT_STARTUP_CASES];
|
||||
}
|
||||
if (options.qaScenarios.length === 0) {
|
||||
options.qaScenarios = [...DEFAULT_QA_SCENARIOS];
|
||||
}
|
||||
return options;
|
||||
}
|
||||
|
||||
function parsePositiveInt(raw, label) {
|
||||
const value = Number(raw);
|
||||
if (!Number.isInteger(value) || value < 1) {
|
||||
throw new Error(`${label} must be a positive integer`);
|
||||
}
|
||||
return value;
|
||||
}
|
||||
|
||||
function parseNonNegativeInt(raw, label) {
|
||||
const value = Number(raw);
|
||||
if (!Number.isInteger(value) || value < 0) {
|
||||
throw new Error(`${label} must be a non-negative integer`);
|
||||
}
|
||||
return value;
|
||||
}
|
||||
|
||||
function parsePositiveNumber(raw, label) {
|
||||
const value = Number(raw);
|
||||
if (!Number.isFinite(value) || value <= 0) {
|
||||
throw new Error(`${label} must be a positive number`);
|
||||
}
|
||||
return value;
|
||||
}
|
||||
|
||||
function printHelp() {
|
||||
console.log(`Usage: pnpm test:gateway:cpu-scenarios [options]
|
||||
|
||||
Runs a small gateway CPU scenario suite against built dist artifacts.
|
||||
|
||||
Options:
|
||||
--output-dir <path> Artifact directory
|
||||
--startup-case <id> Startup bench case, repeatable
|
||||
--qa-scenario <id> QA Lab scenario, repeatable
|
||||
--runs <count> Startup bench runs per case (default: 1)
|
||||
--warmup <count> Startup bench warmup runs per case (default: 0)
|
||||
--cpu-core-warn <ratio> Hot CPU observation threshold (default: 0.9)
|
||||
--hot-wall-warn-ms <ms> Minimum wall time for hot CPU observations (default: 30000)
|
||||
--skip-startup Skip startup bench
|
||||
--skip-qa Skip QA Lab scenario smoke
|
||||
`);
|
||||
}
|
||||
|
||||
function readJsonIfExists(filePath) {
|
||||
if (!fs.existsSync(filePath)) {
|
||||
return null;
|
||||
}
|
||||
return JSON.parse(fs.readFileSync(filePath, "utf8"));
|
||||
}
|
||||
|
||||
function runStep(name, command, args) {
|
||||
console.error(`[gateway-cpu] start ${name}`);
|
||||
const result = spawnSync(command, args, {
|
||||
cwd: process.cwd(),
|
||||
env: process.env,
|
||||
stdio: "inherit",
|
||||
});
|
||||
const status = result.status ?? (result.signal ? 1 : 0);
|
||||
console.error(`[gateway-cpu] ${status === 0 ? "pass" : "fail"} ${name}`);
|
||||
return { name, status, signal: result.signal ?? null };
|
||||
}
|
||||
|
||||
function pnpmCommand() {
|
||||
return process.platform === "win32" ? "pnpm.cmd" : "pnpm";
|
||||
}
|
||||
|
||||
function toRepoRelativePath(absolutePath) {
|
||||
const relativePath = path.relative(process.cwd(), absolutePath);
|
||||
if (!relativePath || relativePath.startsWith("..") || path.isAbsolute(relativePath)) {
|
||||
throw new Error(`Output path must stay inside the repo root: ${absolutePath}`);
|
||||
}
|
||||
return relativePath;
|
||||
}
|
||||
|
||||
function collectObservations(params) {
|
||||
const observations = [];
|
||||
for (const result of params.startup?.results ?? []) {
|
||||
const cpuCoreMax = result.summary?.cpuCoreRatio?.max;
|
||||
const wallMax = result.summary?.readyz?.max ?? result.summary?.healthz?.max;
|
||||
if (
|
||||
typeof cpuCoreMax === "number" &&
|
||||
typeof wallMax === "number" &&
|
||||
cpuCoreMax >= params.cpuCoreWarn &&
|
||||
wallMax >= params.hotWallWarnMs
|
||||
) {
|
||||
observations.push({
|
||||
kind: "startup-cpu-hot",
|
||||
id: result.id,
|
||||
cpuCoreRatioMax: cpuCoreMax,
|
||||
wallMsMax: wallMax,
|
||||
});
|
||||
}
|
||||
}
|
||||
const qaCpuCoreRatio = params.qa?.metrics?.gatewayCpuCoreRatio;
|
||||
const qaWallMs = params.qa?.metrics?.wallMs;
|
||||
if (
|
||||
typeof qaCpuCoreRatio === "number" &&
|
||||
typeof qaWallMs === "number" &&
|
||||
qaCpuCoreRatio >= params.cpuCoreWarn &&
|
||||
qaWallMs >= params.hotWallWarnMs
|
||||
) {
|
||||
observations.push({
|
||||
kind: "qa-cpu-hot",
|
||||
id: "qa-suite",
|
||||
cpuCoreRatio: qaCpuCoreRatio,
|
||||
wallMs: qaWallMs,
|
||||
});
|
||||
}
|
||||
return observations;
|
||||
}
|
||||
|
||||
async function main() {
|
||||
const options = parseArgs(process.argv.slice(2));
|
||||
fs.mkdirSync(options.outputDir, { recursive: true });
|
||||
|
||||
const startupOutput = path.join(options.outputDir, "gateway-startup-bench.json");
|
||||
const qaOutputDir = path.join(options.outputDir, "qa-suite");
|
||||
const qaOutputArg = toRepoRelativePath(qaOutputDir);
|
||||
const steps = [];
|
||||
|
||||
if (!options.skipStartup) {
|
||||
steps.push(
|
||||
runStep("startup bench", process.execPath, [
|
||||
"--import",
|
||||
"tsx",
|
||||
"scripts/bench-gateway-startup.ts",
|
||||
"--runs",
|
||||
String(options.runs),
|
||||
"--warmup",
|
||||
String(options.warmup),
|
||||
"--output",
|
||||
startupOutput,
|
||||
...options.startupCases.flatMap((id) => ["--case", id]),
|
||||
]),
|
||||
);
|
||||
}
|
||||
|
||||
if (!options.skipQa) {
|
||||
steps.push(
|
||||
runStep("qa suite", pnpmCommand(), [
|
||||
"openclaw",
|
||||
"qa",
|
||||
"suite",
|
||||
"--provider-mode",
|
||||
"mock-openai",
|
||||
"--concurrency",
|
||||
"1",
|
||||
"--output-dir",
|
||||
qaOutputArg,
|
||||
...options.qaScenarios.flatMap((id) => ["--scenario", id]),
|
||||
]),
|
||||
);
|
||||
}
|
||||
|
||||
const startup = readJsonIfExists(startupOutput);
|
||||
const qa = readJsonIfExists(path.join(qaOutputDir, "qa-suite-summary.json"));
|
||||
const observations = collectObservations({
|
||||
startup,
|
||||
qa,
|
||||
cpuCoreWarn: options.cpuCoreWarn,
|
||||
hotWallWarnMs: options.hotWallWarnMs,
|
||||
});
|
||||
const summary = {
|
||||
generatedAt: new Date().toISOString(),
|
||||
outputDir: options.outputDir,
|
||||
startupOutput: fs.existsSync(startupOutput) ? startupOutput : null,
|
||||
qaSummary: fs.existsSync(path.join(qaOutputDir, "qa-suite-summary.json"))
|
||||
? path.join(qaOutputDir, "qa-suite-summary.json")
|
||||
: null,
|
||||
options: {
|
||||
startupCases: options.startupCases,
|
||||
qaScenarios: options.qaScenarios,
|
||||
runs: options.runs,
|
||||
warmup: options.warmup,
|
||||
cpuCoreWarn: options.cpuCoreWarn,
|
||||
hotWallWarnMs: options.hotWallWarnMs,
|
||||
},
|
||||
steps,
|
||||
observations,
|
||||
};
|
||||
const summaryPath = path.join(options.outputDir, "summary.json");
|
||||
fs.writeFileSync(summaryPath, `${JSON.stringify(summary, null, 2)}\n`);
|
||||
console.log(JSON.stringify(summary, null, 2));
|
||||
|
||||
if (steps.some((step) => step.status !== 0)) {
|
||||
process.exitCode = 1;
|
||||
}
|
||||
}
|
||||
|
||||
main().catch((error) => {
|
||||
console.error(error instanceof Error ? error.stack : String(error));
|
||||
process.exitCode = 1;
|
||||
});
|
||||
Reference in New Issue
Block a user