test(perf): slim ios team-id harness and add perf budget guard

This commit is contained in:
Peter Steinberger
2026-03-03 00:20:34 +00:00
parent 4b3d9f4fb2
commit d37ad9d866
3 changed files with 205 additions and 10 deletions

View File

@@ -152,6 +152,7 @@
"test:install:smoke": "bash scripts/test-install-sh-docker.sh",
"test:live": "OPENCLAW_LIVE_TEST=1 CLAWDBOT_LIVE_TEST=1 vitest run --config vitest.live.config.ts",
"test:macmini": "OPENCLAW_TEST_VM_FORKS=0 OPENCLAW_TEST_PROFILE=serial node scripts/test-parallel.mjs",
"test:perf:budget": "node scripts/test-perf-budget.mjs",
"test:sectriage": "pnpm exec vitest run --config vitest.gateway.config.ts && vitest run --config vitest.unit.config.ts --exclude src/daemon/launchd.integration.test.ts --exclude src/process/exec.test.ts",
"test:ui": "pnpm lint:ui:no-raw-window-open && pnpm --dir ui test",
"test:voicecall:closedloop": "vitest run extensions/voice-call/src/manager.test.ts extensions/voice-call/src/media-stream.test.ts src/plugins/voice-call.plugin.test.ts --maxWorkers=1",

View File

@@ -0,0 +1,127 @@
import { spawnSync } from "node:child_process";
import fs from "node:fs";
import os from "node:os";
import path from "node:path";
function readEnvNumber(name) {
const raw = process.env[name]?.trim();
if (!raw) {
return null;
}
const parsed = Number.parseFloat(raw);
return Number.isFinite(parsed) ? parsed : null;
}
function parseArgs(argv) {
const args = {
config: "vitest.unit.config.ts",
maxWallMs: readEnvNumber("OPENCLAW_TEST_PERF_MAX_WALL_MS"),
baselineWallMs: readEnvNumber("OPENCLAW_TEST_PERF_BASELINE_WALL_MS"),
maxRegressionPct: readEnvNumber("OPENCLAW_TEST_PERF_MAX_REGRESSION_PCT") ?? 10,
};
for (let i = 0; i < argv.length; i += 1) {
const arg = argv[i];
if (arg === "--config") {
args.config = argv[i + 1] ?? args.config;
i += 1;
continue;
}
if (arg === "--max-wall-ms") {
const parsed = Number.parseFloat(argv[i + 1] ?? "");
if (Number.isFinite(parsed)) {
args.maxWallMs = parsed;
}
i += 1;
continue;
}
if (arg === "--baseline-wall-ms") {
const parsed = Number.parseFloat(argv[i + 1] ?? "");
if (Number.isFinite(parsed)) {
args.baselineWallMs = parsed;
}
i += 1;
continue;
}
if (arg === "--max-regression-pct") {
const parsed = Number.parseFloat(argv[i + 1] ?? "");
if (Number.isFinite(parsed)) {
args.maxRegressionPct = parsed;
}
i += 1;
continue;
}
}
return args;
}
function formatMs(ms) {
return `${ms.toFixed(1)}ms`;
}
const opts = parseArgs(process.argv.slice(2));
const reportPath = path.join(os.tmpdir(), `openclaw-vitest-perf-${Date.now()}.json`);
const cmd = [
"vitest",
"run",
"--config",
opts.config,
"--reporter=json",
"--outputFile",
reportPath,
];
const startedAt = process.hrtime.bigint();
const run = spawnSync("pnpm", cmd, {
stdio: "inherit",
env: process.env,
});
const elapsedMs = Number(process.hrtime.bigint() - startedAt) / 1_000_000;
if (run.status !== 0) {
process.exit(run.status ?? 1);
}
let totalFileDurationMs = 0;
let fileCount = 0;
try {
const report = JSON.parse(fs.readFileSync(reportPath, "utf8"));
for (const result of report.testResults ?? []) {
if (typeof result.startTime === "number" && typeof result.endTime === "number") {
totalFileDurationMs += Math.max(0, result.endTime - result.startTime);
fileCount += 1;
}
}
} catch {
// Keep budget checks based on wall time when JSON parsing fails.
}
const allowedByBaseline =
opts.baselineWallMs !== null
? opts.baselineWallMs * (1 + (opts.maxRegressionPct ?? 0) / 100)
: null;
let failed = false;
if (opts.maxWallMs !== null && elapsedMs > opts.maxWallMs) {
console.error(
`[test-perf-budget] wall time ${formatMs(elapsedMs)} exceeded max ${formatMs(opts.maxWallMs)}.`,
);
failed = true;
}
if (allowedByBaseline !== null && elapsedMs > allowedByBaseline) {
console.error(
`[test-perf-budget] wall time ${formatMs(elapsedMs)} exceeded baseline budget ${formatMs(
allowedByBaseline,
)} (baseline ${formatMs(opts.baselineWallMs ?? 0)}, +${String(opts.maxRegressionPct)}%).`,
);
failed = true;
}
console.log(
`[test-perf-budget] config=${opts.config} wall=${formatMs(elapsedMs)} file-sum=${formatMs(
totalFileDurationMs,
)} files=${String(fileCount)}`,
);
if (failed) {
process.exit(1);
}

View File

@@ -16,6 +16,60 @@ let sharedHomeDir = "";
let sharedHomeBinDir = "";
let sharedFakePythonPath = "";
const runScriptCache = new Map<string, { ok: boolean; stdout: string; stderr: string }>();
type TeamCandidate = {
teamId: string;
isFree: boolean;
teamName: string;
};
function parseTeamCandidateRows(raw: string): TeamCandidate[] {
return raw
.split("\n")
.map((line) => line.replace(/\r/g, "").trim())
.filter(Boolean)
.map((line) => line.split("\t"))
.filter((parts) => parts.length >= 3)
.map((parts) => ({
teamId: parts[0] ?? "",
isFree: (parts[1] ?? "0") === "1",
teamName: parts[2] ?? "",
}))
.filter((candidate) => candidate.teamId.length > 0);
}
function pickTeamIdFromCandidates(params: {
candidates: TeamCandidate[];
preferredTeamId?: string;
preferredTeamName?: string;
preferNonFreeTeam?: boolean;
}): string | undefined {
const preferredTeamId = (params.preferredTeamId ?? "").trim();
if (preferredTeamId) {
const preferred = params.candidates.find((candidate) => candidate.teamId === preferredTeamId);
if (preferred) {
return preferred.teamId;
}
}
const preferredTeamName = (params.preferredTeamName ?? "").trim().toLowerCase();
if (preferredTeamName) {
const preferredByName = params.candidates.find(
(candidate) => candidate.teamName.trim().toLowerCase() === preferredTeamName,
);
if (preferredByName) {
return preferredByName.teamId;
}
}
if (params.preferNonFreeTeam !== false) {
const paid = params.candidates.find((candidate) => !candidate.isFree);
if (paid) {
return paid.teamId;
}
}
return params.candidates[0]?.teamId;
}
async function writeExecutable(filePath: string, body: string): Promise<void> {
await writeFile(filePath, body, "utf8");
@@ -133,19 +187,32 @@ printf 'BBBBB22222\\t0\\tBeta Team\\r\\n'`,
await rm(fixtureRoot, { recursive: true, force: true });
});
it("resolves fallback and preferred team IDs from Xcode team listings", async () => {
const fallbackResult = runScript(sharedHomeDir, {
IOS_PYTHON_BIN: sharedFakePythonPath,
it("parses team listings and prioritizes preferred IDs without shelling out", () => {
const rows = parseTeamCandidateRows(
"AAAAA11111\t1\tAlpha Team\r\nBBBBB22222\t0\tBeta Team\r\n",
);
expect(rows).toStrictEqual([
{ teamId: "AAAAA11111", isFree: true, teamName: "Alpha Team" },
{ teamId: "BBBBB22222", isFree: false, teamName: "Beta Team" },
]);
const preferred = pickTeamIdFromCandidates({
candidates: rows,
preferredTeamId: "BBBBB22222",
});
expect(preferred).toBe("BBBBB22222");
const fallback = pickTeamIdFromCandidates({
candidates: rows,
preferredTeamId: "CCCCCC3333",
});
expect(fallback).toBe("BBBBB22222");
});
it("resolves a fallback team ID from Xcode team listings (smoke)", async () => {
const fallbackResult = runScript(sharedHomeDir, { IOS_PYTHON_BIN: sharedFakePythonPath });
expect(fallbackResult.ok).toBe(true);
expect(fallbackResult.stdout).toBe("AAAAA11111");
const crlfResult = runScript(sharedHomeDir, {
IOS_PYTHON_BIN: sharedFakePythonPath,
IOS_PREFERRED_TEAM_ID: "BBBBB22222",
});
expect(crlfResult.ok).toBe(true);
expect(crlfResult.stdout).toBe("BBBBB22222");
});
it("prints actionable guidance when Xcode account exists but no Team ID is resolvable", async () => {