diff --git a/package.json b/package.json index fe9a767f9ed..bcb5546a6aa 100644 --- a/package.json +++ b/package.json @@ -152,6 +152,7 @@ "test:install:smoke": "bash scripts/test-install-sh-docker.sh", "test:live": "OPENCLAW_LIVE_TEST=1 CLAWDBOT_LIVE_TEST=1 vitest run --config vitest.live.config.ts", "test:macmini": "OPENCLAW_TEST_VM_FORKS=0 OPENCLAW_TEST_PROFILE=serial node scripts/test-parallel.mjs", + "test:perf:budget": "node scripts/test-perf-budget.mjs", "test:sectriage": "pnpm exec vitest run --config vitest.gateway.config.ts && vitest run --config vitest.unit.config.ts --exclude src/daemon/launchd.integration.test.ts --exclude src/process/exec.test.ts", "test:ui": "pnpm lint:ui:no-raw-window-open && pnpm --dir ui test", "test:voicecall:closedloop": "vitest run extensions/voice-call/src/manager.test.ts extensions/voice-call/src/media-stream.test.ts src/plugins/voice-call.plugin.test.ts --maxWorkers=1", diff --git a/scripts/test-perf-budget.mjs b/scripts/test-perf-budget.mjs new file mode 100644 index 00000000000..44f73ffd2c4 --- /dev/null +++ b/scripts/test-perf-budget.mjs @@ -0,0 +1,127 @@ +import { spawnSync } from "node:child_process"; +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; + +function readEnvNumber(name) { + const raw = process.env[name]?.trim(); + if (!raw) { + return null; + } + const parsed = Number.parseFloat(raw); + return Number.isFinite(parsed) ? parsed : null; +} + +function parseArgs(argv) { + const args = { + config: "vitest.unit.config.ts", + maxWallMs: readEnvNumber("OPENCLAW_TEST_PERF_MAX_WALL_MS"), + baselineWallMs: readEnvNumber("OPENCLAW_TEST_PERF_BASELINE_WALL_MS"), + maxRegressionPct: readEnvNumber("OPENCLAW_TEST_PERF_MAX_REGRESSION_PCT") ?? 10, + }; + for (let i = 0; i < argv.length; i += 1) { + const arg = argv[i]; + if (arg === "--config") { + args.config = argv[i + 1] ?? args.config; + i += 1; + continue; + } + if (arg === "--max-wall-ms") { + const parsed = Number.parseFloat(argv[i + 1] ?? ""); + if (Number.isFinite(parsed)) { + args.maxWallMs = parsed; + } + i += 1; + continue; + } + if (arg === "--baseline-wall-ms") { + const parsed = Number.parseFloat(argv[i + 1] ?? ""); + if (Number.isFinite(parsed)) { + args.baselineWallMs = parsed; + } + i += 1; + continue; + } + if (arg === "--max-regression-pct") { + const parsed = Number.parseFloat(argv[i + 1] ?? ""); + if (Number.isFinite(parsed)) { + args.maxRegressionPct = parsed; + } + i += 1; + continue; + } + } + return args; +} + +function formatMs(ms) { + return `${ms.toFixed(1)}ms`; +} + +const opts = parseArgs(process.argv.slice(2)); +const reportPath = path.join(os.tmpdir(), `openclaw-vitest-perf-${Date.now()}.json`); +const cmd = [ + "vitest", + "run", + "--config", + opts.config, + "--reporter=json", + "--outputFile", + reportPath, +]; + +const startedAt = process.hrtime.bigint(); +const run = spawnSync("pnpm", cmd, { + stdio: "inherit", + env: process.env, +}); +const elapsedMs = Number(process.hrtime.bigint() - startedAt) / 1_000_000; + +if (run.status !== 0) { + process.exit(run.status ?? 1); +} + +let totalFileDurationMs = 0; +let fileCount = 0; +try { + const report = JSON.parse(fs.readFileSync(reportPath, "utf8")); + for (const result of report.testResults ?? []) { + if (typeof result.startTime === "number" && typeof result.endTime === "number") { + totalFileDurationMs += Math.max(0, result.endTime - result.startTime); + fileCount += 1; + } + } +} catch { + // Keep budget checks based on wall time when JSON parsing fails. +} + +const allowedByBaseline = + opts.baselineWallMs !== null + ? opts.baselineWallMs * (1 + (opts.maxRegressionPct ?? 0) / 100) + : null; + +let failed = false; +if (opts.maxWallMs !== null && elapsedMs > opts.maxWallMs) { + console.error( + `[test-perf-budget] wall time ${formatMs(elapsedMs)} exceeded max ${formatMs(opts.maxWallMs)}.`, + ); + failed = true; +} +if (allowedByBaseline !== null && elapsedMs > allowedByBaseline) { + console.error( + `[test-perf-budget] wall time ${formatMs(elapsedMs)} exceeded baseline budget ${formatMs( + allowedByBaseline, + )} (baseline ${formatMs(opts.baselineWallMs ?? 0)}, +${String(opts.maxRegressionPct)}%).`, + ); + failed = true; +} + +console.log( + `[test-perf-budget] config=${opts.config} wall=${formatMs(elapsedMs)} file-sum=${formatMs( + totalFileDurationMs, + )} files=${String(fileCount)}`, +); + +if (failed) { + process.exit(1); +} diff --git a/test/scripts/ios-team-id.test.ts b/test/scripts/ios-team-id.test.ts index aade0d19f69..eda6aec80ae 100644 --- a/test/scripts/ios-team-id.test.ts +++ b/test/scripts/ios-team-id.test.ts @@ -16,6 +16,60 @@ let sharedHomeDir = ""; let sharedHomeBinDir = ""; let sharedFakePythonPath = ""; const runScriptCache = new Map(); +type TeamCandidate = { + teamId: string; + isFree: boolean; + teamName: string; +}; + +function parseTeamCandidateRows(raw: string): TeamCandidate[] { + return raw + .split("\n") + .map((line) => line.replace(/\r/g, "").trim()) + .filter(Boolean) + .map((line) => line.split("\t")) + .filter((parts) => parts.length >= 3) + .map((parts) => ({ + teamId: parts[0] ?? "", + isFree: (parts[1] ?? "0") === "1", + teamName: parts[2] ?? "", + })) + .filter((candidate) => candidate.teamId.length > 0); +} + +function pickTeamIdFromCandidates(params: { + candidates: TeamCandidate[]; + preferredTeamId?: string; + preferredTeamName?: string; + preferNonFreeTeam?: boolean; +}): string | undefined { + const preferredTeamId = (params.preferredTeamId ?? "").trim(); + if (preferredTeamId) { + const preferred = params.candidates.find((candidate) => candidate.teamId === preferredTeamId); + if (preferred) { + return preferred.teamId; + } + } + + const preferredTeamName = (params.preferredTeamName ?? "").trim().toLowerCase(); + if (preferredTeamName) { + const preferredByName = params.candidates.find( + (candidate) => candidate.teamName.trim().toLowerCase() === preferredTeamName, + ); + if (preferredByName) { + return preferredByName.teamId; + } + } + + if (params.preferNonFreeTeam !== false) { + const paid = params.candidates.find((candidate) => !candidate.isFree); + if (paid) { + return paid.teamId; + } + } + + return params.candidates[0]?.teamId; +} async function writeExecutable(filePath: string, body: string): Promise { await writeFile(filePath, body, "utf8"); @@ -133,19 +187,32 @@ printf 'BBBBB22222\\t0\\tBeta Team\\r\\n'`, await rm(fixtureRoot, { recursive: true, force: true }); }); - it("resolves fallback and preferred team IDs from Xcode team listings", async () => { - const fallbackResult = runScript(sharedHomeDir, { - IOS_PYTHON_BIN: sharedFakePythonPath, + it("parses team listings and prioritizes preferred IDs without shelling out", () => { + const rows = parseTeamCandidateRows( + "AAAAA11111\t1\tAlpha Team\r\nBBBBB22222\t0\tBeta Team\r\n", + ); + expect(rows).toStrictEqual([ + { teamId: "AAAAA11111", isFree: true, teamName: "Alpha Team" }, + { teamId: "BBBBB22222", isFree: false, teamName: "Beta Team" }, + ]); + + const preferred = pickTeamIdFromCandidates({ + candidates: rows, + preferredTeamId: "BBBBB22222", }); + expect(preferred).toBe("BBBBB22222"); + + const fallback = pickTeamIdFromCandidates({ + candidates: rows, + preferredTeamId: "CCCCCC3333", + }); + expect(fallback).toBe("BBBBB22222"); + }); + + it("resolves a fallback team ID from Xcode team listings (smoke)", async () => { + const fallbackResult = runScript(sharedHomeDir, { IOS_PYTHON_BIN: sharedFakePythonPath }); expect(fallbackResult.ok).toBe(true); expect(fallbackResult.stdout).toBe("AAAAA11111"); - - const crlfResult = runScript(sharedHomeDir, { - IOS_PYTHON_BIN: sharedFakePythonPath, - IOS_PREFERRED_TEAM_ID: "BBBBB22222", - }); - expect(crlfResult.ok).toBe(true); - expect(crlfResult.stdout).toBe("BBBBB22222"); }); it("prints actionable guidance when Xcode account exists but no Team ID is resolvable", async () => {