perf(test): parallelize extension boundary compile

This commit is contained in:
Peter Steinberger
2026-04-07 10:41:35 +01:00
parent b4d0d6fcc9
commit 24d4acb274
5 changed files with 117 additions and 23 deletions

View File

@@ -84,6 +84,10 @@
"types": "./dist/src/plugin-sdk/provider-http.d.ts",
"default": "./src/provider-http.ts"
},
"./provider-model-types": {
"types": "./dist/src/plugin-sdk/provider-model-types.d.ts",
"default": "./src/provider-model-types.ts"
},
"./provider-model-shared": {
"types": "./dist/src/plugin-sdk/provider-model-shared.d.ts",
"default": "./src/provider-model-shared.ts"

View File

@@ -0,0 +1 @@
export * from "../../../src/plugin-sdk/provider-model-types.js";

View File

@@ -1,9 +1,10 @@
#!/usr/bin/env node
import { spawnSync } from "node:child_process";
import { spawn, spawnSync } from "node:child_process";
import { existsSync, mkdirSync, readdirSync, readFileSync, rmSync, writeFileSync } from "node:fs";
import { createRequire } from "node:module";
import { dirname, join, resolve } from "node:path";
import os from "node:os";
const require = createRequire(import.meta.url);
const repoRoot = resolve(import.meta.dirname, "..");
@@ -23,6 +24,15 @@ function parseMode(argv) {
return mode;
}
function resolveCompileConcurrency() {
const raw = process.env.OPENCLAW_EXTENSION_BOUNDARY_CONCURRENCY;
const parsed = raw ? Number.parseInt(raw, 10) : Number.NaN;
if (Number.isInteger(parsed) && parsed > 0) {
return parsed;
}
return Math.max(1, Math.min(6, Math.floor(os.availableParallelism() / 2)));
}
function readJsonFile(filePath) {
return JSON.parse(readFileSync(filePath, "utf8"));
}
@@ -87,6 +97,59 @@ function runNodeStep(label, args, timeoutMs) {
throw failure;
}
function runNodeStepAsync(label, args, timeoutMs) {
return new Promise((resolvePromise, rejectPromise) => {
const child = spawn(process.execPath, args, {
cwd: repoRoot,
env: process.env,
stdio: ["ignore", "pipe", "pipe"],
});
let stdout = "";
let stderr = "";
let settled = false;
const timer = setTimeout(() => {
if (settled) {
return;
}
child.kill("SIGTERM");
settled = true;
rejectPromise(
new Error(`${label}\n${stdout}${stderr}\n${label} timed out after ${timeoutMs}ms`.trim()),
);
}, timeoutMs);
child.stdout.setEncoding("utf8");
child.stderr.setEncoding("utf8");
child.stdout.on("data", (chunk) => {
stdout += chunk;
});
child.stderr.on("data", (chunk) => {
stderr += chunk;
});
child.on("error", (error) => {
if (settled) {
return;
}
clearTimeout(timer);
settled = true;
rejectPromise(new Error(`${label}\n${stdout}${stderr}\n${error.message}`.trim()));
});
child.on("close", (code) => {
if (settled) {
return;
}
clearTimeout(timer);
settled = true;
if (code === 0) {
resolvePromise({ stdout, stderr });
return;
}
rejectPromise(new Error(`${label}\n${stdout}${stderr}`.trim()));
});
});
}
function cleanupCanaryArtifacts(extensionId) {
const extensionRoot = resolve(repoRoot, "extensions", extensionId);
rmSync(resolve(extensionRoot, "__rootdir_boundary_canary__.ts"), { force: true });
@@ -97,29 +160,41 @@ function resolveBoundaryTsBuildInfoPath(extensionId) {
return resolve(repoRoot, "extensions", extensionId, "dist", ".boundary-tsc.tsbuildinfo");
}
function runCompileCheck(extensionIds) {
async function runCompileCheck(extensionIds) {
process.stdout.write(
`preparing plugin-sdk boundary artifacts for ${extensionIds.length} plugins\n`,
);
runNodeStep("plugin-sdk boundary prep", [prepareBoundaryArtifactsBin], 420_000);
for (const [index, extensionId] of extensionIds.entries()) {
const tsBuildInfoPath = resolveBoundaryTsBuildInfoPath(extensionId);
mkdirSync(dirname(tsBuildInfoPath), { recursive: true });
process.stdout.write(`[${index + 1}/${extensionIds.length}] ${extensionId}\n`);
runNodeStep(
extensionId,
[
tscBin,
"-p",
resolve(repoRoot, "extensions", extensionId, "tsconfig.json"),
"--noEmit",
"--incremental",
"--tsBuildInfoFile",
tsBuildInfoPath,
],
120_000,
);
}
const concurrency = resolveCompileConcurrency();
process.stdout.write(`compile concurrency ${concurrency}\n`);
let nextIndex = 0;
const workers = Array.from({ length: Math.min(concurrency, extensionIds.length) }, async () => {
while (true) {
const index = nextIndex;
nextIndex += 1;
if (index >= extensionIds.length) {
return;
}
const extensionId = extensionIds[index];
const tsBuildInfoPath = resolveBoundaryTsBuildInfoPath(extensionId);
mkdirSync(dirname(tsBuildInfoPath), { recursive: true });
process.stdout.write(`[${index + 1}/${extensionIds.length}] ${extensionId}\n`);
await runNodeStepAsync(
extensionId,
[
tscBin,
"-p",
resolve(repoRoot, "extensions", extensionId, "tsconfig.json"),
"--noEmit",
"--incremental",
"--tsBuildInfoFile",
tsBuildInfoPath,
],
120_000,
);
}
});
await Promise.all(workers);
}
function runCanaryCheck(extensionIds) {
@@ -168,17 +243,17 @@ function runCanaryCheck(extensionIds) {
}
}
function main() {
async function main() {
const mode = parseMode(process.argv.slice(2));
const optInExtensionIds = collectOptInExtensionIds();
const canaryExtensionIds = collectCanaryExtensionIds(optInExtensionIds);
if (mode === "all" || mode === "compile") {
runCompileCheck(optInExtensionIds);
await runCompileCheck(optInExtensionIds);
}
if (mode === "all" || mode === "canary") {
runCanaryCheck(canaryExtensionIds);
}
}
main();
await main();

View File

@@ -70,6 +70,17 @@ for (const entry of pluginSdkEntrypoints) {
"utf8",
);
const packageTypeOut = path.join(
process.cwd(),
`packages/plugin-sdk/dist/src/plugin-sdk/${entry}.d.ts`,
);
fs.mkdirSync(path.dirname(packageTypeOut), { recursive: true });
fs.writeFileSync(
packageTypeOut,
`export * from "../../../../../dist/plugin-sdk/${entry}.js";\n`,
"utf8",
);
const runtimeShim = RUNTIME_SHIMS[entry];
if (!runtimeShim) {
continue;

View File

@@ -152,6 +152,9 @@ describe("opt-in extension package boundaries", () => {
expect(packageJson.exports?.["./video-generation"]?.types).toBe(
"./dist/src/plugin-sdk/video-generation.d.ts",
);
expect(packageJson.exports?.["./provider-model-types"]?.types).toBe(
"./dist/src/plugin-sdk/provider-model-types.d.ts",
);
expect(packageJson.exports?.["./zod"]?.types).toBe("./dist/src/plugin-sdk/zod.d.ts");
expect(existsSync(resolve(REPO_ROOT, "packages/plugin-sdk/types/plugin-entry.d.ts"))).toBe(
false,