mirror of
https://github.com/moltbot/moltbot.git
synced 2026-04-20 21:23:23 +00:00
perf(test): parallelize extension boundary compile
This commit is contained in:
@@ -84,6 +84,10 @@
|
||||
"types": "./dist/src/plugin-sdk/provider-http.d.ts",
|
||||
"default": "./src/provider-http.ts"
|
||||
},
|
||||
"./provider-model-types": {
|
||||
"types": "./dist/src/plugin-sdk/provider-model-types.d.ts",
|
||||
"default": "./src/provider-model-types.ts"
|
||||
},
|
||||
"./provider-model-shared": {
|
||||
"types": "./dist/src/plugin-sdk/provider-model-shared.d.ts",
|
||||
"default": "./src/provider-model-shared.ts"
|
||||
|
||||
1
packages/plugin-sdk/src/provider-model-types.ts
Normal file
1
packages/plugin-sdk/src/provider-model-types.ts
Normal file
@@ -0,0 +1 @@
|
||||
export * from "../../../src/plugin-sdk/provider-model-types.js";
|
||||
@@ -1,9 +1,10 @@
|
||||
#!/usr/bin/env node
|
||||
|
||||
import { spawnSync } from "node:child_process";
|
||||
import { spawn, spawnSync } from "node:child_process";
|
||||
import { existsSync, mkdirSync, readdirSync, readFileSync, rmSync, writeFileSync } from "node:fs";
|
||||
import { createRequire } from "node:module";
|
||||
import { dirname, join, resolve } from "node:path";
|
||||
import os from "node:os";
|
||||
|
||||
const require = createRequire(import.meta.url);
|
||||
const repoRoot = resolve(import.meta.dirname, "..");
|
||||
@@ -23,6 +24,15 @@ function parseMode(argv) {
|
||||
return mode;
|
||||
}
|
||||
|
||||
function resolveCompileConcurrency() {
|
||||
const raw = process.env.OPENCLAW_EXTENSION_BOUNDARY_CONCURRENCY;
|
||||
const parsed = raw ? Number.parseInt(raw, 10) : Number.NaN;
|
||||
if (Number.isInteger(parsed) && parsed > 0) {
|
||||
return parsed;
|
||||
}
|
||||
return Math.max(1, Math.min(6, Math.floor(os.availableParallelism() / 2)));
|
||||
}
|
||||
|
||||
function readJsonFile(filePath) {
|
||||
return JSON.parse(readFileSync(filePath, "utf8"));
|
||||
}
|
||||
@@ -87,6 +97,59 @@ function runNodeStep(label, args, timeoutMs) {
|
||||
throw failure;
|
||||
}
|
||||
|
||||
function runNodeStepAsync(label, args, timeoutMs) {
|
||||
return new Promise((resolvePromise, rejectPromise) => {
|
||||
const child = spawn(process.execPath, args, {
|
||||
cwd: repoRoot,
|
||||
env: process.env,
|
||||
stdio: ["ignore", "pipe", "pipe"],
|
||||
});
|
||||
|
||||
let stdout = "";
|
||||
let stderr = "";
|
||||
let settled = false;
|
||||
const timer = setTimeout(() => {
|
||||
if (settled) {
|
||||
return;
|
||||
}
|
||||
child.kill("SIGTERM");
|
||||
settled = true;
|
||||
rejectPromise(
|
||||
new Error(`${label}\n${stdout}${stderr}\n${label} timed out after ${timeoutMs}ms`.trim()),
|
||||
);
|
||||
}, timeoutMs);
|
||||
|
||||
child.stdout.setEncoding("utf8");
|
||||
child.stderr.setEncoding("utf8");
|
||||
child.stdout.on("data", (chunk) => {
|
||||
stdout += chunk;
|
||||
});
|
||||
child.stderr.on("data", (chunk) => {
|
||||
stderr += chunk;
|
||||
});
|
||||
child.on("error", (error) => {
|
||||
if (settled) {
|
||||
return;
|
||||
}
|
||||
clearTimeout(timer);
|
||||
settled = true;
|
||||
rejectPromise(new Error(`${label}\n${stdout}${stderr}\n${error.message}`.trim()));
|
||||
});
|
||||
child.on("close", (code) => {
|
||||
if (settled) {
|
||||
return;
|
||||
}
|
||||
clearTimeout(timer);
|
||||
settled = true;
|
||||
if (code === 0) {
|
||||
resolvePromise({ stdout, stderr });
|
||||
return;
|
||||
}
|
||||
rejectPromise(new Error(`${label}\n${stdout}${stderr}`.trim()));
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
function cleanupCanaryArtifacts(extensionId) {
|
||||
const extensionRoot = resolve(repoRoot, "extensions", extensionId);
|
||||
rmSync(resolve(extensionRoot, "__rootdir_boundary_canary__.ts"), { force: true });
|
||||
@@ -97,29 +160,41 @@ function resolveBoundaryTsBuildInfoPath(extensionId) {
|
||||
return resolve(repoRoot, "extensions", extensionId, "dist", ".boundary-tsc.tsbuildinfo");
|
||||
}
|
||||
|
||||
function runCompileCheck(extensionIds) {
|
||||
async function runCompileCheck(extensionIds) {
|
||||
process.stdout.write(
|
||||
`preparing plugin-sdk boundary artifacts for ${extensionIds.length} plugins\n`,
|
||||
);
|
||||
runNodeStep("plugin-sdk boundary prep", [prepareBoundaryArtifactsBin], 420_000);
|
||||
for (const [index, extensionId] of extensionIds.entries()) {
|
||||
const tsBuildInfoPath = resolveBoundaryTsBuildInfoPath(extensionId);
|
||||
mkdirSync(dirname(tsBuildInfoPath), { recursive: true });
|
||||
process.stdout.write(`[${index + 1}/${extensionIds.length}] ${extensionId}\n`);
|
||||
runNodeStep(
|
||||
extensionId,
|
||||
[
|
||||
tscBin,
|
||||
"-p",
|
||||
resolve(repoRoot, "extensions", extensionId, "tsconfig.json"),
|
||||
"--noEmit",
|
||||
"--incremental",
|
||||
"--tsBuildInfoFile",
|
||||
tsBuildInfoPath,
|
||||
],
|
||||
120_000,
|
||||
);
|
||||
}
|
||||
const concurrency = resolveCompileConcurrency();
|
||||
process.stdout.write(`compile concurrency ${concurrency}\n`);
|
||||
let nextIndex = 0;
|
||||
const workers = Array.from({ length: Math.min(concurrency, extensionIds.length) }, async () => {
|
||||
while (true) {
|
||||
const index = nextIndex;
|
||||
nextIndex += 1;
|
||||
if (index >= extensionIds.length) {
|
||||
return;
|
||||
}
|
||||
const extensionId = extensionIds[index];
|
||||
const tsBuildInfoPath = resolveBoundaryTsBuildInfoPath(extensionId);
|
||||
mkdirSync(dirname(tsBuildInfoPath), { recursive: true });
|
||||
process.stdout.write(`[${index + 1}/${extensionIds.length}] ${extensionId}\n`);
|
||||
await runNodeStepAsync(
|
||||
extensionId,
|
||||
[
|
||||
tscBin,
|
||||
"-p",
|
||||
resolve(repoRoot, "extensions", extensionId, "tsconfig.json"),
|
||||
"--noEmit",
|
||||
"--incremental",
|
||||
"--tsBuildInfoFile",
|
||||
tsBuildInfoPath,
|
||||
],
|
||||
120_000,
|
||||
);
|
||||
}
|
||||
});
|
||||
await Promise.all(workers);
|
||||
}
|
||||
|
||||
function runCanaryCheck(extensionIds) {
|
||||
@@ -168,17 +243,17 @@ function runCanaryCheck(extensionIds) {
|
||||
}
|
||||
}
|
||||
|
||||
function main() {
|
||||
async function main() {
|
||||
const mode = parseMode(process.argv.slice(2));
|
||||
const optInExtensionIds = collectOptInExtensionIds();
|
||||
const canaryExtensionIds = collectCanaryExtensionIds(optInExtensionIds);
|
||||
|
||||
if (mode === "all" || mode === "compile") {
|
||||
runCompileCheck(optInExtensionIds);
|
||||
await runCompileCheck(optInExtensionIds);
|
||||
}
|
||||
if (mode === "all" || mode === "canary") {
|
||||
runCanaryCheck(canaryExtensionIds);
|
||||
}
|
||||
}
|
||||
|
||||
main();
|
||||
await main();
|
||||
|
||||
@@ -70,6 +70,17 @@ for (const entry of pluginSdkEntrypoints) {
|
||||
"utf8",
|
||||
);
|
||||
|
||||
const packageTypeOut = path.join(
|
||||
process.cwd(),
|
||||
`packages/plugin-sdk/dist/src/plugin-sdk/${entry}.d.ts`,
|
||||
);
|
||||
fs.mkdirSync(path.dirname(packageTypeOut), { recursive: true });
|
||||
fs.writeFileSync(
|
||||
packageTypeOut,
|
||||
`export * from "../../../../../dist/plugin-sdk/${entry}.js";\n`,
|
||||
"utf8",
|
||||
);
|
||||
|
||||
const runtimeShim = RUNTIME_SHIMS[entry];
|
||||
if (!runtimeShim) {
|
||||
continue;
|
||||
|
||||
@@ -152,6 +152,9 @@ describe("opt-in extension package boundaries", () => {
|
||||
expect(packageJson.exports?.["./video-generation"]?.types).toBe(
|
||||
"./dist/src/plugin-sdk/video-generation.d.ts",
|
||||
);
|
||||
expect(packageJson.exports?.["./provider-model-types"]?.types).toBe(
|
||||
"./dist/src/plugin-sdk/provider-model-types.d.ts",
|
||||
);
|
||||
expect(packageJson.exports?.["./zod"]?.types).toBe("./dist/src/plugin-sdk/zod.d.ts");
|
||||
expect(existsSync(resolve(REPO_ROOT, "packages/plugin-sdk/types/plugin-entry.d.ts"))).toBe(
|
||||
false,
|
||||
|
||||
Reference in New Issue
Block a user