fix(test): isolate flaky extension lanes

This commit is contained in:
Peter Steinberger
2026-03-23 05:01:21 +00:00
parent dc6c22b812
commit 827c441902
6 changed files with 73 additions and 35 deletions

View File

@@ -55,7 +55,7 @@ Think of the suites as “increasing realism” (and increasing flakiness/cost):
- Scheduler note:
- `pnpm test` now keeps a small checked-in behavioral manifest for true pool/isolation overrides and a separate timing snapshot for the slowest unit files.
- Shared unit coverage now defaults to `threads`, while the manifest keeps the measured fork-only exceptions and heavy singleton lanes explicit.
- The extension suite (`vitest.extensions.config.ts`) also now defaults to `threads`; the March 22, 2026 direct full-suite control run passed clean without extension-specific fork exceptions.
- The shared extension lane still defaults to `threads`; the wrapper keeps explicit fork-only exceptions in `test/fixtures/test-parallel.behavior.json` when a file cannot safely share a non-isolated worker.
- The channel suite (`vitest.channels.config.ts`) now also defaults to `threads`; the March 22, 2026 direct full-suite control run passed clean without channel-specific fork exceptions.
- The wrapper peels the heaviest measured files into dedicated lanes instead of relying on a growing hand-maintained exclusion list.
- Refresh the timing snapshot with `pnpm test:perf:update-timings` after major suite shape changes.

View File

@@ -16,7 +16,7 @@ title: "Tests"
- `pnpm test`: runs the full wrapper. It keeps only a small behavioral override manifest in git, then uses a checked-in timing snapshot to peel the heaviest measured unit files into dedicated lanes.
- Unit files default to `threads` in the wrapper; keep fork-only exceptions documented in `test/fixtures/test-parallel.behavior.json`.
- `pnpm test:channels` now defaults to `threads` via `vitest.channels.config.ts`; the March 22, 2026 direct full-suite control run passed clean without channel-specific fork exceptions.
- `pnpm test:extensions` now defaults to `threads` via `vitest.extensions.config.ts`; the March 22, 2026 direct full-suite control run passed clean without extension-specific fork exceptions.
- `pnpm test:extensions` runs through the wrapper and keeps documented extension fork-only exceptions in `test/fixtures/test-parallel.behavior.json`; the shared extension lane still defaults to `threads`.
- `pnpm test:extensions`: runs extension/plugin suites.
- `pnpm test:perf:imports`: enables Vitest import-duration + import-breakdown reporting for the wrapper.
- `pnpm test:perf:imports:changed`: same import profiling, but only for files changed since `origin/main`.

View File

@@ -710,7 +710,7 @@
"test:e2e": "vitest run --config vitest.e2e.config.ts",
"test:e2e:openshell": "OPENCLAW_E2E_OPENSHELL=1 vitest run --config vitest.e2e.config.ts test/openshell-sandbox.e2e.test.ts",
"test:extension": "node scripts/test-extension.mjs",
"test:extensions": "vitest run --config vitest.extensions.config.ts",
"test:extensions": "OPENCLAW_TEST_SKIP_DEFAULT=1 OPENCLAW_TEST_INCLUDE_EXTENSIONS=1 node scripts/test-parallel.mjs",
"test:extensions:memory": "node scripts/profile-extension-memory.mjs",
"test:fast": "vitest run --config vitest.unit.config.ts",
"test:force": "node --import tsx scripts/test-force.ts",

View File

@@ -50,6 +50,7 @@ const cleanupTempArtifacts = () => {
};
const existingUnitConfigFiles = (entries) => existingFiles(entries).filter(isUnitConfigTestFile);
const baseThreadPinnedFiles = existingFiles(behaviorManifest.base?.threadPinned ?? []);
const extensionForkIsolatedFiles = existingFiles(behaviorManifest.extensions?.isolated ?? []);
const unitForkIsolatedFiles = existingUnitConfigFiles(behaviorManifest.unit.isolated);
const unitThreadPinnedFiles = existingUnitConfigFiles(behaviorManifest.unit.threadPinned);
const unitBehaviorOverrideSet = new Set([...unitForkIsolatedFiles, ...unitThreadPinnedFiles]);
@@ -91,6 +92,7 @@ const disableIsolation =
process.env.OPENCLAW_TEST_NO_ISOLATE !== "false";
const includeGatewaySuite = process.env.OPENCLAW_TEST_INCLUDE_GATEWAY === "1";
const includeExtensionsSuite = process.env.OPENCLAW_TEST_INCLUDE_EXTENSIONS === "1";
const skipDefaultRuns = process.env.OPENCLAW_TEST_SKIP_DEFAULT === "1";
const parsePoolOverride = (value, fallback) => {
if (value === "threads" || value === "forks") {
return value;
@@ -261,7 +263,8 @@ const defaultUnitPool = parsePoolOverride(process.env.OPENCLAW_TEST_UNIT_DEFAULT
const isTargetedIsolatedUnitFile = (fileFilter) =>
unitForkIsolatedFiles.includes(fileFilter) || unitMemoryIsolatedFiles.includes(fileFilter);
const inferTarget = (fileFilter) => {
const isolated = isTargetedIsolatedUnitFile(fileFilter);
const isolated =
isTargetedIsolatedUnitFile(fileFilter) || extensionForkIsolatedFiles.includes(fileFilter);
if (isUnitConfigTestFile(fileFilter)) {
return { owner: "unit", isolated };
}
@@ -392,9 +395,14 @@ const unitFastExcludedFileSet = new Set(unitFastExcludedFiles);
const unitFastCandidateFiles = allKnownUnitFiles.filter(
(file) => !unitFastExcludedFileSet.has(file),
);
const extensionSharedCandidateFiles = allKnownTestFiles.filter((file) =>
file.startsWith("extensions/"),
const extensionForkIsolatedFileSet = new Set(extensionForkIsolatedFiles);
const extensionSharedCandidateFiles = allKnownTestFiles.filter(
(file) => file.startsWith("extensions/") && !extensionForkIsolatedFileSet.has(file),
);
const extensionIsolatedEntries = extensionForkIsolatedFiles.map((file) => ({
name: `extensions-${path.basename(file, ".test.ts")}-isolated`,
args: ["vitest", "run", "--config", "vitest.extensions.config.ts", "--pool=forks", file],
}));
const defaultUnitFastLaneCount = isCI && !isWindows ? 3 : 1;
const unitFastLaneCount = Math.max(
1,
@@ -471,36 +479,39 @@ const unitIsolatedEntries = unitForkIsolatedFiles.map((file) => ({
args: ["vitest", "run", "--config", "vitest.unit.config.ts", "--pool=forks", file],
}));
const baseRuns = [
...(shouldSplitUnitRuns
? [
...unitFastEntries,
...unitIsolatedEntries,
...unitHeavyEntries,
...unitMemoryIsolatedFiles.map((file) => ({
name: `unit-${path.basename(file, ".test.ts")}-memory-isolated`,
args: ["vitest", "run", "--config", "vitest.unit.config.ts", "--pool=forks", file],
})),
...unitThreadEntries,
...channelSingletonFiles.map((file) => ({
name: `${path.basename(file, ".test.ts")}-channels-isolated`,
args: ["vitest", "run", "--config", "vitest.channels.config.ts", "--pool=forks", file],
})),
]
: [
{
name: "unit",
args: [
"vitest",
"run",
"--config",
"vitest.unit.config.ts",
"--pool=forks",
...(disableIsolation ? ["--isolate=false"] : []),
],
},
]),
...(skipDefaultRuns
? []
: shouldSplitUnitRuns
? [
...unitFastEntries,
...unitIsolatedEntries,
...unitHeavyEntries,
...unitMemoryIsolatedFiles.map((file) => ({
name: `unit-${path.basename(file, ".test.ts")}-memory-isolated`,
args: ["vitest", "run", "--config", "vitest.unit.config.ts", "--pool=forks", file],
})),
...unitThreadEntries,
...channelSingletonFiles.map((file) => ({
name: `${path.basename(file, ".test.ts")}-channels-isolated`,
args: ["vitest", "run", "--config", "vitest.channels.config.ts", "--pool=forks", file],
})),
]
: [
{
name: "unit",
args: [
"vitest",
"run",
"--config",
"vitest.unit.config.ts",
"--pool=forks",
...(disableIsolation ? ["--isolate=false"] : []),
],
},
]),
...(includeExtensionsSuite
? [
...extensionIsolatedEntries,
{
name: "extensions",
env:
@@ -583,7 +594,14 @@ const createTargetedEntry = (owner, isolated, filters) => {
if (owner === "extensions") {
return {
name,
args: ["vitest", "run", "--config", "vitest.extensions.config.ts", ...filters],
args: [
"vitest",
"run",
"--config",
"vitest.extensions.config.ts",
...(forceForks ? ["--pool=forks"] : []),
...filters,
],
};
}
if (owner === "gateway") {

View File

@@ -47,10 +47,14 @@ export function loadTestRunnerBehavior() {
const raw = tryReadJsonFile(behaviorManifestPath, {});
const unit = raw.unit ?? {};
const base = raw.base ?? {};
const extensions = raw.extensions ?? {};
return {
base: {
threadPinned: mergeManifestEntries(base, ["threadPinned", "threadSingleton"]),
},
extensions: {
isolated: mergeManifestEntries(extensions, ["isolated"]),
},
unit: {
isolated: mergeManifestEntries(unit, ["isolated"]),
threadPinned: mergeManifestEntries(unit, ["threadPinned", "threadSingleton"]),

View File

@@ -15,6 +15,22 @@
}
]
},
"extensions": {
"isolated": [
{
"file": "extensions/matrix/src/matrix/sdk.test.ts",
"reason": "This suite hoists a matrix-js-sdk module mock that can leak into later Matrix extension files when they share a non-isolated worker."
},
{
"file": "extensions/matrix/src/matrix/client/file-sync-store.test.ts",
"reason": "Matrix sdk.test.ts hoists a matrix-js-sdk module mock; keep the sync-store persistence regression in its own forked lane so non-isolated extension workers stay deterministic."
},
{
"file": "extensions/nextcloud-talk/src/monitor.replay.test.ts",
"reason": "The replay-handling regression is green alone but can inherit disturbed global stream/Response state from the shared extensions lane, so keep it in its own forked lane for deterministic CI."
}
]
},
"unit": {
"isolated": [],
"threadPinned": []