// Docker E2E aggregate scheduler. // Builds shared Docker images, prepares one OpenClaw npm tarball, assigns lanes // to bare/functional images, and runs lanes through weighted resource pools. import { spawn } from "node:child_process"; import fs from "node:fs"; import { mkdir, readFile } from "node:fs/promises"; import path from "node:path"; import { fileURLToPath } from "node:url"; const ROOT_DIR = path.resolve(path.dirname(fileURLToPath(import.meta.url)), ".."); const DEFAULT_E2E_BARE_IMAGE = "openclaw-docker-e2e-bare:local"; const DEFAULT_E2E_FUNCTIONAL_IMAGE = "openclaw-docker-e2e-functional:local"; const DEFAULT_E2E_IMAGE = DEFAULT_E2E_FUNCTIONAL_IMAGE; const DEFAULT_PARALLELISM = 10; const DEFAULT_TAIL_PARALLELISM = 10; const DEFAULT_FAILURE_TAIL_LINES = 80; const DEFAULT_LANE_TIMEOUT_MS = 120 * 60 * 1000; const DEFAULT_LANE_START_STAGGER_MS = 2_000; const DEFAULT_LIVE_RETRIES = 1; const DEFAULT_STATUS_INTERVAL_MS = 30_000; const DEFAULT_PREFLIGHT_RUN_TIMEOUT_MS = 60_000; const DEFAULT_TIMINGS_FILE = path.join(ROOT_DIR, ".artifacts/docker-tests/lane-timings.json"); const DEFAULT_PROFILE = "all"; const RELEASE_PATH_PROFILE = "release-path"; const LIVE_PROFILE_TIMEOUT_MS = 20 * 60 * 1000; const LIVE_CLI_TIMEOUT_MS = 20 * 60 * 1000; const LIVE_ACP_TIMEOUT_MS = 20 * 60 * 1000; const OPENWEBUI_TIMEOUT_MS = 20 * 60 * 1000; const BUNDLED_UPDATE_TIMEOUT_MS = 20 * 60 * 1000; const DEFAULT_RESOURCE_LIMITS = { docker: DEFAULT_PARALLELISM, live: 9, "live:claude": 4, "live:codex": 4, "live:droid": 4, "live:gemini": 4, "live:opencode": 4, npm: 10, service: 7, }; const LIVE_RETRY_PATTERNS = [ /529\b/i, /overloaded/i, /capacity/i, /rate.?limit/i, /gateway closed \(1000 normal closure\)/i, /ECONNRESET|ETIMEDOUT|ENOTFOUND/i, ]; const bundledChannelLaneCommand = "OPENCLAW_SKIP_DOCKER_BUILD=1 OPENCLAW_BUNDLED_CHANNEL_UPDATE_SCENARIO=0 OPENCLAW_BUNDLED_CHANNEL_ROOT_OWNED_SCENARIO=0 OPENCLAW_BUNDLED_CHANNEL_SETUP_ENTRY_SCENARIO=0 OPENCLAW_BUNDLED_CHANNEL_LOAD_FAILURE_SCENARIO=0 OPENCLAW_BUNDLED_CHANNEL_DISABLED_CONFIG_SCENARIO=0 pnpm test:docker:bundled-channel-deps"; function lane(name, command, options = {}) { return { cacheKey: options.cacheKey, command, e2eImageKind: options.e2eImageKind ?? (options.live ? undefined : "functional"), estimateSeconds: options.estimateSeconds, live: options.live === true, name, retryPatterns: options.retryPatterns ?? [], retries: options.retries ?? 0, resources: options.resources ?? [], timeoutMs: options.timeoutMs, weight: options.weight ?? 1, }; } function liveProviderResource(provider) { if (!provider) { return undefined; } if (provider === "claude-cli" || provider === "claude") { return "live:claude"; } if (provider === "codex-cli" || provider === "codex") { return "live:codex"; } if (provider === "droid") { return "live:droid"; } if (provider === "google-gemini-cli" || provider === "gemini") { return "live:gemini"; } if (provider === "opencode") { return "live:opencode"; } if (provider === "openai") { return "live:openai"; } return `live:${provider}`; } function liveProviderResources(options) { const providers = options.providers ?? (options.provider ? [options.provider] : []); return providers.map(liveProviderResource).filter(Boolean); } function liveLane(name, command, options = {}) { return lane(name, command, { ...options, live: true, resources: ["live", ...liveProviderResources(options), ...(options.resources ?? [])], retryPatterns: options.retryPatterns ?? LIVE_RETRY_PATTERNS, retries: options.retries ?? DEFAULT_LIVE_RETRIES, weight: options.weight ?? 3, }); } function npmLane(name, command, options = {}) { return lane(name, command, { ...options, e2eImageKind: options.e2eImageKind ?? "bare", resources: ["npm", ...(options.resources ?? [])], weight: options.weight ?? 2, }); } function serviceLane(name, command, options = {}) { return lane(name, command, { ...options, resources: ["service", ...(options.resources ?? [])], weight: options.weight ?? 2, }); } const bundledScenarioLanes = [ npmLane( "bundled-channel-telegram", `OPENCLAW_BUNDLED_CHANNELS=telegram ${bundledChannelLaneCommand}`, ), npmLane( "bundled-channel-discord", `OPENCLAW_BUNDLED_CHANNELS=discord ${bundledChannelLaneCommand}`, ), npmLane("bundled-channel-slack", `OPENCLAW_BUNDLED_CHANNELS=slack ${bundledChannelLaneCommand}`), npmLane( "bundled-channel-feishu", `OPENCLAW_BUNDLED_CHANNELS=feishu ${bundledChannelLaneCommand}`, ), npmLane( "bundled-channel-memory-lancedb", `OPENCLAW_BUNDLED_CHANNELS=memory-lancedb ${bundledChannelLaneCommand}`, ), npmLane( "bundled-channel-update-telegram", "OPENCLAW_SKIP_DOCKER_BUILD=1 OPENCLAW_BUNDLED_CHANNEL_SCENARIOS=0 OPENCLAW_BUNDLED_CHANNEL_UPDATE_SCENARIO=1 OPENCLAW_BUNDLED_CHANNEL_UPDATE_TARGETS=telegram OPENCLAW_BUNDLED_CHANNEL_ROOT_OWNED_SCENARIO=0 OPENCLAW_BUNDLED_CHANNEL_SETUP_ENTRY_SCENARIO=0 OPENCLAW_BUNDLED_CHANNEL_LOAD_FAILURE_SCENARIO=0 OPENCLAW_BUNDLED_CHANNEL_DISABLED_CONFIG_SCENARIO=0 pnpm test:docker:bundled-channel-deps", { timeoutMs: BUNDLED_UPDATE_TIMEOUT_MS }, ), npmLane( "bundled-channel-update-discord", "OPENCLAW_SKIP_DOCKER_BUILD=1 OPENCLAW_BUNDLED_CHANNEL_SCENARIOS=0 OPENCLAW_BUNDLED_CHANNEL_UPDATE_SCENARIO=1 OPENCLAW_BUNDLED_CHANNEL_UPDATE_TARGETS=discord OPENCLAW_BUNDLED_CHANNEL_ROOT_OWNED_SCENARIO=0 OPENCLAW_BUNDLED_CHANNEL_SETUP_ENTRY_SCENARIO=0 OPENCLAW_BUNDLED_CHANNEL_LOAD_FAILURE_SCENARIO=0 OPENCLAW_BUNDLED_CHANNEL_DISABLED_CONFIG_SCENARIO=0 pnpm test:docker:bundled-channel-deps", { timeoutMs: BUNDLED_UPDATE_TIMEOUT_MS }, ), npmLane( "bundled-channel-update-slack", "OPENCLAW_SKIP_DOCKER_BUILD=1 OPENCLAW_BUNDLED_CHANNEL_SCENARIOS=0 OPENCLAW_BUNDLED_CHANNEL_UPDATE_SCENARIO=1 OPENCLAW_BUNDLED_CHANNEL_UPDATE_TARGETS=slack OPENCLAW_BUNDLED_CHANNEL_ROOT_OWNED_SCENARIO=0 OPENCLAW_BUNDLED_CHANNEL_SETUP_ENTRY_SCENARIO=0 OPENCLAW_BUNDLED_CHANNEL_LOAD_FAILURE_SCENARIO=0 OPENCLAW_BUNDLED_CHANNEL_DISABLED_CONFIG_SCENARIO=0 pnpm test:docker:bundled-channel-deps", { timeoutMs: BUNDLED_UPDATE_TIMEOUT_MS }, ), npmLane( "bundled-channel-update-feishu", "OPENCLAW_SKIP_DOCKER_BUILD=1 OPENCLAW_BUNDLED_CHANNEL_SCENARIOS=0 OPENCLAW_BUNDLED_CHANNEL_UPDATE_SCENARIO=1 OPENCLAW_BUNDLED_CHANNEL_UPDATE_TARGETS=feishu OPENCLAW_BUNDLED_CHANNEL_ROOT_OWNED_SCENARIO=0 OPENCLAW_BUNDLED_CHANNEL_SETUP_ENTRY_SCENARIO=0 OPENCLAW_BUNDLED_CHANNEL_LOAD_FAILURE_SCENARIO=0 OPENCLAW_BUNDLED_CHANNEL_DISABLED_CONFIG_SCENARIO=0 pnpm test:docker:bundled-channel-deps", { timeoutMs: BUNDLED_UPDATE_TIMEOUT_MS }, ), npmLane( "bundled-channel-update-memory-lancedb", "OPENCLAW_SKIP_DOCKER_BUILD=1 OPENCLAW_BUNDLED_CHANNEL_SCENARIOS=0 OPENCLAW_BUNDLED_CHANNEL_UPDATE_SCENARIO=1 OPENCLAW_BUNDLED_CHANNEL_UPDATE_TARGETS=memory-lancedb OPENCLAW_BUNDLED_CHANNEL_ROOT_OWNED_SCENARIO=0 OPENCLAW_BUNDLED_CHANNEL_SETUP_ENTRY_SCENARIO=0 OPENCLAW_BUNDLED_CHANNEL_LOAD_FAILURE_SCENARIO=0 OPENCLAW_BUNDLED_CHANNEL_DISABLED_CONFIG_SCENARIO=0 pnpm test:docker:bundled-channel-deps", { timeoutMs: BUNDLED_UPDATE_TIMEOUT_MS }, ), npmLane( "bundled-channel-update-acpx", "OPENCLAW_SKIP_DOCKER_BUILD=1 OPENCLAW_BUNDLED_CHANNEL_SCENARIOS=0 OPENCLAW_BUNDLED_CHANNEL_UPDATE_SCENARIO=1 OPENCLAW_BUNDLED_CHANNEL_UPDATE_TARGETS=acpx OPENCLAW_BUNDLED_CHANNEL_ROOT_OWNED_SCENARIO=0 OPENCLAW_BUNDLED_CHANNEL_SETUP_ENTRY_SCENARIO=0 OPENCLAW_BUNDLED_CHANNEL_LOAD_FAILURE_SCENARIO=0 OPENCLAW_BUNDLED_CHANNEL_DISABLED_CONFIG_SCENARIO=0 pnpm test:docker:bundled-channel-deps", { timeoutMs: BUNDLED_UPDATE_TIMEOUT_MS }, ), npmLane( "bundled-channel-root-owned", "OPENCLAW_SKIP_DOCKER_BUILD=1 OPENCLAW_BUNDLED_CHANNEL_SCENARIOS=0 OPENCLAW_BUNDLED_CHANNEL_UPDATE_SCENARIO=0 OPENCLAW_BUNDLED_CHANNEL_ROOT_OWNED_SCENARIO=1 OPENCLAW_BUNDLED_CHANNEL_SETUP_ENTRY_SCENARIO=0 OPENCLAW_BUNDLED_CHANNEL_LOAD_FAILURE_SCENARIO=0 OPENCLAW_BUNDLED_CHANNEL_DISABLED_CONFIG_SCENARIO=0 pnpm test:docker:bundled-channel-deps", ), npmLane( "bundled-channel-setup-entry", "OPENCLAW_SKIP_DOCKER_BUILD=1 OPENCLAW_BUNDLED_CHANNEL_SCENARIOS=0 OPENCLAW_BUNDLED_CHANNEL_UPDATE_SCENARIO=0 OPENCLAW_BUNDLED_CHANNEL_ROOT_OWNED_SCENARIO=0 OPENCLAW_BUNDLED_CHANNEL_SETUP_ENTRY_SCENARIO=1 OPENCLAW_BUNDLED_CHANNEL_LOAD_FAILURE_SCENARIO=0 OPENCLAW_BUNDLED_CHANNEL_DISABLED_CONFIG_SCENARIO=0 pnpm test:docker:bundled-channel-deps", ), npmLane( "bundled-channel-load-failure", "OPENCLAW_SKIP_DOCKER_BUILD=1 OPENCLAW_BUNDLED_CHANNEL_SCENARIOS=0 OPENCLAW_BUNDLED_CHANNEL_UPDATE_SCENARIO=0 OPENCLAW_BUNDLED_CHANNEL_ROOT_OWNED_SCENARIO=0 OPENCLAW_BUNDLED_CHANNEL_SETUP_ENTRY_SCENARIO=0 OPENCLAW_BUNDLED_CHANNEL_LOAD_FAILURE_SCENARIO=1 OPENCLAW_BUNDLED_CHANNEL_DISABLED_CONFIG_SCENARIO=0 pnpm test:docker:bundled-channel-deps", ), npmLane( "bundled-channel-disabled-config", "OPENCLAW_SKIP_DOCKER_BUILD=1 OPENCLAW_BUNDLED_CHANNEL_SCENARIOS=0 OPENCLAW_BUNDLED_CHANNEL_UPDATE_SCENARIO=0 OPENCLAW_BUNDLED_CHANNEL_ROOT_OWNED_SCENARIO=0 OPENCLAW_BUNDLED_CHANNEL_SETUP_ENTRY_SCENARIO=0 OPENCLAW_BUNDLED_CHANNEL_LOAD_FAILURE_SCENARIO=0 OPENCLAW_BUNDLED_CHANNEL_DISABLED_CONFIG_SCENARIO=1 pnpm test:docker:bundled-channel-deps", ), ]; const lanes = [ liveLane("live-models", "OPENCLAW_SKIP_DOCKER_BUILD=1 pnpm test:docker:live-models", { providers: ["claude-cli", "codex-cli", "google-gemini-cli"], timeoutMs: LIVE_PROFILE_TIMEOUT_MS, weight: 4, }), liveLane("live-gateway", "OPENCLAW_SKIP_DOCKER_BUILD=1 pnpm test:docker:live-gateway", { providers: ["claude-cli", "codex-cli", "google-gemini-cli"], timeoutMs: LIVE_PROFILE_TIMEOUT_MS, weight: 4, }), liveLane( "live-cli-backend-claude", "OPENCLAW_SKIP_DOCKER_BUILD=1 pnpm test:docker:live-cli-backend:claude", { cacheKey: "cli-backend-claude", provider: "claude-cli", resources: ["npm"], timeoutMs: LIVE_CLI_TIMEOUT_MS, weight: 3, }, ), liveLane( "live-cli-backend-gemini", "OPENCLAW_SKIP_DOCKER_BUILD=1 pnpm test:docker:live-cli-backend:gemini", { cacheKey: "cli-backend-gemini", provider: "google-gemini-cli", resources: ["npm"], timeoutMs: LIVE_CLI_TIMEOUT_MS, weight: 3, }, ), serviceLane("openwebui", "OPENCLAW_SKIP_DOCKER_BUILD=1 pnpm test:docker:openwebui", { timeoutMs: OPENWEBUI_TIMEOUT_MS, weight: 5, }), serviceLane("onboard", "OPENCLAW_SKIP_DOCKER_BUILD=1 pnpm test:docker:onboard", { weight: 2, }), npmLane( "npm-onboard-channel-agent", "OPENCLAW_SKIP_DOCKER_BUILD=1 pnpm test:docker:npm-onboard-channel-agent", { resources: ["service"], weight: 3 }, ), serviceLane("gateway-network", "OPENCLAW_SKIP_DOCKER_BUILD=1 pnpm test:docker:gateway-network"), serviceLane( "agents-delete-shared-workspace", "OPENCLAW_SKIP_DOCKER_BUILD=1 pnpm test:docker:agents-delete-shared-workspace", ), serviceLane("mcp-channels", "OPENCLAW_SKIP_DOCKER_BUILD=1 pnpm test:docker:mcp-channels", { resources: ["npm"], weight: 3, }), lane("pi-bundle-mcp-tools", "OPENCLAW_SKIP_DOCKER_BUILD=1 pnpm test:docker:pi-bundle-mcp-tools"), lane("crestodian-rescue", "OPENCLAW_SKIP_DOCKER_BUILD=1 pnpm test:docker:crestodian-rescue"), lane("crestodian-planner", "OPENCLAW_SKIP_DOCKER_BUILD=1 pnpm test:docker:crestodian-planner"), serviceLane( "cron-mcp-cleanup", "OPENCLAW_SKIP_DOCKER_BUILD=1 pnpm test:docker:cron-mcp-cleanup", { resources: ["npm"], weight: 3 }, ), npmLane("doctor-switch", "OPENCLAW_SKIP_DOCKER_BUILD=1 pnpm test:docker:doctor-switch", { weight: 3, }), npmLane( "update-channel-switch", "OPENCLAW_SKIP_DOCKER_BUILD=1 pnpm test:docker:update-channel-switch", { timeoutMs: 30 * 60 * 1000, weight: 3, }, ), lane("plugins", "OPENCLAW_SKIP_DOCKER_BUILD=1 pnpm test:docker:plugins", { resources: ["npm", "service"], weight: 6, }), npmLane("plugin-update", "OPENCLAW_SKIP_DOCKER_BUILD=1 pnpm test:docker:plugin-update"), serviceLane("config-reload", "OPENCLAW_SKIP_DOCKER_BUILD=1 pnpm test:docker:config-reload"), ...bundledScenarioLanes, lane("openai-image-auth", "OPENCLAW_SKIP_DOCKER_BUILD=1 pnpm test:docker:openai-image-auth"), lane( "crestodian-first-run", "OPENCLAW_SKIP_DOCKER_BUILD=1 pnpm test:docker:crestodian-first-run", ), lane( "session-runtime-context", "OPENCLAW_SKIP_DOCKER_BUILD=1 pnpm test:docker:session-runtime-context", ), lane("qr", "pnpm test:docker:qr"), ]; const exclusiveLanes = [ serviceLane( "openai-web-search-minimal", "OPENCLAW_SKIP_DOCKER_BUILD=1 pnpm test:docker:openai-web-search-minimal", { timeoutMs: 8 * 60 * 1000 }, ), liveLane( "live-codex-harness", "OPENCLAW_SKIP_DOCKER_BUILD=1 pnpm test:docker:live-codex-harness", { cacheKey: "codex-harness", provider: "codex-cli", resources: ["npm"], timeoutMs: LIVE_ACP_TIMEOUT_MS, weight: 3, }, ), liveLane("live-codex-bind", "OPENCLAW_SKIP_DOCKER_BUILD=1 pnpm test:docker:live-codex-bind", { cacheKey: "codex-harness", provider: "codex-cli", resources: ["npm"], timeoutMs: LIVE_ACP_TIMEOUT_MS, weight: 3, }), liveLane( "live-cli-backend-codex", "OPENCLAW_SKIP_DOCKER_BUILD=1 pnpm test:docker:live-cli-backend:codex", { cacheKey: "cli-backend-codex", provider: "codex-cli", resources: ["npm"], timeoutMs: LIVE_CLI_TIMEOUT_MS, weight: 3, }, ), liveLane( "live-acp-bind-claude", "OPENCLAW_SKIP_DOCKER_BUILD=1 pnpm test:docker:live-acp-bind:claude", { cacheKey: "acp-bind-claude", provider: "claude-cli", resources: ["npm"], timeoutMs: LIVE_ACP_TIMEOUT_MS, weight: 3, }, ), liveLane( "live-acp-bind-codex", "OPENCLAW_SKIP_DOCKER_BUILD=1 pnpm test:docker:live-acp-bind:codex", { cacheKey: "acp-bind-codex", provider: "codex-cli", resources: ["npm"], timeoutMs: LIVE_ACP_TIMEOUT_MS, weight: 3, }, ), liveLane( "live-acp-bind-droid", "OPENCLAW_SKIP_DOCKER_BUILD=1 pnpm test:docker:live-acp-bind:droid", { cacheKey: "acp-bind-droid", provider: "droid", resources: ["npm"], timeoutMs: LIVE_ACP_TIMEOUT_MS, weight: 3, }, ), liveLane( "live-acp-bind-gemini", "OPENCLAW_SKIP_DOCKER_BUILD=1 pnpm test:docker:live-acp-bind:gemini", { cacheKey: "acp-bind-gemini", provider: "google-gemini-cli", resources: ["npm"], timeoutMs: LIVE_ACP_TIMEOUT_MS, weight: 3, }, ), liveLane( "live-acp-bind-opencode", "OPENCLAW_SKIP_DOCKER_BUILD=1 pnpm test:docker:live-acp-bind:opencode", { cacheKey: "acp-bind-opencode", provider: "opencode", resources: ["npm"], timeoutMs: LIVE_ACP_TIMEOUT_MS, weight: 3, }, ), ]; const tailLanes = exclusiveLanes; const releasePathChunks = { core: [ lane("qr", "OPENCLAW_SKIP_DOCKER_BUILD=1 pnpm test:docker:qr"), serviceLane("onboard", "OPENCLAW_SKIP_DOCKER_BUILD=1 pnpm test:docker:onboard", { weight: 2, }), serviceLane("gateway-network", "OPENCLAW_SKIP_DOCKER_BUILD=1 pnpm test:docker:gateway-network"), serviceLane("config-reload", "OPENCLAW_SKIP_DOCKER_BUILD=1 pnpm test:docker:config-reload"), lane( "session-runtime-context", "OPENCLAW_SKIP_DOCKER_BUILD=1 pnpm test:docker:session-runtime-context", ), lane( "pi-bundle-mcp-tools", "OPENCLAW_SKIP_DOCKER_BUILD=1 pnpm test:docker:pi-bundle-mcp-tools", ), serviceLane("mcp-channels", "OPENCLAW_SKIP_DOCKER_BUILD=1 pnpm test:docker:mcp-channels", { resources: ["npm"], weight: 3, }), ], "package-update": [ npmLane("install-e2e", "OPENCLAW_E2E_MODELS=both pnpm test:install:e2e", { resources: ["service"], weight: 4, }), npmLane( "npm-onboard-channel-agent", "OPENCLAW_SKIP_DOCKER_BUILD=1 pnpm test:docker:npm-onboard-channel-agent", { resources: ["service"], weight: 3 }, ), npmLane("doctor-switch", "OPENCLAW_SKIP_DOCKER_BUILD=1 pnpm test:docker:doctor-switch", { weight: 3, }), npmLane( "update-channel-switch", "OPENCLAW_SKIP_DOCKER_BUILD=1 pnpm test:docker:update-channel-switch", { timeoutMs: 30 * 60 * 1000, weight: 3, }, ), ], "plugins-integrations": [ lane("plugins", "OPENCLAW_SKIP_DOCKER_BUILD=1 pnpm test:docker:plugins", { resources: ["npm", "service"], weight: 6, }), npmLane("plugin-update", "OPENCLAW_SKIP_DOCKER_BUILD=1 pnpm test:docker:plugin-update"), npmLane( "bundled-channel-deps", "OPENCLAW_SKIP_DOCKER_BUILD=1 pnpm test:docker:bundled-channel-deps", { resources: ["service"], weight: 3 }, ), serviceLane( "cron-mcp-cleanup", "OPENCLAW_SKIP_DOCKER_BUILD=1 pnpm test:docker:cron-mcp-cleanup", { resources: ["npm"], weight: 3, }, ), serviceLane( "openai-web-search-minimal", "OPENCLAW_SKIP_DOCKER_BUILD=1 pnpm test:docker:openai-web-search-minimal", { timeoutMs: 8 * 60 * 1000 }, ), ], }; function releasePathChunkLanes(chunk, options = {}) { const base = releasePathChunks[chunk]; if (!base) { throw new Error( `OPENCLAW_DOCKER_ALL_CHUNK must be one of: ${Object.keys(releasePathChunks).join(", ")}. Got: ${JSON.stringify(chunk)}`, ); } if (chunk !== "plugins-integrations" || !options.includeOpenWebUI) { return base; } return [ ...base, serviceLane("openwebui", "OPENCLAW_SKIP_DOCKER_BUILD=1 pnpm test:docker:openwebui", { timeoutMs: OPENWEBUI_TIMEOUT_MS, weight: 5, }), ]; } function allReleasePathLanes(options = {}) { return Object.keys(releasePathChunks).flatMap((chunk) => releasePathChunkLanes(chunk, { includeOpenWebUI: chunk === "plugins-integrations" && options.includeOpenWebUI, }), ); } function parseLaneSelection(raw) { if (!raw) { return []; } return [ ...new Set( String(raw) .split(/[,\s]+/u) .map((token) => token.trim()) .filter(Boolean), ), ]; } function dedupeLanes(poolLanes) { const byName = new Map(); for (const poolLane of poolLanes) { if (!byName.has(poolLane.name)) { byName.set(poolLane.name, poolLane); } } return [...byName.values()]; } function selectNamedLanes(poolLanes, selectedNames, label) { const byName = new Map(poolLanes.map((poolLane) => [poolLane.name, poolLane])); const missing = selectedNames.filter((name) => !byName.has(name)); if (missing.length > 0) { throw new Error( `${label} unknown lane(s): ${missing.join(", ")}. Available lanes: ${[...byName.keys()] .toSorted((a, b) => a.localeCompare(b)) .join(", ")}`, ); } return selectedNames.map((name) => byName.get(name)); } function parsePositiveInt(raw, fallback, label) { if (!raw) { return fallback; } const parsed = Number(raw); if (!Number.isInteger(parsed) || parsed < 1) { throw new Error(`${label} must be a positive integer. Got: ${JSON.stringify(raw)}`); } return parsed; } function parseNonNegativeInt(raw, fallback, label) { if (!raw) { return fallback; } const parsed = Number(raw); if (!Number.isInteger(parsed) || parsed < 0) { throw new Error(`${label} must be a non-negative integer. Got: ${JSON.stringify(raw)}`); } return parsed; } function parseBool(raw, fallback) { if (raw === undefined || raw === "") { return fallback; } return !/^(?:0|false|no)$/i.test(raw); } function parseLiveMode(raw) { const mode = raw || "all"; if (mode === "all" || mode === "skip" || mode === "only") { return mode; } throw new Error( `OPENCLAW_DOCKER_ALL_LIVE_MODE must be one of: all, skip, only. Got: ${JSON.stringify(raw)}`, ); } function parseProfile(raw) { const profile = raw || DEFAULT_PROFILE; if (profile === DEFAULT_PROFILE || profile === RELEASE_PATH_PROFILE) { return profile; } throw new Error( `OPENCLAW_DOCKER_ALL_PROFILE must be one of: ${DEFAULT_PROFILE}, ${RELEASE_PATH_PROFILE}. Got: ${JSON.stringify(raw)}`, ); } function applyLiveMode(poolLanes, mode) { if (mode === "all") { return poolLanes; } return poolLanes.filter((poolLane) => (mode === "only" ? poolLane.live : !poolLane.live)); } function applyLiveRetries(poolLanes, retries) { return poolLanes.map((poolLane) => (poolLane.live ? { ...poolLane, retries } : poolLane)); } function resourceLimitsSummary(resourceLimits) { return Object.entries(resourceLimits) .map(([resource, limit]) => `${resource}=${String(limit)}`) .join(" "); } function resourceLimitEnvName(resource) { return `OPENCLAW_DOCKER_ALL_${resource.toUpperCase().replace(/[^A-Z0-9]+/g, "_")}_LIMIT`; } function parseResourceLimit(env, resource, parallelism, fallback) { const envName = resourceLimitEnvName(resource); return parsePositiveInt(env[envName], Math.min(parallelism, fallback), envName); } function parseSchedulerOptions(env, parallelism) { const weightLimit = parsePositiveInt( env.OPENCLAW_DOCKER_ALL_WEIGHT_LIMIT, parallelism, "OPENCLAW_DOCKER_ALL_WEIGHT_LIMIT", ); const resourceLimits = {}; for (const [resource, fallback] of Object.entries(DEFAULT_RESOURCE_LIMITS)) { resourceLimits[resource] = parseResourceLimit(env, resource, parallelism, fallback); } return { resourceLimits, weightLimit, }; } function laneWeight(poolLane) { return Math.max(1, poolLane.weight ?? 1); } function laneResources(poolLane) { return ["docker", ...(poolLane.resources ?? [])]; } function laneSummary(poolLane) { const resources = laneResources(poolLane).join(","); const timeout = poolLane.timeoutMs ? ` timeout=${Math.round(poolLane.timeoutMs / 1000)}s` : ""; const retries = poolLane.retries > 0 ? ` retries=${poolLane.retries}` : ""; const cache = poolLane.cacheKey ? ` cache=${poolLane.cacheKey}` : ""; const image = poolLane.e2eImageKind ? ` image=${poolLane.e2eImageKind}` : ""; return `${poolLane.name}(w=${laneWeight(poolLane)} r=${resources}${timeout}${retries}${cache}${image})`; } function sleep(ms) { return new Promise((resolve) => { setTimeout(resolve, ms); }); } function utcStampForPath() { return new Date().toISOString().replaceAll("-", "").replaceAll(":", "").replace(/\..*$/, "Z"); } function utcStamp() { return new Date().toISOString().replace(/\..*$/, "Z"); } function appendExtension(env, extension) { const current = env.OPENCLAW_DOCKER_BUILD_EXTENSIONS ?? env.OPENCLAW_EXTENSIONS ?? ""; const tokens = current.split(/\s+/).filter(Boolean); if (!tokens.includes(extension)) { tokens.push(extension); } env.OPENCLAW_DOCKER_BUILD_EXTENSIONS = tokens.join(" "); } function commandEnv(extra = {}) { return { ...process.env, ...extra, }; } function shellQuote(value) { return `'${String(value).replaceAll("'", "'\\''")}'`; } function buildLaneRerunCommand(name, baseEnv) { const poolLane = findLaneByName(name); const build = name.startsWith("live-") ? "1" : "0"; const image = poolLane ? e2eImageForLane(poolLane, baseEnv) : baseEnv.OPENCLAW_DOCKER_E2E_IMAGE; const env = [ ["OPENCLAW_DOCKER_ALL_LANES", name], ["OPENCLAW_DOCKER_ALL_BUILD", build], ["OPENCLAW_DOCKER_ALL_PREFLIGHT", "0"], ["OPENCLAW_SKIP_DOCKER_BUILD", "1"], ["OPENCLAW_DOCKER_E2E_IMAGE", image || DEFAULT_E2E_IMAGE], ["OPENCLAW_DOCKER_E2E_BARE_IMAGE", baseEnv.OPENCLAW_DOCKER_E2E_BARE_IMAGE], ["OPENCLAW_DOCKER_E2E_FUNCTIONAL_IMAGE", baseEnv.OPENCLAW_DOCKER_E2E_FUNCTIONAL_IMAGE], ["OPENCLAW_CURRENT_PACKAGE_TGZ", baseEnv.OPENCLAW_CURRENT_PACKAGE_TGZ], ]; return `${env .filter(([, value]) => value !== undefined && value !== "") .map(([key, value]) => `${key}=${shellQuote(value)}`) .join(" ")} pnpm test:docker:all`; } function findLaneByName(name) { return dedupeLanes([ ...allReleasePathLanes({ includeOpenWebUI: true }), ...lanes, ...tailLanes, ]).find((poolLane) => poolLane.name === name); } function e2eImageForKind(kind, baseEnv) { if (kind === "bare") { return baseEnv.OPENCLAW_DOCKER_E2E_BARE_IMAGE || baseEnv.OPENCLAW_DOCKER_E2E_IMAGE; } if (kind === "functional") { return baseEnv.OPENCLAW_DOCKER_E2E_FUNCTIONAL_IMAGE || baseEnv.OPENCLAW_DOCKER_E2E_IMAGE; } return baseEnv.OPENCLAW_DOCKER_E2E_IMAGE; } function e2eImageForLane(poolLane, baseEnv) { return e2eImageForKind(poolLane.e2eImageKind, baseEnv); } function lanesNeedE2eImageKind(poolLanes, kind) { return poolLanes.some((poolLane) => poolLane.e2eImageKind === kind); } function timingSeconds(timingStore, poolLane) { const fromStore = timingStore?.lanes?.[poolLane.name]?.durationSeconds; if (typeof fromStore === "number" && Number.isFinite(fromStore) && fromStore > 0) { return fromStore; } return poolLane.estimateSeconds ?? 0; } function orderLanes(poolLanes, timingStore) { return poolLanes .map((poolLane, index) => ({ index, poolLane, seconds: timingSeconds(timingStore, poolLane) })) .toSorted((a, b) => b.seconds - a.seconds || a.index - b.index) .map(({ poolLane }) => poolLane); } async function loadTimingStore(file, enabled) { if (!enabled) { return { enabled: false, file, lanes: {}, version: 1 }; } const raw = await readFile(file, "utf8").catch(() => ""); if (!raw.trim()) { return { enabled: true, file, lanes: {}, version: 1 }; } try { const parsed = JSON.parse(raw); return { enabled: true, file, lanes: parsed && typeof parsed.lanes === "object" && parsed.lanes ? parsed.lanes : {}, version: 1, }; } catch (error) { console.warn( `WARN: ignoring unreadable Docker lane timings ${file}: ${ error instanceof Error ? error.message : String(error) }`, ); return { enabled: true, file, lanes: {}, version: 1 }; } } async function writeTimingStore(timingStore, results) { if (!timingStore.enabled || results.length === 0) { return; } const next = { lanes: { ...timingStore.lanes }, updatedAt: new Date().toISOString(), version: 1, }; for (const result of results) { if (!result || typeof result.elapsedSeconds !== "number") { continue; } next.lanes[result.name] = { durationSeconds: result.elapsedSeconds, status: result.status, timedOut: result.timedOut, updatedAt: new Date().toISOString(), }; } await mkdir(path.dirname(timingStore.file), { recursive: true }); await fs.promises.writeFile(timingStore.file, `${JSON.stringify(next, null, 2)}\n`); timingStore.lanes = next.lanes; console.log(`==> Docker lane timings: ${timingStore.file}`); } async function writeRunSummary(logDir, summary) { const file = path.join(logDir, "summary.json"); const payload = { ...summary, finishedAt: new Date().toISOString(), version: 1, }; await fs.promises.writeFile(file, `${JSON.stringify(payload, null, 2)}\n`); console.log(`==> Docker run summary: ${file}`); } function phaseElapsedSeconds(startedAtMs) { return Math.round((Date.now() - startedAtMs) / 1000); } async function runPhase(phases, name, details, fn) { const startedAtMs = Date.now(); const phase = { ...details, name, startedAt: new Date(startedAtMs).toISOString(), }; try { const result = await fn(); phase.status = "passed"; return result; } catch (error) { phase.status = "failed"; phase.error = error instanceof Error ? error.message : String(error); throw error; } finally { phase.elapsedSeconds = phaseElapsedSeconds(startedAtMs); phase.finishedAt = new Date().toISOString(); phases.push(phase); console.log(`==> Phase ${phase.status}: ${name} ${phase.elapsedSeconds}s`); } } function printLaneManifest(label, poolLanes, timingStore) { console.log(`==> ${label} lanes (${poolLanes.length})`); for (const [index, poolLane] of poolLanes.entries()) { const seconds = timingSeconds(timingStore, poolLane); const estimate = seconds > 0 ? ` last=${Math.round(seconds)}s` : ""; console.log(` ${index + 1}. ${laneSummary(poolLane)}${estimate}`); } } function lanesNeedOpenClawPackage(poolLanes) { return poolLanes.some((poolLane) => poolLane.e2eImageKind); } function dockerPreflightContainerNames(raw) { return raw .split(/\r?\n/) .map((line) => line.trim().split(/\s+/, 1)[0]) .filter((name) => /^(?:openclaw-(?:gateway-e2e|openwebui|openwebui-gateway|config-reload-e2e)-)/.test(name), ); } function runShellCommand({ command, env, label, logFile, timeoutMs }) { return new Promise((resolve) => { const child = spawn("bash", ["-c", command], { cwd: ROOT_DIR, detached: process.platform !== "win32", env, stdio: logFile ? ["ignore", "pipe", "pipe"] : "inherit", }); activeChildren.add(child); let timedOut = false; let killTimer; const timeoutTimer = timeoutMs > 0 ? setTimeout(() => { timedOut = true; if (stream) { stream.write(`\n==> [${label}] timeout after ${timeoutMs}ms; sending SIGTERM\n`); } terminateChild(child, "SIGTERM"); killTimer = setTimeout(() => terminateChild(child, "SIGKILL"), 10_000); killTimer.unref?.(); }, timeoutMs) : undefined; timeoutTimer?.unref?.(); let stream; if (logFile) { stream = fs.createWriteStream(logFile, { flags: "a" }); stream.write(`==> [${label}] command: ${command}\n`); stream.write(`==> [${label}] started: ${utcStamp()}\n`); child.stdout.pipe(stream, { end: false }); child.stderr.pipe(stream, { end: false }); } child.on("close", (status, signal) => { if (timeoutTimer) { clearTimeout(timeoutTimer); } if (killTimer) { clearTimeout(killTimer); } activeChildren.delete(child); const exitCode = typeof status === "number" ? status : signal ? 128 : 1; if (stream) { stream.write(`\n==> [${label}] finished: ${utcStamp()} status=${exitCode}\n`); stream.end(); } resolve({ signal, status: exitCode, timedOut }); }); }); } function runShellCaptureCommand({ command, env, label, timeoutMs }) { return new Promise((resolve) => { const child = spawn("bash", ["-c", command], { cwd: ROOT_DIR, detached: process.platform !== "win32", env, stdio: ["ignore", "pipe", "pipe"], }); activeChildren.add(child); let stdout = ""; let stderr = ""; let timedOut = false; const timeoutTimer = timeoutMs > 0 ? setTimeout(() => { timedOut = true; terminateChild(child, "SIGTERM"); setTimeout(() => terminateChild(child, "SIGKILL"), 10_000).unref?.(); }, timeoutMs) : undefined; timeoutTimer?.unref?.(); child.stdout.on("data", (chunk) => { stdout += String(chunk); }); child.stderr.on("data", (chunk) => { stderr += String(chunk); }); child.on("close", (status, signal) => { if (timeoutTimer) { clearTimeout(timeoutTimer); } activeChildren.delete(child); const exitCode = typeof status === "number" ? status : signal ? 128 : 1; resolve({ label, signal, status: exitCode, stderr, stdout, timedOut }); }); }); } async function runForeground(label, command, env) { console.log(`==> ${label}`); const result = await runShellCommand({ command, env, label }); if (result.status !== 0) { throw new Error(`${label} failed with status ${result.status}`); } } async function runForegroundGroup(entries, env) { const failures = []; for (const entry of entries) { try { const label = entry.label ?? entry[0]; const command = entry.command ?? entry[1]; const entryEnv = { ...env, ...entry.env }; const phases = entry.phases; const details = entry.phaseDetails ?? {}; if (phases) { await runPhase(phases, `build:${label}`, details, async () => { await runForeground(label, command, entryEnv); }); } else { await runForeground(label, command, entryEnv); } } catch (error) { failures.push({ entry, error }); } } if (failures.length > 0) { throw new Error( failures .map( ({ entry, error }) => `${entry.label ?? entry[0]}: ${error instanceof Error ? error.message : String(error)}`, ) .join("\n"), ); } } async function runDockerPreflight(baseEnv, options) { if (!options.enabled) { console.log("==> Docker preflight: skipped"); return; } console.log("==> Docker preflight"); const version = await runShellCaptureCommand({ command: "docker version --format '{{.Server.Version}}'", env: baseEnv, label: "docker-version", timeoutMs: 20_000, }); if (version.status !== 0) { throw new Error( `Docker preflight failed: docker version status=${version.status}\n${version.stderr}${version.stdout}`, ); } console.log(`==> Docker server: ${version.stdout.trim()}`); if (options.cleanup) { const stale = await runShellCaptureCommand({ command: "docker ps -a --filter status=created --filter status=exited --filter status=dead --format '{{.Names}} {{.Status}}'", env: baseEnv, label: "docker-stale-list", timeoutMs: 20_000, }); if (stale.status === 0) { const names = dockerPreflightContainerNames(stale.stdout); if (names.length > 0) { console.log(`==> Docker preflight cleanup: ${names.join(", ")}`); const cleanup = await runShellCommand({ command: `docker rm -f ${names.map(shellQuote).join(" ")}`, env: baseEnv, label: "docker-stale-cleanup", timeoutMs: 90_000, }); if (cleanup.status !== 0) { throw new Error(`Docker preflight cleanup failed with status ${cleanup.status}`); } } } } const startedAt = Date.now(); const run = await runShellCommand({ command: "docker run --rm alpine:3.20 true", env: baseEnv, label: "docker-run-smoke", timeoutMs: options.runTimeoutMs, }); const elapsedSeconds = Math.round((Date.now() - startedAt) / 1000); if (run.status !== 0) { throw new Error( `Docker preflight failed: docker run alpine:3.20 true status=${run.status} elapsed=${elapsedSeconds}s`, ); } console.log(`==> Docker preflight run: ${elapsedSeconds}s`); } async function prepareOpenClawPackage(baseEnv, logDir) { const existing = baseEnv.OPENCLAW_CURRENT_PACKAGE_TGZ || baseEnv.OPENCLAW_BUNDLED_CHANNEL_PACKAGE_TGZ || baseEnv.OPENCLAW_NPM_ONBOARD_PACKAGE_TGZ; if (existing) { const packageTgz = path.resolve(existing); baseEnv.OPENCLAW_CURRENT_PACKAGE_TGZ = packageTgz; baseEnv.OPENCLAW_BUNDLED_CHANNEL_PACKAGE_TGZ ||= packageTgz; baseEnv.OPENCLAW_NPM_ONBOARD_PACKAGE_TGZ ||= packageTgz; baseEnv.OPENCLAW_BUNDLED_CHANNEL_HOST_BUILD = "0"; baseEnv.OPENCLAW_NPM_ONBOARD_HOST_BUILD = "0"; console.log(`==> OpenClaw package: ${packageTgz}`); return; } const packDir = path.join(logDir, "openclaw-package"); await mkdir(packDir, { recursive: true }); await runForeground("Build OpenClaw package artifacts once", "pnpm build", baseEnv); await runForeground( "Write OpenClaw package inventory", "node --import tsx --input-type=module -e \"const { writePackageDistInventory } = await import('./src/infra/package-dist-inventory.ts'); await writePackageDistInventory(process.cwd());\"", baseEnv, ); await runForeground( "Pack OpenClaw package once", `npm pack --silent --ignore-scripts --pack-destination ${shellQuote(packDir)}`, baseEnv, ); const packed = (await fs.promises.readdir(packDir)) .filter((entry) => /^openclaw-.*\.tgz$/.test(entry)) .toSorted() .at(-1); if (!packed) { throw new Error(`missing packed OpenClaw tarball in ${packDir}`); } baseEnv.OPENCLAW_CURRENT_PACKAGE_TGZ = path.join(packDir, packed); baseEnv.OPENCLAW_BUNDLED_CHANNEL_PACKAGE_TGZ = baseEnv.OPENCLAW_CURRENT_PACKAGE_TGZ; baseEnv.OPENCLAW_BUNDLED_CHANNEL_HOST_BUILD = "0"; baseEnv.OPENCLAW_NPM_ONBOARD_PACKAGE_TGZ = baseEnv.OPENCLAW_CURRENT_PACKAGE_TGZ; baseEnv.OPENCLAW_NPM_ONBOARD_HOST_BUILD = "0"; console.log(`==> OpenClaw package: ${baseEnv.OPENCLAW_CURRENT_PACKAGE_TGZ}`); } function laneEnv(poolLane, baseEnv, logDir, cacheKey) { const env = { ...baseEnv, }; const name = poolLane.name; const image = e2eImageForLane(poolLane, baseEnv); if (image) { env.OPENCLAW_DOCKER_E2E_IMAGE = image; } if (poolLane.e2eImageKind) { env.OPENCLAW_DOCKER_E2E_IMAGE_KIND = poolLane.e2eImageKind; } const cacheName = cacheKey || name; if (!process.env.OPENCLAW_DOCKER_CLI_TOOLS_DIR) { env.OPENCLAW_DOCKER_CLI_TOOLS_DIR = path.join(logDir, `${cacheName}-cli-tools`); } if (!process.env.OPENCLAW_DOCKER_CACHE_HOME_DIR) { env.OPENCLAW_DOCKER_CACHE_HOME_DIR = path.join(logDir, `${cacheName}-cache`); } return env; } async function runLane(lane, baseEnv, logDir, fallbackTimeoutMs) { const { command, name } = lane; const timeoutMs = lane.timeoutMs ?? fallbackTimeoutMs; const logFile = path.join(logDir, `${name}.log`); const env = laneEnv(lane, baseEnv, logDir, lane.cacheKey); await mkdir(env.OPENCLAW_DOCKER_CLI_TOOLS_DIR, { recursive: true }); await mkdir(env.OPENCLAW_DOCKER_CACHE_HOME_DIR, { recursive: true }); await fs.promises.writeFile( logFile, [ `==> [${name}] cli tools dir: ${env.OPENCLAW_DOCKER_CLI_TOOLS_DIR}`, `==> [${name}] cache dir: ${env.OPENCLAW_DOCKER_CACHE_HOME_DIR}`, `==> [${name}] timeout: ${timeoutMs}ms`, `==> [${name}] retries: ${lane.retries ?? 0}`, `==> [${name}] e2e image kind: ${lane.e2eImageKind ?? "none"}`, `==> [${name}] e2e image: ${env.OPENCLAW_DOCKER_E2E_IMAGE ?? ""}`, "", ].join("\n"), ); console.log(`==> [${name}] start`); const startedAt = Date.now(); const startedAtIso = new Date(startedAt).toISOString(); let result; const attempts = []; const maxAttempts = 1 + Math.max(0, lane.retries ?? 0); for (let attempt = 1; attempt <= maxAttempts; attempt += 1) { const attemptStartedAt = Date.now(); if (attempt > 1) { await fs.promises.appendFile(logFile, `\n==> [${name}] retry attempt ${attempt}\n`); console.log(`==> [${name}] retry ${attempt}/${maxAttempts}`); } result = await runShellCommand({ command, env, label: name, logFile, timeoutMs }); attempts.push({ attempt, elapsedSeconds: phaseElapsedSeconds(attemptStartedAt), finishedAt: new Date().toISOString(), startedAt: new Date(attemptStartedAt).toISOString(), status: result.status, timedOut: result.timedOut, }); if (result.status === 0 || attempt >= maxAttempts) { break; } const retryable = result.timedOut || (await laneLogMatchesRetryPattern(logFile, lane.retryPatterns)); if (!retryable) { break; } } const elapsedSeconds = Math.round((Date.now() - startedAt) / 1000); if (result.status === 0) { console.log(`==> [${name}] pass ${elapsedSeconds}s`); } else { const timeoutLabel = result.timedOut ? " timeout" : ""; console.error( `==> [${name}] fail${timeoutLabel} status=${result.status} ${elapsedSeconds}s log=${logFile}`, ); } return { command, attempts, finishedAt: new Date().toISOString(), image: env.OPENCLAW_DOCKER_E2E_IMAGE, imageKind: lane.e2eImageKind, logFile, name, elapsedSeconds, rerunCommand: buildLaneRerunCommand(name, baseEnv), startedAt: startedAtIso, status: result.status, timedOut: result.timedOut, }; } async function runLanePool(poolLanes, baseEnv, logDir, parallelism, options) { const failures = []; const results = []; const pending = [...poolLanes]; const running = new Set(); const active = { count: 0, resources: new Map(), weight: 0, }; const activeLanes = new Map(); let lastLaneStartAt = 0; let laneStartQueue = Promise.resolve(); const statusTimer = options.statusIntervalMs > 0 ? setInterval(() => { const runningSummary = [...activeLanes.values()] .map((entry) => `${entry.name}:${Math.round((Date.now() - entry.startedAt) / 1000)}s`) .join(", "); const resources = [...active.resources.entries()] .map(([resource, value]) => `${resource}=${value}`) .join(" "); console.log( `==> [${options.poolLabel}] active=${active.count} pending=${pending.length} ${resources}${ runningSummary ? ` lanes=${runningSummary}` : "" }`, ); }, options.statusIntervalMs) : undefined; statusTimer?.unref?.(); async function waitForLaneStartSlot() { if (options.startStaggerMs <= 0) { return; } const previous = laneStartQueue; let releaseQueue; laneStartQueue = new Promise((resolve) => { releaseQueue = resolve; }); await previous; const waitMs = Math.max(0, lastLaneStartAt + options.startStaggerMs - Date.now()); if (waitMs > 0) { await sleep(waitMs); } lastLaneStartAt = Date.now(); releaseQueue(); } function canStartLane(candidate) { const weight = laneWeight(candidate); if (active.count >= parallelism || active.weight + weight > options.weightLimit) { return false; } for (const resource of laneResources(candidate)) { const limit = options.resourceLimits[resource] ?? options.weightLimit; const current = active.resources.get(resource) ?? 0; if (current + weight > limit) { return false; } } return true; } function reserve(candidate) { const weight = laneWeight(candidate); active.count += 1; active.weight += weight; for (const resource of laneResources(candidate)) { active.resources.set(resource, (active.resources.get(resource) ?? 0) + weight); } } function release(candidate) { const weight = laneWeight(candidate); active.count -= 1; active.weight -= weight; for (const resource of laneResources(candidate)) { const next = (active.resources.get(resource) ?? 0) - weight; if (next > 0) { active.resources.set(resource, next); } else { active.resources.delete(resource); } } } async function startLane(poolLane) { await waitForLaneStartSlot(); reserve(poolLane); activeLanes.set(poolLane.name, { name: poolLane.name, startedAt: Date.now() }); let promise; promise = runLane(poolLane, baseEnv, logDir, options.timeoutMs) .then((result) => ({ lane: poolLane, promise, result })) .finally(() => { activeLanes.delete(poolLane.name); release(poolLane); }); running.add(promise); } try { while (pending.length > 0 || running.size > 0) { let started = false; if (!options.failFast || failures.length === 0) { for (let index = 0; index < pending.length; ) { const candidate = pending[index]; if (!canStartLane(candidate)) { index += 1; continue; } pending.splice(index, 1); await startLane(candidate); started = true; } } if (started) { continue; } if (running.size === 0) { const blocked = pending.map(laneSummary).join(", "); throw new Error(`No Docker lanes fit scheduler limits: ${blocked}`); } const { promise, result } = await Promise.race(running); running.delete(promise); results.push(result); if (result.status !== 0) { failures.push(result); } if (options.failFast && failures.length > 0) { const remainingResults = await Promise.all(running); running.clear(); for (const remaining of remainingResults) { results.push(remaining.result); if (remaining.result.status !== 0) { failures.push(remaining.result); } } break; } } } finally { if (statusTimer) { clearInterval(statusTimer); } } return { failures, results }; } async function tailFile(file, lines) { const content = await readFile(file, "utf8").catch(() => ""); const tail = content.split(/\r?\n/).slice(-lines).join("\n"); return tail.trimEnd(); } async function laneLogMatchesRetryPattern(logFile, patterns) { if (!patterns || patterns.length === 0) { return false; } const tail = await tailFile(logFile, 160); return patterns.some((pattern) => pattern.test(tail)); } async function printFailureSummary(failures, tailLines) { console.error(`ERROR: ${failures.length} Docker lane(s) failed.`); for (const failure of failures) { console.error(`---- ${failure.name} failed (status=${failure.status}): ${failure.logFile}`); const tail = await tailFile(failure.logFile, tailLines); if (tail) { console.error(tail); } } } const activeChildren = new Set(); function terminateChild(child, signal) { if (process.platform !== "win32" && child.pid) { try { process.kill(-child.pid, signal); return; } catch { // Fall back to killing the direct child below. } } child.kill(signal); } function terminateActiveChildren(signal) { for (const child of activeChildren) { terminateChild(child, signal); } } process.on("SIGINT", () => { terminateActiveChildren("SIGINT"); process.exit(130); }); process.on("SIGTERM", () => { terminateActiveChildren("SIGTERM"); process.exit(143); }); async function main() { const runStartedAt = new Date().toISOString(); const phases = []; const parallelism = parsePositiveInt( process.env.OPENCLAW_DOCKER_ALL_PARALLELISM, DEFAULT_PARALLELISM, "OPENCLAW_DOCKER_ALL_PARALLELISM", ); const tailParallelism = parsePositiveInt( process.env.OPENCLAW_DOCKER_ALL_TAIL_PARALLELISM, Math.min(parallelism, DEFAULT_TAIL_PARALLELISM), "OPENCLAW_DOCKER_ALL_TAIL_PARALLELISM", ); const tailLines = parsePositiveInt( process.env.OPENCLAW_DOCKER_ALL_FAILURE_TAIL_LINES, DEFAULT_FAILURE_TAIL_LINES, "OPENCLAW_DOCKER_ALL_FAILURE_TAIL_LINES", ); const laneTimeoutMs = parsePositiveInt( process.env.OPENCLAW_DOCKER_ALL_LANE_TIMEOUT_MS, DEFAULT_LANE_TIMEOUT_MS, "OPENCLAW_DOCKER_ALL_LANE_TIMEOUT_MS", ); const laneStartStaggerMs = parseNonNegativeInt( process.env.OPENCLAW_DOCKER_ALL_START_STAGGER_MS, DEFAULT_LANE_START_STAGGER_MS, "OPENCLAW_DOCKER_ALL_START_STAGGER_MS", ); const statusIntervalMs = parseNonNegativeInt( process.env.OPENCLAW_DOCKER_ALL_STATUS_INTERVAL_MS, DEFAULT_STATUS_INTERVAL_MS, "OPENCLAW_DOCKER_ALL_STATUS_INTERVAL_MS", ); const preflightRunTimeoutMs = parsePositiveInt( process.env.OPENCLAW_DOCKER_ALL_PREFLIGHT_RUN_TIMEOUT_MS, DEFAULT_PREFLIGHT_RUN_TIMEOUT_MS, "OPENCLAW_DOCKER_ALL_PREFLIGHT_RUN_TIMEOUT_MS", ); const failFast = parseBool(process.env.OPENCLAW_DOCKER_ALL_FAIL_FAST, true); const dryRun = parseBool(process.env.OPENCLAW_DOCKER_ALL_DRY_RUN, false); const preflightEnabled = parseBool(process.env.OPENCLAW_DOCKER_ALL_PREFLIGHT, true); const preflightCleanup = parseBool(process.env.OPENCLAW_DOCKER_ALL_PREFLIGHT_CLEANUP, true); const timingsEnabled = parseBool(process.env.OPENCLAW_DOCKER_ALL_TIMINGS, true); const buildEnabled = parseBool(process.env.OPENCLAW_DOCKER_ALL_BUILD, true); const profile = parseProfile(process.env.OPENCLAW_DOCKER_ALL_PROFILE); const releaseChunk = process.env.OPENCLAW_DOCKER_ALL_CHUNK || process.env.DOCKER_E2E_CHUNK || ""; const includeOpenWebUI = parseBool( process.env.OPENCLAW_DOCKER_ALL_INCLUDE_OPENWEBUI ?? process.env.INCLUDE_OPENWEBUI, true, ); const selectedLaneNamesRaw = process.env.OPENCLAW_DOCKER_ALL_LANES || process.env.DOCKER_E2E_LANES || ""; const selectedLaneNames = parseLaneSelection(selectedLaneNamesRaw); if (selectedLaneNamesRaw && selectedLaneNames.length === 0) { throw new Error("OPENCLAW_DOCKER_ALL_LANES must include at least one lane name"); } const liveMode = parseLiveMode(process.env.OPENCLAW_DOCKER_ALL_LIVE_MODE); const liveRetries = parseNonNegativeInt( process.env.OPENCLAW_DOCKER_ALL_LIVE_RETRIES, DEFAULT_LIVE_RETRIES, "OPENCLAW_DOCKER_ALL_LIVE_RETRIES", ); const timingsFile = path.resolve( process.env.OPENCLAW_DOCKER_ALL_TIMINGS_FILE || DEFAULT_TIMINGS_FILE, ); const runId = process.env.OPENCLAW_DOCKER_ALL_RUN_ID || utcStampForPath(); const logDir = path.resolve( process.env.OPENCLAW_DOCKER_ALL_LOG_DIR || path.join(ROOT_DIR, ".artifacts/docker-tests", runId), ); await mkdir(logDir, { recursive: true }); const baseEnv = commandEnv({ OPENCLAW_DOCKER_E2E_BARE_IMAGE: process.env.OPENCLAW_DOCKER_E2E_BARE_IMAGE || process.env.OPENCLAW_DOCKER_E2E_IMAGE || DEFAULT_E2E_BARE_IMAGE, OPENCLAW_DOCKER_E2E_FUNCTIONAL_IMAGE: process.env.OPENCLAW_DOCKER_E2E_FUNCTIONAL_IMAGE || process.env.OPENCLAW_DOCKER_E2E_IMAGE || DEFAULT_E2E_FUNCTIONAL_IMAGE, }); baseEnv.OPENCLAW_DOCKER_E2E_IMAGE = process.env.OPENCLAW_DOCKER_E2E_IMAGE || baseEnv.OPENCLAW_DOCKER_E2E_FUNCTIONAL_IMAGE; appendExtension(baseEnv, "matrix"); appendExtension(baseEnv, "acpx"); appendExtension(baseEnv, "codex"); const timingStore = await loadTimingStore(timingsFile, timingsEnabled); const retriedMainLanes = applyLiveRetries(lanes, liveRetries); const retriedTailLanes = applyLiveRetries(tailLanes, liveRetries); const releaseLanes = selectedLaneNames.length === 0 && profile === RELEASE_PATH_PROFILE ? releasePathChunkLanes(releaseChunk, { includeOpenWebUI }) : undefined; const selectedLanes = selectedLaneNames.length > 0 ? selectNamedLanes( dedupeLanes([ ...allReleasePathLanes({ includeOpenWebUI }), ...retriedMainLanes, ...retriedTailLanes, ]), selectedLaneNames, "OPENCLAW_DOCKER_ALL_LANES", ) : undefined; const configuredLanes = selectedLanes ? selectedLanes : releaseLanes ? releaseLanes : liveMode === "only" ? applyLiveMode([...retriedMainLanes, ...retriedTailLanes], liveMode) : applyLiveMode(retriedMainLanes, liveMode); const configuredTailLanes = selectedLanes || releaseLanes ? [] : liveMode === "only" ? [] : applyLiveMode(retriedTailLanes, liveMode); const orderedLanes = orderLanes(configuredLanes, timingStore); const orderedTailLanes = orderLanes(configuredTailLanes, timingStore); console.log(`==> Docker test logs: ${logDir}`); console.log(`==> Profile: ${profile}${releaseChunk ? ` chunk=${releaseChunk}` : ""}`); console.log(`==> Parallelism: ${parallelism}`); console.log(`==> Tail parallelism: ${tailParallelism}`); console.log(`==> Lane timeout: ${laneTimeoutMs}ms`); console.log(`==> Live mode: ${liveMode}`); console.log(`==> Live retries: ${liveRetries}`); console.log(`==> Lane start stagger: ${laneStartStaggerMs}ms`); console.log(`==> Status interval: ${statusIntervalMs}ms`); console.log(`==> Fail fast: ${failFast ? "yes" : "no"}`); console.log(`==> Dry run: ${dryRun ? "yes" : "no"}`); console.log( `==> Docker preflight: ${preflightEnabled ? "yes" : "no"}${ preflightCleanup ? " cleanup=yes" : " cleanup=no" }`, ); console.log(`==> Build shared Docker images: ${buildEnabled ? "yes" : "no"}`); console.log(`==> Docker E2E bare image: ${baseEnv.OPENCLAW_DOCKER_E2E_BARE_IMAGE}`); console.log(`==> Docker E2E functional image: ${baseEnv.OPENCLAW_DOCKER_E2E_FUNCTIONAL_IMAGE}`); if (profile === RELEASE_PATH_PROFILE) { console.log(`==> Include Open WebUI: ${includeOpenWebUI ? "yes" : "no"}`); } if (selectedLaneNames.length > 0) { console.log(`==> Selected lanes: ${selectedLaneNames.join(", ")}`); } console.log(`==> Docker lane timings: ${timingStore.enabled ? timingsFile : "disabled"}`); console.log(`==> Live-test bundled plugin deps: ${baseEnv.OPENCLAW_DOCKER_BUILD_EXTENSIONS}`); const schedulerOptions = parseSchedulerOptions(process.env, parallelism); const tailSchedulerOptions = parseSchedulerOptions(process.env, tailParallelism); console.log( `==> Scheduler: weight=${schedulerOptions.weightLimit} ${resourceLimitsSummary(schedulerOptions.resourceLimits)}`, ); console.log( `==> Tail scheduler: weight=${tailSchedulerOptions.weightLimit} ${resourceLimitsSummary(tailSchedulerOptions.resourceLimits)}`, ); printLaneManifest("Main", orderedLanes, timingStore); printLaneManifest("Tail", orderedTailLanes, timingStore); if (dryRun) { console.log("==> Dry run complete"); return; } await runPhase( phases, "docker-preflight", { cleanup: preflightCleanup, enabled: preflightEnabled }, async () => { await runDockerPreflight(baseEnv, { cleanup: preflightCleanup, enabled: preflightEnabled, runTimeoutMs: preflightRunTimeoutMs, }); }, ); const scheduledLanes = [...orderedLanes, ...orderedTailLanes]; if (lanesNeedOpenClawPackage(scheduledLanes)) { await runPhase(phases, "prepare-openclaw-package", {}, async () => { await prepareOpenClawPackage(baseEnv, logDir); }); } else { console.log("==> OpenClaw package: not needed for selected lanes"); } if (buildEnabled) { const buildEntries = []; if (scheduledLanes.some((poolLane) => poolLane.live)) { buildEntries.push({ command: "pnpm test:docker:live-build", label: "shared live-test image once", phaseDetails: { imageKind: "live" }, phases, }); } if (lanesNeedE2eImageKind(scheduledLanes, "bare")) { buildEntries.push({ command: "pnpm test:docker:e2e-build", env: { OPENCLAW_DOCKER_E2E_IMAGE: baseEnv.OPENCLAW_DOCKER_E2E_BARE_IMAGE, OPENCLAW_DOCKER_E2E_TARGET: "bare", }, label: `shared bare Docker E2E image once: ${baseEnv.OPENCLAW_DOCKER_E2E_BARE_IMAGE}`, phaseDetails: { image: baseEnv.OPENCLAW_DOCKER_E2E_BARE_IMAGE, imageKind: "bare" }, phases, }); } if (lanesNeedE2eImageKind(scheduledLanes, "functional")) { buildEntries.push({ command: "pnpm test:docker:e2e-build", env: { OPENCLAW_DOCKER_E2E_IMAGE: baseEnv.OPENCLAW_DOCKER_E2E_FUNCTIONAL_IMAGE, OPENCLAW_DOCKER_E2E_TARGET: "functional", }, label: `shared functional Docker E2E image once: ${baseEnv.OPENCLAW_DOCKER_E2E_FUNCTIONAL_IMAGE}`, phaseDetails: { image: baseEnv.OPENCLAW_DOCKER_E2E_FUNCTIONAL_IMAGE, imageKind: "functional", }, phases, }); } await runForegroundGroup(buildEntries, baseEnv); } else { console.log(`==> Shared Docker image builds: skipped`); } const options = { ...schedulerOptions, failFast, poolLabel: "main", startStaggerMs: laneStartStaggerMs, statusIntervalMs, timeoutMs: laneTimeoutMs, }; const mainResult = await runPhase(phases, "main-lane-pool", { lanes: orderedLanes.length }, () => runLanePool(orderedLanes, baseEnv, logDir, parallelism, options), ); const failures = [...mainResult.failures]; const allResults = [...mainResult.results]; await writeTimingStore(timingStore, mainResult.results); if (failFast && failures.length > 0) { await writeRunSummary(logDir, { chunk: releaseChunk || undefined, failures, image: baseEnv.OPENCLAW_DOCKER_E2E_IMAGE, images: { bare: baseEnv.OPENCLAW_DOCKER_E2E_BARE_IMAGE, functional: baseEnv.OPENCLAW_DOCKER_E2E_FUNCTIONAL_IMAGE, }, lanes: allResults, phases, profile, selectedLanes: selectedLaneNames.length > 0 ? selectedLaneNames : undefined, startedAt: runStartedAt, status: "failed", }); await printFailureSummary(failures, tailLines); process.exit(1); } if (orderedTailLanes.length > 0) { console.log("==> Running provider-sensitive Docker tail lanes"); const tailResult = await runPhase( phases, "tail-lane-pool", { lanes: orderedTailLanes.length }, () => runLanePool(orderedTailLanes, baseEnv, logDir, tailParallelism, { ...options, ...tailSchedulerOptions, poolLabel: "tail", }), ); failures.push(...tailResult.failures); allResults.push(...tailResult.results); await writeTimingStore(timingStore, tailResult.results); } else { console.log("==> Provider-sensitive Docker tail lanes: none"); } if (failures.length > 0) { await writeRunSummary(logDir, { chunk: releaseChunk || undefined, failures, image: baseEnv.OPENCLAW_DOCKER_E2E_IMAGE, images: { bare: baseEnv.OPENCLAW_DOCKER_E2E_BARE_IMAGE, functional: baseEnv.OPENCLAW_DOCKER_E2E_FUNCTIONAL_IMAGE, }, lanes: allResults, phases, profile, selectedLanes: selectedLaneNames.length > 0 ? selectedLaneNames : undefined, startedAt: runStartedAt, status: "failed", }); await printFailureSummary(failures, tailLines); process.exit(1); } if (profile === DEFAULT_PROFILE && selectedLaneNames.length === 0) { await runPhase(phases, "cleanup-smoke", {}, async () => { await runForeground( "Run cleanup smoke after parallel lanes", "pnpm test:docker:cleanup", baseEnv, ); }); } else { console.log("==> Cleanup smoke after parallel lanes: skipped for selected/release lanes"); } await writeTimingStore(timingStore, allResults); await writeRunSummary(logDir, { chunk: releaseChunk || undefined, failures, image: baseEnv.OPENCLAW_DOCKER_E2E_IMAGE, images: { bare: baseEnv.OPENCLAW_DOCKER_E2E_BARE_IMAGE, functional: baseEnv.OPENCLAW_DOCKER_E2E_FUNCTIONAL_IMAGE, }, lanes: allResults, phases, profile, selectedLanes: selectedLaneNames.length > 0 ? selectedLaneNames : undefined, startedAt: runStartedAt, status: "passed", }); console.log("==> Docker test suite passed"); } await main().catch((error) => { console.error(error instanceof Error ? error.message : String(error)); process.exit(1); });