diff --git a/.github/actions/setup-node-env/action.yml b/.github/actions/setup-node-env/action.yml index 6b402040896..55ab77971b5 100644 --- a/.github/actions/setup-node-env/action.yml +++ b/.github/actions/setup-node-env/action.yml @@ -1,7 +1,7 @@ name: Setup Node environment description: > - Initialize submodules with retry, install Node 24 by default, pnpm, optionally Bun, - and optionally run pnpm install. Requires actions/checkout to run first. + Install Node 24 by default, pnpm, optionally Bun, and optionally run pnpm + install. Requires actions/checkout to run first. inputs: node-version: description: Node.js version to install. @@ -34,20 +34,6 @@ inputs: runs: using: composite steps: - - name: Checkout submodules (retry) - shell: bash - run: | - set -euo pipefail - git submodule sync --recursive - for attempt in 1 2 3 4 5; do - if git -c protocol.version=2 submodule update --init --force --depth=1 --recursive; then - exit 0 - fi - echo "Submodule update failed (attempt $attempt/5). Retrying…" - sleep $((attempt * 10)) - done - exit 1 - - name: Setup Node.js uses: actions/setup-node@v6 with: diff --git a/.github/workflows/ci-bun.yml b/.github/workflows/ci-bun.yml index aeb20874274..2e0e1b66621 100644 --- a/.github/workflows/ci-bun.yml +++ b/.github/workflows/ci-bun.yml @@ -12,7 +12,42 @@ env: FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: "true" jobs: + preflight: + runs-on: blacksmith-16vcpu-ubuntu-2404 + timeout-minutes: 20 + outputs: + run_bun_checks: ${{ steps.manifest.outputs.run_bun_checks }} + bun_checks_matrix: ${{ steps.manifest.outputs.bun_checks_matrix }} + steps: + - name: Checkout + uses: actions/checkout@v6 + with: + submodules: false + + - name: Setup Node environment + uses: ./.github/actions/setup-node-env + with: + install-bun: "false" + install-deps: "false" + use-sticky-disk: "false" + + - name: Build Bun CI manifest + id: manifest + env: + OPENCLAW_CI_DOCS_ONLY: "false" + OPENCLAW_CI_DOCS_CHANGED: "false" + OPENCLAW_CI_RUN_NODE: "true" + OPENCLAW_CI_RUN_MACOS: "false" + OPENCLAW_CI_RUN_ANDROID: "false" + OPENCLAW_CI_RUN_WINDOWS: "false" + OPENCLAW_CI_RUN_SKILLS_PYTHON: "false" + OPENCLAW_CI_HAS_CHANGED_EXTENSIONS: "false" + OPENCLAW_CI_CHANGED_EXTENSIONS_MATRIX: '{"include":[]}' + run: node scripts/ci-write-manifest-outputs.mjs --workflow ci-bun + build-bun-artifacts: + needs: [preflight] + if: needs.preflight.outputs.run_bun_checks == 'true' runs-on: blacksmith-16vcpu-ubuntu-2404 timeout-minutes: 20 steps: @@ -37,25 +72,14 @@ jobs: path: src/canvas-host/a2ui/ bun-checks: - needs: [build-bun-artifacts] + name: ${{ matrix.check_name }} + needs: [preflight, build-bun-artifacts] + if: needs.preflight.outputs.run_bun_checks == 'true' runs-on: blacksmith-16vcpu-ubuntu-2404 timeout-minutes: 20 strategy: fail-fast: false - matrix: - include: - - shard_index: 1 - shard_count: 4 - command: OPENCLAW_TEST_ISOLATE=1 bunx vitest run --config vitest.unit.config.ts --shard 1/4 - - shard_index: 2 - shard_count: 4 - command: OPENCLAW_TEST_ISOLATE=1 bunx vitest run --config vitest.unit.config.ts --shard 2/4 - - shard_index: 3 - shard_count: 4 - command: OPENCLAW_TEST_ISOLATE=1 bunx vitest run --config vitest.unit.config.ts --shard 3/4 - - shard_index: 4 - shard_count: 4 - command: OPENCLAW_TEST_ISOLATE=1 bunx vitest run --config vitest.unit.config.ts --shard 4/4 + matrix: ${{ fromJson(needs.preflight.outputs.bun_checks_matrix) }} steps: - name: Checkout uses: actions/checkout@v6 @@ -75,4 +99,10 @@ jobs: path: src/canvas-host/a2ui/ - name: Run Bun test shard - run: ${{ matrix.command }} + env: + SHARD_COUNT: ${{ matrix.shard_count }} + SHARD_INDEX: ${{ matrix.shard_index }} + shell: bash + run: | + set -euo pipefail + OPENCLAW_TEST_ISOLATE=1 bunx vitest run --config vitest.unit.config.ts --shard "$SHARD_INDEX/$SHARD_COUNT" diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 7f4e8af5785..85877b195a3 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -17,26 +17,42 @@ env: FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: "true" jobs: - # Scope: establish the fast global truth for this revision before the - # expensive platform and platform-specific lanes fan out. - # Detect docs-only changes to skip heavy jobs (test, build, Windows, macOS, Android). - # Keep this job focused on routing decisions so the rest of CI can fan out sooner. - # Fail-safe: if detection steps are skipped, downstream outputs fall back to - # conservative defaults that keep heavy lanes enabled. - scope: + # Preflight: establish routing truth and planner-owned matrices once, then let + # real work fan out from a single source of truth. + preflight: if: github.event_name != 'pull_request' || !github.event.pull_request.draft runs-on: blacksmith-16vcpu-ubuntu-2404 timeout-minutes: 20 outputs: - docs_only: ${{ steps.docs_scope.outputs.docs_only }} - docs_changed: ${{ steps.docs_scope.outputs.docs_changed }} - run_node: ${{ steps.changed_scope.outputs.run_node || 'false' }} - run_macos: ${{ steps.changed_scope.outputs.run_macos || 'false' }} - run_android: ${{ steps.changed_scope.outputs.run_android || 'false' }} - run_skills_python: ${{ steps.changed_scope.outputs.run_skills_python || 'false' }} - run_windows: ${{ steps.changed_scope.outputs.run_windows || 'false' }} - has_changed_extensions: ${{ steps.changed_extensions.outputs.has_changed_extensions || 'false' }} - changed_extensions_matrix: ${{ steps.changed_extensions.outputs.changed_extensions_matrix || '{"include":[]}' }} + docs_only: ${{ steps.manifest.outputs.docs_only }} + docs_changed: ${{ steps.manifest.outputs.docs_changed }} + run_node: ${{ steps.manifest.outputs.run_node }} + run_macos: ${{ steps.manifest.outputs.run_macos }} + run_android: ${{ steps.manifest.outputs.run_android }} + run_skills_python: ${{ steps.manifest.outputs.run_skills_python }} + run_skills_python_job: ${{ steps.manifest.outputs.run_skills_python_job }} + run_windows: ${{ steps.manifest.outputs.run_windows }} + has_changed_extensions: ${{ steps.manifest.outputs.has_changed_extensions }} + changed_extensions_matrix: ${{ steps.manifest.outputs.changed_extensions_matrix }} + run_build_artifacts: ${{ steps.manifest.outputs.run_build_artifacts }} + run_release_check: ${{ steps.manifest.outputs.run_release_check }} + run_checks_fast: ${{ steps.manifest.outputs.run_checks_fast }} + checks_fast_matrix: ${{ steps.manifest.outputs.checks_fast_matrix }} + run_checks: ${{ steps.manifest.outputs.run_checks }} + checks_matrix: ${{ steps.manifest.outputs.checks_matrix }} + run_extension_fast: ${{ steps.manifest.outputs.run_extension_fast }} + extension_fast_matrix: ${{ steps.manifest.outputs.extension_fast_matrix }} + run_check: ${{ steps.manifest.outputs.run_check }} + run_check_additional: ${{ steps.manifest.outputs.run_check_additional }} + run_build_smoke: ${{ steps.manifest.outputs.run_build_smoke }} + run_check_docs: ${{ steps.manifest.outputs.run_check_docs }} + run_checks_windows: ${{ steps.manifest.outputs.run_checks_windows }} + checks_windows_matrix: ${{ steps.manifest.outputs.checks_windows_matrix }} + run_macos_node: ${{ steps.manifest.outputs.run_macos_node }} + macos_node_matrix: ${{ steps.manifest.outputs.macos_node_matrix }} + run_macos_swift: ${{ steps.manifest.outputs.run_macos_swift }} + run_android_job: ${{ steps.manifest.outputs.run_android_job }} + android_matrix: ${{ steps.manifest.outputs.android_matrix }} steps: - name: Checkout uses: actions/checkout@v6 @@ -56,8 +72,6 @@ jobs: id: docs_scope uses: ./.github/actions/detect-docs-changes - # Detect which heavy areas are touched so CI can skip unrelated expensive jobs. - # Fail-safe: if skipped, downstream lanes run. - name: Detect changed scopes id: changed_scope if: steps.docs_scope.outputs.docs_only != 'true' @@ -104,6 +118,20 @@ jobs: appendFileSync(process.env.GITHUB_OUTPUT, `changed_extensions_matrix=${matrix}\n`, "utf8"); EOF + - name: Build CI manifest + id: manifest + env: + OPENCLAW_CI_DOCS_ONLY: ${{ steps.docs_scope.outputs.docs_only }} + OPENCLAW_CI_DOCS_CHANGED: ${{ steps.docs_scope.outputs.docs_changed }} + OPENCLAW_CI_RUN_NODE: ${{ steps.changed_scope.outputs.run_node || 'false' }} + OPENCLAW_CI_RUN_MACOS: ${{ steps.changed_scope.outputs.run_macos || 'false' }} + OPENCLAW_CI_RUN_ANDROID: ${{ steps.changed_scope.outputs.run_android || 'false' }} + OPENCLAW_CI_RUN_WINDOWS: ${{ steps.changed_scope.outputs.run_windows || 'false' }} + OPENCLAW_CI_RUN_SKILLS_PYTHON: ${{ steps.changed_scope.outputs.run_skills_python || 'false' }} + OPENCLAW_CI_HAS_CHANGED_EXTENSIONS: ${{ steps.changed_extensions.outputs.has_changed_extensions || 'false' }} + OPENCLAW_CI_CHANGED_EXTENSIONS_MATRIX: ${{ steps.changed_extensions.outputs.changed_extensions_matrix || '{"include":[]}' }} + run: node scripts/ci-write-manifest-outputs.mjs --workflow ci + # Run the fast security/SCM checks in parallel with scope detection so the # main Node jobs do not have to wait for Python/pre-commit setup. security-fast: @@ -205,8 +233,8 @@ jobs: # Keep this overlapping with the fast correctness lanes so green PRs get heavy # test/build feedback sooner instead of waiting behind a full `check` pass. build-artifacts: - needs: [scope] - if: needs.scope.outputs.docs_only != 'true' && needs.scope.outputs.run_node == 'true' + needs: [preflight] + if: needs.preflight.outputs.run_build_artifacts == 'true' runs-on: blacksmith-16vcpu-ubuntu-2404 timeout-minutes: 20 steps: @@ -252,8 +280,8 @@ jobs: # Validate npm pack contents after build (only on push to main, not PRs). release-check: - needs: [scope, build-artifacts] - if: github.event_name == 'push' && needs.scope.outputs.docs_only != 'true' + needs: [preflight, build-artifacts] + if: needs.preflight.outputs.run_release_check == 'true' runs-on: blacksmith-16vcpu-ubuntu-2404 timeout-minutes: 20 steps: @@ -279,22 +307,14 @@ jobs: run: pnpm release:check checks-fast: - needs: [scope] - if: needs.scope.outputs.docs_only != 'true' && needs.scope.outputs.run_node == 'true' + name: ${{ matrix.check_name }} + needs: [preflight] + if: needs.preflight.outputs.run_checks_fast == 'true' runs-on: blacksmith-16vcpu-ubuntu-2404 timeout-minutes: 20 strategy: fail-fast: false - matrix: - include: - - runtime: node - task: extensions - command: pnpm test:extensions - - runtime: node - task: contracts-protocol - command: | - pnpm test:contracts - pnpm protocol:check + matrix: ${{ fromJson(needs.preflight.outputs.checks_fast_matrix) }} steps: - name: Checkout uses: actions/checkout@v6 @@ -309,64 +329,34 @@ jobs: use-sticky-disk: "false" - name: Run ${{ matrix.task }} (${{ matrix.runtime }}) - run: ${{ matrix.command }} + env: + TASK: ${{ matrix.task }} + shell: bash + run: | + set -euo pipefail + case "$TASK" in + extensions) + pnpm test:extensions + ;; + contracts|contracts-protocol) + pnpm test:contracts + pnpm protocol:check + ;; + *) + echo "Unsupported checks-fast task: $TASK" >&2 + exit 1 + ;; + esac checks: - needs: [scope, build-artifacts] - if: always() && needs.scope.outputs.docs_only != 'true' && needs.scope.outputs.run_node == 'true' && needs.build-artifacts.result == 'success' + name: ${{ matrix.check_name }} + needs: [preflight, build-artifacts] + if: always() && needs.preflight.outputs.run_checks == 'true' && needs.build-artifacts.result == 'success' runs-on: blacksmith-16vcpu-ubuntu-2404 timeout-minutes: 20 strategy: fail-fast: false - matrix: - include: - - runtime: node - task: test - shard_index: 1 - shard_count: 4 - command: pnpm test - - runtime: node - task: test - shard_index: 2 - shard_count: 4 - command: pnpm test - - runtime: node - task: test - shard_index: 3 - shard_count: 4 - command: pnpm test - - runtime: node - task: test - shard_index: 4 - shard_count: 4 - command: pnpm test - - runtime: node - task: channels - shard_index: 1 - shard_count: 3 - command: pnpm test:channels - - runtime: node - task: channels - shard_index: 2 - shard_count: 3 - command: pnpm test:channels - - runtime: node - task: channels - shard_index: 3 - shard_count: 3 - command: pnpm test:channels - - runtime: node - task: compat-node22 - node_version: "22.x" - cache_key_suffix: "node22" - command: | - pnpm build - pnpm ui:build - node openclaw.mjs --help - node openclaw.mjs status --json --timeout 1 - pnpm test:build:singleton - node scripts/stage-bundled-plugin-runtime-deps.mjs - node --import tsx scripts/release-check.ts + matrix: ${{ fromJson(needs.preflight.outputs.checks_matrix) }} steps: - name: Skip compatibility lanes on pull requests if: github.event_name == 'pull_request' && matrix.task == 'compat-node22' @@ -391,6 +381,7 @@ jobs: - name: Configure Node test resources if: (github.event_name != 'pull_request' || matrix.task != 'compat-node22') && matrix.runtime == 'node' && (matrix.task == 'test' || matrix.task == 'channels' || matrix.task == 'compat-node22') env: + TASK: ${{ matrix.task }} SHARD_COUNT: ${{ matrix.shard_count || '' }} SHARD_INDEX: ${{ matrix.shard_index || '' }} run: | @@ -398,7 +389,7 @@ jobs: # Default heap limits have been too low on Linux CI (V8 OOM near 4GB). echo "OPENCLAW_TEST_WORKERS=2" >> "$GITHUB_ENV" echo "OPENCLAW_TEST_MAX_OLD_SPACE_SIZE_MB=6144" >> "$GITHUB_ENV" - if [ "${{ matrix.task }}" = "channels" ]; then + if [ "$TASK" = "channels" ]; then echo "OPENCLAW_TEST_WORKERS=1" >> "$GITHUB_ENV" echo "OPENCLAW_TEST_ISOLATE=1" >> "$GITHUB_ENV" fi @@ -423,17 +414,42 @@ jobs: - name: Run ${{ matrix.task }} (${{ matrix.runtime }}) if: github.event_name != 'pull_request' || matrix.task != 'compat-node22' - run: ${{ matrix.command }} + env: + TASK: ${{ matrix.task }} + shell: bash + run: | + set -euo pipefail + case "$TASK" in + test) + pnpm test + ;; + channels) + pnpm test:channels + ;; + compat-node22) + pnpm build + pnpm ui:build + node openclaw.mjs --help + node openclaw.mjs status --json --timeout 1 + pnpm test:build:singleton + node scripts/stage-bundled-plugin-runtime-deps.mjs + node --import tsx scripts/release-check.ts + ;; + *) + echo "Unsupported checks task: $TASK" >&2 + exit 1 + ;; + esac extension-fast: name: "extension-fast" - needs: [scope] - if: needs.scope.outputs.docs_only != 'true' && needs.scope.outputs.run_node == 'true' && needs.scope.outputs.has_changed_extensions == 'true' + needs: [preflight] + if: needs.preflight.outputs.run_extension_fast == 'true' runs-on: blacksmith-16vcpu-ubuntu-2404 timeout-minutes: 20 strategy: fail-fast: false - matrix: ${{ fromJson(needs.scope.outputs.changed_extensions_matrix) }} + matrix: ${{ fromJson(needs.preflight.outputs.extension_fast_matrix) }} steps: - name: Checkout uses: actions/checkout@v6 @@ -455,8 +471,8 @@ jobs: # Types, lint, and format check. check: name: "check" - needs: [scope] - if: always() && (github.event_name != 'pull_request' || !github.event.pull_request.draft) && needs.scope.outputs.docs_only != 'true' + needs: [preflight] + if: always() && needs.preflight.outputs.run_check == 'true' runs-on: blacksmith-16vcpu-ubuntu-2404 timeout-minutes: 20 steps: @@ -480,8 +496,8 @@ jobs: check-additional: name: "check-additional" - needs: [scope] - if: always() && (github.event_name != 'pull_request' || !github.event.pull_request.draft) && needs.scope.outputs.docs_only != 'true' + needs: [preflight] + if: always() && needs.preflight.outputs.run_check_additional == 'true' runs-on: blacksmith-16vcpu-ubuntu-2404 timeout-minutes: 20 steps: @@ -579,8 +595,8 @@ jobs: build-smoke: name: "build-smoke" - needs: [scope, build-artifacts] - if: always() && needs.scope.outputs.docs_only != 'true' && needs.scope.outputs.run_node == 'true' && (github.event_name != 'push' || needs.build-artifacts.result == 'success') + needs: [preflight, build-artifacts] + if: always() && needs.preflight.outputs.run_build_smoke == 'true' && (github.event_name != 'push' || needs.build-artifacts.result == 'success') runs-on: blacksmith-16vcpu-ubuntu-2404 timeout-minutes: 20 steps: @@ -621,8 +637,8 @@ jobs: # Validate docs (format, lint, broken links) only when docs files changed. check-docs: - needs: [scope] - if: needs.scope.outputs.docs_changed == 'true' + needs: [preflight] + if: needs.preflight.outputs.run_check_docs == 'true' runs-on: blacksmith-16vcpu-ubuntu-2404 timeout-minutes: 20 steps: @@ -642,8 +658,8 @@ jobs: run: pnpm check:docs skills-python: - needs: [scope] - if: needs.scope.outputs.docs_only != 'true' && (github.event_name == 'push' || needs.scope.outputs.run_skills_python == 'true') + needs: [preflight] + if: needs.preflight.outputs.run_skills_python_job == 'true' runs-on: blacksmith-16vcpu-ubuntu-2404 timeout-minutes: 20 steps: @@ -670,8 +686,9 @@ jobs: run: python -m pytest -q skills checks-windows: - needs: [scope, build-artifacts] - if: always() && needs.scope.outputs.docs_only != 'true' && needs.scope.outputs.run_windows == 'true' && needs.build-artifacts.result == 'success' + name: ${{ matrix.check_name }} + needs: [preflight, build-artifacts] + if: always() && needs.preflight.outputs.run_checks_windows == 'true' && needs.build-artifacts.result == 'success' runs-on: blacksmith-32vcpu-windows-2025 timeout-minutes: 20 env: @@ -684,53 +701,7 @@ jobs: shell: bash strategy: fail-fast: false - matrix: - include: - - runtime: node - task: test - shard_index: 1 - shard_count: 9 - command: pnpm test - - runtime: node - task: test - shard_index: 2 - shard_count: 9 - command: pnpm test - - runtime: node - task: test - shard_index: 3 - shard_count: 9 - command: pnpm test - - runtime: node - task: test - shard_index: 4 - shard_count: 9 - command: pnpm test - - runtime: node - task: test - shard_index: 5 - shard_count: 9 - command: pnpm test - - runtime: node - task: test - shard_index: 6 - shard_count: 9 - command: pnpm test - - runtime: node - task: test - shard_index: 7 - shard_count: 9 - command: pnpm test - - runtime: node - task: test - shard_index: 8 - shard_count: 9 - command: pnpm test - - runtime: node - task: test - shard_index: 9 - shard_count: 9 - command: pnpm test + matrix: ${{ fromJson(needs.preflight.outputs.checks_windows_matrix) }} steps: - name: Checkout uses: actions/checkout@v6 @@ -799,9 +770,12 @@ jobs: - name: Configure test shard (Windows) if: matrix.task == 'test' + env: + SHARD_COUNT: ${{ matrix.shard_count }} + SHARD_INDEX: ${{ matrix.shard_index }} run: | - echo "OPENCLAW_TEST_SHARDS=${{ matrix.shard_count }}" >> "$GITHUB_ENV" - echo "OPENCLAW_TEST_SHARD_INDEX=${{ matrix.shard_index }}" >> "$GITHUB_ENV" + echo "OPENCLAW_TEST_SHARDS=$SHARD_COUNT" >> "$GITHUB_ENV" + echo "OPENCLAW_TEST_SHARD_INDEX=$SHARD_INDEX" >> "$GITHUB_ENV" - name: Download dist artifact if: matrix.task == 'test' @@ -818,13 +792,30 @@ jobs: path: src/canvas-host/a2ui/ - name: Run ${{ matrix.task }} (${{ matrix.runtime }}) - run: ${{ matrix.command }} + env: + TASK: ${{ matrix.task }} + shell: bash + run: | + set -euo pipefail + case "$TASK" in + test) + pnpm test + ;; + *) + echo "Unsupported Windows checks task: $TASK" >&2 + exit 1 + ;; + esac - macos-node-1: - needs: [scope, build-artifacts] - if: always() && github.event_name == 'pull_request' && needs.scope.outputs.docs_only != 'true' && needs.scope.outputs.run_macos == 'true' && needs.build-artifacts.result == 'success' + macos-node: + name: ${{ matrix.check_name }} + needs: [preflight, build-artifacts] + if: always() && needs.preflight.outputs.run_macos_node == 'true' && needs.build-artifacts.result == 'success' runs-on: macos-latest timeout-minutes: 20 + strategy: + fail-fast: false + matrix: ${{ fromJson(needs.preflight.outputs.macos_node_matrix) }} steps: - name: Checkout uses: actions/checkout@v6 @@ -849,136 +840,35 @@ jobs: name: canvas-a2ui-bundle path: src/canvas-host/a2ui/ - - name: Configure test shard (macOS 1/4) + - name: Configure test shard (macOS) + env: + SHARD_COUNT: ${{ matrix.shard_count }} + SHARD_INDEX: ${{ matrix.shard_index }} run: | - echo "OPENCLAW_TEST_SHARDS=4" >> "$GITHUB_ENV" - echo "OPENCLAW_TEST_SHARD_INDEX=1" >> "$GITHUB_ENV" + echo "OPENCLAW_TEST_SHARDS=$SHARD_COUNT" >> "$GITHUB_ENV" + echo "OPENCLAW_TEST_SHARD_INDEX=$SHARD_INDEX" >> "$GITHUB_ENV" - - name: TS tests (macOS 1/4) + - name: TS tests (macOS) env: NODE_OPTIONS: --max-old-space-size=4096 - run: pnpm test - - macos-node-2: - needs: [scope, build-artifacts] - if: always() && github.event_name == 'pull_request' && needs.scope.outputs.docs_only != 'true' && needs.scope.outputs.run_macos == 'true' && needs.build-artifacts.result == 'success' - runs-on: macos-latest - timeout-minutes: 20 - steps: - - name: Checkout - uses: actions/checkout@v6 - with: - persist-credentials: false - submodules: false - - - name: Setup Node environment - uses: ./.github/actions/setup-node-env - with: - install-bun: "false" - - - name: Download dist artifact - uses: actions/download-artifact@v8 - with: - name: dist-build - path: dist/ - - - name: Download A2UI bundle artifact - uses: actions/download-artifact@v8 - with: - name: canvas-a2ui-bundle - path: src/canvas-host/a2ui/ - - - name: Configure test shard (macOS 2/4) + TASK: ${{ matrix.task }} + shell: bash run: | - echo "OPENCLAW_TEST_SHARDS=4" >> "$GITHUB_ENV" - echo "OPENCLAW_TEST_SHARD_INDEX=2" >> "$GITHUB_ENV" - - - name: TS tests (macOS 2/4) - env: - NODE_OPTIONS: --max-old-space-size=4096 - run: pnpm test - - macos-node-3: - needs: [scope, build-artifacts] - if: always() && github.event_name == 'pull_request' && needs.scope.outputs.docs_only != 'true' && needs.scope.outputs.run_macos == 'true' && needs.build-artifacts.result == 'success' - runs-on: macos-latest - timeout-minutes: 20 - steps: - - name: Checkout - uses: actions/checkout@v6 - with: - persist-credentials: false - submodules: false - - - name: Setup Node environment - uses: ./.github/actions/setup-node-env - with: - install-bun: "false" - - - name: Download dist artifact - uses: actions/download-artifact@v8 - with: - name: dist-build - path: dist/ - - - name: Download A2UI bundle artifact - uses: actions/download-artifact@v8 - with: - name: canvas-a2ui-bundle - path: src/canvas-host/a2ui/ - - - name: Configure test shard (macOS 3/4) - run: | - echo "OPENCLAW_TEST_SHARDS=4" >> "$GITHUB_ENV" - echo "OPENCLAW_TEST_SHARD_INDEX=3" >> "$GITHUB_ENV" - - - name: TS tests (macOS 3/4) - env: - NODE_OPTIONS: --max-old-space-size=4096 - run: pnpm test - - macos-node-4: - needs: [scope, build-artifacts] - if: always() && github.event_name == 'pull_request' && needs.scope.outputs.docs_only != 'true' && needs.scope.outputs.run_macos == 'true' && needs.build-artifacts.result == 'success' - runs-on: macos-latest - timeout-minutes: 20 - steps: - - name: Checkout - uses: actions/checkout@v6 - with: - persist-credentials: false - submodules: false - - - name: Setup Node environment - uses: ./.github/actions/setup-node-env - with: - install-bun: "false" - - - name: Download dist artifact - uses: actions/download-artifact@v8 - with: - name: dist-build - path: dist/ - - - name: Download A2UI bundle artifact - uses: actions/download-artifact@v8 - with: - name: canvas-a2ui-bundle - path: src/canvas-host/a2ui/ - - - name: Configure test shard (macOS 4/4) - run: | - echo "OPENCLAW_TEST_SHARDS=4" >> "$GITHUB_ENV" - echo "OPENCLAW_TEST_SHARD_INDEX=4" >> "$GITHUB_ENV" - - - name: TS tests (macOS 4/4) - env: - NODE_OPTIONS: --max-old-space-size=4096 - run: pnpm test + set -euo pipefail + case "$TASK" in + test) + pnpm test + ;; + *) + echo "Unsupported macOS node task: $TASK" >&2 + exit 1 + ;; + esac macos-swift: - needs: [scope] - if: github.event_name == 'pull_request' && needs.scope.outputs.docs_only != 'true' && needs.scope.outputs.run_macos == 'true' + name: "macos-swift" + needs: [preflight] + if: needs.preflight.outputs.run_macos_swift == 'true' runs-on: macos-latest timeout-minutes: 20 steps: @@ -996,6 +886,14 @@ jobs: - name: Install XcodeGen / SwiftLint / SwiftFormat run: brew install xcodegen swiftlint swiftformat + - name: Cache SwiftPM + uses: actions/cache@v5 + with: + path: ~/Library/Caches/org.swift.swiftpm + key: ${{ runner.os }}-swiftpm-${{ hashFiles('apps/macos/Package.resolved') }} + restore-keys: | + ${{ runner.os }}-swiftpm- + - name: Show toolchain run: | sw_vers @@ -1007,14 +905,6 @@ jobs: swiftlint --config .swiftlint.yml swiftformat --lint apps/macos/Sources --config .swiftformat - - name: Cache SwiftPM - uses: actions/cache@v5 - with: - path: ~/Library/Caches/org.swift.swiftpm - key: ${{ runner.os }}-swiftpm-${{ hashFiles('apps/macos/Package.resolved') }} - restore-keys: | - ${{ runner.os }}-swiftpm- - - name: Swift build (release) run: | set -euo pipefail @@ -1040,22 +930,14 @@ jobs: exit 1 android: - needs: [scope] - if: needs.scope.outputs.docs_only != 'true' && needs.scope.outputs.run_android == 'true' + name: ${{ matrix.check_name }} + needs: [preflight] + if: needs.preflight.outputs.run_android_job == 'true' runs-on: blacksmith-16vcpu-ubuntu-2404 timeout-minutes: 20 strategy: fail-fast: false - matrix: - include: - - task: test-play - command: ./gradlew --no-daemon :app:testPlayDebugUnitTest - - task: test-third-party - command: ./gradlew --no-daemon :app:testThirdPartyDebugUnitTest - - task: build-play - command: ./gradlew --no-daemon :app:assemblePlayDebug - - task: build-third-party - command: ./gradlew --no-daemon :app:assembleThirdPartyDebug + matrix: ${{ fromJson(needs.preflight.outputs.android_matrix) }} steps: - name: Checkout uses: actions/checkout@v6 @@ -1104,4 +986,26 @@ jobs: - name: Run Android ${{ matrix.task }} working-directory: apps/android - run: ${{ matrix.command }} + env: + TASK: ${{ matrix.task }} + shell: bash + run: | + set -euo pipefail + case "$TASK" in + test-play) + ./gradlew --no-daemon :app:testPlayDebugUnitTest + ;; + test-third-party) + ./gradlew --no-daemon :app:testThirdPartyDebugUnitTest + ;; + build-play) + ./gradlew --no-daemon :app:assemblePlayDebug + ;; + build-third-party) + ./gradlew --no-daemon :app:assembleThirdPartyDebug + ;; + *) + echo "Unsupported Android task: $TASK" >&2 + exit 1 + ;; + esac diff --git a/.github/workflows/install-smoke.yml b/.github/workflows/install-smoke.yml index 200ce7595d7..8dc93dc09ba 100644 --- a/.github/workflows/install-smoke.yml +++ b/.github/workflows/install-smoke.yml @@ -15,49 +15,34 @@ env: FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: "true" jobs: - docs-scope: + preflight: if: github.event_name != 'pull_request' || !github.event.pull_request.draft runs-on: blacksmith-16vcpu-ubuntu-2404 outputs: - docs_only: ${{ steps.check.outputs.docs_only }} + docs_only: ${{ steps.manifest.outputs.docs_only }} + run_install_smoke: ${{ steps.manifest.outputs.run_install_smoke }} steps: - name: Checkout uses: actions/checkout@v6 with: fetch-depth: 1 fetch-tags: false + persist-credentials: false + submodules: false - - name: Ensure docs-scope base commit + - name: Ensure preflight base commit uses: ./.github/actions/ensure-base-commit with: base-sha: ${{ github.event_name == 'push' && github.event.before || github.event.pull_request.base.sha }} fetch-ref: ${{ github.event_name == 'push' && github.ref_name || github.event.pull_request.base.ref }} - name: Detect docs-only changes - id: check + id: docs_scope uses: ./.github/actions/detect-docs-changes - changed-smoke: - needs: [docs-scope] - if: needs.docs-scope.outputs.docs_only != 'true' - runs-on: blacksmith-16vcpu-ubuntu-2404 - outputs: - run_changed_smoke: ${{ steps.scope.outputs.run_changed_smoke }} - steps: - - name: Checkout - uses: actions/checkout@v6 - with: - fetch-depth: 1 - fetch-tags: false - - - name: Ensure changed-smoke base commit - uses: ./.github/actions/ensure-base-commit - with: - base-sha: ${{ github.event_name == 'push' && github.event.before || github.event.pull_request.base.sha }} - fetch-ref: ${{ github.event_name == 'push' && github.ref_name || github.event.pull_request.base.ref }} - - name: Detect changed smoke scope - id: scope + id: changed_scope + if: steps.docs_scope.outputs.docs_only != 'true' shell: bash run: | set -euo pipefail @@ -70,9 +55,32 @@ jobs: node scripts/ci-changed-scope.mjs --base "$BASE" --head HEAD + - name: Setup Node environment + if: steps.docs_scope.outputs.docs_only != 'true' + uses: ./.github/actions/setup-node-env + with: + install-bun: "false" + install-deps: "false" + use-sticky-disk: "false" + + - name: Build install-smoke CI manifest + id: manifest + env: + OPENCLAW_CI_DOCS_ONLY: ${{ steps.docs_scope.outputs.docs_only }} + OPENCLAW_CI_DOCS_CHANGED: "false" + OPENCLAW_CI_RUN_NODE: "false" + OPENCLAW_CI_RUN_MACOS: "false" + OPENCLAW_CI_RUN_ANDROID: "false" + OPENCLAW_CI_RUN_WINDOWS: "false" + OPENCLAW_CI_RUN_SKILLS_PYTHON: "false" + OPENCLAW_CI_HAS_CHANGED_EXTENSIONS: "false" + OPENCLAW_CI_CHANGED_EXTENSIONS_MATRIX: '{"include":[]}' + OPENCLAW_CI_RUN_CHANGED_SMOKE: ${{ steps.changed_scope.outputs.run_changed_smoke || 'false' }} + run: node scripts/ci-write-manifest-outputs.mjs --workflow install-smoke + install-smoke: - needs: [docs-scope, changed-smoke] - if: (github.event_name != 'pull_request' || !github.event.pull_request.draft) && needs.docs-scope.outputs.docs_only != 'true' && needs.changed-smoke.outputs.run_changed_smoke == 'true' + needs: [preflight] + if: needs.preflight.outputs.run_install_smoke == 'true' runs-on: blacksmith-16vcpu-ubuntu-2404 env: DOCKER_BUILD_SUMMARY: "false" diff --git a/scripts/ci-write-manifest-outputs.mjs b/scripts/ci-write-manifest-outputs.mjs new file mode 100644 index 00000000000..4bf41778dc4 --- /dev/null +++ b/scripts/ci-write-manifest-outputs.mjs @@ -0,0 +1,76 @@ +import { appendFileSync } from "node:fs"; +import { buildCIExecutionManifest } from "./test-planner/planner.mjs"; + +const WORKFLOWS = new Set(["ci", "install-smoke", "ci-bun"]); + +const parseArgs = (argv) => { + const parsed = { + workflow: "ci", + }; + for (let index = 0; index < argv.length; index += 1) { + const arg = argv[index]; + if (arg === "--workflow") { + const nextValue = argv[index + 1] ?? ""; + if (!WORKFLOWS.has(nextValue)) { + throw new Error( + `Unsupported --workflow value "${String(nextValue || "")}". Supported values: ci, install-smoke, ci-bun.`, + ); + } + parsed.workflow = nextValue; + index += 1; + } + } + return parsed; +}; + +const outputPath = process.env.GITHUB_OUTPUT; + +if (!outputPath) { + throw new Error("GITHUB_OUTPUT is required"); +} + +const { workflow } = parseArgs(process.argv.slice(2)); +const manifest = buildCIExecutionManifest(undefined, { env: process.env }); + +const writeOutput = (name, value) => { + appendFileSync(outputPath, `${name}=${value}\n`, "utf8"); +}; + +if (workflow === "ci") { + writeOutput("docs_only", String(manifest.scope.docsOnly)); + writeOutput("docs_changed", String(manifest.scope.docsChanged)); + writeOutput("run_node", String(manifest.scope.runNode)); + writeOutput("run_macos", String(manifest.scope.runMacos)); + writeOutput("run_android", String(manifest.scope.runAndroid)); + writeOutput("run_skills_python", String(manifest.scope.runSkillsPython)); + writeOutput("run_windows", String(manifest.scope.runWindows)); + writeOutput("has_changed_extensions", String(manifest.scope.hasChangedExtensions)); + writeOutput("changed_extensions_matrix", JSON.stringify(manifest.scope.changedExtensionsMatrix)); + writeOutput("run_build_artifacts", String(manifest.jobs.buildArtifacts.enabled)); + writeOutput("run_release_check", String(manifest.jobs.releaseCheck.enabled)); + writeOutput("run_checks_fast", String(manifest.jobs.checksFast.enabled)); + writeOutput("checks_fast_matrix", JSON.stringify(manifest.jobs.checksFast.matrix)); + writeOutput("run_checks", String(manifest.jobs.checks.enabled)); + writeOutput("checks_matrix", JSON.stringify(manifest.jobs.checks.matrix)); + writeOutput("run_extension_fast", String(manifest.jobs.extensionFast.enabled)); + writeOutput("extension_fast_matrix", JSON.stringify(manifest.jobs.extensionFast.matrix)); + writeOutput("run_check", String(manifest.jobs.check.enabled)); + writeOutput("run_check_additional", String(manifest.jobs.checkAdditional.enabled)); + writeOutput("run_build_smoke", String(manifest.jobs.buildSmoke.enabled)); + writeOutput("run_check_docs", String(manifest.jobs.checkDocs.enabled)); + writeOutput("run_skills_python_job", String(manifest.jobs.skillsPython.enabled)); + writeOutput("run_checks_windows", String(manifest.jobs.checksWindows.enabled)); + writeOutput("checks_windows_matrix", JSON.stringify(manifest.jobs.checksWindows.matrix)); + writeOutput("run_macos_node", String(manifest.jobs.macosNode.enabled)); + writeOutput("macos_node_matrix", JSON.stringify(manifest.jobs.macosNode.matrix)); + writeOutput("run_macos_swift", String(manifest.jobs.macosSwift.enabled)); + writeOutput("run_android_job", String(manifest.jobs.android.enabled)); + writeOutput("android_matrix", JSON.stringify(manifest.jobs.android.matrix)); + writeOutput("required_check_names", JSON.stringify(manifest.requiredCheckNames)); +} else if (workflow === "install-smoke") { + writeOutput("docs_only", String(manifest.scope.docsOnly)); + writeOutput("run_install_smoke", String(manifest.jobs.installSmoke.enabled)); +} else if (workflow === "ci-bun") { + writeOutput("run_bun_checks", String(manifest.jobs.bunChecks.enabled)); + writeOutput("bun_checks_matrix", JSON.stringify(manifest.jobs.bunChecks.matrix)); +} diff --git a/scripts/test-parallel.mjs b/scripts/test-parallel.mjs index c530c3fcbf5..3bd053e925e 100644 --- a/scripts/test-parallel.mjs +++ b/scripts/test-parallel.mjs @@ -4,10 +4,15 @@ import { formatExplanation, formatPlanOutput, } from "./test-planner/executor.mjs"; -import { buildExecutionPlan, explainExecutionTarget } from "./test-planner/planner.mjs"; +import { + buildCIExecutionManifest, + buildExecutionPlan, + explainExecutionTarget, +} from "./test-planner/planner.mjs"; const parseCliArgs = (args) => { const wrapper = { + ciManifest: false, plan: false, explain: null, mode: null, @@ -32,6 +37,10 @@ const parseCliArgs = (args) => { wrapper.plan = true; continue; } + if (arg === "--ci-manifest") { + wrapper.ciManifest = true; + continue; + } if (arg === "--help") { wrapper.showHelp = true; continue; @@ -104,6 +113,7 @@ if (rawCli.showHelp) { "", "Wrapper flags:", " --plan Print the resolved execution plan", + " --ci-manifest Print the planner-backed CI execution manifest as JSON", " --explain Explain how a file is classified and run", " --surface Select a surface (repeatable or comma-separated)", " --files Add targeted files/patterns (repeatable)", @@ -131,6 +141,12 @@ if (rawCli.explain) { process.exit(0); } +if (rawCli.ciManifest) { + const manifest = buildCIExecutionManifest(undefined, { env: process.env }); + console.log(`${JSON.stringify(manifest, null, 2)}\n`); + process.exit(0); +} + const artifacts = createExecutionArtifacts(process.env); let plan; try { diff --git a/scripts/test-planner/planner.mjs b/scripts/test-planner/planner.mjs index bdc9e677ee5..e4a9f592f41 100644 --- a/scripts/test-planner/planner.mjs +++ b/scripts/test-planner/planner.mjs @@ -22,6 +22,71 @@ const parseEnvNumber = (env, name, fallback) => { return Number.isFinite(parsed) && parsed >= 0 ? parsed : fallback; }; +const parseBooleanLike = (value, fallback = false) => { + if (typeof value === "boolean") { + return value; + } + if (typeof value === "string") { + const normalized = value.trim().toLowerCase(); + if (normalized === "true" || normalized === "1") { + return true; + } + if (normalized === "false" || normalized === "0" || normalized === "") { + return false; + } + } + return fallback; +}; + +const clamp = (value, min, max) => Math.min(max, Math.max(min, value)); + +const sumKnownManifestDurationsMs = (manifest) => + Object.values(manifest.files ?? {}).reduce((totalMs, entry) => totalMs + entry.durationMs, 0); + +const resolveDynamicShardCount = ({ + estimatedDurationMs, + fileCount, + targetDurationMs, + targetFilesPerShard, + minShards, + maxShards, +}) => { + const durationDriven = + Number.isFinite(targetDurationMs) && targetDurationMs > 0 + ? Math.ceil(estimatedDurationMs / targetDurationMs) + : 1; + const fileDriven = + Number.isFinite(targetFilesPerShard) && targetFilesPerShard > 0 + ? Math.ceil(fileCount / targetFilesPerShard) + : 1; + return clamp(Math.max(minShards, durationDriven, fileDriven), minShards, maxShards); +}; + +const createShardMatrixEntries = ({ checkNamePrefix, runtime, task, command, shardCount }) => + Array.from({ length: shardCount }, (_, index) => ({ + check_name: `${checkNamePrefix}-${String(index + 1)}`, + runtime, + task, + command, + shard_index: index + 1, + shard_count: shardCount, + })); + +const parseChangedExtensionsMatrix = (value) => { + if (typeof value === "object" && value !== null && Array.isArray(value.include)) { + return value; + } + if (typeof value === "string" && value.trim().length > 0) { + try { + const parsed = JSON.parse(value); + if (parsed && typeof parsed === "object" && Array.isArray(parsed.include)) { + return parsed; + } + } catch {} + } + return { include: [] }; +}; + const normalizeSurfaces = (values = []) => [ ...new Set( values @@ -92,6 +157,31 @@ const createPlannerContext = (request, options = {}) => { }; }; +const resolveCIManifestScope = (scope = {}, env = process.env) => ({ + eventName: scope.eventName ?? env.GITHUB_EVENT_NAME ?? "pull_request", + docsOnly: parseBooleanLike(scope.docsOnly ?? env.OPENCLAW_CI_DOCS_ONLY, false), + docsChanged: parseBooleanLike(scope.docsChanged ?? env.OPENCLAW_CI_DOCS_CHANGED, false), + runNode: parseBooleanLike(scope.runNode ?? env.OPENCLAW_CI_RUN_NODE, true), + runMacos: parseBooleanLike(scope.runMacos ?? env.OPENCLAW_CI_RUN_MACOS, true), + runAndroid: parseBooleanLike(scope.runAndroid ?? env.OPENCLAW_CI_RUN_ANDROID, true), + runWindows: parseBooleanLike(scope.runWindows ?? env.OPENCLAW_CI_RUN_WINDOWS, true), + runSkillsPython: parseBooleanLike( + scope.runSkillsPython ?? env.OPENCLAW_CI_RUN_SKILLS_PYTHON, + true, + ), + hasChangedExtensions: parseBooleanLike( + scope.hasChangedExtensions ?? env.OPENCLAW_CI_HAS_CHANGED_EXTENSIONS, + false, + ), + changedExtensionsMatrix: parseChangedExtensionsMatrix( + scope.changedExtensionsMatrix ?? env.OPENCLAW_CI_CHANGED_EXTENSIONS_MATRIX, + ), + runChangedSmoke: parseBooleanLike( + scope.runChangedSmoke ?? env.OPENCLAW_CI_RUN_CHANGED_SMOKE, + true, + ), +}); + const estimateEntryFilesDurationMs = (entry, files, context) => { const estimateDurationMs = resolveEntryTimingEstimator(entry, context); if (!estimateDurationMs) { @@ -854,6 +944,223 @@ const buildTopLevelSingleShardAssignments = (context, units) => { return assignmentMap; }; +export function buildCIExecutionManifest(scopeInput = {}, options = {}) { + const env = options.env ?? process.env; + const scope = resolveCIManifestScope(scopeInput, env); + const context = createPlannerContext({ mode: "ci", profile: null }, { ...options, env }); + const isPullRequest = scope.eventName === "pull_request"; + const isPush = scope.eventName === "push"; + const nodeEligible = !scope.docsOnly && scope.runNode; + const macosEligible = !scope.docsOnly && isPullRequest && scope.runMacos; + const windowsEligible = !scope.docsOnly && scope.runWindows; + const androidEligible = !scope.docsOnly && scope.runAndroid; + const docsEligible = scope.docsChanged; + const skillsPythonEligible = !scope.docsOnly && (isPush || scope.runSkillsPython); + const extensionFastEligible = nodeEligible && scope.hasChangedExtensions; + + const channelCandidateFiles = context.catalog.allKnownTestFiles.filter((file) => + context.catalog.channelTestPrefixes.some((prefix) => file.startsWith(prefix)), + ); + const unitShardCount = resolveDynamicShardCount({ + estimatedDurationMs: sumKnownManifestDurationsMs(context.unitTimingManifest), + fileCount: context.catalog.allKnownUnitFiles.length, + targetDurationMs: 30_000, + targetFilesPerShard: 80, + minShards: 1, + maxShards: 4, + }); + const channelShardCount = resolveDynamicShardCount({ + estimatedDurationMs: sumKnownManifestDurationsMs(context.channelTimingManifest), + fileCount: channelCandidateFiles.length, + targetDurationMs: 90_000, + targetFilesPerShard: 150, + minShards: 1, + maxShards: 4, + }); + const windowsShardCount = resolveDynamicShardCount({ + estimatedDurationMs: sumKnownManifestDurationsMs(context.unitTimingManifest), + fileCount: context.catalog.allKnownUnitFiles.length, + targetDurationMs: 12_000, + targetFilesPerShard: 30, + minShards: 1, + maxShards: 9, + }); + const macosNodeShardCount = windowsShardCount; + const bunShardCount = resolveDynamicShardCount({ + estimatedDurationMs: sumKnownManifestDurationsMs(context.unitTimingManifest), + fileCount: context.catalog.allKnownUnitFiles.length, + targetDurationMs: 30_000, + targetFilesPerShard: 80, + minShards: 1, + maxShards: 4, + }); + + const checksFastInclude = nodeEligible + ? [ + { + check_name: "checks-fast-extensions", + runtime: "node", + task: "extensions", + command: "pnpm test:extensions", + }, + { + check_name: "checks-fast-contracts-protocol", + runtime: "node", + task: "contracts-protocol", + command: "pnpm test:contracts\npnpm protocol:check", + }, + ] + : []; + const checksInclude = nodeEligible + ? [ + ...createShardMatrixEntries({ + checkNamePrefix: "checks-node-test", + runtime: "node", + task: "test", + command: "pnpm test", + shardCount: unitShardCount, + }), + ...createShardMatrixEntries({ + checkNamePrefix: "checks-node-channels", + runtime: "node", + task: "channels", + command: "pnpm test:channels", + shardCount: channelShardCount, + }), + ...(isPush + ? [ + { + check_name: "checks-node-compat-node22", + runtime: "node", + task: "compat-node22", + node_version: "22.x", + cache_key_suffix: "node22", + command: [ + "pnpm build", + "pnpm ui:build", + "node openclaw.mjs --help", + "node openclaw.mjs status --json --timeout 1", + "pnpm test:build:singleton", + "node scripts/stage-bundled-plugin-runtime-deps.mjs", + "node --import tsx scripts/release-check.ts", + ].join("\n"), + }, + ] + : []), + ] + : []; + const checksWindowsInclude = windowsEligible + ? createShardMatrixEntries({ + checkNamePrefix: "checks-windows-node-test", + runtime: "node", + task: "test", + command: "pnpm test", + shardCount: windowsShardCount, + }) + : []; + const macosNodeInclude = macosEligible + ? createShardMatrixEntries({ + checkNamePrefix: "macos-node", + runtime: "node", + task: "test", + command: "pnpm test", + shardCount: macosNodeShardCount, + }) + : []; + const androidInclude = androidEligible + ? [ + { + check_name: "android-test-play", + task: "test-play", + command: "./gradlew --no-daemon :app:testPlayDebugUnitTest", + }, + { + check_name: "android-test-third-party", + task: "test-third-party", + command: "./gradlew --no-daemon :app:testThirdPartyDebugUnitTest", + }, + { + check_name: "android-build-play", + task: "build-play", + command: "./gradlew --no-daemon :app:assemblePlayDebug", + }, + { + check_name: "android-build-third-party", + task: "build-third-party", + command: "./gradlew --no-daemon :app:assembleThirdPartyDebug", + }, + ] + : []; + const bunChecksInclude = createShardMatrixEntries({ + checkNamePrefix: "bun-checks", + runtime: "bun", + task: "test", + command: "bunx vitest run --config vitest.unit.config.ts", + shardCount: bunShardCount, + }); + const extensionFastInclude = extensionFastEligible + ? scope.changedExtensionsMatrix.include.map((entry) => ({ + check_name: `extension-fast-${entry.extension}`, + extension: entry.extension, + })) + : []; + + const jobs = { + buildArtifacts: { enabled: nodeEligible, needsDistArtifacts: false }, + releaseCheck: { enabled: isPush && !scope.docsOnly && nodeEligible }, + checksFast: { enabled: checksFastInclude.length > 0, matrix: { include: checksFastInclude } }, + checks: { enabled: checksInclude.length > 0, matrix: { include: checksInclude } }, + extensionFast: { + enabled: extensionFastInclude.length > 0, + matrix: { include: extensionFastInclude }, + }, + check: { enabled: !scope.docsOnly }, + checkAdditional: { enabled: !scope.docsOnly }, + buildSmoke: { enabled: nodeEligible }, + checkDocs: { enabled: docsEligible }, + skillsPython: { enabled: skillsPythonEligible }, + checksWindows: { + enabled: checksWindowsInclude.length > 0, + matrix: { include: checksWindowsInclude }, + }, + macosNode: { enabled: macosNodeInclude.length > 0, matrix: { include: macosNodeInclude } }, + macosSwift: { enabled: macosEligible }, + android: { enabled: androidInclude.length > 0, matrix: { include: androidInclude } }, + bunChecks: { enabled: bunChecksInclude.length > 0, matrix: { include: bunChecksInclude } }, + installSmoke: { enabled: !scope.docsOnly && scope.runChangedSmoke }, + }; + + return { + runtimeProfile: context.runtime.runtimeProfileName, + scope, + shardCounts: { + unit: unitShardCount, + channels: channelShardCount, + windows: windowsShardCount, + macosNode: macosNodeShardCount, + bun: bunShardCount, + }, + jobs, + requiredCheckNames: [ + ...checksFastInclude.map((entry) => entry.check_name), + ...checksInclude.map((entry) => entry.check_name), + ...checksWindowsInclude.map((entry) => entry.check_name), + ...macosNodeInclude.map((entry) => entry.check_name), + ...(macosEligible ? ["macos-swift"] : []), + ...androidInclude.map((entry) => entry.check_name), + ...extensionFastInclude.map((entry) => entry.check_name), + ...bunChecksInclude.map((entry) => entry.check_name), + "check", + "check-additional", + "build-smoke", + ...(docsEligible ? ["check-docs"] : []), + ...(skillsPythonEligible ? ["skills-python"] : []), + ...(nodeEligible ? ["build-artifacts"] : []), + ...(isPush && !scope.docsOnly && nodeEligible ? ["release-check"] : []), + ], + }; +} + export const formatExecutionUnitSummary = (unit) => `${unit.id} filters=${String(countExplicitEntryFilters(unit.args) || "all")} maxWorkers=${String( unit.maxWorkers ?? "default", diff --git a/test/scripts/test-parallel.test.ts b/test/scripts/test-parallel.test.ts index 276d7c91d40..7e568e94e09 100644 --- a/test/scripts/test-parallel.test.ts +++ b/test/scripts/test-parallel.test.ts @@ -216,6 +216,106 @@ describe("scripts/test-parallel lane planning", () => { expect(output).toContain("pool=forks"); }); + it("prints the planner-backed CI manifest as JSON", () => { + const repoRoot = path.resolve(import.meta.dirname, "../.."); + const output = execFileSync("node", ["scripts/test-parallel.mjs", "--ci-manifest"], { + cwd: repoRoot, + env: { + ...clearPlannerShardEnv(process.env), + GITHUB_EVENT_NAME: "pull_request", + OPENCLAW_CI_DOCS_ONLY: "false", + OPENCLAW_CI_DOCS_CHANGED: "false", + OPENCLAW_CI_RUN_NODE: "true", + OPENCLAW_CI_RUN_MACOS: "true", + OPENCLAW_CI_RUN_ANDROID: "false", + OPENCLAW_CI_RUN_WINDOWS: "true", + OPENCLAW_CI_RUN_SKILLS_PYTHON: "false", + OPENCLAW_CI_HAS_CHANGED_EXTENSIONS: "false", + OPENCLAW_CI_CHANGED_EXTENSIONS_MATRIX: '{"include":[]}', + }, + encoding: "utf8", + }); + + const manifest = JSON.parse(output); + expect(manifest.jobs.checks.enabled).toBe(true); + expect(manifest.jobs.macosNode.enabled).toBe(true); + expect(manifest.jobs.checksWindows.enabled).toBe(true); + }); + + it("writes CI workflow outputs in ci mode", () => { + const repoRoot = path.resolve(import.meta.dirname, "../.."); + const outputPath = path.join(os.tmpdir(), `openclaw-ci-output-${Date.now()}.txt`); + + execFileSync("node", ["scripts/ci-write-manifest-outputs.mjs", "--workflow", "ci"], { + cwd: repoRoot, + env: { + ...clearPlannerShardEnv(process.env), + GITHUB_OUTPUT: outputPath, + GITHUB_EVENT_NAME: "pull_request", + OPENCLAW_CI_DOCS_ONLY: "false", + OPENCLAW_CI_DOCS_CHANGED: "false", + OPENCLAW_CI_RUN_NODE: "true", + OPENCLAW_CI_RUN_MACOS: "true", + OPENCLAW_CI_RUN_ANDROID: "true", + OPENCLAW_CI_RUN_WINDOWS: "true", + OPENCLAW_CI_RUN_SKILLS_PYTHON: "false", + OPENCLAW_CI_HAS_CHANGED_EXTENSIONS: "false", + OPENCLAW_CI_CHANGED_EXTENSIONS_MATRIX: '{"include":[]}', + }, + encoding: "utf8", + }); + + const outputs = fs.readFileSync(outputPath, "utf8"); + expect(outputs).toContain("run_build_artifacts=true"); + expect(outputs).toContain("run_checks_windows=true"); + expect(outputs).toContain("run_macos_node=true"); + expect(outputs).toContain("android_matrix="); + fs.rmSync(outputPath, { force: true }); + }); + + it("writes install-smoke outputs in install-smoke mode", () => { + const repoRoot = path.resolve(import.meta.dirname, "../.."); + const outputPath = path.join(os.tmpdir(), `openclaw-install-output-${Date.now()}.txt`); + + execFileSync("node", ["scripts/ci-write-manifest-outputs.mjs", "--workflow", "install-smoke"], { + cwd: repoRoot, + env: { + ...clearPlannerShardEnv(process.env), + GITHUB_OUTPUT: outputPath, + OPENCLAW_CI_DOCS_ONLY: "false", + OPENCLAW_CI_RUN_CHANGED_SMOKE: "true", + }, + encoding: "utf8", + }); + + const outputs = fs.readFileSync(outputPath, "utf8"); + expect(outputs).toContain("run_install_smoke=true"); + expect(outputs).not.toContain("run_checks="); + fs.rmSync(outputPath, { force: true }); + }); + + it("writes bun outputs in ci-bun mode", () => { + const repoRoot = path.resolve(import.meta.dirname, "../.."); + const outputPath = path.join(os.tmpdir(), `openclaw-bun-output-${Date.now()}.txt`); + + execFileSync("node", ["scripts/ci-write-manifest-outputs.mjs", "--workflow", "ci-bun"], { + cwd: repoRoot, + env: { + ...clearPlannerShardEnv(process.env), + GITHUB_OUTPUT: outputPath, + OPENCLAW_CI_DOCS_ONLY: "false", + OPENCLAW_CI_RUN_NODE: "true", + }, + encoding: "utf8", + }); + + const outputs = fs.readFileSync(outputPath, "utf8"); + expect(outputs).toContain("run_bun_checks=true"); + expect(outputs).toContain("bun_checks_matrix="); + expect(outputs).not.toContain("run_install_smoke="); + fs.rmSync(outputPath, { force: true }); + }); + it("passes through vitest --mode values that are not wrapper runtime overrides", () => { const repoRoot = path.resolve(import.meta.dirname, "../.."); const output = execFileSync( diff --git a/test/scripts/test-planner.test.ts b/test/scripts/test-planner.test.ts index f16ce03eeba..a1ac379e0f5 100644 --- a/test/scripts/test-planner.test.ts +++ b/test/scripts/test-planner.test.ts @@ -5,7 +5,11 @@ import { createExecutionArtifacts, resolvePnpmCommandInvocation, } from "../../scripts/test-planner/executor.mjs"; -import { buildExecutionPlan, explainExecutionTarget } from "../../scripts/test-planner/planner.mjs"; +import { + buildCIExecutionManifest, + buildExecutionPlan, + explainExecutionTarget, +} from "../../scripts/test-planner/planner.mjs"; describe("test planner", () => { it("builds a capability-aware plan for mid-memory local runs", () => { @@ -256,6 +260,92 @@ describe("test planner", () => { artifacts.cleanupTempArtifacts(); expect(fs.existsSync(artifactDir)).toBe(false); }); + + it("builds a CI manifest with planner-owned shard counts and matrices", () => { + const manifest = buildCIExecutionManifest( + { + eventName: "pull_request", + docsOnly: false, + docsChanged: false, + runNode: true, + runMacos: true, + runAndroid: true, + runWindows: true, + runSkillsPython: false, + hasChangedExtensions: true, + changedExtensionsMatrix: { include: [{ extension: "discord" }] }, + }, + { + env: {}, + }, + ); + + expect(manifest.jobs.buildArtifacts.enabled).toBe(true); + expect(manifest.shardCounts.unit).toBe(4); + expect(manifest.shardCounts.channels).toBe(3); + expect(manifest.shardCounts.windows).toBe(9); + expect(manifest.shardCounts.macosNode).toBe(9); + expect(manifest.jobs.checks.matrix.include).toHaveLength(7); + expect(manifest.jobs.checksWindows.matrix.include).toHaveLength(9); + expect(manifest.jobs.macosNode.matrix.include).toHaveLength(9); + expect(manifest.jobs.macosSwift.enabled).toBe(true); + expect(manifest.requiredCheckNames).toContain("macos-swift"); + expect(manifest.requiredCheckNames).not.toContain("macos-swift-lint"); + expect(manifest.requiredCheckNames).not.toContain("macos-swift-build"); + expect(manifest.requiredCheckNames).not.toContain("macos-swift-test"); + expect(manifest.jobs.extensionFast.matrix.include).toEqual([ + { check_name: "extension-fast-discord", extension: "discord" }, + ]); + }); + + it("suppresses heavy CI jobs in docs-only manifests", () => { + const manifest = buildCIExecutionManifest( + { + eventName: "pull_request", + docsOnly: true, + docsChanged: true, + runNode: false, + runMacos: false, + runAndroid: false, + runWindows: false, + runSkillsPython: false, + hasChangedExtensions: false, + }, + { + env: {}, + }, + ); + + expect(manifest.jobs.buildArtifacts.enabled).toBe(false); + expect(manifest.jobs.checks.enabled).toBe(false); + expect(manifest.jobs.checksWindows.enabled).toBe(false); + expect(manifest.jobs.macosNode.enabled).toBe(false); + expect(manifest.jobs.checkDocs.enabled).toBe(true); + }); + + it("adds push-only compat and release lanes to push manifests", () => { + const manifest = buildCIExecutionManifest( + { + eventName: "push", + docsOnly: false, + docsChanged: false, + runNode: true, + runMacos: false, + runAndroid: false, + runWindows: false, + runSkillsPython: false, + hasChangedExtensions: false, + }, + { + env: {}, + }, + ); + + expect(manifest.jobs.releaseCheck.enabled).toBe(true); + expect( + manifest.jobs.checks.matrix.include.some((entry) => entry.task === "compat-node22"), + ).toBe(true); + }); }); describe("resolvePnpmCommandInvocation", () => {