From ceaa56fb12142d6e220c57f260ce3d57b8d1c335 Mon Sep 17 00:00:00 2001 From: Peter Steinberger Date: Wed, 6 May 2026 10:18:12 +0100 Subject: [PATCH] fix(release): stabilize final validation checks --- scripts/e2e/lib/plugin-update/probe.mjs | 72 +++++++++++++++---- scripts/verify-docker-attestations.mjs | 22 ++---- src/agents/models.profiles.live.test.ts | 10 +++ .../verify-docker-attestations.test.ts | 45 ++++++------ 4 files changed, 99 insertions(+), 50 deletions(-) diff --git a/scripts/e2e/lib/plugin-update/probe.mjs b/scripts/e2e/lib/plugin-update/probe.mjs index 580cb1b3863..c77b41099ac 100644 --- a/scripts/e2e/lib/plugin-update/probe.mjs +++ b/scripts/e2e/lib/plugin-update/probe.mjs @@ -121,16 +121,23 @@ function assertCorruptUpdate(updateJsonPath, pluginId) { if (!plugins) { throw new Error(`missing postUpdate.plugins in update output: ${JSON.stringify(payload)}`); } - if (plugins.status !== "warning") { - throw new Error( - `expected post-update plugin status warning, got ${JSON.stringify(plugins.status)}`, - ); - } - assertCorruptPluginDetails(plugins, pluginId); + assertCorruptPluginTolerated(plugins, pluginId); } function assertCorruptPluginResult(pluginJsonPath, pluginId) { const plugins = readJson(pluginJsonPath); + assertCorruptPluginTolerated(plugins, pluginId); +} + +function assertCorruptPluginTolerated(plugins, pluginId) { + const evidence = collectPluginEvidence(plugins, pluginId); + if (plugins.status === "ok") { + if (isCorruptPluginDisabledAfterUpdate(evidence, pluginId)) { + return; + } + assertCorruptPluginCleanOrRepaired(evidence); + return; + } if (plugins.status !== "warning") { throw new Error( `expected post-update plugin status warning, got ${JSON.stringify(plugins.status)}`, @@ -139,23 +146,49 @@ function assertCorruptPluginResult(pluginJsonPath, pluginId) { assertCorruptPluginDetails(plugins, pluginId); } +function isCorruptPluginDisabledAfterUpdate(evidence, pluginId) { + const outcome = evidence.outcome; + const message = typeof outcome?.message === "string" ? outcome.message : ""; + return ( + outcome?.status === "skipped" && + message.includes(`Disabled "${pluginId}" after plugin update failure`) && + message.includes("OpenClaw will continue without it") + ); +} + +function assertCorruptPluginCleanOrRepaired(evidence) { + if (evidence.outcome) { + throw new Error( + `expected clean or repaired corrupt plugin state, got ${JSON.stringify(evidence)}`, + ); + } + if (evidence.warning || evidence.integrityDrift || evidence.syncMessages.length > 0) { + throw new Error( + `expected warning post-update status for corrupt plugin evidence, got ok: ${JSON.stringify( + evidence, + )}`, + ); + } +} + function assertCorruptPluginDetails(plugins, pluginId) { - const outcomes = plugins.npm?.outcomes ?? []; - const outcome = outcomes.find((entry) => entry?.pluginId === pluginId); + const evidence = collectPluginEvidence(plugins, pluginId); + const outcome = evidence.outcome; if (!outcome || outcome.status !== "error") { throw new Error( `expected error outcome for ${pluginId}, got ${JSON.stringify({ - outcomes, + outcomes: plugins.npm?.outcomes ?? [], warnings: plugins.warnings ?? [], sync: plugins.sync, integrityDrifts: plugins.integrityDrifts ?? [], })}`, ); } - const warnings = plugins.warnings ?? []; - const warning = warnings.find((entry) => entry?.pluginId === pluginId); + const warning = evidence.warning; if (!warning) { - throw new Error(`expected warning for ${pluginId}, got ${JSON.stringify(warnings)}`); + throw new Error( + `expected warning for ${pluginId}, got ${JSON.stringify(plugins.warnings ?? [])}`, + ); } const text = JSON.stringify({ outcome, warning }); for (const expected of [ @@ -169,6 +202,21 @@ function assertCorruptPluginDetails(plugins, pluginId) { } } +function collectPluginEvidence(plugins, pluginId) { + const outcomes = plugins.npm?.outcomes ?? []; + const warnings = plugins.warnings ?? []; + const integrityDrifts = plugins.integrityDrifts ?? []; + const syncMessages = [...(plugins.sync?.warnings ?? []), ...(plugins.sync?.errors ?? [])].filter( + (message) => String(message).includes(pluginId), + ); + return { + outcome: outcomes.find((entry) => entry?.pluginId === pluginId), + warning: warnings.find((entry) => entry?.pluginId === pluginId), + integrityDrift: integrityDrifts.find((entry) => entry?.pluginId === pluginId), + syncMessages, + }; +} + function assertLegacyPostUpdatePluginFailure(updateJsonPath) { const payload = readJson(updateJsonPath); if (payload.status !== "error" || payload.reason !== "post-update-plugins") { diff --git a/scripts/verify-docker-attestations.mjs b/scripts/verify-docker-attestations.mjs index 3da6f0295cf..8605df0c0e2 100644 --- a/scripts/verify-docker-attestations.mjs +++ b/scripts/verify-docker-attestations.mjs @@ -4,11 +4,7 @@ import { execFileSync } from "node:child_process"; import process from "node:process"; const ATTESTATION_REFERENCE_TYPE = "attestation-manifest"; -const ATTESTATION_ARTIFACT_TYPE = "application/vnd.docker.attestation.manifest.v1+json"; -const ATTESTATION_MANIFEST_MEDIA_TYPES = new Set([ - "application/vnd.docker.distribution.manifest.v2+json", - "application/vnd.oci.image.manifest.v1+json", -]); +const EXPECTED_ATTESTATION_ARTIFACT_TYPE = "application/vnd.docker.attestation.manifest.v1+json"; const REQUIRED_PREDICATES = ["https://spdx.dev/Document", "https://slsa.dev/provenance/v1"]; export function imageRefForDigest(imageRef, digest) { @@ -44,13 +40,6 @@ function platformMatches(actual, expected) { ); } -function isAttestationManifest(attestation) { - if (attestation?.artifactType !== undefined) { - return attestation.artifactType === ATTESTATION_ARTIFACT_TYPE; - } - return ATTESTATION_MANIFEST_MEDIA_TYPES.has(attestation?.mediaType); -} - function parseJson(raw, label) { try { return JSON.parse(raw); @@ -97,11 +86,14 @@ export function collectDockerAttestationErrors(params) { const predicates = new Set(); for (const descriptor of attestationDescriptors) { const attestation = inspectAttestation(descriptor.digest); - if (!isAttestationManifest(attestation)) { + if ( + attestation?.artifactType !== undefined && + attestation.artifactType !== EXPECTED_ATTESTATION_ARTIFACT_TYPE + ) { errors.push( - `${imageRef}: ${platformLabel} attestation ${descriptor.digest} has unexpected manifest shape artifactType=${JSON.stringify( + `${imageRef}: ${platformLabel} attestation ${descriptor.digest} has unexpected artifactType ${JSON.stringify( attestation?.artifactType, - )} mediaType=${JSON.stringify(attestation?.mediaType)}`, + )}`, ); } for (const layer of attestation?.layers ?? []) { diff --git a/src/agents/models.profiles.live.test.ts b/src/agents/models.profiles.live.test.ts index c5d3236b6c5..bf80d0607a7 100644 --- a/src/agents/models.profiles.live.test.ts +++ b/src/agents/models.profiles.live.test.ts @@ -292,6 +292,7 @@ function isAudioOnlyModelErrorMessage(raw: string): boolean { function isUnsupportedReasoningEffortErrorMessage(raw: string): boolean { return ( /does not support parameter reasoningeffort/i.test(raw) || + /invalid reasoning effort/i.test(raw) || /unsupported value:\s*'low'.*reasoning\.effort.*supported values are:\s*'medium'/i.test(raw) ); } @@ -313,6 +314,15 @@ function isOpenRouterOpaqueBadRequestErrorMessage(raw: string): boolean { ); } +describe("isUnsupportedReasoningEffortErrorMessage", () => { + it("matches provider-native reasoning effort rejections", () => { + expect(isUnsupportedReasoningEffortErrorMessage('Error: 400 "Invalid reasoning effort."')).toBe( + true, + ); + expect(isUnsupportedReasoningEffortErrorMessage("Error: 400 model not found")).toBe(false); + }); +}); + describe("isUnsupportedPlanErrorMessage", () => { it("matches provider plan-gated models", () => { expect(isUnsupportedPlanErrorMessage("current token plan does not support this model")).toBe( diff --git a/test/scripts/verify-docker-attestations.test.ts b/test/scripts/verify-docker-attestations.test.ts index dd14ee5ecde..a85d6a4baa3 100644 --- a/test/scripts/verify-docker-attestations.test.ts +++ b/test/scripts/verify-docker-attestations.test.ts @@ -51,11 +51,6 @@ function createAttestation( }; } -function createAttestationWithoutArtifactType() { - const { artifactType: _artifactType, ...attestation } = createAttestation(); - return attestation; -} - describe("verify-docker-attestations", () => { it("resolves digest refs from tagged image refs", () => { expect(imageRefForDigest("ghcr.io/openclaw/openclaw:2026.4.26", imageDigest)).toBe( @@ -77,17 +72,37 @@ describe("verify-docker-attestations", () => { expect(errors).toEqual([]); }); - it("accepts OCI attestation manifests without artifactType", () => { + it("accepts attestation manifests with omitted artifactType", () => { const errors = collectDockerAttestationErrors({ imageRef: "ghcr.io/openclaw/openclaw:test", index: createIndex(), requiredPlatforms: [parsePlatform("linux/amd64")], - inspectAttestation: () => createAttestationWithoutArtifactType(), + inspectAttestation: () => { + const attestation: Record = createAttestation(); + delete attestation.artifactType; + return attestation; + }, }); expect(errors).toEqual([]); }); + it("reports unexpected attestation artifact types", () => { + const errors = collectDockerAttestationErrors({ + imageRef: "ghcr.io/openclaw/openclaw:test", + index: createIndex(), + requiredPlatforms: [parsePlatform("linux/amd64")], + inspectAttestation: () => ({ + ...createAttestation(), + artifactType: "application/vnd.unknown", + }), + }); + + expect(errors).toEqual([ + `ghcr.io/openclaw/openclaw:test: linux/amd64 attestation ${attestationDigest} has unexpected artifactType "application/vnd.unknown"`, + ]); + }); + it("reports missing attestation manifests", () => { const index = createIndex(); index.manifests = index.manifests.slice(0, 1); @@ -116,20 +131,4 @@ describe("verify-docker-attestations", () => { "ghcr.io/openclaw/openclaw:test: linux/amd64 missing predicate https://slsa.dev/provenance/v1", ]); }); - - it("reports an unexpected attestation manifest shape", () => { - const errors = collectDockerAttestationErrors({ - imageRef: "ghcr.io/openclaw/openclaw:test", - index: createIndex(), - requiredPlatforms: [parsePlatform("linux/amd64")], - inspectAttestation: () => ({ - ...createAttestation(), - artifactType: "application/vnd.example.invalid", - }), - }); - - expect(errors).toEqual([ - `ghcr.io/openclaw/openclaw:test: linux/amd64 attestation ${attestationDigest} has unexpected manifest shape artifactType="application/vnd.example.invalid" mediaType="application/vnd.oci.image.manifest.v1+json"`, - ]); - }); });