fix: ignore unsupported video generation overrides

2026-04-21 13:44:03 +00:00 · 2026-04-06 00:59:02 +01:00
parent c4cc557604
commit ad6c584ce7
10 changed files with 382 additions and 33 deletions
--- a/docs/providers/openai.md
+++ b/docs/providers/openai.md
@@ -145,8 +145,10 @@ The bundled `openai` plugin also registers video generation through the shared
 - Default video model: `openai/sora-2`
 - Modes: text-to-video, image-to-video, and single-video reference/edit flows
 - Current limits: 1 image or 1 video reference input
- Current OpenAI-specific caveat: OpenClaw does not forward `aspectRatio` or
-  `resolution` overrides to the native OpenAI video API today
+- Current OpenAI-specific caveat: OpenClaw currently only forwards `size`
+  overrides for native OpenAI video generation. Unsupported optional overrides
+  such as `aspectRatio`, `resolution`, `audio`, and `watermark` are ignored
+  and reported back as a tool warning.

 To use OpenAI as the default video provider:

--- a/docs/tools/video-generation.md
+++ b/docs/tools/video-generation.md
@@ -82,7 +82,7 @@ Use `action: "list"` to inspect available providers and models at runtime:
 | `watermark`       | boolean  | Toggle provider watermarking when supported                                            |
 | `filename`        | string   | Output filename hint                                                                   |

-Not all providers support all parameters. The tool validates provider capability limits before it submits the request. When a provider or model only supports a discrete set of video lengths, OpenClaw rounds `durationSeconds` to the nearest supported value and reports the normalized duration in the tool result.
+Not all providers support all parameters. Unsupported optional overrides are ignored on a best-effort basis and reported back in the tool result as a warning. Hard capability limits such as too many reference inputs still fail before submission. When a provider or model only supports a discrete set of video lengths, OpenClaw rounds `durationSeconds` to the nearest supported value and reports the normalized duration in the tool result.

 ## Async behavior

--- a/extensions/video-generation-core/src/runtime.test.ts
+++ b/extensions/video-generation-core/src/runtime.test.ts
@@ -108,6 +108,7 @@ describe("video-generation runtime", () => {
    expect(result.provider).toBe("video-plugin");
    expect(result.model).toBe("vid-v1");
    expect(result.attempts).toEqual([]);
+    expect(result.ignoredOverrides).toEqual([]);
    expect(seenAuthStore).toEqual(authStore);
    expect(result.videos).toEqual([
      {
@@ -161,4 +162,66 @@ describe("video-generation runtime", () => {
    );
    await expect(promise).rejects.toThrow("qwen: QWEN_API_KEY");
  });
+
+  it("ignores unsupported optional overrides per provider", async () => {
+    let seenRequest:
+      | {
+          size?: string;
+          aspectRatio?: string;
+          resolution?: string;
+          audio?: boolean;
+          watermark?: boolean;
+        }
+      | undefined;
+    mocks.resolveAgentModelPrimaryValue.mockReturnValue("openai/sora-2");
+    mocks.getVideoGenerationProvider.mockReturnValue({
+      id: "openai",
+      capabilities: {
+        supportsSize: true,
+      },
+      generateVideo: async (req) => {
+        seenRequest = {
+          size: req.size,
+          aspectRatio: req.aspectRatio,
+          resolution: req.resolution,
+          audio: req.audio,
+          watermark: req.watermark,
+        };
+        return {
+          videos: [{ buffer: Buffer.from("mp4-bytes"), mimeType: "video/mp4" }],
+          model: "sora-2",
+        };
+      },
+    });
+
+    const result = await generateVideo({
+      cfg: {
+        agents: {
+          defaults: {
+            videoGenerationModel: { primary: "openai/sora-2" },
+          },
+        },
+      } as OpenClawConfig,
+      prompt: "animate a lobster",
+      size: "1280x720",
+      aspectRatio: "16:9",
+      resolution: "720P",
+      audio: false,
+      watermark: false,
+    });
+
+    expect(seenRequest).toEqual({
+      size: "1280x720",
+      aspectRatio: undefined,
+      resolution: undefined,
+      audio: undefined,
+      watermark: undefined,
+    });
+    expect(result.ignoredOverrides).toEqual([
+      { key: "aspectRatio", value: "16:9" },
+      { key: "resolution", value: "720P" },
+      { key: "audio", value: false },
+      { key: "watermark", value: false },
+    ]);
+  });
 });
--- a/extensions/video-generation-core/src/runtime.ts
+++ b/extensions/video-generation-core/src/runtime.ts
@@ -12,6 +12,7 @@ import {
  type FallbackAttempt,
  type GeneratedVideoAsset,
  type OpenClawConfig,
+  type VideoGenerationIgnoredOverride,
  type VideoGenerationResolution,
  type VideoGenerationResult,
  type VideoGenerationSourceAsset,
@@ -41,6 +42,7 @@ export type GenerateVideoRuntimeResult = {
  model: string;
  attempts: FallbackAttempt[];
  metadata?: Record<string, unknown>;
+  ignoredOverrides: VideoGenerationIgnoredOverride[];
 };

 function resolveVideoGenerationCandidates(params: {
@@ -116,6 +118,57 @@ export function listRuntimeVideoGenerationProviders(params?: { config?: OpenClaw
  return listVideoGenerationProviders(params?.config);
 }

+function resolveProviderVideoGenerationOverrides(params: {
+  provider: NonNullable<ReturnType<typeof getVideoGenerationProvider>>;
+  size?: string;
+  aspectRatio?: string;
+  resolution?: VideoGenerationResolution;
+  audio?: boolean;
+  watermark?: boolean;
+}) {
+  const caps = params.provider.capabilities;
+  const ignoredOverrides: VideoGenerationIgnoredOverride[] = [];
+  let size = params.size;
+  let aspectRatio = params.aspectRatio;
+  let resolution = params.resolution;
+  let audio = params.audio;
+  let watermark = params.watermark;
+
+  if (size && !caps.supportsSize) {
+    ignoredOverrides.push({ key: "size", value: size });
+    size = undefined;
+  }
+
+  if (aspectRatio && !caps.supportsAspectRatio) {
+    ignoredOverrides.push({ key: "aspectRatio", value: aspectRatio });
+    aspectRatio = undefined;
+  }
+
+  if (resolution && !caps.supportsResolution) {
+    ignoredOverrides.push({ key: "resolution", value: resolution });
+    resolution = undefined;
+  }
+
+  if (typeof audio === "boolean" && !caps.supportsAudio) {
+    ignoredOverrides.push({ key: "audio", value: audio });
+    audio = undefined;
+  }
+
+  if (typeof watermark === "boolean" && !caps.supportsWatermark) {
+    ignoredOverrides.push({ key: "watermark", value: watermark });
+    watermark = undefined;
+  }
+
+  return {
+    size,
+    aspectRatio,
+    resolution,
+    audio,
+    watermark,
+    ignoredOverrides,
+  };
+}
+
 export async function generateVideo(
  params: GenerateVideoParams,
 ): Promise<GenerateVideoRuntimeResult> {
@@ -144,6 +197,14 @@ export async function generateVideo(
    }

    try {
+      const sanitized = resolveProviderVideoGenerationOverrides({
+        provider,
+        size: params.size,
+        aspectRatio: params.aspectRatio,
+        resolution: params.resolution,
+        audio: params.audio,
+        watermark: params.watermark,
+      });
      const result: VideoGenerationResult = await provider.generateVideo({
        provider: candidate.provider,
        model: candidate.model,
@@ -151,12 +212,12 @@ export async function generateVideo(
        cfg: params.cfg,
        agentDir: params.agentDir,
        authStore: params.authStore,
-        size: params.size,
-        aspectRatio: params.aspectRatio,
-        resolution: params.resolution,
+        size: sanitized.size,
+        aspectRatio: sanitized.aspectRatio,
+        resolution: sanitized.resolution,
        durationSeconds: params.durationSeconds,
-        audio: params.audio,
-        watermark: params.watermark,
+        audio: sanitized.audio,
+        watermark: sanitized.watermark,
        inputImages: params.inputImages,
        inputVideos: params.inputVideos,
      });
@@ -168,6 +229,7 @@ export async function generateVideo(
        provider: candidate.provider,
        model: result.model ?? candidate.model,
        attempts,
+        ignoredOverrides: sanitized.ignoredOverrides,
        metadata: result.metadata,
      };
    } catch (err) {
--- a/src/agents/tools/video-generate-tool.test.ts
+++ b/src/agents/tools/video-generate-tool.test.ts
@@ -70,6 +70,7 @@ describe("createVideoGenerateTool", () => {
      provider: "qwen",
      model: "wan2.6-t2v",
      attempts: [],
+      ignoredOverrides: [],
      videos: [
        {
          buffer: Buffer.from("video-bytes"),
@@ -240,6 +241,7 @@ describe("createVideoGenerateTool", () => {
      provider: "google",
      model: "veo-3.1-fast-generate-preview",
      attempts: [],
+      ignoredOverrides: [],
      videos: [
        {
          buffer: Buffer.from("video-bytes"),
@@ -320,4 +322,85 @@ describe("createVideoGenerateTool", () => {
    const text = (result.content?.[0] as { text: string } | undefined)?.text ?? "";
    expect(text).toContain("supportedDurationSeconds=4/6/8");
  });
+
+  it("warns when optional provider overrides are ignored", async () => {
+    vi.spyOn(videoGenerationRuntime, "listRuntimeVideoGenerationProviders").mockReturnValue([
+      {
+        id: "openai",
+        defaultModel: "sora-2",
+        models: ["sora-2"],
+        capabilities: {
+          supportsSize: true,
+        },
+        generateVideo: vi.fn(async () => {
+          throw new Error("not used");
+        }),
+      },
+    ]);
+    vi.spyOn(videoGenerationRuntime, "generateVideo").mockResolvedValue({
+      provider: "openai",
+      model: "sora-2",
+      attempts: [],
+      ignoredOverrides: [
+        { key: "resolution", value: "720P" },
+        { key: "audio", value: false },
+        { key: "watermark", value: false },
+      ],
+      videos: [
+        {
+          buffer: Buffer.from("video-bytes"),
+          mimeType: "video/mp4",
+          fileName: "lobster.mp4",
+        },
+      ],
+    });
+    vi.spyOn(mediaStore, "saveMediaBuffer").mockResolvedValueOnce({
+      path: "/tmp/generated-lobster.mp4",
+      id: "generated-lobster.mp4",
+      size: 11,
+      contentType: "video/mp4",
+    });
+
+    const tool = createVideoGenerateTool({
+      config: asConfig({
+        agents: {
+          defaults: {
+            videoGenerationModel: { primary: "openai/sora-2" },
+          },
+        },
+      }),
+    });
+    if (!tool) {
+      throw new Error("expected video_generate tool");
+    }
+
+    const result = await tool.execute("call-openai-generate", {
+      prompt: "A lobster on a neon bridge",
+      size: "1280x720",
+      resolution: "720P",
+      audio: false,
+      watermark: false,
+    });
+    const text = (result.content?.[0] as { text: string } | undefined)?.text ?? "";
+
+    expect(text).toContain("Generated 1 video with openai/sora-2.");
+    expect(text).toContain(
+      "Warning: Ignored unsupported overrides for openai/sora-2: resolution=720P, audio=false, watermark=false.",
+    );
+    expect(result).toMatchObject({
+      details: {
+        size: "1280x720",
+        warning:
+          "Ignored unsupported overrides for openai/sora-2: resolution=720P, audio=false, watermark=false.",
+        ignoredOverrides: [
+          { key: "resolution", value: "720P" },
+          { key: "audio", value: false },
+          { key: "watermark", value: false },
+        ],
+      },
+    });
+    expect(result.details).not.toHaveProperty("resolution");
+    expect(result.details).not.toHaveProperty("audio");
+    expect(result.details).not.toHaveProperty("watermark");
+  });
 });
--- a/src/agents/tools/video-generate-tool.ts
+++ b/src/agents/tools/video-generate-tool.ts
@@ -15,6 +15,7 @@ import {
  listRuntimeVideoGenerationProviders,
 } from "../../video-generation/runtime.js";
 import type {
+  VideoGenerationIgnoredOverride,
  VideoGenerationProvider,
  VideoGenerationResolution,
  VideoGenerationSourceAsset,
@@ -373,15 +374,6 @@ function validateVideoGenerationCapabilities(params: {
      );
    }
  }
-  if (params.size && !caps.supportsSize) {
-    throw new ToolInputError(`${provider.id} does not support size overrides.`);
-  }
-  if (params.aspectRatio && !caps.supportsAspectRatio) {
-    throw new ToolInputError(`${provider.id} does not support aspectRatio overrides.`);
-  }
-  if (params.resolution && !caps.supportsResolution) {
-    throw new ToolInputError(`${provider.id} does not support resolution overrides.`);
-  }
  if (
    typeof params.durationSeconds === "number" &&
    Number.isFinite(params.durationSeconds) &&
@@ -396,12 +388,10 @@ function validateVideoGenerationCapabilities(params: {
      `${provider.id} supports at most ${caps.maxDurationSeconds} seconds per video.`,
    );
  }
-  if (typeof params.audio === "boolean" && !caps.supportsAudio) {
-    throw new ToolInputError(`${provider.id} does not support audio toggles.`);
-  }
-  if (typeof params.watermark === "boolean" && !caps.supportsWatermark) {
-    throw new ToolInputError(`${provider.id} does not support watermark toggles.`);
-  }
+}
+
+function formatIgnoredVideoGenerationOverride(override: VideoGenerationIgnoredOverride): string {
+  return `${override.key}=${String(override.value)}`;
 }

 type VideoGenerateSandboxConfig = {
@@ -605,6 +595,12 @@ async function executeVideoGenerationJob(params: {
    Number.isFinite(result.metadata.requestedDurationSeconds)
      ? result.metadata.requestedDurationSeconds
      : params.durationSeconds;
+  const ignoredOverrides = result.ignoredOverrides ?? [];
+  const ignoredOverrideKeys = new Set(ignoredOverrides.map((entry) => entry.key));
+  const warning =
+    ignoredOverrides.length > 0
+      ? `Ignored unsupported overrides for ${result.provider}/${result.model}: ${ignoredOverrides.map(formatIgnoredVideoGenerationOverride).join(", ")}.`
+      : undefined;
  const normalizedDurationSeconds =
    typeof result.metadata?.normalizedDurationSeconds === "number" &&
    Number.isFinite(result.metadata.normalizedDurationSeconds)
@@ -617,6 +613,7 @@ async function executeVideoGenerationJob(params: {
    : undefined;
  const lines = [
    `Generated ${savedVideos.length} video${savedVideos.length === 1 ? "" : "s"} with ${result.provider}/${result.model}.`,
+    ...(warning ? [`Warning: ${warning}`] : []),
    typeof requestedDurationSeconds === "number" &&
    typeof normalizedDurationSeconds === "number" &&
    requestedDurationSeconds !== normalizedDurationSeconds
@@ -677,9 +674,13 @@ async function executeVideoGenerationJob(params: {
              })),
            }
          : {}),
-      ...(params.size ? { size: params.size } : {}),
-      ...(params.aspectRatio ? { aspectRatio: params.aspectRatio } : {}),
-      ...(params.resolution ? { resolution: params.resolution } : {}),
+      ...(!ignoredOverrideKeys.has("size") && params.size ? { size: params.size } : {}),
+      ...(!ignoredOverrideKeys.has("aspectRatio") && params.aspectRatio
+        ? { aspectRatio: params.aspectRatio }
+        : {}),
+      ...(!ignoredOverrideKeys.has("resolution") && params.resolution
+        ? { resolution: params.resolution }
+        : {}),
      ...(typeof normalizedDurationSeconds === "number"
        ? { durationSeconds: normalizedDurationSeconds }
        : {}),
@@ -691,11 +692,17 @@ async function executeVideoGenerationJob(params: {
      ...(supportedDurationSeconds && supportedDurationSeconds.length > 0
        ? { supportedDurationSeconds }
        : {}),
-      ...(typeof params.audio === "boolean" ? { audio: params.audio } : {}),
-      ...(typeof params.watermark === "boolean" ? { watermark: params.watermark } : {}),
+      ...(!ignoredOverrideKeys.has("audio") && typeof params.audio === "boolean"
+        ? { audio: params.audio }
+        : {}),
+      ...(!ignoredOverrideKeys.has("watermark") && typeof params.watermark === "boolean"
+        ? { watermark: params.watermark }
+        : {}),
      ...(params.filename ? { filename: params.filename } : {}),
      attempts: result.attempts,
      metadata: result.metadata,
+      ...(warning ? { warning } : {}),
+      ...(ignoredOverrides.length > 0 ? { ignoredOverrides } : {}),
    },
  };
 }
--- a/src/plugin-sdk/video-generation-core.ts
+++ b/src/plugin-sdk/video-generation-core.ts
@@ -5,6 +5,7 @@ export type { FallbackAttempt } from "../agents/model-fallback.types.js";
 export type { VideoGenerationProviderPlugin } from "../plugins/types.js";
 export type {
  GeneratedVideoAsset,
+  VideoGenerationIgnoredOverride,
  VideoGenerationProvider,
  VideoGenerationProviderConfiguredContext,
  VideoGenerationRequest,
--- a/src/video-generation/runtime.test.ts
+++ b/src/video-generation/runtime.test.ts
@@ -118,6 +118,7 @@ describe("video-generation runtime", () => {
    expect(result.provider).toBe("video-plugin");
    expect(result.model).toBe("vid-v1");
    expect(result.attempts).toEqual([]);
+    expect(result.ignoredOverrides).toEqual([]);
    expect(seenAuthStore).toEqual(authStore);
    expect(result.videos).toEqual([
      {
@@ -185,6 +186,69 @@ describe("video-generation runtime", () => {
      normalizedDurationSeconds: 6,
      supportedDurationSeconds: [4, 6, 8],
    });
+    expect(result.ignoredOverrides).toEqual([]);
+  });
+
+  it("ignores unsupported optional overrides per provider", async () => {
+    let seenRequest:
+      | {
+          size?: string;
+          aspectRatio?: string;
+          resolution?: string;
+          audio?: boolean;
+          watermark?: boolean;
+        }
+      | undefined;
+    mocks.resolveAgentModelPrimaryValue.mockReturnValue("openai/sora-2");
+    mocks.getVideoGenerationProvider.mockReturnValue({
+      id: "openai",
+      capabilities: {
+        supportsSize: true,
+      },
+      generateVideo: async (req) => {
+        seenRequest = {
+          size: req.size,
+          aspectRatio: req.aspectRatio,
+          resolution: req.resolution,
+          audio: req.audio,
+          watermark: req.watermark,
+        };
+        return {
+          videos: [{ buffer: Buffer.from("mp4-bytes"), mimeType: "video/mp4" }],
+          model: "sora-2",
+        };
+      },
+    });
+
+    const result = await generateVideo({
+      cfg: {
+        agents: {
+          defaults: {
+            videoGenerationModel: { primary: "openai/sora-2" },
+          },
+        },
+      } as OpenClawConfig,
+      prompt: "animate a lobster",
+      size: "1280x720",
+      aspectRatio: "16:9",
+      resolution: "720P",
+      audio: false,
+      watermark: false,
+    });
+
+    expect(seenRequest).toEqual({
+      size: "1280x720",
+      aspectRatio: undefined,
+      resolution: undefined,
+      audio: undefined,
+      watermark: undefined,
+    });
+    expect(result.ignoredOverrides).toEqual([
+      { key: "aspectRatio", value: "16:9" },
+      { key: "resolution", value: "720P" },
+      { key: "audio", value: false },
+      { key: "watermark", value: false },
+    ]);
  });

  it("builds a generic config hint without hardcoded provider ids", async () => {
--- a/src/video-generation/runtime.ts
+++ b/src/video-generation/runtime.ts
@@ -16,6 +16,7 @@ import { parseVideoGenerationModelRef } from "./model-ref.js";
 import { getVideoGenerationProvider, listVideoGenerationProviders } from "./provider-registry.js";
 import type {
  GeneratedVideoAsset,
+  VideoGenerationIgnoredOverride,
  VideoGenerationResolution,
  VideoGenerationResult,
  VideoGenerationSourceAsset,
@@ -45,6 +46,7 @@ export type GenerateVideoRuntimeResult = {
  model: string;
  attempts: FallbackAttempt[];
  metadata?: Record<string, unknown>;
+  ignoredOverrides: VideoGenerationIgnoredOverride[];
 };

 function resolveVideoGenerationCandidates(params: {
@@ -123,6 +125,57 @@ export function listRuntimeVideoGenerationProviders(params?: { config?: OpenClaw
  return listVideoGenerationProviders(params?.config);
 }

+function resolveProviderVideoGenerationOverrides(params: {
+  provider: NonNullable<ReturnType<typeof getVideoGenerationProvider>>;
+  size?: string;
+  aspectRatio?: string;
+  resolution?: VideoGenerationResolution;
+  audio?: boolean;
+  watermark?: boolean;
+}) {
+  const caps = params.provider.capabilities;
+  const ignoredOverrides: VideoGenerationIgnoredOverride[] = [];
+  let size = params.size;
+  let aspectRatio = params.aspectRatio;
+  let resolution = params.resolution;
+  let audio = params.audio;
+  let watermark = params.watermark;
+
+  if (size && !caps.supportsSize) {
+    ignoredOverrides.push({ key: "size", value: size });
+    size = undefined;
+  }
+
+  if (aspectRatio && !caps.supportsAspectRatio) {
+    ignoredOverrides.push({ key: "aspectRatio", value: aspectRatio });
+    aspectRatio = undefined;
+  }
+
+  if (resolution && !caps.supportsResolution) {
+    ignoredOverrides.push({ key: "resolution", value: resolution });
+    resolution = undefined;
+  }
+
+  if (typeof audio === "boolean" && !caps.supportsAudio) {
+    ignoredOverrides.push({ key: "audio", value: audio });
+    audio = undefined;
+  }
+
+  if (typeof watermark === "boolean" && !caps.supportsWatermark) {
+    ignoredOverrides.push({ key: "watermark", value: watermark });
+    watermark = undefined;
+  }
+
+  return {
+    size,
+    aspectRatio,
+    resolution,
+    audio,
+    watermark,
+    ignoredOverrides,
+  };
+}
+
 export async function generateVideo(
  params: GenerateVideoParams,
 ): Promise<GenerateVideoRuntimeResult> {
@@ -151,6 +204,14 @@ export async function generateVideo(
    }

    try {
+      const sanitized = resolveProviderVideoGenerationOverrides({
+        provider,
+        size: params.size,
+        aspectRatio: params.aspectRatio,
+        resolution: params.resolution,
+        audio: params.audio,
+        watermark: params.watermark,
+      });
      const requestedDurationSeconds =
        typeof params.durationSeconds === "number" && Number.isFinite(params.durationSeconds)
          ? Math.max(1, Math.round(params.durationSeconds))
@@ -171,12 +232,12 @@ export async function generateVideo(
        cfg: params.cfg,
        agentDir: params.agentDir,
        authStore: params.authStore,
-        size: params.size,
-        aspectRatio: params.aspectRatio,
-        resolution: params.resolution,
+        size: sanitized.size,
+        aspectRatio: sanitized.aspectRatio,
+        resolution: sanitized.resolution,
        durationSeconds: normalizedDurationSeconds,
-        audio: params.audio,
-        watermark: params.watermark,
+        audio: sanitized.audio,
+        watermark: sanitized.watermark,
        inputImages: params.inputImages,
        inputVideos: params.inputVideos,
      });
@@ -188,6 +249,7 @@ export async function generateVideo(
        provider: candidate.provider,
        model: result.model ?? candidate.model,
        attempts,
+        ignoredOverrides: sanitized.ignoredOverrides,
        metadata:
          typeof requestedDurationSeconds === "number" &&
          typeof normalizedDurationSeconds === "number" &&
--- a/src/video-generation/types.ts
+++ b/src/video-generation/types.ts
@@ -47,6 +47,11 @@ export type VideoGenerationResult = {
  metadata?: Record<string, unknown>;
 };

+export type VideoGenerationIgnoredOverride = {
+  key: "size" | "aspectRatio" | "resolution" | "audio" | "watermark";
+  value: string | boolean;
+};
+
 export type VideoGenerationProviderCapabilities = {
  maxVideos?: number;
  maxInputImages?: number;