From 697d85aefeb4df2bd97a1a3d64235f41d61f3dca Mon Sep 17 00:00:00 2001 From: Peter Steinberger Date: Tue, 28 Apr 2026 00:36:12 +0100 Subject: [PATCH] fix: auto-register bundled computer use marketplace --- docs/plugins/codex-computer-use.md | 59 ++++++- docs/plugins/codex-harness.md | 18 ++- .../codex/src/app-server/computer-use.test.ts | 152 ++++++++++++++++++ .../codex/src/app-server/computer-use.ts | 57 +++++-- 4 files changed, 269 insertions(+), 17 deletions(-) diff --git a/docs/plugins/codex-computer-use.md b/docs/plugins/codex-computer-use.md index b119eebbb3f..24df3bc148c 100644 --- a/docs/plugins/codex-computer-use.md +++ b/docs/plugins/codex-computer-use.md @@ -39,8 +39,9 @@ Computer Use available before a thread starts: agents: { defaults: { model: "openai/gpt-5.5", - embeddedHarness: { - runtime: "codex", + agentRuntime: { + id: "codex", + fallback: "none", }, }, }, @@ -50,13 +51,22 @@ Computer Use available before a thread starts: With this config, OpenClaw checks Codex app-server before each Codex-mode turn. If Computer Use is missing but Codex app-server has already discovered an installable marketplace, OpenClaw asks Codex app-server to install or re-enable -the plugin and reload MCP servers. If setup still cannot make the MCP server -available, the turn fails before the thread starts. +the plugin and reload MCP servers. On macOS, when no matching marketplace is +registered and the standard Codex app bundle exists, OpenClaw also tries to +register the bundled Codex marketplace from +`/Applications/Codex.app/Contents/Resources/plugins/openai-bundled` before it +fails. If setup still cannot make the MCP server available, the turn fails +before the thread starts. + +Existing sessions keep their runtime and Codex thread binding. After changing +`agentRuntime` or Computer Use config, use `/new` or `/reset` in the affected +chat before testing. ## Commands Use the `/codex computer-use` commands from any chat surface where the `codex` -plugin command surface is available: +plugin command surface is available. These are OpenClaw chat/runtime commands, +not `openclaw codex ...` CLI subcommands: ```text /codex computer-use status @@ -93,6 +103,32 @@ If multiple known marketplaces contain Computer Use, OpenClaw prefers `openai-bundled`, then `openai-curated`, then `local`. Unknown ambiguous matches fail closed and ask you to set `marketplaceName` or `marketplacePath`. +## Bundled macOS marketplace + +Recent Codex desktop builds bundle Computer Use here: + +```text +/Applications/Codex.app/Contents/Resources/plugins/openai-bundled/plugins/computer-use +``` + +When `computerUse.autoInstall` is true and no marketplace containing +`computer-use` is registered, OpenClaw tries to add the standard bundled +marketplace root automatically: + +```text +/Applications/Codex.app/Contents/Resources/plugins/openai-bundled +``` + +You can also register it explicitly from a shell with Codex: + +```bash +codex plugin marketplace add /Applications/Codex.app/Contents/Resources/plugins/openai-bundled +``` + +If you use a nonstandard Codex app path, set `computerUse.marketplacePath` to a +local marketplace file path or run `/codex computer-use install --source +` once. + ## Remote catalog limit Codex app-server can list and read remote-only catalog entries, but it does not @@ -125,6 +161,8 @@ Turn-start auto-install intentionally refuses configured `marketplaceSource` values. Adding a new source is an explicit setup operation, so use `/codex computer-use install --source ` once, then let `autoInstall` handle future re-enables from discovered local marketplaces. +Turn-start auto-install can use a configured `marketplacePath`, because that is +already a local path on the host. ## What OpenClaw checks @@ -180,6 +218,17 @@ current app-server API. servers reload. If it remains unavailable, fix the Codex Computer Use app, Codex app-server MCP status, or macOS permissions. +**Status or a probe times out on `computer-use.list_apps`.** The plugin and MCP +server are present, but the local Computer Use bridge did not answer. Quit or +restart Codex Computer Use, relaunch Codex Desktop if needed, then retry in a +fresh OpenClaw session. + +**A Computer Use tool says `Native hook relay unavailable`.** The Codex-native +tool hook reached OpenClaw with a stale or missing relay registration. Start a +fresh OpenClaw session with `/new` or `/reset`. If it keeps happening, restart +the gateway so old app-server threads and hook registrations are dropped, then +retry. + **Turn-start auto-install refuses a source.** This is intentional. Add the source with explicit `/codex computer-use install --source ` first, then future turn-start auto-install can use the discovered local diff --git a/docs/plugins/codex-harness.md b/docs/plugins/codex-harness.md index 022eccc1697..ed8eb98fbfb 100644 --- a/docs/plugins/codex-harness.md +++ b/docs/plugins/codex-harness.md @@ -571,8 +571,9 @@ Minimal config: agents: { defaults: { model: "openai/gpt-5.5", - embeddedHarness: { - runtime: "codex", + agentRuntime: { + id: "codex", + fallback: "none", }, }, }, @@ -593,6 +594,13 @@ silently running without the native Computer Use tools. See [Codex Computer Use](/plugins/codex-computer-use) for marketplace choices, remote catalog limits, status reasons, and troubleshooting. +When `computerUse.autoInstall` is true, OpenClaw can register the standard +bundled Codex Desktop marketplace from +`/Applications/Codex.app/Contents/Resources/plugins/openai-bundled` if Codex +has not discovered a local marketplace yet. Use `/new` or `/reset` after +changing runtime or Computer Use config so existing sessions do not keep an old +PI or Codex thread binding. + ## Common recipes Local Codex with default stdio transport: @@ -853,6 +861,12 @@ and that the remote app-server speaks the same Codex app-server protocol version provider path in `auto` mode. If you force `agentRuntime.id: "codex"`, every embedded turn for that agent must be a Codex-supported OpenAI model. +**Computer Use is installed but tools do not run:** check +`/codex computer-use status` from a fresh session. If a tool reports +`Native hook relay unavailable`, use `/new` or `/reset`; if it persists, restart +the gateway to clear stale native hook registrations. If `computer-use.list_apps` +times out, restart Codex Computer Use or Codex Desktop and retry. + ## Related - [Agent harness plugins](/plugins/sdk-agent-harness) diff --git a/extensions/codex/src/app-server/computer-use.test.ts b/extensions/codex/src/app-server/computer-use.test.ts index 98be0d2d03f..9c597120c10 100644 --- a/extensions/codex/src/app-server/computer-use.test.ts +++ b/extensions/codex/src/app-server/computer-use.test.ts @@ -1,3 +1,6 @@ +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; import { afterEach, describe, expect, it, vi } from "vitest"; import { ensureCodexComputerUse, @@ -7,8 +10,13 @@ import { } from "./computer-use.js"; describe("Codex Computer Use setup", () => { + const cleanupPaths: string[] = []; + afterEach(() => { vi.useRealTimers(); + for (const cleanupPath of cleanupPaths.splice(0)) { + fs.rmSync(cleanupPath, { recursive: true, force: true }); + } }); it("stays disabled until configured", async () => { @@ -253,6 +261,69 @@ describe("Codex Computer Use setup", () => { }); }); + it("auto-registers the bundled Codex app marketplace during auto-install", async () => { + const bundledMarketplacePath = fs.mkdtempSync( + path.join(os.tmpdir(), "openclaw-codex-bundled-marketplace-"), + ); + cleanupPaths.push(bundledMarketplacePath); + const request = createBundledMarketplaceComputerUseRequest(bundledMarketplacePath); + + await expect( + ensureCodexComputerUse({ + pluginConfig: { + computerUse: { + enabled: true, + autoInstall: true, + }, + }, + request, + defaultBundledMarketplacePath: bundledMarketplacePath, + }), + ).resolves.toEqual( + expect.objectContaining({ + ready: true, + reason: "ready", + marketplaceName: "openai-bundled", + message: "Computer Use is ready.", + }), + ); + expect(request).toHaveBeenCalledWith("marketplace/add", { + source: bundledMarketplacePath, + }); + expect(request).toHaveBeenCalledWith("plugin/install", { + marketplacePath: `${bundledMarketplacePath}/.agents/plugins/marketplace.json`, + pluginName: "computer-use", + }); + }); + + it("allows auto-install from a configured local marketplace path", async () => { + const request = createComputerUseRequest({ installed: false }); + + await expect( + ensureCodexComputerUse({ + pluginConfig: { + computerUse: { + enabled: true, + autoInstall: true, + marketplacePath: "/marketplaces/desktop-tools/.agents/plugins/marketplace.json", + }, + }, + request, + }), + ).resolves.toEqual( + expect.objectContaining({ + ready: true, + reason: "ready", + message: "Computer Use is ready.", + }), + ); + expect(request).not.toHaveBeenCalledWith("marketplace/add", expect.anything()); + expect(request).toHaveBeenCalledWith("plugin/install", { + marketplacePath: "/marketplaces/desktop-tools/.agents/plugins/marketplace.json", + pluginName: "computer-use", + }); + }); + it("requires an explicit install command for configured marketplace sources", async () => { const request = createComputerUseRequest({ installed: false }); @@ -607,6 +678,87 @@ function createMultiMarketplaceComputerUseRequest(): CodexComputerUseRequest { }) as CodexComputerUseRequest; } +function createBundledMarketplaceComputerUseRequest( + bundledMarketplacePath: string, +): CodexComputerUseRequest { + let registered = false; + let installed = false; + return vi.fn(async (method: string, requestParams?: unknown) => { + if (method === "experimentalFeature/enablement/set") { + return { enablement: { plugins: true } }; + } + if (method === "marketplace/add") { + expect(requestParams).toEqual({ + source: bundledMarketplacePath, + }); + registered = true; + return { + marketplaceName: "openai-bundled", + installedRoot: bundledMarketplacePath, + alreadyAdded: false, + }; + } + if (method === "plugin/list") { + return { + marketplaces: registered + ? [ + { + name: "openai-bundled", + path: `${bundledMarketplacePath}/.agents/plugins/marketplace.json`, + interface: null, + plugins: [pluginSummary(installed, "openai-bundled")], + }, + ] + : [], + marketplaceLoadErrors: [], + featuredPluginIds: [], + }; + } + if (method === "plugin/read") { + return { + plugin: { + marketplaceName: "openai-bundled", + marketplacePath: `${bundledMarketplacePath}/.agents/plugins/marketplace.json`, + summary: pluginSummary(installed, "openai-bundled"), + description: "Control desktop apps.", + skills: [], + apps: [], + mcpServers: ["computer-use"], + }, + }; + } + if (method === "plugin/install") { + installed = true; + return { authPolicy: "ON_INSTALL", appsNeedingAuth: [] }; + } + if (method === "config/mcpServer/reload") { + return undefined; + } + if (method === "mcpServerStatus/list") { + return { + data: installed + ? [ + { + name: "computer-use", + tools: { + list_apps: { + name: "list_apps", + inputSchema: { type: "object" }, + }, + }, + resources: [], + resourceTemplates: [], + authStatus: "unsupported", + }, + ] + : [], + nextCursor: null, + }; + } + throw new Error(`unexpected request ${method}`); + }) as CodexComputerUseRequest; +} + function marketplaceEntry(marketplaceName: string, installed: boolean) { return { name: marketplaceName, diff --git a/extensions/codex/src/app-server/computer-use.ts b/extensions/codex/src/app-server/computer-use.ts index b4ff01df8fd..861c73efda2 100644 --- a/extensions/codex/src/app-server/computer-use.ts +++ b/extensions/codex/src/app-server/computer-use.ts @@ -1,3 +1,4 @@ +import { existsSync } from "node:fs"; import { describeControlFailure } from "./capabilities.js"; import type { CodexAppServerClient } from "./client.js"; import { @@ -59,6 +60,7 @@ export type CodexComputerUseSetupParams = { timeoutMs?: number; signal?: AbortSignal; forceEnable?: boolean; + defaultBundledMarketplacePath?: string; }; type MarketplaceRef = @@ -90,6 +92,8 @@ type PluginInspection = const CURATED_MARKETPLACE_POLL_INTERVAL_MS = 2_000; const COMPUTER_USE_MARKETPLACE_NAME_PRIORITY = ["openai-bundled", "openai-curated", "local"]; +const DEFAULT_CODEX_BUNDLED_MARKETPLACE_PATH = + "/Applications/Codex.app/Contents/Resources/plugins/openai-bundled"; export async function readCodexComputerUseStatus( params: CodexComputerUseSetupParams = {}, @@ -176,6 +180,7 @@ async function inspectCodexComputerUse(params: { signal?: AbortSignal; config: ResolvedCodexComputerUseConfig; installPlugin: boolean; + defaultBundledMarketplacePath?: string; }): Promise { const request = createComputerUseRequest(params); if (params.installPlugin) { @@ -192,6 +197,7 @@ async function inspectCodexComputerUse(params: { config: params.config, allowAdd: params.installPlugin, signal: params.signal, + defaultBundledMarketplacePath: params.defaultBundledMarketplacePath, }); if (!marketplace.marketplace) { return unavailableStatus( @@ -320,6 +326,7 @@ async function resolveMarketplaceRef(params: { config: ResolvedCodexComputerUseConfig; allowAdd: boolean; signal?: AbortSignal; + defaultBundledMarketplacePath?: string; }): Promise { let preferredMarketplaceName = params.config.marketplaceName; if (params.config.marketplaceSource && params.allowAdd) { @@ -336,16 +343,19 @@ async function resolveMarketplaceRef(params: { return { marketplace }; } - let candidates: MarketplaceRef[] = []; + let candidates = await listComputerUseMarketplaceCandidates(params.request, params.config); + if (candidates.length === 0 && shouldAddBundledComputerUseMarketplace(params)) { + const bundledMarketplacePath = + params.defaultBundledMarketplacePath ?? DEFAULT_CODEX_BUNDLED_MARKETPLACE_PATH; + const added = await params.request("marketplace/add", { + source: bundledMarketplacePath, + } satisfies v2.MarketplaceAddParams); + preferredMarketplaceName ??= added.marketplaceName; + candidates = await listComputerUseMarketplaceCandidates(params.request, params.config); + } + const waitUntil = marketplaceDiscoveryWaitUntil(params); while (candidates.length === 0) { - const listed = await params.request("plugin/list", { - cwds: [], - } satisfies v2.PluginListParams); - candidates = findComputerUseMarketplaces(listed, params.config.pluginName); - if (candidates.length > 0) { - break; - } if (Date.now() >= waitUntil) { break; } @@ -353,6 +363,7 @@ async function resolveMarketplaceRef(params: { Math.min(CURATED_MARKETPLACE_POLL_INTERVAL_MS, waitUntil - Date.now()), params.signal, ); + candidates = await listComputerUseMarketplaceCandidates(params.request, params.config); } if (preferredMarketplaceName) { @@ -383,16 +394,42 @@ async function resolveMarketplaceRef(params: { return marketplace ? { marketplace } : {}; } +async function listComputerUseMarketplaceCandidates( + request: CodexComputerUseRequest, + config: ResolvedCodexComputerUseConfig, +): Promise { + const listed = await request("plugin/list", { + cwds: [], + } satisfies v2.PluginListParams); + return findComputerUseMarketplaces(listed, config.pluginName); +} + function blockUnsafeAutoInstallStatus( config: ResolvedCodexComputerUseConfig, ): CodexComputerUseStatus | undefined { - if (!config.marketplaceSource && !config.marketplacePath) { + if (!config.marketplaceSource) { return undefined; } return unavailableStatus( config, "auto_install_blocked", - "Computer Use auto-install only uses marketplaces Codex app-server has already discovered. Run /codex computer-use install to install from a configured marketplace source or path.", + "Computer Use auto-install only uses marketplaces Codex app-server has already discovered. Run /codex computer-use install to install from a configured marketplace source.", + ); +} + +function shouldAddBundledComputerUseMarketplace(params: { + config: ResolvedCodexComputerUseConfig; + allowAdd: boolean; + defaultBundledMarketplacePath?: string; +}): boolean { + const bundledMarketplacePath = + params.defaultBundledMarketplacePath ?? DEFAULT_CODEX_BUNDLED_MARKETPLACE_PATH; + return ( + params.allowAdd && + !params.config.marketplaceSource && + !params.config.marketplacePath && + !params.config.marketplaceName && + existsSync(bundledMarketplacePath) ); }