refactor: route plugin runtime through bundled seams

This commit is contained in:
Peter Steinberger
2026-03-27 16:36:43 +00:00
parent e425056aa3
commit ed055f44ae
87 changed files with 2129 additions and 1582 deletions

View File

@@ -1522,6 +1522,15 @@
"path": "src/channels/plugins/normalize/whatsapp.ts"
}
},
{
"declaration": "export function normalizeChannelId(raw?: string | null | undefined): ChannelId | null;",
"exportName": "normalizeChannelId",
"kind": "function",
"source": {
"line": 80,
"path": "src/channels/plugins/registry.ts"
}
},
{
"declaration": "export function normalizeChatType(raw?: string | undefined): ChatType | undefined;",
"exportName": "normalizeChatType",

View File

@@ -166,6 +166,7 @@
{"declaration":"export function keepHttpServerTaskAlive(params: { server: CloseAwareServer; abortSignal?: AbortSignal | undefined; onAbort?: (() => void | Promise<void>) | undefined; }): Promise<void>;","entrypoint":"channel-runtime","exportName":"keepHttpServerTaskAlive","importSpecifier":"openclaw/plugin-sdk/channel-runtime","kind":"function","recordType":"export","sourceLine":79,"sourcePath":"src/plugin-sdk/channel-lifecycle.ts"}
{"declaration":"export function looksLikeSignalTargetId(raw: string, normalized?: string | undefined): boolean;","entrypoint":"channel-runtime","exportName":"looksLikeSignalTargetId","importSpecifier":"openclaw/plugin-sdk/channel-runtime","kind":"function","recordType":"export","sourceLine":38,"sourcePath":"src/channels/plugins/normalize/signal.ts"}
{"declaration":"export function looksLikeWhatsAppTargetId(raw: string): boolean;","entrypoint":"channel-runtime","exportName":"looksLikeWhatsAppTargetId","importSpecifier":"openclaw/plugin-sdk/channel-runtime","kind":"function","recordType":"export","sourceLine":20,"sourcePath":"src/channels/plugins/normalize/whatsapp.ts"}
{"declaration":"export function normalizeChannelId(raw?: string | null | undefined): ChannelId | null;","entrypoint":"channel-runtime","exportName":"normalizeChannelId","importSpecifier":"openclaw/plugin-sdk/channel-runtime","kind":"function","recordType":"export","sourceLine":80,"sourcePath":"src/channels/plugins/registry.ts"}
{"declaration":"export function normalizeChatType(raw?: string | undefined): ChatType | undefined;","entrypoint":"channel-runtime","exportName":"normalizeChatType","importSpecifier":"openclaw/plugin-sdk/channel-runtime","kind":"function","recordType":"export","sourceLine":3,"sourcePath":"src/channels/chat-type.ts"}
{"declaration":"export function normalizePollDurationHours(value: number | undefined, options: { defaultHours: number; maxHours: number; }): number;","entrypoint":"channel-runtime","exportName":"normalizePollDurationHours","importSpecifier":"openclaw/plugin-sdk/channel-runtime","kind":"function","recordType":"export","sourceLine":93,"sourcePath":"src/polls.ts"}
{"declaration":"export function normalizePollInput(input: PollInput, options?: NormalizePollOptions): NormalizedPollInput;","entrypoint":"channel-runtime","exportName":"normalizePollInput","importSpecifier":"openclaw/plugin-sdk/channel-runtime","kind":"function","recordType":"export","sourceLine":36,"sourcePath":"src/polls.ts"}

View File

@@ -0,0 +1 @@
export { buildAnthropicCliBackend } from "./cli-backend.js";

View File

@@ -1,2 +1,6 @@
export { bluebubblesPlugin } from "./src/channel.js";
export {
resolveBlueBubblesGroupRequireMention,
resolveBlueBubblesGroupToolPolicy,
} from "./src/group-policy.js";
export { isAllowedBlueBubblesSender } from "./src/targets.js";

View File

@@ -0,0 +1 @@
export { handleDiscordAction } from "./src/actions/runtime.js";

View File

@@ -6,7 +6,7 @@ import {
readStringParam,
} from "openclaw/plugin-sdk/agent-runtime";
import type { ChannelMessageActionContext } from "openclaw/plugin-sdk/channel-contract";
import { handleDiscordAction } from "./runtime.js";
import { handleDiscordAction } from "../../action-runtime-api.js";
import {
isDiscordModerationAction,
readDiscordModerationCommand,

View File

@@ -8,10 +8,10 @@ import { readBooleanParam } from "openclaw/plugin-sdk/boolean-param";
import { resolveReactionMessageId } from "openclaw/plugin-sdk/channel-actions";
import type { ChannelMessageActionContext } from "openclaw/plugin-sdk/channel-contract";
import { normalizeInteractiveReply } from "openclaw/plugin-sdk/interactive-runtime";
import { handleDiscordAction } from "../../action-runtime-api.js";
import { buildDiscordInteractiveComponents } from "../shared-interactive.js";
import { resolveDiscordChannelId } from "../targets.js";
import { tryHandleDiscordMessageActionGuildAdmin } from "./handle-action.guild-admin.js";
import { handleDiscordAction } from "./runtime.js";
import { readDiscordParentIdParam } from "./runtime.shared.js";
const providerId = "discord";

View File

@@ -0,0 +1 @@
export { buildGoogleGeminiCliBackend } from "./cli-backend.js";

View File

@@ -0,0 +1 @@
export * from "openclaw/plugin-sdk/image-generation-core";

View File

@@ -0,0 +1,7 @@
{
"name": "@openclaw/image-generation-core",
"version": "2026.3.26",
"private": true,
"description": "OpenClaw image generation runtime package",
"type": "module"
}

View File

@@ -0,0 +1,6 @@
export {
generateImage,
listRuntimeImageGenerationProviders,
type GenerateImageParams,
type GenerateImageRuntimeResult,
} from "./src/runtime.js";

View File

@@ -0,0 +1,183 @@
import {
createSubsystemLogger,
describeFailoverError,
getImageGenerationProvider,
getProviderEnvVars,
isFailoverError,
listImageGenerationProviders,
parseImageGenerationModelRef,
resolveAgentModelFallbackValues,
resolveAgentModelPrimaryValue,
type AuthProfileStore,
type FallbackAttempt,
type GeneratedImageAsset,
type ImageGenerationResolution,
type ImageGenerationResult,
type ImageGenerationSourceImage,
type OpenClawConfig,
} from "../api.js";
const log = createSubsystemLogger("image-generation");
export type GenerateImageParams = {
cfg: OpenClawConfig;
prompt: string;
agentDir?: string;
authStore?: AuthProfileStore;
modelOverride?: string;
count?: number;
size?: string;
aspectRatio?: string;
resolution?: ImageGenerationResolution;
inputImages?: ImageGenerationSourceImage[];
};
export type GenerateImageRuntimeResult = {
images: GeneratedImageAsset[];
provider: string;
model: string;
attempts: FallbackAttempt[];
metadata?: Record<string, unknown>;
};
function resolveImageGenerationCandidates(params: {
cfg: OpenClawConfig;
modelOverride?: string;
}): Array<{ provider: string; model: string }> {
const candidates: Array<{ provider: string; model: string }> = [];
const seen = new Set<string>();
const add = (raw: string | undefined) => {
const parsed = parseImageGenerationModelRef(raw);
if (!parsed) {
return;
}
const key = `${parsed.provider}/${parsed.model}`;
if (seen.has(key)) {
return;
}
seen.add(key);
candidates.push(parsed);
};
add(params.modelOverride);
add(resolveAgentModelPrimaryValue(params.cfg.agents?.defaults?.imageGenerationModel));
for (const fallback of resolveAgentModelFallbackValues(
params.cfg.agents?.defaults?.imageGenerationModel,
)) {
add(fallback);
}
return candidates;
}
function throwImageGenerationFailure(params: {
attempts: FallbackAttempt[];
lastError: unknown;
}): never {
if (params.attempts.length <= 1 && params.lastError) {
throw params.lastError;
}
const summary =
params.attempts.length > 0
? params.attempts
.map((attempt) => `${attempt.provider}/${attempt.model}: ${attempt.error}`)
.join(" | ")
: "unknown";
throw new Error(`All image generation models failed (${params.attempts.length}): ${summary}`, {
cause: params.lastError instanceof Error ? params.lastError : undefined,
});
}
function buildNoImageGenerationModelConfiguredMessage(cfg: OpenClawConfig): string {
const providers = listImageGenerationProviders(cfg);
const sampleModel =
providers.find((provider) => provider.defaultModel) ??
({ id: "google", defaultModel: "gemini-3-pro-image-preview" } as const);
const authHints = providers
.flatMap((provider) => {
const envVars = getProviderEnvVars(provider.id);
if (envVars.length === 0) {
return [];
}
return [`${provider.id}: ${envVars.join(" / ")}`];
})
.slice(0, 3);
return [
`No image-generation model configured. Set agents.defaults.imageGenerationModel.primary to a provider/model like "${sampleModel.id}/${sampleModel.defaultModel}".`,
authHints.length > 0
? `If you want a specific provider, also configure that provider's auth/API key first (${authHints.join("; ")}).`
: "If you want a specific provider, also configure that provider's auth/API key first.",
].join(" ");
}
export function listRuntimeImageGenerationProviders(params?: { config?: OpenClawConfig }) {
return listImageGenerationProviders(params?.config);
}
export async function generateImage(
params: GenerateImageParams,
): Promise<GenerateImageRuntimeResult> {
const candidates = resolveImageGenerationCandidates({
cfg: params.cfg,
modelOverride: params.modelOverride,
});
if (candidates.length === 0) {
throw new Error(buildNoImageGenerationModelConfiguredMessage(params.cfg));
}
const attempts: FallbackAttempt[] = [];
let lastError: unknown;
for (const candidate of candidates) {
const provider = getImageGenerationProvider(candidate.provider, params.cfg);
if (!provider) {
const error = `No image-generation provider registered for ${candidate.provider}`;
attempts.push({
provider: candidate.provider,
model: candidate.model,
error,
});
lastError = new Error(error);
continue;
}
try {
const result: ImageGenerationResult = await provider.generateImage({
provider: candidate.provider,
model: candidate.model,
prompt: params.prompt,
cfg: params.cfg,
agentDir: params.agentDir,
authStore: params.authStore,
count: params.count,
size: params.size,
aspectRatio: params.aspectRatio,
resolution: params.resolution,
inputImages: params.inputImages,
});
if (!Array.isArray(result.images) || result.images.length === 0) {
throw new Error("Image generation provider returned no images.");
}
return {
images: result.images,
provider: candidate.provider,
model: result.model ?? candidate.model,
attempts,
metadata: result.metadata,
};
} catch (err) {
lastError = err;
const described = isFailoverError(err) ? describeFailoverError(err) : undefined;
attempts.push({
provider: candidate.provider,
model: candidate.model,
error: described?.message ?? (err instanceof Error ? err.message : String(err)),
reason: described?.reason,
status: described?.status,
code: described?.code,
});
log.debug(`image-generation candidate failed: ${candidate.provider}/${candidate.model}`);
}
}
throwImageGenerationFailure({ attempts, lastError });
}

View File

@@ -1,11 +1,13 @@
import { ToolPolicySchema } from "openclaw/plugin-sdk/agent-config-primitives";
import {
AllowFromListSchema,
buildNestedDmConfigSchema,
DmPolicySchema,
GroupPolicySchema,
} from "openclaw/plugin-sdk/channel-config-schema";
MarkdownConfigSchema,
} from "openclaw/plugin-sdk/channel-config-primitives";
import { buildSecretInputSchema } from "openclaw/plugin-sdk/secret-input";
import { z } from "openclaw/plugin-sdk/zod";
import { buildSecretInputSchema, MarkdownConfigSchema, ToolPolicySchema } from "./runtime-api.js";
const matrixActionSchema = z
.object({

View File

@@ -20,7 +20,7 @@ import {
createComputedAccountStatusAdapter,
createDefaultChannelRuntimeState,
} from "openclaw/plugin-sdk/status-helpers";
import { MattermostConfigSchema } from "./config-schema.js";
import { MattermostChannelConfigSchema } from "./config-surface.js";
import { resolveMattermostGroupRequireMention } from "./group-mentions.js";
import {
listMattermostAccountIds,
@@ -40,7 +40,6 @@ import { sendMessageMattermost } from "./mattermost/send.js";
import { resolveMattermostOpaqueTarget } from "./mattermost/target-resolution.js";
import { looksLikeMattermostTargetId, normalizeMattermostMessagingTarget } from "./normalize.js";
import {
buildChannelConfigSchema,
createAccountStatusSink,
DEFAULT_ACCOUNT_ID,
resolveAllowlistProviderRuntimeGroupPolicy,
@@ -306,7 +305,7 @@ export const mattermostPlugin: ChannelPlugin<ResolvedMattermostAccount> = create
blockStreamingCoalesceDefaults: { minChars: 1500, idleMs: 1000 },
},
reload: { configPrefixes: ["channels.mattermost"] },
configSchema: buildChannelConfigSchema(MattermostConfigSchema),
configSchema: MattermostChannelConfigSchema,
config: {
...mattermostConfigAdapter,
isConfigured: (account) => Boolean(account.botToken && account.baseUrl),

View File

@@ -0,0 +1,125 @@
import {
BlockStreamingCoalesceSchema,
DmPolicySchema,
GroupPolicySchema,
MarkdownConfigSchema,
requireOpenAllowFrom,
} from "openclaw/plugin-sdk/channel-config-primitives";
import { z } from "openclaw/plugin-sdk/zod";
import { buildSecretInputSchema } from "./secret-input.js";
function requireMattermostOpenAllowFrom(params: {
policy?: string;
allowFrom?: Array<string | number>;
ctx: z.RefinementCtx;
}) {
requireOpenAllowFrom({
policy: params.policy,
allowFrom: params.allowFrom,
ctx: params.ctx,
path: ["allowFrom"],
message:
'channels.mattermost.dmPolicy="open" requires channels.mattermost.allowFrom to include "*"',
});
}
const DmChannelRetrySchema = z
.object({
/** Maximum number of retry attempts for DM channel creation (default: 3) */
maxRetries: z.number().int().min(0).max(10).optional(),
/** Initial delay in milliseconds before first retry (default: 1000) */
initialDelayMs: z.number().int().min(100).max(60000).optional(),
/** Maximum delay in milliseconds between retries (default: 10000) */
maxDelayMs: z.number().int().min(1000).max(60000).optional(),
/** Timeout for each individual DM channel creation request in milliseconds (default: 30000) */
timeoutMs: z.number().int().min(5000).max(120000).optional(),
})
.strict()
.refine(
(data) => {
if (data.initialDelayMs !== undefined && data.maxDelayMs !== undefined) {
return data.initialDelayMs <= data.maxDelayMs;
}
return true;
},
{
message: "initialDelayMs must be less than or equal to maxDelayMs",
path: ["initialDelayMs"],
},
)
.optional();
const MattermostSlashCommandsSchema = z
.object({
/** Enable native slash commands. "auto" resolves to false (opt-in). */
native: z.union([z.boolean(), z.literal("auto")]).optional(),
/** Also register skill-based commands. */
nativeSkills: z.union([z.boolean(), z.literal("auto")]).optional(),
/** Path for the callback endpoint on the gateway HTTP server. */
callbackPath: z.string().optional(),
/** Explicit callback URL (e.g. behind reverse proxy). */
callbackUrl: z.string().optional(),
})
.strict()
.optional();
const MattermostAccountSchemaBase = z
.object({
name: z.string().optional(),
capabilities: z.array(z.string()).optional(),
dangerouslyAllowNameMatching: z.boolean().optional(),
markdown: MarkdownConfigSchema,
enabled: z.boolean().optional(),
configWrites: z.boolean().optional(),
botToken: buildSecretInputSchema().optional(),
baseUrl: z.string().optional(),
chatmode: z.enum(["oncall", "onmessage", "onchar"]).optional(),
oncharPrefixes: z.array(z.string()).optional(),
requireMention: z.boolean().optional(),
dmPolicy: DmPolicySchema.optional().default("pairing"),
allowFrom: z.array(z.union([z.string(), z.number()])).optional(),
groupAllowFrom: z.array(z.union([z.string(), z.number()])).optional(),
groupPolicy: GroupPolicySchema.optional().default("allowlist"),
textChunkLimit: z.number().int().positive().optional(),
chunkMode: z.enum(["length", "newline"]).optional(),
blockStreaming: z.boolean().optional(),
blockStreamingCoalesce: BlockStreamingCoalesceSchema.optional(),
replyToMode: z.enum(["off", "first", "all"]).optional(),
responsePrefix: z.string().optional(),
actions: z
.object({
reactions: z.boolean().optional(),
})
.optional(),
commands: MattermostSlashCommandsSchema,
interactions: z
.object({
callbackBaseUrl: z.string().optional(),
allowedSourceIps: z.array(z.string()).optional(),
})
.optional(),
/** Allow fetching from private/internal IP addresses (e.g. localhost). Required for self-hosted Mattermost on LAN/VPN. */
allowPrivateNetwork: z.boolean().optional(),
/** Retry configuration for DM channel creation */
dmChannelRetry: DmChannelRetrySchema,
})
.strict();
const MattermostAccountSchema = MattermostAccountSchemaBase.superRefine((value, ctx) => {
requireMattermostOpenAllowFrom({
policy: value.dmPolicy,
allowFrom: value.allowFrom,
ctx,
});
});
export const MattermostConfigSchema = MattermostAccountSchemaBase.extend({
accounts: z.record(z.string(), MattermostAccountSchema.optional()).optional(),
defaultAccount: z.string().optional(),
}).superRefine((value, ctx) => {
requireMattermostOpenAllowFrom({
policy: value.dmPolicy,
allowFrom: value.allowFrom,
ctx,
});
});

View File

@@ -1,115 +1 @@
import { requireChannelOpenAllowFrom } from "openclaw/plugin-sdk/extension-shared";
import { z } from "openclaw/plugin-sdk/zod";
import {
BlockStreamingCoalesceSchema,
DmPolicySchema,
GroupPolicySchema,
MarkdownConfigSchema,
requireOpenAllowFrom,
} from "./runtime-api.js";
import { buildSecretInputSchema } from "./secret-input.js";
const DmChannelRetrySchema = z
.object({
/** Maximum number of retry attempts for DM channel creation (default: 3) */
maxRetries: z.number().int().min(0).max(10).optional(),
/** Initial delay in milliseconds before first retry (default: 1000) */
initialDelayMs: z.number().int().min(100).max(60000).optional(),
/** Maximum delay in milliseconds between retries (default: 10000) */
maxDelayMs: z.number().int().min(1000).max(60000).optional(),
/** Timeout for each individual DM channel creation request in milliseconds (default: 30000) */
timeoutMs: z.number().int().min(5000).max(120000).optional(),
})
.strict()
.refine(
(data) => {
if (data.initialDelayMs !== undefined && data.maxDelayMs !== undefined) {
return data.initialDelayMs <= data.maxDelayMs;
}
return true;
},
{
message: "initialDelayMs must be less than or equal to maxDelayMs",
path: ["initialDelayMs"],
},
)
.optional();
const MattermostSlashCommandsSchema = z
.object({
/** Enable native slash commands. "auto" resolves to false (opt-in). */
native: z.union([z.boolean(), z.literal("auto")]).optional(),
/** Also register skill-based commands. */
nativeSkills: z.union([z.boolean(), z.literal("auto")]).optional(),
/** Path for the callback endpoint on the gateway HTTP server. */
callbackPath: z.string().optional(),
/** Explicit callback URL (e.g. behind reverse proxy). */
callbackUrl: z.string().optional(),
})
.strict()
.optional();
const MattermostAccountSchemaBase = z
.object({
name: z.string().optional(),
capabilities: z.array(z.string()).optional(),
dangerouslyAllowNameMatching: z.boolean().optional(),
markdown: MarkdownConfigSchema,
enabled: z.boolean().optional(),
configWrites: z.boolean().optional(),
botToken: buildSecretInputSchema().optional(),
baseUrl: z.string().optional(),
chatmode: z.enum(["oncall", "onmessage", "onchar"]).optional(),
oncharPrefixes: z.array(z.string()).optional(),
requireMention: z.boolean().optional(),
dmPolicy: DmPolicySchema.optional().default("pairing"),
allowFrom: z.array(z.union([z.string(), z.number()])).optional(),
groupAllowFrom: z.array(z.union([z.string(), z.number()])).optional(),
groupPolicy: GroupPolicySchema.optional().default("allowlist"),
textChunkLimit: z.number().int().positive().optional(),
chunkMode: z.enum(["length", "newline"]).optional(),
blockStreaming: z.boolean().optional(),
blockStreamingCoalesce: BlockStreamingCoalesceSchema.optional(),
replyToMode: z.enum(["off", "first", "all"]).optional(),
responsePrefix: z.string().optional(),
actions: z
.object({
reactions: z.boolean().optional(),
})
.optional(),
commands: MattermostSlashCommandsSchema,
interactions: z
.object({
callbackBaseUrl: z.string().optional(),
allowedSourceIps: z.array(z.string()).optional(),
})
.optional(),
/** Allow fetching from private/internal IP addresses (e.g. localhost). Required for self-hosted Mattermost on LAN/VPN. */
allowPrivateNetwork: z.boolean().optional(),
/** Retry configuration for DM channel creation */
dmChannelRetry: DmChannelRetrySchema,
})
.strict();
const MattermostAccountSchema = MattermostAccountSchemaBase.superRefine((value, ctx) => {
requireChannelOpenAllowFrom({
channel: "mattermost",
policy: value.dmPolicy,
allowFrom: value.allowFrom,
ctx,
requireOpenAllowFrom,
});
});
export const MattermostConfigSchema = MattermostAccountSchemaBase.extend({
accounts: z.record(z.string(), MattermostAccountSchema.optional()).optional(),
defaultAccount: z.string().optional(),
}).superRefine((value, ctx) => {
requireChannelOpenAllowFrom({
channel: "mattermost",
policy: value.dmPolicy,
allowFrom: value.allowFrom,
ctx,
requireOpenAllowFrom,
});
});
export { MattermostConfigSchema } from "./config-schema-core.js";

View File

@@ -0,0 +1,4 @@
import { buildChannelConfigSchema } from "openclaw/plugin-sdk/channel-config-primitives";
import { MattermostConfigSchema } from "./config-schema-core.js";
export const MattermostChannelConfigSchema = buildChannelConfigSchema(MattermostConfigSchema);

View File

@@ -0,0 +1,7 @@
{
"name": "@openclaw/media-understanding-core",
"version": "2026.3.26",
"private": true,
"description": "OpenClaw media understanding runtime package",
"type": "module"
}

View File

@@ -0,0 +1,9 @@
export {
describeImageFile,
describeImageFileWithModel,
describeVideoFile,
runMediaUnderstandingFile,
transcribeAudioFile,
type RunMediaUnderstandingFileParams,
type RunMediaUnderstandingFileResult,
} from "./src/runtime.js";

View File

@@ -0,0 +1,147 @@
import fs from "node:fs/promises";
import path from "node:path";
import type { OpenClawConfig } from "openclaw/plugin-sdk/core";
import {
buildProviderRegistry,
createMediaAttachmentCache,
normalizeMediaAttachments,
normalizeMediaProviderId,
runCapability,
type ActiveMediaModel,
} from "openclaw/plugin-sdk/media-runtime";
type MediaUnderstandingCapability = "image" | "audio" | "video";
type MediaUnderstandingOutput = Awaited<ReturnType<typeof runCapability>>["outputs"][number];
const KIND_BY_CAPABILITY: Record<MediaUnderstandingCapability, MediaUnderstandingOutput["kind"]> = {
audio: "audio.transcription",
image: "image.description",
video: "video.description",
};
export type RunMediaUnderstandingFileParams = {
capability: MediaUnderstandingCapability;
filePath: string;
cfg: OpenClawConfig;
agentDir?: string;
mime?: string;
activeModel?: ActiveMediaModel;
};
export type RunMediaUnderstandingFileResult = {
text: string | undefined;
provider?: string;
model?: string;
output?: MediaUnderstandingOutput;
};
function buildFileContext(params: { filePath: string; mime?: string }) {
return {
MediaPath: params.filePath,
MediaType: params.mime,
};
}
export async function runMediaUnderstandingFile(
params: RunMediaUnderstandingFileParams,
): Promise<RunMediaUnderstandingFileResult> {
const ctx = buildFileContext(params);
const attachments = normalizeMediaAttachments(ctx);
if (attachments.length === 0) {
return { text: undefined };
}
const providerRegistry = buildProviderRegistry(undefined, params.cfg);
const cache = createMediaAttachmentCache(attachments, {
localPathRoots: [path.dirname(params.filePath)],
});
try {
const result = await runCapability({
capability: params.capability,
cfg: params.cfg,
ctx,
attachments: cache,
media: attachments,
agentDir: params.agentDir,
providerRegistry,
config: params.cfg.tools?.media?.[params.capability],
activeModel: params.activeModel,
});
const output = result.outputs.find(
(entry) => entry.kind === KIND_BY_CAPABILITY[params.capability],
);
const text = output?.text?.trim();
return {
text: text || undefined,
provider: output?.provider,
model: output?.model,
output,
};
} finally {
await cache.cleanup();
}
}
export async function describeImageFile(params: {
filePath: string;
cfg: OpenClawConfig;
agentDir?: string;
mime?: string;
activeModel?: ActiveMediaModel;
}): Promise<RunMediaUnderstandingFileResult> {
return await runMediaUnderstandingFile({ ...params, capability: "image" });
}
export async function describeImageFileWithModel(params: {
filePath: string;
cfg: OpenClawConfig;
agentDir?: string;
mime?: string;
provider: string;
model: string;
prompt: string;
maxTokens?: number;
timeoutMs?: number;
}) {
const timeoutMs = params.timeoutMs ?? 30_000;
const providerRegistry = buildProviderRegistry(undefined, params.cfg);
const provider = providerRegistry.get(normalizeMediaProviderId(params.provider));
if (!provider?.describeImage) {
throw new Error(`Provider does not support image analysis: ${params.provider}`);
}
const buffer = await fs.readFile(params.filePath);
return await provider.describeImage({
buffer,
fileName: path.basename(params.filePath),
mime: params.mime,
provider: params.provider,
model: params.model,
prompt: params.prompt,
maxTokens: params.maxTokens,
timeoutMs,
cfg: params.cfg,
agentDir: params.agentDir ?? "",
});
}
export async function describeVideoFile(params: {
filePath: string;
cfg: OpenClawConfig;
agentDir?: string;
mime?: string;
activeModel?: ActiveMediaModel;
}): Promise<RunMediaUnderstandingFileResult> {
return await runMediaUnderstandingFile({ ...params, capability: "video" });
}
export async function transcribeAudioFile(params: {
filePath: string;
cfg: OpenClawConfig;
agentDir?: string;
mime?: string;
activeModel?: ActiveMediaModel;
}): Promise<{ text: string | undefined }> {
const result = await runMediaUnderstandingFile({ ...params, capability: "audio" });
return { text: result.text };
}

View File

@@ -0,0 +1 @@
export { msteamsPlugin } from "./src/channel.js";

View File

@@ -1,17 +1,33 @@
import { requireChannelOpenAllowFrom } from "openclaw/plugin-sdk/extension-shared";
import { z } from "openclaw/plugin-sdk/zod";
import {
ReplyRuntimeConfigSchemaShape,
ToolPolicySchema,
} from "openclaw/plugin-sdk/agent-config-primitives";
import {
BlockStreamingCoalesceSchema,
DmConfigSchema,
DmPolicySchema,
GroupPolicySchema,
MarkdownConfigSchema,
ReplyRuntimeConfigSchemaShape,
ToolPolicySchema,
requireOpenAllowFrom,
} from "../runtime-api.js";
} from "openclaw/plugin-sdk/channel-config-primitives";
import { z } from "openclaw/plugin-sdk/zod";
import { buildSecretInputSchema } from "./secret-input.js";
function requireNextcloudTalkOpenAllowFrom(params: {
policy?: string;
allowFrom?: string[];
ctx: z.RefinementCtx;
}) {
requireOpenAllowFrom({
policy: params.policy,
allowFrom: params.allowFrom,
ctx: params.ctx,
path: ["allowFrom"],
message:
'channels.nextcloud-talk.dmPolicy="open" requires channels.nextcloud-talk.allowFrom to include "*"',
});
}
export const NextcloudTalkRoomSchema = z
.object({
requireMention: z.boolean().optional(),
@@ -51,12 +67,10 @@ export const NextcloudTalkAccountSchemaBase = z
export const NextcloudTalkAccountSchema = NextcloudTalkAccountSchemaBase.superRefine(
(value, ctx) => {
requireChannelOpenAllowFrom({
channel: "nextcloud-talk",
requireNextcloudTalkOpenAllowFrom({
policy: value.dmPolicy,
allowFrom: value.allowFrom,
ctx,
requireOpenAllowFrom,
});
},
);
@@ -65,11 +79,9 @@ export const NextcloudTalkConfigSchema = NextcloudTalkAccountSchemaBase.extend({
accounts: z.record(z.string(), NextcloudTalkAccountSchema.optional()).optional(),
defaultAccount: z.string().optional(),
}).superRefine((value, ctx) => {
requireChannelOpenAllowFrom({
channel: "nextcloud-talk",
requireNextcloudTalkOpenAllowFrom({
policy: value.dmPolicy,
allowFrom: value.allowFrom,
ctx,
requireOpenAllowFrom,
});
});

View File

@@ -1,6 +1,10 @@
import { AllowFromListSchema, DmPolicySchema } from "openclaw/plugin-sdk/channel-config-schema";
import {
AllowFromListSchema,
buildChannelConfigSchema,
DmPolicySchema,
MarkdownConfigSchema,
} from "openclaw/plugin-sdk/channel-config-primitives";
import { z } from "openclaw/plugin-sdk/zod";
import { MarkdownConfigSchema, buildChannelConfigSchema } from "../api.js";
/**
* Validates https:// URLs only (no javascript:, data:, file:, etc.)

View File

@@ -0,0 +1 @@
export { nostrPlugin } from "./src/channel.js";

View File

@@ -1 +1,2 @@
export { buildOpenAICodexCliBackend } from "./cli-backend.js";
export { buildOpenAISpeechProvider } from "./speech-provider.js";

View File

@@ -1 +1,17 @@
export { requireChannelOpenAllowFrom } from "openclaw/plugin-sdk/extension-shared";
import { requireOpenAllowFrom } from "openclaw/plugin-sdk/channel-config-primitives";
import type { z } from "openclaw/plugin-sdk/zod";
export function requireChannelOpenAllowFrom(params: {
channel: string;
policy?: string;
allowFrom?: Array<string | number>;
ctx: z.RefinementCtx;
}) {
requireOpenAllowFrom({
policy: params.policy,
allowFrom: params.allowFrom,
ctx: params.ctx,
path: ["allowFrom"],
message: `channels.${params.channel}.dmPolicy="open" requires channels.${params.channel}.allowFrom to include "*"`,
});
}

View File

@@ -0,0 +1,6 @@
export {
removeReactionSignal,
sendReactionSignal,
type SignalReactionOpts,
type SignalReactionResult,
} from "./src/send-reactions.js";

View File

@@ -4,7 +4,7 @@ import {
resolveMergedAccountConfig,
type OpenClawConfig,
} from "openclaw/plugin-sdk/account-resolution";
import type { SignalAccountConfig } from "openclaw/plugin-sdk/signal";
import type { SignalAccountConfig } from "openclaw/plugin-sdk/signal-core";
export type ResolvedSignalAccount = {
accountId: string;

View File

@@ -4,9 +4,9 @@ import type {
ChannelMessageActionAdapter,
ChannelMessageActionName,
} from "openclaw/plugin-sdk/channel-contract";
import { removeReactionSignal, sendReactionSignal } from "../reaction-runtime-api.js";
import { listEnabledSignalAccounts, resolveSignalAccount } from "./accounts.js";
import { resolveSignalReactionLevel } from "./reaction-level.js";
import { removeReactionSignal, sendReactionSignal } from "./send-reactions.js";
const providerId = "signal";
const GROUP_PREFIX = "group:";

View File

@@ -1,5 +1,7 @@
export type { ResolvedSlackAccount } from "./src/accounts.js";
export type { SlackMessageEvent } from "./src/types.js";
export { slackPlugin } from "./src/channel.js";
export { setSlackRuntime } from "./src/runtime.js";
export { createSlackActions } from "./src/channel-actions.js";
export { prepareSlackMessage } from "./src/monitor/message-handler/prepare.js";
export { createInboundSlackTestContext } from "./src/monitor/message-handler/prepare.test-helpers.js";

View File

@@ -0,0 +1 @@
export * from "openclaw/plugin-sdk/speech-core";

View File

@@ -0,0 +1,7 @@
{
"name": "@openclaw/speech-core",
"version": "2026.3.26",
"private": true,
"description": "OpenClaw speech runtime package",
"type": "module"
}

View File

@@ -0,0 +1,33 @@
export {
buildTtsSystemPromptHint,
getLastTtsAttempt,
getResolvedSpeechProviderConfig,
getTtsMaxLength,
getTtsProvider,
isSummarizationEnabled,
isTtsEnabled,
isTtsProviderConfigured,
listSpeechVoices,
maybeApplyTtsToPayload,
resolveTtsAutoMode,
resolveTtsConfig,
resolveTtsPrefsPath,
resolveTtsProviderOrder,
setLastTtsAttempt,
setSummarizationEnabled,
setTtsAutoMode,
setTtsEnabled,
setTtsMaxLength,
setTtsProvider,
synthesizeSpeech,
textToSpeech,
textToSpeechTelephony,
_test,
type ResolvedTtsConfig,
type ResolvedTtsModelOverrides,
type TtsDirectiveOverrides,
type TtsDirectiveParseResult,
type TtsResult,
type TtsSynthesisResult,
type TtsTelephonyResult,
} from "./src/tts.js";

View File

@@ -0,0 +1,849 @@
import { randomBytes } from "node:crypto";
import {
existsSync,
mkdirSync,
readFileSync,
writeFileSync,
mkdtempSync,
renameSync,
unlinkSync,
} from "node:fs";
import path from "node:path";
import { normalizeChannelId, type ChannelId } from "openclaw/plugin-sdk/channel-runtime";
import type {
OpenClawConfig,
TtsAutoMode,
TtsConfig,
TtsMode,
TtsModelOverrideConfig,
TtsProvider,
} from "openclaw/plugin-sdk/config-runtime";
import { resolveSendableOutboundReplyParts } from "openclaw/plugin-sdk/reply-payload";
import type { ReplyPayload } from "openclaw/plugin-sdk/reply-runtime";
import { logVerbose } from "openclaw/plugin-sdk/runtime-env";
import { resolvePreferredOpenClawTmpDir } from "openclaw/plugin-sdk/sandbox";
import { CONFIG_DIR, resolveUserPath, stripMarkdown } from "openclaw/plugin-sdk/text-runtime";
import {
canonicalizeSpeechProviderId,
getSpeechProvider,
listSpeechProviders,
normalizeTtsAutoMode,
parseTtsDirectives,
scheduleCleanup,
summarizeText,
type SpeechModelOverridePolicy,
type SpeechProviderConfig,
type SpeechVoiceOption,
type TtsDirectiveOverrides,
type TtsDirectiveParseResult,
} from "../api.js";
export type { TtsDirectiveOverrides, TtsDirectiveParseResult };
const DEFAULT_TIMEOUT_MS = 30_000;
const DEFAULT_TTS_MAX_LENGTH = 1500;
const DEFAULT_TTS_SUMMARIZE = true;
const DEFAULT_MAX_TEXT_LENGTH = 4096;
export type ResolvedTtsConfig = {
auto: TtsAutoMode;
mode: TtsMode;
provider: TtsProvider;
providerSource: "config" | "default";
summaryModel?: string;
modelOverrides: ResolvedTtsModelOverrides;
providerConfigs: Record<string, SpeechProviderConfig>;
prefsPath?: string;
maxTextLength: number;
timeoutMs: number;
};
type TtsUserPrefs = {
tts?: {
auto?: TtsAutoMode;
enabled?: boolean;
provider?: TtsProvider;
maxLength?: number;
summarize?: boolean;
};
};
export type ResolvedTtsModelOverrides = SpeechModelOverridePolicy;
export type TtsResult = {
success: boolean;
audioPath?: string;
error?: string;
latencyMs?: number;
provider?: string;
outputFormat?: string;
voiceCompatible?: boolean;
};
export type TtsSynthesisResult = {
success: boolean;
audioBuffer?: Buffer;
error?: string;
latencyMs?: number;
provider?: string;
outputFormat?: string;
voiceCompatible?: boolean;
fileExtension?: string;
};
export type TtsTelephonyResult = {
success: boolean;
audioBuffer?: Buffer;
error?: string;
latencyMs?: number;
provider?: string;
outputFormat?: string;
sampleRate?: number;
};
type TtsStatusEntry = {
timestamp: number;
success: boolean;
textLength: number;
summarized: boolean;
provider?: string;
latencyMs?: number;
error?: string;
};
let lastTtsAttempt: TtsStatusEntry | undefined;
function resolveModelOverridePolicy(
overrides: TtsModelOverrideConfig | undefined,
): ResolvedTtsModelOverrides {
const enabled = overrides?.enabled ?? true;
if (!enabled) {
return {
enabled: false,
allowText: false,
allowProvider: false,
allowVoice: false,
allowModelId: false,
allowVoiceSettings: false,
allowNormalization: false,
allowSeed: false,
};
}
const allow = (value: boolean | undefined, defaultValue = true) => value ?? defaultValue;
return {
enabled: true,
allowText: allow(overrides?.allowText),
allowProvider: allow(overrides?.allowProvider, false),
allowVoice: allow(overrides?.allowVoice),
allowModelId: allow(overrides?.allowModelId),
allowVoiceSettings: allow(overrides?.allowVoiceSettings),
allowNormalization: allow(overrides?.allowNormalization),
allowSeed: allow(overrides?.allowSeed),
};
}
function sortSpeechProvidersForAutoSelection(cfg?: OpenClawConfig) {
return listSpeechProviders(cfg).toSorted((left, right) => {
const leftOrder = left.autoSelectOrder ?? Number.MAX_SAFE_INTEGER;
const rightOrder = right.autoSelectOrder ?? Number.MAX_SAFE_INTEGER;
if (leftOrder !== rightOrder) {
return leftOrder - rightOrder;
}
return left.id.localeCompare(right.id);
});
}
function resolveRegistryDefaultSpeechProviderId(cfg?: OpenClawConfig): TtsProvider {
return sortSpeechProvidersForAutoSelection(cfg)[0]?.id ?? "";
}
function asProviderConfig(value: unknown): SpeechProviderConfig {
return typeof value === "object" && value !== null && !Array.isArray(value)
? (value as SpeechProviderConfig)
: {};
}
function asProviderConfigMap(value: unknown): Record<string, unknown> {
return typeof value === "object" && value !== null && !Array.isArray(value)
? (value as Record<string, unknown>)
: {};
}
function resolveSpeechProviderConfigs(
raw: TtsConfig,
cfg: OpenClawConfig,
timeoutMs: number,
): Record<string, SpeechProviderConfig> {
const providerConfigs: Record<string, SpeechProviderConfig> = {};
const rawProviders = asProviderConfigMap(raw.providers);
for (const provider of listSpeechProviders(cfg)) {
providerConfigs[provider.id] =
provider.resolveConfig?.({
cfg,
rawConfig: {
...(raw as Record<string, unknown>),
providers: rawProviders,
},
timeoutMs,
}) ??
asProviderConfig(rawProviders[provider.id] ?? (raw as Record<string, unknown>)[provider.id]);
}
return providerConfigs;
}
export function getResolvedSpeechProviderConfig(
config: ResolvedTtsConfig,
providerId: string,
cfg?: OpenClawConfig,
): SpeechProviderConfig {
const canonical =
canonicalizeSpeechProviderId(providerId, cfg) ?? providerId.trim().toLowerCase();
return config.providerConfigs[canonical] ?? {};
}
export function resolveTtsConfig(cfg: OpenClawConfig): ResolvedTtsConfig {
const raw: TtsConfig = cfg.messages?.tts ?? {};
const providerSource = raw.provider ? "config" : "default";
const timeoutMs = raw.timeoutMs ?? DEFAULT_TIMEOUT_MS;
const auto = normalizeTtsAutoMode(raw.auto) ?? (raw.enabled ? "always" : "off");
return {
auto,
mode: raw.mode ?? "final",
provider:
canonicalizeSpeechProviderId(raw.provider, cfg) ??
resolveRegistryDefaultSpeechProviderId(cfg),
providerSource,
summaryModel: raw.summaryModel?.trim() || undefined,
modelOverrides: resolveModelOverridePolicy(raw.modelOverrides),
providerConfigs: resolveSpeechProviderConfigs(raw, cfg, timeoutMs),
prefsPath: raw.prefsPath,
maxTextLength: raw.maxTextLength ?? DEFAULT_MAX_TEXT_LENGTH,
timeoutMs,
};
}
export function resolveTtsPrefsPath(config: ResolvedTtsConfig): string {
if (config.prefsPath?.trim()) {
return resolveUserPath(config.prefsPath.trim());
}
const envPath = process.env.OPENCLAW_TTS_PREFS?.trim();
if (envPath) {
return resolveUserPath(envPath);
}
return path.join(CONFIG_DIR, "settings", "tts.json");
}
function resolveTtsAutoModeFromPrefs(prefs: TtsUserPrefs): TtsAutoMode | undefined {
const auto = normalizeTtsAutoMode(prefs.tts?.auto);
if (auto) {
return auto;
}
if (typeof prefs.tts?.enabled === "boolean") {
return prefs.tts.enabled ? "always" : "off";
}
return undefined;
}
export function resolveTtsAutoMode(params: {
config: ResolvedTtsConfig;
prefsPath: string;
sessionAuto?: string;
}): TtsAutoMode {
const sessionAuto = normalizeTtsAutoMode(params.sessionAuto);
if (sessionAuto) {
return sessionAuto;
}
const prefsAuto = resolveTtsAutoModeFromPrefs(readPrefs(params.prefsPath));
if (prefsAuto) {
return prefsAuto;
}
return params.config.auto;
}
export function buildTtsSystemPromptHint(cfg: OpenClawConfig): string | undefined {
const config = resolveTtsConfig(cfg);
const prefsPath = resolveTtsPrefsPath(config);
const autoMode = resolveTtsAutoMode({ config, prefsPath });
if (autoMode === "off") {
return undefined;
}
const maxLength = getTtsMaxLength(prefsPath);
const summarize = isSummarizationEnabled(prefsPath) ? "on" : "off";
const autoHint =
autoMode === "inbound"
? "Only use TTS when the user's last message includes audio/voice."
: autoMode === "tagged"
? "Only use TTS when you include [[tts]] or [[tts:text]] tags."
: undefined;
return [
"Voice (TTS) is enabled.",
autoHint,
`Keep spoken text ≤${maxLength} chars to avoid auto-summary (summary ${summarize}).`,
"Use [[tts:...]] and optional [[tts:text]]...[[/tts:text]] to control voice/expressiveness.",
]
.filter(Boolean)
.join("\n");
}
function readPrefs(prefsPath: string): TtsUserPrefs {
try {
if (!existsSync(prefsPath)) {
return {};
}
return JSON.parse(readFileSync(prefsPath, "utf8")) as TtsUserPrefs;
} catch {
return {};
}
}
function atomicWriteFileSync(filePath: string, content: string): void {
const tmpPath = `${filePath}.tmp.${Date.now()}.${randomBytes(8).toString("hex")}`;
writeFileSync(tmpPath, content, { mode: 0o600 });
try {
renameSync(tmpPath, filePath);
} catch (err) {
try {
unlinkSync(tmpPath);
} catch {
// ignore
}
throw err;
}
}
function updatePrefs(prefsPath: string, update: (prefs: TtsUserPrefs) => void): void {
const prefs = readPrefs(prefsPath);
update(prefs);
mkdirSync(path.dirname(prefsPath), { recursive: true });
atomicWriteFileSync(prefsPath, JSON.stringify(prefs, null, 2));
}
export function isTtsEnabled(
config: ResolvedTtsConfig,
prefsPath: string,
sessionAuto?: string,
): boolean {
return resolveTtsAutoMode({ config, prefsPath, sessionAuto }) !== "off";
}
export function setTtsAutoMode(prefsPath: string, mode: TtsAutoMode): void {
updatePrefs(prefsPath, (prefs) => {
const next = { ...prefs.tts };
delete next.enabled;
next.auto = mode;
prefs.tts = next;
});
}
export function setTtsEnabled(prefsPath: string, enabled: boolean): void {
setTtsAutoMode(prefsPath, enabled ? "always" : "off");
}
export function getTtsProvider(config: ResolvedTtsConfig, prefsPath: string): TtsProvider {
const prefs = readPrefs(prefsPath);
const prefsProvider = canonicalizeSpeechProviderId(prefs.tts?.provider);
if (prefsProvider) {
return prefsProvider;
}
if (config.providerSource === "config") {
return canonicalizeSpeechProviderId(config.provider) ?? config.provider;
}
for (const provider of sortSpeechProvidersForAutoSelection()) {
if (
provider.isConfigured({
providerConfig: config.providerConfigs[provider.id] ?? {},
timeoutMs: config.timeoutMs,
})
) {
return provider.id;
}
}
return config.provider;
}
export function setTtsProvider(prefsPath: string, provider: TtsProvider): void {
updatePrefs(prefsPath, (prefs) => {
prefs.tts = { ...prefs.tts, provider: canonicalizeSpeechProviderId(provider) ?? provider };
});
}
export function getTtsMaxLength(prefsPath: string): number {
const prefs = readPrefs(prefsPath);
return prefs.tts?.maxLength ?? DEFAULT_TTS_MAX_LENGTH;
}
export function setTtsMaxLength(prefsPath: string, maxLength: number): void {
updatePrefs(prefsPath, (prefs) => {
prefs.tts = { ...prefs.tts, maxLength };
});
}
export function isSummarizationEnabled(prefsPath: string): boolean {
const prefs = readPrefs(prefsPath);
return prefs.tts?.summarize ?? DEFAULT_TTS_SUMMARIZE;
}
export function setSummarizationEnabled(prefsPath: string, enabled: boolean): void {
updatePrefs(prefsPath, (prefs) => {
prefs.tts = { ...prefs.tts, summarize: enabled };
});
}
export function getLastTtsAttempt(): TtsStatusEntry | undefined {
return lastTtsAttempt;
}
export function setLastTtsAttempt(entry: TtsStatusEntry | undefined): void {
lastTtsAttempt = entry;
}
const OPUS_CHANNELS = new Set(["telegram", "feishu", "whatsapp", "matrix"]);
function resolveChannelId(channel: string | undefined): ChannelId | null {
return channel ? normalizeChannelId(channel) : null;
}
export function resolveTtsProviderOrder(primary: TtsProvider, cfg?: OpenClawConfig): TtsProvider[] {
const normalizedPrimary = canonicalizeSpeechProviderId(primary, cfg) ?? primary;
const ordered = new Set<TtsProvider>([normalizedPrimary]);
for (const provider of sortSpeechProvidersForAutoSelection(cfg)) {
const normalized = provider.id;
if (normalized !== normalizedPrimary) {
ordered.add(normalized);
}
}
return [...ordered];
}
export function isTtsProviderConfigured(
config: ResolvedTtsConfig,
provider: TtsProvider,
cfg?: OpenClawConfig,
): boolean {
const resolvedProvider = getSpeechProvider(provider, cfg);
if (!resolvedProvider) {
return false;
}
return (
resolvedProvider.isConfigured({
cfg,
providerConfig: getResolvedSpeechProviderConfig(config, resolvedProvider.id, cfg),
timeoutMs: config.timeoutMs,
}) ?? false
);
}
function formatTtsProviderError(provider: TtsProvider, err: unknown): string {
const error = err instanceof Error ? err : new Error(String(err));
if (error.name === "AbortError") {
return `${provider}: request timed out`;
}
return `${provider}: ${error.message}`;
}
function buildTtsFailureResult(errors: string[]): { success: false; error: string } {
return {
success: false,
error: `TTS conversion failed: ${errors.join("; ") || "no providers available"}`,
};
}
function resolveReadySpeechProvider(params: {
provider: TtsProvider;
cfg: OpenClawConfig;
config: ResolvedTtsConfig;
errors: string[];
requireTelephony?: boolean;
}): NonNullable<ReturnType<typeof getSpeechProvider>> | null {
const resolvedProvider = getSpeechProvider(params.provider, params.cfg);
if (!resolvedProvider) {
params.errors.push(`${params.provider}: no provider registered`);
return null;
}
const providerConfig = getResolvedSpeechProviderConfig(
params.config,
resolvedProvider.id,
params.cfg,
);
if (
!resolvedProvider.isConfigured({
cfg: params.cfg,
providerConfig,
timeoutMs: params.config.timeoutMs,
})
) {
params.errors.push(`${params.provider}: not configured`);
return null;
}
if (params.requireTelephony && !resolvedProvider.synthesizeTelephony) {
params.errors.push(`${params.provider}: unsupported for telephony`);
return null;
}
return resolvedProvider;
}
function resolveTtsRequestSetup(params: {
text: string;
cfg: OpenClawConfig;
prefsPath?: string;
providerOverride?: TtsProvider;
disableFallback?: boolean;
}):
| {
config: ResolvedTtsConfig;
providers: TtsProvider[];
}
| {
error: string;
} {
const config = resolveTtsConfig(params.cfg);
const prefsPath = params.prefsPath ?? resolveTtsPrefsPath(config);
if (params.text.length > config.maxTextLength) {
return {
error: `Text too long (${params.text.length} chars, max ${config.maxTextLength})`,
};
}
const userProvider = getTtsProvider(config, prefsPath);
const provider =
canonicalizeSpeechProviderId(params.providerOverride, params.cfg) ?? userProvider;
return {
config,
providers: params.disableFallback ? [provider] : resolveTtsProviderOrder(provider, params.cfg),
};
}
export async function textToSpeech(params: {
text: string;
cfg: OpenClawConfig;
prefsPath?: string;
channel?: string;
overrides?: TtsDirectiveOverrides;
disableFallback?: boolean;
}): Promise<TtsResult> {
const synthesis = await synthesizeSpeech(params);
if (!synthesis.success || !synthesis.audioBuffer || !synthesis.fileExtension) {
return buildTtsFailureResult([synthesis.error ?? "TTS conversion failed"]);
}
const tempRoot = resolvePreferredOpenClawTmpDir();
mkdirSync(tempRoot, { recursive: true, mode: 0o700 });
const tempDir = mkdtempSync(path.join(tempRoot, "tts-"));
const audioPath = path.join(tempDir, `voice-${Date.now()}${synthesis.fileExtension}`);
writeFileSync(audioPath, synthesis.audioBuffer);
scheduleCleanup(tempDir);
return {
success: true,
audioPath,
latencyMs: synthesis.latencyMs,
provider: synthesis.provider,
outputFormat: synthesis.outputFormat,
voiceCompatible: synthesis.voiceCompatible,
};
}
export async function synthesizeSpeech(params: {
text: string;
cfg: OpenClawConfig;
prefsPath?: string;
channel?: string;
overrides?: TtsDirectiveOverrides;
disableFallback?: boolean;
}): Promise<TtsSynthesisResult> {
const setup = resolveTtsRequestSetup({
text: params.text,
cfg: params.cfg,
prefsPath: params.prefsPath,
providerOverride: params.overrides?.provider,
disableFallback: params.disableFallback,
});
if ("error" in setup) {
return { success: false, error: setup.error };
}
const { config, providers } = setup;
const channelId = resolveChannelId(params.channel);
const target = channelId && OPUS_CHANNELS.has(channelId) ? "voice-note" : "audio-file";
const errors: string[] = [];
for (const provider of providers) {
const providerStart = Date.now();
try {
const resolvedProvider = resolveReadySpeechProvider({
provider,
cfg: params.cfg,
config,
errors,
});
if (!resolvedProvider) {
continue;
}
const synthesis = await resolvedProvider.synthesize({
text: params.text,
cfg: params.cfg,
providerConfig: getResolvedSpeechProviderConfig(config, resolvedProvider.id, params.cfg),
target,
providerOverrides: params.overrides?.providerOverrides?.[resolvedProvider.id],
timeoutMs: config.timeoutMs,
});
return {
success: true,
audioBuffer: synthesis.audioBuffer,
latencyMs: Date.now() - providerStart,
provider,
outputFormat: synthesis.outputFormat,
voiceCompatible: synthesis.voiceCompatible,
fileExtension: synthesis.fileExtension,
};
} catch (err) {
errors.push(formatTtsProviderError(provider, err));
}
}
return buildTtsFailureResult(errors);
}
export async function textToSpeechTelephony(params: {
text: string;
cfg: OpenClawConfig;
prefsPath?: string;
}): Promise<TtsTelephonyResult> {
const setup = resolveTtsRequestSetup({
text: params.text,
cfg: params.cfg,
prefsPath: params.prefsPath,
});
if ("error" in setup) {
return { success: false, error: setup.error };
}
const { config, providers } = setup;
const errors: string[] = [];
for (const provider of providers) {
const providerStart = Date.now();
try {
const resolvedProvider = resolveReadySpeechProvider({
provider,
cfg: params.cfg,
config,
errors,
requireTelephony: true,
});
if (!resolvedProvider?.synthesizeTelephony) {
continue;
}
const synthesis = await resolvedProvider.synthesizeTelephony({
text: params.text,
cfg: params.cfg,
providerConfig: getResolvedSpeechProviderConfig(config, resolvedProvider.id, params.cfg),
timeoutMs: config.timeoutMs,
});
return {
success: true,
audioBuffer: synthesis.audioBuffer,
latencyMs: Date.now() - providerStart,
provider,
outputFormat: synthesis.outputFormat,
sampleRate: synthesis.sampleRate,
};
} catch (err) {
errors.push(formatTtsProviderError(provider, err));
}
}
return buildTtsFailureResult(errors);
}
export async function listSpeechVoices(params: {
provider: string;
cfg?: OpenClawConfig;
config?: ResolvedTtsConfig;
apiKey?: string;
baseUrl?: string;
}): Promise<SpeechVoiceOption[]> {
const provider = canonicalizeSpeechProviderId(params.provider, params.cfg);
if (!provider) {
throw new Error("speech provider id is required");
}
const config = params.config ?? (params.cfg ? resolveTtsConfig(params.cfg) : undefined);
if (!config) {
throw new Error(`speech provider ${provider} requires cfg or resolved config`);
}
const resolvedProvider = getSpeechProvider(provider, params.cfg);
if (!resolvedProvider) {
throw new Error(`speech provider ${provider} is not registered`);
}
if (!resolvedProvider.listVoices) {
throw new Error(`speech provider ${provider} does not support voice listing`);
}
return await resolvedProvider.listVoices({
cfg: params.cfg,
providerConfig: getResolvedSpeechProviderConfig(config, resolvedProvider.id, params.cfg),
apiKey: params.apiKey,
baseUrl: params.baseUrl,
});
}
export async function maybeApplyTtsToPayload(params: {
payload: ReplyPayload;
cfg: OpenClawConfig;
channel?: string;
kind?: "tool" | "block" | "final";
inboundAudio?: boolean;
ttsAuto?: string;
}): Promise<ReplyPayload> {
if (params.payload.isCompactionNotice) {
return params.payload;
}
const config = resolveTtsConfig(params.cfg);
const prefsPath = resolveTtsPrefsPath(config);
const autoMode = resolveTtsAutoMode({
config,
prefsPath,
sessionAuto: params.ttsAuto,
});
if (autoMode === "off") {
return params.payload;
}
const reply = resolveSendableOutboundReplyParts(params.payload);
const text = reply.text;
const directives = parseTtsDirectives(text, config.modelOverrides, {
cfg: params.cfg,
providerConfigs: config.providerConfigs,
});
if (directives.warnings.length > 0) {
logVerbose(`TTS: ignored directive overrides (${directives.warnings.join("; ")})`);
}
const cleanedText = directives.cleanedText;
const trimmedCleaned = cleanedText.trim();
const visibleText = trimmedCleaned.length > 0 ? trimmedCleaned : "";
const ttsText = directives.ttsText?.trim() || visibleText;
const nextPayload =
visibleText === text.trim()
? params.payload
: {
...params.payload,
text: visibleText.length > 0 ? visibleText : undefined,
};
if (autoMode === "tagged" && !directives.hasDirective) {
return nextPayload;
}
if (autoMode === "inbound" && params.inboundAudio !== true) {
return nextPayload;
}
const mode = config.mode ?? "final";
if (mode === "final" && params.kind && params.kind !== "final") {
return nextPayload;
}
if (!ttsText.trim()) {
return nextPayload;
}
if (reply.hasMedia) {
return nextPayload;
}
if (text.includes("MEDIA:")) {
return nextPayload;
}
if (ttsText.trim().length < 10) {
return nextPayload;
}
const maxLength = getTtsMaxLength(prefsPath);
let textForAudio = ttsText.trim();
let wasSummarized = false;
if (textForAudio.length > maxLength) {
if (!isSummarizationEnabled(prefsPath)) {
logVerbose(
`TTS: truncating long text (${textForAudio.length} > ${maxLength}), summarization disabled.`,
);
textForAudio = `${textForAudio.slice(0, maxLength - 3)}...`;
} else {
try {
const summary = await summarizeText({
text: textForAudio,
targetLength: maxLength,
cfg: params.cfg,
config,
timeoutMs: config.timeoutMs,
});
textForAudio = summary.summary;
wasSummarized = true;
if (textForAudio.length > config.maxTextLength) {
logVerbose(
`TTS: summary exceeded hard limit (${textForAudio.length} > ${config.maxTextLength}); truncating.`,
);
textForAudio = `${textForAudio.slice(0, config.maxTextLength - 3)}...`;
}
} catch (err) {
const error = err as Error;
logVerbose(`TTS: summarization failed, truncating instead: ${error.message}`);
textForAudio = `${textForAudio.slice(0, maxLength - 3)}...`;
}
}
}
textForAudio = stripMarkdown(textForAudio).trim();
if (textForAudio.length < 10) {
return nextPayload;
}
const ttsStart = Date.now();
const result = await textToSpeech({
text: textForAudio,
cfg: params.cfg,
prefsPath,
channel: params.channel,
overrides: directives.overrides,
});
if (result.success && result.audioPath) {
lastTtsAttempt = {
timestamp: Date.now(),
success: true,
textLength: text.length,
summarized: wasSummarized,
provider: result.provider,
latencyMs: result.latencyMs,
};
const channelId = resolveChannelId(params.channel);
const shouldVoice =
channelId !== null && OPUS_CHANNELS.has(channelId) && result.voiceCompatible === true;
return {
...nextPayload,
mediaUrl: result.audioPath,
audioAsVoice: shouldVoice || params.payload.audioAsVoice,
};
}
lastTtsAttempt = {
timestamp: Date.now(),
success: false,
textLength: text.length,
summarized: wasSummarized,
error: result.error,
};
const latency = Date.now() - ttsStart;
logVerbose(`TTS: conversion failed after ${latency}ms (${result.error ?? "unknown"}).`);
return nextPayload;
}
export const _test = {
parseTtsDirectives,
resolveModelOverridePolicy,
summarizeText,
getResolvedSpeechProviderConfig,
};

View File

@@ -1,8 +1,10 @@
export { buildTelegramMessageContextForTest } from "./src/bot-message-context.test-harness.js";
export { handleTelegramAction } from "./src/action-runtime.js";
export { telegramMessageActionRuntime } from "./src/channel-actions.js";
export { telegramPlugin } from "./src/channel.js";
export { listTelegramAccountIds, resolveTelegramAccount } from "./src/accounts.js";
export { resolveTelegramFetch } from "./src/fetch.js";
export { makeProxyFetch } from "./src/proxy.js";
export { telegramOutbound } from "./src/outbound-adapter.js";
export { setTelegramRuntime } from "./src/runtime.js";
export { sendMessageTelegram, sendPollTelegram, type TelegramApiOverride } from "./src/send.js";

View File

@@ -0,0 +1 @@
export { tlonPlugin } from "./src/channel.js";

View File

@@ -1,5 +1,5 @@
import { MarkdownConfigSchema } from "openclaw/plugin-sdk/channel-config-primitives";
import { z } from "openclaw/plugin-sdk/zod";
import { MarkdownConfigSchema } from "../runtime-api.js";
/**
* Twitch user roles that can be allowed to interact with the bot

View File

@@ -1,3 +1,5 @@
export { whatsappPlugin } from "./src/channel.js";
export { setWhatsAppRuntime } from "./src/runtime.js";
export { whatsappOutbound } from "./src/outbound-adapter.js";
export { deliverWebReply } from "./src/auto-reply/deliver-reply.js";
export {

View File

@@ -3,9 +3,9 @@ import {
buildCatchallMultiAccountChannelSchema,
DmPolicySchema,
GroupPolicySchema,
} from "openclaw/plugin-sdk/channel-config-schema";
MarkdownConfigSchema,
} from "openclaw/plugin-sdk/channel-config-primitives";
import { z } from "openclaw/plugin-sdk/zod";
import { MarkdownConfigSchema } from "./runtime-api.js";
import { buildSecretInputSchema } from "./secret-input.js";
const zaloAccountSchema = z.object({

View File

@@ -1,11 +1,12 @@
import { ToolPolicySchema } from "openclaw/plugin-sdk/agent-config-primitives";
import {
AllowFromListSchema,
buildCatchallMultiAccountChannelSchema,
DmPolicySchema,
GroupPolicySchema,
} from "openclaw/plugin-sdk/channel-config-schema";
MarkdownConfigSchema,
} from "openclaw/plugin-sdk/channel-config-primitives";
import { z } from "openclaw/plugin-sdk/zod";
import { MarkdownConfigSchema, ToolPolicySchema } from "../runtime-api.js";
const groupConfigSchema = z.object({
allow: z.boolean().optional(),

View File

@@ -228,6 +228,10 @@
"types": "./dist/plugin-sdk/account-resolution.d.ts",
"default": "./dist/plugin-sdk/account-resolution.js"
},
"./plugin-sdk/agent-config-primitives": {
"types": "./dist/plugin-sdk/agent-config-primitives.d.ts",
"default": "./dist/plugin-sdk/agent-config-primitives.js"
},
"./plugin-sdk/allow-from": {
"types": "./dist/plugin-sdk/allow-from.d.ts",
"default": "./dist/plugin-sdk/allow-from.js"
@@ -292,6 +296,10 @@
"types": "./dist/plugin-sdk/channel-config-schema.d.ts",
"default": "./dist/plugin-sdk/channel-config-schema.js"
},
"./plugin-sdk/channel-config-primitives": {
"types": "./dist/plugin-sdk/channel-config-primitives.d.ts",
"default": "./dist/plugin-sdk/channel-config-primitives.js"
},
"./plugin-sdk/channel-actions": {
"types": "./dist/plugin-sdk/channel-actions.d.ts",
"default": "./dist/plugin-sdk/channel-actions.js"

View File

@@ -47,6 +47,7 @@
"account-helpers",
"account-id",
"account-resolution",
"agent-config-primitives",
"allow-from",
"allowlist-config-edit",
"bluebubbles",
@@ -62,6 +63,7 @@
"discord-core",
"extension-shared",
"channel-config-helpers",
"channel-config-primitives",
"channel-config-schema",
"channel-actions",
"channel-contract",

View File

@@ -5,17 +5,9 @@ import { existsSync, mkdtempSync, readFileSync, rmSync } from "node:fs";
import { tmpdir } from "node:os";
import { join } from "node:path";
import { pathToFileURL } from "node:url";
import { BUNDLED_RUNTIME_SIDECAR_PATHS } from "../src/plugins/public-artifacts.ts";
import { parseReleaseVersion, resolveNpmCommandInvocation } from "./openclaw-npm-release-check.ts";
const REQUIRED_RUNTIME_SIDECARS = [
"dist/extensions/whatsapp/light-runtime-api.js",
"dist/extensions/whatsapp/runtime-api.js",
"dist/extensions/matrix/helper-api.js",
"dist/extensions/matrix/runtime-api.js",
"dist/extensions/matrix/thread-bindings-runtime.js",
"dist/extensions/msteams/runtime-api.js",
] as const;
type InstalledPackageJson = {
version?: string;
};
@@ -65,7 +57,7 @@ export function collectInstalledPackageErrors(params: {
);
}
for (const relativePath of REQUIRED_RUNTIME_SIDECARS) {
for (const relativePath of BUNDLED_RUNTIME_SIDECAR_PATHS) {
if (!existsSync(join(params.packageRoot, relativePath))) {
errors.push(`installed package is missing required bundled runtime sidecar: ${relativePath}`);
}

View File

@@ -1,8 +1,8 @@
import fs from "node:fs/promises";
import { beforeEach, vi } from "vitest";
import { buildAnthropicCliBackend } from "../../extensions/anthropic/cli-backend.js";
import { buildGoogleGeminiCliBackend } from "../../extensions/google/cli-backend.js";
import { buildOpenAICodexCliBackend } from "../../extensions/openai/cli-backend.js";
import { buildAnthropicCliBackend } from "../../extensions/anthropic/test-api.js";
import { buildGoogleGeminiCliBackend } from "../../extensions/google/test-api.js";
import { buildOpenAICodexCliBackend } from "../../extensions/openai/test-api.js";
import type { OpenClawConfig } from "../config/config.js";
import { createEmptyPluginRegistry } from "../plugins/registry.js";
import { setActivePluginRegistry } from "../plugins/runtime.js";

View File

@@ -1,32 +1,32 @@
import { describe, expect, it, vi } from "vitest";
import { beforeEach, describe, expect, it, vi } from "vitest";
import { SILENT_REPLY_TOKEN } from "../../auto-reply/tokens.js";
vi.mock("../../auto-reply/tokens.js", () => ({
SILENT_REPLY_TOKEN: "QUIET_TOKEN",
}));
vi.mock("../../tts/tts.js", () => ({
textToSpeech: vi.fn(),
}));
const { createTtsTool } = await import("./tts-tool.js");
const { textToSpeech } = await import("../../tts/tts.js");
let textToSpeechSpy: ReturnType<typeof vi.spyOn>;
describe("createTtsTool", () => {
it("uses SILENT_REPLY_TOKEN in guidance text", () => {
beforeEach(async () => {
vi.restoreAllMocks();
vi.resetModules();
const ttsRuntime = await import("../../tts/tts.js");
textToSpeechSpy = vi.spyOn(ttsRuntime, "textToSpeech");
});
it("uses SILENT_REPLY_TOKEN in guidance text", async () => {
const { createTtsTool } = await import("./tts-tool.js");
const tool = createTtsTool();
expect(tool.description).toContain("QUIET_TOKEN");
expect(tool.description).not.toContain("NO_REPLY");
expect(tool.description).toContain(SILENT_REPLY_TOKEN);
});
it("stores audio delivery in details.media", async () => {
vi.mocked(textToSpeech).mockResolvedValue({
textToSpeechSpy.mockResolvedValue({
success: true,
audioPath: "/tmp/reply.opus",
provider: "test",
voiceCompatible: true,
});
const { createTtsTool } = await import("./tts-tool.js");
const tool = createTtsTool();
const result = await tool.execute("call-1", { text: "hello" });

View File

@@ -12,10 +12,6 @@ vi.mock("../../agents/bootstrap-files.js", () => ({
})),
}));
vi.mock("../../agents/pi-tools.js", () => ({
createOpenClawCodingTools: createOpenClawCodingToolsMock,
}));
vi.mock("../../agents/sandbox.js", () => ({
resolveSandboxRuntimeStatus: vi.fn(() => ({ sandboxed: false, mode: "off" })),
}));
@@ -57,12 +53,6 @@ vi.mock("../../infra/skills-remote.js", () => ({
getRemoteSkillEligibility: vi.fn(() => false),
}));
vi.mock("../../tts/tts.js", () => ({
buildTtsSystemPromptHint: vi.fn(() => undefined),
}));
import { resolveCommandsSystemPromptBundle } from "./commands-system-prompt.js";
function makeParams(): HandleCommandsParams {
return {
ctx: {
@@ -107,12 +97,21 @@ function makeParams(): HandleCommandsParams {
}
describe("resolveCommandsSystemPromptBundle", () => {
beforeEach(() => {
beforeEach(async () => {
vi.restoreAllMocks();
vi.resetModules();
createOpenClawCodingToolsMock.mockClear();
createOpenClawCodingToolsMock.mockReturnValue([]);
const piTools = await import("../../agents/pi-tools.js");
vi.spyOn(piTools, "createOpenClawCodingTools").mockImplementation(
createOpenClawCodingToolsMock,
);
const ttsRuntime = await import("../../tts/tts.js");
vi.spyOn(ttsRuntime, "buildTtsSystemPromptHint").mockReturnValue(undefined);
});
it("opts command tool builds into gateway subagent binding", async () => {
const { resolveCommandsSystemPromptBundle } = await import("./commands-system-prompt.js");
await resolveCommandsSystemPromptBundle(makeParams());
expect(createOpenClawCodingToolsMock).toHaveBeenCalledWith(

View File

@@ -0,0 +1 @@
export { promptYesNo } from "./prompt.js";

View File

@@ -4,8 +4,8 @@ import path from "node:path";
import { Command } from "commander";
import { beforeEach, describe, expect, it, vi } from "vitest";
import type { OpenClawConfig, ConfigFileSnapshot } from "../config/types.openclaw.js";
import { BUNDLED_RUNTIME_SIDECAR_PATHS } from "../extensions/public-artifacts.js";
import type { UpdateRunResult } from "../infra/update-runner.js";
import { BUNDLED_RUNTIME_SIDECAR_PATHS } from "../plugins/public-artifacts.js";
import { withEnvAsync } from "../test-utils/env.js";
import { createCliRuntimeCapture } from "./test-runtime-capture.js";

View File

@@ -1,8 +1,8 @@
import { matrixPlugin, setMatrixRuntime } from "../../extensions/matrix/index.js";
import { msteamsPlugin } from "../../extensions/msteams/index.js";
import { nostrPlugin } from "../../extensions/nostr/index.js";
import { tlonPlugin } from "../../extensions/tlon/index.js";
import { whatsappPlugin } from "../../extensions/whatsapp/index.js";
import { matrixPlugin, setMatrixRuntime } from "../../extensions/matrix/test-api.js";
import { msteamsPlugin } from "../../extensions/msteams/test-api.js";
import { nostrPlugin } from "../../extensions/nostr/test-api.js";
import { tlonPlugin } from "../../extensions/tlon/test-api.js";
import { whatsappPlugin } from "../../extensions/whatsapp/test-api.js";
import { bundledChannelPlugins } from "../channels/plugins/bundled.js";
import { setActivePluginRegistry } from "../plugins/runtime.js";
import { createTestRegistry } from "../test-utils/channel-plugins.js";

View File

@@ -1,10 +1,10 @@
import { vi } from "vitest";
import { parseTelegramTarget } from "../../extensions/telegram/api.js";
import { signalOutbound, telegramOutbound } from "../../test/channel-outbounds.js";
import { loadModelCatalog } from "../agents/model-catalog.js";
import { runEmbeddedPiAgent } from "../agents/pi-embedded.js";
import { runSubagentAnnounceFlow } from "../agents/subagent-announce.js";
import { callGateway } from "../gateway/call.js";
import { parseTelegramTarget } from "../plugin-sdk/telegram.js";
import { setActivePluginRegistry } from "../plugins/runtime.js";
import { createOutboundTestPlugin, createTestRegistry } from "../test-utils/channel-plugins.js";

View File

@@ -4,8 +4,8 @@ import fs from "node:fs/promises";
import os from "node:os";
import path from "node:path";
import { Mock, vi } from "vitest";
import { buildElevenLabsSpeechProvider } from "../../extensions/elevenlabs/speech-provider.ts";
import { buildOpenAISpeechProvider } from "../../extensions/openai/speech-provider.ts";
import { buildElevenLabsSpeechProvider } from "../../extensions/elevenlabs/test-api.ts";
import { buildOpenAISpeechProvider } from "../../extensions/openai/test-api.ts";
import type { MsgContext } from "../auto-reply/templating.js";
import type { GetReplyOptions, ReplyPayload } from "../auto-reply/types.js";
import type { ChannelPlugin, ChannelOutboundAdapter } from "../channels/plugins/types.js";

View File

@@ -1,183 +1,6 @@
import type { AuthProfileStore } from "../agents/auth-profiles.js";
import { describeFailoverError, isFailoverError } from "../agents/failover-error.js";
import type { FallbackAttempt } from "../agents/model-fallback.types.js";
import type { OpenClawConfig } from "../config/config.js";
import {
resolveAgentModelFallbackValues,
resolveAgentModelPrimaryValue,
} from "../config/model-input.js";
import { createSubsystemLogger } from "../logging/subsystem.js";
import { getProviderEnvVars } from "../secrets/provider-env-vars.js";
import { parseImageGenerationModelRef } from "./model-ref.js";
import { getImageGenerationProvider, listImageGenerationProviders } from "./provider-registry.js";
import type {
GeneratedImageAsset,
ImageGenerationResolution,
ImageGenerationResult,
ImageGenerationSourceImage,
} from "./types.js";
const log = createSubsystemLogger("image-generation");
export type GenerateImageParams = {
cfg: OpenClawConfig;
prompt: string;
agentDir?: string;
authStore?: AuthProfileStore;
modelOverride?: string;
count?: number;
size?: string;
aspectRatio?: string;
resolution?: ImageGenerationResolution;
inputImages?: ImageGenerationSourceImage[];
};
export type GenerateImageRuntimeResult = {
images: GeneratedImageAsset[];
provider: string;
model: string;
attempts: FallbackAttempt[];
metadata?: Record<string, unknown>;
};
function resolveImageGenerationCandidates(params: {
cfg: OpenClawConfig;
modelOverride?: string;
}): Array<{ provider: string; model: string }> {
const candidates: Array<{ provider: string; model: string }> = [];
const seen = new Set<string>();
const add = (raw: string | undefined) => {
const parsed = parseImageGenerationModelRef(raw);
if (!parsed) {
return;
}
const key = `${parsed.provider}/${parsed.model}`;
if (seen.has(key)) {
return;
}
seen.add(key);
candidates.push(parsed);
};
add(params.modelOverride);
add(resolveAgentModelPrimaryValue(params.cfg.agents?.defaults?.imageGenerationModel));
for (const fallback of resolveAgentModelFallbackValues(
params.cfg.agents?.defaults?.imageGenerationModel,
)) {
add(fallback);
}
return candidates;
}
function throwImageGenerationFailure(params: {
attempts: FallbackAttempt[];
lastError: unknown;
}): never {
if (params.attempts.length <= 1 && params.lastError) {
throw params.lastError;
}
const summary =
params.attempts.length > 0
? params.attempts
.map((attempt) => `${attempt.provider}/${attempt.model}: ${attempt.error}`)
.join(" | ")
: "unknown";
throw new Error(`All image generation models failed (${params.attempts.length}): ${summary}`, {
cause: params.lastError instanceof Error ? params.lastError : undefined,
});
}
function buildNoImageGenerationModelConfiguredMessage(cfg: OpenClawConfig): string {
const providers = listImageGenerationProviders(cfg);
const sampleModel =
providers.find((provider) => provider.defaultModel) ??
({ id: "google", defaultModel: "gemini-3-pro-image-preview" } as const);
const authHints = providers
.flatMap((provider) => {
const envVars = getProviderEnvVars(provider.id);
if (envVars.length === 0) {
return [];
}
return [`${provider.id}: ${envVars.join(" / ")}`];
})
.slice(0, 3);
return [
`No image-generation model configured. Set agents.defaults.imageGenerationModel.primary to a provider/model like "${sampleModel.id}/${sampleModel.defaultModel}".`,
authHints.length > 0
? `If you want a specific provider, also configure that provider's auth/API key first (${authHints.join("; ")}).`
: "If you want a specific provider, also configure that provider's auth/API key first.",
].join(" ");
}
export function listRuntimeImageGenerationProviders(params?: { config?: OpenClawConfig }) {
return listImageGenerationProviders(params?.config);
}
export async function generateImage(
params: GenerateImageParams,
): Promise<GenerateImageRuntimeResult> {
const candidates = resolveImageGenerationCandidates({
cfg: params.cfg,
modelOverride: params.modelOverride,
});
if (candidates.length === 0) {
throw new Error(buildNoImageGenerationModelConfiguredMessage(params.cfg));
}
const attempts: FallbackAttempt[] = [];
let lastError: unknown;
for (const candidate of candidates) {
const provider = getImageGenerationProvider(candidate.provider, params.cfg);
if (!provider) {
const error = `No image-generation provider registered for ${candidate.provider}`;
attempts.push({
provider: candidate.provider,
model: candidate.model,
error,
});
lastError = new Error(error);
continue;
}
try {
const result: ImageGenerationResult = await provider.generateImage({
provider: candidate.provider,
model: candidate.model,
prompt: params.prompt,
cfg: params.cfg,
agentDir: params.agentDir,
authStore: params.authStore,
count: params.count,
size: params.size,
aspectRatio: params.aspectRatio,
resolution: params.resolution,
inputImages: params.inputImages,
});
if (!Array.isArray(result.images) || result.images.length === 0) {
throw new Error("Image generation provider returned no images.");
}
return {
images: result.images,
provider: candidate.provider,
model: result.model ?? candidate.model,
attempts,
metadata: result.metadata,
};
} catch (err) {
lastError = err;
const described = isFailoverError(err) ? describeFailoverError(err) : undefined;
attempts.push({
provider: candidate.provider,
model: candidate.model,
error: described?.message ?? (err instanceof Error ? err.message : String(err)),
reason: described?.reason,
status: described?.status,
code: described?.code,
});
log.debug(`image-generation candidate failed: ${candidate.provider}/${candidate.model}`);
}
}
throwImageGenerationFailure({ attempts, lastError });
}
export {
generateImage,
listRuntimeImageGenerationProviders,
type GenerateImageParams,
type GenerateImageRuntimeResult,
} from "../plugin-sdk/image-generation-runtime.js";

View File

@@ -0,0 +1 @@
export { ensureBinary } from "./binaries.js";

View File

@@ -1,5 +1,4 @@
import { createSubsystemLogger } from "../logging/subsystem.js";
import { parseBooleanValue } from "../utils/boolean.js";
let log: ReturnType<typeof createSubsystemLogger> | null = null;
const loggedEnv = new Set<string>();
@@ -53,7 +52,18 @@ export function normalizeZaiEnv(): void {
}
export function isTruthyEnvValue(value?: string): boolean {
return parseBooleanValue(value) === true;
if (typeof value !== "string") {
return false;
}
switch (value.trim().toLowerCase()) {
case "1":
case "on":
case "true":
case "yes":
return true;
default:
return false;
}
}
export function normalizeEnv(): void {

View File

@@ -1,7 +1,7 @@
import { beforeEach } from "vitest";
import { slackPlugin, setSlackRuntime } from "../../extensions/slack/index.js";
import { telegramPlugin, setTelegramRuntime } from "../../extensions/telegram/index.js";
import { whatsappPlugin, setWhatsAppRuntime } from "../../extensions/whatsapp/index.js";
import { slackPlugin, setSlackRuntime } from "../../extensions/slack/test-api.js";
import { telegramPlugin, setTelegramRuntime } from "../../extensions/telegram/test-api.js";
import { whatsappPlugin, setWhatsAppRuntime } from "../../extensions/whatsapp/test-api.js";
import type { ChannelPlugin } from "../channels/plugins/types.plugin.js";
import { setActivePluginRegistry } from "../plugins/runtime.js";
import { createPluginRuntime } from "../plugins/runtime/index.js";

View File

@@ -2,7 +2,7 @@ import fs from "node:fs/promises";
import os from "node:os";
import path from "node:path";
import { vi } from "vitest";
import { telegramPlugin, setTelegramRuntime } from "../../extensions/telegram/index.js";
import { telegramPlugin, setTelegramRuntime } from "../../extensions/telegram/test-api.js";
import * as replyModule from "../auto-reply/reply.js";
import type { OpenClawConfig } from "../config/config.js";
import { resolveMainSessionKey } from "../config/sessions.js";

View File

@@ -1,5 +1,5 @@
import { slackPlugin, setSlackRuntime } from "../../../extensions/slack/index.js";
import { telegramPlugin, setTelegramRuntime } from "../../../extensions/telegram/index.js";
import { slackPlugin, setSlackRuntime } from "../../../extensions/slack/test-api.js";
import { telegramPlugin, setTelegramRuntime } from "../../../extensions/telegram/test-api.js";
import type { OpenClawConfig } from "../../config/config.js";
import { setActivePluginRegistry } from "../../plugins/runtime.js";
import { createPluginRuntime } from "../../plugins/runtime/index.js";

View File

@@ -1,7 +1,7 @@
import { afterEach, beforeEach, describe, expect, it } from "vitest";
import { parseTelegramTarget } from "../../../extensions/telegram/api.js";
import { telegramOutbound, whatsappOutbound } from "../../../test/channel-outbounds.js";
import type { OpenClawConfig } from "../../config/config.js";
import { parseTelegramTarget } from "../../plugin-sdk/telegram.js";
import { isWhatsAppGroupJid, normalizeWhatsAppTarget } from "../../plugin-sdk/whatsapp-shared.js";
import { setActivePluginRegistry } from "../../plugins/runtime.js";
import { createOutboundTestPlugin, createTestRegistry } from "../../test-utils/channel-plugins.js";

View File

@@ -4,7 +4,11 @@ const resolveProviderUsageAuthWithPluginMock = vi.fn(
async (..._args: unknown[]): Promise<unknown> => null,
);
vi.mock("../plugins/provider-runtime.js", () => ({
const resolveProviderCapabilitiesWithPluginMock = vi.fn(() => undefined);
vi.mock("../plugins/provider-runtime.js", async (importOriginal) => ({
...(await importOriginal<typeof import("../plugins/provider-runtime.js")>()),
resolveProviderCapabilitiesWithPlugin: resolveProviderCapabilitiesWithPluginMock,
resolveProviderUsageAuthWithPlugin: resolveProviderUsageAuthWithPluginMock,
}));

View File

@@ -14,13 +14,13 @@ describe("library module imports", () => {
replyRuntimeLoads();
return await importOriginal<typeof import("./auto-reply/reply.runtime.js")>();
});
vi.doMock("./cli/prompt.js", async (importOriginal) => {
vi.doMock("./cli/prompt.runtime.js", async (importOriginal) => {
promptRuntimeLoads();
return await importOriginal<typeof import("./cli/prompt.js")>();
return await importOriginal<typeof import("./cli/prompt.runtime.js")>();
});
vi.doMock("./infra/binaries.js", async (importOriginal) => {
vi.doMock("./infra/binaries.runtime.js", async (importOriginal) => {
binariesRuntimeLoads();
return await importOriginal<typeof import("./infra/binaries.js")>();
return await importOriginal<typeof import("./infra/binaries.runtime.js")>();
});
vi.doMock("./plugins/runtime/runtime-whatsapp-boundary.js", async (importOriginal) => {
whatsappRuntimeLoads();
@@ -32,12 +32,12 @@ describe("library module imports", () => {
await import("./library.js");
expect(replyRuntimeLoads).not.toHaveBeenCalled();
expect(promptRuntimeLoads).not.toHaveBeenCalled();
expect(binariesRuntimeLoads).not.toHaveBeenCalled();
expect(whatsappRuntimeLoads).not.toHaveBeenCalled();
// Vitest eagerly resolves some manual mocks for runtime-boundary modules
// even when the lazy wrapper is not invoked. Keep the assertion on the
// reply runtime, which is the stable import-time contract this test cares about.
vi.doUnmock("./auto-reply/reply.runtime.js");
vi.doUnmock("./cli/prompt.js");
vi.doUnmock("./infra/binaries.js");
vi.doUnmock("./cli/prompt.runtime.js");
vi.doUnmock("./infra/binaries.runtime.js");
vi.doUnmock("./plugins/runtime/runtime-whatsapp-boundary.js");
});
});

View File

@@ -13,57 +13,51 @@ import {
} from "./infra/ports.js";
import { assertWebChannel, normalizeE164, toWhatsappJid } from "./utils.js";
type GetReplyFromConfig = typeof import("./auto-reply/reply.runtime.js").getReplyFromConfig;
type PromptYesNo = typeof import("./cli/prompt.js").promptYesNo;
type EnsureBinary = typeof import("./infra/binaries.js").ensureBinary;
type RunExec = typeof import("./process/exec.js").runExec;
type RunCommandWithTimeout = typeof import("./process/exec.js").runCommandWithTimeout;
type MonitorWebChannel =
typeof import("./plugins/runtime/runtime-whatsapp-boundary.js").monitorWebChannel;
type ReplyRuntimeModule = typeof import("./auto-reply/reply.runtime.js");
type PromptRuntimeModule = typeof import("./cli/prompt.runtime.js");
type BinariesRuntimeModule = typeof import("./infra/binaries.runtime.js");
type ExecRuntimeModule = typeof import("./process/exec.js");
type WhatsAppRuntimeModule = typeof import("./plugins/runtime/runtime-whatsapp-boundary.js");
let replyRuntimePromise: Promise<typeof import("./auto-reply/reply.runtime.js")> | null = null;
let promptRuntimePromise: Promise<typeof import("./cli/prompt.js")> | null = null;
let binariesRuntimePromise: Promise<typeof import("./infra/binaries.js")> | null = null;
let execRuntimePromise: Promise<typeof import("./process/exec.js")> | null = null;
let whatsappRuntimePromise: Promise<
typeof import("./plugins/runtime/runtime-whatsapp-boundary.js")
> | null = null;
let replyRuntimePromise: Promise<ReplyRuntimeModule> | undefined;
let promptRuntimePromise: Promise<PromptRuntimeModule> | undefined;
let binariesRuntimePromise: Promise<BinariesRuntimeModule> | undefined;
let execRuntimePromise: Promise<ExecRuntimeModule> | undefined;
let whatsappRuntimePromise: Promise<WhatsAppRuntimeModule> | undefined;
function loadReplyRuntime() {
replyRuntimePromise ??= import("./auto-reply/reply.runtime.js");
return replyRuntimePromise;
function loadReplyRuntime(): Promise<ReplyRuntimeModule> {
return (replyRuntimePromise ??= import("./auto-reply/reply.runtime.js"));
}
function loadPromptRuntime() {
promptRuntimePromise ??= import("./cli/prompt.js");
return promptRuntimePromise;
function loadPromptRuntime(): Promise<PromptRuntimeModule> {
return (promptRuntimePromise ??= import("./cli/prompt.runtime.js"));
}
function loadBinariesRuntime() {
binariesRuntimePromise ??= import("./infra/binaries.js");
return binariesRuntimePromise;
function loadBinariesRuntime(): Promise<BinariesRuntimeModule> {
return (binariesRuntimePromise ??= import("./infra/binaries.runtime.js"));
}
function loadExecRuntime() {
execRuntimePromise ??= import("./process/exec.js");
return execRuntimePromise;
function loadExecRuntime(): Promise<ExecRuntimeModule> {
return (execRuntimePromise ??= import("./process/exec.js"));
}
function loadWhatsAppRuntime() {
whatsappRuntimePromise ??= import("./plugins/runtime/runtime-whatsapp-boundary.js");
return whatsappRuntimePromise;
function loadWhatsAppRuntime(): Promise<WhatsAppRuntimeModule> {
return (whatsappRuntimePromise ??= import("./plugins/runtime/runtime-whatsapp-boundary.js"));
}
export const getReplyFromConfig: GetReplyFromConfig = async (...args) =>
export const getReplyFromConfig: ReplyRuntimeModule["getReplyFromConfig"] = async (...args) =>
(await loadReplyRuntime()).getReplyFromConfig(...args);
export const promptYesNo: PromptYesNo = async (...args) =>
export const promptYesNo: PromptRuntimeModule["promptYesNo"] = async (...args) =>
(await loadPromptRuntime()).promptYesNo(...args);
export const ensureBinary: EnsureBinary = async (...args) =>
export const ensureBinary: BinariesRuntimeModule["ensureBinary"] = async (...args) =>
(await loadBinariesRuntime()).ensureBinary(...args);
export const runExec: RunExec = async (...args) => (await loadExecRuntime()).runExec(...args);
export const runCommandWithTimeout: RunCommandWithTimeout = async (...args) =>
export const runExec: ExecRuntimeModule["runExec"] = async (...args) =>
(await loadExecRuntime()).runExec(...args);
export const runCommandWithTimeout: ExecRuntimeModule["runCommandWithTimeout"] = async (
...args
) =>
(await loadExecRuntime()).runCommandWithTimeout(...args);
export const monitorWebChannel: MonitorWebChannel = async (...args) =>
export const monitorWebChannel: WhatsAppRuntimeModule["monitorWebChannel"] = async (...args) =>
(await loadWhatsAppRuntime()).monitorWebChannel(...args);
export {

View File

@@ -1,146 +1,9 @@
import fs from "node:fs/promises";
import path from "node:path";
import type { MsgContext } from "../auto-reply/templating.js";
import type { OpenClawConfig } from "../config/config.js";
import { getMediaUnderstandingProvider } from "./provider-registry.js";
import {
buildProviderRegistry,
createMediaAttachmentCache,
normalizeMediaAttachments,
runCapability,
type ActiveMediaModel,
} from "./runner.js";
import type { MediaUnderstandingCapability, MediaUnderstandingOutput } from "./types.js";
const KIND_BY_CAPABILITY: Record<MediaUnderstandingCapability, MediaUnderstandingOutput["kind"]> = {
audio: "audio.transcription",
image: "image.description",
video: "video.description",
};
export type RunMediaUnderstandingFileParams = {
capability: MediaUnderstandingCapability;
filePath: string;
cfg: OpenClawConfig;
agentDir?: string;
mime?: string;
activeModel?: ActiveMediaModel;
};
export type RunMediaUnderstandingFileResult = {
text: string | undefined;
provider?: string;
model?: string;
output?: MediaUnderstandingOutput;
};
function buildFileContext(params: { filePath: string; mime?: string }): MsgContext {
return {
MediaPath: params.filePath,
MediaType: params.mime,
};
}
export async function runMediaUnderstandingFile(
params: RunMediaUnderstandingFileParams,
): Promise<RunMediaUnderstandingFileResult> {
const ctx = buildFileContext(params);
const attachments = normalizeMediaAttachments(ctx);
if (attachments.length === 0) {
return { text: undefined };
}
const providerRegistry = buildProviderRegistry(undefined, params.cfg);
const cache = createMediaAttachmentCache(attachments, {
localPathRoots: [path.dirname(params.filePath)],
});
try {
const result = await runCapability({
capability: params.capability,
cfg: params.cfg,
ctx,
attachments: cache,
media: attachments,
agentDir: params.agentDir,
providerRegistry,
config: params.cfg.tools?.media?.[params.capability],
activeModel: params.activeModel,
});
const output = result.outputs.find(
(entry) => entry.kind === KIND_BY_CAPABILITY[params.capability],
);
const text = output?.text?.trim();
return {
text: text || undefined,
provider: output?.provider,
model: output?.model,
output,
};
} finally {
await cache.cleanup();
}
}
export async function describeImageFile(params: {
filePath: string;
cfg: OpenClawConfig;
agentDir?: string;
mime?: string;
activeModel?: ActiveMediaModel;
}): Promise<RunMediaUnderstandingFileResult> {
return await runMediaUnderstandingFile({ ...params, capability: "image" });
}
export async function describeImageFileWithModel(params: {
filePath: string;
cfg: OpenClawConfig;
agentDir?: string;
mime?: string;
provider: string;
model: string;
prompt: string;
maxTokens?: number;
timeoutMs?: number;
}) {
const timeoutMs = params.timeoutMs ?? 30_000;
const providerRegistry = buildProviderRegistry(undefined, params.cfg);
const provider = getMediaUnderstandingProvider(params.provider, providerRegistry);
if (!provider?.describeImage) {
throw new Error(`Provider does not support image analysis: ${params.provider}`);
}
const buffer = await fs.readFile(params.filePath);
return await provider.describeImage({
buffer,
fileName: path.basename(params.filePath),
mime: params.mime,
provider: params.provider,
model: params.model,
prompt: params.prompt,
maxTokens: params.maxTokens,
timeoutMs,
cfg: params.cfg,
agentDir: params.agentDir ?? "",
});
}
export async function describeVideoFile(params: {
filePath: string;
cfg: OpenClawConfig;
agentDir?: string;
mime?: string;
activeModel?: ActiveMediaModel;
}): Promise<RunMediaUnderstandingFileResult> {
return await runMediaUnderstandingFile({ ...params, capability: "video" });
}
export async function transcribeAudioFile(params: {
filePath: string;
cfg: OpenClawConfig;
agentDir?: string;
mime?: string;
activeModel?: ActiveMediaModel;
}): Promise<{ text: string | undefined }> {
const result = await runMediaUnderstandingFile({ ...params, capability: "audio" });
return { text: result.text };
}
export {
describeImageFile,
describeImageFileWithModel,
describeVideoFile,
runMediaUnderstandingFile,
transcribeAudioFile,
type RunMediaUnderstandingFileParams,
type RunMediaUnderstandingFileResult,
} from "../plugin-sdk/media-understanding-runtime.js";

View File

@@ -20,19 +20,13 @@ export { normalizeE164, pathExists, resolveUserPath } from "../utils.js";
export {
resolveDiscordAccount,
type ResolvedDiscordAccount,
} from "../../extensions/discord/src/accounts.js";
export {
resolveSlackAccount,
type ResolvedSlackAccount,
} from "../../extensions/slack/src/accounts.js";
} from "../../extensions/discord/api.js";
export { resolveSlackAccount, type ResolvedSlackAccount } from "../../extensions/slack/api.js";
export {
resolveTelegramAccount,
type ResolvedTelegramAccount,
} from "../../extensions/telegram/src/accounts.js";
export {
resolveSignalAccount,
type ResolvedSignalAccount,
} from "../../extensions/signal/src/accounts.js";
} from "../../extensions/telegram/api.js";
export { resolveSignalAccount, type ResolvedSignalAccount } from "../../extensions/signal/api.js";
/** Resolve an account by id, then fall back to the default account when the primary lacks credentials. */
export function resolveAccountWithDefaultFallback<TAccount>(params: {

View File

@@ -0,0 +1,3 @@
/** Narrow agent-runtime schema primitives without broader config/runtime surfaces. */
export { ReplyRuntimeConfigSchemaShape } from "../config/zod-schema.core.js";
export { ToolPolicySchema } from "../config/zod-schema.agent-runtime.js";

View File

@@ -28,7 +28,7 @@ export { buildChannelConfigSchema } from "../channels/plugins/config-schema.js";
export {
resolveBlueBubblesGroupRequireMention,
resolveBlueBubblesGroupToolPolicy,
} from "../../extensions/bluebubbles/src/group-policy.js";
} from "../../extensions/bluebubbles/api.js";
export { formatPairingApproveHint } from "../channels/plugins/helpers.js";
export { resolveChannelMediaMaxBytes } from "../channels/plugins/media-limits.js";
export {
@@ -85,6 +85,7 @@ export {
buildComputedAccountStatusSnapshot,
buildProbeChannelStatusSummary,
} from "./status-helpers.js";
export { isAllowedBlueBubblesSender } from "../../extensions/bluebubbles/api.js";
export { extractToolSend } from "./tool-send.js";
export {
WEBHOOK_RATE_LIMIT_DEFAULTS,

View File

@@ -0,0 +1,16 @@
/** Narrow channel config-schema primitives without provider-schema re-exports. */
export {
AllowFromListSchema,
buildChannelConfigSchema,
buildCatchallMultiAccountChannelSchema,
buildNestedDmConfigSchema,
} from "../channels/plugins/config-schema.js";
export {
BlockStreamingCoalesceSchema,
DmConfigSchema,
DmPolicySchema,
GroupPolicySchema,
MarkdownConfigSchema,
ReplyRuntimeConfigSchemaShape,
requireOpenAllowFrom,
} from "../config/zod-schema.core.js";

View File

@@ -2,7 +2,7 @@ import { readdirSync, readFileSync } from "node:fs";
import { dirname, resolve } from "node:path";
import { fileURLToPath } from "node:url";
import { describe, expect, it } from "vitest";
import { GUARDED_EXTENSION_PUBLIC_SURFACE_BASENAMES } from "../extensions/public-artifacts.js";
import { GUARDED_EXTENSION_PUBLIC_SURFACE_BASENAMES } from "../plugins/public-artifacts.js";
const ROOT_DIR = resolve(dirname(fileURLToPath(import.meta.url)), "..");
const REPO_ROOT = resolve(ROOT_DIR, "..");

View File

@@ -5,6 +5,7 @@ export * from "../channels/chat-type.js";
export * from "../channels/reply-prefix.js";
export * from "../channels/typing.js";
export type * from "../channels/plugins/types.js";
export { normalizeChannelId } from "../channels/plugins/registry.js";
export * from "../channels/plugins/normalize/signal.js";
export * from "../channels/plugins/normalize/whatsapp.js";
export * from "../channels/plugins/outbound/interactive.js";

View File

@@ -48,5 +48,5 @@ export { mapAllowlistResolutionInputs } from "./allow-from.js";
export {
resolveBlueBubblesGroupRequireMention,
resolveBlueBubblesGroupToolPolicy,
} from "../../extensions/bluebubbles/src/group-policy.js";
} from "../../extensions/bluebubbles/api.js";
export { collectBlueBubblesStatusIssues } from "../channels/plugins/status-issues/bluebubbles.js";

View File

@@ -74,7 +74,11 @@ export type {
TelegramInlineButtonsScope,
TelegramNetworkConfig,
TelegramTopicConfig,
TtsAutoMode,
TtsConfig,
TtsMode,
TtsModelOverrideConfig,
TtsProvider,
} from "../config/types.js";
export {
loadSessionStore,

View File

@@ -1,5 +1,7 @@
// Shared image-generation implementation helpers for bundled and third-party plugins.
export type { AuthProfileStore } from "../agents/auth-profiles.js";
export type { FallbackAttempt } from "../agents/model-fallback.types.js";
export type { ImageGenerationProviderPlugin } from "../plugins/types.js";
export type {
GeneratedImageAsset,
@@ -9,8 +11,21 @@ export type {
ImageGenerationResult,
ImageGenerationSourceImage,
} from "../image-generation/types.js";
export type { OpenClawConfig } from "../config/config.js";
export { describeFailoverError, isFailoverError } from "../agents/failover-error.js";
export { resolveApiKeyForProvider } from "../agents/model-auth.js";
export { normalizeGoogleModelId } from "../agents/model-id-normalization.js";
export {
resolveAgentModelFallbackValues,
resolveAgentModelPrimaryValue,
} from "../config/model-input.js";
export { parseGeminiAuth } from "../infra/gemini-auth.js";
export {
getImageGenerationProvider,
listImageGenerationProviders,
} from "../image-generation/provider-registry.js";
export { parseImageGenerationModelRef } from "../image-generation/model-ref.js";
export { createSubsystemLogger } from "../logging/subsystem.js";
export { OPENAI_DEFAULT_IMAGE_MODEL } from "../plugins/provider-model-defaults.js";
export { getProviderEnvVars } from "../secrets/provider-env-vars.js";

View File

@@ -1,3 +1,8 @@
// Public runtime-facing image-generation helpers for feature/channel plugins.
export { generateImage, listRuntimeImageGenerationProviders } from "../image-generation/runtime.js";
export {
generateImage,
listRuntimeImageGenerationProviders,
type GenerateImageParams,
type GenerateImageRuntimeResult,
} from "../../extensions/image-generation-core/runtime-api.js";

View File

@@ -94,3 +94,4 @@ export { getAgentScopedMediaLocalRoots } from "../media/local-roots.js";
export { loadOutboundMediaFromUrl } from "./outbound-media.js";
export { createChannelPairingController } from "./channel-pairing.js";
export { isRequestBodyLimitError, readRequestBodyWithLimit } from "../infra/http-body.js";
export { isMattermostSenderAllowed } from "../../extensions/mattermost/api.js";

View File

@@ -23,6 +23,7 @@ export * from "../media-understanding/audio-preflight.ts";
export * from "../media-understanding/defaults.js";
export * from "../media-understanding/image-runtime.ts";
export * from "../media-understanding/runner.js";
export { normalizeMediaProviderId } from "../media-understanding/provider-registry.js";
export * from "../polls.js";
export {
createDirectTextMediaOutbound,

View File

@@ -6,4 +6,6 @@ export {
describeVideoFile,
runMediaUnderstandingFile,
transcribeAudioFile,
} from "../media-understanding/runtime.js";
type RunMediaUnderstandingFileParams,
type RunMediaUnderstandingFileResult,
} from "../../extensions/media-understanding-core/runtime-api.js";

View File

@@ -4,7 +4,7 @@
export type { ChannelMessageActionAdapter } from "../channels/plugins/types.js";
export type { OpenClawConfig } from "../config/config.js";
export type { SignalAccountConfig } from "../config/types.js";
export type { ResolvedSignalAccount } from "../../extensions/signal/src/accounts.js";
export type { ResolvedSignalAccount } from "../../extensions/signal/api.js";
export type {
ChannelMessageActionContext,
ChannelPlugin,
@@ -54,13 +54,12 @@ export {
listEnabledSignalAccounts,
listSignalAccountIds,
resolveDefaultSignalAccountId,
} from "../../extensions/signal/src/accounts.js";
export { monitorSignalProvider } from "../../extensions/signal/src/monitor.js";
export { probeSignal } from "../../extensions/signal/src/probe.js";
export { resolveSignalReactionLevel } from "../../extensions/signal/src/reaction-level.js";
export {
removeReactionSignal,
sendReactionSignal,
} from "../../extensions/signal/src/send-reactions.js";
export { sendMessageSignal } from "../../extensions/signal/src/send.js";
export { signalMessageActions } from "../../extensions/signal/src/message-actions.js";
} from "../../extensions/signal/api.js";
export { isSignalSenderAllowed } from "../../extensions/signal/api.js";
export type { SignalSender } from "../../extensions/signal/api.js";
export { monitorSignalProvider } from "../../extensions/signal/api.js";
export { probeSignal } from "../../extensions/signal/api.js";
export { resolveSignalReactionLevel } from "../../extensions/signal/api.js";
export { removeReactionSignal, sendReactionSignal } from "../../extensions/signal/api.js";
export { sendMessageSignal } from "../../extensions/signal/api.js";
export { signalMessageActions } from "../../extensions/signal/api.js";

View File

@@ -20,9 +20,18 @@ export type {
} from "../tts/provider-types.js";
export {
scheduleCleanup,
summarizeText,
normalizeApplyTextNormalization,
normalizeLanguageCode,
normalizeSeed,
requireInRange,
} from "../tts/tts-core.js";
export { parseTtsDirectives } from "../tts/directives.js";
export {
canonicalizeSpeechProviderId,
getSpeechProvider,
listSpeechProviders,
normalizeSpeechProviderId,
} from "../tts/provider-registry.js";
export { normalizeTtsAutoMode, TTS_AUTO_MODES } from "../tts/tts-auto-mode.js";

View File

@@ -1,3 +1,35 @@
// Public runtime-facing speech helpers for feature/channel plugins.
export { listSpeechVoices, textToSpeech, textToSpeechTelephony } from "../tts/runtime.js";
export {
_test,
buildTtsSystemPromptHint,
getLastTtsAttempt,
getResolvedSpeechProviderConfig,
getTtsMaxLength,
getTtsProvider,
isSummarizationEnabled,
isTtsEnabled,
isTtsProviderConfigured,
listSpeechVoices,
maybeApplyTtsToPayload,
resolveTtsAutoMode,
resolveTtsConfig,
resolveTtsPrefsPath,
resolveTtsProviderOrder,
setLastTtsAttempt,
setSummarizationEnabled,
setTtsAutoMode,
setTtsEnabled,
setTtsMaxLength,
setTtsProvider,
synthesizeSpeech,
textToSpeech,
textToSpeechTelephony,
type ResolvedTtsConfig,
type ResolvedTtsModelOverrides,
type TtsDirectiveOverrides,
type TtsDirectiveParseResult,
type TtsResult,
type TtsSynthesisResult,
type TtsTelephonyResult,
} from "../../extensions/speech-core/runtime-api.js";

View File

@@ -16,7 +16,7 @@ export function resolvePluginCapabilityProviders<K extends CapabilityProviderReg
cfg?: OpenClawConfig;
useActiveRegistryWhen?: (active: PluginRegistry | undefined) => boolean;
}): CapabilityProviderForKey<K>[] {
const active = getActivePluginRegistry();
const active = getActivePluginRegistry() ?? undefined;
const shouldUseActive =
params.useActiveRegistryWhen?.(active) ?? (active?.[params.key].length ?? 0) > 0;
const registry =

View File

@@ -1,7 +1,9 @@
import {
BUNDLED_IMAGE_GENERATION_PLUGIN_IDS,
BUNDLED_MEDIA_UNDERSTANDING_PLUGIN_IDS,
BUNDLED_PLUGIN_CONTRACT_SNAPSHOTS,
BUNDLED_PROVIDER_PLUGIN_IDS,
BUNDLED_SPEECH_PLUGIN_IDS,
BUNDLED_WEB_SEARCH_PLUGIN_IDS,
} from "../bundled-capability-metadata.js";
import { loadBundledCapabilityRuntimeRegistry } from "../bundled-capability-runtime.js";
@@ -55,33 +57,6 @@ function createProviderContractPluginIdsByProviderId(): Map<string, string[]> {
return result;
}
function createContractSpeechProvider(providerId: string): SpeechProviderPlugin {
return {
id: providerId,
label: providerId,
isConfigured: () => true,
synthesize: async () => ({
audioBuffer: Buffer.alloc(0),
outputFormat: "mp3",
fileExtension: "mp3",
voiceCompatible: true,
}),
listVoices: async () => [],
};
}
function createContractMediaUnderstandingProvider(
providerId: string,
): MediaUnderstandingProviderPlugin {
return {
id: providerId,
capabilities: ["image"],
describeImages: async () => {
throw new Error(`media-understanding contract stub invoked for ${providerId}`);
},
};
}
function uniqueStrings(values: readonly string[]): string[] {
const result: string[] = [];
const seen = new Set<string>();
@@ -163,9 +138,7 @@ function loadProviderContractPluginIds(): string[] {
}
function loadProviderContractCompatPluginIds(): string[] {
return loadProviderContractPluginIds().map((pluginId) =>
pluginId === "kimi-coding" ? "kimi" : pluginId,
);
return loadProviderContractPluginIds();
}
function resolveWebSearchCredentialValue(provider: WebSearchProviderPlugin): unknown {
@@ -199,25 +172,29 @@ function loadWebSearchProviderContractRegistry(): WebSearchProviderContractEntry
function loadSpeechProviderContractRegistry(): SpeechProviderContractEntry[] {
if (!speechProviderContractRegistryCache) {
// Contract tests only need bundled ownership and public speech surface shape.
speechProviderContractRegistryCache = BUNDLED_PLUGIN_CONTRACT_SNAPSHOTS.flatMap((entry) =>
entry.speechProviderIds.map((providerId) => ({
pluginId: entry.pluginId,
provider: createContractSpeechProvider(providerId),
})),
);
const registry = loadBundledCapabilityRuntimeRegistry({
pluginIds: BUNDLED_SPEECH_PLUGIN_IDS,
pluginSdkResolution: "dist",
});
speechProviderContractRegistryCache = registry.speechProviders.map((entry) => ({
pluginId: entry.pluginId,
provider: entry.provider,
}));
}
return speechProviderContractRegistryCache;
}
function loadMediaUnderstandingProviderContractRegistry(): MediaUnderstandingProviderContractEntry[] {
if (!mediaUnderstandingProviderContractRegistryCache) {
mediaUnderstandingProviderContractRegistryCache = BUNDLED_PLUGIN_CONTRACT_SNAPSHOTS.flatMap(
(entry) =>
entry.mediaUnderstandingProviderIds.map((providerId) => ({
pluginId: entry.pluginId,
provider: createContractMediaUnderstandingProvider(providerId),
})),
const registry = loadBundledCapabilityRuntimeRegistry({
pluginIds: BUNDLED_MEDIA_UNDERSTANDING_PLUGIN_IDS,
pluginSdkResolution: "dist",
});
mediaUnderstandingProviderContractRegistryCache = registry.mediaUnderstandingProviders.map(
(entry) => ({
pluginId: entry.pluginId,
provider: entry.provider,
}),
);
}
return mediaUnderstandingProviderContractRegistryCache;

View File

@@ -1,3 +1,5 @@
import { BUNDLED_PLUGIN_METADATA } from "./bundled-plugin-metadata.js";
function assertUniqueValues<T extends string>(values: readonly T[], label: string): readonly T[] {
const seen = new Set<string>();
const duplicates = new Set<string>();
@@ -19,14 +21,11 @@ export function getPublicArtifactBasename(relativePath: string): string {
}
export const BUNDLED_RUNTIME_SIDECAR_PATHS = assertUniqueValues(
[
"dist/extensions/whatsapp/light-runtime-api.js",
"dist/extensions/whatsapp/runtime-api.js",
"dist/extensions/matrix/helper-api.js",
"dist/extensions/matrix/runtime-api.js",
"dist/extensions/matrix/thread-bindings-runtime.js",
"dist/extensions/msteams/runtime-api.js",
] as const,
BUNDLED_PLUGIN_METADATA.flatMap((entry) =>
(entry.runtimeSidecarArtifacts ?? []).map(
(artifact) => `dist/extensions/${entry.dirName}/${artifact}`,
),
).toSorted((left, right) => left.localeCompare(right)),
"bundled runtime sidecar path",
);

View File

@@ -1,4 +1,5 @@
import { createJiti } from "jiti";
import type { MatrixRuntimeBoundaryModule } from "./runtime-matrix-surface.js";
import {
loadPluginBoundaryModuleWithJiti,
resolvePluginRuntimeModulePath,
@@ -7,15 +8,13 @@ import {
const MATRIX_PLUGIN_ID = "matrix";
type MatrixModule = typeof import("../../../extensions/matrix/runtime-api.js");
type MatrixPluginRecord = {
rootDir?: string;
source: string;
};
let cachedModulePath: string | null = null;
let cachedModule: MatrixModule | null = null;
let cachedModule: MatrixRuntimeBoundaryModule | null = null;
const jitiLoaders = new Map<boolean, ReturnType<typeof createJiti>>();
@@ -27,7 +26,7 @@ function resolveMatrixRuntimeModulePath(record: MatrixPluginRecord): string | nu
return resolvePluginRuntimeModulePath(record, "runtime-api");
}
function loadMatrixModule(): MatrixModule | null {
function loadMatrixModule(): MatrixRuntimeBoundaryModule | null {
const record = resolveMatrixPluginRecord();
if (!record) {
return null;
@@ -39,15 +38,18 @@ function loadMatrixModule(): MatrixModule | null {
if (cachedModule && cachedModulePath === modulePath) {
return cachedModule;
}
const loaded = loadPluginBoundaryModuleWithJiti<MatrixModule>(modulePath, jitiLoaders);
const loaded = loadPluginBoundaryModuleWithJiti<MatrixRuntimeBoundaryModule>(
modulePath,
jitiLoaders,
);
cachedModulePath = modulePath;
cachedModule = loaded;
return loaded;
}
export function setMatrixThreadBindingIdleTimeoutBySessionKey(
...args: Parameters<MatrixModule["setMatrixThreadBindingIdleTimeoutBySessionKey"]>
): ReturnType<MatrixModule["setMatrixThreadBindingIdleTimeoutBySessionKey"]> {
...args: Parameters<MatrixRuntimeBoundaryModule["setMatrixThreadBindingIdleTimeoutBySessionKey"]>
): ReturnType<MatrixRuntimeBoundaryModule["setMatrixThreadBindingIdleTimeoutBySessionKey"]> {
const fn = loadMatrixModule()?.setMatrixThreadBindingIdleTimeoutBySessionKey;
if (typeof fn !== "function") {
return [];
@@ -56,8 +58,8 @@ export function setMatrixThreadBindingIdleTimeoutBySessionKey(
}
export function setMatrixThreadBindingMaxAgeBySessionKey(
...args: Parameters<MatrixModule["setMatrixThreadBindingMaxAgeBySessionKey"]>
): ReturnType<MatrixModule["setMatrixThreadBindingMaxAgeBySessionKey"]> {
...args: Parameters<MatrixRuntimeBoundaryModule["setMatrixThreadBindingMaxAgeBySessionKey"]>
): ReturnType<MatrixRuntimeBoundaryModule["setMatrixThreadBindingMaxAgeBySessionKey"]> {
const fn = loadMatrixModule()?.setMatrixThreadBindingMaxAgeBySessionKey;
if (typeof fn !== "function") {
return [];

View File

@@ -0,0 +1,22 @@
import type { SessionBindingRecord } from "../../infra/outbound/session-binding-service.js";
export type MatrixThreadBindingIdleTimeoutParams = {
accountId: string;
targetSessionKey: string;
idleTimeoutMs: number;
};
export type MatrixThreadBindingMaxAgeParams = {
accountId: string;
targetSessionKey: string;
maxAgeMs: number;
};
export type MatrixRuntimeBoundaryModule = {
setMatrixThreadBindingIdleTimeoutBySessionKey: (
params: MatrixThreadBindingIdleTimeoutParams,
) => SessionBindingRecord[];
setMatrixThreadBindingMaxAgeBySessionKey: (
params: MatrixThreadBindingMaxAgeParams,
) => SessionBindingRecord[];
};

View File

@@ -11,12 +11,13 @@ import {
resolvePluginRuntimeModulePath,
resolvePluginRuntimeRecord,
} from "./runtime-plugin-boundary.js";
import type {
WhatsAppHeavyRuntimeModule,
WhatsAppLightRuntimeModule,
} from "./runtime-whatsapp-surface.js";
const WHATSAPP_PLUGIN_ID = "whatsapp";
type WhatsAppLightModule = typeof import("../../../extensions/whatsapp/light-runtime-api.js");
type WhatsAppHeavyModule = typeof import("../../../extensions/whatsapp/runtime-api.js");
type WhatsAppPluginRecord = {
origin: string;
rootDir?: string;
@@ -24,9 +25,9 @@ type WhatsAppPluginRecord = {
};
let cachedHeavyModulePath: string | null = null;
let cachedHeavyModule: WhatsAppHeavyModule | null = null;
let cachedHeavyModule: WhatsAppHeavyRuntimeModule | null = null;
let cachedLightModulePath: string | null = null;
let cachedLightModule: WhatsAppLightModule | null = null;
let cachedLightModule: WhatsAppLightRuntimeModule | null = null;
const jitiLoaders = new Map<boolean, ReturnType<typeof createJiti>>();
@@ -55,12 +56,12 @@ function resolveWhatsAppRuntimeModulePath(
return modulePath;
}
function loadCurrentHeavyModuleSync(): WhatsAppHeavyModule {
function loadCurrentHeavyModuleSync(): WhatsAppHeavyRuntimeModule {
const modulePath = resolveWhatsAppRuntimeModulePath(resolveWhatsAppPluginRecord(), "runtime-api");
return loadPluginBoundaryModuleWithJiti<WhatsAppHeavyModule>(modulePath, jitiLoaders);
return loadPluginBoundaryModuleWithJiti<WhatsAppHeavyRuntimeModule>(modulePath, jitiLoaders);
}
function loadWhatsAppLightModule(): WhatsAppLightModule {
function loadWhatsAppLightModule(): WhatsAppLightRuntimeModule {
const modulePath = resolveWhatsAppRuntimeModulePath(
resolveWhatsAppPluginRecord(),
"light-runtime-api",
@@ -68,143 +69,149 @@ function loadWhatsAppLightModule(): WhatsAppLightModule {
if (cachedLightModule && cachedLightModulePath === modulePath) {
return cachedLightModule;
}
const loaded = loadPluginBoundaryModuleWithJiti<WhatsAppLightModule>(modulePath, jitiLoaders);
const loaded = loadPluginBoundaryModuleWithJiti<WhatsAppLightRuntimeModule>(
modulePath,
jitiLoaders,
);
cachedLightModulePath = modulePath;
cachedLightModule = loaded;
return loaded;
}
async function loadWhatsAppHeavyModule(): Promise<WhatsAppHeavyModule> {
async function loadWhatsAppHeavyModule(): Promise<WhatsAppHeavyRuntimeModule> {
const record = resolveWhatsAppPluginRecord();
const modulePath = resolveWhatsAppRuntimeModulePath(record, "runtime-api");
if (cachedHeavyModule && cachedHeavyModulePath === modulePath) {
return cachedHeavyModule;
}
const loaded = loadPluginBoundaryModuleWithJiti<WhatsAppHeavyModule>(modulePath, jitiLoaders);
const loaded = loadPluginBoundaryModuleWithJiti<WhatsAppHeavyRuntimeModule>(
modulePath,
jitiLoaders,
);
cachedHeavyModulePath = modulePath;
cachedHeavyModule = loaded;
return loaded;
}
function getLightExport<K extends keyof WhatsAppLightModule>(
function getLightExport<K extends keyof WhatsAppLightRuntimeModule>(
exportName: K,
): NonNullable<WhatsAppLightModule[K]> {
): NonNullable<WhatsAppLightRuntimeModule[K]> {
const loaded = loadWhatsAppLightModule();
const value = loaded[exportName];
if (value == null) {
throw new Error(`WhatsApp plugin runtime is missing export '${String(exportName)}'`);
}
return value as NonNullable<WhatsAppLightModule[K]>;
return value as NonNullable<WhatsAppLightRuntimeModule[K]>;
}
async function getHeavyExport<K extends keyof WhatsAppHeavyModule>(
async function getHeavyExport<K extends keyof WhatsAppHeavyRuntimeModule>(
exportName: K,
): Promise<NonNullable<WhatsAppHeavyModule[K]>> {
): Promise<NonNullable<WhatsAppHeavyRuntimeModule[K]>> {
const loaded = await loadWhatsAppHeavyModule();
const value = loaded[exportName];
if (value == null) {
throw new Error(`WhatsApp plugin runtime is missing export '${String(exportName)}'`);
}
return value as NonNullable<WhatsAppHeavyModule[K]>;
return value as NonNullable<WhatsAppHeavyRuntimeModule[K]>;
}
export function getActiveWebListener(
...args: Parameters<WhatsAppLightModule["getActiveWebListener"]>
): ReturnType<WhatsAppLightModule["getActiveWebListener"]> {
...args: Parameters<WhatsAppLightRuntimeModule["getActiveWebListener"]>
): ReturnType<WhatsAppLightRuntimeModule["getActiveWebListener"]> {
return getLightExport("getActiveWebListener")(...args);
}
export function getWebAuthAgeMs(
...args: Parameters<WhatsAppLightModule["getWebAuthAgeMs"]>
): ReturnType<WhatsAppLightModule["getWebAuthAgeMs"]> {
...args: Parameters<WhatsAppLightRuntimeModule["getWebAuthAgeMs"]>
): ReturnType<WhatsAppLightRuntimeModule["getWebAuthAgeMs"]> {
return getLightExport("getWebAuthAgeMs")(...args);
}
export function logWebSelfId(
...args: Parameters<WhatsAppLightModule["logWebSelfId"]>
): ReturnType<WhatsAppLightModule["logWebSelfId"]> {
...args: Parameters<WhatsAppLightRuntimeModule["logWebSelfId"]>
): ReturnType<WhatsAppLightRuntimeModule["logWebSelfId"]> {
return getLightExport("logWebSelfId")(...args);
}
export function loginWeb(
...args: Parameters<WhatsAppHeavyModule["loginWeb"]>
): ReturnType<WhatsAppHeavyModule["loginWeb"]> {
...args: Parameters<WhatsAppHeavyRuntimeModule["loginWeb"]>
): ReturnType<WhatsAppHeavyRuntimeModule["loginWeb"]> {
return loadWhatsAppHeavyModule().then((loaded) => loaded.loginWeb(...args));
}
export function logoutWeb(
...args: Parameters<WhatsAppLightModule["logoutWeb"]>
): ReturnType<WhatsAppLightModule["logoutWeb"]> {
...args: Parameters<WhatsAppLightRuntimeModule["logoutWeb"]>
): ReturnType<WhatsAppLightRuntimeModule["logoutWeb"]> {
return getLightExport("logoutWeb")(...args);
}
export function readWebSelfId(
...args: Parameters<WhatsAppLightModule["readWebSelfId"]>
): ReturnType<WhatsAppLightModule["readWebSelfId"]> {
...args: Parameters<WhatsAppLightRuntimeModule["readWebSelfId"]>
): ReturnType<WhatsAppLightRuntimeModule["readWebSelfId"]> {
return getLightExport("readWebSelfId")(...args);
}
export function webAuthExists(
...args: Parameters<WhatsAppLightModule["webAuthExists"]>
): ReturnType<WhatsAppLightModule["webAuthExists"]> {
...args: Parameters<WhatsAppLightRuntimeModule["webAuthExists"]>
): ReturnType<WhatsAppLightRuntimeModule["webAuthExists"]> {
return getLightExport("webAuthExists")(...args);
}
export function sendMessageWhatsApp(
...args: Parameters<WhatsAppHeavyModule["sendMessageWhatsApp"]>
): ReturnType<WhatsAppHeavyModule["sendMessageWhatsApp"]> {
...args: Parameters<WhatsAppHeavyRuntimeModule["sendMessageWhatsApp"]>
): ReturnType<WhatsAppHeavyRuntimeModule["sendMessageWhatsApp"]> {
return loadWhatsAppHeavyModule().then((loaded) => loaded.sendMessageWhatsApp(...args));
}
export function sendPollWhatsApp(
...args: Parameters<WhatsAppHeavyModule["sendPollWhatsApp"]>
): ReturnType<WhatsAppHeavyModule["sendPollWhatsApp"]> {
...args: Parameters<WhatsAppHeavyRuntimeModule["sendPollWhatsApp"]>
): ReturnType<WhatsAppHeavyRuntimeModule["sendPollWhatsApp"]> {
return loadWhatsAppHeavyModule().then((loaded) => loaded.sendPollWhatsApp(...args));
}
export function sendReactionWhatsApp(
...args: Parameters<WhatsAppHeavyModule["sendReactionWhatsApp"]>
): ReturnType<WhatsAppHeavyModule["sendReactionWhatsApp"]> {
...args: Parameters<WhatsAppHeavyRuntimeModule["sendReactionWhatsApp"]>
): ReturnType<WhatsAppHeavyRuntimeModule["sendReactionWhatsApp"]> {
return loadWhatsAppHeavyModule().then((loaded) => loaded.sendReactionWhatsApp(...args));
}
export function createRuntimeWhatsAppLoginTool(
...args: Parameters<WhatsAppLightModule["createWhatsAppLoginTool"]>
): ReturnType<WhatsAppLightModule["createWhatsAppLoginTool"]> {
...args: Parameters<WhatsAppLightRuntimeModule["createWhatsAppLoginTool"]>
): ReturnType<WhatsAppLightRuntimeModule["createWhatsAppLoginTool"]> {
return getLightExport("createWhatsAppLoginTool")(...args);
}
export function createWaSocket(
...args: Parameters<WhatsAppHeavyModule["createWaSocket"]>
): ReturnType<WhatsAppHeavyModule["createWaSocket"]> {
...args: Parameters<WhatsAppHeavyRuntimeModule["createWaSocket"]>
): ReturnType<WhatsAppHeavyRuntimeModule["createWaSocket"]> {
return loadWhatsAppHeavyModule().then((loaded) => loaded.createWaSocket(...args));
}
export function formatError(
...args: Parameters<WhatsAppLightModule["formatError"]>
): ReturnType<WhatsAppLightModule["formatError"]> {
...args: Parameters<WhatsAppLightRuntimeModule["formatError"]>
): ReturnType<WhatsAppLightRuntimeModule["formatError"]> {
return getLightExport("formatError")(...args);
}
export function getStatusCode(
...args: Parameters<WhatsAppLightModule["getStatusCode"]>
): ReturnType<WhatsAppLightModule["getStatusCode"]> {
...args: Parameters<WhatsAppLightRuntimeModule["getStatusCode"]>
): ReturnType<WhatsAppLightRuntimeModule["getStatusCode"]> {
return getLightExport("getStatusCode")(...args);
}
export function pickWebChannel(
...args: Parameters<WhatsAppLightModule["pickWebChannel"]>
): ReturnType<WhatsAppLightModule["pickWebChannel"]> {
...args: Parameters<WhatsAppLightRuntimeModule["pickWebChannel"]>
): ReturnType<WhatsAppLightRuntimeModule["pickWebChannel"]> {
return getLightExport("pickWebChannel")(...args);
}
export function resolveWaWebAuthDir(): WhatsAppLightModule["WA_WEB_AUTH_DIR"] {
export function resolveWaWebAuthDir(): WhatsAppLightRuntimeModule["WA_WEB_AUTH_DIR"] {
return getLightExport("WA_WEB_AUTH_DIR");
}
export async function handleWhatsAppAction(
...args: Parameters<WhatsAppHeavyModule["handleWhatsAppAction"]>
): ReturnType<WhatsAppHeavyModule["handleWhatsAppAction"]> {
...args: Parameters<WhatsAppHeavyRuntimeModule["handleWhatsAppAction"]>
): ReturnType<WhatsAppHeavyRuntimeModule["handleWhatsAppAction"]> {
return (await getHeavyExport("handleWhatsAppAction"))(...args);
}
@@ -221,14 +228,14 @@ export async function loadWebMediaRaw(
}
export function monitorWebChannel(
...args: Parameters<WhatsAppHeavyModule["monitorWebChannel"]>
): ReturnType<WhatsAppHeavyModule["monitorWebChannel"]> {
...args: Parameters<WhatsAppHeavyRuntimeModule["monitorWebChannel"]>
): ReturnType<WhatsAppHeavyRuntimeModule["monitorWebChannel"]> {
return loadWhatsAppHeavyModule().then((loaded) => loaded.monitorWebChannel(...args));
}
export async function monitorWebInbox(
...args: Parameters<WhatsAppHeavyModule["monitorWebInbox"]>
): ReturnType<WhatsAppHeavyModule["monitorWebInbox"]> {
...args: Parameters<WhatsAppHeavyRuntimeModule["monitorWebInbox"]>
): ReturnType<WhatsAppHeavyRuntimeModule["monitorWebInbox"]> {
return (await getHeavyExport("monitorWebInbox"))(...args);
}
@@ -239,34 +246,34 @@ export async function optimizeImageToJpeg(
}
export async function runWebHeartbeatOnce(
...args: Parameters<WhatsAppHeavyModule["runWebHeartbeatOnce"]>
): ReturnType<WhatsAppHeavyModule["runWebHeartbeatOnce"]> {
...args: Parameters<WhatsAppHeavyRuntimeModule["runWebHeartbeatOnce"]>
): ReturnType<WhatsAppHeavyRuntimeModule["runWebHeartbeatOnce"]> {
return (await getHeavyExport("runWebHeartbeatOnce"))(...args);
}
export async function startWebLoginWithQr(
...args: Parameters<WhatsAppHeavyModule["startWebLoginWithQr"]>
): ReturnType<WhatsAppHeavyModule["startWebLoginWithQr"]> {
...args: Parameters<WhatsAppHeavyRuntimeModule["startWebLoginWithQr"]>
): ReturnType<WhatsAppHeavyRuntimeModule["startWebLoginWithQr"]> {
return (await getHeavyExport("startWebLoginWithQr"))(...args);
}
export async function waitForWaConnection(
...args: Parameters<WhatsAppHeavyModule["waitForWaConnection"]>
): ReturnType<WhatsAppHeavyModule["waitForWaConnection"]> {
...args: Parameters<WhatsAppHeavyRuntimeModule["waitForWaConnection"]>
): ReturnType<WhatsAppHeavyRuntimeModule["waitForWaConnection"]> {
return (await getHeavyExport("waitForWaConnection"))(...args);
}
export async function waitForWebLogin(
...args: Parameters<WhatsAppHeavyModule["waitForWebLogin"]>
): ReturnType<WhatsAppHeavyModule["waitForWebLogin"]> {
...args: Parameters<WhatsAppHeavyRuntimeModule["waitForWebLogin"]>
): ReturnType<WhatsAppHeavyRuntimeModule["waitForWebLogin"]> {
return (await getHeavyExport("waitForWebLogin"))(...args);
}
export const extractMediaPlaceholder = (
...args: Parameters<WhatsAppHeavyModule["extractMediaPlaceholder"]>
...args: Parameters<WhatsAppHeavyRuntimeModule["extractMediaPlaceholder"]>
) => loadCurrentHeavyModuleSync().extractMediaPlaceholder(...args);
export const extractText = (...args: Parameters<WhatsAppHeavyModule["extractText"]>) =>
export const extractText = (...args: Parameters<WhatsAppHeavyRuntimeModule["extractText"]>) =>
loadCurrentHeavyModuleSync().extractText(...args);
export function getDefaultLocalRoots(

View File

@@ -0,0 +1,249 @@
import type { AgentToolResult } from "@mariozechner/pi-agent-core";
import type { AnyMessageContent, makeWASocket } from "@whiskeysockets/baileys";
import type { NormalizedLocation } from "../../channels/location.js";
import type { ChannelAgentTool } from "../../channels/plugins/types.core.js";
import type { OpenClawConfig } from "../../config/config.js";
import type { PollInput } from "../../polls.js";
import type { RuntimeEnv } from "../../runtime.js";
import type { WebChannel } from "../../utils.js";
export type ActiveWebSendOptions = {
gifPlayback?: boolean;
accountId?: string;
fileName?: string;
};
export type ActiveWebListener = {
sendMessage: (
to: string,
text: string,
mediaBuffer?: Buffer,
mediaType?: string,
options?: ActiveWebSendOptions,
) => Promise<{ messageId: string }>;
sendPoll: (to: string, poll: PollInput) => Promise<{ messageId: string }>;
sendReaction: (
chatJid: string,
messageId: string,
emoji: string,
fromMe: boolean,
participant?: string,
) => Promise<void>;
sendComposingTo: (to: string) => Promise<void>;
close?: () => Promise<void>;
};
export type WebListenerCloseReason = {
status?: number;
isLoggedOut: boolean;
error?: unknown;
};
export type WebInboundMessage = {
id?: string;
from: string;
conversationId: string;
to: string;
accountId: string;
body: string;
pushName?: string;
timestamp?: number;
chatType: "direct" | "group";
chatId: string;
sender?: unknown;
senderJid?: string;
senderE164?: string;
senderName?: string;
replyTo?: unknown;
replyToId?: string;
replyToBody?: string;
replyToSender?: string;
replyToSenderJid?: string;
replyToSenderE164?: string;
groupSubject?: string;
groupParticipants?: string[];
mentions?: string[];
mentionedJids?: string[];
self?: unknown;
selfJid?: string | null;
selfLid?: string | null;
selfE164?: string | null;
fromMe?: boolean;
location?: NormalizedLocation;
sendComposing: () => Promise<void>;
reply: (text: string) => Promise<void>;
sendMedia: (payload: AnyMessageContent) => Promise<void>;
mediaPath?: string;
mediaType?: string;
mediaFileName?: string;
mediaUrl?: string;
wasMentioned?: boolean;
};
export type WebChannelHealthState =
| "starting"
| "healthy"
| "stale"
| "reconnecting"
| "conflict"
| "logged-out"
| "stopped";
export type WebChannelStatus = {
running: boolean;
connected: boolean;
reconnectAttempts: number;
lastConnectedAt?: number | null;
lastDisconnect?: {
at: number;
status?: number;
error?: string;
loggedOut?: boolean;
} | null;
lastInboundAt?: number | null;
lastMessageAt?: number | null;
lastEventAt?: number | null;
lastError?: string | null;
healthState?: WebChannelHealthState;
};
export type WebMonitorTuning = {
reconnect?: Partial<{
enabled: boolean;
maxAttempts: number;
baseDelayMs: number;
maxDelayMs: number;
}>;
heartbeatSeconds?: number;
messageTimeoutMs?: number;
watchdogCheckMs?: number;
sleep?: (ms: number, signal?: AbortSignal) => Promise<void>;
statusSink?: (status: WebChannelStatus) => void;
accountId?: string;
debounceMs?: number;
};
export type MonitorWebInboxFactory = (options: {
verbose: boolean;
accountId: string;
authDir: string;
onMessage: (msg: WebInboundMessage) => Promise<void>;
mediaMaxMb?: number;
sendReadReceipts?: boolean;
debounceMs?: number;
shouldDebounce?: (msg: WebInboundMessage) => boolean;
}) => Promise<{
closeReason: Promise<WebListenerCloseReason>;
stop: () => Promise<void>;
}>;
export type ReplyResolver = (...args: unknown[]) => Promise<unknown>;
export type WhatsAppWaSocket = ReturnType<typeof makeWASocket>;
export type WhatsAppLightRuntimeModule = {
getActiveWebListener: (accountId?: string | null) => ActiveWebListener | null;
getWebAuthAgeMs: (authDir?: string) => number | null;
logWebSelfId: (authDir?: string, runtime?: RuntimeEnv, includeChannelPrefix?: boolean) => void;
logoutWeb: (params: {
authDir?: string;
isLegacyAuthDir?: boolean;
runtime?: RuntimeEnv;
}) => Promise<boolean>;
readWebSelfId: (authDir?: string) => {
e164: string | null;
jid: string | null;
lid: string | null;
};
webAuthExists: (authDir?: string) => Promise<boolean>;
createWhatsAppLoginTool: () => ChannelAgentTool;
formatError: (err: unknown) => string;
getStatusCode: (err: unknown) => number | undefined;
pickWebChannel: (pref: WebChannel | "auto", authDir?: string) => Promise<WebChannel>;
WA_WEB_AUTH_DIR: string;
};
export type WhatsAppHeavyRuntimeModule = {
loginWeb: (
verbose: boolean,
waitForConnection?: (sock: WhatsAppWaSocket) => Promise<void>,
runtime?: RuntimeEnv,
accountId?: string,
) => Promise<void>;
sendMessageWhatsApp: (
to: string,
body: string,
options: {
verbose: boolean;
cfg?: OpenClawConfig;
mediaUrl?: string;
mediaLocalRoots?: readonly string[];
gifPlayback?: boolean;
accountId?: string;
},
) => Promise<{ messageId: string; toJid: string }>;
sendPollWhatsApp: (
to: string,
poll: PollInput,
options: { verbose: boolean; accountId?: string; cfg?: OpenClawConfig },
) => Promise<{ messageId: string; toJid: string }>;
sendReactionWhatsApp: (
chatJid: string,
messageId: string,
emoji: string,
options: {
verbose: boolean;
fromMe?: boolean;
participant?: string;
accountId?: string;
},
) => Promise<void>;
createWaSocket: (
printQr: boolean,
verbose: boolean,
opts?: { authDir?: string; onQr?: (qr: string) => void },
) => Promise<WhatsAppWaSocket>;
handleWhatsAppAction: (
params: Record<string, unknown>,
cfg: OpenClawConfig,
) => Promise<AgentToolResult<unknown>>;
monitorWebChannel: (
verbose: boolean,
listenerFactory?: MonitorWebInboxFactory,
keepAlive?: boolean,
replyResolver?: ReplyResolver,
runtime?: RuntimeEnv,
abortSignal?: AbortSignal,
tuning?: WebMonitorTuning,
) => Promise<void>;
monitorWebInbox: MonitorWebInboxFactory;
runWebHeartbeatOnce: (opts: {
cfg?: OpenClawConfig;
to: string;
verbose?: boolean;
replyResolver?: ReplyResolver;
sender?: WhatsAppHeavyRuntimeModule["sendMessageWhatsApp"];
sessionId?: string;
overrideBody?: string;
dryRun?: boolean;
}) => Promise<void>;
startWebLoginWithQr: (opts?: {
verbose?: boolean;
timeoutMs?: number;
force?: boolean;
accountId?: string;
runtime?: RuntimeEnv;
}) => Promise<{ qrDataUrl?: string; message: string }>;
waitForWaConnection: (sock: WhatsAppWaSocket) => Promise<void>;
waitForWebLogin: (opts?: {
timeoutMs?: number;
runtime?: RuntimeEnv;
accountId?: string;
}) => Promise<{ connected: boolean; message: string }>;
extractMediaPlaceholder: (
message: unknown,
mediaDir: string,
verbose?: boolean,
) => Promise<string | null>;
extractText: (message: unknown) => string;
};

View File

@@ -1,6 +1,6 @@
import { normalizeIMessageHandle } from "../../extensions/imessage/api.js";
import { imessageOutbound } from "../../test/channel-outbounds.js";
import type { ChannelOutboundAdapter, ChannelPlugin } from "../channels/plugins/types.js";
import { normalizeIMessageHandle } from "../plugin-sdk/imessage-targets.js";
import { collectStatusIssuesFromLastError } from "../plugin-sdk/status-helpers.js";
export const createIMessageTestPlugin = (params?: {

View File

@@ -1,857 +1,36 @@
import { randomBytes } from "node:crypto";
import {
existsSync,
mkdirSync,
readFileSync,
writeFileSync,
mkdtempSync,
renameSync,
unlinkSync,
} from "node:fs";
import path from "node:path";
import { resolveSendableOutboundReplyParts } from "openclaw/plugin-sdk/reply-payload";
import type { ReplyPayload } from "../auto-reply/types.js";
import { normalizeChannelId } from "../channels/plugins/index.js";
import type { ChannelId } from "../channels/plugins/types.js";
import type { OpenClawConfig } from "../config/config.js";
import type {
TtsConfig,
TtsAutoMode,
TtsMode,
TtsProvider,
TtsModelOverrideConfig,
} from "../config/types.tts.js";
import { logVerbose } from "../globals.js";
import { resolvePreferredOpenClawTmpDir } from "../infra/tmp-openclaw-dir.js";
import { stripMarkdown } from "../shared/text/strip-markdown.js";
import { CONFIG_DIR, resolveUserPath } from "../utils.js";
import { parseTtsDirectives } from "./directives.js";
import {
canonicalizeSpeechProviderId,
getSpeechProvider,
listSpeechProviders,
} from "./provider-registry.js";
import type {
SpeechModelOverridePolicy,
SpeechProviderConfig,
SpeechVoiceOption,
import * as speechRuntime from "../plugin-sdk/speech-runtime.js";
export const buildTtsSystemPromptHint = speechRuntime.buildTtsSystemPromptHint;
export const getLastTtsAttempt = speechRuntime.getLastTtsAttempt;
export const getResolvedSpeechProviderConfig = speechRuntime.getResolvedSpeechProviderConfig;
export const getTtsMaxLength = speechRuntime.getTtsMaxLength;
export const getTtsProvider = speechRuntime.getTtsProvider;
export const isSummarizationEnabled = speechRuntime.isSummarizationEnabled;
export const isTtsEnabled = speechRuntime.isTtsEnabled;
export const isTtsProviderConfigured = speechRuntime.isTtsProviderConfigured;
export const listSpeechVoices = speechRuntime.listSpeechVoices;
export const maybeApplyTtsToPayload = speechRuntime.maybeApplyTtsToPayload;
export const resolveTtsAutoMode = speechRuntime.resolveTtsAutoMode;
export const resolveTtsConfig = speechRuntime.resolveTtsConfig;
export const resolveTtsPrefsPath = speechRuntime.resolveTtsPrefsPath;
export const resolveTtsProviderOrder = speechRuntime.resolveTtsProviderOrder;
export const setLastTtsAttempt = speechRuntime.setLastTtsAttempt;
export const setSummarizationEnabled = speechRuntime.setSummarizationEnabled;
export const setTtsAutoMode = speechRuntime.setTtsAutoMode;
export const setTtsEnabled = speechRuntime.setTtsEnabled;
export const setTtsMaxLength = speechRuntime.setTtsMaxLength;
export const setTtsProvider = speechRuntime.setTtsProvider;
export const synthesizeSpeech = speechRuntime.synthesizeSpeech;
export const textToSpeech = speechRuntime.textToSpeech;
export const textToSpeechTelephony = speechRuntime.textToSpeechTelephony;
export const _test = speechRuntime._test;
export type {
ResolvedTtsConfig,
ResolvedTtsModelOverrides,
TtsDirectiveOverrides,
TtsDirectiveParseResult,
} from "./provider-types.js";
import { normalizeTtsAutoMode } from "./tts-auto-mode.js";
import { scheduleCleanup, summarizeText } from "./tts-core.js";
export type { TtsDirectiveOverrides, TtsDirectiveParseResult } from "./provider-types.js";
const DEFAULT_TIMEOUT_MS = 30_000;
const DEFAULT_TTS_MAX_LENGTH = 1500;
const DEFAULT_TTS_SUMMARIZE = true;
const DEFAULT_MAX_TEXT_LENGTH = 4096;
export type ResolvedTtsConfig = {
auto: TtsAutoMode;
mode: TtsMode;
provider: TtsProvider;
providerSource: "config" | "default";
summaryModel?: string;
modelOverrides: ResolvedTtsModelOverrides;
providerConfigs: Record<string, SpeechProviderConfig>;
prefsPath?: string;
maxTextLength: number;
timeoutMs: number;
};
type TtsUserPrefs = {
tts?: {
auto?: TtsAutoMode;
enabled?: boolean;
provider?: TtsProvider;
maxLength?: number;
summarize?: boolean;
};
};
export type ResolvedTtsModelOverrides = SpeechModelOverridePolicy;
export type TtsResult = {
success: boolean;
audioPath?: string;
error?: string;
latencyMs?: number;
provider?: string;
outputFormat?: string;
voiceCompatible?: boolean;
};
export type TtsSynthesisResult = {
success: boolean;
audioBuffer?: Buffer;
error?: string;
latencyMs?: number;
provider?: string;
outputFormat?: string;
voiceCompatible?: boolean;
fileExtension?: string;
};
export type TtsTelephonyResult = {
success: boolean;
audioBuffer?: Buffer;
error?: string;
latencyMs?: number;
provider?: string;
outputFormat?: string;
sampleRate?: number;
};
type TtsStatusEntry = {
timestamp: number;
success: boolean;
textLength: number;
summarized: boolean;
provider?: string;
latencyMs?: number;
error?: string;
};
let lastTtsAttempt: TtsStatusEntry | undefined;
function resolveModelOverridePolicy(
overrides: TtsModelOverrideConfig | undefined,
): ResolvedTtsModelOverrides {
const enabled = overrides?.enabled ?? true;
if (!enabled) {
return {
enabled: false,
allowText: false,
allowProvider: false,
allowVoice: false,
allowModelId: false,
allowVoiceSettings: false,
allowNormalization: false,
allowSeed: false,
};
}
const allow = (value: boolean | undefined, defaultValue = true) => value ?? defaultValue;
return {
enabled: true,
allowText: allow(overrides?.allowText),
// Provider switching is higher-impact than voice/style tweaks; keep opt-in.
allowProvider: allow(overrides?.allowProvider, false),
allowVoice: allow(overrides?.allowVoice),
allowModelId: allow(overrides?.allowModelId),
allowVoiceSettings: allow(overrides?.allowVoiceSettings),
allowNormalization: allow(overrides?.allowNormalization),
allowSeed: allow(overrides?.allowSeed),
};
}
function sortSpeechProvidersForAutoSelection(cfg?: OpenClawConfig) {
return listSpeechProviders(cfg).toSorted((left, right) => {
const leftOrder = left.autoSelectOrder ?? Number.MAX_SAFE_INTEGER;
const rightOrder = right.autoSelectOrder ?? Number.MAX_SAFE_INTEGER;
if (leftOrder !== rightOrder) {
return leftOrder - rightOrder;
}
return left.id.localeCompare(right.id);
});
}
function resolveRegistryDefaultSpeechProviderId(cfg?: OpenClawConfig): TtsProvider {
return sortSpeechProvidersForAutoSelection(cfg)[0]?.id ?? "";
}
function asProviderConfig(value: unknown): SpeechProviderConfig {
return typeof value === "object" && value !== null && !Array.isArray(value)
? (value as SpeechProviderConfig)
: {};
}
function asProviderConfigMap(value: unknown): Record<string, unknown> {
return typeof value === "object" && value !== null && !Array.isArray(value)
? (value as Record<string, unknown>)
: {};
}
function resolveSpeechProviderConfigs(
raw: TtsConfig,
cfg: OpenClawConfig,
timeoutMs: number,
): Record<string, SpeechProviderConfig> {
const providerConfigs: Record<string, SpeechProviderConfig> = {};
const rawProviders = asProviderConfigMap(raw.providers);
for (const provider of listSpeechProviders(cfg)) {
providerConfigs[provider.id] =
provider.resolveConfig?.({
cfg,
rawConfig: {
...(raw as Record<string, unknown>),
providers: rawProviders,
},
timeoutMs,
}) ??
asProviderConfig(rawProviders[provider.id] ?? (raw as Record<string, unknown>)[provider.id]);
}
return providerConfigs;
}
export function getResolvedSpeechProviderConfig(
config: ResolvedTtsConfig,
providerId: string,
cfg?: OpenClawConfig,
): SpeechProviderConfig {
const canonical =
canonicalizeSpeechProviderId(providerId, cfg) ?? providerId.trim().toLowerCase();
return config.providerConfigs[canonical] ?? {};
}
export function resolveTtsConfig(cfg: OpenClawConfig): ResolvedTtsConfig {
const raw: TtsConfig = cfg.messages?.tts ?? {};
const providerSource = raw.provider ? "config" : "default";
const timeoutMs = raw.timeoutMs ?? DEFAULT_TIMEOUT_MS;
const auto = normalizeTtsAutoMode(raw.auto) ?? (raw.enabled ? "always" : "off");
return {
auto,
mode: raw.mode ?? "final",
provider:
canonicalizeSpeechProviderId(raw.provider, cfg) ??
resolveRegistryDefaultSpeechProviderId(cfg),
providerSource,
summaryModel: raw.summaryModel?.trim() || undefined,
modelOverrides: resolveModelOverridePolicy(raw.modelOverrides),
providerConfigs: resolveSpeechProviderConfigs(raw, cfg, timeoutMs),
prefsPath: raw.prefsPath,
maxTextLength: raw.maxTextLength ?? DEFAULT_MAX_TEXT_LENGTH,
timeoutMs,
};
}
export function resolveTtsPrefsPath(config: ResolvedTtsConfig): string {
if (config.prefsPath?.trim()) {
return resolveUserPath(config.prefsPath.trim());
}
const envPath = process.env.OPENCLAW_TTS_PREFS?.trim();
if (envPath) {
return resolveUserPath(envPath);
}
return path.join(CONFIG_DIR, "settings", "tts.json");
}
function resolveTtsAutoModeFromPrefs(prefs: TtsUserPrefs): TtsAutoMode | undefined {
const auto = normalizeTtsAutoMode(prefs.tts?.auto);
if (auto) {
return auto;
}
if (typeof prefs.tts?.enabled === "boolean") {
return prefs.tts.enabled ? "always" : "off";
}
return undefined;
}
export function resolveTtsAutoMode(params: {
config: ResolvedTtsConfig;
prefsPath: string;
sessionAuto?: string;
}): TtsAutoMode {
const sessionAuto = normalizeTtsAutoMode(params.sessionAuto);
if (sessionAuto) {
return sessionAuto;
}
const prefsAuto = resolveTtsAutoModeFromPrefs(readPrefs(params.prefsPath));
if (prefsAuto) {
return prefsAuto;
}
return params.config.auto;
}
export function buildTtsSystemPromptHint(cfg: OpenClawConfig): string | undefined {
const config = resolveTtsConfig(cfg);
const prefsPath = resolveTtsPrefsPath(config);
const autoMode = resolveTtsAutoMode({ config, prefsPath });
if (autoMode === "off") {
return undefined;
}
const maxLength = getTtsMaxLength(prefsPath);
const summarize = isSummarizationEnabled(prefsPath) ? "on" : "off";
const autoHint =
autoMode === "inbound"
? "Only use TTS when the user's last message includes audio/voice."
: autoMode === "tagged"
? "Only use TTS when you include [[tts]] or [[tts:text]] tags."
: undefined;
return [
"Voice (TTS) is enabled.",
autoHint,
`Keep spoken text ≤${maxLength} chars to avoid auto-summary (summary ${summarize}).`,
"Use [[tts:...]] and optional [[tts:text]]...[[/tts:text]] to control voice/expressiveness.",
]
.filter(Boolean)
.join("\n");
}
function readPrefs(prefsPath: string): TtsUserPrefs {
try {
if (!existsSync(prefsPath)) {
return {};
}
return JSON.parse(readFileSync(prefsPath, "utf8")) as TtsUserPrefs;
} catch {
return {};
}
}
function atomicWriteFileSync(filePath: string, content: string): void {
const tmpPath = `${filePath}.tmp.${Date.now()}.${randomBytes(8).toString("hex")}`;
writeFileSync(tmpPath, content, { mode: 0o600 });
try {
renameSync(tmpPath, filePath);
} catch (err) {
try {
unlinkSync(tmpPath);
} catch {
// ignore
}
throw err;
}
}
function updatePrefs(prefsPath: string, update: (prefs: TtsUserPrefs) => void): void {
const prefs = readPrefs(prefsPath);
update(prefs);
mkdirSync(path.dirname(prefsPath), { recursive: true });
atomicWriteFileSync(prefsPath, JSON.stringify(prefs, null, 2));
}
export function isTtsEnabled(
config: ResolvedTtsConfig,
prefsPath: string,
sessionAuto?: string,
): boolean {
return resolveTtsAutoMode({ config, prefsPath, sessionAuto }) !== "off";
}
export function setTtsAutoMode(prefsPath: string, mode: TtsAutoMode): void {
updatePrefs(prefsPath, (prefs) => {
const next = { ...prefs.tts };
delete next.enabled;
next.auto = mode;
prefs.tts = next;
});
}
export function setTtsEnabled(prefsPath: string, enabled: boolean): void {
setTtsAutoMode(prefsPath, enabled ? "always" : "off");
}
export function getTtsProvider(config: ResolvedTtsConfig, prefsPath: string): TtsProvider {
const prefs = readPrefs(prefsPath);
const prefsProvider = canonicalizeSpeechProviderId(prefs.tts?.provider);
if (prefsProvider) {
return prefsProvider;
}
if (config.providerSource === "config") {
return canonicalizeSpeechProviderId(config.provider) ?? config.provider;
}
for (const provider of sortSpeechProvidersForAutoSelection()) {
if (
provider.isConfigured({
providerConfig: config.providerConfigs[provider.id] ?? {},
timeoutMs: config.timeoutMs,
})
) {
return provider.id;
}
}
return config.provider;
}
export function setTtsProvider(prefsPath: string, provider: TtsProvider): void {
updatePrefs(prefsPath, (prefs) => {
prefs.tts = { ...prefs.tts, provider: canonicalizeSpeechProviderId(provider) ?? provider };
});
}
export function getTtsMaxLength(prefsPath: string): number {
const prefs = readPrefs(prefsPath);
return prefs.tts?.maxLength ?? DEFAULT_TTS_MAX_LENGTH;
}
export function setTtsMaxLength(prefsPath: string, maxLength: number): void {
updatePrefs(prefsPath, (prefs) => {
prefs.tts = { ...prefs.tts, maxLength };
});
}
export function isSummarizationEnabled(prefsPath: string): boolean {
const prefs = readPrefs(prefsPath);
return prefs.tts?.summarize ?? DEFAULT_TTS_SUMMARIZE;
}
export function setSummarizationEnabled(prefsPath: string, enabled: boolean): void {
updatePrefs(prefsPath, (prefs) => {
prefs.tts = { ...prefs.tts, summarize: enabled };
});
}
export function getLastTtsAttempt(): TtsStatusEntry | undefined {
return lastTtsAttempt;
}
export function setLastTtsAttempt(entry: TtsStatusEntry | undefined): void {
lastTtsAttempt = entry;
}
/** Channels that require voice-note-compatible audio */
const OPUS_CHANNELS = new Set(["telegram", "feishu", "whatsapp", "matrix"]);
function resolveChannelId(channel: string | undefined): ChannelId | null {
return channel ? normalizeChannelId(channel) : null;
}
export function resolveTtsProviderOrder(primary: TtsProvider, cfg?: OpenClawConfig): TtsProvider[] {
const normalizedPrimary = canonicalizeSpeechProviderId(primary, cfg) ?? primary;
const ordered = new Set<TtsProvider>([normalizedPrimary]);
for (const provider of sortSpeechProvidersForAutoSelection(cfg)) {
const normalized = provider.id;
if (normalized !== normalizedPrimary) {
ordered.add(normalized);
}
}
return [...ordered];
}
export function isTtsProviderConfigured(
config: ResolvedTtsConfig,
provider: TtsProvider,
cfg?: OpenClawConfig,
): boolean {
const resolvedProvider = getSpeechProvider(provider, cfg);
if (!resolvedProvider) {
return false;
}
return (
resolvedProvider.isConfigured({
cfg,
providerConfig: getResolvedSpeechProviderConfig(config, resolvedProvider.id, cfg),
timeoutMs: config.timeoutMs,
}) ?? false
);
}
function formatTtsProviderError(provider: TtsProvider, err: unknown): string {
const error = err instanceof Error ? err : new Error(String(err));
if (error.name === "AbortError") {
return `${provider}: request timed out`;
}
return `${provider}: ${error.message}`;
}
function buildTtsFailureResult(errors: string[]): { success: false; error: string } {
return {
success: false,
error: `TTS conversion failed: ${errors.join("; ") || "no providers available"}`,
};
}
function resolveReadySpeechProvider(params: {
provider: TtsProvider;
cfg: OpenClawConfig;
config: ResolvedTtsConfig;
errors: string[];
requireTelephony?: boolean;
}): NonNullable<ReturnType<typeof getSpeechProvider>> | null {
const resolvedProvider = getSpeechProvider(params.provider, params.cfg);
if (!resolvedProvider) {
params.errors.push(`${params.provider}: no provider registered`);
return null;
}
const providerConfig = getResolvedSpeechProviderConfig(
params.config,
resolvedProvider.id,
params.cfg,
);
if (
!resolvedProvider.isConfigured({
cfg: params.cfg,
providerConfig,
timeoutMs: params.config.timeoutMs,
})
) {
params.errors.push(`${params.provider}: not configured`);
return null;
}
if (params.requireTelephony && !resolvedProvider.synthesizeTelephony) {
params.errors.push(`${params.provider}: unsupported for telephony`);
return null;
}
return resolvedProvider;
}
function resolveTtsRequestSetup(params: {
text: string;
cfg: OpenClawConfig;
prefsPath?: string;
providerOverride?: TtsProvider;
disableFallback?: boolean;
}):
| {
config: ResolvedTtsConfig;
providers: TtsProvider[];
}
| {
error: string;
} {
const config = resolveTtsConfig(params.cfg);
const prefsPath = params.prefsPath ?? resolveTtsPrefsPath(config);
if (params.text.length > config.maxTextLength) {
return {
error: `Text too long (${params.text.length} chars, max ${config.maxTextLength})`,
};
}
const userProvider = getTtsProvider(config, prefsPath);
const provider =
canonicalizeSpeechProviderId(params.providerOverride, params.cfg) ?? userProvider;
return {
config,
providers: params.disableFallback ? [provider] : resolveTtsProviderOrder(provider, params.cfg),
};
}
export async function textToSpeech(params: {
text: string;
cfg: OpenClawConfig;
prefsPath?: string;
channel?: string;
overrides?: TtsDirectiveOverrides;
disableFallback?: boolean;
}): Promise<TtsResult> {
const synthesis = await synthesizeSpeech(params);
if (!synthesis.success || !synthesis.audioBuffer || !synthesis.fileExtension) {
return buildTtsFailureResult([synthesis.error ?? "TTS conversion failed"]);
}
const tempRoot = resolvePreferredOpenClawTmpDir();
mkdirSync(tempRoot, { recursive: true, mode: 0o700 });
const tempDir = mkdtempSync(path.join(tempRoot, "tts-"));
const audioPath = path.join(tempDir, `voice-${Date.now()}${synthesis.fileExtension}`);
writeFileSync(audioPath, synthesis.audioBuffer);
scheduleCleanup(tempDir);
return {
success: true,
audioPath,
latencyMs: synthesis.latencyMs,
provider: synthesis.provider,
outputFormat: synthesis.outputFormat,
voiceCompatible: synthesis.voiceCompatible,
};
}
export async function synthesizeSpeech(params: {
text: string;
cfg: OpenClawConfig;
prefsPath?: string;
channel?: string;
overrides?: TtsDirectiveOverrides;
disableFallback?: boolean;
}): Promise<TtsSynthesisResult> {
const setup = resolveTtsRequestSetup({
text: params.text,
cfg: params.cfg,
prefsPath: params.prefsPath,
providerOverride: params.overrides?.provider,
disableFallback: params.disableFallback,
});
if ("error" in setup) {
return { success: false, error: setup.error };
}
const { config, providers } = setup;
const channelId = resolveChannelId(params.channel);
const target = channelId && OPUS_CHANNELS.has(channelId) ? "voice-note" : "audio-file";
const errors: string[] = [];
for (const provider of providers) {
const providerStart = Date.now();
try {
const resolvedProvider = resolveReadySpeechProvider({
provider,
cfg: params.cfg,
config,
errors,
});
if (!resolvedProvider) {
continue;
}
const synthesis = await resolvedProvider.synthesize({
text: params.text,
cfg: params.cfg,
providerConfig: getResolvedSpeechProviderConfig(config, resolvedProvider.id, params.cfg),
target,
providerOverrides: params.overrides?.providerOverrides?.[resolvedProvider.id],
timeoutMs: config.timeoutMs,
});
return {
success: true,
audioBuffer: synthesis.audioBuffer,
latencyMs: Date.now() - providerStart,
provider,
outputFormat: synthesis.outputFormat,
voiceCompatible: synthesis.voiceCompatible,
fileExtension: synthesis.fileExtension,
};
} catch (err) {
errors.push(formatTtsProviderError(provider, err));
}
}
return buildTtsFailureResult(errors);
}
export async function textToSpeechTelephony(params: {
text: string;
cfg: OpenClawConfig;
prefsPath?: string;
}): Promise<TtsTelephonyResult> {
const setup = resolveTtsRequestSetup({
text: params.text,
cfg: params.cfg,
prefsPath: params.prefsPath,
});
if ("error" in setup) {
return { success: false, error: setup.error };
}
const { config, providers } = setup;
const errors: string[] = [];
for (const provider of providers) {
const providerStart = Date.now();
try {
const resolvedProvider = resolveReadySpeechProvider({
provider,
cfg: params.cfg,
config,
errors,
requireTelephony: true,
});
if (!resolvedProvider?.synthesizeTelephony) {
continue;
}
const synthesis = await resolvedProvider.synthesizeTelephony({
text: params.text,
cfg: params.cfg,
providerConfig: getResolvedSpeechProviderConfig(config, resolvedProvider.id, params.cfg),
timeoutMs: config.timeoutMs,
});
return {
success: true,
audioBuffer: synthesis.audioBuffer,
latencyMs: Date.now() - providerStart,
provider,
outputFormat: synthesis.outputFormat,
sampleRate: synthesis.sampleRate,
};
} catch (err) {
errors.push(formatTtsProviderError(provider, err));
}
}
return buildTtsFailureResult(errors);
}
export async function listSpeechVoices(params: {
provider: string;
cfg?: OpenClawConfig;
config?: ResolvedTtsConfig;
apiKey?: string;
baseUrl?: string;
}): Promise<SpeechVoiceOption[]> {
const provider = canonicalizeSpeechProviderId(params.provider, params.cfg);
if (!provider) {
throw new Error("speech provider id is required");
}
const config = params.config ?? (params.cfg ? resolveTtsConfig(params.cfg) : undefined);
if (!config) {
throw new Error(`speech provider ${provider} requires cfg or resolved config`);
}
const resolvedProvider = getSpeechProvider(provider, params.cfg);
if (!resolvedProvider) {
throw new Error(`speech provider ${provider} is not registered`);
}
if (!resolvedProvider.listVoices) {
throw new Error(`speech provider ${provider} does not support voice listing`);
}
return await resolvedProvider.listVoices({
cfg: params.cfg,
providerConfig: getResolvedSpeechProviderConfig(config, resolvedProvider.id, params.cfg),
apiKey: params.apiKey,
baseUrl: params.baseUrl,
});
}
export async function maybeApplyTtsToPayload(params: {
payload: ReplyPayload;
cfg: OpenClawConfig;
channel?: string;
kind?: "tool" | "block" | "final";
inboundAudio?: boolean;
ttsAuto?: string;
}): Promise<ReplyPayload> {
// Compaction notices are informational UI signals — never synthesise them as speech.
if (params.payload.isCompactionNotice) {
return params.payload;
}
const config = resolveTtsConfig(params.cfg);
const prefsPath = resolveTtsPrefsPath(config);
const autoMode = resolveTtsAutoMode({
config,
prefsPath,
sessionAuto: params.ttsAuto,
});
if (autoMode === "off") {
return params.payload;
}
const reply = resolveSendableOutboundReplyParts(params.payload);
const text = reply.text;
const directives = parseTtsDirectives(text, config.modelOverrides, {
cfg: params.cfg,
providerConfigs: config.providerConfigs,
});
if (directives.warnings.length > 0) {
logVerbose(`TTS: ignored directive overrides (${directives.warnings.join("; ")})`);
}
const cleanedText = directives.cleanedText;
const trimmedCleaned = cleanedText.trim();
const visibleText = trimmedCleaned.length > 0 ? trimmedCleaned : "";
const ttsText = directives.ttsText?.trim() || visibleText;
const nextPayload =
visibleText === text.trim()
? params.payload
: {
...params.payload,
text: visibleText.length > 0 ? visibleText : undefined,
};
if (autoMode === "tagged" && !directives.hasDirective) {
return nextPayload;
}
if (autoMode === "inbound" && params.inboundAudio !== true) {
return nextPayload;
}
const mode = config.mode ?? "final";
if (mode === "final" && params.kind && params.kind !== "final") {
return nextPayload;
}
if (!ttsText.trim()) {
return nextPayload;
}
if (reply.hasMedia) {
return nextPayload;
}
if (text.includes("MEDIA:")) {
return nextPayload;
}
if (ttsText.trim().length < 10) {
return nextPayload;
}
const maxLength = getTtsMaxLength(prefsPath);
let textForAudio = ttsText.trim();
let wasSummarized = false;
if (textForAudio.length > maxLength) {
if (!isSummarizationEnabled(prefsPath)) {
logVerbose(
`TTS: truncating long text (${textForAudio.length} > ${maxLength}), summarization disabled.`,
);
textForAudio = `${textForAudio.slice(0, maxLength - 3)}...`;
} else {
try {
const summary = await summarizeText({
text: textForAudio,
targetLength: maxLength,
cfg: params.cfg,
config,
timeoutMs: config.timeoutMs,
});
textForAudio = summary.summary;
wasSummarized = true;
if (textForAudio.length > config.maxTextLength) {
logVerbose(
`TTS: summary exceeded hard limit (${textForAudio.length} > ${config.maxTextLength}); truncating.`,
);
textForAudio = `${textForAudio.slice(0, config.maxTextLength - 3)}...`;
}
} catch (err) {
const error = err as Error;
logVerbose(`TTS: summarization failed, truncating instead: ${error.message}`);
textForAudio = `${textForAudio.slice(0, maxLength - 3)}...`;
}
}
}
textForAudio = stripMarkdown(textForAudio).trim(); // strip markdown for TTS (### → "hashtag" etc.)
if (textForAudio.length < 10) {
return nextPayload;
}
const ttsStart = Date.now();
const result = await textToSpeech({
text: textForAudio,
cfg: params.cfg,
prefsPath,
channel: params.channel,
overrides: directives.overrides,
});
if (result.success && result.audioPath) {
lastTtsAttempt = {
timestamp: Date.now(),
success: true,
textLength: text.length,
summarized: wasSummarized,
provider: result.provider,
latencyMs: result.latencyMs,
};
const channelId = resolveChannelId(params.channel);
const shouldVoice =
channelId !== null && OPUS_CHANNELS.has(channelId) && result.voiceCompatible === true;
const finalPayload = {
...nextPayload,
mediaUrl: result.audioPath,
audioAsVoice: shouldVoice || params.payload.audioAsVoice,
};
return finalPayload;
}
lastTtsAttempt = {
timestamp: Date.now(),
success: false,
textLength: text.length,
summarized: wasSummarized,
error: result.error,
};
const latency = Date.now() - ttsStart;
logVerbose(`TTS: conversion failed after ${latency}ms (${result.error ?? "unknown"}).`);
return nextPayload;
}
export const _test = {
parseTtsDirectives,
resolveModelOverridePolicy,
summarizeText,
getResolvedSpeechProviderConfig,
};
TtsResult,
TtsSynthesisResult,
TtsTelephonyResult,
} from "../plugin-sdk/speech-runtime.js";