mirror of
https://github.com/moltbot/moltbot.git
synced 2026-03-30 01:06:11 +00:00
refactor: route plugin runtime through bundled seams
This commit is contained in:
@@ -1522,6 +1522,15 @@
|
||||
"path": "src/channels/plugins/normalize/whatsapp.ts"
|
||||
}
|
||||
},
|
||||
{
|
||||
"declaration": "export function normalizeChannelId(raw?: string | null | undefined): ChannelId | null;",
|
||||
"exportName": "normalizeChannelId",
|
||||
"kind": "function",
|
||||
"source": {
|
||||
"line": 80,
|
||||
"path": "src/channels/plugins/registry.ts"
|
||||
}
|
||||
},
|
||||
{
|
||||
"declaration": "export function normalizeChatType(raw?: string | undefined): ChatType | undefined;",
|
||||
"exportName": "normalizeChatType",
|
||||
|
||||
@@ -166,6 +166,7 @@
|
||||
{"declaration":"export function keepHttpServerTaskAlive(params: { server: CloseAwareServer; abortSignal?: AbortSignal | undefined; onAbort?: (() => void | Promise<void>) | undefined; }): Promise<void>;","entrypoint":"channel-runtime","exportName":"keepHttpServerTaskAlive","importSpecifier":"openclaw/plugin-sdk/channel-runtime","kind":"function","recordType":"export","sourceLine":79,"sourcePath":"src/plugin-sdk/channel-lifecycle.ts"}
|
||||
{"declaration":"export function looksLikeSignalTargetId(raw: string, normalized?: string | undefined): boolean;","entrypoint":"channel-runtime","exportName":"looksLikeSignalTargetId","importSpecifier":"openclaw/plugin-sdk/channel-runtime","kind":"function","recordType":"export","sourceLine":38,"sourcePath":"src/channels/plugins/normalize/signal.ts"}
|
||||
{"declaration":"export function looksLikeWhatsAppTargetId(raw: string): boolean;","entrypoint":"channel-runtime","exportName":"looksLikeWhatsAppTargetId","importSpecifier":"openclaw/plugin-sdk/channel-runtime","kind":"function","recordType":"export","sourceLine":20,"sourcePath":"src/channels/plugins/normalize/whatsapp.ts"}
|
||||
{"declaration":"export function normalizeChannelId(raw?: string | null | undefined): ChannelId | null;","entrypoint":"channel-runtime","exportName":"normalizeChannelId","importSpecifier":"openclaw/plugin-sdk/channel-runtime","kind":"function","recordType":"export","sourceLine":80,"sourcePath":"src/channels/plugins/registry.ts"}
|
||||
{"declaration":"export function normalizeChatType(raw?: string | undefined): ChatType | undefined;","entrypoint":"channel-runtime","exportName":"normalizeChatType","importSpecifier":"openclaw/plugin-sdk/channel-runtime","kind":"function","recordType":"export","sourceLine":3,"sourcePath":"src/channels/chat-type.ts"}
|
||||
{"declaration":"export function normalizePollDurationHours(value: number | undefined, options: { defaultHours: number; maxHours: number; }): number;","entrypoint":"channel-runtime","exportName":"normalizePollDurationHours","importSpecifier":"openclaw/plugin-sdk/channel-runtime","kind":"function","recordType":"export","sourceLine":93,"sourcePath":"src/polls.ts"}
|
||||
{"declaration":"export function normalizePollInput(input: PollInput, options?: NormalizePollOptions): NormalizedPollInput;","entrypoint":"channel-runtime","exportName":"normalizePollInput","importSpecifier":"openclaw/plugin-sdk/channel-runtime","kind":"function","recordType":"export","sourceLine":36,"sourcePath":"src/polls.ts"}
|
||||
|
||||
1
extensions/anthropic/test-api.ts
Normal file
1
extensions/anthropic/test-api.ts
Normal file
@@ -0,0 +1 @@
|
||||
export { buildAnthropicCliBackend } from "./cli-backend.js";
|
||||
@@ -1,2 +1,6 @@
|
||||
export { bluebubblesPlugin } from "./src/channel.js";
|
||||
export {
|
||||
resolveBlueBubblesGroupRequireMention,
|
||||
resolveBlueBubblesGroupToolPolicy,
|
||||
} from "./src/group-policy.js";
|
||||
export { isAllowedBlueBubblesSender } from "./src/targets.js";
|
||||
|
||||
1
extensions/discord/action-runtime-api.ts
Normal file
1
extensions/discord/action-runtime-api.ts
Normal file
@@ -0,0 +1 @@
|
||||
export { handleDiscordAction } from "./src/actions/runtime.js";
|
||||
@@ -6,7 +6,7 @@ import {
|
||||
readStringParam,
|
||||
} from "openclaw/plugin-sdk/agent-runtime";
|
||||
import type { ChannelMessageActionContext } from "openclaw/plugin-sdk/channel-contract";
|
||||
import { handleDiscordAction } from "./runtime.js";
|
||||
import { handleDiscordAction } from "../../action-runtime-api.js";
|
||||
import {
|
||||
isDiscordModerationAction,
|
||||
readDiscordModerationCommand,
|
||||
|
||||
@@ -8,10 +8,10 @@ import { readBooleanParam } from "openclaw/plugin-sdk/boolean-param";
|
||||
import { resolveReactionMessageId } from "openclaw/plugin-sdk/channel-actions";
|
||||
import type { ChannelMessageActionContext } from "openclaw/plugin-sdk/channel-contract";
|
||||
import { normalizeInteractiveReply } from "openclaw/plugin-sdk/interactive-runtime";
|
||||
import { handleDiscordAction } from "../../action-runtime-api.js";
|
||||
import { buildDiscordInteractiveComponents } from "../shared-interactive.js";
|
||||
import { resolveDiscordChannelId } from "../targets.js";
|
||||
import { tryHandleDiscordMessageActionGuildAdmin } from "./handle-action.guild-admin.js";
|
||||
import { handleDiscordAction } from "./runtime.js";
|
||||
import { readDiscordParentIdParam } from "./runtime.shared.js";
|
||||
|
||||
const providerId = "discord";
|
||||
|
||||
1
extensions/google/test-api.ts
Normal file
1
extensions/google/test-api.ts
Normal file
@@ -0,0 +1 @@
|
||||
export { buildGoogleGeminiCliBackend } from "./cli-backend.js";
|
||||
1
extensions/image-generation-core/api.ts
Normal file
1
extensions/image-generation-core/api.ts
Normal file
@@ -0,0 +1 @@
|
||||
export * from "openclaw/plugin-sdk/image-generation-core";
|
||||
7
extensions/image-generation-core/package.json
Normal file
7
extensions/image-generation-core/package.json
Normal file
@@ -0,0 +1,7 @@
|
||||
{
|
||||
"name": "@openclaw/image-generation-core",
|
||||
"version": "2026.3.26",
|
||||
"private": true,
|
||||
"description": "OpenClaw image generation runtime package",
|
||||
"type": "module"
|
||||
}
|
||||
6
extensions/image-generation-core/runtime-api.ts
Normal file
6
extensions/image-generation-core/runtime-api.ts
Normal file
@@ -0,0 +1,6 @@
|
||||
export {
|
||||
generateImage,
|
||||
listRuntimeImageGenerationProviders,
|
||||
type GenerateImageParams,
|
||||
type GenerateImageRuntimeResult,
|
||||
} from "./src/runtime.js";
|
||||
183
extensions/image-generation-core/src/runtime.ts
Normal file
183
extensions/image-generation-core/src/runtime.ts
Normal file
@@ -0,0 +1,183 @@
|
||||
import {
|
||||
createSubsystemLogger,
|
||||
describeFailoverError,
|
||||
getImageGenerationProvider,
|
||||
getProviderEnvVars,
|
||||
isFailoverError,
|
||||
listImageGenerationProviders,
|
||||
parseImageGenerationModelRef,
|
||||
resolveAgentModelFallbackValues,
|
||||
resolveAgentModelPrimaryValue,
|
||||
type AuthProfileStore,
|
||||
type FallbackAttempt,
|
||||
type GeneratedImageAsset,
|
||||
type ImageGenerationResolution,
|
||||
type ImageGenerationResult,
|
||||
type ImageGenerationSourceImage,
|
||||
type OpenClawConfig,
|
||||
} from "../api.js";
|
||||
|
||||
const log = createSubsystemLogger("image-generation");
|
||||
|
||||
export type GenerateImageParams = {
|
||||
cfg: OpenClawConfig;
|
||||
prompt: string;
|
||||
agentDir?: string;
|
||||
authStore?: AuthProfileStore;
|
||||
modelOverride?: string;
|
||||
count?: number;
|
||||
size?: string;
|
||||
aspectRatio?: string;
|
||||
resolution?: ImageGenerationResolution;
|
||||
inputImages?: ImageGenerationSourceImage[];
|
||||
};
|
||||
|
||||
export type GenerateImageRuntimeResult = {
|
||||
images: GeneratedImageAsset[];
|
||||
provider: string;
|
||||
model: string;
|
||||
attempts: FallbackAttempt[];
|
||||
metadata?: Record<string, unknown>;
|
||||
};
|
||||
|
||||
function resolveImageGenerationCandidates(params: {
|
||||
cfg: OpenClawConfig;
|
||||
modelOverride?: string;
|
||||
}): Array<{ provider: string; model: string }> {
|
||||
const candidates: Array<{ provider: string; model: string }> = [];
|
||||
const seen = new Set<string>();
|
||||
const add = (raw: string | undefined) => {
|
||||
const parsed = parseImageGenerationModelRef(raw);
|
||||
if (!parsed) {
|
||||
return;
|
||||
}
|
||||
const key = `${parsed.provider}/${parsed.model}`;
|
||||
if (seen.has(key)) {
|
||||
return;
|
||||
}
|
||||
seen.add(key);
|
||||
candidates.push(parsed);
|
||||
};
|
||||
|
||||
add(params.modelOverride);
|
||||
add(resolveAgentModelPrimaryValue(params.cfg.agents?.defaults?.imageGenerationModel));
|
||||
for (const fallback of resolveAgentModelFallbackValues(
|
||||
params.cfg.agents?.defaults?.imageGenerationModel,
|
||||
)) {
|
||||
add(fallback);
|
||||
}
|
||||
return candidates;
|
||||
}
|
||||
|
||||
function throwImageGenerationFailure(params: {
|
||||
attempts: FallbackAttempt[];
|
||||
lastError: unknown;
|
||||
}): never {
|
||||
if (params.attempts.length <= 1 && params.lastError) {
|
||||
throw params.lastError;
|
||||
}
|
||||
const summary =
|
||||
params.attempts.length > 0
|
||||
? params.attempts
|
||||
.map((attempt) => `${attempt.provider}/${attempt.model}: ${attempt.error}`)
|
||||
.join(" | ")
|
||||
: "unknown";
|
||||
throw new Error(`All image generation models failed (${params.attempts.length}): ${summary}`, {
|
||||
cause: params.lastError instanceof Error ? params.lastError : undefined,
|
||||
});
|
||||
}
|
||||
|
||||
function buildNoImageGenerationModelConfiguredMessage(cfg: OpenClawConfig): string {
|
||||
const providers = listImageGenerationProviders(cfg);
|
||||
const sampleModel =
|
||||
providers.find((provider) => provider.defaultModel) ??
|
||||
({ id: "google", defaultModel: "gemini-3-pro-image-preview" } as const);
|
||||
const authHints = providers
|
||||
.flatMap((provider) => {
|
||||
const envVars = getProviderEnvVars(provider.id);
|
||||
if (envVars.length === 0) {
|
||||
return [];
|
||||
}
|
||||
return [`${provider.id}: ${envVars.join(" / ")}`];
|
||||
})
|
||||
.slice(0, 3);
|
||||
return [
|
||||
`No image-generation model configured. Set agents.defaults.imageGenerationModel.primary to a provider/model like "${sampleModel.id}/${sampleModel.defaultModel}".`,
|
||||
authHints.length > 0
|
||||
? `If you want a specific provider, also configure that provider's auth/API key first (${authHints.join("; ")}).`
|
||||
: "If you want a specific provider, also configure that provider's auth/API key first.",
|
||||
].join(" ");
|
||||
}
|
||||
|
||||
export function listRuntimeImageGenerationProviders(params?: { config?: OpenClawConfig }) {
|
||||
return listImageGenerationProviders(params?.config);
|
||||
}
|
||||
|
||||
export async function generateImage(
|
||||
params: GenerateImageParams,
|
||||
): Promise<GenerateImageRuntimeResult> {
|
||||
const candidates = resolveImageGenerationCandidates({
|
||||
cfg: params.cfg,
|
||||
modelOverride: params.modelOverride,
|
||||
});
|
||||
if (candidates.length === 0) {
|
||||
throw new Error(buildNoImageGenerationModelConfiguredMessage(params.cfg));
|
||||
}
|
||||
|
||||
const attempts: FallbackAttempt[] = [];
|
||||
let lastError: unknown;
|
||||
|
||||
for (const candidate of candidates) {
|
||||
const provider = getImageGenerationProvider(candidate.provider, params.cfg);
|
||||
if (!provider) {
|
||||
const error = `No image-generation provider registered for ${candidate.provider}`;
|
||||
attempts.push({
|
||||
provider: candidate.provider,
|
||||
model: candidate.model,
|
||||
error,
|
||||
});
|
||||
lastError = new Error(error);
|
||||
continue;
|
||||
}
|
||||
|
||||
try {
|
||||
const result: ImageGenerationResult = await provider.generateImage({
|
||||
provider: candidate.provider,
|
||||
model: candidate.model,
|
||||
prompt: params.prompt,
|
||||
cfg: params.cfg,
|
||||
agentDir: params.agentDir,
|
||||
authStore: params.authStore,
|
||||
count: params.count,
|
||||
size: params.size,
|
||||
aspectRatio: params.aspectRatio,
|
||||
resolution: params.resolution,
|
||||
inputImages: params.inputImages,
|
||||
});
|
||||
if (!Array.isArray(result.images) || result.images.length === 0) {
|
||||
throw new Error("Image generation provider returned no images.");
|
||||
}
|
||||
return {
|
||||
images: result.images,
|
||||
provider: candidate.provider,
|
||||
model: result.model ?? candidate.model,
|
||||
attempts,
|
||||
metadata: result.metadata,
|
||||
};
|
||||
} catch (err) {
|
||||
lastError = err;
|
||||
const described = isFailoverError(err) ? describeFailoverError(err) : undefined;
|
||||
attempts.push({
|
||||
provider: candidate.provider,
|
||||
model: candidate.model,
|
||||
error: described?.message ?? (err instanceof Error ? err.message : String(err)),
|
||||
reason: described?.reason,
|
||||
status: described?.status,
|
||||
code: described?.code,
|
||||
});
|
||||
log.debug(`image-generation candidate failed: ${candidate.provider}/${candidate.model}`);
|
||||
}
|
||||
}
|
||||
|
||||
throwImageGenerationFailure({ attempts, lastError });
|
||||
}
|
||||
@@ -1,11 +1,13 @@
|
||||
import { ToolPolicySchema } from "openclaw/plugin-sdk/agent-config-primitives";
|
||||
import {
|
||||
AllowFromListSchema,
|
||||
buildNestedDmConfigSchema,
|
||||
DmPolicySchema,
|
||||
GroupPolicySchema,
|
||||
} from "openclaw/plugin-sdk/channel-config-schema";
|
||||
MarkdownConfigSchema,
|
||||
} from "openclaw/plugin-sdk/channel-config-primitives";
|
||||
import { buildSecretInputSchema } from "openclaw/plugin-sdk/secret-input";
|
||||
import { z } from "openclaw/plugin-sdk/zod";
|
||||
import { buildSecretInputSchema, MarkdownConfigSchema, ToolPolicySchema } from "./runtime-api.js";
|
||||
|
||||
const matrixActionSchema = z
|
||||
.object({
|
||||
|
||||
@@ -20,7 +20,7 @@ import {
|
||||
createComputedAccountStatusAdapter,
|
||||
createDefaultChannelRuntimeState,
|
||||
} from "openclaw/plugin-sdk/status-helpers";
|
||||
import { MattermostConfigSchema } from "./config-schema.js";
|
||||
import { MattermostChannelConfigSchema } from "./config-surface.js";
|
||||
import { resolveMattermostGroupRequireMention } from "./group-mentions.js";
|
||||
import {
|
||||
listMattermostAccountIds,
|
||||
@@ -40,7 +40,6 @@ import { sendMessageMattermost } from "./mattermost/send.js";
|
||||
import { resolveMattermostOpaqueTarget } from "./mattermost/target-resolution.js";
|
||||
import { looksLikeMattermostTargetId, normalizeMattermostMessagingTarget } from "./normalize.js";
|
||||
import {
|
||||
buildChannelConfigSchema,
|
||||
createAccountStatusSink,
|
||||
DEFAULT_ACCOUNT_ID,
|
||||
resolveAllowlistProviderRuntimeGroupPolicy,
|
||||
@@ -306,7 +305,7 @@ export const mattermostPlugin: ChannelPlugin<ResolvedMattermostAccount> = create
|
||||
blockStreamingCoalesceDefaults: { minChars: 1500, idleMs: 1000 },
|
||||
},
|
||||
reload: { configPrefixes: ["channels.mattermost"] },
|
||||
configSchema: buildChannelConfigSchema(MattermostConfigSchema),
|
||||
configSchema: MattermostChannelConfigSchema,
|
||||
config: {
|
||||
...mattermostConfigAdapter,
|
||||
isConfigured: (account) => Boolean(account.botToken && account.baseUrl),
|
||||
|
||||
125
extensions/mattermost/src/config-schema-core.ts
Normal file
125
extensions/mattermost/src/config-schema-core.ts
Normal file
@@ -0,0 +1,125 @@
|
||||
import {
|
||||
BlockStreamingCoalesceSchema,
|
||||
DmPolicySchema,
|
||||
GroupPolicySchema,
|
||||
MarkdownConfigSchema,
|
||||
requireOpenAllowFrom,
|
||||
} from "openclaw/plugin-sdk/channel-config-primitives";
|
||||
import { z } from "openclaw/plugin-sdk/zod";
|
||||
import { buildSecretInputSchema } from "./secret-input.js";
|
||||
|
||||
function requireMattermostOpenAllowFrom(params: {
|
||||
policy?: string;
|
||||
allowFrom?: Array<string | number>;
|
||||
ctx: z.RefinementCtx;
|
||||
}) {
|
||||
requireOpenAllowFrom({
|
||||
policy: params.policy,
|
||||
allowFrom: params.allowFrom,
|
||||
ctx: params.ctx,
|
||||
path: ["allowFrom"],
|
||||
message:
|
||||
'channels.mattermost.dmPolicy="open" requires channels.mattermost.allowFrom to include "*"',
|
||||
});
|
||||
}
|
||||
|
||||
const DmChannelRetrySchema = z
|
||||
.object({
|
||||
/** Maximum number of retry attempts for DM channel creation (default: 3) */
|
||||
maxRetries: z.number().int().min(0).max(10).optional(),
|
||||
/** Initial delay in milliseconds before first retry (default: 1000) */
|
||||
initialDelayMs: z.number().int().min(100).max(60000).optional(),
|
||||
/** Maximum delay in milliseconds between retries (default: 10000) */
|
||||
maxDelayMs: z.number().int().min(1000).max(60000).optional(),
|
||||
/** Timeout for each individual DM channel creation request in milliseconds (default: 30000) */
|
||||
timeoutMs: z.number().int().min(5000).max(120000).optional(),
|
||||
})
|
||||
.strict()
|
||||
.refine(
|
||||
(data) => {
|
||||
if (data.initialDelayMs !== undefined && data.maxDelayMs !== undefined) {
|
||||
return data.initialDelayMs <= data.maxDelayMs;
|
||||
}
|
||||
return true;
|
||||
},
|
||||
{
|
||||
message: "initialDelayMs must be less than or equal to maxDelayMs",
|
||||
path: ["initialDelayMs"],
|
||||
},
|
||||
)
|
||||
.optional();
|
||||
|
||||
const MattermostSlashCommandsSchema = z
|
||||
.object({
|
||||
/** Enable native slash commands. "auto" resolves to false (opt-in). */
|
||||
native: z.union([z.boolean(), z.literal("auto")]).optional(),
|
||||
/** Also register skill-based commands. */
|
||||
nativeSkills: z.union([z.boolean(), z.literal("auto")]).optional(),
|
||||
/** Path for the callback endpoint on the gateway HTTP server. */
|
||||
callbackPath: z.string().optional(),
|
||||
/** Explicit callback URL (e.g. behind reverse proxy). */
|
||||
callbackUrl: z.string().optional(),
|
||||
})
|
||||
.strict()
|
||||
.optional();
|
||||
|
||||
const MattermostAccountSchemaBase = z
|
||||
.object({
|
||||
name: z.string().optional(),
|
||||
capabilities: z.array(z.string()).optional(),
|
||||
dangerouslyAllowNameMatching: z.boolean().optional(),
|
||||
markdown: MarkdownConfigSchema,
|
||||
enabled: z.boolean().optional(),
|
||||
configWrites: z.boolean().optional(),
|
||||
botToken: buildSecretInputSchema().optional(),
|
||||
baseUrl: z.string().optional(),
|
||||
chatmode: z.enum(["oncall", "onmessage", "onchar"]).optional(),
|
||||
oncharPrefixes: z.array(z.string()).optional(),
|
||||
requireMention: z.boolean().optional(),
|
||||
dmPolicy: DmPolicySchema.optional().default("pairing"),
|
||||
allowFrom: z.array(z.union([z.string(), z.number()])).optional(),
|
||||
groupAllowFrom: z.array(z.union([z.string(), z.number()])).optional(),
|
||||
groupPolicy: GroupPolicySchema.optional().default("allowlist"),
|
||||
textChunkLimit: z.number().int().positive().optional(),
|
||||
chunkMode: z.enum(["length", "newline"]).optional(),
|
||||
blockStreaming: z.boolean().optional(),
|
||||
blockStreamingCoalesce: BlockStreamingCoalesceSchema.optional(),
|
||||
replyToMode: z.enum(["off", "first", "all"]).optional(),
|
||||
responsePrefix: z.string().optional(),
|
||||
actions: z
|
||||
.object({
|
||||
reactions: z.boolean().optional(),
|
||||
})
|
||||
.optional(),
|
||||
commands: MattermostSlashCommandsSchema,
|
||||
interactions: z
|
||||
.object({
|
||||
callbackBaseUrl: z.string().optional(),
|
||||
allowedSourceIps: z.array(z.string()).optional(),
|
||||
})
|
||||
.optional(),
|
||||
/** Allow fetching from private/internal IP addresses (e.g. localhost). Required for self-hosted Mattermost on LAN/VPN. */
|
||||
allowPrivateNetwork: z.boolean().optional(),
|
||||
/** Retry configuration for DM channel creation */
|
||||
dmChannelRetry: DmChannelRetrySchema,
|
||||
})
|
||||
.strict();
|
||||
|
||||
const MattermostAccountSchema = MattermostAccountSchemaBase.superRefine((value, ctx) => {
|
||||
requireMattermostOpenAllowFrom({
|
||||
policy: value.dmPolicy,
|
||||
allowFrom: value.allowFrom,
|
||||
ctx,
|
||||
});
|
||||
});
|
||||
|
||||
export const MattermostConfigSchema = MattermostAccountSchemaBase.extend({
|
||||
accounts: z.record(z.string(), MattermostAccountSchema.optional()).optional(),
|
||||
defaultAccount: z.string().optional(),
|
||||
}).superRefine((value, ctx) => {
|
||||
requireMattermostOpenAllowFrom({
|
||||
policy: value.dmPolicy,
|
||||
allowFrom: value.allowFrom,
|
||||
ctx,
|
||||
});
|
||||
});
|
||||
@@ -1,115 +1 @@
|
||||
import { requireChannelOpenAllowFrom } from "openclaw/plugin-sdk/extension-shared";
|
||||
import { z } from "openclaw/plugin-sdk/zod";
|
||||
import {
|
||||
BlockStreamingCoalesceSchema,
|
||||
DmPolicySchema,
|
||||
GroupPolicySchema,
|
||||
MarkdownConfigSchema,
|
||||
requireOpenAllowFrom,
|
||||
} from "./runtime-api.js";
|
||||
import { buildSecretInputSchema } from "./secret-input.js";
|
||||
|
||||
const DmChannelRetrySchema = z
|
||||
.object({
|
||||
/** Maximum number of retry attempts for DM channel creation (default: 3) */
|
||||
maxRetries: z.number().int().min(0).max(10).optional(),
|
||||
/** Initial delay in milliseconds before first retry (default: 1000) */
|
||||
initialDelayMs: z.number().int().min(100).max(60000).optional(),
|
||||
/** Maximum delay in milliseconds between retries (default: 10000) */
|
||||
maxDelayMs: z.number().int().min(1000).max(60000).optional(),
|
||||
/** Timeout for each individual DM channel creation request in milliseconds (default: 30000) */
|
||||
timeoutMs: z.number().int().min(5000).max(120000).optional(),
|
||||
})
|
||||
.strict()
|
||||
.refine(
|
||||
(data) => {
|
||||
if (data.initialDelayMs !== undefined && data.maxDelayMs !== undefined) {
|
||||
return data.initialDelayMs <= data.maxDelayMs;
|
||||
}
|
||||
return true;
|
||||
},
|
||||
{
|
||||
message: "initialDelayMs must be less than or equal to maxDelayMs",
|
||||
path: ["initialDelayMs"],
|
||||
},
|
||||
)
|
||||
.optional();
|
||||
|
||||
const MattermostSlashCommandsSchema = z
|
||||
.object({
|
||||
/** Enable native slash commands. "auto" resolves to false (opt-in). */
|
||||
native: z.union([z.boolean(), z.literal("auto")]).optional(),
|
||||
/** Also register skill-based commands. */
|
||||
nativeSkills: z.union([z.boolean(), z.literal("auto")]).optional(),
|
||||
/** Path for the callback endpoint on the gateway HTTP server. */
|
||||
callbackPath: z.string().optional(),
|
||||
/** Explicit callback URL (e.g. behind reverse proxy). */
|
||||
callbackUrl: z.string().optional(),
|
||||
})
|
||||
.strict()
|
||||
.optional();
|
||||
|
||||
const MattermostAccountSchemaBase = z
|
||||
.object({
|
||||
name: z.string().optional(),
|
||||
capabilities: z.array(z.string()).optional(),
|
||||
dangerouslyAllowNameMatching: z.boolean().optional(),
|
||||
markdown: MarkdownConfigSchema,
|
||||
enabled: z.boolean().optional(),
|
||||
configWrites: z.boolean().optional(),
|
||||
botToken: buildSecretInputSchema().optional(),
|
||||
baseUrl: z.string().optional(),
|
||||
chatmode: z.enum(["oncall", "onmessage", "onchar"]).optional(),
|
||||
oncharPrefixes: z.array(z.string()).optional(),
|
||||
requireMention: z.boolean().optional(),
|
||||
dmPolicy: DmPolicySchema.optional().default("pairing"),
|
||||
allowFrom: z.array(z.union([z.string(), z.number()])).optional(),
|
||||
groupAllowFrom: z.array(z.union([z.string(), z.number()])).optional(),
|
||||
groupPolicy: GroupPolicySchema.optional().default("allowlist"),
|
||||
textChunkLimit: z.number().int().positive().optional(),
|
||||
chunkMode: z.enum(["length", "newline"]).optional(),
|
||||
blockStreaming: z.boolean().optional(),
|
||||
blockStreamingCoalesce: BlockStreamingCoalesceSchema.optional(),
|
||||
replyToMode: z.enum(["off", "first", "all"]).optional(),
|
||||
responsePrefix: z.string().optional(),
|
||||
actions: z
|
||||
.object({
|
||||
reactions: z.boolean().optional(),
|
||||
})
|
||||
.optional(),
|
||||
commands: MattermostSlashCommandsSchema,
|
||||
interactions: z
|
||||
.object({
|
||||
callbackBaseUrl: z.string().optional(),
|
||||
allowedSourceIps: z.array(z.string()).optional(),
|
||||
})
|
||||
.optional(),
|
||||
/** Allow fetching from private/internal IP addresses (e.g. localhost). Required for self-hosted Mattermost on LAN/VPN. */
|
||||
allowPrivateNetwork: z.boolean().optional(),
|
||||
/** Retry configuration for DM channel creation */
|
||||
dmChannelRetry: DmChannelRetrySchema,
|
||||
})
|
||||
.strict();
|
||||
|
||||
const MattermostAccountSchema = MattermostAccountSchemaBase.superRefine((value, ctx) => {
|
||||
requireChannelOpenAllowFrom({
|
||||
channel: "mattermost",
|
||||
policy: value.dmPolicy,
|
||||
allowFrom: value.allowFrom,
|
||||
ctx,
|
||||
requireOpenAllowFrom,
|
||||
});
|
||||
});
|
||||
|
||||
export const MattermostConfigSchema = MattermostAccountSchemaBase.extend({
|
||||
accounts: z.record(z.string(), MattermostAccountSchema.optional()).optional(),
|
||||
defaultAccount: z.string().optional(),
|
||||
}).superRefine((value, ctx) => {
|
||||
requireChannelOpenAllowFrom({
|
||||
channel: "mattermost",
|
||||
policy: value.dmPolicy,
|
||||
allowFrom: value.allowFrom,
|
||||
ctx,
|
||||
requireOpenAllowFrom,
|
||||
});
|
||||
});
|
||||
export { MattermostConfigSchema } from "./config-schema-core.js";
|
||||
|
||||
4
extensions/mattermost/src/config-surface.ts
Normal file
4
extensions/mattermost/src/config-surface.ts
Normal file
@@ -0,0 +1,4 @@
|
||||
import { buildChannelConfigSchema } from "openclaw/plugin-sdk/channel-config-primitives";
|
||||
import { MattermostConfigSchema } from "./config-schema-core.js";
|
||||
|
||||
export const MattermostChannelConfigSchema = buildChannelConfigSchema(MattermostConfigSchema);
|
||||
7
extensions/media-understanding-core/package.json
Normal file
7
extensions/media-understanding-core/package.json
Normal file
@@ -0,0 +1,7 @@
|
||||
{
|
||||
"name": "@openclaw/media-understanding-core",
|
||||
"version": "2026.3.26",
|
||||
"private": true,
|
||||
"description": "OpenClaw media understanding runtime package",
|
||||
"type": "module"
|
||||
}
|
||||
9
extensions/media-understanding-core/runtime-api.ts
Normal file
9
extensions/media-understanding-core/runtime-api.ts
Normal file
@@ -0,0 +1,9 @@
|
||||
export {
|
||||
describeImageFile,
|
||||
describeImageFileWithModel,
|
||||
describeVideoFile,
|
||||
runMediaUnderstandingFile,
|
||||
transcribeAudioFile,
|
||||
type RunMediaUnderstandingFileParams,
|
||||
type RunMediaUnderstandingFileResult,
|
||||
} from "./src/runtime.js";
|
||||
147
extensions/media-understanding-core/src/runtime.ts
Normal file
147
extensions/media-understanding-core/src/runtime.ts
Normal file
@@ -0,0 +1,147 @@
|
||||
import fs from "node:fs/promises";
|
||||
import path from "node:path";
|
||||
import type { OpenClawConfig } from "openclaw/plugin-sdk/core";
|
||||
import {
|
||||
buildProviderRegistry,
|
||||
createMediaAttachmentCache,
|
||||
normalizeMediaAttachments,
|
||||
normalizeMediaProviderId,
|
||||
runCapability,
|
||||
type ActiveMediaModel,
|
||||
} from "openclaw/plugin-sdk/media-runtime";
|
||||
|
||||
type MediaUnderstandingCapability = "image" | "audio" | "video";
|
||||
type MediaUnderstandingOutput = Awaited<ReturnType<typeof runCapability>>["outputs"][number];
|
||||
|
||||
const KIND_BY_CAPABILITY: Record<MediaUnderstandingCapability, MediaUnderstandingOutput["kind"]> = {
|
||||
audio: "audio.transcription",
|
||||
image: "image.description",
|
||||
video: "video.description",
|
||||
};
|
||||
|
||||
export type RunMediaUnderstandingFileParams = {
|
||||
capability: MediaUnderstandingCapability;
|
||||
filePath: string;
|
||||
cfg: OpenClawConfig;
|
||||
agentDir?: string;
|
||||
mime?: string;
|
||||
activeModel?: ActiveMediaModel;
|
||||
};
|
||||
|
||||
export type RunMediaUnderstandingFileResult = {
|
||||
text: string | undefined;
|
||||
provider?: string;
|
||||
model?: string;
|
||||
output?: MediaUnderstandingOutput;
|
||||
};
|
||||
|
||||
function buildFileContext(params: { filePath: string; mime?: string }) {
|
||||
return {
|
||||
MediaPath: params.filePath,
|
||||
MediaType: params.mime,
|
||||
};
|
||||
}
|
||||
|
||||
export async function runMediaUnderstandingFile(
|
||||
params: RunMediaUnderstandingFileParams,
|
||||
): Promise<RunMediaUnderstandingFileResult> {
|
||||
const ctx = buildFileContext(params);
|
||||
const attachments = normalizeMediaAttachments(ctx);
|
||||
if (attachments.length === 0) {
|
||||
return { text: undefined };
|
||||
}
|
||||
|
||||
const providerRegistry = buildProviderRegistry(undefined, params.cfg);
|
||||
const cache = createMediaAttachmentCache(attachments, {
|
||||
localPathRoots: [path.dirname(params.filePath)],
|
||||
});
|
||||
|
||||
try {
|
||||
const result = await runCapability({
|
||||
capability: params.capability,
|
||||
cfg: params.cfg,
|
||||
ctx,
|
||||
attachments: cache,
|
||||
media: attachments,
|
||||
agentDir: params.agentDir,
|
||||
providerRegistry,
|
||||
config: params.cfg.tools?.media?.[params.capability],
|
||||
activeModel: params.activeModel,
|
||||
});
|
||||
const output = result.outputs.find(
|
||||
(entry) => entry.kind === KIND_BY_CAPABILITY[params.capability],
|
||||
);
|
||||
const text = output?.text?.trim();
|
||||
return {
|
||||
text: text || undefined,
|
||||
provider: output?.provider,
|
||||
model: output?.model,
|
||||
output,
|
||||
};
|
||||
} finally {
|
||||
await cache.cleanup();
|
||||
}
|
||||
}
|
||||
|
||||
export async function describeImageFile(params: {
|
||||
filePath: string;
|
||||
cfg: OpenClawConfig;
|
||||
agentDir?: string;
|
||||
mime?: string;
|
||||
activeModel?: ActiveMediaModel;
|
||||
}): Promise<RunMediaUnderstandingFileResult> {
|
||||
return await runMediaUnderstandingFile({ ...params, capability: "image" });
|
||||
}
|
||||
|
||||
export async function describeImageFileWithModel(params: {
|
||||
filePath: string;
|
||||
cfg: OpenClawConfig;
|
||||
agentDir?: string;
|
||||
mime?: string;
|
||||
provider: string;
|
||||
model: string;
|
||||
prompt: string;
|
||||
maxTokens?: number;
|
||||
timeoutMs?: number;
|
||||
}) {
|
||||
const timeoutMs = params.timeoutMs ?? 30_000;
|
||||
const providerRegistry = buildProviderRegistry(undefined, params.cfg);
|
||||
const provider = providerRegistry.get(normalizeMediaProviderId(params.provider));
|
||||
if (!provider?.describeImage) {
|
||||
throw new Error(`Provider does not support image analysis: ${params.provider}`);
|
||||
}
|
||||
const buffer = await fs.readFile(params.filePath);
|
||||
return await provider.describeImage({
|
||||
buffer,
|
||||
fileName: path.basename(params.filePath),
|
||||
mime: params.mime,
|
||||
provider: params.provider,
|
||||
model: params.model,
|
||||
prompt: params.prompt,
|
||||
maxTokens: params.maxTokens,
|
||||
timeoutMs,
|
||||
cfg: params.cfg,
|
||||
agentDir: params.agentDir ?? "",
|
||||
});
|
||||
}
|
||||
|
||||
export async function describeVideoFile(params: {
|
||||
filePath: string;
|
||||
cfg: OpenClawConfig;
|
||||
agentDir?: string;
|
||||
mime?: string;
|
||||
activeModel?: ActiveMediaModel;
|
||||
}): Promise<RunMediaUnderstandingFileResult> {
|
||||
return await runMediaUnderstandingFile({ ...params, capability: "video" });
|
||||
}
|
||||
|
||||
export async function transcribeAudioFile(params: {
|
||||
filePath: string;
|
||||
cfg: OpenClawConfig;
|
||||
agentDir?: string;
|
||||
mime?: string;
|
||||
activeModel?: ActiveMediaModel;
|
||||
}): Promise<{ text: string | undefined }> {
|
||||
const result = await runMediaUnderstandingFile({ ...params, capability: "audio" });
|
||||
return { text: result.text };
|
||||
}
|
||||
1
extensions/msteams/test-api.ts
Normal file
1
extensions/msteams/test-api.ts
Normal file
@@ -0,0 +1 @@
|
||||
export { msteamsPlugin } from "./src/channel.js";
|
||||
@@ -1,17 +1,33 @@
|
||||
import { requireChannelOpenAllowFrom } from "openclaw/plugin-sdk/extension-shared";
|
||||
import { z } from "openclaw/plugin-sdk/zod";
|
||||
import {
|
||||
ReplyRuntimeConfigSchemaShape,
|
||||
ToolPolicySchema,
|
||||
} from "openclaw/plugin-sdk/agent-config-primitives";
|
||||
import {
|
||||
BlockStreamingCoalesceSchema,
|
||||
DmConfigSchema,
|
||||
DmPolicySchema,
|
||||
GroupPolicySchema,
|
||||
MarkdownConfigSchema,
|
||||
ReplyRuntimeConfigSchemaShape,
|
||||
ToolPolicySchema,
|
||||
requireOpenAllowFrom,
|
||||
} from "../runtime-api.js";
|
||||
} from "openclaw/plugin-sdk/channel-config-primitives";
|
||||
import { z } from "openclaw/plugin-sdk/zod";
|
||||
import { buildSecretInputSchema } from "./secret-input.js";
|
||||
|
||||
function requireNextcloudTalkOpenAllowFrom(params: {
|
||||
policy?: string;
|
||||
allowFrom?: string[];
|
||||
ctx: z.RefinementCtx;
|
||||
}) {
|
||||
requireOpenAllowFrom({
|
||||
policy: params.policy,
|
||||
allowFrom: params.allowFrom,
|
||||
ctx: params.ctx,
|
||||
path: ["allowFrom"],
|
||||
message:
|
||||
'channels.nextcloud-talk.dmPolicy="open" requires channels.nextcloud-talk.allowFrom to include "*"',
|
||||
});
|
||||
}
|
||||
|
||||
export const NextcloudTalkRoomSchema = z
|
||||
.object({
|
||||
requireMention: z.boolean().optional(),
|
||||
@@ -51,12 +67,10 @@ export const NextcloudTalkAccountSchemaBase = z
|
||||
|
||||
export const NextcloudTalkAccountSchema = NextcloudTalkAccountSchemaBase.superRefine(
|
||||
(value, ctx) => {
|
||||
requireChannelOpenAllowFrom({
|
||||
channel: "nextcloud-talk",
|
||||
requireNextcloudTalkOpenAllowFrom({
|
||||
policy: value.dmPolicy,
|
||||
allowFrom: value.allowFrom,
|
||||
ctx,
|
||||
requireOpenAllowFrom,
|
||||
});
|
||||
},
|
||||
);
|
||||
@@ -65,11 +79,9 @@ export const NextcloudTalkConfigSchema = NextcloudTalkAccountSchemaBase.extend({
|
||||
accounts: z.record(z.string(), NextcloudTalkAccountSchema.optional()).optional(),
|
||||
defaultAccount: z.string().optional(),
|
||||
}).superRefine((value, ctx) => {
|
||||
requireChannelOpenAllowFrom({
|
||||
channel: "nextcloud-talk",
|
||||
requireNextcloudTalkOpenAllowFrom({
|
||||
policy: value.dmPolicy,
|
||||
allowFrom: value.allowFrom,
|
||||
ctx,
|
||||
requireOpenAllowFrom,
|
||||
});
|
||||
});
|
||||
|
||||
@@ -1,6 +1,10 @@
|
||||
import { AllowFromListSchema, DmPolicySchema } from "openclaw/plugin-sdk/channel-config-schema";
|
||||
import {
|
||||
AllowFromListSchema,
|
||||
buildChannelConfigSchema,
|
||||
DmPolicySchema,
|
||||
MarkdownConfigSchema,
|
||||
} from "openclaw/plugin-sdk/channel-config-primitives";
|
||||
import { z } from "openclaw/plugin-sdk/zod";
|
||||
import { MarkdownConfigSchema, buildChannelConfigSchema } from "../api.js";
|
||||
|
||||
/**
|
||||
* Validates https:// URLs only (no javascript:, data:, file:, etc.)
|
||||
|
||||
1
extensions/nostr/test-api.ts
Normal file
1
extensions/nostr/test-api.ts
Normal file
@@ -0,0 +1 @@
|
||||
export { nostrPlugin } from "./src/channel.js";
|
||||
@@ -1 +1,2 @@
|
||||
export { buildOpenAICodexCliBackend } from "./cli-backend.js";
|
||||
export { buildOpenAISpeechProvider } from "./speech-provider.js";
|
||||
|
||||
@@ -1 +1,17 @@
|
||||
export { requireChannelOpenAllowFrom } from "openclaw/plugin-sdk/extension-shared";
|
||||
import { requireOpenAllowFrom } from "openclaw/plugin-sdk/channel-config-primitives";
|
||||
import type { z } from "openclaw/plugin-sdk/zod";
|
||||
|
||||
export function requireChannelOpenAllowFrom(params: {
|
||||
channel: string;
|
||||
policy?: string;
|
||||
allowFrom?: Array<string | number>;
|
||||
ctx: z.RefinementCtx;
|
||||
}) {
|
||||
requireOpenAllowFrom({
|
||||
policy: params.policy,
|
||||
allowFrom: params.allowFrom,
|
||||
ctx: params.ctx,
|
||||
path: ["allowFrom"],
|
||||
message: `channels.${params.channel}.dmPolicy="open" requires channels.${params.channel}.allowFrom to include "*"`,
|
||||
});
|
||||
}
|
||||
|
||||
6
extensions/signal/reaction-runtime-api.ts
Normal file
6
extensions/signal/reaction-runtime-api.ts
Normal file
@@ -0,0 +1,6 @@
|
||||
export {
|
||||
removeReactionSignal,
|
||||
sendReactionSignal,
|
||||
type SignalReactionOpts,
|
||||
type SignalReactionResult,
|
||||
} from "./src/send-reactions.js";
|
||||
@@ -4,7 +4,7 @@ import {
|
||||
resolveMergedAccountConfig,
|
||||
type OpenClawConfig,
|
||||
} from "openclaw/plugin-sdk/account-resolution";
|
||||
import type { SignalAccountConfig } from "openclaw/plugin-sdk/signal";
|
||||
import type { SignalAccountConfig } from "openclaw/plugin-sdk/signal-core";
|
||||
|
||||
export type ResolvedSignalAccount = {
|
||||
accountId: string;
|
||||
|
||||
@@ -4,9 +4,9 @@ import type {
|
||||
ChannelMessageActionAdapter,
|
||||
ChannelMessageActionName,
|
||||
} from "openclaw/plugin-sdk/channel-contract";
|
||||
import { removeReactionSignal, sendReactionSignal } from "../reaction-runtime-api.js";
|
||||
import { listEnabledSignalAccounts, resolveSignalAccount } from "./accounts.js";
|
||||
import { resolveSignalReactionLevel } from "./reaction-level.js";
|
||||
import { removeReactionSignal, sendReactionSignal } from "./send-reactions.js";
|
||||
|
||||
const providerId = "signal";
|
||||
const GROUP_PREFIX = "group:";
|
||||
|
||||
@@ -1,5 +1,7 @@
|
||||
export type { ResolvedSlackAccount } from "./src/accounts.js";
|
||||
export type { SlackMessageEvent } from "./src/types.js";
|
||||
export { slackPlugin } from "./src/channel.js";
|
||||
export { setSlackRuntime } from "./src/runtime.js";
|
||||
export { createSlackActions } from "./src/channel-actions.js";
|
||||
export { prepareSlackMessage } from "./src/monitor/message-handler/prepare.js";
|
||||
export { createInboundSlackTestContext } from "./src/monitor/message-handler/prepare.test-helpers.js";
|
||||
|
||||
1
extensions/speech-core/api.ts
Normal file
1
extensions/speech-core/api.ts
Normal file
@@ -0,0 +1 @@
|
||||
export * from "openclaw/plugin-sdk/speech-core";
|
||||
7
extensions/speech-core/package.json
Normal file
7
extensions/speech-core/package.json
Normal file
@@ -0,0 +1,7 @@
|
||||
{
|
||||
"name": "@openclaw/speech-core",
|
||||
"version": "2026.3.26",
|
||||
"private": true,
|
||||
"description": "OpenClaw speech runtime package",
|
||||
"type": "module"
|
||||
}
|
||||
33
extensions/speech-core/runtime-api.ts
Normal file
33
extensions/speech-core/runtime-api.ts
Normal file
@@ -0,0 +1,33 @@
|
||||
export {
|
||||
buildTtsSystemPromptHint,
|
||||
getLastTtsAttempt,
|
||||
getResolvedSpeechProviderConfig,
|
||||
getTtsMaxLength,
|
||||
getTtsProvider,
|
||||
isSummarizationEnabled,
|
||||
isTtsEnabled,
|
||||
isTtsProviderConfigured,
|
||||
listSpeechVoices,
|
||||
maybeApplyTtsToPayload,
|
||||
resolveTtsAutoMode,
|
||||
resolveTtsConfig,
|
||||
resolveTtsPrefsPath,
|
||||
resolveTtsProviderOrder,
|
||||
setLastTtsAttempt,
|
||||
setSummarizationEnabled,
|
||||
setTtsAutoMode,
|
||||
setTtsEnabled,
|
||||
setTtsMaxLength,
|
||||
setTtsProvider,
|
||||
synthesizeSpeech,
|
||||
textToSpeech,
|
||||
textToSpeechTelephony,
|
||||
_test,
|
||||
type ResolvedTtsConfig,
|
||||
type ResolvedTtsModelOverrides,
|
||||
type TtsDirectiveOverrides,
|
||||
type TtsDirectiveParseResult,
|
||||
type TtsResult,
|
||||
type TtsSynthesisResult,
|
||||
type TtsTelephonyResult,
|
||||
} from "./src/tts.js";
|
||||
849
extensions/speech-core/src/tts.ts
Normal file
849
extensions/speech-core/src/tts.ts
Normal file
@@ -0,0 +1,849 @@
|
||||
import { randomBytes } from "node:crypto";
|
||||
import {
|
||||
existsSync,
|
||||
mkdirSync,
|
||||
readFileSync,
|
||||
writeFileSync,
|
||||
mkdtempSync,
|
||||
renameSync,
|
||||
unlinkSync,
|
||||
} from "node:fs";
|
||||
import path from "node:path";
|
||||
import { normalizeChannelId, type ChannelId } from "openclaw/plugin-sdk/channel-runtime";
|
||||
import type {
|
||||
OpenClawConfig,
|
||||
TtsAutoMode,
|
||||
TtsConfig,
|
||||
TtsMode,
|
||||
TtsModelOverrideConfig,
|
||||
TtsProvider,
|
||||
} from "openclaw/plugin-sdk/config-runtime";
|
||||
import { resolveSendableOutboundReplyParts } from "openclaw/plugin-sdk/reply-payload";
|
||||
import type { ReplyPayload } from "openclaw/plugin-sdk/reply-runtime";
|
||||
import { logVerbose } from "openclaw/plugin-sdk/runtime-env";
|
||||
import { resolvePreferredOpenClawTmpDir } from "openclaw/plugin-sdk/sandbox";
|
||||
import { CONFIG_DIR, resolveUserPath, stripMarkdown } from "openclaw/plugin-sdk/text-runtime";
|
||||
import {
|
||||
canonicalizeSpeechProviderId,
|
||||
getSpeechProvider,
|
||||
listSpeechProviders,
|
||||
normalizeTtsAutoMode,
|
||||
parseTtsDirectives,
|
||||
scheduleCleanup,
|
||||
summarizeText,
|
||||
type SpeechModelOverridePolicy,
|
||||
type SpeechProviderConfig,
|
||||
type SpeechVoiceOption,
|
||||
type TtsDirectiveOverrides,
|
||||
type TtsDirectiveParseResult,
|
||||
} from "../api.js";
|
||||
|
||||
export type { TtsDirectiveOverrides, TtsDirectiveParseResult };
|
||||
|
||||
const DEFAULT_TIMEOUT_MS = 30_000;
|
||||
const DEFAULT_TTS_MAX_LENGTH = 1500;
|
||||
const DEFAULT_TTS_SUMMARIZE = true;
|
||||
const DEFAULT_MAX_TEXT_LENGTH = 4096;
|
||||
|
||||
export type ResolvedTtsConfig = {
|
||||
auto: TtsAutoMode;
|
||||
mode: TtsMode;
|
||||
provider: TtsProvider;
|
||||
providerSource: "config" | "default";
|
||||
summaryModel?: string;
|
||||
modelOverrides: ResolvedTtsModelOverrides;
|
||||
providerConfigs: Record<string, SpeechProviderConfig>;
|
||||
prefsPath?: string;
|
||||
maxTextLength: number;
|
||||
timeoutMs: number;
|
||||
};
|
||||
|
||||
type TtsUserPrefs = {
|
||||
tts?: {
|
||||
auto?: TtsAutoMode;
|
||||
enabled?: boolean;
|
||||
provider?: TtsProvider;
|
||||
maxLength?: number;
|
||||
summarize?: boolean;
|
||||
};
|
||||
};
|
||||
|
||||
export type ResolvedTtsModelOverrides = SpeechModelOverridePolicy;
|
||||
|
||||
export type TtsResult = {
|
||||
success: boolean;
|
||||
audioPath?: string;
|
||||
error?: string;
|
||||
latencyMs?: number;
|
||||
provider?: string;
|
||||
outputFormat?: string;
|
||||
voiceCompatible?: boolean;
|
||||
};
|
||||
|
||||
export type TtsSynthesisResult = {
|
||||
success: boolean;
|
||||
audioBuffer?: Buffer;
|
||||
error?: string;
|
||||
latencyMs?: number;
|
||||
provider?: string;
|
||||
outputFormat?: string;
|
||||
voiceCompatible?: boolean;
|
||||
fileExtension?: string;
|
||||
};
|
||||
|
||||
export type TtsTelephonyResult = {
|
||||
success: boolean;
|
||||
audioBuffer?: Buffer;
|
||||
error?: string;
|
||||
latencyMs?: number;
|
||||
provider?: string;
|
||||
outputFormat?: string;
|
||||
sampleRate?: number;
|
||||
};
|
||||
|
||||
type TtsStatusEntry = {
|
||||
timestamp: number;
|
||||
success: boolean;
|
||||
textLength: number;
|
||||
summarized: boolean;
|
||||
provider?: string;
|
||||
latencyMs?: number;
|
||||
error?: string;
|
||||
};
|
||||
|
||||
let lastTtsAttempt: TtsStatusEntry | undefined;
|
||||
|
||||
function resolveModelOverridePolicy(
|
||||
overrides: TtsModelOverrideConfig | undefined,
|
||||
): ResolvedTtsModelOverrides {
|
||||
const enabled = overrides?.enabled ?? true;
|
||||
if (!enabled) {
|
||||
return {
|
||||
enabled: false,
|
||||
allowText: false,
|
||||
allowProvider: false,
|
||||
allowVoice: false,
|
||||
allowModelId: false,
|
||||
allowVoiceSettings: false,
|
||||
allowNormalization: false,
|
||||
allowSeed: false,
|
||||
};
|
||||
}
|
||||
const allow = (value: boolean | undefined, defaultValue = true) => value ?? defaultValue;
|
||||
return {
|
||||
enabled: true,
|
||||
allowText: allow(overrides?.allowText),
|
||||
allowProvider: allow(overrides?.allowProvider, false),
|
||||
allowVoice: allow(overrides?.allowVoice),
|
||||
allowModelId: allow(overrides?.allowModelId),
|
||||
allowVoiceSettings: allow(overrides?.allowVoiceSettings),
|
||||
allowNormalization: allow(overrides?.allowNormalization),
|
||||
allowSeed: allow(overrides?.allowSeed),
|
||||
};
|
||||
}
|
||||
|
||||
function sortSpeechProvidersForAutoSelection(cfg?: OpenClawConfig) {
|
||||
return listSpeechProviders(cfg).toSorted((left, right) => {
|
||||
const leftOrder = left.autoSelectOrder ?? Number.MAX_SAFE_INTEGER;
|
||||
const rightOrder = right.autoSelectOrder ?? Number.MAX_SAFE_INTEGER;
|
||||
if (leftOrder !== rightOrder) {
|
||||
return leftOrder - rightOrder;
|
||||
}
|
||||
return left.id.localeCompare(right.id);
|
||||
});
|
||||
}
|
||||
|
||||
function resolveRegistryDefaultSpeechProviderId(cfg?: OpenClawConfig): TtsProvider {
|
||||
return sortSpeechProvidersForAutoSelection(cfg)[0]?.id ?? "";
|
||||
}
|
||||
|
||||
function asProviderConfig(value: unknown): SpeechProviderConfig {
|
||||
return typeof value === "object" && value !== null && !Array.isArray(value)
|
||||
? (value as SpeechProviderConfig)
|
||||
: {};
|
||||
}
|
||||
|
||||
function asProviderConfigMap(value: unknown): Record<string, unknown> {
|
||||
return typeof value === "object" && value !== null && !Array.isArray(value)
|
||||
? (value as Record<string, unknown>)
|
||||
: {};
|
||||
}
|
||||
|
||||
function resolveSpeechProviderConfigs(
|
||||
raw: TtsConfig,
|
||||
cfg: OpenClawConfig,
|
||||
timeoutMs: number,
|
||||
): Record<string, SpeechProviderConfig> {
|
||||
const providerConfigs: Record<string, SpeechProviderConfig> = {};
|
||||
const rawProviders = asProviderConfigMap(raw.providers);
|
||||
for (const provider of listSpeechProviders(cfg)) {
|
||||
providerConfigs[provider.id] =
|
||||
provider.resolveConfig?.({
|
||||
cfg,
|
||||
rawConfig: {
|
||||
...(raw as Record<string, unknown>),
|
||||
providers: rawProviders,
|
||||
},
|
||||
timeoutMs,
|
||||
}) ??
|
||||
asProviderConfig(rawProviders[provider.id] ?? (raw as Record<string, unknown>)[provider.id]);
|
||||
}
|
||||
return providerConfigs;
|
||||
}
|
||||
|
||||
export function getResolvedSpeechProviderConfig(
|
||||
config: ResolvedTtsConfig,
|
||||
providerId: string,
|
||||
cfg?: OpenClawConfig,
|
||||
): SpeechProviderConfig {
|
||||
const canonical =
|
||||
canonicalizeSpeechProviderId(providerId, cfg) ?? providerId.trim().toLowerCase();
|
||||
return config.providerConfigs[canonical] ?? {};
|
||||
}
|
||||
|
||||
export function resolveTtsConfig(cfg: OpenClawConfig): ResolvedTtsConfig {
|
||||
const raw: TtsConfig = cfg.messages?.tts ?? {};
|
||||
const providerSource = raw.provider ? "config" : "default";
|
||||
const timeoutMs = raw.timeoutMs ?? DEFAULT_TIMEOUT_MS;
|
||||
const auto = normalizeTtsAutoMode(raw.auto) ?? (raw.enabled ? "always" : "off");
|
||||
return {
|
||||
auto,
|
||||
mode: raw.mode ?? "final",
|
||||
provider:
|
||||
canonicalizeSpeechProviderId(raw.provider, cfg) ??
|
||||
resolveRegistryDefaultSpeechProviderId(cfg),
|
||||
providerSource,
|
||||
summaryModel: raw.summaryModel?.trim() || undefined,
|
||||
modelOverrides: resolveModelOverridePolicy(raw.modelOverrides),
|
||||
providerConfigs: resolveSpeechProviderConfigs(raw, cfg, timeoutMs),
|
||||
prefsPath: raw.prefsPath,
|
||||
maxTextLength: raw.maxTextLength ?? DEFAULT_MAX_TEXT_LENGTH,
|
||||
timeoutMs,
|
||||
};
|
||||
}
|
||||
|
||||
export function resolveTtsPrefsPath(config: ResolvedTtsConfig): string {
|
||||
if (config.prefsPath?.trim()) {
|
||||
return resolveUserPath(config.prefsPath.trim());
|
||||
}
|
||||
const envPath = process.env.OPENCLAW_TTS_PREFS?.trim();
|
||||
if (envPath) {
|
||||
return resolveUserPath(envPath);
|
||||
}
|
||||
return path.join(CONFIG_DIR, "settings", "tts.json");
|
||||
}
|
||||
|
||||
function resolveTtsAutoModeFromPrefs(prefs: TtsUserPrefs): TtsAutoMode | undefined {
|
||||
const auto = normalizeTtsAutoMode(prefs.tts?.auto);
|
||||
if (auto) {
|
||||
return auto;
|
||||
}
|
||||
if (typeof prefs.tts?.enabled === "boolean") {
|
||||
return prefs.tts.enabled ? "always" : "off";
|
||||
}
|
||||
return undefined;
|
||||
}
|
||||
|
||||
export function resolveTtsAutoMode(params: {
|
||||
config: ResolvedTtsConfig;
|
||||
prefsPath: string;
|
||||
sessionAuto?: string;
|
||||
}): TtsAutoMode {
|
||||
const sessionAuto = normalizeTtsAutoMode(params.sessionAuto);
|
||||
if (sessionAuto) {
|
||||
return sessionAuto;
|
||||
}
|
||||
const prefsAuto = resolveTtsAutoModeFromPrefs(readPrefs(params.prefsPath));
|
||||
if (prefsAuto) {
|
||||
return prefsAuto;
|
||||
}
|
||||
return params.config.auto;
|
||||
}
|
||||
|
||||
export function buildTtsSystemPromptHint(cfg: OpenClawConfig): string | undefined {
|
||||
const config = resolveTtsConfig(cfg);
|
||||
const prefsPath = resolveTtsPrefsPath(config);
|
||||
const autoMode = resolveTtsAutoMode({ config, prefsPath });
|
||||
if (autoMode === "off") {
|
||||
return undefined;
|
||||
}
|
||||
const maxLength = getTtsMaxLength(prefsPath);
|
||||
const summarize = isSummarizationEnabled(prefsPath) ? "on" : "off";
|
||||
const autoHint =
|
||||
autoMode === "inbound"
|
||||
? "Only use TTS when the user's last message includes audio/voice."
|
||||
: autoMode === "tagged"
|
||||
? "Only use TTS when you include [[tts]] or [[tts:text]] tags."
|
||||
: undefined;
|
||||
return [
|
||||
"Voice (TTS) is enabled.",
|
||||
autoHint,
|
||||
`Keep spoken text ≤${maxLength} chars to avoid auto-summary (summary ${summarize}).`,
|
||||
"Use [[tts:...]] and optional [[tts:text]]...[[/tts:text]] to control voice/expressiveness.",
|
||||
]
|
||||
.filter(Boolean)
|
||||
.join("\n");
|
||||
}
|
||||
|
||||
function readPrefs(prefsPath: string): TtsUserPrefs {
|
||||
try {
|
||||
if (!existsSync(prefsPath)) {
|
||||
return {};
|
||||
}
|
||||
return JSON.parse(readFileSync(prefsPath, "utf8")) as TtsUserPrefs;
|
||||
} catch {
|
||||
return {};
|
||||
}
|
||||
}
|
||||
|
||||
function atomicWriteFileSync(filePath: string, content: string): void {
|
||||
const tmpPath = `${filePath}.tmp.${Date.now()}.${randomBytes(8).toString("hex")}`;
|
||||
writeFileSync(tmpPath, content, { mode: 0o600 });
|
||||
try {
|
||||
renameSync(tmpPath, filePath);
|
||||
} catch (err) {
|
||||
try {
|
||||
unlinkSync(tmpPath);
|
||||
} catch {
|
||||
// ignore
|
||||
}
|
||||
throw err;
|
||||
}
|
||||
}
|
||||
|
||||
function updatePrefs(prefsPath: string, update: (prefs: TtsUserPrefs) => void): void {
|
||||
const prefs = readPrefs(prefsPath);
|
||||
update(prefs);
|
||||
mkdirSync(path.dirname(prefsPath), { recursive: true });
|
||||
atomicWriteFileSync(prefsPath, JSON.stringify(prefs, null, 2));
|
||||
}
|
||||
|
||||
export function isTtsEnabled(
|
||||
config: ResolvedTtsConfig,
|
||||
prefsPath: string,
|
||||
sessionAuto?: string,
|
||||
): boolean {
|
||||
return resolveTtsAutoMode({ config, prefsPath, sessionAuto }) !== "off";
|
||||
}
|
||||
|
||||
export function setTtsAutoMode(prefsPath: string, mode: TtsAutoMode): void {
|
||||
updatePrefs(prefsPath, (prefs) => {
|
||||
const next = { ...prefs.tts };
|
||||
delete next.enabled;
|
||||
next.auto = mode;
|
||||
prefs.tts = next;
|
||||
});
|
||||
}
|
||||
|
||||
export function setTtsEnabled(prefsPath: string, enabled: boolean): void {
|
||||
setTtsAutoMode(prefsPath, enabled ? "always" : "off");
|
||||
}
|
||||
|
||||
export function getTtsProvider(config: ResolvedTtsConfig, prefsPath: string): TtsProvider {
|
||||
const prefs = readPrefs(prefsPath);
|
||||
const prefsProvider = canonicalizeSpeechProviderId(prefs.tts?.provider);
|
||||
if (prefsProvider) {
|
||||
return prefsProvider;
|
||||
}
|
||||
if (config.providerSource === "config") {
|
||||
return canonicalizeSpeechProviderId(config.provider) ?? config.provider;
|
||||
}
|
||||
|
||||
for (const provider of sortSpeechProvidersForAutoSelection()) {
|
||||
if (
|
||||
provider.isConfigured({
|
||||
providerConfig: config.providerConfigs[provider.id] ?? {},
|
||||
timeoutMs: config.timeoutMs,
|
||||
})
|
||||
) {
|
||||
return provider.id;
|
||||
}
|
||||
}
|
||||
return config.provider;
|
||||
}
|
||||
|
||||
export function setTtsProvider(prefsPath: string, provider: TtsProvider): void {
|
||||
updatePrefs(prefsPath, (prefs) => {
|
||||
prefs.tts = { ...prefs.tts, provider: canonicalizeSpeechProviderId(provider) ?? provider };
|
||||
});
|
||||
}
|
||||
|
||||
export function getTtsMaxLength(prefsPath: string): number {
|
||||
const prefs = readPrefs(prefsPath);
|
||||
return prefs.tts?.maxLength ?? DEFAULT_TTS_MAX_LENGTH;
|
||||
}
|
||||
|
||||
export function setTtsMaxLength(prefsPath: string, maxLength: number): void {
|
||||
updatePrefs(prefsPath, (prefs) => {
|
||||
prefs.tts = { ...prefs.tts, maxLength };
|
||||
});
|
||||
}
|
||||
|
||||
export function isSummarizationEnabled(prefsPath: string): boolean {
|
||||
const prefs = readPrefs(prefsPath);
|
||||
return prefs.tts?.summarize ?? DEFAULT_TTS_SUMMARIZE;
|
||||
}
|
||||
|
||||
export function setSummarizationEnabled(prefsPath: string, enabled: boolean): void {
|
||||
updatePrefs(prefsPath, (prefs) => {
|
||||
prefs.tts = { ...prefs.tts, summarize: enabled };
|
||||
});
|
||||
}
|
||||
|
||||
export function getLastTtsAttempt(): TtsStatusEntry | undefined {
|
||||
return lastTtsAttempt;
|
||||
}
|
||||
|
||||
export function setLastTtsAttempt(entry: TtsStatusEntry | undefined): void {
|
||||
lastTtsAttempt = entry;
|
||||
}
|
||||
|
||||
const OPUS_CHANNELS = new Set(["telegram", "feishu", "whatsapp", "matrix"]);
|
||||
|
||||
function resolveChannelId(channel: string | undefined): ChannelId | null {
|
||||
return channel ? normalizeChannelId(channel) : null;
|
||||
}
|
||||
|
||||
export function resolveTtsProviderOrder(primary: TtsProvider, cfg?: OpenClawConfig): TtsProvider[] {
|
||||
const normalizedPrimary = canonicalizeSpeechProviderId(primary, cfg) ?? primary;
|
||||
const ordered = new Set<TtsProvider>([normalizedPrimary]);
|
||||
for (const provider of sortSpeechProvidersForAutoSelection(cfg)) {
|
||||
const normalized = provider.id;
|
||||
if (normalized !== normalizedPrimary) {
|
||||
ordered.add(normalized);
|
||||
}
|
||||
}
|
||||
return [...ordered];
|
||||
}
|
||||
|
||||
export function isTtsProviderConfigured(
|
||||
config: ResolvedTtsConfig,
|
||||
provider: TtsProvider,
|
||||
cfg?: OpenClawConfig,
|
||||
): boolean {
|
||||
const resolvedProvider = getSpeechProvider(provider, cfg);
|
||||
if (!resolvedProvider) {
|
||||
return false;
|
||||
}
|
||||
return (
|
||||
resolvedProvider.isConfigured({
|
||||
cfg,
|
||||
providerConfig: getResolvedSpeechProviderConfig(config, resolvedProvider.id, cfg),
|
||||
timeoutMs: config.timeoutMs,
|
||||
}) ?? false
|
||||
);
|
||||
}
|
||||
|
||||
function formatTtsProviderError(provider: TtsProvider, err: unknown): string {
|
||||
const error = err instanceof Error ? err : new Error(String(err));
|
||||
if (error.name === "AbortError") {
|
||||
return `${provider}: request timed out`;
|
||||
}
|
||||
return `${provider}: ${error.message}`;
|
||||
}
|
||||
|
||||
function buildTtsFailureResult(errors: string[]): { success: false; error: string } {
|
||||
return {
|
||||
success: false,
|
||||
error: `TTS conversion failed: ${errors.join("; ") || "no providers available"}`,
|
||||
};
|
||||
}
|
||||
|
||||
function resolveReadySpeechProvider(params: {
|
||||
provider: TtsProvider;
|
||||
cfg: OpenClawConfig;
|
||||
config: ResolvedTtsConfig;
|
||||
errors: string[];
|
||||
requireTelephony?: boolean;
|
||||
}): NonNullable<ReturnType<typeof getSpeechProvider>> | null {
|
||||
const resolvedProvider = getSpeechProvider(params.provider, params.cfg);
|
||||
if (!resolvedProvider) {
|
||||
params.errors.push(`${params.provider}: no provider registered`);
|
||||
return null;
|
||||
}
|
||||
const providerConfig = getResolvedSpeechProviderConfig(
|
||||
params.config,
|
||||
resolvedProvider.id,
|
||||
params.cfg,
|
||||
);
|
||||
if (
|
||||
!resolvedProvider.isConfigured({
|
||||
cfg: params.cfg,
|
||||
providerConfig,
|
||||
timeoutMs: params.config.timeoutMs,
|
||||
})
|
||||
) {
|
||||
params.errors.push(`${params.provider}: not configured`);
|
||||
return null;
|
||||
}
|
||||
if (params.requireTelephony && !resolvedProvider.synthesizeTelephony) {
|
||||
params.errors.push(`${params.provider}: unsupported for telephony`);
|
||||
return null;
|
||||
}
|
||||
return resolvedProvider;
|
||||
}
|
||||
|
||||
function resolveTtsRequestSetup(params: {
|
||||
text: string;
|
||||
cfg: OpenClawConfig;
|
||||
prefsPath?: string;
|
||||
providerOverride?: TtsProvider;
|
||||
disableFallback?: boolean;
|
||||
}):
|
||||
| {
|
||||
config: ResolvedTtsConfig;
|
||||
providers: TtsProvider[];
|
||||
}
|
||||
| {
|
||||
error: string;
|
||||
} {
|
||||
const config = resolveTtsConfig(params.cfg);
|
||||
const prefsPath = params.prefsPath ?? resolveTtsPrefsPath(config);
|
||||
if (params.text.length > config.maxTextLength) {
|
||||
return {
|
||||
error: `Text too long (${params.text.length} chars, max ${config.maxTextLength})`,
|
||||
};
|
||||
}
|
||||
|
||||
const userProvider = getTtsProvider(config, prefsPath);
|
||||
const provider =
|
||||
canonicalizeSpeechProviderId(params.providerOverride, params.cfg) ?? userProvider;
|
||||
return {
|
||||
config,
|
||||
providers: params.disableFallback ? [provider] : resolveTtsProviderOrder(provider, params.cfg),
|
||||
};
|
||||
}
|
||||
|
||||
export async function textToSpeech(params: {
|
||||
text: string;
|
||||
cfg: OpenClawConfig;
|
||||
prefsPath?: string;
|
||||
channel?: string;
|
||||
overrides?: TtsDirectiveOverrides;
|
||||
disableFallback?: boolean;
|
||||
}): Promise<TtsResult> {
|
||||
const synthesis = await synthesizeSpeech(params);
|
||||
if (!synthesis.success || !synthesis.audioBuffer || !synthesis.fileExtension) {
|
||||
return buildTtsFailureResult([synthesis.error ?? "TTS conversion failed"]);
|
||||
}
|
||||
|
||||
const tempRoot = resolvePreferredOpenClawTmpDir();
|
||||
mkdirSync(tempRoot, { recursive: true, mode: 0o700 });
|
||||
const tempDir = mkdtempSync(path.join(tempRoot, "tts-"));
|
||||
const audioPath = path.join(tempDir, `voice-${Date.now()}${synthesis.fileExtension}`);
|
||||
writeFileSync(audioPath, synthesis.audioBuffer);
|
||||
scheduleCleanup(tempDir);
|
||||
|
||||
return {
|
||||
success: true,
|
||||
audioPath,
|
||||
latencyMs: synthesis.latencyMs,
|
||||
provider: synthesis.provider,
|
||||
outputFormat: synthesis.outputFormat,
|
||||
voiceCompatible: synthesis.voiceCompatible,
|
||||
};
|
||||
}
|
||||
|
||||
export async function synthesizeSpeech(params: {
|
||||
text: string;
|
||||
cfg: OpenClawConfig;
|
||||
prefsPath?: string;
|
||||
channel?: string;
|
||||
overrides?: TtsDirectiveOverrides;
|
||||
disableFallback?: boolean;
|
||||
}): Promise<TtsSynthesisResult> {
|
||||
const setup = resolveTtsRequestSetup({
|
||||
text: params.text,
|
||||
cfg: params.cfg,
|
||||
prefsPath: params.prefsPath,
|
||||
providerOverride: params.overrides?.provider,
|
||||
disableFallback: params.disableFallback,
|
||||
});
|
||||
if ("error" in setup) {
|
||||
return { success: false, error: setup.error };
|
||||
}
|
||||
|
||||
const { config, providers } = setup;
|
||||
const channelId = resolveChannelId(params.channel);
|
||||
const target = channelId && OPUS_CHANNELS.has(channelId) ? "voice-note" : "audio-file";
|
||||
|
||||
const errors: string[] = [];
|
||||
|
||||
for (const provider of providers) {
|
||||
const providerStart = Date.now();
|
||||
try {
|
||||
const resolvedProvider = resolveReadySpeechProvider({
|
||||
provider,
|
||||
cfg: params.cfg,
|
||||
config,
|
||||
errors,
|
||||
});
|
||||
if (!resolvedProvider) {
|
||||
continue;
|
||||
}
|
||||
const synthesis = await resolvedProvider.synthesize({
|
||||
text: params.text,
|
||||
cfg: params.cfg,
|
||||
providerConfig: getResolvedSpeechProviderConfig(config, resolvedProvider.id, params.cfg),
|
||||
target,
|
||||
providerOverrides: params.overrides?.providerOverrides?.[resolvedProvider.id],
|
||||
timeoutMs: config.timeoutMs,
|
||||
});
|
||||
return {
|
||||
success: true,
|
||||
audioBuffer: synthesis.audioBuffer,
|
||||
latencyMs: Date.now() - providerStart,
|
||||
provider,
|
||||
outputFormat: synthesis.outputFormat,
|
||||
voiceCompatible: synthesis.voiceCompatible,
|
||||
fileExtension: synthesis.fileExtension,
|
||||
};
|
||||
} catch (err) {
|
||||
errors.push(formatTtsProviderError(provider, err));
|
||||
}
|
||||
}
|
||||
|
||||
return buildTtsFailureResult(errors);
|
||||
}
|
||||
|
||||
export async function textToSpeechTelephony(params: {
|
||||
text: string;
|
||||
cfg: OpenClawConfig;
|
||||
prefsPath?: string;
|
||||
}): Promise<TtsTelephonyResult> {
|
||||
const setup = resolveTtsRequestSetup({
|
||||
text: params.text,
|
||||
cfg: params.cfg,
|
||||
prefsPath: params.prefsPath,
|
||||
});
|
||||
if ("error" in setup) {
|
||||
return { success: false, error: setup.error };
|
||||
}
|
||||
|
||||
const { config, providers } = setup;
|
||||
const errors: string[] = [];
|
||||
|
||||
for (const provider of providers) {
|
||||
const providerStart = Date.now();
|
||||
try {
|
||||
const resolvedProvider = resolveReadySpeechProvider({
|
||||
provider,
|
||||
cfg: params.cfg,
|
||||
config,
|
||||
errors,
|
||||
requireTelephony: true,
|
||||
});
|
||||
if (!resolvedProvider?.synthesizeTelephony) {
|
||||
continue;
|
||||
}
|
||||
const synthesis = await resolvedProvider.synthesizeTelephony({
|
||||
text: params.text,
|
||||
cfg: params.cfg,
|
||||
providerConfig: getResolvedSpeechProviderConfig(config, resolvedProvider.id, params.cfg),
|
||||
timeoutMs: config.timeoutMs,
|
||||
});
|
||||
|
||||
return {
|
||||
success: true,
|
||||
audioBuffer: synthesis.audioBuffer,
|
||||
latencyMs: Date.now() - providerStart,
|
||||
provider,
|
||||
outputFormat: synthesis.outputFormat,
|
||||
sampleRate: synthesis.sampleRate,
|
||||
};
|
||||
} catch (err) {
|
||||
errors.push(formatTtsProviderError(provider, err));
|
||||
}
|
||||
}
|
||||
|
||||
return buildTtsFailureResult(errors);
|
||||
}
|
||||
|
||||
export async function listSpeechVoices(params: {
|
||||
provider: string;
|
||||
cfg?: OpenClawConfig;
|
||||
config?: ResolvedTtsConfig;
|
||||
apiKey?: string;
|
||||
baseUrl?: string;
|
||||
}): Promise<SpeechVoiceOption[]> {
|
||||
const provider = canonicalizeSpeechProviderId(params.provider, params.cfg);
|
||||
if (!provider) {
|
||||
throw new Error("speech provider id is required");
|
||||
}
|
||||
const config = params.config ?? (params.cfg ? resolveTtsConfig(params.cfg) : undefined);
|
||||
if (!config) {
|
||||
throw new Error(`speech provider ${provider} requires cfg or resolved config`);
|
||||
}
|
||||
const resolvedProvider = getSpeechProvider(provider, params.cfg);
|
||||
if (!resolvedProvider) {
|
||||
throw new Error(`speech provider ${provider} is not registered`);
|
||||
}
|
||||
if (!resolvedProvider.listVoices) {
|
||||
throw new Error(`speech provider ${provider} does not support voice listing`);
|
||||
}
|
||||
return await resolvedProvider.listVoices({
|
||||
cfg: params.cfg,
|
||||
providerConfig: getResolvedSpeechProviderConfig(config, resolvedProvider.id, params.cfg),
|
||||
apiKey: params.apiKey,
|
||||
baseUrl: params.baseUrl,
|
||||
});
|
||||
}
|
||||
|
||||
export async function maybeApplyTtsToPayload(params: {
|
||||
payload: ReplyPayload;
|
||||
cfg: OpenClawConfig;
|
||||
channel?: string;
|
||||
kind?: "tool" | "block" | "final";
|
||||
inboundAudio?: boolean;
|
||||
ttsAuto?: string;
|
||||
}): Promise<ReplyPayload> {
|
||||
if (params.payload.isCompactionNotice) {
|
||||
return params.payload;
|
||||
}
|
||||
const config = resolveTtsConfig(params.cfg);
|
||||
const prefsPath = resolveTtsPrefsPath(config);
|
||||
const autoMode = resolveTtsAutoMode({
|
||||
config,
|
||||
prefsPath,
|
||||
sessionAuto: params.ttsAuto,
|
||||
});
|
||||
if (autoMode === "off") {
|
||||
return params.payload;
|
||||
}
|
||||
|
||||
const reply = resolveSendableOutboundReplyParts(params.payload);
|
||||
const text = reply.text;
|
||||
const directives = parseTtsDirectives(text, config.modelOverrides, {
|
||||
cfg: params.cfg,
|
||||
providerConfigs: config.providerConfigs,
|
||||
});
|
||||
if (directives.warnings.length > 0) {
|
||||
logVerbose(`TTS: ignored directive overrides (${directives.warnings.join("; ")})`);
|
||||
}
|
||||
|
||||
const cleanedText = directives.cleanedText;
|
||||
const trimmedCleaned = cleanedText.trim();
|
||||
const visibleText = trimmedCleaned.length > 0 ? trimmedCleaned : "";
|
||||
const ttsText = directives.ttsText?.trim() || visibleText;
|
||||
|
||||
const nextPayload =
|
||||
visibleText === text.trim()
|
||||
? params.payload
|
||||
: {
|
||||
...params.payload,
|
||||
text: visibleText.length > 0 ? visibleText : undefined,
|
||||
};
|
||||
|
||||
if (autoMode === "tagged" && !directives.hasDirective) {
|
||||
return nextPayload;
|
||||
}
|
||||
if (autoMode === "inbound" && params.inboundAudio !== true) {
|
||||
return nextPayload;
|
||||
}
|
||||
|
||||
const mode = config.mode ?? "final";
|
||||
if (mode === "final" && params.kind && params.kind !== "final") {
|
||||
return nextPayload;
|
||||
}
|
||||
|
||||
if (!ttsText.trim()) {
|
||||
return nextPayload;
|
||||
}
|
||||
if (reply.hasMedia) {
|
||||
return nextPayload;
|
||||
}
|
||||
if (text.includes("MEDIA:")) {
|
||||
return nextPayload;
|
||||
}
|
||||
if (ttsText.trim().length < 10) {
|
||||
return nextPayload;
|
||||
}
|
||||
|
||||
const maxLength = getTtsMaxLength(prefsPath);
|
||||
let textForAudio = ttsText.trim();
|
||||
let wasSummarized = false;
|
||||
|
||||
if (textForAudio.length > maxLength) {
|
||||
if (!isSummarizationEnabled(prefsPath)) {
|
||||
logVerbose(
|
||||
`TTS: truncating long text (${textForAudio.length} > ${maxLength}), summarization disabled.`,
|
||||
);
|
||||
textForAudio = `${textForAudio.slice(0, maxLength - 3)}...`;
|
||||
} else {
|
||||
try {
|
||||
const summary = await summarizeText({
|
||||
text: textForAudio,
|
||||
targetLength: maxLength,
|
||||
cfg: params.cfg,
|
||||
config,
|
||||
timeoutMs: config.timeoutMs,
|
||||
});
|
||||
textForAudio = summary.summary;
|
||||
wasSummarized = true;
|
||||
if (textForAudio.length > config.maxTextLength) {
|
||||
logVerbose(
|
||||
`TTS: summary exceeded hard limit (${textForAudio.length} > ${config.maxTextLength}); truncating.`,
|
||||
);
|
||||
textForAudio = `${textForAudio.slice(0, config.maxTextLength - 3)}...`;
|
||||
}
|
||||
} catch (err) {
|
||||
const error = err as Error;
|
||||
logVerbose(`TTS: summarization failed, truncating instead: ${error.message}`);
|
||||
textForAudio = `${textForAudio.slice(0, maxLength - 3)}...`;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
textForAudio = stripMarkdown(textForAudio).trim();
|
||||
if (textForAudio.length < 10) {
|
||||
return nextPayload;
|
||||
}
|
||||
|
||||
const ttsStart = Date.now();
|
||||
const result = await textToSpeech({
|
||||
text: textForAudio,
|
||||
cfg: params.cfg,
|
||||
prefsPath,
|
||||
channel: params.channel,
|
||||
overrides: directives.overrides,
|
||||
});
|
||||
|
||||
if (result.success && result.audioPath) {
|
||||
lastTtsAttempt = {
|
||||
timestamp: Date.now(),
|
||||
success: true,
|
||||
textLength: text.length,
|
||||
summarized: wasSummarized,
|
||||
provider: result.provider,
|
||||
latencyMs: result.latencyMs,
|
||||
};
|
||||
|
||||
const channelId = resolveChannelId(params.channel);
|
||||
const shouldVoice =
|
||||
channelId !== null && OPUS_CHANNELS.has(channelId) && result.voiceCompatible === true;
|
||||
return {
|
||||
...nextPayload,
|
||||
mediaUrl: result.audioPath,
|
||||
audioAsVoice: shouldVoice || params.payload.audioAsVoice,
|
||||
};
|
||||
}
|
||||
|
||||
lastTtsAttempt = {
|
||||
timestamp: Date.now(),
|
||||
success: false,
|
||||
textLength: text.length,
|
||||
summarized: wasSummarized,
|
||||
error: result.error,
|
||||
};
|
||||
|
||||
const latency = Date.now() - ttsStart;
|
||||
logVerbose(`TTS: conversion failed after ${latency}ms (${result.error ?? "unknown"}).`);
|
||||
return nextPayload;
|
||||
}
|
||||
|
||||
export const _test = {
|
||||
parseTtsDirectives,
|
||||
resolveModelOverridePolicy,
|
||||
summarizeText,
|
||||
getResolvedSpeechProviderConfig,
|
||||
};
|
||||
@@ -1,8 +1,10 @@
|
||||
export { buildTelegramMessageContextForTest } from "./src/bot-message-context.test-harness.js";
|
||||
export { handleTelegramAction } from "./src/action-runtime.js";
|
||||
export { telegramMessageActionRuntime } from "./src/channel-actions.js";
|
||||
export { telegramPlugin } from "./src/channel.js";
|
||||
export { listTelegramAccountIds, resolveTelegramAccount } from "./src/accounts.js";
|
||||
export { resolveTelegramFetch } from "./src/fetch.js";
|
||||
export { makeProxyFetch } from "./src/proxy.js";
|
||||
export { telegramOutbound } from "./src/outbound-adapter.js";
|
||||
export { setTelegramRuntime } from "./src/runtime.js";
|
||||
export { sendMessageTelegram, sendPollTelegram, type TelegramApiOverride } from "./src/send.js";
|
||||
|
||||
1
extensions/tlon/test-api.ts
Normal file
1
extensions/tlon/test-api.ts
Normal file
@@ -0,0 +1 @@
|
||||
export { tlonPlugin } from "./src/channel.js";
|
||||
@@ -1,5 +1,5 @@
|
||||
import { MarkdownConfigSchema } from "openclaw/plugin-sdk/channel-config-primitives";
|
||||
import { z } from "openclaw/plugin-sdk/zod";
|
||||
import { MarkdownConfigSchema } from "../runtime-api.js";
|
||||
|
||||
/**
|
||||
* Twitch user roles that can be allowed to interact with the bot
|
||||
|
||||
@@ -1,3 +1,5 @@
|
||||
export { whatsappPlugin } from "./src/channel.js";
|
||||
export { setWhatsAppRuntime } from "./src/runtime.js";
|
||||
export { whatsappOutbound } from "./src/outbound-adapter.js";
|
||||
export { deliverWebReply } from "./src/auto-reply/deliver-reply.js";
|
||||
export {
|
||||
|
||||
@@ -3,9 +3,9 @@ import {
|
||||
buildCatchallMultiAccountChannelSchema,
|
||||
DmPolicySchema,
|
||||
GroupPolicySchema,
|
||||
} from "openclaw/plugin-sdk/channel-config-schema";
|
||||
MarkdownConfigSchema,
|
||||
} from "openclaw/plugin-sdk/channel-config-primitives";
|
||||
import { z } from "openclaw/plugin-sdk/zod";
|
||||
import { MarkdownConfigSchema } from "./runtime-api.js";
|
||||
import { buildSecretInputSchema } from "./secret-input.js";
|
||||
|
||||
const zaloAccountSchema = z.object({
|
||||
|
||||
@@ -1,11 +1,12 @@
|
||||
import { ToolPolicySchema } from "openclaw/plugin-sdk/agent-config-primitives";
|
||||
import {
|
||||
AllowFromListSchema,
|
||||
buildCatchallMultiAccountChannelSchema,
|
||||
DmPolicySchema,
|
||||
GroupPolicySchema,
|
||||
} from "openclaw/plugin-sdk/channel-config-schema";
|
||||
MarkdownConfigSchema,
|
||||
} from "openclaw/plugin-sdk/channel-config-primitives";
|
||||
import { z } from "openclaw/plugin-sdk/zod";
|
||||
import { MarkdownConfigSchema, ToolPolicySchema } from "../runtime-api.js";
|
||||
|
||||
const groupConfigSchema = z.object({
|
||||
allow: z.boolean().optional(),
|
||||
|
||||
@@ -228,6 +228,10 @@
|
||||
"types": "./dist/plugin-sdk/account-resolution.d.ts",
|
||||
"default": "./dist/plugin-sdk/account-resolution.js"
|
||||
},
|
||||
"./plugin-sdk/agent-config-primitives": {
|
||||
"types": "./dist/plugin-sdk/agent-config-primitives.d.ts",
|
||||
"default": "./dist/plugin-sdk/agent-config-primitives.js"
|
||||
},
|
||||
"./plugin-sdk/allow-from": {
|
||||
"types": "./dist/plugin-sdk/allow-from.d.ts",
|
||||
"default": "./dist/plugin-sdk/allow-from.js"
|
||||
@@ -292,6 +296,10 @@
|
||||
"types": "./dist/plugin-sdk/channel-config-schema.d.ts",
|
||||
"default": "./dist/plugin-sdk/channel-config-schema.js"
|
||||
},
|
||||
"./plugin-sdk/channel-config-primitives": {
|
||||
"types": "./dist/plugin-sdk/channel-config-primitives.d.ts",
|
||||
"default": "./dist/plugin-sdk/channel-config-primitives.js"
|
||||
},
|
||||
"./plugin-sdk/channel-actions": {
|
||||
"types": "./dist/plugin-sdk/channel-actions.d.ts",
|
||||
"default": "./dist/plugin-sdk/channel-actions.js"
|
||||
|
||||
@@ -47,6 +47,7 @@
|
||||
"account-helpers",
|
||||
"account-id",
|
||||
"account-resolution",
|
||||
"agent-config-primitives",
|
||||
"allow-from",
|
||||
"allowlist-config-edit",
|
||||
"bluebubbles",
|
||||
@@ -62,6 +63,7 @@
|
||||
"discord-core",
|
||||
"extension-shared",
|
||||
"channel-config-helpers",
|
||||
"channel-config-primitives",
|
||||
"channel-config-schema",
|
||||
"channel-actions",
|
||||
"channel-contract",
|
||||
|
||||
@@ -5,17 +5,9 @@ import { existsSync, mkdtempSync, readFileSync, rmSync } from "node:fs";
|
||||
import { tmpdir } from "node:os";
|
||||
import { join } from "node:path";
|
||||
import { pathToFileURL } from "node:url";
|
||||
import { BUNDLED_RUNTIME_SIDECAR_PATHS } from "../src/plugins/public-artifacts.ts";
|
||||
import { parseReleaseVersion, resolveNpmCommandInvocation } from "./openclaw-npm-release-check.ts";
|
||||
|
||||
const REQUIRED_RUNTIME_SIDECARS = [
|
||||
"dist/extensions/whatsapp/light-runtime-api.js",
|
||||
"dist/extensions/whatsapp/runtime-api.js",
|
||||
"dist/extensions/matrix/helper-api.js",
|
||||
"dist/extensions/matrix/runtime-api.js",
|
||||
"dist/extensions/matrix/thread-bindings-runtime.js",
|
||||
"dist/extensions/msteams/runtime-api.js",
|
||||
] as const;
|
||||
|
||||
type InstalledPackageJson = {
|
||||
version?: string;
|
||||
};
|
||||
@@ -65,7 +57,7 @@ export function collectInstalledPackageErrors(params: {
|
||||
);
|
||||
}
|
||||
|
||||
for (const relativePath of REQUIRED_RUNTIME_SIDECARS) {
|
||||
for (const relativePath of BUNDLED_RUNTIME_SIDECAR_PATHS) {
|
||||
if (!existsSync(join(params.packageRoot, relativePath))) {
|
||||
errors.push(`installed package is missing required bundled runtime sidecar: ${relativePath}`);
|
||||
}
|
||||
|
||||
@@ -1,8 +1,8 @@
|
||||
import fs from "node:fs/promises";
|
||||
import { beforeEach, vi } from "vitest";
|
||||
import { buildAnthropicCliBackend } from "../../extensions/anthropic/cli-backend.js";
|
||||
import { buildGoogleGeminiCliBackend } from "../../extensions/google/cli-backend.js";
|
||||
import { buildOpenAICodexCliBackend } from "../../extensions/openai/cli-backend.js";
|
||||
import { buildAnthropicCliBackend } from "../../extensions/anthropic/test-api.js";
|
||||
import { buildGoogleGeminiCliBackend } from "../../extensions/google/test-api.js";
|
||||
import { buildOpenAICodexCliBackend } from "../../extensions/openai/test-api.js";
|
||||
import type { OpenClawConfig } from "../config/config.js";
|
||||
import { createEmptyPluginRegistry } from "../plugins/registry.js";
|
||||
import { setActivePluginRegistry } from "../plugins/runtime.js";
|
||||
|
||||
@@ -1,32 +1,32 @@
|
||||
import { describe, expect, it, vi } from "vitest";
|
||||
import { beforeEach, describe, expect, it, vi } from "vitest";
|
||||
import { SILENT_REPLY_TOKEN } from "../../auto-reply/tokens.js";
|
||||
|
||||
vi.mock("../../auto-reply/tokens.js", () => ({
|
||||
SILENT_REPLY_TOKEN: "QUIET_TOKEN",
|
||||
}));
|
||||
|
||||
vi.mock("../../tts/tts.js", () => ({
|
||||
textToSpeech: vi.fn(),
|
||||
}));
|
||||
|
||||
const { createTtsTool } = await import("./tts-tool.js");
|
||||
const { textToSpeech } = await import("../../tts/tts.js");
|
||||
let textToSpeechSpy: ReturnType<typeof vi.spyOn>;
|
||||
|
||||
describe("createTtsTool", () => {
|
||||
it("uses SILENT_REPLY_TOKEN in guidance text", () => {
|
||||
beforeEach(async () => {
|
||||
vi.restoreAllMocks();
|
||||
vi.resetModules();
|
||||
const ttsRuntime = await import("../../tts/tts.js");
|
||||
textToSpeechSpy = vi.spyOn(ttsRuntime, "textToSpeech");
|
||||
});
|
||||
|
||||
it("uses SILENT_REPLY_TOKEN in guidance text", async () => {
|
||||
const { createTtsTool } = await import("./tts-tool.js");
|
||||
const tool = createTtsTool();
|
||||
|
||||
expect(tool.description).toContain("QUIET_TOKEN");
|
||||
expect(tool.description).not.toContain("NO_REPLY");
|
||||
expect(tool.description).toContain(SILENT_REPLY_TOKEN);
|
||||
});
|
||||
|
||||
it("stores audio delivery in details.media", async () => {
|
||||
vi.mocked(textToSpeech).mockResolvedValue({
|
||||
textToSpeechSpy.mockResolvedValue({
|
||||
success: true,
|
||||
audioPath: "/tmp/reply.opus",
|
||||
provider: "test",
|
||||
voiceCompatible: true,
|
||||
});
|
||||
|
||||
const { createTtsTool } = await import("./tts-tool.js");
|
||||
const tool = createTtsTool();
|
||||
const result = await tool.execute("call-1", { text: "hello" });
|
||||
|
||||
|
||||
@@ -12,10 +12,6 @@ vi.mock("../../agents/bootstrap-files.js", () => ({
|
||||
})),
|
||||
}));
|
||||
|
||||
vi.mock("../../agents/pi-tools.js", () => ({
|
||||
createOpenClawCodingTools: createOpenClawCodingToolsMock,
|
||||
}));
|
||||
|
||||
vi.mock("../../agents/sandbox.js", () => ({
|
||||
resolveSandboxRuntimeStatus: vi.fn(() => ({ sandboxed: false, mode: "off" })),
|
||||
}));
|
||||
@@ -57,12 +53,6 @@ vi.mock("../../infra/skills-remote.js", () => ({
|
||||
getRemoteSkillEligibility: vi.fn(() => false),
|
||||
}));
|
||||
|
||||
vi.mock("../../tts/tts.js", () => ({
|
||||
buildTtsSystemPromptHint: vi.fn(() => undefined),
|
||||
}));
|
||||
|
||||
import { resolveCommandsSystemPromptBundle } from "./commands-system-prompt.js";
|
||||
|
||||
function makeParams(): HandleCommandsParams {
|
||||
return {
|
||||
ctx: {
|
||||
@@ -107,12 +97,21 @@ function makeParams(): HandleCommandsParams {
|
||||
}
|
||||
|
||||
describe("resolveCommandsSystemPromptBundle", () => {
|
||||
beforeEach(() => {
|
||||
beforeEach(async () => {
|
||||
vi.restoreAllMocks();
|
||||
vi.resetModules();
|
||||
createOpenClawCodingToolsMock.mockClear();
|
||||
createOpenClawCodingToolsMock.mockReturnValue([]);
|
||||
const piTools = await import("../../agents/pi-tools.js");
|
||||
vi.spyOn(piTools, "createOpenClawCodingTools").mockImplementation(
|
||||
createOpenClawCodingToolsMock,
|
||||
);
|
||||
const ttsRuntime = await import("../../tts/tts.js");
|
||||
vi.spyOn(ttsRuntime, "buildTtsSystemPromptHint").mockReturnValue(undefined);
|
||||
});
|
||||
|
||||
it("opts command tool builds into gateway subagent binding", async () => {
|
||||
const { resolveCommandsSystemPromptBundle } = await import("./commands-system-prompt.js");
|
||||
await resolveCommandsSystemPromptBundle(makeParams());
|
||||
|
||||
expect(createOpenClawCodingToolsMock).toHaveBeenCalledWith(
|
||||
|
||||
1
src/cli/prompt.runtime.ts
Normal file
1
src/cli/prompt.runtime.ts
Normal file
@@ -0,0 +1 @@
|
||||
export { promptYesNo } from "./prompt.js";
|
||||
@@ -4,8 +4,8 @@ import path from "node:path";
|
||||
import { Command } from "commander";
|
||||
import { beforeEach, describe, expect, it, vi } from "vitest";
|
||||
import type { OpenClawConfig, ConfigFileSnapshot } from "../config/types.openclaw.js";
|
||||
import { BUNDLED_RUNTIME_SIDECAR_PATHS } from "../extensions/public-artifacts.js";
|
||||
import type { UpdateRunResult } from "../infra/update-runner.js";
|
||||
import { BUNDLED_RUNTIME_SIDECAR_PATHS } from "../plugins/public-artifacts.js";
|
||||
import { withEnvAsync } from "../test-utils/env.js";
|
||||
import { createCliRuntimeCapture } from "./test-runtime-capture.js";
|
||||
|
||||
|
||||
@@ -1,8 +1,8 @@
|
||||
import { matrixPlugin, setMatrixRuntime } from "../../extensions/matrix/index.js";
|
||||
import { msteamsPlugin } from "../../extensions/msteams/index.js";
|
||||
import { nostrPlugin } from "../../extensions/nostr/index.js";
|
||||
import { tlonPlugin } from "../../extensions/tlon/index.js";
|
||||
import { whatsappPlugin } from "../../extensions/whatsapp/index.js";
|
||||
import { matrixPlugin, setMatrixRuntime } from "../../extensions/matrix/test-api.js";
|
||||
import { msteamsPlugin } from "../../extensions/msteams/test-api.js";
|
||||
import { nostrPlugin } from "../../extensions/nostr/test-api.js";
|
||||
import { tlonPlugin } from "../../extensions/tlon/test-api.js";
|
||||
import { whatsappPlugin } from "../../extensions/whatsapp/test-api.js";
|
||||
import { bundledChannelPlugins } from "../channels/plugins/bundled.js";
|
||||
import { setActivePluginRegistry } from "../plugins/runtime.js";
|
||||
import { createTestRegistry } from "../test-utils/channel-plugins.js";
|
||||
|
||||
@@ -1,10 +1,10 @@
|
||||
import { vi } from "vitest";
|
||||
import { parseTelegramTarget } from "../../extensions/telegram/api.js";
|
||||
import { signalOutbound, telegramOutbound } from "../../test/channel-outbounds.js";
|
||||
import { loadModelCatalog } from "../agents/model-catalog.js";
|
||||
import { runEmbeddedPiAgent } from "../agents/pi-embedded.js";
|
||||
import { runSubagentAnnounceFlow } from "../agents/subagent-announce.js";
|
||||
import { callGateway } from "../gateway/call.js";
|
||||
import { parseTelegramTarget } from "../plugin-sdk/telegram.js";
|
||||
import { setActivePluginRegistry } from "../plugins/runtime.js";
|
||||
import { createOutboundTestPlugin, createTestRegistry } from "../test-utils/channel-plugins.js";
|
||||
|
||||
|
||||
@@ -4,8 +4,8 @@ import fs from "node:fs/promises";
|
||||
import os from "node:os";
|
||||
import path from "node:path";
|
||||
import { Mock, vi } from "vitest";
|
||||
import { buildElevenLabsSpeechProvider } from "../../extensions/elevenlabs/speech-provider.ts";
|
||||
import { buildOpenAISpeechProvider } from "../../extensions/openai/speech-provider.ts";
|
||||
import { buildElevenLabsSpeechProvider } from "../../extensions/elevenlabs/test-api.ts";
|
||||
import { buildOpenAISpeechProvider } from "../../extensions/openai/test-api.ts";
|
||||
import type { MsgContext } from "../auto-reply/templating.js";
|
||||
import type { GetReplyOptions, ReplyPayload } from "../auto-reply/types.js";
|
||||
import type { ChannelPlugin, ChannelOutboundAdapter } from "../channels/plugins/types.js";
|
||||
|
||||
@@ -1,183 +1,6 @@
|
||||
import type { AuthProfileStore } from "../agents/auth-profiles.js";
|
||||
import { describeFailoverError, isFailoverError } from "../agents/failover-error.js";
|
||||
import type { FallbackAttempt } from "../agents/model-fallback.types.js";
|
||||
import type { OpenClawConfig } from "../config/config.js";
|
||||
import {
|
||||
resolveAgentModelFallbackValues,
|
||||
resolveAgentModelPrimaryValue,
|
||||
} from "../config/model-input.js";
|
||||
import { createSubsystemLogger } from "../logging/subsystem.js";
|
||||
import { getProviderEnvVars } from "../secrets/provider-env-vars.js";
|
||||
import { parseImageGenerationModelRef } from "./model-ref.js";
|
||||
import { getImageGenerationProvider, listImageGenerationProviders } from "./provider-registry.js";
|
||||
import type {
|
||||
GeneratedImageAsset,
|
||||
ImageGenerationResolution,
|
||||
ImageGenerationResult,
|
||||
ImageGenerationSourceImage,
|
||||
} from "./types.js";
|
||||
|
||||
const log = createSubsystemLogger("image-generation");
|
||||
|
||||
export type GenerateImageParams = {
|
||||
cfg: OpenClawConfig;
|
||||
prompt: string;
|
||||
agentDir?: string;
|
||||
authStore?: AuthProfileStore;
|
||||
modelOverride?: string;
|
||||
count?: number;
|
||||
size?: string;
|
||||
aspectRatio?: string;
|
||||
resolution?: ImageGenerationResolution;
|
||||
inputImages?: ImageGenerationSourceImage[];
|
||||
};
|
||||
|
||||
export type GenerateImageRuntimeResult = {
|
||||
images: GeneratedImageAsset[];
|
||||
provider: string;
|
||||
model: string;
|
||||
attempts: FallbackAttempt[];
|
||||
metadata?: Record<string, unknown>;
|
||||
};
|
||||
|
||||
function resolveImageGenerationCandidates(params: {
|
||||
cfg: OpenClawConfig;
|
||||
modelOverride?: string;
|
||||
}): Array<{ provider: string; model: string }> {
|
||||
const candidates: Array<{ provider: string; model: string }> = [];
|
||||
const seen = new Set<string>();
|
||||
const add = (raw: string | undefined) => {
|
||||
const parsed = parseImageGenerationModelRef(raw);
|
||||
if (!parsed) {
|
||||
return;
|
||||
}
|
||||
const key = `${parsed.provider}/${parsed.model}`;
|
||||
if (seen.has(key)) {
|
||||
return;
|
||||
}
|
||||
seen.add(key);
|
||||
candidates.push(parsed);
|
||||
};
|
||||
|
||||
add(params.modelOverride);
|
||||
add(resolveAgentModelPrimaryValue(params.cfg.agents?.defaults?.imageGenerationModel));
|
||||
for (const fallback of resolveAgentModelFallbackValues(
|
||||
params.cfg.agents?.defaults?.imageGenerationModel,
|
||||
)) {
|
||||
add(fallback);
|
||||
}
|
||||
return candidates;
|
||||
}
|
||||
|
||||
function throwImageGenerationFailure(params: {
|
||||
attempts: FallbackAttempt[];
|
||||
lastError: unknown;
|
||||
}): never {
|
||||
if (params.attempts.length <= 1 && params.lastError) {
|
||||
throw params.lastError;
|
||||
}
|
||||
const summary =
|
||||
params.attempts.length > 0
|
||||
? params.attempts
|
||||
.map((attempt) => `${attempt.provider}/${attempt.model}: ${attempt.error}`)
|
||||
.join(" | ")
|
||||
: "unknown";
|
||||
throw new Error(`All image generation models failed (${params.attempts.length}): ${summary}`, {
|
||||
cause: params.lastError instanceof Error ? params.lastError : undefined,
|
||||
});
|
||||
}
|
||||
|
||||
function buildNoImageGenerationModelConfiguredMessage(cfg: OpenClawConfig): string {
|
||||
const providers = listImageGenerationProviders(cfg);
|
||||
const sampleModel =
|
||||
providers.find((provider) => provider.defaultModel) ??
|
||||
({ id: "google", defaultModel: "gemini-3-pro-image-preview" } as const);
|
||||
const authHints = providers
|
||||
.flatMap((provider) => {
|
||||
const envVars = getProviderEnvVars(provider.id);
|
||||
if (envVars.length === 0) {
|
||||
return [];
|
||||
}
|
||||
return [`${provider.id}: ${envVars.join(" / ")}`];
|
||||
})
|
||||
.slice(0, 3);
|
||||
return [
|
||||
`No image-generation model configured. Set agents.defaults.imageGenerationModel.primary to a provider/model like "${sampleModel.id}/${sampleModel.defaultModel}".`,
|
||||
authHints.length > 0
|
||||
? `If you want a specific provider, also configure that provider's auth/API key first (${authHints.join("; ")}).`
|
||||
: "If you want a specific provider, also configure that provider's auth/API key first.",
|
||||
].join(" ");
|
||||
}
|
||||
|
||||
export function listRuntimeImageGenerationProviders(params?: { config?: OpenClawConfig }) {
|
||||
return listImageGenerationProviders(params?.config);
|
||||
}
|
||||
|
||||
export async function generateImage(
|
||||
params: GenerateImageParams,
|
||||
): Promise<GenerateImageRuntimeResult> {
|
||||
const candidates = resolveImageGenerationCandidates({
|
||||
cfg: params.cfg,
|
||||
modelOverride: params.modelOverride,
|
||||
});
|
||||
if (candidates.length === 0) {
|
||||
throw new Error(buildNoImageGenerationModelConfiguredMessage(params.cfg));
|
||||
}
|
||||
|
||||
const attempts: FallbackAttempt[] = [];
|
||||
let lastError: unknown;
|
||||
|
||||
for (const candidate of candidates) {
|
||||
const provider = getImageGenerationProvider(candidate.provider, params.cfg);
|
||||
if (!provider) {
|
||||
const error = `No image-generation provider registered for ${candidate.provider}`;
|
||||
attempts.push({
|
||||
provider: candidate.provider,
|
||||
model: candidate.model,
|
||||
error,
|
||||
});
|
||||
lastError = new Error(error);
|
||||
continue;
|
||||
}
|
||||
|
||||
try {
|
||||
const result: ImageGenerationResult = await provider.generateImage({
|
||||
provider: candidate.provider,
|
||||
model: candidate.model,
|
||||
prompt: params.prompt,
|
||||
cfg: params.cfg,
|
||||
agentDir: params.agentDir,
|
||||
authStore: params.authStore,
|
||||
count: params.count,
|
||||
size: params.size,
|
||||
aspectRatio: params.aspectRatio,
|
||||
resolution: params.resolution,
|
||||
inputImages: params.inputImages,
|
||||
});
|
||||
if (!Array.isArray(result.images) || result.images.length === 0) {
|
||||
throw new Error("Image generation provider returned no images.");
|
||||
}
|
||||
return {
|
||||
images: result.images,
|
||||
provider: candidate.provider,
|
||||
model: result.model ?? candidate.model,
|
||||
attempts,
|
||||
metadata: result.metadata,
|
||||
};
|
||||
} catch (err) {
|
||||
lastError = err;
|
||||
const described = isFailoverError(err) ? describeFailoverError(err) : undefined;
|
||||
attempts.push({
|
||||
provider: candidate.provider,
|
||||
model: candidate.model,
|
||||
error: described?.message ?? (err instanceof Error ? err.message : String(err)),
|
||||
reason: described?.reason,
|
||||
status: described?.status,
|
||||
code: described?.code,
|
||||
});
|
||||
log.debug(`image-generation candidate failed: ${candidate.provider}/${candidate.model}`);
|
||||
}
|
||||
}
|
||||
|
||||
throwImageGenerationFailure({ attempts, lastError });
|
||||
}
|
||||
export {
|
||||
generateImage,
|
||||
listRuntimeImageGenerationProviders,
|
||||
type GenerateImageParams,
|
||||
type GenerateImageRuntimeResult,
|
||||
} from "../plugin-sdk/image-generation-runtime.js";
|
||||
|
||||
1
src/infra/binaries.runtime.ts
Normal file
1
src/infra/binaries.runtime.ts
Normal file
@@ -0,0 +1 @@
|
||||
export { ensureBinary } from "./binaries.js";
|
||||
@@ -1,5 +1,4 @@
|
||||
import { createSubsystemLogger } from "../logging/subsystem.js";
|
||||
import { parseBooleanValue } from "../utils/boolean.js";
|
||||
|
||||
let log: ReturnType<typeof createSubsystemLogger> | null = null;
|
||||
const loggedEnv = new Set<string>();
|
||||
@@ -53,7 +52,18 @@ export function normalizeZaiEnv(): void {
|
||||
}
|
||||
|
||||
export function isTruthyEnvValue(value?: string): boolean {
|
||||
return parseBooleanValue(value) === true;
|
||||
if (typeof value !== "string") {
|
||||
return false;
|
||||
}
|
||||
switch (value.trim().toLowerCase()) {
|
||||
case "1":
|
||||
case "on":
|
||||
case "true":
|
||||
case "yes":
|
||||
return true;
|
||||
default:
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
export function normalizeEnv(): void {
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
import { beforeEach } from "vitest";
|
||||
import { slackPlugin, setSlackRuntime } from "../../extensions/slack/index.js";
|
||||
import { telegramPlugin, setTelegramRuntime } from "../../extensions/telegram/index.js";
|
||||
import { whatsappPlugin, setWhatsAppRuntime } from "../../extensions/whatsapp/index.js";
|
||||
import { slackPlugin, setSlackRuntime } from "../../extensions/slack/test-api.js";
|
||||
import { telegramPlugin, setTelegramRuntime } from "../../extensions/telegram/test-api.js";
|
||||
import { whatsappPlugin, setWhatsAppRuntime } from "../../extensions/whatsapp/test-api.js";
|
||||
import type { ChannelPlugin } from "../channels/plugins/types.plugin.js";
|
||||
import { setActivePluginRegistry } from "../plugins/runtime.js";
|
||||
import { createPluginRuntime } from "../plugins/runtime/index.js";
|
||||
|
||||
@@ -2,7 +2,7 @@ import fs from "node:fs/promises";
|
||||
import os from "node:os";
|
||||
import path from "node:path";
|
||||
import { vi } from "vitest";
|
||||
import { telegramPlugin, setTelegramRuntime } from "../../extensions/telegram/index.js";
|
||||
import { telegramPlugin, setTelegramRuntime } from "../../extensions/telegram/test-api.js";
|
||||
import * as replyModule from "../auto-reply/reply.js";
|
||||
import type { OpenClawConfig } from "../config/config.js";
|
||||
import { resolveMainSessionKey } from "../config/sessions.js";
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
import { slackPlugin, setSlackRuntime } from "../../../extensions/slack/index.js";
|
||||
import { telegramPlugin, setTelegramRuntime } from "../../../extensions/telegram/index.js";
|
||||
import { slackPlugin, setSlackRuntime } from "../../../extensions/slack/test-api.js";
|
||||
import { telegramPlugin, setTelegramRuntime } from "../../../extensions/telegram/test-api.js";
|
||||
import type { OpenClawConfig } from "../../config/config.js";
|
||||
import { setActivePluginRegistry } from "../../plugins/runtime.js";
|
||||
import { createPluginRuntime } from "../../plugins/runtime/index.js";
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
import { afterEach, beforeEach, describe, expect, it } from "vitest";
|
||||
import { parseTelegramTarget } from "../../../extensions/telegram/api.js";
|
||||
import { telegramOutbound, whatsappOutbound } from "../../../test/channel-outbounds.js";
|
||||
import type { OpenClawConfig } from "../../config/config.js";
|
||||
import { parseTelegramTarget } from "../../plugin-sdk/telegram.js";
|
||||
import { isWhatsAppGroupJid, normalizeWhatsAppTarget } from "../../plugin-sdk/whatsapp-shared.js";
|
||||
import { setActivePluginRegistry } from "../../plugins/runtime.js";
|
||||
import { createOutboundTestPlugin, createTestRegistry } from "../../test-utils/channel-plugins.js";
|
||||
|
||||
@@ -4,7 +4,11 @@ const resolveProviderUsageAuthWithPluginMock = vi.fn(
|
||||
async (..._args: unknown[]): Promise<unknown> => null,
|
||||
);
|
||||
|
||||
vi.mock("../plugins/provider-runtime.js", () => ({
|
||||
const resolveProviderCapabilitiesWithPluginMock = vi.fn(() => undefined);
|
||||
|
||||
vi.mock("../plugins/provider-runtime.js", async (importOriginal) => ({
|
||||
...(await importOriginal<typeof import("../plugins/provider-runtime.js")>()),
|
||||
resolveProviderCapabilitiesWithPlugin: resolveProviderCapabilitiesWithPluginMock,
|
||||
resolveProviderUsageAuthWithPlugin: resolveProviderUsageAuthWithPluginMock,
|
||||
}));
|
||||
|
||||
|
||||
@@ -14,13 +14,13 @@ describe("library module imports", () => {
|
||||
replyRuntimeLoads();
|
||||
return await importOriginal<typeof import("./auto-reply/reply.runtime.js")>();
|
||||
});
|
||||
vi.doMock("./cli/prompt.js", async (importOriginal) => {
|
||||
vi.doMock("./cli/prompt.runtime.js", async (importOriginal) => {
|
||||
promptRuntimeLoads();
|
||||
return await importOriginal<typeof import("./cli/prompt.js")>();
|
||||
return await importOriginal<typeof import("./cli/prompt.runtime.js")>();
|
||||
});
|
||||
vi.doMock("./infra/binaries.js", async (importOriginal) => {
|
||||
vi.doMock("./infra/binaries.runtime.js", async (importOriginal) => {
|
||||
binariesRuntimeLoads();
|
||||
return await importOriginal<typeof import("./infra/binaries.js")>();
|
||||
return await importOriginal<typeof import("./infra/binaries.runtime.js")>();
|
||||
});
|
||||
vi.doMock("./plugins/runtime/runtime-whatsapp-boundary.js", async (importOriginal) => {
|
||||
whatsappRuntimeLoads();
|
||||
@@ -32,12 +32,12 @@ describe("library module imports", () => {
|
||||
await import("./library.js");
|
||||
|
||||
expect(replyRuntimeLoads).not.toHaveBeenCalled();
|
||||
expect(promptRuntimeLoads).not.toHaveBeenCalled();
|
||||
expect(binariesRuntimeLoads).not.toHaveBeenCalled();
|
||||
expect(whatsappRuntimeLoads).not.toHaveBeenCalled();
|
||||
// Vitest eagerly resolves some manual mocks for runtime-boundary modules
|
||||
// even when the lazy wrapper is not invoked. Keep the assertion on the
|
||||
// reply runtime, which is the stable import-time contract this test cares about.
|
||||
vi.doUnmock("./auto-reply/reply.runtime.js");
|
||||
vi.doUnmock("./cli/prompt.js");
|
||||
vi.doUnmock("./infra/binaries.js");
|
||||
vi.doUnmock("./cli/prompt.runtime.js");
|
||||
vi.doUnmock("./infra/binaries.runtime.js");
|
||||
vi.doUnmock("./plugins/runtime/runtime-whatsapp-boundary.js");
|
||||
});
|
||||
});
|
||||
|
||||
@@ -13,57 +13,51 @@ import {
|
||||
} from "./infra/ports.js";
|
||||
import { assertWebChannel, normalizeE164, toWhatsappJid } from "./utils.js";
|
||||
|
||||
type GetReplyFromConfig = typeof import("./auto-reply/reply.runtime.js").getReplyFromConfig;
|
||||
type PromptYesNo = typeof import("./cli/prompt.js").promptYesNo;
|
||||
type EnsureBinary = typeof import("./infra/binaries.js").ensureBinary;
|
||||
type RunExec = typeof import("./process/exec.js").runExec;
|
||||
type RunCommandWithTimeout = typeof import("./process/exec.js").runCommandWithTimeout;
|
||||
type MonitorWebChannel =
|
||||
typeof import("./plugins/runtime/runtime-whatsapp-boundary.js").monitorWebChannel;
|
||||
type ReplyRuntimeModule = typeof import("./auto-reply/reply.runtime.js");
|
||||
type PromptRuntimeModule = typeof import("./cli/prompt.runtime.js");
|
||||
type BinariesRuntimeModule = typeof import("./infra/binaries.runtime.js");
|
||||
type ExecRuntimeModule = typeof import("./process/exec.js");
|
||||
type WhatsAppRuntimeModule = typeof import("./plugins/runtime/runtime-whatsapp-boundary.js");
|
||||
|
||||
let replyRuntimePromise: Promise<typeof import("./auto-reply/reply.runtime.js")> | null = null;
|
||||
let promptRuntimePromise: Promise<typeof import("./cli/prompt.js")> | null = null;
|
||||
let binariesRuntimePromise: Promise<typeof import("./infra/binaries.js")> | null = null;
|
||||
let execRuntimePromise: Promise<typeof import("./process/exec.js")> | null = null;
|
||||
let whatsappRuntimePromise: Promise<
|
||||
typeof import("./plugins/runtime/runtime-whatsapp-boundary.js")
|
||||
> | null = null;
|
||||
let replyRuntimePromise: Promise<ReplyRuntimeModule> | undefined;
|
||||
let promptRuntimePromise: Promise<PromptRuntimeModule> | undefined;
|
||||
let binariesRuntimePromise: Promise<BinariesRuntimeModule> | undefined;
|
||||
let execRuntimePromise: Promise<ExecRuntimeModule> | undefined;
|
||||
let whatsappRuntimePromise: Promise<WhatsAppRuntimeModule> | undefined;
|
||||
|
||||
function loadReplyRuntime() {
|
||||
replyRuntimePromise ??= import("./auto-reply/reply.runtime.js");
|
||||
return replyRuntimePromise;
|
||||
function loadReplyRuntime(): Promise<ReplyRuntimeModule> {
|
||||
return (replyRuntimePromise ??= import("./auto-reply/reply.runtime.js"));
|
||||
}
|
||||
|
||||
function loadPromptRuntime() {
|
||||
promptRuntimePromise ??= import("./cli/prompt.js");
|
||||
return promptRuntimePromise;
|
||||
function loadPromptRuntime(): Promise<PromptRuntimeModule> {
|
||||
return (promptRuntimePromise ??= import("./cli/prompt.runtime.js"));
|
||||
}
|
||||
|
||||
function loadBinariesRuntime() {
|
||||
binariesRuntimePromise ??= import("./infra/binaries.js");
|
||||
return binariesRuntimePromise;
|
||||
function loadBinariesRuntime(): Promise<BinariesRuntimeModule> {
|
||||
return (binariesRuntimePromise ??= import("./infra/binaries.runtime.js"));
|
||||
}
|
||||
|
||||
function loadExecRuntime() {
|
||||
execRuntimePromise ??= import("./process/exec.js");
|
||||
return execRuntimePromise;
|
||||
function loadExecRuntime(): Promise<ExecRuntimeModule> {
|
||||
return (execRuntimePromise ??= import("./process/exec.js"));
|
||||
}
|
||||
|
||||
function loadWhatsAppRuntime() {
|
||||
whatsappRuntimePromise ??= import("./plugins/runtime/runtime-whatsapp-boundary.js");
|
||||
return whatsappRuntimePromise;
|
||||
function loadWhatsAppRuntime(): Promise<WhatsAppRuntimeModule> {
|
||||
return (whatsappRuntimePromise ??= import("./plugins/runtime/runtime-whatsapp-boundary.js"));
|
||||
}
|
||||
|
||||
export const getReplyFromConfig: GetReplyFromConfig = async (...args) =>
|
||||
export const getReplyFromConfig: ReplyRuntimeModule["getReplyFromConfig"] = async (...args) =>
|
||||
(await loadReplyRuntime()).getReplyFromConfig(...args);
|
||||
export const promptYesNo: PromptYesNo = async (...args) =>
|
||||
export const promptYesNo: PromptRuntimeModule["promptYesNo"] = async (...args) =>
|
||||
(await loadPromptRuntime()).promptYesNo(...args);
|
||||
export const ensureBinary: EnsureBinary = async (...args) =>
|
||||
export const ensureBinary: BinariesRuntimeModule["ensureBinary"] = async (...args) =>
|
||||
(await loadBinariesRuntime()).ensureBinary(...args);
|
||||
export const runExec: RunExec = async (...args) => (await loadExecRuntime()).runExec(...args);
|
||||
export const runCommandWithTimeout: RunCommandWithTimeout = async (...args) =>
|
||||
export const runExec: ExecRuntimeModule["runExec"] = async (...args) =>
|
||||
(await loadExecRuntime()).runExec(...args);
|
||||
export const runCommandWithTimeout: ExecRuntimeModule["runCommandWithTimeout"] = async (
|
||||
...args
|
||||
) =>
|
||||
(await loadExecRuntime()).runCommandWithTimeout(...args);
|
||||
export const monitorWebChannel: MonitorWebChannel = async (...args) =>
|
||||
export const monitorWebChannel: WhatsAppRuntimeModule["monitorWebChannel"] = async (...args) =>
|
||||
(await loadWhatsAppRuntime()).monitorWebChannel(...args);
|
||||
|
||||
export {
|
||||
|
||||
@@ -1,146 +1,9 @@
|
||||
import fs from "node:fs/promises";
|
||||
import path from "node:path";
|
||||
import type { MsgContext } from "../auto-reply/templating.js";
|
||||
import type { OpenClawConfig } from "../config/config.js";
|
||||
import { getMediaUnderstandingProvider } from "./provider-registry.js";
|
||||
import {
|
||||
buildProviderRegistry,
|
||||
createMediaAttachmentCache,
|
||||
normalizeMediaAttachments,
|
||||
runCapability,
|
||||
type ActiveMediaModel,
|
||||
} from "./runner.js";
|
||||
import type { MediaUnderstandingCapability, MediaUnderstandingOutput } from "./types.js";
|
||||
|
||||
const KIND_BY_CAPABILITY: Record<MediaUnderstandingCapability, MediaUnderstandingOutput["kind"]> = {
|
||||
audio: "audio.transcription",
|
||||
image: "image.description",
|
||||
video: "video.description",
|
||||
};
|
||||
|
||||
export type RunMediaUnderstandingFileParams = {
|
||||
capability: MediaUnderstandingCapability;
|
||||
filePath: string;
|
||||
cfg: OpenClawConfig;
|
||||
agentDir?: string;
|
||||
mime?: string;
|
||||
activeModel?: ActiveMediaModel;
|
||||
};
|
||||
|
||||
export type RunMediaUnderstandingFileResult = {
|
||||
text: string | undefined;
|
||||
provider?: string;
|
||||
model?: string;
|
||||
output?: MediaUnderstandingOutput;
|
||||
};
|
||||
|
||||
function buildFileContext(params: { filePath: string; mime?: string }): MsgContext {
|
||||
return {
|
||||
MediaPath: params.filePath,
|
||||
MediaType: params.mime,
|
||||
};
|
||||
}
|
||||
|
||||
export async function runMediaUnderstandingFile(
|
||||
params: RunMediaUnderstandingFileParams,
|
||||
): Promise<RunMediaUnderstandingFileResult> {
|
||||
const ctx = buildFileContext(params);
|
||||
const attachments = normalizeMediaAttachments(ctx);
|
||||
if (attachments.length === 0) {
|
||||
return { text: undefined };
|
||||
}
|
||||
|
||||
const providerRegistry = buildProviderRegistry(undefined, params.cfg);
|
||||
const cache = createMediaAttachmentCache(attachments, {
|
||||
localPathRoots: [path.dirname(params.filePath)],
|
||||
});
|
||||
|
||||
try {
|
||||
const result = await runCapability({
|
||||
capability: params.capability,
|
||||
cfg: params.cfg,
|
||||
ctx,
|
||||
attachments: cache,
|
||||
media: attachments,
|
||||
agentDir: params.agentDir,
|
||||
providerRegistry,
|
||||
config: params.cfg.tools?.media?.[params.capability],
|
||||
activeModel: params.activeModel,
|
||||
});
|
||||
const output = result.outputs.find(
|
||||
(entry) => entry.kind === KIND_BY_CAPABILITY[params.capability],
|
||||
);
|
||||
const text = output?.text?.trim();
|
||||
return {
|
||||
text: text || undefined,
|
||||
provider: output?.provider,
|
||||
model: output?.model,
|
||||
output,
|
||||
};
|
||||
} finally {
|
||||
await cache.cleanup();
|
||||
}
|
||||
}
|
||||
|
||||
export async function describeImageFile(params: {
|
||||
filePath: string;
|
||||
cfg: OpenClawConfig;
|
||||
agentDir?: string;
|
||||
mime?: string;
|
||||
activeModel?: ActiveMediaModel;
|
||||
}): Promise<RunMediaUnderstandingFileResult> {
|
||||
return await runMediaUnderstandingFile({ ...params, capability: "image" });
|
||||
}
|
||||
|
||||
export async function describeImageFileWithModel(params: {
|
||||
filePath: string;
|
||||
cfg: OpenClawConfig;
|
||||
agentDir?: string;
|
||||
mime?: string;
|
||||
provider: string;
|
||||
model: string;
|
||||
prompt: string;
|
||||
maxTokens?: number;
|
||||
timeoutMs?: number;
|
||||
}) {
|
||||
const timeoutMs = params.timeoutMs ?? 30_000;
|
||||
const providerRegistry = buildProviderRegistry(undefined, params.cfg);
|
||||
const provider = getMediaUnderstandingProvider(params.provider, providerRegistry);
|
||||
if (!provider?.describeImage) {
|
||||
throw new Error(`Provider does not support image analysis: ${params.provider}`);
|
||||
}
|
||||
const buffer = await fs.readFile(params.filePath);
|
||||
return await provider.describeImage({
|
||||
buffer,
|
||||
fileName: path.basename(params.filePath),
|
||||
mime: params.mime,
|
||||
provider: params.provider,
|
||||
model: params.model,
|
||||
prompt: params.prompt,
|
||||
maxTokens: params.maxTokens,
|
||||
timeoutMs,
|
||||
cfg: params.cfg,
|
||||
agentDir: params.agentDir ?? "",
|
||||
});
|
||||
}
|
||||
|
||||
export async function describeVideoFile(params: {
|
||||
filePath: string;
|
||||
cfg: OpenClawConfig;
|
||||
agentDir?: string;
|
||||
mime?: string;
|
||||
activeModel?: ActiveMediaModel;
|
||||
}): Promise<RunMediaUnderstandingFileResult> {
|
||||
return await runMediaUnderstandingFile({ ...params, capability: "video" });
|
||||
}
|
||||
|
||||
export async function transcribeAudioFile(params: {
|
||||
filePath: string;
|
||||
cfg: OpenClawConfig;
|
||||
agentDir?: string;
|
||||
mime?: string;
|
||||
activeModel?: ActiveMediaModel;
|
||||
}): Promise<{ text: string | undefined }> {
|
||||
const result = await runMediaUnderstandingFile({ ...params, capability: "audio" });
|
||||
return { text: result.text };
|
||||
}
|
||||
export {
|
||||
describeImageFile,
|
||||
describeImageFileWithModel,
|
||||
describeVideoFile,
|
||||
runMediaUnderstandingFile,
|
||||
transcribeAudioFile,
|
||||
type RunMediaUnderstandingFileParams,
|
||||
type RunMediaUnderstandingFileResult,
|
||||
} from "../plugin-sdk/media-understanding-runtime.js";
|
||||
|
||||
@@ -20,19 +20,13 @@ export { normalizeE164, pathExists, resolveUserPath } from "../utils.js";
|
||||
export {
|
||||
resolveDiscordAccount,
|
||||
type ResolvedDiscordAccount,
|
||||
} from "../../extensions/discord/src/accounts.js";
|
||||
export {
|
||||
resolveSlackAccount,
|
||||
type ResolvedSlackAccount,
|
||||
} from "../../extensions/slack/src/accounts.js";
|
||||
} from "../../extensions/discord/api.js";
|
||||
export { resolveSlackAccount, type ResolvedSlackAccount } from "../../extensions/slack/api.js";
|
||||
export {
|
||||
resolveTelegramAccount,
|
||||
type ResolvedTelegramAccount,
|
||||
} from "../../extensions/telegram/src/accounts.js";
|
||||
export {
|
||||
resolveSignalAccount,
|
||||
type ResolvedSignalAccount,
|
||||
} from "../../extensions/signal/src/accounts.js";
|
||||
} from "../../extensions/telegram/api.js";
|
||||
export { resolveSignalAccount, type ResolvedSignalAccount } from "../../extensions/signal/api.js";
|
||||
|
||||
/** Resolve an account by id, then fall back to the default account when the primary lacks credentials. */
|
||||
export function resolveAccountWithDefaultFallback<TAccount>(params: {
|
||||
|
||||
3
src/plugin-sdk/agent-config-primitives.ts
Normal file
3
src/plugin-sdk/agent-config-primitives.ts
Normal file
@@ -0,0 +1,3 @@
|
||||
/** Narrow agent-runtime schema primitives without broader config/runtime surfaces. */
|
||||
export { ReplyRuntimeConfigSchemaShape } from "../config/zod-schema.core.js";
|
||||
export { ToolPolicySchema } from "../config/zod-schema.agent-runtime.js";
|
||||
@@ -28,7 +28,7 @@ export { buildChannelConfigSchema } from "../channels/plugins/config-schema.js";
|
||||
export {
|
||||
resolveBlueBubblesGroupRequireMention,
|
||||
resolveBlueBubblesGroupToolPolicy,
|
||||
} from "../../extensions/bluebubbles/src/group-policy.js";
|
||||
} from "../../extensions/bluebubbles/api.js";
|
||||
export { formatPairingApproveHint } from "../channels/plugins/helpers.js";
|
||||
export { resolveChannelMediaMaxBytes } from "../channels/plugins/media-limits.js";
|
||||
export {
|
||||
@@ -85,6 +85,7 @@ export {
|
||||
buildComputedAccountStatusSnapshot,
|
||||
buildProbeChannelStatusSummary,
|
||||
} from "./status-helpers.js";
|
||||
export { isAllowedBlueBubblesSender } from "../../extensions/bluebubbles/api.js";
|
||||
export { extractToolSend } from "./tool-send.js";
|
||||
export {
|
||||
WEBHOOK_RATE_LIMIT_DEFAULTS,
|
||||
|
||||
16
src/plugin-sdk/channel-config-primitives.ts
Normal file
16
src/plugin-sdk/channel-config-primitives.ts
Normal file
@@ -0,0 +1,16 @@
|
||||
/** Narrow channel config-schema primitives without provider-schema re-exports. */
|
||||
export {
|
||||
AllowFromListSchema,
|
||||
buildChannelConfigSchema,
|
||||
buildCatchallMultiAccountChannelSchema,
|
||||
buildNestedDmConfigSchema,
|
||||
} from "../channels/plugins/config-schema.js";
|
||||
export {
|
||||
BlockStreamingCoalesceSchema,
|
||||
DmConfigSchema,
|
||||
DmPolicySchema,
|
||||
GroupPolicySchema,
|
||||
MarkdownConfigSchema,
|
||||
ReplyRuntimeConfigSchemaShape,
|
||||
requireOpenAllowFrom,
|
||||
} from "../config/zod-schema.core.js";
|
||||
@@ -2,7 +2,7 @@ import { readdirSync, readFileSync } from "node:fs";
|
||||
import { dirname, resolve } from "node:path";
|
||||
import { fileURLToPath } from "node:url";
|
||||
import { describe, expect, it } from "vitest";
|
||||
import { GUARDED_EXTENSION_PUBLIC_SURFACE_BASENAMES } from "../extensions/public-artifacts.js";
|
||||
import { GUARDED_EXTENSION_PUBLIC_SURFACE_BASENAMES } from "../plugins/public-artifacts.js";
|
||||
|
||||
const ROOT_DIR = resolve(dirname(fileURLToPath(import.meta.url)), "..");
|
||||
const REPO_ROOT = resolve(ROOT_DIR, "..");
|
||||
|
||||
@@ -5,6 +5,7 @@ export * from "../channels/chat-type.js";
|
||||
export * from "../channels/reply-prefix.js";
|
||||
export * from "../channels/typing.js";
|
||||
export type * from "../channels/plugins/types.js";
|
||||
export { normalizeChannelId } from "../channels/plugins/registry.js";
|
||||
export * from "../channels/plugins/normalize/signal.js";
|
||||
export * from "../channels/plugins/normalize/whatsapp.js";
|
||||
export * from "../channels/plugins/outbound/interactive.js";
|
||||
|
||||
@@ -48,5 +48,5 @@ export { mapAllowlistResolutionInputs } from "./allow-from.js";
|
||||
export {
|
||||
resolveBlueBubblesGroupRequireMention,
|
||||
resolveBlueBubblesGroupToolPolicy,
|
||||
} from "../../extensions/bluebubbles/src/group-policy.js";
|
||||
} from "../../extensions/bluebubbles/api.js";
|
||||
export { collectBlueBubblesStatusIssues } from "../channels/plugins/status-issues/bluebubbles.js";
|
||||
|
||||
@@ -74,7 +74,11 @@ export type {
|
||||
TelegramInlineButtonsScope,
|
||||
TelegramNetworkConfig,
|
||||
TelegramTopicConfig,
|
||||
TtsAutoMode,
|
||||
TtsConfig,
|
||||
TtsMode,
|
||||
TtsModelOverrideConfig,
|
||||
TtsProvider,
|
||||
} from "../config/types.js";
|
||||
export {
|
||||
loadSessionStore,
|
||||
|
||||
@@ -1,5 +1,7 @@
|
||||
// Shared image-generation implementation helpers for bundled and third-party plugins.
|
||||
|
||||
export type { AuthProfileStore } from "../agents/auth-profiles.js";
|
||||
export type { FallbackAttempt } from "../agents/model-fallback.types.js";
|
||||
export type { ImageGenerationProviderPlugin } from "../plugins/types.js";
|
||||
export type {
|
||||
GeneratedImageAsset,
|
||||
@@ -9,8 +11,21 @@ export type {
|
||||
ImageGenerationResult,
|
||||
ImageGenerationSourceImage,
|
||||
} from "../image-generation/types.js";
|
||||
export type { OpenClawConfig } from "../config/config.js";
|
||||
|
||||
export { describeFailoverError, isFailoverError } from "../agents/failover-error.js";
|
||||
export { resolveApiKeyForProvider } from "../agents/model-auth.js";
|
||||
export { normalizeGoogleModelId } from "../agents/model-id-normalization.js";
|
||||
export {
|
||||
resolveAgentModelFallbackValues,
|
||||
resolveAgentModelPrimaryValue,
|
||||
} from "../config/model-input.js";
|
||||
export { parseGeminiAuth } from "../infra/gemini-auth.js";
|
||||
export {
|
||||
getImageGenerationProvider,
|
||||
listImageGenerationProviders,
|
||||
} from "../image-generation/provider-registry.js";
|
||||
export { parseImageGenerationModelRef } from "../image-generation/model-ref.js";
|
||||
export { createSubsystemLogger } from "../logging/subsystem.js";
|
||||
export { OPENAI_DEFAULT_IMAGE_MODEL } from "../plugins/provider-model-defaults.js";
|
||||
export { getProviderEnvVars } from "../secrets/provider-env-vars.js";
|
||||
|
||||
@@ -1,3 +1,8 @@
|
||||
// Public runtime-facing image-generation helpers for feature/channel plugins.
|
||||
|
||||
export { generateImage, listRuntimeImageGenerationProviders } from "../image-generation/runtime.js";
|
||||
export {
|
||||
generateImage,
|
||||
listRuntimeImageGenerationProviders,
|
||||
type GenerateImageParams,
|
||||
type GenerateImageRuntimeResult,
|
||||
} from "../../extensions/image-generation-core/runtime-api.js";
|
||||
|
||||
@@ -94,3 +94,4 @@ export { getAgentScopedMediaLocalRoots } from "../media/local-roots.js";
|
||||
export { loadOutboundMediaFromUrl } from "./outbound-media.js";
|
||||
export { createChannelPairingController } from "./channel-pairing.js";
|
||||
export { isRequestBodyLimitError, readRequestBodyWithLimit } from "../infra/http-body.js";
|
||||
export { isMattermostSenderAllowed } from "../../extensions/mattermost/api.js";
|
||||
|
||||
@@ -23,6 +23,7 @@ export * from "../media-understanding/audio-preflight.ts";
|
||||
export * from "../media-understanding/defaults.js";
|
||||
export * from "../media-understanding/image-runtime.ts";
|
||||
export * from "../media-understanding/runner.js";
|
||||
export { normalizeMediaProviderId } from "../media-understanding/provider-registry.js";
|
||||
export * from "../polls.js";
|
||||
export {
|
||||
createDirectTextMediaOutbound,
|
||||
|
||||
@@ -6,4 +6,6 @@ export {
|
||||
describeVideoFile,
|
||||
runMediaUnderstandingFile,
|
||||
transcribeAudioFile,
|
||||
} from "../media-understanding/runtime.js";
|
||||
type RunMediaUnderstandingFileParams,
|
||||
type RunMediaUnderstandingFileResult,
|
||||
} from "../../extensions/media-understanding-core/runtime-api.js";
|
||||
|
||||
@@ -4,7 +4,7 @@
|
||||
export type { ChannelMessageActionAdapter } from "../channels/plugins/types.js";
|
||||
export type { OpenClawConfig } from "../config/config.js";
|
||||
export type { SignalAccountConfig } from "../config/types.js";
|
||||
export type { ResolvedSignalAccount } from "../../extensions/signal/src/accounts.js";
|
||||
export type { ResolvedSignalAccount } from "../../extensions/signal/api.js";
|
||||
export type {
|
||||
ChannelMessageActionContext,
|
||||
ChannelPlugin,
|
||||
@@ -54,13 +54,12 @@ export {
|
||||
listEnabledSignalAccounts,
|
||||
listSignalAccountIds,
|
||||
resolveDefaultSignalAccountId,
|
||||
} from "../../extensions/signal/src/accounts.js";
|
||||
export { monitorSignalProvider } from "../../extensions/signal/src/monitor.js";
|
||||
export { probeSignal } from "../../extensions/signal/src/probe.js";
|
||||
export { resolveSignalReactionLevel } from "../../extensions/signal/src/reaction-level.js";
|
||||
export {
|
||||
removeReactionSignal,
|
||||
sendReactionSignal,
|
||||
} from "../../extensions/signal/src/send-reactions.js";
|
||||
export { sendMessageSignal } from "../../extensions/signal/src/send.js";
|
||||
export { signalMessageActions } from "../../extensions/signal/src/message-actions.js";
|
||||
} from "../../extensions/signal/api.js";
|
||||
export { isSignalSenderAllowed } from "../../extensions/signal/api.js";
|
||||
export type { SignalSender } from "../../extensions/signal/api.js";
|
||||
export { monitorSignalProvider } from "../../extensions/signal/api.js";
|
||||
export { probeSignal } from "../../extensions/signal/api.js";
|
||||
export { resolveSignalReactionLevel } from "../../extensions/signal/api.js";
|
||||
export { removeReactionSignal, sendReactionSignal } from "../../extensions/signal/api.js";
|
||||
export { sendMessageSignal } from "../../extensions/signal/api.js";
|
||||
export { signalMessageActions } from "../../extensions/signal/api.js";
|
||||
|
||||
@@ -20,9 +20,18 @@ export type {
|
||||
} from "../tts/provider-types.js";
|
||||
|
||||
export {
|
||||
scheduleCleanup,
|
||||
summarizeText,
|
||||
normalizeApplyTextNormalization,
|
||||
normalizeLanguageCode,
|
||||
normalizeSeed,
|
||||
requireInRange,
|
||||
} from "../tts/tts-core.js";
|
||||
export { parseTtsDirectives } from "../tts/directives.js";
|
||||
export {
|
||||
canonicalizeSpeechProviderId,
|
||||
getSpeechProvider,
|
||||
listSpeechProviders,
|
||||
normalizeSpeechProviderId,
|
||||
} from "../tts/provider-registry.js";
|
||||
export { normalizeTtsAutoMode, TTS_AUTO_MODES } from "../tts/tts-auto-mode.js";
|
||||
|
||||
@@ -1,3 +1,35 @@
|
||||
// Public runtime-facing speech helpers for feature/channel plugins.
|
||||
|
||||
export { listSpeechVoices, textToSpeech, textToSpeechTelephony } from "../tts/runtime.js";
|
||||
export {
|
||||
_test,
|
||||
buildTtsSystemPromptHint,
|
||||
getLastTtsAttempt,
|
||||
getResolvedSpeechProviderConfig,
|
||||
getTtsMaxLength,
|
||||
getTtsProvider,
|
||||
isSummarizationEnabled,
|
||||
isTtsEnabled,
|
||||
isTtsProviderConfigured,
|
||||
listSpeechVoices,
|
||||
maybeApplyTtsToPayload,
|
||||
resolveTtsAutoMode,
|
||||
resolveTtsConfig,
|
||||
resolveTtsPrefsPath,
|
||||
resolveTtsProviderOrder,
|
||||
setLastTtsAttempt,
|
||||
setSummarizationEnabled,
|
||||
setTtsAutoMode,
|
||||
setTtsEnabled,
|
||||
setTtsMaxLength,
|
||||
setTtsProvider,
|
||||
synthesizeSpeech,
|
||||
textToSpeech,
|
||||
textToSpeechTelephony,
|
||||
type ResolvedTtsConfig,
|
||||
type ResolvedTtsModelOverrides,
|
||||
type TtsDirectiveOverrides,
|
||||
type TtsDirectiveParseResult,
|
||||
type TtsResult,
|
||||
type TtsSynthesisResult,
|
||||
type TtsTelephonyResult,
|
||||
} from "../../extensions/speech-core/runtime-api.js";
|
||||
|
||||
@@ -16,7 +16,7 @@ export function resolvePluginCapabilityProviders<K extends CapabilityProviderReg
|
||||
cfg?: OpenClawConfig;
|
||||
useActiveRegistryWhen?: (active: PluginRegistry | undefined) => boolean;
|
||||
}): CapabilityProviderForKey<K>[] {
|
||||
const active = getActivePluginRegistry();
|
||||
const active = getActivePluginRegistry() ?? undefined;
|
||||
const shouldUseActive =
|
||||
params.useActiveRegistryWhen?.(active) ?? (active?.[params.key].length ?? 0) > 0;
|
||||
const registry =
|
||||
|
||||
@@ -1,7 +1,9 @@
|
||||
import {
|
||||
BUNDLED_IMAGE_GENERATION_PLUGIN_IDS,
|
||||
BUNDLED_MEDIA_UNDERSTANDING_PLUGIN_IDS,
|
||||
BUNDLED_PLUGIN_CONTRACT_SNAPSHOTS,
|
||||
BUNDLED_PROVIDER_PLUGIN_IDS,
|
||||
BUNDLED_SPEECH_PLUGIN_IDS,
|
||||
BUNDLED_WEB_SEARCH_PLUGIN_IDS,
|
||||
} from "../bundled-capability-metadata.js";
|
||||
import { loadBundledCapabilityRuntimeRegistry } from "../bundled-capability-runtime.js";
|
||||
@@ -55,33 +57,6 @@ function createProviderContractPluginIdsByProviderId(): Map<string, string[]> {
|
||||
return result;
|
||||
}
|
||||
|
||||
function createContractSpeechProvider(providerId: string): SpeechProviderPlugin {
|
||||
return {
|
||||
id: providerId,
|
||||
label: providerId,
|
||||
isConfigured: () => true,
|
||||
synthesize: async () => ({
|
||||
audioBuffer: Buffer.alloc(0),
|
||||
outputFormat: "mp3",
|
||||
fileExtension: "mp3",
|
||||
voiceCompatible: true,
|
||||
}),
|
||||
listVoices: async () => [],
|
||||
};
|
||||
}
|
||||
|
||||
function createContractMediaUnderstandingProvider(
|
||||
providerId: string,
|
||||
): MediaUnderstandingProviderPlugin {
|
||||
return {
|
||||
id: providerId,
|
||||
capabilities: ["image"],
|
||||
describeImages: async () => {
|
||||
throw new Error(`media-understanding contract stub invoked for ${providerId}`);
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
function uniqueStrings(values: readonly string[]): string[] {
|
||||
const result: string[] = [];
|
||||
const seen = new Set<string>();
|
||||
@@ -163,9 +138,7 @@ function loadProviderContractPluginIds(): string[] {
|
||||
}
|
||||
|
||||
function loadProviderContractCompatPluginIds(): string[] {
|
||||
return loadProviderContractPluginIds().map((pluginId) =>
|
||||
pluginId === "kimi-coding" ? "kimi" : pluginId,
|
||||
);
|
||||
return loadProviderContractPluginIds();
|
||||
}
|
||||
|
||||
function resolveWebSearchCredentialValue(provider: WebSearchProviderPlugin): unknown {
|
||||
@@ -199,25 +172,29 @@ function loadWebSearchProviderContractRegistry(): WebSearchProviderContractEntry
|
||||
|
||||
function loadSpeechProviderContractRegistry(): SpeechProviderContractEntry[] {
|
||||
if (!speechProviderContractRegistryCache) {
|
||||
// Contract tests only need bundled ownership and public speech surface shape.
|
||||
speechProviderContractRegistryCache = BUNDLED_PLUGIN_CONTRACT_SNAPSHOTS.flatMap((entry) =>
|
||||
entry.speechProviderIds.map((providerId) => ({
|
||||
pluginId: entry.pluginId,
|
||||
provider: createContractSpeechProvider(providerId),
|
||||
})),
|
||||
);
|
||||
const registry = loadBundledCapabilityRuntimeRegistry({
|
||||
pluginIds: BUNDLED_SPEECH_PLUGIN_IDS,
|
||||
pluginSdkResolution: "dist",
|
||||
});
|
||||
speechProviderContractRegistryCache = registry.speechProviders.map((entry) => ({
|
||||
pluginId: entry.pluginId,
|
||||
provider: entry.provider,
|
||||
}));
|
||||
}
|
||||
return speechProviderContractRegistryCache;
|
||||
}
|
||||
|
||||
function loadMediaUnderstandingProviderContractRegistry(): MediaUnderstandingProviderContractEntry[] {
|
||||
if (!mediaUnderstandingProviderContractRegistryCache) {
|
||||
mediaUnderstandingProviderContractRegistryCache = BUNDLED_PLUGIN_CONTRACT_SNAPSHOTS.flatMap(
|
||||
(entry) =>
|
||||
entry.mediaUnderstandingProviderIds.map((providerId) => ({
|
||||
pluginId: entry.pluginId,
|
||||
provider: createContractMediaUnderstandingProvider(providerId),
|
||||
})),
|
||||
const registry = loadBundledCapabilityRuntimeRegistry({
|
||||
pluginIds: BUNDLED_MEDIA_UNDERSTANDING_PLUGIN_IDS,
|
||||
pluginSdkResolution: "dist",
|
||||
});
|
||||
mediaUnderstandingProviderContractRegistryCache = registry.mediaUnderstandingProviders.map(
|
||||
(entry) => ({
|
||||
pluginId: entry.pluginId,
|
||||
provider: entry.provider,
|
||||
}),
|
||||
);
|
||||
}
|
||||
return mediaUnderstandingProviderContractRegistryCache;
|
||||
|
||||
@@ -1,3 +1,5 @@
|
||||
import { BUNDLED_PLUGIN_METADATA } from "./bundled-plugin-metadata.js";
|
||||
|
||||
function assertUniqueValues<T extends string>(values: readonly T[], label: string): readonly T[] {
|
||||
const seen = new Set<string>();
|
||||
const duplicates = new Set<string>();
|
||||
@@ -19,14 +21,11 @@ export function getPublicArtifactBasename(relativePath: string): string {
|
||||
}
|
||||
|
||||
export const BUNDLED_RUNTIME_SIDECAR_PATHS = assertUniqueValues(
|
||||
[
|
||||
"dist/extensions/whatsapp/light-runtime-api.js",
|
||||
"dist/extensions/whatsapp/runtime-api.js",
|
||||
"dist/extensions/matrix/helper-api.js",
|
||||
"dist/extensions/matrix/runtime-api.js",
|
||||
"dist/extensions/matrix/thread-bindings-runtime.js",
|
||||
"dist/extensions/msteams/runtime-api.js",
|
||||
] as const,
|
||||
BUNDLED_PLUGIN_METADATA.flatMap((entry) =>
|
||||
(entry.runtimeSidecarArtifacts ?? []).map(
|
||||
(artifact) => `dist/extensions/${entry.dirName}/${artifact}`,
|
||||
),
|
||||
).toSorted((left, right) => left.localeCompare(right)),
|
||||
"bundled runtime sidecar path",
|
||||
);
|
||||
|
||||
|
||||
@@ -1,4 +1,5 @@
|
||||
import { createJiti } from "jiti";
|
||||
import type { MatrixRuntimeBoundaryModule } from "./runtime-matrix-surface.js";
|
||||
import {
|
||||
loadPluginBoundaryModuleWithJiti,
|
||||
resolvePluginRuntimeModulePath,
|
||||
@@ -7,15 +8,13 @@ import {
|
||||
|
||||
const MATRIX_PLUGIN_ID = "matrix";
|
||||
|
||||
type MatrixModule = typeof import("../../../extensions/matrix/runtime-api.js");
|
||||
|
||||
type MatrixPluginRecord = {
|
||||
rootDir?: string;
|
||||
source: string;
|
||||
};
|
||||
|
||||
let cachedModulePath: string | null = null;
|
||||
let cachedModule: MatrixModule | null = null;
|
||||
let cachedModule: MatrixRuntimeBoundaryModule | null = null;
|
||||
|
||||
const jitiLoaders = new Map<boolean, ReturnType<typeof createJiti>>();
|
||||
|
||||
@@ -27,7 +26,7 @@ function resolveMatrixRuntimeModulePath(record: MatrixPluginRecord): string | nu
|
||||
return resolvePluginRuntimeModulePath(record, "runtime-api");
|
||||
}
|
||||
|
||||
function loadMatrixModule(): MatrixModule | null {
|
||||
function loadMatrixModule(): MatrixRuntimeBoundaryModule | null {
|
||||
const record = resolveMatrixPluginRecord();
|
||||
if (!record) {
|
||||
return null;
|
||||
@@ -39,15 +38,18 @@ function loadMatrixModule(): MatrixModule | null {
|
||||
if (cachedModule && cachedModulePath === modulePath) {
|
||||
return cachedModule;
|
||||
}
|
||||
const loaded = loadPluginBoundaryModuleWithJiti<MatrixModule>(modulePath, jitiLoaders);
|
||||
const loaded = loadPluginBoundaryModuleWithJiti<MatrixRuntimeBoundaryModule>(
|
||||
modulePath,
|
||||
jitiLoaders,
|
||||
);
|
||||
cachedModulePath = modulePath;
|
||||
cachedModule = loaded;
|
||||
return loaded;
|
||||
}
|
||||
|
||||
export function setMatrixThreadBindingIdleTimeoutBySessionKey(
|
||||
...args: Parameters<MatrixModule["setMatrixThreadBindingIdleTimeoutBySessionKey"]>
|
||||
): ReturnType<MatrixModule["setMatrixThreadBindingIdleTimeoutBySessionKey"]> {
|
||||
...args: Parameters<MatrixRuntimeBoundaryModule["setMatrixThreadBindingIdleTimeoutBySessionKey"]>
|
||||
): ReturnType<MatrixRuntimeBoundaryModule["setMatrixThreadBindingIdleTimeoutBySessionKey"]> {
|
||||
const fn = loadMatrixModule()?.setMatrixThreadBindingIdleTimeoutBySessionKey;
|
||||
if (typeof fn !== "function") {
|
||||
return [];
|
||||
@@ -56,8 +58,8 @@ export function setMatrixThreadBindingIdleTimeoutBySessionKey(
|
||||
}
|
||||
|
||||
export function setMatrixThreadBindingMaxAgeBySessionKey(
|
||||
...args: Parameters<MatrixModule["setMatrixThreadBindingMaxAgeBySessionKey"]>
|
||||
): ReturnType<MatrixModule["setMatrixThreadBindingMaxAgeBySessionKey"]> {
|
||||
...args: Parameters<MatrixRuntimeBoundaryModule["setMatrixThreadBindingMaxAgeBySessionKey"]>
|
||||
): ReturnType<MatrixRuntimeBoundaryModule["setMatrixThreadBindingMaxAgeBySessionKey"]> {
|
||||
const fn = loadMatrixModule()?.setMatrixThreadBindingMaxAgeBySessionKey;
|
||||
if (typeof fn !== "function") {
|
||||
return [];
|
||||
|
||||
22
src/plugins/runtime/runtime-matrix-surface.ts
Normal file
22
src/plugins/runtime/runtime-matrix-surface.ts
Normal file
@@ -0,0 +1,22 @@
|
||||
import type { SessionBindingRecord } from "../../infra/outbound/session-binding-service.js";
|
||||
|
||||
export type MatrixThreadBindingIdleTimeoutParams = {
|
||||
accountId: string;
|
||||
targetSessionKey: string;
|
||||
idleTimeoutMs: number;
|
||||
};
|
||||
|
||||
export type MatrixThreadBindingMaxAgeParams = {
|
||||
accountId: string;
|
||||
targetSessionKey: string;
|
||||
maxAgeMs: number;
|
||||
};
|
||||
|
||||
export type MatrixRuntimeBoundaryModule = {
|
||||
setMatrixThreadBindingIdleTimeoutBySessionKey: (
|
||||
params: MatrixThreadBindingIdleTimeoutParams,
|
||||
) => SessionBindingRecord[];
|
||||
setMatrixThreadBindingMaxAgeBySessionKey: (
|
||||
params: MatrixThreadBindingMaxAgeParams,
|
||||
) => SessionBindingRecord[];
|
||||
};
|
||||
@@ -11,12 +11,13 @@ import {
|
||||
resolvePluginRuntimeModulePath,
|
||||
resolvePluginRuntimeRecord,
|
||||
} from "./runtime-plugin-boundary.js";
|
||||
import type {
|
||||
WhatsAppHeavyRuntimeModule,
|
||||
WhatsAppLightRuntimeModule,
|
||||
} from "./runtime-whatsapp-surface.js";
|
||||
|
||||
const WHATSAPP_PLUGIN_ID = "whatsapp";
|
||||
|
||||
type WhatsAppLightModule = typeof import("../../../extensions/whatsapp/light-runtime-api.js");
|
||||
type WhatsAppHeavyModule = typeof import("../../../extensions/whatsapp/runtime-api.js");
|
||||
|
||||
type WhatsAppPluginRecord = {
|
||||
origin: string;
|
||||
rootDir?: string;
|
||||
@@ -24,9 +25,9 @@ type WhatsAppPluginRecord = {
|
||||
};
|
||||
|
||||
let cachedHeavyModulePath: string | null = null;
|
||||
let cachedHeavyModule: WhatsAppHeavyModule | null = null;
|
||||
let cachedHeavyModule: WhatsAppHeavyRuntimeModule | null = null;
|
||||
let cachedLightModulePath: string | null = null;
|
||||
let cachedLightModule: WhatsAppLightModule | null = null;
|
||||
let cachedLightModule: WhatsAppLightRuntimeModule | null = null;
|
||||
|
||||
const jitiLoaders = new Map<boolean, ReturnType<typeof createJiti>>();
|
||||
|
||||
@@ -55,12 +56,12 @@ function resolveWhatsAppRuntimeModulePath(
|
||||
return modulePath;
|
||||
}
|
||||
|
||||
function loadCurrentHeavyModuleSync(): WhatsAppHeavyModule {
|
||||
function loadCurrentHeavyModuleSync(): WhatsAppHeavyRuntimeModule {
|
||||
const modulePath = resolveWhatsAppRuntimeModulePath(resolveWhatsAppPluginRecord(), "runtime-api");
|
||||
return loadPluginBoundaryModuleWithJiti<WhatsAppHeavyModule>(modulePath, jitiLoaders);
|
||||
return loadPluginBoundaryModuleWithJiti<WhatsAppHeavyRuntimeModule>(modulePath, jitiLoaders);
|
||||
}
|
||||
|
||||
function loadWhatsAppLightModule(): WhatsAppLightModule {
|
||||
function loadWhatsAppLightModule(): WhatsAppLightRuntimeModule {
|
||||
const modulePath = resolveWhatsAppRuntimeModulePath(
|
||||
resolveWhatsAppPluginRecord(),
|
||||
"light-runtime-api",
|
||||
@@ -68,143 +69,149 @@ function loadWhatsAppLightModule(): WhatsAppLightModule {
|
||||
if (cachedLightModule && cachedLightModulePath === modulePath) {
|
||||
return cachedLightModule;
|
||||
}
|
||||
const loaded = loadPluginBoundaryModuleWithJiti<WhatsAppLightModule>(modulePath, jitiLoaders);
|
||||
const loaded = loadPluginBoundaryModuleWithJiti<WhatsAppLightRuntimeModule>(
|
||||
modulePath,
|
||||
jitiLoaders,
|
||||
);
|
||||
cachedLightModulePath = modulePath;
|
||||
cachedLightModule = loaded;
|
||||
return loaded;
|
||||
}
|
||||
|
||||
async function loadWhatsAppHeavyModule(): Promise<WhatsAppHeavyModule> {
|
||||
async function loadWhatsAppHeavyModule(): Promise<WhatsAppHeavyRuntimeModule> {
|
||||
const record = resolveWhatsAppPluginRecord();
|
||||
const modulePath = resolveWhatsAppRuntimeModulePath(record, "runtime-api");
|
||||
if (cachedHeavyModule && cachedHeavyModulePath === modulePath) {
|
||||
return cachedHeavyModule;
|
||||
}
|
||||
const loaded = loadPluginBoundaryModuleWithJiti<WhatsAppHeavyModule>(modulePath, jitiLoaders);
|
||||
const loaded = loadPluginBoundaryModuleWithJiti<WhatsAppHeavyRuntimeModule>(
|
||||
modulePath,
|
||||
jitiLoaders,
|
||||
);
|
||||
cachedHeavyModulePath = modulePath;
|
||||
cachedHeavyModule = loaded;
|
||||
return loaded;
|
||||
}
|
||||
|
||||
function getLightExport<K extends keyof WhatsAppLightModule>(
|
||||
function getLightExport<K extends keyof WhatsAppLightRuntimeModule>(
|
||||
exportName: K,
|
||||
): NonNullable<WhatsAppLightModule[K]> {
|
||||
): NonNullable<WhatsAppLightRuntimeModule[K]> {
|
||||
const loaded = loadWhatsAppLightModule();
|
||||
const value = loaded[exportName];
|
||||
if (value == null) {
|
||||
throw new Error(`WhatsApp plugin runtime is missing export '${String(exportName)}'`);
|
||||
}
|
||||
return value as NonNullable<WhatsAppLightModule[K]>;
|
||||
return value as NonNullable<WhatsAppLightRuntimeModule[K]>;
|
||||
}
|
||||
|
||||
async function getHeavyExport<K extends keyof WhatsAppHeavyModule>(
|
||||
async function getHeavyExport<K extends keyof WhatsAppHeavyRuntimeModule>(
|
||||
exportName: K,
|
||||
): Promise<NonNullable<WhatsAppHeavyModule[K]>> {
|
||||
): Promise<NonNullable<WhatsAppHeavyRuntimeModule[K]>> {
|
||||
const loaded = await loadWhatsAppHeavyModule();
|
||||
const value = loaded[exportName];
|
||||
if (value == null) {
|
||||
throw new Error(`WhatsApp plugin runtime is missing export '${String(exportName)}'`);
|
||||
}
|
||||
return value as NonNullable<WhatsAppHeavyModule[K]>;
|
||||
return value as NonNullable<WhatsAppHeavyRuntimeModule[K]>;
|
||||
}
|
||||
|
||||
export function getActiveWebListener(
|
||||
...args: Parameters<WhatsAppLightModule["getActiveWebListener"]>
|
||||
): ReturnType<WhatsAppLightModule["getActiveWebListener"]> {
|
||||
...args: Parameters<WhatsAppLightRuntimeModule["getActiveWebListener"]>
|
||||
): ReturnType<WhatsAppLightRuntimeModule["getActiveWebListener"]> {
|
||||
return getLightExport("getActiveWebListener")(...args);
|
||||
}
|
||||
|
||||
export function getWebAuthAgeMs(
|
||||
...args: Parameters<WhatsAppLightModule["getWebAuthAgeMs"]>
|
||||
): ReturnType<WhatsAppLightModule["getWebAuthAgeMs"]> {
|
||||
...args: Parameters<WhatsAppLightRuntimeModule["getWebAuthAgeMs"]>
|
||||
): ReturnType<WhatsAppLightRuntimeModule["getWebAuthAgeMs"]> {
|
||||
return getLightExport("getWebAuthAgeMs")(...args);
|
||||
}
|
||||
|
||||
export function logWebSelfId(
|
||||
...args: Parameters<WhatsAppLightModule["logWebSelfId"]>
|
||||
): ReturnType<WhatsAppLightModule["logWebSelfId"]> {
|
||||
...args: Parameters<WhatsAppLightRuntimeModule["logWebSelfId"]>
|
||||
): ReturnType<WhatsAppLightRuntimeModule["logWebSelfId"]> {
|
||||
return getLightExport("logWebSelfId")(...args);
|
||||
}
|
||||
|
||||
export function loginWeb(
|
||||
...args: Parameters<WhatsAppHeavyModule["loginWeb"]>
|
||||
): ReturnType<WhatsAppHeavyModule["loginWeb"]> {
|
||||
...args: Parameters<WhatsAppHeavyRuntimeModule["loginWeb"]>
|
||||
): ReturnType<WhatsAppHeavyRuntimeModule["loginWeb"]> {
|
||||
return loadWhatsAppHeavyModule().then((loaded) => loaded.loginWeb(...args));
|
||||
}
|
||||
|
||||
export function logoutWeb(
|
||||
...args: Parameters<WhatsAppLightModule["logoutWeb"]>
|
||||
): ReturnType<WhatsAppLightModule["logoutWeb"]> {
|
||||
...args: Parameters<WhatsAppLightRuntimeModule["logoutWeb"]>
|
||||
): ReturnType<WhatsAppLightRuntimeModule["logoutWeb"]> {
|
||||
return getLightExport("logoutWeb")(...args);
|
||||
}
|
||||
|
||||
export function readWebSelfId(
|
||||
...args: Parameters<WhatsAppLightModule["readWebSelfId"]>
|
||||
): ReturnType<WhatsAppLightModule["readWebSelfId"]> {
|
||||
...args: Parameters<WhatsAppLightRuntimeModule["readWebSelfId"]>
|
||||
): ReturnType<WhatsAppLightRuntimeModule["readWebSelfId"]> {
|
||||
return getLightExport("readWebSelfId")(...args);
|
||||
}
|
||||
|
||||
export function webAuthExists(
|
||||
...args: Parameters<WhatsAppLightModule["webAuthExists"]>
|
||||
): ReturnType<WhatsAppLightModule["webAuthExists"]> {
|
||||
...args: Parameters<WhatsAppLightRuntimeModule["webAuthExists"]>
|
||||
): ReturnType<WhatsAppLightRuntimeModule["webAuthExists"]> {
|
||||
return getLightExport("webAuthExists")(...args);
|
||||
}
|
||||
|
||||
export function sendMessageWhatsApp(
|
||||
...args: Parameters<WhatsAppHeavyModule["sendMessageWhatsApp"]>
|
||||
): ReturnType<WhatsAppHeavyModule["sendMessageWhatsApp"]> {
|
||||
...args: Parameters<WhatsAppHeavyRuntimeModule["sendMessageWhatsApp"]>
|
||||
): ReturnType<WhatsAppHeavyRuntimeModule["sendMessageWhatsApp"]> {
|
||||
return loadWhatsAppHeavyModule().then((loaded) => loaded.sendMessageWhatsApp(...args));
|
||||
}
|
||||
|
||||
export function sendPollWhatsApp(
|
||||
...args: Parameters<WhatsAppHeavyModule["sendPollWhatsApp"]>
|
||||
): ReturnType<WhatsAppHeavyModule["sendPollWhatsApp"]> {
|
||||
...args: Parameters<WhatsAppHeavyRuntimeModule["sendPollWhatsApp"]>
|
||||
): ReturnType<WhatsAppHeavyRuntimeModule["sendPollWhatsApp"]> {
|
||||
return loadWhatsAppHeavyModule().then((loaded) => loaded.sendPollWhatsApp(...args));
|
||||
}
|
||||
|
||||
export function sendReactionWhatsApp(
|
||||
...args: Parameters<WhatsAppHeavyModule["sendReactionWhatsApp"]>
|
||||
): ReturnType<WhatsAppHeavyModule["sendReactionWhatsApp"]> {
|
||||
...args: Parameters<WhatsAppHeavyRuntimeModule["sendReactionWhatsApp"]>
|
||||
): ReturnType<WhatsAppHeavyRuntimeModule["sendReactionWhatsApp"]> {
|
||||
return loadWhatsAppHeavyModule().then((loaded) => loaded.sendReactionWhatsApp(...args));
|
||||
}
|
||||
|
||||
export function createRuntimeWhatsAppLoginTool(
|
||||
...args: Parameters<WhatsAppLightModule["createWhatsAppLoginTool"]>
|
||||
): ReturnType<WhatsAppLightModule["createWhatsAppLoginTool"]> {
|
||||
...args: Parameters<WhatsAppLightRuntimeModule["createWhatsAppLoginTool"]>
|
||||
): ReturnType<WhatsAppLightRuntimeModule["createWhatsAppLoginTool"]> {
|
||||
return getLightExport("createWhatsAppLoginTool")(...args);
|
||||
}
|
||||
|
||||
export function createWaSocket(
|
||||
...args: Parameters<WhatsAppHeavyModule["createWaSocket"]>
|
||||
): ReturnType<WhatsAppHeavyModule["createWaSocket"]> {
|
||||
...args: Parameters<WhatsAppHeavyRuntimeModule["createWaSocket"]>
|
||||
): ReturnType<WhatsAppHeavyRuntimeModule["createWaSocket"]> {
|
||||
return loadWhatsAppHeavyModule().then((loaded) => loaded.createWaSocket(...args));
|
||||
}
|
||||
|
||||
export function formatError(
|
||||
...args: Parameters<WhatsAppLightModule["formatError"]>
|
||||
): ReturnType<WhatsAppLightModule["formatError"]> {
|
||||
...args: Parameters<WhatsAppLightRuntimeModule["formatError"]>
|
||||
): ReturnType<WhatsAppLightRuntimeModule["formatError"]> {
|
||||
return getLightExport("formatError")(...args);
|
||||
}
|
||||
|
||||
export function getStatusCode(
|
||||
...args: Parameters<WhatsAppLightModule["getStatusCode"]>
|
||||
): ReturnType<WhatsAppLightModule["getStatusCode"]> {
|
||||
...args: Parameters<WhatsAppLightRuntimeModule["getStatusCode"]>
|
||||
): ReturnType<WhatsAppLightRuntimeModule["getStatusCode"]> {
|
||||
return getLightExport("getStatusCode")(...args);
|
||||
}
|
||||
|
||||
export function pickWebChannel(
|
||||
...args: Parameters<WhatsAppLightModule["pickWebChannel"]>
|
||||
): ReturnType<WhatsAppLightModule["pickWebChannel"]> {
|
||||
...args: Parameters<WhatsAppLightRuntimeModule["pickWebChannel"]>
|
||||
): ReturnType<WhatsAppLightRuntimeModule["pickWebChannel"]> {
|
||||
return getLightExport("pickWebChannel")(...args);
|
||||
}
|
||||
|
||||
export function resolveWaWebAuthDir(): WhatsAppLightModule["WA_WEB_AUTH_DIR"] {
|
||||
export function resolveWaWebAuthDir(): WhatsAppLightRuntimeModule["WA_WEB_AUTH_DIR"] {
|
||||
return getLightExport("WA_WEB_AUTH_DIR");
|
||||
}
|
||||
|
||||
export async function handleWhatsAppAction(
|
||||
...args: Parameters<WhatsAppHeavyModule["handleWhatsAppAction"]>
|
||||
): ReturnType<WhatsAppHeavyModule["handleWhatsAppAction"]> {
|
||||
...args: Parameters<WhatsAppHeavyRuntimeModule["handleWhatsAppAction"]>
|
||||
): ReturnType<WhatsAppHeavyRuntimeModule["handleWhatsAppAction"]> {
|
||||
return (await getHeavyExport("handleWhatsAppAction"))(...args);
|
||||
}
|
||||
|
||||
@@ -221,14 +228,14 @@ export async function loadWebMediaRaw(
|
||||
}
|
||||
|
||||
export function monitorWebChannel(
|
||||
...args: Parameters<WhatsAppHeavyModule["monitorWebChannel"]>
|
||||
): ReturnType<WhatsAppHeavyModule["monitorWebChannel"]> {
|
||||
...args: Parameters<WhatsAppHeavyRuntimeModule["monitorWebChannel"]>
|
||||
): ReturnType<WhatsAppHeavyRuntimeModule["monitorWebChannel"]> {
|
||||
return loadWhatsAppHeavyModule().then((loaded) => loaded.monitorWebChannel(...args));
|
||||
}
|
||||
|
||||
export async function monitorWebInbox(
|
||||
...args: Parameters<WhatsAppHeavyModule["monitorWebInbox"]>
|
||||
): ReturnType<WhatsAppHeavyModule["monitorWebInbox"]> {
|
||||
...args: Parameters<WhatsAppHeavyRuntimeModule["monitorWebInbox"]>
|
||||
): ReturnType<WhatsAppHeavyRuntimeModule["monitorWebInbox"]> {
|
||||
return (await getHeavyExport("monitorWebInbox"))(...args);
|
||||
}
|
||||
|
||||
@@ -239,34 +246,34 @@ export async function optimizeImageToJpeg(
|
||||
}
|
||||
|
||||
export async function runWebHeartbeatOnce(
|
||||
...args: Parameters<WhatsAppHeavyModule["runWebHeartbeatOnce"]>
|
||||
): ReturnType<WhatsAppHeavyModule["runWebHeartbeatOnce"]> {
|
||||
...args: Parameters<WhatsAppHeavyRuntimeModule["runWebHeartbeatOnce"]>
|
||||
): ReturnType<WhatsAppHeavyRuntimeModule["runWebHeartbeatOnce"]> {
|
||||
return (await getHeavyExport("runWebHeartbeatOnce"))(...args);
|
||||
}
|
||||
|
||||
export async function startWebLoginWithQr(
|
||||
...args: Parameters<WhatsAppHeavyModule["startWebLoginWithQr"]>
|
||||
): ReturnType<WhatsAppHeavyModule["startWebLoginWithQr"]> {
|
||||
...args: Parameters<WhatsAppHeavyRuntimeModule["startWebLoginWithQr"]>
|
||||
): ReturnType<WhatsAppHeavyRuntimeModule["startWebLoginWithQr"]> {
|
||||
return (await getHeavyExport("startWebLoginWithQr"))(...args);
|
||||
}
|
||||
|
||||
export async function waitForWaConnection(
|
||||
...args: Parameters<WhatsAppHeavyModule["waitForWaConnection"]>
|
||||
): ReturnType<WhatsAppHeavyModule["waitForWaConnection"]> {
|
||||
...args: Parameters<WhatsAppHeavyRuntimeModule["waitForWaConnection"]>
|
||||
): ReturnType<WhatsAppHeavyRuntimeModule["waitForWaConnection"]> {
|
||||
return (await getHeavyExport("waitForWaConnection"))(...args);
|
||||
}
|
||||
|
||||
export async function waitForWebLogin(
|
||||
...args: Parameters<WhatsAppHeavyModule["waitForWebLogin"]>
|
||||
): ReturnType<WhatsAppHeavyModule["waitForWebLogin"]> {
|
||||
...args: Parameters<WhatsAppHeavyRuntimeModule["waitForWebLogin"]>
|
||||
): ReturnType<WhatsAppHeavyRuntimeModule["waitForWebLogin"]> {
|
||||
return (await getHeavyExport("waitForWebLogin"))(...args);
|
||||
}
|
||||
|
||||
export const extractMediaPlaceholder = (
|
||||
...args: Parameters<WhatsAppHeavyModule["extractMediaPlaceholder"]>
|
||||
...args: Parameters<WhatsAppHeavyRuntimeModule["extractMediaPlaceholder"]>
|
||||
) => loadCurrentHeavyModuleSync().extractMediaPlaceholder(...args);
|
||||
|
||||
export const extractText = (...args: Parameters<WhatsAppHeavyModule["extractText"]>) =>
|
||||
export const extractText = (...args: Parameters<WhatsAppHeavyRuntimeModule["extractText"]>) =>
|
||||
loadCurrentHeavyModuleSync().extractText(...args);
|
||||
|
||||
export function getDefaultLocalRoots(
|
||||
|
||||
249
src/plugins/runtime/runtime-whatsapp-surface.ts
Normal file
249
src/plugins/runtime/runtime-whatsapp-surface.ts
Normal file
@@ -0,0 +1,249 @@
|
||||
import type { AgentToolResult } from "@mariozechner/pi-agent-core";
|
||||
import type { AnyMessageContent, makeWASocket } from "@whiskeysockets/baileys";
|
||||
import type { NormalizedLocation } from "../../channels/location.js";
|
||||
import type { ChannelAgentTool } from "../../channels/plugins/types.core.js";
|
||||
import type { OpenClawConfig } from "../../config/config.js";
|
||||
import type { PollInput } from "../../polls.js";
|
||||
import type { RuntimeEnv } from "../../runtime.js";
|
||||
import type { WebChannel } from "../../utils.js";
|
||||
|
||||
export type ActiveWebSendOptions = {
|
||||
gifPlayback?: boolean;
|
||||
accountId?: string;
|
||||
fileName?: string;
|
||||
};
|
||||
|
||||
export type ActiveWebListener = {
|
||||
sendMessage: (
|
||||
to: string,
|
||||
text: string,
|
||||
mediaBuffer?: Buffer,
|
||||
mediaType?: string,
|
||||
options?: ActiveWebSendOptions,
|
||||
) => Promise<{ messageId: string }>;
|
||||
sendPoll: (to: string, poll: PollInput) => Promise<{ messageId: string }>;
|
||||
sendReaction: (
|
||||
chatJid: string,
|
||||
messageId: string,
|
||||
emoji: string,
|
||||
fromMe: boolean,
|
||||
participant?: string,
|
||||
) => Promise<void>;
|
||||
sendComposingTo: (to: string) => Promise<void>;
|
||||
close?: () => Promise<void>;
|
||||
};
|
||||
|
||||
export type WebListenerCloseReason = {
|
||||
status?: number;
|
||||
isLoggedOut: boolean;
|
||||
error?: unknown;
|
||||
};
|
||||
|
||||
export type WebInboundMessage = {
|
||||
id?: string;
|
||||
from: string;
|
||||
conversationId: string;
|
||||
to: string;
|
||||
accountId: string;
|
||||
body: string;
|
||||
pushName?: string;
|
||||
timestamp?: number;
|
||||
chatType: "direct" | "group";
|
||||
chatId: string;
|
||||
sender?: unknown;
|
||||
senderJid?: string;
|
||||
senderE164?: string;
|
||||
senderName?: string;
|
||||
replyTo?: unknown;
|
||||
replyToId?: string;
|
||||
replyToBody?: string;
|
||||
replyToSender?: string;
|
||||
replyToSenderJid?: string;
|
||||
replyToSenderE164?: string;
|
||||
groupSubject?: string;
|
||||
groupParticipants?: string[];
|
||||
mentions?: string[];
|
||||
mentionedJids?: string[];
|
||||
self?: unknown;
|
||||
selfJid?: string | null;
|
||||
selfLid?: string | null;
|
||||
selfE164?: string | null;
|
||||
fromMe?: boolean;
|
||||
location?: NormalizedLocation;
|
||||
sendComposing: () => Promise<void>;
|
||||
reply: (text: string) => Promise<void>;
|
||||
sendMedia: (payload: AnyMessageContent) => Promise<void>;
|
||||
mediaPath?: string;
|
||||
mediaType?: string;
|
||||
mediaFileName?: string;
|
||||
mediaUrl?: string;
|
||||
wasMentioned?: boolean;
|
||||
};
|
||||
|
||||
export type WebChannelHealthState =
|
||||
| "starting"
|
||||
| "healthy"
|
||||
| "stale"
|
||||
| "reconnecting"
|
||||
| "conflict"
|
||||
| "logged-out"
|
||||
| "stopped";
|
||||
|
||||
export type WebChannelStatus = {
|
||||
running: boolean;
|
||||
connected: boolean;
|
||||
reconnectAttempts: number;
|
||||
lastConnectedAt?: number | null;
|
||||
lastDisconnect?: {
|
||||
at: number;
|
||||
status?: number;
|
||||
error?: string;
|
||||
loggedOut?: boolean;
|
||||
} | null;
|
||||
lastInboundAt?: number | null;
|
||||
lastMessageAt?: number | null;
|
||||
lastEventAt?: number | null;
|
||||
lastError?: string | null;
|
||||
healthState?: WebChannelHealthState;
|
||||
};
|
||||
|
||||
export type WebMonitorTuning = {
|
||||
reconnect?: Partial<{
|
||||
enabled: boolean;
|
||||
maxAttempts: number;
|
||||
baseDelayMs: number;
|
||||
maxDelayMs: number;
|
||||
}>;
|
||||
heartbeatSeconds?: number;
|
||||
messageTimeoutMs?: number;
|
||||
watchdogCheckMs?: number;
|
||||
sleep?: (ms: number, signal?: AbortSignal) => Promise<void>;
|
||||
statusSink?: (status: WebChannelStatus) => void;
|
||||
accountId?: string;
|
||||
debounceMs?: number;
|
||||
};
|
||||
|
||||
export type MonitorWebInboxFactory = (options: {
|
||||
verbose: boolean;
|
||||
accountId: string;
|
||||
authDir: string;
|
||||
onMessage: (msg: WebInboundMessage) => Promise<void>;
|
||||
mediaMaxMb?: number;
|
||||
sendReadReceipts?: boolean;
|
||||
debounceMs?: number;
|
||||
shouldDebounce?: (msg: WebInboundMessage) => boolean;
|
||||
}) => Promise<{
|
||||
closeReason: Promise<WebListenerCloseReason>;
|
||||
stop: () => Promise<void>;
|
||||
}>;
|
||||
|
||||
export type ReplyResolver = (...args: unknown[]) => Promise<unknown>;
|
||||
|
||||
export type WhatsAppWaSocket = ReturnType<typeof makeWASocket>;
|
||||
|
||||
export type WhatsAppLightRuntimeModule = {
|
||||
getActiveWebListener: (accountId?: string | null) => ActiveWebListener | null;
|
||||
getWebAuthAgeMs: (authDir?: string) => number | null;
|
||||
logWebSelfId: (authDir?: string, runtime?: RuntimeEnv, includeChannelPrefix?: boolean) => void;
|
||||
logoutWeb: (params: {
|
||||
authDir?: string;
|
||||
isLegacyAuthDir?: boolean;
|
||||
runtime?: RuntimeEnv;
|
||||
}) => Promise<boolean>;
|
||||
readWebSelfId: (authDir?: string) => {
|
||||
e164: string | null;
|
||||
jid: string | null;
|
||||
lid: string | null;
|
||||
};
|
||||
webAuthExists: (authDir?: string) => Promise<boolean>;
|
||||
createWhatsAppLoginTool: () => ChannelAgentTool;
|
||||
formatError: (err: unknown) => string;
|
||||
getStatusCode: (err: unknown) => number | undefined;
|
||||
pickWebChannel: (pref: WebChannel | "auto", authDir?: string) => Promise<WebChannel>;
|
||||
WA_WEB_AUTH_DIR: string;
|
||||
};
|
||||
|
||||
export type WhatsAppHeavyRuntimeModule = {
|
||||
loginWeb: (
|
||||
verbose: boolean,
|
||||
waitForConnection?: (sock: WhatsAppWaSocket) => Promise<void>,
|
||||
runtime?: RuntimeEnv,
|
||||
accountId?: string,
|
||||
) => Promise<void>;
|
||||
sendMessageWhatsApp: (
|
||||
to: string,
|
||||
body: string,
|
||||
options: {
|
||||
verbose: boolean;
|
||||
cfg?: OpenClawConfig;
|
||||
mediaUrl?: string;
|
||||
mediaLocalRoots?: readonly string[];
|
||||
gifPlayback?: boolean;
|
||||
accountId?: string;
|
||||
},
|
||||
) => Promise<{ messageId: string; toJid: string }>;
|
||||
sendPollWhatsApp: (
|
||||
to: string,
|
||||
poll: PollInput,
|
||||
options: { verbose: boolean; accountId?: string; cfg?: OpenClawConfig },
|
||||
) => Promise<{ messageId: string; toJid: string }>;
|
||||
sendReactionWhatsApp: (
|
||||
chatJid: string,
|
||||
messageId: string,
|
||||
emoji: string,
|
||||
options: {
|
||||
verbose: boolean;
|
||||
fromMe?: boolean;
|
||||
participant?: string;
|
||||
accountId?: string;
|
||||
},
|
||||
) => Promise<void>;
|
||||
createWaSocket: (
|
||||
printQr: boolean,
|
||||
verbose: boolean,
|
||||
opts?: { authDir?: string; onQr?: (qr: string) => void },
|
||||
) => Promise<WhatsAppWaSocket>;
|
||||
handleWhatsAppAction: (
|
||||
params: Record<string, unknown>,
|
||||
cfg: OpenClawConfig,
|
||||
) => Promise<AgentToolResult<unknown>>;
|
||||
monitorWebChannel: (
|
||||
verbose: boolean,
|
||||
listenerFactory?: MonitorWebInboxFactory,
|
||||
keepAlive?: boolean,
|
||||
replyResolver?: ReplyResolver,
|
||||
runtime?: RuntimeEnv,
|
||||
abortSignal?: AbortSignal,
|
||||
tuning?: WebMonitorTuning,
|
||||
) => Promise<void>;
|
||||
monitorWebInbox: MonitorWebInboxFactory;
|
||||
runWebHeartbeatOnce: (opts: {
|
||||
cfg?: OpenClawConfig;
|
||||
to: string;
|
||||
verbose?: boolean;
|
||||
replyResolver?: ReplyResolver;
|
||||
sender?: WhatsAppHeavyRuntimeModule["sendMessageWhatsApp"];
|
||||
sessionId?: string;
|
||||
overrideBody?: string;
|
||||
dryRun?: boolean;
|
||||
}) => Promise<void>;
|
||||
startWebLoginWithQr: (opts?: {
|
||||
verbose?: boolean;
|
||||
timeoutMs?: number;
|
||||
force?: boolean;
|
||||
accountId?: string;
|
||||
runtime?: RuntimeEnv;
|
||||
}) => Promise<{ qrDataUrl?: string; message: string }>;
|
||||
waitForWaConnection: (sock: WhatsAppWaSocket) => Promise<void>;
|
||||
waitForWebLogin: (opts?: {
|
||||
timeoutMs?: number;
|
||||
runtime?: RuntimeEnv;
|
||||
accountId?: string;
|
||||
}) => Promise<{ connected: boolean; message: string }>;
|
||||
extractMediaPlaceholder: (
|
||||
message: unknown,
|
||||
mediaDir: string,
|
||||
verbose?: boolean,
|
||||
) => Promise<string | null>;
|
||||
extractText: (message: unknown) => string;
|
||||
};
|
||||
@@ -1,6 +1,6 @@
|
||||
import { normalizeIMessageHandle } from "../../extensions/imessage/api.js";
|
||||
import { imessageOutbound } from "../../test/channel-outbounds.js";
|
||||
import type { ChannelOutboundAdapter, ChannelPlugin } from "../channels/plugins/types.js";
|
||||
import { normalizeIMessageHandle } from "../plugin-sdk/imessage-targets.js";
|
||||
import { collectStatusIssuesFromLastError } from "../plugin-sdk/status-helpers.js";
|
||||
|
||||
export const createIMessageTestPlugin = (params?: {
|
||||
|
||||
889
src/tts/tts.ts
889
src/tts/tts.ts
@@ -1,857 +1,36 @@
|
||||
import { randomBytes } from "node:crypto";
|
||||
import {
|
||||
existsSync,
|
||||
mkdirSync,
|
||||
readFileSync,
|
||||
writeFileSync,
|
||||
mkdtempSync,
|
||||
renameSync,
|
||||
unlinkSync,
|
||||
} from "node:fs";
|
||||
import path from "node:path";
|
||||
import { resolveSendableOutboundReplyParts } from "openclaw/plugin-sdk/reply-payload";
|
||||
import type { ReplyPayload } from "../auto-reply/types.js";
|
||||
import { normalizeChannelId } from "../channels/plugins/index.js";
|
||||
import type { ChannelId } from "../channels/plugins/types.js";
|
||||
import type { OpenClawConfig } from "../config/config.js";
|
||||
import type {
|
||||
TtsConfig,
|
||||
TtsAutoMode,
|
||||
TtsMode,
|
||||
TtsProvider,
|
||||
TtsModelOverrideConfig,
|
||||
} from "../config/types.tts.js";
|
||||
import { logVerbose } from "../globals.js";
|
||||
import { resolvePreferredOpenClawTmpDir } from "../infra/tmp-openclaw-dir.js";
|
||||
import { stripMarkdown } from "../shared/text/strip-markdown.js";
|
||||
import { CONFIG_DIR, resolveUserPath } from "../utils.js";
|
||||
import { parseTtsDirectives } from "./directives.js";
|
||||
import {
|
||||
canonicalizeSpeechProviderId,
|
||||
getSpeechProvider,
|
||||
listSpeechProviders,
|
||||
} from "./provider-registry.js";
|
||||
import type {
|
||||
SpeechModelOverridePolicy,
|
||||
SpeechProviderConfig,
|
||||
SpeechVoiceOption,
|
||||
import * as speechRuntime from "../plugin-sdk/speech-runtime.js";
|
||||
|
||||
export const buildTtsSystemPromptHint = speechRuntime.buildTtsSystemPromptHint;
|
||||
export const getLastTtsAttempt = speechRuntime.getLastTtsAttempt;
|
||||
export const getResolvedSpeechProviderConfig = speechRuntime.getResolvedSpeechProviderConfig;
|
||||
export const getTtsMaxLength = speechRuntime.getTtsMaxLength;
|
||||
export const getTtsProvider = speechRuntime.getTtsProvider;
|
||||
export const isSummarizationEnabled = speechRuntime.isSummarizationEnabled;
|
||||
export const isTtsEnabled = speechRuntime.isTtsEnabled;
|
||||
export const isTtsProviderConfigured = speechRuntime.isTtsProviderConfigured;
|
||||
export const listSpeechVoices = speechRuntime.listSpeechVoices;
|
||||
export const maybeApplyTtsToPayload = speechRuntime.maybeApplyTtsToPayload;
|
||||
export const resolveTtsAutoMode = speechRuntime.resolveTtsAutoMode;
|
||||
export const resolveTtsConfig = speechRuntime.resolveTtsConfig;
|
||||
export const resolveTtsPrefsPath = speechRuntime.resolveTtsPrefsPath;
|
||||
export const resolveTtsProviderOrder = speechRuntime.resolveTtsProviderOrder;
|
||||
export const setLastTtsAttempt = speechRuntime.setLastTtsAttempt;
|
||||
export const setSummarizationEnabled = speechRuntime.setSummarizationEnabled;
|
||||
export const setTtsAutoMode = speechRuntime.setTtsAutoMode;
|
||||
export const setTtsEnabled = speechRuntime.setTtsEnabled;
|
||||
export const setTtsMaxLength = speechRuntime.setTtsMaxLength;
|
||||
export const setTtsProvider = speechRuntime.setTtsProvider;
|
||||
export const synthesizeSpeech = speechRuntime.synthesizeSpeech;
|
||||
export const textToSpeech = speechRuntime.textToSpeech;
|
||||
export const textToSpeechTelephony = speechRuntime.textToSpeechTelephony;
|
||||
export const _test = speechRuntime._test;
|
||||
|
||||
export type {
|
||||
ResolvedTtsConfig,
|
||||
ResolvedTtsModelOverrides,
|
||||
TtsDirectiveOverrides,
|
||||
TtsDirectiveParseResult,
|
||||
} from "./provider-types.js";
|
||||
import { normalizeTtsAutoMode } from "./tts-auto-mode.js";
|
||||
import { scheduleCleanup, summarizeText } from "./tts-core.js";
|
||||
|
||||
export type { TtsDirectiveOverrides, TtsDirectiveParseResult } from "./provider-types.js";
|
||||
|
||||
const DEFAULT_TIMEOUT_MS = 30_000;
|
||||
const DEFAULT_TTS_MAX_LENGTH = 1500;
|
||||
const DEFAULT_TTS_SUMMARIZE = true;
|
||||
const DEFAULT_MAX_TEXT_LENGTH = 4096;
|
||||
|
||||
export type ResolvedTtsConfig = {
|
||||
auto: TtsAutoMode;
|
||||
mode: TtsMode;
|
||||
provider: TtsProvider;
|
||||
providerSource: "config" | "default";
|
||||
summaryModel?: string;
|
||||
modelOverrides: ResolvedTtsModelOverrides;
|
||||
providerConfigs: Record<string, SpeechProviderConfig>;
|
||||
prefsPath?: string;
|
||||
maxTextLength: number;
|
||||
timeoutMs: number;
|
||||
};
|
||||
|
||||
type TtsUserPrefs = {
|
||||
tts?: {
|
||||
auto?: TtsAutoMode;
|
||||
enabled?: boolean;
|
||||
provider?: TtsProvider;
|
||||
maxLength?: number;
|
||||
summarize?: boolean;
|
||||
};
|
||||
};
|
||||
|
||||
export type ResolvedTtsModelOverrides = SpeechModelOverridePolicy;
|
||||
|
||||
export type TtsResult = {
|
||||
success: boolean;
|
||||
audioPath?: string;
|
||||
error?: string;
|
||||
latencyMs?: number;
|
||||
provider?: string;
|
||||
outputFormat?: string;
|
||||
voiceCompatible?: boolean;
|
||||
};
|
||||
|
||||
export type TtsSynthesisResult = {
|
||||
success: boolean;
|
||||
audioBuffer?: Buffer;
|
||||
error?: string;
|
||||
latencyMs?: number;
|
||||
provider?: string;
|
||||
outputFormat?: string;
|
||||
voiceCompatible?: boolean;
|
||||
fileExtension?: string;
|
||||
};
|
||||
|
||||
export type TtsTelephonyResult = {
|
||||
success: boolean;
|
||||
audioBuffer?: Buffer;
|
||||
error?: string;
|
||||
latencyMs?: number;
|
||||
provider?: string;
|
||||
outputFormat?: string;
|
||||
sampleRate?: number;
|
||||
};
|
||||
|
||||
type TtsStatusEntry = {
|
||||
timestamp: number;
|
||||
success: boolean;
|
||||
textLength: number;
|
||||
summarized: boolean;
|
||||
provider?: string;
|
||||
latencyMs?: number;
|
||||
error?: string;
|
||||
};
|
||||
|
||||
let lastTtsAttempt: TtsStatusEntry | undefined;
|
||||
|
||||
function resolveModelOverridePolicy(
|
||||
overrides: TtsModelOverrideConfig | undefined,
|
||||
): ResolvedTtsModelOverrides {
|
||||
const enabled = overrides?.enabled ?? true;
|
||||
if (!enabled) {
|
||||
return {
|
||||
enabled: false,
|
||||
allowText: false,
|
||||
allowProvider: false,
|
||||
allowVoice: false,
|
||||
allowModelId: false,
|
||||
allowVoiceSettings: false,
|
||||
allowNormalization: false,
|
||||
allowSeed: false,
|
||||
};
|
||||
}
|
||||
const allow = (value: boolean | undefined, defaultValue = true) => value ?? defaultValue;
|
||||
return {
|
||||
enabled: true,
|
||||
allowText: allow(overrides?.allowText),
|
||||
// Provider switching is higher-impact than voice/style tweaks; keep opt-in.
|
||||
allowProvider: allow(overrides?.allowProvider, false),
|
||||
allowVoice: allow(overrides?.allowVoice),
|
||||
allowModelId: allow(overrides?.allowModelId),
|
||||
allowVoiceSettings: allow(overrides?.allowVoiceSettings),
|
||||
allowNormalization: allow(overrides?.allowNormalization),
|
||||
allowSeed: allow(overrides?.allowSeed),
|
||||
};
|
||||
}
|
||||
|
||||
function sortSpeechProvidersForAutoSelection(cfg?: OpenClawConfig) {
|
||||
return listSpeechProviders(cfg).toSorted((left, right) => {
|
||||
const leftOrder = left.autoSelectOrder ?? Number.MAX_SAFE_INTEGER;
|
||||
const rightOrder = right.autoSelectOrder ?? Number.MAX_SAFE_INTEGER;
|
||||
if (leftOrder !== rightOrder) {
|
||||
return leftOrder - rightOrder;
|
||||
}
|
||||
return left.id.localeCompare(right.id);
|
||||
});
|
||||
}
|
||||
|
||||
function resolveRegistryDefaultSpeechProviderId(cfg?: OpenClawConfig): TtsProvider {
|
||||
return sortSpeechProvidersForAutoSelection(cfg)[0]?.id ?? "";
|
||||
}
|
||||
|
||||
function asProviderConfig(value: unknown): SpeechProviderConfig {
|
||||
return typeof value === "object" && value !== null && !Array.isArray(value)
|
||||
? (value as SpeechProviderConfig)
|
||||
: {};
|
||||
}
|
||||
|
||||
function asProviderConfigMap(value: unknown): Record<string, unknown> {
|
||||
return typeof value === "object" && value !== null && !Array.isArray(value)
|
||||
? (value as Record<string, unknown>)
|
||||
: {};
|
||||
}
|
||||
|
||||
function resolveSpeechProviderConfigs(
|
||||
raw: TtsConfig,
|
||||
cfg: OpenClawConfig,
|
||||
timeoutMs: number,
|
||||
): Record<string, SpeechProviderConfig> {
|
||||
const providerConfigs: Record<string, SpeechProviderConfig> = {};
|
||||
const rawProviders = asProviderConfigMap(raw.providers);
|
||||
for (const provider of listSpeechProviders(cfg)) {
|
||||
providerConfigs[provider.id] =
|
||||
provider.resolveConfig?.({
|
||||
cfg,
|
||||
rawConfig: {
|
||||
...(raw as Record<string, unknown>),
|
||||
providers: rawProviders,
|
||||
},
|
||||
timeoutMs,
|
||||
}) ??
|
||||
asProviderConfig(rawProviders[provider.id] ?? (raw as Record<string, unknown>)[provider.id]);
|
||||
}
|
||||
return providerConfigs;
|
||||
}
|
||||
|
||||
export function getResolvedSpeechProviderConfig(
|
||||
config: ResolvedTtsConfig,
|
||||
providerId: string,
|
||||
cfg?: OpenClawConfig,
|
||||
): SpeechProviderConfig {
|
||||
const canonical =
|
||||
canonicalizeSpeechProviderId(providerId, cfg) ?? providerId.trim().toLowerCase();
|
||||
return config.providerConfigs[canonical] ?? {};
|
||||
}
|
||||
|
||||
export function resolveTtsConfig(cfg: OpenClawConfig): ResolvedTtsConfig {
|
||||
const raw: TtsConfig = cfg.messages?.tts ?? {};
|
||||
const providerSource = raw.provider ? "config" : "default";
|
||||
const timeoutMs = raw.timeoutMs ?? DEFAULT_TIMEOUT_MS;
|
||||
const auto = normalizeTtsAutoMode(raw.auto) ?? (raw.enabled ? "always" : "off");
|
||||
return {
|
||||
auto,
|
||||
mode: raw.mode ?? "final",
|
||||
provider:
|
||||
canonicalizeSpeechProviderId(raw.provider, cfg) ??
|
||||
resolveRegistryDefaultSpeechProviderId(cfg),
|
||||
providerSource,
|
||||
summaryModel: raw.summaryModel?.trim() || undefined,
|
||||
modelOverrides: resolveModelOverridePolicy(raw.modelOverrides),
|
||||
providerConfigs: resolveSpeechProviderConfigs(raw, cfg, timeoutMs),
|
||||
prefsPath: raw.prefsPath,
|
||||
maxTextLength: raw.maxTextLength ?? DEFAULT_MAX_TEXT_LENGTH,
|
||||
timeoutMs,
|
||||
};
|
||||
}
|
||||
|
||||
export function resolveTtsPrefsPath(config: ResolvedTtsConfig): string {
|
||||
if (config.prefsPath?.trim()) {
|
||||
return resolveUserPath(config.prefsPath.trim());
|
||||
}
|
||||
const envPath = process.env.OPENCLAW_TTS_PREFS?.trim();
|
||||
if (envPath) {
|
||||
return resolveUserPath(envPath);
|
||||
}
|
||||
return path.join(CONFIG_DIR, "settings", "tts.json");
|
||||
}
|
||||
|
||||
function resolveTtsAutoModeFromPrefs(prefs: TtsUserPrefs): TtsAutoMode | undefined {
|
||||
const auto = normalizeTtsAutoMode(prefs.tts?.auto);
|
||||
if (auto) {
|
||||
return auto;
|
||||
}
|
||||
if (typeof prefs.tts?.enabled === "boolean") {
|
||||
return prefs.tts.enabled ? "always" : "off";
|
||||
}
|
||||
return undefined;
|
||||
}
|
||||
|
||||
export function resolveTtsAutoMode(params: {
|
||||
config: ResolvedTtsConfig;
|
||||
prefsPath: string;
|
||||
sessionAuto?: string;
|
||||
}): TtsAutoMode {
|
||||
const sessionAuto = normalizeTtsAutoMode(params.sessionAuto);
|
||||
if (sessionAuto) {
|
||||
return sessionAuto;
|
||||
}
|
||||
const prefsAuto = resolveTtsAutoModeFromPrefs(readPrefs(params.prefsPath));
|
||||
if (prefsAuto) {
|
||||
return prefsAuto;
|
||||
}
|
||||
return params.config.auto;
|
||||
}
|
||||
|
||||
export function buildTtsSystemPromptHint(cfg: OpenClawConfig): string | undefined {
|
||||
const config = resolveTtsConfig(cfg);
|
||||
const prefsPath = resolveTtsPrefsPath(config);
|
||||
const autoMode = resolveTtsAutoMode({ config, prefsPath });
|
||||
if (autoMode === "off") {
|
||||
return undefined;
|
||||
}
|
||||
const maxLength = getTtsMaxLength(prefsPath);
|
||||
const summarize = isSummarizationEnabled(prefsPath) ? "on" : "off";
|
||||
const autoHint =
|
||||
autoMode === "inbound"
|
||||
? "Only use TTS when the user's last message includes audio/voice."
|
||||
: autoMode === "tagged"
|
||||
? "Only use TTS when you include [[tts]] or [[tts:text]] tags."
|
||||
: undefined;
|
||||
return [
|
||||
"Voice (TTS) is enabled.",
|
||||
autoHint,
|
||||
`Keep spoken text ≤${maxLength} chars to avoid auto-summary (summary ${summarize}).`,
|
||||
"Use [[tts:...]] and optional [[tts:text]]...[[/tts:text]] to control voice/expressiveness.",
|
||||
]
|
||||
.filter(Boolean)
|
||||
.join("\n");
|
||||
}
|
||||
|
||||
function readPrefs(prefsPath: string): TtsUserPrefs {
|
||||
try {
|
||||
if (!existsSync(prefsPath)) {
|
||||
return {};
|
||||
}
|
||||
return JSON.parse(readFileSync(prefsPath, "utf8")) as TtsUserPrefs;
|
||||
} catch {
|
||||
return {};
|
||||
}
|
||||
}
|
||||
|
||||
function atomicWriteFileSync(filePath: string, content: string): void {
|
||||
const tmpPath = `${filePath}.tmp.${Date.now()}.${randomBytes(8).toString("hex")}`;
|
||||
writeFileSync(tmpPath, content, { mode: 0o600 });
|
||||
try {
|
||||
renameSync(tmpPath, filePath);
|
||||
} catch (err) {
|
||||
try {
|
||||
unlinkSync(tmpPath);
|
||||
} catch {
|
||||
// ignore
|
||||
}
|
||||
throw err;
|
||||
}
|
||||
}
|
||||
|
||||
function updatePrefs(prefsPath: string, update: (prefs: TtsUserPrefs) => void): void {
|
||||
const prefs = readPrefs(prefsPath);
|
||||
update(prefs);
|
||||
mkdirSync(path.dirname(prefsPath), { recursive: true });
|
||||
atomicWriteFileSync(prefsPath, JSON.stringify(prefs, null, 2));
|
||||
}
|
||||
|
||||
export function isTtsEnabled(
|
||||
config: ResolvedTtsConfig,
|
||||
prefsPath: string,
|
||||
sessionAuto?: string,
|
||||
): boolean {
|
||||
return resolveTtsAutoMode({ config, prefsPath, sessionAuto }) !== "off";
|
||||
}
|
||||
|
||||
export function setTtsAutoMode(prefsPath: string, mode: TtsAutoMode): void {
|
||||
updatePrefs(prefsPath, (prefs) => {
|
||||
const next = { ...prefs.tts };
|
||||
delete next.enabled;
|
||||
next.auto = mode;
|
||||
prefs.tts = next;
|
||||
});
|
||||
}
|
||||
|
||||
export function setTtsEnabled(prefsPath: string, enabled: boolean): void {
|
||||
setTtsAutoMode(prefsPath, enabled ? "always" : "off");
|
||||
}
|
||||
|
||||
export function getTtsProvider(config: ResolvedTtsConfig, prefsPath: string): TtsProvider {
|
||||
const prefs = readPrefs(prefsPath);
|
||||
const prefsProvider = canonicalizeSpeechProviderId(prefs.tts?.provider);
|
||||
if (prefsProvider) {
|
||||
return prefsProvider;
|
||||
}
|
||||
if (config.providerSource === "config") {
|
||||
return canonicalizeSpeechProviderId(config.provider) ?? config.provider;
|
||||
}
|
||||
|
||||
for (const provider of sortSpeechProvidersForAutoSelection()) {
|
||||
if (
|
||||
provider.isConfigured({
|
||||
providerConfig: config.providerConfigs[provider.id] ?? {},
|
||||
timeoutMs: config.timeoutMs,
|
||||
})
|
||||
) {
|
||||
return provider.id;
|
||||
}
|
||||
}
|
||||
return config.provider;
|
||||
}
|
||||
|
||||
export function setTtsProvider(prefsPath: string, provider: TtsProvider): void {
|
||||
updatePrefs(prefsPath, (prefs) => {
|
||||
prefs.tts = { ...prefs.tts, provider: canonicalizeSpeechProviderId(provider) ?? provider };
|
||||
});
|
||||
}
|
||||
|
||||
export function getTtsMaxLength(prefsPath: string): number {
|
||||
const prefs = readPrefs(prefsPath);
|
||||
return prefs.tts?.maxLength ?? DEFAULT_TTS_MAX_LENGTH;
|
||||
}
|
||||
|
||||
export function setTtsMaxLength(prefsPath: string, maxLength: number): void {
|
||||
updatePrefs(prefsPath, (prefs) => {
|
||||
prefs.tts = { ...prefs.tts, maxLength };
|
||||
});
|
||||
}
|
||||
|
||||
export function isSummarizationEnabled(prefsPath: string): boolean {
|
||||
const prefs = readPrefs(prefsPath);
|
||||
return prefs.tts?.summarize ?? DEFAULT_TTS_SUMMARIZE;
|
||||
}
|
||||
|
||||
export function setSummarizationEnabled(prefsPath: string, enabled: boolean): void {
|
||||
updatePrefs(prefsPath, (prefs) => {
|
||||
prefs.tts = { ...prefs.tts, summarize: enabled };
|
||||
});
|
||||
}
|
||||
|
||||
export function getLastTtsAttempt(): TtsStatusEntry | undefined {
|
||||
return lastTtsAttempt;
|
||||
}
|
||||
|
||||
export function setLastTtsAttempt(entry: TtsStatusEntry | undefined): void {
|
||||
lastTtsAttempt = entry;
|
||||
}
|
||||
|
||||
/** Channels that require voice-note-compatible audio */
|
||||
const OPUS_CHANNELS = new Set(["telegram", "feishu", "whatsapp", "matrix"]);
|
||||
|
||||
function resolveChannelId(channel: string | undefined): ChannelId | null {
|
||||
return channel ? normalizeChannelId(channel) : null;
|
||||
}
|
||||
|
||||
export function resolveTtsProviderOrder(primary: TtsProvider, cfg?: OpenClawConfig): TtsProvider[] {
|
||||
const normalizedPrimary = canonicalizeSpeechProviderId(primary, cfg) ?? primary;
|
||||
const ordered = new Set<TtsProvider>([normalizedPrimary]);
|
||||
for (const provider of sortSpeechProvidersForAutoSelection(cfg)) {
|
||||
const normalized = provider.id;
|
||||
if (normalized !== normalizedPrimary) {
|
||||
ordered.add(normalized);
|
||||
}
|
||||
}
|
||||
return [...ordered];
|
||||
}
|
||||
|
||||
export function isTtsProviderConfigured(
|
||||
config: ResolvedTtsConfig,
|
||||
provider: TtsProvider,
|
||||
cfg?: OpenClawConfig,
|
||||
): boolean {
|
||||
const resolvedProvider = getSpeechProvider(provider, cfg);
|
||||
if (!resolvedProvider) {
|
||||
return false;
|
||||
}
|
||||
return (
|
||||
resolvedProvider.isConfigured({
|
||||
cfg,
|
||||
providerConfig: getResolvedSpeechProviderConfig(config, resolvedProvider.id, cfg),
|
||||
timeoutMs: config.timeoutMs,
|
||||
}) ?? false
|
||||
);
|
||||
}
|
||||
|
||||
function formatTtsProviderError(provider: TtsProvider, err: unknown): string {
|
||||
const error = err instanceof Error ? err : new Error(String(err));
|
||||
if (error.name === "AbortError") {
|
||||
return `${provider}: request timed out`;
|
||||
}
|
||||
return `${provider}: ${error.message}`;
|
||||
}
|
||||
|
||||
function buildTtsFailureResult(errors: string[]): { success: false; error: string } {
|
||||
return {
|
||||
success: false,
|
||||
error: `TTS conversion failed: ${errors.join("; ") || "no providers available"}`,
|
||||
};
|
||||
}
|
||||
|
||||
function resolveReadySpeechProvider(params: {
|
||||
provider: TtsProvider;
|
||||
cfg: OpenClawConfig;
|
||||
config: ResolvedTtsConfig;
|
||||
errors: string[];
|
||||
requireTelephony?: boolean;
|
||||
}): NonNullable<ReturnType<typeof getSpeechProvider>> | null {
|
||||
const resolvedProvider = getSpeechProvider(params.provider, params.cfg);
|
||||
if (!resolvedProvider) {
|
||||
params.errors.push(`${params.provider}: no provider registered`);
|
||||
return null;
|
||||
}
|
||||
const providerConfig = getResolvedSpeechProviderConfig(
|
||||
params.config,
|
||||
resolvedProvider.id,
|
||||
params.cfg,
|
||||
);
|
||||
if (
|
||||
!resolvedProvider.isConfigured({
|
||||
cfg: params.cfg,
|
||||
providerConfig,
|
||||
timeoutMs: params.config.timeoutMs,
|
||||
})
|
||||
) {
|
||||
params.errors.push(`${params.provider}: not configured`);
|
||||
return null;
|
||||
}
|
||||
if (params.requireTelephony && !resolvedProvider.synthesizeTelephony) {
|
||||
params.errors.push(`${params.provider}: unsupported for telephony`);
|
||||
return null;
|
||||
}
|
||||
return resolvedProvider;
|
||||
}
|
||||
|
||||
function resolveTtsRequestSetup(params: {
|
||||
text: string;
|
||||
cfg: OpenClawConfig;
|
||||
prefsPath?: string;
|
||||
providerOverride?: TtsProvider;
|
||||
disableFallback?: boolean;
|
||||
}):
|
||||
| {
|
||||
config: ResolvedTtsConfig;
|
||||
providers: TtsProvider[];
|
||||
}
|
||||
| {
|
||||
error: string;
|
||||
} {
|
||||
const config = resolveTtsConfig(params.cfg);
|
||||
const prefsPath = params.prefsPath ?? resolveTtsPrefsPath(config);
|
||||
if (params.text.length > config.maxTextLength) {
|
||||
return {
|
||||
error: `Text too long (${params.text.length} chars, max ${config.maxTextLength})`,
|
||||
};
|
||||
}
|
||||
|
||||
const userProvider = getTtsProvider(config, prefsPath);
|
||||
const provider =
|
||||
canonicalizeSpeechProviderId(params.providerOverride, params.cfg) ?? userProvider;
|
||||
return {
|
||||
config,
|
||||
providers: params.disableFallback ? [provider] : resolveTtsProviderOrder(provider, params.cfg),
|
||||
};
|
||||
}
|
||||
|
||||
export async function textToSpeech(params: {
|
||||
text: string;
|
||||
cfg: OpenClawConfig;
|
||||
prefsPath?: string;
|
||||
channel?: string;
|
||||
overrides?: TtsDirectiveOverrides;
|
||||
disableFallback?: boolean;
|
||||
}): Promise<TtsResult> {
|
||||
const synthesis = await synthesizeSpeech(params);
|
||||
if (!synthesis.success || !synthesis.audioBuffer || !synthesis.fileExtension) {
|
||||
return buildTtsFailureResult([synthesis.error ?? "TTS conversion failed"]);
|
||||
}
|
||||
|
||||
const tempRoot = resolvePreferredOpenClawTmpDir();
|
||||
mkdirSync(tempRoot, { recursive: true, mode: 0o700 });
|
||||
const tempDir = mkdtempSync(path.join(tempRoot, "tts-"));
|
||||
const audioPath = path.join(tempDir, `voice-${Date.now()}${synthesis.fileExtension}`);
|
||||
writeFileSync(audioPath, synthesis.audioBuffer);
|
||||
scheduleCleanup(tempDir);
|
||||
|
||||
return {
|
||||
success: true,
|
||||
audioPath,
|
||||
latencyMs: synthesis.latencyMs,
|
||||
provider: synthesis.provider,
|
||||
outputFormat: synthesis.outputFormat,
|
||||
voiceCompatible: synthesis.voiceCompatible,
|
||||
};
|
||||
}
|
||||
|
||||
export async function synthesizeSpeech(params: {
|
||||
text: string;
|
||||
cfg: OpenClawConfig;
|
||||
prefsPath?: string;
|
||||
channel?: string;
|
||||
overrides?: TtsDirectiveOverrides;
|
||||
disableFallback?: boolean;
|
||||
}): Promise<TtsSynthesisResult> {
|
||||
const setup = resolveTtsRequestSetup({
|
||||
text: params.text,
|
||||
cfg: params.cfg,
|
||||
prefsPath: params.prefsPath,
|
||||
providerOverride: params.overrides?.provider,
|
||||
disableFallback: params.disableFallback,
|
||||
});
|
||||
if ("error" in setup) {
|
||||
return { success: false, error: setup.error };
|
||||
}
|
||||
|
||||
const { config, providers } = setup;
|
||||
const channelId = resolveChannelId(params.channel);
|
||||
const target = channelId && OPUS_CHANNELS.has(channelId) ? "voice-note" : "audio-file";
|
||||
|
||||
const errors: string[] = [];
|
||||
|
||||
for (const provider of providers) {
|
||||
const providerStart = Date.now();
|
||||
try {
|
||||
const resolvedProvider = resolveReadySpeechProvider({
|
||||
provider,
|
||||
cfg: params.cfg,
|
||||
config,
|
||||
errors,
|
||||
});
|
||||
if (!resolvedProvider) {
|
||||
continue;
|
||||
}
|
||||
const synthesis = await resolvedProvider.synthesize({
|
||||
text: params.text,
|
||||
cfg: params.cfg,
|
||||
providerConfig: getResolvedSpeechProviderConfig(config, resolvedProvider.id, params.cfg),
|
||||
target,
|
||||
providerOverrides: params.overrides?.providerOverrides?.[resolvedProvider.id],
|
||||
timeoutMs: config.timeoutMs,
|
||||
});
|
||||
return {
|
||||
success: true,
|
||||
audioBuffer: synthesis.audioBuffer,
|
||||
latencyMs: Date.now() - providerStart,
|
||||
provider,
|
||||
outputFormat: synthesis.outputFormat,
|
||||
voiceCompatible: synthesis.voiceCompatible,
|
||||
fileExtension: synthesis.fileExtension,
|
||||
};
|
||||
} catch (err) {
|
||||
errors.push(formatTtsProviderError(provider, err));
|
||||
}
|
||||
}
|
||||
|
||||
return buildTtsFailureResult(errors);
|
||||
}
|
||||
|
||||
export async function textToSpeechTelephony(params: {
|
||||
text: string;
|
||||
cfg: OpenClawConfig;
|
||||
prefsPath?: string;
|
||||
}): Promise<TtsTelephonyResult> {
|
||||
const setup = resolveTtsRequestSetup({
|
||||
text: params.text,
|
||||
cfg: params.cfg,
|
||||
prefsPath: params.prefsPath,
|
||||
});
|
||||
if ("error" in setup) {
|
||||
return { success: false, error: setup.error };
|
||||
}
|
||||
|
||||
const { config, providers } = setup;
|
||||
|
||||
const errors: string[] = [];
|
||||
|
||||
for (const provider of providers) {
|
||||
const providerStart = Date.now();
|
||||
try {
|
||||
const resolvedProvider = resolveReadySpeechProvider({
|
||||
provider,
|
||||
cfg: params.cfg,
|
||||
config,
|
||||
errors,
|
||||
requireTelephony: true,
|
||||
});
|
||||
if (!resolvedProvider?.synthesizeTelephony) {
|
||||
continue;
|
||||
}
|
||||
const synthesis = await resolvedProvider.synthesizeTelephony({
|
||||
text: params.text,
|
||||
cfg: params.cfg,
|
||||
providerConfig: getResolvedSpeechProviderConfig(config, resolvedProvider.id, params.cfg),
|
||||
timeoutMs: config.timeoutMs,
|
||||
});
|
||||
|
||||
return {
|
||||
success: true,
|
||||
audioBuffer: synthesis.audioBuffer,
|
||||
latencyMs: Date.now() - providerStart,
|
||||
provider,
|
||||
outputFormat: synthesis.outputFormat,
|
||||
sampleRate: synthesis.sampleRate,
|
||||
};
|
||||
} catch (err) {
|
||||
errors.push(formatTtsProviderError(provider, err));
|
||||
}
|
||||
}
|
||||
|
||||
return buildTtsFailureResult(errors);
|
||||
}
|
||||
|
||||
export async function listSpeechVoices(params: {
|
||||
provider: string;
|
||||
cfg?: OpenClawConfig;
|
||||
config?: ResolvedTtsConfig;
|
||||
apiKey?: string;
|
||||
baseUrl?: string;
|
||||
}): Promise<SpeechVoiceOption[]> {
|
||||
const provider = canonicalizeSpeechProviderId(params.provider, params.cfg);
|
||||
if (!provider) {
|
||||
throw new Error("speech provider id is required");
|
||||
}
|
||||
const config = params.config ?? (params.cfg ? resolveTtsConfig(params.cfg) : undefined);
|
||||
if (!config) {
|
||||
throw new Error(`speech provider ${provider} requires cfg or resolved config`);
|
||||
}
|
||||
const resolvedProvider = getSpeechProvider(provider, params.cfg);
|
||||
if (!resolvedProvider) {
|
||||
throw new Error(`speech provider ${provider} is not registered`);
|
||||
}
|
||||
if (!resolvedProvider.listVoices) {
|
||||
throw new Error(`speech provider ${provider} does not support voice listing`);
|
||||
}
|
||||
return await resolvedProvider.listVoices({
|
||||
cfg: params.cfg,
|
||||
providerConfig: getResolvedSpeechProviderConfig(config, resolvedProvider.id, params.cfg),
|
||||
apiKey: params.apiKey,
|
||||
baseUrl: params.baseUrl,
|
||||
});
|
||||
}
|
||||
|
||||
export async function maybeApplyTtsToPayload(params: {
|
||||
payload: ReplyPayload;
|
||||
cfg: OpenClawConfig;
|
||||
channel?: string;
|
||||
kind?: "tool" | "block" | "final";
|
||||
inboundAudio?: boolean;
|
||||
ttsAuto?: string;
|
||||
}): Promise<ReplyPayload> {
|
||||
// Compaction notices are informational UI signals — never synthesise them as speech.
|
||||
if (params.payload.isCompactionNotice) {
|
||||
return params.payload;
|
||||
}
|
||||
const config = resolveTtsConfig(params.cfg);
|
||||
const prefsPath = resolveTtsPrefsPath(config);
|
||||
const autoMode = resolveTtsAutoMode({
|
||||
config,
|
||||
prefsPath,
|
||||
sessionAuto: params.ttsAuto,
|
||||
});
|
||||
if (autoMode === "off") {
|
||||
return params.payload;
|
||||
}
|
||||
|
||||
const reply = resolveSendableOutboundReplyParts(params.payload);
|
||||
const text = reply.text;
|
||||
const directives = parseTtsDirectives(text, config.modelOverrides, {
|
||||
cfg: params.cfg,
|
||||
providerConfigs: config.providerConfigs,
|
||||
});
|
||||
if (directives.warnings.length > 0) {
|
||||
logVerbose(`TTS: ignored directive overrides (${directives.warnings.join("; ")})`);
|
||||
}
|
||||
|
||||
const cleanedText = directives.cleanedText;
|
||||
const trimmedCleaned = cleanedText.trim();
|
||||
const visibleText = trimmedCleaned.length > 0 ? trimmedCleaned : "";
|
||||
const ttsText = directives.ttsText?.trim() || visibleText;
|
||||
|
||||
const nextPayload =
|
||||
visibleText === text.trim()
|
||||
? params.payload
|
||||
: {
|
||||
...params.payload,
|
||||
text: visibleText.length > 0 ? visibleText : undefined,
|
||||
};
|
||||
|
||||
if (autoMode === "tagged" && !directives.hasDirective) {
|
||||
return nextPayload;
|
||||
}
|
||||
if (autoMode === "inbound" && params.inboundAudio !== true) {
|
||||
return nextPayload;
|
||||
}
|
||||
|
||||
const mode = config.mode ?? "final";
|
||||
if (mode === "final" && params.kind && params.kind !== "final") {
|
||||
return nextPayload;
|
||||
}
|
||||
|
||||
if (!ttsText.trim()) {
|
||||
return nextPayload;
|
||||
}
|
||||
if (reply.hasMedia) {
|
||||
return nextPayload;
|
||||
}
|
||||
if (text.includes("MEDIA:")) {
|
||||
return nextPayload;
|
||||
}
|
||||
if (ttsText.trim().length < 10) {
|
||||
return nextPayload;
|
||||
}
|
||||
|
||||
const maxLength = getTtsMaxLength(prefsPath);
|
||||
let textForAudio = ttsText.trim();
|
||||
let wasSummarized = false;
|
||||
|
||||
if (textForAudio.length > maxLength) {
|
||||
if (!isSummarizationEnabled(prefsPath)) {
|
||||
logVerbose(
|
||||
`TTS: truncating long text (${textForAudio.length} > ${maxLength}), summarization disabled.`,
|
||||
);
|
||||
textForAudio = `${textForAudio.slice(0, maxLength - 3)}...`;
|
||||
} else {
|
||||
try {
|
||||
const summary = await summarizeText({
|
||||
text: textForAudio,
|
||||
targetLength: maxLength,
|
||||
cfg: params.cfg,
|
||||
config,
|
||||
timeoutMs: config.timeoutMs,
|
||||
});
|
||||
textForAudio = summary.summary;
|
||||
wasSummarized = true;
|
||||
if (textForAudio.length > config.maxTextLength) {
|
||||
logVerbose(
|
||||
`TTS: summary exceeded hard limit (${textForAudio.length} > ${config.maxTextLength}); truncating.`,
|
||||
);
|
||||
textForAudio = `${textForAudio.slice(0, config.maxTextLength - 3)}...`;
|
||||
}
|
||||
} catch (err) {
|
||||
const error = err as Error;
|
||||
logVerbose(`TTS: summarization failed, truncating instead: ${error.message}`);
|
||||
textForAudio = `${textForAudio.slice(0, maxLength - 3)}...`;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
textForAudio = stripMarkdown(textForAudio).trim(); // strip markdown for TTS (### → "hashtag" etc.)
|
||||
if (textForAudio.length < 10) {
|
||||
return nextPayload;
|
||||
}
|
||||
|
||||
const ttsStart = Date.now();
|
||||
const result = await textToSpeech({
|
||||
text: textForAudio,
|
||||
cfg: params.cfg,
|
||||
prefsPath,
|
||||
channel: params.channel,
|
||||
overrides: directives.overrides,
|
||||
});
|
||||
|
||||
if (result.success && result.audioPath) {
|
||||
lastTtsAttempt = {
|
||||
timestamp: Date.now(),
|
||||
success: true,
|
||||
textLength: text.length,
|
||||
summarized: wasSummarized,
|
||||
provider: result.provider,
|
||||
latencyMs: result.latencyMs,
|
||||
};
|
||||
|
||||
const channelId = resolveChannelId(params.channel);
|
||||
const shouldVoice =
|
||||
channelId !== null && OPUS_CHANNELS.has(channelId) && result.voiceCompatible === true;
|
||||
const finalPayload = {
|
||||
...nextPayload,
|
||||
mediaUrl: result.audioPath,
|
||||
audioAsVoice: shouldVoice || params.payload.audioAsVoice,
|
||||
};
|
||||
return finalPayload;
|
||||
}
|
||||
|
||||
lastTtsAttempt = {
|
||||
timestamp: Date.now(),
|
||||
success: false,
|
||||
textLength: text.length,
|
||||
summarized: wasSummarized,
|
||||
error: result.error,
|
||||
};
|
||||
|
||||
const latency = Date.now() - ttsStart;
|
||||
logVerbose(`TTS: conversion failed after ${latency}ms (${result.error ?? "unknown"}).`);
|
||||
return nextPayload;
|
||||
}
|
||||
|
||||
export const _test = {
|
||||
parseTtsDirectives,
|
||||
resolveModelOverridePolicy,
|
||||
summarizeText,
|
||||
getResolvedSpeechProviderConfig,
|
||||
};
|
||||
TtsResult,
|
||||
TtsSynthesisResult,
|
||||
TtsTelephonyResult,
|
||||
} from "../plugin-sdk/speech-runtime.js";
|
||||
|
||||
Reference in New Issue
Block a user