diff --git a/docs/concepts/qa-e2e-automation.md b/docs/concepts/qa-e2e-automation.md index d8305037734..03a1a501a01 100644 --- a/docs/concepts/qa-e2e-automation.md +++ b/docs/concepts/qa-e2e-automation.md @@ -572,9 +572,15 @@ Telegram, Discord, Slack, and WhatsApp lanes can lease credentials from a shared Payload shapes the broker validates on `admin/add`: - Telegram (`kind: "telegram"`): `{ groupId: string, driverToken: string, sutToken: string }` - `groupId` must be a numeric chat-id string. +- Telegram real user (`kind: "telegram-user"`): `{ groupId: string, sutToken: string, testerUserId: string, testerUsername: string, telegramApiId: string, telegramApiHash: string, tdlibDatabaseEncryptionKey: string, tdlibArchiveBase64: string, tdlibArchiveSha256: string, desktopTdataArchiveBase64: string, desktopTdataArchiveSha256: string }` - one exclusive burner-account lease used by both the TDLib CLI driver and Telegram Desktop visual witness. - Discord (`kind: "discord"`): `{ guildId: string, channelId: string, driverBotToken: string, sutBotToken: string, sutApplicationId: string }`. - WhatsApp (`kind: "whatsapp"`): `{ driverPhoneE164: string, sutPhoneE164: string, driverAuthArchiveBase64: string, sutAuthArchiveBase64: string, groupJid?: string }` - phone numbers must be distinct E.164 strings. +For visual real-user Telegram proof, prefer `pnpm qa:telegram-user:crabbox -- --text /status`. +It uses one Convex `telegram-user` lease for both the TDLib CLI driver and the +Telegram Desktop witness, captures a Crabbox recording plus motion-trimmed +video/GIF artifacts, and releases the lease on shutdown. + Slack lanes can also use the pool. Slack payload shape checks currently live in the Slack QA runner rather than the broker; use `{ channelId: string, driverBotToken: string, sutBotToken: string, sutAppToken: string }`, with a Slack channel id like `Cxxxxxxxxxx`. See [Setting up the Slack workspace](#setting-up-the-slack-workspace) for app and scope provisioning. Operational env vars and the Convex broker endpoint contract live in [Testing → Shared Telegram credentials via Convex](/help/testing#shared-telegram-credentials-via-convex-v1) (the section name predates the multi-channel pool; the lease semantics are shared across kinds). diff --git a/docs/help/testing.md b/docs/help/testing.md index b03ffb15c15..ef507bdb77b 100644 --- a/docs/help/testing.md +++ b/docs/help/testing.md @@ -404,6 +404,9 @@ Default endpoint contract (`OPENCLAW_QA_CONVEX_SITE_URL` + `/qa-credentials/v1`) - Request: `{ kind, ownerId, actorRole, leaseTtlMs, heartbeatIntervalMs }` - Success: `{ status: "ok", credentialId, leaseToken, payload, leaseTtlMs?, heartbeatIntervalMs? }` - Exhausted/retryable: `{ status: "error", code: "POOL_EXHAUSTED" | "NO_CREDENTIAL_AVAILABLE", ... }` +- `POST /payload-chunk` + - Request: `{ kind, ownerId, actorRole, credentialId, leaseToken, index }` + - Success: `{ status: "ok", index, data }` - `POST /heartbeat` - Request: `{ kind, ownerId, actorRole, credentialId, leaseToken, leaseTtlMs }` - Success: `{ status: "ok" }` (or empty `2xx`) @@ -427,6 +430,46 @@ Payload shape for Telegram kind: - `groupId` must be a numeric Telegram chat id string. - `admin/add` validates this shape for `kind: "telegram"` and rejects malformed payloads. +Payload shape for Telegram real-user kind: + +- `{ groupId: string, sutToken: string, testerUserId: string, testerUsername: string, telegramApiId: string, telegramApiHash: string, tdlibDatabaseEncryptionKey: string, tdlibArchiveBase64: string, tdlibArchiveSha256: string, desktopTdataArchiveBase64: string, desktopTdataArchiveSha256: string }` +- `groupId`, `testerUserId`, and `telegramApiId` must be numeric strings. +- `tdlibArchiveSha256` and `desktopTdataArchiveSha256` must be SHA-256 hex strings. +- `kind: "telegram-user"` represents one Telegram burner account. Treat the lease as account-wide: the TDLib CLI driver and Telegram Desktop visual witness restore from the same payload, and only one job should hold the lease at a time. + +Telegram real-user lease restore: + +```bash +tmp=$(mktemp -d /tmp/openclaw-telegram-user.XXXXXX) +node --import tsx scripts/e2e/telegram-user-credential.ts lease-restore \ + --user-driver-dir "$tmp/user-driver" \ + --desktop-workdir "$tmp/desktop" \ + --lease-file "$tmp/lease.json" +TELEGRAM_USER_DRIVER_STATE_DIR="$tmp/user-driver" \ + uv run ~/.codex/skills/custom/telegram-e2e-bot-to-bot/scripts/user-driver.py status --json +node --import tsx scripts/e2e/telegram-user-credential.ts release --lease-file "$tmp/lease.json" +``` + +Use the restored Desktop profile with `Telegram -workdir "$tmp/desktop"` when a visual recording is needed. In local operator environments, `scripts/e2e/telegram-user-credential.ts` reads `~/.codex/skills/custom/telegram-e2e-bot-to-bot/convex.local.env` by default if process env vars are absent. + +One-command Crabbox proof: + +```bash +pnpm qa:telegram-user:crabbox -- --text /status +``` + +That command leases the `telegram-user` credential, restores the same account +into TDLib and Telegram Desktop on a Crabbox Linux desktop, starts a local mock +SUT gateway from the current checkout, sends the command as the real QA user, +records the visible Telegram Desktop session, trims the recording to the motion +window, writes artifacts under `.artifacts/qa-e2e/telegram-user-crabbox/`, then +releases the credential and stops the box. Use `--id ` to reuse a warm +desktop lease, `--keep-box` to keep VNC open after failure, +`--desktop-chat-title ` to pick the visible chat, and `--tdlib-url ` +when using a prebaked Linux `libtdjson.so` archive instead of building TDLib on +a fresh box. The runner verifies `--tdlib-url` with `--tdlib-sha256 ` or, +by default, a sibling `.sha256` file. + Broker-validated multi-channel payloads: - Discord: `{ guildId: string, channelId: string, driverBotToken: string, sutBotToken: string, sutApplicationId: string, voiceChannelId?: string }` diff --git a/package.json b/package.json index 1f7ad581505..91f3d7b686e 100644 --- a/package.json +++ b/package.json @@ -1506,6 +1506,7 @@ "qa:lab:up:fast": "node --import tsx scripts/qa-lab-up.ts --use-prebuilt-image --bind-ui-dist --skip-ui-build", "qa:lab:watch": "vite build --watch --config extensions/qa-lab/web/vite.config.ts", "qa:otel:smoke": "node --import tsx scripts/qa-otel-smoke.ts", + "qa:telegram-user:crabbox": "node --import tsx scripts/e2e/telegram-user-crabbox-proof.ts", "release-metadata:check": "node scripts/check-release-metadata-only.mjs", "release:beta-smoke": "node --import tsx scripts/release-beta-smoke.ts", "release:check": "pnpm release:generated:check && node --import tsx scripts/release-check.ts", diff --git a/scripts/e2e/telegram-user-crabbox-proof.ts b/scripts/e2e/telegram-user-crabbox-proof.ts new file mode 100644 index 00000000000..89c9236b997 --- /dev/null +++ b/scripts/e2e/telegram-user-crabbox-proof.ts @@ -0,0 +1,1239 @@ +#!/usr/bin/env -S node --import tsx + +import { type ChildProcess, spawn, type SpawnOptionsWithoutStdio } from "node:child_process"; +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; + +type CommandResult = { + stderr: string; + stdout: string; +}; + +type JsonObject = Record; + +type CrabboxInspect = { + host?: string; + id?: string; + slug?: string; + sshKey?: string; + sshPort?: string; + sshUser?: string; + state?: string; +}; + +type Options = { + crabboxBin: string; + desktopChatTitle: string; + dryRun: boolean; + envFile?: string; + expect: string[]; + gatewayPort: number; + idleTimeout: string; + keepBox: boolean; + leaseId?: string; + mockPort: number; + outputDir: string; + provider: string; + recordSeconds: number; + sutUsername?: string; + target: string; + tdlibSha256?: string; + tdlibUrl?: string; + text: string; + timeoutMs: number; + ttl: string; + userDriverScript: string; +}; + +type LocalSut = { + configPath: string; + drained: { + drained: number; + pendingAfter?: number; + pendingBefore?: number; + webhookUrlSet: boolean; + }; + mock: ChildProcess; + mockLog: string; + requestLog: string; + stateDir: string; + tempRoot: string; + workspace: string; + gateway: ChildProcess; + gatewayLog: string; +}; + +const DEFAULT_SKILL_DIR = "~/.codex/skills/custom/telegram-e2e-bot-to-bot"; +const DEFAULT_CONVEX_ENV_FILE = `${DEFAULT_SKILL_DIR}/convex.local.env`; +const DEFAULT_USER_DRIVER = `${DEFAULT_SKILL_DIR}/scripts/user-driver.py`; +const DEFAULT_OUTPUT_ROOT = ".artifacts/qa-e2e/telegram-user-crabbox"; +const REMOTE_ROOT = "/tmp/openclaw-telegram-user-crabbox"; + +function usageText() { + return [ + "Usage:", + " node --import tsx scripts/e2e/telegram-user-crabbox-proof.ts [--text /status] [--expect OpenClaw]", + "", + "Useful options:", + " --desktop-chat-title Telegram Desktop chat to select before recording.", + " --id Reuse an existing Crabbox desktop lease.", + " --keep-box Leave the Crabbox lease running for VNC debugging.", + " --output-dir Artifact directory under the repo.", + " --record-seconds Desktop video duration. Default: 35.", + " --tdlib-sha256 Expected SHA-256 for --tdlib-url. Defaults to .sha256.", + " --tdlib-url Linux tdlib archive containing libtdjson.so.", + " --dry-run Validate local inputs and print the plan.", + ].join("\n"); +} + +function usage(): never { + throw new Error(usageText()); +} + +function expandHome(value: string) { + if (value === "~") { + return os.homedir(); + } + if (value.startsWith("~/")) { + return path.join(os.homedir(), value.slice(2)); + } + return value; +} + +function trimToValue(value: string | undefined) { + const trimmed = value?.trim(); + return trimmed && trimmed.length > 0 ? trimmed : undefined; +} + +function parsePositiveInteger(value: string, label: string) { + const parsed = Number(value); + if (!Number.isInteger(parsed) || parsed < 1) { + throw new Error(`${label} must be a positive integer.`); + } + return parsed; +} + +function parseArgs(argv: string[]): Options { + argv = argv[0] === "--" ? argv.slice(1) : argv; + const stamp = new Date().toISOString().replace(/[:.]/gu, "-"); + const opts: Options = { + crabboxBin: trimToValue(process.env.OPENCLAW_TELEGRAM_USER_CRABBOX_BIN) ?? "crabbox", + desktopChatTitle: + trimToValue(process.env.OPENCLAW_TELEGRAM_USER_DESKTOP_CHAT_TITLE) ?? "OpenClaw Testing", + dryRun: false, + expect: ["OpenClaw"], + gatewayPort: 19_879, + idleTimeout: "60m", + keepBox: false, + mockPort: 19_882, + outputDir: path.join(DEFAULT_OUTPUT_ROOT, stamp), + provider: process.env.OPENCLAW_TELEGRAM_USER_CRABBOX_PROVIDER?.trim() || "aws", + recordSeconds: 35, + target: "linux", + text: "/status", + timeoutMs: 90_000, + ttl: "120m", + userDriverScript: DEFAULT_USER_DRIVER, + }; + let expectWasPassed = false; + for (let index = 0; index < argv.length; index += 1) { + const arg = argv[index]; + const readValue = () => { + const value = argv[index + 1]; + if (!value || value.startsWith("--")) { + usage(); + } + index += 1; + return value; + }; + if (arg === "--crabbox-bin") { + opts.crabboxBin = readValue(); + } else if (arg === "--desktop-chat-title") { + opts.desktopChatTitle = readValue(); + } else if (arg === "--dry-run") { + opts.dryRun = true; + } else if (arg === "--env-file") { + opts.envFile = readValue(); + } else if (arg === "--expect") { + if (!expectWasPassed) { + opts.expect = []; + expectWasPassed = true; + } + opts.expect.push(readValue()); + } else if (arg === "--gateway-port") { + opts.gatewayPort = parsePositiveInteger(readValue(), "--gateway-port"); + } else if (arg === "--id") { + opts.leaseId = readValue(); + } else if (arg === "--idle-timeout") { + opts.idleTimeout = readValue(); + } else if (arg === "--keep-box") { + opts.keepBox = true; + } else if (arg === "--mock-port") { + opts.mockPort = parsePositiveInteger(readValue(), "--mock-port"); + } else if (arg === "--output-dir") { + opts.outputDir = readValue(); + } else if (arg === "--provider") { + opts.provider = readValue(); + } else if (arg === "--record-seconds") { + opts.recordSeconds = parsePositiveInteger(readValue(), "--record-seconds"); + } else if (arg === "--sut-username") { + opts.sutUsername = readValue().replace(/^@/u, ""); + } else if (arg === "--target") { + opts.target = readValue(); + } else if (arg === "--tdlib-sha256") { + opts.tdlibSha256 = readValue().toLowerCase(); + } else if (arg === "--tdlib-url") { + opts.tdlibUrl = readValue(); + } else if (arg === "--text") { + opts.text = readValue(); + } else if (arg === "--timeout-ms") { + opts.timeoutMs = parsePositiveInteger(readValue(), "--timeout-ms"); + } else if (arg === "--ttl") { + opts.ttl = readValue(); + } else if (arg === "--user-driver-script") { + opts.userDriverScript = readValue(); + } else if (arg === "--help" || arg === "-h") { + console.log(usageText()); + process.exit(0); + } else { + throw new Error(`Unknown argument: ${arg}`); + } + } + return opts; +} + +function repoRoot() { + const cwd = process.cwd(); + if ( + !fs.existsSync(path.join(cwd, "package.json")) || + !fs.existsSync(path.join(cwd, "scripts/e2e/mock-openai-server.mjs")) + ) { + throw new Error("Run from the OpenClaw repo root."); + } + return cwd; +} + +function resolveRepoPath(root: string, value: string) { + const resolved = path.isAbsolute(value) ? value : path.resolve(root, value); + const relative = path.relative(root, resolved); + if (relative === "" || relative.startsWith("..") || path.isAbsolute(relative)) { + throw new Error(`Output path must stay inside the repo: ${value}`); + } + return resolved; +} + +function readJsonFile(filePath: string): JsonObject { + try { + return JSON.parse(fs.readFileSync(expandHome(filePath), "utf8")) as JsonObject; + } catch (error) { + if (error && typeof error === "object" && "code" in error && error.code === "ENOENT") { + return {}; + } + throw error; + } +} + +function requireString(source: JsonObject, key: string) { + const value = source[key]; + if (typeof value === "number") { + return String(value); + } + if (typeof value === "string" && value.trim()) { + return value.trim(); + } + throw new Error(`Missing ${key}.`); +} + +function optionalString(source: JsonObject, key: string) { + const value = source[key]; + return typeof value === "string" && value.trim() ? value.trim() : undefined; +} + +function runCommand(params: { + args: string[]; + command: string; + cwd: string; + env?: NodeJS.ProcessEnv; + stdio?: "inherit" | "pipe"; + stdin?: string; +}) { + return new Promise((resolve, reject) => { + const child = spawn(params.command, params.args, { + cwd: params.cwd, + env: params.env ?? process.env, + stdio: ["pipe", "pipe", "pipe"], + }); + let stdout = ""; + let stderr = ""; + child.stdout.on("data", (chunk: Buffer) => { + const text = chunk.toString(); + stdout += text; + if (params.stdio === "inherit") { + process.stdout.write(text); + } + }); + child.stderr.on("data", (chunk: Buffer) => { + const text = chunk.toString(); + stderr += text; + if (params.stdio === "inherit") { + process.stderr.write(text); + } + }); + child.on("error", reject); + child.on("close", (code, signal) => { + if (code === 0) { + resolve({ stdout, stderr }); + return; + } + const detail = signal ? `signal ${signal}` : `exit code ${code ?? "unknown"}`; + reject( + new Error( + `${params.command} ${params.args.join(" ")} failed with ${detail}\n${stdout}${stderr}`, + ), + ); + }); + if (params.stdin) { + child.stdin.end(params.stdin); + } else { + child.stdin.end(); + } + }); +} + +function spawnLogged(command: string, args: string[], options: SpawnOptionsWithoutStdio) { + const child = spawn(command, args, { + ...options, + detached: true, + stdio: ["ignore", "pipe", "pipe"], + }); + child.stdout.setEncoding("utf8"); + child.stderr.setEncoding("utf8"); + let output = ""; + const capture = (chunk: string) => { + output = `${output}${chunk}`.slice(-12000); + }; + child.stdout.on("data", capture); + child.stderr.on("data", capture); + return { + child, + get output() { + return output; + }, + }; +} + +function waitForOutput( + child: ChildProcess, + pattern: RegExp, + output: () => string, + label: string, + timeoutMs: number, +) { + return new Promise((resolve, reject) => { + const timeout = setTimeout(() => { + reject( + new Error(`${label} did not become ready within ${timeoutMs}ms\n${output().slice(-4000)}`), + ); + }, timeoutMs); + const onData = () => { + if (pattern.test(output())) { + cleanup(); + resolve(); + } + }; + const onExit = (code: number | null) => { + cleanup(); + reject( + new Error( + `${label} exited before ready with code ${code ?? "unknown"}\n${output().slice(-4000)}`, + ), + ); + }; + const cleanup = () => { + clearTimeout(timeout); + child.stdout?.off("data", onData); + child.stderr?.off("data", onData); + child.off("exit", onExit); + }; + child.stdout?.on("data", onData); + child.stderr?.on("data", onData); + child.on("exit", onExit); + onData(); + }); +} + +function killTree(child: ChildProcess | undefined) { + if (!child || child.killed || child.exitCode !== null) { + return; + } + if (!child.pid) { + return; + } + try { + process.kill(-child.pid, "SIGTERM"); + } catch { + child.kill("SIGTERM"); + } +} + +async function telegram(token: string, method: string, body: JsonObject = {}) { + const response = await fetch(`https://api.telegram.org/bot${token}/${method}`, { + method: "POST", + headers: { "content-type": "application/json" }, + body: JSON.stringify(body), + }); + const payload = (await response.json()) as JsonObject; + if (!response.ok || payload.ok !== true) { + throw new Error( + optionalString(payload, "description") ?? `${method} failed with HTTP ${response.status}`, + ); + } + return payload.result; +} + +async function drainSutUpdates(sutToken: string) { + const before = telegramResultObject(await telegram(sutToken, "getWebhookInfo"), "getWebhookInfo"); + const rawUpdates = await telegram(sutToken, "getUpdates", { + allowed_updates: ["message", "edited_message"], + timeout: 0, + }); + if (!Array.isArray(rawUpdates)) { + throw new Error("getUpdates returned an invalid payload."); + } + const updates = rawUpdates; + if (updates.length) { + const last = updates.at(-1); + if ( + last && + typeof last === "object" && + "update_id" in last && + typeof last.update_id === "number" + ) { + await telegram(sutToken, "getUpdates", { offset: last.update_id + 1, timeout: 0 }); + } + } + const after = telegramResultObject(await telegram(sutToken, "getWebhookInfo"), "getWebhookInfo"); + return { + drained: updates.length, + pendingAfter: + typeof after.pending_update_count === "number" ? after.pending_update_count : undefined, + pendingBefore: + typeof before.pending_update_count === "number" ? before.pending_update_count : undefined, + webhookUrlSet: typeof before.url === "string" && before.url.length > 0, + }; +} + +async function sutIdentity(sutToken: string) { + const result = telegramResultObject(await telegram(sutToken, "getMe"), "getMe"); + const username = requireString(result, "username").replace(/^@/u, ""); + return { id: requireString(result, "id"), username }; +} + +function telegramResultObject(value: unknown, label: string): JsonObject { + if (!value || typeof value !== "object" || Array.isArray(value)) { + throw new Error(`${label} returned an invalid payload.`); + } + return value as JsonObject; +} + +function writeSutConfig(params: { + gatewayPort: number; + groupId: string; + mockPort: number; + outputDir: string; + testerId: string; +}) { + const tempRoot = fs.mkdtempSync(path.join(os.tmpdir(), "openclaw-tg-crabbox-sut-")); + const stateDir = path.join(tempRoot, "state"); + const workspace = path.join(tempRoot, "workspace"); + fs.mkdirSync(stateDir, { recursive: true }); + fs.mkdirSync(workspace, { recursive: true }); + const configPath = path.join(tempRoot, "openclaw.json"); + const config = { + agents: { + defaults: { + model: { primary: "openai/gpt-5.5" }, + models: { "openai/gpt-5.5": { params: { openaiWsWarmup: false, transport: "sse" } } }, + }, + list: [ + { + default: true, + id: "main", + model: { primary: "openai/gpt-5.5" }, + name: "Main", + workspace, + }, + ], + }, + channels: { + telegram: { + allowFrom: [params.testerId], + botToken: { id: "TELEGRAM_BOT_TOKEN", provider: "default", source: "env" }, + commands: { native: true, nativeSkills: false }, + dmPolicy: "allowlist", + enabled: true, + groupAllowFrom: [params.testerId], + groupPolicy: "allowlist", + groups: { + [params.groupId]: { + allowFrom: [params.testerId], + groupPolicy: "allowlist", + requireMention: false, + }, + }, + replyToMode: "first", + }, + }, + gateway: { auth: { mode: "none" }, bind: "loopback", mode: "local", port: params.gatewayPort }, + messages: { groupChat: { visibleReplies: "automatic" } }, + models: { + providers: { + openai: { + api: "openai-responses", + apiKey: { id: "OPENAI_API_KEY", provider: "default", source: "env" }, + baseUrl: `http://127.0.0.1:${params.mockPort}/v1`, + models: [ + { api: "openai-responses", contextWindow: 128000, id: "gpt-5.5", name: "gpt-5.5" }, + ], + request: { allowPrivateNetwork: true }, + }, + }, + }, + plugins: { + allow: ["telegram", "openai"], + enabled: true, + entries: { openai: { enabled: true }, telegram: { enabled: true } }, + }, + }; + fs.writeFileSync(configPath, `${JSON.stringify(config, null, 2)}\n`); + return { configPath, stateDir, tempRoot, workspace }; +} + +async function startLocalSut(params: { + gatewayPort: number; + groupId: string; + mockPort: number; + outputDir: string; + sutToken: string; + testerId: string; + repoRoot: string; +}) { + const drained = await drainSutUpdates(params.sutToken); + const config = writeSutConfig(params); + const requestLog = path.join(params.outputDir, "mock-openai-requests.ndjson"); + const mock = spawnLogged("node", ["scripts/e2e/mock-openai-server.mjs"], { + cwd: params.repoRoot, + env: { + ...process.env, + MOCK_PORT: String(params.mockPort), + MOCK_REQUEST_LOG: requestLog, + SUCCESS_MARKER: "OPENCLAW_E2E_OK", + }, + }); + await waitForOutput( + mock.child, + /mock-openai listening/u, + () => mock.output, + "mock-openai", + 10_000, + ); + const gateway = spawnLogged( + "pnpm", + ["openclaw", "gateway", "--port", String(params.gatewayPort)], + { + cwd: params.repoRoot, + env: { + ...process.env, + OPENAI_API_KEY: "sk-openclaw-e2e-mock", + OPENCLAW_CONFIG_PATH: config.configPath, + OPENCLAW_STATE_DIR: config.stateDir, + TELEGRAM_BOT_TOKEN: params.sutToken, + }, + }, + ); + await waitForOutput(gateway.child, /\[gateway\] ready/u, () => gateway.output, "gateway", 60_000); + return { + ...config, + drained, + gateway: gateway.child, + get gatewayLog() { + return gateway.output; + }, + mock: mock.child, + get mockLog() { + return mock.output; + }, + requestLog, + }; +} + +function extractLeaseId(output: string) { + return output.match(/\b(?:cbx_[a-f0-9]+|tbx_[A-Za-z0-9_-]+)\b/u)?.[0]; +} + +async function warmupCrabbox(opts: Options, root: string) { + const result = await runCommand({ + command: opts.crabboxBin, + args: [ + "warmup", + "--provider", + opts.provider, + "--target", + opts.target, + "--desktop", + "--browser", + "--idle-timeout", + opts.idleTimeout, + "--ttl", + opts.ttl, + ], + cwd: root, + stdio: "inherit", + }); + const leaseId = extractLeaseId(`${result.stdout}\n${result.stderr}`); + if (!leaseId) { + throw new Error("Crabbox warmup did not print a lease id."); + } + return leaseId; +} + +async function inspectCrabbox(opts: Options, root: string, leaseId: string) { + const result = await runCommand({ + command: opts.crabboxBin, + args: [ + "inspect", + "--provider", + opts.provider, + "--target", + opts.target, + "--id", + leaseId, + "--json", + ], + cwd: root, + }); + return JSON.parse(result.stdout) as CrabboxInspect; +} + +function sshArgs(inspect: CrabboxInspect) { + if (!inspect.host || !inspect.sshKey || !inspect.sshUser) { + throw new Error("Crabbox inspect output is missing SSH details."); + } + return { + base: [ + "-i", + inspect.sshKey, + "-p", + inspect.sshPort ?? "22", + "-o", + "IdentitiesOnly=yes", + "-o", + "BatchMode=yes", + "-o", + "StrictHostKeyChecking=accept-new", + "-o", + "ConnectTimeout=15", + ], + scpBase: [ + "-i", + inspect.sshKey, + "-P", + inspect.sshPort ?? "22", + "-o", + "IdentitiesOnly=yes", + "-o", + "BatchMode=yes", + "-o", + "StrictHostKeyChecking=accept-new", + "-o", + "ConnectTimeout=15", + ], + target: `${inspect.sshUser}@${inspect.host}`, + }; +} + +async function scpToRemote(root: string, inspect: CrabboxInspect, local: string, remote: string) { + const ssh = sshArgs(inspect); + await runCommand({ + command: "scp", + args: [...ssh.scpBase, local, `${ssh.target}:${remote}`], + cwd: root, + stdio: "inherit", + }); +} + +async function scpFromRemote(root: string, inspect: CrabboxInspect, remote: string, local: string) { + const ssh = sshArgs(inspect); + await runCommand({ + command: "scp", + args: [...ssh.scpBase, `${ssh.target}:${remote}`, local], + cwd: root, + stdio: "inherit", + }); +} + +async function sshRun(root: string, inspect: CrabboxInspect, remoteCommand: string) { + const ssh = sshArgs(inspect); + return await runCommand({ + command: "ssh", + args: [...ssh.base, ssh.target, remoteCommand], + cwd: root, + stdio: "inherit", + }); +} + +function renderRemoteSetup(params: { tdlibSha256?: string; tdlibUrl?: string }) { + const tdlibSha256 = JSON.stringify(params.tdlibSha256 ?? ""); + const tdlibUrl = JSON.stringify(params.tdlibUrl ?? ""); + return `#!/usr/bin/env bash +set -euo pipefail +root=${REMOTE_ROOT} +tdlib_sha256=${tdlibSha256} +tdlib_url=${tdlibUrl} +mkdir -p "$root" +tar -xzf "$root/state.tgz" -C "$root" +sudo apt-get update -y +sudo DEBIAN_FRONTEND=noninteractive apt-get install -y curl git cmake g++ make zlib1g-dev libssl-dev python3 ffmpeg scrot xz-utils tar wmctrl xdotool libopengl0 libxcb-cursor0 libxcb-icccm4 libxcb-image0 libxcb-keysyms1 libxcb-randr0 libxcb-render-util0 libxcb-shape0 libxcb-xfixes0 libxcb-xinerama0 libxkbcommon-x11-0 >/tmp/openclaw-telegram-apt.log +if ! command -v python3 >/dev/null 2>&1; then + echo "python3 is required" >&2 + exit 127 +fi +if [ ! -x "$root/Telegram/Telegram" ]; then + curl -fL https://telegram.org/dl/desktop/linux -o "$root/telegram.tar.xz" + tar -xJf "$root/telegram.tar.xz" -C "$root" +fi +if ! ldconfig -p | grep -q libtdjson.so; then + if [ -n "$tdlib_url" ]; then + curl -fL "$tdlib_url" -o "$root/tdlib-linux.tgz" + if [ -z "$tdlib_sha256" ]; then + curl -fL "$tdlib_url.sha256" -o "$root/tdlib-linux.tgz.sha256" + tdlib_sha256="$(awk '{print $1; exit}' "$root/tdlib-linux.tgz.sha256")" + fi + printf '%s %s\\n' "$tdlib_sha256" "$root/tdlib-linux.tgz" | sha256sum -c - + mkdir -p "$root/tdlib-linux" + tar -xzf "$root/tdlib-linux.tgz" -C "$root/tdlib-linux" + lib="$(find "$root/tdlib-linux" -name libtdjson.so -type f | head -n 1)" + test -n "$lib" + sudo install -m 0755 "$lib" /usr/local/lib/libtdjson.so + else + rm -rf "$root/td" "$root/td-build" + git clone --depth 1 --branch v1.8.0 https://github.com/tdlib/td.git "$root/td" + cmake -S "$root/td" -B "$root/td-build" -DCMAKE_BUILD_TYPE=Release -DTD_ENABLE_JNI=OFF + cmake --build "$root/td-build" --target tdjson -j "$(nproc)" + sudo cmake --install "$root/td-build" + fi + sudo ldconfig +fi +TELEGRAM_USER_DRIVER_STATE_DIR="$root/user-driver" python3 "$root/user-driver.py" status --json --timeout-ms 60000 >"$root/status.json" +`; +} + +function renderLaunchDesktop() { + return `#!/usr/bin/env bash +set -euo pipefail +root=${REMOTE_ROOT} +export DISPLAY="\${DISPLAY:-:99}" +pkill -f "$root/Telegram/Telegram" >/dev/null 2>&1 || true +nohup "$root/Telegram/Telegram" -workdir "$root/desktop" >"$root/telegram-desktop.log" 2>&1 & +pid=$! +sleep 8 +if ! kill -0 "$pid" >/dev/null 2>&1; then + cat "$root/telegram-desktop.log" >&2 + exit 1 +fi +if ! wmctrl -l | grep -i telegram >/dev/null 2>&1; then + cat "$root/telegram-desktop.log" >&2 + exit 1 +fi +`; +} + +function renderSelectDesktopChat(params: { chatTitle: string }) { + return `#!/usr/bin/env bash +set -euo pipefail +chat_title=${JSON.stringify(params.chatTitle)} +export DISPLAY="\${DISPLAY:-:99}" +win="$(wmctrl -l | awk 'tolower($0) ~ /telegram/ {print $1; exit}')" +test -n "$win" +left=520 +top=170 +xdotool windowactivate --sync "$win" +xdotool windowsize "$win" 980 720 +xdotool windowmove "$win" "$left" "$top" +sleep 1 +xdotool mousemove "$((left + 180))" "$((top + 50))" click 1 +xdotool key ctrl+a BackSpace +xdotool type --delay 5 -- "$chat_title" +sleep 2 +xdotool mousemove "$((left + 150))" "$((top + 120))" click 1 +sleep 1 +`; +} + +function renderRemoteProbe(params: { + expect: string[]; + sutUsername: string; + text: string; + timeoutMs: number; +}) { + const args = [ + "probe", + "--text", + params.text, + "--timeout-ms", + String(params.timeoutMs), + "--output", + `${REMOTE_ROOT}/probe.json`, + "--json", + ]; + for (const expected of params.expect) { + args.push("--expect", expected); + } + const escapedArgs = args.map((arg) => JSON.stringify(arg)).join(" "); + return `#!/usr/bin/env bash +set -euo pipefail +root=${REMOTE_ROOT} +export TELEGRAM_USER_DRIVER_STATE_DIR="$root/user-driver" +export TELEGRAM_USER_DRIVER_SUT_USERNAME=${JSON.stringify(params.sutUsername)} +python3 "$root/user-driver.py" ${escapedArgs} +`; +} + +async function writeExecutable(filePath: string, content: string) { + fs.writeFileSync(filePath, content); + fs.chmodSync(filePath, 0o700); +} + +async function prepareRemoteState(params: { localRoot: string; opts: Options; root: string }) { + const stateArchive = path.join(params.localRoot, "remote-state.tgz"); + const userDriverScript = expandHome(params.opts.userDriverScript); + if (!fs.existsSync(userDriverScript)) { + throw new Error(`Missing user driver script: ${params.opts.userDriverScript}`); + } + await runCommand({ + command: "cp", + args: [userDriverScript, path.join(params.localRoot, "user-driver.py")], + cwd: params.root, + }); + await runCommand({ + command: "tar", + args: [ + "-C", + params.localRoot, + "-czf", + stateArchive, + "user-driver", + "desktop", + "user-driver.py", + ], + cwd: params.root, + }); + return stateArchive; +} + +async function leaseCredential(params: { localRoot: string; opts: Options; root: string }) { + const userDriverDir = path.join(params.localRoot, "user-driver"); + const desktopWorkdir = path.join(params.localRoot, "desktop"); + const leaseFile = path.join(params.localRoot, "lease.json"); + const payloadFile = path.join(params.localRoot, "payload.json"); + const args = [ + "scripts/e2e/telegram-user-credential.ts", + "lease-restore", + "--user-driver-dir", + userDriverDir, + "--desktop-workdir", + desktopWorkdir, + "--lease-file", + leaseFile, + "--payload-output", + payloadFile, + ]; + if (params.opts.envFile) { + args.push("--env-file", params.opts.envFile); + } + const result = await runCommand({ + command: "node", + args: ["--import", "tsx", ...args], + cwd: params.root, + stdio: "inherit", + }); + const acquired = JSON.parse(result.stdout || "{}") as JsonObject; + const payload = readJsonFile(payloadFile); + return { + acquired, + desktopWorkdir, + groupId: requireString(payload, "groupId"), + leaseFile, + payloadFile, + sutToken: requireString(payload, "sutToken"), + testerUserId: requireString(payload, "testerUserId"), + testerUsername: requireString(payload, "testerUsername"), + userDriverDir, + }; +} + +async function releaseCredential(root: string, opts: Options, leaseFile: string) { + if (!fs.existsSync(leaseFile)) { + return; + } + const args = ["scripts/e2e/telegram-user-credential.ts", "release", "--lease-file", leaseFile]; + if (opts.envFile) { + args.push("--env-file", opts.envFile); + } + await runCommand({ + command: "node", + args: ["--import", "tsx", ...args], + cwd: root, + stdio: "inherit", + }); +} + +async function stopCrabbox(root: string, opts: Options, leaseId: string) { + await runCommand({ + command: opts.crabboxBin, + args: ["stop", "--provider", opts.provider, leaseId], + cwd: root, + stdio: "inherit", + }); +} + +function buildTargetText(text: string, sutUsername: string) { + if (!text.startsWith("/")) { + return text.replaceAll("{sut}", sutUsername); + } + if (/^\/\S+@\w+/u.test(text)) { + return text; + } + const [command, ...rest] = text.split(/\s+/u); + return [`${command}@${sutUsername}`, ...rest].join(" ").trim(); +} + +function summarizeProbe(probePath: string) { + const probe = readJsonFile(probePath); + const reply = probe.reply; + const sent = probe.sent; + return { + ok: probe.ok === true, + replyMessageId: reply && typeof reply === "object" && "id" in reply ? reply.id : undefined, + sentMessageId: sent && typeof sent === "object" && "id" in sent ? sent.id : undefined, + }; +} + +function writeReport(params: { + motionGifPath?: string; + motionVideoPath?: string; + outputDir: string; + screenshotPath?: string; + status: "pass" | "fail"; + summaryPath: string; + videoPath?: string; +}) { + const reportPath = path.join(params.outputDir, "telegram-user-crabbox-proof.md"); + fs.writeFileSync( + reportPath, + [ + "# Telegram User Crabbox Proof", + "", + `Status: ${params.status}`, + `Summary: ${path.basename(params.summaryPath)}`, + params.videoPath ? `Video: ${path.basename(params.videoPath)}` : "Video: missing", + params.motionVideoPath + ? `Motion video: ${path.basename(params.motionVideoPath)}` + : "Motion video: missing", + params.motionGifPath + ? `Motion GIF: ${path.basename(params.motionGifPath)}` + : "Motion GIF: missing", + params.screenshotPath + ? `Screenshot: ${path.basename(params.screenshotPath)}` + : "Screenshot: missing", + "", + ].join("\n"), + ); + return reportPath; +} + +async function main() { + const opts = parseArgs(process.argv.slice(2)); + const root = repoRoot(); + const outputDir = resolveRepoPath(root, opts.outputDir); + fs.mkdirSync(outputDir, { recursive: true }); + opts.outputDir = outputDir; + + const localRoot = fs.mkdtempSync(path.join(os.tmpdir(), "openclaw-telegram-crabbox-")); + const summary: JsonObject = { + artifacts: {}, + crabbox: { provider: opts.provider, target: opts.target }, + outputDir, + startedAt: new Date().toISOString(), + status: "fail", + }; + + let credential: Awaited> | undefined; + let leaseId = opts.leaseId; + let createdLease = false; + let localSut: LocalSut | undefined; + + try { + const convexEnvFile = expandHome(opts.envFile ?? DEFAULT_CONVEX_ENV_FILE); + const hasConvexEnv = + trimToValue(process.env.OPENCLAW_QA_CONVEX_SITE_URL) && + trimToValue(process.env.OPENCLAW_QA_CONVEX_SECRET_CI); + if (!hasConvexEnv && !fs.existsSync(convexEnvFile)) { + throw new Error(`Missing Convex env file: ${opts.envFile ?? DEFAULT_CONVEX_ENV_FILE}`); + } + await runCommand({ command: opts.crabboxBin, args: ["--version"], cwd: root }); + if (opts.dryRun) { + summary.status = "pass"; + summary.plan = { + command: "telegram-user-crabbox-proof", + outputDir, + provider: opts.provider, + target: opts.target, + tdlibSha256: opts.tdlibSha256, + tdlibUrl: opts.tdlibUrl, + text: opts.text, + }; + return; + } + + credential = await leaseCredential({ localRoot, opts, root }); + const sut = opts.sutUsername + ? { id: "", username: opts.sutUsername } + : await sutIdentity(credential.sutToken); + const targetText = buildTargetText(opts.text, sut.username); + summary.telegram = { + groupId: credential.groupId, + sutUsername: sut.username, + testerUserId: credential.testerUserId, + testerUsername: credential.testerUsername, + text: targetText, + }; + + const stateArchive = await prepareRemoteState({ + localRoot, + opts, + root, + }); + if (!leaseId) { + leaseId = await warmupCrabbox(opts, root); + createdLease = true; + } + summary.crabbox = { + createdLease, + id: leaseId, + provider: opts.provider, + target: opts.target, + }; + const inspect = await inspectCrabbox(opts, root, leaseId); + summary.crabbox = { + createdLease, + id: leaseId, + provider: opts.provider, + slug: inspect.slug, + state: inspect.state, + target: opts.target, + }; + + const setupScript = path.join(localRoot, "remote-setup.sh"); + const launchScript = path.join(localRoot, "launch-desktop.sh"); + const selectChatScript = path.join(localRoot, "select-desktop-chat.sh"); + const probeScript = path.join(localRoot, "remote-probe.sh"); + await writeExecutable( + setupScript, + renderRemoteSetup({ tdlibSha256: opts.tdlibSha256, tdlibUrl: opts.tdlibUrl }), + ); + await writeExecutable(launchScript, renderLaunchDesktop()); + await writeExecutable( + selectChatScript, + renderSelectDesktopChat({ chatTitle: opts.desktopChatTitle }), + ); + await writeExecutable( + probeScript, + renderRemoteProbe({ + expect: opts.expect, + sutUsername: sut.username, + text: targetText, + timeoutMs: opts.timeoutMs, + }), + ); + + await sshRun(root, inspect, `rm -rf ${REMOTE_ROOT} && mkdir -p ${REMOTE_ROOT}`); + await scpToRemote(root, inspect, stateArchive, `${REMOTE_ROOT}/state.tgz`); + await scpToRemote(root, inspect, setupScript, `${REMOTE_ROOT}/remote-setup.sh`); + await scpToRemote(root, inspect, launchScript, `${REMOTE_ROOT}/launch-desktop.sh`); + await scpToRemote(root, inspect, selectChatScript, `${REMOTE_ROOT}/select-desktop-chat.sh`); + await scpToRemote(root, inspect, probeScript, `${REMOTE_ROOT}/remote-probe.sh`); + await sshRun(root, inspect, `bash ${REMOTE_ROOT}/remote-setup.sh`); + + const sutRuntime = await startLocalSut({ + gatewayPort: opts.gatewayPort, + groupId: credential.groupId, + mockPort: opts.mockPort, + outputDir, + repoRoot: root, + sutToken: credential.sutToken, + testerId: credential.testerUserId, + }); + localSut = sutRuntime; + summary.localSut = { + drained: sutRuntime.drained, + gatewayPort: opts.gatewayPort, + mockPort: opts.mockPort, + requestLog: path.relative(root, sutRuntime.requestLog), + }; + + await sshRun(root, inspect, `bash ${REMOTE_ROOT}/launch-desktop.sh`); + await sshRun(root, inspect, `bash ${REMOTE_ROOT}/select-desktop-chat.sh`); + const videoPath = path.join(outputDir, "telegram-user-crabbox-proof.mp4"); + const recording = spawn( + opts.crabboxBin, + [ + "artifacts", + "video", + "--provider", + opts.provider, + "--target", + opts.target, + "--id", + leaseId, + "--duration", + `${opts.recordSeconds}s`, + "--output", + videoPath, + ], + { cwd: root, stdio: "inherit" }, + ); + await new Promise((resolve) => setTimeout(resolve, 3_000)); + await sshRun(root, inspect, `bash ${REMOTE_ROOT}/remote-probe.sh`); + const recordCode = await new Promise((resolve) => recording.on("exit", resolve)); + if (recordCode !== 0) { + throw new Error(`Crabbox recording failed with exit code ${recordCode ?? "unknown"}.`); + } + const motionVideoPath = path.join(outputDir, "telegram-user-crabbox-proof-motion.mp4"); + const motionGifPath = path.join(outputDir, "telegram-user-crabbox-proof-motion.gif"); + const preview = await runCommand({ + command: opts.crabboxBin, + args: [ + "media", + "preview", + "--input", + videoPath, + "--output", + motionGifPath, + "--trimmed-video-output", + motionVideoPath, + "--json", + ], + cwd: root, + stdio: "inherit", + }); + summary.mediaPreview = JSON.parse(preview.stdout) as JsonObject; + + const screenshotPath = path.join(outputDir, "telegram-user-crabbox-proof.png"); + await runCommand({ + command: opts.crabboxBin, + args: [ + "screenshot", + "--provider", + opts.provider, + "--target", + opts.target, + "--id", + leaseId, + "--output", + screenshotPath, + ], + cwd: root, + stdio: "inherit", + }); + const probePath = path.join(outputDir, "probe.json"); + const statusPath = path.join(outputDir, "status.json"); + const desktopLogPath = path.join(outputDir, "telegram-desktop.log"); + await scpFromRemote(root, inspect, `${REMOTE_ROOT}/probe.json`, probePath); + await scpFromRemote(root, inspect, `${REMOTE_ROOT}/status.json`, statusPath); + await scpFromRemote(root, inspect, `${REMOTE_ROOT}/telegram-desktop.log`, desktopLogPath); + summary.artifacts = { + desktopLog: path.relative(root, desktopLogPath), + probe: path.relative(root, probePath), + previewGif: path.relative(root, motionGifPath), + screenshot: path.relative(root, screenshotPath), + status: path.relative(root, statusPath), + trimmedVideo: path.relative(root, motionVideoPath), + video: path.relative(root, videoPath), + }; + summary.probe = summarizeProbe(probePath); + summary.status = "pass"; + } finally { + killTree(localSut?.gateway); + killTree(localSut?.mock); + if (credential) { + await releaseCredential(root, opts, credential.leaseFile).catch((error: unknown) => { + summary.credentialReleaseError = error instanceof Error ? error.message : String(error); + }); + } + if (leaseId && createdLease && !opts.keepBox) { + await stopCrabbox(root, opts, leaseId).catch((error: unknown) => { + summary.crabboxStopError = error instanceof Error ? error.message : String(error); + }); + } + if (opts.keepBox && leaseId) { + summary.keepBox = true; + summary.webvnc = `${opts.crabboxBin} webvnc --provider ${opts.provider} --target ${opts.target} --id ${leaseId} --open`; + } + summary.finishedAt = new Date().toISOString(); + const summaryPath = path.join(outputDir, "telegram-user-crabbox-proof-summary.json"); + fs.writeFileSync(summaryPath, `${JSON.stringify(summary, null, 2)}\n`); + const artifacts = summary.artifacts; + const screenshotPath = + artifacts && + typeof artifacts === "object" && + "screenshot" in artifacts && + typeof artifacts.screenshot === "string" + ? path.join(root, artifacts.screenshot) + : undefined; + const motionGifPath = + artifacts && + typeof artifacts === "object" && + "previewGif" in artifacts && + typeof artifacts.previewGif === "string" + ? path.join(root, artifacts.previewGif) + : undefined; + const motionVideoPath = + artifacts && + typeof artifacts === "object" && + "trimmedVideo" in artifacts && + typeof artifacts.trimmedVideo === "string" + ? path.join(root, artifacts.trimmedVideo) + : undefined; + const videoPath = + artifacts && + typeof artifacts === "object" && + "video" in artifacts && + typeof artifacts.video === "string" + ? path.join(root, artifacts.video) + : undefined; + const reportPath = writeReport({ + motionGifPath, + motionVideoPath, + outputDir, + screenshotPath, + status: summary.status === "pass" ? "pass" : "fail", + summaryPath, + videoPath, + }); + summary.report = path.relative(root, reportPath); + fs.writeFileSync(summaryPath, `${JSON.stringify(summary, null, 2)}\n`); + fs.rmSync(localRoot, { force: true, recursive: true }); + console.log(JSON.stringify({ outputDir, reportPath, status: summary.status }, null, 2)); + } + + if (summary.status !== "pass") { + process.exitCode = 1; + } +} + +main().catch((error: unknown) => { + console.error(error instanceof Error ? error.message : String(error)); + process.exit(1); +}); diff --git a/scripts/e2e/telegram-user-credential.ts b/scripts/e2e/telegram-user-credential.ts new file mode 100644 index 00000000000..601d4e30fd2 --- /dev/null +++ b/scripts/e2e/telegram-user-credential.ts @@ -0,0 +1,618 @@ +#!/usr/bin/env -S node --import tsx + +import { spawn } from "node:child_process"; +import { createHash } from "node:crypto"; +import { chmod, copyFile, mkdir, readFile, rm, unlink, writeFile } from "node:fs/promises"; + +type JsonObject = Record; + +const DEFAULT_USER_DRIVER_DIR = "~/.codex/skills/custom/telegram-e2e-bot-to-bot/user-driver"; +const DEFAULT_BOT_CREDENTIALS_FILE = + "~/.codex/skills/custom/telegram-e2e-bot-to-bot/credentials.local.json"; +const DEFAULT_CONVEX_ENV_FILE = "~/.codex/skills/custom/telegram-e2e-bot-to-bot/convex.local.env"; +const TELEGRAM_USER_KIND = "telegram-user"; +const CHUNKED_PAYLOAD_MARKER = "__openclawQaCredentialPayloadChunksV1"; + +function usage(): never { + throw new Error( + [ + "Usage:", + " node --import tsx scripts/e2e/telegram-user-credential.ts export (--desktop-tdata-dir | --desktop-tdata-archive ) --output ", + " node --import tsx scripts/e2e/telegram-user-credential.ts restore --payload-file --user-driver-dir --desktop-workdir ", + " node --import tsx scripts/e2e/telegram-user-credential.ts lease-restore --user-driver-dir --desktop-workdir --lease-file [--payload-output ] [--env-file ]", + " node --import tsx scripts/e2e/telegram-user-credential.ts release --lease-file [--env-file ]", + ].join("\n"), + ); +} + +function printUsage() { + console.log( + [ + "Usage:", + " node --import tsx scripts/e2e/telegram-user-credential.ts export (--desktop-tdata-dir | --desktop-tdata-archive ) --output ", + " node --import tsx scripts/e2e/telegram-user-credential.ts restore --payload-file --user-driver-dir --desktop-workdir ", + " node --import tsx scripts/e2e/telegram-user-credential.ts lease-restore --user-driver-dir --desktop-workdir --lease-file [--payload-output ] [--env-file ]", + " node --import tsx scripts/e2e/telegram-user-credential.ts release --lease-file [--env-file ]", + ].join("\n"), + ); +} + +function expandHome(path: string) { + if (path === "~") { + return process.env.HOME || path; + } + if (path.startsWith("~/")) { + return `${process.env.HOME || "~"}${path.slice(1)}`; + } + return path; +} + +function parseArgs(argv: string[]) { + const args = argv.slice(2); + const command = args[0] || usage(); + if (command === "--help" || command === "-h") { + printUsage(); + process.exit(0); + } + const opts = new Map(); + for (let index = 1; index < args.length; index += 1) { + if (args[index] === "--") { + continue; + } + const key = args[index]; + if (!key.startsWith("--")) { + usage(); + } + const value = args[index + 1]; + if (!value || value.startsWith("--")) { + usage(); + } + opts.set(key.slice(2), value); + index += 1; + } + return { command, opts }; +} + +async function readJson(path: string): Promise { + try { + return JSON.parse(await readFile(expandHome(path), "utf8")) as JsonObject; + } catch (error) { + if (error instanceof Error && "code" in error && error.code === "ENOENT") { + return {}; + } + throw error; + } +} + +function fileExists(path: string) { + return readFile(expandHome(path)) + .then(() => true) + .catch((error: unknown) => { + if (error instanceof Error && "code" in error && error.code === "ENOENT") { + return false; + } + throw error; + }); +} + +async function readEnvFile(path: string) { + if (!(await fileExists(path))) { + return {}; + } + const env: Record = {}; + const text = await readFile(expandHome(path), "utf8"); + for (const line of text.split(/\r?\n/u)) { + const trimmed = line.trim(); + if (!trimmed || trimmed.startsWith("#")) { + continue; + } + const separator = trimmed.indexOf("="); + if (separator < 1) { + throw new Error(`Invalid env line in ${path}.`); + } + const key = trimmed.slice(0, separator).trim(); + const value = trimmed + .slice(separator + 1) + .trim() + .replace(/^['"]|['"]$/gu, ""); + env[key] = value; + } + return env; +} + +function requireString(source: JsonObject, key: string) { + const value = source[key]; + if (typeof value === "number") { + return String(value); + } + if (typeof value === "string" && value.trim()) { + return value.trim(); + } + throw new Error(`Missing ${key}.`); +} + +function optionalString(source: JsonObject, key: string) { + const value = source[key]; + if (typeof value === "number") { + return String(value); + } + if (typeof value === "string" && value.trim()) { + return value.trim(); + } + return undefined; +} + +function optionalPositiveInteger(value: string | undefined, fallback: number) { + if (!value) { + return fallback; + } + const parsed = Number(value); + if (!Number.isInteger(parsed) || parsed < 1) { + throw new Error(`Expected positive integer, got ${value}.`); + } + return parsed; +} + +async function fileSha256(path: string) { + return createHash("sha256") + .update(await readFile(path)) + .digest("hex"); +} + +async function tgzBase64(path: string) { + return (await readFile(path)).toString("base64"); +} + +async function writePrivateJson(path: string, payload: JsonObject) { + const expanded = expandHome(path); + await mkdir(expanded.substring(0, expanded.lastIndexOf("/")), { recursive: true }); + await writeFile(expanded, `${JSON.stringify(payload, null, 2)}\n`, { mode: 0o600 }); + await chmodPrivate(expanded); +} + +async function chmodPrivate(path: string) { + await chmod(path, 0o600); +} + +function runCommand(command: string, args: string[], cwd?: string) { + return new Promise((resolve, reject) => { + const child = spawn(command, args, { + cwd, + stdio: ["ignore", "pipe", "pipe"], + }); + let stdout = ""; + let stderr = ""; + child.stdout.on("data", (chunk: Buffer) => { + stdout += chunk.toString(); + }); + child.stderr.on("data", (chunk: Buffer) => { + stderr += chunk.toString(); + }); + child.on("error", reject); + child.on("close", (code, signal) => { + if (code === 0) { + resolve(); + return; + } + const detail = signal ? `signal ${signal}` : `exit code ${code ?? "unknown"}`; + reject(new Error(`${command} ${args.join(" ")} failed with ${detail}\n${stdout}${stderr}`)); + }); + }); +} + +function joinBrokerEndpoint(siteUrl: string, endpoint: string) { + const normalized = siteUrl.replace(/\/+$/u, ""); + return `${normalized}/qa-credentials/v1/${endpoint}`; +} + +function assertBrokerSuccess(payload: JsonObject, action: string) { + if (payload.status === "error") { + throw new Error( + `${action} failed: ${requireString(payload, "code")} ${optionalString(payload, "message") || ""}`.trim(), + ); + } + if (payload.status !== "ok") { + throw new Error(`${action} returned an invalid response.`); + } +} + +async function postBroker(params: { + action: string; + body: JsonObject; + siteUrl: string; + token: string; +}) { + const response = await fetch(joinBrokerEndpoint(params.siteUrl, params.action), { + method: "POST", + headers: { + authorization: `Bearer ${params.token}`, + "content-type": "application/json", + }, + body: JSON.stringify(params.body), + }); + const payload = (await response.json()) as JsonObject; + if (!response.ok) { + assertBrokerSuccess(payload, params.action); + throw new Error(`${params.action} failed with HTTP ${response.status}.`); + } + assertBrokerSuccess(payload, params.action); + return payload; +} + +async function resolveConvexLeaseConfig(opts: Map) { + const envFile = opts.get("env-file") || DEFAULT_CONVEX_ENV_FILE; + const fileEnv = await readEnvFile(envFile); + const siteUrl = + opts.get("site-url") || + process.env.OPENCLAW_QA_CONVEX_SITE_URL?.trim() || + fileEnv.OPENCLAW_QA_CONVEX_SITE_URL; + const token = + opts.get("ci-secret") || + process.env.OPENCLAW_QA_CONVEX_SECRET_CI?.trim() || + fileEnv.OPENCLAW_QA_CONVEX_SECRET_CI; + if (!siteUrl) { + throw new Error("Missing OPENCLAW_QA_CONVEX_SITE_URL."); + } + if (!token) { + throw new Error("Missing OPENCLAW_QA_CONVEX_SECRET_CI."); + } + return { + siteUrl, + token, + leaseTtlMs: optionalPositiveInteger( + opts.get("lease-ttl-ms") || + process.env.OPENCLAW_QA_CREDENTIAL_LEASE_TTL_MS?.trim() || + fileEnv.OPENCLAW_QA_CREDENTIAL_LEASE_TTL_MS, + 20 * 60 * 1_000, + ), + heartbeatIntervalMs: optionalPositiveInteger( + opts.get("heartbeat-interval-ms") || + process.env.OPENCLAW_QA_CREDENTIAL_HEARTBEAT_INTERVAL_MS?.trim() || + fileEnv.OPENCLAW_QA_CREDENTIAL_HEARTBEAT_INTERVAL_MS, + 30_000, + ), + ownerId: + opts.get("owner-id") || + process.env.OPENCLAW_QA_CREDENTIAL_OWNER_ID?.trim() || + `telegram-user-${Date.now()}-${Math.random().toString(16).slice(2, 10)}`, + }; +} + +function parseChunkedPayloadMarker(payload: unknown) { + if (!payload || typeof payload !== "object" || Array.isArray(payload)) { + return null; + } + const record = payload as Record; + if (record[CHUNKED_PAYLOAD_MARKER] !== true) { + return null; + } + if ( + typeof record.chunkCount !== "number" || + !Number.isInteger(record.chunkCount) || + record.chunkCount < 1 + ) { + throw new Error("Chunked payload marker has invalid chunkCount."); + } + if ( + typeof record.byteLength !== "number" || + !Number.isInteger(record.byteLength) || + record.byteLength < 0 + ) { + throw new Error("Chunked payload marker has invalid byteLength."); + } + return { + chunkCount: record.chunkCount, + byteLength: record.byteLength, + }; +} + +async function hydratePayloadFromLease(params: { + acquired: JsonObject; + ownerId: string; + siteUrl: string; + token: string; +}) { + const marker = parseChunkedPayloadMarker(params.acquired.payload); + if (!marker) { + return params.acquired.payload as JsonObject; + } + const credentialId = requireString(params.acquired, "credentialId"); + const leaseToken = requireString(params.acquired, "leaseToken"); + const chunks: string[] = []; + for (let index = 0; index < marker.chunkCount; index += 1) { + const chunk = await postBroker({ + action: "payload-chunk", + siteUrl: params.siteUrl, + token: params.token, + body: { + kind: TELEGRAM_USER_KIND, + ownerId: params.ownerId, + actorRole: "ci", + credentialId, + leaseToken, + index, + }, + }); + chunks.push(requireString(chunk, "data")); + } + const serialized = chunks.join(""); + if (serialized.length !== marker.byteLength) { + throw new Error("Chunked payload length mismatch."); + } + return JSON.parse(serialized) as JsonObject; +} + +async function createTelegramUserPayload(opts: Map) { + const userDriverDir = expandHome(opts.get("user-driver-dir") || DEFAULT_USER_DRIVER_DIR); + const botCredentialsFile = expandHome( + opts.get("bot-credentials-file") || DEFAULT_BOT_CREDENTIALS_FILE, + ); + const desktopTdataDir = opts.get("desktop-tdata-dir"); + const desktopTdataArchiveInput = opts.get("desktop-tdata-archive"); + const output = opts.get("output"); + if ( + (!desktopTdataDir && !desktopTdataArchiveInput) || + (desktopTdataDir && desktopTdataArchiveInput) || + !output + ) { + usage(); + } + + const config = await readJson(`${userDriverDir}/config.local.json`); + const botCredentials = await readJson(botCredentialsFile); + const sutToken = + process.env.OPENCLAW_QA_TELEGRAM_SUT_BOT_TOKEN?.trim() || + process.env.TELEGRAM_E2E_SUT_BOT_TOKEN?.trim() || + (typeof botCredentials.sutBotToken === "string" ? botCredentials.sutBotToken.trim() : "") || + (typeof botCredentials.botAToken === "string" ? botCredentials.botAToken.trim() : "") || + (typeof botCredentials.BOTA === "string" ? botCredentials.BOTA.trim() : ""); + if (!sutToken) { + throw new Error("Missing SUT token in env or bot credentials file."); + } + + const groupId = + process.env.OPENCLAW_QA_TELEGRAM_GROUP_ID?.trim() || + process.env.TELEGRAM_E2E_GROUP_ID?.trim() || + (typeof config.defaultChatId === "string" ? config.defaultChatId.trim() : "") || + (typeof botCredentials.groupId === "string" ? botCredentials.groupId.trim() : ""); + if (!groupId) { + throw new Error("Missing group id in env, user-driver config, or bot credentials file."); + } + + const tempRoot = `/tmp/openclaw-telegram-user-credential-${Date.now()}-${Math.random() + .toString(16) + .slice(2)}`; + const tdlibArchive = `${tempRoot}/tdlib.tgz`; + const desktopArchive = `${tempRoot}/desktop-tdata.tgz`; + await mkdir(tempRoot, { recursive: true }); + try { + await runCommand("tar", ["-C", userDriverDir, "-czf", tdlibArchive, "db", "files"]); + if (desktopTdataArchiveInput) { + await copyFile(expandHome(desktopTdataArchiveInput), desktopArchive); + } else { + await runCommand("tar", [ + "-C", + `${expandHome(desktopTdataDir!)}/..`, + "--exclude", + "tdata/countries", + "--exclude", + "tdata/dictionaries", + "--exclude", + "tdata/dumps", + "--exclude", + "tdata/emoji", + "--exclude", + "tdata/user_data", + "--exclude", + "tdata/working", + "-czf", + desktopArchive, + "tdata", + ]); + } + + await writePrivateJson(output, { + groupId, + sutToken, + testerUserId: requireString(config, "testerUserId"), + testerUsername: requireString(config, "testerUsername"), + telegramApiId: requireString(config, "apiId"), + telegramApiHash: requireString(config, "apiHash"), + tdlibDatabaseEncryptionKey: requireString(config, "databaseEncryptionKey"), + tdlibArchiveBase64: await tgzBase64(tdlibArchive), + tdlibArchiveSha256: await fileSha256(tdlibArchive), + desktopTdataArchiveBase64: await tgzBase64(desktopArchive), + desktopTdataArchiveSha256: await fileSha256(desktopArchive), + }); + } finally { + await rm(tempRoot, { force: true, recursive: true }); + } +} + +async function restoreTelegramUserPayloadFromFile(opts: Map) { + const payloadFile = opts.get("payload-file"); + if (!payloadFile) { + usage(); + } + await restoreTelegramUserPayload({ + payload: await readJson(payloadFile), + userDriverDir: opts.get("user-driver-dir"), + desktopWorkdir: opts.get("desktop-workdir"), + }); +} + +async function restoreTelegramUserPayload(params: { + payload: JsonObject; + userDriverDir: string | undefined; + desktopWorkdir: string | undefined; +}) { + const userDriverDir = params.userDriverDir; + const desktopWorkdir = params.desktopWorkdir; + if (!userDriverDir || !desktopWorkdir) { + usage(); + } + const payload = params.payload; + const tempRoot = `/tmp/openclaw-telegram-user-restore-${Date.now()}-${Math.random() + .toString(16) + .slice(2)}`; + const tdlibArchive = `${tempRoot}/tdlib.tgz`; + const desktopArchive = `${tempRoot}/desktop-tdata.tgz`; + await mkdir(tempRoot, { recursive: true }); + await mkdir(expandHome(userDriverDir), { recursive: true }); + await mkdir(expandHome(desktopWorkdir), { recursive: true }); + try { + await writeFile( + tdlibArchive, + Buffer.from(requireString(payload, "tdlibArchiveBase64"), "base64"), + ); + await writeFile( + desktopArchive, + Buffer.from(requireString(payload, "desktopTdataArchiveBase64"), "base64"), + ); + if ((await fileSha256(tdlibArchive)) !== requireString(payload, "tdlibArchiveSha256")) { + throw new Error("TDLib archive SHA-256 mismatch."); + } + if ( + (await fileSha256(desktopArchive)) !== requireString(payload, "desktopTdataArchiveSha256") + ) { + throw new Error("Telegram Desktop archive SHA-256 mismatch."); + } + + await runCommand("tar", ["-C", expandHome(userDriverDir), "-xzf", tdlibArchive]); + await runCommand("tar", ["-C", expandHome(desktopWorkdir), "-xzf", desktopArchive]); + await writePrivateJson(`${expandHome(userDriverDir)}/config.local.json`, { + apiId: Number(requireString(payload, "telegramApiId")), + apiHash: requireString(payload, "telegramApiHash"), + databaseEncryptionKey: requireString(payload, "tdlibDatabaseEncryptionKey"), + defaultChatId: requireString(payload, "groupId"), + testerUserId: Number(requireString(payload, "testerUserId")), + testerUsername: requireString(payload, "testerUsername"), + }); + } finally { + await rm(tempRoot, { force: true, recursive: true }); + } +} + +async function leaseAndRestoreTelegramUser(opts: Map) { + const userDriverDir = opts.get("user-driver-dir"); + const desktopWorkdir = opts.get("desktop-workdir"); + const leaseFile = opts.get("lease-file"); + const payloadOutput = opts.get("payload-output"); + if (!userDriverDir || !desktopWorkdir || !leaseFile) { + usage(); + } + const config = await resolveConvexLeaseConfig(opts); + const acquired = await postBroker({ + action: "acquire", + siteUrl: config.siteUrl, + token: config.token, + body: { + kind: TELEGRAM_USER_KIND, + ownerId: config.ownerId, + actorRole: "ci", + leaseTtlMs: config.leaseTtlMs, + heartbeatIntervalMs: config.heartbeatIntervalMs, + }, + }); + const lease = { + siteUrl: config.siteUrl, + kind: TELEGRAM_USER_KIND, + ownerId: config.ownerId, + actorRole: "ci", + credentialId: requireString(acquired, "credentialId"), + leaseToken: requireString(acquired, "leaseToken"), + }; + + try { + const payload = await hydratePayloadFromLease({ + acquired, + siteUrl: config.siteUrl, + token: config.token, + ownerId: config.ownerId, + }); + await restoreTelegramUserPayload({ payload, userDriverDir, desktopWorkdir }); + await writePrivateJson(leaseFile, lease); + if (payloadOutput) { + await writePrivateJson(payloadOutput, payload); + } + console.log( + JSON.stringify( + { + status: "ok", + credentialId: lease.credentialId, + ownerId: lease.ownerId, + leaseFile, + userDriverDir, + desktopWorkdir, + testerUserId: requireString(payload, "testerUserId"), + testerUsername: requireString(payload, "testerUsername"), + groupId: requireString(payload, "groupId"), + }, + null, + 2, + ), + ); + } catch (error) { + await releaseTelegramUserLeaseBody({ + siteUrl: lease.siteUrl, + token: config.token, + lease, + }); + throw error; + } +} + +async function releaseTelegramUserLeaseBody(params: { + siteUrl: string; + token: string; + lease: JsonObject; +}) { + return postBroker({ + action: "release", + siteUrl: params.siteUrl, + token: params.token, + body: { + kind: requireString(params.lease, "kind"), + ownerId: requireString(params.lease, "ownerId"), + actorRole: requireString(params.lease, "actorRole"), + credentialId: requireString(params.lease, "credentialId"), + leaseToken: requireString(params.lease, "leaseToken"), + }, + }); +} + +async function releaseTelegramUserLease(opts: Map) { + const leaseFile = opts.get("lease-file"); + if (!leaseFile) { + usage(); + } + const config = await resolveConvexLeaseConfig(opts); + const lease = await readJson(leaseFile); + await releaseTelegramUserLeaseBody({ + siteUrl: config.siteUrl, + token: config.token, + lease, + }); + await unlink(expandHome(leaseFile)).catch((error: unknown) => { + if (error instanceof Error && "code" in error && error.code === "ENOENT") { + return; + } + throw error; + }); + console.log( + JSON.stringify({ status: "ok", credentialId: requireString(lease, "credentialId") }, null, 2), + ); +} + +const { command, opts } = parseArgs(process.argv); +if (command === "export") { + await createTelegramUserPayload(opts); +} else if (command === "restore") { + await restoreTelegramUserPayloadFromFile(opts); +} else if (command === "lease-restore") { + await leaseAndRestoreTelegramUser(opts); +} else if (command === "release") { + await releaseTelegramUserLease(opts); +} else { + usage(); +}