From 57334cd7d85174d5f951de01114fd5801b063564 Mon Sep 17 00:00:00 2001 From: Peter Steinberger Date: Thu, 26 Feb 2026 16:43:44 +0100 Subject: [PATCH] refactor: unify channel/plugin ssrf fetch policy and auth fallback --- extensions/msteams/src/attachments.test.ts | 107 ++++--- .../msteams/src/attachments/download.ts | 85 +---- extensions/msteams/src/attachments/graph.ts | 146 +++++---- .../msteams/src/attachments/remote-media.ts | 3 + .../msteams/src/attachments/shared.test.ts | 295 ++---------------- extensions/msteams/src/attachments/shared.ts | 153 +-------- package.json | 3 +- scripts/check-no-raw-channel-fetch.mjs | 211 +++++++++++++ src/plugin-sdk/fetch-auth.test.ts | 94 ++++++ src/plugin-sdk/fetch-auth.ts | 71 +++++ src/plugin-sdk/index.ts | 7 + src/plugin-sdk/ssrf-policy.test.ts | 84 +++++ src/plugin-sdk/ssrf-policy.ts | 85 +++++ 13 files changed, 749 insertions(+), 595 deletions(-) create mode 100644 scripts/check-no-raw-channel-fetch.mjs create mode 100644 src/plugin-sdk/fetch-auth.test.ts create mode 100644 src/plugin-sdk/fetch-auth.ts create mode 100644 src/plugin-sdk/ssrf-policy.test.ts create mode 100644 src/plugin-sdk/ssrf-policy.ts diff --git a/extensions/msteams/src/attachments.test.ts b/extensions/msteams/src/attachments.test.ts index b67289aea9d..167075d1c6e 100644 --- a/extensions/msteams/src/attachments.test.ts +++ b/extensions/msteams/src/attachments.test.ts @@ -1,4 +1,4 @@ -import type { PluginRuntime } from "openclaw/plugin-sdk"; +import type { PluginRuntime, SsrFPolicy } from "openclaw/plugin-sdk"; import { beforeEach, describe, expect, it, vi } from "vitest"; import { buildMSTeamsAttachmentPlaceholder, @@ -9,16 +9,6 @@ import { } from "./attachments.js"; import { setMSTeamsRuntime } from "./runtime.js"; -vi.mock("openclaw/plugin-sdk", async (importOriginal) => { - const actual = await importOriginal(); - return { - ...actual, - isPrivateIpAddress: () => false, - }; -}); - -/** Mock DNS resolver that always returns a public IP (for anti-SSRF validation in tests). */ -const publicResolveFn = async () => ({ address: "13.107.136.10" }); const GRAPH_HOST = "graph.microsoft.com"; const SHAREPOINT_HOST = "contoso.sharepoint.com"; const AZUREEDGE_HOST = "azureedge.net"; @@ -50,6 +40,7 @@ type RemoteMediaFetchParams = { url: string; maxBytes?: number; filePathHint?: string; + ssrfPolicy?: SsrFPolicy; fetchImpl?: (input: RequestInfo | URL, init?: RequestInit) => Promise; }; @@ -75,10 +66,44 @@ const readRemoteMediaResponse = async ( fileName: params.filePathHint, }; }; + +function isHostnameAllowedByPattern(hostname: string, pattern: string): boolean { + if (pattern.startsWith("*.")) { + const suffix = pattern.slice(2); + return suffix.length > 0 && hostname !== suffix && hostname.endsWith(`.${suffix}`); + } + return hostname === pattern; +} + +function isUrlAllowedBySsrfPolicy(url: string, policy?: SsrFPolicy): boolean { + if (!policy?.hostnameAllowlist || policy.hostnameAllowlist.length === 0) { + return true; + } + const hostname = new URL(url).hostname.toLowerCase(); + return policy.hostnameAllowlist.some((pattern) => + isHostnameAllowedByPattern(hostname, pattern.toLowerCase()), + ); +} + const fetchRemoteMediaMock = vi.fn(async (params: RemoteMediaFetchParams) => { const fetchFn = params.fetchImpl ?? fetch; - const res = await fetchFn(params.url); - return readRemoteMediaResponse(res, params); + let currentUrl = params.url; + for (let i = 0; i <= MAX_REDIRECT_HOPS; i += 1) { + if (!isUrlAllowedBySsrfPolicy(currentUrl, params.ssrfPolicy)) { + throw new Error(`Blocked hostname (not in allowlist): ${currentUrl}`); + } + const res = await fetchFn(currentUrl, { redirect: "manual" }); + if (REDIRECT_STATUS_CODES.includes(res.status)) { + const location = res.headers.get("location"); + if (!location) { + throw new Error("redirect missing location"); + } + currentUrl = new URL(location, currentUrl).toString(); + continue; + } + return readRemoteMediaResponse(res, params); + } + throw new Error("too many redirects"); }); const runtimeStub = { @@ -100,16 +125,13 @@ type DownloadGraphMediaParams = Parameters[0]; type DownloadedMedia = Awaited>; type MSTeamsMediaPayload = ReturnType; type DownloadAttachmentsBuildOverrides = Partial< - Omit + Omit > & - Pick; + Pick; type DownloadAttachmentsNoFetchOverrides = Partial< - Omit< - DownloadAttachmentsParams, - "attachments" | "maxBytes" | "allowHosts" | "resolveFn" | "fetchFn" - > + Omit > & - Pick; + Pick; type DownloadGraphMediaOverrides = Partial< Omit >; @@ -210,7 +232,6 @@ const buildDownloadParams = ( attachments, maxBytes: DEFAULT_MAX_BYTES, allowHosts: DEFAULT_ALLOW_HOSTS, - resolveFn: publicResolveFn, ...overrides, }; }; @@ -680,13 +701,37 @@ describe("msteams attachments", () => { fetchMock, { allowHosts: [GRAPH_HOST], - resolveFn: undefined, }, { expectFetchCalled: false }, ); expectAttachmentMediaLength(media, 0); }); + + it("blocks redirects to non-https URLs", async () => { + const insecureUrl = "http://x/insecure.png"; + const fetchMock = vi.fn(async (input: RequestInfo | URL) => { + const url = typeof input === "string" ? input : input.toString(); + if (url === TEST_URL_IMAGE) { + return createRedirectResponse(insecureUrl); + } + if (url === insecureUrl) { + return createBufferResponse("insecure", CONTENT_TYPE_IMAGE_PNG); + } + return createNotFoundResponse(); + }); + + const media = await downloadAttachmentsWithFetch( + createImageAttachments(TEST_URL_IMAGE), + fetchMock, + { + allowHosts: [TEST_HOST], + }, + ); + + expectAttachmentMediaLength(media, 0); + expect(fetchMock).toHaveBeenCalledTimes(1); + }); }); describe("buildMSTeamsGraphMessageUrls", () => { @@ -701,24 +746,6 @@ describe("msteams attachments", () => { it("blocks SharePoint redirects to hosts outside allowHosts", async () => { const escapedUrl = "https://evil.example/internal.pdf"; - fetchRemoteMediaMock.mockImplementationOnce(async (params) => { - const fetchFn = params.fetchImpl ?? fetch; - let currentUrl = params.url; - for (let i = 0; i < MAX_REDIRECT_HOPS; i += 1) { - const res = await fetchFn(currentUrl, { redirect: "manual" }); - if (REDIRECT_STATUS_CODES.includes(res.status)) { - const location = res.headers.get("location"); - if (!location) { - throw new Error("redirect missing location"); - } - currentUrl = new URL(location, currentUrl).toString(); - continue; - } - return readRemoteMediaResponse(res, params); - } - throw new Error("too many redirects"); - }); - const { fetchMock, media } = await downloadGraphMediaWithMockOptions( { ...buildDefaultShareReferenceGraphFetchOptions({ diff --git a/extensions/msteams/src/attachments/download.ts b/extensions/msteams/src/attachments/download.ts index bb3c5867205..f6f16ff803e 100644 --- a/extensions/msteams/src/attachments/download.ts +++ b/extensions/msteams/src/attachments/download.ts @@ -1,3 +1,4 @@ +import { fetchWithBearerAuthScopeFallback } from "openclaw/plugin-sdk"; import { getMSTeamsRuntime } from "../runtime.js"; import { downloadAndStoreMSTeamsRemoteMedia } from "./remote-media.js"; import { @@ -7,10 +8,10 @@ import { isRecord, isUrlAllowed, normalizeContentType, + resolveMediaSsrfPolicy, resolveRequestUrl, resolveAuthAllowedHosts, resolveAllowedHosts, - safeFetch, } from "./shared.js"; import type { MSTeamsAccessTokenProvider, @@ -90,81 +91,17 @@ async function fetchWithAuthFallback(params: { tokenProvider?: MSTeamsAccessTokenProvider; fetchFn?: typeof fetch; requestInit?: RequestInit; - allowHosts: string[]; authAllowHosts: string[]; - resolveFn?: (hostname: string) => Promise<{ address: string }>; }): Promise { - const fetchFn = params.fetchFn ?? fetch; - - // Use safeFetch for the initial attempt — redirect: "manual" with - // allowlist + DNS/IP validation on every hop (prevents SSRF via redirect). - const firstAttempt = await safeFetch({ + return await fetchWithBearerAuthScopeFallback({ url: params.url, - allowHosts: params.allowHosts, - fetchFn, + scopes: scopeCandidatesForUrl(params.url), + tokenProvider: params.tokenProvider, + fetchFn: params.fetchFn, requestInit: params.requestInit, - resolveFn: params.resolveFn, + requireHttps: true, + shouldAttachAuth: (url) => isUrlAllowed(url, params.authAllowHosts), }); - if (firstAttempt.ok) { - return firstAttempt; - } - if (!params.tokenProvider) { - return firstAttempt; - } - if (firstAttempt.status !== 401 && firstAttempt.status !== 403) { - return firstAttempt; - } - if (!isUrlAllowed(params.url, params.authAllowHosts)) { - return firstAttempt; - } - - const scopes = scopeCandidatesForUrl(params.url); - for (const scope of scopes) { - try { - const token = await params.tokenProvider.getAccessToken(scope); - const authHeaders = new Headers(params.requestInit?.headers); - authHeaders.set("Authorization", `Bearer ${token}`); - const authAttempt = await safeFetch({ - url: params.url, - allowHosts: params.allowHosts, - fetchFn, - requestInit: { - ...params.requestInit, - headers: authHeaders, - }, - resolveFn: params.resolveFn, - }); - if (authAttempt.ok) { - return authAttempt; - } - if (authAttempt.status !== 401 && authAttempt.status !== 403) { - continue; - } - - const finalUrl = - typeof authAttempt.url === "string" && authAttempt.url ? authAttempt.url : ""; - if (!finalUrl || finalUrl === params.url || !isUrlAllowed(finalUrl, params.authAllowHosts)) { - continue; - } - const redirectedAuthAttempt = await safeFetch({ - url: finalUrl, - allowHosts: params.allowHosts, - fetchFn, - requestInit: { - ...params.requestInit, - headers: authHeaders, - }, - resolveFn: params.resolveFn, - }); - if (redirectedAuthAttempt.ok) { - return redirectedAuthAttempt; - } - } catch { - // Try the next scope. - } - } - - return firstAttempt; } /** @@ -180,8 +117,6 @@ export async function downloadMSTeamsAttachments(params: { fetchFn?: typeof fetch; /** When true, embeds original filename in stored path for later extraction. */ preserveFilenames?: boolean; - /** Override DNS resolver for testing (anti-SSRF IP validation). */ - resolveFn?: (hostname: string) => Promise<{ address: string }>; }): Promise { const list = Array.isArray(params.attachments) ? params.attachments : []; if (list.length === 0) { @@ -189,6 +124,7 @@ export async function downloadMSTeamsAttachments(params: { } const allowHosts = resolveAllowedHosts(params.allowHosts); const authAllowHosts = resolveAuthAllowedHosts(params.authAllowHosts); + const ssrfPolicy = resolveMediaSsrfPolicy(allowHosts); // Download ANY downloadable attachment (not just images) const downloadable = list.filter(isDownloadableAttachment); @@ -257,15 +193,14 @@ export async function downloadMSTeamsAttachments(params: { contentTypeHint: candidate.contentTypeHint, placeholder: candidate.placeholder, preserveFilenames: params.preserveFilenames, + ssrfPolicy, fetchImpl: (input, init) => fetchWithAuthFallback({ url: resolveRequestUrl(input), tokenProvider: params.tokenProvider, fetchFn: params.fetchFn, requestInit: init, - allowHosts, authAllowHosts, - resolveFn: params.resolveFn, }), }); out.push(media); diff --git a/extensions/msteams/src/attachments/graph.ts b/extensions/msteams/src/attachments/graph.ts index 8ae4b3f424b..1097d0caeb1 100644 --- a/extensions/msteams/src/attachments/graph.ts +++ b/extensions/msteams/src/attachments/graph.ts @@ -1,3 +1,4 @@ +import { fetchWithSsrFGuard, type SsrFPolicy } from "openclaw/plugin-sdk"; import { getMSTeamsRuntime } from "../runtime.js"; import { downloadMSTeamsAttachments } from "./download.js"; import { downloadAndStoreMSTeamsRemoteMedia } from "./remote-media.js"; @@ -7,9 +8,9 @@ import { isRecord, isUrlAllowed, normalizeContentType, + resolveMediaSsrfPolicy, resolveRequestUrl, resolveAllowedHosts, - safeFetch, } from "./shared.js"; import type { MSTeamsAccessTokenProvider, @@ -119,20 +120,31 @@ async function fetchGraphCollection(params: { url: string; accessToken: string; fetchFn?: typeof fetch; + ssrfPolicy?: SsrFPolicy; }): Promise<{ status: number; items: T[] }> { const fetchFn = params.fetchFn ?? fetch; - const res = await fetchFn(params.url, { - headers: { Authorization: `Bearer ${params.accessToken}` }, + const { response, release } = await fetchWithSsrFGuard({ + url: params.url, + fetchImpl: fetchFn, + init: { + headers: { Authorization: `Bearer ${params.accessToken}` }, + }, + policy: params.ssrfPolicy, + auditContext: "msteams.graph.collection", }); - const status = res.status; - if (!res.ok) { - return { status, items: [] }; - } try { - const data = (await res.json()) as { value?: T[] }; - return { status, items: Array.isArray(data.value) ? data.value : [] }; - } catch { - return { status, items: [] }; + const status = response.status; + if (!response.ok) { + return { status, items: [] }; + } + try { + const data = (await response.json()) as { value?: T[] }; + return { status, items: Array.isArray(data.value) ? data.value : [] }; + } catch { + return { status, items: [] }; + } + } finally { + await release(); } } @@ -164,11 +176,13 @@ async function downloadGraphHostedContent(params: { maxBytes: number; fetchFn?: typeof fetch; preserveFilenames?: boolean; + ssrfPolicy?: SsrFPolicy; }): Promise<{ media: MSTeamsInboundMedia[]; status: number; count: number }> { const hosted = await fetchGraphCollection({ url: `${params.messageUrl}/hostedContents`, accessToken: params.accessToken, fetchFn: params.fetchFn, + ssrfPolicy: params.ssrfPolicy, }); if (hosted.items.length === 0) { return { media: [], status: hosted.status, count: 0 }; @@ -228,6 +242,7 @@ export async function downloadMSTeamsGraphMedia(params: { return { media: [] }; } const allowHosts = resolveAllowedHosts(params.allowHosts); + const ssrfPolicy = resolveMediaSsrfPolicy(allowHosts); const messageUrl = params.messageUrl; let accessToken: string; try { @@ -241,64 +256,67 @@ export async function downloadMSTeamsGraphMedia(params: { const sharePointMedia: MSTeamsInboundMedia[] = []; const downloadedReferenceUrls = new Set(); try { - const msgRes = await fetchFn(messageUrl, { - headers: { Authorization: `Bearer ${accessToken}` }, + const { response: msgRes, release } = await fetchWithSsrFGuard({ + url: messageUrl, + fetchImpl: fetchFn, + init: { + headers: { Authorization: `Bearer ${accessToken}` }, + }, + policy: ssrfPolicy, + auditContext: "msteams.graph.message", }); - if (msgRes.ok) { - const msgData = (await msgRes.json()) as { - body?: { content?: string; contentType?: string }; - attachments?: Array<{ - id?: string; - contentUrl?: string; - contentType?: string; - name?: string; - }>; - }; + try { + if (msgRes.ok) { + const msgData = (await msgRes.json()) as { + body?: { content?: string; contentType?: string }; + attachments?: Array<{ + id?: string; + contentUrl?: string; + contentType?: string; + name?: string; + }>; + }; - // Extract SharePoint file attachments (contentType: "reference") - // Download any file type, not just images - const spAttachments = (msgData.attachments ?? []).filter( - (a) => a.contentType === "reference" && a.contentUrl && a.name, - ); - for (const att of spAttachments) { - const name = att.name ?? "file"; + // Extract SharePoint file attachments (contentType: "reference") + // Download any file type, not just images + const spAttachments = (msgData.attachments ?? []).filter( + (a) => a.contentType === "reference" && a.contentUrl && a.name, + ); + for (const att of spAttachments) { + const name = att.name ?? "file"; - try { - // SharePoint URLs need to be accessed via Graph shares API - const shareUrl = att.contentUrl!; - if (!isUrlAllowed(shareUrl, allowHosts)) { - continue; + try { + // SharePoint URLs need to be accessed via Graph shares API + const shareUrl = att.contentUrl!; + if (!isUrlAllowed(shareUrl, allowHosts)) { + continue; + } + const encodedUrl = Buffer.from(shareUrl).toString("base64url"); + const sharesUrl = `${GRAPH_ROOT}/shares/u!${encodedUrl}/driveItem/content`; + + const media = await downloadAndStoreMSTeamsRemoteMedia({ + url: sharesUrl, + filePathHint: name, + maxBytes: params.maxBytes, + contentTypeHint: "application/octet-stream", + preserveFilenames: params.preserveFilenames, + ssrfPolicy, + fetchImpl: async (input, init) => { + const requestUrl = resolveRequestUrl(input); + const headers = new Headers(init?.headers); + headers.set("Authorization", `Bearer ${accessToken}`); + return await fetchFn(requestUrl, { ...init, headers }); + }, + }); + sharePointMedia.push(media); + downloadedReferenceUrls.add(shareUrl); + } catch { + // Ignore SharePoint download failures. } - const encodedUrl = Buffer.from(shareUrl).toString("base64url"); - const sharesUrl = `${GRAPH_ROOT}/shares/u!${encodedUrl}/driveItem/content`; - - const media = await downloadAndStoreMSTeamsRemoteMedia({ - url: sharesUrl, - filePathHint: name, - maxBytes: params.maxBytes, - contentTypeHint: "application/octet-stream", - preserveFilenames: params.preserveFilenames, - fetchImpl: async (input, init) => { - const requestUrl = resolveRequestUrl(input); - const headers = new Headers(init?.headers); - headers.set("Authorization", `Bearer ${accessToken}`); - return await safeFetch({ - url: requestUrl, - allowHosts, - fetchFn, - requestInit: { - ...init, - headers, - }, - }); - }, - }); - sharePointMedia.push(media); - downloadedReferenceUrls.add(shareUrl); - } catch { - // Ignore SharePoint download failures. } } + } finally { + await release(); } } catch { // Ignore message fetch failures. @@ -310,12 +328,14 @@ export async function downloadMSTeamsGraphMedia(params: { maxBytes: params.maxBytes, fetchFn: params.fetchFn, preserveFilenames: params.preserveFilenames, + ssrfPolicy, }); const attachments = await fetchGraphCollection({ url: `${messageUrl}/attachments`, accessToken, fetchFn: params.fetchFn, + ssrfPolicy, }); const normalizedAttachments = attachments.items.map(normalizeGraphAttachment); diff --git a/extensions/msteams/src/attachments/remote-media.ts b/extensions/msteams/src/attachments/remote-media.ts index 20842b2b5a0..162a797b57f 100644 --- a/extensions/msteams/src/attachments/remote-media.ts +++ b/extensions/msteams/src/attachments/remote-media.ts @@ -1,3 +1,4 @@ +import type { SsrFPolicy } from "openclaw/plugin-sdk"; import { getMSTeamsRuntime } from "../runtime.js"; import { inferPlaceholder } from "./shared.js"; import type { MSTeamsInboundMedia } from "./types.js"; @@ -9,6 +10,7 @@ export async function downloadAndStoreMSTeamsRemoteMedia(params: { filePathHint: string; maxBytes: number; fetchImpl?: FetchLike; + ssrfPolicy?: SsrFPolicy; contentTypeHint?: string; placeholder?: string; preserveFilenames?: boolean; @@ -18,6 +20,7 @@ export async function downloadAndStoreMSTeamsRemoteMedia(params: { fetchImpl: params.fetchImpl, filePathHint: params.filePathHint, maxBytes: params.maxBytes, + ssrfPolicy: params.ssrfPolicy, }); const mime = await getMSTeamsRuntime().media.detectMime({ buffer: fetched.buffer, diff --git a/extensions/msteams/src/attachments/shared.test.ts b/extensions/msteams/src/attachments/shared.test.ts index 9df64c51ab4..a5d0a4bef5a 100644 --- a/extensions/msteams/src/attachments/shared.test.ts +++ b/extensions/msteams/src/attachments/shared.test.ts @@ -1,281 +1,28 @@ -import { describe, expect, it, vi } from "vitest"; -import { isPrivateOrReservedIP, resolveAndValidateIP, safeFetch } from "./shared.js"; +import { describe, expect, it } from "vitest"; +import { + isUrlAllowed, + resolveAllowedHosts, + resolveAuthAllowedHosts, + resolveMediaSsrfPolicy, +} from "./shared.js"; -// ─── Helpers ───────────────────────────────────────────────────────────────── - -const publicResolve = async () => ({ address: "13.107.136.10" }); -const privateResolve = (ip: string) => async () => ({ address: ip }); -const failingResolve = async () => { - throw new Error("DNS failure"); -}; - -function mockFetchWithRedirect(redirectMap: Record, finalBody = "ok") { - return vi.fn(async (url: string, init?: RequestInit) => { - const target = redirectMap[url]; - if (target && init?.redirect === "manual") { - return new Response(null, { - status: 302, - headers: { location: target }, - }); - } - return new Response(finalBody, { status: 200 }); - }); -} - -// ─── isPrivateOrReservedIP ─────────────────────────────────────────────────── - -describe("isPrivateOrReservedIP", () => { - it.each([ - ["10.0.0.1", true], - ["10.255.255.255", true], - ["172.16.0.1", true], - ["172.31.255.255", true], - ["172.15.0.1", false], - ["172.32.0.1", false], - ["192.168.0.1", true], - ["192.168.255.255", true], - ["127.0.0.1", true], - ["127.255.255.255", true], - ["169.254.0.1", true], - ["169.254.169.254", true], - ["0.0.0.0", true], - ["8.8.8.8", false], - ["13.107.136.10", false], - ["52.96.0.1", false], - ] as const)("IPv4 %s → %s", (ip, expected) => { - expect(isPrivateOrReservedIP(ip)).toBe(expected); +describe("msteams attachment allowlists", () => { + it("normalizes wildcard host lists", () => { + expect(resolveAllowedHosts(["*", "graph.microsoft.com"])).toEqual(["*"]); + expect(resolveAuthAllowedHosts(["*", "graph.microsoft.com"])).toEqual(["*"]); }); - it.each([ - ["::1", true], - ["::", true], - ["fe80::1", true], - ["fc00::1", true], - ["fd12:3456::1", true], - ["2001:0db8::1", false], - ["2620:1ec:c11::200", false], - // IPv4-mapped IPv6 addresses - ["::ffff:127.0.0.1", true], - ["::ffff:10.0.0.1", true], - ["::ffff:192.168.1.1", true], - ["::ffff:169.254.169.254", true], - ["::ffff:8.8.8.8", false], - ["::ffff:13.107.136.10", false], - ] as const)("IPv6 %s → %s", (ip, expected) => { - expect(isPrivateOrReservedIP(ip)).toBe(expected); + it("requires https and host suffix match", () => { + const allowHosts = resolveAllowedHosts(["sharepoint.com"]); + expect(isUrlAllowed("https://contoso.sharepoint.com/file.png", allowHosts)).toBe(true); + expect(isUrlAllowed("http://contoso.sharepoint.com/file.png", allowHosts)).toBe(false); + expect(isUrlAllowed("https://evil.example.com/file.png", allowHosts)).toBe(false); }); - it.each([ - ["999.999.999.999", true], - ["256.0.0.1", true], - ["10.0.0.256", true], - ["-1.0.0.1", false], - ["1.2.3.4.5", false], - ["0:0:0:0:0:0:0:1", true], - ] as const)("malformed/expanded %s → %s (SDK fails closed)", (ip, expected) => { - expect(isPrivateOrReservedIP(ip)).toBe(expected); - }); -}); - -// ─── resolveAndValidateIP ──────────────────────────────────────────────────── - -describe("resolveAndValidateIP", () => { - it("accepts a hostname resolving to a public IP", async () => { - const ip = await resolveAndValidateIP("teams.sharepoint.com", publicResolve); - expect(ip).toBe("13.107.136.10"); - }); - - it("rejects a hostname resolving to 10.x.x.x", async () => { - await expect(resolveAndValidateIP("evil.test", privateResolve("10.0.0.1"))).rejects.toThrow( - "private/reserved IP", - ); - }); - - it("rejects a hostname resolving to 169.254.169.254", async () => { - await expect( - resolveAndValidateIP("evil.test", privateResolve("169.254.169.254")), - ).rejects.toThrow("private/reserved IP"); - }); - - it("rejects a hostname resolving to loopback", async () => { - await expect(resolveAndValidateIP("evil.test", privateResolve("127.0.0.1"))).rejects.toThrow( - "private/reserved IP", - ); - }); - - it("rejects a hostname resolving to IPv6 loopback", async () => { - await expect(resolveAndValidateIP("evil.test", privateResolve("::1"))).rejects.toThrow( - "private/reserved IP", - ); - }); - - it("throws on DNS resolution failure", async () => { - await expect(resolveAndValidateIP("nonexistent.test", failingResolve)).rejects.toThrow( - "DNS resolution failed", - ); - }); -}); - -// ─── safeFetch ─────────────────────────────────────────────────────────────── - -describe("safeFetch", () => { - it("fetches a URL directly when no redirect occurs", async () => { - const fetchMock = vi.fn(async (_url: string, _init?: RequestInit) => { - return new Response("ok", { status: 200 }); - }); - const res = await safeFetch({ - url: "https://teams.sharepoint.com/file.pdf", - allowHosts: ["sharepoint.com"], - fetchFn: fetchMock as unknown as typeof fetch, - resolveFn: publicResolve, - }); - expect(res.status).toBe(200); - expect(fetchMock).toHaveBeenCalledOnce(); - // Should have used redirect: "manual" - expect(fetchMock.mock.calls[0][1]).toHaveProperty("redirect", "manual"); - }); - - it("follows a redirect to an allowlisted host with public IP", async () => { - const fetchMock = mockFetchWithRedirect({ - "https://teams.sharepoint.com/file.pdf": "https://cdn.sharepoint.com/storage/file.pdf", - }); - const res = await safeFetch({ - url: "https://teams.sharepoint.com/file.pdf", - allowHosts: ["sharepoint.com"], - fetchFn: fetchMock as unknown as typeof fetch, - resolveFn: publicResolve, - }); - expect(res.status).toBe(200); - expect(fetchMock).toHaveBeenCalledTimes(2); - }); - - it("blocks a redirect to a non-allowlisted host", async () => { - const fetchMock = mockFetchWithRedirect({ - "https://teams.sharepoint.com/file.pdf": "https://evil.example.com/steal", - }); - await expect( - safeFetch({ - url: "https://teams.sharepoint.com/file.pdf", - allowHosts: ["sharepoint.com"], - fetchFn: fetchMock as unknown as typeof fetch, - resolveFn: publicResolve, - }), - ).rejects.toThrow("blocked by allowlist"); - // Should not have fetched the evil URL - expect(fetchMock).toHaveBeenCalledTimes(1); - }); - - it("blocks a redirect to an allowlisted host that resolves to a private IP (DNS rebinding)", async () => { - let callCount = 0; - const rebindingResolve = async () => { - callCount++; - // First call (initial URL) resolves to public IP - if (callCount === 1) return { address: "13.107.136.10" }; - // Second call (redirect target) resolves to private IP - return { address: "169.254.169.254" }; - }; - - const fetchMock = mockFetchWithRedirect({ - "https://teams.sharepoint.com/file.pdf": "https://evil.trafficmanager.net/metadata", - }); - await expect( - safeFetch({ - url: "https://teams.sharepoint.com/file.pdf", - allowHosts: ["sharepoint.com", "trafficmanager.net"], - fetchFn: fetchMock as unknown as typeof fetch, - resolveFn: rebindingResolve, - }), - ).rejects.toThrow("private/reserved IP"); - expect(fetchMock).toHaveBeenCalledTimes(1); - }); - - it("blocks when the initial URL resolves to a private IP", async () => { - const fetchMock = vi.fn(); - await expect( - safeFetch({ - url: "https://evil.sharepoint.com/file.pdf", - allowHosts: ["sharepoint.com"], - fetchFn: fetchMock as unknown as typeof fetch, - resolveFn: privateResolve("10.0.0.1"), - }), - ).rejects.toThrow("Initial download URL blocked"); - expect(fetchMock).not.toHaveBeenCalled(); - }); - - it("blocks when initial URL DNS resolution fails", async () => { - const fetchMock = vi.fn(); - await expect( - safeFetch({ - url: "https://nonexistent.sharepoint.com/file.pdf", - allowHosts: ["sharepoint.com"], - fetchFn: fetchMock as unknown as typeof fetch, - resolveFn: failingResolve, - }), - ).rejects.toThrow("Initial download URL blocked"); - expect(fetchMock).not.toHaveBeenCalled(); - }); - - it("follows multiple redirects when all are valid", async () => { - const fetchMock = vi.fn(async (url: string, init?: RequestInit) => { - if (url === "https://a.sharepoint.com/1" && init?.redirect === "manual") { - return new Response(null, { - status: 302, - headers: { location: "https://b.sharepoint.com/2" }, - }); - } - if (url === "https://b.sharepoint.com/2" && init?.redirect === "manual") { - return new Response(null, { - status: 302, - headers: { location: "https://c.sharepoint.com/3" }, - }); - } - return new Response("final", { status: 200 }); - }); - - const res = await safeFetch({ - url: "https://a.sharepoint.com/1", - allowHosts: ["sharepoint.com"], - fetchFn: fetchMock as unknown as typeof fetch, - resolveFn: publicResolve, - }); - expect(res.status).toBe(200); - expect(fetchMock).toHaveBeenCalledTimes(3); - }); - - it("throws on too many redirects", async () => { - let counter = 0; - const fetchMock = vi.fn(async (_url: string, init?: RequestInit) => { - if (init?.redirect === "manual") { - counter++; - return new Response(null, { - status: 302, - headers: { location: `https://loop${counter}.sharepoint.com/x` }, - }); - } - return new Response("ok", { status: 200 }); - }); - - await expect( - safeFetch({ - url: "https://start.sharepoint.com/x", - allowHosts: ["sharepoint.com"], - fetchFn: fetchMock as unknown as typeof fetch, - resolveFn: publicResolve, - }), - ).rejects.toThrow("Too many redirects"); - }); - - it("blocks redirect to HTTP (non-HTTPS)", async () => { - const fetchMock = mockFetchWithRedirect({ - "https://teams.sharepoint.com/file": "http://internal.sharepoint.com/file", - }); - await expect( - safeFetch({ - url: "https://teams.sharepoint.com/file", - allowHosts: ["sharepoint.com"], - fetchFn: fetchMock as unknown as typeof fetch, - resolveFn: publicResolve, - }), - ).rejects.toThrow("blocked by allowlist"); + it("builds shared SSRF policy from suffix allowlist", () => { + expect(resolveMediaSsrfPolicy(["sharepoint.com"])).toEqual({ + hostnameAllowlist: ["sharepoint.com", "*.sharepoint.com"], + }); + expect(resolveMediaSsrfPolicy(["*"])).toBeUndefined(); }); }); diff --git a/extensions/msteams/src/attachments/shared.ts b/extensions/msteams/src/attachments/shared.ts index 50221e8eb9a..abb98791b32 100644 --- a/extensions/msteams/src/attachments/shared.ts +++ b/extensions/msteams/src/attachments/shared.ts @@ -1,5 +1,9 @@ -import { lookup } from "node:dns/promises"; -import { isPrivateIpAddress } from "openclaw/plugin-sdk"; +import { + buildHostnameAllowlistPolicyFromSuffixAllowlist, + isHttpsUrlAllowedByHostnameSuffixAllowlist, + normalizeHostnameSuffixAllowlist, +} from "openclaw/plugin-sdk"; +import type { SsrFPolicy } from "openclaw/plugin-sdk"; import type { MSTeamsAttachmentLike } from "./types.js"; type InlineImageCandidate = @@ -252,153 +256,18 @@ export function safeHostForUrl(url: string): string { } } -function normalizeAllowHost(value: string): string { - const trimmed = value.trim().toLowerCase(); - if (!trimmed) { - return ""; - } - if (trimmed === "*") { - return "*"; - } - return trimmed.replace(/^\*\.?/, ""); -} - export function resolveAllowedHosts(input?: string[]): string[] { - if (!Array.isArray(input) || input.length === 0) { - return DEFAULT_MEDIA_HOST_ALLOWLIST.slice(); - } - const normalized = input.map(normalizeAllowHost).filter(Boolean); - if (normalized.includes("*")) { - return ["*"]; - } - return normalized; + return normalizeHostnameSuffixAllowlist(input, DEFAULT_MEDIA_HOST_ALLOWLIST); } export function resolveAuthAllowedHosts(input?: string[]): string[] { - if (!Array.isArray(input) || input.length === 0) { - return DEFAULT_MEDIA_AUTH_HOST_ALLOWLIST.slice(); - } - const normalized = input.map(normalizeAllowHost).filter(Boolean); - if (normalized.includes("*")) { - return ["*"]; - } - return normalized; -} - -function isHostAllowed(host: string, allowlist: string[]): boolean { - if (allowlist.includes("*")) { - return true; - } - const normalized = host.toLowerCase(); - return allowlist.some((entry) => normalized === entry || normalized.endsWith(`.${entry}`)); + return normalizeHostnameSuffixAllowlist(input, DEFAULT_MEDIA_AUTH_HOST_ALLOWLIST); } export function isUrlAllowed(url: string, allowlist: string[]): boolean { - try { - const parsed = new URL(url); - if (parsed.protocol !== "https:") { - return false; - } - return isHostAllowed(parsed.hostname, allowlist); - } catch { - return false; - } + return isHttpsUrlAllowedByHostnameSuffixAllowlist(url, allowlist); } -/** - * Returns true if the given IPv4 or IPv6 address is in a private, loopback, - * or link-local range that must never be reached from media downloads. - * - * Delegates to the SDK's `isPrivateIpAddress` which handles IPv4-mapped IPv6, - * expanded notation, NAT64, 6to4, Teredo, octal IPv4, and fails closed on - * parse errors. - */ -export const isPrivateOrReservedIP: (ip: string) => boolean = isPrivateIpAddress; - -/** - * Resolve a hostname via DNS and reject private/reserved IPs. - * Throws if the resolved IP is private or resolution fails. - */ -export async function resolveAndValidateIP( - hostname: string, - resolveFn?: (hostname: string) => Promise<{ address: string }>, -): Promise { - const resolve = resolveFn ?? lookup; - let resolved: { address: string }; - try { - resolved = await resolve(hostname); - } catch { - throw new Error(`DNS resolution failed for "${hostname}"`); - } - if (isPrivateOrReservedIP(resolved.address)) { - throw new Error(`Hostname "${hostname}" resolves to private/reserved IP (${resolved.address})`); - } - return resolved.address; -} - -/** Maximum number of redirects to follow in safeFetch. */ -const MAX_SAFE_REDIRECTS = 5; - -/** - * Fetch a URL with redirect: "manual", validating each redirect target - * against the hostname allowlist and DNS-resolved IP (anti-SSRF). - * - * This prevents: - * - Auto-following redirects to non-allowlisted hosts - * - DNS rebinding attacks where an allowlisted domain resolves to a private IP - */ -export async function safeFetch(params: { - url: string; - allowHosts: string[]; - fetchFn?: typeof fetch; - requestInit?: RequestInit; - resolveFn?: (hostname: string) => Promise<{ address: string }>; -}): Promise { - const fetchFn = params.fetchFn ?? fetch; - const resolveFn = params.resolveFn; - let currentUrl = params.url; - - // Validate the initial URL's resolved IP - try { - const initialHost = new URL(currentUrl).hostname; - await resolveAndValidateIP(initialHost, resolveFn); - } catch { - throw new Error(`Initial download URL blocked: ${currentUrl}`); - } - - for (let i = 0; i <= MAX_SAFE_REDIRECTS; i++) { - const res = await fetchFn(currentUrl, { - ...params.requestInit, - redirect: "manual", - }); - - if (![301, 302, 303, 307, 308].includes(res.status)) { - return res; - } - - const location = res.headers.get("location"); - if (!location) { - return res; - } - - let redirectUrl: string; - try { - redirectUrl = new URL(location, currentUrl).toString(); - } catch { - throw new Error(`Invalid redirect URL: ${location}`); - } - - // Validate redirect target against hostname allowlist - if (!isUrlAllowed(redirectUrl, params.allowHosts)) { - throw new Error(`Media redirect target blocked by allowlist: ${redirectUrl}`); - } - - // Validate redirect target's resolved IP - const redirectHost = new URL(redirectUrl).hostname; - await resolveAndValidateIP(redirectHost, resolveFn); - - currentUrl = redirectUrl; - } - - throw new Error(`Too many redirects (>${MAX_SAFE_REDIRECTS})`); +export function resolveMediaSsrfPolicy(allowHosts: string[]): SsrFPolicy | undefined { + return buildHostnameAllowlistPolicyFromSuffixAllowlist(allowHosts); } diff --git a/package.json b/package.json index dbb56c4fbd9..e9c9b4b796f 100644 --- a/package.json +++ b/package.json @@ -54,7 +54,7 @@ "build": "pnpm canvas:a2ui:bundle && tsdown && pnpm build:plugin-sdk:dts && node --import tsx scripts/write-plugin-sdk-entry-dts.ts && node --import tsx scripts/canvas-a2ui-copy.ts && node --import tsx scripts/copy-hook-metadata.ts && node --import tsx scripts/copy-export-html-templates.ts && node --import tsx scripts/write-build-info.ts && node --import tsx scripts/write-cli-compat.ts", "build:plugin-sdk:dts": "tsc -p tsconfig.plugin-sdk.dts.json", "canvas:a2ui:bundle": "bash scripts/bundle-a2ui.sh", - "check": "pnpm format:check && pnpm tsgo && pnpm lint && pnpm lint:tmp:no-random-messaging && pnpm lint:tmp:channel-agnostic-boundaries && pnpm lint:auth:no-pairing-store-group", + "check": "pnpm format:check && pnpm tsgo && pnpm lint && pnpm lint:tmp:no-random-messaging && pnpm lint:tmp:channel-agnostic-boundaries && pnpm lint:tmp:no-raw-channel-fetch && pnpm lint:auth:no-pairing-store-group", "check:docs": "pnpm format:docs:check && pnpm lint:docs && pnpm docs:check-links", "check:loc": "node --import tsx scripts/check-ts-max-loc.ts --max 500", "deadcode:ci": "pnpm deadcode:report:ci:knip && pnpm deadcode:report:ci:ts-prune && pnpm deadcode:report:ci:ts-unused", @@ -96,6 +96,7 @@ "lint:swift": "swiftlint lint --config .swiftlint.yml && (cd apps/ios && swiftlint lint --config .swiftlint.yml)", "lint:tmp:channel-agnostic-boundaries": "node scripts/check-channel-agnostic-boundaries.mjs", "lint:tmp:no-random-messaging": "node scripts/check-no-random-messaging-tmp.mjs", + "lint:tmp:no-raw-channel-fetch": "node scripts/check-no-raw-channel-fetch.mjs", "lint:ui:no-raw-window-open": "node scripts/check-no-raw-window-open.mjs", "mac:open": "open dist/OpenClaw.app", "mac:package": "bash scripts/package-mac-app.sh", diff --git a/scripts/check-no-raw-channel-fetch.mjs b/scripts/check-no-raw-channel-fetch.mjs new file mode 100644 index 00000000000..639c698a3f3 --- /dev/null +++ b/scripts/check-no-raw-channel-fetch.mjs @@ -0,0 +1,211 @@ +#!/usr/bin/env node + +import { promises as fs } from "node:fs"; +import path from "node:path"; +import { fileURLToPath } from "node:url"; +import ts from "typescript"; + +const repoRoot = path.resolve(path.dirname(fileURLToPath(import.meta.url)), ".."); +const sourceRoots = [ + path.join(repoRoot, "src", "telegram"), + path.join(repoRoot, "src", "discord"), + path.join(repoRoot, "src", "slack"), + path.join(repoRoot, "src", "signal"), + path.join(repoRoot, "src", "imessage"), + path.join(repoRoot, "src", "web"), + path.join(repoRoot, "src", "channels"), + path.join(repoRoot, "src", "routing"), + path.join(repoRoot, "src", "line"), + path.join(repoRoot, "extensions"), +]; + +// Temporary allowlist for legacy callsites. New raw fetch callsites in channel/plugin runtime +// code should be rejected and migrated to fetchWithSsrFGuard/shared channel helpers. +const allowedRawFetchCallsites = new Set([ + "extensions/bluebubbles/src/types.ts:131", + "extensions/feishu/src/streaming-card.ts:31", + "extensions/feishu/src/streaming-card.ts:100", + "extensions/feishu/src/streaming-card.ts:141", + "extensions/feishu/src/streaming-card.ts:197", + "extensions/google-gemini-cli-auth/oauth.ts:372", + "extensions/google-gemini-cli-auth/oauth.ts:408", + "extensions/google-gemini-cli-auth/oauth.ts:447", + "extensions/google-gemini-cli-auth/oauth.ts:507", + "extensions/google-gemini-cli-auth/oauth.ts:575", + "extensions/googlechat/src/api.ts:22", + "extensions/googlechat/src/api.ts:43", + "extensions/googlechat/src/api.ts:63", + "extensions/googlechat/src/api.ts:184", + "extensions/googlechat/src/auth.ts:82", + "extensions/matrix/src/directory-live.ts:41", + "extensions/matrix/src/matrix/client/config.ts:171", + "extensions/mattermost/src/mattermost/client.ts:211", + "extensions/mattermost/src/mattermost/monitor.ts:234", + "extensions/mattermost/src/mattermost/probe.ts:27", + "extensions/minimax-portal-auth/oauth.ts:71", + "extensions/minimax-portal-auth/oauth.ts:112", + "extensions/msteams/src/graph.ts:39", + "extensions/nextcloud-talk/src/room-info.ts:92", + "extensions/nextcloud-talk/src/send.ts:107", + "extensions/nextcloud-talk/src/send.ts:198", + "extensions/qwen-portal-auth/oauth.ts:46", + "extensions/qwen-portal-auth/oauth.ts:80", + "extensions/talk-voice/index.ts:27", + "extensions/thread-ownership/index.ts:105", + "extensions/voice-call/src/providers/plivo.ts:95", + "extensions/voice-call/src/providers/telnyx.ts:61", + "extensions/voice-call/src/providers/tts-openai.ts:111", + "extensions/voice-call/src/providers/twilio/api.ts:23", + "src/channels/telegram/api.ts:8", + "src/discord/send.outbound.ts:347", + "src/discord/voice-message.ts:267", + "src/slack/monitor/media.ts:64", + "src/slack/monitor/media.ts:68", + "src/slack/monitor/media.ts:82", + "src/slack/monitor/media.ts:108", +]); + +function isTestLikeFile(filePath) { + return ( + filePath.endsWith(".test.ts") || + filePath.endsWith(".test-utils.ts") || + filePath.endsWith(".test-harness.ts") || + filePath.endsWith(".e2e-harness.ts") || + filePath.endsWith(".browser.test.ts") || + filePath.endsWith(".node.test.ts") + ); +} + +async function collectTypeScriptFiles(targetPath) { + const stat = await fs.stat(targetPath); + if (stat.isFile()) { + if (!targetPath.endsWith(".ts") || isTestLikeFile(targetPath)) { + return []; + } + return [targetPath]; + } + const entries = await fs.readdir(targetPath, { withFileTypes: true }); + const files = []; + for (const entry of entries) { + const entryPath = path.join(targetPath, entry.name); + if (entry.isDirectory()) { + files.push(...(await collectTypeScriptFiles(entryPath))); + continue; + } + if (!entry.isFile()) { + continue; + } + if (!entryPath.endsWith(".ts")) { + continue; + } + if (isTestLikeFile(entryPath)) { + continue; + } + files.push(entryPath); + } + return files; +} + +function unwrapExpression(expression) { + let current = expression; + while (true) { + if (ts.isParenthesizedExpression(current)) { + current = current.expression; + continue; + } + if (ts.isAsExpression(current) || ts.isTypeAssertionExpression(current)) { + current = current.expression; + continue; + } + if (ts.isNonNullExpression(current)) { + current = current.expression; + continue; + } + return current; + } +} + +function isRawFetchCall(expression) { + const callee = unwrapExpression(expression); + if (ts.isIdentifier(callee)) { + return callee.text === "fetch"; + } + if (ts.isPropertyAccessExpression(callee)) { + return ( + ts.isIdentifier(callee.expression) && + callee.expression.text === "globalThis" && + callee.name.text === "fetch" + ); + } + return false; +} + +export function findRawFetchCallLines(content, fileName = "source.ts") { + const sourceFile = ts.createSourceFile(fileName, content, ts.ScriptTarget.Latest, true); + const lines = []; + const visit = (node) => { + if (ts.isCallExpression(node) && isRawFetchCall(node.expression)) { + const line = + sourceFile.getLineAndCharacterOfPosition(node.expression.getStart(sourceFile)).line + 1; + lines.push(line); + } + ts.forEachChild(node, visit); + }; + visit(sourceFile); + return lines; +} + +export async function main() { + const files = ( + await Promise.all( + sourceRoots.map(async (sourceRoot) => { + try { + return await collectTypeScriptFiles(sourceRoot); + } catch { + return []; + } + }), + ) + ).flat(); + + const violations = []; + for (const filePath of files) { + const content = await fs.readFile(filePath, "utf8"); + const relPath = path.relative(repoRoot, filePath).replaceAll(path.sep, "/"); + for (const line of findRawFetchCallLines(content, filePath)) { + const callsite = `${relPath}:${line}`; + if (allowedRawFetchCallsites.has(callsite)) { + continue; + } + violations.push(callsite); + } + } + + if (violations.length === 0) { + return; + } + + console.error("Found raw fetch() usage in channel/plugin runtime sources outside allowlist:"); + for (const violation of violations.toSorted()) { + console.error(`- ${violation}`); + } + console.error( + "Use fetchWithSsrFGuard() or existing channel/plugin SDK wrappers for network calls.", + ); + process.exit(1); +} + +const isDirectExecution = (() => { + const entry = process.argv[1]; + if (!entry) { + return false; + } + return path.resolve(entry) === fileURLToPath(import.meta.url); +})(); + +if (isDirectExecution) { + main().catch((error) => { + console.error(error); + process.exit(1); + }); +} diff --git a/src/plugin-sdk/fetch-auth.test.ts b/src/plugin-sdk/fetch-auth.test.ts new file mode 100644 index 00000000000..cc401cc1a3d --- /dev/null +++ b/src/plugin-sdk/fetch-auth.test.ts @@ -0,0 +1,94 @@ +import { describe, expect, it, vi } from "vitest"; +import { fetchWithBearerAuthScopeFallback } from "./fetch-auth.js"; + +describe("fetchWithBearerAuthScopeFallback", () => { + it("rejects non-https urls when https is required", async () => { + await expect( + fetchWithBearerAuthScopeFallback({ + url: "http://example.com/file", + scopes: [], + requireHttps: true, + }), + ).rejects.toThrow("URL must use HTTPS"); + }); + + it("returns immediately when the first attempt succeeds", async () => { + const fetchFn = vi.fn(async () => new Response("ok", { status: 200 })); + const tokenProvider = { getAccessToken: vi.fn(async () => "unused") }; + + const response = await fetchWithBearerAuthScopeFallback({ + url: "https://example.com/file", + scopes: ["https://graph.microsoft.com"], + fetchFn, + tokenProvider, + }); + + expect(response.status).toBe(200); + expect(fetchFn).toHaveBeenCalledTimes(1); + expect(tokenProvider.getAccessToken).not.toHaveBeenCalled(); + }); + + it("retries with auth scopes after a 401 response", async () => { + const fetchFn = vi + .fn() + .mockResolvedValueOnce(new Response("unauthorized", { status: 401 })) + .mockResolvedValueOnce(new Response("ok", { status: 200 })); + const tokenProvider = { getAccessToken: vi.fn(async () => "token-1") }; + + const response = await fetchWithBearerAuthScopeFallback({ + url: "https://graph.microsoft.com/v1.0/me", + scopes: ["https://graph.microsoft.com", "https://api.botframework.com"], + fetchFn, + tokenProvider, + }); + + expect(response.status).toBe(200); + expect(fetchFn).toHaveBeenCalledTimes(2); + expect(tokenProvider.getAccessToken).toHaveBeenCalledWith("https://graph.microsoft.com"); + const secondCall = fetchFn.mock.calls[1] as [string, RequestInit | undefined]; + const secondHeaders = new Headers(secondCall[1]?.headers); + expect(secondHeaders.get("authorization")).toBe("Bearer token-1"); + }); + + it("does not attach auth when host predicate rejects url", async () => { + const fetchFn = vi.fn(async () => new Response("unauthorized", { status: 401 })); + const tokenProvider = { getAccessToken: vi.fn(async () => "token-1") }; + + const response = await fetchWithBearerAuthScopeFallback({ + url: "https://example.com/file", + scopes: ["https://graph.microsoft.com"], + fetchFn, + tokenProvider, + shouldAttachAuth: () => false, + }); + + expect(response.status).toBe(401); + expect(fetchFn).toHaveBeenCalledTimes(1); + expect(tokenProvider.getAccessToken).not.toHaveBeenCalled(); + }); + + it("continues across scopes when token retrieval fails", async () => { + const fetchFn = vi + .fn() + .mockResolvedValueOnce(new Response("unauthorized", { status: 401 })) + .mockResolvedValueOnce(new Response("ok", { status: 200 })); + const tokenProvider = { + getAccessToken: vi + .fn() + .mockRejectedValueOnce(new Error("first scope failed")) + .mockResolvedValueOnce("token-2"), + }; + + const response = await fetchWithBearerAuthScopeFallback({ + url: "https://graph.microsoft.com/v1.0/me", + scopes: ["https://first.example", "https://second.example"], + fetchFn, + tokenProvider, + }); + + expect(response.status).toBe(200); + expect(tokenProvider.getAccessToken).toHaveBeenCalledTimes(2); + expect(tokenProvider.getAccessToken).toHaveBeenNthCalledWith(1, "https://first.example"); + expect(tokenProvider.getAccessToken).toHaveBeenNthCalledWith(2, "https://second.example"); + }); +}); diff --git a/src/plugin-sdk/fetch-auth.ts b/src/plugin-sdk/fetch-auth.ts new file mode 100644 index 00000000000..fc04e4aa910 --- /dev/null +++ b/src/plugin-sdk/fetch-auth.ts @@ -0,0 +1,71 @@ +export type ScopeTokenProvider = { + getAccessToken: (scope: string) => Promise; +}; + +function isAuthFailureStatus(status: number): boolean { + return status === 401 || status === 403; +} + +export async function fetchWithBearerAuthScopeFallback(params: { + url: string; + scopes: readonly string[]; + tokenProvider?: ScopeTokenProvider; + fetchFn?: typeof fetch; + requestInit?: RequestInit; + requireHttps?: boolean; + shouldAttachAuth?: (url: string) => boolean; + shouldRetry?: (response: Response) => boolean; +}): Promise { + const fetchFn = params.fetchFn ?? fetch; + let parsedUrl: URL; + try { + parsedUrl = new URL(params.url); + } catch { + throw new Error(`Invalid URL: ${params.url}`); + } + if (params.requireHttps === true && parsedUrl.protocol !== "https:") { + throw new Error(`URL must use HTTPS: ${params.url}`); + } + + const fetchOnce = (headers?: Headers): Promise => + fetchFn(params.url, { + ...params.requestInit, + ...(headers ? { headers } : {}), + }); + + const firstAttempt = await fetchOnce(); + if (firstAttempt.ok) { + return firstAttempt; + } + if (!params.tokenProvider) { + return firstAttempt; + } + + const shouldRetry = + params.shouldRetry ?? ((response: Response) => isAuthFailureStatus(response.status)); + if (!shouldRetry(firstAttempt)) { + return firstAttempt; + } + if (params.shouldAttachAuth && !params.shouldAttachAuth(params.url)) { + return firstAttempt; + } + + for (const scope of params.scopes) { + try { + const token = await params.tokenProvider.getAccessToken(scope); + const authHeaders = new Headers(params.requestInit?.headers); + authHeaders.set("Authorization", `Bearer ${token}`); + const authAttempt = await fetchOnce(authHeaders); + if (authAttempt.ok) { + return authAttempt; + } + if (!shouldRetry(authAttempt)) { + continue; + } + } catch { + // Ignore token/fetch errors and continue trying remaining scopes. + } + } + + return firstAttempt; +} diff --git a/src/plugin-sdk/index.ts b/src/plugin-sdk/index.ts index 767507f6b84..0d378ec6c34 100644 --- a/src/plugin-sdk/index.ts +++ b/src/plugin-sdk/index.ts @@ -292,6 +292,13 @@ export { isPrivateIpAddress, } from "../infra/net/ssrf.js"; export type { LookupFn, SsrFPolicy } from "../infra/net/ssrf.js"; +export { + buildHostnameAllowlistPolicyFromSuffixAllowlist, + isHttpsUrlAllowedByHostnameSuffixAllowlist, + normalizeHostnameSuffixAllowlist, +} from "./ssrf-policy.js"; +export { fetchWithBearerAuthScopeFallback } from "./fetch-auth.js"; +export type { ScopeTokenProvider } from "./fetch-auth.js"; export { rawDataToString } from "../infra/ws.js"; export { isWSLSync, isWSL2Sync, isWSLEnv } from "../infra/wsl.js"; export { isTruthyEnvValue } from "../infra/env.js"; diff --git a/src/plugin-sdk/ssrf-policy.test.ts b/src/plugin-sdk/ssrf-policy.test.ts new file mode 100644 index 00000000000..20247e7bc2a --- /dev/null +++ b/src/plugin-sdk/ssrf-policy.test.ts @@ -0,0 +1,84 @@ +import { describe, expect, it } from "vitest"; +import { + buildHostnameAllowlistPolicyFromSuffixAllowlist, + isHttpsUrlAllowedByHostnameSuffixAllowlist, + normalizeHostnameSuffixAllowlist, +} from "./ssrf-policy.js"; + +describe("normalizeHostnameSuffixAllowlist", () => { + it("uses defaults when input is missing", () => { + expect(normalizeHostnameSuffixAllowlist(undefined, ["GRAPH.MICROSOFT.COM"])).toEqual([ + "graph.microsoft.com", + ]); + }); + + it("normalizes wildcard prefixes and deduplicates", () => { + expect( + normalizeHostnameSuffixAllowlist([ + "*.TrafficManager.NET", + ".trafficmanager.net.", + " * ", + "x", + ]), + ).toEqual(["*"]); + }); +}); + +describe("isHttpsUrlAllowedByHostnameSuffixAllowlist", () => { + it("requires https", () => { + expect( + isHttpsUrlAllowedByHostnameSuffixAllowlist("http://a.example.com/x", ["example.com"]), + ).toBe(false); + }); + + it("supports exact and suffix match", () => { + expect( + isHttpsUrlAllowedByHostnameSuffixAllowlist("https://example.com/x", ["example.com"]), + ).toBe(true); + expect( + isHttpsUrlAllowedByHostnameSuffixAllowlist("https://a.example.com/x", ["example.com"]), + ).toBe(true); + expect(isHttpsUrlAllowedByHostnameSuffixAllowlist("https://evil.com/x", ["example.com"])).toBe( + false, + ); + }); + + it("supports wildcard allowlist", () => { + expect(isHttpsUrlAllowedByHostnameSuffixAllowlist("https://evil.com/x", ["*"])).toBe(true); + }); +}); + +describe("buildHostnameAllowlistPolicyFromSuffixAllowlist", () => { + it("returns undefined when allowHosts is empty", () => { + expect(buildHostnameAllowlistPolicyFromSuffixAllowlist()).toBeUndefined(); + expect(buildHostnameAllowlistPolicyFromSuffixAllowlist([])).toBeUndefined(); + }); + + it("returns undefined when wildcard host is present", () => { + expect(buildHostnameAllowlistPolicyFromSuffixAllowlist(["*"])).toBeUndefined(); + expect(buildHostnameAllowlistPolicyFromSuffixAllowlist(["example.com", "*"])).toBeUndefined(); + }); + + it("expands a suffix entry to exact + wildcard hostname allowlist patterns", () => { + expect(buildHostnameAllowlistPolicyFromSuffixAllowlist(["sharepoint.com"])).toEqual({ + hostnameAllowlist: ["sharepoint.com", "*.sharepoint.com"], + }); + }); + + it("normalizes wildcard prefixes, leading/trailing dots, and deduplicates patterns", () => { + expect( + buildHostnameAllowlistPolicyFromSuffixAllowlist([ + "*.TrafficManager.NET", + ".trafficmanager.net.", + " blob.core.windows.net ", + ]), + ).toEqual({ + hostnameAllowlist: [ + "trafficmanager.net", + "*.trafficmanager.net", + "blob.core.windows.net", + "*.blob.core.windows.net", + ], + }); + }); +}); diff --git a/src/plugin-sdk/ssrf-policy.ts b/src/plugin-sdk/ssrf-policy.ts new file mode 100644 index 00000000000..351938d0456 --- /dev/null +++ b/src/plugin-sdk/ssrf-policy.ts @@ -0,0 +1,85 @@ +import type { SsrFPolicy } from "../infra/net/ssrf.js"; + +function normalizeHostnameSuffix(value: string): string { + const trimmed = value.trim().toLowerCase(); + if (!trimmed) { + return ""; + } + if (trimmed === "*" || trimmed === "*.") { + return "*"; + } + const withoutWildcard = trimmed.replace(/^\*\.?/, ""); + const withoutLeadingDot = withoutWildcard.replace(/^\.+/, ""); + return withoutLeadingDot.replace(/\.+$/, ""); +} + +function isHostnameAllowedBySuffixAllowlist( + hostname: string, + allowlist: readonly string[], +): boolean { + if (allowlist.includes("*")) { + return true; + } + const normalized = hostname.toLowerCase(); + return allowlist.some((entry) => normalized === entry || normalized.endsWith(`.${entry}`)); +} + +export function normalizeHostnameSuffixAllowlist( + input?: readonly string[], + defaults?: readonly string[], +): string[] { + const source = input && input.length > 0 ? input : defaults; + if (!source || source.length === 0) { + return []; + } + const normalized = source.map(normalizeHostnameSuffix).filter(Boolean); + if (normalized.includes("*")) { + return ["*"]; + } + return Array.from(new Set(normalized)); +} + +export function isHttpsUrlAllowedByHostnameSuffixAllowlist( + url: string, + allowlist: readonly string[], +): boolean { + try { + const parsed = new URL(url); + if (parsed.protocol !== "https:") { + return false; + } + return isHostnameAllowedBySuffixAllowlist(parsed.hostname, allowlist); + } catch { + return false; + } +} + +/** + * Converts suffix-style host allowlists (for example "example.com") into SSRF + * hostname allowlist patterns used by the shared fetch guard. + * + * Suffix semantics: + * - "example.com" allows "example.com" and "*.example.com" + * - "*" disables hostname allowlist restrictions + */ +export function buildHostnameAllowlistPolicyFromSuffixAllowlist( + allowHosts?: readonly string[], +): SsrFPolicy | undefined { + const normalizedAllowHosts = normalizeHostnameSuffixAllowlist(allowHosts); + if (normalizedAllowHosts.length === 0) { + return undefined; + } + const patterns = new Set(); + for (const normalized of normalizedAllowHosts) { + if (normalized === "*") { + return undefined; + } + patterns.add(normalized); + patterns.add(`*.${normalized}`); + } + + if (patterns.size === 0) { + return undefined; + } + return { hostnameAllowlist: Array.from(patterns) }; +}