Download src/media/fetch.test.ts from SaylorTwift/openclaw: direct link, hf CLI and curl.
- Browser
- Download file 58.6 kB
-
https://huggingface.co/SaylorTwift/openclaw/resolve/main/src/media/fetch.test.ts
- Command line
-
hf download hf://SaylorTwift/openclaw/src/media/fetch.test.ts
-
curl -L -o fetch.test.ts https://huggingface.co/SaylorTwift/openclaw/resolve/main/src/media/fetch.test.ts
58.6 kB
| // Media fetch tests cover remote media download limits and validation. | |
| import fs from "node:fs/promises"; | |
| import { createServer } from "node:http"; | |
| import path from "node:path"; | |
| import { MAX_TIMER_TIMEOUT_MS } from "@openclaw/normalization-core/number-coercion"; | |
| import { afterAll, beforeAll, beforeEach, describe, expect, it, vi } from "vitest"; | |
| import { hasErrnoCode } from "../infra/errors.js"; | |
| import { createTempHomeEnv, type TempHomeEnv } from "../test-utils/temp-home.js"; | |
| const fetchWithSsrFGuardMock = vi.hoisted(() => vi.fn()); | |
| vi.mock("../infra/net/fetch-guard.js", () => ({ | |
| fetchWithSsrFGuard: (...args: unknown[]) => fetchWithSsrFGuardMock(...args), | |
| withStrictGuardedFetchMode: <T>(params: T) => params, | |
| withTrustedExplicitProxyGuardedFetchMode: <T>(params: T) => ({ | |
| ...params, | |
| mode: "trusted_explicit_proxy", | |
| }), | |
| })); | |
| type FetchModule = typeof import("./fetch.js"); | |
| type ReadRemoteMediaBuffer = FetchModule["readRemoteMediaBuffer"]; | |
| type SaveRemoteMedia = FetchModule["saveRemoteMedia"]; | |
| type SaveResponseMedia = FetchModule["saveResponseMedia"]; | |
| type LookupFn = NonNullable<Parameters<ReadRemoteMediaBuffer>[0]["lookupFn"]>; | |
| let readRemoteMediaBuffer: ReadRemoteMediaBuffer; | |
| let saveRemoteMedia: SaveRemoteMedia; | |
| let saveResponseMedia: SaveResponseMedia; | |
| let defaultFetchMediaMaxBytes: number; | |
| let tempHome: TempHomeEnv; | |
| function makeStream(chunks: Uint8Array[]) { | |
| return new ReadableStream<Uint8Array>({ | |
| start(controller) { | |
| for (const chunk of chunks) { | |
| controller.enqueue(chunk); | |
| } | |
| controller.close(); | |
| }, | |
| }); | |
| } | |
| function makeCancelableStream(chunks: Uint8Array[]) { | |
| let canceled = false; | |
| const stream = new ReadableStream<Uint8Array>({ | |
| start(controller) { | |
| for (const chunk of chunks) { | |
| controller.enqueue(chunk); | |
| } | |
| }, | |
| cancel() { | |
| canceled = true; | |
| }, | |
| }); | |
| return { stream, wasCanceled: () => canceled }; | |
| } | |
| function makeStallingFetch(firstChunk: Uint8Array) { | |
| return vi.fn(async () => { | |
| return new Response( | |
| new ReadableStream<Uint8Array>({ | |
| start(controller) { | |
| controller.enqueue(firstChunk); | |
| }, | |
| }), | |
| { status: 200 }, | |
| ); | |
| }); | |
| } | |
| function makeResponseHeaderStallingFetch() { | |
| return vi.fn( | |
| async (_input: RequestInfo | URL, init?: RequestInit) => | |
| await new Promise<Response>((_resolve, reject) => { | |
| const signal = init?.signal; | |
| const rejectForAbort = () => reject(abortReasonError(signal)); | |
| if (signal?.aborted) { | |
| rejectForAbort(); | |
| return; | |
| } | |
| signal?.addEventListener("abort", rejectForAbort, { once: true }); | |
| }), | |
| ); | |
| } | |
| function makeLookupFn(): LookupFn { | |
| return vi.fn(async () => ({ address: "149.154.167.220", family: 4 })) as unknown as LookupFn; | |
| } | |
| function abortReasonError(signal?: AbortSignal | null): Error { | |
| return signal?.reason instanceof Error | |
| ? signal.reason | |
| : new Error("request aborted", { cause: signal?.reason }); | |
| } | |
| function requireFetchGuardRequest(): unknown { | |
| const [call] = fetchWithSsrFGuardMock.mock.calls; | |
| if (!call) { | |
| throw new Error("expected fetchWithSsrFGuard call"); | |
| } | |
| return call[0]; | |
| } | |
| async function expectRemoteMediaMaxBytesError(params: { | |
| fetchImpl: Parameters<typeof readRemoteMediaBuffer>[0]["fetchImpl"]; | |
| maxBytes: number; | |
| }) { | |
| await expect( | |
| readRemoteMediaBuffer({ | |
| url: "https://example.com/file.bin", | |
| fetchImpl: params.fetchImpl, | |
| maxBytes: params.maxBytes, | |
| lookupFn: makeLookupFn(), | |
| }), | |
| ).rejects.toThrow("exceeds maxBytes"); | |
| } | |
| async function expectRedactedBotTokenFetchError(params: { | |
| botFileUrl: string; | |
| botToken: string; | |
| expectedErrorText: string; | |
| fetchImpl: Parameters<typeof readRemoteMediaBuffer>[0]["fetchImpl"]; | |
| }) { | |
| const error = await readRemoteMediaBuffer({ | |
| url: params.botFileUrl, | |
| fetchImpl: params.fetchImpl, | |
| lookupFn: makeLookupFn(), | |
| maxBytes: 1024, | |
| ssrfPolicy: { | |
| allowedHostnames: ["files.example.test"], | |
| allowRfc2544BenchmarkRange: true, | |
| }, | |
| }).catch((err: unknown) => err as Error); | |
| expect(error).toBeInstanceOf(Error); | |
| const errorText = error instanceof Error ? String(error) : ""; | |
| expect(errorText).not.toContain(params.botToken); | |
| expect(errorText).toBe(params.expectedErrorText); | |
| } | |
| async function expectReadRemoteMediaBufferRejected(params: { | |
| url: string; | |
| fetchImpl: Parameters<typeof readRemoteMediaBuffer>[0]["fetchImpl"]; | |
| maxBytes?: number; | |
| readIdleTimeoutMs?: number; | |
| lookupFn?: LookupFn; | |
| expectedError: RegExp | string | Record<string, unknown>; | |
| }) { | |
| const request = { | |
| url: params.url, | |
| fetchImpl: params.fetchImpl, | |
| lookupFn: params.lookupFn ?? makeLookupFn(), | |
| maxBytes: params.maxBytes ?? 1024, | |
| ...(params.readIdleTimeoutMs ? { readIdleTimeoutMs: params.readIdleTimeoutMs } : {}), | |
| }; | |
| if (params.expectedError instanceof RegExp || typeof params.expectedError === "string") { | |
| await expect(readRemoteMediaBuffer(request)).rejects.toThrow(params.expectedError); | |
| return; | |
| } | |
| let fetchError: unknown; | |
| try { | |
| await readRemoteMediaBuffer(request); | |
| } catch (error) { | |
| fetchError = error; | |
| } | |
| expect(fetchError).toBeInstanceOf(Error); | |
| for (const [key, value] of Object.entries(params.expectedError)) { | |
| expect((fetchError as Record<string, unknown>)[key]).toStrictEqual(value); | |
| } | |
| } | |
| async function expectReadRemoteMediaBufferResolvesToError( | |
| params: Parameters<typeof readRemoteMediaBuffer>[0], | |
| ): Promise<Error> { | |
| const result = await readRemoteMediaBuffer(params).catch((err: unknown) => err); | |
| expect(result).toBeInstanceOf(Error); | |
| if (!(result instanceof Error)) { | |
| expect.unreachable("expected readRemoteMediaBuffer to reject"); | |
| } | |
| return result; | |
| } | |
| async function expectReadRemoteMediaBufferIdleTimeoutCase(params: { | |
| lookupFn: LookupFn; | |
| fetchImpl: Parameters<typeof readRemoteMediaBuffer>[0]["fetchImpl"]; | |
| readIdleTimeoutMs: number; | |
| expectedError: Record<string, unknown>; | |
| }) { | |
| vi.useFakeTimers(); | |
| try { | |
| const rejection = expectReadRemoteMediaBufferRejected({ | |
| url: "https://example.com/file.bin", | |
| fetchImpl: params.fetchImpl, | |
| lookupFn: params.lookupFn, | |
| readIdleTimeoutMs: params.readIdleTimeoutMs, | |
| expectedError: params.expectedError, | |
| }); | |
| await vi.advanceTimersByTimeAsync(params.readIdleTimeoutMs + 5); | |
| await rejection; | |
| } finally { | |
| vi.useRealTimers(); | |
| } | |
| } | |
| async function expectBoundedErrorBodyCase( | |
| fetchImpl: Parameters<typeof readRemoteMediaBuffer>[0]["fetchImpl"], | |
| ) { | |
| const result = await expectReadRemoteMediaBufferResolvesToError( | |
| createReadRemoteMediaBufferParams({ | |
| url: "https://example.com/file.bin", | |
| fetchImpl, | |
| }), | |
| ); | |
| expect(result.message).not.toContain("BAD"); | |
| expect(result.message).not.toContain("body:"); | |
| } | |
| async function expectPrivateIpFetchBlockedCase() { | |
| const fetchImpl = vi.fn(); | |
| await expectReadRemoteMediaBufferRejected({ | |
| url: "http://127.0.0.1/secret.jpg", | |
| fetchImpl, | |
| expectedError: /private|internal|blocked/i, | |
| }); | |
| expect(fetchImpl).not.toHaveBeenCalled(); | |
| } | |
| function createReadRemoteMediaBufferParams( | |
| params: Omit<Parameters<typeof readRemoteMediaBuffer>[0], "lookupFn"> & { lookupFn?: LookupFn }, | |
| ) { | |
| return { | |
| lookupFn: params.lookupFn ?? makeLookupFn(), | |
| maxBytes: 1024, | |
| ...params, | |
| }; | |
| } | |
| describe("readRemoteMediaBuffer", () => { | |
| const botToken = "123456789:ABCDEFGHIJKLMNOPQRSTUVWXYZabcd"; | |
| const redactedBotToken = `${botToken.slice(0, 6)}…${botToken.slice(-4)}`; | |
| const botFileUrl = `https://files.example.test/file/bot${botToken}/photos/1.jpg`; | |
| beforeAll(async () => { | |
| vi.resetModules(); | |
| tempHome = await createTempHomeEnv("openclaw-test-home-"); | |
| const fetchModule = await import("./fetch.js"); | |
| readRemoteMediaBuffer = fetchModule.readRemoteMediaBuffer; | |
| saveRemoteMedia = fetchModule.saveRemoteMedia; | |
| saveResponseMedia = fetchModule.saveResponseMedia; | |
| // Default cap mirrors the module-private DEFAULT_FETCH_MEDIA_MAX_BYTES. | |
| defaultFetchMediaMaxBytes = (await import("@openclaw/media-core/constants")).MAX_DOCUMENT_BYTES; | |
| }); | |
| beforeEach(() => { | |
| vi.useRealTimers(); | |
| fetchWithSsrFGuardMock.mockReset().mockImplementation(async (paramsUnknown: unknown) => { | |
| const params = paramsUnknown as { | |
| url: string; | |
| fetchImpl?: (input: RequestInfo | URL, init?: RequestInit) => Promise<Response>; | |
| beforeRequest?: () => void; | |
| init?: RequestInit; | |
| signal?: AbortSignal; | |
| }; | |
| if (params.url.startsWith("http://127.0.0.1/")) { | |
| throw new Error("Blocked hostname or private/internal/special-use IP address"); | |
| } | |
| const fetcher = params.fetchImpl ?? globalThis.fetch; | |
| if (!fetcher) { | |
| throw new Error("fetch is not available"); | |
| } | |
| params.beforeRequest?.(); | |
| return { | |
| response: await fetcher(params.url, { | |
| ...params.init, | |
| ...(params.signal ? { signal: params.signal } : {}), | |
| }), | |
| finalUrl: params.url, | |
| release: async () => {}, | |
| }; | |
| }); | |
| }); | |
| afterAll(async () => { | |
| try { | |
| await tempHome.restore(); | |
| } finally { | |
| vi.doUnmock("../infra/net/fetch-guard.js"); | |
| vi.resetModules(); | |
| } | |
| }); | |
| it.each([ | |
| { | |
| name: "rejects when content-length exceeds maxBytes", | |
| fetchImpl: async () => | |
| new Response(makeStream([new Uint8Array([1, 2, 3, 4, 5])]), { | |
| status: 200, | |
| headers: { "content-length": "5" }, | |
| }), | |
| }, | |
| { | |
| name: "rejects when streamed payload exceeds maxBytes", | |
| fetchImpl: async () => | |
| new Response(makeStream([new Uint8Array([1, 2, 3]), new Uint8Array([4, 5, 6])]), { | |
| status: 200, | |
| }), | |
| }, | |
| ] as const)("$name", async ({ fetchImpl }) => { | |
| await expectRemoteMediaMaxBytesError({ fetchImpl, maxBytes: 4 }); | |
| }); | |
| it("cancels ignored content-length overflow bodies for remote buffer reads", async () => { | |
| const body = makeCancelableStream([new Uint8Array([1, 2, 3, 4, 5])]); | |
| const fetchImpl = vi.fn( | |
| async () => | |
| new Response(body.stream, { | |
| status: 200, | |
| headers: { "content-length": "5" }, | |
| }), | |
| ); | |
| await expectRemoteMediaMaxBytesError({ fetchImpl, maxBytes: 4 }); | |
| expect(body.wasCanceled()).toBe(true); | |
| }); | |
| it("rejects malformed content-length before remote buffer reads", async () => { | |
| const body = makeCancelableStream([new Uint8Array([1, 2, 3, 4, 5])]); | |
| const fetchImpl = vi.fn( | |
| async () => | |
| new Response(body.stream, { | |
| status: 200, | |
| headers: { "content-length": "1e9" }, | |
| }), | |
| ); | |
| await expect( | |
| readRemoteMediaBuffer({ | |
| url: "https://example.com/file.bin", | |
| fetchImpl, | |
| maxBytes: 4, | |
| lookupFn: makeLookupFn(), | |
| }), | |
| ).rejects.toThrow("invalid content-length header: 1e9"); | |
| expect(body.wasCanceled()).toBe(true); | |
| }); | |
| it("accepts comma-joined identical content-length values", async () => { | |
| const fetchImpl = vi.fn( | |
| async () => | |
| new Response("hello", { | |
| status: 200, | |
| headers: { "content-length": "5, 5" }, | |
| }), | |
| ); | |
| const result = await readRemoteMediaBuffer({ | |
| url: "https://example.com/file.bin", | |
| fetchImpl, | |
| maxBytes: 5, | |
| lookupFn: makeLookupFn(), | |
| }); | |
| expect(result.buffer.toString()).toBe("hello"); | |
| }); | |
| it("applies a default stream limit when maxBytes is omitted", async () => { | |
| const fetchImpl = vi.fn( | |
| async () => | |
| new Response(makeStream([new Uint8Array([1])]), { | |
| status: 200, | |
| headers: { "content-length": String(defaultFetchMediaMaxBytes + 1) }, | |
| }), | |
| ); | |
| await expect( | |
| readRemoteMediaBuffer({ | |
| url: "https://example.com/file.bin", | |
| fetchImpl, | |
| lookupFn: makeLookupFn(), | |
| }), | |
| ).rejects.toThrow(`exceeds maxBytes ${defaultFetchMediaMaxBytes}`); | |
| }); | |
| it.each([ | |
| { | |
| name: "redacts bot tokens from fetch failure messages", | |
| fetchImpl: vi.fn(async () => { | |
| throw new Error(`dial failed for ${botFileUrl}`); | |
| }), | |
| expectedErrorText: `MediaFetchError: Failed to fetch media from https://files.example.test/file/bot${redactedBotToken}/photos/1.jpg: dial failed for https://files.example.test/file/bot${redactedBotToken}/photos/1.jpg`, | |
| }, | |
| { | |
| name: "redacts bot tokens from HTTP error messages", | |
| fetchImpl: vi.fn(async () => new Response("unauthorized", { status: 401 })), | |
| expectedErrorText: `MediaFetchError: Failed to fetch media from https://files.example.test/file/bot${redactedBotToken}/photos/1.jpg: HTTP 401; body: unauthorized`, | |
| }, | |
| ] as const)("$name", async ({ fetchImpl, expectedErrorText }) => { | |
| await expectRedactedBotTokenFetchError({ | |
| botFileUrl, | |
| botToken, | |
| expectedErrorText, | |
| fetchImpl, | |
| }); | |
| }); | |
| it.each([ | |
| { | |
| name: "aborts stalled body reads when idle timeout expires", | |
| lookupFn: vi.fn(async () => ({ | |
| address: "93.184.216.34", | |
| family: 4, | |
| })) as unknown as LookupFn, | |
| fetchImpl: makeStallingFetch(new Uint8Array([1, 2])), | |
| readIdleTimeoutMs: 20, | |
| expectedError: { | |
| code: "fetch_failed", | |
| name: "MediaFetchError", | |
| cause: expect.objectContaining({ name: "TimeoutError" }), | |
| }, | |
| }, | |
| ] as const)("$name", async ({ lookupFn, fetchImpl, readIdleTimeoutMs, expectedError }) => { | |
| await expectReadRemoteMediaBufferIdleTimeoutCase({ | |
| lookupFn, | |
| fetchImpl, | |
| readIdleTimeoutMs, | |
| expectedError, | |
| }); | |
| }); | |
| it("aborts when response headers exceed their deadline", async () => { | |
| vi.useFakeTimers(); | |
| try { | |
| const fetchImpl = makeResponseHeaderStallingFetch(); | |
| const result = readRemoteMediaBuffer({ | |
| url: "https://example.com/file.bin", | |
| fetchImpl, | |
| lookupFn: makeLookupFn(), | |
| maxBytes: 1024, | |
| responseHeaderTimeoutMs: 20, | |
| }).catch((error: unknown) => error); | |
| await vi.advanceTimersByTimeAsync(25); | |
| await expect(result).resolves.toMatchObject({ | |
| name: "MediaFetchError", | |
| code: "fetch_failed", | |
| cause: { name: "TimeoutError" }, | |
| }); | |
| } finally { | |
| vi.useRealTimers(); | |
| } | |
| }); | |
| it("uses the default response-header deadline for stalled media", async () => { | |
| vi.useFakeTimers(); | |
| try { | |
| const result = readRemoteMediaBuffer({ | |
| url: "https://example.com/file.bin", | |
| fetchImpl: makeResponseHeaderStallingFetch(), | |
| lookupFn: makeLookupFn(), | |
| maxBytes: 1024, | |
| }).catch((error: unknown) => error); | |
| await vi.advanceTimersByTimeAsync(15 * 60_000 + 5); | |
| await expect(result).resolves.toMatchObject({ | |
| name: "MediaFetchError", | |
| code: "fetch_failed", | |
| cause: { name: "TimeoutError" }, | |
| }); | |
| } finally { | |
| vi.useRealTimers(); | |
| } | |
| }); | |
| it("clears the response-header deadline while a healthy body keeps progressing", async () => { | |
| vi.useFakeTimers(); | |
| try { | |
| const fetchImpl = vi.fn(async (_input: RequestInfo | URL, init?: RequestInit) => { | |
| const signal = init?.signal; | |
| return new Response( | |
| new ReadableStream<Uint8Array>({ | |
| start(controller) { | |
| const failForAbort = () => controller.error(signal?.reason); | |
| if (signal?.aborted) { | |
| failForAbort(); | |
| return; | |
| } | |
| signal?.addEventListener("abort", failForAbort, { once: true }); | |
| setTimeout(() => controller.enqueue(new Uint8Array([1])), 25); | |
| setTimeout(() => controller.enqueue(new Uint8Array([2])), 50); | |
| setTimeout(() => { | |
| signal?.removeEventListener("abort", failForAbort); | |
| controller.close(); | |
| }, 75); | |
| }, | |
| }), | |
| { status: 200 }, | |
| ); | |
| }); | |
| const result = readRemoteMediaBuffer({ | |
| url: "https://example.com/file.bin", | |
| fetchImpl, | |
| lookupFn: makeLookupFn(), | |
| maxBytes: 1024, | |
| responseHeaderTimeoutMs: 10, | |
| readIdleTimeoutMs: 30, | |
| }); | |
| await vi.advanceTimersByTimeAsync(80); | |
| await expect(result).resolves.toMatchObject({ buffer: Buffer.from([1, 2]) }); | |
| } finally { | |
| vi.useRealTimers(); | |
| } | |
| }); | |
| it("propagates a parent abort while waiting for response headers", async () => { | |
| const parent = new AbortController(); | |
| const fetchImpl = vi.fn( | |
| async (_input: RequestInfo | URL, init?: RequestInit) => | |
| await new Promise<Response>((_resolve, reject) => { | |
| const signal = init?.signal; | |
| const rejectForAbort = () => reject(abortReasonError(signal)); | |
| if (signal?.aborted) { | |
| rejectForAbort(); | |
| return; | |
| } | |
| signal?.addEventListener("abort", rejectForAbort, { once: true }); | |
| }), | |
| ); | |
| const result = readRemoteMediaBuffer({ | |
| url: "https://example.com/file.bin", | |
| fetchImpl, | |
| requestInit: { signal: parent.signal }, | |
| lookupFn: makeLookupFn(), | |
| maxBytes: 1024, | |
| responseHeaderTimeoutMs: 60_000, | |
| }).catch((error: unknown) => error); | |
| parent.abort(); | |
| await expect(result).resolves.toMatchObject({ | |
| name: "MediaFetchError", | |
| code: "fetch_failed", | |
| cause: { name: "AbortError" }, | |
| }); | |
| }); | |
| it("keeps the parent abort active while reading the response body", async () => { | |
| const parent = new AbortController(); | |
| const fetchImpl = vi.fn(async (_input: RequestInfo | URL, init?: RequestInit) => { | |
| const signal = init?.signal; | |
| return new Response( | |
| new ReadableStream<Uint8Array>({ | |
| start(controller) { | |
| controller.enqueue(new Uint8Array([1])); | |
| const failForAbort = () => controller.error(signal?.reason); | |
| if (signal?.aborted) { | |
| failForAbort(); | |
| return; | |
| } | |
| signal?.addEventListener("abort", failForAbort, { once: true }); | |
| }, | |
| }), | |
| { status: 200 }, | |
| ); | |
| }); | |
| const result = readRemoteMediaBuffer({ | |
| url: "https://example.com/file.bin", | |
| fetchImpl, | |
| requestInit: { signal: parent.signal }, | |
| lookupFn: makeLookupFn(), | |
| maxBytes: 1024, | |
| responseHeaderTimeoutMs: 60_000, | |
| }).catch((error: unknown) => error); | |
| await vi.waitFor(() => expect(fetchImpl).toHaveBeenCalledTimes(1)); | |
| parent.abort(); | |
| await expect(result).resolves.toMatchObject({ | |
| name: "MediaFetchError", | |
| code: "fetch_failed", | |
| cause: { name: "AbortError" }, | |
| }); | |
| }); | |
| it("retries transient fetch failures when retry is enabled", async () => { | |
| const transientError = Object.assign(new Error("socket hang up"), { code: "ECONNRESET" }); | |
| const fetchImpl = vi | |
| .fn() | |
| .mockRejectedValueOnce(transientError) | |
| .mockResolvedValueOnce(new Response("ok", { status: 200 })); | |
| const result = await readRemoteMediaBuffer({ | |
| url: "https://example.com/file.bin", | |
| fetchImpl, | |
| lookupFn: makeLookupFn(), | |
| maxBytes: 1024, | |
| retry: { attempts: 3, minDelayMs: 0, maxDelayMs: 0, jitter: 0 }, | |
| }); | |
| expect(result.buffer.toString()).toBe("ok"); | |
| expect(fetchImpl).toHaveBeenCalledTimes(2); | |
| }); | |
| it("retries 5xx responses when retry is enabled", async () => { | |
| const fetchImpl = vi | |
| .fn() | |
| .mockResolvedValueOnce( | |
| new Response("busy", { status: 503, statusText: "Service Unavailable" }), | |
| ) | |
| .mockResolvedValueOnce(new Response("ok", { status: 200 })); | |
| const result = await readRemoteMediaBuffer({ | |
| url: "https://example.com/file.bin", | |
| fetchImpl, | |
| lookupFn: makeLookupFn(), | |
| maxBytes: 1024, | |
| retry: { attempts: 3, minDelayMs: 0, maxDelayMs: 0, jitter: 0 }, | |
| }); | |
| expect(result.buffer.toString()).toBe("ok"); | |
| expect(fetchImpl).toHaveBeenCalledTimes(2); | |
| }); | |
| it("retries 408 responses when retry is enabled", async () => { | |
| const fetchImpl = vi | |
| .fn() | |
| .mockResolvedValueOnce( | |
| new Response("timeout", { status: 408, statusText: "Request Timeout" }), | |
| ) | |
| .mockResolvedValueOnce(new Response("ok", { status: 200 })); | |
| const result = await readRemoteMediaBuffer({ | |
| url: "https://example.com/file.bin", | |
| fetchImpl, | |
| lookupFn: makeLookupFn(), | |
| maxBytes: 1024, | |
| retry: { attempts: 3, minDelayMs: 0, maxDelayMs: 0, jitter: 0 }, | |
| }); | |
| expect(result.buffer.toString()).toBe("ok"); | |
| expect(fetchImpl).toHaveBeenCalledTimes(2); | |
| }); | |
| it("retries transient response body read failures when retry is enabled", async () => { | |
| const transientError = Object.assign(new Error("socket hang up"), { code: "ECONNRESET" }); | |
| const fetchImpl = vi | |
| .fn() | |
| .mockResolvedValueOnce( | |
| new Response( | |
| new ReadableStream<Uint8Array>({ | |
| start(controller) { | |
| controller.error(transientError); | |
| }, | |
| }), | |
| { status: 200 }, | |
| ), | |
| ) | |
| .mockResolvedValueOnce(new Response("ok", { status: 200 })); | |
| const result = await readRemoteMediaBuffer({ | |
| url: "https://example.com/file.bin", | |
| fetchImpl, | |
| lookupFn: makeLookupFn(), | |
| maxBytes: 1024, | |
| retry: { attempts: 3, minDelayMs: 0, maxDelayMs: 0, jitter: 0 }, | |
| }); | |
| expect(result.buffer.toString()).toBe("ok"); | |
| expect(fetchImpl).toHaveBeenCalledTimes(2); | |
| }); | |
| it("retries a default response-body idle timeout", async () => { | |
| vi.useFakeTimers(); | |
| try { | |
| const fetchImpl = vi | |
| .fn() | |
| .mockResolvedValueOnce( | |
| new Response( | |
| new ReadableStream<Uint8Array>({ | |
| start(controller) { | |
| controller.enqueue(new Uint8Array([1, 2])); | |
| }, | |
| }), | |
| { status: 200 }, | |
| ), | |
| ) | |
| .mockResolvedValueOnce(new Response("ok", { status: 200 })); | |
| const result = readRemoteMediaBuffer({ | |
| url: "https://example.com/file.bin", | |
| fetchImpl, | |
| lookupFn: makeLookupFn(), | |
| maxBytes: 1024, | |
| readIdleTimeoutMs: 20, | |
| retry: { attempts: 2, minDelayMs: 0, maxDelayMs: 0, jitter: 0 }, | |
| }); | |
| await vi.advanceTimersByTimeAsync(25); | |
| await expect(result).resolves.toMatchObject({ buffer: Buffer.from("ok") }); | |
| expect(fetchImpl).toHaveBeenCalledTimes(2); | |
| } finally { | |
| vi.useRealTimers(); | |
| } | |
| }); | |
| it("does not retry 4xx responses", async () => { | |
| const fetchImpl = vi | |
| .fn() | |
| .mockResolvedValueOnce(new Response("missing", { status: 404 })) | |
| .mockResolvedValueOnce(new Response("ok", { status: 200 })); | |
| await expect( | |
| readRemoteMediaBuffer({ | |
| url: "https://example.com/file.bin", | |
| fetchImpl, | |
| lookupFn: makeLookupFn(), | |
| maxBytes: 1024, | |
| retry: { attempts: 3, minDelayMs: 0, maxDelayMs: 0, jitter: 0 }, | |
| }), | |
| ).rejects.toMatchObject({ code: "http_error", status: 404 }); | |
| expect(fetchImpl).toHaveBeenCalledTimes(1); | |
| }); | |
| it("does not retry caller aborts", async () => { | |
| const abortError = new Error("This operation was aborted"); | |
| abortError.name = "AbortError"; | |
| const fetchImpl = vi | |
| .fn() | |
| .mockRejectedValueOnce(abortError) | |
| .mockResolvedValueOnce(new Response("ok", { status: 200 })); | |
| await expect( | |
| readRemoteMediaBuffer({ | |
| url: "https://example.com/file.bin", | |
| fetchImpl, | |
| lookupFn: makeLookupFn(), | |
| maxBytes: 1024, | |
| retry: { attempts: 3, minDelayMs: 0, maxDelayMs: 0, jitter: 0 }, | |
| }), | |
| ).rejects.toMatchObject({ code: "fetch_failed" }); | |
| expect(fetchImpl).toHaveBeenCalledTimes(1); | |
| }); | |
| it.each([ | |
| { | |
| name: "buffer reads", | |
| fetchMedia: (options: Parameters<ReadRemoteMediaBuffer>[0]) => readRemoteMediaBuffer(options), | |
| }, | |
| { | |
| name: "store writes", | |
| fetchMedia: (options: Parameters<ReadRemoteMediaBuffer>[0]) => saveRemoteMedia(options), | |
| }, | |
| ])("cancels retry backoff for $name", async ({ fetchMedia }) => { | |
| let requests = 0; | |
| const server = createServer((_request, response) => { | |
| requests += 1; | |
| response.writeHead(503).end("busy"); | |
| }); | |
| await new Promise<void>((resolve, reject) => { | |
| server.once("error", reject); | |
| server.listen(0, "127.0.0.1", resolve); | |
| }); | |
| try { | |
| const address = server.address(); | |
| if (!address || typeof address === "string") { | |
| throw new Error("expected a local media test server address"); | |
| } | |
| const controller = new AbortController(); | |
| const operation = fetchMedia({ | |
| url: `http://127.0.0.1:${address.port}/retry.bin`, | |
| requestInit: { signal: controller.signal }, | |
| retry: { | |
| attempts: 2, | |
| minDelayMs: 25, | |
| maxDelayMs: 25, | |
| jitter: 0, | |
| onRetry: () => { | |
| setImmediate(() => controller.abort()); | |
| }, | |
| }, | |
| }); | |
| await expect(operation).rejects.toMatchObject({ | |
| name: "MediaFetchError", | |
| code: "fetch_failed", | |
| cause: { name: "AbortError" }, | |
| }); | |
| expect(requests).toBe(1); | |
| expect(fetchWithSsrFGuardMock).toHaveBeenCalledTimes(1); | |
| } finally { | |
| server.closeAllConnections(); | |
| await new Promise<void>((resolve, reject) => { | |
| server.close((error) => (error ? reject(error) : resolve())); | |
| }); | |
| } | |
| }); | |
| it("does not retry SSRF guard blocks", async () => { | |
| const fetchImpl = vi.fn(); | |
| await expect( | |
| readRemoteMediaBuffer({ | |
| url: "http://127.0.0.1/secret.jpg", | |
| fetchImpl, | |
| lookupFn: makeLookupFn(), | |
| maxBytes: 1024, | |
| retry: { attempts: 3, minDelayMs: 0, maxDelayMs: 0, jitter: 0 }, | |
| }), | |
| ).rejects.toThrow(/private|internal|blocked/i); | |
| expect(fetchWithSsrFGuardMock).toHaveBeenCalledTimes(1); | |
| expect(fetchImpl).not.toHaveBeenCalled(); | |
| }); | |
| it("does not retry maxBytes failures", async () => { | |
| const fetchImpl = vi | |
| .fn() | |
| .mockResolvedValueOnce( | |
| new Response("large", { status: 200, headers: { "content-length": "5" } }), | |
| ) | |
| .mockResolvedValueOnce(new Response("ok", { status: 200 })); | |
| await expect( | |
| readRemoteMediaBuffer({ | |
| url: "https://example.com/file.bin", | |
| fetchImpl, | |
| lookupFn: makeLookupFn(), | |
| maxBytes: 4, | |
| retry: { attempts: 3, minDelayMs: 0, maxDelayMs: 0, jitter: 0 }, | |
| }), | |
| ).rejects.toMatchObject({ code: "max_bytes" }); | |
| expect(fetchImpl).toHaveBeenCalledTimes(1); | |
| }); | |
| it.each([ | |
| { | |
| name: "bounds error-body snippets instead of reading the full response", | |
| kind: "bounded-error-body" as const, | |
| fetchImpl: vi.fn( | |
| async () => | |
| new Response(makeStream([new TextEncoder().encode(`${" ".repeat(9_000)}BAD`)]), { | |
| status: 400, | |
| statusText: "Bad Request", | |
| }), | |
| ), | |
| }, | |
| { | |
| name: "blocks private IP literals before fetching", | |
| kind: "private-ip-block" as const, | |
| }, | |
| ] as const)("$name", async (testCase) => { | |
| if (testCase.kind === "private-ip-block") { | |
| await expectPrivateIpFetchBlockedCase(); | |
| return; | |
| } | |
| await expectBoundedErrorBodyCase(testCase.fetchImpl); | |
| }); | |
| it("uses trusted explicit-proxy mode when the caller opts in for proxy-side DNS", async () => { | |
| const fetchImpl = vi.fn(async () => new Response("ok", { status: 200 })); | |
| const lookupFn = makeLookupFn(); | |
| const dispatcherPolicy = { | |
| mode: "explicit-proxy" as const, | |
| proxyUrl: "http://localhost:8888", | |
| allowPrivateProxy: true, | |
| }; | |
| await readRemoteMediaBuffer({ | |
| url: "https://files.example.test/file/bot123/photos/test.jpg", | |
| fetchImpl, | |
| lookupFn, | |
| trustExplicitProxyDns: true, | |
| dispatcherAttempts: [ | |
| { | |
| dispatcherPolicy, | |
| }, | |
| ], | |
| }); | |
| expect(fetchWithSsrFGuardMock).toHaveBeenCalledTimes(1); | |
| expect(requireFetchGuardRequest()).toStrictEqual({ | |
| url: "https://files.example.test/file/bot123/photos/test.jpg", | |
| fetchImpl, | |
| init: undefined, | |
| maxRedirects: undefined, | |
| policy: undefined, | |
| lookupFn, | |
| dispatcherPolicy, | |
| mode: "trusted_explicit_proxy", | |
| signal: expect.any(AbortSignal), | |
| }); | |
| }); | |
| it("passes request timeout through the guarded fetch path", async () => { | |
| const fetchImpl = vi.fn(async () => new Response("ok", { status: 200 })); | |
| const parent = new AbortController(); | |
| await readRemoteMediaBuffer({ | |
| url: "https://example.com/file.bin", | |
| fetchImpl, | |
| requestInit: { signal: parent.signal }, | |
| lookupFn: makeLookupFn(), | |
| maxBytes: 1024, | |
| timeoutMs: 1234, | |
| }); | |
| expect(fetchWithSsrFGuardMock).toHaveBeenCalledTimes(1); | |
| expect(requireFetchGuardRequest()).toMatchObject({ | |
| url: "https://example.com/file.bin", | |
| timeoutMs: 1234, | |
| signal: parent.signal, | |
| }); | |
| }); | |
| it("passes the HTTPS-only redirect policy through the guarded fetch path", async () => { | |
| const fetchImpl = vi.fn(async () => new Response("ok", { status: 200 })); | |
| await readRemoteMediaBuffer({ | |
| url: "https://example.com/favicon.ico", | |
| fetchImpl, | |
| lookupFn: makeLookupFn(), | |
| requireHttps: true, | |
| }); | |
| expect(requireFetchGuardRequest()).toMatchObject({ | |
| url: "https://example.com/favicon.ico", | |
| requireHttps: true, | |
| }); | |
| }); | |
| it("streams successful responses directly into the media store", async () => { | |
| const fetchImpl = vi.fn( | |
| async () => | |
| new Response(makeStream([new Uint8Array([1, 2, 3]), new Uint8Array([4])]), { | |
| status: 200, | |
| headers: { | |
| "content-disposition": 'attachment; filename="photo"', | |
| "content-type": "image/png", | |
| }, | |
| }), | |
| ); | |
| const saved = await saveRemoteMedia({ | |
| url: "https://example.com/download", | |
| fetchImpl, | |
| lookupFn: makeLookupFn(), | |
| maxBytes: 8, | |
| }); | |
| expect(saved.fileName).toBe("photo"); | |
| expect(saved.contentType).toBe("image/png"); | |
| expect(saved.path).toMatch(/[a-f0-9-]{36}\.png$/); | |
| expect(saved.path).not.toMatch(/photo---/); | |
| await expect(fs.readFile(saved.path)).resolves.toStrictEqual(Buffer.from([1, 2, 3, 4])); | |
| }); | |
| it("preserves content-disposition CSV detection for streamed downloads", async () => { | |
| const csv = Buffer.from("name,value\nopenclaw,1\n"); | |
| const fetchImpl = vi.fn( | |
| async () => | |
| new Response(makeStream([csv.subarray(0, 8), csv.subarray(8)]), { | |
| status: 200, | |
| headers: { | |
| "content-disposition": 'attachment; filename="report.csv"', | |
| "content-type": "application/octet-stream", | |
| }, | |
| }), | |
| ); | |
| const saved = await saveRemoteMedia({ | |
| url: "https://example.com/download", | |
| fetchImpl, | |
| lookupFn: makeLookupFn(), | |
| maxBytes: 64, | |
| }); | |
| expect(saved.fileName).toBe("report.csv"); | |
| expect(saved.contentType).toBe("text/csv"); | |
| expect(saved.path).toMatch(/[a-f0-9-]{36}\.csv$/); | |
| expect(saved.path).not.toMatch(/report---/); | |
| await expect(fs.readFile(saved.path)).resolves.toStrictEqual(csv); | |
| }); | |
| it("preserves content-disposition CSV detection for provided response streams", async () => { | |
| const csv = Buffer.from("name,value\nopenclaw,1\n"); | |
| const response = new Response(makeStream([csv.subarray(0, 8), csv.subarray(8)]), { | |
| status: 200, | |
| headers: { | |
| "content-disposition": 'attachment; filename="report.csv"', | |
| "content-type": "application/octet-stream", | |
| }, | |
| }); | |
| const saved = await saveResponseMedia(response, { | |
| sourceUrl: "https://example.com/download", | |
| maxBytes: 64, | |
| }); | |
| expect(saved.fileName).toBe("report.csv"); | |
| expect(saved.contentType).toBe("text/csv"); | |
| expect(saved.path).toMatch(/[a-f0-9-]{36}\.csv$/); | |
| await expect(fs.readFile(saved.path)).resolves.toStrictEqual(csv); | |
| }); | |
| it("preserves content-disposition CSV detection for buffered downloads", async () => { | |
| const csv = Buffer.from("name,value\nopenclaw,1\n"); | |
| const fetchImpl = vi.fn( | |
| async () => | |
| new Response(makeStream([csv.subarray(0, 8), csv.subarray(8)]), { | |
| status: 200, | |
| headers: { | |
| "content-disposition": 'attachment; filename="report.csv"', | |
| "content-type": "application/octet-stream", | |
| }, | |
| }), | |
| ); | |
| const media = await readRemoteMediaBuffer({ | |
| url: "https://example.com/download", | |
| fetchImpl, | |
| lookupFn: makeLookupFn(), | |
| maxBytes: 64, | |
| }); | |
| expect(media.fileName).toBe("report.csv"); | |
| expect(media.contentType).toBe("text/csv"); | |
| expect(media.buffer).toStrictEqual(csv); | |
| }); | |
| it("keeps explicit stream detection hints ahead of content-disposition filenames", async () => { | |
| const fetchImpl = vi.fn( | |
| async () => | |
| new Response(makeStream([new Uint8Array([1, 2, 3])]), { | |
| status: 200, | |
| headers: { | |
| "content-disposition": 'attachment; filename="report.csv"', | |
| "content-type": "application/octet-stream", | |
| }, | |
| }), | |
| ); | |
| const saved = await saveRemoteMedia({ | |
| url: "https://example.com/download", | |
| fetchImpl, | |
| lookupFn: makeLookupFn(), | |
| filePathHint: "document.docx", | |
| maxBytes: 8, | |
| }); | |
| expect(saved.fileName).toBe("report.csv"); | |
| expect(saved.contentType).toBe( | |
| "application/vnd.openxmlformats-officedocument.wordprocessingml.document", | |
| ); | |
| expect(saved.path).toMatch(/[a-f0-9-]{36}\.docx$/); | |
| expect(saved.path).not.toMatch(/\.csv$/); | |
| }); | |
| it("keeps byte-sniffed images ahead of content-disposition stream hints", async () => { | |
| const jpeg = Buffer.from([0xff, 0xd8, 0xff, 0x00]); | |
| const fetchImpl = vi.fn( | |
| async () => | |
| new Response(makeStream([jpeg]), { | |
| status: 200, | |
| headers: { | |
| "content-disposition": 'attachment; filename="report.csv"', | |
| "content-type": "application/octet-stream", | |
| }, | |
| }), | |
| ); | |
| const saved = await saveRemoteMedia({ | |
| url: "https://example.com/download", | |
| fetchImpl, | |
| lookupFn: makeLookupFn(), | |
| maxBytes: 8, | |
| }); | |
| expect(saved.fileName).toBe("report.csv"); | |
| expect(saved.contentType).toBe("image/jpeg"); | |
| expect(saved.path).toMatch(/[a-f0-9-]{36}\.jpg$/); | |
| expect(saved.path).not.toMatch(/\.csv$/); | |
| await expect(fs.readFile(saved.path)).resolves.toStrictEqual(jpeg); | |
| }); | |
| it("keeps saving a healthy streaming body after the response-header deadline", async () => { | |
| vi.useFakeTimers(); | |
| try { | |
| const fetchImpl = vi.fn(async (_input: RequestInfo | URL, init?: RequestInit) => { | |
| const signal = init?.signal; | |
| return new Response( | |
| new ReadableStream<Uint8Array>({ | |
| start(controller) { | |
| const failForAbort = () => controller.error(signal?.reason); | |
| if (signal?.aborted) { | |
| failForAbort(); | |
| return; | |
| } | |
| signal?.addEventListener("abort", failForAbort, { once: true }); | |
| setTimeout(() => controller.enqueue(new Uint8Array([1])), 25); | |
| setTimeout(() => controller.enqueue(new Uint8Array([2])), 50); | |
| setTimeout(() => { | |
| signal?.removeEventListener("abort", failForAbort); | |
| controller.close(); | |
| }, 75); | |
| }, | |
| }), | |
| { status: 200, headers: { "content-type": "application/octet-stream" } }, | |
| ); | |
| }); | |
| const result = saveRemoteMedia({ | |
| url: "https://example.com/download", | |
| fetchImpl, | |
| lookupFn: makeLookupFn(), | |
| maxBytes: 8, | |
| responseHeaderTimeoutMs: 10, | |
| readIdleTimeoutMs: 30, | |
| }); | |
| await vi.advanceTimersByTimeAsync(80); | |
| const saved = await result; | |
| await expect(fs.readFile(saved.path)).resolves.toStrictEqual(Buffer.from([1, 2])); | |
| } finally { | |
| vi.useRealTimers(); | |
| } | |
| }); | |
| it("keeps the parent abort active while saving the response body", async () => { | |
| const parent = new AbortController(); | |
| let bodyStarted!: () => void; | |
| const bodyReady = new Promise<void>((resolve) => { | |
| bodyStarted = resolve; | |
| }); | |
| const fetchImpl = vi.fn(async (_input: RequestInfo | URL, init?: RequestInit) => { | |
| const signal = init?.signal; | |
| return new Response( | |
| new ReadableStream<Uint8Array>({ | |
| start(controller) { | |
| controller.enqueue(new Uint8Array([1])); | |
| const failForAbort = () => controller.error(signal?.reason); | |
| if (signal?.aborted) { | |
| failForAbort(); | |
| return; | |
| } | |
| signal?.addEventListener("abort", failForAbort, { once: true }); | |
| bodyStarted(); | |
| }, | |
| }), | |
| { status: 200 }, | |
| ); | |
| }); | |
| const result = saveRemoteMedia({ | |
| url: "https://example.com/download", | |
| fetchImpl, | |
| requestInit: { signal: parent.signal }, | |
| lookupFn: makeLookupFn(), | |
| maxBytes: 8, | |
| responseHeaderTimeoutMs: 60_000, | |
| }).catch((error: unknown) => error); | |
| await bodyReady; | |
| parent.abort(); | |
| await expect(result).resolves.toMatchObject({ | |
| name: "MediaFetchError", | |
| code: "fetch_failed", | |
| cause: { name: "AbortError" }, | |
| }); | |
| }); | |
| it("clamps oversized saved-response idle timeout timers", async () => { | |
| const setTimeoutSpy = vi.spyOn(globalThis, "setTimeout"); | |
| try { | |
| const fetchImpl = vi.fn( | |
| async () => | |
| new Response(makeStream([new Uint8Array([1, 2, 3])]), { | |
| status: 200, | |
| headers: { "content-type": "application/octet-stream" }, | |
| }), | |
| ); | |
| const saved = await saveRemoteMedia({ | |
| url: "https://example.com/download", | |
| fetchImpl, | |
| lookupFn: makeLookupFn(), | |
| maxBytes: 8, | |
| readIdleTimeoutMs: MAX_TIMER_TIMEOUT_MS + 1, | |
| }); | |
| await expect(fs.readFile(saved.path)).resolves.toStrictEqual(Buffer.from([1, 2, 3])); | |
| expect(setTimeoutSpy).toHaveBeenCalledWith(expect.any(Function), MAX_TIMER_TIMEOUT_MS); | |
| } finally { | |
| setTimeoutSpy.mockRestore(); | |
| } | |
| }); | |
| it.each([ | |
| ["5", "content length 5 exceeds maxBytes 4", false], | |
| ["5", "content length 5 exceeds maxBytes 4", true], | |
| ["1e9", "invalid content-length header: 1e9", false], | |
| ["1e9", "invalid content-length header: 1e9", true], | |
| ] as const)( | |
| "cancels saved-response content-length %s (%s; partially read: %s)", | |
| async (contentLength, message, partiallyRead) => { | |
| const body = makeCancelableStream([new Uint8Array([1]), new Uint8Array([2, 3, 4, 5])]); | |
| const response = new Response(body.stream, { | |
| status: 200, | |
| headers: { "content-length": contentLength }, | |
| }); | |
| try { | |
| if (partiallyRead) { | |
| const reader = body.stream.getReader(); | |
| try { | |
| expect(await reader.read()).toEqual({ done: false, value: new Uint8Array([1]) }); | |
| } finally { | |
| reader.releaseLock(); | |
| } | |
| } | |
| expect(response.bodyUsed).toBe(partiallyRead); | |
| await expect( | |
| saveResponseMedia(response, { | |
| maxBytes: 4, | |
| sourceUrl: "https://example.com/file.bin", | |
| }), | |
| ).rejects.toThrow(message); | |
| expect(body.wasCanceled()).toBe(true); | |
| expect(body.stream.locked).toBe(false); | |
| } finally { | |
| await body.stream.cancel(); | |
| } | |
| }, | |
| ); | |
| it("decodes URL path basenames when deriving remote media filenames", async () => { | |
| const fetchImpl = vi.fn( | |
| async () => | |
| new Response(makeStream([new Uint8Array([1, 2, 3])]), { | |
| status: 200, | |
| headers: { "content-type": "application/pdf" }, | |
| }), | |
| ); | |
| const saved = await saveRemoteMedia({ | |
| url: "https://example.com/files/My%20Report.pdf", | |
| fetchImpl, | |
| lookupFn: makeLookupFn(), | |
| maxBytes: 8, | |
| }); | |
| expect(saved.fileName).toBe("My Report.pdf"); | |
| }); | |
| it("keeps raw URL path basenames when percent escapes are malformed", async () => { | |
| const fetchImpl = vi.fn( | |
| async () => | |
| new Response(makeStream([new Uint8Array([1, 2, 3])]), { | |
| status: 200, | |
| headers: { "content-type": "application/pdf" }, | |
| }), | |
| ); | |
| const saved = await saveRemoteMedia({ | |
| url: "https://example.com/files/bad%E0%A4%A.pdf", | |
| fetchImpl, | |
| lookupFn: makeLookupFn(), | |
| maxBytes: 8, | |
| }); | |
| expect(saved.fileName).toBe("bad%E0%A4%A.pdf"); | |
| }); | |
| it.each([ | |
| ["https://example.com/files/reports%2FQ1.pdf", "reports_Q1.pdf"], | |
| ["https://example.com/files/reports%5CQ1.pdf", "reports_Q1.pdf"], | |
| ["https://example.com/files/reports%2F%2FQ1.pdf", "reports__Q1.pdf"], | |
| ])( | |
| "keeps decoded URL fallback separators inside the selected basename", | |
| async (url, fileName) => { | |
| const fetchImpl = vi.fn( | |
| async () => | |
| new Response(makeStream([new Uint8Array([1, 2, 3])]), { | |
| status: 200, | |
| headers: { "content-type": "application/pdf" }, | |
| }), | |
| ); | |
| const saved = await saveRemoteMedia({ | |
| url, | |
| fetchImpl, | |
| lookupFn: makeLookupFn(), | |
| maxBytes: 8, | |
| }); | |
| expect(saved.fileName).toBe(fileName); | |
| }, | |
| ); | |
| it.each([ | |
| { | |
| name: "quoted filename containing a semicolon", | |
| header: 'attachment; filename="quarter;final.csv"', | |
| fileName: "quarter;final.csv", | |
| }, | |
| { | |
| name: "quoted filename containing an escaped quotation mark", | |
| header: String.raw`attachment; filename="quarter\"final.csv"`, | |
| fileName: 'quarter"final.csv', | |
| }, | |
| { | |
| name: "quoted filename containing an escaped semicolon", | |
| header: String.raw`attachment; filename="quarter\;final.csv"`, | |
| fileName: "quarter;final.csv", | |
| }, | |
| { | |
| name: "quoted filename containing an escaped comma", | |
| header: String.raw`attachment; filename="quarter\,final.csv"`, | |
| fileName: "quarter,final.csv", | |
| }, | |
| { | |
| name: "quoted filename containing an escaped space", | |
| header: String.raw`attachment; filename="quarter\ final.csv"`, | |
| fileName: "quarter final.csv", | |
| }, | |
| { | |
| name: "quoted filename containing an escaped hyphen", | |
| header: String.raw`attachment; filename="quarter\-final.csv"`, | |
| fileName: "quarter-final.csv", | |
| }, | |
| { | |
| name: "quoted filename containing multiple escaped punctuation characters", | |
| header: String.raw`attachment; filename="quarter\ final\,v2.csv"`, | |
| fileName: "quarter final,v2.csv", | |
| }, | |
| { | |
| name: "filename text inside an unrelated quoted parameter is ignored", | |
| header: 'attachment; note="x; filename=spoof.csv; y"; filename=safe.csv', | |
| fileName: "safe.csv", | |
| }, | |
| { | |
| name: "Windows drive paths still decode escaped punctuation", | |
| header: String.raw`attachment; filename="C:/tmp/quarter\;final.csv"`, | |
| fileName: "quarter;final.csv", | |
| }, | |
| { | |
| name: "mixed Windows path separators preserve the final basename", | |
| header: String.raw`attachment; filename="C:/tmp/reports\Q1.csv"`, | |
| fileName: "Q1.csv", | |
| }, | |
| { | |
| name: "mixed relative Windows path separators preserve the final basename", | |
| header: String.raw`attachment; filename="reports/2026\Q1.csv"`, | |
| fileName: "Q1.csv", | |
| }, | |
| { | |
| name: "legacy UNC Windows path is reduced to its basename", | |
| header: String.raw`attachment; filename="\\server\share\photo.csv"`, | |
| fileName: "photo.csv", | |
| }, | |
| { | |
| name: "legacy UNC Windows path preserves a Unicode-leading basename", | |
| header: String.raw`attachment; filename="\\server\share\é.csv"`, | |
| fileName: "é.csv", | |
| }, | |
| { | |
| name: "legacy UNC Windows path preserves a punctuation-leading basename", | |
| header: String.raw`attachment; filename="\\server\share\;photo.csv"`, | |
| fileName: ";photo.csv", | |
| }, | |
| { | |
| name: "legacy UNC Windows path preserves a space-leading basename", | |
| header: String.raw`attachment; filename="\\server\share\ photo.csv"`, | |
| fileName: " photo.csv", | |
| }, | |
| { | |
| name: "legacy relative Windows path is reduced to its basename", | |
| header: String.raw`attachment; filename="reports\Q1.csv"`, | |
| fileName: "Q1.csv", | |
| }, | |
| { | |
| name: "nested legacy relative Windows path is reduced to its basename", | |
| header: String.raw`attachment; filename="reports\2026\Q1.csv"`, | |
| fileName: "Q1.csv", | |
| }, | |
| { | |
| name: "UTF-8 filename with a language tag", | |
| header: "attachment; filename*=UTF-8'en'%E2%82%ACrates.csv", | |
| fileName: "€rates.csv", | |
| }, | |
| { | |
| name: "ISO-8859-1 extended filename", | |
| header: "attachment; filename*=ISO-8859-1''caf%E9.csv", | |
| fileName: "café.csv", | |
| }, | |
| { | |
| name: "valid extended filename preferred over plain fallback", | |
| header: "attachment; filename=legacy.csv; filename*=UTF-8'en'%E2%82%ACrates.csv", | |
| fileName: "€rates.csv", | |
| }, | |
| { | |
| name: "malformed extended filename falls back to plain filename", | |
| header: "attachment; filename=fallback.csv; filename*=UTF-8''%ZZbad.csv", | |
| fileName: "fallback.csv", | |
| }, | |
| { | |
| name: "unusable extended filename dot before plain", | |
| header: "attachment; filename*=UTF-8''.; filename=fallback.csv", | |
| fileName: "fallback.csv", | |
| }, | |
| { | |
| name: "unusable extended filename dot after plain", | |
| header: "attachment; filename=fallback.csv; filename*=UTF-8''.", | |
| fileName: "fallback.csv", | |
| }, | |
| { | |
| name: "unusable extended filename encoded parent before plain", | |
| header: "attachment; filename*=UTF-8''%2E%2E; filename=fallback.csv", | |
| fileName: "fallback.csv", | |
| }, | |
| { | |
| name: "unusable extended filename encoded parent after plain", | |
| header: "attachment; filename=fallback.csv; filename*=UTF-8''%2E%2E", | |
| fileName: "fallback.csv", | |
| }, | |
| { | |
| name: "unsupported extended charset falls back to plain filename", | |
| header: "attachment; filename*=UTF-16''bad.csv; filename=fallback.csv", | |
| fileName: "fallback.csv", | |
| }, | |
| ] as const)("parses $name for buffered and stored remote media", async (testCase) => { | |
| const fetchImpl = vi.fn( | |
| async () => | |
| new Response(makeStream([new Uint8Array([1, 2, 3])]), { | |
| status: 200, | |
| headers: { | |
| "content-disposition": testCase.header, | |
| "content-type": "text/csv", | |
| }, | |
| }), | |
| ); | |
| const request = { | |
| url: "https://example.com/download", | |
| fetchImpl, | |
| lookupFn: makeLookupFn(), | |
| maxBytes: 8, | |
| }; | |
| const buffered = await readRemoteMediaBuffer(request); | |
| const stored = await saveRemoteMedia(request); | |
| expect(buffered.fileName).toBe(testCase.fileName); | |
| expect(stored.fileName).toBe(testCase.fileName); | |
| expect(fetchImpl).toHaveBeenCalledTimes(2); | |
| }); | |
| it.each([ | |
| [`attachment; filename*=UTF-8''reports%2FQ1.pdf`, "reports_Q1.pdf"], | |
| [`attachment; filename*=UTF-8''reports%5CQ1.pdf`, "reports_Q1.pdf"], | |
| [`attachment; filename*=UTF-8''reports%2F%2FQ1.pdf`, "reports__Q1.pdf"], | |
| ])( | |
| "keeps decoded content-disposition filename* separators inside the selected filename", | |
| async (contentDisposition, fileName) => { | |
| const fetchImpl = vi.fn( | |
| async () => | |
| new Response(makeStream([new Uint8Array([1, 2, 3])]), { | |
| status: 200, | |
| headers: { | |
| "content-disposition": contentDisposition, | |
| "content-type": "application/pdf", | |
| }, | |
| }), | |
| ); | |
| const saved = await saveRemoteMedia({ | |
| url: "https://example.com/download", | |
| fetchImpl, | |
| lookupFn: makeLookupFn(), | |
| maxBytes: 8, | |
| }); | |
| expect(saved.fileName).toBe(fileName); | |
| }, | |
| ); | |
| it("rejects bodyless successful responses without saving an empty file", async () => { | |
| const inboundDir = path.join(tempHome.home, ".openclaw", "media", "inbound"); | |
| const listInboundFiles = async () => { | |
| try { | |
| return (await fs.readdir(inboundDir)).toSorted(); | |
| } catch (error) { | |
| if (hasErrnoCode(error, "ENOENT")) { | |
| return []; | |
| } | |
| throw error; | |
| } | |
| }; | |
| const before = await listInboundFiles(); | |
| await expect( | |
| saveResponseMedia(new Response(null, { status: 204 }), { | |
| sourceUrl: "https://example.com/empty", | |
| fallbackContentType: "application/octet-stream", | |
| maxBytes: 8, | |
| }), | |
| ).rejects.toMatchObject({ | |
| name: "MediaFetchError", | |
| code: "http_error", | |
| status: 204, | |
| message: | |
| "Failed to fetch media from https://example.com/empty: HTTP 204; empty response body", | |
| }); | |
| await expect(listInboundFiles()).resolves.toEqual(before); | |
| }); | |
| it("uses caller filename hints for MIME detection without preserving storage basenames", async () => { | |
| const contentType = "application/vnd.openxmlformats-officedocument.wordprocessingml.document"; | |
| const fetchImpl = vi.fn( | |
| async () => | |
| new Response(makeStream([new Uint8Array([1, 2, 3])]), { | |
| status: 200, | |
| headers: { "content-type": "application/octet-stream" }, | |
| }), | |
| ); | |
| const saved = await saveRemoteMedia({ | |
| url: "https://smba.trafficmanager.net/v3/attachments/att-1/views/original", | |
| fetchImpl, | |
| lookupFn: makeLookupFn(), | |
| filePathHint: "document.docx", | |
| maxBytes: 8, | |
| }); | |
| expect(saved.fileName).toBe("document.docx"); | |
| expect(saved.contentType).toBe(contentType); | |
| expect(saved.path).toMatch(/[a-f0-9-]{36}\.docx$/); | |
| expect(saved.path).not.toMatch(/document---/); | |
| await expect(fs.readFile(saved.path)).resolves.toStrictEqual(Buffer.from([1, 2, 3])); | |
| }); | |
| it("normalizes Windows-style response filenames and caller hints on POSIX hosts", async () => { | |
| const fetchImpl = vi.fn( | |
| async () => | |
| new Response(makeStream([new Uint8Array([1, 2, 3])]), { | |
| status: 200, | |
| headers: { | |
| "content-disposition": String.raw`attachment; filename="C:\Users\Ada\Downloads\photo.png"`, | |
| "content-type": "application/octet-stream", | |
| }, | |
| }), | |
| ); | |
| const savedFromHeader = await saveRemoteMedia({ | |
| url: "https://example.com/download", | |
| fetchImpl, | |
| lookupFn: makeLookupFn(), | |
| maxBytes: 8, | |
| }); | |
| expect(savedFromHeader.fileName).toBe("photo.png"); | |
| const savedFromHint = await saveRemoteMedia({ | |
| url: "https://example.com/download", | |
| fetchImpl: vi.fn( | |
| async () => | |
| new Response(makeStream([new Uint8Array([1, 2, 3])]), { | |
| status: 200, | |
| headers: { "content-type": "application/octet-stream" }, | |
| }), | |
| ), | |
| lookupFn: makeLookupFn(), | |
| filePathHint: String.raw`C:\Users\Ada\Downloads\document.docx`, | |
| maxBytes: 8, | |
| }); | |
| expect(savedFromHint.fileName).toBe("document.docx"); | |
| expect(savedFromHint.contentType).toBe( | |
| "application/vnd.openxmlformats-officedocument.wordprocessingml.document", | |
| ); | |
| }); | |
| it("does not let filename hints force stored extensions before byte sniffing", async () => { | |
| const jpeg = Buffer.from([0xff, 0xd8, 0xff, 0x00]); | |
| const fetchImpl = vi.fn( | |
| async () => | |
| new Response(makeStream([jpeg]), { | |
| status: 200, | |
| headers: { "content-type": "application/octet-stream" }, | |
| }), | |
| ); | |
| const saved = await saveRemoteMedia({ | |
| url: "https://example.com/views/original", | |
| fetchImpl, | |
| lookupFn: makeLookupFn(), | |
| filePathHint: "document.docx", | |
| maxBytes: 8, | |
| }); | |
| expect(saved.fileName).toBe("document.docx"); | |
| expect(saved.contentType).toBe("image/jpeg"); | |
| expect(saved.path).toMatch(/[a-f0-9-]{36}\.jpg$/); | |
| expect(saved.path).not.toMatch(/\.docx$/); | |
| expect(saved.path).not.toMatch(/document---/); | |
| await expect(fs.readFile(saved.path)).resolves.toStrictEqual(jpeg); | |
| }); | |
| it("preserves explicit original filenames when saving streams", async () => { | |
| const contentType = "application/vnd.openxmlformats-officedocument.wordprocessingml.document"; | |
| const fetchImpl = vi.fn( | |
| async () => | |
| new Response(makeStream([new Uint8Array([1, 2, 3])]), { | |
| status: 200, | |
| headers: { "content-type": "application/octet-stream" }, | |
| }), | |
| ); | |
| const saved = await saveRemoteMedia({ | |
| url: "https://smba.trafficmanager.net/v3/attachments/att-1/views/original", | |
| fetchImpl, | |
| lookupFn: makeLookupFn(), | |
| filePathHint: "document.docx", | |
| fallbackContentType: contentType, | |
| originalFilename: "document.docx", | |
| maxBytes: 8, | |
| }); | |
| expect(saved.fileName).toBe("document.docx"); | |
| expect(saved.contentType).toBe(contentType); | |
| expect(saved.path).toMatch(/document---.+\.docx$/); | |
| }); | |
| it("uses fallback content type when streamed response headers are generic", async () => { | |
| const contentType = "application/vnd.openxmlformats-officedocument.wordprocessingml.document"; | |
| const fetchImpl = vi.fn( | |
| async () => | |
| new Response(makeStream([new Uint8Array([4, 5, 6])]), { | |
| status: 200, | |
| headers: { "content-type": "application/octet-stream" }, | |
| }), | |
| ); | |
| const saved = await saveRemoteMedia({ | |
| url: "https://example.com/views/original", | |
| fetchImpl, | |
| lookupFn: makeLookupFn(), | |
| filePathHint: "document", | |
| fallbackContentType: contentType, | |
| maxBytes: 8, | |
| }); | |
| expect(saved.fileName).toBe("document"); | |
| expect(saved.contentType).toBe(contentType); | |
| expect(saved.path).toMatch(/[a-f0-9-]{36}\.docx$/); | |
| expect(saved.path).not.toMatch(/document---/); | |
| }); | |
| it("uses audio fallback content type when streamed response headers report matching video container", async () => { | |
| const fetchImpl = vi.fn( | |
| async () => | |
| new Response(makeStream([new Uint8Array([7, 8, 9])]), { | |
| status: 200, | |
| headers: { "content-type": "video/mp4" }, | |
| }), | |
| ); | |
| const saved = await saveRemoteMedia({ | |
| url: "https://example.com/voice.mp4", | |
| fetchImpl, | |
| lookupFn: makeLookupFn(), | |
| filePathHint: "voice.mp4", | |
| fallbackContentType: "audio/mp4", | |
| maxBytes: 8, | |
| }); | |
| expect(saved.contentType).toBe("audio/mp4"); | |
| expect(saved.path).toMatch(/[a-f0-9-]{36}\.m4a$/); | |
| }); | |
| it("cancels streamed response bodies when media save exceeds maxBytes", async () => { | |
| const cancel = vi.fn(); | |
| const fetchImpl = vi.fn( | |
| async () => | |
| new Response( | |
| new ReadableStream<Uint8Array>({ | |
| start(controller) { | |
| controller.enqueue(new Uint8Array([1, 2, 3])); | |
| controller.enqueue(new Uint8Array([4, 5, 6])); | |
| }, | |
| cancel, | |
| }), | |
| { status: 200 }, | |
| ), | |
| ); | |
| await expect( | |
| saveRemoteMedia({ | |
| url: "https://example.com/large.bin", | |
| fetchImpl, | |
| lookupFn: makeLookupFn(), | |
| maxBytes: 4, | |
| }), | |
| ).rejects.toThrow("exceeds maxBytes"); | |
| expect(cancel).toHaveBeenCalledTimes(1); | |
| }); | |
| it.each(["streamed", "content-length"])( | |
| "cleans up %s media overflow before a response clone is released", | |
| async (kind) => { | |
| const body = makeCancelableStream([new Uint8Array([1, 2, 3, 4, 5])]); | |
| const response = new Response(body.stream, { | |
| headers: kind === "content-length" ? { "content-length": "5" } : {}, | |
| }); | |
| const capture = response.clone(); | |
| const subdir = `captured-${kind}`; | |
| let completed = false; | |
| const operation = saveRemoteMedia({ | |
| url: "https://example.com/large.bin", | |
| fetchImpl: async () => response, | |
| lookupFn: makeLookupFn(), | |
| maxBytes: 4, | |
| subdir, | |
| }) | |
| .catch((error: unknown) => error) | |
| .finally(() => { | |
| completed = true; | |
| }); | |
| try { | |
| await vi.waitFor(() => expect(completed).toBe(true), { timeout: 500 }); | |
| await expect(operation).resolves.toMatchObject({ code: "max_bytes" }); | |
| expect(response.body?.locked).toBe(false); | |
| expect(body.wasCanceled()).toBe(false); | |
| const dir = path.join(tempHome.home, ".openclaw", "media", subdir); | |
| await expect( | |
| fs.readdir(dir).catch((error: unknown) => { | |
| if (hasErrnoCode(error, "ENOENT")) { | |
| return []; | |
| } | |
| throw error; | |
| }), | |
| ).resolves.toEqual([]); | |
| } finally { | |
| await capture.body?.cancel(); | |
| await operation; | |
| } | |
| expect(body.wasCanceled()).toBe(true); | |
| }, | |
| ); | |
| it("retries saveRemoteMedia after a transient fetch failure", async () => { | |
| const transientError = Object.assign(new TypeError("socket reset"), { code: "ECONNRESET" }); | |
| const fetchImpl = vi | |
| .fn() | |
| .mockRejectedValueOnce(transientError) | |
| .mockResolvedValueOnce( | |
| new Response(makeStream([new Uint8Array([5, 6])]), { | |
| status: 200, | |
| headers: { "content-type": "image/png" }, | |
| }), | |
| ); | |
| const onRetry = vi.fn(); | |
| const beforeRequest = vi.fn(); | |
| const saved = await saveRemoteMedia({ | |
| url: "https://example.com/retry.png", | |
| fetchImpl, | |
| beforeRequest, | |
| lookupFn: makeLookupFn(), | |
| maxBytes: 8, | |
| retry: { attempts: 2, minDelayMs: 0, maxDelayMs: 0, jitter: 0, onRetry }, | |
| }); | |
| expect(fetchImpl).toHaveBeenCalledTimes(2); | |
| expect(beforeRequest).toHaveBeenCalledTimes(2); | |
| expect(onRetry).toHaveBeenCalledTimes(1); | |
| expect(saved.contentType).toBe("image/png"); | |
| await expect(fs.readFile(saved.path)).resolves.toStrictEqual(Buffer.from([5, 6])); | |
| }); | |
| it("does not retry permanent media limit failures", async () => { | |
| const fetchImpl = vi.fn( | |
| async () => | |
| new Response(makeStream([new Uint8Array([1, 2, 3, 4, 5])]), { | |
| status: 200, | |
| headers: { "content-length": "5" }, | |
| }), | |
| ); | |
| await expect( | |
| saveRemoteMedia({ | |
| url: "https://example.com/too-large.bin", | |
| fetchImpl, | |
| lookupFn: makeLookupFn(), | |
| maxBytes: 4, | |
| retry: { attempts: 2, minDelayMs: 0, maxDelayMs: 0, jitter: 0 }, | |
| }), | |
| ).rejects.toThrow("exceeds maxBytes"); | |
| expect(fetchImpl).toHaveBeenCalledTimes(1); | |
| }); | |
| }); | |
| /* oxlint-disable max-lines -- TODO: split this grandfathered oversized file. */ | |