Skip to content

Commit

Permalink
fix: add timeout and data limit to link verification fetches.
Browse files Browse the repository at this point in the history
  • Loading branch information
zicklag committed Jan 10, 2025
1 parent 94e6482 commit f6a42cb
Show file tree
Hide file tree
Showing 2 changed files with 43 additions and 1 deletion.
41 changes: 41 additions & 0 deletions src/lib/limited-fetch.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,41 @@
export async function limitedFetch(
options: { timeout: number; maxSize: number },
...fetchArgs: Parameters<typeof fetch>
): Promise<Response> {
const signal = AbortSignal.timeout(options.timeout);
const resp = await fetch(fetchArgs[0], { ...fetchArgs[1], signal });

const reader = resp.body?.getReader();
if (!reader) return resp;

let bytesSoFar = 0;
const stream = new ReadableStream({
start(controller) {
return pump();
async function pump(): Promise<typeof pump | undefined> {
const { done, value } = await reader!.read();
// When no more data needs to be consumed, close the stream
if (done) {
controller.close();
return;
}
bytesSoFar += value?.length;
if (bytesSoFar > options.maxSize) {
throw new Error(
`Could not fetch URL ( ${fetchArgs[0]} ) because returned number of \
bytes exceeded limit of ${options.maxSize} bytes.`
);
}
// Enqueue the next data chunk into our target stream
controller.enqueue(value);
return pump();
}
}
});

return new Response(stream, {
headers: resp.headers,
status: resp.status,
statusText: resp.statusText
});
}
3 changes: 2 additions & 1 deletion src/lib/link_verifier/LinkVerifier.ts
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@ import { parseHTML } from 'linkedom';

import type { LinkVerificationStrategyFactory } from './strategy/LinkVerificationStrategy';
import { DefaultLinkVerificationStrategy } from './strategy/DefaultLinkVerificationStrategy';
import { limitedFetch } from '$lib/limited-fetch';

export const VERIFIABLE_ORIGIN_STRATEGY: Record<string, LinkVerificationStrategyFactory> = {
// Put custom link verifiers for specific domains here once we have them.
Expand All @@ -20,7 +21,7 @@ export class LinkVerifier {
}

private static async fetchHtml(webLink: string): Promise<Window> {
const res = await fetch(webLink);
const res = await limitedFetch({ timeout: 3 * 1000, maxSize: 1 * 1024 * 1024 }, webLink);

if (res.status === 200) {
const resText = await res.text();
Expand Down

0 comments on commit f6a42cb

Please sign in to comment.