File size: 4,841 Bytes
6f91ce5
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
import { createHash, randomBytes } from 'node:crypto'
import { createReadStream, existsSync, mkdirSync, renameSync, statSync, writeFileSync } from 'node:fs'
import type { ServerResponse } from 'node:http'
import { join, resolve } from 'node:path'
import { pipeline } from 'node:stream/promises'
import { slug } from '../shared/ids.ts'
import { assertPublicUrl } from './netGuard.ts'

// Open-access PDFs for the paper widgets (T10), proxied because publishers don't send CORS headers.
// The link comes from paper metadata, so it is untrusted: https only, public addresses only (checked
// again on every redirect, at most 3), `%PDF` magic bytes, 50 MB at most. Cached under DATA_DIR/pdfs.

export const MAX_PDF_BYTES = 50 * 1024 * 1024
const MAX_REDIRECTS = 3
const TIMEOUT_MS = 60_000

let pdfDir = join(resolve(process.env.DATA_DIR ?? '.data'), 'pdfs')

/** Tests point the cache at a temp dir. */
export function setPdfDir(dir: string) {
	pdfDir = dir
}

/** `refused`: the link itself isn't allowed (scheme, address); otherwise fetching it failed. */
export class PdfError extends Error {
	refused: boolean
	constructor(message: string, refused = false) {
		super(message)
		this.refused = refused
	}
}

/** Throws a refused PdfError unless the URL is https on a public address. */
export async function checkPdfUrl(url: URL) {
	if (url.protocol !== 'https:') throw new PdfError(`Only https PDF links are fetched, not ${url.protocol}`, true)
	try {
		await assertPublicUrl(url)
	} catch (e) {
		throw new PdfError(e instanceof Error ? e.message : String(e), true)
	}
}

export interface FetchPdfOptions {
	/** Run on the link and every redirect; tests swap it to reach a local server. */
	check?: (url: URL) => Promise<void>
	maxBytes?: number
}

export async function fetchPdf(link: string, { check = checkPdfUrl, maxBytes = MAX_PDF_BYTES }: FetchPdfOptions = {}) {
	let url: URL
	try {
		url = new URL(link)
	} catch {
		throw new PdfError(`Not a URL: ${link.slice(0, 200)}`, true)
	}
	const signal = AbortSignal.timeout(TIMEOUT_MS)
	for (let hop = 0; ; hop++) {
		await check(url)
		let res: Response
		try {
			res = await fetch(url, { redirect: 'manual', signal, headers: { accept: 'application/pdf' } })
		} catch (e) {
			throw new PdfError(`Couldn't fetch ${url.host}: ${e instanceof Error ? e.message : e}`)
		}
		const location = res.headers.get('location')
		if (res.status >= 300 && res.status < 400 && location) {
			await res.body?.cancel()
			if (hop >= MAX_REDIRECTS) throw new PdfError(`More than ${MAX_REDIRECTS} redirects`)
			url = new URL(location, url)
			continue
		}
		if (!res.ok) {
			await res.body?.cancel()
			throw new PdfError(`${url.host} answered ${res.status}`)
		}
		if (Number(res.headers.get('content-length')) > maxBytes) {
			await res.body?.cancel()
			throw new PdfError(`The PDF is larger than ${maxBytes} bytes`)
		}
		const pdf = await readCapped(res, maxBytes, url)
		if (!pdf.subarray(0, 1024).includes('%PDF-')) throw new PdfError(`${url.host} didn't send a PDF`)
		return pdf
	}
}

async function readCapped(res: Response, maxBytes: number, url: URL): Promise<Buffer> {
	const reader = res.body?.getReader()
	const chunks: Uint8Array[] = []
	let size = 0
	try {
		for (let read = await reader?.read(); read && !read.done; read = await reader!.read()) {
			size += read.value.length
			if (size > maxBytes) {
				await reader!.cancel()
				throw new PdfError(`The PDF is larger than ${maxBytes} bytes`)
			}
			chunks.push(read.value)
		}
	} catch (e) {
		if (e instanceof PdfError) throw e
		throw new PdfError(`Couldn't read the PDF from ${url.host}: ${e instanceof Error ? e.message : e}`)
	}
	return Buffer.concat(chunks)
}

// slugs are lossy, so a short hash of the paper ID keeps two papers from sharing a file
const cacheFile = (paperId: string) =>
	join(pdfDir, `${slug(paperId).slice(0, 100)}-${createHash('sha256').update(paperId).digest('hex').slice(0, 8)}.pdf`)

/**
 * Sends the paper's PDF, fetching it from `link()` and caching it the first time. Throws (a PdfError
 * for the PDF itself) before writing anything.
 */
export async function servePdf(res: ServerResponse, paperId: string, link: () => Promise<string>, options?: FetchPdfOptions) {
	const file = cacheFile(paperId)
	if (!existsSync(file)) {
		const pdf = await fetchPdf(await link(), options)
		mkdirSync(pdfDir, { recursive: true })
		// written aside and renamed, so a half-written file is never served
		const part = `${file}.${randomBytes(4).toString('hex')}.part`
		writeFileSync(part, pdf)
		renameSync(part, file)
	}
	res.writeHead(200, {
		'content-type': 'application/pdf',
		'content-length': statSync(file).size,
		'cache-control': 'private, max-age=86400',
		'content-security-policy': "default-src 'none'",
		'x-content-type-options': 'nosniff',
	})
	await pipeline(createReadStream(file), res)
}