File size: 4,841 Bytes
6f91ce5 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 | import { createHash, randomBytes } from 'node:crypto'
import { createReadStream, existsSync, mkdirSync, renameSync, statSync, writeFileSync } from 'node:fs'
import type { ServerResponse } from 'node:http'
import { join, resolve } from 'node:path'
import { pipeline } from 'node:stream/promises'
import { slug } from '../shared/ids.ts'
import { assertPublicUrl } from './netGuard.ts'
// Open-access PDFs for the paper widgets (T10), proxied because publishers don't send CORS headers.
// The link comes from paper metadata, so it is untrusted: https only, public addresses only (checked
// again on every redirect, at most 3), `%PDF` magic bytes, 50 MB at most. Cached under DATA_DIR/pdfs.
export const MAX_PDF_BYTES = 50 * 1024 * 1024
const MAX_REDIRECTS = 3
const TIMEOUT_MS = 60_000
let pdfDir = join(resolve(process.env.DATA_DIR ?? '.data'), 'pdfs')
/** Tests point the cache at a temp dir. */
export function setPdfDir(dir: string) {
pdfDir = dir
}
/** `refused`: the link itself isn't allowed (scheme, address); otherwise fetching it failed. */
export class PdfError extends Error {
refused: boolean
constructor(message: string, refused = false) {
super(message)
this.refused = refused
}
}
/** Throws a refused PdfError unless the URL is https on a public address. */
export async function checkPdfUrl(url: URL) {
if (url.protocol !== 'https:') throw new PdfError(`Only https PDF links are fetched, not ${url.protocol}`, true)
try {
await assertPublicUrl(url)
} catch (e) {
throw new PdfError(e instanceof Error ? e.message : String(e), true)
}
}
export interface FetchPdfOptions {
/** Run on the link and every redirect; tests swap it to reach a local server. */
check?: (url: URL) => Promise<void>
maxBytes?: number
}
export async function fetchPdf(link: string, { check = checkPdfUrl, maxBytes = MAX_PDF_BYTES }: FetchPdfOptions = {}) {
let url: URL
try {
url = new URL(link)
} catch {
throw new PdfError(`Not a URL: ${link.slice(0, 200)}`, true)
}
const signal = AbortSignal.timeout(TIMEOUT_MS)
for (let hop = 0; ; hop++) {
await check(url)
let res: Response
try {
res = await fetch(url, { redirect: 'manual', signal, headers: { accept: 'application/pdf' } })
} catch (e) {
throw new PdfError(`Couldn't fetch ${url.host}: ${e instanceof Error ? e.message : e}`)
}
const location = res.headers.get('location')
if (res.status >= 300 && res.status < 400 && location) {
await res.body?.cancel()
if (hop >= MAX_REDIRECTS) throw new PdfError(`More than ${MAX_REDIRECTS} redirects`)
url = new URL(location, url)
continue
}
if (!res.ok) {
await res.body?.cancel()
throw new PdfError(`${url.host} answered ${res.status}`)
}
if (Number(res.headers.get('content-length')) > maxBytes) {
await res.body?.cancel()
throw new PdfError(`The PDF is larger than ${maxBytes} bytes`)
}
const pdf = await readCapped(res, maxBytes, url)
if (!pdf.subarray(0, 1024).includes('%PDF-')) throw new PdfError(`${url.host} didn't send a PDF`)
return pdf
}
}
async function readCapped(res: Response, maxBytes: number, url: URL): Promise<Buffer> {
const reader = res.body?.getReader()
const chunks: Uint8Array[] = []
let size = 0
try {
for (let read = await reader?.read(); read && !read.done; read = await reader!.read()) {
size += read.value.length
if (size > maxBytes) {
await reader!.cancel()
throw new PdfError(`The PDF is larger than ${maxBytes} bytes`)
}
chunks.push(read.value)
}
} catch (e) {
if (e instanceof PdfError) throw e
throw new PdfError(`Couldn't read the PDF from ${url.host}: ${e instanceof Error ? e.message : e}`)
}
return Buffer.concat(chunks)
}
// slugs are lossy, so a short hash of the paper ID keeps two papers from sharing a file
const cacheFile = (paperId: string) =>
join(pdfDir, `${slug(paperId).slice(0, 100)}-${createHash('sha256').update(paperId).digest('hex').slice(0, 8)}.pdf`)
/**
* Sends the paper's PDF, fetching it from `link()` and caching it the first time. Throws (a PdfError
* for the PDF itself) before writing anything.
*/
export async function servePdf(res: ServerResponse, paperId: string, link: () => Promise<string>, options?: FetchPdfOptions) {
const file = cacheFile(paperId)
if (!existsSync(file)) {
const pdf = await fetchPdf(await link(), options)
mkdirSync(pdfDir, { recursive: true })
// written aside and renamed, so a half-written file is never served
const part = `${file}.${randomBytes(4).toString('hex')}.part`
writeFileSync(part, pdf)
renameSync(part, file)
}
res.writeHead(200, {
'content-type': 'application/pdf',
'content-length': statSync(file).size,
'cache-control': 'private, max-age=86400',
'content-security-policy': "default-src 'none'",
'x-content-type-options': 'nosniff',
})
await pipeline(createReadStream(file), res)
}
|