"""The sample pages the reference is made from, all from Unlimited-OCR's own paper (MIT, in the makers' repository): paper-p1, paper-p3, paper-p5 three of its pages at 200 dpi — a title page with a figure, a page of prose and maths, a page with figures and their captions small-print its text from pages 3 to 9 set on one page in small print: a page long enough that the model writes past its 128-token window many times over two-columns its text from pages 10 to 12 in two columns, scanned: a page read across its columns no-text shapes and gradients with no writing, for the text finder make_samples.py """ import io import random import sys from pathlib import Path import pymupdf from PIL import Image, ImageDraw, ImageFilter, ImageFont pdf, out, font_path = Path(sys.argv[1]), Path(sys.argv[2]), sys.argv[3] out.mkdir(parents=True, exist_ok=True) doc = pymupdf.open(str(pdf)) for number in (1, 3, 5): doc[number - 1].get_pixmap(matrix=pymupdf.Matrix(200 / 72, 200 / 72)).save(str(out / f"paper-p{number}.png")) def words(first, last): return " ".join(" ".join(doc[i - 1].get_text().split()) for i in range(first, last + 1)).split() def wrap(draw, ws, font, width): lines, line = [], "" for word in ws: trial = f"{line} {word}".strip() if draw.textlength(trial, font=font) <= width: line = trial else: lines.append(line) line = word return lines + ([line] if line else []) page = Image.new("RGB", (2480, 3508), "white") draw = ImageDraw.Draw(page) font = ImageFont.truetype(font_path, 30) y = 140 for line in wrap(draw, words(3, 9), font, 2200)[:88]: draw.text((140, y), line, fill="black", font=font) y += 37 page.save(out / "small-print.png") random.seed(7) page = Image.new("RGB", (1700, 2200), "white") draw = ImageDraw.Draw(page) font = ImageFont.truetype(font_path, 26) lines = wrap(draw, words(10, 12), font, 700) for column, x in ((lines[:62], 100), (lines[62:124], 900)): y = 120 for line in column: draw.text((x, y), line, fill="black", font=font) y += 32 page = page.rotate(0.6, resample=Image.BICUBIC, fillcolor=(250, 250, 248)).filter(ImageFilter.GaussianBlur(0.6)) pixels = page.load() for _ in range(page.width * page.height // 60): px, py = random.randrange(page.width), random.randrange(page.height) v = max(0, min(255, pixels[px, py][0] + random.randint(-30, 30))) pixels[px, py] = (v, v, v) buffer = io.BytesIO() page.convert("L").save(buffer, "JPEG", quality=60) Image.open(io.BytesIO(buffer.getvalue())).convert("RGB").save(out / "two-columns.png") page = Image.new("RGB", (1600, 1200)) draw = ImageDraw.Draw(page) for row in range(1200): draw.line([(0, row), (1600, row)], fill=(40 + row // 8, 90 + row // 12, 160 - row // 10)) for i in range(14): cx, cy, r = random.randrange(1600), random.randrange(1200), random.randrange(30, 220) draw.ellipse([cx - r, cy - r, cx + r, cy + r], fill=(random.randrange(256), random.randrange(256), random.randrange(256))) page.filter(ImageFilter.GaussianBlur(3)).save(out / "no-text.png") print(sorted(p.name for p in out.iterdir()))