"""Making extracted text safe to store. PDF text extraction is not guaranteed to produce valid text. Malformed fonts, broken encodings and embedded binary all leak through as C0 control bytes, and one of them - NUL - cannot be stored in a Postgres ``text`` column at all: APIError 22P05: unsupported Unicode escape sequence \\u0000 cannot be converted to text The insert raises, the exception propagates out of the fetch worker, and the document is marked rejected. A source that was found, downloaded, OCR'd and paid for is thrown away over a byte that carries no meaning in the first place. """ from __future__ import annotations # Everything below 0x20 except tab, newline and carriage return, plus DEL. # These are structural artefacts of a bad extraction, never content. _STRIP = {c: None for c in range(0x20) if c not in (0x09, 0x0A, 0x0D)} _STRIP[0x7F] = None def clean(value: str) -> str: """Drop control characters a text column cannot hold.""" return value.translate(_STRIP) def clean_deep(value): """The same, everywhere inside a nested structure. Applied at the storage boundary so it holds for every caller, including ones added later that have no reason to know about any of this. """ if isinstance(value, str): return clean(value) if isinstance(value, dict): return {k: clean_deep(v) for k, v in value.items()} if isinstance(value, (list, tuple)): return [clean_deep(v) for v in value] return value