Download apps/api/app/core/text.py from aliroya/contiai: direct link, hf CLI and curl.
- Browser
- Download file 1.5 kB
-
https://huggingface.co/spaces/aliroya/contiai/resolve/main/apps/api/app/core/text.py
- Command line
-
hf download hf://spaces/aliroya/contiai/apps/api/app/core/text.py
-
curl -L -o text.py https://huggingface.co/spaces/aliroya/contiai/resolve/main/apps/api/app/core/text.py
1.5 kB
| """Making extracted text safe to store. | |
| PDF text extraction is not guaranteed to produce valid text. Malformed fonts, | |
| broken encodings and embedded binary all leak through as C0 control bytes, and | |
| one of them - NUL - cannot be stored in a Postgres ``text`` column at all: | |
| APIError 22P05: unsupported Unicode escape sequence | |
| \\u0000 cannot be converted to text | |
| The insert raises, the exception propagates out of the fetch worker, and the | |
| document is marked rejected. A source that was found, downloaded, OCR'd and | |
| paid for is thrown away over a byte that carries no meaning in the first place. | |
| """ | |
| from __future__ import annotations | |
| # Everything below 0x20 except tab, newline and carriage return, plus DEL. | |
| # These are structural artefacts of a bad extraction, never content. | |
| _STRIP = {c: None for c in range(0x20) if c not in (0x09, 0x0A, 0x0D)} | |
| _STRIP[0x7F] = None | |
| def clean(value: str) -> str: | |
| """Drop control characters a text column cannot hold.""" | |
| return value.translate(_STRIP) | |
| def clean_deep(value): | |
| """The same, everywhere inside a nested structure. | |
| Applied at the storage boundary so it holds for every caller, including | |
| ones added later that have no reason to know about any of this. | |
| """ | |
| if isinstance(value, str): | |
| return clean(value) | |
| if isinstance(value, dict): | |
| return {k: clean_deep(v) for k, v in value.items()} | |
| if isinstance(value, (list, tuple)): | |
| return [clean_deep(v) for v in value] | |
| return value | |