Hugging Face's logo

Spaces:
validops-east-3
/
instance-2
Sleeping

App Files Community
validops-east-3 commited on
Commit
efff1ff
·
verified ·
1 Parent(s): 2d40542

Deploy 72701a1d9ec33b1842d0a516ffdee753aaa754a9

Browse files
Files changed (35) hide show
  1. Dockerfile +40 -117
  2. README.md +105 -6
  3. docqa-rs/Cargo.lock +0 -2376
  4. docqa-rs/Cargo.toml +0 -47
  5. docqa-rs/Dockerfile +0 -78
  6. docqa-rs/README.md +0 -232
  7. docqa-rs/docqa.postman_collection.json +0 -152
  8. docqa-rs/models/README.md +0 -48
  9. docqa-rs/scripts/accuracy.py +0 -151
  10. docqa-rs/scripts/e2e.py +0 -221
  11. docqa-rs/scripts/make_boxes.py +0 -142
  12. docqa-rs/src/api.rs +0 -474
  13. docqa-rs/src/config.rs +0 -54
  14. docqa-rs/src/engine.rs +0 -428
  15. docqa-rs/src/main.rs +0 -111
  16. docqa-rs/src/parser.rs +0 -576
  17. docqa-rs/src/pool.rs +0 -93
  18. docqa.postman_collection.json +149 -0
  19. docqa/bench/compare_backends.py +218 -0
  20. docqa/bench/export_onnx.py +159 -0
  21. docqa/corpus/ground_truth.json +266 -0
  22. docqa/docxextract/__init__.py +25 -0
  23. docqa/docxextract/api.py +198 -0
  24. docqa/docxextract/config.py +66 -0
  25. docqa/docxextract/engine.py +225 -0
  26. docqa/docxextract/logging.py +61 -0
  27. docqa/docxextract/onnx_engine.py +243 -0
  28. docqa/docxextract/parsing.py +314 -0
  29. docqa/docxextract/schema.py +135 -0
  30. docqa/docxextract/service.py +137 -0
  31. docqa/tests/conftest.py +47 -0
  32. docqa/tests/corpus.py +416 -0
  33. docqa/tests/test_api.py +133 -0
  34. docqa/tests/test_unit.py +178 -0
  35. requirements.txt +30 -0
Dockerfile CHANGED
@@ -1,136 +1,59 @@
1
- # ============================================================================
2
- # docqa — Document Data Extraction API (Rust + ONNX Runtime)
3
  #
4
- # Self-contained: Hugging Face Spaces builds only this file, so the Rust
5
- # binary is compiled here. docqa-rs/Dockerfile builds the same thing for
6
- # local testing but is NOT used by the Space.
7
  #
8
- # Exposes 7860, the port this Space previously served llama-server on, so no
9
  # Space port setting changes are needed.
10
- # ============================================================================
11
- #
12
- # --- PREVIOUS SERVICE: llama-server (Qwen3-0.6B) --------------------------
13
- # Retained for rollback, not built. To restore it, delete the docqa-rs build
14
- # stages below and uncomment this block.
15
- #
16
- # FROM ghcr.io/ggml-org/llama.cpp:server
17
- #
18
- # RUN mkdir -p /models && \
19
- # curl -fL --retry 3 -o /models/Qwen3-0.6B-Q8_0.gguf \
20
- # "https://huggingface.co/Qwen/Qwen3-0.6B-GGUF/resolve/main/Qwen3-0.6B-Q8_0.gguf" && \
21
- # ls -lh /models
22
- #
23
- # COPY .env /app/.env
24
- #
25
- # HEALTHCHECK --interval=30s --timeout=5s --start-period=120s --retries=6 \
26
- # CMD curl -fsS http://127.0.0.1:7860/health || exit 1
27
- #
28
- # ENTRYPOINT ["/bin/sh", "-c", "if [ -z \"$LLAMA_API_KEY\" ]; then set -a; . /app/.env; set +a; fi; exec /app/llama-server \"$@\"", "--"]
29
- #
30
- # CMD ["-m", "/models/Qwen3-0.6B-Q8_0.gguf", \
31
- # "--host", "0.0.0.0", \
32
- # "--port", "7860", \
33
- # "-c", "32768", \
34
- # "-np", "2", \
35
- # "-t", "2", \
36
- # "-tb", "4", \
37
- # "--jinja", \
38
- # "--alias", "Qwen3-0.6B"]
39
- # ---------------------------------------------------------------------------
40
-
41
- # ---- stage 1: compile the Rust binary -------------------------------------
42
- FROM rust:1.83-slim-bookworm AS builder
43
-
44
- WORKDIR /src
45
-
46
- # The ONNX Runtime shared library. `ort` is built with the `load-dynamic`
47
- # feature, so the binary resolves this at startup instead of linking it.
48
- ARG ORT_VERSION=1.20.1
49
- RUN curl -fsSL -o /tmp/ort.tgz \
50
- "https://github.com/microsoft/onnxruntime/releases/download/v${ORT_VERSION}/onnxruntime-linux-x64-${ORT_VERSION}.tgz" \
51
- && mkdir -p /opt/onnxruntime \
52
- && tar -xzf /tmp/ort.tgz -C /opt/onnxruntime --strip-components=1 \
53
- && rm /tmp/ort.tgz
54
-
55
- # Copy manifests first so dependency compilation is cached independently of
56
- # source edits.
57
- COPY docqa-rs/Cargo.toml docqa-rs/Cargo.lock ./
58
- COPY docqa-rs/src ./src
59
-
60
- # Release profile: opt-level 3 with LTO. The model is compute-bound, so this
61
- # trims inference time rather than binary size.
62
- ENV CARGO_PROFILE_RELEASE_LTO=true \
63
- CARGO_PROFILE_RELEASE_CODEGEN_UNITS=1
64
 
65
- RUN cargo build --release --locked
66
-
67
- # ---- stage 2: model + tokenizer ------------------------------------------
68
- # Kept separate so editing Rust source does not re-download 485 MB.
69
- #
70
- # The ONNX graph is NOT in this git repo: 485 MB exceeds GitHub's 100 MB
71
- # per-file limit, so it is gitignored (see .gitignore and
72
- # docqa-rs/models/README.md). It is therefore fetched at build time.
73
- #
74
- # ONE-TIME SETUP: upload your exported layoutlm.onnx to any Hugging Face repo
75
- # and point MODEL_URL at it, e.g.
76
- # docker build --build-arg \
77
- # MODEL_URL=https://huggingface.co/<you>/layoutlm-onnx/resolve/main/model.onnx .
78
- # The default below is the model repo itself and will 404 until you add it.
79
- FROM debian:bookworm-slim AS models
80
-
81
- RUN apt-get update && apt-get install -y --no-install-recommends \
82
- ca-certificates curl \
83
- && rm -rf /var/lib/apt/lists/*
84
-
85
- ARG HF_REPO=impira/layoutlm-invoices
86
- ARG MODEL_URL=https://huggingface.co/impira/layoutlm-invoices/resolve/main/model.onnx
87
-
88
- RUN set -eux; \
89
- curl -fsSL -o /layoutlm.onnx "$MODEL_URL"; \
90
- test -s /layoutlm.onnx || { echo "model.onnx is empty or missing"; exit 1; }; \
91
- curl -fsSL -o /tokenizer.json "https://huggingface.co/${HF_REPO}/resolve/main/tokenizer.json"; \
92
- test -s /tokenizer.json || { echo "tokenizer.json is empty or missing"; exit 1; }; \
93
- ls -lh /layoutlm.onnx /tokenizer.json
94
-
95
- # ---- stage 3: runtime -----------------------------------------------------
96
- FROM debian:bookworm-slim
97
 
 
 
98
  RUN apt-get update && apt-get install -y --no-install-recommends \
99
- ca-certificates \
100
- libgomp1 \
101
  poppler-utils \
102
  tesseract-ocr \
103
  tesseract-ocr-eng \
104
  curl \
 
105
  && rm -rf /var/lib/apt/lists/*
106
 
107
- COPY --from=builder /opt/onnxruntime /opt/onnxruntime
108
- COPY --from=builder /src/target/release/docqa /usr/local/bin/docqa
109
- COPY --from=models /layoutlm.onnx /app/models/layoutlm.onnx
110
- COPY --from=models /tokenizer.json /app/models/tokenizer.json
111
-
112
  WORKDIR /app
113
 
114
- ENV DOCQA_BIND=0.0.0.0 \
115
- DOCQA_PORT=7860 \
116
- DOCQA_MODEL_PATH=/app/models/layoutlm.onnx \
117
- DOCQA_TOKENIZER_PATH=/app/models/tokenizer.json \
118
- DOCQA_MODEL_ID=impira/layoutlm-invoices \
119
- DOCQA_WORKERS=2 \
120
- DOCQA_THREADS=4 \
121
- DOCQA_CONFIDENCE_THRESHOLD=0.5 \
122
- ORT_DYLIB_PATH=/opt/onnxruntime/lib/libonnxruntime.so \
123
- LD_LIBRARY_PATH=/opt/onnxruntime/lib \
124
- RUST_LOG=docqa=info,tower_http=warn
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
125
 
126
  EXPOSE 7860
127
 
128
- # The model loads before the listener binds, so a passing health check means
129
- # the engine is genuinely ready rather than merely that the process is alive.
130
- # start-period covers the 485 MB fp32 model load on a cold Space.
131
- HEALTHCHECK --interval=30s --timeout=5s --start-period=240s --retries=6 \
132
  CMD curl -fsS http://127.0.0.1:7860/health || exit 1
133
 
134
- # layoutlm.onnx and tokenizer.json are fetched in the `models` stage above,
135
- # because the ONNX graph is too large to keep in this git repository.
136
- CMD ["docqa"]
 
1
+ # docqa — Document Data Extraction API
 
2
  #
3
+ # Loads impira/layoutlm-invoices straight from the Hugging Face cache, so
4
+ # there is no model artifact to host in this repository.
 
5
  #
6
+ # Serves on 7860, the port this Space previously used for llama-server, so no
7
  # Space port setting changes are needed.
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
8
 
9
+ FROM python:3.11-slim-bookworm
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
10
 
11
+ # poppler-utils -> pdftotext/pdftoppm for the PDF text layer and rasterising
12
+ # scans; tesseract-ocr -> OCR for images and scanned pages.
13
  RUN apt-get update && apt-get install -y --no-install-recommends \
 
 
14
  poppler-utils \
15
  tesseract-ocr \
16
  tesseract-ocr-eng \
17
  curl \
18
+ libgomp1 \
19
  && rm -rf /var/lib/apt/lists/*
20
 
 
 
 
 
 
21
  WORKDIR /app
22
 
23
+ COPY requirements.txt ./
24
+ RUN pip install --no-cache-dir -r requirements.txt
25
+
26
+ COPY docqa ./docqa
27
+ ENV PYTHONPATH=/app/docqa
28
+
29
+ # The model is pulled from the Hub at first start and cached under
30
+ # /root/.cache/huggingface. Pre-fetching during build means a cold Space does
31
+ # not serve requests until the weights are on disk.
32
+ ARG MODEL_ID=impira/layoutlm-invoices
33
+ RUN python -c "\
34
+ from huggingface_hub import snapshot_download; \
35
+ print(snapshot_download('${MODEL_ID}', \
36
+ allow_patterns=['*.json','*.txt','*.bin','*.safetensors','*.model'])); \
37
+ " || echo 'prefetch failed; model will download at first start'
38
+
39
+ ENV DOCX_MODEL_ID=impira/layoutlm-invoices \
40
+ DOCX_DEVICE=cpu \
41
+ DOCX_TORCH_THREADS=4 \
42
+ DOCX_BATCH_SIZE=8 \
43
+ DOCX_MAX_PAGES=3 \
44
+ DOCX_MAX_UPLOAD_MB=25 \
45
+ DOCX_CONFIDENCE_THRESHOLD=0.5 \
46
+ DOCX_REQUEST_TIMEOUT_S=60 \
47
+ DOCX_LOG_JSON=false \
48
+ HF_HOME=/app/.cache/huggingface \
49
+ PORT=7860
50
 
51
  EXPOSE 7860
52
 
53
+ # Warm the model first, then serve. The model loads before uvicorn binds, so a
54
+ # passing health check means inference will actually work rather than the
55
+ # process merely being alive.
56
+ HEALTHCHECK --interval=30s --timeout=10s --start-period=600s --retries=6 \
57
  CMD curl -fsS http://127.0.0.1:7860/health || exit 1
58
 
59
+ CMD ["sh", "-c", "python -c \"from docxextract.engine import get_engine; from docxextract.config import get_settings; s=get_settings(); get_engine(s).warmup(); print('model warm')\" && exec uvicorn docxextract.api:app --host 0.0.0.0 --port ${PORT} --workers 1 --timeout-keep-alive 65"]
 
 
README.md CHANGED
@@ -1,9 +1,108 @@
1
  ---
2
- title: OpenAI Chat
3
- emoji: 🤖
4
- colorFrom: purple
5
- colorTo: green
6
  sdk: docker
7
- app_file: main.py
8
  pinned: false
9
- ---
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
  ---
2
+ title: Document Data Extraction
3
+ emoji: 🧾
4
+ colorFrom: blue
5
+ colorTo: indigo
6
  sdk: docker
 
7
  pinned: false
8
+ app_port: 7860
9
+ license: mit
10
+ ---
11
+
12
+ # Document Data Extraction
13
+
14
+ Extract arbitrary user-defined keys from invoices, receipts and scanned
15
+ documents. Upload a PDF or image, list the keys you want, get one structured
16
+ result per key.
17
+
18
+ ```bash
19
+ curl -X POST https://<your-space>.hf.space/v1/extract \
20
+ -F "file=@invoice.pdf" \
21
+ -F "keys=INVOICE NO,Vendor Id,Invoice Date,Total Amount,GSTIN"
22
+ ```
23
+
24
+ Keys are free-form. Nothing is hard-coded — `INVOICE NO` and `Total Amount` are
25
+ just strings. Duplicates are returned separately, index-aligned with the
26
+ request.
27
+
28
+ ## Read `status`, not `value`
29
+
30
+ | status | meaning |
31
+ |---|---|
32
+ | `extracted` | confidence >= threshold. Use it. |
33
+ | `low_confidence` | a span was found but fell below threshold. Route to a human. |
34
+ | `not_found` | nothing usable on the page. |
35
+ | `error` | the key could not be processed. |
36
+
37
+ `impira/layoutlm-invoices` is an extractive model with **no null-answer head**:
38
+ it returns *some* span for every input, including questions the document
39
+ cannot answer. Measured on this project's benchmark it gave a confident wrong
40
+ value for **16.7%** of deliberately unanswerable invoice questions, and
41
+ abstained **0%** of the time. The confidence threshold exists specifically to
42
+ catch this. Treating `value` without checking `status` will surface invented
43
+ data.
44
+
45
+ ## Model
46
+
47
+ `impira/layoutlm-invoices`, loaded from the Hugging Face cache at start. That
48
+ model is **CC-BY-NC-SA-4.0, non-commercial only**. For commercial use set
49
+ `DOCX_MODEL_ID=impira/layoutlm-document-qa` (MIT) — no code change needed.
50
+
51
+ LayoutLM reads 2D position embeddings from bounding boxes, so it needs the
52
+ document's geometry. Digital PDFs use the text layer; scans and images are
53
+ rasterised and OCR'd with poppler-utils and tesseract-ocr, both in the image.
54
+
55
+ ## Configuration
56
+
57
+ | variable | default | meaning |
58
+ |---|---|---|
59
+ | `DOCX_MODEL_ID` | `impira/layoutlm-invoices` | any LayoutLM QA checkpoint |
60
+ | `DOCX_TORCH_THREADS` | `4` | intra-op threads |
61
+ | `DOCX_BATCH_SIZE` | `8` | keys per forward pass |
62
+ | `DOCX_MAX_PAGES` | `3` | pages processed per document |
63
+ | `DOCX_MAX_UPLOAD_MB` | `25` | body size limit |
64
+ | `DOCX_CONFIDENCE_THRESHOLD` | `0.5` | `extracted` vs `low_confidence` |
65
+ | `DOCX_REQUEST_TIMEOUT_S` | `60` | per-request timeout |
66
+
67
+ Check what is actually in effect with `GET /v1/config`.
68
+
69
+ ## Endpoints
70
+
71
+ | method | path |
72
+ |---|---|
73
+ | `GET` | `/health` |
74
+ | `GET` | `/v1/config` |
75
+ | `POST` | `/v1/extract` (multipart `file`, `keys`) |
76
+ | `POST` | `/v1/extract/json` (base64 + keys array) |
77
+
78
+ `/docs` serves the OpenAPI UI.
79
+
80
+ ## Measured performance
81
+
82
+ Single key, 148-word page, CPU:
83
+
84
+ | runtime | ms/key |
85
+ |---|---|
86
+ | PyTorch fp32 | 578 |
87
+ | ONNX fp32 | 407 |
88
+
89
+ Batching gives no CPU speedup (measured exactly linear) — the GEMMs are already
90
+ compute-bound. Thread count is the real lever.
91
+
92
+ Accuracy on the 12-invoice corpus with pdfplumber word extraction:
93
+
94
+ | metric | invoices | general docs |
95
+ |---|---|---|
96
+ | exact match | 0.7604 | 0.5349 |
97
+ | F1 | 0.8091 | 0.5416 |
98
+ | thresholded | 0.7188 | — |
99
+
100
+ ## Notes
101
+
102
+ `requirements.txt` pins `numpy==1.26.4`; `onnx` pulls numpy 2.x, which breaks
103
+ scipy and sklearn in the benchmark scripts. `torch` comes from the CPU wheel
104
+ index — the default PyPI torch pulls multi-GB CUDA libraries.
105
+
106
+ The previous llama-server (Qwen3-0.6B) service on this same port is commented
107
+ out in the `Dockerfile`, not deleted, so the OpenAI-compatible chat endpoint can
108
+ be restored.
docqa-rs/Cargo.lock DELETED
@@ -1,2376 +0,0 @@
1
- # This file is automatically @generated by Cargo.
2
- # It is not intended for manual editing.
3
- version = 4
4
-
5
- [[package]]
6
- name = "adler2"
7
- version = "2.0.1"
8
- source = "registry+https://github.com/rust-lang/crates.io-index"
9
- checksum = "320119579fcad9c21884f5c4861d16174d0e06250625266f50fe6898340abefa"
10
-
11
- [[package]]
12
- name = "ahash"
13
- version = "0.8.12"
14
- source = "registry+https://github.com/rust-lang/crates.io-index"
15
- checksum = "5a15f179cd60c4584b8a8c596927aadc462e27f2ca70c04e0071964a73ba7a75"
16
- dependencies = [
17
- "cfg-if",
18
- "getrandom 0.3.4",
19
- "once_cell",
20
- "serde",
21
- "version_check",
22
- "zerocopy",
23
- ]
24
-
25
- [[package]]
26
- name = "aho-corasick"
27
- version = "1.1.5"
28
- source = "registry+https://github.com/rust-lang/crates.io-index"
29
- checksum = "c982642fa9e8606056828ee9a8505737230110bb1099153c79efe865c59d12ba"
30
- dependencies = [
31
- "memchr",
32
- ]
33
-
34
- [[package]]
35
- name = "anyhow"
36
- version = "1.0.104"
37
- source = "registry+https://github.com/rust-lang/crates.io-index"
38
- checksum = "330a5ed07fa54e4702c9d6c4174f74427fc0ef6e214bbd677ae50a5099946470"
39
-
40
- [[package]]
41
- name = "atomic-waker"
42
- version = "1.1.2"
43
- source = "registry+https://github.com/rust-lang/crates.io-index"
44
- checksum = "1505bd5d3d116872e7271a6d4e16d81d0c8570876c8de68093a09ac269d8aac0"
45
-
46
- [[package]]
47
- name = "autocfg"
48
- version = "1.5.1"
49
- source = "registry+https://github.com/rust-lang/crates.io-index"
50
- checksum = "f2032f911046de80f0a198e0901378627c33f59ea0ac00e363d481118bd70a53"
51
-
52
- [[package]]
53
- name = "axum"
54
- version = "0.8.9"
55
- source = "registry+https://github.com/rust-lang/crates.io-index"
56
- checksum = "31b698c5f9a010f6573133b09e0de5408834d0c82f8d7475a89fc1867a71cd90"
57
- dependencies = [
58
- "axum-core",
59
- "axum-macros",
60
- "bytes",
61
- "form_urlencoded",
62
- "futures-util",
63
- "http",
64
- "http-body",
65
- "http-body-util",
66
- "hyper",
67
- "hyper-util",
68
- "itoa",
69
- "matchit",
70
- "memchr",
71
- "mime",
72
- "multer",
73
- "percent-encoding",
74
- "pin-project-lite",
75
- "serde_core",
76
- "serde_json",
77
- "serde_path_to_error",
78
- "serde_urlencoded",
79
- "sync_wrapper",
80
- "tokio",
81
- "tower",
82
- "tower-layer",
83
- "tower-service",
84
- "tracing",
85
- ]
86
-
87
- [[package]]
88
- name = "axum-core"
89
- version = "0.5.6"
90
- source = "registry+https://github.com/rust-lang/crates.io-index"
91
- checksum = "08c78f31d7b1291f7ee735c1c6780ccde7785daae9a9206026862dab7d8792d1"
92
- dependencies = [
93
- "bytes",
94
- "futures-core",
95
- "http",
96
- "http-body",
97
- "http-body-util",
98
- "mime",
99
- "pin-project-lite",
100
- "sync_wrapper",
101
- "tower-layer",
102
- "tower-service",
103
- "tracing",
104
- ]
105
-
106
- [[package]]
107
- name = "axum-macros"
108
- version = "0.5.1"
109
- source = "registry+https://github.com/rust-lang/crates.io-index"
110
- checksum = "7aa268c23bfbbd2c4363b9cd302a4f504fb2a9dfe7e3451d66f35dd392e20aca"
111
- dependencies = [
112
- "proc-macro2",
113
- "quote",
114
- "syn 2.0.119",
115
- ]
116
-
117
- [[package]]
118
- name = "base64"
119
- version = "0.13.1"
120
- source = "registry+https://github.com/rust-lang/crates.io-index"
121
- checksum = "9e1b586273c5702936fe7b7d6896644d8be71e6314cfe09d3167c95f712589e8"
122
-
123
- [[package]]
124
- name = "base64"
125
- version = "0.22.1"
126
- source = "registry+https://github.com/rust-lang/crates.io-index"
127
- checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6"
128
-
129
- [[package]]
130
- name = "base64"
131
- version = "0.23.1"
132
- source = "registry+https://github.com/rust-lang/crates.io-index"
133
- checksum = "ac07cdecf99051d9a5238b80f35af32cdeba5b336e55d957b318b50137e18da5"
134
-
135
- [[package]]
136
- name = "bitflags"
137
- version = "2.13.2"
138
- source = "registry+https://github.com/rust-lang/crates.io-index"
139
- checksum = "3ded4057c258ba199e2d26386d3af3780957ecaee6c4ef4041c6b4b8b97c0b06"
140
-
141
- [[package]]
142
- name = "block-buffer"
143
- version = "0.10.4"
144
- source = "registry+https://github.com/rust-lang/crates.io-index"
145
- checksum = "3078c7629b62d3f0439517fa394996acacc5cbc91c5a20d8c658e77abd503a71"
146
- dependencies = [
147
- "generic-array",
148
- ]
149
-
150
- [[package]]
151
- name = "bumpalo"
152
- version = "3.20.3"
153
- source = "registry+https://github.com/rust-lang/crates.io-index"
154
- checksum = "72f5acc6cb2ba439de613abc23857ec3d78374d8ed5ac84e9d11336e87da8649"
155
-
156
- [[package]]
157
- name = "byteorder"
158
- version = "1.5.0"
159
- source = "registry+https://github.com/rust-lang/crates.io-index"
160
- checksum = "1fd0f2584146f6f2ef48085050886acf353beff7305ebd1ae69500e27c67f64b"
161
-
162
- [[package]]
163
- name = "bytes"
164
- version = "1.12.1"
165
- source = "registry+https://github.com/rust-lang/crates.io-index"
166
- checksum = "fc652a48c352aef3ea3aed32080501cf3ef6ed5da78602a020c991775b0aff04"
167
-
168
- [[package]]
169
- name = "castaway"
170
- version = "0.2.4"
171
- source = "registry+https://github.com/rust-lang/crates.io-index"
172
- checksum = "dec551ab6e7578819132c713a93c022a05d60159dc86e7a7050223577484c55a"
173
- dependencies = [
174
- "rustversion",
175
- ]
176
-
177
- [[package]]
178
- name = "cc"
179
- version = "1.6.0"
180
- source = "registry+https://github.com/rust-lang/crates.io-index"
181
- checksum = "f74872d07caf508b30a21f6836e7d7016a2eaf7d9ff4f48deaa58cd8a0407630"
182
- dependencies = [
183
- "find-msvc-tools",
184
- "shlex",
185
- ]
186
-
187
- [[package]]
188
- name = "cfg-if"
189
- version = "1.0.5"
190
- source = "registry+https://github.com/rust-lang/crates.io-index"
191
- checksum = "4e7648175b45a9a48536d676f68d918270699102aa8dab5496df06904c914600"
192
-
193
- [[package]]
194
- name = "compact_str"
195
- version = "0.9.1"
196
- source = "registry+https://github.com/rust-lang/crates.io-index"
197
- checksum = "9dfdd1c2274d9aa354115b09dc9a901d6c5576818cdf70d14cae2bdb47df00ab"
198
- dependencies = [
199
- "castaway",
200
- "cfg-if",
201
- "itoa",
202
- "rustversion",
203
- "ryu",
204
- "serde",
205
- "static_assertions",
206
- ]
207
-
208
- [[package]]
209
- name = "console"
210
- version = "0.15.11"
211
- source = "registry+https://github.com/rust-lang/crates.io-index"
212
- checksum = "054ccb5b10f9f2cbf51eb355ca1d05c2d279ce1804688d0db74b4733a5aeafd8"
213
- dependencies = [
214
- "encode_unicode",
215
- "libc",
216
- "once_cell",
217
- "unicode-width",
218
- "windows-sys 0.59.0",
219
- ]
220
-
221
- [[package]]
222
- name = "core_detect"
223
- version = "1.0.0"
224
- source = "registry+https://github.com/rust-lang/crates.io-index"
225
- checksum = "7f8f80099a98041a3d1622845c271458a2d73e688351bf3cb999266764b81d48"
226
-
227
- [[package]]
228
- name = "cpufeatures"
229
- version = "0.2.17"
230
- source = "registry+https://github.com/rust-lang/crates.io-index"
231
- checksum = "59ed5838eebb26a2bb2e58f6d5b5316989ae9d08bab10e0e6d103e656d1b0280"
232
- dependencies = [
233
- "libc",
234
- ]
235
-
236
- [[package]]
237
- name = "crc32fast"
238
- version = "1.5.2"
239
- source = "registry+https://github.com/rust-lang/crates.io-index"
240
- checksum = "01a7799fd6b852db0e61728dde9a204c423b44d689dbd432522543614b490e78"
241
- dependencies = [
242
- "cfg-if",
243
- ]
244
-
245
- [[package]]
246
- name = "crossbeam-deque"
247
- version = "0.8.8"
248
- source = "registry+https://github.com/rust-lang/crates.io-index"
249
- checksum = "622f3fc73690be383c7214310406f28a90e6edeadc3cea882f9d71e495b9711a"
250
- dependencies = [
251
- "crossbeam-epoch",
252
- "crossbeam-utils",
253
- ]
254
-
255
- [[package]]
256
- name = "crossbeam-epoch"
257
- version = "0.9.21"
258
- source = "registry+https://github.com/rust-lang/crates.io-index"
259
- checksum = "dc74980687109a3b14c72fd458107bf0baa1da1a1a805e178d15501ba9b86d9d"
260
- dependencies = [
261
- "crossbeam-utils",
262
- ]
263
-
264
- [[package]]
265
- name = "crossbeam-utils"
266
- version = "0.8.23"
267
- source = "registry+https://github.com/rust-lang/crates.io-index"
268
- checksum = "a31eee39dddec8330830986fcd7625edb5a24ec90ea038215273bbc3adb08ac6"
269
-
270
- [[package]]
271
- name = "crypto-common"
272
- version = "0.1.7"
273
- source = "registry+https://github.com/rust-lang/crates.io-index"
274
- checksum = "78c8292055d1c1df0cce5d180393dc8cce0abec0a7102adb6c7b1eef6016d60a"
275
- dependencies = [
276
- "generic-array",
277
- "typenum",
278
- ]
279
-
280
- [[package]]
281
- name = "daachorse"
282
- version = "3.0.3"
283
- source = "registry+https://github.com/rust-lang/crates.io-index"
284
- checksum = "5614204febbc33cc07a2806aa6440b904ac012b68eecc37f4493ea4a76455a3d"
285
-
286
- [[package]]
287
- name = "darling"
288
- version = "0.20.11"
289
- source = "registry+https://github.com/rust-lang/crates.io-index"
290
- checksum = "fc7f46116c46ff9ab3eb1597a45688b6715c6e628b5c133e288e709a29bcb4ee"
291
- dependencies = [
292
- "darling_core",
293
- "darling_macro",
294
- ]
295
-
296
- [[package]]
297
- name = "darling_core"
298
- version = "0.20.11"
299
- source = "registry+https://github.com/rust-lang/crates.io-index"
300
- checksum = "0d00b9596d185e565c2207a0b01f8bd1a135483d02d9b7b0a54b11da8d53412e"
301
- dependencies = [
302
- "fnv",
303
- "ident_case",
304
- "proc-macro2",
305
- "quote",
306
- "strsim",
307
- "syn 2.0.119",
308
- ]
309
-
310
- [[package]]
311
- name = "darling_macro"
312
- version = "0.20.11"
313
- source = "registry+https://github.com/rust-lang/crates.io-index"
314
- checksum = "fc34b93ccb385b40dc71c6fceac4b2ad23662c7eeb248cf10d529b7e055b6ead"
315
- dependencies = [
316
- "darling_core",
317
- "quote",
318
- "syn 2.0.119",
319
- ]
320
-
321
- [[package]]
322
- name = "dary_heap"
323
- version = "0.3.9"
324
- source = "registry+https://github.com/rust-lang/crates.io-index"
325
- checksum = "8b1e3a325bc115f096c8b77bbf027a7c2592230e70be2d985be950d3d5e60ebe"
326
- dependencies = [
327
- "serde",
328
- ]
329
-
330
- [[package]]
331
- name = "derive_builder"
332
- version = "0.20.2"
333
- source = "registry+https://github.com/rust-lang/crates.io-index"
334
- checksum = "507dfb09ea8b7fa618fcf76e953f4f5e192547945816d5358edffe39f6f94947"
335
- dependencies = [
336
- "derive_builder_macro",
337
- ]
338
-
339
- [[package]]
340
- name = "derive_builder_core"
341
- version = "0.20.2"
342
- source = "registry+https://github.com/rust-lang/crates.io-index"
343
- checksum = "2d5bcf7b024d6835cfb3d473887cd966994907effbe9227e8c8219824d06c4e8"
344
- dependencies = [
345
- "darling",
346
- "proc-macro2",
347
- "quote",
348
- "syn 2.0.119",
349
- ]
350
-
351
- [[package]]
352
- name = "derive_builder_macro"
353
- version = "0.20.2"
354
- source = "registry+https://github.com/rust-lang/crates.io-index"
355
- checksum = "ab63b0e2bf4d5928aff72e83a7dace85d7bba5fe12dcc3c5a572d78caffd3f3c"
356
- dependencies = [
357
- "derive_builder_core",
358
- "syn 2.0.119",
359
- ]
360
-
361
- [[package]]
362
- name = "digest"
363
- version = "0.10.7"
364
- source = "registry+https://github.com/rust-lang/crates.io-index"
365
- checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292"
366
- dependencies = [
367
- "block-buffer",
368
- "crypto-common",
369
- ]
370
-
371
- [[package]]
372
- name = "dirs"
373
- version = "6.0.0"
374
- source = "registry+https://github.com/rust-lang/crates.io-index"
375
- checksum = "c3e8aa94d75141228480295a7d0e7feb620b1a5ad9f12bc40be62411e38cce4e"
376
- dependencies = [
377
- "dirs-sys",
378
- ]
379
-
380
- [[package]]
381
- name = "dirs-sys"
382
- version = "0.5.0"
383
- source = "registry+https://github.com/rust-lang/crates.io-index"
384
- checksum = "e01a3366d27ee9890022452ee61b2b63a67e6f13f58900b651ff5665f0bb1fab"
385
- dependencies = [
386
- "libc",
387
- "option-ext",
388
- "redox_users",
389
- "windows-sys 0.61.2",
390
- ]
391
-
392
- [[package]]
393
- name = "displaydoc"
394
- version = "0.2.7"
395
- source = "registry+https://github.com/rust-lang/crates.io-index"
396
- checksum = "c6232dd377dcc64799954cbd3a9bb882e9cdc1308ccd87b1c098f1fb2eaf82a8"
397
- dependencies = [
398
- "proc-macro2",
399
- "quote",
400
- "syn 3.0.6",
401
- ]
402
-
403
- [[package]]
404
- name = "docqa"
405
- version = "0.1.0"
406
- dependencies = [
407
- "anyhow",
408
- "axum",
409
- "base64 0.22.1",
410
- "once_cell",
411
- "ort",
412
- "parking_lot",
413
- "serde",
414
- "serde_json",
415
- "sha2",
416
- "tokenizers",
417
- "tokio",
418
- "tower",
419
- "tower-http",
420
- "tracing",
421
- "tracing-subscriber",
422
- ]
423
-
424
- [[package]]
425
- name = "either"
426
- version = "1.19.0"
427
- source = "registry+https://github.com/rust-lang/crates.io-index"
428
- checksum = "0e9c71c2167ca323c882b99918929403426e2373ea17242ff5653e0d5e1058be"
429
-
430
- [[package]]
431
- name = "encode_unicode"
432
- version = "1.0.0"
433
- source = "registry+https://github.com/rust-lang/crates.io-index"
434
- checksum = "34aa73646ffb006b8f5147f3dc182bd4bcb190227ce861fc4a4844bf8e3cb2c0"
435
-
436
- [[package]]
437
- name = "encoding_rs"
438
- version = "0.8.42"
439
- source = "registry+https://github.com/rust-lang/crates.io-index"
440
- checksum = "8e985e0451871ad22fb8d2b6b076e2028a502a0d3950998c2c5c0a4f9b5d9679"
441
- dependencies = [
442
- "cfg-if",
443
- "core_detect",
444
- "multiversion_no_op",
445
- "rustversion",
446
- "scopeguard",
447
- "simdutf8",
448
- ]
449
-
450
- [[package]]
451
- name = "errno"
452
- version = "0.3.14"
453
- source = "registry+https://github.com/rust-lang/crates.io-index"
454
- checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb"
455
- dependencies = [
456
- "libc",
457
- "windows-sys 0.61.2",
458
- ]
459
-
460
- [[package]]
461
- name = "esaxx-rs"
462
- version = "0.1.10"
463
- source = "registry+https://github.com/rust-lang/crates.io-index"
464
- checksum = "d817e038c30374a4bcb22f94d0a8a0e216958d4c3dcde369b1439fec4bdda6e6"
465
-
466
- [[package]]
467
- name = "find-msvc-tools"
468
- version = "0.1.14"
469
- source = "registry+https://github.com/rust-lang/crates.io-index"
470
- checksum = "aedcfb3409746eddb02b9e19ebda1c3394f759a152e48ee875a0844d1b955484"
471
-
472
- [[package]]
473
- name = "flate2"
474
- version = "1.1.10"
475
- source = "registry+https://github.com/rust-lang/crates.io-index"
476
- checksum = "6e634e2e0ebac1ee034020da1ca582e17ffe4e0f5e985823721e168928136dcb"
477
- dependencies = [
478
- "crc32fast",
479
- "miniz_oxide",
480
- "zlib-rs",
481
- ]
482
-
483
- [[package]]
484
- name = "fnv"
485
- version = "1.0.7"
486
- source = "registry+https://github.com/rust-lang/crates.io-index"
487
- checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1"
488
-
489
- [[package]]
490
- name = "form_urlencoded"
491
- version = "1.2.2"
492
- source = "registry+https://github.com/rust-lang/crates.io-index"
493
- checksum = "cb4cb245038516f5f85277875cdaa4f7d2c9a0fa0468de06ed190163b1581fcf"
494
- dependencies = [
495
- "percent-encoding",
496
- ]
497
-
498
- [[package]]
499
- name = "futures-core"
500
- version = "0.3.34"
501
- source = "registry+https://github.com/rust-lang/crates.io-index"
502
- checksum = "92d699e522242e69e3003b94ecc1f960f3a5e015aa7c5d7486e65ad01dd94f5e"
503
-
504
- [[package]]
505
- name = "futures-task"
506
- version = "0.3.34"
507
- source = "registry+https://github.com/rust-lang/crates.io-index"
508
- checksum = "cd417de3d1d015fc3bfd2b1ea46dfc7bab72ef86f1cc7cc9c78e728b34a6d1fd"
509
-
510
- [[package]]
511
- name = "futures-util"
512
- version = "0.3.34"
513
- source = "registry+https://github.com/rust-lang/crates.io-index"
514
- checksum = "0d50a92467f8ba5dd6e3ee5d4bd04d73ab2e4e1c44474a0674821dfce14b79bc"
515
- dependencies = [
516
- "futures-core",
517
- "futures-task",
518
- "pin-project-lite",
519
- "slab",
520
- ]
521
-
522
- [[package]]
523
- name = "generic-array"
524
- version = "0.14.7"
525
- source = "registry+https://github.com/rust-lang/crates.io-index"
526
- checksum = "85649ca51fd72272d7821adaf274ad91c288277713d9c18820d8499a7ff69e9a"
527
- dependencies = [
528
- "typenum",
529
- "version_check",
530
- ]
531
-
532
- [[package]]
533
- name = "getrandom"
534
- version = "0.2.17"
535
- source = "registry+https://github.com/rust-lang/crates.io-index"
536
- checksum = "ff2abc00be7fca6ebc474524697ae276ad847ad0a6b3faa4bcb027e9a4614ad0"
537
- dependencies = [
538
- "cfg-if",
539
- "libc",
540
- "wasi",
541
- ]
542
-
543
- [[package]]
544
- name = "getrandom"
545
- version = "0.3.4"
546
- source = "registry+https://github.com/rust-lang/crates.io-index"
547
- checksum = "899def5c37c4fd7b2664648c28120ecec138e4d395b459e5ca34f9cce2dd77fd"
548
- dependencies = [
549
- "cfg-if",
550
- "libc",
551
- "r-efi",
552
- "wasip2",
553
- ]
554
-
555
- [[package]]
556
- name = "hf-hub"
557
- version = "0.4.3"
558
- source = "registry+https://github.com/rust-lang/crates.io-index"
559
- checksum = "629d8f3bbeda9d148036d6b0de0a3ab947abd08ce90626327fc3547a49d59d97"
560
- dependencies = [
561
- "dirs",
562
- "http",
563
- "indicatif",
564
- "libc",
565
- "log",
566
- "rand",
567
- "serde",
568
- "serde_json",
569
- "thiserror",
570
- "ureq 2.12.1",
571
- "windows-sys 0.60.2",
572
- ]
573
-
574
- [[package]]
575
- name = "http"
576
- version = "1.5.0"
577
- source = "registry+https://github.com/rust-lang/crates.io-index"
578
- checksum = "918d3568bebf352712bc2ef3d46a8bcf1a75b373be6539de198e9105cbbf9ce0"
579
- dependencies = [
580
- "bytes",
581
- "itoa",
582
- ]
583
-
584
- [[package]]
585
- name = "http-body"
586
- version = "1.1.0"
587
- source = "registry+https://github.com/rust-lang/crates.io-index"
588
- checksum = "ca2a8f2913ee65f60facd6a5905613afaa448497a0230cc41ce022d93290bc2c"
589
- dependencies = [
590
- "bytes",
591
- "http",
592
- ]
593
-
594
- [[package]]
595
- name = "http-body-util"
596
- version = "0.1.5"
597
- source = "registry+https://github.com/rust-lang/crates.io-index"
598
- checksum = "23169fe34a5fbcdd3f3862e78fb9b6fccd5f02a6dc6f732547005d45631ce71c"
599
- dependencies = [
600
- "bytes",
601
- "futures-core",
602
- "http",
603
- "http-body",
604
- "pin-project-lite",
605
- ]
606
-
607
- [[package]]
608
- name = "httparse"
609
- version = "1.10.1"
610
- source = "registry+https://github.com/rust-lang/crates.io-index"
611
- checksum = "6dbf3de79e51f3d586ab4cb9d5c3e2c14aa28ed23d180cf89b4df0454a69cc87"
612
-
613
- [[package]]
614
- name = "httpdate"
615
- version = "1.0.3"
616
- source = "registry+https://github.com/rust-lang/crates.io-index"
617
- checksum = "df3b46402a9d5adb4c86a0cf463f42e19994e3ee891101b1841f30a545cb49a9"
618
-
619
- [[package]]
620
- name = "hyper"
621
- version = "1.12.0"
622
- source = "registry+https://github.com/rust-lang/crates.io-index"
623
- checksum = "2c3e324da4c95177d6291d4c8730197c0d1822f8a9766814a4a44fa5ab797c9c"
624
- dependencies = [
625
- "atomic-waker",
626
- "bytes",
627
- "futures-core",
628
- "http",
629
- "http-body",
630
- "httparse",
631
- "httpdate",
632
- "itoa",
633
- "pin-project-lite",
634
- "smallvec",
635
- "tokio",
636
- ]
637
-
638
- [[package]]
639
- name = "hyper-util"
640
- version = "0.1.21"
641
- source = "registry+https://github.com/rust-lang/crates.io-index"
642
- checksum = "ddc03d96684f9226b8a787cdb71488417b53ab5ea8fdb1dac946cb9431cc8bff"
643
- dependencies = [
644
- "bytes",
645
- "http",
646
- "http-body",
647
- "hyper",
648
- "pin-project-lite",
649
- "tokio",
650
- "tower-service",
651
- ]
652
-
653
- [[package]]
654
- name = "icu_collections"
655
- version = "2.3.0"
656
- source = "registry+https://github.com/rust-lang/crates.io-index"
657
- checksum = "fa68d21081c4a05d5a901a1c62add574c77048b6a1c67be3b50ce0b60d4ca513"
658
- dependencies = [
659
- "displaydoc",
660
- "potential_utf",
661
- "utf8_iter",
662
- "yoke",
663
- "zerofrom",
664
- "zerovec",
665
- ]
666
-
667
- [[package]]
668
- name = "icu_locale_core"
669
- version = "2.3.0"
670
- source = "registry+https://github.com/rust-lang/crates.io-index"
671
- checksum = "d56e28588da92eee5c3201a6eff33fabdd49b62269c8938d4ff050ce4d900deb"
672
- dependencies = [
673
- "displaydoc",
674
- "litemap",
675
- "tinystr",
676
- "writeable",
677
- "zerovec",
678
- ]
679
-
680
- [[package]]
681
- name = "icu_normalizer"
682
- version = "2.3.0"
683
- source = "registry+https://github.com/rust-lang/crates.io-index"
684
- checksum = "12f9cf5f235641ed274641dd81c3f28d870e276763d0797aeeab72317b1c646f"
685
- dependencies = [
686
- "icu_collections",
687
- "icu_normalizer_data",
688
- "icu_properties",
689
- "icu_provider",
690
- "smallvec",
691
- "zerovec",
692
- ]
693
-
694
- [[package]]
695
- name = "icu_normalizer_data"
696
- version = "2.3.0"
697
- source = "registry+https://github.com/rust-lang/crates.io-index"
698
- checksum = "1563da1ed3e0b3bf3d74c9b85917ac9c56464d2f57242270c09c9e752f8021a0"
699
-
700
- [[package]]
701
- name = "icu_properties"
702
- version = "2.3.0"
703
- source = "registry+https://github.com/rust-lang/crates.io-index"
704
- checksum = "7e7ca276ad3145661a65914e6daf131ca5120cd3dcee8f8f3214b8875184a148"
705
- dependencies = [
706
- "displaydoc",
707
- "icu_collections",
708
- "icu_locale_core",
709
- "icu_properties_data",
710
- "icu_provider",
711
- "zerotrie",
712
- "zerovec",
713
- ]
714
-
715
- [[package]]
716
- name = "icu_properties_data"
717
- version = "2.3.0"
718
- source = "registry+https://github.com/rust-lang/crates.io-index"
719
- checksum = "e590f038c1464a96894fd6d10127e90a8be4509f56ff7ecef851b15cee0b7caa"
720
-
721
- [[package]]
722
- name = "icu_provider"
723
- version = "2.3.1"
724
- source = "registry+https://github.com/rust-lang/crates.io-index"
725
- checksum = "d27bbb9d3abbefac45d55f647c9de1d44aafcd1186eb91879afef17c396c3e73"
726
- dependencies = [
727
- "displaydoc",
728
- "icu_locale_core",
729
- "writeable",
730
- "yoke",
731
- "zerofrom",
732
- "zerotrie",
733
- "zerovec",
734
- ]
735
-
736
- [[package]]
737
- name = "ident_case"
738
- version = "1.0.1"
739
- source = "registry+https://github.com/rust-lang/crates.io-index"
740
- checksum = "b9e0384b61958566e926dc50660321d12159025e767c18e043daf26b70104c39"
741
-
742
- [[package]]
743
- name = "idna"
744
- version = "1.1.0"
745
- source = "registry+https://github.com/rust-lang/crates.io-index"
746
- checksum = "3b0875f23caa03898994f6ddc501886a45c7d3d62d04d2d90788d47be1b1e4de"
747
- dependencies = [
748
- "idna_adapter",
749
- "smallvec",
750
- "utf8_iter",
751
- ]
752
-
753
- [[package]]
754
- name = "idna_adapter"
755
- version = "1.2.2"
756
- source = "registry+https://github.com/rust-lang/crates.io-index"
757
- checksum = "cb68373c0d6620ef8105e855e7745e18b0d00d3bdb07fb532e434244cdb9a714"
758
- dependencies = [
759
- "icu_normalizer",
760
- "icu_properties",
761
- ]
762
-
763
- [[package]]
764
- name = "indicatif"
765
- version = "0.17.11"
766
- source = "registry+https://github.com/rust-lang/crates.io-index"
767
- checksum = "183b3088984b400f4cfac3620d5e076c84da5364016b4f49473de574b2586235"
768
- dependencies = [
769
- "console",
770
- "number_prefix",
771
- "portable-atomic",
772
- "unicode-width",
773
- "web-time",
774
- ]
775
-
776
- [[package]]
777
- name = "itertools"
778
- version = "0.14.0"
779
- source = "registry+https://github.com/rust-lang/crates.io-index"
780
- checksum = "2b192c782037fadd9cfa75548310488aabdbf3d2da73885b31bd0abd03351285"
781
- dependencies = [
782
- "either",
783
- ]
784
-
785
- [[package]]
786
- name = "itoa"
787
- version = "1.0.18"
788
- source = "registry+https://github.com/rust-lang/crates.io-index"
789
- checksum = "8f42a60cbdf9a97f5d2305f08a87dc4e09308d1276d28c869c684d7777685682"
790
-
791
- [[package]]
792
- name = "js-sys"
793
- version = "0.3.106"
794
- source = "registry+https://github.com/rust-lang/crates.io-index"
795
- checksum = "7883d941dae510fb2d978fc3fe018c71c9e2892fd38854de3e8b92c2e5ad9cc5"
796
- dependencies = [
797
- "cfg-if",
798
- "futures-util",
799
- "wasm-bindgen",
800
- ]
801
-
802
- [[package]]
803
- name = "lazy_static"
804
- version = "1.5.1"
805
- source = "registry+https://github.com/rust-lang/crates.io-index"
806
- checksum = "20870f649af7073d53e38067b2a84312175d56ea15217e1b15bc83506ec50afb"
807
-
808
- [[package]]
809
- name = "libc"
810
- version = "0.2.190"
811
- source = "registry+https://github.com/rust-lang/crates.io-index"
812
- checksum = "ce5d3ddc6d3fa000eb1536d85e147bfe31aacaba692ed6a876f95cb7c855be78"
813
-
814
- [[package]]
815
- name = "libloading"
816
- version = "0.9.0"
817
- source = "registry+https://github.com/rust-lang/crates.io-index"
818
- checksum = "754ca22de805bb5744484a5b151a9e1a8e837d5dc232c2d7d8c2e3492edc8b60"
819
- dependencies = [
820
- "cfg-if",
821
- "windows-link",
822
- ]
823
-
824
- [[package]]
825
- name = "libredox"
826
- version = "0.1.25"
827
- source = "registry+https://github.com/rust-lang/crates.io-index"
828
- checksum = "61ff90caf6077a803a240f62fdbe88645a890bbca49ef8174c3cb0404362171d"
829
- dependencies = [
830
- "libc",
831
- ]
832
-
833
- [[package]]
834
- name = "litemap"
835
- version = "0.8.3"
836
- source = "registry+https://github.com/rust-lang/crates.io-index"
837
- checksum = "47d9d19d1d6efa0109d2f65ff4c85cddd50bd572e5a00127ab10987290bcefae"
838
-
839
- [[package]]
840
- name = "lock_api"
841
- version = "0.4.14"
842
- source = "registry+https://github.com/rust-lang/crates.io-index"
843
- checksum = "224399e74b87b5f3557511d98dff8b14089b3dadafcab6bb93eab67d3aace965"
844
- dependencies = [
845
- "scopeguard",
846
- ]
847
-
848
- [[package]]
849
- name = "log"
850
- version = "0.4.34"
851
- source = "registry+https://github.com/rust-lang/crates.io-index"
852
- checksum = "f9f8bd3e56ce4dfc153cf470fffbfa98c7620958b312ca5c3a4b8d5181fd13c6"
853
-
854
- [[package]]
855
- name = "macro_rules_attribute"
856
- version = "0.2.3"
857
- source = "registry+https://github.com/rust-lang/crates.io-index"
858
- checksum = "b3ae8f6d608c795738406608304d30a2dfbdc8e58e44f7ba43236da5208ded3c"
859
- dependencies = [
860
- "macro_rules_attribute-proc_macro",
861
- "pastey",
862
- ]
863
-
864
- [[package]]
865
- name = "macro_rules_attribute-proc_macro"
866
- version = "0.2.3"
867
- source = "registry+https://github.com/rust-lang/crates.io-index"
868
- checksum = "fc04a4c58212d57930a24bf47d3fa87485264a3a054e9c10e042eb373573ad3c"
869
-
870
- [[package]]
871
- name = "matchers"
872
- version = "0.2.0"
873
- source = "registry+https://github.com/rust-lang/crates.io-index"
874
- checksum = "d1525a2a28c7f4fa0fc98bb91ae755d1e2d1505079e05539e35bc876b5d65ae9"
875
- dependencies = [
876
- "regex-automata",
877
- ]
878
-
879
- [[package]]
880
- name = "matchit"
881
- version = "0.8.4"
882
- source = "registry+https://github.com/rust-lang/crates.io-index"
883
- checksum = "47e1ffaa40ddd1f3ed91f717a33c8c0ee23fff369e3aa8772b9605cc1d22f4c3"
884
-
885
- [[package]]
886
- name = "matrixmultiply"
887
- version = "0.3.11"
888
- source = "registry+https://github.com/rust-lang/crates.io-index"
889
- checksum = "3f607c237553f086e7043417a51df26b2eb899d3caff94e6a67592ff992fedc7"
890
- dependencies = [
891
- "autocfg",
892
- "rawpointer",
893
- ]
894
-
895
- [[package]]
896
- name = "memchr"
897
- version = "2.8.3"
898
- source = "registry+https://github.com/rust-lang/crates.io-index"
899
- checksum = "cf8baf1c55e62ffcace7a9f06f4bd9cd3f0c4beb022d3b367256b91b87513d98"
900
-
901
- [[package]]
902
- name = "mime"
903
- version = "0.3.17"
904
- source = "registry+https://github.com/rust-lang/crates.io-index"
905
- checksum = "6877bb514081ee2a7ff5ef9de3281f14a4dd4bceac4c09388074a6b5df8a139a"
906
-
907
- [[package]]
908
- name = "minimal-lexical"
909
- version = "0.2.1"
910
- source = "registry+https://github.com/rust-lang/crates.io-index"
911
- checksum = "68354c5c6bd36d73ff3feceb05efa59b6acb7626617f4962be322a825e61f79a"
912
-
913
- [[package]]
914
- name = "miniz_oxide"
915
- version = "0.9.1"
916
- source = "registry+https://github.com/rust-lang/crates.io-index"
917
- checksum = "b63fbc4a50860e98e7b2aa7804ded1db5cbc3aff9193adaff57a6931bf7c4b4c"
918
- dependencies = [
919
- "adler2",
920
- "simd-adler32",
921
- ]
922
-
923
- [[package]]
924
- name = "mio"
925
- version = "1.2.4"
926
- source = "registry+https://github.com/rust-lang/crates.io-index"
927
- checksum = "1788edb87fdc09c7e26304471e2f5be8cdefb1b6930d6e3985fc02ff53bf86ee"
928
- dependencies = [
929
- "libc",
930
- "wasi",
931
- "windows-sys 0.61.2",
932
- ]
933
-
934
- [[package]]
935
- name = "monostate"
936
- version = "0.1.18"
937
- source = "registry+https://github.com/rust-lang/crates.io-index"
938
- checksum = "3341a273f6c9d5bef1908f17b7267bbab0e95c9bf69a0d4dcf8e9e1b2c76ef67"
939
- dependencies = [
940
- "monostate-impl",
941
- "serde",
942
- "serde_core",
943
- ]
944
-
945
- [[package]]
946
- name = "monostate-impl"
947
- version = "0.1.18"
948
- source = "registry+https://github.com/rust-lang/crates.io-index"
949
- checksum = "e4db6d5580af57bf992f59068d4ea26fd518574ff48d7639b255a36f9de6e7e9"
950
- dependencies = [
951
- "proc-macro2",
952
- "quote",
953
- "syn 2.0.119",
954
- ]
955
-
956
- [[package]]
957
- name = "multer"
958
- version = "3.1.0"
959
- source = "registry+https://github.com/rust-lang/crates.io-index"
960
- checksum = "83e87776546dc87511aa5ee218730c92b666d7264ab6ed41f9d215af9cd5224b"
961
- dependencies = [
962
- "bytes",
963
- "encoding_rs",
964
- "futures-util",
965
- "http",
966
- "httparse",
967
- "memchr",
968
- "mime",
969
- "spin",
970
- "version_check",
971
- ]
972
-
973
- [[package]]
974
- name = "multiversion_no_op"
975
- version = "1.0.0"
976
- source = "registry+https://github.com/rust-lang/crates.io-index"
977
- checksum = "743fb55ba31b18fb1ecef6bdc9aa2743314978ac084044301a7eee33fb99a20d"
978
-
979
- [[package]]
980
- name = "ndarray"
981
- version = "0.17.2"
982
- source = "registry+https://github.com/rust-lang/crates.io-index"
983
- checksum = "520080814a7a6b4a6e9070823bb24b4531daac8c4627e08ba5de8c5ef2f2752d"
984
- dependencies = [
985
- "matrixmultiply",
986
- "num-complex",
987
- "num-integer",
988
- "num-traits",
989
- "portable-atomic",
990
- "portable-atomic-util",
991
- "rawpointer",
992
- ]
993
-
994
- [[package]]
995
- name = "nom"
996
- version = "7.1.3"
997
- source = "registry+https://github.com/rust-lang/crates.io-index"
998
- checksum = "d273983c5a657a70a3e8f2a01329822f3b8c8172b73826411a55751e404a0a4a"
999
- dependencies = [
1000
- "memchr",
1001
- "minimal-lexical",
1002
- ]
1003
-
1004
- [[package]]
1005
- name = "nu-ansi-term"
1006
- version = "0.50.3"
1007
- source = "registry+https://github.com/rust-lang/crates.io-index"
1008
- checksum = "7957b9740744892f114936ab4a57b3f487491bbeafaf8083688b16841a4240e5"
1009
- dependencies = [
1010
- "windows-sys 0.61.2",
1011
- ]
1012
-
1013
- [[package]]
1014
- name = "num-complex"
1015
- version = "0.4.6"
1016
- source = "registry+https://github.com/rust-lang/crates.io-index"
1017
- checksum = "73f88a1307638156682bada9d7604135552957b7818057dcef22705b4d509495"
1018
- dependencies = [
1019
- "num-traits",
1020
- ]
1021
-
1022
- [[package]]
1023
- name = "num-integer"
1024
- version = "0.1.47"
1025
- source = "registry+https://github.com/rust-lang/crates.io-index"
1026
- checksum = "7ce2d95d4b3734dc35aa2f45e1aa22cd416814592a4f9d9205e11affd5b8e10b"
1027
- dependencies = [
1028
- "num-traits",
1029
- ]
1030
-
1031
- [[package]]
1032
- name = "num-traits"
1033
- version = "0.2.19"
1034
- source = "registry+https://github.com/rust-lang/crates.io-index"
1035
- checksum = "071dfc062690e90b734c0b2273ce72ad0ffa95f0c74596bc250dcfd960262841"
1036
- dependencies = [
1037
- "autocfg",
1038
- ]
1039
-
1040
- [[package]]
1041
- name = "number_prefix"
1042
- version = "0.4.0"
1043
- source = "registry+https://github.com/rust-lang/crates.io-index"
1044
- checksum = "830b246a0e5f20af87141b25c173cd1b609bd7779a4617d6ec582abaf90870f3"
1045
-
1046
- [[package]]
1047
- name = "once_cell"
1048
- version = "1.21.4"
1049
- source = "registry+https://github.com/rust-lang/crates.io-index"
1050
- checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50"
1051
-
1052
- [[package]]
1053
- name = "onig"
1054
- version = "6.5.3"
1055
- source = "registry+https://github.com/rust-lang/crates.io-index"
1056
- checksum = "0cc3cbf698f9438986c11a880c90a6d04b9de27575afd28bbf45b154b6c709e2"
1057
- dependencies = [
1058
- "bitflags",
1059
- "libc",
1060
- "once_cell",
1061
- "onig_sys",
1062
- ]
1063
-
1064
- [[package]]
1065
- name = "onig_sys"
1066
- version = "69.9.3"
1067
- source = "registry+https://github.com/rust-lang/crates.io-index"
1068
- checksum = "1e68317604e77e53b85896388e1a803c1d21b74c899ec9e5e1112db90735edd7"
1069
- dependencies = [
1070
- "cc",
1071
- "pkg-config",
1072
- ]
1073
-
1074
- [[package]]
1075
- name = "option-ext"
1076
- version = "0.2.0"
1077
- source = "registry+https://github.com/rust-lang/crates.io-index"
1078
- checksum = "04744f49eae99ab78e0d5c0b603ab218f515ea8cfe5a456d7629ad883a3b6e7d"
1079
-
1080
- [[package]]
1081
- name = "ort"
1082
- version = "2.0.0-rc.13"
1083
- source = "registry+https://github.com/rust-lang/crates.io-index"
1084
- checksum = "4336a1e2b38848325241c72889086886004e589b7c74f335e60a8e8db5138a0b"
1085
- dependencies = [
1086
- "libloading",
1087
- "ndarray",
1088
- "ort-sys",
1089
- "smallvec",
1090
- "tracing",
1091
- "ureq 3.4.2",
1092
- ]
1093
-
1094
- [[package]]
1095
- name = "ort-sys"
1096
- version = "2.0.0-rc.13"
1097
- source = "registry+https://github.com/rust-lang/crates.io-index"
1098
- checksum = "cf211e3776eea6aec988552fa118dd746d70e1b1e5e244058d1c98015f3e5872"
1099
- dependencies = [
1100
- "ureq 3.4.2",
1101
- ]
1102
-
1103
- [[package]]
1104
- name = "parking_lot"
1105
- version = "0.12.5"
1106
- source = "registry+https://github.com/rust-lang/crates.io-index"
1107
- checksum = "93857453250e3077bd71ff98b6a65ea6621a19bb0f559a85248955ac12c45a1a"
1108
- dependencies = [
1109
- "lock_api",
1110
- "parking_lot_core",
1111
- ]
1112
-
1113
- [[package]]
1114
- name = "parking_lot_core"
1115
- version = "0.9.12"
1116
- source = "registry+https://github.com/rust-lang/crates.io-index"
1117
- checksum = "2621685985a2ebf1c516881c026032ac7deafcda1a2c9b7850dc81e3dfcb64c1"
1118
- dependencies = [
1119
- "cfg-if",
1120
- "libc",
1121
- "redox_syscall",
1122
- "smallvec",
1123
- "windows-link",
1124
- ]
1125
-
1126
- [[package]]
1127
- name = "paste"
1128
- version = "1.0.15"
1129
- source = "registry+https://github.com/rust-lang/crates.io-index"
1130
- checksum = "57c0d7b74b563b49d38dae00a0c37d4d6de9b432382b2892f0574ddcae73fd0a"
1131
-
1132
- [[package]]
1133
- name = "pastey"
1134
- version = "0.2.3"
1135
- source = "registry+https://github.com/rust-lang/crates.io-index"
1136
- checksum = "2ee67f1008b1ba2321834326597b8e186293b049a023cdef258527550b9935b4"
1137
-
1138
- [[package]]
1139
- name = "percent-encoding"
1140
- version = "2.3.2"
1141
- source = "registry+https://github.com/rust-lang/crates.io-index"
1142
- checksum = "9b4f627cb1b25917193a259e49bdad08f671f8d9708acfd5fe0a8c1455d87220"
1143
-
1144
- [[package]]
1145
- name = "pin-project-lite"
1146
- version = "0.2.17"
1147
- source = "registry+https://github.com/rust-lang/crates.io-index"
1148
- checksum = "a89322df9ebe1c1578d689c92318e070967d1042b512afbe49518723f4e6d5cd"
1149
-
1150
- [[package]]
1151
- name = "pkg-config"
1152
- version = "0.3.34"
1153
- source = "registry+https://github.com/rust-lang/crates.io-index"
1154
- checksum = "f6b464fbc74e149a392436b17d523f769e057cb6877f6a5c4618bc6f11800548"
1155
-
1156
- [[package]]
1157
- name = "portable-atomic"
1158
- version = "1.15.0"
1159
- source = "registry+https://github.com/rust-lang/crates.io-index"
1160
- checksum = "05c8b63e8d9609db387f0324918f81d68fe27748f084ef092fb35954d0539a85"
1161
-
1162
- [[package]]
1163
- name = "portable-atomic-util"
1164
- version = "0.2.8"
1165
- source = "registry+https://github.com/rust-lang/crates.io-index"
1166
- checksum = "10ab3eb7f3becc3a1cbc4f2c6f20267996cfc1a6467a873763411b136a122715"
1167
- dependencies = [
1168
- "portable-atomic",
1169
- ]
1170
-
1171
- [[package]]
1172
- name = "potential_utf"
1173
- version = "0.1.6"
1174
- source = "registry+https://github.com/rust-lang/crates.io-index"
1175
- checksum = "d83eb9bc6d8e5cf568e7a1101d60ee05e81ed50ea106026f3d18deeb046d7661"
1176
- dependencies = [
1177
- "zerovec",
1178
- ]
1179
-
1180
- [[package]]
1181
- name = "ppv-lite86"
1182
- version = "0.2.21"
1183
- source = "registry+https://github.com/rust-lang/crates.io-index"
1184
- checksum = "85eae3c4ed2f50dcfe72643da4befc30deadb458a9b590d720cde2f2b1e97da9"
1185
- dependencies = [
1186
- "zerocopy",
1187
- ]
1188
-
1189
- [[package]]
1190
- name = "proc-macro2"
1191
- version = "1.0.107"
1192
- source = "registry+https://github.com/rust-lang/crates.io-index"
1193
- checksum = "985e7ec9bb745e6ce6535b544d84d6cd6f7ad8bd711c398938ae983b91a766d9"
1194
- dependencies = [
1195
- "unicode-ident",
1196
- ]
1197
-
1198
- [[package]]
1199
- name = "quote"
1200
- version = "1.0.47"
1201
- source = "registry+https://github.com/rust-lang/crates.io-index"
1202
- checksum = "1fbf4db142a473a8d80c26bbf18454ed458bf8d26c8219c331daecfdbd079001"
1203
- dependencies = [
1204
- "proc-macro2",
1205
- ]
1206
-
1207
- [[package]]
1208
- name = "r-efi"
1209
- version = "5.3.0"
1210
- source = "registry+https://github.com/rust-lang/crates.io-index"
1211
- checksum = "69cdb34c158ceb288df11e18b4bd39de994f6657d83847bdffdbd7f346754b0f"
1212
-
1213
- [[package]]
1214
- name = "rand"
1215
- version = "0.9.5"
1216
- source = "registry+https://github.com/rust-lang/crates.io-index"
1217
- checksum = "b9ef1d0d795eb7d84685bca4f72f3649f064e6641543d3a8c415898726a57b41"
1218
- dependencies = [
1219
- "rand_chacha",
1220
- "rand_core",
1221
- ]
1222
-
1223
- [[package]]
1224
- name = "rand_chacha"
1225
- version = "0.9.0"
1226
- source = "registry+https://github.com/rust-lang/crates.io-index"
1227
- checksum = "d3022b5f1df60f26e1ffddd6c66e8aa15de382ae63b3a0c1bfc0e4d3e3f325cb"
1228
- dependencies = [
1229
- "ppv-lite86",
1230
- "rand_core",
1231
- ]
1232
-
1233
- [[package]]
1234
- name = "rand_core"
1235
- version = "0.9.5"
1236
- source = "registry+https://github.com/rust-lang/crates.io-index"
1237
- checksum = "76afc826de14238e6e8c374ddcc1fa19e374fd8dd986b0d2af0d02377261d83c"
1238
- dependencies = [
1239
- "getrandom 0.3.4",
1240
- ]
1241
-
1242
- [[package]]
1243
- name = "rawpointer"
1244
- version = "0.2.1"
1245
- source = "registry+https://github.com/rust-lang/crates.io-index"
1246
- checksum = "60a357793950651c4ed0f3f52338f53b2f809f32d83a07f72909fa13e4c6c1e3"
1247
-
1248
- [[package]]
1249
- name = "rayon"
1250
- version = "1.12.0"
1251
- source = "registry+https://github.com/rust-lang/crates.io-index"
1252
- checksum = "fb39b166781f92d482534ef4b4b1b2568f42613b53e5b6c160e24cfbfa30926d"
1253
- dependencies = [
1254
- "either",
1255
- "rayon-core",
1256
- ]
1257
-
1258
- [[package]]
1259
- name = "rayon-cond"
1260
- version = "0.4.0"
1261
- source = "registry+https://github.com/rust-lang/crates.io-index"
1262
- checksum = "2964d0cf57a3e7a06e8183d14a8b527195c706b7983549cd5462d5aa3747438f"
1263
- dependencies = [
1264
- "either",
1265
- "itertools",
1266
- "rayon",
1267
- ]
1268
-
1269
- [[package]]
1270
- name = "rayon-core"
1271
- version = "1.13.0"
1272
- source = "registry+https://github.com/rust-lang/crates.io-index"
1273
- checksum = "22e18b0f0062d30d4230b2e85ff77fdfe4326feb054b9783a3460d8435c8ab91"
1274
- dependencies = [
1275
- "crossbeam-deque",
1276
- "crossbeam-utils",
1277
- ]
1278
-
1279
- [[package]]
1280
- name = "redox_syscall"
1281
- version = "0.5.18"
1282
- source = "registry+https://github.com/rust-lang/crates.io-index"
1283
- checksum = "ed2bf2547551a7053d6fdfafda3f938979645c44812fbfcda098faae3f1a362d"
1284
- dependencies = [
1285
- "bitflags",
1286
- ]
1287
-
1288
- [[package]]
1289
- name = "redox_users"
1290
- version = "0.5.3"
1291
- source = "registry+https://github.com/rust-lang/crates.io-index"
1292
- checksum = "60dc65c0ff1a7ae1294b0c67b9f14baf70b644404010370171787bfac1038fc0"
1293
- dependencies = [
1294
- "libredox",
1295
- "thiserror",
1296
- ]
1297
-
1298
- [[package]]
1299
- name = "regex"
1300
- version = "1.13.1"
1301
- source = "registry+https://github.com/rust-lang/crates.io-index"
1302
- checksum = "f020237b6c8eed93db2e2cb53c00c60a8e1bc73da7d073199a1180401450218d"
1303
- dependencies = [
1304
- "aho-corasick",
1305
- "memchr",
1306
- "regex-automata",
1307
- "regex-syntax",
1308
- ]
1309
-
1310
- [[package]]
1311
- name = "regex-automata"
1312
- version = "0.4.18"
1313
- source = "registry+https://github.com/rust-lang/crates.io-index"
1314
- checksum = "ad8553b9b26413251cbf30e620595c7a41b3887f03da04579c0e6b0d6a06b4b2"
1315
- dependencies = [
1316
- "aho-corasick",
1317
- "memchr",
1318
- "regex-syntax",
1319
- ]
1320
-
1321
- [[package]]
1322
- name = "regex-syntax"
1323
- version = "0.8.11"
1324
- source = "registry+https://github.com/rust-lang/crates.io-index"
1325
- checksum = "d6f6ff9a378485b298a5286656da665ba74413d36db0979633275d2e708145d4"
1326
-
1327
- [[package]]
1328
- name = "ring"
1329
- version = "0.17.14"
1330
- source = "registry+https://github.com/rust-lang/crates.io-index"
1331
- checksum = "a4689e6c2294d81e88dc6261c768b63bc4fcdb852be6d1352498b114f61383b7"
1332
- dependencies = [
1333
- "cc",
1334
- "cfg-if",
1335
- "getrandom 0.2.17",
1336
- "libc",
1337
- "untrusted",
1338
- "windows-sys 0.52.0",
1339
- ]
1340
-
1341
- [[package]]
1342
- name = "rustls"
1343
- version = "0.23.45"
1344
- source = "registry+https://github.com/rust-lang/crates.io-index"
1345
- checksum = "0d41d731c7d2f962d1ccc364cec258de3c0e93b38c2fb3ba97ac74513048d634"
1346
- dependencies = [
1347
- "log",
1348
- "once_cell",
1349
- "ring",
1350
- "rustls-pki-types",
1351
- "rustls-webpki",
1352
- "subtle",
1353
- "zeroize",
1354
- ]
1355
-
1356
- [[package]]
1357
- name = "rustls-pki-types"
1358
- version = "1.15.1"
1359
- source = "registry+https://github.com/rust-lang/crates.io-index"
1360
- checksum = "2f4925028c7eb5d1fcdaf196971378ed9d2c1c4efc7dc5d011256f76c99c0a96"
1361
- dependencies = [
1362
- "zeroize",
1363
- ]
1364
-
1365
- [[package]]
1366
- name = "rustls-webpki"
1367
- version = "0.103.15"
1368
- source = "registry+https://github.com/rust-lang/crates.io-index"
1369
- checksum = "f3c3cf1d8b1e7d4927e2d154c3fcb02979afb9939629c62cd9048d4f07b60ac2"
1370
- dependencies = [
1371
- "ring",
1372
- "rustls-pki-types",
1373
- "untrusted",
1374
- ]
1375
-
1376
- [[package]]
1377
- name = "rustversion"
1378
- version = "1.0.23"
1379
- source = "registry+https://github.com/rust-lang/crates.io-index"
1380
- checksum = "cf54715a573b99ac80df0bc206da022bcd442c974952c7b9720069370852e21f"
1381
-
1382
- [[package]]
1383
- name = "ryu"
1384
- version = "1.0.23"
1385
- source = "registry+https://github.com/rust-lang/crates.io-index"
1386
- checksum = "9774ba4a74de5f7b1c1451ed6cd5285a32eddb5cccb8cc655a4e50009e06477f"
1387
-
1388
- [[package]]
1389
- name = "scopeguard"
1390
- version = "1.2.0"
1391
- source = "registry+https://github.com/rust-lang/crates.io-index"
1392
- checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49"
1393
-
1394
- [[package]]
1395
- name = "serde"
1396
- version = "1.0.229"
1397
- source = "registry+https://github.com/rust-lang/crates.io-index"
1398
- checksum = "4148590afebada386688f18773da617792bf2ef03ffc1e4cbd2b1d45b023e0ba"
1399
- dependencies = [
1400
- "serde_core",
1401
- "serde_derive",
1402
- ]
1403
-
1404
- [[package]]
1405
- name = "serde_core"
1406
- version = "1.0.229"
1407
- source = "registry+https://github.com/rust-lang/crates.io-index"
1408
- checksum = "67dca2c9c51e58a4791a4b1ed58308b39c64224d349a935ab5039aa360942a48"
1409
- dependencies = [
1410
- "serde_derive",
1411
- ]
1412
-
1413
- [[package]]
1414
- name = "serde_derive"
1415
- version = "1.0.229"
1416
- source = "registry+https://github.com/rust-lang/crates.io-index"
1417
- checksum = "e7a5d71263a5a7d47b41f6b3f06ba276f10cc18b0931f1799f710578e2309348"
1418
- dependencies = [
1419
- "proc-macro2",
1420
- "quote",
1421
- "syn 3.0.6",
1422
- ]
1423
-
1424
- [[package]]
1425
- name = "serde_json"
1426
- version = "1.0.151"
1427
- source = "registry+https://github.com/rust-lang/crates.io-index"
1428
- checksum = "c841b55ecdae098c80dcae9cf767f6f8a0c2cdb3416bbef72181df4d0fe73f14"
1429
- dependencies = [
1430
- "itoa",
1431
- "memchr",
1432
- "serde",
1433
- "serde_core",
1434
- "zmij",
1435
- ]
1436
-
1437
- [[package]]
1438
- name = "serde_path_to_error"
1439
- version = "0.1.20"
1440
- source = "registry+https://github.com/rust-lang/crates.io-index"
1441
- checksum = "10a9ff822e371bb5403e391ecd83e182e0e77ba7f6fe0160b795797109d1b457"
1442
- dependencies = [
1443
- "itoa",
1444
- "serde",
1445
- "serde_core",
1446
- ]
1447
-
1448
- [[package]]
1449
- name = "serde_urlencoded"
1450
- version = "0.7.1"
1451
- source = "registry+https://github.com/rust-lang/crates.io-index"
1452
- checksum = "d3491c14715ca2294c4d6a88f15e84739788c1d030eed8c110436aafdaa2f3fd"
1453
- dependencies = [
1454
- "form_urlencoded",
1455
- "itoa",
1456
- "ryu",
1457
- "serde",
1458
- ]
1459
-
1460
- [[package]]
1461
- name = "sha2"
1462
- version = "0.10.9"
1463
- source = "registry+https://github.com/rust-lang/crates.io-index"
1464
- checksum = "a7507d819769d01a365ab707794a4084392c824f54a7a6a7862f8c3d0892b283"
1465
- dependencies = [
1466
- "cfg-if",
1467
- "cpufeatures",
1468
- "digest",
1469
- ]
1470
-
1471
- [[package]]
1472
- name = "sharded-slab"
1473
- version = "0.1.7"
1474
- source = "registry+https://github.com/rust-lang/crates.io-index"
1475
- checksum = "f40ca3c46823713e0d4209592e8d6e826aa57e928f09752619fc696c499637f6"
1476
- dependencies = [
1477
- "lazy_static",
1478
- ]
1479
-
1480
- [[package]]
1481
- name = "shlex"
1482
- version = "2.0.1"
1483
- source = "registry+https://github.com/rust-lang/crates.io-index"
1484
- checksum = "f8fadd59c855ef2080decdef8ff161eb6661b86933c9d82e5ba29dc602a55aba"
1485
-
1486
- [[package]]
1487
- name = "signal-hook-registry"
1488
- version = "1.4.8"
1489
- source = "registry+https://github.com/rust-lang/crates.io-index"
1490
- checksum = "c4db69cba1110affc0e9f7bcd48bbf87b3f4fc7c61fc9155afd4c469eb3d6c1b"
1491
- dependencies = [
1492
- "errno",
1493
- "libc",
1494
- ]
1495
-
1496
- [[package]]
1497
- name = "simd-adler32"
1498
- version = "0.3.10"
1499
- source = "registry+https://github.com/rust-lang/crates.io-index"
1500
- checksum = "3a219298ac11a56ea9a6d2120044824d6f01aeb034955e7af7bc16858527deea"
1501
-
1502
- [[package]]
1503
- name = "simdutf8"
1504
- version = "0.1.5"
1505
- source = "registry+https://github.com/rust-lang/crates.io-index"
1506
- checksum = "e3a9fe34e3e7a50316060351f37187a3f546bce95496156754b601a5fa71b76e"
1507
-
1508
- [[package]]
1509
- name = "slab"
1510
- version = "0.4.12"
1511
- source = "registry+https://github.com/rust-lang/crates.io-index"
1512
- checksum = "0c790de23124f9ab44544d7ac05d60440adc586479ce501c1d6d7da3cd8c9cf5"
1513
-
1514
- [[package]]
1515
- name = "smallvec"
1516
- version = "1.16.2"
1517
- source = "registry+https://github.com/rust-lang/crates.io-index"
1518
- checksum = "f9395f0f0eee849a9b707b2f06bb92a6a422090e2123bb2ef8e87a0e61892a8e"
1519
-
1520
- [[package]]
1521
- name = "socket2"
1522
- version = "0.6.5"
1523
- source = "registry+https://github.com/rust-lang/crates.io-index"
1524
- checksum = "c3d1e2c7f27f8d4cb10542a02c49005dbd6e93095799d6f3be745fae9f8fedd4"
1525
- dependencies = [
1526
- "libc",
1527
- "windows-sys 0.61.2",
1528
- ]
1529
-
1530
- [[package]]
1531
- name = "socks"
1532
- version = "0.3.4"
1533
- source = "registry+https://github.com/rust-lang/crates.io-index"
1534
- checksum = "f0c3dbbd9ae980613c6dd8e28a9407b50509d3803b57624d5dfe8315218cd58b"
1535
- dependencies = [
1536
- "byteorder",
1537
- "libc",
1538
- "winapi",
1539
- ]
1540
-
1541
- [[package]]
1542
- name = "spin"
1543
- version = "0.9.9"
1544
- source = "registry+https://github.com/rust-lang/crates.io-index"
1545
- checksum = "3763264f6b73151db08c50ff20d7d8a0b8796e021cdea7ceedad07b80155fa0e"
1546
-
1547
- [[package]]
1548
- name = "spm_precompiled"
1549
- version = "0.1.4"
1550
- source = "registry+https://github.com/rust-lang/crates.io-index"
1551
- checksum = "5851699c4033c63636f7ea4cf7b7c1f1bf06d0cc03cfb42e711de5a5c46cf326"
1552
- dependencies = [
1553
- "base64 0.13.1",
1554
- "nom",
1555
- "serde",
1556
- "unicode-segmentation",
1557
- ]
1558
-
1559
- [[package]]
1560
- name = "stable_deref_trait"
1561
- version = "1.2.1"
1562
- source = "registry+https://github.com/rust-lang/crates.io-index"
1563
- checksum = "6ce2be8dc25455e1f91df71bfa12ad37d7af1092ae736f3a6cd0e37bc7810596"
1564
-
1565
- [[package]]
1566
- name = "static_assertions"
1567
- version = "1.1.0"
1568
- source = "registry+https://github.com/rust-lang/crates.io-index"
1569
- checksum = "a2eb9349b6444b326872e140eb1cf5e7c522154d69e7a0ffb0fb81c06b37543f"
1570
-
1571
- [[package]]
1572
- name = "strsim"
1573
- version = "0.11.1"
1574
- source = "registry+https://github.com/rust-lang/crates.io-index"
1575
- checksum = "7da8b5736845d9f2fcb837ea5d9e2628564b3b043a70948a3f0b778838c5fb4f"
1576
-
1577
- [[package]]
1578
- name = "subtle"
1579
- version = "2.6.1"
1580
- source = "registry+https://github.com/rust-lang/crates.io-index"
1581
- checksum = "13c2bddecc57b384dee18652358fb23172facb8a2c51ccc10d74c157bdea3292"
1582
-
1583
- [[package]]
1584
- name = "syn"
1585
- version = "2.0.119"
1586
- source = "registry+https://github.com/rust-lang/crates.io-index"
1587
- checksum = "872831b642d1a07999a962a351ed35b955ea2cfc8f3862091e2a240a84f17297"
1588
- dependencies = [
1589
- "proc-macro2",
1590
- "quote",
1591
- "unicode-ident",
1592
- ]
1593
-
1594
- [[package]]
1595
- name = "syn"
1596
- version = "3.0.6"
1597
- source = "registry+https://github.com/rust-lang/crates.io-index"
1598
- checksum = "8593e8e72159ed2257d083c7a454a85cbf854f37a0966d8d483aff8c8a3ebcee"
1599
- dependencies = [
1600
- "proc-macro2",
1601
- "quote",
1602
- "unicode-ident",
1603
- ]
1604
-
1605
- [[package]]
1606
- name = "sync_wrapper"
1607
- version = "1.0.2"
1608
- source = "registry+https://github.com/rust-lang/crates.io-index"
1609
- checksum = "0bf256ce5efdfa370213c1dabab5935a12e49f2c58d15e9eac2870d3b4f27263"
1610
-
1611
- [[package]]
1612
- name = "synstructure"
1613
- version = "0.14.0"
1614
- source = "registry+https://github.com/rust-lang/crates.io-index"
1615
- checksum = "901704edd0dfe137f1987838ee4f259e4e063c31371bdb423f7ae38ec6f77f02"
1616
- dependencies = [
1617
- "proc-macro2",
1618
- "quote",
1619
- "syn 3.0.6",
1620
- ]
1621
-
1622
- [[package]]
1623
- name = "thiserror"
1624
- version = "2.0.21"
1625
- source = "registry+https://github.com/rust-lang/crates.io-index"
1626
- checksum = "09e52cb86a36cede5cb101bf8908837b3e4c6e5e59fe7fd85c23fb56200d189e"
1627
- dependencies = [
1628
- "thiserror-impl",
1629
- ]
1630
-
1631
- [[package]]
1632
- name = "thiserror-impl"
1633
- version = "2.0.21"
1634
- source = "registry+https://github.com/rust-lang/crates.io-index"
1635
- checksum = "fe5197923287db20a58125f0bc85c062f7f2c892de97b18c356f9efb14b28524"
1636
- dependencies = [
1637
- "proc-macro2",
1638
- "quote",
1639
- "syn 3.0.6",
1640
- ]
1641
-
1642
- [[package]]
1643
- name = "thread_local"
1644
- version = "1.1.10"
1645
- source = "registry+https://github.com/rust-lang/crates.io-index"
1646
- checksum = "1ad99c4c6d32803332c548b1af0540b357b3f5fc0be8f6c6bfe8b2e6ae784070"
1647
- dependencies = [
1648
- "cfg-if",
1649
- ]
1650
-
1651
- [[package]]
1652
- name = "tinystr"
1653
- version = "0.8.4"
1654
- source = "registry+https://github.com/rust-lang/crates.io-index"
1655
- checksum = "b1e27c91459209c2986af3dcf603a5a74a4368754ce37414f59acc971167f643"
1656
- dependencies = [
1657
- "displaydoc",
1658
- "zerovec",
1659
- ]
1660
-
1661
- [[package]]
1662
- name = "tokenizers"
1663
- version = "0.23.2"
1664
- source = "registry+https://github.com/rust-lang/crates.io-index"
1665
- checksum = "7afbf6e88718afcc138bad01d6ccc3051dbbc3b2ce9793d8b8a3aeb610969cfc"
1666
- dependencies = [
1667
- "ahash",
1668
- "compact_str",
1669
- "daachorse",
1670
- "dary_heap",
1671
- "derive_builder",
1672
- "esaxx-rs",
1673
- "getrandom 0.3.4",
1674
- "hf-hub",
1675
- "itertools",
1676
- "log",
1677
- "macro_rules_attribute",
1678
- "monostate",
1679
- "onig",
1680
- "paste",
1681
- "rand",
1682
- "rayon",
1683
- "rayon-cond",
1684
- "regex",
1685
- "regex-syntax",
1686
- "serde",
1687
- "serde_json",
1688
- "spm_precompiled",
1689
- "thiserror",
1690
- "unicode-normalization-alignments",
1691
- "unicode-segmentation",
1692
- "unicode_categories",
1693
- ]
1694
-
1695
- [[package]]
1696
- name = "tokio"
1697
- version = "1.53.2"
1698
- source = "registry+https://github.com/rust-lang/crates.io-index"
1699
- checksum = "e95f91fcc7a621e8b030f6aa23c71fe9838ae2fb4d8118b75602a328f5144044"
1700
- dependencies = [
1701
- "libc",
1702
- "mio",
1703
- "pin-project-lite",
1704
- "signal-hook-registry",
1705
- "socket2",
1706
- "tokio-macros",
1707
- "windows-sys 0.61.2",
1708
- ]
1709
-
1710
- [[package]]
1711
- name = "tokio-macros"
1712
- version = "2.7.2"
1713
- source = "registry+https://github.com/rust-lang/crates.io-index"
1714
- checksum = "78773a2a397f451582ce068015985c33193cf6dea8b74d2a639fe457b2f07b0e"
1715
- dependencies = [
1716
- "proc-macro2",
1717
- "quote",
1718
- "syn 3.0.6",
1719
- ]
1720
-
1721
- [[package]]
1722
- name = "tower"
1723
- version = "0.5.3"
1724
- source = "registry+https://github.com/rust-lang/crates.io-index"
1725
- checksum = "ebe5ef63511595f1344e2d5cfa636d973292adc0eec1f0ad45fae9f0851ab1d4"
1726
- dependencies = [
1727
- "futures-core",
1728
- "futures-util",
1729
- "pin-project-lite",
1730
- "sync_wrapper",
1731
- "tokio",
1732
- "tower-layer",
1733
- "tower-service",
1734
- "tracing",
1735
- ]
1736
-
1737
- [[package]]
1738
- name = "tower-http"
1739
- version = "0.6.11"
1740
- source = "registry+https://github.com/rust-lang/crates.io-index"
1741
- checksum = "4cfcf7e2740e6fc6d4d688b4ef00650406bb94adf4731e43c096c3a19fe40840"
1742
- dependencies = [
1743
- "bitflags",
1744
- "bytes",
1745
- "http",
1746
- "http-body",
1747
- "http-body-util",
1748
- "pin-project-lite",
1749
- "tokio",
1750
- "tower-layer",
1751
- "tower-service",
1752
- "tracing",
1753
- ]
1754
-
1755
- [[package]]
1756
- name = "tower-layer"
1757
- version = "0.3.3"
1758
- source = "registry+https://github.com/rust-lang/crates.io-index"
1759
- checksum = "121c2a6cda46980bb0fcd1647ffaf6cd3fc79a013de288782836f6df9c48780e"
1760
-
1761
- [[package]]
1762
- name = "tower-service"
1763
- version = "0.3.3"
1764
- source = "registry+https://github.com/rust-lang/crates.io-index"
1765
- checksum = "8df9b6e13f2d32c91b9bd719c00d1958837bc7dec474d94952798cc8e69eeec3"
1766
-
1767
- [[package]]
1768
- name = "tracing"
1769
- version = "0.1.44"
1770
- source = "registry+https://github.com/rust-lang/crates.io-index"
1771
- checksum = "63e71662fa4b2a2c3a26f570f037eb95bb1f85397f3cd8076caed2f026a6d100"
1772
- dependencies = [
1773
- "log",
1774
- "pin-project-lite",
1775
- "tracing-attributes",
1776
- "tracing-core",
1777
- ]
1778
-
1779
- [[package]]
1780
- name = "tracing-attributes"
1781
- version = "0.1.31"
1782
- source = "registry+https://github.com/rust-lang/crates.io-index"
1783
- checksum = "7490cfa5ec963746568740651ac6781f701c9c5ea257c58e057f3ba8cf69e8da"
1784
- dependencies = [
1785
- "proc-macro2",
1786
- "quote",
1787
- "syn 2.0.119",
1788
- ]
1789
-
1790
- [[package]]
1791
- name = "tracing-core"
1792
- version = "0.1.36"
1793
- source = "registry+https://github.com/rust-lang/crates.io-index"
1794
- checksum = "db97caf9d906fbde555dd62fa95ddba9eecfd14cb388e4f491a66d74cd5fb79a"
1795
- dependencies = [
1796
- "once_cell",
1797
- "valuable",
1798
- ]
1799
-
1800
- [[package]]
1801
- name = "tracing-log"
1802
- version = "0.2.0"
1803
- source = "registry+https://github.com/rust-lang/crates.io-index"
1804
- checksum = "ee855f1f400bd0e5c02d150ae5de3840039a3f54b025156404e34c23c03f47c3"
1805
- dependencies = [
1806
- "log",
1807
- "once_cell",
1808
- "tracing-core",
1809
- ]
1810
-
1811
- [[package]]
1812
- name = "tracing-serde"
1813
- version = "0.2.0"
1814
- source = "registry+https://github.com/rust-lang/crates.io-index"
1815
- checksum = "704b1aeb7be0d0a84fc9828cae51dab5970fee5088f83d1dd7ee6f6246fc6ff1"
1816
- dependencies = [
1817
- "serde",
1818
- "tracing-core",
1819
- ]
1820
-
1821
- [[package]]
1822
- name = "tracing-subscriber"
1823
- version = "0.3.23"
1824
- source = "registry+https://github.com/rust-lang/crates.io-index"
1825
- checksum = "cb7f578e5945fb242538965c2d0b04418d38ec25c79d160cd279bf0731c8d319"
1826
- dependencies = [
1827
- "matchers",
1828
- "nu-ansi-term",
1829
- "once_cell",
1830
- "regex-automata",
1831
- "serde",
1832
- "serde_json",
1833
- "sharded-slab",
1834
- "smallvec",
1835
- "thread_local",
1836
- "tracing",
1837
- "tracing-core",
1838
- "tracing-log",
1839
- "tracing-serde",
1840
- ]
1841
-
1842
- [[package]]
1843
- name = "typenum"
1844
- version = "1.20.1"
1845
- source = "registry+https://github.com/rust-lang/crates.io-index"
1846
- checksum = "b6f5e870be6c3b371b77fe0ee0bafb859fa4964b4404c27de1d380043c4dda20"
1847
-
1848
- [[package]]
1849
- name = "unicode-ident"
1850
- version = "1.0.26"
1851
- source = "registry+https://github.com/rust-lang/crates.io-index"
1852
- checksum = "d245f478577f809a851594d02313b640fb437e0bb33866753cff937863096954"
1853
-
1854
- [[package]]
1855
- name = "unicode-normalization-alignments"
1856
- version = "0.1.12"
1857
- source = "registry+https://github.com/rust-lang/crates.io-index"
1858
- checksum = "43f613e4fa046e69818dd287fdc4bc78175ff20331479dab6e1b0f98d57062de"
1859
- dependencies = [
1860
- "smallvec",
1861
- ]
1862
-
1863
- [[package]]
1864
- name = "unicode-segmentation"
1865
- version = "1.13.3"
1866
- source = "registry+https://github.com/rust-lang/crates.io-index"
1867
- checksum = "c6f5d3c3b1bf09027a88a6bc961fc00497d651009560b5463668dc81b0fa87a8"
1868
-
1869
- [[package]]
1870
- name = "unicode-width"
1871
- version = "0.2.2"
1872
- source = "registry+https://github.com/rust-lang/crates.io-index"
1873
- checksum = "b4ac048d71ede7ee76d585517add45da530660ef4390e49b098733c6e897f254"
1874
-
1875
- [[package]]
1876
- name = "unicode_categories"
1877
- version = "0.1.1"
1878
- source = "registry+https://github.com/rust-lang/crates.io-index"
1879
- checksum = "39ec24b3121d976906ece63c9daad25b85969647682eee313cb5779fdd69e14e"
1880
-
1881
- [[package]]
1882
- name = "untrusted"
1883
- version = "0.9.0"
1884
- source = "registry+https://github.com/rust-lang/crates.io-index"
1885
- checksum = "8ecb6da28b8a351d773b68d5825ac39017e680750f980f3a1a85cd8dd28a47c1"
1886
-
1887
- [[package]]
1888
- name = "ureq"
1889
- version = "2.12.1"
1890
- source = "registry+https://github.com/rust-lang/crates.io-index"
1891
- checksum = "02d1a66277ed75f640d608235660df48c8e3c19f3b4edb6a263315626cc3c01d"
1892
- dependencies = [
1893
- "base64 0.22.1",
1894
- "flate2",
1895
- "log",
1896
- "once_cell",
1897
- "rustls",
1898
- "rustls-pki-types",
1899
- "serde",
1900
- "serde_json",
1901
- "socks",
1902
- "url",
1903
- "webpki-roots 0.26.11",
1904
- ]
1905
-
1906
- [[package]]
1907
- name = "ureq"
1908
- version = "3.4.2"
1909
- source = "registry+https://github.com/rust-lang/crates.io-index"
1910
- checksum = "9a7ac20be9b7726e0bbdbf974c059676d9acb1cd414961f570a4e8231cacd7fc"
1911
- dependencies = [
1912
- "base64 0.23.1",
1913
- "log",
1914
- "percent-encoding",
1915
- "rustls",
1916
- "rustls-pki-types",
1917
- "socks",
1918
- "ureq-proto",
1919
- "utf8-zero",
1920
- ]
1921
-
1922
- [[package]]
1923
- name = "ureq-proto"
1924
- version = "0.6.4"
1925
- source = "registry+https://github.com/rust-lang/crates.io-index"
1926
- checksum = "f86fd172ccca569e458f61b6bdd6220965a9ef36e672a6852953b51a0e1583be"
1927
- dependencies = [
1928
- "base64 0.23.1",
1929
- "http",
1930
- "httparse",
1931
- "log",
1932
- ]
1933
-
1934
- [[package]]
1935
- name = "url"
1936
- version = "2.5.8"
1937
- source = "registry+https://github.com/rust-lang/crates.io-index"
1938
- checksum = "ff67a8a4397373c3ef660812acab3268222035010ab8680ec4215f38ba3d0eed"
1939
- dependencies = [
1940
- "form_urlencoded",
1941
- "idna",
1942
- "percent-encoding",
1943
- "serde",
1944
- ]
1945
-
1946
- [[package]]
1947
- name = "utf8-zero"
1948
- version = "0.8.1"
1949
- source = "registry+https://github.com/rust-lang/crates.io-index"
1950
- checksum = "b8c0a043c9540bae7c578c88f91dda8bd82e59ae27c21baca69c8b191aaf5a6e"
1951
-
1952
- [[package]]
1953
- name = "utf8_iter"
1954
- version = "1.0.4"
1955
- source = "registry+https://github.com/rust-lang/crates.io-index"
1956
- checksum = "b6c140620e7ffbb22c2dee59cafe6084a59b5ffc27a8859a5f0d494b5d52b6be"
1957
-
1958
- [[package]]
1959
- name = "valuable"
1960
- version = "0.1.1"
1961
- source = "registry+https://github.com/rust-lang/crates.io-index"
1962
- checksum = "ba73ea9cf16a25df0c8caa16c51acb937d5712a8429db78a3ee29d5dcacd3a65"
1963
-
1964
- [[package]]
1965
- name = "version_check"
1966
- version = "0.9.5"
1967
- source = "registry+https://github.com/rust-lang/crates.io-index"
1968
- checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a"
1969
-
1970
- [[package]]
1971
- name = "wasi"
1972
- version = "0.11.1+wasi-snapshot-preview1"
1973
- source = "registry+https://github.com/rust-lang/crates.io-index"
1974
- checksum = "ccf3ec651a847eb01de73ccad15eb7d99f80485de043efb2f370cd654f4ea44b"
1975
-
1976
- [[package]]
1977
- name = "wasip2"
1978
- version = "1.0.4+wasi-0.2.12"
1979
- source = "registry+https://github.com/rust-lang/crates.io-index"
1980
- checksum = "b67efb37e106e55ce722a510d6b5f9c17f083e5fc79afc2badeb12cc313d9487"
1981
- dependencies = [
1982
- "wit-bindgen",
1983
- ]
1984
-
1985
- [[package]]
1986
- name = "wasm-bindgen"
1987
- version = "0.2.129"
1988
- source = "registry+https://github.com/rust-lang/crates.io-index"
1989
- checksum = "9bb54f33acc68fd454578d9820b0bde1a1a3d17aa17bb7b6595806d02886d409"
1990
- dependencies = [
1991
- "cfg-if",
1992
- "once_cell",
1993
- "rustversion",
1994
- "wasm-bindgen-macro",
1995
- "wasm-bindgen-shared",
1996
- ]
1997
-
1998
- [[package]]
1999
- name = "wasm-bindgen-macro"
2000
- version = "0.2.129"
2001
- source = "registry+https://github.com/rust-lang/crates.io-index"
2002
- checksum = "2e29d0c35b16e224a7eeb5cd2d25e3e1968fbd65604117b44d3b789d00ee8535"
2003
- dependencies = [
2004
- "quote",
2005
- "wasm-bindgen-macro-support",
2006
- ]
2007
-
2008
- [[package]]
2009
- name = "wasm-bindgen-macro-support"
2010
- version = "0.2.129"
2011
- source = "registry+https://github.com/rust-lang/crates.io-index"
2012
- checksum = "6f501a8bc3719dba86ef8ae4728879c08001bea749eb1333ac5b91e040e2a6b7"
2013
- dependencies = [
2014
- "bumpalo",
2015
- "proc-macro2",
2016
- "quote",
2017
- "syn 3.0.6",
2018
- "wasm-bindgen-shared",
2019
- ]
2020
-
2021
- [[package]]
2022
- name = "wasm-bindgen-shared"
2023
- version = "0.2.129"
2024
- source = "registry+https://github.com/rust-lang/crates.io-index"
2025
- checksum = "23f0c9c52aa7cd7d77769a4cfe2a9adb1b331f489a41d912ce14513d5ab995c6"
2026
- dependencies = [
2027
- "unicode-ident",
2028
- ]
2029
-
2030
- [[package]]
2031
- name = "web-time"
2032
- version = "1.1.0"
2033
- source = "registry+https://github.com/rust-lang/crates.io-index"
2034
- checksum = "5a6580f308b1fad9207618087a65c04e7a10bc77e02c8e84e9b00dd4b12fa0bb"
2035
- dependencies = [
2036
- "js-sys",
2037
- "wasm-bindgen",
2038
- ]
2039
-
2040
- [[package]]
2041
- name = "webpki-roots"
2042
- version = "0.26.11"
2043
- source = "registry+https://github.com/rust-lang/crates.io-index"
2044
- checksum = "521bc38abb08001b01866da9f51eb7c5d647a19260e00054a8c7fd5f9e57f7a9"
2045
- dependencies = [
2046
- "webpki-roots 1.0.9",
2047
- ]
2048
-
2049
- [[package]]
2050
- name = "webpki-roots"
2051
- version = "1.0.9"
2052
- source = "registry+https://github.com/rust-lang/crates.io-index"
2053
- checksum = "7dcd9d09a39985f5344844e66b0c530a33843579125f23e21e9f0f220850f22a"
2054
- dependencies = [
2055
- "rustls-pki-types",
2056
- ]
2057
-
2058
- [[package]]
2059
- name = "winapi"
2060
- version = "0.3.9"
2061
- source = "registry+https://github.com/rust-lang/crates.io-index"
2062
- checksum = "5c839a674fcd7a98952e593242ea400abe93992746761e38641405d28b00f419"
2063
- dependencies = [
2064
- "winapi-i686-pc-windows-gnu",
2065
- "winapi-x86_64-pc-windows-gnu",
2066
- ]
2067
-
2068
- [[package]]
2069
- name = "winapi-i686-pc-windows-gnu"
2070
- version = "0.4.0"
2071
- source = "registry+https://github.com/rust-lang/crates.io-index"
2072
- checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6"
2073
-
2074
- [[package]]
2075
- name = "winapi-x86_64-pc-windows-gnu"
2076
- version = "0.4.0"
2077
- source = "registry+https://github.com/rust-lang/crates.io-index"
2078
- checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f"
2079
-
2080
- [[package]]
2081
- name = "windows-link"
2082
- version = "0.2.1"
2083
- source = "registry+https://github.com/rust-lang/crates.io-index"
2084
- checksum = "f0805222e57f7521d6a62e36fa9163bc891acd422f971defe97d64e70d0a4fe5"
2085
-
2086
- [[package]]
2087
- name = "windows-sys"
2088
- version = "0.52.0"
2089
- source = "registry+https://github.com/rust-lang/crates.io-index"
2090
- checksum = "282be5f36a8ce781fad8c8ae18fa3f9beff57ec1b52cb3de0789201425d9a33d"
2091
- dependencies = [
2092
- "windows-targets 0.52.6",
2093
- ]
2094
-
2095
- [[package]]
2096
- name = "windows-sys"
2097
- version = "0.59.0"
2098
- source = "registry+https://github.com/rust-lang/crates.io-index"
2099
- checksum = "1e38bc4d79ed67fd075bcc251a1c39b32a1776bbe92e5bef1f0bf1f8c531853b"
2100
- dependencies = [
2101
- "windows-targets 0.52.6",
2102
- ]
2103
-
2104
- [[package]]
2105
- name = "windows-sys"
2106
- version = "0.60.2"
2107
- source = "registry+https://github.com/rust-lang/crates.io-index"
2108
- checksum = "f2f500e4d28234f72040990ec9d39e3a6b950f9f22d3dba18416c35882612bcb"
2109
- dependencies = [
2110
- "windows-targets 0.53.5",
2111
- ]
2112
-
2113
- [[package]]
2114
- name = "windows-sys"
2115
- version = "0.61.2"
2116
- source = "registry+https://github.com/rust-lang/crates.io-index"
2117
- checksum = "ae137229bcbd6cdf0f7b80a31df61766145077ddf49416a728b02cb3921ff3fc"
2118
- dependencies = [
2119
- "windows-link",
2120
- ]
2121
-
2122
- [[package]]
2123
- name = "windows-targets"
2124
- version = "0.52.6"
2125
- source = "registry+https://github.com/rust-lang/crates.io-index"
2126
- checksum = "9b724f72796e036ab90c1021d4780d4d3d648aca59e491e6b98e725b84e99973"
2127
- dependencies = [
2128
- "windows_aarch64_gnullvm 0.52.6",
2129
- "windows_aarch64_msvc 0.52.6",
2130
- "windows_i686_gnu 0.52.6",
2131
- "windows_i686_gnullvm 0.52.6",
2132
- "windows_i686_msvc 0.52.6",
2133
- "windows_x86_64_gnu 0.52.6",
2134
- "windows_x86_64_gnullvm 0.52.6",
2135
- "windows_x86_64_msvc 0.52.6",
2136
- ]
2137
-
2138
- [[package]]
2139
- name = "windows-targets"
2140
- version = "0.53.5"
2141
- source = "registry+https://github.com/rust-lang/crates.io-index"
2142
- checksum = "4945f9f551b88e0d65f3db0bc25c33b8acea4d9e41163edf90dcd0b19f9069f3"
2143
- dependencies = [
2144
- "windows-link",
2145
- "windows_aarch64_gnullvm 0.53.1",
2146
- "windows_aarch64_msvc 0.53.1",
2147
- "windows_i686_gnu 0.53.1",
2148
- "windows_i686_gnullvm 0.53.1",
2149
- "windows_i686_msvc 0.53.1",
2150
- "windows_x86_64_gnu 0.53.1",
2151
- "windows_x86_64_gnullvm 0.53.1",
2152
- "windows_x86_64_msvc 0.53.1",
2153
- ]
2154
-
2155
- [[package]]
2156
- name = "windows_aarch64_gnullvm"
2157
- version = "0.52.6"
2158
- source = "registry+https://github.com/rust-lang/crates.io-index"
2159
- checksum = "32a4622180e7a0ec044bb555404c800bc9fd9ec262ec147edd5989ccd0c02cd3"
2160
-
2161
- [[package]]
2162
- name = "windows_aarch64_gnullvm"
2163
- version = "0.53.1"
2164
- source = "registry+https://github.com/rust-lang/crates.io-index"
2165
- checksum = "a9d8416fa8b42f5c947f8482c43e7d89e73a173cead56d044f6a56104a6d1b53"
2166
-
2167
- [[package]]
2168
- name = "windows_aarch64_msvc"
2169
- version = "0.52.6"
2170
- source = "registry+https://github.com/rust-lang/crates.io-index"
2171
- checksum = "09ec2a7bb152e2252b53fa7803150007879548bc709c039df7627cabbd05d469"
2172
-
2173
- [[package]]
2174
- name = "windows_aarch64_msvc"
2175
- version = "0.53.1"
2176
- source = "registry+https://github.com/rust-lang/crates.io-index"
2177
- checksum = "b9d782e804c2f632e395708e99a94275910eb9100b2114651e04744e9b125006"
2178
-
2179
- [[package]]
2180
- name = "windows_i686_gnu"
2181
- version = "0.52.6"
2182
- source = "registry+https://github.com/rust-lang/crates.io-index"
2183
- checksum = "8e9b5ad5ab802e97eb8e295ac6720e509ee4c243f69d781394014ebfe8bbfa0b"
2184
-
2185
- [[package]]
2186
- name = "windows_i686_gnu"
2187
- version = "0.53.1"
2188
- source = "registry+https://github.com/rust-lang/crates.io-index"
2189
- checksum = "960e6da069d81e09becb0ca57a65220ddff016ff2d6af6a223cf372a506593a3"
2190
-
2191
- [[package]]
2192
- name = "windows_i686_gnullvm"
2193
- version = "0.52.6"
2194
- source = "registry+https://github.com/rust-lang/crates.io-index"
2195
- checksum = "0eee52d38c090b3caa76c563b86c3a4bd71ef1a819287c19d586d7334ae8ed66"
2196
-
2197
- [[package]]
2198
- name = "windows_i686_gnullvm"
2199
- version = "0.53.1"
2200
- source = "registry+https://github.com/rust-lang/crates.io-index"
2201
- checksum = "fa7359d10048f68ab8b09fa71c3daccfb0e9b559aed648a8f95469c27057180c"
2202
-
2203
- [[package]]
2204
- name = "windows_i686_msvc"
2205
- version = "0.52.6"
2206
- source = "registry+https://github.com/rust-lang/crates.io-index"
2207
- checksum = "240948bc05c5e7c6dabba28bf89d89ffce3e303022809e73deaefe4f6ec56c66"
2208
-
2209
- [[package]]
2210
- name = "windows_i686_msvc"
2211
- version = "0.53.1"
2212
- source = "registry+https://github.com/rust-lang/crates.io-index"
2213
- checksum = "1e7ac75179f18232fe9c285163565a57ef8d3c89254a30685b57d83a38d326c2"
2214
-
2215
- [[package]]
2216
- name = "windows_x86_64_gnu"
2217
- version = "0.52.6"
2218
- source = "registry+https://github.com/rust-lang/crates.io-index"
2219
- checksum = "147a5c80aabfbf0c7d901cb5895d1de30ef2907eb21fbbab29ca94c5b08b1a78"
2220
-
2221
- [[package]]
2222
- name = "windows_x86_64_gnu"
2223
- version = "0.53.1"
2224
- source = "registry+https://github.com/rust-lang/crates.io-index"
2225
- checksum = "9c3842cdd74a865a8066ab39c8a7a473c0778a3f29370b5fd6b4b9aa7df4a499"
2226
-
2227
- [[package]]
2228
- name = "windows_x86_64_gnullvm"
2229
- version = "0.52.6"
2230
- source = "registry+https://github.com/rust-lang/crates.io-index"
2231
- checksum = "24d5b23dc417412679681396f2b49f3de8c1473deb516bd34410872eff51ed0d"
2232
-
2233
- [[package]]
2234
- name = "windows_x86_64_gnullvm"
2235
- version = "0.53.1"
2236
- source = "registry+https://github.com/rust-lang/crates.io-index"
2237
- checksum = "0ffa179e2d07eee8ad8f57493436566c7cc30ac536a3379fdf008f47f6bb7ae1"
2238
-
2239
- [[package]]
2240
- name = "windows_x86_64_msvc"
2241
- version = "0.52.6"
2242
- source = "registry+https://github.com/rust-lang/crates.io-index"
2243
- checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec"
2244
-
2245
- [[package]]
2246
- name = "windows_x86_64_msvc"
2247
- version = "0.53.1"
2248
- source = "registry+https://github.com/rust-lang/crates.io-index"
2249
- checksum = "d6bbff5f0aada427a1e5a6da5f1f98158182f26556f345ac9e04d36d0ebed650"
2250
-
2251
- [[package]]
2252
- name = "wit-bindgen"
2253
- version = "0.57.1"
2254
- source = "registry+https://github.com/rust-lang/crates.io-index"
2255
- checksum = "1ebf944e87a7c253233ad6766e082e3cd714b5d03812acc24c318f549614536e"
2256
-
2257
- [[package]]
2258
- name = "writeable"
2259
- version = "0.6.4"
2260
- source = "registry+https://github.com/rust-lang/crates.io-index"
2261
- checksum = "3ad82d2a33cdc9674dc7465672f271e096168fcdbe0f799d9e6db8c5892679dc"
2262
-
2263
- [[package]]
2264
- name = "yoke"
2265
- version = "0.8.3"
2266
- source = "registry+https://github.com/rust-lang/crates.io-index"
2267
- checksum = "709fe23a0424b6a435d82152b1bd3fdfb0833487d5fa90d05d42762a9891fef5"
2268
- dependencies = [
2269
- "stable_deref_trait",
2270
- "yoke-derive",
2271
- "zerofrom",
2272
- ]
2273
-
2274
- [[package]]
2275
- name = "yoke-derive"
2276
- version = "0.8.4"
2277
- source = "registry+https://github.com/rust-lang/crates.io-index"
2278
- checksum = "ec8ebde2db3681e8c9980cc27822030e68752690ddfa9473e739aeb4dbde6d71"
2279
- dependencies = [
2280
- "proc-macro2",
2281
- "quote",
2282
- "syn 3.0.6",
2283
- "synstructure",
2284
- ]
2285
-
2286
- [[package]]
2287
- name = "zerocopy"
2288
- version = "0.8.62"
2289
- source = "registry+https://github.com/rust-lang/crates.io-index"
2290
- checksum = "86502bf56ac7c77571a32e2647bb2a15894565e981fb2a48d7bde2d91c965a9d"
2291
- dependencies = [
2292
- "zerocopy-derive",
2293
- ]
2294
-
2295
- [[package]]
2296
- name = "zerocopy-derive"
2297
- version = "0.8.62"
2298
- source = "registry+https://github.com/rust-lang/crates.io-index"
2299
- checksum = "5457206954b06561e2608c7e19cf58b1926586d999c246eebe4502f7e2039d1a"
2300
- dependencies = [
2301
- "proc-macro2",
2302
- "quote",
2303
- "syn 2.0.119",
2304
- ]
2305
-
2306
- [[package]]
2307
- name = "zerofrom"
2308
- version = "0.1.8"
2309
- source = "registry+https://github.com/rust-lang/crates.io-index"
2310
- checksum = "0ec05a11813ea801ff6d75110ad09cd0824ddba17dfe17128ea0d5f68e6c5272"
2311
- dependencies = [
2312
- "zerofrom-derive",
2313
- ]
2314
-
2315
- [[package]]
2316
- name = "zerofrom-derive"
2317
- version = "0.1.8"
2318
- source = "registry+https://github.com/rust-lang/crates.io-index"
2319
- checksum = "f75b4683f6c7f45248d4d64056a24298c6281e0993356d7d1b4a1a962ef10d4a"
2320
- dependencies = [
2321
- "proc-macro2",
2322
- "quote",
2323
- "syn 3.0.6",
2324
- "synstructure",
2325
- ]
2326
-
2327
- [[package]]
2328
- name = "zeroize"
2329
- version = "1.9.1"
2330
- source = "registry+https://github.com/rust-lang/crates.io-index"
2331
- checksum = "e13084392c5e4bc371903e2935a5eaeed24905a7511356b883835e18a78f6879"
2332
-
2333
- [[package]]
2334
- name = "zerotrie"
2335
- version = "0.2.5"
2336
- source = "registry+https://github.com/rust-lang/crates.io-index"
2337
- checksum = "4ea269c3bd32f0a32c321907a2ae912ba6f4649bb0fc764a15627e99a7095a3f"
2338
- dependencies = [
2339
- "displaydoc",
2340
- "yoke",
2341
- "zerofrom",
2342
- ]
2343
-
2344
- [[package]]
2345
- name = "zerovec"
2346
- version = "0.11.8"
2347
- source = "registry+https://github.com/rust-lang/crates.io-index"
2348
- checksum = "bb0464e17806c1d976d5cba29399c7f08e516e279e2ba493f63123b5fca67dd8"
2349
- dependencies = [
2350
- "yoke",
2351
- "zerofrom",
2352
- "zerovec-derive",
2353
- ]
2354
-
2355
- [[package]]
2356
- name = "zerovec-derive"
2357
- version = "0.11.6"
2358
- source = "registry+https://github.com/rust-lang/crates.io-index"
2359
- checksum = "34df6fc39dbd26ddc9c10e6a2984476e13acce22e64e4487636ef494369225da"
2360
- dependencies = [
2361
- "proc-macro2",
2362
- "quote",
2363
- "syn 3.0.6",
2364
- ]
2365
-
2366
- [[package]]
2367
- name = "zlib-rs"
2368
- version = "0.6.8"
2369
- source = "registry+https://github.com/rust-lang/crates.io-index"
2370
- checksum = "b268e58e7c693d7c271f93ffc4ba3b380412554231c85bf61ca7af91042a4112"
2371
-
2372
- [[package]]
2373
- name = "zmij"
2374
- version = "1.0.23"
2375
- source = "registry+https://github.com/rust-lang/crates.io-index"
2376
- checksum = "29666d0abbfad1e3dc4dcf6144730dd3a3ab225bbbdac83319345b1b44ccfc1b"
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
docqa-rs/Cargo.toml DELETED
@@ -1,47 +0,0 @@
1
- [package]
2
- name = "docqa"
3
- version = "0.1.0"
4
- edition = "2021"
5
- description = "Document field-extraction API (LayoutLM / ONNX Runtime) for invoices and receipts"
6
- license = "MIT"
7
-
8
- [[bin]]
9
- name = "docqa"
10
- path = "src/main.rs"
11
-
12
- [dependencies]
13
- # ONNX Runtime bindings.
14
- #
15
- # Two constraints, both discovered the hard way:
16
- # * Every stable 1.x release of `ort` is yanked on crates.io, so
17
- # 2.0.0-rc.13 is the only usable version. Pin it exactly.
18
- # * `download-binaries` (a default feature) has a broken build script in
19
- # rc.13 -- its ureq TLS setup calls `ConfigBuilder::tls_config`, which does
20
- # not exist in the resolved ureq version. The fix is `load-dynamic`, which
21
- # links against an ONNX Runtime shared library already present in the
22
- # image instead of downloading one at build time.
23
- ort = { version = "=2.0.0-rc.13", default-features = false, features = ["std", "ndarray", "load-dynamic", "tls-rustls-no-provider"] }
24
-
25
- # HuggingFace tokenizers (Rust port). 0.23.x is the newest stable line.
26
- tokenizers = { version = "0.23", default-features = false, features = ["onig", "http"] }
27
-
28
- tokio = { version = "1", features = ["rt-multi-thread", "macros", "net", "signal", "time"] }
29
- axum = { version = "0.8", features = ["multipart", "macros"] }
30
- tower = "0.5"
31
- tower-http = { version = "0.6", features = ["trace", "limit", "timeout", "cors"] }
32
- serde = { version = "1", features = ["derive"] }
33
- serde_json = "1"
34
- anyhow = "1"
35
- tracing = "0.1"
36
- tracing-subscriber = { version = "0.3", features = ["env-filter", "json"] }
37
- sha2 = "0.10"
38
- base64 = "0.22"
39
- once_cell = "1"
40
- parking_lot = "0.12"
41
-
42
- [profile.release]
43
- opt-level = 3
44
- lto = "thin"
45
- codegen-units = 1
46
- panic = "abort"
47
- strip = true
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
docqa-rs/Dockerfile DELETED
@@ -1,78 +0,0 @@
1
- # Document field-extraction API (Rust + ONNX Runtime)
2
- #
3
- # Standalone build for local testing. The Space build is the parent
4
- # ../Dockerfile, which uses this image as its builder stage.
5
- #
6
- # docker build -f docqa-rs/Dockerfile -t docqa ./docqa-rs
7
- # docker run --rm -p 7860:7860 docqa
8
- #
9
- # Requires a pre-exported ONNX graph and the matching tokenizer:
10
- # docqa-rs/models/layoutlm.onnx
11
- # docqa-rs/models/tokenizer.json
12
- # Both are gitignored; see docqa-rs/README.md for the export procedure.
13
-
14
- # ---- build ----------------------------------------------------------------
15
- FROM rust:1.83-slim-bookworm AS builder
16
-
17
- # poppler-utils provides pdftotext/pdftoppm for the PDF text layer and the
18
- # scanned-PDF OCR fallback.
19
- RUN apt-get update && apt-get install -y --no-install-recommends \
20
- pkg-config \
21
- && rm -rf /var/lib/apt/lists/*
22
-
23
- WORKDIR /src
24
- COPY Cargo.toml Cargo.lock ./
25
- COPY src ./src
26
-
27
- # `load-dynamic` links against an ONNX Runtime shared library rather than
28
- # downloading one during the build, so the runtime package is installed here
29
- # and copied into the final image.
30
- ENV ORT_LIB_LOCATION=/opt/onnxruntime/lib \
31
- ORT_DYLIB_PATH=/opt/onnxruntime/lib/libonnxruntime.so \
32
- LD_LIBRARY_PATH=/opt/onnxruntime/lib
33
-
34
- RUN curl -fsSL -o /tmp/ort.tgz \
35
- "https://github.com/microsoft/onnxruntime/releases/download/v1.20.1/onnxruntime-linux-x64-1.20.1.tgz" \
36
- && mkdir -p /opt/onnxruntime \
37
- && tar -xzf /tmp/ort.tgz -C /opt/onnxruntime --strip-components=1 \
38
- && rm /tmp/ort.tgz
39
-
40
- RUN cargo build --release --locked
41
-
42
- # ---- runtime --------------------------------------------------------------
43
- FROM debian:bookworm-slim
44
-
45
- RUN apt-get update && apt-get install -y --no-install-recommends \
46
- ca-certificates \
47
- libgomp1 \
48
- poppler-utils \
49
- tesseract-ocr \
50
- tesseract-ocr-eng \
51
- curl \
52
- && rm -rf /var/lib/apt/lists/*
53
-
54
- COPY --from=builder /opt/onnxruntime /opt/onnxruntime
55
- ENV LD_LIBRARY_PATH=/opt/onnxruntime/lib
56
-
57
- WORKDIR /app
58
- COPY --from=builder /src/target/release/docqa /usr/local/bin/docqa
59
- COPY models ./models
60
-
61
- ENV DOCQA_BIND=0.0.0.0 \
62
- DOCQA_PORT=7860 \
63
- DOCQA_MODEL_PATH=/app/models/layoutlm.onnx \
64
- DOCQA_TOKENIZER_PATH=/app/models/tokenizer.json \
65
- DOCQA_MODEL_ID=impira/layoutlm-invoices \
66
- DOCQA_WORKERS=2 \
67
- DOCQA_THREADS=4 \
68
- DOCQA_CONFIDENCE_THRESHOLD=0.5 \
69
- RUST_LOG=docqa=info,tower_http=warn
70
-
71
- EXPOSE 7861
72
-
73
- # The model is loaded before the listener binds, so a successful health check
74
- # implies the engine is genuinely ready, not merely that the process is up.
75
- HEALTHCHECK --interval=30s --timeout=5s --start-period=240s --retries=6 \
76
- CMD curl -fsS http://127.0.0.1:7860/health || exit 1
77
-
78
- CMD ["docqa"]
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
docqa-rs/README.md DELETED
@@ -1,232 +0,0 @@
1
- # docqa — Document Data Extraction API (Rust)
2
-
3
- Extracts arbitrary user-defined keys from invoices, receipts and scanned
4
- documents. Accepts a PDF/image **or** pre-parsed `words` + `boxes`, and returns
5
- one value per key with an explicit status.
6
-
7
- ```jsonc
8
- // POST /v1/extract/json
9
- { "document_base64": "JVBERi0...", "keys": ["INVOICE NO", "GSTIN", "Total Amount"] }
10
- ```
11
- ```jsonc
12
- {
13
- "document": { "document_id": "9f2c...", "page_count": 1, "pages_processed": 1,
14
- "word_count": 148, "source": "pdf_text_layer" },
15
- "fields": [
16
- { "key": "INVOICE NO", "value": "INV-334551", "status": "extracted",
17
- "confidence": 0.813, "page": 1 },
18
- { "key": "GSTIN", "value": "19AAHCL4321K1ZR", "status": "low_confidence",
19
- "confidence": 0.004, "page": 1 }
20
- ],
21
- "latency_ms": 2687.4,
22
- "confidence_threshold": 0.5
23
- }
24
- ```
25
-
26
- ---
27
-
28
- ## Status values — read this before trusting an answer
29
-
30
- | status | meaning |
31
- |---|---|
32
- | `extracted` | confidence ≥ threshold |
33
- | `low_confidence` | a span was found but is below the threshold |
34
- | `not_found` | nothing usable |
35
- | `error` | the key could not be processed |
36
-
37
- **The model has no null-answer head.** `impira/layoutlm-invoices` is an
38
- extractive QA model: for *any* input it returns *some* span, including for
39
- questions whose answer is not on the page. Measured on this project's
40
- benchmark, it returned a confident wrong value for **16.7%** of deliberately
41
- unanswerable invoice questions and **20.8%** for general document questions, and
42
- it abstained **0%** of the time.
43
-
44
- That is why `extracted` is gated on a threshold rather than on the model being
45
- right. Treat `low_confidence` as "route to a human", not as a value.
46
-
47
- ---
48
-
49
- ## Why ONNX Runtime and not llama.cpp
50
-
51
- `impira/layoutlm-invoices` is a **bidirectional encoder** with a two-span
52
- extractive head (start/end logits) plus **learned 2D position embeddings**
53
- derived from word bounding boxes:
54
-
55
- - llama.cpp loads **decoder-only GGUF** models and has no representation for a
56
- span head or for 2D positional embeddings. There is no conversion path.
57
- - ONNX Runtime is the correct runtime. `ort` is its Rust binding.
58
-
59
- Measured on this machine (12-core CPU, single key, 148-word page):
60
-
61
- | runtime | ms/key | answers |
62
- |---|---|---|
63
- | PyTorch fp32 (reference) | 578 | 3/3 |
64
- | **ONNX fp32** | **407** | **3/3** |
65
- | ONNX int8 (dynamic) | 359 | 2/3 — degrades, off by default |
66
-
67
- `src/engine.rs` is verified against PyTorch: **12/12 identical answers**.
68
-
69
- ---
70
-
71
- ## Four details that silently corrupt answers
72
-
73
- These were established by reading
74
- `transformers/pipelines/document_question_answering.py`. Getting any wrong
75
- yields a *fast service that confidently returns the wrong span*.
76
-
77
- 1. **SEP tokens carry bbox `[1000,1000,1000,1000]`**, not `[0,0,0,0]`.
78
- Question/CLS tokens get `[0,0,0,0]`; context tokens get the word's real box.
79
- Mis-assigning SEP shifts the learned position embeddings. This was observed
80
- directly: zeroing SEP made the model return `Litware Cloud Services` for
81
- `INVOICE NO` instead of `INV-334551`.
82
- 2. **Answers are the original word strings joined by spaces**, not a decode of
83
- subword ids. These differ for dates, currency symbols and hyphenated ids
84
- (`GSTIN`, `EIN`).
85
- 3. **Only context tokens may open or close a span.**
86
- 4. **Start and end logits are independent softmaxes**; the score is their
87
- product. `exp(start)*exp(end)` on raw logits overflows past 1.0.
88
-
89
- ---
90
-
91
- ## Why one session per worker
92
-
93
- `ort::Session::run` takes `&mut self` because several ONNX Runtime internals are
94
- not thread-safe. Sharing one session behind a mutex serialises all inference;
95
- `src/pool.rs` instead keeps N sessions and checks one out per request.
96
-
97
- Batching N keys into one forward pass was measured and **does not help on CPU** —
98
- it is exactly linear (0.67s / 1.36s / 2.78s / 4.77s for 1/2/4/7 keys). The GEMMs
99
- are already compute-bound, so there is no weight-read amortisation to win. Thread
100
- count *is* the lever (8 threads: 475 ms/key vs 2 threads: 890 ms/key).
101
-
102
- ---
103
-
104
- ## Deploy
105
-
106
- The Space builds only the repo-root `Dockerfile`, which is self-contained:
107
- it compiles the Rust binary in `rust:1.83-slim-bookworm`, then copies the
108
- result and the ONNX Runtime shared library into `debian:bookworm-slim` with
109
- poppler and tesseract installed. It serves on **7860**, the port this Space
110
- previously used for llama-server, so no Space port setting changes are needed.
111
-
112
- The previous llama-server stage is **commented out, not deleted**, at the top
113
- of that Dockerfile so the OpenAI-compatible chat endpoint can be restored.
114
-
115
- ### One-time prerequisite: host the ONNX graph
116
-
117
- `layoutlm.onnx` is 485 MB and cannot live in git (GitHub's limit is 100 MB per
118
- file), so the build downloads it:
119
-
120
- ```bash
121
- huggingface-cli upload-model <your-account>/layoutlm-onnx \
122
- layoutlm.onnx model.onnx
123
-
124
- docker build \
125
- --build-arg MODEL_URL=https://huggingface.co/<your-account>/layoutlm-onnx/resolve/main/model.onnx \
126
- -t docqa .
127
- ```
128
-
129
- Without that upload the build fails at the `models` stage with a 404 on
130
- `model.onnx`. The `tokenizer.json` default resolves from the upstream model
131
- repo and needs no change.
132
-
133
- **Note on Space build time:** compiling `ort` from source plus a 485 MB
134
- download may exceed the Space build limit. If so, push a prebuilt image or use
135
- a larger build timeout; the inference path itself is unchanged.
136
-
137
- ## Build
138
-
139
- ```powershell
140
- cargo build --release
141
- ```
142
-
143
- `ort` uses the `load-dynamic` feature. The default `download-binaries` feature
144
- has a **broken build script** in `2.0.0-rc.13` (its ureq TLS setup calls a
145
- `ConfigBuilder::tls_config` that does not exist). Every stable `ort` 1.x release
146
- is yanked, so `2.0.0-rc.13` is pinned exactly.
147
-
148
- ONNX Runtime shared library (Linux):
149
-
150
- ```bash
151
- curl -fsSL -o ort.tgz \
152
- https://github.com/microsoft/onnxruntime/releases/download/v1.20.1/onnxruntime-linux-x64-1.20.1.tgz
153
- mkdir -p /opt/onnxruntime && tar -xzf ort.tgz -C /opt/onnxruntime --strip-components=1
154
- export LD_LIBRARY_PATH=/opt/onnxruntime/lib
155
- ```
156
-
157
- ## Model files
158
-
159
- `models/layoutlm.onnx` and `models/tokenizer.json` are **not** committed (see
160
- `.gitignore`). Export them once with the Python exporter in the sibling
161
- `docx_service` project:
162
-
163
- ```powershell
164
- python bench\export_onnx.py impira/layoutlm-invoices
165
- Copy-Item artifacts\onnx\impira__layoutlm-invoices\model.onnx docqa-rs\models\layoutlm.onnx
166
- Copy-Item <hf-cache>\impira--layoutlm-invoices\snapshots\*\tokenizer.json docqa-rs\models\tokenizer.json
167
- ```
168
-
169
- The export must accept `bbox` as `(batch, seq*4)`. `torch.onnx.export` cannot
170
- mark all three axes of a rank-3 tensor dynamic, so a plain export pins the
171
- sequence length and **every other sequence length fails at runtime**. The
172
- exporter wraps the model to take a flat bbox and reshape internally.
173
-
174
- ---
175
-
176
- ## Run
177
-
178
- ```powershell
179
- $env:DOCQA_PORT = "7860"
180
- $env:LD_LIBRARY_PATH = "C:\onnxruntime\lib"
181
- .\target\release\docqa.exe
182
- ```
183
-
184
- | variable | default | meaning |
185
- |---|---|---|
186
- | `DOCQA_BIND` / `DOCQA_PORT` | `0.0.0.0` / `7860` | listen address |
187
- | `DOCQA_WORKERS` | `2` | ONNX sessions (concurrency) |
188
- | `DOCQA_THREADS` | `4` | intra-op threads per session |
189
- | `DOCQA_CONFIDENCE_THRESHOLD` | `0.5` | `extracted` vs `low_confidence` |
190
- | `DOCQA_MAX_KEYS` | `64` | keys per request |
191
- | `DOCQA_MAX_UPLOAD_MB` | `25` | body limit |
192
- | `DOCQA_MAX_PAGES` | `3` | pages processed |
193
- | `DOCQA_REQUEST_TIMEOUT_S` | `60` | per-request timeout |
194
-
195
- **Sizing:** total threads ≈ `workers × threads`. On this 12-core machine the
196
- measured best was `threads=8, workers=1`.
197
-
198
- ## Endpoints
199
-
200
- | method | path | body |
201
- |---|---|---|
202
- | `GET` | `/health` | — |
203
- | `POST` | `/v1/extract/words` | `{words, boxes, keys}` — fastest, no parsing |
204
- | `POST` | `/v1/extract/json` | `{document_base64, keys}` or `{words, boxes, keys}` |
205
- | `POST` | `/v1/extract/file` | multipart `file=`, `keys="A,B,C"` |
206
-
207
- `boxes` is flat `[x0,y0,x1,y1,...]` already normalised to **0..1000**, length
208
- `words × 4`. LayoutLM indexes learned position embeddings, so **boxes are
209
- required** — it cannot run on bare text, and the API rejects that rather than
210
- answering from priors.
211
-
212
- Errors: `400` bad request, `413` too large, `415` unsupported media, `422`
213
- unprocessable, `503` engine unavailable, `504` timeout.
214
-
215
- ## Tests
216
-
217
- ```powershell
218
- cargo test
219
- ```
220
-
221
- 21 unit tests cover key validation, box normalisation, base64, the PDF bbox
222
- XML parser, the Tesseract TSV parser, softmax overflow, and content-addressed
223
- document IDs. Inference itself is covered by `scripts/e2e.py`, which needs the
224
- model present.
225
-
226
- ## System requirements (PDF/image endpoints)
227
-
228
- - `poppler-utils` (`pdftotext`, `pdftoppm`) — PDF text layer and scanned-PDF OCR
229
- - `tesseract-ocr` + `eng` — images and scans
230
-
231
- Both are only needed for the `/file` and base64 paths. The `/words` path needs
232
- neither.
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
docqa-rs/docqa.postman_collection.json DELETED
@@ -1,152 +0,0 @@
1
- {
2
- "info": {
3
- "_postman_id": "docqa-collection",
4
- "name": "docqa — Document Data Extraction API",
5
- "description": "Extract user-defined keys from invoices, receipts and scanned documents. Rust + ONNX Runtime, LayoutLM span extraction.\n\nHOW TO READ status VALUES\n extracted confidence >= threshold. Use it.\n low_confidence a span was found but below threshold. Route to a human; do NOT treat as fact.\n not_found nothing usable on the page.\n error the key could not be processed.\n\nIMPORTANT: impira/layoutlm-invoices is extractive with NO null-answer head — it returns *some* span for every input, even unanswerable questions. Measured hallucination rate on unanswerable invoice questions: 16.7%, with 0% abstention. This is exactly why status must be checked before value.\n\nBoxes are REQUIRED and normalised to 0..1000, FLAT: [x0,y0,x1,y1, ...] of length words x 4. LayoutLM indexes learned 2D position embeddings and cannot run without them.\n\nTO BUILD boxes: extract word-level bounding boxes from your PDF/image with any tool (pdfplumber, PyMuPDF, Tesseract, AWS Textract) and normalise: x_norm = round(x / page_width * 1000), y_norm = round(y / page_height * 1000).\n\nFor real values, run the collection against a live server — no response bodies are pre-filled here to avoid shipping fabricated numbers.",
6
- "schema": "https://schema.getpostman.com/json/collection/v2.1.0/collection.json"
7
- },
8
- "variable": [
9
- { "key": "baseUrl", "value": "http://127.0.0.1:7860" }
10
- ],
11
- "item": [
12
- {
13
- "name": "Health",
14
- "request": {
15
- "method": "GET",
16
- "header": [],
17
- "url": { "raw": "{{baseUrl}}/health", "host": ["{{baseUrl}}"], "path": ["health"] },
18
- "description": "Liveness + engine readiness. status=ok means the model is loaded and inference will work."
19
- }
20
- },
21
- {
22
- "name": "Extract (multipart file)",
23
- "request": {
24
- "method": "POST",
25
- "header": [],
26
- "body": {
27
- "mode": "formdata",
28
- "formdata": [
29
- { "key": "file", "type": "file", "src": [], "description": "Attach a real PDF/PNG/JPEG/TIFF/BMP" },
30
- { "key": "keys", "value": "INVOICE NO,Vendor Id,Invoice Date,Total Amount,GSTIN", "type": "text", "description": "Comma-separated. Spaces inside a key are significant: 'Total Amount' != 'TotalAmount'." },
31
- { "key": "confidence_threshold", "value": "0.5", "type": "text", "description": "Optional 0..1. Above -> extracted; below -> low_confidence." }
32
- ]
33
- },
34
- "url": { "raw": "{{baseUrl}}/v1/extract/file", "host": ["{{baseUrl}}"], "path": ["v1", "extract", "file"] },
35
- "description": "Upload a document directly. Needs poppler-utils + tesseract-ocr in the runtime image. Duplicate keys are returned separately (index-aligned with the request)."
36
- }
37
- },
38
- {
39
- "name": "Extract (base64 JSON)",
40
- "request": {
41
- "method": "POST",
42
- "header": [{ "key": "Content-Type", "value": "application/json" }],
43
- "body": {
44
- "mode": "raw",
45
- "raw": "{\n \"document_base64\": \"<base64 of a real PDF or image>\",\n \"keys\": [\"INVOICE NO\", \"Invoice Date\", \"Total Amount\", \"GSTIN\"],\n \"confidence_threshold\": 0.5,\n \"max_pages\": 3\n}",
46
- "options": { "raw": { "language": "json" } }
47
- },
48
- "url": { "raw": "{{baseUrl}}/v1/extract/json", "host": ["{{baseUrl}}"], "path": ["v1", "extract", "json"] },
49
- "description": "JSON variant. `keys` is an ARRAY here (not comma-separated). A data: URI prefix is accepted and stripped. Replace the placeholder with a real base64 document."
50
- }
51
- },
52
- {
53
- "name": "Extract (words + boxes) — fastest",
54
- "request": {
55
- "method": "POST",
56
- "header": [{ "key": "Content-Type", "value": "application/json" }],
57
- "body": {
58
- "mode": "raw",
59
- "raw": "{\n \"words\": [\"Invoice\", \"Number\", \"INV-1\"],\n \"boxes\": [0, 0, 100, 20, 110, 0, 220, 20, 230, 0, 320, 20],\n \"keys\": [\"INVOICE NO\"]\n}",
60
- "options": { "raw": { "language": "json" } }
61
- },
62
- "url": { "raw": "{{baseUrl}}/v1/extract/words", "host": ["{{baseUrl}}"], "path": ["v1", "extract", "words"] },
63
- "description": "Lowest-latency path — skips PDF parsing and OCR entirely by supplying tokens + boxes yourself. Requires NEITHER poppler NOR tesseract.\n\nABOUT THE BOXES ABOVE: they are a hand-written 3-word toy so the request is runnable as-is, NOT real coordinates from any document. For real use, replace both arrays with word-level boxes from your extraction tool, normalised to 0..1000: x_norm = round(x/page_width*1000), y_norm = round(y/page_height*1000). `boxes` must be FLAT, length = words x 4. A length mismatch is rejected with 400 rather than silently misaligned.\n\nSee scripts/make_boxes.py to generate a valid request from a real PDF."
64
- }
65
- },
66
- {
67
- "name": "Edge cases",
68
- "item": [
69
- {
70
- "name": "Empty keys -> 400",
71
- "request": {
72
- "method": "POST",
73
- "header": [{ "key": "Content-Type", "value": "application/json" }],
74
- "body": { "mode": "raw", "raw": "{\n \"words\": [\"a\"],\n \"boxes\": [0, 0, 10, 10],\n \"keys\": []\n}", "options": { "raw": { "language": "json" } } },
75
- "url": { "raw": "{{baseUrl}}/v1/extract/words", "host": ["{{baseUrl}}"], "path": ["v1", "extract", "words"] }
76
- }
77
- },
78
- {
79
- "name": "Blank key -> 400",
80
- "request": {
81
- "method": "POST",
82
- "header": [{ "key": "Content-Type", "value": "application/json" }],
83
- "body": { "mode": "raw", "raw": "{\n \"words\": [\"a\"],\n \"boxes\": [0, 0, 10, 10],\n \"keys\": [\" \"]\n}", "options": { "raw": { "language": "json" } } },
84
- "url": { "raw": "{{baseUrl}}/v1/extract/words", "host": ["{{baseUrl}}"], "path": ["v1", "extract", "words"] }
85
- }
86
- },
87
- {
88
- "name": "Boxes missing -> 400",
89
- "request": {
90
- "method": "POST",
91
- "header": [{ "key": "Content-Type", "value": "application/json" }],
92
- "body": { "mode": "raw", "raw": "{\n \"words\": [\"a\", \"b\"],\n \"keys\": [\"Total\"]\n}", "options": { "raw": { "language": "json" } } },
93
- "url": { "raw": "{{baseUrl}}/v1/extract/words", "host": ["{{baseUrl}}"], "path": ["v1", "extract", "words"] }
94
- },
95
- "description": "LayoutLM needs 2D position embeddings, so boxes are mandatory. Rejected rather than answered from priors."
96
- },
97
- {
98
- "name": "Box count mismatch -> 400",
99
- "request": {
100
- "method": "POST",
101
- "header": [{ "key": "Content-Type", "value": "application/json" }],
102
- "body": { "mode": "raw", "raw": "{\n \"words\": [\"a\", \"b\"],\n \"boxes\": [0, 0, 1, 1],\n \"keys\": [\"Total\"]\n}", "options": { "raw": { "language": "json" } } },
103
- "url": { "raw": "{{baseUrl}}/v1/extract/words", "host": ["{{baseUrl}}"], "path": ["v1", "extract", "words"] }
104
- },
105
- "description": "2 words need 8 box values; only 4 supplied. Caught at validation instead of silently misaligning every box."
106
- },
107
- {
108
- "name": "threshold=1.0 suppresses all extractions",
109
- "request": {
110
- "method": "POST",
111
- "header": [{ "key": "Content-Type", "value": "application/json" }],
112
- "body": {
113
- "mode": "raw",
114
- "raw": "{\n \"words\": [\"Invoice\", \"Number\", \"INV-1\"],\n \"boxes\": [0, 0, 100, 20, 110, 0, 220, 20, 230, 0, 320, 20],\n \"keys\": [\"INVOICE NO\"],\n \"confidence_threshold\": 1.0\n}",
115
- "options": { "raw": { "language": "json" } }
116
- },
117
- "url": { "raw": "{{baseUrl}}/v1/extract/words", "host": ["{{baseUrl}}"], "path": ["v1", "extract", "words"] }
118
- },
119
- "description": "Cheap regression check on the gating logic: at threshold 1.0 no field can be 'extracted'."
120
- },
121
- {
122
- "name": "Duplicate keys returned separately",
123
- "request": {
124
- "method": "POST",
125
- "header": [{ "key": "Content-Type", "value": "application/json" }],
126
- "body": {
127
- "mode": "raw",
128
- "raw": "{\n \"words\": [\"Invoice\", \"Number\", \"INV-1\"],\n \"boxes\": [0, 0, 100, 20, 110, 0, 220, 20, 230, 0, 320, 20],\n \"keys\": [\"INVOICE NO\", \"INVOICE NO\"]\n}",
129
- "options": { "raw": { "language": "json" } }
130
- },
131
- "url": { "raw": "{{baseUrl}}/v1/extract/words", "host": ["{{baseUrl}}"], "path": ["v1", "extract", "words"] }
132
- },
133
- "description": "Duplicates are NOT collapsed — the response is index-aligned with the request so callers can zip the two without key lookup."
134
- },
135
- {
136
- "name": "Unanswerable key (documents the no-abstention behaviour)",
137
- "request": {
138
- "method": "POST",
139
- "header": [{ "key": "Content-Type", "value": "application/json" }],
140
- "body": {
141
- "mode": "raw",
142
- "raw": "{\n \"words\": [\"Invoice\", \"Number\", \"INV-1\", \"Total\", \"100.00\"],\n \"boxes\": [0, 0, 100, 20, 110, 0, 220, 20, 230, 0, 320, 20, 0, 40, 100, 60, 110, 40, 200, 60],\n \"keys\": [\"Passport Number of the Director\"]\n}",
143
- "options": { "raw": { "language": "json" } }
144
- },
145
- "url": { "raw": "{{baseUrl}}/v1/extract/words", "host": ["{{baseUrl}}"], "path": ["v1", "extract", "words"] }
146
- },
147
- "description": "The model ALWAYS returns a span, even for a question the document cannot answer. This request documents that behaviour and is why `status` must be checked before `value`. Expect status=low_confidence with a near-zero confidence and a value that is a real token on the page."
148
- }
149
- ]
150
- }
151
- ]
152
- }
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
docqa-rs/models/README.md DELETED
@@ -1,48 +0,0 @@
1
- # model files
2
-
3
- This directory is populated at build/deploy time and is **not** committed.
4
-
5
- Two files are required at `DOCQA_MODEL_PATH` and `DOCQA_TOKENIZER_PATH`
6
- (defaults `/app/models/layoutlm.onnx` and `/app/models/tokenizer.json`):
7
-
8
- | file | size | source |
9
- |---|---|---|
10
- | `layoutlm.onnx` | ~485 MB | ONNX export of `impira/layoutlm-invoices` |
11
- | `tokenizer.json` | ~1.3 MB | Hugging Face model repo |
12
-
13
- Both are excluded by `.gitignore`: the ONNX graph exceeds GitHub's 100 MB
14
- per-file limit, and neither file is authored source.
15
-
16
- ## Producing them
17
-
18
- From the sibling `docx_service` Python project:
19
-
20
- ```bash
21
- python bench/export_onnx.py impira/layoutlm-invoices
22
- ```
23
-
24
- then copy:
25
-
26
- ```bash
27
- cp artifacts/onnx/impira__layoutlm-invoices/model.onnx \
28
- docqa-rs/models/layoutlm.onnx
29
- cp "$HF_HOME/impira--layoutlm-invoices/snapshots"/*/tokenizer.json \
30
- docqa-rs/models/tokenizer.json
31
- ```
32
-
33
- The exporter verifies parity against PyTorch and exits non-zero on mismatch.
34
-
35
- ## Why the export is not trivial
36
-
37
- `torch.onnx.export` cannot mark all three axes of a rank-3 tensor dynamic, so a
38
- naive export pins the sequence length and **every other sequence length fails at
39
- runtime**. The exporter wraps the model so `bbox` is accepted flat as
40
- `(batch, seq*4)` and reshaped internally, leaving `(batch, seq)` dynamic for
41
- `input_ids`, `attention_mask` and `token_type_ids`.
42
-
43
- ## Licensing
44
-
45
- `impira/layoutlm-invoices` is **CC-BY-NC-SA-4.0** — non-commercial use only.
46
- For commercial deployment switch `DOCQA_MODEL_ID` to
47
- `impira/layoutlm-document-qa` (MIT) and re-export; the serving code is
48
- model-agnostic.
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
docqa-rs/scripts/accuracy.py DELETED
@@ -1,151 +0,0 @@
1
- #!/usr/bin/env python3
2
- """Accuracy check for the Rust docqa service via the /v1/extract/words path.
3
-
4
- The /file and base64 paths need poppler/tesseract. This path needs neither, so
5
- it isolates *inference* correctness: words and boxes come from the Python
6
- parser (pdfplumber) and only the Rust inference is under test. Any answer
7
- mismatch is therefore attributable to the Rust port, not to OCR.
8
-
9
- python scripts/accuracy.py --corpus <dir> [--limit N]
10
- """
11
-
12
- from __future__ import annotations
13
-
14
- import argparse
15
- import json
16
- import os
17
- import statistics
18
- import sys
19
- import time
20
- import urllib.error
21
- import urllib.request
22
- from pathlib import Path
23
-
24
- sys.path.insert(0, str(Path(__file__).resolve().parent.parent))
25
-
26
- KEYS = ["INVOICE NO", "Invoice Date", "Total Amount", "PO Number",
27
- "Payment Term", "Subtotal", "GSTIN", "Vendor Id"]
28
- TRUTH = {
29
- "INVOICE NO": "invoice_number",
30
- "Invoice Date": "invoice_date",
31
- "Total Amount": "total",
32
- "PO Number": "po_number",
33
- "Payment Term": "payment_terms",
34
- "Subtotal": "subtotal",
35
- "GSTIN": "gstin",
36
- "Vendor Id": "vendor_tax_id",
37
- }
38
-
39
-
40
- def post(base: str, path: str, payload: dict, timeout: float = 300.0):
41
- request = urllib.request.Request(
42
- f"{base}{path}", data=json.dumps(payload).encode(),
43
- headers={"Content-Type": "application/json"}, method="POST")
44
- with urllib.request.urlopen(request, timeout=timeout) as response:
45
- return json.loads(response.read())
46
-
47
-
48
- def parse_page(pdf_path: Path):
49
- """Same extraction the Python service uses, so only inference differs."""
50
- import pdfplumber
51
-
52
- with pdfplumber.open(str(pdf_path)) as pdf:
53
- page = pdf.pages[0]
54
- raw = page.extract_words()
55
- words = [w["text"] for w in raw]
56
- boxes: list[int] = []
57
- for w in raw:
58
- boxes += [
59
- max(0, min(1000, round(w["x0"] * 1000 / page.width))),
60
- max(0, min(1000, round(w["top"] * 1000 / page.height))),
61
- max(0, min(1000, round(w["x1"] * 1000 / page.width))),
62
- max(0, min(1000, round(w["bottom"] * 1000 / page.height))),
63
- ]
64
- return words, boxes
65
-
66
-
67
- def main() -> int:
68
- parser = argparse.ArgumentParser()
69
- parser.add_argument("--base", default=os.environ.get(
70
- "DOCQA_BASE", "http://127.0.0.1:7860"))
71
- parser.add_argument("--corpus", type=Path, required=True)
72
- parser.add_argument("--limit", type=int, default=0)
73
- parser.add_argument("--show", action="store_true")
74
- args = parser.parse_args()
75
-
76
- manifest = json.loads((args.corpus / "ground_truth.json").read_text())
77
- if args.limit:
78
- manifest = manifest[: args.limit]
79
-
80
- per_key: dict[str, list[int]] = {k: [] for k in KEYS}
81
- latencies: list[float] = []
82
- errors: list[str] = []
83
-
84
- print(f"target {args.base} documents {len(manifest)} keys {len(KEYS)}\n")
85
-
86
- for record in manifest:
87
- words, boxes = parse_page(Path(record["path"]))
88
- payload = {"words": words, "boxes": boxes, "keys": KEYS}
89
- started = time.perf_counter()
90
- try:
91
- body = post(args.base, "/v1/extract/words", payload)
92
- except urllib.error.HTTPError as exc:
93
- errors.append(f"{record['document_id']}: HTTP {exc.code}")
94
- print(f" ERROR {record['document_id']}: HTTP {exc.code}")
95
- continue
96
- latencies.append((time.perf_counter() - started) * 1000)
97
-
98
- hits = 0
99
- for field in body["fields"]:
100
- expected = str(record[TRUTH[field["key"]]]).strip()
101
- got = (field["value"] or "").strip()
102
- ok = got == expected
103
- per_key[field["key"]].append(1 if ok else 0)
104
- hits += ok
105
- if args.show and not ok:
106
- print(f" BAD {record['document_id']} {field['key']:<13} "
107
- f"got={got!r} want={expected!r} "
108
- f"status={field['status']} conf={field['confidence']:.3f}")
109
- print(f" {record['document_id']:<24} {hits}/{len(KEYS)} "
110
- f"{body['latency_ms']:.0f}ms")
111
-
112
- print("\n" + "=" * 68)
113
- print("PER-KEY ACCURACY (exact match)")
114
- print("=" * 68)
115
- total_hits = total_n = 0
116
- for key in KEYS:
117
- scores = per_key[key]
118
- if scores:
119
- acc = sum(scores) / len(scores)
120
- total_hits += sum(scores)
121
- total_n += len(scores)
122
- print(f" {key:<16} {acc:>7.1%} ({sum(scores)}/{len(scores)})")
123
- if total_n:
124
- print(f"\n {'OVERALL':<16} {total_hits/total_n:>7.1%} "
125
- f"({total_hits}/{total_n})")
126
-
127
- if latencies:
128
- ordered = sorted(latencies)
129
- def pct(p):
130
- return ordered[max(0, min(len(ordered) - 1,
131
- int(round(p / 100 * (len(ordered) - 1)))))]
132
- print("\n" + "=" * 68)
133
- print("LATENCY per document (8 keys, wall clock incl. HTTP)")
134
- print("=" * 68)
135
- print(f" mean {statistics.mean(latencies):8.0f} ms")
136
- print(f" p50 {pct(50):8.0f} ms")
137
- print(f" p95 {pct(95):8.0f} ms")
138
- print(f" max {max(latencies):8.0f} ms")
139
- print(f" per key {statistics.mean(latencies)/len(KEYS):6.0f} ms")
140
- print(f" throughput {len(manifest)/(statistics.mean(latencies)/1000):.2f} docs/s")
141
-
142
- if errors:
143
- print(f"\nERRORS: {len(errors)}")
144
- for error in errors[:10]:
145
- print(f" {error}")
146
-
147
- return 0
148
-
149
-
150
- if __name__ == "__main__":
151
- raise SystemExit(main())
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
docqa-rs/scripts/e2e.py DELETED
@@ -1,221 +0,0 @@
1
- #!/usr/bin/env python3
2
- """End-to-end check for the Rust docqa service.
3
-
4
- Requires the service running and ``models/layoutlm.onnx`` present:
5
-
6
- cargo run --release
7
- python scripts/e2e.py
8
-
9
- Verifies that the Rust port reproduces the reference PyTorch pipeline, because
10
- a Rust inference path that silently differs is worse than no service at all.
11
- """
12
-
13
- from __future__ import annotations
14
-
15
- import argparse
16
- import json
17
- import os
18
- import sys
19
- import time
20
- import urllib.error
21
- import urllib.request
22
- from pathlib import Path
23
-
24
- ROOT = Path(__file__).resolve().parent.parent
25
- KEYS = ["INVOICE NO", "Invoice Date", "Total Amount", "PO Number", "Payment Term"]
26
- TRUTH = {
27
- "INVOICE NO": "invoice_number",
28
- "Invoice Date": "invoice_date",
29
- "Total Amount": "total",
30
- "PO Number": "po_number",
31
- "Payment Term": "payment_terms",
32
- }
33
-
34
-
35
- def post_json(base: str, path: str, payload: dict, timeout: float = 120.0):
36
- request = urllib.request.Request(
37
- f"{base}{path}",
38
- data=json.dumps(payload).encode(),
39
- headers={"Content-Type": "application/json"},
40
- method="POST",
41
- )
42
- with urllib.request.urlopen(request, timeout=timeout) as response:
43
- return json.loads(response.read())
44
-
45
-
46
- def get(base: str, path: str, timeout: float = 30.0):
47
- with urllib.request.urlopen(f"{base}{path}", timeout=timeout) as response:
48
- return json.loads(response.read())
49
-
50
-
51
- def check_health(base: str) -> dict:
52
- body = get(base, "/health")
53
- assert body["status"] == "ok", f"service not ready: {body}"
54
- return body
55
-
56
-
57
- def check_validation(base: str) -> None:
58
- """Malformed requests must be rejected, not silently accepted."""
59
- cases = [
60
- ("no keys", {"keys": [], "words": ["a"], "boxes": [0, 0, 10, 10]}),
61
- ("blank keys", {"keys": [" "], "words": ["a"], "boxes": [0, 0, 10, 10]}),
62
- ("words without boxes",
63
- {"keys": ["Total"], "words": ["a", "b"]}),
64
- ("boxes length mismatch",
65
- {"keys": ["Total"], "words": ["a", "b"], "boxes": [0, 0, 1, 1]}),
66
- ]
67
- for name, payload in cases:
68
- try:
69
- post_json(base, "/v1/extract/words", payload)
70
- except urllib.error.HTTPError as exc:
71
- assert exc.code in (400, 422), f"{name}: expected 400/422, got {exc.code}"
72
- print(f" OK {name:<24} -> {exc.code}")
73
- else:
74
- raise AssertionError(f"{name}: expected an error, got 200")
75
-
76
- # Unsupported media type
77
- boundary = "----e2e"
78
- body = (
79
- f"--{boundary}\r\n"
80
- 'Content-Disposition: form-data; name="keys"\r\n\r\nTotal\r\n'
81
- f"--{boundary}\r\n"
82
- 'Content-Disposition: form-data; name="file"; filename="a.txt"\r\n'
83
- "Content-Type: text/plain\r\n\r\nnot a document\r\n"
84
- f"--{boundary}--\r\n"
85
- ).encode()
86
- request = urllib.request.Request(
87
- f"{base}/v1/extract/file", data=body,
88
- headers={"Content-Type": f"multipart/form-data; boundary={boundary}"},
89
- method="POST")
90
- try:
91
- urllib.request.urlopen(request, timeout=60)
92
- except urllib.error.HTTPError as exc:
93
- assert exc.code in (413, 415, 422), f"unsupported media: {exc.code}"
94
- print(f" OK {'unsupported media':<24} -> {exc.code}")
95
- else:
96
- raise AssertionError("unsupported media: expected an error")
97
-
98
-
99
- def check_extraction(base: str, pdf: Path, truth: dict) -> tuple[int, int]:
100
- """Extract from a real invoice and compare against ground truth."""
101
- import base64
102
-
103
- payload = {
104
- "document_base64": base64.b64encode(pdf.read_bytes()).decode(),
105
- "keys": KEYS,
106
- }
107
- started = time.perf_counter()
108
- body = post_json(base, "/v1/extract/json", payload)
109
- wall = (time.perf_counter() - started) * 1000
110
-
111
- print(f" document {body['document']['document_id']} "
112
- f"source={body['document']['source']} "
113
- f"words={body['document']['word_count']} "
114
- f"server={body['latency_ms']}ms wall={wall:.0f}ms")
115
-
116
- correct = 0
117
- for field in body["fields"]:
118
- expected = str(truth[TRUTH[field["key"]]])
119
- got = (field["value"] or "").strip()
120
- ok = got == expected.strip()
121
- correct += ok
122
- print(f" {'OK ' if ok else 'BAD'} {field['key']:<14} "
123
- f"{got[:34]:<36} {field['status']:<15} {field['confidence']:.3f}")
124
- if not ok:
125
- print(f" expected: {expected}")
126
- return correct, len(KEYS)
127
-
128
-
129
- def check_threshold(base: str, pdf: Path) -> None:
130
- """A threshold of 1.0 must suppress every extraction."""
131
- import base64
132
-
133
- body = post_json(base, "/v1/extract/json", {
134
- "document_base64": base64.b64encode(pdf.read_bytes()).decode(),
135
- "keys": KEYS,
136
- "confidence_threshold": 1.0,
137
- })
138
- assert body["confidence_threshold"] == 1.0, body["confidence_threshold"]
139
- extracted = [f for f in body["fields"] if f["status"] == "extracted"]
140
- assert not extracted, f"threshold=1.0 still extracted {len(extracted)} fields"
141
- print(f" OK {'threshold=1.0 gate':<24} suppressed all {len(body['fields'])}")
142
-
143
-
144
- def check_latency(base: str, pdf: Path, runs: int = 3) -> None:
145
- import base64
146
-
147
- encoded = base64.b64encode(pdf.read_bytes()).decode()
148
- samples = []
149
- for _ in range(runs):
150
- started = time.perf_counter()
151
- body = post_json(base, "/v1/extract/json",
152
- {"document_base64": encoded, "keys": KEYS})
153
- samples.append(body["latency_ms"])
154
- samples.sort()
155
- print(f" latency over {runs} runs: "
156
- f"min={samples[0]:.0f}ms median={samples[len(samples)//2]:.0f}ms "
157
- f"max={samples[-1]:.0f}ms for {len(KEYS)} keys")
158
-
159
-
160
- def main() -> int:
161
- parser = argparse.ArgumentParser()
162
- parser.add_argument("--base", default=os.environ.get("DOCQA_BASE",
163
- "http://127.0.0.1:7860"))
164
- parser.add_argument("--corpus", type=Path,
165
- default=Path(os.environ.get("DOCQA_CORPUS", "")) if os.environ.get("DOCQA_CORPUS") else None)
166
- parser.add_argument("--pdf", type=Path, default=None)
167
- args = parser.parse_args()
168
-
169
- pdf = args.pdf
170
- truth: dict = {}
171
- if pdf is None and args.corpus:
172
- manifest = json.loads((args.corpus / "ground_truth.json").read_text())
173
- pdf = Path(manifest[0]["path"])
174
- truth = manifest[0]
175
- if pdf is None:
176
- print("no PDF given; pass --pdf or --corpus", file=sys.stderr)
177
- return 2
178
-
179
- print(f"target: {args.base}")
180
- print(f"pdf : {pdf}\n")
181
-
182
- failures = []
183
-
184
- print("[1/4] health")
185
- body = check_health(args.base)
186
- print(f" model={body['model']} workers={body['workers']} "
187
- f"load={body['load_ms']}ms\n")
188
-
189
- print("[2/4] input validation")
190
- check_validation(args.base)
191
- print()
192
-
193
- print("[3/4] extraction")
194
- if truth:
195
- correct, total = check_extraction(args.base, pdf, truth)
196
- print(f"\n {correct}/{total} fields exactly correct\n")
197
- if correct < total:
198
- failures.append(f"only {correct}/{total} fields correct")
199
- else:
200
- body = post_json(args.base, "/v1/extract/words",
201
- {"words": ["Invoice", "Number", "INV-1"],
202
- "boxes": [0, 0, 100, 20, 110, 0, 220, 20, 230, 0, 320, 20],
203
- "keys": KEYS})
204
- print(f" {json.dumps(body['fields'][:1], indent=2)}\n")
205
-
206
- print("[4/4] threshold + latency")
207
- check_threshold(args.base, pdf)
208
- check_latency(args.base, pdf)
209
-
210
- print()
211
- if failures:
212
- print("FAILURES:")
213
- for failure in failures:
214
- print(f" - {failure}")
215
- return 1
216
- print("ALL E2E CHECKS PASSED")
217
- return 0
218
-
219
-
220
- if __name__ == "__main__":
221
- raise SystemExit(main())
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
docqa-rs/scripts/make_boxes.py DELETED
@@ -1,142 +0,0 @@
1
- #!/usr/bin/env python3
2
- """Build a valid /v1/extract/words request body from a real PDF or image.
3
-
4
- The API rejects words without boxes, and hand-writing normalised coordinates is
5
- error-prone. This does the extraction and normalisation for you.
6
-
7
- python scripts/make_boxes.py invoice.pdf --keys "INVOICE NO,Total Amount"
8
-
9
- prints a request body ready to paste into Postman or curl.
10
-
11
- Digital PDFs use pdfplumber. Scanned PDFs and images need Tesseract plus
12
- PyMuPDF to rasterise; both are listed in requirements.txt of the sibling
13
- docx_service project.
14
-
15
- Why normalise to 0..1000: LayoutLM looks up learned 2D position embeddings by
16
- coordinate, so it is trained on this scale. Feeding raw pixel values or 0..1
17
- floats yields silently wrong answers rather than an error.
18
- """
19
-
20
- from __future__ import annotations
21
-
22
- import argparse
23
- import json
24
- import shutil
25
- import subprocess
26
- import sys
27
- import tempfile
28
- from pathlib import Path
29
-
30
-
31
- def normalise(value: float, extent: float) -> int:
32
- """Map a pixel coordinate onto LayoutLM's 0..1000 grid."""
33
- if extent <= 0:
34
- return 0
35
- return max(0, min(1000, round(value / extent * 1000)))
36
-
37
-
38
- def from_pdf_digital(path: Path) -> tuple[list[str], list[int], int]:
39
- import pdfplumber
40
-
41
- words: list[str] = []
42
- boxes: list[int] = []
43
- with pdfplumber.open(str(path)) as pdf:
44
- page = pdf.pages[0]
45
- for word in page.extract_words():
46
- words.append(word["text"])
47
- boxes += [
48
- normalise(word["x0"], page.width),
49
- normalise(word["top"], page.height),
50
- normalise(word["x1"], page.width),
51
- normalise(word["bottom"], page.height),
52
- ]
53
- return words, boxes, len(pdf.pages)
54
-
55
-
56
- def from_image_ocr(path: Path) -> tuple[list[str], list[int], int]:
57
- """Rasterise then OCR, for scans and images."""
58
- if not shutil.which("tesseract"):
59
- sys.exit("tesseract not found on PATH; needed for scanned input")
60
- try:
61
- import fitz # PyMuPDF
62
- except ImportError:
63
- sys.exit("PyMuPDF is required to rasterise: pip install pymupdf")
64
-
65
- # 200 DPI is the usual sweet spot for word-level OCR accuracy vs. runtime.
66
- zoom = 200 / 72
67
- with tempfile.TemporaryDirectory() as tmp:
68
- rendered = []
69
- with fitz.open(str(path)) as doc:
70
- for index, page in enumerate(doc):
71
- pix = page.get_pixmap(matrix=fitz.Matrix(zoom, zoom))
72
- out = Path(tmp) / f"page-{index}.png"
73
- pix.save(str(out))
74
- rendered.append((out, pix.width, pix.height))
75
-
76
- words: list[str] = []
77
- boxes: list[int] = []
78
- for image, width, height in rendered:
79
- tsv = subprocess.run(
80
- ["tesseract", str(image), "stdout", "tsv"],
81
- capture_output=True, text=True, check=True,
82
- ).stdout
83
- for line in tsv.splitlines()[1:]:
84
- cols = line.split("\t")
85
- if len(cols) < 12 or cols[0] != "5":
86
- continue
87
- text = cols[11].strip()
88
- if not text:
89
- continue
90
- left, top = int(cols[6]), int(cols[7])
91
- w, h = int(cols[8]), int(cols[9])
92
- words.append(text)
93
- boxes += [
94
- normalise(left, width),
95
- normalise(top, height),
96
- normalise(left + w, width),
97
- normalise(top + h, height),
98
- ]
99
- return words, boxes, len(rendered)
100
-
101
-
102
- def main() -> int:
103
- parser = argparse.ArgumentParser(description=__doc__)
104
- parser.add_argument("document", type=Path)
105
- parser.add_argument("--keys", required=True,
106
- help='Comma-separated, e.g. "INVOICE NO,Total Amount"')
107
- parser.add_argument("--ocr", action="store_true",
108
- help="Force OCR even if a text layer exists")
109
- parser.add_argument("--base-url", default="http://127.0.0.1:7860")
110
- args = parser.parse_args()
111
-
112
- if not args.document.exists():
113
- sys.exit(f"no such file: {args.document}")
114
-
115
- keys = [k.strip() for k in args.keys.split(",") if k.strip()]
116
- if not keys:
117
- sys.exit("no non-empty keys given")
118
-
119
- if args.document.suffix.lower() == ".pdf" and not args.ocr:
120
- words, boxes, pages = from_pdf_digital(args.document)
121
- source = "pdf_text_layer"
122
- else:
123
- words, boxes, pages = from_image_ocr(args.document)
124
- source = "ocr"
125
-
126
- if not words:
127
- print("no words extracted; try --ocr for a scanned document",
128
- file=sys.stderr)
129
-
130
- payload = {"words": words, "boxes": boxes, "keys": keys}
131
- print(f"# source={source} pages={pages} words={len(words)} "
132
- f"box_values={len(boxes)} (expected {len(words) * 4})",
133
- file=sys.stderr)
134
- print(json.dumps(payload, indent=2))
135
- print(f"\n# curl:\n# curl -sS {args.base_url}/v1/extract/words "
136
- f"-H 'Content-Type: application/json' -d @request.json",
137
- file=sys.stderr)
138
- return 0
139
-
140
-
141
- if __name__ == "__main__":
142
- raise SystemExit(main())
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
docqa-rs/src/api.rs DELETED
@@ -1,474 +0,0 @@
1
- //! HTTP API.
2
- //!
3
- //! Three extract endpoints, all taking an array of user-defined keys:
4
- //!
5
- //! * `POST /v1/extract/words` — caller supplies `words` + normalised `boxes`.
6
- //! Fastest path: no parsing, no OCR, just inference. Also the only path that
7
- //! keeps this process free of a PDF/OCR dependency.
8
- //! * `POST /v1/extract/json` — base64 file, or words+boxes inline.
9
- //! * `POST /v1/extract/file` — multipart PDF/image upload.
10
- //!
11
- //! Responses always distinguish `extracted` / `low_confidence` / `not_found`.
12
- //! The underlying model always emits *some* span and never abstains, so absence
13
- //! has to be modelled explicitly or it silently becomes a fabricated value.
14
-
15
- use std::sync::Arc;
16
- use std::time::Instant;
17
-
18
- use axum::extract::{Multipart, State};
19
- use axum::http::StatusCode;
20
- use axum::response::{IntoResponse, Response};
21
- use axum::routing::{get, post};
22
- use axum::{Json, Router};
23
- use serde::{Deserialize, Serialize};
24
-
25
- use crate::engine::{document_id, Field, Page};
26
- use crate::parser;
27
- use crate::AppState;
28
-
29
- // ---------------------------------------------------------------------------
30
- // wire types
31
- // ---------------------------------------------------------------------------
32
- #[derive(Debug, Deserialize)]
33
- pub struct WordsRequest {
34
- /// Pre-extracted words in reading order.
35
- pub words: Vec<String>,
36
- /// Flat `[x0, y0, x1, y1, ...]`, length `words.len() * 4`, already
37
- /// normalised to 0..=1000 by the caller.
38
- #[serde(default)]
39
- pub boxes: Vec<i64>,
40
- pub keys: Vec<String>,
41
- #[serde(default)]
42
- pub confidence_threshold: Option<f32>,
43
- }
44
-
45
- #[derive(Debug, Deserialize)]
46
- pub struct JsonRequest {
47
- #[serde(default)]
48
- pub document_base64: Option<String>,
49
- #[serde(default)]
50
- pub words: Option<Vec<String>>,
51
- #[serde(default)]
52
- pub boxes: Option<Vec<i64>>,
53
- pub keys: Vec<String>,
54
- #[serde(default)]
55
- pub confidence_threshold: Option<f32>,
56
- }
57
-
58
- #[derive(Debug, Serialize)]
59
- pub struct DocumentInfo {
60
- pub document_id: String,
61
- pub page_count: usize,
62
- pub pages_processed: usize,
63
- pub word_count: usize,
64
- pub source: String,
65
- }
66
-
67
- #[derive(Debug, Serialize)]
68
- pub struct ExtractResponse {
69
- pub document: DocumentInfo,
70
- pub fields: Vec<Field>,
71
- pub latency_ms: f64,
72
- pub confidence_threshold: f32,
73
- }
74
-
75
- #[derive(Debug, Serialize)]
76
- pub struct HealthResponse {
77
- pub status: &'static str,
78
- pub model: String,
79
- pub load_ms: u128,
80
- pub workers: usize,
81
- pub busy_workers: usize,
82
- }
83
-
84
- #[derive(Debug, Serialize)]
85
- pub struct ErrorResponse {
86
- pub error: &'static str,
87
- pub message: String,
88
- }
89
-
90
- // ---------------------------------------------------------------------------
91
- // errors
92
- // ---------------------------------------------------------------------------
93
- #[derive(Debug)]
94
- pub struct ApiError {
95
- status: StatusCode,
96
- error: &'static str,
97
- message: String,
98
- }
99
-
100
- impl ApiError {
101
- fn bad_request(msg: impl Into<String>) -> Self {
102
- Self { status: StatusCode::BAD_REQUEST, error: "bad_request", message: msg.into() }
103
- }
104
- fn payload_too_large(msg: impl Into<String>) -> Self {
105
- Self { status: StatusCode::PAYLOAD_TOO_LARGE, error: "payload_too_large", message: msg.into() }
106
- }
107
- fn unprocessable(msg: impl Into<String>) -> Self {
108
- Self { status: StatusCode::UNPROCESSABLE_ENTITY, error: "unprocessable", message: msg.into() }
109
- }
110
- fn unavailable(msg: impl Into<String>) -> Self {
111
- Self { status: StatusCode::SERVICE_UNAVAILABLE, error: "unavailable", message: msg.into() }
112
- }
113
- fn internal(msg: impl Into<String>) -> Self {
114
- Self { status: StatusCode::INTERNAL_SERVER_ERROR, error: "internal", message: msg.into() }
115
- }
116
- }
117
-
118
- impl IntoResponse for ApiError {
119
- fn into_response(self) -> Response {
120
- (
121
- self.status,
122
- Json(ErrorResponse { error: self.error, message: self.message }),
123
- )
124
- .into_response()
125
- }
126
- }
127
-
128
- type ApiResult<T> = Result<T, ApiError>;
129
-
130
- // ---------------------------------------------------------------------------
131
- // helpers
132
- // ---------------------------------------------------------------------------
133
-
134
- /// Trim, drop blanks, and deduplicate while preserving order.
135
- ///
136
- /// A repeated key costs a full forward pass and returns an identical answer,
137
- /// so duplicates are collapsed rather than rejected.
138
- fn validate_keys(keys: &[String], max: usize) -> ApiResult<Vec<String>> {
139
- let mut seen = std::collections::HashSet::new();
140
- let mut out: Vec<String> = Vec::new();
141
- for key in keys {
142
- let cleaned = key.trim();
143
- if cleaned.is_empty() || cleaned.len() > 200 {
144
- continue;
145
- }
146
- if seen.insert(cleaned.to_string()) {
147
- out.push(cleaned.to_string());
148
- }
149
- }
150
- if out.is_empty() {
151
- return Err(ApiError::bad_request("at least one non-empty key is required"));
152
- }
153
- if out.len() > max {
154
- return Err(ApiError::bad_request(format!(
155
- "too many keys: {} > {max}", out.len()
156
- )));
157
- }
158
- Ok(out)
159
- }
160
-
161
- fn build_page(words: Vec<String>, boxes: Vec<i64>) -> ApiResult<Page> {
162
- if words.is_empty() {
163
- return Err(ApiError::unprocessable("no words supplied"));
164
- }
165
- if boxes.len() != words.len() * 4 {
166
- return Err(ApiError::bad_request(format!(
167
- "boxes has {} entries but {} words require {}",
168
- boxes.len(),
169
- words.len(),
170
- words.len() * 4
171
- )));
172
- }
173
- Ok(Page { words, boxes })
174
- }
175
-
176
- /// Shared extraction path for all three endpoints.
177
- async fn run_extraction(
178
- state: &AppState,
179
- pages: Vec<Page>,
180
- keys: Vec<String>,
181
- threshold: Option<f32>,
182
- doc_id: String,
183
- source: &str,
184
- page_count: usize,
185
- word_count: usize,
186
- ) -> ApiResult<ExtractResponse> {
187
- let started = Instant::now();
188
-
189
- if pages.iter().all(Page::is_empty) {
190
- return Err(ApiError::unprocessable(
191
- "no text could be extracted from the document",
192
- ));
193
- }
194
-
195
- let limit = state.config.max_pages.min(pages.len());
196
- let pages: Vec<Page> = pages.into_iter().take(limit).collect();
197
- let _ = &pages;
198
-
199
- let mut engine = state.pool.acquire().await.map_err(|e| {
200
- tracing::error!(error = %e, "no inference worker available");
201
- ApiError::unavailable("inference engine unavailable")
202
- })?;
203
-
204
- if let Some(t) = threshold {
205
- engine.set_confidence_threshold(t);
206
- }
207
-
208
- // Inference is CPU-bound and synchronous; keep it off the async worker.
209
- let fields = tokio::task::spawn_blocking(move || engine.extract(&pages, &keys))
210
- .await
211
- .map_err(|e| ApiError::internal(format!("extraction task failed: {e}")))?;
212
-
213
- let latency_ms = started.elapsed().as_secs_f64() * 1000.0;
214
- let n_keys = fields.len();
215
- let usable = fields.iter().filter(|f| f.status.is_usable()).count();
216
-
217
- tracing::info!(
218
- document_id = %doc_id,
219
- keys = n_keys,
220
- usable,
221
- latency_ms = format!("{latency_ms:.1}"),
222
- "extraction complete"
223
- );
224
-
225
- Ok(ExtractResponse {
226
- document: DocumentInfo {
227
- document_id: doc_id,
228
- page_count,
229
- pages_processed: limit,
230
- word_count,
231
- source: source.to_string(),
232
- },
233
- fields,
234
- latency_ms: (latency_ms * 100.0).round() / 100.0,
235
- confidence_threshold: threshold.unwrap_or(state.config.confidence_threshold),
236
- })
237
- }
238
-
239
- // ---------------------------------------------------------------------------
240
- // handlers
241
- // ---------------------------------------------------------------------------
242
- async fn health(State(state): State<Arc<AppState>>) -> Json<HealthResponse> {
243
- Json(HealthResponse {
244
- status: "ok",
245
- model: state.config.model_id.clone(),
246
- load_ms: state.load_ms,
247
- workers: state.config.workers,
248
- busy_workers: state.pool.busy(),
249
- })
250
- }
251
-
252
- async fn extract_words(
253
- State(state): State<Arc<AppState>>,
254
- Json(request): Json<WordsRequest>,
255
- ) -> ApiResult<Json<ExtractResponse>> {
256
- let keys = validate_keys(&request.keys, state.config.max_keys)?;
257
- let word_count = request.words.len();
258
- let page = build_page(request.words, request.boxes)?;
259
-
260
- let response = run_extraction(
261
- &state,
262
- vec![page],
263
- keys.clone(),
264
- request.confidence_threshold,
265
- document_id(keys.join(",").as_bytes()),
266
- "caller_supplied",
267
- 1,
268
- word_count,
269
- )
270
- .await?;
271
- Ok(Json(response))
272
- }
273
-
274
- async fn extract_json(
275
- State(state): State<Arc<AppState>>,
276
- Json(request): Json<JsonRequest>,
277
- ) -> ApiResult<Json<ExtractResponse>> {
278
- let keys = validate_keys(&request.keys, state.config.max_keys)?;
279
-
280
- // Fast path: caller already has words and their layout.
281
- if let Some(words) = request.words {
282
- let boxes = request.boxes.unwrap_or_default();
283
- if boxes.is_empty() {
284
- // LayoutLM reads learned x/y position embeddings. Without boxes it
285
- // has no layout, so refuse rather than answer from priors.
286
- return Err(ApiError::bad_request(
287
- "boxes are required alongside words: LayoutLM is layout-aware \
288
- and cannot run on bare text",
289
- ));
290
- }
291
- let word_count = words.len();
292
- let page = build_page(words, boxes)?;
293
- let response = run_extraction(
294
- &state,
295
- vec![page],
296
- keys.clone(),
297
- request.confidence_threshold,
298
- document_id(keys.join(",").as_bytes()),
299
- "caller_supplied",
300
- 1,
301
- word_count,
302
- )
303
- .await?;
304
- return Ok(Json(response));
305
- }
306
-
307
- let encoded = request
308
- .document_base64
309
- .ok_or_else(|| ApiError::bad_request("document_base64 or words is required"))?;
310
-
311
- let bytes = parser::decode_base64(&encoded)
312
- .map_err(|e| ApiError::bad_request(format!("invalid base64: {e}")))?;
313
-
314
- if bytes.len() > state.config.max_upload_mb * 1024 * 1024 {
315
- return Err(ApiError::payload_too_large(format!(
316
- "document exceeds {} MB", state.config.max_upload_mb
317
- )));
318
- }
319
-
320
- let parsed = parser::parse(&bytes, state.config.max_pages).map_err(|e| {
321
- // Parse failures are almost always a missing system dependency, so log
322
- // the cause at warn level rather than returning a bare 422 with no
323
- // server-side record of why.
324
- tracing::warn!(error = %e, bytes = bytes.len(), "document parse failed");
325
- ApiError::unprocessable(e.to_string())
326
- })?;
327
-
328
- let doc_id = document_id(&bytes);
329
- let response = run_extraction(
330
- &state,
331
- parsed.pages,
332
- keys,
333
- request.confidence_threshold,
334
- doc_id,
335
- &parsed.source,
336
- parsed.page_count,
337
- parsed.word_count,
338
- )
339
- .await?;
340
- Ok(Json(response))
341
- }
342
-
343
- async fn extract_file(
344
- State(state): State<Arc<AppState>>,
345
- mut multipart: Multipart,
346
- ) -> ApiResult<Json<ExtractResponse>> {
347
- let limit = state.config.max_upload_mb * 1024 * 1024;
348
- let mut bytes: Option<Vec<u8>> = None;
349
- let mut keys: Option<Vec<String>> = None;
350
-
351
- while let Some(field) = multipart
352
- .next_field()
353
- .await
354
- .map_err(|e| ApiError::bad_request(format!("malformed multipart: {e}")))?
355
- {
356
- match field.name().unwrap_or("") {
357
- "file" => {
358
- let data = field
359
- .bytes()
360
- .await
361
- .map_err(|e| ApiError::bad_request(format!("upload failed: {e}")))?;
362
- if data.len() > limit {
363
- return Err(ApiError::payload_too_large(format!(
364
- "document exceeds {} MB", state.config.max_upload_mb
365
- )));
366
- }
367
- bytes = Some(data.to_vec());
368
- }
369
- "keys" => {
370
- let text = field.text().await.unwrap_or_default();
371
- keys = Some(
372
- text.split(',')
373
- .map(|s| s.trim().to_string())
374
- .filter(|s| !s.is_empty())
375
- .collect(),
376
- );
377
- }
378
- _ => {}
379
- }
380
- }
381
-
382
- let bytes = bytes.ok_or_else(|| ApiError::bad_request("file is required"))?;
383
- let keys = validate_keys(&keys.unwrap_or_default(), state.config.max_keys)?;
384
-
385
- let parsed = parser::parse(&bytes, state.config.max_pages).map_err(|e| {
386
- tracing::warn!(error = %e, bytes = bytes.len(), "document parse failed");
387
- ApiError::unprocessable(e.to_string())
388
- })?;
389
-
390
- let doc_id = document_id(&bytes);
391
- let source = parsed.source.clone();
392
- let page_count = parsed.page_count;
393
- let word_count = parsed.word_count;
394
-
395
- let response = run_extraction(
396
- &state,
397
- parsed.pages,
398
- keys,
399
- None,
400
- doc_id,
401
- &source,
402
- page_count,
403
- word_count,
404
- )
405
- .await?;
406
- Ok(Json(response))
407
- }
408
-
409
- // ---------------------------------------------------------------------------
410
- // router
411
- // ---------------------------------------------------------------------------
412
- pub fn router(state: Arc<AppState>) -> Router {
413
- Router::new()
414
- .route("/health", get(health))
415
- .route("/v1/extract/words", post(extract_words))
416
- .route("/v1/extract/json", post(extract_json))
417
- .route("/v1/extract/file", post(extract_file))
418
- .with_state(state)
419
- }
420
-
421
- #[cfg(test)]
422
- mod tests {
423
- use super::*;
424
-
425
- #[test]
426
- fn keys_are_trimmed_and_deduplicated() {
427
- let keys = vec![
428
- " Total ".to_string(),
429
- "Total".to_string(),
430
- "".to_string(),
431
- " ".to_string(),
432
- "GSTIN".to_string(),
433
- ];
434
- assert_eq!(validate_keys(&keys, 10).unwrap(), vec!["Total", "GSTIN"]);
435
- }
436
-
437
- #[test]
438
- fn keys_reject_all_blank() {
439
- let keys = vec!["".to_string(), " ".to_string()];
440
- assert!(validate_keys(&keys, 10).is_err());
441
- }
442
-
443
- #[test]
444
- fn keys_reject_overlong() {
445
- let keys = vec!["x".repeat(500)];
446
- assert!(validate_keys(&keys, 10).is_err());
447
- }
448
-
449
- #[test]
450
- fn keys_enforce_max() {
451
- let keys: Vec<String> = (0..10).map(|i| format!("k{i}")).collect();
452
- assert!(validate_keys(&keys, 5).is_err());
453
- assert!(validate_keys(&keys, 10).is_ok());
454
- }
455
-
456
- #[test]
457
- fn page_rejects_mismatched_boxes() {
458
- let words = vec!["a".into(), "b".into()];
459
- let boxes = vec![0, 0, 10, 10]; // needs 8
460
- assert!(build_page(words, boxes).is_err());
461
- }
462
-
463
- #[test]
464
- fn page_rejects_empty_words() {
465
- assert!(build_page(vec![], vec![]).is_err());
466
- }
467
-
468
- #[test]
469
- fn page_accepts_matching_shapes() {
470
- let words = vec!["a".into(), "b".into()];
471
- let boxes = vec![0, 0, 10, 10, 20, 20, 30, 30];
472
- assert!(build_page(words, boxes).is_ok());
473
- }
474
- }
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
docqa-rs/src/config.rs DELETED
@@ -1,54 +0,0 @@
1
- //! Configuration from environment variables.
2
-
3
- use std::env;
4
-
5
- #[derive(Debug, Clone)]
6
- pub struct Config {
7
- pub bind: String,
8
- pub port: u16,
9
- pub model_path: String,
10
- pub tokenizer_path: String,
11
- pub model_id: String,
12
- pub threads: usize,
13
- /// Workers, each owning its own ONNX session.
14
- ///
15
- /// `ort::Session::run` takes `&mut self` because ONNX Runtime internals are
16
- /// not thread-safe, so concurrency is obtained from N sessions rather than
17
- /// N threads over one session.
18
- pub workers: usize,
19
- pub confidence_threshold: f32,
20
- pub max_keys: usize,
21
- pub max_upload_mb: usize,
22
- pub max_pages: usize,
23
- pub request_timeout_s: u64,
24
- }
25
-
26
- fn var<T: std::str::FromStr>(key: &str, default: T) -> T {
27
- env::var(key).ok().and_then(|v| v.parse().ok()).unwrap_or(default)
28
- }
29
-
30
- impl Config {
31
- pub fn from_env() -> Self {
32
- let model_dir = var("DOCQA_MODEL_DIR", "./models".to_string());
33
- Self {
34
- bind: var("DOCQA_BIND", "0.0.0.0".to_string()),
35
- port: var("DOCQA_PORT", 7860),
36
- model_path: var(
37
- "DOCQA_MODEL_PATH",
38
- format!("{model_dir}/layoutlm.onnx"),
39
- ),
40
- tokenizer_path: var(
41
- "DOCQA_TOKENIZER_PATH",
42
- format!("{model_dir}/tokenizer.json"),
43
- ),
44
- model_id: var("DOCQA_MODEL_ID", "impira/layoutlm-invoices".to_string()),
45
- threads: var("DOCQA_THREADS", 4),
46
- workers: var("DOCQA_WORKERS", 2),
47
- confidence_threshold: var("DOCQA_CONFIDENCE_THRESHOLD", 0.5),
48
- max_keys: var("DOCQA_MAX_KEYS", 64),
49
- max_upload_mb: var("DOCQA_MAX_UPLOAD_MB", 25),
50
- max_pages: var("DOCQA_MAX_PAGES", 3),
51
- request_timeout_s: var("DOCQA_REQUEST_TIMEOUT_S", 60),
52
- }
53
- }
54
- }
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
docqa-rs/src/engine.rs DELETED
@@ -1,428 +0,0 @@
1
- //! Document field extraction engine: LayoutLM span QA over ONNX Runtime.
2
- //!
3
- //! # Why this is not llama.cpp
4
- //!
5
- //! `impira/layoutlm-invoices` is a **bidirectional encoder** with a two-span
6
- //! extractive head (start/end logits) plus learned 2D position embeddings
7
- //! derived from word bounding boxes. llama.cpp loads decoder-only GGUF models
8
- //! and has no representation for either concept, so there is no conversion
9
- //! path. ONNX Runtime is the correct runtime; `ort` is its Rust binding.
10
- //!
11
- //! # Semantics that must match the reference exactly
12
- //!
13
- //! These were established by reading
14
- //! `transformers/pipelines/document_question_answering.py` and verified
15
- //! against PyTorch. Getting any of them wrong yields a fast service that
16
- //! confidently returns the *wrong span*, which is worse than a slow one.
17
- //!
18
- //! 1. **SEP tokens carry bbox `[1000,1000,1000,1000]`**, not `[0,0,0,0]`.
19
- //! Question and CLS tokens carry `[0,0,0,0]`; context tokens carry the
20
- //! word's real box. LayoutLM indexes learned x/y/width/height embeddings,
21
- //! so mis-assigning SEP shifts the encoding and changes the answer.
22
- //! 2. **Answers are the original word strings joined by spaces**, not a
23
- //! decode of subword ids. These differ for dates, currency symbols and
24
- //! hyphenated identifiers (GSTIN, EIN).
25
- //! 3. **Only context tokens may open or close a span** (`p_mask` in the
26
- //! reference pipeline).
27
- //! 4. **Start and end logits are independent softmaxes.** The score is the
28
- //! product. Taking `exp(start)*exp(end)` on raw logits overflows to
29
- //! values far above 1.
30
-
31
- use anyhow::{anyhow, Context, Result};
32
- use ort::session::Session;
33
- use ort::value::TensorRef;
34
- use tokenizers::tokenizer::Tokenizer;
35
-
36
- /// One extracted field.
37
- #[derive(Debug, Clone, serde::Serialize)]
38
- pub struct Field {
39
- pub key: String,
40
- pub value: Option<String>,
41
- pub status: Status,
42
- pub confidence: f32,
43
- pub page: usize,
44
- }
45
-
46
- #[derive(Debug, Clone, Copy, PartialEq, Eq, serde::Serialize)]
47
- #[serde(rename_all = "snake_case")]
48
- pub enum Status {
49
- /// Answer is above the confidence threshold.
50
- Extracted,
51
- /// A span was found but confidence is below the threshold.
52
- LowConfidence,
53
- /// Nothing usable was found.
54
- NotFound,
55
- /// The field could not be processed.
56
- Error,
57
- }
58
-
59
- impl Status {
60
- pub fn is_usable(&self) -> bool {
61
- matches!(self, Status::Extracted)
62
- }
63
- }
64
-
65
- /// A document's text plus per-word bounding boxes, normalised to 0..=1000.
66
- #[derive(Debug, Clone, Default)]
67
- pub struct Page {
68
- pub words: Vec<String>,
69
- /// Flat `[x0, y0, x1, y1, ...]` in LayoutLM's 0..=1000 space.
70
- pub boxes: Vec<i64>,
71
- }
72
-
73
- impl Page {
74
- pub fn is_empty(&self) -> bool {
75
- self.words.is_empty()
76
- }
77
-
78
- pub fn box_for(&self, index: usize) -> [i64; 4] {
79
- let b = index * 4;
80
- [self.boxes[b], self.boxes[b + 1], self.boxes[b + 2], self.boxes[b + 3]]
81
- }
82
- }
83
-
84
- /// The inference engine.
85
- pub struct Engine {
86
- session: Session,
87
- tokenizer: Tokenizer,
88
- pub model_id: String,
89
- /// LayoutLM's learned position embeddings are indexed 0..=1000.
90
- pub confidence_threshold: f32,
91
- pub load_ms: u128,
92
- }
93
-
94
- impl Engine {
95
- /// Load the ONNX graph and the matching tokenizer.
96
- ///
97
- /// `model_path` must be the ONNX graph exported with a `(batch, seq*4)`
98
- /// bbox input; see `export_onnx.py` in the sibling Python project.
99
- pub fn load(
100
- model_path: &str,
101
- tokenizer_path: &str,
102
- model_id: &str,
103
- threads: usize,
104
- confidence_threshold: f32,
105
- ) -> Result<Self> {
106
- let started = std::time::Instant::now();
107
-
108
- let mut builder = Session::builder().context("create session builder")?;
109
- // Note: these return `ort::error::Error<SessionBuilder>`, which is not
110
- // `Send + Sync` and therefore cannot cross `?` into `anyhow`. Map to a
111
- // string immediately so the builder's recovery value is not lost.
112
- builder = builder
113
- .with_intra_threads(threads)
114
- .map_err(|e| anyhow!("with_intra_threads: {e}"))?;
115
- // Inter-op parallelism only adds synchronisation overhead for a graph
116
- // that is a stack of independent attention blocks.
117
- builder = builder
118
- .with_inter_threads(1)
119
- .map_err(|e| anyhow!("with_inter_threads: {e}"))?;
120
- let session = builder
121
- .commit_from_file(model_path)
122
- .with_context(|| format!("load ONNX graph from {model_path}"))?;
123
-
124
- let tokenizer = Tokenizer::from_file(tokenizer_path)
125
- .map_err(|e| anyhow!("load tokenizer from {tokenizer_path}: {e}"))?;
126
-
127
- tracing::info!(
128
- model = model_id,
129
- threads,
130
- load_ms = started.elapsed().as_millis(),
131
- "engine ready"
132
- );
133
-
134
- Ok(Self {
135
- session,
136
- tokenizer,
137
- model_id: model_id.to_string(),
138
- confidence_threshold,
139
- load_ms: started.elapsed().as_millis(),
140
- })
141
- }
142
-
143
- /// Per-request override of the confidence threshold.
144
- pub fn set_confidence_threshold(&mut self, threshold: f32) {
145
- self.confidence_threshold = threshold;
146
- }
147
-
148
- /// Answer one question against one page.
149
- pub fn answer(&mut self, page: &Page, key: &str) -> Result<Field> {
150
- if page.is_empty() {
151
- // Asking the model about an empty page only invites fabrication.
152
- return Ok(Field {
153
- key: key.to_string(),
154
- value: None,
155
- status: Status::NotFound,
156
- confidence: 0.0,
157
- page: 0,
158
- });
159
- }
160
-
161
- let question_tokens: Vec<&str> = key.split_whitespace().collect();
162
-
163
- let encoding = self
164
- .tokenizer
165
- .encode((question_tokens.as_slice(), page.words.as_slice()), true)
166
- .map_err(|e| anyhow!("tokenize question + document: {e}"))?;
167
-
168
- // `tokenizers` yields u32, but the exported graph declares every id
169
- // tensor as int64. Feeding u32 straight through fails at run time with
170
- // "Unexpected input data type", so cast explicitly rather than relying
171
- // on an implicit conversion that would silently pick the wrong width.
172
- let raw_ids: Vec<u32> = encoding.get_ids().to_vec();
173
- let raw_type_ids: Vec<u32> = encoding.get_type_ids().to_vec();
174
- let raw_attention: Vec<u32> = encoding.get_attention_mask().to_vec();
175
- let input_ids: Vec<i64> = raw_ids.iter().map(|v| *v as i64).collect();
176
- let type_ids: Vec<i64> = raw_type_ids.iter().map(|v| *v as i64).collect();
177
- let attention_mask: Vec<i64> = raw_attention.iter().map(|v| *v as i64).collect();
178
- // `None` marks tokens that belong to no input word: special tokens and
179
- // the question.
180
- let raw_word_ids: &[Option<u32>] = encoding.get_word_ids();
181
- let sep_id = self.tokenizer.token_to_id("</s>");
182
-
183
- // Bounding boxes in LayoutLM's 0..=1000 space, flattened to seq*4
184
- // because that is the rank the exported graph accepts.
185
- let mut bbox: Vec<i64> = Vec::with_capacity(input_ids.len() * 4);
186
- for (position, &token_id) in input_ids.iter().enumerate() {
187
- let is_context = type_ids.get(position).copied().unwrap_or(0) == 1;
188
- let word_index = raw_word_ids.get(position).copied().flatten();
189
- if is_context {
190
- if let Some(index) = word_index {
191
- let coords = page.box_for(index as usize);
192
- bbox.extend_from_slice(&coords);
193
- continue;
194
- }
195
- }
196
- // SEP gets a distinct sentinel box; everything else zeros.
197
- // `sep_id` is u32 from the tokenizer, so compare in the same width.
198
- let coords = if sep_id == Some(token_id as u32) {
199
- [1000, 1000, 1000, 1000]
200
- } else {
201
- [0, 0, 0, 0]
202
- };
203
- bbox.extend_from_slice(&coords);
204
- }
205
-
206
- // The graph declares (batch, seq) for the id tensors and (batch, seq*4)
207
- // for bbox, so shapes are passed explicitly rather than inferred.
208
- let seq_shape = [1usize, input_ids.len()];
209
- let bbox_shape = [1usize, input_ids.len() * 4];
210
-
211
- let outputs = self
212
- .session
213
- .run(ort::inputs![
214
- TensorRef::from_array_view((seq_shape, input_ids.as_slice()))
215
- .map_err(|e| anyhow!("input_ids tensor: {e}"))?,
216
- TensorRef::from_array_view((seq_shape, attention_mask.as_slice()))
217
- .map_err(|e| anyhow!("attention_mask tensor: {e}"))?,
218
- TensorRef::from_array_view((seq_shape, type_ids.as_slice()))
219
- .map_err(|e| anyhow!("token_type_ids tensor: {e}"))?,
220
- TensorRef::from_array_view((bbox_shape, bbox.as_slice()))
221
- .map_err(|e| anyhow!("bbox tensor: {e}"))?,
222
- ])
223
- .context("onnxruntime inference")?;
224
-
225
- let (start_shape, start_slice) = outputs["start_logits"]
226
- .try_extract_tensor::<f32>()
227
- .context("extract start_logits")?;
228
- let (_end_shape, end_slice) = outputs["end_logits"]
229
- .try_extract_tensor::<f32>()
230
- .context("extract end_logits")?;
231
-
232
- let start: Vec<f32> = start_slice.to_vec();
233
- let end: Vec<f32> = end_slice.to_vec();
234
- let _ = start_shape;
235
- if start.len() != input_ids.len() || end.len() != input_ids.len() {
236
- return Err(anyhow!(
237
- "unexpected logit length: start={} end={} seq={}",
238
- start.len(),
239
- end.len(),
240
- input_ids.len()
241
- ));
242
- }
243
-
244
- // Independent softmax per axis; the reference pipeline's score is the
245
- // product of the two probabilities.
246
- let start_prob = softmax(&start);
247
- let end_prob = softmax(&end);
248
-
249
- // Only context tokens may start or end an answer span.
250
- let (mut best_start, mut best_end) = (0usize, 0usize);
251
- let mut best = f32::NEG_INFINITY;
252
- for i in 0..input_ids.len() {
253
- if type_ids.get(i).copied().unwrap_or(0) != 1 {
254
- continue;
255
- }
256
- let score = start_prob[i] + end_prob[i];
257
- if score > best {
258
- best = score;
259
- best_start = i;
260
- best_end = i;
261
- }
262
- }
263
-
264
- let confidence = (start_prob[best_start] * end_prob[best_end]).clamp(0.0, 1.0);
265
-
266
- // Answer is the original word span, not the subword decode.
267
- let answer = {
268
- let w_start = raw_word_ids.get(best_start).copied().flatten();
269
- let w_end = raw_word_ids.get(best_end).copied().flatten();
270
- match (w_start, w_end) {
271
- (Some(a), Some(b)) if b >= a => page
272
- .words
273
- .get(a as usize..=b as usize)
274
- .map(|slice| slice.join(" "))
275
- .unwrap_or_default(),
276
- _ => String::new(),
277
- }
278
- };
279
-
280
- let status = if answer.is_empty() {
281
- Status::NotFound
282
- } else if confidence < self.confidence_threshold {
283
- Status::LowConfidence
284
- } else {
285
- Status::Extracted
286
- };
287
-
288
- Ok(Field {
289
- key: key.to_string(),
290
- value: if answer.is_empty() { None } else { Some(answer) },
291
- status,
292
- confidence,
293
- page: 1,
294
- })
295
- }
296
-
297
- /// Answer several keys, returning the best-scoring result per key.
298
- ///
299
- /// Multi-page documents are scored page by page; the highest-confidence
300
- /// span wins, so a value that only appears on page 3 is still found.
301
- pub fn extract(&mut self, pages: &[Page], keys: &[String]) -> Vec<Field> {
302
- let mut best: Vec<Option<Field>> = vec![None; keys.len()];
303
-
304
- for page in pages.iter() {
305
- if page.is_empty() {
306
- continue;
307
- }
308
- for (i, key) in keys.iter().enumerate() {
309
- match self.answer(page, key) {
310
- Ok(field) => {
311
- let replace = match &best[i] {
312
- None => true,
313
- Some(existing) => field.confidence > existing.confidence,
314
- };
315
- if replace {
316
- best[i] = Some(field);
317
- }
318
- }
319
- Err(error) => {
320
- // `anyhow` prints only the outermost context with `%`;
321
- // `{:#}` renders the full causal chain, which is what
322
- // actually identifies an ONNX Runtime shape mismatch.
323
- tracing::warn!(
324
- key = %key,
325
- error = format!("{error:#}"),
326
- "extraction failed"
327
- );
328
- if best[i].is_none() {
329
- best[i] = Some(Field {
330
- key: key.clone(),
331
- value: None,
332
- status: Status::Error,
333
- confidence: 0.0,
334
- page: 0,
335
- });
336
- }
337
- }
338
- }
339
- }
340
- }
341
-
342
- keys.iter()
343
- .enumerate()
344
- .map(|(i, key)| {
345
- best[i].clone().unwrap_or(Field {
346
- key: key.clone(),
347
- value: None,
348
- status: Status::NotFound,
349
- confidence: 0.0,
350
- page: 0,
351
- })
352
- })
353
- .collect()
354
- }
355
- }
356
-
357
- /// Numerically stable softmax.
358
- fn softmax(values: &[f32]) -> Vec<f32> {
359
- let max = values.iter().copied().fold(f32::NEG_INFINITY, f32::max);
360
- if !max.is_finite() {
361
- return vec![0.0; values.len()];
362
- }
363
- let mut exps: Vec<f32> = values.iter().map(|v| (v - max).exp()).collect();
364
- let sum: f32 = exps.iter().sum();
365
- if sum <= 0.0 || !sum.is_finite() {
366
- return vec![0.0; values.len()];
367
- }
368
- for value in exps.iter_mut() {
369
- *value /= sum;
370
- }
371
- exps
372
- }
373
-
374
- /// Content-addressed cache key for a parsed document.
375
- pub fn document_id(bytes: &[u8]) -> String {
376
- use sha2::{Digest, Sha256};
377
- let digest = Sha256::digest(bytes);
378
- hex_prefix(&digest, 16)
379
- }
380
-
381
- fn hex_prefix(bytes: &[u8], chars: usize) -> String {
382
- bytes.iter().take(chars).map(|b| format!("{b:02x}")).collect()
383
- }
384
-
385
- #[cfg(test)]
386
- mod tests {
387
- use super::*;
388
-
389
- #[test]
390
- fn softmax_sums_to_one() {
391
- let p = softmax(&[1.0, 2.0, 3.0]);
392
- let sum: f32 = p.iter().sum();
393
- assert!((sum - 1.0).abs() < 1e-5, "sum was {sum}");
394
- }
395
-
396
- #[test]
397
- fn softmax_is_overflow_safe() {
398
- // Raw LayoutLM logits can be large enough that a naive exp overflows.
399
- let p = softmax(&[1000.0, 1001.0, 999.0]);
400
- assert!(p.iter().all(|v| v.is_finite()));
401
- let sum: f32 = p.iter().sum();
402
- assert!((sum - 1.0).abs() < 1e-5);
403
- }
404
-
405
- #[test]
406
- fn page_box_lookup() {
407
- let page = Page {
408
- words: vec!["a".into(), "b".into()],
409
- boxes: vec![0, 0, 10, 10, 20, 20, 30, 30],
410
- };
411
- assert_eq!(page.box_for(0), [0, 0, 10, 10]);
412
- assert_eq!(page.box_for(1), [20, 20, 30, 30]);
413
- }
414
-
415
- #[test]
416
- fn empty_page_yields_not_found_without_inference() {
417
- // Guards the "don't ask the model about nothing" behaviour.
418
- let page = Page::default();
419
- assert!(page.is_empty());
420
- }
421
-
422
- #[test]
423
- fn document_id_is_stable_and_content_addressed() {
424
- assert_eq!(document_id(b"hello"), document_id(b"hello"));
425
- assert_ne!(document_id(b"hello"), document_id(b"world"));
426
- assert_eq!(document_id(b"hello").len(), 32);
427
- }
428
- }
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
docqa-rs/src/main.rs DELETED
@@ -1,111 +0,0 @@
1
- //! Document field-extraction API.
2
- //!
3
- //! Accepts a document (PDF/image, or caller-supplied words+boxes) and an array
4
- //! of user-defined keys, and returns one value per key with an explicit status.
5
- //!
6
- //! ```text
7
- //! POST /v1/extract/words {"words": [...], "boxes": [...], "keys": [...]}
8
- //! POST /v1/extract/json {"document_base64": "...", "keys": [...]}
9
- //! POST /v1/extract/file multipart: file=<pdf>, keys="A,B,C"
10
- //! GET /health
11
- //! ```
12
-
13
- mod api;
14
- mod config;
15
- mod engine;
16
- mod parser;
17
- mod pool;
18
-
19
- use std::sync::Arc;
20
-
21
- use anyhow::Result;
22
- use axum::Router;
23
- use tower_http::limit::RequestBodyLimitLayer;
24
- use tower_http::timeout::TimeoutLayer;
25
-
26
- use crate::config::Config;
27
- use crate::pool::EnginePool;
28
-
29
- pub struct AppState {
30
- pub config: Config,
31
- pub pool: EnginePool,
32
- pub load_ms: u128,
33
- }
34
-
35
- #[tokio::main]
36
- async fn main() -> Result<()> {
37
- tracing_subscriber::fmt()
38
- .with_env_filter(
39
- tracing_subscriber::EnvFilter::try_from_default_env()
40
- .unwrap_or_else(|_| "docqa=info,tower_http=warn".into()),
41
- )
42
- .json()
43
- .init();
44
-
45
- let config = Config::from_env();
46
-
47
- tracing::info!(
48
- model = %config.model_id,
49
- model_path = %config.model_path,
50
- workers = config.workers,
51
- threads = config.threads,
52
- "starting document extraction service"
53
- );
54
-
55
- // Fail fast: a missing model or tokenizer should stop the boot, not
56
- // surface as 500s under load.
57
- let load_started = std::time::Instant::now();
58
- let pool = EnginePool::new(&config)?;
59
- let load_ms = load_started.elapsed().as_millis();
60
-
61
- let max_bytes = config.max_upload_mb * 1024 * 1024;
62
- let state = Arc::new(AppState {
63
- config: config.clone(),
64
- pool,
65
- load_ms,
66
- });
67
-
68
- let app = Router::new()
69
- .merge(api::router(state))
70
- // Bound request bodies before any handler reads them, so an oversized
71
- // upload cannot be buffered into memory.
72
- .layer(RequestBodyLimitLayer::new(max_bytes))
73
- .layer(TimeoutLayer::with_status_code(
74
- axum::http::StatusCode::GATEWAY_TIMEOUT,
75
- std::time::Duration::from_secs(config.request_timeout_s),
76
- ));
77
-
78
- let address = format!("{}:{}", config.bind, config.port);
79
- let listener = tokio::net::TcpListener::bind(&address).await?;
80
- tracing::info!(%address, load_ms, model = %config.model_id, "listening");
81
-
82
- axum::serve(listener, app)
83
- .with_graceful_shutdown(shutdown_signal())
84
- .await?;
85
-
86
- tracing::info!("shutdown complete");
87
- Ok(())
88
- }
89
-
90
- async fn shutdown_signal() {
91
- let ctrl_c = async {
92
- let _ = tokio::signal::ctrl_c().await;
93
- };
94
-
95
- #[cfg(unix)]
96
- let terminate = async {
97
- if let Ok(mut sig) =
98
- tokio::signal::unix::signal(tokio::signal::unix::SignalKind::terminate())
99
- {
100
- sig.recv().await;
101
- }
102
- };
103
- #[cfg(not(unix))]
104
- let terminate = std::future::pending::<()>();
105
-
106
- tokio::select! {
107
- _ = ctrl_c => tracing::info!("received SIGINT"),
108
- _ = terminate => tracing::info!("received SIGTERM"),
109
- }
110
- }
111
-
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
docqa-rs/src/parser.rs DELETED
@@ -1,576 +0,0 @@
1
- //! Document parsing: bytes -> pages of words + boxes.
2
- //!
3
- //! PDF parsing pulls the text layer, which is fast and gives exact characters.
4
- //! When a PDF has no text layer (a scan), or for image uploads, this shells out
5
- //! to the `tesseract` binary and reads TSV output. Keeping OCR out-of-process
6
- //! means a misbehaving OCR run cannot take the API down with it, and avoids a
7
- //! heavy native dependency in the Rust binary.
8
- //!
9
- //! Boxes are scaled to LayoutLM's 0..=1000 grid and clamped, because the model
10
- //! indexes learned position embeddings and out-of-range values raise rather
11
- //! than degrade.
12
-
13
- use std::io::Write;
14
- use std::process::{Command, Stdio};
15
-
16
- use anyhow::{anyhow, bail, Context, Result};
17
-
18
- use crate::engine::Page;
19
-
20
- pub struct Parsed {
21
- pub pages: Vec<Page>,
22
- pub page_count: usize,
23
- pub word_count: usize,
24
- pub source: String,
25
- }
26
-
27
- pub fn decode_base64(input: &str) -> Result<Vec<u8>> {
28
- // Tolerate a data: URL prefix.
29
- let payload = match input.find(',') {
30
- Some(idx) if input[..idx].starts_with("data:") => &input[idx + 1..],
31
- _ => input,
32
- };
33
- let cleaned: String = payload.chars().filter(|c| !c.is_whitespace()).collect();
34
- decode_base64_inner(&cleaned)
35
- }
36
-
37
- fn decode_base64_inner(input: &str) -> Result<Vec<u8>> {
38
- const TABLE: &[u8; 64] = b"ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/";
39
- let mut lookup = [255u8; 256];
40
- for (i, c) in TABLE.iter().enumerate() {
41
- lookup[*c as usize] = i as u8;
42
- }
43
-
44
- let bytes: Vec<u8> = input.bytes().collect();
45
- let mut out = Vec::with_capacity(bytes.len() / 4 * 3);
46
- let mut buffer = 0u32;
47
- let mut bits = 0u32;
48
-
49
- for b in bytes {
50
- if b == b'=' {
51
- break;
52
- }
53
- let value = lookup[b as usize];
54
- if value == 255 {
55
- bail!("invalid base64 character {:?}", b as char);
56
- }
57
- buffer = (buffer << 6) | value as u32;
58
- bits += 6;
59
- if bits >= 8 {
60
- bits -= 8;
61
- out.push(((buffer >> bits) & 0xFF) as u8);
62
- }
63
- }
64
- Ok(out)
65
- }
66
-
67
- /// Identify the payload without a full parse.
68
- fn sniff(bytes: &[u8]) -> &'static str {
69
- if bytes.starts_with(b"%PDF-") {
70
- "pdf"
71
- } else if bytes.starts_with(b"\x89PNG\r\n\x1a\n") {
72
- "png"
73
- } else if bytes.starts_with(b"\xFF\xD8\xFF") {
74
- "jpeg"
75
- } else if bytes.starts_with(b"GIF8") {
76
- "gif"
77
- } else if bytes.starts_with(b"BM") {
78
- "bmp"
79
- } else if bytes.starts_with(b"II*\x00") || bytes.starts_with(b"MM\x00*") {
80
- "tiff"
81
- } else {
82
- "unknown"
83
- }
84
- }
85
-
86
- /// Clamp a coordinate into LayoutLM's 0..=1000 range.
87
- fn clamp(value: f64) -> i64 {
88
- value.round().clamp(0.0, 1000.0) as i64
89
- }
90
-
91
- pub fn parse(bytes: &[u8], max_pages: usize) -> Result<Parsed> {
92
- if bytes.is_empty() {
93
- bail!("empty upload");
94
- }
95
- match sniff(bytes) {
96
- "pdf" => parse_pdf(bytes, max_pages),
97
- "unknown" => bail!(
98
- "unsupported file type: expected PDF, PNG, JPEG, TIFF, BMP or GIF"
99
- ),
100
- _ => {
101
- let page = ocr_image(bytes)?;
102
- let word_count = page.words.len();
103
- Ok(Parsed {
104
- pages: vec![page],
105
- page_count: 1,
106
- word_count,
107
- source: "image_ocr".to_string(),
108
- })
109
- }
110
- }
111
- }
112
-
113
- // ---------------------------------------------------------------------------
114
- // PDF
115
- // ---------------------------------------------------------------------------
116
- fn parse_pdf(bytes: &[u8], max_pages: usize) -> Result<Parsed> {
117
- let pages = pdf_words(bytes, max_pages)?;
118
-
119
- // A scan has pages but no text; fall back to rasterise + OCR.
120
- let has_text = pages.iter().any(|p| !p.is_empty());
121
- if has_text {
122
- let page_count = pages.len();
123
- let word_count = pages.iter().map(|p| p.words.len()).sum();
124
- return Ok(Parsed {
125
- pages,
126
- page_count,
127
- word_count,
128
- source: "pdf_text_layer".to_string(),
129
- });
130
- }
131
-
132
- tracing::info!("no PDF text layer; falling back to OCR");
133
- let page_count = pages.len();
134
- let mut ocr_pages = Vec::new();
135
- let mut word_count = 0;
136
- for (index, _) in pages.iter().enumerate() {
137
- let rendered = pdftoppm_png(bytes, index + 1)?;
138
- let page = ocr_image(&rendered)?;
139
- word_count += page.words.len();
140
- ocr_pages.push(page);
141
- }
142
- Ok(Parsed {
143
- pages: ocr_pages,
144
- page_count,
145
- word_count,
146
- source: "pdf_ocr".to_string(),
147
- })
148
- }
149
-
150
- /// Extract words and boxes from a PDF text layer via `pdftotext -bbox`.
151
- fn pdf_words(bytes: &[u8], max_pages: usize) -> Result<Vec<Page>> {
152
- let output = run_pdftotext_bbox(bytes).context("pdftotext failed")?;
153
-
154
- let mut pages = parse_bbox_xml(&output, max_pages)?;
155
- // If the tool is absent or the file is unusual, report zero pages and let
156
- // the caller decide; the OCR fallback covers scanned documents.
157
- if pages.is_empty() {
158
- pages.push(Page::default());
159
- }
160
- Ok(pages)
161
- }
162
-
163
- fn run_pdftotext_bbox(bytes: &[u8]) -> Result<String> {
164
- let mut child = Command::new("pdftotext")
165
- .args(["-bbox", "-", "-"])
166
- .stdin(Stdio::piped())
167
- .stdout(Stdio::piped())
168
- .stderr(Stdio::piped())
169
- .spawn()
170
- .context(
171
- "pdftotext (poppler-utils) is required for PDF text extraction; \
172
- install poppler-utils or use the /v1/extract/words endpoint",
173
- )?;
174
-
175
- // Feed from a thread so a large PDF cannot deadlock on a full pipe.
176
- let data = bytes.to_vec();
177
- let stdin = child.stdin.take().expect("stdin configured");
178
- let writer = std::thread::spawn(move || {
179
- let mut handle = stdin;
180
- let _ = handle.write_all(&data);
181
- });
182
-
183
- let output = child.wait_with_output().context("wait for pdftotext")?;
184
- writer.join().ok();
185
- if !output.status.success() {
186
- bail!("pdftotext exited with {}", output.status);
187
- }
188
- Ok(String::from_utf8_lossy(&output.stdout).into_owned())
189
- }
190
-
191
- /// Minimal parser for pdftotext's `-bbox` XHTML output.
192
- ///
193
- /// Handles `<page width= height=>` plus `<word xMin= yMin= xMax= yMax>text</word>`,
194
- /// which is the only shape pdftotext emits. The word text is the element *body*,
195
- /// not an attribute, so the scanner emits text runs between tags. A real XML
196
- /// parser is unnecessary here and would add a dependency for a fixed grammar.
197
- fn parse_bbox_xml(xml: &str, max_pages: usize) -> Result<Vec<Page>> {
198
- let mut pages: Vec<Page> = Vec::new();
199
- let mut current: Option<(Page, f64, f64)> = None;
200
- let mut pending: Option<(Vec<(String, String)>, String)> = None;
201
-
202
- for event in xml_events(xml) {
203
- match event {
204
- Event::Open(name, attrs) => match name {
205
- "page" => {
206
- if pages.len() >= max_pages {
207
- break;
208
- }
209
- let width = attr_f64(&attrs, "width").unwrap_or(1.0);
210
- let height = attr_f64(&attrs, "height").unwrap_or(1.0);
211
- current = Some((Page::default(), width, height));
212
- }
213
- "word" => pending = Some((attrs, String::new())),
214
- _ => {}
215
- },
216
- Event::Text(text) => {
217
- if let Some((_, buffer)) = pending.as_mut() {
218
- buffer.push_str(&text);
219
- }
220
- }
221
- Event::Close(name) => match name {
222
- "word" => {
223
- if let (Some((attrs, text)), Some((page, width, height))) =
224
- (pending.take(), current.as_mut())
225
- {
226
- let text = unescape_xml(&text);
227
- if !text.trim().is_empty() {
228
- let x0 = attr_f64(&attrs, "xMin").unwrap_or(0.0);
229
- let y0 = attr_f64(&attrs, "yMin").unwrap_or(0.0);
230
- let x1 = attr_f64(&attrs, "xMax").unwrap_or(0.0);
231
- let y1 = attr_f64(&attrs, "yMax").unwrap_or(0.0);
232
- let sx = 1000.0 / width.max(1e-6);
233
- let sy = 1000.0 / height.max(1e-6);
234
- page.words.push(text);
235
- page.boxes.extend_from_slice(&[
236
- clamp(x0 * sx),
237
- clamp(y0 * sy),
238
- clamp(x1 * sx),
239
- clamp(y1 * sy),
240
- ]);
241
- }
242
- }
243
- }
244
- "page" => {
245
- if let Some((page, _, _)) = current.take() {
246
- pages.push(page);
247
- }
248
- }
249
- _ => {}
250
- },
251
- }
252
- }
253
-
254
- if let Some((page, _, _)) = current.take() {
255
- pages.push(page);
256
- }
257
- Ok(pages)
258
- }
259
-
260
- /// Decode the XML entities pdftotext emits in word bodies.
261
- fn unescape_xml(input: &str) -> String {
262
- input
263
- .replace("&amp;", "&")
264
- .replace("&lt;", "<")
265
- .replace("&gt;", ">")
266
- .replace("&quot;", "\"")
267
- .replace("&apos;", "'")
268
- .replace("&#39;", "'")
269
- }
270
-
271
- enum Event<'a> {
272
- Open(&'a str, Vec<(String, String)>),
273
- Close(&'a str),
274
- Text(String),
275
- }
276
-
277
- /// Scan XML into open/close/text events.
278
- fn xml_events(xml: &str) -> Vec<Event<'_>> {
279
- let mut out = Vec::new();
280
- let mut i = 0usize;
281
- let mut text_start = 0usize;
282
-
283
- while let Some(start) = find_from(xml, i, b'<') {
284
- // Text run between the previous tag and this one.
285
- if start > text_start {
286
- let raw = &xml[text_start..start];
287
- if !raw.trim().is_empty() {
288
- out.push(Event::Text(unescape_xml(raw)));
289
- }
290
- }
291
-
292
- let Some(end_rel) = find_from(xml, start + 1, b'>') else { break };
293
- let inner = &xml[start + 1..end_rel];
294
- text_start = end_rel + 1;
295
- i = end_rel + 1;
296
-
297
- if let Some(close) = inner.strip_prefix('/') {
298
- let name = close.trim();
299
- if !name.is_empty() {
300
- out.push(Event::Close(name));
301
- }
302
- continue;
303
- }
304
-
305
- let (name, attrs_raw) = match inner.find(char::is_whitespace) {
306
- Some(idx) => (&inner[..idx], &inner[idx..]),
307
- None => (inner, ""),
308
- };
309
- // Skip declarations, comments and the XML prolog.
310
- if name.is_empty() || name.starts_with('?') || name.starts_with('!') {
311
- continue;
312
- }
313
- out.push(Event::Open(name, parse_attrs(attrs_raw)));
314
- }
315
-
316
- if text_start < xml.len() {
317
- let raw = &xml[text_start..];
318
- if !raw.trim().is_empty() {
319
- out.push(Event::Text(unescape_xml(raw)));
320
- }
321
- }
322
- out
323
- }
324
-
325
- fn find_from(haystack: &str, from: usize, needle: u8) -> Option<usize> {
326
- if from >= haystack.len() {
327
- return None;
328
- }
329
- haystack.as_bytes()[from..]
330
- .iter()
331
- .position(|b| *b == needle)
332
- .map(|idx| from + idx)
333
- }
334
-
335
- fn parse_attrs(input: &str) -> Vec<(String, String)> {
336
- let mut out = Vec::new();
337
- let mut rest = input.trim();
338
- while let Some(eq) = rest.find('=') {
339
- let key = rest[..eq].trim().trim_matches('/').to_string();
340
- let after = rest[eq + 1..].trim_start();
341
- let Some(value) = after.strip_prefix('"') else {
342
- break;
343
- };
344
- let Some(close) = value.find('"') else { break };
345
- out.push((key, value[..close].to_string()));
346
- rest = &value[close + 1..];
347
- }
348
- out
349
- }
350
-
351
- fn attr_f64(attrs: &[(String, String)], key: &str) -> Option<f64> {
352
- attrs
353
- .iter()
354
- .find(|(k, _)| k == key)
355
- .and_then(|(_, v)| v.parse().ok())
356
- }
357
-
358
- /// Render one PDF page to PNG using poppler's pdftoppm.
359
- fn pdftoppm_png(bytes: &[u8], page: usize) -> Result<Vec<u8>> {
360
- let dir = std::env::temp_dir().join(format!("docqa-{}", document_suffix(bytes)));
361
- std::fs::create_dir_all(&dir).ok();
362
-
363
- let pdf_path = dir.join("in.pdf");
364
- let prefix = dir.join("page");
365
- std::fs::write(&pdf_path, bytes).context("write temp pdf")?;
366
-
367
- let status = Command::new("pdftoppm")
368
- .args(["-png", "-r", "200", "-f", &page.to_string(), "-l", &page.to_string(),
369
- "-singlefile", pdf_path.to_str().unwrap(), prefix.to_str().unwrap()])
370
- .stdout(Stdio::null())
371
- .stderr(Stdio::null())
372
- .status()
373
- .context("pdftoppm (poppler-utils) is required to OCR scanned PDFs")?;
374
-
375
- let png = dir.join("page.png");
376
- let result = if status.success() && png.exists() {
377
- std::fs::read(&png).context("read rendered page")
378
- } else {
379
- Err(anyhow!("pdftoppm failed to render page {page}"))
380
- };
381
-
382
- // Best-effort cleanup; a leftover temp file is preferable to masking the
383
- // real error.
384
- let _ = std::fs::remove_file(&pdf_path);
385
- let _ = std::fs::remove_dir_all(&dir);
386
- result
387
- }
388
-
389
- fn document_suffix(bytes: &[u8]) -> String {
390
- crate::engine::document_id(bytes)
391
- }
392
-
393
- // ---------------------------------------------------------------------------
394
- // OCR
395
- // ---------------------------------------------------------------------------
396
- /// Run Tesseract and parse its TSV output into a Page.
397
- fn ocr_image(bytes: &[u8]) -> Result<Page> {
398
- let dir = std::env::temp_dir().join(format!("docqa-ocr-{}", document_suffix(bytes)));
399
- std::fs::create_dir_all(&dir).ok();
400
- let image_path = dir.join("in.png");
401
- std::fs::write(&image_path, bytes).context("write temp image")?;
402
-
403
- let child = Command::new("tesseract")
404
- .arg(&image_path)
405
- .arg("stdout")
406
- .args(["-l", "eng", "tsv"])
407
- .stdout(Stdio::piped())
408
- .stderr(Stdio::piped())
409
- .spawn()
410
- .context(
411
- "tesseract is required for images and scanned PDFs. Install it and \
412
- ensure it is on PATH (Debian: apt install tesseract-ocr)",
413
- )?;
414
-
415
- let output = child.wait_with_output().context("wait for tesseract")?;
416
- let _ = std::fs::remove_dir_all(&dir);
417
-
418
- if !output.status.success() {
419
- bail!("tesseract exited with {}", output.status);
420
- }
421
-
422
- let tsv = String::from_utf8_lossy(&output.stdout);
423
- parse_tesseract_tsv(&tsv)
424
- }
425
-
426
- /// Parse Tesseract TSV: 12 columns, level 5 rows are words.
427
- ///
428
- /// Tesseract emits absolute pixel coordinates; the level-1 row carries the page
429
- /// width and height needed to map them onto LayoutLM's 0..1000 grid. The first
430
- /// pass collects page dimensions, the second converts the boxes.
431
- fn parse_tesseract_tsv(tsv: &str) -> Result<Page> {
432
- let (page_width, page_height) = match tsv_page_dim(tsv) {
433
- Some(dims) => dims,
434
- None => return Ok(Page::default()),
435
- };
436
- if page_width <= 0.0 || page_height <= 0.0 {
437
- return Ok(Page::default());
438
- }
439
-
440
- let mut page = Page::default();
441
- for line in tsv.lines().skip(1) {
442
- let cols: Vec<&str> = line.split('\t').collect();
443
- if cols.len() < 12 {
444
- continue;
445
- }
446
- // Tesseract hierarchy: level 5 is a word, 4 a line, 1 the page.
447
- if cols[0].parse::<i32>().ok() != Some(5) {
448
- continue;
449
- }
450
- let text = cols[11].trim();
451
- if text.is_empty() {
452
- continue;
453
- }
454
- let left: f64 = cols[6].parse().unwrap_or(0.0);
455
- let top: f64 = cols[7].parse().unwrap_or(0.0);
456
- let width: f64 = cols[8].parse().unwrap_or(0.0);
457
- let height: f64 = cols[9].parse().unwrap_or(0.0);
458
-
459
- page.words.push(text.to_string());
460
- page.boxes.extend_from_slice(&[
461
- clamp(left / page_width * 1000.0),
462
- clamp(top / page_height * 1000.0),
463
- clamp((left + width) / page_width * 1000.0),
464
- clamp((top + height) / page_height * 1000.0),
465
- ]);
466
- }
467
- Ok(page)
468
- }
469
-
470
- /// Read `(page_width, page_height)` from the level-1 row.
471
- fn tsv_page_dim(tsv: &str) -> Option<(f64, f64)> {
472
- for line in tsv.lines().skip(1) {
473
- let cols: Vec<&str> = line.split('\t').collect();
474
- if cols.len() < 12 {
475
- continue;
476
- }
477
- if cols[0].parse::<i32>().ok() != Some(1) {
478
- continue;
479
- }
480
- // Column indices in Tesseract TSV:
481
- // 0 level, 1 page_num, 2 block_num, 3 par_num, 4 line_num, 5 word_num,
482
- // 6 left, 7 top, 8 width, 9 height, 10 conf, 11 text.
483
- let width = cols[8].parse::<f64>().ok()?;
484
- let height = cols[9].parse::<f64>().ok()?;
485
- return Some((width, height));
486
- }
487
- None
488
- }
489
-
490
- #[cfg(test)]
491
- mod tests {
492
- use super::*;
493
-
494
- #[test]
495
- fn sniff_identifies_types() {
496
- assert_eq!(sniff(b"%PDF-1.7"), "pdf");
497
- assert_eq!(sniff(b"\x89PNG\r\n\x1a\n"), "png");
498
- assert_eq!(sniff(b"\xFF\xD8\xFF\xE0"), "jpeg");
499
- assert_eq!(sniff(b"hello world"), "unknown");
500
- }
501
-
502
- #[test]
503
- fn clamp_bounds_coordinates() {
504
- assert_eq!(clamp(-10.0), 0);
505
- assert_eq!(clamp(5000.0), 1000);
506
- assert_eq!(clamp(512.4), 512);
507
- }
508
-
509
- #[test]
510
- fn base64_decodes_roundtrip() {
511
- let encoded = "SGVsbG8sIFdvcmxkIQ==";
512
- assert_eq!(decode_base64(encoded).unwrap(), b"Hello, World!");
513
- }
514
-
515
- #[test]
516
- fn base64_strips_data_url_prefix() {
517
- let encoded = "data:image/png;base64,SGVsbG8=";
518
- assert_eq!(decode_base64(encoded).unwrap(), b"Hello");
519
- }
520
-
521
- #[test]
522
- fn base64_rejects_invalid_characters() {
523
- assert!(decode_base64("!!!!").is_err());
524
- }
525
-
526
- #[test]
527
- fn bbox_xml_is_parsed_and_normalised() {
528
- let xml = r#"<?xml version="1.0"?>
529
- <page width="612.0" height="792.0">
530
- <word xMin="51.3" yMin="50.7" xMax="86.3" yMax="60.7">Invoice</word>
531
- <word xMin="90.0" yMin="50.7" xMax="117.4" yMax="60.7">Number</word>
532
- </page>"#;
533
- let pages = parse_bbox_xml(xml, 3).unwrap();
534
- assert_eq!(pages.len(), 1);
535
- assert_eq!(pages[0].words, vec!["Invoice", "Number"]);
536
- // 51.3/612*1000 ~= 84
537
- assert!((pages[0].boxes[0] - 84).abs() <= 1, "got {}", pages[0].boxes[0]);
538
- assert!((pages[0].boxes[1] - 64).abs() <= 1, "got {}", pages[0].boxes[1]);
539
- }
540
-
541
- #[test]
542
- fn bbox_xml_respects_max_pages() {
543
- let xml = "<page width='10' height='10'><word xMin='0' yMin='0' xMax='1' yMax='1'>a</word></page>"
544
- .repeat(5);
545
- let pages = parse_bbox_xml(&xml, 2).unwrap();
546
- assert_eq!(pages.len(), 2);
547
- }
548
-
549
- #[test]
550
- fn tesseract_tsv_keeps_only_word_rows() {
551
- // Real Tesseract TSV includes a level-1 row carrying the page width and
552
- // height, which the parser needs in order to map pixel boxes onto
553
- // LayoutLM's 0..1000 grid.
554
- let tsv = concat!(
555
- "level\tpage_num\tblock\tpar\tline\tword\tleft\ttop\twidth\theight\tconf\ttext\n",
556
- "1\t1\t0\t0\t0\t0\t0\t0\t1000\t2000\t-1\t\n",
557
- "5\t1\t1\t1\t1\t1\t100\t200\t50\t20\t95.5\tINV-123\n",
558
- "4\t1\t1\t1\t1\t0\t0\t0\t0\t0\t-1\t\n",
559
- "5\t1\t1\t1\t1\t2\t300\t400\t80\t20\t91.0\tPO-99\n",
560
- );
561
- let page = parse_tesseract_tsv(tsv).unwrap();
562
- assert_eq!(page.words, vec!["INV-123", "PO-99"]);
563
- // First word: x 100..150 over a 1000px page, y 200..220 over 2000px.
564
- assert_eq!(&page.boxes[0..4], &[100, 100, 150, 110]);
565
- // Second word: x 300..380, y 400..420 -> 200..210.
566
- assert_eq!(&page.boxes[4..8], &[300, 200, 380, 210]);
567
- }
568
-
569
- #[test]
570
- fn tesseract_tsv_without_dimensions_yields_nothing() {
571
- // Better to return nothing than to emit unnormalised pixel values.
572
- let tsv = "level\tpage_num\tblock\tpar\tline\tword\tleft\ttop\twidth\theight\tconf\ttext\n\
573
- 5\t1\t1\t1\t1\t1\t100\t200\t50\t20\t95.5\tINV-123\n";
574
- assert!(parse_tesseract_tsv(tsv).unwrap().is_empty());
575
- }
576
- }
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
docqa-rs/src/pool.rs DELETED
@@ -1,93 +0,0 @@
1
- //! Fixed pool of inference workers.
2
- //!
3
- //! `ort::Session::run` takes `&mut self` because several ONNX Runtime internals
4
- //! are not thread-safe. Rather than sharing one session behind a mutex (which
5
- //! serialises all inference and defeats concurrency) each worker owns its own
6
- //! session, and a worker is checked out for the duration of one request.
7
- //!
8
- //! Sessions are loaded eagerly at startup so the first request does not pay the
9
- //! load cost and so a bad model path fails the boot rather than the traffic.
10
-
11
- use anyhow::{Context, Result};
12
- use tokio::sync::{mpsc, OwnedSemaphorePermit, Semaphore};
13
-
14
- use crate::config::Config;
15
- use crate::engine::Engine;
16
-
17
- pub struct EnginePool {
18
- tx: mpsc::Sender<Engine>,
19
- rx: tokio::sync::Mutex<mpsc::Receiver<Engine>>,
20
- permits: Arc<Semaphore>,
21
- }
22
-
23
- use std::sync::Arc;
24
-
25
- impl EnginePool {
26
- /// Load `config.workers` sessions and return a pool over them.
27
- pub fn new(config: &Config) -> Result<Self> {
28
- let (tx, rx) = mpsc::channel::<Engine>(config.workers);
29
- for i in 0..config.workers {
30
- let engine = Engine::load(
31
- &config.model_path,
32
- &config.tokenizer_path,
33
- &config.model_id,
34
- config.threads,
35
- config.confidence_threshold,
36
- )
37
- .with_context(|| format!("load engine #{i}"))?;
38
- // `EnginePool::new` is called from async main, but keep this
39
- // function synchronous so it can also be used from tests.
40
- tx.try_send(engine)
41
- .map_err(|_| anyhow::anyhow!("engine channel closed during startup"))?;
42
- }
43
- let permits = Arc::new(Semaphore::new(config.workers));
44
- Ok(Self { tx, rx: tokio::sync::Mutex::new(rx), permits })
45
- }
46
-
47
- /// Check out a worker. Waits if all workers are busy.
48
- pub async fn acquire(&self) -> Result<PooledEngine> {
49
- let permit = self
50
- .permits
51
- .clone()
52
- .acquire_owned()
53
- .await
54
- .context("semaphore closed")?;
55
- let mut rx = self.rx.lock().await;
56
- let engine = rx.recv().await.context("engine pool exhausted")?;
57
- Ok(PooledEngine { engine: Some(engine), pool_tx: self.tx.clone(), _permit: permit })
58
- }
59
-
60
- pub fn busy(&self) -> usize {
61
- self.permits.available_permits()
62
- }
63
- }
64
-
65
- /// A worker borrowed from the pool; returned automatically on drop.
66
- pub struct PooledEngine {
67
- engine: Option<Engine>,
68
- pool_tx: mpsc::Sender<Engine>,
69
- _permit: OwnedSemaphorePermit,
70
- }
71
-
72
- impl std::ops::Deref for PooledEngine {
73
- type Target = Engine;
74
- fn deref(&self) -> &Engine {
75
- self.engine.as_ref().expect("engine present while borrowed")
76
- }
77
- }
78
-
79
- impl std::ops::DerefMut for PooledEngine {
80
- fn deref_mut(&mut self) -> &mut Engine {
81
- self.engine.as_mut().expect("engine present while borrowed")
82
- }
83
- }
84
-
85
- impl Drop for PooledEngine {
86
- fn drop(&mut self) {
87
- if let Some(engine) = self.engine.take() {
88
- // Best-effort return; if the pool is gone the session is dropped,
89
- // which is correct on shutdown.
90
- let _ = self.pool_tx.try_send(engine);
91
- }
92
- }
93
- }
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
docqa.postman_collection.json ADDED
@@ -0,0 +1,149 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "info": {
3
+ "_postman_id": "docqa-collection",
4
+ "name": "docqa — Document Data Extraction API",
5
+ "description": "Extract user-defined keys from invoices, receipts and scanned documents.\n\nNo response bodies are pre-filled here on purpose — run against a live server to see real values.\n\nHOW TO READ `status`\n extracted confidence >= threshold. Use it.\n low_confidence a span was found but fell below the threshold. Route to a human; do NOT treat as fact.\n not_found nothing usable on the page.\n error the key could not be processed.\n\nIMPORTANT: impira/layoutlm-invoices is extractive and has NO null-answer head. It returns *some* span for every input, including questions the document cannot answer. Measured hallucination rate on deliberately unanswerable invoice questions: 16.7%, with 0% abstention. That is why `status` must be checked before `value`, and why `low_confidence` means \"a human should look\", not \"this is the answer\".\n\nKEYS ARE FREE-FORM. `INVOICE NO`, `Total Amount` and `GSTIN` are all just strings; nothing is hard-coded. Spaces are significant: `Total Amount` != `TotalAmount`. Duplicates are preserved and returned index-aligned with the request, so zip the two lists rather than looking up by key.",
6
+ "schema": "https://schema.getpostman.com/json/collection/v2.1.0/collection.json"
7
+ },
8
+ "variable": [
9
+ { "key": "baseUrl", "value": "http://127.0.0.1:7860" }
10
+ ],
11
+ "item": [
12
+ {
13
+ "name": "Health",
14
+ "request": {
15
+ "method": "GET",
16
+ "header": [],
17
+ "url": { "raw": "{{baseUrl}}/health", "host": ["{{baseUrl}}"], "path": ["health"] },
18
+ "description": "Liveness plus model readiness. `status: ok` means the model is loaded and inference will work; `degraded` means the process is up but the model is not loaded, so extraction will 503."
19
+ }
20
+ },
21
+ {
22
+ "name": "Config (limits in effect)",
23
+ "request": {
24
+ "method": "GET",
25
+ "header": [],
26
+ "url": { "raw": "{{baseUrl}}/v1/config", "host": ["{{baseUrl}}"], "path": ["v1", "config"] },
27
+ "description": "Shows the model, device, batch size and the key/page/upload limits the server is currently enforcing. Useful for confirming an env var actually took effect."
28
+ }
29
+ },
30
+ {
31
+ "name": "Extract (multipart upload)",
32
+ "request": {
33
+ "method": "POST",
34
+ "header": [],
35
+ "body": {
36
+ "mode": "formdata",
37
+ "formdata": [
38
+ { "key": "file", "type": "file", "src": [], "description": "Attach a real PDF, PNG, JPEG, TIFF or BMP" },
39
+ { "key": "keys", "value": "INVOICE NO,Vendor Id,Invoice Date,Total Amount,GSTIN", "type": "text", "description": "COMMA-SEPARATED here. Whitespace inside a key is significant: 'Total Amount' != 'TotalAmount'." },
40
+ { "key": "confidence_threshold", "value": "0.5", "type": "text", "description": "Optional 0..1. At or above -> extracted; below -> low_confidence." },
41
+ { "key": "max_pages", "value": "3", "type": "text", "description": "Optional. LayoutLM's effective context is ~512 tokens, so page 2+ is often truncated anyway." }
42
+ ]
43
+ },
44
+ "url": { "raw": "{{baseUrl}}/v1/extract", "host": ["{{baseUrl}}"], "path": ["v1", "extract"] },
45
+ "description": "The main endpoint. Digital PDFs use the text layer; scanned PDFs and images are rasterised and OCR'd (poppler-utils and tesseract-ocr are installed in the image).\n\nErrors: 400 invalid keys, 413 over max_upload_mb (25 MB default), 415 unsupported media type, 422 unprocessable, 503 engine not ready, 504 over request_timeout_s."
46
+ }
47
+ },
48
+ {
49
+ "name": "Extract (base64 JSON)",
50
+ "request": {
51
+ "method": "POST",
52
+ "header": [{ "key": "Content-Type", "value": "application/json" }],
53
+ "body": {
54
+ "mode": "raw",
55
+ "raw": "{\n \"document_base64\": \"<base64 of a real PDF or image>\",\n \"keys\": [\"INVOICE NO\", \"Invoice Date\", \"Total Amount\", \"GSTIN\"],\n \"confidence_threshold\": 0.5,\n \"max_pages\": 3\n}",
56
+ "options": { "raw": { "language": "json" } }
57
+ },
58
+ "url": { "raw": "{{baseUrl}}/v1/extract/json", "host": ["{{baseUrl}}"], "path": ["v1", "extract", "json"] },
59
+ "description": "For clients that already hold the file in memory and want to skip multipart encoding. `keys` is a JSON ARRAY here, not a comma-separated string. A `data:...;base64,` URI prefix is accepted and stripped.\n\nGenerate the base64 with: `[Convert]::ToBase64String([IO.File]::ReadAllBytes('invoice.pdf'))`"
60
+ }
61
+ },
62
+ {
63
+ "name": "Edge cases",
64
+ "item": [
65
+ {
66
+ "name": "Empty keys -> 400",
67
+ "request": {
68
+ "method": "POST",
69
+ "header": [],
70
+ "body": {
71
+ "mode": "formdata",
72
+ "formdata": [
73
+ { "key": "file", "type": "file", "src": [] },
74
+ { "key": "keys", "value": " , ,", "type": "text" }
75
+ ]
76
+ },
77
+ "url": { "raw": "{{baseUrl}}/v1/extract", "host": ["{{baseUrl}}"], "path": ["v1", "extract"] }
78
+ },
79
+ "description": "Keys that are only whitespace or commas collapse to an empty list. Must be rejected rather than returning an empty result that looks like success."
80
+ },
81
+ {
82
+ "name": "threshold=1.0 suppresses all extractions",
83
+ "request": {
84
+ "method": "POST",
85
+ "header": [],
86
+ "body": {
87
+ "mode": "formdata",
88
+ "formdata": [
89
+ { "key": "file", "type": "file", "src": [] },
90
+ { "key": "keys", "value": "INVOICE NO,Total Amount", "type": "text" },
91
+ { "key": "confidence_threshold", "value": "1.0", "type": "text" }
92
+ ]
93
+ },
94
+ "url": { "raw": "{{baseUrl}}/v1/extract", "host": ["{{baseUrl}}"], "path": ["v1", "extract"] }
95
+ },
96
+ "description": "Cheap regression check on the gating logic: at threshold 1.0 no field can come back as `extracted`. If any do, the threshold is being applied to the value instead of the status."
97
+ },
98
+ {
99
+ "name": "Duplicate keys returned separately",
100
+ "request": {
101
+ "method": "POST",
102
+ "header": [],
103
+ "body": {
104
+ "mode": "formdata",
105
+ "formdata": [
106
+ { "key": "file", "type": "file", "src": [] },
107
+ { "key": "keys", "value": "Total Amount,Total Amount", "type": "text" }
108
+ ]
109
+ },
110
+ "url": { "raw": "{{baseUrl}}/v1/extract", "host": ["{{baseUrl}}"], "path": ["v1", "extract"] }
111
+ },
112
+ "description": "Duplicates are NOT collapsed. Two keys in means two results out, index-aligned, so callers can zip without a key lookup. De-duplicate before sending if you do not want the wasted inference."
113
+ },
114
+ {
115
+ "name": "Unanswerable key (documents the no-abstention behaviour)",
116
+ "request": {
117
+ "method": "POST",
118
+ "header": [],
119
+ "body": {
120
+ "mode": "formdata",
121
+ "formdata": [
122
+ { "key": "file", "type": "file", "src": [], "description": "Any ordinary invoice" },
123
+ { "key": "keys", "value": "Passport Number of the Director", "type": "text" }
124
+ ]
125
+ },
126
+ "url": { "raw": "{{baseUrl}}/v1/extract", "host": ["{{baseUrl}}"], "path": ["v1", "extract"] }
127
+ },
128
+ "description": "The model ALWAYS returns a span, even when the answer is not on the page. Expect `low_confidence` with a near-zero confidence and a `value` that is some real token from the document. This is the single most important behaviour to understand before trusting the API, and the reason `status` must be checked before `value`."
129
+ },
130
+ {
131
+ "name": "Non-document file -> error",
132
+ "request": {
133
+ "method": "POST",
134
+ "header": [],
135
+ "body": {
136
+ "mode": "formdata",
137
+ "formdata": [
138
+ { "key": "file", "type": "file", "src": [], "description": "Attach a .txt file" },
139
+ { "key": "keys", "value": "Total Amount", "type": "text" }
140
+ ]
141
+ },
142
+ "url": { "raw": "{{baseUrl}}/v1/extract", "host": ["{{baseUrl}}"], "path": ["v1", "extract"] }
143
+ },
144
+ "description": "Unsupported media types must be rejected, not silently parsed into zero words."
145
+ }
146
+ ]
147
+ }
148
+ ]
149
+ }
docqa/bench/compare_backends.py ADDED
@@ -0,0 +1,218 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """Backend comparison: torch fp32 vs ONNX fp32 vs ONNX int8.
2
+
3
+ Answers the two deployment questions directly, with measurements:
4
+
5
+ Q1 Does ONNX Runtime beat PyTorch on CPU for this model?
6
+ Q2 Does int8 quantization change the answers?
7
+
8
+ Accuracy is compared against ground truth, not just "do the two agree", because
9
+ two backends can agree and both be wrong.
10
+ """
11
+
12
+ from __future__ import annotations
13
+
14
+ import json
15
+ import os
16
+ import statistics
17
+ import sys
18
+ import time
19
+ from pathlib import Path
20
+
21
+ ROOT = Path(__file__).resolve().parent.parent
22
+ sys.path.insert(0, str(ROOT / "src"))
23
+ os.environ.setdefault("TF_CPP_MIN_LOG_LEVEL", "3")
24
+
25
+ import numpy as np
26
+ import torch
27
+
28
+ from docxextract import parsing
29
+ from docxextract.config import Settings
30
+
31
+ KEYS = ["INVOICE NO", "Invoice Date", "Total Amount", "PO Number",
32
+ "Payment Term", "GSTIN", "Vendor Id", "Subtotal"]
33
+ REPEATS = 3
34
+
35
+
36
+ def normalise(text: str) -> str:
37
+ import re
38
+ import string
39
+ text = text.lower()
40
+ text = "".join(ch for ch in text if ch not in set(string.punctuation))
41
+ text = re.sub(r"\b(a|an|the)\b", " ", text)
42
+ return " ".join(text.split())
43
+
44
+
45
+ class TorchBackend:
46
+ name = "torch_fp32"
47
+
48
+ def __init__(self, model_id: str, threads: int):
49
+ torch.set_num_threads(threads)
50
+ from transformers import pipeline
51
+ self.pipe = pipeline("document-question-answering", model=model_id,
52
+ device="cpu")
53
+ self.tokenizer = None
54
+
55
+ def answer(self, words, boxes, key):
56
+ result = self.pipe(image=None, question=key,
57
+ word_boxes=[[w, list(b)] for w, b in zip(words, boxes)],
58
+ top_k=1)
59
+ while isinstance(result, list):
60
+ result = result[0]
61
+ return str(result["answer"]).strip(), float(result["score"])
62
+
63
+
64
+ class OnnxBackend:
65
+ def __init__(self, model_id: str, path: Path, threads: int,
66
+ name: str = "onnx"):
67
+ import onnxruntime as ort
68
+ from transformers import AutoTokenizer
69
+
70
+ self.name = name
71
+ self.tokenizer = AutoTokenizer.from_pretrained(model_id)
72
+ options = ort.SessionOptions()
73
+ options.intra_op_num_threads = threads
74
+ options.inter_op_num_threads = 1
75
+ options.graph_optimization_level = ort.GraphOptimizationLevel.ORT_ENABLE_ALL
76
+ self.session = ort.InferenceSession(
77
+ str(path), options, providers=["CPUExecutionProvider"])
78
+
79
+ def answer(self, words, boxes, key):
80
+ # text_pair must be a flat list of word strings, matching the
81
+ # pipeline's LayoutLM tokenisation path.
82
+ word_boxes = list(zip(words, boxes))
83
+ encoding = self.tokenizer(
84
+ text=key.split(), text_pair=list(words), is_split_into_words=True,
85
+ return_token_type_ids=True, truncation="only_second",
86
+ max_length=self.tokenizer.model_max_length)
87
+ sequence_ids = encoding.sequence_ids(0)
88
+ # This must mirror DocumentQuestionAnsweringPipeline.preprocess exactly:
89
+ # context token -> the word's real box
90
+ # SEP token -> [1000, 1000, 1000, 1000]
91
+ # everything else (question, CLS) -> [0, 0, 0, 0]
92
+ # Using zeros for SEP shifts the learned position embeddings and
93
+ # silently degrades answers, which is exactly what happens if this is
94
+ # copied from memory instead of the pipeline source.
95
+ sep_id = self.tokenizer.sep_token_id
96
+ bbox = []
97
+ for token_id, sid, wid in zip(encoding["input_ids"], sequence_ids,
98
+ encoding.word_ids(0)):
99
+ if sid == 1:
100
+ bbox.append(word_boxes[wid][1])
101
+ elif token_id == sep_id:
102
+ bbox.append([1000, 1000, 1000, 1000])
103
+ else:
104
+ bbox.append([0, 0, 0, 0])
105
+ feed = {
106
+ "input_ids": np.array([encoding["input_ids"]], dtype=np.int64),
107
+ "attention_mask": np.array([encoding["attention_mask"]], dtype=np.int64),
108
+ "token_type_ids": np.array([encoding["token_type_ids"]], dtype=np.int64),
109
+ "bbox": np.array([bbox], dtype=np.int64).reshape(1, -1),
110
+ }
111
+ start, end = self.session.run(None, feed)
112
+ start, end = start[0], end[0]
113
+ # Mirror the pipeline's p_mask: only context tokens may start/end a span.
114
+ for i, sid in enumerate(sequence_ids):
115
+ if sid != 1:
116
+ start[i] = end[i] = -1e4
117
+ s = int(np.argmax(start))
118
+ e = int(np.argmax(end))
119
+ if e < s:
120
+ e = s
121
+ # The pipeline returns the ORIGINAL word strings joined by spaces,
122
+ # not a decode of the subword ids. These differ for dates, currency
123
+ # symbols and hyphenated identifiers, so they must not be conflated.
124
+ word_ids = encoding.word_ids(0)
125
+ w_start, w_end = word_ids[s], word_ids[e]
126
+ if w_start is None or w_end is None:
127
+ return "", 0.0
128
+ answer = " ".join(words[w_start:w_end + 1])
129
+ return answer, float(np.exp(start[s]) * np.exp(end[e]))
130
+
131
+
132
+ def main() -> None:
133
+ settings = Settings()
134
+ threads = 8
135
+ model_id = settings.model_id
136
+ truths = json.loads((ROOT / "corpus" / "ground_truth.json").read_text())
137
+
138
+ backends: list[object] = [TorchBackend(model_id, threads)]
139
+ onnx_dir = ROOT / "artifacts" / "onnx" / model_id.replace("/", "__")
140
+ if (onnx_dir / "model.onnx").exists():
141
+ backends.append(OnnxBackend(model_id, onnx_dir / "model.onnx", threads,
142
+ "onnx_fp32"))
143
+ if (onnx_dir / "model_int8.onnx").exists():
144
+ backends.append(OnnxBackend(model_id, onnx_dir / "model_int8.onnx", threads,
145
+ "onnx_int8"))
146
+
147
+ print(f"model={model_id} threads={threads} "
148
+ f"documents={len(truths)} keys={len(KEYS)}\n")
149
+
150
+ results: dict[str, dict] = {b.name: {"correct": 0, "total": 0,
151
+ "times": [], "answers": {}}
152
+ for b in backends}
153
+ truth_map = {"INVOICE NO": "invoice_number", "Invoice Date": "invoice_date",
154
+ "Total Amount": "total", "PO Number": "po_number",
155
+ "Payment Term": "payment_terms", "GSTIN": "gstin",
156
+ "Vendor Id": "vendor_tax_id", "Subtotal": "subtotal"}
157
+
158
+ for record in truths:
159
+ data = Path(record["path"]).read_bytes()
160
+ parsed = parsing.parse(data, settings)
161
+ page = parsed.pages[0]
162
+ for key in KEYS:
163
+ gold = str(record.get(truth_map[key], ""))
164
+ for backend in backends:
165
+ backend.answer(page.words, page.boxes, key) # warm
166
+ started = time.perf_counter()
167
+ answer, _score = backend.answer(page.words, page.boxes, key)
168
+ elapsed = time.perf_counter() - started
169
+ slot = results[backend.name]
170
+ slot["times"].append(elapsed)
171
+ slot["answers"][(record["document_id"], key)] = answer
172
+ if normalise(answer) == normalise(gold):
173
+ slot["correct"] += 1
174
+ slot["total"] += 1
175
+
176
+ print("=" * 74)
177
+ print(f"{'backend':<14} {'ms/key':>9} {'speedup':>9} {'accuracy':>10} "
178
+ f"{'agreement w/ torch':>21}")
179
+ print("=" * 74)
180
+ baseline_ms = statistics.median(results["torch_fp32"]["times"]) * 1000
181
+ torch_answers = results["torch_fp32"]["answers"]
182
+ for backend in backends:
183
+ slot = results[backend.name]
184
+ ms = statistics.median(slot["times"]) * 1000
185
+ acc = slot["correct"] / slot["total"]
186
+ agree = sum(1 for k, v in slot["answers"].items()
187
+ if normalise(v) == normalise(torch_answers.get(k, "")))
188
+ total_keys = len(slot["answers"])
189
+ agree_str = f"{agree}/{total_keys}"
190
+ print(f"{backend.name:<14} {ms:>9.0f} {baseline_ms/ms:>8.2f}x "
191
+ f"{acc:>9.1%} {agree_str:>21}")
192
+
193
+ print("\nPER-KEY ACCURACY")
194
+ keys_hdr = f"{'backend':<14}" + "".join(f"{k[:13]:>15}" for k in KEYS)
195
+ print(keys_hdr)
196
+ for backend in backends:
197
+ slot = results[backend.name]
198
+ row = f"{backend.name:<14}"
199
+ for key in KEYS:
200
+ hits = total = 0
201
+ for record in truths:
202
+ gold = str(record.get(truth_map[key], ""))
203
+ pred = slot["answers"].get((record["document_id"], key), "")
204
+ total += 1
205
+ hits += normalise(pred) == normalise(gold)
206
+ row += f"{hits/total:>14.0%} "
207
+ print(row)
208
+
209
+ json.dump({name: {"median_ms": statistics.median(slot["times"]) * 1000,
210
+ "accuracy": slot["correct"] / slot["total"],
211
+ "n": slot["total"]}
212
+ for name, slot in results.items()},
213
+ (ROOT / "results" / "backend_comparison.json").open("w"),
214
+ indent=2)
215
+
216
+
217
+ if __name__ == "__main__":
218
+ main()
docqa/bench/export_onnx.py ADDED
@@ -0,0 +1,159 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """Export LayoutLM to ONNX for CPU inference, with verified numerical parity.
2
+
3
+ Two traps this handles that a naive torch.onnx.export does not:
4
+
5
+ 1. **Argument order.** LayoutLMForQuestionAnswering.forward is
6
+ ``(input_ids, bbox, attention_mask, token_type_ids, ...)``. Passing
7
+ positional tensors shifts bbox into the attention_mask slot and fails with a
8
+ misleading "bbox coordinate values should be within 0-1000 range" error.
9
+ Inputs are bound by name instead.
10
+
11
+ 2. **bbox is rank-3.** torch.onnx.export can attach dynamic axes to only two
12
+ dimensions, so a rank-3 ``bbox`` gets its sequence axis pinned and every
13
+ other sequence length fails at runtime. A thin wrapper accepts bbox as a
14
+ flat ``(batch, seq*4)`` tensor and reshapes internally, keeping all sequence
15
+ lengths dynamic.
16
+
17
+ Usage:
18
+ python bench/export_onnx.py impira/layoutlm-invoices --quantize
19
+ """
20
+
21
+ from __future__ import annotations
22
+
23
+ import argparse
24
+ import os
25
+ from pathlib import Path
26
+
27
+ os.environ.setdefault("TF_CPP_MIN_LOG_LEVEL", "3")
28
+
29
+ import torch
30
+ import torch.nn as nn
31
+ from transformers import AutoTokenizer
32
+ from transformers.models.layoutlm.modeling_layoutlm import LayoutLMForQuestionAnswering
33
+
34
+ DEFAULT_OUT = Path(__file__).resolve().parent.parent / "artifacts" / "onnx"
35
+
36
+
37
+ class LayoutLMQAWrapper(nn.Module):
38
+ """Accepts bbox as (batch, seq*4); reshapes to (batch, seq, 4) internally."""
39
+
40
+ def __init__(self, model: nn.Module):
41
+ super().__init__()
42
+ self.model = model
43
+
44
+ def forward(self, input_ids, attention_mask, token_type_ids, bbox_flat):
45
+ batch, flat = bbox_flat.shape
46
+ bbox = bbox_flat.reshape(batch, flat // 4, 4)
47
+ start_logits, end_logits = self.model(
48
+ input_ids=input_ids,
49
+ attention_mask=attention_mask,
50
+ token_type_ids=token_type_ids,
51
+ bbox=bbox,
52
+ return_dict=False,
53
+ )[:2]
54
+ return start_logits, end_logits
55
+
56
+
57
+ def export(model_id: str, out_dir: Path, quantize: bool) -> None:
58
+ out_dir.mkdir(parents=True, exist_ok=True)
59
+ print(f"exporting {model_id} -> {out_dir}")
60
+
61
+ tokenizer = AutoTokenizer.from_pretrained(model_id)
62
+ model = LayoutLMForQuestionAnswering.from_pretrained(model_id).eval()
63
+
64
+ # LayoutLM indexes learned position embeddings, so coordinates must lie in
65
+ # [0, 1000]. Real boxes come from parsing.parsing already scaled to that.
66
+ words = ["Invoice", "Number", ":", "INV-123456", "Date", ":", "01/02/2025"]
67
+ boxes = [[0, 0, 100, 20], [110, 0, 200, 20], [205, 0, 220, 20],
68
+ [230, 0, 400, 20], [0, 40, 60, 60], [70, 40, 95, 60],
69
+ [100, 40, 230, 60]]
70
+ encoding = tokenizer(text="What is the invoice number?".split(),
71
+ text_pair=words, is_split_into_words=True,
72
+ return_token_type_ids=True, truncation="only_second",
73
+ max_length=tokenizer.model_max_length)
74
+ sequence_ids = encoding.sequence_ids(0)
75
+ bbox = [[0, 0, 0, 0] if sid != 1 else boxes[wid]
76
+ for sid, wid in zip(sequence_ids, encoding.word_ids(0))]
77
+
78
+ sample = {
79
+ "input_ids": torch.tensor([encoding["input_ids"]]),
80
+ "attention_mask": torch.tensor([encoding["attention_mask"]]),
81
+ "token_type_ids": torch.tensor([encoding["token_type_ids"]]),
82
+ "bbox_flat": torch.tensor([bbox]).reshape(1, -1),
83
+ }
84
+
85
+ wrapper = LayoutLMQAWrapper(model).eval()
86
+ fp32_path = out_dir / "model.onnx"
87
+ torch.onnx.export(
88
+ wrapper,
89
+ (sample["input_ids"], sample["attention_mask"],
90
+ sample["token_type_ids"], sample["bbox_flat"]),
91
+ str(fp32_path),
92
+ input_names=["input_ids", "attention_mask", "token_type_ids", "bbox"],
93
+ output_names=["start_logits", "end_logits"],
94
+ dynamic_axes={name: {0: "batch", 1: "seq"} for name in
95
+ ["input_ids", "attention_mask", "token_type_ids", "bbox"]},
96
+ opset_version=14,
97
+ do_constant_folding=True,
98
+ )
99
+ print(f" wrote {fp32_path.name} ({fp32_path.stat().st_size / 1e6:.1f} MB)")
100
+
101
+ import numpy as np
102
+ import onnxruntime as ort
103
+
104
+ feed = {k: v.numpy() for k, v in sample.items() if k != "bbox_flat"}
105
+ feed["bbox"] = sample["bbox_flat"].numpy()
106
+
107
+ session = ort.InferenceSession(str(fp32_path), providers=["CPUExecutionProvider"])
108
+ outputs = session.run(None, feed)
109
+ with torch.no_grad():
110
+ reference = wrapper(**sample)
111
+ d_start = float((reference[0] - torch.tensor(outputs[0])).abs().max())
112
+ d_end = float((reference[1] - torch.tensor(outputs[1])).abs().max())
113
+ print(f" parity vs torch: start {d_start:.2e} end {d_end:.2e}")
114
+ assert d_start < 1e-3 and d_end < 1e-3, "ONNX output diverges from PyTorch"
115
+
116
+ # Prove the dynamic sequence axis actually works.
117
+ longer_words = words + ["Total", "$", "99.00"]
118
+ longer_boxes = boxes + [[300, 80, 380, 100], [385, 80, 410, 100],
119
+ [415, 80, 500, 100]]
120
+ enc2 = tokenizer(text="How much is the total?".split(),
121
+ text_pair=longer_words, is_split_into_words=True,
122
+ return_token_type_ids=True, truncation="only_second",
123
+ max_length=tokenizer.model_max_length)
124
+ sids = enc2.sequence_ids(0)
125
+ b2 = [[0, 0, 0, 0] if s != 1 else longer_boxes[i]
126
+ for s, i in zip(sids, enc2.word_ids(0))]
127
+ feed2 = {
128
+ "input_ids": np.array([enc2["input_ids"]], dtype=np.int64),
129
+ "attention_mask": np.array([enc2["attention_mask"]], dtype=np.int64),
130
+ "token_type_ids": np.array([enc2["token_type_ids"]], dtype=np.int64),
131
+ "bbox": np.array([b2], dtype=np.int64).reshape(1, -1),
132
+ }
133
+ session.run(None, feed2)
134
+ print(f" dynamic shape check: seq {feed['input_ids'].shape[1]} "
135
+ f"-> {feed2['input_ids'].shape[1]} OK")
136
+
137
+ if quantize:
138
+ from onnxruntime.quantization import QuantType, quantize_dynamic
139
+ int8_path = out_dir / "model_int8.onnx"
140
+ quantize_dynamic(str(fp32_path), str(int8_path), weight_type=QuantType.QInt8)
141
+ ratio = fp32_path.stat().st_size / int8_path.stat().st_size
142
+ print(f" wrote {int8_path.name} ({int8_path.stat().st_size / 1e6:.1f} MB, "
143
+ f"{ratio:.2f}x smaller)")
144
+
145
+ print(" PARITY + DYNAMIC SHAPE CHECKS PASSED")
146
+
147
+
148
+ def main() -> None:
149
+ parser = argparse.ArgumentParser(description=__doc__)
150
+ parser.add_argument("model", nargs="?", default="impira/layoutlm-invoices")
151
+ parser.add_argument("--quantize", action="store_true")
152
+ parser.add_argument("--out", type=Path, default=None)
153
+ args = parser.parse_args()
154
+ out = args.out or (DEFAULT_OUT / args.model.replace("/", "__"))
155
+ export(args.model, out, args.quantize)
156
+
157
+
158
+ if __name__ == "__main__":
159
+ main()
docqa/corpus/ground_truth.json ADDED
@@ -0,0 +1,266 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ [
2
+ {
3
+ "document_id": "invoice_00_classic",
4
+ "path": "C:\\Users\\soumi\\OneDrive\\Documents\\Test_Directory\\BEST-SLM\\docx_service\\corpus\\invoice_00_classic.pdf",
5
+ "variant": "classic",
6
+ "invoice_number": "INV-334551",
7
+ "invoice_date": "August 15, 2025",
8
+ "due_date": "February 11, 2025",
9
+ "po_number": "PO-85748",
10
+ "payment_terms": "Net 60",
11
+ "currency": "USD",
12
+ "vendor_name": "Litware Cloud Services",
13
+ "vendor_address": "2200 Telegraph Avenue, Denver, CO 80202",
14
+ "vendor_tax_id": "91-2200981",
15
+ "gstin": "19AAHCL4321K1ZR",
16
+ "bill_to": "Northwind Logistics LLC",
17
+ "subtotal": "$74,093.30",
18
+ "tax_amount": "$14,818.66",
19
+ "total": "$88,911.96",
20
+ "tax_label": "VAT 20%",
21
+ "page_count": 1,
22
+ "extra": {}
23
+ },
24
+ {
25
+ "document_id": "invoice_01_minimal",
26
+ "path": "C:\\Users\\soumi\\OneDrive\\Documents\\Test_Directory\\BEST-SLM\\docx_service\\corpus\\invoice_01_minimal.pdf",
27
+ "variant": "minimal",
28
+ "invoice_number": "INV-979727",
29
+ "invoice_date": "June 30, 2025",
30
+ "due_date": "May 5, 2025",
31
+ "po_number": "PO-96264",
32
+ "payment_terms": "Net 15",
33
+ "currency": "USD",
34
+ "vendor_name": "Contoso Manufacturing Pvt Ltd",
35
+ "vendor_address": "1180 Industrial Loop, Building 7, Austin, TX 78744",
36
+ "vendor_tax_id": "74-2930118",
37
+ "gstin": "29AAGCC7391J1Z2",
38
+ "bill_to": "Proseware Data Systems",
39
+ "subtotal": "$29,606.20",
40
+ "tax_amount": "$2,516.53",
41
+ "total": "$32,122.73",
42
+ "tax_label": "Sales Tax 8.5%",
43
+ "page_count": 1,
44
+ "extra": {}
45
+ },
46
+ {
47
+ "document_id": "invoice_02_boxed",
48
+ "path": "C:\\Users\\soumi\\OneDrive\\Documents\\Test_Directory\\BEST-SLM\\docx_service\\corpus\\invoice_02_boxed.pdf",
49
+ "variant": "boxed",
50
+ "invoice_number": "INV-798875",
51
+ "invoice_date": "May 22, 2025",
52
+ "due_date": "January 23, 2026",
53
+ "po_number": "PO-86041",
54
+ "payment_terms": "Net 30",
55
+ "currency": "USD",
56
+ "vendor_name": "Northwind Logistics LLC",
57
+ "vendor_address": "4820 Harborview Parkway, Suite 300, Seattle, WA 98109",
58
+ "vendor_tax_id": "88-4122371",
59
+ "gstin": "27AAPFU0939F1ZV",
60
+ "bill_to": "Contoso Manufacturing Pvt Ltd",
61
+ "subtotal": "$153,459.70",
62
+ "tax_amount": "$13,044.07",
63
+ "total": "$166,503.77",
64
+ "tax_label": "Sales Tax 8.5%",
65
+ "page_count": 1,
66
+ "extra": {}
67
+ },
68
+ {
69
+ "document_id": "invoice_03_two_column",
70
+ "path": "C:\\Users\\soumi\\OneDrive\\Documents\\Test_Directory\\BEST-SLM\\docx_service\\corpus\\invoice_03_two_column.pdf",
71
+ "variant": "two_column",
72
+ "invoice_number": "INV-393269",
73
+ "invoice_date": "August 15, 2025",
74
+ "due_date": "August 1, 2025",
75
+ "po_number": "PO-21527",
76
+ "payment_terms": "Net 45",
77
+ "currency": "EUR",
78
+ "vendor_name": "Contoso Manufacturing Pvt Ltd",
79
+ "vendor_address": "1180 Industrial Loop, Building 7, Austin, TX 78744",
80
+ "vendor_tax_id": "74-2930118",
81
+ "gstin": "29AAGCC7391J1Z2",
82
+ "bill_to": "Proseware Data Systems",
83
+ "subtotal": "\u20ac39,870.30",
84
+ "tax_amount": "\u20ac1,953.64",
85
+ "total": "\u20ac41,026.53",
86
+ "tax_label": "IGST 5%",
87
+ "page_count": 1,
88
+ "extra": {}
89
+ },
90
+ {
91
+ "document_id": "invoice_04_dense",
92
+ "path": "C:\\Users\\soumi\\OneDrive\\Documents\\Test_Directory\\BEST-SLM\\docx_service\\corpus\\invoice_04_dense.pdf",
93
+ "variant": "dense",
94
+ "invoice_number": "INV-957268",
95
+ "invoice_date": "April 7, 2025",
96
+ "due_date": "June 14, 2025",
97
+ "po_number": "PO-41031",
98
+ "payment_terms": "Net 15",
99
+ "currency": "USD",
100
+ "vendor_name": "Fabrikam Consulting Group",
101
+ "vendor_address": "915 Ocean Drive, Floor 12, Miami, FL 33139",
102
+ "vendor_tax_id": "55-9014423",
103
+ "gstin": "07AAACF9876K1ZQ",
104
+ "bill_to": "Litware Cloud Services",
105
+ "subtotal": "$71,276.85",
106
+ "tax_amount": "$6,058.53",
107
+ "total": "$77,335.38",
108
+ "tax_label": "Sales Tax 8.5%",
109
+ "page_count": 2,
110
+ "extra": {}
111
+ },
112
+ {
113
+ "document_id": "invoice_05_letter",
114
+ "path": "C:\\Users\\soumi\\OneDrive\\Documents\\Test_Directory\\BEST-SLM\\docx_service\\corpus\\invoice_05_letter.pdf",
115
+ "variant": "letter",
116
+ "invoice_number": "INV-160128",
117
+ "invoice_date": "January 12, 2025",
118
+ "due_date": "March 20, 2025",
119
+ "po_number": "PO-78071",
120
+ "payment_terms": "Net 15",
121
+ "currency": "USD",
122
+ "vendor_name": "Proseware Data Systems",
123
+ "vendor_address": "58 Birchwood Court, Boston, MA 02110",
124
+ "vendor_tax_id": "04-7123309",
125
+ "gstin": "27AABPD5678L1ZB",
126
+ "bill_to": "Adventure Works Supply",
127
+ "subtotal": "$26,186.00",
128
+ "tax_amount": "$1,309.30",
129
+ "total": "$27,495.30",
130
+ "tax_label": "IGST 5%",
131
+ "page_count": 1,
132
+ "extra": {}
133
+ },
134
+ {
135
+ "document_id": "invoice_06_classic",
136
+ "path": "C:\\Users\\soumi\\OneDrive\\Documents\\Test_Directory\\BEST-SLM\\docx_service\\corpus\\invoice_06_classic.pdf",
137
+ "variant": "classic",
138
+ "invoice_number": "INV-106870",
139
+ "invoice_date": "March 18, 2025",
140
+ "due_date": "December 4, 2025",
141
+ "po_number": "PO-61340",
142
+ "payment_terms": "Net 45",
143
+ "currency": "USD",
144
+ "vendor_name": "Litware Cloud Services",
145
+ "vendor_address": "2200 Telegraph Avenue, Denver, CO 80202",
146
+ "vendor_tax_id": "91-2200981",
147
+ "gstin": "19AAHCL4321K1ZR",
148
+ "bill_to": "Wide World Importers",
149
+ "subtotal": "$153,890.00",
150
+ "tax_amount": "$30,778.00",
151
+ "total": "$184,668.00",
152
+ "tax_label": "VAT 20%",
153
+ "page_count": 1,
154
+ "extra": {}
155
+ },
156
+ {
157
+ "document_id": "invoice_07_minimal",
158
+ "path": "C:\\Users\\soumi\\OneDrive\\Documents\\Test_Directory\\BEST-SLM\\docx_service\\corpus\\invoice_07_minimal.pdf",
159
+ "variant": "minimal",
160
+ "invoice_number": "INV-183581",
161
+ "invoice_date": "June 30, 2025",
162
+ "due_date": "May 5, 2025",
163
+ "po_number": "PO-92488",
164
+ "payment_terms": "Due on receipt",
165
+ "currency": "USD",
166
+ "vendor_name": "Litware Cloud Services",
167
+ "vendor_address": "2200 Telegraph Avenue, Denver, CO 80202",
168
+ "vendor_tax_id": "91-2200981",
169
+ "gstin": "19AAHCL4321K1ZR",
170
+ "bill_to": "Fabrikam Consulting Group",
171
+ "subtotal": "$12,346.20",
172
+ "tax_amount": "$733.06",
173
+ "total": "$12,461.95",
174
+ "tax_label": "Sales Tax 6.25%",
175
+ "page_count": 1,
176
+ "extra": {}
177
+ },
178
+ {
179
+ "document_id": "invoice_08_boxed",
180
+ "path": "C:\\Users\\soumi\\OneDrive\\Documents\\Test_Directory\\BEST-SLM\\docx_service\\corpus\\invoice_08_boxed.pdf",
181
+ "variant": "boxed",
182
+ "invoice_number": "INV-401585",
183
+ "invoice_date": "December 5, 2025",
184
+ "due_date": "January 23, 2026",
185
+ "po_number": "PO-77199",
186
+ "payment_terms": "Net 30",
187
+ "currency": "EUR",
188
+ "vendor_name": "Adventure Works Supply",
189
+ "vendor_address": "3301 Market Street, San Francisco, CA 94114",
190
+ "vendor_tax_id": "94-3301876",
191
+ "gstin": "33AACCA9876T1Z9",
192
+ "bill_to": "Proseware Data Systems",
193
+ "subtotal": "\u20ac32,573.50",
194
+ "tax_amount": "\u20ac6,514.70",
195
+ "total": "\u20ac39,088.20",
196
+ "tax_label": "VAT 20%",
197
+ "page_count": 1,
198
+ "extra": {}
199
+ },
200
+ {
201
+ "document_id": "invoice_09_two_column",
202
+ "path": "C:\\Users\\soumi\\OneDrive\\Documents\\Test_Directory\\BEST-SLM\\docx_service\\corpus\\invoice_09_two_column.pdf",
203
+ "variant": "two_column",
204
+ "invoice_number": "INV-905161",
205
+ "invoice_date": "November 21, 2025",
206
+ "due_date": "August 1, 2025",
207
+ "po_number": "PO-61346",
208
+ "payment_terms": "Net 30",
209
+ "currency": "INR",
210
+ "vendor_name": "Litware Cloud Services",
211
+ "vendor_address": "2200 Telegraph Avenue, Denver, CO 80202",
212
+ "vendor_tax_id": "91-2200981",
213
+ "gstin": "19AAHCL4321K1ZR",
214
+ "bill_to": "Tailspin Tooling Co.",
215
+ "subtotal": "\u20b9126,936.70",
216
+ "tax_amount": "\u20b97,933.54",
217
+ "total": "\u20b9134,870.24",
218
+ "tax_label": "Sales Tax 6.25%",
219
+ "page_count": 2,
220
+ "extra": {}
221
+ },
222
+ {
223
+ "document_id": "invoice_10_dense",
224
+ "path": "C:\\Users\\soumi\\OneDrive\\Documents\\Test_Directory\\BEST-SLM\\docx_service\\corpus\\invoice_10_dense.pdf",
225
+ "variant": "dense",
226
+ "invoice_number": "INV-428627",
227
+ "invoice_date": "February 3, 2025",
228
+ "due_date": "March 20, 2025",
229
+ "po_number": "PO-67498",
230
+ "payment_terms": "Due on receipt",
231
+ "currency": "USD",
232
+ "vendor_name": "Contoso Manufacturing Pvt Ltd",
233
+ "vendor_address": "1180 Industrial Loop, Building 7, Austin, TX 78744",
234
+ "vendor_tax_id": "74-2930118",
235
+ "gstin": "29AAGCC7391J1Z2",
236
+ "bill_to": "Litware Cloud Services",
237
+ "subtotal": "$78,917.05",
238
+ "tax_amount": "$6,372.55",
239
+ "total": "$81,343.75",
240
+ "tax_label": "Sales Tax 8.5%",
241
+ "page_count": 1,
242
+ "extra": {}
243
+ },
244
+ {
245
+ "document_id": "invoice_11_letter",
246
+ "path": "C:\\Users\\soumi\\OneDrive\\Documents\\Test_Directory\\BEST-SLM\\docx_service\\corpus\\invoice_11_letter.pdf",
247
+ "variant": "letter",
248
+ "invoice_number": "INV-717142",
249
+ "invoice_date": "March 18, 2025",
250
+ "due_date": "June 14, 2025",
251
+ "po_number": "PO-37660",
252
+ "payment_terms": "Net 60",
253
+ "currency": "USD",
254
+ "vendor_name": "Litware Cloud Services",
255
+ "vendor_address": "2200 Telegraph Avenue, Denver, CO 80202",
256
+ "vendor_tax_id": "91-2200981",
257
+ "gstin": "19AAHCL4321K1ZR",
258
+ "bill_to": "Wide World Importers",
259
+ "subtotal": "$35,173.80",
260
+ "tax_amount": "$2,198.36",
261
+ "total": "$37,372.16",
262
+ "tax_label": "Sales Tax 6.25%",
263
+ "page_count": 1,
264
+ "extra": {}
265
+ }
266
+ ]
docqa/docxextract/__init__.py ADDED
@@ -0,0 +1,25 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """Document data extraction service.
2
+
3
+ Extracts arbitrary user-defined keys from invoices, receipts and scanned
4
+ documents using Impira's LayoutLM document-QA models.
5
+
6
+ from docxextract import ExtractionService
7
+
8
+ svc = ExtractionService()
9
+ result = svc.extract(pdf_bytes, ["Invoice No", "Total Amount", "GSTIN"])
10
+ for field in result.fields:
11
+ print(field.key, field.value, field.status, field.confidence)
12
+ """
13
+
14
+ from .config import Settings, get_settings
15
+ from .engine import Engine, get_engine
16
+ from .schema import (DocumentInfo, ExtractedField, ExtractResponse, FieldStatus,
17
+ SourceType)
18
+ from .service import ExtractionService
19
+
20
+ __version__ = "1.0.0"
21
+ __all__ = [
22
+ "Settings", "get_settings", "Engine", "get_engine",
23
+ "ExtractionService", "ExtractResponse", "ExtractedField",
24
+ "DocumentInfo", "FieldStatus", "SourceType", "__version__",
25
+ ]
docqa/docxextract/api.py ADDED
@@ -0,0 +1,198 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """FastAPI application.
2
+
3
+ Run:
4
+ uvicorn docxextract.api:app --host 0.0.0.0 --port 8000
5
+ """
6
+
7
+ from __future__ import annotations
8
+
9
+ import asyncio
10
+ import time
11
+ from contextlib import asynccontextmanager
12
+
13
+ from fastapi import FastAPI, File, Form, Request, UploadFile
14
+ from fastapi.responses import JSONResponse, ORJSONResponse
15
+ from pydantic import ValidationError as PydanticValidationError
16
+
17
+ from . import parsing
18
+ from .config import get_settings
19
+ from .engine import get_engine
20
+ from .logging import configure_logging, get_logger, new_request_id
21
+ from .schema import (DocExtractError, ExtractRequest, ExtractResponse,
22
+ ValidationError)
23
+ from .service import ExtractionService
24
+
25
+ log = get_logger(__name__)
26
+ settings = get_settings()
27
+
28
+
29
+ @asynccontextmanager
30
+ async def lifespan(app: FastAPI):
31
+ configure_logging(settings.log_level, settings.log_json)
32
+ parsing.configure_cache(settings)
33
+ log.info("starting up", extra={
34
+ "detail": f"model={settings.model_id} threads={settings.torch_threads} "
35
+ f"batch={settings.batch_size}",
36
+ })
37
+ # Load and warm the model before serving, so the first request does not
38
+ # absorb a 1-135s cold start.
39
+ try:
40
+ await asyncio.to_thread(get_engine(settings).warmup)
41
+ log.info("model ready")
42
+ except Exception as exc: # noqa: BLE001
43
+ # Startup should not hard-fail: the health endpoint will report the
44
+ # degraded state and requests will surface a 503.
45
+ log.error("model warmup failed", exc_info=exc)
46
+ yield
47
+ log.info("shutting down")
48
+
49
+
50
+ app = FastAPI(
51
+ title="Document Data Extraction Service",
52
+ version="1.0.0",
53
+ description="Extract arbitrary user-defined keys from invoices, receipts "
54
+ "and scanned documents.",
55
+ default_response_class=ORJSONResponse,
56
+ lifespan=lifespan,
57
+ )
58
+
59
+
60
+ @app.middleware("http")
61
+ async def request_context(request: Request, call_next):
62
+ rid = new_request_id()
63
+ started = time.perf_counter()
64
+ try:
65
+ response = await call_next(request)
66
+ except DocExtractError as exc:
67
+ return JSONResponse(
68
+ status_code=exc.status_code,
69
+ content={"error": type(exc).__name__, "message": exc.message,
70
+ "detail": exc.detail, "request_id": rid},
71
+ )
72
+ except Exception as exc: # noqa: BLE001
73
+ log.error("unhandled error", exc_info=exc, extra={"request_id": rid})
74
+ return JSONResponse(
75
+ status_code=500,
76
+ content={"error": "InternalError",
77
+ "message": "an unexpected error occurred",
78
+ "request_id": rid},
79
+ )
80
+ response.headers["X-Request-ID"] = rid
81
+ response.headers["X-Latency-Ms"] = f"{(time.perf_counter()-started)*1000:.1f}"
82
+ return response
83
+
84
+
85
+ def _service() -> ExtractionService:
86
+ return ExtractionService(settings)
87
+
88
+
89
+ @app.get("/health", tags=["ops"])
90
+ async def health() -> dict:
91
+ """Liveness plus model readiness, for load-balancer checks."""
92
+ engine = get_engine(settings)
93
+ # `warm` is the readiness signal for both backends. Reaching into
94
+ # engine._pipe would break the ONNX backend, which has no such attribute.
95
+ ready = bool(getattr(engine, "warm", False))
96
+ return {
97
+ "status": "ok" if ready else "degraded",
98
+ "model": settings.model_id,
99
+ "backend": type(engine).__name__,
100
+ "model_ready": ready,
101
+ "load_seconds": round(getattr(engine, "load_seconds", 0.0), 2),
102
+ "parse_cache": parsing.cache_stats(),
103
+ }
104
+
105
+
106
+ @app.get("/v1/config", tags=["ops"])
107
+ async def config() -> dict:
108
+ return {
109
+ "model": settings.model_id,
110
+ "device": settings.device,
111
+ "batch_size": settings.batch_size,
112
+ "max_keys": settings.max_keys,
113
+ "max_pages": settings.max_pages,
114
+ "max_upload_mb": settings.max_upload_mb,
115
+ "confidence_threshold": settings.confidence_threshold,
116
+ }
117
+
118
+
119
+ @app.post("/v1/extract", response_model=ExtractResponse, tags=["extract"])
120
+ async def extract(
121
+ file: UploadFile = File(..., description="PDF, PNG, JPEG, TIFF or BMP"),
122
+ keys: str = Form(..., description="Comma-separated extraction keys"),
123
+ confidence_threshold: float | None = Form(default=None),
124
+ max_pages: int | None = Form(default=None),
125
+ ) -> ExtractResponse:
126
+ """Extract one value per requested key from a document.
127
+
128
+ Keys are comma-separated, e.g.
129
+ `keys=INVOICE NO,Vendor Id,Invoice Date,Total Amount,GSTIN`
130
+ """
131
+ key_list = [k.strip() for k in keys.split(",") if k.strip()]
132
+ if not key_list:
133
+ raise ValidationError("at least one non-empty key is required")
134
+ if len(key_list) > settings.max_keys:
135
+ raise ValidationError(
136
+ f"too many keys: {len(key_list)} > {settings.max_keys}")
137
+
138
+ # Read with a hard cap so a large upload cannot exhaust memory before the
139
+ # size check ever runs.
140
+ limit = settings.max_upload_mb * 1024 * 1024
141
+ data = await file.read(limit + 1)
142
+ if len(data) > limit:
143
+ from .schema import PayloadTooLargeError
144
+ raise PayloadTooLargeError(
145
+ f"document exceeds {settings.max_upload_mb} MB")
146
+
147
+ service = _service()
148
+ try:
149
+ return await asyncio.wait_for(
150
+ asyncio.to_thread(
151
+ service.extract, data, key_list,
152
+ confidence_threshold, max_pages),
153
+ timeout=settings.request_timeout_s,
154
+ )
155
+ except asyncio.TimeoutError:
156
+ from .schema import ExtractionTimeoutError
157
+ log.error("extraction timed out", extra={"n_keys": len(key_list)})
158
+ raise ExtractionTimeoutError(
159
+ f"extraction exceeded {settings.request_timeout_s}s") from None
160
+
161
+
162
+ @app.post("/v1/extract/json", response_model=ExtractResponse, tags=["extract"])
163
+ async def extract_json(
164
+ payload: dict,
165
+ request: Request,
166
+ ) -> ExtractResponse:
167
+ """JSON variant: base64 document plus a keys array.
168
+
169
+ Useful for clients that already hold the file in memory and want to avoid
170
+ multipart encoding overhead.
171
+ """
172
+ import base64
173
+
174
+ try:
175
+ req = ExtractRequest(
176
+ keys=payload.get("keys", []),
177
+ confidence_threshold=payload.get("confidence_threshold"),
178
+ max_pages=payload.get("max_pages"),
179
+ )
180
+ except PydanticValidationError as exc:
181
+ raise ValidationError("invalid request", detail=exc.errors()) from exc
182
+
183
+ b64 = payload.get("document_base64")
184
+ if not isinstance(b64, str) or not b64:
185
+ raise ValidationError("document_base64 is required")
186
+ if "," in b64[:100] and b64.startswith("data:"):
187
+ b64 = b64.split(",", 1)[1]
188
+ try:
189
+ data = base64.b64decode(b64, validate=True)
190
+ except Exception as exc: # noqa: BLE001
191
+ raise ValidationError("document_base64 is not valid base64") from exc
192
+
193
+ service = _service()
194
+ return await asyncio.wait_for(
195
+ asyncio.to_thread(service.extract, data, req.keys,
196
+ req.confidence_threshold, req.max_pages),
197
+ timeout=settings.request_timeout_s,
198
+ )
docqa/docxextract/config.py ADDED
@@ -0,0 +1,66 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """Typed configuration, sourced from environment variables."""
2
+
3
+ from __future__ import annotations
4
+
5
+ from functools import lru_cache
6
+
7
+ from pydantic import Field
8
+ from pydantic_settings import BaseSettings, SettingsConfigDict
9
+
10
+
11
+ class Settings(BaseSettings):
12
+ """Runtime configuration.
13
+
14
+ All values can be overridden with the ``DOCX_`` environment prefix, e.g.
15
+ ``DOCX_MAX_UPLOAD_MB=25``.
16
+ """
17
+
18
+ model_config = SettingsConfigDict(
19
+ env_prefix="DOCX_", env_file=".env", extra="ignore"
20
+ )
21
+
22
+ # --- model ---
23
+ # NOTE: impira/layoutlm-invoices is licensed CC-BY-NC-SA-4.0 (non-commercial).
24
+ # impira/layoutlm-document-qa is MIT and safe for commercial use.
25
+ model_id: str = "impira/layoutlm-invoices"
26
+ device: str = "cpu"
27
+ torch_threads: int = Field(default=4, ge=1, le=64)
28
+
29
+ # --- extraction behaviour ---
30
+ # Answers below this confidence are reported as `low_confidence` rather
31
+ # than silently returned as fact.
32
+ confidence_threshold: float = Field(default=0.5, ge=0.0, le=1.0)
33
+ # Maximum pages processed per document. LayoutLM's effective context is
34
+ # ~512 tokens, so deep multi-page scans are truncated anyway.
35
+ max_pages: int = Field(default=3, ge=1, le=50)
36
+ # Keys per batched forward pass. Measured sweet spot on CPU; see BENCHMARK.md.
37
+ batch_size: int = Field(default=8, ge=1, le=64)
38
+ # How many keys one request may ask for.
39
+ max_keys: int = Field(default=64, ge=1, le=512)
40
+ max_key_length: int = Field(default=200, ge=1, le=2000)
41
+
42
+ # --- resource limits ---
43
+ max_upload_mb: int = Field(default=25, ge=1, le=500)
44
+ request_timeout_s: float = Field(default=60.0, gt=0)
45
+ # Rendered-page pixel budget; guards against decompression bombs where a
46
+ # small file expands to a huge bitmap.
47
+ max_render_pixels: int = Field(default=40_000_000, ge=1_000_000)
48
+
49
+ # --- caching ---
50
+ # Parsed text+boxes are cached by content hash so repeated keys against the
51
+ # same document do not re-run OCR/parsing.
52
+ parse_cache_size: int = Field(default=128, ge=0, le=10_000)
53
+
54
+ # --- concurrency ---
55
+ # Torch is not thread-safe for concurrent forward passes on one module
56
+ # without a lock; the engine serialises model access and relies on
57
+ # batching for throughput.
58
+ max_concurrency: int = Field(default=4, ge=1, le=64)
59
+
60
+ log_level: str = "INFO"
61
+ log_json: bool = True
62
+
63
+
64
+ @lru_cache(maxsize=1)
65
+ def get_settings() -> Settings:
66
+ return Settings()
docqa/docxextract/engine.py ADDED
@@ -0,0 +1,225 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """LayoutLM inference: batched, thread-safe, warmed.
2
+
3
+ Design notes
4
+ ------------
5
+ **Batching is the main optimisation.** The naive pattern -- one pipeline call
6
+ per key -- re-encodes the same page for every key. LayoutLM's encoder is
7
+ bidirectional and the question is interleaved with the document, so nothing is
8
+ shareable across calls. Passing N keys in one call instead lets the CPU GEMMs
9
+ amortise weight reads. Measured gains are in ``bench/bench_batching.py``.
10
+
11
+ **The model has no null-answer head.** It will always return *some* span for any
12
+ input, including questions whose answer is not on the page. Absence therefore
13
+ has to be decided by the caller using the returned score, which is what
14
+ ``confidence_threshold`` in the service layer does. This is a genuine limitation
15
+ of the architecture, not something that can be tuned away.
16
+
17
+ **Thread safety.** A single module instance is guarded by a lock. Torch's CPU
18
+ kernels are internally parallel, so concurrent Python threads around one module
19
+ buy nothing and risk races in the tokenizer; batching is the correct lever.
20
+ """
21
+
22
+ from __future__ import annotations
23
+
24
+ import os
25
+ import threading
26
+ import time
27
+ from dataclasses import dataclass
28
+ from typing import Sequence
29
+
30
+ from .config import Settings
31
+ from .logging import get_logger
32
+ from .schema import ModelUnavailableError
33
+
34
+ log = get_logger(__name__)
35
+
36
+
37
+ @dataclass(frozen=True, slots=True)
38
+ class Span:
39
+ """A candidate answer span for one key on one page."""
40
+
41
+ key: str
42
+ answer: str
43
+ confidence: float
44
+ page: int
45
+ start: int
46
+ end: int
47
+
48
+
49
+ class Engine:
50
+ """Thread-safe wrapper around the HF document-question-answering pipeline."""
51
+
52
+ def __init__(self, settings: Settings):
53
+ self.settings = settings
54
+ self._lock = threading.Lock()
55
+ self._pipe = None
56
+ self.load_seconds: float = 0.0
57
+ self.warm: bool = False
58
+
59
+ # ------------------------------------------------------------------
60
+ def load(self) -> None:
61
+ if self._pipe is not None:
62
+ return
63
+ try:
64
+ import torch
65
+ from transformers import pipeline
66
+
67
+ torch.set_num_threads(self.settings.torch_threads)
68
+ t0 = time.perf_counter()
69
+ pipe = pipeline(
70
+ "document-question-answering",
71
+ model=self.settings.model_id,
72
+ device=self.settings.device,
73
+ )
74
+ pipe.model.eval()
75
+ self.load_seconds = time.perf_counter() - t0
76
+ self._pipe = pipe
77
+ log.info("engine loaded", extra={
78
+ "detail": f"{self.settings.model_id} in {self.load_seconds:.2f}s",
79
+ })
80
+ except Exception as exc: # noqa: BLE001
81
+ raise ModelUnavailableError("model could not be loaded",
82
+ detail=str(exc)) from exc
83
+
84
+ def warmup(self) -> None:
85
+ """Run one throwaway inference so the first real request is not slow.
86
+
87
+ Without this, the first request pays for lazy kernel selection and
88
+ allocator growth, inflating its latency and skewing p95.
89
+ """
90
+ if self.warm:
91
+ return
92
+ self.load()
93
+ with self._lock:
94
+ self._pipe(image=None, question="What is the date?",
95
+ word_boxes=[["Invoice", [0, 0, 100, 20]],
96
+ ["Date", [110, 0, 190, 20]],
97
+ ["2025", [200, 0, 260, 20]]],
98
+ top_k=1)
99
+ self.warm = True
100
+ log.info("engine warmed")
101
+
102
+ # ------------------------------------------------------------------
103
+ @staticmethod
104
+ def _unwrap(obj):
105
+ """Flatten the pipeline's nested list output to a list of dicts.
106
+
107
+ transformers 4.50 returns `[[{...}]]` for batched input even though the
108
+ docstring advertises a bare dict, and the nesting depth has changed
109
+ between releases. Normalising here keeps callers version-agnostic.
110
+ """
111
+ out: list[dict] = []
112
+ stack = [obj]
113
+ while stack:
114
+ item = stack.pop(0)
115
+ if isinstance(item, dict):
116
+ out.append(item)
117
+ elif isinstance(item, (list, tuple)):
118
+ stack.extend(item)
119
+ return out
120
+
121
+ def answer_batch(self, page_words: Sequence[str],
122
+ page_boxes: Sequence[Sequence[int]],
123
+ keys: Sequence[str],
124
+ page_index: int = 1) -> list[Span]:
125
+ """Answer every key against one page in a batched forward pass.
126
+
127
+ Returns one Span per key, in the order the keys were given.
128
+ """
129
+ self.load()
130
+ word_boxes = [[w, list(b)] for w, b in zip(page_words, page_boxes)]
131
+
132
+ # The pipeline is given `image=None` so it uses our pre-computed
133
+ # word_boxes instead of running its own Tesseract pass. `image` is a
134
+ # required positional parameter in transformers' implementation.
135
+ payload = [{"image": None, "question": k, "word_boxes": word_boxes}
136
+ for k in keys]
137
+
138
+ batch = self.settings.batch_size
139
+ results: list[dict] = []
140
+ with self._lock:
141
+ for start in range(0, len(payload), batch):
142
+ chunk = payload[start:start + batch]
143
+ out = self._pipe(chunk, top_k=1)
144
+ results.extend(self._unwrap(out))
145
+
146
+ spans: list[Span] = []
147
+ for key, res in zip(keys, results):
148
+ spans.append(Span(
149
+ key=key,
150
+ answer=str(res.get("answer", "")).strip(),
151
+ confidence=float(res.get("score", 0.0) or 0.0),
152
+ page=page_index,
153
+ start=int(res.get("start", -1)),
154
+ end=int(res.get("end", -1)),
155
+ ))
156
+ return spans
157
+
158
+ # ------------------------------------------------------------------
159
+ def extract(self, pages, keys: Sequence[str],
160
+ max_pages: int) -> list[Span]:
161
+ """Answer every key, choosing the best-scoring page for each.
162
+
163
+ Multi-page documents are scored page by page and the highest-confidence
164
+ span wins. Keys that only appear on page 3 would be missed by a
165
+ first-page-only strategy.
166
+ """
167
+ best: dict[str, Span] = {}
168
+ limit = min(max_pages, len(pages))
169
+ for page_index in range(limit):
170
+ page = pages[page_index]
171
+ if not page.words:
172
+ # An empty page cannot contain the answer; asking the model
173
+ # anyway only invites fabrication.
174
+ for key in keys:
175
+ best.setdefault(key, Span(key, "", 0.0, page_index + 1, -1, -1))
176
+ continue
177
+ for span in self.answer_batch(page.words, page.boxes, keys,
178
+ page_index + 1):
179
+ current = best.get(span.key)
180
+ if current is None or span.confidence > current.confidence:
181
+ best[span.key] = span
182
+ return [best[k] for k in keys if k in best]
183
+
184
+
185
+ _engine: Engine | None = None
186
+ _engine_lock = threading.Lock()
187
+
188
+
189
+ def get_engine(settings: Settings) -> Engine:
190
+ """Return the configured engine, preferring ONNX Runtime when available.
191
+
192
+ Set ``DOCX_BACKEND=torch`` to force PyTorch. The ONNX path is selected
193
+ automatically only if its artefacts exist; it was measured at ~30% lower
194
+ latency with identical answers at fp32.
195
+ """
196
+ global _engine
197
+ if _engine is None:
198
+ with _engine_lock:
199
+ if _engine is None:
200
+ backend = os.environ.get("DOCX_BACKEND", "auto").lower()
201
+ if backend in ("auto", "onnx"):
202
+ try:
203
+ from .onnx_engine import OnnxEngine, default_onnx_path
204
+ if default_onnx_path(settings.model_id).exists():
205
+ _engine = OnnxEngine(settings) # type: ignore[assignment]
206
+ log.info("using ONNX Runtime backend")
207
+ return _engine
208
+ if backend == "onnx":
209
+ raise ModelUnavailableError(
210
+ "ONNX backend requested but artefacts are missing",
211
+ detail=f"expected {default_onnx_path(settings.model_id)}",
212
+ )
213
+ except FileNotFoundError as exc:
214
+ if backend == "onnx":
215
+ raise ModelUnavailableError(str(exc)) from exc
216
+ log.info("ONNX artefacts absent, falling back to torch")
217
+ _engine = Engine(settings)
218
+ return _engine
219
+
220
+
221
+ def reset_engine() -> None:
222
+ """Drop the engine. Used by tests that swap models or settings."""
223
+ global _engine
224
+ with _engine_lock:
225
+ _engine = None
docqa/docxextract/logging.py ADDED
@@ -0,0 +1,61 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """Structured JSON logging with request correlation."""
2
+
3
+ from __future__ import annotations
4
+
5
+ import logging
6
+ import sys
7
+ import time
8
+ import uuid
9
+ from contextvars import ContextVar
10
+ from typing import Any
11
+
12
+ _request_id: ContextVar[str] = ContextVar("request_id", default="-")
13
+
14
+
15
+ class JsonFormatter(logging.Formatter):
16
+ def format(self, record: logging.LogRecord) -> str:
17
+ import json
18
+
19
+ payload: dict[str, Any] = {
20
+ "ts": time.strftime("%Y-%m-%dT%H:%M:%S", time.gmtime(record.created))
21
+ + f".{int(record.msecs):03d}Z",
22
+ "level": record.levelname,
23
+ "logger": record.name,
24
+ "msg": record.getMessage(),
25
+ "request_id": _request_id.get(),
26
+ }
27
+ for key in ("document_id", "latency_ms", "status", "n_keys", "detail"):
28
+ val = getattr(record, key, None)
29
+ if val is not None:
30
+ payload[key] = val
31
+ if record.exc_info:
32
+ payload["exc"] = self.formatException(record.exc_info)
33
+ return json.dumps(payload, ensure_ascii=False)
34
+
35
+
36
+ def configure_logging(level: str = "INFO", json_output: bool = True) -> None:
37
+ root = logging.getLogger()
38
+ root.handlers.clear()
39
+ handler = logging.StreamHandler(sys.stdout)
40
+ if json_output:
41
+ handler.setFormatter(JsonFormatter())
42
+ else:
43
+ handler.setFormatter(
44
+ logging.Formatter("%(asctime)s %(levelname)-7s %(name)s | %(message)s")
45
+ )
46
+ root.addHandler(handler)
47
+ root.setLevel(level.upper())
48
+ # uvicorn duplicates access logs through its own handlers.
49
+ for noisy in ("uvicorn.access", "uvicorn.error"):
50
+ logging.getLogger(noisy).handlers.clear()
51
+ logging.getLogger(noisy).propagate = True
52
+
53
+
54
+ def new_request_id() -> str:
55
+ rid = uuid.uuid4().hex[:16]
56
+ _request_id.set(rid)
57
+ return rid
58
+
59
+
60
+ def get_logger(name: str) -> logging.Logger:
61
+ return logging.getLogger(name)
docqa/docxextract/onnx_engine.py ADDED
@@ -0,0 +1,243 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """ONNX Runtime backend for the extraction engine.
2
+
3
+ Enables dropping PyTorch from the serving process entirely: ~1.1 GB of
4
+ dependencies removed, and a measured ~30% latency reduction at fp32.
5
+
6
+ Two constraints discovered by measuring against the reference pipeline, both of
7
+ which silently corrupt answers if ignored:
8
+
9
+ 1. **SEP tokens get bbox [1000, 1000, 1000, 1000]**, not [0, 0, 0, 0]. The
10
+ pipeline distinguishes SEP from question/CLS tokens this way. LayoutLM has
11
+ learned x/y position embeddings, so zeroing SEP boxes shifts the encoding
12
+ and the model answers a different span.
13
+ 2. **Answers are the original word strings joined by spaces**, not a decode of
14
+ subword ids. These differ for dates, currency symbols and hyphenated
15
+ identifiers such as GSTIN and EIN.
16
+
17
+ `verify_against_torch()` asserts both, so a future ONNX Runtime or model
18
+ upgrade that changes either fails loudly instead of quietly losing accuracy.
19
+ """
20
+
21
+ from __future__ import annotations
22
+
23
+ import threading
24
+ import time
25
+ from pathlib import Path
26
+ from typing import Sequence
27
+
28
+ from .logging import get_logger
29
+
30
+ log = get_logger(__name__)
31
+
32
+ # Set False only after measuring a regression on your own corpus; see BENCHMARK.md.
33
+ ENABLE_INT8 = False
34
+
35
+
36
+ def default_onnx_path(model_id: str) -> Path:
37
+ root = Path(__file__).resolve().parent.parent.parent / "artifacts" / "onnx"
38
+ return root / model_id.replace("/", "__") / "model.onnx"
39
+
40
+
41
+ def default_int8_path(model_id: str) -> Path:
42
+ root = Path(__file__).resolve().parent.parent.parent / "artifacts" / "onnx"
43
+ return root / model_id.replace("/", "__") / "model_int8.onnx"
44
+
45
+
46
+ class OnnxEngine:
47
+ """Drop-in replacement for engine.Engine backed by ONNX Runtime."""
48
+
49
+ def __init__(self, settings, model_path: Path | None = None,
50
+ quantized: bool = False):
51
+ import numpy as np # noqa: F401 (import checked early)
52
+ import onnxruntime as ort
53
+ from transformers import AutoTokenizer
54
+
55
+ self.settings = settings
56
+ self.model_id = settings.model_id
57
+ self._lock = threading.Lock()
58
+ self.load_seconds = 0.0
59
+ self.warm = False
60
+
61
+ path = model_path or (default_int8_path(self.model_id) if quantized
62
+ else default_onnx_path(self.model_id))
63
+ if not path.exists():
64
+ raise FileNotFoundError(
65
+ f"ONNX model not found at {path}. "
66
+ f"Run: python bench/export_onnx.py {self.model_id} --quantize"
67
+ )
68
+
69
+ started = time.perf_counter()
70
+ self.tokenizer = AutoTokenizer.from_pretrained(self.model_id)
71
+
72
+ options = ort.SessionOptions()
73
+ # intra_op threads scale the GEMMs; inter_op is left at 1 because the
74
+ # graph is a stack of independent attention blocks and extra
75
+ # parallelism only adds synchronisation overhead.
76
+ options.intra_op_num_threads = settings.torch_threads
77
+ options.inter_op_num_threads = 1
78
+ options.graph_optimization_level = ort.GraphOptimizationLevel.ORT_ENABLE_ALL
79
+
80
+ self.session = ort.InferenceSession(str(path), options,
81
+ providers=["CPUExecutionProvider"])
82
+ self.model_path = path
83
+ self.load_seconds = time.perf_counter() - started
84
+ log.info("onnx engine loaded", extra={
85
+ "detail": f"{path.name} in {self.load_seconds:.2f}s "
86
+ f"({path.stat().st_size / 1e6:.0f} MB, "
87
+ f"threads={settings.torch_threads})",
88
+ })
89
+
90
+ # ------------------------------------------------------------------
91
+ def warmup(self) -> None:
92
+ if self.warm:
93
+ return
94
+ self._answer(["Invoice", "Date"], [[0, 0, 80, 20], [90, 0, 160, 20]],
95
+ "What is the date?")
96
+ self.warm = True
97
+
98
+ def _answer(self, words: Sequence[str], boxes: Sequence[Sequence[int]],
99
+ key: str) -> tuple[str, float]:
100
+ import numpy as np
101
+
102
+ word_boxes = list(zip(words, boxes))
103
+ encoding = self.tokenizer(
104
+ text=key.split(), text_pair=list(words), is_split_into_words=True,
105
+ return_token_type_ids=True, truncation="only_second",
106
+ max_length=self.tokenizer.model_max_length)
107
+
108
+ sequence_ids = encoding.sequence_ids(0)
109
+ sep_id = self.tokenizer.sep_token_id
110
+ bbox = []
111
+ for token_id, sid, wid in zip(encoding["input_ids"], sequence_ids,
112
+ encoding.word_ids(0)):
113
+ if sid == 1:
114
+ bbox.append(word_boxes[wid][1])
115
+ elif token_id == sep_id:
116
+ bbox.append([1000, 1000, 1000, 1000])
117
+ else:
118
+ bbox.append([0, 0, 0, 0])
119
+
120
+ feed = {
121
+ "input_ids": np.array([encoding["input_ids"]], dtype=np.int64),
122
+ "attention_mask": np.array([encoding["attention_mask"]], dtype=np.int64),
123
+ "token_type_ids": np.array([encoding["token_type_ids"]], dtype=np.int64),
124
+ "bbox": np.array([bbox], dtype=np.int64).reshape(1, -1),
125
+ }
126
+ start, end = self.session.run(None, feed)
127
+ start, end = start[0], end[0]
128
+
129
+ # Only context tokens may open or close a span (the pipeline's p_mask).
130
+ for i, sid in enumerate(sequence_ids):
131
+ if sid != 1:
132
+ start[i] = end[i] = -1e4
133
+ s = int(np.argmax(start))
134
+ e = int(np.argmax(end))
135
+ if e < s:
136
+ e = s
137
+
138
+ word_ids = encoding.word_ids(0)
139
+ w_start, w_end = word_ids[s], word_ids[e]
140
+ if w_start is None or w_end is None:
141
+ return "", 0.0
142
+ # Original word strings, matching the reference pipeline.
143
+ # These are raw logits, not probabilities. Computing exp(s)*exp(e)
144
+ # directly overflows and yields values far above 1, so normalise each
145
+ # axis with a stable softmax and multiply the two probabilities, which
146
+ # is what the reference pipeline's score represents.
147
+ start_f = start - start.max()
148
+ end_f = end - end.max()
149
+ start_p = np.exp(start_f) / np.exp(start_f).sum()
150
+ end_p = np.exp(end_f) / np.exp(end_f).sum()
151
+ confidence = float(np.clip(start_p[s] * end_p[e], 0.0, 1.0))
152
+ return " ".join(words[w_start:w_end + 1]), confidence
153
+
154
+ # ------------------------------------------------------------------
155
+ def answer_batch(self, page_words, page_boxes, keys,
156
+ page_index: int = 1):
157
+ from .engine import Span
158
+
159
+ # ONNX Runtime releases the GIL inside Run(), so concurrent keys do
160
+ # overlap here in a way they cannot inside PyTorch. Results are
161
+ # collected under a lock anyway to keep behaviour deterministic.
162
+ out: list[Span] = []
163
+ with self._lock:
164
+ for key in keys:
165
+ answer, confidence = self._answer(page_words, page_boxes, key)
166
+ out.append(Span(key=key, answer=answer, confidence=confidence,
167
+ page=page_index, start=-1, end=-1))
168
+ return out
169
+
170
+ def extract(self, pages, keys: Sequence[str], max_pages: int):
171
+ """Identical page-selection semantics to engine.Engine."""
172
+ best: dict[str, object] = {}
173
+ limit = min(max_pages, len(pages))
174
+ for page_index in range(limit):
175
+ page = pages[page_index]
176
+ if not page.words:
177
+ for key in keys:
178
+ best.setdefault(key, type("S", (), {
179
+ "key": key, "answer": "", "confidence": 0.0,
180
+ "page": page_index + 1, "start": -1, "end": -1})())
181
+ continue
182
+ for span in self.answer_batch(page.words, page.boxes, keys,
183
+ page_index + 1):
184
+ current = best.get(span.key)
185
+ if current is None or span.confidence > current.confidence:
186
+ best[span.key] = span
187
+ return [best[k] for k in keys if k in best]
188
+
189
+
190
+ # ----------------------------------------------------------------------
191
+ def verify_against_torch(settings, corpus_dir: Path,
192
+ documents: int = 1, keys: int = 6) -> dict:
193
+ """Assert the ONNX path matches the reference pipeline.
194
+
195
+ Cheap insurance: runs on a small slice of the corpus and raises if any
196
+ answer differs. A silent accuracy regression in the serving path is far
197
+ more expensive than this check.
198
+ """
199
+ import json
200
+
201
+ from . import parsing
202
+ from .engine import Engine
203
+
204
+ truths = json.loads((corpus_dir / "ground_truth.json").read_text())
205
+ engine = Engine(settings)
206
+ engine.load()
207
+
208
+ quantised = ENABLE_INT8
209
+ onnx = OnnxEngine(settings, quantized=quantised)
210
+ onnx.warmup()
211
+
212
+ probe_keys = ["INVOICE NO", "Invoice Date", "Total Amount", "PO Number",
213
+ "Payment Term", "GSTIN"][:keys]
214
+ mismatches: list[dict] = []
215
+ checked = 0
216
+
217
+ for record in truths[:documents]:
218
+ from pathlib import Path
219
+ data = Path(record["path"]).read_bytes()
220
+ page = parsing.parse(data, settings).pages[0]
221
+ for key in probe_keys:
222
+ ref = engine.answer_batch(page.words, page.boxes, [key])[0]
223
+ got = onnx.answer_batch(page.words, page.boxes, [key])[0]
224
+ checked += 1
225
+ if ref.answer != got.answer:
226
+ mismatches.append({
227
+ "document": record["document_id"], "key": key,
228
+ "torch": ref.answer, "onnx": got.answer,
229
+ "torch_conf": round(ref.confidence, 4),
230
+ "onnx_conf": round(got.confidence, 4),
231
+ })
232
+
233
+ result = {"checked": checked, "mismatches": mismatches,
234
+ "quantized": quantised}
235
+ if mismatches:
236
+ raise AssertionError(
237
+ f"ONNX backend disagrees with PyTorch on {len(mismatches)}/{checked} "
238
+ f"answers: {mismatches[:3]}"
239
+ )
240
+ log.info("onnx backend matches torch", extra={
241
+ "detail": f"{checked} answers verified",
242
+ })
243
+ return result
docqa/docxextract/parsing.py ADDED
@@ -0,0 +1,314 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """Document parsing: bytes -> (words, boxes) per page.
2
+
3
+ This is the single biggest performance lever in the service. The naive pattern
4
+ calls OCR once per extraction key, so a 10-key request pays for parsing ten
5
+ times. Here, parsing happens exactly once per document and the result is cached
6
+ by content hash, so keys 2..N are effectively free on the parse side.
7
+
8
+ Two paths, chosen automatically:
9
+
10
+ * PDF with a text layer -> pdfplumber. Milliseconds, no OCR, exact characters.
11
+ * PDF without a text layer, or an image -> rasterise then Tesseract OCR.
12
+
13
+ Both paths emit the same normalised representation: a list of words and a list
14
+ of ``[x0, y0, x1, y1]`` boxes scaled to 0..1000, which is what LayoutLM expects.
15
+ """
16
+
17
+ from __future__ import annotations
18
+
19
+ import hashlib
20
+ import io
21
+ import threading
22
+ from collections import OrderedDict
23
+ from dataclasses import dataclass
24
+
25
+ from .config import Settings
26
+ from .logging import get_logger
27
+ from .schema import (CorruptDocumentError, DocExtractError, SourceType,
28
+ UnsupportedMediaError)
29
+
30
+ log = get_logger(__name__)
31
+
32
+ PDF_MAGIC = b"%PDF-"
33
+ IMAGE_MAGICS: tuple[tuple[bytes, str], ...] = (
34
+ (b"\x89PNG\r\n\x1a\n", "png"),
35
+ (b"\xff\xd8\xff", "jpeg"),
36
+ (b"GIF87a", "gif"),
37
+ (b"GIF89a", "gif"),
38
+ (b"BM", "bmp"),
39
+ (b"II*\x00", "tiff"),
40
+ (b"MM\x00*", "tiff"),
41
+ )
42
+
43
+
44
+ @dataclass(frozen=True, slots=True)
45
+ class Page:
46
+ words: tuple[str, ...]
47
+ boxes: tuple[tuple[int, int, int, int], ...]
48
+
49
+
50
+ @dataclass(frozen=True, slots=True)
51
+ class ParsedDocument:
52
+ document_id: str
53
+ pages: tuple[Page, ...]
54
+ source_type: SourceType
55
+ page_count: int
56
+ truncated: bool
57
+
58
+ @property
59
+ def word_count(self) -> int:
60
+ return sum(len(p.words) for p in self.pages)
61
+
62
+
63
+ def tesseract_available() -> bool:
64
+ """Whether the Tesseract binary can actually be executed.
65
+
66
+ Images and scanned PDFs depend on it. Probing explicitly lets the API
67
+ return a clear 503 instead of an opaque pytesseract traceback, and lets
68
+ tests skip rather than fail for an environment reason.
69
+ """
70
+ try:
71
+ import pytesseract
72
+ pytesseract.get_tesseract_version()
73
+ return True
74
+ except Exception: # noqa: BLE001
75
+ return False
76
+
77
+
78
+ def sniff(data: bytes) -> str:
79
+ if data.startswith(PDF_MAGIC):
80
+ return "pdf"
81
+ for magic, kind in IMAGE_MAGICS:
82
+ if data.startswith(magic):
83
+ return kind
84
+ return "unknown"
85
+
86
+
87
+ def _normalize(words: list[str], boxes: list[list[float]],
88
+ width: float, height: float) -> tuple[tuple[str, ...],
89
+ tuple[tuple[int, int, int, int], ...]]:
90
+ """Scale pixel boxes to LayoutLM's 0..1000 grid and clamp.
91
+
92
+ LayoutLM indexes learned x/y position embeddings, so out-of-range values
93
+ raise an IndexError rather than degrading gracefully.
94
+ """
95
+ def clamp(v: float) -> int:
96
+ return max(0, min(1000, int(round(v))))
97
+
98
+ out_boxes = []
99
+ for b in boxes:
100
+ x0 = clamp(b[0] * 1000.0 / max(width, 1e-6))
101
+ y0 = clamp(b[1] * 1000.0 / max(height, 1e-6))
102
+ x1 = clamp(b[2] * 1000.0 / max(width, 1e-6))
103
+ y1 = clamp(b[3] * 1000.0 / max(height, 1e-6))
104
+ out_boxes.append((min(x0, x1), min(y0, y1), max(x0, x1), max(y0, y1)))
105
+ return tuple(words), tuple(out_boxes)
106
+
107
+
108
+ # --------------------------------------------------------------------------
109
+ # PDF
110
+ # --------------------------------------------------------------------------
111
+ def _parse_pdf_text_layer(data: bytes, settings: Settings) -> tuple[list[Page], int]:
112
+ """Extract words directly from the PDF text layer via pdfplumber.
113
+
114
+ Returns (pages, total_page_count). ``total_page_count`` comes from the
115
+ document, not the slice, so callers can report truncation accurately.
116
+ """
117
+ import pdfplumber
118
+
119
+ pages: list[Page] = []
120
+ with pdfplumber.open(io.BytesIO(data)) as pdf:
121
+ total = len(pdf.pages)
122
+ for page in pdf.pages[: settings.max_pages]:
123
+ raw = page.extract_words(keep_blank_chars=False, use_text_flow=False)
124
+ words = [w["text"] for w in raw]
125
+ boxes = [[w["x0"], w["top"], w["x1"], w["bottom"]] for w in raw]
126
+ pages.append(Page(*_normalize(words, boxes, page.width, page.height)))
127
+ return pages, total
128
+
129
+
130
+ def _render_pdf_pages(data: bytes, settings: Settings, dpi: int = 200):
131
+ """Rasterise PDF pages with PyMuPDF, used when no text layer exists.
132
+
133
+ DPI is reduced per page if the rendered bitmap would exceed the pixel
134
+ budget, which is how a small crafted PDF that expands to a gigapixel image
135
+ is contained.
136
+ """
137
+ import fitz # PyMuPDF
138
+ from PIL import Image
139
+
140
+ doc = fitz.open(stream=data, filetype="pdf")
141
+ images = []
142
+ for index, page in enumerate(doc):
143
+ if index >= settings.max_pages:
144
+ break
145
+ rect = page.rect
146
+ estimated = (rect.width / 72 * dpi) * (rect.height / 72 * dpi)
147
+ if estimated > settings.max_render_pixels:
148
+ dpi = max(72, int(dpi * (settings.max_render_pixels / estimated) ** 0.5))
149
+ pix = page.get_pixmap(dpi=dpi)
150
+ images.append(Image.frombytes("RGB", (pix.width, pix.height), pix.samples))
151
+ total = doc.page_count
152
+ doc.close()
153
+ return images, total
154
+
155
+
156
+ def _ocr_image(image, settings: Settings) -> Page:
157
+ """Tesseract OCR of a PIL image -> Page."""
158
+ import pytesseract
159
+ from pytesseract import Output
160
+
161
+ if not tesseract_available():
162
+ raise DocExtractError(
163
+ "OCR backend unavailable",
164
+ detail="the Tesseract binary is required for images and scanned "
165
+ "PDFs. Install it and ensure `tesseract` is on PATH "
166
+ "(Windows: winget install UB-Mannheim.TesseractOCR).",
167
+ )
168
+ width, height = image.size
169
+ if width * height > settings.max_render_pixels:
170
+ log.warning("image exceeds pixel budget", extra={"detail": "downscale"})
171
+ scale = (settings.max_render_pixels / (width * height)) ** 0.5
172
+ image = image.resize((max(1, int(width * scale)),
173
+ max(1, int(height * scale))))
174
+ width, height = image.size
175
+
176
+ data = pytesseract.image_to_data(image, lang="eng", output_type=Output.DICT)
177
+ words: list[str] = []
178
+ boxes: list[list[float]] = []
179
+ for i, text in enumerate(data["text"]):
180
+ if not text.strip():
181
+ continue
182
+ x, y = data["left"][i], data["top"][i]
183
+ w, h = data["width"][i], data["height"][i]
184
+ words.append(text)
185
+ boxes.append([x, y, x + w, y + h])
186
+ return Page(*_normalize(words, boxes, width, height))
187
+
188
+
189
+ def _parse_pdf(data: bytes, settings: Settings) -> tuple[list[Page], int, SourceType]:
190
+ try:
191
+ pages, total = _parse_pdf_text_layer(data, settings)
192
+ except Exception as exc: # noqa: BLE001
193
+ raise CorruptDocumentError("PDF could not be parsed", detail=str(exc)) from exc
194
+
195
+ meaningful = [p for p in pages if p.words]
196
+ if meaningful:
197
+ return pages, total, SourceType.PDF_TEXT_LAYER
198
+
199
+ # No text layer: a scanned document. Rasterise and OCR.
200
+ log.info("no text layer detected, falling back to OCR")
201
+ try:
202
+ images, total = _render_pdf_pages(data, settings)
203
+ except Exception as exc: # noqa: BLE001
204
+ raise CorruptDocumentError("PDF could not be rendered", detail=str(exc)) from exc
205
+ return [_ocr_image(im, settings) for im in images], total, SourceType.PDF_OCR
206
+
207
+
208
+ def _parse_image(data: bytes, settings: Settings) -> tuple[list[Page], int, SourceType]:
209
+ from PIL import Image, UnidentifiedImageError
210
+
211
+ try:
212
+ image = Image.open(io.BytesIO(data))
213
+ image.load() # force decode so truncation is caught here
214
+ image = image.convert("RGB")
215
+ except (UnidentifiedImageError, OSError) as exc:
216
+ raise CorruptDocumentError("image could not be decoded", detail=str(exc)) from exc
217
+ return [_ocr_image(image, settings)], 1, SourceType.IMAGE_OCR
218
+
219
+
220
+ # --------------------------------------------------------------------------
221
+ # cache
222
+ # --------------------------------------------------------------------------
223
+ class ParseCache:
224
+ """Small LRU keyed by content hash.
225
+
226
+ Thread-safe because FastAPI serves requests from a thread pool and the
227
+ same document is commonly re-submitted across keys in quick succession.
228
+ """
229
+
230
+ def __init__(self, maxsize: int):
231
+ self._data: OrderedDict[str, ParsedDocument] = OrderedDict()
232
+ self._lock = threading.Lock()
233
+ self._maxsize = maxsize
234
+ self.hits = 0
235
+ self.misses = 0
236
+
237
+ def get(self, key: str) -> ParsedDocument | None:
238
+ if self._maxsize == 0:
239
+ return None
240
+ with self._lock:
241
+ if key in self._data:
242
+ self._data.move_to_end(key)
243
+ self.hits += 1
244
+ return self._data[key]
245
+ self.misses += 1
246
+ return None
247
+
248
+ def put(self, key: str, value: ParsedDocument) -> None:
249
+ if self._maxsize == 0:
250
+ return
251
+ with self._lock:
252
+ self._data[key] = value
253
+ self._data.move_to_end(key)
254
+ while len(self._data) > self._maxsize:
255
+ self._data.popitem(last=False)
256
+
257
+ def clear(self) -> None:
258
+ with self._lock:
259
+ self._data.clear()
260
+
261
+
262
+ _cache = ParseCache(128)
263
+
264
+
265
+ def configure_cache(settings: Settings) -> None:
266
+ global _cache
267
+ _cache = ParseCache(settings.parse_cache_size)
268
+
269
+
270
+ def parse(data: bytes, settings: Settings) -> ParsedDocument:
271
+ """Parse document bytes into per-page words and boxes, with caching."""
272
+ if not data:
273
+ raise CorruptDocumentError("empty upload")
274
+
275
+ kind = sniff(data)
276
+ if kind == "pdf":
277
+ parse_fn = _parse_pdf
278
+ elif kind == "unknown":
279
+ raise UnsupportedMediaError(
280
+ "unsupported file type",
281
+ detail="expected a PDF or a PNG/JPEG/TIFF/BMP image",
282
+ )
283
+ else:
284
+ parse_fn = _parse_image
285
+
286
+ # Cache key is content-addressed, so the same document uploaded twice
287
+ # parses once. Tesseract on a dense page costs ~200ms; this removes it from
288
+ # every request after the first.
289
+ document_id = hashlib.sha256(data).hexdigest()[:16]
290
+ cached = _cache.get(document_id)
291
+ if cached is not None:
292
+ return cached
293
+
294
+ pages, page_count, source_type = parse_fn(data, settings)
295
+
296
+ doc = ParsedDocument(
297
+ document_id=document_id,
298
+ pages=tuple(pages),
299
+ source_type=source_type,
300
+ page_count=page_count,
301
+ truncated=page_count > len(pages),
302
+ )
303
+ _cache.put(document_id, doc)
304
+ log.info("parsed document", extra={
305
+ "document_id": document_id,
306
+ "detail": f"{doc.word_count} words, {len(pages)}/{page_count} pages, "
307
+ f"{source_type.value}",
308
+ })
309
+ return doc
310
+
311
+
312
+ def cache_stats() -> dict:
313
+ return {"hits": _cache.hits, "misses": _cache.misses,
314
+ "size": len(_cache._data), "maxsize": _cache._maxsize}
docqa/docxextract/schema.py ADDED
@@ -0,0 +1,135 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """Typed request/response contracts and the error hierarchy."""
2
+
3
+ from __future__ import annotations
4
+
5
+ import enum
6
+ from typing import Any
7
+
8
+ from pydantic import BaseModel, Field, field_validator
9
+
10
+
11
+ class FieldStatus(str, enum.Enum):
12
+ """Outcome of a single key extraction.
13
+
14
+ `EXTRACTED` is only assigned above the confidence threshold; anything else
15
+ is surfaced to the caller rather than being silently returned as fact.
16
+ This distinction is the whole point of the service: the underlying model
17
+ will always emit *some* span, so absence must be modelled explicitly.
18
+ """
19
+
20
+ EXTRACTED = "extracted"
21
+ LOW_CONFIDENCE = "low_confidence"
22
+ NOT_FOUND = "not_found"
23
+ ERROR = "error"
24
+
25
+
26
+ class SourceType(str, enum.Enum):
27
+ PDF_TEXT_LAYER = "pdf_text_layer"
28
+ PDF_OCR = "pdf_ocr"
29
+ IMAGE_OCR = "image_ocr"
30
+
31
+
32
+ class ExtractedField(BaseModel):
33
+ key: str
34
+ value: str | None = None
35
+ status: FieldStatus = FieldStatus.NOT_FOUND
36
+ confidence: float = Field(default=0.0, ge=0.0, le=1.0)
37
+ page: int | None = None
38
+ source: SourceType | None = None
39
+ error: str | None = None
40
+
41
+ @property
42
+ def is_usable(self) -> bool:
43
+ return self.status is FieldStatus.EXTRACTED
44
+
45
+
46
+ class DocumentInfo(BaseModel):
47
+ document_id: str
48
+ page_count: int
49
+ pages_processed: int
50
+ word_count: int
51
+ source_type: SourceType
52
+ truncated: bool = False
53
+
54
+
55
+ class ExtractRequest(BaseModel):
56
+ """Request body. `file` arrives separately as an UploadFile."""
57
+
58
+ keys: list[str] = Field(..., min_length=1)
59
+ confidence_threshold: float | None = Field(default=None, ge=0.0, le=1.0)
60
+ max_pages: int | None = Field(default=None, ge=1, le=50)
61
+
62
+ @field_validator("keys")
63
+ @classmethod
64
+ def _validate_keys(cls, v: list[str]) -> list[str]:
65
+ cleaned = [k.strip() for k in v if k and k.strip()]
66
+ if not cleaned:
67
+ raise ValueError("at least one non-empty key is required")
68
+ if any(len(k) > 200 for k in cleaned):
69
+ raise ValueError("each key must be 200 characters or fewer")
70
+ return cleaned
71
+
72
+ @field_validator("keys")
73
+ @classmethod
74
+ def _dedupe(cls, v: list[str]) -> list[str]:
75
+ """Preserve order but drop exact duplicates.
76
+
77
+ Repeating a key costs a full forward pass and returns an identical
78
+ answer, so duplicates are collapsed rather than rejected.
79
+ """
80
+ seen: set[str] = set()
81
+ out: list[str] = []
82
+ for k in v:
83
+ if k not in seen:
84
+ seen.add(k)
85
+ out.append(k)
86
+ return out
87
+
88
+
89
+ class ExtractResponse(BaseModel):
90
+ document: DocumentInfo
91
+ fields: list[ExtractedField]
92
+ latency_ms: float
93
+ confidence_threshold: float
94
+
95
+ @property
96
+ def values(self) -> dict[str, str | None]:
97
+ return {f.key: f.value for f in self.fields}
98
+
99
+
100
+ # --------------------------------------------------------------------------
101
+ # errors
102
+ # --------------------------------------------------------------------------
103
+ class DocExtractError(Exception):
104
+ """Base class. `status_code` is what the HTTP layer will return."""
105
+
106
+ status_code = 500
107
+
108
+ def __init__(self, message: str, detail: Any = None):
109
+ super().__init__(message)
110
+ self.message = message
111
+ self.detail = detail
112
+
113
+
114
+ class ValidationError(DocExtractError):
115
+ status_code = 422
116
+
117
+
118
+ class UnsupportedMediaError(DocExtractError):
119
+ status_code = 415
120
+
121
+
122
+ class PayloadTooLargeError(DocExtractError):
123
+ status_code = 413
124
+
125
+
126
+ class CorruptDocumentError(DocExtractError):
127
+ status_code = 422
128
+
129
+
130
+ class ExtractionTimeoutError(DocExtractError):
131
+ status_code = 504
132
+
133
+
134
+ class ModelUnavailableError(DocExtractError):
135
+ status_code = 503
docqa/docxextract/service.py ADDED
@@ -0,0 +1,137 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """Request orchestration: bytes + keys -> structured response.
2
+
3
+ This layer owns the parts that decide whether the service is *trustworthy*
4
+ rather than merely functional:
5
+
6
+ * parse once, extract many
7
+ * refuse to present a low-confidence span as a fact
8
+ * treat an empty document as an error, not as "no fields found"
9
+ * bound everything: bytes, keys, pages, wall-clock time
10
+ """
11
+
12
+ from __future__ import annotations
13
+
14
+ import time
15
+
16
+ from . import parsing
17
+ from .config import Settings, get_settings
18
+ from .engine import Engine, Span, get_engine
19
+ from .logging import get_logger
20
+ from .schema import (DocumentInfo, ExtractedField, ExtractResponse, FieldStatus,
21
+ PayloadTooLargeError, ValidationError)
22
+
23
+ log = get_logger(__name__)
24
+
25
+
26
+ class ExtractionService:
27
+ def __init__(self, settings: Settings | None = None,
28
+ engine: Engine | None = None):
29
+ self.settings = settings or get_settings()
30
+ self.engine = engine or get_engine(self.settings)
31
+
32
+ # ------------------------------------------------------------------
33
+ def _status_for(self, span: Span, threshold: float,
34
+ page_has_words: bool) -> ExtractedField:
35
+ """Map a model span onto an honest status.
36
+
37
+ The model always produces a span. Reporting it as `extracted`
38
+ regardless of confidence is how an extractive QA model ends up
39
+ confidently inventing values -- measured at a 17-21% fabrication rate on
40
+ unanswerable questions in this project's benchmark. The threshold is
41
+ the mitigation, so it is applied here, in one place.
42
+ """
43
+ if not page_has_words:
44
+ return ExtractedField(
45
+ key=span.key, value=None, status=FieldStatus.NOT_FOUND,
46
+ confidence=span.confidence, page=span.page,
47
+ )
48
+ if not span.answer:
49
+ return ExtractedField(
50
+ key=span.key, value=None, status=FieldStatus.NOT_FOUND,
51
+ confidence=span.confidence, page=span.page,
52
+ )
53
+ if span.confidence < threshold:
54
+ return ExtractedField(
55
+ key=span.key, value=span.answer,
56
+ status=FieldStatus.LOW_CONFIDENCE,
57
+ confidence=span.confidence, page=span.page,
58
+ )
59
+ return ExtractedField(
60
+ key=span.key, value=span.answer, status=FieldStatus.EXTRACTED,
61
+ confidence=span.confidence, page=span.page,
62
+ )
63
+
64
+ # ------------------------------------------------------------------
65
+ def extract(self, data: bytes, keys: list[str],
66
+ confidence_threshold: float | None = None,
67
+ max_pages: int | None = None) -> ExtractResponse:
68
+ started = time.perf_counter()
69
+
70
+ if not keys:
71
+ raise ValidationError("at least one key is required")
72
+
73
+ # Deduplicate here rather than only in the JSON request model: a
74
+ # repeated key costs a full forward pass and returns an identical
75
+ # answer. Applied at the service layer it covers both transports.
76
+ seen: set[str] = set()
77
+ unique_keys: list[str] = []
78
+ for key in keys:
79
+ cleaned = key.strip()
80
+ if not cleaned:
81
+ continue
82
+ if cleaned not in seen:
83
+ seen.add(cleaned)
84
+ unique_keys.append(cleaned)
85
+ if not unique_keys:
86
+ raise ValidationError("at least one non-empty key is required")
87
+ keys = unique_keys
88
+
89
+ if len(keys) > self.settings.max_keys:
90
+ raise ValidationError(
91
+ f"too many keys: {len(keys)} > {self.settings.max_keys}")
92
+ max_bytes = self.settings.max_upload_mb * 1024 * 1024
93
+ if len(data) > max_bytes:
94
+ raise PayloadTooLargeError(
95
+ f"document exceeds {self.settings.max_upload_mb} MB")
96
+
97
+ threshold = (confidence_threshold
98
+ if confidence_threshold is not None
99
+ else self.settings.confidence_threshold)
100
+ pages_limit = max_pages or self.settings.max_pages
101
+
102
+ # Reject documents with no extractable text before spending any
103
+ # inference time on them.
104
+ doc = parsing.parse(data, self.settings)
105
+ if doc.word_count == 0:
106
+ raise ValidationError(
107
+ "no text could be extracted from the document",
108
+ detail="the file may be empty, blank, or an unsupported scan",
109
+ )
110
+
111
+ self.engine.warmup()
112
+ spans = self.engine.extract(doc.pages, keys, pages_limit)
113
+
114
+ any_words = any(p.words for p in doc.pages[:pages_limit])
115
+ fields = [self._status_for(s, threshold, any_words) for s in spans]
116
+
117
+ latency_ms = (time.perf_counter() - started) * 1000
118
+ log.info("extraction complete", extra={
119
+ "document_id": doc.document_id,
120
+ "latency_ms": round(latency_ms, 2),
121
+ "n_keys": len(keys),
122
+ "status": sum(1 for f in fields if f.is_usable),
123
+ })
124
+
125
+ return ExtractResponse(
126
+ document=DocumentInfo(
127
+ document_id=doc.document_id,
128
+ page_count=doc.page_count,
129
+ pages_processed=len(doc.pages[:pages_limit]),
130
+ word_count=doc.word_count,
131
+ source_type=doc.source_type,
132
+ truncated=doc.truncated,
133
+ ),
134
+ fields=fields,
135
+ latency_ms=round(latency_ms, 2),
136
+ confidence_threshold=threshold,
137
+ )
docqa/tests/conftest.py ADDED
@@ -0,0 +1,47 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """Shared pytest fixtures."""
2
+
3
+ from __future__ import annotations
4
+
5
+ import json
6
+ import sys
7
+ from pathlib import Path
8
+
9
+ import pytest
10
+
11
+ ROOT = Path(__file__).resolve().parent.parent
12
+ sys.path.insert(0, str(ROOT / "src"))
13
+
14
+ CORPUS = ROOT / "corpus"
15
+
16
+
17
+ @pytest.fixture(scope="session")
18
+ def corpus_dir() -> Path:
19
+ if not (CORPUS / "ground_truth.json").exists():
20
+ pytest.skip("corpus not generated; run: python tests/corpus.py corpus 12")
21
+ return CORPUS
22
+
23
+
24
+ @pytest.fixture(scope="session")
25
+ def ground_truth(corpus_dir: Path) -> list[dict]:
26
+ return json.loads((corpus_dir / "ground_truth.json").read_text())
27
+
28
+
29
+ @pytest.fixture(scope="session")
30
+ def settings():
31
+ from docxextract.config import Settings
32
+ return Settings(torch_threads=4, parse_cache_size=32)
33
+
34
+
35
+ @pytest.fixture(scope="session")
36
+ def sample_pdf(ground_truth) -> bytes:
37
+ return Path(ground_truth[0]["path"]).read_bytes()
38
+
39
+
40
+ @pytest.fixture(scope="session")
41
+ def engine(settings):
42
+ """Session-scoped warmed engine. Loading a model per test is too slow."""
43
+ from docxextract.engine import Engine, reset_engine
44
+ eng = Engine(settings)
45
+ eng.warmup()
46
+ yield eng
47
+ reset_engine()
docqa/tests/corpus.py ADDED
@@ -0,0 +1,416 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """Synthetic invoice/receipt corpus with verified ground truth.
2
+
3
+ Ground truth is written into the PDF itself, so every expected value is known
4
+ exactly without human labelling. Layout variety is deliberate: these models are
5
+ layout-aware, so a single tidy template would overstate accuracy.
6
+
7
+ Vendors carry both US (EIN) and Indian (GSTIN) identifiers because the target
8
+ extraction keys in the brief include `GSTIN` and `Vendor Id`.
9
+ """
10
+
11
+ from __future__ import annotations
12
+
13
+ import json
14
+ import random
15
+ from dataclasses import asdict, dataclass, field
16
+ from pathlib import Path
17
+
18
+ from reportlab.lib import colors
19
+ from reportlab.lib.pagesizes import A4, LETTER
20
+ from reportlab.lib.styles import ParagraphStyle, getSampleStyleSheet
21
+ from reportlab.lib.units import mm
22
+ from reportlab.platypus import (BaseDocTemplate, Frame, PageTemplate, Paragraph,
23
+ Spacer, Table, TableStyle)
24
+
25
+ # --------------------------------------------------------------------------
26
+ VENDORS = [
27
+ dict(name="Northwind Logistics LLC",
28
+ address="4820 Harborview Parkway, Suite 300, Seattle, WA 98109",
29
+ country="US", tax_id="88-4122371", gstin="27AAPFU0939F1ZV",
30
+ vat="VAT: US-8841-223"),
31
+ dict(name="Contoso Manufacturing Pvt Ltd",
32
+ address="1180 Industrial Loop, Building 7, Austin, TX 78744",
33
+ country="IN", tax_id="74-2930118", gstin="29AAGCC7391J1Z2",
34
+ vat="GSTIN: 29AAGCC7391J1Z2"),
35
+ dict(name="Fabrikam Consulting Group",
36
+ address="915 Ocean Drive, Floor 12, Miami, FL 33139",
37
+ country="US", tax_id="55-9014423", gstin="07AAACF9876K1ZQ",
38
+ vat="VAT: US-5512-009"),
39
+ dict(name="Tailspin Tooling Co.",
40
+ address="77 Ironworks Road, Detroit, MI 48226",
41
+ country="US", tax_id="38-2210547", gstin="36AABCT1234M1ZP",
42
+ vat="Tax ID: 38-2210547"),
43
+ dict(name="Adventure Works Supply",
44
+ address="3301 Market Street, San Francisco, CA 94114",
45
+ country="US", tax_id="94-3301876", gstin="33AACCA9876T1Z9",
46
+ vat="EIN: 94-3301876"),
47
+ dict(name="Litware Cloud Services",
48
+ address="2200 Telegraph Avenue, Denver, CO 80202",
49
+ country="US", tax_id="91-2200981", gstin="19AAHCL4321K1ZR",
50
+ vat="VAT: US-7719-556"),
51
+ dict(name="Proseware Data Systems",
52
+ address="58 Birchwood Court, Boston, MA 02110",
53
+ country="IN", tax_id="04-7123309", gstin="27AABPD5678L1ZB",
54
+ vat="GSTIN: 27AABPD5678L1ZB"),
55
+ dict(name="Wide World Importers",
56
+ address="9910 Commerce Way, Seattle, WA 98108",
57
+ country="IN", tax_id="55-1102874", gstin="24AACCW8765M1ZN",
58
+ vat="GSTIN: 24AACCW8765M1ZN"),
59
+ ]
60
+
61
+ PEOPLE = [
62
+ ("Dana Whitfield", "Accounts Payable Manager"),
63
+ ("Marcus Oyelaran", "Procurement Lead"),
64
+ ("Priya Raghunathan", "Finance Director"),
65
+ ("Tomas Lindqvist", "Operations Director"),
66
+ ("Alicia Moreau", "Controller"),
67
+ ("Ravi Deshpande", "Purchasing Analyst"),
68
+ ]
69
+
70
+ ITEMS = [
71
+ ("Warehouse management services", "WMS-ENT", 2450.00),
72
+ ("Pallet racking, steel, 96in", "RCK-96", 189.50),
73
+ ("Forklift rental, monthly", "FLT-MTH", 1150.00),
74
+ ("Customs brokerage service", "CUS-BRK", 210.00),
75
+ ("Data migration consulting", "CONS-MIG", 195.00),
76
+ ("Annual support renewal", "SUP-ANN", 8800.00),
77
+ ("Freight forwarding, LTL", "FRT-LTL", 342.75),
78
+ ("Cold chain packaging supplies", "PKG-CC", 3.20),
79
+ ("Safety compliance audit", "AUD-SFT", 1450.00),
80
+ ("API integration development", "DEV-API", 168.00),
81
+ ("Diesel fuel surcharge", "SUR-DSL", 612.40),
82
+ ("Warehouse safety training", "TRN-SFT", 275.00),
83
+ ]
84
+
85
+ TERMS = [
86
+ ("Net 30", "Payment due within 30 days of invoice date."),
87
+ ("Net 45", "Payment due within 45 days of invoice date."),
88
+ ("Net 15", "Payment due within 15 days of invoice date."),
89
+ ("Due on receipt", "Payment due immediately upon receipt."),
90
+ ("Net 60", "Payment due within 60 days of invoice date."),
91
+ ]
92
+
93
+ TAXES = [("Sales Tax 8.5%", 0.085), ("VAT 20%", 0.20),
94
+ ("GST 18%", 0.18), ("Sales Tax 6.25%", 0.0625), ("IGST 5%", 0.05)]
95
+ CURRENCIES = ["USD", "USD", "USD", "EUR", "INR"]
96
+ DATES = ["January 12, 2025", "February 3, 2025", "March 18, 2025",
97
+ "April 7, 2025", "May 22, 2025", "June 30, 2025",
98
+ "August 15, 2025", "September 2, 2025", "October 9, 2025",
99
+ "November 21, 2025", "December 5, 2025"]
100
+ DUE_DATES = ["February 11, 2025", "March 20, 2025", "May 5, 2025",
101
+ "June 14, 2025", "August 1, 2025", "September 30, 2025",
102
+ "December 4, 2025", "January 23, 2026"]
103
+
104
+ SYM = {"USD": "$", "EUR": "\u20ac", "INR": "\u20b9"}
105
+
106
+
107
+ def money(value: float, currency: str = "USD") -> str:
108
+ return f"{SYM.get(currency, '')}{value:,.2f}"
109
+
110
+
111
+ @dataclass
112
+ class InvoiceTruth:
113
+ """Every value written onto the rendered invoice."""
114
+
115
+ document_id: str
116
+ path: str
117
+ variant: str
118
+ invoice_number: str
119
+ invoice_date: str
120
+ due_date: str
121
+ po_number: str
122
+ payment_terms: str
123
+ currency: str
124
+ vendor_name: str
125
+ vendor_address: str
126
+ vendor_tax_id: str
127
+ gstin: str
128
+ bill_to: str
129
+ subtotal: str
130
+ tax_amount: str
131
+ total: str
132
+ tax_label: str
133
+ page_count: int = 1
134
+ extra: dict = field(default_factory=dict)
135
+
136
+
137
+ class InvoiceBuilder:
138
+ def __init__(self, seed: int, out: Path, variant: str,
139
+ currency: str = "USD", pages: int = 1):
140
+ self.rng = random.Random(seed)
141
+ self.out = out
142
+ self.variant = variant
143
+ self.currency = currency
144
+ self.pages = pages
145
+ self.vendor = self.rng.choice(VENDORS)
146
+ self.bill_to = self.rng.choice([v for v in VENDORS
147
+ if v["name"] != self.vendor["name"]])
148
+ self.contact, self.contact_title = self.rng.choice(PEOPLE)
149
+ self.po = f"PO-{self.rng.randint(10000, 99999)}"
150
+ self.invoice_no = f"INV-{self.rng.randint(100000, 999999)}"
151
+ self.date = self.rng.choice(DATES)
152
+ self.due = self.rng.choice(DUE_DATES)
153
+ self.terms, self.terms_note = self.rng.choice(TERMS)
154
+ self.tax_label, self.tax_rate = self.rng.choice(TAXES)
155
+ self.lines = [(desc, sku, self.rng.randint(1, 20), price)
156
+ for desc, sku, price in
157
+ self.rng.sample(ITEMS, self.rng.randint(3, 8))]
158
+ self.subtotal = round(sum(q * p for _, _, q, p in self.lines), 2)
159
+ self.discount_rate = self.rng.choice([0.0, 0.0, 0.0, 0.02, 0.05])
160
+ self.discount = round(self.subtotal * self.discount_rate, 2)
161
+ self.tax = round((self.subtotal - self.discount) * self.tax_rate, 2)
162
+ self.total = round(self.subtotal - self.discount + self.tax, 2)
163
+
164
+ # ------------------------------------------------------------------
165
+ def build(self) -> InvoiceTruth:
166
+ s = getSampleStyleSheet()
167
+ small = ParagraphStyle("sm", parent=s["BodyText"], fontSize=8.5, leading=11)
168
+ tiny = ParagraphStyle("ti", parent=s["BodyText"], fontSize=6.5, leading=8)
169
+ h1 = ParagraphStyle("h1", parent=s["Title"], fontSize=19, leading=23,
170
+ textColor=colors.HexColor("#1a1a2e"))
171
+ h2 = ParagraphStyle("h2", parent=s["Heading2"], fontSize=12, leading=15,
172
+ textColor=colors.HexColor("#1a1a2e"))
173
+
174
+ pagesize = LETTER if self.variant == "letter" else A4
175
+ doc = BaseDocTemplate(
176
+ str(self.out), pagesize=pagesize,
177
+ leftMargin=16 * mm, rightMargin=16 * mm,
178
+ topMargin=14 * mm, bottomMargin=14 * mm,
179
+ title=f"Invoice {self.invoice_no}", author=self.vendor["name"])
180
+
181
+ frame = Frame(doc.leftMargin, doc.bottomMargin, doc.width, doc.height,
182
+ id="f")
183
+ doc.addPageTemplates([PageTemplate(id="main", frames=[frame])])
184
+
185
+ story: list = []
186
+ v = self.variant
187
+ if v == "minimal":
188
+ story += self._minimal(s)
189
+ elif v == "boxed":
190
+ story += self._boxed(s, small)
191
+ elif v == "two_column":
192
+ story += self._two_column(s)
193
+ elif v == "dense":
194
+ story += self._dense(tiny)
195
+ else:
196
+ story += self._classic(s, h1, h2, small)
197
+
198
+ story += [Spacer(1, 7 * mm), Paragraph(self.terms_note, tiny),
199
+ Paragraph(
200
+ f"Remit to: {self.vendor['name']} &mdash; "
201
+ f"{self.vendor['vat']}. Reference {self.po} with all "
202
+ f"payments. Queries to {self.contact}.", tiny)]
203
+
204
+ # extra pages force multi-page handling in the service
205
+ for p in range(1, self.pages):
206
+ story += [Spacer(1, 150 * mm),
207
+ Paragraph(f"Continued &mdash; page {p + 1}", small),
208
+ Paragraph(
209
+ f"This page continues invoice {self.invoice_no}. "
210
+ f"Total amount due remains {money(self.total, self.currency)}.",
211
+ small)]
212
+
213
+ doc.build(story)
214
+ return self.truth()
215
+
216
+ # ---------------- layouts ----------------
217
+ def _items_table(self, fs: ParagraphStyle) -> Table:
218
+ rows = [[Paragraph("<b>Description</b>", fs), Paragraph("<b>SKU</b>", fs),
219
+ Paragraph("<b>Qty</b>", fs),
220
+ Paragraph("<b>Unit Price</b>", fs),
221
+ Paragraph("<b>Amount</b>", fs)]]
222
+ for desc, sku, qty, price in self.lines:
223
+ rows.append([Paragraph(desc, fs), Paragraph(sku, fs),
224
+ Paragraph(str(qty), fs),
225
+ Paragraph(money(price, self.currency), fs),
226
+ Paragraph(money(qty * price, self.currency), fs)])
227
+ t = Table(rows, colWidths=[76 * mm, 26 * mm, 14 * mm, 30 * mm, 32 * mm])
228
+ t.setStyle(TableStyle([
229
+ ("GRID", (0, 0), (-1, -1), 0.4, colors.HexColor("#999999")),
230
+ ("BACKGROUND", (0, 0), (-1, 0), colors.HexColor("#eeeeee")),
231
+ ("VALIGN", (0, 0), (-1, -1), "MIDDLE"),
232
+ ]))
233
+ return t
234
+
235
+ def _totals(self, fs: ParagraphStyle) -> Table:
236
+ rows = [["Subtotal", money(self.subtotal, self.currency)]]
237
+ if self.discount:
238
+ rows.append([f"Discount ({self.discount_rate*100:.0f}%)",
239
+ f"-{money(self.discount, self.currency)}"])
240
+ rows += [[self.tax_label, money(self.tax, self.currency)],
241
+ ["Total Amount Due", money(self.total, self.currency)]]
242
+ t = Table(rows, colWidths=[60 * mm, 42 * mm], hAlign="RIGHT")
243
+ t.setStyle(TableStyle([
244
+ ("FONTNAME", (0, -1), (-1, -1), "Helvetica-Bold"),
245
+ ("LINEABOVE", (0, -1), (-1, -1), 0.8, colors.black),
246
+ ("FONTSIZE", (0, 0), (-1, -1), 9.5),
247
+ ("ALIGN", (1, 0), (1, -1), "RIGHT"),
248
+ ]))
249
+ return t
250
+
251
+ def _identity(self, fs: ParagraphStyle) -> str:
252
+ return (f"<b>{self.vendor['name']}</b><br/>{self.vendor['address']}"
253
+ f"<br/>{self.vendor['vat']}<br/>GSTIN: {self.vendor['gstin']}")
254
+
255
+ def _classic(self, s, h1, h2, small):
256
+ left = Paragraph(self._identity(small), small)
257
+ right = Paragraph(
258
+ f"<b>INVOICE</b><br/>Invoice Number: {self.invoice_no}<br/>"
259
+ f"Invoice Date: {self.date}<br/>Due Date: {self.due}", small)
260
+ head = Table([[left, right]], colWidths=[89 * mm, 89 * mm])
261
+ head.setStyle(TableStyle([("VALIGN", (0, 0), (-1, -1), "TOP")]))
262
+
263
+ meta = Table([
264
+ ["Bill To:", f"{self.bill_to['name']}<br/>{self.bill_to['address']}"],
265
+ ["Attention:", f"{self.contact}, {self.contact_title}"],
266
+ ["PO Number:", self.po],
267
+ ["Terms:", self.terms],
268
+ ["Currency:", self.currency],
269
+ ["Vendor Id:", self.vendor["tax_id"]],
270
+ ], colWidths=[32 * mm, 120 * mm])
271
+ meta.setStyle(TableStyle([
272
+ ("FONTNAME", (0, 0), (0, -1), "Helvetica-Bold"),
273
+ ("FONTSIZE", (0, 0), (-1, -1), 9.5),
274
+ ("VALIGN", (0, 0), (-1, -1), "TOP"),
275
+ ("BOTTOMPADDING", (0, 0), (-1, -1), 3),
276
+ ]))
277
+ return [head, Spacer(1, 5 * mm), meta, Spacer(1, 5 * mm),
278
+ Paragraph("Line Items", h2), Spacer(1, 2 * mm),
279
+ self._items_table(small), Spacer(1, 4 * mm),
280
+ self._totals(small)]
281
+
282
+ def _minimal(self, s):
283
+ small = s["BodyText"]
284
+ info = Table([
285
+ [Paragraph(f"<b>Invoice No.</b> {self.invoice_no}", small),
286
+ Paragraph(f"<b>Date</b> {self.date}", small),
287
+ Paragraph(f"<b>Due</b> {self.due}", small)],
288
+ [Paragraph(f"<b>Bill to</b> {self.bill_to['name']}", small),
289
+ Paragraph(f"<b>PO</b> {self.po}", small),
290
+ Paragraph(f"<b>Terms</b> {self.terms}", small)],
291
+ [Paragraph(f"<b>Vendor</b> {self.vendor['name']}", small),
292
+ Paragraph(f"<b>Vendor Id</b> {self.vendor['tax_id']}", small),
293
+ Paragraph(f"<b>GSTIN</b> {self.vendor['gstin']}", small)],
294
+ ], colWidths=[62 * mm, 58 * mm, 58 * mm])
295
+ info.setStyle(TableStyle([("TOPPADDING", (0, 0), (-1, -1), 4),
296
+ ("BOTTOMPADDING", (0, 0), (-1, -1), 4)]))
297
+ return [Spacer(1, 8 * mm), Paragraph(self.vendor["name"], h1_small(s)),
298
+ Paragraph(self.vendor["address"], small), Spacer(1, 6 * mm),
299
+ info, Spacer(1, 5 * mm), self._items_table(small),
300
+ Spacer(1, 4 * mm), self._totals(small)]
301
+
302
+ def _boxed(self, s, small):
303
+ head = Table([[Paragraph(self._identity(small), small),
304
+ Paragraph(f"<b>INVOICE {self.invoice_no}</b><br/>"
305
+ f"Date: {self.date}<br/>Due: {self.due}", small)]],
306
+ colWidths=[89 * mm, 89 * mm])
307
+ head.setStyle(TableStyle([("VALIGN", (0, 0), (-1, -1), "TOP")]))
308
+ box = Table([[Paragraph(
309
+ f"<b>SHIP TO / BILL TO</b><br/>{self.bill_to['name']}<br/>"
310
+ f"{self.bill_to['address']}<br/>Attn: {self.contact}, "
311
+ f"{self.contact_title}<br/>PO: {self.po}<br/>"
312
+ f"Currency: {self.currency}", small)]], colWidths=[90 * mm])
313
+ box.setStyle(TableStyle([
314
+ ("BOX", (0, 0), (-1, -1), 1.2, colors.HexColor("#1a1a2e")),
315
+ ("BACKGROUND", (0, 0), (-1, -1), colors.HexColor("#f4f4f8")),
316
+ ("LEFTPADDING", (0, 0), (-1, -1), 8),
317
+ ("TOPPADDING", (0, 0), (-1, -1), 6),
318
+ ("BOTTOMPADDING", (0, 0), (-1, -1), 6),
319
+ ]))
320
+ return [head, Spacer(1, 5 * mm), box, Spacer(1, 5 * mm),
321
+ self._items_table(small), Spacer(1, 4 * mm),
322
+ self._totals(small)]
323
+
324
+ def _two_column(self, s):
325
+ small = s["BodyText"]
326
+ left = [Paragraph(f"<b>{self.vendor['name']}</b>", small),
327
+ Paragraph(self.vendor["address"], small),
328
+ Paragraph(self.vendor["vat"], small),
329
+ Paragraph(f"GSTIN: {self.vendor['gstin']}", small),
330
+ Paragraph(f"Vendor Id: {self.vendor['tax_id']}", small),
331
+ Spacer(1, 4 * mm),
332
+ Paragraph(f"<b>Bill To</b><br/>{self.bill_to['name']}<br/>"
333
+ f"{self.bill_to['address']}", small),
334
+ Paragraph(f"Attn: {self.contact}", small)]
335
+ right = [Paragraph(f"<b>INVOICE {self.invoice_no}</b>", small),
336
+ Paragraph(f"Date: {self.date}<br/>Due: {self.due}<br/>"
337
+ f"PO: {self.po}<br/>Terms: {self.terms}<br/>"
338
+ f"Currency: {self.currency}", small)]
339
+ t = Table([[left, right]], colWidths=[89 * mm, 89 * mm])
340
+ t.setStyle(TableStyle([("VALIGN", (0, 0), (-1, -1), "TOP")]))
341
+ return [t, Spacer(1, 5 * mm), self._items_table(small),
342
+ Spacer(1, 4 * mm), self._totals(small)]
343
+
344
+ def _dense(self, tiny):
345
+ rows = []
346
+ for i in range(26):
347
+ desc, sku, price = ITEMS[i % len(ITEMS)]
348
+ qty = (i % 9) + 1
349
+ rows.append([Paragraph(desc, tiny), Paragraph(sku, tiny),
350
+ Paragraph(str(qty), tiny),
351
+ Paragraph(money(price, self.currency), tiny),
352
+ Paragraph(money(qty * price, self.currency), tiny)])
353
+ rows.insert(0, [Paragraph(
354
+ f"<b>{self.vendor['name']}</b> &mdash; INVOICE {self.invoice_no}"
355
+ f" &mdash; Date {self.date} &mdash; Due {self.due}"
356
+ f" &mdash; PO {self.po} &mdash; Terms {self.terms}"
357
+ f" &mdash; Currency {self.currency} &mdash; Vendor Id "
358
+ f"{self.vendor['tax_id']} &mdash; GSTIN {self.vendor['gstin']}",
359
+ tiny)])
360
+ t = Table(rows, colWidths=[84 * mm, 22 * mm, 14 * mm, 28 * mm, 30 * mm])
361
+ t.setStyle(TableStyle([
362
+ ("GRID", (0, 1), (-1, -1), 0.3, colors.HexColor("#aaaaaa")),
363
+ ("BACKGROUND", (0, 0), (-1, 0), colors.HexColor("#e8e8f0")),
364
+ ]))
365
+ return [t, Spacer(1, 3 * mm), self._totals(tiny),
366
+ Paragraph(f"Bill to: {self.bill_to['name']}", tiny),
367
+ Paragraph(f"Attention: {self.contact}", tiny)]
368
+
369
+ def truth(self) -> InvoiceTruth:
370
+ return InvoiceTruth(
371
+ document_id=self.out.stem, path=str(self.out.resolve()),
372
+ variant=self.variant, invoice_number=self.invoice_no,
373
+ invoice_date=self.date, due_date=self.due, po_number=self.po,
374
+ payment_terms=self.terms, currency=self.currency,
375
+ vendor_name=self.vendor["name"], vendor_address=self.vendor["address"],
376
+ vendor_tax_id=self.vendor["tax_id"], gstin=self.vendor["gstin"],
377
+ bill_to=self.bill_to["name"],
378
+ subtotal=money(self.subtotal, self.currency),
379
+ tax_amount=money(self.tax, self.currency),
380
+ total=money(self.total, self.currency),
381
+ tax_label=self.tax_label, page_count=self.pages,
382
+ )
383
+
384
+
385
+ def h1_small(s):
386
+ st = ParagraphStyle("h1s", parent=s["Title"], fontSize=17, leading=21,
387
+ alignment=1, textColor=colors.HexColor("#1a1a2e"))
388
+ return st
389
+
390
+
391
+ VARIANTS = ["classic", "minimal", "boxed", "two_column", "dense", "letter"]
392
+
393
+
394
+ def generate(out_dir: Path, n: int = 12, seed: int = 7) -> list[InvoiceTruth]:
395
+ out_dir.mkdir(parents=True, exist_ok=True)
396
+ truths: list[InvoiceTruth] = []
397
+ for i in range(n):
398
+ variant = VARIANTS[i % len(VARIANTS)]
399
+ currency = CURRENCIES[i % len(CURRENCIES)] if variant != "dense" else "USD"
400
+ pages = 2 if i % 5 == 4 else 1
401
+ builder = InvoiceBuilder(seed=seed * 1000 + i,
402
+ out=out_dir / f"invoice_{i:02d}_{variant}.pdf",
403
+ variant=variant, currency=currency, pages=pages)
404
+ truths.append(builder.build())
405
+ manifest = out_dir / "ground_truth.json"
406
+ manifest.write_text(json.dumps([asdict(t) for t in truths], indent=2),
407
+ encoding="utf-8")
408
+ return truths
409
+
410
+
411
+ if __name__ == "__main__":
412
+ import sys
413
+ target = Path(sys.argv[1] if len(sys.argv) > 1 else "corpus")
414
+ count = int(sys.argv[2]) if len(sys.argv) > 2 else 12
415
+ rows = generate(target, count)
416
+ print(f"generated {len(rows)} invoices -> {target}")
docqa/tests/test_api.py ADDED
@@ -0,0 +1,133 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """HTTP-layer tests: validation, error mapping, and response shape.
2
+
3
+ Uses FastAPI's TestClient with the real pipeline where inference is needed, so
4
+ these exercise the actual serving path rather than a stub.
5
+ """
6
+
7
+ from __future__ import annotations
8
+
9
+ import base64
10
+
11
+ import pytest
12
+ from fastapi.testclient import TestClient
13
+
14
+ from docxextract.api import app
15
+
16
+
17
+ @pytest.fixture(scope="module")
18
+ def client():
19
+ with TestClient(app) as c:
20
+ yield c
21
+
22
+
23
+ def test_health(client):
24
+ response = client.get("/health")
25
+ assert response.status_code == 200
26
+ body = response.json()
27
+ assert body["status"] in ("ok", "degraded")
28
+ assert "model_ready" in body
29
+
30
+
31
+ def test_config(client):
32
+ body = client.get("/v1/config").json()
33
+ assert body["max_keys"] > 0
34
+ assert 0.0 <= body["confidence_threshold"] <= 1.0
35
+
36
+
37
+ def test_extract_rejects_unsupported_media(client):
38
+ response = client.post(
39
+ "/v1/extract",
40
+ files={"file": ("notes.txt", b"just text", "text/plain")},
41
+ data={"keys": "Total"},
42
+ )
43
+ assert response.status_code in (415, 422)
44
+
45
+
46
+ def test_extract_rejects_empty_keys(client, sample_pdf):
47
+ response = client.post(
48
+ "/v1/extract",
49
+ files={"file": ("invoice.pdf", sample_pdf, "application/pdf")},
50
+ data={"keys": " , , "},
51
+ )
52
+ assert response.status_code == 422
53
+
54
+
55
+ def test_extract_rejects_too_many_keys(client, sample_pdf):
56
+ response = client.post(
57
+ "/v1/extract",
58
+ files={"file": ("invoice.pdf", sample_pdf, "application/pdf")},
59
+ data={"keys": ",".join(f"key{i}" for i in range(500))},
60
+ )
61
+ assert response.status_code == 422
62
+
63
+
64
+ def test_extract_rejects_corrupt_pdf(client):
65
+ response = client.post(
66
+ "/v1/extract",
67
+ files={"file": ("broken.pdf", b"%PDF-1.7\ngarbage garbage",
68
+ "application/pdf")},
69
+ data={"keys": "Total"},
70
+ )
71
+ assert response.status_code in (422, 500)
72
+
73
+
74
+ def test_extract_success_shape(client, sample_pdf):
75
+ keys = ["INVOICE NO", "Total Amount"]
76
+ response = client.post(
77
+ "/v1/extract",
78
+ files={"file": ("invoice.pdf", sample_pdf, "application/pdf")},
79
+ data={"keys": ",".join(keys)},
80
+ )
81
+ assert response.status_code == 200
82
+ body = response.json()
83
+ assert [f["key"] for f in body["fields"]] == keys
84
+ for field in body["fields"]:
85
+ assert field["status"] in ("extracted", "low_confidence", "not_found",
86
+ "error")
87
+ assert 0.0 <= field["confidence"] <= 1.0
88
+ assert body["document"]["word_count"] > 0
89
+ assert body["latency_ms"] >= 0
90
+ assert response.headers.get("X-Request-ID")
91
+
92
+
93
+ def test_duplicate_keys_collapse(client, sample_pdf):
94
+ response = client.post(
95
+ "/v1/extract",
96
+ files={"file": ("invoice.pdf", sample_pdf, "application/pdf")},
97
+ data={"keys": "Total Amount,Total Amount"},
98
+ )
99
+ body = response.json()
100
+ assert len(body["fields"]) == 1
101
+
102
+
103
+ def test_json_endpoint(client, sample_pdf):
104
+ response = client.post("/v1/extract/json", json={
105
+ "keys": ["INVOICE NO"],
106
+ "document_base64": base64.b64encode(sample_pdf).decode(),
107
+ })
108
+ assert response.status_code == 200
109
+ assert response.json()["fields"][0]["key"] == "INVOICE NO"
110
+
111
+
112
+ def test_json_endpoint_rejects_bad_base64(client):
113
+ response = client.post("/v1/extract/json", json={
114
+ "keys": ["Total"], "document_base64": "!!!not-base64!!!"})
115
+ assert response.status_code == 422
116
+
117
+
118
+ def test_json_endpoint_requires_keys(client, sample_pdf):
119
+ response = client.post("/v1/extract/json", json={
120
+ "keys": [], "document_base64": base64.b64encode(sample_pdf).decode()})
121
+ assert response.status_code == 422
122
+
123
+
124
+ def test_confidence_threshold_is_respected(client, sample_pdf):
125
+ """A threshold of 1.0 must suppress every extraction."""
126
+ response = client.post(
127
+ "/v1/extract",
128
+ files={"file": ("invoice.pdf", sample_pdf, "application/pdf")},
129
+ data={"keys": "INVOICE NO,Total Amount", "confidence_threshold": "1.0"},
130
+ )
131
+ body = response.json()
132
+ assert body["confidence_threshold"] == 1.0
133
+ assert all(f["status"] != "extracted" for f in body["fields"])
docqa/tests/test_unit.py ADDED
@@ -0,0 +1,178 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """Unit tests for the pure, fast layers: parsing, validation, status mapping.
2
+
3
+ No model inference here, so this file runs in seconds.
4
+ """
5
+
6
+ from __future__ import annotations
7
+
8
+ import io
9
+
10
+ import pytest
11
+
12
+ from docxextract import parsing
13
+ from docxextract.schema import (ExtractRequest, FieldStatus, UnsupportedMediaError,
14
+ ValidationError)
15
+
16
+
17
+ # ---------------------------------------------------------------- media sniff
18
+ def test_sniff_pdf():
19
+ assert parsing.sniff(b"%PDF-1.7\n...") == "pdf"
20
+
21
+
22
+ @pytest.mark.parametrize("data,expected", [
23
+ (b"\x89PNG\r\n\x1a\n", "png"),
24
+ (b"\xff\xd8\xff\xe0", "jpeg"),
25
+ (b"GIF89a...", "gif"),
26
+ (b"BM....", "bmp"),
27
+ (b"not a document", "unknown"),
28
+ ])
29
+ def test_sniff_images(data, expected):
30
+ assert parsing.sniff(data) == expected
31
+
32
+
33
+ def test_rejects_unknown_type(settings):
34
+ with pytest.raises(UnsupportedMediaError):
35
+ parsing.parse(b"just some text", settings)
36
+
37
+
38
+ def test_rejects_empty(settings):
39
+ with pytest.raises(Exception):
40
+ parsing.parse(b"", settings)
41
+
42
+
43
+ # ---------------------------------------------------------------- normalisation
44
+ def test_boxes_are_clamped_to_1000(settings):
45
+ """LayoutLM indexes learned position embeddings; out-of-range raises."""
46
+ words, boxes = parsing._normalize(
47
+ ["a", "b"], [[-50, -50, 5000, 5000], [0, 0, 100, 100]], 100, 100)
48
+ assert all(0 <= v <= 1000 for b in boxes for v in b)
49
+
50
+
51
+ def test_boxes_ordered_lowercase_first():
52
+ _words, boxes = parsing._normalize(["a"], [[300, 400, 100, 200]], 1000, 1000)
53
+ x0, y0, x1, y1 = boxes[0]
54
+ assert x0 <= x1 and y0 <= y1
55
+
56
+
57
+ def test_zero_size_page_does_not_divide_by_zero():
58
+ words, boxes = parsing._normalize(["a"], [[10, 10, 20, 20]], 0, 0)
59
+ assert len(words) == 1 and len(boxes) == 1
60
+
61
+
62
+ # ---------------------------------------------------------------- parsing
63
+ def test_parses_pdf_text_layer(corpus_dir, settings, ground_truth):
64
+ from pathlib import Path
65
+ data = Path(ground_truth[0]["path"]).read_bytes()
66
+ doc = parsing.parse(data, settings)
67
+ assert doc.word_count > 50
68
+ assert doc.source_type.value == "pdf_text_layer"
69
+ assert doc.page_count >= 1
70
+
71
+
72
+ def test_parse_cache_hits_on_repeat(corpus_dir, settings, ground_truth):
73
+ from pathlib import Path
74
+ data = Path(ground_truth[0]["path"]).read_bytes()
75
+ first = parsing.parse(data, settings)
76
+ before = parsing.cache_stats()["hits"]
77
+ second = parsing.parse(data, settings)
78
+ assert second.document_id == first.document_id
79
+ assert parsing.cache_stats()["hits"] > before
80
+
81
+
82
+ def test_blank_pdf_reports_no_words(tmp_path, settings):
83
+ from reportlab.lib.pagesizes import A4
84
+ from reportlab.pdfgen import canvas
85
+
86
+ pytest.importorskip("pytesseract")
87
+ if not parsing.tesseract_available():
88
+ pytest.skip("Tesseract binary not on PATH")
89
+
90
+ path = tmp_path / "blank.pdf"
91
+ c = canvas.Canvas(str(path), pagesize=A4)
92
+ c.showPage()
93
+ c.save()
94
+ doc = parsing.parse(path.read_bytes(), settings)
95
+ # Either genuinely empty, or OCR of a blank page yields nothing usable.
96
+ assert doc.word_count < 5
97
+
98
+
99
+ def test_tesseract_probe_does_not_raise():
100
+ """The availability probe must never raise, whatever the environment."""
101
+ assert isinstance(parsing.tesseract_available(), bool)
102
+
103
+
104
+ # ---------------------------------------------------------------- validation
105
+ def test_extract_request_rejects_empty_keys():
106
+ with pytest.raises(Exception):
107
+ ExtractRequest(keys=["", " "])
108
+
109
+
110
+ def test_extract_request_deduplicates_preserving_order():
111
+ req = ExtractRequest(keys=["b", "a", "b", "c"])
112
+ assert req.keys == ["b", "a", "c"]
113
+
114
+
115
+ def test_extract_request_rejects_overlong_key():
116
+ with pytest.raises(Exception):
117
+ ExtractRequest(keys=["x" * 500])
118
+
119
+
120
+ # ---------------------------------------------------------------- status mapping
121
+ def test_status_below_threshold_is_low_confidence(settings):
122
+ from docxextract.engine import Span
123
+ from docxextract.service import ExtractionService
124
+
125
+ service = ExtractionService(settings)
126
+ span = Span(key="k", answer="$10.00", confidence=0.2, page=1, start=0, end=0)
127
+ field = service._status_for(span, threshold=0.5, page_has_words=True)
128
+ assert field.status is FieldStatus.LOW_CONFIDENCE
129
+ # The value is still returned so a human reviewer can judge it.
130
+ assert field.value == "$10.00"
131
+
132
+
133
+ def test_status_empty_answer_is_not_found(settings):
134
+ from docxextract.engine import Span
135
+ from docxextract.service import ExtractionService
136
+
137
+ service = ExtractionService(settings)
138
+ span = Span(key="k", answer="", confidence=0.99, page=1, start=-1, end=-1)
139
+ field = service._status_for(span, threshold=0.5, page_has_words=True)
140
+ assert field.status is FieldStatus.NOT_FOUND
141
+ assert field.value is None
142
+
143
+
144
+ def test_status_high_confidence_is_extracted(settings):
145
+ from docxextract.engine import Span
146
+ from docxextract.service import ExtractionService
147
+
148
+ service = ExtractionService(settings)
149
+ span = Span(key="k", answer="INV-1", confidence=0.95, page=1, start=0, end=0)
150
+ field = service._status_for(span, threshold=0.5, page_has_words=True)
151
+ assert field.status is FieldStatus.EXTRACTED
152
+ assert field.is_usable
153
+
154
+
155
+ def test_service_rejects_no_keys(settings, sample_pdf):
156
+ from docxextract.service import ExtractionService
157
+ service = ExtractionService(settings)
158
+ with pytest.raises(ValidationError):
159
+ service.extract(sample_pdf, [])
160
+
161
+
162
+ def test_service_rejects_blank_document(settings, tmp_path):
163
+ from reportlab.lib.pagesizes import A4
164
+ from reportlab.pdfgen import canvas
165
+
166
+ from docxextract.service import ExtractionService
167
+
168
+ pytest.importorskip("pytesseract")
169
+ if not parsing.tesseract_available():
170
+ pytest.skip("Tesseract binary not on PATH")
171
+
172
+ path = tmp_path / "empty.pdf"
173
+ c = canvas.Canvas(str(path), pagesize=A4)
174
+ c.showPage()
175
+ c.save()
176
+ service = ExtractionService(settings)
177
+ with pytest.raises(ValidationError):
178
+ service.extract(path.read_bytes(), ["Total"])
requirements.txt ADDED
@@ -0,0 +1,30 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # Pinned exactly. Loose pins here have broken this build more than once:
2
+ # - onnx pulls numpy 2.x, which breaks scipy and sklearn used by the
3
+ # benchmark scripts.
4
+ # - torch must come from the CPU wheel index; the default PyPI torch pulls
5
+ # multi-GB CUDA libraries and blows the Space image size limit.
6
+ --extra-index-url https://download.pytorch.org/whl/cpu
7
+
8
+ fastapi==0.115.5
9
+ uvicorn[standard]==0.34.0
10
+ pydantic==2.13.4
11
+ pydantic-settings==2.15.0
12
+ orjson==3.11.9
13
+
14
+ # model
15
+ torch==2.5.1+cpu
16
+ transformers==4.50.2
17
+ huggingface_hub>=0.26
18
+
19
+ # parsing / OCR
20
+ pdfplumber==0.11.9
21
+ pytesseract==0.3.13
22
+ Pillow==10.3.0
23
+
24
+ # numpy must stay on 1.x; see note above
25
+ numpy==1.26.4
26
+
27
+ # tests
28
+ pytest==8.3.4
29
+ httpx==0.28.1
30
+ pytest-asyncio==0.24.0