Thrisul-Parser v0.1

CONTRIBUTING.md

@@ -0,0 +1,48 @@
+# Contributing
+
+## Developer Certificate of Origin (DCO)
+
+We require that all contributors "sign-off" on their commits. This certifies that the contribution is your original
+work, or you have rights to submit it under the same license, or a compatible license.
+
+Any contribution which contains commits that are not signed off are not accepted.
+
+To sign off on a commit, use the `--signoff` (or `-s`) option when you commit your changes:
+
+```bash
+git commit --signoff --message "Add cool feature."
+```
+
+This appends the following text to your commit message:
+
+```text
+Signed-off-by: Your Name <your@email.com>
+```
+
+### Full text of the Developer Certificate of Origin
+
+```text
+Developer Certificate of Origin
+Version 1.1
+
+Copyright (C) 2004, 2006 The Linux Foundation and its contributors.
+
+Everyone is permitted to copy and distribute verbatim copies of this license document, but changing it is not allowed.
+
+
+Developer's Certificate of Origin 1.1
+
+By making a contribution to this project, I certify that:
+
+(a) The contribution was created in whole or in part by me and I have the right to submit it under the open source license indicated in the file; or
+
+(b) The contribution is based upon previous work that, to the best of my knowledge, is covered under an appropriate open source license and I have the right under that license to submit that work with modifications, whether created in whole or in part by me, under the same open source license (unless I am permitted to submit under a different license), as indicated in the file; or
+
+(c) The contribution was provided directly to me by some other person who certified (a), (b) or (c) and I have not modified it.
+
+(d) I understand and agree that this project and the contribution are public and that a record of the contribution (including all personal information I submit with it, including my sign-off) is maintained indefinitely and may be redistributed consistent with this project or the open source license(s) involved.
+```
+
+## License
+
+By contributing to this project, you agree that your contributions will be licensed under the [OpenMDW-1.1 License](LICENSE). All contributions must comply with the terms of this license.

Dockerfile

@@ -0,0 +1,19 @@
+# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: OpenMDW-1.1
+
+FROM vllm/vllm-openai:v0.26.0
+LABEL org.opencontainers.image.title="Nemotron 3.5 Parse" \
+      org.opencontainers.image.version="3.5.0.post7" \
+      org.opencontainers.image.licenses="OpenMDW-1.1"
+WORKDIR /workspace
+RUN python3 -m venv /opt/nemotron-parse-venv
+ENV PATH="/opt/nemotron-parse-venv/bin:$PATH" \
+    VIRTUAL_ENV="/opt/nemotron-parse-venv" \
+    PYTHONPATH=""
+COPY pyproject.toml LICENSE THIRD_PARTY_NOTICES.md TOKENIZER_ATTRIBUTION.md postprocessing.py latex2html.py /tmp/nemotron-package/
+COPY nemotron_parse /tmp/nemotron-package/nemotron_parse
+COPY vllm_tied_patch /tmp/nemotron-package/vllm_tied_patch
+RUN python3 -m pip install "/tmp/nemotron-package[vllm,client]" && python3 -m pip check
+COPY . /workspace
+ENTRYPOINT ["python3", "-m", "vllm.entrypoints.openai.api_server"]
+CMD ["--model", "/workspace", "--served-model-name", "nemotron-parse", "--trust-remote-code", "--dtype", "bfloat16", "--distributed-executor-backend", "uni", "--tensor-parallel-size", "1", "--max-model-len", "16384", "--max-num-seqs", "192", "--max-num-batched-tokens", "16384", "--no-enable-prefix-caching", "--attention-config", "{\"flash_attn_version\":3}", "--limit-mm-per-prompt", "{\"image\":1}", "--chat-template", "/workspace/chat_template.jinja"]

LICENSE

@@ -0,0 +1,57 @@
+Unless separately indicated, the model weights and NVIDIA-provided source code
+and documentation are licensed under the OpenMDW-1.1 License.
+The tokenizer included with this model is separately licensed under the
+MIT license. See TOKENIZER_ATTRIBUTION.md for the license text,
+original copyright notice, source, and NVIDIA modification history.
+
+------------
+Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+
+
+OpenMDW License Agreement, version 1.1 (OpenMDW-1.1)
+
+By exercising rights granted to you under this agreement, you accept and agree
+to its terms.
+
+As used in this agreement, "Model Materials" means the materials provided to
+you under this agreement, consisting of: (1) one or more machine learning
+models (including architecture and parameters); and (2) all related artifacts
+(including associated data, documentation and software) that are provided to
+you hereunder.
+
+Subject to your compliance with this agreement, permission is hereby granted,
+free of charge, to deal in the Model Materials without restriction, including
+under all copyright, patent, database, and trade secret rights included or
+embodied therein.
+
+If you distribute any portion of the Model Materials, you shall retain in your
+distribution (1) a copy of this agreement, and (2) all copyright notices and
+other notices of origin included in the Model Materials that are applicable to
+your distribution.
+If you file, maintain, or voluntarily participate in a lawsuit against any
+person or entity asserting that the Model Materials directly or indirectly
+infringe any patent or copyright, then all rights and grants made to you
+hereunder are terminated, unless that lawsuit was in response to a
+corresponding lawsuit first brought against you.
+
+This agreement does not impose any restrictions or obligations with respect to
+any use, modification, or sharing of any outputs generated by using the Model
+Materials.
+
+THE MODEL MATERIALS ARE PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
+OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE, TITLE, NONINFRINGEMENT, ACCURACY, OR THE
+ABSENCE OF LATENT OR OTHER DEFECTS OR ERRORS, WHETHER OR NOT DISCOVERABLE, ALL
+TO THE GREATEST EXTENT PERMISSIBLE UNDER APPLICABLE LAW.
+
+YOU ARE SOLELY RESPONSIBLE FOR (1) CLEARING RIGHTS OF OTHER PERSONS THAT MAY
+APPLY TO THE MODEL MATERIALS OR ANY USE THEREOF, INCLUDING WITHOUT LIMITATION
+ANY PERSON'S COPYRIGHTS OR OTHER RIGHTS INCLUDED OR EMBODIED IN THE MODEL
+MATERIALS; (2) OBTAINING ANY NECESSARY CONSENTS, PERMISSIONS OR OTHER RIGHTS
+REQUIRED FOR ANY USE OF THE MODEL MATERIALS; OR (3) PERFORMING ANY DUE
+DILIGENCE OR UNDERTAKING ANY OTHER INVESTIGATIONS INTO THE MODEL MATERIALS OR
+ANYTHING INCORPORATED OR EMBODIED THEREIN.
+IN NO EVENT SHALL THE PROVIDERS OF THE MODEL MATERIALS BE LIABLE FOR ANY CLAIM,
+DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
+OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE MODEL MATERIALS, THE
+USE THEREOF OR OTHER DEALINGS THEREIN.

README.md

@@ -0,0 +1,102 @@
+---
+license: other
+license_name: openmdw-1.1
+base_model: nvidia/Nemotron-3.5-Parse
+library_name: transformers
+pipeline_tag: image-text-to-text
+tags:
+- document-parsing
+- ocr
+- tables
+- charts
+- thrisul
+---
+# Thrisul-Parser
+
+A derivative of [`nvidia/Nemotron-3.5-Parse`](https://huggingface.co/nvidia/Nemotron-3.5-Parse) (631M parameters) built by TIMPS.
+It adds two things on top of the base parser: **(A) verifier-routed refinement** and **(B) a lightweight resolution self-distillation post-training pass**.
+
+## What is new
+1. **Verifier-routed refinement (inference).** The base model's double/triple pass re-decodes *every* detected table/chart at native resolution.
+   Thrisul runs the page once, scores each table/chart region with a cheap verifier (decoder token uncertainty, structure sanity checks,
+   and a render-vs-crop similarity feature from the vision encoder), and re-decodes **only** the regions a small gradient-boosted router predicts will gain.
+   Code: `thrisul_pipeline.py` (`parse_page(..., mode="routed")`), router: `thrisul_router.joblib`.
+2. **Resolution self-distillation (weights).** LoRA (r=16, decoder linears) trained so that the *page-scale view* of a table/chart produces the output the model
+   itself gives at *native resolution*, filtered by ground-truth quality (teacher score >= 0.8 and not worse than the page-scale output), with native-resolution anchors and text-page replay to avoid forgetting.
+   The LoRA was merged losslessly into the original weights (untouched tensors, including the MTP head, are bit-identical to the base).
+
+## Training data (all exact-ground-truth or model-generated; no benchmark data was used for training)
+- Procedurally rendered tables (merged cells, multi-level headers, numeric/currency/percent columns) and charts (bar/line/pie/stacked, labelled and unlabelled) with exact ground truth.
+- 300 real charts from ChartNet (if available in the build session).
+- Targets are the base model's own native-resolution outputs (so the output format is unchanged), quality-filtered against ground truth.
+- Compute: Kaggle free tier (T4), LoRA, 166 optimizer steps.
+
+## Results (read the caveats)
+All numbers below were measured by us on **held-out synthetic data and a small ParseBench-proxy sample**, on a **T4** GPU with Transformers (no vLLM).
+They are **not** the official ParseBench score and are **not comparable** to the H100 numbers in the base model's card.
+
+Held-out synthetic regions (page-scale input; `routed` refines only regions the router selects):
+
+| system | score | refine_frac | sec_per_region | n |
+|---|---|---|---|---|
+| base_single | 0.800 | 0.000 | 0.723 | 150 |
+| base_double(all) | 0.954 | 1.000 | 1.612 | 150 |
+| base_routed | 0.951 | 0.500 | 1.165 | 150 |
+| thr_single | 0.816 | 0.000 | 0.939 | 150 |
+| thr_double(all) | 0.957 | 1.000 | 1.818 | 150 |
+| thr_routed | 0.956 | 0.680 | 1.517 | 150 |
+
+Full synthetic pages through the real pipeline (table score):
+
+| system | table_score | sec_per_page | pages_per_s | map_fail_rate | regions_refined | n |
+|---|---|---|---|---|---|---|
+| base_single | 0.996 | 5.272 | 0.190 | 0.000 | 0 | 24 |
+| base_double(all) | 0.988 | 7.664 | 0.130 | 0.000 | 24 | 24 |
+| base_routed | 0.988 | 7.305 | 0.137 | 0.000 | 20 | 24 |
+| thr_single | 0.996 | 5.294 | 0.189 | 0.000 | 0 | 24 |
+| thr_routed | 0.997 | 7.504 | 0.133 | 0.000 | 22 | 24 |
+
+ParseBench-proxy on real PDFs (first table/chart of the page vs `expected_markdown`; crude proxy, small n):
+
+not run (dataset load failed in this session)
+
+Verifier ablation (out-of-fold, which signals predict the benefit of refinement):
+
+| features | spearman | auc | refine_frac_for_90pct_gain | gain_captured_at_30pct |
+|---|---|---|---|---|
+| entropy+structure | 0.653 | 0.850 | 0.548 | 0.791 |
+| all (entropy+structure+render) | 0.659 | 0.852 | 0.538 | 0.788 |
+| entropy | 0.646 | 0.844 | 0.552 | 0.773 |
+| render only | 0.417 | 0.732 | 0.767 | 0.606 |
+| structure | 0.287 | 0.663 | 0.917 | 0.541 |
+
+Text regression check (Thrisul-single vs base-single page markdown similarity): 97.0 %
+
+Published reference numbers (self-reported by their authors, different protocol/hardware): Nemotron 3.5 Parse ParseBench 72.74 / 74.92 / 75.20 (single / double / triple pass); Infinity-Parser2-Pro 74.3.
+
+## Usage
+```python
+import torch, sys
+from huggingface_hub import snapshot_download
+from transformers import AutoModel, AutoProcessor, GenerationConfig
+path = snapshot_download("sandeeprdy1729/thrisul-parser")
+sys.path.insert(0, path)
+from postprocessing import postprocess_document          # shipped from the base model
+import thrisul_pipeline as tp
+model = AutoModel.from_pretrained(path, trust_remote_code=True, dtype=torch.bfloat16).cuda().eval()   # use float16 on T4
+processor = AutoProcessor.from_pretrained(path, trust_remote_code=True)
+gen_cfg = GenerationConfig.from_pretrained(path)
+router = tp.Router.load(path + "/thrisul_router.joblib")
+post = lambda text, image_size: postprocess_document(text, image_size=image_size)
+out = tp.parse_page(model, processor, gen_cfg, image, post, mode="routed", router=router)   # or mode="single" / "all"
+print(out["md"])
+```
+
+## Limitations
+- Trained on synthetic + a small number of real charts; real-document gains are unproven beyond the proxy above.
+- Router features depend on this tokenizer's `<class_*>` tokens and on region<->table mapping; pages where mapping fails fall back to single pass.
+- The router was fitted on crop-level labels and applied to page-level generations (distribution shift).
+- Evaluated on a T4 with Transformers; throughput claims do not transfer to H100/vLLM.
+
+## License and attribution
+Derivative work of NVIDIA Nemotron 3.5 Parse; use is governed by the base model's license (OpenMDW 1.1) and the third-party notices in this repository. See `LICENSE` and the notice files copied from the base repo.

TOKENIZER_ATTRIBUTION.md

@@ -0,0 +1,41 @@
+# Tokenizer Attribution
+
+## Meta Platforms, Inc. and affiliates — Nougat tokenizer — MIT License
+
+Component: `nougat/dataset/tokenizer.json`
+
+Source: https://github.com/facebookresearch/nougat/blob/5a92920d342fb6acf05fc9b594ccb4053dbe8e7a/nougat/dataset/tokenizer.json
+
+Source revision: `5a92920d342fb6acf05fc9b594ccb4053dbe8e7a`
+
+Attribution Statements: Modified by NVIDIA. The base 50,000-token vocabulary and token IDs, BPE merge rules, decoder, and original special-token IDs are retained. NVIDIA added multilingual and document-parsing tokens, including chart-related tokens, expanding the tokenizer to 72,505 entries, and modified pre-tokenization.
+
+License Text (https://github.com/facebookresearch/nougat/blob/5a92920d342fb6acf05fc9b594ccb4053dbe8e7a/LICENSE)
+
+MIT License
+
+Copyright (c) Meta Platforms, Inc. and affiliates.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.
+
+## NVIDIA tokenizer modifications
+
+Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES.
+
+License: MIT

bias.md

@@ -0,0 +1,7 @@
+## Bias
+
+Field | Response
+:---- | :----
+Participation considerations from adversely impacted groups [protected classes](https://calcivilrights.ca.gov/disputeresolution/protected-characteristics/) in model design and testing: | None
+Measures taken to mitigate against unwanted bias: | Dataset curation, filtering, and evaluation include review across document types, layouts, languages, and scripts where available. Users should evaluate performance on their target document domains before deployment.
+Bias Metric (If Measured): | Not measured as a demographic fairness metric. Model quality is evaluated through document parsing, OCR, layout, table, chart, and reading-order metrics.

chat_template.jinja

@@ -0,0 +1,3 @@
+{# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. #}
+{# SPDX-License-Identifier: OpenMDW-1.1 #}
+{% for message in messages %}{{ message['content'] }}{% endfor %}

config.json

@@ -0,0 +1,207 @@
+{
+  "architectures": [
+    "RadioMBartPrefixForCausalLM"
+  ],
+  "auto_map": {
+    "AutoConfig": "hf_nemotron_parse_config.NemotronParseConfig",
+    "AutoModel": "hf_nemotron_parse_modeling.NemotronParseForConditionalGeneration",
+    "AutoImageProcessor": "hf_nemotron_parse_processor.NemotronParseImageProcessor",
+    "AutoProcessor": "hf_nemotron_parse_processor.NemotronParseProcessor"
+  },
+  "bos_token_id": 0,
+  "decoder": {
+    "_attn_implementation": "sdpa",
+    "_name_or_path": "",
+    "activation_dropout": 0.0,
+    "activation_function": "gelu",
+    "add_cross_attention": false,
+    "add_final_layer_norm": true,
+    "architectures": null,
+    "attention_dropout": 0.0,
+    "bad_words_ids": null,
+    "begin_suppress_tokens": null,
+    "bos_token_id": 0,
+    "chunk_size_feed_forward": 0,
+    "classifier_dropout": 0.0,
+    "cross_attention_hidden_size": null,
+    "d_model": 1024,
+    "decoder_attention_heads": 16,
+    "decoder_ffn_dim": 4096,
+    "decoder_layerdrop": 0.0,
+    "decoder_layers": 10,
+    "decoder_start_token_id": null,
+    "diversity_penalty": 0.0,
+    "do_sample": false,
+    "dropout": 0.1,
+    "early_stopping": false,
+    "encoder_attention_heads": 16,
+    "encoder_ffn_dim": 4096,
+    "encoder_layerdrop": 0.0,
+    "encoder_layers": 12,
+    "encoder_no_repeat_ngram_size": 0,
+    "eos_token_id": 2,
+    "exponential_decay_length_penalty": null,
+    "finetuning_task": null,
+    "forced_bos_token_id": null,
+    "forced_eos_token_id": 2,
+    "hidden_size": 1024,
+    "id2label": {
+      "0": "LABEL_0",
+      "1": "LABEL_1",
+      "2": "LABEL_2"
+    },
+    "init_std": 0.02,
+    "is_decoder": true,
+    "is_encoder_decoder": false,
+    "label2id": {
+      "LABEL_0": 0,
+      "LABEL_1": 1,
+      "LABEL_2": 2
+    },
+    "length_penalty": 1.0,
+    "max_length": 20,
+    "min_length": 0,
+    "model_type": "nemotron_parse_text",
+    "no_repeat_ngram_size": 0,
+    "num_beam_groups": 1,
+    "num_beams": 1,
+    "num_hidden_layers": 10,
+    "num_return_sequences": 1,
+    "output_attentions": false,
+    "output_hidden_states": false,
+    "output_scores": false,
+    "pad_token_id": 1,
+    "prefix": null,
+    "problem_type": null,
+    "pruned_heads": {},
+    "remove_invalid_values": false,
+    "repetition_penalty": 1.0,
+    "return_dict": true,
+    "return_dict_in_generate": false,
+    "scale_embedding": true,
+    "sep_token_id": null,
+    "suppress_tokens": null,
+    "task_specific_params": null,
+    "temperature": 1.0,
+    "tf_legacy_loss": false,
+    "tie_encoder_decoder": false,
+    "tie_word_embeddings": true,
+    "tokenizer_class": null,
+    "top_k": 50,
+    "top_p": 1.0,
+    "torch_dtype": "bfloat16",
+    "torchscript": false,
+    "transformers_version": "5.6.1",
+    "typical_p": 1.0,
+    "use_bfloat16": true,
+    "use_cache": true,
+    "vocab_size": 72512,
+    "self_attention_kv_heads": 8,
+    "cross_attention_kv_heads": 0,
+    "causal_vision_prefix": true
+  },
+  "decoder_start_token_id": 2,
+  "encoder": {
+    "_name_or_path": "nvidia/C-RADIOv4-SO400M",
+    "adaptor_configs": {},
+    "adaptor_names": null,
+    "architectures": [
+      "RADIOModel"
+    ],
+    "args": {
+      "cls_token_per_teacher": true,
+      "cpe_max_size": 2048,
+      "model": "vit_so400m_patch16_224",
+      "register_multiple": 10,
+      "teachers": [
+        {
+          "name": "siglip2-g",
+          "use_summary": true
+        },
+        {
+          "name": "dino_v3_7b",
+          "use_summary": true
+        },
+        {
+          "name": "sam3",
+          "use_summary": false
+        }
+      ]
+    },
+    "auto_map": {
+      "AutoConfig": "nvidia/C-RADIOv4-SO400M--hf_model.RADIOConfig",
+      "AutoModel": "nvidia/C-RADIOv4-SO400M--hf_model.RADIOModel"
+    },
+    "chunk_size_feed_forward": 0,
+    "dtype": "float32",
+    "feature_normalizer_config": null,
+    "id2label": {
+      "0": "LABEL_0",
+      "1": "LABEL_1"
+    },
+    "inter_feature_normalizer_config": null,
+    "is_encoder_decoder": false,
+    "label2id": {
+      "LABEL_0": 0,
+      "LABEL_1": 1
+    },
+    "max_resolution": 2048,
+    "model_type": "",
+    "output_attentions": false,
+    "output_hidden_states": false,
+    "patch_size": 16,
+    "preferred_resolution": [
+      512,
+      512
+    ],
+    "problem_type": null,
+    "return_dict": true,
+    "version": "c-radio_v4-so400m",
+    "vitdet_window_size": null,
+    "code_revision": "c0457f5dc26ca145f954cd4fc5bb6114e5705ad8"
+  },
+  "eos_token_id": 2,
+  "image_size": [
+    2048,
+    1664
+  ],
+  "is_encoder_decoder": false,
+  "max_sequence_length": 16384,
+  "model_type": "nemotron_parse",
+  "pad_token_id": 1,
+  "tie_word_embeddings": true,
+  "torch_dtype": "bfloat16",
+  "transformers_version": "5.6.1",
+  "vocab_size": 72512,
+  "class_token_start_idx": 52315,
+  "legacy_use_hf_vision_model": true,
+  "vision_backbone_name": "nvidia/C-RADIOv4-SO400M",
+  "vision_spatial_dim": 1152,
+  "vision_summary_dim": 2304,
+  "architecture_benchmark": {
+    "backbone": "nvidia/C-RADIOv4-SO400M",
+    "attention_mode": "global",
+    "spatial_dim": 1152,
+    "summary_dim": 2304
+  },
+  "navit_disable_cpe_augmentation": true,
+  "self_attention_kv_heads": 8,
+  "cross_attention_kv_heads": 0,
+  "legacy_causal_vision_prefix": true,
+  "legacy_causal_prefix_context_limit": 16384,
+  "image_token_index": 1,
+  "num_hidden_layers": 10,
+  "decoder_layers": 10,
+  "hidden_size": 1024,
+  "intermediate_size": 4096,
+  "num_attention_heads": 16,
+  "num_key_value_heads": 8,
+  "max_position_embeddings": 16384,
+  "navit_position_embedding_mode": "content_normalized_2d_interpolate",
+  "navit_packing": true,
+  "navit_coordinate_frame": "content_normalized",
+  "virtual_canvas_width": 1664,
+  "virtual_canvas_height": 2048,
+  "virtual_canvas_placement": "aligned_center",
+  "num_extra_heads": 1
+}

docker-compose.yaml

@@ -0,0 +1,14 @@
+services:
+  nemotron-parse:
+    build: .
+    image: nemotron-parse:3.5.0.post7
+    deploy:
+      resources:
+        reservations:
+          devices:
+            - driver: nvidia
+              count: 1
+              capabilities: [gpu]
+    shm_size: 8gb
+    ports:
+      - "8000:8000"

docs/postprocessing.md

@@ -0,0 +1,446 @@
+# Document postprocessing and multipass parsing
+
+The public helpers operate on model predictions and ordinary page images. They do
+not load benchmark labels, scorer code, category IDs, reference answers, or
+page-specific rules. They are adapted from the general conversion and crop-merge
+components of the bundled ParseBench pipeline. The pinned reproduction bundle
+remains unchanged so its recorded hashes and results stay reproducible.
+
+## Installation
+
+For vLLM, use the model card's repository-source installation:
+
+```bash
+hf download nvidia/Nemotron-3.5-Parse --local-dir nemotron-parse
+cd nemotron-parse
+python -m pip install '.[vllm,client]'
+```
+
+This installs the plugin, public `nemotron_parse` Python package, compatibility
+modules, and `nemotron-parse` command. No prebuilt wheel is required.
+The repository source installs the vLLM plugin, parsing helpers and CLI together.
+Archived release artifacts and their digests are listed in the
+[build evidence](../reference/package/README.md).
+Use version 3.5.0.post5 or later for the restored merge default and post7 or later
+for the corrected third-party notices.
+
+For Transformers, keep the model card's inference dependencies. To download the
+helpers without installing the package:
+
+```bash
+python -m pip install 'beautifulsoup4==4.13.3' 'Pillow==12.2.0' 'numpy==2.2.6' 'httpx==0.28.1'
+hf download nvidia/Nemotron-3.5-Parse \
+  --include 'postprocessing.py' 'latex2html.py' 'nemotron_parse/**' --local-dir .
+```
+
+Run Python in that directory, or put it on your Python import path. Basic text/table
+postprocessing only needs Beautiful Soup; the image wrappers also need Pillow and
+NumPy. The HTTP backend needs httpx. Downloaded source and installed packages
+provide the same public API.
+
+## Render an existing prediction
+
+```python
+from nemotron_parse import postprocess_document, PostprocessOptions
+
+options = PostprocessOptions(
+    infer_headers=True,
+    render_charts=True,
+    split_chart_value_percent=True,
+    relocate_chart_captions=True,
+)
+document = postprocess_document(raw_model_output, image_size=(width, height), options=options)
+print(document['markdown'])
+print(document['blocks'])
+```
+
+`raw_out` contains the cleaned model serialization. Each block has its original
+class, normalized `bbox`, literal `text`, and rendered `content`. Supplying
+`image_size` adds clipped, ordered `bbox_pixels`. `markdown` renders the complete
+page in model order, with section headings and HTML tables for span preservation.
+Raw per-block text remains available independently of presentation changes.
+Malformed table markup that cannot be converted is preserved rather than passed
+through a lossy row splitter; document `warnings` identify these blocks.
+
+Default cleanup preserves empty cells, spans, line breaks, nested inline styles,
+and escaped literals. It removes exact repeated blocks at identical coordinates,
+repairs empty incomplete serialization prefixes, removes internal continuation
+markers, and normalizes existing bullets and soft hyphens in textual roles.
+Code/math roles and marked code/math text are protected from soft-hyphen cleanup.
+Table conversion is applied to Table/Chart blocks; literal LaTeX in prose, code,
+formulas, or Picture blocks retains its source representation.
+Header inference uses predicted span topology and existing bold styling, never an
+English vocabulary list. It can still make mistakes; disable `infer_headers` to
+use only explicit predicted headers. Unknown classes and page numbers are retained.
+
+Chart titles, subtitles, and explicit panel labels can be rendered with their
+associated tables. Two guarded presentation choices are enabled by default and can be disabled:
+
+- `split_chart_value_percent=True` splits consistently predicted numeric pairs
+  such as `32 (16.6%)` into Value and Percent subcolumns. It requires at least two
+  matching nonempty cells, adds those column labels, and does not calculate values.
+- `relocate_chart_captions=True` associates a plain caption immediately above a
+  chart when geometry and text-preservation checks pass. Notes below remain
+  standalone. Set `relocate_chart_captions=False` to retain the standalone caption in its source order.
+
+`normalize_text=False`, `deduplicate=False`, and `render_charts=False` disable the
+corresponding cleanup/rendering choices. The existing root imports continue to
+work:
+
+```python
+from postprocessing import extract_classes_bboxes, postprocess_text
+from latex2html import convert_html_table_to_json_hierarchical
+```
+
+`postprocess_text(..., cls='Table', table_format=...)` supports LaTeX, HTML,
+Markdown, record JSON, hierarchical JSON, and CSV. HTML retains merged-cell
+structure. Flat exports expand spans and use neutral column names for headerless
+tables without consuming the first data row as a header. Duplicate column names
+are disambiguated without overwriting values. Hierarchical JSON retains the
+existing `headers`, `columns`, and flat `data` fields and additionally exposes
+`body_cells` with explicit text, span, and header-role metadata.
+
+## Parse a page with a running vLLM server
+
+Start the server using the model card. Then:
+
+```python
+from nemotron_parse.inference import DocumentParser
+
+with DocumentParser(base_url='http://localhost:8000/v1', model='nemotron-parse') as parser:
+    single = parser.parse('page.png')
+    double = parser.parse_double('page.png')
+    triple = parser.parse_triple('page.png', triple_regions='union')
+    documents = parser.parse_many(['one.png', 'two.png'], mode='double', concurrency=128)
+```
+
+| Mode | Work performed |
+| --- | --- |
+| Single | Full-page parse and document rendering |
+| Double | Full-page parse, then Table/Chart crops from its predicted regions |
+| Triple | Full-page parse, separate detection, then Table/Chart crop parsing |
+
+A pass denotes a stage, not one request: each retained crop needs its own model
+call. `triple_regions='union'` retains detector regions first and adds parser
+regions after overlap filtering; `'detector'` uses only the separate detector.
+The region budget defaults to 80 per page and can be changed with `max_regions`.
+Crops use 4% plus 12-pixel margins and JPEG quality 95, configurable margins, and
+content-normalized coordinates mapped back to the original image. Tiny crops are
+skipped. Crops are made in memory and no temporary image files remain.
+
+The default `merge_policy="alternatives"` preserves differing overlapping
+Table/Chart predictions from the full-page and crop passes. Near-identical blocks
+are deduplicated using geometry and text. Guarded table refinement checks native
+grid validity and compatible column counts; the original block order is preserved
+and crop alternatives are inserted by their predicted geometry. This policy
+matches the reference approach using only model predictions, without labels,
+benchmark categories or scorer feedback.
+
+A returned document can contain overlapping or conflicting Table/Chart hypotheses.
+Use `DocumentParser(merge_policy="guarded")` (CLI: `--merge-policy guarded`) when
+one selected prediction per overlapping same-class region is preferable. This
+option applies conservative structural replacement and rejects overlapping
+same-class additions. It can discard a correct alternative, reducing recall.
+Raw stage responses remain available with either policy.
+
+Returned results include `stages` with original raw responses and usage, and
+`inference` with request counts, regions, sampling, and crop/rendering settings.
+Empty, failed, filtered, or token-limited generations raise `GenerationError`;
+its `stage` and `response` expose the failed response. Successful stage responses
+from the same page are only returned when parsing completes. No automatic retry
+policy changes the request settings.
+
+Sampling is configurable per stage:
+
+```python
+from nemotron_parse.inference import DocumentParser, GenerationSettings
+
+with DocumentParser(
+    fullpage=GenerationSettings(temperature=0, top_k=1, repetition_penalty=1.1),
+    detection=GenerationSettings(temperature=0, top_k=1, max_tokens=4096),
+    crop=GenerationSettings(temperature=0, top_k=1, repetition_penalty=1.0),
+) as parser:
+    document = parser.parse_double('page.png')
+```
+
+Defaults are temperature 0.2 / repetition penalty 1.05 for full-page and detection,
+and 0.1 / 1.0 for crops, top-k 50, top-p 1.0, seed 0. Token caps are 8,744 for
+parsing and 4,096 for detection. These are practical starting settings rather than
+a claim of optimal sampling for every document type.
+
+## Use an already loaded Transformers model
+
+Continue from the model card's Transformers setup:
+
+```python
+from nemotron_parse.inference import DocumentParser, TransformersBackend
+
+backend = TransformersBackend(model, processor, generation_config=generation_config)
+with DocumentParser(backend=backend) as parser:
+    document = parser.parse_triple(image)
+```
+
+This uses the same image/crop orchestration without a server or plugin. The adapter
+serializes calls to the shared model, uses inference mode, applies stage settings,
+and restores the CPU/current-GPU RNG after generation. Model weights and processor
+remain caller-owned. Reaching the token cap is conservatively reported as
+truncation, including when generation forces an EOS at the cap.
+
+Other inference implementations can supply a callable
+`backend(image, *, prompt, stage, settings)` returning
+`{'raw_out': text, 'finish_reason': 'stop', 'usage': {...}}`. A string return is also
+accepted, but then the custom backend must detect truncation itself. Custom
+backends used with `parse_many` should support concurrent calls or use concurrency 1.
+
+## Command line
+
+```bash
+nemotron-parse --image page.png --mode double --output parsed-page
+nemotron-parse --image-dir pages --mode triple --triple-regions union --output parsed-pages
+```
+
+The download-only equivalent is `python -m nemotron_parse.inference` with the same
+arguments. The default outputs are compact `documents.jsonl.gz` and `metrics.json`, containing
+blocks, raw stages, settings and timing. Add `--write-page-files` for individual
+Markdown and JSON exports. Use a new/empty output directory. Files are written only
+after all pages complete. The API accepts PIL images and image paths; the CLI
+accepts image files. Render PDF pages to images before calling it.
+
+## Validation and scope
+
+Run the focused CPU regression suite from the downloaded repository:
+
+```bash
+python -m pip install pytest
+python -m pytest -q tests/test_postprocessing.py tests/test_inference.py tests/test_batch.py
+```
+
+The 86 focused tests passed from both source and the installed wheel. They cover
+table structure, text preservation, chart defaults and opt-outs, crop geometry,
+merge guards, HTTP handling, explicit truncation reporting, batch ordering,
+independent-stage overlap, early crop submission, a shared concurrency bound,
+and staged/pipelined region and output equivalence with deterministic responses.
+The Transformers adapter is checked with a small CPU model stub. The real-model
+H100 runs and official full-page scoring are recorded below and in
+[the validation receipt](../reference/validation/postprocessing.json). The pinned reproduction
+bundle and its recorded headline result remain unchanged.
+
+## Prepared batches and timing
+
+`parser.prepare(images, workers=8)` returns a reusable in-memory batch containing
+decoded RGB pixels and encoded image requests. Existing RGB JPEG/PNG/WebP payloads
+are retained without another lossy encode. Other inputs are encoded as PNG.
+The preparation time is reported separately, and cached images can occupy several
+GB for thousands of document pages.
+
+`parser.parse_prepared(batch, mode="double", concurrency=128, postprocess_workers=8)`
+returns a `BatchResult` with `documents` and `metrics`. The default
+`schedule="pipelined"` overlaps independent full-page and detection requests,
+submits detector crops as soon as their regions are ready, and adds parser-only
+regions in the original union order. Double-pass crops similarly start as each
+page finishes its full-page request. Ready crops take the next available HTTP
+slot, and new pages fill remaining slots. All stages share one bounded
+`concurrency` window, including requests waiting inside vLLM. Crop preparation
+uses separate CPU threads; finished pages merge and render while other pages
+continue inference. Large batches use spawned rendering processes.
+
+Run standalone scripts under a main guard, or choose `postprocess_workers=1`.
+HTTP `parse_many` uses this same pipelined batch engine and returns only the
+documents. Its default request window is 128; a custom backend retains the
+four-worker default. Pass `concurrency=1` for a backend that requires serial calls.
+Use `schedule="staged"` for stage-barrier comparisons. A custom Transformers
+backend still serializes its shared model calls.
+
+Metrics include separately reported initial preparation, pipeline elapsed time,
+request counts, token-limit counts, and processed-page and fully completed-page
+throughput. Pipeline time excludes
+initial preparation and output-file writing; model-dependent crop preparation is
+included. A benchmark should also report serialization and warmup/server-startup
+exclusions explicitly. The same prepared batch can be reused for different modes;
+each call generates its own responses and provides an independent pipeline timing.
+Pipelined stage spans overlap and must not be added together.
+`request_service_seconds` sums concurrent request latency, including server queue
+wait; it is not GPU compute time. `peak_inflight_requests` records the client
+window, while actual server running/waiting requests require vLLM metrics.
+
+Strict generation handling remains the default. Set `allow_partial=True` (CLI:
+`--allow-partial`) only when partial results are useful: capped raw responses are
+preserved, affected pages have `complete=False` and `generation_warnings`, and
+truncated crops are excluded from refinement. HTTP/transport errors and empty
+responses still raise. A successful benchmark execution does not imply that every
+page completed within its token limit; inspect both coverage and throughput.
+
+The CLI supports `--no-split-chart-value-percent` and
+`--no-relocate-chart-captions` to disable the default presentation rules.
+
+## Measured prepared-batch performance
+
+The following matched scheduler comparison used the earlier **guarded** merge
+policy in package 3.5.0.post4. The restored default and its fresh results are in
+[accuracy restoration](#accuracy-restoration-with-the-faster-scheduler).
+
+Measured on all 2,078 unique ParseBench pages with one H100 80GB, vLLM 0.26.0,
+`--distributed-executor-backend uni`, tensor parallelism 1, one-token MTP, FA3,
+`max_num_seqs=192`, `max_num_batched_tokens=16384`, and `max_model_len=16384`.
+Both chart presentation rules were enabled. Each run generated all of its own
+responses. The same-server comparison used the same prepared images and
+per-mode concurrency limit for both schedulers.
+
+| Mode | Concurrency | Staged pages/s | Pipelined pages/s | Pipelined elapsed | Fully completed pages |
+| --- | ---: | ---: | ---: | ---: | ---: |
+| Single | 128 | 14.56 | 14.91 | 139.4 s | 2047/2078 |
+| Double | 256 | 11.16 | 11.87 | 175.0 s | 2038/2078 |
+| Triple | 128 | 7.31 | 10.90 | 190.6 s | 2039/2078 |
+
+Initial decoding/encoding of the full batch took 9.7 seconds and was excluded
+from these rates. Timed work includes all model calls, predicted-region crop
+preparation, merge and rendering; startup, warmup and output serialization are
+excluded. The receipt separately reports serialization-inclusive rates.
+
+These diagnostic runs explicitly enabled `allow_partial=True`. Token-limited
+responses remain visible and affected pages count as incomplete. The table reports
+processed-page throughput; fully completed-page throughput and cap counts are in
+[the benchmark receipt](../reference/validation/batch_benchmarks.json). Strict failure handling remains
+the API default. A successful scoring run does not make truncated generations complete.
+
+Concurrency 128 and 256 were compared on a 256-page pilot; the selected settings
+above were then checked on the full batch. The earlier full single-pass sweep reached
+15.09 pages/s at 256. These are measured serving profiles, not a throughput ceiling.
+Sampling is nonzero: changed batching can change output lengths and crop counts.
+The receipt includes request and completion-token totals for each run.
+Stages share one bounded client request window. In the pipelined runner, crops
+receive available slots while independent full-page/detection requests continue.
+Stage spans overlap; adding them does not produce elapsed time. Queue averages
+include pipeline ramp-up and final drain, so mean running-request counts need not
+equal the configured limit.
+
+### Quality and the reference output contract
+
+| Previous guarded merge, pipelined | Official overall (%) |
+| --- | ---: |
+| Single | 72.79 |
+| Double | 73.28 |
+| Triple | 73.48 |
+
+Each full-run output shown here was freshly scored on all 2,078 pages with the
+pinned official scorer
+`9af4102e26b51f234ab5f037e19fec05c0696ec6`, with all 241 Python source hashes
+verified and zero official scoring errors. Capped generations are included in scoring.
+
+The pinned reference merge can retain an original Table/Chart block and an
+overlapping crop alternative when their text differs. The chart evaluator searches
+rendered tables for the best matching data. The previous guarded policy uses
+replacement and rejects overlapping same-class crop additions; raw alternatives
+remain in `stages.crops` for inspection.
+
+A controlled replay of identical baseline triple responses scored 73.44
+with the previous guarded merge and 75.19 with reference-style merging of
+the same completed crops. The latter inserted 1,660 overlapping alternative blocks.
+This identified the output contract as the cause of the score gap. Package
+3.5.0.post5 restores alternatives as the default, keeping the faster scheduler.
+Use the separately pinned reproduction pipeline for the exact headline 75.20
+contract. The receipt includes category metrics and the merge-only replay checks.
+
+## Accuracy restoration with the faster scheduler
+
+Package 3.5.0.post5 defaults to `merge_policy="alternatives"`. The scheduler,
+model settings, region selection and chart presentation defaults are unchanged.
+The merge retains differing full-page/crop hypotheses as the reference does;
+it adds no model calls. `merge_policy="guarded"` remains available explicitly.
+
+Fresh model inference on all 2,078 pages with one H100, the same uni/FA3/MTP server
+profile, and prepared inputs produced:
+
+| Pipeline | Concurrency | Pages/s | Elapsed | Official overall (%) | Fully completed pages |
+| --- | ---: | ---: | ---: | ---: | ---: |
+| Double | 256 | 11.64 | 178.5 s | 74.92 | 2043/2078 |
+| Triple | 128 | 10.61 | 195.8 s | 75.20 | 2038/2078 |
+
+Timing includes crop preparation and merge/rendering. Preparation, startup, warmup
+and serialization are excluded; the receipt also reports serialization-inclusive
+and fully completed-page rates. These diagnostic runs explicitly used
+`allow_partial=True`; token-limit cases remain marked incomplete.
+
+The same saved responses from the earlier staged and pipelined runs were also
+rescored with the restored merge. This isolates the merge from generation:
+
+| Saved responses | Guarded overall (%) | Alternatives overall (%) |
+| --- | ---: | ---: |
+| Double Staged | 73.25 | 74.88 |
+| Double Pipelined | 73.28 | 74.96 |
+| Triple Staged | 73.51 | 75.23 |
+| Triple Pipelined | 73.48 | 75.23 |
+
+All six outputs were freshly scored on 2,078 unique pages with the pinned official
+scorer and all 241 source hashes verified, with zero official scoring errors.
+Batching at nonzero temperature can change individual responses; this does not
+establish bit-identical generations or a guarantee for every document.
+
+The reference-compatible score includes alternative tables/charts. The evaluator
+can match the best alternative, so score recovery does not prove that a single
+chosen transcription is equally accurate. The alternatives are present in the
+actual returned document and its raw stages. Use the guarded policy for a more
+compact document when that tradeoff fits the application.
+
+See [the restoration receipt](../reference/validation/accuracy_restoration.json) for category
+scores, generation coverage, exact settings, request/token counts, hashes and
+controlled replay results.
+
+
+## Inference throughput with prepared pages
+
+For prediction throughput, use `result.metrics["inference_pages_per_second"]`
+(package 3.5.0.post6 or later). Its wall clock runs from the first model request
+start to the last model response across all passes. It includes server queueing,
+image encoding inside the server, generation, HTTP transfer and any gaps waiting
+for predicted-region selection, crop encoding or dispatch. It excludes initial
+page preparation, startup/warmup, final merge/rendering after inference, and writes.
+`pipeline_seconds` and `processed_pages_per_second` retain their earlier meanings,
+including merge/rendering, for applications that need complete documents.
+Concurrent rendering can still compete for CPU during inference; that observed
+contention remains in the wall time. We do not subtract overlapping CPU time or
+sum concurrent request latencies to claim a higher rate.
+
+The fastest observed profiles compatible with the restored output quality are:
+
+| Pipeline | Concurrency | Schedule | Prediction pages/s | Inference wall time | Fully completed pages |
+| --- | ---: | --- | ---: | ---: | ---: |
+| Single | 256 | Staged | **15.42** | 134.78 s | 2045/2078 |
+| Double | 256 | Pipelined | **11.65** | 178.35 s | 2043/2078 |
+| Triple | 128 | Pipelined | **10.62** | 195.70 s | 2038/2078 |
+
+These use 2,078 unique prepared ParseBench page images on one H100 80GB and the
+same uni/FA3/MTP serving profile. The single-pass measurement predates the merge
+restoration; it has no crops and is unaffected by that merge change. Its saved
+stage clock includes executor setup/teardown and is slightly conservative.
+Double/triple times are reconstructed exactly from saved stage start/end spans,
+including gaps between calls. These are existing measured runs, not a new GPU
+run of the timing-only post6 release. See the [timing receipt](../reference/validation/inference_benchmarks.json).
+
+The earlier guarded-merge runs reached 11.88/10.91 prediction pages/s for
+double/triple, respectively, but had the lower scores documented above. Use the
+restored profiles when comparing speed while preserving the reference output
+contract. The restored double/triple scores are 74.92/75.20.
+
+This is realistic batch prediction throughput when pages are prepared ahead of
+time and enough work is queued. It is not PDF ingestion throughput or single-page
+latency, and is not a measured hardware ceiling. Document complexity, generation
+length, crop count, CPU capacity, network placement and batching affect speed.
+The numerator includes token-limited pages; these diagnostic runs explicitly
+allow partial output. Use `complete_inference_pages_per_second` for the number
+of fully completed pages divided by the same wall time. Strict errors remain the
+default outside these diagnostic runs.
+
+```python
+prepared = parser.prepare(page_images, workers=8)  # outside inference clock
+result = parser.parse_prepared(
+    prepared, mode="triple", concurrency=128, postprocess_workers=8,
+)
+print(result.metrics["inference_seconds"])
+print(result.metrics["inference_pages_per_second"])
+```
+
+For double pass, use `mode="double", concurrency=256`. The measured fastest
+single-pass profile uses `mode="single", concurrency=256, schedule="staged"`;
+a single-pass pipelined run at concurrency 128 measured 14.92 prediction pages/s.
+A pipelined single-pass run at concurrency 256 has not been measured here.

docs/serving.md

@@ -0,0 +1,106 @@
+# Nemotron 3.5 Parse: Transformers and vLLM
+
+## Installation
+
+Use Python 3.12 and a fresh environment.
+
+The reference serving contract is vLLM 0.26.0, Transformers 5.14.1,
+huggingface-hub 1.24.0, and BF16 on one NVIDIA H100. The package declares the exact
+core dependencies; its `[vllm,client]` extras add serving and the HTTP evaluation
+client. See `pyproject.toml` for pinned direct dependencies.
+
+Download the repository and install its serving extras:
+
+```bash
+python3.12 -m venv .venv
+source .venv/bin/activate
+python -m pip install "huggingface-hub==1.24.0"
+hf download nvidia/Nemotron-3.5-Parse --local-dir nemotron-parse
+cd nemotron-parse
+export MODEL_REPO="$PWD"
+python -m pip install ".[vllm,client]"
+python -m pip check
+```
+
+One-token MTP and the compatible FA3 scheduler are enabled automatically for this model.
+
+For Transformers-only inference, no Nemotron package is required. Follow the
+[Quick Start](../README.md#quick-start) to load the model directly from the Hub.
+
+## Generation contract
+
+The recommended prompt is identical in Transformers, vLLM, and evaluation:
+
+`</s><s><predict_bbox><predict_classes><output_markdown><predict_text_in_pic>`
+
+Use one RGB page per request, temperature 0, top-k 1, repetition penalty 1.1,
+`max_tokens=8744`, and `skip_special_tokens=False`. Preserve the coordinate and
+class tokens in the raw prediction used by postprocessing and scoring.
+
+Boxes are normalized to the evaluated image content. Scale x by that image's
+width and y by its height; no legacy center-pad inverse is needed. The image
+processor accepts dynamically sized pages and applies the exported NaViT
+preprocessing configuration.
+
+## One-token MTP
+
+`model.safetensors` contains the main model and all four trained MTP tensors.
+No auxiliary weight sidecar is required. The model has **630,800,496 unique
+trainable parameters**, including **2,099,200 MTP parameters**. The decoder
+embedding and LM head share weights and are counted once. Eight stored scalar
+buffers are excluded from the parameter count.
+
+Transformers generation uses the normal decoder. vLLM enables the trained
+one-token proposer automatically; it shares the decoder embedding and output
+head.
+
+```bash
+vllm serve "$MODEL_REPO" \
+  --served-model-name nemotron-parse \
+  --trust-remote-code \
+  --dtype bfloat16 \
+  --distributed-executor-backend uni \
+  --tensor-parallel-size 1 \
+  --data-parallel-size 1 \
+  --max-model-len 16384 \
+  --max-num-seqs 192 \
+  --max-num-batched-tokens 16384 \
+  --no-enable-prefix-caching \
+  --limit-mm-per-prompt '{"image":1}' \
+  --attention-config '{"flash_attn_version":3}' \
+  --chat-template "$MODEL_REPO/chat_template.jinja" \
+  --host 127.0.0.1 \
+  --port 8000
+```
+
+This is the reference H100 throughput profile. Reduce `--max-num-seqs` when
+using a GPU with less memory. Set `NP_VLLM_MTP=0` before startup to run the matched normal-decoding path. Clients use `model="nemotron-parse"`, matching
+the served model name. Startup must show MTP with one speculative token; vLLM's
+`SpecDecoding metrics` report its measured draft acceptance during generation.
+
+## Verification and references
+
+The current prepared-batch inference and official scoring records are in the
+[validation receipts](../reference/validation/). The
+[reference reproduction records](../reference/README.md) retain their pinned
+model, runtime and page-set contracts. See
+[runtime metadata](../reference/model/runtime_compatibility.json) and
+[package evidence](../reference/package/README.md) for artifact identities and
+verification scope.
+
+Set `NP_VLLM_MTP=0` before starting vLLM to use normal decoding.
+
+
+### Docker alternative
+
+From the downloaded folder, run the included Compose recipe:
+
+```bash
+docker compose up --build
+```
+
+Use the OpenAI-compatible client in the [Quick Start](../README.md#quick-start). Container verification details are in [reference/README.md](https://huggingface.co/nvidia/Nemotron-3.5-Parse/blob/main/reference/README.md).
+
+### HTTP client
+
+See [vllm_example.py](../vllm_example.py) for an OpenAI-compatible HTTP client.

explainability.md

@@ -0,0 +1,15 @@
+## Explainability
+
+Field | Response
+:---- | :----
+Intended Task/Domain: | Document understanding, OCR, layout parsing, table extraction, and text extraction with spatial grounding.
+Model Type: | Transformer-based vision-encoder-decoder model.
+Intended Users: | Developers and teams building document intelligence, retrieval-augmented generation (RAG), data-curation, extraction, and multimodal agent workflows.
+Output: | Structured text containing document element classes, bounding boxes, reading-order information, chart/table annotations, and extracted text.
+Describe how the model works: | Nemotron 3.5 Parse encodes a dynamically sized document image with C-RADIOv4-SO400M, compresses the visual features with a neck, and places them before the OCR prompt as a causal prefix to a 10-layer mBART decoder. The decoder uses grouped-query self-attention and produces structured document annotations with content-normalized boxes. The 72,505-token tokenizer includes multilingual, task-control, and chart-aware tokens such as `<class_Chart>`.
+Name the adversely impacted groups this has been tested to deliver comparable outcomes regardless of: | Not Applicable
+Technical Limitations & Mitigation: | Performance can vary for low-resolution scans, motion blur, handwriting, decorative fonts, dense tables, unusual layouts, unsupported languages or scripts, and documents with heavy compression artifacts. Users should evaluate outputs on representative documents and use human review for high-impact workflows.
+Verified to have met prescribed NVIDIA quality standards: | Yes
+Performance Metrics: | OCR accuracy, layout/class accuracy, table extraction quality, chart extraction quality, reading-order quality, grounding quality, latency, throughput, and qualitative visual inspection.
+Potential Known Risks: | The model may miss text, hallucinate structure, assign incorrect classes, produce inaccurate bounding boxes, or incorrectly order elements. Downstream systems should treat output as model-generated extraction results and apply validation, confidence checks, and human review where appropriate.
+Licensing: | Governing Terms: Your use of this model is governed by the [OpenMDW License Agreement, version 1.1](https://openmdw.ai/license/1-1/). Use of the tokenizer included in this model is governed by the [MIT license and tokenizer attribution](https://huggingface.co/nvidia/Nemotron-3.5-Parse/blob/main/TOKENIZER_ATTRIBUTION.md). See [third-party notices](https://huggingface.co/nvidia/Nemotron-3.5-Parse/blob/main/THIRD_PARTY_NOTICES.md) for separately installed dependencies.

generation_config.json

@@ -0,0 +1,13 @@
+{
+  "_from_model_config": true,
+  "bos_token_id": 0,
+  "decoder_start_token_id": 2,
+  "eos_token_id": 2,
+  "forced_eos_token_id": 2,
+  "pad_token_id": 1,
+  "transformers_version": "5.6.1",
+  "max_new_tokens": 9000,
+  "do_sample": false,
+  "num_beams": 1,
+  "repetition_penalty": 1.1
+}

hf_logits_processor.py

@@ -0,0 +1,352 @@
+# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: OpenMDW-1.1
+
+"""
+LogitsProcessors for document parsing models.
+
+Contains:
+- TableInsertionLogitsProcessor: Forces table structure insertion after coordinates
+- RepetitionStopProcessor: Detects repetition/hallucination and forces coordinate tokens
+"""
+
+import torch
+from collections import Counter
+from transformers import LogitsProcessor
+from typing import Set, List, Tuple
+
+
+class TableInsertionLogitsProcessor(LogitsProcessor):
+    """
+    A LogitsProcessor that inserts \begin{tabular} tokens after coordinate pairs
+    that mark the START of an object (not the END coordinates).
+    
+    Pattern: <x_start><y_start>CONTENT<x_end><y_end><class_...>
+    
+    This processor triggers after the first <x_...><y_...> pair (start coords),
+    but not after the second pair (end coords).
+    
+    Args:
+        tokenizer: The tokenizer used for encoding/decoding
+        table_prefix: The string to insert (default: "\\begin{tabular}")
+    """
+    
+    def __init__(
+        self, 
+        tokenizer, 
+        table_prefix: str = "\\begin{tabular}"
+    ):
+        self.tokenizer = tokenizer
+        self.table_prefix = table_prefix
+        
+        # Tokenize the table prefix to get the sequence of tokens to force
+        self.table_prefix_ids = tokenizer.encode(
+            table_prefix, 
+            add_special_tokens=False
+        )
+        
+        # Build sets of token IDs for detection
+        self._build_token_sets()
+        
+        # State tracking for forced insertion
+        self._insertion_position = {}  # batch_idx -> position in table_prefix_ids
+        self._insertion_active = {}    # batch_idx -> bool
+        
+        # State tracking for coordinate pairs
+        # False = expecting START coordinates (should trigger)
+        # True = expecting END coordinates (should NOT trigger)
+        self._expecting_end_coords = {}  # batch_idx -> bool
+    
+    def _build_token_sets(self):
+        """Build sets of token IDs for x_, y_, class_, and special tokens."""
+        vocab = self.tokenizer.get_vocab()
+        
+        # Collect all x_ coordinate tokens
+        self.x_token_ids: Set[int] = set()
+        for token, token_id in vocab.items():
+            if token.startswith("<x_") and token.endswith(">"):
+                self.x_token_ids.add(token_id)
+        
+        # Collect all y_ coordinate tokens
+        self.y_token_ids: Set[int] = set()
+        for token, token_id in vocab.items():
+            if token.startswith("<y_") and token.endswith(">"):
+                self.y_token_ids.add(token_id)
+        
+        # Collect all class tokens
+        self.class_token_ids: Set[int] = set()
+        for token, token_id in vocab.items():
+            if token.startswith("<class_") and token.endswith(">"):
+                self.class_token_ids.add(token_id)
+    
+    def _is_xy_pair(self, input_ids: torch.Tensor, batch_idx: int) -> bool:
+        """Check if the sequence ends with <x_...><y_...>."""
+        seq = input_ids[batch_idx].tolist()
+        
+        if len(seq) < 2:
+            return False
+        
+        return seq[-1] in self.y_token_ids and seq[-2] in self.x_token_ids
+    
+    def _last_token_is_class(self, input_ids: torch.Tensor, batch_idx: int) -> bool:
+        """Check if the last token is a <class_...> token."""
+        seq = input_ids[batch_idx].tolist()
+        
+        if len(seq) < 1:
+            return False
+        
+        return seq[-1] in self.class_token_ids
+    
+    def reset(self):
+        """Reset the processor state for a new generation."""
+        self._insertion_position = {}
+        self._insertion_active = {}
+        self._expecting_end_coords = {}
+    
+    def __call__(
+        self, 
+        input_ids: torch.LongTensor, 
+        scores: torch.FloatTensor
+    ) -> torch.FloatTensor:
+        """
+        Process logits to force table prefix insertion when appropriate.
+        
+        Args:
+            input_ids: (batch_size, seq_len) - tokens generated so far
+            scores: (batch_size, vocab_size) - logits for next token
+            
+        Returns:
+            Modified scores with forced tokens where appropriate
+        """
+        batch_size = input_ids.shape[0]
+        
+        for batch_idx in range(batch_size):
+            # Check if we're currently in an insertion sequence
+            if self._insertion_active.get(batch_idx, False):
+                pos = self._insertion_position[batch_idx]
+                
+                if pos < len(self.table_prefix_ids):
+                    # Force the next token in the sequence
+                    forced_token_id = self.table_prefix_ids[pos]
+                    scores[batch_idx] = torch.full_like(scores[batch_idx], float('-inf'))
+                    scores[batch_idx, forced_token_id] = 0.0
+                    
+                    # Advance position
+                    self._insertion_position[batch_idx] = pos + 1
+                else:
+                    # Finished inserting, deactivate
+                    self._insertion_active[batch_idx] = False
+                continue
+            
+            # Check if we just saw a <class_...> token - reset state to expect START coords
+            if self._last_token_is_class(input_ids, batch_idx):
+                self._expecting_end_coords[batch_idx] = False
+            
+            # Check if we have an <x_...><y_...> pair
+            if self._is_xy_pair(input_ids, batch_idx):
+                expecting_end = self._expecting_end_coords.get(batch_idx, False)
+                
+                if not expecting_end:
+                    # This is START coordinates - trigger insertion!
+                    self._insertion_active[batch_idx] = True
+                    self._insertion_position[batch_idx] = 0
+                    
+                    # After START coords, we expect END coords next
+                    self._expecting_end_coords[batch_idx] = True
+                    
+                    # Force the first token
+                    if len(self.table_prefix_ids) > 0:
+                        forced_token_id = self.table_prefix_ids[0]
+                        scores[batch_idx] = torch.full_like(scores[batch_idx], float('-inf'))
+                        scores[batch_idx, forced_token_id] = 0.0
+                        self._insertion_position[batch_idx] = 1
+                else:
+                    # This is END coordinates - don't trigger, but reset for next object
+                    # The <class_...> token will come next, which resets expecting_end_coords
+                    pass
+        
+        return scores
+
+
+class RepetitionStopProcessor(LogitsProcessor):
+    """
+    A LogitsProcessor that detects CONSECUTIVE repetition/hallucination in generated 
+    content and forces the model to output <x_...> tokens to close the current object.
+    
+    This detects patterns like:
+    - <x_0.0><x_0.0><x_0.0>... (same token repeating consecutively)
+    - "ABC ABC ABC ABC..." (same phrase repeating consecutively)
+    
+    But NOT patterns like:
+    - "& 1 & 2 & 3 & 4..." (delimiters with different content between them)
+    
+    When consecutive repetition exceeds the threshold, the processor forces an <x_...> 
+    token to start coordinates, effectively ending the current content. After triggering, 
+    it enters a cooldown until a <class_...> token is seen (indicating the object was 
+    closed properly).
+    
+    Args:
+        tokenizer: The tokenizer used for encoding/decoding
+        max_repetitions: Max consecutive repetitions before forcing stop (default: 10)
+        ngram_sizes: List of n-gram sizes to check for repetition (default: [1, 2, 3, 4, 5])
+        window_size: Size of the sliding window to check for repetitions (default: 2 * max_ngram * (max_repetitions + 1))
+    """
+    
+    def __init__(
+        self,
+        tokenizer,
+        max_repetitions: int = 10,
+        ngram_sizes: List[int] = None,
+        window_size: int | None = None
+    ):
+        self.tokenizer = tokenizer
+        self.max_repetitions = max_repetitions
+        # Check various n-gram sizes for consecutive repetition
+        # n=1 catches single token repetition like <x_0.0><x_0.0><x_0.0>...
+        # n=2,3,4,5 catch phrase repetition like "ABC ABC ABC..."
+        self.ngram_sizes = ngram_sizes if ngram_sizes is not None else [1, 2, 3, 4, 5]
+        max_ngram = max((n for n in self.ngram_sizes if n > 0), default=1)
+        default_window = 2 * max_ngram * (self.max_repetitions + 1)
+        self.window_size = int(window_size) if window_size is not None else int(default_window)
+        
+        # Build set of token IDs
+        self._build_token_sets()
+        
+        # State tracking - once triggered, stay in cooldown until we see <class_...>
+        self._in_cooldown = {}  # batch_idx -> bool
+        # Track the start of the "current object segment" (tokens after the last <class_...>)
+        # We only look for repetition inside this segment to avoid old repetition re-triggering immediately.
+        self._segment_start = {}  # batch_idx -> int (seq index)
+    
+    def _build_token_sets(self):
+        """Build sets of token IDs for detection."""
+        vocab = self.tokenizer.get_vocab()
+        
+        # Collect all x_ coordinate tokens
+        self.x_token_ids: Set[int] = set()
+        self.x_token_list: List[int] = []
+        for token, token_id in vocab.items():
+            if token.startswith("<x_") and token.endswith(">"):
+                self.x_token_ids.add(token_id)
+                self.x_token_list.append(token_id)
+        
+        # Sort and pick a middle x token as default
+        self.x_token_list.sort()
+        self.default_x_token = self.x_token_list[len(self.x_token_list) // 2] if self.x_token_list else None
+        
+        # Collect all class tokens (used to reset cooldown)
+        self.class_token_ids: Set[int] = set()
+        for token, token_id in vocab.items():
+            if token.startswith("<class_") and token.endswith(">"):
+                self.class_token_ids.add(token_id)
+    
+    def _count_consecutive_repetitions(self, seq: List[int], n: int) -> int:
+        """
+        Count the maximum number of times an n-gram repeats CONSECUTIVELY.
+        
+        For example, if seq contains "ABC ABC ABC ABC DEF ABC":
+        - The n-gram "ABC" appears 4 times consecutively at the start
+        - Returns 4 (the max consecutive count)
+        
+        Returns:
+            Maximum consecutive repetition count for any n-gram
+        """
+        if len(seq) < n:
+            return 0
+        
+        max_consecutive = 1
+        current_consecutive = 1
+        
+        # Slide through the sequence comparing adjacent n-grams
+        prev_ngram = tuple(seq[0:n])
+        
+        i = n
+        while i <= len(seq) - n:
+            current_ngram = tuple(seq[i:i + n])
+            
+            if current_ngram == prev_ngram:
+                current_consecutive += 1
+                max_consecutive = max(max_consecutive, current_consecutive)
+                i += n  # Skip by n to check the next occurrence
+            else:
+                current_consecutive = 1
+                prev_ngram = current_ngram
+                i += 1  # Move by 1 to find new patterns
+        
+        return max_consecutive
+    
+    def _has_excessive_repetition(self, seq: List[int]) -> bool:
+        """
+        Check if the sequence has excessive CONSECUTIVE repetition of any n-gram.
+        
+        This detects hallucination patterns like:
+        - <x_0.0><x_0.0><x_0.0><x_0.0>... (same token repeating)
+        - "ABC ABC ABC ABC..." (same phrase repeating)
+        
+        But NOT patterns like:
+        - "& 1 & 2 & 3 & 4..." (delimiters with different content)
+        
+        Returns:
+            True if any n-gram repeats consecutively more than max_repetitions times
+        """
+        # Only check the recent window
+        check_seq = seq[-self.window_size:] if len(seq) > self.window_size else seq
+        
+        for n in self.ngram_sizes:
+            consecutive_count = self._count_consecutive_repetitions(check_seq, n)
+            if consecutive_count > self.max_repetitions:
+                return True
+        
+        return False
+    
+    def reset(self):
+        """Reset the processor state for a new generation."""
+        self._in_cooldown = {}
+        self._segment_start = {}
+    
+    def __call__(
+        self,
+        input_ids: torch.LongTensor,
+        scores: torch.FloatTensor
+    ) -> torch.FloatTensor:
+        """
+        Process logits to force <x_...> token when repetition is detected.
+        
+        Args:
+            input_ids: (batch_size, seq_len) - tokens generated so far
+            scores: (batch_size, vocab_size) - logits for next token
+            
+        Returns:
+            Modified scores with forced <x_...> tokens where repetition detected
+        """
+        batch_size = input_ids.shape[0]
+        
+        for batch_idx in range(batch_size):
+            seq = input_ids[batch_idx].tolist()
+            seq_len = len(seq)
+            
+            # Check if we just saw a class token - reset cooldown and start a new segment
+            if seq_len > 0 and seq[-1] in self.class_token_ids:
+                self._in_cooldown[batch_idx] = False
+                self._segment_start[batch_idx] = seq_len
+            
+            # If in cooldown, don't check for repetition (let model generate naturally)
+            if self._in_cooldown.get(batch_idx, False):
+                continue
+            
+            # Check if repetition threshold is exceeded
+            segment_start = self._segment_start.get(batch_idx, 0)
+            segment_seq = seq[segment_start:]
+            if self._has_excessive_repetition(segment_seq):
+                # Enter cooldown to avoid repeated triggering
+                self._in_cooldown[batch_idx] = True
+                
+                # Force one of the <x_...> tokens to start closing the object
+                # Keep the model's original preferences for which x_ coordinate to use
+                original_scores = scores[batch_idx].clone()
+                scores[batch_idx] = torch.full_like(scores[batch_idx], float('-inf'))
+                
+                # Restore original logits for x_ tokens only
+                for x_token_id in self.x_token_ids:
+                    scores[batch_idx, x_token_id] = original_scores[x_token_id]
+        
+        return scores

hf_nemotron_parse_config.py

@@ -0,0 +1,133 @@
+# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: OpenMDW-1.1
+
+from transformers import PretrainedConfig
+from typing import List, Optional
+
+
+class NemotronParseTextConfig(PretrainedConfig):
+    """
+    Configuration class for NemotronParse text decoder (mBART-based).
+    """
+    model_type = "nemotron_parse_text"
+
+    def __init__(
+        self,
+        vocab_size: int = 250027,
+        d_model: int = 1024,
+        encoder_layers: int = 12,
+        decoder_layers: int = 12,
+        encoder_attention_heads: int = 16,
+        decoder_attention_heads: int = 16,
+        decoder_ffn_dim: int = 4096,
+        encoder_ffn_dim: int = 4096,
+        activation_function: str = "gelu",
+        dropout: float = 0.1,
+        attention_dropout: float = 0.0,
+        activation_dropout: float = 0.0,
+        classifier_dropout: float = 0.0,
+        init_std: float = 0.02,
+        encoder_layerdrop: float = 0.0,
+        decoder_layerdrop: float = 0.0,
+        scale_embedding: bool = False,
+        use_cache: bool = True,
+        num_labels: int = 3,
+        forced_eos_token_id: int = 2,
+        add_cross_attention: bool = True,  # Enable cross-attention for vision-encoder-decoder
+        is_decoder: bool = True,  # This is a decoder
+        max_sequence_length: int = 9000,
+        **kwargs
+    ):
+        super().__init__(**kwargs)
+        self.vocab_size = vocab_size
+        self.d_model = d_model
+        self.encoder_layers = encoder_layers
+        self.decoder_layers = decoder_layers
+        self.encoder_attention_heads = encoder_attention_heads
+        self.decoder_attention_heads = decoder_attention_heads
+        self.decoder_ffn_dim = decoder_ffn_dim
+        self.encoder_ffn_dim = encoder_ffn_dim
+        self.activation_function = activation_function
+        self.dropout = dropout
+        self.attention_dropout = attention_dropout
+        self.activation_dropout = activation_dropout
+        self.classifier_dropout = classifier_dropout
+        self.init_std = init_std
+        self.encoder_layerdrop = encoder_layerdrop
+        self.decoder_layerdrop = decoder_layerdrop
+        self.scale_embedding = scale_embedding
+        self.use_cache = use_cache
+        self.num_labels = num_labels
+        self.add_cross_attention = add_cross_attention
+        self.is_decoder = is_decoder
+        
+        # Add hidden_size as alias for d_model (for compatibility)
+        self.hidden_size = self.d_model
+        self.forced_eos_token_id = forced_eos_token_id
+        self.num_attention_heads = self.encoder_attention_heads
+
+        self.max_sequence_length = max_sequence_length
+
+
+class NemotronParseConfig(PretrainedConfig):
+    """
+    Configuration class for NemotronParse model.
+    
+    This configuration class is used to store the configuration of a [`NemotronParseForConditionalGeneration`] model.
+    It is used to instantiate an NemotronParse model according to the specified arguments, defining the vision and text model configs.
+    """
+    model_type = "nemotron_parse"
+    is_composition = True
+    max_sequence_length = 9000
+    
+    def __init__(
+        self,
+        encoder: Optional[dict] = None,
+        decoder: Optional[dict] = None,
+        tie_word_embeddings: bool = False,
+        decoder_start_token_id: int = 2,
+        pad_token_id: int = 1,
+        eos_token_id: int = 2,
+        bos_token_id: int = 0,
+        image_size: List[int] = [2048, 1664],
+        is_encoder_decoder: bool = True,
+        max_sequence_length: int = 9000,
+        **kwargs
+    ):
+        super().__init__(
+            tie_word_embeddings=tie_word_embeddings,
+            decoder_start_token_id=decoder_start_token_id,
+            pad_token_id=pad_token_id,
+            eos_token_id=eos_token_id,
+            bos_token_id=bos_token_id,
+            max_sequence_length=max_sequence_length,
+            **kwargs
+        )
+        
+        
+        if decoder is None:
+            decoder = {}
+            
+        # Encoder tensors and architecture are part of this release. Avoid
+        # importing a second model repository while constructing the config.
+        self.encoder = PretrainedConfig(**(encoder or {}))
+
+        decoder["max_sequence_length"] = max_sequence_length
+        self.decoder = NemotronParseTextConfig(**decoder)
+        self.image_size = image_size
+        
+        # Initialize vocab size from text config
+        self.vocab_size = self.decoder.vocab_size
+        self.is_encoder_decoder = is_encoder_decoder
+        self.max_sequence_length = max_sequence_length
+        
+    def to_dict(self):
+        """
+        Serializes this instance to a Python dictionary. Override the default [`~PretrainedConfig.to_dict`].
+        """
+        output = super().to_dict()
+        output["encoder"] = self.encoder.to_dict()
+        output["decoder"] = self.decoder.to_dict()
+        output["model_type"] = self.model_type
+        output["is_encoder_decoder"] = self.is_encoder_decoder
+        return output 

hf_nemotron_parse_modeling.py

@@ -0,0 +1,275 @@
+# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: OpenMDW-1.1
+
+"""Native Transformers inference for Nemotron 3.5 Parse.
+
+The complete C-RADIOv4-SO400M encoder, content-normalized NaViT geometry,
+causal mBART visual prefix, and trained one-step MTP tensors are loaded from
+this checkpoint. No second encoder model or remote encoder code is fetched.
+"""
+
+import math
+from functools import partial
+
+import torch
+from torch import nn
+from torch.nn import functional as F
+from timm.models.vision_transformer import Block
+from transformers import GenerationMixin, PreTrainedModel
+from transformers.cache_utils import DynamicCache
+from transformers.modeling_outputs import CausalLMOutputWithPast
+
+from .hf_nemotron_parse_config import NemotronParseConfig
+from .hf_prefix_geometry import _normalized_2d_position_pages
+
+
+class _ClassTokens(nn.Module):
+    def __init__(self):
+        super().__init__()
+        self.token = nn.Parameter(torch.empty(10, 1152))
+
+
+class _PatchGenerator(nn.Module):
+    def __init__(self):
+        super().__init__()
+        self.cls_token = _ClassTokens()
+        self.embedder = nn.Linear(768, 1152, bias=False)
+        self.pos_embed = nn.Parameter(torch.empty(1, 16384, 1152))
+
+
+class _VisionTransformer(nn.Module):
+    def __init__(self):
+        super().__init__()
+        self.patch_generator = _PatchGenerator()
+        self.blocks = nn.ModuleList([
+            Block(1152, 16, mlp_ratio=4304 / 1152, qkv_bias=True,
+                  norm_layer=partial(nn.LayerNorm, eps=1e-6))
+            for _ in range(27)
+        ])
+
+    def forward(self, patches, image_size, content_size):
+        pg = self.patch_generator
+        patches = pg.embedder(patches).unsqueeze(0)
+        patches = _normalized_2d_position_pages(
+            torch, patches=patches, pos_embed=pg.pos_embed,
+            source_rows=128, source_cols=128, image_sizes=[image_size],
+            patch_size=16, content_sizes=[content_size], content_normalized=True,
+        )
+        hidden = torch.cat((pg.cls_token.token.unsqueeze(0), patches), dim=1)
+        for block in self.blocks:
+            hidden = block(hidden)
+        return hidden
+
+
+class _InputConditioner(nn.Module):
+    def __init__(self):
+        super().__init__()
+        self.register_buffer("norm_mean", torch.zeros(3, 1, 1))
+        self.register_buffer("norm_std", torch.ones(3, 1, 1))
+
+
+class _RadioModel(nn.Module):
+    def __init__(self):
+        super().__init__()
+        self.register_buffer("summary_idxs", torch.tensor([0, 1]))
+        self.input_conditioner = _InputConditioner()
+        self.model = _VisionTransformer()
+
+
+class _RadioWrapper(nn.Module):
+    def __init__(self):
+        super().__init__()
+        self.radio_model = _RadioModel()
+
+
+class RadioWithNeck(nn.Module):
+    def __init__(self, config):
+        super().__init__()
+        self.model_encoder = _RadioWrapper()
+        self.conv1 = nn.Conv1d(1152, 1024, 1)
+        self.layer_norm1 = nn.LayerNorm(1024, eps=1e-6)
+        self.conv2 = nn.Conv2d(1024, 1024, (1, 4), stride=(1, 4), bias=False)
+        self.layer_norm2 = nn.LayerNorm(1024, eps=1e-6)
+        self.sum_proj = nn.Linear(2304, 1024)
+        self.layer_norm3 = nn.LayerNorm(1024, eps=1e-6)
+
+    def forward(self, pixel_values, image_sizes, content_sizes):
+        # The shipped processor has already applied CLIP normalization and
+        # flattened each 16x16 patch in channel/row/column order.
+        if image_sizes is None or content_sizes is None:
+            raise ValueError("Packed inference requires image_sizes and content_sizes from AutoProcessor")
+        if pixel_values.ndim == 3 and pixel_values.shape[0] == 1:
+            pixel_values = pixel_values[0]
+        if pixel_values.ndim != 2 or pixel_values.shape[1] != 768:
+            raise ValueError(f"Expected packed raw patches [patches,768], got {tuple(pixel_values.shape)}")
+        pixel_values = pixel_values.to(dtype=self.conv1.weight.dtype)
+        pages, cursor = [], 0
+        radio = self.model_encoder.radio_model
+        for size, content in zip(image_sizes.tolist(), content_sizes.tolist()):
+            height, width = map(int, size)
+            count = (height // 16) * (width // 16)
+            hidden = radio.model(pixel_values[cursor:cursor + count], size, content)
+            cursor += count
+            summary = hidden[:, radio.summary_idxs].flatten(1)
+            features = hidden[:, 10:]
+            features = self.layer_norm1(self.conv1(features.transpose(1, 2)).transpose(1, 2))
+            features = features.transpose(1, 2).reshape(1, 1024, height // 16, width // 16)
+            features = self.conv2(features).flatten(2).transpose(1, 2)
+            features = self.layer_norm2(features)
+            summary = self.layer_norm3(self.sum_proj(summary)).unsqueeze(1)
+            pages.append(torch.cat((features, summary), dim=1)[0])
+        if cursor != len(pixel_values):
+            raise ValueError("Image metadata did not consume all packed patches")
+        max_length = max(len(page) for page in pages)
+        output = pages[0].new_zeros(len(pages), max_length, 1024)
+        mask = torch.zeros(len(pages), max_length, device=output.device, dtype=torch.bool)
+        for i, page in enumerate(pages):
+            output[i, :len(page)] = page
+            mask[i, :len(page)] = True
+        return output, mask
+
+
+class _ScaledEmbedding(nn.Embedding):
+    def __init__(self, config):
+        super().__init__(config.vocab_size, config.d_model, padding_idx=config.pad_token_id)
+        self.embed_scale = math.sqrt(config.d_model) if config.scale_embedding else 1.0
+
+    def forward(self, ids):
+        return super().forward(ids) * self.embed_scale
+
+
+class _GQAAttention(nn.Module):
+    def __init__(self, config, index):
+        super().__init__()
+        self.index = index
+        self.num_heads = config.decoder_attention_heads
+        self.num_kv_heads = config.self_attention_kv_heads
+        self.head_dim = config.d_model // self.num_heads
+        self.q_proj = nn.Linear(config.d_model, config.d_model)
+        self.k_proj = nn.Linear(config.d_model, self.num_kv_heads * self.head_dim)
+        self.v_proj = nn.Linear(config.d_model, self.num_kv_heads * self.head_dim)
+        self.out_proj = nn.Linear(config.d_model, config.d_model)
+
+    def forward(self, hidden, cache, key_mask):
+        batch, length, _ = hidden.shape
+        q = self.q_proj(hidden).view(batch, length, self.num_heads, self.head_dim).transpose(1, 2)
+        k = self.k_proj(hidden).view(batch, length, self.num_kv_heads, self.head_dim).transpose(1, 2)
+        v = self.v_proj(hidden).view(batch, length, self.num_kv_heads, self.head_dim).transpose(1, 2)
+        if cache is not None:
+            k, v = cache.update(k, v, self.index)
+        past = k.shape[-2] - length
+        allowed = torch.arange(k.shape[-2], device=hidden.device)[None, :] <= (
+            torch.arange(length, device=hidden.device)[:, None] + past
+        )
+        allowed = allowed[None, None] & key_mask[:, None, None, :]
+        repeat = self.num_heads // self.num_kv_heads
+        output = F.scaled_dot_product_attention(
+            q, k.repeat_interleave(repeat, dim=1), v.repeat_interleave(repeat, dim=1),
+            attn_mask=allowed, dropout_p=0.0,
+        )
+        return self.out_proj(output.transpose(1, 2).reshape(batch, length, -1))
+
+
+class _DecoderLayer(nn.Module):
+    def __init__(self, config, index):
+        super().__init__()
+        self.self_attn = _GQAAttention(config, index)
+        self.self_attn_layer_norm = nn.LayerNorm(config.d_model)
+        self.final_layer_norm = nn.LayerNorm(config.d_model)
+        self.fc1 = nn.Linear(config.d_model, config.decoder_ffn_dim)
+        self.fc2 = nn.Linear(config.decoder_ffn_dim, config.d_model)
+
+    def forward(self, hidden, cache, mask):
+        hidden = hidden + self.self_attn(self.self_attn_layer_norm(hidden), cache, mask)
+        return hidden + self.fc2(F.gelu(self.fc1(self.final_layer_norm(hidden))))
+
+
+class NemotronParseDecoder(nn.Module):
+    def __init__(self, config, num_extra_heads):
+        super().__init__()
+        self.embed_tokens = _ScaledEmbedding(config)
+        self.layers = nn.ModuleList([_DecoderLayer(config, i) for i in range(config.decoder_layers)])
+        self.layernorm_embedding = nn.LayerNorm(config.d_model)
+        self.layer_norm = nn.LayerNorm(config.d_model)
+        self.extra_heads = nn.ModuleList([nn.Linear(config.d_model, config.d_model) for _ in range(num_extra_heads)])
+        self.extra_proj = nn.ModuleList([nn.Linear(config.d_model, config.d_model) for _ in range(num_extra_heads)])
+
+    def forward(self, embeddings, cache, mask):
+        hidden = self.layernorm_embedding(embeddings)
+        for layer in self.layers:
+            hidden = layer(hidden, cache, mask)
+        return self.layer_norm(hidden)
+
+
+class NemotronParseForConditionalGeneration(PreTrainedModel, GenerationMixin):
+    config_class = NemotronParseConfig
+    main_input_name = "input_ids"
+    _tied_weights_keys = {"lm_head.weight": "decoder.embed_tokens.weight"}
+    _no_split_modules = ["RadioWithNeck", "Block", "_DecoderLayer"]
+    _supports_sdpa = True
+
+    def __init__(self, config):
+        super().__init__(config)
+        if not getattr(config, "legacy_causal_vision_prefix", False):
+            raise ValueError("This release implements the trained causal visual-prefix architecture")
+        self.encoder = RadioWithNeck(config.encoder)
+        self.decoder = NemotronParseDecoder(config.decoder, getattr(config, "num_extra_heads", 0))
+        self.lm_head = nn.Linear(config.decoder.d_model, config.decoder.vocab_size, bias=False)
+        self.post_init()
+
+    def get_input_embeddings(self):
+        return self.decoder.embed_tokens
+
+    def get_output_embeddings(self):
+        return self.lm_head
+
+    def _tie_weights(self):
+        self._tie_or_clone_weights(self.lm_head, self.decoder.embed_tokens)
+
+    def forward(self, input_ids=None, decoder_input_ids=None, pixel_values=None,
+                image_sizes=None, content_sizes=None, attention_mask=None,
+                past_key_values=None, use_cache=True, return_dict=True, **kwargs):
+        del kwargs
+        if input_ids is None:
+            input_ids = decoder_input_ids
+        if input_ids is None:
+            raise ValueError("input_ids are required")
+        if past_key_values is None and use_cache:
+            past_key_values = DynamicCache()
+        past_length = past_key_values.get_seq_length() if past_key_values is not None else 0
+        text_embeddings = self.decoder.embed_tokens(input_ids)
+        text_offset = 0
+        if past_length == 0:
+            if pixel_values is None:
+                raise ValueError("pixel_values are required for the first generation step")
+            vision, visual_mask = self.encoder(pixel_values, image_sizes, content_sizes)
+            embeddings = torch.cat((vision, text_embeddings), dim=1)
+            text_offset = vision.shape[1]
+            if past_key_values is not None:
+                past_key_values._nemotron_visual_mask = visual_mask
+        else:
+            embeddings = text_embeddings
+            visual_mask = past_key_values._nemotron_visual_mask
+        if attention_mask is None:
+            text_length = past_length + input_ids.shape[1] - visual_mask.shape[1]
+            if past_length == 0:
+                text_length = input_ids.shape[1]
+            attention_mask = torch.ones(input_ids.shape[0], text_length, device=input_ids.device, dtype=torch.bool)
+        mask = torch.cat((visual_mask, attention_mask.bool()), dim=1)
+        if mask.shape[1] > self.config.max_sequence_length:
+            raise ValueError("Visual prefix plus text exceeds the model context limit")
+        hidden = self.decoder(embeddings, past_key_values if use_cache else None, mask)
+        logits = self.lm_head(hidden[:, text_offset:])
+        output = CausalLMOutputWithPast(logits=logits, past_key_values=past_key_values if use_cache else None)
+        return output if return_dict else (output.logits, output.past_key_values)
+
+    def prepare_inputs_for_generation(self, input_ids, past_key_values=None,
+                                      attention_mask=None, pixel_values=None,
+                                      image_sizes=None, content_sizes=None, **kwargs):
+        del kwargs
+        if past_key_values is not None and past_key_values.get_seq_length() > 0:
+            input_ids = input_ids[:, -1:]
+            pixel_values = None
+        return {"input_ids": input_ids, "attention_mask": attention_mask,
+                "past_key_values": past_key_values, "pixel_values": pixel_values,
+                "image_sizes": image_sizes, "content_sizes": content_sizes, "use_cache": True}

hf_nemotron_parse_processor.py

@@ -0,0 +1,501 @@
+# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: OpenMDW-1.1
+
+import numpy as np
+from PIL import Image
+from typing import List, Optional, Union, Dict, Any
+import torch
+from torchvision import transforms as T
+import json
+
+from transformers import BatchFeature, ProcessorMixin, BaseImageProcessor, ImageProcessingMixin
+from transformers.image_utils import ChannelDimension, ImageInput, PILImageResampling, infer_channel_dimension_format
+from transformers.utils import TensorType
+
+OPENAI_CLIP_MEAN = (0.48145466, 0.4578275, 0.40821073)
+OPENAI_CLIP_STD = (0.26862954, 0.26130258, 0.27577711)
+
+
+from transformers.processing_utils import ImagesKwargs
+
+class NemotronParseImagesKwargs(ImagesKwargs, total=False):
+    content_hw: Optional[List[int]]
+
+class NemotronParseImageProcessor(BaseImageProcessor, ImageProcessingMixin):
+    """
+    Image processor for NemotronParse model.
+    
+    This processor inherits from BaseImageProcessor to be compatible with transformers AutoImageProcessor.
+    """
+    
+    valid_kwargs = NemotronParseImagesKwargs
+    model_input_names = ["pixel_values", "image_sizes", "content_sizes"]
+    
+    def __init__(
+        self,
+        final_size: tuple = (2048, 1664),
+        do_normalize: bool = False,
+        image_mean: Optional[Union[List[float], tuple]] = None,
+        image_std: Optional[Union[List[float], tuple]] = None,
+        **kwargs,
+    ):
+        self.navit_packing = bool(kwargs.pop('navit_packing', False))
+        self.navit_padding_mode = kwargs.pop('navit_padding_mode', 'top_left_white')
+        self.navit_coordinate_frame = kwargs.pop('navit_coordinate_frame', 'content_normalized')
+        clean_kwargs = {}
+        for k, v in kwargs.items():
+            if not k.startswith('_') and k not in ['transform', 'torch_transform']:
+                clean_kwargs[k] = v
+        
+        if 'size' in clean_kwargs:
+            size_config = clean_kwargs.pop('size')
+            if isinstance(size_config, dict):
+                if 'longest_edge' in size_config:
+                    longest_edge = size_config['longest_edge']
+                    if isinstance(longest_edge, (list, tuple)):
+                        final_size = tuple(int(x) for x in longest_edge)
+                    else:
+                        final_size = (int(longest_edge), int(longest_edge))
+                elif 'height' in size_config and 'width' in size_config:
+                    final_size = (int(size_config['height']), int(size_config['width']))
+        
+        super().__init__(**clean_kwargs)
+        self.do_normalize = bool(do_normalize)
+        self.image_mean = list(image_mean or OPENAI_CLIP_MEAN)
+        self.image_std = list(image_std or OPENAI_CLIP_STD)
+        
+        if isinstance(final_size, (list, tuple)) and len(final_size) >= 2:
+            self.final_size = (int(final_size[0]), int(final_size[1]))
+        elif isinstance(final_size, (int, float)):
+            self.final_size = (int(final_size), int(final_size))
+        else:
+            self.final_size = (2048, 1664)  # Default fallback
+        
+        self._create_transforms()
+    
+    def _create_transforms(self):
+        """Create transform objects (not serialized to JSON)."""
+        if isinstance(self.final_size, (list, tuple)):
+            self.target_height, self.target_width = int(self.final_size[0]), int(self.final_size[1])
+        else:
+            self.target_height = self.target_width = int(self.final_size)
+        
+        self.transform = None
+        
+        self.torch_transform = T.Compose([
+            T.ToTensor(),
+        ])
+
+    def to_dict(self):
+        """Override to exclude non-serializable transforms."""
+        output = super().to_dict()
+        output.pop('transform', None)
+        output.pop('torch_transform', None)
+        return output
+    
+    @classmethod
+    def from_dict(cls, config_dict: dict, **kwargs):
+        """Override to recreate transforms after loading."""
+        config_dict = config_dict.copy()
+        config_dict.pop('transform', None)
+        config_dict.pop('torch_transform', None)
+        
+        # Clean any problematic entries
+        for key in list(config_dict.keys()):
+            if key.startswith('_') or config_dict[key] is None:
+                config_dict.pop(key, None)
+        
+        # Ensure numeric types are correct
+        if 'final_size' in config_dict:
+            final_size = config_dict['final_size']
+            if isinstance(final_size, (list, tuple)):
+                config_dict['final_size'] = tuple(int(x) for x in final_size)
+        if 'image_mean' in config_dict:
+            config_dict['image_mean'] = [float(x) for x in config_dict['image_mean']]
+        if 'image_std' in config_dict:
+            config_dict['image_std'] = [float(x) for x in config_dict['image_std']]
+        
+        try:
+            return cls(**config_dict, **kwargs)
+        except Exception as e:
+            print(f"Warning: Error in from_dict: {e}")
+            print("Using default parameters...")
+            return cls(**kwargs)
+    
+    def save_pretrained(self, save_directory, **kwargs):
+        """Save image processor configuration."""
+        import os
+        import json
+        
+        os.makedirs(save_directory, exist_ok=True)
+        
+        # Save preprocessor config in standard HuggingFace format
+        config = {
+            "feature_extractor_type": "NemotronParseImageProcessor",
+            "image_processor_type": "NemotronParseImageProcessor", 
+            "processor_class": "NemotronParseProcessor",
+            "size": {
+                "height": self.final_size[0],
+                "width": self.final_size[1],
+                "longest_edge": self.final_size
+            },
+            "final_size": self.final_size,
+            "do_normalize": self.do_normalize,
+            "navit_packing": self.navit_packing,
+            "navit_padding_mode": self.navit_padding_mode,
+            "navit_coordinate_frame": self.navit_coordinate_frame,
+            "image_mean": self.image_mean,
+            "image_std": self.image_std,
+        }
+        
+        config_path = os.path.join(save_directory, "preprocessor_config.json")
+        with open(config_path, 'w') as f:
+            json.dump(config, f, indent=2)
+
+    def _resize_with_aspect_ratio(self, image: np.ndarray) -> np.ndarray:
+        """Resize image maintaining aspect ratio (exact replica of original LongestMaxSizeHW)."""
+        height, width = image.shape[:2]
+        max_size_height = self.target_height
+        max_size_width = self.target_width
+        
+        # Original LongestMaxSizeHW algorithm from custom_augmentations.py
+        aspect_ratio = width / height
+        new_height = height
+        new_width = width
+
+        if height > max_size_height:
+            new_height = max_size_height
+            new_width = int(new_height * aspect_ratio)
+
+        if new_width > max_size_width:
+            new_width = max_size_width
+            new_height = int(new_width / aspect_ratio)
+        
+        if new_height == height and new_width == width:
+            return image
+        resampling = getattr(Image, "Resampling", Image).BILINEAR
+        return np.asarray(Image.fromarray(image).resize((new_width, new_height), resampling))
+    
+    def _pad_to_size(self, image: np.ndarray) -> np.ndarray:
+        """Center-pad image to target size with white padding."""
+        h, w = image.shape[:2]
+        min_height, min_width = self.target_height, self.target_width
+        
+        pad_h = max(0, min_height - h)
+        pad_w = max(0, min_width - w)
+        
+        if pad_h == 0 and pad_w == 0:
+            return image
+        
+        pad_top = pad_h // 2
+        pad_bottom = pad_h - pad_top
+        pad_left = pad_w // 2
+        pad_right = pad_w - pad_left
+        
+        if len(image.shape) == 3:
+            return np.pad(
+                image,
+                ((pad_top, pad_bottom), (pad_left, pad_right), (0, 0)),
+                mode='constant',
+                constant_values=255,
+            )
+        return np.pad(
+            image,
+            ((pad_top, pad_bottom), (pad_left, pad_right)),
+            mode='constant',
+            constant_values=255,
+        )
+    
+    def _pad_to_navit_grid(self, image: np.ndarray) -> np.ndarray:
+        height, width = image.shape[:2]
+        target_height = ((height + 15) // 16) * 16
+        target_width = ((width + 63) // 64) * 64
+        pad_h, pad_w = target_height - height, target_width - width
+        if not pad_h and not pad_w:
+            return image
+        padding = ((0, pad_h), (0, pad_w), (0, 0)) if image.ndim == 3 else ((0, pad_h), (0, pad_w))
+        return np.pad(image, padding, mode="constant", constant_values=255)
+
+    def preprocess(
+        self,
+        images: ImageInput,
+        return_tensors: Optional[Union[str, TensorType]] = None,
+        **kwargs,
+    ) -> Dict[str, torch.Tensor]:
+        """
+        Preprocess an image or batch of images for the NemotronParse model.
+        
+        Args:
+            images: Input image(s)
+            return_tensors: Type of tensors to return
+        """
+        
+        captured_hw = kwargs.pop("content_hw", None)
+        image_options = kwargs.pop("images_kwargs", {}) or {}
+        nested_hw = image_options.get("content_hw")
+        if captured_hw is not None and nested_hw is not None and list(captured_hw) != list(nested_hw):
+            raise ValueError("conflicting captured content dimensions")
+        if captured_hw is None:
+            captured_hw = nested_hw
+        if captured_hw is not None:
+            if not self.navit_packing or self.navit_coordinate_frame != "content_normalized":
+                raise ValueError("loader capture requires content-normalized NaViT")
+            if self.navit_padding_mode != "top_left_white":
+                raise ValueError("unsupported captured loader pixel contract")
+            pages = images if isinstance(images, list) else [images]
+            if len(pages) != 1 or len(captured_hw) != 2:
+                raise ValueError("one captured page and [content_height, content_width] required")
+            pixels = np.array(pages[0].convert("RGB"), copy=True)
+            height, width = pixels.shape[:2]
+            ch, cw = (int(value) for value in captured_hw)
+            if list(captured_hw) != [ch, cw] or not (0 < ch <= height and 0 < cw <= width):
+                raise ValueError("invalid captured content dimensions")
+            if height % 16 or width % 64:
+                raise ValueError("captured canvas is not aligned to the NaViT grid")
+            page = torch.from_numpy(pixels).permute(2, 0, 1).contiguous().float().div_(255.0)
+            # Native RADIO normalizes inside its encoder; the vLLM boundary
+            # expects the equivalent normalization here, as in ordinary serving.
+            if self.do_normalize:
+                mean = page.new_tensor(self.image_mean).view(-1, 1, 1)
+                std = page.new_tensor(self.image_std).view(-1, 1, 1)
+                page = (page - mean) / std
+            patches = page.unfold(1, 16, 16).unfold(2, 16, 16)
+            patches = patches.permute(1, 2, 0, 3, 4).reshape(-1, 3 * 16 * 16)
+            return BatchFeature(data={
+                "pixel_values": patches,
+                "image_sizes": torch.tensor([[height, width]], dtype=torch.long),
+                "content_sizes": torch.tensor([[ch, cw]], dtype=torch.long),
+            }, tensor_type=return_tensors)
+
+        # Ensure images is a list
+        if not isinstance(images, list):
+            images = [images]
+
+        # Ensure images are RGB
+        for i in range(len(images)):
+            images[i] = images[i].convert('RGB')
+        
+        # Convert PIL images to numpy arrays if needed
+        processed_images = []
+        for image in images:
+            if isinstance(image, Image.Image):
+                image = np.asarray(image)
+            processed_images.append(image)
+        
+        # Apply NemotronParse-specific transforms
+        pixel_values = []
+        content_sizes = []
+        for image in processed_images:
+            processed_image = self._resize_with_aspect_ratio(image)
+            content_sizes.append(
+                [int(processed_image.shape[0]), int(processed_image.shape[1])]
+            )
+            if self.navit_packing:
+                processed_image = self._pad_to_navit_grid(processed_image)
+            
+            if self.transform is not None:
+                transformed = self.transform(image=processed_image)
+                processed_image = transformed["image"]
+            else:
+                # Fixed exports retain the historical full-canvas pad.
+                if not self.navit_packing:
+                    processed_image = self._pad_to_size(processed_image)
+            
+            pixel_values_tensor = self.torch_transform(processed_image)
+            
+            if pixel_values_tensor.shape[0] == 1:
+                pixel_values_tensor = pixel_values_tensor.expand(3, -1, -1)
+                
+            pixel_values.append(pixel_values_tensor)
+                        
+        if self.navit_packing:
+            content_sizes = torch.tensor(content_sizes, dtype=torch.long)
+            image_sizes = torch.tensor(
+                [[int(page.shape[-2]), int(page.shape[-1])] for page in pixel_values],
+                dtype=torch.long,
+            )
+            raw_patches = []
+            for page in pixel_values:
+                if self.do_normalize:
+                    mean = page.new_tensor(self.image_mean).view(-1, 1, 1)
+                    std = page.new_tensor(self.image_std).view(-1, 1, 1)
+                    page = (page - mean) / std
+                patches = page.unfold(1, 16, 16).unfold(2, 16, 16)
+                patches = patches.permute(1, 2, 0, 3, 4).reshape(-1, 3 * 16 * 16)
+                raw_patches.append(patches)
+            pixel_values = torch.cat(raw_patches, dim=0)
+            data = {
+                "pixel_values": pixel_values,
+                "image_sizes": image_sizes,
+                "content_sizes": content_sizes,
+            }
+        else:
+            pixel_values = torch.stack(pixel_values)
+            if self.do_normalize:
+                mean = pixel_values.new_tensor(self.image_mean).view(1, -1, 1, 1)
+                std = pixel_values.new_tensor(self.image_std).view(1, -1, 1, 1)
+                pixel_values = (pixel_values - mean) / std
+            data = {"pixel_values": pixel_values}
+        
+        if return_tensors is not None:
+            data = self._convert_output_format(data, return_tensors)
+            
+        return data
+    
+    def _convert_output_format(self, data: Dict[str, torch.Tensor], return_tensors: Union[str, TensorType]) -> Dict:
+        """Convert output format based on return_tensors parameter."""
+        if return_tensors == "pt" or return_tensors == TensorType.PYTORCH:
+            return data
+        elif return_tensors == "np" or return_tensors == TensorType.NUMPY:
+            return {k: v.numpy() for k, v in data.items()}
+        else:
+            return data
+    
+    def __call__(self, images: Union[Image.Image, List[Image.Image]], **kwargs) -> Dict[str, torch.Tensor]:
+        """Process images for the model (backward compatibility)."""
+        return self.preprocess(images, **kwargs)
+
+
+class NemotronParseProcessor(ProcessorMixin):
+    
+    attributes = ["image_processor", "tokenizer"]
+    image_processor_class = "AutoImageProcessor"
+    tokenizer_class = ("PreTrainedTokenizer", "PreTrainedTokenizerFast")
+    
+    def __init__(self, image_processor=None, tokenizer=None, **kwargs):
+        if image_processor is None:
+            image_processor = NemotronParseImageProcessor(**kwargs)
+               
+        super().__init__(image_processor, tokenizer)
+            
+    
+    def __call__(
+        self,
+        images: Union[Image.Image, List[Image.Image]] = None,
+        text: Union[str, List[str]] = None,
+        add_special_tokens: bool = True,
+        padding: Union[bool, str] = False,
+        truncation: Union[bool, str] = False,
+        max_length: Optional[int] = None,
+        stride: int = 0,
+        pad_to_multiple_of: Optional[int] = None,
+        return_attention_mask: Optional[bool] = None,
+        return_overflowing_tokens: bool = False,
+        return_special_tokens_mask: bool = False,
+        return_offsets_mapping: bool = False,
+        return_token_type_ids: bool = False,
+        return_length: bool = False,
+        verbose: bool = True,
+        return_tensors: Optional[Union[str, "TensorType"]] = None,
+        content_hw: Optional[List[int]] = None,
+        **kwargs
+    ) -> BatchFeature:
+        """
+        Main method to prepare for the model one or several text(s) and image(s).
+        """
+        
+        # Process images
+        if images is not None:
+            image_inputs = self.image_processor(
+                images, content_hw=content_hw, **kwargs)
+        else:
+            image_inputs = {}
+        
+        # Process text
+        if text is not None:
+            text_inputs = self.tokenizer(
+                text,
+                add_special_tokens=add_special_tokens,
+                padding=padding,
+                truncation=truncation,
+                max_length=max_length,
+                stride=stride,
+                pad_to_multiple_of=pad_to_multiple_of,
+                return_attention_mask=return_attention_mask,
+                return_overflowing_tokens=return_overflowing_tokens,
+                return_special_tokens_mask=return_special_tokens_mask,
+                return_offsets_mapping=return_offsets_mapping,
+                return_token_type_ids=return_token_type_ids,
+                return_length=return_length,
+                verbose=verbose,
+                return_tensors=return_tensors,
+                **kwargs,
+            )
+        else:
+            text_inputs = {}
+        
+        # Combine inputs
+        return BatchFeature(data={**image_inputs, **text_inputs})
+    
+    def decode(self, *args, **kwargs):
+        """Decode token ids to strings."""
+        return self.tokenizer.decode(*args, **kwargs)
+    
+    def batch_decode(self, *args, **kwargs):
+        """Batch decode token ids to strings."""
+        return self.tokenizer.batch_decode(*args, **kwargs)
+    
+    def post_process_generation(self, sequences, fix_markdown=False):
+        """Post-process generated sequences."""
+        if hasattr(self.tokenizer, 'post_process_generation'):
+            return self.tokenizer.post_process_generation(sequences, fix_markdown=fix_markdown)
+        else:
+            # Fallback processing
+            if isinstance(sequences, str):
+                sequences = [sequences]
+            
+            processed = []
+            for seq in sequences:
+                # Basic cleaning
+                seq = seq.replace('<s>', '').replace('</s>', '').strip()
+                processed.append(seq)
+            
+            return processed[0] if len(processed) == 1 else processed
+    
+    @classmethod
+    def from_pretrained(cls, pretrained_model_name_or_path, **kwargs):
+        """
+        Load processor from pretrained model.
+        
+        This method is compatible with AutoProcessor.from_pretrained().
+        """
+        # Explicitly load subcomponents via Auto* to ensure remote auto_map is honored.
+        from transformers import AutoImageProcessor, AutoTokenizer
+        trust_remote_code = kwargs.get("trust_remote_code", None)
+        revision = kwargs.get("revision", None)
+        token = kwargs.get("token", None)
+        image_processor = AutoImageProcessor.from_pretrained(
+            pretrained_model_name_or_path,
+            trust_remote_code=trust_remote_code,
+            revision=revision,
+            token=token,
+        )
+        tokenizer = AutoTokenizer.from_pretrained(
+            pretrained_model_name_or_path,
+            trust_remote_code=trust_remote_code,
+            revision=revision,
+            token=token,
+        )
+        return cls(image_processor=image_processor, tokenizer=tokenizer)
+    
+    def save_pretrained(self, save_directory, **kwargs):
+        """
+        Save processor to directory.
+        
+        This method is compatible with AutoProcessor/AutoImageProcessor loading.
+        """
+        import os
+        os.makedirs(save_directory, exist_ok=True)
+        
+        # Save tokenizer with proper configuration for AutoTokenizer
+        print("Saving tokenizer for AutoTokenizer compatibility...")
+        self.tokenizer.save_pretrained(save_directory, **kwargs)
+        
+        # Save image processor  
+        print("Saving image processor...")
+        self.image_processor.save_pretrained(save_directory, **kwargs)
+        
+        # Use the parent class's save_pretrained method for processor config
+        super().save_pretrained(save_directory, **kwargs)
+        print(f"NemotronParseProcessor saved to {save_directory}")
+        print(f"AutoTokenizer.from_pretrained('{save_directory}') should now work!")

hf_prefix_geometry.py

@@ -0,0 +1,149 @@
+# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: OpenMDW-1.1
+
+"""Learned content-normalized positional interpolation shared with serving."""
+
+def _normalized_2d_position_pages(
+    torch_module,
+    *,
+    patches,
+    pos_embed,
+    source_rows: int,
+    source_cols: int,
+    image_sizes,
+    patch_size: int,
+    content_sizes=None,
+    content_normalized: bool = False,
+):
+    """Mirror training's per-page normalized-2D position interpolation.
+
+    Keep this helper independent of vLLM imports so the export contract can be
+    exercised on the login node.  vLLM supplies one concatenated raw-patch
+    sequence, whereas training embeds each page separately; splitting before
+    interpolation makes the two representations numerically identical.
+    """
+
+    source_rows = int(source_rows)
+    source_cols = int(source_cols)
+    patch_size = int(patch_size)
+    if source_rows <= 0 or source_cols <= 0 or patch_size <= 0:
+        raise ValueError(
+            "normalized 2D positions require positive source grid and patch size"
+        )
+    if source_rows * source_cols != int(pos_embed.shape[1]):
+        raise ValueError(
+            "RADIO learned positional field shape is inconsistent: "
+            f"source={source_rows}x{source_cols} tokens={int(pos_embed.shape[1])}"
+        )
+
+    positional_grid = pos_embed.reshape(
+        1, source_rows, source_cols, -1
+    ).permute(0, 3, 1, 2)
+    normalized_content_sizes = None
+    if content_sizes is not None:
+        normalized_content_sizes = [
+            tuple(int(value) for value in size) for size in content_sizes
+        ]
+        if len(normalized_content_sizes) != len(image_sizes):
+            raise ValueError(
+                "Content-size count must match image-size count: "
+                f"content_sizes={len(normalized_content_sizes)} "
+                f"image_sizes={len(image_sizes)}"
+            )
+    if content_normalized and normalized_content_sizes is None:
+        raise ValueError("content-normalized 2D positions require content_sizes")
+
+    positioned_pages = []
+    cursor = 0
+    for image_index, (raw_height, raw_width) in enumerate(image_sizes):
+        height, width = int(raw_height), int(raw_width)
+        if height % patch_size or width % patch_size:
+            raise ValueError(
+                f"Page shape {(height, width)} is not aligned to patch size {patch_size}"
+            )
+        rows, cols = height // patch_size, width // patch_size
+        patch_count = rows * cols
+        page = patches[:, cursor : cursor + patch_count]
+        if int(page.shape[1]) != patch_count:
+            raise ValueError(
+                "Normalized position split ended inside a page: "
+                f"page={(height, width)} expected={patch_count} actual={int(page.shape[1])}"
+            )
+        cursor += patch_count
+        if content_normalized:
+            content_height, content_width = normalized_content_sizes[image_index]
+            if content_height <= 0 or content_width <= 0:
+                raise ValueError(
+                    "content size must be positive, got "
+                    f"{(content_height, content_width)}"
+                )
+            if content_height > height or content_width > width:
+                raise ValueError(
+                    "content size cannot exceed padded image: "
+                    f"content={(content_height, content_width)} image={(height, width)}"
+                )
+            content_rows = (content_height + patch_size - 1) // patch_size
+            content_cols = (content_width + patch_size - 1) // patch_size
+            if min(content_rows, content_cols) == 1:
+                # Match training's safe area-preserving path for augmentation
+                # crops whose visible content spans only one patch in an axis.
+                page_positions = torch_module.nn.functional.adaptive_avg_pool2d(
+                    positional_grid.float(), (content_rows, content_cols)
+                )
+            else:
+                page_positions = torch_module.nn.functional.interpolate(
+                    positional_grid.float(),
+                    size=(content_rows, content_cols),
+                    mode="bilinear",
+                    align_corners=False,
+                    antialias=True,
+                )
+            row_starts = (
+                torch_module.arange(rows, device=page.device, dtype=torch_module.float32)
+                * patch_size
+            )
+            col_starts = (
+                torch_module.arange(cols, device=page.device, dtype=torch_module.float32)
+                * patch_size
+            )
+            row_extents = (content_height - row_starts).clamp(
+                min=0.0, max=float(patch_size)
+            )
+            col_extents = (content_width - col_starts).clamp(
+                min=0.0, max=float(patch_size)
+            )
+            normalized_y = (
+                row_starts + 0.5 * row_extents
+            ).clamp(max=float(content_height)) / float(content_height)
+            normalized_x = (
+                col_starts + 0.5 * col_extents
+            ).clamp(max=float(content_width)) / float(content_width)
+            grid_y, grid_x = torch_module.meshgrid(
+                normalized_y, normalized_x, indexing="ij"
+            )
+            sampling_grid = torch_module.stack(
+                (2.0 * grid_x - 1.0, 2.0 * grid_y - 1.0), dim=-1
+            )
+            page_positions = torch_module.nn.functional.grid_sample(
+                page_positions,
+                sampling_grid.unsqueeze(0),
+                mode="bilinear",
+                padding_mode="border",
+                align_corners=False,
+            ).to(dtype=page.dtype)
+        else:
+            page_positions = torch_module.nn.functional.interpolate(
+                positional_grid.float(),
+                size=(rows, cols),
+                mode="bilinear",
+                align_corners=False,
+                antialias=True,
+            ).to(dtype=page.dtype)
+        page_positions = page_positions.flatten(2).permute(0, 2, 1)
+        positioned_pages.append(page + page_positions)
+    if cursor != int(patches.shape[1]):
+        raise ValueError(
+            f"Normalized position split consumed {cursor} of {int(patches.shape[1])} patches"
+        )
+    return torch_module.cat(positioned_pages, dim=1)
+

latex2html.py

@@ -0,0 +1,11 @@
+# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: OpenMDW-1.1
+"""Compatibility imports; implementation lives in nemotron_parse.tables."""
+from nemotron_parse.tables import (
+    latex_table_to_html,
+    convert_single_table,
+    convert_html_tables_to_markdown,
+    convert_html_table_to_json,
+    convert_html_table_to_json_hierarchical,
+    convert_html_table_to_csv,
+)

logitsprocs/nemotron_parse_vllm_logitprocs.py

@@ -0,0 +1,350 @@
+# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: OpenMDW-1.1
+
+"""
+Custom vLLM v1 logits processors for NVIDIA Nemotron Parse.
+
+Tuning via environment variables:
+  - NEMOTRON_PARSE_TABLE_PREFIX (default: \\begin{tabular})
+  - NEMOTRON_PARSE_REP_MAX      (default: 10)
+  - NEMOTRON_PARSE_REP_WINDOW   (default: 2 * max_ngram * (rep_max + 1))
+  - NEMOTRON_PARSE_REP_NGRAMS   (default: 1,2,3,4,5)
+"""
+
+from __future__ import annotations
+
+import os
+import warnings
+from dataclasses import dataclass
+from typing import Dict, List, Set
+
+import torch
+from transformers import AutoTokenizer
+
+from vllm.v1.sample.logits_processor.builtin import process_dict_updates
+from vllm.v1.sample.logits_processor.interface import BatchUpdate, LogitsProcessor
+
+
+def _strip_trailing_negative_token_ids(token_ids: List[int]) -> List[int]:
+    """vLLM v1 keeps a trailing -1 placeholder in output_token_ids."""
+    i = len(token_ids)
+    while i > 0 and token_ids[i - 1] < 0:
+        i -= 1
+    return token_ids[:i]
+
+
+def _env_int(name: str, default: int) -> int:
+    val = os.environ.get(name, "").strip()
+    if not val:
+        return default
+    try:
+        return int(val)
+    except ValueError:
+        return default
+
+
+def _env_csv_ints(name: str, default: List[int]) -> List[int]:
+    val = os.environ.get(name, "").strip()
+    if not val:
+        return default
+    out: List[int] = []
+    for part in val.split(","):
+        part = part.strip()
+        if not part:
+            continue
+        try:
+            out.append(int(part))
+        except ValueError:
+            return default
+    return out or default
+
+
+def _warn_if_unexpected_arch(vllm_config) -> None:
+    try:
+        arches = getattr(vllm_config.model_config, "architectures", None) or []
+    except Exception:
+        arches = []
+    if any(a == "NemotronParseForConditionalGeneration" for a in arches):
+        return
+    warnings.warn(
+        "Nemotron-Parse logits processors enabled for a model whose "
+        f"architectures={arches!r}. These processors assume Nemotron-Parse-style "
+        "special tokens like <x_...>, <y_...>, and <class_...>; behavior may be "
+        "unsupported for other models.",
+        RuntimeWarning,
+        stacklevel=2,
+    )
+
+
+def _load_hf_tokenizer(vllm_config):
+    model_cfg = vllm_config.model_config
+    tok_name = getattr(model_cfg, "tokenizer", None) or getattr(model_cfg, "model", None)
+    trust_rc = bool(getattr(model_cfg, "trust_remote_code", False))
+    revision = getattr(model_cfg, "tokenizer_revision", None) or getattr(model_cfg, "revision", None)
+    token = getattr(model_cfg, "hf_token", None)
+    kwargs = {"trust_remote_code": trust_rc}
+    if revision:
+        kwargs["revision"] = revision
+    if token:
+        # HF uses `token=` in modern versions.
+        kwargs["token"] = token
+    return AutoTokenizer.from_pretrained(tok_name, **kwargs)
+
+
+def _build_token_sets(tokenizer) -> tuple[Set[int], Set[int], Set[int]]:
+    """Build token-id sets for <x_...>, <y_...>, and <class_...>."""
+
+    def _scan(items) -> tuple[Set[int], Set[int], Set[int]]:
+        x_ids: Set[int] = set()
+        y_ids: Set[int] = set()
+        class_ids: Set[int] = set()
+        for token, tid in items:
+            if token.startswith("<x_") and token.endswith(">") and token.count(">") == 1:
+                x_ids.add(tid)
+            elif token.startswith("<y_") and token.endswith(">") and token.count(">") == 1:
+                y_ids.add(tid)
+            elif token.startswith("<class_") and token.endswith(">") and token.count(">") == 1:
+                class_ids.add(tid)
+        return x_ids, y_ids, class_ids
+
+    added_vocab = {}
+    if hasattr(tokenizer, "get_added_vocab"):
+        try:
+            added_vocab = tokenizer.get_added_vocab() or {}
+        except Exception:
+            added_vocab = {}
+
+    x_ids, y_ids, class_ids = _scan(added_vocab.items())
+
+    if len(x_ids) < 100 or len(y_ids) < 100 or not class_ids:
+        vocab = tokenizer.get_vocab()
+        x2, y2, c2 = _scan(vocab.items())
+        x_ids, y_ids, class_ids = x2, y2, c2
+
+    return x_ids, y_ids, class_ids
+
+
+@dataclass
+class _TableReqState:
+    output_ids: List[int]
+    insertion_active: bool = False
+    insertion_pos: int = 0
+    expecting_end_coords: bool = False
+    _last_class_pos: int = -1
+
+
+class NemotronParseTableInsertionLogitsProcessor(LogitsProcessor):
+    """Force a table prefix right after each START <x_...><y_...> coordinate pair."""
+
+    def __init__(self, vllm_config, device: torch.device, is_pin_memory: bool) -> None:
+        self.enabled = True
+        self.req_states: Dict[int, _TableReqState] = {}
+
+        self.table_prefix = os.environ.get("NEMOTRON_PARSE_TABLE_PREFIX", r"\begin{tabular}")
+        self.table_prefix_ids: List[int] = []
+        self.x_token_ids: Set[int] = set()
+        self.y_token_ids: Set[int] = set()
+        self.class_token_ids: Set[int] = set()
+
+        if self.enabled:
+            _warn_if_unexpected_arch(vllm_config)
+            tok = _load_hf_tokenizer(vllm_config)
+            self.table_prefix_ids = tok.encode(self.table_prefix, add_special_tokens=False)
+            self.x_token_ids, self.y_token_ids, self.class_token_ids = _build_token_sets(tok)
+
+        if not self.x_token_ids or not self.y_token_ids or not self.table_prefix_ids:
+            self.enabled = False
+
+    @classmethod
+    def validate_params(cls, sampling_params):
+        return None
+
+    def is_argmax_invariant(self) -> bool:
+        return False
+
+    def update_state(self, batch_update: BatchUpdate | None) -> None:
+        if not self.enabled:
+            return
+
+        def _new_state(params, prompt_ids, output_ids):
+            return _TableReqState(output_ids=output_ids)
+
+        process_dict_updates(self.req_states, batch_update, _new_state)
+
+        for st in self.req_states.values():
+            out = st.output_ids
+            if not out:
+                continue
+
+            view = _strip_trailing_negative_token_ids(out)
+            if not view:
+                continue
+
+            # If we ended an object (a newly-generated <class_...>), next xy should be START.
+            if self.class_token_ids:
+                last_class_pos = -1
+                for i in range(len(view) - 1, -1, -1):
+                    if view[i] in self.class_token_ids:
+                        last_class_pos = i
+                        break
+                if last_class_pos != -1 and last_class_pos != st._last_class_pos:
+                    st._last_class_pos = last_class_pos
+                    st.expecting_end_coords = False
+
+            if st.insertion_active:
+                continue
+
+            if len(view) >= 2 and view[-2] in self.x_token_ids and view[-1] in self.y_token_ids:
+                if not st.expecting_end_coords:
+                    st.insertion_active = True
+                    st.insertion_pos = 0
+                    st.expecting_end_coords = True
+                else:
+                    pass
+
+    def apply(self, logits: torch.Tensor) -> torch.Tensor:
+        if not self.enabled or not self.req_states:
+            return logits
+
+        for req_idx, st in self.req_states.items():
+            if not st.insertion_active:
+                continue
+            pos = st.insertion_pos
+            if pos >= len(self.table_prefix_ids):
+                st.insertion_active = False
+                continue
+
+            forced_tid = self.table_prefix_ids[pos]
+            logits[req_idx].fill_(-float("inf"))
+            logits[req_idx, forced_tid] = 0.0
+
+            st.insertion_pos = pos + 1
+            if st.insertion_pos >= len(self.table_prefix_ids):
+                st.insertion_active = False
+
+        return logits
+
+
+@dataclass
+class _RepReqState:
+    output_ids: List[int]
+    in_cooldown: bool = False
+    segment_start: int = 0
+    _last_class_pos: int = -1
+
+
+class NemotronParseRepetitionStopLogitsProcessor(LogitsProcessor):
+    """Force an <x_...> token when consecutive repetition exceeds a threshold."""
+
+    def __init__(self, vllm_config, device: torch.device, is_pin_memory: bool) -> None:
+        self.enabled = True
+        self.req_states: Dict[int, _RepReqState] = {}
+
+        self.max_repetitions = _env_int("NEMOTRON_PARSE_REP_MAX", 10)
+        self.ngram_sizes = _env_csv_ints("NEMOTRON_PARSE_REP_NGRAMS", [1, 2, 3, 4, 5])
+        max_ngram = max((n for n in self.ngram_sizes if n > 0), default=1)
+        default_window = 2 * max_ngram * (self.max_repetitions + 1)
+        self.window_size = _env_int("NEMOTRON_PARSE_REP_WINDOW", default_window)
+
+        self.x_token_ids: Set[int] = set()
+        self.class_token_ids: Set[int] = set()
+
+        if self.enabled:
+            _warn_if_unexpected_arch(vllm_config)
+            tok = _load_hf_tokenizer(vllm_config)
+            x_ids, _, class_ids = _build_token_sets(tok)
+            self.x_token_ids = x_ids
+            self.class_token_ids = class_ids
+
+        if not self.x_token_ids:
+            self.enabled = False
+
+    @classmethod
+    def validate_params(cls, sampling_params):
+        return None
+
+    def is_argmax_invariant(self) -> bool:
+        return False
+
+    def update_state(self, batch_update: BatchUpdate | None) -> None:
+        if not self.enabled:
+            return
+
+        def _new_state(params, prompt_ids, output_ids):
+            return _RepReqState(output_ids=output_ids)
+
+        process_dict_updates(self.req_states, batch_update, _new_state)
+
+        for st in self.req_states.values():
+            out = st.output_ids
+            if not out:
+                continue
+            view = _strip_trailing_negative_token_ids(out)
+            if not view:
+                continue
+            # Clear cooldown and advance segment on a newly-generated <class_...> token.
+            if self.class_token_ids:
+                last_class_pos = -1
+                for i in range(len(view) - 1, -1, -1):
+                    if view[i] in self.class_token_ids:
+                        last_class_pos = i
+                        break
+                if last_class_pos != -1 and last_class_pos != st._last_class_pos:
+                    st._last_class_pos = last_class_pos
+                    st.in_cooldown = False
+                    st.segment_start = last_class_pos + 1
+
+    @staticmethod
+    def _max_consecutive_repetitions(seq: List[int], n: int) -> int:
+        if len(seq) < n:
+            return 0
+        max_consec = 1
+        cur = 1
+        prev = tuple(seq[0:n])
+        i = n
+        while i <= len(seq) - n:
+            cur_ng = tuple(seq[i:i + n])
+            if cur_ng == prev:
+                cur += 1
+                if cur > max_consec:
+                    max_consec = cur
+                i += n
+            else:
+                cur = 1
+                prev = cur_ng
+                i += 1
+        return max_consec
+
+    def _has_excessive_repetition(self, seq: List[int]) -> bool:
+        if not seq:
+            return False
+        check_seq = seq[-self.window_size:] if len(seq) > self.window_size else seq
+        for n in self.ngram_sizes:
+            if n <= 0:
+                continue
+            if self._max_consecutive_repetitions(check_seq, n) > self.max_repetitions:
+                return True
+        return False
+
+    def apply(self, logits: torch.Tensor) -> torch.Tensor:
+        if not self.enabled or not self.req_states:
+            return logits
+
+        for req_idx, st in self.req_states.items():
+            if st.in_cooldown:
+                continue
+            view = _strip_trailing_negative_token_ids(st.output_ids)
+            seg = view[st.segment_start:]
+            if not self._has_excessive_repetition(seg):
+                continue
+
+            st.in_cooldown = True
+            row = logits[req_idx]
+            original = row.clone()
+            row.fill_(-float("inf"))
+            for tid in self.x_token_ids:
+                row[tid] = original[tid]
+
+        return logits
+
+

model-cards/nemotron-3-5-parse-subcards/bias.md

@@ -0,0 +1,7 @@
+## Bias
+
+Field | Response
+:---- | :----
+Participation considerations from adversely impacted groups [protected classes](https://calcivilrights.ca.gov/disputeresolution/protected-characteristics/) in model design and testing: | None
+Measures taken to mitigate against unwanted bias: | Dataset curation, filtering, and evaluation include review across document types, layouts, languages, and scripts where available. Users should evaluate performance on their target document domains before deployment.
+Bias Metric (If Measured): | Not measured as a demographic fairness metric. Model quality is evaluated through document parsing, OCR, layout, table, chart, and reading-order metrics.

model-cards/nemotron-3-5-parse-subcards/explainability.md

@@ -0,0 +1,15 @@
+## Explainability
+
+Field | Response
+:---- | :----
+Intended Task/Domain: | Document understanding, OCR, layout parsing, table extraction, and text extraction with spatial grounding.
+Model Type: | Transformer-based vision-encoder-decoder model.
+Intended Users: | Developers and teams building document intelligence, retrieval-augmented generation (RAG), data-curation, extraction, and multimodal agent workflows.
+Output: | Structured text containing document element classes, bounding boxes, reading-order information, chart/table annotations, and extracted text.
+Describe how the model works: | Nemotron 3.5 Parse encodes a dynamically sized document image with C-RADIOv4-SO400M, compresses the visual features with a neck, and places them before the OCR prompt as a causal prefix to a 10-layer mBART decoder. The decoder uses grouped-query self-attention and produces structured document annotations with content-normalized boxes. The 72,505-token tokenizer includes multilingual, task-control, and chart-aware tokens such as `<class_Chart>`.
+Name the adversely impacted groups this has been tested to deliver comparable outcomes regardless of: | Not Applicable
+Technical Limitations & Mitigation: | Performance can vary for low-resolution scans, motion blur, handwriting, decorative fonts, dense tables, unusual layouts, unsupported languages or scripts, and documents with heavy compression artifacts. Users should evaluate outputs on representative documents and use human review for high-impact workflows.
+Verified to have met prescribed NVIDIA quality standards: | Yes
+Performance Metrics: | OCR accuracy, layout/class accuracy, table extraction quality, chart extraction quality, reading-order quality, grounding quality, latency, throughput, and qualitative visual inspection.
+Potential Known Risks: | The model may miss text, hallucinate structure, assign incorrect classes, produce inaccurate bounding boxes, or incorrectly order elements. Downstream systems should treat output as model-generated extraction results and apply validation, confidence checks, and human review where appropriate.
+Licensing: | Governing Terms: Your use of this model is governed by the [OpenMDW License Agreement, version 1.1](https://openmdw.ai/license/1-1/). Use of the tokenizer included in this model is governed by the [MIT license and tokenizer attribution](https://huggingface.co/nvidia/Nemotron-3.5-Parse/blob/main/TOKENIZER_ATTRIBUTION.md). See [third-party notices](https://huggingface.co/nvidia/Nemotron-3.5-Parse/blob/main/THIRD_PARTY_NOTICES.md) for separately installed dependencies.

model-cards/nemotron-3-5-parse-subcards/privacy.md

@@ -0,0 +1,2 @@
+## Privacy
+The datasets used to train this model were reviewed before release. Please review NVIDIA’s Privacy Policy (https://www.nvidia.com/en-us/about-nvidia/privacy-policy/) for more information.

model-cards/nemotron-3-5-parse-subcards/safety_security.md

@@ -0,0 +1,12 @@
+## Safety
+
+Field | Response
+:---- | :----
+Model Application Field(s): | Document intelligence, information retrieval, enterprise automation, data extraction, multimodal data curation, and retrieval-augmented generation workflows.
+Describe the life critical impact (if present). | Not Applicable. This model is not intended to make life-critical, legal, medical, financial, or safety-critical decisions without downstream validation and human oversight.
+Use Case Restrictions: | Governing Terms: Your use of this model is governed by the [OpenMDW License Agreement, version 1.1](https://openmdw.ai/license/1-1/). Use of the tokenizer included in this model is governed by the [MIT license and tokenizer attribution](https://huggingface.co/nvidia/Nemotron-3.5-Parse/blob/main/TOKENIZER_ATTRIBUTION.md). See [third-party notices](https://huggingface.co/nvidia/Nemotron-3.5-Parse/blob/main/THIRD_PARTY_NOTICES.md) for separately installed dependencies.
+Input permissions: | Users are responsible for ensuring they have rights and permissions for input documents and images.
+Model and dataset restrictions: | The Principle of least privilege (PoLP) is applied limiting access for dataset generation and model development. Restrictions enforce dataset access during training, and dataset license constraints adhered to.
+
+
+**You are responsible for ensuring that your use of NVIDIA-provided models complies with all applicable laws.**

model-cards/nemotron-3-5-parse.md

@@ -0,0 +1,158 @@
+# Nemotron 3.5 Parse Overview
+
+## Description:
+Nemotron 3.5 Parse is a document image parser that produces structured, machine-readable text with semantic layout classes, bounding boxes, and reading order. It combines a C-RADIOv4-SO400M vision encoder with a 10-layer mBART decoder that consumes the visual sequence as a causal prefix. The release uses dynamic rectangular NaViT preprocessing, content-normalized coordinates, grouped-query self-attention (16 query heads and 8 key/value heads), a 72,505-token multilingual tokenizer, and chart-aware output through the `<class_Chart>` token. It is intended for document understanding, information retrieval, data extraction, and multimodal data-curation workflows.
+
+Nemotron 3.5 Parse was developed by NVIDIA as a part of Nemotron.
+
+This model is ready for commercial or non-commercial use.
+
+### License/Terms of Use:
+Governing Terms: Your use of this model is governed by the [OpenMDW License Agreement, version 1.1](https://openmdw.ai/license/1-1/). Use of the tokenizer included in this model is governed by the [MIT license and tokenizer attribution](https://huggingface.co/nvidia/Nemotron-3.5-Parse/blob/main/TOKENIZER_ATTRIBUTION.md). See [third-party notices](https://huggingface.co/nvidia/Nemotron-3.5-Parse/blob/main/THIRD_PARTY_NOTICES.md) for separately installed dependencies.
+
+### Deployment Geography:
+Global
+
+### Use Case:
+Nemotron 3.5 Parse is designed for developers and teams building document intelligence, retrieval-augmented generation (RAG), curator, extractor, and agentic AI applications. It can convert scanned or rendered PDFs, presentation slides, forms, reports, tables, and mixed-content document pages into structured outputs for downstream indexing, retrieval, analytics, model training-data creation, and human-in-the-loop review. The model supports document understanding, optical character recognition (OCR), layout parsing, table and chart extraction, reading-order recovery, and text extraction with spatial grounding.
+
+### Release Date:
+Hugging Face 10/17/2026 via https://huggingface.co/nvidia/Nemotron-3.5-Parse
+
+## Reference(s):
+* [Hugging Face Transformers mBART documentation](https://huggingface.co/docs/transformers/en/model_doc/mbart) 
+* [NVIDIA C-RADIOv4-SO400M](https://huggingface.co/nvidia/C-RADIOv4-SO400M)
+
+## Model Architecture:
+**Architecture Type:** Transformer
+
+**Network Architecture:** 
+* Vision Encoder: NVIDIA C-RADIOv4-SO400M (`vit_so400m_patch16_224`) with dynamic rectangular NaViT preprocessing
+* Adapter Layer: 1D convolutions and normalization layers that compress the vision latent sequence before decoding
+* Decoder: 10-layer mBART causal visual-prefix decoder with 16 query heads and 8 key/value heads
+* Auxiliary Prediction Head: One trained one-step multi-token-prediction (MTP) head is included in `model.safetensors`. Transformers uses the normal decoder for generation; the vLLM plugin enables the bundled head for one-token speculative decoding.
+* Tokenizer: 72,505 entries with multilingual and task/control tokens including `<predict_bbox>`, `<predict_classes>`, `<predict_text_in_pic>`, and `<class_Chart>`; the decoder embedding is padded to 72,512 rows. Use of the tokenizer included in this model is governed by the [MIT license and tokenizer attribution](https://huggingface.co/nvidia/Nemotron-3.5-Parse/blob/main/TOKENIZER_ATTRIBUTION.md).
+**Number of model parameters:** 630,800,496 unique trainable parameters (approximately 0.631B), including the MTP head; shared decoder embedding/output weights are counted once
+
+## Input:
+**Input Type(s):** Image, Text
+
+**Input Format(s):** 
+* Image: Red, Green, Blue (RGB)
+* Text: Prompt string
+
+**Input Parameters:** 
+* Image: Two-Dimensional (2D)
+* Text: One-Dimensional (1D)
+
+**Other Properties Related to Input:** 
+* Recommended maximum input resolution (Width, Height): 1664, 2048
+* Channel Count: 3
+* Prompt format: one task prompt composed from the supported control tokens. The default prompt extracts bounding boxes, semantic classes, and text in markdown format: `</s><s><predict_bbox><predict_classes><output_markdown><predict_text_in_pic>`.
+
+## Output:
+**Output Type(s):** Text
+
+**Output Format(s):** String
+
+**Output Parameters:** One-Dimensional (1D)
+
+**Other Properties Related to Output:** Nemotron 3.5 Parse returns a string encoding document text, semantic element classes, and content-normalized bounding boxes. Scale x coordinates by the evaluated image width and y coordinates by its height. Postprocessing utilities can convert table or chart-associated text into LaTeX, HTML, markdown, JSON, or CSV where supported.
+
+Our AI models are designed and/or optimized to run on NVIDIA GPU-accelerated systems. By leveraging NVIDIA hardware and software frameworks, the model achieves faster training and inference times compared to CPU-only solutions.
+
+## Software Integration:
+**Runtime Engine(s):**
+* Transformers 
+* vLLM 
+
+**Reference Hardware:** NVIDIA Hopper H100, BF16.
+
+**Supported Operating System(s):**
+* Linux 
+
+The integration of foundation and fine-tuned models into AI systems requires additional testing using use-case-specific data to ensure safe and effective deployment. Following the V-model methodology, iterative testing and validation at both unit and system levels is essential to mitigate risks, meet technical and functional requirements, and ensure compliance with safety and ethical standards before deployment.
+
+## Model Version(s):
+Nemotron 3.5 Parse · October 17, 2026 release 
+
+This AI model can be embedded as an Application Programming Interface (API) call through the documented Transformers or vLLM serving interfaces.
+
+## Training, Testing, and Evaluation Datasets:
+### Training Dataset
+
+**Data Modality:** 
+* Image 
+* Text 
+
+**Image Training Data Size:** 
+* 1 Million to 1 Billion Images 
+
+**Text Training Data Size:** 
+* 1 Billion to 10 Trillion Tokens 
+
+**Data Collection Method by dataset:** 
+* Automated, Manually-Collected, Synthetic 
+
+**Labeling Method by dataset:** Automated, Manually-Labeled, Synthetic 
+**Labeling Details:** Labels are produced through a combination of automated OCR and layout annotation, manual annotation, and synthetic generation. The combination varies by constituent dataset. 
+
+**Properties:** The training set contains millions of image-text items aggregated across large document, table, and layout datasets. The data consists of document-page and table images paired with OCR text, bounding boxes, and layout labels. Sources include rendered digital documents, scientific papers, PDFs, Wikipedia-style pages, and synthetic document, table, word, and character renderings. Annotations come from OCR and layout models, third-party OCR services, synthetic-generation pipelines, and human labeling. 
+
+### Testing Dataset
+
+**Data Collection Method by dataset:** Automated, Manually-Collected, Synthetic  
+**Collection Details:** Sources combine automated collection, manually collected material, and synthetic generation; methods vary by constituent dataset.
+
+**Labeling Method by dataset:** Automated, Manually-Labeled, Synthetic  
+**Labeling Details:** Labels are produced through a combination of automated OCR and layout annotation, manual annotation, and synthetic generation. The combination varies by constituent dataset.
+
+**Properties:** Tested on internal and public document-understanding benchmarks using document-page images and reference text, layout classes, bounding boxes, and structure annotations. Coverage includes OCR quality, layout structure, table parsing, reading order, and visual grounding across document types and languages.
+
+### Evaluation Dataset
+
+**Data Collection Method by dataset:** Automated, Manually-Collected, Synthetic  
+**Collection Details:** Sources combine automated collection, manually collected material, and synthetic generation; methods vary by constituent dataset.
+
+**Labeling Method by dataset:** Automated, Manually-Labeled, Synthetic  
+**Labeling Details:** Labels are produced through a combination of automated OCR and layout annotation, manual annotation, and synthetic generation. The combination varies by constituent dataset.
+
+**Properties:** Evaluated on internal and public document-understanding benchmarks covering OCR quality, layout structure, table parsing, reading order, and visual grounding. The ParseBench reference evaluation covers 2,078 unique document pages. Additional benchmarks cover Indic-language OCR and multilingual synthetic OCR across the scripts listed below. 
+
+Benchmark coverage: 
+* ParseBench evaluates document parsing across text fidelity, semantic formatting, tables, charts, and visual grounding with layout boxes/classes. 
+* IndicVisionBench evaluates OCR quality for Indic-language pages, including Bengali, Gujarati, Hindi, Kannada, Malayalam, Marathi, Odia, Punjabi, Tamil, and Telugu. 
+* MOSCAR evaluates multilingual synthetic OCR across broad script coverage, including Latin, Arabic, Cyrillic, Han, Hangul, Japanese, Indic scripts, Hebrew, Thai, Greek, and others. 
+
+## ParseBench
+
+| Pipeline | Inference stages | Overall score ↑ | Prepared pages/s ↑ |
+| --- | --- | ---: | ---: |
+| Single | Full-page parse | 72.74 | 15.42 |
+| Double | Full-page parse and predicted Table/Chart crop refinement | 74.92 | 11.65 |
+| Triple | Full-page parse, separate detection, and crop refinement using their region union | 75.20 | 10.62 |
+
+Measured on 2,078 unique pages with one H100. Double captures most of the observed
+accuracy gain; triple adds 0.28 points at about 9% lower throughput than double.
+Timing includes inference waits and dependent crop preparation, excluding initial
+preparation and final rendering/writes. Processed-page rates include token-limited
+outputs. The refinement merge retains differing alternatives. See the
+[measurement sources](https://huggingface.co/nvidia/Nemotron-3.5-Parse/blob/main/reference/parsebench/comparisons/README.md).
+
+## Inference:
+**Acceleration Engine:** Transformers, vLLM 
+**Test Hardware:** 
+* NVIDIA Hopper H100
+
+## Limitations:
+Performance can vary for low-resolution scans, motion blur, handwriting, decorative fonts, dense tables, unusual layouts, unsupported languages or scripts, and documents with heavy compression artifacts. The model may miss text, hallucinate structure, assign incorrect classes, produce inaccurate bounding boxes, or incorrectly order elements. Users should evaluate outputs on representative target documents and use validation and human review for high-impact workflows.
+
+## Ethical Considerations:
+NVIDIA believes Trustworthy AI is a shared responsibility and has established policies and practices to enable development for a wide array of AI applications. Developers should work with their internal model teams to ensure this model meets requirements for the relevant industry and use case and addresses unforeseen product misuse.
+
+Please make sure you have proper rights and permissions for all input image and video content. If an input image contains people, personal health information, confidential business data, or intellectual property, the model may extract or reproduce visible text from that content.
+
+For more detailed information on ethical considerations for this model, please see the Model Card++ [Bias](nemotron-3-5-parse-subcards/bias.md), [Explainability](nemotron-3-5-parse-subcards/explainability.md), [Safety & Security](nemotron-3-5-parse-subcards/safety_security.md), and [Privacy](nemotron-3-5-parse-subcards/privacy.md) subcards.  
+Please report model quality, risk, security vulnerabilities, or NVIDIA AI concerns [here](https://www.nvidia.com/en-us/support/submit-security-vulnerability/).
+
+**Generated by NVIDIA Model Card Generator Toolkit.**

model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d7139ecd62f830bac1ce83b77655f10710ee0da31f2810dd47a40eec368917a3
+size 2523266632

nemotron_parse/SOURCE_MANIFEST.json

@@ -0,0 +1,110 @@
+{
+  "repository": "nvidia/Nemotron-3.5-Parse",
+  "source_revision": "0fad7099302dc11313b7e3157eb8fd6ae633176e",
+  "extracted_components": [
+    {
+      "source": "reference/parsebench/pipeline/engine/__init__.py",
+      "source_sha256": "717a1b9838b948a88d2fb5106069db57dce81dc54854cf2be5fb54150c7ac652",
+      "destination": "nemotron_parse/engine/__init__.py"
+    },
+    {
+      "source": "reference/parsebench/pipeline/engine/caption_relocation.py",
+      "source_sha256": "3a17677b1a43c793cd817f8f3f5745be807203ff75b8e3e0af5a4d51d547dd4b",
+      "destination": "nemotron_parse/engine/caption_relocation.py"
+    },
+    {
+      "source": "reference/parsebench/pipeline/engine/chart_caption.py",
+      "source_sha256": "fe82e0eca23be96d7cc2199657432bd2ddf464313d41fed3984bf5960fddb0e6",
+      "destination": "nemotron_parse/engine/chart_caption.py"
+    },
+    {
+      "source": "reference/parsebench/pipeline/engine/chart_context.py",
+      "source_sha256": "ffe3393bc13bd54b9b65235ca51a4c15a0b40fe6404a5897379eb0dd2748d868",
+      "destination": "nemotron_parse/engine/chart_context.py"
+    },
+    {
+      "source": "reference/parsebench/pipeline/engine/convert/__init__.py",
+      "source_sha256": "717a1b9838b948a88d2fb5106069db57dce81dc54854cf2be5fb54150c7ac652",
+      "destination": "nemotron_parse/engine/convert/__init__.py"
+    },
+    {
+      "source": "reference/parsebench/pipeline/engine/convert/hybrid_cleanup.py",
+      "source_sha256": "f2f319de9bdabcfc5e264cdef6ff9d13235c488a372e9f58f676017f28f7c5c0",
+      "destination": "nemotron_parse/engine/convert/hybrid_cleanup.py"
+    },
+    {
+      "source": "reference/parsebench/pipeline/engine/convert/hybrid_inline.py",
+      "source_sha256": "0296f7827e74ae3ab64e9c9efbd496f096f0a3bbde7c3ad64d017539f1db7cb6",
+      "destination": "nemotron_parse/engine/convert/hybrid_inline.py"
+    },
+    {
+      "source": "reference/parsebench/pipeline/engine/convert/latex2html.py",
+      "source_sha256": "d583b5fa7fdbe181b190d2c8c22034318a51ef25d47203f9c7504bf35b508a72",
+      "destination": "nemotron_parse/engine/convert/latex2html.py"
+    },
+    {
+      "source": "reference/parsebench/pipeline/engine/convert/rich_table_fallback.py",
+      "source_sha256": "c2abbc602b293bf14a499f680de9f7d1584fdcf1438a374e3c75cf3705c0b793",
+      "destination": "nemotron_parse/engine/convert/rich_table_fallback.py"
+    },
+    {
+      "source": "reference/parsebench/pipeline/engine/convert/rich_table_reader.py",
+      "source_sha256": "8a8b615681d3adcb46c5290ad631abffeccb2af0ee18f25e82f0cdb70f5befac",
+      "destination": "nemotron_parse/engine/convert/rich_table_reader.py"
+    },
+    {
+      "source": "reference/parsebench/pipeline/engine/convert/semantic_styles.py",
+      "source_sha256": "3d92146e6c713b9de67cb71e92fe254a1344d9054c3925a85cb778ec68c0d075",
+      "destination": "nemotron_parse/engine/convert/semantic_styles.py"
+    },
+    {
+      "source": "reference/parsebench/pipeline/engine/geometry.py",
+      "source_sha256": "76671df686b574a827205acb757eecf702e970470ffa15f81b3129cbcd95e242",
+      "destination": "nemotron_parse/engine/geometry.py"
+    },
+    {
+      "source": "reference/parsebench/pipeline/engine/header_inference_probe.py",
+      "source_sha256": "02950f1aec5e2dc88a96487bd9124c97e5ce2842fbcfc5264ac2d5f697865722",
+      "destination": "nemotron_parse/engine/header_inference_probe.py"
+    },
+    {
+      "source": "reference/parsebench/pipeline/engine/markdown.py",
+      "source_sha256": "fc57566255ac632b24825973d9533a13fd717dad4e38d096098bac33ecc79957",
+      "destination": "nemotron_parse/engine/markdown.py"
+    },
+    {
+      "source": "reference/parsebench/pipeline/engine/merge.py",
+      "source_sha256": "b1c2e749e8f772b716c5a8a2ebeba7348474eb9368b00d37f1c8987e26d24e4e",
+      "destination": "nemotron_parse/engine/merge.py"
+    },
+    {
+      "source": "reference/parsebench/pipeline/engine/policies.py",
+      "source_sha256": "4c1f78be4d8061e4224fb59c3dc5342dfc21d9d37826814ea3573f339777aebd",
+      "destination": "nemotron_parse/engine/policies.py"
+    },
+    {
+      "source": "reference/parsebench/pipeline/engine/regions.py",
+      "source_sha256": "33058eec02bb378050febc577a2d85de97d34083234a08a2a8fa9c4b8dc62239",
+      "destination": "nemotron_parse/engine/regions.py"
+    },
+    {
+      "source": "reference/parsebench/pipeline/engine/text_cleanup.py",
+      "source_sha256": "f02f472d7b3cb08eca5b11d3ecf80ee6ec0636857ade17fb4cd11c13f371c2e4",
+      "destination": "nemotron_parse/engine/text_cleanup.py"
+    }
+  ],
+  "adaptations": [
+    "No dataset, scorer, or reference-answer dependencies",
+    "Headerless flat exports preserve all rows; collision-free field names",
+    "Exact duplicate cleanup preserves whitespace differences",
+    "Malformed table fallback preserves source",
+    "Presentation heuristics configurable",
+    "Image-based inference wrappers and guarded crop merging",
+    "Code and Formula blocks retain literal table examples",
+    "Enable guarded chart presentation rules by default, retaining opt-outs",
+    "Prepared image batches, bounded pipelined concurrency, and staged comparisons",
+    "Explicit partial-output accounting and per-stage timing",
+    "Retain differing overlapping structural alternatives by default; guarded replacement remains an explicit option. Scheduler unchanged.",
+    "Add first-request-to-last-response inference wall timing, retaining dependent crop preparation gaps and separate total pipeline metrics."
+  ]
+}

nemotron_parse/__init__.py

@@ -0,0 +1,6 @@
+# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: OpenMDW-1.1
+"""General document postprocessing and optional multipass inference."""
+from .postprocessing import PostprocessOptions, postprocess_document, render_markdown
+
+__all__ = ["PostprocessOptions", "postprocess_document", "render_markdown"]

nemotron_parse/batch.py

@@ -0,0 +1,571 @@
+# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: OpenMDW-1.1
+"""Prepared image batches with bounded, overlapping HTTP concurrency and CPU rendering."""
+from concurrent.futures import (
+    ThreadPoolExecutor,
+    ProcessPoolExecutor,
+    wait,
+    FIRST_COMPLETED,
+)
+from dataclasses import dataclass, asdict
+from io import BytesIO
+from pathlib import Path
+from types import SimpleNamespace
+import base64
+import collections
+import multiprocessing
+import time
+
+from PIL import Image
+from .engine import geometry
+from .engine.regions import select_union_regions
+from .postprocessing import clean_raw_output, postprocess_document
+
+
+@dataclass
+class PreparedPage:
+    image: Image.Image
+    data_url: str
+    source: str | None
+
+
+@dataclass
+class PreparedBatch:
+    pages: list[PreparedPage]
+    preprocessing_seconds: float
+
+
+@dataclass
+class BatchResult:
+    documents: list[dict]
+    metrics: dict
+
+
+def _url(data, mime):
+    return f"data:{mime};base64," + base64.b64encode(data).decode("ascii")
+
+
+def prepare_images(images, *, workers=8):
+    if workers < 1:
+        raise ValueError("workers must be positive")
+    start = time.perf_counter()
+
+    def one(value):
+        if isinstance(value, Image.Image):
+            image = value.convert("RGB")
+            stream = BytesIO()
+            image.save(stream, format="PNG")
+            return PreparedPage(image, _url(stream.getvalue(), "image/png"), None)
+        data = Path(value).read_bytes()
+        with Image.open(BytesIO(data)) as opened:
+            mime = {"JPEG": "image/jpeg", "PNG": "image/png", "WEBP": "image/webp"}.get(
+                opened.format
+            )
+            keep_encoded = mime is not None and opened.mode == "RGB"
+            image = opened.convert("RGB")
+        if not keep_encoded:
+            stream = BytesIO()
+            image.save(stream, format="PNG")
+            data, mime = stream.getvalue(), "image/png"
+        return PreparedPage(image, _url(data, mime), str(value))
+
+    with ThreadPoolExecutor(max_workers=workers) as pool:
+        pages = list(pool.map(one, images))
+    if not pages:
+        raise ValueError("No page images provided")
+    return PreparedBatch(pages, time.perf_counter() - start)
+
+
+def _finish(task):
+    raw, crops, size, options, merge_policy = task
+    from .inference import merge_crops
+
+    raw = clean_raw_output(raw, deduplicate=options.deduplicate)
+    if crops:
+        raw = merge_crops(raw, crops, policy=merge_policy)
+    return postprocess_document(raw, image_size=size, options=options)
+
+
+def _crop_preparer(parser, pages, triple_regions):
+    args = SimpleNamespace(
+        include_detector_labels="table,chart",
+        detector_order="model",
+        detector_score_min=0.0,
+        max_elements_per_page=parser.max_regions,
+        model_coordinate_frame="content_normalized",
+        canvas_width=1664.0,
+        canvas_height=2048.0,
+        pad_position="top-left",
+    )
+
+    def page_crops(index, direct_response, detection_response=None, start_region=0):
+        page = pages[index]
+        page_spec = geometry.PageImage(
+            "page", "image.png", Path("image.png"), *page.image.size
+        )
+
+        def boxes(response):
+            return geometry.parse_model_detection_predictions(
+                args,
+                [page_spec],
+                [{"key": "page", "raw_out": clean_raw_output(response["raw_out"])}],
+            )
+
+        selected = (
+            boxes(direct_response)
+            if direct_response is not None
+            else [{"key": "page", "detections": []}]
+        )
+        if detection_response is not None:
+            detected = boxes(detection_response)
+            selected = (
+                select_union_regions(detected, selected, max_regions=parser.max_regions)
+                if triple_regions == "union"
+                else detected
+            )
+        boxes_for_page = selected[0]["detections"]
+        items, metadata = [], []
+        for region_index, region in enumerate(
+            boxes_for_page[start_region:], start_region
+        ):
+            pixels = geometry.crop_bbox_pixels(
+                tuple(region["bbox_norm"]),
+                *page.image.size,
+                margin_px=parser.crop_margin_px,
+                margin_ratio=parser.crop_margin_ratio,
+            )
+            if pixels[2] - pixels[0] < 8 or pixels[3] - pixels[1] < 8:
+                continue
+            stream = BytesIO()
+            page.image.crop(pixels).save(stream, format="JPEG", quality=95)
+            data = stream.getvalue()
+            # HTTP sends the prepared crop directly; custom backends receive decoded pixels.
+            image = None
+            if parser.backend is not None:
+                with Image.open(BytesIO(data)) as decoded:
+                    image = decoded.convert("RGB")
+            items.append((image, _url(data, "image/jpeg")))
+            metadata.append(
+                {
+                    "page_index": index,
+                    "crop_id": f"page__{region_index:03d}",
+                    "page_key": "page",
+                    "detector_label": region["label"],
+                    "detector_score": region["score"],
+                    "detector_bbox_norm": region["bbox_norm"],
+                    "crop_bbox_pixels": pixels,
+                    "page_size": page.image.size,
+                }
+            )
+        return index, boxes_for_page, items, metadata
+
+    return page_crops
+
+
+def _inference_timing(spans):
+    """Wall time across model calls, including gaps needed to prepare later calls."""
+    elapsed = max(end for begin, end in spans.values()) - min(
+        begin for begin, end in spans.values()
+    )
+    return {
+        "inference_seconds": elapsed,
+        "inference_timing_includes": [
+            "all HTTP/model calls, server queueing and response transfer",
+            "gaps between inference calls, including dependent crop preparation",
+        ],
+        "inference_timing_excludes": [
+            "initial image preparation",
+            "server startup",
+            "warmup requests",
+            "merge/render work after the final model response",
+            "output serialization",
+        ],
+        "inference_timing_definition": "first model request start to last model response",
+    }
+
+
+def _pipeline(parser, pages, mode, concurrency, workers, region_policy, allow_partial):
+    """Keep one bounded HTTP window full; dependent crops take the next free slot.
+
+    Detection regions form the stable prefix of the union. They can be submitted
+    before full-page parsing finishes; parser-only additions follow in union order.
+    CPU crop encoding and document rendering never occupy an HTTP worker.
+    """
+    from .inference import GenerationError
+
+    count = len(pages)
+    direct = [None] * count
+    detection = [None] * count if mode == "triple" else None
+    regions = [[] for _ in pages]
+    crops = [{} for _ in pages]
+    documents = [None] * count
+    prep_pending = [0] * count
+    crop_pending = [0] * count
+    sealed = [mode == "single"] * count
+    render_started = set()
+    ready = collections.deque()
+    initial = collections.deque(
+        (i, stage, (page.image, page.data_url), None)
+        for i, page in enumerate(pages)
+        for stage in (["fullpage", "detection"] if mode == "triple" else ["fullpage"])
+    )
+    futures = {}
+    http_active = 0
+    peak = 0
+    spans = {}
+    service = collections.Counter()
+    started = time.perf_counter()
+    page_crops = _crop_preparer(parser, pages, region_policy)
+
+    def generate(stage, item):
+        begin = time.perf_counter()
+        try:
+            result = parser._generate(item[0], stage, data_url=item[1])
+        except GenerationError as exc:
+            if not (
+                allow_partial
+                and exc.response.get("finish_reason") == "length"
+                and exc.response.get("raw_out")
+            ):
+                raise
+            result = dict(exc.response)
+        end = time.perf_counter()
+        return dict(result, latency_s=end - begin), begin, end
+
+    # Small batches avoid process startup; threaded CPU work still cannot block HTTP.
+    render_executor = (
+        ProcessPoolExecutor(
+            max_workers=workers, mp_context=multiprocessing.get_context("spawn")
+        )
+        if workers > 1 and count >= 16
+        else ThreadPoolExecutor(max_workers=1)
+    )
+    with ThreadPoolExecutor(max_workers=concurrency) as http, ThreadPoolExecutor(
+        max_workers=min(8, count)
+    ) as prep, render_executor as render:
+
+        def submit_prep(i, first, second, offset=0):
+            prep_pending[i] += 1
+            futures[prep.submit(page_crops, i, first, second, offset)] = (
+                "prep",
+                i,
+                None,
+            )
+
+        def maybe_render(i):
+            if (
+                i in render_started
+                or direct[i] is None
+                or not sealed[i]
+                or prep_pending[i]
+                or crop_pending[i]
+            ):
+                return
+            render_started.add(i)
+            rows = [crops[i][j] for j in sorted(crops[i])]
+            task = (
+                direct[i]["raw_out"],
+                [r for r in rows if r.get("finish_reason") != "length"],
+                pages[i].image.size,
+                parser.options,
+                parser.merge_policy,
+            )
+            futures[render.submit(_finish, task)] = ("render", i, None)
+
+        while initial or ready or futures:
+            while http_active < concurrency and (ready or initial):
+                i, stage, item, metadata = (
+                    ready.popleft() if ready else initial.popleft()
+                )
+                futures[http.submit(generate, stage, item)] = (stage, i, metadata)
+                http_active += 1
+                peak = max(peak, http_active)
+            if not futures:
+                break
+            done, _ = wait(futures, return_when=FIRST_COMPLETED)
+            for future in done:
+                kind, i, metadata = futures.pop(future)
+                value = future.result()
+                if kind == "prep":
+                    prep_pending[i] -= 1
+                    _, selected, items, rows = value
+                    # Two prep jobs can finish out of order; retain the full union.
+                    if len(selected) > len(regions[i]):
+                        regions[i] = selected
+                    for item, row in zip(items, rows):
+                        ready.append((i, "crop", item, row))
+                        crop_pending[i] += 1
+                elif kind == "render":
+                    documents[i] = value
+                else:
+                    http_active -= 1
+                    response, begin, end = value
+                    span = spans.setdefault(kind, [begin, end])
+                    span[0], span[1] = min(span[0], begin), max(span[1], end)
+                    service[kind] += end - begin
+                    if kind == "crop":
+                        crop_pending[i] -= 1
+                        crops[i][metadata["crop_id"]] = {**metadata, **response}
+                    elif kind == "fullpage":
+                        direct[i] = response
+                        if mode == "double":
+                            sealed[i] = True
+                            submit_prep(i, response, None)
+                        elif detection is not None and detection[i] is not None:
+                            sealed[i] = True
+                            if region_policy == "union":
+                                # The detector prefix is already queued. Only prepare additions.
+                                detector_count = len(
+                                    page_crops(
+                                        i, None, detection[i], parser.max_regions
+                                    )[1]
+                                )
+                                submit_prep(i, response, detection[i], detector_count)
+                    else:
+                        detection[i] = response
+                        submit_prep(i, None, response)
+                        if region_policy == "detector":
+                            sealed[i] = True
+                        elif direct[i] is not None:
+                            sealed[i] = True
+                            detector_count = len(
+                                page_crops(i, None, response, parser.max_regions)[1]
+                            )
+                            submit_prep(i, direct[i], response, detector_count)
+                maybe_render(i)
+    assert all(document is not None for document in documents)
+    return (
+        direct,
+        detection,
+        regions,
+        [[rows[j] for j in sorted(rows)] for rows in crops],
+        documents,
+        {
+            **_inference_timing(spans),
+            "peak_inflight_requests": peak,
+            "stage_times": {
+                stage + "_span_seconds": end - begin
+                for stage, (begin, end) in spans.items()
+            },
+            "request_service_seconds": dict(service),
+            "stage_first_request_seconds": {
+                stage: span[0] - started for stage, span in spans.items()
+            },
+            "overlapping_stage_times": True,
+        },
+    )
+
+
+def parse_prepared(
+    parser,
+    prepared,
+    *,
+    mode="single",
+    concurrency=128,
+    postprocess_workers=8,
+    triple_regions="union",
+    allow_partial=False,
+    schedule="pipelined",
+):
+    from .inference import GenerationError
+
+    if mode not in ("single", "double", "triple") or triple_regions not in (
+        "detector",
+        "union",
+    ):
+        raise ValueError("Invalid pipeline mode or region policy")
+    if concurrency < 1 or postprocess_workers < 1:
+        raise ValueError("Worker counts must be positive")
+    if not isinstance(prepared, PreparedBatch) or not prepared.pages:
+        raise ValueError("Use parser.prepare() to prepare a nonempty image batch")
+    if schedule not in ("staged", "pipelined"):
+        raise ValueError("schedule must be staged or pipelined")
+    pages = prepared.pages
+    start = time.perf_counter()
+    stage_times = {}
+    request_spans = {}
+
+    def requests(items, stage):
+        stage_start = time.perf_counter()
+
+        def one(item):
+            image, url = item
+            request_start = time.perf_counter()
+            try:
+                response = parser._generate(image, stage, data_url=url)
+            except GenerationError as exc:
+                # Opt-in partial output is explicit and never counted as complete.
+                if (
+                    not allow_partial
+                    or exc.response.get("finish_reason") != "length"
+                    or not exc.response.get("raw_out")
+                ):
+                    raise
+                response = dict(exc.response)
+            request_end = time.perf_counter()
+            response = dict(response, latency_s=request_end - request_start)
+            return response, request_start, request_end
+
+        with ThreadPoolExecutor(max_workers=concurrency) as pool:
+            result = list(pool.map(one, items))
+        stage_times[stage + "_seconds"] = time.perf_counter() - stage_start
+        request_spans[stage] = (
+            min(begin for response, begin, end in result),
+            max(end for response, begin, end in result),
+        )
+        return [response for response, begin, end in result]
+
+    if schedule == "pipelined":
+        direct, detection, regions, crops_by_page, documents, stream_metrics = (
+            _pipeline(
+                parser,
+                pages,
+                mode,
+                concurrency,
+                postprocess_workers,
+                triple_regions,
+                allow_partial,
+            )
+        )
+        stage_times.update(stream_metrics.pop("stage_times"))
+        crop_responses = [r for crops in crops_by_page for r in crops]
+    else:
+        direct = requests([(p.image, p.data_url) for p in pages], "fullpage")
+        detection = (
+            requests([(p.image, p.data_url) for p in pages], "detection")
+            if mode == "triple"
+            else None
+        )
+        regions = [[] for _ in pages]
+        crop_metadata = []
+        crop_items = []
+        crop_responses = []
+        if mode != "single":
+            prep_start = time.perf_counter()
+            page_crops = _crop_preparer(parser, pages, triple_regions)
+            with ThreadPoolExecutor(max_workers=min(8, len(pages))) as pool:
+                for index, selected, items, metadata in pool.map(
+                    lambda i: page_crops(
+                        i, direct[i], detection[i] if detection is not None else None
+                    ),
+                    range(len(pages)),
+                ):
+                    regions[index] = selected
+                    crop_items.extend(items)
+                    crop_metadata.extend(metadata)
+            stage_times["crop_preprocessing_seconds"] = time.perf_counter() - prep_start
+            crop_responses = requests(crop_items, "crop") if crop_items else []
+        crops_by_page = [[] for _ in pages]
+        for metadata, response in zip(crop_metadata, crop_responses):
+            crops_by_page[metadata["page_index"]].append({**metadata, **response})
+        post_start = time.perf_counter()
+        tasks = [
+            (
+                response["raw_out"],
+                [r for r in crops if r.get("finish_reason") != "length"],
+                page.image.size,
+                parser.options,
+                parser.merge_policy,
+            )
+            for page, response, crops in zip(pages, direct, crops_by_page)
+        ]
+        if postprocess_workers == 1 or len(tasks) < 16:
+            documents = list(map(_finish, tasks))
+        else:
+            with ProcessPoolExecutor(
+                max_workers=postprocess_workers,
+                mp_context=multiprocessing.get_context("spawn"),
+            ) as pool:
+                documents = list(pool.map(_finish, tasks, chunksize=8))
+        stage_times["postprocessing_seconds"] = time.perf_counter() - post_start
+        stream_metrics = _inference_timing(request_spans)
+    capped = collections.Counter()
+    completed = 0
+    for index, document in enumerate(documents):
+        stages = {"fullpage": direct[index], "crops": crops_by_page[index]}
+        if detection is not None:
+            stages["detection"] = detection[index]
+        errors = []
+        for stage in ["fullpage", "detection"]:
+            if stage in stages and stages[stage].get("finish_reason") == "length":
+                capped[stage] += 1
+                errors.append(stage + " reached its token limit")
+        for crop in stages["crops"]:
+            if crop.get("finish_reason") == "length":
+                capped["crop"] += 1
+                errors.append(
+                    "crop "
+                    + crop["crop_id"]
+                    + " reached its token limit; refinement omitted"
+                )
+        document["complete"] = not errors
+        document["generation_warnings"] = errors
+        document["stages"] = stages
+        document["source"] = pages[index].source
+        document["inference"] = {
+            "mode": mode,
+            "merge_policy": parser.merge_policy,
+            "triple_regions": triple_regions if mode == "triple" else None,
+            "model": parser.model,
+            "coordinate_frame": "content_normalized",
+            "regions": regions[index],
+            "sampling": {
+                name: asdict(value) for name, value in parser.settings.items()
+            },
+            "postprocessing": asdict(parser.options),
+            "crop_jpeg_quality": 95,
+            "crop_margin_ratio": parser.crop_margin_ratio,
+            "crop_margin_px": parser.crop_margin_px,
+        }
+        completed += not errors
+    elapsed = time.perf_counter() - start
+    all_responses = direct + (detection or []) + crop_responses
+    metrics = {
+        "mode": mode,
+        "merge_policy": parser.merge_policy,
+        "schedule": schedule,
+        **stream_metrics,
+        "pages": len(pages),
+        "complete_pages": completed,
+        "incomplete_pages": len(pages) - completed,
+        "concurrency": concurrency,
+        "postprocess_workers": postprocess_workers,
+        "preprocessing_seconds": prepared.preprocessing_seconds,
+        "inference_pages_per_second": (
+            len(pages) / stream_metrics["inference_seconds"]
+            if stream_metrics["inference_seconds"] > 0
+            else None
+        ),
+        "complete_inference_pages_per_second": (
+            completed / stream_metrics["inference_seconds"]
+            if stream_metrics["inference_seconds"] > 0
+            else None
+        ),
+        "pipeline_seconds": elapsed,
+        "processed_pages_per_second": len(pages) / elapsed,
+        "complete_pages_per_second": completed / elapsed,
+        "stage_times": stage_times,
+        "requests": {
+            "fullpage": len(direct),
+            "detection": len(detection or []),
+            "crop": len(crop_responses),
+        },
+        "token_limited_responses": dict(capped),
+        "completion_tokens": sum(
+            int(r.get("usage", {}).get("completion_tokens") or 0) for r in all_responses
+        ),
+        "postprocessing": asdict(parser.options),
+        "allow_partial": allow_partial,
+        "timing_excludes": [
+            "initial image preparation",
+            "output serialization",
+            "server startup",
+            "warmup requests",
+        ],
+        "timing_includes": [
+            "all mode inference stages",
+            "predicted-region crop preparation",
+            "merge and document rendering",
+        ],
+    }
+    return BatchResult(documents, metrics)

nemotron_parse/engine/__init__.py

@@ -0,0 +1,2 @@
+# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: OpenMDW-1.1

nemotron_parse/engine/caption_relocation.py

@@ -0,0 +1,89 @@
+# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: OpenMDW-1.1
+"""Relocate only plain above-chart captions with geometric/preservation guards."""
+import re
+from bs4 import BeautifulSoup
+
+BLOCK = re.compile(
+    r"(<x_([\d.]+)><y_([\d.]+)>)(.*?)(<x_([\d.]+)><y_([\d.]+)><class_([^>]+)>)", re.S
+)
+
+
+def plain(text):
+    text = text.replace("<tbc>", "").strip()
+    if any(marker in text for marker in ["**", "__", "~~", "`"]):
+        return None
+    if re.search(r"(?<!\w)_[^_\n]+_(?!\w)|<[/a-zA-Z!]|\\[A-Za-z&%_]|\]\(", text):
+        return None
+    return text if text and "\n" not in text else None
+
+
+def relocate_above_captions(raw, markdown, render_base):
+    matches = list(BLOCK.finditer(raw))
+    eligible = {}
+    targets = {}
+    moved = set()
+    allowed = {"Chart"}
+    boxes = [tuple(map(float, [m[2], m[3], m[6], m[7]])) for m in matches]
+    for i, m in enumerate(matches):
+        if m[8] != "Caption" or plain(m[4]) is None:
+            continue
+        box = boxes[i]
+        candidates = []
+        for j in [i - 1, i + 1]:
+            if not 0 <= j < len(matches) or matches[j][8] not in allowed:
+                continue
+            other = boxes[j]
+            if box[3] > other[1]:
+                continue  # Attach above-object captions only; retain notes below as standalone.
+            width = min(box[2] - box[0], other[2] - other[0])
+            overlap = max(0, min(box[2], other[2]) - max(box[0], other[0]))
+            gap = max(other[1] - box[3], box[1] - other[3])
+            if width > 0 and overlap / width >= 0.8 and 0 <= gap <= 0.03:
+                candidates.append((gap, j))
+        candidates.sort()
+        if not candidates or (
+            len(candidates) > 1 and abs(candidates[0][0] - candidates[1][0]) < 1e-9
+        ):
+            continue
+        gap, j = candidates[0]
+        eligible.setdefault(j, []).append((gap, i))
+    for j, caps in eligible.items():
+        caps.sort()
+        if len(caps) > 1 and abs(caps[0][0] - caps[1][0]) < 1e-9:
+            continue
+        caption_index = caps[0][1]
+        label = plain(matches[caption_index][4])
+        m = matches[j]
+        body = m[4]
+        if len(re.findall(r"<chart\b", body)) != 1:
+            continue
+        title = re.search(r"(?m)^title:[ \t]*([^\n]*)", body)
+        if title:
+            if label not in title[1]:
+                body = (
+                    body[: title.start()]
+                    + "title: "
+                    + title[1].strip()
+                    + " — "
+                    + label
+                    + body[title.end() :]
+                )
+        else:
+            body = re.sub(
+                r"(<chart\b[^>]*>)", lambda q: q[0] + "\ntitle: " + label, body, count=1
+            )
+        derived = m[1] + body + m[5]
+        rendered = render_base(derived)
+        # Do not move text unless it survives inside an actual table caption.
+        captions = BeautifulSoup(rendered, "html.parser").find_all("caption")
+        if not any(label in cap.get_text(" ", strip=True) for cap in captions):
+            continue
+        targets[m.start()] = derived
+        moved.add(matches[caption_index].start())
+    if not targets:
+        return markdown
+    derived = BLOCK.sub(
+        lambda m: "" if m.start() in moved else targets.get(m.start(), m[0]), raw
+    )
+    return render_base(derived)

nemotron_parse/engine/chart_caption.py

@@ -0,0 +1,71 @@
+# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: OpenMDW-1.1
+
+"""Render titles explicitly declared by native chart metadata as table captions."""
+import re
+from html import escape
+
+
+def transform(text):
+    changed = 0
+
+    def chart(m):
+        nonlocal changed
+        body = m[2]
+        if "<chart" in body:
+            return m[0]
+        title = re.search(r"(?m)^title:[ \t]*([^\n]*)$", body)
+        if not title:
+            return m[0]
+        title_text = title[1].strip()
+        if not title_text:
+            return m[0]
+        count = 0
+        pattern = re.compile(
+            r"(?m)^\|[^\n]*\|[ \t]*\n\|[ \t:|\-]+\|[ \t]*\n(?:\|[^\n]*\|[ \t]*(?:\n|$))+"
+        )
+
+        def table(t):
+            nonlocal count
+            raw = t[0]
+            if r"\|" in raw or "`" in raw:
+                return raw
+            lines = [a.strip() for a in raw.strip().splitlines()]
+            cells = [[c.strip() for c in line.strip("|").split("|")] for line in lines]
+            if (
+                len(cells) < 3
+                or any(len(row) != len(cells[0]) for row in cells)
+                or any(
+                    not re.fullmatch(r":?-+:?", c.replace(" ", "")) for c in cells[1]
+                )
+            ):
+                return raw
+            count += 1
+            result = (
+                "<table><caption>"
+                + escape(title_text)
+                + "</caption><thead><tr>"
+                + "".join('<th scope="col">' + escape(c) + "</th>" for c in cells[0])
+                + "</tr></thead><tbody>"
+            )
+            result += (
+                "".join(
+                    "<tr>"
+                    + "".join("<td>" + escape(c) + "</td>" for c in row)
+                    + "</tr>"
+                    for row in cells[2:]
+                )
+                + "</tbody></table>\n"
+            )
+            return result
+
+        new = pattern.sub(table, body)
+        if not count:
+            return m[0]
+        # The title is rendered in its associated captions, retaining text and roles.
+        new = re.sub(r"(?m)^title:[ \t]*[^\n]*\n?", "", new, count=1)
+        changed += count
+        return m[1] + new + m[3]
+
+    out = re.sub(r"(<chart\b[^>]*>)(.*?)(</chart>)", chart, text, flags=re.S)
+    return out, changed

nemotron_parse/engine/chart_context.py

@@ -0,0 +1,40 @@
+# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: OpenMDW-1.1
+"""Preserve explicit chart subtitles in their existing table-caption context."""
+import re
+from bs4 import BeautifulSoup, NavigableString
+
+TABLE = re.compile(r"<table\b[^>]*>.*?</table>", re.S | re.I)
+CHART = re.compile(r"(<chart\b[^>]*>)(.*?)(</chart>)", re.S | re.I)
+
+
+def add_native_subtitles(text):
+    def chart(match):
+        body = match[2]
+        labels = [
+            m[1].strip()
+            for m in re.finditer(r"(?m)^subtitle:[ \t]*([^\n]*)", body)
+            if m[1].strip()
+        ]
+        if not labels:
+            return match[0]
+
+        def table(m):
+            soup = BeautifulSoup(m[0], "html.parser")
+            node = soup.find("table")
+            cap = node.find("caption", recursive=False)
+            if not cap:
+                return m[0]
+            existing = cap.get_text(" ", strip=True)
+            added = []
+            for value in labels:
+                if value not in existing and value not in added:
+                    added.append(value)
+            if not added:
+                return m[0]
+            cap.append(NavigableString(" — " + " — ".join(added)))
+            return str(node)
+
+        return match[1] + TABLE.sub(table, body) + match[3]
+
+    return CHART.sub(chart, text)

nemotron_parse/engine/convert/__init__.py

@@ -0,0 +1,2 @@
+# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: OpenMDW-1.1

nemotron_parse/engine/convert/hybrid_cleanup.py

@@ -0,0 +1,67 @@
+# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: OpenMDW-1.1
+
+"""Escape LaTeX cell literals while preserving balanced nested inline markup."""
+import re
+
+ESCAPES = {
+    "#": r"\#",
+    "%": r"\%",
+    "$": r"\$",
+    "_": r"\_",
+    "~": r"\textasciitilde{}",
+    "^": r"\textasciicircum{}",
+}
+MATH_OR_TAG = re.compile(r"\$\$.*?\$\$|\$[^$]*\$|\\\(.*?\\\)|\\\[.*?\\\]|<[^>]+>", re.S)
+
+
+def _process_cell(text, escape_literals):
+    def scan(position, closing=None, depth=0):
+        if depth > 64:
+            raise ValueError("inline_markup_nesting_too_deep")
+        output = []
+        while position < len(text):
+            if closing and text.startswith(closing, position):
+                return "".join(output), position + len(closing), True
+            protected = MATH_OR_TAG.match(text, position)
+            if protected:
+                output.append(protected[0])
+                position = protected.end()
+                continue
+            if text[position] == "\\":
+                # Preserve escaped literals and command prefixes. Formatting
+                # inside command arguments is still scanned when reached.
+                output.append(text[position : position + 2])
+                position += 2
+                continue
+            marker = next(
+                (m for m in ("**", "__", "~~", "_") if text.startswith(m, position)),
+                None,
+            )
+            if marker:
+                body, end, balanced = scan(position + len(marker), marker, depth + 1)
+                if balanced:
+                    if escape_literals:
+                        output.extend((marker, body, marker))
+                    else:
+                        tag = {"**": "b", "__": "b", "_": "i", "~~": "s"}[marker]
+                        output.extend(("<" + tag + ">", body, "</" + tag + ">"))
+                    position = end
+                    continue
+            output.append(
+                ESCAPES.get(text[position], text[position])
+                if escape_literals
+                else text[position]
+            )
+            position += 1
+        return "".join(output), position, closing is None
+
+    return scan(0)[0]
+
+
+def escape_cell(text):
+    return _process_cell(text, True)
+
+
+def decode_styles(text):
+    return _process_cell(text, False)

nemotron_parse/engine/convert/hybrid_inline.py

@@ -0,0 +1,164 @@
+# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: OpenMDW-1.1
+
+"""Bounded Markdown/markup inline dialect inside LaTeX table cells.
+
+Escaped punctuation is literal. Formatting is parsed before literal decoding;
+literal text is never reparsed as markup. Unknown syntax fails explicitly.
+"""
+import html
+import re
+from bs4 import NavigableString, BeautifulSoup
+
+try:
+    from .semantic_styles import mark_open, ALIASES
+except ImportError:
+    from semantic_styles import mark_open, ALIASES
+try:
+    from .rich_table_reader import TAGS, LITERALS, argument
+except ImportError:
+    from rich_table_reader import TAGS, LITERALS, argument
+
+ESC = {
+    "&": r"\&",
+    "%": r"\%",
+    "$": r"\$",
+    "#": r"\#",
+    "_": r"\_",
+    "*": r"\*",
+    "{": r"\{",
+    "}": r"\}",
+    "\\": r"\textbackslash{}",
+    "~": r"\textasciitilde{}",
+    "^": r"\textasciicircum{}",
+    "<": r"\textless{}",
+    ">": r"\textgreater{}",
+}
+
+
+def encode_node(node):
+    if isinstance(node, NavigableString):
+        return "".join(ESC.get(char, char) for char in str(node))
+    if node.name == "mark":
+        return (
+            mark_open(node)
+            + "".join(encode_node(child) for child in node.contents)
+            + "</mark>"
+        )
+    if node.name in ALIASES:
+        node = BeautifulSoup(str(node), "html.parser").find()
+        node.name = ALIASES[node.name]
+    if node.name == "br":
+        return r"\linebreak{}"
+    content = "".join(encode_node(child) for child in node.contents)
+    if node.name in {"b", "strong"}:
+        return "**" + content + "**"
+    if node.name in {"i", "em"}:
+        # The historical cleanup deliberately strips single punctuation emphasis.
+        # Markup preserves that uncommon case without changing global cleanup.
+        before = str(node.previous_sibling or "")[-1:]
+        after = str(node.next_sibling or "")[:1]
+        if (
+            re.fullmatch(r"\\?[\W_]", node.get_text())
+            or not node.get_text().strip()
+            or re.match(r"\w", before)
+            or re.match(r"\w", after)
+        ):
+            return "<i>" + content + "</i>"
+        return "_" + content + "_"
+    if node.name == "s":
+        return "~~" + content + "~~"
+    if node.name == "u":
+        return "<u>" + content + "</u>"
+    if node.name in {"sup", "sub"}:
+        return "<" + node.name + ">" + content + "</" + node.name + ">"
+    return content
+
+
+def inline_html(text):
+    def parse(position, closing=None):
+        output = []
+        while position < len(text):
+            if closing and text.startswith(closing, position):
+                return "".join(output), position + len(closing)
+            if text.startswith("<br>", position):
+                output.append("<br>")
+                position += 4
+                continue
+            if text[position] == "\n":
+                output.append("<br>")
+                position += 1
+                continue
+            mark = re.match(r"<mark(?:\s[^<>]*)?>", text[position:], re.I)
+            if mark:
+                opener = mark_open(
+                    BeautifulSoup(mark[0] + "</mark>", "html.parser").mark
+                )
+                inner, position = parse(position + len(mark[0]), "</mark>")
+                output.append(opener + inner + "</mark>")
+                continue
+            marker = next(
+                (
+                    m
+                    for m in ("**", "_", "~~", "<u>", "<s>", "<i>", "<sup>", "<sub>")
+                    if text.startswith(m, position)
+                ),
+                None,
+            )
+            if marker:
+                tag = {
+                    "**": "b",
+                    "_": "i",
+                    "~~": "s",
+                    "<u>": "u",
+                    "<s>": "s",
+                    "<i>": "i",
+                    "<sup>": "sup",
+                    "<sub>": "sub",
+                }[marker]
+                end = marker if marker in {"**", "_", "~~"} else "</" + tag + ">"
+                inner, position = parse(position + len(marker), end)
+                output.append("<" + tag + ">" + inner + "</" + tag + ">")
+                continue
+            if text[position] != "\\":
+                if text[position] in "%$#_*{}^~<>":
+                    raise ValueError("unsupported_unescaped_hybrid_special")
+                output.append(html.escape(text[position]))
+                position += 1
+                continue
+            if position + 1 < len(text) and text[position + 1] in "&%$#_*{}":
+                output.append(html.escape(text[position + 1]))
+                position += 2
+                continue
+            match = re.match(r"\\([A-Za-z]+)", text[position:])
+            if not match:
+                raise ValueError("unsupported_hybrid_escape")
+            command = match[1]
+            position += len(match[0])
+            body, position = argument(text, position)
+            if command == "colorbox":
+                color = body
+                body, position = argument(text, position)
+                node = BeautifulSoup("<mark></mark>", "html.parser").mark
+                node["background"] = color
+                output.append(mark_open(node) + inline_html(body) + "</mark>")
+                continue
+            if command == "hl":
+                output.append("<mark>" + inline_html(body) + "</mark>")
+                continue
+            if command == "linebreak" and not body:
+                output.append("<br>")
+            elif command == "mbox" and not body:
+                pass
+            elif command in TAGS:
+                tag = TAGS[command]
+                output.append("<" + tag + ">" + inline_html(body) + "</" + tag + ">")
+            elif command in LITERALS and not body:
+                output.append(html.escape(LITERALS[command]))
+            else:
+                raise ValueError("unsupported_hybrid_command_" + command)
+        if closing:
+            raise ValueError("unclosed_hybrid_style_" + closing)
+        return "".join(output), position
+
+    return parse(0)[0]

nemotron_parse/engine/convert/latex2html.py

@@ -0,0 +1,653 @@
+# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: OpenMDW-1.1
+
+import csv
+import io
+import json
+import re
+
+from bs4 import BeautifulSoup
+
+
+def skip_whitespace(text, i):
+    """Advance index i past any whitespace."""
+    while i < len(text) and text[i].isspace():
+        i += 1
+    return i
+
+
+def parse_braced_argument(text, i):
+    """
+    Given text and an index i that should point at an opening '{',
+    return a tuple (argument_content, new_index) where argument_content is the full
+    string inside the balanced braces and new_index is the position just after the matching '}'.
+    """
+    if i >= len(text) or text[i] != "{":
+        raise ValueError("Expected '{' at position {}".format(i))
+    i += 1  # skip the opening brace
+    start = i
+    level = 1
+    while i < len(text) and level > 0:
+        if text[i] == "{":
+            level += 1
+        elif text[i] == "}":
+            level -= 1
+        i += 1
+    if level != 0:
+        raise ValueError("Unbalanced braces starting at position {}".format(start - 1))
+    # The argument content is from start to i-1 (excluding the closing brace)
+    return text[start : i - 1], i
+
+
+def parse_command(text, i):
+    r"""
+    Parse a \multirow or \multicolumn command starting at index i.
+    This function assumes the command has exactly three braced arguments.
+
+    It processes each argument recursively. For the third argument, after recursive processing,
+    it replaces any unescaped & with \&.
+
+    Returns a tuple (command_text, new_index) where command_text is the reconstructed command.
+    """
+    # Determine which command we have.
+    if text.startswith(r"\multirow", i):
+        command_name = r"\multirow"
+        i += len(r"\multirow")
+    elif text.startswith(r"\multicolumn", i):
+        command_name = r"\multicolumn"
+        i += len(r"\multicolumn")
+    else:
+        raise ValueError(
+            "Expected \\multirow or \\multicolumn at position {}".format(i)
+        )
+
+    # Skip whitespace between the command name and the first argument.
+    i = skip_whitespace(text, i)
+    args = []
+    # Expect exactly three arguments
+    for arg_index in range(3):
+        if i >= len(text) or text[i] != "{":
+            raise ValueError(
+                "Expected '{' for argument {} at position {}".format(arg_index + 1, i)
+            )
+        arg_content, i = parse_braced_argument(text, i)
+        # Process the content recursively to catch nested commands
+        processed_arg = clean_multi_cells(arg_content)
+        if arg_index == 2:
+            # For the cell text (third argument), replace any unescaped &
+            processed_arg = re.sub(r"(?<!\\)&", r"\\&", processed_arg)
+        args.append(processed_arg)
+        # Only skip whitespace between arguments, not after the last one.
+        if arg_index < 2:
+            i = skip_whitespace(text, i)
+    # Reconstruct the full command with its three arguments
+    command_text = f"{command_name}{{{args[0]}}}{{{args[1]}}}{{{args[2]}}}"
+    return command_text, i
+
+
+def clean_multi_cells(text):
+    r"""
+    Process an arbitrary LaTeX text string and look for occurrences of \multirow or \multicolumn commands.
+    When found, the command is parsed (handling nested braces and nested commands) and its third argument is fixed.
+
+    Returns the processed text.
+    """
+    result = []
+    i = 0
+    while i < len(text):
+        # Find next occurrence of either command.
+        idx_multi = text.find(r"\multirow", i)
+        idx_multiC = text.find(r"\multicolumn", i)
+
+        # Determine the next index among the two (if any)
+        if idx_multi == -1 and idx_multiC == -1:
+            result.append(text[i:])
+            break
+        if idx_multi == -1:
+            next_idx = idx_multiC
+        elif idx_multiC == -1:
+            next_idx = idx_multi
+        else:
+            next_idx = min(idx_multi, idx_multiC)
+
+        # Append text before the command (preserving any whitespace)
+        result.append(text[i:next_idx])
+        # Process the command starting at next_idx
+        command_text, new_index = parse_command(text, next_idx)
+        result.append(command_text)
+        i = new_index
+    return "".join(result)
+
+
+def parse_brace(s, pos):
+    """
+    Given a string s and an index pos pointing to an opening '{',
+    returns a tuple (content, new_pos) where content is the string
+    between the matching braces (handling nested braces) and new_pos is
+    the index just after the closing '}'.
+    """
+    if pos >= len(s) or s[pos] != "{":
+        raise ValueError("Expected '{' at position %d" % pos)
+    pos += 1  # skip the opening brace
+    content = ""
+    depth = 1
+    while pos < len(s) and depth:
+        char = s[pos]
+        if char == "{":
+            depth += 1
+            content += char
+        elif char == "}":
+            depth -= 1
+            if depth:
+                content += char
+        else:
+            content += char
+        pos += 1
+    if depth != 0:
+        raise ValueError("Unmatched '{' in string.")
+    return content, pos
+
+
+def parse_command_merge(s, pos):
+    """
+    Parse a multirow or multicolumn command starting at s[pos]. If the content
+    of the command contains a nested command, then recursively parse the inner
+    command and merge its parameters with the outer ones. The merging is done
+    so that the outer multirow’s parameters (e.g. rowspan and width) are kept
+    while the inner command’s parameters (e.g. colspan, alignment) and its innermost
+    content are returned.
+
+    Returns a tuple (merged_dict, new_pos) where merged_dict is a dictionary
+    containing the combined parameters and new_pos is the updated index after
+    parsing the command.
+    """
+    if s.startswith(r"\multirow", pos):
+        newpos = pos + len(r"\multirow")
+        # Parse the three required arguments for multirow: rowspan, width, and content.
+        rowspan, newpos = parse_brace(s, newpos)
+        width, newpos = parse_brace(s, newpos)
+        content, newpos = parse_brace(s, newpos)
+        # Look for a nested command (either \multirow or \multicolumn) in the content.
+        index_mr = content.find(r"\multirow")
+        index_mc = content.find(r"\multicolumn")
+        if index_mr == -1 and index_mc == -1:
+            # No nested command found; return this command’s details.
+            return {
+                "rowspan": rowspan.strip(),
+                "width": width.strip(),
+                "content": content.strip(),
+            }, newpos
+        else:
+            # At least one nested command is present. Pick the first occurrence.
+            indices = [i for i in (index_mr, index_mc) if i != -1]
+            first_index = min(indices)
+            # Parse the inner (nested) command from within the content.
+            inner, _ = parse_command_merge(content, first_index)
+            # Merge: keep the outer multirow’s parameters and add the inner ones.
+            merged = {"rowspan": rowspan.strip(), "width": width.strip()}
+            merged.update(inner)
+            return merged, newpos
+
+    elif s.startswith(r"\multicolumn", pos):
+        newpos = pos + len(r"\multicolumn")
+        # Parse the three arguments for multicolumn: colspan, alignment, and content.
+        colspan, newpos = parse_brace(s, newpos)
+        alignment, newpos = parse_brace(s, newpos)
+        content, newpos = parse_brace(s, newpos)
+        # Look for a nested command in the content.
+        index_mr = content.find(r"\multirow")
+        index_mc = content.find(r"\multicolumn")
+        if index_mr == -1 and index_mc == -1:
+            return {
+                "colspan": colspan.strip(),
+                "alignment": alignment.strip(),
+                "content": content.strip(),
+            }, newpos
+        else:
+            indices = [i for i in (index_mr, index_mc) if i != -1]
+            first_index = min(indices)
+            inner, _ = parse_command_merge(content, first_index)
+            merged = {"colspan": colspan.strip(), "alignment": alignment.strip()}
+            merged.update(inner)
+            return merged, newpos
+
+    # Not a recognized command starting at pos.
+    return None, pos
+
+
+def extract_merged_commands(s):
+    """
+    Scan through the LaTeX string s and extract merged multirow/multicolumn commands.
+    For each command found, if there is nesting the parser merges the outer and inner
+    parameters so that the final result includes both the rowspan (or width) and the colspan
+    (or alignment) along with the innermost content.
+
+    Returns a list of dictionaries.
+    """
+    pos = 0
+    results = []
+    while pos < len(s):
+        if s[pos] == "\\":
+            res, newpos = parse_command_merge(s, pos)
+            if res is not None:
+                results.append(res)
+                pos = newpos
+                continue
+        pos += 1
+    return results
+
+
+def remove_tags(html, tags_to_remove):
+    soup = BeautifulSoup(html, "html.parser")
+    # Loop through the tags to remove
+    for tag_name in tags_to_remove:
+        for tag in soup.find_all(tag_name):
+            # Move the children of the tag to the parent tag
+            tag.unwrap()  # This removes the tag but keeps its contents
+    # Return the modified HTML as a string
+    return str(soup)
+
+
+def convert_th_to_td(html):
+    """Replace all th tags with td tags"""
+    soup = BeautifulSoup(html, "html.parser")
+    for th_tag in soup.find_all("th"):
+        th_tag.name = "td"
+    return str(soup)
+
+
+def replace_italic(text):
+    pattern = re.compile(r"(?<!\\)_(.*?)(?<!\\)_")
+
+    def italic_replacer(match):
+        # Get the text inside the underscores.
+        content = match.group(1)
+        # Remove the escape (backslash) from any escaped underscores inside.
+        content = content.replace(r"\_", "_")
+        return f"<i>{content}</i>"
+
+    # Replace all occurrences of the pattern using the replacer function.
+    return pattern.sub(italic_replacer, text)
+
+
+def replace_bold(text):
+    pattern = re.compile(r"(?<!\\)\*\*(.*?)(?<!\\)\*\*")
+
+    def bold_replacer(match):
+        content = match.group(1)
+        # Unescape any escaped asterisks within the captured text.
+        content = content.replace(r"\*", "*")
+        return f"<b>{content}</b>"
+
+    return pattern.sub(bold_replacer, text)
+
+
+def latex_table_to_html(latex_str, add_head_body=False):
+    # Strict paired reader for rich distillation targets; legacy syntax follows.
+    try:
+        from .rich_table_reader import to_html as rich_to_html
+    except ImportError:
+        from rich_table_reader import to_html as rich_to_html
+    try:
+        from .rich_table_fallback import decode_known_inline, restore_header_cells
+    except ImportError:
+        from rich_table_fallback import decode_known_inline, restore_header_cells
+    # Pattern to match the entire tabular environment
+    table_pattern = r"\\begin{tabular}{([^}]*)}\s*(.*?)\\end{tabular}"
+
+    def process_cell(cell):
+        # Clean up cell content
+        cell = cell.strip()
+        if re.fullmatch(r"\{\s*\}", cell):
+            cell = ""
+
+        out = extract_merged_commands(cell)
+        if len(out) > 0:
+            cell = process_cell(out[0]["content"])["content"]
+            rowspan = int(out[0].get("rowspan", "1"))
+            colspan = int(out[0].get("colspan", "1"))
+            return {"content": cell, "colspan": colspan, "rowspan": rowspan}
+
+        # Replace latex and markdown formatting with HTML tags
+        cell = re.sub(r"\$([^$]*)\$", r"\1", cell)  # Remove math mode
+        cell = re.sub(r"\\textbf{([^}]*)}", r"<b>\1</b>", cell)  # Convert latex bold
+        cell = re.sub(r"\\textit{([^}]*)}", r"<i>\1</i>", cell)  # Convert latex italic
+        cell = re.sub(r"\\emph{([^}]*)}", r"<i>\1</i>", cell)
+        cell = re.sub(r"\\underline{([^}]*)}", r"<u>\1</u>", cell)
+        cell = re.sub(r"\\(?:sout|st){([^}]*)}", r"<s>\1</s>", cell)
+        cell = replace_italic(cell)
+        cell = replace_bold(cell)
+        cell = (
+            cell.replace("\\$", "$")
+            .replace("\\%", "%")
+            .replace("\\newline", "<br>")
+            .replace("\\textless", "<")
+            .replace("\\textgreater", ">")
+            .replace("\\*", "*")
+            .replace("\\_", "_")
+            .replace("\\backslash", "\\")
+        )
+
+        # Replace \& with & in the cell text
+        cell = cell.replace(r"\&", "&")
+        cell = cell.replace("<tbc>", "")
+        cell = re.sub(r"(?:\r?\n|<br\s*/?>)+", "<br>", cell, flags=re.IGNORECASE)
+        # Preserve in-cell line breaks as <br>; clean other tokens.
+        cell = (
+            cell.replace("\\unknown", "")
+            .replace("\\<|unk|\\>", "")
+            .replace("<underline>", "<u>")
+            .replace("</underline>", "</u>")
+        )
+        return {"content": cell, "colspan": 1, "rowspan": 1}
+
+    def split_row(input_string):
+        # Use a regular expression to split on '&' that is not preceded by a backslash
+        return re.split(r"(?<!\\)&", input_string)
+
+    def convert_table(match):
+        try:
+            rich = rich_to_html(match.group(0))
+        except ValueError:
+            rich = None
+        if rich is not None:
+            if add_head_body:
+                soup = BeautifulSoup(rich, "html.parser")
+                rows = list(soup.table.find_all("tr", recursive=False))
+                head = soup.new_tag("thead")
+                body = soup.new_tag("tbody")
+                in_header = True
+                for row in rows:
+                    cells = row.find_all(["td", "th"], recursive=False)
+                    in_header = (
+                        in_header and bool(cells) and all(c.name == "th" for c in cells)
+                    )
+                    (head if in_header else body).append(row.extract())
+                soup.table.append(head)
+                soup.table.append(body)
+                rich = str(soup)
+            return rich
+        # Extract table content
+        format_spec, content = match.groups()
+        content = decode_known_inline(content)
+
+        # Start building HTML table
+        html = ["<table>"]
+
+        # Track cells for multirow
+        multirow_tracker = set()
+
+        # Process rows
+        rows = re.split(r"\\\\", content)
+        current_row = 0
+
+        for row in rows:
+            if not row.strip():
+                continue
+
+            row = row.strip()
+
+            # Skip \hline
+            if "\\hline" in row:
+                row = row.replace("\\hline", "")
+                if not row.strip():
+                    continue
+
+            row = clean_multi_cells(row)
+
+            # Process cells
+            cells = split_row(row)
+            processed_cells = [process_cell(cell) for cell in cells]
+
+            if add_head_body:
+                if current_row == 0:
+                    html.append(" <thead>")
+                if current_row == 1:
+                    html.append(" <tbody>")
+            html.append("  <tr>")
+            current_col = 0
+
+            for cell in processed_cells:
+                content_segment = cell["content"]
+
+                attrs = []
+                if cell["colspan"] > 1:
+                    attrs.append(f'colspan="{cell["colspan"]}"')
+                if cell["rowspan"] > 1:
+                    attrs.append(f'rowspan="{cell["rowspan"]}"')
+                    for r in range(current_row + 1, current_row + cell["rowspan"]):
+                        for c in range(current_col, current_col + cell["colspan"]):
+                            multirow_tracker.add((r, c))
+
+                # Empty multicolumn placeholders under a prior multirow are
+                # covered cells too. Skip only a fully occupied, empty span;
+                # do not remove content or infer missing/partial occupancy.
+                if (
+                    content_segment == ""
+                    and cell["rowspan"] == 1
+                    and cell["colspan"] >= 1
+                    and all(
+                        (current_row, col) in multirow_tracker
+                        for col in range(current_col, current_col + cell["colspan"])
+                    )
+                ):
+                    current_col += cell["colspan"]
+                    continue
+
+                attr_str = " " + " ".join(attrs) if attrs else ""
+                cell_tag = "td"
+                html.append(f"    <{cell_tag}{attr_str}>{content_segment}</{cell_tag}>")
+                current_col += cell["colspan"]
+
+            if add_head_body and current_row == 0:
+                html.append(" </thead>")
+            html.append("  </tr>")
+            current_row += 1
+        if add_head_body:
+            html.append(" </tbody>")
+        html.append("</table>")
+        return restore_header_cells("\n".join(html))
+
+    # Convert all tabular environments in the input
+    return re.sub(table_pattern, convert_table, latex_str, flags=re.DOTALL)
+
+
+def convert_single_table(table):
+    """
+    Convert a single HTML table to Markdown format.
+
+    Args:
+        table: BeautifulSoup table element
+
+    Returns:
+        str: Markdown table string
+    """
+    markdown_lines = []
+    rows = table.find_all("tr")
+
+    for i, row in enumerate(rows):
+        cells = row.find_all(["td", "th"])
+        if not cells:
+            continue
+
+        # Convert cells to text, handling nested elements
+        row_data = []
+        for cell in cells:
+            # Get text content, handling nested elements
+            cell_text = cell.get_text(separator=" ", strip=True)
+            # Escape pipe characters
+            cell_text = cell_text.replace("|", "\\|")
+            row_data.append(cell_text)
+
+        # Add row to markdown
+        markdown_lines.append("| " + " | ".join(row_data) + " |")
+
+        # Add separator after header row
+        if i == 0:
+            separator = "| " + " | ".join(["---"] * len(cells)) + " |"
+            markdown_lines.append(separator)
+
+    return "\n".join(markdown_lines)
+
+
+def convert_html_tables_to_markdown(html_content):
+    """
+    Find all HTML tables and convert them to Markdown while preserving all other content.
+
+    Args:
+        html_content (str): HTML content that may contain tables
+
+    Returns:
+        str: HTML content with tables converted to Markdown
+    """
+    soup = BeautifulSoup(html_content, "html.parser")
+
+    # Find all tables
+    tables = soup.find_all("table")
+
+    if not tables:
+        return html_content  # Return original content unchanged
+
+    # Convert each table to markdown and replace it
+    for table in tables:
+        markdown_table = convert_single_table(table)
+
+        # Create a new element to replace the table
+        replacement = soup.new_string("\n" + markdown_table + "\n")
+        table.replace_with(replacement)
+
+    return str(soup)
+
+
+def _expand_spans(table):
+    """
+    Expand colspan/rowspan in a BeautifulSoup <table> into a regular grid.
+    Returns a list-of-lists of cell text strings.
+    """
+    rows = table.find_all("tr")
+    if not rows:
+        return []
+
+    num_cols = 0
+    for row in rows:
+        col_count = sum(int(c.get("colspan", 1)) for c in row.find_all(["td", "th"]))
+        num_cols = max(num_cols, col_count)
+
+    grid = [[None] * num_cols for _ in range(len(rows))]
+
+    for r, row in enumerate(rows):
+        cells = row.find_all(["td", "th"])
+        c = 0
+        for cell in cells:
+            while c < num_cols and grid[r][c] is not None:
+                c += 1
+            if c >= num_cols:
+                break
+
+            colspan = int(cell.get("colspan", 1))
+            rowspan = int(cell.get("rowspan", 1))
+            text = cell.get_text(separator=" ", strip=True)
+
+            for dr in range(rowspan):
+                for dc in range(colspan):
+                    rr, cc = r + dr, c + dc
+                    if rr < len(grid) and cc < num_cols:
+                        grid[rr][cc] = text
+            c += colspan
+
+    for row in grid:
+        for i in range(len(row)):
+            if row[i] is None:
+                row[i] = ""
+
+    return grid
+
+
+def _grid_to_records(grid):
+    """
+    Convert a grid (list-of-lists) to a list of dicts using the first row as
+    column headers. Duplicate or empty headers get disambiguated with a suffix.
+    """
+    if len(grid) < 2:
+        return grid
+
+    raw_headers = grid[0]
+    headers = []
+    seen = {}
+    for h in raw_headers:
+        key = h if h else "column"
+        if key in seen:
+            seen[key] += 1
+            key = f"{key}_{seen[key]}"
+        else:
+            seen[key] = 0
+        headers.append(key)
+
+    return [dict(zip(headers, row)) for row in grid[1:]]
+
+
+def convert_html_table_to_json(html_content, records=True):
+    """
+    Convert HTML tables produced by ``latex_table_to_html`` into JSON.
+
+    Parameters
+    ----------
+    html_content : str
+        HTML string (may contain one or more ``<table>`` elements and
+        surrounding non-table text).
+    records : bool
+        If *True* (default) the first row is treated as a header and each
+        subsequent row becomes a ``{header: value, ...}`` dict.
+        If *False* the table is returned as a plain list-of-lists.
+
+    Returns
+    -------
+    str
+        A JSON string.  If the input contains a single table the top-level
+        value is a list (of dicts or lists).  If there are multiple tables
+        the top-level value is a list of such lists.
+    """
+    soup = BeautifulSoup(html_content, "html.parser")
+    tables = soup.find_all("table")
+
+    if not tables:
+        return html_content
+
+    results = []
+    for table in tables:
+        grid = _expand_spans(table)
+        if records:
+            results.append(_grid_to_records(grid))
+        else:
+            results.append(grid)
+
+    payload = results[0] if len(results) == 1 else results
+    return json.dumps(payload, ensure_ascii=False, indent=2)
+
+
+def convert_html_table_to_csv(html_content):
+    """
+    Convert HTML tables produced by ``latex_table_to_html`` into CSV.
+
+    Each table becomes a CSV block separated by a blank line.
+
+    Returns
+    -------
+    str
+        CSV-formatted string.
+    """
+    soup = BeautifulSoup(html_content, "html.parser")
+    tables = soup.find_all("table")
+
+    if not tables:
+        return html_content
+
+    csv_blocks = []
+    for table in tables:
+        grid = _expand_spans(table)
+        buf = io.StringIO()
+        writer = csv.writer(buf)
+        writer.writerows(grid)
+        csv_blocks.append(buf.getvalue().rstrip("\r\n"))
+
+    return "\n\n".join(csv_blocks)

nemotron_parse/engine/convert/rich_table_fallback.py

@@ -0,0 +1,105 @@
+# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: OpenMDW-1.1
+
+"""Decode known rich inline commands for the legacy tabular fallback.
+
+Used only after strict rich-grid parsing fails. This preserves unknown commands
+and malformed arguments; it never invents cell contents or repairs table spans.
+"""
+import html
+import re
+
+try:
+    from .hybrid_cleanup import decode_styles
+except ImportError:
+    from hybrid_cleanup import decode_styles
+
+try:
+    from .rich_table_reader import TAGS, LITERALS, argument
+except ImportError:
+    from rich_table_reader import TAGS, LITERALS, argument
+
+
+def decode_known_inline(text, _depth=0):
+    """Decode balanced, recognized inline arguments without exposing delimiters."""
+    if _depth > 64:
+        return text
+    out = []
+    i = 0
+    known = set(TAGS) | set(LITERALS) | {"thead", "linebreak", "mbox", "hl", "colorbox"}
+    while i < len(text):
+        # Preserve escaped literal characters. In particular, an escaped
+        # backslash followed by 'thead' must not become a formatting command.
+        if text[i] == "\\" and i + 1 < len(text) and not text[i + 1].isalpha():
+            literal = text[i + 1]
+            out.append(
+                "\\&#" + str(ord(literal)) + ";"
+                if literal in "&%$#_*{}"
+                else text[i : i + 2]
+            )
+            i += 2
+            continue
+        match = re.match(r"\\([A-Za-z]+)\{", text[i:])
+        if not match or match[1] not in known:
+            out.append(text[i])
+            i += 1
+            continue
+        command = match[1]
+        try:
+            body, end = argument(text, i + len(command) + 1)
+        except ValueError:
+            out.append(text[i])
+            i += 1
+            continue
+        if command in {"hl", "colorbox"}:
+            try:
+                from .semantic_styles import mark_open
+            except ImportError:
+                from semantic_styles import mark_open
+            from bs4 import BeautifulSoup
+
+            node = BeautifulSoup("<mark></mark>", "html.parser").mark
+            if command == "colorbox":
+                node["background"] = body
+                try:
+                    body, end = argument(text, end)
+                except ValueError:
+                    out.append(text[i:end])
+                    i = end
+                    continue
+            out.append(
+                mark_open(node) + decode_known_inline(body, _depth + 1) + "</mark>"
+            )
+        elif command in TAGS or command == "thead":
+            inner = decode_known_inline(body, _depth + 1)
+            tag = "nvp-header-cell" if command == "thead" else TAGS[command]
+            out.append("<" + tag + ">" + inner + "</" + tag + ">")
+        elif command in LITERALS and not body:
+            # Entities cannot be mistaken for LaTeX row/column delimiters.
+            out.append("\\&#" + str(ord(LITERALS[command])) + ";")
+        elif command == "linebreak" and not body:
+            out.append("<nvp-line-break></nvp-line-break>")
+        elif command == "mbox" and not body:
+            pass
+        else:
+            out.append(text[i:end])
+        i = end
+    return decode_styles("".join(out))
+
+
+def restore_header_cells(html_text):
+    from bs4 import BeautifulSoup
+
+    # Restore explicit breaks after legacy whitespace normalization, which
+    # otherwise collapses consecutive linebreak commands.
+    html_text = html_text.replace("<nvp-line-break></nvp-line-break>", "<br>")
+    if "<nvp-header-cell>" not in html_text:
+        return html_text
+    soup = BeautifulSoup(html_text, "html.parser")
+    for cell in soup.find_all(["td", "th"]):
+        markers = cell.find_all("nvp-header-cell")
+        if markers:
+            cell.name = "th"
+            for marker in markers:
+                marker.unwrap()
+    return str(soup)

nemotron_parse/engine/convert/rich_table_reader.py

@@ -0,0 +1,164 @@
+# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: OpenMDW-1.1
+
+"""Strict rich_tabular_v3 reverse reader. Other LaTeX uses the legacy parser."""
+import html, re
+
+TAGS = {
+    "textstrong": "b",
+    "textem": "i",
+    "textbf": "b",
+    "textit": "i",
+    "underline": "u",
+    "sout": "s",
+    "textsuperscript": "sup",
+    "textsubscript": "sub",
+}
+
+LITERALS = {
+    "textbackslash": "\\",
+    "textasciitilde": "~",
+    "textasciicircum": "^",
+    "textless": "<",
+    "textgreater": ">",
+}
+
+
+def split_top(text, delimiter):
+    parts = []
+    start = 0
+    depth = 0
+    i = 0
+    while i < len(text):
+        if depth == 0 and text.startswith(delimiter, i):
+            parts.append(text[start:i])
+            i += len(delimiter)
+            start = i
+            continue
+        if text[i] == "\\":
+            i += 2
+            continue
+        if text[i] == "{":
+            depth += 1
+        elif text[i] == "}":
+            depth -= 1
+        if depth < 0:
+            raise ValueError("unbalanced_generated_latex")
+        i += 1
+    if depth:
+        raise ValueError("unbalanced_generated_latex")
+    parts.append(text[start:])
+    return parts
+
+
+def argument(text, pos):
+    if pos >= len(text) or text[pos] != "{":
+        raise ValueError("expected_braced_argument")
+    start = pos + 1
+    depth = 1
+    pos += 1
+    while pos < len(text):
+        if text[pos] == "\\":
+            pos += 2
+            continue
+        if text[pos] == "{":
+            depth += 1
+        elif text[pos] == "}":
+            depth -= 1
+            if depth == 0:
+                return text[start:pos], pos + 1
+        pos += 1
+    raise ValueError("unbalanced_generated_latex")
+
+
+def inline_html(text):
+    try:
+        from .hybrid_inline import inline_html as decode
+    except ImportError:
+        from hybrid_inline import inline_html as decode
+    return decode(text)
+
+
+def cell_parts(text):
+    text = text.strip()
+    rs = cs = 1
+    while text.startswith((r"\multicolumn", r"\multirow")):
+        command = "multicolumn" if text.startswith(r"\multicolumn") else "multirow"
+        pos = len(command) + 1
+        count, pos = argument(text, pos)
+        _, pos = argument(text, pos)
+        body, pos = argument(text, pos)
+        if text[pos:].strip():
+            raise ValueError("trailing_span_content")
+        if command == "multicolumn":
+            cs = int(count)
+        else:
+            rs = int(count)
+        text = body
+    header = False
+    if text.startswith(r"\thead{"):
+        body, end = argument(text, len(r"\thead"))
+        if text[end:].strip():
+            raise ValueError("trailing_header_content")
+        text = body
+        header = True
+    return rs, cs, inline_html(text), header
+
+
+def to_html(latex):
+    m = re.fullmatch(
+        r"\\begin\{tabular\}\{[clrx]+\}\n(.*?)\n\\end\{tabular\}", latex, re.S
+    )
+    if not m:
+        raise ValueError("unsupported_generated_tabular")
+    rows = split_top(m.group(1), r"\\")
+    if rows[-1].strip():
+        raise ValueError("missing_final_row_separator")
+    columns = len(re.match(r"\\begin\{tabular\}\{([clrx]+)\}", latex).group(1))
+    rows.pop()
+    occupied = set()
+    out = ["<table>"]
+    if not rows or len(rows) > 4096 or columns > 512:
+        raise ValueError("generated_grid_size")
+    for r, row in enumerate(rows):
+        out.append("<tr>")
+        c = 0
+        for part in split_top(row, "&"):
+            rs, cs, content, header = cell_parts(part)
+            if rs < 1 or cs < 1 or r + rs > len(rows) or c + cs > columns:
+                raise ValueError("invalid_generated_span")
+            if (r, c) in occupied:
+                if content or header or rs != 1:
+                    raise ValueError("nonempty_span_placeholder")
+                if any((r, cc) not in occupied for cc in range(c, c + cs)):
+                    raise ValueError("invalid_span_placeholder")
+                c += cs
+                continue
+            if any(
+                (rr, cc) in occupied
+                for rr in range(r, r + rs)
+                for cc in range(c, c + cs)
+            ):
+                raise ValueError("overlapping_generated_span")
+            tag = "th" if header else "td"
+            out.append(
+                "<"
+                + tag
+                + ' rowspan="'
+                + str(rs)
+                + '" colspan="'
+                + str(cs)
+                + '">'
+                + content
+                + "</"
+                + tag
+                + ">"
+            )
+            for rr in range(r, r + rs):
+                for cc in range(c, c + cs):
+                    occupied.add((rr, cc))
+            c += cs
+        if c != columns:
+            raise ValueError("generated_column_count")
+        out.append("</tr>")
+    return "".join(out) + "</table>"

nemotron_parse/engine/convert/semantic_styles.py

@@ -0,0 +1,68 @@
+# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: OpenMDW-1.1
+
+"""Lossless, bounded semantic inline styles shared by table and page conversion."""
+import html
+import re
+from bs4 import BeautifulSoup
+
+ALIASES = {
+    "strong": "b",
+    "em": "i",
+    "ins": "u",
+    "underline": "u",
+    "del": "s",
+    "strike": "s",
+}
+
+
+# Highlight color is deliberately outside the training target contract.
+def mark_color(node):
+    return None
+
+
+def mark_open(node):
+    return "<mark>"
+
+
+def normalize_semantic_html(value):
+    soup = BeautifulSoup(str(value), "html.parser")
+    for node in list(soup.find_all(True)):
+        node.name = ALIASES.get(node.name, node.name)
+        if node.name == "mark":
+            node.attrs = {}
+        if node.name != "span":
+            continue
+        props = {
+            k.strip().lower(): v.strip().lower()
+            for part in str(node.get("style", "")).split(";")
+            for k, sep, v in [part.partition(":")]
+            if sep
+        }
+        styles = []
+        if props.get("font-weight") in ["bold", "bolder", "600", "700", "800", "900"]:
+            styles.append("b")
+        if props.get("font-style") in ["italic", "oblique"]:
+            styles.append("i")
+        decoration = (
+            props.get("text-decoration", "")
+            + " "
+            + props.get("text-decoration-line", "")
+        )
+        if "underline" in decoration:
+            styles.append("u")
+        if "line-through" in decoration:
+            styles.append("s")
+        if props.get("vertical-align") in ["super", "sub"]:
+            styles.append("sup" if props["vertical-align"] == "super" else "sub")
+        if props.get("background-color") or props.get("background"):
+            styles.append("mark")
+        if not styles:
+            continue
+        for name in reversed(styles):
+            inner = soup.new_tag(name)
+            for child in list(node.contents):
+                inner.append(child.extract())
+            node.append(inner)
+        node.unwrap()
+    return soup

nemotron_parse/engine/geometry.py

@@ -0,0 +1,438 @@
+# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: OpenMDW-1.1
+
+from __future__ import annotations
+
+import argparse
+
+
+import base64
+
+
+import json
+
+
+import math
+
+
+import mimetypes
+
+
+import re
+
+
+from dataclasses import dataclass
+
+
+from pathlib import Path
+
+
+from typing import Any
+
+
+import numpy as np
+
+
+from PIL import Image
+
+
+BLOCK_RE = re.compile(
+    r"<x_(\d+(?:\.\d+)?)><y_(\d+(?:\.\d+)?)>(.*?)"
+    r"<x_(\d+(?:\.\d+)?)><y_(\d+(?:\.\d+)?)><class_([^>]+)>",
+    re.MULTILINE | re.DOTALL,
+)
+
+
+@dataclass(frozen=True)
+class PageImage:
+    key: str
+    image_name: str
+    image_path: Path
+    width: int
+    height: int
+    category: str | None = None
+
+
+@dataclass(frozen=True)
+class CropSpec:
+    crop_id: str
+    page_key: str
+    image_name: str
+    page_image_path: str
+    crop_image_name: str
+    crop_image_path: str
+    element_index: int
+    detector_label: str
+    detector_score: float
+    detector_bbox_norm: tuple[float, float, float, float]
+    crop_bbox_pixels: tuple[int, int, int, int]
+    page_size: tuple[int, int]
+
+
+def write_jsonl(path: Path, rows: list[dict[str, Any]]) -> None:
+    path.parent.mkdir(parents=True, exist_ok=True)
+    with path.open("w", encoding="utf-8") as f:
+        for row in rows:
+            f.write(json.dumps(row, ensure_ascii=False) + "\n")
+
+
+def normalize_detector_label(label: Any, labels: Any = None) -> str:
+    if isinstance(label, (int, float, np.integer, np.floating)) and labels is not None:
+        try:
+            label_idx = int(label)
+            if isinstance(labels, dict):
+                label = labels.get(label_idx, labels.get(str(label_idx), label))
+            else:
+                label = labels[label_idx]
+        except Exception:
+            pass
+    text = str(label).strip().lower()
+    text = (
+        text.replace("&", "and").replace("/", "_").replace("-", "_").replace(" ", "_")
+    )
+    text = text.replace("__", "_")
+    text = re.sub(r"_+", "_", text).strip("_")
+    if text in {
+        "headers_and_footers",
+        "header_and_footer",
+        "header_footer",
+        "headers_footers",
+    }:
+        return "header_footer"
+    if text in {
+        "header",
+        "footer",
+        "page_header",
+        "page_footer",
+        "header_image",
+        "footer_image",
+    }:
+        return "header_footer"
+    if text in {"tables"}:
+        return "table"
+    if text in {"charts"}:
+        return "chart"
+    if text in {
+        "texts",
+        "paragraph",
+        "paragraphs",
+        "paragraph_title",
+        "content",
+        "abstract",
+        "references",
+        "reference",
+        "reference_content",
+        "footnote",
+        "page_number",
+        "number",
+        "formula",
+        "formula_number",
+        "inline_formula",
+        "display_formula",
+        "algorithm",
+        "aside_text",
+        "sidebar_text",
+        "list",
+        "lists",
+        "vertical_text",
+    }:
+        return "text"
+    if text in {
+        "title",
+        "titles",
+        "document_title",
+        "section_header",
+        "figure_title",
+        "table_title",
+        "chart_title",
+        "figure_table_title",
+        "figure_caption",
+        "table_caption",
+        "image_caption",
+    }:
+        return "title"
+    if text in {
+        "image",
+        "images",
+        "figure",
+        "figures",
+        "picture",
+        "pictures",
+        "seal",
+        "stamp",
+    }:
+        return "infographic"
+    if text in {"infographics"}:
+        return "infographic"
+    return text
+
+
+def parse_label_filter(value: str) -> set[str]:
+    if value.strip().lower() in {"", "*", "all"}:
+        return set()
+    return {normalize_detector_label(item) for item in value.split(",") if item.strip()}
+
+
+def clamp(value: float, low: float = 0.0, high: float = 1.0) -> float:
+    return min(high, max(low, value))
+
+
+def crop_bbox_pixels(
+    bbox_norm: tuple[float, float, float, float],
+    width: int,
+    height: int,
+    *,
+    margin_px: int,
+    margin_ratio: float,
+) -> tuple[int, int, int, int]:
+    x1, y1, x2, y2 = bbox_norm
+    px1, py1, px2, py2 = x1 * width, y1 * height, x2 * width, y2 * height
+    mx = margin_px + (px2 - px1) * margin_ratio
+    my = margin_px + (py2 - py1) * margin_ratio
+    left = max(0, math.floor(px1 - mx))
+    top = max(0, math.floor(py1 - my))
+    right = min(width, math.ceil(px2 + mx))
+    bottom = min(height, math.ceil(py2 + my))
+    return left, top, right, bottom
+
+
+def filter_detection_rows(
+    args: argparse.Namespace, rows: list[dict[str, Any]]
+) -> list[dict[str, Any]]:
+    include_labels = parse_label_filter(args.include_detector_labels)
+    filtered_rows = []
+    for row in rows:
+        detections = []
+        for det in row.get("detections") or []:
+            label = normalize_detector_label(det.get("label") or "")
+            if include_labels and label not in include_labels:
+                continue
+            if float(det.get("score") or 0.0) < args.detector_score_min:
+                continue
+            kept = dict(det)
+            kept["label"] = label
+            detections.append(kept)
+        if args.detector_order == "spatial":
+            detections.sort(
+                key=lambda item: (
+                    item["bbox_norm"][1],
+                    item["bbox_norm"][0],
+                    -float(item.get("score") or 0.0),
+                )
+            )
+        if args.max_elements_per_page > 0:
+            detections = detections[: args.max_elements_per_page]
+        out = dict(row)
+        out["detections"] = detections
+        filtered_rows.append(out)
+    return filtered_rows
+
+
+def make_fallback_detection(page: PageImage) -> dict[str, Any]:
+    return {
+        "label": "text",
+        "score": 0.0,
+        "bbox_norm": [0.0, 0.0, 1.0, 1.0],
+        "fallback": True,
+    }
+
+
+def build_crops(
+    args: argparse.Namespace,
+    pages: list[PageImage],
+    detections: list[dict[str, Any]],
+    crops_dir: Path,
+    crops_manifest_path: Path,
+) -> list[CropSpec]:
+    pages_by_name = {page.image_name: page for page in pages}
+    pages_by_key = {page.key: page for page in pages}
+    crops: list[CropSpec] = []
+    for row in detections:
+        page = pages_by_key.get(str(row.get("key"))) or pages_by_name.get(
+            str(row.get("image_name"))
+        )
+        if page is None:
+            continue
+        items = list(row.get("detections") or [])
+        if not items and args.fallback_full_page:
+            items = [make_fallback_detection(page)]
+        if not items:
+            continue
+        with Image.open(page.image_path) as image:
+            image = image.convert("RGB")
+            for element_index, det in enumerate(items):
+                bbox_norm = tuple(float(v) for v in det["bbox_norm"])
+                crop_px = crop_bbox_pixels(
+                    bbox_norm,
+                    page.width,
+                    page.height,
+                    margin_px=args.crop_margin_px,
+                    margin_ratio=args.crop_margin_ratio,
+                )
+                left, top, right, bottom = crop_px
+                if (
+                    right - left < args.min_crop_size
+                    or bottom - top < args.min_crop_size
+                ):
+                    continue
+                crop_image_name = f"{page.key}__pe_{element_index:03d}.jpg"
+                crop_image_path = crops_dir / crop_image_name
+                crop_image_path.parent.mkdir(parents=True, exist_ok=True)
+                if not crop_image_path.is_file() or crop_image_path.stat().st_size == 0:
+                    image.crop((left, top, right, bottom)).save(
+                        crop_image_path, quality=95
+                    )
+                crops.append(
+                    CropSpec(
+                        crop_id=f"{page.key}__pe_{element_index:03d}",
+                        page_key=page.key,
+                        image_name=page.image_name,
+                        page_image_path=str(page.image_path),
+                        crop_image_name=crop_image_name,
+                        crop_image_path=str(crop_image_path),
+                        element_index=element_index,
+                        detector_label=str(det.get("label") or "text"),
+                        detector_score=float(det.get("score") or 0.0),
+                        detector_bbox_norm=bbox_norm,
+                        crop_bbox_pixels=crop_px,
+                        page_size=(page.width, page.height),
+                    )
+                )
+    write_jsonl(crops_manifest_path, [crop.__dict__ for crop in crops])
+    print(f"Prepared {len(crops)} crops under {crops_dir}", flush=True)
+    return crops
+
+
+def image_data_url(path: Path) -> str:
+    mime = mimetypes.guess_type(path.name)[0] or "image/jpeg"
+    data = base64.b64encode(path.read_bytes()).decode("ascii")
+    return f"data:{mime};base64,{data}"
+
+
+def parse_model_detection_predictions(
+    args: argparse.Namespace,
+    pages: list[PageImage],
+    prediction_rows: list[dict[str, Any]],
+) -> list[dict[str, Any]]:
+    page_by_key = {page.key: page for page in pages}
+    rows = []
+    for prediction in prediction_rows:
+        page = page_by_key.get(str(prediction.get("key")))
+        if page is None:
+            continue
+        detections = []
+        for match in BLOCK_RE.finditer(str(prediction.get("raw_out") or "")):
+            x1, y1, _body, x2, y2, cls = match.groups()
+            bbox_norm = model_bbox_to_image_norm(
+                (float(x1), float(y1), float(x2), float(y2)),
+                page.width,
+                page.height,
+                coordinate_frame=args.model_coordinate_frame,
+                canvas_width=args.canvas_width,
+                canvas_height=args.canvas_height,
+                pad_position=args.pad_position,
+            )
+            if bbox_norm[2] <= bbox_norm[0] or bbox_norm[3] <= bbox_norm[1]:
+                continue
+            detections.append(
+                {
+                    "label": normalize_detector_label(cls),
+                    "score": 1.0,
+                    "bbox_norm": list(bbox_norm),
+                    "model_class": cls,
+                    "model_order": len(detections),
+                }
+            )
+        rows.append(
+            {
+                "key": page.key,
+                "image_name": page.image_name,
+                "image_path": str(page.image_path),
+                "page_size": [page.width, page.height],
+                "detector_backend": "model-detection",
+                "model_coordinate_frame": args.model_coordinate_frame,
+                "detections": detections,
+                "detection_error": prediction.get("error"),
+            }
+        )
+    return filter_detection_rows(args, rows)
+
+
+def model_canvas_norm_to_image_norm(
+    bbox: tuple[float, float, float, float],
+    width: int,
+    height: int,
+    *,
+    canvas_width: float,
+    canvas_height: float,
+    pad_position: str,
+) -> tuple[float, float, float, float]:
+    x1, y1, x2, y2 = bbox
+    max_coord = max(abs(x1), abs(y1), abs(x2), abs(y2))
+    if max_coord > 1.5 and max_coord <= 1000.0:
+        norm = (x1 / 1000.0, y1 / 1000.0, x2 / 1000.0, y2 / 1000.0)
+    elif max_coord > 1000.0:
+        norm = (
+            x1 / max(1, width),
+            y1 / max(1, height),
+            x2 / max(1, width),
+            y2 / max(1, height),
+        )
+    else:
+        norm = (x1, y1, x2, y2)
+    scale = min(1.0, canvas_width / max(1, width), canvas_height / max(1, height))
+    resized_width = width * scale
+    resized_height = height * scale
+    if pad_position == "top-left":
+        pad_x = 0.0
+        pad_y = 0.0
+    elif pad_position == "bottom-right":
+        pad_x = canvas_width - resized_width
+        pad_y = canvas_height - resized_height
+    else:
+        pad_x = (canvas_width - resized_width) / 2.0
+        pad_y = (canvas_height - resized_height) / 2.0
+    out = (
+        (norm[0] * canvas_width - pad_x) / max(1.0, resized_width),
+        (norm[1] * canvas_height - pad_y) / max(1.0, resized_height),
+        (norm[2] * canvas_width - pad_x) / max(1.0, resized_width),
+        (norm[3] * canvas_height - pad_y) / max(1.0, resized_height),
+    )
+    left, right = sorted((clamp(out[0]), clamp(out[2])))
+    top, bottom = sorted((clamp(out[1]), clamp(out[3])))
+    return left, top, right, bottom
+
+
+def model_bbox_to_image_norm(
+    bbox: tuple[float, float, float, float],
+    width: int,
+    height: int,
+    *,
+    coordinate_frame: str,
+    canvas_width: float,
+    canvas_height: float,
+    pad_position: str,
+) -> tuple[float, float, float, float]:
+    if coordinate_frame in {
+        "content_normalized",
+        "original_normalized",
+        "navit_content",
+    }:
+        x1, y1, x2, y2 = bbox
+        max_coord = max(abs(x1), abs(y1), abs(x2), abs(y2))
+        if max_coord > 1.5 and max_coord <= 1000.0:
+            x1, y1, x2, y2 = (value / 1000.0 for value in (x1, y1, x2, y2))
+        elif max_coord > 1000.0:
+            x1, x2 = x1 / max(1, width), x2 / max(1, width)
+            y1, y2 = y1 / max(1, height), y2 / max(1, height)
+        left, right = sorted((clamp(x1), clamp(x2)))
+        top, bottom = sorted((clamp(y1), clamp(y2)))
+        return left, top, right, bottom
+    return model_canvas_norm_to_image_norm(
+        bbox,
+        width,
+        height,
+        canvas_width=canvas_width,
+        canvas_height=canvas_height,
+        pad_position=pad_position,
+    )

nemotron_parse/engine/header_inference_probe.py

@@ -0,0 +1,84 @@
+# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: OpenMDW-1.1
+
+"""Infer leading header rows from predicted table spans.
+No benchmark labels, source-image metadata, or dataset IDs are accepted as input.
+"""
+import re
+from bs4 import BeautifulSoup
+
+
+def infer_header_rows(raw):
+    soup = BeautifulSoup(raw, "html.parser")
+    diagnostics = []
+    for table in soup.find_all("table"):
+        if table.find_parent("table"):
+            continue
+        rows = [r for r in table.find_all("tr") if r.find_parent("table") == table]
+        if not rows:
+            continue
+        # Respect existing explicit cell roles; this probe is only for td-only output.
+        if table.find("th"):
+            diagnostics.append({"changed": False, "reason": "existing_th"})
+            continue
+        start = 0
+        while start < min(3, len(rows) - 1):
+            cells = rows[start].find_all(["td", "th"], recursive=False)
+            if (
+                len(cells) == 1
+                and int(cells[0].get("colspan", 1)) > 1
+                and int(cells[0].get("rowspan", 1)) == 1
+            ):
+                start += 1
+            else:
+                break
+        cells = rows[start].find_all(["td", "th"], recursive=False)
+        if not cells:
+            continue
+        numeric = lambda s: bool(re.fullmatch(r"[\s\d.,%$€£()+−-]+", s))
+        texts = [
+            c.get_text(" ", strip=True) for c in cells if c.get_text(" ", strip=True)
+        ]
+        if not texts or sum(numeric(t) for t in texts) / len(texts) > 0.5:
+            diagnostics.append({"changed": False, "reason": "numeric_first_row"})
+            continue
+        end = start + 1
+        rs = max(int(c.get("rowspan", 1)) for c in cells)
+        cs = max(int(c.get("colspan", 1)) for c in cells)
+        if 1 < rs <= 4:
+            end = min(len(rows), start + rs)
+        elif rs > 4:
+            diagnostics.append({"changed": False, "reason": "long_rowspan"})
+            continue
+        elif cs > 1 and start + 1 < len(rows):
+            nxt = [
+                c
+                for c in rows[start + 1].find_all(["td", "th"], recursive=False)
+                if c.get_text(" ", strip=True)
+            ]
+            # A group heading's colspan does not establish the next row's role.
+            # Require explicit header styling on every nonempty next-row cell;
+            # text-heavy data rows and year-valued data must not become headers.
+            if (
+                len(nxt) >= 2
+                and all(c.find(["b", "strong"]) for c in nxt)
+                and sum(numeric(c.get_text(" ", strip=True)) for c in nxt) / len(nxt)
+                < 0.5
+            ):
+                end = start + 2
+        # Only probe multi-level headers. Single-row tables remain byte-for-byte as supplied.
+        if end - start <= 1:
+            diagnostics.append({"changed": False, "reason": "single_level"})
+            continue
+        for row in rows[start:end]:
+            for c in row.find_all(["td", "th"], recursive=False):
+                c.name = "th"
+        diagnostics.append(
+            {
+                "changed": True,
+                "start": start,
+                "end": end,
+                "rule": "bounded_span_header_topology",
+            }
+        )
+    return str(soup), diagnostics

nemotron_parse/engine/markdown.py

@@ -0,0 +1,61 @@
+# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: OpenMDW-1.1
+
+from __future__ import annotations
+
+
+import re
+
+
+_RE_EXTRACT_CLASS_BBOX = re.compile(
+    r"<x_(\d+(?:\.\d+)?)><y_(\d+(?:\.\d+)?)>(.*?)<x_(\d+(?:\.\d+)?)><y_(\d+(?:\.\d+)?)><class_([^>]+)>",
+    re.DOTALL,
+)
+
+
+def extract_classes_bboxes(
+    text: str,
+) -> tuple[list[str], list[tuple[float, float, float, float]], list[str]]:
+    classes: list[str] = []
+    bboxes: list[tuple[float, float, float, float]] = []
+    texts: list[str] = []
+    for match in _RE_EXTRACT_CLASS_BBOX.finditer(text or ""):
+        x1, y1, body, x2, y2, cls = match.groups()
+        classes.append(cls)
+        bboxes.append((float(x1), float(y1), float(x2), float(y2)))
+        texts.append(body)
+    return classes, bboxes, texts
+
+
+def convert_latex_tables_in_markdown(markdown_text: str) -> str:
+    if "\\begin{tabular}" not in markdown_text:
+        return markdown_text
+    try:
+        from .convert.latex2html import latex_table_to_html
+
+        return latex_table_to_html(markdown_text)
+    except (ValueError, IndexError):
+        return markdown_text
+
+
+def mmd_to_markdown(prediction: str, text_transform=None) -> str:
+    classes, _boxes, texts = extract_classes_bboxes(prediction or "")
+    if not texts:
+        cleaned = re.sub(r"<x_[^>]+>|<y_[^>]+>|<class_[^>]+>", "", prediction or "")
+        return convert_latex_tables_in_markdown(cleaned.replace("<tbc>", "").strip())
+
+    blocks = []
+    for cls, text in zip(classes, texts):
+        if text_transform is not None:
+            text = text_transform(cls, text)
+        text = text.replace("<tbc>", "").strip()
+        if not text:
+            continue
+        if cls in {"Table", "Chart"}:
+            text = convert_latex_tables_in_markdown(text)
+        elif cls == "Title" and not text.startswith("#"):
+            text = f"# {text}"
+        elif cls == "Section-header" and not text.startswith("#"):
+            text = f"## {text}"
+        blocks.append(text)
+    return "\n\n".join(blocks).strip()

nemotron_parse/engine/merge.py

@@ -0,0 +1,630 @@
+# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: OpenMDW-1.1
+
+from __future__ import annotations
+
+import argparse
+
+
+import json
+
+
+import re
+
+
+from dataclasses import dataclass
+
+
+from pathlib import Path
+
+
+from typing import Any
+
+
+BLOCK_RE = re.compile(
+    r"<x_(\d+(?:\.\d+)?)><y_(\d+(?:\.\d+)?)>(.*?)"
+    r"<x_(\d+(?:\.\d+)?)><y_(\d+(?:\.\d+)?)><class_([^>]+)>",
+    re.DOTALL,
+)
+
+
+TOKEN_RE = re.compile(r"</?s>|<pad>|<unk>|<image>|<\|[^>]+\|>")
+
+
+@dataclass(frozen=True)
+class Block:
+    x1: float
+    y1: float
+    body: str
+    x2: float
+    y2: float
+    cls: str
+    raw: str
+    detector_label: str = ""
+    origin: str = ""
+    detector_score: float = 0.0
+
+    @property
+    def bbox(self) -> tuple[float, float, float, float]:
+        left, right = sorted((self.x1, self.x2))
+        top, bottom = sorted((self.y1, self.y2))
+        return left, top, right, bottom
+
+
+def load_jsonl(path: Path) -> list[dict[str, Any]]:
+    rows = []
+    with path.open(encoding="utf-8", errors="replace") as f:
+        for line_no, line in enumerate(f, 1):
+            if not line.strip():
+                continue
+            try:
+                rows.append(json.loads(line))
+            except json.JSONDecodeError as exc:
+                raise ValueError(f"{path}:{line_no}: {exc}") from exc
+    return rows
+
+
+def row_aliases(row: dict[str, Any]) -> set[str]:
+    aliases = set()
+    for key in ("key", "image_name"):
+        value = row.get(key)
+        if value:
+            text = str(value)
+            aliases.add(text)
+            aliases.add(Path(text).stem)
+            aliases.add(Path(text).name)
+    path = row.get("image_path")
+    if path:
+        aliases.add(Path(str(path)).name)
+        aliases.add(Path(str(path)).stem)
+    aliases = {alias for alias in aliases if alias}
+    return aliases
+
+
+def index_rows(rows: list[dict[str, Any]]) -> dict[str, dict[str, Any]]:
+    out = {}
+    for row in rows:
+        for alias in row_aliases(row):
+            out.setdefault(alias, row)
+    return out
+
+
+def prediction_text(row: dict[str, Any] | None) -> str:
+    if not row:
+        return ""
+    for key in ("raw_out", "output_text", "prediction", "pred", "text"):
+        value = row.get(key)
+        if isinstance(value, str):
+            return value
+    return ""
+
+
+def parse_blocks(raw: str) -> list[Block]:
+    blocks = []
+    for match in BLOCK_RE.finditer(raw or ""):
+        x1, y1, body, x2, y2, cls = match.groups()
+        blocks.append(
+            Block(float(x1), float(y1), body, float(x2), float(y2), cls, match.group(0))
+        )
+    return blocks
+
+
+def block_with_raw(
+    bbox: tuple[float, float, float, float],
+    cls: str,
+    body: str,
+    *,
+    detector_label: str = "",
+    origin: str = "",
+    detector_score: float = 0.0,
+) -> Block:
+    x1, y1, x2, y2 = bbox
+    raw = make_block(bbox, cls, body)
+    return Block(x1, y1, body, x2, y2, cls, raw, detector_label, origin, detector_score)
+
+
+def rebuild_blocks(blocks: list[Block]) -> str:
+    return "\n\n".join(block.raw for block in blocks).strip()
+
+
+def iou(
+    a: tuple[float, float, float, float], b: tuple[float, float, float, float]
+) -> float:
+    ax1, ay1, ax2, ay2 = a
+    bx1, by1, bx2, by2 = b
+    ix1, iy1 = max(ax1, bx1), max(ay1, by1)
+    ix2, iy2 = min(ax2, bx2), min(ay2, by2)
+    iw, ih = max(0.0, ix2 - ix1), max(0.0, iy2 - iy1)
+    inter = iw * ih
+    if inter <= 0:
+        return 0.0
+    area_a = max(0.0, ax2 - ax1) * max(0.0, ay2 - ay1)
+    area_b = max(0.0, bx2 - bx1) * max(0.0, by2 - by1)
+    denom = area_a + area_b - inter
+    return inter / denom if denom > 0 else 0.0
+
+
+def block_area(block: Block) -> float:
+    left, top, right, bottom = block.bbox
+    return max(0.0, right - left) * max(0.0, bottom - top)
+
+
+def normalized_text(text: str) -> str:
+    return " ".join(re.sub(r"<[^>]+>", " ", text or "").lower().split())
+
+
+def text_similarity(a: str, b: str) -> float:
+    a_tokens = set(normalized_text(a).split())
+    b_tokens = set(normalized_text(b).split())
+    if not a_tokens and not b_tokens:
+        return 1.0
+    if not a_tokens or not b_tokens:
+        return 0.0
+    return len(a_tokens & b_tokens) / len(a_tokens | b_tokens)
+
+
+def clamp(value: float) -> float:
+    return min(1.0, max(0.0, float(value)))
+
+
+def fmt_coord(value: float) -> str:
+    return f"{clamp(value):.6f}".rstrip("0").rstrip(".") or "0"
+
+
+def make_block(bbox: tuple[float, float, float, float], cls: str, text: str) -> str:
+    x1, y1, x2, y2 = bbox
+    return f"<x_{fmt_coord(x1)}><y_{fmt_coord(y1)}>{text}<x_{fmt_coord(x2)}><y_{fmt_coord(y2)}><class_{cls}>"
+
+
+def strip_model_tokens(text: str) -> str:
+    return TOKEN_RE.sub("", text or "").replace("<tbc>", "").strip()
+
+
+def crop_markdown(raw_text: str) -> str:
+    blocks = []
+    for match in BLOCK_RE.finditer(raw_text or ""):
+        body = strip_model_tokens(match.group(3))
+        if body:
+            blocks.append(body)
+    if blocks:
+        return "\n\n".join(blocks).strip()
+    cleaned = re.sub(r"<x_[^>]+>|<y_[^>]+>|<class_[^>]+>", "", raw_text or "")
+    return strip_model_tokens(cleaned)
+
+
+def normalize_detector_label(label: Any) -> str:
+    text = str(label).strip().lower()
+    text = (
+        text.replace("&", "and").replace("/", "_").replace("-", "_").replace(" ", "_")
+    )
+    text = re.sub(r"_+", "_", text).strip("_")
+    return text
+
+
+def parse_label_filter(value: str) -> set[str]:
+    return {normalize_detector_label(item) for item in value.split(",") if item.strip()}
+
+
+def class_for_detector(label: str, bbox_norm: tuple[float, float, float, float]) -> str:
+    label = normalize_detector_label(label)
+    if label == "table":
+        return "Table"
+    if label == "chart":
+        return "Chart"
+    if label == "infographic":
+        return "Picture"
+    if label == "title":
+        return "Title"
+    if label == "header_footer":
+        return (
+            "Page-header"
+            if ((bbox_norm[1] + bbox_norm[3]) / 2.0) < 0.5
+            else "Page-footer"
+        )
+    return "Text"
+
+
+def model_canvas_norm_to_image_norm(
+    bbox: tuple[float, float, float, float],
+    width: int,
+    height: int,
+    *,
+    canvas_width: float,
+    canvas_height: float,
+    pad_position: str,
+) -> tuple[float, float, float, float]:
+    x1, y1, x2, y2 = bbox
+    max_coord = max(abs(x1), abs(y1), abs(x2), abs(y2))
+    if max_coord > 1.5 and max_coord <= 1000.0:
+        norm = (x1 / 1000.0, y1 / 1000.0, x2 / 1000.0, y2 / 1000.0)
+    elif max_coord > 1000.0:
+        norm = (
+            x1 / max(1, width),
+            y1 / max(1, height),
+            x2 / max(1, width),
+            y2 / max(1, height),
+        )
+    else:
+        norm = (x1, y1, x2, y2)
+    scale = min(1.0, canvas_width / max(1, width), canvas_height / max(1, height))
+    resized_width = width * scale
+    resized_height = height * scale
+    if pad_position == "top-left":
+        pad_x = 0.0
+        pad_y = 0.0
+    elif pad_position == "bottom-right":
+        pad_x = canvas_width - resized_width
+        pad_y = canvas_height - resized_height
+    else:
+        pad_x = (canvas_width - resized_width) / 2.0
+        pad_y = (canvas_height - resized_height) / 2.0
+    out = (
+        (norm[0] * canvas_width - pad_x) / max(1.0, resized_width),
+        (norm[1] * canvas_height - pad_y) / max(1.0, resized_height),
+        (norm[2] * canvas_width - pad_x) / max(1.0, resized_width),
+        (norm[3] * canvas_height - pad_y) / max(1.0, resized_height),
+    )
+    left, right = sorted((clamp(out[0]), clamp(out[2])))
+    top, bottom = sorted((clamp(out[1]), clamp(out[3])))
+    return left, top, right, bottom
+
+
+def image_norm_to_model_canvas_norm(
+    bbox: tuple[float, float, float, float],
+    width: int,
+    height: int,
+    *,
+    canvas_width: float,
+    canvas_height: float,
+    pad_position: str,
+) -> tuple[float, float, float, float]:
+    x1, y1, x2, y2 = bbox
+    scale = min(1.0, canvas_width / max(1, width), canvas_height / max(1, height))
+    resized_width = width * scale
+    resized_height = height * scale
+    if pad_position == "top-left":
+        pad_x = 0.0
+        pad_y = 0.0
+    elif pad_position == "bottom-right":
+        pad_x = canvas_width - resized_width
+        pad_y = canvas_height - resized_height
+    else:
+        pad_x = (canvas_width - resized_width) / 2.0
+        pad_y = (canvas_height - resized_height) / 2.0
+    return (
+        clamp((x1 * resized_width + pad_x) / canvas_width),
+        clamp((y1 * resized_height + pad_y) / canvas_height),
+        clamp((x2 * resized_width + pad_x) / canvas_width),
+        clamp((y2 * resized_height + pad_y) / canvas_height),
+    )
+
+
+def page_norm_to_output_frame(
+    bbox: tuple[float, float, float, float],
+    width: int,
+    height: int,
+    args: argparse.Namespace,
+) -> tuple[float, float, float, float]:
+    """Encode a page-normalized box in the direct prediction coordinate frame."""
+    frame = getattr(args, "output_coordinate_frame", None)
+    if frame in {"content_normalized", "original_normalized", "navit_content"}:
+        x1, y1, x2, y2 = bbox
+        left, right = sorted((clamp(x1), clamp(x2)))
+        top, bottom = sorted((clamp(y1), clamp(y2)))
+        return left, top, right, bottom
+    if frame not in {None, "padded_grid", "fixed_canvas", "navit_grid"}:
+        raise ValueError(f"Unsupported hybrid output coordinate frame: {frame!r}")
+    return image_norm_to_model_canvas_norm(
+        bbox,
+        width,
+        height,
+        canvas_width=args.canvas_width,
+        canvas_height=args.canvas_height,
+        pad_position=args.prediction_pad_position,
+    )
+
+
+def crop_box_to_page_canvas(
+    crop_row: dict[str, Any],
+    crop_local_model_bbox: tuple[float, float, float, float],
+    args: argparse.Namespace,
+) -> tuple[float, float, float, float]:
+    left, top, right, bottom = [int(v) for v in crop_row["crop_bbox_pixels"]]
+    crop_width = max(1, right - left)
+    crop_height = max(1, bottom - top)
+    page_width, page_height = [int(v) for v in crop_row["page_size"]]
+    crop_frame = getattr(args, "crop_coordinate_frame", "padded_grid")
+    if crop_frame in {"content_normalized", "original_normalized", "navit_content"}:
+        x1, y1, x2, y2 = crop_local_model_bbox
+        max_coord = max(abs(x1), abs(y1), abs(x2), abs(y2))
+        if max_coord > 1.5 and max_coord <= 1000.0:
+            x1, y1, x2, y2 = (value / 1000.0 for value in (x1, y1, x2, y2))
+        left_norm, right_norm = sorted((clamp(x1), clamp(x2)))
+        top_norm, bottom_norm = sorted((clamp(y1), clamp(y2)))
+        crop_norm = (left_norm, top_norm, right_norm, bottom_norm)
+    else:
+        crop_norm = model_canvas_norm_to_image_norm(
+            crop_local_model_bbox,
+            crop_width,
+            crop_height,
+            canvas_width=args.canvas_width,
+            canvas_height=args.canvas_height,
+            pad_position=args.crop_pad_position,
+        )
+    page_norm = (
+        (left + crop_norm[0] * crop_width) / page_width,
+        (top + crop_norm[1] * crop_height) / page_height,
+        (left + crop_norm[2] * crop_width) / page_width,
+        (top + crop_norm[3] * crop_height) / page_height,
+    )
+    return page_norm_to_output_frame(
+        page_norm,
+        page_width,
+        page_height,
+        args,
+    )
+
+
+def page_bbox_to_page_canvas(
+    crop_row: dict[str, Any], args: argparse.Namespace
+) -> tuple[float, float, float, float]:
+    page_width, page_height = [int(v) for v in crop_row["page_size"]]
+    bbox = tuple(float(v) for v in crop_row["detector_bbox_norm"])
+    return page_norm_to_output_frame(
+        bbox,
+        page_width,
+        page_height,
+        args,
+    )
+
+
+def rebuild_crop_row_blocks(
+    crop_row: dict[str, Any], args: argparse.Namespace
+) -> list[Block]:
+    detector_label = normalize_detector_label(crop_row.get("detector_label") or "")
+    detector_score = float(crop_row.get("detector_score") or 0.0)
+    raw = str(crop_row.get("raw_out") or "")
+    blocks = []
+    for match in BLOCK_RE.finditer(raw):
+        x1, y1, body, x2, y2, cls = match.groups()
+        text = strip_model_tokens(body)
+        if not text:
+            continue
+        bbox = crop_box_to_page_canvas(
+            crop_row, (float(x1), float(y1), float(x2), float(y2)), args
+        )
+        blocks.append(
+            block_with_raw(
+                bbox,
+                cls,
+                text,
+                detector_label=detector_label,
+                origin="crop_element",
+                detector_score=detector_score,
+            )
+        )
+    if blocks:
+        return blocks
+    text = crop_markdown(raw)
+    if not text:
+        return []
+    bbox = page_bbox_to_page_canvas(crop_row, args)
+    cls = class_for_detector(
+        detector_label, tuple(float(v) for v in crop_row["detector_bbox_norm"])
+    )
+    return [
+        block_with_raw(
+            bbox,
+            cls,
+            text,
+            detector_label=detector_label,
+            origin="crop_element_fallback",
+            detector_score=detector_score,
+        )
+    ]
+
+
+def parse_class_set(value: str) -> set[str]:
+    return {item.strip() for item in value.split(",") if item.strip()}
+
+
+def crop_element_blocks_by_page(args: argparse.Namespace) -> dict[str, list[Block]]:
+    if not args.crop_element_predictions:
+        return {}
+    structural_classes = parse_class_set(args.structural_classes)
+    include_detector_labels = parse_label_filter(args.include_detector_labels)
+    by_page: dict[str, list[Block]] = {}
+    for path in args.crop_element_predictions:
+        for row in load_jsonl(path):
+            if row.get("error"):
+                continue
+            detector_label = normalize_detector_label(row.get("detector_label") or "")
+            if (
+                include_detector_labels
+                and detector_label not in include_detector_labels
+            ):
+                continue
+            raw_text = prediction_text(row)
+            if (
+                args.max_crop_output_chars > 0
+                and len(raw_text) > args.max_crop_output_chars
+            ):
+                continue
+            if (
+                args.max_crop_output_blocks > 0
+                and len(BLOCK_RE.findall(raw_text)) > args.max_crop_output_blocks
+            ):
+                continue
+            page_key = str(row.get("page_key") or "")
+            if not page_key:
+                continue
+            for block in rebuild_crop_row_blocks(row, args):
+                if block.cls not in structural_classes:
+                    continue
+                if (
+                    args.min_structural_area > 0
+                    and block_area(block) < args.min_structural_area
+                ):
+                    continue
+                by_page.setdefault(page_key, []).append(block)
+    for blocks in by_page.values():
+        blocks.sort(key=reading_order_key)
+    return by_page
+
+
+def find_row(
+    index: dict[str, dict[str, Any]], label: dict[str, Any]
+) -> dict[str, Any] | None:
+    for alias in row_aliases(label):
+        if alias in index:
+            return index[alias]
+    key = str(label.get("key") or "")
+    image_name = str(label.get("image_name") or "")
+    return index.get(key) or index.get(Path(image_name).stem) or index.get(image_name)
+
+
+def clone_prediction(
+    label: dict[str, Any], source_row: dict[str, Any] | None, *, source_name: str
+) -> dict[str, Any]:
+    raw_out = prediction_text(source_row)
+    row = {
+        "key": label["key"],
+        "image_name": label.get("image_name") or f"{label['key']}.jpg",
+        "image_path": (source_row or {}).get("image_path") or label.get("image_path"),
+        "category": label.get("category"),
+        "raw_out": raw_out,
+        "hybrid_source": source_name,
+        "error": (
+            (source_row or {}).get("error") if source_row else "missing_prediction"
+        ),
+    }
+    for key in (
+        "coordinate_frame",
+        "image_preprocessing",
+        "model_padding",
+        "model_grid_size",
+        "resized_content_size",
+    ):
+        if (source_row or {}).get(key) is not None:
+            row[key] = source_row[key]
+    return row
+
+
+def structural_blocks(
+    args: argparse.Namespace, row: dict[str, Any] | None
+) -> list[Block]:
+    structural_classes = parse_class_set(args.structural_classes)
+    blocks = [
+        block
+        for block in parse_blocks(prediction_text(row))
+        if block.cls in structural_classes
+    ]
+    if args.min_structural_area > 0:
+        blocks = [
+            block for block in blocks if block_area(block) >= args.min_structural_area
+        ]
+    return blocks
+
+
+def candidate_structural_blocks(
+    args: argparse.Namespace,
+    label: dict[str, Any],
+    structural_row: dict[str, Any] | None,
+    element_blocks: dict[str, list[Block]],
+) -> list[Block]:
+    if element_blocks:
+        return list(element_blocks.get(str(label.get("key") or ""), []))
+    return structural_blocks(args, structural_row)
+
+
+def detector_label_counts(blocks: list[Block]) -> dict[str, int]:
+    counts: dict[str, int] = {}
+    for block in blocks:
+        label = block.detector_label or "unknown"
+        counts[label] = counts.get(label, 0) + 1
+    return counts
+
+
+def is_duplicate(
+    candidate: Block, existing: list[Block], args: argparse.Namespace
+) -> bool:
+    for block in existing:
+        if block.cls != candidate.cls:
+            continue
+        if iou(block.bbox, candidate.bbox) < args.dedupe_overlap_threshold:
+            continue
+        if text_similarity(block.body, candidate.body) >= args.dedupe_text_threshold:
+            return True
+    return False
+
+
+def reading_order_key(block: Block) -> tuple[float, float, float, float]:
+    return block.y1, block.x1, block.y2, block.x2
+
+
+def insert_additions_preserving_direct_order(
+    direct_ordered: list[Block], additions: list[Block]
+) -> list[Block]:
+    """Keep the direct model's block sequence stable, inserting new blocks by bbox."""
+    if not additions:
+        return direct_ordered
+    additions_sorted = sorted(additions, key=reading_order_key)
+    merged = []
+    add_idx = 0
+    for block in direct_ordered:
+        block_key = reading_order_key(block)
+        while (
+            add_idx < len(additions_sorted)
+            and reading_order_key(additions_sorted[add_idx]) < block_key
+        ):
+            merged.append(additions_sorted[add_idx])
+            add_idx += 1
+        merged.append(block)
+    merged.extend(additions_sorted[add_idx:])
+    return merged
+
+
+def build_direct_plus_ordered_structural_addonly(
+    args: argparse.Namespace,
+    labels: list[dict[str, Any]],
+    indexes: dict[str, dict[str, dict[str, Any]]],
+    element_blocks: dict[str, list[Block]],
+) -> list[dict[str, Any]]:
+    structural_index = indexes.get(args.structural_source)
+    if structural_index is None and not element_blocks:
+        raise SystemExit(
+            f"Missing --{args.structural_source}-predictions for structural merge"
+        )
+    rows = []
+    for label in labels:
+        direct = find_row(indexes["direct"], label)
+        structural = (
+            find_row(structural_index, label) if structural_index is not None else None
+        )
+        direct_blocks = parse_blocks(prediction_text(direct))
+        added = []
+        skipped_duplicates = 0
+        for block in candidate_structural_blocks(
+            args, label, structural, element_blocks
+        ):
+            if is_duplicate(block, direct_blocks + added, args):
+                skipped_duplicates += 1
+                continue
+            added.append(block)
+        merged_blocks = insert_additions_preserving_direct_order(direct_blocks, added)
+        row = clone_prediction(
+            label,
+            direct,
+            source_name=f"direct-plus-{args.structural_source}-ordered-structural-addonly",
+        )
+        row["raw_out"] = (
+            rebuild_blocks(merged_blocks)
+            or prediction_text(direct)
+            or prediction_text(structural)
+        )
+        row["hybrid_kept_direct_blocks"] = len(direct_blocks)
+        row["hybrid_added_structural_blocks"] = len(added)
+        row["hybrid_skipped_duplicate_blocks"] = skipped_duplicates
+        row["hybrid_added_detector_labels"] = detector_label_counts(added)
+        rows.append(row)
+    return rows

nemotron_parse/engine/policies.py

@@ -0,0 +1,338 @@
+# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: OpenMDW-1.1
+
+"""Fixed single-checkpoint output conversion; no ensemble routing or benchmark inputs."""
+
+import re
+
+from functools import lru_cache
+
+BLOCK = re.compile(
+    r"(<x_[^>]+><y_[^>]+>)(.*?)(<x_[^>]+><y_[^>]+><class_([^>]+)>)", re.S
+)
+
+MDTABLE = re.compile(
+    r"(?m)^\|[^\n]*\|[ \t]*\n\|[ \t:|\-]+\|[ \t]*\n(?:\|[^\n]*\|[ \t]*(?:\n|$))+"
+)
+
+
+def promote_bold_header_row(markdown):
+    """Promote a mostly bold first row to header cells, preserving its styling.
+
+    Existing header cells take precedence. This rule uses predicted markup and
+    table shape; it contains no language vocabulary and never adds bold text.
+    """
+    from bs4 import BeautifulSoup
+
+    def transform(match):
+        original = match[0]
+        soup = BeautifulSoup(original, "html.parser")
+        table = soup.find("table")
+        if table.find("th"):
+            return original
+        rows = [
+            row for row in table.find_all("tr") if row.find_parent("table") == table
+        ]
+        if len(rows) < 2:
+            return original
+        cells = rows[0].find_all(["td", "th"], recursive=False)
+        next_cells = rows[1].find_all(["td", "th"], recursive=False)
+        texts = [cell.get_text(" ", strip=True) for cell in cells]
+        numeric = lambda text: bool(re.fullmatch(r"[\s\d.,%$€£()+−-]+", text))
+        eligible = (
+            2 <= len(cells) <= 20
+            and len(next_cells) >= 2
+            and all(len(text) <= 100 for text in texts)
+            and sum(bool(text) and not numeric(text) for text in texts)
+            >= len(cells) * 0.6
+        )
+        mostly_bold = (
+            sum(bool(cell.find(["b", "strong"])) for cell in cells) >= len(cells) * 0.6
+        )
+        if not eligible or not mostly_bold:
+            return original
+        before = [
+            (cell.get_text(), cell.get("rowspan"), cell.get("colspan"))
+            for cell in table.find_all(["td", "th"])
+        ]
+        for cell in cells:
+            cell.name = "th"
+        after = [
+            (cell.get_text(), cell.get("rowspan"), cell.get("colspan"))
+            for cell in table.find_all(["td", "th"])
+        ]
+        assert before == after, "Header conversion changed cell content or spans"
+        return str(table)
+
+    return re.sub(r"<table\b[^>]*>.*?</table>", transform, markdown, flags=re.S | re.I)
+
+
+def chart_render(raw, panels=False, compound=False):
+    """Render explicit panels and existing numeric value/percent pairs."""
+    from html import escape
+
+    number = r"[+−-]?\d[\d,. ]*(?:[kKmMbB])?"
+    pair = re.compile(r"^\s*(" + number + r")\s*\(\s*(" + number + r"\s*%)\s*\)\s*$")
+
+    def chart(m):
+        body = m[2]
+        title = re.search(r"(?m)^title:[ \t]*([^\n]*)$", body)
+        title = title[1].strip() if title else ""
+        panel_names = {
+            p[1].strip(): p[2].strip()
+            for p in re.finditer(
+                r"(?m)^panel:\s*([^;\n]+)\s*;[^;\n]+;\s*([^\n]+)", body
+            )
+        }
+        matches = list(MDTABLE.finditer(body))
+        changes = {}
+        for table in matches:
+            if r"\|" in table[0] or "`" in table[0]:
+                continue
+            rows = [
+                [c.strip() for c in line.strip().strip("|").split("|")]
+                for line in table[0].strip().splitlines()
+            ]
+            if (
+                len(rows) < 3
+                or any(len(row) != len(rows[0]) for row in rows)
+                or any(not re.fullmatch(r":?-+:?", c.replace(" ", "")) for c in rows[1])
+            ):
+                continue
+            marker = re.search(r"table\[([^\]\n]+)\]:\s*$", body[: table.start()])
+            panel = panel_names.get(marker[1].strip(), "") if marker else ""
+            split = {}
+            if compound:
+                for c in range(1, len(rows[0])):
+                    nonempty = [row[c] for row in rows[2:] if row[c]]
+                    if len(nonempty) >= 2 and all(pair.fullmatch(v) for v in nonempty):
+                        split[c] = True
+            if not split and not (panels and panel):
+                continue
+            caption = " — ".join(v for v in [title, panel if panels else ""] if v)
+            out = ["<table>"]
+            if caption:
+                out.append("<caption>" + escape(caption) + "</caption>")
+            out.append("<thead><tr>")
+            for c, label in enumerate(rows[0]):
+                span = (
+                    ' colspan="2"' if c in split else (' rowspan="2"' if split else "")
+                )
+                out.append('<th scope="col"' + span + ">" + escape(label) + "</th>")
+            out.append("</tr>")
+            if split:
+                out.append(
+                    "<tr>"
+                    + "".join(
+                        '<th scope="col">Value</th><th scope="col">Percent</th>'
+                        for c in split
+                    )
+                    + "</tr>"
+                )
+            out.append("</thead><tbody>")
+            for row in rows[2:]:
+                out.append("<tr>")
+                for c, text in enumerate(row):
+                    values = (
+                        (pair.fullmatch(text).groups() if text else ("", ""))
+                        if c in split
+                        else (text,)
+                    )
+                    out.extend("<td>" + escape(v.strip()) + "</td>" for v in values)
+                out.append("</tr>")
+            out.append("</tbody></table>\n")
+            changes[table.start()] = "".join(out)
+        if not changes:
+            return m[0]
+        new = MDTABLE.sub(lambda t: changes.get(t.start(), t[0]), body)
+        if len(changes) == len(matches) and title:
+            new = re.sub(r"(?m)^title:[ \t]*[^\n]*\n?", "", new, count=1)
+        return m[1] + new + m[3]
+
+    return re.sub(r"(<chart\b[^>]*>)(.*?)(</chart>)", chart, raw, flags=re.S)
+
+
+def native_grid(body, repair=False):
+    from .convert.rich_table_reader import split_top, argument
+
+    pattern = re.compile(
+        r"\\begin\{tabular\}\{([clrx]+)\}\n(.*?)\n\\end\{tabular\}", re.S
+    )
+    matches = list(pattern.finditer(body))
+    if len(matches) != 1:
+        return body, None
+    m = matches[0]
+    new = m[0]
+    changed = False
+
+    def cell(part):
+        text = part.strip()
+        rs = cs = 1
+        while text.startswith((r"\multicolumn", r"\multirow")):
+            command = "multicolumn" if text.startswith(r"\multicolumn") else "multirow"
+            pos = len(command) + 1
+            count, pos = argument(text, pos)
+            _, pos = argument(text, pos)
+            inner, pos = argument(text, pos)
+            if text[pos:].strip():
+                raise ValueError("trailing_span_content")
+            if command == "multicolumn":
+                cs = int(count)
+            else:
+                rs = int(count)
+            text = inner
+        header = False
+        if text.startswith(r"\thead{"):
+            inner, end = argument(text, len(r"\thead"))
+            header = True
+            if text[end:].strip():
+                raise ValueError("trailing_header_content")
+            text = inner
+        return rs, cs, text, header
+
+    try:
+        rows = split_top(m[2], r"\\")
+        if rows[-1].strip():
+            return body, {"valid": False, "reason": "unterminated_row"}
+        rows.pop()
+        parsed = [[cell(c) for c in split_top(row, "&")] for row in rows]
+        widths = [sum(c[1] for c in row) for row in parsed]
+        columns = len(m[1])
+        if (
+            repair
+            and widths
+            and len(set(widths)) == 1
+            and 1 <= widths[0] <= 512
+            and widths[0] != columns
+        ):
+            columns = widths[0]
+            cols = (m[1] + "c" * columns)[:columns]
+            new = r"\begin{tabular}{" + cols + "}\n" + m[2] + "\n" + r"\end{tabular}"
+            changed = True
+        occupied = set()
+        headers = 0
+        if not rows or len(rows) > 4096 or columns > 512:
+            raise ValueError("grid_size")
+        for ri, row in enumerate(parsed):
+            ci = 0
+            for rs, cs, content, header in row:
+                if rs < 1 or cs < 1 or ri + rs > len(rows) or ci + cs > columns:
+                    raise ValueError("invalid_span")
+                if (ri, ci) in occupied:
+                    empty = re.sub(r"\s+", "", content) in ("", r"\mbox{}")
+                    if (
+                        not empty
+                        or header
+                        or rs != 1
+                        or any((ri, cc) not in occupied for cc in range(ci, ci + cs))
+                    ):
+                        raise ValueError("invalid_placeholder")
+                else:
+                    if any(
+                        (rr, cc) in occupied
+                        for rr in range(ri, ri + rs)
+                        for cc in range(ci, ci + cs)
+                    ):
+                        raise ValueError("overlap")
+                    headers += int(header)
+                    for rr in range(ri, ri + rs):
+                        for cc in range(ci, ci + cs):
+                            occupied.add((rr, cc))
+                ci += cs
+            if ci != columns:
+                raise ValueError("row_width")
+        header_shape = []
+        for row in parsed:
+            if not any(c[3] for c in row):
+                break
+            header_shape.append(tuple((c[0], c[1], c[3]) for c in row))
+        status = {
+            "valid": True,
+            "columns": columns,
+            "rows": len(rows),
+            "header_cells": headers,
+            "header_shape": tuple(header_shape),
+            "changed_column_declaration": changed,
+        }
+    except (ValueError, TypeError, IndexError, KeyError):
+        return body, {"valid": False, "reason": "inconsistent_native_grid"}
+    return body[: m.start()] + new + body[m.end() :], status
+
+
+native_grid = lru_cache(maxsize=32768)(native_grid)
+
+
+def replace_guarded(
+    raw,
+    candidates,
+    merger,
+    similarity=0.65,
+    guard="valid",
+    repair=False,
+    min_rows_ratio=0.0,
+    iou_threshold=0.5,
+    protect_headers=False,
+    emit_repair=True,
+):
+    blocks = merger.parse_blocks(raw)
+    pairs = []
+    for i, b in enumerate(blocks):
+        if b.cls != "Table":
+            continue
+        _, oldstatus = native_grid(b.body, repair)
+        for j, c in enumerate(candidates):
+            if c.cls != "Table" or not c.body.strip():
+                continue
+            overlap = merger.iou(b.bbox, c.bbox)
+            if overlap < iou_threshold:
+                continue
+            body, status = native_grid(c.body, repair)
+            if status and not status["valid"]:
+                continue
+            if c.body.count(r"\begin{tabular}") != c.body.count(r"\end{tabular}"):
+                continue
+            if (
+                guard == "columns"
+                and status
+                and oldstatus
+                and oldstatus["valid"]
+                and status["columns"] != oldstatus["columns"]
+            ):
+                continue
+            if (
+                min_rows_ratio
+                and status
+                and oldstatus
+                and oldstatus["valid"]
+                and status["rows"] < oldstatus["rows"] * min_rows_ratio
+            ):
+                continue
+            if (
+                protect_headers
+                and status
+                and oldstatus
+                and oldstatus["valid"]
+                and oldstatus["header_shape"]
+                and status["header_shape"] != oldstatus["header_shape"]
+            ):
+                continue
+            if not emit_repair:
+                body = c.body
+            sim = merger.text_similarity(b.body, body)
+            if sim < similarity:
+                continue
+            pairs.append((overlap, sim, i, j, body))
+    chosen = {}
+    used = set()
+    for overlap, sim, i, j, body in sorted(pairs, reverse=True):
+        if i in chosen or j in used:
+            continue
+        chosen[i] = body
+        used.add(j)
+    index = iter(range(len(blocks)))
+
+    def f(m):
+        i = next(index)
+        return m[1] + chosen[i] + m[3] if i in chosen else m[0]
+
+    return BLOCK.sub(f, raw)