{
 "version": "v1.2-clean",
 "built_from": "/run/media/kushal/Kush/sansar/data/ocr_labels/v1",
 "rules": "(every train / val row of v1.1 is decided by them; test = the 54 bench_v1 pages, unchanged):\n  1. tiers silver_vision, llm_claude, llm_codex and sources textbooks_vision, air_vision, llm_claude, llm_codex: dropped.\n  2. any remaining row that carries an ocr_engine, a cer_vs_vision value or an id prefix of those tiers\n     (tbv_, airv_, llmc_, llmx_): dropped (none expected; a guard).\n  3. synthetic pages that typeset a paragraph whose corpus record lists `vagartha` (Sarvam's Vagartha dataset) among\n     its source_corpora: dropped. The paragraph ids come from --vagartha_ids (id<TAB>source_corpora, made by a\n     grep of train_slice_plus_all_v2 for the synthetic text cache's ids); a page is dropped when it shares one\n     16-letter window (hnorm, stride 1 on the page) with any of those paragraphs.\n  4. kept tiers must be gold_twin, gold_digital (textbooks: text layer, AIR: OCR-free decode) and synthetic (curated\n     e-text typeset): asserted.",
 "splits": {
  "train": {
   "pages": 19377,
   "books": 1057,
   "chars": 27080866,
   "by_tier": {
    "gold_twin": 11074,
    "gold_digital": 7206,
    "synthetic": 1097
   },
   "by_source": {
    "twin": 11074,
    "textbooks": 3367,
    "air": 3839,
    "synthetic": 1097
   }
  },
  "val": {
   "pages": 711,
   "books": 31,
   "chars": 1045871,
   "by_tier": {
    "gold_twin": 209,
    "gold_digital": 478,
    "synthetic": 24
   },
   "by_source": {
    "twin": 209,
    "textbooks": 36,
    "air": 442,
    "synthetic": 24
   }
  },
  "test": {
   "pages": 54,
   "books": 53,
   "chars": 86038,
   "by_tier": {
    "gold_bench": 54
   },
   "by_source": {
    "bench": 54
   }
  }
 },
 "dropped": {
  "train: synthetic page typesets a Vagartha (Sarvam) paragraph": 847,
  "train: tier llm_claude (llm_claude)": 420,
  "train: tier llm_codex (llm_codex)": 219,
  "train: tier silver_vision (air_vision)": 908,
  "train: tier silver_vision (textbooks_vision)": 2881,
  "val: synthetic page typesets a Vagartha (Sarvam) paragraph": 26,
  "val: tier silver_vision (air_vision)": 125
 }
}