{
  "version": 4,
  "toy": false,
  "rung": null,
  "prepared_at": "2026-10-07T01:27:18+00:00",
  "dataset_dir": "dataset-v4-full",
  "manifest_sha256": "6792bfb740b3c7217663afdba60ae75f5ae6d79aa53ac1475971fabe51a981ac",
  "manifest": {
    "pages": 181924,
    "tokens": 330366386,
    "created_at": "2026-10-07T00:58:27+00:00"
  },
  "manifest_shards": [
    {
      "file": "crawl-shard-00000.jsonl",
      "pages": 20000,
      "tokens": 47840786,
      "bytes": 175257859,
      "sha256": "012bf5d772f4731eaeddcd909029d07bbe90afb2088d425cbf903b92280095e3",
      "source": "crawl"
    },
    {
      "file": "crawl-shard-00001.jsonl",
      "pages": 20000,
      "tokens": 44063575,
      "bytes": 162061573,
      "sha256": "ead9a77cebab1660e399cb47db2adee31055847dba512dbb3ef74d03813b2b00",
      "source": "crawl"
    },
    {
      "file": "crawl-shard-00002.jsonl",
      "pages": 20000,
      "tokens": 53181778,
      "bytes": 196369474,
      "sha256": "5ad1e712e52a266041ad1e37cf1b27fdb9c21260a228c4d072ce6b0e38bf460e",
      "source": "crawl"
    },
    {
      "file": "crawl-shard-00003.jsonl",
      "pages": 20000,
      "tokens": 51313378,
      "bytes": 187276813,
      "sha256": "cd1b35c6649443e5e2c31ac7091045dc2a550411d12676cfda7c00918ebd9cbf",
      "source": "crawl"
    },
    {
      "file": "crawl-shard-00004.jsonl",
      "pages": 20000,
      "tokens": 45012511,
      "bytes": 167024162,
      "sha256": "1586d5a71304ef352103f7b6e61f0189f3e63a29d887a53b0c54b70a2f848b59",
      "source": "crawl"
    },
    {
      "file": "crawl-shard-00005.jsonl",
      "pages": 20000,
      "tokens": 42741504,
      "bytes": 157047166,
      "sha256": "4c750107a6ba28f0086af7d3b75f858b1aa76a1db8980cffa126db796c719da2",
      "source": "crawl"
    },
    {
      "file": "crawl-shard-00006.jsonl",
      "pages": 16260,
      "tokens": 35305976,
      "bytes": 129501055,
      "sha256": "b1ad8bae8ceb4f69a291e23a82e358fef26ac1cae7ab4acf6f189ecf556e0b9f",
      "source": "crawl"
    },
    {
      "file": "records-shard-00000.jsonl",
      "pages": 5000,
      "tokens": 1158392,
      "bytes": 4949845,
      "sha256": "c23a21d5a97448f678524a879c9e8cc4a8df3caf49a8fd52824759bb95a88d20",
      "source": "records"
    },
    {
      "file": "records-shard-00001.jsonl",
      "pages": 5000,
      "tokens": 1096599,
      "bytes": 4915698,
      "sha256": "ec75f7ae6a88e48fcb7d1d3373fda9ce74c3a816174e87844e5760f9b95baf6d",
      "source": "records"
    },
    {
      "file": "records-shard-00002.jsonl",
      "pages": 5000,
      "tokens": 918678,
      "bytes": 4308866,
      "sha256": "046f7198ec3f62d71aef5766ac7a11a6ca4480c045bf1e1d0b843fb32b878741",
      "source": "records"
    },
    {
      "file": "records-shard-00003.jsonl",
      "pages": 5000,
      "tokens": 817118,
      "bytes": 3988989,
      "sha256": "a20430453ed74f27a9f121446bf5a62fa31d8377da07e499d1c8d9d1dd779e85",
      "source": "records"
    },
    {
      "file": "records-shard-00004.jsonl",
      "pages": 5000,
      "tokens": 857884,
      "bytes": 4127675,
      "sha256": "844e7fed170a4e884aeda1f66cae789c25567b0ff3bd6cc4b2ad48f8abad62d8",
      "source": "records"
    },
    {
      "file": "records-shard-00005.jsonl",
      "pages": 5000,
      "tokens": 883422,
      "bytes": 4218635,
      "sha256": "42c798f331a149394804573b62e74b14528fc8f8f5bc22ee1ac0a7f3e6918a2b",
      "source": "records"
    },
    {
      "file": "records-shard-00006.jsonl",
      "pages": 5000,
      "tokens": 845402,
      "bytes": 4086717,
      "sha256": "f509992a39f656bdb263ce64f30fe360f39665fda2b8cef52f233e055d4289b0",
      "source": "records"
    },
    {
      "file": "records-shard-00007.jsonl",
      "pages": 5000,
      "tokens": 1921725,
      "bytes": 6913850,
      "sha256": "7d2215063b7c8ea010bae86449b1a1153e22b3c32cb54d0dac711c1f306accf8",
      "source": "records"
    },
    {
      "file": "records-shard-00008.jsonl",
      "pages": 5000,
      "tokens": 789413,
      "bytes": 3765224,
      "sha256": "7e8ea4008c2ef5d5ebf07cc53b1f45c9e0082416ef2227bb3452dcf62a574899",
      "source": "records"
    },
    {
      "file": "records-shard-00009.jsonl",
      "pages": 664,
      "tokens": 1618245,
      "bytes": 5173560,
      "sha256": "46b0e5806746a10a607909c70b4998ff4f6a7cea5b1a783e5e35d700cdb4ecf8",
      "source": "records"
    }
  ],
  "cleaning": {
    "input_pages": 179924,
    "dup_url": 0,
    "dropped_url_class": {
      "casino": 289,
      "convert": 1746,
      "flashnews": 8345,
      "forecast": 48,
      "jobs": 1360,
      "price": 794,
      "price_prediction": 821,
      "zh": 669
    },
    "boilerplate_line_kinds": 19179,
    "boilerplate_lines_removed": 2698066,
    "wiki_tails_cut": {
      "pages": 17,
      "chars": 240631
    },
    "dropped_short": 45,
    "dropped_low_alpha": 15,
    "dropped_x_rewrite": 10446,
    "x_rewrite": {
      "rule": "(@handles + 'posted on X'-style phrases) per 1,000 words > 2",
      "handles": 55752,
      "phrases": 6071,
      "top_domains": [
        [
          "forum.arbitrum.foundation",
          1356
        ],
        [
          "ethereum-magicians.org",
          882
        ],
        [
          "gov.optimism.io",
          863
        ],
        [
          "u.today",
          630
        ],
        [
          "gov.uniswap.org",
          389
        ],
        [
          "eips.ethereum.org",
          329
        ],
        [
          "research.lido.fi",
          279
        ],
        [
          "governance.aave.com",
          279
        ],
        [
          "forum.balancer.fi",
          250
        ],
        [
          "tapchibitcoin.io",
          207
        ],
        [
          "github.com",
          196
        ],
        [
          "bitcoinfoundation.org",
          195
        ],
        [
          "discuss.jup.ag",
          184
        ],
        [
          "coinlore.com",
          132
        ],
        [
          "ethresear.ch",
          117
        ],
        [
          "news.bitcoin.com",
          104
        ],
        [
          "coindesk.com",
          102
        ],
        [
          "coinfomania.com",
          97
        ],
        [
          "cointelegraph.com",
          93
        ],
        [
          "ankr.com",
          90
        ]
      ],
      "x_links_removed": {
        "pages": 2208,
        "links": 6130
      },
      "records_x_lines_removed": {
        "pages": 36,
        "lines": 39
      }
    },
    "dropped_lang": 14915,
    "lang": {
      "model": "fastText lid.176",
      "threshold": 0.65,
      "dropped_by_top_language": {
        "en": 6032,
        "vi": 1879,
        "pt": 731,
        "de": 641,
        "it": 532,
        "ar": 443,
        "id": 428,
        "nl": 423,
        "es": 391,
        "tr": 376,
        "pl": 371,
        "fr": 349,
        "cs": 288,
        "ko": 287,
        "ru": 250,
        "hu": 169,
        "uk": 159,
        "ja": 136,
        "th": 68,
        "ro": 62
      }
    },
    "dup_text": 3,
    "kept_pages": 140428,
    "dropped_heldout": 2000,
    "bad_rows": 0,
    "dropped_val_overlap": 0
  },
  "pages": {
    "train": 140428,
    "val": 2000
  },
  "chunks": {
    "train": 190867,
    "val": 4964,
    "max_chars": null,
    "max_tokens": 2040,
    "train_chunks_longer_than_context": 0
  },
  "chars": {
    "train": 790442138,
    "val": 15792677
  },
  "tokens": {
    "train": 183061543,
    "val": 3726648,
    "chars_per_token": 4.318
  },
  "tokenizer": {
    "trained_on": "English train pages of this dataset only (spider pages, fastText lid.176 >= 0.65)",
    "max_chars": 2000000000,
    "doc_cap": 10000,
    "vocab_size": 16384,
    "trainer": "nanochat rustbpe (GPT-4 style split pattern)",
    "sha256": "5782900e33bdfc9a614fd96502b9df631d90d17854ec79bff0aab9f354a64478",
    "glossary": {
      "terms": 680,
      "tokens": 1031,
      "single_token": 408,
      "mean_tokens": 1.516,
      "ref_tokens": 1106,
      "worse": 1,
      "better": 67,
      "same": 612,
      "worst": [
        {
          "term": "Substrate",
          "tokens": 3,
          "ref": 2
        }
      ],
      "passed": true,
      "ref": "v3"
    }
  },
  "sft_headings": {
    "train": {
      "conversations": 521532,
      "by_kind": {
        "heading": 394646,
        "page": 75707,
        "faq": 51179
      },
      "selection": {
        "input": 521709,
        "dropped_low_edu": 0,
        "dropped_domain_cap": 177,
        "kept": 521532,
        "max_domain_share": 0.03,
        "min_edu": 2
      }
    },
    "val": {
      "conversations": 12012,
      "by_kind": {
        "heading": 9451,
        "page": 1431,
        "faq": 1130
      },
      "selection": {
        "input": 12042,
        "dropped_low_edu": 0,
        "dropped_domain_cap": 30,
        "kept": 12012,
        "max_domain_share": 0.03,
        "min_edu": 2
      }
    }
  },
  "chapters": {
    "none": 60570,
    "whitepapers": 564,
    "blogs": 2028,
    "docs": 14034,
    "solana": 5031,
    "forums": 12574,
    "evm-dex": 127,
    "defillama-chains": 120,
    "investors": 8339,
    "projects": 29154,
    "defillama-protocols": 6884,
    "raises": 163,
    "solana-mints-burns": 120,
    "solana-dex": 364,
    "defillama-stablecoins": 119,
    "token-facts": 230,
    "unlocks": 7
  },
  "domains": 2745,
  "contributors": [
    {
      "spider_id": "queen-records",
      "name": "queen-records",
      "kind": "queen",
      "owner": null,
      "burn_tx": null,
      "pages": 45627,
      "tokens": 10830251
    },
    {
      "spider_id": "spider-05",
      "name": "spider-05",
      "kind": "queen",
      "owner": null,
      "burn_tx": null,
      "pages": 841,
      "tokens": 1692853
    },
    {
      "spider_id": "spider-04",
      "name": "spider-04",
      "kind": "queen",
      "owner": null,
      "burn_tx": null,
      "pages": 926,
      "tokens": 1668484
    },
    {
      "spider_id": "spider-03",
      "name": "spider-03",
      "kind": "queen",
      "owner": null,
      "burn_tx": null,
      "pages": 852,
      "tokens": 1662728
    },
    {
      "spider_id": "spider-02",
      "name": "spider-02",
      "kind": "queen",
      "owner": null,
      "burn_tx": null,
      "pages": 846,
      "tokens": 1615554
    },
    {
      "spider_id": "spider-01",
      "name": "spider-01",
      "kind": "queen",
      "owner": null,
      "burn_tx": null,
      "pages": 854,
      "tokens": 1587959
    },
    {
      "spider_id": "spider-13",
      "name": "spider-13",
      "kind": "queen",
      "owner": null,
      "burn_tx": null,
      "pages": 735,
      "tokens": 1536440
    },
    {
      "spider_id": "spider-07",
      "name": "spider-07",
      "kind": "queen",
      "owner": null,
      "burn_tx": null,
      "pages": 724,
      "tokens": 1517295
    },
    {
      "spider_id": "spider-06",
      "name": "spider-06",
      "kind": "queen",
      "owner": null,
      "burn_tx": null,
      "pages": 863,
      "tokens": 1482433
    },
    {
      "spider_id": "spider-24",
      "name": "spider-24",
      "kind": "queen",
      "owner": null,
      "burn_tx": null,
      "pages": 747,
      "tokens": 1474814
    },
    {
      "spider_id": "spider-32",
      "name": "spider-32",
      "kind": "queen",
      "owner": null,
      "burn_tx": null,
      "pages": 703,
      "tokens": 1458174
    },
    {
      "spider_id": "spider-33",
      "name": "spider-33",
      "kind": "queen",
      "owner": null,
      "burn_tx": null,
      "pages": 647,
      "tokens": 1442197
    },
    {
      "spider_id": "spider-08",
      "name": "spider-08",
      "kind": "queen",
      "owner": null,
      "burn_tx": null,
      "pages": 762,
      "tokens": 1440335
    },
    {
      "spider_id": "spider-69",
      "name": "spider-69",
      "kind": "queen",
      "owner": null,
      "burn_tx": null,
      "pages": 616,
      "tokens": 1426349
    },
    {
      "spider_id": "spider-58",
      "name": "spider-58",
      "kind": "queen",
      "owner": null,
      "burn_tx": null,
      "pages": 653,
      "tokens": 1397465
    },
    {
      "spider_id": "spider-57",
      "name": "spider-57",
      "kind": "queen",
      "owner": null,
      "burn_tx": null,
      "pages": 604,
      "tokens": 1394965
    },
    {
      "spider_id": "spider-41",
      "name": "spider-41",
      "kind": "queen",
      "owner": null,
      "burn_tx": null,
      "pages": 686,
      "tokens": 1378451
    },
    {
      "spider_id": "spider-12",
      "name": "spider-12",
      "kind": "queen",
      "owner": null,
      "burn_tx": null,
      "pages": 708,
      "tokens": 1373638
    },
    {
      "spider_id": "spider-46",
      "name": "spider-46",
      "kind": "queen",
      "owner": null,
      "burn_tx": null,
      "pages": 682,
      "tokens": 1359574
    },
    {
      "spider_id": "spider-23",
      "name": "spider-23",
      "kind": "queen",
      "owner": null,
      "burn_tx": null,
      "pages": 693,
      "tokens": 1338465
    },
    {
      "spider_id": "spider-26",
      "name": "spider-26",
      "kind": "queen",
      "owner": null,
      "burn_tx": null,
      "pages": 652,
      "tokens": 1335145
    },
    {
      "spider_id": "spider-31",
      "name": "spider-31",
      "kind": "queen",
      "owner": null,
      "burn_tx": null,
      "pages": 650,
      "tokens": 1322803
    },
    {
      "spider_id": "spider-11",
      "name": "spider-11",
      "kind": "queen",
      "owner": null,
      "burn_tx": null,
      "pages": 723,
      "tokens": 1313413
    },
    {
      "spider_id": "spider-52",
      "name": "spider-52",
      "kind": "queen",
      "owner": null,
      "burn_tx": null,
      "pages": 667,
      "tokens": 1304428
    },
    {
      "spider_id": "spider-81",
      "name": "spider-81",
      "kind": "queen",
      "owner": null,
      "burn_tx": null,
      "pages": 628,
      "tokens": 1300351
    },
    {
      "spider_id": "spider-17",
      "name": "spider-17",
      "kind": "queen",
      "owner": null,
      "burn_tx": null,
      "pages": 722,
      "tokens": 1295522
    },
    {
      "spider_id": "spider-28",
      "name": "spider-28",
      "kind": "queen",
      "owner": null,
      "burn_tx": null,
      "pages": 666,
      "tokens": 1274072
    },
    {
      "spider_id": "spider-20",
      "name": "spider-20",
      "kind": "queen",
      "owner": null,
      "burn_tx": null,
      "pages": 738,
      "tokens": 1263457
    },
    {
      "spider_id": "spider-19",
      "name": "spider-19",
      "kind": "queen",
      "owner": null,
      "burn_tx": null,
      "pages": 763,
      "tokens": 1262077
    },
    {
      "spider_id": "spider-48",
      "name": "spider-48",
      "kind": "queen",
      "owner": null,
      "burn_tx": null,
      "pages": 622,
      "tokens": 1253761
    },
    {
      "spider_id": "spider-29",
      "name": "spider-29",
      "kind": "queen",
      "owner": null,
      "burn_tx": null,
      "pages": 688,
      "tokens": 1244215
    },
    {
      "spider_id": "spider-25",
      "name": "spider-25",
      "kind": "queen",
      "owner": null,
      "burn_tx": null,
      "pages": 689,
      "tokens": 1241588
    },
    {
      "spider_id": "spider-65",
      "name": "spider-65",
      "kind": "queen",
      "owner": null,
      "burn_tx": null,
      "pages": 662,
      "tokens": 1236627
    },
    {
      "spider_id": "spider-45",
      "name": "spider-45",
      "kind": "queen",
      "owner": null,
      "burn_tx": null,
      "pages": 646,
      "tokens": 1227888
    },
    {
      "spider_id": "spider-38",
      "name": "spider-38",
      "kind": "queen",
      "owner": null,
      "burn_tx": null,
      "pages": 629,
      "tokens": 1223193
    },
    {
      "spider_id": "spider-14",
      "name": "spider-14",
      "kind": "queen",
      "owner": null,
      "burn_tx": null,
      "pages": 700,
      "tokens": 1218348
    },
    {
      "spider_id": "spider-40",
      "name": "spider-40",
      "kind": "queen",
      "owner": null,
      "burn_tx": null,
      "pages": 684,
      "tokens": 1205962
    },
    {
      "spider_id": "spider-21",
      "name": "spider-21",
      "kind": "queen",
      "owner": null,
      "burn_tx": null,
      "pages": 691,
      "tokens": 1205224
    },
    {
      "spider_id": "spider-66",
      "name": "spider-66",
      "kind": "queen",
      "owner": null,
      "burn_tx": null,
      "pages": 676,
      "tokens": 1199029
    },
    {
      "spider_id": "spider-22",
      "name": "spider-22",
      "kind": "queen",
      "owner": null,
      "burn_tx": null,
      "pages": 701,
      "tokens": 1196474
    },
    {
      "spider_id": "spider-15",
      "name": "spider-15",
      "kind": "queen",
      "owner": null,
      "burn_tx": null,
      "pages": 680,
      "tokens": 1187175
    },
    {
      "spider_id": "spider-10",
      "name": "spider-10",
      "kind": "queen",
      "owner": null,
      "burn_tx": null,
      "pages": 675,
      "tokens": 1181301
    },
    {
      "spider_id": "spider-70",
      "name": "spider-70",
      "kind": "queen",
      "owner": null,
      "burn_tx": null,
      "pages": 617,
      "tokens": 1169522
    },
    {
      "spider_id": "spider-49",
      "name": "spider-49",
      "kind": "queen",
      "owner": null,
      "burn_tx": null,
      "pages": 634,
      "tokens": 1159457
    },
    {
      "spider_id": "spider-39",
      "name": "spider-39",
      "kind": "queen",
      "owner": null,
      "burn_tx": null,
      "pages": 664,
      "tokens": 1137328
    },
    {
      "spider_id": "spider-34",
      "name": "spider-34",
      "kind": "queen",
      "owner": null,
      "burn_tx": null,
      "pages": 686,
      "tokens": 1136731
    },
    {
      "spider_id": "spider-42",
      "name": "spider-42",
      "kind": "queen",
      "owner": null,
      "burn_tx": null,
      "pages": 609,
      "tokens": 1136613
    },
    {
      "spider_id": "spider-73",
      "name": "spider-73",
      "kind": "queen",
      "owner": null,
      "burn_tx": null,
      "pages": 595,
      "tokens": 1132373
    },
    {
      "spider_id": "spider-64",
      "name": "spider-64",
      "kind": "queen",
      "owner": null,
      "burn_tx": null,
      "pages": 617,
      "tokens": 1117763
    },
    {
      "spider_id": "spider-82",
      "name": "spider-82",
      "kind": "queen",
      "owner": null,
      "burn_tx": null,
      "pages": 590,
      "tokens": 1117054
    },
    {
      "spider_id": "spider-09",
      "name": "spider-09",
      "kind": "queen",
      "owner": null,
      "burn_tx": null,
      "pages": 693,
      "tokens": 1113423
    },
    {
      "spider_id": "spider-75",
      "name": "spider-75",
      "kind": "queen",
      "owner": null,
      "burn_tx": null,
      "pages": 577,
      "tokens": 1104448
    },
    {
      "spider_id": "spider-101",
      "name": "spider-101",
      "kind": "queen",
      "owner": null,
      "burn_tx": null,
      "pages": 495,
      "tokens": 1090813
    },
    {
      "spider_id": "spider-67",
      "name": "spider-67",
      "kind": "queen",
      "owner": null,
      "burn_tx": null,
      "pages": 600,
      "tokens": 1085070
    },
    {
      "spider_id": "spider-36",
      "name": "spider-36",
      "kind": "queen",
      "owner": null,
      "burn_tx": null,
      "pages": 648,
      "tokens": 1082160
    },
    {
      "spider_id": "spider-27",
      "name": "spider-27",
      "kind": "queen",
      "owner": null,
      "burn_tx": null,
      "pages": 678,
      "tokens": 1075869
    },
    {
      "spider_id": "spider-18",
      "name": "spider-18",
      "kind": "queen",
      "owner": null,
      "burn_tx": null,
      "pages": 677,
      "tokens": 1066376
    },
    {
      "spider_id": "spider-71",
      "name": "spider-71",
      "kind": "queen",
      "owner": null,
      "burn_tx": null,
      "pages": 612,
      "tokens": 1059893
    },
    {
      "spider_id": "spider-30",
      "name": "spider-30",
      "kind": "queen",
      "owner": null,
      "burn_tx": null,
      "pages": 657,
      "tokens": 1056643
    },
    {
      "spider_id": "spider-44",
      "name": "spider-44",
      "kind": "queen",
      "owner": null,
      "burn_tx": null,
      "pages": 590,
      "tokens": 1048690
    },
    {
      "spider_id": "spider-43",
      "name": "spider-43",
      "kind": "queen",
      "owner": null,
      "burn_tx": null,
      "pages": 656,
      "tokens": 1048301
    },
    {
      "spider_id": "spider-35",
      "name": "spider-35",
      "kind": "queen",
      "owner": null,
      "burn_tx": null,
      "pages": 590,
      "tokens": 1047010
    },
    {
      "spider_id": "spider-60",
      "name": "spider-60",
      "kind": "queen",
      "owner": null,
      "burn_tx": null,
      "pages": 609,
      "tokens": 1038729
    },
    {
      "spider_id": "spider-85",
      "name": "spider-85",
      "kind": "queen",
      "owner": null,
      "burn_tx": null,
      "pages": 533,
      "tokens": 1037504
    },
    {
      "spider_id": "spider-86",
      "name": "spider-86",
      "kind": "queen",
      "owner": null,
      "burn_tx": null,
      "pages": 511,
      "tokens": 1031582
    },
    {
      "spider_id": "spider-72",
      "name": "spider-72",
      "kind": "queen",
      "owner": null,
      "burn_tx": null,
      "pages": 617,
      "tokens": 1022998
    },
    {
      "spider_id": "spider-123",
      "name": "spider-123",
      "kind": "queen",
      "owner": null,
      "burn_tx": null,
      "pages": 390,
      "tokens": 1020861
    },
    {
      "spider_id": "spider-62",
      "name": "spider-62",
      "kind": "queen",
      "owner": null,
      "burn_tx": null,
      "pages": 588,
      "tokens": 1008813
    },
    {
      "spider_id": "spider-109",
      "name": "spider-109",
      "kind": "queen",
      "owner": null,
      "burn_tx": null,
      "pages": 519,
      "tokens": 994913
    },
    {
      "spider_id": "spider-87",
      "name": "spider-87",
      "kind": "queen",
      "owner": null,
      "burn_tx": null,
      "pages": 511,
      "tokens": 992120
    },
    {
      "spider_id": "spider-54",
      "name": "spider-54",
      "kind": "queen",
      "owner": null,
      "burn_tx": null,
      "pages": 651,
      "tokens": 991981
    },
    {
      "spider_id": "spider-97",
      "name": "spider-97",
      "kind": "queen",
      "owner": null,
      "burn_tx": null,
      "pages": 524,
      "tokens": 988073
    },
    {
      "spider_id": "spider-68",
      "name": "spider-68",
      "kind": "queen",
      "owner": null,
      "burn_tx": null,
      "pages": 607,
      "tokens": 985389
    },
    {
      "spider_id": "spider-63",
      "name": "spider-63",
      "kind": "queen",
      "owner": null,
      "burn_tx": null,
      "pages": 592,
      "tokens": 981462
    },
    {
      "spider_id": "spider-98",
      "name": "spider-98",
      "kind": "queen",
      "owner": null,
      "burn_tx": null,
      "pages": 568,
      "tokens": 977601
    },
    {
      "spider_id": "spider-61",
      "name": "spider-61",
      "kind": "queen",
      "owner": null,
      "burn_tx": null,
      "pages": 606,
      "tokens": 977574
    },
    {
      "spider_id": "spider-16",
      "name": "spider-16",
      "kind": "queen",
      "owner": null,
      "burn_tx": null,
      "pages": 689,
      "tokens": 975878
    },
    {
      "spider_id": "spider-51",
      "name": "spider-51",
      "kind": "queen",
      "owner": null,
      "burn_tx": null,
      "pages": 649,
      "tokens": 973831
    },
    {
      "spider_id": "spider-77",
      "name": "spider-77",
      "kind": "queen",
      "owner": null,
      "burn_tx": null,
      "pages": 528,
      "tokens": 973816
    },
    {
      "spider_id": "spider-83",
      "name": "spider-83",
      "kind": "queen",
      "owner": null,
      "burn_tx": null,
      "pages": 549,
      "tokens": 966853
    },
    {
      "spider_id": "spider-90",
      "name": "spider-90",
      "kind": "queen",
      "owner": null,
      "burn_tx": null,
      "pages": 509,
      "tokens": 963175
    },
    {
      "spider_id": "spider-135",
      "name": "spider-135",
      "kind": "queen",
      "owner": null,
      "burn_tx": null,
      "pages": 368,
      "tokens": 962209
    },
    {
      "spider_id": "spider-47",
      "name": "spider-47",
      "kind": "queen",
      "owner": null,
      "burn_tx": null,
      "pages": 614,
      "tokens": 960974
    },
    {
      "spider_id": "spider-93",
      "name": "spider-93",
      "kind": "queen",
      "owner": null,
      "burn_tx": null,
      "pages": 485,
      "tokens": 960061
    },
    {
      "spider_id": "spider-79",
      "name": "spider-79",
      "kind": "queen",
      "owner": null,
      "burn_tx": null,
      "pages": 536,
      "tokens": 957583
    },
    {
      "spider_id": "spider-59",
      "name": "spider-59",
      "kind": "queen",
      "owner": null,
      "burn_tx": null,
      "pages": 529,
      "tokens": 953718
    },
    {
      "spider_id": "spider-76",
      "name": "spider-76",
      "kind": "queen",
      "owner": null,
      "burn_tx": null,
      "pages": 575,
      "tokens": 953181
    },
    {
      "spider_id": "spider-92",
      "name": "spider-92",
      "kind": "queen",
      "owner": null,
      "burn_tx": null,
      "pages": 525,
      "tokens": 952317
    },
    {
      "spider_id": "spider-50",
      "name": "spider-50",
      "kind": "queen",
      "owner": null,
      "burn_tx": null,
      "pages": 617,
      "tokens": 950383
    },
    {
      "spider_id": "spider-96",
      "name": "spider-96",
      "kind": "queen",
      "owner": null,
      "burn_tx": null,
      "pages": 545,
      "tokens": 948223
    },
    {
      "spider_id": "spider-89",
      "name": "spider-89",
      "kind": "queen",
      "owner": null,
      "burn_tx": null,
      "pages": 517,
      "tokens": 937528
    },
    {
      "spider_id": "spider-108",
      "name": "spider-108",
      "kind": "queen",
      "owner": null,
      "burn_tx": null,
      "pages": 510,
      "tokens": 933970
    },
    {
      "spider_id": "spider-99",
      "name": "spider-99",
      "kind": "queen",
      "owner": null,
      "burn_tx": null,
      "pages": 540,
      "tokens": 928101
    },
    {
      "spider_id": "spider-53",
      "name": "spider-53",
      "kind": "queen",
      "owner": null,
      "burn_tx": null,
      "pages": 552,
      "tokens": 926608
    },
    {
      "spider_id": "spider-80",
      "name": "spider-80",
      "kind": "queen",
      "owner": null,
      "burn_tx": null,
      "pages": 530,
      "tokens": 923432
    },
    {
      "spider_id": "spider-84",
      "name": "spider-84",
      "kind": "queen",
      "owner": null,
      "burn_tx": null,
      "pages": 515,
      "tokens": 922461
    },
    {
      "spider_id": "spider-95",
      "name": "spider-95",
      "kind": "queen",
      "owner": null,
      "burn_tx": null,
      "pages": 452,
      "tokens": 921857
    },
    {
      "spider_id": "spider-37",
      "name": "spider-37",
      "kind": "queen",
      "owner": null,
      "burn_tx": null,
      "pages": 606,
      "tokens": 920668
    },
    {
      "spider_id": "spider-78",
      "name": "spider-78",
      "kind": "queen",
      "owner": null,
      "burn_tx": null,
      "pages": 536,
      "tokens": 912846
    },
    {
      "spider_id": "spider-144",
      "name": "spider-144",
      "kind": "queen",
      "owner": null,
      "burn_tx": null,
      "pages": 408,
      "tokens": 905626
    },
    {
      "spider_id": "spider-100",
      "name": "spider-100",
      "kind": "queen",
      "owner": null,
      "burn_tx": null,
      "pages": 504,
      "tokens": 896326
    },
    {
      "spider_id": "spider-55",
      "name": "spider-55",
      "kind": "queen",
      "owner": null,
      "burn_tx": null,
      "pages": 581,
      "tokens": 895110
    },
    {
      "spider_id": "spider-56",
      "name": "spider-56",
      "kind": "queen",
      "owner": null,
      "burn_tx": null,
      "pages": 593,
      "tokens": 881178
    },
    {
      "spider_id": "spider-114",
      "name": "spider-114",
      "kind": "queen",
      "owner": null,
      "burn_tx": null,
      "pages": 400,
      "tokens": 872805
    },
    {
      "spider_id": "spider-118",
      "name": "spider-118",
      "kind": "queen",
      "owner": null,
      "burn_tx": null,
      "pages": 355,
      "tokens": 857226
    },
    {
      "spider_id": "spider-139",
      "name": "spider-139",
      "kind": "queen",
      "owner": null,
      "burn_tx": null,
      "pages": 372,
      "tokens": 855311
    },
    {
      "spider_id": "spider-134",
      "name": "spider-134",
      "kind": "queen",
      "owner": null,
      "burn_tx": null,
      "pages": 406,
      "tokens": 854840
    },
    {
      "spider_id": "spider-104",
      "name": "spider-104",
      "kind": "queen",
      "owner": null,
      "burn_tx": null,
      "pages": 515,
      "tokens": 836639
    },
    {
      "spider_id": "spider-149",
      "name": "spider-149",
      "kind": "queen",
      "owner": null,
      "burn_tx": null,
      "pages": 337,
      "tokens": 829398
    },
    {
      "spider_id": "spider-103",
      "name": "spider-103",
      "kind": "queen",
      "owner": null,
      "burn_tx": null,
      "pages": 492,
      "tokens": 812322
    },
    {
      "spider_id": "spider-166",
      "name": "spider-166",
      "kind": "queen",
      "owner": null,
      "burn_tx": null,
      "pages": 349,
      "tokens": 807096
    },
    {
      "spider_id": "spider-91",
      "name": "spider-91",
      "kind": "queen",
      "owner": null,
      "burn_tx": null,
      "pages": 507,
      "tokens": 806440
    },
    {
      "spider_id": "spider-107",
      "name": "spider-107",
      "kind": "queen",
      "owner": null,
      "burn_tx": null,
      "pages": 519,
      "tokens": 804924
    },
    {
      "spider_id": "spider-148",
      "name": "spider-148",
      "kind": "queen",
      "owner": null,
      "burn_tx": null,
      "pages": 355,
      "tokens": 803132
    },
    {
      "spider_id": "spider-88",
      "name": "spider-88",
      "kind": "queen",
      "owner": null,
      "burn_tx": null,
      "pages": 500,
      "tokens": 798800
    },
    {
      "spider_id": "spider-106",
      "name": "spider-106",
      "kind": "queen",
      "owner": null,
      "burn_tx": null,
      "pages": 512,
      "tokens": 796260
    },
    {
      "spider_id": "spider-94",
      "name": "spider-94",
      "kind": "queen",
      "owner": null,
      "burn_tx": null,
      "pages": 532,
      "tokens": 791025
    },
    {
      "spider_id": "spider-156",
      "name": "spider-156",
      "kind": "queen",
      "owner": null,
      "burn_tx": null,
      "pages": 397,
      "tokens": 788783
    },
    {
      "spider_id": "spider-140",
      "name": "spider-140",
      "kind": "queen",
      "owner": null,
      "burn_tx": null,
      "pages": 339,
      "tokens": 783380
    },
    {
      "spider_id": "spider-119",
      "name": "spider-119",
      "kind": "queen",
      "owner": null,
      "burn_tx": null,
      "pages": 407,
      "tokens": 781758
    },
    {
      "spider_id": "spider-161",
      "name": "spider-161",
      "kind": "queen",
      "owner": null,
      "burn_tx": null,
      "pages": 348,
      "tokens": 777115
    },
    {
      "spider_id": "spider-154",
      "name": "spider-154",
      "kind": "queen",
      "owner": null,
      "burn_tx": null,
      "pages": 371,
      "tokens": 775887
    },
    {
      "spider_id": "spider-110",
      "name": "spider-110",
      "kind": "queen",
      "owner": null,
      "burn_tx": null,
      "pages": 465,
      "tokens": 766973
    },
    {
      "spider_id": "spider-102",
      "name": "spider-102",
      "kind": "queen",
      "owner": null,
      "burn_tx": null,
      "pages": 487,
      "tokens": 764842
    },
    {
      "spider_id": "spider-143",
      "name": "spider-143",
      "kind": "queen",
      "owner": null,
      "burn_tx": null,
      "pages": 400,
      "tokens": 758536
    },
    {
      "spider_id": "spider-113",
      "name": "spider-113",
      "kind": "queen",
      "owner": null,
      "burn_tx": null,
      "pages": 373,
      "tokens": 754280
    },
    {
      "spider_id": "spider-74",
      "name": "spider-74",
      "kind": "queen",
      "owner": null,
      "burn_tx": null,
      "pages": 557,
      "tokens": 748295
    },
    {
      "spider_id": "spider-167",
      "name": "spider-167",
      "kind": "queen",
      "owner": null,
      "burn_tx": null,
      "pages": 334,
      "tokens": 748128
    },
    {
      "spider_id": "spider-145",
      "name": "spider-145",
      "kind": "queen",
      "owner": null,
      "burn_tx": null,
      "pages": 385,
      "tokens": 741840
    },
    {
      "spider_id": "spider-151",
      "name": "spider-151",
      "kind": "queen",
      "owner": null,
      "burn_tx": null,
      "pages": 359,
      "tokens": 741547
    },
    {
      "spider_id": "spider-116",
      "name": "spider-116",
      "kind": "queen",
      "owner": null,
      "burn_tx": null,
      "pages": 391,
      "tokens": 738707
    },
    {
      "spider_id": "spider-132",
      "name": "spider-132",
      "kind": "queen",
      "owner": null,
      "burn_tx": null,
      "pages": 364,
      "tokens": 736976
    },
    {
      "spider_id": "spider-105",
      "name": "spider-105",
      "kind": "queen",
      "owner": null,
      "burn_tx": null,
      "pages": 456,
      "tokens": 735612
    },
    {
      "spider_id": "spider-121",
      "name": "spider-121",
      "kind": "queen",
      "owner": null,
      "burn_tx": null,
      "pages": 369,
      "tokens": 729813
    },
    {
      "spider_id": "spider-153",
      "name": "spider-153",
      "kind": "queen",
      "owner": null,
      "burn_tx": null,
      "pages": 397,
      "tokens": 720902
    },
    {
      "spider_id": "spider-138",
      "name": "spider-138",
      "kind": "queen",
      "owner": null,
      "burn_tx": null,
      "pages": 353,
      "tokens": 717669
    },
    {
      "spider_id": "spider-122",
      "name": "spider-122",
      "kind": "queen",
      "owner": null,
      "burn_tx": null,
      "pages": 354,
      "tokens": 714317
    },
    {
      "spider_id": "spider-180",
      "name": "spider-180",
      "kind": "queen",
      "owner": null,
      "burn_tx": null,
      "pages": 306,
      "tokens": 709530
    },
    {
      "spider_id": "spider-125",
      "name": "spider-125",
      "kind": "queen",
      "owner": null,
      "burn_tx": null,
      "pages": 374,
      "tokens": 699434
    },
    {
      "spider_id": "spider-150",
      "name": "spider-150",
      "kind": "queen",
      "owner": null,
      "burn_tx": null,
      "pages": 360,
      "tokens": 698648
    },
    {
      "spider_id": "spider-146",
      "name": "spider-146",
      "kind": "queen",
      "owner": null,
      "burn_tx": null,
      "pages": 401,
      "tokens": 685836
    },
    {
      "spider_id": "spider-126",
      "name": "spider-126",
      "kind": "queen",
      "owner": null,
      "burn_tx": null,
      "pages": 351,
      "tokens": 683075
    },
    {
      "spider_id": "spider-176",
      "name": "spider-176",
      "kind": "queen",
      "owner": null,
      "burn_tx": null,
      "pages": 312,
      "tokens": 680954
    },
    {
      "spider_id": "spider-129",
      "name": "spider-129",
      "kind": "queen",
      "owner": null,
      "burn_tx": null,
      "pages": 383,
      "tokens": 675353
    },
    {
      "spider_id": "spider-158",
      "name": "spider-158",
      "kind": "queen",
      "owner": null,
      "burn_tx": null,
      "pages": 325,
      "tokens": 667011
    },
    {
      "spider_id": "spider-160",
      "name": "spider-160",
      "kind": "queen",
      "owner": null,
      "burn_tx": null,
      "pages": 319,
      "tokens": 661323
    },
    {
      "spider_id": "spider-157",
      "name": "spider-157",
      "kind": "queen",
      "owner": null,
      "burn_tx": null,
      "pages": 380,
      "tokens": 648929
    },
    {
      "spider_id": "spider-127",
      "name": "spider-127",
      "kind": "queen",
      "owner": null,
      "burn_tx": null,
      "pages": 377,
      "tokens": 646950
    },
    {
      "spider_id": "spider-162",
      "name": "spider-162",
      "kind": "queen",
      "owner": null,
      "burn_tx": null,
      "pages": 336,
      "tokens": 643737
    },
    {
      "spider_id": "spider-120",
      "name": "spider-120",
      "kind": "queen",
      "owner": null,
      "burn_tx": null,
      "pages": 350,
      "tokens": 641282
    },
    {
      "spider_id": "spider-159",
      "name": "spider-159",
      "kind": "queen",
      "owner": null,
      "burn_tx": null,
      "pages": 372,
      "tokens": 639210
    },
    {
      "spider_id": "spider-128",
      "name": "spider-128",
      "kind": "queen",
      "owner": null,
      "burn_tx": null,
      "pages": 379,
      "tokens": 629538
    },
    {
      "spider_id": "spider-163",
      "name": "spider-163",
      "kind": "queen",
      "owner": null,
      "burn_tx": null,
      "pages": 375,
      "tokens": 622387
    },
    {
      "spider_id": "spider-137",
      "name": "spider-137",
      "kind": "queen",
      "owner": null,
      "burn_tx": null,
      "pages": 382,
      "tokens": 621373
    },
    {
      "spider_id": "spider-155",
      "name": "spider-155",
      "kind": "queen",
      "owner": null,
      "burn_tx": null,
      "pages": 361,
      "tokens": 616713
    },
    {
      "spider_id": "spider-136",
      "name": "spider-136",
      "kind": "queen",
      "owner": null,
      "burn_tx": null,
      "pages": 359,
      "tokens": 581199
    },
    {
      "spider_id": "spider-173",
      "name": "spider-173",
      "kind": "queen",
      "owner": null,
      "burn_tx": null,
      "pages": 352,
      "tokens": 577298
    },
    {
      "spider_id": "spider-131",
      "name": "spider-131",
      "kind": "queen",
      "owner": null,
      "burn_tx": null,
      "pages": 366,
      "tokens": 576070
    },
    {
      "spider_id": "spider-112",
      "name": "spider-112",
      "kind": "queen",
      "owner": null,
      "burn_tx": null,
      "pages": 355,
      "tokens": 574960
    },
    {
      "spider_id": "spider-111",
      "name": "spider-111",
      "kind": "queen",
      "owner": null,
      "burn_tx": null,
      "pages": 307,
      "tokens": 574400
    },
    {
      "spider_id": "spider-165",
      "name": "spider-165",
      "kind": "queen",
      "owner": null,
      "burn_tx": null,
      "pages": 297,
      "tokens": 573245
    },
    {
      "spider_id": "spider-177",
      "name": "spider-177",
      "kind": "queen",
      "owner": null,
      "burn_tx": null,
      "pages": 338,
      "tokens": 571467
    },
    {
      "spider_id": "spider-147",
      "name": "spider-147",
      "kind": "queen",
      "owner": null,
      "burn_tx": null,
      "pages": 305,
      "tokens": 569516
    },
    {
      "spider_id": "spider-171",
      "name": "spider-171",
      "kind": "queen",
      "owner": null,
      "burn_tx": null,
      "pages": 309,
      "tokens": 567054
    },
    {
      "spider_id": "spider-115",
      "name": "spider-115",
      "kind": "queen",
      "owner": null,
      "burn_tx": null,
      "pages": 341,
      "tokens": 566272
    },
    {
      "spider_id": "spider-141",
      "name": "spider-141",
      "kind": "queen",
      "owner": null,
      "burn_tx": null,
      "pages": 375,
      "tokens": 564651
    },
    {
      "spider_id": "spider-124",
      "name": "spider-124",
      "kind": "queen",
      "owner": null,
      "burn_tx": null,
      "pages": 324,
      "tokens": 561483
    },
    {
      "spider_id": "spider-130",
      "name": "spider-130",
      "kind": "queen",
      "owner": null,
      "burn_tx": null,
      "pages": 346,
      "tokens": 558800
    },
    {
      "spider_id": "spider-133",
      "name": "spider-133",
      "kind": "queen",
      "owner": null,
      "burn_tx": null,
      "pages": 344,
      "tokens": 557752
    },
    {
      "spider_id": "spider-142",
      "name": "spider-142",
      "kind": "queen",
      "owner": null,
      "burn_tx": null,
      "pages": 344,
      "tokens": 552839
    },
    {
      "spider_id": "spider-117",
      "name": "spider-117",
      "kind": "queen",
      "owner": null,
      "burn_tx": null,
      "pages": 361,
      "tokens": 525406
    },
    {
      "spider_id": "spider-183",
      "name": "spider-183",
      "kind": "queen",
      "owner": null,
      "burn_tx": null,
      "pages": 317,
      "tokens": 523941
    },
    {
      "spider_id": "spider-174",
      "name": "spider-174",
      "kind": "queen",
      "owner": null,
      "burn_tx": null,
      "pages": 273,
      "tokens": 490409
    },
    {
      "spider_id": "spider-181",
      "name": "spider-181",
      "kind": "queen",
      "owner": null,
      "burn_tx": null,
      "pages": 257,
      "tokens": 488999
    },
    {
      "spider_id": "spider-169",
      "name": "spider-169",
      "kind": "queen",
      "owner": null,
      "burn_tx": null,
      "pages": 284,
      "tokens": 487042
    },
    {
      "spider_id": "spider-152",
      "name": "spider-152",
      "kind": "queen",
      "owner": null,
      "burn_tx": null,
      "pages": 336,
      "tokens": 483309
    },
    {
      "spider_id": "spider-164",
      "name": "spider-164",
      "kind": "queen",
      "owner": null,
      "burn_tx": null,
      "pages": 295,
      "tokens": 459644
    },
    {
      "spider_id": "spider-182",
      "name": "spider-182",
      "kind": "queen",
      "owner": null,
      "burn_tx": null,
      "pages": 308,
      "tokens": 452045
    },
    {
      "spider_id": "spider-175",
      "name": "spider-175",
      "kind": "queen",
      "owner": null,
      "burn_tx": null,
      "pages": 244,
      "tokens": 436415
    },
    {
      "spider_id": "spider-168",
      "name": "spider-168",
      "kind": "queen",
      "owner": null,
      "burn_tx": null,
      "pages": 309,
      "tokens": 436316
    },
    {
      "spider_id": "spider-179",
      "name": "spider-179",
      "kind": "queen",
      "owner": null,
      "burn_tx": null,
      "pages": 267,
      "tokens": 425945
    },
    {
      "spider_id": "spider-172",
      "name": "spider-172",
      "kind": "queen",
      "owner": null,
      "burn_tx": null,
      "pages": 254,
      "tokens": 399802
    },
    {
      "spider_id": "spider-170",
      "name": "spider-170",
      "kind": "queen",
      "owner": null,
      "burn_tx": null,
      "pages": 246,
      "tokens": 385536
    },
    {
      "spider_id": "spider-178",
      "name": "spider-178",
      "kind": "queen",
      "owner": null,
      "burn_tx": null,
      "pages": 260,
      "tokens": 363115
    },
    {
      "spider_id": "spider-189",
      "name": "spider-189",
      "kind": "queen",
      "owner": null,
      "burn_tx": null,
      "pages": 121,
      "tokens": 328039
    },
    {
      "spider_id": "spider-191",
      "name": "spider-191",
      "kind": "queen",
      "owner": null,
      "burn_tx": null,
      "pages": 125,
      "tokens": 286249
    },
    {
      "spider_id": "spider-188",
      "name": "spider-188",
      "kind": "queen",
      "owner": null,
      "burn_tx": null,
      "pages": 145,
      "tokens": 267474
    },
    {
      "spider_id": "spider-185",
      "name": "spider-185",
      "kind": "queen",
      "owner": null,
      "burn_tx": null,
      "pages": 152,
      "tokens": 246111
    },
    {
      "spider_id": "spider-184",
      "name": "spider-184",
      "kind": "queen",
      "owner": null,
      "burn_tx": null,
      "pages": 178,
      "tokens": 214114
    },
    {
      "spider_id": "spider-186",
      "name": "spider-186",
      "kind": "queen",
      "owner": null,
      "burn_tx": null,
      "pages": 140,
      "tokens": 209374
    },
    {
      "spider_id": "spider-187",
      "name": "spider-187",
      "kind": "queen",
      "owner": null,
      "burn_tx": null,
      "pages": 113,
      "tokens": 158597
    },
    {
      "spider_id": "spider-193",
      "name": "spider-193",
      "kind": "queen",
      "owner": null,
      "burn_tx": null,
      "pages": 75,
      "tokens": 142691
    },
    {
      "spider_id": "spider-190",
      "name": "spider-190",
      "kind": "queen",
      "owner": null,
      "burn_tx": null,
      "pages": 85,
      "tokens": 140273
    },
    {
      "spider_id": "spider-194",
      "name": "spider-194",
      "kind": "queen",
      "owner": null,
      "burn_tx": null,
      "pages": 50,
      "tokens": 105080
    },
    {
      "spider_id": "spider-192",
      "name": "spider-192",
      "kind": "queen",
      "owner": null,
      "burn_tx": null,
      "pages": 48,
      "tokens": 65896
    },
    {
      "spider_id": "spider-195",
      "name": "spider-195",
      "kind": "queen",
      "owner": null,
      "burn_tx": null,
      "pages": 29,
      "tokens": 39995
    }
  ],
  "contributors_exported_at": null,
  "recipe": {
    "lang": true,
    "url_filters": true,
    "x_filter": true,
    "wiki_tails": true,
    "token_chunks": true,
    "stream": true,
    "records_forms": true,
    "qa_decay": true,
    "source_line": true,
    "headings": "v31",
    "pass_scale": 2.0,
    "domain_cap": 0.05,
    "decay_frac": 0.2,
    "qa_share": 0.25,
    "qa_max_heading_pairs": 100000,
    "lid_threshold": 0.65,
    "headings_min_edu": 2,
    "headings_max_domain_share": 0.03,
    "recipe": "v31",
    "chunk_tokens": 2040,
    "labels": null,
    "shard_chars": 200000000,
    "row_group": 256,
    "bench_protocols": null,
    "lid_model": "lid.176.bin"
  },
  "val": {
    "source": "external held-out set",
    "dataset": "heldout-q31",
    "manifest_sha256": "7efdd2af22792322c436c80449fe6153083d59a95a0605ee4eab29459657d584"
  },
  "heldout_exclusions": {
    "files": [
      "exclude.json"
    ],
    "urls": 2000,
    "sha256": 2000,
    "dropped": 2000
  },
  "records": {
    "parsed": {
      "chain": 120,
      "investor": 8339,
      "project": 29103,
      "protocol": 6884,
      "raises": 163,
      "stablecoin": 119,
      "token_facts": 230
    },
    "unparsed": {
      "project": 51
    },
    "forms": {
      "chain": 960,
      "investor": 25017,
      "project": 87270,
      "protocol": 54366,
      "raises": 1304,
      "stablecoin": 952,
      "token_facts": 1840
    },
    "qa_forms": 95325,
    "leak_audit_dropped": {}
  },
  "stream": {
    "pass_scale": 2.0,
    "seed": 0,
    "passes_table": {
      "edu": {
        "0": 0,
        "1": 1,
        "2": 2,
        "3": 4,
        "4": 6,
        "5": 8
      },
      "good_min": 6,
      "unlabeled": {
        "good": 6,
        "none": 2,
        "crawl": 4,
        "records": 4,
        "form": 2
      }
    },
    "domain_cap": 0.05,
    "units": {
      "page": 144359,
      "records": 46508,
      "form": 172975
    },
    "unique_tokens": {
      "page": 172076164,
      "records": 10794512,
      "form": 10382023
    },
    "source_line": {
      "on": true,
      "plain_from": 0.9,
      "units_with_source": 363842
    },
    "classes": {
      "form": {
        "units": 172975,
        "occurrences": 691900,
        "tokens": 41528092,
        "passes_mean": 4.0
      },
      "good": {
        "units": 25214,
        "occurrences": 299656,
        "tokens": 361754490,
        "passes_mean": 11.885
      },
      "normal": {
        "units": 29666,
        "occurrences": 237328,
        "tokens": 277729504,
        "passes_mean": 8.0
      },
      "records": {
        "units": 46508,
        "occurrences": 372064,
        "tokens": 86356096,
        "passes_mean": 8.0
      },
      "weak": {
        "units": 89479,
        "occurrences": 357916,
        "tokens": 427320188,
        "passes_mean": 4.0
      }
    },
    "top_sources": [
      [
        "records",
        0.107
      ],
      [
        "en.wikipedia.org",
        0.05
      ],
      [
        "forum.arbitrum.foundation",
        0.0403
      ],
      [
        "github.com",
        0.0247
      ],
      [
        "ethereum-magicians.org",
        0.0216
      ],
      [
        "gov.optimism.io",
        0.0211
      ],
      [
        "docs.chain.link",
        0.0172
      ],
      [
        "polymarket.com",
        0.016
      ],
      [
        "governance.aave.com",
        0.0149
      ],
      [
        "solanacompass.com",
        0.0148
      ],
      [
        "galaxy.com",
        0.0145
      ],
      [
        "rapidinnovation.io",
        0.0134
      ],
      [
        "cryptoslate.com",
        0.0119
      ],
      [
        "gov.uniswap.org",
        0.0116
      ],
      [
        "research.lido.fi",
        0.0111
      ],
      [
        "ethresear.ch",
        0.0096
      ],
      [
        "ethereum.org",
        0.0094
      ],
      [
        "bitcoin.org",
        0.0091
      ],
      [
        "forum.balancer.fi",
        0.0085
      ],
      [
        "eips.ethereum.org",
        0.0085
      ]
    ],
    "capped_domains": [
      {
        "domain": "en.wikipedia.org",
        "share_before": 0.0536,
        "factor": 0.9284
      }
    ],
    "records_share": 0.107,
    "forms_share": 0.0348,
    "occurrences": 1958864,
    "decay": {
      "frac": 0.2,
      "tokens_target": 254254181,
      "qa": {
        "pairs": 222270,
        "docs": 120947,
        "tokens": 54219003,
        "target_tokens": 63563545,
        "reps": 4,
        "formats": {
          "question_answer": 40240,
          "faq": 40316,
          "q_a": 40391
        },
        "short_of_target": true
      },
      "qa_tokens": 54980953,
      "moved_tokens": 199273784,
      "moved_good_tokens": 199273784,
      "moved_documents": 481891,
      "qa_share": 0.2162,
      "documents": 602838,
      "shards": 6
    },
    "tokens": {
      "main": 1008433578,
      "decay": 254254737,
      "total": 1262688315
    },
    "documents": {
      "main": 1476973,
      "decay": 602838
    },
    "dirs": {
      "main": "nanochat/stream/main",
      "decay": "nanochat/stream/decay"
    },
    "qa_pairs": {
      "records": 90597,
      "faq": 51183,
      "heading": 68002,
      "page": 12488
    }
  },
  "contamination": {
    "bench_files": {
      "arc_easy_test.json": {
        "sha256": "76e55faa15bce1b6ef26212c0c67a242a39952088d3b9e1a4d6071ea33c15190",
        "items": 2376
      },
      "cryptomc.json": {
        "sha256": "c81b3367d654f1e207d2dcbb48f490ff5feb496e2d08a049d39fb8aa8de35f87",
        "items": 150
      },
      "cryptoterms.json": {
        "sha256": "918b257c12deac8818cebf271187eedd88f026f875908b5a2e6987434b991f90",
        "items": 150
      },
      "ctslang.json": {
        "sha256": "d967e9196888027a4055592381892fca6185ee3a41b70be6a7351fa06d0bfc70",
        "items": 150
      },
      "projectid.json": {
        "sha256": "6feec798ea7269d2931fba111c48b33af2030a885e4c589235072ca6e488f51e",
        "items": 200
      },
      "projectid_xl.json": {
        "sha256": "859b5fc0d6a197bdf38e09e980d4577bd32f321e4fb3aec674725b6e4ed9a709",
        "items": 2593
      },
      "qa.json": {
        "sha256": "d5f699bdec9dca94e355332380adf317ef497784a56d3d9277ca6c0f993af9c1",
        "items": 100
      }
    },
    "skipped_files": [
      "._arc_easy_test.json",
      "._cryptomc.json",
      "._cryptoterms.json",
      "._ctslang.json",
      "._fresh_pages.json",
      "._fresh_pages_log.json",
      "._grade-0.json",
      "._grade-1.json",
      "._grade-2.json",
      "._grade-3.json",
      "._grade-4.json",
      "._grade-key.json",
      "._grade2-a.json",
      "._grade2-b.json",
      "._grade2-c.json",
      "._grade2-grades.json",
      "._grade2-key.json",
      "._grade3-a.json",
      "._grade3-b.json",
      "._grade3-c.json",
      "._grade3-grades.json",
      "._grade3-key.json",
      "._models.json",
      "._projectid.json",
      "._projectid_bench_protocols.json",
      "._projectid_defillama_raw.json",
      "._projectid_meta.json",
      "._projectid_results.json",
      "._projectid_results_detail.json",
      "._projectid_split.json",
      "._projectid_xl.json",
      "._projectid_xl_meta.json",
      "._qa.json",
      "._qa_answers.json",
      "._report_q31.json",
      "._results.json",
      "._results_ctslang.json",
      "._results_ctslang_detail.json",
      "._results_detail.json",
      "._results_domain.json",
      "._results_domain_detail.json",
      "._smoke.json",
      "._web_snapshot.json",
      "fresh_pages.json",
      "fresh_pages_log.json",
      "grade-0.json",
      "grade-1.json",
      "grade-2.json",
      "grade-3.json",
      "grade-4.json",
      "grade-key.json",
      "grade2-a.json",
      "grade2-b.json",
      "grade2-c.json",
      "grade2-grades.json",
      "grade2-key.json",
      "grade3-a.json",
      "grade3-b.json",
      "grade3-c.json",
      "grade3-grades.json",
      "grade3-key.json",
      "models.json",
      "projectid_bench_protocols.json",
      "projectid_defillama_raw.json",
      "projectid_meta.json",
      "projectid_results.json",
      "projectid_results_detail.json",
      "projectid_split.json",
      "projectid_xl_meta.json",
      "qa_answers.json",
      "report_q31.json",
      "results.json",
      "results_ctslang.json",
      "results_ctslang_detail.json",
      "results_detail.json",
      "results_domain.json",
      "results_domain_detail.json",
      "smoke.json",
      "web_snapshot.json"
    ],
    "criteria": {
      "ngram13": 13,
      "ngram8": 8,
      "ngram8_share": 0.7,
      "question_min_words_in_text": 5,
      "answer_within_words": 100
    },
    "dropped": {
      "chunks": 0,
      "records_forms": 0,
      "qa_pairs": 753,
      "headings.train": 1136,
      "headings.val": 36
    },
    "checked": "on",
    "by_criterion": {
      "qa_pairs": {
        "question": 752,
        "ngram8_70": 3,
        "question_answer": 1,
        "ngram13": 1
      },
      "headings.train": {
        "question": 1132,
        "ngram13": 4,
        "ngram8_70": 4,
        "question_answer": 4
      },
      "headings.val": {
        "question": 36
      }
    },
    "by_bench": {
      "qa_pairs": {
        "qa.json": 751,
        "cryptomc.json": 8,
        "cryptoterms.json": 1
      },
      "headings.train": {
        "qa.json": 1129,
        "cryptomc.json": 16,
        "cryptoterms.json": 3
      },
      "headings.val": {
        "qa.json": 36
      }
    },
    "bench_fingerprint": "656b4d7aa50c7d85e43d3915aae7dce568d54be7842219a97db529bbc349c125"
  },
  "labels": null,
  "bench_protocols": {
    "list": {
      "file": "projectid_bench_protocols.json",
      "sha256": "4e4f7eaae1e655f2ccc53de0b701ee20a0cea50be540024d6ec4cdeb6eef7471",
      "keys": 2222,
      "urls": 1922
    },
    "protocol_records": 6884,
    "bench_records": 1707,
    "synth_deny_list": {
      "file": "stream_src/synth_deny.txt",
      "urls": 1707,
      "every_protocol": false
    },
    "qa_forms_left_out": 4725
  },
  "restreamed_at": "2026-10-07T01:32:05+00:00"
}