queen-v4 / dataset.json
Crawlnet's picture
queen-v4 (6792bfb740b3)
54cb090 verified
Raw History Blame Contribute Delete
55.5 kB
{
"version": 4,
"toy": false,
"rung": null,
"prepared_at": "2026-10-07T01:27:18+00:00",
"dataset_dir": "dataset-v4-full",
"manifest_sha256": "6792bfb740b3c7217663afdba60ae75f5ae6d79aa53ac1475971fabe51a981ac",
"manifest": {
"pages": 181924,
"tokens": 330366386,
"created_at": "2026-10-07T00:58:27+00:00"
},
"manifest_shards": [
{
"file": "crawl-shard-00000.jsonl",
"pages": 20000,
"tokens": 47840786,
"bytes": 175257859,
"sha256": "012bf5d772f4731eaeddcd909029d07bbe90afb2088d425cbf903b92280095e3",
"source": "crawl"
},
{
"file": "crawl-shard-00001.jsonl",
"pages": 20000,
"tokens": 44063575,
"bytes": 162061573,
"sha256": "ead9a77cebab1660e399cb47db2adee31055847dba512dbb3ef74d03813b2b00",
"source": "crawl"
},
{
"file": "crawl-shard-00002.jsonl",
"pages": 20000,
"tokens": 53181778,
"bytes": 196369474,
"sha256": "5ad1e712e52a266041ad1e37cf1b27fdb9c21260a228c4d072ce6b0e38bf460e",
"source": "crawl"
},
{
"file": "crawl-shard-00003.jsonl",
"pages": 20000,
"tokens": 51313378,
"bytes": 187276813,
"sha256": "cd1b35c6649443e5e2c31ac7091045dc2a550411d12676cfda7c00918ebd9cbf",
"source": "crawl"
},
{
"file": "crawl-shard-00004.jsonl",
"pages": 20000,
"tokens": 45012511,
"bytes": 167024162,
"sha256": "1586d5a71304ef352103f7b6e61f0189f3e63a29d887a53b0c54b70a2f848b59",
"source": "crawl"
},
{
"file": "crawl-shard-00005.jsonl",
"pages": 20000,
"tokens": 42741504,
"bytes": 157047166,
"sha256": "4c750107a6ba28f0086af7d3b75f858b1aa76a1db8980cffa126db796c719da2",
"source": "crawl"
},
{
"file": "crawl-shard-00006.jsonl",
"pages": 16260,
"tokens": 35305976,
"bytes": 129501055,
"sha256": "b1ad8bae8ceb4f69a291e23a82e358fef26ac1cae7ab4acf6f189ecf556e0b9f",
"source": "crawl"
},
{
"file": "records-shard-00000.jsonl",
"pages": 5000,
"tokens": 1158392,
"bytes": 4949845,
"sha256": "c23a21d5a97448f678524a879c9e8cc4a8df3caf49a8fd52824759bb95a88d20",
"source": "records"
},
{
"file": "records-shard-00001.jsonl",
"pages": 5000,
"tokens": 1096599,
"bytes": 4915698,
"sha256": "ec75f7ae6a88e48fcb7d1d3373fda9ce74c3a816174e87844e5760f9b95baf6d",
"source": "records"
},
{
"file": "records-shard-00002.jsonl",
"pages": 5000,
"tokens": 918678,
"bytes": 4308866,
"sha256": "046f7198ec3f62d71aef5766ac7a11a6ca4480c045bf1e1d0b843fb32b878741",
"source": "records"
},
{
"file": "records-shard-00003.jsonl",
"pages": 5000,
"tokens": 817118,
"bytes": 3988989,
"sha256": "a20430453ed74f27a9f121446bf5a62fa31d8377da07e499d1c8d9d1dd779e85",
"source": "records"
},
{
"file": "records-shard-00004.jsonl",
"pages": 5000,
"tokens": 857884,
"bytes": 4127675,
"sha256": "844e7fed170a4e884aeda1f66cae789c25567b0ff3bd6cc4b2ad48f8abad62d8",
"source": "records"
},
{
"file": "records-shard-00005.jsonl",
"pages": 5000,
"tokens": 883422,
"bytes": 4218635,
"sha256": "42c798f331a149394804573b62e74b14528fc8f8f5bc22ee1ac0a7f3e6918a2b",
"source": "records"
},
{
"file": "records-shard-00006.jsonl",
"pages": 5000,
"tokens": 845402,
"bytes": 4086717,
"sha256": "f509992a39f656bdb263ce64f30fe360f39665fda2b8cef52f233e055d4289b0",
"source": "records"
},
{
"file": "records-shard-00007.jsonl",
"pages": 5000,
"tokens": 1921725,
"bytes": 6913850,
"sha256": "7d2215063b7c8ea010bae86449b1a1153e22b3c32cb54d0dac711c1f306accf8",
"source": "records"
},
{
"file": "records-shard-00008.jsonl",
"pages": 5000,
"tokens": 789413,
"bytes": 3765224,
"sha256": "7e8ea4008c2ef5d5ebf07cc53b1f45c9e0082416ef2227bb3452dcf62a574899",
"source": "records"
},
{
"file": "records-shard-00009.jsonl",
"pages": 664,
"tokens": 1618245,
"bytes": 5173560,
"sha256": "46b0e5806746a10a607909c70b4998ff4f6a7cea5b1a783e5e35d700cdb4ecf8",
"source": "records"
}
],
"cleaning": {
"input_pages": 179924,
"dup_url": 0,
"dropped_url_class": {
"casino": 289,
"convert": 1746,
"flashnews": 8345,
"forecast": 48,
"jobs": 1360,
"price": 794,
"price_prediction": 821,
"zh": 669
},
"boilerplate_line_kinds": 19179,
"boilerplate_lines_removed": 2698066,
"wiki_tails_cut": {
"pages": 17,
"chars": 240631
},
"dropped_short": 45,
"dropped_low_alpha": 15,
"dropped_x_rewrite": 10446,
"x_rewrite": {
"rule": "(@handles + 'posted on X'-style phrases) per 1,000 words > 2",
"handles": 55752,
"phrases": 6071,
"top_domains": [
[
"forum.arbitrum.foundation",
1356
],
[
"ethereum-magicians.org",
882
],
[
"gov.optimism.io",
863
],
[
"u.today",
630
],
[
"gov.uniswap.org",
389
],
[
"eips.ethereum.org",
329
],
[
"research.lido.fi",
279
],
[
"governance.aave.com",
279
],
[
"forum.balancer.fi",
250
],
[
"tapchibitcoin.io",
207
],
[
"github.com",
196
],
[
"bitcoinfoundation.org",
195
],
[
"discuss.jup.ag",
184
],
[
"coinlore.com",
132
],
[
"ethresear.ch",
117
],
[
"news.bitcoin.com",
104
],
[
"coindesk.com",
102
],
[
"coinfomania.com",
97
],
[
"cointelegraph.com",
93
],
[
"ankr.com",
90
]
],
"x_links_removed": {
"pages": 2208,
"links": 6130
},
"records_x_lines_removed": {
"pages": 36,
"lines": 39
}
},
"dropped_lang": 14915,
"lang": {
"model": "fastText lid.176",
"threshold": 0.65,
"dropped_by_top_language": {
"en": 6032,
"vi": 1879,
"pt": 731,
"de": 641,
"it": 532,
"ar": 443,
"id": 428,
"nl": 423,
"es": 391,
"tr": 376,
"pl": 371,
"fr": 349,
"cs": 288,
"ko": 287,
"ru": 250,
"hu": 169,
"uk": 159,
"ja": 136,
"th": 68,
"ro": 62
}
},
"dup_text": 3,
"kept_pages": 140428,
"dropped_heldout": 2000,
"bad_rows": 0,
"dropped_val_overlap": 0
},
"pages": {
"train": 140428,
"val": 2000
},
"chunks": {
"train": 190867,
"val": 4964,
"max_chars": null,
"max_tokens": 2040,
"train_chunks_longer_than_context": 0
},
"chars": {
"train": 790442138,
"val": 15792677
},
"tokens": {
"train": 183061543,
"val": 3726648,
"chars_per_token": 4.318
},
"tokenizer": {
"trained_on": "English train pages of this dataset only (spider pages, fastText lid.176 >= 0.65)",
"max_chars": 2000000000,
"doc_cap": 10000,
"vocab_size": 16384,
"trainer": "nanochat rustbpe (GPT-4 style split pattern)",
"sha256": "5782900e33bdfc9a614fd96502b9df631d90d17854ec79bff0aab9f354a64478",
"glossary": {
"terms": 680,
"tokens": 1031,
"single_token": 408,
"mean_tokens": 1.516,
"ref_tokens": 1106,
"worse": 1,
"better": 67,
"same": 612,
"worst": [
{
"term": "Substrate",
"tokens": 3,
"ref": 2
}
],
"passed": true,
"ref": "v3"
}
},
"sft_headings": {
"train": {
"conversations": 521532,
"by_kind": {
"heading": 394646,
"page": 75707,
"faq": 51179
},
"selection": {
"input": 521709,
"dropped_low_edu": 0,
"dropped_domain_cap": 177,
"kept": 521532,
"max_domain_share": 0.03,
"min_edu": 2
}
},
"val": {
"conversations": 12012,
"by_kind": {
"heading": 9451,
"page": 1431,
"faq": 1130
},
"selection": {
"input": 12042,
"dropped_low_edu": 0,
"dropped_domain_cap": 30,
"kept": 12012,
"max_domain_share": 0.03,
"min_edu": 2
}
}
},
"chapters": {
"none": 60570,
"whitepapers": 564,
"blogs": 2028,
"docs": 14034,
"solana": 5031,
"forums": 12574,
"evm-dex": 127,
"defillama-chains": 120,
"investors": 8339,
"projects": 29154,
"defillama-protocols": 6884,
"raises": 163,
"solana-mints-burns": 120,
"solana-dex": 364,
"defillama-stablecoins": 119,
"token-facts": 230,
"unlocks": 7
},
"domains": 2745,
"contributors": [
{
"spider_id": "queen-records",
"name": "queen-records",
"kind": "queen",
"owner": null,
"burn_tx": null,
"pages": 45627,
"tokens": 10830251
},
{
"spider_id": "spider-05",
"name": "spider-05",
"kind": "queen",
"owner": null,
"burn_tx": null,
"pages": 841,
"tokens": 1692853
},
{
"spider_id": "spider-04",
"name": "spider-04",
"kind": "queen",
"owner": null,
"burn_tx": null,
"pages": 926,
"tokens": 1668484
},
{
"spider_id": "spider-03",
"name": "spider-03",
"kind": "queen",
"owner": null,
"burn_tx": null,
"pages": 852,
"tokens": 1662728
},
{
"spider_id": "spider-02",
"name": "spider-02",
"kind": "queen",
"owner": null,
"burn_tx": null,
"pages": 846,
"tokens": 1615554
},
{
"spider_id": "spider-01",
"name": "spider-01",
"kind": "queen",
"owner": null,
"burn_tx": null,
"pages": 854,
"tokens": 1587959
},
{
"spider_id": "spider-13",
"name": "spider-13",
"kind": "queen",
"owner": null,
"burn_tx": null,
"pages": 735,
"tokens": 1536440
},
{
"spider_id": "spider-07",
"name": "spider-07",
"kind": "queen",
"owner": null,
"burn_tx": null,
"pages": 724,
"tokens": 1517295
},
{
"spider_id": "spider-06",
"name": "spider-06",
"kind": "queen",
"owner": null,
"burn_tx": null,
"pages": 863,
"tokens": 1482433
},
{
"spider_id": "spider-24",
"name": "spider-24",
"kind": "queen",
"owner": null,
"burn_tx": null,
"pages": 747,
"tokens": 1474814
},
{
"spider_id": "spider-32",
"name": "spider-32",
"kind": "queen",
"owner": null,
"burn_tx": null,
"pages": 703,
"tokens": 1458174
},
{
"spider_id": "spider-33",
"name": "spider-33",
"kind": "queen",
"owner": null,
"burn_tx": null,
"pages": 647,
"tokens": 1442197
},
{
"spider_id": "spider-08",
"name": "spider-08",
"kind": "queen",
"owner": null,
"burn_tx": null,
"pages": 762,
"tokens": 1440335
},
{
"spider_id": "spider-69",
"name": "spider-69",
"kind": "queen",
"owner": null,
"burn_tx": null,
"pages": 616,
"tokens": 1426349
},
{
"spider_id": "spider-58",
"name": "spider-58",
"kind": "queen",
"owner": null,
"burn_tx": null,
"pages": 653,
"tokens": 1397465
},
{
"spider_id": "spider-57",
"name": "spider-57",
"kind": "queen",
"owner": null,
"burn_tx": null,
"pages": 604,
"tokens": 1394965
},
{
"spider_id": "spider-41",
"name": "spider-41",
"kind": "queen",
"owner": null,
"burn_tx": null,
"pages": 686,
"tokens": 1378451
},
{
"spider_id": "spider-12",
"name": "spider-12",
"kind": "queen",
"owner": null,
"burn_tx": null,
"pages": 708,
"tokens": 1373638
},
{
"spider_id": "spider-46",
"name": "spider-46",
"kind": "queen",
"owner": null,
"burn_tx": null,
"pages": 682,
"tokens": 1359574
},
{
"spider_id": "spider-23",
"name": "spider-23",
"kind": "queen",
"owner": null,
"burn_tx": null,
"pages": 693,
"tokens": 1338465
},
{
"spider_id": "spider-26",
"name": "spider-26",
"kind": "queen",
"owner": null,
"burn_tx": null,
"pages": 652,
"tokens": 1335145
},
{
"spider_id": "spider-31",
"name": "spider-31",
"kind": "queen",
"owner": null,
"burn_tx": null,
"pages": 650,
"tokens": 1322803
},
{
"spider_id": "spider-11",
"name": "spider-11",
"kind": "queen",
"owner": null,
"burn_tx": null,
"pages": 723,
"tokens": 1313413
},
{
"spider_id": "spider-52",
"name": "spider-52",
"kind": "queen",
"owner": null,
"burn_tx": null,
"pages": 667,
"tokens": 1304428
},
{
"spider_id": "spider-81",
"name": "spider-81",
"kind": "queen",
"owner": null,
"burn_tx": null,
"pages": 628,
"tokens": 1300351
},
{
"spider_id": "spider-17",
"name": "spider-17",
"kind": "queen",
"owner": null,
"burn_tx": null,
"pages": 722,
"tokens": 1295522
},
{
"spider_id": "spider-28",
"name": "spider-28",
"kind": "queen",
"owner": null,
"burn_tx": null,
"pages": 666,
"tokens": 1274072
},
{
"spider_id": "spider-20",
"name": "spider-20",
"kind": "queen",
"owner": null,
"burn_tx": null,
"pages": 738,
"tokens": 1263457
},
{
"spider_id": "spider-19",
"name": "spider-19",
"kind": "queen",
"owner": null,
"burn_tx": null,
"pages": 763,
"tokens": 1262077
},
{
"spider_id": "spider-48",
"name": "spider-48",
"kind": "queen",
"owner": null,
"burn_tx": null,
"pages": 622,
"tokens": 1253761
},
{
"spider_id": "spider-29",
"name": "spider-29",
"kind": "queen",
"owner": null,
"burn_tx": null,
"pages": 688,
"tokens": 1244215
},
{
"spider_id": "spider-25",
"name": "spider-25",
"kind": "queen",
"owner": null,
"burn_tx": null,
"pages": 689,
"tokens": 1241588
},
{
"spider_id": "spider-65",
"name": "spider-65",
"kind": "queen",
"owner": null,
"burn_tx": null,
"pages": 662,
"tokens": 1236627
},
{
"spider_id": "spider-45",
"name": "spider-45",
"kind": "queen",
"owner": null,
"burn_tx": null,
"pages": 646,
"tokens": 1227888
},
{
"spider_id": "spider-38",
"name": "spider-38",
"kind": "queen",
"owner": null,
"burn_tx": null,
"pages": 629,
"tokens": 1223193
},
{
"spider_id": "spider-14",
"name": "spider-14",
"kind": "queen",
"owner": null,
"burn_tx": null,
"pages": 700,
"tokens": 1218348
},
{
"spider_id": "spider-40",
"name": "spider-40",
"kind": "queen",
"owner": null,
"burn_tx": null,
"pages": 684,
"tokens": 1205962
},
{
"spider_id": "spider-21",
"name": "spider-21",
"kind": "queen",
"owner": null,
"burn_tx": null,
"pages": 691,
"tokens": 1205224
},
{
"spider_id": "spider-66",
"name": "spider-66",
"kind": "queen",
"owner": null,
"burn_tx": null,
"pages": 676,
"tokens": 1199029
},
{
"spider_id": "spider-22",
"name": "spider-22",
"kind": "queen",
"owner": null,
"burn_tx": null,
"pages": 701,
"tokens": 1196474
},
{
"spider_id": "spider-15",
"name": "spider-15",
"kind": "queen",
"owner": null,
"burn_tx": null,
"pages": 680,
"tokens": 1187175
},
{
"spider_id": "spider-10",
"name": "spider-10",
"kind": "queen",
"owner": null,
"burn_tx": null,
"pages": 675,
"tokens": 1181301
},
{
"spider_id": "spider-70",
"name": "spider-70",
"kind": "queen",
"owner": null,
"burn_tx": null,
"pages": 617,
"tokens": 1169522
},
{
"spider_id": "spider-49",
"name": "spider-49",
"kind": "queen",
"owner": null,
"burn_tx": null,
"pages": 634,
"tokens": 1159457
},
{
"spider_id": "spider-39",
"name": "spider-39",
"kind": "queen",
"owner": null,
"burn_tx": null,
"pages": 664,
"tokens": 1137328
},
{
"spider_id": "spider-34",
"name": "spider-34",
"kind": "queen",
"owner": null,
"burn_tx": null,
"pages": 686,
"tokens": 1136731
},
{
"spider_id": "spider-42",
"name": "spider-42",
"kind": "queen",
"owner": null,
"burn_tx": null,
"pages": 609,
"tokens": 1136613
},
{
"spider_id": "spider-73",
"name": "spider-73",
"kind": "queen",
"owner": null,
"burn_tx": null,
"pages": 595,
"tokens": 1132373
},
{
"spider_id": "spider-64",
"name": "spider-64",
"kind": "queen",
"owner": null,
"burn_tx": null,
"pages": 617,
"tokens": 1117763
},
{
"spider_id": "spider-82",
"name": "spider-82",
"kind": "queen",
"owner": null,
"burn_tx": null,
"pages": 590,
"tokens": 1117054
},
{
"spider_id": "spider-09",
"name": "spider-09",
"kind": "queen",
"owner": null,
"burn_tx": null,
"pages": 693,
"tokens": 1113423
},
{
"spider_id": "spider-75",
"name": "spider-75",
"kind": "queen",
"owner": null,
"burn_tx": null,
"pages": 577,
"tokens": 1104448
},
{
"spider_id": "spider-101",
"name": "spider-101",
"kind": "queen",
"owner": null,
"burn_tx": null,
"pages": 495,
"tokens": 1090813
},
{
"spider_id": "spider-67",
"name": "spider-67",
"kind": "queen",
"owner": null,
"burn_tx": null,
"pages": 600,
"tokens": 1085070
},
{
"spider_id": "spider-36",
"name": "spider-36",
"kind": "queen",
"owner": null,
"burn_tx": null,
"pages": 648,
"tokens": 1082160
},
{
"spider_id": "spider-27",
"name": "spider-27",
"kind": "queen",
"owner": null,
"burn_tx": null,
"pages": 678,
"tokens": 1075869
},
{
"spider_id": "spider-18",
"name": "spider-18",
"kind": "queen",
"owner": null,
"burn_tx": null,
"pages": 677,
"tokens": 1066376
},
{
"spider_id": "spider-71",
"name": "spider-71",
"kind": "queen",
"owner": null,
"burn_tx": null,
"pages": 612,
"tokens": 1059893
},
{
"spider_id": "spider-30",
"name": "spider-30",
"kind": "queen",
"owner": null,
"burn_tx": null,
"pages": 657,
"tokens": 1056643
},
{
"spider_id": "spider-44",
"name": "spider-44",
"kind": "queen",
"owner": null,
"burn_tx": null,
"pages": 590,
"tokens": 1048690
},
{
"spider_id": "spider-43",
"name": "spider-43",
"kind": "queen",
"owner": null,
"burn_tx": null,
"pages": 656,
"tokens": 1048301
},
{
"spider_id": "spider-35",
"name": "spider-35",
"kind": "queen",
"owner": null,
"burn_tx": null,
"pages": 590,
"tokens": 1047010
},
{
"spider_id": "spider-60",
"name": "spider-60",
"kind": "queen",
"owner": null,
"burn_tx": null,
"pages": 609,
"tokens": 1038729
},
{
"spider_id": "spider-85",
"name": "spider-85",
"kind": "queen",
"owner": null,
"burn_tx": null,
"pages": 533,
"tokens": 1037504
},
{
"spider_id": "spider-86",
"name": "spider-86",
"kind": "queen",
"owner": null,
"burn_tx": null,
"pages": 511,
"tokens": 1031582
},
{
"spider_id": "spider-72",
"name": "spider-72",
"kind": "queen",
"owner": null,
"burn_tx": null,
"pages": 617,
"tokens": 1022998
},
{
"spider_id": "spider-123",
"name": "spider-123",
"kind": "queen",
"owner": null,
"burn_tx": null,
"pages": 390,
"tokens": 1020861
},
{
"spider_id": "spider-62",
"name": "spider-62",
"kind": "queen",
"owner": null,
"burn_tx": null,
"pages": 588,
"tokens": 1008813
},
{
"spider_id": "spider-109",
"name": "spider-109",
"kind": "queen",
"owner": null,
"burn_tx": null,
"pages": 519,
"tokens": 994913
},
{
"spider_id": "spider-87",
"name": "spider-87",
"kind": "queen",
"owner": null,
"burn_tx": null,
"pages": 511,
"tokens": 992120
},
{
"spider_id": "spider-54",
"name": "spider-54",
"kind": "queen",
"owner": null,
"burn_tx": null,
"pages": 651,
"tokens": 991981
},
{
"spider_id": "spider-97",
"name": "spider-97",
"kind": "queen",
"owner": null,
"burn_tx": null,
"pages": 524,
"tokens": 988073
},
{
"spider_id": "spider-68",
"name": "spider-68",
"kind": "queen",
"owner": null,
"burn_tx": null,
"pages": 607,
"tokens": 985389
},
{
"spider_id": "spider-63",
"name": "spider-63",
"kind": "queen",
"owner": null,
"burn_tx": null,
"pages": 592,
"tokens": 981462
},
{
"spider_id": "spider-98",
"name": "spider-98",
"kind": "queen",
"owner": null,
"burn_tx": null,
"pages": 568,
"tokens": 977601
},
{
"spider_id": "spider-61",
"name": "spider-61",
"kind": "queen",
"owner": null,
"burn_tx": null,
"pages": 606,
"tokens": 977574
},
{
"spider_id": "spider-16",
"name": "spider-16",
"kind": "queen",
"owner": null,
"burn_tx": null,
"pages": 689,
"tokens": 975878
},
{
"spider_id": "spider-51",
"name": "spider-51",
"kind": "queen",
"owner": null,
"burn_tx": null,
"pages": 649,
"tokens": 973831
},
{
"spider_id": "spider-77",
"name": "spider-77",
"kind": "queen",
"owner": null,
"burn_tx": null,
"pages": 528,
"tokens": 973816
},
{
"spider_id": "spider-83",
"name": "spider-83",
"kind": "queen",
"owner": null,
"burn_tx": null,
"pages": 549,
"tokens": 966853
},
{
"spider_id": "spider-90",
"name": "spider-90",
"kind": "queen",
"owner": null,
"burn_tx": null,
"pages": 509,
"tokens": 963175
},
{
"spider_id": "spider-135",
"name": "spider-135",
"kind": "queen",
"owner": null,
"burn_tx": null,
"pages": 368,
"tokens": 962209
},
{
"spider_id": "spider-47",
"name": "spider-47",
"kind": "queen",
"owner": null,
"burn_tx": null,
"pages": 614,
"tokens": 960974
},
{
"spider_id": "spider-93",
"name": "spider-93",
"kind": "queen",
"owner": null,
"burn_tx": null,
"pages": 485,
"tokens": 960061
},
{
"spider_id": "spider-79",
"name": "spider-79",
"kind": "queen",
"owner": null,
"burn_tx": null,
"pages": 536,
"tokens": 957583
},
{
"spider_id": "spider-59",
"name": "spider-59",
"kind": "queen",
"owner": null,
"burn_tx": null,
"pages": 529,
"tokens": 953718
},
{
"spider_id": "spider-76",
"name": "spider-76",
"kind": "queen",
"owner": null,
"burn_tx": null,
"pages": 575,
"tokens": 953181
},
{
"spider_id": "spider-92",
"name": "spider-92",
"kind": "queen",
"owner": null,
"burn_tx": null,
"pages": 525,
"tokens": 952317
},
{
"spider_id": "spider-50",
"name": "spider-50",
"kind": "queen",
"owner": null,
"burn_tx": null,
"pages": 617,
"tokens": 950383
},
{
"spider_id": "spider-96",
"name": "spider-96",
"kind": "queen",
"owner": null,
"burn_tx": null,
"pages": 545,
"tokens": 948223
},
{
"spider_id": "spider-89",
"name": "spider-89",
"kind": "queen",
"owner": null,
"burn_tx": null,
"pages": 517,
"tokens": 937528
},
{
"spider_id": "spider-108",
"name": "spider-108",
"kind": "queen",
"owner": null,
"burn_tx": null,
"pages": 510,
"tokens": 933970
},
{
"spider_id": "spider-99",
"name": "spider-99",
"kind": "queen",
"owner": null,
"burn_tx": null,
"pages": 540,
"tokens": 928101
},
{
"spider_id": "spider-53",
"name": "spider-53",
"kind": "queen",
"owner": null,
"burn_tx": null,
"pages": 552,
"tokens": 926608
},
{
"spider_id": "spider-80",
"name": "spider-80",
"kind": "queen",
"owner": null,
"burn_tx": null,
"pages": 530,
"tokens": 923432
},
{
"spider_id": "spider-84",
"name": "spider-84",
"kind": "queen",
"owner": null,
"burn_tx": null,
"pages": 515,
"tokens": 922461
},
{
"spider_id": "spider-95",
"name": "spider-95",
"kind": "queen",
"owner": null,
"burn_tx": null,
"pages": 452,
"tokens": 921857
},
{
"spider_id": "spider-37",
"name": "spider-37",
"kind": "queen",
"owner": null,
"burn_tx": null,
"pages": 606,
"tokens": 920668
},
{
"spider_id": "spider-78",
"name": "spider-78",
"kind": "queen",
"owner": null,
"burn_tx": null,
"pages": 536,
"tokens": 912846
},
{
"spider_id": "spider-144",
"name": "spider-144",
"kind": "queen",
"owner": null,
"burn_tx": null,
"pages": 408,
"tokens": 905626
},
{
"spider_id": "spider-100",
"name": "spider-100",
"kind": "queen",
"owner": null,
"burn_tx": null,
"pages": 504,
"tokens": 896326
},
{
"spider_id": "spider-55",
"name": "spider-55",
"kind": "queen",
"owner": null,
"burn_tx": null,
"pages": 581,
"tokens": 895110
},
{
"spider_id": "spider-56",
"name": "spider-56",
"kind": "queen",
"owner": null,
"burn_tx": null,
"pages": 593,
"tokens": 881178
},
{
"spider_id": "spider-114",
"name": "spider-114",
"kind": "queen",
"owner": null,
"burn_tx": null,
"pages": 400,
"tokens": 872805
},
{
"spider_id": "spider-118",
"name": "spider-118",
"kind": "queen",
"owner": null,
"burn_tx": null,
"pages": 355,
"tokens": 857226
},
{
"spider_id": "spider-139",
"name": "spider-139",
"kind": "queen",
"owner": null,
"burn_tx": null,
"pages": 372,
"tokens": 855311
},
{
"spider_id": "spider-134",
"name": "spider-134",
"kind": "queen",
"owner": null,
"burn_tx": null,
"pages": 406,
"tokens": 854840
},
{
"spider_id": "spider-104",
"name": "spider-104",
"kind": "queen",
"owner": null,
"burn_tx": null,
"pages": 515,
"tokens": 836639
},
{
"spider_id": "spider-149",
"name": "spider-149",
"kind": "queen",
"owner": null,
"burn_tx": null,
"pages": 337,
"tokens": 829398
},
{
"spider_id": "spider-103",
"name": "spider-103",
"kind": "queen",
"owner": null,
"burn_tx": null,
"pages": 492,
"tokens": 812322
},
{
"spider_id": "spider-166",
"name": "spider-166",
"kind": "queen",
"owner": null,
"burn_tx": null,
"pages": 349,
"tokens": 807096
},
{
"spider_id": "spider-91",
"name": "spider-91",
"kind": "queen",
"owner": null,
"burn_tx": null,
"pages": 507,
"tokens": 806440
},
{
"spider_id": "spider-107",
"name": "spider-107",
"kind": "queen",
"owner": null,
"burn_tx": null,
"pages": 519,
"tokens": 804924
},
{
"spider_id": "spider-148",
"name": "spider-148",
"kind": "queen",
"owner": null,
"burn_tx": null,
"pages": 355,
"tokens": 803132
},
{
"spider_id": "spider-88",
"name": "spider-88",
"kind": "queen",
"owner": null,
"burn_tx": null,
"pages": 500,
"tokens": 798800
},
{
"spider_id": "spider-106",
"name": "spider-106",
"kind": "queen",
"owner": null,
"burn_tx": null,
"pages": 512,
"tokens": 796260
},
{
"spider_id": "spider-94",
"name": "spider-94",
"kind": "queen",
"owner": null,
"burn_tx": null,
"pages": 532,
"tokens": 791025
},
{
"spider_id": "spider-156",
"name": "spider-156",
"kind": "queen",
"owner": null,
"burn_tx": null,
"pages": 397,
"tokens": 788783
},
{
"spider_id": "spider-140",
"name": "spider-140",
"kind": "queen",
"owner": null,
"burn_tx": null,
"pages": 339,
"tokens": 783380
},
{
"spider_id": "spider-119",
"name": "spider-119",
"kind": "queen",
"owner": null,
"burn_tx": null,
"pages": 407,
"tokens": 781758
},
{
"spider_id": "spider-161",
"name": "spider-161",
"kind": "queen",
"owner": null,
"burn_tx": null,
"pages": 348,
"tokens": 777115
},
{
"spider_id": "spider-154",
"name": "spider-154",
"kind": "queen",
"owner": null,
"burn_tx": null,
"pages": 371,
"tokens": 775887
},
{
"spider_id": "spider-110",
"name": "spider-110",
"kind": "queen",
"owner": null,
"burn_tx": null,
"pages": 465,
"tokens": 766973
},
{
"spider_id": "spider-102",
"name": "spider-102",
"kind": "queen",
"owner": null,
"burn_tx": null,
"pages": 487,
"tokens": 764842
},
{
"spider_id": "spider-143",
"name": "spider-143",
"kind": "queen",
"owner": null,
"burn_tx": null,
"pages": 400,
"tokens": 758536
},
{
"spider_id": "spider-113",
"name": "spider-113",
"kind": "queen",
"owner": null,
"burn_tx": null,
"pages": 373,
"tokens": 754280
},
{
"spider_id": "spider-74",
"name": "spider-74",
"kind": "queen",
"owner": null,
"burn_tx": null,
"pages": 557,
"tokens": 748295
},
{
"spider_id": "spider-167",
"name": "spider-167",
"kind": "queen",
"owner": null,
"burn_tx": null,
"pages": 334,
"tokens": 748128
},
{
"spider_id": "spider-145",
"name": "spider-145",
"kind": "queen",
"owner": null,
"burn_tx": null,
"pages": 385,
"tokens": 741840
},
{
"spider_id": "spider-151",
"name": "spider-151",
"kind": "queen",
"owner": null,
"burn_tx": null,
"pages": 359,
"tokens": 741547
},
{
"spider_id": "spider-116",
"name": "spider-116",
"kind": "queen",
"owner": null,
"burn_tx": null,
"pages": 391,
"tokens": 738707
},
{
"spider_id": "spider-132",
"name": "spider-132",
"kind": "queen",
"owner": null,
"burn_tx": null,
"pages": 364,
"tokens": 736976
},
{
"spider_id": "spider-105",
"name": "spider-105",
"kind": "queen",
"owner": null,
"burn_tx": null,
"pages": 456,
"tokens": 735612
},
{
"spider_id": "spider-121",
"name": "spider-121",
"kind": "queen",
"owner": null,
"burn_tx": null,
"pages": 369,
"tokens": 729813
},
{
"spider_id": "spider-153",
"name": "spider-153",
"kind": "queen",
"owner": null,
"burn_tx": null,
"pages": 397,
"tokens": 720902
},
{
"spider_id": "spider-138",
"name": "spider-138",
"kind": "queen",
"owner": null,
"burn_tx": null,
"pages": 353,
"tokens": 717669
},
{
"spider_id": "spider-122",
"name": "spider-122",
"kind": "queen",
"owner": null,
"burn_tx": null,
"pages": 354,
"tokens": 714317
},
{
"spider_id": "spider-180",
"name": "spider-180",
"kind": "queen",
"owner": null,
"burn_tx": null,
"pages": 306,
"tokens": 709530
},
{
"spider_id": "spider-125",
"name": "spider-125",
"kind": "queen",
"owner": null,
"burn_tx": null,
"pages": 374,
"tokens": 699434
},
{
"spider_id": "spider-150",
"name": "spider-150",
"kind": "queen",
"owner": null,
"burn_tx": null,
"pages": 360,
"tokens": 698648
},
{
"spider_id": "spider-146",
"name": "spider-146",
"kind": "queen",
"owner": null,
"burn_tx": null,
"pages": 401,
"tokens": 685836
},
{
"spider_id": "spider-126",
"name": "spider-126",
"kind": "queen",
"owner": null,
"burn_tx": null,
"pages": 351,
"tokens": 683075
},
{
"spider_id": "spider-176",
"name": "spider-176",
"kind": "queen",
"owner": null,
"burn_tx": null,
"pages": 312,
"tokens": 680954
},
{
"spider_id": "spider-129",
"name": "spider-129",
"kind": "queen",
"owner": null,
"burn_tx": null,
"pages": 383,
"tokens": 675353
},
{
"spider_id": "spider-158",
"name": "spider-158",
"kind": "queen",
"owner": null,
"burn_tx": null,
"pages": 325,
"tokens": 667011
},
{
"spider_id": "spider-160",
"name": "spider-160",
"kind": "queen",
"owner": null,
"burn_tx": null,
"pages": 319,
"tokens": 661323
},
{
"spider_id": "spider-157",
"name": "spider-157",
"kind": "queen",
"owner": null,
"burn_tx": null,
"pages": 380,
"tokens": 648929
},
{
"spider_id": "spider-127",
"name": "spider-127",
"kind": "queen",
"owner": null,
"burn_tx": null,
"pages": 377,
"tokens": 646950
},
{
"spider_id": "spider-162",
"name": "spider-162",
"kind": "queen",
"owner": null,
"burn_tx": null,
"pages": 336,
"tokens": 643737
},
{
"spider_id": "spider-120",
"name": "spider-120",
"kind": "queen",
"owner": null,
"burn_tx": null,
"pages": 350,
"tokens": 641282
},
{
"spider_id": "spider-159",
"name": "spider-159",
"kind": "queen",
"owner": null,
"burn_tx": null,
"pages": 372,
"tokens": 639210
},
{
"spider_id": "spider-128",
"name": "spider-128",
"kind": "queen",
"owner": null,
"burn_tx": null,
"pages": 379,
"tokens": 629538
},
{
"spider_id": "spider-163",
"name": "spider-163",
"kind": "queen",
"owner": null,
"burn_tx": null,
"pages": 375,
"tokens": 622387
},
{
"spider_id": "spider-137",
"name": "spider-137",
"kind": "queen",
"owner": null,
"burn_tx": null,
"pages": 382,
"tokens": 621373
},
{
"spider_id": "spider-155",
"name": "spider-155",
"kind": "queen",
"owner": null,
"burn_tx": null,
"pages": 361,
"tokens": 616713
},
{
"spider_id": "spider-136",
"name": "spider-136",
"kind": "queen",
"owner": null,
"burn_tx": null,
"pages": 359,
"tokens": 581199
},
{
"spider_id": "spider-173",
"name": "spider-173",
"kind": "queen",
"owner": null,
"burn_tx": null,
"pages": 352,
"tokens": 577298
},
{
"spider_id": "spider-131",
"name": "spider-131",
"kind": "queen",
"owner": null,
"burn_tx": null,
"pages": 366,
"tokens": 576070
},
{
"spider_id": "spider-112",
"name": "spider-112",
"kind": "queen",
"owner": null,
"burn_tx": null,
"pages": 355,
"tokens": 574960
},
{
"spider_id": "spider-111",
"name": "spider-111",
"kind": "queen",
"owner": null,
"burn_tx": null,
"pages": 307,
"tokens": 574400
},
{
"spider_id": "spider-165",
"name": "spider-165",
"kind": "queen",
"owner": null,
"burn_tx": null,
"pages": 297,
"tokens": 573245
},
{
"spider_id": "spider-177",
"name": "spider-177",
"kind": "queen",
"owner": null,
"burn_tx": null,
"pages": 338,
"tokens": 571467
},
{
"spider_id": "spider-147",
"name": "spider-147",
"kind": "queen",
"owner": null,
"burn_tx": null,
"pages": 305,
"tokens": 569516
},
{
"spider_id": "spider-171",
"name": "spider-171",
"kind": "queen",
"owner": null,
"burn_tx": null,
"pages": 309,
"tokens": 567054
},
{
"spider_id": "spider-115",
"name": "spider-115",
"kind": "queen",
"owner": null,
"burn_tx": null,
"pages": 341,
"tokens": 566272
},
{
"spider_id": "spider-141",
"name": "spider-141",
"kind": "queen",
"owner": null,
"burn_tx": null,
"pages": 375,
"tokens": 564651
},
{
"spider_id": "spider-124",
"name": "spider-124",
"kind": "queen",
"owner": null,
"burn_tx": null,
"pages": 324,
"tokens": 561483
},
{
"spider_id": "spider-130",
"name": "spider-130",
"kind": "queen",
"owner": null,
"burn_tx": null,
"pages": 346,
"tokens": 558800
},
{
"spider_id": "spider-133",
"name": "spider-133",
"kind": "queen",
"owner": null,
"burn_tx": null,
"pages": 344,
"tokens": 557752
},
{
"spider_id": "spider-142",
"name": "spider-142",
"kind": "queen",
"owner": null,
"burn_tx": null,
"pages": 344,
"tokens": 552839
},
{
"spider_id": "spider-117",
"name": "spider-117",
"kind": "queen",
"owner": null,
"burn_tx": null,
"pages": 361,
"tokens": 525406
},
{
"spider_id": "spider-183",
"name": "spider-183",
"kind": "queen",
"owner": null,
"burn_tx": null,
"pages": 317,
"tokens": 523941
},
{
"spider_id": "spider-174",
"name": "spider-174",
"kind": "queen",
"owner": null,
"burn_tx": null,
"pages": 273,
"tokens": 490409
},
{
"spider_id": "spider-181",
"name": "spider-181",
"kind": "queen",
"owner": null,
"burn_tx": null,
"pages": 257,
"tokens": 488999
},
{
"spider_id": "spider-169",
"name": "spider-169",
"kind": "queen",
"owner": null,
"burn_tx": null,
"pages": 284,
"tokens": 487042
},
{
"spider_id": "spider-152",
"name": "spider-152",
"kind": "queen",
"owner": null,
"burn_tx": null,
"pages": 336,
"tokens": 483309
},
{
"spider_id": "spider-164",
"name": "spider-164",
"kind": "queen",
"owner": null,
"burn_tx": null,
"pages": 295,
"tokens": 459644
},
{
"spider_id": "spider-182",
"name": "spider-182",
"kind": "queen",
"owner": null,
"burn_tx": null,
"pages": 308,
"tokens": 452045
},
{
"spider_id": "spider-175",
"name": "spider-175",
"kind": "queen",
"owner": null,
"burn_tx": null,
"pages": 244,
"tokens": 436415
},
{
"spider_id": "spider-168",
"name": "spider-168",
"kind": "queen",
"owner": null,
"burn_tx": null,
"pages": 309,
"tokens": 436316
},
{
"spider_id": "spider-179",
"name": "spider-179",
"kind": "queen",
"owner": null,
"burn_tx": null,
"pages": 267,
"tokens": 425945
},
{
"spider_id": "spider-172",
"name": "spider-172",
"kind": "queen",
"owner": null,
"burn_tx": null,
"pages": 254,
"tokens": 399802
},
{
"spider_id": "spider-170",
"name": "spider-170",
"kind": "queen",
"owner": null,
"burn_tx": null,
"pages": 246,
"tokens": 385536
},
{
"spider_id": "spider-178",
"name": "spider-178",
"kind": "queen",
"owner": null,
"burn_tx": null,
"pages": 260,
"tokens": 363115
},
{
"spider_id": "spider-189",
"name": "spider-189",
"kind": "queen",
"owner": null,
"burn_tx": null,
"pages": 121,
"tokens": 328039
},
{
"spider_id": "spider-191",
"name": "spider-191",
"kind": "queen",
"owner": null,
"burn_tx": null,
"pages": 125,
"tokens": 286249
},
{
"spider_id": "spider-188",
"name": "spider-188",
"kind": "queen",
"owner": null,
"burn_tx": null,
"pages": 145,
"tokens": 267474
},
{
"spider_id": "spider-185",
"name": "spider-185",
"kind": "queen",
"owner": null,
"burn_tx": null,
"pages": 152,
"tokens": 246111
},
{
"spider_id": "spider-184",
"name": "spider-184",
"kind": "queen",
"owner": null,
"burn_tx": null,
"pages": 178,
"tokens": 214114
},
{
"spider_id": "spider-186",
"name": "spider-186",
"kind": "queen",
"owner": null,
"burn_tx": null,
"pages": 140,
"tokens": 209374
},
{
"spider_id": "spider-187",
"name": "spider-187",
"kind": "queen",
"owner": null,
"burn_tx": null,
"pages": 113,
"tokens": 158597
},
{
"spider_id": "spider-193",
"name": "spider-193",
"kind": "queen",
"owner": null,
"burn_tx": null,
"pages": 75,
"tokens": 142691
},
{
"spider_id": "spider-190",
"name": "spider-190",
"kind": "queen",
"owner": null,
"burn_tx": null,
"pages": 85,
"tokens": 140273
},
{
"spider_id": "spider-194",
"name": "spider-194",
"kind": "queen",
"owner": null,
"burn_tx": null,
"pages": 50,
"tokens": 105080
},
{
"spider_id": "spider-192",
"name": "spider-192",
"kind": "queen",
"owner": null,
"burn_tx": null,
"pages": 48,
"tokens": 65896
},
{
"spider_id": "spider-195",
"name": "spider-195",
"kind": "queen",
"owner": null,
"burn_tx": null,
"pages": 29,
"tokens": 39995
}
],
"contributors_exported_at": null,
"recipe": {
"lang": true,
"url_filters": true,
"x_filter": true,
"wiki_tails": true,
"token_chunks": true,
"stream": true,
"records_forms": true,
"qa_decay": true,
"source_line": true,
"headings": "v31",
"pass_scale": 2.0,
"domain_cap": 0.05,
"decay_frac": 0.2,
"qa_share": 0.25,
"qa_max_heading_pairs": 100000,
"lid_threshold": 0.65,
"headings_min_edu": 2,
"headings_max_domain_share": 0.03,
"recipe": "v31",
"chunk_tokens": 2040,
"labels": null,
"shard_chars": 200000000,
"row_group": 256,
"bench_protocols": null,
"lid_model": "lid.176.bin"
},
"val": {
"source": "external held-out set",
"dataset": "heldout-q31",
"manifest_sha256": "7efdd2af22792322c436c80449fe6153083d59a95a0605ee4eab29459657d584"
},
"heldout_exclusions": {
"files": [
"exclude.json"
],
"urls": 2000,
"sha256": 2000,
"dropped": 2000
},
"records": {
"parsed": {
"chain": 120,
"investor": 8339,
"project": 29103,
"protocol": 6884,
"raises": 163,
"stablecoin": 119,
"token_facts": 230
},
"unparsed": {
"project": 51
},
"forms": {
"chain": 960,
"investor": 25017,
"project": 87270,
"protocol": 54366,
"raises": 1304,
"stablecoin": 952,
"token_facts": 1840
},
"qa_forms": 95325,
"leak_audit_dropped": {}
},
"stream": {
"pass_scale": 2.0,
"seed": 0,
"passes_table": {
"edu": {
"0": 0,
"1": 1,
"2": 2,
"3": 4,
"4": 6,
"5": 8
},
"good_min": 6,
"unlabeled": {
"good": 6,
"none": 2,
"crawl": 4,
"records": 4,
"form": 2
}
},
"domain_cap": 0.05,
"units": {
"page": 144359,
"records": 46508,
"form": 172975
},
"unique_tokens": {
"page": 172076164,
"records": 10794512,
"form": 10382023
},
"source_line": {
"on": true,
"plain_from": 0.9,
"units_with_source": 363842
},
"classes": {
"form": {
"units": 172975,
"occurrences": 691900,
"tokens": 41528092,
"passes_mean": 4.0
},
"good": {
"units": 25214,
"occurrences": 299656,
"tokens": 361754490,
"passes_mean": 11.885
},
"normal": {
"units": 29666,
"occurrences": 237328,
"tokens": 277729504,
"passes_mean": 8.0
},
"records": {
"units": 46508,
"occurrences": 372064,
"tokens": 86356096,
"passes_mean": 8.0
},
"weak": {
"units": 89479,
"occurrences": 357916,
"tokens": 427320188,
"passes_mean": 4.0
}
},
"top_sources": [
[
"records",
0.107
],
[
"en.wikipedia.org",
0.05
],
[
"forum.arbitrum.foundation",
0.0403
],
[
"github.com",
0.0247
],
[
"ethereum-magicians.org",
0.0216
],
[
"gov.optimism.io",
0.0211
],
[
"docs.chain.link",
0.0172
],
[
"polymarket.com",
0.016
],
[
"governance.aave.com",
0.0149
],
[
"solanacompass.com",
0.0148
],
[
"galaxy.com",
0.0145
],
[
"rapidinnovation.io",
0.0134
],
[
"cryptoslate.com",
0.0119
],
[
"gov.uniswap.org",
0.0116
],
[
"research.lido.fi",
0.0111
],
[
"ethresear.ch",
0.0096
],
[
"ethereum.org",
0.0094
],
[
"bitcoin.org",
0.0091
],
[
"forum.balancer.fi",
0.0085
],
[
"eips.ethereum.org",
0.0085
]
],
"capped_domains": [
{
"domain": "en.wikipedia.org",
"share_before": 0.0536,
"factor": 0.9284
}
],
"records_share": 0.107,
"forms_share": 0.0348,
"occurrences": 1958864,
"decay": {
"frac": 0.2,
"tokens_target": 254254181,
"qa": {
"pairs": 222270,
"docs": 120947,
"tokens": 54219003,
"target_tokens": 63563545,
"reps": 4,
"formats": {
"question_answer": 40240,
"faq": 40316,
"q_a": 40391
},
"short_of_target": true
},
"qa_tokens": 54980953,
"moved_tokens": 199273784,
"moved_good_tokens": 199273784,
"moved_documents": 481891,
"qa_share": 0.2162,
"documents": 602838,
"shards": 6
},
"tokens": {
"main": 1008433578,
"decay": 254254737,
"total": 1262688315
},
"documents": {
"main": 1476973,
"decay": 602838
},
"dirs": {
"main": "nanochat/stream/main",
"decay": "nanochat/stream/decay"
},
"qa_pairs": {
"records": 90597,
"faq": 51183,
"heading": 68002,
"page": 12488
}
},
"contamination": {
"bench_files": {
"arc_easy_test.json": {
"sha256": "76e55faa15bce1b6ef26212c0c67a242a39952088d3b9e1a4d6071ea33c15190",
"items": 2376
},
"cryptomc.json": {
"sha256": "c81b3367d654f1e207d2dcbb48f490ff5feb496e2d08a049d39fb8aa8de35f87",
"items": 150
},
"cryptoterms.json": {
"sha256": "918b257c12deac8818cebf271187eedd88f026f875908b5a2e6987434b991f90",
"items": 150
},
"ctslang.json": {
"sha256": "d967e9196888027a4055592381892fca6185ee3a41b70be6a7351fa06d0bfc70",
"items": 150
},
"projectid.json": {
"sha256": "6feec798ea7269d2931fba111c48b33af2030a885e4c589235072ca6e488f51e",
"items": 200
},
"projectid_xl.json": {
"sha256": "859b5fc0d6a197bdf38e09e980d4577bd32f321e4fb3aec674725b6e4ed9a709",
"items": 2593
},
"qa.json": {
"sha256": "d5f699bdec9dca94e355332380adf317ef497784a56d3d9277ca6c0f993af9c1",
"items": 100
}
},
"skipped_files": [
"._arc_easy_test.json",
"._cryptomc.json",
"._cryptoterms.json",
"._ctslang.json",
"._fresh_pages.json",
"._fresh_pages_log.json",
"._grade-0.json",
"._grade-1.json",
"._grade-2.json",
"._grade-3.json",
"._grade-4.json",
"._grade-key.json",
"._grade2-a.json",
"._grade2-b.json",
"._grade2-c.json",
"._grade2-grades.json",
"._grade2-key.json",
"._grade3-a.json",
"._grade3-b.json",
"._grade3-c.json",
"._grade3-grades.json",
"._grade3-key.json",
"._models.json",
"._projectid.json",
"._projectid_bench_protocols.json",
"._projectid_defillama_raw.json",
"._projectid_meta.json",
"._projectid_results.json",
"._projectid_results_detail.json",
"._projectid_split.json",
"._projectid_xl.json",
"._projectid_xl_meta.json",
"._qa.json",
"._qa_answers.json",
"._report_q31.json",
"._results.json",
"._results_ctslang.json",
"._results_ctslang_detail.json",
"._results_detail.json",
"._results_domain.json",
"._results_domain_detail.json",
"._smoke.json",
"._web_snapshot.json",
"fresh_pages.json",
"fresh_pages_log.json",
"grade-0.json",
"grade-1.json",
"grade-2.json",
"grade-3.json",
"grade-4.json",
"grade-key.json",
"grade2-a.json",
"grade2-b.json",
"grade2-c.json",
"grade2-grades.json",
"grade2-key.json",
"grade3-a.json",
"grade3-b.json",
"grade3-c.json",
"grade3-grades.json",
"grade3-key.json",
"models.json",
"projectid_bench_protocols.json",
"projectid_defillama_raw.json",
"projectid_meta.json",
"projectid_results.json",
"projectid_results_detail.json",
"projectid_split.json",
"projectid_xl_meta.json",
"qa_answers.json",
"report_q31.json",
"results.json",
"results_ctslang.json",
"results_ctslang_detail.json",
"results_detail.json",
"results_domain.json",
"results_domain_detail.json",
"smoke.json",
"web_snapshot.json"
],
"criteria": {
"ngram13": 13,
"ngram8": 8,
"ngram8_share": 0.7,
"question_min_words_in_text": 5,
"answer_within_words": 100
},
"dropped": {
"chunks": 0,
"records_forms": 0,
"qa_pairs": 753,
"headings.train": 1136,
"headings.val": 36
},
"checked": "on",
"by_criterion": {
"qa_pairs": {
"question": 752,
"ngram8_70": 3,
"question_answer": 1,
"ngram13": 1
},
"headings.train": {
"question": 1132,
"ngram13": 4,
"ngram8_70": 4,
"question_answer": 4
},
"headings.val": {
"question": 36
}
},
"by_bench": {
"qa_pairs": {
"qa.json": 751,
"cryptomc.json": 8,
"cryptoterms.json": 1
},
"headings.train": {
"qa.json": 1129,
"cryptomc.json": 16,
"cryptoterms.json": 3
},
"headings.val": {
"qa.json": 36
}
},
"bench_fingerprint": "656b4d7aa50c7d85e43d3915aae7dce568d54be7842219a97db529bbc349c125"
},
"labels": null,
"bench_protocols": {
"list": {
"file": "projectid_bench_protocols.json",
"sha256": "4e4f7eaae1e655f2ccc53de0b701ee20a0cea50be540024d6ec4cdeb6eef7471",
"keys": 2222,
"urls": 1922
},
"protocol_records": 6884,
"bench_records": 1707,
"synth_deny_list": {
"file": "stream_src/synth_deny.txt",
"urls": 1707,
"every_protocol": false
},
"qa_forms_left_out": 4725
},
"restreamed_at": "2026-10-07T01:32:05+00:00"
}