spencernorton commited on
Commit
bc1d09f
·
verified ·
1 Parent(s): 0da2731

Standardize Chart 2/3 labels and Per-Category Recall table/caption parity

Browse files

Remove 'Cheap' prefix in Chart 2, replace tc2000 with zero_cue_2000 in Chart 3 Panel 4 and tables, and sync SHA-256 provenance.

benchmarks/CHARTS_MANIFEST.json CHANGED
@@ -36,10 +36,10 @@
36
  ],
37
  "provenance_sha256": {
38
  "canonical_eval_vectors.json": "058473f4ba21bc07db2c2b6981d5e62f3c584311fc134d92cb523de7f261243a",
39
- "generate_canonical_charts.py": "409e9cd324b8e14acd8db58f46ebab9d0ea623e57cf0b7eabb2343f0a9237daa",
40
  "assets/armiger_vs_llms_overall_and_6dims.png": "0a0bc3a260b3c3b1188a92fb9cc015c6bdb786e9033924d9daa7a36be332fc62",
41
- "assets/armiger_vs_llms_cost_comparison.png": "3cf46fd59817d3bffafce1f4f5a15379ffef7bf779b8d4b40b866a8d9671eff4",
42
- "assets/armiger_vs_torrent_classifiers_6dims.png": "3768e7ec5bef503027375f87f089362798ab2ec74f96a7ea450458de325154dd",
43
  "assets/armiger_categorical_breakdown.png": "1a77a64340f7e05af318105d83270b29396450d01f22f71cffdf808b45e10d98",
44
  "validated_zero_cue_catalog.json": "a43193a7d9982f1b83e9e6968c47dbbe8d058b113af5020d6d5ea45ed2af7fcd",
45
  "armiger_ultra.fp32.onnx": "24c65e1af1be68c923bfdf774afb862c54b66a30f40c4b65cf5961929f149d48",
@@ -529,13 +529,13 @@
529
  }
530
  },
531
  "armiger_vs_llms_cost_comparison.png": {
532
- "sha256": "3cf46fd59817d3bffafce1f4f5a15379ffef7bf779b8d4b40b866a8d9671eff4",
533
  "models": [
534
  "Armiger (8.3 MB M4 Pro CPU INT8)",
535
  "Armiger Ultra (M4 Pro / RTX GPU FP32)",
536
  "Local 8B Instruct LLM (FP16)",
537
  "Google Gemini Flash",
538
- "Cheap OpenAI GPT (Fast Tier)",
539
  "Anthropic Claude Haiku 4.5"
540
  ],
541
  "batched_usd_per_1m": [
@@ -556,7 +556,7 @@
556
  ]
557
  },
558
  "armiger_vs_torrent_classifiers_6dims.png": {
559
- "sha256": "3768e7ec5bef503027375f87f089362798ab2ec74f96a7ea450458de325154dd",
560
  "panel_1_category_11class_oct15": {
561
  "panel_name": "chart3_1_oct15_cat11",
562
  "n": 2835,
@@ -997,8 +997,8 @@
997
  }
998
  }
999
  },
1000
- "panel_4_anime_dub_tc2000": {
1001
- "panel_name": "chart3_4_tc2000_anime_dub",
1002
  "n": 2000,
1003
  "metric_type": "mean",
1004
  "reference_model": "RTN (rank-torrent-name)",
@@ -1332,7 +1332,7 @@
1332
  },
1333
  "five_split_category_summary": {
1334
  "oct15_confirmation": {
1335
- "panel_name": "five_split_oct15",
1336
  "n": 2835,
1337
  "metric_type": "mean",
1338
  "reference_model": "armiger_ultra_hyb",
@@ -1380,7 +1380,7 @@
1380
  }
1381
  },
1382
  "fresh_confirmation": {
1383
- "panel_name": "five_split_fresh",
1384
  "n": 1402,
1385
  "metric_type": "mean",
1386
  "reference_model": "armiger_ultra_hyb",
@@ -1428,7 +1428,7 @@
1428
  }
1429
  },
1430
  "beyond_confirmation": {
1431
- "panel_name": "five_split_beyond",
1432
  "n": 1025,
1433
  "metric_type": "mean",
1434
  "reference_model": "armiger_ultra_hyb",
@@ -1476,7 +1476,7 @@
1476
  }
1477
  },
1478
  "zero_cue_2000": {
1479
- "panel_name": "five_split_tc2000",
1480
  "n": 2000,
1481
  "metric_type": "mean",
1482
  "reference_model": "armiger_ultra_hyb",
@@ -1524,7 +1524,7 @@
1524
  }
1525
  },
1526
  "multilang_3000": {
1527
- "panel_name": "five_split_tc3000",
1528
  "n": 3000,
1529
  "metric_type": "mean",
1530
  "reference_model": "armiger_ultra_hyb",
 
36
  ],
37
  "provenance_sha256": {
38
  "canonical_eval_vectors.json": "058473f4ba21bc07db2c2b6981d5e62f3c584311fc134d92cb523de7f261243a",
39
+ "generate_canonical_charts.py": "1f35073d35e7e7e82d6c3274ef2c29238424fa5885e36c2e68a86148e38edb78",
40
  "assets/armiger_vs_llms_overall_and_6dims.png": "0a0bc3a260b3c3b1188a92fb9cc015c6bdb786e9033924d9daa7a36be332fc62",
41
+ "assets/armiger_vs_llms_cost_comparison.png": "7344ba71b9041ac737296c98079c30bcae33c237c5b6549509596d6eb5c3ec52",
42
+ "assets/armiger_vs_torrent_classifiers_6dims.png": "51e1f2293aada2d42fc8d0feaef976c2b59eb5f648a27e5c32df041c1c7f3edc",
43
  "assets/armiger_categorical_breakdown.png": "1a77a64340f7e05af318105d83270b29396450d01f22f71cffdf808b45e10d98",
44
  "validated_zero_cue_catalog.json": "a43193a7d9982f1b83e9e6968c47dbbe8d058b113af5020d6d5ea45ed2af7fcd",
45
  "armiger_ultra.fp32.onnx": "24c65e1af1be68c923bfdf774afb862c54b66a30f40c4b65cf5961929f149d48",
 
529
  }
530
  },
531
  "armiger_vs_llms_cost_comparison.png": {
532
+ "sha256": "7344ba71b9041ac737296c98079c30bcae33c237c5b6549509596d6eb5c3ec52",
533
  "models": [
534
  "Armiger (8.3 MB M4 Pro CPU INT8)",
535
  "Armiger Ultra (M4 Pro / RTX GPU FP32)",
536
  "Local 8B Instruct LLM (FP16)",
537
  "Google Gemini Flash",
538
+ "OpenAI GPT (Fast Tier)",
539
  "Anthropic Claude Haiku 4.5"
540
  ],
541
  "batched_usd_per_1m": [
 
556
  ]
557
  },
558
  "armiger_vs_torrent_classifiers_6dims.png": {
559
+ "sha256": "51e1f2293aada2d42fc8d0feaef976c2b59eb5f648a27e5c32df041c1c7f3edc",
560
  "panel_1_category_11class_oct15": {
561
  "panel_name": "chart3_1_oct15_cat11",
562
  "n": 2835,
 
997
  }
998
  }
999
  },
1000
+ "panel_4_anime_dub_zero_cue_2000": {
1001
+ "panel_name": "chart3_4_zero_cue_2000_anime_dub",
1002
  "n": 2000,
1003
  "metric_type": "mean",
1004
  "reference_model": "RTN (rank-torrent-name)",
 
1332
  },
1333
  "five_split_category_summary": {
1334
  "oct15_confirmation": {
1335
+ "panel_name": "five_split_oct15_confirmation",
1336
  "n": 2835,
1337
  "metric_type": "mean",
1338
  "reference_model": "armiger_ultra_hyb",
 
1380
  }
1381
  },
1382
  "fresh_confirmation": {
1383
+ "panel_name": "five_split_fresh_confirmation",
1384
  "n": 1402,
1385
  "metric_type": "mean",
1386
  "reference_model": "armiger_ultra_hyb",
 
1428
  }
1429
  },
1430
  "beyond_confirmation": {
1431
+ "panel_name": "five_split_beyond_confirmation",
1432
  "n": 1025,
1433
  "metric_type": "mean",
1434
  "reference_model": "armiger_ultra_hyb",
 
1476
  }
1477
  },
1478
  "zero_cue_2000": {
1479
+ "panel_name": "five_split_zero_cue_2000",
1480
  "n": 2000,
1481
  "metric_type": "mean",
1482
  "reference_model": "armiger_ultra_hyb",
 
1524
  }
1525
  },
1526
  "multilang_3000": {
1527
+ "panel_name": "five_split_multilang_3000",
1528
  "n": 3000,
1529
  "metric_type": "mean",
1530
  "reference_model": "armiger_ultra_hyb",
benchmarks/generate_canonical_charts.py CHANGED
@@ -586,7 +586,7 @@ def build_all_charts_and_manifest(
586
  "Armiger Ultra (M4 Pro / RTX GPU FP32)",
587
  "Local 8B Instruct LLM (FP16)",
588
  "Google Gemini Flash",
589
- "Cheap OpenAI GPT (Fast Tier)",
590
  "Anthropic Claude Haiku 4.5",
591
  ]
592
  colors_cost = [C_INT8, C_ULTRA, C_VINCI, C_GEMINI, C_GPT, C_CLAUDE]
@@ -788,7 +788,7 @@ def build_all_charts_and_manifest(
788
  "Armiger Ultra (M4 Pro / RTX GPU FP32)",
789
  "Local 8B Instruct LLM (FP16)",
790
  "Google Gemini Flash",
791
- "Cheap OpenAI GPT (Fast Tier)",
792
  "Anthropic Claude Haiku 4.5",
793
  ]
794
  colors_cost = [C_INT8, C_ULTRA, C_VINCI, C_GEMINI, C_GPT, C_CLAUDE]
@@ -948,7 +948,7 @@ def build_all_charts_and_manifest(
948
 
949
  y_tc2k_ani = d["tc2000_ani_y_true"]
950
  c3_p4_tc2k_stats = compute_paired_panel_stats(
951
- "chart3_4_tc2000_anime_dub",
952
  {
953
  models_10[4]: (d["tc2000_ani_pred_rtn"] == y_tc2k_ani).astype(np.float32),
954
  models_10[5]: (d["tc2000_ani_pred_ptt"] == y_tc2k_ani).astype(np.float32),
@@ -1177,7 +1177,7 @@ def build_all_charts_and_manifest(
1177
  v = s_item["point_pct"]
1178
  ax4.barh(idx, v, height=0.58, color=colors_10[idx], zorder=3)
1179
  draw_ci_whiskers(ax4, np.array([idx]), [s_item])
1180
- ax4.text(max(v, s_item["ci95_hi"]) + 0.6, idx, f"{v:.2f}% (95.8% tc2000)", va="center", ha="left", fontsize=8.4, fontweight="bold", color=TEXT_MAIN)
1181
  elif m in c3_p4_tc2k_stats["models"]:
1182
  s_item = c3_p4_tc2k_stats["models"][m]
1183
  v = s_item["point_pct"]
@@ -1190,7 +1190,7 @@ def build_all_charts_and_manifest(
1190
  ax4.set_yticks(y10)
1191
  ax4.set_yticklabels(models_10, fontsize=9.5, fontweight="bold", color=TEXT_MAIN)
1192
  ax4.invert_yaxis()
1193
- ax4.set_xlim(45, 122)
1194
  ax4.set_xticks([50, 60, 70, 80, 90, 100])
1195
  ax4.set_xlabel("99.61% on oct15_confirmation (505/507); 95.80% on zero_cue_2000 (N=2,000)", fontsize=8.6, color=TEXT_SUB, labelpad=6)
1196
  ax4.set_title("4. Anime & Anime Dub Detection (%)\n(oct15_confirmation N = 507 & zero_cue_2000 N = 2,000)", fontsize=11, fontweight="bold", color=TEXT_MAIN, pad=8)
@@ -1433,7 +1433,7 @@ def build_all_charts_and_manifest(
1433
  ]:
1434
  y_sp = d[f"split5_{sp_code}_y_cat"]
1435
  sp_stats = compute_paired_panel_stats(
1436
- f"five_split_{sp_code}",
1437
  {
1438
  "armiger_ultra_hyb": (d[f"split5_{sp_code}_fp32_hyb_pred"] == y_sp).astype(np.float32),
1439
  "armiger_ultra_raw": (d[f"split5_{sp_code}_fp32_raw_pred"] == y_sp).astype(np.float32),
@@ -1506,7 +1506,7 @@ def build_all_charts_and_manifest(
1506
  "panel_2_catalog_matching_oct15": c3_p2_stats,
1507
  "panel_3_audio_strict_oct15": c3_p3_stats,
1508
  "panel_4_anime_oct15": c3_p4_oct_stats,
1509
- "panel_4_anime_dub_tc2000": c3_p4_tc2k_stats,
1510
  "panel_5_adult_oct15": c3_p5_stats,
1511
  "panel_6a_junk_oct15": c3_p6a_stats,
1512
  },
 
586
  "Armiger Ultra (M4 Pro / RTX GPU FP32)",
587
  "Local 8B Instruct LLM (FP16)",
588
  "Google Gemini Flash",
589
+ "OpenAI GPT (Fast Tier)",
590
  "Anthropic Claude Haiku 4.5",
591
  ]
592
  colors_cost = [C_INT8, C_ULTRA, C_VINCI, C_GEMINI, C_GPT, C_CLAUDE]
 
788
  "Armiger Ultra (M4 Pro / RTX GPU FP32)",
789
  "Local 8B Instruct LLM (FP16)",
790
  "Google Gemini Flash",
791
+ "OpenAI GPT (Fast Tier)",
792
  "Anthropic Claude Haiku 4.5",
793
  ]
794
  colors_cost = [C_INT8, C_ULTRA, C_VINCI, C_GEMINI, C_GPT, C_CLAUDE]
 
948
 
949
  y_tc2k_ani = d["tc2000_ani_y_true"]
950
  c3_p4_tc2k_stats = compute_paired_panel_stats(
951
+ "chart3_4_zero_cue_2000_anime_dub",
952
  {
953
  models_10[4]: (d["tc2000_ani_pred_rtn"] == y_tc2k_ani).astype(np.float32),
954
  models_10[5]: (d["tc2000_ani_pred_ptt"] == y_tc2k_ani).astype(np.float32),
 
1177
  v = s_item["point_pct"]
1178
  ax4.barh(idx, v, height=0.58, color=colors_10[idx], zorder=3)
1179
  draw_ci_whiskers(ax4, np.array([idx]), [s_item])
1180
+ ax4.text(max(v, s_item["ci95_hi"]) + 0.6, idx, f"{v:.2f}% (95.8% zero_cue_2000)", va="center", ha="left", fontsize=8.4, fontweight="bold", color=TEXT_MAIN)
1181
  elif m in c3_p4_tc2k_stats["models"]:
1182
  s_item = c3_p4_tc2k_stats["models"][m]
1183
  v = s_item["point_pct"]
 
1190
  ax4.set_yticks(y10)
1191
  ax4.set_yticklabels(models_10, fontsize=9.5, fontweight="bold", color=TEXT_MAIN)
1192
  ax4.invert_yaxis()
1193
+ ax4.set_xlim(45, 128)
1194
  ax4.set_xticks([50, 60, 70, 80, 90, 100])
1195
  ax4.set_xlabel("99.61% on oct15_confirmation (505/507); 95.80% on zero_cue_2000 (N=2,000)", fontsize=8.6, color=TEXT_SUB, labelpad=6)
1196
  ax4.set_title("4. Anime & Anime Dub Detection (%)\n(oct15_confirmation N = 507 & zero_cue_2000 N = 2,000)", fontsize=11, fontweight="bold", color=TEXT_MAIN, pad=8)
 
1433
  ]:
1434
  y_sp = d[f"split5_{sp_code}_y_cat"]
1435
  sp_stats = compute_paired_panel_stats(
1436
+ f"five_split_{sp_label}",
1437
  {
1438
  "armiger_ultra_hyb": (d[f"split5_{sp_code}_fp32_hyb_pred"] == y_sp).astype(np.float32),
1439
  "armiger_ultra_raw": (d[f"split5_{sp_code}_fp32_raw_pred"] == y_sp).astype(np.float32),
 
1506
  "panel_2_catalog_matching_oct15": c3_p2_stats,
1507
  "panel_3_audio_strict_oct15": c3_p3_stats,
1508
  "panel_4_anime_oct15": c3_p4_oct_stats,
1509
+ "panel_4_anime_dub_zero_cue_2000": c3_p4_tc2k_stats,
1510
  "panel_5_adult_oct15": c3_p5_stats,
1511
  "panel_6a_junk_oct15": c3_p6a_stats,
1512
  },
benchmarks/test_chart_integrity.py CHANGED
@@ -188,5 +188,5 @@ def test_readme_tables_match_charts_manifest():
188
  assert required_token in text, f"Missing canonical metric {required_token} in {readme_path}"
189
 
190
  # Stale pre-audit numbers that must never reappear in README
191
- for stale_token in ("97.13%", "96.35%", "72.35%", "35.29%", "97.55%", "true" + "spec", "tru" + "spec"):
192
  assert stale_token not in text, f"Stale metric {stale_token} found in {readme_path}"
 
188
  assert required_token in text, f"Missing canonical metric {required_token} in {readme_path}"
189
 
190
  # Stale pre-audit numbers that must never reappear in README
191
+ for stale_token in ("97.13%", "96.35%", "72.35%", "35.29%", "97.55%", "true" + "spec", "tru" + "spec", "tc" + "2000", "multilang" + "3000"):
192
  assert stale_token not in text, f"Stale metric {stale_token} found in {readme_path}"
docs/CHART_PIPELINE.md CHANGED
@@ -9,11 +9,11 @@ Every benchmark figure and headline accuracy table across the **Armiger** and **
9
  | Artifact | SHA-256 Digest | Description |
10
  | :--- | :--- | :--- |
11
  | `benchmarks/canonical_eval_vectors.json` | `058473f4ba21bc07db2c2b6981d5e62f3c584311fc134d92cb523de7f261243a` | Plain-text JSON containing `148` 1D numeric `int16`/`float32` prediction and ground-truth label arrays across all 5 evaluation splits (`N=10,262`). Contains **zero** strings, release titles, file paths, or infohashes. |
12
- | `benchmarks/generate_canonical_charts.py` | `409e9cd324b8e14acd8db58f46ebab9d0ea623e57cf0b7eabb2343f0a9237daa` | Deterministic chart generator and statistical test runner (`BOOTSTRAP_SEED = 20261018`, `BOOTSTRAP_RESAMPLES = 10000`). |
13
  | `benchmarks/CHARTS_MANIFEST.json` | *(Tracked by `test_chart_integrity.py`)* | Complete machine-readable statistical ledger recording point estimates, 95% paired bootstrap CIs, exact McNemar $p$-values, LOTO anti-circularity audits, and SHA-256 provenance hashes. |
14
  | `assets/armiger_vs_llms_overall_and_6dims.png` | `0a0bc3a260b3c3b1188a92fb9cc015c6bdb786e9033924d9daa7a36be332fc62` | 6-dimension accuracy & composite mean comparison against frontier cloud LLMs and local GPU LLMs on identical evaluation cohorts. |
15
- | `assets/armiger_vs_llms_cost_comparison.png` | `3cf46fd59817d3bffafce1f4f5a15379ffef7bf779b8d4b40b866a8d9671eff4` | Operational cost per 1M torrents and token/hardware rate comparison. |
16
- | `assets/armiger_vs_torrent_classifiers_6dims.png` | `3768e7ec5bef503027375f87f089362798ab2ec74f96a7ea450458de325154dd` | 7-dimension comparison against open-source torrent classifiers and regex parsers (`MIMMO BERT`, `RTN`, `Parsett`, `GuessIt`, `PTN`, `Anitomy`, `Bitmagnet`). |
17
  | `assets/armiger_categorical_breakdown.png` | `1a77a64340f7e05af318105d83270b29396450d01f22f71cffdf808b45e10d98` | Per-category recall breakdown across all 10 media & software categories on `oct15_confirmation` (`N=2,835`) and `beyond_confirmation` (`N=1,025`). |
18
 
19
  ---
 
9
  | Artifact | SHA-256 Digest | Description |
10
  | :--- | :--- | :--- |
11
  | `benchmarks/canonical_eval_vectors.json` | `058473f4ba21bc07db2c2b6981d5e62f3c584311fc134d92cb523de7f261243a` | Plain-text JSON containing `148` 1D numeric `int16`/`float32` prediction and ground-truth label arrays across all 5 evaluation splits (`N=10,262`). Contains **zero** strings, release titles, file paths, or infohashes. |
12
+ | `benchmarks/generate_canonical_charts.py` | `1f35073d35e7e7e82d6c3274ef2c29238424fa5885e36c2e68a86148e38edb78` | Deterministic chart generator and statistical test runner (`BOOTSTRAP_SEED = 20261018`, `BOOTSTRAP_RESAMPLES = 10000`). |
13
  | `benchmarks/CHARTS_MANIFEST.json` | *(Tracked by `test_chart_integrity.py`)* | Complete machine-readable statistical ledger recording point estimates, 95% paired bootstrap CIs, exact McNemar $p$-values, LOTO anti-circularity audits, and SHA-256 provenance hashes. |
14
  | `assets/armiger_vs_llms_overall_and_6dims.png` | `0a0bc3a260b3c3b1188a92fb9cc015c6bdb786e9033924d9daa7a36be332fc62` | 6-dimension accuracy & composite mean comparison against frontier cloud LLMs and local GPU LLMs on identical evaluation cohorts. |
15
+ | `assets/armiger_vs_llms_cost_comparison.png` | `7344ba71b9041ac737296c98079c30bcae33c237c5b6549509596d6eb5c3ec52` | Operational cost per 1M torrents and token/hardware rate comparison. |
16
+ | `assets/armiger_vs_torrent_classifiers_6dims.png` | `51e1f2293aada2d42fc8d0feaef976c2b59eb5f648a27e5c32df041c1c7f3edc` | 7-dimension comparison against open-source torrent classifiers and regex parsers (`MIMMO BERT`, `RTN`, `Parsett`, `GuessIt`, `PTN`, `Anitomy`, `Bitmagnet`). |
17
  | `assets/armiger_categorical_breakdown.png` | `1a77a64340f7e05af318105d83270b29396450d01f22f71cffdf808b45e10d98` | Per-category recall breakdown across all 10 media & software categories on `oct15_confirmation` (`N=2,835`) and `beyond_confirmation` (`N=1,025`). |
18
 
19
  ---
docs/base-model.md CHANGED
@@ -21,7 +21,7 @@
21
 
22
  ![Armiger vs Open-Source Torrent Classifiers Across 7 Core Dimensions](../assets/armiger_vs_torrent_classifiers_6dims.png)
23
 
24
- ![Categorical Accuracy Breakdown Across All 10 Media & Software Categories](../assets/armiger_categorical_breakdown.png)
25
 
26
  ---
27
 
 
21
 
22
  ![Armiger vs Open-Source Torrent Classifiers Across 7 Core Dimensions](../assets/armiger_vs_torrent_classifiers_6dims.png)
23
 
24
+ ![Per-Category Recall Breakdown Across All 10 Media & Software Categories](../assets/armiger_categorical_breakdown.png)
25
 
26
  ---
27