{ "schema_version": "1.0.0", "bootstrap_seed": 20261018, "bootstrap_resamples": 10000, "invariants_verified": { "identical_denominator_lock": true, "anti_circularity_lock": true, "paired_uncertainty_and_significance": true, "png_metadata_stripped_ihdr_idat_iend_only": true }, "anti_circularity_audit": [ { "model_name": "OpenAI GPT (Fast Tier)", "is_teacher": true, "eval_mode": "leave_one_teacher_out", "self_vote_weight": 0.0, "non_circular_pct": 95.12, "circular_self_vote_pct": 96.59, "circularity_inflation_pp": 1.46 }, { "model_name": "Google Gemini Flash", "is_teacher": true, "eval_mode": "leave_one_teacher_out", "self_vote_weight": 0.0, "non_circular_pct": 93.8, "circular_self_vote_pct": 95.61, "circularity_inflation_pp": 1.8 }, { "model_name": "Anthropic Claude Haiku 4.5", "is_teacher": true, "eval_mode": "independent_ground_truth", "self_vote_weight": 0.0 } ], "provenance_sha256": { "canonical_eval_vectors.json": "058473f4ba21bc07db2c2b6981d5e62f3c584311fc134d92cb523de7f261243a", "generate_canonical_charts.py": "1f35073d35e7e7e82d6c3274ef2c29238424fa5885e36c2e68a86148e38edb78", "assets/armiger_vs_llms_overall_and_6dims.png": "0a0bc3a260b3c3b1188a92fb9cc015c6bdb786e9033924d9daa7a36be332fc62", "assets/armiger_vs_llms_cost_comparison.png": "7344ba71b9041ac737296c98079c30bcae33c237c5b6549509596d6eb5c3ec52", "assets/armiger_vs_torrent_classifiers_6dims.png": "51e1f2293aada2d42fc8d0feaef976c2b59eb5f648a27e5c32df041c1c7f3edc", "assets/armiger_categorical_breakdown.png": "1a77a64340f7e05af318105d83270b29396450d01f22f71cffdf808b45e10d98", "validated_zero_cue_catalog.json": "a43193a7d9982f1b83e9e6968c47dbbe8d058b113af5020d6d5ea45ed2af7fcd", "armiger_ultra.fp32.onnx": "24c65e1af1be68c923bfdf774afb862c54b66a30f40c4b65cf5961929f149d48", "armiger.int8.onnx": "c750e034aace4ff28c0eb1e1112b28f744a5770abd78618127258f283e54e4da" }, "charts": { "armiger_vs_llms_overall_and_6dims.png": { "sha256": "0a0bc3a260b3c3b1188a92fb9cc015c6bdb786e9033924d9daa7a36be332fc62", "composite_6dim_mean": { "Armiger Ultra (Fair Hybrid)": { "point_pct": 96.86, "ci95_lo": 96.35, "ci95_hi": 97.38, "point_with_corpus_swarm_pct": 97.6 }, "Armiger (8.3 MB INT8 + Plugins)": { "point_pct": 96.85, "ci95_lo": 96.33, "ci95_hi": 97.36 }, "Google Gemini Flash": { "point_pct": 94.56, "ci95_lo": 93.88, "ci95_hi": 95.23 }, "Anthropic Claude Haiku 4.5": { "point_pct": 92.88, "ci95_lo": 92.04, "ci95_hi": 93.72 }, "OpenAI GPT (Fast Tier)": { "point_pct": 86.24, "ci95_lo": 85.56, "ci95_hi": 86.91 }, "Local 8B Instruct LLM (FP16)": { "point_pct": 84.79, "ci95_lo": 83.65, "ci95_hi": 85.94 } }, "panel_2a_category_11class": { "panel_name": "chart1_2a_beyond_cat11", "n": 1025, "metric_type": "mean", "reference_model": "Armiger Ultra (Fair Hybrid)", "models": { "Armiger Ultra (Fair Hybrid)": { "n": 1025, "point_pct": 95.12, "ci95_lo": 93.76, "ci95_hi": 96.39, "delta_vs_ref_pct": 0.0, "delta_ci95_lo": 0.0, "delta_ci95_hi": 0.0, "p_value_vs_ref": 1.0 }, "Armiger (8.3 MB INT8 + Plugins)": { "n": 1025, "point_pct": 95.12, "ci95_lo": 93.76, "ci95_hi": 96.39, "delta_vs_ref_pct": 0.0, "delta_ci95_lo": 0.0, "delta_ci95_hi": 0.0, "p_value_vs_ref": 1.0 }, "Google Gemini Flash": { "n": 1025, "point_pct": 93.8, "ci95_lo": 92.39, "ci95_hi": 95.12, "delta_vs_ref_pct": 1.32, "delta_ci95_lo": -0.29, "delta_ci95_hi": 2.93, "p_value_vs_ref": 0.1073516 }, "Anthropic Claude Haiku 4.5": { "n": 1025, "point_pct": 93.76, "ci95_lo": 92.2, "ci95_hi": 95.22, "delta_vs_ref_pct": 1.37, "delta_ci95_lo": 0.68, "delta_ci95_hi": 2.15, "p_value_vs_ref": 0.0001220703 }, "OpenAI GPT (Fast Tier)": { "n": 1025, "point_pct": 95.12, "ci95_lo": 93.85, "ci95_hi": 96.29, "delta_vs_ref_pct": 0.0, "delta_ci95_lo": -1.46, "delta_ci95_hi": 1.41, "p_value_vs_ref": 0.5715867 }, "Local 8B Instruct LLM (FP16)": { "n": 1025, "point_pct": 80.78, "ci95_lo": 78.34, "ci95_hi": 83.22, "delta_vs_ref_pct": 14.34, "delta_ci95_lo": 12.2, "delta_ci95_hi": 16.49, "p_value_vs_ref": 1.121039e-44 } } }, "panel_2b_catalog_matching": { "panel_name": "chart1_2b_beyond_matching", "n": 1025, "metric_type": "mean", "reference_model": "Armiger Ultra (Fair Hybrid)", "models": { "Armiger Ultra (Fair Hybrid)": { "n": 1025, "point_pct": 96.39, "ci95_lo": 95.22, "ci95_hi": 97.46, "delta_vs_ref_pct": 0.0, "delta_ci95_lo": 0.0, "delta_ci95_hi": 0.0, "p_value_vs_ref": 1.0 }, "Armiger (8.3 MB INT8 + Plugins)": { "n": 1025, "point_pct": 96.29, "ci95_lo": 95.02, "ci95_hi": 97.37, "delta_vs_ref_pct": 0.1, "delta_ci95_lo": -0.69, "delta_ci95_hi": 0.98, "p_value_vs_ref": 1.0 }, "Google Gemini Flash": { "n": 1025, "point_pct": 90.88, "ci95_lo": 89.17, "ci95_hi": 92.49, "delta_vs_ref_pct": 5.51, "delta_ci95_lo": 3.71, "delta_ci95_hi": 7.32, "p_value_vs_ref": 3.767158e-09 }, "Anthropic Claude Haiku 4.5": { "n": 1025, "point_pct": 86.15, "ci95_lo": 84.0, "ci95_hi": 88.2, "delta_vs_ref_pct": 10.24, "delta_ci95_lo": 8.39, "delta_ci95_hi": 12.1, "p_value_vs_ref": 4.930381e-32 }, "OpenAI GPT (Fast Tier)": { "n": 1025, "point_pct": 89.07, "ci95_lo": 87.17, "ci95_hi": 90.88, "delta_vs_ref_pct": 7.32, "delta_ci95_lo": 5.37, "delta_ci95_hi": 9.32, "p_value_vs_ref": 2.797646e-13 }, "Local 8B Instruct LLM (FP16)": { "n": 1025, "point_pct": 77.85, "ci95_lo": 75.32, "ci95_hi": 80.29, "delta_vs_ref_pct": 18.54, "delta_ci95_lo": 16.29, "delta_ci95_hi": 20.98, "p_value_vs_ref": 1.274474e-57 } } }, "panel_2c_audio_strict_229": { "panel_name": "chart1_2c_beyond_audio_strict", "n": 229, "metric_type": "mean", "reference_model": "Armiger Ultra (Fair Hybrid)", "models": { "Armiger Ultra (Fair Hybrid)": { "n": 229, "point_pct": 99.13, "ci95_lo": 97.82, "ci95_hi": 100.0, "delta_vs_ref_pct": 0.0, "delta_ci95_lo": 0.0, "delta_ci95_hi": 0.0, "p_value_vs_ref": 1.0 }, "Armiger (8.3 MB INT8 + Plugins)": { "n": 229, "point_pct": 99.13, "ci95_lo": 97.82, "ci95_hi": 100.0, "delta_vs_ref_pct": 0.0, "delta_ci95_lo": 0.0, "delta_ci95_hi": 0.0, "p_value_vs_ref": 1.0 }, "Google Gemini Flash": { "n": 229, "point_pct": 98.69, "ci95_lo": 96.94, "ci95_hi": 100.0, "delta_vs_ref_pct": 0.44, "delta_ci95_lo": -1.31, "delta_ci95_hi": 2.18, "p_value_vs_ref": 1.0 }, "Anthropic Claude Haiku 4.5": { "n": 229, "point_pct": 95.63, "ci95_lo": 92.58, "ci95_hi": 98.25, "delta_vs_ref_pct": 3.49, "delta_ci95_lo": 1.31, "delta_ci95_hi": 6.11, "p_value_vs_ref": 0.0078125 }, "OpenAI GPT (Fast Tier)": { "n": 229, "point_pct": 97.82, "ci95_lo": 95.63, "ci95_hi": 99.56, "delta_vs_ref_pct": 1.31, "delta_ci95_lo": 0.0, "delta_ci95_hi": 3.06, "p_value_vs_ref": 0.25 }, "Local 8B Instruct LLM (FP16)": { "n": 229, "point_pct": 93.45, "ci95_lo": 89.96, "ci95_hi": 96.51, "delta_vs_ref_pct": 5.68, "delta_ci95_lo": 3.06, "delta_ci95_hi": 8.73, "p_value_vs_ref": 0.0002441406 } } }, "panel_2d_anime_valid_636": { "panel_name": "chart1_2d_beyond_anime_valid636", "n": 636, "metric_type": "mean", "reference_model": "Armiger Ultra (Fair Hybrid)", "models": { "Armiger Ultra (Fair Hybrid)": { "n": 636, "point_pct": 98.9, "ci95_lo": 97.96, "ci95_hi": 99.69, "delta_vs_ref_pct": 0.0, "delta_ci95_lo": 0.0, "delta_ci95_hi": 0.0, "p_value_vs_ref": 1.0 }, "Armiger (8.3 MB INT8 + Plugins)": { "n": 636, "point_pct": 98.9, "ci95_lo": 97.96, "ci95_hi": 99.69, "delta_vs_ref_pct": 0.0, "delta_ci95_lo": 0.0, "delta_ci95_hi": 0.0, "p_value_vs_ref": 1.0 }, "Google Gemini Flash": { "n": 636, "point_pct": 95.6, "ci95_lo": 93.87, "ci95_hi": 97.17, "delta_vs_ref_pct": 3.3, "delta_ci95_lo": 1.73, "delta_ci95_hi": 5.03, "p_value_vs_ref": 4.923344e-05 }, "Anthropic Claude Haiku 4.5": { "n": 636, "point_pct": 98.9, "ci95_lo": 97.96, "ci95_hi": 99.69, "delta_vs_ref_pct": 0.0, "delta_ci95_lo": 0.0, "delta_ci95_hi": 0.0, "p_value_vs_ref": 1.0 }, "OpenAI GPT (Fast Tier)": { "n": 636, "point_pct": 95.6, "ci95_lo": 93.87, "ci95_hi": 97.17, "delta_vs_ref_pct": 3.3, "delta_ci95_lo": 1.73, "delta_ci95_hi": 5.03, "p_value_vs_ref": 4.923344e-05 }, "Local 8B Instruct LLM (FP16)": { "n": 636, "point_pct": 92.45, "ci95_lo": 90.25, "ci95_hi": 94.5, "delta_vs_ref_pct": 6.45, "delta_ci95_lo": 4.56, "delta_ci95_hi": 8.33, "p_value_vs_ref": 9.094947e-13 } } }, "panel_2e_junk_balanced_977": { "panel_name": "chart1_2e_beyond_junk977_balanced", "n": 977, "metric_type": "balanced", "reference_model": "Armiger Ultra (Fair Hybrid)", "models": { "Armiger Ultra (Fair Hybrid)": { "n": 977, "point_pct": 99.23, "ci95_lo": 97.69, "ci95_hi": 100.0, "delta_vs_ref_pct": 0.0, "delta_ci95_lo": 0.0, "delta_ci95_hi": 0.0, "p_value_vs_ref": 1.0 }, "Armiger (8.3 MB INT8 + Plugins)": { "n": 977, "point_pct": 99.23, "ci95_lo": 97.69, "ci95_hi": 100.0, "delta_vs_ref_pct": 0.0, "delta_ci95_lo": 0.0, "delta_ci95_hi": 0.0, "p_value_vs_ref": 1.0 }, "Google Gemini Flash": { "n": 977, "point_pct": 99.18, "ci95_lo": 97.53, "ci95_hi": 100.0, "delta_vs_ref_pct": 0.05, "delta_ci95_lo": -2.25, "delta_ci95_hi": 2.36, "p_value_vs_ref": 1.0 }, "Anthropic Claude Haiku 4.5": { "n": 977, "point_pct": 93.09, "ci95_lo": 90.56, "ci95_hi": 95.12, "delta_vs_ref_pct": 6.14, "delta_ci95_lo": 3.39, "delta_ci95_hi": 9.0, "p_value_vs_ref": 8.046381e-29 }, "OpenAI GPT (Fast Tier)": { "n": 977, "point_pct": 50.0, "ci95_lo": 50.0, "ci95_hi": 50.0, "delta_vs_ref_pct": 49.23, "delta_ci95_lo": 47.69, "delta_ci95_hi": 50.0, "p_value_vs_ref": 1.084202e-19 }, "Local 8B Instruct LLM (FP16)": { "n": 977, "point_pct": 97.91, "ci95_lo": 95.5, "ci95_hi": 99.62, "delta_vs_ref_pct": 1.32, "delta_ci95_lo": -1.15, "delta_ci95_hi": 4.01, "p_value_vs_ref": 0.003417969 } } }, "panel_2f_swarm_public_757": { "panel_name": "chart1_2f_swarm_public_757", "n": 757, "metric_type": "auroc", "reference_model": "Armiger Ultra (Fair Hybrid)", "models": { "Armiger Ultra (Fair Hybrid)": { "n": 757, "point_pct": 92.42, "ci95_lo": 90.55, "ci95_hi": 94.13, "delta_vs_ref_pct": 0.0, "delta_ci95_lo": 0.0, "delta_ci95_hi": 0.0, "p_value_vs_ref": 1.0 }, "Armiger (8.3 MB INT8 + Plugins)": { "n": 757, "point_pct": 92.42, "ci95_lo": 90.55, "ci95_hi": 94.13, "delta_vs_ref_pct": 0.0, "delta_ci95_lo": 0.0, "delta_ci95_hi": 0.0, "p_value_vs_ref": 1.0 }, "Google Gemini Flash": { "n": 757, "point_pct": 89.2, "ci95_lo": 86.89, "ci95_hi": 91.42, "delta_vs_ref_pct": 3.22, "delta_ci95_lo": 0.32, "delta_ci95_hi": 6.1, "p_value_vs_ref": 2.227484e-06 }, "Anthropic Claude Haiku 4.5": { "n": 757, "point_pct": 89.76, "ci95_lo": 87.53, "ci95_hi": 91.94, "delta_vs_ref_pct": 2.66, "delta_ci95_lo": -0.16, "delta_ci95_hi": 5.47, "p_value_vs_ref": 4.115658e-10 }, "OpenAI GPT (Fast Tier)": { "n": 757, "point_pct": 89.82, "ci95_lo": 87.51, "ci95_hi": 91.93, "delta_vs_ref_pct": 2.6, "delta_ci95_lo": -0.26, "delta_ci95_hi": 5.47, "p_value_vs_ref": 6.473227e-07 }, "Local 8B Instruct LLM (FP16)": { "n": 757, "point_pct": 66.32, "ci95_lo": 62.34, "ci95_hi": 70.17, "delta_vs_ref_pct": 26.1, "delta_ci95_lo": 21.73, "delta_ci95_hi": 30.6, "p_value_vs_ref": 1.035307e-81 } } }, "panel_2f_swarm_warehouse_10278": { "panel_name": "chart1_2f_swarm_warehouse_10278", "n": 10278, "metric_type": "auroc", "reference_model": "Armiger Ultra (Fair Hybrid)", "models": { "Armiger Ultra (Fair Hybrid)": { "n": 10278, "point_pct": 96.83, "ci95_lo": 96.52, "ci95_hi": 97.12, "delta_vs_ref_pct": 0.0, "delta_ci95_lo": 0.0, "delta_ci95_hi": 0.0, "p_value_vs_ref": 1.0 }, "Armiger 4-Way Priors": { "n": 10278, "point_pct": 86.64, "ci95_lo": 85.91, "ci95_hi": 87.36, "delta_vs_ref_pct": 10.19, "delta_ci95_lo": 9.39, "delta_ci95_hi": 10.99, "p_value_vs_ref": 0.0 }, "Raw Seeder Count": { "n": 10278, "point_pct": 53.02, "ci95_lo": 51.88, "ci95_hi": 54.22, "delta_vs_ref_pct": 43.81, "delta_ci95_lo": 42.57, "delta_ci95_hi": 45.0, "p_value_vs_ref": 0.0 } } } }, "armiger_vs_llms_cost_comparison.png": { "sha256": "7344ba71b9041ac737296c98079c30bcae33c237c5b6549509596d6eb5c3ec52", "models": [ "Armiger (8.3 MB M4 Pro CPU INT8)", "Armiger Ultra (M4 Pro / RTX GPU FP32)", "Local 8B Instruct LLM (FP16)", "Google Gemini Flash", "OpenAI GPT (Fast Tier)", "Anthropic Claude Haiku 4.5" ], "batched_usd_per_1m": [ 0.002, 0.004, 1.85, 94.9, 127.3, 712.0 ], "realtime_usd_per_1m": [ 0.002, 0.004, 18.0, 147.0, 220.5, 1220.0 ] }, "armiger_vs_torrent_classifiers_6dims.png": { "sha256": "51e1f2293aada2d42fc8d0feaef976c2b59eb5f648a27e5c32df041c1c7f3edc", "panel_1_category_11class_oct15": { "panel_name": "chart3_1_oct15_cat11", "n": 2835, "metric_type": "mean", "reference_model": "Armiger Ultra (Fair Hybrid)", "models": { "Armiger Ultra (Fair Hybrid)": { "n": 2835, "point_pct": 95.49, "ci95_lo": 94.71, "ci95_hi": 96.23, "delta_vs_ref_pct": 0.0, "delta_ci95_lo": 0.0, "delta_ci95_hi": 0.0, "p_value_vs_ref": 1.0 }, "Armiger (8.3 MB INT8 + Plugins)": { "n": 2835, "point_pct": 95.49, "ci95_lo": 94.71, "ci95_hi": 96.23, "delta_vs_ref_pct": 0.0, "delta_ci95_lo": 0.0, "delta_ci95_hi": 0.0, "p_value_vs_ref": 1.0 }, "MIMMO BERT (360M)": { "n": 2835, "point_pct": 72.24, "ci95_lo": 70.58, "ci95_hi": 73.86, "delta_vs_ref_pct": 23.25, "delta_ci95_lo": 21.62, "delta_ci95_hi": 24.9, "p_value_vs_ref": 2.797454e-162 }, "MIMMO Cascade": { "n": 2835, "point_pct": 71.32, "ci95_lo": 69.63, "ci95_hi": 72.95, "delta_vs_ref_pct": 24.16, "delta_ci95_lo": 22.5, "delta_ci95_hi": 25.86, "p_value_vs_ref": 7.860859e-169 }, "RTN (rank-torrent-name)": { "n": 2835, "point_pct": 53.58, "ci95_lo": 51.78, "ci95_hi": 55.41, "delta_vs_ref_pct": 41.9, "delta_ci95_lo": 40.07, "delta_ci95_hi": 43.77, "p_value_vs_ref": 0.0 }, "PTT / Parsett": { "n": 2835, "point_pct": 53.58, "ci95_lo": 51.78, "ci95_hi": 55.41, "delta_vs_ref_pct": 41.9, "delta_ci95_lo": 40.07, "delta_ci95_hi": 43.77, "p_value_vs_ref": 0.0 }, "GuessIt": { "n": 2835, "point_pct": 38.69, "ci95_lo": 36.93, "ci95_hi": 40.46, "delta_vs_ref_pct": 56.79, "delta_ci95_lo": 54.96, "delta_ci95_hi": 58.62, "p_value_vs_ref": 0.0 }, "PTN (parse-torrent-title)": { "n": 2835, "point_pct": 35.59, "ci95_lo": 33.83, "ci95_hi": 37.35, "delta_vs_ref_pct": 59.89, "delta_ci95_lo": 58.06, "delta_ci95_hi": 61.73, "p_value_vs_ref": 0.0 }, "Anitomy": { "n": 2835, "point_pct": 33.4, "ci95_lo": 31.68, "ci95_hi": 35.13, "delta_vs_ref_pct": 62.08, "delta_ci95_lo": 60.25, "delta_ci95_hi": 63.88, "p_value_vs_ref": 0.0 }, "Bitmagnet Built-In": { "n": 2835, "point_pct": 35.17, "ci95_lo": 33.47, "ci95_hi": 36.9, "delta_vs_ref_pct": 60.32, "delta_ci95_lo": 58.52, "delta_ci95_hi": 62.12, "p_value_vs_ref": 0.0 } } }, "panel_1_video_mvtv_oct15": { "panel_name": "chart3_1_oct15_video_mvtv", "n": 1296, "metric_type": "mean", "reference_model": "Armiger Ultra (Fair Hybrid)", "models": { "Armiger Ultra (Fair Hybrid)": { "n": 1296, "point_pct": 97.3, "ci95_lo": 96.37, "ci95_hi": 98.15, "delta_vs_ref_pct": 0.0, "delta_ci95_lo": 0.0, "delta_ci95_hi": 0.0, "p_value_vs_ref": 1.0 }, "Armiger (8.3 MB INT8 + Plugins)": { "n": 1296, "point_pct": 97.3, "ci95_lo": 96.37, "ci95_hi": 98.15, "delta_vs_ref_pct": 0.0, "delta_ci95_lo": 0.0, "delta_ci95_hi": 0.0, "p_value_vs_ref": 1.0 }, "MIMMO BERT (360M)": { "n": 1296, "point_pct": 76.0, "ci95_lo": 73.61, "ci95_hi": 78.32, "delta_vs_ref_pct": 21.3, "delta_ci95_lo": 18.9, "delta_ci95_hi": 23.77, "p_value_vs_ref": 1.427447e-63 }, "MIMMO Cascade": { "n": 1296, "point_pct": 76.08, "ci95_lo": 73.69, "ci95_hi": 78.4, "delta_vs_ref_pct": 21.22, "delta_ci95_lo": 18.83, "delta_ci95_hi": 23.69, "p_value_vs_ref": 2.667039e-63 }, "RTN (rank-torrent-name)": { "n": 1296, "point_pct": 81.64, "ci95_lo": 79.48, "ci95_hi": 83.72, "delta_vs_ref_pct": 15.66, "delta_ci95_lo": 13.5, "delta_ci95_hi": 17.82, "p_value_vs_ref": 4.447751e-45 }, "PTT / Parsett": { "n": 1296, "point_pct": 81.64, "ci95_lo": 79.48, "ci95_hi": 83.72, "delta_vs_ref_pct": 15.66, "delta_ci95_lo": 13.5, "delta_ci95_hi": 17.82, "p_value_vs_ref": 4.447751e-45 }, "GuessIt": { "n": 1296, "point_pct": 84.65, "ci95_lo": 82.64, "ci95_hi": 86.57, "delta_vs_ref_pct": 12.65, "delta_ci95_lo": 10.65, "delta_ci95_hi": 14.66, "p_value_vs_ref": 2.56753e-35 }, "PTN (parse-torrent-title)": { "n": 1296, "point_pct": 77.85, "ci95_lo": 75.54, "ci95_hi": 80.09, "delta_vs_ref_pct": 19.44, "delta_ci95_lo": 17.05, "delta_ci95_hi": 21.84, "p_value_vs_ref": 1.163065e-57 }, "Anitomy": { "n": 1296, "point_pct": 73.07, "ci95_lo": 70.6, "ci95_hi": 75.46, "delta_vs_ref_pct": 24.23, "delta_ci95_lo": 21.68, "delta_ci95_hi": 26.77, "p_value_vs_ref": 5.732874e-76 }, "Bitmagnet Built-In": { "n": 1296, "point_pct": 43.21, "ci95_lo": 40.51, "ci95_hi": 45.91, "delta_vs_ref_pct": 54.09, "delta_ci95_lo": 51.23, "delta_ci95_hi": 56.87, "p_value_vs_ref": 8.363314e-201 } } }, "panel_2_catalog_matching_oct15": { "panel_name": "chart3_2_oct15_catalog_matching", "n": 1296, "metric_type": "mean", "reference_model": "Armiger Ultra (Fair Hybrid)", "models": { "Armiger Ultra (Fair Hybrid)": { "n": 1296, "point_pct": 96.53, "ci95_lo": 95.52, "ci95_hi": 97.53, "delta_vs_ref_pct": 0.0, "delta_ci95_lo": 0.0, "delta_ci95_hi": 0.0, "p_value_vs_ref": 1.0 }, "Armiger (8.3 MB INT8 + Plugins)": { "n": 1296, "point_pct": 96.53, "ci95_lo": 95.52, "ci95_hi": 97.53, "delta_vs_ref_pct": 0.0, "delta_ci95_lo": 0.0, "delta_ci95_hi": 0.0, "p_value_vs_ref": 1.0 }, "MIMMO BERT (360M)": { "n": 1296, "point_pct": 60.26, "ci95_lo": 57.56, "ci95_hi": 62.96, "delta_vs_ref_pct": 36.27, "delta_ci95_lo": 33.49, "delta_ci95_hi": 39.12, "p_value_vs_ref": 1.578863e-117 }, "MIMMO Cascade": { "n": 1296, "point_pct": 60.34, "ci95_lo": 57.56, "ci95_hi": 62.96, "delta_vs_ref_pct": 36.19, "delta_ci95_lo": 33.33, "delta_ci95_hi": 39.04, "p_value_vs_ref": 3.028483e-117 }, "RTN (rank-torrent-name)": { "n": 1296, "point_pct": 80.48, "ci95_lo": 78.24, "ci95_hi": 82.64, "delta_vs_ref_pct": 16.05, "delta_ci95_lo": 13.81, "delta_ci95_hi": 18.29, "p_value_vs_ref": 7.158345e-45 }, "PTT / Parsett": { "n": 1296, "point_pct": 80.48, "ci95_lo": 78.24, "ci95_hi": 82.64, "delta_vs_ref_pct": 16.05, "delta_ci95_lo": 13.81, "delta_ci95_hi": 18.29, "p_value_vs_ref": 7.158345e-45 }, "GuessIt": { "n": 1296, "point_pct": 84.03, "ci95_lo": 82.02, "ci95_hi": 85.96, "delta_vs_ref_pct": 12.5, "delta_ci95_lo": 10.42, "delta_ci95_hi": 14.58, "p_value_vs_ref": 7.451219e-32 }, "PTN (parse-torrent-title)": { "n": 1296, "point_pct": 77.62, "ci95_lo": 75.31, "ci95_hi": 79.86, "delta_vs_ref_pct": 18.9, "delta_ci95_lo": 16.51, "delta_ci95_hi": 21.37, "p_value_vs_ref": 3.559008e-51 }, "Anitomy": { "n": 1296, "point_pct": 72.76, "ci95_lo": 70.29, "ci95_hi": 75.15, "delta_vs_ref_pct": 23.77, "delta_ci95_lo": 21.22, "delta_ci95_hi": 26.31, "p_value_vs_ref": 7.077119e-70 }, "Bitmagnet Built-In": { "n": 1296, "point_pct": 43.21, "ci95_lo": 40.51, "ci95_hi": 45.91, "delta_vs_ref_pct": 53.32, "delta_ci95_lo": 50.54, "delta_ci95_hi": 56.1, "p_value_vs_ref": 7.866753e-198 } } }, "panel_3_audio_strict_oct15": { "panel_name": "chart3_3_oct15_audio_strict545", "n": 545, "metric_type": "mean", "reference_model": "Armiger Ultra (Fair Hybrid)", "models": { "Armiger Ultra (Fair Hybrid)": { "n": 545, "point_pct": 97.8, "ci95_lo": 96.51, "ci95_hi": 98.9, "delta_vs_ref_pct": 0.0, "delta_ci95_lo": 0.0, "delta_ci95_hi": 0.0, "p_value_vs_ref": 1.0 }, "Armiger (8.3 MB INT8 + Plugins)": { "n": 545, "point_pct": 97.8, "ci95_lo": 96.51, "ci95_hi": 98.9, "delta_vs_ref_pct": 0.0, "delta_ci95_lo": 0.0, "delta_ci95_hi": 0.0, "p_value_vs_ref": 1.0 }, "RTN (rank-torrent-name)": { "n": 545, "point_pct": 93.76, "ci95_lo": 91.74, "ci95_hi": 95.78, "delta_vs_ref_pct": 4.04, "delta_ci95_lo": 1.83, "delta_ci95_hi": 6.24, "p_value_vs_ref": 0.0006795483 }, "PTT / Parsett": { "n": 545, "point_pct": 93.76, "ci95_lo": 91.74, "ci95_hi": 95.78, "delta_vs_ref_pct": 4.04, "delta_ci95_lo": 1.83, "delta_ci95_hi": 6.24, "p_value_vs_ref": 0.0006795483 }, "GuessIt": { "n": 545, "point_pct": 94.86, "ci95_lo": 93.03, "ci95_hi": 96.7, "delta_vs_ref_pct": 2.94, "delta_ci95_lo": 0.92, "delta_ci95_hi": 4.95, "p_value_vs_ref": 0.009041185 }, "PTN (parse-torrent-title)": { "n": 545, "point_pct": 93.21, "ci95_lo": 91.01, "ci95_hi": 95.23, "delta_vs_ref_pct": 4.59, "delta_ci95_lo": 2.39, "delta_ci95_hi": 6.97, "p_value_vs_ref": 0.0001701552 }, "Anitomy": { "n": 545, "point_pct": 90.46, "ci95_lo": 88.07, "ci95_hi": 92.84, "delta_vs_ref_pct": 7.34, "delta_ci95_lo": 4.77, "delta_ci95_hi": 9.91, "p_value_vs_ref": 4.688184e-08 }, "Bitmagnet Built-In": { "n": 545, "point_pct": 88.26, "ci95_lo": 85.5, "ci95_hi": 90.83, "delta_vs_ref_pct": 9.54, "delta_ci95_lo": 6.79, "delta_ci95_hi": 12.29, "p_value_vs_ref": 2.383493e-11 } } }, "panel_4_anime_oct15": { "panel_name": "chart3_4_oct15_anime507", "n": 507, "metric_type": "mean", "reference_model": "Armiger Ultra (Fair Hybrid)", "models": { "Armiger Ultra (Fair Hybrid)": { "n": 507, "point_pct": 99.61, "ci95_lo": 99.01, "ci95_hi": 100.0, "delta_vs_ref_pct": 0.0, "delta_ci95_lo": 0.0, "delta_ci95_hi": 0.0, "p_value_vs_ref": 1.0 }, "Armiger (8.3 MB INT8 + Plugins)": { "n": 507, "point_pct": 99.61, "ci95_lo": 99.01, "ci95_hi": 100.0, "delta_vs_ref_pct": 0.0, "delta_ci95_lo": 0.0, "delta_ci95_hi": 0.0, "p_value_vs_ref": 1.0 } } }, "panel_4_anime_dub_zero_cue_2000": { "panel_name": "chart3_4_zero_cue_2000_anime_dub", "n": 2000, "metric_type": "mean", "reference_model": "RTN (rank-torrent-name)", "models": { "RTN (rank-torrent-name)": { "n": 2000, "point_pct": 77.65, "ci95_lo": 75.75, "ci95_hi": 79.45, "delta_vs_ref_pct": 0.0, "delta_ci95_lo": 0.0, "delta_ci95_hi": 0.0, "p_value_vs_ref": 1.0 }, "PTT / Parsett": { "n": 2000, "point_pct": 77.65, "ci95_lo": 75.75, "ci95_hi": 79.45, "delta_vs_ref_pct": 0.0, "delta_ci95_lo": 0.0, "delta_ci95_hi": 0.0, "p_value_vs_ref": 1.0 }, "GuessIt": { "n": 2000, "point_pct": 78.8, "ci95_lo": 76.95, "ci95_hi": 80.55, "delta_vs_ref_pct": -1.15, "delta_ci95_lo": -2.45, "delta_ci95_hi": 0.15, "p_value_vs_ref": 0.09792971 }, "PTN (parse-torrent-title)": { "n": 2000, "point_pct": 68.65, "ci95_lo": 66.6, "ci95_hi": 70.65, "delta_vs_ref_pct": 9.0, "delta_ci95_lo": 7.15, "delta_ci95_hi": 10.85, "p_value_vs_ref": 7.57291e-21 }, "Anitomy": { "n": 2000, "point_pct": 64.55, "ci95_lo": 62.45, "ci95_hi": 66.6, "delta_vs_ref_pct": 13.1, "delta_ci95_lo": 11.2, "delta_ci95_hi": 15.05, "p_value_vs_ref": 4.440189e-40 } } }, "panel_5_adult_oct15": { "panel_name": "chart3_5_oct15_adult2781", "n": 2781, "metric_type": "mean", "reference_model": "Armiger Ultra (Fair Hybrid)", "models": { "Armiger Ultra (Fair Hybrid)": { "n": 2781, "point_pct": 98.53, "ci95_lo": 98.06, "ci95_hi": 98.96, "delta_vs_ref_pct": 0.0, "delta_ci95_lo": 0.0, "delta_ci95_hi": 0.0, "p_value_vs_ref": 1.0 }, "Armiger (8.3 MB INT8 + Plugins)": { "n": 2781, "point_pct": 98.49, "ci95_lo": 98.02, "ci95_hi": 98.92, "delta_vs_ref_pct": 0.04, "delta_ci95_lo": 0.0, "delta_ci95_hi": 0.11, "p_value_vs_ref": 1.0 }, "MIMMO BERT (360M)": { "n": 2781, "point_pct": 91.05, "ci95_lo": 89.93, "ci95_hi": 92.09, "delta_vs_ref_pct": 7.48, "delta_ci95_lo": 6.44, "delta_ci95_hi": 8.59, "p_value_vs_ref": 2.280742e-45 }, "MIMMO Cascade": { "n": 2781, "point_pct": 89.68, "ci95_lo": 88.53, "ci95_hi": 90.79, "delta_vs_ref_pct": 8.85, "delta_ci95_lo": 7.73, "delta_ci95_hi": 10.03, "p_value_vs_ref": 1.795157e-55 }, "RTN (rank-torrent-name)": { "n": 2781, "point_pct": 79.0, "ci95_lo": 77.45, "ci95_hi": 80.55, "delta_vs_ref_pct": 19.53, "delta_ci95_lo": 17.98, "delta_ci95_hi": 21.07, "p_value_vs_ref": 2.877898e-138 }, "PTT / Parsett": { "n": 2781, "point_pct": 79.0, "ci95_lo": 77.45, "ci95_hi": 80.55, "delta_vs_ref_pct": 19.53, "delta_ci95_lo": 17.98, "delta_ci95_hi": 21.07, "p_value_vs_ref": 2.877898e-138 }, "Bitmagnet Built-In": { "n": 2781, "point_pct": 76.73, "ci95_lo": 75.19, "ci95_hi": 78.32, "delta_vs_ref_pct": 21.79, "delta_ci95_lo": 20.17, "delta_ci95_hi": 23.37, "p_value_vs_ref": 1.549894e-154 } } }, "panel_6a_junk_oct15": { "panel_name": "chart3_6a_oct15_junk2170", "n": 2170, "metric_type": "balanced", "reference_model": "Armiger Ultra", "models": { "Armiger Ultra": { "n": 2170, "point_pct": 100.0, "ci95_lo": 100.0, "ci95_hi": 100.0, "delta_vs_ref_pct": 0.0, "delta_ci95_lo": 0.0, "delta_ci95_hi": 0.0, "p_value_vs_ref": 1.0 }, "Armiger INT8": { "n": 2170, "point_pct": 100.0, "ci95_lo": 100.0, "ci95_hi": 100.0, "delta_vs_ref_pct": 0.0, "delta_ci95_lo": 0.0, "delta_ci95_hi": 0.0, "p_value_vs_ref": 1.0 }, "Bitmagnet": { "n": 2170, "point_pct": 50.0, "ci95_lo": 50.0, "ci95_hi": 50.0, "delta_vs_ref_pct": 50.0, "delta_ci95_lo": 50.0, "delta_ci95_hi": 50.0, "p_value_vs_ref": 1.0 }, "RTN Filter": { "n": 2170, "point_pct": 49.29, "ci95_lo": 49.03, "ci95_hi": 49.52, "delta_vs_ref_pct": 50.71, "delta_ci95_lo": 50.48, "delta_ci95_hi": 50.97, "p_value_vs_ref": 4.656613e-10 } } } }, "armiger_categorical_breakdown.png": { "sha256": "1a77a64340f7e05af318105d83270b29396450d01f22f71cffdf808b45e10d98", "left_oct15_confirmation_n2835": { "Audiobook": { "n": 95, "armiger_ultra_pct": 97.89, "armiger_int8_pct": 97.89, "mimmo_bert_pct": 0.0, "bitmagnet_pct": 9.47 }, "Adult / XXX": { "n": 1044, "armiger_ultra_pct": 97.8, "armiger_int8_pct": 97.8, "mimmo_bert_pct": 95.5, "bitmagnet_pct": 39.37 }, "TV Show": { "n": 601, "armiger_ultra_pct": 97.5, "armiger_int8_pct": 97.5, "mimmo_bert_pct": 58.4, "bitmagnet_pct": 28.62 }, "Movie": { "n": 695, "armiger_ultra_pct": 97.12, "armiger_int8_pct": 97.12, "mimmo_bert_pct": 78.85, "bitmagnet_pct": 55.83 }, "Software": { "n": 32, "armiger_ultra_pct": 93.75, "armiger_int8_pct": 93.75, "mimmo_bert_pct": 59.38, "bitmagnet_pct": 15.62 }, "Music": { "n": 140, "armiger_ultra_pct": 90.71, "armiger_int8_pct": 90.71, "mimmo_bert_pct": 77.86, "bitmagnet_pct": 7.86 }, "Game": { "n": 61, "armiger_ultra_pct": 90.16, "armiger_int8_pct": 90.16, "mimmo_bert_pct": 0.0, "bitmagnet_pct": 0.0 }, "Ebook": { "n": 25, "armiger_ultra_pct": 88.0, "armiger_int8_pct": 88.0, "mimmo_bert_pct": 80.0, "bitmagnet_pct": 4.0 }, "Course": { "n": 33, "armiger_ultra_pct": 81.82, "armiger_int8_pct": 81.82, "mimmo_bert_pct": 0.0, "bitmagnet_pct": 0.0 }, "Comic": { "n": 99, "armiger_ultra_pct": 66.67, "armiger_int8_pct": 66.67, "mimmo_bert_pct": 0.0, "bitmagnet_pct": 0.0 } }, "right_beyond_confirmation_n1025_loto": { "Audiobook": { "n": 75, "armiger_ultra_pct": 100.0, "armiger_int8_pct": 100.0, "gpt_noncirc_pct": 98.67, "gemini_noncirc_pct": 98.0 }, "Ebook": { "n": 125, "armiger_ultra_pct": 98.4, "armiger_int8_pct": 98.4, "gpt_noncirc_pct": 96.0, "gemini_noncirc_pct": 96.8 }, "Comic": { "n": 120, "armiger_ultra_pct": 98.33, "armiger_int8_pct": 98.33, "gpt_noncirc_pct": 96.67, "gemini_noncirc_pct": 95.0 }, "Movie": { "n": 95, "armiger_ultra_pct": 96.84, "armiger_int8_pct": 96.84, "gpt_noncirc_pct": 94.21, "gemini_noncirc_pct": 92.63 }, "TV Show": { "n": 136, "armiger_ultra_pct": 96.32, "armiger_int8_pct": 96.32, "gpt_noncirc_pct": 97.79, "gemini_noncirc_pct": 93.75 }, "Game": { "n": 135, "armiger_ultra_pct": 96.3, "armiger_int8_pct": 96.3, "gpt_noncirc_pct": 98.52, "gemini_noncirc_pct": 98.52 }, "Software": { "n": 37, "armiger_ultra_pct": 94.59, "armiger_int8_pct": 94.59, "gpt_noncirc_pct": 97.3, "gemini_noncirc_pct": 77.03 }, "Course": { "n": 100, "armiger_ultra_pct": 92.0, "armiger_int8_pct": 92.0, "gpt_noncirc_pct": 95.5, "gemini_noncirc_pct": 97.5 }, "Adult / XXX": { "n": 174, "armiger_ultra_pct": 91.95, "armiger_int8_pct": 91.95, "gpt_noncirc_pct": 94.83, "gemini_noncirc_pct": 95.69 }, "Music": { "n": 12, "armiger_ultra_pct": 83.33, "armiger_int8_pct": 83.33, "gpt_noncirc_pct": 100.0, "gemini_noncirc_pct": 100.0 } } } }, "five_split_category_summary": { "oct15_confirmation": { "panel_name": "five_split_oct15_confirmation", "n": 2835, "metric_type": "mean", "reference_model": "armiger_ultra_hyb", "models": { "armiger_ultra_hyb": { "n": 2835, "point_pct": 95.49, "ci95_lo": 94.71, "ci95_hi": 96.23, "delta_vs_ref_pct": 0.0, "delta_ci95_lo": 0.0, "delta_ci95_hi": 0.0, "p_value_vs_ref": 1.0 }, "armiger_ultra_raw": { "n": 2835, "point_pct": 95.03, "ci95_lo": 94.22, "ci95_hi": 95.84, "delta_vs_ref_pct": 0.46, "delta_ci95_lo": 0.14, "delta_ci95_hi": 0.78, "p_value_vs_ref": 0.00719738 }, "armiger_int8_hyb": { "n": 2835, "point_pct": 95.49, "ci95_lo": 94.71, "ci95_hi": 96.23, "delta_vs_ref_pct": 0.0, "delta_ci95_lo": 0.0, "delta_ci95_hi": 0.0, "p_value_vs_ref": 1.0 }, "armiger_int8_raw": { "n": 2835, "point_pct": 95.06, "ci95_lo": 94.25, "ci95_hi": 95.87, "delta_vs_ref_pct": 0.42, "delta_ci95_lo": 0.14, "delta_ci95_hi": 0.74, "p_value_vs_ref": 0.01181793 } } }, "fresh_confirmation": { "panel_name": "five_split_fresh_confirmation", "n": 1402, "metric_type": "mean", "reference_model": "armiger_ultra_hyb", "models": { "armiger_ultra_hyb": { "n": 1402, "point_pct": 94.58, "ci95_lo": 93.37, "ci95_hi": 95.72, "delta_vs_ref_pct": 0.0, "delta_ci95_lo": 0.0, "delta_ci95_hi": 0.0, "p_value_vs_ref": 1.0 }, "armiger_ultra_raw": { "n": 1402, "point_pct": 94.51, "ci95_lo": 93.3, "ci95_hi": 95.65, "delta_vs_ref_pct": 0.07, "delta_ci95_lo": -0.21, "delta_ci95_hi": 0.36, "p_value_vs_ref": 1.0 }, "armiger_int8_hyb": { "n": 1402, "point_pct": 94.58, "ci95_lo": 93.37, "ci95_hi": 95.72, "delta_vs_ref_pct": 0.0, "delta_ci95_lo": 0.0, "delta_ci95_hi": 0.0, "p_value_vs_ref": 1.0 }, "armiger_int8_raw": { "n": 1402, "point_pct": 94.51, "ci95_lo": 93.3, "ci95_hi": 95.65, "delta_vs_ref_pct": 0.07, "delta_ci95_lo": -0.21, "delta_ci95_hi": 0.36, "p_value_vs_ref": 1.0 } } }, "beyond_confirmation": { "panel_name": "five_split_beyond_confirmation", "n": 1025, "metric_type": "mean", "reference_model": "armiger_ultra_hyb", "models": { "armiger_ultra_hyb": { "n": 1025, "point_pct": 95.12, "ci95_lo": 93.76, "ci95_hi": 96.39, "delta_vs_ref_pct": 0.0, "delta_ci95_lo": 0.0, "delta_ci95_hi": 0.0, "p_value_vs_ref": 1.0 }, "armiger_ultra_raw": { "n": 1025, "point_pct": 92.88, "ci95_lo": 91.22, "ci95_hi": 94.44, "delta_vs_ref_pct": 2.24, "delta_ci95_lo": 1.37, "delta_ci95_hi": 3.22, "p_value_vs_ref": 2.384186e-07 }, "armiger_int8_hyb": { "n": 1025, "point_pct": 95.12, "ci95_lo": 93.76, "ci95_hi": 96.39, "delta_vs_ref_pct": 0.0, "delta_ci95_lo": 0.0, "delta_ci95_hi": 0.0, "p_value_vs_ref": 1.0 }, "armiger_int8_raw": { "n": 1025, "point_pct": 92.88, "ci95_lo": 91.22, "ci95_hi": 94.44, "delta_vs_ref_pct": 2.24, "delta_ci95_lo": 1.37, "delta_ci95_hi": 3.22, "p_value_vs_ref": 2.384186e-07 } } }, "zero_cue_2000": { "panel_name": "five_split_zero_cue_2000", "n": 2000, "metric_type": "mean", "reference_model": "armiger_ultra_hyb", "models": { "armiger_ultra_hyb": { "n": 2000, "point_pct": 91.5, "ci95_lo": 90.3, "ci95_hi": 92.65, "delta_vs_ref_pct": 0.0, "delta_ci95_lo": 0.0, "delta_ci95_hi": 0.0, "p_value_vs_ref": 1.0 }, "armiger_ultra_raw": { "n": 2000, "point_pct": 87.4, "ci95_lo": 85.95, "ci95_hi": 88.8, "delta_vs_ref_pct": 4.1, "delta_ci95_lo": 2.95, "delta_ci95_hi": 5.25, "p_value_vs_ref": 1.091317e-12 }, "armiger_int8_hyb": { "n": 2000, "point_pct": 91.4, "ci95_lo": 90.2, "ci95_hi": 92.55, "delta_vs_ref_pct": 0.1, "delta_ci95_lo": 0.0, "delta_ci95_hi": 0.25, "p_value_vs_ref": 0.5 }, "armiger_int8_raw": { "n": 2000, "point_pct": 87.3, "ci95_lo": 85.85, "ci95_hi": 88.7, "delta_vs_ref_pct": 4.2, "delta_ci95_lo": 3.05, "delta_ci95_hi": 5.35, "p_value_vs_ref": 4.246281e-13 } } }, "multilang_3000": { "panel_name": "five_split_multilang_3000", "n": 3000, "metric_type": "mean", "reference_model": "armiger_ultra_hyb", "models": { "armiger_ultra_hyb": { "n": 3000, "point_pct": 98.47, "ci95_lo": 98.0, "ci95_hi": 98.9, "delta_vs_ref_pct": 0.0, "delta_ci95_lo": 0.0, "delta_ci95_hi": 0.0, "p_value_vs_ref": 1.0 }, "armiger_ultra_raw": { "n": 3000, "point_pct": 97.5, "ci95_lo": 96.9, "ci95_hi": 98.03, "delta_vs_ref_pct": 0.97, "delta_ci95_lo": 0.57, "delta_ci95_hi": 1.4, "p_value_vs_ref": 4.873598e-06 }, "armiger_int8_hyb": { "n": 3000, "point_pct": 98.43, "ci95_lo": 97.97, "ci95_hi": 98.87, "delta_vs_ref_pct": 0.03, "delta_ci95_lo": 0.0, "delta_ci95_hi": 0.1, "p_value_vs_ref": 1.0 }, "armiger_int8_raw": { "n": 3000, "point_pct": 97.43, "ci95_lo": 96.83, "ci95_hi": 97.97, "delta_vs_ref_pct": 1.03, "delta_ci95_lo": 0.63, "delta_ci95_hi": 1.47, "p_value_vs_ref": 1.636124e-06 } } } } }