{ "step14": { "notebook": "14_step4_v12", "gpu": "NVIDIA RTX PRO 6000 Blackwell Server Edition", "time_utc": "2026-11-14T18:26:34+00:00", "student": "C · trained on the teacher's song vectors only (epoch 3)", "clips": { "train": 68404, "validation": 2901, "vibebench_pool": 1831, "shards": 78 }, "translations": { "es used": 38204, "es unchanged": 38, "es failed": 1758, "de used": 37036, "de unchanged": 426, "de failed": 2538, "fr used": 37928, "fr unchanged": 103, "fr failed": 1969, "pt used": 38331, "pt unchanged": 27, "pt failed": 1642, "ja used": 36715, "ja unchanged": 201, "ja failed": 3084 }, "search_2b": { "english": { "incompe_dev_lenient": 0.7105, "incompe_test_lenient": 0.7368, "incompe_full_lenient": 0.7235, "sdd_dev_R@10": 50.2752, "sdd_test_R@10": 44.385, "sdd_full_R@10": 47.2875 }, "languages_dev": 0.6013 }, "training": { "steps": 2143, "minutes": 21.8, "final_loss": 0.1463, "checks": [ { "check": 1, "step": 357, "english_incompe_dev": 0.7069, "english_sdd_dev_R@10": 50.5, "languages_incompe_dev": 0.6693, "select": 1.8808 }, { "check": 2, "step": 714, "english_incompe_dev": 0.7085, "english_sdd_dev_R@10": 49.9, "languages_incompe_dev": 0.6738, "select": 1.8814 }, { "check": 3, "step": 1072, "english_incompe_dev": 0.7056, "english_sdd_dev_R@10": 50.1, "languages_incompe_dev": 0.6737, "select": 1.8802 }, { "check": 4, "step": 1429, "english_incompe_dev": 0.7139, "english_sdd_dev_R@10": 49.7, "languages_incompe_dev": 0.6785, "select": 1.8896 }, { "check": 5, "step": 1786, "english_incompe_dev": 0.7116, "english_sdd_dev_R@10": 49.9, "languages_incompe_dev": 0.6801, "select": 1.8908 }, { "check": 6, "step": 2143, "english_incompe_dev": 0.7122, "english_sdd_dev_R@10": 50.1, "languages_incompe_dev": 0.6796, "select": 1.8927 } ] }, "language_rule": { "english_incompe_dev": 0.0017, "english_sdd_dev_R@10": -0.18, "languages_incompe_dev": 0.0784, "adopted": true, "chosen": "trained with translations (check 6)" }, "languages_board": [ { "language": "English", "2b dev": 0.7105, "chosen dev": 0.7122, "2b test": 0.7368, "chosen test": 0.7289 }, { "language": "Spanish", "2b dev": 0.6219, "chosen dev": 0.6817, "2b test": 0.6252, "chosen test": 0.6958 }, { "language": "German", "2b dev": 0.6164, "chosen dev": 0.679, "2b test": 0.5986, "chosen test": 0.6806 }, { "language": "French", "2b dev": 0.6365, "chosen dev": 0.6802, "2b test": 0.6355, "chosen test": 0.6792 }, { "language": "Brazilian Portuguese", "2b dev": 0.5764, "chosen dev": 0.6904, "2b test": 0.5857, "chosen test": 0.6907 }, { "language": "Japanese", "2b dev": 0.5551, "chosen dev": 0.6669, "2b test": 0.6049, "chosen test": 0.6842 } ], "search_chosen": { "english": { "incompe_dev_lenient": 0.7122, "incompe_test_lenient": 0.7289, "incompe_full_lenient": 0.7204, "sdd_dev_R@10": 50.0917, "sdd_test_R@10": 44.2068, "sdd_full_R@10": 47.1067 } }, "export_checks": { "audio_fp32_min_cos": 1.0, "audio_int8_min_cos": 0.99979, "text_onnx_vs_pytorch_min_cos": 1.0, "text_fp32": { "cos_mean": 1.0, "cos_min": 1.0, "plugin_parity_min": 1.0, "incompe_full": 0.7199, "ms_per_query_4thread": 71.1, "MB": 2395.7 }, "text_int8": { "cos_mean": 0.99976, "cos_min": 0.99919, "plugin_parity_min": 0.99979, "incompe_full": 0.7187, "ms_per_query_4thread": 55.8, "MB": 818.5 }, "incompe_full_pytorch": 0.7204, "incompe_full_all_int8": 0.7184, "incompe_test_all_int8": 0.7276, "sdd_full_R@10_int8_text": 47.1, "ok": true }, "cpu_speed_this_machine": { "audio_encoder": { "ms_per_10s_1thread": 43.4, "ms_per_10s_4thread": 26.2, "MB": 72.1 }, "audio_encoder_int8": { "ms_per_10s_1thread": 42.7, "ms_per_10s_4thread": 25.6, "MB": 35.3 }, "text_encoder_int8": { "ms_per_query_4thread": 55.8, "MB": 818.5 } }, "installed": true, "total_hours": 1.35 }, "step15": { "notebook": "15_vibebench_v12", "gpu": "NVIDIA RTX PRO 6000 Blackwell Server Edition", "time_utc": "2026-11-14T20:39:02+00:00", "installed_version": "v1.2", "new_pairs": 7675, "judge_minutes": 9.9, "judge": { "cuts": [ 0.9949183095027374, 1.1303260818861856, 2.9999961623507643 ], "dev_kappa": 0.445, "test_kappa": 0.47, "kappa_as_answered": 0.384 }, "main_system": "v1.2 student + v1.2 search side (ships)", "requests_scored_rescaled": 500, "scoreboard_rescaled": { "teacher, its own 7B search side": { "nDCG@10 all": 0.8031, "nDCG@10 test": 0.7922, "nDCG@10 dev": 0.8382, "judged@10": 1.0 }, "combined teacher, its own 7B search side": { "nDCG@10 all": 0.7939, "nDCG@10 test": 0.7866, "nDCG@10 dev": 0.8176, "judged@10": 1.0 }, "combined teacher, the v1.2 search side": { "nDCG@10 all": 0.7644, "nDCG@10 test": 0.7537, "nDCG@10 dev": 0.7991, "judged@10": 1.0 }, "teacher, the student's search side (Qwen3)": { "nDCG@10 all": 0.7536, "nDCG@10 test": 0.7397, "nDCG@10 dev": 0.7983, "judged@10": 1.0 }, "LCO-Omni-7B as is": { "nDCG@10 all": 0.7494, "nDCG@10 test": 0.7415, "nDCG@10 dev": 0.775, "judged@10": 1.0 }, "v1.2 student + v1.2 search side (ships)": { "nDCG@10 all": 0.7329, "nDCG@10 test": 0.7191, "nDCG@10 dev": 0.7775, "judged@10": 1.0 }, "e5-omni-7B as is": { "nDCG@10 all": 0.7309, "nDCG@10 test": 0.7214, "nDCG@10 dev": 0.7615, "judged@10": 1.0 }, "v1 student + v1.1 search side": { "nDCG@10 all": 0.718, "nDCG@10 test": 0.708, "nDCG@10 dev": 0.7507, "judged@10": 1.0 }, "v1 student + v1 search side (07)": { "nDCG@10 all": 0.6861, "nDCG@10 test": 0.6769, "nDCG@10 dev": 0.7159, "judged@10": 1.0 }, "DCLAP (AudioMuse-AI)": { "nDCG@10 all": 0.6371, "nDCG@10 test": 0.6312, "nDCG@10 dev": 0.6561, "judged@10": 1.0 }, "LAION CLAP music+speech": { "nDCG@10 all": 0.574, "nDCG@10 test": 0.5683, "nDCG@10 dev": 0.5923, "judged@10": 1.0 }, "BM25 over the song cards": { "nDCG@10 all": 0.5307, "nDCG@10 test": 0.5245, "nDCG@10 dev": 0.5508, "judged@10": 1.0 } }, "scoreboard_as_answered": { "teacher, its own 7B search side": { "nDCG@10 all": 0.8283, "nDCG@10 test": 0.8238, "nDCG@10 dev": 0.8427, "judged@10": 1.0 }, "combined teacher, its own 7B search side": { "nDCG@10 all": 0.8132, "nDCG@10 test": 0.8115, "nDCG@10 dev": 0.8186, "judged@10": 1.0 }, "combined teacher, the v1.2 search side": { "nDCG@10 all": 0.7847, "nDCG@10 test": 0.782, "nDCG@10 dev": 0.7936, "judged@10": 1.0 }, "teacher, the student's search side (Qwen3)": { "nDCG@10 all": 0.7773, "nDCG@10 test": 0.771, "nDCG@10 dev": 0.7977, "judged@10": 1.0 }, "LCO-Omni-7B as is": { "nDCG@10 all": 0.7635, "nDCG@10 test": 0.7631, "nDCG@10 dev": 0.7649, "judged@10": 1.0 }, "v1.2 student + v1.2 search side (ships)": { "nDCG@10 all": 0.7509, "nDCG@10 test": 0.7449, "nDCG@10 dev": 0.7705, "judged@10": 1.0 }, "v1 student + v1.1 search side": { "nDCG@10 all": 0.744, "nDCG@10 test": 0.7391, "nDCG@10 dev": 0.76, "judged@10": 1.0 }, "e5-omni-7B as is": { "nDCG@10 all": 0.7421, "nDCG@10 test": 0.7383, "nDCG@10 dev": 0.7545, "judged@10": 1.0 }, "v1 student + v1 search side (07)": { "nDCG@10 all": 0.7039, "nDCG@10 test": 0.6994, "nDCG@10 dev": 0.7184, "judged@10": 1.0 }, "DCLAP (AudioMuse-AI)": { "nDCG@10 all": 0.6513, "nDCG@10 test": 0.6472, "nDCG@10 dev": 0.6646, "judged@10": 1.0 }, "LAION CLAP music+speech": { "nDCG@10 all": 0.5857, "nDCG@10 test": 0.5881, "nDCG@10 dev": 0.5779, "judged@10": 1.0 }, "BM25 over the song cards": { "nDCG@10 all": 0.5496, "nDCG@10 test": 0.5486, "nDCG@10 dev": 0.5529, "judged@10": 1.0 } }, "deltas": [ { "v1.2 student + v1.2 search side minus": "DCLAP (AudioMuse-AI)", "rescaled, all [95% CI]": "+0.096 [+0.078, +0.115]", "rescaled, test [95% CI]": "+0.088 [+0.067, +0.109]", "as answered, all [95% CI]": "+0.100 [+0.083, +0.117]" }, { "v1.2 student + v1.2 search side minus": "LAION CLAP music+speech", "rescaled, all [95% CI]": "+0.159 [+0.139, +0.179]", "rescaled, test [95% CI]": "+0.151 [+0.128, +0.175]", "as answered, all [95% CI]": "+0.165 [+0.147, +0.185]" }, { "v1.2 student + v1.2 search side minus": "v1 student + v1.1 search side", "rescaled, all [95% CI]": "+0.015 [+0.004, +0.026]", "rescaled, test [95% CI]": "+0.011 [-0.002, +0.025]", "as answered, all [95% CI]": "+0.007 [-0.002, +0.016]" }, { "v1.2 student + v1.2 search side minus": "v1 student + v1 search side (07)", "rescaled, all [95% CI]": "+0.047 [+0.033, +0.060]", "rescaled, test [95% CI]": "+0.042 [+0.028, +0.057]", "as answered, all [95% CI]": "+0.047 [+0.036, +0.059]" }, { "v1.2 student + v1.2 search side minus": "LCO-Omni-7B as is", "rescaled, all [95% CI]": "-0.017 [-0.031, -0.002]", "rescaled, test [95% CI]": "-0.022 [-0.040, -0.005]", "as answered, all [95% CI]": "-0.013 [-0.026, +0.001]" }, { "v1.2 student + v1.2 search side minus": "combined teacher, its own 7B search side", "rescaled, all [95% CI]": "-0.061 [-0.074, -0.048]", "rescaled, test [95% CI]": "-0.068 [-0.084, -0.052]", "as answered, all [95% CI]": "-0.062 [-0.074, -0.051]" } ], "total_minutes": 48.8 }, "decision13b": { "adopted": true, "student": "mn20_as", "chosen": "C · trained on the teacher's song vectors only (epoch 3)", "teacher": "combined (teacher/combined)", "search_side_for_selection": "student/text_v12_base", "scores": { "incompe_dev_lenient": 0.7105, "incompe_test_lenient": 0.7368, "incompe_full_lenient": 0.7235, "sdd_dev_R@10": 50.2752, "sdd_test_R@10": 44.385, "sdd_full_R@10": 47.2875 } }, "release": { "plugin_checks": { "int8": { "audio_cos": 0.99982, "text_cos_min": 0.99976, "tokenizer": "tokenizers" }, "fp32": { "audio_cos": 1.0, "text_cos_min": 1.0, "tokenizer": "tokenizers" } }, "sizes": { "audio_int8_MB": 35.3, "audio_fp32_MB": 72.1, "text_int8_MB": 818.5, "text_fp32_MB": 2395.7 }, "catalogue": { "songs": 1831, "left_out_licence": 0, "left_out_unreadable": 0, "cos_to_pytorch_mean": 0.9998, "cos_to_pytorch_min": 0.9925, "minutes": 6.4 }, "speed": { "song_seconds": 180.0, "board": [ { "threads": 4, "ours int8 (s/song)": 0.902, "ours fp32 (s/song)": 0.903, "DCLAP (s/song)": 4.243, "DCLAP / ours int8": 4.7 }, { "threads": 1, "ours int8 (s/song)": 1.897, "ours fp32 (s/song)": 1.909, "DCLAP (s/song)": 2.677, "DCLAP / ours int8": 1.41 } ], "gate_passed": true } } }