Download results/summary.json from Divi19/vibe-embed: direct link, hf CLI and curl.
- Browser
- Download file 13.2 kB
-
https://huggingface.co/Divi19/vibe-embed/resolve/main/results/summary.json
- Command line
-
hf download hf://Divi19/vibe-embed/results/summary.json
-
curl -L -o summary.json https://huggingface.co/Divi19/vibe-embed/resolve/main/results/summary.json
13.2 kB
| { | |
| "step14": { | |
| "notebook": "14_step4_v12", | |
| "gpu": "NVIDIA RTX PRO 6000 Blackwell Server Edition", | |
| "time_utc": "2026-11-14T18:26:34+00:00", | |
| "student": "C · trained on the teacher's song vectors only (epoch 3)", | |
| "clips": { | |
| "train": 68404, | |
| "validation": 2901, | |
| "vibebench_pool": 1831, | |
| "shards": 78 | |
| }, | |
| "translations": { | |
| "es used": 38204, | |
| "es unchanged": 38, | |
| "es failed": 1758, | |
| "de used": 37036, | |
| "de unchanged": 426, | |
| "de failed": 2538, | |
| "fr used": 37928, | |
| "fr unchanged": 103, | |
| "fr failed": 1969, | |
| "pt used": 38331, | |
| "pt unchanged": 27, | |
| "pt failed": 1642, | |
| "ja used": 36715, | |
| "ja unchanged": 201, | |
| "ja failed": 3084 | |
| }, | |
| "search_2b": { | |
| "english": { | |
| "incompe_dev_lenient": 0.7105, | |
| "incompe_test_lenient": 0.7368, | |
| "incompe_full_lenient": 0.7235, | |
| "sdd_dev_R@10": 50.2752, | |
| "sdd_test_R@10": 44.385, | |
| "sdd_full_R@10": 47.2875 | |
| }, | |
| "languages_dev": 0.6013 | |
| }, | |
| "training": { | |
| "steps": 2143, | |
| "minutes": 21.8, | |
| "final_loss": 0.1463, | |
| "checks": [ | |
| { | |
| "check": 1, | |
| "step": 357, | |
| "english_incompe_dev": 0.7069, | |
| "english_sdd_dev_R@10": 50.5, | |
| "languages_incompe_dev": 0.6693, | |
| "select": 1.8808 | |
| }, | |
| { | |
| "check": 2, | |
| "step": 714, | |
| "english_incompe_dev": 0.7085, | |
| "english_sdd_dev_R@10": 49.9, | |
| "languages_incompe_dev": 0.6738, | |
| "select": 1.8814 | |
| }, | |
| { | |
| "check": 3, | |
| "step": 1072, | |
| "english_incompe_dev": 0.7056, | |
| "english_sdd_dev_R@10": 50.1, | |
| "languages_incompe_dev": 0.6737, | |
| "select": 1.8802 | |
| }, | |
| { | |
| "check": 4, | |
| "step": 1429, | |
| "english_incompe_dev": 0.7139, | |
| "english_sdd_dev_R@10": 49.7, | |
| "languages_incompe_dev": 0.6785, | |
| "select": 1.8896 | |
| }, | |
| { | |
| "check": 5, | |
| "step": 1786, | |
| "english_incompe_dev": 0.7116, | |
| "english_sdd_dev_R@10": 49.9, | |
| "languages_incompe_dev": 0.6801, | |
| "select": 1.8908 | |
| }, | |
| { | |
| "check": 6, | |
| "step": 2143, | |
| "english_incompe_dev": 0.7122, | |
| "english_sdd_dev_R@10": 50.1, | |
| "languages_incompe_dev": 0.6796, | |
| "select": 1.8927 | |
| } | |
| ] | |
| }, | |
| "language_rule": { | |
| "english_incompe_dev": 0.0017, | |
| "english_sdd_dev_R@10": -0.18, | |
| "languages_incompe_dev": 0.0784, | |
| "adopted": true, | |
| "chosen": "trained with translations (check 6)" | |
| }, | |
| "languages_board": [ | |
| { | |
| "language": "English", | |
| "2b dev": 0.7105, | |
| "chosen dev": 0.7122, | |
| "2b test": 0.7368, | |
| "chosen test": 0.7289 | |
| }, | |
| { | |
| "language": "Spanish", | |
| "2b dev": 0.6219, | |
| "chosen dev": 0.6817, | |
| "2b test": 0.6252, | |
| "chosen test": 0.6958 | |
| }, | |
| { | |
| "language": "German", | |
| "2b dev": 0.6164, | |
| "chosen dev": 0.679, | |
| "2b test": 0.5986, | |
| "chosen test": 0.6806 | |
| }, | |
| { | |
| "language": "French", | |
| "2b dev": 0.6365, | |
| "chosen dev": 0.6802, | |
| "2b test": 0.6355, | |
| "chosen test": 0.6792 | |
| }, | |
| { | |
| "language": "Brazilian Portuguese", | |
| "2b dev": 0.5764, | |
| "chosen dev": 0.6904, | |
| "2b test": 0.5857, | |
| "chosen test": 0.6907 | |
| }, | |
| { | |
| "language": "Japanese", | |
| "2b dev": 0.5551, | |
| "chosen dev": 0.6669, | |
| "2b test": 0.6049, | |
| "chosen test": 0.6842 | |
| } | |
| ], | |
| "search_chosen": { | |
| "english": { | |
| "incompe_dev_lenient": 0.7122, | |
| "incompe_test_lenient": 0.7289, | |
| "incompe_full_lenient": 0.7204, | |
| "sdd_dev_R@10": 50.0917, | |
| "sdd_test_R@10": 44.2068, | |
| "sdd_full_R@10": 47.1067 | |
| } | |
| }, | |
| "export_checks": { | |
| "audio_fp32_min_cos": 1.0, | |
| "audio_int8_min_cos": 0.99979, | |
| "text_onnx_vs_pytorch_min_cos": 1.0, | |
| "text_fp32": { | |
| "cos_mean": 1.0, | |
| "cos_min": 1.0, | |
| "plugin_parity_min": 1.0, | |
| "incompe_full": 0.7199, | |
| "ms_per_query_4thread": 71.1, | |
| "MB": 2395.7 | |
| }, | |
| "text_int8": { | |
| "cos_mean": 0.99976, | |
| "cos_min": 0.99919, | |
| "plugin_parity_min": 0.99979, | |
| "incompe_full": 0.7187, | |
| "ms_per_query_4thread": 55.8, | |
| "MB": 818.5 | |
| }, | |
| "incompe_full_pytorch": 0.7204, | |
| "incompe_full_all_int8": 0.7184, | |
| "incompe_test_all_int8": 0.7276, | |
| "sdd_full_R@10_int8_text": 47.1, | |
| "ok": true | |
| }, | |
| "cpu_speed_this_machine": { | |
| "audio_encoder": { | |
| "ms_per_10s_1thread": 43.4, | |
| "ms_per_10s_4thread": 26.2, | |
| "MB": 72.1 | |
| }, | |
| "audio_encoder_int8": { | |
| "ms_per_10s_1thread": 42.7, | |
| "ms_per_10s_4thread": 25.6, | |
| "MB": 35.3 | |
| }, | |
| "text_encoder_int8": { | |
| "ms_per_query_4thread": 55.8, | |
| "MB": 818.5 | |
| } | |
| }, | |
| "installed": true, | |
| "total_hours": 1.35 | |
| }, | |
| "step15": { | |
| "notebook": "15_vibebench_v12", | |
| "gpu": "NVIDIA RTX PRO 6000 Blackwell Server Edition", | |
| "time_utc": "2026-11-14T20:39:02+00:00", | |
| "installed_version": "v1.2", | |
| "new_pairs": 7675, | |
| "judge_minutes": 9.9, | |
| "judge": { | |
| "cuts": [ | |
| 0.9949183095027374, | |
| 1.1303260818861856, | |
| 2.9999961623507643 | |
| ], | |
| "dev_kappa": 0.445, | |
| "test_kappa": 0.47, | |
| "kappa_as_answered": 0.384 | |
| }, | |
| "main_system": "v1.2 student + v1.2 search side (ships)", | |
| "requests_scored_rescaled": 500, | |
| "scoreboard_rescaled": { | |
| "teacher, its own 7B search side": { | |
| "nDCG@10 all": 0.8031, | |
| "nDCG@10 test": 0.7922, | |
| "nDCG@10 dev": 0.8382, | |
| "judged@10": 1.0 | |
| }, | |
| "combined teacher, its own 7B search side": { | |
| "nDCG@10 all": 0.7939, | |
| "nDCG@10 test": 0.7866, | |
| "nDCG@10 dev": 0.8176, | |
| "judged@10": 1.0 | |
| }, | |
| "combined teacher, the v1.2 search side": { | |
| "nDCG@10 all": 0.7644, | |
| "nDCG@10 test": 0.7537, | |
| "nDCG@10 dev": 0.7991, | |
| "judged@10": 1.0 | |
| }, | |
| "teacher, the student's search side (Qwen3)": { | |
| "nDCG@10 all": 0.7536, | |
| "nDCG@10 test": 0.7397, | |
| "nDCG@10 dev": 0.7983, | |
| "judged@10": 1.0 | |
| }, | |
| "LCO-Omni-7B as is": { | |
| "nDCG@10 all": 0.7494, | |
| "nDCG@10 test": 0.7415, | |
| "nDCG@10 dev": 0.775, | |
| "judged@10": 1.0 | |
| }, | |
| "v1.2 student + v1.2 search side (ships)": { | |
| "nDCG@10 all": 0.7329, | |
| "nDCG@10 test": 0.7191, | |
| "nDCG@10 dev": 0.7775, | |
| "judged@10": 1.0 | |
| }, | |
| "e5-omni-7B as is": { | |
| "nDCG@10 all": 0.7309, | |
| "nDCG@10 test": 0.7214, | |
| "nDCG@10 dev": 0.7615, | |
| "judged@10": 1.0 | |
| }, | |
| "v1 student + v1.1 search side": { | |
| "nDCG@10 all": 0.718, | |
| "nDCG@10 test": 0.708, | |
| "nDCG@10 dev": 0.7507, | |
| "judged@10": 1.0 | |
| }, | |
| "v1 student + v1 search side (07)": { | |
| "nDCG@10 all": 0.6861, | |
| "nDCG@10 test": 0.6769, | |
| "nDCG@10 dev": 0.7159, | |
| "judged@10": 1.0 | |
| }, | |
| "DCLAP (AudioMuse-AI)": { | |
| "nDCG@10 all": 0.6371, | |
| "nDCG@10 test": 0.6312, | |
| "nDCG@10 dev": 0.6561, | |
| "judged@10": 1.0 | |
| }, | |
| "LAION CLAP music+speech": { | |
| "nDCG@10 all": 0.574, | |
| "nDCG@10 test": 0.5683, | |
| "nDCG@10 dev": 0.5923, | |
| "judged@10": 1.0 | |
| }, | |
| "BM25 over the song cards": { | |
| "nDCG@10 all": 0.5307, | |
| "nDCG@10 test": 0.5245, | |
| "nDCG@10 dev": 0.5508, | |
| "judged@10": 1.0 | |
| } | |
| }, | |
| "scoreboard_as_answered": { | |
| "teacher, its own 7B search side": { | |
| "nDCG@10 all": 0.8283, | |
| "nDCG@10 test": 0.8238, | |
| "nDCG@10 dev": 0.8427, | |
| "judged@10": 1.0 | |
| }, | |
| "combined teacher, its own 7B search side": { | |
| "nDCG@10 all": 0.8132, | |
| "nDCG@10 test": 0.8115, | |
| "nDCG@10 dev": 0.8186, | |
| "judged@10": 1.0 | |
| }, | |
| "combined teacher, the v1.2 search side": { | |
| "nDCG@10 all": 0.7847, | |
| "nDCG@10 test": 0.782, | |
| "nDCG@10 dev": 0.7936, | |
| "judged@10": 1.0 | |
| }, | |
| "teacher, the student's search side (Qwen3)": { | |
| "nDCG@10 all": 0.7773, | |
| "nDCG@10 test": 0.771, | |
| "nDCG@10 dev": 0.7977, | |
| "judged@10": 1.0 | |
| }, | |
| "LCO-Omni-7B as is": { | |
| "nDCG@10 all": 0.7635, | |
| "nDCG@10 test": 0.7631, | |
| "nDCG@10 dev": 0.7649, | |
| "judged@10": 1.0 | |
| }, | |
| "v1.2 student + v1.2 search side (ships)": { | |
| "nDCG@10 all": 0.7509, | |
| "nDCG@10 test": 0.7449, | |
| "nDCG@10 dev": 0.7705, | |
| "judged@10": 1.0 | |
| }, | |
| "v1 student + v1.1 search side": { | |
| "nDCG@10 all": 0.744, | |
| "nDCG@10 test": 0.7391, | |
| "nDCG@10 dev": 0.76, | |
| "judged@10": 1.0 | |
| }, | |
| "e5-omni-7B as is": { | |
| "nDCG@10 all": 0.7421, | |
| "nDCG@10 test": 0.7383, | |
| "nDCG@10 dev": 0.7545, | |
| "judged@10": 1.0 | |
| }, | |
| "v1 student + v1 search side (07)": { | |
| "nDCG@10 all": 0.7039, | |
| "nDCG@10 test": 0.6994, | |
| "nDCG@10 dev": 0.7184, | |
| "judged@10": 1.0 | |
| }, | |
| "DCLAP (AudioMuse-AI)": { | |
| "nDCG@10 all": 0.6513, | |
| "nDCG@10 test": 0.6472, | |
| "nDCG@10 dev": 0.6646, | |
| "judged@10": 1.0 | |
| }, | |
| "LAION CLAP music+speech": { | |
| "nDCG@10 all": 0.5857, | |
| "nDCG@10 test": 0.5881, | |
| "nDCG@10 dev": 0.5779, | |
| "judged@10": 1.0 | |
| }, | |
| "BM25 over the song cards": { | |
| "nDCG@10 all": 0.5496, | |
| "nDCG@10 test": 0.5486, | |
| "nDCG@10 dev": 0.5529, | |
| "judged@10": 1.0 | |
| } | |
| }, | |
| "deltas": [ | |
| { | |
| "v1.2 student + v1.2 search side minus": "DCLAP (AudioMuse-AI)", | |
| "rescaled, all [95% CI]": "+0.096 [+0.078, +0.115]", | |
| "rescaled, test [95% CI]": "+0.088 [+0.067, +0.109]", | |
| "as answered, all [95% CI]": "+0.100 [+0.083, +0.117]" | |
| }, | |
| { | |
| "v1.2 student + v1.2 search side minus": "LAION CLAP music+speech", | |
| "rescaled, all [95% CI]": "+0.159 [+0.139, +0.179]", | |
| "rescaled, test [95% CI]": "+0.151 [+0.128, +0.175]", | |
| "as answered, all [95% CI]": "+0.165 [+0.147, +0.185]" | |
| }, | |
| { | |
| "v1.2 student + v1.2 search side minus": "v1 student + v1.1 search side", | |
| "rescaled, all [95% CI]": "+0.015 [+0.004, +0.026]", | |
| "rescaled, test [95% CI]": "+0.011 [-0.002, +0.025]", | |
| "as answered, all [95% CI]": "+0.007 [-0.002, +0.016]" | |
| }, | |
| { | |
| "v1.2 student + v1.2 search side minus": "v1 student + v1 search side (07)", | |
| "rescaled, all [95% CI]": "+0.047 [+0.033, +0.060]", | |
| "rescaled, test [95% CI]": "+0.042 [+0.028, +0.057]", | |
| "as answered, all [95% CI]": "+0.047 [+0.036, +0.059]" | |
| }, | |
| { | |
| "v1.2 student + v1.2 search side minus": "LCO-Omni-7B as is", | |
| "rescaled, all [95% CI]": "-0.017 [-0.031, -0.002]", | |
| "rescaled, test [95% CI]": "-0.022 [-0.040, -0.005]", | |
| "as answered, all [95% CI]": "-0.013 [-0.026, +0.001]" | |
| }, | |
| { | |
| "v1.2 student + v1.2 search side minus": "combined teacher, its own 7B search side", | |
| "rescaled, all [95% CI]": "-0.061 [-0.074, -0.048]", | |
| "rescaled, test [95% CI]": "-0.068 [-0.084, -0.052]", | |
| "as answered, all [95% CI]": "-0.062 [-0.074, -0.051]" | |
| } | |
| ], | |
| "total_minutes": 48.8 | |
| }, | |
| "decision13b": { | |
| "adopted": true, | |
| "student": "mn20_as", | |
| "chosen": "C · trained on the teacher's song vectors only (epoch 3)", | |
| "teacher": "combined (teacher/combined)", | |
| "search_side_for_selection": "student/text_v12_base", | |
| "scores": { | |
| "incompe_dev_lenient": 0.7105, | |
| "incompe_test_lenient": 0.7368, | |
| "incompe_full_lenient": 0.7235, | |
| "sdd_dev_R@10": 50.2752, | |
| "sdd_test_R@10": 44.385, | |
| "sdd_full_R@10": 47.2875 | |
| } | |
| }, | |
| "release": { | |
| "plugin_checks": { | |
| "int8": { | |
| "audio_cos": 0.99982, | |
| "text_cos_min": 0.99976, | |
| "tokenizer": "tokenizers" | |
| }, | |
| "fp32": { | |
| "audio_cos": 1.0, | |
| "text_cos_min": 1.0, | |
| "tokenizer": "tokenizers" | |
| } | |
| }, | |
| "sizes": { | |
| "audio_int8_MB": 35.3, | |
| "audio_fp32_MB": 72.1, | |
| "text_int8_MB": 818.5, | |
| "text_fp32_MB": 2395.7 | |
| }, | |
| "catalogue": { | |
| "songs": 1831, | |
| "left_out_licence": 0, | |
| "left_out_unreadable": 0, | |
| "cos_to_pytorch_mean": 0.9998, | |
| "cos_to_pytorch_min": 0.9925, | |
| "minutes": 6.4 | |
| }, | |
| "speed": { | |
| "song_seconds": 180.0, | |
| "board": [ | |
| { | |
| "threads": 4, | |
| "ours int8 (s/song)": 0.902, | |
| "ours fp32 (s/song)": 0.903, | |
| "DCLAP (s/song)": 4.243, | |
| "DCLAP / ours int8": 4.7 | |
| }, | |
| { | |
| "threads": 1, | |
| "ours int8 (s/song)": 1.897, | |
| "ours fp32 (s/song)": 1.909, | |
| "DCLAP (s/song)": 2.677, | |
| "DCLAP / ours int8": 1.41 | |
| } | |
| ], | |
| "gate_passed": true | |
| } | |
| } | |
| } |