vibe-embed / results /summary.json
Divi19's picture
v1.2 release (16)
8732fc1 verified
Raw History Blame Contribute Delete
13.2 kB
{
"step14": {
"notebook": "14_step4_v12",
"gpu": "NVIDIA RTX PRO 6000 Blackwell Server Edition",
"time_utc": "2026-11-14T18:26:34+00:00",
"student": "C · trained on the teacher's song vectors only (epoch 3)",
"clips": {
"train": 68404,
"validation": 2901,
"vibebench_pool": 1831,
"shards": 78
},
"translations": {
"es used": 38204,
"es unchanged": 38,
"es failed": 1758,
"de used": 37036,
"de unchanged": 426,
"de failed": 2538,
"fr used": 37928,
"fr unchanged": 103,
"fr failed": 1969,
"pt used": 38331,
"pt unchanged": 27,
"pt failed": 1642,
"ja used": 36715,
"ja unchanged": 201,
"ja failed": 3084
},
"search_2b": {
"english": {
"incompe_dev_lenient": 0.7105,
"incompe_test_lenient": 0.7368,
"incompe_full_lenient": 0.7235,
"sdd_dev_R@10": 50.2752,
"sdd_test_R@10": 44.385,
"sdd_full_R@10": 47.2875
},
"languages_dev": 0.6013
},
"training": {
"steps": 2143,
"minutes": 21.8,
"final_loss": 0.1463,
"checks": [
{
"check": 1,
"step": 357,
"english_incompe_dev": 0.7069,
"english_sdd_dev_R@10": 50.5,
"languages_incompe_dev": 0.6693,
"select": 1.8808
},
{
"check": 2,
"step": 714,
"english_incompe_dev": 0.7085,
"english_sdd_dev_R@10": 49.9,
"languages_incompe_dev": 0.6738,
"select": 1.8814
},
{
"check": 3,
"step": 1072,
"english_incompe_dev": 0.7056,
"english_sdd_dev_R@10": 50.1,
"languages_incompe_dev": 0.6737,
"select": 1.8802
},
{
"check": 4,
"step": 1429,
"english_incompe_dev": 0.7139,
"english_sdd_dev_R@10": 49.7,
"languages_incompe_dev": 0.6785,
"select": 1.8896
},
{
"check": 5,
"step": 1786,
"english_incompe_dev": 0.7116,
"english_sdd_dev_R@10": 49.9,
"languages_incompe_dev": 0.6801,
"select": 1.8908
},
{
"check": 6,
"step": 2143,
"english_incompe_dev": 0.7122,
"english_sdd_dev_R@10": 50.1,
"languages_incompe_dev": 0.6796,
"select": 1.8927
}
]
},
"language_rule": {
"english_incompe_dev": 0.0017,
"english_sdd_dev_R@10": -0.18,
"languages_incompe_dev": 0.0784,
"adopted": true,
"chosen": "trained with translations (check 6)"
},
"languages_board": [
{
"language": "English",
"2b dev": 0.7105,
"chosen dev": 0.7122,
"2b test": 0.7368,
"chosen test": 0.7289
},
{
"language": "Spanish",
"2b dev": 0.6219,
"chosen dev": 0.6817,
"2b test": 0.6252,
"chosen test": 0.6958
},
{
"language": "German",
"2b dev": 0.6164,
"chosen dev": 0.679,
"2b test": 0.5986,
"chosen test": 0.6806
},
{
"language": "French",
"2b dev": 0.6365,
"chosen dev": 0.6802,
"2b test": 0.6355,
"chosen test": 0.6792
},
{
"language": "Brazilian Portuguese",
"2b dev": 0.5764,
"chosen dev": 0.6904,
"2b test": 0.5857,
"chosen test": 0.6907
},
{
"language": "Japanese",
"2b dev": 0.5551,
"chosen dev": 0.6669,
"2b test": 0.6049,
"chosen test": 0.6842
}
],
"search_chosen": {
"english": {
"incompe_dev_lenient": 0.7122,
"incompe_test_lenient": 0.7289,
"incompe_full_lenient": 0.7204,
"sdd_dev_R@10": 50.0917,
"sdd_test_R@10": 44.2068,
"sdd_full_R@10": 47.1067
}
},
"export_checks": {
"audio_fp32_min_cos": 1.0,
"audio_int8_min_cos": 0.99979,
"text_onnx_vs_pytorch_min_cos": 1.0,
"text_fp32": {
"cos_mean": 1.0,
"cos_min": 1.0,
"plugin_parity_min": 1.0,
"incompe_full": 0.7199,
"ms_per_query_4thread": 71.1,
"MB": 2395.7
},
"text_int8": {
"cos_mean": 0.99976,
"cos_min": 0.99919,
"plugin_parity_min": 0.99979,
"incompe_full": 0.7187,
"ms_per_query_4thread": 55.8,
"MB": 818.5
},
"incompe_full_pytorch": 0.7204,
"incompe_full_all_int8": 0.7184,
"incompe_test_all_int8": 0.7276,
"sdd_full_R@10_int8_text": 47.1,
"ok": true
},
"cpu_speed_this_machine": {
"audio_encoder": {
"ms_per_10s_1thread": 43.4,
"ms_per_10s_4thread": 26.2,
"MB": 72.1
},
"audio_encoder_int8": {
"ms_per_10s_1thread": 42.7,
"ms_per_10s_4thread": 25.6,
"MB": 35.3
},
"text_encoder_int8": {
"ms_per_query_4thread": 55.8,
"MB": 818.5
}
},
"installed": true,
"total_hours": 1.35
},
"step15": {
"notebook": "15_vibebench_v12",
"gpu": "NVIDIA RTX PRO 6000 Blackwell Server Edition",
"time_utc": "2026-11-14T20:39:02+00:00",
"installed_version": "v1.2",
"new_pairs": 7675,
"judge_minutes": 9.9,
"judge": {
"cuts": [
0.9949183095027374,
1.1303260818861856,
2.9999961623507643
],
"dev_kappa": 0.445,
"test_kappa": 0.47,
"kappa_as_answered": 0.384
},
"main_system": "v1.2 student + v1.2 search side (ships)",
"requests_scored_rescaled": 500,
"scoreboard_rescaled": {
"teacher, its own 7B search side": {
"nDCG@10 all": 0.8031,
"nDCG@10 test": 0.7922,
"nDCG@10 dev": 0.8382,
"judged@10": 1.0
},
"combined teacher, its own 7B search side": {
"nDCG@10 all": 0.7939,
"nDCG@10 test": 0.7866,
"nDCG@10 dev": 0.8176,
"judged@10": 1.0
},
"combined teacher, the v1.2 search side": {
"nDCG@10 all": 0.7644,
"nDCG@10 test": 0.7537,
"nDCG@10 dev": 0.7991,
"judged@10": 1.0
},
"teacher, the student's search side (Qwen3)": {
"nDCG@10 all": 0.7536,
"nDCG@10 test": 0.7397,
"nDCG@10 dev": 0.7983,
"judged@10": 1.0
},
"LCO-Omni-7B as is": {
"nDCG@10 all": 0.7494,
"nDCG@10 test": 0.7415,
"nDCG@10 dev": 0.775,
"judged@10": 1.0
},
"v1.2 student + v1.2 search side (ships)": {
"nDCG@10 all": 0.7329,
"nDCG@10 test": 0.7191,
"nDCG@10 dev": 0.7775,
"judged@10": 1.0
},
"e5-omni-7B as is": {
"nDCG@10 all": 0.7309,
"nDCG@10 test": 0.7214,
"nDCG@10 dev": 0.7615,
"judged@10": 1.0
},
"v1 student + v1.1 search side": {
"nDCG@10 all": 0.718,
"nDCG@10 test": 0.708,
"nDCG@10 dev": 0.7507,
"judged@10": 1.0
},
"v1 student + v1 search side (07)": {
"nDCG@10 all": 0.6861,
"nDCG@10 test": 0.6769,
"nDCG@10 dev": 0.7159,
"judged@10": 1.0
},
"DCLAP (AudioMuse-AI)": {
"nDCG@10 all": 0.6371,
"nDCG@10 test": 0.6312,
"nDCG@10 dev": 0.6561,
"judged@10": 1.0
},
"LAION CLAP music+speech": {
"nDCG@10 all": 0.574,
"nDCG@10 test": 0.5683,
"nDCG@10 dev": 0.5923,
"judged@10": 1.0
},
"BM25 over the song cards": {
"nDCG@10 all": 0.5307,
"nDCG@10 test": 0.5245,
"nDCG@10 dev": 0.5508,
"judged@10": 1.0
}
},
"scoreboard_as_answered": {
"teacher, its own 7B search side": {
"nDCG@10 all": 0.8283,
"nDCG@10 test": 0.8238,
"nDCG@10 dev": 0.8427,
"judged@10": 1.0
},
"combined teacher, its own 7B search side": {
"nDCG@10 all": 0.8132,
"nDCG@10 test": 0.8115,
"nDCG@10 dev": 0.8186,
"judged@10": 1.0
},
"combined teacher, the v1.2 search side": {
"nDCG@10 all": 0.7847,
"nDCG@10 test": 0.782,
"nDCG@10 dev": 0.7936,
"judged@10": 1.0
},
"teacher, the student's search side (Qwen3)": {
"nDCG@10 all": 0.7773,
"nDCG@10 test": 0.771,
"nDCG@10 dev": 0.7977,
"judged@10": 1.0
},
"LCO-Omni-7B as is": {
"nDCG@10 all": 0.7635,
"nDCG@10 test": 0.7631,
"nDCG@10 dev": 0.7649,
"judged@10": 1.0
},
"v1.2 student + v1.2 search side (ships)": {
"nDCG@10 all": 0.7509,
"nDCG@10 test": 0.7449,
"nDCG@10 dev": 0.7705,
"judged@10": 1.0
},
"v1 student + v1.1 search side": {
"nDCG@10 all": 0.744,
"nDCG@10 test": 0.7391,
"nDCG@10 dev": 0.76,
"judged@10": 1.0
},
"e5-omni-7B as is": {
"nDCG@10 all": 0.7421,
"nDCG@10 test": 0.7383,
"nDCG@10 dev": 0.7545,
"judged@10": 1.0
},
"v1 student + v1 search side (07)": {
"nDCG@10 all": 0.7039,
"nDCG@10 test": 0.6994,
"nDCG@10 dev": 0.7184,
"judged@10": 1.0
},
"DCLAP (AudioMuse-AI)": {
"nDCG@10 all": 0.6513,
"nDCG@10 test": 0.6472,
"nDCG@10 dev": 0.6646,
"judged@10": 1.0
},
"LAION CLAP music+speech": {
"nDCG@10 all": 0.5857,
"nDCG@10 test": 0.5881,
"nDCG@10 dev": 0.5779,
"judged@10": 1.0
},
"BM25 over the song cards": {
"nDCG@10 all": 0.5496,
"nDCG@10 test": 0.5486,
"nDCG@10 dev": 0.5529,
"judged@10": 1.0
}
},
"deltas": [
{
"v1.2 student + v1.2 search side minus": "DCLAP (AudioMuse-AI)",
"rescaled, all [95% CI]": "+0.096 [+0.078, +0.115]",
"rescaled, test [95% CI]": "+0.088 [+0.067, +0.109]",
"as answered, all [95% CI]": "+0.100 [+0.083, +0.117]"
},
{
"v1.2 student + v1.2 search side minus": "LAION CLAP music+speech",
"rescaled, all [95% CI]": "+0.159 [+0.139, +0.179]",
"rescaled, test [95% CI]": "+0.151 [+0.128, +0.175]",
"as answered, all [95% CI]": "+0.165 [+0.147, +0.185]"
},
{
"v1.2 student + v1.2 search side minus": "v1 student + v1.1 search side",
"rescaled, all [95% CI]": "+0.015 [+0.004, +0.026]",
"rescaled, test [95% CI]": "+0.011 [-0.002, +0.025]",
"as answered, all [95% CI]": "+0.007 [-0.002, +0.016]"
},
{
"v1.2 student + v1.2 search side minus": "v1 student + v1 search side (07)",
"rescaled, all [95% CI]": "+0.047 [+0.033, +0.060]",
"rescaled, test [95% CI]": "+0.042 [+0.028, +0.057]",
"as answered, all [95% CI]": "+0.047 [+0.036, +0.059]"
},
{
"v1.2 student + v1.2 search side minus": "LCO-Omni-7B as is",
"rescaled, all [95% CI]": "-0.017 [-0.031, -0.002]",
"rescaled, test [95% CI]": "-0.022 [-0.040, -0.005]",
"as answered, all [95% CI]": "-0.013 [-0.026, +0.001]"
},
{
"v1.2 student + v1.2 search side minus": "combined teacher, its own 7B search side",
"rescaled, all [95% CI]": "-0.061 [-0.074, -0.048]",
"rescaled, test [95% CI]": "-0.068 [-0.084, -0.052]",
"as answered, all [95% CI]": "-0.062 [-0.074, -0.051]"
}
],
"total_minutes": 48.8
},
"decision13b": {
"adopted": true,
"student": "mn20_as",
"chosen": "C · trained on the teacher's song vectors only (epoch 3)",
"teacher": "combined (teacher/combined)",
"search_side_for_selection": "student/text_v12_base",
"scores": {
"incompe_dev_lenient": 0.7105,
"incompe_test_lenient": 0.7368,
"incompe_full_lenient": 0.7235,
"sdd_dev_R@10": 50.2752,
"sdd_test_R@10": 44.385,
"sdd_full_R@10": 47.2875
}
},
"release": {
"plugin_checks": {
"int8": {
"audio_cos": 0.99982,
"text_cos_min": 0.99976,
"tokenizer": "tokenizers"
},
"fp32": {
"audio_cos": 1.0,
"text_cos_min": 1.0,
"tokenizer": "tokenizers"
}
},
"sizes": {
"audio_int8_MB": 35.3,
"audio_fp32_MB": 72.1,
"text_int8_MB": 818.5,
"text_fp32_MB": 2395.7
},
"catalogue": {
"songs": 1831,
"left_out_licence": 0,
"left_out_unreadable": 0,
"cos_to_pytorch_mean": 0.9998,
"cos_to_pytorch_min": 0.9925,
"minutes": 6.4
},
"speed": {
"song_seconds": 180.0,
"board": [
{
"threads": 4,
"ours int8 (s/song)": 0.902,
"ours fp32 (s/song)": 0.903,
"DCLAP (s/song)": 4.243,
"DCLAP / ours int8": 4.7
},
{
"threads": 1,
"ours int8 (s/song)": 1.897,
"ours fp32 (s/song)": 1.909,
"DCLAP (s/song)": 2.677,
"DCLAP / ours int8": 1.41
}
],
"gate_passed": true
}
}
}