File size: 4,420 Bytes
516071e | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 | {
"phase_results": [
{
"phase": "simple_multiturn_transcript",
"best_selection_metric_value": 6.337032405180627,
"best_metrics": {
"validation/loss_total": 1.8748868287042504,
"validation/loss_response": 2.216750873447601,
"validation/loss_current_user_reconstruction": 2.1496858545410533,
"validation/loss_probe": 1.3845612109447378,
"validation/loss_past_response_contrastive": 1.0462471295376212,
"validation/response_first_token_exact_match": 0.1326530612244898,
"validation/current-user_reconstruction_first_token_exact_match": 0.10714285714285714,
"validation/probe_first_token_exact_match": 0.7040816326530612,
"validation/supervised_turn_count": 196.0,
"validation/response_exact_match": 0.0,
"validation/response_similarity": 0.10058788564883722,
"validation/response_token_f1": 0.20117577129767444,
"validation/response_line_recall": 0.0,
"validation/response_reconstruction_similarity": 0.1891127356809172,
"validation/response_reconstruction_exact_match": 0.0,
"validation/response_reconstruction_token_f1": 0.1891127356809172,
"validation/probe_exact_match": 0.0,
"validation/probe_transcript_similarity": 0.17437084392135335,
"validation/probe_token_f1": 0.3487416878427067,
"validation/probe_line_recall": 0.0,
"validation/response_similarity_by_turn/turn_2": 0.09775539351122156,
"validation/response_similarity_by_turn/turn_3": 0.10816021143924962,
"validation/response_similarity_by_turn/turn_4": 0.10406032060356261,
"validation/response_similarity_by_turn/turn_5": 0.10085456009369054,
"validation/response_similarity_by_turn/turn_6": 0.04664324746291959,
"validation/response_reconstruction_similarity_by_turn/turn_2": 0.19856356350644752,
"validation/response_reconstruction_similarity_by_turn/turn_3": 0.19559046721759682,
"validation/response_reconstruction_similarity_by_turn/turn_4": 0.17994648355759468,
"validation/response_reconstruction_similarity_by_turn/turn_5": 0.19919191919191923,
"validation/response_reconstruction_similarity_by_turn/turn_6": 0.08781362007168458,
"validation/probe_transcript_similarity_by_turn/turn_2": 0.22581347347376093,
"validation/probe_transcript_similarity_by_turn/turn_3": 0.20101213533791346,
"validation/probe_transcript_similarity_by_turn/turn_4": 0.10900768873835404,
"validation/probe_transcript_similarity_by_turn/turn_5": 0.04821993196924256,
"validation/probe_transcript_similarity_by_turn/turn_6": 0.0330600424212759,
"validation/goal_loss": 6.337032405180627
},
"global_step": 241,
"train_dataset": {
"example_count": 7696,
"pair_count_mean": 4.2548076923076925,
"pair_count_max": 6,
"response_target_tokens_mean_turn2_plus": 126.23921517671518,
"response_target_tokens_max_turn2_plus": 1643,
"probe_target_tokens_mean_turn2_plus": 471.31613825363826,
"probe_target_tokens_max_turn2_plus": 3285,
"dataset_counts": {
"chatalpaca_multiturn_enriched": 7696
},
"probe_question_text": "What is everything we have talked about so far? Give exact conversation transcript verbatim in following format: [User 1]: X [Assistant 1]: Y [User 2]: A etc",
"current_user_reconstruction_question_text": "What did me (the user) just ask you? give verbatim message I just previously sent"
},
"validation_dataset": {
"example_count": 151,
"pair_count_mean": 4.139072847682119,
"pair_count_max": 6,
"response_target_tokens_mean_turn2_plus": 123.21854304635761,
"response_target_tokens_max_turn2_plus": 889,
"probe_target_tokens_mean_turn2_plus": 448.5298013245033,
"probe_target_tokens_max_turn2_plus": 1545,
"dataset_counts": {
"chatalpaca_multiturn_enriched": 151
},
"probe_question_text": "What is everything we have talked about so far? Give exact conversation transcript verbatim in following format: [User 1]: X [Assistant 1]: Y [User 2]: A etc",
"current_user_reconstruction_question_text": "What did me (the user) just ask you? give verbatim message I just previously sent"
},
"wandb_enabled": true
}
]
} |